From bee25e93b5012b470c9f9c282ecd4364ae3cf8cb Mon Sep 17 00:00:00 2001
From: Shuning <shuning@aukilabs.com>
Date: Thu, 13 Mar 2025 16:58:09 +0800
Subject: [PATCH 01/89] integrated with new SDK

---
 .dockerignore                      |    9 +
 .github/workflows/docker-build.yml |   36 +
 .gitignore                         |    2 +
 docker/Dockerfile                  |   48 +-
 server/Cargo.lock                  | 5420 ++++++++++++++++++++++++++++
 server/Cargo.toml                  |   23 +
 server/go.mod                      |   27 -
 server/go.sum                      |   95 -
 server/helper.go                   |  960 -----
 server/main.go                     |  130 -
 server/src/global_refinement.rs    |  205 ++
 server/src/local_refinement.rs     |  287 ++
 server/src/main.rs                 |   52 +
 server/src/utils.rs                |   39 +
 14 files changed, 6108 insertions(+), 1225 deletions(-)
 create mode 100644 .dockerignore
 create mode 100644 .github/workflows/docker-build.yml
 create mode 100644 server/Cargo.lock
 create mode 100644 server/Cargo.toml
 delete mode 100644 server/go.mod
 delete mode 100644 server/go.sum
 delete mode 100644 server/helper.go
 delete mode 100644 server/main.go
 create mode 100644 server/src/global_refinement.rs
 create mode 100644 server/src/local_refinement.rs
 create mode 100644 server/src/main.rs
 create mode 100644 server/src/utils.rs

diff --git a/.dockerignore b/.dockerignore
new file mode 100644
index 0000000..830632b
--- /dev/null
+++ b/.dockerignore
@@ -0,0 +1,9 @@
+target/
+.git/
+.gitignore
+.env
+*.log
+.DS_Store
+README.md
+Dockerfile
+.dockerignore 
diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml
new file mode 100644
index 0000000..7dbd8a2
--- /dev/null
+++ b/.github/workflows/docker-build.yml
@@ -0,0 +1,36 @@
+name: Build Docker Image
+
+on:
+  push:
+    branches:
+      - feature/sdk-integration
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Build Docker image
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          file: docker/Dockerfile
+          platforms: linux/amd64
+          load: true
+          tags: reconstruction-server:latest
+
+      - name: Save Docker image
+        run: |
+          docker save reconstruction-server:latest > reconstruction-server.tar
+
+      - name: Upload Docker image as artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: reconstruction-server-image
+          path: reconstruction-server.tar
+          retention-days: 7 
diff --git a/.gitignore b/.gitignore
index 6689682..a73191c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -8,3 +8,5 @@ reconstruction
 .venv
 /venv
 /.idea
+target
+volume
diff --git a/docker/Dockerfile b/docker/Dockerfile
index 945afff..b7263ec 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -1,23 +1,44 @@
 ##
-## Go build for server
+## Rust build for server
 ##
-FROM --platform=$BUILDPLATFORM golang:1.21 AS go-build
-RUN mkdir -p /app
-WORKDIR /app
-ADD server /app
+FROM --platform=$BUILDPLATFORM rust:1.84-slim-bullseye AS rust-build
 ARG TARGETOS
 ARG TARGETARCH
 ARG VERSION
-RUN CGO_ENABLED=0 GOOS="${TARGETOS}" GOARCH="${TARGETARCH}" go build
 
+# Install build dependencies
+RUN apt-get update && apt-get install -y \
+    python3 \
+    python3-pip \
+    python3-dev \
+    build-essential \
+    pkg-config \
+    git \
+    && rm -rf /var/lib/apt/lists/*
+
+# Install Python packages
+RUN pip3 install --no-cache-dir numpy scipy matplotlib
+
+WORKDIR /app
+COPY server /app/server
+
+# Build the application
+RUN cd server && cargo build --release --target-dir ../target
 
-FROM nvidia/cuda:11.0.3-base-ubuntu20.04
+##
+## Runtime image
+##
+FROM --platform=$TARGETPLATFORM nvidia/cuda:11.0.3-base-ubuntu20.04
 
 ARG USERNAME=reconstruction-server
 ARG USER_UID=1000
 ARG USER_GID=$USER_UID
 ARG DEBIAN_FRONTEND=noninteractive
 
+# Set default environment variables
+ENV PORT=18802
+ENV DOMAIN_MANAGER_ADDR="/ip4/127.0.0.1/udp/18800/quic-v1/p2p/12D3KooWDHaDQeuYeLM8b5zhNjqS7Pkh7KefqzCpDGpdwj5iE8pq"
+
 RUN apt-get update && apt-get install -y \
     wget \
     curl \
@@ -155,12 +176,13 @@ RUN groupadd --gid $USER_GID $USERNAME \
     && useradd --uid $USER_UID --gid $USER_GID -m $USERNAME
 
 RUN mkdir -p /app/jobs && chown -R reconstruction-server:reconstruction-server /app/jobs
-
+RUN mkdir -p /app/volume && chown -R reconstruction-server:reconstruction-server /app/volume
 USER $USERNAME
 
-COPY --from=go-build /app/reconstruction ./reconstruction
-ENTRYPOINT ["./reconstruction"]
+# Copy the built binary from rust-build
+COPY --from=rust-build /app/target/release/server /app/server
+
+ENTRYPOINT ["./server"]
+EXPOSE ${PORT}
 
-# Overridable using 'docker run ... image_name -api-key <your-api-key>'
-# TODO: we probably don't need api key, or make it optional. Just check domain token.
-CMD ["-api-key", "kaffekopp123"]
+CMD ["${PORT}", "${DOMAIN_MANAGER_ADDR}"]
diff --git a/server/Cargo.lock b/server/Cargo.lock
new file mode 100644
index 0000000..e21a0d4
--- /dev/null
+++ b/server/Cargo.lock
@@ -0,0 +1,5420 @@
+# This file is automatically @generated by Cargo.
+# It is not intended for manual editing.
+version = 4
+
+[[package]]
+name = "addr2line"
+version = "0.24.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dfbe277e56a376000877090da837660b4427aad530e3028d44e0bffe4f89a1c1"
+dependencies = [
+ "gimli",
+]
+
+[[package]]
+name = "adler2"
+version = "2.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627"
+
+[[package]]
+name = "aead"
+version = "0.5.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0"
+dependencies = [
+ "crypto-common",
+ "generic-array",
+]
+
+[[package]]
+name = "aes"
+version = "0.8.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0"
+dependencies = [
+ "cfg-if",
+ "cipher",
+ "cpufeatures",
+]
+
+[[package]]
+name = "aes-gcm"
+version = "0.10.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "831010a0f742e1209b3bcea8fab6a8e149051ba6099432c8cb2cc117dec3ead1"
+dependencies = [
+ "aead",
+ "aes",
+ "cipher",
+ "ctr",
+ "ghash",
+ "subtle",
+]
+
+[[package]]
+name = "aho-corasick"
+version = "1.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916"
+dependencies = [
+ "memchr",
+]
+
+[[package]]
+name = "allocator-api2"
+version = "0.2.21"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923"
+
+[[package]]
+name = "android-tzdata"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0"
+
+[[package]]
+name = "android_system_properties"
+version = "0.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311"
+dependencies = [
+ "libc",
+]
+
+[[package]]
+name = "ansi_term"
+version = "0.12.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2"
+dependencies = [
+ "winapi",
+]
+
+[[package]]
+name = "anstream"
+version = "0.6.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8acc5369981196006228e28809f761875c0327210a891e941f4c683b3a99529b"
+dependencies = [
+ "anstyle",
+ "anstyle-parse",
+ "anstyle-query",
+ "anstyle-wincon",
+ "colorchoice",
+ "is_terminal_polyfill",
+ "utf8parse",
+]
+
+[[package]]
+name = "anstyle"
+version = "1.0.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9"
+
+[[package]]
+name = "anstyle-parse"
+version = "0.2.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3b2d16507662817a6a20a9ea92df6652ee4f94f914589377d69f3b21bc5798a9"
+dependencies = [
+ "utf8parse",
+]
+
+[[package]]
+name = "anstyle-query"
+version = "1.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "79947af37f4177cfead1110013d678905c37501914fba0efea834c3fe9a8d60c"
+dependencies = [
+ "windows-sys 0.59.0",
+]
+
+[[package]]
+name = "anstyle-wincon"
+version = "3.0.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ca3534e77181a9cc07539ad51f2141fe32f6c3ffd4df76db8ad92346b003ae4e"
+dependencies = [
+ "anstyle",
+ "once_cell",
+ "windows-sys 0.59.0",
+]
+
+[[package]]
+name = "anyhow"
+version = "1.0.97"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dcfed56ad506cb2c684a14971b8861fdc3baaaae314b9e5f9bb532cbe3ba7a4f"
+
+[[package]]
+name = "arbitrary"
+version = "1.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dde20b3d026af13f561bdd0f15edf01fc734f0dafcedbaf42bba506a9517f223"
+dependencies = [
+ "derive_arbitrary",
+]
+
+[[package]]
+name = "arc-swap"
+version = "1.7.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "69f7f8c3906b62b754cd5326047894316021dcfe5a194c8ea52bdd94934a3457"
+
+[[package]]
+name = "arrayref"
+version = "0.3.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb"
+
+[[package]]
+name = "arrayvec"
+version = "0.7.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50"
+
+[[package]]
+name = "asn1-rs"
+version = "0.5.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7f6fd5ddaf0351dff5b8da21b2fb4ff8e08ddd02857f0bf69c47639106c0fff0"
+dependencies = [
+ "asn1-rs-derive 0.4.0",
+ "asn1-rs-impl 0.1.0",
+ "displaydoc",
+ "nom",
+ "num-traits",
+ "rusticata-macros",
+ "thiserror 1.0.69",
+ "time",
+]
+
+[[package]]
+name = "asn1-rs"
+version = "0.6.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5493c3bedbacf7fd7382c6346bbd66687d12bbaad3a89a2d2c303ee6cf20b048"
+dependencies = [
+ "asn1-rs-derive 0.5.1",
+ "asn1-rs-impl 0.2.0",
+ "displaydoc",
+ "nom",
+ "num-traits",
+ "rusticata-macros",
+ "thiserror 1.0.69",
+ "time",
+]
+
+[[package]]
+name = "asn1-rs-derive"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "726535892e8eae7e70657b4c8ea93d26b8553afb1ce617caee529ef96d7dee6c"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 1.0.109",
+ "synstructure 0.12.6",
+]
+
+[[package]]
+name = "asn1-rs-derive"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "965c2d33e53cb6b267e148a4cb0760bc01f4904c1cd4bb4002a085bb016d1490"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.100",
+ "synstructure 0.13.1",
+]
+
+[[package]]
+name = "asn1-rs-impl"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2777730b2039ac0f95f093556e61b6d26cebed5393ca6f152717777cec3a42ed"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 1.0.109",
+]
+
+[[package]]
+name = "asn1-rs-impl"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.100",
+]
+
+[[package]]
+name = "async-io"
+version = "2.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "43a2b323ccce0a1d90b449fd71f2a06ca7faa7c54c2751f06c9bd851fc061059"
+dependencies = [
+ "async-lock",
+ "cfg-if",
+ "concurrent-queue",
+ "futures-io",
+ "futures-lite",
+ "parking",
+ "polling",
+ "rustix 0.38.44",
+ "slab",
+ "tracing",
+ "windows-sys 0.59.0",
+]
+
+[[package]]
+name = "async-lock"
+version = "3.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ff6e472cdea888a4bd64f342f09b3f50e1886d32afe8df3d663c01140b811b18"
+dependencies = [
+ "event-listener",
+ "event-listener-strategy",
+ "pin-project-lite",
+]
+
+[[package]]
+name = "async-trait"
+version = "0.1.87"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d556ec1359574147ec0c4fc5eb525f3f23263a592b1a9c07e0a75b427de55c97"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.100",
+]
+
+[[package]]
+name = "asynchronous-codec"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a860072022177f903e59730004fb5dc13db9275b79bb2aef7ba8ce831956c233"
+dependencies = [
+ "bytes",
+ "futures-sink",
+ "futures-util",
+ "memchr",
+ "pin-project-lite",
+]
+
+[[package]]
+name = "atomic-waker"
+version = "1.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0"
+
+[[package]]
+name = "attohttpc"
+version = "0.24.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8d9a9bf8b79a749ee0b911b91b671cc2b6c670bdbc7e3dfd537576ddc94bb2a2"
+dependencies = [
+ "http 0.2.12",
+ "log",
+ "url",
+]
+
+[[package]]
+name = "atty"
+version = "0.2.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8"
+dependencies = [
+ "hermit-abi 0.1.19",
+ "libc",
+ "winapi",
+]
+
+[[package]]
+name = "autocfg"
+version = "1.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26"
+
+[[package]]
+name = "backtrace"
+version = "0.3.74"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8d82cb332cdfaed17ae235a638438ac4d4839913cc2af585c3c6746e8f8bee1a"
+dependencies = [
+ "addr2line",
+ "cfg-if",
+ "libc",
+ "miniz_oxide",
+ "object",
+ "rustc-demangle",
+ "windows-targets 0.52.6",
+]
+
+[[package]]
+name = "base-x"
+version = "0.2.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4cbbc9d0964165b47557570cce6c952866c2678457aca742aafc9fb771d30270"
+
+[[package]]
+name = "base16ct"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4c7f02d4ea65f2c1853089ffd8d2787bdbc63de2f0d29dedbcf8ccdfa0ccd4cf"
+
+[[package]]
+name = "base64"
+version = "0.21.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567"
+
+[[package]]
+name = "base64"
+version = "0.22.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
+
+[[package]]
+name = "base64ct"
+version = "1.7.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bb97d56060ee67d285efb8001fec9d2a4c710c32efd2e14b5cbb5ba71930fc2d"
+
+[[package]]
+name = "bincode"
+version = "1.3.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad"
+dependencies = [
+ "serde",
+]
+
+[[package]]
+name = "bitflags"
+version = "1.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
+
+[[package]]
+name = "bitflags"
+version = "2.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5c8214115b7bf84099f1309324e63141d4c5d7cc26862f97a0a857dbefe165bd"
+
+[[package]]
+name = "blake2"
+version = "0.10.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe"
+dependencies = [
+ "digest",
+]
+
+[[package]]
+name = "block-buffer"
+version = "0.10.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71"
+dependencies = [
+ "generic-array",
+]
+
+[[package]]
+name = "block-padding"
+version = "0.3.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a8894febbff9f758034a5b8e12d87918f56dfc64a8e1fe757d65e29041538d93"
+dependencies = [
+ "generic-array",
+]
+
+[[package]]
+name = "bs58"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bf88ba1141d185c399bee5288d850d63b8369520c1eafc32a0430b5b6c287bf4"
+dependencies = [
+ "tinyvec",
+]
+
+[[package]]
+name = "bumpalo"
+version = "3.17.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1628fb46dfa0b37568d12e5edd512553eccf6a22a78e8bde00bb4aed84d5bdbf"
+
+[[package]]
+name = "byteorder"
+version = "1.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
+
+[[package]]
+name = "bytes"
+version = "1.10.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a"
+dependencies = [
+ "serde",
+]
+
+[[package]]
+name = "bzip2"
+version = "0.5.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "49ecfb22d906f800d4fe833b6282cf4dc1c298f5057ca0b5445e5c209735ca47"
+dependencies = [
+ "bzip2-sys",
+]
+
+[[package]]
+name = "bzip2-sys"
+version = "0.1.13+1.0.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "225bff33b2141874fe80d71e07d6eec4f85c5c216453dd96388240f96e1acc14"
+dependencies = [
+ "cc",
+ "pkg-config",
+]
+
+[[package]]
+name = "cbc"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "26b52a9543ae338f279b96b0b9fed9c8093744685043739079ce85cd58f289a6"
+dependencies = [
+ "cipher",
+]
+
+[[package]]
+name = "cbindgen"
+version = "0.28.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "eadd868a2ce9ca38de7eeafdcec9c7065ef89b42b32f0839278d55f35c54d1ff"
+dependencies = [
+ "clap 4.5.32",
+ "heck 0.4.1",
+ "indexmap",
+ "log",
+ "proc-macro2",
+ "quote",
+ "serde",
+ "serde_json",
+ "syn 2.0.100",
+ "tempfile",
+ "toml",
+]
+
+[[package]]
+name = "cc"
+version = "1.2.16"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "be714c154be609ec7f5dad223a33bf1482fff90472de28f7362806e6d4832b8c"
+dependencies = [
+ "jobserver",
+ "libc",
+ "shlex",
+]
+
+[[package]]
+name = "ccm"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9ae3c82e4355234767756212c570e29833699ab63e6ffd161887314cc5b43847"
+dependencies = [
+ "aead",
+ "cipher",
+ "ctr",
+ "subtle",
+]
+
+[[package]]
+name = "cfg-if"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
+
+[[package]]
+name = "cfg_aliases"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724"
+
+[[package]]
+name = "chacha20"
+version = "0.9.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c3613f74bd2eac03dad61bd53dbe620703d4371614fe0bc3b9f04dd36fe4e818"
+dependencies = [
+ "cfg-if",
+ "cipher",
+ "cpufeatures",
+]
+
+[[package]]
+name = "chacha20poly1305"
+version = "0.10.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "10cd79432192d1c0f4e1a0fef9527696cc039165d729fb41b3f4f4f354c2dc35"
+dependencies = [
+ "aead",
+ "chacha20",
+ "cipher",
+ "poly1305",
+ "zeroize",
+]
+
+[[package]]
+name = "chrono"
+version = "0.4.40"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1a7964611d71df112cb1730f2ee67324fcf4d0fc6606acbbe9bfe06df124637c"
+dependencies = [
+ "android-tzdata",
+ "iana-time-zone",
+ "js-sys",
+ "num-traits",
+ "wasm-bindgen",
+ "windows-link",
+]
+
+[[package]]
+name = "cipher"
+version = "0.4.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad"
+dependencies = [
+ "crypto-common",
+ "inout",
+ "zeroize",
+]
+
+[[package]]
+name = "clap"
+version = "2.34.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c"
+dependencies = [
+ "ansi_term",
+ "atty",
+ "bitflags 1.3.2",
+ "strsim 0.8.0",
+ "textwrap",
+ "unicode-width",
+ "vec_map",
+]
+
+[[package]]
+name = "clap"
+version = "4.5.32"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6088f3ae8c3608d19260cd7445411865a485688711b78b5be70d78cd96136f83"
+dependencies = [
+ "clap_builder",
+]
+
+[[package]]
+name = "clap_builder"
+version = "4.5.32"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "22a7ef7f676155edfb82daa97f99441f3ebf4a58d5e32f295a56259f1b6facc8"
+dependencies = [
+ "anstream",
+ "anstyle",
+ "clap_lex",
+ "strsim 0.11.1",
+]
+
+[[package]]
+name = "clap_lex"
+version = "0.7.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f46ad14479a25103f283c0f10005961cf086d8dc42205bb44c46ac563475dca6"
+
+[[package]]
+name = "colorchoice"
+version = "1.0.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990"
+
+[[package]]
+name = "concurrent-queue"
+version = "2.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973"
+dependencies = [
+ "crossbeam-utils",
+]
+
+[[package]]
+name = "console_error_panic_hook"
+version = "0.1.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a06aeb73f470f66dcdbf7223caeebb85984942f22f1adb2a088cf9668146bbbc"
+dependencies = [
+ "cfg-if",
+ "wasm-bindgen",
+]
+
+[[package]]
+name = "const-oid"
+version = "0.9.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8"
+
+[[package]]
+name = "constant_time_eq"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6"
+
+[[package]]
+name = "core-foundation"
+version = "0.9.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f"
+dependencies = [
+ "core-foundation-sys",
+ "libc",
+]
+
+[[package]]
+name = "core-foundation-sys"
+version = "0.8.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b"
+
+[[package]]
+name = "core2"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b49ba7ef1ad6107f8824dbe97de947cbaac53c44e7f9756a1fba0d37c1eec505"
+dependencies = [
+ "memchr",
+]
+
+[[package]]
+name = "cpufeatures"
+version = "0.2.17"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280"
+dependencies = [
+ "libc",
+]
+
+[[package]]
+name = "crc"
+version = "3.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "69e6e4d7b33a94f0991c26729976b10ebde1d34c3ee82408fb536164fa10d636"
+dependencies = [
+ "crc-catalog",
+]
+
+[[package]]
+name = "crc-catalog"
+version = "2.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5"
+
+[[package]]
+name = "crc32fast"
+version = "1.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3"
+dependencies = [
+ "cfg-if",
+]
+
+[[package]]
+name = "crossbeam-utils"
+version = "0.8.21"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28"
+
+[[package]]
+name = "crunchy"
+version = "0.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "43da5946c66ffcc7745f48db692ffbb10a83bfe0afd96235c5c2a4fb23994929"
+
+[[package]]
+name = "crypto-bigint"
+version = "0.5.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76"
+dependencies = [
+ "generic-array",
+ "rand_core 0.6.4",
+ "subtle",
+ "zeroize",
+]
+
+[[package]]
+name = "crypto-common"
+version = "0.1.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3"
+dependencies = [
+ "generic-array",
+ "rand_core 0.6.4",
+ "typenum",
+]
+
+[[package]]
+name = "ctr"
+version = "0.9.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0369ee1ad671834580515889b80f2ea915f23b8be8d0daa4bbaf2ac5c7590835"
+dependencies = [
+ "cipher",
+]
+
+[[package]]
+name = "curve25519-dalek"
+version = "4.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "97fb8b7c4503de7d6ae7b42ab72a5a59857b4c937ec27a3d4539dba95b5ab2be"
+dependencies = [
+ "cfg-if",
+ "cpufeatures",
+ "curve25519-dalek-derive",
+ "digest",
+ "fiat-crypto",
+ "rustc_version",
+ "subtle",
+ "zeroize",
+]
+
+[[package]]
+name = "curve25519-dalek-derive"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.100",
+]
+
+[[package]]
+name = "data-encoding"
+version = "2.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "575f75dfd25738df5b91b8e43e14d44bda14637a58fae779fd2b064f8bf3e010"
+
+[[package]]
+name = "data-encoding-macro"
+version = "0.1.17"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9f9724adfcf41f45bf652b3995837669d73c4d49a1b5ac1ff82905ac7d9b5558"
+dependencies = [
+ "data-encoding",
+ "data-encoding-macro-internal",
+]
+
+[[package]]
+name = "data-encoding-macro-internal"
+version = "0.1.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "18e4fdb82bd54a12e42fb58a800dcae6b9e13982238ce2296dc3570b92148e1f"
+dependencies = [
+ "data-encoding",
+ "syn 2.0.100",
+]
+
+[[package]]
+name = "deflate64"
+version = "0.1.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "da692b8d1080ea3045efaab14434d40468c3d8657e42abddfffca87b428f4c1b"
+
+[[package]]
+name = "der"
+version = "0.7.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f55bf8e7b65898637379c1b74eb1551107c8294ed26d855ceb9fd1a09cfc9bc0"
+dependencies = [
+ "const-oid",
+ "pem-rfc7468",
+ "zeroize",
+]
+
+[[package]]
+name = "der-parser"
+version = "8.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dbd676fbbab537128ef0278adb5576cf363cff6aa22a7b24effe97347cfab61e"
+dependencies = [
+ "asn1-rs 0.5.2",
+ "displaydoc",
+ "nom",
+ "num-bigint",
+ "num-traits",
+ "rusticata-macros",
+]
+
+[[package]]
+name = "der-parser"
+version = "9.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5cd0a5c643689626bec213c4d8bd4d96acc8ffdb4ad4bb6bc16abf27d5f4b553"
+dependencies = [
+ "asn1-rs 0.6.2",
+ "displaydoc",
+ "nom",
+ "num-bigint",
+ "num-traits",
+ "rusticata-macros",
+]
+
+[[package]]
+name = "deranged"
+version = "0.3.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4"
+dependencies = [
+ "powerfmt",
+]
+
+[[package]]
+name = "derive_arbitrary"
+version = "1.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "30542c1ad912e0e3d22a1935c290e12e8a29d704a420177a31faad4a601a0800"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.100",
+]
+
+[[package]]
+name = "digest"
+version = "0.10.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"
+dependencies = [
+ "block-buffer",
+ "const-oid",
+ "crypto-common",
+ "subtle",
+]
+
+[[package]]
+name = "displaydoc"
+version = "0.2.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.100",
+]
+
+[[package]]
+name = "domain"
+version = "0.1.0"
+source = "git+https://github.com/aukilabs/posemesh.git?branch=chore%2Freconstruction-integration#78d50d88988c5e731764efb6a73797edc5c2c1c6"
+dependencies = [
+ "async-trait",
+ "cbindgen",
+ "console_error_panic_hook",
+ "futures",
+ "jsonwebtoken",
+ "libp2p",
+ "networking",
+ "pb-rs",
+ "quick-protobuf",
+ "quick-protobuf-codec 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "runtime",
+ "serde",
+ "serde-wasm-bindgen",
+ "serde_json",
+ "tokio",
+ "tracing",
+ "tracing-subscriber",
+ "tracing-wasm",
+ "uuid",
+ "wasm-bindgen",
+ "wasm-bindgen-futures",
+]
+
+[[package]]
+name = "dtoa"
+version = "1.0.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d6add3b8cff394282be81f3fc1a0605db594ed69890078ca6e2cab1c408bcf04"
+
+[[package]]
+name = "ecdsa"
+version = "0.16.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ee27f32b5c5292967d2d4a9d7f1e0b0aed2c15daded5a60300e4abb9d8020bca"
+dependencies = [
+ "der",
+ "digest",
+ "elliptic-curve",
+ "rfc6979",
+ "signature",
+ "spki",
+]
+
+[[package]]
+name = "ed25519"
+version = "2.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "115531babc129696a58c64a4fef0a8bf9e9698629fb97e9e40767d235cfbcd53"
+dependencies = [
+ "pkcs8",
+ "signature",
+]
+
+[[package]]
+name = "ed25519-dalek"
+version = "2.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4a3daa8e81a3963a60642bcc1f90a670680bd4a77535faa384e9d1c79d620871"
+dependencies = [
+ "curve25519-dalek",
+ "ed25519",
+ "rand_core 0.6.4",
+ "serde",
+ "sha2",
+ "subtle",
+ "zeroize",
+]
+
+[[package]]
+name = "either"
+version = "1.15.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719"
+
+[[package]]
+name = "elliptic-curve"
+version = "0.13.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b5e6043086bf7973472e0c7dff2142ea0b680d30e18d9cc40f267efbf222bd47"
+dependencies = [
+ "base16ct",
+ "crypto-bigint",
+ "digest",
+ "ff",
+ "generic-array",
+ "group",
+ "hkdf",
+ "pem-rfc7468",
+ "pkcs8",
+ "rand_core 0.6.4",
+ "sec1",
+ "subtle",
+ "zeroize",
+]
+
+[[package]]
+name = "enum-as-inner"
+version = "0.6.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a1e6a265c649f3f5979b601d26f1d05ada116434c87741c9493cb56218f76cbc"
+dependencies = [
+ "heck 0.5.0",
+ "proc-macro2",
+ "quote",
+ "syn 2.0.100",
+]
+
+[[package]]
+name = "env_logger"
+version = "0.7.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "44533bbbb3bb3c1fa17d9f2e4e38bbbaf8396ba82193c4cb1b6445d711445d36"
+dependencies = [
+ "atty",
+ "humantime",
+ "log",
+ "regex",
+ "termcolor",
+]
+
+[[package]]
+name = "equivalent"
+version = "1.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f"
+
+[[package]]
+name = "errno"
+version = "0.3.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "33d852cb9b869c2a9b3df2f71a3074817f01e1844f839a144f5fcef059a4eb5d"
+dependencies = [
+ "libc",
+ "windows-sys 0.59.0",
+]
+
+[[package]]
+name = "event-listener"
+version = "5.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3492acde4c3fc54c845eaab3eed8bd00c7a7d881f78bfc801e43a93dec1331ae"
+dependencies = [
+ "concurrent-queue",
+ "parking",
+ "pin-project-lite",
+]
+
+[[package]]
+name = "event-listener-strategy"
+version = "0.5.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3c3e4e0dd3673c1139bf041f3008816d9cf2946bbfac2945c09e523b8d7b05b2"
+dependencies = [
+ "event-listener",
+ "pin-project-lite",
+]
+
+[[package]]
+name = "fastrand"
+version = "2.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be"
+
+[[package]]
+name = "ff"
+version = "0.13.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c0b50bfb653653f9ca9095b427bed08ab8d75a137839d9ad64eb11810d5b6393"
+dependencies = [
+ "rand_core 0.6.4",
+ "subtle",
+]
+
+[[package]]
+name = "fiat-crypto"
+version = "0.2.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d"
+
+[[package]]
+name = "flate2"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "11faaf5a5236997af9848be0bef4db95824b1d534ebc64d0f0c6cf3e67bd38dc"
+dependencies = [
+ "crc32fast",
+ "miniz_oxide",
+]
+
+[[package]]
+name = "fnv"
+version = "1.0.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
+
+[[package]]
+name = "foldhash"
+version = "0.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a0d2fde1f7b3d48b8395d5f2de76c18a528bd6a9cdde438df747bfcba3e05d6f"
+
+[[package]]
+name = "form_urlencoded"
+version = "1.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456"
+dependencies = [
+ "percent-encoding",
+]
+
+[[package]]
+name = "futures"
+version = "0.3.31"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876"
+dependencies = [
+ "futures-channel",
+ "futures-core",
+ "futures-executor",
+ "futures-io",
+ "futures-sink",
+ "futures-task",
+ "futures-util",
+]
+
+[[package]]
+name = "futures-bounded"
+version = "0.2.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "91f328e7fb845fc832912fb6a34f40cf6d1888c92f974d1893a54e97b5ff542e"
+dependencies = [
+ "futures-timer",
+ "futures-util",
+]
+
+[[package]]
+name = "futures-channel"
+version = "0.3.31"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10"
+dependencies = [
+ "futures-core",
+ "futures-sink",
+]
+
+[[package]]
+name = "futures-core"
+version = "0.3.31"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e"
+
+[[package]]
+name = "futures-executor"
+version = "0.3.31"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f"
+dependencies = [
+ "futures-core",
+ "futures-task",
+ "futures-util",
+ "num_cpus",
+]
+
+[[package]]
+name = "futures-io"
+version = "0.3.31"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6"
+
+[[package]]
+name = "futures-lite"
+version = "2.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f5edaec856126859abb19ed65f39e90fea3a9574b9707f13539acf4abf7eb532"
+dependencies = [
+ "futures-core",
+ "pin-project-lite",
+]
+
+[[package]]
+name = "futures-macro"
+version = "0.3.31"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.100",
+]
+
+[[package]]
+name = "futures-rustls"
+version = "0.26.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a8f2f12607f92c69b12ed746fabf9ca4f5c482cba46679c1a75b874ed7c26adb"
+dependencies = [
+ "futures-io",
+ "rustls 0.23.23",
+ "rustls-pki-types",
+]
+
+[[package]]
+name = "futures-sink"
+version = "0.3.31"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7"
+
+[[package]]
+name = "futures-task"
+version = "0.3.31"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988"
+
+[[package]]
+name = "futures-ticker"
+version = "0.0.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9763058047f713632a52e916cc7f6a4b3fc6e9fc1ff8c5b1dc49e5a89041682e"
+dependencies = [
+ "futures",
+ "futures-timer",
+ "instant",
+]
+
+[[package]]
+name = "futures-timer"
+version = "3.0.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f288b0a4f20f9a56b5d1da57e2227c661b7b16168e2f72365f57b63326e29b24"
+dependencies = [
+ "gloo-timers 0.2.6",
+ "send_wrapper 0.4.0",
+]
+
+[[package]]
+name = "futures-util"
+version = "0.3.31"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81"
+dependencies = [
+ "futures-channel",
+ "futures-core",
+ "futures-io",
+ "futures-macro",
+ "futures-sink",
+ "futures-task",
+ "memchr",
+ "pin-project-lite",
+ "pin-utils",
+ "slab",
+]
+
+[[package]]
+name = "generic-array"
+version = "0.14.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a"
+dependencies = [
+ "typenum",
+ "version_check",
+ "zeroize",
+]
+
+[[package]]
+name = "getrandom"
+version = "0.2.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7"
+dependencies = [
+ "cfg-if",
+ "js-sys",
+ "libc",
+ "wasi 0.11.0+wasi-snapshot-preview1",
+ "wasm-bindgen",
+]
+
+[[package]]
+name = "getrandom"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "43a49c392881ce6d5c3b8cb70f98717b7c07aabbdff06687b9030dbfbe2725f8"
+dependencies = [
+ "cfg-if",
+ "libc",
+ "wasi 0.13.3+wasi-0.2.2",
+ "windows-targets 0.52.6",
+]
+
+[[package]]
+name = "ghash"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f0d8a4362ccb29cb0b265253fb0a2728f592895ee6854fd9bc13f2ffda266ff1"
+dependencies = [
+ "opaque-debug",
+ "polyval",
+]
+
+[[package]]
+name = "gimli"
+version = "0.31.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f"
+
+[[package]]
+name = "gloo-timers"
+version = "0.2.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9b995a66bb87bebce9a0f4a95aed01daca4872c050bfcb21653361c03bc35e5c"
+dependencies = [
+ "futures-channel",
+ "futures-core",
+ "js-sys",
+ "wasm-bindgen",
+]
+
+[[package]]
+name = "gloo-timers"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bbb143cf96099802033e0d4f4963b19fd2e0b728bcf076cd9cf7f6634f092994"
+dependencies = [
+ "futures-channel",
+ "futures-core",
+ "js-sys",
+ "wasm-bindgen",
+]
+
+[[package]]
+name = "group"
+version = "0.13.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f0f9ef7462f7c099f518d754361858f86d8a07af53ba9af0fe635bbccb151a63"
+dependencies = [
+ "ff",
+ "rand_core 0.6.4",
+ "subtle",
+]
+
+[[package]]
+name = "h2"
+version = "0.4.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5017294ff4bb30944501348f6f8e42e6ad28f42c8bbef7a74029aff064a4e3c2"
+dependencies = [
+ "atomic-waker",
+ "bytes",
+ "fnv",
+ "futures-core",
+ "futures-sink",
+ "http 1.3.1",
+ "indexmap",
+ "slab",
+ "tokio",
+ "tokio-util",
+ "tracing",
+]
+
+[[package]]
+name = "hashbrown"
+version = "0.15.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289"
+dependencies = [
+ "allocator-api2",
+ "equivalent",
+ "foldhash",
+]
+
+[[package]]
+name = "heck"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8"
+
+[[package]]
+name = "heck"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
+
+[[package]]
+name = "hermit-abi"
+version = "0.1.19"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33"
+dependencies = [
+ "libc",
+]
+
+[[package]]
+name = "hermit-abi"
+version = "0.3.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024"
+
+[[package]]
+name = "hermit-abi"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fbf6a919d6cf397374f7dfeeea91d974c7c0a7221d0d0f4f20d859d329e53fcc"
+
+[[package]]
+name = "hex"
+version = "0.4.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70"
+
+[[package]]
+name = "hex_fmt"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b07f60793ff0a4d9cef0f18e63b5357e06209987153a64648c972c1e5aff336f"
+
+[[package]]
+name = "hickory-proto"
+version = "0.24.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "92652067c9ce6f66ce53cc38d1169daa36e6e7eb7dd3b63b5103bd9d97117248"
+dependencies = [
+ "async-trait",
+ "cfg-if",
+ "data-encoding",
+ "enum-as-inner",
+ "futures-channel",
+ "futures-io",
+ "futures-util",
+ "idna",
+ "ipnet",
+ "once_cell",
+ "rand 0.8.5",
+ "socket2",
+ "thiserror 1.0.69",
+ "tinyvec",
+ "tokio",
+ "tracing",
+ "url",
+]
+
+[[package]]
+name = "hickory-resolver"
+version = "0.24.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cbb117a1ca520e111743ab2f6688eddee69db4e0ea242545a604dce8a66fd22e"
+dependencies = [
+ "cfg-if",
+ "futures-util",
+ "hickory-proto",
+ "ipconfig",
+ "lru-cache",
+ "once_cell",
+ "parking_lot",
+ "rand 0.8.5",
+ "resolv-conf",
+ "smallvec",
+ "thiserror 1.0.69",
+ "tokio",
+ "tracing",
+]
+
+[[package]]
+name = "hkdf"
+version = "0.12.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7b5f8eb2ad728638ea2c7d47a21db23b7b58a72ed6a38256b8a1849f15fbbdf7"
+dependencies = [
+ "hmac",
+]
+
+[[package]]
+name = "hmac"
+version = "0.12.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e"
+dependencies = [
+ "digest",
+]
+
+[[package]]
+name = "hostname"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3c731c3e10504cc8ed35cfe2f1db4c9274c3d35fa486e3b31df46f068ef3e867"
+dependencies = [
+ "libc",
+ "match_cfg",
+ "winapi",
+]
+
+[[package]]
+name = "http"
+version = "0.2.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1"
+dependencies = [
+ "bytes",
+ "fnv",
+ "itoa",
+]
+
+[[package]]
+name = "http"
+version = "1.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f4a85d31aea989eead29a3aaf9e1115a180df8282431156e533de47660892565"
+dependencies = [
+ "bytes",
+ "fnv",
+ "itoa",
+]
+
+[[package]]
+name = "http-body"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184"
+dependencies = [
+ "bytes",
+ "http 1.3.1",
+]
+
+[[package]]
+name = "http-body-util"
+version = "0.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a"
+dependencies = [
+ "bytes",
+ "futures-core",
+ "http 1.3.1",
+ "http-body",
+ "pin-project-lite",
+]
+
+[[package]]
+name = "httparse"
+version = "1.10.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87"
+
+[[package]]
+name = "humantime"
+version = "1.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "df004cfca50ef23c36850aaaa59ad52cc70d0e90243c3c7737a4dd32dc7a3c4f"
+dependencies = [
+ "quick-error",
+]
+
+[[package]]
+name = "hyper"
+version = "1.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cc2b571658e38e0c01b1fdca3bbbe93c00d3d71693ff2770043f8c29bc7d6f80"
+dependencies = [
+ "bytes",
+ "futures-channel",
+ "futures-util",
+ "h2",
+ "http 1.3.1",
+ "http-body",
+ "httparse",
+ "itoa",
+ "pin-project-lite",
+ "smallvec",
+ "tokio",
+ "want",
+]
+
+[[package]]
+name = "hyper-util"
+version = "0.1.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "df2dcfbe0677734ab2f3ffa7fa7bfd4706bfdc1ef393f2ee30184aed67e631b4"
+dependencies = [
+ "bytes",
+ "futures-channel",
+ "futures-util",
+ "http 1.3.1",
+ "http-body",
+ "hyper",
+ "pin-project-lite",
+ "socket2",
+ "tokio",
+ "tower-service",
+ "tracing",
+]
+
+[[package]]
+name = "iana-time-zone"
+version = "0.1.61"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "235e081f3925a06703c2d0117ea8b91f042756fd6e7a6e5d901e8ca1a996b220"
+dependencies = [
+ "android_system_properties",
+ "core-foundation-sys",
+ "iana-time-zone-haiku",
+ "js-sys",
+ "wasm-bindgen",
+ "windows-core 0.52.0",
+]
+
+[[package]]
+name = "iana-time-zone-haiku"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f"
+dependencies = [
+ "cc",
+]
+
+[[package]]
+name = "icu_collections"
+version = "1.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "db2fa452206ebee18c4b5c2274dbf1de17008e874b4dc4f0aea9d01ca79e4526"
+dependencies = [
+ "displaydoc",
+ "yoke",
+ "zerofrom",
+ "zerovec",
+]
+
+[[package]]
+name = "icu_locid"
+version = "1.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "13acbb8371917fc971be86fc8057c41a64b521c184808a698c02acc242dbf637"
+dependencies = [
+ "displaydoc",
+ "litemap",
+ "tinystr",
+ "writeable",
+ "zerovec",
+]
+
+[[package]]
+name = "icu_locid_transform"
+version = "1.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "01d11ac35de8e40fdeda00d9e1e9d92525f3f9d887cdd7aa81d727596788b54e"
+dependencies = [
+ "displaydoc",
+ "icu_locid",
+ "icu_locid_transform_data",
+ "icu_provider",
+ "tinystr",
+ "zerovec",
+]
+
+[[package]]
+name = "icu_locid_transform_data"
+version = "1.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fdc8ff3388f852bede6b579ad4e978ab004f139284d7b28715f773507b946f6e"
+
+[[package]]
+name = "icu_normalizer"
+version = "1.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "19ce3e0da2ec68599d193c93d088142efd7f9c5d6fc9b803774855747dc6a84f"
+dependencies = [
+ "displaydoc",
+ "icu_collections",
+ "icu_normalizer_data",
+ "icu_properties",
+ "icu_provider",
+ "smallvec",
+ "utf16_iter",
+ "utf8_iter",
+ "write16",
+ "zerovec",
+]
+
+[[package]]
+name = "icu_normalizer_data"
+version = "1.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f8cafbf7aa791e9b22bec55a167906f9e1215fd475cd22adfcf660e03e989516"
+
+[[package]]
+name = "icu_properties"
+version = "1.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "93d6020766cfc6302c15dbbc9c8778c37e62c14427cb7f6e601d849e092aeef5"
+dependencies = [
+ "displaydoc",
+ "icu_collections",
+ "icu_locid_transform",
+ "icu_properties_data",
+ "icu_provider",
+ "tinystr",
+ "zerovec",
+]
+
+[[package]]
+name = "icu_properties_data"
+version = "1.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "67a8effbc3dd3e4ba1afa8ad918d5684b8868b3b26500753effea8d2eed19569"
+
+[[package]]
+name = "icu_provider"
+version = "1.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6ed421c8a8ef78d3e2dbc98a973be2f3770cb42b606e3ab18d6237c4dfde68d9"
+dependencies = [
+ "displaydoc",
+ "icu_locid",
+ "icu_provider_macros",
+ "stable_deref_trait",
+ "tinystr",
+ "writeable",
+ "yoke",
+ "zerofrom",
+ "zerovec",
+]
+
+[[package]]
+name = "icu_provider_macros"
+version = "1.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.100",
+]
+
+[[package]]
+name = "idna"
+version = "1.0.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "686f825264d630750a544639377bae737628043f20d38bbc029e8f29ea968a7e"
+dependencies = [
+ "idna_adapter",
+ "smallvec",
+ "utf8_iter",
+]
+
+[[package]]
+name = "idna_adapter"
+version = "1.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "daca1df1c957320b2cf139ac61e7bd64fed304c5040df000a745aa1de3b4ef71"
+dependencies = [
+ "icu_normalizer",
+ "icu_properties",
+]
+
+[[package]]
+name = "if-addrs"
+version = "0.10.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cabb0019d51a643781ff15c9c8a3e5dedc365c47211270f4e8f82812fedd8f0a"
+dependencies = [
+ "libc",
+ "windows-sys 0.48.0",
+]
+
+[[package]]
+name = "if-watch"
+version = "3.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cdf9d64cfcf380606e64f9a0bcf493616b65331199f984151a6fa11a7b3cde38"
+dependencies = [
+ "async-io",
+ "core-foundation",
+ "fnv",
+ "futures",
+ "if-addrs",
+ "ipnet",
+ "log",
+ "netlink-packet-core",
+ "netlink-packet-route",
+ "netlink-proto",
+ "netlink-sys",
+ "rtnetlink",
+ "system-configuration",
+ "tokio",
+ "windows",
+]
+
+[[package]]
+name = "igd-next"
+version = "0.15.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "76b0d7d4541def58a37bf8efc559683f21edce7c82f0d866c93ac21f7e098f93"
+dependencies = [
+ "async-trait",
+ "attohttpc",
+ "bytes",
+ "futures",
+ "http 1.3.1",
+ "http-body-util",
+ "hyper",
+ "hyper-util",
+ "log",
+ "rand 0.8.5",
+ "tokio",
+ "url",
+ "xmltree",
+]
+
+[[package]]
+name = "indexmap"
+version = "2.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3954d50fe15b02142bf25d3b8bdadb634ec3948f103d04ffe3031bc8fe9d7058"
+dependencies = [
+ "equivalent",
+ "hashbrown",
+]
+
+[[package]]
+name = "inout"
+version = "0.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01"
+dependencies = [
+ "block-padding",
+ "generic-array",
+]
+
+[[package]]
+name = "instant"
+version = "0.1.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222"
+dependencies = [
+ "cfg-if",
+ "js-sys",
+ "wasm-bindgen",
+ "web-sys",
+]
+
+[[package]]
+name = "interceptor"
+version = "0.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5927883184e6a819b22d5e4f5f7bc7ca134fde9b2026fbddd8d95249746ba21e"
+dependencies = [
+ "async-trait",
+ "bytes",
+ "log",
+ "rand 0.8.5",
+ "rtcp",
+ "rtp 0.9.0",
+ "thiserror 1.0.69",
+ "tokio",
+ "waitgroup",
+ "webrtc-srtp",
+ "webrtc-util 0.8.1",
+]
+
+[[package]]
+name = "ipconfig"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b58db92f96b720de98181bbbe63c831e87005ab460c1bf306eb2622b4707997f"
+dependencies = [
+ "socket2",
+ "widestring",
+ "windows-sys 0.48.0",
+ "winreg",
+]
+
+[[package]]
+name = "ipnet"
+version = "2.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130"
+
+[[package]]
+name = "is_terminal_polyfill"
+version = "1.70.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf"
+
+[[package]]
+name = "itoa"
+version = "1.0.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c"
+
+[[package]]
+name = "jobserver"
+version = "0.1.32"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "48d1dbcbbeb6a7fec7e059840aa538bd62aaccf972c7346c4d9d2059312853d0"
+dependencies = [
+ "libc",
+]
+
+[[package]]
+name = "js-sys"
+version = "0.3.77"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1cfaf33c695fc6e08064efbc1f72ec937429614f25eef83af942d0e227c3a28f"
+dependencies = [
+ "once_cell",
+ "wasm-bindgen",
+]
+
+[[package]]
+name = "jsonwebtoken"
+version = "9.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5a87cc7a48537badeae96744432de36f4be2b4a34a05a5ef32e9dd8a1c169dde"
+dependencies = [
+ "base64 0.22.1",
+ "js-sys",
+ "pem",
+ "ring 0.17.14",
+ "serde",
+ "serde_json",
+ "simple_asn1",
+]
+
+[[package]]
+name = "lazy_static"
+version = "1.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"
+
+[[package]]
+name = "libc"
+version = "0.2.171"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c19937216e9d3aa9956d9bb8dfc0b0c8beb6058fc4f7a4dc4d850edf86a237d6"
+
+[[package]]
+name = "libp2p"
+version = "0.54.1"
+source = "git+https://github.com/aukilabs/rust-libp2p.git?branch=fix%2Fgossipsub-wasm#b6bd176a869cb1b760a5cac68bc67bd0ba571536"
+dependencies = [
+ "bytes",
+ "either",
+ "futures",
+ "futures-timer",
+ "getrandom 0.2.15",
+ "libp2p-allow-block-list",
+ "libp2p-autonat",
+ "libp2p-connection-limits",
+ "libp2p-core",
+ "libp2p-dns",
+ "libp2p-gossipsub",
+ "libp2p-identify",
+ "libp2p-identity",
+ "libp2p-kad",
+ "libp2p-mdns",
+ "libp2p-metrics",
+ "libp2p-noise",
+ "libp2p-quic",
+ "libp2p-relay",
+ "libp2p-swarm",
+ "libp2p-tcp",
+ "libp2p-upnp",
+ "libp2p-yamux",
+ "multiaddr",
+ "pin-project",
+ "rw-stream-sink",
+ "thiserror 1.0.69",
+]
+
+[[package]]
+name = "libp2p-allow-block-list"
+version = "0.4.1"
+source = "git+https://github.com/aukilabs/rust-libp2p.git?branch=fix%2Fgossipsub-wasm#b6bd176a869cb1b760a5cac68bc67bd0ba571536"
+dependencies = [
+ "libp2p-core",
+ "libp2p-identity",
+ "libp2p-swarm",
+ "void",
+]
+
+[[package]]
+name = "libp2p-autonat"
+version = "0.13.1"
+source = "git+https://github.com/aukilabs/rust-libp2p.git?branch=fix%2Fgossipsub-wasm#b6bd176a869cb1b760a5cac68bc67bd0ba571536"
+dependencies = [
+ "async-trait",
+ "asynchronous-codec",
+ "bytes",
+ "either",
+ "futures",
+ "futures-bounded",
+ "futures-timer",
+ "libp2p-core",
+ "libp2p-identity",
+ "libp2p-request-response",
+ "libp2p-swarm",
+ "quick-protobuf",
+ "quick-protobuf-codec 0.3.1 (git+https://github.com/aukilabs/rust-libp2p.git?branch=fix%2Fgossipsub-wasm)",
+ "rand 0.8.5",
+ "rand_core 0.6.4",
+ "thiserror 1.0.69",
+ "tracing",
+ "void",
+ "web-time",
+]
+
+[[package]]
+name = "libp2p-connection-limits"
+version = "0.4.0"
+source = "git+https://github.com/aukilabs/rust-libp2p.git?branch=fix%2Fgossipsub-wasm#b6bd176a869cb1b760a5cac68bc67bd0ba571536"
+dependencies = [
+ "libp2p-core",
+ "libp2p-identity",
+ "libp2p-swarm",
+ "void",
+]
+
+[[package]]
+name = "libp2p-core"
+version = "0.42.0"
+source = "git+https://github.com/aukilabs/rust-libp2p.git?branch=fix%2Fgossipsub-wasm#b6bd176a869cb1b760a5cac68bc67bd0ba571536"
+dependencies = [
+ "either",
+ "fnv",
+ "futures",
+ "futures-timer",
+ "libp2p-identity",
+ "multiaddr",
+ "multihash",
+ "multistream-select",
+ "once_cell",
+ "parking_lot",
+ "pin-project",
+ "quick-protobuf",
+ "rand 0.8.5",
+ "rw-stream-sink",
+ "serde",
+ "smallvec",
+ "thiserror 1.0.69",
+ "tracing",
+ "unsigned-varint",
+ "void",
+ "web-time",
+]
+
+[[package]]
+name = "libp2p-dns"
+version = "0.42.0"
+source = "git+https://github.com/aukilabs/rust-libp2p.git?branch=fix%2Fgossipsub-wasm#b6bd176a869cb1b760a5cac68bc67bd0ba571536"
+dependencies = [
+ "async-trait",
+ "futures",
+ "hickory-resolver",
+ "libp2p-core",
+ "libp2p-identity",
+ "parking_lot",
+ "smallvec",
+ "tracing",
+]
+
+[[package]]
+name = "libp2p-gossipsub"
+version = "0.47.2"
+source = "git+https://github.com/aukilabs/rust-libp2p.git?branch=fix%2Fgossipsub-wasm#b6bd176a869cb1b760a5cac68bc67bd0ba571536"
+dependencies = [
+ "asynchronous-codec",
+ "base64 0.22.1",
+ "byteorder",
+ "bytes",
+ "either",
+ "fnv",
+ "futures",
+ "futures-ticker",
+ "getrandom 0.2.15",
+ "hex_fmt",
+ "instant",
+ "libp2p-core",
+ "libp2p-identity",
+ "libp2p-swarm",
+ "prometheus-client",
+ "quick-protobuf",
+ "quick-protobuf-codec 0.3.1 (git+https://github.com/aukilabs/rust-libp2p.git?branch=fix%2Fgossipsub-wasm)",
+ "rand 0.8.5",
+ "regex",
+ "serde",
+ "sha2",
+ "smallvec",
+ "tracing",
+ "void",
+ "web-time",
+]
+
+[[package]]
+name = "libp2p-identify"
+version = "0.45.1"
+source = "git+https://github.com/aukilabs/rust-libp2p.git?branch=fix%2Fgossipsub-wasm#b6bd176a869cb1b760a5cac68bc67bd0ba571536"
+dependencies = [
+ "asynchronous-codec",
+ "either",
+ "futures",
+ "futures-bounded",
+ "futures-timer",
+ "libp2p-core",
+ "libp2p-identity",
+ "libp2p-swarm",
+ "lru",
+ "quick-protobuf",
+ "quick-protobuf-codec 0.3.1 (git+https://github.com/aukilabs/rust-libp2p.git?branch=fix%2Fgossipsub-wasm)",
+ "smallvec",
+ "thiserror 1.0.69",
+ "tracing",
+ "void",
+]
+
+[[package]]
+name = "libp2p-identity"
+version = "0.2.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "257b5621d159b32282eac446bed6670c39c7dc68a200a992d8f056afa0066f6d"
+dependencies = [
+ "bs58",
+ "ed25519-dalek",
+ "hkdf",
+ "multihash",
+ "quick-protobuf",
+ "rand 0.8.5",
+ "serde",
+ "sha2",
+ "thiserror 1.0.69",
+ "tracing",
+ "zeroize",
+]
+
+[[package]]
+name = "libp2p-kad"
+version = "0.47.0"
+source = "git+https://github.com/aukilabs/rust-libp2p.git?branch=fix%2Fgossipsub-wasm#b6bd176a869cb1b760a5cac68bc67bd0ba571536"
+dependencies = [
+ "arrayvec",
+ "asynchronous-codec",
+ "bytes",
+ "either",
+ "fnv",
+ "futures",
+ "futures-bounded",
+ "futures-timer",
+ "libp2p-core",
+ "libp2p-identity",
+ "libp2p-swarm",
+ "quick-protobuf",
+ "quick-protobuf-codec 0.3.1 (git+https://github.com/aukilabs/rust-libp2p.git?branch=fix%2Fgossipsub-wasm)",
+ "rand 0.8.5",
+ "serde",
+ "sha2",
+ "smallvec",
+ "thiserror 1.0.69",
+ "tracing",
+ "uint",
+ "void",
+ "web-time",
+]
+
+[[package]]
+name = "libp2p-mdns"
+version = "0.46.0"
+source = "git+https://github.com/aukilabs/rust-libp2p.git?branch=fix%2Fgossipsub-wasm#b6bd176a869cb1b760a5cac68bc67bd0ba571536"
+dependencies = [
+ "data-encoding",
+ "futures",
+ "hickory-proto",
+ "if-watch",
+ "libp2p-core",
+ "libp2p-identity",
+ "libp2p-swarm",
+ "rand 0.8.5",
+ "smallvec",
+ "socket2",
+ "tokio",
+ "tracing",
+ "void",
+]
+
+[[package]]
+name = "libp2p-metrics"
+version = "0.15.0"
+source = "git+https://github.com/aukilabs/rust-libp2p.git?branch=fix%2Fgossipsub-wasm#b6bd176a869cb1b760a5cac68bc67bd0ba571536"
+dependencies = [
+ "futures",
+ "libp2p-core",
+ "libp2p-gossipsub",
+ "libp2p-identify",
+ "libp2p-identity",
+ "libp2p-kad",
+ "libp2p-relay",
+ "libp2p-swarm",
+ "pin-project",
+ "prometheus-client",
+ "web-time",
+]
+
+[[package]]
+name = "libp2p-noise"
+version = "0.45.0"
+source = "git+https://github.com/aukilabs/rust-libp2p.git?branch=fix%2Fgossipsub-wasm#b6bd176a869cb1b760a5cac68bc67bd0ba571536"
+dependencies = [
+ "asynchronous-codec",
+ "bytes",
+ "curve25519-dalek",
+ "futures",
+ "libp2p-core",
+ "libp2p-identity",
+ "multiaddr",
+ "multihash",
+ "once_cell",
+ "quick-protobuf",
+ "rand 0.8.5",
+ "sha2",
+ "snow",
+ "static_assertions",
+ "thiserror 1.0.69",
+ "tracing",
+ "x25519-dalek",
+ "zeroize",
+]
+
+[[package]]
+name = "libp2p-quic"
+version = "0.11.1"
+source = "git+https://github.com/aukilabs/rust-libp2p.git?branch=fix%2Fgossipsub-wasm#b6bd176a869cb1b760a5cac68bc67bd0ba571536"
+dependencies = [
+ "bytes",
+ "futures",
+ "futures-timer",
+ "if-watch",
+ "libp2p-core",
+ "libp2p-identity",
+ "libp2p-tls",
+ "parking_lot",
+ "quinn",
+ "rand 0.8.5",
+ "ring 0.17.14",
+ "rustls 0.23.23",
+ "socket2",
+ "thiserror 1.0.69",
+ "tokio",
+ "tracing",
+]
+
+[[package]]
+name = "libp2p-relay"
+version = "0.18.0"
+source = "git+https://github.com/aukilabs/rust-libp2p.git?branch=fix%2Fgossipsub-wasm#b6bd176a869cb1b760a5cac68bc67bd0ba571536"
+dependencies = [
+ "asynchronous-codec",
+ "bytes",
+ "either",
+ "futures",
+ "futures-bounded",
+ "futures-timer",
+ "libp2p-core",
+ "libp2p-identity",
+ "libp2p-swarm",
+ "quick-protobuf",
+ "quick-protobuf-codec 0.3.1 (git+https://github.com/aukilabs/rust-libp2p.git?branch=fix%2Fgossipsub-wasm)",
+ "rand 0.8.5",
+ "static_assertions",
+ "thiserror 1.0.69",
+ "tracing",
+ "void",
+ "web-time",
+]
+
+[[package]]
+name = "libp2p-request-response"
+version = "0.27.0"
+source = "git+https://github.com/aukilabs/rust-libp2p.git?branch=fix%2Fgossipsub-wasm#b6bd176a869cb1b760a5cac68bc67bd0ba571536"
+dependencies = [
+ "async-trait",
+ "futures",
+ "futures-bounded",
+ "futures-timer",
+ "libp2p-core",
+ "libp2p-identity",
+ "libp2p-swarm",
+ "rand 0.8.5",
+ "smallvec",
+ "tracing",
+ "void",
+ "web-time",
+]
+
+[[package]]
+name = "libp2p-stream"
+version = "0.2.0-alpha"
+source = "git+https://github.com/aukilabs/rust-libp2p.git?branch=fix%2Fgossipsub-wasm#b6bd176a869cb1b760a5cac68bc67bd0ba571536"
+dependencies = [
+ "futures",
+ "libp2p-core",
+ "libp2p-identity",
+ "libp2p-swarm",
+ "rand 0.8.5",
+ "tracing",
+ "void",
+]
+
+[[package]]
+name = "libp2p-swarm"
+version = "0.45.2"
+source = "git+https://github.com/aukilabs/rust-libp2p.git?branch=fix%2Fgossipsub-wasm#b6bd176a869cb1b760a5cac68bc67bd0ba571536"
+dependencies = [
+ "either",
+ "fnv",
+ "futures",
+ "futures-timer",
+ "getrandom 0.2.15",
+ "libp2p-core",
+ "libp2p-identity",
+ "libp2p-swarm-derive",
+ "lru",
+ "multistream-select",
+ "once_cell",
+ "rand 0.8.5",
+ "smallvec",
+ "tokio",
+ "tracing",
+ "void",
+ "wasm-bindgen-futures",
+ "web-time",
+]
+
+[[package]]
+name = "libp2p-swarm-derive"
+version = "0.35.0"
+source = "git+https://github.com/aukilabs/rust-libp2p.git?branch=fix%2Fgossipsub-wasm#b6bd176a869cb1b760a5cac68bc67bd0ba571536"
+dependencies = [
+ "heck 0.5.0",
+ "proc-macro2",
+ "quote",
+ "syn 2.0.100",
+]
+
+[[package]]
+name = "libp2p-tcp"
+version = "0.42.0"
+source = "git+https://github.com/aukilabs/rust-libp2p.git?branch=fix%2Fgossipsub-wasm#b6bd176a869cb1b760a5cac68bc67bd0ba571536"
+dependencies = [
+ "futures",
+ "futures-timer",
+ "if-watch",
+ "libc",
+ "libp2p-core",
+ "libp2p-identity",
+ "socket2",
+ "tokio",
+ "tracing",
+]
+
+[[package]]
+name = "libp2p-tls"
+version = "0.5.0"
+source = "git+https://github.com/aukilabs/rust-libp2p.git?branch=fix%2Fgossipsub-wasm#b6bd176a869cb1b760a5cac68bc67bd0ba571536"
+dependencies = [
+ "futures",
+ "futures-rustls",
+ "libp2p-core",
+ "libp2p-identity",
+ "rcgen",
+ "ring 0.17.14",
+ "rustls 0.23.23",
+ "rustls-webpki 0.101.7",
+ "thiserror 1.0.69",
+ "x509-parser 0.16.0",
+ "yasna",
+]
+
+[[package]]
+name = "libp2p-upnp"
+version = "0.3.1"
+source = "git+https://github.com/aukilabs/rust-libp2p.git?branch=fix%2Fgossipsub-wasm#b6bd176a869cb1b760a5cac68bc67bd0ba571536"
+dependencies = [
+ "futures",
+ "futures-timer",
+ "igd-next",
+ "libp2p-core",
+ "libp2p-swarm",
+ "tokio",
+ "tracing",
+ "void",
+]
+
+[[package]]
+name = "libp2p-webrtc"
+version = "0.8.0-alpha"
+source = "git+https://github.com/aukilabs/rust-libp2p.git?branch=fix%2Fgossipsub-wasm#b6bd176a869cb1b760a5cac68bc67bd0ba571536"
+dependencies = [
+ "async-trait",
+ "bytes",
+ "futures",
+ "futures-timer",
+ "hex",
+ "if-watch",
+ "libp2p-core",
+ "libp2p-identity",
+ "libp2p-noise",
+ "libp2p-webrtc-utils",
+ "multihash",
+ "rand 0.8.5",
+ "rcgen",
+ "serde",
+ "stun 0.6.0",
+ "thiserror 1.0.69",
+ "tinytemplate",
+ "tokio",
+ "tokio-util",
+ "tracing",
+ "webrtc",
+]
+
+[[package]]
+name = "libp2p-webrtc-utils"
+version = "0.3.0"
+source = "git+https://github.com/aukilabs/rust-libp2p.git?branch=fix%2Fgossipsub-wasm#b6bd176a869cb1b760a5cac68bc67bd0ba571536"
+dependencies = [
+ "asynchronous-codec",
+ "bytes",
+ "futures",
+ "hex",
+ "libp2p-core",
+ "libp2p-identity",
+ "libp2p-noise",
+ "quick-protobuf",
+ "quick-protobuf-codec 0.3.1 (git+https://github.com/aukilabs/rust-libp2p.git?branch=fix%2Fgossipsub-wasm)",
+ "rand 0.8.5",
+ "serde",
+ "sha2",
+ "thiserror 1.0.69",
+ "tinytemplate",
+ "tracing",
+]
+
+[[package]]
+name = "libp2p-webrtc-websys"
+version = "0.4.0-alpha.2"
+source = "git+https://github.com/aukilabs/rust-libp2p.git?branch=fix%2Fgossipsub-wasm#b6bd176a869cb1b760a5cac68bc67bd0ba571536"
+dependencies = [
+ "bytes",
+ "futures",
+ "getrandom 0.2.15",
+ "hex",
+ "js-sys",
+ "libp2p-core",
+ "libp2p-identity",
+ "libp2p-webrtc-utils",
+ "send_wrapper 0.6.0",
+ "thiserror 1.0.69",
+ "tracing",
+ "wasm-bindgen",
+ "wasm-bindgen-futures",
+ "web-sys",
+]
+
+[[package]]
+name = "libp2p-yamux"
+version = "0.46.0"
+source = "git+https://github.com/aukilabs/rust-libp2p.git?branch=fix%2Fgossipsub-wasm#b6bd176a869cb1b760a5cac68bc67bd0ba571536"
+dependencies = [
+ "either",
+ "futures",
+ "libp2p-core",
+ "thiserror 1.0.69",
+ "tracing",
+ "yamux 0.12.1",
+ "yamux 0.13.4",
+]
+
+[[package]]
+name = "linked-hash-map"
+version = "0.5.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f"
+
+[[package]]
+name = "linux-raw-sys"
+version = "0.4.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab"
+
+[[package]]
+name = "linux-raw-sys"
+version = "0.9.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6db9c683daf087dc577b7506e9695b3d556a9f3849903fa28186283afd6809e9"
+
+[[package]]
+name = "litemap"
+version = "0.7.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "23fb14cb19457329c82206317a5663005a4d404783dc74f4252769b0d5f42856"
+
+[[package]]
+name = "lock_api"
+version = "0.4.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17"
+dependencies = [
+ "autocfg",
+ "scopeguard",
+]
+
+[[package]]
+name = "lockfree-object-pool"
+version = "0.1.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9374ef4228402d4b7e403e5838cb880d9ee663314b0a900d5a6aabf0c213552e"
+
+[[package]]
+name = "log"
+version = "0.4.26"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "30bde2b3dc3671ae49d8e2e9f044c7c005836e7a023ee57cffa25ab82764bb9e"
+
+[[package]]
+name = "lru"
+version = "0.12.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38"
+dependencies = [
+ "hashbrown",
+]
+
+[[package]]
+name = "lru-cache"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "31e24f1ad8321ca0e8a1e0ac13f23cb668e6f5466c2c57319f6a5cf1cc8e3b1c"
+dependencies = [
+ "linked-hash-map",
+]
+
+[[package]]
+name = "lzma-rs"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "297e814c836ae64db86b36cf2a557ba54368d03f6afcd7d947c266692f71115e"
+dependencies = [
+ "byteorder",
+ "crc",
+]
+
+[[package]]
+name = "match_cfg"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ffbee8634e0d45d258acb448e7eaab3fce7a0a467395d4d9f228e3c1f01fb2e4"
+
+[[package]]
+name = "matchers"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558"
+dependencies = [
+ "regex-automata 0.1.10",
+]
+
+[[package]]
+name = "md-5"
+version = "0.10.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf"
+dependencies = [
+ "cfg-if",
+ "digest",
+]
+
+[[package]]
+name = "memchr"
+version = "2.7.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3"
+
+[[package]]
+name = "memoffset"
+version = "0.7.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5de893c32cde5f383baa4c04c5d6dbdd735cfd4a794b0debdb2bb1b421da5ff4"
+dependencies = [
+ "autocfg",
+]
+
+[[package]]
+name = "minimal-lexical"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
+
+[[package]]
+name = "miniz_oxide"
+version = "0.8.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8e3e04debbb59698c15bacbb6d93584a8c0ca9cc3213cb423d31f760d8843ce5"
+dependencies = [
+ "adler2",
+]
+
+[[package]]
+name = "mio"
+version = "1.0.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2886843bf800fba2e3377cff24abf6379b4c4d5c6681eaf9ea5b0d15090450bd"
+dependencies = [
+ "libc",
+ "wasi 0.11.0+wasi-snapshot-preview1",
+ "windows-sys 0.52.0",
+]
+
+[[package]]
+name = "multiaddr"
+version = "0.18.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fe6351f60b488e04c1d21bc69e56b89cb3f5e8f5d22557d6e8031bdfd79b6961"
+dependencies = [
+ "arrayref",
+ "byteorder",
+ "data-encoding",
+ "libp2p-identity",
+ "multibase",
+ "multihash",
+ "percent-encoding",
+ "serde",
+ "static_assertions",
+ "unsigned-varint",
+ "url",
+]
+
+[[package]]
+name = "multibase"
+version = "0.9.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9b3539ec3c1f04ac9748a260728e855f261b4977f5c3406612c884564f329404"
+dependencies = [
+ "base-x",
+ "data-encoding",
+ "data-encoding-macro",
+]
+
+[[package]]
+name = "multihash"
+version = "0.19.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6b430e7953c29dd6a09afc29ff0bb69c6e306329ee6794700aee27b76a1aea8d"
+dependencies = [
+ "core2",
+ "serde",
+ "unsigned-varint",
+]
+
+[[package]]
+name = "multistream-select"
+version = "0.13.0"
+source = "git+https://github.com/aukilabs/rust-libp2p.git?branch=fix%2Fgossipsub-wasm#b6bd176a869cb1b760a5cac68bc67bd0ba571536"
+dependencies = [
+ "bytes",
+ "futures",
+ "pin-project",
+ "smallvec",
+ "tracing",
+ "unsigned-varint",
+]
+
+[[package]]
+name = "netlink-packet-core"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "72724faf704479d67b388da142b186f916188505e7e0b26719019c525882eda4"
+dependencies = [
+ "anyhow",
+ "byteorder",
+ "netlink-packet-utils",
+]
+
+[[package]]
+name = "netlink-packet-route"
+version = "0.17.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "053998cea5a306971f88580d0829e90f270f940befd7cf928da179d4187a5a66"
+dependencies = [
+ "anyhow",
+ "bitflags 1.3.2",
+ "byteorder",
+ "libc",
+ "netlink-packet-core",
+ "netlink-packet-utils",
+]
+
+[[package]]
+name = "netlink-packet-utils"
+version = "0.5.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0ede8a08c71ad5a95cdd0e4e52facd37190977039a4704eb82a283f713747d34"
+dependencies = [
+ "anyhow",
+ "byteorder",
+ "paste",
+ "thiserror 1.0.69",
+]
+
+[[package]]
+name = "netlink-proto"
+version = "0.11.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "72452e012c2f8d612410d89eea01e2d9b56205274abb35d53f60200b2ec41d60"
+dependencies = [
+ "bytes",
+ "futures",
+ "log",
+ "netlink-packet-core",
+ "netlink-sys",
+ "thiserror 2.0.12",
+]
+
+[[package]]
+name = "netlink-sys"
+version = "0.8.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "16c903aa70590cb93691bf97a767c8d1d6122d2cc9070433deb3bbf36ce8bd23"
+dependencies = [
+ "bytes",
+ "futures",
+ "libc",
+ "log",
+ "tokio",
+]
+
+[[package]]
+name = "networking"
+version = "0.1.0"
+source = "git+https://github.com/aukilabs/posemesh.git?branch=chore%2Freconstruction-integration#78d50d88988c5e731764efb6a73797edc5c2c1c6"
+dependencies = [
+ "cbindgen",
+ "console_error_panic_hook",
+ "futures",
+ "gloo-timers 0.3.0",
+ "libp2p",
+ "libp2p-stream",
+ "libp2p-webrtc",
+ "libp2p-webrtc-websys",
+ "rand 0.8.5",
+ "runtime",
+ "serde",
+ "serde-wasm-bindgen",
+ "serde_json",
+ "tokio",
+ "tracing",
+ "tracing-subscriber",
+ "tracing-wasm",
+ "wasm-bindgen",
+ "wasm-bindgen-futures",
+]
+
+[[package]]
+name = "nix"
+version = "0.26.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "598beaf3cc6fdd9a5dfb1630c2800c7acd31df7aaf0f565796fba2b53ca1af1b"
+dependencies = [
+ "bitflags 1.3.2",
+ "cfg-if",
+ "libc",
+ "memoffset",
+ "pin-utils",
+]
+
+[[package]]
+name = "nohash-hasher"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2bf50223579dc7cdcfb3bfcacf7069ff68243f8c363f62ffa99cf000a6b9c451"
+
+[[package]]
+name = "nom"
+version = "7.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a"
+dependencies = [
+ "memchr",
+ "minimal-lexical",
+]
+
+[[package]]
+name = "nu-ansi-term"
+version = "0.46.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84"
+dependencies = [
+ "overload",
+ "winapi",
+]
+
+[[package]]
+name = "num-bigint"
+version = "0.4.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9"
+dependencies = [
+ "num-integer",
+ "num-traits",
+]
+
+[[package]]
+name = "num-conv"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9"
+
+[[package]]
+name = "num-integer"
+version = "0.1.46"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f"
+dependencies = [
+ "num-traits",
+]
+
+[[package]]
+name = "num-traits"
+version = "0.2.19"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841"
+dependencies = [
+ "autocfg",
+]
+
+[[package]]
+name = "num_cpus"
+version = "1.16.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43"
+dependencies = [
+ "hermit-abi 0.3.9",
+ "libc",
+]
+
+[[package]]
+name = "object"
+version = "0.36.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "62948e14d923ea95ea2c7c86c71013138b66525b86bdc08d2dcc262bdb497b87"
+dependencies = [
+ "memchr",
+]
+
+[[package]]
+name = "oid-registry"
+version = "0.6.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9bedf36ffb6ba96c2eb7144ef6270557b52e54b20c0a8e1eb2ff99a6c6959bff"
+dependencies = [
+ "asn1-rs 0.5.2",
+]
+
+[[package]]
+name = "oid-registry"
+version = "0.7.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a8d8034d9489cdaf79228eb9f6a3b8d7bb32ba00d6645ebd48eef4077ceb5bd9"
+dependencies = [
+ "asn1-rs 0.6.2",
+]
+
+[[package]]
+name = "once_cell"
+version = "1.21.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cde51589ab56b20a6f686b2c68f7a0bd6add753d697abf720d63f8db3ab7b1ad"
+
+[[package]]
+name = "opaque-debug"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381"
+
+[[package]]
+name = "overload"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39"
+
+[[package]]
+name = "p256"
+version = "0.13.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c9863ad85fa8f4460f9c48cb909d38a0d689dba1f6f6988a5e3e0d31071bcd4b"
+dependencies = [
+ "ecdsa",
+ "elliptic-curve",
+ "primeorder",
+ "sha2",
+]
+
+[[package]]
+name = "p384"
+version = "0.13.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fe42f1670a52a47d448f14b6a5c61dd78fce51856e68edaa38f7ae3a46b8d6b6"
+dependencies = [
+ "ecdsa",
+ "elliptic-curve",
+ "primeorder",
+ "sha2",
+]
+
+[[package]]
+name = "parking"
+version = "2.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba"
+
+[[package]]
+name = "parking_lot"
+version = "0.12.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27"
+dependencies = [
+ "lock_api",
+ "parking_lot_core",
+]
+
+[[package]]
+name = "parking_lot_core"
+version = "0.9.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8"
+dependencies = [
+ "cfg-if",
+ "libc",
+ "redox_syscall",
+ "smallvec",
+ "windows-targets 0.52.6",
+]
+
+[[package]]
+name = "paste"
+version = "1.0.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a"
+
+[[package]]
+name = "pb-rs"
+version = "0.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "354a34df9c65b596152598001c0fe3393379ec2db03ae30b9985659422e2607e"
+dependencies = [
+ "clap 2.34.0",
+ "env_logger",
+ "log",
+ "nom",
+]
+
+[[package]]
+name = "pbkdf2"
+version = "0.12.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2"
+dependencies = [
+ "digest",
+ "hmac",
+]
+
+[[package]]
+name = "pem"
+version = "3.0.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "38af38e8470ac9dee3ce1bae1af9c1671fffc44ddfd8bd1d0a3445bf349a8ef3"
+dependencies = [
+ "base64 0.22.1",
+ "serde",
+]
+
+[[package]]
+name = "pem-rfc7468"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412"
+dependencies = [
+ "base64ct",
+]
+
+[[package]]
+name = "percent-encoding"
+version = "2.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e"
+
+[[package]]
+name = "pin-project"
+version = "1.1.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "677f1add503faace112b9f1373e43e9e054bfdd22ff1a63c1bc485eaec6a6a8a"
+dependencies = [
+ "pin-project-internal",
+]
+
+[[package]]
+name = "pin-project-internal"
+version = "1.1.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.100",
+]
+
+[[package]]
+name = "pin-project-lite"
+version = "0.2.16"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b"
+
+[[package]]
+name = "pin-utils"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
+
+[[package]]
+name = "pkcs8"
+version = "0.10.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7"
+dependencies = [
+ "der",
+ "spki",
+]
+
+[[package]]
+name = "pkg-config"
+version = "0.3.32"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c"
+
+[[package]]
+name = "polling"
+version = "3.7.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a604568c3202727d1507653cb121dbd627a58684eb09a820fd746bee38b4442f"
+dependencies = [
+ "cfg-if",
+ "concurrent-queue",
+ "hermit-abi 0.4.0",
+ "pin-project-lite",
+ "rustix 0.38.44",
+ "tracing",
+ "windows-sys 0.59.0",
+]
+
+[[package]]
+name = "poly1305"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8159bd90725d2df49889a078b54f4f79e87f1f8a8444194cdca81d38f5393abf"
+dependencies = [
+ "cpufeatures",
+ "opaque-debug",
+ "universal-hash",
+]
+
+[[package]]
+name = "polyval"
+version = "0.6.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9d1fe60d06143b2430aa532c94cfe9e29783047f06c0d7fd359a9a51b729fa25"
+dependencies = [
+ "cfg-if",
+ "cpufeatures",
+ "opaque-debug",
+ "universal-hash",
+]
+
+[[package]]
+name = "portable-atomic"
+version = "1.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "350e9b48cbc6b0e028b0473b114454c6316e57336ee184ceab6e53f72c178b3e"
+
+[[package]]
+name = "powerfmt"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391"
+
+[[package]]
+name = "ppv-lite86"
+version = "0.2.21"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9"
+dependencies = [
+ "zerocopy",
+]
+
+[[package]]
+name = "primeorder"
+version = "0.13.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "353e1ca18966c16d9deb1c69278edbc5f194139612772bd9537af60ac231e1e6"
+dependencies = [
+ "elliptic-curve",
+]
+
+[[package]]
+name = "proc-macro2"
+version = "1.0.94"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a31971752e70b8b2686d7e46ec17fb38dad4051d94024c88df49b667caea9c84"
+dependencies = [
+ "unicode-ident",
+]
+
+[[package]]
+name = "prometheus-client"
+version = "0.22.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "504ee9ff529add891127c4827eb481bd69dc0ebc72e9a682e187db4caa60c3ca"
+dependencies = [
+ "dtoa",
+ "itoa",
+ "parking_lot",
+ "prometheus-client-derive-encode",
+]
+
+[[package]]
+name = "prometheus-client-derive-encode"
+version = "0.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "440f724eba9f6996b75d63681b0a92b06947f1457076d503a4d2e2c8f56442b8"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.100",
+]
+
+[[package]]
+name = "quick-error"
+version = "1.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0"
+
+[[package]]
+name = "quick-protobuf"
+version = "0.8.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9d6da84cc204722a989e01ba2f6e1e276e190f22263d0cb6ce8526fcdb0d2e1f"
+dependencies = [
+ "byteorder",
+]
+
+[[package]]
+name = "quick-protobuf-codec"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "15a0580ab32b169745d7a39db2ba969226ca16738931be152a3209b409de2474"
+dependencies = [
+ "asynchronous-codec",
+ "bytes",
+ "quick-protobuf",
+ "thiserror 1.0.69",
+ "unsigned-varint",
+]
+
+[[package]]
+name = "quick-protobuf-codec"
+version = "0.3.1"
+source = "git+https://github.com/aukilabs/rust-libp2p.git?branch=fix%2Fgossipsub-wasm#b6bd176a869cb1b760a5cac68bc67bd0ba571536"
+dependencies = [
+ "asynchronous-codec",
+ "bytes",
+ "quick-protobuf",
+ "thiserror 1.0.69",
+ "unsigned-varint",
+]
+
+[[package]]
+name = "quinn"
+version = "0.11.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "62e96808277ec6f97351a2380e6c25114bc9e67037775464979f3037c92d05ef"
+dependencies = [
+ "bytes",
+ "futures-io",
+ "pin-project-lite",
+ "quinn-proto",
+ "quinn-udp",
+ "rustc-hash",
+ "rustls 0.23.23",
+ "socket2",
+ "thiserror 2.0.12",
+ "tokio",
+ "tracing",
+]
+
+[[package]]
+name = "quinn-proto"
+version = "0.11.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a2fe5ef3495d7d2e377ff17b1a8ce2ee2ec2a18cde8b6ad6619d65d0701c135d"
+dependencies = [
+ "bytes",
+ "getrandom 0.2.15",
+ "rand 0.8.5",
+ "ring 0.17.14",
+ "rustc-hash",
+ "rustls 0.23.23",
+ "rustls-pki-types",
+ "slab",
+ "thiserror 2.0.12",
+ "tinyvec",
+ "tracing",
+ "web-time",
+]
+
+[[package]]
+name = "quinn-udp"
+version = "0.5.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e46f3055866785f6b92bc6164b76be02ca8f2eb4b002c0354b28cf4c119e5944"
+dependencies = [
+ "cfg_aliases",
+ "libc",
+ "once_cell",
+ "socket2",
+ "tracing",
+ "windows-sys 0.59.0",
+]
+
+[[package]]
+name = "quote"
+version = "1.0.40"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d"
+dependencies = [
+ "proc-macro2",
+]
+
+[[package]]
+name = "rand"
+version = "0.8.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404"
+dependencies = [
+ "libc",
+ "rand_chacha 0.3.1",
+ "rand_core 0.6.4",
+]
+
+[[package]]
+name = "rand"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3779b94aeb87e8bd4e834cee3650289ee9e0d5677f976ecdb6d219e5f4f6cd94"
+dependencies = [
+ "rand_chacha 0.9.0",
+ "rand_core 0.9.3",
+ "zerocopy",
+]
+
+[[package]]
+name = "rand_chacha"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88"
+dependencies = [
+ "ppv-lite86",
+ "rand_core 0.6.4",
+]
+
+[[package]]
+name = "rand_chacha"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb"
+dependencies = [
+ "ppv-lite86",
+ "rand_core 0.9.3",
+]
+
+[[package]]
+name = "rand_core"
+version = "0.6.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c"
+dependencies = [
+ "getrandom 0.2.15",
+]
+
+[[package]]
+name = "rand_core"
+version = "0.9.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38"
+dependencies = [
+ "getrandom 0.3.1",
+]
+
+[[package]]
+name = "rcgen"
+version = "0.11.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "52c4f3084aa3bc7dfbba4eff4fab2a54db4324965d8872ab933565e6fbd83bc6"
+dependencies = [
+ "pem",
+ "ring 0.16.20",
+ "time",
+ "x509-parser 0.15.1",
+ "yasna",
+]
+
+[[package]]
+name = "redox_syscall"
+version = "0.5.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0b8c0c260b63a8219631167be35e6a988e9554dbd323f8bd08439c8ed1302bd1"
+dependencies = [
+ "bitflags 2.9.0",
+]
+
+[[package]]
+name = "regex"
+version = "1.11.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191"
+dependencies = [
+ "aho-corasick",
+ "memchr",
+ "regex-automata 0.4.9",
+ "regex-syntax 0.8.5",
+]
+
+[[package]]
+name = "regex-automata"
+version = "0.1.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132"
+dependencies = [
+ "regex-syntax 0.6.29",
+]
+
+[[package]]
+name = "regex-automata"
+version = "0.4.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908"
+dependencies = [
+ "aho-corasick",
+ "memchr",
+ "regex-syntax 0.8.5",
+]
+
+[[package]]
+name = "regex-syntax"
+version = "0.6.29"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1"
+
+[[package]]
+name = "regex-syntax"
+version = "0.8.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c"
+
+[[package]]
+name = "regexp"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9b045baed9fc5b2e285a599a42fb158bc483e1b6104de3a93d24bf4addd4727e"
+dependencies = [
+ "lazy_static",
+ "regex",
+]
+
+[[package]]
+name = "resolv-conf"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "52e44394d2086d010551b14b53b1f24e31647570cd1deb0379e2c21b329aba00"
+dependencies = [
+ "hostname",
+ "quick-error",
+]
+
+[[package]]
+name = "rfc6979"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f8dd2a808d456c4a54e300a23e9f5a67e122c3024119acbfd73e3bf664491cb2"
+dependencies = [
+ "hmac",
+ "subtle",
+]
+
+[[package]]
+name = "ring"
+version = "0.16.20"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc"
+dependencies = [
+ "cc",
+ "libc",
+ "once_cell",
+ "spin",
+ "untrusted 0.7.1",
+ "web-sys",
+ "winapi",
+]
+
+[[package]]
+name = "ring"
+version = "0.17.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7"
+dependencies = [
+ "cc",
+ "cfg-if",
+ "getrandom 0.2.15",
+ "libc",
+ "untrusted 0.9.0",
+ "windows-sys 0.52.0",
+]
+
+[[package]]
+name = "rtcp"
+version = "0.10.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "33648a781874466a62d89e265fee9f17e32bc7d05a256e6cca41bf97eadcd8aa"
+dependencies = [
+ "bytes",
+ "thiserror 1.0.69",
+ "webrtc-util 0.8.1",
+]
+
+[[package]]
+name = "rtnetlink"
+version = "0.13.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7a552eb82d19f38c3beed3f786bd23aa434ceb9ac43ab44419ca6d67a7e186c0"
+dependencies = [
+ "futures",
+ "log",
+ "netlink-packet-core",
+ "netlink-packet-route",
+ "netlink-packet-utils",
+ "netlink-proto",
+ "netlink-sys",
+ "nix",
+ "thiserror 1.0.69",
+ "tokio",
+]
+
+[[package]]
+name = "rtp"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e60482acbe8afb31edf6b1413103b7bca7a65004c423b3c3993749a083994fbe"
+dependencies = [
+ "bytes",
+ "rand 0.8.5",
+ "serde",
+ "thiserror 1.0.69",
+ "webrtc-util 0.8.1",
+]
+
+[[package]]
+name = "rtp"
+version = "0.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "47fca9bd66ae0b1f3f649b8f5003d6176433d7293b78b0fce7e1031816bdd99d"
+dependencies = [
+ "bytes",
+ "rand 0.8.5",
+ "serde",
+ "thiserror 1.0.69",
+ "webrtc-util 0.8.1",
+]
+
+[[package]]
+name = "runtime"
+version = "0.1.0"
+source = "git+https://github.com/aukilabs/posemesh.git?branch=chore%2Freconstruction-integration#78d50d88988c5e731764efb6a73797edc5c2c1c6"
+dependencies = [
+ "once_cell",
+ "tokio",
+]
+
+[[package]]
+name = "rustc-demangle"
+version = "0.1.24"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f"
+
+[[package]]
+name = "rustc-hash"
+version = "2.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d"
+
+[[package]]
+name = "rustc_version"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92"
+dependencies = [
+ "semver",
+]
+
+[[package]]
+name = "rusticata-macros"
+version = "4.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "faf0c4a6ece9950b9abdb62b1cfcf2a68b3b67a10ba445b3bb85be2a293d0632"
+dependencies = [
+ "nom",
+]
+
+[[package]]
+name = "rustix"
+version = "0.38.44"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154"
+dependencies = [
+ "bitflags 2.9.0",
+ "errno",
+ "libc",
+ "linux-raw-sys 0.4.15",
+ "windows-sys 0.59.0",
+]
+
+[[package]]
+name = "rustix"
+version = "1.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f7178faa4b75a30e269c71e61c353ce2748cf3d76f0c44c393f4e60abf49b825"
+dependencies = [
+ "bitflags 2.9.0",
+ "errno",
+ "libc",
+ "linux-raw-sys 0.9.2",
+ "windows-sys 0.59.0",
+]
+
+[[package]]
+name = "rustls"
+version = "0.21.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e"
+dependencies = [
+ "log",
+ "ring 0.17.14",
+ "rustls-webpki 0.101.7",
+ "sct",
+]
+
+[[package]]
+name = "rustls"
+version = "0.23.23"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "47796c98c480fce5406ef69d1c76378375492c3b0a0de587be0c1d9feb12f395"
+dependencies = [
+ "once_cell",
+ "ring 0.17.14",
+ "rustls-pki-types",
+ "rustls-webpki 0.102.8",
+ "subtle",
+ "zeroize",
+]
+
+[[package]]
+name = "rustls-pki-types"
+version = "1.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "917ce264624a4b4db1c364dcc35bfca9ded014d0a958cd47ad3e960e988ea51c"
+dependencies = [
+ "web-time",
+]
+
+[[package]]
+name = "rustls-webpki"
+version = "0.101.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765"
+dependencies = [
+ "ring 0.17.14",
+ "untrusted 0.9.0",
+]
+
+[[package]]
+name = "rustls-webpki"
+version = "0.102.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "64ca1bc8749bd4cf37b5ce386cc146580777b4e8572c7b97baf22c83f444bee9"
+dependencies = [
+ "ring 0.17.14",
+ "rustls-pki-types",
+ "untrusted 0.9.0",
+]
+
+[[package]]
+name = "rustversion"
+version = "1.0.20"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "eded382c5f5f786b989652c49544c4877d9f015cc22e145a5ea8ea66c2921cd2"
+
+[[package]]
+name = "rw-stream-sink"
+version = "0.4.0"
+source = "git+https://github.com/aukilabs/rust-libp2p.git?branch=fix%2Fgossipsub-wasm#b6bd176a869cb1b760a5cac68bc67bd0ba571536"
+dependencies = [
+ "futures",
+ "pin-project",
+ "static_assertions",
+]
+
+[[package]]
+name = "ryu"
+version = "1.0.20"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f"
+
+[[package]]
+name = "scopeguard"
+version = "1.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
+
+[[package]]
+name = "sct"
+version = "0.7.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414"
+dependencies = [
+ "ring 0.17.14",
+ "untrusted 0.9.0",
+]
+
+[[package]]
+name = "sdp"
+version = "0.6.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "13254db766b17451aced321e7397ebf0a446ef0c8d2942b6e67a95815421093f"
+dependencies = [
+ "rand 0.8.5",
+ "substring",
+ "thiserror 1.0.69",
+ "url",
+]
+
+[[package]]
+name = "sec1"
+version = "0.7.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc"
+dependencies = [
+ "base16ct",
+ "der",
+ "generic-array",
+ "pkcs8",
+ "subtle",
+ "zeroize",
+]
+
+[[package]]
+name = "semver"
+version = "1.0.26"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "56e6fa9c48d24d85fb3de5ad847117517440f6beceb7798af16b4a87d616b8d0"
+
+[[package]]
+name = "send_wrapper"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f638d531eccd6e23b980caf34876660d38e265409d8e99b397ab71eb3612fad0"
+
+[[package]]
+name = "send_wrapper"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cd0b0ec5f1c1ca621c432a25813d8d60c88abe6d3e08a3eb9cf37d97a0fe3d73"
+dependencies = [
+ "futures-core",
+]
+
+[[package]]
+name = "serde"
+version = "1.0.219"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5f0e2c6ed6606019b4e29e69dbaba95b11854410e5347d525002456dbbb786b6"
+dependencies = [
+ "serde_derive",
+]
+
+[[package]]
+name = "serde-wasm-bindgen"
+version = "0.6.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8302e169f0eddcc139c70f139d19d6467353af16f9fce27e8c30158036a1e16b"
+dependencies = [
+ "js-sys",
+ "serde",
+ "wasm-bindgen",
+]
+
+[[package]]
+name = "serde_derive"
+version = "1.0.219"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.100",
+]
+
+[[package]]
+name = "serde_json"
+version = "1.0.140"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "20068b6e96dc6c9bd23e01df8827e6c7e1f2fddd43c21810382803c136b99373"
+dependencies = [
+ "itoa",
+ "memchr",
+ "ryu",
+ "serde",
+]
+
+[[package]]
+name = "serde_spanned"
+version = "0.6.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "87607cb1398ed59d48732e575a4c28a7a8ebf2454b964fe3f224f2afc07909e1"
+dependencies = [
+ "serde",
+]
+
+[[package]]
+name = "server"
+version = "0.1.0"
+dependencies = [
+ "chrono",
+ "domain",
+ "futures",
+ "jsonwebtoken",
+ "libp2p",
+ "networking",
+ "quick-protobuf",
+ "rand 0.9.0",
+ "regex",
+ "regexp",
+ "serde",
+ "serde_json",
+ "tokio",
+ "tracing",
+ "tracing-subscriber",
+ "uuid",
+ "zip",
+]
+
+[[package]]
+name = "sha1"
+version = "0.10.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba"
+dependencies = [
+ "cfg-if",
+ "cpufeatures",
+ "digest",
+]
+
+[[package]]
+name = "sha2"
+version = "0.10.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8"
+dependencies = [
+ "cfg-if",
+ "cpufeatures",
+ "digest",
+]
+
+[[package]]
+name = "sharded-slab"
+version = "0.1.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6"
+dependencies = [
+ "lazy_static",
+]
+
+[[package]]
+name = "shlex"
+version = "1.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"
+
+[[package]]
+name = "signal-hook-registry"
+version = "1.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a9e9e0b4211b72e7b8b6e85c807d36c212bdb33ea8587f7569562a84df5465b1"
+dependencies = [
+ "libc",
+]
+
+[[package]]
+name = "signature"
+version = "2.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de"
+dependencies = [
+ "digest",
+ "rand_core 0.6.4",
+]
+
+[[package]]
+name = "simd-adler32"
+version = "0.3.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe"
+
+[[package]]
+name = "simple_asn1"
+version = "0.6.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "297f631f50729c8c99b84667867963997ec0b50f32b2a7dbcab828ef0541e8bb"
+dependencies = [
+ "num-bigint",
+ "num-traits",
+ "thiserror 2.0.12",
+ "time",
+]
+
+[[package]]
+name = "slab"
+version = "0.4.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67"
+dependencies = [
+ "autocfg",
+]
+
+[[package]]
+name = "smallvec"
+version = "1.14.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7fcf8323ef1faaee30a44a340193b1ac6814fd9b7b4e88e9d4519a3e4abe1cfd"
+
+[[package]]
+name = "smol_str"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dd538fb6910ac1099850255cf94a94df6551fbdd602454387d0adb2d1ca6dead"
+dependencies = [
+ "serde",
+]
+
+[[package]]
+name = "snow"
+version = "0.9.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "850948bee068e713b8ab860fe1adc4d109676ab4c3b621fd8147f06b261f2f85"
+dependencies = [
+ "aes-gcm",
+ "blake2",
+ "chacha20poly1305",
+ "curve25519-dalek",
+ "rand_core 0.6.4",
+ "ring 0.17.14",
+ "rustc_version",
+ "sha2",
+ "subtle",
+]
+
+[[package]]
+name = "socket2"
+version = "0.5.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c970269d99b64e60ec3bd6ad27270092a5394c4e309314b18ae3fe575695fbe8"
+dependencies = [
+ "libc",
+ "windows-sys 0.52.0",
+]
+
+[[package]]
+name = "spin"
+version = "0.5.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d"
+
+[[package]]
+name = "spki"
+version = "0.7.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d"
+dependencies = [
+ "base64ct",
+ "der",
+]
+
+[[package]]
+name = "stable_deref_trait"
+version = "1.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3"
+
+[[package]]
+name = "static_assertions"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f"
+
+[[package]]
+name = "strsim"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a"
+
+[[package]]
+name = "strsim"
+version = "0.11.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
+
+[[package]]
+name = "stun"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d3f371788132e9d623e6eab4ba28aac083763a4133f045e6ebaee5ceb869803d"
+dependencies = [
+ "base64 0.21.7",
+ "crc",
+ "lazy_static",
+ "md-5",
+ "rand 0.8.5",
+ "ring 0.17.14",
+ "subtle",
+ "thiserror 1.0.69",
+ "tokio",
+ "url",
+ "webrtc-util 0.8.1",
+]
+
+[[package]]
+name = "stun"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "28fad383a1cc63ae141e84e48eaef44a1063e9d9e55bcb8f51a99b886486e01b"
+dependencies = [
+ "base64 0.21.7",
+ "crc",
+ "lazy_static",
+ "md-5",
+ "rand 0.8.5",
+ "ring 0.17.14",
+ "subtle",
+ "thiserror 1.0.69",
+ "tokio",
+ "url",
+ "webrtc-util 0.9.0",
+]
+
+[[package]]
+name = "substring"
+version = "1.4.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "42ee6433ecef213b2e72f587ef64a2f5943e7cd16fbd82dbe8bc07486c534c86"
+dependencies = [
+ "autocfg",
+]
+
+[[package]]
+name = "subtle"
+version = "2.6.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292"
+
+[[package]]
+name = "syn"
+version = "1.0.109"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-ident",
+]
+
+[[package]]
+name = "syn"
+version = "2.0.100"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b09a44accad81e1ba1cd74a32461ba89dee89095ba17b32f5d03683b1b1fc2a0"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-ident",
+]
+
+[[package]]
+name = "synstructure"
+version = "0.12.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f36bdaa60a83aca3921b5259d5400cbf5e90fc51931376a9bd4a0eb79aa7210f"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 1.0.109",
+ "unicode-xid",
+]
+
+[[package]]
+name = "synstructure"
+version = "0.13.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.100",
+]
+
+[[package]]
+name = "system-configuration"
+version = "0.6.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3c879d448e9d986b661742763247d3693ed13609438cf3d006f51f5368a5ba6b"
+dependencies = [
+ "bitflags 2.9.0",
+ "core-foundation",
+ "system-configuration-sys",
+]
+
+[[package]]
+name = "system-configuration-sys"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4"
+dependencies = [
+ "core-foundation-sys",
+ "libc",
+]
+
+[[package]]
+name = "tempfile"
+version = "3.18.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2c317e0a526ee6120d8dabad239c8dadca62b24b6f168914bbbc8e2fb1f0e567"
+dependencies = [
+ "cfg-if",
+ "fastrand",
+ "getrandom 0.3.1",
+ "once_cell",
+ "rustix 1.0.2",
+ "windows-sys 0.59.0",
+]
+
+[[package]]
+name = "termcolor"
+version = "1.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755"
+dependencies = [
+ "winapi-util",
+]
+
+[[package]]
+name = "textwrap"
+version = "0.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060"
+dependencies = [
+ "unicode-width",
+]
+
+[[package]]
+name = "thiserror"
+version = "1.0.69"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52"
+dependencies = [
+ "thiserror-impl 1.0.69",
+]
+
+[[package]]
+name = "thiserror"
+version = "2.0.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "567b8a2dae586314f7be2a752ec7474332959c6460e02bde30d702a66d488708"
+dependencies = [
+ "thiserror-impl 2.0.12",
+]
+
+[[package]]
+name = "thiserror-impl"
+version = "1.0.69"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.100",
+]
+
+[[package]]
+name = "thiserror-impl"
+version = "2.0.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.100",
+]
+
+[[package]]
+name = "thread_local"
+version = "1.1.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8b9ef9bad013ada3808854ceac7b46812a6465ba368859a37e2100283d2d719c"
+dependencies = [
+ "cfg-if",
+ "once_cell",
+]
+
+[[package]]
+name = "time"
+version = "0.3.39"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dad298b01a40a23aac4580b67e3dbedb7cc8402f3592d7f49469de2ea4aecdd8"
+dependencies = [
+ "deranged",
+ "itoa",
+ "num-conv",
+ "powerfmt",
+ "serde",
+ "time-core",
+ "time-macros",
+]
+
+[[package]]
+name = "time-core"
+version = "0.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "765c97a5b985b7c11d7bc27fa927dc4fe6af3a6dfb021d28deb60d3bf51e76ef"
+
+[[package]]
+name = "time-macros"
+version = "0.2.20"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e8093bc3e81c3bc5f7879de09619d06c9a5a5e45ca44dfeeb7225bae38005c5c"
+dependencies = [
+ "num-conv",
+ "time-core",
+]
+
+[[package]]
+name = "tinystr"
+version = "0.7.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9117f5d4db391c1cf6927e7bea3db74b9a1c1add8f7eda9ffd5364f40f57b82f"
+dependencies = [
+ "displaydoc",
+ "zerovec",
+]
+
+[[package]]
+name = "tinytemplate"
+version = "1.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc"
+dependencies = [
+ "serde",
+ "serde_json",
+]
+
+[[package]]
+name = "tinyvec"
+version = "1.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "09b3661f17e86524eccd4371ab0429194e0d7c008abb45f7a7495b1719463c71"
+dependencies = [
+ "tinyvec_macros",
+]
+
+[[package]]
+name = "tinyvec_macros"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
+
+[[package]]
+name = "tokio"
+version = "1.44.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9975ea0f48b5aa3972bf2d888c238182458437cc2a19374b81b25cdf1023fb3a"
+dependencies = [
+ "backtrace",
+ "bytes",
+ "libc",
+ "mio",
+ "parking_lot",
+ "pin-project-lite",
+ "signal-hook-registry",
+ "socket2",
+ "tokio-macros",
+ "windows-sys 0.52.0",
+]
+
+[[package]]
+name = "tokio-macros"
+version = "2.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.100",
+]
+
+[[package]]
+name = "tokio-util"
+version = "0.7.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d7fcaa8d55a2bdd6b83ace262b016eca0d79ee02818c5c1bcdf0305114081078"
+dependencies = [
+ "bytes",
+ "futures-core",
+ "futures-io",
+ "futures-sink",
+ "pin-project-lite",
+ "tokio",
+]
+
+[[package]]
+name = "toml"
+version = "0.8.20"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cd87a5cdd6ffab733b2f74bc4fd7ee5fff6634124999ac278c35fc78c6120148"
+dependencies = [
+ "serde",
+ "serde_spanned",
+ "toml_datetime",
+ "toml_edit",
+]
+
+[[package]]
+name = "toml_datetime"
+version = "0.6.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41"
+dependencies = [
+ "serde",
+]
+
+[[package]]
+name = "toml_edit"
+version = "0.22.24"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "17b4795ff5edd201c7cd6dca065ae59972ce77d1b80fa0a84d94950ece7d1474"
+dependencies = [
+ "indexmap",
+ "serde",
+ "serde_spanned",
+ "toml_datetime",
+ "winnow",
+]
+
+[[package]]
+name = "tower-service"
+version = "0.3.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3"
+
+[[package]]
+name = "tracing"
+version = "0.1.41"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0"
+dependencies = [
+ "pin-project-lite",
+ "tracing-attributes",
+ "tracing-core",
+]
+
+[[package]]
+name = "tracing-attributes"
+version = "0.1.28"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "395ae124c09f9e6918a2310af6038fba074bcf474ac352496d5910dd59a2226d"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.100",
+]
+
+[[package]]
+name = "tracing-core"
+version = "0.1.33"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e672c95779cf947c5311f83787af4fa8fffd12fb27e4993211a84bdfd9610f9c"
+dependencies = [
+ "once_cell",
+ "valuable",
+]
+
+[[package]]
+name = "tracing-log"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3"
+dependencies = [
+ "log",
+ "once_cell",
+ "tracing-core",
+]
+
+[[package]]
+name = "tracing-subscriber"
+version = "0.3.19"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e8189decb5ac0fa7bc8b96b7cb9b2701d60d48805aca84a238004d665fcc4008"
+dependencies = [
+ "matchers",
+ "nu-ansi-term",
+ "once_cell",
+ "regex",
+ "sharded-slab",
+ "smallvec",
+ "thread_local",
+ "tracing",
+ "tracing-core",
+ "tracing-log",
+]
+
+[[package]]
+name = "tracing-wasm"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4575c663a174420fa2d78f4108ff68f65bf2fbb7dd89f33749b6e826b3626e07"
+dependencies = [
+ "tracing",
+ "tracing-subscriber",
+ "wasm-bindgen",
+]
+
+[[package]]
+name = "try-lock"
+version = "0.2.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b"
+
+[[package]]
+name = "turn"
+version = "0.7.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ffb2ac4f331064513ad510b7a36edc0df555bd61672986607f7c9ff46f98f415"
+dependencies = [
+ "async-trait",
+ "base64 0.21.7",
+ "futures",
+ "log",
+ "md-5",
+ "rand 0.8.5",
+ "ring 0.17.14",
+ "stun 0.5.1",
+ "thiserror 1.0.69",
+ "tokio",
+ "tokio-util",
+ "webrtc-util 0.8.1",
+]
+
+[[package]]
+name = "typenum"
+version = "1.18.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1dccffe3ce07af9386bfd29e80c0ab1a8205a2fc34e4bcd40364df902cfa8f3f"
+
+[[package]]
+name = "uint"
+version = "0.9.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "76f64bba2c53b04fcab63c01a7d7427eadc821e3bc48c34dc9ba29c501164b52"
+dependencies = [
+ "byteorder",
+ "crunchy",
+ "hex",
+ "static_assertions",
+]
+
+[[package]]
+name = "unicode-ident"
+version = "1.0.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512"
+
+[[package]]
+name = "unicode-width"
+version = "0.1.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af"
+
+[[package]]
+name = "unicode-xid"
+version = "0.2.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853"
+
+[[package]]
+name = "universal-hash"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fc1de2c688dc15305988b563c3854064043356019f97a4b46276fe734c4f07ea"
+dependencies = [
+ "crypto-common",
+ "subtle",
+]
+
+[[package]]
+name = "unsigned-varint"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "eb066959b24b5196ae73cb057f45598450d2c5f71460e98c49b738086eff9c06"
+
+[[package]]
+name = "untrusted"
+version = "0.7.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a"
+
+[[package]]
+name = "untrusted"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1"
+
+[[package]]
+name = "url"
+version = "2.5.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "32f8b686cadd1473f4bd0117a5d28d36b1ade384ea9b5069a1c40aefed7fda60"
+dependencies = [
+ "form_urlencoded",
+ "idna",
+ "percent-encoding",
+]
+
+[[package]]
+name = "utf16_iter"
+version = "1.0.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246"
+
+[[package]]
+name = "utf8_iter"
+version = "1.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be"
+
+[[package]]
+name = "utf8parse"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
+
+[[package]]
+name = "uuid"
+version = "1.15.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e0f540e3240398cce6128b64ba83fdbdd86129c16a3aa1a3a252efd66eb3d587"
+dependencies = [
+ "getrandom 0.3.1",
+ "js-sys",
+ "wasm-bindgen",
+]
+
+[[package]]
+name = "valuable"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65"
+
+[[package]]
+name = "vec_map"
+version = "0.8.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191"
+
+[[package]]
+name = "version_check"
+version = "0.9.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
+
+[[package]]
+name = "void"
+version = "1.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d"
+
+[[package]]
+name = "waitgroup"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d1f50000a783467e6c0200f9d10642f4bc424e39efc1b770203e88b488f79292"
+dependencies = [
+ "atomic-waker",
+]
+
+[[package]]
+name = "want"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e"
+dependencies = [
+ "try-lock",
+]
+
+[[package]]
+name = "wasi"
+version = "0.11.0+wasi-snapshot-preview1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
+
+[[package]]
+name = "wasi"
+version = "0.13.3+wasi-0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "26816d2e1a4a36a2940b96c5296ce403917633dff8f3440e9b236ed6f6bacad2"
+dependencies = [
+ "wit-bindgen-rt",
+]
+
+[[package]]
+name = "wasm-bindgen"
+version = "0.2.100"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1edc8929d7499fc4e8f0be2262a241556cfc54a0bea223790e71446f2aab1ef5"
+dependencies = [
+ "cfg-if",
+ "once_cell",
+ "rustversion",
+ "wasm-bindgen-macro",
+]
+
+[[package]]
+name = "wasm-bindgen-backend"
+version = "0.2.100"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2f0a0651a5c2bc21487bde11ee802ccaf4c51935d0d3d42a6101f98161700bc6"
+dependencies = [
+ "bumpalo",
+ "log",
+ "proc-macro2",
+ "quote",
+ "syn 2.0.100",
+ "wasm-bindgen-shared",
+]
+
+[[package]]
+name = "wasm-bindgen-futures"
+version = "0.4.50"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "555d470ec0bc3bb57890405e5d4322cc9ea83cebb085523ced7be4144dac1e61"
+dependencies = [
+ "cfg-if",
+ "js-sys",
+ "once_cell",
+ "wasm-bindgen",
+ "web-sys",
+]
+
+[[package]]
+name = "wasm-bindgen-macro"
+version = "0.2.100"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7fe63fc6d09ed3792bd0897b314f53de8e16568c2b3f7982f468c0bf9bd0b407"
+dependencies = [
+ "quote",
+ "wasm-bindgen-macro-support",
+]
+
+[[package]]
+name = "wasm-bindgen-macro-support"
+version = "0.2.100"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.100",
+ "wasm-bindgen-backend",
+ "wasm-bindgen-shared",
+]
+
+[[package]]
+name = "wasm-bindgen-shared"
+version = "0.2.100"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1a05d73b933a847d6cccdda8f838a22ff101ad9bf93e33684f39c1f5f0eece3d"
+dependencies = [
+ "unicode-ident",
+]
+
+[[package]]
+name = "web-sys"
+version = "0.3.77"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "33b6dd2ef9186f1f2072e409e99cd22a975331a6b3591b12c764e0e55c60d5d2"
+dependencies = [
+ "js-sys",
+ "wasm-bindgen",
+]
+
+[[package]]
+name = "web-time"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb"
+dependencies = [
+ "js-sys",
+ "wasm-bindgen",
+]
+
+[[package]]
+name = "webrtc"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d91e7cf018f7185552bf6a5dd839f4ed9827aea33b746763c9a215f84a0d0b34"
+dependencies = [
+ "arc-swap",
+ "async-trait",
+ "bytes",
+ "cfg-if",
+ "hex",
+ "interceptor",
+ "lazy_static",
+ "log",
+ "rand 0.8.5",
+ "rcgen",
+ "regex",
+ "ring 0.16.20",
+ "rtcp",
+ "rtp 0.9.0",
+ "rustls 0.21.12",
+ "sdp",
+ "serde",
+ "serde_json",
+ "sha2",
+ "smol_str",
+ "stun 0.5.1",
+ "thiserror 1.0.69",
+ "time",
+ "tokio",
+ "turn",
+ "url",
+ "waitgroup",
+ "webrtc-data",
+ "webrtc-dtls",
+ "webrtc-ice",
+ "webrtc-mdns",
+ "webrtc-media",
+ "webrtc-sctp",
+ "webrtc-srtp",
+ "webrtc-util 0.8.1",
+]
+
+[[package]]
+name = "webrtc-data"
+version = "0.8.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e8c08e648e10572b9edbe741074e0f4d3cb221aa7cdf9a814ee71606de312f33"
+dependencies = [
+ "bytes",
+ "log",
+ "thiserror 1.0.69",
+ "tokio",
+ "webrtc-sctp",
+ "webrtc-util 0.8.1",
+]
+
+[[package]]
+name = "webrtc-dtls"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "32b140b953f986e97828aa33ec6318186b05d862bee689efbc57af04a243e832"
+dependencies = [
+ "aes",
+ "aes-gcm",
+ "async-trait",
+ "bincode",
+ "byteorder",
+ "cbc",
+ "ccm",
+ "der-parser 8.2.0",
+ "hkdf",
+ "hmac",
+ "log",
+ "p256",
+ "p384",
+ "rand 0.8.5",
+ "rand_core 0.6.4",
+ "rcgen",
+ "ring 0.16.20",
+ "rustls 0.21.12",
+ "sec1",
+ "serde",
+ "sha1",
+ "sha2",
+ "subtle",
+ "thiserror 1.0.69",
+ "tokio",
+ "webrtc-util 0.8.1",
+ "x25519-dalek",
+ "x509-parser 0.15.1",
+]
+
+[[package]]
+name = "webrtc-ice"
+version = "0.10.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a1bbd6b3dea22cc6e961e22b012e843d8869e2ac8e76b96e54d4a25e311857ad"
+dependencies = [
+ "arc-swap",
+ "async-trait",
+ "crc",
+ "log",
+ "rand 0.8.5",
+ "serde",
+ "serde_json",
+ "stun 0.5.1",
+ "thiserror 1.0.69",
+ "tokio",
+ "turn",
+ "url",
+ "uuid",
+ "waitgroup",
+ "webrtc-mdns",
+ "webrtc-util 0.8.1",
+]
+
+[[package]]
+name = "webrtc-mdns"
+version = "0.6.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ce981f93104a8debb3563bb0cedfe4aa2f351fdf6b53f346ab50009424125c08"
+dependencies = [
+ "log",
+ "socket2",
+ "thiserror 1.0.69",
+ "tokio",
+ "webrtc-util 0.8.1",
+]
+
+[[package]]
+name = "webrtc-media"
+version = "0.7.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "280017b6b9625ef7329146332518b339c3cceff231cc6f6a9e0e6acab25ca4af"
+dependencies = [
+ "byteorder",
+ "bytes",
+ "rand 0.8.5",
+ "rtp 0.10.0",
+ "thiserror 1.0.69",
+]
+
+[[package]]
+name = "webrtc-sctp"
+version = "0.9.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "df75ec042002fe995194712cbeb2029107a60a7eab646f1b789eb1be94d0e367"
+dependencies = [
+ "arc-swap",
+ "async-trait",
+ "bytes",
+ "crc",
+ "log",
+ "rand 0.8.5",
+ "thiserror 1.0.69",
+ "tokio",
+ "webrtc-util 0.8.1",
+]
+
+[[package]]
+name = "webrtc-srtp"
+version = "0.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c1db1f36c1c81e4b1e531c0b9678ba0c93809e196ce62122d87259bb71c03b9f"
+dependencies = [
+ "aead",
+ "aes",
+ "aes-gcm",
+ "byteorder",
+ "bytes",
+ "ctr",
+ "hmac",
+ "log",
+ "rtcp",
+ "rtp 0.9.0",
+ "sha1",
+ "subtle",
+ "thiserror 1.0.69",
+ "tokio",
+ "webrtc-util 0.8.1",
+]
+
+[[package]]
+name = "webrtc-util"
+version = "0.8.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5e85154ef743d9a2a116d104faaaa82740a281b8b4bed5ee691a2df6c133d873"
+dependencies = [
+ "async-trait",
+ "bitflags 1.3.2",
+ "bytes",
+ "ipnet",
+ "lazy_static",
+ "libc",
+ "log",
+ "nix",
+ "rand 0.8.5",
+ "thiserror 1.0.69",
+ "tokio",
+ "winapi",
+]
+
+[[package]]
+name = "webrtc-util"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dc8d9bc631768958ed97b8d68b5d301e63054ae90b09083d43e2fefb939fd77e"
+dependencies = [
+ "async-trait",
+ "bitflags 1.3.2",
+ "bytes",
+ "ipnet",
+ "lazy_static",
+ "libc",
+ "log",
+ "nix",
+ "portable-atomic",
+ "rand 0.8.5",
+ "thiserror 1.0.69",
+ "tokio",
+ "winapi",
+]
+
+[[package]]
+name = "widestring"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7219d36b6eac893fa81e84ebe06485e7dcbb616177469b142df14f1f4deb1311"
+
+[[package]]
+name = "winapi"
+version = "0.3.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
+dependencies = [
+ "winapi-i686-pc-windows-gnu",
+ "winapi-x86_64-pc-windows-gnu",
+]
+
+[[package]]
+name = "winapi-i686-pc-windows-gnu"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
+
+[[package]]
+name = "winapi-util"
+version = "0.1.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb"
+dependencies = [
+ "windows-sys 0.59.0",
+]
+
+[[package]]
+name = "winapi-x86_64-pc-windows-gnu"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
+
+[[package]]
+name = "windows"
+version = "0.53.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "efc5cf48f83140dcaab716eeaea345f9e93d0018fb81162753a3f76c3397b538"
+dependencies = [
+ "windows-core 0.53.0",
+ "windows-targets 0.52.6",
+]
+
+[[package]]
+name = "windows-core"
+version = "0.52.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9"
+dependencies = [
+ "windows-targets 0.52.6",
+]
+
+[[package]]
+name = "windows-core"
+version = "0.53.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9dcc5b895a6377f1ab9fa55acedab1fd5ac0db66ad1e6c7f47e28a22e446a5dd"
+dependencies = [
+ "windows-result",
+ "windows-targets 0.52.6",
+]
+
+[[package]]
+name = "windows-link"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6dccfd733ce2b1753b03b6d3c65edf020262ea35e20ccdf3e288043e6dd620e3"
+
+[[package]]
+name = "windows-result"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5e383302e8ec8515204254685643de10811af0ed97ea37210dc26fb0032647f8"
+dependencies = [
+ "windows-targets 0.52.6",
+]
+
+[[package]]
+name = "windows-sys"
+version = "0.48.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9"
+dependencies = [
+ "windows-targets 0.48.5",
+]
+
+[[package]]
+name = "windows-sys"
+version = "0.52.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d"
+dependencies = [
+ "windows-targets 0.52.6",
+]
+
+[[package]]
+name = "windows-sys"
+version = "0.59.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b"
+dependencies = [
+ "windows-targets 0.52.6",
+]
+
+[[package]]
+name = "windows-targets"
+version = "0.48.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c"
+dependencies = [
+ "windows_aarch64_gnullvm 0.48.5",
+ "windows_aarch64_msvc 0.48.5",
+ "windows_i686_gnu 0.48.5",
+ "windows_i686_msvc 0.48.5",
+ "windows_x86_64_gnu 0.48.5",
+ "windows_x86_64_gnullvm 0.48.5",
+ "windows_x86_64_msvc 0.48.5",
+]
+
+[[package]]
+name = "windows-targets"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973"
+dependencies = [
+ "windows_aarch64_gnullvm 0.52.6",
+ "windows_aarch64_msvc 0.52.6",
+ "windows_i686_gnu 0.52.6",
+ "windows_i686_gnullvm",
+ "windows_i686_msvc 0.52.6",
+ "windows_x86_64_gnu 0.52.6",
+ "windows_x86_64_gnullvm 0.52.6",
+ "windows_x86_64_msvc 0.52.6",
+]
+
+[[package]]
+name = "windows_aarch64_gnullvm"
+version = "0.48.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8"
+
+[[package]]
+name = "windows_aarch64_gnullvm"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3"
+
+[[package]]
+name = "windows_aarch64_msvc"
+version = "0.48.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc"
+
+[[package]]
+name = "windows_aarch64_msvc"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469"
+
+[[package]]
+name = "windows_i686_gnu"
+version = "0.48.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e"
+
+[[package]]
+name = "windows_i686_gnu"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b"
+
+[[package]]
+name = "windows_i686_gnullvm"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66"
+
+[[package]]
+name = "windows_i686_msvc"
+version = "0.48.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406"
+
+[[package]]
+name = "windows_i686_msvc"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66"
+
+[[package]]
+name = "windows_x86_64_gnu"
+version = "0.48.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e"
+
+[[package]]
+name = "windows_x86_64_gnu"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78"
+
+[[package]]
+name = "windows_x86_64_gnullvm"
+version = "0.48.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc"
+
+[[package]]
+name = "windows_x86_64_gnullvm"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d"
+
+[[package]]
+name = "windows_x86_64_msvc"
+version = "0.48.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538"
+
+[[package]]
+name = "windows_x86_64_msvc"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
+
+[[package]]
+name = "winnow"
+version = "0.7.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0e7f4ea97f6f78012141bcdb6a216b2609f0979ada50b20ca5b52dde2eac2bb1"
+dependencies = [
+ "memchr",
+]
+
+[[package]]
+name = "winreg"
+version = "0.50.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1"
+dependencies = [
+ "cfg-if",
+ "windows-sys 0.48.0",
+]
+
+[[package]]
+name = "wit-bindgen-rt"
+version = "0.33.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3268f3d866458b787f390cf61f4bbb563b922d091359f9608842999eaee3943c"
+dependencies = [
+ "bitflags 2.9.0",
+]
+
+[[package]]
+name = "write16"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d1890f4022759daae28ed4fe62859b1236caebfc61ede2f63ed4e695f3f6d936"
+
+[[package]]
+name = "writeable"
+version = "0.5.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51"
+
+[[package]]
+name = "x25519-dalek"
+version = "2.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c7e468321c81fb07fa7f4c636c3972b9100f0346e5b6a9f2bd0603a52f7ed277"
+dependencies = [
+ "curve25519-dalek",
+ "rand_core 0.6.4",
+ "serde",
+ "zeroize",
+]
+
+[[package]]
+name = "x509-parser"
+version = "0.15.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7069fba5b66b9193bd2c5d3d4ff12b839118f6bcbef5328efafafb5395cf63da"
+dependencies = [
+ "asn1-rs 0.5.2",
+ "data-encoding",
+ "der-parser 8.2.0",
+ "lazy_static",
+ "nom",
+ "oid-registry 0.6.1",
+ "ring 0.16.20",
+ "rusticata-macros",
+ "thiserror 1.0.69",
+ "time",
+]
+
+[[package]]
+name = "x509-parser"
+version = "0.16.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fcbc162f30700d6f3f82a24bf7cc62ffe7caea42c0b2cba8bf7f3ae50cf51f69"
+dependencies = [
+ "asn1-rs 0.6.2",
+ "data-encoding",
+ "der-parser 9.0.0",
+ "lazy_static",
+ "nom",
+ "oid-registry 0.7.1",
+ "rusticata-macros",
+ "thiserror 1.0.69",
+ "time",
+]
+
+[[package]]
+name = "xml-rs"
+version = "0.8.25"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c5b940ebc25896e71dd073bad2dbaa2abfe97b0a391415e22ad1326d9c54e3c4"
+
+[[package]]
+name = "xmltree"
+version = "0.10.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d7d8a75eaf6557bb84a65ace8609883db44a29951042ada9b393151532e41fcb"
+dependencies = [
+ "xml-rs",
+]
+
+[[package]]
+name = "yamux"
+version = "0.12.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9ed0164ae619f2dc144909a9f082187ebb5893693d8c0196e8085283ccd4b776"
+dependencies = [
+ "futures",
+ "log",
+ "nohash-hasher",
+ "parking_lot",
+ "pin-project",
+ "rand 0.8.5",
+ "static_assertions",
+]
+
+[[package]]
+name = "yamux"
+version = "0.13.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "17610762a1207ee816c6fadc29220904753648aba0a9ed61c7b8336e80a559c4"
+dependencies = [
+ "futures",
+ "log",
+ "nohash-hasher",
+ "parking_lot",
+ "pin-project",
+ "rand 0.8.5",
+ "static_assertions",
+ "web-time",
+]
+
+[[package]]
+name = "yasna"
+version = "0.5.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e17bb3549cc1321ae1296b9cdc2698e2b6cb1992adfa19a8c72e5b7a738f44cd"
+dependencies = [
+ "time",
+]
+
+[[package]]
+name = "yoke"
+version = "0.7.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "120e6aef9aa629e3d4f52dc8cc43a015c7724194c97dfaf45180d2daf2b77f40"
+dependencies = [
+ "serde",
+ "stable_deref_trait",
+ "yoke-derive",
+ "zerofrom",
+]
+
+[[package]]
+name = "yoke-derive"
+version = "0.7.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.100",
+ "synstructure 0.13.1",
+]
+
+[[package]]
+name = "zerocopy"
+version = "0.8.23"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fd97444d05a4328b90e75e503a34bad781f14e28a823ad3557f0750df1ebcbc6"
+dependencies = [
+ "zerocopy-derive",
+]
+
+[[package]]
+name = "zerocopy-derive"
+version = "0.8.23"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6352c01d0edd5db859a63e2605f4ea3183ddbd15e2c4a9e7d32184df75e4f154"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.100",
+]
+
+[[package]]
+name = "zerofrom"
+version = "0.1.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "50cc42e0333e05660c3587f3bf9d0478688e15d870fab3346451ce7f8c9fbea5"
+dependencies = [
+ "zerofrom-derive",
+]
+
+[[package]]
+name = "zerofrom-derive"
+version = "0.1.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.100",
+ "synstructure 0.13.1",
+]
+
+[[package]]
+name = "zeroize"
+version = "1.8.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde"
+dependencies = [
+ "zeroize_derive",
+]
+
+[[package]]
+name = "zeroize_derive"
+version = "1.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.100",
+]
+
+[[package]]
+name = "zerovec"
+version = "0.10.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "aa2b893d79df23bfb12d5461018d408ea19dfafe76c2c7ef6d4eba614f8ff079"
+dependencies = [
+ "yoke",
+ "zerofrom",
+ "zerovec-derive",
+]
+
+[[package]]
+name = "zerovec-derive"
+version = "0.10.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.100",
+]
+
+[[package]]
+name = "zip"
+version = "2.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b280484c454e74e5fff658bbf7df8fdbe7a07c6b2de4a53def232c15ef138f3a"
+dependencies = [
+ "aes",
+ "arbitrary",
+ "bzip2",
+ "constant_time_eq",
+ "crc32fast",
+ "crossbeam-utils",
+ "deflate64",
+ "displaydoc",
+ "flate2",
+ "hmac",
+ "indexmap",
+ "lzma-rs",
+ "memchr",
+ "pbkdf2",
+ "rand 0.8.5",
+ "sha1",
+ "thiserror 2.0.12",
+ "time",
+ "zeroize",
+ "zopfli",
+ "zstd",
+]
+
+[[package]]
+name = "zopfli"
+version = "0.8.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e5019f391bac5cf252e93bbcc53d039ffd62c7bfb7c150414d61369afe57e946"
+dependencies = [
+ "bumpalo",
+ "crc32fast",
+ "lockfree-object-pool",
+ "log",
+ "once_cell",
+ "simd-adler32",
+]
+
+[[package]]
+name = "zstd"
+version = "0.13.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a"
+dependencies = [
+ "zstd-safe",
+]
+
+[[package]]
+name = "zstd-safe"
+version = "7.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f3051792fbdc2e1e143244dc28c60f73d8470e93f3f9cbd0ead44da5ed802722"
+dependencies = [
+ "zstd-sys",
+]
+
+[[package]]
+name = "zstd-sys"
+version = "2.0.14+zstd.1.5.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8fb060d4926e4ac3a3ad15d864e99ceb5f343c6b34f5bd6d81ae6ed417311be5"
+dependencies = [
+ "cc",
+ "pkg-config",
+]
diff --git a/server/Cargo.toml b/server/Cargo.toml
new file mode 100644
index 0000000..a254e14
--- /dev/null
+++ b/server/Cargo.toml
@@ -0,0 +1,23 @@
+[package]
+name = "server"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]
+libp2p = { git = "https://github.com/aukilabs/rust-libp2p.git", branch = "fix/gossipsub-wasm" }
+networking = { git = "https://github.com/aukilabs/posemesh.git", branch = "chore/reconstruction-integration" }
+futures = "0.3.30"
+jsonwebtoken = "9.3.0"
+uuid = "1.13.2"
+rand = "0.9.0"
+domain = { git = "https://github.com/aukilabs/posemesh.git", branch = "chore/reconstruction-integration" }
+tracing = "0.1.40"
+tracing-subscriber = "0.3.18"
+tokio = "1.44.0"
+quick-protobuf = "0.8.1"
+serde = "1.0.210"
+serde_json = "1.0.128"
+zip = "2.2.3"
+regex = "1.10.3"
+regexp = "0.3.2"
+chrono = "0.4"
diff --git a/server/go.mod b/server/go.mod
deleted file mode 100644
index 6899a38..0000000
--- a/server/go.mod
+++ /dev/null
@@ -1,27 +0,0 @@
-module github.com/aukilabs/reconstruction
-
-go 1.21.5
-
-require (
-	github.com/go-chi/chi v1.5.5
-	github.com/google/uuid v1.6.0
-	github.com/lestrrat-go/jwx v1.2.29
-)
-
-require (
-	go.opentelemetry.io/otel v1.28.0 // indirect
-	go.opentelemetry.io/otel/trace v1.28.0 // indirect
-)
-
-require (
-	github.com/aukilabs/go-tooling v0.15.8
-	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect
-	github.com/goccy/go-json v0.10.2 // indirect
-	github.com/lestrrat-go/backoff/v2 v2.0.8 // indirect
-	github.com/lestrrat-go/blackmagic v1.0.2 // indirect
-	github.com/lestrrat-go/httpcc v1.0.1 // indirect
-	github.com/lestrrat-go/iter v1.0.2 // indirect
-	github.com/lestrrat-go/option v1.0.1 // indirect
-	github.com/pkg/errors v0.9.1 // indirect
-	golang.org/x/crypto v0.31.0 // indirect
-)
diff --git a/server/go.sum b/server/go.sum
deleted file mode 100644
index 024000b..0000000
--- a/server/go.sum
+++ /dev/null
@@ -1,95 +0,0 @@
-github.com/aukilabs/go-tooling v0.15.8 h1:BLrdW3KGrkLQAkxjuZbqPzC1p+LJfbOGLlXsC52FCyo=
-github.com/aukilabs/go-tooling v0.15.8/go.mod h1:Cs1/wCUK1l1kBtvIY7fASXGGkyF6DX0IKIdh8TjcQFY=
-github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
-github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/decred/dcrd/crypto/blake256 v1.0.1/go.mod h1:2OfgNZ5wDpcsFmHmCK5gZTPcCXqlm2ArzUIkw9czNJo=
-github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 h1:8UrgZ3GkP4i/CLijOJx79Yu+etlyjdBU4sfcs2WYQMs=
-github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0/go.mod h1:v57UDF4pDQJcEfFUCRop3lJL149eHGSe9Jvczhzjo/0=
-github.com/go-chi/chi v1.5.5 h1:vOB/HbEMt9QqBqErz07QehcOKHaWFtuj87tTDVz2qXE=
-github.com/go-chi/chi v1.5.5/go.mod h1:C9JqLr3tIYjDOZpzn+BCuxY8z8vmca43EeMgyZt7irw=
-github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=
-github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
-github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
-github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
-github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
-github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/lestrrat-go/backoff/v2 v2.0.8 h1:oNb5E5isby2kiro9AgdHLv5N5tint1AnDVVf2E2un5A=
-github.com/lestrrat-go/backoff/v2 v2.0.8/go.mod h1:rHP/q/r9aT27n24JQLa7JhSQZCKBBOiM/uP402WwN8Y=
-github.com/lestrrat-go/blackmagic v1.0.2 h1:Cg2gVSc9h7sz9NOByczrbUvLopQmXrfFx//N+AkAr5k=
-github.com/lestrrat-go/blackmagic v1.0.2/go.mod h1:UrEqBzIR2U6CnzVyUtfM6oZNMt/7O7Vohk2J0OGSAtU=
-github.com/lestrrat-go/httpcc v1.0.1 h1:ydWCStUeJLkpYyjLDHihupbn2tYmZ7m22BGkcvZZrIE=
-github.com/lestrrat-go/httpcc v1.0.1/go.mod h1:qiltp3Mt56+55GPVCbTdM9MlqhvzyuL6W/NMDA8vA5E=
-github.com/lestrrat-go/iter v1.0.2 h1:gMXo1q4c2pHmC3dn8LzRhJfP1ceCbgSiT9lUydIzltI=
-github.com/lestrrat-go/iter v1.0.2/go.mod h1:Momfcq3AnRlRjI5b5O8/G5/BvpzrhoFTZcn06fEOPt4=
-github.com/lestrrat-go/jwx v1.2.29 h1:QT0utmUJ4/12rmsVQrJ3u55bycPkKqGYuGT4tyRhxSQ=
-github.com/lestrrat-go/jwx v1.2.29/go.mod h1:hU8k2l6WF0ncx20uQdOmik/Gjg6E3/wIRtXSNFeZuB8=
-github.com/lestrrat-go/option v1.0.0/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I=
-github.com/lestrrat-go/option v1.0.1 h1:oAzP2fvZGQKWkvHa1/SAcFolBEca1oN+mQ7eooNBEYU=
-github.com/lestrrat-go/option v1.0.1/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I=
-github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
-github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
-github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
-github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
-github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
-github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
-github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
-github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
-github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
-github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
-github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
-go.opentelemetry.io/otel v1.28.0 h1:/SqNcYk+idO0CxKEUOtKQClMK/MimZihKYMruSMViUo=
-go.opentelemetry.io/otel v1.28.0/go.mod h1:q68ijF8Fc8CnMHKyzqL6akLO46ePnjkgfIMIjUIX9z4=
-go.opentelemetry.io/otel/trace v1.28.0 h1:GhQ9cUuQGmNDd5BTCP2dAvv75RdMxEfTmYejp+lkx9g=
-go.opentelemetry.io/otel/trace v1.28.0/go.mod h1:jPyXzNPg6da9+38HEwElrQiHlVMTnVfM3/yv2OlIHaI=
-golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
-golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
-golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs=
-golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U=
-golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
-golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
-golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
-golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
-golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
-golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
-golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
-golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
-golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
-golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
-golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
-golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
-golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
-golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58=
-golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
-golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
-golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
-golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
-golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
-golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
-golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
-gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
diff --git a/server/helper.go b/server/helper.go
deleted file mode 100644
index a9b572e..0000000
--- a/server/helper.go
+++ /dev/null
@@ -1,960 +0,0 @@
-package main
-
-import (
-	"bytes"
-	"context"
-	"encoding/json"
-	"slices"
-	"sort"
-
-	"fmt"
-	"io"
-
-	"mime"
-	"mime/multipart"
-	"net/http"
-	"net/textproto"
-	"os"
-	"os/exec"
-	"path"
-	"regexp"
-	"strconv"
-	"strings"
-	"sync"
-	"time"
-
-	"github.com/aukilabs/go-tooling/pkg/errors"
-	"github.com/aukilabs/go-tooling/pkg/logs"
-	"github.com/google/uuid"
-	"github.com/lestrrat-go/jwx/jwt"
-)
-
-type ExpectedOutput struct {
-	FilePath string // relative to job folder
-	Name     string
-	DataType string
-}
-
-type jobList struct {
-	lock sync.RWMutex
-	list map[string]job
-}
-
-func (js *jobList) AddJob(j *job) {
-	js.lock.Lock()
-	defer js.lock.Unlock()
-
-	js.list[j.ID] = *j
-}
-
-func ParseStatusFromManifest(manifestPath string) (string, error) {
-	content, err := os.ReadFile(manifestPath)
-	if err != nil {
-		return "", err
-	}
-
-	var parsedManifest map[string]interface{}
-	if err := json.Unmarshal(content, &parsedManifest); err != nil {
-		return "", err
-	}
-
-	status, ok := parsedManifest["jobStatus"].(string)
-	if !ok {
-		return "", fmt.Errorf("cannot parse jobStatus in existing manifest json file: %s", manifestPath)
-	}
-	return status, nil
-}
-
-func WriteJobManifestFile(j *job, status string) {
-	if status == "failed" {
-		// If python script has already written a manifest with status "failed", don't overwrite it
-		statusFromManifest, err := ParseStatusFromManifest(path.Join(j.JobPath, "job_manifest.json"))
-
-		if err == nil && statusFromManifest == "failed" {
-			logs.WithTag("job_id", j.ID).
-				WithTag("domain_id", j.DomainID).
-				Infof("job %s python script has already written a failed manifest, won't overwrite.", j.ID)
-		} else {
-			WriteFailedJobManifestFile(j, "Reconstruction job script failed")
-		}
-
-	} else if status == "processing" {
-		progress := 0
-		WriteJobManifestFileHelper(j, status, progress, "Request received by reconstruction server")
-	} else {
-		return
-	}
-
-	err := UploadJobManifestToDomain(j)
-	if err != nil {
-		logs.WithTag("job_id", j.ID).
-			WithTag("domain_id", j.DomainID).
-			Error(errors.New("failed to upload job manifest to domain").Wrap(err))
-	}
-}
-
-func (js *jobList) UpdateJob(id string, status string) {
-	js.lock.Lock()
-	j, ok := js.list[id]
-	if ok {
-		j.Status = status
-		js.list[id] = j
-	}
-	js.lock.Unlock()
-
-	if ok {
-		WriteJobManifestFile(&j, status)
-	}
-}
-
-func (js *jobList) List() []job {
-	js.lock.RLock()
-	defer js.lock.RUnlock()
-
-	logs.Info("job list count: ", len(js.list))
-	logs.Info("job list null? ", js.list == nil)
-	// log.Println("job list count: ", len(js.list))
-	// log.Println("job list null? ", js.list == nil)
-
-	var jobs []job
-	for _, j := range js.list {
-		jobs = append(jobs, j)
-		logs.Info("APPEND! new job list count: ", len(jobs))
-		// log.Println("APPEND! new job list count: ", len(jobs))
-	}
-	logs.Info("job list null? ", js.list == nil)
-	// log.Println("job list null? ", js.list == nil)
-	return jobs
-}
-
-var jobs = jobList{
-	lock: sync.RWMutex{},
-	list: map[string]job{},
-}
-
-type JobMetadata struct {
-	ID                      string    `json:"id"`
-	Name                    string    `json:"name"`
-	DomainID                string    `json:"domain_id"`
-	ProcessingType          string    `json:"processing_type"`
-	CreatedAt               time.Time `json:"created_at"`
-	DomainServerURL         string    `json:"domain_server_url"`
-	ReconstructionServerURL string    `json:"reconstruction_server_url"`
-	AccessToken             string    `json:"-"`
-	DataIDs                 []string  `json:"data_ids"`
-}
-
-type job struct {
-	JobMetadata
-	JobPath         string            `json:"job_path"`
-	Status          string            `json:"status"`
-	UploadedDataIDs map[string]string `json:"-"`
-}
-
-type JobRequestData struct {
-	DataIDs         []string `json:"data_ids"`
-	DomainID        string   `json:"domain_id"`
-	AccessToken     string   `json:"access_token"`
-	ProcessingType  string   `json:"processing_type"`
-	DomainServerURL string   `json:"domain_server_url"` // Optional. Default: "issuer" of the incoming request, since jobs are triggered via the domain server.
-}
-
-type DomainDataMetadata struct {
-	ID       string `json:"id"`
-	DomainID string `json:"domain_id"`
-	EditableDomainDataMetadata
-}
-
-type EditableDomainDataMetadata struct {
-	Name     string `json:"name"`
-	DataType string `json:"data_type"`
-}
-
-type DomainData struct {
-	DomainDataMetadata ``
-	Data               io.ReadCloser `json:"-"`
-}
-
-type PostDomainDataResponse struct {
-	Data []DomainDataMetadata `json:"data"`
-}
-
-var quoteEscaper = strings.NewReplacer("\\", "\\\\", `"`, "\\\"")
-
-var MaxBytesError = &http.MaxBytesError{}
-
-func escapeQuotes(s string) string {
-	return quoteEscaper.Replace(s)
-}
-
-func WriteScanDataSummary(datasetsRootPath string, allScanFolders []os.DirEntry, summaryJsonPath string) error {
-	scanCount := 0
-	totalFrameCount := 0
-	totalScanDuration := 0.0
-	scanDurations := []float64{}
-	uniquePortalIDs := []string{}
-	portalSizes := []float64{} // Size list is used when saving manifest, to output same physical size, without asking domain server
-	devicesUsed := []string{}
-	appVersionsUsed := []string{}
-
-	for _, scanFolder := range allScanFolders {
-		if !scanFolder.IsDir() {
-			continue
-		}
-
-		manifestPath := path.Join(datasetsRootPath, scanFolder.Name(), "Manifest.json")
-		if _, err := os.Stat(manifestPath); os.IsNotExist(err) {
-			continue
-		}
-
-		manifestData, err := os.ReadFile(manifestPath)
-		if err != nil {
-			return err
-		}
-
-		var manifest map[string]interface{}
-		if err := json.Unmarshal(manifestData, &manifest); err != nil {
-			return err
-		}
-
-		scanCount++
-
-		frameCount := int(manifest["frameCount"].(float64))
-		duration := manifest["duration"].(float64)
-		totalFrameCount += frameCount
-		totalScanDuration += duration
-		scanDurations = append(scanDurations, duration)
-
-		if portals, ok := manifest["portals"].([]interface{}); ok {
-			for _, portal := range portals {
-				if portalMap, ok := portal.(map[string]interface{}); ok {
-					if portalID, ok := portalMap["shortId"].(string); ok {
-						if !slices.Contains(uniquePortalIDs, portalID) {
-							uniquePortalIDs = append(uniquePortalIDs, portalID)
-							portalSizes = append(portalSizes, portalMap["physicalSize"].(float64))
-						}
-					}
-				}
-			}
-		}
-
-		device := "unknown"
-		if manifest["brand"] != nil && manifest["model"] != nil && manifest["systemName"] != nil && manifest["systemVersion"] != nil {
-			device = manifest["brand"].(string) + " " + manifest["model"].(string) + " " + manifest["systemName"].(string) + " " + manifest["systemVersion"].(string)
-			device = strings.TrimSpace(device)
-		}
-		if !slices.Contains(devicesUsed, device) {
-			devicesUsed = append(devicesUsed, device)
-		}
-
-		appVersion := "unknown"
-		if manifest["appVersion"] != nil && manifest["buildId"] != nil {
-			appVersion = manifest["appVersion"].(string) + " (build " + manifest["buildId"].(string) + ")"
-		}
-		if !slices.Contains(appVersionsUsed, appVersion) {
-			appVersionsUsed = append(appVersionsUsed, appVersion)
-		}
-	}
-
-	sort.Float64s(scanDurations)
-	shortestScanDuration := scanDurations[0]
-	longestScanDuration := scanDurations[len(scanDurations)-1]
-	medianScanDuration := scanDurations[len(scanDurations)/2]
-
-	averageScanDuration := totalScanDuration / float64(len(allScanFolders))
-	averageScanFrameCount := float64(totalFrameCount) / float64(len(allScanFolders))
-	averageScanFrameRate := float64(totalFrameCount) / totalScanDuration
-
-	summary := map[string]interface{}{
-		"scanCount":             scanCount,
-		"totalFrameCount":       totalFrameCount,
-		"totalScanDuration":     totalScanDuration,
-		"averageScanDuration":   averageScanDuration,
-		"averageScanFrameCount": averageScanFrameCount,
-		"averageFrameRate":      averageScanFrameRate,
-		"shortestScanDuration":  shortestScanDuration,
-		"longestScanDuration":   longestScanDuration,
-		"medianScanDuration":    medianScanDuration,
-		"portalCount":           len(uniquePortalIDs),
-		"portalIDs":             uniquePortalIDs,
-		"portalSizes":           portalSizes,
-		"deviceVersionsUsed":    devicesUsed,
-		"appVersionsUsed":       appVersionsUsed,
-	}
-
-	summaryJson, err := json.MarshalIndent(summary, "", "  ")
-	if err != nil {
-		return err
-	}
-
-	return os.WriteFile(summaryJsonPath, summaryJson, 0644)
-}
-
-func WriteDomainData(mw *multipart.Writer, data *DomainData) error {
-	h := make(textproto.MIMEHeader)
-	h.Set("Content-Type", "application/octet-stream")
-	h.Set("Content-Disposition",
-		fmt.Sprintf(
-			`form-data; name="%s"; data-type="%s"; id="%s"; domain-id="%s"`,
-			escapeQuotes(data.Name),
-			escapeQuotes(data.DataType),
-			escapeQuotes(data.ID),
-			escapeQuotes(data.DomainID),
-		),
-	)
-	fw, err := mw.CreatePart(h)
-	if err != nil {
-		return err
-	}
-	_, err = io.Copy(fw, data.Data)
-	return err
-}
-
-func WriteFailedJobManifestFile(j *job, errorMessage string) error {
-	pythonSnippet := `
-from utils.data_utils import save_failed_manifest_json; 
-save_failed_manifest_json('` + j.JobPath + `/job_manifest.json', '` + j.JobPath + `', '` + errorMessage + `')
-`
-	logs.WithTag("job_id", j.ID).
-		WithTag("domain_id", j.DomainID).
-		Infof("writing failed manifest, error message: %s", errorMessage)
-
-	cmd := exec.Command("python3", "-c", pythonSnippet)
-	cmd.Stdout = os.Stdout
-	cmd.Stderr = os.Stderr
-
-	return cmd.Run()
-}
-
-func WriteJobManifestFileHelper(j *job, status string, progress int, statusDetails string) error {
-	pythonSnippet := `
-from utils.data_utils import save_manifest_json;
-save_manifest_json({},
-	'` + j.JobPath + `/job_manifest.json',
-	'` + j.JobPath + `',
-	job_status='` + status + `',
-	job_progress=` + strconv.Itoa(progress) + `,
-	job_status_details='` + statusDetails + `'
-)`
-
-	logs.WithTag("job_id", j.ID).
-		WithTag("domain_id", j.DomainID).
-		Info("Writing manifest with status: ", status, ", progress: ", progress, ", status details: ", statusDetails)
-
-	cmd := exec.Command("python3", "-c", pythonSnippet)
-	cmd.Stdout = os.Stdout
-	cmd.Stderr = os.Stderr
-
-	return cmd.Run()
-}
-
-func UploadJobManifestToDomain(j *job) error {
-	logs.WithTag("job_id", j.ID).
-		WithTag("domain_id", j.DomainID).
-		Infof("Upload job manifest to domain, for job")
-
-	output := ExpectedOutput{
-		FilePath: "job_manifest.json",
-		Name:     "refined_manifest",
-		DataType: "refined_manifest_json",
-	}
-
-	return UploadOutputToDomain(j, output)
-}
-
-func UploadRefinedOutputsToDomain(j *job) (int, error) {
-	refinedOutput := path.Join("refined", "global")
-	expectedOutputs := []ExpectedOutput{
-		{
-			FilePath: path.Join(refinedOutput, "refined_manifest.json"),
-			Name:     "refined_manifest",
-			DataType: "refined_manifest_json",
-		},
-		{
-			FilePath: path.Join(refinedOutput, "RefinedPointCloud.ply"),
-			Name:     "refined_pointcloud",
-			DataType: "refined_pointcloud_ply",
-		},
-		{
-			// The unrefined point cloud after just basic stitch from overlap QR codes
-			// Not really useful to apps, but for debugging the refinement
-			FilePath: path.Join(refinedOutput, "BasicStitchPointCloud.ply"),
-			Name:     "unrefined_pointcloud",
-			DataType: "unrefined_pointcloud_ply",
-		},
-		/*{
-			FilePath: path.Join(refinedOutput, "occlusion", "meshes.obj"),
-			Name:     "occlusionmesh_v1",
-			DataType: "obj",
-		},*/
-	}
-
-	outputCount := 0
-	for _, output := range expectedOutputs {
-		if _, err := os.Stat(path.Join(j.JobPath, output.FilePath)); !os.IsNotExist(err) {
-			outputCount++
-		}
-	}
-
-	// Upload manifest using PUT since it already exists from start of the job
-	if err := UploadOutputToDomain(j, expectedOutputs[0]); err != nil {
-		return outputCount, errors.New("failed to upload refined manifest to domain").Wrap(err)
-	}
-
-	if err := UploadOutputsToDomain(j, expectedOutputs[1:]); err != nil {
-		return outputCount, errors.New("failed to upload refined outputs to domain").Wrap(err)
-	}
-
-	return outputCount, nil
-}
-
-func UploadOutputsToDomain(j *job, expectedOutputs []ExpectedOutput) error {
-	firstErr := error(nil)
-	for _, output := range expectedOutputs {
-		if err := UploadOutputToDomain(j, output); err != nil {
-			if firstErr == nil {
-				firstErr = err
-			}
-		}
-	}
-	return firstErr
-}
-
-func UploadOutputToDomain(j *job, output ExpectedOutput) error {
-	outputPath := j.JobPath
-	if _, err := os.Stat(outputPath); os.IsNotExist(err) {
-		return err
-	}
-
-	f, err := os.Open(path.Join(outputPath, output.FilePath))
-	if err != nil {
-		return fmt.Errorf("failed to open output file %s: %s", output.FilePath, err.Error())
-	}
-	defer f.Close()
-
-	nameSuffix := j.CreatedAt.Format("2006-01-02_15-04-05")
-	domainData := DomainData{
-		DomainDataMetadata: DomainDataMetadata{
-			EditableDomainDataMetadata: EditableDomainDataMetadata{
-				Name:     output.Name + "_" + nameSuffix,
-				DataType: output.DataType,
-			},
-			DomainID: j.DomainID,
-		},
-		Data: f,
-	}
-
-	httpMethod := http.MethodPost
-	alreadyUploadedID := j.UploadedDataIDs[output.Name+"."+output.DataType]
-	if alreadyUploadedID != "" {
-		logs.WithTag("job_id", j.ID).
-			WithTag("domain_id", j.DomainID).
-			Infof("%s.%s already uploaded. Updating it instead.", output.Name, output.DataType)
-		domainData.ID = alreadyUploadedID
-		httpMethod = http.MethodPut
-	}
-
-	body := &bytes.Buffer{}
-	writer := multipart.NewWriter(body)
-
-	if err := WriteDomainData(writer, &domainData); err != nil {
-		return fmt.Errorf("failed to write domain data to message body: %s", err.Error())
-	}
-
-	if err := writer.Close(); err != nil {
-		return err
-	}
-
-	reqUrl := j.DomainServerURL + "/api/v1/domains/" + j.DomainID + "/data"
-	req, err := http.NewRequest(httpMethod, reqUrl, body)
-	if err != nil {
-		return err
-	}
-	req.Header.Set("Content-Type", writer.FormDataContentType())
-	req.Header.Set("Authorization", "Bearer "+j.AccessToken)
-
-	resp, err := http.DefaultClient.Do(req)
-	if err != nil {
-		return err
-	}
-	defer resp.Body.Close()
-
-	if resp.StatusCode != http.StatusOK {
-		return fmt.Errorf("domain server returned status %d", resp.StatusCode)
-	}
-
-	responseBody, err := io.ReadAll(resp.Body)
-	if err != nil {
-		return err
-	}
-	logs.WithTag("job_id", j.ID).
-		WithTag("domain_id", j.DomainID).
-		Infof("Uploaded domain data! response: %s", string(responseBody))
-	var parsedResp PostDomainDataResponse
-	if err := json.Unmarshal(responseBody, &parsedResp); err != nil {
-		return err
-	}
-	logs.WithTag("job_id", j.ID).
-		WithTag("domain_id", j.DomainID).
-		Infof("Uploaded domain data! parsed response: %+v", parsedResp)
-	j.UploadedDataIDs[output.Name+"."+output.DataType] = parsedResp.Data[0].ID
-	return nil
-}
-
-func DownloadDomainDataFromDomain(ctx context.Context, j *job, ids ...string) error {
-
-	logs.WithTag("job_id", j.ID).
-		WithTag("domain_id", j.DomainID).
-		Infof("downloading %d data from domain", len(ids))
-	if len(ids) == 0 {
-		return errors.New("no data ids provided")
-	}
-
-	scan_data_ids := strings.Join(ids, ",")
-
-	req, err := http.NewRequestWithContext(ctx, http.MethodGet, j.DomainServerURL+"/api/v1/domains/"+j.DomainID+"/data?ids="+scan_data_ids, nil)
-	if err != nil {
-		return err
-	}
-	req.Header.Add("Authorization", "Bearer "+j.AccessToken)
-	req.Header.Add("Accept", "multipart/form-data")
-
-	logs.WithTag("job_id", j.ID).
-		WithTag("domain_id", j.DomainID).
-		Info("Downloading data from domain, request:\n", req)
-
-	resp, err := http.DefaultClient.Do(req)
-	if err != nil {
-		return err
-	}
-	defer resp.Body.Close()
-
-	if resp.StatusCode != http.StatusOK {
-		return fmt.Errorf("domain server returned status %d", resp.StatusCode)
-	}
-
-	_, params, err := mime.ParseMediaType(resp.Header.Get("Content-Type"))
-	if err != nil {
-		return err
-	}
-	var i int
-	mr := multipart.NewReader(resp.Body, params["boundary"])
-	for {
-		part, err := mr.NextPart()
-		if err != nil {
-			if err == io.EOF {
-				break
-			}
-			return err
-		}
-
-		meta, err := ReadDomainDataMetadata(part)
-		if err != nil {
-			return err
-		}
-
-		// For now match multiple data objects from same scan by the timestamp in the name:
-		// dmt_manifest_2024-09-27_01-13-50
-		// dmt_pointcloud_2024-09-27_01-13-50
-		// dmt_arposes_2024-09-27_01-13-50
-		// dmt_portal_detections_2024-09-27_01-13-50
-		// dmt_intrinsics_2024-09-27_01-13-50
-		// dmt_frames_2024-09-27_01-13-50
-
-		dateTimeRegex := regexp.MustCompile(`\d{4}-\d{2}-\d{2}[_-]\d{2}-\d{2}-\d{2}`)
-		scanFolderName := ""
-		if match := dateTimeRegex.FindString(meta.Name); match != "" {
-			scanFolderName = match
-		}
-
-		var fileName string
-		switch meta.DataType {
-		case "dmt_manifest_json":
-			fileName = "Manifest.json"
-		case "dmt_featurepoints_ply", "dmt_pointcloud_ply":
-			fileName = "FeaturePoints.ply"
-		case "dmt_arposes_csv":
-			fileName = "ARposes.csv"
-		case "dmt_portal_detections_csv", "dmt_observations_csv":
-			fileName = "PortalDetections.csv"
-		case "dmt_intrinsics_csv", "dmt_cameraintrinsics_csv":
-			fileName = "CameraIntrinsics.csv"
-		case "dmt_frames_csv":
-			fileName = "Frames.csv"
-		case "dmt_gyro_csv":
-			fileName = "Gyro.csv"
-		case "dmt_accel_csv":
-			fileName = "Accel.csv"
-		case "dmt_gyroaccel_csv":
-			fileName = "gyro_accel.csv"
-		case "dmt_recording_mp4":
-			fileName = "Frames.mp4"
-		default:
-			logs.Infof("unknown domain data type: %s", meta.DataType)
-			fileName = meta.Name + "." + meta.DataType
-		}
-
-		scanFolder := path.Join(j.JobPath, "datasets", scanFolderName)
-		if err := os.MkdirAll(scanFolder, 0755); err != nil {
-			return err
-		}
-
-		f, err := os.Create(path.Join(scanFolder, fileName))
-		if err != nil {
-			return err
-		}
-		defer f.Close()
-
-		if _, err := io.Copy(f, part); err != nil {
-			return err
-		}
-
-		i++
-	}
-	logs.WithTag("job_id", j.ID).
-		WithTag("domain_id", j.DomainID).
-		Infof("downloaded %d data objects from domain", i)
-	return nil
-}
-
-func ReadDispositionParams(part *multipart.Part) (map[string]string, error) {
-	d := part.Header.Get("Content-Disposition")
-	var err error
-	_, dispositionParams, err := mime.ParseMediaType(d)
-	if err != nil {
-		return nil, err
-	}
-	return dispositionParams, nil
-}
-
-func ReadDomainDataMetadata(part *multipart.Part) (*DomainDataMetadata, error) {
-	dispositionParams, err := ReadDispositionParams(part)
-	if err != nil {
-		return nil, err
-	}
-	return &DomainDataMetadata{
-		EditableDomainDataMetadata: EditableDomainDataMetadata{
-			Name:     dispositionParams["name"],
-			DataType: dispositionParams["data-type"],
-		},
-		DomainID: dispositionParams["domain-id"],
-	}, nil
-}
-
-func ReadJobRequestFromJson(requestJson string) (*JobRequestData, error) {
-
-	var jobRequest JobRequestData
-	if err := json.Unmarshal([]byte(requestJson), &jobRequest); err != nil {
-		return nil, err
-	}
-
-	// Debug printing the extracted metadata
-	logs.Debug("Parsed Metadata:\n")
-	logs.Debug("Data IDs: %s\n", jobRequest.DataIDs)
-	logs.Debug("DomainID: %s\n", jobRequest.DomainID)
-	logs.Debug("Processing Type: %s\n", jobRequest.ProcessingType)
-	logs.Debug("Access Token: %s\n", jobRequest.AccessToken)
-	logs.Debug("Domain Server URL: %s\n", jobRequest.DomainServerURL)
-
-	return &jobRequest, nil
-}
-
-func CreateJobMetadata(dirPath string, requestJson string, reconstructionServerURL string) (*job, error) {
-
-	logs.Info("Will mkdir path ", dirPath)
-	if err := os.MkdirAll(dirPath, 0750); err != nil {
-		return nil, err
-	}
-
-	logs.Info("Refinement job requested")
-	jobRequest, err := ReadJobRequestFromJson(requestJson)
-
-	if err != nil {
-		return nil, err
-	}
-
-	logs.WithTag("domain_id", jobRequest.DomainID).
-		Info("Parsing domain access token: ", jobRequest.AccessToken)
-	t, err := jwt.ParseString(jobRequest.AccessToken, jwt.WithValidate(false))
-	if err != nil {
-		return nil, errors.New("Error parsing domain access token from job request").
-			WithTag("domain_id", jobRequest.DomainID).Wrap(err)
-	}
-
-	domainServerURL := jobRequest.DomainServerURL
-	if domainServerURL == "" {
-		domainServerURL = t.Issuer()
-		if domainServerURL == "" {
-			return nil, errors.New("domain server URL is not set in job request or domain access token")
-		}
-		logs.WithTag("domain_id", jobRequest.DomainID).
-			Info("Using domain server URL from domain access token: ", domainServerURL)
-
-	} else {
-		logs.WithTag("domain_id", jobRequest.DomainID).
-			Info("Using domain server URL from job request: ", domainServerURL)
-	}
-
-	startTime := time.Now()
-	jobID := uuid.NewString()
-	jobName := "job_" + jobID
-
-	j := job{
-		JobMetadata: JobMetadata{
-			CreatedAt:               startTime,
-			ID:                      jobID,
-			Name:                    jobName,
-			DomainID:                jobRequest.DomainID,
-			DataIDs:                 jobRequest.DataIDs,
-			ProcessingType:          jobRequest.ProcessingType,
-			DomainServerURL:         domainServerURL,
-			ReconstructionServerURL: reconstructionServerURL,
-			AccessToken:             jobRequest.AccessToken,
-		},
-		Status:          "started",
-		UploadedDataIDs: map[string]string{},
-		JobPath:         path.Join(dirPath, jobRequest.DomainID, jobName),
-	}
-
-	if err := os.MkdirAll(j.JobPath, 0755); err != nil {
-		return nil, errors.New("failed to create job directory").Wrap(err).
-			WithTag("job_id", j.ID).
-			WithTag("domain_id", j.DomainID)
-	}
-
-	// write the requestJson to file, for debugging purposes
-	requestFile := path.Join(j.JobPath, "job_request.json")
-	if err := os.WriteFile(requestFile, []byte(requestJson), 0644); err != nil {
-		return nil, errors.New("failed to write jobrequest json file to disk").Wrap(err).
-			WithTag("job_id", j.ID).
-			WithTag("domain_id", j.DomainID)
-	}
-
-	// write job metadata to file, gets added into refined manifest file
-	metadataFile := path.Join(j.JobPath, "job_metadata.json")
-	jobMetadataJson, err := json.Marshal(j.JobMetadata)
-	if err != nil {
-		return nil, errors.New("failed to marshal job metadata to json").Wrap(err).
-			WithTag("job_id", j.ID).
-			WithTag("domain_id", j.DomainID)
-	}
-	if err := os.WriteFile(metadataFile, jobMetadataJson, 0644); err != nil {
-		return nil, errors.New("failed to write job metadata json file to disk").Wrap(err).
-			WithTag("job_id", j.ID).
-			WithTag("domain_id", j.DomainID)
-	}
-
-	logs.WithTag("job_id", j.ID).
-		WithTag("domain_id", j.DomainID).
-		Infof("Job Request File: %s", requestFile)
-
-	logs.WithTag("job_id", j.ID).
-		WithTag("domain_id", j.DomainID).
-		Infof("Job Metadata File: %s", metadataFile)
-
-	jobs.AddJob(&j)
-	logs.WithTag("job_id", j.ID).
-		WithTag("domain_id", j.DomainID).
-		Infof("Job added to job list: %s", j.ID)
-
-	return &j, nil
-}
-
-func executeJob(j *job) {
-
-	// Write in-progress manifest as soon as job starts.
-	// DMT uses this to show job status to the user.
-	WriteJobManifestFile(j, "processing")
-
-	// Download domain data in batches of 20 ids
-	batchSize := 20
-	for i := 0; i < len(j.DataIDs); i += batchSize {
-		end := i + batchSize
-		if end > len(j.DataIDs) {
-			end = len(j.DataIDs)
-		}
-		batch := j.DataIDs[i:end]
-
-		if err := DownloadDomainDataFromDomain(context.Background(), j, batch...); err != nil {
-			logs.WithTag("job_id", j.ID).
-				WithTag("domain_id", j.DomainID).
-				Error(errors.Newf("failed to download data batch %d-%d", i, end).Wrap(err))
-			jobs.UpdateJob(j.ID, "failed")
-			return
-		}
-	}
-
-	refinementPython := "main.py"
-
-	outputPath := path.Join(j.JobPath, "refined")
-	logFilePath := path.Join(j.JobPath, "log.txt")
-
-	params := []string{
-		refinementPython,
-		"--mode", j.ProcessingType,
-		"--job_root_path", j.JobPath,
-		"--output", outputPath,
-		"--domain_id", j.DomainID,
-		"--job_id", j.Name,
-		"--scans"}
-
-	datasetsRootPath := path.Join(j.JobPath, "datasets")
-	if allScanFolders, err := os.ReadDir(datasetsRootPath); err != nil {
-		logs.WithTag("job_id", j.ID).
-			WithTag("domain_id", j.DomainID).
-			Error(errors.Newf("failed to to read input directory").Wrap(err))
-		jobs.UpdateJob(j.ID, "failed")
-		return
-	} else {
-		logs.WithTag("job_id", j.ID).
-			WithTag("domain_id", j.DomainID).
-			Infof("read %d scan folders", len(allScanFolders))
-
-		scanDataSummaryPath := path.Join(j.JobPath, "scan_data_summary.json")
-		WriteScanDataSummary(datasetsRootPath, allScanFolders, scanDataSummaryPath)
-
-		for _, folder := range allScanFolders {
-			params = append(params, folder.Name())
-		}
-	}
-
-	startTime := time.Now()
-	logs.WithTag("job_id", j.ID).
-		WithTag("domain_id", j.DomainID).
-		Debugf("executing main.py with params: %s", params)
-	cmd := exec.Command("python3", params...)
-
-	// Create log file
-	logFile, err := os.Create(logFilePath)
-	if err != nil {
-		logs.WithTag("job_id", j.ID).
-			WithTag("domain_id", j.DomainID).
-			Error(errors.Newf("failed to create log file").Wrap(err))
-		jobs.UpdateJob(j.ID, "failed")
-		return
-	}
-	defer logFile.Close()
-
-	// Write to both log file and stdout/stderr
-	stdoutWriter := io.MultiWriter(logFile, os.Stdout)
-	stderrWriter := io.MultiWriter(logFile, os.Stderr)
-	cmd.Stdout = stdoutWriter
-	cmd.Stderr = stderrWriter
-
-	logs.WithTag("job_id", j.ID).
-		WithTag("domain_id", j.DomainID).
-		Infof("started, logging to %s", logFilePath)
-
-	// Run the refinement python
-	if err := cmd.Start(); err != nil {
-		logs.WithTag("job_id", j.ID).
-			WithTag("domain_id", j.DomainID).
-			Error(errors.Newf("job failed to start").Wrap(err))
-		jobs.UpdateJob(j.ID, "failed")
-		return
-	}
-
-	// Monitor progress in a separate goroutine
-	progressDone := make(chan bool)
-	go func() {
-		refinedPath := path.Join(outputPath, "local")
-
-		for {
-			select {
-			case <-progressDone:
-				return
-			default:
-				// Get total number of datasets
-				datasetFolders, err := os.ReadDir(datasetsRootPath)
-				if err != nil {
-					logs.WithTag("job_id", j.ID).
-						WithTag("domain_id", j.DomainID).
-						Error(errors.Newf("Error reading datasets directory").Wrap(err))
-					return
-				}
-				totalCount := len(datasetFolders)
-
-				// Check number of completed datasets by matching dataset names
-				refinedCount := 0
-				for _, dataset := range datasetFolders {
-					if _, err := os.Stat(path.Join(refinedPath, dataset.Name())); err == nil {
-						refinedCount++
-					}
-				}
-				refinedCount-- // Last folder created is still refining, remove it
-				if refinedCount < 0 {
-					refinedCount = 0 // Just in case
-				}
-				progress := int((float64(refinedCount) / float64(totalCount)) * 100)
-				if progress > 100 {
-					progress = 100
-				}
-
-				// Update manifest with current progress
-				statusText := fmt.Sprintf("Processed %d of %d scans", refinedCount, totalCount)
-				logs.WithTag("job_id", j.ID).
-					WithTag("domain_id", j.DomainID).
-					Infof("progress: %d%% - %s", progress, statusText)
-
-				err = WriteJobManifestFileHelper(j, "processing", progress, statusText)
-				if err != nil {
-					logs.WithTag("job_id", j.ID).
-						WithTag("domain_id", j.DomainID).
-						Error(errors.Newf("failed to write job manifest").Wrap(err))
-				}
-
-				err = UploadJobManifestToDomain(j)
-				if err != nil {
-					logs.WithTag("job_id", j.ID).
-						WithTag("domain_id", j.DomainID).
-						Error(errors.Newf("failed to upload job manifest").Wrap(err))
-				}
-
-				time.Sleep(30 * time.Second)
-			}
-		}
-	}()
-
-	if err := cmd.Wait(); err != nil {
-		progressDone <- true
-		logs.WithTag("job_id", j.ID).
-			WithTag("domain_id", j.DomainID).
-			Error(errors.Newf("job failed").Wrap(err))
-		jobs.UpdateJob(j.ID, "failed")
-		return
-	}
-	progressDone <- true
-
-	logs.WithTag("job_id", j.ID).
-		WithTag("domain_id", j.DomainID).
-		Infof("Refinement python script finished.")
-
-	timeTaken := time.Since(startTime)
-	logs.WithTag("job_id", j.ID).
-		WithTag("domain_id", j.DomainID).
-		Infof("Refinement algorithm took %s", timeTaken)
-
-	logs.WithTag("job_id", j.ID).
-		WithTag("domain_id", j.DomainID).
-		Infof("Going to upload results to domain %s", j.DomainID)
-
-	if _, err := UploadRefinedOutputsToDomain(j); err != nil {
-		logs.WithTag("job_id", j.ID).
-			WithTag("domain_id", j.DomainID).
-			Error(errors.New("failed to upload refined outputs to domain").Wrap(err))
-		jobs.UpdateJob(j.ID, "failed")
-		return
-	}
-
-	// remove the job directory (disable for now)
-	// TODO: keep scan inputs downloaded in some local cache while still setting up the domain.
-	/*if err := os.RemoveAll(j.JobPath); err != nil {
-		log.Printf("job %s failed to remove output directory: %s", j.ID, err)
-	}
-	*/
-
-	logs.WithTag("job_id", j.ID).
-		WithTag("domain_id", j.DomainID).
-		Infof("job succeeded!")
-	jobs.UpdateJob(j.ID, "succeeded")
-}
diff --git a/server/main.go b/server/main.go
deleted file mode 100644
index 4e65e1e..0000000
--- a/server/main.go
+++ /dev/null
@@ -1,130 +0,0 @@
-package main
-
-import (
-	"encoding/json"
-	"flag"
-	"io"
-
-	"net/http"
-	"net/http/httputil"
-	"reflect"
-	"sync"
-
-	"github.com/aukilabs/go-tooling/pkg/errors"
-	"github.com/aukilabs/go-tooling/pkg/logs"
-	"github.com/go-chi/chi"
-)
-
-// Add a global variable to track if a job is in progress
-var (
-	jobInProgress bool
-	jobMutex      sync.Mutex
-)
-
-func main() {
-
-	apiKey := flag.String("api-key", "", "API key for the server")
-	port := flag.String("port", ":8080", "Port to run the server on")
-	loglevel := flag.String("log-level", "info", "Log Level")
-	flag.Parse()
-
-	// Configure logging to include file name, line number, and timestamp
-	logs.SetLevel(logs.ParseLevel(*loglevel))
-	logs.Encoder = json.Marshal
-
-	if apiKey == nil || *apiKey == "" {
-		logs.Fatal(errors.New("API key is required"))
-	}
-	// create a new router
-	r := chi.NewRouter()
-
-	// Endpoint for triggering refinement jobs (from DMT)
-	r.Post("/jobs", func(w http.ResponseWriter, r *http.Request) {
-		logs.Info("[POST] /jobs endpoint called")
-
-		debug, _ := httputil.DumpRequest(r, true)
-		logs.Debug(debug)
-
-		if apiKey != nil && *apiKey != "" {
-			inApiKey := r.Header.Get("X-API-Key")
-			if inApiKey != *apiKey {
-				http.Error(w, "Unauthorized", http.StatusUnauthorized)
-				return
-			}
-		}
-
-		// Check if a job is already in progress
-		jobMutex.Lock()
-		if jobInProgress {
-			jobMutex.Unlock()
-			logs.Info("Job already in progress, rejecting incoming job request.")
-			http.Error(w, "Reconstruction server is busy processing another job", http.StatusServiceUnavailable)
-			return
-		}
-		jobInProgress = true
-		jobMutex.Unlock()
-
-		// Read request body
-		reqBodyBytes, err := io.ReadAll(r.Body)
-		if err != nil {
-			logs.Error(errors.New("Failed to read request body for job request: " + err.Error()))
-			http.Error(w, err.Error(), http.StatusInternalServerError)
-			jobMutex.Lock()
-			jobInProgress = false
-			jobMutex.Unlock()
-			return
-		}
-		reqBodyString := string(reqBodyBytes)
-		logs.Infof("Request body: %s", reqBodyString)
-
-		reconstructionServerURL := r.Host
-
-		// Create job metadata
-		j, err := CreateJobMetadata("jobs", reqBodyString, reconstructionServerURL)
-		if err != nil {
-			logs.Error(errors.New("Job creation failed with error: " + err.Error()))
-			http.Error(w, err.Error(), http.StatusInternalServerError)
-			jobMutex.Lock()
-			jobInProgress = false
-			jobMutex.Unlock()
-			return
-		}
-
-		// Execute Job
-		go func(j job) {
-			defer func() {
-				jobMutex.Lock()
-				jobInProgress = false
-				jobMutex.Unlock()
-			}()
-			executeJob(&j)
-		}(*j)
-
-		w.WriteHeader(http.StatusOK)
-	})
-
-	// Endpoint for fetching current job list
-	r.Get("/jobs", func(w http.ResponseWriter, r *http.Request) {
-		logs.Info("[GET] /jobs endpoint called")
-
-		debug, _ := httputil.DumpRequest(r, true)
-		logs.Debug(debug)
-
-		jobList := jobs.List()
-		logs.Info("Number of jobs to list: ", len(jobList), " type: ", reflect.TypeOf(jobList))
-
-		encoder := json.NewEncoder(w)
-		encoder.SetIndent("", "  ")
-
-		err := encoder.Encode(jobList)
-		if err != nil {
-			logs.Error(errors.Newf("Error encoding jobs list: %v", err))
-			http.Error(w, "Internal Server Error", http.StatusInternalServerError)
-			return
-		}
-		logs.Info("Jobs list returned successfully")
-	})
-	// start the server
-	logs.Info("Server running on ", *port)
-	logs.Fatal(http.ListenAndServe(*port, r))
-}
diff --git a/server/src/global_refinement.rs b/server/src/global_refinement.rs
new file mode 100644
index 0000000..4fb776d
--- /dev/null
+++ b/server/src/global_refinement.rs
@@ -0,0 +1,205 @@
+use std::{collections::HashMap, fs::{self, File}, io::Cursor, path::{Path, PathBuf}, process::Command, time::SystemTime};
+
+use chrono::Utc;
+use domain::{cluster::DomainCluster, datastore::{self, common::Datastore, remote::RemoteDatastore}, message::read_prefix_size_message, protobuf::{domain_data::{Data, Metadata, Query},task::{self, LocalRefinementInputV1, LocalRefinementOutputV1}}};
+use jsonwebtoken::{decode, DecodingKey,Validation, Algorithm};
+use libp2p::Stream;
+use networking::{client::Client, libp2p::Networking};
+use quick_protobuf::{deserialize_from_slice, serialize_into_vec, BytesReader};
+use regex::Regex;
+use tokio::{self, select, time::{sleep, Duration}};
+use futures::{stream::Zip, AsyncReadExt, StreamExt};
+use uuid::Uuid;
+use serde::{Serialize, Deserialize};
+use zip::ZipArchive;
+
+use crate::utils::handshake;
+
+fn unzip_bytes(path: PathBuf, zip_bytes: Vec<u8>) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    let cursor = Cursor::new(zip_bytes);
+    let mut archive = ZipArchive::new(cursor)?;
+    
+    for i in 0..archive.len() {
+        let mut input_file = archive.by_index(i)?;
+        let file_name = input_file.name().to_string();
+        let path = path.join(file_name);
+        let mut output_file: File = File::create(path)?;
+        std::io::copy(&mut input_file, &mut output_file)?;
+
+    }
+    Ok(())
+}
+
+pub(crate) async fn v1(base_path: String, mut stream: Stream, mut datastore: Box<dyn Datastore>, mut c: Client) {
+    let claim = handshake(&mut stream).await.expect("Failed to handshake");
+    let job_id = claim.job_id.clone();
+    c.subscribe(job_id.clone()).await.expect("Failed to subscribe to job");
+    let t = &mut task::Task {
+        name: claim.task_name.clone(),
+        receiver: claim.receiver.clone(),
+        sender: claim.sender.clone(),
+        endpoint: "/global-refinement/v1".to_string(),
+        status: task::Status::STARTED,
+        access_token: "".to_string(),
+        job_id: job_id.clone(),
+        output: None,
+    };
+    /*
+        | volumn/node_name
+        | | job_id
+        | | | datasets/suffix
+        | | | refined/local/suffix/sfm => input for global refinement
+        | | | refined/global => output for global refinement
+    
+    */
+
+    let input = read_prefix_size_message::<task::GlobalRefinementInputV1>(stream).await.expect("Failed to read global refinement input");
+    println!("Received global refinement input: {:?}", input);
+    // merge input.local_refinement_output into query
+    let mut query = Query::default();
+    for result in input.local_refinement_results {
+        query.ids.extend(result.result_ids);
+    }
+    
+    // download the local refinement output
+    let mut downloader = datastore.consume(claim.domain_id.clone(), query, false).await;
+    let task_path = Path::new(&base_path).join(job_id.clone());
+    let dataset_path = Path::new(&task_path).join("datasets");
+    let input_path = Path::new(&task_path).join("refined").join("local");
+    let output_path = Path::new(&task_path).join("refined").join("global");
+    fs::create_dir_all(input_path.clone()).expect("Failed to create input folder");
+    fs::create_dir_all(dataset_path.clone()).expect("Failed to create dataset folder");
+    fs::create_dir_all(output_path.clone()).expect("Failed to create output folder");
+
+    loop {
+        match downloader.next().await {
+            Some(Ok(data)) => {
+                // parse suffix from data.metadata.name
+                let date_time_regex = Regex::new(r"\d{4}-\d{2}-\d{2}[_-]\d{2}-\d{2}-\d{2}").unwrap();
+                let res = date_time_regex.find(&data.metadata.name)
+                    .map(|m| m.as_str().to_string());
+                if res.is_none() {
+                    t.status = task::Status::FAILED;
+                    t.output = Some(task::Any {
+                        type_url: "Error".to_string(),
+                        value: "Failed to parse suffix from data.metadata.name".to_string().into_bytes(),
+                    });
+                    let message = serialize_into_vec(t).expect("failed to serialize task update");
+                    c.publish(job_id.clone(), message).await.expect("failed to publish task update");
+                    return;
+                }
+                let suffix = res.unwrap();
+                
+                fs::create_dir_all(Path::new(&dataset_path).join(&suffix)).expect("Failed to create dataset folder");
+                let path = Path::new(&input_path).join(&suffix).join("sfm");
+                if let Err(e) = unzip_bytes(path, data.content) {
+                    t.status = task::Status::FAILED;
+                    t.output = Some(task::Any {
+                        type_url: "Error".to_string(),
+                        value: e.to_string().into_bytes(),
+                    });
+                    let message = serialize_into_vec(t).expect("failed to serialize task update");
+                    c.publish(job_id.clone(), message).await.expect("failed to publish task update");
+                    return;
+                }
+                println!("downloaded {}", data.metadata.name);
+            }
+            Some(Err(_)) => {
+                t.status = task::Status::RETRY;
+                let message = serialize_into_vec(t).expect("failed to serialize task update");
+                c.publish(job_id.clone(), message).await.expect("failed to publish task update");
+                return;
+            }
+            None => {
+                break;
+            }
+        }
+    }
+    let params = vec![
+        "main.py",
+        "--mode", "global_refinement",
+        "--job_root_path", task_path.to_str().unwrap(),
+        "--output", output_path.to_str().unwrap(),
+        "--domain_id", &claim.domain_id,
+        "--job_id", &claim.job_id,
+        "--scans"
+    ];
+    c.publish(job_id.clone(), serialize_into_vec(t).expect("failed to serialize task update")).await.expect("failed to publish task update");
+
+    let output = Command::new("python3")
+        .args(params)
+        .output();
+    match output {
+        Err(e) => {
+            eprintln!("Failed to execute global refinement: {}", e);
+            t.status = task::Status::FAILED;
+            t.output = Some(task::Any {
+                type_url: "Error".to_string(),
+                value: serialize_into_vec(&task::Error {
+                    message: e.to_string(),
+                }).unwrap(),
+            });
+            let message = serialize_into_vec(t).expect("failed to serialize task update");
+            c.publish(job_id.clone(), message).await.expect("failed to publish task update");
+            return;
+        }
+        Ok(_) => println!("Finished executing {}", claim.task_name)
+    }
+
+    let mut uploader = datastore.produce(claim.domain_id.clone()).await;
+    // open output_path and upload to datastore
+    let files = fs::read_dir(output_path).expect("Failed to read output folder");
+    for file in files {
+        let file = file.unwrap();
+        let path = file.path();
+        let content = fs::read(path).expect("Failed to read file");
+        let metadata: Metadata = match file.file_name().to_str().unwrap() {
+            "refined_manifest.json" => Metadata {
+                name: "refined_manifest".to_string(),
+                data_type: "refined_manifest_json".to_string(),
+                size: content.len() as u32,
+                id: None,
+                properties: HashMap::new(),
+            },
+            "RefinedPointCloud.ply" => Metadata {
+                name: "refined_pointcloud".to_string(),
+                data_type: "refined_pointcloud_ply".to_string(),
+                size: content.len() as u32,
+                id: None,
+                properties: HashMap::new(),
+            },
+            "BasicStitchPointCloud.ply" => Metadata {
+                name: "unrefined_pointcloud".to_string(),
+                data_type: "unrefined_pointcloud_ply".to_string(),
+                size: content.len() as u32,
+                id: None,
+                properties: HashMap::new(),
+            },
+            _ => continue
+        };
+        uploader.push(&Data {
+            domain_id: claim.domain_id.clone(),
+            metadata,
+            content,
+        }).await.expect("failed to publish file");
+    }
+
+    while !uploader.is_completed().await {
+        sleep(Duration::from_secs(3)).await;
+    }
+    uploader.close().await;
+
+    let event = task::Task {
+        name: claim.task_name.clone(),
+        receiver: claim.receiver.clone(),
+        sender: claim.sender.clone(),
+        endpoint: "/global-refinement/v1".to_string(),
+        status: task::Status::DONE,
+        access_token: "".to_string(),
+        job_id: job_id.clone(),
+        output: None,
+    };
+    let buf = serialize_into_vec(&event).expect("failed to serialize task update");
+    c.publish(job_id.clone(), buf).await.expect("failed to publish task update");
+    println!("Finished executing {}", claim.task_name);
+}
diff --git a/server/src/local_refinement.rs b/server/src/local_refinement.rs
new file mode 100644
index 0000000..492b20d
--- /dev/null
+++ b/server/src/local_refinement.rs
@@ -0,0 +1,287 @@
+use std::{collections::HashMap, fs, path::Path, process::{Command, ExitCode, ExitStatus, Stdio}, time::Duration};
+use domain::{datastore::common::Datastore, message::read_prefix_size_message, protobuf::{domain_data::{Data, Metadata, Query},task::{self, LocalRefinementInputV1, LocalRefinementOutputV1}}};
+use libp2p::Stream;
+use networking::client::Client;
+use quick_protobuf::serialize_into_vec;
+use futures::StreamExt;
+use tokio::time::sleep;
+use uuid::Uuid;
+use std::io::{self, BufReader, BufWriter, Read, Write};
+use zip::{write::{FileOptions, SimpleFileOptions}, ZipWriter};
+use std::io::BufRead;
+
+use crate::utils::handshake;
+
+pub(crate) async fn v1(base_path: String, mut stream: Stream, mut datastore: Box<dyn Datastore>, mut c: Client) {
+    let claim = handshake(&mut stream).await.expect("Failed to handshake");
+    let job_id = claim.job_id.clone();
+    c.subscribe(job_id.clone()).await.expect("Failed to subscribe to job");
+    let t = &mut task::Task {
+        name: claim.task_name.clone(),
+        receiver: claim.receiver.clone(),
+        sender: claim.sender.clone(),
+        endpoint: "/local-refinement/v1".to_string(),
+        status: task::Status::STARTED,
+        access_token: "".to_string(),
+        job_id: job_id.clone(),
+        output: None,
+    };
+
+    let res = read_prefix_size_message::<LocalRefinementInputV1>(stream).await;
+    if let Err(e) = res {
+        eprintln!("Failed to load local refinement input {}", e);
+        t.status = task::Status::FAILED;
+        let message = serialize_into_vec(t).expect("failed to serialize task update");
+        c.publish(job_id.clone(), message).await.expect("failed to publish task update");
+        return;
+    }
+    let input = res.unwrap();
+
+    println!("Start executing {}, {:?}", claim.task_name, input);
+
+    // input.name_regexp looks .*_date
+    // get date from regexp
+    if input.query.is_none() {
+        t.status = task::Status::FAILED;
+        t.output = Some(task::Any {
+            type_url: "Error".to_string(),
+            value: "Query is empty".as_bytes().to_vec(),
+        });
+        let message = serialize_into_vec(t).expect("failed to serialize task update");
+        c.publish(job_id.clone(), message).await.expect("failed to publish task update");
+        return;
+    }
+    let query = input.query.clone().unwrap();
+    let query_clone = query.clone();
+    let name_regexp = query.name_regexp;
+    if name_regexp.is_none() {
+        t.status = task::Status::FAILED;
+        t.output = Some(task::Any {
+            type_url: "Error".to_string(),
+            value: "Name regexp is empty".as_bytes().to_vec(),
+        });
+        let message = serialize_into_vec(t).expect("failed to serialize task update");
+        c.publish(job_id.clone(), message).await.expect("failed to publish task update");
+        return;
+    }
+
+    /*
+        get suffix from regexp
+
+        | volumn/node_name
+        | | job_id
+        | | | datasets/suffix => input for local refinement
+        | | | refined/local/suffix
+        | | | | sfm/.(txt|bin|csv) => output for local refinement
+        upload refined_scan_2024-12-03_22-57-12 with type refined_scan_zip_v1
+
+     */
+    let suffix = name_regexp.unwrap().replace(".*_", "");
+
+    let task_folder = Path::new(&base_path).join(&claim.job_id);
+    let scan_folder = Path::new(&task_folder).join("datasets");
+    let input_folder = Path::new(&scan_folder).join(&suffix.clone());
+    let output_folder = Path::new(&task_folder).join("refined").join("local").join(&suffix);
+    fs::create_dir_all(&scan_folder).expect("Failed to create directory");
+    fs::create_dir_all(&output_folder).expect("Failed to create directory");
+    fs::create_dir_all(&input_folder).expect("Failed to create directory");
+
+    let mut downloader = datastore.consume("".to_string(), query_clone, false).await;
+    let mut i = 0;
+    loop {
+        match downloader.next().await {
+            Some(Ok(data)) => {
+                let filename = match data.metadata.data_type.as_str() {
+                    "dmt_manifest_json" => "Manifest.json".to_string(),
+                    "dmt_featurepoints_ply" | "dmt_pointcloud_ply" => "FeaturePoints.ply".to_string(),
+                    "dmt_arposes_csv" => "ARposes.csv".to_string(),
+                    "dmt_portal_detections_csv" | "dmt_observations_csv" => "PortalDetections.csv".to_string(),
+                    "dmt_intrinsics_csv" | "dmt_cameraintrinsics_csv" => "CameraIntrinsics.csv".to_string(),
+                    "dmt_frames_csv" => "Frames.csv".to_string(),
+                    "dmt_gyro_csv" => "Gyro.csv".to_string(),
+                    "dmt_accel_csv" => "Accel.csv".to_string(),
+                    "dmt_gyroaccel_csv" => "gyro_accel.csv".to_string(),
+                    "dmt_recording_mp4" => "Frames.mp4".to_string(),
+                    _ => {
+                        println!("unknown domain data type: {}", data.metadata.data_type);
+                        format!("{}.{}", data.metadata.name, data.metadata.data_type)
+                    }
+                };
+                let path = input_folder.join(&filename);
+                fs::write(path, &data
+                    .content)
+                    .expect("Failed to write data to file");
+                i+=1;
+                println!("downloaded {}", filename);
+            }
+            Some(Err(_)) => {
+                t.status = task::Status::RETRY;
+                let message = serialize_into_vec(t).expect("failed to serialize task update");
+                c.publish(job_id.clone(), message).await.expect("failed to publish task update");
+                return;
+            }
+            None => {
+                break;
+            }
+        }
+    }
+    println!("Finished downloading {} data for {}", i, claim.task_name);
+
+    let params = vec![
+        "main.py",
+        "--mode", "local_refinement",
+        "--job_root_path", task_folder.to_str().unwrap(),
+        "--output", output_folder.to_str().unwrap(),
+        "--domain_id", &claim.domain_id,
+        "--job_id", &claim.job_id,
+        "--scans"
+    ];
+    c.publish(job_id.clone(), serialize_into_vec(t).expect("failed to serialize task update")).await.expect("failed to publish task update");
+
+    let child = Command::new("python3")
+    .args(params)
+    .stdout(Stdio::piped())
+    .stderr(Stdio::piped())
+    .spawn();
+
+    if let Err(e) = child {
+        eprintln!("Failed to execute local refinement: {}", e);
+        t.status = task::Status::FAILED;
+        let message = serialize_into_vec(t).expect("failed to serialize task update");
+        c.publish(job_id.clone(), message).await.expect("failed to publish task update");
+        return;
+    }
+    let mut child = child.unwrap();
+
+    // Read stdout in real-time
+    if let Some(stdout) = child.stdout.take() {
+        let stdout_reader = BufReader::new(stdout);
+        tokio::spawn(async move {
+            for line in stdout_reader.lines() {
+                if let Ok(line) = line {
+                    println!("stdout: {}", line);
+                }
+            }
+        });
+    }
+
+    // Read stderr in real-time
+    if let Some(stderr) = child.stderr.take() {
+        let stderr_reader = BufReader::new(stderr);
+        tokio::spawn(async move {
+            for line in stderr_reader.lines() {
+                if let Ok(line) = line {
+                    eprintln!("stderr: {}", line);
+                }
+            }
+        });
+    }
+
+    // Wait for the command to complete
+    let status = child.wait();
+
+    match status {
+        Err(e) => {
+            eprintln!("Failed to execute local refinement: {}", e);
+            t.status = task::Status::FAILED;
+            t.output = Some(task::Any {
+                type_url: "Error".to_string(),
+                value: serialize_into_vec(&task::Error {
+                    message: e.to_string(),
+                }).unwrap(),
+            });
+            let message = serialize_into_vec(t).expect("failed to serialize task update");
+            c.publish(job_id.clone(), message).await.expect("failed to publish task update");
+            return;
+        }
+        Ok(exit_status) => {
+            if !exit_status.success() {
+                eprintln!("Failed to execute local refinement: {}", exit_status.code().unwrap_or(-1));
+                t.status = task::Status::FAILED;
+                let message = serialize_into_vec(t).expect("failed to serialize task update");
+                c.publish(job_id.clone(), message).await.expect("failed to publish task update");
+                return;
+            }
+            println!("Finished executing {}", claim.task_name);
+        }
+    }
+
+    let mut producer = datastore.produce(claim.domain_id.clone()).await;
+
+    let zip_path = output_folder.join("sfm").join(suffix.clone() + ".zip");
+    let mut zip = zip::ZipWriter::new(fs::File::create(&zip_path).expect("Failed to create zip file"));
+    let options = SimpleFileOptions::default().compression_method(zip::CompressionMethod::Stored); // No compression
+    
+    // open output folder/sfm, zip all txt, bin, csv files and upload
+    for entry in fs::read_dir(output_folder.join("sfm")).expect("Failed to read directory") {
+        let entry = entry.expect("Failed to read entry");
+        let path = entry.path();
+        let ext = path.extension().expect("Failed to get extension").to_str().expect("Failed to convert extension to string");
+        if ext == "txt" || ext == "bin" || ext == "csv" {
+            let str_path = path.to_str().expect("Failed to convert path to string");
+            let file = fs::File::open(str_path).expect("Failed to open file");
+            let mut reader = BufReader::new(file);
+            zip.start_file(str_path, options).expect("Failed to start zip file");
+
+            let mut buffer = [0u8; 8192]; // Use a buffer to stream in chunks
+            loop {
+                let bytes_read = reader.read(&mut buffer).expect("Failed to read file");
+                if bytes_read == 0 {
+                    break;
+                }
+                zip.write_all(&buffer[..bytes_read]).expect("Failed to write to zip");
+            }
+        }
+    }
+    zip.finish().expect("Failed to finish zip");
+    let zip_file_metadata = fs::metadata(&zip_path).expect("Failed to get metadata");
+
+    let res = producer.push(&Data {
+        domain_id: claim.domain_id.clone(),
+        metadata: Metadata {
+            size: zip_file_metadata.len() as u32,
+            name: format!("refined_scan_{}", suffix.clone()),
+            data_type: "refined_scan_zip_v1".to_string(),
+            id: None,
+            properties: HashMap::new(),
+        },
+        content: fs::read(&zip_path).expect("Failed to read zip file"),
+    }).await;
+    if let Err(e) = res {
+        eprintln!("Failed to upload refined scan: {}", e);
+        t.status = task::Status::FAILED;
+        t.output = Some(task::Any {
+            type_url: "Error".to_string(),
+            value: serialize_into_vec(&task::Error {
+                message: e.to_string(),
+            }).unwrap(),
+        });
+        let message = serialize_into_vec(t).expect("failed to serialize task update");
+        c.publish(job_id.clone(), message).await.expect("failed to publish task update");
+        return;
+    }
+    while !producer.is_completed().await {
+        sleep(Duration::from_secs(3)).await;
+    }
+    producer.close().await;
+    println!("Finished uploading refined scan for {}", suffix.clone());
+
+    let output = LocalRefinementOutputV1 {
+        result_ids: vec![res.unwrap()],
+    };
+    let event = task::Task {
+        name: claim.task_name.clone(),
+        receiver: claim.receiver.clone(),
+        sender: claim.sender.clone(),
+        endpoint: "/local-refinement/v1".to_string(),
+        status: task::Status::DONE,
+        access_token: "".to_string(),
+        job_id: job_id.clone(),
+        output: Some(task::Any {
+            type_url: "LocalRefinementOutputV1".to_string(),
+            value: serialize_into_vec(&output).expect("Failed to serialize local refinement output"),
+        }),
+    };
+    let buf = serialize_into_vec(&event).expect("failed to serialize task update");
+    c.publish(job_id.clone(), buf).await.expect("failed to publish task update");
+}
diff --git a/server/src/main.rs b/server/src/main.rs
new file mode 100644
index 0000000..51c171d
--- /dev/null
+++ b/server/src/main.rs
@@ -0,0 +1,52 @@
+use domain::{cluster::DomainCluster, datastore::{common::Datastore, remote::{self, RemoteDatastore}}, message::read_prefix_size_message, protobuf::{domain_data::Query,task::{self, LocalRefinementInputV1, LocalRefinementOutputV1}}};
+use jsonwebtoken::{decode, DecodingKey,Validation, Algorithm};
+use libp2p::Stream;
+use networking::{client::Client, libp2p::Networking};
+use quick_protobuf::{deserialize_from_slice, serialize_into_vec};
+use tokio::{self, select, time::{sleep, Duration}};
+use futures::{AsyncReadExt, StreamExt};
+use uuid::Uuid;
+use regex::Regex;
+mod local_refinement;
+mod global_refinement;
+mod utils;
+
+/*
+    * This is a simple example of a reconstruction node. It will connect to a set of bootstraps and execute reconstruction jobs.
+    * Usage: cargo run <port> <name> <domain_manager> 
+    * Example: cargo run 18808 reconstruction 
+ */
+#[tokio::main]
+async fn main() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    let args: Vec<String> = std::env::args().collect();
+    if args.len() < 3 {
+        println!("Usage: {} <port> <name> <domain_manager>", args[0]);
+        return Ok(());
+    }
+    let port = args[1].parse::<u16>().unwrap();
+    let name = args[2].clone();
+    let base_path = format!("./volume/{}", name);
+    let domain_manager = args[3].clone();
+    let private_key_path = format!("{}/pkey", base_path);
+
+    let domain_manager_id = domain_manager.split("/").last().unwrap().to_string();
+    let domain_cluster = DomainCluster::new(domain_manager.clone(), name, false, None, Some(private_key_path));
+    let mut n = domain_cluster.peer.clone();
+    let mut local_refinement_v1_handler = n.client.set_stream_handler("/local-refinement/v1".to_string()).await.unwrap();
+    let mut global_refinement_v1_handler = n.client.set_stream_handler("/global-refinement/v1".to_string()).await.unwrap();
+    let remote_storage = RemoteDatastore::new(domain_cluster);
+
+    loop {
+        select! {
+            Some((_, stream)) = local_refinement_v1_handler.next() => {
+                let _ = tokio::spawn(local_refinement::v1(base_path.clone(), stream, Box::new(remote_storage.clone()), n.client.clone()));
+            }
+            Some((_, stream)) = global_refinement_v1_handler.next() => {
+                let _ = tokio::spawn(global_refinement::v1(base_path.clone(), stream, Box::new(remote_storage.clone()), n.client.clone()));
+            }
+            else => break
+        }
+    }
+
+    Ok(())
+}
diff --git a/server/src/utils.rs b/server/src/utils.rs
new file mode 100644
index 0000000..ef62641
--- /dev/null
+++ b/server/src/utils.rs
@@ -0,0 +1,39 @@
+use domain::{cluster::DomainCluster, datastore::{common::Datastore, remote::RemoteDatastore}, message::read_prefix_size_message, protobuf::{domain_data::Query,task::{self, LocalRefinementInputV1, LocalRefinementOutputV1}}};
+use jsonwebtoken::{decode, DecodingKey,Validation, Algorithm};
+use libp2p::Stream;
+use networking::{client::Client, libp2p::Networking};
+use quick_protobuf::{deserialize_from_slice, serialize_into_vec};
+use tokio::{self, select, time::{sleep, Duration}};
+use futures::{AsyncReadExt, StreamExt};
+use uuid::Uuid;
+use serde::{Serialize, Deserialize};
+
+#[derive(Debug, Serialize, Deserialize)]
+pub struct TaskTokenClaim {
+    pub domain_id: String,
+    pub task_name: String,
+    pub job_id: String,
+    pub sender: String,
+    pub receiver: String,
+    pub exp: usize,
+    pub iat: usize,
+    pub sub: String,
+}
+
+pub fn decode_jwt(token: &str) -> Result<TaskTokenClaim, Box<dyn std::error::Error + Send + Sync>> {
+    let token_data = decode::<TaskTokenClaim>(token, &DecodingKey::from_secret("secret".as_ref()), &Validation::new(Algorithm::HS256))?;
+    Ok(token_data.claims)
+}
+
+pub async fn handshake(stream: &mut Stream) -> Result<TaskTokenClaim, Box<dyn std::error::Error + Send + Sync>> {
+    let mut length_buf = [0u8; 4];
+    stream.read_exact(&mut length_buf).await?;
+
+    let length = u32::from_be_bytes(length_buf) as usize;
+    let mut buffer = vec![0u8; length];
+    stream.read_exact(&mut buffer).await?;
+        
+    let header = deserialize_from_slice::<task::DomainClusterHandshake>(&buffer)?;
+
+    decode_jwt(header.access_token.as_str())
+}

From 1b2a06aba5ebbd304dfe7b65dca6f3d132dc3083 Mon Sep 17 00:00:00 2001
From: Shuning <shuning@aukilabs.com>
Date: Thu, 13 Mar 2025 18:15:55 +0800
Subject: [PATCH 02/89] disable cache and add concurrency

---
 .github/workflows/docker-build.yml | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml
index 7dbd8a2..c6820a5 100644
--- a/.github/workflows/docker-build.yml
+++ b/.github/workflows/docker-build.yml
@@ -5,6 +5,10 @@ on:
     branches:
       - feature/sdk-integration
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref_name }}
+  cancel-in-progress: true
+
 jobs:
   build:
     runs-on: ubuntu-latest
@@ -23,6 +27,7 @@ jobs:
           platforms: linux/amd64
           load: true
           tags: reconstruction-server:latest
+          no-cache: true
 
       - name: Save Docker image
         run: |

From 0a05d629469ea57a9005b2ec209efb0210f39209 Mon Sep 17 00:00:00 2001
From: Shuning <shuning@aukilabs.com>
Date: Thu, 13 Mar 2025 19:07:18 +0800
Subject: [PATCH 03/89] upload to ECR

---
 .github/workflows/docker-build.yml | 25 ++++++++++++-------------
 1 file changed, 12 insertions(+), 13 deletions(-)

diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml
index c6820a5..19d7c0f 100644
--- a/.github/workflows/docker-build.yml
+++ b/.github/workflows/docker-build.yml
@@ -16,6 +16,16 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@v4
 
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          aws-access-key-id: ${{ secrets.GLOBAL_COMMON_AWS_ACCESS_KEY }}
+          aws-secret-access-key: ${{ secrets.GLOBAL_COMMON_AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ vars.GLOBAL_COMMON_AWS_REGION }}
+      - name: Log in to AWS ECR
+        id: ecr-login
+        uses: aws-actions/amazon-ecr-login@v2
+
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
 
@@ -25,17 +35,6 @@ jobs:
           context: .
           file: docker/Dockerfile
           platforms: linux/amd64
-          load: true
-          tags: reconstruction-server:latest
+          push: true
+          tags: ${{ steps.ecr-login.outputs.registry }}/reconstruction-server:new-sdk-integration-${{ github.sha }}
           no-cache: true
-
-      - name: Save Docker image
-        run: |
-          docker save reconstruction-server:latest > reconstruction-server.tar
-
-      - name: Upload Docker image as artifact
-        uses: actions/upload-artifact@v4
-        with:
-          name: reconstruction-server-image
-          path: reconstruction-server.tar
-          retention-days: 7 

From aa29ba54a334f71c21656b4cb4f1ccf563269c35 Mon Sep 17 00:00:00 2001
From: Shuning <shuning@aukilabs.com>
Date: Fri, 14 Mar 2025 10:34:57 +0800
Subject: [PATCH 04/89] free up space

---
 .github/workflows/docker-build.yml | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml
index 19d7c0f..4d28852 100644
--- a/.github/workflows/docker-build.yml
+++ b/.github/workflows/docker-build.yml
@@ -16,6 +16,15 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@v4
 
+      - name: Free up space
+        run: |
+          sudo rm -rf /usr/share/dotnet  # ~20GB
+          sudo rm -rf /opt/ghc           # ~4GB
+          sudo rm -rf /usr/local/lib/android  # ~10GB
+
+      - name: Check Available Space
+        run: df -h
+
       - name: Configure AWS credentials
         uses: aws-actions/configure-aws-credentials@v4
         with:

From 5777a02c1ab97805433a7309b00de7dccd61aa21 Mon Sep 17 00:00:00 2001
From: Shuning <shuning@aukilabs.com>
Date: Fri, 14 Mar 2025 10:46:24 +0800
Subject: [PATCH 05/89] get larger disk space

---
 .github/workflows/docker-build.yml | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml
index 4d28852..db3eda7 100644
--- a/.github/workflows/docker-build.yml
+++ b/.github/workflows/docker-build.yml
@@ -11,7 +11,7 @@ concurrency:
 
 jobs:
   build:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04-8-cores
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
@@ -47,3 +47,6 @@ jobs:
           push: true
           tags: ${{ steps.ecr-login.outputs.registry }}/reconstruction-server:new-sdk-integration-${{ github.sha }}
           no-cache: true
+
+      - name: Check Available Space
+        run: df -h

From a084d92edb4fb8398e3ec3c965cef3694efdfb7e Mon Sep 17 00:00:00 2001
From: Shuning <shuning@aukilabs.com>
Date: Fri, 14 Mar 2025 10:57:34 +0800
Subject: [PATCH 06/89] no one picks up the job

---
 .github/workflows/docker-build.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml
index db3eda7..8fe080b 100644
--- a/.github/workflows/docker-build.yml
+++ b/.github/workflows/docker-build.yml
@@ -11,7 +11,7 @@ concurrency:
 
 jobs:
   build:
-    runs-on: ubuntu-22.04-8-cores
+    runs-on: ubuntu-22.04-4-cores
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4

From 5b2708e5a1c9d31824919c4742c962cadecdc93f Mon Sep 17 00:00:00 2001
From: Shuning <shuning@aukilabs.com>
Date: Fri, 14 Mar 2025 11:08:52 +0800
Subject: [PATCH 07/89] no one picks up

---
 .github/workflows/docker-build.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml
index 8fe080b..0a821b1 100644
--- a/.github/workflows/docker-build.yml
+++ b/.github/workflows/docker-build.yml
@@ -11,7 +11,7 @@ concurrency:
 
 jobs:
   build:
-    runs-on: ubuntu-22.04-4-cores
+    runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4

From 8e9bceaf10dd164449cf6ceb878da9d3564c9af4 Mon Sep 17 00:00:00 2001
From: Shuning <shuning@aukilabs.com>
Date: Fri, 14 Mar 2025 14:23:01 +0800
Subject: [PATCH 08/89] add heartbeat

---
 server/src/global_refinement.rs | 41 +++++++++++++++++++++++++++++----
 server/src/local_refinement.rs  | 39 ++++++++++++++++++++++++++++---
 2 files changed, 73 insertions(+), 7 deletions(-)

diff --git a/server/src/global_refinement.rs b/server/src/global_refinement.rs
index 4fb776d..19d3f69 100644
--- a/server/src/global_refinement.rs
+++ b/server/src/global_refinement.rs
@@ -7,7 +7,7 @@ use libp2p::Stream;
 use networking::{client::Client, libp2p::Networking};
 use quick_protobuf::{deserialize_from_slice, serialize_into_vec, BytesReader};
 use regex::Regex;
-use tokio::{self, select, time::{sleep, Duration}};
+use tokio::{self, select, sync::watch, time::{interval, sleep, Duration}};
 use futures::{stream::Zip, AsyncReadExt, StreamExt};
 use uuid::Uuid;
 use serde::{Serialize, Deserialize};
@@ -61,8 +61,9 @@ pub(crate) async fn v1(base_path: String, mut stream: Stream, mut datastore: Box
         query.ids.extend(result.result_ids);
     }
     
+    c.publish(job_id.clone(), serialize_into_vec(t).expect("failed to serialize task update")).await.expect("failed to publish task update");
+
     // download the local refinement output
-    let mut downloader = datastore.consume(claim.domain_id.clone(), query, false).await;
     let task_path = Path::new(&base_path).join(job_id.clone());
     let dataset_path = Path::new(&task_path).join("datasets");
     let input_path = Path::new(&task_path).join("refined").join("local");
@@ -71,6 +72,36 @@ pub(crate) async fn v1(base_path: String, mut stream: Stream, mut datastore: Box
     fs::create_dir_all(dataset_path.clone()).expect("Failed to create dataset folder");
     fs::create_dir_all(output_path.clone()).expect("Failed to create output folder");
 
+    let (tx, rx) = watch::channel(false);
+    let mut c_clone = c.clone();
+    let job_id_clone = job_id.clone();
+    let task_clone = t.clone();
+    let heartbeat_handle = tokio::spawn(async move {
+        let mut rx = rx;
+        let mut interval = interval(Duration::from_secs(30)); // Send heartbeat every 30 seconds
+        
+        loop {
+            tokio::select! {
+                _ = interval.tick() => {
+                    let mut progress_task = task_clone.clone();
+                    progress_task.status = task::Status::PROCESSING;
+                    if let Ok(message) = serialize_into_vec(&progress_task) {
+                        let _ = c_clone.publish(job_id_clone.clone(), message).await;
+                    }
+                }
+                Ok(_) = rx.changed() => {
+                    break;
+                }
+            }
+        }
+    });
+
+    let _cleanup = scopeguard::guard(tx, |tx| {
+        let _ = tx.send(true); // Signal heartbeat task to stop
+    });
+
+    let mut downloader = datastore.consume(claim.domain_id.clone(), query, false).await;
+
     loop {
         match downloader.next().await {
             Some(Ok(data)) => {
@@ -124,8 +155,6 @@ pub(crate) async fn v1(base_path: String, mut stream: Stream, mut datastore: Box
         "--job_id", &claim.job_id,
         "--scans"
     ];
-    c.publish(job_id.clone(), serialize_into_vec(t).expect("failed to serialize task update")).await.expect("failed to publish task update");
-
     let output = Command::new("python3")
         .args(params)
         .output();
@@ -202,4 +231,8 @@ pub(crate) async fn v1(base_path: String, mut stream: Stream, mut datastore: Box
     let buf = serialize_into_vec(&event).expect("failed to serialize task update");
     c.publish(job_id.clone(), buf).await.expect("failed to publish task update");
     println!("Finished executing {}", claim.task_name);
+
+    let _ = heartbeat_handle.await;
+    println!("Heartbeat task stopped");
+    return;
 }
diff --git a/server/src/local_refinement.rs b/server/src/local_refinement.rs
index 492b20d..06889c5 100644
--- a/server/src/local_refinement.rs
+++ b/server/src/local_refinement.rs
@@ -4,7 +4,7 @@ use libp2p::Stream;
 use networking::client::Client;
 use quick_protobuf::serialize_into_vec;
 use futures::StreamExt;
-use tokio::time::sleep;
+use tokio::{sync::watch, time::{interval, sleep}};
 use uuid::Uuid;
 use std::io::{self, BufReader, BufWriter, Read, Write};
 use zip::{write::{FileOptions, SimpleFileOptions}, ZipWriter};
@@ -86,6 +86,37 @@ pub(crate) async fn v1(base_path: String, mut stream: Stream, mut datastore: Box
     fs::create_dir_all(&output_folder).expect("Failed to create directory");
     fs::create_dir_all(&input_folder).expect("Failed to create directory");
 
+    c.publish(job_id.clone(), serialize_into_vec(t).expect("failed to serialize task update")).await.expect("failed to publish task update");
+
+    let (tx, rx) = watch::channel(false);
+    let mut c_clone = c.clone();
+    let task_clone = t.clone();
+    let job_id_clone = job_id.clone();
+    let heartbeat_handle = tokio::spawn(async move {
+        let mut rx = rx;
+        let mut interval = interval(Duration::from_secs(30)); // Send heartbeat every 30 seconds
+        
+        loop {
+            tokio::select! {
+                _ = interval.tick() => {
+                    let mut progress_task = task_clone.clone();
+                    progress_task.status = task::Status::PROCESSING;
+                    if let Ok(message) = serialize_into_vec(&progress_task) {
+                        let _ = c_clone.publish(job_id_clone.clone(), message).await;
+                    }
+                }
+                // Check if we should stop
+                Ok(_) = rx.changed() => {
+                    break;
+                }
+            }
+        }
+    });
+
+    let _cleanup = scopeguard::guard(tx, |tx| {
+        let _ = tx.send(true); // Signal heartbeat task to stop
+    });
+
     let mut downloader = datastore.consume("".to_string(), query_clone, false).await;
     let mut i = 0;
     loop {
@@ -136,8 +167,6 @@ pub(crate) async fn v1(base_path: String, mut stream: Stream, mut datastore: Box
         "--job_id", &claim.job_id,
         "--scans"
     ];
-    c.publish(job_id.clone(), serialize_into_vec(t).expect("failed to serialize task update")).await.expect("failed to publish task update");
-
     let child = Command::new("python3")
     .args(params)
     .stdout(Stdio::piped())
@@ -284,4 +313,8 @@ pub(crate) async fn v1(base_path: String, mut stream: Stream, mut datastore: Box
     };
     let buf = serialize_into_vec(&event).expect("failed to serialize task update");
     c.publish(job_id.clone(), buf).await.expect("failed to publish task update");
+
+    let _ = heartbeat_handle.await;
+    println!("Heartbeat task stopped");
+    return;
 }

From 13073f8beb3764a5960416641a7b735ebc51b533 Mon Sep 17 00:00:00 2001
From: Shuning <shuning@aukilabs.com>
Date: Fri, 14 Mar 2025 15:26:08 +0800
Subject: [PATCH 09/89] fix Dockerfile and terminate the program gracefully

---
 docker/Dockerfile  |  5 -----
 server/Cargo.lock  |  1 +
 server/Cargo.toml  |  1 +
 server/src/main.rs | 19 ++++++++++++++++---
 4 files changed, 18 insertions(+), 8 deletions(-)

diff --git a/docker/Dockerfile b/docker/Dockerfile
index b7263ec..c266896 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -181,8 +181,3 @@ USER $USERNAME
 
 # Copy the built binary from rust-build
 COPY --from=rust-build /app/target/release/server /app/server
-
-ENTRYPOINT ["./server"]
-EXPOSE ${PORT}
-
-CMD ["${PORT}", "${DOMAIN_MANAGER_ADDR}"]
diff --git a/server/Cargo.lock b/server/Cargo.lock
index e21a0d4..32aeacb 100644
--- a/server/Cargo.lock
+++ b/server/Cargo.lock
@@ -3772,6 +3772,7 @@ dependencies = [
  "rand 0.9.0",
  "regex",
  "regexp",
+ "scopeguard",
  "serde",
  "serde_json",
  "tokio",
diff --git a/server/Cargo.toml b/server/Cargo.toml
index a254e14..e680233 100644
--- a/server/Cargo.toml
+++ b/server/Cargo.toml
@@ -21,3 +21,4 @@ zip = "2.2.3"
 regex = "1.10.3"
 regexp = "0.3.2"
 chrono = "0.4"
+scopeguard = "1.2.0"
diff --git a/server/src/main.rs b/server/src/main.rs
index 51c171d..72092cf 100644
--- a/server/src/main.rs
+++ b/server/src/main.rs
@@ -3,7 +3,7 @@ use jsonwebtoken::{decode, DecodingKey,Validation, Algorithm};
 use libp2p::Stream;
 use networking::{client::Client, libp2p::Networking};
 use quick_protobuf::{deserialize_from_slice, serialize_into_vec};
-use tokio::{self, select, time::{sleep, Duration}};
+use tokio::{self, select, signal::unix::{signal, SignalKind}, time::{sleep, Duration}};
 use futures::{AsyncReadExt, StreamExt};
 use uuid::Uuid;
 use regex::Regex;
@@ -11,15 +11,24 @@ mod local_refinement;
 mod global_refinement;
 mod utils;
 
+async fn shutdown_signal() {
+    let mut term_signal = signal(SignalKind::terminate()).expect("Failed to register SIGTERM handler");
+    let mut int_signal = signal(SignalKind::interrupt()).expect("Failed to register SIGINT handler");
+
+    tokio::select! {
+        _ = term_signal.recv() => println!("Received SIGTERM, exiting..."),
+        _ = int_signal.recv() => println!("Received SIGINT, exiting..."),
+    }
+}
 /*
     * This is a simple example of a reconstruction node. It will connect to a set of bootstraps and execute reconstruction jobs.
     * Usage: cargo run <port> <name> <domain_manager> 
-    * Example: cargo run 18808 reconstruction 
+    * Example: cargo run 18808 reconstruction /ip4/127.0.0.1/udp/18800/quic-v1/p2p/12D3KooWDHaDQeuYeLM8b5zhNjqS7Pkh7KefqzCpDGpdwj5iE8pq
  */
 #[tokio::main]
 async fn main() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     let args: Vec<String> = std::env::args().collect();
-    if args.len() < 3 {
+    if args.len() < 4 {
         println!("Usage: {} <port> <name> <domain_manager>", args[0]);
         return Ok(());
     }
@@ -44,6 +53,10 @@ async fn main() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
             Some((_, stream)) = global_refinement_v1_handler.next() => {
                 let _ = tokio::spawn(global_refinement::v1(base_path.clone(), stream, Box::new(remote_storage.clone()), n.client.clone()));
             }
+            _ = shutdown_signal() => {
+                println!("Received termination signal, shutting down...");
+                break;
+            }
             else => break
         }
     }

From 703bed43656bd6fb5778d62a75f83cfd18a99e5a Mon Sep 17 00:00:00 2001
From: Shuning <shuning@aukilabs.com>
Date: Fri, 14 Mar 2025 18:22:00 +0800
Subject: [PATCH 10/89] pass in port

---
 server/src/main.rs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/server/src/main.rs b/server/src/main.rs
index 72092cf..688e881 100644
--- a/server/src/main.rs
+++ b/server/src/main.rs
@@ -22,14 +22,14 @@ async fn shutdown_signal() {
 }
 /*
     * This is a simple example of a reconstruction node. It will connect to a set of bootstraps and execute reconstruction jobs.
-    * Usage: cargo run <port> <name> <domain_manager> 
+    * Usage: cargo run <port> <name> <domain_manager_addr> 
     * Example: cargo run 18808 reconstruction /ip4/127.0.0.1/udp/18800/quic-v1/p2p/12D3KooWDHaDQeuYeLM8b5zhNjqS7Pkh7KefqzCpDGpdwj5iE8pq
  */
 #[tokio::main]
 async fn main() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     let args: Vec<String> = std::env::args().collect();
     if args.len() < 4 {
-        println!("Usage: {} <port> <name> <domain_manager>", args[0]);
+        println!("Usage: {} <port> <name> <domain_manager_addr>", args[0]);
         return Ok(());
     }
     let port = args[1].parse::<u16>().unwrap();
@@ -39,7 +39,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     let private_key_path = format!("{}/pkey", base_path);
 
     let domain_manager_id = domain_manager.split("/").last().unwrap().to_string();
-    let domain_cluster = DomainCluster::new(domain_manager.clone(), name, false, None, Some(private_key_path));
+    let domain_cluster = DomainCluster::new(domain_manager.clone(), name, false, port, None, Some(private_key_path));
     let mut n = domain_cluster.peer.clone();
     let mut local_refinement_v1_handler = n.client.set_stream_handler("/local-refinement/v1".to_string()).await.unwrap();
     let mut global_refinement_v1_handler = n.client.set_stream_handler("/global-refinement/v1".to_string()).await.unwrap();

From 431204d8f63b2da352220e2d02ae38dd9e138953 Mon Sep 17 00:00:00 2001
From: Shuning <shuning@aukilabs.com>
Date: Fri, 14 Mar 2025 21:27:33 +0800
Subject: [PATCH 11/89] pull the latest posemesh

---
 server/Cargo.lock | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/server/Cargo.lock b/server/Cargo.lock
index 32aeacb..1fbf12d 100644
--- a/server/Cargo.lock
+++ b/server/Cargo.lock
@@ -919,7 +919,7 @@ dependencies = [
 [[package]]
 name = "domain"
 version = "0.1.0"
-source = "git+https://github.com/aukilabs/posemesh.git?branch=chore%2Freconstruction-integration#78d50d88988c5e731764efb6a73797edc5c2c1c6"
+source = "git+https://github.com/aukilabs/posemesh.git?branch=chore%2Freconstruction-integration#2bae6379625aea5a063c507b030ab9b68b3531d2"
 dependencies = [
  "async-trait",
  "cbindgen",
@@ -2753,7 +2753,7 @@ dependencies = [
 [[package]]
 name = "networking"
 version = "0.1.0"
-source = "git+https://github.com/aukilabs/posemesh.git?branch=chore%2Freconstruction-integration#78d50d88988c5e731764efb6a73797edc5c2c1c6"
+source = "git+https://github.com/aukilabs/posemesh.git?branch=chore%2Freconstruction-integration#2bae6379625aea5a063c507b030ab9b68b3531d2"
 dependencies = [
  "cbindgen",
  "console_error_panic_hook",
@@ -3503,7 +3503,7 @@ dependencies = [
 [[package]]
 name = "runtime"
 version = "0.1.0"
-source = "git+https://github.com/aukilabs/posemesh.git?branch=chore%2Freconstruction-integration#78d50d88988c5e731764efb6a73797edc5c2c1c6"
+source = "git+https://github.com/aukilabs/posemesh.git?branch=chore%2Freconstruction-integration#2bae6379625aea5a063c507b030ab9b68b3531d2"
 dependencies = [
  "once_cell",
  "tokio",

From 71752449839122214b62f16f43051aba698e0e0f Mon Sep 17 00:00:00 2001
From: Shuning <shuning@aukilabs.com>
Date: Fri, 14 Mar 2025 21:48:41 +0800
Subject: [PATCH 12/89] fix missing dependency

---
 server/Cargo.lock | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/server/Cargo.lock b/server/Cargo.lock
index 1fbf12d..a1b2811 100644
--- a/server/Cargo.lock
+++ b/server/Cargo.lock
@@ -919,7 +919,7 @@ dependencies = [
 [[package]]
 name = "domain"
 version = "0.1.0"
-source = "git+https://github.com/aukilabs/posemesh.git?branch=chore%2Freconstruction-integration#2bae6379625aea5a063c507b030ab9b68b3531d2"
+source = "git+https://github.com/aukilabs/posemesh.git?branch=chore%2Freconstruction-integration#1a2596262e1b82aeedf6d1a6ebbc895f25d88959"
 dependencies = [
  "async-trait",
  "cbindgen",
@@ -2753,7 +2753,7 @@ dependencies = [
 [[package]]
 name = "networking"
 version = "0.1.0"
-source = "git+https://github.com/aukilabs/posemesh.git?branch=chore%2Freconstruction-integration#2bae6379625aea5a063c507b030ab9b68b3531d2"
+source = "git+https://github.com/aukilabs/posemesh.git?branch=chore%2Freconstruction-integration#1a2596262e1b82aeedf6d1a6ebbc895f25d88959"
 dependencies = [
  "cbindgen",
  "console_error_panic_hook",
@@ -3503,7 +3503,7 @@ dependencies = [
 [[package]]
 name = "runtime"
 version = "0.1.0"
-source = "git+https://github.com/aukilabs/posemesh.git?branch=chore%2Freconstruction-integration#2bae6379625aea5a063c507b030ab9b68b3531d2"
+source = "git+https://github.com/aukilabs/posemesh.git?branch=chore%2Freconstruction-integration#1a2596262e1b82aeedf6d1a6ebbc895f25d88959"
 dependencies = [
  "once_cell",
  "tokio",

From 20e40bf9cd8b0ae337536d90f0becf516168698e Mon Sep 17 00:00:00 2001
From: Shuning <shuning@aukilabs.com>
Date: Fri, 14 Mar 2025 22:47:43 +0800
Subject: [PATCH 13/89] update tags

---
 .github/workflows/docker-build.yml | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml
index 0a821b1..e9f03a5 100644
--- a/.github/workflows/docker-build.yml
+++ b/.github/workflows/docker-build.yml
@@ -31,10 +31,21 @@ jobs:
           aws-access-key-id: ${{ secrets.GLOBAL_COMMON_AWS_ACCESS_KEY }}
           aws-secret-access-key: ${{ secrets.GLOBAL_COMMON_AWS_SECRET_ACCESS_KEY }}
           aws-region: ${{ vars.GLOBAL_COMMON_AWS_REGION }}
+      
       - name: Log in to AWS ECR
         id: ecr-login
         uses: aws-actions/amazon-ecr-login@v2
 
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: |
+            ${{ steps.ecr-login.outputs.registry }}
+          tags: |
+            type=ref,event=branch
+            type=sha,prefix=,format=long
+
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
 
@@ -45,8 +56,7 @@ jobs:
           file: docker/Dockerfile
           platforms: linux/amd64
           push: true
-          tags: ${{ steps.ecr-login.outputs.registry }}/reconstruction-server:new-sdk-integration-${{ github.sha }}
-          no-cache: true
+          tags: ${{ steps.meta.outputs.tags }}
 
       - name: Check Available Space
         run: df -h

From ccb921e53bcf14d96cfa9725de3ebe5f0e49f94b Mon Sep 17 00:00:00 2001
From: Shuning <shuning@aukilabs.com>
Date: Mon, 17 Mar 2025 11:54:14 +0800
Subject: [PATCH 14/89] update image

---
 .github/workflows/docker-build.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml
index e9f03a5..f793bd7 100644
--- a/.github/workflows/docker-build.yml
+++ b/.github/workflows/docker-build.yml
@@ -41,7 +41,7 @@ jobs:
         uses: docker/metadata-action@v5
         with:
           images: |
-            ${{ steps.ecr-login.outputs.registry }}
+            ${{ steps.ecr-login.outputs.registry }}/reconstruction-server
           tags: |
             type=ref,event=branch
             type=sha,prefix=,format=long

From 7f16972116e6c015c544ad0fae6d35d4adcd742d Mon Sep 17 00:00:00 2001
From: Shuning <shuning@aukilabs.com>
Date: Tue, 18 Mar 2025 13:27:46 +0800
Subject: [PATCH 15/89] update domain package

---
 server/Cargo.lock | 30 +++++++++++++++++++++++++++---
 1 file changed, 27 insertions(+), 3 deletions(-)

diff --git a/server/Cargo.lock b/server/Cargo.lock
index a1b2811..1ec3efe 100644
--- a/server/Cargo.lock
+++ b/server/Cargo.lock
@@ -919,7 +919,7 @@ dependencies = [
 [[package]]
 name = "domain"
 version = "0.1.0"
-source = "git+https://github.com/aukilabs/posemesh.git?branch=chore%2Freconstruction-integration#1a2596262e1b82aeedf6d1a6ebbc895f25d88959"
+source = "git+https://github.com/aukilabs/posemesh.git?branch=chore%2Freconstruction-integration#6c25ade9ae278090692394008f235f1b80db8ec6"
 dependencies = [
  "async-trait",
  "cbindgen",
@@ -1966,6 +1966,7 @@ dependencies = [
  "libp2p-autonat",
  "libp2p-connection-limits",
  "libp2p-core",
+ "libp2p-dcutr",
  "libp2p-dns",
  "libp2p-gossipsub",
  "libp2p-identify",
@@ -2062,6 +2063,28 @@ dependencies = [
  "web-time",
 ]
 
+[[package]]
+name = "libp2p-dcutr"
+version = "0.12.0"
+source = "git+https://github.com/aukilabs/rust-libp2p.git?branch=fix%2Fgossipsub-wasm#b6bd176a869cb1b760a5cac68bc67bd0ba571536"
+dependencies = [
+ "asynchronous-codec",
+ "either",
+ "futures",
+ "futures-bounded",
+ "futures-timer",
+ "libp2p-core",
+ "libp2p-identity",
+ "libp2p-swarm",
+ "lru",
+ "quick-protobuf",
+ "quick-protobuf-codec 0.3.1 (git+https://github.com/aukilabs/rust-libp2p.git?branch=fix%2Fgossipsub-wasm)",
+ "thiserror 1.0.69",
+ "tracing",
+ "void",
+ "web-time",
+]
+
 [[package]]
 name = "libp2p-dns"
 version = "0.42.0"
@@ -2206,6 +2229,7 @@ source = "git+https://github.com/aukilabs/rust-libp2p.git?branch=fix%2Fgossipsub
 dependencies = [
  "futures",
  "libp2p-core",
+ "libp2p-dcutr",
  "libp2p-gossipsub",
  "libp2p-identify",
  "libp2p-identity",
@@ -2753,7 +2777,7 @@ dependencies = [
 [[package]]
 name = "networking"
 version = "0.1.0"
-source = "git+https://github.com/aukilabs/posemesh.git?branch=chore%2Freconstruction-integration#1a2596262e1b82aeedf6d1a6ebbc895f25d88959"
+source = "git+https://github.com/aukilabs/posemesh.git?branch=chore%2Freconstruction-integration#6c25ade9ae278090692394008f235f1b80db8ec6"
 dependencies = [
  "cbindgen",
  "console_error_panic_hook",
@@ -3503,7 +3527,7 @@ dependencies = [
 [[package]]
 name = "runtime"
 version = "0.1.0"
-source = "git+https://github.com/aukilabs/posemesh.git?branch=chore%2Freconstruction-integration#1a2596262e1b82aeedf6d1a6ebbc895f25d88959"
+source = "git+https://github.com/aukilabs/posemesh.git?branch=chore%2Freconstruction-integration#6c25ade9ae278090692394008f235f1b80db8ec6"
 dependencies = [
  "once_cell",
  "tokio",

From 06618c7b1f3ad874f09ae7ff41f5acd20044be53 Mon Sep 17 00:00:00 2001
From: Shuning <shuning@aukilabs.com>
Date: Tue, 18 Mar 2025 16:39:49 +0800
Subject: [PATCH 16/89] fix empty scans parameter

---
 server/src/local_refinement.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/server/src/local_refinement.rs b/server/src/local_refinement.rs
index 06889c5..459bc27 100644
--- a/server/src/local_refinement.rs
+++ b/server/src/local_refinement.rs
@@ -165,7 +165,7 @@ pub(crate) async fn v1(base_path: String, mut stream: Stream, mut datastore: Box
         "--output", output_folder.to_str().unwrap(),
         "--domain_id", &claim.domain_id,
         "--job_id", &claim.job_id,
-        "--scans"
+        "--scans", &suffix.clone(),
     ];
     let child = Command::new("python3")
     .args(params)

From e2df524314dac5b65f3d89c3876d922175aa2a91 Mon Sep 17 00:00:00 2001
From: Shuning <shuning@aukilabs.com>
Date: Tue, 18 Mar 2025 17:04:52 +0800
Subject: [PATCH 17/89] fix rust

---
 server/src/local_refinement.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/server/src/local_refinement.rs b/server/src/local_refinement.rs
index 459bc27..5f4a819 100644
--- a/server/src/local_refinement.rs
+++ b/server/src/local_refinement.rs
@@ -81,7 +81,7 @@ pub(crate) async fn v1(base_path: String, mut stream: Stream, mut datastore: Box
     let task_folder = Path::new(&base_path).join(&claim.job_id);
     let scan_folder = Path::new(&task_folder).join("datasets");
     let input_folder = Path::new(&scan_folder).join(&suffix.clone());
-    let output_folder = Path::new(&task_folder).join("refined").join("local").join(&suffix);
+    let output_folder = Path::new(&task_folder).join("refined").join("local").join(&suffix.clone());
     fs::create_dir_all(&scan_folder).expect("Failed to create directory");
     fs::create_dir_all(&output_folder).expect("Failed to create directory");
     fs::create_dir_all(&input_folder).expect("Failed to create directory");
@@ -165,7 +165,7 @@ pub(crate) async fn v1(base_path: String, mut stream: Stream, mut datastore: Box
         "--output", output_folder.to_str().unwrap(),
         "--domain_id", &claim.domain_id,
         "--job_id", &claim.job_id,
-        "--scans", &suffix.clone(),
+        "--scans", suffix.as_str(),
     ];
     let child = Command::new("python3")
     .args(params)

From 92a0bcf6fd8c8dc7526de8e4754d6dfab391fbd1 Mon Sep 17 00:00:00 2001
From: Shuning <shuning@aukilabs.com>
Date: Wed, 19 Mar 2025 16:22:16 +0800
Subject: [PATCH 18/89] flush log and print to stdout by default

---
 main.py                        |  4 ++--
 server/src/local_refinement.rs | 36 +++++++++++++++++++---------------
 server/src/main.rs             |  2 ++
 utils/data_utils.py            |  8 ++++++--
 4 files changed, 30 insertions(+), 20 deletions(-)

diff --git a/main.py b/main.py
index 5c35a36..c544b66 100644
--- a/main.py
+++ b/main.py
@@ -1,6 +1,6 @@
 from pathlib import Path
 import argparse
-
+import sys
 from local_main import main as local_main
 from global_main import main as global_main
 from occlusion_box import main as occlusion_box_main
@@ -190,7 +190,7 @@ def main(args):
         process_refinement(args, logger)
     except Exception as e:
         handle_refinement_error(e, args, logger)
-        raise e
+        raise sys.exit(1)
 
 
 def parse_args():
diff --git a/server/src/local_refinement.rs b/server/src/local_refinement.rs
index 5f4a819..476bfda 100644
--- a/server/src/local_refinement.rs
+++ b/server/src/local_refinement.rs
@@ -206,33 +206,37 @@ pub(crate) async fn v1(base_path: String, mut stream: Stream, mut datastore: Box
         });
     }
 
-    // Wait for the command to complete
-    let status = child.wait();
-
-    match status {
+    // Wait for the process to complete
+    match child.wait() {
+        Ok(status) => {
+            if !status.success() {
+                eprintln!("Python process exited with status: {}", status);
+                t.status = task::Status::FAILED;
+                t.output = Some(task::Any {
+                    type_url: "Error".to_string(),
+                    value: serialize_into_vec(&task::Error {
+                        message: format!("Python process exited with status: {}", status),
+                    }).unwrap(),
+                });
+                let message = serialize_into_vec(t).expect("failed to serialize task update");
+                c.publish(job_id.clone(), message).await.expect("failed to publish task update");
+                return;
+            }
+            println!("Finished executing {}", claim.task_name);
+        }
         Err(e) => {
-            eprintln!("Failed to execute local refinement: {}", e);
+            eprintln!("Failed to wait for Python process: {}", e);
             t.status = task::Status::FAILED;
             t.output = Some(task::Any {
                 type_url: "Error".to_string(),
                 value: serialize_into_vec(&task::Error {
-                    message: e.to_string(),
+                    message: format!("Failed to wait for Python process: {}", e),
                 }).unwrap(),
             });
             let message = serialize_into_vec(t).expect("failed to serialize task update");
             c.publish(job_id.clone(), message).await.expect("failed to publish task update");
             return;
         }
-        Ok(exit_status) => {
-            if !exit_status.success() {
-                eprintln!("Failed to execute local refinement: {}", exit_status.code().unwrap_or(-1));
-                t.status = task::Status::FAILED;
-                let message = serialize_into_vec(t).expect("failed to serialize task update");
-                c.publish(job_id.clone(), message).await.expect("failed to publish task update");
-                return;
-            }
-            println!("Finished executing {}", claim.task_name);
-        }
     }
 
     let mut producer = datastore.produce(claim.domain_id.clone()).await;
diff --git a/server/src/main.rs b/server/src/main.rs
index 688e881..c586b22 100644
--- a/server/src/main.rs
+++ b/server/src/main.rs
@@ -61,5 +61,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
         }
     }
 
+    println!("Exit");
+
     Ok(())
 }
diff --git a/utils/data_utils.py b/utils/data_utils.py
index 71cd94e..ac50035 100644
--- a/utils/data_utils.py
+++ b/utils/data_utils.py
@@ -18,6 +18,7 @@
 from dateutil import parser
 from pathlib import Path
 from typing import NamedTuple, Dict
+import sys
 
 floor_rotation = pycolmap.Rotation3d(np.array([0, 0.7071068, 0, 0.7071068]))
 floor_rotation_inv = pycolmap.Rotation3d(np.array([0, -0.7071068, 0, 0.7071068]))
@@ -859,10 +860,13 @@ def setup_logger(name=None, log_file=None, domain_id="", job_id="", dataset_id=N
     if log_file:
         logger, _ = add_file_handler(logger, log_file)
 
-    console_handler = logging.StreamHandler()
+    print("Setting up JSON formatter")
+    console_handler = logging.StreamHandler(sys.stdout)
     console_handler.setFormatter(JsonFormatter(datefmt='%Y-%m-%dT%H:%M:%S',
         domain_id=domain_id, job_id=job_id, dataset_id=dataset_id))
     logger.addHandler(console_handler)
+    sys.stdout.reconfigure(line_buffering=True) 
+    logger.info("JSON formatter setup complete")
 
     return logger
 
@@ -1001,4 +1005,4 @@ def load_dataset_metadata(
         ar_poses_per_timestamp=ar_poses_per_timestamp,
         qr_detections_per_timestamp=qr_detections_per_timestamp,
         portal_sizes=portal_sizes 
-    )
\ No newline at end of file
+    )

From 9ac1c1748218b407e5cee1a2c961155e137fdbdd Mon Sep 17 00:00:00 2001
From: Shuning <shuning@aukilabs.com>
Date: Thu, 20 Mar 2025 13:49:48 +0800
Subject: [PATCH 19/89] update Dockerfile

---
 docker/Dockerfile              | 59 +++++++++++++---------------------
 server/src/local_refinement.rs |  2 +-
 2 files changed, 23 insertions(+), 38 deletions(-)

diff --git a/docker/Dockerfile b/docker/Dockerfile
index c266896..a320d63 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -1,43 +1,7 @@
-##
-## Rust build for server
-##
-FROM --platform=$BUILDPLATFORM rust:1.84-slim-bullseye AS rust-build
-ARG TARGETOS
-ARG TARGETARCH
-ARG VERSION
-
-# Install build dependencies
-RUN apt-get update && apt-get install -y \
-    python3 \
-    python3-pip \
-    python3-dev \
-    build-essential \
-    pkg-config \
-    git \
-    && rm -rf /var/lib/apt/lists/*
-
-# Install Python packages
-RUN pip3 install --no-cache-dir numpy scipy matplotlib
-
-WORKDIR /app
-COPY server /app/server
-
-# Build the application
-RUN cd server && cargo build --release --target-dir ../target
-
 ##
 ## Runtime image
 ##
-FROM --platform=$TARGETPLATFORM nvidia/cuda:11.0.3-base-ubuntu20.04
-
-ARG USERNAME=reconstruction-server
-ARG USER_UID=1000
-ARG USER_GID=$USER_UID
-ARG DEBIAN_FRONTEND=noninteractive
-
-# Set default environment variables
-ENV PORT=18802
-ENV DOMAIN_MANAGER_ADDR="/ip4/127.0.0.1/udp/18800/quic-v1/p2p/12D3KooWDHaDQeuYeLM8b5zhNjqS7Pkh7KefqzCpDGpdwj5iE8pq"
+FROM --platform=$BUILDPLATFORM nvidia/cuda:11.0.3-base-ubuntu20.04 AS deps
 
 RUN apt-get update && apt-get install -y \
     wget \
@@ -155,6 +119,27 @@ RUN python3 -m pip install enlighten evo
 
 RUN python3 -m pip install open3d trimesh alphashape
 
+##
+## Rust build for server
+##
+FROM --platform=$BUILDPLATFORM rust:1.84-slim-bullseye AS rust-build
+ARG TARGETOS
+ARG TARGETARCH
+ARG VERSION
+
+WORKDIR /app
+COPY server /app/server
+
+# Build the application
+RUN cd server && cargo build --release --target-dir ../target
+
+FROM deps AS server-build
+
+ARG USERNAME=reconstruction-server
+ARG USER_UID=1000
+ARG USER_GID=$USER_UID
+ARG DEBIAN_FRONTEND=noninteractive
+
 WORKDIR /app
 
 COPY scripts /app/scripts
diff --git a/server/src/local_refinement.rs b/server/src/local_refinement.rs
index 476bfda..0ddc767 100644
--- a/server/src/local_refinement.rs
+++ b/server/src/local_refinement.rs
@@ -143,7 +143,7 @@ pub(crate) async fn v1(base_path: String, mut stream: Stream, mut datastore: Box
                     .content)
                     .expect("Failed to write data to file");
                 i+=1;
-                println!("downloaded {}", filename);
+                println!("downloaded {} into {}", filename, path.to_str().unwrap());
             }
             Some(Err(_)) => {
                 t.status = task::Status::RETRY;

From ad56ff69c38a580955d4b57db689a8622cf08fdc Mon Sep 17 00:00:00 2001
From: Shuning <shuning@aukilabs.com>
Date: Thu, 20 Mar 2025 17:46:46 +0800
Subject: [PATCH 20/89] fix dockerfile

---
 docker/Dockerfile              |  8 +++-
 server/Cargo.lock              | 67 ++++++++++++++++++++++++++++++++--
 server/src/local_refinement.rs | 44 ++++++++++++++--------
 server/src/main.rs             |  2 +-
 4 files changed, 100 insertions(+), 21 deletions(-)

diff --git a/docker/Dockerfile b/docker/Dockerfile
index a320d63..5dc225a 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -127,11 +127,15 @@ ARG TARGETOS
 ARG TARGETARCH
 ARG VERSION
 
+RUN apt-get update && apt-get install -y \
+    git \
+    && rm -rf /var/lib/apt/lists/*
+
 WORKDIR /app
 COPY server /app/server
 
 # Build the application
-RUN cd server && cargo build --release --target-dir ../target
+RUN cd server && cargo build --release
 
 FROM deps AS server-build
 
@@ -165,4 +169,4 @@ RUN mkdir -p /app/volume && chown -R reconstruction-server:reconstruction-server
 USER $USERNAME
 
 # Copy the built binary from rust-build
-COPY --from=rust-build /app/target/release/server /app/server
+COPY --from=rust-build /app/server/target/release/server /app/server
diff --git a/server/Cargo.lock b/server/Cargo.lock
index 1ec3efe..968bd44 100644
--- a/server/Cargo.lock
+++ b/server/Cargo.lock
@@ -919,7 +919,7 @@ dependencies = [
 [[package]]
 name = "domain"
 version = "0.1.0"
-source = "git+https://github.com/aukilabs/posemesh.git?branch=chore%2Freconstruction-integration#6c25ade9ae278090692394008f235f1b80db8ec6"
+source = "git+https://github.com/aukilabs/posemesh.git?branch=chore%2Freconstruction-integration#2514e20eb33634337e51fb37b0bc79df8957a8a7"
 dependencies = [
  "async-trait",
  "cbindgen",
@@ -1980,6 +1980,7 @@ dependencies = [
  "libp2p-swarm",
  "libp2p-tcp",
  "libp2p-upnp",
+ "libp2p-websocket",
  "libp2p-yamux",
  "multiaddr",
  "pin-project",
@@ -2502,6 +2503,43 @@ dependencies = [
  "web-sys",
 ]
 
+[[package]]
+name = "libp2p-websocket"
+version = "0.44.0"
+source = "git+https://github.com/aukilabs/rust-libp2p.git?branch=fix%2Fgossipsub-wasm#b6bd176a869cb1b760a5cac68bc67bd0ba571536"
+dependencies = [
+ "either",
+ "futures",
+ "futures-rustls",
+ "libp2p-core",
+ "libp2p-identity",
+ "parking_lot",
+ "pin-project-lite",
+ "rw-stream-sink",
+ "soketto",
+ "thiserror 1.0.69",
+ "tracing",
+ "url",
+ "webpki-roots",
+]
+
+[[package]]
+name = "libp2p-websocket-websys"
+version = "0.4.0"
+source = "git+https://github.com/aukilabs/rust-libp2p.git?branch=fix%2Fgossipsub-wasm#b6bd176a869cb1b760a5cac68bc67bd0ba571536"
+dependencies = [
+ "bytes",
+ "futures",
+ "js-sys",
+ "libp2p-core",
+ "parking_lot",
+ "send_wrapper 0.6.0",
+ "thiserror 1.0.69",
+ "tracing",
+ "wasm-bindgen",
+ "web-sys",
+]
+
 [[package]]
 name = "libp2p-yamux"
 version = "0.46.0"
@@ -2777,7 +2815,7 @@ dependencies = [
 [[package]]
 name = "networking"
 version = "0.1.0"
-source = "git+https://github.com/aukilabs/posemesh.git?branch=chore%2Freconstruction-integration#6c25ade9ae278090692394008f235f1b80db8ec6"
+source = "git+https://github.com/aukilabs/posemesh.git?branch=chore%2Freconstruction-integration#2514e20eb33634337e51fb37b0bc79df8957a8a7"
 dependencies = [
  "cbindgen",
  "console_error_panic_hook",
@@ -2787,6 +2825,8 @@ dependencies = [
  "libp2p-stream",
  "libp2p-webrtc",
  "libp2p-webrtc-websys",
+ "libp2p-websocket",
+ "libp2p-websocket-websys",
  "rand 0.8.5",
  "runtime",
  "serde",
@@ -3527,7 +3567,7 @@ dependencies = [
 [[package]]
 name = "runtime"
 version = "0.1.0"
-source = "git+https://github.com/aukilabs/posemesh.git?branch=chore%2Freconstruction-integration#6c25ade9ae278090692394008f235f1b80db8ec6"
+source = "git+https://github.com/aukilabs/posemesh.git?branch=chore%2Freconstruction-integration#2514e20eb33634337e51fb37b0bc79df8957a8a7"
 dependencies = [
  "once_cell",
  "tokio",
@@ -3931,6 +3971,21 @@ dependencies = [
  "windows-sys 0.52.0",
 ]
 
+[[package]]
+name = "soketto"
+version = "0.8.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2e859df029d160cb88608f5d7df7fb4753fd20fdfb4de5644f3d8b8440841721"
+dependencies = [
+ "base64 0.22.1",
+ "bytes",
+ "futures",
+ "httparse",
+ "log",
+ "rand 0.8.5",
+ "sha1",
+]
+
 [[package]]
 name = "spin"
 version = "0.5.2"
@@ -4671,6 +4726,12 @@ dependencies = [
  "wasm-bindgen",
 ]
 
+[[package]]
+name = "webpki-roots"
+version = "0.25.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5f20c57d8d7db6d3b86154206ae5d8fba62dd39573114de97c2cb0578251f8e1"
+
 [[package]]
 name = "webrtc"
 version = "0.9.0"
diff --git a/server/src/local_refinement.rs b/server/src/local_refinement.rs
index 0ddc767..f38a2a5 100644
--- a/server/src/local_refinement.rs
+++ b/server/src/local_refinement.rs
@@ -81,7 +81,7 @@ pub(crate) async fn v1(base_path: String, mut stream: Stream, mut datastore: Box
     let task_folder = Path::new(&base_path).join(&claim.job_id);
     let scan_folder = Path::new(&task_folder).join("datasets");
     let input_folder = Path::new(&scan_folder).join(&suffix.clone());
-    let output_folder = Path::new(&task_folder).join("refined").join("local").join(&suffix.clone());
+    let output_folder = Path::new(&task_folder).join("refined").join("local");
     fs::create_dir_all(&scan_folder).expect("Failed to create directory");
     fs::create_dir_all(&output_folder).expect("Failed to create directory");
     fs::create_dir_all(&input_folder).expect("Failed to create directory");
@@ -122,20 +122,34 @@ pub(crate) async fn v1(base_path: String, mut stream: Stream, mut datastore: Box
     loop {
         match downloader.next().await {
             Some(Ok(data)) => {
-                let filename = match data.metadata.data_type.as_str() {
-                    "dmt_manifest_json" => "Manifest.json".to_string(),
-                    "dmt_featurepoints_ply" | "dmt_pointcloud_ply" => "FeaturePoints.ply".to_string(),
-                    "dmt_arposes_csv" => "ARposes.csv".to_string(),
-                    "dmt_portal_detections_csv" | "dmt_observations_csv" => "PortalDetections.csv".to_string(),
-                    "dmt_intrinsics_csv" | "dmt_cameraintrinsics_csv" => "CameraIntrinsics.csv".to_string(),
-                    "dmt_frames_csv" => "Frames.csv".to_string(),
-                    "dmt_gyro_csv" => "Gyro.csv".to_string(),
-                    "dmt_accel_csv" => "Accel.csv".to_string(),
-                    "dmt_gyroaccel_csv" => "gyro_accel.csv".to_string(),
-                    "dmt_recording_mp4" => "Frames.mp4".to_string(),
+                let filename = match data.metadata.name.as_str() {
+                    "Manifest.json" => "Manifest.json".to_string(),
+                    "FeaturePoints.ply" => "FeaturePoints.ply".to_string(),
+                    "ARposes.csv" => "ARposes.csv".to_string(),
+                    "PortalDetections.csv" => "PortalDetections.csv".to_string(),
+                    "CameraIntrinsics.csv" => "CameraIntrinsics.csv".to_string(),
+                    "Frames.csv" => "Frames.csv".to_string(),
+                    "Gyro.csv" => "Gyro.csv".to_string(),
+                    "Accel.csv" => "Accel.csv".to_string(),
+                    "gyro_accel.csv" => "gyro_accel.csv".to_string(),
+                    "Frames.mp4" => "Frames.mp4".to_string(),
                     _ => {
-                        println!("unknown domain data type: {}", data.metadata.data_type);
-                        format!("{}.{}", data.metadata.name, data.metadata.data_type)
+                        match data.metadata.data_type.as_str() {
+                            "dmt_manifest_json" => "Manifest.json".to_string(),
+                            "dmt_featurepoints_ply" | "dmt_pointcloud_ply" => "FeaturePoints.ply".to_string(),
+                            "dmt_arposes_csv" => "ARposes.csv".to_string(),
+                            "dmt_portal_detections_csv" | "dmt_observations_csv" => "PortalDetections.csv".to_string(),
+                            "dmt_intrinsics_csv" | "dmt_cameraintrinsics_csv" => "CameraIntrinsics.csv".to_string(),
+                            "dmt_frames_csv" => "Frames.csv".to_string(),
+                            "dmt_gyro_csv" => "Gyro.csv".to_string(),
+                            "dmt_accel_csv" => "Accel.csv".to_string(),
+                            "dmt_gyroaccel_csv" => "gyro_accel.csv".to_string(),
+                            "dmt_recording_mp4" => "Frames.mp4".to_string(),
+                            _ => {
+                                println!("unknown domain data type: {}", data.metadata.data_type);
+                                format!("{}.{}", data.metadata.name, data.metadata.data_type)
+                            }
+                        }
                     }
                 };
                 let path = input_folder.join(&filename);
@@ -143,7 +157,7 @@ pub(crate) async fn v1(base_path: String, mut stream: Stream, mut datastore: Box
                     .content)
                     .expect("Failed to write data to file");
                 i+=1;
-                println!("downloaded {} into {}", filename, path.to_str().unwrap());
+                println!("downloaded {}", filename);
             }
             Some(Err(_)) => {
                 t.status = task::Status::RETRY;
diff --git a/server/src/main.rs b/server/src/main.rs
index c586b22..7e6b6e7 100644
--- a/server/src/main.rs
+++ b/server/src/main.rs
@@ -39,7 +39,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     let private_key_path = format!("{}/pkey", base_path);
 
     let domain_manager_id = domain_manager.split("/").last().unwrap().to_string();
-    let domain_cluster = DomainCluster::new(domain_manager.clone(), name, false, port, None, Some(private_key_path));
+    let domain_cluster = DomainCluster::new(domain_manager.clone(), name, false, port, false, false, None, Some(private_key_path));
     let mut n = domain_cluster.peer.clone();
     let mut local_refinement_v1_handler = n.client.set_stream_handler("/local-refinement/v1".to_string()).await.unwrap();
     let mut global_refinement_v1_handler = n.client.set_stream_handler("/global-refinement/v1".to_string()).await.unwrap();

From 411414317abe51083cded5de7ddd5c5419ed306f Mon Sep 17 00:00:00 2001
From: Shuning <shuning@aukilabs.com>
Date: Fri, 21 Mar 2025 00:23:00 +0800
Subject: [PATCH 21/89] fix stuck docker

---
 docker/Dockerfile | 35 +++++++++++++++++++++++++----------
 1 file changed, 25 insertions(+), 10 deletions(-)

diff --git a/docker/Dockerfile b/docker/Dockerfile
index 5dc225a..bd0bb3f 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -3,26 +3,38 @@
 ##
 FROM --platform=$BUILDPLATFORM nvidia/cuda:11.0.3-base-ubuntu20.04 AS deps
 
-RUN apt-get update && apt-get install -y \
+# Prevent interactive prompts during package installation
+ENV DEBIAN_FRONTEND=noninteractive
+
+# Basic build tools
+RUN apt-get update && apt-get install -y --no-install-recommends \
     wget \
     curl \
-    vim \
     git \
-    nano \
-    jq \
-    less \
     cmake \
+    build-essential \
+    && rm -rf /var/lib/apt/lists/*
+
+# Development tools
+RUN apt-get update && apt-get install -y --no-install-recommends \
     autoconf \
     automake \
     libtool \
     libffi-dev \
     ninja-build \
-    build-essential \
+    && rm -rf /var/lib/apt/lists/*
+
+# Boost and Eigen dependencies
+RUN apt-get update && apt-get install -y --no-install-recommends \
     libboost-program-options-dev \
     libboost-filesystem-dev \
     libboost-graph-dev \
     libboost-system-dev \
     libeigen3-dev \
+    && rm -rf /var/lib/apt/lists/*
+
+# COLMAP dependencies
+RUN apt-get update && apt-get install -y --no-install-recommends \
     libflann-dev \
     libfreeimage-dev \
     libmetis-dev \
@@ -33,16 +45,19 @@ RUN apt-get update && apt-get install -y \
     libqt5opengl5-dev \
     libcgal-dev \
     libsuitesparse-dev \
-    python3-pip \
-    python3-tk
+    && rm -rf /var/lib/apt/lists/*
 
-RUN apt-get install -y --no-install-recommends \
+# Python and networking tools
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    python3-pip \
+    python3-tk \
     ssh \
     netcat-openbsd \
     ca-certificates \
     iproute2 \
     iputils-ping \
-    bind9-dnsutils
+    bind9-dnsutils \
+    && rm -rf /var/lib/apt/lists/*
 
 RUN pip install --upgrade pip setuptools wheel
 

From de500c2dc3d641d11533b9101181f279f83328f5 Mon Sep 17 00:00:00 2001
From: Shuning <shuning@aukilabs.com>
Date: Fri, 21 Mar 2025 09:00:53 +0800
Subject: [PATCH 22/89] fix dockerfile

---
 docker/Dockerfile | 33 ++++++++++-----------------------
 1 file changed, 10 insertions(+), 23 deletions(-)

diff --git a/docker/Dockerfile b/docker/Dockerfile
index bd0bb3f..7ec05a0 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -2,39 +2,27 @@
 ## Runtime image
 ##
 FROM --platform=$BUILDPLATFORM nvidia/cuda:11.0.3-base-ubuntu20.04 AS deps
-
-# Prevent interactive prompts during package installation
-ENV DEBIAN_FRONTEND=noninteractive
-
-# Basic build tools
-RUN apt-get update && apt-get install -y --no-install-recommends \
+ARG DEBIAN_FRONTEND=noninteractive
+RUN apt-get update && apt-get install -y \
     wget \
     curl \
+    vim \
     git \
+    nano \
+    jq \
+    less \
     cmake \
-    build-essential \
-    && rm -rf /var/lib/apt/lists/*
-
-# Development tools
-RUN apt-get update && apt-get install -y --no-install-recommends \
     autoconf \
     automake \
     libtool \
     libffi-dev \
     ninja-build \
-    && rm -rf /var/lib/apt/lists/*
-
-# Boost and Eigen dependencies
-RUN apt-get update && apt-get install -y --no-install-recommends \
+    build-essential \
     libboost-program-options-dev \
     libboost-filesystem-dev \
     libboost-graph-dev \
     libboost-system-dev \
     libeigen3-dev \
-    && rm -rf /var/lib/apt/lists/*
-
-# COLMAP dependencies
-RUN apt-get update && apt-get install -y --no-install-recommends \
     libflann-dev \
     libfreeimage-dev \
     libmetis-dev \
@@ -45,12 +33,11 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     libqt5opengl5-dev \
     libcgal-dev \
     libsuitesparse-dev \
-    && rm -rf /var/lib/apt/lists/*
-
-# Python and networking tools
-RUN apt-get update && apt-get install -y --no-install-recommends \
     python3-pip \
     python3-tk \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN apt-get install -y --no-install-recommends \
     ssh \
     netcat-openbsd \
     ca-certificates \

From b6e66364cfbb4fe93eb96445089387f04e43996c Mon Sep 17 00:00:00 2001
From: Shuning <shuning@aukilabs.com>
Date: Fri, 21 Mar 2025 09:16:22 +0800
Subject: [PATCH 23/89] ha?

---
 docker/Dockerfile | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/docker/Dockerfile b/docker/Dockerfile
index 7ec05a0..a5246af 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -1,8 +1,10 @@
 ##
 ## Runtime image
 ##
-FROM --platform=$BUILDPLATFORM nvidia/cuda:11.0.3-base-ubuntu20.04 AS deps
+FROM nvidia/cuda:11.0.3-base-ubuntu20.04 AS deps
+
 ARG DEBIAN_FRONTEND=noninteractive
+
 RUN apt-get update && apt-get install -y \
     wget \
     curl \
@@ -34,8 +36,7 @@ RUN apt-get update && apt-get install -y \
     libcgal-dev \
     libsuitesparse-dev \
     python3-pip \
-    python3-tk \
-    && rm -rf /var/lib/apt/lists/*
+    python3-tk
 
 RUN apt-get install -y --no-install-recommends \
     ssh \
@@ -43,8 +44,7 @@ RUN apt-get install -y --no-install-recommends \
     ca-certificates \
     iproute2 \
     iputils-ping \
-    bind9-dnsutils \
-    && rm -rf /var/lib/apt/lists/*
+    bind9-dnsutils
 
 RUN pip install --upgrade pip setuptools wheel
 

From 9558bc84d6623b886cbdc16c8db7e6d9e467cf08 Mon Sep 17 00:00:00 2001
From: Shuning <shuning@aukilabs.com>
Date: Fri, 21 Mar 2025 13:30:26 +0800
Subject: [PATCH 24/89] debug save_portal_csv

---
 utils/data_utils.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/utils/data_utils.py b/utils/data_utils.py
index ac50035..8e4035c 100644
--- a/utils/data_utils.py
+++ b/utils/data_utils.py
@@ -408,6 +408,7 @@ def save_qr_poses_csv(poses_per_qr, csv_path):
                 csv_writer.writerow(row)
 
 def save_portal_csv(poses_per_qr, csv_path, image_ids_per_qr, portal_sizes, corners_per_qr):
+    print(f"Saving portal CSV to {csv_path}")
     with open(csv_path, mode='w', newline='') as csvfile:
         csv_writer = csv.writer(csvfile)
 
@@ -433,6 +434,7 @@ def save_portal_csv(poses_per_qr, csv_path, image_ids_per_qr, portal_sizes, corn
                 row.extend(corner_array)
                 # Write the row to the CSV file
                 csv_writer.writerow(row)
+    print(f"Saved portal CSV to {csv_path}")
 
 def save_failed_manifest_json(json_path, job_root_path, job_status_details):
     save_manifest_json({}, json_path, job_root_path, job_status="failed", job_progress=100, job_status_details=job_status_details)

From 6a6262f72e1c560b5d761be82d44eba1d61e9575 Mon Sep 17 00:00:00 2001
From: Shuning <shuning@aukilabs.com>
Date: Fri, 21 Mar 2025 15:15:45 +0800
Subject: [PATCH 25/89] failed to save portal csv

---
 utils/data_utils.py | 17 ++++++++++-------
 1 file changed, 10 insertions(+), 7 deletions(-)

diff --git a/utils/data_utils.py b/utils/data_utils.py
index 8e4035c..cb62442 100644
--- a/utils/data_utils.py
+++ b/utils/data_utils.py
@@ -409,8 +409,9 @@ def save_qr_poses_csv(poses_per_qr, csv_path):
 
 def save_portal_csv(poses_per_qr, csv_path, image_ids_per_qr, portal_sizes, corners_per_qr):
     print(f"Saving portal CSV to {csv_path}")
-    with open(csv_path, mode='w', newline='') as csvfile:
-        csv_writer = csv.writer(csvfile)
+    try:
+        with open(csv_path, mode='w', newline='') as csvfile:
+            csv_writer = csv.writer(csvfile)
 
         for short_id, qr_poses in poses_per_qr.items():
 
@@ -429,12 +430,14 @@ def save_portal_csv(poses_per_qr, csv_path, image_ids_per_qr, portal_sizes, corn
                     portal_sizes[short_id],
                     pos[0], pos[1], pos[2],
                     quat[0], quat[1], quat[2], quat[3]
-                ]
+                    ]
 
-                row.extend(corner_array)
-                # Write the row to the CSV file
-                csv_writer.writerow(row)
-    print(f"Saved portal CSV to {csv_path}")
+                    row.extend(corner_array)
+                    # Write the row to the CSV file
+                    csv_writer.writerow(row)
+    except Exception as e:
+        print(f"Error saving portal CSV to {csv_path}")
+        print(e)
 
 def save_failed_manifest_json(json_path, job_root_path, job_status_details):
     save_manifest_json({}, json_path, job_root_path, job_status="failed", job_progress=100, job_status_details=job_status_details)

From 306567525a6b26f6b079a320764ab222155d166e Mon Sep 17 00:00:00 2001
From: Shuning <shuning@aukilabs.com>
Date: Fri, 21 Mar 2025 15:16:09 +0800
Subject: [PATCH 26/89] raise

---
 utils/data_utils.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/utils/data_utils.py b/utils/data_utils.py
index cb62442..3060d5d 100644
--- a/utils/data_utils.py
+++ b/utils/data_utils.py
@@ -438,6 +438,7 @@ def save_portal_csv(poses_per_qr, csv_path, image_ids_per_qr, portal_sizes, corn
     except Exception as e:
         print(f"Error saving portal CSV to {csv_path}")
         print(e)
+        raise e
 
 def save_failed_manifest_json(json_path, job_root_path, job_status_details):
     save_manifest_json({}, json_path, job_root_path, job_status="failed", job_progress=100, job_status_details=job_status_details)

From 97a55eb68f7565d338aa8f4c85b3398463eb5bdf Mon Sep 17 00:00:00 2001
From: Shuning <shuning@aukilabs.com>
Date: Fri, 21 Mar 2025 17:48:41 +0800
Subject: [PATCH 27/89] add debugger

---
 utils/data_utils.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/utils/data_utils.py b/utils/data_utils.py
index 3060d5d..2354007 100644
--- a/utils/data_utils.py
+++ b/utils/data_utils.py
@@ -413,14 +413,17 @@ def save_portal_csv(poses_per_qr, csv_path, image_ids_per_qr, portal_sizes, corn
         with open(csv_path, mode='w', newline='') as csvfile:
             csv_writer = csv.writer(csvfile)
 
+        print(f"Opening portal CSV file {csv_path}")
         for short_id, qr_poses in poses_per_qr.items():
 
             corresponding_image_ids = image_ids_per_qr[short_id]
             corresponding_corners = corners_per_qr[short_id]
 
+            print(f"Zipping portal {short_id} with images {corresponding_image_ids}")
             for image_id, qr_pose, qr_corners in zip(corresponding_image_ids, qr_poses, corresponding_corners):
                 pos, quat = qr_pose.translation, qr_pose.rotation.quat
                 corner_array = [coord for coords in qr_corners for coord in coords]
+                print(f"Corner array: {corner_array}")
                 # Create a row for the CSV
                 # Format 
                 # image_id, portal_id, portal_size, px, py, pz, qx, qy, qz, qw
@@ -434,7 +437,10 @@ def save_portal_csv(poses_per_qr, csv_path, image_ids_per_qr, portal_sizes, corn
 
                     row.extend(corner_array)
                     # Write the row to the CSV file
+                    print(f"Writing row to CSV: {row}")
                     csv_writer.writerow(row)
+                    print(f"Wrote row to CSV: {row}")
+        print(f"Saved portal CSV to {csv_path}")
     except Exception as e:
         print(f"Error saving portal CSV to {csv_path}")
         print(e)

From 0953e81248dbe473c9616d396781775d39ac92e1 Mon Sep 17 00:00:00 2001
From: Shuning <shuning@aukilabs.com>
Date: Mon, 24 Mar 2025 11:22:44 +0800
Subject: [PATCH 28/89] fix indent

---
 utils/data_utils.py | 54 ++++++++++++++++++++++-----------------------
 1 file changed, 27 insertions(+), 27 deletions(-)

diff --git a/utils/data_utils.py b/utils/data_utils.py
index 2354007..6388c0c 100644
--- a/utils/data_utils.py
+++ b/utils/data_utils.py
@@ -413,33 +413,33 @@ def save_portal_csv(poses_per_qr, csv_path, image_ids_per_qr, portal_sizes, corn
         with open(csv_path, mode='w', newline='') as csvfile:
             csv_writer = csv.writer(csvfile)
 
-        print(f"Opening portal CSV file {csv_path}")
-        for short_id, qr_poses in poses_per_qr.items():
-
-            corresponding_image_ids = image_ids_per_qr[short_id]
-            corresponding_corners = corners_per_qr[short_id]
-
-            print(f"Zipping portal {short_id} with images {corresponding_image_ids}")
-            for image_id, qr_pose, qr_corners in zip(corresponding_image_ids, qr_poses, corresponding_corners):
-                pos, quat = qr_pose.translation, qr_pose.rotation.quat
-                corner_array = [coord for coords in qr_corners for coord in coords]
-                print(f"Corner array: {corner_array}")
-                # Create a row for the CSV
-                # Format 
-                # image_id, portal_id, portal_size, px, py, pz, qx, qy, qz, qw
-                row = [
-                    image_id,
-                    short_id,
-                    portal_sizes[short_id],
-                    pos[0], pos[1], pos[2],
-                    quat[0], quat[1], quat[2], quat[3]
-                    ]
-
-                    row.extend(corner_array)
-                    # Write the row to the CSV file
-                    print(f"Writing row to CSV: {row}")
-                    csv_writer.writerow(row)
-                    print(f"Wrote row to CSV: {row}")
+            print(f"Opening portal CSV file {csv_path}")
+            for short_id, qr_poses in poses_per_qr.items():
+
+                corresponding_image_ids = image_ids_per_qr[short_id]
+                corresponding_corners = corners_per_qr[short_id]
+
+                print(f"Zipping portal {short_id} with images {corresponding_image_ids}")
+                for image_id, qr_pose, qr_corners in zip(corresponding_image_ids, qr_poses, corresponding_corners):
+                    pos, quat = qr_pose.translation, qr_pose.rotation.quat
+                    corner_array = [coord for coords in qr_corners for coord in coords]
+                    print(f"Corner array: {corner_array}")
+                    # Create a row for the CSV
+                    # Format 
+                    # image_id, portal_id, portal_size, px, py, pz, qx, qy, qz, qw
+                    row = [
+                        image_id,
+                        short_id,
+                        portal_sizes[short_id],
+                        pos[0], pos[1], pos[2],
+                        quat[0], quat[1], quat[2], quat[3]
+                        ]
+
+                        row.extend(corner_array)
+                        # Write the row to the CSV file
+                        print(f"Writing row to CSV: {row}")
+                        csv_writer.writerow(row)
+                        print(f"Wrote row to CSV: {row}")
         print(f"Saved portal CSV to {csv_path}")
     except Exception as e:
         print(f"Error saving portal CSV to {csv_path}")

From d652d8f00527ecc4d90ab006af3768f3fd1dc363 Mon Sep 17 00:00:00 2001
From: Shuning <shuning@aukilabs.com>
Date: Mon, 24 Mar 2025 14:05:36 +0800
Subject: [PATCH 29/89] fix indent

---
 utils/data_utils.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/utils/data_utils.py b/utils/data_utils.py
index 6388c0c..28cc541 100644
--- a/utils/data_utils.py
+++ b/utils/data_utils.py
@@ -435,11 +435,11 @@ def save_portal_csv(poses_per_qr, csv_path, image_ids_per_qr, portal_sizes, corn
                         quat[0], quat[1], quat[2], quat[3]
                         ]
 
-                        row.extend(corner_array)
-                        # Write the row to the CSV file
-                        print(f"Writing row to CSV: {row}")
-                        csv_writer.writerow(row)
-                        print(f"Wrote row to CSV: {row}")
+                    row.extend(corner_array)
+                    # Write the row to the CSV file
+                    print(f"Writing row to CSV: {row}")
+                    csv_writer.writerow(row)
+                    print(f"Wrote row to CSV: {row}")
         print(f"Saved portal CSV to {csv_path}")
     except Exception as e:
         print(f"Error saving portal CSV to {csv_path}")

From 72f2264f845191d3edf994576bea3d9158ba66e8 Mon Sep 17 00:00:00 2001
From: Shuning <shuning@aukilabs.com>
Date: Mon, 24 Mar 2025 17:22:03 +0800
Subject: [PATCH 30/89] add more debuggers

---
 utils/data_utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/utils/data_utils.py b/utils/data_utils.py
index 28cc541..09a2f19 100644
--- a/utils/data_utils.py
+++ b/utils/data_utils.py
@@ -434,7 +434,7 @@ def save_portal_csv(poses_per_qr, csv_path, image_ids_per_qr, portal_sizes, corn
                         pos[0], pos[1], pos[2],
                         quat[0], quat[1], quat[2], quat[3]
                         ]
-
+                    print(f"Extend row: {row}")
                     row.extend(corner_array)
                     # Write the row to the CSV file
                     print(f"Writing row to CSV: {row}")

From 32fbf8bd19eb56795bb84f39beb583cd967df2f9 Mon Sep 17 00:00:00 2001
From: Shuning <shuning@aukilabs.com>
Date: Mon, 24 Mar 2025 17:59:00 +0800
Subject: [PATCH 31/89] maybe portal_sizes is empty

---
 utils/data_utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/utils/data_utils.py b/utils/data_utils.py
index 09a2f19..3eb1619 100644
--- a/utils/data_utils.py
+++ b/utils/data_utils.py
@@ -423,7 +423,7 @@ def save_portal_csv(poses_per_qr, csv_path, image_ids_per_qr, portal_sizes, corn
                 for image_id, qr_pose, qr_corners in zip(corresponding_image_ids, qr_poses, corresponding_corners):
                     pos, quat = qr_pose.translation, qr_pose.rotation.quat
                     corner_array = [coord for coords in qr_corners for coord in coords]
-                    print(f"Corner array: {corner_array}")
+                    print(f"Corner array: {corner_array}, {portal_sizes}")
                     # Create a row for the CSV
                     # Format 
                     # image_id, portal_id, portal_size, px, py, pz, qx, qy, qz, qw

From 4c641b63f9c82d21f533779078fb2d22b82c94a5 Mon Sep 17 00:00:00 2001
From: Shuning <shuning@aukilabs.com>
Date: Tue, 25 Mar 2025 09:59:07 +0800
Subject: [PATCH 32/89] add missing portal sizes

---
 server/src/local_refinement.rs |  16 ++++-
 server/src/utils.rs            | 105 +++++++++++++++++++++++++++++++++
 2 files changed, 120 insertions(+), 1 deletion(-)

diff --git a/server/src/local_refinement.rs b/server/src/local_refinement.rs
index f38a2a5..2959c06 100644
--- a/server/src/local_refinement.rs
+++ b/server/src/local_refinement.rs
@@ -10,7 +10,7 @@ use std::io::{self, BufReader, BufWriter, Read, Write};
 use zip::{write::{FileOptions, SimpleFileOptions}, ZipWriter};
 use std::io::BufRead;
 
-use crate::utils::handshake;
+use crate::utils::{handshake, write_scan_data_summary};
 
 pub(crate) async fn v1(base_path: String, mut stream: Stream, mut datastore: Box<dyn Datastore>, mut c: Client) {
     let claim = handshake(&mut stream).await.expect("Failed to handshake");
@@ -172,6 +172,20 @@ pub(crate) async fn v1(base_path: String, mut stream: Stream, mut datastore: Box
     }
     println!("Finished downloading {} data for {}", i, claim.task_name);
 
+    if let Err(e) = write_scan_data_summary(scan_folder.as_path(), task_folder.as_path().join("scan_data_summary.json").as_path()) {
+        eprintln!("Failed to write scan data summary: {}", e);
+        t.status = task::Status::FAILED;
+        t.output = Some(task::Any {
+            type_url: "Error".to_string(),
+            value: serialize_into_vec(&task::Error {
+                message: format!("Failed to write scan data summary: {}", e),
+            }).unwrap(),
+        });
+        let message = serialize_into_vec(t).expect("failed to serialize task update");
+        c.publish(job_id.clone(), message).await.expect("failed to publish task update");
+        return;
+    }
+
     let params = vec![
         "main.py",
         "--mode", "local_refinement",
diff --git a/server/src/utils.rs b/server/src/utils.rs
index ef62641..ab83150 100644
--- a/server/src/utils.rs
+++ b/server/src/utils.rs
@@ -7,6 +7,11 @@ use tokio::{self, select, time::{sleep, Duration}};
 use futures::{AsyncReadExt, StreamExt};
 use uuid::Uuid;
 use serde::{Serialize, Deserialize};
+use std::fs::{self, DirEntry};
+use std::path::Path;
+use serde_json::{json, Value};
+use std::collections::HashSet;
+use std::error::Error;
 
 #[derive(Debug, Serialize, Deserialize)]
 pub struct TaskTokenClaim {
@@ -37,3 +42,103 @@ pub async fn handshake(stream: &mut Stream) -> Result<TaskTokenClaim, Box<dyn st
 
     decode_jwt(header.access_token.as_str())
 }
+
+pub fn write_scan_data_summary(
+    scan_folder:&Path,
+    summary_json_path: &Path,
+) -> Result<(), Box<dyn Error + Send + Sync>> {
+    let mut scan_count = 0;
+    let mut total_frame_count = 0;
+    let mut total_scan_duration = 0.0;
+    let mut scan_durations = Vec::new();
+    let mut unique_portal_ids = HashSet::new();
+    let mut portal_sizes = Vec::new();
+    let mut devices_used = HashSet::new();
+    let mut app_versions_used = HashSet::new();
+    
+    let manifest_path = scan_folder.join("Manifest.json");
+    if !manifest_path.exists() {
+        return Err("Manifest.json not found".into());
+    }
+
+    let manifest_data = fs::read_to_string(&manifest_path)?;
+    let manifest: Value = serde_json::from_str(&manifest_data)?;
+
+    scan_count += 1;
+
+    let frame_count = manifest["frameCount"].as_f64().unwrap_or(0.0) as i32;
+    let duration = manifest["duration"].as_f64().unwrap_or(0.0);
+    total_frame_count += frame_count;
+    total_scan_duration += duration;
+    scan_durations.push(duration);
+
+    if let Some(portals) = manifest.get("portals").and_then(|p| p.as_array()) {
+        for portal in portals {
+            if let Some(portal_map) = portal.as_object() {
+                if let Some(portal_id) = portal_map.get("shortId").and_then(|id| id.as_str()) {
+                    if unique_portal_ids.insert(portal_id.to_string()) {
+                        if let Some(size) = portal_map.get("physicalSize").and_then(|s| s.as_f64()) {
+                            portal_sizes.push(size);
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+    let device = if let (Some(brand), Some(model), Some(system_name), Some(system_version)) = (
+        manifest.get("brand").and_then(|b| b.as_str()),
+        manifest.get("model").and_then(|m| m.as_str()),
+        manifest.get("systemName").and_then(|s| s.as_str()),
+        manifest.get("systemVersion").and_then(|v| v.as_str()),
+    ) {
+        format!("{} {} {} {}", brand, model, system_name, system_version)
+    } else {
+        "unknown".to_string()
+    };
+    devices_used.insert(device);
+
+    let app_version = if let (Some(version), Some(build_id)) = (
+        manifest.get("appVersion").and_then(|v| v.as_str()),
+        manifest.get("buildId").and_then(|b| b.as_str()),
+    ) {
+        format!("{} (build {})", version, build_id)
+    } else {
+        "unknown".to_string()
+    };
+    app_versions_used.insert(app_version);
+    
+
+    scan_durations.sort_by(|a, b| a.partial_cmp(b).unwrap());
+    let shortest_scan_duration = scan_durations.first().unwrap_or(&0.0);
+    let longest_scan_duration = scan_durations.last().unwrap_or(&0.0);
+    let median_scan_duration = scan_durations.get(scan_durations.len() / 2).unwrap_or(&0.0);
+
+    let average_scan_duration = total_scan_duration / scan_count as f64;
+    let average_scan_frame_count = total_frame_count as f64 / scan_count as f64;
+    let average_scan_frame_rate = total_frame_count as f64 / total_scan_duration;
+
+    let summary = json!({
+        "scanCount": scan_count,
+        "totalFrameCount": total_frame_count,
+        "totalScanDuration": total_scan_duration,
+        "averageScanDuration": average_scan_duration,
+        "averageScanFrameCount": average_scan_frame_count,
+        "averageFrameRate": average_scan_frame_rate,
+        "shortestScanDuration": shortest_scan_duration,
+        "longestScanDuration": longest_scan_duration,
+        "medianScanDuration": median_scan_duration,
+        "portalCount": unique_portal_ids.len(),
+        "portalIDs": unique_portal_ids,
+        "portalSizes": portal_sizes,
+        "deviceVersionsUsed": devices_used,
+        "appVersionsUsed": app_versions_used,
+    });
+
+    fs::write(
+        summary_json_path,
+        serde_json::to_string_pretty(&summary)?,
+    )?;
+
+    Ok(())
+} 

From 5b661fd8906cf59fed63cb18999695d333138ed8 Mon Sep 17 00:00:00 2001
From: Shuning <shuning@aukilabs.com>
Date: Tue, 25 Mar 2025 13:34:55 +0800
Subject: [PATCH 33/89] fix Manifest location

---
 server/src/local_refinement.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/server/src/local_refinement.rs b/server/src/local_refinement.rs
index 2959c06..fcc281f 100644
--- a/server/src/local_refinement.rs
+++ b/server/src/local_refinement.rs
@@ -172,7 +172,7 @@ pub(crate) async fn v1(base_path: String, mut stream: Stream, mut datastore: Box
     }
     println!("Finished downloading {} data for {}", i, claim.task_name);
 
-    if let Err(e) = write_scan_data_summary(scan_folder.as_path(), task_folder.as_path().join("scan_data_summary.json").as_path()) {
+    if let Err(e) = write_scan_data_summary(input_folder.as_path(), task_folder.as_path().join("scan_data_summary.json").as_path()) {
         eprintln!("Failed to write scan data summary: {}", e);
         t.status = task::Status::FAILED;
         t.output = Some(task::Any {

From 1c9110f697df62b1ed84aae8639193635010de91 Mon Sep 17 00:00:00 2001
From: Shuning <shuning@aukilabs.com>
Date: Tue, 25 Mar 2025 19:21:09 +0800
Subject: [PATCH 34/89] fix paths

---
 server/Cargo.lock               |  65 ++++---
 server/Cargo.toml               |   4 +-
 server/src/global_refinement.rs |  97 ++++++++--
 server/src/local_refinement.rs  | 317 ++++++++++++++++----------------
 4 files changed, 269 insertions(+), 214 deletions(-)

diff --git a/server/Cargo.lock b/server/Cargo.lock
index 968bd44..f3653e9 100644
--- a/server/Cargo.lock
+++ b/server/Cargo.lock
@@ -919,7 +919,6 @@ dependencies = [
 [[package]]
 name = "domain"
 version = "0.1.0"
-source = "git+https://github.com/aukilabs/posemesh.git?branch=chore%2Freconstruction-integration#2514e20eb33634337e51fb37b0bc79df8957a8a7"
 dependencies = [
  "async-trait",
  "cbindgen",
@@ -1955,7 +1954,7 @@ checksum = "c19937216e9d3aa9956d9bb8dfc0b0c8beb6058fc4f7a4dc4d850edf86a237d6"
 [[package]]
 name = "libp2p"
 version = "0.54.1"
-source = "git+https://github.com/aukilabs/rust-libp2p.git?branch=fix%2Fgossipsub-wasm#b6bd176a869cb1b760a5cac68bc67bd0ba571536"
+source = "git+https://github.com/aukilabs/rust-libp2p.git?branch=fix%2Fgossipsub-wasm#9b97c007650c3a3e29cf90b896a159fc3e0e0b5b"
 dependencies = [
  "bytes",
  "either",
@@ -1991,7 +1990,7 @@ dependencies = [
 [[package]]
 name = "libp2p-allow-block-list"
 version = "0.4.1"
-source = "git+https://github.com/aukilabs/rust-libp2p.git?branch=fix%2Fgossipsub-wasm#b6bd176a869cb1b760a5cac68bc67bd0ba571536"
+source = "git+https://github.com/aukilabs/rust-libp2p.git?branch=fix%2Fgossipsub-wasm#9b97c007650c3a3e29cf90b896a159fc3e0e0b5b"
 dependencies = [
  "libp2p-core",
  "libp2p-identity",
@@ -2002,7 +2001,7 @@ dependencies = [
 [[package]]
 name = "libp2p-autonat"
 version = "0.13.1"
-source = "git+https://github.com/aukilabs/rust-libp2p.git?branch=fix%2Fgossipsub-wasm#b6bd176a869cb1b760a5cac68bc67bd0ba571536"
+source = "git+https://github.com/aukilabs/rust-libp2p.git?branch=fix%2Fgossipsub-wasm#9b97c007650c3a3e29cf90b896a159fc3e0e0b5b"
 dependencies = [
  "async-trait",
  "asynchronous-codec",
@@ -2028,7 +2027,7 @@ dependencies = [
 [[package]]
 name = "libp2p-connection-limits"
 version = "0.4.0"
-source = "git+https://github.com/aukilabs/rust-libp2p.git?branch=fix%2Fgossipsub-wasm#b6bd176a869cb1b760a5cac68bc67bd0ba571536"
+source = "git+https://github.com/aukilabs/rust-libp2p.git?branch=fix%2Fgossipsub-wasm#9b97c007650c3a3e29cf90b896a159fc3e0e0b5b"
 dependencies = [
  "libp2p-core",
  "libp2p-identity",
@@ -2039,7 +2038,7 @@ dependencies = [
 [[package]]
 name = "libp2p-core"
 version = "0.42.0"
-source = "git+https://github.com/aukilabs/rust-libp2p.git?branch=fix%2Fgossipsub-wasm#b6bd176a869cb1b760a5cac68bc67bd0ba571536"
+source = "git+https://github.com/aukilabs/rust-libp2p.git?branch=fix%2Fgossipsub-wasm#9b97c007650c3a3e29cf90b896a159fc3e0e0b5b"
 dependencies = [
  "either",
  "fnv",
@@ -2067,7 +2066,7 @@ dependencies = [
 [[package]]
 name = "libp2p-dcutr"
 version = "0.12.0"
-source = "git+https://github.com/aukilabs/rust-libp2p.git?branch=fix%2Fgossipsub-wasm#b6bd176a869cb1b760a5cac68bc67bd0ba571536"
+source = "git+https://github.com/aukilabs/rust-libp2p.git?branch=fix%2Fgossipsub-wasm#9b97c007650c3a3e29cf90b896a159fc3e0e0b5b"
 dependencies = [
  "asynchronous-codec",
  "either",
@@ -2089,7 +2088,7 @@ dependencies = [
 [[package]]
 name = "libp2p-dns"
 version = "0.42.0"
-source = "git+https://github.com/aukilabs/rust-libp2p.git?branch=fix%2Fgossipsub-wasm#b6bd176a869cb1b760a5cac68bc67bd0ba571536"
+source = "git+https://github.com/aukilabs/rust-libp2p.git?branch=fix%2Fgossipsub-wasm#9b97c007650c3a3e29cf90b896a159fc3e0e0b5b"
 dependencies = [
  "async-trait",
  "futures",
@@ -2104,7 +2103,7 @@ dependencies = [
 [[package]]
 name = "libp2p-gossipsub"
 version = "0.47.2"
-source = "git+https://github.com/aukilabs/rust-libp2p.git?branch=fix%2Fgossipsub-wasm#b6bd176a869cb1b760a5cac68bc67bd0ba571536"
+source = "git+https://github.com/aukilabs/rust-libp2p.git?branch=fix%2Fgossipsub-wasm#9b97c007650c3a3e29cf90b896a159fc3e0e0b5b"
 dependencies = [
  "asynchronous-codec",
  "base64 0.22.1",
@@ -2136,7 +2135,7 @@ dependencies = [
 [[package]]
 name = "libp2p-identify"
 version = "0.45.1"
-source = "git+https://github.com/aukilabs/rust-libp2p.git?branch=fix%2Fgossipsub-wasm#b6bd176a869cb1b760a5cac68bc67bd0ba571536"
+source = "git+https://github.com/aukilabs/rust-libp2p.git?branch=fix%2Fgossipsub-wasm#9b97c007650c3a3e29cf90b896a159fc3e0e0b5b"
 dependencies = [
  "asynchronous-codec",
  "either",
@@ -2177,7 +2176,7 @@ dependencies = [
 [[package]]
 name = "libp2p-kad"
 version = "0.47.0"
-source = "git+https://github.com/aukilabs/rust-libp2p.git?branch=fix%2Fgossipsub-wasm#b6bd176a869cb1b760a5cac68bc67bd0ba571536"
+source = "git+https://github.com/aukilabs/rust-libp2p.git?branch=fix%2Fgossipsub-wasm#9b97c007650c3a3e29cf90b896a159fc3e0e0b5b"
 dependencies = [
  "arrayvec",
  "asynchronous-codec",
@@ -2206,7 +2205,7 @@ dependencies = [
 [[package]]
 name = "libp2p-mdns"
 version = "0.46.0"
-source = "git+https://github.com/aukilabs/rust-libp2p.git?branch=fix%2Fgossipsub-wasm#b6bd176a869cb1b760a5cac68bc67bd0ba571536"
+source = "git+https://github.com/aukilabs/rust-libp2p.git?branch=fix%2Fgossipsub-wasm#9b97c007650c3a3e29cf90b896a159fc3e0e0b5b"
 dependencies = [
  "data-encoding",
  "futures",
@@ -2226,7 +2225,7 @@ dependencies = [
 [[package]]
 name = "libp2p-metrics"
 version = "0.15.0"
-source = "git+https://github.com/aukilabs/rust-libp2p.git?branch=fix%2Fgossipsub-wasm#b6bd176a869cb1b760a5cac68bc67bd0ba571536"
+source = "git+https://github.com/aukilabs/rust-libp2p.git?branch=fix%2Fgossipsub-wasm#9b97c007650c3a3e29cf90b896a159fc3e0e0b5b"
 dependencies = [
  "futures",
  "libp2p-core",
@@ -2245,7 +2244,7 @@ dependencies = [
 [[package]]
 name = "libp2p-noise"
 version = "0.45.0"
-source = "git+https://github.com/aukilabs/rust-libp2p.git?branch=fix%2Fgossipsub-wasm#b6bd176a869cb1b760a5cac68bc67bd0ba571536"
+source = "git+https://github.com/aukilabs/rust-libp2p.git?branch=fix%2Fgossipsub-wasm#9b97c007650c3a3e29cf90b896a159fc3e0e0b5b"
 dependencies = [
  "asynchronous-codec",
  "bytes",
@@ -2270,7 +2269,7 @@ dependencies = [
 [[package]]
 name = "libp2p-quic"
 version = "0.11.1"
-source = "git+https://github.com/aukilabs/rust-libp2p.git?branch=fix%2Fgossipsub-wasm#b6bd176a869cb1b760a5cac68bc67bd0ba571536"
+source = "git+https://github.com/aukilabs/rust-libp2p.git?branch=fix%2Fgossipsub-wasm#9b97c007650c3a3e29cf90b896a159fc3e0e0b5b"
 dependencies = [
  "bytes",
  "futures",
@@ -2293,7 +2292,7 @@ dependencies = [
 [[package]]
 name = "libp2p-relay"
 version = "0.18.0"
-source = "git+https://github.com/aukilabs/rust-libp2p.git?branch=fix%2Fgossipsub-wasm#b6bd176a869cb1b760a5cac68bc67bd0ba571536"
+source = "git+https://github.com/aukilabs/rust-libp2p.git?branch=fix%2Fgossipsub-wasm#9b97c007650c3a3e29cf90b896a159fc3e0e0b5b"
 dependencies = [
  "asynchronous-codec",
  "bytes",
@@ -2317,7 +2316,7 @@ dependencies = [
 [[package]]
 name = "libp2p-request-response"
 version = "0.27.0"
-source = "git+https://github.com/aukilabs/rust-libp2p.git?branch=fix%2Fgossipsub-wasm#b6bd176a869cb1b760a5cac68bc67bd0ba571536"
+source = "git+https://github.com/aukilabs/rust-libp2p.git?branch=fix%2Fgossipsub-wasm#9b97c007650c3a3e29cf90b896a159fc3e0e0b5b"
 dependencies = [
  "async-trait",
  "futures",
@@ -2336,7 +2335,7 @@ dependencies = [
 [[package]]
 name = "libp2p-stream"
 version = "0.2.0-alpha"
-source = "git+https://github.com/aukilabs/rust-libp2p.git?branch=fix%2Fgossipsub-wasm#b6bd176a869cb1b760a5cac68bc67bd0ba571536"
+source = "git+https://github.com/aukilabs/rust-libp2p.git?branch=fix%2Fgossipsub-wasm#9b97c007650c3a3e29cf90b896a159fc3e0e0b5b"
 dependencies = [
  "futures",
  "libp2p-core",
@@ -2350,7 +2349,7 @@ dependencies = [
 [[package]]
 name = "libp2p-swarm"
 version = "0.45.2"
-source = "git+https://github.com/aukilabs/rust-libp2p.git?branch=fix%2Fgossipsub-wasm#b6bd176a869cb1b760a5cac68bc67bd0ba571536"
+source = "git+https://github.com/aukilabs/rust-libp2p.git?branch=fix%2Fgossipsub-wasm#9b97c007650c3a3e29cf90b896a159fc3e0e0b5b"
 dependencies = [
  "either",
  "fnv",
@@ -2375,7 +2374,7 @@ dependencies = [
 [[package]]
 name = "libp2p-swarm-derive"
 version = "0.35.0"
-source = "git+https://github.com/aukilabs/rust-libp2p.git?branch=fix%2Fgossipsub-wasm#b6bd176a869cb1b760a5cac68bc67bd0ba571536"
+source = "git+https://github.com/aukilabs/rust-libp2p.git?branch=fix%2Fgossipsub-wasm#9b97c007650c3a3e29cf90b896a159fc3e0e0b5b"
 dependencies = [
  "heck 0.5.0",
  "proc-macro2",
@@ -2386,7 +2385,7 @@ dependencies = [
 [[package]]
 name = "libp2p-tcp"
 version = "0.42.0"
-source = "git+https://github.com/aukilabs/rust-libp2p.git?branch=fix%2Fgossipsub-wasm#b6bd176a869cb1b760a5cac68bc67bd0ba571536"
+source = "git+https://github.com/aukilabs/rust-libp2p.git?branch=fix%2Fgossipsub-wasm#9b97c007650c3a3e29cf90b896a159fc3e0e0b5b"
 dependencies = [
  "futures",
  "futures-timer",
@@ -2402,7 +2401,7 @@ dependencies = [
 [[package]]
 name = "libp2p-tls"
 version = "0.5.0"
-source = "git+https://github.com/aukilabs/rust-libp2p.git?branch=fix%2Fgossipsub-wasm#b6bd176a869cb1b760a5cac68bc67bd0ba571536"
+source = "git+https://github.com/aukilabs/rust-libp2p.git?branch=fix%2Fgossipsub-wasm#9b97c007650c3a3e29cf90b896a159fc3e0e0b5b"
 dependencies = [
  "futures",
  "futures-rustls",
@@ -2420,7 +2419,7 @@ dependencies = [
 [[package]]
 name = "libp2p-upnp"
 version = "0.3.1"
-source = "git+https://github.com/aukilabs/rust-libp2p.git?branch=fix%2Fgossipsub-wasm#b6bd176a869cb1b760a5cac68bc67bd0ba571536"
+source = "git+https://github.com/aukilabs/rust-libp2p.git?branch=fix%2Fgossipsub-wasm#9b97c007650c3a3e29cf90b896a159fc3e0e0b5b"
 dependencies = [
  "futures",
  "futures-timer",
@@ -2435,7 +2434,7 @@ dependencies = [
 [[package]]
 name = "libp2p-webrtc"
 version = "0.8.0-alpha"
-source = "git+https://github.com/aukilabs/rust-libp2p.git?branch=fix%2Fgossipsub-wasm#b6bd176a869cb1b760a5cac68bc67bd0ba571536"
+source = "git+https://github.com/aukilabs/rust-libp2p.git?branch=fix%2Fgossipsub-wasm#9b97c007650c3a3e29cf90b896a159fc3e0e0b5b"
 dependencies = [
  "async-trait",
  "bytes",
@@ -2463,7 +2462,7 @@ dependencies = [
 [[package]]
 name = "libp2p-webrtc-utils"
 version = "0.3.0"
-source = "git+https://github.com/aukilabs/rust-libp2p.git?branch=fix%2Fgossipsub-wasm#b6bd176a869cb1b760a5cac68bc67bd0ba571536"
+source = "git+https://github.com/aukilabs/rust-libp2p.git?branch=fix%2Fgossipsub-wasm#9b97c007650c3a3e29cf90b896a159fc3e0e0b5b"
 dependencies = [
  "asynchronous-codec",
  "bytes",
@@ -2485,7 +2484,7 @@ dependencies = [
 [[package]]
 name = "libp2p-webrtc-websys"
 version = "0.4.0-alpha.2"
-source = "git+https://github.com/aukilabs/rust-libp2p.git?branch=fix%2Fgossipsub-wasm#b6bd176a869cb1b760a5cac68bc67bd0ba571536"
+source = "git+https://github.com/aukilabs/rust-libp2p.git?branch=fix%2Fgossipsub-wasm#9b97c007650c3a3e29cf90b896a159fc3e0e0b5b"
 dependencies = [
  "bytes",
  "futures",
@@ -2506,7 +2505,7 @@ dependencies = [
 [[package]]
 name = "libp2p-websocket"
 version = "0.44.0"
-source = "git+https://github.com/aukilabs/rust-libp2p.git?branch=fix%2Fgossipsub-wasm#b6bd176a869cb1b760a5cac68bc67bd0ba571536"
+source = "git+https://github.com/aukilabs/rust-libp2p.git?branch=fix%2Fgossipsub-wasm#9b97c007650c3a3e29cf90b896a159fc3e0e0b5b"
 dependencies = [
  "either",
  "futures",
@@ -2526,7 +2525,7 @@ dependencies = [
 [[package]]
 name = "libp2p-websocket-websys"
 version = "0.4.0"
-source = "git+https://github.com/aukilabs/rust-libp2p.git?branch=fix%2Fgossipsub-wasm#b6bd176a869cb1b760a5cac68bc67bd0ba571536"
+source = "git+https://github.com/aukilabs/rust-libp2p.git?branch=fix%2Fgossipsub-wasm#9b97c007650c3a3e29cf90b896a159fc3e0e0b5b"
 dependencies = [
  "bytes",
  "futures",
@@ -2543,7 +2542,7 @@ dependencies = [
 [[package]]
 name = "libp2p-yamux"
 version = "0.46.0"
-source = "git+https://github.com/aukilabs/rust-libp2p.git?branch=fix%2Fgossipsub-wasm#b6bd176a869cb1b760a5cac68bc67bd0ba571536"
+source = "git+https://github.com/aukilabs/rust-libp2p.git?branch=fix%2Fgossipsub-wasm#9b97c007650c3a3e29cf90b896a159fc3e0e0b5b"
 dependencies = [
  "either",
  "futures",
@@ -2738,7 +2737,7 @@ dependencies = [
 [[package]]
 name = "multistream-select"
 version = "0.13.0"
-source = "git+https://github.com/aukilabs/rust-libp2p.git?branch=fix%2Fgossipsub-wasm#b6bd176a869cb1b760a5cac68bc67bd0ba571536"
+source = "git+https://github.com/aukilabs/rust-libp2p.git?branch=fix%2Fgossipsub-wasm#9b97c007650c3a3e29cf90b896a159fc3e0e0b5b"
 dependencies = [
  "bytes",
  "futures",
@@ -2815,7 +2814,6 @@ dependencies = [
 [[package]]
 name = "networking"
 version = "0.1.0"
-source = "git+https://github.com/aukilabs/posemesh.git?branch=chore%2Freconstruction-integration#2514e20eb33634337e51fb37b0bc79df8957a8a7"
 dependencies = [
  "cbindgen",
  "console_error_panic_hook",
@@ -3253,7 +3251,7 @@ dependencies = [
 [[package]]
 name = "quick-protobuf-codec"
 version = "0.3.1"
-source = "git+https://github.com/aukilabs/rust-libp2p.git?branch=fix%2Fgossipsub-wasm#b6bd176a869cb1b760a5cac68bc67bd0ba571536"
+source = "git+https://github.com/aukilabs/rust-libp2p.git?branch=fix%2Fgossipsub-wasm#9b97c007650c3a3e29cf90b896a159fc3e0e0b5b"
 dependencies = [
  "asynchronous-codec",
  "bytes",
@@ -3567,7 +3565,6 @@ dependencies = [
 [[package]]
 name = "runtime"
 version = "0.1.0"
-source = "git+https://github.com/aukilabs/posemesh.git?branch=chore%2Freconstruction-integration#2514e20eb33634337e51fb37b0bc79df8957a8a7"
 dependencies = [
  "once_cell",
  "tokio",
@@ -3694,7 +3691,7 @@ checksum = "eded382c5f5f786b989652c49544c4877d9f015cc22e145a5ea8ea66c2921cd2"
 [[package]]
 name = "rw-stream-sink"
 version = "0.4.0"
-source = "git+https://github.com/aukilabs/rust-libp2p.git?branch=fix%2Fgossipsub-wasm#b6bd176a869cb1b760a5cac68bc67bd0ba571536"
+source = "git+https://github.com/aukilabs/rust-libp2p.git?branch=fix%2Fgossipsub-wasm#9b97c007650c3a3e29cf90b896a159fc3e0e0b5b"
 dependencies = [
  "futures",
  "pin-project",
diff --git a/server/Cargo.toml b/server/Cargo.toml
index e680233..7a1668e 100644
--- a/server/Cargo.toml
+++ b/server/Cargo.toml
@@ -5,12 +5,12 @@ edition = "2021"
 
 [dependencies]
 libp2p = { git = "https://github.com/aukilabs/rust-libp2p.git", branch = "fix/gossipsub-wasm" }
-networking = { git = "https://github.com/aukilabs/posemesh.git", branch = "chore/reconstruction-integration" }
+networking = { path = "../../posemesh/core/networking" }
 futures = "0.3.30"
 jsonwebtoken = "9.3.0"
 uuid = "1.13.2"
 rand = "0.9.0"
-domain = { git = "https://github.com/aukilabs/posemesh.git", branch = "chore/reconstruction-integration" }
+domain = { path = "../../posemesh/core/domain" }
 tracing = "0.1.40"
 tracing-subscriber = "0.3.18"
 tokio = "1.44.0"
diff --git a/server/src/global_refinement.rs b/server/src/global_refinement.rs
index 19d3f69..aa29b43 100644
--- a/server/src/global_refinement.rs
+++ b/server/src/global_refinement.rs
@@ -1,5 +1,4 @@
-use std::{collections::HashMap, fs::{self, File}, io::Cursor, path::{Path, PathBuf}, process::Command, time::SystemTime};
-
+use std::{collections::HashMap, fs::{self, File}, io::{self, BufReader, BufWriter, Read, Write, BufRead, Cursor}, path::{Path, PathBuf}, process::{Command, Stdio}, time::SystemTime};
 use chrono::Utc;
 use domain::{cluster::DomainCluster, datastore::{self, common::Datastore, remote::RemoteDatastore}, message::read_prefix_size_message, protobuf::{domain_data::{Data, Metadata, Query},task::{self, LocalRefinementInputV1, LocalRefinementOutputV1}}};
 use jsonwebtoken::{decode, DecodingKey,Validation, Algorithm};
@@ -16,17 +15,29 @@ use zip::ZipArchive;
 use crate::utils::handshake;
 
 fn unzip_bytes(path: PathBuf, zip_bytes: Vec<u8>) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    println!("Starting to unzip bytes to path: {:?}", path);
     let cursor = Cursor::new(zip_bytes);
     let mut archive = ZipArchive::new(cursor)?;
+    println!("Zip archive opened, contains {} files", archive.len());
     
     for i in 0..archive.len() {
+        println!("Processing file {}/{}", i + 1, archive.len());
         let mut input_file = archive.by_index(i)?;
         let file_name = input_file.name().to_string();
-        let path = path.join(file_name);
-        let mut output_file: File = File::create(path)?;
-        std::io::copy(&mut input_file, &mut output_file)?;
-
+        println!("Extracting file: {}", file_name);
+        
+        let file_path = path.join(&file_name);
+        
+        // Create parent directory if it doesn't exist
+        if let Some(parent) = file_path.parent() {
+            fs::create_dir_all(parent)?;
+        }
+        
+        let mut output_file: File = File::create(&file_path)?;
+        let bytes_copied = std::io::copy(&mut input_file, &mut output_file)?;
+        println!("Extracted {} bytes to {:?}", bytes_copied, file_path);
     }
+    println!("Finished unzipping all files");
     Ok(())
 }
 
@@ -36,11 +47,11 @@ pub(crate) async fn v1(base_path: String, mut stream: Stream, mut datastore: Box
     c.subscribe(job_id.clone()).await.expect("Failed to subscribe to job");
     let t = &mut task::Task {
         name: claim.task_name.clone(),
-        receiver: claim.receiver.clone(),
+        receiver: Some(claim.receiver.clone()),
         sender: claim.sender.clone(),
         endpoint: "/global-refinement/v1".to_string(),
         status: task::Status::STARTED,
-        access_token: "".to_string(),
+        access_token: None,
         job_id: job_id.clone(),
         output: None,
     };
@@ -155,24 +166,76 @@ pub(crate) async fn v1(base_path: String, mut stream: Stream, mut datastore: Box
         "--job_id", &claim.job_id,
         "--scans"
     ];
-    let output = Command::new("python3")
-        .args(params)
-        .output();
-    match output {
+    let child = Command::new("python3")
+    .args(params)
+    .stdout(Stdio::piped())
+    .stderr(Stdio::piped())
+    .spawn();
+
+    if let Err(e) = child {
+        eprintln!("Failed to execute global refinement: {}", e);
+        t.status = task::Status::FAILED;
+        let message = serialize_into_vec(t).expect("failed to serialize task update");
+        c.publish(job_id.clone(), message).await.expect("failed to publish task update");
+        return;
+    }
+    let mut child = child.unwrap();
+
+    // Read stdout in real-time
+    if let Some(stdout) = child.stdout.take() {
+        let stdout_reader = BufReader::new(stdout);
+        tokio::spawn(async move {
+            for line in stdout_reader.lines() {
+                if let Ok(line) = line {
+                    println!("stdout: {}", line);
+                }
+            }
+        });
+    }
+
+    // Read stderr in real-time
+    if let Some(stderr) = child.stderr.take() {
+        let stderr_reader = BufReader::new(stderr);
+        tokio::spawn(async move {
+            for line in stderr_reader.lines() {
+                if let Ok(line) = line {
+                    eprintln!("stderr: {}", line);
+                }
+            }
+        });
+    }
+
+    // Wait for the process to complete
+    match child.wait() {
+        Ok(status) => {
+            if !status.success() {
+                eprintln!("Python process exited with status: {}", status);
+                t.status = task::Status::FAILED;
+                t.output = Some(task::Any {
+                    type_url: "Error".to_string(),
+                    value: serialize_into_vec(&task::Error {
+                        message: format!("Python process exited with status: {}", status),
+                    }).unwrap(),
+                });
+                let message = serialize_into_vec(t).expect("failed to serialize task update");
+                c.publish(job_id.clone(), message).await.expect("failed to publish task update");
+                return;
+            }
+            println!("Finished executing {}", claim.task_name);
+        }
         Err(e) => {
-            eprintln!("Failed to execute global refinement: {}", e);
+            eprintln!("Failed to wait for Python process: {}", e);
             t.status = task::Status::FAILED;
             t.output = Some(task::Any {
                 type_url: "Error".to_string(),
                 value: serialize_into_vec(&task::Error {
-                    message: e.to_string(),
+                    message: format!("Failed to wait for Python process: {}", e),
                 }).unwrap(),
             });
             let message = serialize_into_vec(t).expect("failed to serialize task update");
             c.publish(job_id.clone(), message).await.expect("failed to publish task update");
             return;
         }
-        Ok(_) => println!("Finished executing {}", claim.task_name)
     }
 
     let mut uploader = datastore.produce(claim.domain_id.clone()).await;
@@ -220,11 +283,11 @@ pub(crate) async fn v1(base_path: String, mut stream: Stream, mut datastore: Box
 
     let event = task::Task {
         name: claim.task_name.clone(),
-        receiver: claim.receiver.clone(),
+        receiver: Some(claim.receiver.clone()),
         sender: claim.sender.clone(),
         endpoint: "/global-refinement/v1".to_string(),
         status: task::Status::DONE,
-        access_token: "".to_string(),
+        access_token: None,
         job_id: job_id.clone(),
         output: None,
     };
diff --git a/server/src/local_refinement.rs b/server/src/local_refinement.rs
index fcc281f..eb2af3e 100644
--- a/server/src/local_refinement.rs
+++ b/server/src/local_refinement.rs
@@ -18,11 +18,11 @@ pub(crate) async fn v1(base_path: String, mut stream: Stream, mut datastore: Box
     c.subscribe(job_id.clone()).await.expect("Failed to subscribe to job");
     let t = &mut task::Task {
         name: claim.task_name.clone(),
-        receiver: claim.receiver.clone(),
+        receiver: Some(claim.receiver.clone()),
         sender: claim.sender.clone(),
         endpoint: "/local-refinement/v1".to_string(),
         status: task::Status::STARTED,
-        access_token: "".to_string(),
+        access_token: None,
         job_id: job_id.clone(),
         output: None,
     };
@@ -41,17 +41,7 @@ pub(crate) async fn v1(base_path: String, mut stream: Stream, mut datastore: Box
 
     // input.name_regexp looks .*_date
     // get date from regexp
-    if input.query.is_none() {
-        t.status = task::Status::FAILED;
-        t.output = Some(task::Any {
-            type_url: "Error".to_string(),
-            value: "Query is empty".as_bytes().to_vec(),
-        });
-        let message = serialize_into_vec(t).expect("failed to serialize task update");
-        c.publish(job_id.clone(), message).await.expect("failed to publish task update");
-        return;
-    }
-    let query = input.query.clone().unwrap();
+    let query = input.query.clone();
     let query_clone = query.clone();
     let name_regexp = query.name_regexp;
     if name_regexp.is_none() {
@@ -113,170 +103,174 @@ pub(crate) async fn v1(base_path: String, mut stream: Stream, mut datastore: Box
         }
     });
 
-    let _cleanup = scopeguard::guard(tx, |tx| {
+    let _cleanup = scopeguard::guard(tx.clone(), |tx| {
         let _ = tx.send(true); // Signal heartbeat task to stop
     });
 
-    let mut downloader = datastore.consume("".to_string(), query_clone, false).await;
-    let mut i = 0;
-    loop {
-        match downloader.next().await {
-            Some(Ok(data)) => {
-                let filename = match data.metadata.name.as_str() {
-                    "Manifest.json" => "Manifest.json".to_string(),
-                    "FeaturePoints.ply" => "FeaturePoints.ply".to_string(),
-                    "ARposes.csv" => "ARposes.csv".to_string(),
-                    "PortalDetections.csv" => "PortalDetections.csv".to_string(),
-                    "CameraIntrinsics.csv" => "CameraIntrinsics.csv".to_string(),
-                    "Frames.csv" => "Frames.csv".to_string(),
-                    "Gyro.csv" => "Gyro.csv".to_string(),
-                    "Accel.csv" => "Accel.csv".to_string(),
-                    "gyro_accel.csv" => "gyro_accel.csv".to_string(),
-                    "Frames.mp4" => "Frames.mp4".to_string(),
-                    _ => {
-                        match data.metadata.data_type.as_str() {
-                            "dmt_manifest_json" => "Manifest.json".to_string(),
-                            "dmt_featurepoints_ply" | "dmt_pointcloud_ply" => "FeaturePoints.ply".to_string(),
-                            "dmt_arposes_csv" => "ARposes.csv".to_string(),
-                            "dmt_portal_detections_csv" | "dmt_observations_csv" => "PortalDetections.csv".to_string(),
-                            "dmt_intrinsics_csv" | "dmt_cameraintrinsics_csv" => "CameraIntrinsics.csv".to_string(),
-                            "dmt_frames_csv" => "Frames.csv".to_string(),
-                            "dmt_gyro_csv" => "Gyro.csv".to_string(),
-                            "dmt_accel_csv" => "Accel.csv".to_string(),
-                            "dmt_gyroaccel_csv" => "gyro_accel.csv".to_string(),
-                            "dmt_recording_mp4" => "Frames.mp4".to_string(),
-                            _ => {
-                                println!("unknown domain data type: {}", data.metadata.data_type);
-                                format!("{}.{}", data.metadata.name, data.metadata.data_type)
-                            }
-                        }
-                    }
-                };
-                let path = input_folder.join(&filename);
-                fs::write(path, &data
-                    .content)
-                    .expect("Failed to write data to file");
-                i+=1;
-                println!("downloaded {}", filename);
-            }
-            Some(Err(_)) => {
-                t.status = task::Status::RETRY;
-                let message = serialize_into_vec(t).expect("failed to serialize task update");
-                c.publish(job_id.clone(), message).await.expect("failed to publish task update");
-                return;
-            }
-            None => {
-                break;
-            }
-        }
-    }
-    println!("Finished downloading {} data for {}", i, claim.task_name);
+    // let mut downloader = datastore.consume("".to_string(), query_clone, false).await;
+    // let mut i = 0;
+    // loop {
+    //     match downloader.next().await {
+    //         Some(Ok(data)) => {
+    //             let filename = match data.metadata.name.as_str() {
+    //                 "Manifest.json" => "Manifest.json".to_string(),
+    //                 "FeaturePoints.ply" => "FeaturePoints.ply".to_string(),
+    //                 "ARposes.csv" => "ARposes.csv".to_string(),
+    //                 "PortalDetections.csv" => "PortalDetections.csv".to_string(),
+    //                 "CameraIntrinsics.csv" => "CameraIntrinsics.csv".to_string(),
+    //                 "Frames.csv" => "Frames.csv".to_string(),
+    //                 "Gyro.csv" => "Gyro.csv".to_string(),
+    //                 "Accel.csv" => "Accel.csv".to_string(),
+    //                 "gyro_accel.csv" => "gyro_accel.csv".to_string(),
+    //                 "Frames.mp4" => "Frames.mp4".to_string(),
+    //                 _ => {
+    //                     match data.metadata.data_type.as_str() {
+    //                         "dmt_manifest_json" => "Manifest.json".to_string(),
+    //                         "dmt_featurepoints_ply" | "dmt_pointcloud_ply" => "FeaturePoints.ply".to_string(),
+    //                         "dmt_arposes_csv" => "ARposes.csv".to_string(),
+    //                         "dmt_portal_detections_csv" | "dmt_observations_csv" => "PortalDetections.csv".to_string(),
+    //                         "dmt_intrinsics_csv" | "dmt_cameraintrinsics_csv" => "CameraIntrinsics.csv".to_string(),
+    //                         "dmt_frames_csv" => "Frames.csv".to_string(),
+    //                         "dmt_gyro_csv" => "Gyro.csv".to_string(),
+    //                         "dmt_accel_csv" => "Accel.csv".to_string(),
+    //                         "dmt_gyroaccel_csv" => "gyro_accel.csv".to_string(),
+    //                         "dmt_recording_mp4" => "Frames.mp4".to_string(),
+    //                         _ => {
+    //                             println!("unknown domain data type: {}", data.metadata.data_type);
+    //                             format!("{}.{}", data.metadata.name, data.metadata.data_type)
+    //                         }
+    //                     }
+    //                 }
+    //             };
+    //             let path = input_folder.join(&filename);
+    //             fs::write(path, &data
+    //                 .content)
+    //                 .expect("Failed to write data to file");
+    //             i+=1;
+    //             println!("downloaded {}", filename);
+    //         }
+    //         Some(Err(_)) => {
+    //             t.status = task::Status::RETRY;
+    //             let message = serialize_into_vec(t).expect("failed to serialize task update");
+    //             c.publish(job_id.clone(), message).await.expect("failed to publish task update");
+    //             return;
+    //         }
+    //         None => {
+    //             break;
+    //         }
+    //     }
+    // }
+    // println!("Finished downloading {} data for {}", i, claim.task_name);
 
-    if let Err(e) = write_scan_data_summary(input_folder.as_path(), task_folder.as_path().join("scan_data_summary.json").as_path()) {
-        eprintln!("Failed to write scan data summary: {}", e);
-        t.status = task::Status::FAILED;
-        t.output = Some(task::Any {
-            type_url: "Error".to_string(),
-            value: serialize_into_vec(&task::Error {
-                message: format!("Failed to write scan data summary: {}", e),
-            }).unwrap(),
-        });
-        let message = serialize_into_vec(t).expect("failed to serialize task update");
-        c.publish(job_id.clone(), message).await.expect("failed to publish task update");
-        return;
-    }
+    // if let Err(e) = write_scan_data_summary(input_folder.as_path(), task_folder.as_path().join("scan_data_summary.json").as_path()) {
+    //     eprintln!("Failed to write scan data summary: {}", e);
+    //     t.status = task::Status::FAILED;
+    //     t.output = Some(task::Any {
+    //         type_url: "Error".to_string(),
+    //         value: serialize_into_vec(&task::Error {
+    //             message: format!("Failed to write scan data summary: {}", e),
+    //         }).unwrap(),
+    //     });
+    //     let message = serialize_into_vec(t).expect("failed to serialize task update");
+    //     c.publish(job_id.clone(), message).await.expect("failed to publish task update");
+    //     return;
+    // }
 
-    let params = vec![
-        "main.py",
-        "--mode", "local_refinement",
-        "--job_root_path", task_folder.to_str().unwrap(),
-        "--output", output_folder.to_str().unwrap(),
-        "--domain_id", &claim.domain_id,
-        "--job_id", &claim.job_id,
-        "--scans", suffix.as_str(),
-    ];
-    let child = Command::new("python3")
-    .args(params)
-    .stdout(Stdio::piped())
-    .stderr(Stdio::piped())
-    .spawn();
+    // let params = vec![
+    //     "main.py",
+    //     "--mode", "local_refinement",
+    //     "--job_root_path", task_folder.to_str().unwrap(),
+    //     "--output", output_folder.to_str().unwrap(),
+    //     "--domain_id", &claim.domain_id,
+    //     "--job_id", &claim.job_id,
+    //     "--scans", suffix.as_str(),
+    // ];
+    // let child = Command::new("python3")
+    // .args(params)
+    // .stdout(Stdio::piped())
+    // .stderr(Stdio::piped())
+    // .spawn();
 
-    if let Err(e) = child {
-        eprintln!("Failed to execute local refinement: {}", e);
-        t.status = task::Status::FAILED;
-        let message = serialize_into_vec(t).expect("failed to serialize task update");
-        c.publish(job_id.clone(), message).await.expect("failed to publish task update");
-        return;
-    }
-    let mut child = child.unwrap();
+    // if let Err(e) = child {
+    //     eprintln!("Failed to execute local refinement: {}", e);
+    //     t.status = task::Status::FAILED;
+    //     let message = serialize_into_vec(t).expect("failed to serialize task update");
+    //     c.publish(job_id.clone(), message).await.expect("failed to publish task update");
+    //     return;
+    // }
+    // let mut child = child.unwrap();
 
-    // Read stdout in real-time
-    if let Some(stdout) = child.stdout.take() {
-        let stdout_reader = BufReader::new(stdout);
-        tokio::spawn(async move {
-            for line in stdout_reader.lines() {
-                if let Ok(line) = line {
-                    println!("stdout: {}", line);
-                }
-            }
-        });
-    }
+    // // Read stdout in real-time
+    // if let Some(stdout) = child.stdout.take() {
+    //     let stdout_reader = BufReader::new(stdout);
+    //     tokio::spawn(async move {
+    //         for line in stdout_reader.lines() {
+    //             if let Ok(line) = line {
+    //                 println!("stdout: {}", line);
+    //             }
+    //         }
+    //     });
+    // }
 
-    // Read stderr in real-time
-    if let Some(stderr) = child.stderr.take() {
-        let stderr_reader = BufReader::new(stderr);
-        tokio::spawn(async move {
-            for line in stderr_reader.lines() {
-                if let Ok(line) = line {
-                    eprintln!("stderr: {}", line);
-                }
-            }
-        });
-    }
+    // // Read stderr in real-time
+    // if let Some(stderr) = child.stderr.take() {
+    //     let stderr_reader = BufReader::new(stderr);
+    //     tokio::spawn(async move {
+    //         for line in stderr_reader.lines() {
+    //             if let Ok(line) = line {
+    //                 eprintln!("stderr: {}", line);
+    //             }
+    //         }
+    //     });
+    // }
 
-    // Wait for the process to complete
-    match child.wait() {
-        Ok(status) => {
-            if !status.success() {
-                eprintln!("Python process exited with status: {}", status);
-                t.status = task::Status::FAILED;
-                t.output = Some(task::Any {
-                    type_url: "Error".to_string(),
-                    value: serialize_into_vec(&task::Error {
-                        message: format!("Python process exited with status: {}", status),
-                    }).unwrap(),
-                });
-                let message = serialize_into_vec(t).expect("failed to serialize task update");
-                c.publish(job_id.clone(), message).await.expect("failed to publish task update");
-                return;
-            }
-            println!("Finished executing {}", claim.task_name);
-        }
-        Err(e) => {
-            eprintln!("Failed to wait for Python process: {}", e);
-            t.status = task::Status::FAILED;
-            t.output = Some(task::Any {
-                type_url: "Error".to_string(),
-                value: serialize_into_vec(&task::Error {
-                    message: format!("Failed to wait for Python process: {}", e),
-                }).unwrap(),
-            });
-            let message = serialize_into_vec(t).expect("failed to serialize task update");
-            c.publish(job_id.clone(), message).await.expect("failed to publish task update");
-            return;
-        }
-    }
+    // // Wait for the process to complete
+    // match child.wait() {
+    //     Ok(status) => {
+    //         if !status.success() {
+    //             eprintln!("Python process exited with status: {}", status);
+    //             t.status = task::Status::FAILED;
+    //             t.output = Some(task::Any {
+    //                 type_url: "Error".to_string(),
+    //                 value: serialize_into_vec(&task::Error {
+    //                     message: format!("Python process exited with status: {}", status),
+    //                 }).unwrap(),
+    //             });
+    //             let message = serialize_into_vec(t).expect("failed to serialize task update");
+    //             c.publish(job_id.clone(), message).await.expect("failed to publish task update");
+    //             return;
+    //         }
+    //         println!("Finished executing {}", claim.task_name);
+    //     }
+    //     Err(e) => {
+    //         eprintln!("Failed to wait for Python process: {}", e);
+    //         t.status = task::Status::FAILED;
+    //         t.output = Some(task::Any {
+    //             type_url: "Error".to_string(),
+    //             value: serialize_into_vec(&task::Error {
+    //                 message: format!("Failed to wait for Python process: {}", e),
+    //             }).unwrap(),
+    //         });
+    //         let message = serialize_into_vec(t).expect("failed to serialize task update");
+    //         c.publish(job_id.clone(), message).await.expect("failed to publish task update");
+    //         return;
+    //     }
+    // }
 
     let mut producer = datastore.produce(claim.domain_id.clone()).await;
 
-    let zip_path = output_folder.join("sfm").join(suffix.clone() + ".zip");
+    let sfm = output_folder.join(suffix.clone()).join("sfm");
+    let zip_path = sfm.join(suffix.clone() + ".zip");
     let mut zip = zip::ZipWriter::new(fs::File::create(&zip_path).expect("Failed to create zip file"));
     let options = SimpleFileOptions::default().compression_method(zip::CompressionMethod::Stored); // No compression
     
     // open output folder/sfm, zip all txt, bin, csv files and upload
-    for entry in fs::read_dir(output_folder.join("sfm")).expect("Failed to read directory") {
+    for entry in fs::read_dir(sfm).expect("Failed to read directory") {
         let entry = entry.expect("Failed to read entry");
         let path = entry.path();
+        if path.is_dir() {
+            continue;
+        }
         let ext = path.extension().expect("Failed to get extension").to_str().expect("Failed to convert extension to string");
         if ext == "txt" || ext == "bin" || ext == "csv" {
             let str_path = path.to_str().expect("Failed to convert path to string");
@@ -332,11 +326,11 @@ pub(crate) async fn v1(base_path: String, mut stream: Stream, mut datastore: Box
     };
     let event = task::Task {
         name: claim.task_name.clone(),
-        receiver: claim.receiver.clone(),
+        receiver: Some(claim.receiver.clone()),
         sender: claim.sender.clone(),
         endpoint: "/local-refinement/v1".to_string(),
         status: task::Status::DONE,
-        access_token: "".to_string(),
+        access_token: None,
         job_id: job_id.clone(),
         output: Some(task::Any {
             type_url: "LocalRefinementOutputV1".to_string(),
@@ -346,6 +340,7 @@ pub(crate) async fn v1(base_path: String, mut stream: Stream, mut datastore: Box
     let buf = serialize_into_vec(&event).expect("failed to serialize task update");
     c.publish(job_id.clone(), buf).await.expect("failed to publish task update");
 
+    tx.send(true).unwrap();
     let _ = heartbeat_handle.await;
     println!("Heartbeat task stopped");
     return;

From 4efd8d955c81336071287bdf36584e63c659d899 Mon Sep 17 00:00:00 2001
From: Shuning <shuning@aukilabs.com>
Date: Tue, 25 Mar 2025 19:22:25 +0800
Subject: [PATCH 35/89] uncomment local refinement

---
 server/src/local_refinement.rs | 286 ++++++++++++++++-----------------
 1 file changed, 143 insertions(+), 143 deletions(-)

diff --git a/server/src/local_refinement.rs b/server/src/local_refinement.rs
index eb2af3e..4ed10cc 100644
--- a/server/src/local_refinement.rs
+++ b/server/src/local_refinement.rs
@@ -107,155 +107,155 @@ pub(crate) async fn v1(base_path: String, mut stream: Stream, mut datastore: Box
         let _ = tx.send(true); // Signal heartbeat task to stop
     });
 
-    // let mut downloader = datastore.consume("".to_string(), query_clone, false).await;
-    // let mut i = 0;
-    // loop {
-    //     match downloader.next().await {
-    //         Some(Ok(data)) => {
-    //             let filename = match data.metadata.name.as_str() {
-    //                 "Manifest.json" => "Manifest.json".to_string(),
-    //                 "FeaturePoints.ply" => "FeaturePoints.ply".to_string(),
-    //                 "ARposes.csv" => "ARposes.csv".to_string(),
-    //                 "PortalDetections.csv" => "PortalDetections.csv".to_string(),
-    //                 "CameraIntrinsics.csv" => "CameraIntrinsics.csv".to_string(),
-    //                 "Frames.csv" => "Frames.csv".to_string(),
-    //                 "Gyro.csv" => "Gyro.csv".to_string(),
-    //                 "Accel.csv" => "Accel.csv".to_string(),
-    //                 "gyro_accel.csv" => "gyro_accel.csv".to_string(),
-    //                 "Frames.mp4" => "Frames.mp4".to_string(),
-    //                 _ => {
-    //                     match data.metadata.data_type.as_str() {
-    //                         "dmt_manifest_json" => "Manifest.json".to_string(),
-    //                         "dmt_featurepoints_ply" | "dmt_pointcloud_ply" => "FeaturePoints.ply".to_string(),
-    //                         "dmt_arposes_csv" => "ARposes.csv".to_string(),
-    //                         "dmt_portal_detections_csv" | "dmt_observations_csv" => "PortalDetections.csv".to_string(),
-    //                         "dmt_intrinsics_csv" | "dmt_cameraintrinsics_csv" => "CameraIntrinsics.csv".to_string(),
-    //                         "dmt_frames_csv" => "Frames.csv".to_string(),
-    //                         "dmt_gyro_csv" => "Gyro.csv".to_string(),
-    //                         "dmt_accel_csv" => "Accel.csv".to_string(),
-    //                         "dmt_gyroaccel_csv" => "gyro_accel.csv".to_string(),
-    //                         "dmt_recording_mp4" => "Frames.mp4".to_string(),
-    //                         _ => {
-    //                             println!("unknown domain data type: {}", data.metadata.data_type);
-    //                             format!("{}.{}", data.metadata.name, data.metadata.data_type)
-    //                         }
-    //                     }
-    //                 }
-    //             };
-    //             let path = input_folder.join(&filename);
-    //             fs::write(path, &data
-    //                 .content)
-    //                 .expect("Failed to write data to file");
-    //             i+=1;
-    //             println!("downloaded {}", filename);
-    //         }
-    //         Some(Err(_)) => {
-    //             t.status = task::Status::RETRY;
-    //             let message = serialize_into_vec(t).expect("failed to serialize task update");
-    //             c.publish(job_id.clone(), message).await.expect("failed to publish task update");
-    //             return;
-    //         }
-    //         None => {
-    //             break;
-    //         }
-    //     }
-    // }
-    // println!("Finished downloading {} data for {}", i, claim.task_name);
+    let mut downloader = datastore.consume("".to_string(), query_clone, false).await;
+    let mut i = 0;
+    loop {
+        match downloader.next().await {
+            Some(Ok(data)) => {
+                let filename = match data.metadata.name.as_str() {
+                    "Manifest.json" => "Manifest.json".to_string(),
+                    "FeaturePoints.ply" => "FeaturePoints.ply".to_string(),
+                    "ARposes.csv" => "ARposes.csv".to_string(),
+                    "PortalDetections.csv" => "PortalDetections.csv".to_string(),
+                    "CameraIntrinsics.csv" => "CameraIntrinsics.csv".to_string(),
+                    "Frames.csv" => "Frames.csv".to_string(),
+                    "Gyro.csv" => "Gyro.csv".to_string(),
+                    "Accel.csv" => "Accel.csv".to_string(),
+                    "gyro_accel.csv" => "gyro_accel.csv".to_string(),
+                    "Frames.mp4" => "Frames.mp4".to_string(),
+                    _ => {
+                        match data.metadata.data_type.as_str() {
+                            "dmt_manifest_json" => "Manifest.json".to_string(),
+                            "dmt_featurepoints_ply" | "dmt_pointcloud_ply" => "FeaturePoints.ply".to_string(),
+                            "dmt_arposes_csv" => "ARposes.csv".to_string(),
+                            "dmt_portal_detections_csv" | "dmt_observations_csv" => "PortalDetections.csv".to_string(),
+                            "dmt_intrinsics_csv" | "dmt_cameraintrinsics_csv" => "CameraIntrinsics.csv".to_string(),
+                            "dmt_frames_csv" => "Frames.csv".to_string(),
+                            "dmt_gyro_csv" => "Gyro.csv".to_string(),
+                            "dmt_accel_csv" => "Accel.csv".to_string(),
+                            "dmt_gyroaccel_csv" => "gyro_accel.csv".to_string(),
+                            "dmt_recording_mp4" => "Frames.mp4".to_string(),
+                            _ => {
+                                println!("unknown domain data type: {}", data.metadata.data_type);
+                                format!("{}.{}", data.metadata.name, data.metadata.data_type)
+                            }
+                        }
+                    }
+                };
+                let path = input_folder.join(&filename);
+                fs::write(path, &data
+                    .content)
+                    .expect("Failed to write data to file");
+                i+=1;
+                println!("downloaded {}", filename);
+            }
+            Some(Err(_)) => {
+                t.status = task::Status::RETRY;
+                let message = serialize_into_vec(t).expect("failed to serialize task update");
+                c.publish(job_id.clone(), message).await.expect("failed to publish task update");
+                return;
+            }
+            None => {
+                break;
+            }
+        }
+    }
+    println!("Finished downloading {} data for {}", i, claim.task_name);
 
-    // if let Err(e) = write_scan_data_summary(input_folder.as_path(), task_folder.as_path().join("scan_data_summary.json").as_path()) {
-    //     eprintln!("Failed to write scan data summary: {}", e);
-    //     t.status = task::Status::FAILED;
-    //     t.output = Some(task::Any {
-    //         type_url: "Error".to_string(),
-    //         value: serialize_into_vec(&task::Error {
-    //             message: format!("Failed to write scan data summary: {}", e),
-    //         }).unwrap(),
-    //     });
-    //     let message = serialize_into_vec(t).expect("failed to serialize task update");
-    //     c.publish(job_id.clone(), message).await.expect("failed to publish task update");
-    //     return;
-    // }
+    if let Err(e) = write_scan_data_summary(input_folder.as_path(), task_folder.as_path().join("scan_data_summary.json").as_path()) {
+        eprintln!("Failed to write scan data summary: {}", e);
+        t.status = task::Status::FAILED;
+        t.output = Some(task::Any {
+            type_url: "Error".to_string(),
+            value: serialize_into_vec(&task::Error {
+                message: format!("Failed to write scan data summary: {}", e),
+            }).unwrap(),
+        });
+        let message = serialize_into_vec(t).expect("failed to serialize task update");
+        c.publish(job_id.clone(), message).await.expect("failed to publish task update");
+        return;
+    }
 
-    // let params = vec![
-    //     "main.py",
-    //     "--mode", "local_refinement",
-    //     "--job_root_path", task_folder.to_str().unwrap(),
-    //     "--output", output_folder.to_str().unwrap(),
-    //     "--domain_id", &claim.domain_id,
-    //     "--job_id", &claim.job_id,
-    //     "--scans", suffix.as_str(),
-    // ];
-    // let child = Command::new("python3")
-    // .args(params)
-    // .stdout(Stdio::piped())
-    // .stderr(Stdio::piped())
-    // .spawn();
+    let params = vec![
+        "main.py",
+        "--mode", "local_refinement",
+        "--job_root_path", task_folder.to_str().unwrap(),
+        "--output", output_folder.to_str().unwrap(),
+        "--domain_id", &claim.domain_id,
+        "--job_id", &claim.job_id,
+        "--scans", suffix.as_str(),
+    ];
+    let child = Command::new("python3")
+    .args(params)
+    .stdout(Stdio::piped())
+    .stderr(Stdio::piped())
+    .spawn();
 
-    // if let Err(e) = child {
-    //     eprintln!("Failed to execute local refinement: {}", e);
-    //     t.status = task::Status::FAILED;
-    //     let message = serialize_into_vec(t).expect("failed to serialize task update");
-    //     c.publish(job_id.clone(), message).await.expect("failed to publish task update");
-    //     return;
-    // }
-    // let mut child = child.unwrap();
+    if let Err(e) = child {
+        eprintln!("Failed to execute local refinement: {}", e);
+        t.status = task::Status::FAILED;
+        let message = serialize_into_vec(t).expect("failed to serialize task update");
+        c.publish(job_id.clone(), message).await.expect("failed to publish task update");
+        return;
+    }
+    let mut child = child.unwrap();
 
-    // // Read stdout in real-time
-    // if let Some(stdout) = child.stdout.take() {
-    //     let stdout_reader = BufReader::new(stdout);
-    //     tokio::spawn(async move {
-    //         for line in stdout_reader.lines() {
-    //             if let Ok(line) = line {
-    //                 println!("stdout: {}", line);
-    //             }
-    //         }
-    //     });
-    // }
+    // Read stdout in real-time
+    if let Some(stdout) = child.stdout.take() {
+        let stdout_reader = BufReader::new(stdout);
+        tokio::spawn(async move {
+            for line in stdout_reader.lines() {
+                if let Ok(line) = line {
+                    println!("stdout: {}", line);
+                }
+            }
+        });
+    }
 
-    // // Read stderr in real-time
-    // if let Some(stderr) = child.stderr.take() {
-    //     let stderr_reader = BufReader::new(stderr);
-    //     tokio::spawn(async move {
-    //         for line in stderr_reader.lines() {
-    //             if let Ok(line) = line {
-    //                 eprintln!("stderr: {}", line);
-    //             }
-    //         }
-    //     });
-    // }
+    // Read stderr in real-time
+    if let Some(stderr) = child.stderr.take() {
+        let stderr_reader = BufReader::new(stderr);
+        tokio::spawn(async move {
+            for line in stderr_reader.lines() {
+                if let Ok(line) = line {
+                    eprintln!("stderr: {}", line);
+                }
+            }
+        });
+    }
 
-    // // Wait for the process to complete
-    // match child.wait() {
-    //     Ok(status) => {
-    //         if !status.success() {
-    //             eprintln!("Python process exited with status: {}", status);
-    //             t.status = task::Status::FAILED;
-    //             t.output = Some(task::Any {
-    //                 type_url: "Error".to_string(),
-    //                 value: serialize_into_vec(&task::Error {
-    //                     message: format!("Python process exited with status: {}", status),
-    //                 }).unwrap(),
-    //             });
-    //             let message = serialize_into_vec(t).expect("failed to serialize task update");
-    //             c.publish(job_id.clone(), message).await.expect("failed to publish task update");
-    //             return;
-    //         }
-    //         println!("Finished executing {}", claim.task_name);
-    //     }
-    //     Err(e) => {
-    //         eprintln!("Failed to wait for Python process: {}", e);
-    //         t.status = task::Status::FAILED;
-    //         t.output = Some(task::Any {
-    //             type_url: "Error".to_string(),
-    //             value: serialize_into_vec(&task::Error {
-    //                 message: format!("Failed to wait for Python process: {}", e),
-    //             }).unwrap(),
-    //         });
-    //         let message = serialize_into_vec(t).expect("failed to serialize task update");
-    //         c.publish(job_id.clone(), message).await.expect("failed to publish task update");
-    //         return;
-    //     }
-    // }
+    // Wait for the process to complete
+    match child.wait() {
+        Ok(status) => {
+            if !status.success() {
+                eprintln!("Python process exited with status: {}", status);
+                t.status = task::Status::FAILED;
+                t.output = Some(task::Any {
+                    type_url: "Error".to_string(),
+                    value: serialize_into_vec(&task::Error {
+                        message: format!("Python process exited with status: {}", status),
+                    }).unwrap(),
+                });
+                let message = serialize_into_vec(t).expect("failed to serialize task update");
+                c.publish(job_id.clone(), message).await.expect("failed to publish task update");
+                return;
+            }
+            println!("Finished executing {}", claim.task_name);
+        }
+        Err(e) => {
+            eprintln!("Failed to wait for Python process: {}", e);
+            t.status = task::Status::FAILED;
+            t.output = Some(task::Any {
+                type_url: "Error".to_string(),
+                value: serialize_into_vec(&task::Error {
+                    message: format!("Failed to wait for Python process: {}", e),
+                }).unwrap(),
+            });
+            let message = serialize_into_vec(t).expect("failed to serialize task update");
+            c.publish(job_id.clone(), message).await.expect("failed to publish task update");
+            return;
+        }
+    }
 
     let mut producer = datastore.produce(claim.domain_id.clone()).await;
 

From c02ab1a92ee789d5d1eb1db7266ffaccc6b02e45 Mon Sep 17 00:00:00 2001
From: Shuning <shuning@aukilabs.com>
Date: Tue, 25 Mar 2025 21:15:07 +0800
Subject: [PATCH 36/89] fix import

---
 server/Cargo.lock | 3 +++
 server/Cargo.toml | 4 ++--
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/server/Cargo.lock b/server/Cargo.lock
index f3653e9..e79fcd5 100644
--- a/server/Cargo.lock
+++ b/server/Cargo.lock
@@ -919,6 +919,7 @@ dependencies = [
 [[package]]
 name = "domain"
 version = "0.1.0"
+source = "git+https://github.com/aukilabs/posemesh.git?branch=chore%2Freconstruction-integration#8b5978edab2714f601161f39404d8147872a20e3"
 dependencies = [
  "async-trait",
  "cbindgen",
@@ -2814,6 +2815,7 @@ dependencies = [
 [[package]]
 name = "networking"
 version = "0.1.0"
+source = "git+https://github.com/aukilabs/posemesh.git?branch=chore%2Freconstruction-integration#8b5978edab2714f601161f39404d8147872a20e3"
 dependencies = [
  "cbindgen",
  "console_error_panic_hook",
@@ -3565,6 +3567,7 @@ dependencies = [
 [[package]]
 name = "runtime"
 version = "0.1.0"
+source = "git+https://github.com/aukilabs/posemesh.git?branch=chore%2Freconstruction-integration#8b5978edab2714f601161f39404d8147872a20e3"
 dependencies = [
  "once_cell",
  "tokio",
diff --git a/server/Cargo.toml b/server/Cargo.toml
index 7a1668e..e680233 100644
--- a/server/Cargo.toml
+++ b/server/Cargo.toml
@@ -5,12 +5,12 @@ edition = "2021"
 
 [dependencies]
 libp2p = { git = "https://github.com/aukilabs/rust-libp2p.git", branch = "fix/gossipsub-wasm" }
-networking = { path = "../../posemesh/core/networking" }
+networking = { git = "https://github.com/aukilabs/posemesh.git", branch = "chore/reconstruction-integration" }
 futures = "0.3.30"
 jsonwebtoken = "9.3.0"
 uuid = "1.13.2"
 rand = "0.9.0"
-domain = { path = "../../posemesh/core/domain" }
+domain = { git = "https://github.com/aukilabs/posemesh.git", branch = "chore/reconstruction-integration" }
 tracing = "0.1.40"
 tracing-subscriber = "0.3.18"
 tokio = "1.44.0"

From ee439754fd7c0fb332375167472ca188325d8696 Mon Sep 17 00:00:00 2001
From: Shuning <shuning@aukilabs.com>
Date: Wed, 26 Mar 2025 11:31:58 +0800
Subject: [PATCH 37/89] resolve handshake timeout

---
 server/Cargo.lock              | 6 +++---
 server/src/local_refinement.rs | 3 +--
 utils/data_utils.py            | 3 ++-
 3 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/server/Cargo.lock b/server/Cargo.lock
index e79fcd5..8a209ac 100644
--- a/server/Cargo.lock
+++ b/server/Cargo.lock
@@ -919,7 +919,7 @@ dependencies = [
 [[package]]
 name = "domain"
 version = "0.1.0"
-source = "git+https://github.com/aukilabs/posemesh.git?branch=chore%2Freconstruction-integration#8b5978edab2714f601161f39404d8147872a20e3"
+source = "git+https://github.com/aukilabs/posemesh.git?branch=chore%2Freconstruction-integration#ac99aaf25112409137382d14d23dacc2e75ee45b"
 dependencies = [
  "async-trait",
  "cbindgen",
@@ -2815,7 +2815,7 @@ dependencies = [
 [[package]]
 name = "networking"
 version = "0.1.0"
-source = "git+https://github.com/aukilabs/posemesh.git?branch=chore%2Freconstruction-integration#8b5978edab2714f601161f39404d8147872a20e3"
+source = "git+https://github.com/aukilabs/posemesh.git?branch=chore%2Freconstruction-integration#ac99aaf25112409137382d14d23dacc2e75ee45b"
 dependencies = [
  "cbindgen",
  "console_error_panic_hook",
@@ -3567,7 +3567,7 @@ dependencies = [
 [[package]]
 name = "runtime"
 version = "0.1.0"
-source = "git+https://github.com/aukilabs/posemesh.git?branch=chore%2Freconstruction-integration#8b5978edab2714f601161f39404d8147872a20e3"
+source = "git+https://github.com/aukilabs/posemesh.git?branch=chore%2Freconstruction-integration#ac99aaf25112409137382d14d23dacc2e75ee45b"
 dependencies = [
  "once_cell",
  "tokio",
diff --git a/server/src/local_refinement.rs b/server/src/local_refinement.rs
index 4ed10cc..ac9bb75 100644
--- a/server/src/local_refinement.rs
+++ b/server/src/local_refinement.rs
@@ -257,8 +257,6 @@ pub(crate) async fn v1(base_path: String, mut stream: Stream, mut datastore: Box
         }
     }
 
-    let mut producer = datastore.produce(claim.domain_id.clone()).await;
-
     let sfm = output_folder.join(suffix.clone()).join("sfm");
     let zip_path = sfm.join(suffix.clone() + ".zip");
     let mut zip = zip::ZipWriter::new(fs::File::create(&zip_path).expect("Failed to create zip file"));
@@ -291,6 +289,7 @@ pub(crate) async fn v1(base_path: String, mut stream: Stream, mut datastore: Box
     zip.finish().expect("Failed to finish zip");
     let zip_file_metadata = fs::metadata(&zip_path).expect("Failed to get metadata");
 
+    let mut producer = datastore.produce(claim.domain_id.clone()).await;
     let res = producer.push(&Data {
         domain_id: claim.domain_id.clone(),
         metadata: Metadata {
diff --git a/utils/data_utils.py b/utils/data_utils.py
index 3eb1619..2c4d27a 100644
--- a/utils/data_utils.py
+++ b/utils/data_utils.py
@@ -877,7 +877,8 @@ def setup_logger(name=None, log_file=None, domain_id="", job_id="", dataset_id=N
     console_handler.setFormatter(JsonFormatter(datefmt='%Y-%m-%dT%H:%M:%S',
         domain_id=domain_id, job_id=job_id, dataset_id=dataset_id))
     logger.addHandler(console_handler)
-    sys.stdout.reconfigure(line_buffering=True) 
+    sys.stdout.reconfigure(line_buffering=True)
+    sys.stderr.reconfigure(line_buffering=True)
     logger.info("JSON formatter setup complete")
 
     return logger

From f87de8f93ecde8d61d1cc51c744ae302313eccac Mon Sep 17 00:00:00 2001
From: Shuning <shuning@aukilabs.com>
Date: Wed, 26 Mar 2025 16:42:42 +0800
Subject: [PATCH 38/89] update lock

---
 server/Cargo.lock  | 6 +++---
 server/src/main.rs | 1 -
 2 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/server/Cargo.lock b/server/Cargo.lock
index 8a209ac..a742770 100644
--- a/server/Cargo.lock
+++ b/server/Cargo.lock
@@ -919,7 +919,7 @@ dependencies = [
 [[package]]
 name = "domain"
 version = "0.1.0"
-source = "git+https://github.com/aukilabs/posemesh.git?branch=chore%2Freconstruction-integration#ac99aaf25112409137382d14d23dacc2e75ee45b"
+source = "git+https://github.com/aukilabs/posemesh.git?branch=chore%2Freconstruction-integration#7c9111ba356f33d621adccf986dc9fcc72721d83"
 dependencies = [
  "async-trait",
  "cbindgen",
@@ -2815,7 +2815,7 @@ dependencies = [
 [[package]]
 name = "networking"
 version = "0.1.0"
-source = "git+https://github.com/aukilabs/posemesh.git?branch=chore%2Freconstruction-integration#ac99aaf25112409137382d14d23dacc2e75ee45b"
+source = "git+https://github.com/aukilabs/posemesh.git?branch=chore%2Freconstruction-integration#7c9111ba356f33d621adccf986dc9fcc72721d83"
 dependencies = [
  "cbindgen",
  "console_error_panic_hook",
@@ -3567,7 +3567,7 @@ dependencies = [
 [[package]]
 name = "runtime"
 version = "0.1.0"
-source = "git+https://github.com/aukilabs/posemesh.git?branch=chore%2Freconstruction-integration#ac99aaf25112409137382d14d23dacc2e75ee45b"
+source = "git+https://github.com/aukilabs/posemesh.git?branch=chore%2Freconstruction-integration#7c9111ba356f33d621adccf986dc9fcc72721d83"
 dependencies = [
  "once_cell",
  "tokio",
diff --git a/server/src/main.rs b/server/src/main.rs
index 7e6b6e7..0f70e0a 100644
--- a/server/src/main.rs
+++ b/server/src/main.rs
@@ -57,7 +57,6 @@ async fn main() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
                 println!("Received termination signal, shutting down...");
                 break;
             }
-            else => break
         }
     }
 

From 2112e07aaabef04f23d84dcd60cbdb45c38c799b Mon Sep 17 00:00:00 2001
From: Shuning <shuning@aukilabs.com>
Date: Wed, 26 Mar 2025 16:43:44 +0800
Subject: [PATCH 39/89] fix rust

---
 server/src/main.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/server/src/main.rs b/server/src/main.rs
index 0f70e0a..7e6b6e7 100644
--- a/server/src/main.rs
+++ b/server/src/main.rs
@@ -57,6 +57,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
                 println!("Received termination signal, shutting down...");
                 break;
             }
+            else => break
         }
     }
 

From 912195f572b072308950f8b22290fe8abf1088b5 Mon Sep 17 00:00:00 2001
From: Shuning <shuning@aukilabs.com>
Date: Wed, 26 Mar 2025 17:49:51 +0800
Subject: [PATCH 40/89] use self hosted machine to leverage docker build cache

---
 .github/workflows/docker-build.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml
index f793bd7..d559ffe 100644
--- a/.github/workflows/docker-build.yml
+++ b/.github/workflows/docker-build.yml
@@ -11,7 +11,7 @@ concurrency:
 
 jobs:
   build:
-    runs-on: ubuntu-latest
+    runs-on: self-hosted
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4

From 6fae60ff1d9764e252c90acdca9011052fbeacad Mon Sep 17 00:00:00 2001
From: Shuning <shuning@aukilabs.com>
Date: Wed, 26 Mar 2025 17:50:42 +0800
Subject: [PATCH 41/89] remove space saver

---
 .github/workflows/docker-build.yml | 12 ------------
 1 file changed, 12 deletions(-)

diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml
index d559ffe..009592e 100644
--- a/.github/workflows/docker-build.yml
+++ b/.github/workflows/docker-build.yml
@@ -16,15 +16,6 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@v4
 
-      - name: Free up space
-        run: |
-          sudo rm -rf /usr/share/dotnet  # ~20GB
-          sudo rm -rf /opt/ghc           # ~4GB
-          sudo rm -rf /usr/local/lib/android  # ~10GB
-
-      - name: Check Available Space
-        run: df -h
-
       - name: Configure AWS credentials
         uses: aws-actions/configure-aws-credentials@v4
         with:
@@ -57,6 +48,3 @@ jobs:
           platforms: linux/amd64
           push: true
           tags: ${{ steps.meta.outputs.tags }}
-
-      - name: Check Available Space
-        run: df -h

From b4ea0c20737972e4ef82971f43b1007dc904b3fc Mon Sep 17 00:00:00 2001
From: Shuning <shuning@aukilabs.com>
Date: Wed, 26 Mar 2025 18:23:51 +0800
Subject: [PATCH 42/89] update dockerfile

---
 docker/Dockerfile | 54 ++++++++++++++++++++++++-----------------------
 1 file changed, 28 insertions(+), 26 deletions(-)

diff --git a/docker/Dockerfile b/docker/Dockerfile
index a5246af..81976c5 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -1,8 +1,29 @@
+##
+## Rust build for server
+##
+FROM --platform=$BUILDPLATFORM rust:1.84-slim-bullseye AS rust-build
+ARG TARGETOS
+ARG TARGETARCH
+ARG VERSION
+
+RUN apt-get update && apt-get install -y \
+    git \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app
+COPY server /app/server
+
+# Build the application
+RUN cd server && cargo build --release
+
 ##
 ## Runtime image
 ##
-FROM nvidia/cuda:11.0.3-base-ubuntu20.04 AS deps
+FROM nvidia/cuda:11.0.3-base-ubuntu20.04
 
+ARG USERNAME=reconstruction-server
+ARG USER_UID=1000
+ARG USER_GID=$USER_UID
 ARG DEBIAN_FRONTEND=noninteractive
 
 RUN apt-get update && apt-get install -y \
@@ -121,30 +142,12 @@ RUN python3 -m pip install enlighten evo
 
 RUN python3 -m pip install open3d trimesh alphashape
 
-##
-## Rust build for server
-##
-FROM --platform=$BUILDPLATFORM rust:1.84-slim-bullseye AS rust-build
-ARG TARGETOS
-ARG TARGETARCH
-ARG VERSION
-
-RUN apt-get update && apt-get install -y \
-    git \
-    && rm -rf /var/lib/apt/lists/*
-
-WORKDIR /app
-COPY server /app/server
-
-# Build the application
-RUN cd server && cargo build --release
-
-FROM deps AS server-build
-
-ARG USERNAME=reconstruction-server
-ARG USER_UID=1000
-ARG USER_GID=$USER_UID
-ARG DEBIAN_FRONTEND=noninteractive
+RUN git clone https://github.com/google/draco.git && \
+    cd draco && \
+    mkdir build && \
+    cd build && \
+    cmake .. && \
+    make
 
 WORKDIR /app
 
@@ -169,6 +172,5 @@ RUN groupadd --gid $USER_GID $USERNAME \
 RUN mkdir -p /app/jobs && chown -R reconstruction-server:reconstruction-server /app/jobs
 RUN mkdir -p /app/volume && chown -R reconstruction-server:reconstruction-server /app/volume
 USER $USERNAME
-
 # Copy the built binary from rust-build
 COPY --from=rust-build /app/server/target/release/server /app/server

From 0633658228d4ae4a7d6f4e4d8d624eae07b4ffcd Mon Sep 17 00:00:00 2001
From: Shuning <shuning@aukilabs.com>
Date: Wed, 26 Mar 2025 18:56:37 +0800
Subject: [PATCH 43/89] call python in unbuffered mode

---
 server/src/global_refinement.rs | 1 +
 server/src/local_refinement.rs  | 1 +
 utils/data_utils.py             | 2 --
 3 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/server/src/global_refinement.rs b/server/src/global_refinement.rs
index aa29b43..6c1626f 100644
--- a/server/src/global_refinement.rs
+++ b/server/src/global_refinement.rs
@@ -158,6 +158,7 @@ pub(crate) async fn v1(base_path: String, mut stream: Stream, mut datastore: Box
         }
     }
     let params = vec![
+        "-u",
         "main.py",
         "--mode", "global_refinement",
         "--job_root_path", task_path.to_str().unwrap(),
diff --git a/server/src/local_refinement.rs b/server/src/local_refinement.rs
index ac9bb75..1bced68 100644
--- a/server/src/local_refinement.rs
+++ b/server/src/local_refinement.rs
@@ -177,6 +177,7 @@ pub(crate) async fn v1(base_path: String, mut stream: Stream, mut datastore: Box
     }
 
     let params = vec![
+        "-u",
         "main.py",
         "--mode", "local_refinement",
         "--job_root_path", task_folder.to_str().unwrap(),
diff --git a/utils/data_utils.py b/utils/data_utils.py
index 2c4d27a..ad3955b 100644
--- a/utils/data_utils.py
+++ b/utils/data_utils.py
@@ -877,8 +877,6 @@ def setup_logger(name=None, log_file=None, domain_id="", job_id="", dataset_id=N
     console_handler.setFormatter(JsonFormatter(datefmt='%Y-%m-%dT%H:%M:%S',
         domain_id=domain_id, job_id=job_id, dataset_id=dataset_id))
     logger.addHandler(console_handler)
-    sys.stdout.reconfigure(line_buffering=True)
-    sys.stderr.reconfigure(line_buffering=True)
     logger.info("JSON formatter setup complete")
 
     return logger

From 262602e2df1ee05cecdd2daafa8609eb2d0958e0 Mon Sep 17 00:00:00 2001
From: Shuning <shuning@aukilabs.com>
Date: Wed, 26 Mar 2025 19:02:06 +0800
Subject: [PATCH 44/89] remove platform

---
 .github/workflows/docker-build.yml | 1 -
 docker/Dockerfile                  | 4 +---
 2 files changed, 1 insertion(+), 4 deletions(-)

diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml
index 009592e..7f77603 100644
--- a/.github/workflows/docker-build.yml
+++ b/.github/workflows/docker-build.yml
@@ -45,6 +45,5 @@ jobs:
         with:
           context: .
           file: docker/Dockerfile
-          platforms: linux/amd64
           push: true
           tags: ${{ steps.meta.outputs.tags }}
diff --git a/docker/Dockerfile b/docker/Dockerfile
index 81976c5..4bd7874 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -1,9 +1,7 @@
 ##
 ## Rust build for server
 ##
-FROM --platform=$BUILDPLATFORM rust:1.84-slim-bullseye AS rust-build
-ARG TARGETOS
-ARG TARGETARCH
+FROM rust:1.84-slim-bullseye AS rust-build
 ARG VERSION
 
 RUN apt-get update && apt-get install -y \

From 0fe8aafd6b0fe345fcc21670f043a75f9a1faadb Mon Sep 17 00:00:00 2001
From: Shuning <shuning@aukilabs.com>
Date: Wed, 26 Mar 2025 21:19:19 +0800
Subject: [PATCH 45/89] even scans parameter is ignored, it is still required

---
 server/src/global_refinement.rs | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/server/src/global_refinement.rs b/server/src/global_refinement.rs
index 6c1626f..7773e76 100644
--- a/server/src/global_refinement.rs
+++ b/server/src/global_refinement.rs
@@ -112,6 +112,7 @@ pub(crate) async fn v1(base_path: String, mut stream: Stream, mut datastore: Box
     });
 
     let mut downloader = datastore.consume(claim.domain_id.clone(), query, false).await;
+    let mut scan_ids = Vec::new();
 
     loop {
         match downloader.next().await {
@@ -131,6 +132,7 @@ pub(crate) async fn v1(base_path: String, mut stream: Stream, mut datastore: Box
                     return;
                 }
                 let suffix = res.unwrap();
+                scan_ids.push(suffix.clone());
                 
                 fs::create_dir_all(Path::new(&dataset_path).join(&suffix)).expect("Failed to create dataset folder");
                 let path = Path::new(&input_path).join(&suffix).join("sfm");
@@ -157,7 +159,7 @@ pub(crate) async fn v1(base_path: String, mut stream: Stream, mut datastore: Box
             }
         }
     }
-    let params = vec![
+    let mut params = vec![
         "-u",
         "main.py",
         "--mode", "global_refinement",
@@ -167,6 +169,7 @@ pub(crate) async fn v1(base_path: String, mut stream: Stream, mut datastore: Box
         "--job_id", &claim.job_id,
         "--scans"
     ];
+    params.extend(scan_ids.iter().map(|s| s.as_str()));
     let child = Command::new("python3")
     .args(params)
     .stdout(Stdio::piped())

From eef3e86c70b6fb281c9c2a5efde2fea53ef31e1d Mon Sep 17 00:00:00 2001
From: Shuning <shuning@aukilabs.com>
Date: Thu, 27 Mar 2025 10:09:12 +0800
Subject: [PATCH 46/89] self hosted runner doesn't work

---
 .github/workflows/docker-build.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml
index 7f77603..431ba39 100644
--- a/.github/workflows/docker-build.yml
+++ b/.github/workflows/docker-build.yml
@@ -11,7 +11,7 @@ concurrency:
 
 jobs:
   build:
-    runs-on: self-hosted
+    runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4

From f52aac5040bbbee562ad1a1844f72b2b512a2624 Mon Sep 17 00:00:00 2001
From: Shuning <shuning@aukilabs.com>
Date: Thu, 27 Mar 2025 11:02:49 +0800
Subject: [PATCH 47/89] free up space

---
 .github/workflows/docker-build.yml | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml
index 431ba39..1f5f7ef 100644
--- a/.github/workflows/docker-build.yml
+++ b/.github/workflows/docker-build.yml
@@ -16,6 +16,12 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@v4
 
+      - name: Free up space
+        run: |
+          sudo sudo rm -rf /usr/share/dotnet  # ~20GB
+          sudo rm -rf /opt/ghc           # ~4GB
+          sudo rm -rf /usr/local/lib/android  # ~10GB
+
       - name: Configure AWS credentials
         uses: aws-actions/configure-aws-credentials@v4
         with:

From 17bf077cb056060fb21b2a09de5c576bc4fd0553 Mon Sep 17 00:00:00 2001
From: Shuning <shuning@aukilabs.com>
Date: Thu, 27 Mar 2025 12:36:35 +0800
Subject: [PATCH 48/89] only upload some files

---
 server/src/global_refinement.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/server/src/global_refinement.rs b/server/src/global_refinement.rs
index 7773e76..93ace40 100644
--- a/server/src/global_refinement.rs
+++ b/server/src/global_refinement.rs
@@ -248,7 +248,6 @@ pub(crate) async fn v1(base_path: String, mut stream: Stream, mut datastore: Box
     for file in files {
         let file = file.unwrap();
         let path = file.path();
-        let content = fs::read(path).expect("Failed to read file");
         let metadata: Metadata = match file.file_name().to_str().unwrap() {
             "refined_manifest.json" => Metadata {
                 name: "refined_manifest".to_string(),
@@ -273,6 +272,7 @@ pub(crate) async fn v1(base_path: String, mut stream: Stream, mut datastore: Box
             },
             _ => continue
         };
+        let content = fs::read(path).expect("Failed to read file");
         uploader.push(&Data {
             domain_id: claim.domain_id.clone(),
             metadata,

From 46d9259df8abcea4691e8e58f0e33885d9fd8e62 Mon Sep 17 00:00:00 2001
From: Shuning <shuning@aukilabs.com>
Date: Thu, 27 Mar 2025 15:00:15 +0800
Subject: [PATCH 49/89] catch error

---
 server/src/global_refinement.rs | 99 +++++++++++++++++++--------------
 1 file changed, 58 insertions(+), 41 deletions(-)

diff --git a/server/src/global_refinement.rs b/server/src/global_refinement.rs
index 93ace40..3f11bf8 100644
--- a/server/src/global_refinement.rs
+++ b/server/src/global_refinement.rs
@@ -14,6 +14,52 @@ use zip::ZipArchive;
 
 use crate::utils::handshake;
 
+async fn upload_results(domain_id: &str, output_path: PathBuf, mut datastore: Box<dyn Datastore>) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    let mut uploader = datastore.produce(domain_id.to_string()).await;
+    // open output_path and upload to datastore
+    let files = fs::read_dir(output_path)?;
+    for file in files {
+        let file = file?;
+        let path = file.path();
+        let metadata: Metadata = match file.file_name().to_str().unwrap() {
+            "refined_manifest.json" => Metadata {
+                name: "refined_manifest".to_string(),
+                data_type: "refined_manifest_json".to_string(),
+                size: file.metadata()?.len() as u32,
+                id: None,
+                properties: HashMap::new(),
+            },
+            "RefinedPointCloud.ply" => Metadata {
+                name: "refined_pointcloud".to_string(),
+                data_type: "refined_pointcloud_ply".to_string(),
+                size: file.metadata()?.len() as u32,
+                id: None,
+                properties: HashMap::new(),
+            },
+            "BasicStitchPointCloud.ply" => Metadata {
+                name: "unrefined_pointcloud".to_string(),
+                data_type: "unrefined_pointcloud_ply".to_string(),
+                size: file.metadata()?.len() as u32,
+                id: None,
+                properties: HashMap::new(),
+            },
+            _ => continue
+        };
+        let content = fs::read(path)?;
+        uploader.push(&Data {
+            domain_id: domain_id.to_string(),
+            metadata,
+            content,
+        }).await?;
+    }
+
+    while !uploader.is_completed().await {
+        sleep(Duration::from_secs(3)).await;
+    }
+    uploader.close().await;
+    Ok(())
+}
+
 fn unzip_bytes(path: PathBuf, zip_bytes: Vec<u8>) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     println!("Starting to unzip bytes to path: {:?}", path);
     let cursor = Cursor::new(zip_bytes);
@@ -242,48 +288,19 @@ pub(crate) async fn v1(base_path: String, mut stream: Stream, mut datastore: Box
         }
     }
 
-    let mut uploader = datastore.produce(claim.domain_id.clone()).await;
-    // open output_path and upload to datastore
-    let files = fs::read_dir(output_path).expect("Failed to read output folder");
-    for file in files {
-        let file = file.unwrap();
-        let path = file.path();
-        let metadata: Metadata = match file.file_name().to_str().unwrap() {
-            "refined_manifest.json" => Metadata {
-                name: "refined_manifest".to_string(),
-                data_type: "refined_manifest_json".to_string(),
-                size: content.len() as u32,
-                id: None,
-                properties: HashMap::new(),
-            },
-            "RefinedPointCloud.ply" => Metadata {
-                name: "refined_pointcloud".to_string(),
-                data_type: "refined_pointcloud_ply".to_string(),
-                size: content.len() as u32,
-                id: None,
-                properties: HashMap::new(),
-            },
-            "BasicStitchPointCloud.ply" => Metadata {
-                name: "unrefined_pointcloud".to_string(),
-                data_type: "unrefined_pointcloud_ply".to_string(),
-                size: content.len() as u32,
-                id: None,
-                properties: HashMap::new(),
-            },
-            _ => continue
-        };
-        let content = fs::read(path).expect("Failed to read file");
-        uploader.push(&Data {
-            domain_id: claim.domain_id.clone(),
-            metadata,
-            content,
-        }).await.expect("failed to publish file");
-    }
-
-    while !uploader.is_completed().await {
-        sleep(Duration::from_secs(3)).await;
+    if let Err(e) = upload_results(&claim.domain_id, output_path, datastore).await {
+        eprintln!("Failed to upload results: {}", e);
+        t.status = task::Status::FAILED;
+        t.output = Some(task::Any {
+            type_url: "Error".to_string(),
+            value: serialize_into_vec(&task::Error {
+                message: format!("Failed to upload results: {}", e),
+            }).unwrap(),
+        });
+        let message = serialize_into_vec(t).expect("failed to serialize task update");
+        c.publish(job_id.clone(), message).await.expect("failed to publish task update");
+        return;
     }
-    uploader.close().await;
 
     let event = task::Task {
         name: claim.task_name.clone(),

From 821651c3b7ac34d2ac3606f87ab70f6c7b0d17da Mon Sep 17 00:00:00 2001
From: Shuning <shuning@aukilabs.com>
Date: Fri, 28 Mar 2025 11:26:20 +0800
Subject: [PATCH 50/89] make everything async

---
 server/src/global_refinement.rs | 26 ++++++++++++-------------
 server/src/local_refinement.rs  | 34 +++++++++++++++++----------------
 utils/data_utils.py             |  4 +---
 3 files changed, 32 insertions(+), 32 deletions(-)

diff --git a/server/src/global_refinement.rs b/server/src/global_refinement.rs
index 3f11bf8..e2c0122 100644
--- a/server/src/global_refinement.rs
+++ b/server/src/global_refinement.rs
@@ -1,4 +1,4 @@
-use std::{collections::HashMap, fs::{self, File}, io::{self, BufReader, BufWriter, Read, Write, BufRead, Cursor}, path::{Path, PathBuf}, process::{Command, Stdio}, time::SystemTime};
+use std::{collections::HashMap, fs::{self, File}, io::{self, BufReader, BufWriter, Read, Write, BufRead, Cursor}, path::{Path, PathBuf}, process::Stdio, time::SystemTime};
 use chrono::Utc;
 use domain::{cluster::DomainCluster, datastore::{self, common::Datastore, remote::RemoteDatastore}, message::read_prefix_size_message, protobuf::{domain_data::{Data, Metadata, Query},task::{self, LocalRefinementInputV1, LocalRefinementOutputV1}}};
 use jsonwebtoken::{decode, DecodingKey,Validation, Algorithm};
@@ -11,6 +11,8 @@ use futures::{stream::Zip, AsyncReadExt, StreamExt};
 use uuid::Uuid;
 use serde::{Serialize, Deserialize};
 use zip::ZipArchive;
+use tokio::io::{AsyncBufReadExt, BufReader as TokioBufReader};
+use tokio::process::Command;
 
 use crate::utils::handshake;
 
@@ -233,30 +235,28 @@ pub(crate) async fn v1(base_path: String, mut stream: Stream, mut datastore: Box
 
     // Read stdout in real-time
     if let Some(stdout) = child.stdout.take() {
-        let stdout_reader = BufReader::new(stdout);
+        let stdout_reader = TokioBufReader::new(stdout);
         tokio::spawn(async move {
-            for line in stdout_reader.lines() {
-                if let Ok(line) = line {
-                    println!("stdout: {}", line);
-                }
+            let mut lines = stdout_reader.lines();
+            while let Ok(Some(line)) = lines.next_line().await {
+                println!("stdout: {}", line);
             }
         });
     }
 
     // Read stderr in real-time
     if let Some(stderr) = child.stderr.take() {
-        let stderr_reader = BufReader::new(stderr);
+        let stderr_reader = TokioBufReader::new(stderr);
         tokio::spawn(async move {
-            for line in stderr_reader.lines() {
-                if let Ok(line) = line {
-                    eprintln!("stderr: {}", line);
-                }
+            let mut lines = stderr_reader.lines();
+            while let Ok(Some(line)) = lines.next_line().await {
+                eprintln!("stderr: {}", line);
             }
         });
     }
 
-    // Wait for the process to complete
-    match child.wait() {
+    // Wait for the process to complete (non-blocking)
+    match child.wait().await {
         Ok(status) => {
             if !status.success() {
                 eprintln!("Python process exited with status: {}", status);
diff --git a/server/src/local_refinement.rs b/server/src/local_refinement.rs
index 1bced68..caba758 100644
--- a/server/src/local_refinement.rs
+++ b/server/src/local_refinement.rs
@@ -1,4 +1,4 @@
-use std::{collections::HashMap, fs, path::Path, process::{Command, ExitCode, ExitStatus, Stdio}, time::Duration};
+use std::{collections::HashMap, fs, path::Path, process::{ExitCode, ExitStatus, Stdio}, time::Duration};
 use domain::{datastore::common::Datastore, message::read_prefix_size_message, protobuf::{domain_data::{Data, Metadata, Query},task::{self, LocalRefinementInputV1, LocalRefinementOutputV1}}};
 use libp2p::Stream;
 use networking::client::Client;
@@ -9,6 +9,8 @@ use uuid::Uuid;
 use std::io::{self, BufReader, BufWriter, Read, Write};
 use zip::{write::{FileOptions, SimpleFileOptions}, ZipWriter};
 use std::io::BufRead;
+use tokio::io::{AsyncBufReadExt, BufReader as TokioBufReader};
+use tokio::process::Command;
 
 use crate::utils::{handshake, write_scan_data_summary};
 
@@ -186,11 +188,12 @@ pub(crate) async fn v1(base_path: String, mut stream: Stream, mut datastore: Box
         "--job_id", &claim.job_id,
         "--scans", suffix.as_str(),
     ];
+    
     let child = Command::new("python3")
-    .args(params)
-    .stdout(Stdio::piped())
-    .stderr(Stdio::piped())
-    .spawn();
+        .args(params)
+        .stdout(Stdio::piped())
+        .stderr(Stdio::piped())
+        .spawn();
 
     if let Err(e) = child {
         eprintln!("Failed to execute local refinement: {}", e);
@@ -199,34 +202,33 @@ pub(crate) async fn v1(base_path: String, mut stream: Stream, mut datastore: Box
         c.publish(job_id.clone(), message).await.expect("failed to publish task update");
         return;
     }
+
     let mut child = child.unwrap();
 
     // Read stdout in real-time
     if let Some(stdout) = child.stdout.take() {
-        let stdout_reader = BufReader::new(stdout);
+        let stdout_reader = TokioBufReader::new(stdout);
         tokio::spawn(async move {
-            for line in stdout_reader.lines() {
-                if let Ok(line) = line {
-                    println!("stdout: {}", line);
-                }
+            let mut lines = stdout_reader.lines();
+            while let Ok(Some(line)) = lines.next_line().await {
+                println!("stdout: {}", line);
             }
         });
     }
 
     // Read stderr in real-time
     if let Some(stderr) = child.stderr.take() {
-        let stderr_reader = BufReader::new(stderr);
+        let stderr_reader = TokioBufReader::new(stderr);
         tokio::spawn(async move {
-            for line in stderr_reader.lines() {
-                if let Ok(line) = line {
-                    eprintln!("stderr: {}", line);
-                }
+            let mut lines = stderr_reader.lines();
+            while let Ok(Some(line)) = lines.next_line().await {
+                eprintln!("stderr: {}", line);
             }
         });
     }
 
     // Wait for the process to complete
-    match child.wait() {
+    match child.wait().await {
         Ok(status) => {
             if !status.success() {
                 eprintln!("Python process exited with status: {}", status);
diff --git a/utils/data_utils.py b/utils/data_utils.py
index ad3955b..8ccc5b8 100644
--- a/utils/data_utils.py
+++ b/utils/data_utils.py
@@ -872,12 +872,10 @@ def setup_logger(name=None, log_file=None, domain_id="", job_id="", dataset_id=N
     if log_file:
         logger, _ = add_file_handler(logger, log_file)
 
-    print("Setting up JSON formatter")
-    console_handler = logging.StreamHandler(sys.stdout)
+    console_handler = logging.StreamHandler()
     console_handler.setFormatter(JsonFormatter(datefmt='%Y-%m-%dT%H:%M:%S',
         domain_id=domain_id, job_id=job_id, dataset_id=dataset_id))
     logger.addHandler(console_handler)
-    logger.info("JSON formatter setup complete")
 
     return logger
 

From 7e89ce93207b0b7eba42a8ab41a804190b68a8bf Mon Sep 17 00:00:00 2001
From: Shuning <shuning@aukilabs.com>
Date: Fri, 28 Mar 2025 11:57:34 +0800
Subject: [PATCH 51/89] terminate global

---
 server/src/global_refinement.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/server/src/global_refinement.rs b/server/src/global_refinement.rs
index e2c0122..0a5942f 100644
--- a/server/src/global_refinement.rs
+++ b/server/src/global_refinement.rs
@@ -316,6 +316,7 @@ pub(crate) async fn v1(base_path: String, mut stream: Stream, mut datastore: Box
     c.publish(job_id.clone(), buf).await.expect("failed to publish task update");
     println!("Finished executing {}", claim.task_name);
 
+    tx.send(true).unwrap();
     let _ = heartbeat_handle.await;
     println!("Heartbeat task stopped");
     return;

From 789b0388671123204a93d70cb4ad5a33e4cc0cee Mon Sep 17 00:00:00 2001
From: Shuning <shuning@aukilabs.com>
Date: Fri, 28 Mar 2025 12:10:05 +0800
Subject: [PATCH 52/89] remove results after uploading

---
 server/Cargo.toml               | 4 ++--
 server/src/global_refinement.rs | 2 ++
 server/src/local_refinement.rs  | 1 +
 3 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/server/Cargo.toml b/server/Cargo.toml
index e680233..77d0add 100644
--- a/server/Cargo.toml
+++ b/server/Cargo.toml
@@ -5,12 +5,12 @@ edition = "2021"
 
 [dependencies]
 libp2p = { git = "https://github.com/aukilabs/rust-libp2p.git", branch = "fix/gossipsub-wasm" }
-networking = { git = "https://github.com/aukilabs/posemesh.git", branch = "chore/reconstruction-integration" }
+networking = { git = "https://github.com/aukilabs/posemesh.git", rev = "e51bd80a6ce7a00d1d4cddadce566ccddc7e4cac" }
 futures = "0.3.30"
 jsonwebtoken = "9.3.0"
 uuid = "1.13.2"
 rand = "0.9.0"
-domain = { git = "https://github.com/aukilabs/posemesh.git", branch = "chore/reconstruction-integration" }
+domain = { git = "https://github.com/aukilabs/posemesh.git", rev = "e51bd80a6ce7a00d1d4cddadce566ccddc7e4cac" }
 tracing = "0.1.40"
 tracing-subscriber = "0.3.18"
 tokio = "1.44.0"
diff --git a/server/src/global_refinement.rs b/server/src/global_refinement.rs
index 0a5942f..baaa98e 100644
--- a/server/src/global_refinement.rs
+++ b/server/src/global_refinement.rs
@@ -302,6 +302,8 @@ pub(crate) async fn v1(base_path: String, mut stream: Stream, mut datastore: Box
         return;
     }
 
+    remove_dir_all(&task_path).expect("Failed to remove task folder");
+
     let event = task::Task {
         name: claim.task_name.clone(),
         receiver: Some(claim.receiver.clone()),
diff --git a/server/src/local_refinement.rs b/server/src/local_refinement.rs
index caba758..0c20935 100644
--- a/server/src/local_refinement.rs
+++ b/server/src/local_refinement.rs
@@ -322,6 +322,7 @@ pub(crate) async fn v1(base_path: String, mut stream: Stream, mut datastore: Box
     }
     producer.close().await;
     println!("Finished uploading refined scan for {}", suffix.clone());
+    remove_dir_all(&task_folder).expect("Failed to remove task folder");
 
     let output = LocalRefinementOutputV1 {
         result_ids: vec![res.unwrap()],

From 7b255b9cd67569b7d9e2aee719fe53100285aade Mon Sep 17 00:00:00 2001
From: Shuning <shuning@aukilabs.com>
Date: Fri, 28 Mar 2025 12:32:40 +0800
Subject: [PATCH 53/89] fix rust

---
 server/src/global_refinement.rs | 2 +-
 server/src/local_refinement.rs  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/server/src/global_refinement.rs b/server/src/global_refinement.rs
index baaa98e..9de997f 100644
--- a/server/src/global_refinement.rs
+++ b/server/src/global_refinement.rs
@@ -302,7 +302,7 @@ pub(crate) async fn v1(base_path: String, mut stream: Stream, mut datastore: Box
         return;
     }
 
-    remove_dir_all(&task_path).expect("Failed to remove task folder");
+    std::fs::remove_dir_all(&task_path).expect("Failed to remove task folder");
 
     let event = task::Task {
         name: claim.task_name.clone(),
diff --git a/server/src/local_refinement.rs b/server/src/local_refinement.rs
index 0c20935..d71a487 100644
--- a/server/src/local_refinement.rs
+++ b/server/src/local_refinement.rs
@@ -322,7 +322,7 @@ pub(crate) async fn v1(base_path: String, mut stream: Stream, mut datastore: Box
     }
     producer.close().await;
     println!("Finished uploading refined scan for {}", suffix.clone());
-    remove_dir_all(&task_folder).expect("Failed to remove task folder");
+    std::fs::remove_dir_all(&task_folder).expect("Failed to remove task folder");
 
     let output = LocalRefinementOutputV1 {
         result_ids: vec![res.unwrap()],

From 7be4df33ce4585249fd0254601e00063334702fb Mon Sep 17 00:00:00 2001
From: Shuning <shuning@aukilabs.com>
Date: Fri, 28 Mar 2025 14:37:21 +0800
Subject: [PATCH 54/89] upgrade domain

---
 server/Cargo.lock               | 6 +++---
 server/Cargo.toml               | 4 ++--
 server/src/global_refinement.rs | 2 +-
 3 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/server/Cargo.lock b/server/Cargo.lock
index a742770..56d6f94 100644
--- a/server/Cargo.lock
+++ b/server/Cargo.lock
@@ -919,7 +919,7 @@ dependencies = [
 [[package]]
 name = "domain"
 version = "0.1.0"
-source = "git+https://github.com/aukilabs/posemesh.git?branch=chore%2Freconstruction-integration#7c9111ba356f33d621adccf986dc9fcc72721d83"
+source = "git+https://github.com/aukilabs/posemesh.git?branch=chore%2Freconstruction-integration#e51bd80a6ce7a00d1d4cddadce566ccddc7e4cac"
 dependencies = [
  "async-trait",
  "cbindgen",
@@ -2815,7 +2815,7 @@ dependencies = [
 [[package]]
 name = "networking"
 version = "0.1.0"
-source = "git+https://github.com/aukilabs/posemesh.git?branch=chore%2Freconstruction-integration#7c9111ba356f33d621adccf986dc9fcc72721d83"
+source = "git+https://github.com/aukilabs/posemesh.git?branch=chore%2Freconstruction-integration#e51bd80a6ce7a00d1d4cddadce566ccddc7e4cac"
 dependencies = [
  "cbindgen",
  "console_error_panic_hook",
@@ -3567,7 +3567,7 @@ dependencies = [
 [[package]]
 name = "runtime"
 version = "0.1.0"
-source = "git+https://github.com/aukilabs/posemesh.git?branch=chore%2Freconstruction-integration#7c9111ba356f33d621adccf986dc9fcc72721d83"
+source = "git+https://github.com/aukilabs/posemesh.git?branch=chore%2Freconstruction-integration#e51bd80a6ce7a00d1d4cddadce566ccddc7e4cac"
 dependencies = [
  "once_cell",
  "tokio",
diff --git a/server/Cargo.toml b/server/Cargo.toml
index 77d0add..e680233 100644
--- a/server/Cargo.toml
+++ b/server/Cargo.toml
@@ -5,12 +5,12 @@ edition = "2021"
 
 [dependencies]
 libp2p = { git = "https://github.com/aukilabs/rust-libp2p.git", branch = "fix/gossipsub-wasm" }
-networking = { git = "https://github.com/aukilabs/posemesh.git", rev = "e51bd80a6ce7a00d1d4cddadce566ccddc7e4cac" }
+networking = { git = "https://github.com/aukilabs/posemesh.git", branch = "chore/reconstruction-integration" }
 futures = "0.3.30"
 jsonwebtoken = "9.3.0"
 uuid = "1.13.2"
 rand = "0.9.0"
-domain = { git = "https://github.com/aukilabs/posemesh.git", rev = "e51bd80a6ce7a00d1d4cddadce566ccddc7e4cac" }
+domain = { git = "https://github.com/aukilabs/posemesh.git", branch = "chore/reconstruction-integration" }
 tracing = "0.1.40"
 tracing-subscriber = "0.3.18"
 tokio = "1.44.0"
diff --git a/server/src/global_refinement.rs b/server/src/global_refinement.rs
index 9de997f..feb8f07 100644
--- a/server/src/global_refinement.rs
+++ b/server/src/global_refinement.rs
@@ -155,7 +155,7 @@ pub(crate) async fn v1(base_path: String, mut stream: Stream, mut datastore: Box
         }
     });
 
-    let _cleanup = scopeguard::guard(tx, |tx| {
+    let _cleanup = scopeguard::guard(tx.clone(), |tx| {
         let _ = tx.send(true); // Signal heartbeat task to stop
     });
 

From a0d70651a4ea1c813ae594f74b35b0a23d7a1eb9 Mon Sep 17 00:00:00 2001
From: Shuning <shuning@aukilabs.com>
Date: Fri, 28 Mar 2025 14:37:55 +0800
Subject: [PATCH 55/89] try self hosted machine

---
 .github/workflows/docker-build.yml | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml
index 1f5f7ef..98c98fa 100644
--- a/.github/workflows/docker-build.yml
+++ b/.github/workflows/docker-build.yml
@@ -11,16 +11,16 @@ concurrency:
 
 jobs:
   build:
-    runs-on: ubuntu-latest
+    runs-on: self-hosted
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
 
-      - name: Free up space
-        run: |
-          sudo sudo rm -rf /usr/share/dotnet  # ~20GB
-          sudo rm -rf /opt/ghc           # ~4GB
-          sudo rm -rf /usr/local/lib/android  # ~10GB
+      # - name: Free up space
+      #   run: |
+      #     sudo sudo rm -rf /usr/share/dotnet  # ~20GB
+      #     sudo rm -rf /opt/ghc           # ~4GB
+      #     sudo rm -rf /usr/local/lib/android  # ~10GB
 
       - name: Configure AWS credentials
         uses: aws-actions/configure-aws-credentials@v4

From 8e158df0ff153540640fd583cf10df4f6de9f577 Mon Sep 17 00:00:00 2001
From: Shuning <shuning@aukilabs.com>
Date: Fri, 28 Mar 2025 14:39:21 +0800
Subject: [PATCH 56/89] revert to ubuntu latest

---
 .github/workflows/docker-build.yml | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml
index 98c98fa..1f5f7ef 100644
--- a/.github/workflows/docker-build.yml
+++ b/.github/workflows/docker-build.yml
@@ -11,16 +11,16 @@ concurrency:
 
 jobs:
   build:
-    runs-on: self-hosted
+    runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
 
-      # - name: Free up space
-      #   run: |
-      #     sudo sudo rm -rf /usr/share/dotnet  # ~20GB
-      #     sudo rm -rf /opt/ghc           # ~4GB
-      #     sudo rm -rf /usr/local/lib/android  # ~10GB
+      - name: Free up space
+        run: |
+          sudo sudo rm -rf /usr/share/dotnet  # ~20GB
+          sudo rm -rf /opt/ghc           # ~4GB
+          sudo rm -rf /usr/local/lib/android  # ~10GB
 
       - name: Configure AWS credentials
         uses: aws-actions/configure-aws-credentials@v4

From 07ec373c50960c2808f7fca003d97d6898b848f0 Mon Sep 17 00:00:00 2001
From: Shuning <shuning@aukilabs.com>
Date: Fri, 28 Mar 2025 15:20:24 +0800
Subject: [PATCH 57/89] use self hosted runner

---
 .github/workflows/docker-build.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml
index 1f5f7ef..ec80b9c 100644
--- a/.github/workflows/docker-build.yml
+++ b/.github/workflows/docker-build.yml
@@ -11,7 +11,7 @@ concurrency:
 
 jobs:
   build:
-    runs-on: ubuntu-latest
+    runs-on: [self-hosted, Linux]
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4

From bf2d8a9a7fb21392ee52a7b07c92e7209eff93c0 Mon Sep 17 00:00:00 2001
From: Shuning <shuning@aukilabs.com>
Date: Mon, 31 Mar 2025 13:53:31 +0800
Subject: [PATCH 58/89] upgrade domain module so it handles backpressure

---
 server/Cargo.lock | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/server/Cargo.lock b/server/Cargo.lock
index 56d6f94..b38cbb2 100644
--- a/server/Cargo.lock
+++ b/server/Cargo.lock
@@ -919,7 +919,7 @@ dependencies = [
 [[package]]
 name = "domain"
 version = "0.1.0"
-source = "git+https://github.com/aukilabs/posemesh.git?branch=chore%2Freconstruction-integration#e51bd80a6ce7a00d1d4cddadce566ccddc7e4cac"
+source = "git+https://github.com/aukilabs/posemesh.git?branch=chore%2Freconstruction-integration#9e87fafe17435a9ee92b55d4c3c4164625d9ed40"
 dependencies = [
  "async-trait",
  "cbindgen",
@@ -2815,7 +2815,7 @@ dependencies = [
 [[package]]
 name = "networking"
 version = "0.1.0"
-source = "git+https://github.com/aukilabs/posemesh.git?branch=chore%2Freconstruction-integration#e51bd80a6ce7a00d1d4cddadce566ccddc7e4cac"
+source = "git+https://github.com/aukilabs/posemesh.git?branch=chore%2Freconstruction-integration#9e87fafe17435a9ee92b55d4c3c4164625d9ed40"
 dependencies = [
  "cbindgen",
  "console_error_panic_hook",
@@ -3567,7 +3567,7 @@ dependencies = [
 [[package]]
 name = "runtime"
 version = "0.1.0"
-source = "git+https://github.com/aukilabs/posemesh.git?branch=chore%2Freconstruction-integration#e51bd80a6ce7a00d1d4cddadce566ccddc7e4cac"
+source = "git+https://github.com/aukilabs/posemesh.git?branch=chore%2Freconstruction-integration#9e87fafe17435a9ee92b55d4c3c4164625d9ed40"
 dependencies = [
  "once_cell",
  "tokio",

From bfa1fcf69959c824bf73c4b5b8e6d70ff5d9cc75 Mon Sep 17 00:00:00 2001
From: Shuning <shuning@aukilabs.com>
Date: Mon, 31 Mar 2025 13:53:56 +0800
Subject: [PATCH 59/89] disable space freeing up

---
 .github/workflows/docker-build.yml | 10 +++++-----
 server/Cargo.lock                  |  6 +++---
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml
index ec80b9c..55d2250 100644
--- a/.github/workflows/docker-build.yml
+++ b/.github/workflows/docker-build.yml
@@ -16,11 +16,11 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@v4
 
-      - name: Free up space
-        run: |
-          sudo sudo rm -rf /usr/share/dotnet  # ~20GB
-          sudo rm -rf /opt/ghc           # ~4GB
-          sudo rm -rf /usr/local/lib/android  # ~10GB
+      # - name: Free up space
+      #   run: |
+      #     sudo sudo rm -rf /usr/share/dotnet  # ~20GB
+      #     sudo rm -rf /opt/ghc           # ~4GB
+      #     sudo rm -rf /usr/local/lib/android  # ~10GB
 
       - name: Configure AWS credentials
         uses: aws-actions/configure-aws-credentials@v4
diff --git a/server/Cargo.lock b/server/Cargo.lock
index b38cbb2..53574c8 100644
--- a/server/Cargo.lock
+++ b/server/Cargo.lock
@@ -919,7 +919,7 @@ dependencies = [
 [[package]]
 name = "domain"
 version = "0.1.0"
-source = "git+https://github.com/aukilabs/posemesh.git?branch=chore%2Freconstruction-integration#9e87fafe17435a9ee92b55d4c3c4164625d9ed40"
+source = "git+https://github.com/aukilabs/posemesh.git?branch=chore%2Freconstruction-integration#c25181b59861d41025e86cd400417edc51954995"
 dependencies = [
  "async-trait",
  "cbindgen",
@@ -2815,7 +2815,7 @@ dependencies = [
 [[package]]
 name = "networking"
 version = "0.1.0"
-source = "git+https://github.com/aukilabs/posemesh.git?branch=chore%2Freconstruction-integration#9e87fafe17435a9ee92b55d4c3c4164625d9ed40"
+source = "git+https://github.com/aukilabs/posemesh.git?branch=chore%2Freconstruction-integration#c25181b59861d41025e86cd400417edc51954995"
 dependencies = [
  "cbindgen",
  "console_error_panic_hook",
@@ -3567,7 +3567,7 @@ dependencies = [
 [[package]]
 name = "runtime"
 version = "0.1.0"
-source = "git+https://github.com/aukilabs/posemesh.git?branch=chore%2Freconstruction-integration#9e87fafe17435a9ee92b55d4c3c4164625d9ed40"
+source = "git+https://github.com/aukilabs/posemesh.git?branch=chore%2Freconstruction-integration#c25181b59861d41025e86cd400417edc51954995"
 dependencies = [
  "once_cell",
  "tokio",

From 8ec355beb8734ff6e5445f23560cb7cd89878311 Mon Sep 17 00:00:00 2001
From: Shuning <shuning@aukilabs.com>
Date: Tue, 1 Apr 2025 10:35:39 +0800
Subject: [PATCH 60/89] update domain module

---
 server/Cargo.lock | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/server/Cargo.lock b/server/Cargo.lock
index 53574c8..290fab9 100644
--- a/server/Cargo.lock
+++ b/server/Cargo.lock
@@ -919,7 +919,7 @@ dependencies = [
 [[package]]
 name = "domain"
 version = "0.1.0"
-source = "git+https://github.com/aukilabs/posemesh.git?branch=chore%2Freconstruction-integration#c25181b59861d41025e86cd400417edc51954995"
+source = "git+https://github.com/aukilabs/posemesh.git?branch=chore%2Freconstruction-integration#5036029e878bb67c034da4305108febe155fb860"
 dependencies = [
  "async-trait",
  "cbindgen",
@@ -2815,7 +2815,7 @@ dependencies = [
 [[package]]
 name = "networking"
 version = "0.1.0"
-source = "git+https://github.com/aukilabs/posemesh.git?branch=chore%2Freconstruction-integration#c25181b59861d41025e86cd400417edc51954995"
+source = "git+https://github.com/aukilabs/posemesh.git?branch=chore%2Freconstruction-integration#5036029e878bb67c034da4305108febe155fb860"
 dependencies = [
  "cbindgen",
  "console_error_panic_hook",
@@ -3567,7 +3567,7 @@ dependencies = [
 [[package]]
 name = "runtime"
 version = "0.1.0"
-source = "git+https://github.com/aukilabs/posemesh.git?branch=chore%2Freconstruction-integration#c25181b59861d41025e86cd400417edc51954995"
+source = "git+https://github.com/aukilabs/posemesh.git?branch=chore%2Freconstruction-integration#5036029e878bb67c034da4305108febe155fb860"
 dependencies = [
  "once_cell",
  "tokio",

From b752d0c0e5e866813a62d71268b6d60a8f2fa8da Mon Sep 17 00:00:00 2001
From: Shuning <shuning@aukilabs.com>
Date: Tue, 1 Apr 2025 15:13:33 +0800
Subject: [PATCH 61/89] extract file name properly

---
 server/src/global_refinement.rs | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/server/src/global_refinement.rs b/server/src/global_refinement.rs
index feb8f07..aa4e090 100644
--- a/server/src/global_refinement.rs
+++ b/server/src/global_refinement.rs
@@ -71,10 +71,15 @@ fn unzip_bytes(path: PathBuf, zip_bytes: Vec<u8>) -> Result<(), Box<dyn std::err
     for i in 0..archive.len() {
         println!("Processing file {}/{}", i + 1, archive.len());
         let mut input_file = archive.by_index(i)?;
-        let file_name = input_file.name().to_string();
+        let file_name = input_file.enclosed_name()
+            .ok_or("Invalid file name in zip")?
+            .file_name()
+            .ok_or("Invalid file name")?
+            .to_str()
+            .ok_or("Invalid UTF-8 in file name")?;
         println!("Extracting file: {}", file_name);
         
-        let file_path = path.join(&file_name);
+        let file_path = path.join(file_name);
         
         // Create parent directory if it doesn't exist
         if let Some(parent) = file_path.parent() {

From 40872eb59ad97b62341675477738d8d5cb34d635 Mon Sep 17 00:00:00 2001
From: Shuning <shuning@aukilabs.com>
Date: Tue, 1 Apr 2025 15:23:49 +0800
Subject: [PATCH 62/89] self hosted stops working again

---
 .github/workflows/docker-build.yml | 12 ++++++------
 server/Cargo.lock                  | 14 ++++++++++++++
 2 files changed, 20 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml
index 55d2250..1f5f7ef 100644
--- a/.github/workflows/docker-build.yml
+++ b/.github/workflows/docker-build.yml
@@ -11,16 +11,16 @@ concurrency:
 
 jobs:
   build:
-    runs-on: [self-hosted, Linux]
+    runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
 
-      # - name: Free up space
-      #   run: |
-      #     sudo sudo rm -rf /usr/share/dotnet  # ~20GB
-      #     sudo rm -rf /opt/ghc           # ~4GB
-      #     sudo rm -rf /usr/local/lib/android  # ~10GB
+      - name: Free up space
+        run: |
+          sudo sudo rm -rf /usr/share/dotnet  # ~20GB
+          sudo rm -rf /opt/ghc           # ~4GB
+          sudo rm -rf /usr/local/lib/android  # ~10GB
 
       - name: Configure AWS credentials
         uses: aws-actions/configure-aws-credentials@v4
diff --git a/server/Cargo.lock b/server/Cargo.lock
index 290fab9..163cb56 100644
--- a/server/Cargo.lock
+++ b/server/Cargo.lock
@@ -2836,6 +2836,7 @@ dependencies = [
  "tracing",
  "tracing-subscriber",
  "tracing-wasm",
+ "utils",
  "wasm-bindgen",
  "wasm-bindgen-futures",
 ]
@@ -4567,6 +4568,19 @@ version = "0.2.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
 
+[[package]]
+name = "utils"
+version = "0.1.0"
+source = "git+https://github.com/aukilabs/posemesh.git?branch=chore%2Freconstruction-integration#5036029e878bb67c034da4305108febe155fb860"
+dependencies = [
+ "js-sys",
+ "tokio",
+ "tracing",
+ "wasm-bindgen",
+ "wasm-bindgen-futures",
+ "web-sys",
+]
+
 [[package]]
 name = "uuid"
 version = "1.15.1"

From 2de622c25ae0d734487cdb7b180e2334f4daef0c Mon Sep 17 00:00:00 2001
From: Shuning <shuning@aukilabs.com>
Date: Tue, 1 Apr 2025 15:25:47 +0800
Subject: [PATCH 63/89] fix file name

---
 server/src/global_refinement.rs | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/server/src/global_refinement.rs b/server/src/global_refinement.rs
index aa4e090..713c059 100644
--- a/server/src/global_refinement.rs
+++ b/server/src/global_refinement.rs
@@ -71,12 +71,8 @@ fn unzip_bytes(path: PathBuf, zip_bytes: Vec<u8>) -> Result<(), Box<dyn std::err
     for i in 0..archive.len() {
         println!("Processing file {}/{}", i + 1, archive.len());
         let mut input_file = archive.by_index(i)?;
-        let file_name = input_file.enclosed_name()
-            .ok_or("Invalid file name in zip")?
-            .file_name()
-            .ok_or("Invalid file name")?
-            .to_str()
-            .ok_or("Invalid UTF-8 in file name")?;
+        let file_name = input_file.enclosed_name().unwrap();
+        let file_name = file_name.file_name().unwrap().to_str().unwrap();
         println!("Extracting file: {}", file_name);
         
         let file_path = path.join(file_name);

From 3765cd5aed7ea2f7c7f9545ac93293fa61016a2e Mon Sep 17 00:00:00 2001
From: Shuning <shuning@aukilabs.com>
Date: Tue, 1 Apr 2025 15:26:34 +0800
Subject: [PATCH 64/89] try self hosted linux again

---
 .github/workflows/docker-build.yml | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml
index 1f5f7ef..55d2250 100644
--- a/.github/workflows/docker-build.yml
+++ b/.github/workflows/docker-build.yml
@@ -11,16 +11,16 @@ concurrency:
 
 jobs:
   build:
-    runs-on: ubuntu-latest
+    runs-on: [self-hosted, Linux]
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
 
-      - name: Free up space
-        run: |
-          sudo sudo rm -rf /usr/share/dotnet  # ~20GB
-          sudo rm -rf /opt/ghc           # ~4GB
-          sudo rm -rf /usr/local/lib/android  # ~10GB
+      # - name: Free up space
+      #   run: |
+      #     sudo sudo rm -rf /usr/share/dotnet  # ~20GB
+      #     sudo rm -rf /opt/ghc           # ~4GB
+      #     sudo rm -rf /usr/local/lib/android  # ~10GB
 
       - name: Configure AWS credentials
         uses: aws-actions/configure-aws-credentials@v4

From d2d043141f692f49708838816aabcb28cc3357ec Mon Sep 17 00:00:00 2001
From: Shuning <shuning@aukilabs.com>
Date: Tue, 1 Apr 2025 15:43:25 +0800
Subject: [PATCH 65/89] use github runner

---
 .github/workflows/docker-build.yml | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml
index 55d2250..1f5f7ef 100644
--- a/.github/workflows/docker-build.yml
+++ b/.github/workflows/docker-build.yml
@@ -11,16 +11,16 @@ concurrency:
 
 jobs:
   build:
-    runs-on: [self-hosted, Linux]
+    runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
 
-      # - name: Free up space
-      #   run: |
-      #     sudo sudo rm -rf /usr/share/dotnet  # ~20GB
-      #     sudo rm -rf /opt/ghc           # ~4GB
-      #     sudo rm -rf /usr/local/lib/android  # ~10GB
+      - name: Free up space
+        run: |
+          sudo sudo rm -rf /usr/share/dotnet  # ~20GB
+          sudo rm -rf /opt/ghc           # ~4GB
+          sudo rm -rf /usr/local/lib/android  # ~10GB
 
       - name: Configure AWS credentials
         uses: aws-actions/configure-aws-credentials@v4

From 4606c4876989326fdcf2ab6784cd8801cfb09117 Mon Sep 17 00:00:00 2001
From: RobinLindh <robin@aukilabs.com>
Date: Wed, 2 Apr 2025 08:12:19 +0200
Subject: [PATCH 66/89] use scans parameter to fix issue with parallel
 refinements

---
 main.py | 21 +--------------------
 1 file changed, 1 insertion(+), 20 deletions(-)

diff --git a/main.py b/main.py
index c544b66..53a660a 100644
--- a/main.py
+++ b/main.py
@@ -116,21 +116,6 @@ def local_and_global_main_wrapper(args, logger):
     #occlusion_box_wrapper(ply_output_path, global_out_folder / "occlusion", logger)
 
 
-def get_available_scans(datasets_path):
-    """Get list of available scans in the datasets directory.
-    
-    Args:
-        datasets_path: Path to datasets directory
-        
-    Returns:
-        List of scan names
-    """
-    return [
-        scan.name for scan in datasets_path.iterdir()
-        if scan.is_dir() or scan.suffix == ".zip"
-    ]
-
-
 def process_refinement(args, logger):
     """Process refinement based on specified mode.
     
@@ -182,10 +167,6 @@ def main(args):
         level=args.log_level
     )
 
-    # TODO: ignoring the scans parameter from go for now since it's incorrect (fix after redeploy)
-    # Get available scans from datasets directory
-    args.scans = get_available_scans(args.job_root_path / "datasets")
-
     try:
         process_refinement(args, logger)
     except Exception as e:
@@ -204,7 +185,7 @@ def parse_args():
         help="Set the logging level (default: INFO)"
     )
 
-    parser.add_argument("--scans", nargs="+", default=[], help="List of scans to process")
+    parser.add_argument("--scans", nargs="+", help="List of scans to process")
     return parser.parse_args()
 
 

From b316079e9736558bb0cfa0e07b78e972a2f674da Mon Sep 17 00:00:00 2001
From: RobinLindh <robin@aukilabs.com>
Date: Wed, 2 Apr 2025 08:13:47 +0200
Subject: [PATCH 67/89] catch error to remove task folder

---
 server/src/local_refinement.rs | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/server/src/local_refinement.rs b/server/src/local_refinement.rs
index d71a487..39c5d83 100644
--- a/server/src/local_refinement.rs
+++ b/server/src/local_refinement.rs
@@ -322,7 +322,9 @@ pub(crate) async fn v1(base_path: String, mut stream: Stream, mut datastore: Box
     }
     producer.close().await;
     println!("Finished uploading refined scan for {}", suffix.clone());
-    std::fs::remove_dir_all(&task_folder).expect("Failed to remove task folder");
+    if let Err(e) = std::fs::remove_dir_all(&task_folder) {
+        eprintln!("Failed to remove task folder: {:?}: {}", task_folder, e);
+    }
 
     let output = LocalRefinementOutputV1 {
         result_ids: vec![res.unwrap()],

From c62037ba0ebd8e1dbb8f108babc98bf071aedbb4 Mon Sep 17 00:00:00 2001
From: RobinLindh <robin@aukilabs.com>
Date: Wed, 2 Apr 2025 08:16:57 +0200
Subject: [PATCH 68/89] no image timestamps in global refinement

Worked before when running everything on same machine.
We don't save them to domain data from local refinements.
We don't use them anyway in the global refinement
---
 utils/dataset_utils.py | 14 --------------
 1 file changed, 14 deletions(-)

diff --git a/utils/dataset_utils.py b/utils/dataset_utils.py
index 96f7ac6..1b557a6 100644
--- a/utils/dataset_utils.py
+++ b/utils/dataset_utils.py
@@ -45,7 +45,6 @@ def __init__(self, message='No overlaps!'):
 class StitchingData:
     detections_per_qr: Dict[str, List[pycolmap.Rigid3d]] = None
     image_ids_per_qr: Dict[str, List[int]] = None
-    timestamp_per_image: Dict[str, int] = None
     arkit_precomputed: Dict = None
     placed_portal: Dict[str, pycolmap.Rigid3d] = None
     chunks_image_ids: List[List[int]] = None
@@ -55,7 +54,6 @@ class StitchingData:
     def __post_init__(self):
         self.detections_per_qr = self.detections_per_qr or {}
         self.image_ids_per_qr = self.image_ids_per_qr or {}
-        self.timestamp_per_image = self.timestamp_per_image or {}
         self.arkit_precomputed = self.arkit_precomputed or {}
         self.placed_portal = self.placed_portal or {}
         self.chunks_image_ids = self.chunks_image_ids or []
@@ -208,11 +206,6 @@ def load_partial(
     if loaded_rec is None:
         return stitch_data
     
-    # Load frame timestamps
-    timestamp_chunk = _load_frame_timestamps(unzip_folder, logger)
-    if not timestamp_chunk:
-        return stitch_data
-    
     # Extract and process QR code detections
     qr_detections = _process_qr_detections(loaded_rec)
 
@@ -243,7 +236,6 @@ def load_partial(
         loaded_rec,
         alignment_transform,
         qr_detections,
-        timestamp_chunk,
         stitch_data,
         with_3dpoints,
         logger
@@ -367,7 +359,6 @@ def _process_reconstruction(
     loaded_rec: Model,
     alignment_transform: Optional[pycolmap.Rigid3d],
     qr_detections: List[Dict],
-    timestamp_chunk: Dict[str, int],
     stitch_data: StitchingData,
     with_3dpoints: bool,
     logger
@@ -437,11 +428,6 @@ def _process_reconstruction(
                 element.image_id = image_id_old_to_new[element.image_id]
                 stitch_data.combined_rec.add_observation(point3D_id_new, element)
 
-    # Update timestamps
-    for filename, timestamp in timestamp_chunk.items():
-        assert filename not in stitch_data.timestamp_per_image
-        stitch_data.timestamp_per_image[filename] = timestamp
-
     # Process sorted image IDs
     sorted_image_ids = sorted(list(rec2.images.keys()))
     stitch_data.chunks_image_ids.append(sorted_image_ids)

From 2a1cecb7b255e6b40950376d05b3f5ea11b894f6 Mon Sep 17 00:00:00 2001
From: RobinLindh <robin@aukilabs.com>
Date: Thu, 3 Apr 2025 00:35:48 +0200
Subject: [PATCH 69/89] use same dockerfile for dev container

avoids duplication and extra work keeping dockerfiles synced
---
 .devcontainer/build_golang_server.sh |   5 -
 .devcontainer/build_server.sh        |   4 +
 .devcontainer/devcontainer.json      |   8 +-
 docker/Dockerfile                    |  37 +++++--
 docker/Dockerfile.dev                | 141 ---------------------------
 5 files changed, 40 insertions(+), 155 deletions(-)
 delete mode 100755 .devcontainer/build_golang_server.sh
 create mode 100644 .devcontainer/build_server.sh
 delete mode 100644 docker/Dockerfile.dev

diff --git a/.devcontainer/build_golang_server.sh b/.devcontainer/build_golang_server.sh
deleted file mode 100755
index 9b3253a..0000000
--- a/.devcontainer/build_golang_server.sh
+++ /dev/null
@@ -1,5 +0,0 @@
-#!/bin/bash
-
-cd server && go build
-
-cp reconstruction ../
\ No newline at end of file
diff --git a/.devcontainer/build_server.sh b/.devcontainer/build_server.sh
new file mode 100644
index 0000000..42c1e98
--- /dev/null
+++ b/.devcontainer/build_server.sh
@@ -0,0 +1,4 @@
+#!/bin/bash
+
+cd server && cargo build --release
+cp target/release/server ../reconstruction
diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json
index dfc53f4..9e6b1ed 100644
--- a/.devcontainer/devcontainer.json
+++ b/.devcontainer/devcontainer.json
@@ -2,7 +2,11 @@
     "name": "Reconstruction Server Devcontainer",
     "remoteUser": "reconstruction-server",
     "build": {
-        "dockerfile": "../docker/Dockerfile.dev"
+        "dockerfile": "../docker/Dockerfile",
+        "context": "../",
+        "args": {
+            "IS_DEVELOPMENT": "true"
+        }
     },
     "workspaceFolder": "/app",
     "workspaceMount": "source=${localWorkspaceFolder},target=/app,type=bind",
@@ -32,5 +36,5 @@
         // Uncomment if you want display forward
         // "source=/tmp/.X11-unix,target=/tmp/.X11-unix,type=bind,consistency=cached"
     ],
-    "postCreateCommand": "chown -R reconstruction-server:reconstruction-server /app && .devcontainer/build_golang_server.sh"
+    "postCreateCommand": "sudo chown -R reconstruction-server:reconstruction-server /app && .devcontainer/build_server.sh"
 }
\ No newline at end of file
diff --git a/docker/Dockerfile b/docker/Dockerfile
index 4bd7874..956dc30 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -1,23 +1,23 @@
+ARG IS_DEVELOPMENT="false"
+
 ##
 ## Rust build for server
 ##
 FROM rust:1.84-slim-bullseye AS rust-build
-ARG VERSION
 
 RUN apt-get update && apt-get install -y \
     git \
     && rm -rf /var/lib/apt/lists/*
 
 WORKDIR /app
+RUN mkdir -p /app/server
 COPY server /app/server
-
-# Build the application
 RUN cd server && cargo build --release
 
 ##
 ## Runtime image
 ##
-FROM nvidia/cuda:11.0.3-base-ubuntu20.04
+FROM nvidia/cuda:11.0.3-base-ubuntu20.04 AS runtime
 
 ARG USERNAME=reconstruction-server
 ARG USER_UID=1000
@@ -165,10 +165,33 @@ RUN echo "/usr/local/lib" > /etc/ld.so.conf.d/local.conf && ldconfig
 
 # Run reconstruction server as separate user, not root
 RUN groupadd --gid $USER_GID $USERNAME \
-    && useradd --uid $USER_UID --gid $USER_GID -m $USERNAME
+    && useradd --uid $USER_UID --gid $USER_GID -m $USERNAME \
+    && chown -R $USERNAME:$USERNAME /app
 
 RUN mkdir -p /app/jobs && chown -R reconstruction-server:reconstruction-server /app/jobs
 RUN mkdir -p /app/volume && chown -R reconstruction-server:reconstruction-server /app/volume
-USER $USERNAME
-# Copy the built binary from rust-build
+# Dev container: Copy rust tools so we can compile inside the dev container
+FROM runtime AS reconstruction_dev
+
+# Sudo enabled in dev container only
+RUN apt-get install -y sudo \
+    && echo $USERNAME ALL=\(root\) NOPASSWD:ALL > /etc/sudoers.d/$USERNAME \
+    && chmod 0440 /etc/sudoers.d/$USERNAME
+
+COPY --from=rust-build /usr/local/rustup /usr/local/rustup
+COPY --from=rust-build /usr/local/cargo /usr/local/cargo
+ENV PATH="/usr/local/cargo/bin:$PATH"
+ENV RUSTUP_HOME="/usr/local/rustup"
+ENV CARGO_HOME="/usr/local/cargo"
+
+# Standalone image: Use the compiled server binary
+FROM runtime AS reconstruction
 COPY --from=rust-build /app/server/target/release/server /app/server
+RUN chmod +x /app/server
+
+# Final stage: Select between dev and prod
+FROM reconstruction${IS_DEVELOPMENT:+_dev} AS final
+
+WORKDIR /app
+USER $USERNAME
+
diff --git a/docker/Dockerfile.dev b/docker/Dockerfile.dev
deleted file mode 100644
index b7c0e2c..0000000
--- a/docker/Dockerfile.dev
+++ /dev/null
@@ -1,141 +0,0 @@
-FROM nvidia/cuda:11.0.3-base-ubuntu20.04
-
-ARG USERNAME=reconstruction-server
-ARG USER_UID=1000
-ARG USER_GID=$USER_UID
-ARG DEBIAN_FRONTEND=noninteractive
-
-RUN apt-get update && apt-get install -y \
-    wget \
-    curl \
-    vim \
-    git \
-    nano \
-    jq \
-    less \
-    cmake \
-    autoconf \
-    automake \
-    libtool \
-    libffi-dev \
-    ninja-build \
-    build-essential \
-    libboost-program-options-dev \
-    libboost-filesystem-dev \
-    libboost-graph-dev \
-    libboost-system-dev \
-    libeigen3-dev \
-    libflann-dev \
-    libfreeimage-dev \
-    libmetis-dev \
-    libgtest-dev \
-    libsqlite3-dev \
-    libglew-dev \
-    qtbase5-dev \
-    libqt5opengl5-dev \
-    libcgal-dev \
-    libsuitesparse-dev \
-    python3-pip \
-    python3-tk
-
-RUN apt-get install -y --no-install-recommends \
-    ssh \
-    netcat-openbsd \
-    ca-certificates \
-    iproute2 \
-    iputils-ping \
-    bind9-dnsutils
-
-RUN pip install --upgrade pip setuptools wheel
-
-RUN wget https://cmake.org/files/v3.27/cmake-3.27.9-linux-x86_64.tar.gz \
-    && tar xvf cmake-3.27.9-linux-x86_64.tar.gz \
-    && cd cmake-3.27.9-linux-x86_64 \
-    && cp -r bin /usr/ \
-    && cp -r share /usr/ \
-    && cp -r doc /usr/share/ \
-    && cp -r man /usr/share/ \
-    && cd .. \
-    && rm -rf cmake*
-
-RUN git clone https://github.com/google/glog.git && cd glog && \
-    git checkout tags/v0.6.0 && \
-    cmake -S . -B build -G "Unix Makefiles" && \
-    cmake --build build --target install
-
-RUN pip install "pybind11[global]==2.12.0"
-
-RUN pip install psutil gputil
-
-WORKDIR /src
-
-RUN git clone https://github.com/NVIDIA/libglvnd && \
-    cd libglvnd && \
-    ./autogen.sh && \
-    ./configure && \
-    make -j4 && \
-    make install
-
-# Ceres
-RUN git clone --recursive https://ceres-solver.googlesource.com/ceres-solver \
-    && cd ceres-solver \
-    && git checkout tags/2.2.0 \
-    && mkdir build \
-    && cd build \
-    && cmake .. -DBUILD_TESTING=OFF -DBUILD_EXAMPLES=OFF \
-    && make -j4 \
-    && make install
-
-# pyCeres
-RUN git clone https://github.com/cvg/pyceres.git \
-    && cd pyceres \
-    && git checkout tags/v2.3 \
-    && python3 -m pip install -e .
-
-# COLMAP
-RUN git clone https://github.com/colmap/colmap.git && \
-    cd colmap \
-    && git checkout release/3.10 \
-    && mkdir build \
-    && cd build \
-    && cmake -DCUDA_ENABLED=ON -DCMAKE_CUDA_ARCHITECTURES="60;61;70;75;80;86;89" .. \
-    && make -j 4 \
-    && make install \
-    && cd ../pycolmap \
-    && python3 -m pip install -e .
-
-# Hloc
-RUN git clone --recursive https://github.com/cvg/Hierarchical-Localization && \
-    cd Hierarchical-Localization && \
-    git checkout --recurse-submodules abb252080282e31147db6291206ca102c43353f7 && \
-    python3 -m pip install -e . --config-settings editable_mode=compat && \
-    python3 -m pip install --upgrade plotly
-
-# Hloc installs latest opencv-python as dependency,
-# but the latest opencv 4.11.0.86 introduced a bug
-# causing unpacked mp4 frames to be rotated wrong.
-# For now we need to downgrade opencv (after hloc).
-RUN python3 -m pip install opencv-python==4.10.0.84
-
-RUN python3 -m pip install enlighten evo
-
-RUN python3 -m pip install open3d trimesh alphashape
-
-# Server Stuff
-COPY --from=golang:1.21 /usr/local/go/ /usr/local/go/
-ENV PATH="$PATH:/usr/local/go/bin"
-
-# User Stuff
-RUN groupadd --gid $USER_GID $USERNAME \
-    && useradd --uid $USER_UID --gid $USER_GID -m $USERNAME \
-    #
-    # [Optional] Add sudo support. Omit if you don't need to install software after connecting.
-    && apt-get update \
-    && apt-get install -y sudo \
-    && echo $USERNAME ALL=\(root\) NOPASSWD:ALL > /etc/sudoers.d/$USERNAME \
-    && chmod 0440 /etc/sudoers.d/$USERNAME
-
-USER $USERNAME
-
-WORKDIR /app
-ENTRYPOINT /bin/bash
\ No newline at end of file

From 465def8db2f3c44de740d6e4edce09e3f397cde9 Mon Sep 17 00:00:00 2001
From: Shuning <shuning@aukilabs.com>
Date: Thu, 3 Apr 2025 14:20:03 +0800
Subject: [PATCH 70/89] update domain module

---
 server/Cargo.lock | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/server/Cargo.lock b/server/Cargo.lock
index 163cb56..2a3e936 100644
--- a/server/Cargo.lock
+++ b/server/Cargo.lock
@@ -919,7 +919,7 @@ dependencies = [
 [[package]]
 name = "domain"
 version = "0.1.0"
-source = "git+https://github.com/aukilabs/posemesh.git?branch=chore%2Freconstruction-integration#5036029e878bb67c034da4305108febe155fb860"
+source = "git+https://github.com/aukilabs/posemesh.git?branch=chore%2Freconstruction-integration#ee37586e6cad8b2c910ec29152b0d939e5e80e64"
 dependencies = [
  "async-trait",
  "cbindgen",
@@ -2815,7 +2815,7 @@ dependencies = [
 [[package]]
 name = "networking"
 version = "0.1.0"
-source = "git+https://github.com/aukilabs/posemesh.git?branch=chore%2Freconstruction-integration#5036029e878bb67c034da4305108febe155fb860"
+source = "git+https://github.com/aukilabs/posemesh.git?branch=chore%2Freconstruction-integration#ee37586e6cad8b2c910ec29152b0d939e5e80e64"
 dependencies = [
  "cbindgen",
  "console_error_panic_hook",
@@ -3568,7 +3568,7 @@ dependencies = [
 [[package]]
 name = "runtime"
 version = "0.1.0"
-source = "git+https://github.com/aukilabs/posemesh.git?branch=chore%2Freconstruction-integration#5036029e878bb67c034da4305108febe155fb860"
+source = "git+https://github.com/aukilabs/posemesh.git?branch=chore%2Freconstruction-integration#ee37586e6cad8b2c910ec29152b0d939e5e80e64"
 dependencies = [
  "once_cell",
  "tokio",
@@ -4571,8 +4571,10 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
 [[package]]
 name = "utils"
 version = "0.1.0"
-source = "git+https://github.com/aukilabs/posemesh.git?branch=chore%2Freconstruction-integration#5036029e878bb67c034da4305108febe155fb860"
+source = "git+https://github.com/aukilabs/posemesh.git?branch=chore%2Freconstruction-integration#ee37586e6cad8b2c910ec29152b0d939e5e80e64"
 dependencies = [
+ "futures",
+ "gloo-timers 0.3.0",
  "js-sys",
  "tokio",
  "tracing",

From e9c1fd26d71cccdadf9387d4070f0c6d16fb7e80 Mon Sep 17 00:00:00 2001
From: Shuning <shuning@aukilabs.com>
Date: Tue, 8 Apr 2025 13:44:28 +0800
Subject: [PATCH 71/89] use feature/data-node branch

---
 server/Cargo.lock               | 159 +++++++++++++-
 server/Cargo.toml               |   5 +-
 server/src/global_refinement.rs |  38 ++--
 server/src/local_refinement.rs  | 355 ++++++++++++--------------------
 server/src/main.rs              |  17 +-
 server/src/utils.rs             |  81 ++++++--
 6 files changed, 383 insertions(+), 272 deletions(-)

diff --git a/server/Cargo.lock b/server/Cargo.lock
index 2a3e936..fc1d64c 100644
--- a/server/Cargo.lock
+++ b/server/Cargo.lock
@@ -919,23 +919,27 @@ dependencies = [
 [[package]]
 name = "domain"
 version = "0.1.0"
-source = "git+https://github.com/aukilabs/posemesh.git?branch=chore%2Freconstruction-integration#ee37586e6cad8b2c910ec29152b0d939e5e80e64"
+source = "git+https://github.com/aukilabs/posemesh.git?branch=feature%2Fdata-node#36aae24b3e6eb4e087509ef371e64a0e667997cc"
 dependencies = [
  "async-trait",
  "cbindgen",
  "console_error_panic_hook",
  "futures",
+ "hex",
  "jsonwebtoken",
  "libp2p",
  "networking",
  "pb-rs",
  "quick-protobuf",
  "quick-protobuf-codec 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rs_merkle",
  "runtime",
  "serde",
  "serde-wasm-bindgen",
  "serde_json",
+ "sha2",
  "tokio",
+ "tokio-postgres",
  "tracing",
  "tracing-subscriber",
  "tracing-wasm",
@@ -1078,6 +1082,12 @@ dependencies = [
  "pin-project-lite",
 ]
 
+[[package]]
+name = "fallible-iterator"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7"
+
 [[package]]
 name = "fastrand"
 version = "2.3.0"
@@ -2815,7 +2825,7 @@ dependencies = [
 [[package]]
 name = "networking"
 version = "0.1.0"
-source = "git+https://github.com/aukilabs/posemesh.git?branch=chore%2Freconstruction-integration#ee37586e6cad8b2c910ec29152b0d939e5e80e64"
+source = "git+https://github.com/aukilabs/posemesh.git?branch=feature%2Fdata-node#36aae24b3e6eb4e087509ef371e64a0e667997cc"
 dependencies = [
  "cbindgen",
  "console_error_panic_hook",
@@ -3075,6 +3085,24 @@ version = "2.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e"
 
+[[package]]
+name = "phf"
+version = "0.11.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078"
+dependencies = [
+ "phf_shared",
+]
+
+[[package]]
+name = "phf_shared"
+version = "0.11.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "67eabc2ef2a60eb7faa00097bd1ffdb5bd28e62bf39990626a582201b7a754e5"
+dependencies = [
+ "siphasher",
+]
+
 [[package]]
 name = "pin-project"
 version = "1.1.10"
@@ -3167,6 +3195,38 @@ version = "1.11.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "350e9b48cbc6b0e028b0473b114454c6316e57336ee184ceab6e53f72c178b3e"
 
+[[package]]
+name = "postgres-protocol"
+version = "0.6.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "76ff0abab4a9b844b93ef7b81f1efc0a366062aaef2cd702c76256b5dc075c54"
+dependencies = [
+ "base64 0.22.1",
+ "byteorder",
+ "bytes",
+ "fallible-iterator",
+ "hmac",
+ "md-5",
+ "memchr",
+ "rand 0.9.0",
+ "sha2",
+ "stringprep",
+]
+
+[[package]]
+name = "postgres-types"
+version = "0.2.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "613283563cd90e1dfc3518d548caee47e0e725455ed619881f5cf21f36de4b48"
+dependencies = [
+ "bytes",
+ "fallible-iterator",
+ "postgres-protocol",
+ "serde",
+ "serde_json",
+ "uuid",
+]
+
 [[package]]
 name = "powerfmt"
 version = "0.2.0"
@@ -3510,6 +3570,15 @@ dependencies = [
  "windows-sys 0.52.0",
 ]
 
+[[package]]
+name = "rs_merkle"
+version = "1.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bb09b49230ba22e8c676e7b75dfe2887dea8121f18b530ae0ba519ce442d2b21"
+dependencies = [
+ "sha2",
+]
+
 [[package]]
 name = "rtcp"
 version = "0.10.1"
@@ -3568,7 +3637,7 @@ dependencies = [
 [[package]]
 name = "runtime"
 version = "0.1.0"
-source = "git+https://github.com/aukilabs/posemesh.git?branch=chore%2Freconstruction-integration#ee37586e6cad8b2c910ec29152b0d939e5e80e64"
+source = "git+https://github.com/aukilabs/posemesh.git?branch=feature%2Fdata-node#36aae24b3e6eb4e087509ef371e64a0e667997cc"
 dependencies = [
  "once_cell",
  "tokio",
@@ -3831,7 +3900,6 @@ dependencies = [
  "domain",
  "futures",
  "jsonwebtoken",
- "libp2p",
  "networking",
  "quick-protobuf",
  "rand 0.9.0",
@@ -3921,6 +3989,12 @@ dependencies = [
  "time",
 ]
 
+[[package]]
+name = "siphasher"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "56199f7ddabf13fe5074ce809e7d3f42b42ae711800501b5b16ea82ad029c39d"
+
 [[package]]
 name = "slab"
 version = "0.4.9"
@@ -4015,6 +4089,17 @@ version = "1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f"
 
+[[package]]
+name = "stringprep"
+version = "0.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7b4df3d392d81bd458a8a621b8bffbd2302a12ffe288a9d931670948749463b1"
+dependencies = [
+ "unicode-bidi",
+ "unicode-normalization",
+ "unicode-properties",
+]
+
 [[package]]
 name = "strsim"
 version = "0.8.0"
@@ -4323,6 +4408,32 @@ dependencies = [
  "syn 2.0.100",
 ]
 
+[[package]]
+name = "tokio-postgres"
+version = "0.7.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6c95d533c83082bb6490e0189acaa0bbeef9084e60471b696ca6988cd0541fb0"
+dependencies = [
+ "async-trait",
+ "byteorder",
+ "bytes",
+ "fallible-iterator",
+ "futures-channel",
+ "futures-util",
+ "log",
+ "parking_lot",
+ "percent-encoding",
+ "phf",
+ "pin-project-lite",
+ "postgres-protocol",
+ "postgres-types",
+ "rand 0.9.0",
+ "socket2",
+ "tokio",
+ "tokio-util",
+ "whoami",
+]
+
 [[package]]
 name = "tokio-util"
 version = "0.7.13"
@@ -4493,12 +4604,33 @@ dependencies = [
  "static_assertions",
 ]
 
+[[package]]
+name = "unicode-bidi"
+version = "0.3.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5"
+
 [[package]]
 name = "unicode-ident"
 version = "1.0.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512"
 
+[[package]]
+name = "unicode-normalization"
+version = "0.1.24"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5033c97c4262335cded6d6fc3e5c18ab755e1a3dc96376350f3d8e9f009ad956"
+dependencies = [
+ "tinyvec",
+]
+
+[[package]]
+name = "unicode-properties"
+version = "0.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e70f2a8b45122e719eb623c01822704c4e0907e7e426a05927e1a1cfff5b75d0"
+
 [[package]]
 name = "unicode-width"
 version = "0.1.14"
@@ -4571,7 +4703,7 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
 [[package]]
 name = "utils"
 version = "0.1.0"
-source = "git+https://github.com/aukilabs/posemesh.git?branch=chore%2Freconstruction-integration#ee37586e6cad8b2c910ec29152b0d939e5e80e64"
+source = "git+https://github.com/aukilabs/posemesh.git?branch=feature%2Fdata-node#36aae24b3e6eb4e087509ef371e64a0e667997cc"
 dependencies = [
  "futures",
  "gloo-timers 0.3.0",
@@ -4651,6 +4783,12 @@ dependencies = [
  "wit-bindgen-rt",
 ]
 
+[[package]]
+name = "wasite"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b"
+
 [[package]]
 name = "wasm-bindgen"
 version = "0.2.100"
@@ -4972,6 +5110,17 @@ dependencies = [
  "winapi",
 ]
 
+[[package]]
+name = "whoami"
+version = "1.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6994d13118ab492c3c80c1f81928718159254c53c472bf9ce36f8dae4add02a7"
+dependencies = [
+ "redox_syscall",
+ "wasite",
+ "web-sys",
+]
+
 [[package]]
 name = "widestring"
 version = "1.1.0"
diff --git a/server/Cargo.toml b/server/Cargo.toml
index e680233..926757e 100644
--- a/server/Cargo.toml
+++ b/server/Cargo.toml
@@ -4,13 +4,12 @@ version = "0.1.0"
 edition = "2021"
 
 [dependencies]
-libp2p = { git = "https://github.com/aukilabs/rust-libp2p.git", branch = "fix/gossipsub-wasm" }
-networking = { git = "https://github.com/aukilabs/posemesh.git", branch = "chore/reconstruction-integration" }
+networking = { git = "https://github.com/aukilabs/posemesh.git", branch = "feature/data-node" }
 futures = "0.3.30"
 jsonwebtoken = "9.3.0"
 uuid = "1.13.2"
 rand = "0.9.0"
-domain = { git = "https://github.com/aukilabs/posemesh.git", branch = "chore/reconstruction-integration" }
+domain = { git = "https://github.com/aukilabs/posemesh.git", branch = "feature/data-node" }
 tracing = "0.1.40"
 tracing-subscriber = "0.3.18"
 tokio = "1.44.0"
diff --git a/server/src/global_refinement.rs b/server/src/global_refinement.rs
index 713c059..051a4bd 100644
--- a/server/src/global_refinement.rs
+++ b/server/src/global_refinement.rs
@@ -1,23 +1,18 @@
-use std::{collections::HashMap, fs::{self, File}, io::{self, BufReader, BufWriter, Read, Write, BufRead, Cursor}, path::{Path, PathBuf}, process::Stdio, time::SystemTime};
-use chrono::Utc;
-use domain::{cluster::DomainCluster, datastore::{self, common::Datastore, remote::RemoteDatastore}, message::read_prefix_size_message, protobuf::{domain_data::{Data, Metadata, Query},task::{self, LocalRefinementInputV1, LocalRefinementOutputV1}}};
-use jsonwebtoken::{decode, DecodingKey,Validation, Algorithm};
-use libp2p::Stream;
-use networking::{client::Client, libp2p::Networking};
-use quick_protobuf::{deserialize_from_slice, serialize_into_vec, BytesReader};
+use std::{collections::HashMap, fs::{self, File}, io::Cursor, path::{Path, PathBuf}, process::Stdio};
+use domain::{datastore::{common::Datastore, remote::RemoteDatastore}, message::read_prefix_size_message, protobuf::{domain_data::{Data, Metadata, Query}, task}};
+use networking::{client::Client, AsyncStream};
+use quick_protobuf::serialize_into_vec;
 use regex::Regex;
-use tokio::{self, select, sync::watch, time::{interval, sleep, Duration}};
-use futures::{stream::Zip, AsyncReadExt, StreamExt};
-use uuid::Uuid;
-use serde::{Serialize, Deserialize};
+use tokio::{self, sync::watch, time::{interval, sleep, Duration}};
+use futures::StreamExt;
 use zip::ZipArchive;
 use tokio::io::{AsyncBufReadExt, BufReader as TokioBufReader};
 use tokio::process::Command;
 
 use crate::utils::handshake;
 
-async fn upload_results(domain_id: &str, output_path: PathBuf, mut datastore: Box<dyn Datastore>) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
-    let mut uploader = datastore.produce(domain_id.to_string()).await;
+async fn upload_results(domain_id: &str, output_path: PathBuf, mut datastore: RemoteDatastore) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    let mut uploader = datastore.upsert(domain_id.to_string()).await;
     // open output_path and upload to datastore
     let files = fs::read_dir(output_path)?;
     for file in files {
@@ -30,6 +25,8 @@ async fn upload_results(domain_id: &str, output_path: PathBuf, mut datastore: Bo
                 size: file.metadata()?.len() as u32,
                 id: None,
                 properties: HashMap::new(),
+                link: None,
+                hash: None,
             },
             "RefinedPointCloud.ply" => Metadata {
                 name: "refined_pointcloud".to_string(),
@@ -37,6 +34,8 @@ async fn upload_results(domain_id: &str, output_path: PathBuf, mut datastore: Bo
                 size: file.metadata()?.len() as u32,
                 id: None,
                 properties: HashMap::new(),
+                link: None,
+                hash: None,
             },
             "BasicStitchPointCloud.ply" => Metadata {
                 name: "unrefined_pointcloud".to_string(),
@@ -44,15 +43,14 @@ async fn upload_results(domain_id: &str, output_path: PathBuf, mut datastore: Bo
                 size: file.metadata()?.len() as u32,
                 id: None,
                 properties: HashMap::new(),
+                link: None,
+                hash: None,
             },
             _ => continue
         };
         let content = fs::read(path)?;
-        uploader.push(&Data {
-            domain_id: domain_id.to_string(),
-            metadata,
-            content,
-        }).await?;
+        let mut producer = uploader.push(&metadata).await?;
+        producer.push_chunk(&content, false).await?;
     }
 
     while !uploader.is_completed().await {
@@ -90,7 +88,7 @@ fn unzip_bytes(path: PathBuf, zip_bytes: Vec<u8>) -> Result<(), Box<dyn std::err
     Ok(())
 }
 
-pub(crate) async fn v1(base_path: String, mut stream: Stream, mut datastore: Box<dyn Datastore>, mut c: Client) {
+pub(crate) async fn v1<S: AsyncStream>(base_path: String, mut stream: S, mut datastore: RemoteDatastore, mut c: Client) {
     let claim = handshake(&mut stream).await.expect("Failed to handshake");
     let job_id = claim.job_id.clone();
     c.subscribe(job_id.clone()).await.expect("Failed to subscribe to job");
@@ -160,7 +158,7 @@ pub(crate) async fn v1(base_path: String, mut stream: Stream, mut datastore: Box
         let _ = tx.send(true); // Signal heartbeat task to stop
     });
 
-    let mut downloader = datastore.consume(claim.domain_id.clone(), query, false).await;
+    let mut downloader = datastore.load(claim.domain_id.clone(), query, false).await;
     let mut scan_ids = Vec::new();
 
     loop {
diff --git a/server/src/local_refinement.rs b/server/src/local_refinement.rs
index 39c5d83..10cf979 100644
--- a/server/src/local_refinement.rs
+++ b/server/src/local_refinement.rs
@@ -1,20 +1,15 @@
-use std::{collections::HashMap, fs, path::Path, process::{ExitCode, ExitStatus, Stdio}, time::Duration};
-use domain::{datastore::common::Datastore, message::read_prefix_size_message, protobuf::{domain_data::{Data, Metadata, Query},task::{self, LocalRefinementInputV1, LocalRefinementOutputV1}}};
-use libp2p::Stream;
-use networking::client::Client;
+use std::{collections::HashMap, fs, path::{Path, PathBuf}, time::Duration};
+use domain::{datastore::{common::Datastore, remote::RemoteDatastore}, message::read_prefix_size_message, protobuf::{domain_data::{Data, Metadata, Query},task::{self, LocalRefinementInputV1, LocalRefinementOutputV1}}};
+use networking::{client::Client, AsyncStream};
 use quick_protobuf::serialize_into_vec;
-use futures::StreamExt;
+use futures::{StreamExt, AsyncRead};
 use tokio::{sync::watch, time::{interval, sleep}};
-use uuid::Uuid;
-use std::io::{self, BufReader, BufWriter, Read, Write};
-use zip::{write::{FileOptions, SimpleFileOptions}, ZipWriter};
-use std::io::BufRead;
-use tokio::io::{AsyncBufReadExt, BufReader as TokioBufReader};
-use tokio::process::Command;
+use std::io::{BufReader, Read, Write};
+use zip::write::SimpleFileOptions;
 
-use crate::utils::{handshake, write_scan_data_summary};
+use crate::utils::{handshake, health, write_scan_data_summary};
 
-pub(crate) async fn v1(base_path: String, mut stream: Stream, mut datastore: Box<dyn Datastore>, mut c: Client) {
+pub(crate) async fn v1<S: AsyncStream>(base_path: String, mut stream: S, datastore: RemoteDatastore, mut c: Client) {
     let claim = handshake(&mut stream).await.expect("Failed to handshake");
     let job_id = claim.job_id.clone();
     c.subscribe(job_id.clone()).await.expect("Failed to subscribe to job");
@@ -29,32 +24,88 @@ pub(crate) async fn v1(base_path: String, mut stream: Stream, mut datastore: Box
         output: None,
     };
 
-    let res = read_prefix_size_message::<LocalRefinementInputV1>(stream).await;
-    if let Err(e) = res {
-        eprintln!("Failed to load local refinement input {}", e);
-        t.status = task::Status::FAILED;
-        let message = serialize_into_vec(t).expect("failed to serialize task update");
-        c.publish(job_id.clone(), message).await.expect("failed to publish task update");
-        return;
+    let res = start(base_path, &job_id, &claim.task_name, &mut stream).await;
+    match res {
+        Ok(context) => {
+            t.status = task::Status::STARTED;
+            let message = serialize_into_vec(t).expect("failed to serialize task update");
+            c.publish(job_id.clone(), message).await.expect("failed to publish task update");
+            let (tx, rx) = watch::channel(false);
+
+            let heartbeat_handle = health(t.clone(), c.clone(), &job_id, rx);
+
+            let _cleanup = scopeguard::guard(tx.clone(), |tx| {
+                let _ = tx.send(true); // Signal heartbeat task to stop
+            });
+
+            let output = run(&claim.domain_id, &job_id, &claim.task_name, context, datastore).await;
+            match output {
+                Ok(output) => {
+                    t.output = Some(task::Any {
+                        type_url: "LocalRefinementOutputV1".to_string(),
+                        value: serialize_into_vec(&output).expect("failed to serialize local refinement output"),
+                    });
+                    t.status = task::Status::DONE;
+                }
+                Err(e) => {
+                    eprintln!("Failed to run local refinement: {}", e);
+                    t.status = task::Status::FAILED;
+                    t.output = Some(task::Any {
+                        type_url: "Error".to_string(),
+                        value: serialize_into_vec(&task::Error {
+                            message: e.to_string(),
+                        }).expect("failed to serialize local refinement output"),
+                    });
+                }
+            }
+
+            tx.send(true).unwrap();
+            let _ = heartbeat_handle.await;
+        }
+        Err(e) => {
+            eprintln!("Failed to start local refinement: {}", e);
+            t.status = task::Status::FAILED;
+            t.output = Some(task::Any {
+                type_url: "Error".to_string(),
+                value: serialize_into_vec(&task::Error {
+                    message: e.to_string(),
+                }).expect("failed to serialize local refinement output"),
+            });
+        }
     }
-    let input = res.unwrap();
+    let message = serialize_into_vec(t).expect("failed to serialize task update");
+    c.publish(job_id.clone(), message).await.expect("failed to publish task update");
+}
+
+struct LocalRefinementContext {
+    query: Query,
+    task_folder: PathBuf,
+    scan_folder: PathBuf,
+    input_folder: PathBuf,
+    output_folder: PathBuf,
+    suffix: String,
+}
+
+async fn start<S: AsyncStream>(base_path: String, job_id: &str, task_name: &str, stream: S) -> Result<LocalRefinementContext, Box<dyn std::error::Error + Send + Sync>> {
+    let input = read_prefix_size_message::<LocalRefinementInputV1>(stream).await?;
+    // if let Err(e) = res {
+    //     eprintln!("Failed to load local refinement input {}", e);
+    //     t.status = task::Status::FAILED;
+    //     let message = serialize_into_vec(t).expect("failed to serialize task update");
+    //     c.publish(job_id.clone(), message).await.expect("failed to publish task update");
+    //     return;
+    // }
+    // let input = res.unwrap();
 
-    println!("Start executing {}, {:?}", claim.task_name, input);
+    println!("Start executing {}, {:?}", task_name, input);
 
     // input.name_regexp looks .*_date
     // get date from regexp
-    let query = input.query.clone();
+    let query = input.query;
     let query_clone = query.clone();
     let name_regexp = query.name_regexp;
     if name_regexp.is_none() {
-        t.status = task::Status::FAILED;
-        t.output = Some(task::Any {
-            type_url: "Error".to_string(),
-            value: "Name regexp is empty".as_bytes().to_vec(),
-        });
-        let message = serialize_into_vec(t).expect("failed to serialize task update");
-        c.publish(job_id.clone(), message).await.expect("failed to publish task update");
-        return;
+        return Err("Name regexp is empty".into());
     }
 
     /*
@@ -70,46 +121,29 @@ pub(crate) async fn v1(base_path: String, mut stream: Stream, mut datastore: Box
      */
     let suffix = name_regexp.unwrap().replace(".*_", "");
 
-    let task_folder = Path::new(&base_path).join(&claim.job_id);
+    let task_folder = Path::new(&base_path).join(job_id);
+
     let scan_folder = Path::new(&task_folder).join("datasets");
     let input_folder = Path::new(&scan_folder).join(&suffix.clone());
     let output_folder = Path::new(&task_folder).join("refined").join("local");
-    fs::create_dir_all(&scan_folder).expect("Failed to create directory");
-    fs::create_dir_all(&output_folder).expect("Failed to create directory");
-    fs::create_dir_all(&input_folder).expect("Failed to create directory");
-
-    c.publish(job_id.clone(), serialize_into_vec(t).expect("failed to serialize task update")).await.expect("failed to publish task update");
+    fs::create_dir_all(&scan_folder)?;
+    fs::create_dir_all(&output_folder)?;
+    fs::create_dir_all(&input_folder)?;
 
-    let (tx, rx) = watch::channel(false);
-    let mut c_clone = c.clone();
-    let task_clone = t.clone();
-    let job_id_clone = job_id.clone();
-    let heartbeat_handle = tokio::spawn(async move {
-        let mut rx = rx;
-        let mut interval = interval(Duration::from_secs(30)); // Send heartbeat every 30 seconds
-        
-        loop {
-            tokio::select! {
-                _ = interval.tick() => {
-                    let mut progress_task = task_clone.clone();
-                    progress_task.status = task::Status::PROCESSING;
-                    if let Ok(message) = serialize_into_vec(&progress_task) {
-                        let _ = c_clone.publish(job_id_clone.clone(), message).await;
-                    }
-                }
-                // Check if we should stop
-                Ok(_) = rx.changed() => {
-                    break;
-                }
-            }
-        }
-    });
+    Ok(LocalRefinementContext {
+        query: query_clone,
+        task_folder,
+        scan_folder,
+        input_folder,
+        output_folder,
+        suffix,
+    })
+}
 
-    let _cleanup = scopeguard::guard(tx.clone(), |tx| {
-        let _ = tx.send(true); // Signal heartbeat task to stop
-    });
+async fn run(domain_id: &str, job_id: &str, task_name: &str, context: LocalRefinementContext, mut datastore: RemoteDatastore) -> Result<LocalRefinementOutputV1, Box<dyn std::error::Error + Send + Sync>> { 
+    let query = context.query;
 
-    let mut downloader = datastore.consume("".to_string(), query_clone, false).await;
+    let mut downloader = datastore.load(domain_id.to_string(), query.clone(), false).await;
     let mut i = 0;
     loop {
         match downloader.next().await {
@@ -144,130 +178,42 @@ pub(crate) async fn v1(base_path: String, mut stream: Stream, mut datastore: Box
                         }
                     }
                 };
-                let path = input_folder.join(&filename);
-                fs::write(path, &data
-                    .content)
-                    .expect("Failed to write data to file");
+                let path = context.input_folder.join(&filename);
+                fs::write(path, &data.content)?;
                 i+=1;
                 println!("downloaded {}", filename);
             }
-            Some(Err(_)) => {
-                t.status = task::Status::RETRY;
-                let message = serialize_into_vec(t).expect("failed to serialize task update");
-                c.publish(job_id.clone(), message).await.expect("failed to publish task update");
-                return;
+            Some(Err(e)) => {
+                return Err(e.into());
             }
             None => {
                 break;
             }
         }
     }
-    println!("Finished downloading {} data for {}", i, claim.task_name);
+    println!("Finished downloading {} data for {}", i, task_name);
 
-    if let Err(e) = write_scan_data_summary(input_folder.as_path(), task_folder.as_path().join("scan_data_summary.json").as_path()) {
-        eprintln!("Failed to write scan data summary: {}", e);
-        t.status = task::Status::FAILED;
-        t.output = Some(task::Any {
-            type_url: "Error".to_string(),
-            value: serialize_into_vec(&task::Error {
-                message: format!("Failed to write scan data summary: {}", e),
-            }).unwrap(),
-        });
-        let message = serialize_into_vec(t).expect("failed to serialize task update");
-        c.publish(job_id.clone(), message).await.expect("failed to publish task update");
-        return;
-    }
+    write_scan_data_summary(context.input_folder.as_path(), context.task_folder.as_path().join("scan_data_summary.json").as_path())?;
 
     let params = vec![
         "-u",
         "main.py",
         "--mode", "local_refinement",
-        "--job_root_path", task_folder.to_str().unwrap(),
-        "--output", output_folder.to_str().unwrap(),
-        "--domain_id", &claim.domain_id,
-        "--job_id", &claim.job_id,
-        "--scans", suffix.as_str(),
+        "--job_root_path", context.task_folder.to_str().unwrap(),
+        "--output", context.output_folder.to_str().unwrap(),
+        "--domain_id", domain_id,
+        "--job_id", job_id,
+        "--scans", context.suffix.as_str(),
     ];
-    
-    let child = Command::new("python3")
-        .args(params)
-        .stdout(Stdio::piped())
-        .stderr(Stdio::piped())
-        .spawn();
 
-    if let Err(e) = child {
-        eprintln!("Failed to execute local refinement: {}", e);
-        t.status = task::Status::FAILED;
-        let message = serialize_into_vec(t).expect("failed to serialize task update");
-        c.publish(job_id.clone(), message).await.expect("failed to publish task update");
-        return;
-    }
-
-    let mut child = child.unwrap();
-
-    // Read stdout in real-time
-    if let Some(stdout) = child.stdout.take() {
-        let stdout_reader = TokioBufReader::new(stdout);
-        tokio::spawn(async move {
-            let mut lines = stdout_reader.lines();
-            while let Ok(Some(line)) = lines.next_line().await {
-                println!("stdout: {}", line);
-            }
-        });
-    }
-
-    // Read stderr in real-time
-    if let Some(stderr) = child.stderr.take() {
-        let stderr_reader = TokioBufReader::new(stderr);
-        tokio::spawn(async move {
-            let mut lines = stderr_reader.lines();
-            while let Ok(Some(line)) = lines.next_line().await {
-                eprintln!("stderr: {}", line);
-            }
-        });
-    }
-
-    // Wait for the process to complete
-    match child.wait().await {
-        Ok(status) => {
-            if !status.success() {
-                eprintln!("Python process exited with status: {}", status);
-                t.status = task::Status::FAILED;
-                t.output = Some(task::Any {
-                    type_url: "Error".to_string(),
-                    value: serialize_into_vec(&task::Error {
-                        message: format!("Python process exited with status: {}", status),
-                    }).unwrap(),
-                });
-                let message = serialize_into_vec(t).expect("failed to serialize task update");
-                c.publish(job_id.clone(), message).await.expect("failed to publish task update");
-                return;
-            }
-            println!("Finished executing {}", claim.task_name);
-        }
-        Err(e) => {
-            eprintln!("Failed to wait for Python process: {}", e);
-            t.status = task::Status::FAILED;
-            t.output = Some(task::Any {
-                type_url: "Error".to_string(),
-                value: serialize_into_vec(&task::Error {
-                    message: format!("Failed to wait for Python process: {}", e),
-                }).unwrap(),
-            });
-            let message = serialize_into_vec(t).expect("failed to serialize task update");
-            c.publish(job_id.clone(), message).await.expect("failed to publish task update");
-            return;
-        }
-    }
-
-    let sfm = output_folder.join(suffix.clone()).join("sfm");
-    let zip_path = sfm.join(suffix.clone() + ".zip");
-    let mut zip = zip::ZipWriter::new(fs::File::create(&zip_path).expect("Failed to create zip file"));
+    let sfm = context.output_folder.join(context.suffix.clone()).join("sfm");
+    let zip_path = sfm.join(context.suffix.clone() + ".zip");
+    let mut zip = zip::ZipWriter::new(fs::File::create(&zip_path)?);
     let options = SimpleFileOptions::default().compression_method(zip::CompressionMethod::Stored); // No compression
     
     // open output folder/sfm, zip all txt, bin, csv files and upload
-    for entry in fs::read_dir(sfm).expect("Failed to read directory") {
-        let entry = entry.expect("Failed to read entry");
+    for entry in fs::read_dir(sfm)? {
+        let entry = entry?;
         let path = entry.path();
         if path.is_dir() {
             continue;
@@ -275,78 +221,47 @@ pub(crate) async fn v1(base_path: String, mut stream: Stream, mut datastore: Box
         let ext = path.extension().expect("Failed to get extension").to_str().expect("Failed to convert extension to string");
         if ext == "txt" || ext == "bin" || ext == "csv" {
             let str_path = path.to_str().expect("Failed to convert path to string");
-            let file = fs::File::open(str_path).expect("Failed to open file");
+            let file = fs::File::open(str_path)?;
             let mut reader = BufReader::new(file);
-            zip.start_file(str_path, options).expect("Failed to start zip file");
+            zip.start_file(str_path, options)?;
 
             let mut buffer = [0u8; 8192]; // Use a buffer to stream in chunks
             loop {
-                let bytes_read = reader.read(&mut buffer).expect("Failed to read file");
+                let bytes_read = reader.read(&mut buffer)?;
                 if bytes_read == 0 {
                     break;
                 }
-                zip.write_all(&buffer[..bytes_read]).expect("Failed to write to zip");
+                zip.write_all(&buffer[..bytes_read])?;
             }
         }
     }
-    zip.finish().expect("Failed to finish zip");
-    let zip_file_metadata = fs::metadata(&zip_path).expect("Failed to get metadata");
+    zip.finish()?;
+    let zip_file_metadata = fs::metadata(&zip_path)?;
 
-    let mut producer = datastore.produce(claim.domain_id.clone()).await;
-    let res = producer.push(&Data {
-        domain_id: claim.domain_id.clone(),
-        metadata: Metadata {
+    let mut producer = datastore.upsert(domain_id.to_string()).await;
+    let mut chunks = producer.push(
+        &Metadata {
             size: zip_file_metadata.len() as u32,
-            name: format!("refined_scan_{}", suffix.clone()),
+            name: format!("refined_scan_{}", context.suffix.clone()),
             data_type: "refined_scan_zip_v1".to_string(),
             id: None,
             properties: HashMap::new(),
+            link: None,
+            hash: None,
         },
-        content: fs::read(&zip_path).expect("Failed to read zip file"),
-    }).await;
-    if let Err(e) = res {
-        eprintln!("Failed to upload refined scan: {}", e);
-        t.status = task::Status::FAILED;
-        t.output = Some(task::Any {
-            type_url: "Error".to_string(),
-            value: serialize_into_vec(&task::Error {
-                message: e.to_string(),
-            }).unwrap(),
-        });
-        let message = serialize_into_vec(t).expect("failed to serialize task update");
-        c.publish(job_id.clone(), message).await.expect("failed to publish task update");
-        return;
-    }
+    ).await?;
+    
+    let hash = chunks.push_chunk(&fs::read(&zip_path)?, false).await?;
     while !producer.is_completed().await {
         sleep(Duration::from_secs(3)).await;
     }
     producer.close().await;
-    println!("Finished uploading refined scan for {}", suffix.clone());
-    if let Err(e) = std::fs::remove_dir_all(&task_folder) {
-        eprintln!("Failed to remove task folder: {:?}: {}", task_folder, e);
-    }
+    println!("Finished uploading refined scan for {}", context.suffix.clone());
+    std::fs::remove_dir_all(&context.task_folder)?;
 
     let output = LocalRefinementOutputV1 {
-        result_ids: vec![res.unwrap()],
-    };
-    let event = task::Task {
-        name: claim.task_name.clone(),
-        receiver: Some(claim.receiver.clone()),
-        sender: claim.sender.clone(),
-        endpoint: "/local-refinement/v1".to_string(),
-        status: task::Status::DONE,
-        access_token: None,
-        job_id: job_id.clone(),
-        output: Some(task::Any {
-            type_url: "LocalRefinementOutputV1".to_string(),
-            value: serialize_into_vec(&output).expect("Failed to serialize local refinement output"),
-        }),
+        result_ids: vec![hash],
     };
-    let buf = serialize_into_vec(&event).expect("failed to serialize task update");
-    c.publish(job_id.clone(), buf).await.expect("failed to publish task update");
-
-    tx.send(true).unwrap();
-    let _ = heartbeat_handle.await;
-    println!("Heartbeat task stopped");
-    return;
+    
+    return Ok(output);
 }
diff --git a/server/src/main.rs b/server/src/main.rs
index 7e6b6e7..d98e71c 100644
--- a/server/src/main.rs
+++ b/server/src/main.rs
@@ -1,12 +1,6 @@
-use domain::{cluster::DomainCluster, datastore::{common::Datastore, remote::{self, RemoteDatastore}}, message::read_prefix_size_message, protobuf::{domain_data::Query,task::{self, LocalRefinementInputV1, LocalRefinementOutputV1}}};
-use jsonwebtoken::{decode, DecodingKey,Validation, Algorithm};
-use libp2p::Stream;
-use networking::{client::Client, libp2p::Networking};
-use quick_protobuf::{deserialize_from_slice, serialize_into_vec};
-use tokio::{self, select, signal::unix::{signal, SignalKind}, time::{sleep, Duration}};
-use futures::{AsyncReadExt, StreamExt};
-use uuid::Uuid;
-use regex::Regex;
+use domain::{cluster::DomainCluster, datastore::remote::RemoteDatastore};
+use tokio::{self, select, signal::unix::{signal, SignalKind}};
+use futures::StreamExt;
 mod local_refinement;
 mod global_refinement;
 mod utils;
@@ -38,7 +32,6 @@ async fn main() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     let domain_manager = args[3].clone();
     let private_key_path = format!("{}/pkey", base_path);
 
-    let domain_manager_id = domain_manager.split("/").last().unwrap().to_string();
     let domain_cluster = DomainCluster::new(domain_manager.clone(), name, false, port, false, false, None, Some(private_key_path));
     let mut n = domain_cluster.peer.clone();
     let mut local_refinement_v1_handler = n.client.set_stream_handler("/local-refinement/v1".to_string()).await.unwrap();
@@ -48,10 +41,10 @@ async fn main() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     loop {
         select! {
             Some((_, stream)) = local_refinement_v1_handler.next() => {
-                let _ = tokio::spawn(local_refinement::v1(base_path.clone(), stream, Box::new(remote_storage.clone()), n.client.clone()));
+                let _ = tokio::spawn(local_refinement::v1(base_path.clone(), stream, remote_storage.clone(), n.client.clone()));
             }
             Some((_, stream)) = global_refinement_v1_handler.next() => {
-                let _ = tokio::spawn(global_refinement::v1(base_path.clone(), stream, Box::new(remote_storage.clone()), n.client.clone()));
+                let _ = tokio::spawn(global_refinement::v1(base_path.clone(), stream, remote_storage.clone(), n.client.clone()));
             }
             _ = shutdown_signal() => {
                 println!("Received termination signal, shutting down...");
diff --git a/server/src/utils.rs b/server/src/utils.rs
index ab83150..de65350 100644
--- a/server/src/utils.rs
+++ b/server/src/utils.rs
@@ -1,17 +1,13 @@
-use domain::{cluster::DomainCluster, datastore::{common::Datastore, remote::RemoteDatastore}, message::read_prefix_size_message, protobuf::{domain_data::Query,task::{self, LocalRefinementInputV1, LocalRefinementOutputV1}}};
+use domain::protobuf::task::{DomainClusterHandshake, Status, Task};
 use jsonwebtoken::{decode, DecodingKey,Validation, Algorithm};
-use libp2p::Stream;
-use networking::{client::Client, libp2p::Networking};
+use networking::{client::Client, AsyncStream};
 use quick_protobuf::{deserialize_from_slice, serialize_into_vec};
-use tokio::{self, select, time::{sleep, Duration}};
-use futures::{AsyncReadExt, StreamExt};
-use uuid::Uuid;
+use futures::AsyncReadExt;
 use serde::{Serialize, Deserialize};
-use std::fs::{self, DirEntry};
-use std::path::Path;
+use tokio::{sync::watch, task::JoinHandle, time::interval};
+use std::{collections::HashSet, error::Error, fs, path::Path, process::Stdio, time::Duration};
 use serde_json::{json, Value};
-use std::collections::HashSet;
-use std::error::Error;
+use tokio::{io::{AsyncBufReadExt, BufReader as TokioBufReader}, process::Command};
 
 #[derive(Debug, Serialize, Deserialize)]
 pub struct TaskTokenClaim {
@@ -30,7 +26,7 @@ pub fn decode_jwt(token: &str) -> Result<TaskTokenClaim, Box<dyn std::error::Err
     Ok(token_data.claims)
 }
 
-pub async fn handshake(stream: &mut Stream) -> Result<TaskTokenClaim, Box<dyn std::error::Error + Send + Sync>> {
+pub async fn handshake<S: AsyncStream>(stream: &mut S) -> Result<TaskTokenClaim, Box<dyn std::error::Error + Send + Sync>> {
     let mut length_buf = [0u8; 4];
     stream.read_exact(&mut length_buf).await?;
 
@@ -38,7 +34,7 @@ pub async fn handshake(stream: &mut Stream) -> Result<TaskTokenClaim, Box<dyn st
     let mut buffer = vec![0u8; length];
     stream.read_exact(&mut buffer).await?;
         
-    let header = deserialize_from_slice::<task::DomainClusterHandshake>(&buffer)?;
+    let header = deserialize_from_slice::<DomainClusterHandshake>(&buffer)?;
 
     decode_jwt(header.access_token.as_str())
 }
@@ -142,3 +138,64 @@ pub fn write_scan_data_summary(
 
     Ok(())
 } 
+
+pub async fn health(task: Task, mut c: Client, job_id: &str, rx: watch::Receiver<bool>) -> JoinHandle<()> {
+    let job_id_clone = job_id.to_string();
+    tokio::spawn(async move {
+        let mut rx = rx;
+        let mut interval = interval(Duration::from_secs(30)); // Send heartbeat every 30 seconds
+        
+        loop {
+            tokio::select! {
+                _ = interval.tick() => {
+                    let mut progress_task = task.clone();
+                    progress_task.status = Status::PROCESSING;
+                    if let Ok(message) = serialize_into_vec(&progress_task) {
+                        let _ = c.publish(job_id_clone.clone(), message).await;
+                    }
+                }
+                // Check if we should stop
+                Ok(_) = rx.changed() => {
+                    break;
+                }
+            }
+        }
+    })
+}
+
+pub async fn execute_python(params: Vec<&str>) -> Result<(), Box<dyn Error + Send + Sync>> {
+    let mut child = Command::new("python3")
+        .args(params)
+        .stdout(Stdio::piped())
+        .stderr(Stdio::piped())
+        .spawn()?;
+
+    // Read stdout in real-time
+    if let Some(stdout) = child.stdout.take() {
+        let stdout_reader = TokioBufReader::new(stdout);
+        tokio::spawn(async move {
+            let mut lines = stdout_reader.lines();
+            while let Ok(Some(line)) = lines.next_line().await {
+                println!("stdout: {}", line);
+            }
+        });
+    }
+
+    // Read stderr in real-time
+    if let Some(stderr) = child.stderr.take() {
+        let stderr_reader = TokioBufReader::new(stderr);
+        tokio::spawn(async move {
+            let mut lines = stderr_reader.lines();
+            while let Ok(Some(line)) = lines.next_line().await {
+                eprintln!("stderr: {}", line);
+            }
+        });
+    }
+
+    // Wait for the process to complete
+    let status = child.wait().await?;
+    if !status.success() {
+        return Err("Python process failed".into());
+    }
+    Ok(())
+}

From 82e03b601b5b06a3dc7ff1a998fa2070609f8a94 Mon Sep 17 00:00:00 2001
From: Shuning <shuning@aukilabs.com>
Date: Thu, 10 Apr 2025 14:39:43 +0800
Subject: [PATCH 72/89] remove unused imports

---
 server/src/global_refinement.rs | 2 +-
 server/src/local_refinement.rs  | 6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/server/src/global_refinement.rs b/server/src/global_refinement.rs
index 051a4bd..0af7bb7 100644
--- a/server/src/global_refinement.rs
+++ b/server/src/global_refinement.rs
@@ -1,5 +1,5 @@
 use std::{collections::HashMap, fs::{self, File}, io::Cursor, path::{Path, PathBuf}, process::Stdio};
-use domain::{datastore::{common::Datastore, remote::RemoteDatastore}, message::read_prefix_size_message, protobuf::{domain_data::{Data, Metadata, Query}, task}};
+use domain::{datastore::{common::Datastore, remote::RemoteDatastore}, message::read_prefix_size_message, protobuf::{domain_data::{Metadata, Query}, task}};
 use networking::{client::Client, AsyncStream};
 use quick_protobuf::serialize_into_vec;
 use regex::Regex;
diff --git a/server/src/local_refinement.rs b/server/src/local_refinement.rs
index 10cf979..13d556c 100644
--- a/server/src/local_refinement.rs
+++ b/server/src/local_refinement.rs
@@ -1,9 +1,9 @@
 use std::{collections::HashMap, fs, path::{Path, PathBuf}, time::Duration};
-use domain::{datastore::{common::Datastore, remote::RemoteDatastore}, message::read_prefix_size_message, protobuf::{domain_data::{Data, Metadata, Query},task::{self, LocalRefinementInputV1, LocalRefinementOutputV1}}};
+use domain::{datastore::{common::Datastore, remote::RemoteDatastore}, message::read_prefix_size_message, protobuf::{domain_data::{Metadata, Query},task::{self, LocalRefinementInputV1, LocalRefinementOutputV1}}};
 use networking::{client::Client, AsyncStream};
 use quick_protobuf::serialize_into_vec;
-use futures::{StreamExt, AsyncRead};
-use tokio::{sync::watch, time::{interval, sleep}};
+use futures::StreamExt;
+use tokio::{sync::watch, time::sleep};
 use std::io::{BufReader, Read, Write};
 use zip::write::SimpleFileOptions;
 

From 4be8c98cbd18d089cb3cc454bbd6982886909abc Mon Sep 17 00:00:00 2001
From: Shuning <shuning@aukilabs.com>
Date: Wed, 16 Apr 2025 18:06:22 +0800
Subject: [PATCH 73/89] update domain

---
 server/Cargo.lock | 375 +++++++++++++++++++++++++++-------------------
 1 file changed, 222 insertions(+), 153 deletions(-)

diff --git a/server/Cargo.lock b/server/Cargo.lock
index fc1d64c..09ed9f6 100644
--- a/server/Cargo.lock
+++ b/server/Cargo.lock
@@ -143,9 +143,9 @@ dependencies = [
 
 [[package]]
 name = "anyhow"
-version = "1.0.97"
+version = "1.0.98"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dcfed56ad506cb2c684a14971b8861fdc3baaaae314b9e5f9bb532cbe3ba7a4f"
+checksum = "e16d2d3311acee920a9eb8d33b8cbc1787ce4a264e85f964c2404b969bdcd487"
 
 [[package]]
 name = "arbitrary"
@@ -284,9 +284,9 @@ dependencies = [
 
 [[package]]
 name = "async-trait"
-version = "0.1.87"
+version = "0.1.88"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d556ec1359574147ec0c4fc5eb525f3f23263a592b1a9c07e0a75b427de55c97"
+checksum = "e539d3fca749fcee5236ab05e93a52867dd549cc157c8cb7f99595f3cedffdb5"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -381,9 +381,9 @@ checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
 
 [[package]]
 name = "base64ct"
-version = "1.7.1"
+version = "1.7.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bb97d56060ee67d285efb8001fec9d2a4c710c32efd2e14b5cbb5ba71930fc2d"
+checksum = "89e25b6adfb930f02d1981565a6e5d9c547ac15a96606256d3b59040e5cd4ca3"
 
 [[package]]
 name = "bincode"
@@ -497,7 +497,7 @@ version = "0.28.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "eadd868a2ce9ca38de7eeafdcec9c7065ef89b42b32f0839278d55f35c54d1ff"
 dependencies = [
- "clap 4.5.32",
+ "clap 4.5.36",
  "heck 0.4.1",
  "indexmap",
  "log",
@@ -512,9 +512,9 @@ dependencies = [
 
 [[package]]
 name = "cc"
-version = "1.2.16"
+version = "1.2.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "be714c154be609ec7f5dad223a33bf1482fff90472de28f7362806e6d4832b8c"
+checksum = "8e3a13707ac958681c13b39b458c073d0d9bc8a22cb1b2f4c8e55eb72c13f362"
 dependencies = [
  "jobserver",
  "libc",
@@ -611,18 +611,18 @@ dependencies = [
 
 [[package]]
 name = "clap"
-version = "4.5.32"
+version = "4.5.36"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6088f3ae8c3608d19260cd7445411865a485688711b78b5be70d78cd96136f83"
+checksum = "2df961d8c8a0d08aa9945718ccf584145eee3f3aa06cddbeac12933781102e04"
 dependencies = [
  "clap_builder",
 ]
 
 [[package]]
 name = "clap_builder"
-version = "4.5.32"
+version = "4.5.36"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "22a7ef7f676155edfb82daa97f99441f3ebf4a58d5e32f295a56259f1b6facc8"
+checksum = "132dbda40fb6753878316a489d5a1242a8ef2f0d9e47ba01c951ea8aa7d013a5"
 dependencies = [
  "anstream",
  "anstyle",
@@ -804,15 +804,15 @@ dependencies = [
 
 [[package]]
 name = "data-encoding"
-version = "2.8.0"
+version = "2.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "575f75dfd25738df5b91b8e43e14d44bda14637a58fae779fd2b064f8bf3e010"
+checksum = "2a2330da5de22e8a3cb63252ce2abb30116bf5265e89c0e01bc17015ce30a476"
 
 [[package]]
 name = "data-encoding-macro"
-version = "0.1.17"
+version = "0.1.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9f9724adfcf41f45bf652b3995837669d73c4d49a1b5ac1ff82905ac7d9b5558"
+checksum = "47ce6c96ea0102f01122a185683611bd5ac8d99e62bc59dd12e6bda344ee673d"
 dependencies = [
  "data-encoding",
  "data-encoding-macro-internal",
@@ -820,9 +820,9 @@ dependencies = [
 
 [[package]]
 name = "data-encoding-macro-internal"
-version = "0.1.15"
+version = "0.1.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "18e4fdb82bd54a12e42fb58a800dcae6b9e13982238ce2296dc3570b92148e1f"
+checksum = "8d162beedaa69905488a8da94f5ac3edb4dd4788b732fadb7bd120b2625c1976"
 dependencies = [
  "data-encoding",
  "syn 2.0.100",
@@ -875,9 +875,9 @@ dependencies = [
 
 [[package]]
 name = "deranged"
-version = "0.3.11"
+version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4"
+checksum = "9c9e6a11ca8224451684bc0d7d5a7adbf8f2fd6887261a1cfc3c0432f9d4068e"
 dependencies = [
  "powerfmt",
 ]
@@ -919,7 +919,7 @@ dependencies = [
 [[package]]
 name = "domain"
 version = "0.1.0"
-source = "git+https://github.com/aukilabs/posemesh.git?branch=feature%2Fdata-node#36aae24b3e6eb4e087509ef371e64a0e667997cc"
+source = "git+https://github.com/aukilabs/posemesh.git?branch=feature%2Fdata-node#2690043823011c811290797294fac3b2e6d0d6b4"
 dependencies = [
  "async-trait",
  "cbindgen",
@@ -938,6 +938,7 @@ dependencies = [
  "serde-wasm-bindgen",
  "serde_json",
  "sha2",
+ "thiserror 2.0.12",
  "tokio",
  "tokio-postgres",
  "tracing",
@@ -1053,9 +1054,9 @@ checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f"
 
 [[package]]
 name = "errno"
-version = "0.3.10"
+version = "0.3.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "33d852cb9b869c2a9b3df2f71a3074817f01e1844f839a144f5fcef059a4eb5d"
+checksum = "976dd42dc7e85965fe702eb8164f21f450704bdde31faefd6471dba214cb594e"
 dependencies = [
  "libc",
  "windows-sys 0.59.0",
@@ -1074,9 +1075,9 @@ dependencies = [
 
 [[package]]
 name = "event-listener-strategy"
-version = "0.5.3"
+version = "0.5.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3c3e4e0dd3673c1139bf041f3008816d9cf2946bbfac2945c09e523b8d7b05b2"
+checksum = "8be9f3dfaaffdae2972880079a491a1a8bb7cbed0b8dd7a347f668b4150a3b93"
 dependencies = [
  "event-listener",
  "pin-project-lite",
@@ -1112,9 +1113,9 @@ checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d"
 
 [[package]]
 name = "flate2"
-version = "1.1.0"
+version = "1.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "11faaf5a5236997af9848be0bef4db95824b1d534ebc64d0f0c6cf3e67bd38dc"
+checksum = "7ced92e76e966ca2fd84c8f7aa01a4aea65b0eb6648d72f7c8f3e2764a67fece"
 dependencies = [
  "crc32fast",
  "miniz_oxide",
@@ -1128,9 +1129,9 @@ checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
 
 [[package]]
 name = "foldhash"
-version = "0.1.4"
+version = "0.1.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a0d2fde1f7b3d48b8395d5f2de76c18a528bd6a9cdde438df747bfcba3e05d6f"
+checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2"
 
 [[package]]
 name = "form_urlencoded"
@@ -1228,7 +1229,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a8f2f12607f92c69b12ed746fabf9ca4f5c482cba46679c1a75b874ed7c26adb"
 dependencies = [
  "futures-io",
- "rustls 0.23.23",
+ "rustls 0.23.26",
  "rustls-pki-types",
 ]
 
@@ -1309,14 +1310,16 @@ dependencies = [
 
 [[package]]
 name = "getrandom"
-version = "0.3.1"
+version = "0.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "43a49c392881ce6d5c3b8cb70f98717b7c07aabbdff06687b9030dbfbe2725f8"
+checksum = "73fea8450eea4bac3940448fb7ae50d91f034f941199fcd9d909a5a07aa455f0"
 dependencies = [
  "cfg-if",
+ "js-sys",
  "libc",
- "wasi 0.13.3+wasi-0.2.2",
- "windows-targets 0.52.6",
+ "r-efi",
+ "wasi 0.14.2+wasi-0.2.4",
+ "wasm-bindgen",
 ]
 
 [[package]]
@@ -1372,9 +1375,9 @@ dependencies = [
 
 [[package]]
 name = "h2"
-version = "0.4.8"
+version = "0.4.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5017294ff4bb30944501348f6f8e42e6ad28f42c8bbef7a74029aff064a4e3c2"
+checksum = "75249d144030531f8dee69fe9cea04d3edf809a017ae445e2abdff6629e86633"
 dependencies = [
  "atomic-waker",
  "bytes",
@@ -1511,13 +1514,13 @@ dependencies = [
 
 [[package]]
 name = "hostname"
-version = "0.3.1"
+version = "0.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3c731c3e10504cc8ed35cfe2f1db4c9274c3d35fa486e3b31df46f068ef3e867"
+checksum = "a56f203cd1c76362b69e3863fd987520ac36cf70a8c92627449b2f64a8cf7d65"
 dependencies = [
+ "cfg-if",
  "libc",
- "match_cfg",
- "winapi",
+ "windows-link",
 ]
 
 [[package]]
@@ -1602,9 +1605,9 @@ dependencies = [
 
 [[package]]
 name = "hyper-util"
-version = "0.1.10"
+version = "0.1.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "df2dcfbe0677734ab2f3ffa7fa7bfd4706bfdc1ef393f2ee30184aed67e631b4"
+checksum = "497bbc33a26fdd4af9ed9c70d63f61cf56a938375fbb32df34db9b1cd6d643f2"
 dependencies = [
  "bytes",
  "futures-channel",
@@ -1612,6 +1615,7 @@ dependencies = [
  "http 1.3.1",
  "http-body",
  "hyper",
+ "libc",
  "pin-project-lite",
  "socket2",
  "tokio",
@@ -1621,16 +1625,17 @@ dependencies = [
 
 [[package]]
 name = "iana-time-zone"
-version = "0.1.61"
+version = "0.1.63"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "235e081f3925a06703c2d0117ea8b91f042756fd6e7a6e5d901e8ca1a996b220"
+checksum = "b0c919e5debc312ad217002b8048a17b7d83f80703865bbfcfebb0458b0b27d8"
 dependencies = [
  "android_system_properties",
  "core-foundation-sys",
  "iana-time-zone-haiku",
  "js-sys",
+ "log",
  "wasm-bindgen",
- "windows-core 0.52.0",
+ "windows-core 0.61.0",
 ]
 
 [[package]]
@@ -1683,9 +1688,9 @@ dependencies = [
 
 [[package]]
 name = "icu_locid_transform_data"
-version = "1.5.0"
+version = "1.5.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fdc8ff3388f852bede6b579ad4e978ab004f139284d7b28715f773507b946f6e"
+checksum = "7515e6d781098bf9f7205ab3fc7e9709d34554ae0b21ddbcb5febfa4bc7df11d"
 
 [[package]]
 name = "icu_normalizer"
@@ -1707,9 +1712,9 @@ dependencies = [
 
 [[package]]
 name = "icu_normalizer_data"
-version = "1.5.0"
+version = "1.5.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f8cafbf7aa791e9b22bec55a167906f9e1215fd475cd22adfcf660e03e989516"
+checksum = "c5e8338228bdc8ab83303f16b797e177953730f601a96c25d10cb3ab0daa0cb7"
 
 [[package]]
 name = "icu_properties"
@@ -1728,9 +1733,9 @@ dependencies = [
 
 [[package]]
 name = "icu_properties_data"
-version = "1.5.0"
+version = "1.5.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "67a8effbc3dd3e4ba1afa8ad918d5684b8868b3b26500753effea8d2eed19569"
+checksum = "85fb8799753b75aee8d2a21d7c14d9f38921b54b3dbda10f5a3c7a7b82dba5e2"
 
 [[package]]
 name = "icu_provider"
@@ -1837,9 +1842,9 @@ dependencies = [
 
 [[package]]
 name = "indexmap"
-version = "2.8.0"
+version = "2.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3954d50fe15b02142bf25d3b8bdadb634ec3948f103d04ffe3031bc8fe9d7058"
+checksum = "cea70ddb795996207ad57735b50c5982d8844f38ba9ee5f1aedcfb708a2aa11e"
 dependencies = [
  "equivalent",
  "hashbrown",
@@ -1918,10 +1923,11 @@ checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c"
 
 [[package]]
 name = "jobserver"
-version = "0.1.32"
+version = "0.1.33"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "48d1dbcbbeb6a7fec7e059840aa538bd62aaccf972c7346c4d9d2059312853d0"
+checksum = "38f262f097c174adebe41eb73d66ae9c06b2844fb0da69969647bbddd9b0538a"
 dependencies = [
+ "getrandom 0.3.2",
  "libc",
 ]
 
@@ -1958,9 +1964,9 @@ checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"
 
 [[package]]
 name = "libc"
-version = "0.2.171"
+version = "0.2.172"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c19937216e9d3aa9956d9bb8dfc0b0c8beb6058fc4f7a4dc4d850edf86a237d6"
+checksum = "d750af042f7ef4f724306de029d18836c26c1765a54a6a3f094cbd23a7267ffa"
 
 [[package]]
 name = "libp2p"
@@ -2293,7 +2299,7 @@ dependencies = [
  "quinn",
  "rand 0.8.5",
  "ring 0.17.14",
- "rustls 0.23.23",
+ "rustls 0.23.26",
  "socket2",
  "thiserror 1.0.69",
  "tokio",
@@ -2420,7 +2426,7 @@ dependencies = [
  "libp2p-identity",
  "rcgen",
  "ring 0.17.14",
- "rustls 0.23.23",
+ "rustls 0.23.26",
  "rustls-webpki 0.101.7",
  "thiserror 1.0.69",
  "x509-parser 0.16.0",
@@ -2578,9 +2584,9 @@ checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab"
 
 [[package]]
 name = "linux-raw-sys"
-version = "0.9.2"
+version = "0.9.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6db9c683daf087dc577b7506e9695b3d556a9f3849903fa28186283afd6809e9"
+checksum = "cd945864f07fe9f5371a27ad7b52a172b4b499999f1d97574c9fa68373937e12"
 
 [[package]]
 name = "litemap"
@@ -2606,9 +2612,9 @@ checksum = "9374ef4228402d4b7e403e5838cb880d9ee663314b0a900d5a6aabf0c213552e"
 
 [[package]]
 name = "log"
-version = "0.4.26"
+version = "0.4.27"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "30bde2b3dc3671ae49d8e2e9f044c7c005836e7a023ee57cffa25ab82764bb9e"
+checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94"
 
 [[package]]
 name = "lru"
@@ -2639,10 +2645,15 @@ dependencies = [
 ]
 
 [[package]]
-name = "match_cfg"
-version = "0.1.0"
+name = "lzma-sys"
+version = "0.1.20"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ffbee8634e0d45d258acb448e7eaab3fce7a0a467395d4d9f228e3c1f01fb2e4"
+checksum = "5fda04ab3764e6cde78b9974eec4f779acaba7c4e84b36eca3cf77c581b85d27"
+dependencies = [
+ "cc",
+ "libc",
+ "pkg-config",
+]
 
 [[package]]
 name = "matchers"
@@ -2686,9 +2697,9 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
 
 [[package]]
 name = "miniz_oxide"
-version = "0.8.5"
+version = "0.8.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8e3e04debbb59698c15bacbb6d93584a8c0ca9cc3213cb423d31f760d8843ce5"
+checksum = "3be647b768db090acb35d5ec5db2b0e1f1de11133ca123b9eacf5137868f892a"
 dependencies = [
  "adler2",
 ]
@@ -2825,7 +2836,7 @@ dependencies = [
 [[package]]
 name = "networking"
 version = "0.1.0"
-source = "git+https://github.com/aukilabs/posemesh.git?branch=feature%2Fdata-node#36aae24b3e6eb4e087509ef371e64a0e667997cc"
+source = "git+https://github.com/aukilabs/posemesh.git?branch=feature%2Fdata-node#2690043823011c811290797294fac3b2e6d0d6b4"
 dependencies = [
  "cbindgen",
  "console_error_panic_hook",
@@ -2963,9 +2974,9 @@ dependencies = [
 
 [[package]]
 name = "once_cell"
-version = "1.21.0"
+version = "1.21.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cde51589ab56b20a6f686b2c68f7a0bd6add753d697abf720d63f8db3ab7b1ad"
+checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d"
 
 [[package]]
 name = "opaque-debug"
@@ -3253,9 +3264,9 @@ dependencies = [
 
 [[package]]
 name = "proc-macro2"
-version = "1.0.94"
+version = "1.0.95"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a31971752e70b8b2686d7e46ec17fb38dad4051d94024c88df49b667caea9c84"
+checksum = "02b3e5e68a3a1a02aad3ec490a98007cbc13c37cbe84a3cd7b8e406d76e7f778"
 dependencies = [
  "unicode-ident",
 ]
@@ -3325,35 +3336,37 @@ dependencies = [
 
 [[package]]
 name = "quinn"
-version = "0.11.6"
+version = "0.11.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "62e96808277ec6f97351a2380e6c25114bc9e67037775464979f3037c92d05ef"
+checksum = "c3bd15a6f2967aef83887dcb9fec0014580467e33720d073560cf015a5683012"
 dependencies = [
  "bytes",
+ "cfg_aliases",
  "futures-io",
  "pin-project-lite",
  "quinn-proto",
  "quinn-udp",
  "rustc-hash",
- "rustls 0.23.23",
+ "rustls 0.23.26",
  "socket2",
  "thiserror 2.0.12",
  "tokio",
  "tracing",
+ "web-time",
 ]
 
 [[package]]
 name = "quinn-proto"
-version = "0.11.9"
+version = "0.11.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a2fe5ef3495d7d2e377ff17b1a8ce2ee2ec2a18cde8b6ad6619d65d0701c135d"
+checksum = "b820744eb4dc9b57a3398183639c511b5a26d2ed702cedd3febaa1393caa22cc"
 dependencies = [
  "bytes",
- "getrandom 0.2.15",
- "rand 0.8.5",
+ "getrandom 0.3.2",
+ "rand 0.9.0",
  "ring 0.17.14",
  "rustc-hash",
- "rustls 0.23.23",
+ "rustls 0.23.26",
  "rustls-pki-types",
  "slab",
  "thiserror 2.0.12",
@@ -3364,9 +3377,9 @@ dependencies = [
 
 [[package]]
 name = "quinn-udp"
-version = "0.5.10"
+version = "0.5.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e46f3055866785f6b92bc6164b76be02ca8f2eb4b002c0354b28cf4c119e5944"
+checksum = "541d0f57c6ec747a90738a52741d3221f7960e8ac2f0ff4b1a63680e033b4ab5"
 dependencies = [
  "cfg_aliases",
  "libc",
@@ -3385,6 +3398,12 @@ dependencies = [
  "proc-macro2",
 ]
 
+[[package]]
+name = "r-efi"
+version = "5.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "74765f6d916ee2faa39bc8e68e4f3ed8949b48cccdac59983d287a7cb71ce9c5"
+
 [[package]]
 name = "rand"
 version = "0.8.5"
@@ -3442,7 +3461,7 @@ version = "0.9.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38"
 dependencies = [
- "getrandom 0.3.1",
+ "getrandom 0.3.2",
 ]
 
 [[package]]
@@ -3460,9 +3479,9 @@ dependencies = [
 
 [[package]]
 name = "redox_syscall"
-version = "0.5.10"
+version = "0.5.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0b8c0c260b63a8219631167be35e6a988e9554dbd323f8bd08439c8ed1302bd1"
+checksum = "d2f103c6d277498fbceb16e84d317e2a400f160f46904d5f5410848c829511a3"
 dependencies = [
  "bitflags 2.9.0",
 ]
@@ -3523,12 +3542,11 @@ dependencies = [
 
 [[package]]
 name = "resolv-conf"
-version = "0.7.0"
+version = "0.7.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "52e44394d2086d010551b14b53b1f24e31647570cd1deb0379e2c21b329aba00"
+checksum = "48375394603e3dd4b2d64371f7148fd8c7baa2680e28741f2cb8d23b59e3d4c4"
 dependencies = [
  "hostname",
- "quick-error",
 ]
 
 [[package]]
@@ -3637,7 +3655,7 @@ dependencies = [
 [[package]]
 name = "runtime"
 version = "0.1.0"
-source = "git+https://github.com/aukilabs/posemesh.git?branch=feature%2Fdata-node#36aae24b3e6eb4e087509ef371e64a0e667997cc"
+source = "git+https://github.com/aukilabs/posemesh.git?branch=feature%2Fdata-node#2690043823011c811290797294fac3b2e6d0d6b4"
 dependencies = [
  "once_cell",
  "tokio",
@@ -3688,14 +3706,14 @@ dependencies = [
 
 [[package]]
 name = "rustix"
-version = "1.0.2"
+version = "1.0.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f7178faa4b75a30e269c71e61c353ce2748cf3d76f0c44c393f4e60abf49b825"
+checksum = "d97817398dd4bb2e6da002002db259209759911da105da92bec29ccb12cf58bf"
 dependencies = [
  "bitflags 2.9.0",
  "errno",
  "libc",
- "linux-raw-sys 0.9.2",
+ "linux-raw-sys 0.9.4",
  "windows-sys 0.59.0",
 ]
 
@@ -3713,14 +3731,14 @@ dependencies = [
 
 [[package]]
 name = "rustls"
-version = "0.23.23"
+version = "0.23.26"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "47796c98c480fce5406ef69d1c76378375492c3b0a0de587be0c1d9feb12f395"
+checksum = "df51b5869f3a441595eac5e8ff14d486ff285f7b8c0df8770e49c3b56351f0f0"
 dependencies = [
  "once_cell",
  "ring 0.17.14",
  "rustls-pki-types",
- "rustls-webpki 0.102.8",
+ "rustls-webpki 0.103.1",
  "subtle",
  "zeroize",
 ]
@@ -3746,9 +3764,9 @@ dependencies = [
 
 [[package]]
 name = "rustls-webpki"
-version = "0.102.8"
+version = "0.103.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "64ca1bc8749bd4cf37b5ce386cc146580777b4e8572c7b97baf22c83f444bee9"
+checksum = "fef8b8769aaccf73098557a87cd1816b4f9c7c16811c9c77142aa695c16f2c03"
 dependencies = [
  "ring 0.17.14",
  "rustls-pki-types",
@@ -4006,9 +4024,9 @@ dependencies = [
 
 [[package]]
 name = "smallvec"
-version = "1.14.0"
+version = "1.15.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7fcf8323ef1faaee30a44a340193b1ac6814fd9b7b4e88e9d4519a3e4abe1cfd"
+checksum = "8917285742e9f3e1683f0a9c4e6b57960b7314d0b08d30d1ecd426713ee2eee9"
 
 [[package]]
 name = "smol_str"
@@ -4038,9 +4056,9 @@ dependencies = [
 
 [[package]]
 name = "socket2"
-version = "0.5.8"
+version = "0.5.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c970269d99b64e60ec3bd6ad27270092a5394c4e309314b18ae3fe575695fbe8"
+checksum = "4f5fd57c80058a56cf5c777ab8a126398ece8e442983605d280a44ce79d0edef"
 dependencies = [
  "libc",
  "windows-sys 0.52.0",
@@ -4233,15 +4251,14 @@ dependencies = [
 
 [[package]]
 name = "tempfile"
-version = "3.18.0"
+version = "3.19.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2c317e0a526ee6120d8dabad239c8dadca62b24b6f168914bbbc8e2fb1f0e567"
+checksum = "7437ac7763b9b123ccf33c338a5cc1bac6f69b45a136c19bdd8a65e3916435bf"
 dependencies = [
- "cfg-if",
  "fastrand",
- "getrandom 0.3.1",
+ "getrandom 0.3.2",
  "once_cell",
- "rustix 1.0.2",
+ "rustix 1.0.5",
  "windows-sys 0.59.0",
 ]
 
@@ -4315,9 +4332,9 @@ dependencies = [
 
 [[package]]
 name = "time"
-version = "0.3.39"
+version = "0.3.41"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dad298b01a40a23aac4580b67e3dbedb7cc8402f3592d7f49469de2ea4aecdd8"
+checksum = "8a7619e19bc266e0f9c5e6686659d394bc57973859340060a69221e57dbc0c40"
 dependencies = [
  "deranged",
  "itoa",
@@ -4330,15 +4347,15 @@ dependencies = [
 
 [[package]]
 name = "time-core"
-version = "0.1.3"
+version = "0.1.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "765c97a5b985b7c11d7bc27fa927dc4fe6af3a6dfb021d28deb60d3bf51e76ef"
+checksum = "c9e9a38711f559d9e3ce1cdb06dd7c5b8ea546bc90052da6d06bb76da74bb07c"
 
 [[package]]
 name = "time-macros"
-version = "0.2.20"
+version = "0.2.22"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e8093bc3e81c3bc5f7879de09619d06c9a5a5e45ca44dfeeb7225bae38005c5c"
+checksum = "3526739392ec93fd8b359c8e98514cb3e8e021beb4e5f597b00a0221f8ed8a49"
 dependencies = [
  "num-conv",
  "time-core",
@@ -4381,9 +4398,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
 
 [[package]]
 name = "tokio"
-version = "1.44.0"
+version = "1.44.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9975ea0f48b5aa3972bf2d888c238182458437cc2a19374b81b25cdf1023fb3a"
+checksum = "e6b88822cbe49de4185e3a4cbf8321dd487cf5fe0c5c65695fef6346371e9c48"
 dependencies = [
  "backtrace",
  "bytes",
@@ -4436,9 +4453,9 @@ dependencies = [
 
 [[package]]
 name = "tokio-util"
-version = "0.7.13"
+version = "0.7.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d7fcaa8d55a2bdd6b83ace262b016eca0d79ee02818c5c1bcdf0305114081078"
+checksum = "6b9590b93e6fcc1739458317cccd391ad3955e2bde8913edf6f95f9e65a8f034"
 dependencies = [
  "bytes",
  "futures-core",
@@ -4703,7 +4720,7 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
 [[package]]
 name = "utils"
 version = "0.1.0"
-source = "git+https://github.com/aukilabs/posemesh.git?branch=feature%2Fdata-node#36aae24b3e6eb4e087509ef371e64a0e667997cc"
+source = "git+https://github.com/aukilabs/posemesh.git?branch=feature%2Fdata-node#2690043823011c811290797294fac3b2e6d0d6b4"
 dependencies = [
  "futures",
  "gloo-timers 0.3.0",
@@ -4717,11 +4734,11 @@ dependencies = [
 
 [[package]]
 name = "uuid"
-version = "1.15.1"
+version = "1.16.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e0f540e3240398cce6128b64ba83fdbdd86129c16a3aa1a3a252efd66eb3d587"
+checksum = "458f7a779bf54acc9f347480ac654f68407d3aab21269a6e3c9f922acd9e2da9"
 dependencies = [
- "getrandom 0.3.1",
+ "getrandom 0.3.2",
  "js-sys",
  "wasm-bindgen",
 ]
@@ -4776,9 +4793,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
 
 [[package]]
 name = "wasi"
-version = "0.13.3+wasi-0.2.2"
+version = "0.14.2+wasi-0.2.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "26816d2e1a4a36a2940b96c5296ce403917633dff8f3440e9b236ed6f6bacad2"
+checksum = "9683f9a5a998d873c0d21fcbe3c083009670149a8fab228644b8bd36b2c48cb3"
 dependencies = [
  "wit-bindgen-rt",
 ]
@@ -5123,9 +5140,9 @@ dependencies = [
 
 [[package]]
 name = "widestring"
-version = "1.1.0"
+version = "1.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7219d36b6eac893fa81e84ebe06485e7dcbb616177469b142df14f1f4deb1311"
+checksum = "dd7cf3379ca1aac9eea11fba24fd7e315d621f8dfe35c8d7d2be8b793726e07d"
 
 [[package]]
 name = "winapi"
@@ -5170,28 +5187,54 @@ dependencies = [
 
 [[package]]
 name = "windows-core"
-version = "0.52.0"
+version = "0.53.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9"
+checksum = "9dcc5b895a6377f1ab9fa55acedab1fd5ac0db66ad1e6c7f47e28a22e446a5dd"
 dependencies = [
+ "windows-result 0.1.2",
  "windows-targets 0.52.6",
 ]
 
 [[package]]
 name = "windows-core"
-version = "0.53.0"
+version = "0.61.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9dcc5b895a6377f1ab9fa55acedab1fd5ac0db66ad1e6c7f47e28a22e446a5dd"
+checksum = "4763c1de310c86d75a878046489e2e5ba02c649d185f21c67d4cf8a56d098980"
 dependencies = [
- "windows-result",
- "windows-targets 0.52.6",
+ "windows-implement",
+ "windows-interface",
+ "windows-link",
+ "windows-result 0.3.2",
+ "windows-strings",
+]
+
+[[package]]
+name = "windows-implement"
+version = "0.60.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a47fddd13af08290e67f4acabf4b459f647552718f683a7b415d290ac744a836"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.100",
+]
+
+[[package]]
+name = "windows-interface"
+version = "0.59.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bd9211b69f8dcdfa817bfd14bf1c97c9188afa36f4750130fcdf3f400eca9fa8"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.100",
 ]
 
 [[package]]
 name = "windows-link"
-version = "0.1.0"
+version = "0.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6dccfd733ce2b1753b03b6d3c65edf020262ea35e20ccdf3e288043e6dd620e3"
+checksum = "76840935b766e1b0a05c0066835fb9ec80071d4c09a16f6bd5f7e655e3c14c38"
 
 [[package]]
 name = "windows-result"
@@ -5202,6 +5245,24 @@ dependencies = [
  "windows-targets 0.52.6",
 ]
 
+[[package]]
+name = "windows-result"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c64fd11a4fd95df68efcfee5f44a294fe71b8bc6a91993e2791938abcc712252"
+dependencies = [
+ "windows-link",
+]
+
+[[package]]
+name = "windows-strings"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7a2ba9642430ee452d5a7aa78d72907ebe8cfda358e8cb7918a2050581322f97"
+dependencies = [
+ "windows-link",
+]
+
 [[package]]
 name = "windows-sys"
 version = "0.48.0"
@@ -5352,9 +5413,9 @@ checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
 
 [[package]]
 name = "winnow"
-version = "0.7.3"
+version = "0.7.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0e7f4ea97f6f78012141bcdb6a216b2609f0979ada50b20ca5b52dde2eac2bb1"
+checksum = "63d3fcd9bba44b03821e7d699eeee959f3126dcc4aa8e4ae18ec617c2a5cea10"
 dependencies = [
  "memchr",
 ]
@@ -5371,9 +5432,9 @@ dependencies = [
 
 [[package]]
 name = "wit-bindgen-rt"
-version = "0.33.0"
+version = "0.39.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3268f3d866458b787f390cf61f4bbb563b922d091359f9608842999eaee3943c"
+checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1"
 dependencies = [
  "bitflags 2.9.0",
 ]
@@ -5439,9 +5500,9 @@ dependencies = [
 
 [[package]]
 name = "xml-rs"
-version = "0.8.25"
+version = "0.8.26"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c5b940ebc25896e71dd073bad2dbaa2abfe97b0a391415e22ad1326d9c54e3c4"
+checksum = "a62ce76d9b56901b19a74f19431b0d8b3bc7ca4ad685a746dfd78ca8f4fc6bda"
 
 [[package]]
 name = "xmltree"
@@ -5452,6 +5513,15 @@ dependencies = [
  "xml-rs",
 ]
 
+[[package]]
+name = "xz2"
+version = "0.1.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "388c44dc09d76f1536602ead6d325eb532f5c122f17782bd57fb47baeeb767e2"
+dependencies = [
+ "lzma-sys",
+]
+
 [[package]]
 name = "yamux"
 version = "0.12.1"
@@ -5518,18 +5588,18 @@ dependencies = [
 
 [[package]]
 name = "zerocopy"
-version = "0.8.23"
+version = "0.8.24"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fd97444d05a4328b90e75e503a34bad781f14e28a823ad3557f0750df1ebcbc6"
+checksum = "2586fea28e186957ef732a5f8b3be2da217d65c5969d4b1e17f973ebbe876879"
 dependencies = [
  "zerocopy-derive",
 ]
 
 [[package]]
 name = "zerocopy-derive"
-version = "0.8.23"
+version = "0.8.24"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6352c01d0edd5db859a63e2605f4ea3183ddbd15e2c4a9e7d32184df75e4f154"
+checksum = "a996a8f63c5c4448cd959ac1bab0aaa3306ccfd060472f85943ee0750f0169be"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -5601,9 +5671,9 @@ dependencies = [
 
 [[package]]
 name = "zip"
-version = "2.2.3"
+version = "2.6.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b280484c454e74e5fff658bbf7df8fdbe7a07c6b2de4a53def232c15ef138f3a"
+checksum = "1dcb24d0152526ae49b9b96c1dcf71850ca1e0b882e4e28ed898a93c41334744"
 dependencies = [
  "aes",
  "arbitrary",
@@ -5612,17 +5682,16 @@ dependencies = [
  "crc32fast",
  "crossbeam-utils",
  "deflate64",
- "displaydoc",
  "flate2",
+ "getrandom 0.3.2",
  "hmac",
  "indexmap",
  "lzma-rs",
  "memchr",
  "pbkdf2",
- "rand 0.8.5",
  "sha1",
- "thiserror 2.0.12",
  "time",
+ "xz2",
  "zeroize",
  "zopfli",
  "zstd",
@@ -5653,18 +5722,18 @@ dependencies = [
 
 [[package]]
 name = "zstd-safe"
-version = "7.2.3"
+version = "7.2.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f3051792fbdc2e1e143244dc28c60f73d8470e93f3f9cbd0ead44da5ed802722"
+checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d"
 dependencies = [
  "zstd-sys",
 ]
 
 [[package]]
 name = "zstd-sys"
-version = "2.0.14+zstd.1.5.7"
+version = "2.0.15+zstd.1.5.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8fb060d4926e4ac3a3ad15d864e99ceb5f343c6b34f5bd6d81ae6ed417311be5"
+checksum = "eb81183ddd97d0c74cedf1d50d85c8d08c1b8b68ee863bdee9e706eedba1a237"
 dependencies = [
  "cc",
  "pkg-config",

From c80440ea19adce495c57642e707c96e8c6ba1c4e Mon Sep 17 00:00:00 2001
From: Shuning <shuning@aukilabs.com>
Date: Wed, 16 Apr 2025 20:43:51 +0800
Subject: [PATCH 74/89] cancel networking and use auki pc for building

---
 .github/workflows/docker-build.yml | 12 ++++++------
 server/src/main.rs                 |  4 ++++
 2 files changed, 10 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml
index 1f5f7ef..ca0b020 100644
--- a/.github/workflows/docker-build.yml
+++ b/.github/workflows/docker-build.yml
@@ -11,16 +11,16 @@ concurrency:
 
 jobs:
   build:
-    runs-on: ubuntu-latest
+    runs-on: [self-hosted, Linux] 
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
 
-      - name: Free up space
-        run: |
-          sudo sudo rm -rf /usr/share/dotnet  # ~20GB
-          sudo rm -rf /opt/ghc           # ~4GB
-          sudo rm -rf /usr/local/lib/android  # ~10GB
+      # - name: Free up space
+      #   run: |
+      #     sudo sudo rm -rf /usr/share/dotnet  # ~20GB
+      #     sudo rm -rf /opt/ghc           # ~4GB
+      #     sudo rm -rf /usr/local/lib/android  # ~10GB
 
       - name: Configure AWS credentials
         uses: aws-actions/configure-aws-credentials@v4
diff --git a/server/src/main.rs b/server/src/main.rs
index d98e71c..9345967 100644
--- a/server/src/main.rs
+++ b/server/src/main.rs
@@ -39,6 +39,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     let remote_storage = RemoteDatastore::new(domain_cluster);
 
     loop {
+        let mut c = n.client.clone();
         select! {
             Some((_, stream)) = local_refinement_v1_handler.next() => {
                 let _ = tokio::spawn(local_refinement::v1(base_path.clone(), stream, remote_storage.clone(), n.client.clone()));
@@ -47,6 +48,9 @@ async fn main() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
                 let _ = tokio::spawn(global_refinement::v1(base_path.clone(), stream, remote_storage.clone(), n.client.clone()));
             }
             _ = shutdown_signal() => {
+                c.cancel().await.unwrap_or_else(|e| {
+                    eprintln!("Failed to cancel client: {}", e);
+                });
                 println!("Received termination signal, shutting down...");
                 break;
             }

From 808573297fa157a4417a7bbe68f5b8af9df76c52 Mon Sep 17 00:00:00 2001
From: Shuning <shuning@aukilabs.com>
Date: Thu, 17 Apr 2025 16:47:06 +0800
Subject: [PATCH 75/89] fix networking

---
 server/Cargo.lock | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/server/Cargo.lock b/server/Cargo.lock
index 09ed9f6..8d829dc 100644
--- a/server/Cargo.lock
+++ b/server/Cargo.lock
@@ -919,7 +919,7 @@ dependencies = [
 [[package]]
 name = "domain"
 version = "0.1.0"
-source = "git+https://github.com/aukilabs/posemesh.git?branch=feature%2Fdata-node#2690043823011c811290797294fac3b2e6d0d6b4"
+source = "git+https://github.com/aukilabs/posemesh.git?branch=feature%2Fdata-node#15d38d39f62acabfda4d590462d3556ff0b543d0"
 dependencies = [
  "async-trait",
  "cbindgen",
@@ -2836,7 +2836,7 @@ dependencies = [
 [[package]]
 name = "networking"
 version = "0.1.0"
-source = "git+https://github.com/aukilabs/posemesh.git?branch=feature%2Fdata-node#2690043823011c811290797294fac3b2e6d0d6b4"
+source = "git+https://github.com/aukilabs/posemesh.git?branch=feature%2Fdata-node#15d38d39f62acabfda4d590462d3556ff0b543d0"
 dependencies = [
  "cbindgen",
  "console_error_panic_hook",
@@ -3655,7 +3655,7 @@ dependencies = [
 [[package]]
 name = "runtime"
 version = "0.1.0"
-source = "git+https://github.com/aukilabs/posemesh.git?branch=feature%2Fdata-node#2690043823011c811290797294fac3b2e6d0d6b4"
+source = "git+https://github.com/aukilabs/posemesh.git?branch=feature%2Fdata-node#15d38d39f62acabfda4d590462d3556ff0b543d0"
 dependencies = [
  "once_cell",
  "tokio",
@@ -4720,7 +4720,7 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
 [[package]]
 name = "utils"
 version = "0.1.0"
-source = "git+https://github.com/aukilabs/posemesh.git?branch=feature%2Fdata-node#2690043823011c811290797294fac3b2e6d0d6b4"
+source = "git+https://github.com/aukilabs/posemesh.git?branch=feature%2Fdata-node#15d38d39f62acabfda4d590462d3556ff0b543d0"
 dependencies = [
  "futures",
  "gloo-timers 0.3.0",

From 64d4a5e696c01ab9a92c31f28521fbd0eeec65b3 Mon Sep 17 00:00:00 2001
From: Shuning <shuning@aukilabs.com>
Date: Thu, 24 Apr 2025 10:12:23 +0800
Subject: [PATCH 76/89] update domain

---
 server/Cargo.lock  | 6 +++---
 server/Cargo.toml  | 4 ++--
 server/src/main.rs | 3 ++-
 3 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/server/Cargo.lock b/server/Cargo.lock
index 8d829dc..2405ec1 100644
--- a/server/Cargo.lock
+++ b/server/Cargo.lock
@@ -919,7 +919,7 @@ dependencies = [
 [[package]]
 name = "domain"
 version = "0.1.0"
-source = "git+https://github.com/aukilabs/posemesh.git?branch=feature%2Fdata-node#15d38d39f62acabfda4d590462d3556ff0b543d0"
+source = "git+https://github.com/aukilabs/posemesh.git?branch=feature%2Fnamespace#4c491cf17e9b8f2a55b8b894c7671204b4c39aab"
 dependencies = [
  "async-trait",
  "cbindgen",
@@ -2836,7 +2836,7 @@ dependencies = [
 [[package]]
 name = "networking"
 version = "0.1.0"
-source = "git+https://github.com/aukilabs/posemesh.git?branch=feature%2Fdata-node#15d38d39f62acabfda4d590462d3556ff0b543d0"
+source = "git+https://github.com/aukilabs/posemesh.git?branch=feature%2Fnamespace#4c491cf17e9b8f2a55b8b894c7671204b4c39aab"
 dependencies = [
  "cbindgen",
  "console_error_panic_hook",
@@ -4720,7 +4720,7 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
 [[package]]
 name = "utils"
 version = "0.1.0"
-source = "git+https://github.com/aukilabs/posemesh.git?branch=feature%2Fdata-node#15d38d39f62acabfda4d590462d3556ff0b543d0"
+source = "git+https://github.com/aukilabs/posemesh.git?branch=feature%2Fnamespace#4c491cf17e9b8f2a55b8b894c7671204b4c39aab"
 dependencies = [
  "futures",
  "gloo-timers 0.3.0",
diff --git a/server/Cargo.toml b/server/Cargo.toml
index 926757e..0ea6acd 100644
--- a/server/Cargo.toml
+++ b/server/Cargo.toml
@@ -4,12 +4,12 @@ version = "0.1.0"
 edition = "2021"
 
 [dependencies]
-networking = { git = "https://github.com/aukilabs/posemesh.git", branch = "feature/data-node" }
+networking = { git = "https://github.com/aukilabs/posemesh.git", branch = "feature/namespace" }
 futures = "0.3.30"
 jsonwebtoken = "9.3.0"
 uuid = "1.13.2"
 rand = "0.9.0"
-domain = { git = "https://github.com/aukilabs/posemesh.git", branch = "feature/data-node" }
+domain = { git = "https://github.com/aukilabs/posemesh.git", branch = "feature/namespace" }
 tracing = "0.1.40"
 tracing-subscriber = "0.3.18"
 tokio = "1.44.0"
diff --git a/server/src/main.rs b/server/src/main.rs
index 9345967..f12e8ac 100644
--- a/server/src/main.rs
+++ b/server/src/main.rs
@@ -31,8 +31,9 @@ async fn main() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     let base_path = format!("./volume/{}", name);
     let domain_manager = args[3].clone();
     let private_key_path = format!("{}/pkey", base_path);
+    tracing_subscriber::fmt().with_env_filter(tracing_subscriber::EnvFilter::from_default_env()).init();
 
-    let domain_cluster = DomainCluster::new(domain_manager.clone(), name, false, port, false, false, None, Some(private_key_path));
+    let domain_cluster = DomainCluster::new("".to_string(), domain_manager.clone(), name, false, port, false, false, None, Some(private_key_path), vec![domain_manager.clone()]);
     let mut n = domain_cluster.peer.clone();
     let mut local_refinement_v1_handler = n.client.set_stream_handler("/local-refinement/v1".to_string()).await.unwrap();
     let mut global_refinement_v1_handler = n.client.set_stream_handler("/global-refinement/v1".to_string()).await.unwrap();

From 12edd2c93f820884915404f112b2a40d4a8af6b3 Mon Sep 17 00:00:00 2001
From: Shuning <shuning@aukilabs.com>
Date: Thu, 24 Apr 2025 22:33:32 +0800
Subject: [PATCH 77/89] update Cargo.lock

---
 server/Cargo.lock | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/server/Cargo.lock b/server/Cargo.lock
index 2405ec1..b2c66f5 100644
--- a/server/Cargo.lock
+++ b/server/Cargo.lock
@@ -2853,6 +2853,7 @@ dependencies = [
  "serde",
  "serde-wasm-bindgen",
  "serde_json",
+ "thiserror 2.0.12",
  "tokio",
  "tracing",
  "tracing-subscriber",
@@ -3655,7 +3656,7 @@ dependencies = [
 [[package]]
 name = "runtime"
 version = "0.1.0"
-source = "git+https://github.com/aukilabs/posemesh.git?branch=feature%2Fdata-node#15d38d39f62acabfda4d590462d3556ff0b543d0"
+source = "git+https://github.com/aukilabs/posemesh.git?branch=feature%2Fnamespace#4c491cf17e9b8f2a55b8b894c7671204b4c39aab"
 dependencies = [
  "once_cell",
  "tokio",

From af653dfc9ca521aaba31e237fa530354a70b5335 Mon Sep 17 00:00:00 2001
From: Shuning <shuning@aukilabs.com>
Date: Fri, 25 Apr 2025 10:15:43 +0800
Subject: [PATCH 78/89] update domain

---
 server/Cargo.lock | 8 ++++----
 server/Cargo.toml | 4 ++--
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/server/Cargo.lock b/server/Cargo.lock
index b2c66f5..f983c5f 100644
--- a/server/Cargo.lock
+++ b/server/Cargo.lock
@@ -919,7 +919,7 @@ dependencies = [
 [[package]]
 name = "domain"
 version = "0.1.0"
-source = "git+https://github.com/aukilabs/posemesh.git?branch=feature%2Fnamespace#4c491cf17e9b8f2a55b8b894c7671204b4c39aab"
+source = "git+https://github.com/aukilabs/posemesh.git?branch=main#f617f1bd2f72bac6a1d3d31447bcc19dafb7bef8"
 dependencies = [
  "async-trait",
  "cbindgen",
@@ -2836,7 +2836,7 @@ dependencies = [
 [[package]]
 name = "networking"
 version = "0.1.0"
-source = "git+https://github.com/aukilabs/posemesh.git?branch=feature%2Fnamespace#4c491cf17e9b8f2a55b8b894c7671204b4c39aab"
+source = "git+https://github.com/aukilabs/posemesh.git?branch=main#f617f1bd2f72bac6a1d3d31447bcc19dafb7bef8"
 dependencies = [
  "cbindgen",
  "console_error_panic_hook",
@@ -3656,7 +3656,7 @@ dependencies = [
 [[package]]
 name = "runtime"
 version = "0.1.0"
-source = "git+https://github.com/aukilabs/posemesh.git?branch=feature%2Fnamespace#4c491cf17e9b8f2a55b8b894c7671204b4c39aab"
+source = "git+https://github.com/aukilabs/posemesh.git?branch=main#f617f1bd2f72bac6a1d3d31447bcc19dafb7bef8"
 dependencies = [
  "once_cell",
  "tokio",
@@ -4721,7 +4721,7 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
 [[package]]
 name = "utils"
 version = "0.1.0"
-source = "git+https://github.com/aukilabs/posemesh.git?branch=feature%2Fnamespace#4c491cf17e9b8f2a55b8b894c7671204b4c39aab"
+source = "git+https://github.com/aukilabs/posemesh.git?branch=main#f617f1bd2f72bac6a1d3d31447bcc19dafb7bef8"
 dependencies = [
  "futures",
  "gloo-timers 0.3.0",
diff --git a/server/Cargo.toml b/server/Cargo.toml
index 0ea6acd..8e1836d 100644
--- a/server/Cargo.toml
+++ b/server/Cargo.toml
@@ -4,12 +4,12 @@ version = "0.1.0"
 edition = "2021"
 
 [dependencies]
-networking = { git = "https://github.com/aukilabs/posemesh.git", branch = "feature/namespace" }
+networking = { git = "https://github.com/aukilabs/posemesh.git", branch = "main" }
 futures = "0.3.30"
 jsonwebtoken = "9.3.0"
 uuid = "1.13.2"
 rand = "0.9.0"
-domain = { git = "https://github.com/aukilabs/posemesh.git", branch = "feature/namespace" }
+domain = { git = "https://github.com/aukilabs/posemesh.git", branch = "main" }
 tracing = "0.1.40"
 tracing-subscriber = "0.3.18"
 tokio = "1.44.0"

From 8a9be85d5c2add6d46193c5f19965b6f2cf619f8 Mon Sep 17 00:00:00 2001
From: RobinLindh <robin@aukilabs.com>
Date: Tue, 29 Apr 2025 11:14:49 +0200
Subject: [PATCH 79/89] fixes

---
 server/src/local_refinement.rs | 5 +++--
 server/src/utils.rs            | 2 +-
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/server/src/local_refinement.rs b/server/src/local_refinement.rs
index 13d556c..69fc16b 100644
--- a/server/src/local_refinement.rs
+++ b/server/src/local_refinement.rs
@@ -6,8 +6,7 @@ use futures::StreamExt;
 use tokio::{sync::watch, time::sleep};
 use std::io::{BufReader, Read, Write};
 use zip::write::SimpleFileOptions;
-
-use crate::utils::{handshake, health, write_scan_data_summary};
+use crate::utils::{handshake, health, write_scan_data_summary, execute_python};
 
 pub(crate) async fn v1<S: AsyncStream>(base_path: String, mut stream: S, datastore: RemoteDatastore, mut c: Client) {
     let claim = handshake(&mut stream).await.expect("Failed to handshake");
@@ -206,6 +205,8 @@ async fn run(domain_id: &str, job_id: &str, task_name: &str, context: LocalRefin
         "--scans", context.suffix.as_str(),
     ];
 
+    execute_python(params).await?;
+
     let sfm = context.output_folder.join(context.suffix.clone()).join("sfm");
     let zip_path = sfm.join(context.suffix.clone() + ".zip");
     let mut zip = zip::ZipWriter::new(fs::File::create(&zip_path)?);
diff --git a/server/src/utils.rs b/server/src/utils.rs
index de65350..76a1fa5 100644
--- a/server/src/utils.rs
+++ b/server/src/utils.rs
@@ -22,7 +22,7 @@ pub struct TaskTokenClaim {
 }
 
 pub fn decode_jwt(token: &str) -> Result<TaskTokenClaim, Box<dyn std::error::Error + Send + Sync>> {
-    let token_data = decode::<TaskTokenClaim>(token, &DecodingKey::from_secret("secret".as_ref()), &Validation::new(Algorithm::HS256))?;
+    let token_data = decode::<TaskTokenClaim>(token, &DecodingKey::from_secret("d5b0966e-8876-4b7c-b6e3-863a0c72c7fd".as_ref()), &Validation::new(Algorithm::HS256))?;
     Ok(token_data.claims)
 }
 

From 7da3976807413d5ec52d9b5261124b4d3e3f5dd5 Mon Sep 17 00:00:00 2001
From: Shuning <shuning@aukilabs.com>
Date: Mon, 5 May 2025 09:15:06 +0800
Subject: [PATCH 80/89] load public key

---
 server/Cargo.lock               | 498 ++++++++------------------------
 server/Cargo.toml               |   9 +-
 server/src/global_refinement.rs |  72 +++--
 server/src/local_refinement.rs  |  24 +-
 server/src/main.rs              |  18 +-
 server/src/utils.rs             |  39 +--
 6 files changed, 179 insertions(+), 481 deletions(-)

diff --git a/server/Cargo.lock b/server/Cargo.lock
index f983c5f..3c5516e 100644
--- a/server/Cargo.lock
+++ b/server/Cargo.lock
@@ -67,21 +67,6 @@ version = "0.2.21"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923"
 
-[[package]]
-name = "android-tzdata"
-version = "0.1.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0"
-
-[[package]]
-name = "android_system_properties"
-version = "0.1.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311"
-dependencies = [
- "libc",
-]
-
 [[package]]
 name = "ansi_term"
 version = "0.12.1"
@@ -226,8 +211,8 @@ checksum = "965c2d33e53cb6b267e148a4cb0760bc01f4904c1cd4bb4002a085bb016d1490"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.100",
- "synstructure 0.13.1",
+ "syn 2.0.101",
+ "synstructure 0.13.2",
 ]
 
 [[package]]
@@ -249,7 +234,7 @@ checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.100",
+ "syn 2.0.101",
 ]
 
 [[package]]
@@ -282,6 +267,17 @@ dependencies = [
  "pin-project-lite",
 ]
 
+[[package]]
+name = "async-timer"
+version = "0.7.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ba5fa6ed76cb2aa820707b4eb9ec46f42da9ce70b0eafab5e5e34942b38a44d5"
+dependencies = [
+ "libc",
+ "wasm-bindgen",
+ "winapi",
+]
+
 [[package]]
 name = "async-trait"
 version = "0.1.88"
@@ -290,7 +286,7 @@ checksum = "e539d3fca749fcee5236ab05e93a52867dd549cc157c8cb7f99595f3cedffdb5"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.100",
+ "syn 2.0.101",
 ]
 
 [[package]]
@@ -497,7 +493,7 @@ version = "0.28.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "eadd868a2ce9ca38de7eeafdcec9c7065ef89b42b32f0839278d55f35c54d1ff"
 dependencies = [
- "clap 4.5.36",
+ "clap 4.5.37",
  "heck 0.4.1",
  "indexmap",
  "log",
@@ -505,16 +501,16 @@ dependencies = [
  "quote",
  "serde",
  "serde_json",
- "syn 2.0.100",
+ "syn 2.0.101",
  "tempfile",
  "toml",
 ]
 
 [[package]]
 name = "cc"
-version = "1.2.19"
+version = "1.2.21"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8e3a13707ac958681c13b39b458c073d0d9bc8a22cb1b2f4c8e55eb72c13f362"
+checksum = "8691782945451c1c383942c4874dbe63814f61cb57ef773cda2972682b7bb3c0"
 dependencies = [
  "jobserver",
  "libc",
@@ -569,20 +565,6 @@ dependencies = [
  "zeroize",
 ]
 
-[[package]]
-name = "chrono"
-version = "0.4.40"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1a7964611d71df112cb1730f2ee67324fcf4d0fc6606acbbe9bfe06df124637c"
-dependencies = [
- "android-tzdata",
- "iana-time-zone",
- "js-sys",
- "num-traits",
- "wasm-bindgen",
- "windows-link",
-]
-
 [[package]]
 name = "cipher"
 version = "0.4.4"
@@ -611,18 +593,18 @@ dependencies = [
 
 [[package]]
 name = "clap"
-version = "4.5.36"
+version = "4.5.37"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2df961d8c8a0d08aa9945718ccf584145eee3f3aa06cddbeac12933781102e04"
+checksum = "eccb054f56cbd38340b380d4a8e69ef1f02f1af43db2f0cc817a4774d80ae071"
 dependencies = [
  "clap_builder",
 ]
 
 [[package]]
 name = "clap_builder"
-version = "4.5.36"
+version = "4.5.37"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "132dbda40fb6753878316a489d5a1242a8ef2f0d9e47ba01c951ea8aa7d013a5"
+checksum = "efd9466fac8543255d3b1fcad4762c5e116ffe808c8a3043d4263cd4fd4862a2"
 dependencies = [
  "anstream",
  "anstyle",
@@ -799,7 +781,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.100",
+ "syn 2.0.101",
 ]
 
 [[package]]
@@ -825,7 +807,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8d162beedaa69905488a8da94f5ac3edb4dd4788b732fadb7bd120b2625c1976"
 dependencies = [
  "data-encoding",
- "syn 2.0.100",
+ "syn 2.0.101",
 ]
 
 [[package]]
@@ -836,9 +818,9 @@ checksum = "da692b8d1080ea3045efaab14434d40468c3d8657e42abddfffca87b428f4c1b"
 
 [[package]]
 name = "der"
-version = "0.7.9"
+version = "0.7.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f55bf8e7b65898637379c1b74eb1551107c8294ed26d855ceb9fd1a09cfc9bc0"
+checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb"
 dependencies = [
  "const-oid",
  "pem-rfc7468",
@@ -890,7 +872,7 @@ checksum = "30542c1ad912e0e3d22a1935c290e12e8a29d704a420177a31faad4a601a0800"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.100",
+ "syn 2.0.101",
 ]
 
 [[package]]
@@ -913,15 +895,17 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.100",
+ "syn 2.0.101",
 ]
 
 [[package]]
 name = "domain"
 version = "0.1.0"
-source = "git+https://github.com/aukilabs/posemesh.git?branch=main#f617f1bd2f72bac6a1d3d31447bcc19dafb7bef8"
+source = "git+https://github.com/aukilabs/posemesh.git?branch=feature%2Ffix-auth#192cf534f0eb26f08b592f52dc3fee3f72a59efe"
 dependencies = [
+ "async-timer",
  "async-trait",
+ "base64 0.22.1",
  "cbindgen",
  "console_error_panic_hook",
  "futures",
@@ -932,6 +916,7 @@ dependencies = [
  "pb-rs",
  "quick-protobuf",
  "quick-protobuf-codec 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "ring 0.17.14",
  "rs_merkle",
  "runtime",
  "serde",
@@ -940,10 +925,10 @@ dependencies = [
  "sha2",
  "thiserror 2.0.12",
  "tokio",
- "tokio-postgres",
  "tracing",
  "tracing-subscriber",
  "tracing-wasm",
+ "utils",
  "uuid",
  "wasm-bindgen",
  "wasm-bindgen-futures",
@@ -1030,7 +1015,7 @@ dependencies = [
  "heck 0.5.0",
  "proc-macro2",
  "quote",
- "syn 2.0.100",
+ "syn 2.0.101",
 ]
 
 [[package]]
@@ -1083,12 +1068,6 @@ dependencies = [
  "pin-project-lite",
 ]
 
-[[package]]
-name = "fallible-iterator"
-version = "0.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7"
-
 [[package]]
 name = "fastrand"
 version = "2.3.0"
@@ -1219,7 +1198,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.100",
+ "syn 2.0.101",
 ]
 
 [[package]]
@@ -1297,9 +1276,9 @@ dependencies = [
 
 [[package]]
 name = "getrandom"
-version = "0.2.15"
+version = "0.2.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7"
+checksum = "335ff9f135e4384c8150d6f27c6daed433577f86b4750418338c01a1a2528592"
 dependencies = [
  "cfg-if",
  "js-sys",
@@ -1394,9 +1373,9 @@ dependencies = [
 
 [[package]]
 name = "hashbrown"
-version = "0.15.2"
+version = "0.15.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289"
+checksum = "84b26c544d002229e640969970a2e74021aadf6e2f96372b9c58eff97de08eb3"
 dependencies = [
  "allocator-api2",
  "equivalent",
@@ -1512,17 +1491,6 @@ dependencies = [
  "digest",
 ]
 
-[[package]]
-name = "hostname"
-version = "0.4.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a56f203cd1c76362b69e3863fd987520ac36cf70a8c92627449b2f64a8cf7d65"
-dependencies = [
- "cfg-if",
- "libc",
- "windows-link",
-]
-
 [[package]]
 name = "http"
 version = "0.2.12"
@@ -1623,30 +1591,6 @@ dependencies = [
  "tracing",
 ]
 
-[[package]]
-name = "iana-time-zone"
-version = "0.1.63"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b0c919e5debc312ad217002b8048a17b7d83f80703865bbfcfebb0458b0b27d8"
-dependencies = [
- "android_system_properties",
- "core-foundation-sys",
- "iana-time-zone-haiku",
- "js-sys",
- "log",
- "wasm-bindgen",
- "windows-core 0.61.0",
-]
-
-[[package]]
-name = "iana-time-zone-haiku"
-version = "0.1.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f"
-dependencies = [
- "cc",
-]
-
 [[package]]
 name = "icu_collections"
 version = "1.5.0"
@@ -1762,7 +1706,7 @@ checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.100",
+ "syn 2.0.101",
 ]
 
 [[package]]
@@ -1977,7 +1921,7 @@ dependencies = [
  "either",
  "futures",
  "futures-timer",
- "getrandom 0.2.15",
+ "getrandom 0.2.16",
  "libp2p-allow-block-list",
  "libp2p-autonat",
  "libp2p-connection-limits",
@@ -2130,7 +2074,7 @@ dependencies = [
  "fnv",
  "futures",
  "futures-ticker",
- "getrandom 0.2.15",
+ "getrandom 0.2.16",
  "hex_fmt",
  "instant",
  "libp2p-core",
@@ -2173,9 +2117,9 @@ dependencies = [
 
 [[package]]
 name = "libp2p-identity"
-version = "0.2.10"
+version = "0.2.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "257b5621d159b32282eac446bed6670c39c7dc68a200a992d8f056afa0066f6d"
+checksum = "fbb68ea10844211a59ce46230909fd0ea040e8a192454d4cc2ee0d53e12280eb"
 dependencies = [
  "bs58",
  "ed25519-dalek",
@@ -2185,7 +2129,7 @@ dependencies = [
  "rand 0.8.5",
  "serde",
  "sha2",
- "thiserror 1.0.69",
+ "thiserror 2.0.12",
  "tracing",
  "zeroize",
 ]
@@ -2372,7 +2316,7 @@ dependencies = [
  "fnv",
  "futures",
  "futures-timer",
- "getrandom 0.2.15",
+ "getrandom 0.2.16",
  "libp2p-core",
  "libp2p-identity",
  "libp2p-swarm-derive",
@@ -2396,7 +2340,7 @@ dependencies = [
  "heck 0.5.0",
  "proc-macro2",
  "quote",
- "syn 2.0.100",
+ "syn 2.0.101",
 ]
 
 [[package]]
@@ -2505,7 +2449,7 @@ source = "git+https://github.com/aukilabs/rust-libp2p.git?branch=fix%2Fgossipsub
 dependencies = [
  "bytes",
  "futures",
- "getrandom 0.2.15",
+ "getrandom 0.2.16",
  "hex",
  "js-sys",
  "libp2p-core",
@@ -2604,12 +2548,6 @@ dependencies = [
  "scopeguard",
 ]
 
-[[package]]
-name = "lockfree-object-pool"
-version = "0.1.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9374ef4228402d4b7e403e5838cb880d9ee663314b0a900d5a6aabf0c213552e"
-
 [[package]]
 name = "log"
 version = "0.4.27"
@@ -2836,7 +2774,7 @@ dependencies = [
 [[package]]
 name = "networking"
 version = "0.1.0"
-source = "git+https://github.com/aukilabs/posemesh.git?branch=main#f617f1bd2f72bac6a1d3d31447bcc19dafb7bef8"
+source = "git+https://github.com/aukilabs/posemesh.git?branch=feature%2Ffix-auth#192cf534f0eb26f08b592f52dc3fee3f72a59efe"
 dependencies = [
  "cbindgen",
  "console_error_panic_hook",
@@ -3097,24 +3035,6 @@ version = "2.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e"
 
-[[package]]
-name = "phf"
-version = "0.11.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078"
-dependencies = [
- "phf_shared",
-]
-
-[[package]]
-name = "phf_shared"
-version = "0.11.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "67eabc2ef2a60eb7faa00097bd1ffdb5bd28e62bf39990626a582201b7a754e5"
-dependencies = [
- "siphasher",
-]
-
 [[package]]
 name = "pin-project"
 version = "1.1.10"
@@ -3132,7 +3052,7 @@ checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.100",
+ "syn 2.0.101",
 ]
 
 [[package]]
@@ -3207,38 +3127,6 @@ version = "1.11.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "350e9b48cbc6b0e028b0473b114454c6316e57336ee184ceab6e53f72c178b3e"
 
-[[package]]
-name = "postgres-protocol"
-version = "0.6.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "76ff0abab4a9b844b93ef7b81f1efc0a366062aaef2cd702c76256b5dc075c54"
-dependencies = [
- "base64 0.22.1",
- "byteorder",
- "bytes",
- "fallible-iterator",
- "hmac",
- "md-5",
- "memchr",
- "rand 0.9.0",
- "sha2",
- "stringprep",
-]
-
-[[package]]
-name = "postgres-types"
-version = "0.2.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "613283563cd90e1dfc3518d548caee47e0e725455ed619881f5cf21f36de4b48"
-dependencies = [
- "bytes",
- "fallible-iterator",
- "postgres-protocol",
- "serde",
- "serde_json",
- "uuid",
-]
-
 [[package]]
 name = "powerfmt"
 version = "0.2.0"
@@ -3292,7 +3180,7 @@ checksum = "440f724eba9f6996b75d63681b0a92b06947f1457076d503a4d2e2c8f56442b8"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.100",
+ "syn 2.0.101",
 ]
 
 [[package]]
@@ -3358,13 +3246,13 @@ dependencies = [
 
 [[package]]
 name = "quinn-proto"
-version = "0.11.10"
+version = "0.11.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b820744eb4dc9b57a3398183639c511b5a26d2ed702cedd3febaa1393caa22cc"
+checksum = "bcbafbbdbb0f638fe3f35f3c56739f77a8a1d070cb25603226c83339b391472b"
 dependencies = [
  "bytes",
  "getrandom 0.3.2",
- "rand 0.9.0",
+ "rand 0.9.1",
  "ring 0.17.14",
  "rustc-hash",
  "rustls 0.23.26",
@@ -3378,9 +3266,9 @@ dependencies = [
 
 [[package]]
 name = "quinn-udp"
-version = "0.5.11"
+version = "0.5.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "541d0f57c6ec747a90738a52741d3221f7960e8ac2f0ff4b1a63680e033b4ab5"
+checksum = "ee4e529991f949c5e25755532370b8af5d114acae52326361d68d47af64aa842"
 dependencies = [
  "cfg_aliases",
  "libc",
@@ -3418,13 +3306,12 @@ dependencies = [
 
 [[package]]
 name = "rand"
-version = "0.9.0"
+version = "0.9.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3779b94aeb87e8bd4e834cee3650289ee9e0d5677f976ecdb6d219e5f4f6cd94"
+checksum = "9fbfd9d094a40bf3ae768db9361049ace4c0e04a4fd6b359518bd7b73a73dd97"
 dependencies = [
  "rand_chacha 0.9.0",
  "rand_core 0.9.3",
- "zerocopy",
 ]
 
 [[package]]
@@ -3453,7 +3340,7 @@ version = "0.6.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c"
 dependencies = [
- "getrandom 0.2.15",
+ "getrandom 0.2.16",
 ]
 
 [[package]]
@@ -3531,24 +3418,11 @@ version = "0.8.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c"
 
-[[package]]
-name = "regexp"
-version = "0.3.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9b045baed9fc5b2e285a599a42fb158bc483e1b6104de3a93d24bf4addd4727e"
-dependencies = [
- "lazy_static",
- "regex",
-]
-
 [[package]]
 name = "resolv-conf"
-version = "0.7.1"
+version = "0.7.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "48375394603e3dd4b2d64371f7148fd8c7baa2680e28741f2cb8d23b59e3d4c4"
-dependencies = [
- "hostname",
-]
+checksum = "fc7c8f7f733062b66dc1c63f9db168ac0b97a9210e247fa90fdc9ad08f51b302"
 
 [[package]]
 name = "rfc6979"
@@ -3583,7 +3457,7 @@ checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7"
 dependencies = [
  "cc",
  "cfg-if",
- "getrandom 0.2.15",
+ "getrandom 0.2.16",
  "libc",
  "untrusted 0.9.0",
  "windows-sys 0.52.0",
@@ -3656,7 +3530,7 @@ dependencies = [
 [[package]]
 name = "runtime"
 version = "0.1.0"
-source = "git+https://github.com/aukilabs/posemesh.git?branch=main#f617f1bd2f72bac6a1d3d31447bcc19dafb7bef8"
+source = "git+https://github.com/aukilabs/posemesh.git?branch=feature%2Ffix-auth#192cf534f0eb26f08b592f52dc3fee3f72a59efe"
 dependencies = [
  "once_cell",
  "tokio",
@@ -3707,9 +3581,9 @@ dependencies = [
 
 [[package]]
 name = "rustix"
-version = "1.0.5"
+version = "1.0.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d97817398dd4bb2e6da002002db259209759911da105da92bec29ccb12cf58bf"
+checksum = "c71e83d6afe7ff64890ec6b71d6a69bb8a610ab78ce364b3352876bb4c801266"
 dependencies = [
  "bitflags 2.9.0",
  "errno",
@@ -3887,7 +3761,7 @@ checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.100",
+ "syn 2.0.101",
 ]
 
 [[package]]
@@ -3915,22 +3789,17 @@ dependencies = [
 name = "server"
 version = "0.1.0"
 dependencies = [
- "chrono",
  "domain",
  "futures",
- "jsonwebtoken",
  "networking",
  "quick-protobuf",
- "rand 0.9.0",
  "regex",
- "regexp",
  "scopeguard",
  "serde",
  "serde_json",
  "tokio",
  "tracing",
  "tracing-subscriber",
- "uuid",
  "zip",
 ]
 
@@ -3947,9 +3816,9 @@ dependencies = [
 
 [[package]]
 name = "sha2"
-version = "0.10.8"
+version = "0.10.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8"
+checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283"
 dependencies = [
  "cfg-if",
  "cpufeatures",
@@ -3973,9 +3842,9 @@ checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"
 
 [[package]]
 name = "signal-hook-registry"
-version = "1.4.2"
+version = "1.4.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a9e9e0b4211b72e7b8b6e85c807d36c212bdb33ea8587f7569562a84df5465b1"
+checksum = "9203b8055f63a2a00e2f593bb0510367fe707d7ff1e5c872de2f537b339e5410"
 dependencies = [
  "libc",
 ]
@@ -4008,12 +3877,6 @@ dependencies = [
  "time",
 ]
 
-[[package]]
-name = "siphasher"
-version = "1.0.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "56199f7ddabf13fe5074ce809e7d3f42b42ae711800501b5b16ea82ad029c39d"
-
 [[package]]
 name = "slab"
 version = "0.4.9"
@@ -4108,17 +3971,6 @@ version = "1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f"
 
-[[package]]
-name = "stringprep"
-version = "0.1.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7b4df3d392d81bd458a8a621b8bffbd2302a12ffe288a9d931670948749463b1"
-dependencies = [
- "unicode-bidi",
- "unicode-normalization",
- "unicode-properties",
-]
-
 [[package]]
 name = "strsim"
 version = "0.8.0"
@@ -4197,9 +4049,9 @@ dependencies = [
 
 [[package]]
 name = "syn"
-version = "2.0.100"
+version = "2.0.101"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b09a44accad81e1ba1cd74a32461ba89dee89095ba17b32f5d03683b1b1fc2a0"
+checksum = "8ce2b7fc941b3a24138a0a7cf8e858bfc6a992e7978a068a5c760deb0ed43caf"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -4220,13 +4072,13 @@ dependencies = [
 
 [[package]]
 name = "synstructure"
-version = "0.13.1"
+version = "0.13.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971"
+checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.100",
+ "syn 2.0.101",
 ]
 
 [[package]]
@@ -4259,7 +4111,7 @@ dependencies = [
  "fastrand",
  "getrandom 0.3.2",
  "once_cell",
- "rustix 1.0.5",
+ "rustix 1.0.7",
  "windows-sys 0.59.0",
 ]
 
@@ -4307,7 +4159,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.100",
+ "syn 2.0.101",
 ]
 
 [[package]]
@@ -4318,7 +4170,7 @@ checksum = "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.100",
+ "syn 2.0.101",
 ]
 
 [[package]]
@@ -4423,40 +4275,14 @@ checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.100",
-]
-
-[[package]]
-name = "tokio-postgres"
-version = "0.7.13"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6c95d533c83082bb6490e0189acaa0bbeef9084e60471b696ca6988cd0541fb0"
-dependencies = [
- "async-trait",
- "byteorder",
- "bytes",
- "fallible-iterator",
- "futures-channel",
- "futures-util",
- "log",
- "parking_lot",
- "percent-encoding",
- "phf",
- "pin-project-lite",
- "postgres-protocol",
- "postgres-types",
- "rand 0.9.0",
- "socket2",
- "tokio",
- "tokio-util",
- "whoami",
+ "syn 2.0.101",
 ]
 
 [[package]]
 name = "tokio-util"
-version = "0.7.14"
+version = "0.7.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6b9590b93e6fcc1739458317cccd391ad3955e2bde8913edf6f95f9e65a8f034"
+checksum = "66a539a9ad6d5d281510d5bd368c973d636c02dbf8a67300bfb6b950696ad7df"
 dependencies = [
  "bytes",
  "futures-core",
@@ -4468,9 +4294,9 @@ dependencies = [
 
 [[package]]
 name = "toml"
-version = "0.8.20"
+version = "0.8.22"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cd87a5cdd6ffab733b2f74bc4fd7ee5fff6634124999ac278c35fc78c6120148"
+checksum = "05ae329d1f08c4d17a59bed7ff5b5a769d062e64a62d34a3261b219e62cd5aae"
 dependencies = [
  "serde",
  "serde_spanned",
@@ -4480,26 +4306,33 @@ dependencies = [
 
 [[package]]
 name = "toml_datetime"
-version = "0.6.8"
+version = "0.6.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41"
+checksum = "3da5db5a963e24bc68be8b17b6fa82814bb22ee8660f192bb182771d498f09a3"
 dependencies = [
  "serde",
 ]
 
 [[package]]
 name = "toml_edit"
-version = "0.22.24"
+version = "0.22.26"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "17b4795ff5edd201c7cd6dca065ae59972ce77d1b80fa0a84d94950ece7d1474"
+checksum = "310068873db2c5b3e7659d2cc35d21855dbafa50d1ce336397c666e3cb08137e"
 dependencies = [
  "indexmap",
  "serde",
  "serde_spanned",
  "toml_datetime",
+ "toml_write",
  "winnow",
 ]
 
+[[package]]
+name = "toml_write"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bfb942dfe1d8e29a7ee7fcbde5bd2b9a25fb89aa70caea2eba3bee836ff41076"
+
 [[package]]
 name = "tower-service"
 version = "0.3.3"
@@ -4525,7 +4358,7 @@ checksum = "395ae124c09f9e6918a2310af6038fba074bcf474ac352496d5910dd59a2226d"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.100",
+ "syn 2.0.101",
 ]
 
 [[package]]
@@ -4622,33 +4455,12 @@ dependencies = [
  "static_assertions",
 ]
 
-[[package]]
-name = "unicode-bidi"
-version = "0.3.18"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5"
-
 [[package]]
 name = "unicode-ident"
 version = "1.0.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512"
 
-[[package]]
-name = "unicode-normalization"
-version = "0.1.24"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5033c97c4262335cded6d6fc3e5c18ab755e1a3dc96376350f3d8e9f009ad956"
-dependencies = [
- "tinyvec",
-]
-
-[[package]]
-name = "unicode-properties"
-version = "0.1.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e70f2a8b45122e719eb623c01822704c4e0907e7e426a05927e1a1cfff5b75d0"
-
 [[package]]
 name = "unicode-width"
 version = "0.1.14"
@@ -4721,7 +4533,7 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
 [[package]]
 name = "utils"
 version = "0.1.0"
-source = "git+https://github.com/aukilabs/posemesh.git?branch=main#f617f1bd2f72bac6a1d3d31447bcc19dafb7bef8"
+source = "git+https://github.com/aukilabs/posemesh.git?branch=feature%2Ffix-auth#192cf534f0eb26f08b592f52dc3fee3f72a59efe"
 dependencies = [
  "futures",
  "gloo-timers 0.3.0",
@@ -4801,12 +4613,6 @@ dependencies = [
  "wit-bindgen-rt",
 ]
 
-[[package]]
-name = "wasite"
-version = "0.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b"
-
 [[package]]
 name = "wasm-bindgen"
 version = "0.2.100"
@@ -4829,7 +4635,7 @@ dependencies = [
  "log",
  "proc-macro2",
  "quote",
- "syn 2.0.100",
+ "syn 2.0.101",
  "wasm-bindgen-shared",
 ]
 
@@ -4864,7 +4670,7 @@ checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.100",
+ "syn 2.0.101",
  "wasm-bindgen-backend",
  "wasm-bindgen-shared",
 ]
@@ -5128,17 +4934,6 @@ dependencies = [
  "winapi",
 ]
 
-[[package]]
-name = "whoami"
-version = "1.6.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6994d13118ab492c3c80c1f81928718159254c53c472bf9ce36f8dae4add02a7"
-dependencies = [
- "redox_syscall",
- "wasite",
- "web-sys",
-]
-
 [[package]]
 name = "widestring"
 version = "1.2.0"
@@ -5182,7 +4977,7 @@ version = "0.53.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "efc5cf48f83140dcaab716eeaea345f9e93d0018fb81162753a3f76c3397b538"
 dependencies = [
- "windows-core 0.53.0",
+ "windows-core",
  "windows-targets 0.52.6",
 ]
 
@@ -5192,51 +4987,10 @@ version = "0.53.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9dcc5b895a6377f1ab9fa55acedab1fd5ac0db66ad1e6c7f47e28a22e446a5dd"
 dependencies = [
- "windows-result 0.1.2",
+ "windows-result",
  "windows-targets 0.52.6",
 ]
 
-[[package]]
-name = "windows-core"
-version = "0.61.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4763c1de310c86d75a878046489e2e5ba02c649d185f21c67d4cf8a56d098980"
-dependencies = [
- "windows-implement",
- "windows-interface",
- "windows-link",
- "windows-result 0.3.2",
- "windows-strings",
-]
-
-[[package]]
-name = "windows-implement"
-version = "0.60.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a47fddd13af08290e67f4acabf4b459f647552718f683a7b415d290ac744a836"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 2.0.100",
-]
-
-[[package]]
-name = "windows-interface"
-version = "0.59.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bd9211b69f8dcdfa817bfd14bf1c97c9188afa36f4750130fcdf3f400eca9fa8"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 2.0.100",
-]
-
-[[package]]
-name = "windows-link"
-version = "0.1.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "76840935b766e1b0a05c0066835fb9ec80071d4c09a16f6bd5f7e655e3c14c38"
-
 [[package]]
 name = "windows-result"
 version = "0.1.2"
@@ -5246,24 +5000,6 @@ dependencies = [
  "windows-targets 0.52.6",
 ]
 
-[[package]]
-name = "windows-result"
-version = "0.3.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c64fd11a4fd95df68efcfee5f44a294fe71b8bc6a91993e2791938abcc712252"
-dependencies = [
- "windows-link",
-]
-
-[[package]]
-name = "windows-strings"
-version = "0.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7a2ba9642430ee452d5a7aa78d72907ebe8cfda358e8cb7918a2050581322f97"
-dependencies = [
- "windows-link",
-]
-
 [[package]]
 name = "windows-sys"
 version = "0.48.0"
@@ -5414,9 +5150,9 @@ checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
 
 [[package]]
 name = "winnow"
-version = "0.7.6"
+version = "0.7.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "63d3fcd9bba44b03821e7d699eeee959f3126dcc4aa8e4ae18ec617c2a5cea10"
+checksum = "9e27d6ad3dac991091e4d35de9ba2d2d00647c5d0fc26c5496dee55984ae111b"
 dependencies = [
  "memchr",
 ]
@@ -5583,28 +5319,28 @@ checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.100",
- "synstructure 0.13.1",
+ "syn 2.0.101",
+ "synstructure 0.13.2",
 ]
 
 [[package]]
 name = "zerocopy"
-version = "0.8.24"
+version = "0.8.25"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2586fea28e186957ef732a5f8b3be2da217d65c5969d4b1e17f973ebbe876879"
+checksum = "a1702d9583232ddb9174e01bb7c15a2ab8fb1bc6f227aa1233858c351a3ba0cb"
 dependencies = [
  "zerocopy-derive",
 ]
 
 [[package]]
 name = "zerocopy-derive"
-version = "0.8.24"
+version = "0.8.25"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a996a8f63c5c4448cd959ac1bab0aaa3306ccfd060472f85943ee0750f0169be"
+checksum = "28a6e20d751156648aa063f3800b706ee209a32c0b4d9f24be3d980b01be55ef"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.100",
+ "syn 2.0.101",
 ]
 
 [[package]]
@@ -5624,8 +5360,8 @@ checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.100",
- "synstructure 0.13.1",
+ "syn 2.0.101",
+ "synstructure 0.13.2",
 ]
 
 [[package]]
@@ -5645,7 +5381,7 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.100",
+ "syn 2.0.101",
 ]
 
 [[package]]
@@ -5667,7 +5403,7 @@ checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.100",
+ "syn 2.0.101",
 ]
 
 [[package]]
@@ -5700,15 +5436,13 @@ dependencies = [
 
 [[package]]
 name = "zopfli"
-version = "0.8.1"
+version = "0.8.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e5019f391bac5cf252e93bbcc53d039ffd62c7bfb7c150414d61369afe57e946"
+checksum = "edfc5ee405f504cd4984ecc6f14d02d55cfda60fa4b689434ef4102aae150cd7"
 dependencies = [
  "bumpalo",
  "crc32fast",
- "lockfree-object-pool",
  "log",
- "once_cell",
  "simd-adler32",
 ]
 
diff --git a/server/Cargo.toml b/server/Cargo.toml
index 8e1836d..1929744 100644
--- a/server/Cargo.toml
+++ b/server/Cargo.toml
@@ -4,12 +4,9 @@ version = "0.1.0"
 edition = "2021"
 
 [dependencies]
-networking = { git = "https://github.com/aukilabs/posemesh.git", branch = "main" }
+networking = { git = "https://github.com/aukilabs/posemesh.git", branch = "feature/fix-auth" }
 futures = "0.3.30"
-jsonwebtoken = "9.3.0"
-uuid = "1.13.2"
-rand = "0.9.0"
-domain = { git = "https://github.com/aukilabs/posemesh.git", branch = "main" }
+domain = { git = "https://github.com/aukilabs/posemesh.git", branch = "feature/fix-auth" }
 tracing = "0.1.40"
 tracing-subscriber = "0.3.18"
 tokio = "1.44.0"
@@ -18,6 +15,4 @@ serde = "1.0.210"
 serde_json = "1.0.128"
 zip = "2.2.3"
 regex = "1.10.3"
-regexp = "0.3.2"
-chrono = "0.4"
 scopeguard = "1.2.0"
diff --git a/server/src/global_refinement.rs b/server/src/global_refinement.rs
index 0af7bb7..5b86eae 100644
--- a/server/src/global_refinement.rs
+++ b/server/src/global_refinement.rs
@@ -1,5 +1,5 @@
 use std::{collections::HashMap, fs::{self, File}, io::Cursor, path::{Path, PathBuf}, process::Stdio};
-use domain::{datastore::{common::Datastore, remote::RemoteDatastore}, message::read_prefix_size_message, protobuf::{domain_data::{Metadata, Query}, task}};
+use domain::{auth::{handshake, AuthClient}, datastore::{common::{data_id_generator, Datastore}, remote::RemoteDatastore}, message::read_prefix_size_message, protobuf::{domain_data::{Metadata, Query, UpsertMetadata}, task}};
 use networking::{client::Client, AsyncStream};
 use quick_protobuf::serialize_into_vec;
 use regex::Regex;
@@ -9,48 +9,43 @@ use zip::ZipArchive;
 use tokio::io::{AsyncBufReadExt, BufReader as TokioBufReader};
 use tokio::process::Command;
 
-use crate::utils::handshake;
-
 async fn upload_results(domain_id: &str, output_path: PathBuf, mut datastore: RemoteDatastore) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
-    let mut uploader = datastore.upsert(domain_id.to_string()).await;
+    let mut uploader = datastore.upsert(domain_id.to_string()).await?;
     // open output_path and upload to datastore
     let files = fs::read_dir(output_path)?;
     for file in files {
         let file = file?;
         let path = file.path();
-        let metadata: Metadata = match file.file_name().to_str().unwrap() {
-            "refined_manifest.json" => Metadata {
+        let metadata: UpsertMetadata = match file.file_name().to_str().unwrap() {
+            "refined_manifest.json" => UpsertMetadata {
                 name: "refined_manifest".to_string(),
                 data_type: "refined_manifest_json".to_string(),
                 size: file.metadata()?.len() as u32,
-                id: None,
+                id: data_id_generator(),
                 properties: HashMap::new(),
-                link: None,
-                hash: None,
+                is_new: true,
             },
-            "RefinedPointCloud.ply" => Metadata {
+            "RefinedPointCloud.ply" => UpsertMetadata {
                 name: "refined_pointcloud".to_string(),
                 data_type: "refined_pointcloud_ply".to_string(),
                 size: file.metadata()?.len() as u32,
-                id: None,
+                id: data_id_generator(),
                 properties: HashMap::new(),
-                link: None,
-                hash: None,
+                is_new: true
             },
-            "BasicStitchPointCloud.ply" => Metadata {
+            "BasicStitchPointCloud.ply" => UpsertMetadata {
                 name: "unrefined_pointcloud".to_string(),
                 data_type: "unrefined_pointcloud_ply".to_string(),
                 size: file.metadata()?.len() as u32,
-                id: None,
+                id: data_id_generator(),
                 properties: HashMap::new(),
-                link: None,
-                hash: None,
+                is_new: true
             },
             _ => continue
         };
         let content = fs::read(path)?;
         let mut producer = uploader.push(&metadata).await?;
-        producer.push_chunk(&content, false).await?;
+        producer.next_chunk(&content, false).await?;
     }
 
     while !uploader.is_completed().await {
@@ -88,8 +83,8 @@ fn unzip_bytes(path: PathBuf, zip_bytes: Vec<u8>) -> Result<(), Box<dyn std::err
     Ok(())
 }
 
-pub(crate) async fn v1<S: AsyncStream>(base_path: String, mut stream: S, mut datastore: RemoteDatastore, mut c: Client) {
-    let claim = handshake(&mut stream).await.expect("Failed to handshake");
+pub(crate) async fn v1<S: AsyncStream>(public_key: Vec<u8>, base_path: String, mut stream: S, mut datastore: RemoteDatastore, mut c: Client) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    let claim = handshake(&public_key, &mut stream).await.expect("Failed to handshake");
     let job_id = claim.job_id.clone();
     c.subscribe(job_id.clone()).await.expect("Failed to subscribe to job");
     let t = &mut task::Task {
@@ -158,7 +153,7 @@ pub(crate) async fn v1<S: AsyncStream>(base_path: String, mut stream: S, mut dat
         let _ = tx.send(true); // Signal heartbeat task to stop
     });
 
-    let mut downloader = datastore.load(claim.domain_id.clone(), query, false).await;
+    let mut downloader = datastore.load(claim.domain_id.clone(), query, false).await?;
     let mut scan_ids = Vec::new();
 
     loop {
@@ -175,8 +170,8 @@ pub(crate) async fn v1<S: AsyncStream>(base_path: String, mut stream: S, mut dat
                         value: "Failed to parse suffix from data.metadata.name".to_string().into_bytes(),
                     });
                     let message = serialize_into_vec(t).expect("failed to serialize task update");
-                    c.publish(job_id.clone(), message).await.expect("failed to publish task update");
-                    return;
+                    c.publish(job_id.clone(), message).await?;
+                    return Err(Box::new(std::io::Error::new(std::io::ErrorKind::Other, "Failed to parse suffix from data.metadata.name")));
                 }
                 let suffix = res.unwrap();
                 scan_ids.push(suffix.clone());
@@ -190,16 +185,16 @@ pub(crate) async fn v1<S: AsyncStream>(base_path: String, mut stream: S, mut dat
                         value: e.to_string().into_bytes(),
                     });
                     let message = serialize_into_vec(t).expect("failed to serialize task update");
-                    c.publish(job_id.clone(), message).await.expect("failed to publish task update");
-                    return;
+                    c.publish(job_id.clone(), message).await?;
+                    return Err(e);
                 }
                 println!("downloaded {}", data.metadata.name);
             }
-            Some(Err(_)) => {
+            Some(Err(e)) => {
                 t.status = task::Status::RETRY;
                 let message = serialize_into_vec(t).expect("failed to serialize task update");
-                c.publish(job_id.clone(), message).await.expect("failed to publish task update");
-                return;
+                c.publish(job_id.clone(), message).await?;
+                return Err(Box::new(e));
             }
             None => {
                 break;
@@ -227,8 +222,8 @@ pub(crate) async fn v1<S: AsyncStream>(base_path: String, mut stream: S, mut dat
         eprintln!("Failed to execute global refinement: {}", e);
         t.status = task::Status::FAILED;
         let message = serialize_into_vec(t).expect("failed to serialize task update");
-        c.publish(job_id.clone(), message).await.expect("failed to publish task update");
-        return;
+        c.publish(job_id.clone(), message).await?;
+        return Err(Box::new(e));
     }
     let mut child = child.unwrap();
 
@@ -260,15 +255,16 @@ pub(crate) async fn v1<S: AsyncStream>(base_path: String, mut stream: S, mut dat
             if !status.success() {
                 eprintln!("Python process exited with status: {}", status);
                 t.status = task::Status::FAILED;
+                let msg = format!("Python process exited with status: {}", status);
                 t.output = Some(task::Any {
                     type_url: "Error".to_string(),
                     value: serialize_into_vec(&task::Error {
-                        message: format!("Python process exited with status: {}", status),
+                        message: msg.clone(),
                     }).unwrap(),
                 });
                 let message = serialize_into_vec(t).expect("failed to serialize task update");
-                c.publish(job_id.clone(), message).await.expect("failed to publish task update");
-                return;
+                c.publish(job_id.clone(), message).await?;
+                return Err(Box::new(std::io::Error::new(std::io::ErrorKind::Other, msg.clone())));
             }
             println!("Finished executing {}", claim.task_name);
         }
@@ -282,8 +278,8 @@ pub(crate) async fn v1<S: AsyncStream>(base_path: String, mut stream: S, mut dat
                 }).unwrap(),
             });
             let message = serialize_into_vec(t).expect("failed to serialize task update");
-            c.publish(job_id.clone(), message).await.expect("failed to publish task update");
-            return;
+            c.publish(job_id.clone(), message).await?;
+            return Err(Box::new(e));
         }
     }
 
@@ -297,8 +293,8 @@ pub(crate) async fn v1<S: AsyncStream>(base_path: String, mut stream: S, mut dat
             }).unwrap(),
         });
         let message = serialize_into_vec(t).expect("failed to serialize task update");
-        c.publish(job_id.clone(), message).await.expect("failed to publish task update");
-        return;
+        c.publish(job_id.clone(), message).await?;
+        return Err(e);
     }
 
     std::fs::remove_dir_all(&task_path).expect("Failed to remove task folder");
@@ -320,5 +316,5 @@ pub(crate) async fn v1<S: AsyncStream>(base_path: String, mut stream: S, mut dat
     tx.send(true).unwrap();
     let _ = heartbeat_handle.await;
     println!("Heartbeat task stopped");
-    return;
+    Ok(())
 }
diff --git a/server/src/local_refinement.rs b/server/src/local_refinement.rs
index 69fc16b..665a7a8 100644
--- a/server/src/local_refinement.rs
+++ b/server/src/local_refinement.rs
@@ -1,15 +1,15 @@
 use std::{collections::HashMap, fs, path::{Path, PathBuf}, time::Duration};
-use domain::{datastore::{common::Datastore, remote::RemoteDatastore}, message::read_prefix_size_message, protobuf::{domain_data::{Metadata, Query},task::{self, LocalRefinementInputV1, LocalRefinementOutputV1}}};
+use domain::{auth::{handshake, AuthClient}, datastore::{common::{data_id_generator, Datastore}, remote::RemoteDatastore}, message::read_prefix_size_message, protobuf::{domain_data::{Metadata, Query, UpsertMetadata},task::{self, LocalRefinementInputV1, LocalRefinementOutputV1}}};
 use networking::{client::Client, AsyncStream};
 use quick_protobuf::serialize_into_vec;
 use futures::StreamExt;
 use tokio::{sync::watch, time::sleep};
 use std::io::{BufReader, Read, Write};
 use zip::write::SimpleFileOptions;
-use crate::utils::{handshake, health, write_scan_data_summary, execute_python};
+use crate::utils::{execute_python, health, write_scan_data_summary};
 
-pub(crate) async fn v1<S: AsyncStream>(base_path: String, mut stream: S, datastore: RemoteDatastore, mut c: Client) {
-    let claim = handshake(&mut stream).await.expect("Failed to handshake");
+pub(crate) async fn v1<S: AsyncStream>(public_key: Vec<u8>, base_path: String, mut stream: S, datastore: RemoteDatastore, mut c: Client) {
+    let claim = handshake(&public_key,  &mut stream).await.expect("Failed to handshake");
     let job_id = claim.job_id.clone();
     c.subscribe(job_id.clone()).await.expect("Failed to subscribe to job");
     let t = &mut task::Task {
@@ -142,7 +142,7 @@ async fn start<S: AsyncStream>(base_path: String, job_id: &str, task_name: &str,
 async fn run(domain_id: &str, job_id: &str, task_name: &str, context: LocalRefinementContext, mut datastore: RemoteDatastore) -> Result<LocalRefinementOutputV1, Box<dyn std::error::Error + Send + Sync>> { 
     let query = context.query;
 
-    let mut downloader = datastore.load(domain_id.to_string(), query.clone(), false).await;
+    let mut downloader = datastore.load(domain_id.to_string(), query.clone(), false).await?;
     let mut i = 0;
     loop {
         match downloader.next().await {
@@ -239,20 +239,20 @@ async fn run(domain_id: &str, job_id: &str, task_name: &str, context: LocalRefin
     zip.finish()?;
     let zip_file_metadata = fs::metadata(&zip_path)?;
 
-    let mut producer = datastore.upsert(domain_id.to_string()).await;
+    let mut producer = datastore.upsert(domain_id.to_string()).await?;
+    let id = data_id_generator();
     let mut chunks = producer.push(
-        &Metadata {
+        &UpsertMetadata {
             size: zip_file_metadata.len() as u32,
             name: format!("refined_scan_{}", context.suffix.clone()),
             data_type: "refined_scan_zip_v1".to_string(),
-            id: None,
+            id: id.clone(),
+            is_new: true,
             properties: HashMap::new(),
-            link: None,
-            hash: None,
         },
     ).await?;
     
-    let hash = chunks.push_chunk(&fs::read(&zip_path)?, false).await?;
+    let _ = chunks.next_chunk(&fs::read(&zip_path)?, false).await?;
     while !producer.is_completed().await {
         sleep(Duration::from_secs(3)).await;
     }
@@ -261,7 +261,7 @@ async fn run(domain_id: &str, job_id: &str, task_name: &str, context: LocalRefin
     std::fs::remove_dir_all(&context.task_folder)?;
 
     let output = LocalRefinementOutputV1 {
-        result_ids: vec![hash],
+        result_ids: vec![id],
     };
     
     return Ok(output);
diff --git a/server/src/main.rs b/server/src/main.rs
index f12e8ac..545ac52 100644
--- a/server/src/main.rs
+++ b/server/src/main.rs
@@ -1,4 +1,6 @@
-use domain::{cluster::DomainCluster, datastore::remote::RemoteDatastore};
+use std::time::Duration;
+
+use domain::{auth::AuthClient, cluster::DomainCluster, datastore::remote::RemoteDatastore};
 use tokio::{self, select, signal::unix::{signal, SignalKind}};
 use futures::StreamExt;
 mod local_refinement;
@@ -22,19 +24,21 @@ async fn shutdown_signal() {
 #[tokio::main]
 async fn main() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     let args: Vec<String> = std::env::args().collect();
-    if args.len() < 4 {
-        println!("Usage: {} <port> <name> <domain_manager_addr>", args[0]);
+    if args.len() < 5 {
+        println!("Usage: {} <port> <name> <domain_manager_addr> <domain_id>", args[0]);
         return Ok(());
     }
     let port = args[1].parse::<u16>().unwrap();
     let name = args[2].clone();
     let base_path = format!("./volume/{}", name);
     let domain_manager = args[3].clone();
+    let domain_id = args[4].clone();
     let private_key_path = format!("{}/pkey", base_path);
     tracing_subscriber::fmt().with_env_filter(tracing_subscriber::EnvFilter::from_default_env()).init();
 
-    let domain_cluster = DomainCluster::new("".to_string(), domain_manager.clone(), name, false, port, false, false, None, Some(private_key_path), vec![domain_manager.clone()]);
+    let domain_cluster = DomainCluster::new(domain_id.clone(), domain_manager.clone(), name, false, port, false, false, None, Some(private_key_path), vec![domain_manager.clone()]);
     let mut n = domain_cluster.peer.clone();
+    let auth_client = AuthClient::initialize(n.client.clone(), &domain_cluster.manager_id.clone(), Duration::from_secs(60), Some(&format!("/{}/public-key", domain_id.clone()))).await?;
     let mut local_refinement_v1_handler = n.client.set_stream_handler("/local-refinement/v1".to_string()).await.unwrap();
     let mut global_refinement_v1_handler = n.client.set_stream_handler("/global-refinement/v1".to_string()).await.unwrap();
     let remote_storage = RemoteDatastore::new(domain_cluster);
@@ -43,10 +47,12 @@ async fn main() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
         let mut c = n.client.clone();
         select! {
             Some((_, stream)) = local_refinement_v1_handler.next() => {
-                let _ = tokio::spawn(local_refinement::v1(base_path.clone(), stream, remote_storage.clone(), n.client.clone()));
+                let public_key = auth_client.public_key().await;
+                let _ = tokio::spawn(local_refinement::v1(public_key, base_path.clone(), stream, remote_storage.clone(), n.client.clone()));
             }
             Some((_, stream)) = global_refinement_v1_handler.next() => {
-                let _ = tokio::spawn(global_refinement::v1(base_path.clone(), stream, remote_storage.clone(), n.client.clone()));
+                let public_key = auth_client.public_key().await;
+                let _ = tokio::spawn(global_refinement::v1(public_key, base_path.clone(), stream, remote_storage.clone(), n.client.clone()));
             }
             _ = shutdown_signal() => {
                 c.cancel().await.unwrap_or_else(|e| {
diff --git a/server/src/utils.rs b/server/src/utils.rs
index 76a1fa5..0df46df 100644
--- a/server/src/utils.rs
+++ b/server/src/utils.rs
@@ -1,44 +1,11 @@
-use domain::protobuf::task::{DomainClusterHandshake, Status, Task};
-use jsonwebtoken::{decode, DecodingKey,Validation, Algorithm};
-use networking::{client::Client, AsyncStream};
-use quick_protobuf::{deserialize_from_slice, serialize_into_vec};
-use futures::AsyncReadExt;
-use serde::{Serialize, Deserialize};
+use domain::protobuf::task::{Status, Task};
+use networking::client::Client;
+use quick_protobuf::serialize_into_vec;
 use tokio::{sync::watch, task::JoinHandle, time::interval};
 use std::{collections::HashSet, error::Error, fs, path::Path, process::Stdio, time::Duration};
 use serde_json::{json, Value};
 use tokio::{io::{AsyncBufReadExt, BufReader as TokioBufReader}, process::Command};
 
-#[derive(Debug, Serialize, Deserialize)]
-pub struct TaskTokenClaim {
-    pub domain_id: String,
-    pub task_name: String,
-    pub job_id: String,
-    pub sender: String,
-    pub receiver: String,
-    pub exp: usize,
-    pub iat: usize,
-    pub sub: String,
-}
-
-pub fn decode_jwt(token: &str) -> Result<TaskTokenClaim, Box<dyn std::error::Error + Send + Sync>> {
-    let token_data = decode::<TaskTokenClaim>(token, &DecodingKey::from_secret("d5b0966e-8876-4b7c-b6e3-863a0c72c7fd".as_ref()), &Validation::new(Algorithm::HS256))?;
-    Ok(token_data.claims)
-}
-
-pub async fn handshake<S: AsyncStream>(stream: &mut S) -> Result<TaskTokenClaim, Box<dyn std::error::Error + Send + Sync>> {
-    let mut length_buf = [0u8; 4];
-    stream.read_exact(&mut length_buf).await?;
-
-    let length = u32::from_be_bytes(length_buf) as usize;
-    let mut buffer = vec![0u8; length];
-    stream.read_exact(&mut buffer).await?;
-        
-    let header = deserialize_from_slice::<DomainClusterHandshake>(&buffer)?;
-
-    decode_jwt(header.access_token.as_str())
-}
-
 pub fn write_scan_data_summary(
     scan_folder:&Path,
     summary_json_path: &Path,

From e22d176b08a308f4111ef6edd6d9683f536a6785 Mon Sep 17 00:00:00 2001
From: Shuning <shuning@aukilabs.com>
Date: Mon, 5 May 2025 17:26:14 +0800
Subject: [PATCH 81/89] upgrade authentication

---
 server/Cargo.lock  | 8 ++++----
 server/src/main.rs | 4 ++--
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/server/Cargo.lock b/server/Cargo.lock
index 3c5516e..81ae2e1 100644
--- a/server/Cargo.lock
+++ b/server/Cargo.lock
@@ -901,7 +901,7 @@ dependencies = [
 [[package]]
 name = "domain"
 version = "0.1.0"
-source = "git+https://github.com/aukilabs/posemesh.git?branch=feature%2Ffix-auth#192cf534f0eb26f08b592f52dc3fee3f72a59efe"
+source = "git+https://github.com/aukilabs/posemesh.git?branch=feature%2Ffix-auth#960cbcb6259b53469d17091bbcb87f2796cf8f79"
 dependencies = [
  "async-timer",
  "async-trait",
@@ -2774,7 +2774,7 @@ dependencies = [
 [[package]]
 name = "networking"
 version = "0.1.0"
-source = "git+https://github.com/aukilabs/posemesh.git?branch=feature%2Ffix-auth#192cf534f0eb26f08b592f52dc3fee3f72a59efe"
+source = "git+https://github.com/aukilabs/posemesh.git?branch=feature%2Ffix-auth#960cbcb6259b53469d17091bbcb87f2796cf8f79"
 dependencies = [
  "cbindgen",
  "console_error_panic_hook",
@@ -3530,7 +3530,7 @@ dependencies = [
 [[package]]
 name = "runtime"
 version = "0.1.0"
-source = "git+https://github.com/aukilabs/posemesh.git?branch=feature%2Ffix-auth#192cf534f0eb26f08b592f52dc3fee3f72a59efe"
+source = "git+https://github.com/aukilabs/posemesh.git?branch=feature%2Ffix-auth#960cbcb6259b53469d17091bbcb87f2796cf8f79"
 dependencies = [
  "once_cell",
  "tokio",
@@ -4533,7 +4533,7 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
 [[package]]
 name = "utils"
 version = "0.1.0"
-source = "git+https://github.com/aukilabs/posemesh.git?branch=feature%2Ffix-auth#192cf534f0eb26f08b592f52dc3fee3f72a59efe"
+source = "git+https://github.com/aukilabs/posemesh.git?branch=feature%2Ffix-auth#960cbcb6259b53469d17091bbcb87f2796cf8f79"
 dependencies = [
  "futures",
  "gloo-timers 0.3.0",
diff --git a/server/src/main.rs b/server/src/main.rs
index 545ac52..b1daf93 100644
--- a/server/src/main.rs
+++ b/server/src/main.rs
@@ -36,9 +36,9 @@ async fn main() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     let private_key_path = format!("{}/pkey", base_path);
     tracing_subscriber::fmt().with_env_filter(tracing_subscriber::EnvFilter::from_default_env()).init();
 
-    let domain_cluster = DomainCluster::new(domain_id.clone(), domain_manager.clone(), name, false, port, false, false, None, Some(private_key_path), vec![domain_manager.clone()]);
+    let domain_cluster = DomainCluster::new(domain_manager.clone(), name, false, port, false, false, None, Some(private_key_path), vec![domain_manager.clone()]);
     let mut n = domain_cluster.peer.clone();
-    let auth_client = AuthClient::initialize(n.client.clone(), &domain_cluster.manager_id.clone(), Duration::from_secs(60), Some(&format!("/{}/public-key", domain_id.clone()))).await?;
+    let auth_client = AuthClient::initialize(n.client.clone(), &domain_cluster.manager_id.clone(), Duration::from_secs(60), &domain_id).await?;
     let mut local_refinement_v1_handler = n.client.set_stream_handler("/local-refinement/v1".to_string()).await.unwrap();
     let mut global_refinement_v1_handler = n.client.set_stream_handler("/global-refinement/v1".to_string()).await.unwrap();
     let remote_storage = RemoteDatastore::new(domain_cluster);

From 7199c1c76963cf035c0038e7bc7aedea1e253942 Mon Sep 17 00:00:00 2001
From: Shuning <shuning@aukilabs.com>
Date: Thu, 8 May 2025 10:50:44 +0800
Subject: [PATCH 82/89] each domain should have its own key

---
 server/Cargo.lock               |  9 ++---
 server/Cargo.toml               |  5 +--
 server/src/global_refinement.rs |  6 ++--
 server/src/local_refinement.rs  | 18 +++-------
 server/src/main.rs              | 62 ++++++++++++++++++++++++++-------
 5 files changed, 65 insertions(+), 35 deletions(-)

diff --git a/server/Cargo.lock b/server/Cargo.lock
index 81ae2e1..8618fa8 100644
--- a/server/Cargo.lock
+++ b/server/Cargo.lock
@@ -901,7 +901,7 @@ dependencies = [
 [[package]]
 name = "domain"
 version = "0.1.0"
-source = "git+https://github.com/aukilabs/posemesh.git?branch=feature%2Ffix-auth#960cbcb6259b53469d17091bbcb87f2796cf8f79"
+source = "git+https://github.com/aukilabs/posemesh.git?rev=60c1b4d8575aac53d7517487e51b89a3316dc970#60c1b4d8575aac53d7517487e51b89a3316dc970"
 dependencies = [
  "async-timer",
  "async-trait",
@@ -2774,7 +2774,7 @@ dependencies = [
 [[package]]
 name = "networking"
 version = "0.1.0"
-source = "git+https://github.com/aukilabs/posemesh.git?branch=feature%2Ffix-auth#960cbcb6259b53469d17091bbcb87f2796cf8f79"
+source = "git+https://github.com/aukilabs/posemesh.git?rev=60c1b4d8575aac53d7517487e51b89a3316dc970#60c1b4d8575aac53d7517487e51b89a3316dc970"
 dependencies = [
  "cbindgen",
  "console_error_panic_hook",
@@ -3530,7 +3530,7 @@ dependencies = [
 [[package]]
 name = "runtime"
 version = "0.1.0"
-source = "git+https://github.com/aukilabs/posemesh.git?branch=feature%2Ffix-auth#960cbcb6259b53469d17091bbcb87f2796cf8f79"
+source = "git+https://github.com/aukilabs/posemesh.git?rev=60c1b4d8575aac53d7517487e51b89a3316dc970#60c1b4d8575aac53d7517487e51b89a3316dc970"
 dependencies = [
  "once_cell",
  "tokio",
@@ -3789,6 +3789,7 @@ dependencies = [
 name = "server"
 version = "0.1.0"
 dependencies = [
+ "async-trait",
  "domain",
  "futures",
  "networking",
@@ -4533,7 +4534,7 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
 [[package]]
 name = "utils"
 version = "0.1.0"
-source = "git+https://github.com/aukilabs/posemesh.git?branch=feature%2Ffix-auth#960cbcb6259b53469d17091bbcb87f2796cf8f79"
+source = "git+https://github.com/aukilabs/posemesh.git?rev=60c1b4d8575aac53d7517487e51b89a3316dc970#60c1b4d8575aac53d7517487e51b89a3316dc970"
 dependencies = [
  "futures",
  "gloo-timers 0.3.0",
diff --git a/server/Cargo.toml b/server/Cargo.toml
index 1929744..834ac04 100644
--- a/server/Cargo.toml
+++ b/server/Cargo.toml
@@ -4,9 +4,9 @@ version = "0.1.0"
 edition = "2021"
 
 [dependencies]
-networking = { git = "https://github.com/aukilabs/posemesh.git", branch = "feature/fix-auth" }
+networking = { git = "https://github.com/aukilabs/posemesh.git", rev = "60c1b4d8575aac53d7517487e51b89a3316dc970" }
 futures = "0.3.30"
-domain = { git = "https://github.com/aukilabs/posemesh.git", branch = "feature/fix-auth" }
+domain = { git = "https://github.com/aukilabs/posemesh.git", rev = "60c1b4d8575aac53d7517487e51b89a3316dc970" }
 tracing = "0.1.40"
 tracing-subscriber = "0.3.18"
 tokio = "1.44.0"
@@ -16,3 +16,4 @@ serde_json = "1.0.128"
 zip = "2.2.3"
 regex = "1.10.3"
 scopeguard = "1.2.0"
+async-trait = "0.1.88"
diff --git a/server/src/global_refinement.rs b/server/src/global_refinement.rs
index 5b86eae..04a905f 100644
--- a/server/src/global_refinement.rs
+++ b/server/src/global_refinement.rs
@@ -1,5 +1,5 @@
 use std::{collections::HashMap, fs::{self, File}, io::Cursor, path::{Path, PathBuf}, process::Stdio};
-use domain::{auth::{handshake, AuthClient}, datastore::{common::{data_id_generator, Datastore}, remote::RemoteDatastore}, message::read_prefix_size_message, protobuf::{domain_data::{Metadata, Query, UpsertMetadata}, task}};
+use domain::{auth::{handshake, AuthClient}, capabilities::public_key::PublicKeyStorage, datastore::{common::{data_id_generator, Datastore}, remote::RemoteDatastore}, message::read_prefix_size_message, protobuf::{domain_data::{Metadata, Query, UpsertMetadata}, task}};
 use networking::{client::Client, AsyncStream};
 use quick_protobuf::serialize_into_vec;
 use regex::Regex;
@@ -83,8 +83,8 @@ fn unzip_bytes(path: PathBuf, zip_bytes: Vec<u8>) -> Result<(), Box<dyn std::err
     Ok(())
 }
 
-pub(crate) async fn v1<S: AsyncStream>(public_key: Vec<u8>, base_path: String, mut stream: S, mut datastore: RemoteDatastore, mut c: Client) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
-    let claim = handshake(&public_key, &mut stream).await.expect("Failed to handshake");
+pub(crate) async fn v1<S: AsyncStream, P: PublicKeyStorage>(base_path: String, mut stream: S, mut datastore: RemoteDatastore, mut c: Client, key_loader: P) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    let claim = handshake(&mut stream, key_loader).await.expect("Failed to handshake");
     let job_id = claim.job_id.clone();
     c.subscribe(job_id.clone()).await.expect("Failed to subscribe to job");
     let t = &mut task::Task {
diff --git a/server/src/local_refinement.rs b/server/src/local_refinement.rs
index 665a7a8..88dd352 100644
--- a/server/src/local_refinement.rs
+++ b/server/src/local_refinement.rs
@@ -1,15 +1,15 @@
-use std::{collections::HashMap, fs, path::{Path, PathBuf}, time::Duration};
-use domain::{auth::{handshake, AuthClient}, datastore::{common::{data_id_generator, Datastore}, remote::RemoteDatastore}, message::read_prefix_size_message, protobuf::{domain_data::{Metadata, Query, UpsertMetadata},task::{self, LocalRefinementInputV1, LocalRefinementOutputV1}}};
+use std::{collections::HashMap, fs, path::{Path, PathBuf}, sync::Arc, time::Duration};
+use domain::{auth::{handshake, AuthClient}, capabilities::public_key::PublicKeyStorage, datastore::{common::{data_id_generator, Datastore}, remote::RemoteDatastore}, message::read_prefix_size_message, protobuf::{domain_data::{Metadata, Query, UpsertMetadata},task::{self, LocalRefinementInputV1, LocalRefinementOutputV1}}};
 use networking::{client::Client, AsyncStream};
 use quick_protobuf::serialize_into_vec;
-use futures::StreamExt;
+use futures::{lock::Mutex, StreamExt};
 use tokio::{sync::watch, time::sleep};
 use std::io::{BufReader, Read, Write};
 use zip::write::SimpleFileOptions;
 use crate::utils::{execute_python, health, write_scan_data_summary};
 
-pub(crate) async fn v1<S: AsyncStream>(public_key: Vec<u8>, base_path: String, mut stream: S, datastore: RemoteDatastore, mut c: Client) {
-    let claim = handshake(&public_key,  &mut stream).await.expect("Failed to handshake");
+pub(crate) async fn v1<S: AsyncStream, P: PublicKeyStorage>(base_path: String, mut stream: S, datastore: RemoteDatastore, mut c: Client, key_loader: P) {
+    let claim = handshake(&mut stream, key_loader).await.expect("Failed to handshake");
     let job_id = claim.job_id.clone();
     c.subscribe(job_id.clone()).await.expect("Failed to subscribe to job");
     let t = &mut task::Task {
@@ -87,14 +87,6 @@ struct LocalRefinementContext {
 
 async fn start<S: AsyncStream>(base_path: String, job_id: &str, task_name: &str, stream: S) -> Result<LocalRefinementContext, Box<dyn std::error::Error + Send + Sync>> {
     let input = read_prefix_size_message::<LocalRefinementInputV1>(stream).await?;
-    // if let Err(e) = res {
-    //     eprintln!("Failed to load local refinement input {}", e);
-    //     t.status = task::Status::FAILED;
-    //     let message = serialize_into_vec(t).expect("failed to serialize task update");
-    //     c.publish(job_id.clone(), message).await.expect("failed to publish task update");
-    //     return;
-    // }
-    // let input = res.unwrap();
 
     println!("Start executing {}, {:?}", task_name, input);
 
diff --git a/server/src/main.rs b/server/src/main.rs
index b1daf93..94b75d6 100644
--- a/server/src/main.rs
+++ b/server/src/main.rs
@@ -1,8 +1,10 @@
-use std::time::Duration;
+use std::{collections::HashMap, hash::Hash, sync::Arc, time::Duration};
 
-use domain::{auth::AuthClient, cluster::DomainCluster, datastore::remote::RemoteDatastore};
+use async_trait::async_trait;
+use domain::{auth::{AuthClient, AuthError}, capabilities::public_key::PublicKeyStorage, cluster::DomainCluster, datastore::remote::RemoteDatastore};
+use networking::client::Client;
 use tokio::{self, select, signal::unix::{signal, SignalKind}};
-use futures::StreamExt;
+use futures::{lock::Mutex, StreamExt};
 mod local_refinement;
 mod global_refinement;
 mod utils;
@@ -16,6 +18,38 @@ async fn shutdown_signal() {
         _ = int_signal.recv() => println!("Received SIGINT, exiting..."),
     }
 }
+
+#[derive(Clone)]
+struct PublicKeyLoader {
+    auth_clients: Arc<Mutex<HashMap<String, AuthClient>>>,
+    client: Client,
+    cache_ttl: Duration,
+    domain_manager_id: String
+}
+
+#[async_trait]
+impl PublicKeyStorage for PublicKeyLoader {
+    async fn get_by_domain_id(&self, domain_id: String) -> Result<Vec<u8>, AuthError> {
+        let auth_clients = self.auth_clients.lock().await;
+        if let Some(client) = auth_clients.get(&domain_id) {
+            Ok(client.public_key().await)
+        } else {
+            drop(auth_clients);
+            let auth_client = AuthClient::initialize(self.client.clone(), &self.domain_manager_id.clone(), self.cache_ttl, &domain_id).await?;
+            let public_key = auth_client.public_key().await;
+            let mut auth_clients = self.auth_clients.lock().await;
+            auth_clients.insert(domain_id.clone(), auth_client);
+            Ok(public_key)
+        }
+    }
+}
+
+impl PublicKeyLoader {
+    async fn cleanup(&self) {
+        let mut auth_clients = self.auth_clients.lock().await;
+        auth_clients.clear();
+    }
+}
 /*
     * This is a simple example of a reconstruction node. It will connect to a set of bootstraps and execute reconstruction jobs.
     * Usage: cargo run <port> <name> <domain_manager_addr> 
@@ -24,40 +58,44 @@ async fn shutdown_signal() {
 #[tokio::main]
 async fn main() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     let args: Vec<String> = std::env::args().collect();
-    if args.len() < 5 {
-        println!("Usage: {} <port> <name> <domain_manager_addr> <domain_id>", args[0]);
+    if args.len() < 4 {
+        println!("Usage: {} <port> <name> <domain_manager_addr>", args[0]);
         return Ok(());
     }
     let port = args[1].parse::<u16>().unwrap();
     let name = args[2].clone();
     let base_path = format!("./volume/{}", name);
     let domain_manager = args[3].clone();
-    let domain_id = args[4].clone();
     let private_key_path = format!("{}/pkey", base_path);
     tracing_subscriber::fmt().with_env_filter(tracing_subscriber::EnvFilter::from_default_env()).init();
 
     let domain_cluster = DomainCluster::new(domain_manager.clone(), name, false, port, false, false, None, Some(private_key_path), vec![domain_manager.clone()]);
+    let domain_manager_id = domain_cluster.manager_id.clone();
     let mut n = domain_cluster.peer.clone();
-    let auth_client = AuthClient::initialize(n.client.clone(), &domain_cluster.manager_id.clone(), Duration::from_secs(60), &domain_id).await?;
     let mut local_refinement_v1_handler = n.client.set_stream_handler("/local-refinement/v1".to_string()).await.unwrap();
     let mut global_refinement_v1_handler = n.client.set_stream_handler("/global-refinement/v1".to_string()).await.unwrap();
     let remote_storage = RemoteDatastore::new(domain_cluster);
+    let keys_loader = PublicKeyLoader {
+        auth_clients: Arc::new(Mutex::new(HashMap::new())),
+        client: n.client.clone(),
+        cache_ttl: Duration::from_secs(60*60*24),
+        domain_manager_id,
+    };
 
     loop {
         let mut c = n.client.clone();
         select! {
             Some((_, stream)) = local_refinement_v1_handler.next() => {
-                let public_key = auth_client.public_key().await;
-                let _ = tokio::spawn(local_refinement::v1(public_key, base_path.clone(), stream, remote_storage.clone(), n.client.clone()));
+                let _ = tokio::spawn(local_refinement::v1(base_path.clone(), stream, remote_storage.clone(), n.client.clone(), keys_loader.clone()));
             }
             Some((_, stream)) = global_refinement_v1_handler.next() => {
-                let public_key = auth_client.public_key().await;
-                let _ = tokio::spawn(global_refinement::v1(public_key, base_path.clone(), stream, remote_storage.clone(), n.client.clone()));
+                let _ = tokio::spawn(global_refinement::v1(base_path.clone(), stream, remote_storage.clone(), n.client.clone(), keys_loader.clone()));
             }
             _ = shutdown_signal() => {
                 c.cancel().await.unwrap_or_else(|e| {
                     eprintln!("Failed to cancel client: {}", e);
                 });
+                keys_loader.cleanup().await;
                 println!("Received termination signal, shutting down...");
                 break;
             }
@@ -65,7 +103,5 @@ async fn main() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
         }
     }
 
-    println!("Exit");
-
     Ok(())
 }

From 0baeb3aa37ede9d0bc985466d5d5d5ce552cb15f Mon Sep 17 00:00:00 2001
From: Shuning <shuning@aukilabs.com>
Date: Tue, 13 May 2025 11:49:29 +0800
Subject: [PATCH 83/89] skip local refinement if the scan has already been
 refined

---
 server/Cargo.lock               |   9 +-
 server/Cargo.toml               |   4 +-
 server/src/global_refinement.rs | 327 ++++++++++++--------------------
 server/src/local_refinement.rs  | 255 ++++++++++++++-----------
 server/src/main.rs              |  19 +-
 server/src/utils.rs             |  94 ++++++++-
 6 files changed, 374 insertions(+), 334 deletions(-)

diff --git a/server/Cargo.lock b/server/Cargo.lock
index 8618fa8..b9797be 100644
--- a/server/Cargo.lock
+++ b/server/Cargo.lock
@@ -901,7 +901,7 @@ dependencies = [
 [[package]]
 name = "domain"
 version = "0.1.0"
-source = "git+https://github.com/aukilabs/posemesh.git?rev=60c1b4d8575aac53d7517487e51b89a3316dc970#60c1b4d8575aac53d7517487e51b89a3316dc970"
+source = "git+https://github.com/aukilabs/posemesh.git?rev=5512cf097bf262130c95296b77c93d9bc2b384f6#5512cf097bf262130c95296b77c93d9bc2b384f6"
 dependencies = [
  "async-timer",
  "async-trait",
@@ -2774,8 +2774,9 @@ dependencies = [
 [[package]]
 name = "networking"
 version = "0.1.0"
-source = "git+https://github.com/aukilabs/posemesh.git?rev=60c1b4d8575aac53d7517487e51b89a3316dc970#60c1b4d8575aac53d7517487e51b89a3316dc970"
+source = "git+https://github.com/aukilabs/posemesh.git?rev=5512cf097bf262130c95296b77c93d9bc2b384f6#5512cf097bf262130c95296b77c93d9bc2b384f6"
 dependencies = [
+ "async-trait",
  "cbindgen",
  "console_error_panic_hook",
  "futures",
@@ -3530,7 +3531,7 @@ dependencies = [
 [[package]]
 name = "runtime"
 version = "0.1.0"
-source = "git+https://github.com/aukilabs/posemesh.git?rev=60c1b4d8575aac53d7517487e51b89a3316dc970#60c1b4d8575aac53d7517487e51b89a3316dc970"
+source = "git+https://github.com/aukilabs/posemesh.git?rev=5512cf097bf262130c95296b77c93d9bc2b384f6#5512cf097bf262130c95296b77c93d9bc2b384f6"
 dependencies = [
  "once_cell",
  "tokio",
@@ -4534,7 +4535,7 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
 [[package]]
 name = "utils"
 version = "0.1.0"
-source = "git+https://github.com/aukilabs/posemesh.git?rev=60c1b4d8575aac53d7517487e51b89a3316dc970#60c1b4d8575aac53d7517487e51b89a3316dc970"
+source = "git+https://github.com/aukilabs/posemesh.git?rev=5512cf097bf262130c95296b77c93d9bc2b384f6#5512cf097bf262130c95296b77c93d9bc2b384f6"
 dependencies = [
  "futures",
  "gloo-timers 0.3.0",
diff --git a/server/Cargo.toml b/server/Cargo.toml
index 834ac04..7cce020 100644
--- a/server/Cargo.toml
+++ b/server/Cargo.toml
@@ -4,9 +4,9 @@ version = "0.1.0"
 edition = "2021"
 
 [dependencies]
-networking = { git = "https://github.com/aukilabs/posemesh.git", rev = "60c1b4d8575aac53d7517487e51b89a3316dc970" }
+networking = { git = "https://github.com/aukilabs/posemesh.git", rev = "5512cf097bf262130c95296b77c93d9bc2b384f6" }
 futures = "0.3.30"
-domain = { git = "https://github.com/aukilabs/posemesh.git", rev = "60c1b4d8575aac53d7517487e51b89a3316dc970" }
+domain = { git = "https://github.com/aukilabs/posemesh.git", rev = "5512cf097bf262130c95296b77c93d9bc2b384f6" }
 tracing = "0.1.40"
 tracing-subscriber = "0.3.18"
 tokio = "1.44.0"
diff --git a/server/src/global_refinement.rs b/server/src/global_refinement.rs
index 04a905f..eab8ac8 100644
--- a/server/src/global_refinement.rs
+++ b/server/src/global_refinement.rs
@@ -1,15 +1,15 @@
-use std::{collections::HashMap, fs::{self, File}, io::Cursor, path::{Path, PathBuf}, process::Stdio};
-use domain::{auth::{handshake, AuthClient}, capabilities::public_key::PublicKeyStorage, datastore::{common::{data_id_generator, Datastore}, remote::RemoteDatastore}, message::read_prefix_size_message, protobuf::{domain_data::{Metadata, Query, UpsertMetadata}, task}};
-use networking::{client::Client, AsyncStream};
+use std::{collections::HashMap, fs::{self, File}, io::Cursor, path::{Path, PathBuf}};
+use domain::{auth::{handshake, TaskTokenClaim}, capabilities::public_key::PublicKeyStorage, datastore::{common::{data_id_generator, Datastore}, remote::RemoteDatastore}, message::read_prefix_size_message, protobuf::{domain_data::{Metadata, Query, UpsertMetadata}, task::{self, Task}}};
+use networking::{client::{Client, TClient}, AsyncStream};
 use quick_protobuf::serialize_into_vec;
 use regex::Regex;
-use tokio::{self, sync::watch, time::{interval, sleep, Duration}};
+use tokio::{self, sync::watch, time::{sleep, Duration}};
 use futures::StreamExt;
 use zip::ZipArchive;
-use tokio::io::{AsyncBufReadExt, BufReader as TokioBufReader};
-use tokio::process::Command;
 
-async fn upload_results(domain_id: &str, output_path: PathBuf, mut datastore: RemoteDatastore) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+use crate::utils::{execute_python, health};
+
+async fn upload_results(domain_id: &str, output_path: PathBuf, datastore: &mut RemoteDatastore) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     let mut uploader = datastore.upsert(domain_id.to_string()).await?;
     // open output_path and upload to datastore
     let files = fs::read_dir(output_path)?;
@@ -56,17 +56,14 @@ async fn upload_results(domain_id: &str, output_path: PathBuf, mut datastore: Re
 }
 
 fn unzip_bytes(path: PathBuf, zip_bytes: Vec<u8>) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
-    println!("Starting to unzip bytes to path: {:?}", path);
     let cursor = Cursor::new(zip_bytes);
     let mut archive = ZipArchive::new(cursor)?;
-    println!("Zip archive opened, contains {} files", archive.len());
+    tracing::info!("Zip archive opened, contains {} files", archive.len());
     
     for i in 0..archive.len() {
-        println!("Processing file {}/{}", i + 1, archive.len());
         let mut input_file = archive.by_index(i)?;
         let file_name = input_file.enclosed_name().unwrap();
         let file_name = file_name.file_name().unwrap().to_str().unwrap();
-        println!("Extracting file: {}", file_name);
         
         let file_path = path.join(file_name);
         
@@ -77,17 +74,20 @@ fn unzip_bytes(path: PathBuf, zip_bytes: Vec<u8>) -> Result<(), Box<dyn std::err
         
         let mut output_file: File = File::create(&file_path)?;
         let bytes_copied = std::io::copy(&mut input_file, &mut output_file)?;
-        println!("Extracted {} bytes to {:?}", bytes_copied, file_path);
     }
-    println!("Finished unzipping all files");
     Ok(())
 }
 
-pub(crate) async fn v1<S: AsyncStream, P: PublicKeyStorage>(base_path: String, mut stream: S, mut datastore: RemoteDatastore, mut c: Client, key_loader: P) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
-    let claim = handshake(&mut stream, key_loader).await.expect("Failed to handshake");
+async fn initialize_task<S: AsyncStream, P: PublicKeyStorage, C: TClient + Clone + Send + Sync + 'static>(
+    base_path: &str,
+    stream: &mut S,
+    c: &mut C,
+    key_loader: P,
+) -> Result<(task::Task, String, TaskTokenClaim, PathBuf, PathBuf, PathBuf), Box<dyn std::error::Error + Send + Sync>> {
+    let claim = handshake(stream, key_loader).await.expect("Failed to handshake");
     let job_id = claim.job_id.clone();
     c.subscribe(job_id.clone()).await.expect("Failed to subscribe to job");
-    let t = &mut task::Task {
+    let task = task::Task {
         name: claim.task_name.clone(),
         receiver: Some(claim.receiver.clone()),
         sender: claim.sender.clone(),
@@ -97,224 +97,149 @@ pub(crate) async fn v1<S: AsyncStream, P: PublicKeyStorage>(base_path: String, m
         job_id: job_id.clone(),
         output: None,
     };
+
+    let task_path = Path::new(base_path).join(&job_id);
+    let dataset_path = task_path.join("datasets");
+    let input_path = task_path.join("refined").join("local");
+    let output_path = task_path.join("refined").join("global");
+
     /*
         | volumn/node_name
         | | job_id
         | | | datasets/suffix
         | | | refined/local/suffix/sfm => input for global refinement
         | | | refined/global => output for global refinement
-    
     */
 
-    let input = read_prefix_size_message::<task::GlobalRefinementInputV1>(stream).await.expect("Failed to read global refinement input");
-    println!("Received global refinement input: {:?}", input);
-    // merge input.local_refinement_output into query
-    let mut query = Query::default();
-    for result in input.local_refinement_results {
-        query.ids.extend(result.result_ids);
-    }
-    
-    c.publish(job_id.clone(), serialize_into_vec(t).expect("failed to serialize task update")).await.expect("failed to publish task update");
+    fs::create_dir_all(&input_path)?;
+    fs::create_dir_all(&dataset_path)?;
+    fs::create_dir_all(&output_path)?;
 
-    // download the local refinement output
-    let task_path = Path::new(&base_path).join(job_id.clone());
-    let dataset_path = Path::new(&task_path).join("datasets");
-    let input_path = Path::new(&task_path).join("refined").join("local");
-    let output_path = Path::new(&task_path).join("refined").join("global");
-    fs::create_dir_all(input_path.clone()).expect("Failed to create input folder");
-    fs::create_dir_all(dataset_path.clone()).expect("Failed to create dataset folder");
-    fs::create_dir_all(output_path.clone()).expect("Failed to create output folder");
-
-    let (tx, rx) = watch::channel(false);
-    let mut c_clone = c.clone();
-    let job_id_clone = job_id.clone();
-    let task_clone = t.clone();
-    let heartbeat_handle = tokio::spawn(async move {
-        let mut rx = rx;
-        let mut interval = interval(Duration::from_secs(30)); // Send heartbeat every 30 seconds
-        
-        loop {
-            tokio::select! {
-                _ = interval.tick() => {
-                    let mut progress_task = task_clone.clone();
-                    progress_task.status = task::Status::PROCESSING;
-                    if let Ok(message) = serialize_into_vec(&progress_task) {
-                        let _ = c_clone.publish(job_id_clone.clone(), message).await;
-                    }
-                }
-                Ok(_) = rx.changed() => {
-                    break;
-                }
-            }
-        }
-    });
-
-    let _cleanup = scopeguard::guard(tx.clone(), |tx| {
-        let _ = tx.send(true); // Signal heartbeat task to stop
-    });
+    Ok((task, job_id, claim, task_path, input_path, output_path))
+}
 
-    let mut downloader = datastore.load(claim.domain_id.clone(), query, false).await?;
+async fn download_data(
+    datastore: &mut RemoteDatastore,
+    domain_id: &str,
+    query: Query,
+    input_path: &Path,
+    dataset_path: &Path,
+) -> Result<Vec<String>, Box<dyn std::error::Error + Send + Sync>> {
+    let mut downloader = datastore.load(domain_id.to_string(), query, false).await?;
     let mut scan_ids = Vec::new();
 
-    loop {
-        match downloader.next().await {
-            Some(Ok(data)) => {
-                // parse suffix from data.metadata.name
-                let date_time_regex = Regex::new(r"\d{4}-\d{2}-\d{2}[_-]\d{2}-\d{2}-\d{2}").unwrap();
-                let res = date_time_regex.find(&data.metadata.name)
-                    .map(|m| m.as_str().to_string());
-                if res.is_none() {
-                    t.status = task::Status::FAILED;
-                    t.output = Some(task::Any {
-                        type_url: "Error".to_string(),
-                        value: "Failed to parse suffix from data.metadata.name".to_string().into_bytes(),
-                    });
-                    let message = serialize_into_vec(t).expect("failed to serialize task update");
-                    c.publish(job_id.clone(), message).await?;
-                    return Err(Box::new(std::io::Error::new(std::io::ErrorKind::Other, "Failed to parse suffix from data.metadata.name")));
-                }
-                let suffix = res.unwrap();
+    while let Some(data) = downloader.next().await {
+        match data {
+            Ok(data) => {
+                let suffix = extract_suffix(&data.metadata.name)?;
                 scan_ids.push(suffix.clone());
-                
-                fs::create_dir_all(Path::new(&dataset_path).join(&suffix)).expect("Failed to create dataset folder");
-                let path = Path::new(&input_path).join(&suffix).join("sfm");
-                if let Err(e) = unzip_bytes(path, data.content) {
-                    t.status = task::Status::FAILED;
-                    t.output = Some(task::Any {
-                        type_url: "Error".to_string(),
-                        value: e.to_string().into_bytes(),
-                    });
-                    let message = serialize_into_vec(t).expect("failed to serialize task update");
-                    c.publish(job_id.clone(), message).await?;
-                    return Err(e);
-                }
-                println!("downloaded {}", data.metadata.name);
+
+                fs::create_dir_all(dataset_path.join(&suffix))?;
+                let path = input_path.join(&suffix).join("sfm");
+                unzip_bytes(path, data.content)?;
+                tracing::info!("downloaded {}", data.metadata.name);
             }
-            Some(Err(e)) => {
-                t.status = task::Status::RETRY;
-                let message = serialize_into_vec(t).expect("failed to serialize task update");
-                c.publish(job_id.clone(), message).await?;
+            Err(e) => {
                 return Err(Box::new(e));
             }
-            None => {
-                break;
-            }
         }
     }
-    let mut params = vec![
-        "-u",
-        "main.py",
-        "--mode", "global_refinement",
-        "--job_root_path", task_path.to_str().unwrap(),
-        "--output", output_path.to_str().unwrap(),
-        "--domain_id", &claim.domain_id,
-        "--job_id", &claim.job_id,
-        "--scans"
-    ];
-    params.extend(scan_ids.iter().map(|s| s.as_str()));
-    let child = Command::new("python3")
-    .args(params)
-    .stdout(Stdio::piped())
-    .stderr(Stdio::piped())
-    .spawn();
 
-    if let Err(e) = child {
-        eprintln!("Failed to execute global refinement: {}", e);
-        t.status = task::Status::FAILED;
-        let message = serialize_into_vec(t).expect("failed to serialize task update");
-        c.publish(job_id.clone(), message).await?;
-        return Err(Box::new(e));
+    if scan_ids.is_empty() {
+        return Err("No scans to refine".into());
     }
-    let mut child = child.unwrap();
 
-    // Read stdout in real-time
-    if let Some(stdout) = child.stdout.take() {
-        let stdout_reader = TokioBufReader::new(stdout);
-        tokio::spawn(async move {
-            let mut lines = stdout_reader.lines();
-            while let Ok(Some(line)) = lines.next_line().await {
-                println!("stdout: {}", line);
-            }
-        });
-    }
+    Ok(scan_ids)
+}
 
-    // Read stderr in real-time
-    if let Some(stderr) = child.stderr.take() {
-        let stderr_reader = TokioBufReader::new(stderr);
-        tokio::spawn(async move {
-            let mut lines = stderr_reader.lines();
-            while let Ok(Some(line)) = lines.next_line().await {
-                eprintln!("stderr: {}", line);
-            }
-        });
-    }
+fn extract_suffix(name: &str) -> Result<String, Box<dyn std::error::Error + Send + Sync>> {
+    let date_time_regex = Regex::new(r"\d{4}-\d{2}-\d{2}[_-]\d{2}-\d{2}-\d{2}")?;
+    date_time_regex
+        .find(name)
+        .map(|m| m.as_str().to_string())
+        .ok_or_else(|| "Failed to parse suffix from data.metadata.name".into())
+}
 
-    // Wait for the process to complete (non-blocking)
-    match child.wait().await {
-        Ok(status) => {
-            if !status.success() {
-                eprintln!("Python process exited with status: {}", status);
-                t.status = task::Status::FAILED;
-                let msg = format!("Python process exited with status: {}", status);
-                t.output = Some(task::Any {
-                    type_url: "Error".to_string(),
-                    value: serialize_into_vec(&task::Error {
-                        message: msg.clone(),
-                    }).unwrap(),
-                });
-                let message = serialize_into_vec(t).expect("failed to serialize task update");
-                c.publish(job_id.clone(), message).await?;
-                return Err(Box::new(std::io::Error::new(std::io::ErrorKind::Other, msg.clone())));
-            }
-            println!("Finished executing {}", claim.task_name);
-        }
-        Err(e) => {
-            eprintln!("Failed to wait for Python process: {}", e);
-            t.status = task::Status::FAILED;
-            t.output = Some(task::Any {
-                type_url: "Error".to_string(),
-                value: serialize_into_vec(&task::Error {
-                    message: format!("Failed to wait for Python process: {}", e),
-                }).unwrap(),
-            });
-            let message = serialize_into_vec(t).expect("failed to serialize task update");
-            c.publish(job_id.clone(), message).await?;
-            return Err(Box::new(e));
-        }
+pub(crate) async fn v1<S: AsyncStream, P: PublicKeyStorage>(
+    base_path: String,
+    mut stream: S,
+    datastore: RemoteDatastore,
+    mut c: Client,
+    key_loader: P,
+) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    let (mut t, job_id, claim, task_path, input_path, output_path) =
+        initialize_task(&base_path, &mut stream, &mut c, key_loader).await?;
+
+    let input = read_prefix_size_message::<task::GlobalRefinementInputV1>(&mut stream).await?;
+    tracing::info!("Received global refinement input: {:?}", input);
+
+    let mut query = Query::default();
+    for result in input.local_refinement_results {
+        query.ids.extend(result.result_ids);
     }
 
-    if let Err(e) = upload_results(&claim.domain_id, output_path, datastore).await {
-        eprintln!("Failed to upload results: {}", e);
+    c.publish(job_id.clone(), serialize_into_vec(&t).expect("failed to serialize task update")).await?;
+
+    let (tx, rx) = watch::channel(false);
+
+    let heartbeat_handle = health(t.clone(), c.clone(), &job_id, rx).await;
+
+    let _cleanup = scopeguard::guard(tx.clone(), |tx| {
+        let _ = tx.send(true); // Signal heartbeat task to stop
+    });
+
+    let domain_id = claim.domain_id.clone();
+    let res = run(&domain_id, &input_path, &task_path, &output_path, datastore, query, &job_id).await;
+
+    if let Err(e) = res {
         t.status = task::Status::FAILED;
         t.output = Some(task::Any {
             type_url: "Error".to_string(),
             value: serialize_into_vec(&task::Error {
-                message: format!("Failed to upload results: {}", e),
-            }).unwrap(),
+                message: e.to_string(),
+            }).expect("failed to serialize local refinement output"),
         });
-        let message = serialize_into_vec(t).expect("failed to serialize task update");
-        c.publish(job_id.clone(), message).await?;
-        return Err(e);
+        tracing::error!("Error: {}", e);
+    } else {
+        t.status = task::Status::DONE;
     }
 
-    std::fs::remove_dir_all(&task_path).expect("Failed to remove task folder");
-
-    let event = task::Task {
-        name: claim.task_name.clone(),
-        receiver: Some(claim.receiver.clone()),
-        sender: claim.sender.clone(),
-        endpoint: "/global-refinement/v1".to_string(),
-        status: task::Status::DONE,
-        access_token: None,
-        job_id: job_id.clone(),
-        output: None,
-    };
-    let buf = serialize_into_vec(&event).expect("failed to serialize task update");
-    c.publish(job_id.clone(), buf).await.expect("failed to publish task update");
-    println!("Finished executing {}", claim.task_name);
-
     tx.send(true).unwrap();
     let _ = heartbeat_handle.await;
-    println!("Heartbeat task stopped");
+
+    let buf = serialize_into_vec(&t)?;
+    c.publish(job_id.to_string(), buf).await?;
+
+    tracing::info!("Finished executing {}", claim.task_name);
+
+    Ok(())
+}
+
+async fn run(
+    domain_id: &str,
+    input_path: &Path,
+    task_path: &Path,
+    output_path: &Path,
+    mut datastore: RemoteDatastore,
+    query: Query,
+    job_id: &str,
+) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    let scan_ids = download_data(&mut datastore, &domain_id, query, &input_path, &task_path.join("datasets")).await?;
+        
+    let mut params = vec![
+        "-u",
+        "main.py",
+        "--mode", "global_refinement",
+        "--job_root_path", task_path.to_str().unwrap(),
+        "--output", output_path.to_str().unwrap(),
+        "--domain_id", &domain_id,
+        "--job_id", &job_id,
+        "--scans",
+    ];
+    params.extend(scan_ids.iter().map(|s| s.as_str()));
+
+    execute_python(params).await?;
+    upload_results(&domain_id, output_path.to_path_buf(), &mut datastore).await?;
+    std::fs::remove_dir_all(output_path)?;
     Ok(())
 }
diff --git a/server/src/local_refinement.rs b/server/src/local_refinement.rs
index 88dd352..5a2d242 100644
--- a/server/src/local_refinement.rs
+++ b/server/src/local_refinement.rs
@@ -1,14 +1,14 @@
-use std::{collections::HashMap, fs, path::{Path, PathBuf}, sync::Arc, time::Duration};
-use domain::{auth::{handshake, AuthClient}, capabilities::public_key::PublicKeyStorage, datastore::{common::{data_id_generator, Datastore}, remote::RemoteDatastore}, message::read_prefix_size_message, protobuf::{domain_data::{Metadata, Query, UpsertMetadata},task::{self, LocalRefinementInputV1, LocalRefinementOutputV1}}};
-use networking::{client::Client, AsyncStream};
+use std::{collections::HashMap, fs, path::{Path, PathBuf}, time::Duration};
+use domain::{auth::handshake, capabilities::public_key::PublicKeyStorage, datastore::{common::{data_id_generator, Datastore}, remote::RemoteDatastore}, message::read_prefix_size_message, protobuf::{domain_data::{Query, UpsertMetadata},task::{self, LocalRefinementInputV1, LocalRefinementOutputV1}}};
+use networking::{client::TClient, AsyncStream};
 use quick_protobuf::serialize_into_vec;
-use futures::{lock::Mutex, StreamExt};
+use futures::StreamExt;
 use tokio::{sync::watch, time::sleep};
 use std::io::{BufReader, Read, Write};
 use zip::write::SimpleFileOptions;
 use crate::utils::{execute_python, health, write_scan_data_summary};
 
-pub(crate) async fn v1<S: AsyncStream, P: PublicKeyStorage>(base_path: String, mut stream: S, datastore: RemoteDatastore, mut c: Client, key_loader: P) {
+pub(crate) async fn v1<S: AsyncStream, P: PublicKeyStorage, C: TClient + Clone + Send + Sync + 'static>(base_path: String, mut stream: S, datastore: RemoteDatastore, mut c: C, key_loader: P) {
     let claim = handshake(&mut stream, key_loader).await.expect("Failed to handshake");
     let job_id = claim.job_id.clone();
     c.subscribe(job_id.clone()).await.expect("Failed to subscribe to job");
@@ -31,7 +31,7 @@ pub(crate) async fn v1<S: AsyncStream, P: PublicKeyStorage>(base_path: String, m
             c.publish(job_id.clone(), message).await.expect("failed to publish task update");
             let (tx, rx) = watch::channel(false);
 
-            let heartbeat_handle = health(t.clone(), c.clone(), &job_id, rx);
+            let heartbeat_handle = health(t.clone(), c.clone(), &job_id, rx).await;
 
             let _cleanup = scopeguard::guard(tx.clone(), |tx| {
                 let _ = tx.send(true); // Signal heartbeat task to stop
@@ -47,7 +47,7 @@ pub(crate) async fn v1<S: AsyncStream, P: PublicKeyStorage>(base_path: String, m
                     t.status = task::Status::DONE;
                 }
                 Err(e) => {
-                    eprintln!("Failed to run local refinement: {}", e);
+                    tracing::error!("Failed to run local refinement: {}", e);
                     t.status = task::Status::FAILED;
                     t.output = Some(task::Any {
                         type_url: "Error".to_string(),
@@ -62,7 +62,7 @@ pub(crate) async fn v1<S: AsyncStream, P: PublicKeyStorage>(base_path: String, m
             let _ = heartbeat_handle.await;
         }
         Err(e) => {
-            eprintln!("Failed to start local refinement: {}", e);
+            tracing::error!("Failed to start local refinement: {}", e);
             t.status = task::Status::FAILED;
             t.output = Some(task::Any {
                 type_url: "Error".to_string(),
@@ -74,6 +74,8 @@ pub(crate) async fn v1<S: AsyncStream, P: PublicKeyStorage>(base_path: String, m
     }
     let message = serialize_into_vec(t).expect("failed to serialize task update");
     c.publish(job_id.clone(), message).await.expect("failed to publish task update");
+
+    tracing::info!("finished local refinement {}", job_id);
 }
 
 struct LocalRefinementContext {
@@ -88,7 +90,7 @@ struct LocalRefinementContext {
 async fn start<S: AsyncStream>(base_path: String, job_id: &str, task_name: &str, stream: S) -> Result<LocalRefinementContext, Box<dyn std::error::Error + Send + Sync>> {
     let input = read_prefix_size_message::<LocalRefinementInputV1>(stream).await?;
 
-    println!("Start executing {}, {:?}", task_name, input);
+    tracing::info!("Start executing {}, {:?}", task_name, input);
 
     // input.name_regexp looks .*_date
     // get date from regexp
@@ -133,127 +135,152 @@ async fn start<S: AsyncStream>(base_path: String, job_id: &str, task_name: &str,
 
 async fn run(domain_id: &str, job_id: &str, task_name: &str, context: LocalRefinementContext, mut datastore: RemoteDatastore) -> Result<LocalRefinementOutputV1, Box<dyn std::error::Error + Send + Sync>> { 
     let query = context.query;
+    let mut res_id: String = "".to_string();
+
+    let mut check_refined = datastore.load(domain_id.to_string(), Query {
+        ids: vec![],
+        name_regexp: None,
+        data_type_regexp: None,
+        names: vec![format!("refined_scan_{}", context.suffix.clone())],
+        data_types: vec![],
+        metadata_only: true,
+    }, false).await.unwrap_or_else(|e| {
+        panic!("Failed to load refined scan {}: {}", context.suffix.clone(), e);
+    });
+    while let Some(Ok(data)) = check_refined.next().await {
+        tracing::info!("found refined scan: {}", data.metadata.name);
+        res_id = data.metadata.id.clone();
+    }
+    check_refined.close();
 
-    let mut downloader = datastore.load(domain_id.to_string(), query.clone(), false).await?;
-    let mut i = 0;
-    loop {
-        match downloader.next().await {
-            Some(Ok(data)) => {
-                let filename = match data.metadata.name.as_str() {
-                    "Manifest.json" => "Manifest.json".to_string(),
-                    "FeaturePoints.ply" => "FeaturePoints.ply".to_string(),
-                    "ARposes.csv" => "ARposes.csv".to_string(),
-                    "PortalDetections.csv" => "PortalDetections.csv".to_string(),
-                    "CameraIntrinsics.csv" => "CameraIntrinsics.csv".to_string(),
-                    "Frames.csv" => "Frames.csv".to_string(),
-                    "Gyro.csv" => "Gyro.csv".to_string(),
-                    "Accel.csv" => "Accel.csv".to_string(),
-                    "gyro_accel.csv" => "gyro_accel.csv".to_string(),
-                    "Frames.mp4" => "Frames.mp4".to_string(),
-                    _ => {
-                        match data.metadata.data_type.as_str() {
-                            "dmt_manifest_json" => "Manifest.json".to_string(),
-                            "dmt_featurepoints_ply" | "dmt_pointcloud_ply" => "FeaturePoints.ply".to_string(),
-                            "dmt_arposes_csv" => "ARposes.csv".to_string(),
-                            "dmt_portal_detections_csv" | "dmt_observations_csv" => "PortalDetections.csv".to_string(),
-                            "dmt_intrinsics_csv" | "dmt_cameraintrinsics_csv" => "CameraIntrinsics.csv".to_string(),
-                            "dmt_frames_csv" => "Frames.csv".to_string(),
-                            "dmt_gyro_csv" => "Gyro.csv".to_string(),
-                            "dmt_accel_csv" => "Accel.csv".to_string(),
-                            "dmt_gyroaccel_csv" => "gyro_accel.csv".to_string(),
-                            "dmt_recording_mp4" => "Frames.mp4".to_string(),
-                            _ => {
-                                println!("unknown domain data type: {}", data.metadata.data_type);
-                                format!("{}.{}", data.metadata.name, data.metadata.data_type)
+    if res_id.is_empty() {
+        let mut downloader = datastore.load(domain_id.to_string(), Query {
+            metadata_only: false,
+            ..query
+        }, false).await?;
+        let mut i = 0;
+        loop {
+            match downloader.next().await {
+                Some(Ok(data)) => {
+                    let filename = match data.metadata.name.as_str() {
+                        "Manifest.json" => "Manifest.json".to_string(),
+                        "FeaturePoints.ply" => "FeaturePoints.ply".to_string(),
+                        "ARposes.csv" => "ARposes.csv".to_string(),
+                        "PortalDetections.csv" => "PortalDetections.csv".to_string(),
+                        "CameraIntrinsics.csv" => "CameraIntrinsics.csv".to_string(),
+                        "Frames.csv" => "Frames.csv".to_string(),
+                        "Gyro.csv" => "Gyro.csv".to_string(),
+                        "Accel.csv" => "Accel.csv".to_string(),
+                        "gyro_accel.csv" => "gyro_accel.csv".to_string(),
+                        "Frames.mp4" => "Frames.mp4".to_string(),
+                        _ => {
+                            match data.metadata.data_type.as_str() {
+                                "dmt_manifest_json" => "Manifest.json".to_string(),
+                                "dmt_featurepoints_ply" | "dmt_pointcloud_ply" => "FeaturePoints.ply".to_string(),
+                                "dmt_arposes_csv" => "ARposes.csv".to_string(),
+                                "dmt_portal_detections_csv" | "dmt_observations_csv" => "PortalDetections.csv".to_string(),
+                                "dmt_intrinsics_csv" | "dmt_cameraintrinsics_csv" => "CameraIntrinsics.csv".to_string(),
+                                "dmt_frames_csv" => "Frames.csv".to_string(),
+                                "dmt_gyro_csv" => "Gyro.csv".to_string(),
+                                "dmt_accel_csv" => "Accel.csv".to_string(),
+                                "dmt_gyroaccel_csv" => "gyro_accel.csv".to_string(),
+                                "dmt_recording_mp4" => "Frames.mp4".to_string(),
+                                _ => {
+                                    tracing::info!("unknown domain data type: {}", data.metadata.data_type);
+                                    format!("{}.{}", data.metadata.name, data.metadata.data_type)
+                                }
                             }
                         }
-                    }
-                };
-                let path = context.input_folder.join(&filename);
-                fs::write(path, &data.content)?;
-                i+=1;
-                println!("downloaded {}", filename);
-            }
-            Some(Err(e)) => {
-                return Err(e.into());
-            }
-            None => {
-                break;
+                    };
+
+                    let path = context.input_folder.join(&filename);
+                    fs::write(path, &data.content)?;
+                    i += 1;
+                    tracing::info!("downloaded {}", filename);
+                }
+                Some(Err(e)) => {
+                    return Err(e.into());
+                }
+                None => {
+                    tracing::info!("no more data to download");
+                    break;
+                }
             }
         }
-    }
-    println!("Finished downloading {} data for {}", i, task_name);
 
-    write_scan_data_summary(context.input_folder.as_path(), context.task_folder.as_path().join("scan_data_summary.json").as_path())?;
+        tracing::info!("Finished downloading {} data for {}", i, task_name);
 
-    let params = vec![
-        "-u",
-        "main.py",
-        "--mode", "local_refinement",
-        "--job_root_path", context.task_folder.to_str().unwrap(),
-        "--output", context.output_folder.to_str().unwrap(),
-        "--domain_id", domain_id,
-        "--job_id", job_id,
-        "--scans", context.suffix.as_str(),
-    ];
+        write_scan_data_summary(context.input_folder.as_path(), context.task_folder.as_path().join("scan_data_summary.json").as_path())?;
 
-    execute_python(params).await?;
+        let params = vec![
+            "-u",
+            "main.py",
+            "--mode", "local_refinement",
+            "--job_root_path", context.task_folder.to_str().unwrap(),
+            "--output", context.output_folder.to_str().unwrap(),
+            "--domain_id", domain_id,
+            "--job_id", job_id,
+            "--scans", context.suffix.as_str(),
+        ];
 
-    let sfm = context.output_folder.join(context.suffix.clone()).join("sfm");
-    let zip_path = sfm.join(context.suffix.clone() + ".zip");
-    let mut zip = zip::ZipWriter::new(fs::File::create(&zip_path)?);
-    let options = SimpleFileOptions::default().compression_method(zip::CompressionMethod::Stored); // No compression
-    
-    // open output folder/sfm, zip all txt, bin, csv files and upload
-    for entry in fs::read_dir(sfm)? {
-        let entry = entry?;
-        let path = entry.path();
-        if path.is_dir() {
-            continue;
-        }
-        let ext = path.extension().expect("Failed to get extension").to_str().expect("Failed to convert extension to string");
-        if ext == "txt" || ext == "bin" || ext == "csv" {
-            let str_path = path.to_str().expect("Failed to convert path to string");
-            let file = fs::File::open(str_path)?;
-            let mut reader = BufReader::new(file);
-            zip.start_file(str_path, options)?;
+        execute_python(params).await?;
 
-            let mut buffer = [0u8; 8192]; // Use a buffer to stream in chunks
-            loop {
-                let bytes_read = reader.read(&mut buffer)?;
-                if bytes_read == 0 {
-                    break;
+        let sfm = context.output_folder.join(context.suffix.clone()).join("sfm");
+        fs::create_dir_all(&sfm)?;
+        let zip_path = sfm.join(context.suffix.clone() + ".zip");
+        let mut zip = zip::ZipWriter::new(fs::File::create(&zip_path)?);
+        let options = SimpleFileOptions::default().compression_method(zip::CompressionMethod::Stored); // No compression
+        
+        // open output folder/sfm, zip all txt, bin, csv files and upload
+        for entry in fs::read_dir(sfm)? {
+            let entry = entry?;
+            let path = entry.path();
+            if path.is_dir() {
+                continue;
+            }
+            let ext = path.extension().expect("Failed to get extension").to_str().expect("Failed to convert extension to string");
+            if ext == "txt" || ext == "bin" || ext == "csv" {
+                let str_path = path.to_str().expect("Failed to convert path to string");
+                let file = fs::File::open(str_path)?;
+                let mut reader = BufReader::new(file);
+                zip.start_file(str_path, options)?;
+
+                let mut buffer = [0u8; 8192]; // Use a buffer to stream in chunks
+                loop {
+                    let bytes_read = reader.read(&mut buffer)?;
+                    if bytes_read == 0 {
+                        break;
+                    }
+                    zip.write_all(&buffer[..bytes_read])?;
                 }
-                zip.write_all(&buffer[..bytes_read])?;
             }
         }
-    }
-    zip.finish()?;
-    let zip_file_metadata = fs::metadata(&zip_path)?;
+        zip.finish()?;
+        let zip_file_metadata = fs::metadata(&zip_path)?;
 
-    let mut producer = datastore.upsert(domain_id.to_string()).await?;
-    let id = data_id_generator();
-    let mut chunks = producer.push(
-        &UpsertMetadata {
-            size: zip_file_metadata.len() as u32,
-            name: format!("refined_scan_{}", context.suffix.clone()),
-            data_type: "refined_scan_zip_v1".to_string(),
-            id: id.clone(),
-            is_new: true,
-            properties: HashMap::new(),
-        },
-    ).await?;
-    
-    let _ = chunks.next_chunk(&fs::read(&zip_path)?, false).await?;
-    while !producer.is_completed().await {
-        sleep(Duration::from_secs(3)).await;
+        let mut producer = datastore.upsert(domain_id.to_string()).await?;
+        res_id = data_id_generator();
+        let mut chunks = producer.push(
+            &UpsertMetadata {
+                size: zip_file_metadata.len() as u32,
+                name: format!("refined_scan_{}", context.suffix.clone()),
+                data_type: "refined_scan_zip_v1".to_string(),
+                id: res_id.clone(),
+                is_new: true,
+                properties: HashMap::new(),
+            },
+        ).await?;
+        
+        let _ = chunks.next_chunk(&fs::read(&zip_path)?, false).await?;
+        while !producer.is_completed().await {
+            sleep(Duration::from_secs(3)).await;
+        }
+        producer.close().await;
+        std::fs::remove_dir_all(&context.task_folder)?;
     }
-    producer.close().await;
-    println!("Finished uploading refined scan for {}", context.suffix.clone());
-    std::fs::remove_dir_all(&context.task_folder)?;
-
+    
     let output = LocalRefinementOutputV1 {
-        result_ids: vec![id],
+        result_ids: vec![res_id],
     };
     
     return Ok(output);
diff --git a/server/src/main.rs b/server/src/main.rs
index 94b75d6..aa5181f 100644
--- a/server/src/main.rs
+++ b/server/src/main.rs
@@ -5,6 +5,7 @@ use domain::{auth::{AuthClient, AuthError}, capabilities::public_key::PublicKeyS
 use networking::client::Client;
 use tokio::{self, select, signal::unix::{signal, SignalKind}};
 use futures::{lock::Mutex, StreamExt};
+use tracing_subscriber::{fmt, prelude::__tracing_subscriber_SubscriberExt, EnvFilter, Registry};
 mod local_refinement;
 mod global_refinement;
 mod utils;
@@ -14,8 +15,8 @@ async fn shutdown_signal() {
     let mut int_signal = signal(SignalKind::interrupt()).expect("Failed to register SIGINT handler");
 
     tokio::select! {
-        _ = term_signal.recv() => println!("Received SIGTERM, exiting..."),
-        _ = int_signal.recv() => println!("Received SIGINT, exiting..."),
+        _ = term_signal.recv() => tracing::info!("Received SIGTERM, exiting..."),
+        _ = int_signal.recv() => tracing::info!("Received SIGINT, exiting..."),
     }
 }
 
@@ -67,7 +68,10 @@ async fn main() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     let base_path = format!("./volume/{}", name);
     let domain_manager = args[3].clone();
     let private_key_path = format!("{}/pkey", base_path);
-    tracing_subscriber::fmt().with_env_filter(tracing_subscriber::EnvFilter::from_default_env()).init();
+    let subscriber = Registry::default()
+            .with(fmt::layer().with_file(true).with_line_number(true))
+            .with(EnvFilter::new("info"));
+    tracing::subscriber::set_global_default(subscriber).expect("failed to set subscriber");
 
     let domain_cluster = DomainCluster::new(domain_manager.clone(), name, false, port, false, false, None, Some(private_key_path), vec![domain_manager.clone()]);
     let domain_manager_id = domain_cluster.manager_id.clone();
@@ -89,14 +93,17 @@ async fn main() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
                 let _ = tokio::spawn(local_refinement::v1(base_path.clone(), stream, remote_storage.clone(), n.client.clone(), keys_loader.clone()));
             }
             Some((_, stream)) = global_refinement_v1_handler.next() => {
-                let _ = tokio::spawn(global_refinement::v1(base_path.clone(), stream, remote_storage.clone(), n.client.clone(), keys_loader.clone()));
+                let res = tokio::spawn(global_refinement::v1(base_path.clone(), stream, remote_storage.clone(), n.client.clone(), keys_loader.clone()));
+                if let Err(e) = res.await {
+                    tracing::error!("Error: {}", e);
+                }
             }
             _ = shutdown_signal() => {
                 c.cancel().await.unwrap_or_else(|e| {
-                    eprintln!("Failed to cancel client: {}", e);
+                    tracing::error!("Failed to cancel client: {}", e);
                 });
                 keys_loader.cleanup().await;
-                println!("Received termination signal, shutting down...");
+                tracing::info!("Received termination signal, shutting down...");
                 break;
             }
             else => break
diff --git a/server/src/utils.rs b/server/src/utils.rs
index 0df46df..0661bf0 100644
--- a/server/src/utils.rs
+++ b/server/src/utils.rs
@@ -1,5 +1,5 @@
 use domain::protobuf::task::{Status, Task};
-use networking::client::Client;
+use networking::client::TClient;
 use quick_protobuf::serialize_into_vec;
 use tokio::{sync::watch, task::JoinHandle, time::interval};
 use std::{collections::HashSet, error::Error, fs, path::Path, process::Stdio, time::Duration};
@@ -106,19 +106,23 @@ pub fn write_scan_data_summary(
     Ok(())
 } 
 
-pub async fn health(task: Task, mut c: Client, job_id: &str, rx: watch::Receiver<bool>) -> JoinHandle<()> {
+pub async fn health<C: TClient + Send + Sync + 'static>(task: Task, mut c: C, job_id: &str, rx: watch::Receiver<bool>) -> JoinHandle<()> {
     let job_id_clone = job_id.to_string();
     tokio::spawn(async move {
-        let mut rx = rx;
+        let mut rx = rx.clone();
         let mut interval = interval(Duration::from_secs(30)); // Send heartbeat every 30 seconds
-        
         loop {
             tokio::select! {
                 _ = interval.tick() => {
                     let mut progress_task = task.clone();
                     progress_task.status = Status::PROCESSING;
                     if let Ok(message) = serialize_into_vec(&progress_task) {
-                        let _ = c.publish(job_id_clone.clone(), message).await;
+                        if let Err(e) = c.publish(job_id_clone.clone(), message).await {
+                            tracing::error!("Error publishing heartbeat: {}", e);
+                            continue;
+                        }
+                    } else {
+                        tracing::error!("Failed to serialize heartbeat");
                     }
                 }
                 // Check if we should stop
@@ -154,7 +158,7 @@ pub async fn execute_python(params: Vec<&str>) -> Result<(), Box<dyn Error + Sen
         tokio::spawn(async move {
             let mut lines = stderr_reader.lines();
             while let Ok(Some(line)) = lines.next_line().await {
-                eprintln!("stderr: {}", line);
+                println!("stderr: {}", line);
             }
         });
     }
@@ -162,7 +166,83 @@ pub async fn execute_python(params: Vec<&str>) -> Result<(), Box<dyn Error + Sen
     // Wait for the process to complete
     let status = child.wait().await?;
     if !status.success() {
-        return Err("Python process failed".into());
+        tracing::info!("Python process failed");
+        return Err(format!("Python process failed with status {}", status).into());
     }
     Ok(())
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use async_trait::async_trait;
+    use networking::client::TClient;
+    use networking::libp2p::NetworkError;
+    use tokio::sync::watch;
+    use tokio::time::{sleep, Duration};
+    use std::sync::{Arc, Mutex};
+
+    // Mock Client to capture published messages
+    #[derive(Clone)]
+    struct MockClient {
+        published_messages: Arc<Mutex<Vec<Vec<u8>>>>,
+    }
+
+    impl MockClient {
+        fn new() -> Self {
+            MockClient {
+                published_messages: Arc::new(Mutex::new(Vec::new())),
+            }
+        }
+
+        fn get_published_messages(&self) -> Vec<Vec<u8>> {
+            self.published_messages.lock().unwrap().clone()
+        }
+    }
+
+    #[async_trait]
+    impl TClient for MockClient {
+        async fn publish(&mut self, _job_id: String, message: Vec<u8>) -> Result<(), NetworkError> {
+            self.published_messages.lock().unwrap().push(message);
+            Ok(())
+        }
+        async fn subscribe(&mut self, _job_id: String) -> Result<(), NetworkError> {
+            Ok(())
+        }
+    }
+
+    #[tokio::test]
+    async fn test_health_sends_heartbeat() {
+        let task = Task {
+            status: Status::PROCESSING,
+            ..Default::default()
+        };
+        let client = MockClient::new();
+        let (tx, rx) = watch::channel(false);
+        // Start the health task
+        let heartbeat_handle = health(task, client.clone(), "job_id", rx).await;
+
+        // Allow some time for heartbeats to be sent
+        sleep(Duration::from_secs(32)).await;
+
+        // Check that heartbeats were sent
+        let messages = client.get_published_messages();
+        assert!(messages.len() >= 1, "Expected at least 1 heartbeat, got {}", messages.len());
+
+        // Stop the heartbeat task
+        tx.send(true).unwrap();
+        let _ = heartbeat_handle.await;
+    }
+
+    #[tokio::test]
+    async fn test_execute_python_should_fail_when_python_script_does_not_exist() {
+        async fn run() -> Result<(), Box<dyn core::error::Error + Send + Sync>> {
+            let params = vec!["non_existent_script.py"];
+            execute_python(params).await?;
+            Ok(())
+        }
+
+        let res = run().await;
+        assert!(res.is_err(), "Expected error when script does not exist");
+    }
+}

From 50cdc269b5c1266f9454d1fba8c9f53bb2b898af Mon Sep 17 00:00:00 2001
From: RobinLindh <robin@aukilabs.com>
Date: Thu, 15 May 2025 01:01:54 +0200
Subject: [PATCH 84/89] wip

---
 .devcontainer/devcontainer.json | 5 +++--
 docker/Dockerfile               | 4 +++-
 2 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json
index 9e6b1ed..49ec737 100644
--- a/.devcontainer/devcontainer.json
+++ b/.devcontainer/devcontainer.json
@@ -15,7 +15,8 @@
             "extensions":[
                 "golang.Go",
                 "ms-python.python",
-                "ms-python.debugpy"
+                "ms-python.debugpy",
+                "rust-lang.rust-analyzer"
             ]
         }
     },
@@ -36,5 +37,5 @@
         // Uncomment if you want display forward
         // "source=/tmp/.X11-unix,target=/tmp/.X11-unix,type=bind,consistency=cached"
     ],
-    "postCreateCommand": "sudo chown -R reconstruction-server:reconstruction-server /app && .devcontainer/build_server.sh"
+    "postCreateCommand": "sudo .devcontainer/build_server.sh && sudo chown -R reconstruction-server:reconstruction-server /app"
 }
\ No newline at end of file
diff --git a/docker/Dockerfile b/docker/Dockerfile
index 956dc30..24389a5 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -186,7 +186,7 @@ ENV CARGO_HOME="/usr/local/cargo"
 
 # Standalone image: Use the compiled server binary
 FROM runtime AS reconstruction
-COPY --from=rust-build /app/server/target/release/server /app/server
+COPY --from=rust-build /app/server/target/release/server /app/reconstruction
 RUN chmod +x /app/server
 
 # Final stage: Select between dev and prod
@@ -194,4 +194,6 @@ FROM reconstruction${IS_DEVELOPMENT:+_dev} AS final
 
 WORKDIR /app
 USER $USERNAME
+ENTRYPOINT ["/app/reconstruction"]
+
 

From d26fe0ae5b92a842428bcf19eaa24a8e5820f2c0 Mon Sep 17 00:00:00 2001
From: Shuning <shuning@aukilabs.com>
Date: Thu, 29 May 2025 16:58:54 +0800
Subject: [PATCH 85/89] download data in chunks and write them to file system
 as stream

---
 server/Cargo.lock               | 256 +++++++++++++++---------
 server/Cargo.toml               |   4 +-
 server/src/global_refinement.rs | 119 ++++++++---
 server/src/local_refinement.rs  | 343 +++++++++++++++++++-------------
 server/src/main.rs              |  35 +++-
 server/src/utils.rs             |   8 +-
 6 files changed, 491 insertions(+), 274 deletions(-)

diff --git a/server/Cargo.lock b/server/Cargo.lock
index b9797be..f2662ac 100644
--- a/server/Cargo.lock
+++ b/server/Cargo.lock
@@ -807,7 +807,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8d162beedaa69905488a8da94f5ac3edb4dd4788b732fadb7bd120b2625c1976"
 dependencies = [
  "data-encoding",
- "syn 2.0.101",
+ "syn 1.0.109",
 ]
 
 [[package]]
@@ -899,40 +899,10 @@ dependencies = [
 ]
 
 [[package]]
-name = "domain"
-version = "0.1.0"
-source = "git+https://github.com/aukilabs/posemesh.git?rev=5512cf097bf262130c95296b77c93d9bc2b384f6#5512cf097bf262130c95296b77c93d9bc2b384f6"
-dependencies = [
- "async-timer",
- "async-trait",
- "base64 0.22.1",
- "cbindgen",
- "console_error_panic_hook",
- "futures",
- "hex",
- "jsonwebtoken",
- "libp2p",
- "networking",
- "pb-rs",
- "quick-protobuf",
- "quick-protobuf-codec 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
- "ring 0.17.14",
- "rs_merkle",
- "runtime",
- "serde",
- "serde-wasm-bindgen",
- "serde_json",
- "sha2",
- "thiserror 2.0.12",
- "tokio",
- "tracing",
- "tracing-subscriber",
- "tracing-wasm",
- "utils",
- "uuid",
- "wasm-bindgen",
- "wasm-bindgen-futures",
-]
+name = "downcast"
+version = "0.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1435fa1053d8b2fbbe9be7e97eca7f33d37b28409959813daefc1446a14247f1"
 
 [[package]]
 name = "dtoa"
@@ -1121,6 +1091,12 @@ dependencies = [
  "percent-encoding",
 ]
 
+[[package]]
+name = "fragile"
+version = "2.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "28dd6caf6059519a65843af8fe2a3ae298b14b80179855aeb4adc2c1934ee619"
+
 [[package]]
 name = "futures"
 version = "0.3.31"
@@ -2653,6 +2629,32 @@ dependencies = [
  "windows-sys 0.52.0",
 ]
 
+[[package]]
+name = "mockall"
+version = "0.13.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "39a6bfcc6c8c7eed5ee98b9c3e33adc726054389233e201c95dab2d41a3839d2"
+dependencies = [
+ "cfg-if",
+ "downcast",
+ "fragile",
+ "mockall_derive",
+ "predicates",
+ "predicates-tree",
+]
+
+[[package]]
+name = "mockall_derive"
+version = "0.13.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "25ca3004c2efe9011bd4e461bd8256445052b9615405b4f7ea43fc8ca5c20898"
+dependencies = [
+ "cfg-if",
+ "proc-macro2",
+ "quote",
+ "syn 2.0.101",
+]
+
 [[package]]
 name = "multiaddr"
 version = "0.18.2"
@@ -2771,37 +2773,6 @@ dependencies = [
  "tokio",
 ]
 
-[[package]]
-name = "networking"
-version = "0.1.0"
-source = "git+https://github.com/aukilabs/posemesh.git?rev=5512cf097bf262130c95296b77c93d9bc2b384f6#5512cf097bf262130c95296b77c93d9bc2b384f6"
-dependencies = [
- "async-trait",
- "cbindgen",
- "console_error_panic_hook",
- "futures",
- "gloo-timers 0.3.0",
- "libp2p",
- "libp2p-stream",
- "libp2p-webrtc",
- "libp2p-webrtc-websys",
- "libp2p-websocket",
- "libp2p-websocket-websys",
- "rand 0.8.5",
- "runtime",
- "serde",
- "serde-wasm-bindgen",
- "serde_json",
- "thiserror 2.0.12",
- "tokio",
- "tracing",
- "tracing-subscriber",
- "tracing-wasm",
- "utils",
- "wasm-bindgen",
- "wasm-bindgen-futures",
-]
-
 [[package]]
 name = "nix"
 version = "0.26.4"
@@ -3128,6 +3099,99 @@ version = "1.11.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "350e9b48cbc6b0e028b0473b114454c6316e57336ee184ceab6e53f72c178b3e"
 
+[[package]]
+name = "posemesh-domain"
+version = "0.1.0"
+source = "git+https://github.com/aukilabs/posemesh.git?rev=06e93d86ea214b51087799c805797e50c41519ed#06e93d86ea214b51087799c805797e50c41519ed"
+dependencies = [
+ "async-timer",
+ "async-trait",
+ "base64 0.22.1",
+ "cbindgen",
+ "console_error_panic_hook",
+ "futures",
+ "hex",
+ "jsonwebtoken",
+ "libp2p",
+ "mockall",
+ "pb-rs",
+ "posemesh-networking",
+ "posemesh-runtime",
+ "posemesh-utils",
+ "quick-protobuf",
+ "quick-protobuf-codec 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "ring 0.17.14",
+ "rs_merkle",
+ "serde",
+ "serde-wasm-bindgen",
+ "serde_json",
+ "sha2",
+ "thiserror 2.0.12",
+ "tokio",
+ "tracing",
+ "tracing-subscriber",
+ "tracing-wasm",
+ "uuid",
+ "wasm-bindgen",
+ "wasm-bindgen-futures",
+]
+
+[[package]]
+name = "posemesh-networking"
+version = "0.1.0"
+source = "git+https://github.com/aukilabs/posemesh.git?rev=06e93d86ea214b51087799c805797e50c41519ed#06e93d86ea214b51087799c805797e50c41519ed"
+dependencies = [
+ "async-trait",
+ "cbindgen",
+ "console_error_panic_hook",
+ "futures",
+ "gloo-timers 0.3.0",
+ "libp2p",
+ "libp2p-stream",
+ "libp2p-webrtc",
+ "libp2p-webrtc-websys",
+ "libp2p-websocket",
+ "libp2p-websocket-websys",
+ "mockall",
+ "posemesh-runtime",
+ "posemesh-utils",
+ "rand 0.8.5",
+ "serde",
+ "serde-wasm-bindgen",
+ "serde_json",
+ "thiserror 2.0.12",
+ "tokio",
+ "tracing",
+ "tracing-subscriber",
+ "tracing-wasm",
+ "wasm-bindgen",
+ "wasm-bindgen-futures",
+]
+
+[[package]]
+name = "posemesh-runtime"
+version = "0.1.0"
+source = "git+https://github.com/aukilabs/posemesh.git?rev=06e93d86ea214b51087799c805797e50c41519ed#06e93d86ea214b51087799c805797e50c41519ed"
+dependencies = [
+ "once_cell",
+ "tokio",
+]
+
+[[package]]
+name = "posemesh-utils"
+version = "0.1.0"
+source = "git+https://github.com/aukilabs/posemesh.git?rev=06e93d86ea214b51087799c805797e50c41519ed#06e93d86ea214b51087799c805797e50c41519ed"
+dependencies = [
+ "futures",
+ "gloo-timers 0.3.0",
+ "js-sys",
+ "tokio",
+ "tracing",
+ "wasm-bindgen",
+ "wasm-bindgen-futures",
+ "web-sys",
+]
+
 [[package]]
 name = "powerfmt"
 version = "0.2.0"
@@ -3143,6 +3207,32 @@ dependencies = [
  "zerocopy",
 ]
 
+[[package]]
+name = "predicates"
+version = "3.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a5d19ee57562043d37e82899fade9a22ebab7be9cef5026b07fda9cdd4293573"
+dependencies = [
+ "anstyle",
+ "predicates-core",
+]
+
+[[package]]
+name = "predicates-core"
+version = "1.0.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "727e462b119fe9c93fd0eb1429a5f7647394014cf3c04ab2c0350eeb09095ffa"
+
+[[package]]
+name = "predicates-tree"
+version = "1.0.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "72dd2d6d381dfb73a193c7fca536518d7caee39fc8503f74e7dc0be0531b425c"
+dependencies = [
+ "predicates-core",
+ "termtree",
+]
+
 [[package]]
 name = "primeorder"
 version = "0.13.6"
@@ -3528,15 +3618,6 @@ dependencies = [
  "webrtc-util 0.8.1",
 ]
 
-[[package]]
-name = "runtime"
-version = "0.1.0"
-source = "git+https://github.com/aukilabs/posemesh.git?rev=5512cf097bf262130c95296b77c93d9bc2b384f6#5512cf097bf262130c95296b77c93d9bc2b384f6"
-dependencies = [
- "once_cell",
- "tokio",
-]
-
 [[package]]
 name = "rustc-demangle"
 version = "0.1.24"
@@ -3791,9 +3872,9 @@ name = "server"
 version = "0.1.0"
 dependencies = [
  "async-trait",
- "domain",
  "futures",
- "networking",
+ "posemesh-domain",
+ "posemesh-networking",
  "quick-protobuf",
  "regex",
  "scopeguard",
@@ -4126,6 +4207,12 @@ dependencies = [
  "winapi-util",
 ]
 
+[[package]]
+name = "termtree"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8f50febec83f5ee1df3015341d8bd429f2d1cc62bcba7ea2076759d315084683"
+
 [[package]]
 name = "textwrap"
 version = "0.11.0"
@@ -4532,21 +4619,6 @@ version = "0.2.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
 
-[[package]]
-name = "utils"
-version = "0.1.0"
-source = "git+https://github.com/aukilabs/posemesh.git?rev=5512cf097bf262130c95296b77c93d9bc2b384f6#5512cf097bf262130c95296b77c93d9bc2b384f6"
-dependencies = [
- "futures",
- "gloo-timers 0.3.0",
- "js-sys",
- "tokio",
- "tracing",
- "wasm-bindgen",
- "wasm-bindgen-futures",
- "web-sys",
-]
-
 [[package]]
 name = "uuid"
 version = "1.16.0"
diff --git a/server/Cargo.toml b/server/Cargo.toml
index 7cce020..67a380c 100644
--- a/server/Cargo.toml
+++ b/server/Cargo.toml
@@ -4,9 +4,9 @@ version = "0.1.0"
 edition = "2021"
 
 [dependencies]
-networking = { git = "https://github.com/aukilabs/posemesh.git", rev = "5512cf097bf262130c95296b77c93d9bc2b384f6" }
+posemesh-networking = { git = "https://github.com/aukilabs/posemesh.git", rev = "06e93d86ea214b51087799c805797e50c41519ed" }
 futures = "0.3.30"
-domain = { git = "https://github.com/aukilabs/posemesh.git", rev = "5512cf097bf262130c95296b77c93d9bc2b384f6" }
+posemesh-domain = { git = "https://github.com/aukilabs/posemesh.git", rev = "06e93d86ea214b51087799c805797e50c41519ed" }
 tracing = "0.1.40"
 tracing-subscriber = "0.3.18"
 tokio = "1.44.0"
diff --git a/server/src/global_refinement.rs b/server/src/global_refinement.rs
index eab8ac8..bc0e879 100644
--- a/server/src/global_refinement.rs
+++ b/server/src/global_refinement.rs
@@ -1,10 +1,10 @@
-use std::{collections::HashMap, fs::{self, File}, io::Cursor, path::{Path, PathBuf}};
-use domain::{auth::{handshake, TaskTokenClaim}, capabilities::public_key::PublicKeyStorage, datastore::{common::{data_id_generator, Datastore}, remote::RemoteDatastore}, message::read_prefix_size_message, protobuf::{domain_data::{Metadata, Query, UpsertMetadata}, task::{self, Task}}};
-use networking::{client::{Client, TClient}, AsyncStream};
-use quick_protobuf::serialize_into_vec;
+use std::{collections::HashMap, fs::{self, File}, io::{self, Cursor, Error, ErrorKind}, path::{Path, PathBuf}, pin::Pin, sync::{Arc, Mutex}, task::{Context, Poll}};
+use posemesh_domain::{auth::{handshake, TaskTokenClaim}, capabilities::public_key::PublicKeyStorage, datastore::{common::{data_id_generator, Datastore}, remote::RemoteDatastore}, message::read_prefix_size_message, protobuf::{domain_data::{self, Metadata, Query, UpsertMetadata}, task}};
+use posemesh_networking::{client::{Client, TClient}, AsyncStream};
+use quick_protobuf::{deserialize_from_slice, serialize_into_vec};
 use regex::Regex;
 use tokio::{self, sync::watch, time::{sleep, Duration}};
-use futures::StreamExt;
+use futures::AsyncWrite;
 use zip::ZipArchive;
 
 use crate::utils::{execute_python, health};
@@ -55,7 +55,7 @@ async fn upload_results(domain_id: &str, output_path: PathBuf, datastore: &mut R
     Ok(())
 }
 
-fn unzip_bytes(path: PathBuf, zip_bytes: Vec<u8>) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+fn unzip_bytes(path: PathBuf, zip_bytes: &[u8]) -> io::Result<()> {
     let cursor = Cursor::new(zip_bytes);
     let mut archive = ZipArchive::new(cursor)?;
     tracing::info!("Zip archive opened, contains {} files", archive.len());
@@ -73,7 +73,7 @@ fn unzip_bytes(path: PathBuf, zip_bytes: Vec<u8>) -> Result<(), Box<dyn std::err
         }
         
         let mut output_file: File = File::create(&file_path)?;
-        let bytes_copied = std::io::copy(&mut input_file, &mut output_file)?;
+        let _ = std::io::copy(&mut input_file, &mut output_file)?;
     }
     Ok(())
 }
@@ -118,6 +118,81 @@ async fn initialize_task<S: AsyncStream, P: PublicKeyStorage, C: TClient + Clone
     Ok((task, job_id, claim, task_path, input_path, output_path))
 }
 
+fn download_local_refinement_result(input_path: &Path, dataset_path: &Path, name: &str, data: &[u8]) -> io::Result<String> {
+    let suffix = extract_suffix(name).map_err(|_| Error::new(ErrorKind::Other, "Failed to extract suffix"))?;
+    fs::create_dir_all(dataset_path.join(&suffix))?;
+    let path = input_path.join(&suffix).join("sfm");
+    unzip_bytes(path, data)?;
+    Ok(suffix)
+}
+
+#[derive(Clone)]
+struct ToZipArchive {
+    input_path: PathBuf,
+    dataset_path: PathBuf,
+    size: u32,
+    name: String,
+    content: Vec<u8>,
+    scan_ids: Arc<Mutex<Vec<String>>>,
+}
+impl ToZipArchive {
+    fn new(input_path: PathBuf, dataset_path: PathBuf) -> Self {
+        Self {
+            input_path,
+            dataset_path,
+            size: 0,
+            name: "".to_string(),
+            content: vec![],
+            scan_ids: Arc::new(Mutex::new(vec![])),
+        }
+    }
+
+    pub fn scan_ids(&self) -> Vec<String> {
+        self.scan_ids.lock().unwrap().clone()
+    }
+}
+
+impl AsyncWrite for ToZipArchive {
+    fn poll_write(mut self: Pin<&mut Self>, _: &mut Context<'_>, content: &[u8]) -> Poll<Result<usize, Error>> {
+        if self.size == 0 {
+            if content.len() < 4 {
+                return Poll::Ready(Err(Error::new(ErrorKind::Other, "Content is too short")));
+            }
+            let metadata = deserialize_from_slice::<Metadata>(&content[..4]).map_err(|_| Error::new(ErrorKind::Other, "Failed to deserialize metadata"))?;
+            self.size = metadata.size;
+            self.name = metadata.name;
+            self.content = vec![];
+            return Poll::Ready(Ok(content.len()));
+        } else {
+            self.content.extend_from_slice(content);
+            return Poll::Ready(Ok(content.len()));
+        }
+    }
+    
+    fn poll_flush(mut self: Pin<&mut Self>, _: &mut Context<'_>) -> Poll<Result<(), Error>> {
+        if self.size == 0 {
+            return Poll::Ready(Ok(()));
+        }
+        if self.size > self.content.len() as u32 {
+            return Poll::Ready(Ok(()));
+        }
+        let suffix = download_local_refinement_result(&self.input_path, &self.dataset_path, &self.name, &self.content)?;
+        self.scan_ids.lock().unwrap().push(suffix);
+        self.content = vec![];
+        self.size = 0;
+        self.name = "".to_string();
+        
+        Poll::Ready(Ok(()))
+    }
+
+    fn poll_close(mut self: Pin<&mut Self>, _: &mut Context<'_>) -> Poll<Result<(), Error>> {
+        self.content = vec![];
+        self.size = 0;
+        self.name = "".to_string();
+        Poll::Ready(Ok(()))
+    }
+}
+
 async fn download_data(
     datastore: &mut RemoteDatastore,
     domain_id: &str,
@@ -125,34 +200,18 @@ async fn download_data(
     input_path: &Path,
     dataset_path: &Path,
 ) -> Result<Vec<String>, Box<dyn std::error::Error + Send + Sync>> {
-    let mut downloader = datastore.load(domain_id.to_string(), query, false).await?;
-    let mut scan_ids = Vec::new();
-
-    while let Some(data) = downloader.next().await {
-        match data {
-            Ok(data) => {
-                let suffix = extract_suffix(&data.metadata.name)?;
-                scan_ids.push(suffix.clone());
-
-                fs::create_dir_all(dataset_path.join(&suffix))?;
-                let path = input_path.join(&suffix).join("sfm");
-                unzip_bytes(path, data.content)?;
-                tracing::info!("downloaded {}", data.metadata.name);
-            }
-            Err(e) => {
-                return Err(Box::new(e));
-            }
-        }
-    }
+    let to_file_system = ToZipArchive::new(input_path.to_path_buf(), dataset_path.to_path_buf());
+    let mut downloader = datastore.load(domain_id.to_string(), query, false, to_file_system.clone()).await?;
+    downloader.wait_for_done().await?;
 
-    if scan_ids.is_empty() {
+    if to_file_system.scan_ids().is_empty() {
         return Err("No scans to refine".into());
     }
 
-    Ok(scan_ids)
+    Ok(to_file_system.scan_ids())
 }
 
-fn extract_suffix(name: &str) -> Result<String, Box<dyn std::error::Error + Send + Sync>> {
+fn extract_suffix(name: &str) -> Result<String, Box<dyn std::error::Error>> {
     let date_time_regex = Regex::new(r"\d{4}-\d{2}-\d{2}[_-]\d{2}-\d{2}-\d{2}")?;
     date_time_regex
         .find(name)
@@ -170,7 +229,7 @@ pub(crate) async fn v1<S: AsyncStream, P: PublicKeyStorage>(
     let (mut t, job_id, claim, task_path, input_path, output_path) =
         initialize_task(&base_path, &mut stream, &mut c, key_loader).await?;
 
-    let input = read_prefix_size_message::<task::GlobalRefinementInputV1>(&mut stream).await?;
+    let input = read_prefix_size_message::<task::GlobalRefinementInputV1, _>(&mut stream).await?;
     tracing::info!("Received global refinement input: {:?}", input);
 
     let mut query = Query::default();
diff --git a/server/src/local_refinement.rs b/server/src/local_refinement.rs
index 5a2d242..1536d78 100644
--- a/server/src/local_refinement.rs
+++ b/server/src/local_refinement.rs
@@ -1,13 +1,136 @@
-use std::{collections::HashMap, fs, path::{Path, PathBuf}, time::Duration};
-use domain::{auth::handshake, capabilities::public_key::PublicKeyStorage, datastore::{common::{data_id_generator, Datastore}, remote::RemoteDatastore}, message::read_prefix_size_message, protobuf::{domain_data::{Query, UpsertMetadata},task::{self, LocalRefinementInputV1, LocalRefinementOutputV1}}};
-use networking::{client::TClient, AsyncStream};
-use quick_protobuf::serialize_into_vec;
-use futures::StreamExt;
+use std::{collections::HashMap, fs::{self, OpenOptions}, io::{Error, ErrorKind}, path::{Path, PathBuf}, pin::Pin, sync::{Arc, Mutex}, task::{Context, Poll}, time::Duration};
+use posemesh_domain::{auth::handshake, capabilities::public_key::PublicKeyStorage, datastore::{common::{data_id_generator, Datastore}, remote::RemoteDatastore}, message::read_prefix_size_message, protobuf::{domain_data::{Metadata, Query, UpsertMetadata},task::{self, LocalRefinementInputV1, LocalRefinementOutputV1}}};
+use posemesh_networking::{client::TClient, AsyncStream};
+use quick_protobuf::{deserialize_from_slice, serialize_into_vec};
+use futures::{AsyncWrite, StreamExt};
 use tokio::{sync::watch, time::sleep};
 use std::io::{BufReader, Read, Write};
 use zip::write::SimpleFileOptions;
 use crate::utils::{execute_python, health, write_scan_data_summary};
 
+const REFINED_SCAN_NAME: &str = "refined_scan_";
+
+#[derive(Clone)]
+struct ToFileSystem {
+    size: u32,
+    content: Option<Vec<u8>>,
+    path: Option<PathBuf>,
+    input_folder: PathBuf,
+    count: Arc<Mutex<u32>>,
+    written: u32,
+    refined_res_id: Arc<Mutex<Option<String>>>,
+}
+impl ToFileSystem {
+    fn new(input_folder: PathBuf) -> Self {
+        Self {
+            input_folder,
+            path: None,
+            size: 0,
+            content: None,
+            count: Arc::new(Mutex::new(0)),
+            written: 0,
+            refined_res_id: Arc::new(Mutex::new(None)),
+        }
+    }
+
+    fn count(&self) -> u32 {
+        self.count.lock().unwrap().clone()
+    }
+
+    fn refined_res_id(&self) -> Option<String> {
+        self.refined_res_id.lock().unwrap().clone()
+    }
+}
+
+impl AsyncWrite for ToFileSystem {
+    fn poll_write(mut self: Pin<&mut Self>, _: &mut Context<'_>, content: &[u8]) -> Poll<Result<usize, Error>> {
+        if self.path.is_none() {
+            if content.len() < 4 {
+                return Poll::Ready(Err(Error::new(ErrorKind::Other, "Content is too short")));
+            }
+            match deserialize_from_slice::<Metadata>(&content[..4]) {
+                Ok(metadata) => {
+                    if let Some(_) = metadata.name.strip_prefix("refined_scan_") {
+                        self.refined_res_id.lock().unwrap().insert(metadata.id.clone());
+                        return Poll::Ready(Err(Error::new(ErrorKind::Other, format!("RefinedAlready({})", metadata.id))));
+                    }
+                    let filename = match metadata.name.as_str() {
+                        "Manifest.json" => "Manifest.json".to_string(),
+                        "FeaturePoints.ply" => "FeaturePoints.ply".to_string(),
+                        "ARposes.csv" => "ARposes.csv".to_string(),
+                        "PortalDetections.csv" => "PortalDetections.csv".to_string(),
+                        "CameraIntrinsics.csv" => "CameraIntrinsics.csv".to_string(),
+                        "Frames.csv" => "Frames.csv".to_string(),
+                        "Gyro.csv" => "Gyro.csv".to_string(),
+                        "Accel.csv" => "Accel.csv".to_string(),
+                        "gyro_accel.csv" => "gyro_accel.csv".to_string(),
+                        "Frames.mp4" => "Frames.mp4".to_string(),
+                        _ => {
+                            match metadata.data_type.as_str() {
+                                "dmt_manifest_json" => "Manifest.json".to_string(),
+                                "dmt_featurepoints_ply" | "dmt_pointcloud_ply" => "FeaturePoints.ply".to_string(),
+                                "dmt_arposes_csv" => "ARposes.csv".to_string(),
+                                "dmt_portal_detections_csv" | "dmt_observations_csv" => "PortalDetections.csv".to_string(),
+                                "dmt_intrinsics_csv" | "dmt_cameraintrinsics_csv" => "CameraIntrinsics.csv".to_string(),
+                                "dmt_frames_csv" => "Frames.csv".to_string(),
+                                "dmt_gyro_csv" => "Gyro.csv".to_string(),
+                                "dmt_accel_csv" => "Accel.csv".to_string(),
+                                "dmt_gyroaccel_csv" => "gyro_accel.csv".to_string(),
+                                "dmt_recording_mp4" => "Frames.mp4".to_string(),
+                                _ => {
+                                    tracing::info!("unknown domain data type: {}", metadata.data_type);
+                                    format!("{}.{}", metadata.name, metadata.data_type)
+                                }
+                            }
+                        }
+                    };
+
+                    self.path = Some(self.input_folder.join(&filename));
+                    self.written = 0;
+                    return Poll::Ready(Ok(content.len()));
+                }
+                Err(_) => {
+                    return Poll::Ready(Err(Error::new(ErrorKind::Other, "Failed to deserialize metadata")));
+                }
+            }
+        } else {
+            self.content
+                .get_or_insert_with(Vec::new)
+                .extend_from_slice(content);
+            return Poll::Ready(Ok(content.len()));
+        }
+    }
+    
+    fn poll_flush(mut self: Pin<&mut Self>, _: &mut Context<'_>) -> Poll<Result<(), Error>> {
+        if self.path.is_none() {
+            return Poll::Ready(Ok(()));
+        }
+        let path = self.path.as_ref().unwrap();
+        let mut file = OpenOptions::new()
+            .create(true)
+            .append(true)
+            .open(path)?;
+        let content = self.content.take().unwrap_or_default();
+        file.write_all(&content)?;
+
+        self.written += content.len() as u32;
+        if self.written == self.size {
+            self.path = None;
+            self.written = 0;
+            let mut count = self.count.lock().unwrap();
+            *count += 1;
+        }
+        
+        Poll::Ready(Ok(()))
+    }
+
+    fn poll_close(mut self: Pin<&mut Self>, _: &mut Context<'_>) -> Poll<Result<(), Error>> {
+        self.content = None;
+        self.path = None;
+        Poll::Ready(Ok(()))
+    }
+}
+
 pub(crate) async fn v1<S: AsyncStream, P: PublicKeyStorage, C: TClient + Clone + Send + Sync + 'static>(base_path: String, mut stream: S, datastore: RemoteDatastore, mut c: C, key_loader: P) {
     let claim = handshake(&mut stream, key_loader).await.expect("Failed to handshake");
     let job_id = claim.job_id.clone();
@@ -87,8 +210,8 @@ struct LocalRefinementContext {
     suffix: String,
 }
 
-async fn start<S: AsyncStream>(base_path: String, job_id: &str, task_name: &str, stream: S) -> Result<LocalRefinementContext, Box<dyn std::error::Error + Send + Sync>> {
-    let input = read_prefix_size_message::<LocalRefinementInputV1>(stream).await?;
+async fn start<S: AsyncStream>(base_path: String, job_id: &str, task_name: &str, mut stream: S) -> Result<LocalRefinementContext, Box<dyn std::error::Error + Send + Sync>> {
+    let input = read_prefix_size_message::<LocalRefinementInputV1, _>(&mut stream).await?;
 
     tracing::info!("Start executing {}, {:?}", task_name, input);
 
@@ -135,149 +258,91 @@ async fn start<S: AsyncStream>(base_path: String, job_id: &str, task_name: &str,
 
 async fn run(domain_id: &str, job_id: &str, task_name: &str, context: LocalRefinementContext, mut datastore: RemoteDatastore) -> Result<LocalRefinementOutputV1, Box<dyn std::error::Error + Send + Sync>> { 
     let query = context.query;
-    let mut res_id: String = "".to_string();
-
-    let mut check_refined = datastore.load(domain_id.to_string(), Query {
-        ids: vec![],
-        name_regexp: None,
-        data_type_regexp: None,
-        names: vec![format!("refined_scan_{}", context.suffix.clone())],
-        data_types: vec![],
-        metadata_only: true,
-    }, false).await.unwrap_or_else(|e| {
-        panic!("Failed to load refined scan {}: {}", context.suffix.clone(), e);
-    });
-    while let Some(Ok(data)) = check_refined.next().await {
-        tracing::info!("found refined scan: {}", data.metadata.name);
-        res_id = data.metadata.id.clone();
-    }
-    check_refined.close();
-
-    if res_id.is_empty() {
-        let mut downloader = datastore.load(domain_id.to_string(), Query {
-            metadata_only: false,
-            ..query
-        }, false).await?;
-        let mut i = 0;
-        loop {
-            match downloader.next().await {
-                Some(Ok(data)) => {
-                    let filename = match data.metadata.name.as_str() {
-                        "Manifest.json" => "Manifest.json".to_string(),
-                        "FeaturePoints.ply" => "FeaturePoints.ply".to_string(),
-                        "ARposes.csv" => "ARposes.csv".to_string(),
-                        "PortalDetections.csv" => "PortalDetections.csv".to_string(),
-                        "CameraIntrinsics.csv" => "CameraIntrinsics.csv".to_string(),
-                        "Frames.csv" => "Frames.csv".to_string(),
-                        "Gyro.csv" => "Gyro.csv".to_string(),
-                        "Accel.csv" => "Accel.csv".to_string(),
-                        "gyro_accel.csv" => "gyro_accel.csv".to_string(),
-                        "Frames.mp4" => "Frames.mp4".to_string(),
-                        _ => {
-                            match data.metadata.data_type.as_str() {
-                                "dmt_manifest_json" => "Manifest.json".to_string(),
-                                "dmt_featurepoints_ply" | "dmt_pointcloud_ply" => "FeaturePoints.ply".to_string(),
-                                "dmt_arposes_csv" => "ARposes.csv".to_string(),
-                                "dmt_portal_detections_csv" | "dmt_observations_csv" => "PortalDetections.csv".to_string(),
-                                "dmt_intrinsics_csv" | "dmt_cameraintrinsics_csv" => "CameraIntrinsics.csv".to_string(),
-                                "dmt_frames_csv" => "Frames.csv".to_string(),
-                                "dmt_gyro_csv" => "Gyro.csv".to_string(),
-                                "dmt_accel_csv" => "Accel.csv".to_string(),
-                                "dmt_gyroaccel_csv" => "gyro_accel.csv".to_string(),
-                                "dmt_recording_mp4" => "Frames.mp4".to_string(),
-                                _ => {
-                                    tracing::info!("unknown domain data type: {}", data.metadata.data_type);
-                                    format!("{}.{}", data.metadata.name, data.metadata.data_type)
-                                }
-                            }
-                        }
-                    };
 
-                    let path = context.input_folder.join(&filename);
-                    fs::write(path, &data.content)?;
-                    i += 1;
-                    tracing::info!("downloaded {}", filename);
-                }
-                Some(Err(e)) => {
-                    return Err(e.into());
-                }
-                None => {
-                    tracing::info!("no more data to download");
-                    break;
-                }
-            }
+    let writer = ToFileSystem::new(context.output_folder.clone());
+    let mut downloader = datastore.load(domain_id.to_string(), Query {
+        metadata_only: false,
+        ..query
+    }, false, writer.clone()).await?;
+    if let Err(e) = downloader.wait_for_done().await {
+        let refined_res_id = writer.refined_res_id();
+        if refined_res_id.is_some() {
+            return Ok(LocalRefinementOutputV1 {
+                result_ids: vec![refined_res_id.unwrap()],
+            });
         }
+        return Err(Box::new(e));
+    }
 
-        tracing::info!("Finished downloading {} data for {}", i, task_name);
+    tracing::info!("Finished downloading {} data for {}", writer.count(), task_name);
 
-        write_scan_data_summary(context.input_folder.as_path(), context.task_folder.as_path().join("scan_data_summary.json").as_path())?;
+    write_scan_data_summary(context.input_folder.as_path(), context.task_folder.as_path().join("scan_data_summary.json").as_path())?;
 
-        let params = vec![
-            "-u",
-            "main.py",
-            "--mode", "local_refinement",
-            "--job_root_path", context.task_folder.to_str().unwrap(),
-            "--output", context.output_folder.to_str().unwrap(),
-            "--domain_id", domain_id,
-            "--job_id", job_id,
-            "--scans", context.suffix.as_str(),
-        ];
+    let params = vec![
+        "-u",
+        "main.py",
+        "--mode", "local_refinement",
+        "--job_root_path", context.task_folder.to_str().unwrap(),
+        "--output", context.output_folder.to_str().unwrap(),
+        "--domain_id", domain_id,
+        "--job_id", job_id,
+        "--scans", context.suffix.as_str(),
+    ];
 
-        execute_python(params).await?;
+    execute_python(params).await?;
 
-        let sfm = context.output_folder.join(context.suffix.clone()).join("sfm");
-        fs::create_dir_all(&sfm)?;
-        let zip_path = sfm.join(context.suffix.clone() + ".zip");
-        let mut zip = zip::ZipWriter::new(fs::File::create(&zip_path)?);
-        let options = SimpleFileOptions::default().compression_method(zip::CompressionMethod::Stored); // No compression
-        
-        // open output folder/sfm, zip all txt, bin, csv files and upload
-        for entry in fs::read_dir(sfm)? {
-            let entry = entry?;
-            let path = entry.path();
-            if path.is_dir() {
-                continue;
-            }
-            let ext = path.extension().expect("Failed to get extension").to_str().expect("Failed to convert extension to string");
-            if ext == "txt" || ext == "bin" || ext == "csv" {
-                let str_path = path.to_str().expect("Failed to convert path to string");
-                let file = fs::File::open(str_path)?;
-                let mut reader = BufReader::new(file);
-                zip.start_file(str_path, options)?;
-
-                let mut buffer = [0u8; 8192]; // Use a buffer to stream in chunks
-                loop {
-                    let bytes_read = reader.read(&mut buffer)?;
-                    if bytes_read == 0 {
-                        break;
-                    }
-                    zip.write_all(&buffer[..bytes_read])?;
+    let sfm = context.output_folder.join(context.suffix.clone()).join("sfm");
+    fs::create_dir_all(&sfm)?;
+    let zip_path = sfm.join(context.suffix.clone() + ".zip");
+    let mut zip = zip::ZipWriter::new(fs::File::create(&zip_path)?);
+    let options = SimpleFileOptions::default().compression_method(zip::CompressionMethod::Stored); // No compression
+    
+    // open output folder/sfm, zip all txt, bin, csv files and upload
+    for entry in fs::read_dir(sfm)? {
+        let entry = entry?;
+        let path = entry.path();
+        if path.is_dir() {
+            continue;
+        }
+        let ext = path.extension().expect("Failed to get extension").to_str().expect("Failed to convert extension to string");
+        if ext == "txt" || ext == "bin" || ext == "csv" {
+            let str_path = path.to_str().expect("Failed to convert path to string");
+            let file = fs::File::open(str_path)?;
+            let mut reader = BufReader::new(file);
+            zip.start_file(str_path, options)?;
+
+            let mut buffer = [0u8; 8192]; // Use a buffer to stream in chunks
+            loop {
+                let bytes_read = reader.read(&mut buffer)?;
+                if bytes_read == 0 {
+                    break;
                 }
+                zip.write_all(&buffer[..bytes_read])?;
             }
         }
-        zip.finish()?;
-        let zip_file_metadata = fs::metadata(&zip_path)?;
-
-        let mut producer = datastore.upsert(domain_id.to_string()).await?;
-        res_id = data_id_generator();
-        let mut chunks = producer.push(
-            &UpsertMetadata {
-                size: zip_file_metadata.len() as u32,
-                name: format!("refined_scan_{}", context.suffix.clone()),
-                data_type: "refined_scan_zip_v1".to_string(),
-                id: res_id.clone(),
-                is_new: true,
-                properties: HashMap::new(),
-            },
-        ).await?;
-        
-        let _ = chunks.next_chunk(&fs::read(&zip_path)?, false).await?;
-        while !producer.is_completed().await {
-            sleep(Duration::from_secs(3)).await;
-        }
-        producer.close().await;
-        std::fs::remove_dir_all(&context.task_folder)?;
     }
+    zip.finish()?;
+    let zip_file_metadata = fs::metadata(&zip_path)?;
+
+    let mut producer = datastore.upsert(domain_id.to_string()).await?;
+    let res_id = data_id_generator();
+    let mut chunks = producer.push(
+        &UpsertMetadata {
+            size: zip_file_metadata.len() as u32,
+            name: format!("refined_scan_{}", context.suffix.clone()),
+            data_type: "refined_scan_zip_v1".to_string(),
+            id: res_id.clone(),
+            is_new: true,
+            properties: HashMap::new(),
+        },
+    ).await?;
+    
+    let _ = chunks.next_chunk(&fs::read(&zip_path)?, false).await?;
+    while !producer.is_completed().await {
+        sleep(Duration::from_secs(3)).await;
+    }
+    producer.close().await;
+    std::fs::remove_dir_all(&context.task_folder)?;
     
     let output = LocalRefinementOutputV1 {
         result_ids: vec![res_id],
diff --git a/server/src/main.rs b/server/src/main.rs
index aa5181f..61dfdc1 100644
--- a/server/src/main.rs
+++ b/server/src/main.rs
@@ -1,8 +1,8 @@
 use std::{collections::HashMap, hash::Hash, sync::Arc, time::Duration};
 
 use async_trait::async_trait;
-use domain::{auth::{AuthClient, AuthError}, capabilities::public_key::PublicKeyStorage, cluster::DomainCluster, datastore::remote::RemoteDatastore};
-use networking::client::Client;
+use posemesh_domain::{auth::{AuthClient, AuthError}, capabilities::public_key::PublicKeyStorage, cluster::DomainCluster, datastore::remote::RemoteDatastore};
+use posemesh_networking::client::Client;
 use tokio::{self, select, signal::unix::{signal, SignalKind}};
 use futures::{lock::Mutex, StreamExt};
 use tracing_subscriber::{fmt, prelude::__tracing_subscriber_SubscriberExt, EnvFilter, Registry};
@@ -90,13 +90,34 @@ async fn main() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
         let mut c = n.client.clone();
         select! {
             Some((_, stream)) = local_refinement_v1_handler.next() => {
-                let _ = tokio::spawn(local_refinement::v1(base_path.clone(), stream, remote_storage.clone(), n.client.clone(), keys_loader.clone()));
+                static LOCAL_REFINEMENT_RUNNING: tokio::sync::Mutex<bool> = tokio::sync::Mutex::const_new(false);
+                let mut lock = LOCAL_REFINEMENT_RUNNING.lock().await;
+                if *lock {
+                    tracing::warn!("There is already a local refinement running, skipping...");
+                    continue;
+                }
+                *lock = true;
+                drop(lock);
+                let _guard = scopeguard::guard((), |_| {
+                    tokio::spawn(async move {
+                        let mut lock = LOCAL_REFINEMENT_RUNNING.lock().await;
+                        *lock = false;
+                    });
+                });
+                if let Err(e) = tokio::spawn(local_refinement::v1(base_path.clone(), stream, remote_storage.clone(), c, keys_loader.clone())).await {
+                    tracing::error!("Local Refinement Error: {}", e);
+                }
             }
             Some((_, stream)) = global_refinement_v1_handler.next() => {
-                let res = tokio::spawn(global_refinement::v1(base_path.clone(), stream, remote_storage.clone(), n.client.clone(), keys_loader.clone()));
-                if let Err(e) = res.await {
-                    tracing::error!("Error: {}", e);
-                }
+                let base_path = base_path.clone();
+                let remote_storage = remote_storage.clone();
+                let keys_loader = keys_loader.clone();
+                let c = c.clone();
+                tokio::spawn(async move {
+                    if let Err(e) = global_refinement::v1(base_path, stream, remote_storage, c, keys_loader).await {
+                        tracing::error!("Global Refinement Error: {}", e);
+                    }
+                });
             }
             _ = shutdown_signal() => {
                 c.cancel().await.unwrap_or_else(|e| {
diff --git a/server/src/utils.rs b/server/src/utils.rs
index 0661bf0..a7fe288 100644
--- a/server/src/utils.rs
+++ b/server/src/utils.rs
@@ -1,5 +1,5 @@
-use domain::protobuf::task::{Status, Task};
-use networking::client::TClient;
+use posemesh_domain::{datastore::common::DomainError, protobuf::task::{Status, Task}};
+use posemesh_networking::{client::TClient, libp2p::NetworkError};
 use quick_protobuf::serialize_into_vec;
 use tokio::{sync::watch, task::JoinHandle, time::interval};
 use std::{collections::HashSet, error::Error, fs, path::Path, process::Stdio, time::Duration};
@@ -176,8 +176,8 @@ pub async fn execute_python(params: Vec<&str>) -> Result<(), Box<dyn Error + Sen
 mod tests {
     use super::*;
     use async_trait::async_trait;
-    use networking::client::TClient;
-    use networking::libp2p::NetworkError;
+    use posemesh_networking::client::TClient;
+    use posemesh_networking::libp2p::NetworkError;
     use tokio::sync::watch;
     use tokio::time::{sleep, Duration};
     use std::sync::{Arc, Mutex};

From e1144e0ddca61e548b80429e805d2158a1fac943 Mon Sep 17 00:00:00 2001
From: Shuning <shuning@aukilabs.com>
Date: Tue, 3 Jun 2025 14:06:05 +0800
Subject: [PATCH 86/89] upgrade to more memory efficient downloading domain
 module

---
 server/Cargo.lock               |  14 +--
 server/Cargo.toml               |   4 +-
 server/src/global_refinement.rs |  37 ++++----
 server/src/local_refinement.rs  | 156 +++++++++++++++-----------------
 server/src/main.rs              |  16 +++-
 5 files changed, 113 insertions(+), 114 deletions(-)

diff --git a/server/Cargo.lock b/server/Cargo.lock
index f2662ac..33f5751 100644
--- a/server/Cargo.lock
+++ b/server/Cargo.lock
@@ -807,7 +807,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8d162beedaa69905488a8da94f5ac3edb4dd4788b732fadb7bd120b2625c1976"
 dependencies = [
  "data-encoding",
- "syn 1.0.109",
+ "syn 2.0.101",
 ]
 
 [[package]]
@@ -3102,7 +3102,7 @@ checksum = "350e9b48cbc6b0e028b0473b114454c6316e57336ee184ceab6e53f72c178b3e"
 [[package]]
 name = "posemesh-domain"
 version = "0.1.0"
-source = "git+https://github.com/aukilabs/posemesh.git?rev=06e93d86ea214b51087799c805797e50c41519ed#06e93d86ea214b51087799c805797e50c41519ed"
+source = "git+https://github.com/aukilabs/posemesh.git?rev=422341bd87588aad02bf8b18052031e97ef64a4d#422341bd87588aad02bf8b18052031e97ef64a4d"
 dependencies = [
  "async-timer",
  "async-trait",
@@ -3139,7 +3139,7 @@ dependencies = [
 [[package]]
 name = "posemesh-networking"
 version = "0.1.0"
-source = "git+https://github.com/aukilabs/posemesh.git?rev=06e93d86ea214b51087799c805797e50c41519ed#06e93d86ea214b51087799c805797e50c41519ed"
+source = "git+https://github.com/aukilabs/posemesh.git?rev=422341bd87588aad02bf8b18052031e97ef64a4d#422341bd87588aad02bf8b18052031e97ef64a4d"
 dependencies = [
  "async-trait",
  "cbindgen",
@@ -3171,7 +3171,7 @@ dependencies = [
 [[package]]
 name = "posemesh-runtime"
 version = "0.1.0"
-source = "git+https://github.com/aukilabs/posemesh.git?rev=06e93d86ea214b51087799c805797e50c41519ed#06e93d86ea214b51087799c805797e50c41519ed"
+source = "git+https://github.com/aukilabs/posemesh.git?rev=422341bd87588aad02bf8b18052031e97ef64a4d#422341bd87588aad02bf8b18052031e97ef64a4d"
 dependencies = [
  "once_cell",
  "tokio",
@@ -3180,7 +3180,7 @@ dependencies = [
 [[package]]
 name = "posemesh-utils"
 version = "0.1.0"
-source = "git+https://github.com/aukilabs/posemesh.git?rev=06e93d86ea214b51087799c805797e50c41519ed#06e93d86ea214b51087799c805797e50c41519ed"
+source = "git+https://github.com/aukilabs/posemesh.git?rev=422341bd87588aad02bf8b18052031e97ef64a4d#422341bd87588aad02bf8b18052031e97ef64a4d"
 dependencies = [
  "futures",
  "gloo-timers 0.3.0",
@@ -4187,9 +4187,9 @@ dependencies = [
 
 [[package]]
 name = "tempfile"
-version = "3.19.1"
+version = "3.20.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7437ac7763b9b123ccf33c338a5cc1bac6f69b45a136c19bdd8a65e3916435bf"
+checksum = "e8a64e3985349f2441a1a9ef0b853f869006c3855f2cda6862a94d26ebb9d6a1"
 dependencies = [
  "fastrand",
  "getrandom 0.3.2",
diff --git a/server/Cargo.toml b/server/Cargo.toml
index 67a380c..d5b5f13 100644
--- a/server/Cargo.toml
+++ b/server/Cargo.toml
@@ -4,9 +4,9 @@ version = "0.1.0"
 edition = "2021"
 
 [dependencies]
-posemesh-networking = { git = "https://github.com/aukilabs/posemesh.git", rev = "06e93d86ea214b51087799c805797e50c41519ed" }
+posemesh-networking = { git = "https://github.com/aukilabs/posemesh.git", rev = "422341bd87588aad02bf8b18052031e97ef64a4d" }
 futures = "0.3.30"
-posemesh-domain = { git = "https://github.com/aukilabs/posemesh.git", rev = "06e93d86ea214b51087799c805797e50c41519ed" }
+posemesh-domain = { git = "https://github.com/aukilabs/posemesh.git", rev = "422341bd87588aad02bf8b18052031e97ef64a4d" }
 tracing = "0.1.40"
 tracing-subscriber = "0.3.18"
 tokio = "1.44.0"
diff --git a/server/src/global_refinement.rs b/server/src/global_refinement.rs
index bc0e879..3156fc4 100644
--- a/server/src/global_refinement.rs
+++ b/server/src/global_refinement.rs
@@ -83,7 +83,7 @@ async fn initialize_task<S: AsyncStream, P: PublicKeyStorage, C: TClient + Clone
     stream: &mut S,
     c: &mut C,
     key_loader: P,
-) -> Result<(task::Task, String, TaskTokenClaim, PathBuf, PathBuf, PathBuf), Box<dyn std::error::Error + Send + Sync>> {
+) -> Result<(task::Task, String, TaskTokenClaim, PathBuf, PathBuf, PathBuf, PathBuf), Box<dyn std::error::Error + Send + Sync>> {
     let claim = handshake(stream, key_loader).await.expect("Failed to handshake");
     let job_id = claim.job_id.clone();
     c.subscribe(job_id.clone()).await.expect("Failed to subscribe to job");
@@ -115,31 +115,33 @@ async fn initialize_task<S: AsyncStream, P: PublicKeyStorage, C: TClient + Clone
     fs::create_dir_all(&dataset_path)?;
     fs::create_dir_all(&output_path)?;
 
-    Ok((task, job_id, claim, task_path, input_path, output_path))
+    Ok((task, job_id, claim, task_path, dataset_path, input_path, output_path))
 }
 
-fn download_local_refinement_result(input_path: &Path, dataset_path: &Path, name: &str, data: &[u8]) -> io::Result<String> {
+fn download_local_refinement_result(dataset_path: &Path, input_path: &Path, name: &str, data: &[u8]) -> io::Result<String> {
     let suffix = extract_suffix(name).map_err(|_| Error::new(ErrorKind::Other, "Failed to extract suffix"))?;
-    fs::create_dir_all(dataset_path.join(&suffix))?;
+    
+    fs::create_dir_all(dataset_path.join(&suffix))?; // This is because python may expect there is dataset folder even it is empty
     let path = input_path.join(&suffix).join("sfm");
+    fs::create_dir_all(&path)?;
     unzip_bytes(path, data)?;
     Ok(suffix)
 }
 
 #[derive(Clone)]
 struct ToZipArchive {
-    input_path: PathBuf,
     dataset_path: PathBuf,
+    input_path: PathBuf,
     size: u32,
     name: String,
     content: Vec<u8>,
     scan_ids: Arc<Mutex<Vec<String>>>,
 }
 impl ToZipArchive {
-    fn new(input_path: PathBuf, dataset_path: PathBuf) -> Self {
+    fn new(dataset_path: PathBuf, input_path: PathBuf) -> Self {
         Self {
-            input_path,
             dataset_path,
+            input_path,
             size: 0,
             name: "".to_string(),
             content: vec![],
@@ -158,7 +160,7 @@ impl AsyncWrite for ToZipArchive {
             if content.len() < 4 {
                 return Poll::Ready(Err(Error::new(ErrorKind::Other, "Content is too short")));
             }
-            let metadata = deserialize_from_slice::<Metadata>(&content[..4]).map_err(|_| Error::new(ErrorKind::Other, "Failed to deserialize metadata"))?;
+            let metadata = deserialize_from_slice::<Metadata>(&content[4..]).map_err(|_| Error::new(ErrorKind::Other, "Failed to deserialize metadata"))?;
             self.size = metadata.size;
             self.name = metadata.name;
             self.content = vec![];
@@ -176,7 +178,7 @@ impl AsyncWrite for ToZipArchive {
         if self.size > self.content.len() as u32 {
             return Poll::Ready(Ok(()));
         }
-        let suffix = download_local_refinement_result(&self.input_path, &self.dataset_path, &self.name, &self.content)?;
+        let suffix = download_local_refinement_result(&self.dataset_path, &self.input_path, &self.name, &self.content)?;
         self.scan_ids.lock().unwrap().push(suffix);
         self.content = vec![];
         self.size = 0;
@@ -197,10 +199,10 @@ async fn download_data(
     datastore: &mut RemoteDatastore,
     domain_id: &str,
     query: Query,
-    input_path: &Path,
     dataset_path: &Path,
+    input_path: &Path,
 ) -> Result<Vec<String>, Box<dyn std::error::Error + Send + Sync>> {
-    let to_file_system = ToZipArchive::new(input_path.to_path_buf(), dataset_path.to_path_buf());
+    let to_file_system = ToZipArchive::new(dataset_path.to_path_buf(), input_path.to_path_buf());
     let mut downloader = datastore.load(domain_id.to_string(), query, false, to_file_system.clone()).await?;
     downloader.wait_for_done().await?;
 
@@ -226,7 +228,7 @@ pub(crate) async fn v1<S: AsyncStream, P: PublicKeyStorage>(
     mut c: Client,
     key_loader: P,
 ) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
-    let (mut t, job_id, claim, task_path, input_path, output_path) =
+    let (mut t, job_id, claim, task_path, dataset_path, input_path, output_path) =
         initialize_task(&base_path, &mut stream, &mut c, key_loader).await?;
 
     let input = read_prefix_size_message::<task::GlobalRefinementInputV1, _>(&mut stream).await?;
@@ -248,8 +250,9 @@ pub(crate) async fn v1<S: AsyncStream, P: PublicKeyStorage>(
     });
 
     let domain_id = claim.domain_id.clone();
-    let res = run(&domain_id, &input_path, &task_path, &output_path, datastore, query, &job_id).await;
+    let res = run(&domain_id, &task_path, &dataset_path, &input_path, &output_path, datastore, query, &job_id).await;
 
+    tx.send(true).unwrap();
     if let Err(e) = res {
         t.status = task::Status::FAILED;
         t.output = Some(task::Any {
@@ -263,7 +266,6 @@ pub(crate) async fn v1<S: AsyncStream, P: PublicKeyStorage>(
         t.status = task::Status::DONE;
     }
 
-    tx.send(true).unwrap();
     let _ = heartbeat_handle.await;
 
     let buf = serialize_into_vec(&t)?;
@@ -276,14 +278,15 @@ pub(crate) async fn v1<S: AsyncStream, P: PublicKeyStorage>(
 
 async fn run(
     domain_id: &str,
-    input_path: &Path,
     task_path: &Path,
+    dataset_path: &Path,
+    input_path: &Path,
     output_path: &Path,
     mut datastore: RemoteDatastore,
     query: Query,
     job_id: &str,
 ) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
-    let scan_ids = download_data(&mut datastore, &domain_id, query, &input_path, &task_path.join("datasets")).await?;
+    let scan_ids = download_data(&mut datastore, &domain_id, query, dataset_path, input_path).await?;
         
     let mut params = vec![
         "-u",
@@ -299,6 +302,6 @@ async fn run(
 
     execute_python(params).await?;
     upload_results(&domain_id, output_path.to_path_buf(), &mut datastore).await?;
-    std::fs::remove_dir_all(output_path)?;
+    std::fs::remove_dir_all(task_path)?;
     Ok(())
 }
diff --git a/server/src/local_refinement.rs b/server/src/local_refinement.rs
index 1536d78..b42a1a7 100644
--- a/server/src/local_refinement.rs
+++ b/server/src/local_refinement.rs
@@ -13,7 +13,6 @@ const REFINED_SCAN_NAME: &str = "refined_scan_";
 #[derive(Clone)]
 struct ToFileSystem {
     size: u32,
-    content: Option<Vec<u8>>,
     path: Option<PathBuf>,
     input_folder: PathBuf,
     count: Arc<Mutex<u32>>,
@@ -26,7 +25,6 @@ impl ToFileSystem {
             input_folder,
             path: None,
             size: 0,
-            content: None,
             count: Arc::new(Mutex::new(0)),
             written: 0,
             refined_res_id: Arc::new(Mutex::new(None)),
@@ -48,93 +46,82 @@ impl AsyncWrite for ToFileSystem {
             if content.len() < 4 {
                 return Poll::Ready(Err(Error::new(ErrorKind::Other, "Content is too short")));
             }
-            match deserialize_from_slice::<Metadata>(&content[..4]) {
-                Ok(metadata) => {
-                    if let Some(_) = metadata.name.strip_prefix("refined_scan_") {
-                        self.refined_res_id.lock().unwrap().insert(metadata.id.clone());
-                        return Poll::Ready(Err(Error::new(ErrorKind::Other, format!("RefinedAlready({})", metadata.id))));
-                    }
-                    let filename = match metadata.name.as_str() {
-                        "Manifest.json" => "Manifest.json".to_string(),
-                        "FeaturePoints.ply" => "FeaturePoints.ply".to_string(),
-                        "ARposes.csv" => "ARposes.csv".to_string(),
-                        "PortalDetections.csv" => "PortalDetections.csv".to_string(),
-                        "CameraIntrinsics.csv" => "CameraIntrinsics.csv".to_string(),
-                        "Frames.csv" => "Frames.csv".to_string(),
-                        "Gyro.csv" => "Gyro.csv".to_string(),
-                        "Accel.csv" => "Accel.csv".to_string(),
-                        "gyro_accel.csv" => "gyro_accel.csv".to_string(),
-                        "Frames.mp4" => "Frames.mp4".to_string(),
+            let metadata = deserialize_from_slice::<Metadata>(&content[4..]).map_err(|e| Error::other(e))?;
+            if let Some(_) = metadata.name.strip_prefix(REFINED_SCAN_NAME) {
+                tracing::warn!("found refined result");
+                let _ = self.refined_res_id.lock().unwrap().insert(metadata.id.clone());
+                return Poll::Ready(Err(Error::new(ErrorKind::Other, format!("RefinedAlready({})", metadata.id))));
+            }
+            let filename = match metadata.name.as_str() {
+                "Manifest.json" => "Manifest.json".to_string(),
+                "FeaturePoints.ply" => "FeaturePoints.ply".to_string(),
+                "ARposes.csv" => "ARposes.csv".to_string(),
+                "PortalDetections.csv" => "PortalDetections.csv".to_string(),
+                "CameraIntrinsics.csv" => "CameraIntrinsics.csv".to_string(),
+                "Frames.csv" => "Frames.csv".to_string(),
+                "Gyro.csv" => "Gyro.csv".to_string(),
+                "Accel.csv" => "Accel.csv".to_string(),
+                "gyro_accel.csv" => "gyro_accel.csv".to_string(),
+                "Frames.mp4" => "Frames.mp4".to_string(),
+                _ => {
+                    match metadata.data_type.as_str() {
+                        "dmt_manifest_json" => "Manifest.json".to_string(),
+                        "dmt_featurepoints_ply" | "dmt_pointcloud_ply" => "FeaturePoints.ply".to_string(),
+                        "dmt_arposes_csv" => "ARposes.csv".to_string(),
+                        "dmt_portal_detections_csv" | "dmt_observations_csv" => "PortalDetections.csv".to_string(),
+                        "dmt_intrinsics_csv" | "dmt_cameraintrinsics_csv" => "CameraIntrinsics.csv".to_string(),
+                        "dmt_frames_csv" => "Frames.csv".to_string(),
+                        "dmt_gyro_csv" => "Gyro.csv".to_string(),
+                        "dmt_accel_csv" => "Accel.csv".to_string(),
+                        "dmt_gyroaccel_csv" => "gyro_accel.csv".to_string(),
+                        "dmt_recording_mp4" => "Frames.mp4".to_string(),
                         _ => {
-                            match metadata.data_type.as_str() {
-                                "dmt_manifest_json" => "Manifest.json".to_string(),
-                                "dmt_featurepoints_ply" | "dmt_pointcloud_ply" => "FeaturePoints.ply".to_string(),
-                                "dmt_arposes_csv" => "ARposes.csv".to_string(),
-                                "dmt_portal_detections_csv" | "dmt_observations_csv" => "PortalDetections.csv".to_string(),
-                                "dmt_intrinsics_csv" | "dmt_cameraintrinsics_csv" => "CameraIntrinsics.csv".to_string(),
-                                "dmt_frames_csv" => "Frames.csv".to_string(),
-                                "dmt_gyro_csv" => "Gyro.csv".to_string(),
-                                "dmt_accel_csv" => "Accel.csv".to_string(),
-                                "dmt_gyroaccel_csv" => "gyro_accel.csv".to_string(),
-                                "dmt_recording_mp4" => "Frames.mp4".to_string(),
-                                _ => {
-                                    tracing::info!("unknown domain data type: {}", metadata.data_type);
-                                    format!("{}.{}", metadata.name, metadata.data_type)
-                                }
-                            }
+                            tracing::warn!("unknown domain data type: {}", metadata.data_type);
+                            return Poll::Ready(Ok(0));
                         }
-                    };
-
-                    self.path = Some(self.input_folder.join(&filename));
-                    self.written = 0;
-                    return Poll::Ready(Ok(content.len()));
-                }
-                Err(_) => {
-                    return Poll::Ready(Err(Error::new(ErrorKind::Other, "Failed to deserialize metadata")));
+                    }
                 }
-            }
+            };
+            tracing::debug!("write to: {}", self.input_folder.join(&filename).to_str().unwrap());
+
+            self.path = Some(self.input_folder.join(&filename));
+            self.written = 0;
+            self.size = metadata.size;
+            return Poll::Ready(Ok(content.len()));
         } else {
-            self.content
-                .get_or_insert_with(Vec::new)
-                .extend_from_slice(content);
+            self.written += content.len() as u32;
+            let path = self.path.as_ref().unwrap();
+            let mut file = OpenOptions::new()
+                .create(true)
+                .append(true)
+                .open(path)?;
+            file.write_all(&content)?;
+            file.flush()?;
+            if self.written == self.size {
+                self.path = None;
+                self.written = 0;
+                self.size = 0;
+                let mut count = self.count.lock().unwrap();
+                *count += 1;
+            }
             return Poll::Ready(Ok(content.len()));
         }
     }
     
-    fn poll_flush(mut self: Pin<&mut Self>, _: &mut Context<'_>) -> Poll<Result<(), Error>> {
-        if self.path.is_none() {
-            return Poll::Ready(Ok(()));
-        }
-        let path = self.path.as_ref().unwrap();
-        let mut file = OpenOptions::new()
-            .create(true)
-            .append(true)
-            .open(path)?;
-        let content = self.content.take().unwrap_or_default();
-        file.write_all(&content)?;
-
-        self.written += content.len() as u32;
-        if self.written == self.size {
-            self.path = None;
-            self.written = 0;
-            let mut count = self.count.lock().unwrap();
-            *count += 1;
-        }
-        
+    fn poll_flush(self: Pin<&mut Self>, _: &mut Context<'_>) -> Poll<Result<(), Error>> {
         Poll::Ready(Ok(()))
     }
 
     fn poll_close(mut self: Pin<&mut Self>, _: &mut Context<'_>) -> Poll<Result<(), Error>> {
-        self.content = None;
         self.path = None;
         Poll::Ready(Ok(()))
     }
 }
 
-pub(crate) async fn v1<S: AsyncStream, P: PublicKeyStorage, C: TClient + Clone + Send + Sync + 'static>(base_path: String, mut stream: S, datastore: RemoteDatastore, mut c: C, key_loader: P) {
-    let claim = handshake(&mut stream, key_loader).await.expect("Failed to handshake");
+pub(crate) async fn v1<S: AsyncStream, P: PublicKeyStorage, C: TClient + Clone + Send + Sync + 'static>(base_path: String, mut stream: S, datastore: RemoteDatastore, mut c: C, key_loader: P) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    let claim = handshake(&mut stream, key_loader).await?;
     let job_id = claim.job_id.clone();
-    c.subscribe(job_id.clone()).await.expect("Failed to subscribe to job");
+    c.subscribe(job_id.clone()).await?;
     let t = &mut task::Task {
         name: claim.task_name.clone(),
         receiver: Some(claim.receiver.clone()),
@@ -150,8 +137,8 @@ pub(crate) async fn v1<S: AsyncStream, P: PublicKeyStorage, C: TClient + Clone +
     match res {
         Ok(context) => {
             t.status = task::Status::STARTED;
-            let message = serialize_into_vec(t).expect("failed to serialize task update");
-            c.publish(job_id.clone(), message).await.expect("failed to publish task update");
+            let message = serialize_into_vec(t)?;
+            c.publish(job_id.clone(), message).await?;
             let (tx, rx) = watch::channel(false);
 
             let heartbeat_handle = health(t.clone(), c.clone(), &job_id, rx).await;
@@ -161,6 +148,7 @@ pub(crate) async fn v1<S: AsyncStream, P: PublicKeyStorage, C: TClient + Clone +
             });
 
             let output = run(&claim.domain_id, &job_id, &claim.task_name, context, datastore).await;
+            let _ = tx.send(true); // stop heartbeat task
             match output {
                 Ok(output) => {
                     t.output = Some(task::Any {
@@ -180,8 +168,6 @@ pub(crate) async fn v1<S: AsyncStream, P: PublicKeyStorage, C: TClient + Clone +
                     });
                 }
             }
-
-            tx.send(true).unwrap();
             let _ = heartbeat_handle.await;
         }
         Err(e) => {
@@ -195,10 +181,12 @@ pub(crate) async fn v1<S: AsyncStream, P: PublicKeyStorage, C: TClient + Clone +
             });
         }
     }
-    let message = serialize_into_vec(t).expect("failed to serialize task update");
-    c.publish(job_id.clone(), message).await.expect("failed to publish task update");
+    let message = serialize_into_vec(t)?;
+    c.publish(job_id.clone(), message).await?;
 
     tracing::info!("finished local refinement {}", job_id);
+    
+    Ok(())
 }
 
 struct LocalRefinementContext {
@@ -259,14 +247,12 @@ async fn start<S: AsyncStream>(base_path: String, job_id: &str, task_name: &str,
 async fn run(domain_id: &str, job_id: &str, task_name: &str, context: LocalRefinementContext, mut datastore: RemoteDatastore) -> Result<LocalRefinementOutputV1, Box<dyn std::error::Error + Send + Sync>> { 
     let query = context.query;
 
-    let writer = ToFileSystem::new(context.output_folder.clone());
-    let mut downloader = datastore.load(domain_id.to_string(), Query {
-        metadata_only: false,
-        ..query
-    }, false, writer.clone()).await?;
+    let writer = ToFileSystem::new(context.input_folder.clone());
+    let mut downloader = datastore.load(domain_id.to_string(), query, false, writer.clone()).await?;
     if let Err(e) = downloader.wait_for_done().await {
         let refined_res_id = writer.refined_res_id();
         if refined_res_id.is_some() {
+            let _ = fs::remove_dir_all(context.input_folder);
             return Ok(LocalRefinementOutputV1 {
                 result_ids: vec![refined_res_id.unwrap()],
             });
@@ -274,6 +260,10 @@ async fn run(domain_id: &str, job_id: &str, task_name: &str, context: LocalRefin
         return Err(Box::new(e));
     }
 
+    if writer.count() == 0 {
+        return Err("No data downloaded".into());
+    }
+
     tracing::info!("Finished downloading {} data for {}", writer.count(), task_name);
 
     write_scan_data_summary(context.input_folder.as_path(), context.task_folder.as_path().join("scan_data_summary.json").as_path())?;
@@ -342,7 +332,7 @@ async fn run(domain_id: &str, job_id: &str, task_name: &str, context: LocalRefin
         sleep(Duration::from_secs(3)).await;
     }
     producer.close().await;
-    std::fs::remove_dir_all(&context.task_folder)?;
+    std::fs::remove_dir_all(&context.input_folder)?;
     
     let output = LocalRefinementOutputV1 {
         result_ids: vec![res_id],
diff --git a/server/src/main.rs b/server/src/main.rs
index 61dfdc1..ad592da 100644
--- a/server/src/main.rs
+++ b/server/src/main.rs
@@ -1,4 +1,4 @@
-use std::{collections::HashMap, hash::Hash, sync::Arc, time::Duration};
+use std::{collections::HashMap, sync::Arc, time::Duration};
 
 use async_trait::async_trait;
 use posemesh_domain::{auth::{AuthClient, AuthError}, capabilities::public_key::PublicKeyStorage, cluster::DomainCluster, datastore::remote::RemoteDatastore};
@@ -70,7 +70,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     let private_key_path = format!("{}/pkey", base_path);
     let subscriber = Registry::default()
             .with(fmt::layer().with_file(true).with_line_number(true))
-            .with(EnvFilter::new("info"));
+            .with(EnvFilter::try_new(std::env::var("RUST_LOG").unwrap_or_else(|_| "info".to_string())).unwrap_or_else(|_| EnvFilter::new("info")));
     tracing::subscriber::set_global_default(subscriber).expect("failed to set subscriber");
 
     let domain_cluster = DomainCluster::new(domain_manager.clone(), name, false, port, false, false, None, Some(private_key_path), vec![domain_manager.clone()]);
@@ -104,9 +104,15 @@ async fn main() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
                         *lock = false;
                     });
                 });
-                if let Err(e) = tokio::spawn(local_refinement::v1(base_path.clone(), stream, remote_storage.clone(), c, keys_loader.clone())).await {
-                    tracing::error!("Local Refinement Error: {}", e);
-                }
+                let base_path = base_path.clone();
+                let remote_storage = remote_storage.clone();
+                let c = c.clone();
+                let keys_loader = keys_loader.clone();
+                tokio::spawn(async move {
+                    if let Err(e) = local_refinement::v1(base_path, stream, remote_storage, c, keys_loader).await {
+                        tracing::error!("Local refinement error: {}", e);
+                    }
+                });
             }
             Some((_, stream)) = global_refinement_v1_handler.next() => {
                 let base_path = base_path.clone();

From 1e155a9acb6694a7c298d3471be855e9c574241b Mon Sep 17 00:00:00 2001
From: Shuning <shuning@aukilabs.com>
Date: Mon, 9 Jun 2025 11:11:39 +0800
Subject: [PATCH 87/89] - upgraded to new domain module so it joins a domain
 cluster - fix how we check if the scan has been refined already - added
 capacity to local refinement, and once it can't take more requests, it sends
 Retry back to domain manager

---
 server/Cargo.lock              | 28 ++++++++++---
 server/Cargo.toml              |  4 +-
 server/src/local_refinement.rs | 73 ++++++++++++++++++++++++++++++----
 server/src/main.rs             | 47 ++++++++++++----------
 4 files changed, 115 insertions(+), 37 deletions(-)

diff --git a/server/Cargo.lock b/server/Cargo.lock
index 33f5751..f4a78f4 100644
--- a/server/Cargo.lock
+++ b/server/Cargo.lock
@@ -1911,6 +1911,7 @@ dependencies = [
  "libp2p-mdns",
  "libp2p-metrics",
  "libp2p-noise",
+ "libp2p-ping",
  "libp2p-quic",
  "libp2p-relay",
  "libp2p-swarm",
@@ -2171,6 +2172,7 @@ dependencies = [
  "libp2p-identify",
  "libp2p-identity",
  "libp2p-kad",
+ "libp2p-ping",
  "libp2p-relay",
  "libp2p-swarm",
  "pin-project",
@@ -2203,6 +2205,23 @@ dependencies = [
  "zeroize",
 ]
 
+[[package]]
+name = "libp2p-ping"
+version = "0.45.0"
+source = "git+https://github.com/aukilabs/rust-libp2p.git?branch=fix%2Fgossipsub-wasm#9b97c007650c3a3e29cf90b896a159fc3e0e0b5b"
+dependencies = [
+ "either",
+ "futures",
+ "futures-timer",
+ "libp2p-core",
+ "libp2p-identity",
+ "libp2p-swarm",
+ "rand 0.8.5",
+ "tracing",
+ "void",
+ "web-time",
+]
+
 [[package]]
 name = "libp2p-quic"
 version = "0.11.1"
@@ -3102,7 +3121,7 @@ checksum = "350e9b48cbc6b0e028b0473b114454c6316e57336ee184ceab6e53f72c178b3e"
 [[package]]
 name = "posemesh-domain"
 version = "0.1.0"
-source = "git+https://github.com/aukilabs/posemesh.git?rev=422341bd87588aad02bf8b18052031e97ef64a4d#422341bd87588aad02bf8b18052031e97ef64a4d"
+source = "git+https://github.com/aukilabs/posemesh.git?rev=0176b1d0998d13875580917b6708dfe9ce4b27ed#0176b1d0998d13875580917b6708dfe9ce4b27ed"
 dependencies = [
  "async-timer",
  "async-trait",
@@ -3113,7 +3132,6 @@ dependencies = [
  "hex",
  "jsonwebtoken",
  "libp2p",
- "mockall",
  "pb-rs",
  "posemesh-networking",
  "posemesh-runtime",
@@ -3139,7 +3157,7 @@ dependencies = [
 [[package]]
 name = "posemesh-networking"
 version = "0.1.0"
-source = "git+https://github.com/aukilabs/posemesh.git?rev=422341bd87588aad02bf8b18052031e97ef64a4d#422341bd87588aad02bf8b18052031e97ef64a4d"
+source = "git+https://github.com/aukilabs/posemesh.git?rev=0176b1d0998d13875580917b6708dfe9ce4b27ed#0176b1d0998d13875580917b6708dfe9ce4b27ed"
 dependencies = [
  "async-trait",
  "cbindgen",
@@ -3171,7 +3189,7 @@ dependencies = [
 [[package]]
 name = "posemesh-runtime"
 version = "0.1.0"
-source = "git+https://github.com/aukilabs/posemesh.git?rev=422341bd87588aad02bf8b18052031e97ef64a4d#422341bd87588aad02bf8b18052031e97ef64a4d"
+source = "git+https://github.com/aukilabs/posemesh.git?rev=0176b1d0998d13875580917b6708dfe9ce4b27ed#0176b1d0998d13875580917b6708dfe9ce4b27ed"
 dependencies = [
  "once_cell",
  "tokio",
@@ -3180,7 +3198,7 @@ dependencies = [
 [[package]]
 name = "posemesh-utils"
 version = "0.1.0"
-source = "git+https://github.com/aukilabs/posemesh.git?rev=422341bd87588aad02bf8b18052031e97ef64a4d#422341bd87588aad02bf8b18052031e97ef64a4d"
+source = "git+https://github.com/aukilabs/posemesh.git?rev=0176b1d0998d13875580917b6708dfe9ce4b27ed#0176b1d0998d13875580917b6708dfe9ce4b27ed"
 dependencies = [
  "futures",
  "gloo-timers 0.3.0",
diff --git a/server/Cargo.toml b/server/Cargo.toml
index d5b5f13..7d324f8 100644
--- a/server/Cargo.toml
+++ b/server/Cargo.toml
@@ -4,9 +4,9 @@ version = "0.1.0"
 edition = "2021"
 
 [dependencies]
-posemesh-networking = { git = "https://github.com/aukilabs/posemesh.git", rev = "422341bd87588aad02bf8b18052031e97ef64a4d" }
+posemesh-networking = { git = "https://github.com/aukilabs/posemesh.git", rev = "0176b1d0998d13875580917b6708dfe9ce4b27ed" }
 futures = "0.3.30"
-posemesh-domain = { git = "https://github.com/aukilabs/posemesh.git", rev = "422341bd87588aad02bf8b18052031e97ef64a4d" }
+posemesh-domain = { git = "https://github.com/aukilabs/posemesh.git", rev = "0176b1d0998d13875580917b6708dfe9ce4b27ed" }
 tracing = "0.1.40"
 tracing-subscriber = "0.3.18"
 tokio = "1.44.0"
diff --git a/server/src/local_refinement.rs b/server/src/local_refinement.rs
index b42a1a7..d7f1ead 100644
--- a/server/src/local_refinement.rs
+++ b/server/src/local_refinement.rs
@@ -1,8 +1,8 @@
-use std::{collections::HashMap, fs::{self, OpenOptions}, io::{Error, ErrorKind}, path::{Path, PathBuf}, pin::Pin, sync::{Arc, Mutex}, task::{Context, Poll}, time::Duration};
+use std::{collections::HashMap, fs::{self, OpenOptions}, io::{Error, ErrorKind}, path::{Path, PathBuf}, pin::Pin, sync::{atomic::{AtomicI32, Ordering}, Arc, Mutex}, task::{Context, Poll}, time::Duration};
 use posemesh_domain::{auth::handshake, capabilities::public_key::PublicKeyStorage, datastore::{common::{data_id_generator, Datastore}, remote::RemoteDatastore}, message::read_prefix_size_message, protobuf::{domain_data::{Metadata, Query, UpsertMetadata},task::{self, LocalRefinementInputV1, LocalRefinementOutputV1}}};
 use posemesh_networking::{client::TClient, AsyncStream};
 use quick_protobuf::{deserialize_from_slice, serialize_into_vec};
-use futures::{AsyncWrite, StreamExt};
+use futures::AsyncWrite;
 use tokio::{sync::watch, time::sleep};
 use std::io::{BufReader, Read, Write};
 use zip::write::SimpleFileOptions;
@@ -10,6 +10,40 @@ use crate::utils::{execute_python, health, write_scan_data_summary};
 
 const REFINED_SCAN_NAME: &str = "refined_scan_";
 
+#[derive(Clone)]
+struct FindScanResult {
+    id: Arc<Mutex<String>>,
+}
+impl FindScanResult {
+    fn new() -> Self {
+        Self {
+            id: Arc::new(Mutex::new("".to_string()))
+        }
+    }
+    fn scan_id(&self) -> String {
+        self.id.lock().unwrap().to_string()
+    }
+}
+impl AsyncWrite for FindScanResult {
+    fn poll_write(self: Pin<&mut Self>, _: &mut Context<'_>, content: &[u8]) -> Poll<Result<usize, Error>> {
+        if content.len() < 4 {
+            return Poll::Ready(Err(Error::new(ErrorKind::Other, "Content is too short")));
+        }
+        let metadata = deserialize_from_slice::<Metadata>(&content[4..]).map_err(|e| Error::other(e))?;
+        let mut id = self.id.lock().unwrap();
+        *id = metadata.id;
+        Poll::Ready(Ok(content.len()))
+    }
+
+    fn poll_flush(self: Pin<&mut Self>, _: &mut Context<'_>) -> Poll<Result<(), Error>> {
+        Poll::Ready(Ok(()))
+    }
+
+    fn poll_close(self: Pin<&mut Self>, _: &mut Context<'_>) -> Poll<Result<(), Error>> {
+        Poll::Ready(Ok(()))
+    }
+}
+
 #[derive(Clone)]
 struct ToFileSystem {
     size: u32,
@@ -47,11 +81,6 @@ impl AsyncWrite for ToFileSystem {
                 return Poll::Ready(Err(Error::new(ErrorKind::Other, "Content is too short")));
             }
             let metadata = deserialize_from_slice::<Metadata>(&content[4..]).map_err(|e| Error::other(e))?;
-            if let Some(_) = metadata.name.strip_prefix(REFINED_SCAN_NAME) {
-                tracing::warn!("found refined result");
-                let _ = self.refined_res_id.lock().unwrap().insert(metadata.id.clone());
-                return Poll::Ready(Err(Error::new(ErrorKind::Other, format!("RefinedAlready({})", metadata.id))));
-            }
             let filename = match metadata.name.as_str() {
                 "Manifest.json" => "Manifest.json".to_string(),
                 "FeaturePoints.ply" => "FeaturePoints.ply".to_string(),
@@ -118,7 +147,7 @@ impl AsyncWrite for ToFileSystem {
     }
 }
 
-pub(crate) async fn v1<S: AsyncStream, P: PublicKeyStorage, C: TClient + Clone + Send + Sync + 'static>(base_path: String, mut stream: S, datastore: RemoteDatastore, mut c: C, key_loader: P) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+pub(crate) async fn v1<S: AsyncStream, P: PublicKeyStorage, C: TClient + Clone + Send + Sync + 'static>(left_slots: Arc<AtomicI32>, base_path: String, mut stream: S, datastore: RemoteDatastore, mut c: C, key_loader: P) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     let claim = handshake(&mut stream, key_loader).await?;
     let job_id = claim.job_id.clone();
     c.subscribe(job_id.clone()).await?;
@@ -132,6 +161,20 @@ pub(crate) async fn v1<S: AsyncStream, P: PublicKeyStorage, C: TClient + Clone +
         job_id: job_id.clone(),
         output: None,
     };
+    let current_slots = left_slots.fetch_sub(1, Ordering::SeqCst);
+    let _guard = scopeguard::guard((), |_| {
+        left_slots.fetch_add(1, Ordering::SeqCst);
+    });
+
+    if current_slots < 0 {
+        tracing::warn!("Reconstruction has no vacancy, skipping this local refinement request");
+        t.receiver = None;
+        t.status = task::Status::RETRY;
+        t.access_token = None;
+        let message = serialize_into_vec(t)?;
+        c.publish(job_id.clone(), message).await?;
+        return Ok(());
+    }
 
     let res = start(base_path, &job_id, &claim.task_name, &mut stream).await;
     match res {
@@ -247,6 +290,20 @@ async fn start<S: AsyncStream>(base_path: String, job_id: &str, task_name: &str,
 async fn run(domain_id: &str, job_id: &str, task_name: &str, context: LocalRefinementContext, mut datastore: RemoteDatastore) -> Result<LocalRefinementOutputV1, Box<dyn std::error::Error + Send + Sync>> { 
     let query = context.query;
 
+    let find_refine_result = FindScanResult::new();
+    let mut find_refine_query = Query::default();
+    find_refine_query.metadata_only = true;
+    find_refine_query.name_regexp = Some(format!("{}.*", REFINED_SCAN_NAME));
+    let mut finding = datastore.load(domain_id.to_string(), find_refine_query, false, find_refine_result.clone()).await?;
+    if let Err(e) = finding.wait_for_done().await {
+        return Err(Box::new(e));
+    }
+    if find_refine_result.scan_id() != "".to_string() {
+        return Ok(LocalRefinementOutputV1 {
+            result_ids: vec![find_refine_result.scan_id()],
+        });
+    }
+
     let writer = ToFileSystem::new(context.input_folder.clone());
     let mut downloader = datastore.load(domain_id.to_string(), query, false, writer.clone()).await?;
     if let Err(e) = downloader.wait_for_done().await {
diff --git a/server/src/main.rs b/server/src/main.rs
index ad592da..2e38a04 100644
--- a/server/src/main.rs
+++ b/server/src/main.rs
@@ -1,7 +1,7 @@
-use std::{collections::HashMap, sync::Arc, time::Duration};
+use std::{collections::HashMap, sync::{atomic::{AtomicI32, Ordering}, Arc}, time::Duration};
 
 use async_trait::async_trait;
-use posemesh_domain::{auth::{AuthClient, AuthError}, capabilities::public_key::PublicKeyStorage, cluster::DomainCluster, datastore::remote::RemoteDatastore};
+use posemesh_domain::{auth::{AuthClient, AuthError}, capabilities::public_key::PublicKeyStorage, cluster::DomainCluster, datastore::remote::RemoteDatastore, protobuf::discovery::Capability};
 use posemesh_networking::client::Client;
 use tokio::{self, select, signal::unix::{signal, SignalKind}};
 use futures::{lock::Mutex, StreamExt};
@@ -51,6 +51,7 @@ impl PublicKeyLoader {
         auth_clients.clear();
     }
 }
+
 /*
     * This is a simple example of a reconstruction node. It will connect to a set of bootstraps and execute reconstruction jobs.
     * Usage: cargo run <port> <name> <domain_manager_addr> 
@@ -60,7 +61,7 @@ impl PublicKeyLoader {
 async fn main() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     let args: Vec<String> = std::env::args().collect();
     if args.len() < 4 {
-        println!("Usage: {} <port> <name> <domain_manager_addr>", args[0]);
+        println!("Usage: {} <port> <name> <domain_manager_addr> <initial_slot>", args[0]);
         return Ok(());
     }
     let port = args[1].parse::<u16>().unwrap();
@@ -68,16 +69,31 @@ async fn main() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     let base_path = format!("./volume/{}", name);
     let domain_manager = args[3].clone();
     let private_key_path = format!("{}/pkey", base_path);
+    let mut slots: i32 = 1;
+    if args.len() == 5 {
+        slots = args[4].parse::<i32>().unwrap();
+    }
+    let left_slots: Arc<AtomicI32> = Arc::new(AtomicI32::new(slots));
     let subscriber = Registry::default()
             .with(fmt::layer().with_file(true).with_line_number(true))
             .with(EnvFilter::try_new(std::env::var("RUST_LOG").unwrap_or_else(|_| "info".to_string())).unwrap_or_else(|_| EnvFilter::new("info")));
     tracing::subscriber::set_global_default(subscriber).expect("failed to set subscriber");
 
-    let domain_cluster = DomainCluster::new(domain_manager.clone(), name, false, port, false, false, None, Some(private_key_path), vec![domain_manager.clone()]);
+
+    let local_refinement_capability = Capability {
+        endpoint: "/local-refinement/v1".to_string(),
+        capacity: slots,
+    };
+    let global_refinement_capability = Capability {
+        endpoint: "/global-refinement/v1".to_string(),
+        capacity: -1, // unlimited capacity
+    };
+    let mut domain_cluster = DomainCluster::join(&domain_manager, &name, false, port, false, false, None, Some(private_key_path), vec![domain_manager.clone()]).await?;
     let domain_manager_id = domain_cluster.manager_id.clone();
-    let mut n = domain_cluster.peer.clone();
-    let mut local_refinement_v1_handler = n.client.set_stream_handler("/local-refinement/v1".to_string()).await.unwrap();
-    let mut global_refinement_v1_handler = n.client.set_stream_handler("/global-refinement/v1".to_string()).await.unwrap();
+    let n = domain_cluster.peer.clone();
+    let mut handlers = domain_cluster.with_capabilities(&vec![local_refinement_capability, global_refinement_capability]).await?;
+    let mut local_refinement_v1_handler = handlers.remove(0);
+    let mut global_refinement_v1_handler = handlers.remove(0);
     let remote_storage = RemoteDatastore::new(domain_cluster);
     let keys_loader = PublicKeyLoader {
         auth_clients: Arc::new(Mutex::new(HashMap::new())),
@@ -90,26 +106,13 @@ async fn main() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
         let mut c = n.client.clone();
         select! {
             Some((_, stream)) = local_refinement_v1_handler.next() => {
-                static LOCAL_REFINEMENT_RUNNING: tokio::sync::Mutex<bool> = tokio::sync::Mutex::const_new(false);
-                let mut lock = LOCAL_REFINEMENT_RUNNING.lock().await;
-                if *lock {
-                    tracing::warn!("There is already a local refinement running, skipping...");
-                    continue;
-                }
-                *lock = true;
-                drop(lock);
-                let _guard = scopeguard::guard((), |_| {
-                    tokio::spawn(async move {
-                        let mut lock = LOCAL_REFINEMENT_RUNNING.lock().await;
-                        *lock = false;
-                    });
-                });
                 let base_path = base_path.clone();
                 let remote_storage = remote_storage.clone();
                 let c = c.clone();
                 let keys_loader = keys_loader.clone();
+                let left_slots = left_slots.clone();
                 tokio::spawn(async move {
-                    if let Err(e) = local_refinement::v1(base_path, stream, remote_storage, c, keys_loader).await {
+                    if let Err(e) = local_refinement::v1(left_slots, base_path, stream, remote_storage, c, keys_loader).await {
                         tracing::error!("Local refinement error: {}", e);
                     }
                 });

From 01d06c6a015867e48f5bd23e4a572ae2e6314e72 Mon Sep 17 00:00:00 2001
From: Shuning <shuning@aukilabs.com>
Date: Mon, 9 Jun 2025 11:12:08 +0800
Subject: [PATCH 88/89] added suffix to global refined results

---
 server/src/global_refinement.rs | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/server/src/global_refinement.rs b/server/src/global_refinement.rs
index 3156fc4..1a6867e 100644
--- a/server/src/global_refinement.rs
+++ b/server/src/global_refinement.rs
@@ -9,7 +9,11 @@ use zip::ZipArchive;
 
 use crate::utils::{execute_python, health};
 
-async fn upload_results(domain_id: &str, output_path: PathBuf, datastore: &mut RemoteDatastore) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+fn add_suffix(name: &str, suffix: &str) -> String {
+    format!("{name}_{suffix}")
+}
+
+async fn upload_results(domain_id: &str, job_id: &str, output_path: PathBuf, datastore: &mut RemoteDatastore) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     let mut uploader = datastore.upsert(domain_id.to_string()).await?;
     // open output_path and upload to datastore
     let files = fs::read_dir(output_path)?;
@@ -18,7 +22,7 @@ async fn upload_results(domain_id: &str, output_path: PathBuf, datastore: &mut R
         let path = file.path();
         let metadata: UpsertMetadata = match file.file_name().to_str().unwrap() {
             "refined_manifest.json" => UpsertMetadata {
-                name: "refined_manifest".to_string(),
+                name: add_suffix("refined_manifest", job_id),
                 data_type: "refined_manifest_json".to_string(),
                 size: file.metadata()?.len() as u32,
                 id: data_id_generator(),
@@ -26,7 +30,7 @@ async fn upload_results(domain_id: &str, output_path: PathBuf, datastore: &mut R
                 is_new: true,
             },
             "RefinedPointCloud.ply" => UpsertMetadata {
-                name: "refined_pointcloud".to_string(),
+                name: add_suffix("refined_pointcloud", job_id),
                 data_type: "refined_pointcloud_ply".to_string(),
                 size: file.metadata()?.len() as u32,
                 id: data_id_generator(),
@@ -34,7 +38,7 @@ async fn upload_results(domain_id: &str, output_path: PathBuf, datastore: &mut R
                 is_new: true
             },
             "BasicStitchPointCloud.ply" => UpsertMetadata {
-                name: "unrefined_pointcloud".to_string(),
+                name: add_suffix("unrefined_pointcloud", job_id),
                 data_type: "unrefined_pointcloud_ply".to_string(),
                 size: file.metadata()?.len() as u32,
                 id: data_id_generator(),
@@ -301,7 +305,7 @@ async fn run(
     params.extend(scan_ids.iter().map(|s| s.as_str()));
 
     execute_python(params).await?;
-    upload_results(&domain_id, output_path.to_path_buf(), &mut datastore).await?;
+    upload_results(&domain_id, job_id, output_path.to_path_buf(), &mut datastore).await?;
     std::fs::remove_dir_all(task_path)?;
     Ok(())
 }

From 6c5015b57c16de6efe63008af27b574c15ea04c6 Mon Sep 17 00:00:00 2001
From: Shuning <shuning@aukilabs.com>
Date: Mon, 9 Jun 2025 17:05:29 +0800
Subject: [PATCH 89/89] update domain module

---
 server/Cargo.lock | 8 ++++----
 server/Cargo.toml | 4 ++--
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/server/Cargo.lock b/server/Cargo.lock
index f4a78f4..e915b55 100644
--- a/server/Cargo.lock
+++ b/server/Cargo.lock
@@ -3121,7 +3121,7 @@ checksum = "350e9b48cbc6b0e028b0473b114454c6316e57336ee184ceab6e53f72c178b3e"
 [[package]]
 name = "posemesh-domain"
 version = "0.1.0"
-source = "git+https://github.com/aukilabs/posemesh.git?rev=0176b1d0998d13875580917b6708dfe9ce4b27ed#0176b1d0998d13875580917b6708dfe9ce4b27ed"
+source = "git+https://github.com/aukilabs/posemesh.git?rev=a6acafc44ede38afe07c4887b139186358f6d02c#a6acafc44ede38afe07c4887b139186358f6d02c"
 dependencies = [
  "async-timer",
  "async-trait",
@@ -3157,7 +3157,7 @@ dependencies = [
 [[package]]
 name = "posemesh-networking"
 version = "0.1.0"
-source = "git+https://github.com/aukilabs/posemesh.git?rev=0176b1d0998d13875580917b6708dfe9ce4b27ed#0176b1d0998d13875580917b6708dfe9ce4b27ed"
+source = "git+https://github.com/aukilabs/posemesh.git?rev=a6acafc44ede38afe07c4887b139186358f6d02c#a6acafc44ede38afe07c4887b139186358f6d02c"
 dependencies = [
  "async-trait",
  "cbindgen",
@@ -3189,7 +3189,7 @@ dependencies = [
 [[package]]
 name = "posemesh-runtime"
 version = "0.1.0"
-source = "git+https://github.com/aukilabs/posemesh.git?rev=0176b1d0998d13875580917b6708dfe9ce4b27ed#0176b1d0998d13875580917b6708dfe9ce4b27ed"
+source = "git+https://github.com/aukilabs/posemesh.git?rev=a6acafc44ede38afe07c4887b139186358f6d02c#a6acafc44ede38afe07c4887b139186358f6d02c"
 dependencies = [
  "once_cell",
  "tokio",
@@ -3198,7 +3198,7 @@ dependencies = [
 [[package]]
 name = "posemesh-utils"
 version = "0.1.0"
-source = "git+https://github.com/aukilabs/posemesh.git?rev=0176b1d0998d13875580917b6708dfe9ce4b27ed#0176b1d0998d13875580917b6708dfe9ce4b27ed"
+source = "git+https://github.com/aukilabs/posemesh.git?rev=a6acafc44ede38afe07c4887b139186358f6d02c#a6acafc44ede38afe07c4887b139186358f6d02c"
 dependencies = [
  "futures",
  "gloo-timers 0.3.0",
diff --git a/server/Cargo.toml b/server/Cargo.toml
index 7d324f8..2c8f294 100644
--- a/server/Cargo.toml
+++ b/server/Cargo.toml
@@ -4,9 +4,9 @@ version = "0.1.0"
 edition = "2021"
 
 [dependencies]
-posemesh-networking = { git = "https://github.com/aukilabs/posemesh.git", rev = "0176b1d0998d13875580917b6708dfe9ce4b27ed" }
+posemesh-networking = { git = "https://github.com/aukilabs/posemesh.git", rev = "a6acafc44ede38afe07c4887b139186358f6d02c" }
 futures = "0.3.30"
-posemesh-domain = { git = "https://github.com/aukilabs/posemesh.git", rev = "0176b1d0998d13875580917b6708dfe9ce4b27ed" }
+posemesh-domain = { git = "https://github.com/aukilabs/posemesh.git", rev = "a6acafc44ede38afe07c4887b139186358f6d02c" }
 tracing = "0.1.40"
 tracing-subscriber = "0.3.18"
 tokio = "1.44.0"