From da88215aa3b07262aea9f2568ac89833485bac8d Mon Sep 17 00:00:00 2001 From: David Dal Busco Date: Tue, 30 Sep 2025 12:32:12 +0200 Subject: [PATCH 01/26] feat: download snapshot offline --- package-lock.json | 22 ++-- package.json | 2 +- src/api/ic.api.ts | 28 ++++ src/commands/snapshot.ts | 9 ++ src/constants/snapshot.constants.ts | 3 + .../snapshot/snapshot.offline.services.ts | 124 ++++++++++++++++++ .../snapshot/snapshot.satellite.services.ts | 8 +- .../modules/snapshot/snapshot.services.ts | 58 +++++++- 8 files changed, 235 insertions(+), 19 deletions(-) create mode 100644 src/constants/snapshot.constants.ts create mode 100644 src/services/modules/snapshot/snapshot.offline.services.ts diff --git a/package-lock.json b/package-lock.json index 80cee7d3..ae272327 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,7 +12,7 @@ "@dfinity/agent": "^3.2.6", "@dfinity/auth-client": "^3.2.6", "@dfinity/candid": "^3.2.6", - "@dfinity/ic-management": "^7.0.1", + "@dfinity/ic-management": "^7.0.2-beta-2025-09-30", "@dfinity/identity": "^3.2.6", "@dfinity/principal": "^3.2.6", "@dfinity/zod-schemas": "^2.1.0", @@ -614,15 +614,15 @@ "license": "Apache-2.0" }, "node_modules/@dfinity/ic-management": { - "version": "7.0.1", - "resolved": "https://registry.npmjs.org/@dfinity/ic-management/-/ic-management-7.0.1.tgz", - "integrity": "sha512-zatpUqzf9k3bYkLeikurwYXOwzvJgKf+y8+u1Vfb/cS58gOu4OMB2buc9/Rw5frtN9TRbe/vsTQSodnLLlbZIw==", + "version": "7.0.2-beta-2025-09-30", + "resolved": "https://registry.npmjs.org/@dfinity/ic-management/-/ic-management-7.0.2-beta-2025-09-30.tgz", + "integrity": "sha512-UcNNIFKulBW+hd9zgDIL2pzPI/MJ6k9rg0uDbPS66q0lVebZ3xvNUP83JTwn143K2PNsYr3SBt9zC/l1OxtvYQ==", "license": "Apache-2.0", "peerDependencies": { - "@dfinity/agent": "^3", - "@dfinity/candid": "^3", - "@dfinity/principal": "^3", - "@dfinity/utils": "^3" + "@dfinity/agent": "*", + "@dfinity/candid": "*", + "@dfinity/principal": "*", + "@dfinity/utils": "*" } }, "node_modules/@dfinity/identity": { @@ -6855,9 +6855,9 @@ "integrity": "sha512-GPJpH73kDEKbUBdUjY80lz7cq9l0vm1h/7ppejPV6O0ZTqCLrYspssYvqjRmK4aNnJ/SKXsP0rg9LYX7zpegaA==" }, "@dfinity/ic-management": { - "version": "7.0.1", - "resolved": "https://registry.npmjs.org/@dfinity/ic-management/-/ic-management-7.0.1.tgz", - "integrity": "sha512-zatpUqzf9k3bYkLeikurwYXOwzvJgKf+y8+u1Vfb/cS58gOu4OMB2buc9/Rw5frtN9TRbe/vsTQSodnLLlbZIw==", + "version": "7.0.2-beta-2025-09-30", + "resolved": "https://registry.npmjs.org/@dfinity/ic-management/-/ic-management-7.0.2-beta-2025-09-30.tgz", + "integrity": "sha512-UcNNIFKulBW+hd9zgDIL2pzPI/MJ6k9rg0uDbPS66q0lVebZ3xvNUP83JTwn143K2PNsYr3SBt9zC/l1OxtvYQ==", "requires": {} }, "@dfinity/identity": { diff --git a/package.json b/package.json index d5e95674..fc580b22 100644 --- a/package.json +++ b/package.json @@ -26,7 +26,7 @@ "@dfinity/agent": "^3.2.6", "@dfinity/auth-client": "^3.2.6", "@dfinity/candid": "^3.2.6", - "@dfinity/ic-management": "^7.0.1", + "@dfinity/ic-management": "^7.0.2-beta-2025-09-30", "@dfinity/identity": "^3.2.6", "@dfinity/principal": "^3.2.6", "@dfinity/zod-schemas": "^2.1.0", diff --git a/src/api/ic.api.ts b/src/api/ic.api.ts index fc3b4fed..89b53ae9 100644 --- a/src/api/ic.api.ts +++ b/src/api/ic.api.ts @@ -1,8 +1,11 @@ import {ICManagementCanister} from '@dfinity/ic-management'; import type { list_canister_snapshots_result, + read_canister_snapshot_data_response, snapshot_id } from '@dfinity/ic-management/dist/candid/ic-management'; +import type {ReadCanisterSnapshotMetadataParams} from '@dfinity/ic-management/dist/types/types/snapshot.params'; +import type {ReadCanisterSnapshotMetadataResponse} from '@dfinity/ic-management/dist/types/types/snapshot.responses'; import type {Principal} from '@dfinity/principal'; import {initAgent} from './agent.api'; @@ -76,3 +79,28 @@ export const deleteCanisterSnapshot = async (params: { await deleteCanisterSnapshot(params); }; + +export const readCanisterSnapshotMetadata = async (params: { + canisterId: Principal; + snapshotId: snapshot_id; +}): Promise => { + const agent = await initAgent(); + + const {readCanisterSnapshotMetadata} = ICManagementCanister.create({ + agent + }); + + return await readCanisterSnapshotMetadata(params); +}; + +export const readCanisterSnapshotData = async ( + params: ReadCanisterSnapshotMetadataParams +): Promise => { + const agent = await initAgent(); + + const {readCanisterSnapshotData} = ICManagementCanister.create({ + agent + }); + + return await readCanisterSnapshotData(params); +}; diff --git a/src/commands/snapshot.ts b/src/commands/snapshot.ts index 3ff62092..0b944945 100644 --- a/src/commands/snapshot.ts +++ b/src/commands/snapshot.ts @@ -14,6 +14,7 @@ import { import { createSnapshotSatellite, deleteSnapshotSatellite, + downloadSnapshotSatellite, restoreSnapshotSatellite } from '../services/modules/snapshot/snapshot.satellite.services'; @@ -45,6 +46,14 @@ export const snapshot = async (args?: string[]) => { orbiterFn: deleteSnapshotOrbiter }); break; + case 'download': + await executeSnapshotFn({ + args, + satelliteFn: downloadSnapshotSatellite, + missionControlFn: deleteSnapshotMissionControl, + orbiterFn: deleteSnapshotOrbiter + }); + break; default: console.log(red('Unknown subcommand.')); logHelpSnapshot(args); diff --git a/src/constants/snapshot.constants.ts b/src/constants/snapshot.constants.ts new file mode 100644 index 00000000..37d15314 --- /dev/null +++ b/src/constants/snapshot.constants.ts @@ -0,0 +1,3 @@ +import {join} from 'node:path'; + +export const SNAPSHOTS_PATH = join(process.cwd(), '.snapshots'); \ No newline at end of file diff --git a/src/services/modules/snapshot/snapshot.offline.services.ts b/src/services/modules/snapshot/snapshot.offline.services.ts new file mode 100644 index 00000000..b5dff395 --- /dev/null +++ b/src/services/modules/snapshot/snapshot.offline.services.ts @@ -0,0 +1,124 @@ +import {CanisterSnapshotMetadataKind, encodeSnapshotId, snapshot_id} from '@dfinity/ic-management'; +import {read_canister_snapshot_data_response} from '@dfinity/ic-management/dist/candid/ic-management'; +import {Principal} from '@dfinity/principal'; +import {jsonReplacer} from '@dfinity/utils'; +import {red} from 'kleur'; +import {existsSync} from 'node:fs'; +import {mkdir, writeFile} from 'node:fs/promises'; +import {join} from 'node:path'; +import ora from 'ora'; +import {readCanisterSnapshotData, readCanisterSnapshotMetadata} from '../../../api/ic.api'; +import {SNAPSHOTS_PATH} from '../../../constants/snapshot.constants'; +import type {AssetKey} from '../../../types/asset-key'; +import {displaySegment} from '../../../utils/display.utils'; + +interface SnapshotParams { + canisterId: Principal; + snapshotId: snapshot_id; +} + +export const downloadExistingSnapshot = async ({ + segment, + ...params +}: SnapshotParams & { + segment: AssetKey; +}): Promise => { + const spinner = ora('Downloading the snapshot...').start(); + + try { + const result = await downloadSnapshotMetadataAndMemory(params); + + spinner.stop(); + + if (result.status === 'error') { + console.log( + `${red('Cannot proceed with download.')}\nDestination ${result.err.folder} already exists.` + ); + return; + } + + console.log(`✅ The snapshot for your ${displaySegment(segment)} was downloaded.`); + } catch (error: unknown) { + spinner.stop(); + + throw error; + } +}; + +class SnapshotFsError extends Error { + constructor(public readonly folder: string) { + super(); + } +} + +const downloadSnapshotMetadataAndMemory = async ({ + snapshotId, + ...rest +}: SnapshotParams): Promise< + {status: 'success'; snapshotIdText: string} | {status: 'error'; err: SnapshotFsError} +> => { + const snapshotIdText = `0x${encodeSnapshotId(snapshotId)}`; + const folder = join(SNAPSHOTS_PATH, snapshotIdText); + + if (existsSync(folder)) { + return {status: 'error', err: new SnapshotFsError(folder)}; + } + + await mkdir(folder, {recursive: true}); + + const metadata = await readCanisterSnapshotMetadata({snapshotId, ...rest}); + + const destination = join(folder, 'metadata.json'); + await writeFile(destination, JSON.stringify(metadata, jsonReplacer, 2), 'utf-8'); + + return {status: 'success', snapshotIdText}; +}; + +type RequestChunk = Exclude; + +type Chunk = read_canister_snapshot_data_response['chunk']; + +const downloadWasmModule = async ({}: SnapshotParams) => {}; + +const batchDownloadChunks = async (params: SnapshotParams) => { + let chunks: Chunk[] = []; + for await (const results of batchUploadChunks({requestedChunks: [], ...params})) { + chunks = [...chunks, ...results]; + } +}; + +async function* batchUploadChunks({ + requestedChunks, + limit = 12, + ...params +}: SnapshotParams & { + requestedChunks: RequestChunk[]; + limit?: number; +}): AsyncGenerator { + for (let i = 0; i < requestedChunks.length; i = i + limit) { + const batch = requestedChunks.slice(i, i + limit); + const result = await Promise.all( + batch.map((requestChunk) => + downloadChunk({ + ...params, + requestChunk + }) + ) + ); + yield result; + } +} + +const downloadChunk = async ({ + requestChunk: kind, + ...rest +}: SnapshotParams & { + requestChunk: RequestChunk; +}): Promise => { + const {chunk} = await readCanisterSnapshotData({ + ...rest, + kind + }); + + return chunk; +}; diff --git a/src/services/modules/snapshot/snapshot.satellite.services.ts b/src/services/modules/snapshot/snapshot.satellite.services.ts index 2e1dbdd9..9159ed3b 100644 --- a/src/services/modules/snapshot/snapshot.satellite.services.ts +++ b/src/services/modules/snapshot/snapshot.satellite.services.ts @@ -3,7 +3,7 @@ import {noJunoConfig} from '../../../configs/juno.config'; import type {AssetKey} from '../../../types/asset-key'; import {consoleNoConfigFound} from '../../../utils/msg.utils'; import {assertConfigAndLoadSatelliteContext} from '../../../utils/satellite.utils'; -import {createSnapshot, deleteSnapshot, restoreSnapshot} from './snapshot.services'; +import {createSnapshot, deleteSnapshot, downloadSnapshot, restoreSnapshot} from './snapshot.services'; export const createSnapshotSatellite = async () => { await executeSnapshotFn({ @@ -23,6 +23,12 @@ export const deleteSnapshotSatellite = async () => { }); }; +export const downloadSnapshotSatellite = async () => { + await executeSnapshotFn({ + fn: downloadSnapshot + }); +}; + const executeSnapshotFn = async ({ fn }: { diff --git a/src/services/modules/snapshot/snapshot.services.ts b/src/services/modules/snapshot/snapshot.services.ts index dff42afa..b85c1e29 100644 --- a/src/services/modules/snapshot/snapshot.services.ts +++ b/src/services/modules/snapshot/snapshot.services.ts @@ -13,6 +13,7 @@ import { import type {AssetKey} from '../../../types/asset-key'; import {displaySegment} from '../../../utils/display.utils'; import {confirmAndExit} from '../../../utils/prompt.utils'; +import {downloadExistingSnapshot} from './snapshot.offline.services'; export const createSnapshot = async ({ canisterId: cId, @@ -47,13 +48,14 @@ export const restoreSnapshot = async ({ }) => { const canisterId = Principal.fromText(cId); - const existingSnapshotId = await loadSnapshot({canisterId}); + const result = await assertAndLoadSnapshot({canisterId, segment}); - if (isNullish(existingSnapshotId)) { - console.log(red(`No snapshot found for your ${displaySegment(segment)}.`)); + if (result.result === 'not_found') { return; } + const {snapshotId: existingSnapshotId} = result; + await confirmAndExit( `Restoring the snapshot 0x${encodeSnapshotId(existingSnapshotId)} will permanently overwrite the current state of your ${displaySegment(segment)}. Are you sure you want to proceed?` ); @@ -74,13 +76,14 @@ export const deleteSnapshot = async ({ }) => { const canisterId = Principal.fromText(cId); - const existingSnapshotId = await loadSnapshot({canisterId}); + const result = await assertAndLoadSnapshot({canisterId, segment}); - if (isNullish(existingSnapshotId)) { - console.log(red(`No snapshot found for your ${displaySegment(segment)}.`)); + if (result.result === 'not_found') { return; } + const {snapshotId: existingSnapshotId} = result; + await confirmAndExit( `Are you sure you want to delete the snapshot 0x${encodeSnapshotId(existingSnapshotId)} of your ${displaySegment(segment)}?` ); @@ -92,6 +95,32 @@ export const deleteSnapshot = async ({ }); }; +export const downloadSnapshot = async ({ + canisterId: cId, + segment +}: { + canisterId: string; + segment: AssetKey; +}) => { + const canisterId = Principal.fromText(cId); + + const result = await assertAndLoadSnapshot({canisterId, segment}); + + if (result.result === 'not_found') { + return; + } + + + + const {snapshotId: existingSnapshotId} = result; + + await downloadExistingSnapshot({ + canisterId, + snapshotId: existingSnapshotId, + segment + }) +}; + const restoreExistingSnapshot = async ({ segment, ...rest @@ -166,3 +195,20 @@ const loadSnapshot = async ({ spinner.stop(); } }; + +const assertAndLoadSnapshot = async ({ + canisterId, + segment +}: { + canisterId: Principal; + segment: AssetKey; +}): Promise<{result: 'ok'; snapshotId: snapshot_id} | {result: 'not_found'}> => { + const existingSnapshotId = await loadSnapshot({canisterId}); + + if (isNullish(existingSnapshotId)) { + console.log(red(`No snapshot found for your ${displaySegment(segment)}.`)); + return {result: 'not_found'}; + } + + return {result: 'ok', snapshotId: existingSnapshotId}; +}; From 3400b471368419fed950a51471f59017dabf4eca Mon Sep 17 00:00:00 2001 From: David Dal Busco Date: Tue, 30 Sep 2025 14:00:56 +0200 Subject: [PATCH 02/26] feat: download chunks --- src/constants/snapshot.constants.ts | 5 +- .../snapshot/snapshot.offline.services.ts | 119 +++++++++++++----- 2 files changed, 94 insertions(+), 30 deletions(-) diff --git a/src/constants/snapshot.constants.ts b/src/constants/snapshot.constants.ts index 37d15314..a294c4c6 100644 --- a/src/constants/snapshot.constants.ts +++ b/src/constants/snapshot.constants.ts @@ -1,3 +1,6 @@ import {join} from 'node:path'; -export const SNAPSHOTS_PATH = join(process.cwd(), '.snapshots'); \ No newline at end of file +export const SNAPSHOTS_PATH = join(process.cwd(), '.snapshots'); + +// https://forum.dfinity.org/t/canister-snapshot-up-download/57397?u=peterparker +export const SNAPSHOT_CHUNK_SIZE = 1_000_000n; \ No newline at end of file diff --git a/src/services/modules/snapshot/snapshot.offline.services.ts b/src/services/modules/snapshot/snapshot.offline.services.ts index b5dff395..315045ad 100644 --- a/src/services/modules/snapshot/snapshot.offline.services.ts +++ b/src/services/modules/snapshot/snapshot.offline.services.ts @@ -1,16 +1,17 @@ import {CanisterSnapshotMetadataKind, encodeSnapshotId, snapshot_id} from '@dfinity/ic-management'; -import {read_canister_snapshot_data_response} from '@dfinity/ic-management/dist/candid/ic-management'; +import type {ReadCanisterSnapshotMetadataResponse} from '@dfinity/ic-management/dist/types/types/snapshot.responses'; import {Principal} from '@dfinity/principal'; -import {jsonReplacer} from '@dfinity/utils'; +import {arrayOfNumberToUint8Array, jsonReplacer} from '@dfinity/utils'; import {red} from 'kleur'; import {existsSync} from 'node:fs'; import {mkdir, writeFile} from 'node:fs/promises'; import {join} from 'node:path'; import ora from 'ora'; import {readCanisterSnapshotData, readCanisterSnapshotMetadata} from '../../../api/ic.api'; -import {SNAPSHOTS_PATH} from '../../../constants/snapshot.constants'; +import {SNAPSHOT_CHUNK_SIZE, SNAPSHOTS_PATH} from '../../../constants/snapshot.constants'; import type {AssetKey} from '../../../types/asset-key'; import {displaySegment} from '../../../utils/display.utils'; +import {size} from 'zod'; interface SnapshotParams { canisterId: Principal; @@ -66,59 +67,119 @@ const downloadSnapshotMetadataAndMemory = async ({ await mkdir(folder, {recursive: true}); - const metadata = await readCanisterSnapshotMetadata({snapshotId, ...rest}); + const { + metadata: {wasmModuleSize} + } = await downloadMetadata({folder, snapshotId, ...rest}); - const destination = join(folder, 'metadata.json'); - await writeFile(destination, JSON.stringify(metadata, jsonReplacer, 2), 'utf-8'); + await downloadWasmModule({ + folder, + snapshotId, + size: wasmModuleSize, + ...rest + }); return {status: 'success', snapshotIdText}; }; -type RequestChunk = Exclude; +type Chunk = Exclude & {orderId: bigint}; -type Chunk = read_canister_snapshot_data_response['chunk']; +const downloadMetadata = async ({ + folder, + ...rest +}: SnapshotParams & {folder: string}): Promise<{ + metadata: ReadCanisterSnapshotMetadataResponse; +}> => { + const metadata = await readCanisterSnapshotMetadata(rest); -const downloadWasmModule = async ({}: SnapshotParams) => {}; + const destination = join(folder, 'metadata.json'); + await writeFile(destination, JSON.stringify(metadata, jsonReplacer, 2), 'utf-8'); -const batchDownloadChunks = async (params: SnapshotParams) => { - let chunks: Chunk[] = []; - for await (const results of batchUploadChunks({requestedChunks: [], ...params})) { - chunks = [...chunks, ...results]; + return {metadata}; +}; + +const downloadWasmModule = async ({ + size, + ...params +}: SnapshotParams & {size: bigint; folder: string}) => { + const {chunks} = prepareDownloadChunks({size}); + + for await (const progress of batchDownloadChunks({ + chunks, + filename: 'wasm-module', + limit: 12, + ...params + })) { + console.log(`Batch ${progress.index} of ${progress.total} done.`); } }; -async function* batchUploadChunks({ - requestedChunks, +const prepareDownloadChunks = ({size: totalSize}: {size: bigint}): {chunks: Chunk[]} => { + let orderId = 0n; + + const chunks: Chunk[] = []; + + for (let offset = 0n; offset < totalSize; offset += SNAPSHOT_CHUNK_SIZE) { + const size = + offset + SNAPSHOT_CHUNK_SIZE <= totalSize ? SNAPSHOT_CHUNK_SIZE : totalSize - offset; + + chunks.push({ + wasmModule: { + offset, + size + }, + orderId + }); + + orderId++; + } + + return {chunks}; +} + +async function* batchDownloadChunks({ + chunks, limit = 12, ...params -}: SnapshotParams & { - requestedChunks: RequestChunk[]; +}: SnapshotChunkFsParams & { + chunks: Chunk[]; limit?: number; -}): AsyncGenerator { - for (let i = 0; i < requestedChunks.length; i = i + limit) { - const batch = requestedChunks.slice(i, i + limit); - const result = await Promise.all( +}): AsyncGenerator<{index: number; total: number}, void> { + for (let i = 0; i < chunks.length; i = i + limit) { + const batch = chunks.slice(i, i + limit); + await Promise.all( batch.map((requestChunk) => downloadChunk({ ...params, - requestChunk + chunk: requestChunk }) ) ); - yield result; + yield {index: i, total: chunks.length}; } } +interface SnapshotChunkFsParams extends SnapshotParams { + folder: string; + filename: string; +} + const downloadChunk = async ({ - requestChunk: kind, + chunk: {orderId, ...kind}, + folder, + filename, ...rest -}: SnapshotParams & { - requestChunk: RequestChunk; -}): Promise => { - const {chunk} = await readCanisterSnapshotData({ +}: SnapshotChunkFsParams & { + chunk: Chunk; +}): Promise => { + const {chunk: downloadedChunk} = await readCanisterSnapshotData({ ...rest, kind }); - return chunk; + // Note: we would not win much at gzipping the data. + const destination = join(folder, `${filename}-${orderId}.bin`); + await writeFile( + destination, + downloadedChunk instanceof Uint8Array ? downloadedChunk : arrayOfNumberToUint8Array(downloadedChunk) + ); }; From 98b98fa255f77a3d089af5e39844d3b4e0d76c42 Mon Sep 17 00:00:00 2001 From: David Dal Busco Date: Tue, 30 Sep 2025 14:40:51 +0200 Subject: [PATCH 03/26] feat: download all memories --- src/constants/snapshot.constants.ts | 2 +- .../snapshot/snapshot.offline.services.ts | 71 +++++++++++++------ .../modules/snapshot/snapshot.services.ts | 4 +- 3 files changed, 51 insertions(+), 26 deletions(-) diff --git a/src/constants/snapshot.constants.ts b/src/constants/snapshot.constants.ts index a294c4c6..c23c0f7b 100644 --- a/src/constants/snapshot.constants.ts +++ b/src/constants/snapshot.constants.ts @@ -3,4 +3,4 @@ import {join} from 'node:path'; export const SNAPSHOTS_PATH = join(process.cwd(), '.snapshots'); // https://forum.dfinity.org/t/canister-snapshot-up-download/57397?u=peterparker -export const SNAPSHOT_CHUNK_SIZE = 1_000_000n; \ No newline at end of file +export const SNAPSHOT_CHUNK_SIZE = 1_000_000n; diff --git a/src/services/modules/snapshot/snapshot.offline.services.ts b/src/services/modules/snapshot/snapshot.offline.services.ts index 315045ad..401503d1 100644 --- a/src/services/modules/snapshot/snapshot.offline.services.ts +++ b/src/services/modules/snapshot/snapshot.offline.services.ts @@ -1,6 +1,6 @@ -import {CanisterSnapshotMetadataKind, encodeSnapshotId, snapshot_id} from '@dfinity/ic-management'; +import {type CanisterSnapshotMetadataKind, encodeSnapshotId, type snapshot_id} from '@dfinity/ic-management'; import type {ReadCanisterSnapshotMetadataResponse} from '@dfinity/ic-management/dist/types/types/snapshot.responses'; -import {Principal} from '@dfinity/principal'; +import {type Principal} from '@dfinity/principal'; import {arrayOfNumberToUint8Array, jsonReplacer} from '@dfinity/utils'; import {red} from 'kleur'; import {existsSync} from 'node:fs'; @@ -11,7 +11,6 @@ import {readCanisterSnapshotData, readCanisterSnapshotMetadata} from '../../../a import {SNAPSHOT_CHUNK_SIZE, SNAPSHOTS_PATH} from '../../../constants/snapshot.constants'; import type {AssetKey} from '../../../types/asset-key'; import {displaySegment} from '../../../utils/display.utils'; -import {size} from 'zod'; interface SnapshotParams { canisterId: Principal; @@ -71,17 +70,37 @@ const downloadSnapshotMetadataAndMemory = async ({ metadata: {wasmModuleSize} } = await downloadMetadata({folder, snapshotId, ...rest}); - await downloadWasmModule({ + await downloadChunks({ folder, snapshotId, size: wasmModuleSize, + build: (param) => ({wasmModule: param}), + ...rest + }); + + await downloadChunks({ + folder, + snapshotId, + size: wasmModuleSize, + build: (param) => ({wasmMemory: param}), + ...rest + }); + + await downloadChunks({ + folder, + snapshotId, + size: wasmModuleSize, + build: (param) => ({stableMemory: param}), ...rest }); return {status: 'success', snapshotIdText}; }; -type Chunk = Exclude & {orderId: bigint}; +type Chunk = Exclude; +type OrderedChunk = Chunk & {orderId: bigint}; + +type BuildChunkFn = (params: {offset: bigint; size: bigint}) => Chunk; const downloadMetadata = async ({ folder, @@ -97,15 +116,15 @@ const downloadMetadata = async ({ return {metadata}; }; -const downloadWasmModule = async ({ +const downloadChunks = async ({ size, + build, ...params -}: SnapshotParams & {size: bigint; folder: string}) => { - const {chunks} = prepareDownloadChunks({size}); +}: SnapshotParams & {folder: string; size: bigint; build: BuildChunkFn}) => { + const {chunks} = prepareDownloadChunks({size, build}); for await (const progress of batchDownloadChunks({ chunks, - filename: 'wasm-module', limit: 12, ...params })) { @@ -113,20 +132,26 @@ const downloadWasmModule = async ({ } }; -const prepareDownloadChunks = ({size: totalSize}: {size: bigint}): {chunks: Chunk[]} => { +const prepareDownloadChunks = ({ + size: totalSize, + build +}: { + size: bigint; + build: BuildChunkFn; +}): {chunks: OrderedChunk[]} => { let orderId = 0n; - const chunks: Chunk[] = []; + const chunks: OrderedChunk[] = []; for (let offset = 0n; offset < totalSize; offset += SNAPSHOT_CHUNK_SIZE) { const size = offset + SNAPSHOT_CHUNK_SIZE <= totalSize ? SNAPSHOT_CHUNK_SIZE : totalSize - offset; chunks.push({ - wasmModule: { + ...build({ offset, size - }, + }), orderId }); @@ -134,24 +159,24 @@ const prepareDownloadChunks = ({size: totalSize}: {size: bigint}): {chunks: Chun } return {chunks}; -} +}; async function* batchDownloadChunks({ chunks, limit = 12, ...params }: SnapshotChunkFsParams & { - chunks: Chunk[]; + chunks: OrderedChunk[]; limit?: number; }): AsyncGenerator<{index: number; total: number}, void> { for (let i = 0; i < chunks.length; i = i + limit) { const batch = chunks.slice(i, i + limit); await Promise.all( - batch.map((requestChunk) => - downloadChunk({ + batch.map(async (requestChunk) => + { await downloadChunk({ ...params, chunk: requestChunk - }) + }); } ) ); yield {index: i, total: chunks.length}; @@ -160,26 +185,28 @@ async function* batchDownloadChunks({ interface SnapshotChunkFsParams extends SnapshotParams { folder: string; - filename: string; } const downloadChunk = async ({ chunk: {orderId, ...kind}, folder, - filename, ...rest }: SnapshotChunkFsParams & { - chunk: Chunk; + chunk: OrderedChunk; }): Promise => { const {chunk: downloadedChunk} = await readCanisterSnapshotData({ ...rest, kind }); + const filename = Object.keys(kind)[0].toLowerCase(); + // Note: we would not win much at gzipping the data. const destination = join(folder, `${filename}-${orderId}.bin`); await writeFile( destination, - downloadedChunk instanceof Uint8Array ? downloadedChunk : arrayOfNumberToUint8Array(downloadedChunk) + downloadedChunk instanceof Uint8Array + ? downloadedChunk + : arrayOfNumberToUint8Array(downloadedChunk) ); }; diff --git a/src/services/modules/snapshot/snapshot.services.ts b/src/services/modules/snapshot/snapshot.services.ts index b85c1e29..b6b84fc9 100644 --- a/src/services/modules/snapshot/snapshot.services.ts +++ b/src/services/modules/snapshot/snapshot.services.ts @@ -110,15 +110,13 @@ export const downloadSnapshot = async ({ return; } - - const {snapshotId: existingSnapshotId} = result; await downloadExistingSnapshot({ canisterId, snapshotId: existingSnapshotId, segment - }) + }); }; const restoreExistingSnapshot = async ({ From a5e11486a7b78f711d7a34d728735a0e5ed1fec0 Mon Sep 17 00:00:00 2001 From: David Dal Busco Date: Tue, 30 Sep 2025 14:56:58 +0200 Subject: [PATCH 04/26] feat: print --- .../snapshot/snapshot.offline.services.ts | 53 +++++++++++++------ .../snapshot/snapshot.satellite.services.ts | 7 ++- 2 files changed, 44 insertions(+), 16 deletions(-) diff --git a/src/services/modules/snapshot/snapshot.offline.services.ts b/src/services/modules/snapshot/snapshot.offline.services.ts index 401503d1..15e9e1f2 100644 --- a/src/services/modules/snapshot/snapshot.offline.services.ts +++ b/src/services/modules/snapshot/snapshot.offline.services.ts @@ -1,4 +1,8 @@ -import {type CanisterSnapshotMetadataKind, encodeSnapshotId, type snapshot_id} from '@dfinity/ic-management'; +import { + type CanisterSnapshotMetadataKind, + encodeSnapshotId, + type snapshot_id +} from '@dfinity/ic-management'; import type {ReadCanisterSnapshotMetadataResponse} from '@dfinity/ic-management/dist/types/types/snapshot.responses'; import {type Principal} from '@dfinity/principal'; import {arrayOfNumberToUint8Array, jsonReplacer} from '@dfinity/utils'; @@ -26,7 +30,10 @@ export const downloadExistingSnapshot = async ({ const spinner = ora('Downloading the snapshot...').start(); try { - const result = await downloadSnapshotMetadataAndMemory(params); + const result = await downloadSnapshotMetadataAndMemory({ + ...params, + log: (text) => (spinner.text = text) + }); spinner.stop(); @@ -51,10 +58,15 @@ class SnapshotFsError extends Error { } } +interface Log { + log: (text: string) => void; +} + const downloadSnapshotMetadataAndMemory = async ({ snapshotId, + log, ...rest -}: SnapshotParams): Promise< +}: SnapshotParams & Log): Promise< {status: 'success'; snapshotIdText: string} | {status: 'error'; err: SnapshotFsError} > => { const snapshotIdText = `0x${encodeSnapshotId(snapshotId)}`; @@ -68,13 +80,14 @@ const downloadSnapshotMetadataAndMemory = async ({ const { metadata: {wasmModuleSize} - } = await downloadMetadata({folder, snapshotId, ...rest}); + } = await downloadMetadata({folder, snapshotId, log, ...rest}); await downloadChunks({ folder, snapshotId, size: wasmModuleSize, build: (param) => ({wasmModule: param}), + log: (text) => log(`[WASM module] ${text}`), ...rest }); @@ -83,6 +96,7 @@ const downloadSnapshotMetadataAndMemory = async ({ snapshotId, size: wasmModuleSize, build: (param) => ({wasmMemory: param}), + log: (text) => log(`[Heap memory] ${text}`), ...rest }); @@ -91,6 +105,7 @@ const downloadSnapshotMetadataAndMemory = async ({ snapshotId, size: wasmModuleSize, build: (param) => ({stableMemory: param}), + log: (text) => log(`[Stable memory] ${text}`), ...rest }); @@ -104,10 +119,13 @@ type BuildChunkFn = (params: {offset: bigint; size: bigint}) => Chunk; const downloadMetadata = async ({ folder, + log, ...rest -}: SnapshotParams & {folder: string}): Promise<{ +}: SnapshotParams & {folder: string} & Log): Promise<{ metadata: ReadCanisterSnapshotMetadataResponse; }> => { + log('Downloading the snapshot metadata...'); + const metadata = await readCanisterSnapshotMetadata(rest); const destination = join(folder, 'metadata.json'); @@ -119,16 +137,19 @@ const downloadMetadata = async ({ const downloadChunks = async ({ size, build, + log, ...params -}: SnapshotParams & {folder: string; size: bigint; build: BuildChunkFn}) => { +}: SnapshotParams & {folder: string; size: bigint; build: BuildChunkFn} & Log) => { const {chunks} = prepareDownloadChunks({size, build}); + log("Downloading chunks..."); + for await (const progress of batchDownloadChunks({ chunks, limit: 12, ...params })) { - console.log(`Batch ${progress.index} of ${progress.total} done.`); + log(`Chunks ${progress.done}/${progress.total} downloaded. Continuing`); } }; @@ -155,7 +176,7 @@ const prepareDownloadChunks = ({ orderId }); - orderId++; + orderId += 1n; } return {chunks}; @@ -168,18 +189,20 @@ async function* batchDownloadChunks({ }: SnapshotChunkFsParams & { chunks: OrderedChunk[]; limit?: number; -}): AsyncGenerator<{index: number; total: number}, void> { - for (let i = 0; i < chunks.length; i = i + limit) { +}): AsyncGenerator<{index: number; done: number; total: number}, void> { + const total = chunks.length; + + for (let i = 0; i < total; i = i + limit) { const batch = chunks.slice(i, i + limit); await Promise.all( - batch.map(async (requestChunk) => - { await downloadChunk({ + batch.map(async (requestChunk) => { + await downloadChunk({ ...params, chunk: requestChunk - }); } - ) + }); + }) ); - yield {index: i, total: chunks.length}; + yield {index: i, done: Math.min(i + limit, total), total}; } } diff --git a/src/services/modules/snapshot/snapshot.satellite.services.ts b/src/services/modules/snapshot/snapshot.satellite.services.ts index 9159ed3b..d204cd31 100644 --- a/src/services/modules/snapshot/snapshot.satellite.services.ts +++ b/src/services/modules/snapshot/snapshot.satellite.services.ts @@ -3,7 +3,12 @@ import {noJunoConfig} from '../../../configs/juno.config'; import type {AssetKey} from '../../../types/asset-key'; import {consoleNoConfigFound} from '../../../utils/msg.utils'; import {assertConfigAndLoadSatelliteContext} from '../../../utils/satellite.utils'; -import {createSnapshot, deleteSnapshot, downloadSnapshot, restoreSnapshot} from './snapshot.services'; +import { + createSnapshot, + deleteSnapshot, + downloadSnapshot, + restoreSnapshot +} from './snapshot.services'; export const createSnapshotSatellite = async () => { await executeSnapshotFn({ From b22f961a14deed8beba45977bf8cb8b8434b2a1d Mon Sep 17 00:00:00 2001 From: David Dal Busco Date: Tue, 30 Sep 2025 15:14:08 +0200 Subject: [PATCH 05/26] feat: logs --- .../modules/snapshot/snapshot.offline.services.ts | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/services/modules/snapshot/snapshot.offline.services.ts b/src/services/modules/snapshot/snapshot.offline.services.ts index 15e9e1f2..a6b307e5 100644 --- a/src/services/modules/snapshot/snapshot.offline.services.ts +++ b/src/services/modules/snapshot/snapshot.offline.services.ts @@ -44,7 +44,12 @@ export const downloadExistingSnapshot = async ({ return; } - console.log(`✅ The snapshot for your ${displaySegment(segment)} was downloaded.`); + const {snapshotIdText, folder} = result; + + console.log( + `✅ The snapshot ${snapshotIdText} for your ${displaySegment(segment)} has been downloaded.` + ); + console.log(`🗂️ Files saved to ${folder}`); } catch (error: unknown) { spinner.stop(); @@ -67,7 +72,8 @@ const downloadSnapshotMetadataAndMemory = async ({ log, ...rest }: SnapshotParams & Log): Promise< - {status: 'success'; snapshotIdText: string} | {status: 'error'; err: SnapshotFsError} + | {status: 'success'; snapshotIdText: string; folder: string} + | {status: 'error'; err: SnapshotFsError} > => { const snapshotIdText = `0x${encodeSnapshotId(snapshotId)}`; const folder = join(SNAPSHOTS_PATH, snapshotIdText); @@ -109,7 +115,7 @@ const downloadSnapshotMetadataAndMemory = async ({ ...rest }); - return {status: 'success', snapshotIdText}; + return {status: 'success', snapshotIdText, folder}; }; type Chunk = Exclude; @@ -142,7 +148,7 @@ const downloadChunks = async ({ }: SnapshotParams & {folder: string; size: bigint; build: BuildChunkFn} & Log) => { const {chunks} = prepareDownloadChunks({size, build}); - log("Downloading chunks..."); + log('Downloading chunks...'); for await (const progress of batchDownloadChunks({ chunks, From a625aa4c72ebfd6493fed1d3d9d000a5f3054cec Mon Sep 17 00:00:00 2001 From: David Dal Busco Date: Tue, 30 Sep 2025 15:20:09 +0200 Subject: [PATCH 06/26] feat: skip size zero --- .../snapshot/snapshot.offline.services.ts | 26 ++++++++++++++++--- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/src/services/modules/snapshot/snapshot.offline.services.ts b/src/services/modules/snapshot/snapshot.offline.services.ts index a6b307e5..f2139d0d 100644 --- a/src/services/modules/snapshot/snapshot.offline.services.ts +++ b/src/services/modules/snapshot/snapshot.offline.services.ts @@ -88,7 +88,7 @@ const downloadSnapshotMetadataAndMemory = async ({ metadata: {wasmModuleSize} } = await downloadMetadata({folder, snapshotId, log, ...rest}); - await downloadChunks({ + await assertSizeAndDownloadChunks({ folder, snapshotId, size: wasmModuleSize, @@ -97,7 +97,7 @@ const downloadSnapshotMetadataAndMemory = async ({ ...rest }); - await downloadChunks({ + await assertSizeAndDownloadChunks({ folder, snapshotId, size: wasmModuleSize, @@ -106,7 +106,7 @@ const downloadSnapshotMetadataAndMemory = async ({ ...rest }); - await downloadChunks({ + await assertSizeAndDownloadChunks({ folder, snapshotId, size: wasmModuleSize, @@ -140,6 +140,24 @@ const downloadMetadata = async ({ return {metadata}; }; +const assertSizeAndDownloadChunks = async ({ + size, + log, + ...params +}: SnapshotParams & {folder: string; size: bigint; build: BuildChunkFn} & Log) => { + if (size === 0n) { + log('No chunks to download (size = 0). Skipping.'); + await new Promise((resolve) => setTimeout(resolve, 2500)); + return; + } + + await downloadChunks({ + size, + log, + ...params + }); +}; + const downloadChunks = async ({ size, build, @@ -155,7 +173,7 @@ const downloadChunks = async ({ limit: 12, ...params })) { - log(`Chunks ${progress.done}/${progress.total} downloaded. Continuing`); + log(`Chunks ${progress.done}/${progress.total} downloaded. Continuing...`); } }; From e9450660947c7a77976f159e837fdfd776963e6d Mon Sep 17 00:00:00 2001 From: David Dal Busco Date: Tue, 30 Sep 2025 15:22:58 +0200 Subject: [PATCH 07/26] feat: use heap and stable --- src/services/modules/snapshot/snapshot.offline.services.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/services/modules/snapshot/snapshot.offline.services.ts b/src/services/modules/snapshot/snapshot.offline.services.ts index f2139d0d..78a5ba14 100644 --- a/src/services/modules/snapshot/snapshot.offline.services.ts +++ b/src/services/modules/snapshot/snapshot.offline.services.ts @@ -85,7 +85,7 @@ const downloadSnapshotMetadataAndMemory = async ({ await mkdir(folder, {recursive: true}); const { - metadata: {wasmModuleSize} + metadata: {wasmModuleSize, wasmMemorySize, stableMemorySize} } = await downloadMetadata({folder, snapshotId, log, ...rest}); await assertSizeAndDownloadChunks({ @@ -100,7 +100,7 @@ const downloadSnapshotMetadataAndMemory = async ({ await assertSizeAndDownloadChunks({ folder, snapshotId, - size: wasmModuleSize, + size: wasmMemorySize, build: (param) => ({wasmMemory: param}), log: (text) => log(`[Heap memory] ${text}`), ...rest @@ -109,7 +109,7 @@ const downloadSnapshotMetadataAndMemory = async ({ await assertSizeAndDownloadChunks({ folder, snapshotId, - size: wasmModuleSize, + size: stableMemorySize, build: (param) => ({stableMemory: param}), log: (text) => log(`[Stable memory] ${text}`), ...rest From 2a8e49d7c9fdb1c938568480d80e302c43541dac Mon Sep 17 00:00:00 2001 From: David Dal Busco Date: Tue, 30 Sep 2025 15:31:50 +0200 Subject: [PATCH 08/26] feat: download chunk store --- .../snapshot/snapshot.offline.services.ts | 64 +++++++++++++++++-- 1 file changed, 57 insertions(+), 7 deletions(-) diff --git a/src/services/modules/snapshot/snapshot.offline.services.ts b/src/services/modules/snapshot/snapshot.offline.services.ts index 78a5ba14..9cdbc301 100644 --- a/src/services/modules/snapshot/snapshot.offline.services.ts +++ b/src/services/modules/snapshot/snapshot.offline.services.ts @@ -85,7 +85,7 @@ const downloadSnapshotMetadataAndMemory = async ({ await mkdir(folder, {recursive: true}); const { - metadata: {wasmModuleSize, wasmMemorySize, stableMemorySize} + metadata: {wasmModuleSize, wasmMemorySize, stableMemorySize, wasmChunkStore} } = await downloadMetadata({folder, snapshotId, log, ...rest}); await assertSizeAndDownloadChunks({ @@ -115,11 +115,19 @@ const downloadSnapshotMetadataAndMemory = async ({ ...rest }); + await assertAndDownloadWasmChunks({ + folder, + snapshotId, + wasmChunkStore, + log: (text) => log(`[WASM store] ${text}`), + ...rest + }); + return {status: 'success', snapshotIdText, folder}; }; -type Chunk = Exclude; -type OrderedChunk = Chunk & {orderId: bigint}; +type Chunk = CanisterSnapshotMetadataKind; +type OrderedChunk = Chunk & {orderId: number}; type BuildChunkFn = (params: {offset: bigint; size: bigint}) => Chunk; @@ -151,14 +159,36 @@ const assertSizeAndDownloadChunks = async ({ return; } - await downloadChunks({ + await downloadMemoryChunks({ size, log, ...params }); }; -const downloadChunks = async ({ +const assertAndDownloadWasmChunks = async ({ + wasmChunkStore, + log, + ...params +}: SnapshotParams & {folder: string} & Pick< + ReadCanisterSnapshotMetadataResponse, + 'wasmChunkStore' + > & + Log) => { + if (wasmChunkStore.length === 0) { + log('No chunks to download (length = 0). Skipping.'); + await new Promise((resolve) => setTimeout(resolve, 2500)); + return; + } + + await downloadWasmChunks({ + wasmChunkStore, + log, + ...params + }); +}; + +const downloadMemoryChunks = async ({ size, build, log, @@ -177,6 +207,26 @@ const downloadChunks = async ({ } }; +const downloadWasmChunks = async ({ + wasmChunkStore, + log, + ...params +}: SnapshotParams & {folder: string} & Pick< + ReadCanisterSnapshotMetadataResponse, + 'wasmChunkStore' + > & + Log) => { + log('[WASM store] Downloading chunks...'); + + for await (const progress of batchDownloadChunks({ + chunks: wasmChunkStore.map((chunk, orderId) => ({wasmChunk: chunk, orderId})), + limit: 12, + ...params + })) { + log(`Chunks ${progress.done}/${progress.total} downloaded. Continuing...`); + } +}; + const prepareDownloadChunks = ({ size: totalSize, build @@ -184,7 +234,7 @@ const prepareDownloadChunks = ({ size: bigint; build: BuildChunkFn; }): {chunks: OrderedChunk[]} => { - let orderId = 0n; + let orderId = 0; const chunks: OrderedChunk[] = []; @@ -200,7 +250,7 @@ const prepareDownloadChunks = ({ orderId }); - orderId += 1n; + orderId += 1; } return {chunks}; From ffd1e4fe4537b6fe4f7ec75182f28d42b5df25b2 Mon Sep 17 00:00:00 2001 From: David Dal Busco Date: Tue, 30 Sep 2025 15:33:28 +0200 Subject: [PATCH 09/26] feat: log --- src/services/modules/snapshot/snapshot.offline.services.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/services/modules/snapshot/snapshot.offline.services.ts b/src/services/modules/snapshot/snapshot.offline.services.ts index 9cdbc301..90ba2fc1 100644 --- a/src/services/modules/snapshot/snapshot.offline.services.ts +++ b/src/services/modules/snapshot/snapshot.offline.services.ts @@ -86,7 +86,7 @@ const downloadSnapshotMetadataAndMemory = async ({ const { metadata: {wasmModuleSize, wasmMemorySize, stableMemorySize, wasmChunkStore} - } = await downloadMetadata({folder, snapshotId, log, ...rest}); + } = await downloadMetadata({folder, snapshotId, log, snapshotIdText, ...rest}); await assertSizeAndDownloadChunks({ folder, @@ -133,12 +133,13 @@ type BuildChunkFn = (params: {offset: bigint; size: bigint}) => Chunk; const downloadMetadata = async ({ folder, + snapshotIdText, log, ...rest -}: SnapshotParams & {folder: string} & Log): Promise<{ +}: SnapshotParams & {folder: string; snapshotIdText: string} & Log): Promise<{ metadata: ReadCanisterSnapshotMetadataResponse; }> => { - log('Downloading the snapshot metadata...'); + log(`[Metadata] Downloading snapshot ${snapshotIdText}...`); const metadata = await readCanisterSnapshotMetadata(rest); From f5f8ec52ee3e01c839697eeafe9bae50834a5faf Mon Sep 17 00:00:00 2001 From: David Dal Busco Date: Tue, 30 Sep 2025 15:33:38 +0200 Subject: [PATCH 10/26] feat: more chunks --- src/services/modules/snapshot/snapshot.offline.services.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/services/modules/snapshot/snapshot.offline.services.ts b/src/services/modules/snapshot/snapshot.offline.services.ts index 90ba2fc1..c8dfb760 100644 --- a/src/services/modules/snapshot/snapshot.offline.services.ts +++ b/src/services/modules/snapshot/snapshot.offline.services.ts @@ -201,7 +201,7 @@ const downloadMemoryChunks = async ({ for await (const progress of batchDownloadChunks({ chunks, - limit: 12, + limit: 50, ...params })) { log(`Chunks ${progress.done}/${progress.total} downloaded. Continuing...`); From 7c766c1ca47695a6911fd0be0489a64d19458fdf Mon Sep 17 00:00:00 2001 From: David Dal Busco Date: Tue, 30 Sep 2025 17:19:11 +0200 Subject: [PATCH 11/26] feat: rename and integrate for all modules --- src/commands/snapshot.ts | 6 ++++-- ...ne.services.ts => snapshot.download.services.ts} | 0 .../snapshot/snapshot.mission-control.services.ts | 8 +++++++- .../modules/snapshot/snapshot.orbiter.services.ts | 13 ++++++++++++- src/services/modules/snapshot/snapshot.services.ts | 2 +- 5 files changed, 24 insertions(+), 5 deletions(-) rename src/services/modules/snapshot/{snapshot.offline.services.ts => snapshot.download.services.ts} (100%) diff --git a/src/commands/snapshot.ts b/src/commands/snapshot.ts index 0b944945..887c5c87 100644 --- a/src/commands/snapshot.ts +++ b/src/commands/snapshot.ts @@ -4,11 +4,13 @@ import {logHelpSnapshot} from '../help/snapshot.help'; import { createSnapshotMissionControl, deleteSnapshotMissionControl, + downloadSnapshotMissionControl, restoreSnapshotMissionControl } from '../services/modules/snapshot/snapshot.mission-control.services'; import { createSnapshotOrbiter, deleteSnapshotOrbiter, + downloadSnapshotOrbiter, restoreSnapshotOrbiter } from '../services/modules/snapshot/snapshot.orbiter.services'; import { @@ -50,8 +52,8 @@ export const snapshot = async (args?: string[]) => { await executeSnapshotFn({ args, satelliteFn: downloadSnapshotSatellite, - missionControlFn: deleteSnapshotMissionControl, - orbiterFn: deleteSnapshotOrbiter + missionControlFn: downloadSnapshotMissionControl, + orbiterFn: downloadSnapshotOrbiter }); break; default: diff --git a/src/services/modules/snapshot/snapshot.offline.services.ts b/src/services/modules/snapshot/snapshot.download.services.ts similarity index 100% rename from src/services/modules/snapshot/snapshot.offline.services.ts rename to src/services/modules/snapshot/snapshot.download.services.ts diff --git a/src/services/modules/snapshot/snapshot.mission-control.services.ts b/src/services/modules/snapshot/snapshot.mission-control.services.ts index d064dbfb..136d6fc7 100644 --- a/src/services/modules/snapshot/snapshot.mission-control.services.ts +++ b/src/services/modules/snapshot/snapshot.mission-control.services.ts @@ -2,7 +2,7 @@ import {isNullish} from '@dfinity/utils'; import {red} from 'kleur'; import {getCliMissionControl} from '../../../configs/cli.config'; import type {AssetKey} from '../../../types/asset-key'; -import {createSnapshot, deleteSnapshot, restoreSnapshot} from './snapshot.services'; +import {createSnapshot, deleteSnapshot, downloadSnapshot, restoreSnapshot} from './snapshot.services'; export const createSnapshotMissionControl = async () => { await executeSnapshotFn({ @@ -22,6 +22,12 @@ export const deleteSnapshotMissionControl = async () => { }); }; +export const downloadSnapshotMissionControl = async () => { + await executeSnapshotFn({ + fn: downloadSnapshot + }); +}; + const executeSnapshotFn = async ({ fn }: { diff --git a/src/services/modules/snapshot/snapshot.orbiter.services.ts b/src/services/modules/snapshot/snapshot.orbiter.services.ts index b4df3aaf..04a8a275 100644 --- a/src/services/modules/snapshot/snapshot.orbiter.services.ts +++ b/src/services/modules/snapshot/snapshot.orbiter.services.ts @@ -1,6 +1,11 @@ import {getCliOrbiters} from '../../../configs/cli.config'; import type {AssetKey} from '../../../types/asset-key'; -import {createSnapshot, deleteSnapshot, restoreSnapshot} from './snapshot.services'; +import { + createSnapshot, + deleteSnapshot, + downloadSnapshot, + restoreSnapshot +} from './snapshot.services'; export const createSnapshotOrbiter = async () => { await executeSnapshotFn({ @@ -20,6 +25,12 @@ export const deleteSnapshotOrbiter = async () => { }); }; +export const downloadSnapshotOrbiter = async () => { + await executeSnapshotFn({ + fn: downloadSnapshot + }); +}; + const executeSnapshotFn = async ({ fn }: { diff --git a/src/services/modules/snapshot/snapshot.services.ts b/src/services/modules/snapshot/snapshot.services.ts index b6b84fc9..4d9cd874 100644 --- a/src/services/modules/snapshot/snapshot.services.ts +++ b/src/services/modules/snapshot/snapshot.services.ts @@ -13,7 +13,7 @@ import { import type {AssetKey} from '../../../types/asset-key'; import {displaySegment} from '../../../utils/display.utils'; import {confirmAndExit} from '../../../utils/prompt.utils'; -import {downloadExistingSnapshot} from './snapshot.offline.services'; +import {downloadExistingSnapshot} from './snapshot.download.services'; export const createSnapshot = async ({ canisterId: cId, From 78bf70a0a390d5574c5ab8d32b6aa9f4fbab134b Mon Sep 17 00:00:00 2001 From: David Dal Busco Date: Tue, 30 Sep 2025 17:30:30 +0200 Subject: [PATCH 12/26] feat: smaller limit for batching --- src/services/modules/snapshot/snapshot.download.services.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/services/modules/snapshot/snapshot.download.services.ts b/src/services/modules/snapshot/snapshot.download.services.ts index c8dfb760..737c24ec 100644 --- a/src/services/modules/snapshot/snapshot.download.services.ts +++ b/src/services/modules/snapshot/snapshot.download.services.ts @@ -201,7 +201,7 @@ const downloadMemoryChunks = async ({ for await (const progress of batchDownloadChunks({ chunks, - limit: 50, + limit: 20, ...params })) { log(`Chunks ${progress.done}/${progress.total} downloaded. Continuing...`); From da399ac768971863d400b1e96eb01867a38225ca Mon Sep 17 00:00:00 2001 From: David Dal Busco Date: Tue, 30 Sep 2025 17:30:41 +0200 Subject: [PATCH 13/26] feat: add command to help --- src/help/snapshot.help.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/help/snapshot.help.ts b/src/help/snapshot.help.ts index 8d687a63..157cbb0c 100644 --- a/src/help/snapshot.help.ts +++ b/src/help/snapshot.help.ts @@ -8,8 +8,9 @@ const usage = `Usage: ${green('juno')} ${cyan('snapshot')} ${magenta(' Date: Wed, 1 Oct 2025 14:34:36 +0200 Subject: [PATCH 14/26] feat: init upload --- src/commands/snapshot.ts | 42 ++++++++-- src/constants/help.constants.ts | 2 + src/help/snapshot.help.ts | 3 +- src/help/snapshot.upload.help.ts | 34 ++++++++ src/index.ts | 5 +- .../snapshot/snapshot.download.services.ts | 60 +++++++------- .../snapshot.mission-control.services.ts | 14 +++- .../snapshot/snapshot.orbiter.services.ts | 9 +- .../snapshot/snapshot.satellite.services.ts | 9 +- .../modules/snapshot/snapshot.services.ts | 58 ++++++++++--- .../snapshot/snapshot.upload.services.ts | 83 +++++++++++++++++++ 11 files changed, 261 insertions(+), 58 deletions(-) create mode 100644 src/help/snapshot.upload.help.ts create mode 100644 src/services/modules/snapshot/snapshot.upload.services.ts diff --git a/src/commands/snapshot.ts b/src/commands/snapshot.ts index 887c5c87..002d8c43 100644 --- a/src/commands/snapshot.ts +++ b/src/commands/snapshot.ts @@ -5,20 +5,24 @@ import { createSnapshotMissionControl, deleteSnapshotMissionControl, downloadSnapshotMissionControl, - restoreSnapshotMissionControl + restoreSnapshotMissionControl, + uploadSnapshotMissionControl } from '../services/modules/snapshot/snapshot.mission-control.services'; import { createSnapshotOrbiter, deleteSnapshotOrbiter, downloadSnapshotOrbiter, - restoreSnapshotOrbiter + restoreSnapshotOrbiter, + uploadSnapshotOrbiter } from '../services/modules/snapshot/snapshot.orbiter.services'; import { createSnapshotSatellite, deleteSnapshotSatellite, downloadSnapshotSatellite, - restoreSnapshotSatellite + restoreSnapshotSatellite, + uploadSnapshotSatellite } from '../services/modules/snapshot/snapshot.satellite.services'; +import {logHelpSnapshotUpload} from '../help/snapshot.upload.help'; export const snapshot = async (args?: string[]) => { const [subCommand] = args ?? []; @@ -56,6 +60,14 @@ export const snapshot = async (args?: string[]) => { orbiterFn: downloadSnapshotOrbiter }); break; + case 'upload': + await executeSnapshotFn({ + args, + satelliteFn: uploadSnapshotSatellite, + missionControlFn: uploadSnapshotMissionControl, + orbiterFn: uploadSnapshotOrbiter + }); + break; default: console.log(red('Unknown subcommand.')); logHelpSnapshot(args); @@ -69,27 +81,39 @@ const executeSnapshotFn = async ({ orbiterFn }: { args?: string[]; - satelliteFn: () => Promise; - missionControlFn: () => Promise; - orbiterFn: () => Promise; + satelliteFn: (args?: string[]) => Promise; + missionControlFn: (args?: string[]) => Promise; + orbiterFn: (args?: string[]) => Promise; }) => { const target = nextArg({args, option: '-t'}) ?? nextArg({args, option: '--target'}); switch (target) { case 's': case 'satellite': - await satelliteFn(); + await satelliteFn(args); break; case 'm': case 'mission-control': - await missionControlFn(); + await missionControlFn(args); break; case 'o': case 'orbiter': - await orbiterFn(); + await orbiterFn(args); break; default: console.log(red('Unknown target.')); logHelpSnapshot(args); } }; + +export const helpSnapshot = (args?: string[]) => { + const [subCommand] = args ?? []; + + switch (subCommand) { + case 'upload': + logHelpSnapshotUpload(args); + break; + default: + logHelpSnapshot(args); + } +} \ No newline at end of file diff --git a/src/constants/help.constants.ts b/src/constants/help.constants.ts index 7b320fb2..a570bb55 100644 --- a/src/constants/help.constants.ts +++ b/src/constants/help.constants.ts @@ -48,6 +48,8 @@ export const CHANGES_LIST_DESCRIPTION = 'List all submitted or applied changes.' export const CHANGES_APPLY_DESCRIPTION = 'Apply a submitted change.'; export const CHANGES_REJECT_DESCRIPTION = 'Reject a change.'; +export const SNAPSHOT_UPLOAD_DESCRIPTION = 'Upload a snapshot from offline files.'; + export const OPTION_KEEP_STAGED = `${yellow('-k, --keep-staged')} Keep staged assets in memory after applying the change.`; export const OPTION_HASH = `${yellow('--hash')} The expected hash of all included changes (for verification).`; export const OPTION_HELP = `${yellow('-h, --help')} Output usage information.`; diff --git a/src/help/snapshot.help.ts b/src/help/snapshot.help.ts index 157cbb0c..22c61737 100644 --- a/src/help/snapshot.help.ts +++ b/src/help/snapshot.help.ts @@ -1,5 +1,5 @@ import {cyan, green, magenta, yellow} from 'kleur'; -import {OPTIONS_ENV, OPTION_HELP, SNAPSHOT_DESCRIPTION} from '../constants/help.constants'; +import {OPTIONS_ENV, OPTION_HELP, SNAPSHOT_DESCRIPTION, SNAPSHOT_UPLOAD_DESCRIPTION} from '../constants/help.constants'; import {helpOutput} from './common.help'; import {TITLE} from './help'; import {TARGET_OPTION_NOTE, targetOption} from './target.help'; @@ -10,6 +10,7 @@ Subcommands: ${magenta('create')} Create a snapshot of your current state. ${magenta('delete')} Delete an existing snapshot. ${magenta('download')} Download a snapshot to offline files. + ${magenta('upload')} ${SNAPSHOT_UPLOAD_DESCRIPTION} ${magenta('restore')} Restore a previously created snapshot. Options: diff --git a/src/help/snapshot.upload.help.ts b/src/help/snapshot.upload.help.ts new file mode 100644 index 00000000..5be9a86a --- /dev/null +++ b/src/help/snapshot.upload.help.ts @@ -0,0 +1,34 @@ +import {cyan, green, magenta, yellow} from 'kleur'; +import { + FUNCTIONS_BUILD_DESCRIPTION, + FUNCTIONS_BUILD_NOTES, + OPTION_HELP, + OPTIONS_BUILD, OPTIONS_ENV, SNAPSHOT_UPLOAD_DESCRIPTION +} from '../constants/help.constants'; +import {helpOutput} from './common.help'; +import {TITLE} from './help'; + +const usage = `Usage: ${green('juno')} ${cyan('snapshot')} ${magenta('upload')} ${yellow('[options]')} + +Options: + ${yellow('--dir')} Path to the snapshot directory that contains the metadata.json and chunks. + ${OPTIONS_ENV} + ${OPTION_HELP}`; + +const doc = `${SNAPSHOT_UPLOAD_DESCRIPTION} + +\`\`\` +${usage} +\`\`\` +`; + +const help = `${TITLE} + +${SNAPSHOT_UPLOAD_DESCRIPTION} + +${usage} +`; + +export const logHelpSnapshotUpload = (args?: string[]) => { + console.log(helpOutput(args) === 'doc' ? doc : help); +}; diff --git a/src/index.ts b/src/index.ts index b62b4399..e85a213c 100644 --- a/src/index.ts +++ b/src/index.ts @@ -12,7 +12,7 @@ import {functions, helpFunctions} from './commands/functions'; import {helpHosting, hosting} from './commands/hosting'; import {open} from './commands/open'; import {helpRun, run as runCmd} from './commands/run'; -import {snapshot} from './commands/snapshot'; +import {helpSnapshot, snapshot} from './commands/snapshot'; import {startStop} from './commands/start-stop'; import {status} from './commands/status'; import {upgrade} from './commands/upgrade'; @@ -22,7 +22,6 @@ import {help} from './help/help'; import {logHelpLogin} from './help/login.help'; import {logHelpLogout} from './help/logout.help'; import {logHelpOpen} from './help/open.help'; -import {logHelpSnapshot} from './help/snapshot.help'; import {logHelpStart} from './help/start.help'; import {logHelpStatus} from './help/status.help'; import {logHelpStop} from './help/stop.help'; @@ -94,7 +93,7 @@ export const run = async () => { helpFunctions(args); break; case 'snapshot': - logHelpSnapshot(args); + helpSnapshot(args); break; case 'init': helpInit(args); diff --git a/src/services/modules/snapshot/snapshot.download.services.ts b/src/services/modules/snapshot/snapshot.download.services.ts index 737c24ec..b85621ed 100644 --- a/src/services/modules/snapshot/snapshot.download.services.ts +++ b/src/services/modules/snapshot/snapshot.download.services.ts @@ -1,10 +1,7 @@ -import { - type CanisterSnapshotMetadataKind, - encodeSnapshotId, - type snapshot_id -} from '@dfinity/ic-management'; +import type {snapshot_id} from '@dfinity/ic-management'; +import {type CanisterSnapshotMetadataKind, encodeSnapshotId} from '@dfinity/ic-management'; import type {ReadCanisterSnapshotMetadataResponse} from '@dfinity/ic-management/dist/types/types/snapshot.responses'; -import {type Principal} from '@dfinity/principal'; +import type {Principal} from '@dfinity/principal'; import {arrayOfNumberToUint8Array, jsonReplacer} from '@dfinity/utils'; import {red} from 'kleur'; import {existsSync} from 'node:fs'; @@ -13,14 +10,32 @@ import {join} from 'node:path'; import ora from 'ora'; import {readCanisterSnapshotData, readCanisterSnapshotMetadata} from '../../../api/ic.api'; import {SNAPSHOT_CHUNK_SIZE, SNAPSHOTS_PATH} from '../../../constants/snapshot.constants'; -import type {AssetKey} from '../../../types/asset-key'; +import {AssetKey} from '../../../types/asset-key'; import {displaySegment} from '../../../utils/display.utils'; +// We override the ic-mgmt interface because we solely want snapshotId as Principal here interface SnapshotParams { canisterId: Principal; snapshotId: snapshot_id; } +type Chunk = CanisterSnapshotMetadataKind; +type OrderedChunk = Chunk & {orderId: number}; + +type BuildChunkFn = (params: {offset: bigint; size: bigint}) => Chunk; + +class SnapshotFsFolderError extends Error { + constructor(public readonly folder: string) { + super(); + } +} + +// A handy wrapper to pass down a function that updates +// the spinner log. +interface SnapshotLog { + log: (text: string) => void; +} + export const downloadExistingSnapshot = async ({ segment, ...params @@ -57,29 +72,19 @@ export const downloadExistingSnapshot = async ({ } }; -class SnapshotFsError extends Error { - constructor(public readonly folder: string) { - super(); - } -} - -interface Log { - log: (text: string) => void; -} - const downloadSnapshotMetadataAndMemory = async ({ snapshotId, log, ...rest -}: SnapshotParams & Log): Promise< +}: SnapshotParams & SnapshotLog): Promise< | {status: 'success'; snapshotIdText: string; folder: string} - | {status: 'error'; err: SnapshotFsError} + | {status: 'error'; err: SnapshotFsFolderError} > => { const snapshotIdText = `0x${encodeSnapshotId(snapshotId)}`; const folder = join(SNAPSHOTS_PATH, snapshotIdText); if (existsSync(folder)) { - return {status: 'error', err: new SnapshotFsError(folder)}; + return {status: 'error', err: new SnapshotFsFolderError(folder)}; } await mkdir(folder, {recursive: true}); @@ -126,17 +131,12 @@ const downloadSnapshotMetadataAndMemory = async ({ return {status: 'success', snapshotIdText, folder}; }; -type Chunk = CanisterSnapshotMetadataKind; -type OrderedChunk = Chunk & {orderId: number}; - -type BuildChunkFn = (params: {offset: bigint; size: bigint}) => Chunk; - const downloadMetadata = async ({ folder, snapshotIdText, log, ...rest -}: SnapshotParams & {folder: string; snapshotIdText: string} & Log): Promise<{ +}: SnapshotParams & {folder: string; snapshotIdText: string} & SnapshotLog): Promise<{ metadata: ReadCanisterSnapshotMetadataResponse; }> => { log(`[Metadata] Downloading snapshot ${snapshotIdText}...`); @@ -153,7 +153,7 @@ const assertSizeAndDownloadChunks = async ({ size, log, ...params -}: SnapshotParams & {folder: string; size: bigint; build: BuildChunkFn} & Log) => { +}: SnapshotParams & {folder: string; size: bigint; build: BuildChunkFn} & SnapshotLog) => { if (size === 0n) { log('No chunks to download (size = 0). Skipping.'); await new Promise((resolve) => setTimeout(resolve, 2500)); @@ -175,7 +175,7 @@ const assertAndDownloadWasmChunks = async ({ ReadCanisterSnapshotMetadataResponse, 'wasmChunkStore' > & - Log) => { + SnapshotLog) => { if (wasmChunkStore.length === 0) { log('No chunks to download (length = 0). Skipping.'); await new Promise((resolve) => setTimeout(resolve, 2500)); @@ -194,7 +194,7 @@ const downloadMemoryChunks = async ({ build, log, ...params -}: SnapshotParams & {folder: string; size: bigint; build: BuildChunkFn} & Log) => { +}: SnapshotParams & {folder: string; size: bigint; build: BuildChunkFn} & SnapshotLog) => { const {chunks} = prepareDownloadChunks({size, build}); log('Downloading chunks...'); @@ -216,7 +216,7 @@ const downloadWasmChunks = async ({ ReadCanisterSnapshotMetadataResponse, 'wasmChunkStore' > & - Log) => { + SnapshotLog) => { log('[WASM store] Downloading chunks...'); for await (const progress of batchDownloadChunks({ diff --git a/src/services/modules/snapshot/snapshot.mission-control.services.ts b/src/services/modules/snapshot/snapshot.mission-control.services.ts index 136d6fc7..85937e90 100644 --- a/src/services/modules/snapshot/snapshot.mission-control.services.ts +++ b/src/services/modules/snapshot/snapshot.mission-control.services.ts @@ -2,7 +2,13 @@ import {isNullish} from '@dfinity/utils'; import {red} from 'kleur'; import {getCliMissionControl} from '../../../configs/cli.config'; import type {AssetKey} from '../../../types/asset-key'; -import {createSnapshot, deleteSnapshot, downloadSnapshot, restoreSnapshot} from './snapshot.services'; +import { + createSnapshot, + deleteSnapshot, + downloadSnapshot, + restoreSnapshot, + uploadSnapshot +} from './snapshot.services'; export const createSnapshotMissionControl = async () => { await executeSnapshotFn({ @@ -28,6 +34,12 @@ export const downloadSnapshotMissionControl = async () => { }); }; +export const uploadSnapshotMissionControl = async (args?: string[]) => { + await executeSnapshotFn({ + fn: (params) => uploadSnapshot({...params, ...args}), + }); +}; + const executeSnapshotFn = async ({ fn }: { diff --git a/src/services/modules/snapshot/snapshot.orbiter.services.ts b/src/services/modules/snapshot/snapshot.orbiter.services.ts index 04a8a275..2a48cc76 100644 --- a/src/services/modules/snapshot/snapshot.orbiter.services.ts +++ b/src/services/modules/snapshot/snapshot.orbiter.services.ts @@ -4,7 +4,8 @@ import { createSnapshot, deleteSnapshot, downloadSnapshot, - restoreSnapshot + restoreSnapshot, + uploadSnapshot } from './snapshot.services'; export const createSnapshotOrbiter = async () => { @@ -31,6 +32,12 @@ export const downloadSnapshotOrbiter = async () => { }); }; +export const uploadSnapshotOrbiter = async (args?: string[]) => { + await executeSnapshotFn({ + fn: (params) => uploadSnapshot({...params, ...args}), + }); +}; + const executeSnapshotFn = async ({ fn }: { diff --git a/src/services/modules/snapshot/snapshot.satellite.services.ts b/src/services/modules/snapshot/snapshot.satellite.services.ts index d204cd31..0f642521 100644 --- a/src/services/modules/snapshot/snapshot.satellite.services.ts +++ b/src/services/modules/snapshot/snapshot.satellite.services.ts @@ -7,7 +7,8 @@ import { createSnapshot, deleteSnapshot, downloadSnapshot, - restoreSnapshot + restoreSnapshot, + uploadSnapshot } from './snapshot.services'; export const createSnapshotSatellite = async () => { @@ -34,6 +35,12 @@ export const downloadSnapshotSatellite = async () => { }); }; +export const uploadSnapshotSatellite = async (args?: string[]) => { + await executeSnapshotFn({ + fn: (params) => uploadSnapshot({...params, ...args}), + }); +}; + const executeSnapshotFn = async ({ fn }: { diff --git a/src/services/modules/snapshot/snapshot.services.ts b/src/services/modules/snapshot/snapshot.services.ts index 4d9cd874..bb569b6e 100644 --- a/src/services/modules/snapshot/snapshot.services.ts +++ b/src/services/modules/snapshot/snapshot.services.ts @@ -14,6 +14,7 @@ import type {AssetKey} from '../../../types/asset-key'; import {displaySegment} from '../../../utils/display.utils'; import {confirmAndExit} from '../../../utils/prompt.utils'; import {downloadExistingSnapshot} from './snapshot.download.services'; +import {uploadExistingSnapshot} from './snapshot.upload.services'; export const createSnapshot = async ({ canisterId: cId, @@ -24,17 +25,11 @@ export const createSnapshot = async ({ }) => { const canisterId = Principal.fromText(cId); - const existingSnapshotId = await loadSnapshot({canisterId}); - - if (nonNullish(existingSnapshotId)) { - await confirmAndExit( - `A snapshot for your ${displaySegment(segment)} already exists with ID 0x${encodeSnapshotId(existingSnapshotId)}. Do you want to overwrite it?` - ); - } + const {snapshotId} = await loadSnapshotAndAssertOverwrite({canisterId, segment}); await takeSnapshot({ canisterId, - snapshotId: existingSnapshotId, + snapshotId, segment }); }; @@ -48,7 +43,7 @@ export const restoreSnapshot = async ({ }) => { const canisterId = Principal.fromText(cId); - const result = await assertAndLoadSnapshot({canisterId, segment}); + const result = await loadSnapshotAndAssertExist({canisterId, segment}); if (result.result === 'not_found') { return; @@ -76,7 +71,7 @@ export const deleteSnapshot = async ({ }) => { const canisterId = Principal.fromText(cId); - const result = await assertAndLoadSnapshot({canisterId, segment}); + const result = await loadSnapshotAndAssertExist({canisterId, segment}); if (result.result === 'not_found') { return; @@ -104,7 +99,7 @@ export const downloadSnapshot = async ({ }) => { const canisterId = Principal.fromText(cId); - const result = await assertAndLoadSnapshot({canisterId, segment}); + const result = await loadSnapshotAndAssertExist({canisterId, segment}); if (result.result === 'not_found') { return; @@ -119,6 +114,27 @@ export const downloadSnapshot = async ({ }); }; +export const uploadSnapshot = async ({ + canisterId: cId, + segment, + args +}: { + canisterId: string; + segment: AssetKey; + args?: string[]; +}) => { + const canisterId = Principal.fromText(cId); + + const {snapshotId} = await loadSnapshotAndAssertOverwrite({canisterId, segment}); + + await uploadExistingSnapshot({ + canisterId, + snapshotId, + segment, + args + }); +}; + const restoreExistingSnapshot = async ({ segment, ...rest @@ -194,7 +210,7 @@ const loadSnapshot = async ({ } }; -const assertAndLoadSnapshot = async ({ +const loadSnapshotAndAssertExist = async ({ canisterId, segment }: { @@ -210,3 +226,21 @@ const assertAndLoadSnapshot = async ({ return {result: 'ok', snapshotId: existingSnapshotId}; }; + +const loadSnapshotAndAssertOverwrite = async ({ + canisterId, + segment +}: { + canisterId: Principal; + segment: AssetKey; +}): Promise<{snapshotId: snapshot_id | undefined}> => { + const existingSnapshotId = await loadSnapshot({canisterId}); + + if (nonNullish(existingSnapshotId)) { + await confirmAndExit( + `A snapshot for your ${displaySegment(segment)} already exists with ID 0x${encodeSnapshotId(existingSnapshotId)}. Do you want to overwrite it?` + ); + } + + return {snapshotId: existingSnapshotId}; +}; diff --git a/src/services/modules/snapshot/snapshot.upload.services.ts b/src/services/modules/snapshot/snapshot.upload.services.ts new file mode 100644 index 00000000..c79aa1ac --- /dev/null +++ b/src/services/modules/snapshot/snapshot.upload.services.ts @@ -0,0 +1,83 @@ +import type {snapshot_id} from '@dfinity/ic-management'; +import type {Principal} from '@dfinity/principal'; +import {isEmptyString} from '@dfinity/utils'; +import {nextArg} from '@junobuild/cli-tools'; +import {red, yellow} from 'kleur'; +import {existsSync, lstatSync} from 'node:fs'; +import ora from 'ora'; +import type {AssetKey} from '../../../types/asset-key'; +import {displaySegment} from '../../../utils/display.utils'; + +// We override the ic-mgmt interface because we solely want snapshotId as Principal here +interface SnapshotParams { + canisterId: Principal; + snapshotId?: snapshot_id; +} + +// A handy wrapper to pass down a function that updates +// the spinner log. +interface SnapshotLog { + log: (text: string) => void; +} + +export const uploadExistingSnapshot = async ({ + segment, + args, + ...params +}: SnapshotParams & { + segment: AssetKey; + args?: string[]; +}): Promise => { + const folder = nextArg({args, option: '--dir'}); + + if (isEmptyString(folder)) { + console.log( + `You did not provide a ${yellow('directory')} that contains metadata.json and chunks to upload.` + ); + return; + } + + if (!existsSync(folder)) { + console.log(`The directory ${yellow('directory')} does not exist.`); + return; + } + + if (!lstatSync(folder).isDirectory()) { + console.log(red(`${folder} is not a directory.`)); + return; + } + + // TODO: extract assertions + // TODO: more assertion like is there a metadata.json and chunk files + + const spinner = ora('Uploading the snapshot...').start(); + + try { + const result = await uploadSnapshotMetadataAndMemory({ + ...params, + log: (text) => (spinner.text = text) + }); + + spinner.stop(); + + const {snapshotIdText} = result; + + console.log( + `✅ The snapshot ${snapshotIdText} for your ${displaySegment(segment)} has been uploaded.` + ); + } catch (error: unknown) { + spinner.stop(); + + throw error; + } +}; + +const uploadSnapshotMetadataAndMemory = async ({ + snapshotId, + log, + ...rest +}: SnapshotParams & SnapshotLog): Promise<{snapshotIdText: string}> => { + // TODO: yep todo + + return {snapshotIdText: ""} +}; From 7499a971133602005b38fcb0dc866e153bbcd453 Mon Sep 17 00:00:00 2001 From: David Dal Busco Date: Wed, 1 Oct 2025 18:34:13 +0200 Subject: [PATCH 15/26] feat: write stream to a single file --- .../snapshot/snapshot.download.services.ts | 180 +++++++++++------- 1 file changed, 111 insertions(+), 69 deletions(-) diff --git a/src/services/modules/snapshot/snapshot.download.services.ts b/src/services/modules/snapshot/snapshot.download.services.ts index b85621ed..a78e16c9 100644 --- a/src/services/modules/snapshot/snapshot.download.services.ts +++ b/src/services/modules/snapshot/snapshot.download.services.ts @@ -4,9 +4,11 @@ import type {ReadCanisterSnapshotMetadataResponse} from '@dfinity/ic-management/ import type {Principal} from '@dfinity/principal'; import {arrayOfNumberToUint8Array, jsonReplacer} from '@dfinity/utils'; import {red} from 'kleur'; -import {existsSync} from 'node:fs'; +import {createWriteStream, existsSync} from 'node:fs'; import {mkdir, writeFile} from 'node:fs/promises'; -import {join} from 'node:path'; +import {join, relative} from 'node:path'; +import {Readable, Transform} from 'node:stream'; +import {pipeline} from 'node:stream/promises'; import ora from 'ora'; import {readCanisterSnapshotData, readCanisterSnapshotMetadata} from '../../../api/ic.api'; import {SNAPSHOT_CHUNK_SIZE, SNAPSHOTS_PATH} from '../../../constants/snapshot.constants'; @@ -19,10 +21,9 @@ interface SnapshotParams { snapshotId: snapshot_id; } -type Chunk = CanisterSnapshotMetadataKind; -type OrderedChunk = Chunk & {orderId: number}; - -type BuildChunkFn = (params: {offset: bigint; size: bigint}) => Chunk; +type RequestedChunk = CanisterSnapshotMetadataKind; +type DownloadedChunk = Uint8Array; +type BuildChunkFn = (params: {offset: bigint; size: bigint}) => RequestedChunk; class SnapshotFsFolderError extends Error { constructor(public readonly folder: string) { @@ -36,6 +37,11 @@ interface SnapshotLog { log: (text: string) => void; } +interface BatchResult { + downloadedChunks: DownloadedChunk[]; + progress: {index: number; done: number; total: number}; +} + export const downloadExistingSnapshot = async ({ segment, ...params @@ -95,36 +101,40 @@ const downloadSnapshotMetadataAndMemory = async ({ await assertSizeAndDownloadChunks({ folder, + filename: 'wasm-code', snapshotId, size: wasmModuleSize, build: (param) => ({wasmModule: param}), - log: (text) => log(`[WASM module] ${text}`), + log, ...rest }); await assertSizeAndDownloadChunks({ folder, + filename: 'heap', snapshotId, size: wasmMemorySize, build: (param) => ({wasmMemory: param}), - log: (text) => log(`[Heap memory] ${text}`), + log, ...rest }); await assertSizeAndDownloadChunks({ folder, + filename: 'stable', snapshotId, size: stableMemorySize, build: (param) => ({stableMemory: param}), - log: (text) => log(`[Stable memory] ${text}`), + log, ...rest }); await assertAndDownloadWasmChunks({ folder, + filename: 'chunks-store', snapshotId, wasmChunkStore, - log: (text) => log(`[WASM store] ${text}`), + log, ...rest }); @@ -139,10 +149,11 @@ const downloadMetadata = async ({ }: SnapshotParams & {folder: string; snapshotIdText: string} & SnapshotLog): Promise<{ metadata: ReadCanisterSnapshotMetadataResponse; }> => { - log(`[Metadata] Downloading snapshot ${snapshotIdText}...`); + log(`Downloading snapshot metadata ${snapshotIdText}...`); const metadata = await readCanisterSnapshotMetadata(rest); + // TODO: write the metadata at the end of the process. It's safer. const destination = join(folder, 'metadata.json'); await writeFile(destination, JSON.stringify(metadata, jsonReplacer, 2), 'utf-8'); @@ -152,10 +163,16 @@ const downloadMetadata = async ({ const assertSizeAndDownloadChunks = async ({ size, log, + filename, ...params -}: SnapshotParams & {folder: string; size: bigint; build: BuildChunkFn} & SnapshotLog) => { +}: SnapshotParams & { + folder: string; + filename: string; + size: bigint; + build: BuildChunkFn; +} & SnapshotLog) => { if (size === 0n) { - log('No chunks to download (size = 0). Skipping.'); + log(`No chunks to download for ${filename} (size = 0). Skipping.`); await new Promise((resolve) => setTimeout(resolve, 2500)); return; } @@ -163,6 +180,7 @@ const assertSizeAndDownloadChunks = async ({ await downloadMemoryChunks({ size, log, + filename, ...params }); }; @@ -171,13 +189,13 @@ const assertAndDownloadWasmChunks = async ({ wasmChunkStore, log, ...params -}: SnapshotParams & {folder: string} & Pick< +}: SnapshotParams & {folder: string; filename: string} & Pick< ReadCanisterSnapshotMetadataResponse, 'wasmChunkStore' > & SnapshotLog) => { if (wasmChunkStore.length === 0) { - log('No chunks to download (length = 0). Skipping.'); + log('Nothing to download from the WASM chunks store (length = 0). Skipping.'); await new Promise((resolve) => setTimeout(resolve, 2500)); return; } @@ -193,39 +211,80 @@ const downloadMemoryChunks = async ({ size, build, log, + folder, + filename, ...params -}: SnapshotParams & {folder: string; size: bigint; build: BuildChunkFn} & SnapshotLog) => { +}: SnapshotParams & { + folder: string; + filename: string; + size: bigint; + build: BuildChunkFn; +} & SnapshotLog) => { const {chunks} = prepareDownloadChunks({size, build}); - log('Downloading chunks...'); + const readable = Readable.from( + batchDownloadChunks({ + chunks, + limit: 20, + ...params + }) + ); - for await (const progress of batchDownloadChunks({ - chunks, - limit: 20, - ...params - })) { - log(`Chunks ${progress.done}/${progress.total} downloaded. Continuing...`); - } + await downloadAndWrite({readable, folder, filename, log}); }; const downloadWasmChunks = async ({ wasmChunkStore, log, + folder, + filename, ...params -}: SnapshotParams & {folder: string} & Pick< +}: SnapshotParams & {folder: string; filename: string} & Pick< ReadCanisterSnapshotMetadataResponse, 'wasmChunkStore' > & SnapshotLog) => { - log('[WASM store] Downloading chunks...'); + const readable = Readable.from( + batchDownloadChunks({ + chunks: wasmChunkStore.map((chunk, orderId) => ({wasmChunk: chunk, orderId})), + limit: 12, + ...params + }) + ); - for await (const progress of batchDownloadChunks({ - chunks: wasmChunkStore.map((chunk, orderId) => ({wasmChunk: chunk, orderId})), - limit: 12, - ...params - })) { - log(`Chunks ${progress.done}/${progress.total} downloaded. Continuing...`); - } + await downloadAndWrite({readable, folder, filename, log}); +}; + +const downloadAndWrite = async ({ + readable, + folder, + filename, + log +}: {readable: Readable; folder: string; filename: string} & SnapshotLog) => { + // Note: we would not win much at gzipping the data. + const destination = join(folder, `${filename}.bin`); + + log(`Downloading chunks to ${relative(process.cwd(), destination)}...`); + + const transformer = new Transform({ + objectMode: true, + writableObjectMode: true, + readableObjectMode: false, + transform({downloadedChunks: chunks, progress}: BatchResult, _enc, cb) { + try { + log(`Chunks ${progress.done}/${progress.total} downloaded. Continuing...`); + + for (const chunk of chunks) { + this.push(chunk); + } + cb(); + } catch (err: unknown) { + cb(err as Error); + } + } + }); + + await pipeline(readable, transformer, createWriteStream(destination)); }; const prepareDownloadChunks = ({ @@ -234,10 +293,8 @@ const prepareDownloadChunks = ({ }: { size: bigint; build: BuildChunkFn; -}): {chunks: OrderedChunk[]} => { - let orderId = 0; - - const chunks: OrderedChunk[] = []; +}): {chunks: RequestedChunk[]} => { + const chunks: RequestedChunk[] = []; for (let offset = 0n; offset < totalSize; offset += SNAPSHOT_CHUNK_SIZE) { const size = @@ -247,11 +304,8 @@ const prepareDownloadChunks = ({ ...build({ offset, size - }), - orderId + }) }); - - orderId += 1; } return {chunks}; @@ -261,50 +315,38 @@ async function* batchDownloadChunks({ chunks, limit = 12, ...params -}: SnapshotChunkFsParams & { - chunks: OrderedChunk[]; +}: SnapshotParams & { + chunks: RequestedChunk[]; limit?: number; -}): AsyncGenerator<{index: number; done: number; total: number}, void> { +}): AsyncGenerator { const total = chunks.length; for (let i = 0; i < total; i = i + limit) { const batch = chunks.slice(i, i + limit); - await Promise.all( - batch.map(async (requestChunk) => { - await downloadChunk({ + const downloadedChunks = await Promise.all( + batch.map((requestChunk) => + downloadChunk({ ...params, chunk: requestChunk - }); - }) + }) + ) ); - yield {index: i, done: Math.min(i + limit, total), total}; + yield {downloadedChunks, progress: {index: i, done: Math.min(i + limit, total), total}}; } } -interface SnapshotChunkFsParams extends SnapshotParams { - folder: string; -} - const downloadChunk = async ({ - chunk: {orderId, ...kind}, - folder, + chunk: kind, ...rest -}: SnapshotChunkFsParams & { - chunk: OrderedChunk; -}): Promise => { +}: SnapshotParams & { + chunk: RequestedChunk; +}): Promise => { const {chunk: downloadedChunk} = await readCanisterSnapshotData({ ...rest, kind }); - const filename = Object.keys(kind)[0].toLowerCase(); - - // Note: we would not win much at gzipping the data. - const destination = join(folder, `${filename}-${orderId}.bin`); - await writeFile( - destination, - downloadedChunk instanceof Uint8Array - ? downloadedChunk - : arrayOfNumberToUint8Array(downloadedChunk) - ); + return downloadedChunk instanceof Uint8Array + ? downloadedChunk + : arrayOfNumberToUint8Array(downloadedChunk); }; From ef378d1adeb45dd6a22a6b744c7838b6ca1d383d Mon Sep 17 00:00:00 2001 From: David Dal Busco Date: Thu, 2 Oct 2025 08:55:38 +0200 Subject: [PATCH 16/26] feat: assert size and write metadata once the process is complete --- .../snapshot/snapshot.download.services.ts | 176 +++++++++++++----- src/types/snapshot.ts | 69 +++++++ 2 files changed, 197 insertions(+), 48 deletions(-) create mode 100644 src/types/snapshot.ts diff --git a/src/services/modules/snapshot/snapshot.download.services.ts b/src/services/modules/snapshot/snapshot.download.services.ts index a78e16c9..6a19043c 100644 --- a/src/services/modules/snapshot/snapshot.download.services.ts +++ b/src/services/modules/snapshot/snapshot.download.services.ts @@ -4,7 +4,8 @@ import type {ReadCanisterSnapshotMetadataResponse} from '@dfinity/ic-management/ import type {Principal} from '@dfinity/principal'; import {arrayOfNumberToUint8Array, jsonReplacer} from '@dfinity/utils'; import {red} from 'kleur'; -import {createWriteStream, existsSync} from 'node:fs'; +import {createHash} from 'node:crypto'; +import {createWriteStream, existsSync, statSync} from 'node:fs'; import {mkdir, writeFile} from 'node:fs/promises'; import {join, relative} from 'node:path'; import {Readable, Transform} from 'node:stream'; @@ -13,6 +14,7 @@ import ora from 'ora'; import {readCanisterSnapshotData, readCanisterSnapshotMetadata} from '../../../api/ic.api'; import {SNAPSHOT_CHUNK_SIZE, SNAPSHOTS_PATH} from '../../../constants/snapshot.constants'; import {AssetKey} from '../../../types/asset-key'; +import {SnapshotFile, SnapshotFilename, SnapshotMetadata} from '../../../types/snapshot'; import {displaySegment} from '../../../utils/display.utils'; // We override the ic-mgmt interface because we solely want snapshotId as Principal here @@ -31,6 +33,16 @@ class SnapshotFsFolderError extends Error { } } +class SnapshotFsSizeError extends Error { + constructor( + public readonly filename: string, + public readonly expectedSize: bigint, + public readonly downloadedSize: bigint + ) { + super(); + } +} + // A handy wrapper to pass down a function that updates // the spinner log. interface SnapshotLog { @@ -48,7 +60,7 @@ export const downloadExistingSnapshot = async ({ }: SnapshotParams & { segment: AssetKey; }): Promise => { - const spinner = ora('Downloading the snapshot...').start(); + const spinner = ora().start(); try { const result = await downloadSnapshotMetadataAndMemory({ @@ -58,13 +70,6 @@ export const downloadExistingSnapshot = async ({ spinner.stop(); - if (result.status === 'error') { - console.log( - `${red('Cannot proceed with download.')}\nDestination ${result.err.folder} already exists.` - ); - return; - } - const {snapshotIdText, folder} = result; console.log( @@ -74,6 +79,20 @@ export const downloadExistingSnapshot = async ({ } catch (error: unknown) { spinner.stop(); + if (error instanceof SnapshotFsFolderError) { + console.log( + `${red('Cannot proceed with download.')}\nDestination ${error.folder} already exists.` + ); + return; + } + + if (error instanceof SnapshotFsSizeError) { + console.log( + `${red('Download size mismatch.')}\n${error.filename}: expected ${error.expectedSize} bytes, got ${error.downloadedSize} bytes.` + ); + return; + } + throw error; } }; @@ -82,26 +101,29 @@ const downloadSnapshotMetadataAndMemory = async ({ snapshotId, log, ...rest -}: SnapshotParams & SnapshotLog): Promise< - | {status: 'success'; snapshotIdText: string; folder: string} - | {status: 'error'; err: SnapshotFsFolderError} -> => { +}: SnapshotParams & SnapshotLog): Promise<{ + status: 'success'; + snapshotIdText: string; + folder: string; +}> => { const snapshotIdText = `0x${encodeSnapshotId(snapshotId)}`; const folder = join(SNAPSHOTS_PATH, snapshotIdText); if (existsSync(folder)) { - return {status: 'error', err: new SnapshotFsFolderError(folder)}; + throw new SnapshotFsFolderError(folder); } await mkdir(folder, {recursive: true}); - const { - metadata: {wasmModuleSize, wasmMemorySize, stableMemorySize, wasmChunkStore} - } = await downloadMetadata({folder, snapshotId, log, snapshotIdText, ...rest}); + // 1. Load the snapshot metadata + const {metadata} = await loadMetadata({snapshotId, log, snapshotIdText, ...rest}); + + // 2. Download the snapshot data (WASM program code, heap and stable memory, WASM chunks store) + const {wasmModuleSize, wasmMemorySize, stableMemorySize, wasmChunkStore} = metadata; - await assertSizeAndDownloadChunks({ + const wasmModuleResult = await assertSizeAndDownloadChunks({ folder, - filename: 'wasm-code', + filename: 'wasm-code.bin', snapshotId, size: wasmModuleSize, build: (param) => ({wasmModule: param}), @@ -109,9 +131,9 @@ const downloadSnapshotMetadataAndMemory = async ({ ...rest }); - await assertSizeAndDownloadChunks({ + const wasmMemoryResult = await assertSizeAndDownloadChunks({ folder, - filename: 'heap', + filename: 'heap.bin', snapshotId, size: wasmMemorySize, build: (param) => ({wasmMemory: param}), @@ -119,9 +141,9 @@ const downloadSnapshotMetadataAndMemory = async ({ ...rest }); - await assertSizeAndDownloadChunks({ + const stableMemoryResult = await assertSizeAndDownloadChunks({ folder, - filename: 'stable', + filename: 'stable.bin', snapshotId, size: stableMemorySize, build: (param) => ({stableMemory: param}), @@ -129,35 +151,57 @@ const downloadSnapshotMetadataAndMemory = async ({ ...rest }); - await assertAndDownloadWasmChunks({ + const wasmChunkStoreResult = await assertAndDownloadWasmChunks({ folder, - filename: 'chunks-store', + filename: 'chunks-store.bin', snapshotId, wasmChunkStore, log, ...rest }); + // 3. Save the metadata of the offline snapshot + await saveMetadata({ + log, + folder, + metadata: { + snapshotId: snapshotIdText, + metadata, + data: { + wasmModule: 'ok' === wasmModuleResult.status ? wasmModuleResult.snapshotFile : null, + wasmMemory: 'ok' === wasmMemoryResult.status ? wasmMemoryResult.snapshotFile : null, + stableMemory: 'ok' === stableMemoryResult.status ? stableMemoryResult.snapshotFile : null, + wasmChunkStore: + 'ok' === wasmChunkStoreResult.status ? wasmChunkStoreResult.snapshotFile : null + } + } + }); + return {status: 'success', snapshotIdText, folder}; }; -const downloadMetadata = async ({ - folder, +const loadMetadata = async ({ snapshotIdText, log, ...rest -}: SnapshotParams & {folder: string; snapshotIdText: string} & SnapshotLog): Promise<{ +}: SnapshotParams & {snapshotIdText: string} & SnapshotLog): Promise<{ metadata: ReadCanisterSnapshotMetadataResponse; }> => { log(`Downloading snapshot metadata ${snapshotIdText}...`); const metadata = await readCanisterSnapshotMetadata(rest); + return {metadata}; +}; + +const saveMetadata = async ({ + folder, + log, + metadata +}: {folder: string; metadata: SnapshotMetadata} & SnapshotLog) => { + log(`Saving snapshot metadata...`); - // TODO: write the metadata at the end of the process. It's safer. const destination = join(folder, 'metadata.json'); await writeFile(destination, JSON.stringify(metadata, jsonReplacer, 2), 'utf-8'); - - return {metadata}; }; const assertSizeAndDownloadChunks = async ({ @@ -167,44 +211,59 @@ const assertSizeAndDownloadChunks = async ({ ...params }: SnapshotParams & { folder: string; - filename: string; + filename: SnapshotFilename; size: bigint; build: BuildChunkFn; -} & SnapshotLog) => { +} & SnapshotLog): Promise<{status: 'ok'; snapshotFile: SnapshotFile} | {status: 'skip'}> => { if (size === 0n) { log(`No chunks to download for ${filename} (size = 0). Skipping.`); await new Promise((resolve) => setTimeout(resolve, 2500)); - return; + return {status: 'skip'}; } - await downloadMemoryChunks({ + const {size: downloadedSize, hash} = await downloadMemoryChunks({ size, log, filename, ...params }); + + if (downloadedSize !== size) { + throw new SnapshotFsSizeError(filename, size, downloadedSize); + } + + return { + status: 'ok', + snapshotFile: { + filename, + size: downloadedSize, + hash + } + }; }; const assertAndDownloadWasmChunks = async ({ wasmChunkStore, log, ...params -}: SnapshotParams & {folder: string; filename: string} & Pick< +}: SnapshotParams & {folder: string; filename: SnapshotFilename} & Pick< ReadCanisterSnapshotMetadataResponse, 'wasmChunkStore' > & - SnapshotLog) => { + SnapshotLog): Promise<{status: 'ok'; snapshotFile: SnapshotFile} | {status: 'skip'}> => { if (wasmChunkStore.length === 0) { log('Nothing to download from the WASM chunks store (length = 0). Skipping.'); await new Promise((resolve) => setTimeout(resolve, 2500)); - return; + return {status: 'skip'}; } - await downloadWasmChunks({ + const snapshotFile = await downloadWasmChunks({ wasmChunkStore, log, ...params }); + + return {status: 'ok', snapshotFile}; }; const downloadMemoryChunks = async ({ @@ -216,10 +275,10 @@ const downloadMemoryChunks = async ({ ...params }: SnapshotParams & { folder: string; - filename: string; + filename: SnapshotFilename; size: bigint; build: BuildChunkFn; -} & SnapshotLog) => { +} & SnapshotLog): Promise => { const {chunks} = prepareDownloadChunks({size, build}); const readable = Readable.from( @@ -230,7 +289,7 @@ const downloadMemoryChunks = async ({ }) ); - await downloadAndWrite({readable, folder, filename, log}); + return await downloadAndWrite({readable, folder, filename, log}); }; const downloadWasmChunks = async ({ @@ -239,11 +298,11 @@ const downloadWasmChunks = async ({ folder, filename, ...params -}: SnapshotParams & {folder: string; filename: string} & Pick< +}: SnapshotParams & {folder: string; filename: SnapshotFilename} & Pick< ReadCanisterSnapshotMetadataResponse, 'wasmChunkStore' > & - SnapshotLog) => { + SnapshotLog): Promise => { const readable = Readable.from( batchDownloadChunks({ chunks: wasmChunkStore.map((chunk, orderId) => ({wasmChunk: chunk, orderId})), @@ -252,7 +311,7 @@ const downloadWasmChunks = async ({ }) ); - await downloadAndWrite({readable, folder, filename, log}); + return await downloadAndWrite({readable, folder, filename, log}); }; const downloadAndWrite = async ({ @@ -260,12 +319,17 @@ const downloadAndWrite = async ({ folder, filename, log -}: {readable: Readable; folder: string; filename: string} & SnapshotLog) => { +}: { + readable: Readable; + folder: string; + filename: SnapshotFilename; +} & SnapshotLog): Promise => { // Note: we would not win much at gzipping the data. - const destination = join(folder, `${filename}.bin`); + const destination = join(folder, filename); log(`Downloading chunks to ${relative(process.cwd(), destination)}...`); + // A transformer use to flatten the back of chunks when writing to the file const transformer = new Transform({ objectMode: true, writableObjectMode: true, @@ -284,7 +348,23 @@ const downloadAndWrite = async ({ } }); - await pipeline(readable, transformer, createWriteStream(destination)); + // We want to compute a sha256 to assert the file in the upload process + const hash = createHash('sha256'); + + const hasher = new Transform({ + transform(chunk, _enc, cb) { + hash.update(chunk); + cb(null, chunk); + } + }); + + await pipeline(readable, transformer, hasher, createWriteStream(destination)); + + return { + filename, + size: BigInt(statSync(destination).size), + hash: hash.digest('hex') + }; }; const prepareDownloadChunks = ({ diff --git a/src/types/snapshot.ts b/src/types/snapshot.ts new file mode 100644 index 00000000..6066481b --- /dev/null +++ b/src/types/snapshot.ts @@ -0,0 +1,69 @@ +import * as z from 'zod'; + +const Uint8ArrayLike = z.instanceof(Uint8Array) as z.ZodType>; + +// A Zod schema for the ic-management ReadCanisterSnapshotMetadataResponse type +const ReadCanisterSnapshotMetadataResponseSchema = z.strictObject({ + globals: z.array( + z.union([ + z.object({f32: z.number()}), + z.object({f64: z.number()}), + z.object({i32: z.number()}), + z.object({i64: z.bigint()}), + z.object({v128: z.bigint()}) + ]) + ), + canisterVersion: z.bigint(), + source: z.union([ + z.object({metadataUpload: z.unknown()}), + z.object({takenFromCanister: z.unknown()}) + ]), + certifiedData: z.union([Uint8ArrayLike, z.array(z.number())]), + globalTimer: z.union([z.object({active: z.bigint()}), z.object({inactive: z.null()})]).optional(), + onLowWasmMemoryHookStatus: z + .union([ + z.object({conditionNotSatisfied: z.null()}), + z.object({executed: z.null()}), + z.object({ready: z.null()}) + ]) + .optional(), + wasmModuleSize: z.bigint(), + stableMemorySize: z.bigint(), + wasmChunkStore: z.array( + z.object({ + hash: z.union([Uint8ArrayLike, z.array(z.number())]) + }) + ), + takenAtTimestamp: z.bigint(), + wasmMemorySize: z.bigint() +}); + +const SnapshotFilenameSchema = z.enum([ + 'wasm-code.bin', + 'heap.bin', + 'stable.bin', + 'chunks-store.bin' +]); + +const SnapshotFileSchema = z.strictObject({ + filename: SnapshotFilenameSchema, + size: z.bigint(), + hash: z.hash('sha256') +}); + +const SnapshotDataSchema = z.strictObject({ + wasmModule: SnapshotFileSchema.nullable(), + wasmMemory: SnapshotFileSchema.nullable(), + stableMemory: SnapshotFileSchema.nullable(), + wasmChunkStore: SnapshotFileSchema.nullable() +}); + +export const SnapshotMetadataSchema = z.strictObject({ + snapshotId: z.string(), + data: SnapshotDataSchema, + metadata: ReadCanisterSnapshotMetadataResponseSchema +}); + +export type SnapshotFilename = z.infer; +export type SnapshotFile = z.infer; +export type SnapshotMetadata = z.infer; From 513a1c3ec04702bc6afe77869ddbb9d21e2b49d0 Mon Sep 17 00:00:00 2001 From: David Dal Busco Date: Thu, 2 Oct 2025 09:34:13 +0200 Subject: [PATCH 17/26] feat: read metadata --- .../snapshot/snapshot.upload.services.ts | 24 +++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/src/services/modules/snapshot/snapshot.upload.services.ts b/src/services/modules/snapshot/snapshot.upload.services.ts index c79aa1ac..b32c6c3a 100644 --- a/src/services/modules/snapshot/snapshot.upload.services.ts +++ b/src/services/modules/snapshot/snapshot.upload.services.ts @@ -1,11 +1,13 @@ import type {snapshot_id} from '@dfinity/ic-management'; import type {Principal} from '@dfinity/principal'; -import {isEmptyString} from '@dfinity/utils'; +import {isEmptyString, jsonReviver} from '@dfinity/utils'; import {nextArg} from '@junobuild/cli-tools'; import {red, yellow} from 'kleur'; import {existsSync, lstatSync} from 'node:fs'; +import {readFile} from 'node:fs/promises'; import ora from 'ora'; import type {AssetKey} from '../../../types/asset-key'; +import {SnapshotMetadata, SnapshotMetadataSchema} from '../../../types/snapshot'; import {displaySegment} from '../../../utils/display.utils'; // We override the ic-mgmt interface because we solely want snapshotId as Principal here @@ -55,6 +57,7 @@ export const uploadExistingSnapshot = async ({ try { const result = await uploadSnapshotMetadataAndMemory({ ...params, + folder, log: (text) => (spinner.text = text) }); @@ -75,9 +78,22 @@ export const uploadExistingSnapshot = async ({ const uploadSnapshotMetadataAndMemory = async ({ snapshotId, log, + folder, ...rest -}: SnapshotParams & SnapshotLog): Promise<{snapshotIdText: string}> => { - // TODO: yep todo +}: SnapshotParams & {folder: string} & SnapshotLog): Promise<{snapshotIdText: string}> => { + const {metadata} = await readMetadata({folder, log}); - return {snapshotIdText: ""} + return {snapshotIdText: ''}; +}; + +const readMetadata = async ({ + folder, + log +}: {folder: string} & SnapshotLog): Promise<{metadata: SnapshotMetadata}> => { + log('Loading metadata...'); + + const data = await readFile(folder, 'utf-8'); + const metadata = JSON.parse(data, jsonReviver); + + return {metadata: SnapshotMetadataSchema.parse(metadata)}; }; From 4b3febe00e2b64682573ca89b99557df0270a5a4 Mon Sep 17 00:00:00 2001 From: David Dal Busco Date: Thu, 2 Oct 2025 18:31:24 +0200 Subject: [PATCH 18/26] feat: upload snapshot --- src/api/ic.api.ts | 41 ++- src/help/snapshot.upload.help.ts | 2 +- .../snapshot/snapshot.download.services.ts | 41 +-- .../snapshot/snapshot.upload.services.ts | 269 +++++++++++++++++- src/types/snapshot.ts | 1 + src/utils/snapshot.utils.ts | 48 ++++ 6 files changed, 355 insertions(+), 47 deletions(-) create mode 100644 src/utils/snapshot.utils.ts diff --git a/src/api/ic.api.ts b/src/api/ic.api.ts index 89b53ae9..2c030afd 100644 --- a/src/api/ic.api.ts +++ b/src/api/ic.api.ts @@ -2,9 +2,15 @@ import {ICManagementCanister} from '@dfinity/ic-management'; import type { list_canister_snapshots_result, read_canister_snapshot_data_response, - snapshot_id + snapshot_id, + upload_canister_snapshot_metadata_response } from '@dfinity/ic-management/dist/candid/ic-management'; -import type {ReadCanisterSnapshotMetadataParams} from '@dfinity/ic-management/dist/types/types/snapshot.params'; +import { + ReadCanisterSnapshotMetadataParams, + SnapshotParams, + UploadCanisterSnapshotDataParams, + UploadCanisterSnapshotMetadataParams +} from '@dfinity/ic-management/dist/types/types/snapshot.params'; import type {ReadCanisterSnapshotMetadataResponse} from '@dfinity/ic-management/dist/types/types/snapshot.responses'; import type {Principal} from '@dfinity/principal'; import {initAgent} from './agent.api'; @@ -80,10 +86,9 @@ export const deleteCanisterSnapshot = async (params: { await deleteCanisterSnapshot(params); }; -export const readCanisterSnapshotMetadata = async (params: { - canisterId: Principal; - snapshotId: snapshot_id; -}): Promise => { +export const readCanisterSnapshotMetadata = async ( + params: SnapshotParams +): Promise => { const agent = await initAgent(); const {readCanisterSnapshotMetadata} = ICManagementCanister.create({ @@ -104,3 +109,27 @@ export const readCanisterSnapshotData = async ( return await readCanisterSnapshotData(params); }; + +export const uploadCanisterSnapshotMetadata = async ( + params: UploadCanisterSnapshotMetadataParams +): Promise => { + const agent = await initAgent(); + + const {uploadCanisterSnapshotMetadata} = ICManagementCanister.create({ + agent + }); + + return await uploadCanisterSnapshotMetadata(params); +}; + +export const uploadCanisterSnapshotData = async ( + params: UploadCanisterSnapshotDataParams +): Promise => { + const agent = await initAgent(); + + const {uploadCanisterSnapshotData} = ICManagementCanister.create({ + agent + }); + + await uploadCanisterSnapshotData(params); +}; diff --git a/src/help/snapshot.upload.help.ts b/src/help/snapshot.upload.help.ts index 5be9a86a..b6a0b74f 100644 --- a/src/help/snapshot.upload.help.ts +++ b/src/help/snapshot.upload.help.ts @@ -11,7 +11,7 @@ import {TITLE} from './help'; const usage = `Usage: ${green('juno')} ${cyan('snapshot')} ${magenta('upload')} ${yellow('[options]')} Options: - ${yellow('--dir')} Path to the snapshot directory that contains the metadata.json and chunks. + ${yellow('--dir')} Path to the snapshot directory that contains the metadata.json and chunks. ${OPTIONS_ENV} ${OPTION_HELP}`; diff --git a/src/services/modules/snapshot/snapshot.download.services.ts b/src/services/modules/snapshot/snapshot.download.services.ts index 6a19043c..aef1e056 100644 --- a/src/services/modules/snapshot/snapshot.download.services.ts +++ b/src/services/modules/snapshot/snapshot.download.services.ts @@ -16,6 +16,7 @@ import {SNAPSHOT_CHUNK_SIZE, SNAPSHOTS_PATH} from '../../../constants/snapshot.c import {AssetKey} from '../../../types/asset-key'; import {SnapshotFile, SnapshotFilename, SnapshotMetadata} from '../../../types/snapshot'; import {displaySegment} from '../../../utils/display.utils'; +import {BuildChunkFn, prepareDataChunks} from '../../../utils/snapshot.utils'; // We override the ic-mgmt interface because we solely want snapshotId as Principal here interface SnapshotParams { @@ -23,9 +24,8 @@ interface SnapshotParams { snapshotId: snapshot_id; } -type RequestedChunk = CanisterSnapshotMetadataKind; +type DataChunk = CanisterSnapshotMetadataKind; type DownloadedChunk = Uint8Array; -type BuildChunkFn = (params: {offset: bigint; size: bigint}) => RequestedChunk; class SnapshotFsFolderError extends Error { constructor(public readonly folder: string) { @@ -213,7 +213,7 @@ const assertSizeAndDownloadChunks = async ({ folder: string; filename: SnapshotFilename; size: bigint; - build: BuildChunkFn; + build: BuildChunkFn; } & SnapshotLog): Promise<{status: 'ok'; snapshotFile: SnapshotFile} | {status: 'skip'}> => { if (size === 0n) { log(`No chunks to download for ${filename} (size = 0). Skipping.`); @@ -277,14 +277,13 @@ const downloadMemoryChunks = async ({ folder: string; filename: SnapshotFilename; size: bigint; - build: BuildChunkFn; + build: BuildChunkFn; } & SnapshotLog): Promise => { - const {chunks} = prepareDownloadChunks({size, build}); + const {chunks} = prepareDataChunks({size, build}); const readable = Readable.from( batchDownloadChunks({ chunks, - limit: 20, ...params }) ); @@ -367,36 +366,12 @@ const downloadAndWrite = async ({ }; }; -const prepareDownloadChunks = ({ - size: totalSize, - build -}: { - size: bigint; - build: BuildChunkFn; -}): {chunks: RequestedChunk[]} => { - const chunks: RequestedChunk[] = []; - - for (let offset = 0n; offset < totalSize; offset += SNAPSHOT_CHUNK_SIZE) { - const size = - offset + SNAPSHOT_CHUNK_SIZE <= totalSize ? SNAPSHOT_CHUNK_SIZE : totalSize - offset; - - chunks.push({ - ...build({ - offset, - size - }) - }); - } - - return {chunks}; -}; - async function* batchDownloadChunks({ chunks, - limit = 12, + limit = 20, ...params }: SnapshotParams & { - chunks: RequestedChunk[]; + chunks: DataChunk[]; limit?: number; }): AsyncGenerator { const total = chunks.length; @@ -419,7 +394,7 @@ const downloadChunk = async ({ chunk: kind, ...rest }: SnapshotParams & { - chunk: RequestedChunk; + chunk: DataChunk; }): Promise => { const {chunk: downloadedChunk} = await readCanisterSnapshotData({ ...rest, diff --git a/src/services/modules/snapshot/snapshot.upload.services.ts b/src/services/modules/snapshot/snapshot.upload.services.ts index b32c6c3a..53bb0694 100644 --- a/src/services/modules/snapshot/snapshot.upload.services.ts +++ b/src/services/modules/snapshot/snapshot.upload.services.ts @@ -1,14 +1,30 @@ -import type {snapshot_id} from '@dfinity/ic-management'; +import {encodeSnapshotId, snapshot_id} from '@dfinity/ic-management'; +import {UploadCanisterSnapshotDataKind} from '@dfinity/ic-management/dist/types/types/snapshot.params'; import type {Principal} from '@dfinity/principal'; -import {isEmptyString, jsonReviver} from '@dfinity/utils'; +import { + arrayBufferToUint8Array, + isEmptyString, + isNullish, + jsonReviver, + nonNullish +} from '@dfinity/utils'; import {nextArg} from '@junobuild/cli-tools'; +import {FileHandle} from 'fs/promises'; import {red, yellow} from 'kleur'; import {existsSync, lstatSync} from 'node:fs'; -import {readFile} from 'node:fs/promises'; +import {open as openFile, readFile} from 'node:fs/promises'; +import {join, relative} from 'node:path'; import ora from 'ora'; +import {uploadCanisterSnapshotData, uploadCanisterSnapshotMetadata} from '../../../api/ic.api'; import type {AssetKey} from '../../../types/asset-key'; -import {SnapshotMetadata, SnapshotMetadataSchema} from '../../../types/snapshot'; +import { + ReadCanisterSnapshotMetadataResponse, + SnapshotFile, + SnapshotMetadata, + SnapshotMetadataSchema +} from '../../../types/snapshot'; import {displaySegment} from '../../../utils/display.utils'; +import {BuildChunkFn, computeLargeFileHash, prepareDataChunks} from '../../../utils/snapshot.utils'; // We override the ic-mgmt interface because we solely want snapshotId as Principal here interface SnapshotParams { @@ -22,6 +38,20 @@ interface SnapshotLog { log: (text: string) => void; } +interface BatchResult { + progress: {index: number; done: number; total: number}; +} + +interface DataChunk { + kind: UploadCanisterSnapshotDataKind; + offset: number; + size: number; +} + +class SnapshotUploadError extends Error {} +class SnapshotAssertError extends Error {} +class SnapshotFsReadError extends Error {} + export const uploadExistingSnapshot = async ({ segment, args, @@ -71,19 +101,111 @@ export const uploadExistingSnapshot = async ({ } catch (error: unknown) { spinner.stop(); + if (error instanceof SnapshotUploadError) { + console.log(error.message); + return; + } + + if (error instanceof SnapshotFsReadError || error instanceof SnapshotAssertError) { + console.log(red(error.message)); + return; + } + throw error; } }; const uploadSnapshotMetadataAndMemory = async ({ - snapshotId, log, folder, + snapshotId: replaceSnapshotId, ...rest }: SnapshotParams & {folder: string} & SnapshotLog): Promise<{snapshotIdText: string}> => { - const {metadata} = await readMetadata({folder, log}); + // 1. Read the snapshot metadata + const { + metadata: { + metadata, + data: {wasmModule, wasmMemory, stableMemory, wasmChunkStore} + } + } = await readMetadata({folder, log}); + + // 2. Upload the snapshot metadata - i.e. we need to upload first the metadata + // because an existing snapshot is required to upload the chunk. Technically, + // we can probably assert here, in case of replacement, if the existing snapshot + // is equals to the one we upload but for simplicity reasons we upload every time. + const {snapshotId} = await uploadMetadata({ + ...rest, + snapshotId: replaceSnapshotId, + metadata, + log + }); + + const snapshotIdText = `0x${encodeSnapshotId(snapshotId)}`; + + // Unlikely but, just in case let's stop here if the snapshot was not overwritten as expected. + if ( + nonNullish(replaceSnapshotId) && + `0x${encodeSnapshotId(replaceSnapshotId)}` !== snapshotIdText + ) { + throw new SnapshotUploadError( + `⚠️ The existing snapshot 0x${encodeSnapshotId(replaceSnapshotId)} was not overwritten. A new snapshot ${snapshotIdText} was created instead. This is unexpected.` + ); + } + + // 3. Upload chunks + await assertAndUploadChunks({ + ...rest, + snapshotId, + log, + folder, + file: wasmModule, + build: ({offset, size}) => ({ + kind: {wasmModule: {offset}}, + offset: Number(offset), + size: Number(size) + }) + }); + + await assertAndUploadChunks({ + ...rest, + snapshotId, + log, + folder, + file: wasmMemory, + build: ({offset, size}) => ({ + kind: {wasmMemory: {offset}}, + offset: Number(offset), + size: Number(size) + }) + }); - return {snapshotIdText: ''}; + await assertAndUploadChunks({ + ...rest, + snapshotId, + log, + folder, + file: stableMemory, + build: ({offset, size}) => ({ + kind: {stableMemory: {offset}}, + offset: Number(offset), + size: Number(size) + }) + }); + + await assertAndUploadChunks({ + ...rest, + snapshotId, + log, + folder, + file: wasmChunkStore, + build: ({offset, size}) => ({ + kind: {wasmChunk: null}, + offset: Number(offset), + size: Number(size) + }) + }); + + return {snapshotIdText}; }; const readMetadata = async ({ @@ -97,3 +219,136 @@ const readMetadata = async ({ return {metadata: SnapshotMetadataSchema.parse(metadata)}; }; + +const uploadMetadata = async ({ + log, + metadata: { + globals, + certifiedData, + globalTimer, + onLowWasmMemoryHookStatus, + wasmModuleSize, + stableMemorySize, + wasmMemorySize + }, + ...rest +}: SnapshotParams & {metadata: ReadCanisterSnapshotMetadataResponse} & SnapshotLog): Promise<{ + snapshotId: snapshot_id; +}> => { + log('Uploading snapshot metadata...'); + + const {snapshot_id: snapshotId} = await uploadCanisterSnapshotMetadata({ + // Handpicked data to avoid sending unexpected data over the wire + metadata: { + globals, + certifiedData, + globalTimer, + onLowWasmMemoryHookStatus, + wasmModuleSize, + stableMemorySize, + wasmMemorySize + }, + ...rest + }); + + return {snapshotId}; +}; + +const assertAndUploadChunks = async ({ + folder, + file, + log, + build, + ...rest +}: Required & { + folder: string; + file: SnapshotFile | null; + build: BuildChunkFn; +} & SnapshotLog): Promise<{status: 'success' | 'skip'}> => { + if (isNullish(file)) { + // We do not log a message to not make the developer think something is missing. + return {status: 'skip'}; + } + + const {filename, size, hash} = file; + + const source = join(folder, filename); + + const actualSize = BigInt(lstatSync(source).size); + if (size !== actualSize) { + throw new SnapshotAssertError( + `Size mismatch for ${filename}: expected ${size} bytes, got ${actualSize} bytes` + ); + } + + const actualHash = await computeLargeFileHash(source); + if (hash !== (await computeLargeFileHash(source))) { + throw new SnapshotAssertError( + `Hash mismatch for ${filename}: expected ${hash}, got ${actualHash}` + ); + } + + const {chunks} = prepareDataChunks({size, build}); + + log(`Uploading chunks from ${relative(process.cwd(), source)}...`); + + for await (const {progress} of batchUploadChunks({ + source, + chunks, + ...rest + })) { + log(`Chunks ${progress.done}/${progress.total} uploaded. Continuing...`); + } + + return {status: 'success'}; +}; + +async function* batchUploadChunks({ + source, + chunks, + limit = 20, + ...params +}: Required & { + source: string; + chunks: DataChunk[]; + limit?: number; +}): AsyncGenerator { + const sourceHandler = await openFile(source); + + const total = chunks.length; + + for (let i = 0; i < total; i = i + limit) { + const batch = chunks.slice(i, i + limit); + await Promise.all( + batch.map((requestChunk) => + uploadChunk({ + ...params, + sourceHandler, + chunk: requestChunk + }) + ) + ); + yield {progress: {index: i, done: Math.min(i + limit, total), total}}; + } +} + +const uploadChunk = async ({ + chunk: {kind, size, offset}, + sourceHandler, + ...rest +}: Required & { + sourceHandler: FileHandle; + chunk: DataChunk; +}): Promise => { + const {buffer, bytesRead} = await sourceHandler.read(Buffer.alloc(size), 0, size, offset); + + if (bytesRead !== size) { + throw new SnapshotFsReadError(`Unexpected bytes read: expected ${size} but got ${bytesRead}`); + } + + await uploadCanisterSnapshotData({ + ...rest, + kind, + chunk: arrayBufferToUint8Array(buffer.buffer) + }); +}; diff --git a/src/types/snapshot.ts b/src/types/snapshot.ts index 6066481b..2d01d3a0 100644 --- a/src/types/snapshot.ts +++ b/src/types/snapshot.ts @@ -67,3 +67,4 @@ export const SnapshotMetadataSchema = z.strictObject({ export type SnapshotFilename = z.infer; export type SnapshotFile = z.infer; export type SnapshotMetadata = z.infer; +export type ReadCanisterSnapshotMetadataResponse = z.infer; diff --git a/src/utils/snapshot.utils.ts b/src/utils/snapshot.utils.ts new file mode 100644 index 00000000..842ad3a4 --- /dev/null +++ b/src/utils/snapshot.utils.ts @@ -0,0 +1,48 @@ +import {createHash} from 'node:crypto'; +import {createReadStream} from 'node:fs'; +import {Writable} from 'node:stream'; +import {pipeline} from 'node:stream/promises'; +import {SNAPSHOT_CHUNK_SIZE} from '../constants/snapshot.constants'; + +export type BuildChunkFn = (params: {offset: bigint; size: bigint}) => T; + +export const prepareDataChunks = ({ + size: totalSize, + build +}: { + size: bigint; + build: BuildChunkFn; +}): {chunks: T[]} => { + const chunks: T[] = []; + + for (let offset = 0n; offset < totalSize; offset += SNAPSHOT_CHUNK_SIZE) { + const size = + offset + SNAPSHOT_CHUNK_SIZE <= totalSize ? SNAPSHOT_CHUNK_SIZE : totalSize - offset; + + chunks.push({ + ...build({ + offset, + size + }) + }); + } + + return {chunks}; +}; + +// TODO: we maybe want to move this elsewhere as it is not stricly related to snapshots +export const computeLargeFileHash = async (filepath: string): Promise => { + const hash = createHash('sha256'); + + await pipeline( + createReadStream(filepath), + new Writable({ + write(chunk, _enc, cb) { + hash.update(chunk); + cb(); + } + }) + ); + + return hash.digest('hex'); +}; From 44d5413c915e0614ef1fd72b0a9aca30c1f7da9d Mon Sep 17 00:00:00 2001 From: David Dal Busco Date: Thu, 2 Oct 2025 18:45:43 +0200 Subject: [PATCH 19/26] feat: check dir before loading snapshot and fix passing down args --- .../snapshot.mission-control.services.ts | 2 +- .../snapshot/snapshot.orbiter.services.ts | 2 +- .../snapshot/snapshot.satellite.services.ts | 2 +- .../modules/snapshot/snapshot.services.ts | 30 +++++++++++-- .../snapshot/snapshot.upload.services.ts | 43 +++---------------- 5 files changed, 37 insertions(+), 42 deletions(-) diff --git a/src/services/modules/snapshot/snapshot.mission-control.services.ts b/src/services/modules/snapshot/snapshot.mission-control.services.ts index 85937e90..16df7e90 100644 --- a/src/services/modules/snapshot/snapshot.mission-control.services.ts +++ b/src/services/modules/snapshot/snapshot.mission-control.services.ts @@ -36,7 +36,7 @@ export const downloadSnapshotMissionControl = async () => { export const uploadSnapshotMissionControl = async (args?: string[]) => { await executeSnapshotFn({ - fn: (params) => uploadSnapshot({...params, ...args}), + fn: (params) => uploadSnapshot({...params, args}), }); }; diff --git a/src/services/modules/snapshot/snapshot.orbiter.services.ts b/src/services/modules/snapshot/snapshot.orbiter.services.ts index 2a48cc76..a13ba2db 100644 --- a/src/services/modules/snapshot/snapshot.orbiter.services.ts +++ b/src/services/modules/snapshot/snapshot.orbiter.services.ts @@ -34,7 +34,7 @@ export const downloadSnapshotOrbiter = async () => { export const uploadSnapshotOrbiter = async (args?: string[]) => { await executeSnapshotFn({ - fn: (params) => uploadSnapshot({...params, ...args}), + fn: (params) => uploadSnapshot({...params, args}), }); }; diff --git a/src/services/modules/snapshot/snapshot.satellite.services.ts b/src/services/modules/snapshot/snapshot.satellite.services.ts index 0f642521..311d42ef 100644 --- a/src/services/modules/snapshot/snapshot.satellite.services.ts +++ b/src/services/modules/snapshot/snapshot.satellite.services.ts @@ -37,7 +37,7 @@ export const downloadSnapshotSatellite = async () => { export const uploadSnapshotSatellite = async (args?: string[]) => { await executeSnapshotFn({ - fn: (params) => uploadSnapshot({...params, ...args}), + fn: (params) => uploadSnapshot({...params, args}), }); }; diff --git a/src/services/modules/snapshot/snapshot.services.ts b/src/services/modules/snapshot/snapshot.services.ts index bb569b6e..915b063c 100644 --- a/src/services/modules/snapshot/snapshot.services.ts +++ b/src/services/modules/snapshot/snapshot.services.ts @@ -1,8 +1,8 @@ import type {snapshot_id} from '@dfinity/ic-management'; import {encodeSnapshotId} from '@dfinity/ic-management'; import {Principal} from '@dfinity/principal'; -import {isNullish, nonNullish} from '@dfinity/utils'; -import {red} from 'kleur'; +import {isEmptyString, isNullish, nonNullish} from '@dfinity/utils'; +import {red, yellow} from 'kleur'; import ora from 'ora'; import { deleteCanisterSnapshot, @@ -15,6 +15,8 @@ import {displaySegment} from '../../../utils/display.utils'; import {confirmAndExit} from '../../../utils/prompt.utils'; import {downloadExistingSnapshot} from './snapshot.download.services'; import {uploadExistingSnapshot} from './snapshot.upload.services'; +import {nextArg} from '@junobuild/cli-tools'; +import {existsSync, lstatSync} from 'node:fs'; export const createSnapshot = async ({ canisterId: cId, @@ -125,13 +127,35 @@ export const uploadSnapshot = async ({ }) => { const canisterId = Principal.fromText(cId); + const folder = nextArg({args, option: '--dir'}); + + if (isEmptyString(folder)) { + console.log( + `You did not provide a ${yellow('directory')} that contains metadata.json and chunks to upload.` + ); + return; + } + + if (!existsSync(folder)) { + console.log(`The directory ${yellow('directory')} does not exist.`); + return; + } + + if (!lstatSync(folder).isDirectory()) { + console.log(red(`${folder} is not a directory.`)); + return; + } + + // TODO: extract assertions + // TODO: more assertion like is there a metadata.json and chunk files + const {snapshotId} = await loadSnapshotAndAssertOverwrite({canisterId, segment}); await uploadExistingSnapshot({ canisterId, snapshotId, segment, - args + folder }); }; diff --git a/src/services/modules/snapshot/snapshot.upload.services.ts b/src/services/modules/snapshot/snapshot.upload.services.ts index 53bb0694..f4e8b8e1 100644 --- a/src/services/modules/snapshot/snapshot.upload.services.ts +++ b/src/services/modules/snapshot/snapshot.upload.services.ts @@ -1,17 +1,10 @@ import {encodeSnapshotId, snapshot_id} from '@dfinity/ic-management'; import {UploadCanisterSnapshotDataKind} from '@dfinity/ic-management/dist/types/types/snapshot.params'; import type {Principal} from '@dfinity/principal'; -import { - arrayBufferToUint8Array, - isEmptyString, - isNullish, - jsonReviver, - nonNullish -} from '@dfinity/utils'; -import {nextArg} from '@junobuild/cli-tools'; +import {arrayBufferToUint8Array, isNullish, jsonReviver, nonNullish} from '@dfinity/utils'; import {FileHandle} from 'fs/promises'; -import {red, yellow} from 'kleur'; -import {existsSync, lstatSync} from 'node:fs'; +import {red} from 'kleur'; +import {lstatSync} from 'node:fs'; import {open as openFile, readFile} from 'node:fs/promises'; import {join, relative} from 'node:path'; import ora from 'ora'; @@ -54,40 +47,16 @@ class SnapshotFsReadError extends Error {} export const uploadExistingSnapshot = async ({ segment, - args, ...params }: SnapshotParams & { segment: AssetKey; - args?: string[]; + folder: string; }): Promise => { - const folder = nextArg({args, option: '--dir'}); - - if (isEmptyString(folder)) { - console.log( - `You did not provide a ${yellow('directory')} that contains metadata.json and chunks to upload.` - ); - return; - } - - if (!existsSync(folder)) { - console.log(`The directory ${yellow('directory')} does not exist.`); - return; - } - - if (!lstatSync(folder).isDirectory()) { - console.log(red(`${folder} is not a directory.`)); - return; - } - - // TODO: extract assertions - // TODO: more assertion like is there a metadata.json and chunk files - const spinner = ora('Uploading the snapshot...').start(); try { const result = await uploadSnapshotMetadataAndMemory({ ...params, - folder, log: (text) => (spinner.text = text) }); @@ -214,7 +183,9 @@ const readMetadata = async ({ }: {folder: string} & SnapshotLog): Promise<{metadata: SnapshotMetadata}> => { log('Loading metadata...'); - const data = await readFile(folder, 'utf-8'); + const source = join(folder, 'metadata.json'); + + const data = await readFile(source, 'utf-8'); const metadata = JSON.parse(data, jsonReviver); return {metadata: SnapshotMetadataSchema.parse(metadata)}; From 562edad666f7d77b5cbab350e8193c39d245bee9 Mon Sep 17 00:00:00 2001 From: David Dal Busco Date: Thu, 2 Oct 2025 18:50:58 +0200 Subject: [PATCH 20/26] fix: always a new snapshot id --- .../snapshot/snapshot.upload.services.ts | 42 +++++++------------ 1 file changed, 15 insertions(+), 27 deletions(-) diff --git a/src/services/modules/snapshot/snapshot.upload.services.ts b/src/services/modules/snapshot/snapshot.upload.services.ts index f4e8b8e1..cf7bcde2 100644 --- a/src/services/modules/snapshot/snapshot.upload.services.ts +++ b/src/services/modules/snapshot/snapshot.upload.services.ts @@ -1,7 +1,7 @@ import {encodeSnapshotId, snapshot_id} from '@dfinity/ic-management'; import {UploadCanisterSnapshotDataKind} from '@dfinity/ic-management/dist/types/types/snapshot.params'; import type {Principal} from '@dfinity/principal'; -import {arrayBufferToUint8Array, isNullish, jsonReviver, nonNullish} from '@dfinity/utils'; +import {arrayBufferToUint8Array, isNullish, jsonReviver} from '@dfinity/utils'; import {FileHandle} from 'fs/promises'; import {red} from 'kleur'; import {lstatSync} from 'node:fs'; @@ -41,7 +41,6 @@ interface DataChunk { size: number; } -class SnapshotUploadError extends Error {} class SnapshotAssertError extends Error {} class SnapshotFsReadError extends Error {} @@ -70,11 +69,6 @@ export const uploadExistingSnapshot = async ({ } catch (error: unknown) { spinner.stop(); - if (error instanceof SnapshotUploadError) { - console.log(error.message); - return; - } - if (error instanceof SnapshotFsReadError || error instanceof SnapshotAssertError) { console.log(red(error.message)); return; @@ -111,16 +105,6 @@ const uploadSnapshotMetadataAndMemory = async ({ const snapshotIdText = `0x${encodeSnapshotId(snapshotId)}`; - // Unlikely but, just in case let's stop here if the snapshot was not overwritten as expected. - if ( - nonNullish(replaceSnapshotId) && - `0x${encodeSnapshotId(replaceSnapshotId)}` !== snapshotIdText - ) { - throw new SnapshotUploadError( - `⚠️ The existing snapshot 0x${encodeSnapshotId(replaceSnapshotId)} was not overwritten. A new snapshot ${snapshotIdText} was created instead. This is unexpected.` - ); - } - // 3. Upload chunks await assertAndUploadChunks({ ...rest, @@ -263,29 +247,33 @@ const assertAndUploadChunks = async ({ log(`Uploading chunks from ${relative(process.cwd(), source)}...`); - for await (const {progress} of batchUploadChunks({ - source, - chunks, - ...rest - })) { - log(`Chunks ${progress.done}/${progress.total} uploaded. Continuing...`); + const sourceHandler = await openFile(source); + + try { + for await (const {progress} of batchUploadChunks({ + sourceHandler, + chunks, + ...rest + })) { + log(`Chunks ${progress.done}/${progress.total} uploaded. Continuing...`); + } + } finally { + await sourceHandler.close(); } return {status: 'success'}; }; async function* batchUploadChunks({ - source, + sourceHandler, chunks, limit = 20, ...params }: Required & { - source: string; + sourceHandler: FileHandle; chunks: DataChunk[]; limit?: number; }): AsyncGenerator { - const sourceHandler = await openFile(source); - const total = chunks.length; for (let i = 0; i < total; i = i + limit) { From 0d64357a6440d9adffee6b6ab922ea1749a2f4bd Mon Sep 17 00:00:00 2001 From: David Dal Busco Date: Thu, 2 Oct 2025 19:02:51 +0200 Subject: [PATCH 21/26] feat: cleanup --- src/services/modules/snapshot/snapshot.upload.services.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/services/modules/snapshot/snapshot.upload.services.ts b/src/services/modules/snapshot/snapshot.upload.services.ts index cf7bcde2..405e9d07 100644 --- a/src/services/modules/snapshot/snapshot.upload.services.ts +++ b/src/services/modules/snapshot/snapshot.upload.services.ts @@ -237,7 +237,7 @@ const assertAndUploadChunks = async ({ } const actualHash = await computeLargeFileHash(source); - if (hash !== (await computeLargeFileHash(source))) { + if (hash !== actualHash) { throw new SnapshotAssertError( `Hash mismatch for ${filename}: expected ${hash}, got ${actualHash}` ); From ad4845ca8018e33a968ea18389d1371de11dd955 Mon Sep 17 00:00:00 2001 From: David Dal Busco Date: Fri, 3 Oct 2025 07:20:28 +0200 Subject: [PATCH 22/26] chore: rename constant --- src/constants/snapshot.constants.ts | 3 ++- src/services/modules/snapshot/snapshot.download.services.ts | 2 +- src/utils/snapshot.utils.ts | 6 +++--- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/constants/snapshot.constants.ts b/src/constants/snapshot.constants.ts index c23c0f7b..99672f66 100644 --- a/src/constants/snapshot.constants.ts +++ b/src/constants/snapshot.constants.ts @@ -3,4 +3,5 @@ import {join} from 'node:path'; export const SNAPSHOTS_PATH = join(process.cwd(), '.snapshots'); // https://forum.dfinity.org/t/canister-snapshot-up-download/57397?u=peterparker -export const SNAPSHOT_CHUNK_SIZE = 1_000_000n; +// Same value as INSTALL_MAX_CHUNK_SIZE in @dfinity/admin +export const SNAPSHOT_MAX_CHUNK_SIZE = 1_000_000n; diff --git a/src/services/modules/snapshot/snapshot.download.services.ts b/src/services/modules/snapshot/snapshot.download.services.ts index aef1e056..f451bc8d 100644 --- a/src/services/modules/snapshot/snapshot.download.services.ts +++ b/src/services/modules/snapshot/snapshot.download.services.ts @@ -12,7 +12,7 @@ import {Readable, Transform} from 'node:stream'; import {pipeline} from 'node:stream/promises'; import ora from 'ora'; import {readCanisterSnapshotData, readCanisterSnapshotMetadata} from '../../../api/ic.api'; -import {SNAPSHOT_CHUNK_SIZE, SNAPSHOTS_PATH} from '../../../constants/snapshot.constants'; +import {SNAPSHOTS_PATH} from '../../../constants/snapshot.constants'; import {AssetKey} from '../../../types/asset-key'; import {SnapshotFile, SnapshotFilename, SnapshotMetadata} from '../../../types/snapshot'; import {displaySegment} from '../../../utils/display.utils'; diff --git a/src/utils/snapshot.utils.ts b/src/utils/snapshot.utils.ts index 842ad3a4..c40186b5 100644 --- a/src/utils/snapshot.utils.ts +++ b/src/utils/snapshot.utils.ts @@ -2,7 +2,7 @@ import {createHash} from 'node:crypto'; import {createReadStream} from 'node:fs'; import {Writable} from 'node:stream'; import {pipeline} from 'node:stream/promises'; -import {SNAPSHOT_CHUNK_SIZE} from '../constants/snapshot.constants'; +import {SNAPSHOT_MAX_CHUNK_SIZE} from '../constants/snapshot.constants'; export type BuildChunkFn = (params: {offset: bigint; size: bigint}) => T; @@ -15,9 +15,9 @@ export const prepareDataChunks = ({ }): {chunks: T[]} => { const chunks: T[] = []; - for (let offset = 0n; offset < totalSize; offset += SNAPSHOT_CHUNK_SIZE) { + for (let offset = 0n; offset < totalSize; offset += SNAPSHOT_MAX_CHUNK_SIZE) { const size = - offset + SNAPSHOT_CHUNK_SIZE <= totalSize ? SNAPSHOT_CHUNK_SIZE : totalSize - offset; + offset + SNAPSHOT_MAX_CHUNK_SIZE <= totalSize ? SNAPSHOT_MAX_CHUNK_SIZE : totalSize - offset; chunks.push({ ...build({ From 7e8e2088b7c0c064f158552bfb0f6455dc1cec7f Mon Sep 17 00:00:00 2001 From: David Dal Busco Date: Fri, 3 Oct 2025 07:26:50 +0200 Subject: [PATCH 23/26] refactor: extract and move types and schemas --- src/schema/snapshot.schema.ts | 65 +++++++++++++ .../snapshot/snapshot.download.services.ts | 48 ++++------ .../snapshot/snapshot.upload.services.ts | 36 +++----- src/types/snapshot.ts | 92 ++++++------------- 4 files changed, 124 insertions(+), 117 deletions(-) create mode 100644 src/schema/snapshot.schema.ts diff --git a/src/schema/snapshot.schema.ts b/src/schema/snapshot.schema.ts new file mode 100644 index 00000000..998c7641 --- /dev/null +++ b/src/schema/snapshot.schema.ts @@ -0,0 +1,65 @@ +import * as z from 'zod'; + +const Uint8ArrayLike = z.instanceof(Uint8Array) as z.ZodType>; + +// A Zod schema for the ic-management ReadCanisterSnapshotMetadataResponse type +export const ReadCanisterSnapshotMetadataResponseSchema = z.strictObject({ + globals: z.array( + z.union([ + z.object({f32: z.number()}), + z.object({f64: z.number()}), + z.object({i32: z.number()}), + z.object({i64: z.bigint()}), + z.object({v128: z.bigint()}) + ]) + ), + canisterVersion: z.bigint(), + source: z.union([ + z.object({metadataUpload: z.unknown()}), + z.object({takenFromCanister: z.unknown()}) + ]), + certifiedData: z.union([Uint8ArrayLike, z.array(z.number())]), + globalTimer: z.union([z.object({active: z.bigint()}), z.object({inactive: z.null()})]).optional(), + onLowWasmMemoryHookStatus: z + .union([ + z.object({conditionNotSatisfied: z.null()}), + z.object({executed: z.null()}), + z.object({ready: z.null()}) + ]) + .optional(), + wasmModuleSize: z.bigint(), + stableMemorySize: z.bigint(), + wasmChunkStore: z.array( + z.object({ + hash: z.union([Uint8ArrayLike, z.array(z.number())]) + }) + ), + takenAtTimestamp: z.bigint(), + wasmMemorySize: z.bigint() +}); + +export const SnapshotFilenameSchema = z.enum([ + 'wasm-code.bin', + 'heap.bin', + 'stable.bin', + 'chunks-store.bin' +]); + +export const SnapshotFileSchema = z.strictObject({ + filename: SnapshotFilenameSchema, + size: z.bigint(), + hash: z.hash('sha256') +}); + +const SnapshotDataSchema = z.strictObject({ + wasmModule: SnapshotFileSchema.nullable(), + wasmMemory: SnapshotFileSchema.nullable(), + stableMemory: SnapshotFileSchema.nullable(), + wasmChunkStore: SnapshotFileSchema.nullable() +}); + +export const SnapshotMetadataSchema = z.strictObject({ + snapshotId: z.string(), + data: SnapshotDataSchema, + metadata: ReadCanisterSnapshotMetadataResponseSchema +}); diff --git a/src/services/modules/snapshot/snapshot.download.services.ts b/src/services/modules/snapshot/snapshot.download.services.ts index f451bc8d..e5befb3e 100644 --- a/src/services/modules/snapshot/snapshot.download.services.ts +++ b/src/services/modules/snapshot/snapshot.download.services.ts @@ -1,7 +1,5 @@ -import type {snapshot_id} from '@dfinity/ic-management'; import {type CanisterSnapshotMetadataKind, encodeSnapshotId} from '@dfinity/ic-management'; import type {ReadCanisterSnapshotMetadataResponse} from '@dfinity/ic-management/dist/types/types/snapshot.responses'; -import type {Principal} from '@dfinity/principal'; import {arrayOfNumberToUint8Array, jsonReplacer} from '@dfinity/utils'; import {red} from 'kleur'; import {createHash} from 'node:crypto'; @@ -14,16 +12,17 @@ import ora from 'ora'; import {readCanisterSnapshotData, readCanisterSnapshotMetadata} from '../../../api/ic.api'; import {SNAPSHOTS_PATH} from '../../../constants/snapshot.constants'; import {AssetKey} from '../../../types/asset-key'; -import {SnapshotFile, SnapshotFilename, SnapshotMetadata} from '../../../types/snapshot'; +import { + DownloadSnapshotParams, + SnapshotBatchResult, + SnapshotFile, + SnapshotFilename, + SnapshotLog, + SnapshotMetadata +} from '../../../types/snapshot'; import {displaySegment} from '../../../utils/display.utils'; import {BuildChunkFn, prepareDataChunks} from '../../../utils/snapshot.utils'; -// We override the ic-mgmt interface because we solely want snapshotId as Principal here -interface SnapshotParams { - canisterId: Principal; - snapshotId: snapshot_id; -} - type DataChunk = CanisterSnapshotMetadataKind; type DownloadedChunk = Uint8Array; @@ -43,21 +42,14 @@ class SnapshotFsSizeError extends Error { } } -// A handy wrapper to pass down a function that updates -// the spinner log. -interface SnapshotLog { - log: (text: string) => void; -} - -interface BatchResult { +interface DownloadSnapshotBatchResult extends SnapshotBatchResult { downloadedChunks: DownloadedChunk[]; - progress: {index: number; done: number; total: number}; } export const downloadExistingSnapshot = async ({ segment, ...params -}: SnapshotParams & { +}: DownloadSnapshotParams & { segment: AssetKey; }): Promise => { const spinner = ora().start(); @@ -101,7 +93,7 @@ const downloadSnapshotMetadataAndMemory = async ({ snapshotId, log, ...rest -}: SnapshotParams & SnapshotLog): Promise<{ +}: DownloadSnapshotParams & SnapshotLog): Promise<{ status: 'success'; snapshotIdText: string; folder: string; @@ -184,7 +176,7 @@ const loadMetadata = async ({ snapshotIdText, log, ...rest -}: SnapshotParams & {snapshotIdText: string} & SnapshotLog): Promise<{ +}: DownloadSnapshotParams & {snapshotIdText: string} & SnapshotLog): Promise<{ metadata: ReadCanisterSnapshotMetadataResponse; }> => { log(`Downloading snapshot metadata ${snapshotIdText}...`); @@ -209,7 +201,7 @@ const assertSizeAndDownloadChunks = async ({ log, filename, ...params -}: SnapshotParams & { +}: DownloadSnapshotParams & { folder: string; filename: SnapshotFilename; size: bigint; @@ -246,7 +238,7 @@ const assertAndDownloadWasmChunks = async ({ wasmChunkStore, log, ...params -}: SnapshotParams & {folder: string; filename: SnapshotFilename} & Pick< +}: DownloadSnapshotParams & {folder: string; filename: SnapshotFilename} & Pick< ReadCanisterSnapshotMetadataResponse, 'wasmChunkStore' > & @@ -273,7 +265,7 @@ const downloadMemoryChunks = async ({ folder, filename, ...params -}: SnapshotParams & { +}: DownloadSnapshotParams & { folder: string; filename: SnapshotFilename; size: bigint; @@ -297,7 +289,7 @@ const downloadWasmChunks = async ({ folder, filename, ...params -}: SnapshotParams & {folder: string; filename: SnapshotFilename} & Pick< +}: DownloadSnapshotParams & {folder: string; filename: SnapshotFilename} & Pick< ReadCanisterSnapshotMetadataResponse, 'wasmChunkStore' > & @@ -333,7 +325,7 @@ const downloadAndWrite = async ({ objectMode: true, writableObjectMode: true, readableObjectMode: false, - transform({downloadedChunks: chunks, progress}: BatchResult, _enc, cb) { + transform({downloadedChunks: chunks, progress}: DownloadSnapshotBatchResult, _enc, cb) { try { log(`Chunks ${progress.done}/${progress.total} downloaded. Continuing...`); @@ -370,10 +362,10 @@ async function* batchDownloadChunks({ chunks, limit = 20, ...params -}: SnapshotParams & { +}: DownloadSnapshotParams & { chunks: DataChunk[]; limit?: number; -}): AsyncGenerator { +}): AsyncGenerator { const total = chunks.length; for (let i = 0; i < total; i = i + limit) { @@ -393,7 +385,7 @@ async function* batchDownloadChunks({ const downloadChunk = async ({ chunk: kind, ...rest -}: SnapshotParams & { +}: DownloadSnapshotParams & { chunk: DataChunk; }): Promise => { const {chunk: downloadedChunk} = await readCanisterSnapshotData({ diff --git a/src/services/modules/snapshot/snapshot.upload.services.ts b/src/services/modules/snapshot/snapshot.upload.services.ts index 405e9d07..168a662b 100644 --- a/src/services/modules/snapshot/snapshot.upload.services.ts +++ b/src/services/modules/snapshot/snapshot.upload.services.ts @@ -1,6 +1,5 @@ import {encodeSnapshotId, snapshot_id} from '@dfinity/ic-management'; import {UploadCanisterSnapshotDataKind} from '@dfinity/ic-management/dist/types/types/snapshot.params'; -import type {Principal} from '@dfinity/principal'; import {arrayBufferToUint8Array, isNullish, jsonReviver} from '@dfinity/utils'; import {FileHandle} from 'fs/promises'; import {red} from 'kleur'; @@ -9,32 +8,19 @@ import {open as openFile, readFile} from 'node:fs/promises'; import {join, relative} from 'node:path'; import ora from 'ora'; import {uploadCanisterSnapshotData, uploadCanisterSnapshotMetadata} from '../../../api/ic.api'; +import {SnapshotMetadataSchema} from '../../../schema/snapshot.schema'; import type {AssetKey} from '../../../types/asset-key'; import { ReadCanisterSnapshotMetadataResponse, + SnapshotBatchResult, SnapshotFile, + SnapshotLog, SnapshotMetadata, - SnapshotMetadataSchema + UploadSnapshotParams } from '../../../types/snapshot'; import {displaySegment} from '../../../utils/display.utils'; import {BuildChunkFn, computeLargeFileHash, prepareDataChunks} from '../../../utils/snapshot.utils'; -// We override the ic-mgmt interface because we solely want snapshotId as Principal here -interface SnapshotParams { - canisterId: Principal; - snapshotId?: snapshot_id; -} - -// A handy wrapper to pass down a function that updates -// the spinner log. -interface SnapshotLog { - log: (text: string) => void; -} - -interface BatchResult { - progress: {index: number; done: number; total: number}; -} - interface DataChunk { kind: UploadCanisterSnapshotDataKind; offset: number; @@ -47,7 +33,7 @@ class SnapshotFsReadError extends Error {} export const uploadExistingSnapshot = async ({ segment, ...params -}: SnapshotParams & { +}: UploadSnapshotParams & { segment: AssetKey; folder: string; }): Promise => { @@ -83,7 +69,7 @@ const uploadSnapshotMetadataAndMemory = async ({ folder, snapshotId: replaceSnapshotId, ...rest -}: SnapshotParams & {folder: string} & SnapshotLog): Promise<{snapshotIdText: string}> => { +}: UploadSnapshotParams & {folder: string} & SnapshotLog): Promise<{snapshotIdText: string}> => { // 1. Read the snapshot metadata const { metadata: { @@ -187,7 +173,7 @@ const uploadMetadata = async ({ wasmMemorySize }, ...rest -}: SnapshotParams & {metadata: ReadCanisterSnapshotMetadataResponse} & SnapshotLog): Promise<{ +}: UploadSnapshotParams & {metadata: ReadCanisterSnapshotMetadataResponse} & SnapshotLog): Promise<{ snapshotId: snapshot_id; }> => { log('Uploading snapshot metadata...'); @@ -215,7 +201,7 @@ const assertAndUploadChunks = async ({ log, build, ...rest -}: Required & { +}: Required & { folder: string; file: SnapshotFile | null; build: BuildChunkFn; @@ -269,11 +255,11 @@ async function* batchUploadChunks({ chunks, limit = 20, ...params -}: Required & { +}: Required & { sourceHandler: FileHandle; chunks: DataChunk[]; limit?: number; -}): AsyncGenerator { +}): AsyncGenerator { const total = chunks.length; for (let i = 0; i < total; i = i + limit) { @@ -295,7 +281,7 @@ const uploadChunk = async ({ chunk: {kind, size, offset}, sourceHandler, ...rest -}: Required & { +}: Required & { sourceHandler: FileHandle; chunk: DataChunk; }): Promise => { diff --git a/src/types/snapshot.ts b/src/types/snapshot.ts index 2d01d3a0..cb33cbee 100644 --- a/src/types/snapshot.ts +++ b/src/types/snapshot.ts @@ -1,70 +1,34 @@ +import type {snapshot_id} from '@dfinity/ic-management'; +import type {Principal} from '@dfinity/principal'; import * as z from 'zod'; +import { + ReadCanisterSnapshotMetadataResponseSchema, + SnapshotFilenameSchema, + SnapshotFileSchema, + SnapshotMetadataSchema +} from '../schema/snapshot.schema'; -const Uint8ArrayLike = z.instanceof(Uint8Array) as z.ZodType>; - -// A Zod schema for the ic-management ReadCanisterSnapshotMetadataResponse type -const ReadCanisterSnapshotMetadataResponseSchema = z.strictObject({ - globals: z.array( - z.union([ - z.object({f32: z.number()}), - z.object({f64: z.number()}), - z.object({i32: z.number()}), - z.object({i64: z.bigint()}), - z.object({v128: z.bigint()}) - ]) - ), - canisterVersion: z.bigint(), - source: z.union([ - z.object({metadataUpload: z.unknown()}), - z.object({takenFromCanister: z.unknown()}) - ]), - certifiedData: z.union([Uint8ArrayLike, z.array(z.number())]), - globalTimer: z.union([z.object({active: z.bigint()}), z.object({inactive: z.null()})]).optional(), - onLowWasmMemoryHookStatus: z - .union([ - z.object({conditionNotSatisfied: z.null()}), - z.object({executed: z.null()}), - z.object({ready: z.null()}) - ]) - .optional(), - wasmModuleSize: z.bigint(), - stableMemorySize: z.bigint(), - wasmChunkStore: z.array( - z.object({ - hash: z.union([Uint8ArrayLike, z.array(z.number())]) - }) - ), - takenAtTimestamp: z.bigint(), - wasmMemorySize: z.bigint() -}); - -const SnapshotFilenameSchema = z.enum([ - 'wasm-code.bin', - 'heap.bin', - 'stable.bin', - 'chunks-store.bin' -]); +export type SnapshotFilename = z.infer; +export type SnapshotFile = z.infer; +export type SnapshotMetadata = z.infer; +export type ReadCanisterSnapshotMetadataResponse = z.infer< + typeof ReadCanisterSnapshotMetadataResponseSchema +>; -const SnapshotFileSchema = z.strictObject({ - filename: SnapshotFilenameSchema, - size: z.bigint(), - hash: z.hash('sha256') -}); +export interface DownloadSnapshotParams { + canisterId: Principal; + snapshotId: snapshot_id; +} -const SnapshotDataSchema = z.strictObject({ - wasmModule: SnapshotFileSchema.nullable(), - wasmMemory: SnapshotFileSchema.nullable(), - stableMemory: SnapshotFileSchema.nullable(), - wasmChunkStore: SnapshotFileSchema.nullable() -}); +export type UploadSnapshotParams = Omit & + Partial>; -export const SnapshotMetadataSchema = z.strictObject({ - snapshotId: z.string(), - data: SnapshotDataSchema, - metadata: ReadCanisterSnapshotMetadataResponseSchema -}); +// A handy wrapper to pass down a function that updates +// the spinner log. +export interface SnapshotLog { + log: (text: string) => void; +} -export type SnapshotFilename = z.infer; -export type SnapshotFile = z.infer; -export type SnapshotMetadata = z.infer; -export type ReadCanisterSnapshotMetadataResponse = z.infer; +export interface SnapshotBatchResult { + progress: {index: number; done: number; total: number}; +} \ No newline at end of file From b5891bbf91276f02e509d8a4b85a140c52cfd28d Mon Sep 17 00:00:00 2001 From: David Dal Busco Date: Fri, 3 Oct 2025 07:54:27 +0200 Subject: [PATCH 24/26] refactor: move hash utils --- .../snapshot/snapshot.upload.services.ts | 3 ++- src/utils/hash.utils.ts | 20 +++++++++++++++++++ src/utils/snapshot.utils.ts | 20 ------------------- 3 files changed, 22 insertions(+), 21 deletions(-) create mode 100644 src/utils/hash.utils.ts diff --git a/src/services/modules/snapshot/snapshot.upload.services.ts b/src/services/modules/snapshot/snapshot.upload.services.ts index 168a662b..11cfe252 100644 --- a/src/services/modules/snapshot/snapshot.upload.services.ts +++ b/src/services/modules/snapshot/snapshot.upload.services.ts @@ -19,7 +19,8 @@ import { UploadSnapshotParams } from '../../../types/snapshot'; import {displaySegment} from '../../../utils/display.utils'; -import {BuildChunkFn, computeLargeFileHash, prepareDataChunks} from '../../../utils/snapshot.utils'; +import {BuildChunkFn, prepareDataChunks} from '../../../utils/snapshot.utils'; +import {computeLargeFileHash} from '../../../utils/hash.utils'; interface DataChunk { kind: UploadCanisterSnapshotDataKind; diff --git a/src/utils/hash.utils.ts b/src/utils/hash.utils.ts new file mode 100644 index 00000000..5d30f652 --- /dev/null +++ b/src/utils/hash.utils.ts @@ -0,0 +1,20 @@ +import {createHash} from 'node:crypto'; +import {pipeline} from 'node:stream/promises'; +import {createReadStream} from 'node:fs'; +import {Writable} from 'node:stream'; + +export const computeLargeFileHash = async (filepath: string): Promise => { + const hash = createHash('sha256'); + + await pipeline( + createReadStream(filepath), + new Writable({ + write(chunk, _enc, cb) { + hash.update(chunk); + cb(); + } + }) + ); + + return hash.digest('hex'); +}; \ No newline at end of file diff --git a/src/utils/snapshot.utils.ts b/src/utils/snapshot.utils.ts index c40186b5..bedfc8cb 100644 --- a/src/utils/snapshot.utils.ts +++ b/src/utils/snapshot.utils.ts @@ -1,7 +1,3 @@ -import {createHash} from 'node:crypto'; -import {createReadStream} from 'node:fs'; -import {Writable} from 'node:stream'; -import {pipeline} from 'node:stream/promises'; import {SNAPSHOT_MAX_CHUNK_SIZE} from '../constants/snapshot.constants'; export type BuildChunkFn = (params: {offset: bigint; size: bigint}) => T; @@ -30,19 +26,3 @@ export const prepareDataChunks = ({ return {chunks}; }; -// TODO: we maybe want to move this elsewhere as it is not stricly related to snapshots -export const computeLargeFileHash = async (filepath: string): Promise => { - const hash = createHash('sha256'); - - await pipeline( - createReadStream(filepath), - new Writable({ - write(chunk, _enc, cb) { - hash.update(chunk); - cb(); - } - }) - ); - - return hash.digest('hex'); -}; From 1be8754b999279ed16e07563c425ef8af9884ed0 Mon Sep 17 00:00:00 2001 From: David Dal Busco Date: Fri, 3 Oct 2025 07:58:44 +0200 Subject: [PATCH 25/26] chore: fmt and lint --- src/api/ic.api.ts | 8 ++-- src/commands/snapshot.ts | 4 +- src/help/snapshot.help.ts | 7 ++- src/help/snapshot.upload.help.ts | 7 +-- src/schema/snapshot.schema.ts | 2 +- .../snapshot/snapshot.download.services.ts | 45 ++++++++++--------- .../snapshot.mission-control.services.ts | 4 +- .../snapshot/snapshot.orbiter.services.ts | 4 +- .../snapshot/snapshot.satellite.services.ts | 4 +- .../modules/snapshot/snapshot.services.ts | 4 +- .../snapshot/snapshot.upload.services.ts | 29 ++++++------ src/types/snapshot.ts | 12 ++--- src/utils/hash.utils.ts | 6 +-- src/utils/snapshot.utils.ts | 1 - 14 files changed, 73 insertions(+), 64 deletions(-) diff --git a/src/api/ic.api.ts b/src/api/ic.api.ts index 2c030afd..12039da8 100644 --- a/src/api/ic.api.ts +++ b/src/api/ic.api.ts @@ -6,10 +6,10 @@ import type { upload_canister_snapshot_metadata_response } from '@dfinity/ic-management/dist/candid/ic-management'; import { - ReadCanisterSnapshotMetadataParams, - SnapshotParams, - UploadCanisterSnapshotDataParams, - UploadCanisterSnapshotMetadataParams + type ReadCanisterSnapshotMetadataParams, + type SnapshotParams, + type UploadCanisterSnapshotDataParams, + type UploadCanisterSnapshotMetadataParams } from '@dfinity/ic-management/dist/types/types/snapshot.params'; import type {ReadCanisterSnapshotMetadataResponse} from '@dfinity/ic-management/dist/types/types/snapshot.responses'; import type {Principal} from '@dfinity/principal'; diff --git a/src/commands/snapshot.ts b/src/commands/snapshot.ts index 002d8c43..30b1c3ad 100644 --- a/src/commands/snapshot.ts +++ b/src/commands/snapshot.ts @@ -1,6 +1,7 @@ import {nextArg} from '@junobuild/cli-tools'; import {red} from 'kleur'; import {logHelpSnapshot} from '../help/snapshot.help'; +import {logHelpSnapshotUpload} from '../help/snapshot.upload.help'; import { createSnapshotMissionControl, deleteSnapshotMissionControl, @@ -22,7 +23,6 @@ import { restoreSnapshotSatellite, uploadSnapshotSatellite } from '../services/modules/snapshot/snapshot.satellite.services'; -import {logHelpSnapshotUpload} from '../help/snapshot.upload.help'; export const snapshot = async (args?: string[]) => { const [subCommand] = args ?? []; @@ -116,4 +116,4 @@ export const helpSnapshot = (args?: string[]) => { default: logHelpSnapshot(args); } -} \ No newline at end of file +}; diff --git a/src/help/snapshot.help.ts b/src/help/snapshot.help.ts index 22c61737..6e57624d 100644 --- a/src/help/snapshot.help.ts +++ b/src/help/snapshot.help.ts @@ -1,5 +1,10 @@ import {cyan, green, magenta, yellow} from 'kleur'; -import {OPTIONS_ENV, OPTION_HELP, SNAPSHOT_DESCRIPTION, SNAPSHOT_UPLOAD_DESCRIPTION} from '../constants/help.constants'; +import { + OPTIONS_ENV, + OPTION_HELP, + SNAPSHOT_DESCRIPTION, + SNAPSHOT_UPLOAD_DESCRIPTION +} from '../constants/help.constants'; import {helpOutput} from './common.help'; import {TITLE} from './help'; import {TARGET_OPTION_NOTE, targetOption} from './target.help'; diff --git a/src/help/snapshot.upload.help.ts b/src/help/snapshot.upload.help.ts index b6a0b74f..07d0223c 100644 --- a/src/help/snapshot.upload.help.ts +++ b/src/help/snapshot.upload.help.ts @@ -1,10 +1,5 @@ import {cyan, green, magenta, yellow} from 'kleur'; -import { - FUNCTIONS_BUILD_DESCRIPTION, - FUNCTIONS_BUILD_NOTES, - OPTION_HELP, - OPTIONS_BUILD, OPTIONS_ENV, SNAPSHOT_UPLOAD_DESCRIPTION -} from '../constants/help.constants'; +import {OPTION_HELP, OPTIONS_ENV, SNAPSHOT_UPLOAD_DESCRIPTION} from '../constants/help.constants'; import {helpOutput} from './common.help'; import {TITLE} from './help'; diff --git a/src/schema/snapshot.schema.ts b/src/schema/snapshot.schema.ts index 998c7641..12b78100 100644 --- a/src/schema/snapshot.schema.ts +++ b/src/schema/snapshot.schema.ts @@ -1,6 +1,6 @@ import * as z from 'zod'; -const Uint8ArrayLike = z.instanceof(Uint8Array) as z.ZodType>; +const Uint8ArrayLike = z.instanceof(Uint8Array) as z.ZodType; // A Zod schema for the ic-management ReadCanisterSnapshotMetadataResponse type export const ReadCanisterSnapshotMetadataResponseSchema = z.strictObject({ diff --git a/src/services/modules/snapshot/snapshot.download.services.ts b/src/services/modules/snapshot/snapshot.download.services.ts index e5befb3e..71d8574f 100644 --- a/src/services/modules/snapshot/snapshot.download.services.ts +++ b/src/services/modules/snapshot/snapshot.download.services.ts @@ -11,17 +11,17 @@ import {pipeline} from 'node:stream/promises'; import ora from 'ora'; import {readCanisterSnapshotData, readCanisterSnapshotMetadata} from '../../../api/ic.api'; import {SNAPSHOTS_PATH} from '../../../constants/snapshot.constants'; -import {AssetKey} from '../../../types/asset-key'; +import {type AssetKey} from '../../../types/asset-key'; import { - DownloadSnapshotParams, - SnapshotBatchResult, - SnapshotFile, - SnapshotFilename, - SnapshotLog, - SnapshotMetadata + type DownloadSnapshotParams, + type SnapshotBatchResult, + type SnapshotFile, + type SnapshotFilename, + type SnapshotLog, + type SnapshotMetadata } from '../../../types/snapshot'; import {displaySegment} from '../../../utils/display.utils'; -import {BuildChunkFn, prepareDataChunks} from '../../../utils/snapshot.utils'; +import {type BuildChunkFn, prepareDataChunks} from '../../../utils/snapshot.utils'; type DataChunk = CanisterSnapshotMetadataKind; type DownloadedChunk = Uint8Array; @@ -160,11 +160,11 @@ const downloadSnapshotMetadataAndMemory = async ({ snapshotId: snapshotIdText, metadata, data: { - wasmModule: 'ok' === wasmModuleResult.status ? wasmModuleResult.snapshotFile : null, - wasmMemory: 'ok' === wasmMemoryResult.status ? wasmMemoryResult.snapshotFile : null, - stableMemory: 'ok' === stableMemoryResult.status ? stableMemoryResult.snapshotFile : null, + wasmModule: wasmModuleResult.status === 'ok' ? wasmModuleResult.snapshotFile : null, + wasmMemory: wasmMemoryResult.status === 'ok' ? wasmMemoryResult.snapshotFile : null, + stableMemory: stableMemoryResult.status === 'ok' ? stableMemoryResult.snapshotFile : null, wasmChunkStore: - 'ok' === wasmChunkStoreResult.status ? wasmChunkStoreResult.snapshotFile : null + wasmChunkStoreResult.status === 'ok' ? wasmChunkStoreResult.snapshotFile : null } } }); @@ -209,7 +209,7 @@ const assertSizeAndDownloadChunks = async ({ } & SnapshotLog): Promise<{status: 'ok'; snapshotFile: SnapshotFile} | {status: 'skip'}> => { if (size === 0n) { log(`No chunks to download for ${filename} (size = 0). Skipping.`); - await new Promise((resolve) => setTimeout(resolve, 2500)); + await sleep(); return {status: 'skip'}; } @@ -234,6 +234,9 @@ const assertSizeAndDownloadChunks = async ({ }; }; +// eslint-disable-next-line promise/avoid-new +const sleep = async () => await new Promise((resolve) => setTimeout(resolve, 2500)); + const assertAndDownloadWasmChunks = async ({ wasmChunkStore, log, @@ -245,7 +248,7 @@ const assertAndDownloadWasmChunks = async ({ SnapshotLog): Promise<{status: 'ok'; snapshotFile: SnapshotFile} | {status: 'skip'}> => { if (wasmChunkStore.length === 0) { log('Nothing to download from the WASM chunks store (length = 0). Skipping.'); - await new Promise((resolve) => setTimeout(resolve, 2500)); + await sleep(); return {status: 'skip'}; } @@ -334,6 +337,7 @@ const downloadAndWrite = async ({ } cb(); } catch (err: unknown) { + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion cb(err as Error); } } @@ -343,7 +347,7 @@ const downloadAndWrite = async ({ const hash = createHash('sha256'); const hasher = new Transform({ - transform(chunk, _enc, cb) { + transform(chunk: DownloadedChunk, _enc, cb) { hash.update(chunk); cb(null, chunk); } @@ -371,11 +375,12 @@ async function* batchDownloadChunks({ for (let i = 0; i < total; i = i + limit) { const batch = chunks.slice(i, i + limit); const downloadedChunks = await Promise.all( - batch.map((requestChunk) => - downloadChunk({ - ...params, - chunk: requestChunk - }) + batch.map( + async (requestChunk) => + await downloadChunk({ + ...params, + chunk: requestChunk + }) ) ); yield {downloadedChunks, progress: {index: i, done: Math.min(i + limit, total), total}}; diff --git a/src/services/modules/snapshot/snapshot.mission-control.services.ts b/src/services/modules/snapshot/snapshot.mission-control.services.ts index 16df7e90..36b64417 100644 --- a/src/services/modules/snapshot/snapshot.mission-control.services.ts +++ b/src/services/modules/snapshot/snapshot.mission-control.services.ts @@ -36,7 +36,9 @@ export const downloadSnapshotMissionControl = async () => { export const uploadSnapshotMissionControl = async (args?: string[]) => { await executeSnapshotFn({ - fn: (params) => uploadSnapshot({...params, args}), + fn: async (params) => { + await uploadSnapshot({...params, args}); + } }); }; diff --git a/src/services/modules/snapshot/snapshot.orbiter.services.ts b/src/services/modules/snapshot/snapshot.orbiter.services.ts index a13ba2db..d2bb6c17 100644 --- a/src/services/modules/snapshot/snapshot.orbiter.services.ts +++ b/src/services/modules/snapshot/snapshot.orbiter.services.ts @@ -34,7 +34,9 @@ export const downloadSnapshotOrbiter = async () => { export const uploadSnapshotOrbiter = async (args?: string[]) => { await executeSnapshotFn({ - fn: (params) => uploadSnapshot({...params, args}), + fn: async (params) => { + await uploadSnapshot({...params, args}); + } }); }; diff --git a/src/services/modules/snapshot/snapshot.satellite.services.ts b/src/services/modules/snapshot/snapshot.satellite.services.ts index 311d42ef..53af5658 100644 --- a/src/services/modules/snapshot/snapshot.satellite.services.ts +++ b/src/services/modules/snapshot/snapshot.satellite.services.ts @@ -37,7 +37,9 @@ export const downloadSnapshotSatellite = async () => { export const uploadSnapshotSatellite = async (args?: string[]) => { await executeSnapshotFn({ - fn: (params) => uploadSnapshot({...params, args}), + fn: async (params) => { + await uploadSnapshot({...params, args}); + } }); }; diff --git a/src/services/modules/snapshot/snapshot.services.ts b/src/services/modules/snapshot/snapshot.services.ts index 915b063c..57bda4ab 100644 --- a/src/services/modules/snapshot/snapshot.services.ts +++ b/src/services/modules/snapshot/snapshot.services.ts @@ -2,7 +2,9 @@ import type {snapshot_id} from '@dfinity/ic-management'; import {encodeSnapshotId} from '@dfinity/ic-management'; import {Principal} from '@dfinity/principal'; import {isEmptyString, isNullish, nonNullish} from '@dfinity/utils'; +import {nextArg} from '@junobuild/cli-tools'; import {red, yellow} from 'kleur'; +import {existsSync, lstatSync} from 'node:fs'; import ora from 'ora'; import { deleteCanisterSnapshot, @@ -15,8 +17,6 @@ import {displaySegment} from '../../../utils/display.utils'; import {confirmAndExit} from '../../../utils/prompt.utils'; import {downloadExistingSnapshot} from './snapshot.download.services'; import {uploadExistingSnapshot} from './snapshot.upload.services'; -import {nextArg} from '@junobuild/cli-tools'; -import {existsSync, lstatSync} from 'node:fs'; export const createSnapshot = async ({ canisterId: cId, diff --git a/src/services/modules/snapshot/snapshot.upload.services.ts b/src/services/modules/snapshot/snapshot.upload.services.ts index 11cfe252..a5c36af7 100644 --- a/src/services/modules/snapshot/snapshot.upload.services.ts +++ b/src/services/modules/snapshot/snapshot.upload.services.ts @@ -1,26 +1,25 @@ -import {encodeSnapshotId, snapshot_id} from '@dfinity/ic-management'; -import {UploadCanisterSnapshotDataKind} from '@dfinity/ic-management/dist/types/types/snapshot.params'; +import {encodeSnapshotId, type snapshot_id} from '@dfinity/ic-management'; +import {type UploadCanisterSnapshotDataKind} from '@dfinity/ic-management/dist/types/types/snapshot.params'; import {arrayBufferToUint8Array, isNullish, jsonReviver} from '@dfinity/utils'; -import {FileHandle} from 'fs/promises'; import {red} from 'kleur'; import {lstatSync} from 'node:fs'; -import {open as openFile, readFile} from 'node:fs/promises'; +import {type FileHandle, open as openFile, readFile} from 'node:fs/promises'; import {join, relative} from 'node:path'; import ora from 'ora'; import {uploadCanisterSnapshotData, uploadCanisterSnapshotMetadata} from '../../../api/ic.api'; import {SnapshotMetadataSchema} from '../../../schema/snapshot.schema'; import type {AssetKey} from '../../../types/asset-key'; import { - ReadCanisterSnapshotMetadataResponse, - SnapshotBatchResult, - SnapshotFile, - SnapshotLog, - SnapshotMetadata, - UploadSnapshotParams + type ReadCanisterSnapshotMetadataResponse, + type SnapshotBatchResult, + type SnapshotFile, + type SnapshotLog, + type SnapshotMetadata, + type UploadSnapshotParams } from '../../../types/snapshot'; import {displaySegment} from '../../../utils/display.utils'; -import {BuildChunkFn, prepareDataChunks} from '../../../utils/snapshot.utils'; import {computeLargeFileHash} from '../../../utils/hash.utils'; +import {type BuildChunkFn, prepareDataChunks} from '../../../utils/snapshot.utils'; interface DataChunk { kind: UploadCanisterSnapshotDataKind; @@ -266,13 +265,13 @@ async function* batchUploadChunks({ for (let i = 0; i < total; i = i + limit) { const batch = chunks.slice(i, i + limit); await Promise.all( - batch.map((requestChunk) => - uploadChunk({ + batch.map(async (requestChunk) => { + await uploadChunk({ ...params, sourceHandler, chunk: requestChunk - }) - ) + }); + }) ); yield {progress: {index: i, done: Math.min(i + limit, total), total}}; } diff --git a/src/types/snapshot.ts b/src/types/snapshot.ts index cb33cbee..ecf8cd3b 100644 --- a/src/types/snapshot.ts +++ b/src/types/snapshot.ts @@ -1,11 +1,11 @@ import type {snapshot_id} from '@dfinity/ic-management'; import type {Principal} from '@dfinity/principal'; -import * as z from 'zod'; +import type * as z from 'zod'; import { - ReadCanisterSnapshotMetadataResponseSchema, - SnapshotFilenameSchema, - SnapshotFileSchema, - SnapshotMetadataSchema + type ReadCanisterSnapshotMetadataResponseSchema, + type SnapshotFilenameSchema, + type SnapshotFileSchema, + type SnapshotMetadataSchema } from '../schema/snapshot.schema'; export type SnapshotFilename = z.infer; @@ -31,4 +31,4 @@ export interface SnapshotLog { export interface SnapshotBatchResult { progress: {index: number; done: number; total: number}; -} \ No newline at end of file +} diff --git a/src/utils/hash.utils.ts b/src/utils/hash.utils.ts index 5d30f652..908fc5cb 100644 --- a/src/utils/hash.utils.ts +++ b/src/utils/hash.utils.ts @@ -1,7 +1,7 @@ import {createHash} from 'node:crypto'; -import {pipeline} from 'node:stream/promises'; import {createReadStream} from 'node:fs'; import {Writable} from 'node:stream'; +import {pipeline} from 'node:stream/promises'; export const computeLargeFileHash = async (filepath: string): Promise => { const hash = createHash('sha256'); @@ -9,7 +9,7 @@ export const computeLargeFileHash = async (filepath: string): Promise => await pipeline( createReadStream(filepath), new Writable({ - write(chunk, _enc, cb) { + write(chunk: Uint8Array, _enc, cb) { hash.update(chunk); cb(); } @@ -17,4 +17,4 @@ export const computeLargeFileHash = async (filepath: string): Promise => ); return hash.digest('hex'); -}; \ No newline at end of file +}; diff --git a/src/utils/snapshot.utils.ts b/src/utils/snapshot.utils.ts index bedfc8cb..9ea500c3 100644 --- a/src/utils/snapshot.utils.ts +++ b/src/utils/snapshot.utils.ts @@ -25,4 +25,3 @@ export const prepareDataChunks = ({ return {chunks}; }; - From cd0f5cf7034b2f886a750dcc801d784f153fd8dd Mon Sep 17 00:00:00 2001 From: David Dal Busco Date: Fri, 3 Oct 2025 09:38:56 +0200 Subject: [PATCH 26/26] docs: incorrect referenced lib --- src/constants/snapshot.constants.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/constants/snapshot.constants.ts b/src/constants/snapshot.constants.ts index 99672f66..dd56969f 100644 --- a/src/constants/snapshot.constants.ts +++ b/src/constants/snapshot.constants.ts @@ -3,5 +3,5 @@ import {join} from 'node:path'; export const SNAPSHOTS_PATH = join(process.cwd(), '.snapshots'); // https://forum.dfinity.org/t/canister-snapshot-up-download/57397?u=peterparker -// Same value as INSTALL_MAX_CHUNK_SIZE in @dfinity/admin +// Same value as INSTALL_MAX_CHUNK_SIZE in @junobuild/admin export const SNAPSHOT_MAX_CHUNK_SIZE = 1_000_000n;