From 5c8ce6dc8ad28bd529eb8939c99aa1747a7d22c1 Mon Sep 17 00:00:00 2001 From: Mateusz Sikora Date: Wed, 4 Mar 2026 16:22:55 +0100 Subject: [PATCH 01/11] add lite interpreter --- bin/test-runner/common.ts | 5 ++++- bin/test-runner/javajam-071.ts | 2 +- bin/test-runner/w3f/accumulate.ts | 8 ++++---- bin/test-runner/w3f/runners.ts | 2 +- package-lock.json | 9 +++++++++ packages/core/pvm-host-calls/package.json | 1 + packages/core/pvm-host-calls/pvm-instance-manager.ts | 4 ++++ packages/jam/config/pvm-backend.ts | 4 +++- 8 files changed, 27 insertions(+), 8 deletions(-) diff --git a/bin/test-runner/common.ts b/bin/test-runner/common.ts index 20e3fe4cc..866470f24 100644 --- a/bin/test-runner/common.ts +++ b/bin/test-runner/common.ts @@ -19,14 +19,17 @@ export const logger = Logger.new(import.meta.filename, "test-runner"); export enum SelectedPvm { Ananas = "ananas", Builtin = "builtin", + Lite = "lite", } -export const ALL_PVMS = [SelectedPvm.Ananas, SelectedPvm.Builtin]; +export const ALL_PVMS = [SelectedPvm.Ananas, SelectedPvm.Builtin, SelectedPvm.Lite]; export function selectedPvmToBackend(pvm: SelectedPvm): PvmBackend { switch (pvm) { case SelectedPvm.Ananas: return PvmBackend.Ananas; case SelectedPvm.Builtin: return PvmBackend.BuiltIn; + case SelectedPvm.Lite: + return PvmBackend.Lite; default: assertNever(pvm); } diff --git a/bin/test-runner/javajam-071.ts b/bin/test-runner/javajam-071.ts index 875707663..878f949be 100644 --- a/bin/test-runner/javajam-071.ts +++ b/bin/test-runner/javajam-071.ts @@ -6,7 +6,7 @@ const runners = [ runner("state_transition", runStateTransition) .fromJson(StateTransition.fromJson) .fromBin(StateTransition.Codec) - .withVariants([SelectedPvm.Ananas, SelectedPvm.Builtin]), + .withVariants([SelectedPvm.Ananas, SelectedPvm.Builtin, SelectedPvm.Lite]), ].map((x) => x.build()); main(runners, "test-vectors/javajam_071", { diff --git a/bin/test-runner/w3f/accumulate.ts b/bin/test-runner/w3f/accumulate.ts index 4d763e81f..032c20240 100644 --- a/bin/test-runner/w3f/accumulate.ts +++ b/bin/test-runner/w3f/accumulate.ts @@ -12,7 +12,7 @@ import type { WorkPackageHash } from "@typeberry/block/refine-context.js"; import type { WorkReport } from "@typeberry/block/work-report.js"; import { fromJson, workReportFromJson } from "@typeberry/block-json"; import { asKnownSize, HashSet } from "@typeberry/collections"; -import { type ChainSpec, PvmBackend, PvmBackendNames } from "@typeberry/config"; +import { type ChainSpec, PvmBackendNames } from "@typeberry/config"; import { Blake2b } from "@typeberry/hash"; import { type FromJson, json } from "@typeberry/json-parser"; import { @@ -26,7 +26,7 @@ import { JsonService, JsonServicePre072 } from "@typeberry/state-json"; import { AccumulateOutput } from "@typeberry/transition/accumulate/accumulate-output.js"; import { Accumulate, type AccumulateRoot } from "@typeberry/transition/accumulate/index.js"; import { Compatibility, deepEqual, GpVersion, Result } from "@typeberry/utils"; -import type { RunOptions } from "../common.js"; +import { type RunOptions, type SelectedPvm, selectedPvmToBackend } from "../common.js"; class Input { static fromJson: FromJson = { @@ -142,9 +142,9 @@ export class AccumulateTest { export async function runAccumulateTest( test: AccumulateTest, { chainSpec, accumulateSequentially }: RunOptions, - variant: "ananas" | "builtin", + variant: SelectedPvm, ) { - const pvm = variant === "ananas" ? PvmBackend.Ananas : PvmBackend.BuiltIn; + const pvm = selectedPvmToBackend(variant); const options = { pvm, accumulateSequentially }; /** * entropy has to be moved to input because state is incompatibile - diff --git a/bin/test-runner/w3f/runners.ts b/bin/test-runner/w3f/runners.ts index 2cb318a6e..3d9c40b9f 100644 --- a/bin/test-runner/w3f/runners.ts +++ b/bin/test-runner/w3f/runners.ts @@ -47,7 +47,7 @@ import { runShufflingTests, shufflingTestsFromJson } from "./shuffling.js"; import { runStatisticsTestFull, runStatisticsTestTiny, StatisticsTestFull, StatisticsTestTiny } from "./statistics.js"; import { runTrieTest, trieTestSuiteFromJson } from "./trie.js"; -const pvms: SelectedPvm[] = [SelectedPvm.Ananas, SelectedPvm.Builtin]; +const pvms: SelectedPvm[] = [SelectedPvm.Ananas, SelectedPvm.Builtin, SelectedPvm.Lite]; const tiny = [tinyChainSpec]; const full = [fullChainSpec]; const tinyFull = [...tiny, ...full]; diff --git a/package-lock.json b/package-lock.json index 7c2ea80d3..0c7b29b26 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1197,6 +1197,14 @@ "integrity": "sha512-TRCCd5npU38ozMBPPj/0wzCrZ+brsnXL9oZQ1lC38s6OnBeg8cznLMaMNU3GCpR2FpU52dusq7ZgNeoUcAmoSQ==", "license": "MPL-2.0" }, + "node_modules/@fluffylabs/pvm-interpreter-lite": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/@fluffylabs/pvm-interpreter-lite/-/pvm-interpreter-lite-0.1.2.tgz", + "integrity": "sha512-JmqWAVCWkxz01uP70U1MgOO6ay6XHzIlrWTfKGpL4XJwaZ65BDR/5CPAf8o/VbLJGqiD4b46PZrKYR7jcwhvtw==", + "peerDependencies": { + "@typeberry/lib": "*" + } + }, "node_modules/@gerrit0/mini-shiki": { "version": "3.15.0", "resolved": "https://registry.npmjs.org/@gerrit0/mini-shiki/-/mini-shiki-3.15.0.tgz", @@ -8733,6 +8741,7 @@ "version": "0.5.9", "license": "MPL-2.0", "dependencies": { + "@fluffylabs/pvm-interpreter-lite": "^0.1.2", "@typeberry/bytes": "*", "@typeberry/config": "*", "@typeberry/logger": "*", diff --git a/packages/core/pvm-host-calls/package.json b/packages/core/pvm-host-calls/package.json index 9f432c092..9f70cc238 100644 --- a/packages/core/pvm-host-calls/package.json +++ b/packages/core/pvm-host-calls/package.json @@ -11,6 +11,7 @@ "@typeberry/pvm-interface": "*", "@typeberry/pvm-interpreter": "*", "@typeberry/pvm-interpreter-ananas": "*", + "@fluffylabs/pvm-interpreter-lite": "^0.1.2", "@typeberry/utils": "*" }, "scripts": { diff --git a/packages/core/pvm-host-calls/pvm-instance-manager.ts b/packages/core/pvm-host-calls/pvm-instance-manager.ts index 1cd0059db..b0a024701 100644 --- a/packages/core/pvm-host-calls/pvm-instance-manager.ts +++ b/packages/core/pvm-host-calls/pvm-instance-manager.ts @@ -1,3 +1,4 @@ +import { Interpreter as LiteInterpreter } from "@fluffylabs/pvm-interpreter-lite"; import { PvmBackend } from "@typeberry/config"; import type { IPvmInterpreter } from "@typeberry/pvm-interface"; import { Interpreter } from "@typeberry/pvm-interpreter"; @@ -25,6 +26,9 @@ export class PvmInstanceManager { case PvmBackend.Ananas: instances.push(await AnanasInterpreter.new()); break; + case PvmBackend.Lite: + instances.push(new LiteInterpreter()); + break; default: assertNever(interpreter); } diff --git a/packages/jam/config/pvm-backend.ts b/packages/jam/config/pvm-backend.ts index 8e07a0ed5..b8260ba68 100644 --- a/packages/jam/config/pvm-backend.ts +++ b/packages/jam/config/pvm-backend.ts @@ -1,5 +1,5 @@ /** Implemented PVM Backends names in THE SAME ORDER as enum. */ -export const PvmBackendNames = ["built-in", "ananas"]; +export const PvmBackendNames = ["built-in", "ananas", "lite"]; /** Implemented PVM Backends to choose from. */ export enum PvmBackend { @@ -7,4 +7,6 @@ export enum PvmBackend { BuiltIn = 0, /** Ananas 🍍 interpreter. */ Ananas = 1, + /** Lite interpreter. */ + Lite = 2, } From 72093264d60dad1d2a2255ec15f62089ffaf1526 Mon Sep 17 00:00:00 2001 From: Mateusz Sikora Date: Wed, 4 Mar 2026 17:59:03 +0100 Subject: [PATCH 02/11] integrate the lite interpreter across the entire codebase --- README.md | 10 +++++++--- bin/jam/README.md | 4 ++-- bin/test-runner/common.test.ts | 10 +++++----- package-lock.json | 8 ++++---- packages/README.md | 3 ++- packages/core/pvm-host-calls/package.json | 2 +- packages/jam/transition/accumulate/accumulate.test.ts | 2 +- packages/workers/importer/protocol.ts | 3 +++ 8 files changed, 25 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 7ac184ff0..56450e3e7 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ JAM Prize requirements - [x] Performance optimisations - [ ] Milestone 2 - [x] Networking (partial) - - [x] Fast PVM (ananas) + - [x] Fast PVM (ananas, lite) - [ ] Milestone 3 - [ ] PVM Recompiler - [ ] Milestone 4 @@ -107,6 +107,7 @@ $ npm start -w @typeberry/rpc - [PVM Debugger](https://github.com/fluffylabs/pvm-debugger) - load & inspect a PVM program - [Gray Paper Reader](https://github.com/fluffylabs/graypaper-reader) - view the Gray Paper - [Ananas](https://github.com/tomusdrw/anan-as) - AssemblyScript PVM interpreter +- [Lite](https://www.npmjs.com/package/@fluffylabs/pvm-interpreter-lite) - Lightweight TypeScript PVM interpreter ### Formatting & linting @@ -171,7 +172,7 @@ cases by altering the glob pattern in the path. #### Selecting PVM Backend -By default, test vectors are run with both PVM backends (built-in and Ananas). +By default, test vectors are run with all PVM backends (built-in, Ananas, and Lite). You can select a specific PVM backend using the `--pvm` option: ```bash @@ -181,7 +182,10 @@ $ npm run w3f-davxy:0.7.1 -w @typeberry/test-runner -- --pvm builtin # Run tests with Ananas PVM only $ npm run w3f-davxy:0.7.1 -w @typeberry/test-runner -- --pvm ananas -# Run tests with both PVMs (default) +# Run tests with Lite PVM only +$ npm run w3f-davxy:0.7.1 -w @typeberry/test-runner -- --pvm lite + +# Run tests with all PVMs (default) $ npm run w3f-davxy:0.7.1 -w @typeberry/test-runner ``` diff --git a/bin/jam/README.md b/bin/jam/README.md index d0cec8d3b..edc688280 100644 --- a/bin/jam/README.md +++ b/bin/jam/README.md @@ -113,12 +113,12 @@ jam --config=custom-config.json ### `--pvm` -Select the PVM (Polkavm) backend to use. Available options: `wasmtime`, `interpreter`, `ananas`. +Select the PVM (Polkavm) backend to use. Available options: `built-in`, `ananas`, `lite`. Default: `ananas` ```bash -jam --pvm=wasmtime +jam --pvm=lite ``` ## Environment Variables diff --git a/bin/test-runner/common.test.ts b/bin/test-runner/common.test.ts index 50f6877c5..2bd64f5da 100644 --- a/bin/test-runner/common.test.ts +++ b/bin/test-runner/common.test.ts @@ -23,7 +23,7 @@ describe("test runner common", () => { deepEqual(result, { initialFiles: ["file1.json", "file2.json"], - pvms: [SelectedPvm.Ananas, SelectedPvm.Builtin], + pvms: [SelectedPvm.Ananas, SelectedPvm.Builtin, SelectedPvm.Lite], accumulateSequentially: false, }); }); @@ -36,7 +36,7 @@ describe("test runner common", () => { const _result = parseArgs(args); }, { - message: "Unknown pvm value: invalid. Use one of ananas, builtin.", + message: "Unknown pvm value: invalid. Use one of ananas, builtin, lite.", }, ); }); @@ -48,7 +48,7 @@ describe("test runner common", () => { deepEqual(result, { initialFiles: ["file1.json"], - pvms: [SelectedPvm.Ananas, SelectedPvm.Builtin], + pvms: [SelectedPvm.Ananas, SelectedPvm.Builtin, SelectedPvm.Lite], accumulateSequentially: true, }); }); @@ -60,7 +60,7 @@ describe("test runner common", () => { deepEqual(result, { initialFiles: ["file1.json"], - pvms: [SelectedPvm.Ananas, SelectedPvm.Builtin], + pvms: [SelectedPvm.Ananas, SelectedPvm.Builtin, SelectedPvm.Lite], accumulateSequentially: true, }); }); @@ -72,7 +72,7 @@ describe("test runner common", () => { deepEqual(result, { initialFiles: ["file1.json"], - pvms: [SelectedPvm.Ananas, SelectedPvm.Builtin], + pvms: [SelectedPvm.Ananas, SelectedPvm.Builtin, SelectedPvm.Lite], accumulateSequentially: false, }); }); diff --git a/package-lock.json b/package-lock.json index 0c7b29b26..fa99bbf17 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1198,9 +1198,9 @@ "license": "MPL-2.0" }, "node_modules/@fluffylabs/pvm-interpreter-lite": { - "version": "0.1.2", - "resolved": "https://registry.npmjs.org/@fluffylabs/pvm-interpreter-lite/-/pvm-interpreter-lite-0.1.2.tgz", - "integrity": "sha512-JmqWAVCWkxz01uP70U1MgOO6ay6XHzIlrWTfKGpL4XJwaZ65BDR/5CPAf8o/VbLJGqiD4b46PZrKYR7jcwhvtw==", + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/@fluffylabs/pvm-interpreter-lite/-/pvm-interpreter-lite-0.1.3.tgz", + "integrity": "sha512-thAPhSyrZMhuP35Q7z44iu3gZIsJRDUE1vXUuAtpfYDz3x8BRFDuDcKITJjVoLZOgdr6rf4ic/MvhcyANs7sAQ==", "peerDependencies": { "@typeberry/lib": "*" } @@ -8741,7 +8741,7 @@ "version": "0.5.9", "license": "MPL-2.0", "dependencies": { - "@fluffylabs/pvm-interpreter-lite": "^0.1.2", + "@fluffylabs/pvm-interpreter-lite": "^0.1.3", "@typeberry/bytes": "*", "@typeberry/config": "*", "@typeberry/logger": "*", diff --git a/packages/README.md b/packages/README.md index c05b83c54..042a6e8ed 100644 --- a/packages/README.md +++ b/packages/README.md @@ -24,7 +24,8 @@ Contains fundamental packages that provide low-level functionality and utilities - **pvm-host-calls** - PVM host call implementations - **pvm-interface** - PVM interface definitions - **pvm-interpreter** - PVM interpreter functionality -- **pvm-interpreter-ananas** - Ananas PVM interpreter implementation +- **pvm-interpreter-ananas** - Ananas PVM interpreter implementation (AssemblyScript/WASM) +- **pvm-interpreter-lite** - Lightweight PVM interpreter (`@fluffylabs/pvm-interpreter-lite`) - **shuffling** - Shuffling algorithms - **telemetry** - OpenTelemetry initialization utilities - **trie** - Trie data structure implementation diff --git a/packages/core/pvm-host-calls/package.json b/packages/core/pvm-host-calls/package.json index 9f70cc238..95e7e51b3 100644 --- a/packages/core/pvm-host-calls/package.json +++ b/packages/core/pvm-host-calls/package.json @@ -11,7 +11,7 @@ "@typeberry/pvm-interface": "*", "@typeberry/pvm-interpreter": "*", "@typeberry/pvm-interpreter-ananas": "*", - "@fluffylabs/pvm-interpreter-lite": "^0.1.2", + "@fluffylabs/pvm-interpreter-lite": "^0.1.3", "@typeberry/utils": "*" }, "scripts": { diff --git a/packages/jam/transition/accumulate/accumulate.test.ts b/packages/jam/transition/accumulate/accumulate.test.ts index 6140aa681..2fdc99f39 100644 --- a/packages/jam/transition/accumulate/accumulate.test.ts +++ b/packages/jam/transition/accumulate/accumulate.test.ts @@ -46,7 +46,7 @@ type TestServiceInfo = { lastAccumulation?: TimeSlot; }; -[PvmBackend.BuiltIn, PvmBackend.Ananas].forEach((pvm) => { +[PvmBackend.BuiltIn, PvmBackend.Ananas, PvmBackend.Lite].forEach((pvm) => { [false, true].forEach((accumulateSequentially) => { const options = { pvm, accumulateSequentially }; describe(`accumulate: ${PvmBackendNames[pvm]} (sequential accumulation: ${accumulateSequentially})`, () => { diff --git a/packages/workers/importer/protocol.ts b/packages/workers/importer/protocol.ts index 99c9377c9..9ae3e73b8 100644 --- a/packages/workers/importer/protocol.ts +++ b/packages/workers/importer/protocol.ts @@ -89,6 +89,9 @@ export class ImporterConfig { if (o === PvmBackend.Ananas) { return PvmBackend.Ananas; } + if (o === PvmBackend.Lite) { + return PvmBackend.Lite; + } throw new Error(`Invalid PvmBackend: ${o}`); }, ), From 6381f2280fff9b35c7f1603a4a2d1f03127f002e Mon Sep 17 00:00:00 2001 From: Mateusz Sikora Date: Wed, 4 Mar 2026 21:48:36 +0100 Subject: [PATCH 03/11] bump lite interpreter --- package-lock.json | 8 ++++---- packages/core/pvm-host-calls/package.json | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package-lock.json b/package-lock.json index fa99bbf17..15781bb78 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1198,9 +1198,9 @@ "license": "MPL-2.0" }, "node_modules/@fluffylabs/pvm-interpreter-lite": { - "version": "0.1.3", - "resolved": "https://registry.npmjs.org/@fluffylabs/pvm-interpreter-lite/-/pvm-interpreter-lite-0.1.3.tgz", - "integrity": "sha512-thAPhSyrZMhuP35Q7z44iu3gZIsJRDUE1vXUuAtpfYDz3x8BRFDuDcKITJjVoLZOgdr6rf4ic/MvhcyANs7sAQ==", + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/@fluffylabs/pvm-interpreter-lite/-/pvm-interpreter-lite-0.1.4.tgz", + "integrity": "sha512-PLbLoVFfQbgStBjveMZMvIT4oVn9eaWIL/tqfemzlGCRrt2Zo/N138PaAAz9aAuiYidoU4dYFRAQ/kEnqg9l4Q==", "peerDependencies": { "@typeberry/lib": "*" } @@ -8741,7 +8741,7 @@ "version": "0.5.9", "license": "MPL-2.0", "dependencies": { - "@fluffylabs/pvm-interpreter-lite": "^0.1.3", + "@fluffylabs/pvm-interpreter-lite": "^0.1.4", "@typeberry/bytes": "*", "@typeberry/config": "*", "@typeberry/logger": "*", diff --git a/packages/core/pvm-host-calls/package.json b/packages/core/pvm-host-calls/package.json index 95e7e51b3..8523bcc71 100644 --- a/packages/core/pvm-host-calls/package.json +++ b/packages/core/pvm-host-calls/package.json @@ -11,7 +11,7 @@ "@typeberry/pvm-interface": "*", "@typeberry/pvm-interpreter": "*", "@typeberry/pvm-interpreter-ananas": "*", - "@fluffylabs/pvm-interpreter-lite": "^0.1.3", + "@fluffylabs/pvm-interpreter-lite": "^0.1.4", "@typeberry/utils": "*" }, "scripts": { From c10173e6c3bd5b86d06820e6dfacfcff34fdad31 Mon Sep 17 00:00:00 2001 From: Mateusz Sikora Date: Wed, 4 Mar 2026 23:14:50 +0100 Subject: [PATCH 04/11] bump lite interpreter --- package-lock.json | 8 ++++---- packages/core/pvm-host-calls/package.json | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package-lock.json b/package-lock.json index 15781bb78..331409d58 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1198,9 +1198,9 @@ "license": "MPL-2.0" }, "node_modules/@fluffylabs/pvm-interpreter-lite": { - "version": "0.1.4", - "resolved": "https://registry.npmjs.org/@fluffylabs/pvm-interpreter-lite/-/pvm-interpreter-lite-0.1.4.tgz", - "integrity": "sha512-PLbLoVFfQbgStBjveMZMvIT4oVn9eaWIL/tqfemzlGCRrt2Zo/N138PaAAz9aAuiYidoU4dYFRAQ/kEnqg9l4Q==", + "version": "0.1.5", + "resolved": "https://registry.npmjs.org/@fluffylabs/pvm-interpreter-lite/-/pvm-interpreter-lite-0.1.5.tgz", + "integrity": "sha512-qxnU76UEObWicx2sUifWYWpz9hb1dNZFSdgsQeLaY5jhG+3plfOMRD0cy0DnLGdRxaEJMJgI56Ei+WfcD7mu3g==", "peerDependencies": { "@typeberry/lib": "*" } @@ -8741,7 +8741,7 @@ "version": "0.5.9", "license": "MPL-2.0", "dependencies": { - "@fluffylabs/pvm-interpreter-lite": "^0.1.4", + "@fluffylabs/pvm-interpreter-lite": "^0.1.5", "@typeberry/bytes": "*", "@typeberry/config": "*", "@typeberry/logger": "*", diff --git a/packages/core/pvm-host-calls/package.json b/packages/core/pvm-host-calls/package.json index 8523bcc71..ba2521de3 100644 --- a/packages/core/pvm-host-calls/package.json +++ b/packages/core/pvm-host-calls/package.json @@ -11,7 +11,7 @@ "@typeberry/pvm-interface": "*", "@typeberry/pvm-interpreter": "*", "@typeberry/pvm-interpreter-ananas": "*", - "@fluffylabs/pvm-interpreter-lite": "^0.1.4", + "@fluffylabs/pvm-interpreter-lite": "^0.1.5", "@typeberry/utils": "*" }, "scripts": { From 42708e61b3b6c54771680b0964a45a55705746f3 Mon Sep 17 00:00:00 2001 From: Mateusz Sikora Date: Thu, 5 Mar 2026 08:30:40 +0100 Subject: [PATCH 05/11] review fixes --- bin/test-runner/javajam-071.ts | 4 ++-- bin/test-runner/w3f/runners.ts | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/bin/test-runner/javajam-071.ts b/bin/test-runner/javajam-071.ts index 878f949be..387c6ec49 100644 --- a/bin/test-runner/javajam-071.ts +++ b/bin/test-runner/javajam-071.ts @@ -1,12 +1,12 @@ import { StateTransition } from "@typeberry/state-vectors"; -import { logger, main, parseArgs, runner, SelectedPvm } from "./common.js"; +import { ALL_PVMS, logger, main, parseArgs, runner } from "./common.js"; import { runStateTransition } from "./state-transition/state-transition.js"; const runners = [ runner("state_transition", runStateTransition) .fromJson(StateTransition.fromJson) .fromBin(StateTransition.Codec) - .withVariants([SelectedPvm.Ananas, SelectedPvm.Builtin, SelectedPvm.Lite]), + .withVariants(ALL_PVMS), ].map((x) => x.build()); main(runners, "test-vectors/javajam_071", { diff --git a/bin/test-runner/w3f/runners.ts b/bin/test-runner/w3f/runners.ts index 3d9c40b9f..7a70ed33d 100644 --- a/bin/test-runner/w3f/runners.ts +++ b/bin/test-runner/w3f/runners.ts @@ -13,7 +13,7 @@ import { } from "@typeberry/block-json"; import { fullChainSpec, tinyChainSpec } from "@typeberry/config"; import { StateTransition } from "@typeberry/state-vectors"; -import { runner, SelectedPvm } from "../common.js"; +import { ALL_PVMS, runner } from "../common.js"; import { runStateTransition } from "../state-transition/state-transition.js"; import { AccumulateTest, runAccumulateTest } from "./accumulate.js"; import { AssurancesTestFull, AssurancesTestTiny, runAssurancesTestFull, runAssurancesTestTiny } from "./assurances.js"; @@ -47,7 +47,7 @@ import { runShufflingTests, shufflingTestsFromJson } from "./shuffling.js"; import { runStatisticsTestFull, runStatisticsTestTiny, StatisticsTestFull, StatisticsTestTiny } from "./statistics.js"; import { runTrieTest, trieTestSuiteFromJson } from "./trie.js"; -const pvms: SelectedPvm[] = [SelectedPvm.Ananas, SelectedPvm.Builtin, SelectedPvm.Lite]; +const pvms = ALL_PVMS; const tiny = [tinyChainSpec]; const full = [fullChainSpec]; const tinyFull = [...tiny, ...full]; From 6f9b9640a08978a4af02702893f7e85fcf52e5ab Mon Sep 17 00:00:00 2001 From: Mateusz Sikora Date: Thu, 5 Mar 2026 11:44:50 +0100 Subject: [PATCH 06/11] change default interpreter to lite --- packages/jam/config-node/node-config.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/jam/config-node/node-config.ts b/packages/jam/config-node/node-config.ts index 8531b6adb..24c8b0025 100644 --- a/packages/jam/config-node/node-config.ts +++ b/packages/jam/config-node/node-config.ts @@ -19,7 +19,7 @@ export const DEFAULT_CONFIG = "default"; export const NODE_DEFAULTS = { name: isBrowser() ? "browser" : os.hostname(), config: [DEFAULT_CONFIG], - pvm: PvmBackend.Ananas, + pvm: PvmBackend.Lite, }; /** Chain spec chooser. */ From a412f337996cd6231d860635609abe0a54015314 Mon Sep 17 00:00:00 2001 From: Mateusz Sikora Date: Fri, 6 Mar 2026 14:18:08 +0100 Subject: [PATCH 07/11] bump interpreter --- package-lock.json | 11 ++++------- packages/core/pvm-host-calls/package.json | 2 +- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/package-lock.json b/package-lock.json index 331409d58..63c3ae394 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1198,12 +1198,9 @@ "license": "MPL-2.0" }, "node_modules/@fluffylabs/pvm-interpreter-lite": { - "version": "0.1.5", - "resolved": "https://registry.npmjs.org/@fluffylabs/pvm-interpreter-lite/-/pvm-interpreter-lite-0.1.5.tgz", - "integrity": "sha512-qxnU76UEObWicx2sUifWYWpz9hb1dNZFSdgsQeLaY5jhG+3plfOMRD0cy0DnLGdRxaEJMJgI56Ei+WfcD7mu3g==", - "peerDependencies": { - "@typeberry/lib": "*" - } + "version": "0.1.6", + "resolved": "https://registry.npmjs.org/@fluffylabs/pvm-interpreter-lite/-/pvm-interpreter-lite-0.1.6.tgz", + "integrity": "sha512-1dgeox4ifmFl1/U+2Tne9o8DopEWQ6FylZjpgy6fErhDFjy0FUNf/5QI0X5Is0yOcUh6yZGEj2/74UmtBSYMXA==" }, "node_modules/@gerrit0/mini-shiki": { "version": "3.15.0", @@ -8741,7 +8738,7 @@ "version": "0.5.9", "license": "MPL-2.0", "dependencies": { - "@fluffylabs/pvm-interpreter-lite": "^0.1.5", + "@fluffylabs/pvm-interpreter-lite": "^0.1.6", "@typeberry/bytes": "*", "@typeberry/config": "*", "@typeberry/logger": "*", diff --git a/packages/core/pvm-host-calls/package.json b/packages/core/pvm-host-calls/package.json index ba2521de3..cf9c6e19d 100644 --- a/packages/core/pvm-host-calls/package.json +++ b/packages/core/pvm-host-calls/package.json @@ -11,7 +11,7 @@ "@typeberry/pvm-interface": "*", "@typeberry/pvm-interpreter": "*", "@typeberry/pvm-interpreter-ananas": "*", - "@fluffylabs/pvm-interpreter-lite": "^0.1.5", + "@fluffylabs/pvm-interpreter-lite": "^0.1.6", "@typeberry/utils": "*" }, "scripts": { From f14f103d5e92d1c9b51c236c46cc3ebda37e1522 Mon Sep 17 00:00:00 2001 From: Mateusz Sikora Date: Tue, 10 Mar 2026 12:01:55 +0100 Subject: [PATCH 08/11] add lite interpreter wrapper --- package-lock.json | 18 ++- package.json | 1 + packages/core/pvm-host-calls/package.json | 2 +- .../pvm-host-calls/pvm-instance-manager.ts | 2 +- packages/core/pvm-interpreter-lite/index.ts | 117 ++++++++++++++++++ .../core/pvm-interpreter-lite/package.json | 18 +++ 6 files changed, 155 insertions(+), 3 deletions(-) create mode 100644 packages/core/pvm-interpreter-lite/index.ts create mode 100644 packages/core/pvm-interpreter-lite/package.json diff --git a/package-lock.json b/package-lock.json index 63c3ae394..2c4f2ec91 100644 --- a/package-lock.json +++ b/package-lock.json @@ -37,6 +37,7 @@ "packages/core/pvm-interface", "packages/core/pvm-interpreter", "packages/core/pvm-interpreter-ananas", + "packages/core/pvm-interpreter-lite", "packages/core/shuffling", "packages/core/telemetry", "packages/core/trie", @@ -3796,6 +3797,10 @@ "resolved": "packages/core/pvm-interpreter-ananas", "link": true }, + "node_modules/@typeberry/pvm-interpreter-lite": { + "resolved": "packages/core/pvm-interpreter-lite", + "link": true + }, "node_modules/@typeberry/rpc": { "resolved": "bin/rpc", "link": true @@ -8738,7 +8743,6 @@ "version": "0.5.9", "license": "MPL-2.0", "dependencies": { - "@fluffylabs/pvm-interpreter-lite": "^0.1.6", "@typeberry/bytes": "*", "@typeberry/config": "*", "@typeberry/logger": "*", @@ -8746,6 +8750,7 @@ "@typeberry/pvm-interface": "*", "@typeberry/pvm-interpreter": "*", "@typeberry/pvm-interpreter-ananas": "*", + "@typeberry/pvm-interpreter-lite": "*", "@typeberry/utils": "*" } }, @@ -8784,6 +8789,17 @@ "assemblyscript-loader": "^0.3.0" } }, + "packages/core/pvm-interpreter-lite": { + "name": "@typeberry/pvm-interpreter-lite", + "version": "0.5.9", + "license": "MPL-2.0", + "dependencies": { + "@fluffylabs/pvm-interpreter-lite": "^0.1.6", + "@typeberry/numbers": "*", + "@typeberry/pvm-interface": "*", + "@typeberry/utils": "*" + } + }, "packages/core/shuffling": { "name": "@typeberry/shuffling", "version": "0.5.9", diff --git a/package.json b/package.json index 6ba7eba75..4b6fb29c2 100644 --- a/package.json +++ b/package.json @@ -32,6 +32,7 @@ "packages/core/pvm-interface", "packages/core/pvm-interpreter", "packages/core/pvm-interpreter-ananas", + "packages/core/pvm-interpreter-lite", "packages/core/shuffling", "packages/core/telemetry", "packages/core/trie", diff --git a/packages/core/pvm-host-calls/package.json b/packages/core/pvm-host-calls/package.json index cf9c6e19d..0c61afc2e 100644 --- a/packages/core/pvm-host-calls/package.json +++ b/packages/core/pvm-host-calls/package.json @@ -11,7 +11,7 @@ "@typeberry/pvm-interface": "*", "@typeberry/pvm-interpreter": "*", "@typeberry/pvm-interpreter-ananas": "*", - "@fluffylabs/pvm-interpreter-lite": "^0.1.6", + "@typeberry/pvm-interpreter-lite": "*", "@typeberry/utils": "*" }, "scripts": { diff --git a/packages/core/pvm-host-calls/pvm-instance-manager.ts b/packages/core/pvm-host-calls/pvm-instance-manager.ts index b0a024701..ee2a4a551 100644 --- a/packages/core/pvm-host-calls/pvm-instance-manager.ts +++ b/packages/core/pvm-host-calls/pvm-instance-manager.ts @@ -1,8 +1,8 @@ -import { Interpreter as LiteInterpreter } from "@fluffylabs/pvm-interpreter-lite"; import { PvmBackend } from "@typeberry/config"; import type { IPvmInterpreter } from "@typeberry/pvm-interface"; import { Interpreter } from "@typeberry/pvm-interpreter"; import { AnanasInterpreter } from "@typeberry/pvm-interpreter-ananas"; +import { LiteInterpreter } from "@typeberry/pvm-interpreter-lite"; import { assertNever } from "@typeberry/utils"; type ResolveFn = (pvm: IPvmInterpreter) => void; diff --git a/packages/core/pvm-interpreter-lite/index.ts b/packages/core/pvm-interpreter-lite/index.ts new file mode 100644 index 000000000..f80009604 --- /dev/null +++ b/packages/core/pvm-interpreter-lite/index.ts @@ -0,0 +1,117 @@ +import { Interpreter } from "@fluffylabs/pvm-interpreter-lite"; +import { tryAsU32, type U32 } from "@typeberry/numbers"; +import { + type Gas, + getPageStartAddress, + type IGasCounter, + type IMemory, + type IPvmInterpreter, + type IRegisters, + type PageFault, + Status, + tryAsBigGas, + tryAsGas, +} from "@typeberry/pvm-interface"; +import { OK, Result } from "@typeberry/utils"; + +export class LiteInterpreter implements IPvmInterpreter { + readonly registers: IRegisters; + readonly memory: IMemory; + readonly gas: IGasCounter; + + private readonly inner: Interpreter; + + constructor() { + this.inner = new Interpreter(); + + const regs = this.inner.registers; + this.registers = { + getAllEncoded(): Uint8Array { + return regs.getAllEncoded(); + }, + setAllEncoded(bytes: Uint8Array): void { + regs.setAllEncoded(bytes); + }, + }; + + // Memory: thin adapter for return type (number → Result) + const mem = this.inner.memory; + this.memory = { + store(address: U32, bytes: Uint8Array): Result { + if (mem.fastStore(address, bytes) === 0) { + return Result.ok(OK); + } + return Result.error({ address: getPageStartAddress(address) }, () => "Memory is unwritable!"); + }, + read(address: U32, result: Uint8Array): Result { + if (result.length === 0) { + return Result.ok(OK); + } + if (mem.fastLoad(result, address) === 0) { + return Result.ok(OK); + } + return Result.error({ address: getPageStartAddress(address) }, () => "Memory is inaccessible!"); + }, + }; + + // Gas: wrap to convert between Gas types + const innerRef = this.inner; + this.gas = { + initialGas: tryAsGas(0n), + get(): Gas { + return tryAsGas(innerRef.gas.get()); + }, + set(g: Gas): void { + innerRef.gas.set(BigInt(g)); + }, + sub(g: Gas): boolean { + return innerRef.gas.sub(BigInt(g)); + }, + used(): Gas { + const gasConsumed = BigInt(this.initialGas) - BigInt(innerRef.gas.get()); + if (gasConsumed < 0) { + return this.initialGas; + } + return tryAsBigGas(gasConsumed); + }, + }; + } + + resetJam(program: Uint8Array, args: Uint8Array, pc: number, gas: Gas): void { + this.gas.initialGas = gas; + this.inner.resetJam(program, args, pc, BigInt(gas)); + } + + runProgram(): void { + this.inner.runProgram(); + } + + getStatus(): Status { + const status = this.inner.getStatus(); + switch (status) { + case 255: + return Status.OK; + case 0: + return Status.HALT; + case 1: + return Status.PANIC; + case 2: + return Status.FAULT; + case 3: + return Status.HOST; + case 4: + return Status.OOG; + default: + return Status.PANIC; + } + } + + getPC(): number { + return this.inner.getPC(); + } + + getExitParam(): U32 | null { + const param = this.inner.getExitParam(); + return param === null ? null : tryAsU32(param); + } +} diff --git a/packages/core/pvm-interpreter-lite/package.json b/packages/core/pvm-interpreter-lite/package.json new file mode 100644 index 000000000..349540dfe --- /dev/null +++ b/packages/core/pvm-interpreter-lite/package.json @@ -0,0 +1,18 @@ +{ + "name": "@typeberry/pvm-interpreter-lite", + "version": "0.5.9", + "description": "Lite PVM interpreter wrapper.", + "main": "index.ts", + "dependencies": { + "@fluffylabs/pvm-interpreter-lite": "^0.1.6", + "@typeberry/numbers": "*", + "@typeberry/pvm-interface": "*", + "@typeberry/utils": "*" + }, + "scripts": { + "test": "tsx --test $(find . -type f -name '*.test.ts' | tr '\\n' ' ')" + }, + "author": "Fluffy Labs", + "license": "MPL-2.0", + "type": "module" +} From 22b1c9a7a857f0853a729a8dee6feb27c7a608f7 Mon Sep 17 00:00:00 2001 From: Mateusz Sikora Date: Sun, 29 Mar 2026 12:56:07 +0200 Subject: [PATCH 09/11] an attempt to run PVM in worker --- bin/test-runner/common.test.ts | 5 + bin/test-runner/common.ts | 15 +- .../state-transition/state-transition.ts | 2 +- bin/test-runner/w3f/accumulate.ts | 4 +- .../jam/jam-host-calls/accumulate/eject.ts | 2 +- .../jam/jam-host-calls/accumulate/forget.ts | 2 +- packages/jam/jam-host-calls/accumulate/new.ts | 2 +- .../jam/jam-host-calls/accumulate/provide.ts | 2 +- .../jam/jam-host-calls/accumulate/query.ts | 2 +- .../jam/jam-host-calls/accumulate/solicit.ts | 2 +- .../jam/jam-host-calls/accumulate/transfer.ts | 2 +- .../jam/jam-host-calls/accumulate/upgrade.ts | 2 +- .../externalities/partial-state-mock.ts | 20 +- .../externalities/partial-state.ts | 16 +- .../externalities/state-update.ts | 125 ++++- .../externalities/test-accounts.ts | 8 +- packages/jam/jam-host-calls/general/info.ts | 4 +- packages/jam/jam-host-calls/general/lookup.ts | 4 +- packages/jam/jam-host-calls/general/read.ts | 4 +- packages/jam/jam-host-calls/general/write.ts | 4 +- packages/jam/state/state.ts | 7 + .../transition/accumulate/accumulate.test.ts | 2 +- .../jam/transition/accumulate/accumulate.ts | 82 +++ packages/jam/transition/accumulate/options.ts | 1 + .../jam/transition/accumulate/worker/pool.ts | 122 +++++ .../transition/accumulate/worker/protocol.ts | 57 ++ .../accumulate/worker/serialization.ts | 512 ++++++++++++++++++ .../transition/accumulate/worker/worker.ts | 250 +++++++++ .../accumulate-externalities.test.ts | 242 ++++----- .../externalities/accumulate-externalities.ts | 105 ++-- packages/workers/importer/importer.ts | 8 +- 31 files changed, 1406 insertions(+), 209 deletions(-) create mode 100644 packages/jam/transition/accumulate/worker/pool.ts create mode 100644 packages/jam/transition/accumulate/worker/protocol.ts create mode 100644 packages/jam/transition/accumulate/worker/serialization.ts create mode 100644 packages/jam/transition/accumulate/worker/worker.ts diff --git a/bin/test-runner/common.test.ts b/bin/test-runner/common.test.ts index 2bd64f5da..dfbba3081 100644 --- a/bin/test-runner/common.test.ts +++ b/bin/test-runner/common.test.ts @@ -13,6 +13,7 @@ describe("test runner common", () => { initialFiles: ["file1.json", "file2.json"], pvms: [SelectedPvm.Ananas], accumulateSequentially: false, + accumulateWorkers: 1, }); }); @@ -25,6 +26,7 @@ describe("test runner common", () => { initialFiles: ["file1.json", "file2.json"], pvms: [SelectedPvm.Ananas, SelectedPvm.Builtin, SelectedPvm.Lite], accumulateSequentially: false, + accumulateWorkers: 1, }); }); @@ -50,6 +52,7 @@ describe("test runner common", () => { initialFiles: ["file1.json"], pvms: [SelectedPvm.Ananas, SelectedPvm.Builtin, SelectedPvm.Lite], accumulateSequentially: true, + accumulateWorkers: 1, }); }); @@ -62,6 +65,7 @@ describe("test runner common", () => { initialFiles: ["file1.json"], pvms: [SelectedPvm.Ananas, SelectedPvm.Builtin, SelectedPvm.Lite], accumulateSequentially: true, + accumulateWorkers: 1, }); }); @@ -74,6 +78,7 @@ describe("test runner common", () => { initialFiles: ["file1.json"], pvms: [SelectedPvm.Ananas, SelectedPvm.Builtin, SelectedPvm.Lite], accumulateSequentially: false, + accumulateWorkers: 1, }); }); diff --git a/bin/test-runner/common.ts b/bin/test-runner/common.ts index 866470f24..cd8a73d2c 100644 --- a/bin/test-runner/common.ts +++ b/bin/test-runner/common.ts @@ -38,6 +38,7 @@ export function selectedPvmToBackend(pvm: SelectedPvm): PvmBackend { export type GlobalsOptions = { pvms: SelectedPvm[]; accumulateSequentially: boolean; + accumulateWorkers: number; }; export class RunnerBuilder implements Runner { @@ -100,6 +101,7 @@ export type RunOptions = { chainSpec: ChainSpec; path: string; accumulateSequentially: boolean; + accumulateWorkers: number; }; export type RunFunction = (test: T, options: RunOptions, variant: V) => Promise; @@ -141,6 +143,7 @@ export namespace testFile { const PVM_OPTION = "pvm"; const ACCUMULATE_SEQUENTIALLY_OPTION = "accumulate-sequentially"; +const ACCUMULATE_WORKERS_OPTION = "accumulate-workers"; const HELP_OPTION = "help"; export const HELP_MESSAGE = ` Usage: test-runner [options] [files...] @@ -153,6 +156,10 @@ Options: --accumulate-sequentially Run accumulation sequentially instead of in parallel. Default: false + --accumulate-workers Number of worker threads for accumulation. + 0 = no workers (current behavior), 1+ = worker threads. + Default: 0 + -h, --help Show this help message. Examples: @@ -166,7 +173,7 @@ export function parseArgs(argv: string[]) { const parsed = minimist(argv, { boolean: [ACCUMULATE_SEQUENTIALLY_OPTION, HELP_OPTION], alias: { h: HELP_OPTION }, - default: { [ACCUMULATE_SEQUENTIALLY_OPTION]: false }, + default: { [ACCUMULATE_SEQUENTIALLY_OPTION]: false, [ACCUMULATE_WORKERS_OPTION]: 1 }, }); const shouldShowHelp = getBooleanOption(parsed[HELP_OPTION]); @@ -178,11 +185,13 @@ export function parseArgs(argv: string[]) { const pvms = getPvms(parsed[PVM_OPTION]); const accumulateSequentially = getBooleanOption(parsed[ACCUMULATE_SEQUENTIALLY_OPTION]); + const accumulateWorkers = Number(parsed[ACCUMULATE_WORKERS_OPTION]) || 0; return { initialFiles: parsed._, pvms, accumulateSequentially, + accumulateWorkers, }; function getBooleanOption(value: unknown): boolean { @@ -221,6 +230,7 @@ export async function main( initialFiles, pvms, accumulateSequentially, + accumulateWorkers, patterns = [testFile.bin, testFile.json], accepted, ignored, @@ -228,6 +238,7 @@ export async function main( initialFiles: string[]; pvms: SelectedPvm[]; accumulateSequentially: boolean; + accumulateWorkers: number; patterns?: (testFile.bin | testFile.json)[]; accepted?: { [testFile.bin]?: string[]; @@ -287,6 +298,7 @@ export async function main( const testVariants = prepareTest(runners, testFileContent, testFilePath, absolutePath, { pvms, accumulateSequentially, + accumulateWorkers, }); for (const test of testVariants) { test.shouldSkip = !isAccepted; @@ -467,6 +479,7 @@ function prepareTest( path: fullPath, chainSpec, accumulateSequentially: globalOptions.accumulateSequentially, + accumulateWorkers: globalOptions.accumulateWorkers, }, variant, ); diff --git a/bin/test-runner/state-transition/state-transition.ts b/bin/test-runner/state-transition/state-transition.ts index aaf9ec050..93b602171 100644 --- a/bin/test-runner/state-transition/state-transition.ts +++ b/bin/test-runner/state-transition/state-transition.ts @@ -86,7 +86,7 @@ export async function runStateTransition(testContent: StateTransition, options: spec, preState, hasher, - { pvm, accumulateSequentially: options.accumulateSequentially }, + { pvm, accumulateSequentially: options.accumulateSequentially, accumulateWorkers: options.accumulateWorkers }, DbHeaderChain.new(blocksDb), ); diff --git a/bin/test-runner/w3f/accumulate.ts b/bin/test-runner/w3f/accumulate.ts index 032c20240..6018b8fbf 100644 --- a/bin/test-runner/w3f/accumulate.ts +++ b/bin/test-runner/w3f/accumulate.ts @@ -141,11 +141,11 @@ export class AccumulateTest { export async function runAccumulateTest( test: AccumulateTest, - { chainSpec, accumulateSequentially }: RunOptions, + { chainSpec, accumulateSequentially, accumulateWorkers }: RunOptions, variant: SelectedPvm, ) { const pvm = selectedPvmToBackend(variant); - const options = { pvm, accumulateSequentially }; + const options = { pvm, accumulateSequentially, accumulateWorkers }; /** * entropy has to be moved to input because state is incompatibile - * in test state we have: `entropy: EntropyHash;` diff --git a/packages/jam/jam-host-calls/accumulate/eject.ts b/packages/jam/jam-host-calls/accumulate/eject.ts index 6debf17db..146486aeb 100644 --- a/packages/jam/jam-host-calls/accumulate/eject.ts +++ b/packages/jam/jam-host-calls/accumulate/eject.ts @@ -49,7 +49,7 @@ export class Eject implements HostCallHandler { return; } - const result = this.partialState.eject(serviceId, previousCodeHash); + const result = await this.partialState.eject(serviceId, previousCodeHash); // All good! if (result.isOk) { diff --git a/packages/jam/jam-host-calls/accumulate/forget.ts b/packages/jam/jam-host-calls/accumulate/forget.ts index 096493eec..ff5c43648 100644 --- a/packages/jam/jam-host-calls/accumulate/forget.ts +++ b/packages/jam/jam-host-calls/accumulate/forget.ts @@ -40,7 +40,7 @@ export class Forget implements HostCallHandler { return PvmExecution.Panic; } - const result = this.partialState.forgetPreimage(hash.asOpaque(), length); + const result = await this.partialState.forgetPreimage(hash.asOpaque(), length); logger.trace`[${this.currentServiceId}] FORGET(${hash}, ${length}) <- ${resultToString(result)}`; if (result.isOk) { diff --git a/packages/jam/jam-host-calls/accumulate/new.ts b/packages/jam/jam-host-calls/accumulate/new.ts index 75d14994e..46ef183f0 100644 --- a/packages/jam/jam-host-calls/accumulate/new.ts +++ b/packages/jam/jam-host-calls/accumulate/new.ts @@ -50,7 +50,7 @@ export class New implements HostCallHandler { return PvmExecution.Panic; } - const assignedId = this.partialState.newService( + const assignedId = await this.partialState.newService( codeHash.asOpaque(), codeLength, gas, diff --git a/packages/jam/jam-host-calls/accumulate/provide.ts b/packages/jam/jam-host-calls/accumulate/provide.ts index 7ca5b0487..a292472fe 100644 --- a/packages/jam/jam-host-calls/accumulate/provide.ts +++ b/packages/jam/jam-host-calls/accumulate/provide.ts @@ -51,7 +51,7 @@ export class Provide implements HostCallHandler { return PvmExecution.Panic; } - const result = this.partialState.providePreimage(serviceId, preimage); + const result = await this.partialState.providePreimage(serviceId, preimage); logger.trace`[${this.currentServiceId}] PROVIDE(${serviceId}, ${preimage.toStringTruncated()}) <- ${resultToString(result)}`; logger.insane`[${this.currentServiceId}] PROVIDE(${serviceId}, ${preimage}) <- ${resultToString(result)}`; diff --git a/packages/jam/jam-host-calls/accumulate/query.ts b/packages/jam/jam-host-calls/accumulate/query.ts index f2c06e921..48ec2fbc4 100644 --- a/packages/jam/jam-host-calls/accumulate/query.ts +++ b/packages/jam/jam-host-calls/accumulate/query.ts @@ -50,7 +50,7 @@ export class Query implements HostCallHandler { return PvmExecution.Panic; } - const result = this.partialState.checkPreimageStatus(hash.asOpaque(), length); + const result = await this.partialState.checkPreimageStatus(hash.asOpaque(), length); const zero = tryAsU64(0n); if (result === null) { diff --git a/packages/jam/jam-host-calls/accumulate/solicit.ts b/packages/jam/jam-host-calls/accumulate/solicit.ts index 42b6135f7..f19b052e0 100644 --- a/packages/jam/jam-host-calls/accumulate/solicit.ts +++ b/packages/jam/jam-host-calls/accumulate/solicit.ts @@ -39,7 +39,7 @@ export class Solicit implements HostCallHandler { return PvmExecution.Panic; } - const result = this.partialState.requestPreimage(hash.asOpaque(), length); + const result = await this.partialState.requestPreimage(hash.asOpaque(), length); logger.trace`[${this.currentServiceId}] SOLICIT(${hash}, ${length}) <- ${resultToString(result)}`; if (result.isOk) { diff --git a/packages/jam/jam-host-calls/accumulate/transfer.ts b/packages/jam/jam-host-calls/accumulate/transfer.ts index 39d970e8b..c08dad28b 100644 --- a/packages/jam/jam-host-calls/accumulate/transfer.ts +++ b/packages/jam/jam-host-calls/accumulate/transfer.ts @@ -61,7 +61,7 @@ export class Transfer implements HostCallHandler { return PvmExecution.Panic; } - const transferResult = this.partialState.transfer(destination, amount, transferGasFee, memo); + const transferResult = await this.partialState.transfer(destination, amount, transferGasFee, memo); logger.trace`[${this.currentServiceId}] TRANSFER(${destination}, ${amount}, ${transferGasFee}, ${memo}) <- ${resultToString(transferResult)}`; // All good! diff --git a/packages/jam/jam-host-calls/accumulate/upgrade.ts b/packages/jam/jam-host-calls/accumulate/upgrade.ts index db04cb54a..a1b562981 100644 --- a/packages/jam/jam-host-calls/accumulate/upgrade.ts +++ b/packages/jam/jam-host-calls/accumulate/upgrade.ts @@ -43,7 +43,7 @@ export class Upgrade implements HostCallHandler { return PvmExecution.Panic; } - this.partialState.upgradeService(codeHash.asOpaque(), gas, allowance); + await this.partialState.upgradeService(codeHash.asOpaque(), gas, allowance); logger.trace`[${this.currentServiceId}] UPGRADE(${codeHash}, ${gas}, ${allowance})`; regs.set(IN_OUT_REG, HostCallResult.OK); diff --git a/packages/jam/jam-host-calls/externalities/partial-state-mock.ts b/packages/jam/jam-host-calls/externalities/partial-state-mock.ts index aa95fe6c3..84340fe81 100644 --- a/packages/jam/jam-host-calls/externalities/partial-state-mock.ts +++ b/packages/jam/jam-host-calls/externalities/partial-state-mock.ts @@ -53,32 +53,32 @@ export class PartialStateMock implements PartialState { public validatorDataResponse: Result = Result.ok(OK); public providePreimageResponse: Result = Result.ok(OK); - eject(from: ServiceId | null, previousCode: PreimageHash): Result { + async eject(from: ServiceId | null, previousCode: PreimageHash): Promise> { this.ejectData.push([from, previousCode]); return this.ejectReturnValue; } - checkPreimageStatus(hash: Blake2bHash, length: U64): PreimageStatus | null { + async checkPreimageStatus(hash: Blake2bHash, length: U64): Promise { this.checkPreimageStatusData.push([hash, length]); return this.checkPreimageStatusResponse; } - requestPreimage(hash: Blake2bHash, length: U64): Result { + async requestPreimage(hash: Blake2bHash, length: U64): Promise> { this.requestPreimageData.push([hash, length]); return this.requestPreimageResponse; } - forgetPreimage(hash: Blake2bHash, length: U64): Result { + async forgetPreimage(hash: Blake2bHash, length: U64): Promise> { this.forgetPreimageData.push([hash, length]); return this.forgetPreimageResponse; } - transfer( + async transfer( destination: ServiceId | null, amount: U64, suppliedGas: ServiceGas, memo: Bytes, - ): Result { + ): Promise> { if (destination === null) { return Result.error(TransferError.DestinationNotFound, () => "Mock: destination is null for transfer"); } @@ -90,19 +90,19 @@ export class PartialStateMock implements PartialState { return this.transferReturnValue; } - newService( + async newService( codeHash: CodeHash, codeLength: U64, gas: ServiceGas, balance: ServiceGas, gratisStorage: U64, serviceId: U64, - ): Result { + ): Promise> { this.newServiceCalled.push([codeHash, codeLength, gas, balance, gratisStorage, serviceId]); return this.newServiceResponse; } - upgradeService(codeHash: CodeHash, gas: U64, allowance: U64): void { + async upgradeService(codeHash: CodeHash, gas: U64, allowance: U64): Promise { this.upgradeData.push([codeHash, gas, allowance]); } @@ -145,7 +145,7 @@ export class PartialStateMock implements PartialState { this.yieldHash = hash; } - providePreimage(service: ServiceId | null, preimage: BytesBlob): Result { + async providePreimage(service: ServiceId | null, preimage: BytesBlob): Promise> { if (service === null) { return Result.error(ProvidePreimageError.ServiceNotFound, () => "Mock: service is null for providePreimage"); } diff --git a/packages/jam/jam-host-calls/externalities/partial-state.ts b/packages/jam/jam-host-calls/externalities/partial-state.ts index bc39919a0..bf24a891d 100644 --- a/packages/jam/jam-host-calls/externalities/partial-state.ts +++ b/packages/jam/jam-host-calls/externalities/partial-state.ts @@ -174,7 +174,7 @@ export interface PartialState { * States: * https://graypaper.fluffylabs.dev/#/579bd12/116f00116f00 */ - checkPreimageStatus(hash: PreimageHash, length: U64): PreimageStatus | null; + checkPreimageStatus(hash: PreimageHash, length: U64): Promise; /** * Request (solicit) a preimage to be (re-)available. @@ -182,21 +182,21 @@ export interface PartialState { * States: * https://graypaper.fluffylabs.dev/#/579bd12/116f00116f00 */ - requestPreimage(hash: PreimageHash, length: U64): Result; + requestPreimage(hash: PreimageHash, length: U64): Promise>; /** * Mark a preimage hash as unavailable (forget it). * * https://graypaper.fluffylabs.dev/#/579bd12/335602335602 */ - forgetPreimage(hash: PreimageHash, length: U64): Result; + forgetPreimage(hash: PreimageHash, length: U64): Promise>; /** * Remove the provided source account and transfer the remaining account balance to current service. * * https://graypaper.fluffylabs.dev/#/9a08063/37b60137b601?v=0.6.6 */ - eject(from: ServiceId | null, previousCode: PreimageHash): Result; + eject(from: ServiceId | null, previousCode: PreimageHash): Promise>; /** * Transfer given `amount` of funds to the `destination`, @@ -207,7 +207,7 @@ export interface PartialState { amount: U64, gas: ServiceGas, memo: Bytes, - ): Result; + ): Promise>; /** * Create a new service with given codeHash, length, gas, allowance, gratisStorage and wantedServiceId. @@ -230,10 +230,10 @@ export interface PartialState { allowance: ServiceGas, gratisStorage: U64, wantedServiceId: U64, - ): Result; + ): Promise>; /** Upgrade code of currently running service. */ - upgradeService(codeHash: CodeHash, gas: U64, allowance: U64): void; + upgradeService(codeHash: CodeHash, gas: U64, allowance: U64): Promise; /** Designate new validators given their key and meta data. */ updateValidatorsData(validatorsData: PerValidator): Result; @@ -276,5 +276,5 @@ export interface PartialState { yield(hash: OpaqueHash): void; /** Provide a preimage for given service. */ - providePreimage(service: ServiceId | null, preimage: BytesBlob): Result; + providePreimage(service: ServiceId | null, preimage: BytesBlob): Promise>; } diff --git a/packages/jam/jam-host-calls/externalities/state-update.ts b/packages/jam/jam-host-calls/externalities/state-update.ts index b66170815..8343a3c60 100644 --- a/packages/jam/jam-host-calls/externalities/state-update.ts +++ b/packages/jam/jam-host-calls/externalities/state-update.ts @@ -9,6 +9,7 @@ import { type AUTHORIZATION_QUEUE_SIZE, LookupHistoryItem, PrivilegedServices, + type Service, ServiceAccountInfo, type ServicesUpdate, type State, @@ -137,7 +138,9 @@ export class AccumulationStateUpdate { } } -type StateSlice = Pick; +type StateSlice = Pick & { + getServiceAsync?(id: ServiceId): Promise; +}; export class PartiallyUpdatedState { /** A collection of state updates. */ @@ -152,6 +155,14 @@ export class PartiallyUpdatedState { stateUpdate === undefined ? AccumulationStateUpdate.empty() : AccumulationStateUpdate.copyFrom(stateUpdate); } + /** Resolve a service from base state, using async variant if available. */ + private getServiceFromBase(id: ServiceId): Promise { + if (this.state.getServiceAsync !== undefined) { + return this.state.getServiceAsync(id); + } + return Promise.resolve(this.state.getService(id)); + } + /** * Retrieve info of service with given id. * @@ -295,6 +306,118 @@ export class PartiallyUpdatedState { assertNever(action); } + // ── Async variants (for worker threads) ────────────────────────────────── + + async getServiceInfoAsync(destination: ServiceId | null): Promise { + if (destination === null) { + return null; + } + + const maybeUpdatedServiceInfo = this.stateUpdate.services.updated.get(destination); + if (maybeUpdatedServiceInfo !== undefined) { + return maybeUpdatedServiceInfo.action.account; + } + + const maybeService = await this.getServiceFromBase(destination); + if (maybeService === null) { + return null; + } + + return maybeService.getInfo(); + } + + async getStorageAsync(serviceId: ServiceId, rawKey: StorageKey): Promise { + const storages = this.stateUpdate.services.storage.get(serviceId) ?? []; + const item = storages.find((x) => x.key.isEqualTo(rawKey)); + if (item !== undefined) { + return item.value; + } + + const service = await this.getServiceFromBase(serviceId); + return service?.getStorage(rawKey) ?? null; + } + + async hasPreimageAsync(serviceId: ServiceId, hash: PreimageHash): Promise { + const preimages = this.stateUpdate.services.preimages.get(serviceId) ?? []; + const providedPreimage = preimages.find((p) => p.hash.isEqualTo(hash)); + if (providedPreimage !== undefined) { + return true; + } + + const service = await this.getServiceFromBase(serviceId); + if (service === null) { + return false; + } + + return service.hasPreimage(hash); + } + + async getPreimageAsync(serviceId: ServiceId, hash: PreimageHash): Promise { + const preimages = this.stateUpdate.services.preimages.get(serviceId) ?? []; + const freshlyProvided = preimages.find((x) => x.hash.isEqualTo(hash)); + if (freshlyProvided !== undefined && freshlyProvided.action.kind === UpdatePreimageKind.Provide) { + return freshlyProvided.action.preimage.blob; + } + + const service = await this.getServiceFromBase(serviceId); + return service?.getPreimage(hash) ?? null; + } + + async getLookupHistoryAsync( + currentTimeslot: TimeSlot, + serviceId: ServiceId, + hash: PreimageHash, + length: U64, + ): Promise { + const updatedService = this.stateUpdate.services.updated.get(serviceId); + + if (updatedService !== undefined && updatedService.action.kind === UpdateServiceKind.Create) { + const lookupHistoryItem = updatedService.action.lookupHistory; + + if ( + lookupHistoryItem !== null && + hash.isEqualTo(lookupHistoryItem.hash) && + length === BigInt(lookupHistoryItem.length) + ) { + return lookupHistoryItem; + } + } + + const preimages = this.stateUpdate.services.preimages.get(serviceId) ?? []; + const updatedPreimage = preimages.findLast( + (update) => update.hash.isEqualTo(hash) && BigInt(update.length) === length, + ); + + const stateFallback = async () => { + const service = await this.getServiceFromBase(serviceId); + const lenU32 = preimageLenAsU32(length); + if (lenU32 === null || service === null) { + return null; + } + const slots = service.getLookupHistory(hash, lenU32); + return slots === null ? null : new LookupHistoryItem(hash, lenU32, slots); + }; + + if (updatedPreimage === undefined) { + return stateFallback(); + } + + const { action } = updatedPreimage; + switch (action.kind) { + case UpdatePreimageKind.Provide: { + return new LookupHistoryItem(hash, updatedPreimage.length, tryAsLookupHistorySlots([currentTimeslot])); + } + case UpdatePreimageKind.Remove: { + return null; + } + case UpdatePreimageKind.UpdateOrAdd: { + return action.item; + } + } + + assertNever(action); + } + /* State update functions. */ updateStorage(serviceId: ServiceId, key: StorageKey, value: BytesBlob | null) { const update = diff --git a/packages/jam/jam-host-calls/externalities/test-accounts.ts b/packages/jam/jam-host-calls/externalities/test-accounts.ts index 4722d4f52..927960d45 100644 --- a/packages/jam/jam-host-calls/externalities/test-accounts.ts +++ b/packages/jam/jam-host-calls/externalities/test-accounts.ts @@ -21,7 +21,7 @@ export class TestAccounts implements AccountsLookup, AccountsRead, AccountsWrite ]); public readonly details = new Map(); - lookup(serviceId: ServiceId | null, hash: Blake2bHash): BytesBlob | null { + async lookup(serviceId: ServiceId | null, hash: Blake2bHash): Promise { if (serviceId === null) { return null; } @@ -32,7 +32,7 @@ export class TestAccounts implements AccountsLookup, AccountsRead, AccountsWrite return preImage; } - read(serviceId: ServiceId | null, hash: StorageKey): BytesBlob | null { + async read(serviceId: ServiceId | null, hash: StorageKey): Promise { if (serviceId === null) { return null; } @@ -43,7 +43,7 @@ export class TestAccounts implements AccountsLookup, AccountsRead, AccountsWrite return d; } - write(hash: StorageKey, data: BytesBlob | null): Result { + async write(hash: StorageKey, data: BytesBlob | null): Promise> { if (this.isStorageFull()) { return Result.error("full", () => "Test accounts: storage is full"); } @@ -72,7 +72,7 @@ export class TestAccounts implements AccountsLookup, AccountsRead, AccountsWrite ); } - getServiceInfo(serviceId: ServiceId | null): ServiceAccountInfo | null { + async getServiceInfo(serviceId: ServiceId | null): Promise { if (serviceId === null) { return null; } diff --git a/packages/jam/jam-host-calls/general/info.ts b/packages/jam/jam-host-calls/general/info.ts index 7ec27bcb7..d7a828cf7 100644 --- a/packages/jam/jam-host-calls/general/info.ts +++ b/packages/jam/jam-host-calls/general/info.ts @@ -15,7 +15,7 @@ import { HostCallResult } from "./results.js"; /** Account data interface for info host calls. */ export interface AccountsInfo { /** Get account info. */ - getServiceInfo(serviceId: ServiceId | null): ServiceAccountInfo | null; + getServiceInfo(serviceId: ServiceId | null): Promise; } const IN_OUT_REG = 7; @@ -60,7 +60,7 @@ export class Info implements HostCallHandler { const outputStart = regs.get(8); // v - const accountInfo = this.account.getServiceInfo(serviceId); + const accountInfo = await this.account.getServiceInfo(serviceId); const encodedInfo = accountInfo === null diff --git a/packages/jam/jam-host-calls/general/lookup.ts b/packages/jam/jam-host-calls/general/lookup.ts index c915c2758..3ff07ee5b 100644 --- a/packages/jam/jam-host-calls/general/lookup.ts +++ b/packages/jam/jam-host-calls/general/lookup.ts @@ -13,7 +13,7 @@ import { HostCallResult } from "./results.js"; /** Account data interface for lookup host calls. */ export interface AccountsLookup { /** Lookup a preimage. */ - lookup(serviceId: ServiceId | null, hash: Blake2bHash): BytesBlob | null; + lookup(serviceId: ServiceId | null, hash: Blake2bHash): Promise; } const IN_OUT_REG = 7; @@ -51,7 +51,7 @@ export class Lookup implements HostCallHandler { } // v - const preImage = this.account.lookup(serviceId, preImageHash); + const preImage = await this.account.lookup(serviceId, preImageHash); logger.trace`[${this.currentServiceId}] LOOKUP(${serviceId}, ${preImageHash}) <- ${preImage?.toStringTruncated() ?? ""}...`; logger.insane`[${this.currentServiceId}] LOOKUP(${serviceId}, ${preImageHash}) <- ${preImage ?? ""}`; diff --git a/packages/jam/jam-host-calls/general/read.ts b/packages/jam/jam-host-calls/general/read.ts index aebb7c39a..80a31a2e3 100644 --- a/packages/jam/jam-host-calls/general/read.ts +++ b/packages/jam/jam-host-calls/general/read.ts @@ -12,7 +12,7 @@ import { HostCallResult } from "./results.js"; /** Account data interface for read host calls. */ export interface AccountsRead { /** Read service storage. */ - read(serviceId: ServiceId | null, rawKey: BytesBlob): BytesBlob | null; + read(serviceId: ServiceId | null, rawKey: BytesBlob): Promise; } const IN_OUT_REG = 7; @@ -54,7 +54,7 @@ export class Read implements HostCallHandler { } // v - const value = this.account.read(serviceId, rawKey); + const value = await this.account.read(serviceId, rawKey); const valueLength = value === null ? tryAsU64(0) : tryAsU64(value.raw.length); const valueBlobOffset = regs.get(11); diff --git a/packages/jam/jam-host-calls/general/write.ts b/packages/jam/jam-host-calls/general/write.ts index 5fe80b2ab..811de6f54 100644 --- a/packages/jam/jam-host-calls/general/write.ts +++ b/packages/jam/jam-host-calls/general/write.ts @@ -21,7 +21,7 @@ export interface AccountsWrite { * * https://graypaper.fluffylabs.dev/#/9a08063/331002331402?v=0.6.6 */ - write(rawKey: BytesBlob, data: BytesBlob | null): Result; + write(rawKey: BytesBlob, data: BytesBlob | null): Promise>; } const IN_OUT_REG = 7; @@ -76,7 +76,7 @@ export class Write implements HostCallHandler { const maybeValue = valueLength === 0n ? null : BytesBlob.blobFrom(value); // a - const result = this.account.write(storageKey, maybeValue); + const result = await this.account.write(storageKey, maybeValue); logger.trace`[${this.currentServiceId}] WRITE(${storageKey}, ${maybeValue?.toStringTruncated() ?? "remove"}) <- ${resultToString(result)}`; logger.insane`[${this.currentServiceId}] WRITE(${storageKey}, ${maybeValue ?? "remove"}) <- ${resultToString(result)}`; diff --git a/packages/jam/state/state.ts b/packages/jam/state/state.ts index 09989497f..b935bb10f 100644 --- a/packages/jam/state/state.ts +++ b/packages/jam/state/state.ts @@ -207,6 +207,13 @@ export type State = { * Retrieve details about single service. */ getService(id: ServiceId): Service | null; + + /** + * Async variant of getService. Used by worker threads that fetch + * service data from the main thread asynchronously. + * When not provided, callers fall back to the sync getService. + */ + getServiceAsync?(id: ServiceId): Promise; }; /** Service details. */ diff --git a/packages/jam/transition/accumulate/accumulate.test.ts b/packages/jam/transition/accumulate/accumulate.test.ts index 2fdc99f39..26678f545 100644 --- a/packages/jam/transition/accumulate/accumulate.test.ts +++ b/packages/jam/transition/accumulate/accumulate.test.ts @@ -48,7 +48,7 @@ type TestServiceInfo = { [PvmBackend.BuiltIn, PvmBackend.Ananas, PvmBackend.Lite].forEach((pvm) => { [false, true].forEach((accumulateSequentially) => { - const options = { pvm, accumulateSequentially }; + const options = { pvm, accumulateSequentially, accumulateWorkers: 0 }; describe(`accumulate: ${PvmBackendNames[pvm]} (sequential accumulation: ${accumulateSequentially})`, () => { // based on tiny/enqueue_and_unlock_chain_wraps-5.json it("should do correct state transition", async () => { diff --git a/packages/jam/transition/accumulate/accumulate.ts b/packages/jam/transition/accumulate/accumulate.ts index 9914e40b4..4b17901a8 100644 --- a/packages/jam/transition/accumulate/accumulate.ts +++ b/packages/jam/transition/accumulate/accumulate.ts @@ -50,6 +50,15 @@ import { } from "./accumulation-result-merge-utils.js"; import type { Operand } from "./operand.js"; import type { AccumulateOptions } from "./options.js"; +import { AccumulateWorkerPool } from "./worker/pool.js"; +import { type AccumulateRequest, MessageType } from "./worker/protocol.js"; +import { + deserializeAccumulationStateUpdate, + serializeAccumulationStateUpdate, + serializeOperand, + serializePendingTransfer, + serializePrivilegedServices, +} from "./worker/serialization.js"; export const ACCUMULATION_ERROR = "duplicate service created"; export type ACCUMULATION_ERROR = typeof ACCUMULATION_ERROR; @@ -80,6 +89,8 @@ const ARGS_CODEC = codec.object({ }); export class Accumulate { + private workerPool: AccumulateWorkerPool | null = null; + constructor( public readonly chainSpec: ChainSpec, public readonly blake2b: Blake2b, @@ -89,6 +100,11 @@ export class Accumulate { if (options.accumulateSequentially === true) { logger.warn`⚠️ Parallel accumulation is disabled. Running in sequential mode.`; } + if (options.accumulateWorkers > 0) { + this.workerPool = new AccumulateWorkerPool(options.accumulateWorkers); + this.workerPool.setGetServiceFn((serviceId) => this.state.getService(serviceId)); + logger.log`Accumulate worker pool created with ${options.accumulateWorkers} workers.`; + } } /** @@ -427,6 +443,72 @@ export class Accumulate { slot: TimeSlot, entropy: EntropyHash, inputStateUpdate: AccumulationStateUpdate, + ): Promise { + if (this.workerPool !== null) { + return this.accumulateInParallelViaWorkers(this.workerPool, accumulateData, slot, entropy, inputStateUpdate); + } + return this.accumulateInParallelInProcess(accumulateData, slot, entropy, inputStateUpdate); + } + + private async accumulateInParallelViaWorkers( + workerPool: AccumulateWorkerPool, + accumulateData: AccumulateData, + slot: TimeSlot, + entropy: EntropyHash, + inputStateUpdate: AccumulationStateUpdate, + ): Promise { + const serviceIds = accumulateData.getServiceIds(); + const serviceIdsLength = serviceIds.length; + const serializedInputState = serializeAccumulationStateUpdate(inputStateUpdate); + + const resultPromises: Promise< + readonly [ServiceId, { consumedGas: ServiceGas; stateUpdate: AccumulationStateUpdate }] + >[] = new Array(serviceIdsLength); + + for (let serviceIndex = 0; serviceIndex < serviceIdsLength; serviceIndex += 1) { + const serviceId = serviceIds[serviceIndex]; + const checkpoint = AccumulationStateUpdate.copyFrom(inputStateUpdate); + + const request: AccumulateRequest = { + type: MessageType.AccumulateRequest, + serviceId, + transfers: accumulateData.getTransfers(serviceId).map(serializePendingTransfer), + operands: accumulateData.getOperands(serviceId).map(serializeOperand), + gasCost: accumulateData.getGasLimit(serviceId), + slot, + entropy: entropy.raw, + inputStateUpdate: serializedInputState, + privilegedServices: serializePrivilegedServices(this.state.privilegedServices), + chainSpec: this.chainSpec, + pvmBackend: this.options.pvm, + }; + + const promise = workerPool.dispatch(request).then((response) => { + if (response.error !== undefined) { + logger.warn`Worker error for service ${serviceId}: ${response.error}`; + } + const resultEntry: readonly [ServiceId, { consumedGas: ServiceGas; stateUpdate: AccumulationStateUpdate }] = [ + serviceId, + { + consumedGas: response.consumedGas, + stateUpdate: + response.stateUpdate !== null ? deserializeAccumulationStateUpdate(response.stateUpdate) : checkpoint, + }, + ]; + return resultEntry; + }); + + resultPromises[serviceIndex] = promise; + } + + return Promise.all(resultPromises).then((results) => new Map(results)); + } + + private async accumulateInParallelInProcess( + accumulateData: AccumulateData, + slot: TimeSlot, + entropy: EntropyHash, + inputStateUpdate: AccumulationStateUpdate, ): Promise { const serviceIds = accumulateData.getServiceIds(); const serviceIdsLength = serviceIds.length; diff --git a/packages/jam/transition/accumulate/options.ts b/packages/jam/transition/accumulate/options.ts index 45c3de19b..8b756c994 100644 --- a/packages/jam/transition/accumulate/options.ts +++ b/packages/jam/transition/accumulate/options.ts @@ -3,4 +3,5 @@ import type { PvmBackend } from "@typeberry/config"; export type AccumulateOptions = { pvm: PvmBackend; accumulateSequentially: boolean; + accumulateWorkers: number; }; diff --git a/packages/jam/transition/accumulate/worker/pool.ts b/packages/jam/transition/accumulate/worker/pool.ts new file mode 100644 index 000000000..a5149f456 --- /dev/null +++ b/packages/jam/transition/accumulate/worker/pool.ts @@ -0,0 +1,122 @@ +/** + * Worker pool for parallel PVM accumulation. + * + * Workers request service data from the main thread asynchronously + * via MessagePort request-response protocol. + */ +import { MessageChannel, type MessagePort, Worker } from "node:worker_threads"; +import type { ServiceId } from "@typeberry/block"; +import { Logger } from "@typeberry/logger"; +import type { Service } from "@typeberry/state"; +import { type AccumulateRequest, type AccumulateResponse, type GetServiceRequest, MessageType } from "./protocol.js"; +import { type PlainService, serializeService } from "./serialization.js"; + +const logger = Logger.new(import.meta.filename, "worker-pool"); + +type PendingAccumulation = { + resolve: (response: AccumulateResponse) => void; + reject: (error: Error) => void; +}; + +type WorkerHandle = { + worker: Worker; + /** Dedicated port for structured clone data (main side). */ + dataPort: MessagePort; + /** FIFO queue of pending accumulations for this worker. */ + pendingQueue: PendingAccumulation[]; +}; + +export class AccumulateWorkerPool { + private readonly workers: WorkerHandle[]; + private nextWorkerIndex = 0; + private getServiceFn: ((serviceId: ServiceId) => Service | null) | null = null; + private terminated = false; + + constructor(workerCount: number) { + const workerUrl = new URL("./worker.ts", import.meta.url); + + this.workers = new Array(workerCount); + for (let i = 0; i < workerCount; i++) { + const { port1: mainPort, port2: workerPort } = new MessageChannel(); + + const worker = new Worker(workerUrl, { + workerData: { dataPort: workerPort }, + transferList: [workerPort], + execArgv: ["--import", "tsx"], + }); + + mainPort.on("message", (msg: AccumulateResponse | GetServiceRequest) => { + this.handleWorkerMessage(i, msg); + }); + + worker.on("error", (err) => { + logger.warn`Worker ${i} error: ${err.message}`; + const handle = this.workers[i]; + const pending = handle.pendingQueue.shift(); + if (pending !== undefined) { + pending.reject(err); + } + }); + + this.workers[i] = { worker, dataPort: mainPort, pendingQueue: [] }; + } + + logger.log`Worker pool created with ${workerCount} workers.`; + } + + setGetServiceFn(fn: (serviceId: ServiceId) => Service | null) { + this.getServiceFn = fn; + } + + dispatch(request: AccumulateRequest): Promise { + const workerIndex = this.nextWorkerIndex; + this.nextWorkerIndex = (this.nextWorkerIndex + 1) % this.workers.length; + const handle = this.workers[workerIndex]; + + return new Promise((resolve, reject) => { + handle.pendingQueue.push({ resolve, reject }); + handle.dataPort.postMessage(request); + }); + } + + private handleWorkerMessage(workerIndex: number, msg: AccumulateResponse | GetServiceRequest) { + const handle = this.workers[workerIndex]; + + if ((msg as GetServiceRequest).type === MessageType.GetServiceRequest) { + this.handleGetServiceRequest(handle, msg as GetServiceRequest); + return; + } + + const response = msg as AccumulateResponse; + const pending = handle.pendingQueue.shift(); + if (pending !== undefined) { + pending.resolve(response); + } + } + + private handleGetServiceRequest(handle: WorkerHandle, request: GetServiceRequest) { + if (this.getServiceFn === null) { + throw new Error("getServiceFn not set on worker pool"); + } + + const service = this.getServiceFn(request.serviceId); + const plainService: PlainService | null = service !== null ? serializeService(service) : null; + + handle.dataPort.postMessage({ + type: MessageType.GetServiceResponse, + service: plainService, + }); + } + + async terminate() { + if (this.terminated) { + return; + } + this.terminated = true; + for (const handle of this.workers) { + handle.dataPort.close(); + await handle.worker.terminate(); + } + logger.log`Worker pool terminated.`; + } +} diff --git a/packages/jam/transition/accumulate/worker/protocol.ts b/packages/jam/transition/accumulate/worker/protocol.ts new file mode 100644 index 000000000..61c23cd03 --- /dev/null +++ b/packages/jam/transition/accumulate/worker/protocol.ts @@ -0,0 +1,57 @@ +/** + * Message protocol for main thread ↔ worker communication. + * + * All types here must be structured-clone-compatible (no class instances). + * We use raw Uint8Array instead of Bytes/BytesBlob, plain objects instead of class instances. + */ +import type { ServiceGas, ServiceId, TimeSlot } from "@typeberry/block"; +import type { ChainSpec, PvmBackend } from "@typeberry/config"; +import type { + PlainAccumulationStateUpdate, + PlainOperand, + PlainPendingTransfer, + PlainPrivilegedServices, + PlainService, +} from "./serialization.js"; + +export enum MessageType { + AccumulateRequest = 0, + AccumulateResponse = 1, + GetServiceRequest = 2, + GetServiceResponse = 3, +} + +/** Main → Worker: request to accumulate a single service. */ +export type AccumulateRequest = { + type: MessageType.AccumulateRequest; + serviceId: ServiceId; + transfers: PlainPendingTransfer[]; + operands: PlainOperand[]; + gasCost: ServiceGas; + slot: TimeSlot; + entropy: Uint8Array; + inputStateUpdate: PlainAccumulationStateUpdate; + privilegedServices: PlainPrivilegedServices; + chainSpec: ChainSpec; + pvmBackend: PvmBackend; +}; + +/** Worker → Main: result of accumulation. */ +export type AccumulateResponse = { + type: MessageType.AccumulateResponse; + consumedGas: ServiceGas; + stateUpdate: PlainAccumulationStateUpdate | null; + error?: string; +}; + +/** Worker → Main: synchronous request for service data from base state. */ +export type GetServiceRequest = { + type: MessageType.GetServiceRequest; + serviceId: ServiceId; +}; + +/** Main → Worker: response with serialized service data. */ +export type GetServiceResponse = { + type: MessageType.GetServiceResponse; + service: PlainService | null; +}; diff --git a/packages/jam/transition/accumulate/worker/serialization.ts b/packages/jam/transition/accumulate/worker/serialization.ts new file mode 100644 index 000000000..1197eda48 --- /dev/null +++ b/packages/jam/transition/accumulate/worker/serialization.ts @@ -0,0 +1,512 @@ +/** + * Serialization layer for worker communication. + * + * Converts class instances to/from plain objects that survive structured clone. + * All "Plain" types use only: primitives, bigint, Uint8Array, Map, arrays. + */ +import type { CodeHash, CoreIndex, PerValidator, ServiceGas, ServiceId, TimeSlot } from "@typeberry/block"; +import type { PreimageHash } from "@typeberry/block/preimage.js"; +import type { AuthorizerHash, ExportsRootHash, WorkPackageHash } from "@typeberry/block/refine-context.js"; +import { WorkExecResult, WorkExecResultKind } from "@typeberry/block/work-result.js"; +import { Bytes, BytesBlob } from "@typeberry/bytes"; +import type { FixedSizeArray } from "@typeberry/collections"; +import { asKnownSize } from "@typeberry/collections"; +import { BANDERSNATCH_KEY_BYTES, BLS_KEY_BYTES, ED25519_KEY_BYTES } from "@typeberry/crypto"; +import { HASH_SIZE, type OpaqueHash } from "@typeberry/hash"; +import { PendingTransfer, type TRANSFER_MEMO_BYTES } from "@typeberry/jam-host-calls"; +import { AccumulationStateUpdate } from "@typeberry/jam-host-calls/externalities/state-update.js"; +import type { U32, U64 } from "@typeberry/numbers"; +import { + type AUTHORIZATION_QUEUE_SIZE, + InMemoryService, + LookupHistoryItem, + PreimageItem, + PrivilegedServices, + type Service, + ServiceAccountInfo, + StorageItem, + type StorageKey, + tryAsLookupHistorySlots, + UpdatePreimage, + UpdatePreimageKind, + UpdateService, + UpdateServiceKind, + UpdateStorage, + UpdateStorageKind, + VALIDATOR_META_BYTES, + ValidatorData, +} from "@typeberry/state"; +import { Operand } from "../operand.js"; + +// ── Plain types (structured-clone-safe) ────────────────────────────────────── + +export type PlainServiceAccountInfo = { + codeHash: Uint8Array; + balance: U64; + accumulateMinGas: ServiceGas; + onTransferMinGas: ServiceGas; + storageUtilisationBytes: U64; + gratisStorage: U64; + storageUtilisationCount: U32; + created: TimeSlot; + lastAccumulation: TimeSlot; + parentService: ServiceId; +}; + +export type PlainLookupHistoryItem = { + hash: Uint8Array; + length: U32; + slots: TimeSlot[]; +}; + +export type PlainPreimageItem = { + hash: Uint8Array; + blob: Uint8Array; +}; + +export type PlainStorageItem = { + key: Uint8Array; + value: Uint8Array; +}; + +export type PlainUpdatePreimage = + | { kind: UpdatePreimageKind.Provide; preimage: PlainPreimageItem; slot: TimeSlot | null; providedFor: ServiceId } + | { kind: UpdatePreimageKind.Remove; hash: Uint8Array; length: U32 } + | { kind: UpdatePreimageKind.UpdateOrAdd; item: PlainLookupHistoryItem }; + +export type PlainUpdateService = + | { kind: UpdateServiceKind.Update; account: PlainServiceAccountInfo } + | { + kind: UpdateServiceKind.Create; + account: PlainServiceAccountInfo; + lookupHistory: PlainLookupHistoryItem | null; + }; + +export type PlainUpdateStorage = + | { kind: UpdateStorageKind.Set; storage: PlainStorageItem } + | { kind: UpdateStorageKind.Remove; key: Uint8Array }; + +export type PlainPrivilegedServices = { + manager: ServiceId; + delegator: ServiceId; + registrar: ServiceId; + assigners: ServiceId[]; + autoAccumulateServices: Map; +}; + +export type PlainAccumulationStateUpdate = { + created: ServiceId[]; + updated: Map; + removed: ServiceId[]; + preimages: Map; + storage: Map; + transfers: PlainPendingTransfer[]; + yieldedRoot: Uint8Array | null; + authorizationQueues: Map; + validatorsData: PlainValidatorData[] | null; + privilegedServices: PlainPrivilegedServices | null; +}; + +export type PlainPendingTransfer = { + source: ServiceId; + destination: ServiceId; + amount: U64; + memo: Uint8Array; + gas: ServiceGas; +}; + +export type PlainOperand = { + hash: Uint8Array; + exportsRoot: Uint8Array; + authorizerHash: Uint8Array; + payloadHash: Uint8Array; + gas: ServiceGas; + result: { kind: number; okBlob?: Uint8Array }; + authorizationOutput: Uint8Array; +}; + +export type PlainValidatorData = { + bandersnatch: Uint8Array; + ed25519: Uint8Array; + bls: Uint8Array; + metadata: Uint8Array; +}; + +export type PlainService = { + serviceId: ServiceId; + info: PlainServiceAccountInfo; + preimages: [Uint8Array, PlainPreimageItem][]; + lookupHistory: [Uint8Array, PlainLookupHistoryItem[]][]; + storage: [string, PlainStorageItem][]; +}; + +// ── Serializers ────────────────────────────────────────────────────────────── + +function toRaw(bytes: { raw: Uint8Array }): Uint8Array { + return bytes.raw; +} + +export function serializeServiceAccountInfo(info: ServiceAccountInfo): PlainServiceAccountInfo { + return { + codeHash: toRaw(info.codeHash), + balance: info.balance, + accumulateMinGas: info.accumulateMinGas, + onTransferMinGas: info.onTransferMinGas, + storageUtilisationBytes: info.storageUtilisationBytes, + gratisStorage: info.gratisStorage, + storageUtilisationCount: info.storageUtilisationCount, + created: info.created, + lastAccumulation: info.lastAccumulation, + parentService: info.parentService, + }; +} + +export function serializeLookupHistoryItem(item: LookupHistoryItem): PlainLookupHistoryItem { + return { hash: toRaw(item.hash), length: item.length, slots: [...item.slots] }; +} + +export function serializePendingTransfer(t: PendingTransfer): PlainPendingTransfer { + return { source: t.source, destination: t.destination, amount: t.amount, memo: toRaw(t.memo), gas: t.gas }; +} + +export function serializeOperand(o: Operand): PlainOperand { + return { + hash: toRaw(o.hash), + exportsRoot: toRaw(o.exportsRoot), + authorizerHash: toRaw(o.authorizerHash), + payloadHash: toRaw(o.payloadHash), + gas: o.gas, + result: + o.result.kind === WorkExecResultKind.ok + ? { kind: o.result.kind, okBlob: o.result.okBlob !== null ? toRaw(o.result.okBlob) : new Uint8Array() } + : { kind: o.result.kind }, + authorizationOutput: toRaw(o.authorizationOutput), + }; +} + +function serializeUpdatePreimage(up: UpdatePreimage): PlainUpdatePreimage { + switch (up.action.kind) { + case UpdatePreimageKind.Provide: + return { + kind: UpdatePreimageKind.Provide, + preimage: { hash: toRaw(up.action.preimage.hash), blob: toRaw(up.action.preimage.blob) }, + slot: up.action.slot, + providedFor: up.action.providedFor, + }; + case UpdatePreimageKind.Remove: + return { kind: UpdatePreimageKind.Remove, hash: toRaw(up.action.hash), length: up.action.length }; + case UpdatePreimageKind.UpdateOrAdd: + return { kind: UpdatePreimageKind.UpdateOrAdd, item: serializeLookupHistoryItem(up.action.item) }; + } +} + +function serializeUpdateService(us: UpdateService): PlainUpdateService { + if (us.action.kind === UpdateServiceKind.Create) { + return { + kind: UpdateServiceKind.Create, + account: serializeServiceAccountInfo(us.action.account), + lookupHistory: us.action.lookupHistory !== null ? serializeLookupHistoryItem(us.action.lookupHistory) : null, + }; + } + return { kind: UpdateServiceKind.Update, account: serializeServiceAccountInfo(us.action.account) }; +} + +function serializeUpdateStorage(us: UpdateStorage): PlainUpdateStorage { + if (us.action.kind === UpdateStorageKind.Set) { + return { + kind: UpdateStorageKind.Set, + storage: { key: toRaw(us.action.storage.key), value: toRaw(us.action.storage.value) }, + }; + } + return { kind: UpdateStorageKind.Remove, key: toRaw(us.action.key) }; +} + +export function serializePrivilegedServices(ps: PrivilegedServices): PlainPrivilegedServices { + return { + manager: ps.manager, + delegator: ps.delegator, + registrar: ps.registrar, + assigners: [...ps.assigners], + autoAccumulateServices: new Map(ps.autoAccumulateServices), + }; +} + +export function serializeAccumulationStateUpdate(su: AccumulationStateUpdate): PlainAccumulationStateUpdate { + const updated = new Map(); + for (const [id, us] of su.services.updated) { + updated.set(id, serializeUpdateService(us)); + } + + const preimages = new Map(); + for (const [id, ups] of su.services.preimages) { + preimages.set(id, ups.map(serializeUpdatePreimage)); + } + + const storage = new Map(); + for (const [id, uss] of su.services.storage) { + storage.set(id, uss.map(serializeUpdateStorage)); + } + + const authorizationQueues = new Map(); + for (const [core, queue] of su.authorizationQueues) { + authorizationQueues.set( + core, + [...queue].map((h) => toRaw(h)), + ); + } + + return { + created: [...su.services.created], + updated, + removed: [...su.services.removed], + preimages, + storage, + transfers: su.transfers.map(serializePendingTransfer), + yieldedRoot: su.yieldedRoot !== null ? toRaw(su.yieldedRoot) : null, + authorizationQueues, + validatorsData: + su.validatorsData !== null + ? [...su.validatorsData].map((vd) => ({ + bandersnatch: toRaw(vd.bandersnatch), + ed25519: toRaw(vd.ed25519), + bls: toRaw(vd.bls), + metadata: toRaw(vd.metadata), + })) + : null, + privilegedServices: su.privilegedServices !== null ? serializePrivilegedServices(su.privilegedServices) : null, + }; +} + +/** + * Serialize a full InMemoryService including all its data. + * Falls back to info-only for non-InMemoryService implementations. + */ +export function serializeService(service: Service): PlainService { + const info = serializeServiceAccountInfo(service.getInfo()); + + if (service instanceof InMemoryService) { + const preimages: [Uint8Array, PlainPreimageItem][] = []; + for (const [hash, item] of service.data.preimages.entries()) { + preimages.push([toRaw(hash), { hash: toRaw(item.hash), blob: toRaw(item.blob) }]); + } + + const lookupHistory: [Uint8Array, PlainLookupHistoryItem[]][] = []; + for (const [hash, items] of service.data.lookupHistory.entries()) { + lookupHistory.push([toRaw(hash), items.map(serializeLookupHistoryItem)]); + } + + const storage: [string, PlainStorageItem][] = []; + for (const [key, item] of service.data.storage) { + storage.push([key, { key: toRaw(item.key), value: toRaw(item.value) }]); + } + + return { serviceId: service.serviceId, info, preimages, lookupHistory, storage }; + } + + return { serviceId: service.serviceId, info, preimages: [], lookupHistory: [], storage: [] }; +} + +// ── Deserializers ──────────────────────────────────────────────────────────── + +function bytesFromRaw(raw: Uint8Array, len: T) { + return Bytes.fromBlob(raw, len); +} + +function hashFromRaw(raw: Uint8Array) { + return bytesFromRaw(raw, HASH_SIZE); +} + +export function deserializeServiceAccountInfo(p: PlainServiceAccountInfo): ServiceAccountInfo { + return ServiceAccountInfo.create({ + codeHash: hashFromRaw(p.codeHash).asOpaque(), + balance: p.balance, + accumulateMinGas: p.accumulateMinGas, + onTransferMinGas: p.onTransferMinGas, + storageUtilisationBytes: p.storageUtilisationBytes, + gratisStorage: p.gratisStorage, + storageUtilisationCount: p.storageUtilisationCount, + created: p.created, + lastAccumulation: p.lastAccumulation, + parentService: p.parentService, + }); +} + +function deserializeLookupHistoryItem(p: PlainLookupHistoryItem): LookupHistoryItem { + return new LookupHistoryItem( + hashFromRaw(p.hash).asOpaque(), + p.length, + tryAsLookupHistorySlots(p.slots), + ); +} + +export function deserializePendingTransfer(p: PlainPendingTransfer): PendingTransfer { + return PendingTransfer.create({ + source: p.source, + destination: p.destination, + amount: p.amount, + memo: bytesFromRaw(p.memo, 32 as TRANSFER_MEMO_BYTES), + gas: p.gas, + }); +} + +export function deserializeOperand(p: PlainOperand): Operand { + return Operand.new({ + hash: hashFromRaw(p.hash).asOpaque(), + exportsRoot: hashFromRaw(p.exportsRoot).asOpaque(), + authorizerHash: hashFromRaw(p.authorizerHash).asOpaque(), + payloadHash: hashFromRaw(p.payloadHash).asOpaque(), + gas: p.gas, + result: + p.result.kind === WorkExecResultKind.ok + ? WorkExecResult.ok(BytesBlob.blobFrom(p.result.okBlob ?? new Uint8Array())) + : WorkExecResult.error(p.result.kind as Exclude), + authorizationOutput: BytesBlob.blobFrom(p.authorizationOutput), + }); +} + +function deserializeUpdatePreimage(p: PlainUpdatePreimage): UpdatePreimage { + switch (p.kind) { + case UpdatePreimageKind.Provide: + return UpdatePreimage.provide({ + preimage: PreimageItem.create({ + hash: hashFromRaw(p.preimage.hash).asOpaque(), + blob: BytesBlob.blobFrom(p.preimage.blob), + }), + slot: p.slot, + providedFor: p.providedFor, + }); + case UpdatePreimageKind.Remove: + return UpdatePreimage.remove({ + hash: hashFromRaw(p.hash).asOpaque(), + length: p.length, + }); + case UpdatePreimageKind.UpdateOrAdd: + return UpdatePreimage.updateOrAdd({ + lookupHistory: deserializeLookupHistoryItem(p.item), + }); + } +} + +function deserializeUpdateService(p: PlainUpdateService): UpdateService { + if (p.kind === UpdateServiceKind.Create) { + return UpdateService.create({ + serviceInfo: deserializeServiceAccountInfo(p.account), + lookupHistory: p.lookupHistory !== null ? deserializeLookupHistoryItem(p.lookupHistory) : null, + }); + } + return UpdateService.update({ + serviceInfo: deserializeServiceAccountInfo(p.account), + }); +} + +function deserializeUpdateStorage(p: PlainUpdateStorage): UpdateStorage { + if (p.kind === UpdateStorageKind.Set) { + return UpdateStorage.set({ + storage: StorageItem.create({ + key: BytesBlob.blobFrom(p.storage.key) as StorageKey, + value: BytesBlob.blobFrom(p.storage.value), + }), + }); + } + return UpdateStorage.remove({ key: BytesBlob.blobFrom(p.key) as StorageKey }); +} + +export function deserializePrivilegedServices(p: PlainPrivilegedServices): PrivilegedServices { + return PrivilegedServices.create({ + manager: p.manager, + delegator: p.delegator, + registrar: p.registrar, + assigners: asKnownSize(p.assigners), + autoAccumulateServices: new Map(p.autoAccumulateServices), + }); +} + +export function deserializeAccumulationStateUpdate(p: PlainAccumulationStateUpdate): AccumulationStateUpdate { + const su = AccumulationStateUpdate.empty(); + + su.services.created.push(...p.created); + su.services.removed.push(...p.removed); + + for (const [id, pus] of p.updated) { + su.services.updated.set(id, deserializeUpdateService(pus)); + } + for (const [id, pups] of p.preimages) { + su.services.preimages.set(id, pups.map(deserializeUpdatePreimage)); + } + for (const [id, puss] of p.storage) { + su.services.storage.set(id, puss.map(deserializeUpdateStorage)); + } + + su.transfers.push(...p.transfers.map(deserializePendingTransfer)); + su.yieldedRoot = p.yieldedRoot !== null ? hashFromRaw(p.yieldedRoot).asOpaque() : null; + + for (const [core, queue] of p.authorizationQueues) { + su.authorizationQueues.set( + core, + asKnownSize(queue.map((h) => hashFromRaw(h).asOpaque())) as FixedSizeArray< + AuthorizerHash, + AUTHORIZATION_QUEUE_SIZE + >, + ); + } + + if (p.validatorsData !== null) { + su.validatorsData = asKnownSize( + p.validatorsData.map((d) => + ValidatorData.create({ + bandersnatch: Bytes.fromBlob(d.bandersnatch, BANDERSNATCH_KEY_BYTES).asOpaque(), + ed25519: Bytes.fromBlob(d.ed25519, ED25519_KEY_BYTES).asOpaque(), + bls: Bytes.fromBlob(d.bls, BLS_KEY_BYTES).asOpaque(), + metadata: Bytes.fromBlob(d.metadata, VALIDATOR_META_BYTES), + }), + ), + ) as PerValidator; + } + + if (p.privilegedServices !== null) { + su.privilegedServices = deserializePrivilegedServices(p.privilegedServices); + } + + return su; +} + +/** + * Reconstruct a Service-like object from PlainService. + * Returns an object implementing the Service interface for use in PartiallyUpdatedState. + */ +export function deserializeService(p: PlainService): Service { + const info = deserializeServiceAccountInfo(p.info); + + const preimageMap = new Map(); + for (const [hashRaw, pi] of p.preimages) { + const hash = hashFromRaw(hashRaw).asOpaque(); + preimageMap.set(hash.toString(), { hash, blob: BytesBlob.blobFrom(pi.blob) }); + } + + const lookupMap = new Map(); + for (const [hashRaw, items] of p.lookupHistory) { + const hash = hashFromRaw(hashRaw); + lookupMap.set(hash.toString(), items.map(deserializeLookupHistoryItem)); + } + + const storageMap = new Map(); + for (const [key, si] of p.storage) { + storageMap.set(key, BytesBlob.blobFrom(si.value)); + } + + return { + serviceId: p.serviceId, + getInfo: () => info, + getStorage: (rawKey) => storageMap.get(rawKey.toString()) ?? null, + hasPreimage: (hash) => preimageMap.has(hash.toString()), + getPreimage: (hash) => preimageMap.get(hash.toString())?.blob ?? null, + getLookupHistory: (hash, len) => { + const items = lookupMap.get(hash.toString()); + if (items === undefined) { + return null; + } + const found = items.find((x) => x.length === len); + return found?.slots ?? null; + }, + }; +} diff --git a/packages/jam/transition/accumulate/worker/worker.ts b/packages/jam/transition/accumulate/worker/worker.ts new file mode 100644 index 000000000..605d58fa3 --- /dev/null +++ b/packages/jam/transition/accumulate/worker/worker.ts @@ -0,0 +1,250 @@ +/** + * Worker thread entry point for PVM accumulation. + * + * Runs accumulateSingleService + pvmAccumulateInvocation in an isolated + * V8 context for better JIT optimization of the PVM interpreter. + */ +import { type MessagePort, workerData } from "node:worker_threads"; +import type { EntropyHash, TimeSlot } from "@typeberry/block"; +import { type ServiceGas, type ServiceId, tryAsServiceGas } from "@typeberry/block"; +import { W_C } from "@typeberry/block/gp-constants.js"; +import { Bytes } from "@typeberry/bytes"; +import { codec, Encoder } from "@typeberry/codec"; +import type { ChainSpec, PvmBackend } from "@typeberry/config"; +import { PvmExecutor, ReturnStatus } from "@typeberry/executor"; +import { Blake2b, HASH_SIZE } from "@typeberry/hash"; +import type { PendingTransfer } from "@typeberry/jam-host-calls"; +import { + type AccumulationStateUpdate, + PartiallyUpdatedState, +} from "@typeberry/jam-host-calls/externalities/state-update.js"; +import { sumU64, tryAsU32 } from "@typeberry/numbers"; +import { type Service, ServiceAccountInfo } from "@typeberry/state"; +import { Result } from "@typeberry/utils"; +import { AccumulateExternalities } from "../../externalities/accumulate-externalities.js"; +import { AccumulateFetchExternalities } from "../../externalities/accumulate-fetch-externalities.js"; +import { generateNextServiceId } from "../accumulate-utils.js"; +import type { Operand } from "../operand.js"; +import { type AccumulateRequest, type AccumulateResponse, type GetServiceResponse, MessageType } from "./protocol.js"; +import { + deserializeAccumulationStateUpdate, + deserializeOperand, + deserializePendingTransfer, + deserializePrivilegedServices, + deserializeService, + serializeAccumulationStateUpdate, +} from "./serialization.js"; + +const ARGS_CODEC = codec.object({ + slot: codec.varU32.asOpaque(), + serviceId: codec.varU32.asOpaque(), + argsLength: codec.varU32, +}); + +// ── Worker state ───────────────────────────────────────────────────────────── + +const { dataPort } = workerData as { dataPort: MessagePort }; + +/** Blake2b hasher — created once per worker lifetime. */ +let blake2bInstance: Blake2b | null = null; + +async function getBlake2b(): Promise { + if (blake2bInstance === null) { + blake2bInstance = await Blake2b.createHasher(); + } + return blake2bInstance; +} + +/** Cache of services fetched from main thread (per-request). */ +const serviceCache = new Map(); + +/** Pending getService requests waiting for main thread response. */ +let pendingGetService: ((service: Service | null) => void) | null = null; + +// ── Async getService via MessagePort ───────────────────────────────────────── + +function getServiceAsync(serviceId: ServiceId): Promise { + const cached = serviceCache.get(serviceId); + if (cached !== undefined) { + return Promise.resolve(cached); + } + + return new Promise((resolve) => { + pendingGetService = (service) => { + serviceCache.set(serviceId, service); + resolve(service); + }; + dataPort.postMessage({ type: MessageType.GetServiceRequest, serviceId }); + }); +} + +/** Handle incoming GetServiceResponse from main thread. */ +function handleGetServiceResponse(msg: GetServiceResponse) { + const service = msg.service !== null ? deserializeService(msg.service) : null; + if (pendingGetService !== null) { + const resolve = pendingGetService; + pendingGetService = null; + resolve(service); + } +} + +// ── Accumulation logic (mirrors accumulate.ts) ─────────────────────────────── + +async function runAccumulation(request: AccumulateRequest): Promise { + const blake2b = await getBlake2b(); + + const transfers = request.transfers.map(deserializePendingTransfer); + const operands = request.operands.map(deserializeOperand); + const entropy = Bytes.fromBlob(request.entropy, HASH_SIZE).asOpaque(); + const inputStateUpdate = deserializeAccumulationStateUpdate(request.inputStateUpdate); + const privilegedServices = deserializePrivilegedServices(request.privilegedServices); + const { serviceId, gasCost, slot, chainSpec, pvmBackend } = request; + + serviceCache.clear(); + + // State proxy — getService is sync (returns null, overlay handles it), + // getServiceAsync fetches from main thread. + const state = { + getService: (_id: ServiceId) => null as Service | null, + getServiceAsync, + privilegedServices, + }; + + const updatedState = new PartiallyUpdatedState(state, inputStateUpdate); + + // Update balance from incoming transfers + const serviceInfo = await updatedState.getServiceInfoAsync(serviceId); + if (serviceInfo !== null) { + const newBalance = sumU64(serviceInfo.balance, ...transfers.map((item) => item.amount)); + + if (newBalance.overflow) { + return { + type: MessageType.AccumulateResponse, + consumedGas: tryAsServiceGas(0n), + stateUpdate: null, + }; + } + + const newInfo = ServiceAccountInfo.create({ ...serviceInfo, balance: newBalance.value }); + updatedState.updateServiceInfo(serviceId, newInfo); + } + + const result = await pvmAccumulateInvocation( + blake2b, + chainSpec, + pvmBackend, + slot, + serviceId, + transfers, + operands, + gasCost, + entropy, + updatedState, + ); + + if (result.isError) { + return { + type: MessageType.AccumulateResponse, + consumedGas: tryAsServiceGas(0n), + stateUpdate: serializeAccumulationStateUpdate(updatedState.stateUpdate), + }; + } + + return { + type: MessageType.AccumulateResponse, + consumedGas: result.ok.consumedGas, + stateUpdate: serializeAccumulationStateUpdate(result.ok.stateUpdate), + }; +} + +enum PvmInvocationError { + NoService = 0, + NoPreimage = 1, + PreimageTooLong = 2, +} + +async function pvmAccumulateInvocation( + blake2b: Blake2b, + chainSpec: ChainSpec, + pvmBackend: PvmBackend, + slot: TimeSlot, + serviceId: ServiceId, + transfers: PendingTransfer[], + operands: Operand[], + gas: ServiceGas, + entropy: EntropyHash, + updatedState: PartiallyUpdatedState, +): Promise> { + const serviceInfo = await updatedState.getServiceInfoAsync(serviceId); + if (serviceInfo === null) { + return Result.error(PvmInvocationError.NoService, () => `Accumulate: service ${serviceId} not found`); + } + + const codeHash = serviceInfo.codeHash; + const code = await updatedState.getPreimageAsync(serviceId, codeHash.asOpaque()); + if (code === null) { + return Result.error(PvmInvocationError.NoPreimage, () => `Accumulate: code not found for service ${serviceId}`); + } + + if (code.length > W_C) { + return Result.error(PvmInvocationError.PreimageTooLong, () => `Accumulate: code too long for service ${serviceId}`); + } + + const nextServiceId = generateNextServiceId({ serviceId, entropy, timeslot: slot }, chainSpec, blake2b); + const partialState = new AccumulateExternalities(chainSpec, blake2b, updatedState, serviceId, nextServiceId, slot); + const fetchExternalities = new AccumulateFetchExternalities(entropy, transfers, operands, chainSpec); + + const externalities = { + partialState, + serviceExternalities: partialState, + fetchExternalities, + }; + + const executor = await PvmExecutor.createAccumulateExecutor(serviceId, code, externalities, chainSpec, pvmBackend); + + const invocationArgs = Encoder.encodeObject(ARGS_CODEC, { + slot, + serviceId, + argsLength: tryAsU32(transfers.length + operands.length), + }); + const result = await executor.run(invocationArgs, gas); + const [newState, checkpoint] = partialState.getStateUpdates(); + + if (result.status !== ReturnStatus.OK) { + return Result.ok({ stateUpdate: checkpoint, consumedGas: tryAsServiceGas(result.consumedGas) }); + } + + if (result.memorySlice.length === HASH_SIZE) { + const memorySlice = Bytes.fromBlob(result.memorySlice, HASH_SIZE); + newState.yieldedRoot = memorySlice.asOpaque(); + } + + return Result.ok({ stateUpdate: newState, consumedGas: tryAsServiceGas(result.consumedGas) }); +} + +// ── Message handler ────────────────────────────────────────────────────────── + +dataPort.on("message", (msg: AccumulateRequest | GetServiceResponse) => { + if ((msg as GetServiceResponse).type === MessageType.GetServiceResponse) { + handleGetServiceResponse(msg as GetServiceResponse); + return; + } + + if ((msg as AccumulateRequest).type !== MessageType.AccumulateRequest) { + return; + } + + runAccumulation(msg as AccumulateRequest) + .then((response) => { + dataPort.postMessage(response); + }) + .catch((err) => { + const error = err instanceof Error ? err.message : String(err); + dataPort.postMessage({ + type: MessageType.AccumulateResponse, + consumedGas: tryAsServiceGas(0n), + stateUpdate: null, + error, + } satisfies AccumulateResponse); + }); +}); diff --git a/packages/jam/transition/externalities/accumulate-externalities.test.ts b/packages/jam/transition/externalities/accumulate-externalities.test.ts index 9a0d3bfd3..c18fdd003 100644 --- a/packages/jam/transition/externalities/accumulate-externalities.test.ts +++ b/packages/jam/transition/externalities/accumulate-externalities.test.ts @@ -65,7 +65,7 @@ function partiallyUpdatedState() { const INVALID_SERVICE_ID_ERROR = "Either manager or delegator or registrar is not a valid service id."; describe("PartialState.checkPreimageStatus", () => { - it("should check preimage status from state", () => { + it("should check preimage status from state", async () => { const state = partiallyUpdatedState(); const partialState = new AccumulateExternalities( tinyChainSpec, @@ -80,14 +80,14 @@ describe("PartialState.checkPreimageStatus", () => { HASH_SIZE, ).asOpaque(); - const status = partialState.checkPreimageStatus(preimageHash, tryAsU64(35)); + const status = await partialState.checkPreimageStatus(preimageHash, tryAsU64(35)); assert.deepStrictEqual(status, { status: PreimageStatusKind.Available, data: [0], }); }); - it("should return preimage status when its in updated state", () => { + it("should return preimage status when its in updated state", async () => { const state = partiallyUpdatedState(); const serviceId = tryAsServiceId(0); const partialState = new AccumulateExternalities( @@ -113,7 +113,7 @@ describe("PartialState.checkPreimageStatus", () => { ); state.stateUpdate.services.preimages.set(serviceId, updates); - const status = partialState.checkPreimageStatus(preimageHash, length); + const status = await partialState.checkPreimageStatus(preimageHash, length); assert.deepStrictEqual(status, { status: PreimageStatusKind.Requested, }); @@ -121,7 +121,7 @@ describe("PartialState.checkPreimageStatus", () => { }); describe("PartialState.requestPreimage", () => { - it("should request a preimage and update service info", () => { + it("should request a preimage and update service info", async () => { const state = partiallyUpdatedState(); const serviceId = tryAsServiceId(0); const maybeService = state.state.services.get(serviceId); @@ -140,7 +140,7 @@ describe("PartialState.requestPreimage", () => { ); const preimageHash = Bytes.fill(HASH_SIZE, 0xa).asOpaque(); - const status = partialState.requestPreimage(preimageHash, tryAsU64(5)); + const status = await partialState.requestPreimage(preimageHash, tryAsU64(5)); assert.deepStrictEqual(status, Result.ok(OK)); assert.deepStrictEqual( @@ -173,7 +173,7 @@ describe("PartialState.requestPreimage", () => { ); }); - it("should request a preimage and update service info", () => { + it("should request a preimage and update service info", async () => { const state = partiallyUpdatedState(); const serviceId = tryAsServiceId(0); const maybeService = state.state.services.get(serviceId); @@ -192,7 +192,7 @@ describe("PartialState.requestPreimage", () => { ); const preimageHash = Bytes.fill(HASH_SIZE, 0xa).asOpaque(); - const status = partialState.requestPreimage(preimageHash, tryAsU64(5)); + const status = await partialState.requestPreimage(preimageHash, tryAsU64(5)); assert.deepStrictEqual(status, Result.ok(OK)); assert.deepStrictEqual( @@ -225,7 +225,7 @@ describe("PartialState.requestPreimage", () => { ); }); - it("should fail if preimage is already requested", () => { + it("should fail if preimage is already requested", async () => { const state = partiallyUpdatedState(); const partialState = new AccumulateExternalities( tinyChainSpec, @@ -237,10 +237,10 @@ describe("PartialState.requestPreimage", () => { ); const preimageHash = Bytes.fill(HASH_SIZE, 0xa).asOpaque(); - const status = partialState.requestPreimage(preimageHash, tryAsU64(5)); + const status = await partialState.requestPreimage(preimageHash, tryAsU64(5)); assert.deepStrictEqual(status, Result.ok(OK)); - const status2 = partialState.requestPreimage(preimageHash, tryAsU64(5)); + const status2 = await partialState.requestPreimage(preimageHash, tryAsU64(5)); deepEqual( status2, Result.error( @@ -250,7 +250,7 @@ describe("PartialState.requestPreimage", () => { ); }); - it("should fail if preimage is already available", () => { + it("should fail if preimage is already available", async () => { const state = partiallyUpdatedState(); const partialState = new AccumulateExternalities( tinyChainSpec, @@ -265,7 +265,7 @@ describe("PartialState.requestPreimage", () => { HASH_SIZE, ).asOpaque(); - const status = partialState.requestPreimage(preimageHash, tryAsU64(35)); + const status = await partialState.requestPreimage(preimageHash, tryAsU64(35)); deepEqual( status, Result.error( @@ -275,7 +275,7 @@ describe("PartialState.requestPreimage", () => { ); }); - it("should fail if balance is insufficient", () => { + it("should fail if balance is insufficient", async () => { const state = partiallyUpdatedState(); const partialState = new AccumulateExternalities( tinyChainSpec, @@ -287,7 +287,7 @@ describe("PartialState.requestPreimage", () => { ); const preimageHash = Bytes.fill(HASH_SIZE, 0xa).asOpaque(); - const status = partialState.requestPreimage(preimageHash, tryAsU64(2n ** 34n - 1n)); + const status = await partialState.requestPreimage(preimageHash, tryAsU64(2n ** 34n - 1n)); deepEqual( status, Result.error( @@ -299,7 +299,7 @@ describe("PartialState.requestPreimage", () => { }); describe("PartialState.forgetPreimage", () => { - it("should error if preimage does not exist", () => { + it("should error if preimage does not exist", async () => { const state = partiallyUpdatedState(); const partialState = new AccumulateExternalities( tinyChainSpec, @@ -311,7 +311,7 @@ describe("PartialState.forgetPreimage", () => { ); const hash = Bytes.fill(HASH_SIZE, 0x01).asOpaque(); - const result = partialState.forgetPreimage(hash, tryAsU64(42)); + const result = await partialState.forgetPreimage(hash, tryAsU64(42)); deepEqual( result, Result.error( @@ -321,7 +321,7 @@ describe("PartialState.forgetPreimage", () => { ); }); - it("should error if preimage is already forgotten", () => { + it("should error if preimage is already forgotten", async () => { const state = partiallyUpdatedState(); const serviceId = tryAsServiceId(0); const hash = Bytes.parseBytes( @@ -350,12 +350,12 @@ describe("PartialState.forgetPreimage", () => { ); state.stateUpdate.services.preimages.set(serviceId, updates); - const result1 = partialState.forgetPreimage(hash, length); + const result1 = await partialState.forgetPreimage(hash, length); assert.deepStrictEqual(result1, Result.ok(OK)); state.state.applyUpdate(state.stateUpdate.services); - const result2 = partialState.forgetPreimage(hash, length); + const result2 = await partialState.forgetPreimage(hash, length); deepEqual( result2, Result.error( @@ -365,7 +365,7 @@ describe("PartialState.forgetPreimage", () => { ); }); - it("should forget a requested preimage", () => { + it("should forget a requested preimage", async () => { const state = partiallyUpdatedState(); const serviceId = tryAsServiceId(0); const hash = Bytes.fill(HASH_SIZE, 0x03).asOpaque(); @@ -379,9 +379,9 @@ describe("PartialState.forgetPreimage", () => { tryAsServiceId(10), tryAsTimeSlot(16), ); - partialState.requestPreimage(hash, length); + await partialState.requestPreimage(hash, length); - const result = partialState.forgetPreimage(hash, length); + const result = await partialState.forgetPreimage(hash, length); assert.deepStrictEqual(result, Result.ok(OK)); assert.deepStrictEqual( @@ -403,7 +403,7 @@ describe("PartialState.forgetPreimage", () => { ); }); - it("should forget an unavailable preimage if it is old enough", () => { + it("should forget an unavailable preimage if it is old enough", async () => { const state = partiallyUpdatedState(); state.state.applyUpdate({ timeslot: tryAsTimeSlot(100000), @@ -433,7 +433,7 @@ describe("PartialState.forgetPreimage", () => { }), ); state.stateUpdate.services.preimages.set(serviceId, updates); - const result = partialState.forgetPreimage(hash, length); + const result = await partialState.forgetPreimage(hash, length); assert.deepStrictEqual(result, Result.ok(OK)); assert.deepStrictEqual( @@ -459,7 +459,7 @@ describe("PartialState.forgetPreimage", () => { ); }); - it("should not forget an unavailable preimage if it is recent", () => { + it("should not forget an unavailable preimage if it is recent", async () => { const state = partiallyUpdatedState(); state.state.applyUpdate({ timeslot: tryAsTimeSlot(100), @@ -486,11 +486,11 @@ describe("PartialState.forgetPreimage", () => { ); state.stateUpdate.services.preimages.set(serviceId, updates); - const result = partialState.forgetPreimage(hash, length); + const result = await partialState.forgetPreimage(hash, length); assert.deepStrictEqual(result, Result.ok(OK)); }); - it("should update lookup history for available preimage", () => { + it("should update lookup history for available preimage", async () => { const state = partiallyUpdatedState(); state.state.applyUpdate({ timeslot: tryAsTimeSlot(100), @@ -517,7 +517,7 @@ describe("PartialState.forgetPreimage", () => { ); state.stateUpdate.services.preimages.set(serviceId, updates); - const result = partialState.forgetPreimage(hash, length); + const result = await partialState.forgetPreimage(hash, length); assert.deepStrictEqual(result, Result.ok(OK)); assert.deepStrictEqual( @@ -546,7 +546,7 @@ describe("PartialState.forgetPreimage", () => { ); }); - it("should update history for reavailable preimage if old", () => { + it("should update history for reavailable preimage if old", async () => { const state = partiallyUpdatedState(); state.state.applyUpdate({ timeslot: tryAsTimeSlot(100000), @@ -578,7 +578,7 @@ describe("PartialState.forgetPreimage", () => { ); state.stateUpdate.services.preimages.set(serviceId, updates); - const result = partialState.forgetPreimage(hash, length); + const result = await partialState.forgetPreimage(hash, length); assert.deepStrictEqual(result, Result.ok(OK)); assert.deepStrictEqual( @@ -607,7 +607,7 @@ describe("PartialState.forgetPreimage", () => { ); }); - it("should not forget reavailable preimage if too recent", () => { + it("should not forget reavailable preimage if too recent", async () => { const state = partiallyUpdatedState(); state.state.applyUpdate({ timeslot: tryAsTimeSlot(100), @@ -639,14 +639,14 @@ describe("PartialState.forgetPreimage", () => { ); state.stateUpdate.services.preimages.set(serviceId, updates); - const result = partialState.forgetPreimage(hash, length); + const result = await partialState.forgetPreimage(hash, length); deepEqual( result, Result.error(ForgetPreimageError.NotExpired, () => "Preimage not expired: y=95, timeslot=100, period=32"), ); }); - it("should not forget unavailable preimage if too recent", () => { + it("should not forget unavailable preimage if too recent", async () => { const state = partiallyUpdatedState(); const hash = Bytes.fill(HASH_SIZE, 0x08).asOpaque(); @@ -673,7 +673,7 @@ describe("PartialState.forgetPreimage", () => { ); state.stateUpdate.services.preimages.set(serviceId, updates); - const result = partialState.forgetPreimage(hash, length); + const result = await partialState.forgetPreimage(hash, length); deepEqual( result, Result.error(ForgetPreimageError.NotExpired, () => "Preimage not expired: y=1, timeslot=2, period=32"), @@ -682,7 +682,7 @@ describe("PartialState.forgetPreimage", () => { }); describe("PartialState.newService", () => { - it("should create a new service and update balance + next service ID", () => { + it("should create a new service and update balance + next service ID", async () => { const state = partiallyUpdatedState(); const maybeService = state.state.services.get(tryAsServiceId(0)); if (maybeService === undefined) { @@ -712,7 +712,7 @@ describe("PartialState.newService", () => { const expectedBalance = tryAsU64(service.data.info.balance - thresholdForNew); // when - const result = partialState.newService( + const result = await partialState.newService( codeHash, codeLengthU64, accumulateMinGas, @@ -765,7 +765,7 @@ describe("PartialState.newService", () => { assert.deepStrictEqual(partialState.getNextNewServiceId(), tryAsServiceId(4294901556)); }); - it("should create a new service with given id and update balance + not changed next service ID", () => { + it("should create a new service with given id and update balance + not changed next service ID", async () => { const state = partiallyUpdatedState(); const serviceId = 0; // setting registrar privileges for our service @@ -803,7 +803,7 @@ describe("PartialState.newService", () => { const expectedBalance = tryAsU64(service.data.info.balance - thresholdForNew); // when - const result = partialState.newService( + const result = await partialState.newService( codeHash, codeLengthU64, accumulateMinGas, @@ -856,7 +856,7 @@ describe("PartialState.newService", () => { assert.deepStrictEqual(partialState.getNextNewServiceId(), tryAsServiceId(10)); }); - it("should return an error if there are insufficient funds", () => { + it("should return an error if there are insufficient funds", async () => { const state = partiallyUpdatedState(); const maybeService = state.state.services.get(tryAsServiceId(0)); if (maybeService === undefined) { @@ -891,7 +891,7 @@ describe("PartialState.newService", () => { const gratisStorage = tryAsU64(1024); // when - const result = partialState.newService( + const result = await partialState.newService( codeHash, codeLength, accumulateMinGas, @@ -913,7 +913,7 @@ describe("PartialState.newService", () => { assert.deepStrictEqual(state.stateUpdate.services.updated, new Map()); }); - it("should return an error if service is unprivileged to set gratis storage", () => { + it("should return an error if service is unprivileged to set gratis storage", async () => { const state = partiallyUpdatedState(); // setting different service than our privileged manager state.stateUpdate.privilegedServices = { @@ -952,7 +952,7 @@ describe("PartialState.newService", () => { const gratisStorage = tryAsU64(1024); // when - const result = partialState.newService( + const result = await partialState.newService( codeHash, codeLength, accumulateMinGas, @@ -971,7 +971,7 @@ describe("PartialState.newService", () => { assert.deepStrictEqual(state.stateUpdate.services.updated, new Map()); }); - it("should return an error if attempting to create new service with selected id that already exists", () => { + it("should return an error if attempting to create new service with selected id that already exists", async () => { const state = partiallyUpdatedState(); const serviceId = 0; // setting registrar privileges for our service @@ -1001,7 +1001,7 @@ describe("PartialState.newService", () => { const gratisStorage = tryAsU64(50); // when - const result = partialState.newService( + const result = await partialState.newService( codeHash, codeLengthU64, accumulateMinGas, @@ -1023,7 +1023,7 @@ describe("PartialState.newService", () => { assert.deepStrictEqual(partialState.getNextNewServiceId(), tryAsServiceId(10)); }); - it("should create a new service with random id if service is unprivileged to select new service id + next service id", () => { + it("should create a new service with random id if service is unprivileged to select new service id + next service id", async () => { const state = partiallyUpdatedState(); // setting different service than our privileged registrar state.stateUpdate.privilegedServices = { @@ -1053,7 +1053,7 @@ describe("PartialState.newService", () => { const serviceId = tryAsU64(42); // when - const result = partialState.newService( + const result = await partialState.newService( codeHash, codeLength, accumulateMinGas, @@ -1134,7 +1134,7 @@ describe("PartialState.updateValidatorsData", () => { }); describe("PartialState.checkpoint", () => { - it("should checkpoint the updates", () => { + it("should checkpoint the updates", async () => { const state = partiallyUpdatedState(); const partialState = new AccumulateExternalities( tinyChainSpec, @@ -1146,7 +1146,7 @@ describe("PartialState.checkpoint", () => { ); const preimageHash = Bytes.fill(HASH_SIZE, 0xa).asOpaque(); // put something into updated state - const status = partialState.requestPreimage(preimageHash, tryAsU64(5)); + const status = await partialState.requestPreimage(preimageHash, tryAsU64(5)); assert.deepStrictEqual(status, Result.ok(OK)); // when @@ -1158,7 +1158,7 @@ describe("PartialState.checkpoint", () => { }); describe("PartialState.upgradeService", () => { - it("should update the service with new code hash and gas limits", () => { + it("should update the service with new code hash and gas limits", async () => { const state = partiallyUpdatedState(); const maybeService = state.state.services.get(tryAsServiceId(0)); if (maybeService === undefined) { @@ -1180,7 +1180,7 @@ describe("PartialState.upgradeService", () => { const allowance = tryAsU64(2_000n); // when - partialState.upgradeService(codeHash, gas, allowance); + await partialState.upgradeService(codeHash, gas, allowance); // then assert.deepStrictEqual( @@ -1489,7 +1489,7 @@ describe("PartialState.transfer", () => { }; }; - it("should perform a successful transfer", () => { + it("should perform a successful transfer", async () => { const { state, service } = partiallyUpdatedStateWithSecondService(); const partialState = new AccumulateExternalities( tinyChainSpec, @@ -1508,7 +1508,7 @@ describe("PartialState.transfer", () => { const newBalance = service.data.info.balance - amount; // when - const result = partialState.transfer(destinationId, amount, gas, memo); + const result = await partialState.transfer(destinationId, amount, gas, memo); // then assert.deepStrictEqual(result, Result.ok(OK)); @@ -1537,7 +1537,7 @@ describe("PartialState.transfer", () => { ); }); - it("should return DestinationNotFound error if destination doesnt exist", () => { + it("should return DestinationNotFound error if destination doesnt exist", async () => { const { state } = partiallyUpdatedStateWithSecondService(); const partialState = new AccumulateExternalities( tinyChainSpec, @@ -1553,7 +1553,7 @@ describe("PartialState.transfer", () => { const memo = Bytes.fill(TRANSFER_MEMO_BYTES, 0xbb); // when - const result = partialState.transfer(tryAsServiceId(4), amount, gas, memo); + const result = await partialState.transfer(tryAsServiceId(4), amount, gas, memo); // then deepEqual( @@ -1562,7 +1562,7 @@ describe("PartialState.transfer", () => { ); }); - it("should return GasTooLow error if gas is below destination's minimum", () => { + it("should return GasTooLow error if gas is below destination's minimum", async () => { const { state } = partiallyUpdatedStateWithSecondService(); const partialState = new AccumulateExternalities( tinyChainSpec, @@ -1579,7 +1579,7 @@ describe("PartialState.transfer", () => { const memo = Bytes.fill(TRANSFER_MEMO_BYTES, 0xcc); // when - const result = partialState.transfer(destinationId, amount, gas, memo); + const result = await partialState.transfer(destinationId, amount, gas, memo); // then deepEqual( @@ -1588,7 +1588,7 @@ describe("PartialState.transfer", () => { ); }); - it("should return BalanceBelowThreshold error if balance would fall too low", () => { + it("should return BalanceBelowThreshold error if balance would fall too low", async () => { const { state } = partiallyUpdatedStateWithSecondService(); const partialState = new AccumulateExternalities( tinyChainSpec, @@ -1605,7 +1605,7 @@ describe("PartialState.transfer", () => { const memo = Bytes.fill(TRANSFER_MEMO_BYTES, 0xdd); // when - const result = partialState.transfer(destinationId, amount, gas, memo); + const result = await partialState.transfer(destinationId, amount, gas, memo); // then deepEqual( @@ -1700,7 +1700,7 @@ describe("PartialState.providePreimage", () => { }; }; - it("should provide a preimage for other service", () => { + it("should provide a preimage for other service", async () => { const { state, preimage } = partiallyUpdatedStateWithSecondService({ self: false, requested: true, @@ -1719,7 +1719,7 @@ describe("PartialState.providePreimage", () => { assert.deepStrictEqual(state.stateUpdate.services.preimages.size, 0); // when - const result = partialState.providePreimage(serviceId, preimage.blob); + const result = await partialState.providePreimage(serviceId, preimage.blob); // then assert.deepStrictEqual(result, Result.ok(OK)); @@ -1757,7 +1757,7 @@ describe("PartialState.providePreimage", () => { ); }); - it("should provide a preimage for itself", () => { + it("should provide a preimage for itself", async () => { const { state, preimage } = partiallyUpdatedStateWithSecondService({ self: true, requested: true }); const partialState = new AccumulateExternalities( @@ -1773,7 +1773,7 @@ describe("PartialState.providePreimage", () => { assert.deepStrictEqual(state.stateUpdate.services.preimages.size, 0); // when - const result = partialState.providePreimage(serviceId, preimage.blob); + const result = await partialState.providePreimage(serviceId, preimage.blob); // then assert.deepStrictEqual(result, Result.ok(OK)); @@ -1797,7 +1797,7 @@ describe("PartialState.providePreimage", () => { ); }); - it("should return error if preimage was not requested", () => { + it("should return error if preimage was not requested", async () => { const { state, preimage } = partiallyUpdatedStateWithSecondService({ self: false, requested: false, @@ -1816,7 +1816,7 @@ describe("PartialState.providePreimage", () => { assert.deepStrictEqual(state.stateUpdate.services.preimages.size, 0); // when - const result = partialState.providePreimage(serviceId, preimage.blob); + const result = await partialState.providePreimage(serviceId, preimage.blob); // then deepEqual( @@ -1830,7 +1830,7 @@ describe("PartialState.providePreimage", () => { assert.deepStrictEqual(state.stateUpdate.services.preimages.size, 0); }); - it("should return error if preimage is requested and already available for other service", () => { + it("should return error if preimage is requested and already available for other service", async () => { const { state, preimage } = partiallyUpdatedStateWithSecondService({ self: false, requested: true, @@ -1860,7 +1860,7 @@ describe("PartialState.providePreimage", () => { state.stateUpdate.services.preimages.set(serviceId, updates); // when - const result = partialState.providePreimage(serviceId, preimage.blob); + const result = await partialState.providePreimage(serviceId, preimage.blob); // then deepEqual( @@ -1891,7 +1891,7 @@ describe("PartialState.providePreimage", () => { ); }); - it("should return error if preimage is requested and already provided for self", () => { + it("should return error if preimage is requested and already provided for self", async () => { const { state, preimage } = partiallyUpdatedStateWithSecondService({ self: true, requested: true, @@ -1909,7 +1909,7 @@ describe("PartialState.providePreimage", () => { const serviceId = tryAsServiceId(0); // when - const result = partialState.providePreimage(serviceId, preimage.blob); + const result = await partialState.providePreimage(serviceId, preimage.blob); // then deepEqual( @@ -1923,7 +1923,7 @@ describe("PartialState.providePreimage", () => { assert.deepStrictEqual(state.stateUpdate.services.preimages, new Map()); }); - it("should return ok and then error if preimage is provided twice for self", () => { + it("should return ok and then error if preimage is provided twice for self", async () => { const { state, preimage } = partiallyUpdatedStateWithSecondService({ self: true, requested: true, @@ -1943,8 +1943,8 @@ describe("PartialState.providePreimage", () => { assert.deepStrictEqual(state.stateUpdate.services.preimages, new Map()); // when - const resultok = partialState.providePreimage(serviceId, preimage.blob); - const resulterr = partialState.providePreimage(serviceId, preimage.blob); + const resultok = await partialState.providePreimage(serviceId, preimage.blob); + const resulterr = await partialState.providePreimage(serviceId, preimage.blob); // then assert.deepStrictEqual(resultok, Result.ok(OK)); @@ -1976,7 +1976,7 @@ describe("PartialState.providePreimage", () => { ); }); - it("should return ok and then error if preimage is provided twice for other", () => { + it("should return ok and then error if preimage is provided twice for other", async () => { const { state, preimage } = partiallyUpdatedStateWithSecondService({ self: false, requested: true, @@ -1996,8 +1996,8 @@ describe("PartialState.providePreimage", () => { assert.deepStrictEqual(state.stateUpdate.services.preimages, new Map()); // when - const resultok = partialState.providePreimage(serviceId, preimage.blob); - const resulterr = partialState.providePreimage(serviceId, preimage.blob); + const resultok = await partialState.providePreimage(serviceId, preimage.blob); + const resulterr = await partialState.providePreimage(serviceId, preimage.blob); // then assert.deepStrictEqual(resultok, Result.ok(OK)); @@ -2110,7 +2110,7 @@ describe("PartialState.eject", () => { stateUpdate.services.set(destinationId, destinationService); return destinationId; } - it("should return InvalidService if destination is null", () => { + it("should return InvalidService if destination is null", async () => { const state = partiallyUpdatedState(); const partialState = new AccumulateExternalities( tinyChainSpec, @@ -2124,7 +2124,7 @@ describe("PartialState.eject", () => { const tombstone = Bytes.fill(HASH_SIZE, 0xef).asOpaque(); // when - const result = partialState.eject(null, tombstone); + const result = await partialState.eject(null, tombstone); // then deepEqual( @@ -2134,7 +2134,7 @@ describe("PartialState.eject", () => { assert.deepStrictEqual(state.stateUpdate.services.removed, []); }); - it("should return InvalidService if destination service does not exist", () => { + it("should return InvalidService if destination service does not exist", async () => { const state = partiallyUpdatedState(); const partialState = new AccumulateExternalities( tinyChainSpec, @@ -2149,7 +2149,7 @@ describe("PartialState.eject", () => { const tombstone = Bytes.fill(HASH_SIZE, 0xee).asOpaque(); // when - const result = partialState.eject(nonExistentServiceId, tombstone); + const result = await partialState.eject(nonExistentServiceId, tombstone); // then deepEqual( @@ -2159,7 +2159,7 @@ describe("PartialState.eject", () => { assert.deepStrictEqual(state.stateUpdate.services.removed, []); }); - it("should return InvalidService if destination service is already ejected", () => { + it("should return InvalidService if destination service is already ejected", async () => { const state = partiallyUpdatedState(); state.state.applyUpdate({ timeslot: tryAsTimeSlot(1_000_000), @@ -2185,11 +2185,11 @@ describe("PartialState.eject", () => { ); // when - const correctEjectResult = partialState.eject(destinationId, tombstone); // correct eject + const correctEjectResult = await partialState.eject(destinationId, tombstone); // correct eject assert.strictEqual(correctEjectResult.isOk, true); assert.deepStrictEqual(state.stateUpdate.services.removed, [destinationId]); - const incorrectResult = partialState.eject(destinationId, tombstone); // incorrect eject + const incorrectResult = await partialState.eject(destinationId, tombstone); // incorrect eject // then deepEqual( @@ -2198,7 +2198,7 @@ describe("PartialState.eject", () => { ); }); - it("should return InvalidService if destination service codeHash does not match expected pattern", () => { + it("should return InvalidService if destination service codeHash does not match expected pattern", async () => { const state = partiallyUpdatedState(); const destinationId = setupEjectableService(state.state, { codeHash: Bytes.fill(HASH_SIZE, 0x99).asOpaque(), // wrong codeHash @@ -2215,7 +2215,7 @@ describe("PartialState.eject", () => { ); // when - const result = partialState.eject(destinationId, tombstone); + const result = await partialState.eject(destinationId, tombstone); // then deepEqual( @@ -2225,7 +2225,7 @@ describe("PartialState.eject", () => { assert.deepStrictEqual(state.stateUpdate.services.removed, []); }); - it("should return InvalidPreimage if storageUtilisationCount is not equal to required value", () => { + it("should return InvalidPreimage if storageUtilisationCount is not equal to required value", async () => { const state = partiallyUpdatedState(); const destinationId = setupEjectableService(state.state, { storageUtilisationCount: tryAsU32(2 + 1), // off by 1 @@ -2242,7 +2242,7 @@ describe("PartialState.eject", () => { ); // when - const result = partialState.eject(destinationId, tombstone); + const result = await partialState.eject(destinationId, tombstone); // then deepEqual( @@ -2252,7 +2252,7 @@ describe("PartialState.eject", () => { assert.deepStrictEqual(state.stateUpdate.services.removed, []); }); - it("should return InvalidPreimage if the tombstone preimage is missing", () => { + it("should return InvalidPreimage if the tombstone preimage is missing", async () => { const state = partiallyUpdatedState(); const tombstone = Bytes.fill(HASH_SIZE, 0xea).asOpaque(); @@ -2269,7 +2269,7 @@ describe("PartialState.eject", () => { ); // when - const result = partialState.eject(destinationId, tombstone); + const result = await partialState.eject(destinationId, tombstone); // then deepEqual( @@ -2279,7 +2279,7 @@ describe("PartialState.eject", () => { assert.deepStrictEqual(state.stateUpdate.services.removed, []); }); - it("should return InvalidPreimage if tombstone preimage exists but has wrong status", () => { + it("should return InvalidPreimage if tombstone preimage exists but has wrong status", async () => { const state = partiallyUpdatedState(); const tombstone = Bytes.fill(HASH_SIZE, 0xe9).asOpaque(); const length = tryAsU32(100); @@ -2303,7 +2303,7 @@ describe("PartialState.eject", () => { ); // when - const result = partialState.eject(destinationId, tombstone); + const result = await partialState.eject(destinationId, tombstone); // then deepEqual( @@ -2313,7 +2313,7 @@ describe("PartialState.eject", () => { assert.deepStrictEqual(state.stateUpdate.services.removed, []); }); - it("should return InvalidPreimage if tombstone preimage exists but is not expired", () => { + it("should return InvalidPreimage if tombstone preimage exists but is not expired", async () => { const state = partiallyUpdatedState(); const tombstone = Bytes.fill(HASH_SIZE, 0xe9).asOpaque(); const length = tryAsU32(13); @@ -2337,7 +2337,7 @@ describe("PartialState.eject", () => { ); // when - const result = partialState.eject(destinationId, tombstone); + const result = await partialState.eject(destinationId, tombstone); // then deepEqual( @@ -2347,7 +2347,7 @@ describe("PartialState.eject", () => { assert.deepStrictEqual(state.stateUpdate.services.removed, []); }); - it("should return InvalidService if summing balances would overflow", () => { + it("should return InvalidService if summing balances would overflow", async () => { const state = partiallyUpdatedState(); state.state.applyUpdate({ timeslot: tryAsTimeSlot(1_000_000), @@ -2387,7 +2387,7 @@ describe("PartialState.eject", () => { ); // when - const result = partialState.eject(destinationId, tombstone); + const result = await partialState.eject(destinationId, tombstone); // then deepEqual( @@ -2397,7 +2397,7 @@ describe("PartialState.eject", () => { assert.deepStrictEqual(state.stateUpdate.services.removed, []); }); - it("should return OK", () => { + it("should return OK", async () => { const state = partiallyUpdatedState(); state.state.applyUpdate({ timeslot: tryAsTimeSlot(1_000_000), @@ -2423,7 +2423,7 @@ describe("PartialState.eject", () => { ); // when - const result = partialState.eject(destinationId, tombstone); + const result = await partialState.eject(destinationId, tombstone); // then assert.deepStrictEqual(result, Result.ok(OK)); @@ -2498,7 +2498,7 @@ describe("AccumulateServiceExternalities", () => { }; describe("getInfo", () => { - it("should return null when serviceId is null", () => { + it("should return null when serviceId is null", async () => { const currentServiceId = tryAsServiceId(10_000); const serviceId: ServiceId | null = null; const state = prepareState([prepareService(currentServiceId)]); @@ -2513,12 +2513,12 @@ describe("AccumulateServiceExternalities", () => { tryAsTimeSlot(16), ); - const serviceInfo = accumulateServiceExternalities.getServiceInfo(serviceId); + const serviceInfo = await accumulateServiceExternalities.getServiceInfo(serviceId); assert.strictEqual(serviceInfo, expectedServiceInfo); }); - it("should return null when serviceId is incorrect", () => { + it("should return null when serviceId is incorrect", async () => { const currentServiceId = tryAsServiceId(10_000); const serviceId = tryAsServiceId(5); const state = prepareState([prepareService(currentServiceId)]); @@ -2533,12 +2533,12 @@ describe("AccumulateServiceExternalities", () => { tryAsTimeSlot(16), ); - const serviceInfo = accumulateServiceExternalities.getServiceInfo(serviceId); + const serviceInfo = await accumulateServiceExternalities.getServiceInfo(serviceId); assert.strictEqual(serviceInfo, expectedServiceInfo); }); - it("should return correct service info", () => { + it("should return correct service info", async () => { const currentServiceId = tryAsServiceId(10_000); const serviceId = tryAsServiceId(5); const state = prepareState([prepareService(currentServiceId), prepareService(serviceId)]); @@ -2553,14 +2553,14 @@ describe("AccumulateServiceExternalities", () => { tryAsTimeSlot(16), ); - const serviceInfo = accumulateServiceExternalities.getServiceInfo(serviceId); + const serviceInfo = await accumulateServiceExternalities.getServiceInfo(serviceId); assert.deepStrictEqual(serviceInfo, expectedServiceInfo); }); }); describe("lookup", () => { - it("should return null when serviceId is null", () => { + it("should return null when serviceId is null", async () => { const currentServiceId = tryAsServiceId(10_000); const serviceId: ServiceId | null = null; const hash = Bytes.fill(HASH_SIZE, 1).asOpaque(); @@ -2576,12 +2576,12 @@ describe("AccumulateServiceExternalities", () => { tryAsTimeSlot(16), ); - const result = accumulateServiceExternalities.lookup(serviceId, hash); + const result = await accumulateServiceExternalities.lookup(serviceId, hash); assert.strictEqual(result, expectedResult); }); - it("should return null when service does not exist", () => { + it("should return null when service does not exist", async () => { const currentServiceId = tryAsServiceId(10_000); const serviceId = tryAsServiceId(0); const hash = Bytes.fill(HASH_SIZE, 1).asOpaque(); @@ -2597,12 +2597,12 @@ describe("AccumulateServiceExternalities", () => { tryAsTimeSlot(16), ); - const result = accumulateServiceExternalities.lookup(serviceId, hash); + const result = await accumulateServiceExternalities.lookup(serviceId, hash); assert.strictEqual(result, expectedResult); }); - it("should return null when preimage does not exists", () => { + it("should return null when preimage does not exists", async () => { const currentServiceId = tryAsServiceId(10_000); const requestedHash = Bytes.fill(HASH_SIZE, 1).asOpaque(); const otherHash = Bytes.fill(HASH_SIZE, 2).asOpaque(); @@ -2620,12 +2620,12 @@ describe("AccumulateServiceExternalities", () => { tryAsTimeSlot(16), ); - const result = accumulateServiceExternalities.lookup(currentServiceId, requestedHash); + const result = await accumulateServiceExternalities.lookup(currentServiceId, requestedHash); assert.strictEqual(result, expectedResult); }); - it("should return return a correct preimage", () => { + it("should return return a correct preimage", async () => { const serviceId = tryAsServiceId(0); const expectedResult = BytesBlob.empty(); const requestedHash = Bytes.fill(HASH_SIZE, 1).asOpaque(); @@ -2642,14 +2642,14 @@ describe("AccumulateServiceExternalities", () => { tryAsTimeSlot(16), ); - const result = accumulateServiceExternalities.lookup(serviceId, requestedHash); + const result = await accumulateServiceExternalities.lookup(serviceId, requestedHash); assert.deepStrictEqual(result, expectedResult); }); }); describe("read / write", () => { - it("should return null when serviceId is null ", () => { + it("should return null when serviceId is null ", async () => { const currentServiceId = tryAsServiceId(10_000); const serviceId: ServiceId | null = null; const hash = Bytes.fill(HASH_SIZE, 1).asOpaque(); @@ -2664,12 +2664,12 @@ describe("AccumulateServiceExternalities", () => { tryAsTimeSlot(16), ); - const result = accumulateServiceExternalities.read(serviceId, hash); + const result = await accumulateServiceExternalities.read(serviceId, hash); assert.strictEqual(result, null); }); - it("should return null when service does not exist ", () => { + it("should return null when service does not exist ", async () => { const currentServiceId = tryAsServiceId(10_000); const serviceId = tryAsServiceId(33); const hash = Bytes.fill(HASH_SIZE, 1).asOpaque(); @@ -2683,12 +2683,12 @@ describe("AccumulateServiceExternalities", () => { tryAsTimeSlot(16), ); - const result = accumulateServiceExternalities.read(serviceId, hash); + const result = await accumulateServiceExternalities.read(serviceId, hash); assert.strictEqual(result, null); }); - it("should correctly read from storage", () => { + it("should correctly read from storage", async () => { const currentServiceId = tryAsServiceId(10_000); const serviceId = tryAsServiceId(33); const key: StorageKey = Bytes.fill(HASH_SIZE, 1).asOpaque(); @@ -2713,11 +2713,11 @@ describe("AccumulateServiceExternalities", () => { tryAsTimeSlot(16), ); - const result = accumulateServiceExternalities.read(serviceId, key); + const result = await accumulateServiceExternalities.read(serviceId, key); assert.strictEqual(result, value); }); - it("should correctly write to storage", () => { + it("should correctly write to storage", async () => { const currentServiceId = tryAsServiceId(10_000); const hash = Bytes.fill(HASH_SIZE, 1).asOpaque(); const blob = BytesBlob.empty(); @@ -2733,12 +2733,12 @@ describe("AccumulateServiceExternalities", () => { assert.strictEqual(state.stateUpdate.services.storage.size, 0); - accumulateServiceExternalities.write(hash, blob); + await accumulateServiceExternalities.write(hash, blob); assert.strictEqual(state.stateUpdate.services.storage.size, 1); }); - it("should return new value if there was a write", () => { + it("should return new value if there was a write", async () => { const currentServiceId = tryAsServiceId(10_000); const key: StorageKey = Bytes.fill(HASH_SIZE, 2).asOpaque(); const initialStorage = new Map(); @@ -2756,11 +2756,11 @@ describe("AccumulateServiceExternalities", () => { tryAsTimeSlot(16), ); - accumulateServiceExternalities.write(key, newBlob); + await accumulateServiceExternalities.write(key, newBlob); assert.strictEqual(state.stateUpdate.services.storage.size, 1); - const result = accumulateServiceExternalities.read(currentServiceId, key); + const result = await accumulateServiceExternalities.read(currentServiceId, key); assert.deepStrictEqual(result, newBlob); }); diff --git a/packages/jam/transition/externalities/accumulate-externalities.ts b/packages/jam/transition/externalities/accumulate-externalities.ts index 1ba81f384..2c9ac2580 100644 --- a/packages/jam/transition/externalities/accumulate-externalities.ts +++ b/packages/jam/transition/externalities/accumulate-externalities.ts @@ -110,8 +110,8 @@ export class AccumulateExternalities * * Takes into account updates over the state. */ - private getCurrentServiceInfo(): ServiceAccountInfo { - const serviceInfo = this.updatedState.getServiceInfo(this.currentServiceId); + private async getCurrentServiceInfo(): Promise { + const serviceInfo = await this.updatedState.getServiceInfoAsync(this.currentServiceId); if (serviceInfo === null) { throw new Error(`Missing service info for current service! ${this.currentServiceId}`); } @@ -125,8 +125,8 @@ export class AccumulateExternalities * * Takes into account newly created services as well. */ - getServiceInfo(destination: ServiceId | null): ServiceAccountInfo | null { - return this.updatedState.getServiceInfo(destination); + async getServiceInfo(destination: ServiceId | null): Promise { + return this.updatedState.getServiceInfoAsync(destination); } /** @@ -142,8 +142,17 @@ export class AccumulateExternalities * - cannot be "freshly provided", since we defer updating the * lookup status. */ - private isPreviousCodeExpired(destination: ServiceId, previousCodeHash: PreimageHash, len: U64): [boolean, string] { - const slots = this.updatedState.getLookupHistory(this.currentTimeslot, destination, previousCodeHash, len); + private async isPreviousCodeExpired( + destination: ServiceId, + previousCodeHash: PreimageHash, + len: U64, + ): Promise<[boolean, string]> { + const slots = await this.updatedState.getLookupHistoryAsync( + this.currentTimeslot, + destination, + previousCodeHash, + len, + ); const status = slots === null ? null : slotsToPreimageStatus(slots.slots); // The previous code needs to be forgotten and expired. if (status?.status !== PreimageStatusKind.Unavailable) { @@ -160,7 +169,7 @@ export class AccumulateExternalities const mod = 2 ** 32 - MIN_PUBLIC_SERVICE_INDEX - 2 ** 8; for (;;) { - const service = this.getServiceInfo(currentServiceId); + const service = this.updatedState.getServiceInfo(currentServiceId); // we found an empty id if (service === null) { return currentServiceId; @@ -172,9 +181,14 @@ export class AccumulateExternalities } } - checkPreimageStatus(hash: PreimageHash, length: U64): PreimageStatus | null { + async checkPreimageStatus(hash: PreimageHash, length: U64): Promise { // https://graypaper.fluffylabs.dev/#/9a08063/378102378102?v=0.6.6 - const status = this.updatedState.getLookupHistory(this.currentTimeslot, this.currentServiceId, hash, length); + const status = await this.updatedState.getLookupHistoryAsync( + this.currentTimeslot, + this.currentServiceId, + hash, + length, + ); if (status === null) { return null; } @@ -182,8 +196,8 @@ export class AccumulateExternalities return slotsToPreimageStatus(status.slots); } - requestPreimage(hash: PreimageHash, length: U64): Result { - const existingPreimage = this.updatedState.getLookupHistory( + async requestPreimage(hash: PreimageHash, length: U64): Promise> { + const existingPreimage = await this.updatedState.getLookupHistoryAsync( this.currentTimeslot, this.currentServiceId, hash, @@ -207,7 +221,7 @@ export class AccumulateExternalities // make sure we have enough balance for this update // https://graypaper.fluffylabs.dev/#/9a08063/381201381601?v=0.6.6 - const serviceInfo = this.getCurrentServiceInfo(); + const serviceInfo = await this.getCurrentServiceInfo(); const hasPreimage = existingPreimage !== null; const countDiff = hasPreimage ? 0 : 2; const lenDiff = length - BigInt(existingPreimage?.length ?? 0); @@ -252,17 +266,22 @@ export class AccumulateExternalities return Result.ok(OK); } - forgetPreimage(hash: PreimageHash, length: U64): Result { + async forgetPreimage(hash: PreimageHash, length: U64): Promise> { const serviceId = this.currentServiceId; - const status = this.updatedState.getLookupHistory(this.currentTimeslot, this.currentServiceId, hash, length); + const status = await this.updatedState.getLookupHistoryAsync( + this.currentTimeslot, + this.currentServiceId, + hash, + length, + ); if (status === null) { return Result.error(ForgetPreimageError.NotFound, () => `Preimage not found: hash=${hash}, length=${length}`); } const s = slotsToPreimageStatus(status.slots); - const updateStorageUtilisation = () => { - const serviceInfo = this.getCurrentServiceInfo(); + const updateStorageUtilisation = async () => { + const serviceInfo = await this.getCurrentServiceInfo(); const items = serviceInfo.storageUtilisationCount - 2; // subtracting 1 for lookup history item and 1 for the preimage const bytes = serviceInfo.storageUtilisationBytes - length - LOOKUP_HISTORY_ENTRY_BYTES; return this.updatedState.updateServiceStorageUtilisation(this.currentServiceId, items, bytes, serviceInfo); @@ -270,7 +289,7 @@ export class AccumulateExternalities // https://graypaper.fluffylabs.dev/#/ab2cdbd/380802380802?v=0.7.2 if (s.status === PreimageStatusKind.Requested) { - const res = updateStorageUtilisation(); + const res = await updateStorageUtilisation(); if (res.isError) { return Result.error(ForgetPreimageError.StorageUtilisationError, res.details); } @@ -289,7 +308,7 @@ export class AccumulateExternalities if (s.status === PreimageStatusKind.Unavailable) { const y = s.data[1]; if (y < t - this.chainSpec.preimageExpungePeriod) { - const res = updateStorageUtilisation(); + const res = await updateStorageUtilisation(); if (res.isError) { return Result.error(ForgetPreimageError.StorageUtilisationError, res.details); } @@ -343,14 +362,14 @@ export class AccumulateExternalities assertNever(s); } - transfer( + async transfer( destinationId: ServiceId | null, amount: U64, gas: ServiceGas, memo: Bytes, - ): Result { - const source = this.getCurrentServiceInfo(); - const destination = this.getServiceInfo(destinationId); + ): Promise> { + const source = await this.getCurrentServiceInfo(); + const destination = await this.getServiceInfo(destinationId); /** https://graypaper.fluffylabs.dev/#/9a08063/370401370401?v=0.6.6 */ if (destination === null || destinationId === null) { return Result.error(TransferError.DestinationNotFound, () => `Destination service not found: ${destinationId}`); @@ -397,14 +416,14 @@ export class AccumulateExternalities return Result.ok(OK); } - newService( + async newService( codeHash: CodeHash, codeLength: U64, accumulateMinGas: ServiceGas, onTransferMinGas: ServiceGas, gratisStorage: U64, wantedServiceId: U64, - ): Result { + ): Promise> { // calculate the threshold. Storage is empty, one preimage requested. // https://graypaper.fluffylabs.dev/#/7e6ff6a/115901115901?v=0.6.7 const items = tryAsU32(2 * 1 + 0); @@ -424,7 +443,7 @@ export class AccumulateExternalities // check if we have enough balance // https://graypaper.fluffylabs.dev/#/7e6ff6a/369e0336a303?v=0.6.7 const thresholdForNew = ServiceAccountInfo.calculateThresholdBalance(items, bytes.value, gratisStorage); - const currentService = this.getCurrentServiceInfo(); + const currentService = await this.getCurrentServiceInfo(); const thresholdForCurrent = ServiceAccountInfo.calculateThresholdBalance( currentService.storageUtilisationCount, currentService.storageUtilisationBytes, @@ -467,7 +486,7 @@ export class AccumulateExternalities ) { // NOTE: It's safe to cast to `Number` here, bcs here service ID cannot be bigger than 2**16 const newServiceId = tryAsServiceId(Number(wantedServiceId)); - if (this.getServiceInfo(newServiceId) !== null) { + if ((await this.getServiceInfo(newServiceId)) !== null) { return Result.error( NewServiceError.RegistrarServiceIdAlreadyTaken, () => `Service ID ${newServiceId} already taken`, @@ -501,9 +520,9 @@ export class AccumulateExternalities return Result.ok(newServiceId); } - upgradeService(codeHash: CodeHash, gas: U64, allowance: U64): void { + async upgradeService(codeHash: CodeHash, gas: U64, allowance: U64): Promise { /** https://graypaper.fluffylabs.dev/#/9a08063/36c80336c803?v=0.6.6 */ - const serviceInfo = this.getCurrentServiceInfo(); + const serviceInfo = await this.getCurrentServiceInfo(); this.updatedState.updateServiceInfo( this.currentServiceId, ServiceAccountInfo.create({ @@ -597,11 +616,11 @@ export class AccumulateExternalities this.updatedState.stateUpdate.yieldedRoot = hash; } - providePreimage(serviceId: ServiceId | null, preimage: BytesBlob): Result { + async providePreimage(serviceId: ServiceId | null, preimage: BytesBlob): Promise> { // we need to explicitly check if service exists, since it's a different error. // we also check if it's in newly created // https://graypaper.fluffylabs.dev/#/ab2cdbd/384e03384e03?v=0.7.2 - const service = serviceId === null ? null : this.updatedState.getServiceInfo(serviceId); + const service = serviceId === null ? null : await this.updatedState.getServiceInfoAsync(serviceId); if (service === null || serviceId === null) { return Result.error(ProvidePreimageError.ServiceNotFound, () => `Service not found: ${serviceId}`); } @@ -610,7 +629,7 @@ export class AccumulateExternalities const preimageHash = this.blake2b.hashBytes(preimage).asOpaque(); // checking service internal lookup - const stateLookup = this.updatedState.getLookupHistory( + const stateLookup = await this.updatedState.getLookupHistoryAsync( this.currentTimeslot, serviceId, preimageHash, @@ -624,7 +643,7 @@ export class AccumulateExternalities } // checking already provided preimages - const hasPreimage = this.updatedState.hasPreimage(serviceId, preimageHash); + const hasPreimage = await this.updatedState.hasPreimageAsync(serviceId, preimageHash); if (hasPreimage) { return Result.error( ProvidePreimageError.AlreadyProvided, @@ -652,8 +671,8 @@ export class AccumulateExternalities return Result.ok(OK); } - eject(destination: ServiceId | null, previousCodeHash: PreimageHash): Result { - const service = this.getServiceInfo(destination); + async eject(destination: ServiceId | null, previousCodeHash: PreimageHash): Promise> { + const service = await this.getServiceInfo(destination); const isRemoved = this.updatedState.stateUpdate.services.removed.find((serviceId) => serviceId === destination) !== undefined; @@ -661,7 +680,7 @@ export class AccumulateExternalities return Result.error(EjectError.InvalidService, () => "Service missing"); } - const currentService = this.getCurrentServiceInfo(); + const currentService = await this.getCurrentServiceInfo(); // check if the service expects to be ejected by us: const expectedCodeHash = Bytes.zero(HASH_SIZE).asOpaque(); @@ -681,7 +700,7 @@ export class AccumulateExternalities ); // check if we have a preimage with the entire storage. - const [isPreviousCodeExpired, errorReason] = this.isPreviousCodeExpired(destination, previousCodeHash, l); + const [isPreviousCodeExpired, errorReason] = await this.isPreviousCodeExpired(destination, previousCodeHash, l); if (!isPreviousCodeExpired) { return Result.error(EjectError.InvalidPreimage, () => `Previous code available: ${errorReason}`); } @@ -714,16 +733,16 @@ export class AccumulateExternalities return Result.ok(OK); } - read(serviceId: ServiceId | null, rawKey: StorageKey): BytesBlob | null { + async read(serviceId: ServiceId | null, rawKey: StorageKey): Promise { if (serviceId === null) { return null; } - return this.updatedState.getStorage(serviceId, rawKey); + return this.updatedState.getStorageAsync(serviceId, rawKey); } - write(rawKey: StorageKey, data: BytesBlob | null): Result { + async write(rawKey: StorageKey, data: BytesBlob | null): Promise> { const rawKeyBytes = tryAsU64(rawKey.length); - const current = this.read(this.currentServiceId, rawKey); + const current = await this.read(this.currentServiceId, rawKey); const isAddingNew = current === null && data !== null; const isRemoving = current !== null && data === null; const countDiff = isAddingNew ? 1 : isRemoving ? -1 : 0; @@ -733,7 +752,7 @@ export class AccumulateExternalities const keyDiffAdding = isAddingNew ? rawKeyBytes : 0n; const rawKeyDiff = keyDiffRemoving + keyDiffAdding; - const serviceInfo = this.getCurrentServiceInfo(); + const serviceInfo = await this.getCurrentServiceInfo(); const items = serviceInfo.storageUtilisationCount + countDiff; const bytes = serviceInfo.storageUtilisationBytes + BigInt(lenDiff) + baseStorageDiff + rawKeyDiff; const res = this.updatedState.updateServiceStorageUtilisation(this.currentServiceId, items, bytes, serviceInfo); @@ -746,12 +765,12 @@ export class AccumulateExternalities return Result.ok(current === null ? null : current.length); } - lookup(serviceId: ServiceId | null, hash: PreimageHash): BytesBlob | null { + async lookup(serviceId: ServiceId | null, hash: PreimageHash): Promise { if (serviceId === null) { return null; } - return this.updatedState.getPreimage(serviceId, hash); + return this.updatedState.getPreimageAsync(serviceId, hash); } } diff --git a/packages/workers/importer/importer.ts b/packages/workers/importer/importer.ts index def610c9e..ef332468e 100644 --- a/packages/workers/importer/importer.ts +++ b/packages/workers/importer/importer.ts @@ -60,7 +60,13 @@ export class Importer { } this.verifier = new BlockVerifier(hasher, blocks); - this.stf = new OnChain(spec, state, hasher, { pvm, accumulateSequentially: false }, DbHeaderChain.new(blocks)); + this.stf = new OnChain( + spec, + state, + hasher, + { pvm, accumulateSequentially: false, accumulateWorkers: 1 }, + DbHeaderChain.new(blocks), + ); this.state = state; this.currentHash = currentBestHeaderHash; this.prepareForNextEpoch(); From ff30c694ed3c0072b70feb94570db0b4c7e5d649 Mon Sep 17 00:00:00 2001 From: Mateusz Sikora Date: Sun, 29 Mar 2026 13:14:34 +0200 Subject: [PATCH 10/11] fix: worker compatibility with Node strip-only mode --- .../jam/transition/accumulate/accumulate.ts | 4 +-- .../jam/transition/accumulate/worker/pool.ts | 7 +++-- .../transition/accumulate/worker/protocol.ts | 20 +++++++------- .../transition/accumulate/worker/worker.ts | 27 ++++++++++--------- 4 files changed, 28 insertions(+), 30 deletions(-) diff --git a/packages/jam/transition/accumulate/accumulate.ts b/packages/jam/transition/accumulate/accumulate.ts index 4b17901a8..a45c7cbed 100644 --- a/packages/jam/transition/accumulate/accumulate.ts +++ b/packages/jam/transition/accumulate/accumulate.ts @@ -51,7 +51,7 @@ import { import type { Operand } from "./operand.js"; import type { AccumulateOptions } from "./options.js"; import { AccumulateWorkerPool } from "./worker/pool.js"; -import { type AccumulateRequest, MessageType } from "./worker/protocol.js"; +import { type AccumulateRequest, MSG_ACCUMULATE_REQUEST } from "./worker/protocol.js"; import { deserializeAccumulationStateUpdate, serializeAccumulationStateUpdate, @@ -470,7 +470,7 @@ export class Accumulate { const checkpoint = AccumulationStateUpdate.copyFrom(inputStateUpdate); const request: AccumulateRequest = { - type: MessageType.AccumulateRequest, + type: MSG_ACCUMULATE_REQUEST, serviceId, transfers: accumulateData.getTransfers(serviceId).map(serializePendingTransfer), operands: accumulateData.getOperands(serviceId).map(serializeOperand), diff --git a/packages/jam/transition/accumulate/worker/pool.ts b/packages/jam/transition/accumulate/worker/pool.ts index a5149f456..0053470d8 100644 --- a/packages/jam/transition/accumulate/worker/pool.ts +++ b/packages/jam/transition/accumulate/worker/pool.ts @@ -8,7 +8,7 @@ import { MessageChannel, type MessagePort, Worker } from "node:worker_threads"; import type { ServiceId } from "@typeberry/block"; import { Logger } from "@typeberry/logger"; import type { Service } from "@typeberry/state"; -import { type AccumulateRequest, type AccumulateResponse, type GetServiceRequest, MessageType } from "./protocol.js"; +import { type AccumulateRequest, type AccumulateResponse, type GetServiceRequest, MSG_GET_SERVICE_REQUEST, MSG_GET_SERVICE_RESPONSE } from "./protocol.js"; import { type PlainService, serializeService } from "./serialization.js"; const logger = Logger.new(import.meta.filename, "worker-pool"); @@ -42,7 +42,6 @@ export class AccumulateWorkerPool { const worker = new Worker(workerUrl, { workerData: { dataPort: workerPort }, transferList: [workerPort], - execArgv: ["--import", "tsx"], }); mainPort.on("message", (msg: AccumulateResponse | GetServiceRequest) => { @@ -82,7 +81,7 @@ export class AccumulateWorkerPool { private handleWorkerMessage(workerIndex: number, msg: AccumulateResponse | GetServiceRequest) { const handle = this.workers[workerIndex]; - if ((msg as GetServiceRequest).type === MessageType.GetServiceRequest) { + if ((msg as GetServiceRequest).type === MSG_GET_SERVICE_REQUEST) { this.handleGetServiceRequest(handle, msg as GetServiceRequest); return; } @@ -103,7 +102,7 @@ export class AccumulateWorkerPool { const plainService: PlainService | null = service !== null ? serializeService(service) : null; handle.dataPort.postMessage({ - type: MessageType.GetServiceResponse, + type: MSG_GET_SERVICE_RESPONSE, service: plainService, }); } diff --git a/packages/jam/transition/accumulate/worker/protocol.ts b/packages/jam/transition/accumulate/worker/protocol.ts index 61c23cd03..49b94b96d 100644 --- a/packages/jam/transition/accumulate/worker/protocol.ts +++ b/packages/jam/transition/accumulate/worker/protocol.ts @@ -14,16 +14,14 @@ import type { PlainService, } from "./serialization.js"; -export enum MessageType { - AccumulateRequest = 0, - AccumulateResponse = 1, - GetServiceRequest = 2, - GetServiceResponse = 3, -} +export const MSG_ACCUMULATE_REQUEST = 0; +export const MSG_ACCUMULATE_RESPONSE = 1; +export const MSG_GET_SERVICE_REQUEST = 2; +export const MSG_GET_SERVICE_RESPONSE = 3; /** Main → Worker: request to accumulate a single service. */ export type AccumulateRequest = { - type: MessageType.AccumulateRequest; + type: typeof MSG_ACCUMULATE_REQUEST; serviceId: ServiceId; transfers: PlainPendingTransfer[]; operands: PlainOperand[]; @@ -38,20 +36,20 @@ export type AccumulateRequest = { /** Worker → Main: result of accumulation. */ export type AccumulateResponse = { - type: MessageType.AccumulateResponse; + type: typeof MSG_ACCUMULATE_RESPONSE; consumedGas: ServiceGas; stateUpdate: PlainAccumulationStateUpdate | null; error?: string; }; -/** Worker → Main: synchronous request for service data from base state. */ +/** Worker → Main: request for service data from base state. */ export type GetServiceRequest = { - type: MessageType.GetServiceRequest; + type: typeof MSG_GET_SERVICE_REQUEST; serviceId: ServiceId; }; /** Main → Worker: response with serialized service data. */ export type GetServiceResponse = { - type: MessageType.GetServiceResponse; + type: typeof MSG_GET_SERVICE_RESPONSE; service: PlainService | null; }; diff --git a/packages/jam/transition/accumulate/worker/worker.ts b/packages/jam/transition/accumulate/worker/worker.ts index 605d58fa3..916ce2d4c 100644 --- a/packages/jam/transition/accumulate/worker/worker.ts +++ b/packages/jam/transition/accumulate/worker/worker.ts @@ -25,7 +25,7 @@ import { AccumulateExternalities } from "../../externalities/accumulate-external import { AccumulateFetchExternalities } from "../../externalities/accumulate-fetch-externalities.js"; import { generateNextServiceId } from "../accumulate-utils.js"; import type { Operand } from "../operand.js"; -import { type AccumulateRequest, type AccumulateResponse, type GetServiceResponse, MessageType } from "./protocol.js"; +import { type AccumulateRequest, type AccumulateResponse, type GetServiceResponse, MSG_ACCUMULATE_REQUEST, MSG_ACCUMULATE_RESPONSE, MSG_GET_SERVICE_REQUEST, MSG_GET_SERVICE_RESPONSE } from "./protocol.js"; import { deserializeAccumulationStateUpdate, deserializeOperand, @@ -74,7 +74,7 @@ function getServiceAsync(serviceId: ServiceId): Promise { serviceCache.set(serviceId, service); resolve(service); }; - dataPort.postMessage({ type: MessageType.GetServiceRequest, serviceId }); + dataPort.postMessage({ type: MSG_GET_SERVICE_REQUEST, serviceId }); }); } @@ -119,7 +119,7 @@ async function runAccumulation(request: AccumulateRequest): Promise { - if ((msg as GetServiceResponse).type === MessageType.GetServiceResponse) { + if ((msg as GetServiceResponse).type === MSG_GET_SERVICE_RESPONSE) { handleGetServiceResponse(msg as GetServiceResponse); return; } - if ((msg as AccumulateRequest).type !== MessageType.AccumulateRequest) { + if ((msg as AccumulateRequest).type !== MSG_ACCUMULATE_REQUEST) { return; } @@ -241,7 +242,7 @@ dataPort.on("message", (msg: AccumulateRequest | GetServiceResponse) => { .catch((err) => { const error = err instanceof Error ? err.message : String(err); dataPort.postMessage({ - type: MessageType.AccumulateResponse, + type: MSG_ACCUMULATE_RESPONSE, consumedGas: tryAsServiceGas(0n), stateUpdate: null, error, From f21533aea558fa7a86af2c60aecda3d5f8541f5b Mon Sep 17 00:00:00 2001 From: Mateusz Sikora Date: Sun, 29 Mar 2026 13:18:08 +0200 Subject: [PATCH 11/11] qa-fix --- packages/jam/transition/accumulate/worker/pool.ts | 8 +++++++- packages/jam/transition/accumulate/worker/worker.ts | 10 +++++++++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/packages/jam/transition/accumulate/worker/pool.ts b/packages/jam/transition/accumulate/worker/pool.ts index 0053470d8..dd48f5869 100644 --- a/packages/jam/transition/accumulate/worker/pool.ts +++ b/packages/jam/transition/accumulate/worker/pool.ts @@ -8,7 +8,13 @@ import { MessageChannel, type MessagePort, Worker } from "node:worker_threads"; import type { ServiceId } from "@typeberry/block"; import { Logger } from "@typeberry/logger"; import type { Service } from "@typeberry/state"; -import { type AccumulateRequest, type AccumulateResponse, type GetServiceRequest, MSG_GET_SERVICE_REQUEST, MSG_GET_SERVICE_RESPONSE } from "./protocol.js"; +import { + type AccumulateRequest, + type AccumulateResponse, + type GetServiceRequest, + MSG_GET_SERVICE_REQUEST, + MSG_GET_SERVICE_RESPONSE, +} from "./protocol.js"; import { type PlainService, serializeService } from "./serialization.js"; const logger = Logger.new(import.meta.filename, "worker-pool"); diff --git a/packages/jam/transition/accumulate/worker/worker.ts b/packages/jam/transition/accumulate/worker/worker.ts index 916ce2d4c..146b5c797 100644 --- a/packages/jam/transition/accumulate/worker/worker.ts +++ b/packages/jam/transition/accumulate/worker/worker.ts @@ -25,7 +25,15 @@ import { AccumulateExternalities } from "../../externalities/accumulate-external import { AccumulateFetchExternalities } from "../../externalities/accumulate-fetch-externalities.js"; import { generateNextServiceId } from "../accumulate-utils.js"; import type { Operand } from "../operand.js"; -import { type AccumulateRequest, type AccumulateResponse, type GetServiceResponse, MSG_ACCUMULATE_REQUEST, MSG_ACCUMULATE_RESPONSE, MSG_GET_SERVICE_REQUEST, MSG_GET_SERVICE_RESPONSE } from "./protocol.js"; +import { + type AccumulateRequest, + type AccumulateResponse, + type GetServiceResponse, + MSG_ACCUMULATE_REQUEST, + MSG_ACCUMULATE_RESPONSE, + MSG_GET_SERVICE_REQUEST, + MSG_GET_SERVICE_RESPONSE, +} from "./protocol.js"; import { deserializeAccumulationStateUpdate, deserializeOperand,