diff --git a/package-lock.json b/package-lock.json index fa21b1399..c1aea339e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9160,6 +9160,7 @@ "@typeberry/database": "*", "@typeberry/executor": "*", "@typeberry/hash": "*", + "@typeberry/jam-host-calls": "*", "@typeberry/logger": "*", "@typeberry/numbers": "*", "@typeberry/state": "*", diff --git a/packages/jam/block/work-item-segment.ts b/packages/jam/block/work-item-segment.ts index e804dd591..319ed1437 100644 --- a/packages/jam/block/work-item-segment.ts +++ b/packages/jam/block/work-item-segment.ts @@ -2,17 +2,35 @@ import type { Bytes } from "@typeberry/bytes"; import { tryAsU16, type U16 } from "@typeberry/numbers"; import { asOpaqueType, type Opaque } from "@typeberry/utils"; -/** `W_E`: The basic size of erasure-coded pieces in octets. See equation H.6. */ +/** + * `W_E`: The basic size of erasure-coded pieces in octets. See equation H.6. + * https://graypaper.fluffylabs.dev/#/ab2cdbd/449600449700?v=0.7.2 + */ export const W_E = 684; -/** `W_S`: The size of an exported segment in erasure-coded pieces in octets. */ -export const W_S = 6; +/** + * `W_P`: The size of an exported segment in erasure-coded pieces in octets. + * https://graypaper.fluffylabs.dev/#/ab2cdbd/44b10044b200?v=0.7.2 + */ +export const W_P = 6; -/** `W_M`: The maximum number of entries in a work-package manifest. */ -export const MAX_NUMBER_OF_SEGMENTS = 2048; // 2**11 +/** + * `W_M`: The maximum number of imports in a work-package manifest. + * https://graypaper.fluffylabs.dev/#/ab2cdbd/44ad0044ae00?v=0.7.2 + */ +export const MAX_NUMBER_OF_IMPORTS_WP = 3072; -/** `W_E * W_S`: Exported segment size in bytes. */ -export const SEGMENT_BYTES = W_E * W_S; +/** + * `W_X`: The maximum number of exports in a work-package manifest. + * https://graypaper.fluffylabs.dev/#/ab2cdbd/44be0044bf00?v=0.7.2 + */ +export const MAX_NUMBER_OF_EXPORTS_WP = 3072; + +/** + * `W_G = W_E * W_P`: Exported segment size in bytes. + * https://graypaper.fluffylabs.dev/#/ab2cdbd/449a00449b00?v=0.7.2 + */ +export const SEGMENT_BYTES = W_E * W_P; export type SEGMENT_BYTES = typeof SEGMENT_BYTES; /** Exported segment data. */ diff --git a/packages/jam/block/work-item.ts b/packages/jam/block/work-item.ts index 80a542f1b..5908671c6 100644 --- a/packages/jam/block/work-item.ts +++ b/packages/jam/block/work-item.ts @@ -7,7 +7,7 @@ import { type Opaque, WithDebug } from "@typeberry/utils"; import { codecKnownSizeArray } from "./codec-utils.js"; import type { ServiceGas, ServiceId } from "./common.js"; import type { CodeHash } from "./hash.js"; -import { MAX_NUMBER_OF_SEGMENTS, type SegmentIndex } from "./work-item-segment.js"; +import { MAX_NUMBER_OF_IMPORTS_WP, type SegmentIndex } from "./work-item-segment.js"; type WorkItemExtrinsicHash = Opaque; @@ -126,13 +126,17 @@ export class WorkItem extends WithDebug { codeHash: codec.bytes(HASH_SIZE).asOpaque(), refineGasLimit: codec.u64.asOpaque(), accumulateGasLimit: codec.u64.asOpaque(), + // TODO: [MaSo] It should be validated to not exceed W_X + // https://graypaper.fluffylabs.dev/#/ab2cdbd/1a0b011a1c01?v=0.7.2 exportCount: codec.u16, payload: codec.blob, importSegments: codecKnownSizeArray(ImportSpec.Codec, { minLength: 0, - maxLength: MAX_NUMBER_OF_SEGMENTS, - typicalLength: MAX_NUMBER_OF_SEGMENTS, + maxLength: MAX_NUMBER_OF_IMPORTS_WP, + typicalLength: MAX_NUMBER_OF_IMPORTS_WP, }), + // TODO: [MaSo] It should be validated to not exceed T = 128 + // https://graypaper.fluffylabs.dev/#/ab2cdbd/1a0b011a1c01?v=0.7.2 extrinsic: codec.sequenceVarLen(WorkItemExtrinsicSpec.Codec), }); @@ -175,7 +179,7 @@ export class WorkItem extends WithDebug { /** `a`: accumulate execution gas limit */ public readonly accumulateGasLimit: ServiceGas, /** `i`: sequence of imported data segments, which identify a prior exported segment. */ - public readonly importSegments: KnownSizeArray, + public readonly importSegments: KnownSizeArray, /** `x`: sequence of blob hashes and lengths to be introduced in this block */ public readonly extrinsic: WorkItemExtrinsicSpec[], /** `e`: number of data segments exported by this work item. */ diff --git a/packages/jam/in-core/externalities/refine.test.ts b/packages/jam/in-core/externalities/refine.test.ts index ec11deea8..cb30bbad5 100644 --- a/packages/jam/in-core/externalities/refine.test.ts +++ b/packages/jam/in-core/externalities/refine.test.ts @@ -1,14 +1,34 @@ import assert from "node:assert"; import { describe, it } from "node:test"; -import { type PreimageHash, type ServiceId, tryAsServiceGas, tryAsServiceId, tryAsTimeSlot } from "@typeberry/block"; +import { + MAX_NUMBER_OF_EXPORTS_WP, + type PreimageHash, + SEGMENT_BYTES, + type Segment, + type ServiceId, + tryAsServiceGas, + tryAsServiceId, + tryAsTimeSlot, +} from "@typeberry/block"; import { Bytes, BytesBlob } from "@typeberry/bytes"; import { HashDictionary } from "@typeberry/collections"; import { tinyChainSpec } from "@typeberry/config"; import { HASH_SIZE } from "@typeberry/hash"; +import { SegmentExportError } from "@typeberry/jam-host-calls"; import { tryAsU32, tryAsU64 } from "@typeberry/numbers"; import { InMemoryService, InMemoryState, PreimageItem, ServiceAccountInfo, type State } from "@typeberry/state"; import { RefineExternalitiesImpl, type RefineExternalitiesParams } from "./refine.js"; +function createSegment(byte = 0xab): Segment { + return Bytes.fill(SEGMENT_BYTES, byte); +} + +function createSmallSegment(bytes: number[]): Segment { + const data = new Uint8Array(SEGMENT_BYTES); + data.set(bytes); + return Bytes.fromBlob(data, SEGMENT_BYTES); +} + /** * Create a mock State that has specified services with preimages. */ @@ -60,6 +80,7 @@ function createExt(overrides: Partial = {}) { return RefineExternalitiesImpl.create({ currentServiceId: tryAsServiceId(42), lookupState: overrides.lookupState ?? defaultState, + exportOffset: overrides.exportOffset ?? 0, ...overrides, }); } @@ -123,4 +144,72 @@ describe("RefineExternalitiesImpl", () => { assert.strictEqual(r2?.raw[0], 0x02); }); }); + + describe("exportSegment", () => { + it("should export a segment and return its index", () => { + const ext = createExt(); + const segment = createSegment(0x01); + const result = ext.exportSegment(segment); + + assert.strictEqual(result.isOk, true); + assert.strictEqual(result.ok, 0); // first export at offset 0 + assert.strictEqual(ext.getExportedSegments().length, 1); + }); + + it("should return sequential indices for multiple exports", () => { + const ext = createExt(); + + const r1 = ext.exportSegment(createSegment(0x01)); + const r2 = ext.exportSegment(createSegment(0x02)); + const r3 = ext.exportSegment(createSegment(0x03)); + + assert.strictEqual(r1.isOk, true); + assert.strictEqual(r1.ok, 0); + assert.strictEqual(r2.isOk, true); + assert.strictEqual(r2.ok, 1); + assert.strictEqual(r3.isOk, true); + assert.strictEqual(r3.ok, 2); + assert.strictEqual(ext.getExportedSegments().length, 3); + }); + + it("should apply exportOffset to segment indices", () => { + const ext = createExt({ exportOffset: 100 }); + const result = ext.exportSegment(createSegment()); + + assert.strictEqual(result.isOk, true); + assert.strictEqual(result.ok, 100); + }); + + it("should return SegmentExportError when MAX_NUMBER_OF_EXPORTS_WP exceeded", () => { + const ext = createExt({ exportOffset: MAX_NUMBER_OF_EXPORTS_WP }); + const result = ext.exportSegment(createSegment()); + + assert.strictEqual(result.isError, true); + assert.strictEqual(result.error, SegmentExportError); + }); + + it("should return SegmentExportError at exactly MAX_NUMBER_OF_EXPORTS_WP - 1 + 1", () => { + const ext = createExt({ exportOffset: MAX_NUMBER_OF_EXPORTS_WP - 1 }); + + // This one should succeed (index = MAX_NUMBER_OF_EXPORTS_WP - 1) + const r1 = ext.exportSegment(createSegment(0x01)); + assert.strictEqual(r1.isOk, true); + assert.strictEqual(r1.ok, MAX_NUMBER_OF_EXPORTS_WP - 1); + + // This one should fail + const r2 = ext.exportSegment(createSegment(0x02)); + assert.strictEqual(r2.isError, true); + assert.strictEqual(r2.error, SegmentExportError); + }); + + it("should store exact segment data", () => { + const ext = createExt(); + const segment = createSmallSegment([1, 2, 3, 4, 5]); + ext.exportSegment(segment); + + const exported = ext.getExportedSegments(); + assert.strictEqual(exported.length, 1); + assert.deepStrictEqual(exported[0].raw.subarray(0, 5), new Uint8Array([1, 2, 3, 4, 5])); + }); + }); }); diff --git a/packages/jam/in-core/externalities/refine.ts b/packages/jam/in-core/externalities/refine.ts index 452aca46c..d202373fe 100644 --- a/packages/jam/in-core/externalities/refine.ts +++ b/packages/jam/in-core/externalities/refine.ts @@ -1,24 +1,30 @@ -import type { Segment, SegmentIndex, ServiceId } from "@typeberry/block"; +import { + MAX_NUMBER_OF_EXPORTS_WP, + type Segment, + type SegmentIndex, + type ServiceId, + tryAsSegmentIndex, +} from "@typeberry/block"; import type { BytesBlob } from "@typeberry/bytes"; import type { Blake2bHash } from "@typeberry/hash"; -import type { - MachineId, - MachineResult, - MemoryOperation, - NoMachineError, - PagesError, - PeekPokeError, - ProgramCounter, - RefineExternalities, +import { + type MachineId, + type MachineResult, + type MemoryOperation, + type NoMachineError, + type PagesError, + type PeekPokeError, + type ProgramCounter, + type RefineExternalities, SegmentExportError, - ZeroVoidError, + type ZeroVoidError, } from "@typeberry/jam-host-calls"; import type { U64 } from "@typeberry/numbers"; import type { HostCallMemory, HostCallRegisters } from "@typeberry/pvm-host-calls"; import type { BigGas } from "@typeberry/pvm-interface"; import type { ProgramDecoderError } from "@typeberry/pvm-interpreter"; import type { State } from "@typeberry/state"; -import type { OK, Result } from "@typeberry/utils"; +import { type OK, Result } from "@typeberry/utils"; /** * Parameters required to create a RefineExternalitiesImpl. @@ -28,6 +34,8 @@ export type RefineExternalitiesParams = { currentServiceId: ServiceId; /** State at the lookup anchor block, used for historical preimage lookups. */ lookupState: State; + /** Export offset -- sum of exports from prior work items in this package. */ + exportOffset: number; }; export class RefineExternalitiesImpl implements RefineExternalities { @@ -35,6 +43,10 @@ export class RefineExternalitiesImpl implements RefineExternalities { private readonly currentServiceId: ServiceId; /** State at the lookup anchor for preimage lookups. */ private readonly lookupState: State; + /** Segments exported by this work item during refinement. */ + private readonly exportedSegments: Segment[] = []; + /** Offset for segment indexing (sum of exports from prior items). */ + private readonly exportOffset: number; static create(params: RefineExternalitiesParams) { return new RefineExternalitiesImpl(params); @@ -43,6 +55,11 @@ export class RefineExternalitiesImpl implements RefineExternalities { private constructor(params: RefineExternalitiesParams) { this.currentServiceId = params.currentServiceId; this.lookupState = params.lookupState; + this.exportOffset = params.exportOffset; + } + + getExportedSegments(): readonly Segment[] { + return this.exportedSegments; } machineExpunge(_machineIndex: MachineId): Promise> { @@ -98,8 +115,19 @@ export class RefineExternalitiesImpl implements RefineExternalities { throw new Error("Method not implemented."); } - exportSegment(_segment: Segment): Result { - throw new Error("Method not implemented."); + exportSegment(segment: Segment): Result { + // https://graypaper.fluffylabs.dev/#/ab2cdbd/335d03335d03?v=0.7.2 + const currentIndex = this.exportOffset + this.exportedSegments.length; + if (currentIndex >= MAX_NUMBER_OF_EXPORTS_WP) { + return Result.error( + SegmentExportError, + () => + `Maximum number of exported segments exceeded (offset: ${this.exportOffset}, exported: ${this.exportedSegments.length})`, + ); + } + // https://graypaper.fluffylabs.dev/#/ab2cdbd/337303337303?v=0.7.2 + this.exportedSegments.push(segment); + return Result.ok(tryAsSegmentIndex(currentIndex)); } historicalLookup(serviceId: ServiceId | null, hash: Blake2bHash): Promise { diff --git a/packages/jam/in-core/in-core.ts b/packages/jam/in-core/in-core.ts index 594dab7c9..acbde75e6 100644 --- a/packages/jam/in-core/in-core.ts +++ b/packages/jam/in-core/in-core.ts @@ -40,7 +40,7 @@ export type RefineResult = { export type RefineItemResult = { result: WorkResult; - exports: Segment[]; + exports: readonly Segment[]; }; export enum RefineError { @@ -171,13 +171,24 @@ export class InCore { logger.log`[core:${core}] Authorized. Proceeding with work items verification. Anchor=${context.anchor}`; // Verify the work items + let exportOffset = 0; const refineResults: Awaited>[] = []; for (const [idx, item] of items.entries()) { logger.info`[core:${core}][i:${idx}] Refining item for service ${item.service}.`; - refineResults.push( - await this.refineItem(state, lookupState, idx, item, imports, extrinsics, core, workPackageHash), + const result = await this.refineItem( + state, + lookupState, + idx, + item, + imports, + extrinsics, + core, + workPackageHash, + exportOffset, ); + refineResults.push(result); + exportOffset += result.exports.length; } // amalgamate the work report now @@ -268,6 +279,7 @@ export class InCore { allExtrinsics: PerWorkItem, coreIndex: CoreIndex, workPackageHash: WorkPackageHash, + exportOffset: number, ): Promise { const payloadHash = this.blake2b.hashBytes(item.payload); const baseResult = { @@ -311,6 +323,7 @@ export class InCore { extrinsics: allExtrinsics, currentServiceId: item.service, lookupState, + exportOffset, }); const executor = await PvmExecutor.createRefineExecutor(item.service, code, externalities, this.pvmBackend); @@ -325,8 +338,7 @@ export class InCore { const execResult = await executor.run(args, item.refineGasLimit); - // TODO [ToDr] get exports from externalities - const exports: Segment[] = []; + const exports = externalities.refine.getExportedSegments(); if (exports.length !== item.exportCount) { return { exports, @@ -422,6 +434,7 @@ export class InCore { extrinsics: PerWorkItem; currentServiceId: ServiceId; lookupState: State; + exportOffset: number; }): RefineHostCallExternalities { // TODO [ToDr] Pass all required fetch data const fetchExternalities = FetchExternalities.createForRefine( @@ -434,6 +447,7 @@ export class InCore { const refine = RefineExternalitiesImpl.create({ currentServiceId: args.currentServiceId, lookupState: args.lookupState, + exportOffset: args.exportOffset, }); return { diff --git a/packages/jam/in-core/package.json b/packages/jam/in-core/package.json index 73624f265..887a985ce 100644 --- a/packages/jam/in-core/package.json +++ b/packages/jam/in-core/package.json @@ -15,6 +15,7 @@ "@typeberry/database": "*", "@typeberry/executor": "*", "@typeberry/hash": "*", + "@typeberry/jam-host-calls": "*", "@typeberry/logger": "*", "@typeberry/numbers": "*", "@typeberry/state": "*", diff --git a/packages/jam/jam-host-calls/externalities/refine-externalities.test.ts b/packages/jam/jam-host-calls/externalities/refine-externalities.test.ts index 58744303a..d53fc0435 100644 --- a/packages/jam/jam-host-calls/externalities/refine-externalities.test.ts +++ b/packages/jam/jam-host-calls/externalities/refine-externalities.test.ts @@ -60,6 +60,12 @@ export class TestRefineExt implements RefineExternalities { public machineInvokeStatus: MachineStatus = { status: Status.OK }; + private readonly exportSegments: Segment[] = []; + + getExportedSegments(): readonly Segment[] { + return this.exportSegments; + } + machineExpunge(machineIndex: MachineId): Promise> { const val = this.machineExpungeData.get(machineIndex); if (val === undefined) { @@ -151,6 +157,7 @@ export class TestRefineExt implements RefineExternalities { if (result === undefined) { throw new Error(`Unexpected call to exportSegment with: ${segment}`); } + this.exportSegments.push(segment); return result; } diff --git a/packages/jam/jam-host-calls/externalities/refine-externalities.ts b/packages/jam/jam-host-calls/externalities/refine-externalities.ts index c2e62a78b..bc858b467 100644 --- a/packages/jam/jam-host-calls/externalities/refine-externalities.ts +++ b/packages/jam/jam-host-calls/externalities/refine-externalities.ts @@ -107,6 +107,9 @@ export type SegmentExportError = typeof SegmentExportError; /** Host functions external invocations available during refine phase. */ export interface RefineExternalities { + /** Get the segments exported during this work item's refinement. */ + getExportedSegments(): readonly Segment[]; + /** Forget a previously started nested VM. */ machineExpunge(machineIndex: MachineId): Promise>; diff --git a/packages/jam/jam-host-calls/refine/export.test.ts b/packages/jam/jam-host-calls/refine/export.test.ts index 4e782256d..844ce04e4 100644 --- a/packages/jam/jam-host-calls/refine/export.test.ts +++ b/packages/jam/jam-host-calls/refine/export.test.ts @@ -57,6 +57,7 @@ describe("HostCalls: Export", () => { // then assert.deepStrictEqual(result, undefined); assert.deepStrictEqual(registers.get(RESULT_REG), 15n); + assert.strictEqual(refine.getExportedSegments().length, 1); }); it("should zero-pad when exported value is small", async () => { @@ -75,6 +76,7 @@ describe("HostCalls: Export", () => { // then assert.deepStrictEqual(result, undefined); assert.deepStrictEqual(registers.get(RESULT_REG), 5n); + assert.strictEqual(refine.getExportedSegments().length, 1); }); it("should panic if memory is not readable", async () => { @@ -89,6 +91,7 @@ describe("HostCalls: Export", () => { // then assert.deepStrictEqual(result, PvmExecution.Panic); + assert.strictEqual(refine.getExportedSegments().length, 0); }); it("should fail with FULL if export limit is reached", async () => { diff --git a/packages/jam/jam-host-calls/refine/export.ts b/packages/jam/jam-host-calls/refine/export.ts index 5d9321a9d..30f15cd5b 100644 --- a/packages/jam/jam-host-calls/refine/export.ts +++ b/packages/jam/jam-host-calls/refine/export.ts @@ -21,7 +21,7 @@ const IN_OUT_REG = 7; /** * Export a segment to be imported by some future `refine` invokation. * - * https://graypaper.fluffylabs.dev/#/7e6ff6a/341d01341d01?v=0.6.7 + * https://graypaper.fluffylabs.dev/#/ab2cdbd/33db0233db02?v=0.7.2 */ export class Export implements HostCallHandler { index = tryAsHostCallIndex(7); diff --git a/packages/jam/jam-host-calls/refine/historical-lookup.ts b/packages/jam/jam-host-calls/refine/historical-lookup.ts index 3c20c6827..f8d6e045e 100644 --- a/packages/jam/jam-host-calls/refine/historical-lookup.ts +++ b/packages/jam/jam-host-calls/refine/historical-lookup.ts @@ -20,7 +20,7 @@ const IN_OUT_REG = 7; /** * Lookup a historical preimage. * - * https://graypaper.fluffylabs.dev/#/7e6ff6a/343b00343b00?v=0.6.7 + * https://graypaper.fluffylabs.dev/#/ab2cdbd/33c90133c901?v=0.7.2 */ export class HistoricalLookup implements HostCallHandler { index = tryAsHostCallIndex(6);