Implement sharded v2 inclusion proof composition

README.md

@@ -366,7 +366,6 @@ for d in 255..=0:
         h = H(0x01 || d || h || siblings[j])
 assert j == 0 and h == UC.IR.h
 ```
-```
 
 #### `get_block_height`
 Retrieve the current blockchain height.

cmd/performance-test/main.go

@@ -5,12 +5,10 @@ import (
 	"bytes"
 	"context"
 	"crypto/rand"
-	"encoding/hex"
 	"encoding/json"
 	"fmt"
 	"log"
 	"math"
-	"math/big"
 	"os"
 	"path/filepath"
 	"runtime"
@@ -198,32 +196,19 @@ func selectShardIndex(requestID api.StateID, shardClients []*ShardClient) int {
 	if len(imprint) == 0 {
 		return 0
 	}
-	return int(imprint[len(imprint)-1]) % shardCount
+	keyBytes := requestID.DataBytes()
+	if len(keyBytes) == 0 {
+		return 0
+	}
+	// Fallback only: keep distribution aligned with LSB-first key layout.
+	return int(keyBytes[0]) % shardCount
 }
 
 func matchesShardMask(requestIDHex string, shardMask int) (bool, error) {
 	if shardMask <= 0 {
 		return false, nil
 	}
-
-	bytes, err := hex.DecodeString(requestIDHex)
-	if err != nil {
-		return false, fmt.Errorf("failed to decode request ID: %w", err)
-	}
-
-	requestBig := new(big.Int).SetBytes(bytes)
-	maskBig := new(big.Int).SetInt64(int64(shardMask))
-
-	msbPos := maskBig.BitLen() - 1
-	if msbPos < 0 {
-		return false, fmt.Errorf("invalid shard mask: %d", shardMask)
-	}
-
-	compareMask := new(big.Int).Sub(new(big.Int).Lsh(big.NewInt(1), uint(msbPos)), big.NewInt(1))
-	expected := new(big.Int).And(maskBig, compareMask)
-	requestLowBits := new(big.Int).And(requestBig, compareMask)
-
-	return requestLowBits.Cmp(expected) == 0, nil
+	return api.MatchesShardPrefixFromHex(requestIDHex, shardMask)
 }
 
 // matchesAnyShardTarget checks if a request ID matches any of the configured shard targets
@@ -409,7 +394,7 @@ func commitmentWorker(ctx context.Context, shardClients []*ShardClient, metrics
 					if proofQueue != nil {
 						metrics.recordSubmissionTimestamp(requestIDStr)
 						select {
-						case proofQueue <- proofJob{shardIdx: shardIdx, requestID: requestIDStr}:
+						case proofQueue <- proofJob{shardIdx: shardIdx, request: req}:
 						default:
 							// Queue full, skip proof verification for this one
 						}
@@ -435,10 +420,21 @@ func proofVerificationWorker(ctx context.Context, shardClients []*ShardClient, m
 		case <-ctx.Done():
 			return
 		case job := <-proofQueue:
-			go func(reqID string, shardIdx int) {
+			go func(job proofJob) {
+				if job.request == nil {
+					metrics.recordError("Missing original request for proof verification")
+					atomic.AddInt64(&metrics.proofVerifyFailed, 1)
+					if sm := metrics.shard(job.shardIdx); sm != nil {
+						sm.proofVerifyFailed.Add(1)
+					}
+					return
+				}
+
+				reqID := normalizeRequestID(job.request.StateID.String())
+				shardIdx := job.shardIdx
 				time.Sleep(proofInitialDelay)
 				startTime := time.Now()
-				normalizedID := normalizeRequestID(reqID)
+				normalizedID := reqID
 				client := shardClients[shardIdx].proofClient // Use separate proof client pool
 
 				for attempt := 0; attempt < proofMaxRetries; attempt++ {
@@ -534,16 +530,12 @@ func proofVerificationWorker(ctx context.Context, shardClients []*ShardClient, m
 					}
 					metrics.addProofLatency(totalLatency)
 
-					// TODO: Wire api.InclusionProofV2.Verify(*CertificationRequest)
-					// verification here. For now we only check that the response carries a
-					// non-empty inclusion cert; perf tests care about throughput, not
-					// cryptographic verification correctness.
-					if len(proofResp.InclusionProof.CertificateBytes) == 0 {
+					if err := proofResp.InclusionProof.Verify(job.request); err != nil {
 						if attempt < proofMaxRetries-1 {
 							time.Sleep(proofRetryDelay)
 							continue
 						}
-						metrics.recordError(fmt.Sprintf("Empty certificate bytes for request %s", reqID))
+						metrics.recordError(fmt.Sprintf("Proof verification failed for request %s: %v", reqID, err))
 						atomic.AddInt64(&metrics.proofVerifyFailed, 1)
 						if sm := metrics.shard(shardIdx); sm != nil {
 							sm.proofVerifyFailed.Add(1)
@@ -563,7 +555,7 @@ func proofVerificationWorker(ctx context.Context, shardClients []*ShardClient, m
 				if sm := metrics.shard(shardIdx); sm != nil {
 					sm.proofFailed.Add(1)
 				}
-			}(job.requestID, job.shardIdx)
+			}(job)
 		}
 	}
 }

cmd/performance-test/types.go

@@ -17,6 +17,7 @@ import (
 	"sync/atomic"
 	"time"
 
+	"github.com/unicitynetwork/aggregator-go/pkg/api"
 	"golang.org/x/net/http2"
 )
 
@@ -109,8 +110,8 @@ type ShardClient struct {
 }
 
 type proofJob struct {
-	shardIdx  int
-	requestID string
+	shardIdx int
+	request  *api.CertificationRequest
 }
 
 // RequestRateCounters tracks per-second client-side request activity.

internal/bft/client_stub_test.go

@@ -47,7 +47,6 @@ func TestBFTClientStub_CertificationRequest_PopulatesSyntheticUC(t *testing.T) {
 		api.HexBytes("0123"),
 		nil,
 		nil,
-		nil,
 	)
 
 	err = client.CertificationRequest(t.Context(), block)

internal/ha/block_syncer_test.go

@@ -122,7 +122,7 @@ func createBlock(t *testing.T, storage *mongodb.Storage, blockNum int64) api.Hex
 	rootHash := api.HexBytes(tmpSMT.GetRootHashRaw())
 
 	// persist block
-	block := models.NewBlock(blockNumber, "unicity", 0, "1.0", "mainnet", rootHash, nil, nil, nil)
+	block := models.NewBlock(blockNumber, "unicity", 0, "1.0", "mainnet", rootHash, nil, nil)
 	block.Finalized = true // Mark as finalized so GetLatestNumber finds it
 	err = storage.BlockStorage().Store(ctx, block)
 	require.NoError(t, err)

internal/models/block.go

@@ -1,7 +1,6 @@
 package models
 
 import (
-	"encoding/json"
 	"fmt"
 	"time"
 
@@ -12,18 +11,19 @@ import (
 
 // Block represents a blockchain block
 type Block struct {
-	Index                *api.BigInt         `json:"index"`
-	ChainID              string              `json:"chainId"`
-	ShardID              api.ShardID         `json:"shardId"`
-	Version              string              `json:"version"`
-	ForkID               string              `json:"forkId"`
-	RootHash             api.HexBytes        `json:"rootHash"`
-	PreviousBlockHash    api.HexBytes        `json:"previousBlockHash"`
-	NoDeletionProofHash  api.HexBytes        `json:"noDeletionProofHash"`
-	CreatedAt            *api.Timestamp      `json:"createdAt"`
-	UnicityCertificate   api.HexBytes        `json:"unicityCertificate"`
-	ParentMerkleTreePath *api.MerkleTreePath `json:"parentMerkleTreePath,omitempty"` // child mode only
-	Finalized            bool                `json:"finalized"`                      // true when all data is persisted
+	Index               *api.BigInt                  `json:"index"`
+	ChainID             string                       `json:"chainId"`
+	ShardID             api.ShardID                  `json:"shardId"`
+	Version             string                       `json:"version"`
+	ForkID              string                       `json:"forkId"`
+	RootHash            api.HexBytes                 `json:"rootHash"`
+	PreviousBlockHash   api.HexBytes                 `json:"previousBlockHash"`
+	NoDeletionProofHash api.HexBytes                 `json:"noDeletionProofHash"`
+	CreatedAt           *api.Timestamp               `json:"createdAt"`
+	UnicityCertificate  api.HexBytes                 `json:"unicityCertificate"`
+	ParentFragment      *api.ParentInclusionFragment `json:"parentFragment,omitempty"`    // child mode only
+	ParentBlockNumber   uint64                       `json:"parentBlockNumber,omitempty"` // child mode only
+	Finalized           bool                         `json:"finalized"`                   // true when all data is persisted
 }
 
 // BlockBSON represents the BSON version of Block for MongoDB storage
@@ -38,23 +38,29 @@ type BlockBSON struct {
 	NoDeletionProofHash string               `bson:"noDeletionProofHash,omitempty"`
 	CreatedAt           time.Time            `bson:"createdAt"`
 	UnicityCertificate  string               `bson:"unicityCertificate"`
-	MerkleTreePath      string               `bson:"merkleTreePath,omitempty"` // child mode only
+	ParentFragment      *ParentFragmentBSON  `bson:"parentFragment,omitempty"` // child mode only
+	ParentBlockNumber   uint64               `bson:"parentBlockNumber,omitempty"`
 	Finalized           bool                 `bson:"finalized"`
 }
 
+// ParentFragmentBSON is the BSON representation of ParentInclusionFragment.
+type ParentFragmentBSON struct {
+	CertificateBytes []byte `bson:"certificateBytes"`
+	ShardLeafValue   []byte `bson:"shardLeafValue"`
+}
+
 // ToBSON converts Block to BlockBSON for MongoDB storage
 func (b *Block) ToBSON() (*BlockBSON, error) {
 	indexDecimal, err := primitive.ParseDecimal128(b.Index.String())
 	if err != nil {
 		return nil, fmt.Errorf("error converting block index to decimal-128: %w", err)
 	}
-	var merkleTreePath string
-	if b.ParentMerkleTreePath != nil {
-		merkleTreePathJson, err := json.Marshal(b.ParentMerkleTreePath)
-		if err != nil {
-			return nil, fmt.Errorf("failed to marshal parent merkle tree path: %w", err)
+	var parentFragment *ParentFragmentBSON
+	if b.ParentFragment != nil {
+		parentFragment = &ParentFragmentBSON{
+			CertificateBytes: append([]byte(nil), b.ParentFragment.CertificateBytes...),
+			ShardLeafValue:   append([]byte(nil), b.ParentFragment.ShardLeafValue...),
 		}
-		merkleTreePath = api.NewHexBytes(merkleTreePathJson).String()
 	}
 	return &BlockBSON{
 		Index:               indexDecimal,
@@ -67,7 +73,8 @@ func (b *Block) ToBSON() (*BlockBSON, error) {
 		NoDeletionProofHash: b.NoDeletionProofHash.String(),
 		CreatedAt:           b.CreatedAt.Time,
 		UnicityCertificate:  b.UnicityCertificate.String(),
-		MerkleTreePath:      merkleTreePath,
+		ParentFragment:      parentFragment,
+		ParentBlockNumber:   b.ParentBlockNumber,
 		Finalized:           b.Finalized,
 	}, nil
 }
@@ -94,15 +101,11 @@ func (bb *BlockBSON) FromBSON() (*Block, error) {
 		return nil, fmt.Errorf("failed to parse unicityCertificate: %w", err)
 	}
 
-	var parentMerkleTreePath *api.MerkleTreePath
-	if bb.MerkleTreePath != "" {
-		hexBytes, err := api.NewHexBytesFromString(bb.MerkleTreePath)
-		if err != nil {
-			return nil, fmt.Errorf("failed to parse parentMerkleTreePath: %w", err)
-		}
-		parentMerkleTreePath = &api.MerkleTreePath{}
-		if err := json.Unmarshal(hexBytes, parentMerkleTreePath); err != nil {
-			return nil, fmt.Errorf("failed to parse parentMerkleTreePath: %w", err)
+	var parentFragment *api.ParentInclusionFragment
+	if bb.ParentFragment != nil {
+		parentFragment = &api.ParentInclusionFragment{
+			CertificateBytes: append([]byte(nil), bb.ParentFragment.CertificateBytes...),
+			ShardLeafValue:   append([]byte(nil), bb.ParentFragment.ShardLeafValue...),
 		}
 	}
 
@@ -112,33 +115,41 @@ func (bb *BlockBSON) FromBSON() (*Block, error) {
 	}
 
 	return &Block{
-		Index:                index,
-		ChainID:              bb.ChainID,
-		ShardID:              bb.ShardID,
-		Version:              bb.Version,
-		ForkID:               bb.ForkID,
-		RootHash:             rootHash,
-		PreviousBlockHash:    previousBlockHash,
-		NoDeletionProofHash:  noDeletionProofHash,
-		CreatedAt:            api.NewTimestamp(bb.CreatedAt),
-		UnicityCertificate:   unicityCertificate,
-		ParentMerkleTreePath: parentMerkleTreePath,
-		Finalized:            bb.Finalized,
+		Index:               index,
+		ChainID:             bb.ChainID,
+		ShardID:             bb.ShardID,
+		Version:             bb.Version,
+		ForkID:              bb.ForkID,
+		RootHash:            rootHash,
+		PreviousBlockHash:   previousBlockHash,
+		NoDeletionProofHash: noDeletionProofHash,
+		CreatedAt:           api.NewTimestamp(bb.CreatedAt),
+		UnicityCertificate:  unicityCertificate,
+		ParentFragment:      parentFragment,
+		ParentBlockNumber:   bb.ParentBlockNumber,
+		Finalized:           bb.Finalized,
 	}, nil
 }
 
 // NewBlock creates a new block
-func NewBlock(index *api.BigInt, chainID string, shardID api.ShardID, version, forkID string, rootHash, previousBlockHash, uc api.HexBytes, parentMerkleTreePath *api.MerkleTreePath) *Block {
+func NewBlock(index *api.BigInt, chainID string, shardID api.ShardID, version, forkID string, rootHash, previousBlockHash, uc api.HexBytes) *Block {
 	return &Block{
-		Index:                index,
-		ChainID:              chainID,
-		ShardID:              shardID,
-		Version:              version,
-		ForkID:               forkID,
-		RootHash:             rootHash,
-		PreviousBlockHash:    previousBlockHash,
-		CreatedAt:            api.Now(),
-		UnicityCertificate:   uc,
-		ParentMerkleTreePath: parentMerkleTreePath,
+		Index:              index,
+		ChainID:            chainID,
+		ShardID:            shardID,
+		Version:            version,
+		ForkID:             forkID,
+		RootHash:           rootHash,
+		PreviousBlockHash:  previousBlockHash,
+		CreatedAt:          api.Now(),
+		UnicityCertificate: uc,
 	}
 }
+
+// NewChildBlock creates a block for child mode with required parent proof metadata.
+func NewChildBlock(index *api.BigInt, chainID string, shardID api.ShardID, version, forkID string, rootHash, previousBlockHash, uc api.HexBytes, parentFragment *api.ParentInclusionFragment, parentBlockNumber uint64) *Block {
+	block := NewBlock(index, chainID, shardID, version, forkID, rootHash, previousBlockHash, uc)
+	block.ParentFragment = parentFragment
+	block.ParentBlockNumber = parentBlockNumber
+	return block
+}

internal/models/block_test.go

@@ -35,8 +35,10 @@ func createTestBlock() *Block {
 		NoDeletionProofHash: randomHash,
 		CreatedAt:           api.Now(),
 		UnicityCertificate:  randomHash,
-		ParentMerkleTreePath: &api.MerkleTreePath{
-			Root: randomHash.String(),
+		ParentFragment: &api.ParentInclusionFragment{
+			CertificateBytes: randomHash,
+			ShardLeafValue:   randomHash,
 		},
+		ParentBlockNumber: 7,
 	}
 }