Implement v2 tree and inclusion proof format

README.md

@@ -310,47 +310,63 @@ type CertificationData struct {
 - `INVALID_TRANSACTION_HASH_FORMAT` - TransactionHash not in proper DataHash imprint format
 - `INVALID_SHARD` - The certification request was sent to the wrong shard
 
-#### `get_inclusion_proof`
-Retrieve the Sparse Merkle Tree inclusion proof for a specific state transition request.
+#### `get_inclusion_proof.v2`
+Retrieve the v2 inclusion proof for a submitted certification request.
+
+The `stateId` must be exactly 64 hex characters (32 raw bytes).
 
 **Request:**
 ```json
 {
   "jsonrpc": "2.0",
-  "method": "get_inclusion_proof",
+  "method": "get_inclusion_proof.v2",
   "params": {
-    "stateId": "0000c7aa6962316c0eeb1469dc3d7793e39e140c005e6eea0e188dcc73035d765937"
+    "stateId": "c7aa6962316c0eeb1469dc3d7793e39e140c005e6eea0e188dcc73035d765937"
   },
   "id": 2
 }
 ```
 
 **Response:**
+
 ```json
 {
-  "jsonrpc":"2.0",
-  "result":{
-    "certificationData":{
-      "publicKey":"027c4fdf89e8138b360397a7285ca99b863499d26f3c1652251fcf680f4d64882c",
-      "signature":"65ed0261e093aa2df02c0e8fb0aa46144e053ea705ce7053023745b3626c60550b2a5e90eacb93416df116af96872547608a31de1f8ef25dc5a79104e6b69c8d00",
-      "sourceStateHash":"0000539cb40d7450fa842ac13f4ea50a17e56c5b1ee544257d46b6ec8bb48a63e647",
-      "transactionHash":"0000c5f9a1f02e6475c599449250bb741b49bd8858afe8a42059ac1522bff47c6297"
-    },
-    "merkleTreePath":{
-      "root":"0000342d44bb4f43b2de5661cf3690254b95b49e46820b90f13fbe2798f428459ba4",
-      "steps":[
-        {
-          "branch":["0000f00a106493f8bee8846b84325fe71411ea01b8a7c5d7cc0853888b1ef9cbf83b"],
-          "path":"7588619140208316325429861720569170962648734570557434545963804239233978322458521890",
-          "sibling":null
-        }
-      ]
-    }
-  },
-  "id":2
+  "jsonrpc": "2.0",
+  "result": "<hex-encoded CBOR>",
+  "id": 2
 }
 ```
 
+The `result` field is a hex-encoded CBOR array:
+```
+[blockNumber, [certificationData, certificateBytes, unicityCertificate]]
+```
+
+- `certificationData` is the certification data for inclusion proofs, or `null` for non-inclusion proofs.
+- For inclusion proofs, `certificateBytes` is the binary inclusion certificate: `bitmap[32] || sibling_1[32] || ... || sibling_n[32]`, where `n = popcount(bitmap)`. Siblings are in root-to-leaf order. For non-inclusion proofs, `certificateBytes` is an exclusion certificate: `k_l[32] || h_l[32] || bitmap[32] || siblings...` (exclusion proof generation is not yet implemented).
+- The expected SMT root is always taken from `UC.IR.h` (input record hash of the Unicity Certificate). No root field appears in the certificate itself.
+
+**Hash rules (Yellowpaper-aligned):**
+- Leaf: `H(0x00 || key || value)` where value is the raw transaction hash bytes
+- Inner node (two children): `H(0x01 || depth_byte || left || right)`
+- Inner node (one child): passthrough (child hash unchanged)
+
+**Key encoding:** 32 bytes, LSB-first bit addressing. `bit(key, d) = (key[d/8] >> (d%8)) & 1`.
+
+**Verification pseudocode:**
+```
+h = H(0x00 || key || value)
+j = len(siblings)
+for d in 255..=0:
+    if bitmap bit d is not set: continue
+    j -= 1
+    if bit(key, d) == 1:
+        h = H(0x01 || d || siblings[j] || h)
+    else:
+        h = H(0x01 || d || h || siblings[j])
+assert j == 0 and h == UC.IR.h
+```
+
 #### `get_block_height`
 Retrieve the current blockchain height.
 

cmd/aggregator/main.go

@@ -168,9 +168,9 @@ func main() {
 	var smtInstance *smt.SparseMerkleTree
 	switch cfg.Sharding.Mode {
 	case config.ShardingModeStandalone:
-		smtInstance = smt.NewSparseMerkleTree(api.SHA256, 16+256)
+		smtInstance = smt.NewSparseMerkleTree(api.SHA256, api.StateTreeKeyLengthBits)
 	case config.ShardingModeChild:
-		smtInstance = smt.NewChildSparseMerkleTree(api.SHA256, 16+256, cfg.Sharding.Child.ShardID)
+		smtInstance = smt.NewChildSparseMerkleTree(api.SHA256, api.StateTreeKeyLengthBits, cfg.Sharding.Child.ShardID)
 	case config.ShardingModeParent:
 		smtInstance = smt.NewParentSparseMerkleTree(api.SHA256, cfg.Sharding.ShardIDLength)
 	default:

cmd/commitment/main.go

@@ -9,7 +9,6 @@ import (
 	"flag"
 	"fmt"
 	"log"
-	"math/big"
 	"net/http"
 	"os"
 	"time"
@@ -67,138 +66,101 @@ func main() {
 		},
 	}
 
-	commitReq := generateCommitmentRequest()
+	certReq := generateCertificationRequest()
 	if *flagVerbose {
-		if payload, err := json.MarshalIndent(commitReq, "", "  "); err == nil {
-			logger.Printf("submit_commitment request:\n%s", payload)
+		if payload, err := json.MarshalIndent(certReq, "", "  "); err == nil {
+			logger.Printf("certification_request request:\n%s", payload)
 		}
 	}
 
-	logger.Printf("Submitting commitment to URL: %s", *flagURL)
+	logger.Printf("Submitting certification request to URL: %s", *flagURL)
 
-	submitResp, err := callJSONRPC(ctx, client, *flagURL, *flagAuth, "submit_commitment", commitReq)
+	submitResp, err := callJSONRPC(ctx, client, *flagURL, *flagAuth, "certification_request", certReq)
 	if err != nil {
-		logger.Fatalf("submit_commitment call failed: %v", err)
+		logger.Fatalf("certification_request call failed: %v", err)
 	}
 
 	if *flagVerbose {
 		if payload, err := json.MarshalIndent(submitResp, "", "  "); err == nil {
-			logger.Printf("submit_commitment response:\n%s", payload)
+			logger.Printf("certification_request response:\n%s", payload)
 		}
 	}
 
-	var submitResult api.SubmitCommitmentResponse
+	var submitResult api.CertificationResponse
 	if submitResp.Error != nil {
-		logger.Fatalf("submit_commitment returned error: %s (code %d)", submitResp.Error.Message, submitResp.Error.Code)
+		logger.Fatalf("certification_request returned error: %s (code %d)", submitResp.Error.Message, submitResp.Error.Code)
 	}
 	if err := json.Unmarshal(submitResp.Result, &submitResult); err != nil {
-		logger.Fatalf("failed to decode submit_commitment result: %v", err)
+		logger.Fatalf("failed to decode certification_request result: %v", err)
 	}
-	if submitResult.Status != "SUCCESS" {
-		logger.Fatalf("submit_commitment status was %q", submitResult.Status)
-	}
-
-	// Verify and display the receipt if present
-	if submitResult.Receipt != nil {
-		logger.Printf("Receipt received:")
-		logger.Printf("  Algorithm: %s", submitResult.Receipt.Algorithm)
-		logger.Printf("  PublicKey: %s", submitResult.Receipt.PublicKey)
-		logger.Printf("  Signature: %s", submitResult.Receipt.Signature)
-		logger.Printf("  Request.Service: %s", submitResult.Receipt.Request.Service)
-		logger.Printf("  Request.Method: %s", submitResult.Receipt.Request.Method)
-		logger.Printf("  Request.RequestID: %s", submitResult.Receipt.Request.RequestID)
-
-		// Verify the receipt signature
-		requestBytes, err := json.Marshal(submitResult.Receipt.Request)
-		if err != nil {
-			logger.Printf("Warning: failed to marshal receipt request for verification: %v", err)
-		} else {
-			signingService := signing.NewSigningService()
-			valid, err := signingService.VerifyWithPublicKey(requestBytes, submitResult.Receipt.Signature, submitResult.Receipt.PublicKey)
-			if err != nil {
-				logger.Printf("Warning: receipt signature verification error: %v", err)
-			} else if valid {
-				logger.Printf("Receipt signature VERIFIED successfully!")
-			} else {
-				logger.Printf("Warning: receipt signature verification FAILED!")
-			}
-		}
-	} else {
-		logger.Printf("No receipt received (receipt was requested: %v)", *commitReq.Receipt)
-	}
-
-	logger.Printf("Commitment %s accepted. Polling for inclusion proof...", commitReq.RequestID)
-
-	path, err := commitReq.RequestID.GetPath()
-	if err != nil {
-		logger.Fatalf("failed to derive SMT path: %v", err)
+	if submitResult.Status != "SUCCESS" && submitResult.Status != "STATE_ID_EXISTS" {
+		logger.Fatalf("certification_request status was %q", submitResult.Status)
 	}
+	logger.Printf("Certification request %s accepted. Polling for inclusion proof...", certReq.StateID)
 
 	submittedAt := time.Now()
-	inclusionProof, verification, attempts, err := waitForInclusionProof(ctx, client, commitReq.RequestID, path, logger)
+	inclusionProof, attempts, err := waitForInclusionProof(ctx, client, certReq, logger)
 	if err != nil {
 		logger.Fatalf("failed to retrieve inclusion proof after %d attempt(s): %v", attempts, err)
 	}
 
 	if *flagVerbose {
 		if payload, err := json.MarshalIndent(inclusionProof, "", "  "); err == nil {
-			logger.Printf("get_inclusion_proof response:\n%s", payload)
+			logger.Printf("get_inclusion_proof.v2 response:\n%s", payload)
 		}
 	}
 
-	logger.Printf("Proof verification result: pathValid=%t pathIncluded=%t overall=%t",
-		verification.PathValid, verification.PathIncluded, verification.Result)
+	if err := inclusionProof.InclusionProof.Verify(certReq); err != nil {
+		logger.Fatalf("proof verification failed: %v", err)
+	}
+	logger.Printf("Proof verified successfully against block %d.", inclusionProof.BlockNumber)
 
 	elapsed := time.Since(submittedAt)
 	logger.Printf("Valid inclusion proof received in %s after %d attempt(s).", elapsed.Round(time.Millisecond), attempts)
 
-	logger.Printf("Commitment %s successfully submitted and verified.", commitReq.RequestID)
+	logger.Printf("Certification request %s successfully submitted and verified.", certReq.StateID)
 }
 
-func generateCommitmentRequest() *api.SubmitCommitmentRequest {
+func generateCertificationRequest() *api.CertificationRequest {
 	privateKey, err := btcec.NewPrivateKey()
 	if err != nil {
 		panic(fmt.Sprintf("failed to generate private key: %v", err))
 	}
 
 	publicKeyBytes := privateKey.PubKey().SerializeCompressed()
+	ownerPredicate := api.NewPayToPublicKeyPredicate(publicKeyBytes)
 
 	stateData := make([]byte, 32)
 	if _, err := rand.Read(stateData); err != nil {
 		panic(fmt.Sprintf("failed to read random state bytes: %v", err))
 	}
 
-	stateHashImprint := signing.CreateDataHash(stateData)
+	sourceStateHash := signing.CreateDataHash(stateData)
 
-	requestID, err := api.CreateRequestID(publicKeyBytes, stateHashImprint)
+	stateID, err := api.CreateStateID(ownerPredicate, sourceStateHash)
 	if err != nil {
-		panic(fmt.Sprintf("failed to create request ID: %v", err))
+		panic(fmt.Sprintf("failed to create state ID: %v", err))
 	}
 
 	transactionData := make([]byte, 32)
 	if _, err := rand.Read(transactionData); err != nil {
 		panic(fmt.Sprintf("failed to read random transaction bytes: %v", err))
 	}
 
-	transactionHashImprint := signing.CreateDataHash(transactionData)
-	transactionHashBytes := transactionHashImprint.DataBytes()
-
-	signature, err := signing.NewSigningService().SignHash(transactionHashBytes, privateKey.Serialize())
-	if err != nil {
-		panic(fmt.Sprintf("failed to sign transaction hash: %v", err))
+	transactionHash := signing.CreateDataHash(transactionData)
+	certData := api.CertificationData{
+		OwnerPredicate:  ownerPredicate,
+		SourceStateHash: sourceStateHash,
+		TransactionHash: transactionHash,
+	}
+	if err := signing.NewSigningService().SignCertData(&certData, privateKey.Serialize()); err != nil {
+		panic(fmt.Sprintf("failed to sign certification data: %v", err))
 	}
 
-	receipt := true
-	return &api.SubmitCommitmentRequest{
-		RequestID:       requestID,
-		TransactionHash: transactionHashImprint,
-		Authenticator: api.Authenticator{
-			Algorithm: "secp256k1",
-			PublicKey: publicKeyBytes,
-			Signature: signature,
-			StateHash: stateHashImprint,
-		},
-		Receipt: &receipt,
+	return &api.CertificationRequest{
+		StateID:               stateID,
+		CertificationData:     certData,
+		AggregateRequestCount: 1,
 	}
 }
 
@@ -219,10 +181,9 @@ func callJSONRPC(ctx context.Context, client *http.Client, url, authHeader, meth
 	}
 
 	req.Header.Set("Content-Type", "application/json")
-	/*if authHeader != "" {
-		req.Header.Set("Authorization", "supersecret")
-	}*/
-	req.Header.Set("Authorization", "Bearer supersecret")
+	if authHeader != "" {
+		req.Header.Set("Authorization", authHeader)
+	}
 
 	resp, err := client.Do(req)
 	if err != nil {
@@ -238,7 +199,7 @@ func callJSONRPC(ctx context.Context, client *http.Client, url, authHeader, meth
 	return &rpcResp, nil
 }
 
-func waitForInclusionProof(ctx context.Context, client *http.Client, requestID api.RequestID, requestPath *big.Int, logger *log.Logger) (*api.GetInclusionProofResponseV2, *api.PathVerificationResult, int, error) {
+func waitForInclusionProof(ctx context.Context, client *http.Client, req *api.CertificationRequest, logger *log.Logger) (*api.GetInclusionProofResponseV2, int, error) {
 	deadline, ok := ctx.Deadline()
 	if !ok {
 		deadline = time.Now().Add(45 * time.Second)
@@ -249,65 +210,46 @@ func waitForInclusionProof(ctx context.Context, client *http.Client, requestID a
 		attempts++
 		select {
 		case <-ctx.Done():
-			return nil, nil, attempts, ctx.Err()
+			return nil, attempts, ctx.Err()
 		default:
 		}
 
-		proofResp, err := callJSONRPC(ctx, client, *flagURL, *flagAuth, "get_inclusion_proof", api.GetInclusionProofRequestV2{
-			StateID: requestID,
+		proofResp, err := callJSONRPC(ctx, client, *flagURL, *flagAuth, "get_inclusion_proof.v2", api.GetInclusionProofRequestV2{
+			StateID: req.StateID,
 		})
 		if err != nil {
-			logger.Printf("get_inclusion_proof attempt %d failed: %v", attempts, err)
+			logger.Printf("get_inclusion_proof.v2 attempt %d failed: %v", attempts, err)
 			time.Sleep(*flagPollInterval)
 			continue
 		}
 
 		if proofResp.Error != nil {
-			logger.Printf("get_inclusion_proof attempt %d returned error: %s (code %d)", attempts, proofResp.Error.Message, proofResp.Error.Code)
+			logger.Printf("get_inclusion_proof.v2 attempt %d returned error: %s (code %d)", attempts, proofResp.Error.Message, proofResp.Error.Code)
 			time.Sleep(*flagPollInterval)
 			continue
 		}
 
 		var payload api.GetInclusionProofResponseV2
 		if err := json.Unmarshal(proofResp.Result, &payload); err != nil {
-			logger.Printf("get_inclusion_proof attempt %d decode error: %v", attempts, err)
+			logger.Printf("get_inclusion_proof.v2 attempt %d decode error: %v", attempts, err)
 			time.Sleep(*flagPollInterval)
 			continue
 		}
 
-		if payload.InclusionProof == nil || payload.InclusionProof.MerkleTreePath == nil {
-			logger.Printf("get_inclusion_proof attempt %d: proof payload incomplete, retrying...", attempts)
+		if payload.InclusionProof == nil || len(payload.InclusionProof.UnicityCertificate) == 0 {
+			logger.Printf("get_inclusion_proof.v2 attempt %d: proof payload incomplete, retrying...", attempts)
 			time.Sleep(*flagPollInterval)
 			continue
 		}
 
-		result, err := verifyProof(&payload, requestPath)
-		if err != nil {
-			logger.Printf("get_inclusion_proof attempt %d verification error: %v", attempts, err)
+		if err := payload.InclusionProof.Verify(req); err != nil {
+			logger.Printf("get_inclusion_proof.v2 attempt %d verification error: %v", attempts, err)
 			time.Sleep(*flagPollInterval)
 			continue
 		}
 
-		if result.PathIncluded {
-			return &payload, result, attempts, nil
-		}
-
-		logger.Printf("get_inclusion_proof attempt %d: proof returned but path not included yet (pathValid=%t). Waiting...",
-			attempts, result.PathValid)
-		time.Sleep(*flagPollInterval)
-	}
-
-	return nil, nil, attempts, fmt.Errorf("timed out waiting for inclusion proof for request %s", requestID)
-}
-
-func verifyProof(resp *api.GetInclusionProofResponseV2, path *big.Int) (*api.PathVerificationResult, error) {
-	if resp == nil || resp.InclusionProof == nil {
-		return nil, fmt.Errorf("inclusion proof payload was empty")
-	}
-
-	if resp.InclusionProof.MerkleTreePath == nil {
-		return nil, fmt.Errorf("merkle tree path missing from inclusion proof")
+		return &payload, attempts, nil
 	}
 
-	return resp.InclusionProof.MerkleTreePath.Verify(path)
+	return nil, attempts, fmt.Errorf("timed out waiting for inclusion proof for state ID %s", req.StateID)
 }