diff --git a/.gitignore b/.gitignore index 9a830862b..79261ef2b 100644 --- a/.gitignore +++ b/.gitignore @@ -57,3 +57,6 @@ benchmark-runs/ /perfetto_traces/ *.pb *benchmark_results.json +.omc/ +optimize-history/ +redteam-history/ diff --git a/Cargo.lock b/Cargo.lock index fb2cca639..7e070baa4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -65,6 +65,15 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "aho-corasick" +version = "0.7.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc936419f96fa211c1b9166887b38e5e40b19958e5b895be7c1f93adec7071ac" +dependencies = [ + "memchr", +] + [[package]] name = "aho-corasick" version = "1.1.4" @@ -564,6 +573,15 @@ version = "1.0.102" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" +[[package]] +name = "arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" +dependencies = [ + "derive_arbitrary", +] + [[package]] name = "ark-bn254" version = "0.5.0" @@ -1668,6 +1686,17 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "derive_arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e567bd82dcff979e4b03460c307b3cdc9e96fde3d73bed1496d2bc75d9dd62a" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "derive_more" version = "2.1.1" @@ -1884,6 +1913,27 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "enumset" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25b07a8dfbbbfc0064c0a6bdf9edcf966de6b1c33ce344bdeca3b41615452634" +dependencies = [ + "enumset_derive", +] + +[[package]] +name = "enumset_derive" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f43e744e4ea338060faee68ed933e46e722fb7f3617e722a5772d7e856d8b3ce" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "env_filter" version = "1.0.0" @@ -2119,6 +2169,15 @@ dependencies = [ "slab", ] +[[package]] +name = "fxhash" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c" +dependencies = [ + "byteorder", +] + [[package]] name = "generic-array" version = "0.14.7" @@ -2623,7 +2682,7 @@ dependencies = [ "jolt-optimizations", "memory-stats", "num", - "num-derive", + "num-derive 0.4.2", "num-traits", "postcard", "pprof", @@ -2645,6 +2704,42 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "jolt-eval" +version = "0.1.0" +dependencies = [ + "arbitrary", + "ark-bn254", + "clap", + "common", + "criterion", + "enumset", + "eyre", + "jolt-core", + "jolt-eval-macros", + "jolt-inlines-secp256k1", + "jolt-inlines-sha2", + "postcard", + "rand 0.8.5", + "rust-code-analysis", + "schemars 0.8.22", + "serde", + "serde_json", + "tempfile", + "tracer", + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "jolt-eval-macros" +version = "0.1.0" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "jolt-field" version = "0.1.0" @@ -3190,6 +3285,17 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf97ec579c3c42f953ef76dbf8d55ac91fb219dde70e49aa4a6b7d74e9919050" +[[package]] +name = "num-derive" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "876a53fff98e03a936a674b29568b0e605f06b29372c2489ff4de23f1949743d" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "num-derive" version = "0.4.2" @@ -4141,7 +4247,7 @@ version = "1.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" dependencies = [ - "aho-corasick", + "aho-corasick 1.1.4", "memchr", "regex-automata", "regex-syntax", @@ -4153,7 +4259,7 @@ version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" dependencies = [ - "aho-corasick", + "aho-corasick 1.1.4", "memchr", "regex-syntax", ] @@ -4372,6 +4478,35 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "48fd7bd8a6377e15ad9d42a8ec25371b94ddc67abe7c8b9127bec79bebaaae18" +[[package]] +name = "rust-code-analysis" +version = "0.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92a0f85e044428a7b58538f95fa58a157d89d5bcc5b37df6e7024957e52bdc5a" +dependencies = [ + "aho-corasick 0.7.20", + "fxhash", + "lazy_static", + "num", + "num-derive 0.3.3", + "num-format", + "num-traits", + "petgraph", + "regex", + "serde", + "termcolor", + "tree-sitter", + "tree-sitter-ccomment", + "tree-sitter-java", + "tree-sitter-javascript", + "tree-sitter-mozcpp", + "tree-sitter-mozjs", + "tree-sitter-preproc", + "tree-sitter-python", + "tree-sitter-rust", + "tree-sitter-typescript", +] + [[package]] name = "rustc-demangle" version = "0.1.27" @@ -4463,6 +4598,13 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "sandbox-guest" +version = "0.1.0" +dependencies = [ + "jolt-sdk", +] + [[package]] name = "scc" version = "2.4.0" @@ -4472,6 +4614,18 @@ dependencies = [ "sdd", ] +[[package]] +name = "schemars" +version = "0.8.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fbf2ae1b8bc8e02df939598064d22402220cd5bbcca1c76f7d6a310974d5615" +dependencies = [ + "dyn-clone", + "schemars_derive", + "serde", + "serde_json", +] + [[package]] name = "schemars" version = "0.9.0" @@ -4496,6 +4650,18 @@ dependencies = [ "serde_json", ] +[[package]] +name = "schemars_derive" +version = "0.8.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32e265784ad618884abaea0600a9adf15393368d840e0222d101a072f3f7534d" +dependencies = [ + "proc-macro2", + "quote", + "serde_derive_internals", + "syn 2.0.117", +] + [[package]] name = "scopeguard" version = "1.2.0" @@ -4638,6 +4804,17 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "serde_derive_internals" +version = "0.29.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "serde_json" version = "1.0.149" @@ -5158,6 +5335,15 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "termcolor" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" +dependencies = [ + "winapi-util", +] + [[package]] name = "text-block-macros" version = "0.2.0" @@ -5397,6 +5583,118 @@ dependencies = [ "tracing-log", ] +[[package]] +name = "tree-sitter" +version = "0.19.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f41201fed3db3b520405a9c01c61773a250d4c3f43e9861c14b2bb232c981ab" +dependencies = [ + "cc", + "regex", +] + +[[package]] +name = "tree-sitter-ccomment" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3b402bc539927bb457e5ab59aac7260e2c3b97c5fcfc043575788654eedd69a" +dependencies = [ + "cc", + "tree-sitter", +] + +[[package]] +name = "tree-sitter-cpp" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7bd90c7b7db59369ed00fbc40458d9c9b2b8ed145640e337e839ac07aa63e15" +dependencies = [ + "cc", + "tree-sitter", +] + +[[package]] +name = "tree-sitter-java" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "301ae2ee7813e1bf935dc06db947642400645bbea8878431e1b31131488d5430" +dependencies = [ + "cc", + "tree-sitter", +] + +[[package]] +name = "tree-sitter-javascript" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "840bb4d5f3c384cb76b976ff07297f5a24b6e61a708baa4464f53e395caaa5f9" +dependencies = [ + "cc", + "tree-sitter", +] + +[[package]] +name = "tree-sitter-mozcpp" +version = "0.19.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5439f32b7685af19efcd0165d28dab80261e1cc922ed259c9c7909c96ac4cc6" +dependencies = [ + "cc", + "tree-sitter", + "tree-sitter-cpp", +] + +[[package]] +name = "tree-sitter-mozjs" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "def6b21c10157d3d79b912191fa4549008885da827451a62be9f30abeb7319c8" +dependencies = [ + "cc", + "tree-sitter", + "tree-sitter-javascript", +] + +[[package]] +name = "tree-sitter-preproc" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "226b2a77578e83efa7a193919660ffc88c22e357f9c2d9f27b5b11898a8682d3" +dependencies = [ + "cc", + "tree-sitter", +] + +[[package]] +name = "tree-sitter-python" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5646bfe71c4eb1c21b714ce0c38334c311eab767095582859e85da6281e9fd6c" +dependencies = [ + "cc", + "tree-sitter", +] + +[[package]] +name = "tree-sitter-rust" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "784f7ef9cdbd4c895dc2d4bb785e95b4a5364a602eec803681db83d1927ddf15" +dependencies = [ + "cc", + "tree-sitter", +] + +[[package]] +name = "tree-sitter-typescript" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3f62d49c6e56bf291c412ee5e178ea14dff40f14a5f01a8847933f56d65bf3b" +dependencies = [ + "cc", + "tree-sitter", +] + [[package]] name = "twox-hash" version = "2.1.2" diff --git a/Cargo.toml b/Cargo.toml index 935b71d83..1f82bff0f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -93,7 +93,11 @@ members = [ "examples/sig-recovery/guest", "zklean-extractor", "z3-verifier", + "jolt-eval", + "jolt-eval/macros", + "jolt-eval/guest-sandbox", ] +exclude = ["jolt-eval/fuzz"] [features] host = ["jolt-sdk/host"] diff --git a/jolt-eval/Cargo.toml b/jolt-eval/Cargo.toml new file mode 100644 index 000000000..4aa2ff1f7 --- /dev/null +++ b/jolt-eval/Cargo.toml @@ -0,0 +1,70 @@ +[package] +name = "jolt-eval" +version = "0.1.0" +edition = "2021" + +[dependencies] +jolt-core = { workspace = true, features = ["host"] } +common = { workspace = true, features = ["std"] } +tracer = { workspace = true } + +ark-bn254 = { workspace = true } +serde = { workspace = true, features = ["derive"] } +serde_json = { workspace = true, features = ["std"] } +postcard = { workspace = true, features = ["use-std"] } +eyre = { workspace = true } +tracing = { workspace = true } +clap = { workspace = true, features = ["derive"] } +rand = { workspace = true } +tracing-subscriber = { workspace = true } + +arbitrary = { version = "1", features = ["derive"] } +enumset = "1" +schemars = "0.8" +tempfile = "3" + +rust-code-analysis = "0.0.24" + +jolt-inlines-secp256k1 = { workspace = true, features = ["host"] } +jolt-inlines-sha2 = { workspace = true, features = ["host"] } + +jolt-eval-macros = { path = "macros" } + +[dev-dependencies] +criterion = { workspace = true } + +[[bench]] +name = "bind_parallel_high_to_low" +harness = false + +[[bench]] +name = "naive_sort_time" +harness = false + +[[bench]] +name = "bind_parallel_low_to_high" +harness = false + +[[bench]] +name = "prover_time_fibonacci" +harness = false + +[[bench]] +name = "prover_time_secp256k1_ecdsa_verify" +harness = false + +[[bench]] +name = "prover_time_sha2_chain" +harness = false + +[[bin]] +name = "measure-objectives" +path = "bin/measure_objectives.rs" + +[[bin]] +name = "optimize" +path = "bin/optimize.rs" + +[[bin]] +name = "redteam" +path = "bin/redteam.rs" diff --git a/jolt-eval/README.md b/jolt-eval/README.md new file mode 100644 index 000000000..062ebce7c --- /dev/null +++ b/jolt-eval/README.md @@ -0,0 +1,243 @@ +# jolt-eval + +Mechanically checkable **invariants** and **objectives** for the Jolt zkVM. + +The motivation is twofold: +1. **Maximize agent productivity** — give AI agents a way to check their work without a human in the loop. +2. **Minimize the human verification surface** — humans gain assurance about the larger codebase while only focusing on a smaller kernel of invariants and objectives. + +## Concepts + +**Invariants** are evaluations with a binary outcome — things that must always hold. From a single invariant description (a small amount of Rust), the framework can synthesize: +- A `#[test]` (via the `#[invariant(Test, Fuzz)]` macro) +- A `libfuzzer_sys` fuzz target (via the `fuzz_invariant!` macro) +- A "red team" harness for AI agents to try to find a violation + +**Objectives** are measurable properties of the codebase. They come in two flavors: +- **Code quality** (static analysis) — measured via `rust-code-analysis`: LLOC, cognitive complexity, Halstead bugs +- **Performance** (benchmarks) — measured via Criterion: polynomial binding, end-to-end prover time + +**Objective functions** combine one or more objectives into a single scalar that the optimizer minimizes. They are declared as `const` structs with a name, input objectives, and an evaluate function. + +## Built-in invariants + +| Invariant | Targets | Description | +|---|---|---| +| `split_eq_bind_low_high` | Test, Fuzz, RedTeam | `GruenSplitEqPolynomial::bind` (LowToHigh) matches `DensePolynomial::bound_poly_var_bot` | +| `split_eq_bind_high_low` | Test, Fuzz | `GruenSplitEqPolynomial::bind` (HighToLow) matches `DensePolynomial::bound_poly_var_top` | +| `soundness` | RedTeam | For any deterministic guest program + input, only one (output, panic) pair is accepted by the verifier | + +## Built-in objectives + +### Code quality (static analysis) + +| Objective | Units | Description | +|---|---|---| +| `lloc` | lines | Total logical lines of code in `jolt-core/src/` | +| `cognitive_complexity_avg` | — | Average cognitive complexity per function | +| `halstead_bugs` | — | Estimated delivered bugs (Halstead volume / 3000) | + +### Performance (Criterion benchmarks) + +| Benchmark | Description | +|---|---| +| `bind_parallel_low_to_high` | `DensePolynomial::bind_parallel` with LowToHigh binding (2^20 evaluations) | +| `bind_parallel_high_to_low` | `DensePolynomial::bind_parallel` with HighToLow binding (2^20 evaluations) | +| `prover_time_fibonacci_100` | End-to-end prover time for `fibonacci(100)` | +| `prover_time_sha2_chain_100` | End-to-end prover time for 100 iterations of SHA-256 chain | +| `prover_time_secp256k1_ecdsa_verify` | End-to-end prover time for secp256k1 ECDSA signature verification | + +Note: `prover_time_*` benchmarks are standalone Criterion bench targets (run via `cargo bench -p jolt-eval --bench `). They are **not** included in `PerformanceObjective::all()` and are not tracked by the `optimize` or `measure-objectives` binaries. + +### Objective functions + +| Name | Inputs | Description | +|---|---|---| +| `minimize_lloc` | lloc | Minimize logical lines of code | +| `minimize_cognitive_complexity` | cognitive_complexity_avg | Minimize average cognitive complexity | +| `minimize_halstead_bugs` | halstead_bugs | Minimize estimated delivered bugs | +| `minimize_bind_low_to_high` | bind_parallel_low_to_high | Minimize LowToHigh binding time | +| `minimize_bind_high_to_low` | bind_parallel_high_to_low | Minimize HighToLow binding time | + +Custom composite objective functions can be defined as `ObjectiveFunction` structs: + +```rust +use jolt_eval::objective::objective_fn::ObjectiveFunction; +use jolt_eval::objective::{normalized, LLOC, HALSTEAD_BUGS}; + +const WEIGHTED_QUALITY: ObjectiveFunction = ObjectiveFunction { + name: "weighted_quality", + inputs: &[LLOC, HALSTEAD_BUGS], + evaluate: |m, _b| { + 2.0 * m.get(&LLOC).unwrap_or(&0.0) + m.get(&HALSTEAD_BUGS).unwrap_or(&0.0) + }, +}; +``` + +## Usage + +### Defining an invariant + +```rust +use jolt_eval::{Invariant, CheckError, InvariantViolation}; + +#[jolt_eval_macros::invariant(Test, Fuzz)] +#[derive(Default)] +pub struct MyInvariant; + +impl Invariant for MyInvariant { + type Setup = (); + type Input = u64; + + fn name(&self) -> &str { "my_invariant" } + fn description(&self) -> String { + "Human-readable description, also used as context for AI red-teaming.".into() + } + fn setup(&self) -> Self::Setup {} + fn check(&self, _setup: &(), input: u64) -> Result<(), CheckError> { + // ... check the invariant ... + Ok(()) + } + fn seed_corpus(&self) -> Vec { + vec![0, 1, u64::MAX] + } +} +``` + +The `#[invariant(Test, Fuzz)]` macro generates: +- `InvariantTargets` impl returning the listed targets +- `#[test] fn seed_corpus()` — runs every seed input through `check` +- `#[test] fn random_inputs()` — runs `JOLT_RANDOM_ITERS` (default 10) randomly-generated inputs + +### Fuzzing + +Each fuzz target is a 3-line file in `fuzz/fuzz_targets/`: + +```rust +#![no_main] +use jolt_eval::invariant::split_eq_bind::SplitEqBindLowHighInvariant; +jolt_eval::fuzz_invariant!(SplitEqBindLowHighInvariant::default()); +``` + +Run with `cargo fuzz`: +```bash +cd jolt-eval/fuzz +cargo fuzz run split_eq_bind_low_high +``` + +### Measuring objectives + +```bash +# All objectives (static analysis + Criterion benchmarks) +cargo run -p jolt-eval --bin measure-objectives + +# Static analysis only (skip benchmarks) +cargo run -p jolt-eval --bin measure-objectives -- --no-bench + +# A specific objective +cargo run -p jolt-eval --bin measure-objectives -- --objective lloc +``` + +### Running Criterion benchmarks directly + +```bash +# All benchmarks +cargo bench -p jolt-eval + +# A specific benchmark +cargo bench -p jolt-eval --bench prover_time_fibonacci + +# Quick mode (fewer samples) +cargo bench -p jolt-eval --bench bind_parallel_low_to_high -- --quick +``` + +Criterion results are saved to `target/criterion/` (symlinked from `jolt-eval/benches/results/`). + +### AI red-teaming + +```bash +# List available invariants +cargo run --release -p jolt-eval --bin redteam -- --list + +# Red-team a specific invariant +cargo run --release -p jolt-eval --bin redteam -- \ + --invariant soundness --iterations 10 \ + --hint "Look for edge cases in the memory layout" + +# Run the built-in e2e sort test +cargo run --release -p jolt-eval --bin redteam -- --test --verbose +``` + +The red-team harness runs the AI agent in an isolated git worktree. For the soundness invariant, the agent can edit `guest-sandbox/` directly — the harness captures the diff automatically via `git diff`. + +### AI-driven optimization + +```bash +# List available objective functions +cargo run --release -p jolt-eval --bin optimize -- --list + +# Optimize a specific objective function +cargo run --release -p jolt-eval --bin optimize -- \ + --objective minimize_lloc --iterations 5 \ + --hint "Focus on reducing complexity in jolt-core/src/subprotocols/" + +# Run the built-in e2e sort test +cargo run --release -p jolt-eval --bin optimize -- --test --verbose +``` + +Each iteration: the agent works in an isolated worktree, the diff is applied, objectives are re-measured (including Criterion benchmarks with `--save-baseline`), invariants are checked, and the change is committed or reverted. The optimizer creates a git branch `jolt-eval/optimize/{name}` and commits each accepted iteration. + +### Defining a performance benchmark + +Implement the `Objective` trait and create a bench file: + +```rust +// src/objective/performance/my_bench.rs +use crate::objective::Objective; + +#[derive(Default)] +pub struct MyBenchObjective; + +impl Objective for MyBenchObjective { + type Setup = MySetup; + fn name(&self) -> &str { "my_bench" } + fn setup(&self) -> MySetup { /* one-time setup */ } + fn run(&self, setup: MySetup) { /* hot path */ } +} +``` + +```rust +// benches/my_bench.rs +use jolt_eval::objective::performance::my_bench::MyBenchObjective; +jolt_eval::bench_objective!(MyBenchObjective); + +// Or with custom Criterion config for slow benchmarks: +jolt_eval::bench_objective!( + MyBenchObjective::default(), + config: sample_size(10), sampling_mode(::criterion::SamplingMode::Flat), +); +``` + +Then run `./sync_targets.sh` to update `Cargo.toml` bench entries. + +## Syncing targets + +```bash +./jolt-eval/sync_targets.sh +``` + +This script: +- Scans `src/invariant/` for `#[invariant(...Fuzz...)]` structs and generates/removes fuzz target files + `fuzz/Cargo.toml` entries +- Scans `benches/*.rs` and updates `Cargo.toml` `[[bench]]` entries + +Bench files are hand-authored (they carry domain-specific Criterion config). The script only syncs `Cargo.toml` entries from existing files. + +## Framing tasks in terms of invariants and objectives + +| Task | Invariants | Objectives | +|---|---|---| +| **New feature** | Add new invariants capturing the feature's behavior; modify existing invariants as necessary | Document expected impact; mechanically validate | +| **Bug fix** | Add/modify invariant to fail without the fix; verify all others still hold | Document impact | +| **Security review** | Try to find a counterexample to some invariant (via red-team) | — | +| **Optimization** | Ensure all invariants still hold | Maximize an objective function f(o₁, …, oₙ) | +| **Refactor** | Ensure all invariants still hold | Special case of optimization where the objective captures code quality | diff --git a/jolt-eval/benches/bind_parallel_high_to_low.rs b/jolt-eval/benches/bind_parallel_high_to_low.rs new file mode 100644 index 000000000..e776a5fb2 --- /dev/null +++ b/jolt-eval/benches/bind_parallel_high_to_low.rs @@ -0,0 +1,2 @@ +use jolt_eval::objective::performance::binding::BindHighToLowObjective; +jolt_eval::bench_objective!(BindHighToLowObjective); diff --git a/jolt-eval/benches/bind_parallel_low_to_high.rs b/jolt-eval/benches/bind_parallel_low_to_high.rs new file mode 100644 index 000000000..2af064f94 --- /dev/null +++ b/jolt-eval/benches/bind_parallel_low_to_high.rs @@ -0,0 +1,2 @@ +use jolt_eval::objective::performance::binding::BindLowToHighObjective; +jolt_eval::bench_objective!(BindLowToHighObjective); diff --git a/jolt-eval/benches/naive_sort_time.rs b/jolt-eval/benches/naive_sort_time.rs new file mode 100644 index 000000000..0f03400ce --- /dev/null +++ b/jolt-eval/benches/naive_sort_time.rs @@ -0,0 +1,2 @@ +use jolt_eval::objective::performance::naive_sort::NaiveSortObjective; +jolt_eval::bench_objective!(NaiveSortObjective); diff --git a/jolt-eval/benches/prover_time_fibonacci.rs b/jolt-eval/benches/prover_time_fibonacci.rs new file mode 100644 index 000000000..bbc999bcd --- /dev/null +++ b/jolt-eval/benches/prover_time_fibonacci.rs @@ -0,0 +1,10 @@ +use jolt_eval::guests::Fibonacci; +use jolt_eval::objective::performance::prover_time::ProverTimeObjective; + +jolt_eval::bench_objective!( + ProverTimeObjective::new(Fibonacci(100)), + config: + sample_size(10), + sampling_mode(::criterion::SamplingMode::Flat), + measurement_time(std::time::Duration::from_secs(30)), +); diff --git a/jolt-eval/benches/prover_time_secp256k1_ecdsa_verify.rs b/jolt-eval/benches/prover_time_secp256k1_ecdsa_verify.rs new file mode 100644 index 000000000..3b429c286 --- /dev/null +++ b/jolt-eval/benches/prover_time_secp256k1_ecdsa_verify.rs @@ -0,0 +1,10 @@ +use jolt_eval::guests::Secp256k1EcdsaVerify; +use jolt_eval::objective::performance::prover_time::ProverTimeObjective; + +jolt_eval::bench_objective!( + ProverTimeObjective::new(Secp256k1EcdsaVerify::default()), + config: + sample_size(10), + sampling_mode(::criterion::SamplingMode::Flat), + measurement_time(std::time::Duration::from_secs(60)), +); diff --git a/jolt-eval/benches/prover_time_sha2_chain.rs b/jolt-eval/benches/prover_time_sha2_chain.rs new file mode 100644 index 000000000..0a95ed7b9 --- /dev/null +++ b/jolt-eval/benches/prover_time_sha2_chain.rs @@ -0,0 +1,10 @@ +use jolt_eval::guests::Sha2Chain; +use jolt_eval::objective::performance::prover_time::ProverTimeObjective; + +jolt_eval::bench_objective!( + ProverTimeObjective::new(Sha2Chain::default()), + config: + sample_size(10), + sampling_mode(::criterion::SamplingMode::Flat), + measurement_time(std::time::Duration::from_secs(60)), +); diff --git a/jolt-eval/benches/results b/jolt-eval/benches/results new file mode 120000 index 000000000..32ab80389 --- /dev/null +++ b/jolt-eval/benches/results @@ -0,0 +1 @@ +../../target/criterion \ No newline at end of file diff --git a/jolt-eval/bin/measure_objectives.rs b/jolt-eval/bin/measure_objectives.rs new file mode 100644 index 000000000..10569abfd --- /dev/null +++ b/jolt-eval/bin/measure_objectives.rs @@ -0,0 +1,107 @@ +use clap::Parser; + +use jolt_eval::objective::performance::read_criterion_estimate; +use jolt_eval::objective::{PerformanceObjective, StaticAnalysisObjective}; + +#[derive(Parser)] +#[command(name = "measure-objectives")] +#[command(about = "Measure Jolt code quality and performance objectives")] +struct Cli { + /// Only measure the named objective (default: all) + #[arg(long)] + objective: Option, + + /// Skip Criterion benchmarks (only show static-analysis objectives) + #[arg(long)] + no_bench: bool, +} + +fn print_header() { + println!("{:<35} {:>15} {:>8}", "Objective", "Value", "Units"); + println!("{}", "-".repeat(60)); +} + +fn print_row(name: &str, val: f64, units: &str) { + println!("{:<35} {:>15.6} {:>8}", name, val, units); +} + +fn main() -> eyre::Result<()> { + tracing_subscriber::fmt::init(); + let cli = Cli::parse(); + + // Performance objectives (from Criterion) + if !cli.no_bench { + let perf = PerformanceObjective::all(); + let run_bench = cli + .objective + .as_ref() + .is_none_or(|name| perf.iter().any(|p| p.name() == name.as_str())); + + if run_bench { + eprintln!("Running Criterion benchmarks..."); + let mut any_succeeded = false; + for p in &perf { + if let Some(ref filter) = cli.objective { + if p.name() != filter.as_str() { + continue; + } + } + let status = std::process::Command::new("cargo") + .args([ + "bench", + "-p", + "jolt-eval", + "--bench", + p.name(), + "--", + "--quick", + ]) + .status(); + if matches!(status, Ok(s) if s.success()) { + any_succeeded = true; + } + } + + if any_succeeded { + println!(); + print_header(); + for p in &perf { + if let Some(ref filter) = cli.objective { + if p.name() != filter.as_str() { + continue; + } + } + match read_criterion_estimate(p.name(), "new") { + Some(secs) => print_row(p.name(), secs, "s"), + None => { + println!("{:<35} {:>15}", p.name(), "NO DATA"); + } + } + } + } + } + } else { + println!(); + print_header(); + } + + // Static-analysis objectives + for sa in StaticAnalysisObjective::all() { + if let Some(ref name) = cli.objective { + if sa.name() != name.as_str() { + continue; + } + } + match sa.collect_measurement() { + Ok(val) => { + let units = sa.units().unwrap_or("-"); + print_row(sa.name(), val, units); + } + Err(e) => { + println!("{:<35} {:>15}", sa.name(), format!("ERROR: {e}")); + } + } + } + + Ok(()) +} diff --git a/jolt-eval/bin/optimize.rs b/jolt-eval/bin/optimize.rs new file mode 100644 index 000000000..7be59914a --- /dev/null +++ b/jolt-eval/bin/optimize.rs @@ -0,0 +1,247 @@ +use std::collections::HashMap; +use std::process::Command; + +use clap::Parser; + +use jolt_eval::agent::ClaudeCodeAgent; +use jolt_eval::invariant::JoltInvariants; +use jolt_eval::objective::objective_fn::ObjectiveFunction; +use jolt_eval::objective::optimize::{auto_optimize, OptimizeConfig, OptimizeEnv}; +use jolt_eval::objective::performance::read_criterion_estimate; +use jolt_eval::objective::{OptimizationObjective, PerformanceObjective, StaticAnalysisObjective}; + +#[derive(Parser)] +#[command(name = "optimize")] +#[command(about = "AI-driven optimization of Jolt codebase objectives")] +struct Cli { + /// Objective function to minimize (mutually exclusive with --test). + #[arg(long, conflicts_with = "test")] + objective: Option, + + /// Run the built-in e2e sort optimization test. + #[arg(long, conflicts_with = "objective")] + test: bool, + + /// List all available objective functions and exit. + #[arg(long)] + list: bool, + + /// Number of optimization iterations + #[arg(long, default_value = "5")] + iterations: usize, + + /// AI model to use + #[arg(long, default_value = "claude-sonnet-4-20250514")] + model: String, + + /// Maximum number of Claude agentic turns per iteration + #[arg(long, default_value = "30")] + max_turns: usize, + + /// Extra context to include in the optimization prompt + #[arg(long)] + hint: Option, + + /// Print agent prompts and responses to stderr. + #[arg(long)] + verbose: bool, +} + +struct RealEnv { + repo_dir: std::path::PathBuf, + invariants: Vec, + bench_perf: bool, +} + +impl OptimizeEnv for RealEnv { + fn measure(&mut self) -> HashMap { + let mut results = HashMap::new(); + + for sa in StaticAnalysisObjective::all() { + if let Ok(v) = sa.collect_measurement() { + results.insert(OptimizationObjective::StaticAnalysis(sa), v); + } + } + + if self.bench_perf { + for p in PerformanceObjective::all() { + let status = Command::new("cargo") + .current_dir(&self.repo_dir) + .args([ + "bench", + "-p", + "jolt-eval", + "--bench", + p.name(), + "--", + "--quick", + "--save-baseline", + "optimize", + ]) + .status(); + + if matches!(status, Ok(s) if s.success()) { + if let Some(secs) = read_criterion_estimate(p.name(), "optimize") { + results.insert(OptimizationObjective::Performance(p), secs); + } + } + } + } + + results + } + + fn check_invariants(&mut self) -> bool { + self.invariants.iter().all(|inv| { + let results = inv.run_checks(0); + results.iter().all(|r| r.is_ok()) + }) + } + + fn apply_diff(&mut self, diff: &str) { + if let Err(e) = jolt_eval::agent::apply_diff(&self.repo_dir, diff) { + tracing::warn!("Failed to apply diff: {e}"); + } + } + + fn accept(&mut self, iteration: usize) { + println!(" Improvement found -- keeping changes (iteration {iteration})."); + } + + fn reject(&mut self) { + println!(" Reverting changes."); + } +} + +fn main() -> eyre::Result<()> { + tracing_subscriber::fmt::init(); + let cli = Cli::parse(); + + if cli.list { + println!("Available objective functions:\n"); + for f in ObjectiveFunction::all() { + let inputs: Vec<_> = f.inputs.iter().map(|i| i.name().to_string()).collect(); + println!(" {:<35} inputs: {}", f.name, inputs.join(", ")); + } + println!("\nBuilt-in e2e targets (use --test):"); + println!(" naive_sort"); + return Ok(()); + } + + if cli.test { + const SORT_TARGETS_PATH: &str = "jolt-eval/src/sort_targets.rs"; + let objective = ObjectiveFunction::by_name("minimize_naive_sort_time").unwrap(); + let repo_dir = std::env::current_dir()?; + let invariants = JoltInvariants::all(); + let mut env = RealEnv { + repo_dir: repo_dir.clone(), + invariants, + bench_perf: true, + }; + let baseline = env.measure(); + let baseline_score = (objective.evaluate)(&baseline, &baseline); + let hint = cli.hint.unwrap_or_else(|| { + format!( + "The target is the `naive_sort` function in {SORT_TARGETS_PATH}. \ + Replace it with a faster sorting algorithm. \ + You MAY modify that file for this task." + ) + }); + let config = OptimizeConfig { + num_iterations: cli.iterations, + hint: Some(hint), + verbose: cli.verbose, + }; + println!("=== Optimize e2e: naive bubble sort ==="); + println!( + "model={}, max_turns={}, iterations={}", + cli.model, cli.max_turns, cli.iterations + ); + println!("Baseline sort time: {baseline_score:.6}s"); + println!(); + let agent = ClaudeCodeAgent::new(&cli.model, cli.max_turns); + let result = auto_optimize(&agent, &mut env, objective, &config, &repo_dir); + println!("Best score: {:.6}s", result.best_score); + println!( + "Improvement: {:.1}%", + (1.0 - result.best_score / baseline_score) * 100.0 + ); + for (i, a) in result.attempts.iter().enumerate() { + println!( + " attempt {}: score={:.6}, invariants={}", + i + 1, + a.score, + a.invariants_passed + ); + } + return Ok(()); + } + + let objective_name = cli + .objective + .as_deref() + .expect("--objective or --test is required (use --list to see options)"); + + let objective = ObjectiveFunction::by_name(objective_name).unwrap_or_else(|| { + eprintln!("Unknown objective function: {objective_name}"); + eprintln!("Available:"); + for f in ObjectiveFunction::all() { + eprintln!(" {}", f.name); + } + std::process::exit(1); + }); + + let repo_dir = std::env::current_dir()?; + + let bench_perf = objective.inputs.iter().any(|i| i.is_perf()); + let invariants = JoltInvariants::all(); + + let mut env = RealEnv { + repo_dir: repo_dir.clone(), + invariants, + bench_perf, + }; + + let baseline = env.measure(); + + println!("=== Baseline ==="); + print_measurements(&baseline); + let baseline_score = (objective.evaluate)(&baseline, &baseline); + println!("Objective: {} = {:.6}\n", objective.name, baseline_score); + + let agent = ClaudeCodeAgent::new(&cli.model, cli.max_turns); + let config = OptimizeConfig { + num_iterations: cli.iterations, + hint: cli.hint.clone(), + verbose: cli.verbose, + }; + + let result = auto_optimize(&agent, &mut env, objective, &config, &repo_dir); + + println!("=== Summary ==="); + println!( + "{}/{} iterations improved the objective.", + result + .attempts + .iter() + .filter(|a| a.invariants_passed && a.score < result.baseline_score) + .count(), + result.attempts.len() + ); + println!( + "Score: {:.6} -> {:.6}", + result.baseline_score, result.best_score + ); + println!("\nFinal measurements:"); + print_measurements(&result.best_measurements); + + Ok(()) +} + +fn print_measurements(measurements: &HashMap) { + let mut entries: Vec<_> = measurements.iter().collect(); + entries.sort_by_key(|(k, _)| k.name()); + for (key, val) in entries { + println!(" {:<35} {:>15.6}", key.name(), val); + } +} diff --git a/jolt-eval/bin/redteam.rs b/jolt-eval/bin/redteam.rs new file mode 100644 index 000000000..c2111f5e9 --- /dev/null +++ b/jolt-eval/bin/redteam.rs @@ -0,0 +1,131 @@ +use clap::Parser; +use tracing::info; + +use jolt_eval::agent::ClaudeCodeAgent; +use jolt_eval::invariant::synthesis::redteam::{RedTeamConfig, RedTeamResult}; +use jolt_eval::invariant::{JoltInvariants, SynthesisTarget}; +use jolt_eval::sort_e2e; + +#[derive(Parser)] +#[command(name = "redteam")] +#[command(about = "AI-driven red team testing of Jolt invariants")] +struct Cli { + /// Name of the invariant to test (mutually exclusive with --test). + #[arg(long, conflicts_with = "test")] + invariant: Option, + + /// Run the built-in e2e sort test instead of a named invariant. + #[arg(long, conflicts_with = "invariant")] + test: bool, + + /// List all red-teamable invariants and exit. + #[arg(long)] + list: bool, + + /// Number of red-team iterations + #[arg(long, default_value = "10")] + iterations: usize, + + /// AI model to use + #[arg(long, default_value = "claude-sonnet-4-20250514")] + model: String, + + /// Maximum number of Claude agentic turns per iteration + #[arg(long, default_value = "30")] + max_turns: usize, + + /// Extra context or guidance for the red-team agent + #[arg(long)] + hint: Option, + + /// Print agent prompts and responses to stderr. + #[arg(long)] + verbose: bool, +} + +fn main() -> eyre::Result<()> { + tracing_subscriber::fmt::init(); + let cli = Cli::parse(); + + if cli.list { + println!("Red-teamable invariants:"); + for inv in &JoltInvariants::all() { + if inv.targets().contains(SynthesisTarget::RedTeam) { + println!(" {}", inv.name()); + } + } + println!("\nBuilt-in e2e targets (use --test):"); + println!(" candidate_sort"); + return Ok(()); + } + + if cli.test { + sort_e2e::run_redteam_test( + &cli.model, + cli.max_turns, + cli.iterations, + cli.hint, + cli.verbose, + ); + return Ok(()); + } + + let invariant_name = cli + .invariant + .as_deref() + .expect("--invariant or --test is required (use --list to see options)"); + + let all = JoltInvariants::all(); + let invariant = all + .iter() + .filter(|inv| inv.targets().contains(SynthesisTarget::RedTeam)) + .find(|inv| inv.name() == invariant_name); + + let Some(invariant) = invariant else { + eprintln!("Invariant '{invariant_name}' not found or not red-teamable."); + eprintln!("Run with --list to see available invariants."); + std::process::exit(1); + }; + + let config = RedTeamConfig { + num_iterations: cli.iterations, + hint: cli.hint, + verbose: cli.verbose, + }; + let agent = ClaudeCodeAgent::new(&cli.model, cli.max_turns); + let repo_dir = std::env::current_dir()?; + + info!( + "Starting red team: invariant={invariant_name}, iterations={}, model={}", + cli.iterations, cli.model + ); + + let result = invariant.red_team(&config, &agent, &repo_dir); + + match result { + RedTeamResult::Violation { + approach, + input_json, + error, + } => { + println!(); + println!("==== VIOLATION FOUND ===="); + println!("Approach: {approach}"); + println!("Input: {input_json}"); + println!("Error: {error}"); + std::process::exit(1); + } + RedTeamResult::NoViolation { attempts } => { + println!(); + println!("No violations found after {} iterations.", attempts.len()); + for attempt in &attempts { + println!( + " {}: {} -- {}", + attempt.description, attempt.approach, attempt.failure_reason + ); + } + } + } + + Ok(()) +} diff --git a/jolt-eval/fuzz/.gitignore b/jolt-eval/fuzz/.gitignore new file mode 100644 index 000000000..1a45eee77 --- /dev/null +++ b/jolt-eval/fuzz/.gitignore @@ -0,0 +1,4 @@ +target +corpus +artifacts +coverage diff --git a/jolt-eval/fuzz/Cargo.lock b/jolt-eval/fuzz/Cargo.lock new file mode 100644 index 000000000..7747142bb --- /dev/null +++ b/jolt-eval/fuzz/Cargo.lock @@ -0,0 +1,2500 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "addr2line" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59317f77929f0e679d39364702289274de2f0f0b22cbf50b2b8cff2169a0b27a" +dependencies = [ + "cpp_demangle", + "fallible-iterator", + "gimli", + "memmap2", + "object 0.39.0", + "rustc-demangle", + "smallvec", + "typed-arena", +] + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "once_cell", + "version_check", + "zerocopy", +] + +[[package]] +name = "aho-corasick" +version = "0.7.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc936419f96fa211c1b9166887b38e5e40b19958e5b895be7c1f93adec7071ac" +dependencies = [ + "memchr", +] + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "allocative" +version = "0.3.4" +source = "git+https://github.com/facebookexperimental/allocative?rev=85b773d85d526d068ce94724ff7a7b81203fc95e#85b773d85d526d068ce94724ff7a7b81203fc95e" +dependencies = [ + "allocative_derive", + "ctor", +] + +[[package]] +name = "allocative_derive" +version = "0.3.3" +source = "git+https://github.com/facebookexperimental/allocative?rev=85b773d85d526d068ce94724ff7a7b81203fc95e#85b773d85d526d068ce94724ff7a7b81203fc95e" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys 0.61.2", +] + +[[package]] +name = "anyhow" +version = "1.0.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" + +[[package]] +name = "arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" +dependencies = [ + "derive_arbitrary", +] + +[[package]] +name = "ark-bn254" +version = "0.5.0" +source = "git+https://github.com/a16z/arkworks-algebra?branch=dev%2Ftwist-shout#76bb3a4518928f1ff7f15875f940d614bb9845e6" +dependencies = [ + "ark-ec", + "ark-ff", + "ark-serialize", + "ark-std", +] + +[[package]] +name = "ark-ec" +version = "0.5.0" +source = "git+https://github.com/a16z/arkworks-algebra?branch=dev%2Ftwist-shout#76bb3a4518928f1ff7f15875f940d614bb9845e6" +dependencies = [ + "ahash", + "ark-ff", + "ark-poly", + "ark-serialize", + "ark-std", + "educe", + "fnv", + "hashbrown 0.15.5", + "itertools 0.13.0", + "num-bigint", + "num-integer", + "num-traits", + "rayon", + "zeroize", +] + +[[package]] +name = "ark-ff" +version = "0.5.0" +source = "git+https://github.com/a16z/arkworks-algebra?branch=dev%2Ftwist-shout#76bb3a4518928f1ff7f15875f940d614bb9845e6" +dependencies = [ + "allocative", + "ark-ff-asm", + "ark-ff-macros", + "ark-serialize", + "ark-std", + "arrayvec", + "digest", + "educe", + "itertools 0.13.0", + "num-bigint", + "num-traits", + "paste", + "rayon", + "zeroize", +] + +[[package]] +name = "ark-ff-asm" +version = "0.5.0" +source = "git+https://github.com/a16z/arkworks-algebra?branch=dev%2Ftwist-shout#76bb3a4518928f1ff7f15875f940d614bb9845e6" +dependencies = [ + "quote", + "syn 2.0.117", +] + +[[package]] +name = "ark-ff-macros" +version = "0.5.0" +source = "git+https://github.com/a16z/arkworks-algebra?branch=dev%2Ftwist-shout#76bb3a4518928f1ff7f15875f940d614bb9845e6" +dependencies = [ + "num-bigint", + "num-traits", + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "ark-poly" +version = "0.5.0" +source = "git+https://github.com/a16z/arkworks-algebra?branch=dev%2Ftwist-shout#76bb3a4518928f1ff7f15875f940d614bb9845e6" +dependencies = [ + "ahash", + "ark-ff", + "ark-serialize", + "ark-std", + "educe", + "fnv", + "hashbrown 0.15.5", +] + +[[package]] +name = "ark-serialize" +version = "0.5.0" +source = "git+https://github.com/a16z/arkworks-algebra?branch=dev%2Ftwist-shout#76bb3a4518928f1ff7f15875f940d614bb9845e6" +dependencies = [ + "ark-serialize-derive", + "ark-std", + "arrayvec", + "digest", + "num-bigint", + "rayon", +] + +[[package]] +name = "ark-serialize-derive" +version = "0.5.0" +source = "git+https://github.com/a16z/arkworks-algebra?branch=dev%2Ftwist-shout#76bb3a4518928f1ff7f15875f940d614bb9845e6" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "ark-std" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "246a225cc6131e9ee4f24619af0f19d67761fff15d7ccc22e42b80846e69449a" +dependencies = [ + "num-traits", + "rand", + "rayon", +] + +[[package]] +name = "arrayvec" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" + +[[package]] +name = "autocfg" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" + +[[package]] +name = "bincode" +version = "1.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" +dependencies = [ + "serde", +] + +[[package]] +name = "bincode" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36eaf5d7b090263e8150820482d5d93cd964a81e4019913c972f4edcc6edb740" +dependencies = [ + "bincode_derive", + "serde", + "unty", +] + +[[package]] +name = "bincode_derive" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf95709a440f45e986983918d0e8a1f30a9b1df04918fc828670606804ac3c09" +dependencies = [ + "virtue", +] + +[[package]] +name = "bitflags" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843867be96c8daad0d758b57df9392b6d8d271134fce549de6ce169ff98a92af" + +[[package]] +name = "blake2" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe" +dependencies = [ + "digest", +] + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bumpalo" +version = "3.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "cc" +version = "1.2.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1e928d4b69e3077709075a938a05ffbedfa53a84c8f766efbf8220bb1ff60e1" +dependencies = [ + "find-msvc-tools", + "jobserver", + "libc", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "chrono" +version = "0.4.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c673075a2e0e5f4a1dde27ce9dee1ea4558c7ffe648f576438a20ca1d2acc4b0" +dependencies = [ + "iana-time-zone", + "js-sys", + "num-traits", + "wasm-bindgen", + "windows-link", +] + +[[package]] +name = "clap" +version = "4.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b193af5b67834b676abd72466a96c1024e6a6ad978a1f484bd90b85c94041351" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1110bd8a634a1ab8cb04345d8d878267d57c3cf1b38d91b71af6686408bbca6a" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "clap_lex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" + +[[package]] +name = "cobs" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fa961b519f0b462e3a3b4a34b64d119eeaca1d59af726fe450bbba07a9fc0a1" +dependencies = [ + "thiserror", +] + +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + +[[package]] +name = "common" +version = "0.2.0" +dependencies = [ + "allocative", + "ark-serialize", + "serde", + "syn 2.0.117", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "cpp_demangle" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0667304c32ea56cb4cd6d2d7c0cfe9a2f8041229db8c033af7f8d69492429def" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "ctor" +version = "0.1.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d2301688392eb071b0bf1a37be05c469d3cc4dbbd95df672fe28ab021e6a096" +dependencies = [ + "quote", + "syn 1.0.109", +] + +[[package]] +name = "darling" +version = "0.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9cdf337090841a411e2a7f3deb9187445851f91b309c0c0a29e05f74a00a48c0" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1247195ecd7e3c85f83c8d2a366e4210d588e802133e1e355180a9870b517ea4" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "darling_macro" +version = "0.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d38308df82d1080de0afee5d069fa14b0326a88c14f15c5ccda35b4a6c414c81" +dependencies = [ + "darling_core", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "derive_arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e567bd82dcff979e4b03460c307b3cdc9e96fde3d73bed1496d2bc75d9dd62a" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "derive_more" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d751e9e49156b02b44f9c1815bcb94b984cdcc4396ecc32521c739452808b134" +dependencies = [ + "derive_more-impl", +] + +[[package]] +name = "derive_more-impl" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "799a97264921d8623a957f6c3b9011f3b5492f557bbb7a5a19b7fa6d06ba8dcb" +dependencies = [ + "proc-macro2", + "quote", + "rustc_version", + "syn 2.0.117", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", + "subtle", +] + +[[package]] +name = "dory-derive" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc9c63de9e3d87d5be179ce2ddde4f31d95c12c1f20ccdbc3a70b004813959ca" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "dory-pcs" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8c58baea9f0ed973489cd1981b0e6a8c91aafddb05e3903b1dd54175ddcb52d" +dependencies = [ + "ark-bn254", + "ark-ec", + "ark-ff", + "ark-serialize", + "ark-std", + "bincode 1.3.3", + "blake2", + "digest", + "dory-derive", + "rand_core", + "rayon", + "serde", + "thiserror", + "tracing", +] + +[[package]] +name = "dyn-clone" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" + +[[package]] +name = "educe" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d7bc049e1bd8cdeb31b68bbd586a9464ecf9f3944af3958a7a9d0f8b9799417" +dependencies = [ + "enum-ordinalize", + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "either" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" + +[[package]] +name = "embedded-io" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef1a6892d9eef45c8fa6b9e0086428a2cca8491aca8f787c534a3d6d0bcb3ced" + +[[package]] +name = "embedded-io" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edd0f118536f44f5ccd48bcb8b111bdc3de888b58c74639dfb034a357d0f206d" + +[[package]] +name = "enum-ordinalize" +version = "4.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a1091a7bb1f8f2c4b28f1fe2cef4980ca2d410a3d727d67ecc3178c9b0800f0" +dependencies = [ + "enum-ordinalize-derive", +] + +[[package]] +name = "enum-ordinalize-derive" +version = "4.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ca9601fb2d62598ee17836250842873a413586e5d7ed88b356e38ddbb0ec631" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "enumset" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25b07a8dfbbbfc0064c0a6bdf9edcf966de6b1c33ce344bdeca3b41615452634" +dependencies = [ + "enumset_derive", +] + +[[package]] +name = "enumset_derive" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f43e744e4ea338060faee68ed933e46e722fb7f3617e722a5772d7e856d8b3ce" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "eyre" +version = "0.6.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cd915d99f24784cdc19fd37ef22b97e3ff0ae756c7e492e9fbfe897d61e2aec" +dependencies = [ + "indenter", + "once_cell", +] + +[[package]] +name = "fallible-iterator" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" + +[[package]] +name = "fastrand" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "fixedbitset" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" + +[[package]] +name = "fixedbitset" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" + +[[package]] +name = "flate2" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + +[[package]] +name = "fxhash" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c" +dependencies = [ + "byteorder", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "wasi", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "r-efi 5.3.0", + "wasip2", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" +dependencies = [ + "cfg-if", + "libc", + "r-efi 6.0.0", + "wasip2", + "wasip3", +] + +[[package]] +name = "gimli" +version = "0.33.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19e16c5073773ccf057c282be832a59ee53ef5ff98db3aeff7f8314f52ffc196" +dependencies = [ + "stable_deref_trait", +] + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "allocator-api2", + "foldhash 0.1.5", +] + +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "foldhash 0.2.0", +] + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "id-arena" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "indenter" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "964de6e86d545b246d84badc0fef527924ace5134f30641c203ef52ba83f58d5" + +[[package]] +name = "indexmap" +version = "2.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7714e70437a7dc3ac8eb7e6f8df75fd8eb422675fc7678aff7364301092b1017" +dependencies = [ + "equivalent", + "hashbrown 0.16.1", + "serde", + "serde_core", +] + +[[package]] +name = "inventory" +version = "0.3.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4f0c30c76f2f4ccee3fe55a2435f691ca00c0e4bd87abe4f4a851b1d4dac39b" +dependencies = [ + "rustversion", +] + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "jobserver" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" +dependencies = [ + "getrandom 0.3.4", + "libc", +] + +[[package]] +name = "jolt-core" +version = "0.1.0" +dependencies = [ + "allocative", + "ark-bn254", + "ark-ec", + "ark-ff", + "ark-serialize", + "ark-std", + "bincode 2.0.1", + "blake2", + "chrono", + "clap", + "common", + "derive_more", + "dory-pcs", + "eyre", + "fixedbitset 0.5.7", + "itertools 0.14.0", + "jolt-inlines-keccak256", + "jolt-inlines-sha2", + "jolt-optimizations", + "memory-stats", + "num", + "num-derive 0.4.2", + "num-traits", + "postcard", + "rand", + "rand_chacha", + "rand_core", + "rayon", + "serde", + "sha3", + "strum", + "strum_macros", + "thiserror", + "tracer", + "tracing", + "tracing-chrome", + "tracing-subscriber", +] + +[[package]] +name = "jolt-eval" +version = "0.1.0" +dependencies = [ + "arbitrary", + "ark-bn254", + "clap", + "common", + "enumset", + "eyre", + "jolt-core", + "jolt-eval-macros", + "postcard", + "rand", + "rust-code-analysis", + "schemars", + "serde", + "serde_json", + "tempfile", + "tracer", + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "jolt-eval-fuzz" +version = "0.0.0" +dependencies = [ + "jolt-eval", + "libfuzzer-sys", +] + +[[package]] +name = "jolt-eval-macros" +version = "0.1.0" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "jolt-inlines-keccak256" +version = "0.1.0" +dependencies = [ + "jolt-inlines-sdk", +] + +[[package]] +name = "jolt-inlines-sdk" +version = "0.1.0" +dependencies = [ + "inventory", + "jolt-platform", + "num-bigint", + "tracer", +] + +[[package]] +name = "jolt-inlines-sha2" +version = "0.1.0" +dependencies = [ + "jolt-inlines-sdk", +] + +[[package]] +name = "jolt-optimizations" +version = "0.5.0" +source = "git+https://github.com/a16z/arkworks-algebra?branch=dev%2Ftwist-shout#76bb3a4518928f1ff7f15875f940d614bb9845e6" +dependencies = [ + "ark-bn254", + "ark-ec", + "ark-ff", + "ark-serialize", + "ark-std", + "arrayvec", + "num-bigint", + "num-integer", + "num-rational", + "num-traits", + "rayon", +] + +[[package]] +name = "jolt-platform" +version = "0.1.0" +dependencies = [ + "getrandom 0.2.17", + "getrandom 0.3.4", + "rand", +] + +[[package]] +name = "js-sys" +version = "0.3.94" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e04e2ef80ce82e13552136fabeef8a5ed1f985a96805761cbb9a2c34e7664d9" +dependencies = [ + "once_cell", + "wasm-bindgen", +] + +[[package]] +name = "keccak" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb26cec98cce3a3d96cbb7bced3c4b16e3d13f27ec56dbd62cbc8f39cfb9d653" +dependencies = [ + "cpufeatures", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "leb128fmt" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" + +[[package]] +name = "libc" +version = "0.2.184" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48f5d2a454e16a5ea0f4ced81bd44e4cfc7bd3a507b61887c99fd3538b28e4af" + +[[package]] +name = "libfuzzer-sys" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f12a681b7dd8ce12bff52488013ba614b869148d54dd79836ab85aafdd53f08d" +dependencies = [ + "arbitrary", + "cc", +] + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "log" +version = "0.4.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" + +[[package]] +name = "matchers" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" +dependencies = [ + "regex-automata", +] + +[[package]] +name = "memchr" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" + +[[package]] +name = "memmap2" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "714098028fe011992e1c3962653c96b2d578c4b4bce9036e15ff220319b1e0e3" +dependencies = [ + "libc", +] + +[[package]] +name = "memory-stats" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c73f5c649995a115e1a0220b35e4df0a1294500477f97a91d0660fb5abeb574a" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "nu-ansi-term" +version = "0.50.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "num" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" +dependencies = [ + "num-bigint", + "num-complex", + "num-integer", + "num-iter", + "num-rational", + "num-traits", +] + +[[package]] +name = "num-bigint" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-derive" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "876a53fff98e03a936a674b29568b0e605f06b29372c2489ff4de23f1949743d" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "num-derive" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed3955f1a9c7c0c15e092f9c887db08b1fc683305fdf6eb6684f22555355e202" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "num-format" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a652d9771a63711fd3c3deb670acfbe5c30a4072e664d7a3bf5a9e1056ac72c3" +dependencies = [ + "arrayvec", + "itoa", +] + +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-iter" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1429034a0490724d0075ebb2bc9e875d6503c3cf69e235a8941aa757d83ef5bf" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" +dependencies = [ + "num-bigint", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "object" +version = "0.38.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "271638cd5fa9cca89c4c304675ca658efc4e64a66c716b7cfe1afb4b9611dbbc" +dependencies = [ + "crc32fast", + "hashbrown 0.16.1", + "indexmap", + "memchr", +] + +[[package]] +name = "object" +version = "0.39.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63944c133d03f44e75866bbd160b95af0ec3f6a13d936d69d31c81078cbc5baf" +dependencies = [ + "flate2", + "memchr", + "ruzstd", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + +[[package]] +name = "petgraph" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" +dependencies = [ + "fixedbitset 0.4.2", + "indexmap", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "postcard" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6764c3b5dd454e283a30e6dfe78e9b31096d9e32036b5d1eaac7a6119ccb9a24" +dependencies = [ + "cobs", + "embedded-io 0.4.0", + "embedded-io 0.6.1", + "serde", +] + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn 2.0.117", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.17", +] + +[[package]] +name = "rayon" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "368f01d005bf8fd9b1206fb6fa653e6c4a81ceb1466406b81792d87c5677a58f" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "regex" +version = "1.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" +dependencies = [ + "aho-corasick 1.1.4", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +dependencies = [ + "aho-corasick 1.1.4", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" + +[[package]] +name = "rust-code-analysis" +version = "0.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92a0f85e044428a7b58538f95fa58a157d89d5bcc5b37df6e7024957e52bdc5a" +dependencies = [ + "aho-corasick 0.7.20", + "fxhash", + "lazy_static", + "num", + "num-derive 0.3.3", + "num-format", + "num-traits", + "petgraph", + "regex", + "serde", + "termcolor", + "tree-sitter", + "tree-sitter-ccomment", + "tree-sitter-java", + "tree-sitter-javascript", + "tree-sitter-mozcpp", + "tree-sitter-mozjs", + "tree-sitter-preproc", + "tree-sitter-python", + "tree-sitter-rust", + "tree-sitter-typescript", +] + +[[package]] +name = "rustc-demangle" +version = "0.1.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b50b8869d9fc858ce7266cce0194bd74df58b9d0e3f6df3a9fc8eb470d95c09d" + +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + +[[package]] +name = "ruzstd" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5ff0cc5e135c8870a775d3320910cd9b564ec036b4dc0b8741629020be63f01" +dependencies = [ + "twox-hash", +] + +[[package]] +name = "schemars" +version = "0.8.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fbf2ae1b8bc8e02df939598064d22402220cd5bbcca1c76f7d6a310974d5615" +dependencies = [ + "dyn-clone", + "schemars_derive", + "serde", + "serde_json", +] + +[[package]] +name = "schemars_derive" +version = "0.8.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32e265784ad618884abaea0600a9adf15393368d840e0222d101a072f3f7534d" +dependencies = [ + "proc-macro2", + "quote", + "serde_derive_internals", + "syn 2.0.117", +] + +[[package]] +name = "semver" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2" + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "serde_derive_internals" +version = "0.29.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "serde_json" +version = "1.0.149" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "sha3" +version = "0.10.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75872d278a8f37ef87fa0ddbda7802605cb18344497949862c0d4dcb291eba60" +dependencies = [ + "digest", + "keccak", +] + +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "simd-adler32" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" + +[[package]] +name = "smallvec" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "strum" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9628de9b8791db39ceda2b119bbe13134770b56c138ec1d3af810d045c04f9bd" + +[[package]] +name = "strum_macros" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab85eea0270ee17587ed4156089e10b9e6880ee688791d45a905f5b1ca36f664" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "2.0.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.4.2", + "once_cell", + "rustix", + "windows-sys 0.61.2", +] + +[[package]] +name = "termcolor" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "thiserror" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "thread_local" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "tracer" +version = "0.2.0" +dependencies = [ + "addr2line", + "ark-serialize", + "clap", + "common", + "derive_more", + "fnv", + "inventory", + "itertools 0.14.0", + "jolt-platform", + "object 0.38.1", + "paste", + "postcard", + "serde", + "serde_json", + "strum", + "strum_macros", + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "tracing-chrome" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf0a738ed5d6450a9fb96e86a23ad808de2b727fd1394585da5cdd6788ffe724" +dependencies = [ + "serde_json", + "tracing-core", + "tracing-subscriber", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex-automata", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", +] + +[[package]] +name = "tree-sitter" +version = "0.19.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f41201fed3db3b520405a9c01c61773a250d4c3f43e9861c14b2bb232c981ab" +dependencies = [ + "cc", + "regex", +] + +[[package]] +name = "tree-sitter-ccomment" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3b402bc539927bb457e5ab59aac7260e2c3b97c5fcfc043575788654eedd69a" +dependencies = [ + "cc", + "tree-sitter", +] + +[[package]] +name = "tree-sitter-cpp" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7bd90c7b7db59369ed00fbc40458d9c9b2b8ed145640e337e839ac07aa63e15" +dependencies = [ + "cc", + "tree-sitter", +] + +[[package]] +name = "tree-sitter-java" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "301ae2ee7813e1bf935dc06db947642400645bbea8878431e1b31131488d5430" +dependencies = [ + "cc", + "tree-sitter", +] + +[[package]] +name = "tree-sitter-javascript" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "840bb4d5f3c384cb76b976ff07297f5a24b6e61a708baa4464f53e395caaa5f9" +dependencies = [ + "cc", + "tree-sitter", +] + +[[package]] +name = "tree-sitter-mozcpp" +version = "0.19.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5439f32b7685af19efcd0165d28dab80261e1cc922ed259c9c7909c96ac4cc6" +dependencies = [ + "cc", + "tree-sitter", + "tree-sitter-cpp", +] + +[[package]] +name = "tree-sitter-mozjs" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "def6b21c10157d3d79b912191fa4549008885da827451a62be9f30abeb7319c8" +dependencies = [ + "cc", + "tree-sitter", + "tree-sitter-javascript", +] + +[[package]] +name = "tree-sitter-preproc" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "226b2a77578e83efa7a193919660ffc88c22e357f9c2d9f27b5b11898a8682d3" +dependencies = [ + "cc", + "tree-sitter", +] + +[[package]] +name = "tree-sitter-python" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5646bfe71c4eb1c21b714ce0c38334c311eab767095582859e85da6281e9fd6c" +dependencies = [ + "cc", + "tree-sitter", +] + +[[package]] +name = "tree-sitter-rust" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "784f7ef9cdbd4c895dc2d4bb785e95b4a5364a602eec803681db83d1927ddf15" +dependencies = [ + "cc", + "tree-sitter", +] + +[[package]] +name = "tree-sitter-typescript" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3f62d49c6e56bf291c412ee5e178ea14dff40f14a5f01a8847933f56d65bf3b" +dependencies = [ + "cc", + "tree-sitter", +] + +[[package]] +name = "twox-hash" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c" + +[[package]] +name = "typed-arena" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6af6ae20167a9ece4bcb41af5b80f8a1f1df981f6391189ce00fd257af04126a" + +[[package]] +name = "typenum" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + +[[package]] +name = "unty" +version = "0.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d49784317cd0d1ee7ec5c716dd598ec5b4483ea832a2dced265471cc0f690ae" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "virtue" +version = "0.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "051eb1abcf10076295e815102942cc58f9d5e3b4560e46e53c21e8ff6f3af7b1" + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasip2" +version = "1.0.2+wasi-0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9517f9239f02c069db75e65f174b3da828fe5f5b945c4dd26bd25d89c03ebcf5" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasip3" +version = "0.4.0+wasi-0.3.0-rc-2026-01-06" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0551fc1bb415591e3372d0bc4780db7e587d84e2a7e79da121051c5c4b89d0b0" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fbdf9a35adf44786aecd5ff89b4563a90325f9da0923236f6104e603c7e86be" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dca9693ef2bab6d4e6707234500350d8dad079eb508dca05530c85dc3a529ff2" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 2.0.117", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39129a682a6d2d841b6c429d0c51e5cb0ed1a03829d8b3d1e69a011e62cb3d3b" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "wasm-encoder" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319" +dependencies = [ + "leb128fmt", + "wasmparser", +] + +[[package]] +name = "wasm-metadata" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" +dependencies = [ + "anyhow", + "indexmap", + "wasm-encoder", + "wasmparser", +] + +[[package]] +name = "wasmparser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" +dependencies = [ + "bitflags", + "hashbrown 0.15.5", + "indexmap", + "semver", +] + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "wit-bindgen" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" +dependencies = [ + "wit-bindgen-rust-macro", +] + +[[package]] +name = "wit-bindgen-core" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc" +dependencies = [ + "anyhow", + "heck", + "wit-parser", +] + +[[package]] +name = "wit-bindgen-rust" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" +dependencies = [ + "anyhow", + "heck", + "indexmap", + "prettyplease", + "syn 2.0.117", + "wasm-metadata", + "wit-bindgen-core", + "wit-component", +] + +[[package]] +name = "wit-bindgen-rust-macro" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a" +dependencies = [ + "anyhow", + "prettyplease", + "proc-macro2", + "quote", + "syn 2.0.117", + "wit-bindgen-core", + "wit-bindgen-rust", +] + +[[package]] +name = "wit-component" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" +dependencies = [ + "anyhow", + "bitflags", + "indexmap", + "log", + "serde", + "serde_derive", + "serde_json", + "wasm-encoder", + "wasm-metadata", + "wasmparser", + "wit-parser", +] + +[[package]] +name = "wit-parser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" +dependencies = [ + "anyhow", + "id-arena", + "indexmap", + "log", + "semver", + "serde", + "serde_derive", + "serde_json", + "unicode-xid", + "wasmparser", +] + +[[package]] +name = "zerocopy" +version = "0.8.48" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eed437bf9d6692032087e337407a86f04cd8d6a16a37199ed57949d415bd68e9" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.48" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70e3cd084b1788766f53af483dd21f93881ff30d7320490ec3ef7526d203bad4" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "zeroize" +version = "1.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" +dependencies = [ + "zeroize_derive", +] + +[[package]] +name = "zeroize_derive" +version = "1.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85a5b4158499876c763cb03bc4e49185d3cccbabb15b33c627f7884f43db852e" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "zmij" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" diff --git a/jolt-eval/fuzz/Cargo.toml b/jolt-eval/fuzz/Cargo.toml new file mode 100644 index 000000000..4eae4f2e4 --- /dev/null +++ b/jolt-eval/fuzz/Cargo.toml @@ -0,0 +1,39 @@ +[package] +name = "jolt-eval-fuzz" +version = "0.0.0" +publish = false +edition = "2021" + +[workspace] + +[patch.crates-io] +ark-bn254 = { git = "https://github.com/a16z/arkworks-algebra", branch = "dev/twist-shout" } +ark-ff = { git = "https://github.com/a16z/arkworks-algebra", branch = "dev/twist-shout" } +ark-ec = { git = "https://github.com/a16z/arkworks-algebra", branch = "dev/twist-shout" } +ark-serialize = { git = "https://github.com/a16z/arkworks-algebra", branch = "dev/twist-shout" } +allocative = { git = "https://github.com/facebookexperimental/allocative", rev = "85b773d85d526d068ce94724ff7a7b81203fc95e" } + +[package.metadata] +cargo-fuzz = true + +[package.metadata.cargo-machete] +ignored = ["libfuzzer-sys"] + +[dependencies] +libfuzzer-sys = "0.4" +jolt-eval = { path = ".." } + +[[bin]] +name = "split_eq_bind_high_low" +path = "fuzz_targets/split_eq_bind_high_low.rs" +test = false +doc = false +bench = false + +[[bin]] +name = "split_eq_bind_low_high" +path = "fuzz_targets/split_eq_bind_low_high.rs" +test = false +doc = false +bench = false + diff --git a/jolt-eval/fuzz/fuzz_targets/split_eq_bind_high_low.rs b/jolt-eval/fuzz/fuzz_targets/split_eq_bind_high_low.rs new file mode 100644 index 000000000..dda196175 --- /dev/null +++ b/jolt-eval/fuzz/fuzz_targets/split_eq_bind_high_low.rs @@ -0,0 +1,3 @@ +#![no_main] +use jolt_eval::invariant::split_eq_bind::SplitEqBindHighLowInvariant; +jolt_eval::fuzz_invariant!(SplitEqBindHighLowInvariant::default()); diff --git a/jolt-eval/fuzz/fuzz_targets/split_eq_bind_low_high.rs b/jolt-eval/fuzz/fuzz_targets/split_eq_bind_low_high.rs new file mode 100644 index 000000000..7057af400 --- /dev/null +++ b/jolt-eval/fuzz/fuzz_targets/split_eq_bind_low_high.rs @@ -0,0 +1,3 @@ +#![no_main] +use jolt_eval::invariant::split_eq_bind::SplitEqBindLowHighInvariant; +jolt_eval::fuzz_invariant!(SplitEqBindLowHighInvariant::default()); diff --git a/jolt-eval/guest-sandbox/.gitignore b/jolt-eval/guest-sandbox/.gitignore new file mode 100644 index 000000000..b83d22266 --- /dev/null +++ b/jolt-eval/guest-sandbox/.gitignore @@ -0,0 +1 @@ +/target/ diff --git a/jolt-eval/guest-sandbox/Cargo.toml b/jolt-eval/guest-sandbox/Cargo.toml new file mode 100644 index 000000000..b5ab76e51 --- /dev/null +++ b/jolt-eval/guest-sandbox/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "sandbox-guest" +version = "0.1.0" +edition = "2021" + +[features] +guest = [] + +[dependencies] +jolt = { package = "jolt-sdk", path = "../../jolt-sdk" } diff --git a/jolt-eval/guest-sandbox/README.md b/jolt-eval/guest-sandbox/README.md new file mode 100644 index 000000000..82f065f96 --- /dev/null +++ b/jolt-eval/guest-sandbox/README.md @@ -0,0 +1,21 @@ +# guest-sandbox + +Template guest program for the soundness invariant's red-team harness. + +During a red-team session (`cargo run --bin redteam -- --invariant soundness`), an AI agent produces a **unified diff** against this directory. The harness applies the patch in-place via `git apply`, then compiles the patched guest with `jolt build`, proves execution, and checks that the verifier rejects any dishonest output/panic claim. A `PatchGuard` RAII guard reverts the changes (via `git checkout .`) on drop, even if the check panics. + +The default guest computes a simple wrapping hash of the input bytes (`h = 0; for b in input { h = h * 31 + b }`). The agent's goal is to patch it into a program that exposes a soundness bug in Jolt — i.e. one where the verifier accepts a proof paired with an incorrect output or panic flag. + +## Structure + +``` +guest-sandbox/ +├── Cargo.toml # depends on jolt-sdk +└── src/ + ├── lib.rs # #[jolt::provable] function (patch target) + └── main.rs # no_main entry point (rarely needs patching) +``` + +## Safety + +The harness filters patches before applying them: any diff hunk referencing a path containing `..` is dropped, so the agent cannot modify files outside this directory. diff --git a/jolt-eval/guest-sandbox/src/lib.rs b/jolt-eval/guest-sandbox/src/lib.rs new file mode 100644 index 000000000..2f0f1b124 --- /dev/null +++ b/jolt-eval/guest-sandbox/src/lib.rs @@ -0,0 +1,12 @@ +#![cfg_attr(feature = "guest", no_std)] + +#[jolt::provable(heap_size = 32768, stack_size = 65536, max_trace_length = 1048576)] +fn sandbox(input: &[u8]) -> u32 { + // Simple hash — the red-team agent patches this to explore + // code paths that might break soundness. + let mut h: u32 = 0; + for &b in input { + h = h.wrapping_mul(31).wrapping_add(b as u32); + } + h +} diff --git a/jolt-eval/guest-sandbox/src/main.rs b/jolt-eval/guest-sandbox/src/main.rs new file mode 100644 index 000000000..6d8f0a47b --- /dev/null +++ b/jolt-eval/guest-sandbox/src/main.rs @@ -0,0 +1,5 @@ +#![cfg_attr(feature = "guest", no_std)] +#![no_main] + +#[allow(unused_imports)] +use sandbox_guest::*; diff --git a/jolt-eval/macros/Cargo.toml b/jolt-eval/macros/Cargo.toml new file mode 100644 index 000000000..78cd57312 --- /dev/null +++ b/jolt-eval/macros/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "jolt-eval-macros" +version = "0.1.0" +edition = "2021" + +[lib] +proc-macro = true + +[dependencies] +syn = { workspace = true } +quote = { workspace = true } +proc-macro2 = { workspace = true } diff --git a/jolt-eval/macros/src/lib.rs b/jolt-eval/macros/src/lib.rs new file mode 100644 index 000000000..c577e4c08 --- /dev/null +++ b/jolt-eval/macros/src/lib.rs @@ -0,0 +1,155 @@ +extern crate proc_macro; + +use proc_macro::TokenStream; +use quote::quote; +use syn::{parse_macro_input, DeriveInput, Ident}; + +/// Attribute macro for invariant structs. +/// +/// Generates an `InvariantTargets` implementation from the listed targets, +/// and optionally a `#[cfg(test)]` module with tests if `Test` is included. +/// +/// The number of random test iterations defaults to 10 and can be +/// overridden with the `JOLT_RANDOM_ITERS` environment variable. +/// +/// The struct must implement `Invariant + Default`. +/// +/// # Usage +/// +/// ```ignore +/// #[jolt_eval_macros::invariant(Test, Fuzz)] +/// #[derive(Default)] +/// pub struct MyInvariant; +/// ``` +#[proc_macro_attribute] +pub fn invariant(attr: TokenStream, item: TokenStream) -> TokenStream { + let input = parse_macro_input!(item as DeriveInput); + let struct_name = &input.ident; + let snake_name = to_snake_case(&struct_name.to_string()); + let test_mod_name = Ident::new(&format!("{snake_name}_synthesized"), struct_name.span()); + + let targets = parse_targets(attr); + let has_test = targets.contains(&"Test".to_string()); + + // Build the EnumSet expression for InvariantTargets::targets() + let target_exprs: Vec = targets + .iter() + .map(|t| { + let ident = Ident::new(t, proc_macro2::Span::call_site()); + quote! { jolt_eval::SynthesisTarget::#ident } + }) + .collect(); + + let targets_body = if target_exprs.is_empty() { + quote! { enumset::EnumSet::empty() } + } else { + let first = &target_exprs[0]; + let rest = &target_exprs[1..]; + quote! { enumset::EnumSet::only(#first) #(| #rest)* } + }; + + let targets_impl = quote! { + impl jolt_eval::InvariantTargets for #struct_name { + fn targets(&self) -> enumset::EnumSet { + #targets_body + } + } + }; + + let test_block = if has_test { + quote! { + #[cfg(test)] + mod #test_mod_name { + use super::*; + use jolt_eval::Invariant; + + fn assert_no_violation( + name: &str, + result: Result<(), jolt_eval::CheckError>, + context: &str, + ) { + match result { + Ok(()) | Err(jolt_eval::CheckError::InvalidInput(_)) => {} + Err(jolt_eval::CheckError::Violation(v)) => { + panic!("Invariant '{name}' violated {context}: {v}"); + } + } + } + + #[test] + fn seed_corpus() { + let invariant = #struct_name::default(); + let setup = invariant.setup(); + for (i, input) in invariant.seed_corpus().into_iter().enumerate() { + assert_no_violation( + invariant.name(), + invariant.check(&setup, input), + &format!("on seed {i}"), + ); + } + } + + #[test] + fn random_inputs() { + use jolt_eval::rand::RngCore; + let num_iters: usize = std::env::var("JOLT_RANDOM_ITERS") + .ok() + .and_then(|s| s.parse().ok()) + .unwrap_or(10); + let invariant = #struct_name::default(); + let setup = invariant.setup(); + let mut rng = jolt_eval::rand::thread_rng(); + for _ in 0..num_iters { + let mut raw = vec![0u8; 4096]; + rng.fill_bytes(&mut raw); + let mut u = jolt_eval::arbitrary::Unstructured::new(&raw); + if let Ok(input) = < + <#struct_name as jolt_eval::Invariant>::Input + as jolt_eval::arbitrary::Arbitrary + >::arbitrary(&mut u) { + assert_no_violation( + invariant.name(), + invariant.check(&setup, input), + "", + ); + } + } + } + } + } + } else { + quote! {} + }; + + let expanded = quote! { + #input + + #targets_impl + #test_block + }; + + expanded.into() +} + +fn to_snake_case(s: &str) -> String { + let mut result = String::new(); + for (i, c) in s.chars().enumerate() { + if c.is_uppercase() { + if i > 0 { + result.push('_'); + } + result.push(c.to_lowercase().next().unwrap()); + } else { + result.push(c); + } + } + result +} + +fn parse_targets(attr: TokenStream) -> Vec { + attr.to_string() + .split(',') + .map(|s| s.trim().to_string()) + .filter(|s| !s.is_empty()) + .collect() +} diff --git a/jolt-eval/src/agent/claude.rs b/jolt-eval/src/agent/claude.rs new file mode 100644 index 000000000..0c4baf2c8 --- /dev/null +++ b/jolt-eval/src/agent/claude.rs @@ -0,0 +1,235 @@ +use std::path::{Path, PathBuf}; +use std::process::Command; + +use super::{AgentError, AgentHarness, AgentResponse, DiffScope}; + +/// Agent implementation that invokes the Claude Code CLI in an isolated +/// git worktree. +pub struct ClaudeCodeAgent { + pub model: String, + pub max_turns: usize, +} + +impl ClaudeCodeAgent { + pub fn new(model: impl Into, max_turns: usize) -> Self { + Self { + model: model.into(), + max_turns, + } + } + + fn run_cli( + &self, + worktree_dir: &Path, + prompt: &str, + extra_args: &[&str], + verbose: bool, + ) -> Result { + tracing::info!( + "Invoking claude (model={}, max_turns={})...", + self.model, + self.max_turns + ); + let mut cmd = Command::new("claude"); + cmd.current_dir(worktree_dir) + .arg("-p") + .arg(prompt) + .arg("--model") + .arg(&self.model) + .arg("--max-turns") + .arg(self.max_turns.to_string()) + .arg("--dangerously-skip-permissions"); + if verbose { + cmd.arg("--verbose"); + } + for arg in extra_args { + cmd.arg(arg); + } + cmd.output().map_err(|e| { + AgentError::new(format!( + "Failed to invoke claude: {e}. \ + Make sure the `claude` CLI is installed and on your PATH. \ + Install via: npm install -g @anthropic-ai/claude-code" + )) + }) + } +} + +impl AgentHarness for ClaudeCodeAgent { + fn invoke( + &self, + repo_dir: &Path, + prompt: &str, + diff_scope: &DiffScope, + ) -> Result { + let worktree_dir = create_worktree(repo_dir)?; + tracing::info!("Created worktree at {}", worktree_dir.display()); + + let result = self.run_cli(&worktree_dir, prompt, &[], true); + + let diff = capture_diff(&worktree_dir, diff_scope); + + tracing::info!("Cleaning up worktree..."); + remove_worktree(repo_dir, &worktree_dir); + let _ = std::fs::remove_dir_all(&worktree_dir); + + let output = result?; + let stdout = String::from_utf8_lossy(&output.stdout); + let stderr = String::from_utf8_lossy(&output.stderr); + + if !output.status.success() { + tracing::warn!("claude exited with status {}", output.status); + if !stderr.is_empty() { + tracing::warn!("stderr: {}", super::truncate(&stderr, 500)); + } + } + + let text = if stdout.trim().is_empty() { + stderr.to_string() + } else { + stdout.to_string() + }; + + if text.trim().is_empty() && diff.is_none() { + return Err(AgentError::new("Agent produced no output")); + } + + Ok(AgentResponse { text, diff }) + } + + fn invoke_structured( + &self, + repo_dir: &Path, + prompt: &str, + schema: &serde_json::Value, + diff_scope: &DiffScope, + ) -> Result { + let worktree_dir = create_worktree(repo_dir)?; + tracing::info!("Created worktree at {}", worktree_dir.display()); + + let schema_str = serde_json::to_string(schema) + .map_err(|e| AgentError::new(format!("schema serialization: {e}")))?; + + let result = self.run_cli( + &worktree_dir, + prompt, + &["--output-format", "json", "--json-schema", &schema_str], + false, + ); + + let diff = capture_diff(&worktree_dir, diff_scope); + + tracing::info!("Cleaning up worktree..."); + remove_worktree(repo_dir, &worktree_dir); + let _ = std::fs::remove_dir_all(&worktree_dir); + + let output = result?; + let stdout = String::from_utf8_lossy(&output.stdout); + + // Parse the JSON envelope — even on non-zero exit (e.g. max_turns + // reached), Claude may still have produced structured output. + let envelope: serde_json::Value = match serde_json::from_str(&stdout) { + Ok(v) => v, + Err(e) => { + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + let detail = if stderr.trim().is_empty() { + super::truncate(&stdout, 1000) + } else { + super::truncate(&stderr, 1000) + }; + return Err(AgentError::new(format!( + "claude exited with status {}: {}", + output.status, detail + ))); + } + return Err(AgentError::new(format!( + "failed to parse CLI JSON envelope: {e}" + ))); + } + }; + + let text = if let Some(structured) = envelope.get("structured_output") { + serde_json::to_string(structured) + .map_err(|e| AgentError::new(format!("re-serialize structured_output: {e}")))? + } else if let Some(result) = envelope.get("result") { + match result { + serde_json::Value::String(s) => s.clone(), + other => serde_json::to_string(other) + .map_err(|e| AgentError::new(format!("re-serialize result: {e}")))?, + } + } else if !output.status.success() { + let errors = envelope + .get("errors") + .and_then(|e| serde_json::to_string(e).ok()) + .unwrap_or_default(); + return Err(AgentError::new(format!( + "claude exited with status {}: {}", + output.status, errors + ))); + } else { + return Err(AgentError::new( + "CLI JSON envelope contained neither structured_output nor result", + )); + }; + + Ok(AgentResponse { text, diff }) + } +} + +/// Capture a unified diff of changes in a worktree relative to HEAD, +/// filtered by the given [`DiffScope`]. +fn capture_diff(worktree_dir: &Path, scope: &DiffScope) -> Option { + let mut cmd = Command::new("git"); + cmd.current_dir(worktree_dir).args(["diff", "HEAD", "--"]); + match scope { + DiffScope::All => {} + DiffScope::Include(paths) => { + for p in paths { + cmd.arg(p); + } + } + DiffScope::Exclude(paths) => { + for p in paths { + cmd.arg(format!(":!{p}")); + } + } + } + cmd.output().ok().and_then(|o| { + let s = String::from_utf8_lossy(&o.stdout).to_string(); + if s.trim().is_empty() { + None + } else { + Some(s) + } + }) +} + +/// Create an isolated detached git worktree from `repo_dir`. +pub fn create_worktree(repo_dir: &Path) -> Result { + let tmp = tempfile::tempdir().map_err(|e| AgentError::new(format!("tempdir: {e}")))?; + let worktree_dir = tmp.path().to_path_buf(); + std::mem::forget(tmp); + + let status = Command::new("git") + .current_dir(repo_dir) + .args(["worktree", "add", "--detach"]) + .arg(&worktree_dir) + .status() + .map_err(|e| AgentError::new(format!("git worktree: {e}")))?; + + if !status.success() { + return Err(AgentError::new("git worktree add failed")); + } + + Ok(worktree_dir) +} + +/// Remove a git worktree. +pub fn remove_worktree(repo_dir: &Path, worktree_dir: &Path) { + let _ = Command::new("git") + .current_dir(repo_dir) + .args(["worktree", "remove", "--force"]) + .arg(worktree_dir) + .status(); +} diff --git a/jolt-eval/src/agent/mock.rs b/jolt-eval/src/agent/mock.rs new file mode 100644 index 000000000..8fad2804a --- /dev/null +++ b/jolt-eval/src/agent/mock.rs @@ -0,0 +1,91 @@ +use std::path::Path; + +use super::{AgentError, AgentHarness, AgentResponse, DiffScope}; + +/// A mock agent for testing. Returns pre-configured responses and records +/// every prompt it receives. +/// +/// # Usage +/// +/// ```ignore +/// use jolt_eval::agent::{MockAgent, AgentResponse}; +/// +/// // Agent that always succeeds with a fixed response +/// let agent = MockAgent::always_ok("I found nothing."); +/// +/// // Agent that returns a sequence of responses +/// let agent = MockAgent::from_responses(vec![ +/// Ok(AgentResponse { text: "attempt 1".into(), diff: None }), +/// Err(AgentError::new("network timeout")), +/// Ok(AgentResponse { text: "attempt 3".into(), diff: Some("diff".into()) }), +/// ]); +/// +/// // After invoking, inspect the prompts the agent received +/// let prompts = agent.recorded_prompts(); +/// ``` +pub struct MockAgent { + responses: std::sync::Mutex>>, + prompts: std::sync::Mutex>, +} + +impl MockAgent { + /// Create a mock that always returns `Ok` with the given text and no diff. + pub fn always_ok(text: &str) -> Self { + let text = text.to_string(); + Self { + responses: std::sync::Mutex::new(vec![Ok(AgentResponse { text, diff: None })]), + prompts: std::sync::Mutex::new(Vec::new()), + } + } + + /// Create a mock that always returns `Err`. + pub fn always_err(message: &str) -> Self { + Self { + responses: std::sync::Mutex::new(vec![Err(AgentError::new(message))]), + prompts: std::sync::Mutex::new(Vec::new()), + } + } + + /// Create a mock that returns responses from a queue. + /// After the queue is exhausted, subsequent calls return an error. + pub fn from_responses(responses: Vec>) -> Self { + let mut reversed = responses; + reversed.reverse(); // so we can pop from the back + Self { + responses: std::sync::Mutex::new(reversed), + prompts: std::sync::Mutex::new(Vec::new()), + } + } + + /// Return all prompts that were passed to `invoke`, in order. + pub fn recorded_prompts(&self) -> Vec { + self.prompts.lock().unwrap().clone() + } +} + +impl AgentHarness for MockAgent { + fn invoke( + &self, + _repo_dir: &Path, + prompt: &str, + _diff_scope: &DiffScope, + ) -> Result { + self.prompts.lock().unwrap().push(prompt.to_string()); + + let mut responses = self.responses.lock().unwrap(); + if responses.is_empty() { + return Err(AgentError::new("MockAgent: no more responses")); + } + // If only one response left, clone it (repeating) instead of popping + if responses.len() == 1 { + return match &responses[0] { + Ok(r) => Ok(AgentResponse { + text: r.text.clone(), + diff: r.diff.clone(), + }), + Err(e) => Err(AgentError::new(&e.message)), + }; + } + responses.pop().unwrap() + } +} diff --git a/jolt-eval/src/agent/mod.rs b/jolt-eval/src/agent/mod.rs new file mode 100644 index 000000000..13a935d4e --- /dev/null +++ b/jolt-eval/src/agent/mod.rs @@ -0,0 +1,119 @@ +pub mod claude; +pub mod mock; +#[cfg(test)] +mod tests; + +use std::fmt; +use std::path::Path; + +pub use claude::ClaudeCodeAgent; +pub use mock::MockAgent; + +/// Output from an agent invocation. +#[derive(Debug)] +pub struct AgentResponse { + /// The agent's textual output/analysis. + pub text: String, + /// A unified diff of code changes the agent produced, if any. + pub diff: Option, +} + +/// Error during agent invocation. +#[derive(Debug, Clone)] +pub struct AgentError { + pub message: String, +} + +impl fmt::Display for AgentError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.message) + } +} + +impl std::error::Error for AgentError {} + +impl AgentError { + pub fn new(message: impl Into) -> Self { + Self { + message: message.into(), + } + } +} + +/// Git pathspec filter for controlling which files appear in the +/// captured diff after an agent run. +pub enum DiffScope { + /// Capture all changes. + All, + /// Only include changes under these paths. + Include(Vec), + /// Include everything except changes under these paths. + Exclude(Vec), +} + +/// A coding agent that can analyze or modify a repository given a prompt. +/// +/// The `diff_scope` parameter controls which file changes are captured +/// in `AgentResponse::diff` after the agent finishes. +pub trait AgentHarness: Send + Sync { + /// Invoke the agent with a prompt. The agent can read and modify + /// files in its worktree; changes matching `diff_scope` are captured. + fn invoke( + &self, + repo_dir: &Path, + prompt: &str, + diff_scope: &DiffScope, + ) -> Result; + + /// Invoke the agent with a JSON Schema constraint on the response. + /// + /// The default falls back to [`invoke`](Self::invoke). + fn invoke_structured( + &self, + repo_dir: &Path, + prompt: &str, + _schema: &serde_json::Value, + diff_scope: &DiffScope, + ) -> Result { + self.invoke(repo_dir, prompt, diff_scope) + } +} + +/// Apply a unified diff to `repo_dir`. +pub fn apply_diff(repo_dir: &Path, diff: &str) -> Result<(), AgentError> { + use std::process::Command; + + let mut child = Command::new("git") + .current_dir(repo_dir) + .args(["apply", "--allow-empty"]) + .stdin(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .spawn() + .map_err(|e| AgentError::new(format!("git apply spawn: {e}")))?; + + if let Some(stdin) = child.stdin.as_mut() { + use std::io::Write; + let _ = stdin.write_all(diff.as_bytes()); + } + + let output = child + .wait_with_output() + .map_err(|e| AgentError::new(format!("git apply wait: {e}")))?; + + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + return Err(AgentError::new(format!("git apply failed: {stderr}"))); + } + Ok(()) +} + +pub fn truncate(s: &str, max_len: usize) -> &str { + if s.len() <= max_len { + return s; + } + let mut end = max_len; + while end > 0 && !s.is_char_boundary(end) { + end -= 1; + } + &s[..end] +} diff --git a/jolt-eval/src/agent/tests.rs b/jolt-eval/src/agent/tests.rs new file mode 100644 index 000000000..5ebb6f79a --- /dev/null +++ b/jolt-eval/src/agent/tests.rs @@ -0,0 +1,1032 @@ +use std::collections::HashMap; +use std::path::Path; + +use enumset::EnumSet; + +use crate::agent::{AgentError, AgentHarness, AgentResponse, DiffScope, MockAgent}; +use crate::invariant::synthesis::redteam::{auto_redteam, RedTeamConfig, RedTeamResult}; +use crate::invariant::{ + CheckError, Invariant, InvariantTargets, InvariantViolation, SynthesisTarget, +}; +use crate::objective::objective_fn::ObjectiveFunction; +use crate::objective::optimize::{auto_optimize, OptimizeConfig, OptimizeEnv}; +use crate::objective::{OptimizationObjective, HALSTEAD_BUGS, LLOC}; + +// Test invariants + +struct AlwaysPassInvariant; +impl InvariantTargets for AlwaysPassInvariant { + fn targets(&self) -> EnumSet { + SynthesisTarget::Test | SynthesisTarget::RedTeam + } +} +impl Invariant for AlwaysPassInvariant { + type Setup = (); + type Input = u8; + fn name(&self) -> &str { + "always_pass" + } + fn description(&self) -> String { + "This invariant always passes.".into() + } + fn setup(&self) {} + fn check(&self, _: &(), _: u8) -> Result<(), CheckError> { + Ok(()) + } + fn seed_corpus(&self) -> Vec { + vec![0, 1, 255] + } +} + +struct AlwaysFailInvariant; +impl InvariantTargets for AlwaysFailInvariant { + fn targets(&self) -> EnumSet { + SynthesisTarget::Test | SynthesisTarget::RedTeam + } +} +impl Invariant for AlwaysFailInvariant { + type Setup = (); + type Input = u8; + fn name(&self) -> &str { + "always_fail" + } + fn description(&self) -> String { + "This invariant always fails.".into() + } + fn setup(&self) {} + fn check(&self, _: &(), input: u8) -> Result<(), CheckError> { + Err(CheckError::Violation(InvariantViolation::new(format!( + "always fails ({input})" + )))) + } + fn seed_corpus(&self) -> Vec { + vec![42] + } +} + +struct FailsOnZeroInvariant; +impl InvariantTargets for FailsOnZeroInvariant { + fn targets(&self) -> EnumSet { + SynthesisTarget::Test | SynthesisTarget::RedTeam + } +} +impl Invariant for FailsOnZeroInvariant { + type Setup = (); + type Input = u8; + fn name(&self) -> &str { + "fails_on_zero" + } + fn description(&self) -> String { + "Fails when input is 0.".into() + } + fn setup(&self) {} + fn check(&self, _: &(), input: u8) -> Result<(), CheckError> { + if input == 0 { + Err(CheckError::Violation(InvariantViolation::new( + "input was zero", + ))) + } else { + Ok(()) + } + } + fn seed_corpus(&self) -> Vec { + vec![1, 2, 3] + } +} + +// MockAgent tests + +#[test] +fn mock_always_ok_returns_text() { + let agent = MockAgent::always_ok("hello world"); + let resp = agent + .invoke(Path::new("/tmp"), "test prompt", &DiffScope::All) + .unwrap(); + assert_eq!(resp.text, "hello world"); + assert!(resp.diff.is_none()); +} + +#[test] +fn mock_always_err_returns_error() { + let agent = MockAgent::always_err("boom"); + let err = agent + .invoke(Path::new("/tmp"), "test", &DiffScope::All) + .unwrap_err(); + assert_eq!(err.message, "boom"); +} + +#[test] +fn mock_records_prompts() { + let agent = MockAgent::always_ok("ok"); + agent + .invoke(Path::new("/tmp"), "prompt 1", &DiffScope::All) + .unwrap(); + agent + .invoke(Path::new("/tmp"), "prompt 2", &DiffScope::All) + .unwrap(); + agent + .invoke(Path::new("/tmp"), "prompt 3", &DiffScope::All) + .unwrap(); + + let prompts = agent.recorded_prompts(); + assert_eq!(prompts.len(), 3); + assert_eq!(prompts[0], "prompt 1"); + assert_eq!(prompts[1], "prompt 2"); + assert_eq!(prompts[2], "prompt 3"); +} + +#[test] +fn mock_always_ok_repeats_indefinitely() { + let agent = MockAgent::always_ok("same"); + for _ in 0..100 { + let resp = agent + .invoke(Path::new("/tmp"), "x", &DiffScope::All) + .unwrap(); + assert_eq!(resp.text, "same"); + } +} + +#[test] +fn mock_always_err_repeats_indefinitely() { + let agent = MockAgent::always_err("fail"); + for _ in 0..100 { + let err = agent + .invoke(Path::new("/tmp"), "x", &DiffScope::All) + .unwrap_err(); + assert_eq!(err.message, "fail"); + } +} + +#[test] +fn mock_from_responses_returns_in_order() { + let agent = MockAgent::from_responses(vec![ + Ok(AgentResponse { + text: "first".into(), + diff: None, + }), + Ok(AgentResponse { + text: "second".into(), + diff: Some("diff".into()), + }), + Err(AgentError::new("third fails")), + ]); + + let r1 = agent + .invoke(Path::new("/tmp"), "a", &DiffScope::All) + .unwrap(); + assert_eq!(r1.text, "first"); + assert!(r1.diff.is_none()); + + let r2 = agent + .invoke(Path::new("/tmp"), "b", &DiffScope::All) + .unwrap(); + assert_eq!(r2.text, "second"); + assert_eq!(r2.diff.as_deref(), Some("diff")); + + let r3 = agent + .invoke(Path::new("/tmp"), "c", &DiffScope::All) + .unwrap_err(); + assert_eq!(r3.message, "third fails"); +} + +#[test] +fn mock_from_responses_last_entry_repeats() { + let agent = MockAgent::from_responses(vec![ + Ok(AgentResponse { + text: "first".into(), + diff: None, + }), + Ok(AgentResponse { + text: "last".into(), + diff: None, + }), + ]); + + agent + .invoke(Path::new("/tmp"), "a", &DiffScope::All) + .unwrap(); + let r2 = agent + .invoke(Path::new("/tmp"), "b", &DiffScope::All) + .unwrap(); + assert_eq!(r2.text, "last"); + let r3 = agent + .invoke(Path::new("/tmp"), "c", &DiffScope::All) + .unwrap(); + assert_eq!(r3.text, "last"); +} + +#[test] +fn mock_with_diff() { + let agent = MockAgent::from_responses(vec![Ok(AgentResponse { + text: "I optimized the code".into(), + diff: Some("--- a/foo\n+++ b/foo\n@@ ...\n-old\n+new".into()), + })]); + + let resp = agent + .invoke(Path::new("/tmp"), "optimize", &DiffScope::All) + .unwrap(); + assert!(resp.diff.is_some()); + assert!(resp.diff.unwrap().contains("+new")); +} + +// auto_redteam tests with MockAgent + +fn envelope(analysis: &str, counterexample: impl serde::Serialize) -> String { + serde_json::json!({ + "analysis": analysis, + "counterexample": counterexample, + }) + .to_string() +} + +#[test] +fn redteam_no_violation_when_invariant_always_passes() { + let invariant = AlwaysPassInvariant; + let agent = MockAgent::always_ok(&envelope("I analyzed the code.", 42)); + let config = RedTeamConfig { + num_iterations: 3, + ..Default::default() + }; + + let result = auto_redteam(&invariant, &config, &agent, Path::new("/tmp")); + + match result { + RedTeamResult::NoViolation { attempts } => { + assert_eq!(attempts.len(), 3); + for a in &attempts { + assert!(a.failure_reason.contains("did not violate")); + } + } + RedTeamResult::Violation { .. } => { + panic!("Expected no violation for AlwaysPassInvariant"); + } + } + + assert_eq!(agent.recorded_prompts().len(), 3); +} + +#[test] +fn redteam_finds_violation_with_structured_response() { + let invariant = AlwaysFailInvariant; + let agent = MockAgent::always_ok(&envelope("I found a bug!", 99)); + let config = RedTeamConfig { + num_iterations: 10, + ..Default::default() + }; + + let result = auto_redteam(&invariant, &config, &agent, Path::new("/tmp")); + + match result { + RedTeamResult::Violation { + input_json, error, .. + } => { + assert_eq!(input_json, "99"); + assert!(error.contains("always fails")); + } + RedTeamResult::NoViolation { .. } => { + panic!("Expected violation for AlwaysFailInvariant"); + } + } + + assert_eq!(agent.recorded_prompts().len(), 1); +} + +#[test] +fn redteam_finds_violation_with_targeted_input() { + let invariant = FailsOnZeroInvariant; + let agent = MockAgent::always_ok(&envelope("Try zero", 0)); + let config = RedTeamConfig { + num_iterations: 5, + ..Default::default() + }; + + let result = auto_redteam(&invariant, &config, &agent, Path::new("/tmp")); + + match result { + RedTeamResult::Violation { + input_json, error, .. + } => { + assert_eq!(input_json, "0"); + assert!(error.contains("zero")); + } + RedTeamResult::NoViolation { .. } => { + panic!("Expected violation for FailsOnZeroInvariant with input 0"); + } + } +} + +#[test] +fn redteam_no_violation_when_agent_misses() { + let invariant = FailsOnZeroInvariant; + let agent = MockAgent::always_ok(&envelope("Trying 1", 1)); + let config = RedTeamConfig { + num_iterations: 2, + ..Default::default() + }; + + let result = auto_redteam(&invariant, &config, &agent, Path::new("/tmp")); + + match result { + RedTeamResult::NoViolation { attempts } => { + assert_eq!(attempts.len(), 2); + for a in &attempts { + assert!(a.failure_reason.contains("did not violate")); + } + } + _ => panic!("Expected NoViolation since agent never guesses 0"), + } +} + +#[test] +fn redteam_handles_agent_errors_gracefully() { + let invariant = AlwaysPassInvariant; + let agent = MockAgent::always_err("network timeout"); + let config = RedTeamConfig { + num_iterations: 3, + ..Default::default() + }; + + let result = auto_redteam(&invariant, &config, &agent, Path::new("/tmp")); + + match result { + RedTeamResult::NoViolation { attempts } => { + assert_eq!(attempts.len(), 3); + for a in &attempts { + assert_eq!(a.approach, "Agent invocation failed"); + assert!(a.failure_reason.contains("network timeout")); + } + } + RedTeamResult::Violation { .. } => { + panic!("Expected no violation when agent always errors"); + } + } +} + +#[test] +fn redteam_handles_no_json_in_response() { + let invariant = AlwaysPassInvariant; + let agent = MockAgent::always_ok("I looked around but have no candidate to offer."); + let config = RedTeamConfig { + num_iterations: 1, + ..Default::default() + }; + + let result = auto_redteam(&invariant, &config, &agent, Path::new("/tmp")); + + match result { + RedTeamResult::NoViolation { attempts } => { + assert_eq!(attempts.len(), 1); + assert!(attempts[0] + .failure_reason + .contains("did not contain valid JSON")); + } + _ => panic!("Expected NoViolation"), + } +} + +#[test] +fn redteam_handles_invalid_counterexample_type() { + let invariant = AlwaysPassInvariant; + let agent = MockAgent::always_ok(&envelope("Here", "not_a_number")); + let config = RedTeamConfig { + num_iterations: 1, + ..Default::default() + }; + + let result = auto_redteam(&invariant, &config, &agent, Path::new("/tmp")); + + match result { + RedTeamResult::NoViolation { attempts } => { + assert_eq!(attempts.len(), 1); + assert!(attempts[0].failure_reason.contains("Could not deserialize")); + } + RedTeamResult::Violation { .. } => { + panic!("Parse error should not be treated as a violation"); + } + } +} + +#[test] +fn redteam_fallback_extracts_json_from_freeform_text() { + let invariant = AlwaysFailInvariant; + let agent = MockAgent::always_ok("Found it!\n```json\n77\n```"); + let config = RedTeamConfig { + num_iterations: 1, + ..Default::default() + }; + + let result = auto_redteam(&invariant, &config, &agent, Path::new("/tmp")); + + match result { + RedTeamResult::Violation { input_json, .. } => { + assert_eq!(input_json, "77"); + } + _ => panic!("Expected violation via extract_json fallback"), + } +} + +#[test] +fn redteam_prompt_includes_invariant_description() { + let invariant = AlwaysPassInvariant; + let agent = MockAgent::always_ok(&envelope("ok", 0)); + let config = RedTeamConfig { + num_iterations: 1, + ..Default::default() + }; + + auto_redteam(&invariant, &config, &agent, Path::new("/tmp")); + + let prompts = agent.recorded_prompts(); + assert_eq!(prompts.len(), 1); + assert!(prompts[0].contains("This invariant always passes.")); + assert!(prompts[0].contains("VIOLATES")); +} + +#[test] +fn redteam_prompt_includes_input_example() { + let invariant = AlwaysPassInvariant; + let agent = MockAgent::always_ok(&envelope("ok", 0)); + let config = RedTeamConfig { + num_iterations: 1, + ..Default::default() + }; + + auto_redteam(&invariant, &config, &agent, Path::new("/tmp")); + + let prompts = agent.recorded_prompts(); + assert!(prompts[0].contains("Input format")); + assert!(prompts[0].contains("```json")); +} + +#[test] +fn redteam_prompt_includes_failed_attempts_after_first_iteration() { + let invariant = AlwaysPassInvariant; + let agent = MockAgent::always_ok(&envelope("Tried something", 42)); + let config = RedTeamConfig { + num_iterations: 3, + ..Default::default() + }; + + auto_redteam(&invariant, &config, &agent, Path::new("/tmp")); + + let prompts = agent.recorded_prompts(); + assert_eq!(prompts.len(), 3); + + assert!(!prompts[0].contains("Previous failed attempts")); + assert!(prompts[1].contains("Previous failed attempts")); + assert!(prompts[2].contains("Iteration 1")); + assert!(prompts[2].contains("Iteration 2")); +} + +#[test] +fn redteam_zero_iterations_returns_immediately() { + let invariant = AlwaysPassInvariant; + let agent = MockAgent::always_ok("should not be called"); + let config = RedTeamConfig { + num_iterations: 0, + ..Default::default() + }; + + let result = auto_redteam(&invariant, &config, &agent, Path::new("/tmp")); + + match result { + RedTeamResult::NoViolation { attempts } => { + assert!(attempts.is_empty()); + } + _ => panic!("Expected NoViolation with empty attempts"), + } + + assert!(agent.recorded_prompts().is_empty()); +} + +#[test] +fn redteam_mixed_agent_responses() { + let invariant = AlwaysPassInvariant; + let agent = MockAgent::from_responses(vec![ + Ok(AgentResponse { + text: envelope("first try", 1), + diff: None, + }), + Err(AgentError::new("transient error")), + Ok(AgentResponse { + text: envelope("third try", 3), + diff: None, + }), + ]); + let config = RedTeamConfig { + num_iterations: 3, + ..Default::default() + }; + + let result = auto_redteam(&invariant, &config, &agent, Path::new("/tmp")); + + match result { + RedTeamResult::NoViolation { attempts } => { + assert_eq!(attempts.len(), 3); + assert!(attempts[0].approach.contains("first try")); + assert_eq!(attempts[1].approach, "Agent invocation failed"); + assert!(attempts[2].approach.contains("third try")); + } + _ => panic!("Expected NoViolation"), + } +} + +// AgentHarness trait object tests + +#[test] +fn agent_harness_is_object_safe() { + let agent: Box = Box::new(MockAgent::always_ok("hi")); + let resp = agent + .invoke(Path::new("/tmp"), "hello", &DiffScope::All) + .unwrap(); + assert_eq!(resp.text, "hi"); +} + +#[test] +fn agent_harness_works_with_arc() { + use std::sync::Arc; + let agent: Arc = Arc::new(MockAgent::always_ok("shared")); + let resp = agent + .invoke(Path::new("/tmp"), "test", &DiffScope::All) + .unwrap(); + assert_eq!(resp.text, "shared"); +} + +struct FirstSuccessHarness { + agents: Vec>, +} + +impl AgentHarness for FirstSuccessHarness { + fn invoke( + &self, + repo_dir: &Path, + prompt: &str, + diff_scope: &DiffScope, + ) -> Result { + for agent in &self.agents { + if let Ok(resp) = agent.invoke(repo_dir, prompt, diff_scope) { + return Ok(resp); + } + } + Err(AgentError::new("All agents failed")) + } +} + +#[test] +fn custom_multi_agent_harness() { + let harness = FirstSuccessHarness { + agents: vec![ + Box::new(MockAgent::always_err("agent 1 down")), + Box::new(MockAgent::always_err("agent 2 down")), + Box::new(MockAgent::always_ok("agent 3 succeeded")), + ], + }; + + let resp = harness + .invoke(Path::new("/tmp"), "test", &DiffScope::All) + .unwrap(); + assert_eq!(resp.text, "agent 3 succeeded"); +} + +#[test] +fn custom_multi_agent_all_fail() { + let harness = FirstSuccessHarness { + agents: vec![ + Box::new(MockAgent::always_err("nope")), + Box::new(MockAgent::always_err("nope")), + ], + }; + + let err = harness + .invoke(Path::new("/tmp"), "test", &DiffScope::All) + .unwrap_err(); + assert_eq!(err.message, "All agents failed"); +} + +#[test] +fn custom_harness_plugs_into_auto_redteam() { + let harness = FirstSuccessHarness { + agents: vec![ + Box::new(MockAgent::always_err("agent 1 down")), + Box::new(MockAgent::always_ok(&envelope("agent 2 found nothing", 7))), + ], + }; + + let invariant = AlwaysPassInvariant; + let config = RedTeamConfig { + num_iterations: 2, + ..Default::default() + }; + + let result = auto_redteam(&invariant, &config, &harness, Path::new("/tmp")); + + match result { + RedTeamResult::NoViolation { attempts } => { + assert_eq!(attempts.len(), 2); + assert!(attempts[0].approach.contains("agent 2 found nothing")); + } + _ => panic!("Expected NoViolation"), + } +} + +// Mock OptimizeEnv + +fn lloc() -> OptimizationObjective { + LLOC +} + +fn halstead() -> OptimizationObjective { + HALSTEAD_BUGS +} + +struct MockOptimizeEnv { + measurements: Vec>, + measure_index: usize, + invariants_pass: Vec, + invariant_index: usize, + applied_diffs: Vec, + accepted: Vec, + rejected: usize, +} + +impl MockOptimizeEnv { + fn new() -> Self { + Self { + measurements: vec![], + measure_index: 0, + invariants_pass: vec![true], + invariant_index: 0, + applied_diffs: vec![], + accepted: vec![], + rejected: 0, + } + } + + fn with_measurements(mut self, measurements: Vec>) -> Self { + self.measurements = measurements; + self + } + + fn with_invariants(mut self, pass: Vec) -> Self { + self.invariants_pass = pass; + self + } +} + +impl OptimizeEnv for MockOptimizeEnv { + fn measure(&mut self) -> HashMap { + if self.measurements.is_empty() { + return HashMap::new(); + } + let idx = self.measure_index.min(self.measurements.len() - 1); + self.measure_index += 1; + self.measurements[idx].clone() + } + + fn check_invariants(&mut self) -> bool { + if self.invariants_pass.is_empty() { + return true; + } + let idx = self.invariant_index.min(self.invariants_pass.len() - 1); + self.invariant_index += 1; + self.invariants_pass[idx] + } + + fn apply_diff(&mut self, diff: &str) { + self.applied_diffs.push(diff.to_string()); + } + + fn accept(&mut self, iteration: usize) { + self.accepted.push(iteration); + } + + fn reject(&mut self) { + self.rejected += 1; + } +} + +fn m(pairs: &[(OptimizationObjective, f64)]) -> HashMap { + pairs.iter().cloned().collect() +} + +fn lloc_obj() -> ObjectiveFunction { + const INPUTS: &[OptimizationObjective] = &[LLOC]; + ObjectiveFunction { + name: "test_lloc", + inputs: INPUTS, + evaluate: |m, _| m.get(&LLOC).copied().unwrap_or(f64::INFINITY), + } +} + +fn opt_config(iterations: usize) -> OptimizeConfig { + OptimizeConfig { + num_iterations: iterations, + ..Default::default() + } +} + +// auto_optimize tests + +#[test] +fn optimize_accepts_improvement() { + let agent = MockAgent::from_responses(vec![Ok(AgentResponse { + text: "I optimized X".into(), + diff: Some("fake diff".into()), + })]); + + let mut env = + MockOptimizeEnv::new().with_measurements(vec![m(&[(lloc(), 10.0)]), m(&[(lloc(), 8.0)])]); + + let config = opt_config(1); + let obj = lloc_obj(); + let result = auto_optimize(&agent, &mut env, &obj, &config, Path::new("/tmp")); + + assert_eq!(result.attempts.len(), 1); + assert!(result.attempts[0].invariants_passed); + assert_eq!(result.best_score, 8.0); + assert_eq!(env.accepted, vec![1]); + assert_eq!(env.rejected, 0); +} + +#[test] +fn optimize_rejects_regression() { + let agent = MockAgent::from_responses(vec![Ok(AgentResponse { + text: "I tried something".into(), + diff: Some("bad diff".into()), + })]); + + let mut env = + MockOptimizeEnv::new().with_measurements(vec![m(&[(lloc(), 10.0)]), m(&[(lloc(), 12.0)])]); + + let config = opt_config(1); + let obj = lloc_obj(); + let result = auto_optimize(&agent, &mut env, &obj, &config, Path::new("/tmp")); + + assert_eq!(result.attempts.len(), 1); + assert_eq!(result.best_score, 10.0); + assert!(env.accepted.is_empty()); + assert_eq!(env.rejected, 1); +} + +#[test] +fn optimize_rejects_when_invariants_fail() { + let agent = MockAgent::from_responses(vec![Ok(AgentResponse { + text: "I broke something".into(), + diff: Some("breaking diff".into()), + })]); + + let mut env = MockOptimizeEnv::new() + .with_measurements(vec![m(&[(lloc(), 10.0)]), m(&[(lloc(), 5.0)])]) + .with_invariants(vec![false]); + + let config = opt_config(1); + let obj = lloc_obj(); + let result = auto_optimize(&agent, &mut env, &obj, &config, Path::new("/tmp")); + + assert!(!result.attempts[0].invariants_passed); + assert_eq!(result.best_score, 10.0); + assert!(env.accepted.is_empty()); + assert_eq!(env.rejected, 1); +} + +#[test] +fn optimize_custom_objective_function() { + const INPUTS: &[OptimizationObjective] = &[LLOC, HALSTEAD_BUGS]; + let weighted = ObjectiveFunction { + name: "weighted", + inputs: INPUTS, + evaluate: |m, _| 2.0 * m.get(&LLOC).unwrap_or(&0.0) + m.get(&HALSTEAD_BUGS).unwrap_or(&0.0), + }; + + let agent = MockAgent::from_responses(vec![Ok(AgentResponse { + text: "optimized".into(), + diff: Some("diff".into()), + })]); + + let mut env = MockOptimizeEnv::new().with_measurements(vec![ + m(&[(lloc(), 10.0), (halstead(), 100.0)]), // score = 120 + m(&[(lloc(), 8.0), (halstead(), 110.0)]), // score = 126 (regression!) + ]); + + let config = opt_config(1); + let result = auto_optimize(&agent, &mut env, &weighted, &config, Path::new("/tmp")); + + assert_eq!(result.best_score, 120.0); + assert!(env.accepted.is_empty()); + assert_eq!(env.rejected, 1); +} + +#[test] +fn optimize_multi_iteration_progressive_improvement() { + let agent = MockAgent::from_responses(vec![ + Ok(AgentResponse { + text: "iter 1".into(), + diff: Some("diff1".into()), + }), + Ok(AgentResponse { + text: "iter 2".into(), + diff: Some("diff2".into()), + }), + Ok(AgentResponse { + text: "iter 3".into(), + diff: Some("diff3".into()), + }), + ]); + + let mut env = MockOptimizeEnv::new().with_measurements(vec![ + m(&[(lloc(), 10.0)]), + m(&[(lloc(), 8.0)]), + m(&[(lloc(), 9.0)]), + m(&[(lloc(), 6.0)]), + ]); + + let config = opt_config(3); + let obj = lloc_obj(); + let result = auto_optimize(&agent, &mut env, &obj, &config, Path::new("/tmp")); + + assert_eq!(result.attempts.len(), 3); + assert_eq!(result.best_score, 6.0); + assert_eq!(env.accepted, vec![1, 3]); + assert_eq!(env.rejected, 1); +} + +#[test] +fn optimize_stops_when_agent_produces_no_diff() { + let agent = MockAgent::from_responses(vec![ + Ok(AgentResponse { + text: "changed".into(), + diff: Some("diff1".into()), + }), + Ok(AgentResponse { + text: "nothing else".into(), + diff: None, + }), + ]); + + let mut env = + MockOptimizeEnv::new().with_measurements(vec![m(&[(lloc(), 10.0)]), m(&[(lloc(), 9.0)])]); + + let config = opt_config(5); + let obj = lloc_obj(); + let result = auto_optimize(&agent, &mut env, &obj, &config, Path::new("/tmp")); + + assert_eq!(result.attempts.len(), 1); +} + +#[test] +fn optimize_stops_when_agent_errors() { + let agent = MockAgent::from_responses(vec![ + Ok(AgentResponse { + text: "change".into(), + diff: Some("diff".into()), + }), + Err(AgentError::new("agent crashed")), + ]); + + let mut env = + MockOptimizeEnv::new().with_measurements(vec![m(&[(lloc(), 10.0)]), m(&[(lloc(), 10.0)])]); + + let config = opt_config(5); + let obj = lloc_obj(); + let result = auto_optimize(&agent, &mut env, &obj, &config, Path::new("/tmp")); + + assert_eq!(result.attempts.len(), 1); +} + +#[test] +fn optimize_zero_iterations() { + let agent = MockAgent::always_ok("should not be called"); + let mut env = MockOptimizeEnv::new().with_measurements(vec![m(&[(lloc(), 10.0)])]); + + let config = opt_config(0); + let obj = lloc_obj(); + let result = auto_optimize(&agent, &mut env, &obj, &config, Path::new("/tmp")); + + assert!(result.attempts.is_empty()); + assert_eq!(result.baseline_score, 10.0); + assert_eq!(result.best_score, 10.0); + assert!(agent.recorded_prompts().is_empty()); +} + +#[test] +fn optimize_prompt_includes_measurements_and_hint() { + let agent = MockAgent::from_responses(vec![Ok(AgentResponse { + text: "done".into(), + diff: Some("diff".into()), + })]); + + let mut env = + MockOptimizeEnv::new().with_measurements(vec![m(&[(lloc(), 42.0)]), m(&[(lloc(), 42.0)])]); + + let config = OptimizeConfig { + num_iterations: 1, + hint: Some("Focus on the inner loop".into()), + ..Default::default() + }; + let obj = lloc_obj(); + auto_optimize(&agent, &mut env, &obj, &config, Path::new("/tmp")); + + let prompts = agent.recorded_prompts(); + assert_eq!(prompts.len(), 1); + assert!(prompts[0].contains("42.0")); + assert!(prompts[0].contains("Focus on the inner loop")); +} + +#[test] +fn optimize_prompt_includes_past_attempts() { + let agent = MockAgent::from_responses(vec![ + Ok(AgentResponse { + text: "attempt 1".into(), + diff: Some("d1".into()), + }), + Ok(AgentResponse { + text: "attempt 2".into(), + diff: Some("d2".into()), + }), + ]); + + let mut env = MockOptimizeEnv::new().with_measurements(vec![ + m(&[(lloc(), 10.0)]), + m(&[(lloc(), 10.0)]), + m(&[(lloc(), 10.0)]), + ]); + + let config = opt_config(2); + let obj = lloc_obj(); + auto_optimize(&agent, &mut env, &obj, &config, Path::new("/tmp")); + + let prompts = agent.recorded_prompts(); + assert_eq!(prompts.len(), 2); + assert!(!prompts[0].contains("Previous attempts")); + assert!(prompts[1].contains("Previous attempts")); + assert!(prompts[1].contains("Iteration 1")); +} + +#[test] +fn optimize_diff_is_applied() { + let agent = MockAgent::from_responses(vec![Ok(AgentResponse { + text: "changed something".into(), + diff: Some("--- a/x\n+++ b/x\n".into()), + })]); + + let mut env = + MockOptimizeEnv::new().with_measurements(vec![m(&[(lloc(), 10.0)]), m(&[(lloc(), 10.0)])]); + + let config = opt_config(1); + let obj = lloc_obj(); + auto_optimize(&agent, &mut env, &obj, &config, Path::new("/tmp")); + + assert_eq!(env.applied_diffs.len(), 1); + assert!(env.applied_diffs[0].contains("--- a/x")); +} + +#[test] +fn optimize_invariant_failure_mid_sequence() { + let agent = MockAgent::from_responses(vec![ + Ok(AgentResponse { + text: "i1".into(), + diff: Some("d1".into()), + }), + Ok(AgentResponse { + text: "i2".into(), + diff: Some("d2".into()), + }), + Ok(AgentResponse { + text: "i3".into(), + diff: Some("d3".into()), + }), + ]); + + let mut env = MockOptimizeEnv::new() + .with_measurements(vec![ + m(&[(lloc(), 10.0)]), + m(&[(lloc(), 8.0)]), + m(&[(lloc(), 5.0)]), + m(&[(lloc(), 7.0)]), + ]) + .with_invariants(vec![true, false, true]); + + let config = opt_config(3); + let obj = lloc_obj(); + let result = auto_optimize(&agent, &mut env, &obj, &config, Path::new("/tmp")); + + assert_eq!(result.attempts.len(), 3); + assert!(result.attempts[0].invariants_passed); + assert!(!result.attempts[1].invariants_passed); + assert!(result.attempts[2].invariants_passed); + assert_eq!(env.accepted, vec![1, 3]); + assert_eq!(env.rejected, 1); + assert_eq!(result.best_score, 7.0); +} + +#[test] +fn objective_function_struct_evaluates() { + let obj = lloc_obj(); + let mut m = HashMap::new(); + m.insert(lloc(), 3.5); + let b = HashMap::new(); + assert_eq!((obj.evaluate)(&m, &b), 3.5); + + let empty = HashMap::new(); + assert_eq!((obj.evaluate)(&empty, &b), f64::INFINITY); +} diff --git a/jolt-eval/src/guests/fibonacci.rs b/jolt-eval/src/guests/fibonacci.rs new file mode 100644 index 000000000..69b161505 --- /dev/null +++ b/jolt-eval/src/guests/fibonacci.rs @@ -0,0 +1,22 @@ +use super::GuestConfig; + +/// Fibonacci guest: computes fib(n). +pub struct Fibonacci(pub u32); + +impl Default for Fibonacci { + fn default() -> Self { + Self(100) + } +} + +impl GuestConfig for Fibonacci { + fn package(&self) -> &str { + "fibonacci-guest" + } + fn input(&self) -> Vec { + postcard::to_stdvec(&self.0).unwrap() + } + fn bench_name(&self) -> String { + format!("prover_time_fibonacci_{}", self.0) + } +} diff --git a/jolt-eval/src/guests/mod.rs b/jolt-eval/src/guests/mod.rs new file mode 100644 index 000000000..dbd178688 --- /dev/null +++ b/jolt-eval/src/guests/mod.rs @@ -0,0 +1,127 @@ +pub mod fibonacci; +pub mod secp256k1_ecdsa; +pub mod sha2_chain; + +use ark_bn254::Fr; +use jolt_core::curve::Bn254Curve; +use jolt_core::poly::commitment::dory::DoryCommitmentScheme; +use jolt_core::transcripts::Blake2bTranscript; + +use common::constants::{DEFAULT_MAX_TRUSTED_ADVICE_SIZE, DEFAULT_MAX_UNTRUSTED_ADVICE_SIZE}; +use common::jolt_device::MemoryConfig; + +pub use fibonacci::Fibonacci; +pub use jolt_core::guest::program::Program as GuestProgram; +pub use jolt_core::utils::errors::ProofVerifyError; +pub use secp256k1_ecdsa::Secp256k1EcdsaVerify; +pub use sha2_chain::Sha2Chain; +pub use tracer::JoltDevice; + +pub type F = Fr; +pub type C = Bn254Curve; +pub type PCS = DoryCommitmentScheme; +pub type FS = Blake2bTranscript; + +pub type Proof = jolt_core::zkvm::proof_serialization::JoltProof; +pub type ProverPreprocessing = jolt_core::zkvm::prover::JoltProverPreprocessing; +pub type VerifierPreprocessing = jolt_core::zkvm::verifier::JoltVerifierPreprocessing; + +pub fn prover_preprocessing( + program: &GuestProgram, + max_trace_length: usize, +) -> ProverPreprocessing { + jolt_core::guest::prover::preprocess(program, max_trace_length) + .expect("prover preprocessing failed") +} + +pub fn verifier_preprocessing(prover_pp: &ProverPreprocessing) -> VerifierPreprocessing { + VerifierPreprocessing::from(prover_pp) +} + +pub fn prove( + program: &GuestProgram, + prover_pp: &ProverPreprocessing, + inputs: &[u8], +) -> (Proof, JoltDevice) { + let mut output_bytes = vec![0u8; program.memory_config.max_output_size as usize]; + let (proof, io_device, _debug) = jolt_core::guest::prover::prove::( + program, + inputs, + &[], + &[], + None, + None, + &mut output_bytes, + prover_pp, + ); + (proof, io_device) +} + +pub fn verify( + verifier_pp: &VerifierPreprocessing, + proof: Proof, + io_device: &JoltDevice, +) -> Result<(), ProofVerifyError> { + jolt_core::guest::verifier::verify::( + &io_device.inputs, + None, + &io_device.outputs, + proof, + verifier_pp, + ) +} + +/// Verify a proof against claimed (potentially malicious) outputs and panic flag. +pub fn verify_with_claims( + verifier_pp: &VerifierPreprocessing, + proof: Proof, + inputs: &[u8], + claimed_outputs: &[u8], + claimed_panic: bool, +) -> Result<(), ProofVerifyError> { + use jolt_core::zkvm::verifier::JoltVerifier; + + let memory_layout = &verifier_pp.shared.memory_layout; + let memory_config = MemoryConfig { + max_untrusted_advice_size: memory_layout.max_untrusted_advice_size, + max_trusted_advice_size: memory_layout.max_trusted_advice_size, + max_input_size: memory_layout.max_input_size, + max_output_size: memory_layout.max_output_size, + stack_size: memory_layout.stack_size, + heap_size: memory_layout.heap_size, + program_size: Some(memory_layout.program_size), + }; + let mut io_device = JoltDevice::new(&memory_config); + io_device.inputs = inputs.to_vec(); + io_device.outputs = claimed_outputs.to_vec(); + io_device.panic = claimed_panic; + + let verifier = JoltVerifier::::new(verifier_pp, proof, io_device, None, None)?; + verifier.verify() +} + +// ── GuestConfig ───────────────────────────────────────────────────── + +/// Trait for configuring which guest program to benchmark. +pub trait GuestConfig: Default + Send + Sync { + /// Cargo package name (e.g. "fibonacci-guest"). + fn package(&self) -> &str; + + fn memory_config(&self) -> MemoryConfig { + MemoryConfig { + max_input_size: 4096, + max_output_size: 4096, + max_untrusted_advice_size: DEFAULT_MAX_UNTRUSTED_ADVICE_SIZE, + max_trusted_advice_size: DEFAULT_MAX_TRUSTED_ADVICE_SIZE, + stack_size: 4096, + heap_size: 32768, + program_size: None, + } + } + + /// Serialized program input (postcard-encoded). + fn input(&self) -> Vec; + + /// Display name for the benchmark. + fn bench_name(&self) -> String; +} diff --git a/jolt-eval/src/guests/secp256k1_ecdsa.rs b/jolt-eval/src/guests/secp256k1_ecdsa.rs new file mode 100644 index 000000000..e4675921e --- /dev/null +++ b/jolt-eval/src/guests/secp256k1_ecdsa.rs @@ -0,0 +1,73 @@ +use common::jolt_device::MemoryConfig; + +use super::GuestConfig; + +/// Secp256k1 ECDSA signature verification guest. +pub struct Secp256k1EcdsaVerify { + pub z: [u64; 4], + pub r: [u64; 4], + pub s: [u64; 4], + pub q: [u64; 8], +} + +impl Default for Secp256k1EcdsaVerify { + fn default() -> Self { + // Test vector from examples/secp256k1-ecdsa-verify: "hello world" + Self { + z: [ + 0x9088f7ace2efcde9, + 0xc484efe37a5380ee, + 0xa52e52d7da7dabfa, + 0xb94d27b9934d3e08, + ], + r: [ + 0xb8fc413b4b967ed8, + 0x248d4b0b2829ab00, + 0x587f69296af3cd88, + 0x3a5d6a386e6cf7c0, + ], + s: [ + 0x66a82f274e3dcafc, + 0x299a02486be40321, + 0x6212d714118f617e, + 0x9d452f63cf91018d, + ], + q: [ + 0x0012563f32ed0216, + 0xee00716af6a73670, + 0x91fc70e34e00e6c8, + 0xeeb6be8b9e68868b, + 0x4780de3d5fda972d, + 0xcb1b42d72491e47f, + 0xdc7f31262e4ba2b7, + 0xdc7b004d3bb2800d, + ], + } + } +} + +impl GuestConfig for Secp256k1EcdsaVerify { + fn package(&self) -> &str { + "secp256k1-ecdsa-verify-guest" + } + fn memory_config(&self) -> MemoryConfig { + use common::constants::{ + DEFAULT_MAX_TRUSTED_ADVICE_SIZE, DEFAULT_MAX_UNTRUSTED_ADVICE_SIZE, + }; + MemoryConfig { + max_input_size: 4096, + max_output_size: 4096, + max_untrusted_advice_size: DEFAULT_MAX_UNTRUSTED_ADVICE_SIZE, + max_trusted_advice_size: DEFAULT_MAX_TRUSTED_ADVICE_SIZE, + stack_size: 4096, + heap_size: 100000, + program_size: None, + } + } + fn input(&self) -> Vec { + postcard::to_stdvec(&(self.z, self.r, self.s, self.q)).unwrap() + } + fn bench_name(&self) -> String { + "prover_time_secp256k1_ecdsa_verify".to_string() + } +} diff --git a/jolt-eval/src/guests/sha2_chain.rs b/jolt-eval/src/guests/sha2_chain.rs new file mode 100644 index 000000000..f4c14eb59 --- /dev/null +++ b/jolt-eval/src/guests/sha2_chain.rs @@ -0,0 +1,28 @@ +use super::GuestConfig; + +/// SHA-2 chain guest: iteratively hashes input `num_iters` times. +pub struct Sha2Chain { + pub input: [u8; 32], + pub num_iters: u32, +} + +impl Default for Sha2Chain { + fn default() -> Self { + Self { + input: [5u8; 32], + num_iters: 100, + } + } +} + +impl GuestConfig for Sha2Chain { + fn package(&self) -> &str { + "sha2-chain-guest" + } + fn input(&self) -> Vec { + postcard::to_stdvec(&(self.input, self.num_iters)).unwrap() + } + fn bench_name(&self) -> String { + format!("prover_time_sha2_chain_{}", self.num_iters) + } +} diff --git a/jolt-eval/src/invariant/macro_tests.rs b/jolt-eval/src/invariant/macro_tests.rs new file mode 100644 index 000000000..f0d430e6c --- /dev/null +++ b/jolt-eval/src/invariant/macro_tests.rs @@ -0,0 +1,86 @@ +use crate::invariant::{CheckError, Invariant, InvariantViolation}; + +// AlwaysPass: trivial invariant to test macro synthesis + +#[jolt_eval_macros::invariant(Test, Fuzz, RedTeam)] +#[derive(Default)] +pub struct AlwaysPassInvariant; + +impl Invariant for AlwaysPassInvariant { + type Setup = (); + type Input = u8; + + fn name(&self) -> &str { + "always_pass" + } + fn description(&self) -> String { + "Trivial invariant that always passes — used to test macro synthesis.".to_string() + } + fn setup(&self) -> Self::Setup {} + fn check(&self, _: &(), _input: u8) -> Result<(), CheckError> { + Ok(()) + } + fn seed_corpus(&self) -> Vec { + vec![0, 1, 42, 128, 255] + } +} + +// BoundsCheck: uses a struct Input type + +#[derive( + Debug, + Clone, + jolt_eval::arbitrary::Arbitrary, + serde::Serialize, + serde::Deserialize, + schemars::JsonSchema, +)] +pub struct RangeInput { + pub lo: u32, + pub hi: u32, +} + +#[jolt_eval_macros::invariant(Test, Fuzz)] +#[derive(Default)] +pub struct BoundsCheckInvariant; + +impl Invariant for BoundsCheckInvariant { + type Setup = (); + type Input = RangeInput; + + fn name(&self) -> &str { + "bounds_check" + } + fn description(&self) -> String { + "Checks that max(lo,hi) >= min(lo,hi).".to_string() + } + fn setup(&self) -> Self::Setup {} + fn check(&self, _: &(), input: RangeInput) -> Result<(), CheckError> { + let lo = input.lo.min(input.hi); + let hi = input.lo.max(input.hi); + if hi >= lo { + Ok(()) + } else { + Err(CheckError::Violation(InvariantViolation::new( + "max < min — impossible", + ))) + } + } + fn seed_corpus(&self) -> Vec { + vec![ + RangeInput { lo: 0, hi: 0 }, + RangeInput { + lo: 0, + hi: u32::MAX, + }, + RangeInput { + lo: u32::MAX, + hi: 0, + }, + RangeInput { lo: 100, hi: 50 }, + ] + } +} + +// The #[test] functions `seed_corpus` and `random_inputs` inside the +// generated `*_synthesized` modules are auto-discovered by nextest. diff --git a/jolt-eval/src/invariant/mod.rs b/jolt-eval/src/invariant/mod.rs new file mode 100644 index 000000000..2117c5490 --- /dev/null +++ b/jolt-eval/src/invariant/mod.rs @@ -0,0 +1,335 @@ +#[cfg(test)] +mod macro_tests; +pub mod soundness; +pub mod split_eq_bind; +pub mod synthesis; + +use std::fmt; + +use arbitrary::Arbitrary; +use enumset::{EnumSet, EnumSetType}; +use rand::RngCore; +use schemars::JsonSchema; +use serde::de::DeserializeOwned; +use serde::Serialize; + +/// What to synthesize from an invariant definition. +#[derive(Debug, EnumSetType)] +pub enum SynthesisTarget { + Test, + Fuzz, + RedTeam, +} + +/// Error indicating an invariant was violated. +#[derive(Debug, Clone)] +pub struct InvariantViolation { + pub message: String, + pub details: Option, +} + +impl fmt::Display for InvariantViolation { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.message)?; + if let Some(details) = &self.details { + write!(f, ": {details}")?; + } + Ok(()) + } +} + +impl std::error::Error for InvariantViolation {} + +impl InvariantViolation { + pub fn new(message: impl Into) -> Self { + Self { + message: message.into(), + details: None, + } + } + + pub fn with_details(message: impl Into, details: impl Into) -> Self { + Self { + message: message.into(), + details: Some(details.into()), + } + } +} + +/// Result of checking an invariant against a single input. +#[derive(Debug)] +pub enum CheckError { + /// The invariant was violated. + Violation(InvariantViolation), + /// The input is degenerate or uninteresting and should be skipped. + InvalidInput(String), +} + +impl fmt::Display for CheckError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Violation(v) => write!(f, "violation: {v}"), + Self::InvalidInput(msg) => write!(f, "invalid input: {msg}"), + } + } +} + +impl std::error::Error for CheckError {} + +/// Core invariant trait. Each invariant defines a setup phase (run once) +/// and a check phase (run per input). The `Input` type must support +/// `Arbitrary` for fuzzing, and `Serialize`/`DeserializeOwned` so an AI +/// agent can produce counterexamples as JSON. +pub trait Invariant: Send + Sync { + type Setup: Send + Sync + 'static; + type Input: for<'a> Arbitrary<'a> + + fmt::Debug + + Clone + + Serialize + + DeserializeOwned + + JsonSchema; + + fn name(&self) -> &str; + + /// Human-readable description, also used as context for AI red-teaming. + fn description(&self) -> String; + + /// One-time setup (e.g. preprocessing, generating an honest proof). + fn setup(&self) -> Self::Setup; + + /// Check the invariant for a single input against the pre-computed setup. + /// + /// Returns `Ok(())` if the invariant holds, `Err(CheckError::Violation)` + /// if it is violated, or `Err(CheckError::InvalidInput)` if the input + /// is degenerate and should be skipped. + fn check(&self, setup: &Self::Setup, input: Self::Input) -> Result<(), CheckError>; + + /// Known-interesting inputs for deterministic test generation. + fn seed_corpus(&self) -> Vec { + vec![] + } + + /// Post-process a deserialized input with context from the agent run. + /// + /// Called by the red-team harness after deserializing the agent's JSON + /// counterexample. The `diff` is the unified diff captured from the + /// agent's worktree (if any files were modified). The default is a no-op. + fn enrich_input(&self, input: Self::Input, _diff: Option<&str>) -> Self::Input { + input + } +} + +/// Declares which synthesis targets an invariant supports. +/// +/// Defaults to an empty set. Use the `#[invariant(Test, Fuzz)]` macro +/// attribute to generate the implementation automatically. +pub trait InvariantTargets { + fn targets(&self) -> EnumSet { + EnumSet::empty() + } +} + +/// Enum collecting all Jolt invariants. Methods dispatch via match. +pub enum JoltInvariants { + SplitEqBindLowHigh(split_eq_bind::SplitEqBindLowHighInvariant), + SplitEqBindHighLow(split_eq_bind::SplitEqBindHighLowInvariant), + Soundness(soundness::SoundnessInvariant), +} + +macro_rules! dispatch { + ($self:expr, |$inv:ident| $body:expr) => { + match $self { + JoltInvariants::SplitEqBindLowHigh($inv) => $body, + JoltInvariants::SplitEqBindHighLow($inv) => $body, + JoltInvariants::Soundness($inv) => $body, + } + }; +} + +impl JoltInvariants { + pub fn all() -> Vec { + vec![ + Self::SplitEqBindLowHigh(split_eq_bind::SplitEqBindLowHighInvariant), + Self::SplitEqBindHighLow(split_eq_bind::SplitEqBindHighLowInvariant), + Self::Soundness(soundness::SoundnessInvariant), + ] + } + + pub fn name(&self) -> &str { + dispatch!(self, |inv| inv.name()) + } + + pub fn description(&self) -> String { + dispatch!(self, |inv| inv.description()) + } + + pub fn targets(&self) -> EnumSet { + dispatch!(self, |inv| InvariantTargets::targets(inv)) + } + + pub fn run_checks(&self, num_random: usize) -> Vec> { + dispatch!(self, |inv| run_checks_impl(inv, num_random)) + } + + pub fn red_team( + &self, + config: &synthesis::redteam::RedTeamConfig, + agent: &dyn crate::agent::AgentHarness, + repo_dir: &std::path::Path, + ) -> synthesis::redteam::RedTeamResult { + dispatch!(self, |inv| synthesis::redteam::auto_redteam( + inv, config, agent, repo_dir + )) + } +} + +fn run_checks_impl( + inv: &I, + num_random: usize, +) -> Vec> { + let setup = inv.setup(); + let mut results = Vec::new(); + + let mut record = |r: Result<(), CheckError>| match r { + Ok(()) => results.push(Ok(())), + Err(CheckError::Violation(v)) => results.push(Err(v)), + Err(CheckError::InvalidInput(_)) => {} + }; + + for input in inv.seed_corpus() { + record(inv.check(&setup, input)); + } + + let mut rng = rand::thread_rng(); + for _ in 0..num_random { + let mut raw = vec![0u8; 4096]; + rng.fill_bytes(&mut raw); + let mut u = arbitrary::Unstructured::new(&raw); + if let Ok(input) = I::Input::arbitrary(&mut u) { + record(inv.check(&setup, input)); + } + } + + results +} + +/// Record of a red-team attempt that failed to find a violation. +pub struct FailedAttempt { + pub description: String, + pub approach: String, + pub failure_reason: String, + /// Path to the persisted attempt directory (relative to repo root). + pub path: Option, +} + +/// Try to extract a JSON object from free-form text. Looks for a +/// ````json` code block first, then falls back to the last `{…}` that +/// parses as valid JSON. +pub fn extract_json(text: &str) -> Option { + // 1. ```json ... ``` + if let Some(start) = text.find("```json") { + let json_start = start + "```json".len(); + if let Some(end) = text[json_start..].find("```") { + let candidate = text[json_start..json_start + end].trim(); + if serde_json::from_str::(candidate).is_ok() { + return Some(candidate.to_string()); + } + } + } + + // 2. Last balanced {…} that is valid JSON + let bytes = text.as_bytes(); + let mut i = bytes.len(); + while i > 0 { + i -= 1; + if bytes[i] == b'}' { + let end = i; + let mut depth: i32 = 0; + let mut j = end + 1; + while j > 0 { + j -= 1; + match bytes[j] { + b'}' => depth += 1, + b'{' => { + depth -= 1; + if depth == 0 { + let candidate = &text[j..=end]; + if serde_json::from_str::(candidate).is_ok() { + return Some(candidate.to_string()); + } + break; + } + } + _ => {} + } + } + } + } + + None +} + +#[cfg(test)] +mod integration_tests { + use super::*; + + struct TrivialInvariant; + impl InvariantTargets for TrivialInvariant {} + + impl Invariant for TrivialInvariant { + type Setup = (); + type Input = u8; + fn name(&self) -> &str { + "trivial" + } + fn description(&self) -> String { + "Always passes".into() + } + fn setup(&self) {} + fn check(&self, _: &(), _: u8) -> Result<(), CheckError> { + Ok(()) + } + fn seed_corpus(&self) -> Vec { + vec![0, 1, 255] + } + } + + struct FailingInvariant; + impl InvariantTargets for FailingInvariant {} + + impl Invariant for FailingInvariant { + type Setup = (); + type Input = u8; + fn name(&self) -> &str { + "failing" + } + fn description(&self) -> String { + "Always fails".into() + } + fn setup(&self) {} + fn check(&self, _: &(), input: u8) -> Result<(), CheckError> { + Err(CheckError::Violation(InvariantViolation::new(format!( + "failed for input {input}" + )))) + } + fn seed_corpus(&self) -> Vec { + vec![42] + } + } + + #[test] + fn trivial_invariant_passes() { + let inv = TrivialInvariant; + for input in inv.seed_corpus() { + inv.check(&(), input).unwrap(); + } + } + + #[test] + fn failing_invariant_reports_violations() { + let inv = FailingInvariant; + for input in inv.seed_corpus() { + assert!(inv.check(&(), input).is_err()); + } + } +} diff --git a/jolt-eval/src/invariant/soundness.rs b/jolt-eval/src/invariant/soundness.rs new file mode 100644 index 000000000..ea0bae86e --- /dev/null +++ b/jolt-eval/src/invariant/soundness.rs @@ -0,0 +1,554 @@ +use std::path::{Path, PathBuf}; +use std::process::Command; + +use arbitrary::{Arbitrary, Unstructured}; + +use common::constants::{DEFAULT_MAX_TRUSTED_ADVICE_SIZE, DEFAULT_MAX_UNTRUSTED_ADVICE_SIZE}; +use common::jolt_device::MemoryConfig; +use jolt_core::host::Program; + +use super::{CheckError, Invariant, InvariantViolation}; +use crate::guests; + +/// Guest memory layout parameters. +/// +/// Serializable mirror of `common::jolt_device::MemoryConfig` for use +/// in JSON-based counterexamples. +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, schemars::JsonSchema)] +pub struct GuestMemoryConfig { + pub max_input_size: u64, + pub max_output_size: u64, + pub stack_size: u64, + pub heap_size: u64, +} + +impl Default for GuestMemoryConfig { + fn default() -> Self { + Self { + max_input_size: 4096, + max_output_size: 4096, + stack_size: 65536, + heap_size: 32768, + } + } +} + +/// Maximum allowed values for memory config parameters. +const MAX_INPUT_SIZE: u64 = 1 << 16; +const MAX_OUTPUT_SIZE: u64 = 1 << 16; +const MAX_STACK_SIZE: u64 = 1 << 16; +const MAX_HEAP_SIZE: u64 = 1 << 20; +const MAX_TRACE_LENGTH: usize = 1 << 20; + +impl GuestMemoryConfig { + pub fn validate(&self) -> Result<(), CheckError> { + if self.max_input_size > MAX_INPUT_SIZE + || self.max_output_size > MAX_OUTPUT_SIZE + || self.stack_size > MAX_STACK_SIZE + || self.heap_size > MAX_HEAP_SIZE + { + return Err(CheckError::InvalidInput(format!( + "memory config exceeds limits: \ + input={}, output={}, stack={}, heap={}; \ + limits: input<={MAX_INPUT_SIZE}, output<={MAX_OUTPUT_SIZE}, stack<={MAX_STACK_SIZE}, heap<={MAX_HEAP_SIZE}", + self.max_input_size, self.max_output_size, self.stack_size, self.heap_size, + ))); + } + Ok(()) + } + + fn to_memory_config(&self) -> MemoryConfig { + MemoryConfig { + max_input_size: self.max_input_size, + max_output_size: self.max_output_size, + max_untrusted_advice_size: DEFAULT_MAX_UNTRUSTED_ADVICE_SIZE, + max_trusted_advice_size: DEFAULT_MAX_TRUSTED_ADVICE_SIZE, + stack_size: self.stack_size, + heap_size: self.heap_size, + program_size: None, + } + } +} + +/// Input for the soundness invariant. +/// +/// The red-team agent produces a `patch` (unified diff) to apply to +/// the `guest-sandbox/` template project, along with the program input +/// and a dishonest claim about the output and panic flag. +/// +/// The invariant proves the patched guest honestly, then checks that +/// the verifier rejects the dishonest claim. +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, schemars::JsonSchema)] +pub struct SoundnessInput { + /// Unified diff to apply to `guest-sandbox/`. + /// Only hunks touching files within the sandbox are applied. + pub patch: String, + /// Guest memory layout. Defaults are reasonable for most programs. + #[serde(default)] + pub memory: GuestMemoryConfig, + /// Input bytes fed to the guest program. + pub program_input: Vec, + /// The output the malicious prover claims. + pub claimed_output: Vec, + /// The panic flag the malicious prover claims. + pub claimed_panic: bool, +} + +impl<'a> Arbitrary<'a> for SoundnessInput { + fn arbitrary(_u: &mut Unstructured<'a>) -> arbitrary::Result { + // Soundness is RedTeam-only; Arbitrary is not meaningful. + Err(arbitrary::Error::IncorrectFormat) + } +} + +/// Cached paths resolved once during setup. +pub struct SoundnessSetup { + sandbox_dir: PathBuf, +} + +#[jolt_eval_macros::invariant(RedTeam)] +#[derive(Default)] +pub struct SoundnessInvariant; + +impl Invariant for SoundnessInvariant { + type Setup = SoundnessSetup; + type Input = SoundnessInput; + + fn name(&self) -> &str { + "soundness" + } + + fn description(&self) -> String { + format!( + "For any deterministic guest program (no advice) and fixed input, \ + there is only one (output, panic) pair that the verifier accepts. \ + A counterexample is a guest patch + input + dishonest (output, panic) \ + claim that the verifier incorrectly accepts. \ + For full context, read the invariant file: jolt-eval/src/invariant/soundness.rs \n\n\ + ## Guest sandbox\n\n\ + The guest template is at `jolt-eval/guest-sandbox/`. It contains:\n\ + - `Cargo.toml` — depends on `jolt-sdk`\n\ + - `src/lib.rs` — the `#[jolt::provable]` function (main patch target)\n\ + - `src/main.rs` — no_main entry point (rarely needs patching)\n\n\ + ## Producing a patch\n\n\ + Simply edit the files inside `jolt-eval/guest-sandbox/` directly. \ + The harness automatically captures your changes as a `git diff` \ + from the worktree before cleanup and uses it as the patch. \ + You do NOT need to put the patch in the JSON counterexample — \ + leave the `patch` field empty and the harness fills it in.\n\n\ + Alternatively, you can provide a patch explicitly in the JSON \ + `patch` field. If non-empty, it takes precedence over the \ + worktree diff. Hunks referencing paths with `..` are filtered out.\n\n\ + ## Limits\n\n\ + Memory config: max_input_size <= {MAX_INPUT_SIZE}, \ + max_output_size <= {MAX_OUTPUT_SIZE}, \ + stack_size <= {MAX_STACK_SIZE}, heap_size <= {MAX_HEAP_SIZE}. \ + The program's execution trace must not exceed {MAX_TRACE_LENGTH} cycles." + ) + } + + fn setup(&self) -> SoundnessSetup { + let manifest_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + SoundnessSetup { + sandbox_dir: manifest_dir.join("guest-sandbox"), + } + } + + fn check(&self, setup: &SoundnessSetup, input: SoundnessInput) -> Result<(), CheckError> { + // 1. Validate memory config + input.memory.validate()?; + let mut memory_config = input.memory.to_memory_config(); + + // 2. Apply patch to sandbox in-place, revert on exit + let _guard = apply_patch(&setup.sandbox_dir, &input.patch)?; + + // 3. Compile the patched guest + let elf_bytes = compile_guest(&setup.sandbox_dir, &memory_config)?; + + // _guard drops here (or on early return), reverting the patch + + // 4. Decode to get program_size, then trace to get actual length + let (_bytecode, _memory_init, program_size, _e_entry) = + jolt_core::guest::program::decode(&elf_bytes); + memory_config.program_size = Some(program_size); + + let program = guests::GuestProgram::new(&elf_bytes, &memory_config); + let (_lazy_trace, trace, _memory, _io) = program.trace(&input.program_input, &[], &[]); + let max_trace_length = (trace.len() + 1).next_power_of_two(); + drop(trace); + + if max_trace_length > MAX_TRACE_LENGTH { + return Err(CheckError::InvalidInput(format!( + "trace length {max_trace_length} exceeds limit {MAX_TRACE_LENGTH}" + ))); + } + + // 5. Prove and verify + let prover_pp = guests::prover_preprocessing(&program, max_trace_length); + let verifier_pp = guests::verifier_preprocessing(&prover_pp); + let (proof, honest_device) = guests::prove(&program, &prover_pp, &input.program_input); + + // 6. Skip no-op claims (the claim matches the honest execution) + if input.claimed_output == honest_device.outputs + && input.claimed_panic == honest_device.panic + { + return Err(CheckError::InvalidInput( + "claimed output/panic matches honest execution".into(), + )); + } + + // 7. Verify with the dishonest claim — this SHOULD fail + match guests::verify_with_claims( + &verifier_pp, + proof, + &honest_device.inputs, + &input.claimed_output, + input.claimed_panic, + ) { + Ok(()) => Err(CheckError::Violation(InvariantViolation::with_details( + "Verifier accepted dishonest claim", + format!( + "honest_output={} bytes (panic={}), claimed_output={} bytes (panic={})", + honest_device.outputs.len(), + honest_device.panic, + input.claimed_output.len(), + input.claimed_panic, + ), + ))), + Err(_) => Ok(()), + } + } + + fn seed_corpus(&self) -> Vec { + vec![SoundnessInput { + patch: String::new(), + memory: GuestMemoryConfig::default(), + program_input: postcard::to_stdvec::<[u8]>(&[1, 2, 3]).unwrap(), + claimed_output: vec![0xFF], + claimed_panic: false, + }] + } + + /// If the agent modified `guest-sandbox/` in its worktree, use that + /// diff as the patch (unless the agent already provided one in JSON). + fn enrich_input(&self, mut input: SoundnessInput, diff: Option<&str>) -> SoundnessInput { + if input.patch.trim().is_empty() { + if let Some(diff) = diff { + input.patch = diff.to_string(); + } + } + input + } +} + +/// RAII guard that reverts a patch on drop via `git checkout`. +struct PatchGuard { + dir: PathBuf, + applied: bool, +} + +impl Drop for PatchGuard { + fn drop(&mut self) { + if self.applied { + let _ = Command::new("git") + .current_dir(&self.dir) + .args(["checkout", "."]) + .status(); + } + } +} + +/// Apply a filtered patch to `sandbox_dir` in-place. Returns a guard +/// that reverts the changes on drop (even on panic). +fn apply_patch(sandbox_dir: &Path, patch: &str) -> Result { + let guard = PatchGuard { + dir: sandbox_dir.to_path_buf(), + applied: false, + }; + + if patch.trim().is_empty() { + return Ok(guard); + } + + let safe_patch = filter_patch(patch); + if safe_patch.trim().is_empty() { + return Ok(guard); + } + + let mut child = Command::new("git") + .current_dir(sandbox_dir) + .args(["apply", "--allow-empty", "-"]) + .stdin(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .spawn() + .map_err(|e| CheckError::InvalidInput(format!("git apply spawn: {e}")))?; + + if let Some(stdin) = child.stdin.as_mut() { + use std::io::Write; + let _ = stdin.write_all(safe_patch.as_bytes()); + } + + let output = child + .wait_with_output() + .map_err(|e| CheckError::InvalidInput(format!("git apply wait: {e}")))?; + + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + return Err(CheckError::InvalidInput(format!( + "patch failed to apply: {stderr}" + ))); + } + + Ok(PatchGuard { + dir: sandbox_dir.to_path_buf(), + applied: true, + }) +} + +/// Remove diff hunks that reference paths containing `..` to prevent +/// escaping the sandbox. +pub fn filter_patch(patch: &str) -> String { + let mut result = String::new(); + let mut include_hunk = true; + + for line in patch.lines() { + if line.starts_with("diff --git") || line.starts_with("--- ") || line.starts_with("+++ ") { + include_hunk = !line.contains(".."); + } + if include_hunk { + result.push_str(line); + result.push('\n'); + } + } + + result +} + +/// Compile the sandbox guest and return the ELF bytes. +/// +/// `Program::build` panics on compilation failure, so we catch it. +fn compile_guest(sandbox_dir: &Path, memory_config: &MemoryConfig) -> Result, CheckError> { + let target_dir = sandbox_dir.join("target").to_string_lossy().to_string(); + let mc = *memory_config; + let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + let mut program = Program::new("sandbox-guest"); + program.set_memory_config(mc); + program.build(&target_dir); + program.get_elf_contents() + })); + match result { + Ok(Some(elf)) => Ok(elf), + Ok(None) => Err(CheckError::InvalidInput( + "guest ELF not found after build".into(), + )), + Err(_) => Err(CheckError::InvalidInput( + "guest compilation panicked".into(), + )), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::Invariant; + + fn default_input() -> SoundnessInput { + SoundnessInput { + patch: String::new(), + memory: GuestMemoryConfig::default(), + program_input: postcard::to_stdvec::<[u8]>(&[1, 2, 3]).unwrap(), + claimed_output: vec![0xFF], + claimed_panic: false, + } + } + + // ── filter_patch ──────────────────────────────────────────────── + + #[test] + fn filter_keeps_safe_hunks() { + let patch = "\ +diff --git a/src/lib.rs b/src/lib.rs +--- a/src/lib.rs ++++ b/src/lib.rs +@@ -1,3 +1,3 @@ +-fn foo() {} ++fn bar() {} +"; + let filtered = filter_patch(patch); + assert!(filtered.contains("+fn bar() {}")); + } + + #[test] + fn filter_drops_path_traversal() { + let patch = "\ +diff --git a/../../jolt-core/src/lib.rs b/../../jolt-core/src/lib.rs +--- a/../../jolt-core/src/lib.rs ++++ b/../../jolt-core/src/lib.rs +@@ -1 +1 @@ +-safe ++malicious +"; + let filtered = filter_patch(patch); + assert!(!filtered.contains("malicious")); + } + + #[test] + fn filter_mixed_safe_and_unsafe() { + let patch = "\ +diff --git a/src/lib.rs b/src/lib.rs +--- a/src/lib.rs ++++ b/src/lib.rs +@@ -1 +1 @@ +-old ++new +diff --git a/../../../etc/passwd b/../../../etc/passwd +--- a/../../../etc/passwd ++++ b/../../../etc/passwd +@@ -1 +1 @@ +-root ++hacked +diff --git a/Cargo.toml b/Cargo.toml +--- a/Cargo.toml ++++ b/Cargo.toml +@@ -1 +1 @@ +-v1 ++v2 +"; + let filtered = filter_patch(patch); + assert!(filtered.contains("+new")); + assert!(!filtered.contains("hacked")); + assert!(filtered.contains("+v2")); + } + + #[test] + fn filter_empty_patch() { + assert!(filter_patch("").is_empty()); + assert!(filter_patch(" \n ").trim().is_empty()); + } + + // ── memory config validation ──────────────────────────────────── + + #[test] + fn validate_accepts_defaults() { + assert!(GuestMemoryConfig::default().validate().is_ok()); + } + + #[test] + fn validate_rejects_oversized_input() { + let c = GuestMemoryConfig { + max_input_size: u64::MAX, + ..Default::default() + }; + assert!(matches!(c.validate(), Err(CheckError::InvalidInput(_)))); + } + + #[test] + fn validate_rejects_oversized_output() { + let c = GuestMemoryConfig { + max_output_size: u64::MAX, + ..Default::default() + }; + assert!(matches!(c.validate(), Err(CheckError::InvalidInput(_)))); + } + + #[test] + fn validate_rejects_oversized_stack() { + let c = GuestMemoryConfig { + stack_size: u64::MAX, + ..Default::default() + }; + assert!(matches!(c.validate(), Err(CheckError::InvalidInput(_)))); + } + + #[test] + fn validate_rejects_oversized_heap() { + let c = GuestMemoryConfig { + heap_size: u64::MAX, + ..Default::default() + }; + assert!(matches!(c.validate(), Err(CheckError::InvalidInput(_)))); + } + + #[test] + fn check_rejects_oversized_memory_before_compilation() { + let inv = SoundnessInvariant; + let setup = inv.setup(); + let input = SoundnessInput { + memory: GuestMemoryConfig { + heap_size: u64::MAX, + ..Default::default() + }, + ..default_input() + }; + assert!(matches!( + inv.check(&setup, input), + Err(CheckError::InvalidInput(_)) + )); + } + + // ── patching ──────────────────────────────────────────────────── + + #[test] + fn check_garbage_patch_is_noop() { + let inv = SoundnessInvariant; + let setup = inv.setup(); + let input = SoundnessInput { + patch: "this is not a valid unified diff\n+garbage".into(), + ..default_input() + }; + // Garbage with no diff headers passes filter_patch unchanged. + // git apply --allow-empty treats it as a no-op (no hunks), + // so the unpatched sandbox compiles and the check proceeds normally. + assert!(inv.check(&setup, input).is_ok()); + } + + // ── compilation + prove/verify (slow) ─────────────────────────── + + #[test] + fn check_path_traversal_filtered_then_compiles() { + let inv = SoundnessInvariant; + let setup = inv.setup(); + let input = SoundnessInput { + patch: "\ +diff --git a/../../etc/passwd b/../../etc/passwd +--- a/../../etc/passwd ++++ b/../../etc/passwd +@@ -1 +1 @@ +-root ++hacked +" + .into(), + ..default_input() + }; + // Traversal hunks are filtered out → empty patch → compiles + // unpatched sandbox → proves → verifier rejects dishonest claim. + assert!(inv.check(&setup, input).is_ok()); + } + + #[test] + fn check_unpatched_sandbox_rejects_dishonest_output() { + let inv = SoundnessInvariant; + let setup = inv.setup(); + // claimed_output=[0xFF] doesn't match the identity function's + // honest output for input [1,2,3]. Verifier should reject. + assert!(inv.check(&setup, default_input()).is_ok()); + } + + #[test] + fn check_noop_claim_returns_invalid_input() { + let inv = SoundnessInvariant; + let setup = inv.setup(); + // The sandbox computes h = wrapping hash of input bytes. + // For input [1,2,3]: h = ((0*31+1)*31+2)*31+3 = 1026 + let honest_output = postcard::to_stdvec(&1026u32).unwrap(); + let input = SoundnessInput { + claimed_output: honest_output, + claimed_panic: false, + ..default_input() + }; + assert!(matches!( + inv.check(&setup, input), + Err(CheckError::InvalidInput(_)) + )); + } +} diff --git a/jolt-eval/src/invariant/split_eq_bind.rs b/jolt-eval/src/invariant/split_eq_bind.rs new file mode 100644 index 000000000..f9a65516e --- /dev/null +++ b/jolt-eval/src/invariant/split_eq_bind.rs @@ -0,0 +1,192 @@ +#![allow(non_snake_case)] + +use arbitrary::{Arbitrary, Unstructured}; + +use ark_bn254::Fr; +use jolt_core::field::JoltField; +use jolt_core::poly::dense_mlpoly::DensePolynomial; +use jolt_core::poly::eq_poly::EqPolynomial; +use jolt_core::poly::multilinear_polynomial::BindingOrder; +use jolt_core::poly::split_eq_poly::GruenSplitEqPolynomial; + +use super::{CheckError, Invariant, InvariantViolation}; + +type Challenge = ::Challenge; + +/// Input for the split-eq bind invariants. +/// +/// `w` are the initial eq-polynomial challenges, `rs` are the binding +/// round challenges. Stored as `u128` for serde/Arbitrary compatibility; +/// converted to `Challenge` via `From` in the check methods. +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, schemars::JsonSchema)] +pub struct SplitEqBindInput { + pub w: Vec, + pub rs: Vec, +} + +impl<'a> Arbitrary<'a> for SplitEqBindInput { + fn arbitrary(u: &mut Unstructured<'a>) -> arbitrary::Result { + let num_vars = u.int_in_range(2u8..=16)? as usize; + let w: Vec = (0..num_vars) + .map(|_| u.arbitrary()) + .collect::>()?; + let rs: Vec = (0..num_vars) + .map(|_| u.arbitrary()) + .collect::>()?; + Ok(Self { w, rs }) + } +} + +fn to_challenges(vals: &[u128]) -> Vec { + vals.iter().copied().map(Challenge::from).collect() +} + +// ── LowToHigh ──────────────────────────────────────────────────────── + +#[jolt_eval_macros::invariant(Test, Fuzz, RedTeam)] +#[derive(Default)] +pub struct SplitEqBindLowHighInvariant; + +impl Invariant for SplitEqBindLowHighInvariant { + type Setup = (); + type Input = SplitEqBindInput; + + fn name(&self) -> &str { + "split_eq_bind_low_high" + } + + fn description(&self) -> String { + "GruenSplitEqPolynomial::bind (LowToHigh) must match \ + DensePolynomial::bound_poly_var_bot at every round." + .to_string() + } + + fn setup(&self) {} + + fn check(&self, _setup: &(), input: SplitEqBindInput) -> Result<(), CheckError> { + if input.w.len() < 2 { + return Err(CheckError::InvalidInput("w.len() < 2".into())); + } + let w = to_challenges(&input.w); + let rs = to_challenges(&input.rs); + let num_vars = w.len(); + + let mut regular_eq = DensePolynomial::::new(EqPolynomial::evals(&w)); + let mut split_eq = GruenSplitEqPolynomial::::new(&w, BindingOrder::LowToHigh); + + let merged = split_eq.merge(); + if regular_eq.Z[..regular_eq.len()] != merged.Z[..merged.len()] { + return Err(CheckError::Violation(InvariantViolation::with_details( + "Initial merge mismatch (LowToHigh)", + format!("num_vars={num_vars}"), + ))); + } + + for (round, r) in rs.iter().enumerate() { + regular_eq.bound_poly_var_bot(r); + split_eq.bind(*r); + + let merged = split_eq.merge(); + if regular_eq.Z[..regular_eq.len()] != merged.Z[..merged.len()] { + return Err(CheckError::Violation(InvariantViolation::with_details( + "Bind mismatch (LowToHigh)", + format!("num_vars={num_vars}, round={round}"), + ))); + } + } + + Ok(()) + } + + fn seed_corpus(&self) -> Vec { + vec![ + SplitEqBindInput { + w: vec![0, 1], + rs: vec![2, 3], + }, + SplitEqBindInput { + w: (0..10).collect(), + rs: (10..20).collect(), + }, + SplitEqBindInput { + w: (0..17).map(|i| u128::MAX - i).collect(), + rs: (0..17).map(|i| i * 1000).collect(), + }, + ] + } +} + +// ── HighToLow ──────────────────────────────────────────────────────── + +#[jolt_eval_macros::invariant(Test, Fuzz)] +#[derive(Default)] +pub struct SplitEqBindHighLowInvariant; + +impl Invariant for SplitEqBindHighLowInvariant { + type Setup = (); + type Input = SplitEqBindInput; + + fn name(&self) -> &str { + "split_eq_bind_high_low" + } + + fn description(&self) -> String { + "GruenSplitEqPolynomial::bind (HighToLow) must match \ + DensePolynomial::bound_poly_var_top at every round." + .to_string() + } + + fn setup(&self) {} + + fn check(&self, _setup: &(), input: SplitEqBindInput) -> Result<(), CheckError> { + if input.w.len() < 2 { + return Err(CheckError::InvalidInput("w.len() < 2".into())); + } + let w = to_challenges(&input.w); + let rs = to_challenges(&input.rs); + let num_vars = w.len(); + + let mut regular_eq = DensePolynomial::::new(EqPolynomial::evals(&w)); + let mut split_eq = GruenSplitEqPolynomial::::new(&w, BindingOrder::HighToLow); + + let merged = split_eq.merge(); + if regular_eq.Z[..regular_eq.len()] != merged.Z[..merged.len()] { + return Err(CheckError::Violation(InvariantViolation::with_details( + "Initial merge mismatch (HighToLow)", + format!("num_vars={num_vars}"), + ))); + } + + for (round, r) in rs.iter().enumerate() { + regular_eq.bound_poly_var_top(r); + split_eq.bind(*r); + + let merged = split_eq.merge(); + if regular_eq.Z[..regular_eq.len()] != merged.Z[..merged.len()] { + return Err(CheckError::Violation(InvariantViolation::with_details( + "Bind mismatch (HighToLow)", + format!("num_vars={num_vars}, round={round}"), + ))); + } + } + + Ok(()) + } + + fn seed_corpus(&self) -> Vec { + vec![ + SplitEqBindInput { + w: vec![0, 1], + rs: vec![2, 3], + }, + SplitEqBindInput { + w: (0..10).collect(), + rs: (10..20).collect(), + }, + SplitEqBindInput { + w: (0..16).map(|i| u128::MAX - i).collect(), + rs: (0..16).map(|i| i * 1000).collect(), + }, + ] + } +} diff --git a/jolt-eval/src/invariant/synthesis/fuzz.rs b/jolt-eval/src/invariant/synthesis/fuzz.rs new file mode 100644 index 000000000..2335ccae4 --- /dev/null +++ b/jolt-eval/src/invariant/synthesis/fuzz.rs @@ -0,0 +1,65 @@ +/// Macro that generates a libfuzzer fuzz target for an invariant. +/// +/// Takes a concrete invariant expression. Setup is performed once; +/// each fuzz iteration produces an `Input` via `Arbitrary` and checks it. +/// +/// # Usage +/// +/// ```ignore +/// #![no_main] +/// use jolt_eval::invariant::split_eq_bind::SplitEqBindLowHighInvariant; +/// jolt_eval::fuzz_invariant!(SplitEqBindLowHighInvariant::default()); +/// ``` +#[macro_export] +macro_rules! fuzz_invariant { + ($inv:expr) => { + use $crate::Invariant as _; + use $crate::InvariantTargets as _; + + // Assert at init time that this invariant includes the Fuzz target. + fn __assert_fuzz_target(inv: &I) { + assert!( + inv.targets() + .contains($crate::SynthesisTarget::Fuzz), + "Invariant does not include SynthesisTarget::Fuzz" + ); + } + + static __FUZZ_SETUP: ::std::sync::OnceLock< + ::std::boxed::Box, + > = ::std::sync::OnceLock::new(); + + fn __fuzz_init(inv: &I) { + __FUZZ_SETUP + .set(::std::boxed::Box::new(inv.setup())) + .ok(); + } + + fn __fuzz_check(inv: &I, data: &[u8]) { + let setup = __FUZZ_SETUP + .get() + .expect("SETUP not initialized") + .downcast_ref::() + .expect("wrong setup type"); + let mut u = $crate::arbitrary::Unstructured::new(data); + if let Ok(input) = ::arbitrary(&mut u) { + match inv.check(setup, input) { + Ok(()) | Err($crate::CheckError::InvalidInput(_)) => {} + Err($crate::CheckError::Violation(v)) => { + panic!("Invariant violated: {v}"); + } + } + } + } + + ::libfuzzer_sys::fuzz_target!( + init: { + __assert_fuzz_target(&$inv); + __fuzz_init(&$inv); + }, + |data: &[u8]| { + __fuzz_check(&$inv, data); + } + ); + }; +} diff --git a/jolt-eval/src/invariant/synthesis/mod.rs b/jolt-eval/src/invariant/synthesis/mod.rs new file mode 100644 index 000000000..7b79988f8 --- /dev/null +++ b/jolt-eval/src/invariant/synthesis/mod.rs @@ -0,0 +1,2 @@ +pub mod fuzz; +pub mod redteam; diff --git a/jolt-eval/src/invariant/synthesis/redteam.rs b/jolt-eval/src/invariant/synthesis/redteam.rs new file mode 100644 index 000000000..51fcc841f --- /dev/null +++ b/jolt-eval/src/invariant/synthesis/redteam.rs @@ -0,0 +1,332 @@ +use std::path::Path; + +use super::super::{CheckError, FailedAttempt, Invariant}; +use crate::agent::{AgentHarness, DiffScope}; + +/// Result of a red-team session. +pub enum RedTeamResult { + /// The agent produced a counterexample that violates the invariant. + Violation { + approach: String, + input_json: String, + error: String, + }, + /// All attempts failed to find a violation. + NoViolation { attempts: Vec }, +} + +/// Configuration for an AI red-team session. +pub struct RedTeamConfig { + pub num_iterations: usize, + pub hint: Option, + pub verbose: bool, +} + +impl Default for RedTeamConfig { + fn default() -> Self { + Self { + num_iterations: 10, + hint: None, + verbose: false, + } + } +} + +/// Run an AI red-team session against a single invariant. +pub fn auto_redteam( + invariant: &I, + config: &RedTeamConfig, + agent: &dyn AgentHarness, + repo_dir: &Path, +) -> RedTeamResult { + let description = invariant.description(); + let input_example: Option = invariant + .seed_corpus() + .into_iter() + .next() + .and_then(|input| serde_json::to_string_pretty(&input).ok()); + let input_schema = serde_json::to_value(schemars::schema_for!(I::Input)).unwrap(); + let envelope_schema = build_envelope_schema(&input_schema); + let setup = invariant.setup(); + let mut failed_attempts = Vec::new(); + + for iteration in 0..config.num_iterations { + let iter = iteration + 1; + tracing::info!( + "Red team iteration {iter}/{} for '{}'", + config.num_iterations, + invariant.name() + ); + + let prompt = build_redteam_prompt( + &description, + input_example.as_deref(), + &input_schema, + config.hint.as_deref(), + &failed_attempts, + iter, + config.num_iterations, + ); + + if config.verbose { + eprintln!("── Iteration {iter} prompt ──"); + eprintln!("{prompt}"); + eprintln!("────────────────────────"); + } + + let diff_scope = DiffScope::Include(vec!["jolt-eval/guest-sandbox/".into()]); + let response = + match agent.invoke_structured(repo_dir, &prompt, &envelope_schema, &diff_scope) { + Ok(r) => r, + Err(e) => { + tracing::warn!("Agent invocation failed: {e}"); + let path = persist_redteam_attempt( + repo_dir, + invariant.name(), + iter, + "Agent invocation failed", + &e.to_string(), + ); + failed_attempts.push(FailedAttempt { + description: format!("Iteration {iter}"), + approach: "Agent invocation failed".to_string(), + failure_reason: e.to_string(), + path, + }); + continue; + } + }; + + if config.verbose { + eprintln!("── Iteration {iter} response ──"); + eprintln!("{}", response.text); + if let Some(ref d) = response.diff { + eprintln!("── diff ({} bytes) ──", d.len()); + eprintln!("{d}"); + } + eprintln!("──────────────────────────"); + } + + let (analysis, counterexample_json) = match parse_envelope(&response.text) { + Some(pair) => pair, + None => match super::super::extract_json(&response.text) { + Some(json) => match parse_envelope(&json) { + Some(pair) => pair, + None => (response.text.clone(), json), + }, + None => { + let failure = "Agent response did not contain valid JSON".to_string(); + let path = persist_redteam_attempt( + repo_dir, + invariant.name(), + iter, + &response.text, + &failure, + ); + failed_attempts.push(FailedAttempt { + description: format!("Iteration {iter}"), + approach: response.text, + failure_reason: failure, + path, + }); + continue; + } + }, + }; + + let input: I::Input = match serde_json::from_str(&counterexample_json) { + Ok(v) => v, + Err(e) => { + tracing::info!("Agent produced unparsable input: {e}"); + let failure = format!("Could not deserialize response JSON into Input type: {e}"); + let path = + persist_redteam_attempt(repo_dir, invariant.name(), iter, &analysis, &failure); + failed_attempts.push(FailedAttempt { + description: format!("Iteration {iter}"), + approach: analysis, + failure_reason: failure, + path, + }); + continue; + } + }; + + // Let the invariant fill in fields from the agent's worktree diff + // (e.g. SoundnessInvariant uses it to populate the patch field). + let input = invariant.enrich_input(input, response.diff.as_deref()); + + match invariant.check(&setup, input) { + Ok(()) => { + let failure = + format!("Candidate input did not violate the invariant: {counterexample_json}"); + let path = + persist_redteam_attempt(repo_dir, invariant.name(), iter, &analysis, &failure); + failed_attempts.push(FailedAttempt { + description: format!("Iteration {iter}"), + approach: analysis, + failure_reason: failure, + path, + }); + } + Err(CheckError::Violation(violation)) => { + tracing::info!("Counterexample CONFIRMED: {violation}"); + return RedTeamResult::Violation { + approach: analysis, + input_json: counterexample_json, + error: violation.to_string(), + }; + } + Err(CheckError::InvalidInput(reason)) => { + let failure = format!("Invalid input: {reason}"); + let path = + persist_redteam_attempt(repo_dir, invariant.name(), iter, &analysis, &failure); + failed_attempts.push(FailedAttempt { + description: format!("Iteration {iter}"), + approach: analysis, + failure_reason: failure, + path, + }); + } + } + } + + RedTeamResult::NoViolation { + attempts: failed_attempts, + } +} + +/// Persist a red-team attempt's approach to disk and return the relative path. +fn persist_redteam_attempt( + repo_dir: &Path, + invariant_name: &str, + iteration: usize, + approach: &str, + failure_reason: &str, +) -> Option { + let dir = repo_dir + .join("jolt-eval/redteam-history") + .join(invariant_name) + .join(format!("attempt-{iteration}")); + std::fs::create_dir_all(&dir).ok()?; + std::fs::write(dir.join("approach.md"), approach).ok()?; + std::fs::write(dir.join("failure_reason.txt"), failure_reason).ok()?; + Some( + dir.strip_prefix(repo_dir) + .ok()? + .to_string_lossy() + .to_string(), + ) +} + +fn build_envelope_schema(input_schema: &serde_json::Value) -> serde_json::Value { + serde_json::json!({ + "type": "object", + "properties": { + "analysis": { + "type": "string", + "description": "Your analysis of the invariant and approach to finding a violation" + }, + "counterexample": input_schema + }, + "required": ["analysis", "counterexample"] + }) +} + +fn parse_envelope(text: &str) -> Option<(String, String)> { + let val: serde_json::Value = serde_json::from_str(text).ok()?; + let analysis = val.get("analysis")?.as_str()?.to_string(); + let counterexample = val.get("counterexample")?; + Some((analysis, serde_json::to_string(counterexample).ok()?)) +} + +fn build_redteam_prompt( + invariant_description: &str, + input_example: Option<&str>, + input_schema: &serde_json::Value, + hint: Option<&str>, + failed_attempts: &[FailedAttempt], + current_iteration: usize, + total_iterations: usize, +) -> String { + let mut prompt = String::new(); + + prompt.push_str(&format!( + "You are a security researcher red-teaming a zkVM (Jolt). \ + Your goal is to find a concrete input that VIOLATES the following invariant.\n\ + Iteration {current_iteration}/{total_iterations}.\n\n" + )); + + prompt.push_str("## Invariant\n\n"); + prompt.push_str(invariant_description); + prompt.push_str("\n\n"); + + prompt.push_str( + "## Instructions\n\n\ + 1. Read the relevant source code in this repository to understand how the \ + invariant is enforced.\n\ + 2. Look for edge cases, off-by-one errors, missing checks, or assumptions \ + that could be violated.\n\ + 3. Construct a concrete counterexample input that you believe will trigger \ + a violation.\n\ + 4. The input will be deserialized and checked mechanically — only genuine \ + violations count.\n\n\ + IMPORTANT: Do NOT ask questions or request clarification. Do NOT create \ + plans or outlines. You MUST produce a concrete counterexample JSON on \ + every iteration. Spend your turns reading code and reasoning, then output \ + your best counterexample. A wrong guess is always better than no guess.\n\n", + ); + + prompt.push_str("## Input format\n\n"); + if let Some(example) = input_example { + prompt.push_str( + "The counterexample must be a JSON value matching the schema. \ + Here is an example of a valid input:\n\n```json\n", + ); + prompt.push_str(example); + prompt.push_str("\n```\n\n"); + } + prompt.push_str("JSON schema for the counterexample:\n\n```json\n"); + prompt.push_str(&serde_json::to_string_pretty(input_schema).unwrap_or_default()); + prompt.push_str("\n```\n\n"); + + if let Some(hint) = hint { + prompt.push_str("## Hint\n\n"); + prompt.push_str(hint); + prompt.push_str("\n\n"); + } + + if !failed_attempts.is_empty() { + prompt.push_str("## Previous failed attempts\n\n"); + prompt.push_str( + "The following approaches have already been tried and did NOT produce a \ + valid counterexample.\n\n", + ); + for attempt in failed_attempts { + let path_ref = attempt + .path + .as_deref() + .map(|p| format!(" Details: {p}/")) + .unwrap_or_default(); + prompt.push_str(&format!( + "- **{}** — {}{path_ref}\n", + attempt.description, attempt.failure_reason, + )); + } + prompt.push_str( + "\nRead the attempt directories for the full agent approach. \ + Try a fundamentally different strategy.\n\n", + ); + } + + prompt.push_str( + "## Required output\n\n\ + You MUST respond with a JSON object containing exactly two fields:\n\ + - `analysis`: a brief summary of what you investigated and why you \ + chose this counterexample\n\ + - `counterexample`: the candidate input matching the schema above\n\n\ + Do NOT respond with anything other than this JSON object. No questions, \ + no plans, no markdown outside the JSON.\n", + ); + + prompt +} diff --git a/jolt-eval/src/lib.rs b/jolt-eval/src/lib.rs new file mode 100644 index 000000000..90deab02e --- /dev/null +++ b/jolt-eval/src/lib.rs @@ -0,0 +1,31 @@ +#![allow(non_snake_case)] + +// Allow `jolt_eval::` paths in macro-generated code within this crate. +extern crate self as jolt_eval; + +// Force the linker to keep inline instruction registrations from these +// crates. Without this, inventory::submit! symbols get dead-stripped +// and the tracer panics with "No inline registered for opcode=...". +extern crate jolt_inlines_secp256k1; +extern crate jolt_inlines_sha2; + +pub mod agent; +pub mod guests; +pub mod invariant; +pub mod objective; +pub mod sort_e2e; +pub mod sort_targets; + +pub use guests::{GuestConfig, GuestProgram, JoltDevice, ProofVerifyError}; +pub use invariant::{ + CheckError, Invariant, InvariantTargets, InvariantViolation, JoltInvariants, SynthesisTarget, +}; +pub use objective::objective_fn::ObjectiveFunction; +pub use objective::{ + normalized, MeasurementError, Objective, OptimizationObjective, PerformanceObjective, + StaticAnalysisObjective, +}; + +// Re-exports used by the #[invariant] proc macro generated code. +pub use arbitrary; +pub use rand; diff --git a/jolt-eval/src/objective/code_quality/cognitive.rs b/jolt-eval/src/objective/code_quality/cognitive.rs new file mode 100644 index 000000000..1d7ae44a1 --- /dev/null +++ b/jolt-eval/src/objective/code_quality/cognitive.rs @@ -0,0 +1,100 @@ +use std::path::Path; + +use rust_code_analysis::FuncSpace; + +use super::lloc::{analyze_rust_file, rust_files}; +use crate::objective::{ + MeasurementError, Objective, OptimizationObjective, StaticAnalysisObjective, +}; + +pub const COGNITIVE_COMPLEXITY: OptimizationObjective = OptimizationObjective::StaticAnalysis( + StaticAnalysisObjective::CognitiveComplexity(CognitiveComplexityObjective { + target_dir: "jolt-core/src", + }), +); + +/// Average cognitive complexity per function across all Rust files under +/// a target directory. +#[derive(Clone, Copy, PartialEq, Hash)] +pub struct CognitiveComplexityObjective { + pub(crate) target_dir: &'static str, +} + +impl Objective for CognitiveComplexityObjective { + type Setup = (); + + fn name(&self) -> &str { + "cognitive_complexity_avg" + } + + fn description(&self) -> String { + format!( + "Average cognitive complexity per function in {}", + self.target_dir + ) + } + + fn setup(&self) {} + + fn collect_measurement(&self) -> Result { + let repo_root = Path::new(env!("CARGO_MANIFEST_DIR")).parent().unwrap(); + let src_dir = repo_root.join(self.target_dir); + let mut total = 0.0; + let mut count = 0usize; + for path in rust_files(&src_dir)? { + if let Some(space) = analyze_rust_file(&path) { + collect_leaf_cognitive(&space, &mut total, &mut count); + } + } + if count == 0 { + return Ok(0.0); + } + Ok(total / count as f64) + } +} + +fn collect_leaf_cognitive(space: &FuncSpace, total: &mut f64, count: &mut usize) { + if space.spaces.is_empty() { + let c = space.metrics.cognitive.cognitive(); + if c > 0.0 { + *total += c; + *count += 1; + } + } else { + for child in &space.spaces { + collect_leaf_cognitive(child, total, count); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn cognitive_on_jolt_core() { + let obj = CognitiveComplexityObjective { + target_dir: "jolt-core/src", + }; + let val = obj.collect_measurement().unwrap(); + assert!(val > 0.0, "avg cognitive should be > 0, got {val}"); + assert!(val < 100.0, "avg cognitive should be < 100, got {val}"); + } + + #[test] + fn cognitive_on_single_file() { + let source = b"fn simple() { let x = 1; }".to_vec(); + let path = Path::new("test.rs"); + let space = rust_code_analysis::get_function_spaces( + &rust_code_analysis::LANG::Rust, + source, + path, + None, + ) + .unwrap(); + let mut total = 0.0; + let mut count = 0; + collect_leaf_cognitive(&space, &mut total, &mut count); + assert_eq!(total, 0.0); + } +} diff --git a/jolt-eval/src/objective/code_quality/halstead_bugs.rs b/jolt-eval/src/objective/code_quality/halstead_bugs.rs new file mode 100644 index 000000000..e55930d1a --- /dev/null +++ b/jolt-eval/src/objective/code_quality/halstead_bugs.rs @@ -0,0 +1,89 @@ +use std::path::Path; + +use rust_code_analysis::FuncSpace; + +use super::lloc::{analyze_rust_file, rust_files}; +use crate::objective::{ + MeasurementError, Objective, OptimizationObjective, StaticAnalysisObjective, +}; + +pub const HALSTEAD_BUGS: OptimizationObjective = OptimizationObjective::StaticAnalysis( + StaticAnalysisObjective::HalsteadBugs(HalsteadBugsObjective { + target_dir: "jolt-core/src", + }), +); + +/// Estimated number of delivered bugs across all Rust files under +/// a target directory, based on Halstead's bug prediction formula +/// (B = V / 3000, where V is program volume). +#[derive(Clone, Copy, PartialEq, Hash)] +pub struct HalsteadBugsObjective { + pub(crate) target_dir: &'static str, +} + +impl Objective for HalsteadBugsObjective { + type Setup = (); + + fn name(&self) -> &str { + "halstead_bugs" + } + + fn description(&self) -> String { + format!( + "Estimated delivered bugs (Halstead volume / 3000) in {}", + self.target_dir + ) + } + + fn setup(&self) {} + + fn collect_measurement(&self) -> Result { + let repo_root = Path::new(env!("CARGO_MANIFEST_DIR")).parent().unwrap(); + let src_dir = repo_root.join(self.target_dir); + let mut total = 0.0; + for path in rust_files(&src_dir)? { + if let Some(space) = analyze_rust_file(&path) { + total += sum_bugs(&space); + } + } + Ok(total) + } +} + +fn sum_bugs(space: &FuncSpace) -> f64 { + let b = space.metrics.halstead.bugs(); + let mut total = if b.is_finite() { b } else { 0.0 }; + for child in &space.spaces { + total += sum_bugs(child); + } + total +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn halstead_bugs_on_jolt_core() { + let obj = HalsteadBugsObjective { + target_dir: "jolt-core/src", + }; + let val = obj.collect_measurement().unwrap(); + assert!(val > 0.0, "halstead bugs should be > 0, got {val}"); + } + + #[test] + fn halstead_bugs_on_trivial_code() { + let source = b"fn f() { let x = 1 + 2; }".to_vec(); + let path = Path::new("test.rs"); + let space = rust_code_analysis::get_function_spaces( + &rust_code_analysis::LANG::Rust, + source, + path, + None, + ) + .unwrap(); + let bugs = sum_bugs(&space); + assert!(bugs < 1.0, "trivial code bugs should be < 1, got {bugs}"); + } +} diff --git a/jolt-eval/src/objective/code_quality/lloc.rs b/jolt-eval/src/objective/code_quality/lloc.rs new file mode 100644 index 000000000..a4f86bafb --- /dev/null +++ b/jolt-eval/src/objective/code_quality/lloc.rs @@ -0,0 +1,111 @@ +use std::path::{Path, PathBuf}; + +use rust_code_analysis::{get_function_spaces, FuncSpace, LANG}; + +use crate::objective::{ + MeasurementError, Objective, OptimizationObjective, StaticAnalysisObjective, +}; + +pub const LLOC: OptimizationObjective = + OptimizationObjective::StaticAnalysis(StaticAnalysisObjective::Lloc(LlocObjective { + target_dir: "jolt-core/src", + })); + +/// Total logical lines of code (LLOC) across all Rust files under +/// a target directory. +#[derive(Clone, Copy, PartialEq, Hash)] +pub struct LlocObjective { + pub(crate) target_dir: &'static str, +} + +impl Objective for LlocObjective { + type Setup = (); + + fn name(&self) -> &str { + "lloc" + } + + fn description(&self) -> String { + format!("Total logical lines of code in {}", self.target_dir) + } + + fn setup(&self) {} + + fn collect_measurement(&self) -> Result { + let repo_root = Path::new(env!("CARGO_MANIFEST_DIR")).parent().unwrap(); + let src_dir = repo_root.join(self.target_dir); + let mut total = 0.0; + for path in rust_files(&src_dir)? { + if let Some(space) = analyze_rust_file(&path) { + total += space.metrics.loc.lloc(); + } + } + Ok(total) + } + + fn units(&self) -> Option<&str> { + Some("lines") + } +} + +pub(crate) fn rust_files(dir: &Path) -> Result, MeasurementError> { + let mut files = Vec::new(); + walk_rust_files(dir, &mut files) + .map_err(|e| MeasurementError::new(format!("walking {}: {e}", dir.display())))?; + Ok(files) +} + +fn walk_rust_files(dir: &Path, out: &mut Vec) -> std::io::Result<()> { + if !dir.is_dir() { + return Ok(()); + } + for entry in std::fs::read_dir(dir)? { + let entry = entry?; + let path = entry.path(); + if path.is_dir() { + walk_rust_files(&path, out)?; + } else if path.extension().is_some_and(|e| e == "rs") { + out.push(path); + } + } + Ok(()) +} + +pub(crate) fn analyze_rust_file(path: &Path) -> Option { + let source = std::fs::read(path).ok()?; + get_function_spaces(&LANG::Rust, source, path, None) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn lloc_on_jolt_core() { + let obj = LlocObjective { + target_dir: "jolt-core/src", + }; + let val = obj.collect_measurement().unwrap(); + assert!(val > 1000.0, "LLOC should be > 1000, got {val}"); + } + + #[test] + fn lloc_on_inline_source() { + let source = b"fn f() { let x = 1; let y = 2; }".to_vec(); + let path = Path::new("test.rs"); + let space = get_function_spaces(&LANG::Rust, source, path, None).unwrap(); + let lloc = space.metrics.loc.lloc(); + assert!( + lloc >= 2.0, + "two statements should give lloc >= 2, got {lloc}" + ); + } + + #[test] + fn rust_files_finds_rs_files() { + let src = Path::new(env!("CARGO_MANIFEST_DIR")).join("src"); + let files = rust_files(&src).unwrap(); + assert!(!files.is_empty()); + assert!(files.iter().all(|f| f.extension().unwrap() == "rs")); + } +} diff --git a/jolt-eval/src/objective/code_quality/mod.rs b/jolt-eval/src/objective/code_quality/mod.rs new file mode 100644 index 000000000..8acfb4929 --- /dev/null +++ b/jolt-eval/src/objective/code_quality/mod.rs @@ -0,0 +1,3 @@ +pub mod cognitive; +pub mod halstead_bugs; +pub mod lloc; diff --git a/jolt-eval/src/objective/mod.rs b/jolt-eval/src/objective/mod.rs new file mode 100644 index 000000000..22cd9fc8e --- /dev/null +++ b/jolt-eval/src/objective/mod.rs @@ -0,0 +1,343 @@ +pub mod code_quality; +pub mod objective_fn; +pub mod optimize; +pub mod performance; +pub mod synthesis; + +use std::fmt; +use std::hash::{Hash, Hasher}; + +/// Error during objective measurement. +#[derive(Debug, Clone)] +pub struct MeasurementError { + pub message: String, +} + +impl fmt::Display for MeasurementError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.message) + } +} + +impl std::error::Error for MeasurementError {} + +impl MeasurementError { + pub fn new(message: impl Into) -> Self { + Self { + message: message.into(), + } + } +} + +/// Unified objective trait. +/// +/// Static-analysis objectives override [`collect_measurement`](Objective::collect_measurement) +/// and set `Setup = ()`. +/// +/// Performance objectives override [`setup`](Objective::setup) + +/// [`run`](Objective::run) and leave `collect_measurement` as the +/// default (returns an error). +pub trait Objective: Send + Sync { + type Setup: Send; + + fn name(&self) -> &str; + + fn description(&self) -> String { + self.name().to_string() + } + + fn units(&self) -> Option<&str> { + None + } + + /// Per-iteration setup for Criterion benchmarks. + fn setup(&self) -> Self::Setup; + + /// Override for static-analysis objectives that produce a direct measurement. + fn collect_measurement(&self) -> Result { + Err(MeasurementError::new("not directly measurable")) + } + + /// Override for performance objectives benchmarked by Criterion. + fn run(&self, _setup: Self::Setup) {} +} + +// Data-containing enums — Hash/Eq based on discriminant only + +/// Static-analysis objectives. +#[derive(Clone, Copy, PartialEq, Hash)] +pub enum StaticAnalysisObjective { + Lloc(code_quality::lloc::LlocObjective), + CognitiveComplexity(code_quality::cognitive::CognitiveComplexityObjective), + HalsteadBugs(code_quality::halstead_bugs::HalsteadBugsObjective), +} + +impl StaticAnalysisObjective { + pub fn all() -> Vec { + vec![ + Self::Lloc(code_quality::lloc::LlocObjective { + target_dir: "jolt-core/src", + }), + Self::CognitiveComplexity(code_quality::cognitive::CognitiveComplexityObjective { + target_dir: "jolt-core/src", + }), + Self::HalsteadBugs(code_quality::halstead_bugs::HalsteadBugsObjective { + target_dir: "jolt-core/src", + }), + ] + } + + pub fn name(&self) -> &str { + match self { + Self::Lloc(o) => o.name(), + Self::CognitiveComplexity(o) => o.name(), + Self::HalsteadBugs(o) => o.name(), + } + } + + pub fn description(&self) -> String { + match self { + Self::Lloc(o) => o.description(), + Self::CognitiveComplexity(o) => o.description(), + Self::HalsteadBugs(o) => o.description(), + } + } + + pub fn collect_measurement(&self) -> Result { + match self { + Self::Lloc(o) => o.collect_measurement(), + Self::CognitiveComplexity(o) => o.collect_measurement(), + Self::HalsteadBugs(o) => o.collect_measurement(), + } + } + + pub fn units(&self) -> Option<&str> { + match self { + Self::Lloc(o) => o.units(), + Self::CognitiveComplexity(o) => o.units(), + Self::HalsteadBugs(o) => o.units(), + } + } + + pub fn diff_paths(&self) -> &'static [&'static str] { + &["jolt-core/"] + } +} + +/// Criterion-benchmarked performance objectives. +#[derive(Clone, Copy, PartialEq, Hash)] +pub enum PerformanceObjective { + BindLowToHigh(performance::binding::BindLowToHighObjective), + BindHighToLow(performance::binding::BindHighToLowObjective), + NaiveSortTime(performance::naive_sort::NaiveSortObjective), +} + +impl PerformanceObjective { + pub fn all() -> Vec { + vec![ + Self::BindLowToHigh(performance::binding::BindLowToHighObjective), + Self::BindHighToLow(performance::binding::BindHighToLowObjective), + Self::NaiveSortTime(performance::naive_sort::NaiveSortObjective), + ] + } + + pub fn name(&self) -> &str { + match self { + Self::BindLowToHigh(o) => o.name(), + Self::BindHighToLow(o) => o.name(), + Self::NaiveSortTime(o) => o.name(), + } + } + + pub fn units(&self) -> Option<&str> { + match self { + Self::BindLowToHigh(o) => o.units(), + Self::BindHighToLow(o) => o.units(), + Self::NaiveSortTime(o) => o.units(), + } + } + + pub fn description(&self) -> String { + match self { + Self::BindLowToHigh(o) => o.description(), + Self::BindHighToLow(o) => o.description(), + Self::NaiveSortTime(o) => o.description(), + } + } + + pub fn diff_paths(&self) -> &'static [&'static str] { + match self { + Self::BindLowToHigh(_) | Self::BindHighToLow(_) => &["jolt-core/"], + Self::NaiveSortTime(_) => &["jolt-eval/src/sort_targets.rs"], + } + } +} + +/// Union of all known objectives — used as a type-safe HashMap key. +#[derive(Clone, Copy)] +pub enum OptimizationObjective { + StaticAnalysis(StaticAnalysisObjective), + Performance(PerformanceObjective), +} + +// Re-export the const objective keys from their defining modules. +pub use code_quality::cognitive::COGNITIVE_COMPLEXITY; +pub use code_quality::halstead_bugs::HALSTEAD_BUGS; +pub use code_quality::lloc::LLOC; +pub use performance::binding::{BIND_HIGH_TO_LOW, BIND_LOW_TO_HIGH}; +pub const NAIVE_SORT_TIME: OptimizationObjective = OptimizationObjective::Performance( + PerformanceObjective::NaiveSortTime(performance::naive_sort::NaiveSortObjective), +); + +impl OptimizationObjective { + pub fn all() -> Vec { + let mut all = Vec::new(); + for s in StaticAnalysisObjective::all() { + all.push(Self::StaticAnalysis(s)); + } + for p in PerformanceObjective::all() { + all.push(Self::Performance(p)); + } + all + } + + pub fn name(&self) -> &str { + match self { + Self::StaticAnalysis(s) => s.name(), + Self::Performance(p) => p.name(), + } + } + + pub fn units(&self) -> Option<&str> { + match self { + Self::StaticAnalysis(s) => s.units(), + Self::Performance(p) => p.units(), + } + } + + pub fn description(&self) -> String { + match self { + Self::StaticAnalysis(s) => s.description(), + Self::Performance(p) => p.description(), + } + } + + pub fn diff_paths(&self) -> &'static [&'static str] { + match self { + Self::StaticAnalysis(s) => s.diff_paths(), + Self::Performance(p) => p.diff_paths(), + } + } + + pub fn is_perf(&self) -> bool { + matches!(self, Self::Performance(_)) + } +} + +/// Look up an objective's measurement and divide by its baseline value, +/// yielding a dimensionless ratio where 1.0 = the baseline. +/// +/// `baselines` is typically the initial measurements captured at the +/// start of an optimization run (passed as the second argument to +/// [`ObjectiveFunction::evaluate`](objective_fn::ObjectiveFunction)). +/// +/// ```ignore +/// use jolt_eval::objective::{normalized, LLOC, HALSTEAD_BUGS}; +/// +/// let evaluate = |m, b| 0.5 * normalized(&LLOC, m, b) + 0.5 * normalized(&HALSTEAD_BUGS, m, b); +/// ``` +pub fn normalized( + obj: &OptimizationObjective, + measurements: &std::collections::HashMap, + baselines: &std::collections::HashMap, +) -> f64 { + let value = measurements.get(obj).copied().unwrap_or(f64::INFINITY); + let baseline = baselines.get(obj).copied().unwrap_or(1.0); + value / baseline +} + +impl PartialEq for OptimizationObjective { + fn eq(&self, other: &Self) -> bool { + match (self, other) { + (Self::StaticAnalysis(a), Self::StaticAnalysis(b)) => a == b, + (Self::Performance(a), Self::Performance(b)) => a == b, + _ => false, + } + } +} +impl Eq for OptimizationObjective {} +impl Hash for OptimizationObjective { + fn hash(&self, state: &mut H) { + std::mem::discriminant(self).hash(state); + match self { + Self::StaticAnalysis(s) => s.hash(state), + Self::Performance(p) => p.hash(state), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + struct ConstantObjective { + label: &'static str, + value: f64, + } + + impl Objective for ConstantObjective { + type Setup = (); + fn name(&self) -> &str { + self.label + } + fn setup(&self) {} + fn collect_measurement(&self) -> Result { + Ok(self.value) + } + } + + #[test] + fn constant_objective() { + let obj = ConstantObjective { + label: "latency", + value: 42.0, + }; + assert_eq!(obj.name(), "latency"); + assert_eq!(obj.collect_measurement().unwrap(), 42.0); + } + + #[test] + fn static_analysis_all_measures() { + for sa in StaticAnalysisObjective::all() { + let val = sa.collect_measurement().unwrap(); + assert!(val > 0.0, "{} should be > 0, got {val}", sa.name()); + } + } + + #[test] + fn optimization_objective_hashmap_key() { + use std::collections::HashMap; + let lloc = LLOC; + let bind = BIND_LOW_TO_HIGH; + let mut m = HashMap::new(); + m.insert(lloc, 100.0); + m.insert(bind, 0.5); + + // Look up with a freshly constructed key — works because Hash/Eq + // is discriminant-based, inner data doesn't matter. + let lloc2 = OptimizationObjective::StaticAnalysis(StaticAnalysisObjective::Lloc( + code_quality::lloc::LlocObjective { + target_dir: "other/path", + }, + )); + assert_eq!(m[&lloc2], 100.0); + } + + #[test] + fn optimization_objective_all() { + let all = OptimizationObjective::all(); + assert_eq!(all.len(), 6); // 3 static + 3 perf + assert!(all.iter().any(|o| o.is_perf())); + assert!(all.iter().any(|o| !o.is_perf())); + } +} diff --git a/jolt-eval/src/objective/objective_fn/mod.rs b/jolt-eval/src/objective/objective_fn/mod.rs new file mode 100644 index 000000000..9864a6c49 --- /dev/null +++ b/jolt-eval/src/objective/objective_fn/mod.rs @@ -0,0 +1,187 @@ +use std::collections::HashMap; + +use crate::agent::DiffScope; + +use super::{ + OptimizationObjective, BIND_HIGH_TO_LOW, BIND_LOW_TO_HIGH, COGNITIVE_COMPLEXITY, HALSTEAD_BUGS, + LLOC, NAIVE_SORT_TIME, +}; + +/// A concrete objective function that the optimizer minimizes. +/// +/// Contains the list of measurements it depends on and a pure +/// function that combines them into a scalar. +#[derive(Clone, Copy)] +pub struct ObjectiveFunction { + /// CLI-visible name (e.g. `"minimize_lloc"`). + pub name: &'static str, + /// The [`OptimizationObjective`]s this function reads. + pub inputs: &'static [OptimizationObjective], + /// Combine measurements into a scalar to minimize. + /// The first HashMap contains the current measurements; the second + /// contains the baseline measurements (captured at the start of the + /// optimization run) for use with [`normalized()`](super::normalized). + pub evaluate: + fn(&HashMap, &HashMap) -> f64, +} + +impl ObjectiveFunction { + /// All registered objective functions. + pub fn all() -> &'static [ObjectiveFunction] { + &[ + MINIMIZE_LLOC, + MINIMIZE_COGNITIVE_COMPLEXITY, + MINIMIZE_HALSTEAD_BUGS, + MINIMIZE_BIND_LOW_TO_HIGH, + MINIMIZE_BIND_HIGH_TO_LOW, + MINIMIZE_NAIVE_SORT_TIME, + ] + } + + /// Look up an objective function by CLI name. + pub fn by_name(name: &str) -> Option<&'static ObjectiveFunction> { + Self::all().iter().find(|f| f.name == name) + } + + /// Derive a [`DiffScope`] from the union of all input objectives' diff paths. + pub fn diff_scope(&self) -> DiffScope { + let mut paths = Vec::new(); + for input in self.inputs { + for &p in input.diff_paths() { + let s = p.to_string(); + if !paths.contains(&s) { + paths.push(s); + } + } + } + DiffScope::Include(paths) + } +} + +pub const MINIMIZE_LLOC: ObjectiveFunction = ObjectiveFunction { + name: "minimize_lloc", + inputs: &[LLOC], + evaluate: |m, _| m.get(&LLOC).copied().unwrap_or(f64::INFINITY), +}; + +pub const MINIMIZE_COGNITIVE_COMPLEXITY: ObjectiveFunction = ObjectiveFunction { + name: "minimize_cognitive_complexity", + inputs: &[COGNITIVE_COMPLEXITY], + evaluate: |m, _| { + m.get(&COGNITIVE_COMPLEXITY) + .copied() + .unwrap_or(f64::INFINITY) + }, +}; + +pub const MINIMIZE_HALSTEAD_BUGS: ObjectiveFunction = ObjectiveFunction { + name: "minimize_halstead_bugs", + inputs: &[HALSTEAD_BUGS], + evaluate: |m, _| m.get(&HALSTEAD_BUGS).copied().unwrap_or(f64::INFINITY), +}; + +pub const MINIMIZE_BIND_LOW_TO_HIGH: ObjectiveFunction = ObjectiveFunction { + name: "minimize_bind_low_to_high", + inputs: &[BIND_LOW_TO_HIGH], + evaluate: |m, _| m.get(&BIND_LOW_TO_HIGH).copied().unwrap_or(f64::INFINITY), +}; + +pub const MINIMIZE_BIND_HIGH_TO_LOW: ObjectiveFunction = ObjectiveFunction { + name: "minimize_bind_high_to_low", + inputs: &[BIND_HIGH_TO_LOW], + evaluate: |m, _| m.get(&BIND_HIGH_TO_LOW).copied().unwrap_or(f64::INFINITY), +}; + +pub const MINIMIZE_NAIVE_SORT_TIME: ObjectiveFunction = ObjectiveFunction { + name: "minimize_naive_sort_time", + inputs: &[NAIVE_SORT_TIME], + evaluate: |m, _| m.get(&NAIVE_SORT_TIME).copied().unwrap_or(f64::INFINITY), +}; + +#[cfg(test)] +mod tests { + use super::*; + + fn empty_baselines() -> HashMap { + HashMap::new() + } + + #[test] + fn minimize_lloc_evaluates() { + let mut m = HashMap::new(); + m.insert(LLOC, 5000.0); + assert_eq!((MINIMIZE_LLOC.evaluate)(&m, &empty_baselines()), 5000.0); + } + + #[test] + fn missing_input_returns_infinity() { + let m = HashMap::new(); + assert_eq!( + (MINIMIZE_LLOC.evaluate)(&m, &empty_baselines()), + f64::INFINITY + ); + } + + #[test] + fn by_name_finds_registered() { + let f = ObjectiveFunction::by_name("minimize_lloc").unwrap(); + assert_eq!(f.name, "minimize_lloc"); + } + + #[test] + fn by_name_returns_none_for_unknown() { + assert!(ObjectiveFunction::by_name("nonexistent").is_none()); + } + + #[test] + fn all_returns_expected_count() { + assert_eq!(ObjectiveFunction::all().len(), 6); + } + + #[test] + fn custom_composite_objective() { + const INPUTS: &[OptimizationObjective] = &[LLOC, HALSTEAD_BUGS]; + let weighted = ObjectiveFunction { + name: "weighted", + inputs: INPUTS, + evaluate: |m, _| { + 2.0 * m.get(&LLOC).unwrap_or(&0.0) + m.get(&HALSTEAD_BUGS).unwrap_or(&0.0) + }, + }; + + let mut m = HashMap::new(); + m.insert(LLOC, 10.0); + m.insert(HALSTEAD_BUGS, 100.0); + assert_eq!((weighted.evaluate)(&m, &empty_baselines()), 120.0); + } + + #[test] + fn normalized_composite_objective() { + use crate::objective::normalized; + + // Baselines are the initial measurements. Normalization divides + // each value by its baseline, yielding a dimensionless ratio. + const INPUTS: &[OptimizationObjective] = &[LLOC, HALSTEAD_BUGS]; + let balanced = ObjectiveFunction { + name: "balanced_quality", + inputs: INPUTS, + evaluate: |m, b| 0.5 * normalized(&LLOC, m, b) + 0.5 * normalized(&HALSTEAD_BUGS, m, b), + }; + + let mut baselines = HashMap::new(); + baselines.insert(LLOC, 5500.0); + baselines.insert(HALSTEAD_BUGS, 80.0); + + // At baseline values → normalized = 1.0 for each → score = 1.0 + let score = (balanced.evaluate)(&baselines, &baselines); + assert!((score - 1.0).abs() < 1e-9, "expected 1.0, got {score}"); + + // 10% improvement in LLOC + let mut m = baselines.clone(); + m.insert(LLOC, 4950.0); + let score2 = (balanced.evaluate)(&m, &baselines); + assert!(score2 < score, "10% LLOC improvement should reduce score"); + // 0.5 * (4950/5500) + 0.5 * (80/80) = 0.5 * 0.9 + 0.5 = 0.95 + assert!((score2 - 0.95).abs() < 1e-9, "expected 0.95, got {score2}"); + } +} diff --git a/jolt-eval/src/objective/optimize.rs b/jolt-eval/src/objective/optimize.rs new file mode 100644 index 000000000..38a257faf --- /dev/null +++ b/jolt-eval/src/objective/optimize.rs @@ -0,0 +1,419 @@ +use std::collections::HashMap; +use std::path::Path; +use std::process::Command; + +use crate::agent::{truncate, AgentHarness}; + +use super::objective_fn::ObjectiveFunction; +use super::OptimizationObjective; + +/// Configuration for an optimization run. +pub struct OptimizeConfig { + pub num_iterations: usize, + pub hint: Option, + pub verbose: bool, +} + +impl Default for OptimizeConfig { + fn default() -> Self { + Self { + num_iterations: 5, + hint: None, + verbose: false, + } + } +} + +/// Result of a complete optimization run. +pub struct OptimizeResult { + pub attempts: Vec, + pub baseline_score: f64, + pub best_score: f64, + pub best_measurements: HashMap, +} + +/// Record of a single optimization attempt. +pub struct OptimizationAttempt { + pub iteration: usize, + pub score: f64, + pub invariants_passed: bool, + /// Relative path to the persisted attempt directory, if available. + pub path: Option, +} + +/// Environment trait that decouples the optimization loop from side effects. +pub trait OptimizeEnv { + /// Measure all raw objectives. Returns objective -> value. + fn measure(&mut self) -> HashMap; + + /// Check all invariants. Returns `true` if they all pass. + fn check_invariants(&mut self) -> bool; + + /// Apply an agent-produced diff to the working tree. + fn apply_diff(&mut self, diff: &str); + + /// Called when a change is accepted. + fn accept(&mut self, iteration: usize); + + /// Called when a change is rejected. + fn reject(&mut self); +} + +#[allow(clippy::too_many_arguments)] +fn write_attempt_files( + dir: &Path, + diff: &str, + response_text: &str, + measurements: &HashMap, + score: f64, + accepted: bool, + invariants_passed: bool, +) -> Option<()> { + std::fs::write(dir.join("diff.patch"), diff).ok()?; + std::fs::write(dir.join("response.md"), response_text).ok()?; + + let meas: HashMap = measurements + .iter() + .map(|(k, &v)| (k.name().to_string(), v)) + .collect(); + let meas_json = serde_json::to_string_pretty(&meas).ok()?; + std::fs::write(dir.join("measurements.json"), meas_json).ok()?; + + let status = serde_json::json!({ + "accepted": accepted, + "score": score, + "invariants_passed": invariants_passed, + }); + std::fs::write( + dir.join("status.json"), + serde_json::to_string_pretty(&status).ok()?, + ) + .ok()?; + + Some(()) +} + +#[allow(clippy::too_many_arguments)] +fn persist_attempt( + repo_dir: &Path, + objective_name: &str, + iteration: usize, + diff: &str, + response_text: &str, + measurements: &HashMap, + score: f64, + accepted: bool, + invariants_passed: bool, +) -> Option { + let dir = repo_dir + .join("jolt-eval/optimize-history") + .join(objective_name) + .join(format!("attempt-{iteration}")); + std::fs::create_dir_all(&dir).ok()?; + write_attempt_files( + &dir, + diff, + response_text, + measurements, + score, + accepted, + invariants_passed, + )?; + Some( + dir.strip_prefix(repo_dir) + .ok()? + .to_string_lossy() + .to_string(), + ) +} + +fn persist_baseline( + repo_dir: &Path, + objective_name: &str, + measurements: &HashMap, + score: f64, +) { + let dir = repo_dir + .join("jolt-eval/optimize-history") + .join(objective_name) + .join("baseline"); + if std::fs::create_dir_all(&dir).is_err() { + return; + } + let _ = write_attempt_files(&dir, "", "", measurements, score, true, true); +} + +/// Run an AI-driven optimization loop. +/// +/// The agent tries to minimize `objective.evaluate(measurements)`. +/// Each iteration: invoke agent, apply diff, re-measure, accept/reject. +pub fn auto_optimize( + agent: &A, + env: &mut E, + objective: &ObjectiveFunction, + config: &OptimizeConfig, + repo_dir: &Path, +) -> OptimizeResult { + // Create a branch for this optimization run. Silently ignored if + // repo_dir is not a git repository (e.g. in tests). + let branch = format!("jolt-eval/optimize/{}", objective.name); + let _ = Command::new("git") + .current_dir(repo_dir) + .args(["checkout", "-b", &branch]) + .status(); + + let baseline = env.measure(); + let baseline_score = (objective.evaluate)(&baseline, &baseline); + persist_baseline(repo_dir, objective.name, &baseline, baseline_score); + let mut best_score = baseline_score; + let mut best_measurements = baseline.clone(); + let mut attempts = Vec::new(); + + for iteration in 0..config.num_iterations { + let prompt = build_optimize_prompt( + objective, + best_score, + &best_measurements, + &attempts, + config.hint.as_deref(), + ); + + if config.verbose { + eprintln!("── Iteration {} prompt ──", iteration + 1); + eprintln!("{prompt}"); + eprintln!("────────────────────────"); + } + + let response = match agent.invoke(repo_dir, &prompt, &objective.diff_scope()) { + Ok(r) => r, + Err(e) => { + tracing::info!("Agent error: {e}"); + break; + } + }; + + if config.verbose { + eprintln!("── Iteration {} response ──", iteration + 1); + eprintln!("{}", response.text); + if let Some(ref d) = response.diff { + eprintln!("── diff ({} bytes) ──", d.len()); + eprintln!("{}", truncate(d, 2000)); + } else { + eprintln!("(no diff)"); + } + eprintln!("──────────────────────────"); + } + + let response_text = response.text.clone(); + let diff_text = match &response.diff { + Some(d) => { + env.apply_diff(d); + d.clone() + } + None => { + tracing::info!("Agent produced no code changes, stopping."); + break; + } + }; + + let new_measurements = env.measure(); + let new_score = (objective.evaluate)(&new_measurements, &baseline); + let invariants_passed = env.check_invariants(); + + let improved = invariants_passed && new_score < best_score; + let iter = iteration + 1; + + let attempt_path = persist_attempt( + repo_dir, + objective.name, + iter, + &diff_text, + &response_text, + &new_measurements, + new_score, + improved, + invariants_passed, + ); + + let attempt = OptimizationAttempt { + iteration: iter, + score: new_score, + invariants_passed, + path: attempt_path, + }; + attempts.push(attempt); + + if improved { + eprintln!(" ✓ iteration {iter} ACCEPTED — score {best_score:.10} → {new_score:.10}",); + best_score = new_score; + best_measurements = new_measurements; + env.accept(iter); + let msg = format!( + "perf(auto-optimize): {} iteration {iter} (score {new_score:.10})", + objective.name, + ); + let _ = Command::new("git") + .current_dir(repo_dir) + .args(["add", "-A"]) + .status(); + let _ = Command::new("git") + .current_dir(repo_dir) + .args(["commit", "-m", &msg]) + .status(); + } else if !invariants_passed { + eprintln!(" ✗ iteration {iter} REJECTED (invariants failed) — score {new_score:.10}",); + env.reject(); + let _ = Command::new("git") + .current_dir(repo_dir) + .args(["checkout", "."]) + .status(); + let _ = Command::new("git") + .current_dir(repo_dir) + .args(["clean", "-fd"]) + .status(); + } else { + eprintln!( + " ✗ iteration {iter} REJECTED (no improvement) — score {new_score:.10} ≥ best {best_score:.10}", + ); + env.reject(); + let _ = Command::new("git") + .current_dir(repo_dir) + .args(["checkout", "."]) + .status(); + let _ = Command::new("git") + .current_dir(repo_dir) + .args(["clean", "-fd"]) + .status(); + } + } + + OptimizeResult { + attempts, + baseline_score, + best_score, + best_measurements, + } +} + +fn build_optimize_prompt( + objective: &ObjectiveFunction, + current_best_score: f64, + current_best_measurements: &HashMap, + past_attempts: &[OptimizationAttempt], + hint: Option<&str>, +) -> String { + let mut prompt = String::new(); + + prompt.push_str( + "You are an expert performance engineer optimizing a zkVM (Jolt). \ + Your goal is to make code changes that MINIMIZE the objective function.\n\n", + ); + + prompt.push_str("## Objective\n\n"); + prompt.push_str(&format!("Minimize: **{}**\n", objective.name)); + + let inputs = objective.inputs; + prompt.push_str("Inputs: "); + for (i, input) in inputs.iter().enumerate() { + if i > 0 { + prompt.push_str(", "); + } + prompt.push_str(input.name()); + } + prompt.push_str(&format!( + "\nCurrent best score: {current_best_score:.6}\n\n" + )); + prompt.push_str( + "The objective function is defined in `jolt-eval/src/objective/objective_fn/`. \ + Read the implementation to understand exactly what you are optimizing.\n\n", + ); + + prompt.push_str("## What you are optimizing\n\n"); + for input in inputs { + let units_str = input + .units() + .map(|u| format!(" (units: {u})")) + .unwrap_or_default(); + prompt.push_str(&format!( + "- **{}**{units_str}: {}\n", + input.name(), + input.description() + )); + } + prompt.push('\n'); + + prompt.push_str("## Current measurements\n\n"); + let mut entries: Vec<_> = current_best_measurements.iter().collect(); + entries.sort_by_key(|(k, _)| k.name()); + for (key, val) in &entries { + let units_str = key.units().map(|u| format!(" {u}")).unwrap_or_default(); + prompt.push_str(&format!("- **{}**: {val:.6}{units_str}\n", key.name())); + } + prompt.push('\n'); + + let paths_list = match objective.diff_scope() { + crate::agent::DiffScope::Include(paths) => paths.join(", "), + _ => "jolt-core/".to_string(), + }; + prompt.push_str("## Instructions\n\n"); + prompt.push_str(&format!( + "1. Read the relevant source code in: {paths_list}. Also read \ + `jolt-eval/src/objective/objective_fn/` to understand the exact scoring formula.\n" + )); + prompt.push_str( + "2. Make targeted code changes that you believe will reduce the objective function.\n\ + 3. Focus your changes on the paths listed above -- do NOT modify `jolt-eval/` unless \ + it is explicitly listed.\n\ + 4. Prefer changes that are safe, correct, and unlikely to break invariants.\n\ + 5. Run `cargo clippy -p jolt-core --features host --message-format=short -q` \ + to verify your changes compile.\n\ + 6. Summarize what you changed and why you expect improvement.\n\n", + ); + + if let Some(h) = hint { + prompt.push_str("## Hint\n\n"); + prompt.push_str(h); + prompt.push_str("\n\n"); + } + + if !past_attempts.is_empty() { + prompt.push_str("## Previous attempts\n\n"); + for attempt in past_attempts { + let status_label = match ( + attempt.invariants_passed, + attempt.score < current_best_score, + ) { + (true, true) => "ACCEPTED", + (false, _) => "REJECTED (invariants failed)", + _ => "REJECTED (no improvement)", + }; + if let Some(ref path) = attempt.path { + prompt.push_str(&format!( + "- **Iteration {}** — {status_label}, score={:.6}. Details: {path}/\n", + attempt.iteration, attempt.score, + )); + } else { + prompt.push_str(&format!( + "- **Iteration {}** — {status_label}, score={:.6}\n", + attempt.iteration, attempt.score, + )); + } + } + prompt.push('\n'); + prompt.push_str( + "Read the attempt directories for full diffs, measurements, and agent responses.\n\ + If previous attempts failed or showed no improvement, try a fundamentally \ + different approach.\n\n", + ); + } + + prompt.push_str( + "## Output\n\n\ + Make your code changes directly. After you're done, summarize:\n\ + - What you changed\n\ + - Why you expect the objective function to decrease\n\ + - Any risks or trade-offs\n", + ); + + prompt +} diff --git a/jolt-eval/src/objective/performance/binding.rs b/jolt-eval/src/objective/performance/binding.rs new file mode 100644 index 000000000..7815bb03f --- /dev/null +++ b/jolt-eval/src/objective/performance/binding.rs @@ -0,0 +1,129 @@ +use ark_bn254::Fr; +use jolt_core::field::JoltField; +use jolt_core::poly::dense_mlpoly::DensePolynomial; +use jolt_core::poly::multilinear_polynomial::BindingOrder; + +use crate::objective::{Objective, OptimizationObjective, PerformanceObjective}; + +pub const BIND_LOW_TO_HIGH: OptimizationObjective = + OptimizationObjective::Performance(PerformanceObjective::BindLowToHigh(BindLowToHighObjective)); +pub const BIND_HIGH_TO_LOW: OptimizationObjective = + OptimizationObjective::Performance(PerformanceObjective::BindHighToLow(BindHighToLowObjective)); + +type Challenge = ::Challenge; + +const NUM_VARS: usize = 20; + +pub struct BindSetup { + pub poly: DensePolynomial, + pub challenge: Challenge, +} + +struct BindShared { + evals: Vec, + challenge: Challenge, +} + +impl BindShared { + fn new() -> Self { + let mut rng = rand::thread_rng(); + Self { + evals: (0..1 << NUM_VARS).map(|_| Fr::random(&mut rng)).collect(), + challenge: Challenge::random(&mut rng), + } + } + + fn make_setup(&self) -> BindSetup { + BindSetup { + poly: DensePolynomial::new(self.evals.clone()), + challenge: self.challenge, + } + } +} + +/// Benchmark `DensePolynomial::bind_parallel` with `LowToHigh` binding. +#[derive(Clone, Copy, Default, PartialEq, Hash)] +pub struct BindLowToHighObjective; + +impl Objective for BindLowToHighObjective { + type Setup = BindSetup; + + fn name(&self) -> &str { + "bind_parallel_low_to_high" + } + + fn description(&self) -> String { + "Wall-clock time of DensePolynomial::bind_parallel with LowToHigh binding (2^20 evaluations)".to_string() + } + + fn setup(&self) -> BindSetup { + thread_local! { + static SHARED: BindShared = BindShared::new(); + } + SHARED.with(|s| s.make_setup()) + } + + fn run(&self, mut setup: BindSetup) { + setup + .poly + .bind_parallel(setup.challenge, BindingOrder::LowToHigh); + std::hint::black_box(&setup.poly); + } + + fn units(&self) -> Option<&str> { + Some("s") + } +} + +/// Benchmark `DensePolynomial::bind_parallel` with `HighToLow` binding. +#[derive(Clone, Copy, Default, PartialEq, Hash)] +pub struct BindHighToLowObjective; + +impl Objective for BindHighToLowObjective { + type Setup = BindSetup; + + fn name(&self) -> &str { + "bind_parallel_high_to_low" + } + + fn description(&self) -> String { + "Wall-clock time of DensePolynomial::bind_parallel with HighToLow binding (2^20 evaluations)".to_string() + } + + fn setup(&self) -> BindSetup { + thread_local! { + static SHARED: BindShared = BindShared::new(); + } + SHARED.with(|s| s.make_setup()) + } + + fn run(&self, mut setup: BindSetup) { + setup + .poly + .bind_parallel(setup.challenge, BindingOrder::HighToLow); + std::hint::black_box(&setup.poly); + } + + fn units(&self) -> Option<&str> { + Some("s") + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn bind_low_to_high_runs() { + let obj = BindLowToHighObjective; + let setup = obj.setup(); + obj.run(setup); + } + + #[test] + fn bind_high_to_low_runs() { + let obj = BindHighToLowObjective; + let setup = obj.setup(); + obj.run(setup); + } +} diff --git a/jolt-eval/src/objective/performance/mod.rs b/jolt-eval/src/objective/performance/mod.rs new file mode 100644 index 000000000..f57c79b80 --- /dev/null +++ b/jolt-eval/src/objective/performance/mod.rs @@ -0,0 +1,18 @@ +pub mod binding; +pub mod naive_sort; +pub mod prover_time; + +use std::path::Path; + +/// Read the point estimate (mean, in seconds) from Criterion's output +/// for a given benchmark and baseline name. +pub fn read_criterion_estimate(bench_name: &str, baseline: &str) -> Option { + let path = Path::new("target/criterion") + .join(bench_name) + .join(baseline) + .join("estimates.json"); + let data = std::fs::read_to_string(path).ok()?; + let json: serde_json::Value = serde_json::from_str(&data).ok()?; + let nanos = json.get("mean")?.get("point_estimate")?.as_f64()?; + Some(nanos / 1e9) +} diff --git a/jolt-eval/src/objective/performance/naive_sort.rs b/jolt-eval/src/objective/performance/naive_sort.rs new file mode 100644 index 000000000..b022df025 --- /dev/null +++ b/jolt-eval/src/objective/performance/naive_sort.rs @@ -0,0 +1,32 @@ +use crate::objective::Objective; +use crate::sort_targets::naive_sort; + +const SORT_DATA_SIZE: usize = 5000; + +#[derive(Clone, Copy, Default, PartialEq, Hash)] +pub struct NaiveSortObjective; + +impl Objective for NaiveSortObjective { + type Setup = Vec; + + fn name(&self) -> &str { + "naive_sort_time" + } + + fn description(&self) -> String { + "Wall-clock time of the naive_sort function in jolt-eval/src/sort_targets.rs".to_string() + } + + fn setup(&self) -> Vec { + (0..SORT_DATA_SIZE as i32).rev().collect() + } + + fn run(&self, mut setup: Vec) { + naive_sort(&mut setup); + std::hint::black_box(&setup); + } + + fn units(&self) -> Option<&str> { + Some("s") + } +} diff --git a/jolt-eval/src/objective/performance/prover_time.rs b/jolt-eval/src/objective/performance/prover_time.rs new file mode 100644 index 000000000..d5c7f90dd --- /dev/null +++ b/jolt-eval/src/objective/performance/prover_time.rs @@ -0,0 +1,92 @@ +use jolt_core::host::Program; + +use crate::guests::{self, GuestConfig, GuestProgram, ProverPreprocessing}; +use crate::objective::Objective; + +/// Per-iteration state: everything needed to call `prove`. +pub struct ProverTimeSetup { + pub program: GuestProgram, + pub prover_pp: ProverPreprocessing, + pub input: Vec, +} + +/// Benchmarks end-to-end prover time for a guest program. +/// +/// Setup compiles the guest, traces to determine trace length, +/// and preprocesses. Each iteration calls `prove`. +#[derive(Default)] +pub struct ProverTimeObjective { + guest: G, + name: String, +} + +impl ProverTimeObjective { + pub fn new(guest: G) -> Self { + let name = format!("{} prover time", guest.bench_name()); + Self { guest, name } + } +} + +impl Objective for ProverTimeObjective { + type Setup = ProverTimeSetup; + + fn name(&self) -> &str { + self.name.as_str() + } + + fn setup(&self) -> ProverTimeSetup { + let mut mc = self.guest.memory_config(); + let input = self.guest.input(); + + // Compile + let target_dir = "/tmp/jolt-eval-bench-targets"; + let mut host_program = Program::new(self.guest.package()); + host_program.set_memory_config(mc); + host_program.build(target_dir); + let elf_bytes = host_program + .get_elf_contents() + .expect("guest ELF not found after build"); + + // Decode to get program_size, trace to get trace length + let (_bytecode, _memory_init, program_size, _e_entry) = + jolt_core::guest::program::decode(&elf_bytes); + mc.program_size = Some(program_size); + + let program = GuestProgram::new(&elf_bytes, &mc); + let (_lazy_trace, trace, _memory, _io) = program.trace(&input, &[], &[]); + let max_trace_length = (trace.len() + 1).next_power_of_two(); + drop(trace); + + let prover_pp = guests::prover_preprocessing(&program, max_trace_length); + + ProverTimeSetup { + program, + prover_pp, + input, + } + } + + fn run(&self, setup: ProverTimeSetup) { + let (_proof, _io) = guests::prove(&setup.program, &setup.prover_pp, &setup.input); + std::hint::black_box(()); + } + + fn units(&self) -> Option<&str> { + Some("s") + } +} + +#[cfg(test)] +mod tests { + use crate::guests::Fibonacci; + + use super::*; + + #[test] + fn fibonacci_config() { + let g = Fibonacci(100); + assert_eq!(g.package(), "fibonacci-guest"); + assert!(!g.input().is_empty()); + assert_eq!(g.bench_name(), "prover_time_fibonacci_100"); + } +} diff --git a/jolt-eval/src/objective/synthesis.rs b/jolt-eval/src/objective/synthesis.rs new file mode 100644 index 000000000..00d2e035b --- /dev/null +++ b/jolt-eval/src/objective/synthesis.rs @@ -0,0 +1,62 @@ +/// Macro that generates a Criterion benchmark harness for an [`Objective`]. +/// +/// Uses `iter_batched` with `BatchSize::LargeInput` so that per-iteration +/// setup (e.g. polynomial clone) is excluded from the measurement. +/// +/// # Usage +/// +/// ```ignore +/// // Fast benchmark (default Criterion settings, type must impl Default): +/// jolt_eval::bench_objective!(BindLowToHighObjective); +/// +/// // Slow benchmark with custom Criterion config: +/// jolt_eval::bench_objective!( +/// ProverTimeObjective::new(Fibonacci(100)), +/// config: sample_size(10), sampling_mode(Flat), measurement_time(30s) +/// ); +/// ``` +#[macro_export] +macro_rules! bench_objective { + // Expression form with config methods + ($obj_expr:expr, config: $($method:ident($($arg:expr),*)),* $(,)?) => { + use $crate::Objective as _; + + fn __bench(c: &mut ::criterion::Criterion) { + let obj = $obj_expr; + let mut group = c.benchmark_group(obj.name()); + $( + group.$method($($arg),*); + )* + group.bench_function("prove", |b| { + b.iter_batched( + || obj.setup(), + |setup| obj.run(setup), + ::criterion::BatchSize::LargeInput, + ); + }); + group.finish(); + } + + ::criterion::criterion_group!(benches, __bench); + ::criterion::criterion_main!(benches); + }; + + // Simple form: just a type (uses Default + default Criterion config) + ($obj_ty:ty) => { + use $crate::Objective as _; + + fn __bench(c: &mut ::criterion::Criterion) { + let obj = <$obj_ty>::default(); + c.bench_function(obj.name(), |b| { + b.iter_batched( + || obj.setup(), + |setup| obj.run(setup), + ::criterion::BatchSize::LargeInput, + ); + }); + } + + ::criterion::criterion_group!(benches, __bench); + ::criterion::criterion_main!(benches); + }; +} diff --git a/jolt-eval/src/sort_e2e.rs b/jolt-eval/src/sort_e2e.rs new file mode 100644 index 000000000..ddacc98fd --- /dev/null +++ b/jolt-eval/src/sort_e2e.rs @@ -0,0 +1,226 @@ +//! End-to-end test harnesses for the optimization and red-team loops, +//! using simple sorting functions as the target domain. + +use crate::agent::ClaudeCodeAgent; +use crate::invariant::synthesis::redteam::{auto_redteam, RedTeamConfig, RedTeamResult}; +use crate::invariant::{CheckError, Invariant, InvariantViolation}; +use crate::sort_targets::{candidate_sort, naive_sort}; + +/// Invariant: a sort function must preserve all elements (multiset +/// equality) and produce sorted output. +#[jolt_eval_macros::invariant(RedTeam)] +pub struct CandidateSortInvariant; + +impl Invariant for CandidateSortInvariant { + type Setup = (); + type Input = Vec; + + fn name(&self) -> &str { + "candidate_sort" + } + + fn description(&self) -> String { + "The sort function `candidate_sort` in \ + jolt-eval/src/sort_targets.rs must return a \ + permutation of its input in non-decreasing order. \ + Any dropped, duplicated, or misplaced elements are a violation." + .to_string() + } + + fn setup(&self) {} + + fn check(&self, _: &(), input: Vec) -> Result<(), CheckError> { + if input.len() > 1_000 { + return Err(CheckError::InvalidInput( + "input too large (max 1000)".into(), + )); + } + + let mut output = input.clone(); + candidate_sort(&mut output); + + let mut expected = input; + expected.sort(); + + if output != expected { + return Err(CheckError::Violation(InvariantViolation::new(format!( + "sort incorrect: expected {expected:?}, got {output:?}" + )))); + } + + Ok(()) + } + + fn seed_corpus(&self) -> Vec> { + vec![ + vec![], + vec![1], + vec![3, 1, 2], + vec![5, 4, 3, 2, 1], + vec![1, 1, 1], + ] + } +} + +/// Invariant for the naive (correct) sort — used in the optimization +/// loop to verify that the "optimized" sort is still correct. +pub struct NaiveSortInvariant; + +impl Invariant for NaiveSortInvariant { + type Setup = (); + type Input = Vec; + + fn name(&self) -> &str { + "naive_sort_correctness" + } + + fn description(&self) -> String { + "The naive sort must return a permutation of its input in \ + non-decreasing order." + .to_string() + } + + fn setup(&self) {} + + fn check(&self, _: &(), input: Vec) -> Result<(), CheckError> { + let mut output = input.clone(); + naive_sort(&mut output); + + let mut expected = input; + expected.sort(); + + if output != expected { + return Err(CheckError::Violation(InvariantViolation::new(format!( + "naive sort incorrect: expected {expected:?}, got {output:?}" + )))); + } + + Ok(()) + } + + fn seed_corpus(&self) -> Vec> { + vec![vec![], vec![1], vec![3, 1, 2], vec![5, 4, 3, 2, 1]] + } +} + +/// Run the red-team e2e test against `CandidateSortInvariant`. +pub fn run_redteam_test( + model: &str, + max_turns: usize, + iterations: usize, + hint: Option, + verbose: bool, +) { + let invariant = CandidateSortInvariant; + let agent = ClaudeCodeAgent::new(model, max_turns); + let repo_dir = std::env::current_dir().expect("current dir"); + let config = RedTeamConfig { + num_iterations: iterations, + hint, + verbose, + }; + + println!("=== Red-team e2e: candidate_sort ==="); + println!("model={model}, max_turns={max_turns}, iterations={iterations}"); + println!(); + + let result = auto_redteam(&invariant, &config, &agent, &repo_dir); + + match &result { + RedTeamResult::Violation { + approach, + input_json, + error, + } => { + println!("VIOLATION FOUND"); + println!(" Approach: {approach}"); + println!(" Input: {input_json}"); + println!(" Error: {error}"); + } + RedTeamResult::NoViolation { attempts } => { + println!("No violation found after {} attempts.", attempts.len()); + for a in attempts { + println!( + " {}: {} -- {}", + a.description, a.approach, a.failure_reason + ); + } + } + } +} + +#[cfg(test)] +mod tests { + use std::path::Path; + + use crate::agent::MockAgent; + use crate::invariant::synthesis::redteam::{auto_redteam, RedTeamConfig, RedTeamResult}; + + use super::*; + + #[test] + fn redteam_e2e_finds_sort_violation() { + let invariant = CandidateSortInvariant; + + // 17 elements (exceeds the small-array threshold), with the + // minimum value at the end — triggers the bug. + let bad_input: Vec = (1..=17).rev().collect(); + let response = serde_json::json!({ + "analysis": "Trying a reversed sequence of 17 elements.", + "counterexample": bad_input, + }); + let agent = MockAgent::always_ok(&response.to_string()); + let config = RedTeamConfig { + num_iterations: 5, + ..Default::default() + }; + + let result = auto_redteam(&invariant, &config, &agent, Path::new("/tmp")); + + match result { + RedTeamResult::Violation { error, .. } => { + assert!( + error.contains("sort incorrect"), + "unexpected error: {error}" + ); + } + RedTeamResult::NoViolation { .. } => { + panic!("Expected violation for large reversed input"); + } + } + + assert_eq!(agent.recorded_prompts().len(), 1); + } + + #[test] + fn redteam_e2e_no_violation_for_small_input() { + let invariant = CandidateSortInvariant; + + let response = serde_json::json!({ + "analysis": "Trying a small permutation.", + "counterexample": [5, 3, 1, 4, 2], + }); + let agent = MockAgent::always_ok(&response.to_string()); + let config = RedTeamConfig { + num_iterations: 3, + ..Default::default() + }; + + let result = auto_redteam(&invariant, &config, &agent, Path::new("/tmp")); + + match result { + RedTeamResult::NoViolation { attempts } => { + assert_eq!(attempts.len(), 3); + } + RedTeamResult::Violation { .. } => { + panic!("Small inputs should not trigger a violation"); + } + } + } + + #[test] + #[ignore] // Requires Claude API access + fn redteam_e2e_real_agent() { + run_redteam_test("claude-sonnet-4-20250514", 10, 5, None, false); + } +} diff --git a/jolt-eval/src/sort_targets.rs b/jolt-eval/src/sort_targets.rs new file mode 100644 index 000000000..153e7421a --- /dev/null +++ b/jolt-eval/src/sort_targets.rs @@ -0,0 +1,34 @@ +//! Sorting functions used as targets for e2e optimization and red-team tests. + +/// Naive bubble sort — the optimization target. +/// Intentionally O(n²) so a "smarter" sort is measurably faster. +pub fn naive_sort(data: &mut [i32]) { + let n = data.len(); + for i in 0..n { + for j in 0..n.saturating_sub(1 + i) { + if data[j] > data[j + 1] { + data.swap(j, j + 1); + } + } + } +} + +/// A sorting routine used as a red-team target. +pub fn candidate_sort(data: &mut [i32]) { + if data.len() <= 16 { + // Small-array path: insertion sort. + for i in 1..data.len() { + let key = data[i]; + let mut j = i; + while j > 0 && data[j - 1] > key { + data[j] = data[j - 1]; + j -= 1; + } + data[j] = key; + } + } else { + // Large-array path: delegate to an optimized routine. + let last = data.len() - 1; + data[..last].sort(); + } +} diff --git a/jolt-eval/sync_targets.sh b/jolt-eval/sync_targets.sh new file mode 100755 index 000000000..fc2d1ba42 --- /dev/null +++ b/jolt-eval/sync_targets.sh @@ -0,0 +1,139 @@ +#!/usr/bin/env bash +# +# Synchronizes fuzz targets and Criterion benchmarks with the invariant +# and objective definitions in jolt-eval source code. +# +# Run from the repo root: +# ./jolt-eval/sync_targets.sh +# +# Idempotent: running twice produces no changes. + +set -euo pipefail + +EVAL_DIR="$(cd "$(dirname "$0")" && pwd)" +FUZZ_DIR="$EVAL_DIR/fuzz" +BENCH_DIR="$EVAL_DIR/benches" + +# ── Helpers ────────────────────────────────────────────────────────── + +# Convert CamelCase to snake_case, stripping Invariant/Objective suffix +to_snake() { + echo "$1" \ + | sed 's/Invariant$//' \ + | sed 's/Objective$//' \ + | sed 's/\([A-Z]\)/_\1/g' \ + | sed 's/^_//' \ + | tr '[:upper:]' '[:lower:]' +} + +# ── Fuzz targets ───────────────────────────────────────────────────── + +echo "=== Syncing fuzz targets ===" + +mkdir -p "$FUZZ_DIR/fuzz_targets" + +# Find (snake_name, module_path, struct_name) for each fuzzable invariant +fuzz_entries="" +for file in "$EVAL_DIR"/src/invariant/*.rs; do + [ -f "$file" ] || continue + basename_rs=$(basename "$file" .rs) + [ "$basename_rs" = "mod" ] && continue + + # Look for #[invariant(...Fuzz...)] annotations + { grep -n 'invariant.*Fuzz' "$file" 2>/dev/null || true; } | while IFS=: read -r line _; do + struct=$(sed -n "$((line+1)),$((line+5))p" "$file" \ + | grep -o 'pub struct [A-Za-z_]*' | head -1 | awk '{print $3}') + [ -z "$struct" ] && continue + snake=$(to_snake "$struct") + echo "$snake invariant::${basename_rs}::${struct}" + done +done | sort -u > /tmp/jolt_fuzz_entries + +# Generate missing fuzz target files +while read -r snake mod_struct; do + [ -z "$snake" ] && continue + struct="${mod_struct##*::}" + target_file="$FUZZ_DIR/fuzz_targets/${snake}.rs" + if [ ! -f "$target_file" ]; then + echo " Creating fuzz target: $snake" + cat > "$target_file" </dev/null; then + echo " Removing stale fuzz target: $base" + rm "$f" + fi +done + +# Regenerate fuzz/Cargo.toml [[bin]] entries +{ + sed '/^\[\[bin\]\]/,$d' "$FUZZ_DIR/Cargo.toml" + while read -r snake _; do + [ -z "$snake" ] && continue + cat < "$FUZZ_DIR/Cargo.toml.tmp" +mv "$FUZZ_DIR/Cargo.toml.tmp" "$FUZZ_DIR/Cargo.toml" + +# ── Criterion benchmarks ───────────────────────────────────────────── +# +# Bench files are hand-authored (they carry domain-specific config). +# This script only syncs Cargo.toml [[bench]] entries from whatever +# .rs files exist in benches/. + +echo "=== Syncing Criterion bench entries ===" + +mkdir -p "$BENCH_DIR" + +# Collect bench names from existing .rs files (excluding symlinks) +for f in "$BENCH_DIR"/*.rs; do + [ -f "$f" ] || continue + basename "$f" .rs +done | sort -u > /tmp/jolt_bench_entries + +# Update Cargo.toml [[bench]] entries +CARGO_TOML="$EVAL_DIR/Cargo.toml" +tmpfile=$(mktemp) + +# Remove existing [[bench]] blocks +awk ' + /^\[\[bench\]\]/ { skip=1; next } + skip && /^$/ { skip=0; next } + skip && /^\[/ { skip=0 } + !skip { print } +' "$CARGO_TOML" > "$tmpfile" + +# Insert new [[bench]] entries before the first [[bin]] +{ + sed '/^\[\[bin\]\]/,$d' "$tmpfile" + while read -r name; do + [ -z "$name" ] && continue + cat < "$CARGO_TOML" +rm -f "$tmpfile" /tmp/jolt_fuzz_entries /tmp/jolt_bench_entries + +echo "=== Done ==="