diff --git a/.cargo/config.toml b/.cargo/config.toml new file mode 100644 index 00000000..5592118f --- /dev/null +++ b/.cargo/config.toml @@ -0,0 +1,2 @@ +[alias] +xtask = "run -p xtask --" diff --git a/Cargo.lock b/Cargo.lock index 61e8c869..368ae5dc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1902,6 +1902,7 @@ checksum = "765c9198f173dd59ce26ff9f95ef0aafd0a0fe01fb9d72841bc5066a4c06511d" dependencies = [ "base64 0.21.7", "byteorder", + "crossbeam-channel", "flate2", "nom 7.1.3", "num-traits", @@ -6409,16 +6410,20 @@ version = "0.1.0" dependencies = [ "anyhow", "bech32 0.11.1", + "chrono", "clap", "dolos", "dolos-cardano", "dolos-core", + "hdrhistogram", "hex", "pallas", "postgres", "postgres-native-tls", + "reqwest", "serde", "serde_json", + "tokio", "toml 0.8.20", "xshell", ] diff --git a/src/minibf_bench/vectors/preprod.json b/src/minibf_bench/vectors/preprod.json new file mode 100644 index 00000000..0af748cb --- /dev/null +++ b/src/minibf_bench/vectors/preprod.json @@ -0,0 +1,429 @@ +{ + "network": "preprod", + "generated_at": "2026-02-09T16:51:45.580710+00:00", + "dbsync_source": "cardano-preprod.dbsync-v3.demeter.run:5432", + "addresses": [ + { + "address": "addr_test1wqt7v9crzydwpwx7f4sjp7w7wsy7rr3ahkldels6tsc707q3xhdsk", + "address_type": "shelley_payment_only", + "tx_count": 315980 + }, + { + "address": "addr_test1wzn5ee2qaqvly3hx7e0nk3vhm240n5muq3plhjcnvx9ppjgf62u6a", + "address_type": "shelley_payment_only", + "tx_count": 272746 + }, + { + "address": "addr_test1wqlcn3pks3xdptxjw9pqrqtcx6ev694sstsruw3phd57ttg0lh0zq", + "address_type": "shelley_payment_only", + "tx_count": 260935 + }, + { + "address": "addr_test1wr64gtafm8rpkndue4ck2nx95u4flhwf643l2qmg9emjajg2ww0nj", + "address_type": "shelley_payment_only", + "tx_count": 243461 + }, + { + "address": "addr_test1qphe4ktmglgyhwqh42ltf8y2nxxgqvdv6c8tcgp7d637xqwsy7cw3eq0wqtup2fyh54q3e0r0ulvjhd6aewm2l6y7k9q7952rc", + "address_type": "shelley_payment_stake", + "tx_count": 176505 + }, + { + "address": "addr_test1wp8rka0tdzpfe20x8d2p7dj97790arg3nye20gjjs9c7wjgdefdv0", + "address_type": "shelley_payment_only", + "tx_count": 156143 + }, + { + "address": "addr_test1wzc86g4ym366hkaphryqqvaptwznqkmk2gdqz9930u534pcx58ahw", + "address_type": "shelley_payment_only", + "tx_count": 155785 + }, + { + "address": "addr_test1wqd2qdzhh3x280q6erdksyku6k3eknrqsmrpkjnn6hsvjegk9gghn", + "address_type": "shelley_payment_only", + "tx_count": 145287 + }, + { + "address": "addr_test1qqzk45kfse4ppwywm0n0n39j3r9y66lchznx9t4p5ume6uqcutrjdtuyrf7g4nr45636k5fnzkv4rz8kcqwect9dx78qqgwtpv", + "address_type": "shelley_payment_stake", + "tx_count": 140672 + }, + { + "address": "addr_test1vrghqljgzecagulwt2x4vx42cjslf6xfxl8xrew3rlqxz8crj5as6", + "address_type": "shelley_payment_only", + "tx_count": 114319 + }, + { + "address": "addr_test1vqe33tv0p36r5e6gx8tfunrpcrvqtmdxjkc0yzj07rk7h3gp7wqvu", + "address_type": "shelley_payment_only", + "tx_count": 113283 + }, + { + "address": "addr_test1qr273sxu9mg679ya3c406pwe2nl0scy6dwzcj7n7ask5jhelu5pu3xlq09au0h20q7ykx5nc027sltg0thyvxstfmytq2syuef", + "address_type": "shelley_payment_stake", + "tx_count": 112160 + }, + { + "address": "addr_test1vpwzy2m5v2svadxjqkj62z83xatq4hugvv9eklwv2wcc8hg3ecszk", + "address_type": "shelley_payment_only", + "tx_count": 111784 + }, + { + "address": "addr_test1qqtu7g3e7pqwt9f4dqlkme9qj80cejr5v0jwugthywvgjyyxn4dgcgtv9swlj6snz3cdx59ru4u7mhrvg5rl50wunlls5z3e57", + "address_type": "shelley_payment_stake", + "tx_count": 109572 + }, + { + "address": "addr_test1wq5yehcpw4e3r32rltrww40e6ezdckr9v9l0ehptsxeynlg630pts", + "address_type": "shelley_payment_only", + "tx_count": 107719 + }, + { + "address": "addr_test1vqz5nu55v72gqs6ev4rkxurqaaut7l9747akt2kpezgg9gcjw8kk6", + "address_type": "shelley_payment_only", + "tx_count": 105388 + }, + { + "address": "addr_test1qrkzc4slyjag0aj73sahznxjarqqwdchjzrp8h02097vzuxv9jcm6htzrttl7pdtfkhtvnxx4y4gz0rxcq7fkdde2vmsrlzzfk", + "address_type": "shelley_payment_stake", + "tx_count": 102027 + }, + { + "address": "addr_test1wpg6eael9tk4jzrnugcn000u9ej65wwm0c86uxcsugud89qdjvl6t", + "address_type": "shelley_payment_only", + "tx_count": 100069 + }, + { + "address": "addr_test1wppg9l6relcpls4u667twqyggkrpfrs5cdge9hhl9cv2upchtch0h", + "address_type": "shelley_payment_only", + "tx_count": 98875 + }, + { + "address": "addr_test1vzpwq95z3xyum8vqndgdd9mdnmafh3djcxnc6jemlgdmswcve6tkw", + "address_type": "shelley_payment_only", + "tx_count": 93703 + }, + { + "address": "addr_test1vr80076l3x5uw6n94nwhgmv7ssgy6muzf47ugn6z0l92rhg2mgtu0", + "address_type": "shelley_payment_only", + "tx_count": 90135 + }, + { + "address": "addr_test1vp5afqy4kzj3229luvsmf0626v6ury5lzlngzrw7fl3z3hgkc4qkl", + "address_type": "shelley_payment_only", + "tx_count": 88905 + }, + { + "address": "addr_test1wz4h6068hs93n8j5ar88fgzz6sfnw8krng09xx0mmf36m8c7j9yap", + "address_type": "shelley_payment_only", + "tx_count": 87810 + }, + { + "address": "addr_test1vp9dz4c70hmr6ntvfyjqeqmjad3e747qaanfxwxq6af09xc2pd49l", + "address_type": "shelley_payment_only", + "tx_count": 87246 + }, + { + "address": "addr_test1wr4se9nuh57rnwu350mzy7ltztnhekpptmpdkzwupaj49nqkldg8j", + "address_type": "shelley_payment_only", + "tx_count": 87112 + }, + { + "address": "addr_test1wzs7xqd6y04p6nyeqjzt8gpuktw4x82p4ve9fmg5vut22ksl6el0e", + "address_type": "shelley_payment_only", + "tx_count": 86303 + }, + { + "address": "addr_test1wpt4ekrdt386wusdgg82ylwm4gv5s9fk996tehj7ch94nesj2v0ku", + "address_type": "shelley_payment_only", + "tx_count": 85458 + }, + { + "address": "addr_test1wzk66pus8w3g08v8uzklz8jyu7u255cu329tj200xwfr59s4p9qtd", + "address_type": "shelley_payment_only", + "tx_count": 85182 + }, + { + "address": "addr_test1wraqlpezmu3h9n9mxey6y03u2sdd0e8cyx9n2qxscz6staczrlnuj", + "address_type": "shelley_payment_only", + "tx_count": 82716 + }, + { + "address": "addr_test1vz7st3e4f5tqzyldkwdr9gkwvpzlfr6364egl7ha4ck7emctt2gnq", + "address_type": "shelley_payment_only", + "tx_count": 81096 + }, + { + "address": "addr_test1vptykfqquzgpuvm98zl2966g7zav32nhkrp6jj8s0ng4luq3kcv6w", + "address_type": "shelley_payment_only", + "tx_count": 80886 + }, + { + "address": "addr_test1vpv3s36futt85l4zegc77j8up0hm8sl66k5g4aexg5c6upc3hygac", + "address_type": "shelley_payment_only", + "tx_count": 80719 + }, + { + "address": "addr_test1wp3e3h5t69ma37zv37ra9ux3nstt0f3clsqakwaz2l8hyxgvedxsr", + "address_type": "shelley_payment_only", + "tx_count": 80135 + }, + { + "address": "addr_test1qzmph7666hj5yadja34pdlm8n5un0fhc9p0hv2g9ask7m49n3kr657fpa3q8mzwmjqvl9lqdn9g2pm3ejhlgwpprwy2srzd2xd", + "address_type": "shelley_payment_stake", + "tx_count": 74756 + }, + { + "address": "addr_test1vrqrt84m05rg34usvj73rryeezu8kkznuwh4jfzmh9lgf5swrdhze", + "address_type": "shelley_payment_only", + "tx_count": 73274 + }, + { + "address": "addr_test1wz7uytdxstxe4nhdtl2gj9rcnlyce99tc707mz6qewxyx9qac0urr", + "address_type": "shelley_payment_only", + "tx_count": 72229 + }, + { + "address": "addr_test1vqcdlelfsk5l509lnlq2tfhrkj62rgvycwul3shjqq693usptapx4", + "address_type": "shelley_payment_only", + "tx_count": 68447 + }, + { + "address": "addr_test1vpymmxpazg6n5jxnntg4yy3zp67hrhfl39lt9x4cnu7ttrsh9ldm4", + "address_type": "shelley_payment_only", + "tx_count": 65740 + }, + { + "address": "addr_test1qqxfprwjvj2mlulhwvwz7kajgtg2p4svk60w07ppdrqt7r7a5ynn9ddhk88t352n3e9myjylu58h97h0sfjdugrjtrwq5r8ga5", + "address_type": "shelley_payment_stake", + "tx_count": 65425 + }, + { + "address": "addr_test1vz7et4vz3z9vmft6yqjkhvp7f39tk67lpxj867yxq4qjc5c378xa3", + "address_type": "shelley_payment_only", + "tx_count": 65004 + }, + { + "address": "addr_test1vz8fa5ucdfmnmvjzpvrqgdt6vh6an5ndp9p8nyv26szlmjswj2gcv", + "address_type": "shelley_payment_only", + "tx_count": 64624 + }, + { + "address": "addr_test1wqkgja788mjxwfrte22hwszqreqfl60p8apkvpz9cmdpkfshcu4rl", + "address_type": "shelley_payment_only", + "tx_count": 62383 + }, + { + "address": "addr_test1qrc2ym7pwzkc9dj284paahsgu782dcsz3dgszpvdqf36szjvpmpraw365fayhrtpzpl4nulq6f9hhdkh4cdyh0tgnjxsaxg4ak", + "address_type": "shelley_payment_stake", + "tx_count": 61079 + }, + { + "address": "addr_test1vquqz9efsnka08p3uluvmjs8zwjkqg2m58smqpywve7p96sfn22vf", + "address_type": "shelley_payment_only", + "tx_count": 59651 + }, + { + "address": "addr_test1wpmuql0ykw6fjzl2z39chf44y6xtj72ag27cfevey5lmuvqmrws5q", + "address_type": "shelley_payment_only", + "tx_count": 59287 + }, + { + "address": "addr_test1vrpndfxnauuezwmruc0eyqyd2xg42mknc0zuzx6swe9ureqrthyny", + "address_type": "shelley_payment_only", + "tx_count": 58950 + }, + { + "address": "addr_test1zpd7cjjwcqllzps9uaqtdjy7ytxzk6sg2tmvxzwm9am0pq8xq98rskrmxdgx4hmax4uuqrklkzg0n52fk5gug57g7cts8wt3jz", + "address_type": "shelley_payment_stake", + "tx_count": 58096 + }, + { + "address": "addr_test1vqnku6rsllyln4fa5s4tlv5ujx0y6kvu4mzzfh5jaht8nfq8584jf", + "address_type": "shelley_payment_only", + "tx_count": 54319 + }, + { + "address": "addr_test1vr7g6edlpquqfl02pjr2cz0rdpwdwvlr3hea3qxxe6wa6qggygcsd", + "address_type": "shelley_payment_only", + "tx_count": 52487 + }, + { + "address": "addr_test1vry3appc9j737z67gperyapaszy5u74wt0k8eq6nxjzpuusw3kd9e", + "address_type": "shelley_payment_only", + "tx_count": 50302 + } + ], + "stake_addresses": [ + { + "stake_address": "stake_test1uzycrl3729mp93et47x9rhsr6dfwxuyn7zs9g2uu7myzsgc06p9sg", + "delegation_count": 1922, + "reward_count": 16 + }, + { + "stake_address": "stake_test1urczp2hz9u2gsdu7lqh7lgvv8mmegqt3hzrt6zm25u63uyc77tleh", + "delegation_count": 1708, + "reward_count": 20 + }, + { + "stake_address": "stake_test1uzx2s8rk2a925v4rfq5r3um8avypq70vfsszfequ247ygwg6mzl4u", + "delegation_count": 1480, + "reward_count": 41 + }, + { + "stake_address": "stake_test1uq2lfrauuqz5f75xqp8sl3qahspa7l3ef65nzcu3fm75m7glkzhp3", + "delegation_count": 1323, + "reward_count": 4 + }, + { + "stake_address": "stake_test1ur6l9f5l9jw44kl2nf6nm5kca3nwqqkccwynnjm0h2cv60ccngdwa", + "delegation_count": 1259, + "reward_count": 55 + }, + { + "stake_address": "stake_test1upgxfdn3vdx3fjud2sl8rhvwksm6glhmg7cty2yzsekyyrgm428xq", + "delegation_count": 1019, + "reward_count": 2 + }, + { + "stake_address": "stake_test1ur7tpuvdkjfjs6wae9p26rvvzr4msmm4xlpffhlkfl4uvzseeul3m", + "delegation_count": 936, + "reward_count": 162 + }, + { + "stake_address": "stake_test17q8snfkq7uhtcaksmwd6mpn85f84su623aagk2peaswvs3shre8vr", + "delegation_count": 762, + "reward_count": 0 + }, + { + "stake_address": "stake_test1uqsykuy9pvcd3nm34yvf5tfuqyhf90zya7xpy83rh86l3rqxxmkyx", + "delegation_count": 679, + "reward_count": 0 + }, + { + "stake_address": "stake_test1uqp805rr47yly7g858q26y9wwehr3jjps9u4v5797pr732s0754el", + "delegation_count": 593, + "reward_count": 0 + }, + { + "stake_address": "stake_test1upr9pnn0vul7dmqtksksyjg2qc65ewjzej02pfwcshdj9ls5ldzfp", + "delegation_count": 591, + "reward_count": 0 + }, + { + "stake_address": "stake_test1uzk9r7mnwsghr8fd72dw8zcxqm804h3dqf9lwzu7yea2p3gl3wk3l", + "delegation_count": 591, + "reward_count": 0 + }, + { + "stake_address": "stake_test1urvp7670m5ec50fz5dekdu5e8pycx62ys2fkuu80m5v823q59qmel", + "delegation_count": 570, + "reward_count": 0 + }, + { + "stake_address": "stake_test1uz5c800m3kegt0zuhyyup703agjg72g5tp299p0y2d4q70qdtrr9j", + "delegation_count": 565, + "reward_count": 107 + }, + { + "stake_address": "stake_test1ur74mnnv32h23mhwa2f8g5s78xql8af6zg34ukmn6assywgulh4cc", + "delegation_count": 540, + "reward_count": 0 + }, + { + "stake_address": "stake_test1uznjv7kaj0rcg07vsxg708w9xv2lu2tfaax29gudsjrfktq9qk99f", + "delegation_count": 540, + "reward_count": 0 + }, + { + "stake_address": "stake_test1uzmumppm3p8vpau7se3646t37dqy3cy4uy3k4cm08jefr6qwx93cc", + "delegation_count": 540, + "reward_count": 0 + }, + { + "stake_address": "stake_test1ur9gwqky69yemg3qf2526qp20rltl9ez84cqlgl002t3qsgx7w3p2", + "delegation_count": 540, + "reward_count": 0 + }, + { + "stake_address": "stake_test1uzr5ad223jghze3ql7lfms25xlmk8xtd26ycn6twcns7sjcmzx0fu", + "delegation_count": 540, + "reward_count": 0 + }, + { + "stake_address": "stake_test1urtyeyl0qz20tsteu5uqzz0tamczyfzegn3ezn6mej360ycky7cg5", + "delegation_count": 534, + "reward_count": 0 + }, + { + "stake_address": "stake_test17rs4wd04fx4ls9rpdelrp9qwnh3w6cexlmgj42h5t0tvv8gjrqqjc", + "delegation_count": 462, + "reward_count": 0 + }, + { + "stake_address": "stake_test17qedgvzpjdv6shzz974fuaedrsa7rdenwzcce4k9wyetfqsqfm4kt", + "delegation_count": 388, + "reward_count": 0 + }, + { + "stake_address": "stake_test1ur7kvvy4ukkuyexuy7pdyhk48lyr2tufk3a6xdcyrdwd5kqmwc45y", + "delegation_count": 275, + "reward_count": 0 + }, + { + "stake_address": "stake_test1uqhtnq65j6gxxepjka4swdsvgdx2jcdqulws6m4fkr3uw7ckxp06v", + "delegation_count": 251, + "reward_count": 0 + }, + { + "stake_address": "stake_test1ur8u69tv9k3fnc7q8a3xqw6ugs5t4nq4dmjy02zw3w0zjgg7tewuu", + "delegation_count": 240, + "reward_count": 0 + }, + { + "stake_address": "stake_test17psajmusqzl46vja59e93msxj0se63quanhxfqjj38hxklglcp8y8", + "delegation_count": 218, + "reward_count": 0 + }, + { + "stake_address": "stake_test17rhka4r6dyt68ja7k3jkr6y98k5kjdphjntxz2ze3g627tqtlqdeh", + "delegation_count": 209, + "reward_count": 0 + }, + { + "stake_address": "stake_test1uz99t67x3ypduu8xdtcwh527p5s9dyhgjh99qjptvqsgtpg32nec5", + "delegation_count": 204, + "reward_count": 7 + }, + { + "stake_address": "stake_test1urkmj2vzdey7ac065rleyrc03fzp7gxxhw32pzgxv8dwuasaqtjuz", + "delegation_count": 189, + "reward_count": 0 + }, + { + "stake_address": "stake_test1urt4qclt9nafll5xkjmx6gam2f6fdrezjkz4xzjzxn34llqjlke05", + "delegation_count": 188, + "reward_count": 174 + } + ], + "reference_blocks": { + "early": { + "height": 440647, + "hash": "\\xdd811d450847ac086901e6736a85509c2026f2f66ae0c4335ff85c7f2df7d66e", + "epoch": 41, + "slot": 16281734 + }, + "mid": { + "height": 2203237, + "hash": "\\x2944dd290435227f96340addb4b616a75035a55fc8119e7752fe4c05642d7889", + "epoch": 140, + "slot": 59122449 + }, + "recent": { + "height": 3965826, + "hash": "\\xdfc06de3d85ad3c8c7f223b4304a67a50377d1de7eb29b136f3e128f07c89778", + "epoch": 243, + "slot": 103708196 + } + } +} \ No newline at end of file diff --git a/xtask/Cargo.toml b/xtask/Cargo.toml index 3bdfdaa3..7a19cf76 100644 --- a/xtask/Cargo.toml +++ b/xtask/Cargo.toml @@ -23,3 +23,7 @@ postgres = "0.19" postgres-native-tls = "0.5" bech32 = { workspace = true } pallas = { workspace = true } +hdrhistogram = "7.5" +chrono = { version = "0.4", features = ["serde"] } +reqwest = { version = "0.12", features = ["json"] } +tokio = { version = "1.0", features = ["rt-multi-thread", "macros"] } diff --git a/xtask/src/ground_truth/delegation.rs b/xtask/src/ground_truth/delegation.rs index e10df4e3..c784926c 100644 --- a/xtask/src/ground_truth/delegation.rs +++ b/xtask/src/ground_truth/delegation.rs @@ -46,8 +46,8 @@ pub(super) fn fetch(dbsync_url: &str, epoch: u64) -> Result Result<()> { - let mut file = File::create(path) - .with_context(|| format!("writing pools csv: {}", path.display()))?; + let mut file = + File::create(path).with_context(|| format!("writing pools csv: {}", path.display()))?; writeln!(file, "pool_bech32,pool_hash,total_lovelace")?; for row in rows { writeln!( diff --git a/xtask/src/ground_truth/epochs.rs b/xtask/src/ground_truth/epochs.rs index a1c0e9e5..f0286f82 100644 --- a/xtask/src/ground_truth/epochs.rs +++ b/xtask/src/ground_truth/epochs.rs @@ -59,9 +59,7 @@ pub(super) fn fetch(dbsync_url: &str, max_epoch: u64) -> Result> { let fees: Option = row.get(8); let block_count: Option = row.get(9); - let nonce = epoch_nonce - .map(|b| hex::encode(&b)) - .unwrap_or_default(); + let nonce = epoch_nonce.map(|b| hex::encode(&b)).unwrap_or_default(); epochs.push(EpochRow { epoch_no, diff --git a/xtask/src/ground_truth/eras.rs b/xtask/src/ground_truth/eras.rs index b3a1983a..af31b403 100644 --- a/xtask/src/ground_truth/eras.rs +++ b/xtask/src/ground_truth/eras.rs @@ -81,10 +81,7 @@ pub(super) fn fetch(dbsync_url: &str, max_epoch: u64, network: &Network) -> Resu pub(super) fn write_csv(path: &std::path::Path, eras: &[EraRow]) -> Result<()> { let mut file = std::fs::File::create(path) .with_context(|| format!("creating eras csv: {}", path.display()))?; - writeln!( - file, - "protocol,start_epoch,epoch_length,slot_length" - )?; + writeln!(file, "protocol,start_epoch,epoch_length,slot_length")?; for era in eras { writeln!( diff --git a/xtask/src/ground_truth/query.rs b/xtask/src/ground_truth/query.rs index a255dfd0..0122269b 100644 --- a/xtask/src/ground_truth/query.rs +++ b/xtask/src/ground_truth/query.rs @@ -46,7 +46,10 @@ pub fn run(args: &QueryArgs) -> Result<()> { let rows = super::delegation::fetch(dbsync_url, args.epoch)?; println!("pool_bech32,pool_hash,total_lovelace"); for row in rows { - println!("{},{},{}", row.pool_bech32, row.pool_hash, row.total_lovelace); + println!( + "{},{},{}", + row.pool_bech32, row.pool_hash, row.total_lovelace + ); } } QueryEntity::Accounts => { diff --git a/xtask/src/ground_truth/rewards.rs b/xtask/src/ground_truth/rewards.rs index 50c29ee1..09473e81 100644 --- a/xtask/src/ground_truth/rewards.rs +++ b/xtask/src/ground_truth/rewards.rs @@ -54,8 +54,8 @@ pub(super) fn fetch(dbsync_url: &str, epoch: u64) -> Result> { } pub(super) fn write_csv(path: &Path, rows: &[RewardRow]) -> Result<()> { - let mut file = File::create(path) - .with_context(|| format!("writing rewards csv: {}", path.display()))?; + let mut file = + File::create(path).with_context(|| format!("writing rewards csv: {}", path.display()))?; writeln!(file, "stake,pool,amount,type,earned_epoch")?; for row in rows { writeln!( diff --git a/xtask/src/ground_truth/stake.rs b/xtask/src/ground_truth/stake.rs index d8e9b028..799ebc5b 100644 --- a/xtask/src/ground_truth/stake.rs +++ b/xtask/src/ground_truth/stake.rs @@ -46,8 +46,8 @@ pub(super) fn fetch(dbsync_url: &str, epoch: u64) -> Result } pub(super) fn write_csv(path: &Path, rows: &[AccountStakeRow]) -> Result<()> { - let mut file = File::create(path) - .with_context(|| format!("writing accounts csv: {}", path.display()))?; + let mut file = + File::create(path).with_context(|| format!("writing accounts csv: {}", path.display()))?; writeln!(file, "stake,pool,lovelace")?; for row in rows { writeln!(file, "{},{},{}", row.stake, row.pool, row.lovelace)?; diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 4345d07b..782b4807 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -5,6 +5,7 @@ use xshell::{cmd, Shell}; mod bootstrap; mod config; mod ground_truth; +mod minibf_bench; mod test_instance; mod util; @@ -31,6 +32,9 @@ enum Commands { /// Test instance management commands (create, delete) #[command(subcommand)] TestInstance(test_instance::TestInstanceCmd), + + /// Benchmark minibf endpoints + MinibfBench(minibf_bench::args::BenchArgs), } fn main() -> Result<()> { @@ -45,6 +49,7 @@ fn main() -> Result<()> { Commands::BootstrapMithrilLocal(args) => bootstrap::run(&sh, &args)?, Commands::GroundTruth(cmd) => ground_truth::run(cmd)?, Commands::TestInstance(cmd) => test_instance::run(&sh, cmd)?, + Commands::MinibfBench(args) => minibf_bench::run(&sh, &args)?, } Ok(()) diff --git a/xtask/src/minibf_bench.rs b/xtask/src/minibf_bench.rs new file mode 100644 index 00000000..0dd5e8d3 --- /dev/null +++ b/xtask/src/minibf_bench.rs @@ -0,0 +1,217 @@ +//! Minibf endpoint benchmarking tool. +//! +//! Measures p50/p95/p99 latency for Dolos minibf endpoints with +//! pagination boundary detection and chain history spread. + +pub mod args; + +mod report; +mod runner; +mod sampler; +mod stats; +mod vectors; + +use std::time::Instant; + +use anyhow::{Context, Result}; +use reqwest::Method; +use xshell::Shell; + +use crate::config::{load_xtask_config, Network}; +use args::BenchArgs; +use runner::{BenchmarkRunner, TestRequest}; +use vectors::{load_vectors, TestVectors}; + +pub fn run(_sh: &Shell, args: &BenchArgs) -> Result<()> { + let start_time = Instant::now(); + let repo_root = std::env::current_dir().context("detecting repo root")?; + + // Load config + let config = load_xtask_config(&repo_root)?; + let dbsync_url = config + .dbsync + .url_for_network(&args.network) + .ok_or_else(|| { + anyhow::anyhow!( + "No DBSync URL configured for {} in xtask.toml", + args.network.as_str() + ) + })?; + + // Load or generate test vectors + let vectors = load_vectors(&args.network, dbsync_url, args.generate_vectors)?; + println!("Loaded test vectors for {}", vectors.network); + + // Build chain samples from reference blocks in test vectors + println!("Sampling chain history from reference blocks..."); + let rt = tokio::runtime::Runtime::new()?; + let samples = rt.block_on(async { + let client = reqwest::Client::new(); + sampler::HistoricalSamples::from_vectors(&client, &args.url, &vectors.reference_blocks) + .await + })?; + + // Generate test requests + let requests = generate_requests(args, &vectors, &samples); + println!( + "Generated {} test requests across {} endpoints", + requests.len(), + requests + .iter() + .map(|r| &r.endpoint_name) + .collect::>() + .len() + ); + + // Run benchmark + let runner = BenchmarkRunner::new(args.clone())?; + + // Warmup + rt.block_on(runner.run_warmup(requests.clone()))?; + + // Benchmark + let (results, _histogram) = rt.block_on(runner.run_benchmark(requests))?; + + // Calculate stats + let endpoint_stats = stats::calculate_endpoint_stats(&results)?; + + // Generate report + let duration = start_time.elapsed(); + let report = report::generate_report(args, duration, endpoint_stats, &vectors)?; + + // Output + report::write_report(&report, args.output_file.as_deref())?; + + Ok(()) +} + +fn generate_requests( + args: &BenchArgs, + vectors: &TestVectors, + samples: &crate::minibf_bench::sampler::HistoricalSamples, +) -> Vec { + let mut requests = Vec::new(); + + // Accounts endpoints (priority) + if let Some(stake) = vectors.stake_addresses.first() { + // /accounts/{stake} + requests.push(TestRequest { + endpoint_name: "accounts_by_stake".to_string(), + path: format!("/accounts/{}", stake.stake_address), + method: Method::GET, + }); + + if !args.skip_paginated { + // /accounts/{stake}/addresses with pagination + for page in [1u64, 5, args.max_page] { + requests.push(TestRequest { + endpoint_name: "accounts_addresses_paginated".to_string(), + path: format!( + "/accounts/{}/addresses?page={}&count=20", + stake.stake_address, page + ), + method: Method::GET, + }); + } + + // /accounts/{stake}/utxos with pagination + for page in [1u64, 5] { + requests.push(TestRequest { + endpoint_name: "accounts_utxos_paginated".to_string(), + path: format!( + "/accounts/{}/utxos?page={}&count=20", + stake.stake_address, page + ), + method: Method::GET, + }); + } + + // /accounts/{stake}/delegations + requests.push(TestRequest { + endpoint_name: "accounts_delegations".to_string(), + path: format!( + "/accounts/{}/delegations?page=1&count=20", + stake.stake_address + ), + method: Method::GET, + }); + } + } + + // Address endpoints + for addr in vectors.addresses.iter().take(3) { + // /addresses/{address}/utxos + requests.push(TestRequest { + endpoint_name: "addresses_utxos".to_string(), + path: format!("/addresses/{}/utxos", addr.address), + method: Method::GET, + }); + + if !args.skip_paginated { + // /addresses/{address}/transactions with pagination (scan limit testing) + for page in [1u64, 5, 10, args.max_page] { + requests.push(TestRequest { + endpoint_name: "addresses_transactions_paginated".to_string(), + path: format!( + "/addresses/{}/transactions?page={}&count=100", + addr.address, page + ), + method: Method::GET, + }); + } + } + } + + // Block endpoints with history spread + for (label, sample) in [ + ("early", &samples.early), + ("mid", &samples.mid), + ("recent", &samples.recent), + ] { + // /blocks/{hash} + requests.push(TestRequest { + endpoint_name: format!("blocks_by_hash_{}", label), + path: format!("/blocks/{}", sample.block_hash), + method: Method::GET, + }); + + // /blocks/{hash}/txs + requests.push(TestRequest { + endpoint_name: format!("blocks_txs_{}", label), + path: format!("/blocks/{}/txs", sample.block_hash), + method: Method::GET, + }); + + // Test some transactions from this block + for (i, tx) in sample.txs.iter().take(3).enumerate() { + requests.push(TestRequest { + endpoint_name: format!("txs_by_hash_{}_{}", label, i), + path: format!("/txs/{}", tx), + method: Method::GET, + }); + } + } + + // Latest block + requests.push(TestRequest { + endpoint_name: "blocks_latest".to_string(), + path: "/blocks/latest".to_string(), + method: Method::GET, + }); + + // Epoch endpoints + requests.push(TestRequest { + endpoint_name: "epochs_latest_parameters".to_string(), + path: "/epochs/latest/parameters".to_string(), + method: Method::GET, + }); + + // Network endpoint + requests.push(TestRequest { + endpoint_name: "network".to_string(), + path: "/network".to_string(), + method: Method::GET, + }); + + requests +} diff --git a/xtask/src/minibf_bench/args.rs b/xtask/src/minibf_bench/args.rs new file mode 100644 index 00000000..ad255b49 --- /dev/null +++ b/xtask/src/minibf_bench/args.rs @@ -0,0 +1,59 @@ +//! CLI arguments for minibf-bench command. + +use clap::{Args, ValueEnum}; +use std::path::PathBuf; + +use crate::config::Network; + +#[derive(Debug, Clone, Args)] +pub struct BenchArgs { + /// Target minibf URL + #[arg(long, default_value = "http://localhost:3000")] + pub url: String, + + /// Target network (determines test vectors) + #[arg(long, value_enum, default_value = "preprod")] + pub network: Network, + + /// Generate fresh test vectors from DBSync + #[arg(long, action)] + pub generate_vectors: bool, + + /// Total number of requests + #[arg(long, default_value = "10000")] + pub requests: usize, + + /// Concurrent connections + #[arg(long, default_value = "10")] + pub concurrency: usize, + + /// Warmup requests (not counted in stats) + #[arg(long, default_value = "1000")] + pub warmup: usize, + + /// Output format + #[arg(long, value_enum, default_value = "json")] + pub output: OutputFormat, + + /// Output file (default: stdout) + #[arg(long)] + pub output_file: Option, + + /// Maximum page to test for pagination endpoints + #[arg(long, default_value = "11")] + pub max_page: u64, + + /// Page sizes to test (comma-separated) + #[arg(long, default_value = "20,50,100")] + pub page_sizes: String, + + /// Skip paginated endpoints (focus on simple lookups) + #[arg(long, action)] + pub skip_paginated: bool, +} + +#[derive(Debug, Clone, ValueEnum)] +pub enum OutputFormat { + Json, + Table, +} diff --git a/xtask/src/minibf_bench/report.rs b/xtask/src/minibf_bench/report.rs new file mode 100644 index 00000000..1220a38e --- /dev/null +++ b/xtask/src/minibf_bench/report.rs @@ -0,0 +1,213 @@ +//! JSON report generation for CI ingestion. + +use anyhow::Result; +use serde::{Deserialize, Serialize}; + +use crate::minibf_bench::{args::BenchArgs, stats::EndpointStats, vectors::TestVectors}; + +#[derive(Debug, Serialize, Deserialize)] +pub struct BenchmarkReport { + pub metadata: ReportMetadata, + pub summary: ReportSummary, + pub endpoints: Vec, + pub pagination_analysis: PaginationAnalysis, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct ReportMetadata { + pub timestamp: String, + pub url: String, + pub network: String, + pub total_requests: usize, + pub concurrency: usize, + pub warmup_requests: usize, + pub duration_secs: f64, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct ReportSummary { + pub total_requests: usize, + pub successful_requests: usize, + pub failed_requests: usize, + pub requests_per_second: f64, + pub overall_p50_ms: f64, + pub overall_p95_ms: f64, + pub overall_p99_ms: f64, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct EndpointReport { + pub name: String, + pub path: String, + pub requests: u64, + pub success_rate: f64, + pub latency_ms: LatencyStats, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct LatencyStats { + pub p50: f64, + pub p95: f64, + pub p99: f64, + pub min: f64, + pub max: f64, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct PaginationAnalysis { + pub scan_limit: u64, + pub page_size_tested: u64, + pub tests: Vec, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct PaginationTestResult { + pub page: u64, + pub count: u64, + pub p50_ms: Option, + pub success_rate: f64, + pub error: Option, +} + +pub fn generate_report( + args: &BenchArgs, + duration: std::time::Duration, + endpoint_stats: Vec, + vectors: &TestVectors, +) -> Result { + let total_requests: usize = endpoint_stats.iter().map(|s| s.requests as usize).sum(); + let successful_requests: usize = endpoint_stats + .iter() + .map(|s| s.success_count as usize) + .sum(); + let failed_requests = total_requests - successful_requests; + let rps = total_requests as f64 / duration.as_secs_f64(); + + // Calculate overall latencies from all endpoints + let all_p50: Vec = endpoint_stats + .iter() + .map(|s| s.latency_p50_micros) + .collect(); + let all_p95: Vec = endpoint_stats + .iter() + .map(|s| s.latency_p95_micros) + .collect(); + let all_p99: Vec = endpoint_stats + .iter() + .map(|s| s.latency_p99_micros) + .collect(); + + let overall_p50 = all_p50.iter().sum::() as f64 / all_p50.len().max(1) as f64 / 1000.0; + let overall_p95 = all_p95.iter().sum::() as f64 / all_p95.len().max(1) as f64 / 1000.0; + let overall_p99 = all_p99.iter().sum::() as f64 / all_p99.len().max(1) as f64 / 1000.0; + + let endpoints: Vec = endpoint_stats + .into_iter() + .map(|stat| { + let success_rate = stat.success_rate(); + EndpointReport { + name: stat.name, + path: stat.path, + requests: stat.requests, + success_rate, + latency_ms: LatencyStats { + p50: stat.latency_p50_micros as f64 / 1000.0, + p95: stat.latency_p95_micros as f64 / 1000.0, + p99: stat.latency_p99_micros as f64 / 1000.0, + min: stat.latency_min_micros as f64 / 1000.0, + max: stat.latency_max_micros as f64 / 1000.0, + }, + } + }) + .collect(); + + // Generate pagination analysis + let pagination_analysis = generate_pagination_analysis(&endpoints); + + Ok(BenchmarkReport { + metadata: ReportMetadata { + timestamp: chrono::Utc::now().to_rfc3339(), + url: args.url.clone(), + network: vectors.network.clone(), + total_requests: args.requests, + concurrency: args.concurrency, + warmup_requests: args.warmup, + duration_secs: duration.as_secs_f64(), + }, + summary: ReportSummary { + total_requests, + successful_requests, + failed_requests, + requests_per_second: rps, + overall_p50_ms: overall_p50, + overall_p95_ms: overall_p95, + overall_p99_ms: overall_p99, + }, + endpoints, + pagination_analysis, + }) +} + +fn generate_pagination_analysis(endpoints: &[EndpointReport]) -> PaginationAnalysis { + // Find pagination endpoints and analyze their performance + let mut tests = Vec::new(); + + // Check if we have scan limit boundary tests + for endpoint in endpoints { + if endpoint.path.contains("page=") { + // Extract page number from path + let page = extract_page(&endpoint.path); + let count = extract_count(&endpoint.path); + + tests.push(PaginationTestResult { + page, + count, + p50_ms: Some(endpoint.latency_ms.p50), + success_rate: endpoint.success_rate, + error: if endpoint.success_rate < 1.0 { + Some("scan_limit_exceeded".to_string()) + } else { + None + }, + }); + } + } + + PaginationAnalysis { + scan_limit: 1000, + page_size_tested: 100, + tests, + } +} + +fn extract_page(path: &str) -> u64 { + path.split("page=") + .nth(1) + .and_then(|s| s.split('&').next()) + .and_then(|s| s.parse().ok()) + .unwrap_or(1) +} + +fn extract_count(path: &str) -> u64 { + path.split("count=") + .nth(1) + .and_then(|s| s.split('&').next()) + .and_then(|s| s.parse().ok()) + .unwrap_or(100) +} + +pub fn write_report(report: &BenchmarkReport, output_path: Option<&std::path::Path>) -> Result<()> { + let json = serde_json::to_string_pretty(report)?; + + match output_path { + Some(path) => { + std::fs::write(path, json)?; + println!("Report written to {}", path.display()); + } + None => { + println!("{}", json); + } + } + + Ok(()) +} diff --git a/xtask/src/minibf_bench/runner.rs b/xtask/src/minibf_bench/runner.rs new file mode 100644 index 00000000..da43f2e8 --- /dev/null +++ b/xtask/src/minibf_bench/runner.rs @@ -0,0 +1,145 @@ +//! HTTP benchmarking with concurrent execution. + +use anyhow::Result; +use hdrhistogram::Histogram; +use reqwest::{Client, Method, StatusCode}; +use std::sync::Arc; +use std::time::{Duration, Instant}; +use tokio::sync::{mpsc, Semaphore}; +use tokio::task::JoinHandle; + +use crate::minibf_bench::args::BenchArgs; + +#[derive(Debug, Clone)] +pub struct TestRequest { + pub endpoint_name: String, + pub path: String, + pub method: Method, +} + +#[derive(Debug, Clone)] +pub struct RequestResult { + pub endpoint_name: String, + pub path: String, + pub latency_micros: u64, + pub status: StatusCode, + pub success: bool, + pub error: Option, +} + +pub struct BenchmarkRunner { + client: Client, + args: BenchArgs, +} + +impl BenchmarkRunner { + pub fn new(args: BenchArgs) -> Result { + let client = Client::builder() + .timeout(Duration::from_secs(30)) + .pool_max_idle_per_host(args.concurrency) + .build()?; + + Ok(Self { client, args }) + } + + pub async fn run_warmup(&self, requests: Vec) -> Result<()> { + println!("Running {} warmup requests...", self.args.warmup); + + let semaphore = Arc::new(Semaphore::new(self.args.concurrency)); + + for (i, req) in requests.iter().cycle().take(self.args.warmup).enumerate() { + let permit = semaphore.clone().acquire_owned().await?; + let client = self.client.clone(); + let request = req.clone(); + let base_url = self.args.url.clone(); + + tokio::spawn(async move { + let url = format!("{}{}", base_url, request.path); + let _ = client.request(request.method, url).send().await; + drop(permit); + }); + + if (i + 1) % 100 == 0 { + println!(" Warmup {}/{}", i + 1, self.args.warmup); + } + } + + Ok(()) + } + + pub async fn run_benchmark( + &self, + requests: Vec, + ) -> Result<(Vec, Histogram)> { + println!("Running {} benchmark requests...", self.args.requests); + + let semaphore = Arc::new(Semaphore::new(self.args.concurrency)); + let (tx, mut rx) = mpsc::channel::(1000); + + // Start result collector before spawning workers to avoid deadlock + let collector = tokio::spawn(async move { + let mut results = Vec::new(); + let mut histogram = Histogram::::new_with_bounds(1, 60_000_000, 3).unwrap(); + + while let Some(result) = rx.recv().await { + let _ = histogram.record(result.latency_micros); + results.push(result); + } + + (results, histogram) + }); + + // Spawn workers + let total = self.args.requests; + for (i, req) in requests.iter().cycle().take(total).enumerate() { + let permit = semaphore.clone().acquire_owned().await?; + let client = self.client.clone(); + let request = req.clone(); + let base_url = self.args.url.clone(); + let tx = tx.clone(); + + tokio::spawn(async move { + let start = Instant::now(); + let url = format!("{}{}", base_url, request.path); + + let result = match client.request(request.method.clone(), &url).send().await { + Ok(resp) => { + let status = resp.status(); + let success = status.is_success(); + RequestResult { + endpoint_name: request.endpoint_name.clone(), + path: request.path, + latency_micros: start.elapsed().as_micros() as u64, + status, + success, + error: None, + } + } + Err(e) => RequestResult { + endpoint_name: request.endpoint_name.clone(), + path: request.path, + latency_micros: start.elapsed().as_micros() as u64, + status: StatusCode::REQUEST_TIMEOUT, + success: false, + error: Some(e.to_string()), + }, + }; + + let _ = tx.send(result).await; + drop(permit); + }); + + if (i + 1) % 1000 == 0 { + println!(" Queued {}/{} requests", i + 1, total); + } + } + + // Drop original sender so collector knows when done + drop(tx); + + // Wait for collector to finish + let (results, histogram) = collector.await?; + + Ok((results, histogram)) + } +} diff --git a/xtask/src/minibf_bench/sampler.rs b/xtask/src/minibf_bench/sampler.rs new file mode 100644 index 00000000..845a4d99 --- /dev/null +++ b/xtask/src/minibf_bench/sampler.rs @@ -0,0 +1,68 @@ +//! Dynamic chain sampling to test endpoints across chain history. + +use anyhow::Result; +use reqwest::Client; + +/// Historical samples from different chain positions +#[derive(Debug, Clone)] +pub struct HistoricalSamples { + pub early: ChainSample, + pub mid: ChainSample, + pub recent: ChainSample, +} + +#[derive(Debug, Clone)] +pub struct ChainSample { + pub block_hash: String, + pub block_number: u64, + pub slot: u64, + pub epoch: u64, + pub txs: Vec, +} + +impl HistoricalSamples { + /// Build samples from pre-computed reference blocks in test vectors, + /// fetching block details and txs via the API by block height. + pub async fn from_vectors( + client: &Client, + base_url: &str, + refs: &crate::minibf_bench::vectors::ReferenceBlocks, + ) -> Result { + Ok(Self { + early: fetch_sample(client, base_url, refs.early.height).await?, + mid: fetch_sample(client, base_url, refs.mid.height).await?, + recent: fetch_sample(client, base_url, refs.recent.height).await?, + }) + } +} + +async fn fetch_sample(client: &Client, base_url: &str, height: i64) -> Result { + let url = format!("{}/blocks/{}", base_url, height); + let resp: serde_json::Value = client.get(&url).send().await?.json().await?; + + let block_hash = resp["hash"].as_str().unwrap_or_default().to_string(); + let block_number = resp["height"].as_u64().unwrap_or(height as u64); + let slot = resp["slot"].as_u64().unwrap_or_default(); + let epoch = resp["epoch"].as_u64().unwrap_or_default(); + + // Get transactions for this block + let txs_url = format!("{}/blocks/{}/txs?count=100", base_url, block_hash); + let txs_resp: serde_json::Value = client.get(&txs_url).send().await?.json().await?; + + let txs = match txs_resp.as_array() { + Some(arr) => arr + .iter() + .filter_map(|v| v["tx_hash"].as_str().or_else(|| v.as_str()).map(String::from)) + .take(20) + .collect(), + None => Vec::new(), + }; + + Ok(ChainSample { + block_hash, + block_number, + slot, + epoch, + txs, + }) +} diff --git a/xtask/src/minibf_bench/stats.rs b/xtask/src/minibf_bench/stats.rs new file mode 100644 index 00000000..c304f30e --- /dev/null +++ b/xtask/src/minibf_bench/stats.rs @@ -0,0 +1,79 @@ +//! Statistical analysis using HDR histogram. + +use anyhow::Result; +use hdrhistogram::Histogram; + +use crate::minibf_bench::runner::RequestResult; + +/// Statistics for a single endpoint +#[derive(Debug, Clone)] +pub struct EndpointStats { + pub name: String, + pub path: String, + pub requests: u64, + pub success_count: u64, + pub failure_count: u64, + pub latency_p50_micros: u64, + pub latency_p95_micros: u64, + pub latency_p99_micros: u64, + pub latency_min_micros: u64, + pub latency_max_micros: u64, +} + +impl EndpointStats { + pub fn success_rate(&self) -> f64 { + if self.requests == 0 { + 0.0 + } else { + self.success_count as f64 / self.requests as f64 + } + } +} + +pub fn calculate_endpoint_stats(results: &[RequestResult]) -> Result> { + use std::collections::HashMap; + + let mut endpoint_data: HashMap> = HashMap::new(); + + for result in results { + endpoint_data + .entry(result.endpoint_name.clone()) + .or_default() + .push(result); + } + + let mut stats = Vec::new(); + + for (name, results) in endpoint_data { + let mut histogram = Histogram::::new_with_bounds(1, 60_000_000, 3)?; + let mut success_count = 0u64; + let mut failure_count = 0u64; + + for result in &results { + histogram.record(result.latency_micros)?; + if result.success { + success_count += 1; + } else { + failure_count += 1; + } + } + + stats.push(EndpointStats { + path: results.first().map(|r| r.path.clone()).unwrap_or_default(), + name, + requests: results.len() as u64, + success_count, + failure_count, + latency_p50_micros: histogram.value_at_quantile(0.50), + latency_p95_micros: histogram.value_at_quantile(0.95), + latency_p99_micros: histogram.value_at_quantile(0.99), + latency_min_micros: histogram.min(), + latency_max_micros: histogram.max(), + }); + } + + // Sort by name for consistent output + stats.sort_by(|a, b| a.name.cmp(&b.name)); + + Ok(stats) +} diff --git a/xtask/src/minibf_bench/vectors.rs b/xtask/src/minibf_bench/vectors.rs new file mode 100644 index 00000000..e1cdc373 --- /dev/null +++ b/xtask/src/minibf_bench/vectors.rs @@ -0,0 +1,238 @@ +//! Test vector management: query DBSync and cache results. + +use anyhow::{Context, Result}; +use postgres::{Client, NoTls}; +use serde::{Deserialize, Serialize}; +use std::fs; +use std::path::PathBuf; +use std::time::{Duration, SystemTime}; + +use crate::config::Network; + +const CACHE_TTL_HOURS: u64 = 24; +const VECTORS_DIR: &str = "src/minibf_bench/vectors"; + +/// Test vectors for a specific network +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct TestVectors { + pub network: String, + pub generated_at: String, + pub dbsync_source: String, + pub addresses: Vec, + pub stake_addresses: Vec, + pub reference_blocks: ReferenceBlocks, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct AddressVector { + pub address: String, + pub address_type: String, + pub tx_count: i64, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct StakeAddressVector { + pub stake_address: String, + pub delegation_count: i64, + pub reward_count: i64, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ReferenceBlocks { + pub early: BlockRef, + pub mid: BlockRef, + pub recent: BlockRef, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct BlockRef { + pub height: i64, + pub hash: String, + pub epoch: i64, + pub slot: i64, +} + +pub struct VectorGenerator { + client: Client, +} + +impl VectorGenerator { + pub fn new(dbsync_url: &str) -> Result { + let client = Client::connect(dbsync_url, NoTls).context("Failed to connect to DBSync")?; + Ok(Self { client }) + } + + pub fn generate_vectors( + &mut self, + network: &Network, + dbsync_host: &str, + ) -> Result { + let network_str = network.as_str(); + + Ok(TestVectors { + network: network_str.to_string(), + generated_at: chrono::Utc::now().to_rfc3339(), + dbsync_source: dbsync_host.to_string(), + addresses: self.query_addresses(network)?, + stake_addresses: self.query_stake_addresses()?, + reference_blocks: self.query_reference_blocks()?, + }) + } + + fn query_addresses(&mut self, network: &Network) -> Result> { + let pattern = match network { + Network::Mainnet => "addr1%", + Network::Preprod => "addr_test1%", + Network::Preview => "addr_test1%", + }; + + let rows = self.client.query( + "SELECT DISTINCT tx_out.address, sa.view as stake_address, COUNT(tx.id) as tx_count + FROM tx_out + LEFT JOIN stake_address sa ON tx_out.stake_address_id = sa.id + JOIN tx ON tx_out.tx_id = tx.id + WHERE tx_out.address LIKE $1 + GROUP BY tx_out.address, sa.view + ORDER BY tx_count DESC + LIMIT 50", + &[&pattern], + )?; + + rows.iter() + .map(|row| { + let address: String = row.get(0); + let stake: Option = row.get(1); + let tx_count: i64 = row.get(2); + + let address_type = if stake.is_some() { + "shelley_payment_stake" + } else { + "shelley_payment_only" + }; + + Ok(AddressVector { + address, + address_type: address_type.to_string(), + tx_count, + }) + }) + .collect() + } + + fn query_stake_addresses(&mut self) -> Result> { + let rows = self.client.query( + "SELECT sa.view, + COALESCE(d.cnt, 0) as delegation_count, + COALESCE(r.cnt, 0) as reward_count + FROM stake_address sa + LEFT JOIN (SELECT addr_id, COUNT(*) as cnt FROM delegation GROUP BY addr_id) d ON sa.id = d.addr_id + LEFT JOIN (SELECT addr_id, COUNT(*) as cnt FROM reward GROUP BY addr_id) r ON sa.id = r.addr_id + WHERE COALESCE(d.cnt, 0) > 0 OR COALESCE(r.cnt, 0) > 0 + ORDER BY delegation_count DESC, reward_count DESC + LIMIT 30", + &[], + )?; + + rows.iter() + .map(|row| { + Ok(StakeAddressVector { + stake_address: row.get(0), + delegation_count: row.get(1), + reward_count: row.get(2), + }) + }) + .collect() + } + + fn query_reference_blocks(&mut self) -> Result { + // Get tip + let tip_row = self.client.query_one( + "SELECT block_no, hash::text, epoch_no, slot_no + FROM block + WHERE block_no IS NOT NULL + ORDER BY block_no DESC + LIMIT 1", + &[], + )?; + + let tip_height: i32 = tip_row.get(0); + + // Calculate positions for 10%, 50%, 90% + let early_height = tip_height / 10; + let mid_height = tip_height / 2; + let recent_height = (tip_height * 9) / 10; + + Ok(ReferenceBlocks { + early: self.get_block_at_height(early_height)?, + mid: self.get_block_at_height(mid_height)?, + recent: self.get_block_at_height(recent_height)?, + }) + } + + fn get_block_at_height(&mut self, height: i32) -> Result { + let row = self.client.query_one( + "SELECT block_no, hash::text, epoch_no, slot_no + FROM block + WHERE block_no = $1", + &[&height], + )?; + + let height: i32 = row.get(0); + let epoch: i32 = row.get(2); + let slot: i64 = row.get(3); + + Ok(BlockRef { + height: height as i64, + hash: row.get(1), + epoch: epoch as i64, + slot, + }) + } +} + +/// Load or generate test vectors for a network +pub fn load_vectors( + network: &Network, + dbsync_url: &str, + force_refresh: bool, +) -> Result { + let vectors_path = PathBuf::from(VECTORS_DIR).join(format!("{}.json", network.as_str())); + + // Check if we can use cached vectors + if !force_refresh && vectors_path.exists() { + let metadata = fs::metadata(&vectors_path)?; + let modified = metadata.modified()?; + let age = SystemTime::now().duration_since(modified)?; + + if age < Duration::from_secs(CACHE_TTL_HOURS * 3600) { + let content = fs::read_to_string(&vectors_path)?; + return Ok(serde_json::from_str(&content)?); + } + } + + // Generate new vectors + println!( + "Generating test vectors for {} from DBSync...", + network.as_str() + ); + + let mut generator = VectorGenerator::new(dbsync_url)?; + let dbsync_host = dbsync_url + .split('@') + .nth(1) + .and_then(|s| s.split('/').next()) + .unwrap_or("unknown") + .to_string(); + + let vectors = generator.generate_vectors(network, &dbsync_host)?; + + // Ensure directory exists + fs::create_dir_all(&vectors_path.parent().unwrap())?; + + // Cache to file + let json = serde_json::to_string_pretty(&vectors)?; + fs::write(&vectors_path, json)?; + println!("Cached test vectors to {}", vectors_path.display()); + + Ok(vectors) +}