From f5e366341b4abc5078b41ddff7037e992fdcbf89 Mon Sep 17 00:00:00 2001 From: dmelotik Date: Fri, 9 Dec 2022 18:56:18 -0700 Subject: [PATCH 01/36] initial commit on solana balances --- sol-balance/Cargo.toml | 22 ++++++++++++++++++++++ sol-balance/Makefile | 11 +++++++++++ sol-balance/proto/v1/sol_balance.proto | 12 ++++++++++++ sol-balance/rust-toolchain.toml | 4 ++++ sol-balance/src/lib.rs | 7 +++++++ sol-balance/substreams.yaml | 26 ++++++++++++++++++++++++++ 6 files changed, 82 insertions(+) create mode 100644 sol-balance/Cargo.toml create mode 100644 sol-balance/Makefile create mode 100644 sol-balance/proto/v1/sol_balance.proto create mode 100644 sol-balance/rust-toolchain.toml create mode 100644 sol-balance/src/lib.rs create mode 100644 sol-balance/substreams.yaml diff --git a/sol-balance/Cargo.toml b/sol-balance/Cargo.toml new file mode 100644 index 00000000..8a898f72 --- /dev/null +++ b/sol-balance/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "sol-balance" +version = "0.1.0" +description = "Messari's standardized substream for wallets sol balance" +edition = "2021" +repository = "https://github.com/messari/substreams" + +[lib] +crate-type = ["cdylib"] + +[dependencies] +prost = "0.11.2" +ethabi = "17.2.0" +num-bigint = "0.4" +hex-literal = "0.3.4" +substreams = { workspace = true } +substreams-solana = { workspace = true } +substreams-helper = { path = "../substreams-helper" } + +[build-dependencies] +anyhow = "1" +substreams-common = { path = "../common" } \ No newline at end of file diff --git a/sol-balance/Makefile b/sol-balance/Makefile new file mode 100644 index 00000000..151f018c --- /dev/null +++ b/sol-balance/Makefile @@ -0,0 +1,11 @@ +.PHONY: codegen +codegen: + substreams protogen ./substreams.yaml --exclude-paths="sf/solana,sf/substreams,google" + +.PHONY: build +build: + cargo build --target wasm32-unknown-unknown --release + +.PHONY: run +run: + substreams run -e mainnet.sol.streamingfast.io:443 substreams.yaml store_balance -s 0 -t +10 diff --git a/sol-balance/proto/v1/sol_balance.proto b/sol-balance/proto/v1/sol_balance.proto new file mode 100644 index 00000000..260bc705 --- /dev/null +++ b/sol-balance/proto/v1/sol_balance.proto @@ -0,0 +1,12 @@ +syntax = "proto3"; + +package messari.sol_balance.v1; + +message SolBalance { + string address = 1; + string value = 2; +} + +message SolBalances { + repeated SolBalance items = 1; +} \ No newline at end of file diff --git a/sol-balance/rust-toolchain.toml b/sol-balance/rust-toolchain.toml new file mode 100644 index 00000000..0a0b90ae --- /dev/null +++ b/sol-balance/rust-toolchain.toml @@ -0,0 +1,4 @@ +[toolchain] +channel = "1.64.0" +components = [ "rustfmt" ] +targets = [ "wasm32-unknown-unknown" ] diff --git a/sol-balance/src/lib.rs b/sol-balance/src/lib.rs new file mode 100644 index 00000000..57605217 --- /dev/null +++ b/sol-balance/src/lib.rs @@ -0,0 +1,7 @@ +use substreams::{log}; +use substreams_solana::pb::sol as solana; + +#[substreams::handlers::store] +fn store_balance(block: solana::v1::Block) { + log::info!("block height: {}", block.blockhash); +} diff --git a/sol-balance/substreams.yaml b/sol-balance/substreams.yaml new file mode 100644 index 00000000..58798322 --- /dev/null +++ b/sol-balance/substreams.yaml @@ -0,0 +1,26 @@ +specVersion: v0.1.0 +package: + name: sol_balance + version: v0.1.0 + +imports: + sol: https://github.com/streamingfast/firehose-solana/releases/download/v0.1.0/solana-v0.1.0.spkg + +protobuf: + files: + - sol_balance.proto + importPaths: + - proto/v1 + +binaries: + default: + type: wasm/rust-v1 + file: "../target/wasm32-unknown-unknown/release/sol_balance.wasm" + +modules: + - name: store_balance + kind: store + updatePolicy: set + valueType: string + inputs: + - source: sf.solana.type.v1.Block From ab018e24a89281e7224ebd7a054b0b10b1eb21ca Mon Sep 17 00:00:00 2001 From: dmelotik Date: Tue, 13 Dec 2022 21:29:18 -0700 Subject: [PATCH 02/36] WIP: trying to get accounts linked to balances... --- sol-balance/Cargo.toml | 3 +- sol-balance/Makefile | 2 +- sol-balance/proto/v1/sol_balance.proto | 2 +- sol-balance/src/lib.rs | 44 ++++++++++++++++++++++++-- 4 files changed, 45 insertions(+), 6 deletions(-) diff --git a/sol-balance/Cargo.toml b/sol-balance/Cargo.toml index 8a898f72..689d2ab5 100644 --- a/sol-balance/Cargo.toml +++ b/sol-balance/Cargo.toml @@ -9,11 +9,12 @@ repository = "https://github.com/messari/substreams" crate-type = ["cdylib"] [dependencies] +bs58 = "0.4.0" prost = "0.11.2" ethabi = "17.2.0" num-bigint = "0.4" hex-literal = "0.3.4" -substreams = { workspace = true } +substreams = "0.0.11" substreams-solana = { workspace = true } substreams-helper = { path = "../substreams-helper" } diff --git a/sol-balance/Makefile b/sol-balance/Makefile index 151f018c..adcdf3e9 100644 --- a/sol-balance/Makefile +++ b/sol-balance/Makefile @@ -8,4 +8,4 @@ build: .PHONY: run run: - substreams run -e mainnet.sol.streamingfast.io:443 substreams.yaml store_balance -s 0 -t +10 + substreams run -e mainnet.sol.streamingfast.io:443 substreams.yaml store_balance -s 1000 -t +10 diff --git a/sol-balance/proto/v1/sol_balance.proto b/sol-balance/proto/v1/sol_balance.proto index 260bc705..8442a338 100644 --- a/sol-balance/proto/v1/sol_balance.proto +++ b/sol-balance/proto/v1/sol_balance.proto @@ -9,4 +9,4 @@ message SolBalance { message SolBalances { repeated SolBalance items = 1; -} \ No newline at end of file +} diff --git a/sol-balance/src/lib.rs b/sol-balance/src/lib.rs index 57605217..89c86ca5 100644 --- a/sol-balance/src/lib.rs +++ b/sol-balance/src/lib.rs @@ -1,7 +1,45 @@ -use substreams::{log}; +use substreams::log; +use substreams::store::StoreSet; use substreams_solana::pb::sol as solana; +// use bs58; #[substreams::handlers::store] -fn store_balance(block: solana::v1::Block) { - log::info!("block height: {}", block.blockhash); +fn store_balance(block: solana::v1::Block, output: StoreSet) { + log::info!("block hash: {}", block.previous_blockhash); + log::info!("block hash: {}", block.parent_slot); + + // TODO: try to get balances in this for loop + + for tx in block.transactions { + if let Some(meta) = tx.meta { + if let Some(_) = meta.err { + continue; + } + log::info!("bal len: {}", meta.post_balances.len()); + + for bal in meta.post_balances { + // TODO: I don't think this is the right address, but I am struggling to figure out another way + + let change_address = + + output.set( + 0, + + ) + } + // log::info!("{}", meta.log_messages_none); + // for bal in meta.post_balances { + // log::info!("post bal: {}", bal); + // } + if let Some(transaction) = tx.transaction { + if let Some(msg) = transaction.message { + log::info!("acct len: {}", msg.account_keys.len()); + for accts in msg.account_keys { + // let program_id = &msg.account_keys[inst.program_id_index as usize]; + // log::info!("acc: {}", bs58::encode(program_id).into_string()); + } + } + } + } + } } From 6c6d8f0e5447d060bafa6e84ab5416d11d1ac602 Mon Sep 17 00:00:00 2001 From: dmelotik Date: Tue, 13 Dec 2022 21:48:49 -0700 Subject: [PATCH 03/36] update substreams-solana --- sol-balance/src/lib.rs | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/sol-balance/src/lib.rs b/sol-balance/src/lib.rs index 89c86ca5..383698f6 100644 --- a/sol-balance/src/lib.rs +++ b/sol-balance/src/lib.rs @@ -20,14 +20,15 @@ fn store_balance(block: solana::v1::Block, output: StoreSet) { for bal in meta.post_balances { // TODO: I don't think this is the right address, but I am struggling to figure out another way - let change_address = - - output.set( - 0, - - ) + // let change_address = + // + // output.set( + // 0, + // + // ) } - // log::info!("{}", meta.log_messages_none); + log::info!("{}", meta.loaded_writable_addresses.len()); + log::info!("{}", meta.post_balances.len()); // for bal in meta.post_balances { // log::info!("post bal: {}", bal); // } From 6a0ec2c202b3cce4b90acd488d223ead7feaa8bd Mon Sep 17 00:00:00 2001 From: dmelotik Date: Wed, 14 Dec 2022 18:33:39 -0700 Subject: [PATCH 04/36] set output --- sol-balance/Cargo.toml | 2 +- sol-balance/Makefile | 2 +- sol-balance/src/lib.rs | 39 +++++++++------------------------------ 3 files changed, 11 insertions(+), 32 deletions(-) diff --git a/sol-balance/Cargo.toml b/sol-balance/Cargo.toml index 689d2ab5..a390e922 100644 --- a/sol-balance/Cargo.toml +++ b/sol-balance/Cargo.toml @@ -15,7 +15,7 @@ ethabi = "17.2.0" num-bigint = "0.4" hex-literal = "0.3.4" substreams = "0.0.11" -substreams-solana = { workspace = true } +substreams-solana = "0.1.0" substreams-helper = { path = "../substreams-helper" } [build-dependencies] diff --git a/sol-balance/Makefile b/sol-balance/Makefile index adcdf3e9..fad4489a 100644 --- a/sol-balance/Makefile +++ b/sol-balance/Makefile @@ -8,4 +8,4 @@ build: .PHONY: run run: - substreams run -e mainnet.sol.streamingfast.io:443 substreams.yaml store_balance -s 1000 -t +10 + substreams run -e mainnet.sol.streamingfast.io:443 substreams.yaml store_balance -s 10000 -t +100 diff --git a/sol-balance/src/lib.rs b/sol-balance/src/lib.rs index 383698f6..98d3315b 100644 --- a/sol-balance/src/lib.rs +++ b/sol-balance/src/lib.rs @@ -1,45 +1,24 @@ use substreams::log; use substreams::store::StoreSet; use substreams_solana::pb::sol as solana; -// use bs58; +use bs58; #[substreams::handlers::store] fn store_balance(block: solana::v1::Block, output: StoreSet) { - log::info!("block hash: {}", block.previous_blockhash); - log::info!("block hash: {}", block.parent_slot); - - // TODO: try to get balances in this for loop - for tx in block.transactions { if let Some(meta) = tx.meta { if let Some(_) = meta.err { continue; } - log::info!("bal len: {}", meta.post_balances.len()); - - for bal in meta.post_balances { + log::info!("bal len: {}", &meta.post_balances.len()); // TODO: remove + for i in 0..meta.post_balances.len() { // TODO: I don't think this is the right address, but I am struggling to figure out another way - - // let change_address = - // - // output.set( - // 0, - // - // ) - } - log::info!("{}", meta.loaded_writable_addresses.len()); - log::info!("{}", meta.post_balances.len()); - // for bal in meta.post_balances { - // log::info!("post bal: {}", bal); - // } - if let Some(transaction) = tx.transaction { - if let Some(msg) = transaction.message { - log::info!("acct len: {}", msg.account_keys.len()); - for accts in msg.account_keys { - // let program_id = &msg.account_keys[inst.program_id_index as usize]; - // log::info!("acc: {}", bs58::encode(program_id).into_string()); - } - } + let account_id = bs58::encode(&meta.loaded_writable_addresses[i]).into_string(); + output.set( + i as u64, + account_id, + &meta.post_balances[i] // TODO: not sure why this wants a &Vec + ); } } } From 59d9d742441066ce781429cc1a85f629f2c6c628 Mon Sep 17 00:00:00 2001 From: dmelotik Date: Wed, 14 Dec 2022 21:38:15 -0700 Subject: [PATCH 05/36] add README --- sol-balance/README.md | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 sol-balance/README.md diff --git a/sol-balance/README.md b/sol-balance/README.md new file mode 100644 index 00000000..e69de29b From 19669897c1c0131bea312e3e4f8dd9f2c8f41408 Mon Sep 17 00:00:00 2001 From: dmelotik Date: Wed, 14 Dec 2022 21:38:46 -0700 Subject: [PATCH 06/36] add README --- sol-balance/README.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/sol-balance/README.md b/sol-balance/README.md index e69de29b..3c784e51 100644 --- a/sol-balance/README.md +++ b/sol-balance/README.md @@ -0,0 +1,10 @@ +# SOL Balance Substream + +This substream is designed to store the SOL balance of every account. + +### Build & Run + +```bash +make codegen +make run +``` From 3ea48ecb247c71c24ca0b6703ee5b5a669f006f4 Mon Sep 17 00:00:00 2001 From: dmelotik Date: Fri, 16 Dec 2022 00:27:19 -0700 Subject: [PATCH 07/36] Add proper commands for sol-balance --- erc20-holdings/README.md | 0 sol-balance/src/pb/messari.sol_balance.v1.rs | 45 ++++++++++++++++++++ 2 files changed, 45 insertions(+) create mode 100644 erc20-holdings/README.md create mode 100644 sol-balance/src/pb/messari.sol_balance.v1.rs diff --git a/erc20-holdings/README.md b/erc20-holdings/README.md new file mode 100644 index 00000000..e69de29b diff --git a/sol-balance/src/pb/messari.sol_balance.v1.rs b/sol-balance/src/pb/messari.sol_balance.v1.rs new file mode 100644 index 00000000..3694a7ee --- /dev/null +++ b/sol-balance/src/pb/messari.sol_balance.v1.rs @@ -0,0 +1,45 @@ +// @generated +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct SolBalance { + #[prost(string, tag="1")] + pub address: ::prost::alloc::string::String, + #[prost(string, tag="2")] + pub value: ::prost::alloc::string::String, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct SolBalances { + #[prost(message, repeated, tag="1")] + pub items: ::prost::alloc::vec::Vec, +} +/// Encoded file descriptor set for the `messari.sol_balance.v1` package +pub const FILE_DESCRIPTOR_SET: &[u8] = &[ + 0x0a, 0xbc, 0x03, 0x0a, 0x11, 0x73, 0x6f, 0x6c, 0x5f, 0x62, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, + 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x16, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x72, 0x69, 0x2e, + 0x73, 0x6f, 0x6c, 0x5f, 0x62, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x22, 0x3c, + 0x0a, 0x0a, 0x53, 0x6f, 0x6c, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x12, 0x18, 0x0a, 0x07, + 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x61, + 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x22, 0x47, 0x0a, 0x0b, + 0x53, 0x6f, 0x6c, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x73, 0x12, 0x38, 0x0a, 0x05, 0x69, + 0x74, 0x65, 0x6d, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x6d, 0x65, 0x73, + 0x73, 0x61, 0x72, 0x69, 0x2e, 0x73, 0x6f, 0x6c, 0x5f, 0x62, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, + 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x6f, 0x6c, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x52, 0x05, + 0x69, 0x74, 0x65, 0x6d, 0x73, 0x4a, 0xff, 0x01, 0x0a, 0x06, 0x12, 0x04, 0x00, 0x00, 0x0b, 0x01, + 0x0a, 0x08, 0x0a, 0x01, 0x0c, 0x12, 0x03, 0x00, 0x00, 0x12, 0x0a, 0x08, 0x0a, 0x01, 0x02, 0x12, + 0x03, 0x02, 0x00, 0x1f, 0x0a, 0x0a, 0x0a, 0x02, 0x04, 0x00, 0x12, 0x04, 0x04, 0x00, 0x07, 0x01, + 0x0a, 0x0a, 0x0a, 0x03, 0x04, 0x00, 0x01, 0x12, 0x03, 0x04, 0x08, 0x12, 0x0a, 0x0b, 0x0a, 0x04, + 0x04, 0x00, 0x02, 0x00, 0x12, 0x03, 0x05, 0x02, 0x15, 0x0a, 0x0c, 0x0a, 0x05, 0x04, 0x00, 0x02, + 0x00, 0x05, 0x12, 0x03, 0x05, 0x02, 0x08, 0x0a, 0x0c, 0x0a, 0x05, 0x04, 0x00, 0x02, 0x00, 0x01, + 0x12, 0x03, 0x05, 0x09, 0x10, 0x0a, 0x0c, 0x0a, 0x05, 0x04, 0x00, 0x02, 0x00, 0x03, 0x12, 0x03, + 0x05, 0x13, 0x14, 0x0a, 0x0b, 0x0a, 0x04, 0x04, 0x00, 0x02, 0x01, 0x12, 0x03, 0x06, 0x02, 0x13, + 0x0a, 0x0c, 0x0a, 0x05, 0x04, 0x00, 0x02, 0x01, 0x05, 0x12, 0x03, 0x06, 0x02, 0x08, 0x0a, 0x0c, + 0x0a, 0x05, 0x04, 0x00, 0x02, 0x01, 0x01, 0x12, 0x03, 0x06, 0x09, 0x0e, 0x0a, 0x0c, 0x0a, 0x05, + 0x04, 0x00, 0x02, 0x01, 0x03, 0x12, 0x03, 0x06, 0x11, 0x12, 0x0a, 0x0a, 0x0a, 0x02, 0x04, 0x01, + 0x12, 0x04, 0x09, 0x00, 0x0b, 0x01, 0x0a, 0x0a, 0x0a, 0x03, 0x04, 0x01, 0x01, 0x12, 0x03, 0x09, + 0x08, 0x13, 0x0a, 0x0b, 0x0a, 0x04, 0x04, 0x01, 0x02, 0x00, 0x12, 0x03, 0x0a, 0x02, 0x20, 0x0a, + 0x0c, 0x0a, 0x05, 0x04, 0x01, 0x02, 0x00, 0x04, 0x12, 0x03, 0x0a, 0x02, 0x0a, 0x0a, 0x0c, 0x0a, + 0x05, 0x04, 0x01, 0x02, 0x00, 0x06, 0x12, 0x03, 0x0a, 0x0b, 0x15, 0x0a, 0x0c, 0x0a, 0x05, 0x04, + 0x01, 0x02, 0x00, 0x01, 0x12, 0x03, 0x0a, 0x16, 0x1b, 0x0a, 0x0c, 0x0a, 0x05, 0x04, 0x01, 0x02, + 0x00, 0x03, 0x12, 0x03, 0x0a, 0x1e, 0x1f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +]; +// @@protoc_insertion_point(module) \ No newline at end of file From 22a4c46dc482fb3f3a6e10b1acc093f2d2e70653 Mon Sep 17 00:00:00 2001 From: dmelotik Date: Wed, 21 Dec 2022 23:05:29 -0700 Subject: [PATCH 08/36] add cargo and makefile for sol --- Cargo.lock | 22 ++++++++++++++++++++++ Cargo.toml | 3 ++- Makefile | 2 ++ 3 files changed, 26 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 8cef384b..396c8da7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -76,6 +76,12 @@ dependencies = [ "generic-array", ] +[[package]] +name = "bs58" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "771fe0050b883fcc3ea2359b1a96bcfbc090b7116eae7c3c512c7a083fdf23d3" + [[package]] name = "bumpalo" version = "3.11.0" @@ -899,6 +905,22 @@ dependencies = [ "keccak", ] +[[package]] +name = "sol-balance" +version = "0.1.0" +dependencies = [ + "anyhow", + "bs58", + "ethabi", + "hex-literal", + "num-bigint", + "prost 0.11.2", + "substreams 0.0.11", + "substreams-common", + "substreams-helper", + "substreams-solana", +] + [[package]] name = "solana-sample" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index e455ffe6..4a3d3816 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,13 +11,14 @@ members = [ "substreams-helper", "uniswap-v2", "solana-sample", +d "sol-balance", ] exclude = ["messari-cli"] [workspace.dependencies] substreams = "0.3.2" substreams-ethereum = "0.6.2" -substreams-solana = "0.1.0" +substreams-solana = "0.2.0" [build] target = "wasm32-unknown-unknown" diff --git a/Makefile b/Makefile index 187c2d41..c302cbc8 100644 --- a/Makefile +++ b/Makefile @@ -10,6 +10,7 @@ build-all: $(MAKE) -C network build $(MAKE) -C solana-sample build $(MAKE) -C eth-balance build + $(MAKE) -C sol-balance build .PHONY: run-all run-all: @@ -18,6 +19,7 @@ run-all: $(MAKE) -C erc20-market-cap run $(MAKE) -C erc721 run $(MAKE) -C network run + $(MAKE) -C sol-balance run .PHONY: test test: From af3f71d383edc0d5ce6139a516d6c659fed00748 Mon Sep 17 00:00:00 2001 From: dmelotik Date: Wed, 21 Dec 2022 23:06:59 -0700 Subject: [PATCH 09/36] remove typo --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 4a3d3816..6b27cf0e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,7 +11,7 @@ members = [ "substreams-helper", "uniswap-v2", "solana-sample", -d "sol-balance", + "sol-balance", ] exclude = ["messari-cli"] From def13f0aadb6fc30b46b3038d3cb03e56cf07966 Mon Sep 17 00:00:00 2001 From: dmelotik Date: Thu, 22 Dec 2022 11:05:05 -0700 Subject: [PATCH 10/36] encode to vec and update readmes --- README.md | 1 + sol-balance/README.md | 5 +++++ sol-balance/src/lib.rs | 5 ++--- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 8704de8e..0501edd1 100644 --- a/README.md +++ b/README.md @@ -43,6 +43,7 @@ | ENS Look Up | 🔨 | ENS records for lookup and reverse lookup | | Uniswap v2 | 🔨 | Substreams for Uniswap v2 | | Compound v2 | 🔨 | Substreams for Compound v2 | +| SOL Balance | 🛠 | SOL balance for every Solana address | ## Workflow diff --git a/sol-balance/README.md b/sol-balance/README.md index 3c784e51..ac707b94 100644 --- a/sol-balance/README.md +++ b/sol-balance/README.md @@ -8,3 +8,8 @@ This substream is designed to store the SOL balance of every account. make codegen make run ``` + +### Notes + +- Currently [`transaction.loaded_writable_addresses`](https://github.com/streamingfast/firehose-solana/blob/develop/proto/sf/solana/type/v1/type.proto#L65) is an unknown field on firesol blocks. +- diff --git a/sol-balance/src/lib.rs b/sol-balance/src/lib.rs index 98d3315b..b351adbb 100644 --- a/sol-balance/src/lib.rs +++ b/sol-balance/src/lib.rs @@ -1,7 +1,7 @@ -use substreams::log; use substreams::store::StoreSet; use substreams_solana::pb::sol as solana; use bs58; +use prost::Message; #[substreams::handlers::store] fn store_balance(block: solana::v1::Block, output: StoreSet) { @@ -10,14 +10,13 @@ fn store_balance(block: solana::v1::Block, output: StoreSet) { if let Some(_) = meta.err { continue; } - log::info!("bal len: {}", &meta.post_balances.len()); // TODO: remove for i in 0..meta.post_balances.len() { // TODO: I don't think this is the right address, but I am struggling to figure out another way let account_id = bs58::encode(&meta.loaded_writable_addresses[i]).into_string(); output.set( i as u64, account_id, - &meta.post_balances[i] // TODO: not sure why this wants a &Vec + &meta.post_balances[i].encode_to_vec() ); } } From 6e45941adf20d6af3a191d66038bc890081a8cc6 Mon Sep 17 00:00:00 2001 From: dmelotik Date: Fri, 6 Jan 2023 20:36:54 -0700 Subject: [PATCH 11/36] add sol token proto to common --- common/proto/sol.token.proto | 44 ++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 common/proto/sol.token.proto diff --git a/common/proto/sol.token.proto b/common/proto/sol.token.proto new file mode 100644 index 00000000..02b4417c --- /dev/null +++ b/common/proto/sol.token.proto @@ -0,0 +1,44 @@ +syntax = "proto3"; + +package messari.sol.token.v1; + +message Transfers { + repeated Transfer transfers = 1; +} + +message Transfer { + string signature = 1; + uint64 ordinal = 2; + string from = 3; + string to = 4; + TokenAccount token = 5; + uint64 native_amount = 6; + uint64 amount = 7; +} + +message Tokens { + repeated TokenAccount tokens = 1; +} + +message TokenAccount { + string address = 1; + string name = 2; + string symbol = 3; + string decimals = 4; + string owner = 5; + string mint = 6; +} + +message Balance { + TokenAccount token = 1; + string balance = 2; +} + +message Account { + string account_id = 1; + repeated Balance balances = 2; +} + +message Accounts { + repeated Account items = 1; +} From a72906a5ce1b82b2e79e953eefd8620fac2a2af1 Mon Sep 17 00:00:00 2001 From: dmelotik Date: Fri, 6 Jan 2023 20:54:54 -0700 Subject: [PATCH 12/36] update to map module --- sol-balance/proto/v1/sol_balance.proto | 12 ------------ sol-balance/substreams.yaml | 13 +++++++------ 2 files changed, 7 insertions(+), 18 deletions(-) delete mode 100644 sol-balance/proto/v1/sol_balance.proto diff --git a/sol-balance/proto/v1/sol_balance.proto b/sol-balance/proto/v1/sol_balance.proto deleted file mode 100644 index 8442a338..00000000 --- a/sol-balance/proto/v1/sol_balance.proto +++ /dev/null @@ -1,12 +0,0 @@ -syntax = "proto3"; - -package messari.sol_balance.v1; - -message SolBalance { - string address = 1; - string value = 2; -} - -message SolBalances { - repeated SolBalance items = 1; -} diff --git a/sol-balance/substreams.yaml b/sol-balance/substreams.yaml index 58798322..b62aeec9 100644 --- a/sol-balance/substreams.yaml +++ b/sol-balance/substreams.yaml @@ -8,9 +8,9 @@ imports: protobuf: files: - - sol_balance.proto + - sol.token.proto importPaths: - - proto/v1 + - ../common/proto binaries: default: @@ -18,9 +18,10 @@ binaries: file: "../target/wasm32-unknown-unknown/release/sol_balance.wasm" modules: - - name: store_balance - kind: store - updatePolicy: set - valueType: string + - name: map_balances + kind: map + initialBlock: 0 inputs: - source: sf.solana.type.v1.Block + output: + type: proto:messari.sol.token.Account From 13690d79d90f83629622ea11f9007b506eb437d5 Mon Sep 17 00:00:00 2001 From: dmelotik Date: Fri, 6 Jan 2023 21:01:52 -0700 Subject: [PATCH 13/36] begin map handler --- sol-balance/src/lib.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/sol-balance/src/lib.rs b/sol-balance/src/lib.rs index b351adbb..c3b06b72 100644 --- a/sol-balance/src/lib.rs +++ b/sol-balance/src/lib.rs @@ -1,10 +1,13 @@ use substreams::store::StoreSet; +use crate::pb::token::v1::Token; +use pb::token::v1 as proto; use substreams_solana::pb::sol as solana; use bs58; use prost::Message; +use substreams_solana::pb; -#[substreams::handlers::store] -fn store_balance(block: solana::v1::Block, output: StoreSet) { +#[substreams::handlers::map] +fn map(block: solana::v1::Block) -> Result { for tx in block.transactions { if let Some(meta) = tx.meta { if let Some(_) = meta.err { From 3a4cbbfc7bc7f44e277876bb3b2b6b68f9075728 Mon Sep 17 00:00:00 2001 From: dmelotik Date: Thu, 12 Jan 2023 12:38:05 -0700 Subject: [PATCH 14/36] fix sol balance build setup --- .../{sol.token.proto => sol_token.proto} | 4 ++ sol-balance/Makefile | 6 +-- sol-balance/build.rs | 10 ++++ sol-balance/src/abi.rs | 1 + sol-balance/src/lib.rs | 53 +++++++++++-------- sol-balance/src/pb.rs | 19 +++++++ sol-balance/src/pb/messari.sol_balance.v1.rs | 45 ---------------- sol-balance/substreams.yaml | 18 ++++--- 8 files changed, 78 insertions(+), 78 deletions(-) rename common/proto/{sol.token.proto => sol_token.proto} (93%) create mode 100644 sol-balance/build.rs create mode 100644 sol-balance/src/abi.rs create mode 100644 sol-balance/src/pb.rs delete mode 100644 sol-balance/src/pb/messari.sol_balance.v1.rs diff --git a/common/proto/sol.token.proto b/common/proto/sol_token.proto similarity index 93% rename from common/proto/sol.token.proto rename to common/proto/sol_token.proto index 02b4417c..5326ad83 100644 --- a/common/proto/sol.token.proto +++ b/common/proto/sol_token.proto @@ -42,3 +42,7 @@ message Account { message Accounts { repeated Account items = 1; } + +message OuterString { + string inner_string = 1; +} diff --git a/sol-balance/Makefile b/sol-balance/Makefile index fad4489a..2f559204 100644 --- a/sol-balance/Makefile +++ b/sol-balance/Makefile @@ -1,11 +1,7 @@ -.PHONY: codegen -codegen: - substreams protogen ./substreams.yaml --exclude-paths="sf/solana,sf/substreams,google" - .PHONY: build build: cargo build --target wasm32-unknown-unknown --release .PHONY: run run: - substreams run -e mainnet.sol.streamingfast.io:443 substreams.yaml store_balance -s 10000 -t +100 + substreams run -e mainnet.sol.streamingfast.io:443 substreams.yaml store_balance -s 10000 -t +10001 diff --git a/sol-balance/build.rs b/sol-balance/build.rs new file mode 100644 index 00000000..fb321d31 --- /dev/null +++ b/sol-balance/build.rs @@ -0,0 +1,10 @@ +use anyhow::{Ok, Result}; +use substreams_common::codegen; + +fn main() -> Result<(), anyhow::Error> { + println!("cargo:rerun-if-changed=proto"); + println!("cargo:rerun-if-changed=abi"); + codegen::generate(None)?; + + Ok(()) +} diff --git a/sol-balance/src/abi.rs b/sol-balance/src/abi.rs new file mode 100644 index 00000000..d55d99b3 --- /dev/null +++ b/sol-balance/src/abi.rs @@ -0,0 +1 @@ +// DO NOT EDIT - the file is generated by build script diff --git a/sol-balance/src/lib.rs b/sol-balance/src/lib.rs index c3b06b72..f1f44135 100644 --- a/sol-balance/src/lib.rs +++ b/sol-balance/src/lib.rs @@ -1,27 +1,36 @@ -use substreams::store::StoreSet; -use crate::pb::token::v1::Token; -use pb::token::v1 as proto; -use substreams_solana::pb::sol as solana; +use crate::pb::sol as token; use bs58; +// use pb::sol::Account; use prost::Message; +use substreams::store::StoreSet; use substreams_solana::pb; +use substreams_solana::pb::sol as solana; -#[substreams::handlers::map] -fn map(block: solana::v1::Block) -> Result { - for tx in block.transactions { - if let Some(meta) = tx.meta { - if let Some(_) = meta.err { - continue; - } - for i in 0..meta.post_balances.len() { - // TODO: I don't think this is the right address, but I am struggling to figure out another way - let account_id = bs58::encode(&meta.loaded_writable_addresses[i]).into_string(); - output.set( - i as u64, - account_id, - &meta.post_balances[i].encode_to_vec() - ); - } - } - } +#[substreams::handlers::store] +fn store_balance(block: solana::v1::Block, output: StoreSet) { + output.set( + 0, + "block".to_string(), + &format!("{:?}", block).as_bytes().to_vec(), + ); } + +// #[substreams::handlers::map] +// fn map(block: solana::v1::Block) -> Result { +// for tx in block.transactions { +// if let Some(meta) = tx.meta { +// if let Some(_) = meta.err { +// continue; +// } +// for i in 0..meta.post_balances.len() { +// // TODO: I don't think this is the right address, but I am struggling to figure out another way +// let account_id = bs58::encode(&meta.loaded_writable_addresses[i]).into_string(); +// output.set( +// i as u64, +// account_id, +// &meta.post_balances[i].encode_to_vec() +// ); +// } +// } +// } +// } diff --git a/sol-balance/src/pb.rs b/sol-balance/src/pb.rs new file mode 100644 index 00000000..dc8da007 --- /dev/null +++ b/sol-balance/src/pb.rs @@ -0,0 +1,19 @@ +#[rustfmt::skip] +#[path = "../target/pb/messari.sol.token.rs"] +pub(in crate::pb) mod sol_token; + +pub mod sol { + pub mod token { + pub use super::super::sol_token::*; + } +} + +#[rustfmt::skip] +#[path = "../target/pb/messari.solana.type.rs"] +pub(in crate::pb) mod solana_type; + +pub mod solana { + pub mod type { + pub use super::super::solana_type::*; + } +} diff --git a/sol-balance/src/pb/messari.sol_balance.v1.rs b/sol-balance/src/pb/messari.sol_balance.v1.rs deleted file mode 100644 index 3694a7ee..00000000 --- a/sol-balance/src/pb/messari.sol_balance.v1.rs +++ /dev/null @@ -1,45 +0,0 @@ -// @generated -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct SolBalance { - #[prost(string, tag="1")] - pub address: ::prost::alloc::string::String, - #[prost(string, tag="2")] - pub value: ::prost::alloc::string::String, -} -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct SolBalances { - #[prost(message, repeated, tag="1")] - pub items: ::prost::alloc::vec::Vec, -} -/// Encoded file descriptor set for the `messari.sol_balance.v1` package -pub const FILE_DESCRIPTOR_SET: &[u8] = &[ - 0x0a, 0xbc, 0x03, 0x0a, 0x11, 0x73, 0x6f, 0x6c, 0x5f, 0x62, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, - 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x16, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x72, 0x69, 0x2e, - 0x73, 0x6f, 0x6c, 0x5f, 0x62, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x22, 0x3c, - 0x0a, 0x0a, 0x53, 0x6f, 0x6c, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x12, 0x18, 0x0a, 0x07, - 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x61, - 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x22, 0x47, 0x0a, 0x0b, - 0x53, 0x6f, 0x6c, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x73, 0x12, 0x38, 0x0a, 0x05, 0x69, - 0x74, 0x65, 0x6d, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x6d, 0x65, 0x73, - 0x73, 0x61, 0x72, 0x69, 0x2e, 0x73, 0x6f, 0x6c, 0x5f, 0x62, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, - 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x6f, 0x6c, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x52, 0x05, - 0x69, 0x74, 0x65, 0x6d, 0x73, 0x4a, 0xff, 0x01, 0x0a, 0x06, 0x12, 0x04, 0x00, 0x00, 0x0b, 0x01, - 0x0a, 0x08, 0x0a, 0x01, 0x0c, 0x12, 0x03, 0x00, 0x00, 0x12, 0x0a, 0x08, 0x0a, 0x01, 0x02, 0x12, - 0x03, 0x02, 0x00, 0x1f, 0x0a, 0x0a, 0x0a, 0x02, 0x04, 0x00, 0x12, 0x04, 0x04, 0x00, 0x07, 0x01, - 0x0a, 0x0a, 0x0a, 0x03, 0x04, 0x00, 0x01, 0x12, 0x03, 0x04, 0x08, 0x12, 0x0a, 0x0b, 0x0a, 0x04, - 0x04, 0x00, 0x02, 0x00, 0x12, 0x03, 0x05, 0x02, 0x15, 0x0a, 0x0c, 0x0a, 0x05, 0x04, 0x00, 0x02, - 0x00, 0x05, 0x12, 0x03, 0x05, 0x02, 0x08, 0x0a, 0x0c, 0x0a, 0x05, 0x04, 0x00, 0x02, 0x00, 0x01, - 0x12, 0x03, 0x05, 0x09, 0x10, 0x0a, 0x0c, 0x0a, 0x05, 0x04, 0x00, 0x02, 0x00, 0x03, 0x12, 0x03, - 0x05, 0x13, 0x14, 0x0a, 0x0b, 0x0a, 0x04, 0x04, 0x00, 0x02, 0x01, 0x12, 0x03, 0x06, 0x02, 0x13, - 0x0a, 0x0c, 0x0a, 0x05, 0x04, 0x00, 0x02, 0x01, 0x05, 0x12, 0x03, 0x06, 0x02, 0x08, 0x0a, 0x0c, - 0x0a, 0x05, 0x04, 0x00, 0x02, 0x01, 0x01, 0x12, 0x03, 0x06, 0x09, 0x0e, 0x0a, 0x0c, 0x0a, 0x05, - 0x04, 0x00, 0x02, 0x01, 0x03, 0x12, 0x03, 0x06, 0x11, 0x12, 0x0a, 0x0a, 0x0a, 0x02, 0x04, 0x01, - 0x12, 0x04, 0x09, 0x00, 0x0b, 0x01, 0x0a, 0x0a, 0x0a, 0x03, 0x04, 0x01, 0x01, 0x12, 0x03, 0x09, - 0x08, 0x13, 0x0a, 0x0b, 0x0a, 0x04, 0x04, 0x01, 0x02, 0x00, 0x12, 0x03, 0x0a, 0x02, 0x20, 0x0a, - 0x0c, 0x0a, 0x05, 0x04, 0x01, 0x02, 0x00, 0x04, 0x12, 0x03, 0x0a, 0x02, 0x0a, 0x0a, 0x0c, 0x0a, - 0x05, 0x04, 0x01, 0x02, 0x00, 0x06, 0x12, 0x03, 0x0a, 0x0b, 0x15, 0x0a, 0x0c, 0x0a, 0x05, 0x04, - 0x01, 0x02, 0x00, 0x01, 0x12, 0x03, 0x0a, 0x16, 0x1b, 0x0a, 0x0c, 0x0a, 0x05, 0x04, 0x01, 0x02, - 0x00, 0x03, 0x12, 0x03, 0x0a, 0x1e, 0x1f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, -]; -// @@protoc_insertion_point(module) \ No newline at end of file diff --git a/sol-balance/substreams.yaml b/sol-balance/substreams.yaml index b62aeec9..b06fed55 100644 --- a/sol-balance/substreams.yaml +++ b/sol-balance/substreams.yaml @@ -8,7 +8,7 @@ imports: protobuf: files: - - sol.token.proto + - sol_token.proto importPaths: - ../common/proto @@ -18,10 +18,16 @@ binaries: file: "../target/wasm32-unknown-unknown/release/sol_balance.wasm" modules: - - name: map_balances - kind: map - initialBlock: 0 + - name: store_balance + kind: store + updatePolicy: set + valueType: string inputs: - source: sf.solana.type.v1.Block - output: - type: proto:messari.sol.token.Account +# - name: map_balances +# kind: map +# initialBlock: 0 +# inputs: +# - source: sf.solana.type.v1.Block +# output: +# type: proto:messari.sol.token.Account From 61d4b083888abb553dca82a67889d4045217385d Mon Sep 17 00:00:00 2001 From: dmelotik Date: Thu, 12 Jan 2023 18:39:25 -0700 Subject: [PATCH 15/36] change to map module --- common/proto/sol_token.proto | 4 ++- sol-balance/Cargo.toml | 2 +- sol-balance/src/lib.rs | 59 ++++++++++++++++++------------------ sol-balance/substreams.yaml | 16 +++------- 4 files changed, 39 insertions(+), 42 deletions(-) diff --git a/common/proto/sol_token.proto b/common/proto/sol_token.proto index 5326ad83..2032052c 100644 --- a/common/proto/sol_token.proto +++ b/common/proto/sol_token.proto @@ -31,7 +31,9 @@ message TokenAccount { message Balance { TokenAccount token = 1; - string balance = 2; + string balance = 2; // BigInt in token's native units + uint64 block_number = 3; + uint64 timestamp = 4; } message Account { diff --git a/sol-balance/Cargo.toml b/sol-balance/Cargo.toml index a390e922..455afa7a 100644 --- a/sol-balance/Cargo.toml +++ b/sol-balance/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "sol-balance" version = "0.1.0" -description = "Messari's standardized substream for wallets sol balance" +description = "Messari's substream for solana SOL balances" edition = "2021" repository = "https://github.com/messari/substreams" diff --git a/sol-balance/src/lib.rs b/sol-balance/src/lib.rs index f1f44135..72f6f4ab 100644 --- a/sol-balance/src/lib.rs +++ b/sol-balance/src/lib.rs @@ -1,36 +1,37 @@ -use crate::pb::sol as token; +use crate::pb::sol::v1 as token; // TODO: I can't get this import to work :( use bs58; -// use pb::sol::Account; use prost::Message; use substreams::store::StoreSet; use substreams_solana::pb; use substreams_solana::pb::sol as solana; -#[substreams::handlers::store] -fn store_balance(block: solana::v1::Block, output: StoreSet) { - output.set( - 0, - "block".to_string(), - &format!("{:?}", block).as_bytes().to_vec(), - ); -} +#[substreams::handlers::map] +fn map_balances(block: solana::v1::Block) -> Result { + let accounts = vec![]; + for tx in block.transactions { + if let Some(meta) = tx.meta { + if let Some(_) = meta.err { + continue; + } + for i in 0..meta.post_balances.len() { + let new_balance = &meta.post_balances[i].encode_to_vec(); + let new_token_balance = vec![token::Balance { + token: get_sol_token(), + balance: new_balance, + block_number: block.block_height.block_height, + timestamp: block.block_time.timestamp, + }]; + // TODO: Not sure if this is the right place to get the address + // AND loaded_writeable_addresses is not being built in the proto stucts + // let account_id = bs58::encode(&meta.loaded_writable_addresses[i]).into_string(); + let account = token::Account { + account_id: "0x0", + balances: new_token_balance, + }; + accounts.push(account); + } + } + } -// #[substreams::handlers::map] -// fn map(block: solana::v1::Block) -> Result { -// for tx in block.transactions { -// if let Some(meta) = tx.meta { -// if let Some(_) = meta.err { -// continue; -// } -// for i in 0..meta.post_balances.len() { -// // TODO: I don't think this is the right address, but I am struggling to figure out another way -// let account_id = bs58::encode(&meta.loaded_writable_addresses[i]).into_string(); -// output.set( -// i as u64, -// account_id, -// &meta.post_balances[i].encode_to_vec() -// ); -// } -// } -// } -// } + Ok(token::Accounts { items: accounts }) +} diff --git a/sol-balance/substreams.yaml b/sol-balance/substreams.yaml index b06fed55..4c8654a7 100644 --- a/sol-balance/substreams.yaml +++ b/sol-balance/substreams.yaml @@ -18,16 +18,10 @@ binaries: file: "../target/wasm32-unknown-unknown/release/sol_balance.wasm" modules: - - name: store_balance - kind: store - updatePolicy: set - valueType: string + - name: map_balances + kind: map + initialBlock: 0 inputs: - source: sf.solana.type.v1.Block -# - name: map_balances -# kind: map -# initialBlock: 0 -# inputs: -# - source: sf.solana.type.v1.Block -# output: -# type: proto:messari.sol.token.Account + output: + type: proto:messari.sol.token.Account From 61583d00518d4cb73c0508fa39eb2fc6a21abe3f Mon Sep 17 00:00:00 2001 From: dmelotik Date: Fri, 9 Dec 2022 18:56:18 -0700 Subject: [PATCH 16/36] initial commit on solana balances --- sol-balance/Cargo.toml | 22 ++++++++++++++++++++++ sol-balance/Makefile | 11 +++++++++++ sol-balance/proto/v1/sol_balance.proto | 12 ++++++++++++ sol-balance/rust-toolchain.toml | 4 ++++ sol-balance/src/lib.rs | 7 +++++++ sol-balance/substreams.yaml | 26 ++++++++++++++++++++++++++ 6 files changed, 82 insertions(+) create mode 100644 sol-balance/Cargo.toml create mode 100644 sol-balance/Makefile create mode 100644 sol-balance/proto/v1/sol_balance.proto create mode 100644 sol-balance/rust-toolchain.toml create mode 100644 sol-balance/src/lib.rs create mode 100644 sol-balance/substreams.yaml diff --git a/sol-balance/Cargo.toml b/sol-balance/Cargo.toml new file mode 100644 index 00000000..8a898f72 --- /dev/null +++ b/sol-balance/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "sol-balance" +version = "0.1.0" +description = "Messari's standardized substream for wallets sol balance" +edition = "2021" +repository = "https://github.com/messari/substreams" + +[lib] +crate-type = ["cdylib"] + +[dependencies] +prost = "0.11.2" +ethabi = "17.2.0" +num-bigint = "0.4" +hex-literal = "0.3.4" +substreams = { workspace = true } +substreams-solana = { workspace = true } +substreams-helper = { path = "../substreams-helper" } + +[build-dependencies] +anyhow = "1" +substreams-common = { path = "../common" } \ No newline at end of file diff --git a/sol-balance/Makefile b/sol-balance/Makefile new file mode 100644 index 00000000..151f018c --- /dev/null +++ b/sol-balance/Makefile @@ -0,0 +1,11 @@ +.PHONY: codegen +codegen: + substreams protogen ./substreams.yaml --exclude-paths="sf/solana,sf/substreams,google" + +.PHONY: build +build: + cargo build --target wasm32-unknown-unknown --release + +.PHONY: run +run: + substreams run -e mainnet.sol.streamingfast.io:443 substreams.yaml store_balance -s 0 -t +10 diff --git a/sol-balance/proto/v1/sol_balance.proto b/sol-balance/proto/v1/sol_balance.proto new file mode 100644 index 00000000..260bc705 --- /dev/null +++ b/sol-balance/proto/v1/sol_balance.proto @@ -0,0 +1,12 @@ +syntax = "proto3"; + +package messari.sol_balance.v1; + +message SolBalance { + string address = 1; + string value = 2; +} + +message SolBalances { + repeated SolBalance items = 1; +} \ No newline at end of file diff --git a/sol-balance/rust-toolchain.toml b/sol-balance/rust-toolchain.toml new file mode 100644 index 00000000..0a0b90ae --- /dev/null +++ b/sol-balance/rust-toolchain.toml @@ -0,0 +1,4 @@ +[toolchain] +channel = "1.64.0" +components = [ "rustfmt" ] +targets = [ "wasm32-unknown-unknown" ] diff --git a/sol-balance/src/lib.rs b/sol-balance/src/lib.rs new file mode 100644 index 00000000..57605217 --- /dev/null +++ b/sol-balance/src/lib.rs @@ -0,0 +1,7 @@ +use substreams::{log}; +use substreams_solana::pb::sol as solana; + +#[substreams::handlers::store] +fn store_balance(block: solana::v1::Block) { + log::info!("block height: {}", block.blockhash); +} diff --git a/sol-balance/substreams.yaml b/sol-balance/substreams.yaml new file mode 100644 index 00000000..58798322 --- /dev/null +++ b/sol-balance/substreams.yaml @@ -0,0 +1,26 @@ +specVersion: v0.1.0 +package: + name: sol_balance + version: v0.1.0 + +imports: + sol: https://github.com/streamingfast/firehose-solana/releases/download/v0.1.0/solana-v0.1.0.spkg + +protobuf: + files: + - sol_balance.proto + importPaths: + - proto/v1 + +binaries: + default: + type: wasm/rust-v1 + file: "../target/wasm32-unknown-unknown/release/sol_balance.wasm" + +modules: + - name: store_balance + kind: store + updatePolicy: set + valueType: string + inputs: + - source: sf.solana.type.v1.Block From f7f2a1780d6830cc4ba9c7d46377d9d02c9db786 Mon Sep 17 00:00:00 2001 From: dmelotik Date: Tue, 13 Dec 2022 21:29:18 -0700 Subject: [PATCH 17/36] WIP: trying to get accounts linked to balances... --- sol-balance/Cargo.toml | 3 +- sol-balance/Makefile | 2 +- sol-balance/proto/v1/sol_balance.proto | 2 +- sol-balance/src/lib.rs | 44 ++++++++++++++++++++++++-- 4 files changed, 45 insertions(+), 6 deletions(-) diff --git a/sol-balance/Cargo.toml b/sol-balance/Cargo.toml index 8a898f72..689d2ab5 100644 --- a/sol-balance/Cargo.toml +++ b/sol-balance/Cargo.toml @@ -9,11 +9,12 @@ repository = "https://github.com/messari/substreams" crate-type = ["cdylib"] [dependencies] +bs58 = "0.4.0" prost = "0.11.2" ethabi = "17.2.0" num-bigint = "0.4" hex-literal = "0.3.4" -substreams = { workspace = true } +substreams = "0.0.11" substreams-solana = { workspace = true } substreams-helper = { path = "../substreams-helper" } diff --git a/sol-balance/Makefile b/sol-balance/Makefile index 151f018c..adcdf3e9 100644 --- a/sol-balance/Makefile +++ b/sol-balance/Makefile @@ -8,4 +8,4 @@ build: .PHONY: run run: - substreams run -e mainnet.sol.streamingfast.io:443 substreams.yaml store_balance -s 0 -t +10 + substreams run -e mainnet.sol.streamingfast.io:443 substreams.yaml store_balance -s 1000 -t +10 diff --git a/sol-balance/proto/v1/sol_balance.proto b/sol-balance/proto/v1/sol_balance.proto index 260bc705..8442a338 100644 --- a/sol-balance/proto/v1/sol_balance.proto +++ b/sol-balance/proto/v1/sol_balance.proto @@ -9,4 +9,4 @@ message SolBalance { message SolBalances { repeated SolBalance items = 1; -} \ No newline at end of file +} diff --git a/sol-balance/src/lib.rs b/sol-balance/src/lib.rs index 57605217..89c86ca5 100644 --- a/sol-balance/src/lib.rs +++ b/sol-balance/src/lib.rs @@ -1,7 +1,45 @@ -use substreams::{log}; +use substreams::log; +use substreams::store::StoreSet; use substreams_solana::pb::sol as solana; +// use bs58; #[substreams::handlers::store] -fn store_balance(block: solana::v1::Block) { - log::info!("block height: {}", block.blockhash); +fn store_balance(block: solana::v1::Block, output: StoreSet) { + log::info!("block hash: {}", block.previous_blockhash); + log::info!("block hash: {}", block.parent_slot); + + // TODO: try to get balances in this for loop + + for tx in block.transactions { + if let Some(meta) = tx.meta { + if let Some(_) = meta.err { + continue; + } + log::info!("bal len: {}", meta.post_balances.len()); + + for bal in meta.post_balances { + // TODO: I don't think this is the right address, but I am struggling to figure out another way + + let change_address = + + output.set( + 0, + + ) + } + // log::info!("{}", meta.log_messages_none); + // for bal in meta.post_balances { + // log::info!("post bal: {}", bal); + // } + if let Some(transaction) = tx.transaction { + if let Some(msg) = transaction.message { + log::info!("acct len: {}", msg.account_keys.len()); + for accts in msg.account_keys { + // let program_id = &msg.account_keys[inst.program_id_index as usize]; + // log::info!("acc: {}", bs58::encode(program_id).into_string()); + } + } + } + } + } } From 43ccb3b04c73cd60e235bda4b9994b08667bdd2b Mon Sep 17 00:00:00 2001 From: dmelotik Date: Tue, 13 Dec 2022 21:48:49 -0700 Subject: [PATCH 18/36] update substreams-solana --- sol-balance/src/lib.rs | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/sol-balance/src/lib.rs b/sol-balance/src/lib.rs index 89c86ca5..383698f6 100644 --- a/sol-balance/src/lib.rs +++ b/sol-balance/src/lib.rs @@ -20,14 +20,15 @@ fn store_balance(block: solana::v1::Block, output: StoreSet) { for bal in meta.post_balances { // TODO: I don't think this is the right address, but I am struggling to figure out another way - let change_address = - - output.set( - 0, - - ) + // let change_address = + // + // output.set( + // 0, + // + // ) } - // log::info!("{}", meta.log_messages_none); + log::info!("{}", meta.loaded_writable_addresses.len()); + log::info!("{}", meta.post_balances.len()); // for bal in meta.post_balances { // log::info!("post bal: {}", bal); // } From 618e52efa397645da58c9839bd7d8a0815f8540d Mon Sep 17 00:00:00 2001 From: dmelotik Date: Wed, 14 Dec 2022 18:33:39 -0700 Subject: [PATCH 19/36] set output --- sol-balance/Cargo.toml | 2 +- sol-balance/Makefile | 2 +- sol-balance/src/lib.rs | 39 +++++++++------------------------------ 3 files changed, 11 insertions(+), 32 deletions(-) diff --git a/sol-balance/Cargo.toml b/sol-balance/Cargo.toml index 689d2ab5..a390e922 100644 --- a/sol-balance/Cargo.toml +++ b/sol-balance/Cargo.toml @@ -15,7 +15,7 @@ ethabi = "17.2.0" num-bigint = "0.4" hex-literal = "0.3.4" substreams = "0.0.11" -substreams-solana = { workspace = true } +substreams-solana = "0.1.0" substreams-helper = { path = "../substreams-helper" } [build-dependencies] diff --git a/sol-balance/Makefile b/sol-balance/Makefile index adcdf3e9..fad4489a 100644 --- a/sol-balance/Makefile +++ b/sol-balance/Makefile @@ -8,4 +8,4 @@ build: .PHONY: run run: - substreams run -e mainnet.sol.streamingfast.io:443 substreams.yaml store_balance -s 1000 -t +10 + substreams run -e mainnet.sol.streamingfast.io:443 substreams.yaml store_balance -s 10000 -t +100 diff --git a/sol-balance/src/lib.rs b/sol-balance/src/lib.rs index 383698f6..98d3315b 100644 --- a/sol-balance/src/lib.rs +++ b/sol-balance/src/lib.rs @@ -1,45 +1,24 @@ use substreams::log; use substreams::store::StoreSet; use substreams_solana::pb::sol as solana; -// use bs58; +use bs58; #[substreams::handlers::store] fn store_balance(block: solana::v1::Block, output: StoreSet) { - log::info!("block hash: {}", block.previous_blockhash); - log::info!("block hash: {}", block.parent_slot); - - // TODO: try to get balances in this for loop - for tx in block.transactions { if let Some(meta) = tx.meta { if let Some(_) = meta.err { continue; } - log::info!("bal len: {}", meta.post_balances.len()); - - for bal in meta.post_balances { + log::info!("bal len: {}", &meta.post_balances.len()); // TODO: remove + for i in 0..meta.post_balances.len() { // TODO: I don't think this is the right address, but I am struggling to figure out another way - - // let change_address = - // - // output.set( - // 0, - // - // ) - } - log::info!("{}", meta.loaded_writable_addresses.len()); - log::info!("{}", meta.post_balances.len()); - // for bal in meta.post_balances { - // log::info!("post bal: {}", bal); - // } - if let Some(transaction) = tx.transaction { - if let Some(msg) = transaction.message { - log::info!("acct len: {}", msg.account_keys.len()); - for accts in msg.account_keys { - // let program_id = &msg.account_keys[inst.program_id_index as usize]; - // log::info!("acc: {}", bs58::encode(program_id).into_string()); - } - } + let account_id = bs58::encode(&meta.loaded_writable_addresses[i]).into_string(); + output.set( + i as u64, + account_id, + &meta.post_balances[i] // TODO: not sure why this wants a &Vec + ); } } } From 9ed710ca5df715e3232eefca5fbf47959a1754ef Mon Sep 17 00:00:00 2001 From: dmelotik Date: Wed, 14 Dec 2022 21:38:15 -0700 Subject: [PATCH 20/36] add README --- sol-balance/README.md | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 sol-balance/README.md diff --git a/sol-balance/README.md b/sol-balance/README.md new file mode 100644 index 00000000..e69de29b From 6cce6f6095619af0054e23d8fec5baf9280b3bfd Mon Sep 17 00:00:00 2001 From: dmelotik Date: Wed, 14 Dec 2022 21:38:46 -0700 Subject: [PATCH 21/36] add README --- sol-balance/README.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/sol-balance/README.md b/sol-balance/README.md index e69de29b..3c784e51 100644 --- a/sol-balance/README.md +++ b/sol-balance/README.md @@ -0,0 +1,10 @@ +# SOL Balance Substream + +This substream is designed to store the SOL balance of every account. + +### Build & Run + +```bash +make codegen +make run +``` From 1b9360634602c5ccc86581a9e2371cba7588d544 Mon Sep 17 00:00:00 2001 From: dmelotik Date: Fri, 16 Dec 2022 00:27:19 -0700 Subject: [PATCH 22/36] Add proper commands for sol-balance --- erc20-holdings/README.md | 0 sol-balance/src/pb/messari.sol_balance.v1.rs | 45 ++++++++++++++++++++ 2 files changed, 45 insertions(+) create mode 100644 erc20-holdings/README.md create mode 100644 sol-balance/src/pb/messari.sol_balance.v1.rs diff --git a/erc20-holdings/README.md b/erc20-holdings/README.md new file mode 100644 index 00000000..e69de29b diff --git a/sol-balance/src/pb/messari.sol_balance.v1.rs b/sol-balance/src/pb/messari.sol_balance.v1.rs new file mode 100644 index 00000000..3694a7ee --- /dev/null +++ b/sol-balance/src/pb/messari.sol_balance.v1.rs @@ -0,0 +1,45 @@ +// @generated +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct SolBalance { + #[prost(string, tag="1")] + pub address: ::prost::alloc::string::String, + #[prost(string, tag="2")] + pub value: ::prost::alloc::string::String, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct SolBalances { + #[prost(message, repeated, tag="1")] + pub items: ::prost::alloc::vec::Vec, +} +/// Encoded file descriptor set for the `messari.sol_balance.v1` package +pub const FILE_DESCRIPTOR_SET: &[u8] = &[ + 0x0a, 0xbc, 0x03, 0x0a, 0x11, 0x73, 0x6f, 0x6c, 0x5f, 0x62, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, + 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x16, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x72, 0x69, 0x2e, + 0x73, 0x6f, 0x6c, 0x5f, 0x62, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x22, 0x3c, + 0x0a, 0x0a, 0x53, 0x6f, 0x6c, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x12, 0x18, 0x0a, 0x07, + 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x61, + 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x22, 0x47, 0x0a, 0x0b, + 0x53, 0x6f, 0x6c, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x73, 0x12, 0x38, 0x0a, 0x05, 0x69, + 0x74, 0x65, 0x6d, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x6d, 0x65, 0x73, + 0x73, 0x61, 0x72, 0x69, 0x2e, 0x73, 0x6f, 0x6c, 0x5f, 0x62, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, + 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x6f, 0x6c, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x52, 0x05, + 0x69, 0x74, 0x65, 0x6d, 0x73, 0x4a, 0xff, 0x01, 0x0a, 0x06, 0x12, 0x04, 0x00, 0x00, 0x0b, 0x01, + 0x0a, 0x08, 0x0a, 0x01, 0x0c, 0x12, 0x03, 0x00, 0x00, 0x12, 0x0a, 0x08, 0x0a, 0x01, 0x02, 0x12, + 0x03, 0x02, 0x00, 0x1f, 0x0a, 0x0a, 0x0a, 0x02, 0x04, 0x00, 0x12, 0x04, 0x04, 0x00, 0x07, 0x01, + 0x0a, 0x0a, 0x0a, 0x03, 0x04, 0x00, 0x01, 0x12, 0x03, 0x04, 0x08, 0x12, 0x0a, 0x0b, 0x0a, 0x04, + 0x04, 0x00, 0x02, 0x00, 0x12, 0x03, 0x05, 0x02, 0x15, 0x0a, 0x0c, 0x0a, 0x05, 0x04, 0x00, 0x02, + 0x00, 0x05, 0x12, 0x03, 0x05, 0x02, 0x08, 0x0a, 0x0c, 0x0a, 0x05, 0x04, 0x00, 0x02, 0x00, 0x01, + 0x12, 0x03, 0x05, 0x09, 0x10, 0x0a, 0x0c, 0x0a, 0x05, 0x04, 0x00, 0x02, 0x00, 0x03, 0x12, 0x03, + 0x05, 0x13, 0x14, 0x0a, 0x0b, 0x0a, 0x04, 0x04, 0x00, 0x02, 0x01, 0x12, 0x03, 0x06, 0x02, 0x13, + 0x0a, 0x0c, 0x0a, 0x05, 0x04, 0x00, 0x02, 0x01, 0x05, 0x12, 0x03, 0x06, 0x02, 0x08, 0x0a, 0x0c, + 0x0a, 0x05, 0x04, 0x00, 0x02, 0x01, 0x01, 0x12, 0x03, 0x06, 0x09, 0x0e, 0x0a, 0x0c, 0x0a, 0x05, + 0x04, 0x00, 0x02, 0x01, 0x03, 0x12, 0x03, 0x06, 0x11, 0x12, 0x0a, 0x0a, 0x0a, 0x02, 0x04, 0x01, + 0x12, 0x04, 0x09, 0x00, 0x0b, 0x01, 0x0a, 0x0a, 0x0a, 0x03, 0x04, 0x01, 0x01, 0x12, 0x03, 0x09, + 0x08, 0x13, 0x0a, 0x0b, 0x0a, 0x04, 0x04, 0x01, 0x02, 0x00, 0x12, 0x03, 0x0a, 0x02, 0x20, 0x0a, + 0x0c, 0x0a, 0x05, 0x04, 0x01, 0x02, 0x00, 0x04, 0x12, 0x03, 0x0a, 0x02, 0x0a, 0x0a, 0x0c, 0x0a, + 0x05, 0x04, 0x01, 0x02, 0x00, 0x06, 0x12, 0x03, 0x0a, 0x0b, 0x15, 0x0a, 0x0c, 0x0a, 0x05, 0x04, + 0x01, 0x02, 0x00, 0x01, 0x12, 0x03, 0x0a, 0x16, 0x1b, 0x0a, 0x0c, 0x0a, 0x05, 0x04, 0x01, 0x02, + 0x00, 0x03, 0x12, 0x03, 0x0a, 0x1e, 0x1f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +]; +// @@protoc_insertion_point(module) \ No newline at end of file From dba9cf564c94ebad95607af0a355216df1814d44 Mon Sep 17 00:00:00 2001 From: dmelotik Date: Thu, 12 Jan 2023 18:54:43 -0700 Subject: [PATCH 23/36] rebase merge --- Cargo.lock | 22 ++++++++++++++++++++++ Cargo.toml | 3 ++- Makefile | 2 ++ 3 files changed, 26 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index c0db0194..c0932858 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -76,6 +76,12 @@ dependencies = [ "generic-array", ] +[[package]] +name = "bs58" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "771fe0050b883fcc3ea2359b1a96bcfbc090b7116eae7c3c512c7a083fdf23d3" + [[package]] name = "bumpalo" version = "3.11.0" @@ -914,6 +920,22 @@ dependencies = [ "keccak", ] +[[package]] +name = "sol-balance" +version = "0.1.0" +dependencies = [ + "anyhow", + "bs58", + "ethabi", + "hex-literal", + "num-bigint", + "prost 0.11.2", + "substreams 0.0.11", + "substreams-common", + "substreams-helper", + "substreams-solana", +] + [[package]] name = "solana-sample" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 5a1bd6a3..23899b1e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,6 +11,7 @@ members = [ "substreams-helper", "uniswap-v2", "solana-sample", + "sol-balance", "ens-names", ] exclude = ["messari-cli"] @@ -18,7 +19,7 @@ exclude = ["messari-cli"] [workspace.dependencies] substreams = "0.3.2" substreams-ethereum = "0.6.2" -substreams-solana = "0.1.0" +substreams-solana = "0.2.0" [build] target = "wasm32-unknown-unknown" diff --git a/Makefile b/Makefile index 150c34c2..3af031e4 100644 --- a/Makefile +++ b/Makefile @@ -10,6 +10,7 @@ build-all: $(MAKE) -C network build $(MAKE) -C solana-sample build $(MAKE) -C eth-balance build + $(MAKE) -C sol-balance build $(MAKE) -C ens-names build .PHONY: run-all @@ -19,6 +20,7 @@ run-all: $(MAKE) -C erc20-market-cap run $(MAKE) -C erc721 run $(MAKE) -C network run + $(MAKE) -C sol-balance run $(MAKE) -C ens-names run .PHONY: test From 3ce1bf69d4598e48ace57efa3aa41cc19641f60b Mon Sep 17 00:00:00 2001 From: dmelotik Date: Thu, 22 Dec 2022 11:05:05 -0700 Subject: [PATCH 24/36] encode to vec and update readmes --- README.md | 1 + sol-balance/README.md | 5 +++++ sol-balance/src/lib.rs | 5 ++--- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 8704de8e..0501edd1 100644 --- a/README.md +++ b/README.md @@ -43,6 +43,7 @@ | ENS Look Up | 🔨 | ENS records for lookup and reverse lookup | | Uniswap v2 | 🔨 | Substreams for Uniswap v2 | | Compound v2 | 🔨 | Substreams for Compound v2 | +| SOL Balance | 🛠 | SOL balance for every Solana address | ## Workflow diff --git a/sol-balance/README.md b/sol-balance/README.md index 3c784e51..ac707b94 100644 --- a/sol-balance/README.md +++ b/sol-balance/README.md @@ -8,3 +8,8 @@ This substream is designed to store the SOL balance of every account. make codegen make run ``` + +### Notes + +- Currently [`transaction.loaded_writable_addresses`](https://github.com/streamingfast/firehose-solana/blob/develop/proto/sf/solana/type/v1/type.proto#L65) is an unknown field on firesol blocks. +- diff --git a/sol-balance/src/lib.rs b/sol-balance/src/lib.rs index 98d3315b..b351adbb 100644 --- a/sol-balance/src/lib.rs +++ b/sol-balance/src/lib.rs @@ -1,7 +1,7 @@ -use substreams::log; use substreams::store::StoreSet; use substreams_solana::pb::sol as solana; use bs58; +use prost::Message; #[substreams::handlers::store] fn store_balance(block: solana::v1::Block, output: StoreSet) { @@ -10,14 +10,13 @@ fn store_balance(block: solana::v1::Block, output: StoreSet) { if let Some(_) = meta.err { continue; } - log::info!("bal len: {}", &meta.post_balances.len()); // TODO: remove for i in 0..meta.post_balances.len() { // TODO: I don't think this is the right address, but I am struggling to figure out another way let account_id = bs58::encode(&meta.loaded_writable_addresses[i]).into_string(); output.set( i as u64, account_id, - &meta.post_balances[i] // TODO: not sure why this wants a &Vec + &meta.post_balances[i].encode_to_vec() ); } } From 482a24a7255b3622e634d705285da607d09be086 Mon Sep 17 00:00:00 2001 From: dmelotik Date: Fri, 6 Jan 2023 20:36:54 -0700 Subject: [PATCH 25/36] add sol token proto to common --- common/proto/sol.token.proto | 44 ++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 common/proto/sol.token.proto diff --git a/common/proto/sol.token.proto b/common/proto/sol.token.proto new file mode 100644 index 00000000..02b4417c --- /dev/null +++ b/common/proto/sol.token.proto @@ -0,0 +1,44 @@ +syntax = "proto3"; + +package messari.sol.token.v1; + +message Transfers { + repeated Transfer transfers = 1; +} + +message Transfer { + string signature = 1; + uint64 ordinal = 2; + string from = 3; + string to = 4; + TokenAccount token = 5; + uint64 native_amount = 6; + uint64 amount = 7; +} + +message Tokens { + repeated TokenAccount tokens = 1; +} + +message TokenAccount { + string address = 1; + string name = 2; + string symbol = 3; + string decimals = 4; + string owner = 5; + string mint = 6; +} + +message Balance { + TokenAccount token = 1; + string balance = 2; +} + +message Account { + string account_id = 1; + repeated Balance balances = 2; +} + +message Accounts { + repeated Account items = 1; +} From 05b67e401480be7083669c8dc2f20a80405a869d Mon Sep 17 00:00:00 2001 From: dmelotik Date: Fri, 6 Jan 2023 20:54:54 -0700 Subject: [PATCH 26/36] update to map module --- sol-balance/proto/v1/sol_balance.proto | 12 ------------ sol-balance/substreams.yaml | 13 +++++++------ 2 files changed, 7 insertions(+), 18 deletions(-) delete mode 100644 sol-balance/proto/v1/sol_balance.proto diff --git a/sol-balance/proto/v1/sol_balance.proto b/sol-balance/proto/v1/sol_balance.proto deleted file mode 100644 index 8442a338..00000000 --- a/sol-balance/proto/v1/sol_balance.proto +++ /dev/null @@ -1,12 +0,0 @@ -syntax = "proto3"; - -package messari.sol_balance.v1; - -message SolBalance { - string address = 1; - string value = 2; -} - -message SolBalances { - repeated SolBalance items = 1; -} diff --git a/sol-balance/substreams.yaml b/sol-balance/substreams.yaml index 58798322..b62aeec9 100644 --- a/sol-balance/substreams.yaml +++ b/sol-balance/substreams.yaml @@ -8,9 +8,9 @@ imports: protobuf: files: - - sol_balance.proto + - sol.token.proto importPaths: - - proto/v1 + - ../common/proto binaries: default: @@ -18,9 +18,10 @@ binaries: file: "../target/wasm32-unknown-unknown/release/sol_balance.wasm" modules: - - name: store_balance - kind: store - updatePolicy: set - valueType: string + - name: map_balances + kind: map + initialBlock: 0 inputs: - source: sf.solana.type.v1.Block + output: + type: proto:messari.sol.token.Account From b3215797734f0eb7868c7a1d900f9d6f9bf604ff Mon Sep 17 00:00:00 2001 From: dmelotik Date: Fri, 6 Jan 2023 21:01:52 -0700 Subject: [PATCH 27/36] begin map handler --- sol-balance/src/lib.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/sol-balance/src/lib.rs b/sol-balance/src/lib.rs index b351adbb..c3b06b72 100644 --- a/sol-balance/src/lib.rs +++ b/sol-balance/src/lib.rs @@ -1,10 +1,13 @@ use substreams::store::StoreSet; +use crate::pb::token::v1::Token; +use pb::token::v1 as proto; use substreams_solana::pb::sol as solana; use bs58; use prost::Message; +use substreams_solana::pb; -#[substreams::handlers::store] -fn store_balance(block: solana::v1::Block, output: StoreSet) { +#[substreams::handlers::map] +fn map(block: solana::v1::Block) -> Result { for tx in block.transactions { if let Some(meta) = tx.meta { if let Some(_) = meta.err { From 2497eaefc025fe9f143bad30353f49cb62feccfa Mon Sep 17 00:00:00 2001 From: dmelotik Date: Thu, 12 Jan 2023 12:38:05 -0700 Subject: [PATCH 28/36] fix sol balance build setup --- .../{sol.token.proto => sol_token.proto} | 4 ++ sol-balance/Makefile | 6 +-- sol-balance/build.rs | 10 ++++ sol-balance/src/abi.rs | 1 + sol-balance/src/lib.rs | 53 +++++++++++-------- sol-balance/src/pb.rs | 19 +++++++ sol-balance/src/pb/messari.sol_balance.v1.rs | 45 ---------------- sol-balance/substreams.yaml | 18 ++++--- 8 files changed, 78 insertions(+), 78 deletions(-) rename common/proto/{sol.token.proto => sol_token.proto} (93%) create mode 100644 sol-balance/build.rs create mode 100644 sol-balance/src/abi.rs create mode 100644 sol-balance/src/pb.rs delete mode 100644 sol-balance/src/pb/messari.sol_balance.v1.rs diff --git a/common/proto/sol.token.proto b/common/proto/sol_token.proto similarity index 93% rename from common/proto/sol.token.proto rename to common/proto/sol_token.proto index 02b4417c..5326ad83 100644 --- a/common/proto/sol.token.proto +++ b/common/proto/sol_token.proto @@ -42,3 +42,7 @@ message Account { message Accounts { repeated Account items = 1; } + +message OuterString { + string inner_string = 1; +} diff --git a/sol-balance/Makefile b/sol-balance/Makefile index fad4489a..2f559204 100644 --- a/sol-balance/Makefile +++ b/sol-balance/Makefile @@ -1,11 +1,7 @@ -.PHONY: codegen -codegen: - substreams protogen ./substreams.yaml --exclude-paths="sf/solana,sf/substreams,google" - .PHONY: build build: cargo build --target wasm32-unknown-unknown --release .PHONY: run run: - substreams run -e mainnet.sol.streamingfast.io:443 substreams.yaml store_balance -s 10000 -t +100 + substreams run -e mainnet.sol.streamingfast.io:443 substreams.yaml store_balance -s 10000 -t +10001 diff --git a/sol-balance/build.rs b/sol-balance/build.rs new file mode 100644 index 00000000..fb321d31 --- /dev/null +++ b/sol-balance/build.rs @@ -0,0 +1,10 @@ +use anyhow::{Ok, Result}; +use substreams_common::codegen; + +fn main() -> Result<(), anyhow::Error> { + println!("cargo:rerun-if-changed=proto"); + println!("cargo:rerun-if-changed=abi"); + codegen::generate(None)?; + + Ok(()) +} diff --git a/sol-balance/src/abi.rs b/sol-balance/src/abi.rs new file mode 100644 index 00000000..d55d99b3 --- /dev/null +++ b/sol-balance/src/abi.rs @@ -0,0 +1 @@ +// DO NOT EDIT - the file is generated by build script diff --git a/sol-balance/src/lib.rs b/sol-balance/src/lib.rs index c3b06b72..f1f44135 100644 --- a/sol-balance/src/lib.rs +++ b/sol-balance/src/lib.rs @@ -1,27 +1,36 @@ -use substreams::store::StoreSet; -use crate::pb::token::v1::Token; -use pb::token::v1 as proto; -use substreams_solana::pb::sol as solana; +use crate::pb::sol as token; use bs58; +// use pb::sol::Account; use prost::Message; +use substreams::store::StoreSet; use substreams_solana::pb; +use substreams_solana::pb::sol as solana; -#[substreams::handlers::map] -fn map(block: solana::v1::Block) -> Result { - for tx in block.transactions { - if let Some(meta) = tx.meta { - if let Some(_) = meta.err { - continue; - } - for i in 0..meta.post_balances.len() { - // TODO: I don't think this is the right address, but I am struggling to figure out another way - let account_id = bs58::encode(&meta.loaded_writable_addresses[i]).into_string(); - output.set( - i as u64, - account_id, - &meta.post_balances[i].encode_to_vec() - ); - } - } - } +#[substreams::handlers::store] +fn store_balance(block: solana::v1::Block, output: StoreSet) { + output.set( + 0, + "block".to_string(), + &format!("{:?}", block).as_bytes().to_vec(), + ); } + +// #[substreams::handlers::map] +// fn map(block: solana::v1::Block) -> Result { +// for tx in block.transactions { +// if let Some(meta) = tx.meta { +// if let Some(_) = meta.err { +// continue; +// } +// for i in 0..meta.post_balances.len() { +// // TODO: I don't think this is the right address, but I am struggling to figure out another way +// let account_id = bs58::encode(&meta.loaded_writable_addresses[i]).into_string(); +// output.set( +// i as u64, +// account_id, +// &meta.post_balances[i].encode_to_vec() +// ); +// } +// } +// } +// } diff --git a/sol-balance/src/pb.rs b/sol-balance/src/pb.rs new file mode 100644 index 00000000..dc8da007 --- /dev/null +++ b/sol-balance/src/pb.rs @@ -0,0 +1,19 @@ +#[rustfmt::skip] +#[path = "../target/pb/messari.sol.token.rs"] +pub(in crate::pb) mod sol_token; + +pub mod sol { + pub mod token { + pub use super::super::sol_token::*; + } +} + +#[rustfmt::skip] +#[path = "../target/pb/messari.solana.type.rs"] +pub(in crate::pb) mod solana_type; + +pub mod solana { + pub mod type { + pub use super::super::solana_type::*; + } +} diff --git a/sol-balance/src/pb/messari.sol_balance.v1.rs b/sol-balance/src/pb/messari.sol_balance.v1.rs deleted file mode 100644 index 3694a7ee..00000000 --- a/sol-balance/src/pb/messari.sol_balance.v1.rs +++ /dev/null @@ -1,45 +0,0 @@ -// @generated -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct SolBalance { - #[prost(string, tag="1")] - pub address: ::prost::alloc::string::String, - #[prost(string, tag="2")] - pub value: ::prost::alloc::string::String, -} -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct SolBalances { - #[prost(message, repeated, tag="1")] - pub items: ::prost::alloc::vec::Vec, -} -/// Encoded file descriptor set for the `messari.sol_balance.v1` package -pub const FILE_DESCRIPTOR_SET: &[u8] = &[ - 0x0a, 0xbc, 0x03, 0x0a, 0x11, 0x73, 0x6f, 0x6c, 0x5f, 0x62, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, - 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x16, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x72, 0x69, 0x2e, - 0x73, 0x6f, 0x6c, 0x5f, 0x62, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x22, 0x3c, - 0x0a, 0x0a, 0x53, 0x6f, 0x6c, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x12, 0x18, 0x0a, 0x07, - 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x61, - 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x22, 0x47, 0x0a, 0x0b, - 0x53, 0x6f, 0x6c, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x73, 0x12, 0x38, 0x0a, 0x05, 0x69, - 0x74, 0x65, 0x6d, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x6d, 0x65, 0x73, - 0x73, 0x61, 0x72, 0x69, 0x2e, 0x73, 0x6f, 0x6c, 0x5f, 0x62, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, - 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x6f, 0x6c, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x52, 0x05, - 0x69, 0x74, 0x65, 0x6d, 0x73, 0x4a, 0xff, 0x01, 0x0a, 0x06, 0x12, 0x04, 0x00, 0x00, 0x0b, 0x01, - 0x0a, 0x08, 0x0a, 0x01, 0x0c, 0x12, 0x03, 0x00, 0x00, 0x12, 0x0a, 0x08, 0x0a, 0x01, 0x02, 0x12, - 0x03, 0x02, 0x00, 0x1f, 0x0a, 0x0a, 0x0a, 0x02, 0x04, 0x00, 0x12, 0x04, 0x04, 0x00, 0x07, 0x01, - 0x0a, 0x0a, 0x0a, 0x03, 0x04, 0x00, 0x01, 0x12, 0x03, 0x04, 0x08, 0x12, 0x0a, 0x0b, 0x0a, 0x04, - 0x04, 0x00, 0x02, 0x00, 0x12, 0x03, 0x05, 0x02, 0x15, 0x0a, 0x0c, 0x0a, 0x05, 0x04, 0x00, 0x02, - 0x00, 0x05, 0x12, 0x03, 0x05, 0x02, 0x08, 0x0a, 0x0c, 0x0a, 0x05, 0x04, 0x00, 0x02, 0x00, 0x01, - 0x12, 0x03, 0x05, 0x09, 0x10, 0x0a, 0x0c, 0x0a, 0x05, 0x04, 0x00, 0x02, 0x00, 0x03, 0x12, 0x03, - 0x05, 0x13, 0x14, 0x0a, 0x0b, 0x0a, 0x04, 0x04, 0x00, 0x02, 0x01, 0x12, 0x03, 0x06, 0x02, 0x13, - 0x0a, 0x0c, 0x0a, 0x05, 0x04, 0x00, 0x02, 0x01, 0x05, 0x12, 0x03, 0x06, 0x02, 0x08, 0x0a, 0x0c, - 0x0a, 0x05, 0x04, 0x00, 0x02, 0x01, 0x01, 0x12, 0x03, 0x06, 0x09, 0x0e, 0x0a, 0x0c, 0x0a, 0x05, - 0x04, 0x00, 0x02, 0x01, 0x03, 0x12, 0x03, 0x06, 0x11, 0x12, 0x0a, 0x0a, 0x0a, 0x02, 0x04, 0x01, - 0x12, 0x04, 0x09, 0x00, 0x0b, 0x01, 0x0a, 0x0a, 0x0a, 0x03, 0x04, 0x01, 0x01, 0x12, 0x03, 0x09, - 0x08, 0x13, 0x0a, 0x0b, 0x0a, 0x04, 0x04, 0x01, 0x02, 0x00, 0x12, 0x03, 0x0a, 0x02, 0x20, 0x0a, - 0x0c, 0x0a, 0x05, 0x04, 0x01, 0x02, 0x00, 0x04, 0x12, 0x03, 0x0a, 0x02, 0x0a, 0x0a, 0x0c, 0x0a, - 0x05, 0x04, 0x01, 0x02, 0x00, 0x06, 0x12, 0x03, 0x0a, 0x0b, 0x15, 0x0a, 0x0c, 0x0a, 0x05, 0x04, - 0x01, 0x02, 0x00, 0x01, 0x12, 0x03, 0x0a, 0x16, 0x1b, 0x0a, 0x0c, 0x0a, 0x05, 0x04, 0x01, 0x02, - 0x00, 0x03, 0x12, 0x03, 0x0a, 0x1e, 0x1f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, -]; -// @@protoc_insertion_point(module) \ No newline at end of file diff --git a/sol-balance/substreams.yaml b/sol-balance/substreams.yaml index b62aeec9..b06fed55 100644 --- a/sol-balance/substreams.yaml +++ b/sol-balance/substreams.yaml @@ -8,7 +8,7 @@ imports: protobuf: files: - - sol.token.proto + - sol_token.proto importPaths: - ../common/proto @@ -18,10 +18,16 @@ binaries: file: "../target/wasm32-unknown-unknown/release/sol_balance.wasm" modules: - - name: map_balances - kind: map - initialBlock: 0 + - name: store_balance + kind: store + updatePolicy: set + valueType: string inputs: - source: sf.solana.type.v1.Block - output: - type: proto:messari.sol.token.Account +# - name: map_balances +# kind: map +# initialBlock: 0 +# inputs: +# - source: sf.solana.type.v1.Block +# output: +# type: proto:messari.sol.token.Account From 24753daa7ba34bcff609078b893db774445b4fdc Mon Sep 17 00:00:00 2001 From: dmelotik Date: Thu, 12 Jan 2023 18:39:25 -0700 Subject: [PATCH 29/36] change to map module --- common/proto/sol_token.proto | 4 ++- sol-balance/Cargo.toml | 2 +- sol-balance/src/lib.rs | 59 ++++++++++++++++++------------------ sol-balance/substreams.yaml | 16 +++------- 4 files changed, 39 insertions(+), 42 deletions(-) diff --git a/common/proto/sol_token.proto b/common/proto/sol_token.proto index 5326ad83..2032052c 100644 --- a/common/proto/sol_token.proto +++ b/common/proto/sol_token.proto @@ -31,7 +31,9 @@ message TokenAccount { message Balance { TokenAccount token = 1; - string balance = 2; + string balance = 2; // BigInt in token's native units + uint64 block_number = 3; + uint64 timestamp = 4; } message Account { diff --git a/sol-balance/Cargo.toml b/sol-balance/Cargo.toml index a390e922..455afa7a 100644 --- a/sol-balance/Cargo.toml +++ b/sol-balance/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "sol-balance" version = "0.1.0" -description = "Messari's standardized substream for wallets sol balance" +description = "Messari's substream for solana SOL balances" edition = "2021" repository = "https://github.com/messari/substreams" diff --git a/sol-balance/src/lib.rs b/sol-balance/src/lib.rs index f1f44135..72f6f4ab 100644 --- a/sol-balance/src/lib.rs +++ b/sol-balance/src/lib.rs @@ -1,36 +1,37 @@ -use crate::pb::sol as token; +use crate::pb::sol::v1 as token; // TODO: I can't get this import to work :( use bs58; -// use pb::sol::Account; use prost::Message; use substreams::store::StoreSet; use substreams_solana::pb; use substreams_solana::pb::sol as solana; -#[substreams::handlers::store] -fn store_balance(block: solana::v1::Block, output: StoreSet) { - output.set( - 0, - "block".to_string(), - &format!("{:?}", block).as_bytes().to_vec(), - ); -} +#[substreams::handlers::map] +fn map_balances(block: solana::v1::Block) -> Result { + let accounts = vec![]; + for tx in block.transactions { + if let Some(meta) = tx.meta { + if let Some(_) = meta.err { + continue; + } + for i in 0..meta.post_balances.len() { + let new_balance = &meta.post_balances[i].encode_to_vec(); + let new_token_balance = vec![token::Balance { + token: get_sol_token(), + balance: new_balance, + block_number: block.block_height.block_height, + timestamp: block.block_time.timestamp, + }]; + // TODO: Not sure if this is the right place to get the address + // AND loaded_writeable_addresses is not being built in the proto stucts + // let account_id = bs58::encode(&meta.loaded_writable_addresses[i]).into_string(); + let account = token::Account { + account_id: "0x0", + balances: new_token_balance, + }; + accounts.push(account); + } + } + } -// #[substreams::handlers::map] -// fn map(block: solana::v1::Block) -> Result { -// for tx in block.transactions { -// if let Some(meta) = tx.meta { -// if let Some(_) = meta.err { -// continue; -// } -// for i in 0..meta.post_balances.len() { -// // TODO: I don't think this is the right address, but I am struggling to figure out another way -// let account_id = bs58::encode(&meta.loaded_writable_addresses[i]).into_string(); -// output.set( -// i as u64, -// account_id, -// &meta.post_balances[i].encode_to_vec() -// ); -// } -// } -// } -// } + Ok(token::Accounts { items: accounts }) +} diff --git a/sol-balance/substreams.yaml b/sol-balance/substreams.yaml index b06fed55..4c8654a7 100644 --- a/sol-balance/substreams.yaml +++ b/sol-balance/substreams.yaml @@ -18,16 +18,10 @@ binaries: file: "../target/wasm32-unknown-unknown/release/sol_balance.wasm" modules: - - name: store_balance - kind: store - updatePolicy: set - valueType: string + - name: map_balances + kind: map + initialBlock: 0 inputs: - source: sf.solana.type.v1.Block -# - name: map_balances -# kind: map -# initialBlock: 0 -# inputs: -# - source: sf.solana.type.v1.Block -# output: -# type: proto:messari.sol.token.Account + output: + type: proto:messari.sol.token.Account From b7a77265acc6dd279c027954da814a2dca9b39f4 Mon Sep 17 00:00:00 2001 From: dmelotik Date: Fri, 13 Jan 2023 14:37:15 -0700 Subject: [PATCH 30/36] remove outer string message --- common/proto/sol_token.proto | 4 ---- 1 file changed, 4 deletions(-) diff --git a/common/proto/sol_token.proto b/common/proto/sol_token.proto index 2032052c..3cc7d595 100644 --- a/common/proto/sol_token.proto +++ b/common/proto/sol_token.proto @@ -44,7 +44,3 @@ message Account { message Accounts { repeated Account items = 1; } - -message OuterString { - string inner_string = 1; -} From b1320639787bec4750d966b8ca5fdb2abc77b2f3 Mon Sep 17 00:00:00 2001 From: dmelotik Date: Thu, 19 Jan 2023 14:54:15 -0700 Subject: [PATCH 31/36] update schema --- Cargo.lock | 2 +- common/proto/sol_token.proto | 51 +++++++++++++++++++++--------------- sol-balance/src/lib.rs | 2 +- sol-balance/src/pb.rs | 10 +++---- sol-balance/substreams.yaml | 2 +- 5 files changed, 38 insertions(+), 29 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c0932858..f72ee003 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -929,7 +929,7 @@ dependencies = [ "ethabi", "hex-literal", "num-bigint", - "prost 0.11.2", + "prost 0.11.3", "substreams 0.0.11", "substreams-common", "substreams-helper", diff --git a/common/proto/sol_token.proto b/common/proto/sol_token.proto index 3cc7d595..1ec26697 100644 --- a/common/proto/sol_token.proto +++ b/common/proto/sol_token.proto @@ -1,6 +1,17 @@ syntax = "proto3"; -package messari.sol.token.v1; +package messari.sol_token.v1; + +message Mints { + repeated Mint mints = 1; +} + +message Mint { + string address = 1; + uint32 decimals = 2; + string mint_authority = 3; + string freeze_authority = 4; +} message Transfers { repeated Transfer transfers = 1; @@ -8,12 +19,26 @@ message Transfers { message Transfer { string signature = 1; - uint64 ordinal = 2; - string from = 3; - string to = 4; - TokenAccount token = 5; + string from = 2; + string to = 3; + TokenAccount token = 4; + string mint = 5; uint64 native_amount = 6; uint64 amount = 7; + BalanceChanges balance_changes = 8; +} + +message BalanceChanges { + repeated TokenBalance items = 1; +} + +// balance change +message TokenBalance { + TokenAccount token = 1; + uint64 block_height = 2; + string address = 3; // account address of the balance change + string old_balance = 4; // BigInt, in token's native amount + string new_balance = 5; // BigInt, in token's native amount } message Tokens { @@ -28,19 +53,3 @@ message TokenAccount { string owner = 5; string mint = 6; } - -message Balance { - TokenAccount token = 1; - string balance = 2; // BigInt in token's native units - uint64 block_number = 3; - uint64 timestamp = 4; -} - -message Account { - string account_id = 1; - repeated Balance balances = 2; -} - -message Accounts { - repeated Account items = 1; -} diff --git a/sol-balance/src/lib.rs b/sol-balance/src/lib.rs index 72f6f4ab..fbd713fa 100644 --- a/sol-balance/src/lib.rs +++ b/sol-balance/src/lib.rs @@ -6,7 +6,7 @@ use substreams_solana::pb; use substreams_solana::pb::sol as solana; #[substreams::handlers::map] -fn map_balances(block: solana::v1::Block) -> Result { +fn map_balances(block: solana::v1::Block) -> Result { let accounts = vec![]; for tx in block.transactions { if let Some(meta) = tx.meta { diff --git a/sol-balance/src/pb.rs b/sol-balance/src/pb.rs index dc8da007..42aed8b4 100644 --- a/sol-balance/src/pb.rs +++ b/sol-balance/src/pb.rs @@ -1,10 +1,10 @@ #[rustfmt::skip] -#[path = "../target/pb/messari.sol.token.rs"] -pub(in crate::pb) mod sol_token; +#[path = "../target/pb/messari.sol_token.v1.rs"] +pub(in crate::pb) mod sol_token_v1; -pub mod sol { - pub mod token { - pub use super::super::sol_token::*; +pub mod sol_token { + pub mod v1 { + pub use super::super::sol_token_v1::*; } } diff --git a/sol-balance/substreams.yaml b/sol-balance/substreams.yaml index 4c8654a7..d4a27f6a 100644 --- a/sol-balance/substreams.yaml +++ b/sol-balance/substreams.yaml @@ -24,4 +24,4 @@ modules: inputs: - source: sf.solana.type.v1.Block output: - type: proto:messari.sol.token.Account + type: proto:messari.sol.token.BalanceChanges From 3b067b96cd7025a892f2336c0879613daff327e1 Mon Sep 17 00:00:00 2001 From: dmelotik Date: Tue, 24 Jan 2023 22:36:44 -0700 Subject: [PATCH 32/36] change map module to output balance changes --- Cargo.lock | 17 +++++++++++--- common/proto/sol_token.proto | 6 ++--- erc20-holdings/src/pb.rs | 38 ++++--------------------------- erc20-market-cap/src/pb.rs | 10 ++++++++ sol-balance/Cargo.toml | 4 ++-- sol-balance/Makefile | 2 +- sol-balance/src/lib.rs | 33 +++++++++------------------ sol-balance/substreams.yaml | 1 - substreams-helper/build.rs | 1 + substreams-helper/src/lib.rs | 2 ++ substreams-helper/src/pb.rs | 9 ++++++++ substreams-helper/src/token.rs | 14 ++++++++++++ substreams-helper/substreams.yaml | 19 ++++++++++++++++ 13 files changed, 90 insertions(+), 66 deletions(-) create mode 100644 substreams-helper/src/pb.rs create mode 100644 substreams-helper/src/token.rs create mode 100644 substreams-helper/substreams.yaml diff --git a/Cargo.lock b/Cargo.lock index f72ee003..287d5462 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -930,10 +930,10 @@ dependencies = [ "hex-literal", "num-bigint", "prost 0.11.3", - "substreams 0.0.11", + "substreams 0.3.2", "substreams-common", "substreams-helper", - "substreams-solana", + "substreams-solana 0.2.0", ] [[package]] @@ -947,7 +947,7 @@ dependencies = [ "prost-build 0.10.4", "prost-types 0.10.1", "substreams 0.0.11", - "substreams-solana", + "substreams-solana 0.1.0", ] [[package]] @@ -1177,6 +1177,17 @@ dependencies = [ "substreams 0.0.12", ] +[[package]] +name = "substreams-solana" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5e5ad4a64b8af93729ca06debc69b2c72eaec3ee44872dd593e16dfcc56dae7" +dependencies = [ + "prost 0.11.3", + "prost-build 0.11.1", + "prost-types 0.11.1", +] + [[package]] name = "substreams-uniswap-v2" version = "0.1.0" diff --git a/common/proto/sol_token.proto b/common/proto/sol_token.proto index 1ec26697..b6ef533a 100644 --- a/common/proto/sol_token.proto +++ b/common/proto/sol_token.proto @@ -37,8 +37,8 @@ message TokenBalance { TokenAccount token = 1; uint64 block_height = 2; string address = 3; // account address of the balance change - string old_balance = 4; // BigInt, in token's native amount - string new_balance = 5; // BigInt, in token's native amount + string pre_balance = 4; // BigInt, in token's native amount + string post_balance = 5; // BigInt, in token's native amount } message Tokens { @@ -49,7 +49,7 @@ message TokenAccount { string address = 1; string name = 2; string symbol = 3; - string decimals = 4; + uint64 decimals = 4; string owner = 5; string mint = 6; } diff --git a/erc20-holdings/src/pb.rs b/erc20-holdings/src/pb.rs index bf5cf178..6d3d4de7 100644 --- a/erc20-holdings/src/pb.rs +++ b/erc20-holdings/src/pb.rs @@ -1,39 +1,9 @@ #[rustfmt::skip] -#[path = "../target/pb/messari.chainlink.v1.rs"] -pub(in crate::pb) mod chainlink_v1; +#[path = "../target/pb/messari.sol_token.v1.rs"] +pub(in crate::pb) mod sol_token_v1; -pub mod chainlink { +pub mod sol_token { pub mod v1 { - pub use super::super::chainlink_v1::*; - } -} - -#[rustfmt::skip] -#[path = "../target/pb/messari.common.v1.rs"] -pub(in crate::pb) mod common_v1; - -pub mod common { - pub mod v1 { - pub use super::super::common_v1::*; - } -} - -#[rustfmt::skip] -#[path = "../target/pb/messari.erc20.v1.rs"] -pub(in crate::pb) mod erc20_v1; - -pub mod erc20 { - pub mod v1 { - pub use super::super::erc20_v1::*; - } -} - -#[rustfmt::skip] -#[path = "../target/pb/messari.erc20_price.v1.rs"] -pub(in crate::pb) mod erc20_price_v1; - -pub mod erc20_price { - pub mod v1 { - pub use super::super::erc20_price_v1::*; + pub use super::super::sol_token_v1::*; } } diff --git a/erc20-market-cap/src/pb.rs b/erc20-market-cap/src/pb.rs index 9a8fb3eb..eb61bfbe 100644 --- a/erc20-market-cap/src/pb.rs +++ b/erc20-market-cap/src/pb.rs @@ -37,3 +37,13 @@ pub mod erc20_price { pub use super::super::erc20_price_v1::*; } } + +#[rustfmt::skip] +#[path = "../target/pb/messari.uniswap.v1.rs"] +pub(in crate::pb) mod uniswap_v1; + +pub mod uniswap { + pub mod v1 { + pub use super::super::uniswap_v1::*; + } +} diff --git a/sol-balance/Cargo.toml b/sol-balance/Cargo.toml index 455afa7a..e9382f9d 100644 --- a/sol-balance/Cargo.toml +++ b/sol-balance/Cargo.toml @@ -14,8 +14,8 @@ prost = "0.11.2" ethabi = "17.2.0" num-bigint = "0.4" hex-literal = "0.3.4" -substreams = "0.0.11" -substreams-solana = "0.1.0" +substreams = { workspace = true } +substreams-solana = { workspace = true } substreams-helper = { path = "../substreams-helper" } [build-dependencies] diff --git a/sol-balance/Makefile b/sol-balance/Makefile index 2f559204..241a52ba 100644 --- a/sol-balance/Makefile +++ b/sol-balance/Makefile @@ -4,4 +4,4 @@ build: .PHONY: run run: - substreams run -e mainnet.sol.streamingfast.io:443 substreams.yaml store_balance -s 10000 -t +10001 + substreams run -e mainnet.sol.streamingfast.io:443 substreams.yaml map_balances -s 174257750 -t +1 diff --git a/sol-balance/src/lib.rs b/sol-balance/src/lib.rs index fbd713fa..71ef66a6 100644 --- a/sol-balance/src/lib.rs +++ b/sol-balance/src/lib.rs @@ -1,37 +1,26 @@ -use crate::pb::sol::v1 as token; // TODO: I can't get this import to work :( -use bs58; -use prost::Message; -use substreams::store::StoreSet; -use substreams_solana::pb; use substreams_solana::pb::sol as solana; +use substreams_helper::token::get_sol_token; +use substreams_helper::pb::sol_token::v1 as proto; #[substreams::handlers::map] -fn map_balances(block: solana::v1::Block) -> Result { - let accounts = vec![]; +fn map_balances(block: solana::v1::Block) -> Result { + let mut balances = vec![]; for tx in block.transactions { if let Some(meta) = tx.meta { if let Some(_) = meta.err { continue; } for i in 0..meta.post_balances.len() { - let new_balance = &meta.post_balances[i].encode_to_vec(); - let new_token_balance = vec![token::Balance { + balances.push(proto::TokenBalance { token: get_sol_token(), - balance: new_balance, - block_number: block.block_height.block_height, - timestamp: block.block_time.timestamp, - }]; - // TODO: Not sure if this is the right place to get the address - // AND loaded_writeable_addresses is not being built in the proto stucts - // let account_id = bs58::encode(&meta.loaded_writable_addresses[i]).into_string(); - let account = token::Account { - account_id: "0x0", - balances: new_token_balance, - }; - accounts.push(account); + block_height: block.block_height.as_ref().unwrap().block_height, + address: "TODO".to_string(), + pre_balance: meta.pre_balances[i].to_string(), + post_balance: meta.post_balances[i].to_string() + }); } } } - Ok(token::Accounts { items: accounts }) + Ok(proto::BalanceChanges { items: balances}) } diff --git a/sol-balance/substreams.yaml b/sol-balance/substreams.yaml index d4a27f6a..2d2a07a1 100644 --- a/sol-balance/substreams.yaml +++ b/sol-balance/substreams.yaml @@ -20,7 +20,6 @@ binaries: modules: - name: map_balances kind: map - initialBlock: 0 inputs: - source: sf.solana.type.v1.Block output: diff --git a/substreams-helper/build.rs b/substreams-helper/build.rs index 0f1a2864..05e44fdb 100644 --- a/substreams-helper/build.rs +++ b/substreams-helper/build.rs @@ -2,6 +2,7 @@ use anyhow::{Ok, Result}; use substreams_common::codegen; fn main() -> Result<(), anyhow::Error> { + codegen::generate(None); codegen::generate_abi(None)?; Ok(()) } diff --git a/substreams-helper/src/lib.rs b/substreams-helper/src/lib.rs index b6cf1767..76cb6853 100644 --- a/substreams-helper/src/lib.rs +++ b/substreams-helper/src/lib.rs @@ -5,3 +5,5 @@ pub mod math; pub mod price; pub mod types; pub mod utils; +pub mod pb; +pub mod token; diff --git a/substreams-helper/src/pb.rs b/substreams-helper/src/pb.rs new file mode 100644 index 00000000..6d3d4de7 --- /dev/null +++ b/substreams-helper/src/pb.rs @@ -0,0 +1,9 @@ +#[rustfmt::skip] +#[path = "../target/pb/messari.sol_token.v1.rs"] +pub(in crate::pb) mod sol_token_v1; + +pub mod sol_token { + pub mod v1 { + pub use super::super::sol_token_v1::*; + } +} diff --git a/substreams-helper/src/token.rs b/substreams-helper/src/token.rs new file mode 100644 index 00000000..c48cb278 --- /dev/null +++ b/substreams-helper/src/token.rs @@ -0,0 +1,14 @@ +use crate::pb::sol_token::v1::TokenAccount; + +pub fn get_sol_token() -> Option { + let sol_token = TokenAccount { + address: "TODO".to_string(), + name: "Solana".to_string(), + symbol: "SOL".to_string(), + decimals: 18_u64, + owner: "TODO".to_string(), + mint: "TODO".to_string() + }; + + Some(sol_token) +} \ No newline at end of file diff --git a/substreams-helper/substreams.yaml b/substreams-helper/substreams.yaml new file mode 100644 index 00000000..ef91ee98 --- /dev/null +++ b/substreams-helper/substreams.yaml @@ -0,0 +1,19 @@ +specVersion: v0.1.0 +package: + name: substreams_helper + version: v0.1.0 + +imports: + eth: https://github.com/streamingfast/sf-ethereum/releases/download/v0.10.2/ethereum-v0.10.4.spkg + +protobuf: + files: + - sol_token.proto + - + importPaths: + - ../common/proto + +binaries: + default: + type: wasm/rust-v1 + file: "../target/wasm32-unknown-unknown/release/substreams_helper.wasm" \ No newline at end of file From f730c0f9e16bf52983a431925994e4a7d15d02b4 Mon Sep 17 00:00:00 2001 From: dmelotik Date: Wed, 25 Jan 2023 20:53:06 -0700 Subject: [PATCH 33/36] find accounts in sol balance --- common/proto/sol_token.proto | 15 ++------------ sol-balance/src/lib.rs | 38 +++++++++++++++++++++++----------- sol-balance/substreams.yaml | 1 + substreams-helper/src/lib.rs | 4 ++-- substreams-helper/src/token.rs | 10 ++++----- 5 files changed, 36 insertions(+), 32 deletions(-) diff --git a/common/proto/sol_token.proto b/common/proto/sol_token.proto index b6ef533a..32d77035 100644 --- a/common/proto/sol_token.proto +++ b/common/proto/sol_token.proto @@ -2,17 +2,6 @@ syntax = "proto3"; package messari.sol_token.v1; -message Mints { - repeated Mint mints = 1; -} - -message Mint { - string address = 1; - uint32 decimals = 2; - string mint_authority = 3; - string freeze_authority = 4; -} - message Transfers { repeated Transfer transfers = 1; } @@ -50,6 +39,6 @@ message TokenAccount { string name = 2; string symbol = 3; uint64 decimals = 4; - string owner = 5; - string mint = 6; + string freeze_authority = 5; + string mint_authority = 6; } diff --git a/sol-balance/src/lib.rs b/sol-balance/src/lib.rs index 71ef66a6..b4df74eb 100644 --- a/sol-balance/src/lib.rs +++ b/sol-balance/src/lib.rs @@ -1,26 +1,40 @@ -use substreams_solana::pb::sol as solana; -use substreams_helper::token::get_sol_token; +use bs58; use substreams_helper::pb::sol_token::v1 as proto; +use substreams_helper::token::get_sol_token; +use substreams_solana::pb::sol as solana; #[substreams::handlers::map] -fn map_balances(block: solana::v1::Block) -> Result { +fn map_balances( + block: solana::v1::Block, +) -> Result { let mut balances = vec![]; for tx in block.transactions { if let Some(meta) = tx.meta { if let Some(_) = meta.err { continue; } - for i in 0..meta.post_balances.len() { - balances.push(proto::TokenBalance { - token: get_sol_token(), - block_height: block.block_height.as_ref().unwrap().block_height, - address: "TODO".to_string(), - pre_balance: meta.pre_balances[i].to_string(), - post_balance: meta.post_balances[i].to_string() - }); + if let Some(transaction) = tx.transaction { + if let Some(msg) = transaction.message { + for i in 0..meta.post_balances.len() { + let account_id: String; + if i < msg.account_keys.len() { + account_id = bs58::encode(&msg.account_keys[i]).into_string(); + } else { + account_id = "TODO".to_string(); + // TODO: use msg.address_table_lookup table, but it is not avail + } + balances.push(proto::TokenBalance { + token: get_sol_token(), + block_height: block.block_height.as_ref().unwrap().block_height, + address: account_id, + pre_balance: meta.pre_balances[i].to_string(), + post_balance: meta.post_balances[i].to_string(), + }); + } + } } } } - Ok(proto::BalanceChanges { items: balances}) + Ok(proto::BalanceChanges { items: balances }) } diff --git a/sol-balance/substreams.yaml b/sol-balance/substreams.yaml index 2d2a07a1..038c966f 100644 --- a/sol-balance/substreams.yaml +++ b/sol-balance/substreams.yaml @@ -18,6 +18,7 @@ binaries: file: "../target/wasm32-unknown-unknown/release/sol_balance.wasm" modules: + # get the SOL balance change of each account in each transaction in a block - name: map_balances kind: map inputs: diff --git a/substreams-helper/src/lib.rs b/substreams-helper/src/lib.rs index 76cb6853..ad0b4954 100644 --- a/substreams-helper/src/lib.rs +++ b/substreams-helper/src/lib.rs @@ -2,8 +2,8 @@ pub mod abi; pub mod erc20; pub mod keyer; pub mod math; +pub mod pb; pub mod price; +pub mod token; pub mod types; pub mod utils; -pub mod pb; -pub mod token; diff --git a/substreams-helper/src/token.rs b/substreams-helper/src/token.rs index c48cb278..a460c926 100644 --- a/substreams-helper/src/token.rs +++ b/substreams-helper/src/token.rs @@ -2,13 +2,13 @@ use crate::pb::sol_token::v1::TokenAccount; pub fn get_sol_token() -> Option { let sol_token = TokenAccount { - address: "TODO".to_string(), + address: "So11111111111111111111111111111111111111111".to_string(), name: "Solana".to_string(), symbol: "SOL".to_string(), - decimals: 18_u64, - owner: "TODO".to_string(), - mint: "TODO".to_string() + decimals: 9_u64, + freeze_authority: "NA".to_string(), + mint_authority: "NA".to_string(), }; Some(sol_token) -} \ No newline at end of file +} From 81da0bc22692b4460bdb366e6151a55ddb950f60 Mon Sep 17 00:00:00 2001 From: dmelotik Date: Wed, 25 Jan 2023 23:48:49 -0700 Subject: [PATCH 34/36] add transaction id to balance change --- common/proto/sol_token.proto | 9 +++++---- sol-balance/src/lib.rs | 1 + 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/common/proto/sol_token.proto b/common/proto/sol_token.proto index 32d77035..754df164 100644 --- a/common/proto/sol_token.proto +++ b/common/proto/sol_token.proto @@ -24,10 +24,11 @@ message BalanceChanges { // balance change message TokenBalance { TokenAccount token = 1; - uint64 block_height = 2; - string address = 3; // account address of the balance change - string pre_balance = 4; // BigInt, in token's native amount - string post_balance = 5; // BigInt, in token's native amount + string transaction_id = 2; + uint64 block_height = 3; + string address = 4; // account address of the balance change + string pre_balance = 5; // BigInt, in token's native amount + string post_balance = 6; // BigInt, in token's native amount } message Tokens { diff --git a/sol-balance/src/lib.rs b/sol-balance/src/lib.rs index b4df74eb..f40387ea 100644 --- a/sol-balance/src/lib.rs +++ b/sol-balance/src/lib.rs @@ -25,6 +25,7 @@ fn map_balances( } balances.push(proto::TokenBalance { token: get_sol_token(), + transaction_id: bs58::encode(&transaction.signatures[0]).into_string(), block_height: block.block_height.as_ref().unwrap().block_height, address: account_id, pre_balance: meta.pre_balances[i].to_string(), From 0222147bbd9dc43daac33fa36c253b56bd269f63 Mon Sep 17 00:00:00 2001 From: dmelotik Date: Wed, 1 Feb 2023 14:18:17 -0700 Subject: [PATCH 35/36] add a few fields to the SOL proto --- common/proto/sol_token.proto | 2 ++ sol-balance/README.md | 14 ++++---------- substreams-helper/src/token.rs | 6 ++++-- 3 files changed, 10 insertions(+), 12 deletions(-) diff --git a/common/proto/sol_token.proto b/common/proto/sol_token.proto index 754df164..fba92695 100644 --- a/common/proto/sol_token.proto +++ b/common/proto/sol_token.proto @@ -42,4 +42,6 @@ message TokenAccount { uint64 decimals = 4; string freeze_authority = 5; string mint_authority = 6; + string tx_created = 7; + uint64 block_created = 8; } diff --git a/sol-balance/README.md b/sol-balance/README.md index ac707b94..3fd318b8 100644 --- a/sol-balance/README.md +++ b/sol-balance/README.md @@ -2,14 +2,8 @@ This substream is designed to store the SOL balance of every account. -### Build & Run +### Notes -```bash -make codegen -make run -``` - -### Notes - -- Currently [`transaction.loaded_writable_addresses`](https://github.com/streamingfast/firehose-solana/blob/develop/proto/sf/solana/type/v1/type.proto#L65) is an unknown field on firesol blocks. -- +- We are not able fully map all the accounts to SOL balance changes. + - There is a missing field [`address_table_lookup`](https://github.com/streamingfast/firehose-solana/blob/develop/proto/sf/solana/type/v1/type.proto#L38) that stores the rest of the addresses. + - See the issue filed [here](https://github.com/streamingfast/substreams/issues/144) to track the status. diff --git a/substreams-helper/src/token.rs b/substreams-helper/src/token.rs index a460c926..7292295f 100644 --- a/substreams-helper/src/token.rs +++ b/substreams-helper/src/token.rs @@ -6,8 +6,10 @@ pub fn get_sol_token() -> Option { name: "Solana".to_string(), symbol: "SOL".to_string(), decimals: 9_u64, - freeze_authority: "NA".to_string(), - mint_authority: "NA".to_string(), + freeze_authority: "".to_string(), + mint_authority: "".to_string(), + tx_created: "".to_string(), + block_created: 0_u64, }; Some(sol_token) From 43b8731aead6b3742d0956c1f96e4b6dc9481fa1 Mon Sep 17 00:00:00 2001 From: dmelotik Date: Thu, 2 Feb 2023 14:45:00 -0700 Subject: [PATCH 36/36] add comment about account_table_lookup --- sol-balance/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/sol-balance/README.md b/sol-balance/README.md index 3fd318b8..9ceb7534 100644 --- a/sol-balance/README.md +++ b/sol-balance/README.md @@ -7,3 +7,4 @@ This substream is designed to store the SOL balance of every account. - We are not able fully map all the accounts to SOL balance changes. - There is a missing field [`address_table_lookup`](https://github.com/streamingfast/firehose-solana/blob/develop/proto/sf/solana/type/v1/type.proto#L38) that stores the rest of the addresses. - See the issue filed [here](https://github.com/streamingfast/substreams/issues/144) to track the status. + - This field is available, but there is decoding needed on the substream part in order to actually get that data. Next steps are understanding the Solana data structure better and seeing firesol output to get a better understanding of where this data lives.