From d61d5fad1acb3c601eca6e2e25b0360322c95df7 Mon Sep 17 00:00:00 2001 From: rikoeldon <106416799+rikoeldon@users.noreply.github.com> Date: Wed, 19 Mar 2025 14:58:56 -0400 Subject: [PATCH 01/38] git ignore update --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 18dfef8f..9c5ef416 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,5 @@ /target *.vscode /chain/assets -/types/pkg \ No newline at end of file +/types/pkg +.idea \ No newline at end of file From 83b2456bd26da8bdde3b16ad31ae089316e36a3d Mon Sep 17 00:00:00 2001 From: rikoeldon <106416799+rikoeldon@users.noreply.github.com> Date: Thu, 20 Mar 2025 11:22:25 -0400 Subject: [PATCH 02/38] setting up actions --- actions/msg.rs | 21 +++++++++++++++++++++ actions/transfer.rs | 0 types/src/codec.rs | 11 +++++++++++ types/src/lib.rs | 1 + 4 files changed, 33 insertions(+) create mode 100644 actions/msg.rs create mode 100644 actions/transfer.rs create mode 100644 types/src/codec.rs diff --git a/actions/msg.rs b/actions/msg.rs new file mode 100644 index 00000000..30dcccbe --- /dev/null +++ b/actions/msg.rs @@ -0,0 +1,21 @@ +use types::Address; +pub struct SequencerMsg { + pub chain_id: Vec, + pub data: Vec, + pub from_address: Address::new(), + pub relayer_id: u64, +} + +impl SequencerMsg { + pub fn new() -> Self { + Self { + chain_id: vec![], + data: vec![], + from_address: Address::new(), + relayer_id: 0, + } + } + pub fn get_type_id(&self) -> u8 { + 0 + } +} \ No newline at end of file diff --git a/actions/transfer.rs b/actions/transfer.rs new file mode 100644 index 00000000..e69de29b diff --git a/types/src/codec.rs b/types/src/codec.rs new file mode 100644 index 00000000..f1a6b529 --- /dev/null +++ b/types/src/codec.rs @@ -0,0 +1,11 @@ +pub enum Codec { + +} +const ADDRESSLEN: usize = 33; +struct Address([u8;ADDRESSLEN]); + +impl Address { + pub fn new() -> Self { + Self([0;ADDRESSLEN]) + } +} \ No newline at end of file diff --git a/types/src/lib.rs b/types/src/lib.rs index 9ffac637..75a601a8 100644 --- a/types/src/lib.rs +++ b/types/src/lib.rs @@ -5,6 +5,7 @@ pub use block::{Block, Finalized, Notarized}; mod consensus; pub use consensus::{leader_index, Finalization, Kind, Notarization, Nullification, Seed}; pub mod wasm; +mod codec; // We don't use functions here to guard against silent changes. pub const NAMESPACE: &[u8] = b"_ALTO"; From 1da828e2a1b13f6adc3e0f337b6014390ee42f20 Mon Sep 17 00:00:00 2001 From: NateAtNodeKit Date: Fri, 21 Mar 2025 15:34:26 -0400 Subject: [PATCH 03/38] prototyping --- Cargo.lock | 11 ++++++ Cargo.toml | 2 ++ actions/transfer.rs | 16 +++++++++ storage/Cargo.toml | 7 ++++ storage/src/account.rs | 15 ++++++++ storage/src/database.rs | 9 +++++ storage/src/hashmap_db.rs | 74 +++++++++++++++++++++++++++++++++++++++ storage/src/lib.rs | 3 ++ types/src/codec.rs | 4 ++- types/src/lib.rs | 11 ++++++ vm/Cargo.toml | 6 ++++ vm/src/lib.rs | 1 + vm/src/vm.rs | 17 +++++++++ 13 files changed, 175 insertions(+), 1 deletion(-) create mode 100644 storage/Cargo.toml create mode 100644 storage/src/account.rs create mode 100644 storage/src/database.rs create mode 100644 storage/src/hashmap_db.rs create mode 100644 storage/src/lib.rs create mode 100644 vm/Cargo.toml create mode 100644 vm/src/lib.rs create mode 100644 vm/src/vm.rs diff --git a/Cargo.lock b/Cargo.lock index a3d67ca7..afa95dcc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -102,6 +102,13 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "alto-storage" +version = "0.1.0" +dependencies = [ + "alto-types", +] + [[package]] name = "alto-types" version = "0.0.4" @@ -117,6 +124,10 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "alto-vm" +version = "0.1.0" + [[package]] name = "anstream" version = "0.6.18" diff --git a/Cargo.toml b/Cargo.toml index bfa926c9..2e211b94 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,7 +3,9 @@ members = [ "chain", "client", "inspector", + "storage", "types", + "vm", ] resolver = "2" diff --git a/actions/transfer.rs b/actions/transfer.rs index e69de29b..799f84dc 100644 --- a/actions/transfer.rs +++ b/actions/transfer.rs @@ -0,0 +1,16 @@ + +pub struct Transfer { + pub from_address: Address::new(), + pub to_address: Address::new(), + pub value: u64, +} + +impl Transfer { + pub fn new(from: Address, to: Address, value: u64) -> Transfer { + Self { + from_address: from, + to_address: to, + value: value, + } + } +} \ No newline at end of file diff --git a/storage/Cargo.toml b/storage/Cargo.toml new file mode 100644 index 00000000..12168468 --- /dev/null +++ b/storage/Cargo.toml @@ -0,0 +1,7 @@ +[package] +name = "alto-storage" +version = "0.1.0" +edition = "2024" + +[dependencies] +alto-types = { workspace = true } diff --git a/storage/src/account.rs b/storage/src/account.rs new file mode 100644 index 00000000..c4694b5d --- /dev/null +++ b/storage/src/account.rs @@ -0,0 +1,15 @@ + +pub(crate) type Balance = u64; + +#[derive(Clone, Debug)] +pub struct Account { + pub balance: Balance, +} + +impl Account { + pub fn new(initial_balance: u64) -> Self { + Self { + balance: initial_balance, + } + } +} \ No newline at end of file diff --git a/storage/src/database.rs b/storage/src/database.rs new file mode 100644 index 00000000..90afc817 --- /dev/null +++ b/storage/src/database.rs @@ -0,0 +1,9 @@ +use alto_types::Address; +use crate::account::Balance; + +// Define database interface that will be used for all impls +pub trait Database { + fn get_balance(&self, address: Address) -> Option; + fn add_balance(&mut self, address: Address, amt: u64) -> Option; + fn sub_balance(&mut self, address: Address, amt: u64) -> Option; +} \ No newline at end of file diff --git a/storage/src/hashmap_db.rs b/storage/src/hashmap_db.rs new file mode 100644 index 00000000..ae0881b8 --- /dev/null +++ b/storage/src/hashmap_db.rs @@ -0,0 +1,74 @@ +use std::collections::HashMap; +use alto_types::Address; +use crate::account::{Account, Balance}; +use crate::database::Database; + +pub struct HashmapDb { + accts: HashMap, +} + +impl HashmapDb { + pub fn new() -> Self { + Self { + accts: HashMap::new(), + } + } +} + +impl Database for HashmapDb { + fn get_balance(&self, address: Address) -> Option { + match self.accts.get(&address) { + Some(acct) => Some(acct.balance), + None => None, + } + } + + fn add_balance(&mut self, address: Address, amt: u64) -> Option { + let curr_balance = self.accts.get(&address); + match curr_balance { + Some(account) => { + let updated_account = account.clone(); + updated_account.balance.checked_add(amt)?; + self.accts.insert(address, updated_account.clone()); + Some(updated_account.balance) + }, + None => { + let new_account = Account::new(amt); + self.accts.insert(address, new_account.clone()); + Some(new_account.balance) + }, + } + } + + fn sub_balance(&mut self, address: Address, amt: u64) -> Option { + let curr_balance = self.accts.get(&address); + match curr_balance { + Some(account) => { + let updated_account = account.clone(); + updated_account.balance.checked_add(amt)?; + self.accts.insert(address, updated_account.clone()); + Some(updated_account.balance) + }, + None => { + let new_account = Account::new(amt); + self.accts.insert(address, new_account.clone()); + Some(new_account.balance) + }, + } + } +} + +#[cfg(test)] // Compile these only when running tests +mod tests { + use super::*; // Import items from the parent module + + #[test] // Mark this function as a test + fn test_addition() { + let a = 2; + let b = 3; + let result = a + b; + + // Use assertions to validate + assert_eq!(result, 6); // Test passes if result == 5 + } +} diff --git a/storage/src/lib.rs b/storage/src/lib.rs new file mode 100644 index 00000000..0b99d3b6 --- /dev/null +++ b/storage/src/lib.rs @@ -0,0 +1,3 @@ +pub mod account; +pub mod database; +pub mod hashmap_db; \ No newline at end of file diff --git a/types/src/codec.rs b/types/src/codec.rs index f1a6b529..583acc66 100644 --- a/types/src/codec.rs +++ b/types/src/codec.rs @@ -1,6 +1,7 @@ pub enum Codec { } +/* const ADDRESSLEN: usize = 33; struct Address([u8;ADDRESSLEN]); @@ -8,4 +9,5 @@ impl Address { pub fn new() -> Self { Self([0;ADDRESSLEN]) } -} \ No newline at end of file +} + */ \ No newline at end of file diff --git a/types/src/lib.rs b/types/src/lib.rs index 75a601a8..1d6d80eb 100644 --- a/types/src/lib.rs +++ b/types/src/lib.rs @@ -15,6 +15,17 @@ pub const NOTARIZE_NAMESPACE: &[u8] = b"_ALTO_NOTARIZE"; pub const NULLIFY_NAMESPACE: &[u8] = b"_ALTO_NULLIFY"; pub const FINALIZE_NAMESPACE: &[u8] = b"_ALTO_FINALIZE"; +const ADDRESSLEN: usize = 33; + +#[derive(Hash, Eq, PartialEq)] +pub struct Address([u8;ADDRESSLEN]); + +impl Address { + pub fn new() -> Self { + Self([0;ADDRESSLEN]) + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/vm/Cargo.toml b/vm/Cargo.toml new file mode 100644 index 00000000..5d1e5c51 --- /dev/null +++ b/vm/Cargo.toml @@ -0,0 +1,6 @@ +[package] +name = "alto-vm" +version = "0.1.0" +edition = "2024" + +[dependencies] diff --git a/vm/src/lib.rs b/vm/src/lib.rs new file mode 100644 index 00000000..77e9423e --- /dev/null +++ b/vm/src/lib.rs @@ -0,0 +1 @@ +pub mod vm; \ No newline at end of file diff --git a/vm/src/vm.rs b/vm/src/vm.rs new file mode 100644 index 00000000..3390c221 --- /dev/null +++ b/vm/src/vm.rs @@ -0,0 +1,17 @@ +use std::collections::HashMap; + +struct VM { + state_db: Box +} + +impl VM { + pub fn new() { + Self { + state_db: Box::new(HashMapDb::new()) + } + } + + pub fn execute(msg: Transfer) { + // process the transfer + } +} \ No newline at end of file From acd89076237c021bd2d6c2db296b22a15fe9e37e Mon Sep 17 00:00:00 2001 From: NateAtNodeKit Date: Fri, 21 Mar 2025 17:23:19 -0400 Subject: [PATCH 04/38] fix project settings --- Cargo.lock | 11 +++++++++++ Cargo.toml | 5 ++++- actions/Cargo.toml | 7 +++++++ actions/src/lib.rs | 2 ++ actions/{ => src}/msg.rs | 5 +++-- actions/{ => src}/transfer.rs | 5 +++-- vm/Cargo.toml | 4 ++++ vm/src/vm.rs | 8 +++++--- 8 files changed, 39 insertions(+), 8 deletions(-) create mode 100644 actions/Cargo.toml create mode 100644 actions/src/lib.rs rename actions/{ => src}/msg.rs (86%) rename actions/{ => src}/transfer.rs (74%) diff --git a/Cargo.lock b/Cargo.lock index afa95dcc..8e1578fb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -36,6 +36,13 @@ dependencies = [ "memchr", ] +[[package]] +name = "alto-actions" +version = "0.1.0" +dependencies = [ + "alto-types", +] + [[package]] name = "alto-chain" version = "0.0.4" @@ -127,6 +134,10 @@ dependencies = [ [[package]] name = "alto-vm" version = "0.1.0" +dependencies = [ + "alto-actions", + "alto-storage", +] [[package]] name = "anstream" diff --git a/Cargo.toml b/Cargo.toml index 2e211b94..d270f159 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,5 @@ [workspace] -members = [ +members = [ "actions", "chain", "client", "inspector", @@ -12,6 +12,9 @@ resolver = "2" [workspace.dependencies] alto-client = { version = "0.0.4", path = "client" } alto-types = { version = "0.0.4", path = "types" } +alto-actions = { version = "0.1.0", path = "actions"} +alto-storage = { version = "0.1.0", path = "storage"} +alto-vm = { version = "0.1.0", path = "vm"} commonware-consensus = { version = "0.0.40" } commonware-cryptography = { version = "0.0.40" } commonware-deployer = { version = "0.0.40" } diff --git a/actions/Cargo.toml b/actions/Cargo.toml new file mode 100644 index 00000000..189380eb --- /dev/null +++ b/actions/Cargo.toml @@ -0,0 +1,7 @@ +[package] +name = "alto-actions" +version = "0.1.0" +edition = "2024" + +[dependencies] +alto-types = { workspace = true } diff --git a/actions/src/lib.rs b/actions/src/lib.rs new file mode 100644 index 00000000..6b3f5403 --- /dev/null +++ b/actions/src/lib.rs @@ -0,0 +1,2 @@ +pub mod msg; +pub mod transfer; \ No newline at end of file diff --git a/actions/msg.rs b/actions/src/msg.rs similarity index 86% rename from actions/msg.rs rename to actions/src/msg.rs index 30dcccbe..b71c7065 100644 --- a/actions/msg.rs +++ b/actions/src/msg.rs @@ -1,8 +1,9 @@ -use types::Address; +use alto_types::Address; + pub struct SequencerMsg { pub chain_id: Vec, pub data: Vec, - pub from_address: Address::new(), + pub from_address: Address, pub relayer_id: u64, } diff --git a/actions/transfer.rs b/actions/src/transfer.rs similarity index 74% rename from actions/transfer.rs rename to actions/src/transfer.rs index 799f84dc..be863b50 100644 --- a/actions/transfer.rs +++ b/actions/src/transfer.rs @@ -1,7 +1,8 @@ +use alto_types::Address; pub struct Transfer { - pub from_address: Address::new(), - pub to_address: Address::new(), + pub from_address: Address, + pub to_address: Address, pub value: u64, } diff --git a/vm/Cargo.toml b/vm/Cargo.toml index 5d1e5c51..1725c336 100644 --- a/vm/Cargo.toml +++ b/vm/Cargo.toml @@ -4,3 +4,7 @@ version = "0.1.0" edition = "2024" [dependencies] +alto-storage = { workspace = true } +alto-actions = { workspace = true } +#alto-storage = { path = "../storage" } +#alto-actions = { path = "../actions" } diff --git a/vm/src/vm.rs b/vm/src/vm.rs index 3390c221..2616fd4e 100644 --- a/vm/src/vm.rs +++ b/vm/src/vm.rs @@ -1,4 +1,6 @@ -use std::collections::HashMap; +use alto_storage::database::Database; +use alto_storage::hashmap_db::HashmapDb; +use alto_actions::transfer::Transfer; struct VM { state_db: Box @@ -7,8 +9,8 @@ struct VM { impl VM { pub fn new() { Self { - state_db: Box::new(HashMapDb::new()) - } + state_db: Box::new(HashmapDb::new()) + }; } pub fn execute(msg: Transfer) { From 384a3d2cabb4e7dc8278179c29292185c11dfb41 Mon Sep 17 00:00:00 2001 From: rikoeldon <106416799+rikoeldon@users.noreply.github.com> Date: Mon, 24 Mar 2025 15:02:31 -0400 Subject: [PATCH 05/38] defined base accounts, balance, address, and working on VM. --- Cargo.lock | 9 +++++++++ Cargo.toml | 1 + actions/src/transfer.rs | 39 ++++++++++++++++++++++++++++++++----- storage/Cargo.toml | 2 ++ storage/src/account.rs | 20 +++++++++++++++++-- types/Cargo.toml | 1 + types/src/lib.rs | 17 +++++++++++++--- vm/src/vm.rs | 43 ++++++++++++++++++++++++++++++++++++----- 8 files changed, 117 insertions(+), 15 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8e1578fb..f5571977 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -114,6 +114,8 @@ name = "alto-storage" version = "0.1.0" dependencies = [ "alto-types", + "commonware-cryptography", + "rand", ] [[package]] @@ -124,6 +126,7 @@ dependencies = [ "commonware-cryptography", "commonware-utils", "getrandom 0.2.15", + "more-asserts", "rand", "serde", "serde-wasm-bindgen", @@ -2093,6 +2096,12 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "more-asserts" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fafa6961cabd9c63bcd77a45d7e3b7f3b552b70417831fb0f56db717e72407e" + [[package]] name = "multimap" version = "0.10.0" diff --git a/Cargo.toml b/Cargo.toml index d270f159..660e31d0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -37,6 +37,7 @@ tracing-subscriber = "0.3.19" governor = "0.6.3" prometheus-client = "0.22.3" clap = "4.5.18" +more-asserts = "0.3.1" [profile.bench] # Because we enable overflow checks in "release," we should benchmark with them. diff --git a/actions/src/transfer.rs b/actions/src/transfer.rs index be863b50..6cdd0ef7 100644 --- a/actions/src/transfer.rs +++ b/actions/src/transfer.rs @@ -1,17 +1,46 @@ use alto_types::Address; +const MAX_MEMO_SIZE: usize = 256; pub struct Transfer { pub from_address: Address, pub to_address: Address, pub value: u64, + pub memo: Vec, +} + +pub enum TransferError { + DuplicateAddress, + InvalidToAddress, + InvalidFromAddress, + + InsufficientFunds, + InvalidMemoSize, } impl Transfer { - pub fn new(from: Address, to: Address, value: u64) -> Transfer { - Self { - from_address: from, - to_address: to, - value: value, + pub fn new(from: Address, to: Address, value: u64, memo: Vec) -> Result { + if value == 0 { + Err(TransferError::InsufficientFunds) + } + else if memo.len() > MAX_MEMO_SIZE { + Err(TransferError::InvalidMemoSize) + } + else if from.is_empty() { + Err(TransferError::InvalidFromAddress) + } + else if to.is_empty() { + Err(TransferError::InvalidToAddress) + } + else if from == to { + Err(TransferError::DuplicateAddress) + } + else { + Ok(Self { + from_address: from, + to_address: to, + value, + memo, + }) } } } \ No newline at end of file diff --git a/storage/Cargo.toml b/storage/Cargo.toml index 12168468..1d99074f 100644 --- a/storage/Cargo.toml +++ b/storage/Cargo.toml @@ -5,3 +5,5 @@ edition = "2024" [dependencies] alto-types = { workspace = true } +commonware-cryptography = { workspace = true } +rand = "0.8.5" diff --git a/storage/src/account.rs b/storage/src/account.rs index c4694b5d..a50712a6 100644 --- a/storage/src/account.rs +++ b/storage/src/account.rs @@ -1,15 +1,31 @@ +use commonware_cryptography::{ed25519::PublicKey, Ed25519, Scheme}; +use alto_types::Address; +use rand::rngs::OsRng; pub(crate) type Balance = u64; #[derive(Clone, Debug)] pub struct Account { + pub address: Address, + pub public_key: PublicKey, pub balance: Balance, } impl Account { - pub fn new(initial_balance: u64) -> Self { + pub fn new() -> Self { + let mut rng = OsRng; + // generates keypair using random number generator + let keypair = Ed25519::new(&mut rng); + + let public_key = keypair.public_key(); + + // takes addr from pk (truncating or hashing if necessary) + let address = Address::new(&public_key.as_ref()[..32]); + Self { - balance: initial_balance, + address, + public_key, + balance: 0, } } } \ No newline at end of file diff --git a/types/Cargo.toml b/types/Cargo.toml index 34cc41ed..d45e2773 100644 --- a/types/Cargo.toml +++ b/types/Cargo.toml @@ -22,6 +22,7 @@ thiserror = { workspace = true } wasm-bindgen = "0.2.100" serde = { version = "1.0.219", features = ["derive"] } serde-wasm-bindgen = "0.6.5" +more-asserts = "0.3.1" # Enable "js" feature when WASM is target [target.'cfg(target_arch = "wasm32")'.dependencies.getrandom] diff --git a/types/src/lib.rs b/types/src/lib.rs index 1d6d80eb..05465228 100644 --- a/types/src/lib.rs +++ b/types/src/lib.rs @@ -6,6 +6,8 @@ mod consensus; pub use consensus::{leader_index, Finalization, Kind, Notarization, Nullification, Seed}; pub mod wasm; mod codec; +use more_asserts; +use more_asserts::assert_le; // We don't use functions here to guard against silent changes. pub const NAMESPACE: &[u8] = b"_ALTO"; @@ -15,15 +17,24 @@ pub const NOTARIZE_NAMESPACE: &[u8] = b"_ALTO_NOTARIZE"; pub const NULLIFY_NAMESPACE: &[u8] = b"_ALTO_NULLIFY"; pub const FINALIZE_NAMESPACE: &[u8] = b"_ALTO_FINALIZE"; -const ADDRESSLEN: usize = 33; +const ADDRESSLEN: usize = 32; -#[derive(Hash, Eq, PartialEq)] +#[derive(Hash, Eq, PartialEq, Clone, Debug)] pub struct Address([u8;ADDRESSLEN]); impl Address { - pub fn new() -> Self { + pub fn new(slice: &[u8]) -> Self { + assert_le!(slice.len(), ADDRESSLEN); + let mut address = Self::empty(); + address.0.copy_from_slice(slice); + address + } + pub fn empty() -> Self { Self([0;ADDRESSLEN]) } + pub fn is_empty(&self) -> bool { + self.0 == Self::empty().0 + } } #[cfg(test)] diff --git a/vm/src/vm.rs b/vm/src/vm.rs index 2616fd4e..291ca75e 100644 --- a/vm/src/vm.rs +++ b/vm/src/vm.rs @@ -1,19 +1,52 @@ +use alto_actions::transfer::{Transfer, TransferError}; +use alto_storage::account::Account; use alto_storage::database::Database; use alto_storage::hashmap_db::HashmapDb; -use alto_actions::transfer::Transfer; struct VM { - state_db: Box + state_db: Box, } impl VM { pub fn new() { Self { - state_db: Box::new(HashmapDb::new()) + state_db: Box::new(HashmapDb::new()), }; } - pub fn execute(msg: Transfer) { + pub fn execute(&mut self, msg: Transfer) -> Result<(), TransferError> { // process the transfer + + // add balance to account + let from_balance = self.state_db.get_balance(msg.from_address); + if from_balance.is_none() { + return Err(TransferError::InvalidFromAddress); + } + let to_balance = self.state_db.get_balance(msg.to_address); + if to_balance.is_none() { + return Err(TransferError::InvalidToAddress); + } + // TODO: + // make check for sending funds > 0 so reject any negative values or balances. + // make sure address isn't a duplicate of self. + // need from balance to be greater than or equal to the transfer amount + // need the to_balance to not overflow + + let updated_from_balance = from_balance.unwrap().checked_sub(msg.value); + let updated_to_balance = to_balance.unwrap().checked_add(msg.value); + + match curr_balance { + Some(account) => { + let updated_account = account.clone(); + updated_account.balance.checked_add(amt)?; + self.accts.insert(address, updated_account.clone()); + Some(updated_account.balance) + } + None => { + let new_account = Account::new(amt); + self.accts.insert(address, new_account.clone()); + Some(new_account.balance) + } + } } -} \ No newline at end of file +} From e31d470d8e9a81886e732f2d02e915779b255f78 Mon Sep 17 00:00:00 2001 From: NateAtNodeKit Date: Mon, 24 Mar 2025 17:24:11 -0400 Subject: [PATCH 06/38] transfer logic --- Cargo.lock | 1 + actions/src/msg.rs | 2 +- actions/src/transfer.rs | 13 ++++- storage/src/account.rs | 2 +- storage/src/database.rs | 5 +- storage/src/hashmap_db.rs | 56 +++--------------- types/src/lib.rs | 10 ++-- vm/Cargo.toml | 3 +- vm/src/vm.rs | 119 +++++++++++++++++++++++++++++--------- 9 files changed, 124 insertions(+), 87 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f5571977..951dbe1e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -140,6 +140,7 @@ version = "0.1.0" dependencies = [ "alto-actions", "alto-storage", + "alto-types", ] [[package]] diff --git a/actions/src/msg.rs b/actions/src/msg.rs index b71c7065..e8e1d093 100644 --- a/actions/src/msg.rs +++ b/actions/src/msg.rs @@ -12,7 +12,7 @@ impl SequencerMsg { Self { chain_id: vec![], data: vec![], - from_address: Address::new(), + from_address: Address::empty(), relayer_id: 0, } } diff --git a/actions/src/transfer.rs b/actions/src/transfer.rs index 6cdd0ef7..84a16129 100644 --- a/actions/src/transfer.rs +++ b/actions/src/transfer.rs @@ -1,6 +1,8 @@ use alto_types::Address; const MAX_MEMO_SIZE: usize = 256; + +#[derive(Debug)] pub struct Transfer { pub from_address: Address, pub to_address: Address, @@ -8,17 +10,24 @@ pub struct Transfer { pub memo: Vec, } +#[derive(Debug)] pub enum TransferError { DuplicateAddress, InvalidToAddress, InvalidFromAddress, - InsufficientFunds, + TooMuchFunds, InvalidMemoSize, + StorageError, } impl Transfer { - pub fn new(from: Address, to: Address, value: u64, memo: Vec) -> Result { + pub fn new(from_address: Address, to_address: Address, value: u64) -> Result { + let empty_memo = b"".to_vec(); + Self::new_with_memo(from_address, to_address, value, empty_memo) + } + + pub fn new_with_memo(from: Address, to: Address, value: u64, memo: Vec) -> Result { if value == 0 { Err(TransferError::InsufficientFunds) } diff --git a/storage/src/account.rs b/storage/src/account.rs index a50712a6..53e31da5 100644 --- a/storage/src/account.rs +++ b/storage/src/account.rs @@ -2,7 +2,7 @@ use commonware_cryptography::{ed25519::PublicKey, Ed25519, Scheme}; use alto_types::Address; use rand::rngs::OsRng; -pub(crate) type Balance = u64; +pub type Balance = u64; #[derive(Clone, Debug)] pub struct Account { diff --git a/storage/src/database.rs b/storage/src/database.rs index 90afc817..1486787b 100644 --- a/storage/src/database.rs +++ b/storage/src/database.rs @@ -3,7 +3,6 @@ use crate::account::Balance; // Define database interface that will be used for all impls pub trait Database { - fn get_balance(&self, address: Address) -> Option; - fn add_balance(&mut self, address: Address, amt: u64) -> Option; - fn sub_balance(&mut self, address: Address, amt: u64) -> Option; + fn get_balance(&self, address: &Address) -> Option; + fn set_balance(&mut self, address: &Address, amt: u64) -> bool; } \ No newline at end of file diff --git a/storage/src/hashmap_db.rs b/storage/src/hashmap_db.rs index ae0881b8..a845e657 100644 --- a/storage/src/hashmap_db.rs +++ b/storage/src/hashmap_db.rs @@ -16,59 +16,19 @@ impl HashmapDb { } impl Database for HashmapDb { - fn get_balance(&self, address: Address) -> Option { + fn get_balance(&self, address: &Address) -> Option { match self.accts.get(&address) { Some(acct) => Some(acct.balance), None => None, } } - fn add_balance(&mut self, address: Address, amt: u64) -> Option { - let curr_balance = self.accts.get(&address); - match curr_balance { - Some(account) => { - let updated_account = account.clone(); - updated_account.balance.checked_add(amt)?; - self.accts.insert(address, updated_account.clone()); - Some(updated_account.balance) - }, - None => { - let new_account = Account::new(amt); - self.accts.insert(address, new_account.clone()); - Some(new_account.balance) - }, - } - } - - fn sub_balance(&mut self, address: Address, amt: u64) -> Option { - let curr_balance = self.accts.get(&address); - match curr_balance { - Some(account) => { - let updated_account = account.clone(); - updated_account.balance.checked_add(amt)?; - self.accts.insert(address, updated_account.clone()); - Some(updated_account.balance) - }, - None => { - let new_account = Account::new(amt); - self.accts.insert(address, new_account.clone()); - Some(new_account.balance) - }, - } - } -} - -#[cfg(test)] // Compile these only when running tests -mod tests { - use super::*; // Import items from the parent module - - #[test] // Mark this function as a test - fn test_addition() { - let a = 2; - let b = 3; - let result = a + b; - - // Use assertions to validate - assert_eq!(result, 6); // Test passes if result == 5 + fn set_balance(&mut self, address: &Address, amt: Balance) -> bool { + // TODO: Add in public key update later + let mut updated_account = Account::new(); + updated_account.address = address.clone(); + updated_account.balance = amt; + self.accts.insert(address.clone(), updated_account); + true } } diff --git a/types/src/lib.rs b/types/src/lib.rs index 05465228..be687dae 100644 --- a/types/src/lib.rs +++ b/types/src/lib.rs @@ -24,14 +24,16 @@ pub struct Address([u8;ADDRESSLEN]); impl Address { pub fn new(slice: &[u8]) -> Self { - assert_le!(slice.len(), ADDRESSLEN); - let mut address = Self::empty(); - address.0.copy_from_slice(slice); - address + assert_le!(slice.len(), ADDRESSLEN, "address slice is too large"); + let mut arr = [0u8; ADDRESSLEN]; + arr[..slice.len()].copy_from_slice(slice); + Address(arr) } + pub fn empty() -> Self { Self([0;ADDRESSLEN]) } + pub fn is_empty(&self) -> bool { self.0 == Self::empty().0 } diff --git a/vm/Cargo.toml b/vm/Cargo.toml index 1725c336..8e773002 100644 --- a/vm/Cargo.toml +++ b/vm/Cargo.toml @@ -6,5 +6,4 @@ edition = "2024" [dependencies] alto-storage = { workspace = true } alto-actions = { workspace = true } -#alto-storage = { path = "../storage" } -#alto-actions = { path = "../actions" } +alto-types = { workspace = true } diff --git a/vm/src/vm.rs b/vm/src/vm.rs index 291ca75e..06a4c417 100644 --- a/vm/src/vm.rs +++ b/vm/src/vm.rs @@ -1,52 +1,119 @@ use alto_actions::transfer::{Transfer, TransferError}; -use alto_storage::account::Account; use alto_storage::database::Database; use alto_storage::hashmap_db::HashmapDb; +use alto_storage::account::Balance; +use alto_types::Address; + +const TEST_FAUCET_ADDRESS: &[u8] = b"0x0123456789abcdef0123456789abcd"; +const TEST_FAUCET_BALANCE: Balance = 10_000_000; struct VM { state_db: Box, } impl VM { - pub fn new() { + pub fn new() -> Self { Self { state_db: Box::new(HashmapDb::new()), - }; + } } - pub fn execute(&mut self, msg: Transfer) -> Result<(), TransferError> { - // process the transfer + pub fn new_test_vm() -> Self { + let mut state_db = Box::new(HashmapDb::new()); + state_db.set_balance(&Self::test_faucet_address(), TEST_FAUCET_BALANCE); + + Self { + state_db + } + } - // add balance to account - let from_balance = self.state_db.get_balance(msg.from_address); + // TODO: + // make check for sending funds > 0 so reject any negative values or balances. + // make sure address isn't a duplicate of self. + // need from balance to be greater than or equal to the transfer amount + // need the to_balance to not overflow + pub fn execute(&mut self, msg: Transfer) -> Result<(), TransferError> { + let from_balance = self.state_db.get_balance(&msg.from_address); if from_balance.is_none() { return Err(TransferError::InvalidFromAddress); } - let to_balance = self.state_db.get_balance(msg.to_address); + + + let mut to_balance = self.state_db.get_balance(&msg.to_address); if to_balance.is_none() { - return Err(TransferError::InvalidToAddress); + to_balance = Some(0) } - // TODO: - // make check for sending funds > 0 so reject any negative values or balances. - // make sure address isn't a duplicate of self. - // need from balance to be greater than or equal to the transfer amount - // need the to_balance to not overflow let updated_from_balance = from_balance.unwrap().checked_sub(msg.value); + if updated_from_balance.is_none() { + return Err(TransferError::InsufficientFunds); + } + let updated_to_balance = to_balance.unwrap().checked_add(msg.value); + if updated_to_balance.is_none() { + return Err(TransferError::TooMuchFunds); + } - match curr_balance { - Some(account) => { - let updated_account = account.clone(); - updated_account.balance.checked_add(amt)?; - self.accts.insert(address, updated_account.clone()); - Some(updated_account.balance) - } - None => { - let new_account = Account::new(amt); - self.accts.insert(address, new_account.clone()); - Some(new_account.balance) - } + // TODO: Below doesn't rollback cases. Fix later. + if !self.state_db.set_balance(&msg.from_address, updated_from_balance.unwrap()) { + return Err(TransferError::StorageError); + } + if !self.state_db.set_balance(&msg.to_address, updated_to_balance.unwrap()) { + return Err(TransferError::StorageError); } + + Ok(()) + } + + pub fn query_balance(&self, address: &Address) -> Option { + self.state_db.get_balance(address) + } + + pub fn test_faucet_address() -> Address { + Address::new(TEST_FAUCET_ADDRESS) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_basic() -> Result<(), String> { + let from_address = Address::new(b"0x10000"); + let to_address = Address::new(b"0x20000"); + let faucet_address = VM::test_faucet_address(); + + let mut test_vm = VM::new_test_vm(); + + // should be empty + let from_balance1 = test_vm.query_balance(&from_address); + assert!(from_balance1.is_none()); + + let faucet_balance = test_vm.query_balance(&VM::test_faucet_address()); + assert_eq!(faucet_balance, Some(TEST_FAUCET_BALANCE)); + + // process faucet transfers to addresses + let transfer1 = Transfer::new(faucet_address.clone(), from_address.clone(), 100); + test_vm.execute(transfer1.unwrap()).unwrap(); + let transfer2 = Transfer::new(faucet_address.clone(), to_address.clone(), 200); + test_vm.execute(transfer2.unwrap()).unwrap(); + + // check balances have been updated from faucet transfers + let from_balance2 = test_vm.query_balance(&from_address); + assert_eq!(from_balance2, Some(100)); + let from_balance3 = test_vm.query_balance(&to_address); + assert_eq!(from_balance3, Some(200)); + + let transfer3 = Transfer::new(from_address.clone(), to_address.clone(), 50); + test_vm.execute(transfer3.unwrap()).unwrap(); + + // check updated balances + let from_balance2 = test_vm.query_balance(&from_address); + assert_eq!(from_balance2, Some(50)); + let from_balance3 = test_vm.query_balance(&to_address); + assert_eq!(from_balance3, Some(250)); + + Ok(()) } } From bb418a45f41ab5f8378392a14388acc1a174dfdb Mon Sep 17 00:00:00 2001 From: NateAtNodeKit Date: Tue, 25 Mar 2025 14:28:40 -0400 Subject: [PATCH 07/38] rocksdb update --- Cargo.lock | 148 +++++++++++++++++++++++++++++++++++++- storage/Cargo.toml | 1 + storage/src/database.rs | 5 ++ storage/src/hashmap_db.rs | 24 +++++-- storage/src/lib.rs | 3 +- storage/src/rocks_db.rs | 51 +++++++++++++ vm/src/vm.rs | 6 +- 7 files changed, 225 insertions(+), 13 deletions(-) create mode 100644 storage/src/rocks_db.rs diff --git a/Cargo.lock b/Cargo.lock index 951dbe1e..f2567793 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -116,6 +116,7 @@ dependencies = [ "alto-types", "commonware-cryptography", "rand", + "rocksdb", ] [[package]] @@ -670,6 +671,26 @@ version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "230c5f1ca6a325a32553f8640d31ac9b49f2411e901e427570154868b46da4f7" +[[package]] +name = "bindgen" +version = "0.69.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "271383c67ccabffb7381723dea0672a673f292304fcb45c01cc648c7a8d58088" +dependencies = [ + "bitflags", + "cexpr", + "clang-sys", + "itertools 0.12.1", + "lazy_static", + "lazycell", + "proc-macro2", + "quote", + "regex", + "rustc-hash", + "shlex", + "syn 2.0.99", +] + [[package]] name = "bitflags" version = "2.9.0" @@ -746,6 +767,16 @@ dependencies = [ "either", ] +[[package]] +name = "bzip2-sys" +version = "0.1.13+1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "225bff33b2141874fe80d71e07d6eec4f85c5c216453dd96388240f96e1acc14" +dependencies = [ + "cc", + "pkg-config", +] + [[package]] name = "cc" version = "1.2.16" @@ -757,6 +788,15 @@ dependencies = [ "shlex", ] +[[package]] +name = "cexpr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" +dependencies = [ + "nom", +] + [[package]] name = "cfg-if" version = "1.0.0" @@ -798,6 +838,17 @@ dependencies = [ "zeroize", ] +[[package]] +name = "clang-sys" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" +dependencies = [ + "glob", + "libc", + "libloading", +] + [[package]] name = "clap" version = "4.5.31" @@ -1979,6 +2030,15 @@ version = "1.70.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" +[[package]] +name = "itertools" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" +dependencies = [ + "either", +] + [[package]] name = "itertools" version = "0.14.0" @@ -2019,18 +2079,60 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +[[package]] +name = "lazycell" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" + [[package]] name = "libc" version = "0.2.170" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "875b3680cb2f8f71bdcf9a30f38d48282f5d3c95cbf9b3fa57269bb5d5c06828" +[[package]] +name = "libloading" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc2f4eb4bc735547cfed7c0a4922cbd04a4655978c09b54f1f7b228750664c34" +dependencies = [ + "cfg-if", + "windows-targets", +] + [[package]] name = "libm" version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8355be11b20d696c8f18f6cc018c4e372165b1fa8126cef092399c9951984ffa" +[[package]] +name = "librocksdb-sys" +version = "0.17.1+9.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b7869a512ae9982f4d46ba482c2a304f1efd80c6412a3d4bf57bb79a619679f" +dependencies = [ + "bindgen", + "bzip2-sys", + "cc", + "libc", + "libz-sys", + "lz4-sys", + "zstd-sys", +] + +[[package]] +name = "libz-sys" +version = "1.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b70e7a7df205e92a1a4cd9aaae7898dac0aa555503cc0a649494d0d60e7651d" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + [[package]] name = "linux-raw-sys" version = "0.9.2" @@ -2059,6 +2161,16 @@ version = "0.4.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "30bde2b3dc3671ae49d8e2e9f044c7c005836e7a023ee57cffa25ab82764bb9e" +[[package]] +name = "lz4-sys" +version = "1.11.1+lz4-1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bd8c0d6c6ed0cd30b3652886bb8711dc4bb01d637a68105a3d5158039b418e6" +dependencies = [ + "cc", + "libc", +] + [[package]] name = "matchit" version = "0.8.4" @@ -2077,6 +2189,12 @@ version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + [[package]] name = "miniz_oxide" version = "0.8.5" @@ -2132,6 +2250,16 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b93853da6d84c2e3c7d730d6473e8817692dd89be387eb01b94d7f108ecb5b8c" +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + [[package]] name = "nonzero_ext" version = "0.3.0" @@ -2457,7 +2585,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" dependencies = [ "heck", - "itertools", + "itertools 0.14.0", "log", "multimap", "once_cell", @@ -2477,7 +2605,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" dependencies = [ "anyhow", - "itertools", + "itertools 0.14.0", "proc-macro2", "quote", "syn 2.0.99", @@ -2709,12 +2837,28 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rocksdb" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26ec73b20525cb235bad420f911473b69f9fe27cc856c5461bccd7e4af037f43" +dependencies = [ + "libc", + "librocksdb-sys", +] + [[package]] name = "rustc-demangle" version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + [[package]] name = "rustc_version" version = "0.4.1" diff --git a/storage/Cargo.toml b/storage/Cargo.toml index 1d99074f..02c6487f 100644 --- a/storage/Cargo.toml +++ b/storage/Cargo.toml @@ -7,3 +7,4 @@ edition = "2024" alto-types = { workspace = true } commonware-cryptography = { workspace = true } rand = "0.8.5" +rocksdb = "0.23.0" diff --git a/storage/src/database.rs b/storage/src/database.rs index 1486787b..08c88ab9 100644 --- a/storage/src/database.rs +++ b/storage/src/database.rs @@ -4,5 +4,10 @@ use crate::account::Balance; // Define database interface that will be used for all impls pub trait Database { fn get_balance(&self, address: &Address) -> Option; + fn set_balance(&mut self, address: &Address, amt: u64) -> bool; + + fn put(&mut self, key: &[u8], value: &[u8]) -> Result<(), Box>; + + fn get(&self, key: &[u8]) -> Result>, Box>; } \ No newline at end of file diff --git a/storage/src/hashmap_db.rs b/storage/src/hashmap_db.rs index a845e657..6a7bb979 100644 --- a/storage/src/hashmap_db.rs +++ b/storage/src/hashmap_db.rs @@ -3,21 +3,23 @@ use alto_types::Address; use crate::account::{Account, Balance}; use crate::database::Database; -pub struct HashmapDb { - accts: HashMap, +pub struct HashmapDatabase { + accounts: HashMap, + data: HashMap, } -impl HashmapDb { +impl HashmapDatabase { pub fn new() -> Self { Self { - accts: HashMap::new(), + accounts: HashMap::new(), + data: HashMap::new(), } } } -impl Database for HashmapDb { +impl Database for HashmapDatabase { fn get_balance(&self, address: &Address) -> Option { - match self.accts.get(&address) { + match self.accounts.get(&address) { Some(acct) => Some(acct.balance), None => None, } @@ -28,7 +30,15 @@ impl Database for HashmapDb { let mut updated_account = Account::new(); updated_account.address = address.clone(); updated_account.balance = amt; - self.accts.insert(address.clone(), updated_account); + self.accounts.insert(address.clone(), updated_account); true } + + fn put(&mut self, key: &[u8], value: &[u8]) -> Result<(), Box> { + self.data.insert(String::from_utf8(key.into())?, value.into()); + } + + fn get(&self, key: &[u8]) -> Result>, Box> { + + } } diff --git a/storage/src/lib.rs b/storage/src/lib.rs index 0b99d3b6..53447579 100644 --- a/storage/src/lib.rs +++ b/storage/src/lib.rs @@ -1,3 +1,4 @@ pub mod account; pub mod database; -pub mod hashmap_db; \ No newline at end of file +pub mod hashmap_db; +mod rocks_db; \ No newline at end of file diff --git a/storage/src/rocks_db.rs b/storage/src/rocks_db.rs new file mode 100644 index 00000000..276e7a94 --- /dev/null +++ b/storage/src/rocks_db.rs @@ -0,0 +1,51 @@ +use alto_types::Address; +use rocksdb::{DB, Options}; +use crate::account::{Account, Balance}; +use crate::database::Database; +use std::path::Path; + +const SAL_ROCKS_DB_PATH: &str = "rocksdb"; + +pub struct RocksDbDatabase { + db: DB, +} + +impl RocksDbDatabase { + pub fn new() -> Result> { + let mut opts = Options::default(); + opts.create_if_missing(true); + + let db_path = Path::new(SAL_ROCKS_DB_PATH); + let db = DB::open(&opts, &db_path)?; + Ok(RocksDbDatabase { db }) + } + + pub fn put(&self, key: &[u8], value: &[u8]) -> Result<(), Box> { + self.db.put(key, value)?; + Ok(()) + } + + pub fn get(&self, key: &[u8]) -> Result>, Box> { + let result = self.db.get(key)?; + Ok(result) + } +} + +impl Database for RocksDbDatabase { + fn get_balance(&self, address: &Address) -> Option { + match self.accts.get(&address) { + Some(acct) => Some(acct.balance), + None => None, + } + } + + fn set_balance(&mut self, address: &Address, amt: Balance) -> bool { + // TODO: Add in public key update later + let mut updated_account = Account::new(); + updated_account.address = address.clone(); + updated_account.balance = amt; + self.accts.insert(address.clone(), updated_account); + true + } +} +*/ \ No newline at end of file diff --git a/vm/src/vm.rs b/vm/src/vm.rs index 06a4c417..8859df3a 100644 --- a/vm/src/vm.rs +++ b/vm/src/vm.rs @@ -1,6 +1,6 @@ use alto_actions::transfer::{Transfer, TransferError}; use alto_storage::database::Database; -use alto_storage::hashmap_db::HashmapDb; +use alto_storage::hashmap_db::HashmapDatabase; use alto_storage::account::Balance; use alto_types::Address; @@ -14,12 +14,12 @@ struct VM { impl VM { pub fn new() -> Self { Self { - state_db: Box::new(HashmapDb::new()), + state_db: Box::new(HashmapDatabase::new()), } } pub fn new_test_vm() -> Self { - let mut state_db = Box::new(HashmapDb::new()); + let mut state_db = Box::new(HashmapDatabase::new()); state_db.set_balance(&Self::test_faucet_address(), TEST_FAUCET_BALANCE); Self { From 8813a1630dcda1c79b2ed4004257d0f833a46c3d Mon Sep 17 00:00:00 2001 From: rikoeldon <106416799+rikoeldon@users.noreply.github.com> Date: Tue, 25 Mar 2025 16:40:58 -0400 Subject: [PATCH 08/38] added to rocksDB --- Cargo.lock | 20 ++++++++++ storage/Cargo.toml | 2 + storage/src/account.rs | 38 +++++++++++++++++++ storage/src/database.rs | 11 ++++-- storage/src/rocks_db.rs | 84 ++++++++++++++++++++++++++++++++--------- types/Cargo.toml | 1 + types/src/codec.rs | 21 +++++------ types/src/lib.rs | 8 ++++ 8 files changed, 153 insertions(+), 32 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f2567793..f109dbcf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -114,6 +114,8 @@ name = "alto-storage" version = "0.1.0" dependencies = [ "alto-types", + "bytes", + "commonware-codec", "commonware-cryptography", "rand", "rocksdb", @@ -124,6 +126,7 @@ name = "alto-types" version = "0.0.4" dependencies = [ "bytes", + "commonware-codec", "commonware-cryptography", "commonware-utils", "getrandom 0.2.15", @@ -882,6 +885,17 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" +[[package]] +name = "commonware-codec" +version = "0.0.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65ebb5fc2fbf7d3fbdaf8d3dd3afdb80f03ee02a7d5a9fefc979235a2c06569a" +dependencies = [ + "bytes", + "paste", + "thiserror 2.0.12", +] + [[package]] name = "commonware-consensus" version = "0.0.40" @@ -2432,6 +2446,12 @@ dependencies = [ "windows-targets", ] +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + [[package]] name = "pem-rfc7468" version = "0.7.0" diff --git a/storage/Cargo.toml b/storage/Cargo.toml index 02c6487f..b5ea0322 100644 --- a/storage/Cargo.toml +++ b/storage/Cargo.toml @@ -8,3 +8,5 @@ alto-types = { workspace = true } commonware-cryptography = { workspace = true } rand = "0.8.5" rocksdb = "0.23.0" +commonware-codec = { version = "0.0.43" } +bytes = "1.7.1" \ No newline at end of file diff --git a/storage/src/account.rs b/storage/src/account.rs index 53e31da5..c313959f 100644 --- a/storage/src/account.rs +++ b/storage/src/account.rs @@ -1,4 +1,6 @@ +use commonware_codec::{Codec, Error, Reader, Writer}; use commonware_cryptography::{ed25519::PublicKey, Ed25519, Scheme}; +use commonware_cryptography::ed25519::PrivateKey; use alto_types::Address; use rand::rngs::OsRng; @@ -8,6 +10,7 @@ pub type Balance = u64; pub struct Account { pub address: Address, pub public_key: PublicKey, + pub private_key: PrivateKey, pub balance: Balance, } @@ -19,13 +22,48 @@ impl Account { let public_key = keypair.public_key(); + let private_key = keypair.private_key(); + // takes addr from pk (truncating or hashing if necessary) let address = Address::new(&public_key.as_ref()[..32]); Self { address, public_key, + private_key, balance: 0, } } +} +impl Codec for Account { + fn write(&self, writer: &mut impl Writer) { + let _ = self.address.write(writer); + self.balance.write(writer); + let len: u32 = self.public_key.as_ref().len() as u32; + len.write(writer); + let _ = self.public_key.as_ref().write(writer); + let priv_len: u32 = self.private_key.as_ref().len() as u32; + priv_len.write(writer); + let _ = self.private_key.as_ref().write(writer); + } + + fn read(reader: &mut impl Reader) -> Result { + let addr = <[u8; 32]>::read(reader)?; + let balance = ::read(reader)?; + // setting up public key + let pk_len = u32::read(reader)?; + let pk_bytes = reader.read_n_bytes(pk_len as usize)?; + let public_key = PublicKey::try_from(pk_bytes).unwrap(); + // setting up private key + let priv_key_len = u32::read(reader)?; + let priv_key_bytes = reader.read_n_bytes(priv_key_len as usize)?; + let private_key = PrivateKey::try_from(priv_key_bytes).unwrap(); + let address = Address::try_from(addr).unwrap(); + + Ok(Self{ address, public_key, private_key, balance}) + } + + fn len_encoded(&self) -> usize { + todo!() + } } \ No newline at end of file diff --git a/storage/src/database.rs b/storage/src/database.rs index 08c88ab9..2023e797 100644 --- a/storage/src/database.rs +++ b/storage/src/database.rs @@ -1,13 +1,18 @@ use alto_types::Address; -use crate::account::Balance; +use crate::account::{Balance, Account}; +use std::error::Error; // Define database interface that will be used for all impls pub trait Database { + fn get_account(&self, address: &Address) -> Option; + + fn set_account(&mut self, acc: &Account) -> Result<(), Box>; + fn get_balance(&self, address: &Address) -> Option; fn set_balance(&mut self, address: &Address, amt: u64) -> bool; - fn put(&mut self, key: &[u8], value: &[u8]) -> Result<(), Box>; + fn put(&mut self, key: &[u8], value: &[u8]) -> Result<(), Box>; - fn get(&self, key: &[u8]) -> Result>, Box>; + fn get(&self, key: &[u8]) -> Result>, Box>; } \ No newline at end of file diff --git a/storage/src/rocks_db.rs b/storage/src/rocks_db.rs index 276e7a94..5e922aa3 100644 --- a/storage/src/rocks_db.rs +++ b/storage/src/rocks_db.rs @@ -1,17 +1,24 @@ +use std::error::Error; use alto_types::Address; use rocksdb::{DB, Options}; +use commonware_codec::{Codec, ReadBuffer, WriteBuffer}; use crate::account::{Account, Balance}; use crate::database::Database; use std::path::Path; +use bytes::{BufMut, Bytes}; + const SAL_ROCKS_DB_PATH: &str = "rocksdb"; +const ACCOUNTS_PREFIX: &[u8] = b"sal_accounts"; + +const WRITE_BUFFER_CAPACITY: u64 = 500; pub struct RocksDbDatabase { db: DB, } impl RocksDbDatabase { - pub fn new() -> Result> { + pub fn new() -> Result> { let mut opts = Options::default(); opts.create_if_missing(true); @@ -20,32 +27,75 @@ impl RocksDbDatabase { Ok(RocksDbDatabase { db }) } - pub fn put(&self, key: &[u8], value: &[u8]) -> Result<(), Box> { - self.db.put(key, value)?; - Ok(()) + pub fn key_accounts(addr: &Address) -> Vec { + Self::make_multi_key(ACCOUNTS_PREFIX, addr.as_slice()) } - pub fn get(&self, key: &[u8]) -> Result>, Box> { - let result = self.db.get(key)?; - Ok(result) + fn make_multi_key(prefix: &[u8], sub_id: &[u8]) -> Vec { + let mut key = Vec::with_capacity(prefix.len() + sub_id.len() + 1); + key.extend_from_slice(prefix); + key.push(b':'); + key.extend_from_slice(sub_id); + key } } impl Database for RocksDbDatabase { + + fn get_account(&self, address: &Address) -> Result, Box> { + let key = Self::key_accounts(address); + let result = self.get(key.as_slice())?; + match result { + None => Ok(None), + Some(value) => { + let bytes = Bytes::copy_from_slice(&value); + let mut read_buf = ReadBuffer::new(bytes); + let acc = Account::read(&mut read_buf)?; + Ok(Some(acc)) + } + } + } + + fn set_account(&mut self, acc: &Account) -> Result<(), Box> { + let key = Self::key_accounts(&acc.address); + let mut write_buf = WriteBuffer::new(WRITE_BUFFER_CAPACITY as usize); + acc.write(&mut write_buf); + self.put(&key, write_buf.as_ref())?; + Ok(()) + } + fn get_balance(&self, address: &Address) -> Option { - match self.accts.get(&address) { - Some(acct) => Some(acct.balance), + let result = self.get_account(address).and_then(|acc| { + match acc { + Some(acc) => Ok(acc.balance), + None => Ok(0), + } + }); + match result { + Some(balance) => Some(balance), None => None, } } fn set_balance(&mut self, address: &Address, amt: Balance) -> bool { - // TODO: Add in public key update later - let mut updated_account = Account::new(); - updated_account.address = address.clone(); - updated_account.balance = amt; - self.accts.insert(address.clone(), updated_account); - true + let result = self.get_account(address); + match result { + Some(mut acc) => { + acc.balance = amt; + let result = self.set_account(&acc); + result.is_ok() + }, + None => false, + } } -} -*/ \ No newline at end of file + + fn put(&self, key: &[u8], value: &[u8]) -> Result<(), Box> { + self.db.put(key, value)?; + Ok(()) + } + + fn get(&self, key: &[u8]) -> Result>, Box> { + let result = self.db.get(key)?; + Ok(result) + } +} \ No newline at end of file diff --git a/types/Cargo.toml b/types/Cargo.toml index d45e2773..b6b721b3 100644 --- a/types/Cargo.toml +++ b/types/Cargo.toml @@ -23,6 +23,7 @@ wasm-bindgen = "0.2.100" serde = { version = "1.0.219", features = ["derive"] } serde-wasm-bindgen = "0.6.5" more-asserts = "0.3.1" +commonware-codec = { version = "0.0.43" } # Enable "js" feature when WASM is target [target.'cfg(target_arch = "wasm32")'.dependencies.getrandom] diff --git a/types/src/codec.rs b/types/src/codec.rs index 583acc66..79120e97 100644 --- a/types/src/codec.rs +++ b/types/src/codec.rs @@ -1,13 +1,10 @@ -pub enum Codec { +use commonware_codec::{Codec, Error, Reader, Writer}; +use commonware_cryptography::{ed25519::PublicKey, Ed25519, Scheme}; +use commonware_cryptography::ed25519::PrivateKey; +use serde::Serialize; -} -/* -const ADDRESSLEN: usize = 33; -struct Address([u8;ADDRESSLEN]); - -impl Address { - pub fn new() -> Self { - Self([0;ADDRESSLEN]) - } -} - */ \ No newline at end of file +pub fn serialize_pk(pk: &PublicKey, writer: &mut impl Writer) { + // let slice: &[u8] = pk.as_ref(); + // let length = slice.len(); + // length.serialize(&mut *writer).unwrap(); +} \ No newline at end of file diff --git a/types/src/lib.rs b/types/src/lib.rs index be687dae..6c608813 100644 --- a/types/src/lib.rs +++ b/types/src/lib.rs @@ -37,6 +37,14 @@ impl Address { pub fn is_empty(&self) -> bool { self.0 == Self::empty().0 } + + pub fn as_slice(&self) -> &[u8] { + &self.0 + } + + pub fn as_bytes(&self) -> &[u8;ADDRESSLEN] { + &self.0 + } } #[cfg(test)] From 70d07bb58a8c5161790f7a674524cfbb74311bb7 Mon Sep 17 00:00:00 2001 From: NateAtNodeKit Date: Wed, 26 Mar 2025 13:17:19 -0400 Subject: [PATCH 09/38] db refactoring --- storage/src/account.rs | 47 +++++++------------------- storage/src/database.rs | 69 +++++++++++++++++++++++++++++++++++++++ storage/src/hashmap_db.rs | 41 +++++++++++++---------- storage/src/rocks_db.rs | 11 ++++--- types/src/lib.rs | 10 +++++- 5 files changed, 119 insertions(+), 59 deletions(-) diff --git a/storage/src/account.rs b/storage/src/account.rs index c313959f..dca506a2 100644 --- a/storage/src/account.rs +++ b/storage/src/account.rs @@ -1,4 +1,4 @@ -use commonware_codec::{Codec, Error, Reader, Writer}; +use commonware_codec::{Codec, Error, Reader, SizedCodec, Writer}; use commonware_cryptography::{ed25519::PublicKey, Ed25519, Scheme}; use commonware_cryptography::ed25519::PrivateKey; use alto_types::Address; @@ -9,61 +9,38 @@ pub type Balance = u64; #[derive(Clone, Debug)] pub struct Account { pub address: Address, - pub public_key: PublicKey, - pub private_key: PrivateKey, pub balance: Balance, } impl Account { pub fn new() -> Self { - let mut rng = OsRng; - // generates keypair using random number generator - let keypair = Ed25519::new(&mut rng); - - let public_key = keypair.public_key(); - - let private_key = keypair.private_key(); - - // takes addr from pk (truncating or hashing if necessary) - let address = Address::new(&public_key.as_ref()[..32]); + Self { + address: Address::empty(), + balance: 0, + } + } + pub fn from_address(address: Address) -> Self { Self { address, - public_key, - private_key, balance: 0, } } } impl Codec for Account { fn write(&self, writer: &mut impl Writer) { - let _ = self.address.write(writer); + writer.write_bytes(self.address.0.as_slice()); self.balance.write(writer); - let len: u32 = self.public_key.as_ref().len() as u32; - len.write(writer); - let _ = self.public_key.as_ref().write(writer); - let priv_len: u32 = self.private_key.as_ref().len() as u32; - priv_len.write(writer); - let _ = self.private_key.as_ref().write(writer); } fn read(reader: &mut impl Reader) -> Result { - let addr = <[u8; 32]>::read(reader)?; + let addr_bytes = <[u8; 32]>::read(reader)?; + let address = Address::from_bytes(&addr_bytes).unwrap(); let balance = ::read(reader)?; - // setting up public key - let pk_len = u32::read(reader)?; - let pk_bytes = reader.read_n_bytes(pk_len as usize)?; - let public_key = PublicKey::try_from(pk_bytes).unwrap(); - // setting up private key - let priv_key_len = u32::read(reader)?; - let priv_key_bytes = reader.read_n_bytes(priv_key_len as usize)?; - let private_key = PrivateKey::try_from(priv_key_bytes).unwrap(); - let address = Address::try_from(addr).unwrap(); - - Ok(Self{ address, public_key, private_key, balance}) + Ok(Self{address, balance}) } fn len_encoded(&self) -> usize { - todo!() + Codec::len_encoded(&self.address.0) + Codec::len_encoded(&self.balance) } } \ No newline at end of file diff --git a/storage/src/database.rs b/storage/src/database.rs index 2023e797..68744de3 100644 --- a/storage/src/database.rs +++ b/storage/src/database.rs @@ -1,9 +1,16 @@ use alto_types::Address; use crate::account::{Balance, Account}; use std::error::Error; +use bytes::Bytes; +use commonware_codec::{Codec, ReadBuffer, WriteBuffer}; + +const ACCOUNTS_PREFIX: &[u8] = b"sal_accounts"; + +const DB_WRITE_BUFFER_CAPACITY: usize = 500; // Define database interface that will be used for all impls pub trait Database { + /* fn get_account(&self, address: &Address) -> Option; fn set_account(&mut self, acc: &Account) -> Result<(), Box>; @@ -12,6 +19,68 @@ pub trait Database { fn set_balance(&mut self, address: &Address, amt: u64) -> bool; + */ + + + fn get_account(&self, address: &Address) -> Result, Box> { + let key = Self::key_accounts(address); + let result = self.get(key.as_slice())?; + match result { + None => Ok(None), + Some(value) => { + let bytes = Bytes::copy_from_slice(&value); + let mut read_buf = ReadBuffer::new(bytes); + let acc = Account::read(&mut read_buf)?; + Ok(Some(acc)) + } + } + } + + fn set_account(&mut self, acc: &Account) -> Result<(), Box> { + let key = Self::key_accounts(&acc.address); + let mut write_buf = WriteBuffer::new(DB_WRITE_BUFFER_CAPACITY); + acc.write(&mut write_buf); + self.put(&key, write_buf.as_ref())?; + Ok(()) + } + + fn get_balance(&self, address: &Address) -> Option { + let result = self.get_account(address).and_then(|acc| { + match acc { + Some(acc) => Ok(acc.balance), + None => Ok(0), + } + }); + match result { + Ok(balance) => Some(balance), + _ => None, + } + } + + fn set_balance(&mut self, address: &Address, amt: Balance) -> bool { + let result = self.get_account(address); + match result { + Ok(Some(mut acc)) => { + acc.balance = amt; + let result = self.set_account(&acc); + result.is_ok() + }, + _ => false, + } + } + + fn key_accounts(addr: &Address) -> Vec { + Self::make_multi_key(ACCOUNTS_PREFIX, addr.as_slice()) + } + + fn make_multi_key(prefix: &[u8], sub_id: &[u8]) -> Vec { + let mut key = Vec::with_capacity(prefix.len() + sub_id.len() + 1); + key.extend_from_slice(prefix); + key.push(b':'); + key.extend_from_slice(sub_id); + key + } + fn put(&mut self, key: &[u8], value: &[u8]) -> Result<(), Box>; fn get(&self, key: &[u8]) -> Result>, Box>; diff --git a/storage/src/hashmap_db.rs b/storage/src/hashmap_db.rs index 6a7bb979..1684e89a 100644 --- a/storage/src/hashmap_db.rs +++ b/storage/src/hashmap_db.rs @@ -4,41 +4,46 @@ use crate::account::{Account, Balance}; use crate::database::Database; pub struct HashmapDatabase { - accounts: HashMap, data: HashMap, } impl HashmapDatabase { pub fn new() -> Self { Self { - accounts: HashMap::new(), data: HashMap::new(), } } } impl Database for HashmapDatabase { - fn get_balance(&self, address: &Address) -> Option { - match self.accounts.get(&address) { - Some(acct) => Some(acct.balance), - None => None, + fn put(&mut self, key: &[u8], value: &[u8]) -> Result<(), Box> { + //self.data.insert(String::from_utf8(key.into())?, value.into()) + let str_value: String = String::from_utf8(value.into()).unwrap(); + match self.data.insert(String::from_utf8(key.into())?, str_value) { + Some(_) => Ok(()), + None => Err("hashmap db insert failed".into()), } } - fn set_balance(&mut self, address: &Address, amt: Balance) -> bool { - // TODO: Add in public key update later - let mut updated_account = Account::new(); - updated_account.address = address.clone(); - updated_account.balance = amt; - self.accounts.insert(address.clone(), updated_account); - true - } - - fn put(&mut self, key: &[u8], value: &[u8]) -> Result<(), Box> { - self.data.insert(String::from_utf8(key.into())?, value.into()); + fn get(&self, key: &[u8]) -> Result>, Box> { + let str_key: String = String::from_utf8(key.into()).unwrap(); + self.data.get(&str_key).map_or( + Ok(None), + |v| Ok(Some(v.clone().into()))) } +} - fn get(&self, key: &[u8]) -> Result>, Box> { +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn test_hashmap_db() { + let mut db = HashmapDatabase::new(); + let key = b"key1"; + let value = b"value1"; + db.put(key, value).unwrap(); + let retrieved = db.get(key).unwrap().unwrap(); + assert_eq!(retrieved.as_slice(), value); } } diff --git a/storage/src/rocks_db.rs b/storage/src/rocks_db.rs index 5e922aa3..a8af5a63 100644 --- a/storage/src/rocks_db.rs +++ b/storage/src/rocks_db.rs @@ -9,9 +9,6 @@ use bytes::{BufMut, Bytes}; const SAL_ROCKS_DB_PATH: &str = "rocksdb"; -const ACCOUNTS_PREFIX: &[u8] = b"sal_accounts"; - -const WRITE_BUFFER_CAPACITY: u64 = 500; pub struct RocksDbDatabase { db: DB, @@ -27,6 +24,7 @@ impl RocksDbDatabase { Ok(RocksDbDatabase { db }) } + /* pub fn key_accounts(addr: &Address) -> Vec { Self::make_multi_key(ACCOUNTS_PREFIX, addr.as_slice()) } @@ -38,10 +36,11 @@ impl RocksDbDatabase { key.extend_from_slice(sub_id); key } + */ } impl Database for RocksDbDatabase { - +/* fn get_account(&self, address: &Address) -> Result, Box> { let key = Self::key_accounts(address); let result = self.get(key.as_slice())?; @@ -89,7 +88,9 @@ impl Database for RocksDbDatabase { } } - fn put(&self, key: &[u8], value: &[u8]) -> Result<(), Box> { + */ + + fn put(&mut self, key: &[u8], value: &[u8]) -> Result<(), Box> { self.db.put(key, value)?; Ok(()) } diff --git a/types/src/lib.rs b/types/src/lib.rs index 6c608813..a03ed207 100644 --- a/types/src/lib.rs +++ b/types/src/lib.rs @@ -20,7 +20,7 @@ pub const FINALIZE_NAMESPACE: &[u8] = b"_ALTO_FINALIZE"; const ADDRESSLEN: usize = 32; #[derive(Hash, Eq, PartialEq, Clone, Debug)] -pub struct Address([u8;ADDRESSLEN]); +pub struct Address(pub [u8;ADDRESSLEN]); impl Address { pub fn new(slice: &[u8]) -> Self { @@ -30,6 +30,14 @@ impl Address { Address(arr) } + pub fn from_bytes(bytes: &[u8]) -> Result { + if bytes.len() != 32 { + return Err("Address must be 32 bytes."); + } + + Ok(Address(<[u8; 32]>::try_from(bytes.clone()).unwrap())) + } + pub fn empty() -> Self { Self([0;ADDRESSLEN]) } From 701b2c5de5fe12c8be77a22c53a328d70447e056 Mon Sep 17 00:00:00 2001 From: NateAtNodeKit Date: Wed, 26 Mar 2025 17:45:14 -0400 Subject: [PATCH 10/38] fixes for db --- .gitignore | 3 +- Cargo.lock | 1 + storage/Cargo.toml | 3 +- storage/src/account.rs | 4 +- storage/src/hashmap_db.rs | 11 ++-- storage/src/rocks_db.rs | 118 ++++++++++++++++---------------------- types/src/lib.rs | 15 +++++ 7 files changed, 78 insertions(+), 77 deletions(-) diff --git a/.gitignore b/.gitignore index 9c5ef416..2ae10116 100644 --- a/.gitignore +++ b/.gitignore @@ -4,4 +4,5 @@ *.vscode /chain/assets /types/pkg -.idea \ No newline at end of file +/storage/rocksdb +.idea diff --git a/Cargo.lock b/Cargo.lock index f109dbcf..fabca603 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -119,6 +119,7 @@ dependencies = [ "commonware-cryptography", "rand", "rocksdb", + "tempfile", ] [[package]] diff --git a/storage/Cargo.toml b/storage/Cargo.toml index b5ea0322..7685351c 100644 --- a/storage/Cargo.toml +++ b/storage/Cargo.toml @@ -9,4 +9,5 @@ commonware-cryptography = { workspace = true } rand = "0.8.5" rocksdb = "0.23.0" commonware-codec = { version = "0.0.43" } -bytes = "1.7.1" \ No newline at end of file +bytes = "1.7.1" +tempfile = "3.18.0" \ No newline at end of file diff --git a/storage/src/account.rs b/storage/src/account.rs index dca506a2..a6d46ab6 100644 --- a/storage/src/account.rs +++ b/storage/src/account.rs @@ -34,8 +34,8 @@ impl Codec for Account { } fn read(reader: &mut impl Reader) -> Result { - let addr_bytes = <[u8; 32]>::read(reader)?; - let address = Address::from_bytes(&addr_bytes).unwrap(); + let addr_bytes = <[u8; 33]>::read(reader)?; + let address = Address::from_bytes(&addr_bytes[1..]).unwrap(); let balance = ::read(reader)?; Ok(Self{address, balance}) } diff --git a/storage/src/hashmap_db.rs b/storage/src/hashmap_db.rs index 1684e89a..6b2ce71b 100644 --- a/storage/src/hashmap_db.rs +++ b/storage/src/hashmap_db.rs @@ -17,12 +17,11 @@ impl HashmapDatabase { impl Database for HashmapDatabase { fn put(&mut self, key: &[u8], value: &[u8]) -> Result<(), Box> { - //self.data.insert(String::from_utf8(key.into())?, value.into()) - let str_value: String = String::from_utf8(value.into()).unwrap(); - match self.data.insert(String::from_utf8(key.into())?, str_value) { - Some(_) => Ok(()), - None => Err("hashmap db insert failed".into()), - } + let str_value: String = String::from_utf8(value.into())?; + let key_value: String = String::from_utf8(key.into())?; + + self.data.insert(key_value, str_value); + Ok(()) } fn get(&self, key: &[u8]) -> Result>, Box> { diff --git a/storage/src/rocks_db.rs b/storage/src/rocks_db.rs index a8af5a63..68c00fc0 100644 --- a/storage/src/rocks_db.rs +++ b/storage/src/rocks_db.rs @@ -1,12 +1,10 @@ use std::error::Error; -use alto_types::Address; use rocksdb::{DB, Options}; -use commonware_codec::{Codec, ReadBuffer, WriteBuffer}; -use crate::account::{Account, Balance}; +use commonware_codec::{Codec}; use crate::database::Database; use std::path::Path; -use bytes::{BufMut, Bytes}; - +use bytes::{BufMut}; +use tempfile::TempDir; const SAL_ROCKS_DB_PATH: &str = "rocksdb"; @@ -16,87 +14,73 @@ pub struct RocksDbDatabase { impl RocksDbDatabase { pub fn new() -> Result> { + Self::new_with_path(SAL_ROCKS_DB_PATH) + } + + pub fn new_with_path(path: &str) -> Result> { let mut opts = Options::default(); opts.create_if_missing(true); - let db_path = Path::new(SAL_ROCKS_DB_PATH); + let db_path = Path::new(path); let db = DB::open(&opts, &db_path)?; Ok(RocksDbDatabase { db }) } - /* - pub fn key_accounts(addr: &Address) -> Vec { - Self::make_multi_key(ACCOUNTS_PREFIX, addr.as_slice()) + pub fn new_tmp_db() -> Result> { + let temp_dir = TempDir::new()?; + let db_path = temp_dir.path().join("testdb"); + Self::new_with_path(db_path.to_str().unwrap()) } - fn make_multi_key(prefix: &[u8], sub_id: &[u8]) -> Vec { - let mut key = Vec::with_capacity(prefix.len() + sub_id.len() + 1); - key.extend_from_slice(prefix); - key.push(b':'); - key.extend_from_slice(sub_id); - key - } - */ } impl Database for RocksDbDatabase { -/* - fn get_account(&self, address: &Address) -> Result, Box> { - let key = Self::key_accounts(address); - let result = self.get(key.as_slice())?; - match result { - None => Ok(None), - Some(value) => { - let bytes = Bytes::copy_from_slice(&value); - let mut read_buf = ReadBuffer::new(bytes); - let acc = Account::read(&mut read_buf)?; - Ok(Some(acc)) - } - } - } - - fn set_account(&mut self, acc: &Account) -> Result<(), Box> { - let key = Self::key_accounts(&acc.address); - let mut write_buf = WriteBuffer::new(WRITE_BUFFER_CAPACITY as usize); - acc.write(&mut write_buf); - self.put(&key, write_buf.as_ref())?; + fn put(&mut self, key: &[u8], value: &[u8]) -> Result<(), Box> { + self.db.put(key, value)?; Ok(()) } - fn get_balance(&self, address: &Address) -> Option { - let result = self.get_account(address).and_then(|acc| { - match acc { - Some(acc) => Ok(acc.balance), - None => Ok(0), - } - }); - match result { - Some(balance) => Some(balance), - None => None, - } + fn get(&self, key: &[u8]) -> Result>, Box> { + let result = self.db.get(key)?; + Ok(result) } +} - fn set_balance(&mut self, address: &Address, amt: Balance) -> bool { - let result = self.get_account(address); - match result { - Some(mut acc) => { - acc.balance = amt; - let result = self.set_account(&acc); - result.is_ok() - }, - None => false, - } +#[cfg(test)] +mod tests { + use alto_types::Address; + use crate::account::Account; + use super::*; + #[test] + fn test_rocks_db_basic() { + let mut db = RocksDbDatabase::new().expect("db could not be created"); + let key = b"key1"; + let value = b"value1"; + db.put(key, value).unwrap(); + let retrieved = db.get(key).unwrap().unwrap(); + assert_eq!(retrieved.as_slice(), value); } - */ + #[test] + fn test_rocks_db_accounts() { + let mut db = RocksDbDatabase::new_tmp_db().expect("db could not be created"); - fn put(&mut self, key: &[u8], value: &[u8]) -> Result<(), Box> { - self.db.put(key, value)?; - Ok(()) - } + let mut account = Account::new(); + let test_address = Address::new(b"0xBEEF"); + account.address = test_address.clone(); + account.balance = 100; - fn get(&self, key: &[u8]) -> Result>, Box> { - let result = self.db.get(key)?; - Ok(result) + // get account for test address is empty + let empty_result = db.get_account(&test_address); + empty_result.unwrap().is_none(); + + // set account + db.set_account(&account).unwrap(); + + let acct_result = db.get_account(&test_address).unwrap(); + assert!(acct_result.is_some()); + let account = acct_result.unwrap(); + assert_eq!(account.address, test_address); + assert_eq!(account.balance, 100); } -} \ No newline at end of file +} diff --git a/types/src/lib.rs b/types/src/lib.rs index a03ed207..b9beda3c 100644 --- a/types/src/lib.rs +++ b/types/src/lib.rs @@ -1,6 +1,9 @@ //! Common types used throughout `alto`. mod block; + +use commonware_cryptography::{Ed25519, Scheme}; +use commonware_cryptography::ed25519::{PrivateKey, PublicKey}; pub use block::{Block, Finalized, Notarized}; mod consensus; pub use consensus::{leader_index, Finalization, Kind, Notarization, Nullification, Seed}; @@ -8,6 +11,7 @@ pub mod wasm; mod codec; use more_asserts; use more_asserts::assert_le; +use rand::rngs::OsRng; // We don't use functions here to guard against silent changes. pub const NAMESPACE: &[u8] = b"_ALTO"; @@ -55,6 +59,17 @@ impl Address { } } +pub fn create_test_keypair() -> (PublicKey, PrivateKey) { + let mut rng = OsRng; + // generates keypair using random number generator + let keypair = Ed25519::new(&mut rng); + + let public_key = keypair.public_key(); + let private_key = keypair.private_key(); + + (public_key, private_key) +} + #[cfg(test)] mod tests { use super::*; From 71a7172ad4ba5b6abb6a816653e7be00d3ef262f Mon Sep 17 00:00:00 2001 From: NateAtNodeKit Date: Thu, 27 Mar 2025 14:24:08 -0400 Subject: [PATCH 11/38] router scaffold --- Cargo.lock | 1 + chain/Cargo.toml | 1 + chain/src/actors/application/actor.rs | 1 + chain/src/actors/application/mod.rs | 2 + chain/src/actors/application/router.rs | 105 +++++++++++++++++++++++++ chain/src/engine.rs | 3 + 6 files changed, 113 insertions(+) create mode 100644 chain/src/actors/application/router.rs diff --git a/Cargo.lock b/Cargo.lock index fabca603..26ed7941 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -70,6 +70,7 @@ dependencies = [ "serde_yaml", "sysinfo", "thiserror 2.0.12", + "tokio", "tracing", "tracing-subscriber", "uuid", diff --git a/chain/Cargo.toml b/chain/Cargo.toml index 81ee85ea..e59b1946 100644 --- a/chain/Cargo.toml +++ b/chain/Cargo.toml @@ -37,6 +37,7 @@ axum = "0.8.1" uuid = "1.15.1" serde = { version = "1.0.218", features = ["derive"] } serde_yaml = "0.9.34" +tokio = "1.44.0" [[bin]] name = "validator" diff --git a/chain/src/actors/application/actor.rs b/chain/src/actors/application/actor.rs index 89094f93..c70cf1b3 100644 --- a/chain/src/actors/application/actor.rs +++ b/chain/src/actors/application/actor.rs @@ -89,6 +89,7 @@ impl Actor { let genesis_digest = genesis.digest(); let built: Option = None; let built = Arc::new(Mutex::new(built)); + while let Some(message) = self.mailbox.next().await { match message { Message::Genesis { response } => { diff --git a/chain/src/actors/application/mod.rs b/chain/src/actors/application/mod.rs index 70f2dcbc..72bb076a 100644 --- a/chain/src/actors/application/mod.rs +++ b/chain/src/actors/application/mod.rs @@ -10,6 +10,8 @@ pub use actor::Actor; mod ingress; pub use ingress::Mailbox; mod supervisor; +mod router; + pub use supervisor::Supervisor; /// Configuration for the application. diff --git a/chain/src/actors/application/router.rs b/chain/src/actors/application/router.rs new file mode 100644 index 00000000..c6975526 --- /dev/null +++ b/chain/src/actors/application/router.rs @@ -0,0 +1,105 @@ +use std::io; +use axum::response::IntoResponse; +use axum::routing::get; +use commonware_runtime::{Clock, Handle, Metrics, Spawner}; +use rand::Rng; +use tokio::net::TcpListener; +use tracing::{event, Level}; +use crate::actors::syncer; + +pub struct RouterConfig { + port: i32, +} + +impl RouterConfig { + pub fn default_config() -> RouterConfig { + RouterConfig { + port: 7844, + } + } +} + +pub struct Router { + context: R, + cfg: RouterConfig, + listener: Option, + router: Option, + is_active: bool, +} + +impl Router { + const PATH_SUBMIT_BLOCK: &'static str = "/builder/submit"; + + pub fn new(context: R, cfg: RouterConfig) -> Self { + if cfg.port == 0 { + panic!("Invalid port number"); + } + + Router { + context, + cfg, + listener: None, + router: None, + is_active: false, + } + } + + pub async fn start(mut self) -> Handle<()> { + self.context.spawn_ref()(self.run()) + } + + pub fn stop(&self) { + if !self.is_active { + return + } + + event!(Level::INFO, "stopped router service"); + } + + async fn init_listener(&mut self) -> io::Result { + let listener = TcpListener::bind(format!("127.0.0.1:{}", self.cfg.port)).await?; + Ok(listener) + } + + async fn handle_default() -> impl IntoResponse { + "Hello world!" + } + + async fn handle_submit_block() -> impl IntoResponse { + "Submit block" + } + + fn init_router(&mut self) { + let router = axum::Router::new() + .route("/", get(Router::handle_default)) + .route(Router::PATH_SUBMIT_BLOCK, get(Router::handle_submit_block())); + + self.router = Some(router) + } + + async fn serve(&mut self) -> Result<(), Box> { + let listener = self.listener.take().ok_or("serve failed because listener is None"); + let router = self.router.take().ok_or("serve failed because router is None"); + axum::serve(listener.unwrap(), router.unwrap()).await?; + Ok(()) + } + + async fn run(mut self) { + event!(Level::INFO, "starting router service"); + + let listener_res = self.init_listener(); + match listener_res.await { + Ok(value) => self.listener = Some(value), + Err(error) => { + println!("Error during listener init: {}", error); + return + }, + } + + self.init_router(); + self.serve().await.unwrap(); + self.is_active = true; + + event!(Level::INFO, "finished starting router service"); + } +} diff --git a/chain/src/engine.rs b/chain/src/engine.rs index 2bb8dabc..096065a1 100644 --- a/chain/src/engine.rs +++ b/chain/src/engine.rs @@ -201,6 +201,9 @@ impl + Metri // Start consensus let consensus_handle = self.consensus.start(voter_network, resolver_network); + // Start the router + let router_config = RouterConfig::default_config(); + // Wait for any actor to finish if let Err(e) = try_join_all(vec![application_handle, syncer_handle, consensus_handle]).await From ab18e5412aaeb244ef491923162700df0e73f194 Mon Sep 17 00:00:00 2001 From: rikoeldon <106416799+rikoeldon@users.noreply.github.com> Date: Thu, 27 Mar 2025 15:04:52 -0400 Subject: [PATCH 12/38] init types for SAL --- chain/src/engine.rs | 2 +- storage/src/database.rs | 12 ------ storage/src/lib.rs | 3 +- storage/src/tx_state_view.rs | 37 +++++++++++++++++ types/src/lib.rs | 5 +++ types/src/signed_tx.rs | 26 ++++++++++++ types/src/state_db.rs | 11 +++++ types/src/tx.rs | 78 ++++++++++++++++++++++++++++++++++++ types/src/wallet.rs | 37 +++++++++++++++++ 9 files changed, 197 insertions(+), 14 deletions(-) create mode 100644 storage/src/tx_state_view.rs create mode 100644 types/src/signed_tx.rs create mode 100644 types/src/state_db.rs create mode 100644 types/src/tx.rs create mode 100644 types/src/wallet.rs diff --git a/chain/src/engine.rs b/chain/src/engine.rs index 096065a1..8536fbd6 100644 --- a/chain/src/engine.rs +++ b/chain/src/engine.rs @@ -202,7 +202,7 @@ impl + Metri let consensus_handle = self.consensus.start(voter_network, resolver_network); // Start the router - let router_config = RouterConfig::default_config(); + // let router_config = RouterConfig::default_config(); // Wait for any actor to finish if let Err(e) = diff --git a/storage/src/database.rs b/storage/src/database.rs index 68744de3..e600426b 100644 --- a/storage/src/database.rs +++ b/storage/src/database.rs @@ -10,18 +10,6 @@ const DB_WRITE_BUFFER_CAPACITY: usize = 500; // Define database interface that will be used for all impls pub trait Database { - /* - fn get_account(&self, address: &Address) -> Option; - - fn set_account(&mut self, acc: &Account) -> Result<(), Box>; - - fn get_balance(&self, address: &Address) -> Option; - - fn set_balance(&mut self, address: &Address, amt: u64) -> bool; - - */ - - fn get_account(&self, address: &Address) -> Result, Box> { let key = Self::key_accounts(address); let result = self.get(key.as_slice())?; diff --git a/storage/src/lib.rs b/storage/src/lib.rs index 53447579..6b6c512a 100644 --- a/storage/src/lib.rs +++ b/storage/src/lib.rs @@ -1,4 +1,5 @@ pub mod account; pub mod database; pub mod hashmap_db; -mod rocks_db; \ No newline at end of file +mod rocks_db; +mod tx_state_view; \ No newline at end of file diff --git a/storage/src/tx_state_view.rs b/storage/src/tx_state_view.rs new file mode 100644 index 00000000..0d2ef71e --- /dev/null +++ b/storage/src/tx_state_view.rs @@ -0,0 +1,37 @@ +use std::collections::HashMap; +use crate::database::Database; +use std::error::Error; + +type Key<'a> = &'a [u8;33]; // 1st byte denotes the type of key. 0b for account key, 1b for others. + +pub enum OpAction { + Read, // key was read + Create, // key was created + Update, // key was updated + Delete, // key got deleted +} + +pub struct Op<'a>{ + pub action: OpAction, + pub key: Key<'a>, + pub value: Vec, +} + +pub struct TxStateView<'a, DB: Database> { + pub cache: HashMap, Vec>, // key-value, state view cache before tx execution. This is not an exhaustive list of all state keys read/write during tx. If cache misses occur, the state view will read from the underlying storage. + pub ops: Vec>, // list of state ops applied. + pub touched: HashMap, Vec>, // key-value pairs that were changed during tx execution. This is a subset of the cache. + pub db: DB, // underlying state storage, to use when cache misses occur. +} + + +pub trait TxStateViewTrait { + fn new(db: DB) -> Self; // create a new instance of TxStateView. + fn init_cache(&mut self, cache: HashMap>); // initialize the cache with an already available hashmap of key-value pairs. + fn get(&mut self, key: Key) -> Result>, Box>; // get a key from the state view. If the key is not in the touched and cache, it will read from the underlying storage. add the key-value pair to `touched` and append op to `ops`. + fn get_from_cache(&self, key: Key) -> Result>, Box>; // get a key from the cache. If the key is not in the cache, it will return an error. + fn get_from_state(&self, key: Key) -> Result>, Box>; // get a key from the underlying storage. If the key is not in the storage, it will return an error. + fn update(&mut self, key: Key, value: Vec) -> Result<(), Box>; // update a key in the state view. add the key-value pair to `touched` and append op to `ops`. + fn delete(&mut self, key: Key) -> Result<(), Box>; // delete a key in the state view. add the key-value pair to `touched` and append op to `ops`. + fn rollback(&mut self); // rollback the state view to the state before the tx execution. +} diff --git a/types/src/lib.rs b/types/src/lib.rs index b9beda3c..4b675f12 100644 --- a/types/src/lib.rs +++ b/types/src/lib.rs @@ -9,6 +9,11 @@ mod consensus; pub use consensus::{leader_index, Finalization, Kind, Notarization, Nullification, Seed}; pub mod wasm; mod codec; +mod wallet; +mod tx; +mod signed_tx; +mod state_db; + use more_asserts; use more_asserts::assert_le; use rand::rngs::OsRng; diff --git a/types/src/signed_tx.rs b/types/src/signed_tx.rs new file mode 100644 index 00000000..eb0c9b0b --- /dev/null +++ b/types/src/signed_tx.rs @@ -0,0 +1,26 @@ +use crate::Address; +use crate::wallet::{AuthTypes, Wallet}; +use crate::tx::{Tx, Unit}; + +// this is sent by the user to the validators. +pub struct SignedTx<'a> { + pub tx: Tx<'a>, + pub auth_type: AuthTypes, + + + pub_key: Vec, + address: Address, + signature: Vec, +} + + +pub trait SignedTxChars { + fn new(tx: Tx, auth_type: AuthTypes) -> Self; + fn sign(&self, wallet: Wallet) -> SignedTx; + fn verify(&self) -> bool; + fn signature(&self) -> Vec; + fn public_key(&self) -> Vec; + fn address(&self) -> Address; + fn encode(&self) -> Vec; + fn decode(&self, bytes: &[u8]) -> Self; +} diff --git a/types/src/state_db.rs b/types/src/state_db.rs new file mode 100644 index 00000000..6ec7e9bd --- /dev/null +++ b/types/src/state_db.rs @@ -0,0 +1,11 @@ +use std::error::Error; + +type Key<'a> = &'a [u8;33]; // 1st byte denotes the type of key. 0b for account key, 1b for others. +pub trait StateDB { + fn get(&self, key: Key) -> Result>, Box>; + fn get_multi_key(&self, key: Vec) -> Result>, Box>; + fn update(&mut self, key: Key, value: Vec) -> Result<(), Box>; + fn delete(&mut self, key: Key) -> Result<(), Box>; + fn commit(&mut self) -> Result<(), Box>; + fn rollback(&mut self) -> Result<(), Box>; +} \ No newline at end of file diff --git a/types/src/tx.rs b/types/src/tx.rs new file mode 100644 index 00000000..ba09ae2d --- /dev/null +++ b/types/src/tx.rs @@ -0,0 +1,78 @@ +use std::error::Error; +use crate::Address; +use crate::state_db::StateDB; +use crate::wallet::Wallet; +use crate::signed_tx::SignedTx; + +pub enum UnitType { + Transfer, + SequencerMsg, +} + + +pub struct UnitContext { + pub timestamp: u64, // timestamp of the tx. + pub chain_id: u64, // chain id of the tx. + pub sender: Address, // sender of the tx. +} + + +// unit need to be simple and easy to be packed in the tx and executed by the vm. +pub trait Unit { + // type DB: Database; + // type T: TxStateViewTrait; + + + fn unit_type(&self) -> UnitType; // return the unit type. + fn encode(&self) -> Vec; // encode the unit. + fn decode(bytes: &[u8]) -> Self; // decode the unit. + fn apply(&self, context: &UnitContext, state: &mut dyn StateDB) -> Result>, Box >; // apply the unit to the state view. +} + + +// pub struct Tx<'a, U: Unit> { +pub struct Tx<'a> { + // timestamp of the tx creation. set by the user. + // will be verified if the tx is in the valid window once received by validators. + // if the timestamp is not in the valid window, the tx will be rejected. + // if tx is in a valid window it is added to mempool. + // timestamp is used to prevent replay attacks. and counter infinite spam attacks as Tx does not have nonce. + pub timestamp: u64, + // units are fundamental unit of a tx. similar to actions. + pub units: Vec, + // max fee is the maximum fee the user is willing to pay for the tx. + pub max_fee: u64, + // priority fee is the fee the user is willing to pay for the tx to be included in the next block. + pub priority_fee: u64, + // chain id is the id of the chain the tx is intended for. + pub chain_id: u64, + + + // id is the transaction id. It is the hash of digest. + id: &'a [u8;32], + // digest is encoded tx. + digest: Vec, +} + + +pub trait TxChars { + // init is used to create a new instance of Tx. + fn init() -> Self; + // new is used to create a new instance of Tx with given units and chain id. + fn new(units: Vec, chain_id: u64) -> Self; + // set_fee is used to set the max fee and priority fee of the tx. + fn set_fee(&mut self, max_fee: u64, priority_fee: u64); + // sign is used to sign the tx with the given wallet. + fn sign(&self, wallet: Wallet) -> SignedTx; + + + // returns tx id. + fn id(&self) -> &[u8;32]; + // returns digest of the tx. + fn digest(&self) -> Vec; + // encodes the tx, writes to digest and returns the digest. + fn encode(&mut self) -> Vec; + + + fn decode(bytes: &[u8]) -> Self; +} diff --git a/types/src/wallet.rs b/types/src/wallet.rs new file mode 100644 index 00000000..2e93bfef --- /dev/null +++ b/types/src/wallet.rs @@ -0,0 +1,37 @@ +use crate::Address; +pub enum AuthTypes { + ED25519, +} + + +// auth should have a method to verify signatures. +// also batch signature verification. +pub trait Auth { + fn public_key(&self) -> Vec; // return the public key of the signer. + fn address(&self) -> Address; // return the account address of the signer. + fn verify(&self, data: &[u8], signature: &[u8]) -> bool; // verify a signature. + fn batch_verify(&self, data: &[u8], signatures: Vec<&[u8]>) -> bool; // batch verify signatures. returns error if batch verification fails. +} + + +pub struct Wallet { + auth: AuthTypes, // auth type + p_key: Vec, // private key + pub_key: Vec, // public key + address: Address, // account address +} + + +// wallet generation, management, and signing should be functions of the wallet. +pub trait ImplWallet { + fn generate(auth_type: AuthTypes) -> Self; // create a new wallet of given auth type. + fn load(&self, auth_type: AuthTypes, p_key: &[u8]) -> Self; // load a new wallet from private key. + fn sign(&self, data: &[u8]) -> Vec; // sign data with the private key. + fn verify(&self,data: &[u8], signature: &[u8]) -> bool; // verify a signature with the public key. + fn address(&self) -> Address; // return the account address. + fn public_key(&self) -> Vec; // return the public key. + fn auth_type(&self) -> AuthTypes; // return the auth type. + fn private_key(&self) -> Vec; // returns the private key. + fn store_private_key(&self); // store the private key. + fn new_address(&mut self); // generate a new address. +} From 6883900d4b67f311df9cc533ad13b48f0ab78edd Mon Sep 17 00:00:00 2001 From: rikoeldon <106416799+rikoeldon@users.noreply.github.com> Date: Thu, 27 Mar 2025 16:22:26 -0400 Subject: [PATCH 13/38] state & DB setup --- storage/src/database.rs | 68 ++------------------------- storage/src/hashmap_db.rs | 7 ++- storage/src/lib.rs | 4 +- storage/src/rocks_db.rs | 3 +- storage/src/state_db.rs | 75 +++++++++++++++++++++++++++++ storage/src/tx_state_view.rs | 78 ++++++++++++++++++++++++------- {storage => types}/src/account.rs | 7 +-- types/src/address.rs | 38 +++++++++++++++ types/src/lib.rs | 51 ++++---------------- types/src/signed_tx.rs | 8 ++-- types/src/state.rs | 11 +++++ types/src/state_db.rs | 11 ----- types/src/tx.rs | 4 +- types/src/wallet.rs | 10 ++-- vm/src/vm.rs | 2 +- 15 files changed, 222 insertions(+), 155 deletions(-) create mode 100644 storage/src/state_db.rs rename {storage => types}/src/account.rs (80%) create mode 100644 types/src/address.rs create mode 100644 types/src/state.rs delete mode 100644 types/src/state_db.rs diff --git a/storage/src/database.rs b/storage/src/database.rs index e600426b..87425cda 100644 --- a/storage/src/database.rs +++ b/storage/src/database.rs @@ -1,75 +1,13 @@ -use alto_types::Address; -use crate::account::{Balance, Account}; +use alto_types::account::{Balance, Account}; use std::error::Error; use bytes::Bytes; use commonware_codec::{Codec, ReadBuffer, WriteBuffer}; - -const ACCOUNTS_PREFIX: &[u8] = b"sal_accounts"; - -const DB_WRITE_BUFFER_CAPACITY: usize = 500; +use alto_types::address::Address; // Define database interface that will be used for all impls pub trait Database { - fn get_account(&self, address: &Address) -> Result, Box> { - let key = Self::key_accounts(address); - let result = self.get(key.as_slice())?; - match result { - None => Ok(None), - Some(value) => { - let bytes = Bytes::copy_from_slice(&value); - let mut read_buf = ReadBuffer::new(bytes); - let acc = Account::read(&mut read_buf)?; - Ok(Some(acc)) - } - } - } - - fn set_account(&mut self, acc: &Account) -> Result<(), Box> { - let key = Self::key_accounts(&acc.address); - let mut write_buf = WriteBuffer::new(DB_WRITE_BUFFER_CAPACITY); - acc.write(&mut write_buf); - self.put(&key, write_buf.as_ref())?; - Ok(()) - } - - fn get_balance(&self, address: &Address) -> Option { - let result = self.get_account(address).and_then(|acc| { - match acc { - Some(acc) => Ok(acc.balance), - None => Ok(0), - } - }); - match result { - Ok(balance) => Some(balance), - _ => None, - } - } - - fn set_balance(&mut self, address: &Address, amt: Balance) -> bool { - let result = self.get_account(address); - match result { - Ok(Some(mut acc)) => { - acc.balance = amt; - let result = self.set_account(&acc); - result.is_ok() - }, - _ => false, - } - } - - fn key_accounts(addr: &Address) -> Vec { - Self::make_multi_key(ACCOUNTS_PREFIX, addr.as_slice()) - } - - fn make_multi_key(prefix: &[u8], sub_id: &[u8]) -> Vec { - let mut key = Vec::with_capacity(prefix.len() + sub_id.len() + 1); - key.extend_from_slice(prefix); - key.push(b':'); - key.extend_from_slice(sub_id); - key - } - fn put(&mut self, key: &[u8], value: &[u8]) -> Result<(), Box>; fn get(&self, key: &[u8]) -> Result>, Box>; + fn delete(&mut self, key: &[u8]) -> Result<(), Box>; } \ No newline at end of file diff --git a/storage/src/hashmap_db.rs b/storage/src/hashmap_db.rs index 6b2ce71b..23f256f8 100644 --- a/storage/src/hashmap_db.rs +++ b/storage/src/hashmap_db.rs @@ -1,6 +1,5 @@ use std::collections::HashMap; -use alto_types::Address; -use crate::account::{Account, Balance}; +use std::error::Error; use crate::database::Database; pub struct HashmapDatabase { @@ -30,6 +29,10 @@ impl Database for HashmapDatabase { Ok(None), |v| Ok(Some(v.clone().into()))) } + + fn delete(&mut self, key: &[u8]) -> Result<(), Box> { + todo!() + } } diff --git a/storage/src/lib.rs b/storage/src/lib.rs index 6b6c512a..ae0cda68 100644 --- a/storage/src/lib.rs +++ b/storage/src/lib.rs @@ -1,5 +1,5 @@ -pub mod account; pub mod database; pub mod hashmap_db; mod rocks_db; -mod tx_state_view; \ No newline at end of file +mod tx_state_view; +mod state_db; \ No newline at end of file diff --git a/storage/src/rocks_db.rs b/storage/src/rocks_db.rs index 68c00fc0..f559289f 100644 --- a/storage/src/rocks_db.rs +++ b/storage/src/rocks_db.rs @@ -49,7 +49,8 @@ impl Database for RocksDbDatabase { #[cfg(test)] mod tests { use alto_types::Address; - use crate::account::Account; + use alto_types::account::Account; + use alto_types::address::Address; use super::*; #[test] fn test_rocks_db_basic() { diff --git a/storage/src/state_db.rs b/storage/src/state_db.rs new file mode 100644 index 00000000..e5c124f8 --- /dev/null +++ b/storage/src/state_db.rs @@ -0,0 +1,75 @@ +use crate::database::Database; +use alto_types::account::{Account, Balance}; +use alto_types::address::Address; +use bytes::Bytes; +use commonware_codec::{Codec, ReadBuffer, WriteBuffer}; +use std::error::Error; + +const ACCOUNTS_PREFIX: &[u8] = b"sal_accounts"; +const DB_WRITE_BUFFER_CAPACITY: usize = 500; + +pub struct StateDb { + db: Box, +} +// can use like a redis from Arcadia like get and set for diff types? +impl StateDb { + pub fn new(db: Box) -> StateDb { + StateDb { db } + } + + pub fn get_account(&self, address: &Address) -> Result, Box> { + let key = Self::key_accounts(address); + let result = self.db.get(key.as_slice())?; + match result { + None => Ok(None), + Some(value) => { + let bytes = Bytes::copy_from_slice(&value); + let mut read_buf = ReadBuffer::new(bytes); + let acc = Account::read(&mut read_buf)?; + Ok(Some(acc)) + } + } + } + + pub fn set_account(&mut self, acc: &Account) -> Result<(), Box> { + let key = Self::key_accounts(&acc.address); + let mut write_buf = WriteBuffer::new(DB_WRITE_BUFFER_CAPACITY); + acc.write(&mut write_buf); + self.db.put(&key, write_buf.as_ref()).expect("TODO: panic message"); + Ok(()) + } + + pub fn get_balance(&self, address: &Address) -> Option { + let result = self.get_account(address).and_then(|acc| match acc { + Some(acc) => Ok(acc.balance), + None => Ok(0), + }); + match result { + Ok(balance) => Some(balance), + _ => None, + } + } + + pub fn set_balance(&mut self, address: &Address, amt: Balance) -> bool { + let result = self.get_account(address); + match result { + Ok(Some(mut acc)) => { + acc.balance = amt; + let result = self.set_account(&acc); + result.is_ok() + } + _ => false, + } + } + + fn key_accounts(addr: &Address) -> Vec { + Self::make_multi_key(ACCOUNTS_PREFIX, addr.as_slice()) + } + fn make_multi_key(prefix: &[u8], sub_id: &[u8]) -> Vec { + let mut key = Vec::with_capacity(prefix.len() + sub_id.len() + 1); + key.extend_from_slice(prefix); + key.push(b':'); + key.extend_from_slice(sub_id); + key + } +} diff --git a/storage/src/tx_state_view.rs b/storage/src/tx_state_view.rs index 0d2ef71e..29ef5af8 100644 --- a/storage/src/tx_state_view.rs +++ b/storage/src/tx_state_view.rs @@ -1,8 +1,12 @@ use std::collections::HashMap; use crate::database::Database; use std::error::Error; +use rocksdb::LogLevel::Error; +use alto_types::state::{State}; +use crate::state_db::StateDb; -type Key<'a> = &'a [u8;33]; // 1st byte denotes the type of key. 0b for account key, 1b for others. +const ACCOUNT_KEY_TYPE: u8 = 0; +type UnitKey<'a> = alto_types::state::UnitKey<'a>; // 1st byte denotes the type of key. 0b for account key, 1b for others. pub enum OpAction { Read, // key was read @@ -13,25 +17,67 @@ pub enum OpAction { pub struct Op<'a>{ pub action: OpAction, - pub key: Key<'a>, + pub key: UnitKey<'a>, pub value: Vec, } -pub struct TxStateView<'a, DB: Database> { - pub cache: HashMap, Vec>, // key-value, state view cache before tx execution. This is not an exhaustive list of all state keys read/write during tx. If cache misses occur, the state view will read from the underlying storage. +pub trait TxStateViewTrait: State { + fn init_cache(&mut self, cache: HashMap>); // initialize the cache with an already available hashmap of key-value pairs. + fn get_from_cache(&self, key: UnitKey) -> Result>, Box>; // get a key from the cache. If the key is not in the cache, it will return an error. + fn get_from_state(&self, key: UnitKey) -> Result>, Box>; // get a key from the underlying storage. If the key is not in the storage, it will return an error. +} + +pub struct TxStateView<'a> { + pub cache: HashMap, Vec>, // key-value, state view cache before tx execution. This is not an exhaustive list of all state keys read/write during tx. If cache misses occur, the state view will read from the underlying storage. pub ops: Vec>, // list of state ops applied. - pub touched: HashMap, Vec>, // key-value pairs that were changed during tx execution. This is a subset of the cache. - pub db: DB, // underlying state storage, to use when cache misses occur. + pub touched: HashMap, Vec>, // key-value pairs that were changed during tx execution. This is a subset of the cache. + pub state_db: StateDb, // underlying state storage, to use when cache misses occur. } +impl<'a> TxStateView<'a> { + pub fn new(state_db: StateDb) -> Self { + Self{ + cache: HashMap::new(), + ops: Vec::new(), + touched: HashMap::new(), + state_db, + } + } + // initialize the cache with an already available hashmap of key-value pairs. + pub fn init_cache(&mut self, cache: HashMap>) { + self.cache = cache; + } + pub fn get_from_cache(&self, key: UnitKey) -> Result>, Box> { + self.cache.get(&key).map_or( + Ok(None), + |v| Ok(Some(v.clone().into()))) + } -pub trait TxStateViewTrait { - fn new(db: DB) -> Self; // create a new instance of TxStateView. - fn init_cache(&mut self, cache: HashMap>); // initialize the cache with an already available hashmap of key-value pairs. - fn get(&mut self, key: Key) -> Result>, Box>; // get a key from the state view. If the key is not in the touched and cache, it will read from the underlying storage. add the key-value pair to `touched` and append op to `ops`. - fn get_from_cache(&self, key: Key) -> Result>, Box>; // get a key from the cache. If the key is not in the cache, it will return an error. - fn get_from_state(&self, key: Key) -> Result>, Box>; // get a key from the underlying storage. If the key is not in the storage, it will return an error. - fn update(&mut self, key: Key, value: Vec) -> Result<(), Box>; // update a key in the state view. add the key-value pair to `touched` and append op to `ops`. - fn delete(&mut self, key: Key) -> Result<(), Box>; // delete a key in the state view. add the key-value pair to `touched` and append op to `ops`. - fn rollback(&mut self); // rollback the state view to the state before the tx execution. -} + pub fn get_from_state(&self, key: UnitKey) -> Result>, Box> { + match key[0] { + ACCOUNT_KEY_TYPE => { + self.get_from_state(key) + }, + _ => Err(format!("invalid state key {:?}", key[0]).into()) + } + } + + pub fn get(&self, key: alto_types::state::UnitKey) -> Result>, Box>{ + todo!() + } + pub fn get_multi_key(&self, key: Vec) -> Result>, Box> { + todo!() + } + pub fn update(&mut self, key: alto_types::state::UnitKey, value: Vec) -> Result<(), Box>{ + todo!() + } + pub fn delete(&mut self, key: alto_types::state::UnitKey) -> Result<(), Box> { + todo!() + } + pub fn commit(&mut self) -> Result<(), Box> { + todo!() + } + pub fn rollback(&mut self) -> Result<(), Box> { + todo!() + } +} \ No newline at end of file diff --git a/storage/src/account.rs b/types/src/account.rs similarity index 80% rename from storage/src/account.rs rename to types/src/account.rs index a6d46ab6..fb4faea5 100644 --- a/storage/src/account.rs +++ b/types/src/account.rs @@ -1,8 +1,5 @@ -use commonware_codec::{Codec, Error, Reader, SizedCodec, Writer}; -use commonware_cryptography::{ed25519::PublicKey, Ed25519, Scheme}; -use commonware_cryptography::ed25519::PrivateKey; -use alto_types::Address; -use rand::rngs::OsRng; +use commonware_codec::{Codec, Error, Reader, Writer}; +use crate::address::Address; pub type Balance = u64; diff --git a/types/src/address.rs b/types/src/address.rs new file mode 100644 index 00000000..53aff458 --- /dev/null +++ b/types/src/address.rs @@ -0,0 +1,38 @@ +use more_asserts::assert_le; +use crate::ADDRESSLEN; + +#[derive(Hash, Eq, PartialEq, Clone, Debug)] +pub struct Address(pub [u8;ADDRESSLEN]); + +impl Address { + pub fn new(slice: &[u8]) -> Self { + assert_le!(slice.len(), ADDRESSLEN, "address slice is too large"); + let mut arr = [0u8; ADDRESSLEN]; + arr[..slice.len()].copy_from_slice(slice); + Address(arr) + } + + pub fn from_bytes(bytes: &[u8]) -> Result { + if bytes.len() != 32 { + return Err("Address must be 32 bytes."); + } + + Ok(Address(<[u8; 32]>::try_from(bytes.clone()).unwrap())) + } + + pub fn empty() -> Self { + Self([0;ADDRESSLEN]) + } + + pub fn is_empty(&self) -> bool { + self.0 == Self::empty().0 + } + + pub fn as_slice(&self) -> &[u8] { + &self.0 + } + + pub fn as_bytes(&self) -> &[u8;ADDRESSLEN] { + &self.0 + } +} \ No newline at end of file diff --git a/types/src/lib.rs b/types/src/lib.rs index 4b675f12..5dd5075e 100644 --- a/types/src/lib.rs +++ b/types/src/lib.rs @@ -3,19 +3,19 @@ mod block; use commonware_cryptography::{Ed25519, Scheme}; -use commonware_cryptography::ed25519::{PrivateKey, PublicKey}; pub use block::{Block, Finalized, Notarized}; mod consensus; pub use consensus::{leader_index, Finalization, Kind, Notarization, Nullification, Seed}; pub mod wasm; -mod codec; -mod wallet; -mod tx; -mod signed_tx; -mod state_db; +pub mod codec; +pub mod wallet; +pub mod tx; +pub mod signed_tx; +pub mod state; +pub mod address; +pub mod account; use more_asserts; -use more_asserts::assert_le; use rand::rngs::OsRng; // We don't use functions here to guard against silent changes. @@ -28,41 +28,8 @@ pub const FINALIZE_NAMESPACE: &[u8] = b"_ALTO_FINALIZE"; const ADDRESSLEN: usize = 32; -#[derive(Hash, Eq, PartialEq, Clone, Debug)] -pub struct Address(pub [u8;ADDRESSLEN]); - -impl Address { - pub fn new(slice: &[u8]) -> Self { - assert_le!(slice.len(), ADDRESSLEN, "address slice is too large"); - let mut arr = [0u8; ADDRESSLEN]; - arr[..slice.len()].copy_from_slice(slice); - Address(arr) - } - - pub fn from_bytes(bytes: &[u8]) -> Result { - if bytes.len() != 32 { - return Err("Address must be 32 bytes."); - } - - Ok(Address(<[u8; 32]>::try_from(bytes.clone()).unwrap())) - } - - pub fn empty() -> Self { - Self([0;ADDRESSLEN]) - } - - pub fn is_empty(&self) -> bool { - self.0 == Self::empty().0 - } - - pub fn as_slice(&self) -> &[u8] { - &self.0 - } - - pub fn as_bytes(&self) -> &[u8;ADDRESSLEN] { - &self.0 - } -} +type PublicKey = commonware_cryptography::ed25519::PublicKey; +type PrivateKey = commonware_cryptography::ed25519::PrivateKey; pub fn create_test_keypair() -> (PublicKey, PrivateKey) { let mut rng = OsRng; diff --git a/types/src/signed_tx.rs b/types/src/signed_tx.rs index eb0c9b0b..1aec2917 100644 --- a/types/src/signed_tx.rs +++ b/types/src/signed_tx.rs @@ -1,14 +1,14 @@ -use crate::Address; +use crate::address::Address; +use crate::PublicKey; use crate::wallet::{AuthTypes, Wallet}; -use crate::tx::{Tx, Unit}; +use crate::tx::Tx; // this is sent by the user to the validators. pub struct SignedTx<'a> { pub tx: Tx<'a>, pub auth_type: AuthTypes, - - pub_key: Vec, + pub_key: PublicKey, address: Address, signature: Vec, } diff --git a/types/src/state.rs b/types/src/state.rs new file mode 100644 index 00000000..cdd3e5f9 --- /dev/null +++ b/types/src/state.rs @@ -0,0 +1,11 @@ +use std::error::Error; + +pub type UnitKey<'a> = &'a [u8;33]; // 1st byte denotes the type of key. 0b for account key, 1b for others. +pub trait State { + fn get(&self, key: UnitKey) -> Result>, Box>; + fn get_multi_key(&self, key: Vec) -> Result>, Box>; + fn update(&mut self, key: UnitKey, value: Vec) -> Result<(), Box>; + fn delete(&mut self, key: UnitKey) -> Result<(), Box>; + fn commit(&mut self) -> Result<(), Box>; + fn rollback(&mut self) -> Result<(), Box>; +} \ No newline at end of file diff --git a/types/src/state_db.rs b/types/src/state_db.rs deleted file mode 100644 index 6ec7e9bd..00000000 --- a/types/src/state_db.rs +++ /dev/null @@ -1,11 +0,0 @@ -use std::error::Error; - -type Key<'a> = &'a [u8;33]; // 1st byte denotes the type of key. 0b for account key, 1b for others. -pub trait StateDB { - fn get(&self, key: Key) -> Result>, Box>; - fn get_multi_key(&self, key: Vec) -> Result>, Box>; - fn update(&mut self, key: Key, value: Vec) -> Result<(), Box>; - fn delete(&mut self, key: Key) -> Result<(), Box>; - fn commit(&mut self) -> Result<(), Box>; - fn rollback(&mut self) -> Result<(), Box>; -} \ No newline at end of file diff --git a/types/src/tx.rs b/types/src/tx.rs index ba09ae2d..53a251fb 100644 --- a/types/src/tx.rs +++ b/types/src/tx.rs @@ -1,6 +1,6 @@ use std::error::Error; -use crate::Address; -use crate::state_db::StateDB; +use crate::address::Address; +use crate::state::StateDB; use crate::wallet::Wallet; use crate::signed_tx::SignedTx; diff --git a/types/src/wallet.rs b/types/src/wallet.rs index 2e93bfef..106686da 100644 --- a/types/src/wallet.rs +++ b/types/src/wallet.rs @@ -1,4 +1,6 @@ -use crate::Address; +use crate::address::Address; +use crate::{PrivateKey, PublicKey}; + pub enum AuthTypes { ED25519, } @@ -7,7 +9,7 @@ pub enum AuthTypes { // auth should have a method to verify signatures. // also batch signature verification. pub trait Auth { - fn public_key(&self) -> Vec; // return the public key of the signer. + fn public_key(&self) -> PublicKey; // return the public key of the signer. fn address(&self) -> Address; // return the account address of the signer. fn verify(&self, data: &[u8], signature: &[u8]) -> bool; // verify a signature. fn batch_verify(&self, data: &[u8], signatures: Vec<&[u8]>) -> bool; // batch verify signatures. returns error if batch verification fails. @@ -16,8 +18,8 @@ pub trait Auth { pub struct Wallet { auth: AuthTypes, // auth type - p_key: Vec, // private key - pub_key: Vec, // public key + p_key: PrivateKey, // private key + pub_key: PublicKey, // public key address: Address, // account address } diff --git a/vm/src/vm.rs b/vm/src/vm.rs index 8859df3a..e6d65bb1 100644 --- a/vm/src/vm.rs +++ b/vm/src/vm.rs @@ -1,7 +1,7 @@ use alto_actions::transfer::{Transfer, TransferError}; use alto_storage::database::Database; use alto_storage::hashmap_db::HashmapDatabase; -use alto_storage::account::Balance; +use alto_types::account::Balance; use alto_types::Address; const TEST_FAUCET_ADDRESS: &[u8] = b"0x0123456789abcdef0123456789abcd"; From 24b820baae75a217451fbe36da48290618b080fd Mon Sep 17 00:00:00 2001 From: NateAtNodeKit Date: Thu, 27 Mar 2025 18:02:21 -0400 Subject: [PATCH 14/38] state update --- storage/src/database.rs | 3 ++- storage/src/hashmap_db.rs | 6 +++-- storage/src/rocks_db.rs | 31 ++++-------------------- storage/src/state_db.rs | 47 ++++++++++++++++++++++++++++++++++++ storage/src/tx_state_view.rs | 41 +++++++++++++++++++++++++------ types/src/tx.rs | 25 ++++++++++--------- 6 files changed, 105 insertions(+), 48 deletions(-) diff --git a/storage/src/database.rs b/storage/src/database.rs index 87425cda..032938f4 100644 --- a/storage/src/database.rs +++ b/storage/src/database.rs @@ -8,6 +8,7 @@ use alto_types::address::Address; pub trait Database { fn put(&mut self, key: &[u8], value: &[u8]) -> Result<(), Box>; - fn get(&self, key: &[u8]) -> Result>, Box>; + fn get(&self, key: &[u8]) -> Result, Box>; + fn delete(&mut self, key: &[u8]) -> Result<(), Box>; } \ No newline at end of file diff --git a/storage/src/hashmap_db.rs b/storage/src/hashmap_db.rs index 23f256f8..d3396498 100644 --- a/storage/src/hashmap_db.rs +++ b/storage/src/hashmap_db.rs @@ -16,8 +16,8 @@ impl HashmapDatabase { impl Database for HashmapDatabase { fn put(&mut self, key: &[u8], value: &[u8]) -> Result<(), Box> { - let str_value: String = String::from_utf8(value.into())?; let key_value: String = String::from_utf8(key.into())?; + let str_value: String = String::from_utf8(value.into())?; self.data.insert(key_value, str_value); Ok(()) @@ -31,7 +31,9 @@ impl Database for HashmapDatabase { } fn delete(&mut self, key: &[u8]) -> Result<(), Box> { - todo!() + let key_value: String = String::from_utf8(key.into())?; + self.data.remove(&key_value); + Ok(()) } } diff --git a/storage/src/rocks_db.rs b/storage/src/rocks_db.rs index f559289f..5e39d5d7 100644 --- a/storage/src/rocks_db.rs +++ b/storage/src/rocks_db.rs @@ -44,13 +44,15 @@ impl Database for RocksDbDatabase { let result = self.db.get(key)?; Ok(result) } + + fn delete(&mut self, key: &[u8]) -> Result<(), Box> { + self.db.delete(key)?; + Ok(()) + } } #[cfg(test)] mod tests { - use alto_types::Address; - use alto_types::account::Account; - use alto_types::address::Address; use super::*; #[test] fn test_rocks_db_basic() { @@ -61,27 +63,4 @@ mod tests { let retrieved = db.get(key).unwrap().unwrap(); assert_eq!(retrieved.as_slice(), value); } - - #[test] - fn test_rocks_db_accounts() { - let mut db = RocksDbDatabase::new_tmp_db().expect("db could not be created"); - - let mut account = Account::new(); - let test_address = Address::new(b"0xBEEF"); - account.address = test_address.clone(); - account.balance = 100; - - // get account for test address is empty - let empty_result = db.get_account(&test_address); - empty_result.unwrap().is_none(); - - // set account - db.set_account(&account).unwrap(); - - let acct_result = db.get_account(&test_address).unwrap(); - assert!(acct_result.is_some()); - let account = acct_result.unwrap(); - assert_eq!(account.address, test_address); - assert_eq!(account.balance, 100); - } } diff --git a/storage/src/state_db.rs b/storage/src/state_db.rs index e5c124f8..ac72a58a 100644 --- a/storage/src/state_db.rs +++ b/storage/src/state_db.rs @@ -4,6 +4,7 @@ use alto_types::address::Address; use bytes::Bytes; use commonware_codec::{Codec, ReadBuffer, WriteBuffer}; use std::error::Error; +use crate::rocks_db::RocksDbDatabase; const ACCOUNTS_PREFIX: &[u8] = b"sal_accounts"; const DB_WRITE_BUFFER_CAPACITY: usize = 500; @@ -73,3 +74,49 @@ impl StateDb { key } } + +impl Database for StateDb { + fn put(&mut self, key: &[u8], value: &[u8]) -> Result<(), Box> { + self.db.put(key, value) + } + + fn get(&self, key: &[u8]) -> Result>, Box> { + self.db.get(key) + } + + fn delete(&mut self, key: &[u8]) -> Result<(), Box> { + self.db.delete(key) + } +} + +#[cfg(test)] +mod tests { + use alto_types::address::Address; + use alto_types::account::Account; + use alto_types::address::Address; + use super::*; + + #[test] + fn test_rocks_db_accounts() { + let db = RocksDbDatabase::new_tmp_db().expect("db could not be created"); + let mut state_db = StateDb::new(Box::new(db)); + + let mut account = Account::new(); + let test_address = Address::new(b"0xBEEF"); + account.address = test_address.clone(); + account.balance = 100; + + // get account for test address is empty + let empty_result = state_db.get_account(&test_address); + empty_result.unwrap().is_none(); + + // set account + state_db.set_account(&account).unwrap(); + + let acct_result = state_db.get_account(&test_address).unwrap(); + assert!(acct_result.is_some()); + let account = acct_result.unwrap(); + assert_eq!(account.address, test_address); + assert_eq!(account.balance, 100); + } +} \ No newline at end of file diff --git a/storage/src/tx_state_view.rs b/storage/src/tx_state_view.rs index 29ef5af8..6a171111 100644 --- a/storage/src/tx_state_view.rs +++ b/storage/src/tx_state_view.rs @@ -1,13 +1,20 @@ use std::collections::HashMap; -use crate::database::Database; use std::error::Error; -use rocksdb::LogLevel::Error; +use alto_types::address::Address; +use crate::database::Database; use alto_types::state::{State}; use crate::state_db::StateDb; const ACCOUNT_KEY_TYPE: u8 = 0; + type UnitKey<'a> = alto_types::state::UnitKey<'a>; // 1st byte denotes the type of key. 0b for account key, 1b for others. +pub fn decode_unit_key(key: UnitKey) -> (u8, Address) { + let key_type: u8 = key[0]; + let address_bytes: &[u8] = &key[1..]; + (key_type, address_bytes.into()) +} + pub enum OpAction { Read, // key was read Create, // key was created @@ -47,6 +54,7 @@ impl<'a> TxStateView<'a> { pub fn init_cache(&mut self, cache: HashMap>) { self.cache = cache; } + pub fn get_from_cache(&self, key: UnitKey) -> Result>, Box> { self.cache.get(&key).map_or( Ok(None), @@ -54,6 +62,7 @@ impl<'a> TxStateView<'a> { } pub fn get_from_state(&self, key: UnitKey) -> Result>, Box> { + let (key_type, address) match key[0] { ACCOUNT_KEY_TYPE => { self.get_from_state(key) @@ -62,16 +71,16 @@ impl<'a> TxStateView<'a> { } } - pub fn get(&self, key: alto_types::state::UnitKey) -> Result>, Box>{ - todo!() + pub fn get(&self, key: UnitKey) -> Result>, Box>{ } - pub fn get_multi_key(&self, key: Vec) -> Result>, Box> { + + pub fn get_multi_key(&self, key: UnitKey) -> Result>, Box> { todo!() } - pub fn update(&mut self, key: alto_types::state::UnitKey, value: Vec) -> Result<(), Box>{ + pub fn update(&mut self, key: UnitKey, value: Vec) -> Result<(), Box>{ todo!() } - pub fn delete(&mut self, key: alto_types::state::UnitKey) -> Result<(), Box> { + pub fn delete(&mut self, key: UnitKey) -> Result<(), Box> { todo!() } pub fn commit(&mut self) -> Result<(), Box> { @@ -80,4 +89,22 @@ impl<'a> TxStateView<'a> { pub fn rollback(&mut self) -> Result<(), Box> { todo!() } + + pub fn process_get_action(&mut self, cmd_type: u8, key: &UnitKey) -> Result>, Box> { + match cmd_type { + ACCOUNT_KEY_TYPE => { + self.state_db.get(key) + } + _ => Err(format!("invalid state key {:?}", key).into()) + } + } + + pub fn process_put_action(&mut self, cmd_type: u8, key: &UnitKey) -> Result>, Box> { + match cmd_type { + ACCOUNT_KEY_TYPE => { + self.state_db.get(key) + } + _ => Err(format!("invalid state key {:?}", key).into()) + } + } } \ No newline at end of file diff --git a/types/src/tx.rs b/types/src/tx.rs index 53a251fb..05bf2021 100644 --- a/types/src/tx.rs +++ b/types/src/tx.rs @@ -1,8 +1,8 @@ use std::error::Error; use crate::address::Address; -use crate::state::StateDB; use crate::wallet::Wallet; use crate::signed_tx::SignedTx; +use crate::state::State; pub enum UnitType { Transfer, @@ -18,15 +18,16 @@ pub struct UnitContext { // unit need to be simple and easy to be packed in the tx and executed by the vm. -pub trait Unit { - // type DB: Database; - // type T: TxStateViewTrait; - - - fn unit_type(&self) -> UnitType; // return the unit type. - fn encode(&self) -> Vec; // encode the unit. - fn decode(bytes: &[u8]) -> Self; // decode the unit. - fn apply(&self, context: &UnitContext, state: &mut dyn StateDB) -> Result>, Box >; // apply the unit to the state view. +pub trait Unit : Send + Sync { + fn unit_type(&self) -> UnitType; + fn encode(&self) -> Vec; + fn decode(&mut self, bytes: &[u8]); + + fn apply( + &self, + context: &UnitContext, + state: &mut Box, + ) -> Result>, Box>; } @@ -39,7 +40,7 @@ pub struct Tx<'a> { // timestamp is used to prevent replay attacks. and counter infinite spam attacks as Tx does not have nonce. pub timestamp: u64, // units are fundamental unit of a tx. similar to actions. - pub units: Vec, + pub units: Vec>, // max fee is the maximum fee the user is willing to pay for the tx. pub max_fee: u64, // priority fee is the fee the user is willing to pay for the tx to be included in the next block. @@ -59,7 +60,7 @@ pub trait TxChars { // init is used to create a new instance of Tx. fn init() -> Self; // new is used to create a new instance of Tx with given units and chain id. - fn new(units: Vec, chain_id: u64) -> Self; + fn new(units: Vec>, chain_id: u64) -> Self; // set_fee is used to set the max fee and priority fee of the tx. fn set_fee(&mut self, max_fee: u64, priority_fee: u64); // sign is used to sign the tx with the given wallet. From 69135e51f4b67ca00aac675360739fe4f2c7304f Mon Sep 17 00:00:00 2001 From: manojkgorle Date: Fri, 28 Mar 2025 04:05:19 +0000 Subject: [PATCH 15/38] edition2024 to edition2021 --- actions/Cargo.toml | 2 +- storage/Cargo.toml | 2 +- vm/Cargo.toml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/actions/Cargo.toml b/actions/Cargo.toml index 189380eb..60a2bf23 100644 --- a/actions/Cargo.toml +++ b/actions/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "alto-actions" version = "0.1.0" -edition = "2024" +edition = "2021" [dependencies] alto-types = { workspace = true } diff --git a/storage/Cargo.toml b/storage/Cargo.toml index 7685351c..cba09b57 100644 --- a/storage/Cargo.toml +++ b/storage/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "alto-storage" version = "0.1.0" -edition = "2024" +edition = "2021" [dependencies] alto-types = { workspace = true } diff --git a/vm/Cargo.toml b/vm/Cargo.toml index 8e773002..3ef612cc 100644 --- a/vm/Cargo.toml +++ b/vm/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "alto-vm" version = "0.1.0" -edition = "2024" +edition = "2021" [dependencies] alto-storage = { workspace = true } From 13f1a7f4e9738b5239df3ad02f3a25b5a911375f Mon Sep 17 00:00:00 2001 From: manojkgorle Date: Fri, 28 Mar 2025 04:18:42 +0000 Subject: [PATCH 16/38] clippy --- types/src/account.rs | 6 ++++++ types/src/address.rs | 2 +- types/src/codec.rs | 6 ++---- types/src/lib.rs | 1 - types/src/tx.rs | 1 - 5 files changed, 9 insertions(+), 7 deletions(-) diff --git a/types/src/account.rs b/types/src/account.rs index fb4faea5..061b7ca4 100644 --- a/types/src/account.rs +++ b/types/src/account.rs @@ -9,6 +9,12 @@ pub struct Account { pub balance: Balance, } +impl Default for Account { + fn default() -> Self { + Self::new() + } +} + impl Account { pub fn new() -> Self { Self { diff --git a/types/src/address.rs b/types/src/address.rs index 53aff458..594e4d65 100644 --- a/types/src/address.rs +++ b/types/src/address.rs @@ -17,7 +17,7 @@ impl Address { return Err("Address must be 32 bytes."); } - Ok(Address(<[u8; 32]>::try_from(bytes.clone()).unwrap())) + Ok(Address(<[u8; 32]>::try_from(bytes).unwrap())) } pub fn empty() -> Self { diff --git a/types/src/codec.rs b/types/src/codec.rs index 79120e97..d8688a2a 100644 --- a/types/src/codec.rs +++ b/types/src/codec.rs @@ -1,7 +1,5 @@ -use commonware_codec::{Codec, Error, Reader, Writer}; -use commonware_cryptography::{ed25519::PublicKey, Ed25519, Scheme}; -use commonware_cryptography::ed25519::PrivateKey; -use serde::Serialize; +use commonware_codec::Writer; +use commonware_cryptography::ed25519::PublicKey; pub fn serialize_pk(pk: &PublicKey, writer: &mut impl Writer) { // let slice: &[u8] = pk.as_ref(); diff --git a/types/src/lib.rs b/types/src/lib.rs index 5dd5075e..b9e06500 100644 --- a/types/src/lib.rs +++ b/types/src/lib.rs @@ -15,7 +15,6 @@ pub mod state; pub mod address; pub mod account; -use more_asserts; use rand::rngs::OsRng; // We don't use functions here to guard against silent changes. diff --git a/types/src/tx.rs b/types/src/tx.rs index 05bf2021..fb5400a9 100644 --- a/types/src/tx.rs +++ b/types/src/tx.rs @@ -1,4 +1,3 @@ -use std::error::Error; use crate::address::Address; use crate::wallet::Wallet; use crate::signed_tx::SignedTx; From 52bc1adba9c157fbc4c830a856341b553ff42ec1 Mon Sep 17 00:00:00 2001 From: bianyuanop Date: Fri, 28 Mar 2025 14:44:02 -0400 Subject: [PATCH 17/38] mempool-rpc --- Cargo.lock | 134 +++++++++++++++--- chain/Cargo.toml | 4 +- chain/src/actors/application/mod.rs | 1 - chain/src/actors/mempool/mempool.rs | 11 +- chain/src/actors/mod.rs | 1 + chain/src/actors/net/actor.rs | 35 +++++ chain/src/actors/net/ingress.rs | 36 +++++ chain/src/actors/net/mod.rs | 59 ++++++++ .../src/actors/{application => net}/router.rs | 78 +++++++--- 9 files changed, 312 insertions(+), 47 deletions(-) create mode 100644 chain/src/actors/net/actor.rs create mode 100644 chain/src/actors/net/ingress.rs create mode 100644 chain/src/actors/net/mod.rs rename chain/src/actors/{application => net}/router.rs (62%) diff --git a/Cargo.lock b/Cargo.lock index 6211c74e..375768cf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -68,10 +68,12 @@ dependencies = [ "prometheus-client", "rand", "serde", + "serde_bytes", "serde_yaml", "sysinfo", "thiserror 2.0.12", "tokio", + "tower 0.4.13", "tracing", "tracing-subscriber", "uuid", @@ -564,11 +566,12 @@ dependencies = [ [[package]] name = "axum" -version = "0.8.1" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d6fd624c75e18b3b4c6b9caf42b1afe24437daaee904069137d8bab077be8b8" +checksum = "de45108900e1f9b9242f7f2e254aa3e2c029c921c258fe9e6b4217eeebd54288" dependencies = [ "axum-core", + "axum-macros", "bytes", "form_urlencoded", "futures-util", @@ -590,7 +593,7 @@ dependencies = [ "serde_urlencoded", "sync_wrapper", "tokio", - "tower", + "tower 0.5.2", "tower-layer", "tower-service", "tracing", @@ -598,12 +601,12 @@ dependencies = [ [[package]] name = "axum-core" -version = "0.5.0" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df1362f362fd16024ae199c1970ce98f9661bf5ef94b9808fee734bc3698b733" +checksum = "68464cd0412f486726fb3373129ef5d2993f90c34bc2bc1c1e9943b2f4fc7ca6" dependencies = [ "bytes", - "futures-util", + "futures-core", "http 1.2.0", "http-body 1.0.1", "http-body-util", @@ -616,6 +619,17 @@ dependencies = [ "tracing", ] +[[package]] +name = "axum-macros" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "604fde5e028fea851ce1d8570bbdc034bec850d157f7569d10f347d06808c05c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.99", +] + [[package]] name = "backtrace" version = "0.3.74" @@ -888,17 +902,6 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" -[[package]] -name = "commonware-codec" -version = "0.0.43" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65ebb5fc2fbf7d3fbdaf8d3dd3afdb80f03ee02a7d5a9fefc979235a2c06569a" -dependencies = [ - "bytes", - "paste", - "thiserror 2.0.12", -] - [[package]] name = "commonware-broadcast" version = "0.0.43" @@ -921,6 +924,17 @@ dependencies = [ "tracing", ] +[[package]] +name = "commonware-codec" +version = "0.0.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65ebb5fc2fbf7d3fbdaf8d3dd3afdb80f03ee02a7d5a9fefc979235a2c06569a" +dependencies = [ + "bytes", + "paste", + "thiserror 2.0.12", +] + [[package]] name = "commonware-consensus" version = "0.0.43" @@ -1653,7 +1667,7 @@ dependencies = [ "futures-sink", "futures-util", "http 0.2.12", - "indexmap", + "indexmap 2.7.1", "slab", "tokio", "tokio-util", @@ -1672,13 +1686,19 @@ dependencies = [ "futures-core", "futures-sink", "http 1.2.0", - "indexmap", + "indexmap 2.7.1", "slab", "tokio", "tokio-util", "tracing", ] +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + [[package]] name = "hashbrown" version = "0.14.5" @@ -1691,6 +1711,16 @@ version = "0.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289" +[[package]] +name = "hdrhistogram" +version = "7.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "765c9198f173dd59ce26ff9f95ef0aafd0a0fe01fb9d72841bc5066a4c06511d" +dependencies = [ + "byteorder", + "num-traits", +] + [[package]] name = "heck" version = "0.5.0" @@ -2038,6 +2068,16 @@ dependencies = [ "icu_properties", ] +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown 0.12.3", +] + [[package]] name = "indexmap" version = "2.7.1" @@ -2499,7 +2539,27 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772" dependencies = [ "fixedbitset", - "indexmap", + "indexmap 2.7.1", +] + +[[package]] +name = "pin-project" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677f1add503faace112b9f1373e43e9e054bfdd22ff1a63c1bc485eaec6a6a8a" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.99", ] [[package]] @@ -2849,7 +2909,7 @@ dependencies = [ "system-configuration", "tokio", "tokio-native-tls", - "tower", + "tower 0.5.2", "tower-service", "url", "wasm-bindgen", @@ -3108,6 +3168,15 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "serde_bytes" +version = "0.11.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8437fd221bde2d4ca316d61b90e337e9e702b3820b87d63caa9ba6c02bd06d96" +dependencies = [ + "serde", +] + [[package]] name = "serde_derive" version = "1.0.219" @@ -3159,7 +3228,7 @@ version = "0.9.34+deprecated" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" dependencies = [ - "indexmap", + "indexmap 2.7.1", "itoa", "ryu", "serde", @@ -3585,6 +3654,27 @@ dependencies = [ "tokio", ] +[[package]] +name = "tower" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" +dependencies = [ + "futures-core", + "futures-util", + "hdrhistogram", + "indexmap 1.9.3", + "pin-project", + "pin-project-lite", + "rand", + "slab", + "tokio", + "tokio-util", + "tower-layer", + "tower-service", + "tracing", +] + [[package]] name = "tower" version = "0.5.2" diff --git a/chain/Cargo.toml b/chain/Cargo.toml index 2db1361e..c6f81c5e 100644 --- a/chain/Cargo.toml +++ b/chain/Cargo.toml @@ -34,11 +34,13 @@ governor = { workspace = true } prometheus-client = { workspace = true } clap = { workspace = true } sysinfo = "0.33.1" -axum = "0.8.1" +axum = { version = "0.8.1", features = ["macros"] } uuid = "1.15.1" serde = { version = "1.0.218", features = ["derive"] } serde_yaml = "0.9.34" +serde_bytes = "0.11.17" tokio = "1.44.0" +tower = { version = "0.4", features = ["full"] } [[bin]] name = "validator" diff --git a/chain/src/actors/application/mod.rs b/chain/src/actors/application/mod.rs index 72bb076a..54ff87d8 100644 --- a/chain/src/actors/application/mod.rs +++ b/chain/src/actors/application/mod.rs @@ -10,7 +10,6 @@ pub use actor::Actor; mod ingress; pub use ingress::Mailbox; mod supervisor; -mod router; pub use supervisor::Supervisor; diff --git a/chain/src/actors/mempool/mempool.rs b/chain/src/actors/mempool/mempool.rs index a743977e..a199d770 100644 --- a/chain/src/actors/mempool/mempool.rs +++ b/chain/src/actors/mempool/mempool.rs @@ -17,7 +17,7 @@ use rand::Rng; use tracing::{debug, warn, info}; use governor::clock::Clock as GClock; use super::{handler::{Handler, self}, key::{self, MultiIndex, Value}, ingress, coordinator::Coordinator, archive::Wrapped}; -use crate::maybe_delay_between; +use crate::{actors::net, maybe_delay_between}; #[derive(Clone, Debug)] pub struct Batch { @@ -139,6 +139,15 @@ impl RawTransaction } } +impl From for RawTransaction + where Sha256: Hasher +{ + fn from(value: net::router::DummyTransaction) -> Self { + let raw = Bytes::from(value.payload); + RawTransaction::new(raw) + } +} + pub enum Message { // mark batch as accepted by the netowrk through the broadcast protocol BatchAcknowledged { diff --git a/chain/src/actors/mod.rs b/chain/src/actors/mod.rs index 3bac1080..4d8ed485 100644 --- a/chain/src/actors/mod.rs +++ b/chain/src/actors/mod.rs @@ -1,3 +1,4 @@ pub mod application; pub mod syncer; pub mod mempool; +pub mod net; \ No newline at end of file diff --git a/chain/src/actors/net/actor.rs b/chain/src/actors/net/actor.rs new file mode 100644 index 00000000..0dee8764 --- /dev/null +++ b/chain/src/actors/net/actor.rs @@ -0,0 +1,35 @@ +use futures::{channel::mpsc, StreamExt}; +use rand::Rng; +use commonware_runtime::{Clock, Handle, Metrics, Spawner}; + +use super::ingress::{Mailbox, Message}; +use tracing::{debug}; + +pub struct Actor { + mailbox: mpsc::Receiver +} + +impl Actor { + pub fn new() -> (Self, Mailbox) { + let (sender, receiver) = mpsc::channel(1024); + + ( + Actor { + mailbox: receiver, + }, + Mailbox::new(sender) + ) + } + + pub async fn run(mut self) { + while let Some(msg) = self.mailbox.next().await { + match msg { + Message::TestMsg { payload, response } => { + debug!(?payload, "received test message"); + let _ = response.send(format!("echo: {}", payload)); + }, + _ => unimplemented!() + } + } + } +} \ No newline at end of file diff --git a/chain/src/actors/net/ingress.rs b/chain/src/actors/net/ingress.rs new file mode 100644 index 00000000..eb348b74 --- /dev/null +++ b/chain/src/actors/net/ingress.rs @@ -0,0 +1,36 @@ +use bytes::Bytes; +use commonware_cryptography::sha256::Digest; +use futures::{channel::{mpsc, oneshot}, SinkExt}; + +pub enum Message { + SubmitTx { + payload: Bytes, + response: oneshot::Sender + }, + TestMsg { + payload: String, + response: oneshot::Sender + } +} + +#[derive(Clone)] +pub struct Mailbox { + sender: mpsc::Sender +} + +impl Mailbox { + pub(super) fn new(sender: mpsc::Sender) -> Self { + Self { sender } + } + + pub async fn test(&mut self, payload: String) -> String { + let (response, receiver) = oneshot::channel(); + + self.sender + .send(Message::TestMsg { payload, response: response }) + .await + .expect("failed to send test msg"); + + receiver.await.expect("failed to receive test msg") + } +} \ No newline at end of file diff --git a/chain/src/actors/net/mod.rs b/chain/src/actors/net/mod.rs new file mode 100644 index 00000000..866f40eb --- /dev/null +++ b/chain/src/actors/net/mod.rs @@ -0,0 +1,59 @@ +pub mod router; +pub mod ingress; +pub mod actor; + +#[cfg(test)] +mod tests { + use core::panic; + use std::time::Duration; + use axum::{ + body::Body, + http::{Request, StatusCode} + }; + use commonware_macros::test_traced; + use commonware_runtime::{deterministic::{Context, Executor}, Clock, Metrics, Runner, Spawner}; + use futures::channel::mpsc; + use tower::ServiceExt; + + use super::{actor::Actor, ingress::Mailbox, router}; + + #[test_traced] + fn test_msg() { + let (runner, mut context, _) = Executor::timed(Duration::from_secs(30)); + runner.start(async move { + let (actor, mailbox) = Actor::new(); + + context.with_label("net_actor").spawn(|_| { + actor.run() + }); + + let app = router::Router::new( + context.with_label("net_router"), + router::RouterConfig { + port: 7890, + mailbox + }); + + let Some(router) = app.router else { + panic!("router not initalized"); + }; + + // Construct a GET request. + // Note: the handler expects a payload (a String). Since GET requests normally have no body, + // you might decide to pass the payload as a query parameter or in the body if that's what you intend. + // Here, we'll assume the payload is extracted from the request body. + let payload = "test payload"; + let request = Request::builder() + .method("GET") + .uri("/mempool/submit") + .body(Body::from(payload)) + .unwrap(); + + // Send the request to the app. + let response = router.oneshot(request).await.unwrap(); + + // Check that the response status is OK. + assert_eq!(response.status(), StatusCode::OK); + }) + } +} \ No newline at end of file diff --git a/chain/src/actors/application/router.rs b/chain/src/actors/net/router.rs similarity index 62% rename from chain/src/actors/application/router.rs rename to chain/src/actors/net/router.rs index c6975526..1b987022 100644 --- a/chain/src/actors/application/router.rs +++ b/chain/src/actors/net/router.rs @@ -1,47 +1,75 @@ -use std::io; +use std::{ + io, + sync::{Arc, RwLock}, +}; use axum::response::IntoResponse; -use axum::routing::get; +use axum::{ + routing::get, + extract::{Path, State}, +}; + +use bytes::Bytes; +use commonware_cryptography::{sha256, Digest}; use commonware_runtime::{Clock, Handle, Metrics, Spawner}; use rand::Rng; +use serde::Deserialize; use tokio::net::TcpListener; use tracing::{event, Level}; -use crate::actors::syncer; + +use super::ingress; pub struct RouterConfig { - port: i32, + pub port: i32, + pub mailbox: ingress::Mailbox } -impl RouterConfig { - pub fn default_config() -> RouterConfig { - RouterConfig { - port: 7844, - } - } +#[derive(Deserialize)] +pub struct DummyTransaction { + #[serde(with = "serde_bytes")] + pub payload: Vec, +} + +type SharedState = Arc>; + +#[derive(Clone)] +struct AppState { + mailbox: ingress::Mailbox } pub struct Router { context: R, - cfg: RouterConfig, + port: i32, listener: Option, - router: Option, + pub router: Option, is_active: bool, + + state: SharedState } impl Router { - const PATH_SUBMIT_BLOCK: &'static str = "/builder/submit"; + pub const PATH_SUBMIT_TX: &'static str = "/mempool/submit"; pub fn new(context: R, cfg: RouterConfig) -> Self { if cfg.port == 0 { panic!("Invalid port number"); } - Router { + let state = AppState { + mailbox: cfg.mailbox, + }; + + let mut router = Router { context, - cfg, + port: cfg.port, listener: None, router: None, is_active: false, - } + + state: Arc::new(RwLock::new(state)) + }; + router.init_router(); + + router } pub async fn start(mut self) -> Handle<()> { @@ -57,7 +85,7 @@ impl Router { } async fn init_listener(&mut self) -> io::Result { - let listener = TcpListener::bind(format!("127.0.0.1:{}", self.cfg.port)).await?; + let listener = TcpListener::bind(format!("127.0.0.1:{}", self.port)).await?; Ok(listener) } @@ -65,15 +93,21 @@ impl Router { "Hello world!" } - async fn handle_submit_block() -> impl IntoResponse { - "Submit block" + async fn handle_submit_tx( + State(state): State, + payload: String, + ) -> impl IntoResponse { + let mut mailbox = state.write().unwrap().mailbox.clone(); + mailbox.test(payload).await } + fn init_router(&mut self) { let router = axum::Router::new() - .route("/", get(Router::handle_default)) - .route(Router::PATH_SUBMIT_BLOCK, get(Router::handle_submit_block())); - + .route( + Router::::PATH_SUBMIT_TX, + get(Router::::handle_submit_tx).with_state(Arc::clone(&self.state)) + ); self.router = Some(router) } From ea0e781725c97b50b13e22c92fec6a33ddd2171a Mon Sep 17 00:00:00 2001 From: bianyuanop Date: Mon, 31 Mar 2025 16:56:57 -0400 Subject: [PATCH 18/38] basic block listening --- Cargo.lock | 164 ++++++++++++++++++++----- chain/Cargo.toml | 3 +- chain/src/actors/net/ingress.rs | 4 + chain/src/actors/net/mod.rs | 77 +++++++++++- chain/src/actors/net/router.rs | 204 +++++++++++++++++++++++++++++--- 5 files changed, 394 insertions(+), 58 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 375768cf..d432ed3d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -66,13 +66,14 @@ dependencies = [ "futures", "governor", "prometheus-client", - "rand", + "rand 0.8.5", "serde", "serde_bytes", "serde_yaml", "sysinfo", "thiserror 2.0.12", "tokio", + "tokio-tungstenite 0.26.2", "tower 0.4.13", "tracing", "tracing-subscriber", @@ -88,11 +89,11 @@ dependencies = [ "commonware-cryptography", "commonware-utils", "futures", - "rand", + "rand 0.8.5", "reqwest", "thiserror 2.0.12", "tokio", - "tokio-tungstenite", + "tokio-tungstenite 0.17.2", ] [[package]] @@ -106,7 +107,7 @@ dependencies = [ "commonware-cryptography", "commonware-utils", "futures", - "rand", + "rand 0.8.5", "thiserror 2.0.12", "tokio", "tracing", @@ -121,7 +122,7 @@ dependencies = [ "bytes", "commonware-codec", "commonware-cryptography", - "rand", + "rand 0.8.5", "rocksdb", "tempfile", ] @@ -136,7 +137,7 @@ dependencies = [ "commonware-utils", "getrandom 0.2.15", "more-asserts", - "rand", + "rand 0.8.5", "serde", "serde-wasm-bindgen", "thiserror 2.0.12", @@ -572,6 +573,7 @@ checksum = "de45108900e1f9b9242f7f2e254aa3e2c029c921c258fe9e6b4217eeebd54288" dependencies = [ "axum-core", "axum-macros", + "base64 0.22.1", "bytes", "form_urlencoded", "futures-util", @@ -591,8 +593,10 @@ dependencies = [ "serde_json", "serde_path_to_error", "serde_urlencoded", + "sha1", "sync_wrapper", "tokio", + "tokio-tungstenite 0.26.2", "tower 0.5.2", "tower-layer", "tower-service", @@ -919,7 +923,7 @@ dependencies = [ "prometheus-client", "prost", "prost-build", - "rand", + "rand 0.8.5", "thiserror 2.0.12", "tracing", ] @@ -954,7 +958,7 @@ dependencies = [ "prometheus-client", "prost", "prost-build", - "rand", + "rand 0.8.5", "rand_distr", "thiserror 2.0.12", "tracing", @@ -973,7 +977,7 @@ dependencies = [ "getrandom 0.2.15", "p256", "prost-build", - "rand", + "rand 0.8.5", "rayon", "sha2 0.10.8", "thiserror 2.0.12", @@ -1033,7 +1037,7 @@ dependencies = [ "prometheus-client", "prost", "prost-build", - "rand", + "rand 0.8.5", "rand_distr", "thiserror 2.0.12", "tracing", @@ -1059,7 +1063,7 @@ dependencies = [ "prometheus-client", "prost", "prost-build", - "rand", + "rand 0.8.5", "thiserror 2.0.12", "tracing", ] @@ -1079,7 +1083,7 @@ dependencies = [ "getrandom 0.2.15", "governor", "prometheus-client", - "rand", + "rand 0.8.5", "sha2 0.10.8", "thiserror 2.0.12", "tokio", @@ -1122,7 +1126,7 @@ dependencies = [ "futures", "prost", "prost-build", - "rand", + "rand 0.8.5", "thiserror 2.0.12", "x25519-dalek", ] @@ -1211,7 +1215,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76" dependencies = [ "generic-array", - "rand_core", + "rand_core 0.6.4", "subtle", "zeroize", ] @@ -1223,7 +1227,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" dependencies = [ "generic-array", - "rand_core", + "rand_core 0.6.4", "typenum", ] @@ -1261,7 +1265,7 @@ checksum = "1c359b7249347e46fb28804470d071c921156ad62b3eef5d34e2ba867533dec8" dependencies = [ "byteorder", "digest 0.9.0", - "rand_core", + "rand_core 0.6.4", "subtle-ng", "zeroize", ] @@ -1279,6 +1283,12 @@ dependencies = [ "parking_lot_core", ] +[[package]] +name = "data-encoding" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "575f75dfd25738df5b91b8e43e14d44bda14637a58fae779fd2b064f8bf3e010" + [[package]] name = "der" version = "0.7.9" @@ -1359,7 +1369,7 @@ checksum = "3c8465edc8ee7436ffea81d21a019b16676ee3db267aa8d5a8d729581ecf998b" dependencies = [ "curve25519-dalek-ng", "hex", - "rand_core", + "rand_core 0.6.4", "serde", "sha2 0.9.9", "thiserror 1.0.69", @@ -1386,7 +1396,7 @@ dependencies = [ "group", "pem-rfc7468", "pkcs8", - "rand_core", + "rand_core 0.6.4", "sec1", "subtle", "zeroize", @@ -1429,7 +1439,7 @@ version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ded41244b729663b1e574f1b4fb731469f69f79c17667b5d776b16cda0479449" dependencies = [ - "rand_core", + "rand_core 0.6.4", "subtle", ] @@ -1639,7 +1649,7 @@ dependencies = [ "parking_lot", "portable-atomic", "quanta", - "rand", + "rand 0.8.5", "smallvec", "spinning_top", ] @@ -1651,7 +1661,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0f9ef7462f7c099f518d754361858f86d8a07af53ba9af0fe635bbccb151a63" dependencies = [ "ff", - "rand_core", + "rand_core 0.6.4", "subtle", ] @@ -2619,7 +2629,7 @@ version = "0.2.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04" dependencies = [ - "zerocopy", + "zerocopy 0.7.35", ] [[package]] @@ -2762,8 +2772,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" dependencies = [ "libc", - "rand_chacha", - "rand_core", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + +[[package]] +name = "rand" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3779b94aeb87e8bd4e834cee3650289ee9e0d5677f976ecdb6d219e5f4f6cd94" +dependencies = [ + "rand_chacha 0.9.0", + "rand_core 0.9.3", + "zerocopy 0.8.24", ] [[package]] @@ -2773,7 +2794,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" dependencies = [ "ppv-lite86", - "rand_core", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.3", ] [[package]] @@ -2785,6 +2816,15 @@ dependencies = [ "getrandom 0.2.15", ] +[[package]] +name = "rand_core" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38" +dependencies = [ + "getrandom 0.3.1", +] + [[package]] name = "rand_distr" version = "0.4.3" @@ -2792,7 +2832,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32cb0b9bc82b0a0876c2dd994a7e7a2683d3e7390ca40e6886785ef0c7e3ee31" dependencies = [ "num-traits", - "rand", + "rand 0.8.5", ] [[package]] @@ -3246,6 +3286,17 @@ dependencies = [ "digest 0.10.7", ] +[[package]] +name = "sha1" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest 0.10.7", +] + [[package]] name = "sha2" version = "0.9.9" @@ -3301,7 +3352,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" dependencies = [ "digest 0.10.7", - "rand_core", + "rand_core 0.6.4", ] [[package]] @@ -3638,7 +3689,19 @@ dependencies = [ "native-tls", "tokio", "tokio-native-tls", - "tungstenite", + "tungstenite 0.17.3", +] + +[[package]] +name = "tokio-tungstenite" +version = "0.26.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a9daff607c6d2bf6c16fd681ccb7eecc83e4e2cdc1ca067ffaadfca5de7f084" +dependencies = [ + "futures-util", + "log", + "tokio", + "tungstenite 0.26.2", ] [[package]] @@ -3666,7 +3729,7 @@ dependencies = [ "indexmap 1.9.3", "pin-project", "pin-project-lite", - "rand", + "rand 0.8.5", "slab", "tokio", "tokio-util", @@ -3793,13 +3856,30 @@ dependencies = [ "httparse", "log", "native-tls", - "rand", + "rand 0.8.5", "sha-1", "thiserror 1.0.69", "url", "utf-8", ] +[[package]] +name = "tungstenite" +version = "0.26.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4793cb5e56680ecbb1d843515b23b6de9a75eb04b66643e256a396d43be33c13" +dependencies = [ + "bytes", + "data-encoding", + "http 1.2.0", + "httparse", + "log", + "rand 0.9.0", + "sha1", + "thiserror 2.0.12", + "utf-8", +] + [[package]] name = "typenum" version = "1.18.0" @@ -4237,7 +4317,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c7e468321c81fb07fa7f4c636c3972b9100f0346e5b6a9f2bd0603a52f7ed277" dependencies = [ "curve25519-dalek", - "rand_core", + "rand_core 0.6.4", "serde", "zeroize", ] @@ -4279,7 +4359,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" dependencies = [ "byteorder", - "zerocopy-derive", + "zerocopy-derive 0.7.35", +] + +[[package]] +name = "zerocopy" +version = "0.8.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2586fea28e186957ef732a5f8b3be2da217d65c5969d4b1e17f973ebbe876879" +dependencies = [ + "zerocopy-derive 0.8.24", ] [[package]] @@ -4293,6 +4382,17 @@ dependencies = [ "syn 2.0.99", ] +[[package]] +name = "zerocopy-derive" +version = "0.8.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a996a8f63c5c4448cd959ac1bab0aaa3306ccfd060472f85943ee0750f0169be" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.99", +] + [[package]] name = "zerofrom" version = "0.1.6" diff --git a/chain/Cargo.toml b/chain/Cargo.toml index c6f81c5e..2bf7d701 100644 --- a/chain/Cargo.toml +++ b/chain/Cargo.toml @@ -34,11 +34,12 @@ governor = { workspace = true } prometheus-client = { workspace = true } clap = { workspace = true } sysinfo = "0.33.1" -axum = { version = "0.8.1", features = ["macros"] } +axum = { version = "0.8.1", features = ["macros", "ws"] } uuid = "1.15.1" serde = { version = "1.0.218", features = ["derive"] } serde_yaml = "0.9.34" serde_bytes = "0.11.17" +tokio-tungstenite = "0.26.2" tokio = "1.44.0" tower = { version = "0.4", features = ["full"] } diff --git a/chain/src/actors/net/ingress.rs b/chain/src/actors/net/ingress.rs index eb348b74..df1ee140 100644 --- a/chain/src/actors/net/ingress.rs +++ b/chain/src/actors/net/ingress.rs @@ -1,8 +1,12 @@ +use alto_types::Block; use bytes::Bytes; use commonware_cryptography::sha256::Digest; use futures::{channel::{mpsc, oneshot}, SinkExt}; pub enum Message { + Block { + block: Block, + }, SubmitTx { payload: Bytes, response: oneshot::Sender diff --git a/chain/src/actors/net/mod.rs b/chain/src/actors/net/mod.rs index 866f40eb..ccc87c67 100644 --- a/chain/src/actors/net/mod.rs +++ b/chain/src/actors/net/mod.rs @@ -6,20 +6,27 @@ pub mod actor; mod tests { use core::panic; use std::time::Duration; + use alto_types::Block; use axum::{ body::Body, http::{Request, StatusCode} }; - use commonware_macros::test_traced; - use commonware_runtime::{deterministic::{Context, Executor}, Clock, Metrics, Runner, Spawner}; - use futures::channel::mpsc; + use commonware_cryptography::sha256; + use commonware_macros::{test_async, test_traced}; + use commonware_runtime::{tokio::{self, Context, Executor}, Clock, Metrics, Runner, Spawner}; + use futures::{channel::mpsc, future::join_all, SinkExt, StreamExt}; + use tokio_tungstenite::{connect_async, tungstenite::Message as WsClientMessage}; use tower::ServiceExt; + use tracing_subscriber::field::debug; - use super::{actor::Actor, ingress::Mailbox, router}; + use crate::actors::net::router::decode_router_message; + + use super::{actor::Actor, ingress::Mailbox, router::{self, RouterMessage}}; + use tracing::debug; #[test_traced] fn test_msg() { - let (runner, mut context, _) = Executor::timed(Duration::from_secs(30)); + let (runner, mut context) = Executor::init(tokio::Config::default()); runner.start(async move { let (actor, mailbox) = Actor::new(); @@ -27,7 +34,7 @@ mod tests { actor.run() }); - let app = router::Router::new( + let (app, _) = router::Router::new( context.with_label("net_router"), router::RouterConfig { port: 7890, @@ -56,4 +63,62 @@ mod tests { assert_eq!(response.status(), StatusCode::OK); }) } + + #[test_traced] + fn test_ws() { + let (runner, mut context) = Executor::default(); + runner.start(async move { + let (actor, actor_mailbox) = Actor::new(); + + context.with_label("net_actor").spawn(|_| { + actor.run() + }); + + let (app, mut router_mailbox) = router::Router::new( + context.with_label("net_router"), + router::RouterConfig { + port: 7890, + mailbox: actor_mailbox + }); + + debug!("starting router"); + let app_handler = app.start().await; + + debug!("launching ws client"); + // instantiate websocket client listening block + let url = format!("ws://127.0.0.1:7890/ws"); + let (ws_stream, response) = connect_async(url).await.expect("Failed to connect"); + assert_eq!(response.status(), StatusCode::SWITCHING_PROTOCOLS); + + let (_, mut read) = ws_stream.split(); + let client_handler = context.with_label("ws_client").spawn(async move |_| { + while let Ok(msg) = read.next().await.unwrap() { + match msg { + WsClientMessage::Binary(bin) => { + match decode_router_message(bin.into()).unwrap() { + RouterMessage::Block { block } => { + debug!(?block, "received a block from server"); + return; + } } + } , + _ => { + debug!("unknown message") + } + } + }; + }); + + // send a dummy block + debug!("mock sending dummy block from another service"); + let parent_digest = sha256::hash(&[0; 32]); + let height = 0; + let timestamp = 1; + let block = Block::new(parent_digest, height, timestamp); + router_mailbox.broadcast_block(block).await; + + context.sleep(Duration::from_millis(1000)).await; + + // join_all(vec![app_handler]).await; + }) + } } \ No newline at end of file diff --git a/chain/src/actors/net/router.rs b/chain/src/actors/net/router.rs index 1b987022..1e95923b 100644 --- a/chain/src/actors/net/router.rs +++ b/chain/src/actors/net/router.rs @@ -1,20 +1,24 @@ use std::{ - io, - sync::{Arc, RwLock}, + collections::HashMap, io, ops::Deref, sync::{Arc, RwLock} }; +use alto_types::Block; use axum::response::IntoResponse; use axum::{ routing::get, - extract::{Path, State}, + extract::{Path, State, ws::{ + WebSocket, WebSocketUpgrade, Message as WSMessage + }}, }; use bytes::Bytes; use commonware_cryptography::{sha256, Digest}; use commonware_runtime::{Clock, Handle, Metrics, Spawner}; +use futures::{channel::{mpsc, oneshot}, lock::Mutex, SinkExt, StreamExt}; use rand::Rng; use serde::Deserialize; use tokio::net::TcpListener; -use tracing::{event, Level}; +use tokio_tungstenite::tungstenite::client; +use tracing::{debug, event, warn, Level, error}; use super::ingress; @@ -29,11 +33,37 @@ pub struct DummyTransaction { pub payload: Vec, } -type SharedState = Arc>; +#[derive(Debug, Clone)] +pub enum RouterMessage { + Block { + block: Block + } +} + +pub struct Mailbox { + sender: mpsc::Sender +} + +impl Mailbox { + pub async fn broadcast_block(&mut self, block: Block) { + self.sender.send(RouterMessage::Block { block }) + .await + .expect("failed to broadcast block") + } +} + +type ClientSender = mpsc::Sender; + +type ClientID = String; +type Clients = Arc>>; + +type SharedState = Arc>>; #[derive(Clone)] -struct AppState { - mailbox: ingress::Mailbox +struct AppState { + context: R, + mailbox: ingress::Mailbox, + clients: Clients } pub struct Router { @@ -43,20 +73,45 @@ pub struct Router { pub router: Option, is_active: bool, - state: SharedState + state: SharedState } impl Router { + pub const WEBSOCKET_PREFIX: &'static str = "/ws"; + pub const RPC_PREFIX: &'static str = "/api"; pub const PATH_SUBMIT_TX: &'static str = "/mempool/submit"; - pub fn new(context: R, cfg: RouterConfig) -> Self { + pub fn new(context: R, cfg: RouterConfig) -> (Self, Mailbox) { if cfg.port == 0 { panic!("Invalid port number"); } - let state = AppState { + let state = AppState:: { + context: context.with_label("app_state"), mailbox: cfg.mailbox, + clients: Arc::new(RwLock::new(HashMap::new())) }; + let state = Arc::new(RwLock::new(state)); + let (sender, mut receiver) = mpsc::channel::(1024); + + let receiver_state = state.clone(); + + context.with_label("receiver").spawn(async move |_| { + println!("starting receiving mailbox messages"); + while let Some(msg) = receiver.next().await { + println!("received message from mailbox: {:?}", msg); + let guard = receiver_state.write().unwrap().clients.clone(); + let clients = guard.write().unwrap().clone(); + println!("clients connected: {:?}", clients.keys()); + for (client_id, mut sender) in clients.into_iter() { + debug!(?client_id, ?msg, "broadcasting message to client"); + print!("broadcasting message to client: {}", client_id); + let _ = sender.send(msg.clone()).await; + } + println!("finishing up broadcasting") + } + }); + let mut router = Router { context, @@ -64,12 +119,14 @@ impl Router { listener: None, router: None, is_active: false, - - state: Arc::new(RwLock::new(state)) + state }; router.init_router(); - router + ( + router, + Mailbox { sender } + ) } pub async fn start(mut self) -> Handle<()> { @@ -89,12 +146,73 @@ impl Router { Ok(listener) } - async fn handle_default() -> impl IntoResponse { - "Hello world!" + async fn ws_handler( + ws: WebSocketUpgrade, + State(state): State>, + ) -> impl IntoResponse { + let client_id = uuid::Uuid::new_v4().to_string(); + ws.on_upgrade(move |socket| Self::handle_socket(socket, client_id, state)) + } + + async fn handle_socket(mut socket: WebSocket, client_id: ClientID, state: SharedState) { + let (mut socket_sender, mut socket_receiver) = socket.split(); + + let (tx, mut rx) = mpsc::channel::(1024); + + // Insert the sender into the shared state + { + let mut state = state.write().unwrap(); + state.clients.write().unwrap().insert(client_id.clone(), tx); + print!("inserting client {}\n", client_id); + + state.context.with_label(format!("client-{}", client_id).deref()).spawn(async move |_| { + println!("starting client rx listener"); + while let Some(msg) = rx.next().await { + print!("received message from client receiver chan: {:?}", msg); + match encode_router_message(msg) { + Ok(raw) => { + socket_sender.send(WSMessage::Binary(Bytes::from(raw))).await.unwrap(); + }, + Err(err) => { + warn!(?err, "received unsupported message"); + print!("received unsupporated message: {}", err) + } + } + } + }); + } + + + while let Some(msg) = socket_receiver.next().await { + match msg { + Ok(WSMessage::Text(text)) => { + debug!(?text, "receiving text"); + } + Ok(WSMessage::Binary(bin)) => { + match decode_router_message(bin.into()) { + Ok(msg) => { + debug!(?msg, "received msg from client") + }, + Err(err) => { + // TODO: possibly terminate the connection as malicious message is sent? + error!(?err, "received unsupported message") + } + } + } + Ok(WSMessage::Close(_)) => break, + _ => {} + } + } + + { + let mut state = state.write().unwrap(); + state.clients.write().unwrap().remove(&client_id); + print!("removing client {}\n", client_id); + } } async fn handle_submit_tx( - State(state): State, + State(state): State>, payload: String, ) -> impl IntoResponse { let mut mailbox = state.write().unwrap().mailbox.clone(); @@ -105,8 +223,12 @@ impl Router { fn init_router(&mut self) { let router = axum::Router::new() .route( - Router::::PATH_SUBMIT_TX, - get(Router::::handle_submit_tx).with_state(Arc::clone(&self.state)) + Self::PATH_SUBMIT_TX, + get(Self::handle_submit_tx).with_state(Arc::clone(&self.state)) + ) + .route( + Self::WEBSOCKET_PREFIX, + get(Self::ws_handler).with_state(Arc::clone(&self.state)) ); self.router = Some(router) } @@ -121,6 +243,7 @@ impl Router { async fn run(mut self) { event!(Level::INFO, "starting router service"); + println!("init listener"); let listener_res = self.init_listener(); match listener_res.await { Ok(value) => self.listener = Some(value), @@ -130,10 +253,53 @@ impl Router { }, } + println!("init router & serve"); self.init_router(); self.serve().await.unwrap(); self.is_active = true; - event!(Level::INFO, "finished starting router service"); + event!(Level::INFO, "server stopping..."); + + } +} + +pub enum RouterMessageType { + Block = 1, +} + +impl TryFrom for RouterMessageType { + type Error = (); + fn try_from(value: u8) -> Result { + match value { + x if x == RouterMessageType::Block as u8 => Ok(RouterMessageType::Block), + _ => Err(()) + } + } +} + +pub fn encode_router_message(msg: RouterMessage) -> Result, String> { + match msg { + RouterMessage::Block { block } => { + let mut raw = block.serialize(); + raw.insert(0, RouterMessageType::Block as u8); + Ok(raw) + } + } +} + +pub fn decode_router_message(raw: Vec) -> Result { + if raw.len() == 0 { + return Err(format!("zero len raw message provided")) + } + + let msg_type = RouterMessageType::try_from(raw[0]).unwrap(); + match msg_type { + RouterMessageType::Block => { + let Some(block) = Block::deserialize(&raw[1..]) else { + return Err(format!("unable to deserialize block")) + }; + + Ok(RouterMessage::Block { block }) + } } } From 4d14f17274352e1bcd231081176a2a1252406332 Mon Sep 17 00:00:00 2001 From: bianyuanop Date: Tue, 1 Apr 2025 15:18:57 -0400 Subject: [PATCH 19/38] refactor --- chain/src/actors/mempool/mempool.rs | 4 +- chain/src/actors/net/actor.rs | 284 ++++++++++++++++++++++++-- chain/src/actors/net/ingress.rs | 144 +++++++++++-- chain/src/actors/net/mod.rs | 59 +++--- chain/src/actors/net/router.rs | 305 ---------------------------- 5 files changed, 416 insertions(+), 380 deletions(-) delete mode 100644 chain/src/actors/net/router.rs diff --git a/chain/src/actors/mempool/mempool.rs b/chain/src/actors/mempool/mempool.rs index a199d770..ebffb985 100644 --- a/chain/src/actors/mempool/mempool.rs +++ b/chain/src/actors/mempool/mempool.rs @@ -139,10 +139,10 @@ impl RawTransaction } } -impl From for RawTransaction +impl From for RawTransaction where Sha256: Hasher { - fn from(value: net::router::DummyTransaction) -> Self { + fn from(value: net::actor::DummyTransaction) -> Self { let raw = Bytes::from(value.payload); RawTransaction::new(raw) } diff --git a/chain/src/actors/net/actor.rs b/chain/src/actors/net/actor.rs index 0dee8764..c9dc57f8 100644 --- a/chain/src/actors/net/actor.rs +++ b/chain/src/actors/net/actor.rs @@ -1,35 +1,277 @@ -use futures::{channel::mpsc, StreamExt}; -use rand::Rng; +use std::{ + collections::{HashMap, HashSet}, io, ops::Deref, sync::{Arc, RwLock} +}; +use alto_client::Client; +use alto_types::Block; +use axum::response::IntoResponse; +use axum::{ + routing::get, + extract::{Path, State, ws::{ + WebSocket, WebSocketUpgrade, Message as WSMessage + }}, +}; + +use bytes::Bytes; +use commonware_cryptography::{sha256, Digest}; use commonware_runtime::{Clock, Handle, Metrics, Spawner}; +use futures::{channel::{mpsc, oneshot}, lock::Mutex, SinkExt, StreamExt}; +use rand::Rng; +use serde::Deserialize; +use tokio::net::TcpListener; +use tracing::{debug, event, Level, error}; +use tracing_subscriber::fmt::format; + +use super::ingress::{Mailbox, Message, WebsocketClientMessage}; -use super::ingress::{Mailbox, Message}; -use tracing::{debug}; +#[derive(Deserialize)] +pub struct DummyTransaction { + #[serde(with = "serde_bytes")] + pub payload: Vec, +} + +type ClientSender = mpsc::Sender>; +type ClientID = String; +type Clients = Arc>>; -pub struct Actor { - mailbox: mpsc::Receiver +type SharedState = Arc>>; + +#[derive(Clone)] +struct AppState { + context: R, + mailbox: Mailbox, + clients: Clients, + block_listeners: Arc>>, + tx_listeners: Arc>> } -impl Actor { - pub fn new() -> (Self, Mailbox) { - let (sender, receiver) = mpsc::channel(1024); +pub struct Config { + pub port: i32, +} + +pub struct Actor { + context: R, + port: i32, + listener: Option, + pub router: Option, + is_active: bool, + + state: SharedState +} + +impl Actor { + pub const WEBSOCKET_PREFIX: &'static str = "/ws"; + pub const RPC_PREFIX: &'static str = "/api"; + pub const PATH_SUBMIT_TX: &'static str = "/mempool/submit"; + + pub fn new(context: R, cfg: Config) -> (Self, Mailbox) { + if cfg.port == 0 { + panic!("Invalid port number"); + } + + let (sender, mut receiver) = mpsc::channel(1024); + let mailbox = Mailbox::new(sender); + + let state = AppState:: { + context: context.with_label("app_state"), + mailbox: mailbox.clone(), + clients: Arc::new(RwLock::new(HashMap::new())), + block_listeners: Arc::new(RwLock::new(HashSet::new())), + tx_listeners: Arc::new(RwLock::new(HashSet::new())), + }; + let state = Arc::new(RwLock::new(state)); + let receiver_state = state.clone(); + + context.with_label("receiver").spawn(async move |_| { + println!("starting receiving mailbox messages"); + while let Some(msg) = receiver.next().await { + Self::handle_message(receiver_state.clone(), msg).await; + } + }); + + + let mut router = Actor { + context, + port: cfg.port, + listener: None, + router: None, + is_active: false, + state + }; + router.init_router(); ( - Actor { - mailbox: receiver, - }, - Mailbox::new(sender) + router, + mailbox ) } - pub async fn run(mut self) { - while let Some(msg) = self.mailbox.next().await { + pub async fn start(mut self) -> Handle<()> { + self.context.spawn_ref()(self.run()) + } + + pub fn stop(&self) { + if !self.is_active { + return + } + + event!(Level::INFO, "stopped router service"); + } + + async fn init_listener(&mut self) -> io::Result { + let listener = TcpListener::bind(format!("127.0.0.1:{}", self.port)).await?; + Ok(listener) + } + + /// handles messages from other services within a node such as block messages + async fn handle_message(state: SharedState, msg: Message) { + println!("handling msg {:?}", msg); + let block_listeners = state.read().unwrap().block_listeners.clone(); + let clients = state.read().unwrap().clients.clone(); + + match msg { + Message::PublishBlock { block } => { + let msg = Arc::new(Message::PublishBlock { block }); + let listeners: Vec<_> = { + block_listeners.read().unwrap().iter().cloned().collect() + }; + + for listener in listeners { + if let Some(tx) = { + let guard = clients.read().unwrap(); + guard.get(&listener).cloned() + } { + let _ = tx.clone().send(msg.clone()).await; + } + } + } + } + } + + async fn ws_handler( + ws: WebSocketUpgrade, + State(state): State>, + ) -> impl IntoResponse { + let client_id = uuid::Uuid::new_v4().to_string(); + ws.on_upgrade(move |socket| Self::handle_socket(socket, client_id, state)) + } + + async fn handle_socket(mut socket: WebSocket, client_id: ClientID, state: SharedState) { + let (mut socket_sender, mut socket_receiver) = socket.split(); + + let (tx, mut rx) = mpsc::channel::>(1024); + + // Insert the sender into the shared state + { + let state = state.write().unwrap(); + state.clients.write().unwrap().insert(client_id.clone(), tx); + print!("inserting client {}\n", client_id); + + state.context.with_label(format!("client-{}", client_id).deref()).spawn(async move |_| { + println!("starting client rx listener"); + while let Some(msg) = rx.next().await { + print!("received message from client receiver chan: {:?}", msg); + let raw = msg.serialize(); + socket_sender.send(WSMessage::Binary(Bytes::from(raw))).await.unwrap(); + } + }); + } + + + while let Some(msg) = socket_receiver.next().await { match msg { - Message::TestMsg { payload, response } => { - debug!(?payload, "received test message"); - let _ = response.send(format!("echo: {}", payload)); - }, - _ => unimplemented!() - } + Ok(WSMessage::Text(text)) => { + debug!(?text, "receiving text"); + } + Ok(WSMessage::Binary(bin)) => { + match WebsocketClientMessage::deserialize(bin.deref()) { + Ok(msg) => { + debug!(?msg, "received msg from client"); + let state = state.write().unwrap(); + match msg { + WebsocketClientMessage::RegisterBlock => { + println!("adding block listener {}", client_id); + state.block_listeners.write().unwrap().insert(client_id.clone()); + }, + WebsocketClientMessage::RegisterTx => { + state.tx_listeners.write().unwrap().insert(client_id.clone()); + }, + WebsocketClientMessage::SubmitTxs(txs) => { + unimplemented!() + } + } + }, + Err(err) => { + // TODO: possibly terminate the connection as malicious message is sent? + error!(?err, "received unsupported message") + } + } + } + Ok(WSMessage::Close(_)) => { + let state = state.write().unwrap(); + state.clients.write().unwrap().remove(&client_id); + state.block_listeners.write().unwrap().remove(&client_id); + state.tx_listeners.write().unwrap().remove(&client_id); + break; + } + _ => {} + } + } + + { + let state = state.write().unwrap(); + state.clients.write().unwrap().remove(&client_id); + print!("removing client {}\n", client_id); + } + } + + async fn handle_submit_tx( + State(state): State>, + payload: Bytes, + ) -> impl IntoResponse { + // TODO: send to mempool mailbox + format!("submitted") + } + + + fn init_router(&mut self) { + let router = axum::Router::new() + .route( + Self::PATH_SUBMIT_TX, + get(Self::handle_submit_tx).with_state(Arc::clone(&self.state)) + ) + .route( + Self::WEBSOCKET_PREFIX, + get(Self::ws_handler).with_state(Arc::clone(&self.state)) + ); + self.router = Some(router) + } + + async fn serve(&mut self) -> Result<(), Box> { + let listener = self.listener.take().ok_or("serve failed because listener is None"); + let router = self.router.take().ok_or("serve failed because router is None"); + axum::serve(listener.unwrap(), router.unwrap()).await?; + Ok(()) + } + + async fn run(mut self) { + event!(Level::INFO, "starting router service"); + + println!("init listener"); + let listener_res = self.init_listener(); + match listener_res.await { + Ok(value) => self.listener = Some(value), + Err(error) => { + println!("Error during listener init: {}", error); + return + }, } + + println!("init router & serve"); + self.init_router(); + self.serve().await.unwrap(); + self.is_active = true; + + event!(Level::INFO, "server stopping..."); + } } \ No newline at end of file diff --git a/chain/src/actors/net/ingress.rs b/chain/src/actors/net/ingress.rs index df1ee140..50674ba9 100644 --- a/chain/src/actors/net/ingress.rs +++ b/chain/src/actors/net/ingress.rs @@ -1,20 +1,139 @@ +use std::vec; + use alto_types::Block; -use bytes::Bytes; +use bytes::{Buf, BufMut, Bytes}; use commonware_cryptography::sha256::Digest; -use futures::{channel::{mpsc, oneshot}, SinkExt}; +use futures::{channel::{mpsc, oneshot}, stream::SelectNextSome, SinkExt}; +// message from other services +#[derive(Debug)] pub enum Message { - Block { + PublishBlock { block: Block, }, +} + +impl Message { + pub fn serialize(&self) -> Vec { + match self { + Self::PublishBlock { block } => { + let raw_block = block.serialize(); + let mut raw = Vec::with_capacity(1 + raw_block.len()); + raw.push(0); + raw.extend_from_slice(&raw_block); + raw + } + } + } + + pub fn deserialize(raw: &[u8]) -> Result { + if raw.is_empty() { + return Err("Empty payload provided".into()); + } + // The first byte indicates the message type. + match raw[0] { + 0 => { + let Some(block) = Block::deserialize(&raw[1..]) else { + return Err(format!("unable to deserialize into block")); + }; + Ok(Message::PublishBlock { block }) + } + other => Err(format!("Unsupported message type: {}", other)), + } + } +} + +/// Messages sent from client +pub enum ClientMessage { + WSMessage(WebsocketClientMessage), + RpcMessage(RpcMessage) +} + +/// Websocket Message sent from client +#[derive(Debug)] +#[repr(u8)] +pub enum WebsocketClientMessage { + RegisterBlock = 0, + RegisterTx, + + SubmitTxs(Vec) +} + +impl WebsocketClientMessage { + // TODO: cache the serialization result + pub fn serialize(&self) -> Vec { + match self { + Self::RegisterBlock => vec![0], + Self::RegisterTx => vec![1], + Self::SubmitTxs(txs) => { + let mut raw = vec![2]; // first byte indicates the message type + raw.put_u64(txs.len() as u64); + for tx in txs.into_iter() { + raw.put_u64(tx.len() as u64); + raw.put(tx.clone()); + } + raw + } + } + } + + pub fn deserialize(raw: &[u8]) -> Result { + use bytes::Buf; + + if raw.is_empty() { + return Err("empty payload provided".into()); + } + + // Create a mutable buffer view over the input slice + let mut buf = raw; + + // Read the message type byte. + let msg_type = buf.get_u8(); + match msg_type { + 0 => Ok(WebsocketClientMessage::RegisterBlock), + 1 => Ok(WebsocketClientMessage::RegisterTx), + 2 => { + // Ensure there are enough bytes for the number of transactions. + if buf.remaining() < 8 { + return Err("payload too short for number of transactions".into()); + } + let num_txs = buf.get_u64(); + + let mut txs = Vec::with_capacity(num_txs as usize); + // Loop over each transaction. + for _ in 0..num_txs { + if buf.remaining() < 8 { + return Err("payload too short for transaction length".into()); + } + let tx_len = buf.get_u64() as usize; + if buf.remaining() < tx_len { + return Err("payload too short for transaction data".into()); + } + // Extract the transaction bytes. + let tx_data = buf.copy_to_bytes(tx_len); + txs.push(tx_data); + } + Ok(WebsocketClientMessage::SubmitTxs(txs)) + } + other => Err(format!("unsupported message type: {}", other)), + } + } +} + +#[derive(Debug)] +pub enum RpcMessage { + // for rpc SubmitTx { payload: Bytes, response: oneshot::Sender }, - TestMsg { - payload: String, - response: oneshot::Sender - } + GetBlockHeight { + response: oneshot::Sender + }, + GetBlock { + height: u64, + response: oneshot::Sender>, + }, } #[derive(Clone)] @@ -27,14 +146,7 @@ impl Mailbox { Self { sender } } - pub async fn test(&mut self, payload: String) -> String { - let (response, receiver) = oneshot::channel(); - - self.sender - .send(Message::TestMsg { payload, response: response }) - .await - .expect("failed to send test msg"); - - receiver.await.expect("failed to receive test msg") + pub async fn broadcast_block(&mut self, block: Block) { + let _ = self.sender.send(Message::PublishBlock { block }).await; } } \ No newline at end of file diff --git a/chain/src/actors/net/mod.rs b/chain/src/actors/net/mod.rs index ccc87c67..beb6b65b 100644 --- a/chain/src/actors/net/mod.rs +++ b/chain/src/actors/net/mod.rs @@ -1,6 +1,5 @@ -pub mod router; -pub mod ingress; pub mod actor; +pub mod ingress; #[cfg(test)] mod tests { @@ -15,33 +14,24 @@ mod tests { use commonware_macros::{test_async, test_traced}; use commonware_runtime::{tokio::{self, Context, Executor}, Clock, Metrics, Runner, Spawner}; use futures::{channel::mpsc, future::join_all, SinkExt, StreamExt}; - use tokio_tungstenite::{connect_async, tungstenite::Message as WsClientMessage}; + use tokio_tungstenite::{connect_async, tungstenite::{client, Message as WsClientMessage}}; use tower::ServiceExt; use tracing_subscriber::field::debug; - use crate::actors::net::router::decode_router_message; + use crate::actors::net::ingress::WebsocketClientMessage; - use super::{actor::Actor, ingress::Mailbox, router::{self, RouterMessage}}; + use super::{actor::Actor, ingress::Message, actor::{self}}; use tracing::debug; #[test_traced] fn test_msg() { let (runner, mut context) = Executor::init(tokio::Config::default()); runner.start(async move { - let (actor, mailbox) = Actor::new(); - - context.with_label("net_actor").spawn(|_| { - actor.run() - }); - - let (app, _) = router::Router::new( - context.with_label("net_router"), - router::RouterConfig { - port: 7890, - mailbox + let (actor, mailbox) = Actor::new(context, actor::Config { + port: 7890 }); - let Some(router) = app.router else { + let Some(router) = actor.router else { panic!("router not initalized"); }; @@ -68,21 +58,12 @@ mod tests { fn test_ws() { let (runner, mut context) = Executor::default(); runner.start(async move { - let (actor, actor_mailbox) = Actor::new(); - - context.with_label("net_actor").spawn(|_| { - actor.run() - }); - - let (app, mut router_mailbox) = router::Router::new( - context.with_label("net_router"), - router::RouterConfig { - port: 7890, - mailbox: actor_mailbox + let (actor, mut mailbox) = Actor::new(context.with_label("router"), actor::Config { + port: 7890 }); debug!("starting router"); - let app_handler = app.start().await; + let app_handler = actor.start().await; debug!("launching ws client"); // instantiate websocket client listening block @@ -90,16 +71,22 @@ mod tests { let (ws_stream, response) = connect_async(url).await.expect("Failed to connect"); assert_eq!(response.status(), StatusCode::SWITCHING_PROTOCOLS); - let (_, mut read) = ws_stream.split(); + let (mut write, mut read) = ws_stream.split(); + // register block + let _ = write.send(WsClientMessage::binary(WebsocketClientMessage::RegisterBlock.serialize())).await; + + // listening block let client_handler = context.with_label("ws_client").spawn(async move |_| { while let Ok(msg) = read.next().await.unwrap() { match msg { WsClientMessage::Binary(bin) => { - match decode_router_message(bin.into()).unwrap() { - RouterMessage::Block { block } => { - debug!(?block, "received a block from server"); + let msg = Message::deserialize(&bin).unwrap(); + match msg { + Message::PublishBlock { block } => { + print!("received a block from server: {:?}", block); return; - } } + } + } } , _ => { debug!("unknown message") @@ -114,11 +101,11 @@ mod tests { let height = 0; let timestamp = 1; let block = Block::new(parent_digest, height, timestamp); - router_mailbox.broadcast_block(block).await; + mailbox.broadcast_block(block).await; context.sleep(Duration::from_millis(1000)).await; - // join_all(vec![app_handler]).await; + join_all(vec![client_handler]).await; }) } } \ No newline at end of file diff --git a/chain/src/actors/net/router.rs b/chain/src/actors/net/router.rs deleted file mode 100644 index 1e95923b..00000000 --- a/chain/src/actors/net/router.rs +++ /dev/null @@ -1,305 +0,0 @@ -use std::{ - collections::HashMap, io, ops::Deref, sync::{Arc, RwLock} -}; -use alto_types::Block; -use axum::response::IntoResponse; -use axum::{ - routing::get, - extract::{Path, State, ws::{ - WebSocket, WebSocketUpgrade, Message as WSMessage - }}, -}; - -use bytes::Bytes; -use commonware_cryptography::{sha256, Digest}; -use commonware_runtime::{Clock, Handle, Metrics, Spawner}; -use futures::{channel::{mpsc, oneshot}, lock::Mutex, SinkExt, StreamExt}; -use rand::Rng; -use serde::Deserialize; -use tokio::net::TcpListener; -use tokio_tungstenite::tungstenite::client; -use tracing::{debug, event, warn, Level, error}; - -use super::ingress; - -pub struct RouterConfig { - pub port: i32, - pub mailbox: ingress::Mailbox -} - -#[derive(Deserialize)] -pub struct DummyTransaction { - #[serde(with = "serde_bytes")] - pub payload: Vec, -} - -#[derive(Debug, Clone)] -pub enum RouterMessage { - Block { - block: Block - } -} - -pub struct Mailbox { - sender: mpsc::Sender -} - -impl Mailbox { - pub async fn broadcast_block(&mut self, block: Block) { - self.sender.send(RouterMessage::Block { block }) - .await - .expect("failed to broadcast block") - } -} - -type ClientSender = mpsc::Sender; - -type ClientID = String; -type Clients = Arc>>; - -type SharedState = Arc>>; - -#[derive(Clone)] -struct AppState { - context: R, - mailbox: ingress::Mailbox, - clients: Clients -} - -pub struct Router { - context: R, - port: i32, - listener: Option, - pub router: Option, - is_active: bool, - - state: SharedState -} - -impl Router { - pub const WEBSOCKET_PREFIX: &'static str = "/ws"; - pub const RPC_PREFIX: &'static str = "/api"; - pub const PATH_SUBMIT_TX: &'static str = "/mempool/submit"; - - pub fn new(context: R, cfg: RouterConfig) -> (Self, Mailbox) { - if cfg.port == 0 { - panic!("Invalid port number"); - } - - let state = AppState:: { - context: context.with_label("app_state"), - mailbox: cfg.mailbox, - clients: Arc::new(RwLock::new(HashMap::new())) - }; - let state = Arc::new(RwLock::new(state)); - let (sender, mut receiver) = mpsc::channel::(1024); - - let receiver_state = state.clone(); - - context.with_label("receiver").spawn(async move |_| { - println!("starting receiving mailbox messages"); - while let Some(msg) = receiver.next().await { - println!("received message from mailbox: {:?}", msg); - let guard = receiver_state.write().unwrap().clients.clone(); - let clients = guard.write().unwrap().clone(); - println!("clients connected: {:?}", clients.keys()); - for (client_id, mut sender) in clients.into_iter() { - debug!(?client_id, ?msg, "broadcasting message to client"); - print!("broadcasting message to client: {}", client_id); - let _ = sender.send(msg.clone()).await; - } - println!("finishing up broadcasting") - } - }); - - - let mut router = Router { - context, - port: cfg.port, - listener: None, - router: None, - is_active: false, - state - }; - router.init_router(); - - ( - router, - Mailbox { sender } - ) - } - - pub async fn start(mut self) -> Handle<()> { - self.context.spawn_ref()(self.run()) - } - - pub fn stop(&self) { - if !self.is_active { - return - } - - event!(Level::INFO, "stopped router service"); - } - - async fn init_listener(&mut self) -> io::Result { - let listener = TcpListener::bind(format!("127.0.0.1:{}", self.port)).await?; - Ok(listener) - } - - async fn ws_handler( - ws: WebSocketUpgrade, - State(state): State>, - ) -> impl IntoResponse { - let client_id = uuid::Uuid::new_v4().to_string(); - ws.on_upgrade(move |socket| Self::handle_socket(socket, client_id, state)) - } - - async fn handle_socket(mut socket: WebSocket, client_id: ClientID, state: SharedState) { - let (mut socket_sender, mut socket_receiver) = socket.split(); - - let (tx, mut rx) = mpsc::channel::(1024); - - // Insert the sender into the shared state - { - let mut state = state.write().unwrap(); - state.clients.write().unwrap().insert(client_id.clone(), tx); - print!("inserting client {}\n", client_id); - - state.context.with_label(format!("client-{}", client_id).deref()).spawn(async move |_| { - println!("starting client rx listener"); - while let Some(msg) = rx.next().await { - print!("received message from client receiver chan: {:?}", msg); - match encode_router_message(msg) { - Ok(raw) => { - socket_sender.send(WSMessage::Binary(Bytes::from(raw))).await.unwrap(); - }, - Err(err) => { - warn!(?err, "received unsupported message"); - print!("received unsupporated message: {}", err) - } - } - } - }); - } - - - while let Some(msg) = socket_receiver.next().await { - match msg { - Ok(WSMessage::Text(text)) => { - debug!(?text, "receiving text"); - } - Ok(WSMessage::Binary(bin)) => { - match decode_router_message(bin.into()) { - Ok(msg) => { - debug!(?msg, "received msg from client") - }, - Err(err) => { - // TODO: possibly terminate the connection as malicious message is sent? - error!(?err, "received unsupported message") - } - } - } - Ok(WSMessage::Close(_)) => break, - _ => {} - } - } - - { - let mut state = state.write().unwrap(); - state.clients.write().unwrap().remove(&client_id); - print!("removing client {}\n", client_id); - } - } - - async fn handle_submit_tx( - State(state): State>, - payload: String, - ) -> impl IntoResponse { - let mut mailbox = state.write().unwrap().mailbox.clone(); - mailbox.test(payload).await - } - - - fn init_router(&mut self) { - let router = axum::Router::new() - .route( - Self::PATH_SUBMIT_TX, - get(Self::handle_submit_tx).with_state(Arc::clone(&self.state)) - ) - .route( - Self::WEBSOCKET_PREFIX, - get(Self::ws_handler).with_state(Arc::clone(&self.state)) - ); - self.router = Some(router) - } - - async fn serve(&mut self) -> Result<(), Box> { - let listener = self.listener.take().ok_or("serve failed because listener is None"); - let router = self.router.take().ok_or("serve failed because router is None"); - axum::serve(listener.unwrap(), router.unwrap()).await?; - Ok(()) - } - - async fn run(mut self) { - event!(Level::INFO, "starting router service"); - - println!("init listener"); - let listener_res = self.init_listener(); - match listener_res.await { - Ok(value) => self.listener = Some(value), - Err(error) => { - println!("Error during listener init: {}", error); - return - }, - } - - println!("init router & serve"); - self.init_router(); - self.serve().await.unwrap(); - self.is_active = true; - - event!(Level::INFO, "server stopping..."); - - } -} - -pub enum RouterMessageType { - Block = 1, -} - -impl TryFrom for RouterMessageType { - type Error = (); - fn try_from(value: u8) -> Result { - match value { - x if x == RouterMessageType::Block as u8 => Ok(RouterMessageType::Block), - _ => Err(()) - } - } -} - -pub fn encode_router_message(msg: RouterMessage) -> Result, String> { - match msg { - RouterMessage::Block { block } => { - let mut raw = block.serialize(); - raw.insert(0, RouterMessageType::Block as u8); - Ok(raw) - } - } -} - -pub fn decode_router_message(raw: Vec) -> Result { - if raw.len() == 0 { - return Err(format!("zero len raw message provided")) - } - - let msg_type = RouterMessageType::try_from(raw[0]).unwrap(); - match msg_type { - RouterMessageType::Block => { - let Some(block) = Block::deserialize(&raw[1..]) else { - return Err(format!("unable to deserialize block")) - }; - - Ok(RouterMessage::Block { block }) - } - } -} From 8b49ee35639c077d3c9fe0a15eadb687d0a1c5b6 Mon Sep 17 00:00:00 2001 From: bianyuanop Date: Tue, 1 Apr 2025 15:19:54 -0400 Subject: [PATCH 20/38] remove net::actor mempool from app state --- chain/src/actors/net/actor.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/chain/src/actors/net/actor.rs b/chain/src/actors/net/actor.rs index c9dc57f8..56f4aea0 100644 --- a/chain/src/actors/net/actor.rs +++ b/chain/src/actors/net/actor.rs @@ -38,7 +38,6 @@ type SharedState = Arc>>; #[derive(Clone)] struct AppState { context: R, - mailbox: Mailbox, clients: Clients, block_listeners: Arc>>, tx_listeners: Arc>> @@ -73,7 +72,6 @@ impl Actor { let state = AppState:: { context: context.with_label("app_state"), - mailbox: mailbox.clone(), clients: Arc::new(RwLock::new(HashMap::new())), block_listeners: Arc::new(RwLock::new(HashSet::new())), tx_listeners: Arc::new(RwLock::new(HashSet::new())), From 5827e906908f681452577315cb3d0f67a56e702f Mon Sep 17 00:00:00 2001 From: rikoeldon <106416799+rikoeldon@users.noreply.github.com> Date: Tue, 1 Apr 2025 15:25:14 -0400 Subject: [PATCH 21/38] skeleton client still WIP. --- client/src/client.rs | 50 ++++++++++++++++++++++++++++++++++++++++++++ client/src/lib.rs | 1 + 2 files changed, 51 insertions(+) create mode 100644 client/src/client.rs diff --git a/client/src/client.rs b/client/src/client.rs new file mode 100644 index 00000000..9a20e995 --- /dev/null +++ b/client/src/client.rs @@ -0,0 +1,50 @@ +use reqwest::Client; + +pub const WEBSOCKET_PREFIX: &'static str = "/ws"; +pub const RPC_PREFIX: &'static str = "/api"; +pub const PATH_SUBMIT_TX: &'static str = "/mempool/submit"; + +//todo add more const once we have server + +//todo define all structs needed from server + +#[derive(Debug)] +pub struct JSONRPCClient { + http_client: Client, + base_url: String, + chain_id: String, +} + +impl JSONRPCClient { + pub fn new(mut uri: String, chain_id: String) -> Self { + if uri.ends_with('/') { + uri.pop(); + } + let final_url = format!("{}/jsonrpc", uri); + + Self { + http_client: Client::new(), + base_url: final_url, + chain_id, + } + } + //todo implement methods needed to communicate with server + pub fn submit_tx(&self, data: Vec) { + todo!() + } + pub fn submit_batch_txs(&self, data: Vec) { + todo!() + } + pub fn get_block(&self, data: Vec) { + todo!() + } + pub fn get_prev_block(&self, data: Vec) { + todo!() + } + pub fn get_block_height(&self, data: Vec) { + todo!() + } + pub fn get_payload(&self, data: Vec) { + todo!() + } +} diff --git a/client/src/lib.rs b/client/src/lib.rs index f2068b13..9ec7c672 100644 --- a/client/src/lib.rs +++ b/client/src/lib.rs @@ -6,6 +6,7 @@ use thiserror::Error; pub mod consensus; pub mod utils; +mod client; const LATEST: &str = "latest"; From ce797953b8f4fa3533f66b9af40ee52cd73e0664 Mon Sep 17 00:00:00 2001 From: bianyuanop Date: Wed, 2 Apr 2025 13:36:56 -0400 Subject: [PATCH 22/38] mempool generic fix --- chain/src/actors/mempool/actor.rs | 14 +-- chain/src/actors/mempool/collector.rs | 26 ++-- chain/src/actors/mempool/handler.rs | 23 ++-- chain/src/actors/mempool/ingress.rs | 24 ++-- chain/src/actors/mempool/key.rs | 102 +++++++++------- chain/src/actors/mempool/mempool.rs | 164 +++++++++++++------------- chain/src/actors/mempool/mod.rs | 24 ++-- chain/src/actors/net/actor.rs | 39 +++--- chain/src/actors/net/mod.rs | 14 ++- 9 files changed, 233 insertions(+), 197 deletions(-) diff --git a/chain/src/actors/mempool/actor.rs b/chain/src/actors/mempool/actor.rs index ae47cd76..6ee7066b 100644 --- a/chain/src/actors/mempool/actor.rs +++ b/chain/src/actors/mempool/actor.rs @@ -1,6 +1,6 @@ use super::{ ingress::{Mailbox, Message}, mempool}; use commonware_broadcast::Broadcaster; -use commonware_cryptography::Digest; +use commonware_cryptography::{Digest, Hasher}; use commonware_utils::Array; use futures::{ channel::mpsc, @@ -9,19 +9,19 @@ use futures::{ use tracing::{error, warn, debug}; -pub struct Actor { - mailbox: mpsc::Receiver>, +pub struct Actor { + mailbox: mpsc::Receiver>, } -impl Actor { - pub fn new() -> (Self, Mailbox) { +impl Actor { + pub fn new() -> (Self, Mailbox) { let (sender, receiver) = mpsc::channel(1024); (Actor { mailbox: receiver }, Mailbox::new(sender)) } pub async fn run(mut self, - mut engine: impl Broadcaster, - mut mempool: mempool::Mailbox + mut engine: impl Broadcaster, + mut mempool: mempool::Mailbox ) { // it passes msgs in the mailbox of the actor to the engine mailbox while let Some(msg) = self.mailbox.next().await { diff --git a/chain/src/actors/mempool/collector.rs b/chain/src/actors/mempool/collector.rs index 9b746887..21347b41 100644 --- a/chain/src/actors/mempool/collector.rs +++ b/chain/src/actors/mempool/collector.rs @@ -1,5 +1,5 @@ use commonware_broadcast::{linked::Prover, Collector as Z, Proof, }; -use commonware_cryptography::{bls12381::primitives::group, Digest, Scheme}; +use commonware_cryptography::{bls12381::primitives::group, Digest, Hasher, Scheme}; use futures::{ channel::mpsc, SinkExt, StreamExt, @@ -8,13 +8,13 @@ use tracing::error; use super::mempool; -enum Message { - Acknowledged(Proof, D), +enum Message { + Acknowledged(Proof, H::Digest), _Phantom(C::PublicKey), } -pub struct Collector { - mailbox: mpsc::Receiver>, +pub struct Collector { + mailbox: mpsc::Receiver>, // Application namespace namespace: Vec, @@ -23,8 +23,8 @@ pub struct Collector { public: group::Public, } -impl Collector { - pub fn new(namespace: &[u8], public: group::Public) -> (Self, Mailbox) { +impl Collector { + pub fn new(namespace: &[u8], public: group::Public) -> (Self, Mailbox) { let (sender, receiver) = mpsc::channel(1024); ( Collector { @@ -36,13 +36,13 @@ impl Collector { ) } - pub async fn run(mut self, mut mempool: mempool::Mailbox) { + pub async fn run(mut self, mut mempool: mempool::Mailbox) { while let Some(msg) = self.mailbox.next().await { match msg { Message::Acknowledged(proof, payload) => { // Check proof. // The prover checks the validity of the threshold signature when deserializing - let prover = Prover::::new(&self.namespace, self.public); + let prover = Prover::::new(&self.namespace, self.public); let _ = match prover.deserialize_threshold(proof) { Some((context, _payload, _epoch, _threshold)) => context, None => { @@ -64,12 +64,12 @@ impl Collector { } #[derive(Clone)] -pub struct Mailbox { - sender: mpsc::Sender>, +pub struct Mailbox { + sender: mpsc::Sender>, } -impl Z for Mailbox { - type Digest = D; +impl Z for Mailbox { + type Digest = H::Digest; async fn acknowledged(&mut self, proof: Proof, payload: Self::Digest) { self.sender .send(Message::Acknowledged(proof, payload)) diff --git a/chain/src/actors/mempool/handler.rs b/chain/src/actors/mempool/handler.rs index e0f014e2..9b8221f1 100644 --- a/chain/src/actors/mempool/handler.rs +++ b/chain/src/actors/mempool/handler.rs @@ -1,5 +1,6 @@ use super::key::MultiIndex; use bytes::Bytes; +use commonware_cryptography::{Digest, Hasher}; use commonware_resolver::{p2p::Producer, Consumer}; use futures::{ channel::{mpsc, oneshot}, @@ -7,32 +8,32 @@ use futures::{ }; use tracing::warn; -pub enum Message { +pub enum Message { Deliver { - key: MultiIndex, + key: MultiIndex, value: Bytes, response: oneshot::Sender, }, Produce { - key: MultiIndex, + key: MultiIndex, response: oneshot::Sender, }, } /// Mailbox for resolver #[derive(Clone)] -pub struct Handler { - sender: mpsc::Sender, +pub struct Handler { + sender: mpsc::Sender>, } -impl Handler { - pub(super) fn new(sender: mpsc::Sender) -> Self { +impl Handler { + pub(super) fn new(sender: mpsc::Sender>) -> Self { Self { sender } } } -impl Consumer for Handler { - type Key = MultiIndex; +impl Consumer for Handler { + type Key = MultiIndex; type Value = Bytes; type Failure = (); @@ -54,8 +55,8 @@ impl Consumer for Handler { } } -impl Producer for Handler { - type Key = MultiIndex; +impl Producer for Handler { + type Key = MultiIndex; async fn produce(&mut self, key: Self::Key) -> oneshot::Receiver { let (response, receiver) = oneshot::channel(); diff --git a/chain/src/actors/mempool/ingress.rs b/chain/src/actors/mempool/ingress.rs index e8dd1def..c02adc15 100644 --- a/chain/src/actors/mempool/ingress.rs +++ b/chain/src/actors/mempool/ingress.rs @@ -1,5 +1,7 @@ +use std::hash::Hash; + use commonware_utils::Array; -use commonware_cryptography::Digest; +use commonware_cryptography::{Digest, Hasher}; use commonware_broadcast::{linked::Context, Application as A}; use futures::{ channel::{mpsc, oneshot}, SinkExt}; @@ -8,31 +10,31 @@ pub struct Payload { data: Vec } -pub enum Message { - Broadcast(D), - Verify(Context

, D, oneshot::Sender), +pub enum Message { + Broadcast(H::Digest), + Verify(Context

, H::Digest, oneshot::Sender), } #[derive(Clone)] -pub struct Mailbox { - sender: mpsc::Sender>, +pub struct Mailbox { + sender: mpsc::Sender>, } -impl Mailbox { - pub(super) fn new(sender: mpsc::Sender>) -> Self { +impl Mailbox { + pub(super) fn new(sender: mpsc::Sender>) -> Self { Self { sender } } - pub async fn broadcast(&mut self, payload: D) { + pub async fn broadcast(&mut self, payload: H::Digest) { let _ = self.sender.send(Message::Broadcast(payload)).await; } } -impl A for Mailbox { +impl A for Mailbox { type Context = Context

; - type Digest = D; + type Digest = H::Digest; async fn verify( &mut self, diff --git a/chain/src/actors/mempool/key.rs b/chain/src/actors/mempool/key.rs index d6ed6b98..b4cb79bd 100644 --- a/chain/src/actors/mempool/key.rs +++ b/chain/src/actors/mempool/key.rs @@ -1,14 +1,14 @@ -use commonware_cryptography::sha256::Digest; +use commonware_cryptography::Digest; use commonware_utils::{Array, SizedSerialize}; use std::{ - cmp::{Ord, PartialOrd}, - fmt::{Debug, Display}, - hash::Hash, - ops::Deref, + cmp::{Ord, PartialOrd}, fmt::{Debug, Display}, hash::Hash, marker::PhantomData, ops::Deref }; use thiserror::Error; -const SERIALIZED_LEN: usize = 1 + Digest::SERIALIZED_LEN; +// to resolve issue of https://github.com/rust-lang/rust/issues/76560 +// the first byte of MultiIndex indicates the index type +// the rest bytes stores key for that type, e.g. a sha256 index would be [0 | digest(32) | rest(31)] +const SERIALIZED_LEN: usize = 64; #[derive(Error, Debug, PartialEq)] pub enum Error { @@ -16,31 +16,42 @@ pub enum Error { InvalidLength, } -pub enum Value { - Digest(Digest), +pub enum Value { + Digest(D), } #[derive(Clone, Eq, PartialEq, Ord, PartialOrd, Hash)] #[repr(transparent)] -pub struct MultiIndex([u8; SERIALIZED_LEN]); +pub struct MultiIndex { + index: [u8; SERIALIZED_LEN], -impl MultiIndex { - pub fn new(value: Value) -> Self { + _marker: PhantomData +} + + +impl MultiIndex { + const DIGEST_LENGTH: usize = D::SERIALIZED_LEN; + + pub fn new(value: Value) -> Self { let mut bytes = [0; SERIALIZED_LEN]; match value { Value::Digest(digest) => { bytes[0] = 0; - bytes[1..].copy_from_slice(&digest); + bytes[1..(1+D::SERIALIZED_LEN)].copy_from_slice(&digest); } } - Self(bytes) + Self { + index: bytes, + + _marker: PhantomData + } } - pub fn to_value(&self) -> Value { - match self.0[0] { + pub fn to_value(&self) -> Value { + match self.index[0] { 0 => { - let bytes: [u8; Digest::SERIALIZED_LEN] = self.0[1..].try_into().unwrap(); - let digest = Digest::from(bytes); + let bytes: Vec = self.index[1..(1+Self::DIGEST_LENGTH)].to_vec(); + let digest = D::try_from(bytes).unwrap(); Value::Digest(digest) } _ => unreachable!(), @@ -48,34 +59,42 @@ impl MultiIndex { } } -impl Array for MultiIndex { +impl Array for MultiIndex { type Error = Error; } -impl SizedSerialize for MultiIndex { +impl SizedSerialize for MultiIndex { const SERIALIZED_LEN: usize = SERIALIZED_LEN; } -impl From<[u8; MultiIndex::SERIALIZED_LEN]> for MultiIndex { - fn from(value: [u8; MultiIndex::SERIALIZED_LEN]) -> Self { - Self(value) +impl From<[u8; SERIALIZED_LEN]> for MultiIndex { + fn from(value: [u8; SERIALIZED_LEN]) -> Self { + + Self { + index: value, + + _marker: PhantomData + } } } -impl TryFrom<&[u8]> for MultiIndex { +impl TryFrom<&[u8]> for MultiIndex { type Error = Error; fn try_from(value: &[u8]) -> Result { - if value.len() != MultiIndex::SERIALIZED_LEN { + if value.len() != SERIALIZED_LEN { return Err(Error::InvalidLength); } - let array: [u8; MultiIndex::SERIALIZED_LEN] = + let array: [u8; SERIALIZED_LEN] = value.try_into().map_err(|_| Error::InvalidLength)?; - Ok(Self(array)) + Ok(Self{ + index: array, + _marker: PhantomData + }) } } -impl TryFrom<&Vec> for MultiIndex { +impl TryFrom<&Vec> for MultiIndex { type Error = Error; fn try_from(value: &Vec) -> Result { @@ -83,49 +102,52 @@ impl TryFrom<&Vec> for MultiIndex { } } -impl TryFrom> for MultiIndex { +impl TryFrom> for MultiIndex { type Error = Error; fn try_from(value: Vec) -> Result { - if value.len() != MultiIndex::SERIALIZED_LEN { + if value.len() != SERIALIZED_LEN { return Err(Error::InvalidLength); } // If the length is correct, we can safely convert the vector into a boxed slice without any // copies. let boxed_slice = value.into_boxed_slice(); - let boxed_array: Box<[u8; MultiIndex::SERIALIZED_LEN]> = + let boxed_array: Box<[u8; SERIALIZED_LEN]> = boxed_slice.try_into().map_err(|_| Error::InvalidLength)?; - Ok(Self(*boxed_array)) + Ok(Self { + index: *boxed_array, + _marker: PhantomData + }) } } -impl AsRef<[u8]> for MultiIndex { +impl AsRef<[u8]> for MultiIndex { fn as_ref(&self) -> &[u8] { - &self.0 + &self.index } } -impl Deref for MultiIndex { +impl Deref for MultiIndex { type Target = [u8]; fn deref(&self) -> &[u8] { - &self.0 + &self.index } } -impl Debug for MultiIndex { +impl Debug for MultiIndex { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self.0[0] { + match self.index[0] { 0 => { - let bytes: [u8; Digest::SERIALIZED_LEN] = self.0[1..].try_into().unwrap(); - write!(f, "digest({})", Digest::from(bytes)) + let bytes: Vec = self.index[1..(1+D::SERIALIZED_LEN)].to_vec(); + write!(f, "digest({})", D::try_from(bytes).unwrap()) } _ => unreachable!(), } } } -impl Display for MultiIndex { +impl Display for MultiIndex { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { Debug::fmt(self, f) } diff --git a/chain/src/actors/mempool/mempool.rs b/chain/src/actors/mempool/mempool.rs index ebffb985..2fb838db 100644 --- a/chain/src/actors/mempool/mempool.rs +++ b/chain/src/actors/mempool/mempool.rs @@ -1,4 +1,4 @@ -use std::{collections::{HashMap}, time::{Duration, SystemTime}}; +use std::{collections::HashMap, hash::Hash, time::{Duration, SystemTime}}; use bytes::{BufMut, Bytes}; use commonware_cryptography::{ed25519::PublicKey, sha256, Digest, Hasher, Sha256}; @@ -20,26 +20,25 @@ use super::{handler::{Handler, self}, key::{self, MultiIndex, Value}, ingress, c use crate::{actors::net, maybe_delay_between}; #[derive(Clone, Debug)] -pub struct Batch { +pub struct Batch { pub timestamp: SystemTime, - pub txs: Vec>, - pub digest: D, + // TODO: store real transactions not just raws + pub txs: Vec>, + pub digest: H::Digest, } -impl Batch - where Sha256: Hasher -{ - fn compute_digest(txs: &Vec>) -> D { - let mut hasher = Sha256::new(); +impl Batch { + fn compute_digest(txs: &Vec>) -> H::Digest { + let mut hasher = H::new(); - for tx in txs.into_iter() { - hasher.update(tx.raw.as_ref()); + for tx in txs.iter() { + hasher.update(&tx.raw); } hasher.finalize() } - pub fn new(txs: Vec>, timestamp: SystemTime) -> Self { + pub fn new(txs: Vec>, timestamp: SystemTime) -> Self { let digest = Self::compute_digest(&txs); Self { @@ -96,27 +95,25 @@ impl Batch }) } - pub fn contain_tx(&self, digest: &D) -> bool { + pub fn contain_tx(&self, digest: &H::Digest) -> bool { self.txs.iter().any(|tx| &tx.digest == digest) } - pub fn tx(&self, digest: &D) -> Option> { + pub fn tx(&self, digest: &H::Digest) -> Option> { self.txs.iter().find(|tx| &tx.digest == digest).cloned() } } #[derive(Clone, Debug)] -pub struct RawTransaction { +pub struct RawTransaction { pub raw: Bytes, - pub digest: D + pub digest: H::Digest } -impl RawTransaction - where Sha256: Hasher -{ - fn compute_digest(raw: &Bytes) -> D { - let mut hasher = Sha256::new(); +impl RawTransaction { + fn compute_digest(raw: &Bytes) -> H::Digest { + let mut hasher = H::new(); hasher.update(&raw); hasher.finalize() } @@ -139,62 +136,60 @@ impl RawTransaction } } -impl From for RawTransaction - where Sha256: Hasher -{ +impl From for RawTransaction { fn from(value: net::actor::DummyTransaction) -> Self { let raw = Bytes::from(value.payload); RawTransaction::new(raw) } } -pub enum Message { +pub enum Message { // mark batch as accepted by the netowrk through the broadcast protocol BatchAcknowledged { - digest: D, + digest: H::Digest, response: oneshot::Sender }, // from rpc or websocket - SubmitTx { - payload: RawTransaction, - response: oneshot::Sender + SubmitTxs { + payload: Vec>, + response: oneshot::Sender> }, BatchConsumed { - digests: Vec, + digests: Vec, block_number: u64, response: oneshot::Sender, }, // proposer consume batches to produce a block ConsumeBatches { - response: oneshot::Sender>> + response: oneshot::Sender>> }, GetTx { - digest: D, - response: oneshot::Sender>> + digest: H::Digest, + response: oneshot::Sender>> }, GetBatch { - digest: D, - response: oneshot::Sender>> + digest: H::Digest, + response: oneshot::Sender>> }, GetBatchContainTx { - digest: D, - response: oneshot::Sender>> + digest: H::Digest, + response: oneshot::Sender>> } } #[derive(Clone)] -pub struct Mailbox { - sender: mpsc::Sender> +pub struct Mailbox { + sender: mpsc::Sender> } -impl Mailbox { - pub fn new(sender: mpsc::Sender>) -> Self { +impl Mailbox { + pub fn new(sender: mpsc::Sender>) -> Self { Self { sender } } - pub async fn acknowledge_batch(&mut self, digest: D) -> bool { + pub async fn acknowledge_batch(&mut self, digest: H::Digest) -> bool { let (response, receiver) = oneshot::channel(); self.sender .send(Message::BatchAcknowledged { digest, response}) @@ -204,17 +199,17 @@ impl Mailbox { receiver.await.expect("failed to receive batch acknowledge") } - pub async fn issue_tx(&mut self, tx: RawTransaction) -> bool { + pub async fn submit_txs(&mut self, payload: Vec>) -> Vec { let (response, receiver) = oneshot::channel(); self.sender - .send(Message::SubmitTx { payload: tx, response }) + .send(Message::SubmitTxs { payload, response }) .await .expect("failed to issue tx"); receiver.await.expect("failed to receive tx issue status") } - pub async fn consume_batches(&mut self) -> Vec> { + pub async fn consume_batches(&mut self) -> Vec> { let (response, receiver) = oneshot::channel(); self.sender .send(Message::ConsumeBatches { response }) @@ -224,7 +219,7 @@ impl Mailbox { receiver.await.expect("failed to receive batches") } - pub async fn consumed_batches(&mut self, digests: Vec, block_number: u64) -> bool { + pub async fn consumed_batches(&mut self, digests: Vec, block_number: u64) -> bool { let (response, receiver) = oneshot::channel(); self.sender .send(Message::BatchConsumed { digests, block_number, response }) @@ -234,7 +229,7 @@ impl Mailbox { receiver.await.expect("failed to mark batches as consumed") } - pub async fn get_tx(&mut self, digest: D) -> Option> { + pub async fn get_tx(&mut self, digest: H::Digest) -> Option> { let (response, receiver) = oneshot::channel(); self.sender .send(Message::GetTx { digest, response }) @@ -244,7 +239,7 @@ impl Mailbox { receiver.await.expect("failed to receive tx") } - pub async fn get_batch(&mut self, digest: D) -> Option> { + pub async fn get_batch(&mut self, digest: H::Digest) -> Option> { let (response, receiver) = oneshot::channel(); self.sender .send(Message::GetBatch { digest, response }) @@ -254,7 +249,7 @@ impl Mailbox { receiver.await.expect("failed to receive batch") } - pub async fn get_batch_contain_tx(&mut self, digest: D) -> Option> { + pub async fn get_batch_contain_tx(&mut self, digest: H::Digest) -> Option> { let (response, receiver) = oneshot::channel(); self.sender .send(Message::GetBatchContainTx { digest, response }) @@ -278,24 +273,24 @@ pub struct Config { pub struct Mempool< B: Blob, R: Rng + Clock + GClock + Spawner + Metrics + Storage, - D: Digest + Into + From + H: Hasher > { context: R, public_key: PublicKey, - batches: HashMap>, + batches: HashMap>, - acknowledged: Vec, + acknowledged: Vec, //TODO: replace the following two - accepted: Archive, - consumed: Archive, + accepted: Archive, + consumed: Archive, - txs: Vec>, + txs: Vec>, - mailbox: mpsc::Receiver>, + mailbox: mpsc::Receiver>, mailbox_size: usize, block_height_seen: u64, @@ -308,12 +303,9 @@ pub struct Mempool< impl< B: Blob, R: Rng + Clock + GClock + Spawner + Metrics + Storage, - D: Digest + Into + From -> Mempool - where - Sha256: Hasher, -{ - pub async fn init(context: R, cfg: Config) -> (Self, Mailbox) { + H: Hasher, +> Mempool { + pub async fn init(context: R, cfg: Config) -> (Self, Mailbox) { let accepted_journal = Journal::init( context.with_label("accepted_journal"), journal::variable::Config { @@ -387,7 +379,7 @@ impl< impl Receiver, ), coordinator: Coordinator, - app_mailbox: ingress::Mailbox + app_mailbox: ingress::Mailbox ) -> Handle<()> { self.context.spawn_ref()(self.run(batch_network, backfill_network, coordinator, app_mailbox)) } @@ -403,10 +395,10 @@ impl< impl Receiver, ), coordinator: Coordinator, - mut app_mailbox: ingress::Mailbox + mut app_mailbox: ingress::Mailbox ) { let (handler_sender, mut handler_receiver) = mpsc::channel(self.mailbox_size); - let handler = Handler::new(handler_sender); + let handler = Handler::::new(handler_sender); let (resolver_engine, mut resolver) = p2p::Engine::new( self.context.with_label("resolver"), p2p::Config { @@ -427,7 +419,7 @@ impl< ); resolver_engine.start(backfill_network); - let mut waiters: HashMap>>>> = HashMap::new(); + let mut waiters: HashMap>>>> = HashMap::new(); let mut propose_timeout = self.context.current() + self.batch_propose_interval; let accepted = Wrapped::new(self.accepted); let consumed = Wrapped::new(self.consumed); @@ -445,14 +437,17 @@ impl< return; }; match message { - Message::SubmitTx { payload, response } => { - if !payload.validate() { - let _ = response.send(false); - return; + Message::SubmitTxs { payload, response } => { + let mut result = Vec::with_capacity(payload.len()); + for tx in payload.into_iter() { + if !tx.validate() { + result.push(false); + continue + } + self.txs.push(tx); + result.push(true); } - - self.txs.push(payload); - let _ = response.send(true); + let _ = response.send(result); }, // batch ackowledged by the network Message::BatchAcknowledged { digest, response } => { @@ -483,7 +478,7 @@ impl< // update the seen height self.block_height_seen = block_number; - let consumed_keys: Vec = self.batches.iter() + let consumed_keys: Vec = self.batches.iter() .filter_map(|(digest, batch)| { if digests.contains(&batch.digest) { Some(digest.clone()) @@ -500,7 +495,7 @@ impl< } // remove digests and batches - let consumed_batches: Vec> = consumed_keys.into_iter() + let consumed_batches: Vec> = consumed_keys.into_iter() .filter_map(|key| self.batches.remove(&key)) .collect(); @@ -606,18 +601,19 @@ impl< handler::Message::Produce { key, response } => { match key.to_value() { key::Value::Digest(digest) => { - if let Some(batch) = self.batches.get(&D::from(digest)).cloned() { + if let Some(batch) = self.batches.get(&digest).cloned() { let _ = response.send(batch.serialize().into()); continue; }; - let consumed = consumed.get(Identifier::Key(&D::from(digest))) - .await - .expect("Failed to get accepted batch"); - if let Some(consumed) = consumed { - let _ = response.send(consumed); - continue; - } + // TODO: replace with new key-value db + // let consumed = consumed.get(Identifier::Key(&H::Digest::from(digest))) + // .await + // .expect("Failed to get accepted batch"); + // if let Some(consumed) = consumed { + // let _ = response.send(consumed); + // continue; + // } debug!(?digest, "missing batch"); } } @@ -626,7 +622,7 @@ impl< match key.to_value() { key::Value::Digest(digest) => { let batch = Batch::deserialize(&value).expect("Failed to deserialize batch"); - if batch.digest.into() != digest { + if batch.digest != digest { let _ = response.send(false); continue; } diff --git a/chain/src/actors/mempool/mod.rs b/chain/src/actors/mempool/mod.rs index 0b73aece..0516af63 100644 --- a/chain/src/actors/mempool/mod.rs +++ b/chain/src/actors/mempool/mod.rs @@ -17,7 +17,7 @@ mod tests { use governor::Quota; use tracing::{debug, info, warn}; - use commonware_cryptography::{bls12381::{dkg, primitives::{group::Share, poly}}, ed25519::PublicKey, sha256, Ed25519, Scheme}; + use commonware_cryptography::{bls12381::{dkg, primitives::{group::Share, poly}}, ed25519::PublicKey, sha256, Ed25519, Scheme, Sha256}; use commonware_macros::test_traced; use commonware_p2p::simulated::{Oracle, Receiver, Sender, Link, Network}; use commonware_runtime::{deterministic::{Context, Executor}, Clock, Metrics, Runner, Spawner}; @@ -142,10 +142,10 @@ mod tests { pks: &[PublicKey], validators: &[(PublicKey, Ed25519, Share)], registrations: &mut Registrations, - collectors: &mut BTreeMap>, + collectors: &mut BTreeMap>, refresh_epoch_timeout: Duration, rebroadcast_timeout: Duration, - ) -> BTreeMap> { + ) -> BTreeMap> { let mut mailboxes = BTreeMap::new(); let namespace = b"my testing namespace"; for (validator, scheme, share) in validators.iter() { @@ -170,11 +170,11 @@ mod tests { coordinator.set_view(111); let (app, app_mailbox) = - super::actor::Actor::::new(); + super::actor::Actor::::new(); let collector_mempool_mailbox = mempool_mailbox.clone(); let (collector, collector_mailbox) = - super::collector::Collector::::new( + super::collector::Collector::::new( namespace, *poly::public(&identity), ); @@ -215,9 +215,9 @@ mod tests { pks: &[PublicKey], validators: &[(PublicKey, Ed25519, Share)], registrations: &mut Registrations, - app_mailbox: &mut ingress::Mailbox, + app_mailbox: &mut ingress::Mailbox, // mailboxes: &mut BTreeMap>, - ) -> BTreeMap> { + ) -> BTreeMap> { let mut mailboxes= BTreeMap::new(); for (validator, _, share) in validators.iter() { let context = context.with_label(&validator.to_string()); @@ -250,7 +250,7 @@ mod tests { async fn spawn_tx_issuer_and_wait( context: Context, - mailboxes: Arc>>>, + mailboxes: Arc>>>, num_txs: u32, wait_batch_acknowlegement: bool, consume_batch: bool, @@ -259,7 +259,7 @@ mod tests { .clone() .with_label("tx issuer") .spawn(move |context| async move { - let mut mailbox_vec: Vec> = { + let mut mailbox_vec: Vec> = { let guard = mailboxes.lock().unwrap(); guard.values().cloned().collect() }; @@ -277,8 +277,8 @@ mod tests { let mut digests = Vec::new(); for i in 0..num_txs { let tx = RawTransaction::new(Bytes::from(format!("tx-{}", i))); - let submission_res = mailbox.issue_tx(tx.clone()).await; - if !submission_res { + let submission_res = mailbox.submit_txs(vec![tx.clone()]).await; + if !submission_res[0] { warn!(?tx.digest, "failed to submit tx"); continue; } @@ -339,7 +339,7 @@ mod tests { context.with_label("simulation"), num_validators, &mut shares_vec).await; - let mut collectors = BTreeMap::>::new(); + let mut collectors = BTreeMap::>::new(); let mailboxes = spawn_validator_engines( context.with_label("validator"), identity.clone(), diff --git a/chain/src/actors/net/actor.rs b/chain/src/actors/net/actor.rs index 56f4aea0..0b7054d8 100644 --- a/chain/src/actors/net/actor.rs +++ b/chain/src/actors/net/actor.rs @@ -1,5 +1,5 @@ use std::{ - collections::{HashMap, HashSet}, io, ops::Deref, sync::{Arc, RwLock} + collections::{HashMap, HashSet}, hash::Hash, io, ops::Deref, sync::{Arc, RwLock} }; use alto_client::Client; use alto_types::Block; @@ -12,7 +12,7 @@ use axum::{ }; use bytes::Bytes; -use commonware_cryptography::{sha256, Digest}; +use commonware_cryptography::{sha256, Digest, Hasher}; use commonware_runtime::{Clock, Handle, Metrics, Spawner}; use futures::{channel::{mpsc, oneshot}, lock::Mutex, SinkExt, StreamExt}; use rand::Rng; @@ -21,6 +21,8 @@ use tokio::net::TcpListener; use tracing::{debug, event, Level, error}; use tracing_subscriber::fmt::format; +use crate::actors::mempool::mempool; + use super::ingress::{Mailbox, Message, WebsocketClientMessage}; #[derive(Deserialize)] @@ -33,36 +35,39 @@ type ClientSender = mpsc::Sender>; type ClientID = String; type Clients = Arc>>; -type SharedState = Arc>>; +type SharedState = Arc>>; #[derive(Clone)] -struct AppState { +struct AppState { context: R, clients: Clients, + mempool: mempool::Mailbox, block_listeners: Arc>>, tx_listeners: Arc>> } -pub struct Config { +pub struct Config { pub port: i32, + + pub mempool: mempool::Mailbox } -pub struct Actor { +pub struct Actor { context: R, port: i32, listener: Option, pub router: Option, is_active: bool, - state: SharedState + state: SharedState } -impl Actor { +impl Actor { pub const WEBSOCKET_PREFIX: &'static str = "/ws"; pub const RPC_PREFIX: &'static str = "/api"; pub const PATH_SUBMIT_TX: &'static str = "/mempool/submit"; - pub fn new(context: R, cfg: Config) -> (Self, Mailbox) { + pub fn new(context: R, cfg: Config) -> (Self, Mailbox) { if cfg.port == 0 { panic!("Invalid port number"); } @@ -70,8 +75,9 @@ impl Actor { let (sender, mut receiver) = mpsc::channel(1024); let mailbox = Mailbox::new(sender); - let state = AppState:: { + let state = AppState:: { context: context.with_label("app_state"), + mempool: cfg.mempool, clients: Arc::new(RwLock::new(HashMap::new())), block_listeners: Arc::new(RwLock::new(HashSet::new())), tx_listeners: Arc::new(RwLock::new(HashSet::new())), @@ -121,7 +127,7 @@ impl Actor { } /// handles messages from other services within a node such as block messages - async fn handle_message(state: SharedState, msg: Message) { + async fn handle_message(state: SharedState, msg: Message) { println!("handling msg {:?}", msg); let block_listeners = state.read().unwrap().block_listeners.clone(); let clients = state.read().unwrap().clients.clone(); @@ -147,13 +153,13 @@ impl Actor { async fn ws_handler( ws: WebSocketUpgrade, - State(state): State>, + State(state): State>, ) -> impl IntoResponse { let client_id = uuid::Uuid::new_v4().to_string(); ws.on_upgrade(move |socket| Self::handle_socket(socket, client_id, state)) } - async fn handle_socket(mut socket: WebSocket, client_id: ClientID, state: SharedState) { + async fn handle_socket(mut socket: WebSocket, client_id: ClientID, state: SharedState) { let (mut socket_sender, mut socket_receiver) = socket.split(); let (tx, mut rx) = mpsc::channel::>(1024); @@ -194,7 +200,10 @@ impl Actor { state.tx_listeners.write().unwrap().insert(client_id.clone()); }, WebsocketClientMessage::SubmitTxs(txs) => { - unimplemented!() + let mut mempool = state.mempool.clone(); + let txs: Vec<_> = txs.into_iter().map(|tx| mempool::RawTransaction::::new(tx)).collect(); + // let submission_res = mempool.submit_txs(txs).await; + // debug!(?submission_res, "txs submission result") } } }, @@ -223,7 +232,7 @@ impl Actor { } async fn handle_submit_tx( - State(state): State>, + State(state): State>, payload: Bytes, ) -> impl IntoResponse { // TODO: send to mempool mailbox diff --git a/chain/src/actors/net/mod.rs b/chain/src/actors/net/mod.rs index beb6b65b..c685c18f 100644 --- a/chain/src/actors/net/mod.rs +++ b/chain/src/actors/net/mod.rs @@ -10,7 +10,7 @@ mod tests { body::Body, http::{Request, StatusCode} }; - use commonware_cryptography::sha256; + use commonware_cryptography::{sha256, Sha256}; use commonware_macros::{test_async, test_traced}; use commonware_runtime::{tokio::{self, Context, Executor}, Clock, Metrics, Runner, Spawner}; use futures::{channel::mpsc, future::join_all, SinkExt, StreamExt}; @@ -18,7 +18,7 @@ mod tests { use tower::ServiceExt; use tracing_subscriber::field::debug; - use crate::actors::net::ingress::WebsocketClientMessage; + use crate::actors::{mempool::mempool, net::ingress::WebsocketClientMessage}; use super::{actor::Actor, ingress::Message, actor::{self}}; use tracing::debug; @@ -27,8 +27,11 @@ mod tests { fn test_msg() { let (runner, mut context) = Executor::init(tokio::Config::default()); runner.start(async move { + let (mempool_sender, mempool_receiver) = mpsc::channel(1024); + let mempool_mailbox: mempool::Mailbox = mempool::Mailbox::new(mempool_sender); let (actor, mailbox) = Actor::new(context, actor::Config { - port: 7890 + port: 7890, + mempool: mempool_mailbox }); let Some(router) = actor.router else { @@ -58,8 +61,11 @@ mod tests { fn test_ws() { let (runner, mut context) = Executor::default(); runner.start(async move { + let (mempool_sender, mempool_receiver) = mpsc::channel(1024); + let mempool_mailbox: mempool::Mailbox = mempool::Mailbox::new(mempool_sender); let (actor, mut mailbox) = Actor::new(context.with_label("router"), actor::Config { - port: 7890 + port: 7890, + mempool: mempool_mailbox }); debug!("starting router"); From 5dc650163a5b0da380116c24d27dcba6b3ffe565 Mon Sep 17 00:00:00 2001 From: bianyuanop Date: Wed, 2 Apr 2025 14:35:02 -0400 Subject: [PATCH 23/38] tx submission --- chain/src/actors/net/actor.rs | 28 ++++++++++++++++++++-------- chain/src/actors/net/mod.rs | 24 +++++++++++++++++++----- chain/src/bin/validator.rs | 6 +++--- 3 files changed, 42 insertions(+), 16 deletions(-) diff --git a/chain/src/actors/net/actor.rs b/chain/src/actors/net/actor.rs index 0b7054d8..2d021155 100644 --- a/chain/src/actors/net/actor.rs +++ b/chain/src/actors/net/actor.rs @@ -21,7 +21,7 @@ use tokio::net::TcpListener; use tracing::{debug, event, Level, error}; use tracing_subscriber::fmt::format; -use crate::actors::mempool::mempool; +use crate::actors::mempool::mempool::{self, RawTransaction}; use super::ingress::{Mailbox, Message, WebsocketClientMessage}; @@ -190,20 +190,24 @@ impl Actor { match WebsocketClientMessage::deserialize(bin.deref()) { Ok(msg) => { debug!(?msg, "received msg from client"); - let state = state.write().unwrap(); match msg { WebsocketClientMessage::RegisterBlock => { println!("adding block listener {}", client_id); + let state = state.write().unwrap(); state.block_listeners.write().unwrap().insert(client_id.clone()); }, WebsocketClientMessage::RegisterTx => { + let state = state.write().unwrap(); state.tx_listeners.write().unwrap().insert(client_id.clone()); }, WebsocketClientMessage::SubmitTxs(txs) => { - let mut mempool = state.mempool.clone(); - let txs: Vec<_> = txs.into_iter().map(|tx| mempool::RawTransaction::::new(tx)).collect(); - // let submission_res = mempool.submit_txs(txs).await; - // debug!(?submission_res, "txs submission result") + let mut mempool = { + let state = state.write().unwrap(); + state.mempool.clone() + }; + let txs = txs.into_iter().map(|tx| mempool::RawTransaction::::new(tx)).collect(); + let submission_res = mempool.submit_txs(txs).await; + debug!(?submission_res, "txs submission result") } } }, @@ -235,8 +239,16 @@ impl Actor { State(state): State>, payload: Bytes, ) -> impl IntoResponse { - // TODO: send to mempool mailbox - format!("submitted") + let mut mempool = { + state.read().unwrap().mempool.clone() + }; + + let success = mempool.submit_txs(vec![RawTransaction::new(payload)]).await[0]; + if success { + format!("submitted") + } else { + format!("failed to submit tx") + } } diff --git a/chain/src/actors/net/mod.rs b/chain/src/actors/net/mod.rs index c685c18f..06a9162c 100644 --- a/chain/src/actors/net/mod.rs +++ b/chain/src/actors/net/mod.rs @@ -13,10 +13,10 @@ mod tests { use commonware_cryptography::{sha256, Sha256}; use commonware_macros::{test_async, test_traced}; use commonware_runtime::{tokio::{self, Context, Executor}, Clock, Metrics, Runner, Spawner}; - use futures::{channel::mpsc, future::join_all, SinkExt, StreamExt}; + use futures::{channel::mpsc, future::{join_all, try_join_all}, SinkExt, StreamExt}; use tokio_tungstenite::{connect_async, tungstenite::{client, Message as WsClientMessage}}; use tower::ServiceExt; - use tracing_subscriber::field::debug; + use tracing_subscriber::{field::debug, fmt::format}; use crate::actors::{mempool::mempool, net::ingress::WebsocketClientMessage}; @@ -27,9 +27,9 @@ mod tests { fn test_msg() { let (runner, mut context) = Executor::init(tokio::Config::default()); runner.start(async move { - let (mempool_sender, mempool_receiver) = mpsc::channel(1024); + let (mempool_sender, mut mempool_receiver) = mpsc::channel(1024); let mempool_mailbox: mempool::Mailbox = mempool::Mailbox::new(mempool_sender); - let (actor, mailbox) = Actor::new(context, actor::Config { + let (actor, mailbox) = Actor::new(context.with_label("router"), actor::Config { port: 7890, mempool: mempool_mailbox }); @@ -38,11 +38,24 @@ mod tests { panic!("router not initalized"); }; + let mempool_handler = context.with_label("mock_mempool").spawn(async move |_| { + while let Some(msg) = mempool_receiver.next().await { + match msg { + mempool::Message::SubmitTxs { payload, response } => { + print!("received txs from rpc: {:?}", payload); + let _ = response.send(vec![true; payload.len()]); + return; + }, + _ => unreachable!() + } + } + }); + // Construct a GET request. // Note: the handler expects a payload (a String). Since GET requests normally have no body, // you might decide to pass the payload as a query parameter or in the body if that's what you intend. // Here, we'll assume the payload is extracted from the request body. - let payload = "test payload"; + let payload = "test-tx"; let request = Request::builder() .method("GET") .uri("/mempool/submit") @@ -54,6 +67,7 @@ mod tests { // Check that the response status is OK. assert_eq!(response.status(), StatusCode::OK); + let _ = try_join_all(vec![mempool_handler]).await; }) } diff --git a/chain/src/bin/validator.rs b/chain/src/bin/validator.rs index d1d730c4..e9566626 100644 --- a/chain/src/bin/validator.rs +++ b/chain/src/bin/validator.rs @@ -8,7 +8,7 @@ use commonware_cryptography::{ bls12381::primitives::{ group::{self, Element}, poly, - }, ed25519::{PrivateKey, PublicKey}, sha256, Ed25519, Scheme + }, ed25519::{PrivateKey, PublicKey}, sha256, Ed25519, Scheme, Sha256 }; use commonware_deployer::ec2::Peers; use commonware_p2p::authenticated; @@ -241,9 +241,9 @@ fn main() { // Create mempool/broadcast/Proof of Availability engine let mempool_namespace = b"mempool"; - let (mempool_application, mempool_app_mailbox) = mempool::actor::Actor::::new(); + let (mempool_application, mempool_app_mailbox) = mempool::actor::Actor::::new(); let broadcast_coordinator = mempool::coordinator::Coordinator::new(identity.clone(), peer_keys.clone(), share); - let (_, collector_mailbox) = mempool::collector::Collector::::new(mempool_namespace, identity_public); + let (_, collector_mailbox) = mempool::collector::Collector::::new(mempool_namespace, identity_public); let (broadcast_engine, broadcast_mailbox) = linked::Engine::new(context.with_label("broadcast_engine"), linked::Config { crypto: signer.clone(), coordinator: broadcast_coordinator.clone(), From 43371e2f7d577687ba5cf4e74747954569c03ad4 Mon Sep 17 00:00:00 2001 From: rikoeldon <106416799+rikoeldon@users.noreply.github.com> Date: Wed, 2 Apr 2025 15:18:09 -0400 Subject: [PATCH 24/38] checkpoint rpc --- Cargo.lock | 3 ++ Cargo.toml | 1 + actions/src/msg.rs | 2 +- actions/src/transfer.rs | 2 +- chain/src/actors/net/actor.rs | 8 +-- chain/src/actors/net/ingress.rs | 78 +-------------------------- client/Cargo.toml | 3 ++ client/src/client.rs | 22 ++++---- client/src/client_types.rs | 95 +++++++++++++++++++++++++++++++++ client/src/lib.rs | 1 + 10 files changed, 120 insertions(+), 95 deletions(-) create mode 100644 client/src/client_types.rs diff --git a/Cargo.lock b/Cargo.lock index d432ed3d..f8433257 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -91,6 +91,9 @@ dependencies = [ "futures", "rand 0.8.5", "reqwest", + "serde", + "serde_bytes", + "serde_yaml", "thiserror 2.0.12", "tokio", "tokio-tungstenite 0.17.2", diff --git a/Cargo.toml b/Cargo.toml index 8d3d4d5a..2221f9dc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,6 +14,7 @@ alto-client = { version = "0.0.6", path = "client" } alto-types = { version = "0.0.6", path = "types" } alto-actions = { version = "0.1.0", path = "actions"} alto-storage = { version = "0.1.0", path = "storage"} +alto-chain = { version = "0.0.6", path = "chain"} alto-vm = { version = "0.1.0", path = "vm"} commonware-broadcast = { version = "0.0.43" } commonware-consensus = { version = "0.0.43" } diff --git a/actions/src/msg.rs b/actions/src/msg.rs index e8e1d093..ac211393 100644 --- a/actions/src/msg.rs +++ b/actions/src/msg.rs @@ -1,4 +1,4 @@ -use alto_types::Address; +use alto_types::address::Address; pub struct SequencerMsg { pub chain_id: Vec, diff --git a/actions/src/transfer.rs b/actions/src/transfer.rs index 84a16129..9d026c33 100644 --- a/actions/src/transfer.rs +++ b/actions/src/transfer.rs @@ -1,4 +1,4 @@ -use alto_types::Address; +use alto_types::address::Address; const MAX_MEMO_SIZE: usize = 256; diff --git a/chain/src/actors/net/actor.rs b/chain/src/actors/net/actor.rs index 2d021155..69c4dd57 100644 --- a/chain/src/actors/net/actor.rs +++ b/chain/src/actors/net/actor.rs @@ -23,8 +23,8 @@ use tracing_subscriber::fmt::format; use crate::actors::mempool::mempool::{self, RawTransaction}; -use super::ingress::{Mailbox, Message, WebsocketClientMessage}; - +use super::ingress::{Mailbox, Message}; +use alto_client::client_types::WebsocketClientMessage; #[derive(Deserialize)] pub struct DummyTransaction { #[serde(with = "serde_bytes")] @@ -245,9 +245,9 @@ impl Actor { let success = mempool.submit_txs(vec![RawTransaction::new(payload)]).await[0]; if success { - format!("submitted") + "submitted".to_string() } else { - format!("failed to submit tx") + "failed to submit tx".to_string() } } diff --git a/chain/src/actors/net/ingress.rs b/chain/src/actors/net/ingress.rs index 50674ba9..b0aab50b 100644 --- a/chain/src/actors/net/ingress.rs +++ b/chain/src/actors/net/ingress.rs @@ -43,83 +43,6 @@ impl Message { } } -/// Messages sent from client -pub enum ClientMessage { - WSMessage(WebsocketClientMessage), - RpcMessage(RpcMessage) -} - -/// Websocket Message sent from client -#[derive(Debug)] -#[repr(u8)] -pub enum WebsocketClientMessage { - RegisterBlock = 0, - RegisterTx, - - SubmitTxs(Vec) -} - -impl WebsocketClientMessage { - // TODO: cache the serialization result - pub fn serialize(&self) -> Vec { - match self { - Self::RegisterBlock => vec![0], - Self::RegisterTx => vec![1], - Self::SubmitTxs(txs) => { - let mut raw = vec![2]; // first byte indicates the message type - raw.put_u64(txs.len() as u64); - for tx in txs.into_iter() { - raw.put_u64(tx.len() as u64); - raw.put(tx.clone()); - } - raw - } - } - } - - pub fn deserialize(raw: &[u8]) -> Result { - use bytes::Buf; - - if raw.is_empty() { - return Err("empty payload provided".into()); - } - - // Create a mutable buffer view over the input slice - let mut buf = raw; - - // Read the message type byte. - let msg_type = buf.get_u8(); - match msg_type { - 0 => Ok(WebsocketClientMessage::RegisterBlock), - 1 => Ok(WebsocketClientMessage::RegisterTx), - 2 => { - // Ensure there are enough bytes for the number of transactions. - if buf.remaining() < 8 { - return Err("payload too short for number of transactions".into()); - } - let num_txs = buf.get_u64(); - - let mut txs = Vec::with_capacity(num_txs as usize); - // Loop over each transaction. - for _ in 0..num_txs { - if buf.remaining() < 8 { - return Err("payload too short for transaction length".into()); - } - let tx_len = buf.get_u64() as usize; - if buf.remaining() < tx_len { - return Err("payload too short for transaction data".into()); - } - // Extract the transaction bytes. - let tx_data = buf.copy_to_bytes(tx_len); - txs.push(tx_data); - } - Ok(WebsocketClientMessage::SubmitTxs(txs)) - } - other => Err(format!("unsupported message type: {}", other)), - } - } -} - #[derive(Debug)] pub enum RpcMessage { // for rpc @@ -136,6 +59,7 @@ pub enum RpcMessage { }, } + #[derive(Clone)] pub struct Mailbox { sender: mpsc::Sender diff --git a/client/Cargo.toml b/client/Cargo.toml index 90aa40ac..038b8209 100644 --- a/client/Cargo.toml +++ b/client/Cargo.toml @@ -21,3 +21,6 @@ futures = { workspace = true } reqwest = "0.12.12" tokio-tungstenite = { version = "0.17", features = ["native-tls"] } tokio = { version = "1.40.0", features = ["full"] } +serde = { version = "1.0.218", features = ["derive"] } +serde_yaml = "0.9.34" +serde_bytes = "0.11.17" \ No newline at end of file diff --git a/client/src/client.rs b/client/src/client.rs index 9a20e995..358d06ca 100644 --- a/client/src/client.rs +++ b/client/src/client.rs @@ -1,5 +1,8 @@ use reqwest::Client; - +use crate::client_types::ClientRpcMessage; +use bytes::Bytes; +use serde::Deserialize; +use alto_types::tx::Tx; pub const WEBSOCKET_PREFIX: &'static str = "/ws"; pub const RPC_PREFIX: &'static str = "/api"; pub const PATH_SUBMIT_TX: &'static str = "/mempool/submit"; @@ -29,22 +32,17 @@ impl JSONRPCClient { } } //todo implement methods needed to communicate with server - pub fn submit_tx(&self, data: Vec) { - todo!() - } - pub fn submit_batch_txs(&self, data: Vec) { - todo!() - } - pub fn get_block(&self, data: Vec) { + pub fn submit_tx(&self, tx: Tx) -> Vec { + //todo safety checks todo!() } - pub fn get_prev_block(&self, data: Vec) { + + pub fn get_block(&self, data: Vec) -> u64{ todo!() } + pub fn get_block_height(&self, data: Vec) { todo!() } - pub fn get_payload(&self, data: Vec) { - todo!() - } + } diff --git a/client/src/client_types.rs b/client/src/client_types.rs new file mode 100644 index 00000000..dc2634a2 --- /dev/null +++ b/client/src/client_types.rs @@ -0,0 +1,95 @@ +use bytes::{BufMut, Bytes}; +use commonware_cryptography::sha256::Digest; +use futures::channel::oneshot; +use alto_types::Block; + +/// Messages sent from client +pub enum ClientMessage { + WSMessage(WebsocketClientMessage), + RpcMessage(ClientRpcMessage) +} + +/// Websocket Message sent from client +#[derive(Debug)] +#[repr(u8)] +pub enum WebsocketClientMessage { + RegisterBlock = 0, + RegisterTx, + + SubmitTxs(Vec) +} + +impl WebsocketClientMessage { + // TODO: cache the serialization result + pub fn serialize(&self) -> Vec { + match self { + Self::RegisterBlock => vec![0], + Self::RegisterTx => vec![1], + Self::SubmitTxs(txs) => { + let mut raw = vec![2]; // first byte indicates the message type + raw.put_u64(txs.len() as u64); + for tx in txs.into_iter() { + raw.put_u64(tx.len() as u64); + raw.put(tx.clone()); + } + raw + } + } + } + + pub fn deserialize(raw: &[u8]) -> Result { + use bytes::Buf; + + if raw.is_empty() { + return Err("empty payload provided".into()); + } + + // Create a mutable buffer view over the input slice + let mut buf = raw; + + // Read the message type byte. + let msg_type = buf.get_u8(); + match msg_type { + 0 => Ok(WebsocketClientMessage::RegisterBlock), + 1 => Ok(WebsocketClientMessage::RegisterTx), + 2 => { + // Ensure there are enough bytes for the number of transactions. + if buf.remaining() < 8 { + return Err("payload too short for number of transactions".into()); + } + let num_txs = buf.get_u64(); + + let mut txs = Vec::with_capacity(num_txs as usize); + // Loop over each transaction. + for _ in 0..num_txs { + if buf.remaining() < 8 { + return Err("payload too short for transaction length".into()); + } + let tx_len = buf.get_u64() as usize; + if buf.remaining() < tx_len { + return Err("payload too short for transaction data".into()); + } + // Extract the transaction bytes. + let tx_data = buf.copy_to_bytes(tx_len); + txs.push(tx_data); + } + Ok(WebsocketClientMessage::SubmitTxs(txs)) + } + other => Err(format!("unsupported message type: {}", other)), + } + } +} + +#[derive(Debug)] +pub enum ClientRpcMessage { + // for rpc + SubmitTx { + payload: Bytes, + }, + GetBlockHeight { + response: oneshot::Sender + }, + GetBlock { + height: u64, + }, +} diff --git a/client/src/lib.rs b/client/src/lib.rs index 9ec7c672..01425f5a 100644 --- a/client/src/lib.rs +++ b/client/src/lib.rs @@ -7,6 +7,7 @@ use thiserror::Error; pub mod consensus; pub mod utils; mod client; +pub mod client_types; const LATEST: &str = "latest"; From 26ebc7f610ed6248933a29392515f05d3e5f1600 Mon Sep 17 00:00:00 2001 From: NateAtNodeKit Date: Wed, 2 Apr 2025 15:42:56 -0400 Subject: [PATCH 25/38] types sync with vm branch --- Cargo.lock | 17 +- Cargo.toml | 3 +- types/Cargo.toml | 6 +- types/src/account.rs | 9 +- types/src/address.rs | 23 ++- types/src/block.rs | 77 ++++++-- types/src/codec.rs | 8 - types/src/consensus.rs | 2 +- types/src/lib.rs | 32 +-- types/src/null_error.rs | 15 ++ types/src/signed_tx.rs | 196 +++++++++++++++++-- types/src/state.rs | 11 -- types/src/state_view.rs | 10 + types/src/tx.rs | 379 ++++++++++++++++++++++++++++++++---- types/src/units/mod.rs | 2 + types/src/units/msg.rs | 106 ++++++++++ types/src/units/transfer.rs | 141 ++++++++++++++ types/src/wallet.rs | 139 ++++++++++--- types/src/wasm.rs | 2 +- 19 files changed, 1044 insertions(+), 134 deletions(-) delete mode 100644 types/src/codec.rs create mode 100644 types/src/null_error.rs delete mode 100644 types/src/state.rs create mode 100644 types/src/state_view.rs create mode 100644 types/src/units/mod.rs create mode 100644 types/src/units/msg.rs create mode 100644 types/src/units/transfer.rs diff --git a/Cargo.lock b/Cargo.lock index f8433257..088bff10 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -123,7 +123,7 @@ version = "0.1.0" dependencies = [ "alto-types", "bytes", - "commonware-codec", + "commonware-codec 0.0.43", "commonware-cryptography", "rand 0.8.5", "rocksdb", @@ -132,10 +132,10 @@ dependencies = [ [[package]] name = "alto-types" -version = "0.0.6" +version = "0.0.4" dependencies = [ "bytes", - "commonware-codec", + "commonware-codec 0.0.40", "commonware-cryptography", "commonware-utils", "getrandom 0.2.15", @@ -931,6 +931,17 @@ dependencies = [ "tracing", ] +[[package]] +name = "commonware-codec" +version = "0.0.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ea2a8daeca3c1346a30a4c92af7791b1c37e669fad9e1907203e2cc6f92eec7" +dependencies = [ + "bytes", + "paste", + "thiserror 2.0.12", +] + [[package]] name = "commonware-codec" version = "0.0.43" diff --git a/Cargo.toml b/Cargo.toml index 2221f9dc..ac14ebfe 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,7 +11,7 @@ resolver = "2" [workspace.dependencies] alto-client = { version = "0.0.6", path = "client" } -alto-types = { version = "0.0.6", path = "types" } +alto-types = { version = "0.0.4", path = "types" } alto-actions = { version = "0.1.0", path = "actions"} alto-storage = { version = "0.1.0", path = "storage"} alto-chain = { version = "0.0.6", path = "chain"} @@ -27,6 +27,7 @@ commonware-runtime = { version = "0.0.43" } commonware-storage = { version = "0.0.43" } commonware-stream = { version = "0.0.43" } commonware-utils = { version = "0.0.43" } +commonware-codec = { version = "0.0.40"} thiserror = "2.0.12" bytes = "1.7.1" rand = "0.8.5" diff --git a/types/Cargo.toml b/types/Cargo.toml index 7d174831..d58be7c7 100644 --- a/types/Cargo.toml +++ b/types/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "alto-types" -version = "0.0.6" +version = "0.0.4" publish = true edition = "2021" license = "MIT OR Apache-2.0" @@ -16,6 +16,7 @@ crate-type = ["rlib", "cdylib"] [dependencies] commonware-cryptography = { workspace = true } commonware-utils = { workspace = true } +commonware-codec = { workspace = true} bytes = { workspace = true } rand = { workspace = true } thiserror = { workspace = true } @@ -23,9 +24,8 @@ wasm-bindgen = "0.2.100" serde = { version = "1.0.219", features = ["derive"] } serde-wasm-bindgen = "0.6.5" more-asserts = "0.3.1" -commonware-codec = { version = "0.0.43" } # Enable "js" feature when WASM is target [target.'cfg(target_arch = "wasm32")'.dependencies.getrandom] version = "0.2.15" -features = ["js"] +features = ["js"] \ No newline at end of file diff --git a/types/src/account.rs b/types/src/account.rs index 061b7ca4..fbe98dab 100644 --- a/types/src/account.rs +++ b/types/src/account.rs @@ -1,9 +1,9 @@ -use commonware_codec::{Codec, Error, Reader, Writer}; use crate::address::Address; +use commonware_codec::{Codec, Error, Reader, Writer}; pub type Balance = u64; -#[derive(Clone, Debug)] +#[derive(Clone, Debug, PartialEq, Eq)] pub struct Account { pub address: Address, pub balance: Balance, @@ -30,8 +30,11 @@ impl Account { } } } + impl Codec for Account { fn write(&self, writer: &mut impl Writer) { + // @rikoeldon I think we don't need to write account address into the state. + // account address is part of the key. writer.write_bytes(self.address.0.as_slice()); self.balance.write(writer); } @@ -40,7 +43,7 @@ impl Codec for Account { let addr_bytes = <[u8; 33]>::read(reader)?; let address = Address::from_bytes(&addr_bytes[1..]).unwrap(); let balance = ::read(reader)?; - Ok(Self{address, balance}) + Ok(Self { address, balance }) } fn len_encoded(&self) -> usize { diff --git a/types/src/address.rs b/types/src/address.rs index 594e4d65..fdd837c3 100644 --- a/types/src/address.rs +++ b/types/src/address.rs @@ -1,8 +1,9 @@ +use crate::{PublicKey, ADDRESSLEN}; use more_asserts::assert_le; -use crate::ADDRESSLEN; +use rand::Rng; #[derive(Hash, Eq, PartialEq, Clone, Debug)] -pub struct Address(pub [u8;ADDRESSLEN]); +pub struct Address(pub [u8; ADDRESSLEN]); impl Address { pub fn new(slice: &[u8]) -> Self { @@ -12,6 +13,20 @@ impl Address { Address(arr) } + pub fn create_random_address() -> Self { + let mut arr = [0u8; ADDRESSLEN]; + rand::thread_rng().fill(&mut arr); + Address(arr) + } + + pub fn from_pub_key(pub_key: &PublicKey) -> Self { + // @todo implement a hasher to derive the address from the public key. + assert_le!(pub_key.len(), ADDRESSLEN, "public key is too large"); + let mut arr = [0u8; ADDRESSLEN]; + arr[..pub_key.len()].copy_from_slice(pub_key.as_ref()); + Address(arr) + } + pub fn from_bytes(bytes: &[u8]) -> Result { if bytes.len() != 32 { return Err("Address must be 32 bytes."); @@ -21,7 +36,7 @@ impl Address { } pub fn empty() -> Self { - Self([0;ADDRESSLEN]) + Self([0; ADDRESSLEN]) } pub fn is_empty(&self) -> bool { @@ -32,7 +47,7 @@ impl Address { &self.0 } - pub fn as_bytes(&self) -> &[u8;ADDRESSLEN] { + pub fn as_bytes(&self) -> &[u8; ADDRESSLEN] { &self.0 } } \ No newline at end of file diff --git a/types/src/block.rs b/types/src/block.rs index c247541f..34697fc6 100644 --- a/types/src/block.rs +++ b/types/src/block.rs @@ -1,9 +1,12 @@ +use crate::signed_tx::{pack_signed_txs, unpack_signed_txs, SignedTx, SignedTxChars}; use crate::{Finalization, Notarization}; use bytes::{Buf, BufMut}; -use commonware_cryptography::{bls12381::PublicKey, sha256::Digest, Hasher, Sha256}; +use commonware_cryptography::{bls12381::PublicKey, sha256, sha256::Digest, Hasher, Sha256}; use commonware_utils::{Array, SizedSerialize}; -#[derive(Clone, Debug, PartialEq, Eq)] +// @todo add state root, fee manager and results to the block struct. +// what method of state root generation should be used? +#[derive(Clone, Debug)] pub struct Block { /// The parent block's digest. pub parent: Digest, @@ -14,25 +17,52 @@ pub struct Block { /// The timestamp of the block (in milliseconds since the Unix epoch). pub timestamp: u64, + /// The raw transactions in the block. + pub raw_txs: Vec, + + /// The state root of the block. + pub state_root: Digest, + + txs: Vec, /// Pre-computed digest of the block. digest: Digest, } impl Block { - fn compute_digest(parent: &Digest, height: u64, timestamp: u64) -> Digest { + fn compute_digest( + parent: &Digest, + height: u64, + timestamp: u64, + raw_txs: Vec, + state_root: &Digest, + ) -> Digest { let mut hasher = Sha256::new(); hasher.update(parent); hasher.update(&height.to_be_bytes()); hasher.update(×tamp.to_be_bytes()); + hasher.update(&raw_txs); + hasher.update(state_root); hasher.finalize() } - pub fn new(parent: Digest, height: u64, timestamp: u64) -> Self { - let digest = Self::compute_digest(&parent, height, timestamp); + pub fn new( + parent: Digest, + height: u64, + timestamp: u64, + txs: Vec, + state_root: Digest, + ) -> Self { + // let mut txs = txs; + // @todo this is packing txs in a block. + let raw_txs = pack_signed_txs(txs.clone()); + let digest = Self::compute_digest(&parent, height, timestamp, raw_txs.clone(), &state_root); Self { parent, height, timestamp, + raw_txs, + state_root, + txs, digest, } } @@ -42,30 +72,55 @@ impl Block { bytes.extend_from_slice(&self.parent); bytes.put_u64(self.height); bytes.put_u64(self.timestamp); + bytes.extend_from_slice(&self.state_root); + bytes.extend_from_slice(self.raw_txs.as_slice()); bytes } pub fn deserialize(mut bytes: &[u8]) -> Option { // Parse the block - if bytes.len() != Self::SERIALIZED_LEN { - return None; - } + // if bytes.len() != Self::SERIALIZED_LEN { + // return None; + // } let parent = Digest::read_from(&mut bytes).ok()?; let height = bytes.get_u64(); let timestamp = bytes.get_u64(); + let state_root = Digest::read_from(&mut bytes).ok()?; + let raw_txs = bytes.to_vec(); + let digest = Self::compute_digest(&parent, height, timestamp, raw_txs.clone(), &state_root); + let txs = unpack_signed_txs(raw_txs.clone()); // Return block - let digest = Self::compute_digest(&parent, height, timestamp); Some(Self { parent, height, timestamp, + raw_txs, + state_root, + txs, digest, }) } pub fn digest(&self) -> Digest { - self.digest + self.digest.clone() + } + //todo check logic below + pub fn encode(&mut self) -> Vec { + let mut bytes: Vec = Vec::new(); + bytes.extend_from_slice(&self.parent); + bytes.extend_from_slice(&(self.height.to_be_bytes())); + bytes.extend_from_slice(&(self.timestamp.to_be_bytes())); + bytes.extend_from_slice(self.raw_txs.as_slice()); + bytes.extend_from_slice(&self.state_root); + // encoding signed txs + for tx in self.txs.iter_mut() { + bytes.extend_from_slice(&tx.encode()); + } + + // return encoded digest. + self.digest = sha256::hash(&bytes); + bytes } } @@ -136,4 +191,4 @@ impl Finalized { } Some(Self { proof, block }) } -} +} \ No newline at end of file diff --git a/types/src/codec.rs b/types/src/codec.rs deleted file mode 100644 index d8688a2a..00000000 --- a/types/src/codec.rs +++ /dev/null @@ -1,8 +0,0 @@ -use commonware_codec::Writer; -use commonware_cryptography::ed25519::PublicKey; - -pub fn serialize_pk(pk: &PublicKey, writer: &mut impl Writer) { - // let slice: &[u8] = pk.as_ref(); - // let length = slice.len(); - // length.serialize(&mut *writer).unwrap(); -} \ No newline at end of file diff --git a/types/src/consensus.rs b/types/src/consensus.rs index 5fca5214..6ae6a5ed 100644 --- a/types/src/consensus.rs +++ b/types/src/consensus.rs @@ -292,4 +292,4 @@ impl SizedSerialize for Finalization { pub fn leader_index(seed: &[u8], participants: usize) -> usize { modulo(seed, participants as u64) as usize -} +} \ No newline at end of file diff --git a/types/src/lib.rs b/types/src/lib.rs index b9e06500..8d6ed98e 100644 --- a/types/src/lib.rs +++ b/types/src/lib.rs @@ -2,23 +2,27 @@ mod block; -use commonware_cryptography::{Ed25519, Scheme}; pub use block::{Block, Finalized, Notarized}; +use commonware_cryptography::{Ed25519, Scheme}; +use commonware_utils::SystemTimeExt; +use std::time::SystemTime; mod consensus; pub use consensus::{leader_index, Finalization, Kind, Notarization, Nullification, Seed}; -pub mod wasm; -pub mod codec; -pub mod wallet; -pub mod tx; -pub mod signed_tx; -pub mod state; -pub mod address; pub mod account; +pub mod address; +pub mod null_error; +pub mod signed_tx; +pub mod state_view; +pub mod tx; +pub mod units; +pub mod wallet; +pub mod wasm; use rand::rngs::OsRng; // We don't use functions here to guard against silent changes. pub const NAMESPACE: &[u8] = b"_ALTO"; +pub const TX_NAMESPACE: &[u8] = b"_tx_namespace_"; pub const P2P_NAMESPACE: &[u8] = b"_ALTO_P2P"; pub const SEED_NAMESPACE: &[u8] = b"_ALTO_SEED"; pub const NOTARIZE_NAMESPACE: &[u8] = b"_ALTO_NOTARIZE"; @@ -29,6 +33,7 @@ const ADDRESSLEN: usize = 32; type PublicKey = commonware_cryptography::ed25519::PublicKey; type PrivateKey = commonware_cryptography::ed25519::PrivateKey; +type Signature = commonware_cryptography::ed25519::Signature; pub fn create_test_keypair() -> (PublicKey, PrivateKey) { let mut rng = OsRng; @@ -41,6 +46,10 @@ pub fn create_test_keypair() -> (PublicKey, PrivateKey) { (public_key, private_key) } +pub fn curr_timestamp() -> u64 { + SystemTime::now().epoch_millis() +} + #[cfg(test)] mod tests { use super::*; @@ -144,7 +153,7 @@ mod tests { let parent_digest = hash(&[0; 32]); let height = 0; let timestamp = 1; - let block = Block::new(parent_digest, height, timestamp); + let block = Block::new(parent_digest, height, timestamp, Vec::new(), [0; 32].into()); let block_digest = block.digest(); // Check block serialization @@ -154,6 +163,7 @@ mod tests { assert_eq!(block.parent, deserialized.parent); assert_eq!(block.height, deserialized.height); assert_eq!(block.timestamp, deserialized.timestamp); + // @todo add deserialization checks for signed transactions. // Create notarization let view = 0; @@ -195,7 +205,7 @@ mod tests { let parent_digest = hash(&[0; 32]); let height = 0; let timestamp = 1; - let block = Block::new(parent_digest, height, timestamp); + let block = Block::new(parent_digest, height, timestamp, Vec::new(), [0; 32].into()); // Create notarization let view = 0; @@ -233,4 +243,4 @@ mod tests { let result = Finalization::deserialize(None, &serialized); assert!(result.is_some()); } -} +} \ No newline at end of file diff --git a/types/src/null_error.rs b/types/src/null_error.rs new file mode 100644 index 00000000..5e3050bc --- /dev/null +++ b/types/src/null_error.rs @@ -0,0 +1,15 @@ +use std::{ + error::Error, + fmt::{Display, Formatter, Result}, +}; + +#[derive(Debug)] +pub struct NullError; + +impl Display for NullError { + fn fmt(&self, f: &mut Formatter<'_>) -> Result { + write!(f, "NoError") + } +} + +impl Error for NullError {} \ No newline at end of file diff --git a/types/src/signed_tx.rs b/types/src/signed_tx.rs index 1aec2917..3db2551b 100644 --- a/types/src/signed_tx.rs +++ b/types/src/signed_tx.rs @@ -1,26 +1,194 @@ use crate::address::Address; -use crate::PublicKey; -use crate::wallet::{AuthTypes, Wallet}; -use crate::tx::Tx; - +use crate::tx::{Tx, TxMethods}; +use crate::wallet::{Wallet, WalletMethods}; +use crate::{PublicKey, Signature, TX_NAMESPACE}; +use commonware_cryptography::{Ed25519, Scheme}; // this is sent by the user to the validators. -pub struct SignedTx<'a> { - pub tx: Tx<'a>, - pub auth_type: AuthTypes, +#[derive(Clone, Debug)] +pub struct SignedTx { + pub tx: Tx, pub_key: PublicKey, address: Address, signature: Vec, } - -pub trait SignedTxChars { - fn new(tx: Tx, auth_type: AuthTypes) -> Self; - fn sign(&self, wallet: Wallet) -> SignedTx; - fn verify(&self) -> bool; +// function names are self explanatory. +pub trait SignedTxChars: Sized { + fn new(tx: Tx, pub_key: PublicKey, signature: Vec) -> Self; + // fn sign(&mut self, wallet: Wallet) -> SignedTx; + fn verify(&mut self) -> bool; fn signature(&self) -> Vec; fn public_key(&self) -> Vec; fn address(&self) -> Address; - fn encode(&self) -> Vec; - fn decode(&self, bytes: &[u8]) -> Self; + fn encode(&mut self) -> Vec; + fn decode(bytes: &[u8]) -> Result; +} + +impl SignedTxChars for SignedTx { + // @todo either have all fields initialized or none. + fn new(tx: Tx, pub_key: PublicKey, signature: Vec) -> Self { + Self { + tx, + pub_key: pub_key.clone(), + address: Address::from_pub_key(&pub_key), + signature: signature.clone(), + } + } + + fn verify(&mut self) -> bool { + let tx_data = self.tx.encode(); + let signature = Signature::try_from(self.signature.as_slice()); + if signature.is_err() { + return false; + } + let signature = signature.unwrap(); + Ed25519::verify(Some(TX_NAMESPACE), &tx_data, &self.pub_key, &signature) + } + + fn signature(&self) -> Vec { + self.signature.clone() + } + + fn public_key(&self) -> Vec { + self.pub_key.to_vec() + } + + fn address(&self) -> Address { + self.address.clone() + } + + // @todo add syntactic checks. + fn encode(&mut self) -> Vec { + let mut bytes = Vec::new(); + + let raw_tx = self.tx.encode(); + let raw_tx_len = raw_tx.len() as u64; + bytes.extend(raw_tx_len.to_be_bytes()); + bytes.extend_from_slice(&raw_tx); + bytes.extend_from_slice(&self.pub_key); + bytes.extend_from_slice(&self.signature); + bytes + } + + // @todo add syntactic checks and use methods consume. + fn decode(bytes: &[u8]) -> Result { + // @todo this method seems untidy. + + let raw_tx_len = u64::from_be_bytes(bytes[0..8].try_into().unwrap()); + let raw_tx = &bytes[8..8 + raw_tx_len as usize]; + let pub_key = &bytes[8 + raw_tx_len as usize..8 + raw_tx_len as usize + 32]; + let signature = &bytes[8 + raw_tx_len as usize + 32..]; + let public_key = PublicKey::try_from(pub_key); + if public_key.is_err() { + return Err(public_key.unwrap_err().to_string()); + } + let public_key = public_key.unwrap(); + let tx = Tx::decode(raw_tx); + if tx.is_err() { + return Err(tx.unwrap_err()); + } + + Ok(SignedTx { + tx: tx.unwrap(), + pub_key: public_key.clone(), + address: Address::from_pub_key(&public_key), + signature: signature.to_vec(), + }) + } +} + +impl SignedTx { + pub fn sign(mut tx: Tx, mut wallet: Wallet) -> SignedTx { + let tx_data = tx.encode(); + SignedTx { + tx: tx.clone(), + signature: wallet.sign(&tx_data), + address: wallet.address(), + pub_key: wallet.public_key(), + } + } +} + +pub fn pack_signed_txs(signed_txs: Vec) -> Vec { + let mut bytes = Vec::new(); + bytes.extend((signed_txs.len() as u64).to_be_bytes()); + for signed_tx in signed_txs { + // @todo improvise + let mut signed_tx = signed_tx; + let signed_tx_bytes = signed_tx.encode(); + bytes.extend((signed_tx_bytes.len() as u64).to_be_bytes()); + bytes.extend_from_slice(&signed_tx_bytes); + } + bytes } + +pub fn unpack_signed_txs(bytes: Vec) -> Vec { + let signed_txs_len = u64::from_be_bytes(bytes[0..8].try_into().unwrap()); + let mut signed_txs = Vec::with_capacity(signed_txs_len as usize); + let mut offset = 8; + for _ in 0..signed_txs_len { + let signed_tx_len = u64::from_be_bytes(bytes[offset..offset + 8].try_into().unwrap()); + offset += 8; + let signed_tx_bytes = &bytes[offset..offset + signed_tx_len as usize]; + offset += signed_tx_len as usize; + let signed_tx = SignedTx::decode(signed_tx_bytes); + if signed_tx.is_err() { + panic!("Failed to unpack signed tx: {}", signed_tx.unwrap_err()); + } + signed_txs.push(signed_tx.unwrap()); + } + signed_txs +} + +#[cfg(test)] +mod tests { + use std::default; + use std::error::Error; + + use super::*; + use crate::tx::Unit; + use crate::units::transfer::Transfer; + use crate::{create_test_keypair, curr_timestamp}; + use commonware_cryptography::sha256::Digest; + use more_asserts::assert_gt; + + #[test] + fn test_encode_decode() -> Result<(), Box> { + let timestamp = curr_timestamp(); + let max_fee = 100; + let priority_fee = 75; + let chain_id = 45205; + let transfer = Transfer::new(); + let units: Vec> = vec![Box::new(transfer)]; + let digest: [u8; 32] = [0; 32]; + let id = Digest::from(digest.clone()); + let (pk, sk) = create_test_keypair(); + // TODO: the .encode call on next line gave error and said origin_msg needed to be mut? but why? + // shouldn't encode be able to encode without changing the msg? + let tx = Tx { + timestamp, + max_fee, + priority_fee, + chain_id, + units: units.clone(), + id, + digest: digest.to_vec(), + actor: Address::empty(), + }; + let mut origin_msg = SignedTx { + tx, + pub_key: pk, + address: Address::create_random_address(), + signature: vec![], + }; + let encoded_bytes = origin_msg.encode(); + assert_gt!(encoded_bytes.len(), 0); + let decoded_msg = SignedTx::decode(&encoded_bytes)?; + assert_eq!(origin_msg.pub_key, decoded_msg.pub_key); + assert_eq!(origin_msg.address, decoded_msg.address); + assert_eq!(origin_msg.signature, decoded_msg.signature); + // @todo make helper to compare fields in tx and units. same issue when testing in tx.rs file. + Ok(()) + } +} \ No newline at end of file diff --git a/types/src/state.rs b/types/src/state.rs deleted file mode 100644 index cdd3e5f9..00000000 --- a/types/src/state.rs +++ /dev/null @@ -1,11 +0,0 @@ -use std::error::Error; - -pub type UnitKey<'a> = &'a [u8;33]; // 1st byte denotes the type of key. 0b for account key, 1b for others. -pub trait State { - fn get(&self, key: UnitKey) -> Result>, Box>; - fn get_multi_key(&self, key: Vec) -> Result>, Box>; - fn update(&mut self, key: UnitKey, value: Vec) -> Result<(), Box>; - fn delete(&mut self, key: UnitKey) -> Result<(), Box>; - fn commit(&mut self) -> Result<(), Box>; - fn rollback(&mut self) -> Result<(), Box>; -} \ No newline at end of file diff --git a/types/src/state_view.rs b/types/src/state_view.rs new file mode 100644 index 00000000..2574b168 --- /dev/null +++ b/types/src/state_view.rs @@ -0,0 +1,10 @@ +use crate::account::{Account, Balance}; +use crate::address::Address; +use std::error::Error; + +pub trait StateView { + fn get_account(&mut self, address: &Address) -> Result, Box>; + fn set_account(&mut self, acc: &Account) -> Result<(), Box>; + fn get_balance(&mut self, address: &Address) -> Option; + fn set_balance(&mut self, address: &Address, amt: Balance) -> bool; +} \ No newline at end of file diff --git a/types/src/tx.rs b/types/src/tx.rs index fb5400a9..ba727a3b 100644 --- a/types/src/tx.rs +++ b/types/src/tx.rs @@ -1,23 +1,57 @@ +use commonware_cryptography::sha256; +use commonware_cryptography::sha256::Digest; +use std::any::Any; + use crate::address::Address; -use crate::wallet::Wallet; use crate::signed_tx::SignedTx; -use crate::state::State; +use crate::state_view::StateView; +use crate::units; +use crate::wallet::Wallet; +use commonware_utils::SystemTimeExt; +use std::time::SystemTime; +#[derive(Debug)] pub enum UnitType { Transfer, SequencerMsg, } +impl TryFrom for UnitType { + type Error = String; + + fn try_from(value: u8) -> Result { + match value { + 0 => Ok(UnitType::Transfer), + 1 => Ok(UnitType::SequencerMsg), + _ => Err(format!("unknown unit type: {}", value)), + } + } +} pub struct UnitContext { - pub timestamp: u64, // timestamp of the tx. - pub chain_id: u64, // chain id of the tx. - pub sender: Address, // sender of the tx. + // timestamp of the tx. + pub timestamp: u64, + // chain id of the tx. + pub chain_id: u64, + // sender of the tx. + pub sender: Address, +} + +pub trait UnitClone { + fn clone_box(&self) -> Box; } +impl UnitClone for T +where + T: 'static + Unit + Clone, +{ + fn clone_box(&self) -> Box { + Box::new(self.clone()) + } +} // unit need to be simple and easy to be packed in the tx and executed by the vm. -pub trait Unit : Send + Sync { +pub trait Unit: UnitClone + Send + Sync + std::fmt::Debug { fn unit_type(&self) -> UnitType; fn encode(&self) -> Vec; fn decode(&mut self, bytes: &[u8]); @@ -25,54 +59,323 @@ pub trait Unit : Send + Sync { fn apply( &self, context: &UnitContext, - state: &mut Box, + state: &mut Box<&mut dyn StateView>, ) -> Result>, Box>; + + fn as_any(&self) -> &dyn Any; } +impl Clone for Box { + fn clone(&self) -> Box { + self.clone_box() + } +} -// pub struct Tx<'a, U: Unit> { -pub struct Tx<'a> { - // timestamp of the tx creation. set by the user. - // will be verified if the tx is in the valid window once received by validators. - // if the timestamp is not in the valid window, the tx will be rejected. - // if tx is in a valid window it is added to mempool. - // timestamp is used to prevent replay attacks. and counter infinite spam attacks as Tx does not have nonce. +#[derive(Clone, Debug)] +pub struct Tx { + /// timestamp of the tx creation. set by the user. + /// will be verified if the tx is in the valid window once received by validators. + /// if the timestamp is not in the valid window, the tx will be rejected. + /// if tx is in a valid window it is added to mempool. + /// timestamp is used to prevent replay attacks. and counter infinite spam attacks as Tx does not have nonce. pub timestamp: u64, - // units are fundamental unit of a tx. similar to actions. - pub units: Vec>, - // max fee is the maximum fee the user is willing to pay for the tx. + /// max fee is the maximum fee the user is willing to pay for the tx. pub max_fee: u64, - // priority fee is the fee the user is willing to pay for the tx to be included in the next block. + /// priority fee is the fee the user is willing to pay for the tx to be included in the next block. pub priority_fee: u64, - // chain id is the id of the chain the tx is intended for. + /// chain id is the id of the chain the tx is intended for. pub chain_id: u64, + /// units are fundamental unit of a tx. similar to actions. + pub units: Vec>, - - // id is the transaction id. It is the hash of digest. - id: &'a [u8;32], - // digest is encoded tx. - digest: Vec, + /// id is the transaction id. It is the hash of digest. + pub id: Digest, + /// digest is encoded tx. + pub digest: Vec, + /// address of the tx sender. wrap this in a better way. + pub actor: Address, } - -pub trait TxChars { - // init is used to create a new instance of Tx. - fn init() -> Self; - // new is used to create a new instance of Tx with given units and chain id. +pub trait TxMethods: Sized { + /// new is used to create a new instance of Tx with given units and chain id. fn new(units: Vec>, chain_id: u64) -> Self; - // set_fee is used to set the max fee and priority fee of the tx. + /// set_fee is used to set the max fee and priority fee of the tx. fn set_fee(&mut self, max_fee: u64, priority_fee: u64); - // sign is used to sign the tx with the given wallet. - fn sign(&self, wallet: Wallet) -> SignedTx; - + /// sign is used to sign the tx with the given wallet. + fn sign(&mut self, wallet: Wallet) -> SignedTx; + fn from( + timestamp: u64, + units: Vec>, + priority_fee: u64, + max_fee: u64, + chain_id: u64, + actor: Address, + ) -> Self; - // returns tx id. - fn id(&self) -> &[u8;32]; - // returns digest of the tx. + /// returns tx id. + fn id(&mut self) -> Digest; + /// returns digest of the tx. fn digest(&self) -> Vec; - // encodes the tx, writes to digest and returns the digest. + /// encodes the tx, writes to digest and returns the digest. + /// ensure all fields are properly set before calling this function. fn encode(&mut self) -> Vec; + fn decode(bytes: &[u8]) -> Result; + + fn set_actor(&mut self, actor: Address); + + fn actor(&self) -> Address; +} + +impl Default for Tx { + fn default() -> Self { + Self { + timestamp: 0, + units: vec![], + max_fee: 0, + priority_fee: 0, + chain_id: 19517, + id: [0; 32].into(), + digest: vec![], + actor: Address::empty(), + } + } +} + +impl TxMethods for Tx { + fn new(units: Vec>, chain_id: u64) -> Self { + let mut tx = Self::default(); + tx.timestamp = SystemTime::now().epoch_millis(); + tx.units = units; + tx.chain_id = chain_id; + + // do not encode and generate tx_id as Tx::new doesnot yet have priority fee and max fee. + tx + } - fn decode(bytes: &[u8]) -> Self; + fn set_fee(&mut self, max_fee: u64, priority_fee: u64) { + self.max_fee = max_fee; + self.priority_fee = priority_fee; + } + + fn sign(&mut self, wallet: Wallet) -> SignedTx { + SignedTx::sign(self.clone(), wallet) + } + + fn from( + timestamp: u64, + units: Vec>, + priority_fee: u64, + max_fee: u64, + chain_id: u64, + actor: Address, + ) -> Self { + let mut tx = Self::default(); + tx.timestamp = timestamp; + tx.units = units; + tx.max_fee = max_fee; + tx.priority_fee = priority_fee; + tx.chain_id = chain_id; + tx.actor = actor; + tx.encode(); + tx + } + + fn id(&mut self) -> Digest { + if self.digest.len() == 0 { + self.encode(); + } + self.id.clone() + } + + fn digest(&self) -> Vec { + self.digest.clone() + } + + fn encode(&mut self) -> Vec { + if self.digest.is_empty() { + return self.digest.clone(); + } + // pack tx timestamp. + self.digest.extend(self.timestamp.to_be_bytes()); + // pack max fee + self.digest.extend(self.max_fee.to_be_bytes()); + // pack priority fee + self.digest.extend(self.priority_fee.to_be_bytes()); + // pack chain id + self.digest.extend(self.chain_id.to_be_bytes()); + // pack # of units. + self.digest.extend((self.units.len() as u64).to_be_bytes()); + // pack individual units + self.units.iter().for_each(|unit| { + let unit_bytes = unit.encode(); + // pack the unit type info. + self.digest.extend((unit.unit_type() as u8).to_be_bytes()); + // pack len of inidividual unit. + self.digest.extend((unit_bytes.len() as u64).to_be_bytes()); + // pack individual unit. + self.digest.extend_from_slice(&unit_bytes); + }); + + // generate tx id. + self.id = sha256::hash(&self.digest); + + // return encoded digest. + self.digest.clone() + } + + fn decode(bytes: &[u8]) -> Result { + if bytes.is_empty() { + return Err("Empty bytes".to_string()); + } + let mut tx = Self::default(); + tx.digest = bytes.to_vec(); // @todo ?? + tx.timestamp = u64::from_be_bytes(bytes[0..8].try_into().unwrap()); + tx.max_fee = u64::from_be_bytes(bytes[8..16].try_into().unwrap()); + tx.priority_fee = u64::from_be_bytes(bytes[16..24].try_into().unwrap()); + tx.chain_id = u64::from_be_bytes(bytes[24..32].try_into().unwrap()); + let units = unpack_units(&bytes[32..]); + if units.is_err() { + return Err(format!("Failed to unpack units: {}", units.unwrap_err())); + } + tx.units = units?; + // generate tx id. + tx.id = sha256::hash(&tx.digest); + // return transaction. + Ok(tx) + } + + fn set_actor(&mut self, actor: Address) { + self.actor = actor; + } + + fn actor(&self) -> Address { + self.actor.clone() + } +} + +fn unpack_units(digest: &[u8]) -> Result>, String> { + let mut offset = 0; + + fn read_u8(input: &[u8], offset: &mut usize) -> Result { + if input.len() < *offset + 1 { + return Err("Unexpected end of input when reading u8".into()); + } + let val = input[*offset]; + *offset += 1; + Ok(val) + } + + fn read_u64(input: &[u8], offset: &mut usize) -> Result { + if input.len() < *offset + 8 { + return Err("Unexpected end of input when reading u64".into()); + } + let val = u64::from_be_bytes(input[*offset..*offset + 8].try_into().unwrap()); + *offset += 8; + Ok(val) + } + + fn read_bytes<'a>( + input: &'a [u8], + offset: &'a mut usize, + len: usize, + ) -> Result<&'a [u8], String> { + if input.len() < *offset + len { + return Err("Unexpected end of input when reading bytes".into()); + } + let bytes = &input[*offset..*offset + len]; + *offset += len; + Ok(bytes) + } + + let unit_count = read_u64(digest, &mut offset)?; + + let mut units: Vec> = Vec::with_capacity(unit_count as usize); + + for _ in 0..unit_count { + let unit_type = read_u8(digest, &mut offset)?; + let unit_len = read_u64(digest, &mut offset)?; + let unit_bytes = read_bytes(digest, &mut offset, unit_len as usize)?.to_vec(); + let unit_type = UnitType::try_from(unit_type); + if unit_type.is_err() { + return Err(format!("Invalid unit type: {}", unit_type.unwrap_err())); + } + let unit_type = unit_type?; + let unit: Box = match unit_type { + UnitType::Transfer => { + let mut transfer = units::transfer::Transfer::default(); + transfer.decode(&unit_bytes); + Box::new(transfer) + } + UnitType::SequencerMsg => { + let mut msg = units::msg::SequencerMsg::default(); + msg.decode(&unit_bytes); + Box::new(msg) + } + }; + units.push(unit); + } + + Ok(units) } + +// @todo implement tests for encoding and decoding of tx. +#[cfg(test)] +mod tests { + use super::*; + use crate::curr_timestamp; + use crate::units::transfer::Transfer; + use more_asserts::assert_gt; + use std::error::Error; + + #[test] + fn test_encode_decode() -> Result<(), Box> { + let timestamp = curr_timestamp(); + let max_fee = 100; + let priority_fee = 75; + let chain_id = 45205; + let transfer = Transfer::new(); + let units: Vec> = vec![Box::new(transfer)]; + let digest: [u8; 32] = [0; 32]; + let id = Digest::from(digest.clone()); + // TODO: the .encode call on next line gave error and said origin_msg needed to be mut? but why? + // shouldn't encode be able to encode without changing the msg? + let mut origin_msg = Tx { + timestamp, + max_fee, + priority_fee, + chain_id, + units: units.clone(), + id, + actor: Address::empty(), + digest: digest.to_vec(), + }; + let encoded_bytes = origin_msg.encode(); + assert_gt!(encoded_bytes.len(), 0); + let decoded_msg = Tx::decode(&encoded_bytes)?; + let origin_transfer = origin_msg.units[0] + .as_ref() + .as_any() + .downcast_ref::() + .expect("Failed to downcast to Transfer"); + + let decode_transfer = decoded_msg.units[0] + .as_ref() + .as_any() + .downcast_ref::() + .expect("Failed to downcast to Transfer"); + + assert_eq!(origin_msg.timestamp, decoded_msg.timestamp); + assert_eq!(origin_msg.max_fee, decoded_msg.max_fee); + assert_eq!(origin_msg.priority_fee, decoded_msg.priority_fee); + assert_eq!(origin_msg.chain_id, decoded_msg.chain_id); + assert_eq!(origin_msg.id, decoded_msg.id); + assert_eq!(origin_msg.digest, decoded_msg.digest); + + // units + assert_eq!(origin_transfer.to_address, decode_transfer.to_address); + assert_eq!(origin_transfer.value, decode_transfer.value); + assert_eq!(origin_transfer.memo, decode_transfer.memo); + Ok(()) + } +} \ No newline at end of file diff --git a/types/src/units/mod.rs b/types/src/units/mod.rs new file mode 100644 index 00000000..ab28bbd8 --- /dev/null +++ b/types/src/units/mod.rs @@ -0,0 +1,2 @@ +pub(crate) mod msg; +pub(crate) mod transfer; \ No newline at end of file diff --git a/types/src/units/msg.rs b/types/src/units/msg.rs new file mode 100644 index 00000000..39a54587 --- /dev/null +++ b/types/src/units/msg.rs @@ -0,0 +1,106 @@ +use crate::{ + address::Address, + state_view::StateView, + tx::{Unit, UnitContext, UnitType}, +}; +use std::any::Any; + +// @todo couple SequencerMsg with DA. +// and skip execution no-op. +#[derive(Clone, Debug)] +pub struct SequencerMsg { + pub chain_id: u64, + pub data: Vec, + pub from_address: Address, +} + +impl SequencerMsg { + pub fn new() -> SequencerMsg { + Self { + chain_id: 0, + data: Vec::new(), + from_address: Address::empty(), + } + } +} + +impl Unit for SequencerMsg { + fn unit_type(&self) -> UnitType { + UnitType::SequencerMsg + } + + fn encode(&self) -> Vec { + let mut bytes: Vec = Vec::new(); + // data length is 8 bytes. + let data_len = self.data.len() as u64; + // chain id length is 8 bytes.n store chain id. + bytes.extend(&self.chain_id.to_be_bytes()); + // address length is 32. store address. + bytes.extend_from_slice(self.from_address.as_slice()); + // store data length. + bytes.extend(data_len.to_be_bytes()); + // store data. + bytes.extend_from_slice(&self.data); + + bytes + } + + // @todo introduce syntactic checks. + fn decode(&mut self, bytes: &[u8]) { + self.chain_id = u64::from_be_bytes(bytes[0..8].try_into().unwrap()); + self.from_address = Address::from_bytes(&bytes[8..40]).unwrap(); + let data_len = u64::from_be_bytes(bytes[40..48].try_into().unwrap()); + self.data = bytes[48..(48 + data_len as usize)].to_vec(); + } + + fn apply( + &self, + _: &UnitContext, + _: &mut Box<&mut dyn StateView>, + ) -> Result>, Box> { + Ok(None) + } + + fn as_any(&self) -> &dyn Any { + self + } +} + +impl Default for SequencerMsg { + fn default() -> Self { + Self { + chain_id: 0, + data: vec![], + from_address: Address::empty(), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use more_asserts::assert_gt; + use std::error::Error; + + #[test] + fn test_encode_decode() -> Result<(), Box> { + let chain_id = 4502; + let data = vec![0xDE, 0xAD, 0xBE, 0xEF]; + let from_address = Address::create_random_address(); + let relayer_id = 1; + let origin_msg = SequencerMsg { + chain_id, + data, + from_address, + }; + let encoded_bytes = origin_msg.encode(); + assert_gt!(encoded_bytes.len(), 0); + let mut decoded_msg = SequencerMsg::new(); + decoded_msg.decode(&encoded_bytes); + assert_eq!(origin_msg.chain_id, decoded_msg.chain_id); + assert_eq!(origin_msg.data.len(), decoded_msg.data.len()); + assert_eq!(origin_msg.data, decoded_msg.data); + assert_eq!(origin_msg.from_address, decoded_msg.from_address); + Ok(()) + } +} \ No newline at end of file diff --git a/types/src/units/transfer.rs b/types/src/units/transfer.rs new file mode 100644 index 00000000..22d949b7 --- /dev/null +++ b/types/src/units/transfer.rs @@ -0,0 +1,141 @@ +use crate::address::Address; +use crate::state_view::StateView; +use crate::tx::{Unit, UnitContext, UnitType}; +use std::{any::Any, error::Error, fmt::Display}; + +const MAX_MEMO_SIZE: usize = 256; + +#[derive(Debug, Clone)] +pub struct Transfer { + pub to_address: Address, + pub value: u64, + pub memo: Vec, +} + +impl Transfer { + pub fn new() -> Transfer { + Self { + to_address: Address::empty(), + value: 0, + memo: Vec::new(), + } + } +} + +#[derive(Debug)] +pub enum TransferError { + SenderAccountNotFound, + InsufficientFunds, + InvalidMemoSize, + StorageError, +} + +impl Display for TransferError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + TransferError::SenderAccountNotFound => write!(f, "Sender account not found"), + TransferError::InsufficientFunds => write!(f, "Insufficient funds"), + TransferError::InvalidMemoSize => write!(f, "Invalid memo size"), + TransferError::StorageError => write!(f, "Storage error"), + } + } +} + +impl Error for TransferError {} + +impl Unit for Transfer { + fn unit_type(&self) -> UnitType { + UnitType::Transfer + } + + fn encode(&self) -> Vec { + let mut bytes = Vec::new(); + let memo_len = self.memo.len() as u64; + bytes.extend_from_slice(self.to_address.as_slice()); + bytes.extend(self.value.to_be_bytes()); + bytes.extend(memo_len.to_be_bytes()); + if memo_len > 0 { + bytes.extend_from_slice(&self.memo); + } + + bytes + } + + // @todo introduce syntactic checks. + fn decode(&mut self, bytes: &[u8]) { + self.to_address = Address::from_bytes(&bytes[0..32]).unwrap(); + self.value = u64::from_be_bytes(bytes[32..40].try_into().unwrap()); + let memo_len = u64::from_be_bytes(bytes[40..48].try_into().unwrap()); + if memo_len > 0 { + self.memo = bytes[48..(48 + memo_len as usize)].to_vec(); + } + } + + fn apply( + &self, + context: &UnitContext, + state: &mut Box<&mut dyn StateView>, + ) -> Result>, Box> { + if self.memo.len() > MAX_MEMO_SIZE { + return Err(TransferError::InvalidMemoSize.into()); + } + + if let Some(bal) = state.get_balance(&context.sender) { + if bal < self.value { + return Err(TransferError::InsufficientFunds.into()); + } + let receiver_bal = state.get_balance(&self.to_address).unwrap_or(0); + + if !state.set_balance(&context.sender, bal - self.value) + || !state.set_balance(&self.to_address, receiver_bal + self.value) + { + return Err(TransferError::StorageError.into()); + } + } else { + return Err(TransferError::SenderAccountNotFound.into()); + } + + Ok(None) + } + + fn as_any(&self) -> &dyn Any { + self + } +} + +impl Default for Transfer { + fn default() -> Self { + Self { + to_address: Address::empty(), + value: 0, + memo: vec![], + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use more_asserts::assert_gt; + use std::error::Error; + + #[test] + fn test_encode_decode() -> Result<(), Box> { + let to_address = Address::create_random_address(); + let value = 5; + let memo = vec![0xDE, 0xAD, 0xBE, 0xEF]; + let origin_msg = Transfer { + to_address, + value, + memo, + }; + let encoded_bytes = origin_msg.encode(); + assert_gt!(encoded_bytes.len(), 0); + let mut decoded_msg = Transfer::new(); + decoded_msg.decode(&encoded_bytes); + assert_eq!(origin_msg.to_address, decoded_msg.to_address); + assert_eq!(origin_msg.value, decoded_msg.value); + assert_eq!(origin_msg.memo, decoded_msg.memo); + Ok(()) + } +} \ No newline at end of file diff --git a/types/src/wallet.rs b/types/src/wallet.rs index 106686da..beace065 100644 --- a/types/src/wallet.rs +++ b/types/src/wallet.rs @@ -1,39 +1,128 @@ use crate::address::Address; -use crate::{PrivateKey, PublicKey}; +use crate::{PrivateKey, PublicKey, Signature, TX_NAMESPACE}; +use commonware_cryptography::ed25519::Ed25519; +use commonware_cryptography::Scheme; +use rand::{CryptoRng, Rng}; +use std::fmt::Error; +#[derive(Clone, Debug)] pub enum AuthTypes { ED25519, } - -// auth should have a method to verify signatures. -// also batch signature verification. +/// auth should have a method to verify signatures. +/// also batch signature verification. pub trait Auth { - fn public_key(&self) -> PublicKey; // return the public key of the signer. - fn address(&self) -> Address; // return the account address of the signer. - fn verify(&self, data: &[u8], signature: &[u8]) -> bool; // verify a signature. - fn batch_verify(&self, data: &[u8], signatures: Vec<&[u8]>) -> bool; // batch verify signatures. returns error if batch verification fails. + // returns the public key of the signer. + fn public_key(&self) -> PublicKey; + // returns the account address of the signer. + fn address(&self) -> Address; + // verifys the signature. + fn verify(&self, data: &[u8], signature: &[u8]) -> bool; + // batch verify signatures. returns false if batch verification fails. + fn batch_verify(&self, data: &[u8], signatures: Vec<&[u8]>) -> bool; } - +/// Wallet is the module used by the user to sign transactions. Wallet uses Ed25519 signature scheme. pub struct Wallet { - auth: AuthTypes, // auth type - p_key: PrivateKey, // private key - pub_key: PublicKey, // public key - address: Address, // account address + // Private key + priv_key: PrivateKey, + // Public key + pub_key: PublicKey, + // Account Address, is derived from the public key. + address: Address, + // Signer + signer: Ed25519, } - // wallet generation, management, and signing should be functions of the wallet. -pub trait ImplWallet { - fn generate(auth_type: AuthTypes) -> Self; // create a new wallet of given auth type. - fn load(&self, auth_type: AuthTypes, p_key: &[u8]) -> Self; // load a new wallet from private key. - fn sign(&self, data: &[u8]) -> Vec; // sign data with the private key. - fn verify(&self,data: &[u8], signature: &[u8]) -> bool; // verify a signature with the public key. - fn address(&self) -> Address; // return the account address. - fn public_key(&self) -> Vec; // return the public key. - fn auth_type(&self) -> AuthTypes; // return the auth type. - fn private_key(&self) -> Vec; // returns the private key. - fn store_private_key(&self); // store the private key. - fn new_address(&mut self); // generate a new address. +pub trait WalletMethods { + // create a new wallet using the given randomness. + fn generate(r: &mut R) -> Self; + // load signer from bytes rep of a private key and initialize the wallet. + fn load(&self, priv_key: &[u8]) -> Self; + // sign the given arbitary data with the private key of the wallet. + fn sign(&mut self, data: &[u8]) -> Vec; + // verify the signature of the given data with the public key of the wallet. + fn verify(&self, data: &[u8], signature: &[u8]) + -> Result; + // return corresponding wallet's address. + fn address(&self) -> Address; + // return corresponding wallet's public key. + fn public_key(&self) -> PublicKey; + // return corresponding wallet's private key. + fn private_key(&self) -> Vec; + // store the private key at the given path. + fn store_private_key(&self, path: &str) -> Result<(), Error>; + // @todo remove this? + fn init_address(&mut self); } + +impl WalletMethods for Wallet { + fn generate(r: &mut R) -> Self { + let signer = Ed25519::new(r); + let pub_key = signer.public_key(); + let address = Address::from_pub_key(&pub_key); + Self { + priv_key: signer.private_key(), + pub_key: signer.public_key(), + address, + signer, + } + } + + fn load(&self, priv_key: &[u8]) -> Self { + let private_key = PrivateKey::try_from(priv_key).expect("Invalid private key"); + let signer = ::from(private_key).unwrap(); + Self { + priv_key: signer.private_key(), + pub_key: signer.public_key(), + address: Address::from_pub_key(&signer.public_key()), + signer, + } + } + + fn sign(&mut self, data: &[u8]) -> Vec { + self.signer.sign(Some(TX_NAMESPACE), data).as_ref().to_vec() + } + + fn verify( + &self, + data: &[u8], + signature: &[u8], + ) -> Result { + let signature = Signature::try_from(signature); + if let Err(e) = signature { + return Err(e); + } + + let signature = signature.unwrap(); + let pub_key = self.signer.public_key(); + Ok(Ed25519::verify( + Some(TX_NAMESPACE), + data, + &pub_key, + &signature, + )) + } + + fn address(&self) -> Address { + self.address.clone() + } + + fn public_key(&self) -> PublicKey { + self.pub_key.clone() + } + + fn private_key(&self) -> Vec { + self.priv_key.as_ref().to_vec() + } + + fn store_private_key(&self, _path: &str) -> Result<(), Error> { + todo!() + } + + fn init_address(&mut self) { + todo!() + } +} \ No newline at end of file diff --git a/types/src/wasm.rs b/types/src/wasm.rs index f9626fca..36c23dc7 100644 --- a/types/src/wasm.rs +++ b/types/src/wasm.rs @@ -130,4 +130,4 @@ pub fn parse_block(bytes: Vec) -> JsValue { #[wasm_bindgen] pub fn leader_index(seed: Vec, participants: usize) -> usize { compute_leader_index(&seed, participants) -} +} \ No newline at end of file From 8366c49f3002bbcad34ddf55bc1b36afb60aea93 Mon Sep 17 00:00:00 2001 From: NateAtNodeKit Date: Wed, 2 Apr 2025 16:13:48 -0400 Subject: [PATCH 26/38] initial client --- Cargo.lock | 1 + client/Cargo.toml | 3 +- client/src/client.rs | 58 +++++++++++++++++++++++++++++--------- client/src/client_types.rs | 23 +++++++++++++-- types/src/tx.rs | 2 +- 5 files changed, 69 insertions(+), 18 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 088bff10..80e1b67e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -97,6 +97,7 @@ dependencies = [ "thiserror 2.0.12", "tokio", "tokio-tungstenite 0.17.2", + "url", ] [[package]] diff --git a/client/Cargo.toml b/client/Cargo.toml index 038b8209..fb84b3a1 100644 --- a/client/Cargo.toml +++ b/client/Cargo.toml @@ -23,4 +23,5 @@ tokio-tungstenite = { version = "0.17", features = ["native-tls"] } tokio = { version = "1.40.0", features = ["full"] } serde = { version = "1.0.218", features = ["derive"] } serde_yaml = "0.9.34" -serde_bytes = "0.11.17" \ No newline at end of file +serde_bytes = "0.11.17" +url = "2.5.4" \ No newline at end of file diff --git a/client/src/client.rs b/client/src/client.rs index 358d06ca..a321ef32 100644 --- a/client/src/client.rs +++ b/client/src/client.rs @@ -1,15 +1,15 @@ -use reqwest::Client; -use crate::client_types::ClientRpcMessage; +use std::error::Error; +use reqwest::{Client, Url}; +use crate::client_types::{ClientRpcMessage, ClientRpcMessageResp}; use bytes::Bytes; use serde::Deserialize; -use alto_types::tx::Tx; +use alto_types::tx::{Tx, TxMethods}; pub const WEBSOCKET_PREFIX: &'static str = "/ws"; pub const RPC_PREFIX: &'static str = "/api"; +// TODO: Update the below endpoints when we know what they are pub const PATH_SUBMIT_TX: &'static str = "/mempool/submit"; - -//todo add more const once we have server - -//todo define all structs needed from server +pub const PATH_GET_BLOCK: &'static str = "/api/get_block"; +pub const PATH_GET_BLOCK_HEIGHT: &'static str = "/api/get_block_height"; #[derive(Debug)] pub struct JSONRPCClient { @@ -32,17 +32,47 @@ impl JSONRPCClient { } } //todo implement methods needed to communicate with server - pub fn submit_tx(&self, tx: Tx) -> Vec { - //todo safety checks - todo!() + pub async fn submit_tx(&self, mut tx: Tx) -> Result> { + let encoded_tx_bytes = tx.encode(); + let mut submit_request = ClientRpcMessage::SubmitTx { + payload: encoded_tx_bytes.into(), + }; + + let full_url = Url::parse(&self.base_url) + .and_then(|base| base.join(PATH_SUBMIT_TX)) + .expect("Invalid base_url or path for submit tx"); + + self.send_request(full_url.to_string(), submit_request.into()) + } + + pub async fn get_block(&self, height: u64) -> Result> { + let mut get_block_req = ClientRpcMessage::GetBlock { + height + }; + + let full_url = Url::parse(&self.base_url) + .and_then(|base| base.join(PATH_GET_BLOCK)) + .expect("Invalid base_url or path for get block"); + + self.send_request(full_url.to_string(), get_block_req.into()) } - pub fn get_block(&self, data: Vec) -> u64{ - todo!() + pub async fn get_block_height(&self) -> Result> { + let mut get_block_height_req = ClientRpcMessage::GetBlockHeight {}; + + let full_url = Url::parse(&self.base_url) + .and_then(|base| base.join(PATH_GET_BLOCK_HEIGHT)) + .expect("Invalid base_url or path for get block"); + + self.send_request(full_url.to_string(), get_block_height_req.into()) } - pub fn get_block_height(&self, data: Vec) { - todo!() + async fn send_request(&self, uri: String, data: Vec) -> Result> { + let resp = self.http_client.post(uri) + .body(data) + .send() + .await?; + Ok(resp) } } diff --git a/client/src/client_types.rs b/client/src/client_types.rs index dc2634a2..d958da5b 100644 --- a/client/src/client_types.rs +++ b/client/src/client_types.rs @@ -1,6 +1,7 @@ use bytes::{BufMut, Bytes}; use commonware_cryptography::sha256::Digest; use futures::channel::oneshot; +use serde::{Deserialize, Serialize}; use alto_types::Block; /// Messages sent from client @@ -80,16 +81,34 @@ impl WebsocketClientMessage { } } -#[derive(Debug)] +#[derive(Serialize, Debug)] pub enum ClientRpcMessage { // for rpc SubmitTx { payload: Bytes, }, GetBlockHeight { - response: oneshot::Sender }, GetBlock { height: u64, }, } + +#[derive(Deserialize, Debug)] +pub enum ClientRpcMessageResp { + // for rpc + SubmitTxResp { + ok: bool, + digest: Vec, + err: String, + }, + GetBlockHeightResp { + chain_id: Vec, + height: u64, + err: String, + }, + GetBlockResp { + height: u64, + err: String, + }, +} diff --git a/types/src/tx.rs b/types/src/tx.rs index ba727a3b..e3682664 100644 --- a/types/src/tx.rs +++ b/types/src/tx.rs @@ -211,7 +211,7 @@ impl TxMethods for Tx { let unit_bytes = unit.encode(); // pack the unit type info. self.digest.extend((unit.unit_type() as u8).to_be_bytes()); - // pack len of inidividual unit. + // pack len of individual unit. self.digest.extend((unit_bytes.len() as u64).to_be_bytes()); // pack individual unit. self.digest.extend_from_slice(&unit_bytes); From a1ec9a81532f6616963fb9501e8acbb749dfc1d1 Mon Sep 17 00:00:00 2001 From: bianyuanop Date: Wed, 2 Apr 2025 21:15:01 -0400 Subject: [PATCH 27/38] resolve errors --- chain/src/actors/application/actor.rs | 6 ++--- chain/src/actors/net/actor.rs | 4 +-- chain/src/actors/net/mod.rs | 10 ++++--- client/src/client.rs | 20 ++++++++------ client/src/client_types.rs | 38 ++++++++++++++++++++++++--- 5 files changed, 56 insertions(+), 22 deletions(-) diff --git a/chain/src/actors/application/actor.rs b/chain/src/actors/application/actor.rs index aa1afd56..a7f7bbd1 100644 --- a/chain/src/actors/application/actor.rs +++ b/chain/src/actors/application/actor.rs @@ -6,7 +6,7 @@ use super::{ use crate::actors::syncer; use alto_types::{Block, Finalization, Notarization, Seed}; use commonware_consensus::threshold_simplex::Prover; -use commonware_cryptography::{sha256::Digest, Hasher, Sha256}; +use commonware_cryptography::{hash, sha256::{self, Digest}, Hasher, Sha256}; use commonware_macros::select; use commonware_runtime::{Clock, Handle, Metrics, Spawner}; use commonware_utils::SystemTimeExt; @@ -85,7 +85,7 @@ impl Actor { // Compute genesis digest self.hasher.update(GENESIS); let genesis_parent = self.hasher.finalize(); - let genesis = Block::new(genesis_parent, 0, 0); + let genesis = Block::new(genesis_parent, 0, 0, vec![], sha256::hash(&[0; 32])); let genesis_digest = genesis.digest(); let built: Option = None; let built = Arc::new(Mutex::new(built)); @@ -125,7 +125,7 @@ impl Actor { if current <= parent.timestamp { current = parent.timestamp + 1; } - let block = Block::new(parent.digest(), parent.height+1, current); + let block = Block::new(parent.digest(), parent.height+1, current, vec![], sha256::hash(&[0; 32])); let digest = block.digest(); { let mut built = built.lock().unwrap(); diff --git a/chain/src/actors/net/actor.rs b/chain/src/actors/net/actor.rs index 69c4dd57..3719e545 100644 --- a/chain/src/actors/net/actor.rs +++ b/chain/src/actors/net/actor.rs @@ -1,8 +1,7 @@ use std::{ collections::{HashMap, HashSet}, hash::Hash, io, ops::Deref, sync::{Arc, RwLock} }; -use alto_client::Client; -use alto_types::Block; +use alto_client::client_types::{WebsocketClientMessage}; use axum::response::IntoResponse; use axum::{ routing::get, @@ -24,7 +23,6 @@ use tracing_subscriber::fmt::format; use crate::actors::mempool::mempool::{self, RawTransaction}; use super::ingress::{Mailbox, Message}; -use alto_client::client_types::WebsocketClientMessage; #[derive(Deserialize)] pub struct DummyTransaction { #[serde(with = "serde_bytes")] diff --git a/chain/src/actors/net/mod.rs b/chain/src/actors/net/mod.rs index 06a9162c..91a033e6 100644 --- a/chain/src/actors/net/mod.rs +++ b/chain/src/actors/net/mod.rs @@ -5,7 +5,7 @@ pub mod ingress; mod tests { use core::panic; use std::time::Duration; - use alto_types::Block; + use alto_types::{Block}; use axum::{ body::Body, http::{Request, StatusCode} @@ -17,14 +17,16 @@ mod tests { use tokio_tungstenite::{connect_async, tungstenite::{client, Message as WsClientMessage}}; use tower::ServiceExt; use tracing_subscriber::{field::debug, fmt::format}; + use alto_client::client_types::{WebsocketClientMessage}; - use crate::actors::{mempool::mempool, net::ingress::WebsocketClientMessage}; + use crate::actors::{mempool::mempool}; + use super::{actor::Actor, ingress::Message, actor::{self}}; use tracing::debug; #[test_traced] - fn test_msg() { + fn test_submit_tx() { let (runner, mut context) = Executor::init(tokio::Config::default()); runner.start(async move { let (mempool_sender, mut mempool_receiver) = mpsc::channel(1024); @@ -120,7 +122,7 @@ mod tests { let parent_digest = sha256::hash(&[0; 32]); let height = 0; let timestamp = 1; - let block = Block::new(parent_digest, height, timestamp); + let block = Block::new(parent_digest, height, timestamp, vec![], sha256::hash(&[0; 32])); mailbox.broadcast_block(block).await; context.sleep(Duration::from_millis(1000)).await; diff --git a/client/src/client.rs b/client/src/client.rs index a321ef32..79780c1a 100644 --- a/client/src/client.rs +++ b/client/src/client.rs @@ -1,6 +1,6 @@ use std::error::Error; use reqwest::{Client, Url}; -use crate::client_types::{ClientRpcMessage, ClientRpcMessageResp}; +use super::client_types::{ClientRpcMessageResp, ClientRpcMessage}; use bytes::Bytes; use serde::Deserialize; use alto_types::tx::{Tx, TxMethods}; @@ -32,7 +32,7 @@ impl JSONRPCClient { } } //todo implement methods needed to communicate with server - pub async fn submit_tx(&self, mut tx: Tx) -> Result> { + pub async fn submit_tx(&self, mut tx: Tx) -> Result> { let encoded_tx_bytes = tx.encode(); let mut submit_request = ClientRpcMessage::SubmitTx { payload: encoded_tx_bytes.into(), @@ -42,10 +42,11 @@ impl JSONRPCClient { .and_then(|base| base.join(PATH_SUBMIT_TX)) .expect("Invalid base_url or path for submit tx"); - self.send_request(full_url.to_string(), submit_request.into()) + todo!() + // self.send_request(full_url.to_string(), submit_request.into()).await } - pub async fn get_block(&self, height: u64) -> Result> { + pub async fn get_block(&self, height: u64) -> Result> { let mut get_block_req = ClientRpcMessage::GetBlock { height }; @@ -54,17 +55,19 @@ impl JSONRPCClient { .and_then(|base| base.join(PATH_GET_BLOCK)) .expect("Invalid base_url or path for get block"); - self.send_request(full_url.to_string(), get_block_req.into()) + todo!() + // self.send_request(full_url.to_string(), get_block_req.into()).await } - pub async fn get_block_height(&self) -> Result> { + pub async fn get_block_height(&self) -> Result> { let mut get_block_height_req = ClientRpcMessage::GetBlockHeight {}; let full_url = Url::parse(&self.base_url) .and_then(|base| base.join(PATH_GET_BLOCK_HEIGHT)) .expect("Invalid base_url or path for get block"); - self.send_request(full_url.to_string(), get_block_height_req.into()) + todo!() + // self.send_request(full_url.to_string(), get_block_height_req.into()).await } async fn send_request(&self, uri: String, data: Vec) -> Result> { @@ -72,7 +75,8 @@ impl JSONRPCClient { .body(data) .send() .await?; - Ok(resp) + todo!() + // Ok(resp) } } diff --git a/client/src/client_types.rs b/client/src/client_types.rs index d958da5b..c90fcf21 100644 --- a/client/src/client_types.rs +++ b/client/src/client_types.rs @@ -1,8 +1,6 @@ use bytes::{BufMut, Bytes}; use commonware_cryptography::sha256::Digest; -use futures::channel::oneshot; use serde::{Deserialize, Serialize}; -use alto_types::Block; /// Messages sent from client pub enum ClientMessage { @@ -81,7 +79,7 @@ impl WebsocketClientMessage { } } -#[derive(Serialize, Debug)] +#[derive(Debug)] pub enum ClientRpcMessage { // for rpc SubmitTx { @@ -94,7 +92,23 @@ pub enum ClientRpcMessage { }, } -#[derive(Deserialize, Debug)] +impl Serialize for ClientRpcMessage { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer { + todo!() + } +} + +impl<'de> Deserialize<'de> for ClientRpcMessage { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de> { + todo!() + } +} + +#[derive(Debug)] pub enum ClientRpcMessageResp { // for rpc SubmitTxResp { @@ -112,3 +126,19 @@ pub enum ClientRpcMessageResp { err: String, }, } + +impl Serialize for ClientRpcMessageResp { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer { + todo!() + } +} + +impl<'de> Deserialize<'de> for ClientRpcMessageResp { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de> { + todo!() + } +} From 18cbe1f79ce52d53ef361895f1f02430af4f867c Mon Sep 17 00:00:00 2001 From: bianyuanop Date: Wed, 2 Apr 2025 22:13:54 -0400 Subject: [PATCH 28/38] use signed tx in mempool & a lot todos --- chain/src/actors/mempool/mempool.rs | 43 +++++---- chain/src/actors/mempool/mod.rs | 3 +- chain/src/actors/net/actor.rs | 20 +++-- chain/src/actors/net/ingress.rs | 17 ---- chain/src/actors/net/mod.rs | 2 +- client/src/client.rs | 5 +- client/src/client_types.rs | 80 +++++++++-------- types/src/block.rs | 6 +- types/src/signed_tx.rs | 93 +++++++++++++------ types/src/tx.rs | 133 +++++++++++++++------------- 10 files changed, 227 insertions(+), 175 deletions(-) diff --git a/chain/src/actors/mempool/mempool.rs b/chain/src/actors/mempool/mempool.rs index 2fb838db..d93d561c 100644 --- a/chain/src/actors/mempool/mempool.rs +++ b/chain/src/actors/mempool/mempool.rs @@ -18,27 +18,28 @@ use tracing::{debug, warn, info}; use governor::clock::Clock as GClock; use super::{handler::{Handler, self}, key::{self, MultiIndex, Value}, ingress, coordinator::Coordinator, archive::Wrapped}; use crate::{actors::net, maybe_delay_between}; +use alto_types::{signed_tx::SignedTx, tx::Tx}; #[derive(Clone, Debug)] pub struct Batch { pub timestamp: SystemTime, // TODO: store real transactions not just raws - pub txs: Vec>, + pub txs: Vec>, pub digest: H::Digest, } impl Batch { - fn compute_digest(txs: &Vec>) -> H::Digest { + fn compute_digest(txs: &Vec>) -> H::Digest { let mut hasher = H::new(); for tx in txs.iter() { - hasher.update(&tx.raw); + hasher.update(&tx.payload()); } hasher.finalize() } - pub fn new(txs: Vec>, timestamp: SystemTime) -> Self { + pub fn new(txs: Vec>, timestamp: SystemTime) -> Self { let digest = Self::compute_digest(&txs); Self { @@ -53,17 +54,17 @@ impl Batch { bytes.put_u64(self.timestamp.epoch_millis()); bytes.put_u64(self.txs.len() as u64); for tx in self.txs.iter() { - bytes.put_u64(tx.size()); - bytes.extend_from_slice(&tx.raw); + bytes.put_u64(tx.size() as u64); + bytes.extend_from_slice(&tx.payload()); } bytes } - pub fn deserialize(mut bytes: &[u8]) -> Option { + pub fn deserialize(mut bytes: &[u8]) -> Result { use bytes::Buf; // We expect at least 18 bytes for the header if bytes.remaining() < 18 { - return None; + return Err(format!("not enough bytes for header")); } let timestamp = bytes.get_u64(); let timestamp = SystemTime::UNIX_EPOCH + Duration::from_millis(timestamp); @@ -73,22 +74,22 @@ impl Batch { for _ in 0..tx_count { // For each transaction, first read the size (u64). if bytes.remaining() < 8 { - return None; + return Err(format!("not enough bytes for tx size")); } let tx_size = bytes.get_u64() as usize; // Ensure there are enough bytes left. if bytes.remaining() < tx_size { - return None; + return Err(format!("not enough bytes for tx payload, needed: {}, actual: {}", tx_size, bytes.remaining())); } // Extract tx_size bytes. let tx_bytes = bytes.copy_to_bytes(tx_size); - txs.push(RawTransaction::new(tx_bytes)); + txs.push(SignedTx::deserialize(&tx_bytes)?); } // Compute the digest from the transactions. let digest = Self::compute_digest(&txs); // Since serialize did not include accepted and timestamp, we set accepted to false // and set timestamp to the current time. - Some(Self { + Ok(Self { timestamp, txs, digest, @@ -96,11 +97,13 @@ impl Batch { } pub fn contain_tx(&self, digest: &H::Digest) -> bool { - self.txs.iter().any(|tx| &tx.digest == digest) + todo!() + // self.txs.iter().any(|tx| &tx.digest == digest) } pub fn tx(&self, digest: &H::Digest) -> Option> { - self.txs.iter().find(|tx| &tx.digest == digest).cloned() + // self.txs.iter().find(|tx| &tx.digest == digest).cloned() + todo!() } } @@ -151,7 +154,7 @@ pub enum Message { }, // from rpc or websocket SubmitTxs { - payload: Vec>, + payload: Vec>, response: oneshot::Sender> }, BatchConsumed { @@ -199,7 +202,7 @@ impl Mailbox { receiver.await.expect("failed to receive batch acknowledge") } - pub async fn submit_txs(&mut self, payload: Vec>) -> Vec { + pub async fn submit_txs(&mut self, payload: Vec>) -> Vec { let (response, receiver) = oneshot::channel(); self.sender .send(Message::SubmitTxs { payload, response }) @@ -288,7 +291,7 @@ pub struct Mempool< accepted: Archive, consumed: Archive, - txs: Vec>, + txs: Vec>, mailbox: mpsc::Receiver>, mailbox_size: usize, @@ -438,6 +441,8 @@ impl< }; match message { Message::SubmitTxs { payload, response } => { + // TODO: add timestamp verification here + let mut result = Vec::with_capacity(payload.len()); for tx in payload.into_iter() { if !tx.validate() { @@ -557,7 +562,7 @@ impl< let mut size = 0; let mut txs_cnt = 0; for tx in self.txs.iter() { - size += tx.size(); + size += tx.size() as u64; txs_cnt += 1; if size > self.batch_size_limit { @@ -585,7 +590,7 @@ impl< }, batch_message = batch_network.1.recv() => { let (sender, message) = batch_message.expect("Batch broadcast closed"); - let Some(batch) = Batch::deserialize(&message) else { + let Ok(batch) = Batch::deserialize(&message) else { warn!(?sender, "failed to deserialize batch"); continue; }; diff --git a/chain/src/actors/mempool/mod.rs b/chain/src/actors/mempool/mod.rs index 0516af63..762472e5 100644 --- a/chain/src/actors/mempool/mod.rs +++ b/chain/src/actors/mempool/mod.rs @@ -11,6 +11,7 @@ pub mod archive; mod tests { use core::panic; use std::{collections::{BTreeMap, HashMap}, num::NonZeroU32, sync::{Arc, Mutex}, time::Duration}; + use alto_types::{signed_tx::SignedTx, tx::Tx}; use bytes::Bytes; use commonware_broadcast::linked::{Config, Engine}; @@ -276,7 +277,7 @@ mod tests { // issue tx to the first validator let mut digests = Vec::new(); for i in 0..num_txs { - let tx = RawTransaction::new(Bytes::from(format!("tx-{}", i))); + let tx = SignedTx::random(); let submission_res = mailbox.submit_txs(vec![tx.clone()]).await; if !submission_res[0] { warn!(?tx.digest, "failed to submit tx"); diff --git a/chain/src/actors/net/actor.rs b/chain/src/actors/net/actor.rs index 3719e545..aeef046d 100644 --- a/chain/src/actors/net/actor.rs +++ b/chain/src/actors/net/actor.rs @@ -2,6 +2,7 @@ use std::{ collections::{HashMap, HashSet}, hash::Hash, io, ops::Deref, sync::{Arc, RwLock} }; use alto_client::client_types::{WebsocketClientMessage}; +use alto_types::signed_tx::SignedTx; use axum::response::IntoResponse; use axum::{ routing::get, @@ -203,7 +204,6 @@ impl Actor { let state = state.write().unwrap(); state.mempool.clone() }; - let txs = txs.into_iter().map(|tx| mempool::RawTransaction::::new(tx)).collect(); let submission_res = mempool.submit_txs(txs).await; debug!(?submission_res, "txs submission result") } @@ -240,12 +240,18 @@ impl Actor { let mut mempool = { state.read().unwrap().mempool.clone() }; - - let success = mempool.submit_txs(vec![RawTransaction::new(payload)]).await[0]; - if success { - "submitted".to_string() - } else { - "failed to submit tx".to_string() + match SignedTx::::deserialize(&payload) { + Ok(tx) => { + let success = mempool.submit_txs(vec![]).await[0]; + if success { + "submitted".to_string() + } else { + "failed to submit tx".to_string() + } + }, + Err(err) => { + format!("failed to submit tx {}", err) + } } } diff --git a/chain/src/actors/net/ingress.rs b/chain/src/actors/net/ingress.rs index b0aab50b..00793e57 100644 --- a/chain/src/actors/net/ingress.rs +++ b/chain/src/actors/net/ingress.rs @@ -43,23 +43,6 @@ impl Message { } } -#[derive(Debug)] -pub enum RpcMessage { - // for rpc - SubmitTx { - payload: Bytes, - response: oneshot::Sender - }, - GetBlockHeight { - response: oneshot::Sender - }, - GetBlock { - height: u64, - response: oneshot::Sender>, - }, -} - - #[derive(Clone)] pub struct Mailbox { sender: mpsc::Sender diff --git a/chain/src/actors/net/mod.rs b/chain/src/actors/net/mod.rs index 91a033e6..c9c8524d 100644 --- a/chain/src/actors/net/mod.rs +++ b/chain/src/actors/net/mod.rs @@ -95,7 +95,7 @@ mod tests { let (mut write, mut read) = ws_stream.split(); // register block - let _ = write.send(WsClientMessage::binary(WebsocketClientMessage::RegisterBlock.serialize())).await; + let _ = write.send(WsClientMessage::binary(WebsocketClientMessage::::RegisterBlock.serialize())).await; // listening block let client_handler = context.with_label("ws_client").spawn(async move |_| { diff --git a/client/src/client.rs b/client/src/client.rs index 79780c1a..81c106e2 100644 --- a/client/src/client.rs +++ b/client/src/client.rs @@ -1,9 +1,10 @@ use std::error::Error; +use commonware_cryptography::Sha256; use reqwest::{Client, Url}; use super::client_types::{ClientRpcMessageResp, ClientRpcMessage}; use bytes::Bytes; use serde::Deserialize; -use alto_types::tx::{Tx, TxMethods}; +use alto_types::tx::{Tx}; pub const WEBSOCKET_PREFIX: &'static str = "/ws"; pub const RPC_PREFIX: &'static str = "/api"; // TODO: Update the below endpoints when we know what they are @@ -32,7 +33,7 @@ impl JSONRPCClient { } } //todo implement methods needed to communicate with server - pub async fn submit_tx(&self, mut tx: Tx) -> Result> { + pub async fn submit_tx(&self, mut tx: Tx) -> Result> { let encoded_tx_bytes = tx.encode(); let mut submit_request = ClientRpcMessage::SubmitTx { payload: encoded_tx_bytes.into(), diff --git a/client/src/client_types.rs b/client/src/client_types.rs index c90fcf21..6761bcf5 100644 --- a/client/src/client_types.rs +++ b/client/src/client_types.rs @@ -1,42 +1,51 @@ +use std::fmt::Debug; + +use alto_types::signed_tx::SignedTx; use bytes::{BufMut, Bytes}; -use commonware_cryptography::sha256::Digest; +use commonware_cryptography::{sha256::Digest, Hasher, Sha256}; use serde::{Deserialize, Serialize}; /// Messages sent from client pub enum ClientMessage { - WSMessage(WebsocketClientMessage), + WSMessage(WebsocketClientMessage), RpcMessage(ClientRpcMessage) } /// Websocket Message sent from client -#[derive(Debug)] #[repr(u8)] -pub enum WebsocketClientMessage { +pub enum WebsocketClientMessage { RegisterBlock = 0, RegisterTx, + SubmitTxs(Vec>) +} - SubmitTxs(Vec) +impl Debug for WebsocketClientMessage { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + todo!() + } } -impl WebsocketClientMessage { + +impl WebsocketClientMessage { // TODO: cache the serialization result pub fn serialize(&self) -> Vec { match self { Self::RegisterBlock => vec![0], Self::RegisterTx => vec![1], Self::SubmitTxs(txs) => { - let mut raw = vec![2]; // first byte indicates the message type - raw.put_u64(txs.len() as u64); - for tx in txs.into_iter() { - raw.put_u64(tx.len() as u64); - raw.put(tx.clone()); - } - raw + todo!() + // let mut raw = vec![2]; // first byte indicates the message type + // raw.put_u64(txs.len() as u64); + // for tx in txs.into_iter() { + // raw.put_u64(tx.size() as u64); + // raw.put(tx.payload().into()); + // } + // raw } } } - pub fn deserialize(raw: &[u8]) -> Result { + pub fn deserialize(raw: &[u8]) -> Result, String> { use bytes::Buf; if raw.is_empty() { @@ -52,27 +61,28 @@ impl WebsocketClientMessage { 0 => Ok(WebsocketClientMessage::RegisterBlock), 1 => Ok(WebsocketClientMessage::RegisterTx), 2 => { - // Ensure there are enough bytes for the number of transactions. - if buf.remaining() < 8 { - return Err("payload too short for number of transactions".into()); - } - let num_txs = buf.get_u64(); - - let mut txs = Vec::with_capacity(num_txs as usize); - // Loop over each transaction. - for _ in 0..num_txs { - if buf.remaining() < 8 { - return Err("payload too short for transaction length".into()); - } - let tx_len = buf.get_u64() as usize; - if buf.remaining() < tx_len { - return Err("payload too short for transaction data".into()); - } - // Extract the transaction bytes. - let tx_data = buf.copy_to_bytes(tx_len); - txs.push(tx_data); - } - Ok(WebsocketClientMessage::SubmitTxs(txs)) + todo!() + // // Ensure there are enough bytes for the number of transactions. + // if buf.remaining() < 8 { + // return Err("payload too short for number of transactions".into()); + // } + // let num_txs = buf.get_u64(); + + // let mut txs = Vec::with_capacity(num_txs as usize); + // // Loop over each transaction. + // for _ in 0..num_txs { + // if buf.remaining() < 8 { + // return Err("payload too short for transaction length".into()); + // } + // let tx_len = buf.get_u64() as usize; + // if buf.remaining() < tx_len { + // return Err("payload too short for transaction data".into()); + // } + // // Extract the transaction bytes. + // let tx_data = buf.copy_to_bytes(tx_len); + // txs.push(tx_data); + // } + // Ok(WebsocketClientMessage::SubmitTxs(txs)) } other => Err(format!("unsupported message type: {}", other)), } diff --git a/types/src/block.rs b/types/src/block.rs index 34697fc6..2205c108 100644 --- a/types/src/block.rs +++ b/types/src/block.rs @@ -1,4 +1,4 @@ -use crate::signed_tx::{pack_signed_txs, unpack_signed_txs, SignedTx, SignedTxChars}; +use crate::signed_tx::{pack_signed_txs, unpack_signed_txs, SignedTx}; use crate::{Finalization, Notarization}; use bytes::{Buf, BufMut}; use commonware_cryptography::{bls12381::PublicKey, sha256, sha256::Digest, Hasher, Sha256}; @@ -23,7 +23,7 @@ pub struct Block { /// The state root of the block. pub state_root: Digest, - txs: Vec, + txs: Vec>, /// Pre-computed digest of the block. digest: Digest, } @@ -49,7 +49,7 @@ impl Block { parent: Digest, height: u64, timestamp: u64, - txs: Vec, + txs: Vec>, state_root: Digest, ) -> Self { // let mut txs = txs; diff --git a/types/src/signed_tx.rs b/types/src/signed_tx.rs index 3db2551b..d9be6832 100644 --- a/types/src/signed_tx.rs +++ b/types/src/signed_tx.rs @@ -1,38 +1,66 @@ +use std::fmt::Debug; +use std::hash::Hash; + use crate::address::Address; -use crate::tx::{Tx, TxMethods}; +use crate::tx::{Tx}; use crate::wallet::{Wallet, WalletMethods}; use crate::{PublicKey, Signature, TX_NAMESPACE}; -use commonware_cryptography::{Ed25519, Scheme}; +use commonware_cryptography::{Ed25519, Hasher, Scheme}; // this is sent by the user to the validators. -#[derive(Clone, Debug)] -pub struct SignedTx { - pub tx: Tx, +#[derive(Clone)] +pub struct SignedTx { + pub tx: Tx, + + pub digest: H::Digest, pub_key: PublicKey, address: Address, signature: Vec, } -// function names are self explanatory. -pub trait SignedTxChars: Sized { - fn new(tx: Tx, pub_key: PublicKey, signature: Vec) -> Self; - // fn sign(&mut self, wallet: Wallet) -> SignedTx; - fn verify(&mut self) -> bool; - fn signature(&self) -> Vec; - fn public_key(&self) -> Vec; - fn address(&self) -> Address; - fn encode(&mut self) -> Vec; - fn decode(bytes: &[u8]) -> Result; +impl Debug for SignedTx { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + todo!() + } } -impl SignedTxChars for SignedTx { + +impl SignedTx { + pub fn payload(&self) -> Vec { + todo!() + } + pub fn size(&self) -> usize { + todo!() + } + + pub fn serialize(&self) -> Vec { + todo!() + } + + pub fn deserialize(raw: &[u8]) -> Result { + todo!() + } + + pub fn validate(&self) -> bool { + todo!() + } + + pub fn random() -> Self { + todo!() + } + + + // @todo either have all fields initialized or none. - fn new(tx: Tx, pub_key: PublicKey, signature: Vec) -> Self { + fn new(tx: Tx, pub_key: PublicKey, signature: Vec) -> Self { + let mut hasher = H::new(); + let digest = hasher.finalize(); Self { tx, pub_key: pub_key.clone(), address: Address::from_pub_key(&pub_key), signature: signature.clone(), + digest } } @@ -59,7 +87,7 @@ impl SignedTxChars for SignedTx { } // @todo add syntactic checks. - fn encode(&mut self) -> Vec { + pub fn encode(&mut self) -> Vec { let mut bytes = Vec::new(); let raw_tx = self.tx.encode(); @@ -89,28 +117,37 @@ impl SignedTxChars for SignedTx { return Err(tx.unwrap_err()); } + let mut hasher = H::new(); + hasher.update(raw_tx); + let digest = hasher.finalize(); + Ok(SignedTx { tx: tx.unwrap(), pub_key: public_key.clone(), address: Address::from_pub_key(&public_key), signature: signature.to_vec(), + digest }) } -} -impl SignedTx { - pub fn sign(mut tx: Tx, mut wallet: Wallet) -> SignedTx { + pub fn sign(mut tx: Tx, mut wallet: Wallet) -> SignedTx { let tx_data = tx.encode(); + + let mut hasher = H::new(); + hasher.update(&tx_data); + let digest = hasher.finalize(); + SignedTx { tx: tx.clone(), signature: wallet.sign(&tx_data), address: wallet.address(), pub_key: wallet.public_key(), + digest } } } -pub fn pack_signed_txs(signed_txs: Vec) -> Vec { +pub fn pack_signed_txs(signed_txs: Vec>) -> Vec { let mut bytes = Vec::new(); bytes.extend((signed_txs.len() as u64).to_be_bytes()); for signed_tx in signed_txs { @@ -123,7 +160,7 @@ pub fn pack_signed_txs(signed_txs: Vec) -> Vec { bytes } -pub fn unpack_signed_txs(bytes: Vec) -> Vec { +pub fn unpack_signed_txs(bytes: Vec) -> Vec> { let signed_txs_len = u64::from_be_bytes(bytes[0..8].try_into().unwrap()); let mut signed_txs = Vec::with_capacity(signed_txs_len as usize); let mut offset = 8; @@ -145,12 +182,14 @@ pub fn unpack_signed_txs(bytes: Vec) -> Vec { mod tests { use std::default; use std::error::Error; + use std::hash::Hash; use super::*; use crate::tx::Unit; use crate::units::transfer::Transfer; use crate::{create_test_keypair, curr_timestamp}; - use commonware_cryptography::sha256::Digest; + use commonware_cryptography::sha256::{self, Digest}; + use commonware_cryptography::Sha256; use more_asserts::assert_gt; #[test] @@ -166,7 +205,7 @@ mod tests { let (pk, sk) = create_test_keypair(); // TODO: the .encode call on next line gave error and said origin_msg needed to be mut? but why? // shouldn't encode be able to encode without changing the msg? - let tx = Tx { + let tx = Tx:: { timestamp, max_fee, priority_fee, @@ -176,15 +215,17 @@ mod tests { digest: digest.to_vec(), actor: Address::empty(), }; + let digest = sha256::hash(&[0; 32]); let mut origin_msg = SignedTx { tx, pub_key: pk, address: Address::create_random_address(), signature: vec![], + digest }; let encoded_bytes = origin_msg.encode(); assert_gt!(encoded_bytes.len(), 0); - let decoded_msg = SignedTx::decode(&encoded_bytes)?; + let decoded_msg = SignedTx::::decode(&encoded_bytes)?; assert_eq!(origin_msg.pub_key, decoded_msg.pub_key); assert_eq!(origin_msg.address, decoded_msg.address); assert_eq!(origin_msg.signature, decoded_msg.signature); diff --git a/types/src/tx.rs b/types/src/tx.rs index e3682664..07009e03 100644 --- a/types/src/tx.rs +++ b/types/src/tx.rs @@ -1,6 +1,9 @@ -use commonware_cryptography::sha256; +use commonware_cryptography::{hash, sha256, Hasher}; use commonware_cryptography::sha256::Digest; +use core::hash; use std::any::Any; +use std::cell::OnceCell; +use std::fmt::Debug; use crate::address::Address; use crate::signed_tx::SignedTx; @@ -71,8 +74,9 @@ impl Clone for Box { } } -#[derive(Clone, Debug)] -pub struct Tx { +// TODO: add a commonware_cryptography::Hasher trait for Tx, and the digest should be labeled as H::Digest +#[derive(Clone)] +pub struct Tx { /// timestamp of the tx creation. set by the user. /// will be verified if the tx is in the valid window once received by validators. /// if the timestamp is not in the valid window, the tx will be rejected. @@ -88,61 +92,53 @@ pub struct Tx { /// units are fundamental unit of a tx. similar to actions. pub units: Vec>, + // TODO: the digest and the id should be the same thing, which is the hash of the tx payload /// id is the transaction id. It is the hash of digest. - pub id: Digest, + pub id: H::Digest, /// digest is encoded tx. pub digest: Vec, /// address of the tx sender. wrap this in a better way. pub actor: Address, + + // TODO: add a payload referenced by OnceCell here possibly to avoid repeated serialization/deserialization } -pub trait TxMethods: Sized { - /// new is used to create a new instance of Tx with given units and chain id. - fn new(units: Vec>, chain_id: u64) -> Self; - /// set_fee is used to set the max fee and priority fee of the tx. - fn set_fee(&mut self, max_fee: u64, priority_fee: u64); - /// sign is used to sign the tx with the given wallet. - fn sign(&mut self, wallet: Wallet) -> SignedTx; - fn from( - timestamp: u64, - units: Vec>, - priority_fee: u64, - max_fee: u64, - chain_id: u64, - actor: Address, - ) -> Self; +impl Debug for Tx { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + todo!() + } +} - /// returns tx id. - fn id(&mut self) -> Digest; - /// returns digest of the tx. - fn digest(&self) -> Vec; - /// encodes the tx, writes to digest and returns the digest. - /// ensure all fields are properly set before calling this function. - fn encode(&mut self) -> Vec; +impl Tx { + pub fn digest(&self) -> H::Digest { + todo!() + } - fn decode(bytes: &[u8]) -> Result; + pub fn validate(&self) -> bool { + todo!() + } - fn set_actor(&mut self, actor: Address); + pub fn payload(&self) -> Vec { + todo!() + } - fn actor(&self) -> Address; -} + pub fn serialize(&self) -> Vec { + todo!() + } -impl Default for Tx { - fn default() -> Self { - Self { - timestamp: 0, - units: vec![], - max_fee: 0, - priority_fee: 0, - chain_id: 19517, - id: [0; 32].into(), - digest: vec![], - actor: Address::empty(), - } + pub fn deserialize(raw: &[u8]) -> Result { + todo!() + } + + // size of the payload + pub fn size(&self) -> usize { + todo!() + } + + pub fn random() -> Self { + todo!() } -} -impl TxMethods for Tx { fn new(units: Vec>, chain_id: u64) -> Self { let mut tx = Self::default(); tx.timestamp = SystemTime::now().epoch_millis(); @@ -158,10 +154,12 @@ impl TxMethods for Tx { self.priority_fee = priority_fee; } - fn sign(&mut self, wallet: Wallet) -> SignedTx { + fn sign(&mut self, wallet: Wallet) -> SignedTx { SignedTx::sign(self.clone(), wallet) } + //TODO: rename possibly as this method signature is more like a `new` for me instead of a from method + // the from method usually is taken by the From trait fn from( timestamp: u64, units: Vec>, @@ -181,18 +179,7 @@ impl TxMethods for Tx { tx } - fn id(&mut self) -> Digest { - if self.digest.len() == 0 { - self.encode(); - } - self.id.clone() - } - - fn digest(&self) -> Vec { - self.digest.clone() - } - - fn encode(&mut self) -> Vec { + pub fn encode(&mut self) -> Vec { if self.digest.is_empty() { return self.digest.clone(); } @@ -217,14 +204,11 @@ impl TxMethods for Tx { self.digest.extend_from_slice(&unit_bytes); }); - // generate tx id. - self.id = sha256::hash(&self.digest); - // return encoded digest. self.digest.clone() } - fn decode(bytes: &[u8]) -> Result { + pub fn decode(bytes: &[u8]) -> Result { if bytes.is_empty() { return Err("Empty bytes".to_string()); } @@ -240,7 +224,9 @@ impl TxMethods for Tx { } tx.units = units?; // generate tx id. - tx.id = sha256::hash(&tx.digest); + let mut hasher = H::new(); + hasher.update(&tx.digest()); + tx.id = hasher.finalize(); // return transaction. Ok(tx) } @@ -254,6 +240,24 @@ impl TxMethods for Tx { } } +impl Default for Tx { + fn default() -> Self { + let mut hasher = H::new(); + hasher.update(&[0; 32]); + + Self { + timestamp: 0, + units: vec![], + max_fee: 0, + priority_fee: 0, + chain_id: 19517, + id: hasher.finalize(), + digest: vec![], + actor: Address::empty(), + } + } +} + fn unpack_units(digest: &[u8]) -> Result>, String> { let mut offset = 0; @@ -325,6 +329,7 @@ mod tests { use super::*; use crate::curr_timestamp; use crate::units::transfer::Transfer; + use commonware_cryptography::Sha256; use more_asserts::assert_gt; use std::error::Error; @@ -340,7 +345,7 @@ mod tests { let id = Digest::from(digest.clone()); // TODO: the .encode call on next line gave error and said origin_msg needed to be mut? but why? // shouldn't encode be able to encode without changing the msg? - let mut origin_msg = Tx { + let mut origin_msg = Tx:: { timestamp, max_fee, priority_fee, @@ -352,7 +357,7 @@ mod tests { }; let encoded_bytes = origin_msg.encode(); assert_gt!(encoded_bytes.len(), 0); - let decoded_msg = Tx::decode(&encoded_bytes)?; + let decoded_msg = Tx::::decode(&encoded_bytes)?; let origin_transfer = origin_msg.units[0] .as_ref() .as_any() From ddfec1a66bf0ad1cdf36305687370c56bf5a4ee3 Mon Sep 17 00:00:00 2001 From: bianyuanop Date: Thu, 3 Apr 2025 10:07:51 -0400 Subject: [PATCH 29/38] move batch to types --- chain/src/actors/mempool/mempool.rs | 132 +--------------------------- chain/src/actors/mempool/mod.rs | 2 +- chain/src/actors/net/actor.rs | 2 +- types/src/batch.rs | 94 ++++++++++++++++++++ types/src/lib.rs | 2 + 5 files changed, 101 insertions(+), 131 deletions(-) create mode 100644 types/src/batch.rs diff --git a/chain/src/actors/mempool/mempool.rs b/chain/src/actors/mempool/mempool.rs index d93d561c..ce28b5d4 100644 --- a/chain/src/actors/mempool/mempool.rs +++ b/chain/src/actors/mempool/mempool.rs @@ -18,133 +18,7 @@ use tracing::{debug, warn, info}; use governor::clock::Clock as GClock; use super::{handler::{Handler, self}, key::{self, MultiIndex, Value}, ingress, coordinator::Coordinator, archive::Wrapped}; use crate::{actors::net, maybe_delay_between}; -use alto_types::{signed_tx::SignedTx, tx::Tx}; - -#[derive(Clone, Debug)] -pub struct Batch { - pub timestamp: SystemTime, - // TODO: store real transactions not just raws - pub txs: Vec>, - pub digest: H::Digest, -} - -impl Batch { - fn compute_digest(txs: &Vec>) -> H::Digest { - let mut hasher = H::new(); - - for tx in txs.iter() { - hasher.update(&tx.payload()); - } - - hasher.finalize() - } - - pub fn new(txs: Vec>, timestamp: SystemTime) -> Self { - let digest = Self::compute_digest(&txs); - - Self { - txs, - digest, - timestamp - } - } - - pub fn serialize(&self) -> Vec { - let mut bytes = Vec::new(); - bytes.put_u64(self.timestamp.epoch_millis()); - bytes.put_u64(self.txs.len() as u64); - for tx in self.txs.iter() { - bytes.put_u64(tx.size() as u64); - bytes.extend_from_slice(&tx.payload()); - } - bytes - } - - pub fn deserialize(mut bytes: &[u8]) -> Result { - use bytes::Buf; - // We expect at least 18 bytes for the header - if bytes.remaining() < 18 { - return Err(format!("not enough bytes for header")); - } - let timestamp = bytes.get_u64(); - let timestamp = SystemTime::UNIX_EPOCH + Duration::from_millis(timestamp); - - let tx_count = bytes.get_u64(); - let mut txs = Vec::with_capacity(tx_count as usize); - for _ in 0..tx_count { - // For each transaction, first read the size (u64). - if bytes.remaining() < 8 { - return Err(format!("not enough bytes for tx size")); - } - let tx_size = bytes.get_u64() as usize; - // Ensure there are enough bytes left. - if bytes.remaining() < tx_size { - return Err(format!("not enough bytes for tx payload, needed: {}, actual: {}", tx_size, bytes.remaining())); - } - // Extract tx_size bytes. - let tx_bytes = bytes.copy_to_bytes(tx_size); - txs.push(SignedTx::deserialize(&tx_bytes)?); - } - // Compute the digest from the transactions. - let digest = Self::compute_digest(&txs); - // Since serialize did not include accepted and timestamp, we set accepted to false - // and set timestamp to the current time. - Ok(Self { - timestamp, - txs, - digest, - }) - } - - pub fn contain_tx(&self, digest: &H::Digest) -> bool { - todo!() - // self.txs.iter().any(|tx| &tx.digest == digest) - } - - pub fn tx(&self, digest: &H::Digest) -> Option> { - // self.txs.iter().find(|tx| &tx.digest == digest).cloned() - todo!() - } -} - -#[derive(Clone, Debug)] -pub struct RawTransaction { - pub raw: Bytes, - - pub digest: H::Digest -} - -impl RawTransaction { - fn compute_digest(raw: &Bytes) -> H::Digest { - let mut hasher = H::new(); - hasher.update(&raw); - hasher.finalize() - } - - pub fn new(raw: Bytes) -> Self { - let digest = Self::compute_digest(&raw); - Self { - raw, - digest - } - } - - pub fn validate(&self) -> bool { - // TODO: implement validate here - true - } - - pub fn size(&self) -> u64 { - self.raw.len() as u64 - } -} - -impl From for RawTransaction { - fn from(value: net::actor::DummyTransaction) -> Self { - let raw = Bytes::from(value.payload); - RawTransaction::new(raw) - } -} +use alto_types::{signed_tx::SignedTx, tx::Tx, Batch}; pub enum Message { // mark batch as accepted by the netowrk through the broadcast protocol @@ -168,7 +42,7 @@ pub enum Message { }, GetTx { digest: H::Digest, - response: oneshot::Sender>> + response: oneshot::Sender>> }, GetBatch { digest: H::Digest, @@ -232,7 +106,7 @@ impl Mailbox { receiver.await.expect("failed to mark batches as consumed") } - pub async fn get_tx(&mut self, digest: H::Digest) -> Option> { + pub async fn get_tx(&mut self, digest: H::Digest) -> Option> { let (response, receiver) = oneshot::channel(); self.sender .send(Message::GetTx { digest, response }) diff --git a/chain/src/actors/mempool/mod.rs b/chain/src/actors/mempool/mod.rs index 762472e5..bb80906c 100644 --- a/chain/src/actors/mempool/mod.rs +++ b/chain/src/actors/mempool/mod.rs @@ -24,7 +24,7 @@ mod tests { use commonware_runtime::{deterministic::{Context, Executor}, Clock, Metrics, Runner, Spawner}; use futures::channel::mpsc; - use super::{ingress, mempool::{self, Mempool, RawTransaction}}; + use super::{ingress, mempool::{self, Mempool}}; type Registrations

= HashMap, Receiver

), diff --git a/chain/src/actors/net/actor.rs b/chain/src/actors/net/actor.rs index aeef046d..ff7029cc 100644 --- a/chain/src/actors/net/actor.rs +++ b/chain/src/actors/net/actor.rs @@ -21,7 +21,7 @@ use tokio::net::TcpListener; use tracing::{debug, event, Level, error}; use tracing_subscriber::fmt::format; -use crate::actors::mempool::mempool::{self, RawTransaction}; +use crate::actors::mempool::mempool; use super::ingress::{Mailbox, Message}; #[derive(Deserialize)] diff --git a/types/src/batch.rs b/types/src/batch.rs new file mode 100644 index 00000000..6cd79f8b --- /dev/null +++ b/types/src/batch.rs @@ -0,0 +1,94 @@ +use std::time::{Duration, SystemTime}; + +use bytes::BufMut; +use commonware_cryptography::Hasher; +use commonware_utils::SystemTimeExt; + +use crate::signed_tx::SignedTx; + +#[derive(Clone, Debug)] +pub struct Batch { + pub timestamp: SystemTime, + // TODO: store real transactions not just raws + pub txs: Vec>, + pub digest: H::Digest, +} + +impl Batch { + fn compute_digest(txs: &Vec>) -> H::Digest { + let mut hasher = H::new(); + + for tx in txs.iter() { + hasher.update(&tx.payload()); + } + + hasher.finalize() + } + + pub fn new(txs: Vec>, timestamp: SystemTime) -> Self { + let digest = Self::compute_digest(&txs); + + Self { + txs, + digest, + timestamp + } + } + + pub fn serialize(&self) -> Vec { + let mut bytes = Vec::new(); + bytes.put_u64(self.timestamp.epoch_millis()); + bytes.put_u64(self.txs.len() as u64); + for tx in self.txs.iter() { + bytes.put_u64(tx.size() as u64); + bytes.extend_from_slice(&tx.payload()); + } + bytes + } + + pub fn deserialize(mut bytes: &[u8]) -> Result { + use bytes::Buf; + // We expect at least 18 bytes for the header + if bytes.remaining() < 18 { + return Err(format!("not enough bytes for header")); + } + let timestamp = bytes.get_u64(); + let timestamp = SystemTime::UNIX_EPOCH + Duration::from_millis(timestamp); + + let tx_count = bytes.get_u64(); + let mut txs = Vec::with_capacity(tx_count as usize); + for _ in 0..tx_count { + // For each transaction, first read the size (u64). + if bytes.remaining() < 8 { + return Err(format!("not enough bytes for tx size")); + } + let tx_size = bytes.get_u64() as usize; + // Ensure there are enough bytes left. + if bytes.remaining() < tx_size { + return Err(format!("not enough bytes for tx payload, needed: {}, actual: {}", tx_size, bytes.remaining())); + } + // Extract tx_size bytes. + let tx_bytes = bytes.copy_to_bytes(tx_size); + txs.push(SignedTx::deserialize(&tx_bytes)?); + } + // Compute the digest from the transactions. + let digest = Self::compute_digest(&txs); + // Since serialize did not include accepted and timestamp, we set accepted to false + // and set timestamp to the current time. + Ok(Self { + timestamp, + txs, + digest, + }) + } + + pub fn contain_tx(&self, digest: &H::Digest) -> bool { + todo!() + // self.txs.iter().any(|tx| &tx.digest == digest) + } + + pub fn tx(&self, digest: &H::Digest) -> Option> { + // self.txs.iter().find(|tx| &tx.digest == digest).cloned() + todo!() + } +} \ No newline at end of file diff --git a/types/src/lib.rs b/types/src/lib.rs index 8d6ed98e..172c4185 100644 --- a/types/src/lib.rs +++ b/types/src/lib.rs @@ -1,8 +1,10 @@ //! Common types used throughout `alto`. mod block; +mod batch; pub use block::{Block, Finalized, Notarized}; +pub use batch::Batch; use commonware_cryptography::{Ed25519, Scheme}; use commonware_utils::SystemTimeExt; use std::time::SystemTime; From 6f38c51bc5164c74b8ae9500413a703f0d24ae6c Mon Sep 17 00:00:00 2001 From: bianyuanop Date: Thu, 3 Apr 2025 10:43:12 -0400 Subject: [PATCH 30/38] block references batches --- chain/src/actors/net/mod.rs | 2 +- types/src/block.rs | 94 ++++++++++++++++++------------------- 2 files changed, 47 insertions(+), 49 deletions(-) diff --git a/chain/src/actors/net/mod.rs b/chain/src/actors/net/mod.rs index c9c8524d..981d6e7c 100644 --- a/chain/src/actors/net/mod.rs +++ b/chain/src/actors/net/mod.rs @@ -109,7 +109,7 @@ mod tests { return; } } - } , + }, _ => { debug!("unknown message") } diff --git a/types/src/block.rs b/types/src/block.rs index 2205c108..bf57b984 100644 --- a/types/src/block.rs +++ b/types/src/block.rs @@ -1,7 +1,9 @@ +use std::hash::Hash; + use crate::signed_tx::{pack_signed_txs, unpack_signed_txs, SignedTx}; -use crate::{Finalization, Notarization}; +use crate::{batch, Batch, Finalization, Notarization}; use bytes::{Buf, BufMut}; -use commonware_cryptography::{bls12381::PublicKey, sha256, sha256::Digest, Hasher, Sha256}; +use commonware_cryptography::{bls12381::PublicKey, sha256, sha256::Digest as Sha256Digest, Hasher, Sha256}; use commonware_utils::{Array, SizedSerialize}; // @todo add state root, fee manager and results to the block struct. @@ -9,7 +11,7 @@ use commonware_utils::{Array, SizedSerialize}; #[derive(Clone, Debug)] pub struct Block { /// The parent block's digest. - pub parent: Digest, + pub parent: Sha256Digest, /// The height of the block in the blockchain. pub height: u64, @@ -17,52 +19,55 @@ pub struct Block { /// The timestamp of the block (in milliseconds since the Unix epoch). pub timestamp: u64, - /// The raw transactions in the block. - pub raw_txs: Vec, - /// The state root of the block. - pub state_root: Digest, + pub state_root: Sha256Digest, + + pub batches: Vec, + + _batches: Vec>, - txs: Vec>, /// Pre-computed digest of the block. - digest: Digest, + digest: Sha256Digest, } impl Block { fn compute_digest( - parent: &Digest, + parent: &Sha256Digest, height: u64, timestamp: u64, - raw_txs: Vec, - state_root: &Digest, - ) -> Digest { + batch_digests: &Vec, + state_root: &Sha256Digest, + ) -> Sha256Digest { let mut hasher = Sha256::new(); hasher.update(parent); hasher.update(&height.to_be_bytes()); hasher.update(×tamp.to_be_bytes()); - hasher.update(&raw_txs); + for digest in batch_digests.iter() { + hasher.update(digest); + } hasher.update(state_root); hasher.finalize() } pub fn new( - parent: Digest, + parent: Sha256Digest, height: u64, timestamp: u64, - txs: Vec>, - state_root: Digest, + batches: Vec>, + state_root: Sha256Digest, ) -> Self { // let mut txs = txs; // @todo this is packing txs in a block. - let raw_txs = pack_signed_txs(txs.clone()); - let digest = Self::compute_digest(&parent, height, timestamp, raw_txs.clone(), &state_root); + let batch_digests = batches.iter().map(|batch| batch.digest).collect(); + + let digest = Self::compute_digest(&parent, height, timestamp, &batch_digests, &state_root); Self { parent, height, timestamp, - raw_txs, state_root, - txs, + batches: batch_digests, + _batches: batches, digest, } } @@ -73,7 +78,10 @@ impl Block { bytes.put_u64(self.height); bytes.put_u64(self.timestamp); bytes.extend_from_slice(&self.state_root); - bytes.extend_from_slice(self.raw_txs.as_slice()); + bytes.put_u64(self.batches.len() as u64); + for digest in self.batches.iter() { + bytes.extend_from_slice(&digest); + } bytes } @@ -82,51 +90,41 @@ impl Block { // if bytes.len() != Self::SERIALIZED_LEN { // return None; // } - let parent = Digest::read_from(&mut bytes).ok()?; + let parent = Sha256Digest::read_from(&mut bytes).ok()?; let height = bytes.get_u64(); let timestamp = bytes.get_u64(); - let state_root = Digest::read_from(&mut bytes).ok()?; - let raw_txs = bytes.to_vec(); - let digest = Self::compute_digest(&parent, height, timestamp, raw_txs.clone(), &state_root); - let txs = unpack_signed_txs(raw_txs.clone()); + let state_root = Sha256Digest::read_from(&mut bytes).ok()?; + let num_batches = bytes.get_u64(); + let mut batch_digests = Vec::with_capacity(num_batches as usize); + for _ in 0..num_batches { + let batch_digest = Sha256Digest::read_from(&mut bytes).ok()?; + batch_digests.push(batch_digest); + } + + let digest = Self::compute_digest(&parent, height, timestamp, &batch_digests, &state_root); // Return block Some(Self { parent, height, timestamp, - raw_txs, state_root, - txs, + batches: batch_digests, + _batches: vec![], digest, }) } - pub fn digest(&self) -> Digest { + pub fn digest(&self) -> Sha256Digest { self.digest.clone() } - //todo check logic below - pub fn encode(&mut self) -> Vec { - let mut bytes: Vec = Vec::new(); - bytes.extend_from_slice(&self.parent); - bytes.extend_from_slice(&(self.height.to_be_bytes())); - bytes.extend_from_slice(&(self.timestamp.to_be_bytes())); - bytes.extend_from_slice(self.raw_txs.as_slice()); - bytes.extend_from_slice(&self.state_root); - // encoding signed txs - for tx in self.txs.iter_mut() { - bytes.extend_from_slice(&tx.encode()); - } - - // return encoded digest. - self.digest = sha256::hash(&bytes); - bytes - } } +// TODO: this should be an estimate of the size of one block since batches size can be variable impl SizedSerialize for Block { + // there is an assumed factor `5` multiply by Sha256Digest::SERIALIZED_LEN, which is an estimate how average many batches will be included in one block const SERIALIZED_LEN: usize = - Digest::SERIALIZED_LEN + u64::SERIALIZED_LEN + u64::SERIALIZED_LEN; + Sha256Digest::SERIALIZED_LEN + u64::SERIALIZED_LEN + u64::SERIALIZED_LEN + Sha256Digest::SERIALIZED_LEN + 5 * Sha256Digest::SERIALIZED_LEN; } pub struct Notarized { From 9c020fad9d9463050b18b6230e291521f3a075d0 Mon Sep 17 00:00:00 2001 From: rikoeldon <106416799+rikoeldon@users.noreply.github.com> Date: Thu, 3 Apr 2025 16:11:39 -0400 Subject: [PATCH 31/38] worked on todos in types --- types/src/account.rs | 3 +- types/src/batch.rs | 9 +++--- types/src/signed_tx.rs | 59 +++++++++++++++++++------------------ types/src/tx.rs | 35 +++++++++++++--------- types/src/units/msg.rs | 17 ++++++++--- types/src/units/transfer.rs | 16 +++++++--- 6 files changed, 83 insertions(+), 56 deletions(-) diff --git a/types/src/account.rs b/types/src/account.rs index fbe98dab..bcdb85f0 100644 --- a/types/src/account.rs +++ b/types/src/account.rs @@ -35,7 +35,8 @@ impl Codec for Account { fn write(&self, writer: &mut impl Writer) { // @rikoeldon I think we don't need to write account address into the state. // account address is part of the key. - writer.write_bytes(self.address.0.as_slice()); + // todo: might not need to write address in since address is already part of the key? + // writer.write_bytes(self.address.0.as_slice()); self.balance.write(writer); } diff --git a/types/src/batch.rs b/types/src/batch.rs index 6cd79f8b..c021eed8 100644 --- a/types/src/batch.rs +++ b/types/src/batch.rs @@ -1,3 +1,4 @@ +use std::hash::Hash; use std::time::{Duration, SystemTime}; use bytes::BufMut; @@ -60,7 +61,7 @@ impl Batch { for _ in 0..tx_count { // For each transaction, first read the size (u64). if bytes.remaining() < 8 { - return Err(format!("not enough bytes for tx size")); + return Err("not enough bytes for tx size".to_string()); } let tx_size = bytes.get_u64() as usize; // Ensure there are enough bytes left. @@ -83,12 +84,10 @@ impl Batch { } pub fn contain_tx(&self, digest: &H::Digest) -> bool { - todo!() - // self.txs.iter().any(|tx| &tx.digest == digest) + self.txs.iter().any(|tx| &tx.digest == digest) } pub fn tx(&self, digest: &H::Digest) -> Option> { - // self.txs.iter().find(|tx| &tx.digest == digest).cloned() - todo!() + self.txs.iter().find(|tx| &tx.hash() == digest).map_or(None, |tx| Some(tx.clone())) } } \ No newline at end of file diff --git a/types/src/signed_tx.rs b/types/src/signed_tx.rs index d9be6832..dfd3d5d0 100644 --- a/types/src/signed_tx.rs +++ b/types/src/signed_tx.rs @@ -1,6 +1,6 @@ use std::fmt::Debug; use std::hash::Hash; - +use commonware_codec::Codec; use crate::address::Address; use crate::tx::{Tx}; use crate::wallet::{Wallet, WalletMethods}; @@ -14,52 +14,60 @@ pub struct SignedTx { pub digest: H::Digest, pub_key: PublicKey, - address: Address, signature: Vec, + // cached is encode of SignedTx + cached_payload: Vec, } impl Debug for SignedTx { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - todo!() + // todo! do any of these need to be hex encoded? + f.debug_struct("SignedTx") + .field("tx", &self.tx) + .field("digest", &self.digest) + // .field("address", &self.address) + .field("signature", &self.signature) + .finish() } } impl SignedTx { - pub fn payload(&self) -> Vec { - todo!() - } - pub fn size(&self) -> usize { - todo!() + pub fn payload(&mut self) -> Vec { + if self.cached_payload.is_empty() { + self.cached_payload = self.encode(); + } + self.cached_payload.clone() } - pub fn serialize(&self) -> Vec { - todo!() + pub fn size(&mut self) -> usize { + self.payload().len() + } + + pub fn serialize(&mut self) -> Vec { + self.payload() } pub fn deserialize(raw: &[u8]) -> Result { - todo!() + Self::decode(raw) } pub fn validate(&self) -> bool { - todo!() + } pub fn random() -> Self { todo!() } - - - // @todo either have all fields initialized or none. fn new(tx: Tx, pub_key: PublicKey, signature: Vec) -> Self { let mut hasher = H::new(); let digest = hasher.finalize(); Self { tx, pub_key: pub_key.clone(), - address: Address::from_pub_key(&pub_key), signature: signature.clone(), + cached_payload: Vec::new(), digest } } @@ -82,10 +90,6 @@ impl SignedTx { self.pub_key.to_vec() } - fn address(&self) -> Address { - self.address.clone() - } - // @todo add syntactic checks. pub fn encode(&mut self) -> Vec { let mut bytes = Vec::new(); @@ -122,11 +126,11 @@ impl SignedTx { let digest = hasher.finalize(); Ok(SignedTx { - tx: tx.unwrap(), + tx: tx?, pub_key: public_key.clone(), - address: Address::from_pub_key(&public_key), signature: signature.to_vec(), - digest + digest, + cached_payload: Vec::new(), }) } @@ -140,9 +144,9 @@ impl SignedTx { SignedTx { tx: tx.clone(), signature: wallet.sign(&tx_data), - address: wallet.address(), pub_key: wallet.public_key(), - digest + digest, + cached_payload: Vec::new(), } } } @@ -219,15 +223,14 @@ mod tests { let mut origin_msg = SignedTx { tx, pub_key: pk, - address: Address::create_random_address(), signature: vec![], - digest + digest, + cached_payload: vec![], }; let encoded_bytes = origin_msg.encode(); assert_gt!(encoded_bytes.len(), 0); let decoded_msg = SignedTx::::decode(&encoded_bytes)?; assert_eq!(origin_msg.pub_key, decoded_msg.pub_key); - assert_eq!(origin_msg.address, decoded_msg.address); assert_eq!(origin_msg.signature, decoded_msg.signature); // @todo make helper to compare fields in tx and units. same issue when testing in tx.rs file. Ok(()) diff --git a/types/src/tx.rs b/types/src/tx.rs index 07009e03..6bd3580e 100644 --- a/types/src/tx.rs +++ b/types/src/tx.rs @@ -105,34 +105,44 @@ pub struct Tx { impl Debug for Tx { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - todo!() + // todo! do any of these need to be hex encoded? + f.debug_struct("SignedTx") + .field("timestamp", &self.timestamp) + .field("max_fee", &self.max_fee) + .field("priority_fee", &self.priority_fee) + .field("chain_id", &self.chain_id) + .field("units", &self.units) + .field("id", &self.id) + .field("digest", &self.digest) + .field("actor", &self.actor) + .finish() } } impl Tx { - pub fn digest(&self) -> H::Digest { - todo!() + pub fn digest(&mut self) -> H::Digest { + self.encode() } pub fn validate(&self) -> bool { todo!() } - pub fn payload(&self) -> Vec { - todo!() + pub fn payload(&mut self) -> Vec { + self.encode() } - pub fn serialize(&self) -> Vec { - todo!() + pub fn serialize(&mut self) -> Vec { + self.encode() } pub fn deserialize(raw: &[u8]) -> Result { - todo!() + Self::decode(raw) } // size of the payload - pub fn size(&self) -> usize { - todo!() + pub fn size(&mut self) -> usize { + self.payload().len() } pub fn random() -> Self { @@ -158,9 +168,7 @@ impl Tx { SignedTx::sign(self.clone(), wallet) } - //TODO: rename possibly as this method signature is more like a `new` for me instead of a from method - // the from method usually is taken by the From trait - fn from( + fn new_from_params( timestamp: u64, units: Vec>, priority_fee: u64, @@ -323,7 +331,6 @@ fn unpack_units(digest: &[u8]) -> Result>, String> { Ok(units) } -// @todo implement tests for encoding and decoding of tx. #[cfg(test)] mod tests { use super::*; diff --git a/types/src/units/msg.rs b/types/src/units/msg.rs index 39a54587..ebca636c 100644 --- a/types/src/units/msg.rs +++ b/types/src/units/msg.rs @@ -4,6 +4,7 @@ use crate::{ tx::{Unit, UnitContext, UnitType}, }; use std::any::Any; +use std::error::Error; // @todo couple SequencerMsg with DA. // and skip execution no-op. @@ -46,11 +47,19 @@ impl Unit for SequencerMsg { } // @todo introduce syntactic checks. - fn decode(&mut self, bytes: &[u8]) { - self.chain_id = u64::from_be_bytes(bytes[0..8].try_into().unwrap()); - self.from_address = Address::from_bytes(&bytes[8..40]).unwrap(); - let data_len = u64::from_be_bytes(bytes[40..48].try_into().unwrap()); + fn decode(&mut self, bytes: &[u8]) -> Result<(), Box>{ + let expected_size = size_of::() * 2 + size_of::

(); + if expected_size < bytes.len() { + return Err("Not enough data to decode sequencer message".into()); + } + self.chain_id = u64::from_be_bytes(bytes[0..8].try_into()?); + self.from_address = Address::from_bytes(&bytes[8..40])?; + let data_len:usize = u64::from_be_bytes(bytes[40..48].try_into()?) as usize; + if (expected_size + data_len) != bytes.len() { + return Err("Incorrect sequencer message length".into()); + } self.data = bytes[48..(48 + data_len as usize)].to_vec(); + Ok(()) } fn apply( diff --git a/types/src/units/transfer.rs b/types/src/units/transfer.rs index 22d949b7..dc4df2c1 100644 --- a/types/src/units/transfer.rs +++ b/types/src/units/transfer.rs @@ -62,13 +62,21 @@ impl Unit for Transfer { } // @todo introduce syntactic checks. - fn decode(&mut self, bytes: &[u8]) { - self.to_address = Address::from_bytes(&bytes[0..32]).unwrap(); - self.value = u64::from_be_bytes(bytes[32..40].try_into().unwrap()); - let memo_len = u64::from_be_bytes(bytes[40..48].try_into().unwrap()); + fn decode(&mut self, bytes: &[u8]) -> Result<(), Box> { + let expected_size = size_of::() * 2 + size_of::
(); + if expected_size < bytes.len() { + return Err("Not enough data to decode sequencer message".into()); + } + self.to_address = Address::from_bytes(&bytes[0..32])?; + self.value = u64::from_be_bytes(bytes[32..40].try_into()?); + let memo_len :usize = u64::from_be_bytes(bytes[40..48].try_into()?) as usize; + if (expected_size + memo_len) != bytes.len() { + return Err("Incorrect sequencer message length".into()); + } if memo_len > 0 { self.memo = bytes[48..(48 + memo_len as usize)].to_vec(); } + Ok(()) } fn apply( From 9142aeb6080e3235b21f7081596af41703d5b1bf Mon Sep 17 00:00:00 2001 From: rikoeldon <106416799+rikoeldon@users.noreply.github.com> Date: Thu, 3 Apr 2025 16:32:39 -0400 Subject: [PATCH 32/38] using OnceCell as solution to mut self --- types/src/signed_tx.rs | 28 ++++++++++++++++------------ types/src/tx.rs | 35 ++++++++++++++++++----------------- 2 files changed, 34 insertions(+), 29 deletions(-) diff --git a/types/src/signed_tx.rs b/types/src/signed_tx.rs index dfd3d5d0..a87c3ad4 100644 --- a/types/src/signed_tx.rs +++ b/types/src/signed_tx.rs @@ -6,6 +6,9 @@ use crate::tx::{Tx}; use crate::wallet::{Wallet, WalletMethods}; use crate::{PublicKey, Signature, TX_NAMESPACE}; use commonware_cryptography::{Ed25519, Hasher, Scheme}; +use std::cell::{Cell, OnceCell, RefCell}; +use std::iter::Once; + // this is sent by the user to the validators. #[derive(Clone)] pub struct SignedTx { @@ -16,7 +19,8 @@ pub struct SignedTx { pub_key: PublicKey, signature: Vec, // cached is encode of SignedTx - cached_payload: Vec, + // todo use OnceCell since payload encode is set once or use RefCell for mutable access? + cached_payload: OnceCell>, } impl Debug for SignedTx { @@ -33,11 +37,11 @@ impl Debug for SignedTx { impl SignedTx { - pub fn payload(&mut self) -> Vec { - if self.cached_payload.is_empty() { - self.cached_payload = self.encode(); + pub fn payload(&self) -> Vec { + if self.cached_payload.get().is_none() { + self.cached_payload.set(self.encode()).expect("could not set cache payload"); } - self.cached_payload.clone() + self.cached_payload.get().unwrap().to_vec() } pub fn size(&mut self) -> usize { @@ -53,7 +57,7 @@ impl SignedTx { } pub fn validate(&self) -> bool { - + todo!() } pub fn random() -> Self { @@ -67,7 +71,7 @@ impl SignedTx { tx, pub_key: pub_key.clone(), signature: signature.clone(), - cached_payload: Vec::new(), + cached_payload: OnceCell::new(), digest } } @@ -91,7 +95,7 @@ impl SignedTx { } // @todo add syntactic checks. - pub fn encode(&mut self) -> Vec { + pub fn encode(&self) -> Vec { let mut bytes = Vec::new(); let raw_tx = self.tx.encode(); @@ -130,7 +134,7 @@ impl SignedTx { pub_key: public_key.clone(), signature: signature.to_vec(), digest, - cached_payload: Vec::new(), + cached_payload: OnceCell::new(), }) } @@ -146,7 +150,7 @@ impl SignedTx { signature: wallet.sign(&tx_data), pub_key: wallet.public_key(), digest, - cached_payload: Vec::new(), + cached_payload: OnceCell::new(), } } } @@ -216,7 +220,7 @@ mod tests { chain_id, units: units.clone(), id, - digest: digest.to_vec(), + digest: OnceCell::from(digest.to_vec()), actor: Address::empty(), }; let digest = sha256::hash(&[0; 32]); @@ -225,7 +229,7 @@ mod tests { pub_key: pk, signature: vec![], digest, - cached_payload: vec![], + cached_payload: OnceCell::new(), }; let encoded_bytes = origin_msg.encode(); assert_gt!(encoded_bytes.len(), 0); diff --git a/types/src/tx.rs b/types/src/tx.rs index 6bd3580e..8fecd5a4 100644 --- a/types/src/tx.rs +++ b/types/src/tx.rs @@ -96,7 +96,7 @@ pub struct Tx { /// id is the transaction id. It is the hash of digest. pub id: H::Digest, /// digest is encoded tx. - pub digest: Vec, + pub digest: OnceCell>, /// address of the tx sender. wrap this in a better way. pub actor: Address, @@ -187,33 +187,34 @@ impl Tx { tx } - pub fn encode(&mut self) -> Vec { - if self.digest.is_empty() { - return self.digest.clone(); + pub fn encode(&self) -> Vec { + if self.digest.get().is_some() { + return self.digest.get().unwrap().to_vec(); } + let mut digest: Vec = Vec::new(); // pack tx timestamp. - self.digest.extend(self.timestamp.to_be_bytes()); + digest.extend(self.timestamp.to_be_bytes()); // pack max fee - self.digest.extend(self.max_fee.to_be_bytes()); + digest.extend(self.max_fee.to_be_bytes()); // pack priority fee - self.digest.extend(self.priority_fee.to_be_bytes()); + digest.extend(self.priority_fee.to_be_bytes()); // pack chain id - self.digest.extend(self.chain_id.to_be_bytes()); + digest.extend(self.chain_id.to_be_bytes()); // pack # of units. - self.digest.extend((self.units.len() as u64).to_be_bytes()); + digest.extend((self.units.len() as u64).to_be_bytes()); // pack individual units self.units.iter().for_each(|unit| { let unit_bytes = unit.encode(); // pack the unit type info. - self.digest.extend((unit.unit_type() as u8).to_be_bytes()); + digest.extend((unit.unit_type() as u8).to_be_bytes()); // pack len of individual unit. - self.digest.extend((unit_bytes.len() as u64).to_be_bytes()); + digest.extend((unit_bytes.len() as u64).to_be_bytes()); // pack individual unit. - self.digest.extend_from_slice(&unit_bytes); + digest.extend_from_slice(&unit_bytes); }); - + self.digest.set(digest).expect("cannot set digest"); // return encoded digest. - self.digest.clone() + self.digest.get().unwrap().to_vec() } pub fn decode(bytes: &[u8]) -> Result { @@ -221,7 +222,7 @@ impl Tx { return Err("Empty bytes".to_string()); } let mut tx = Self::default(); - tx.digest = bytes.to_vec(); // @todo ?? + tx.digest = OnceCell::from(bytes.to_vec()); // @todo ?? tx.timestamp = u64::from_be_bytes(bytes[0..8].try_into().unwrap()); tx.max_fee = u64::from_be_bytes(bytes[8..16].try_into().unwrap()); tx.priority_fee = u64::from_be_bytes(bytes[16..24].try_into().unwrap()); @@ -260,7 +261,7 @@ impl Default for Tx { priority_fee: 0, chain_id: 19517, id: hasher.finalize(), - digest: vec![], + digest: OnceCell::new(), actor: Address::empty(), } } @@ -360,7 +361,7 @@ mod tests { units: units.clone(), id, actor: Address::empty(), - digest: digest.to_vec(), + digest: OnceCell::from(digest.to_vec()), }; let encoded_bytes = origin_msg.encode(); assert_gt!(encoded_bytes.len(), 0); From af3673529ae480bec8660e875f744a1d1e2886c7 Mon Sep 17 00:00:00 2001 From: bianyuanop Date: Thu, 3 Apr 2025 18:00:39 -0400 Subject: [PATCH 33/38] compiling --- chain/src/actors/application/actor.rs | 2 +- chain/src/actors/net/actor.rs | 6 ++-- chain/src/actors/net/ingress.rs | 2 +- chain/src/actors/net/mod.rs | 3 ++ chain/src/actors/syncer/actor.rs | 12 +++++-- chain/src/bin/validator.rs | 21 +++++++++--- chain/src/engine.rs | 11 ++++--- chain/src/lib.rs | 47 +++++++++++++++++++++++---- types/src/batch.rs | 2 +- types/src/lib.rs | 1 + types/src/signed_tx.rs | 2 +- 11 files changed, 84 insertions(+), 25 deletions(-) diff --git a/chain/src/actors/application/actor.rs b/chain/src/actors/application/actor.rs index a7f7bbd1..adbaefec 100644 --- a/chain/src/actors/application/actor.rs +++ b/chain/src/actors/application/actor.rs @@ -3,7 +3,7 @@ use super::{ supervisor::Supervisor, Config, }; -use crate::actors::syncer; +use crate::actors::{net, syncer}; use alto_types::{Block, Finalization, Notarization, Seed}; use commonware_consensus::threshold_simplex::Prover; use commonware_cryptography::{hash, sha256::{self, Digest}, Hasher, Sha256}; diff --git a/chain/src/actors/net/actor.rs b/chain/src/actors/net/actor.rs index ff7029cc..ed08ac8c 100644 --- a/chain/src/actors/net/actor.rs +++ b/chain/src/actors/net/actor.rs @@ -46,14 +46,14 @@ struct AppState { } pub struct Config { - pub port: i32, + pub port: u16, pub mempool: mempool::Mailbox } pub struct Actor { context: R, - port: i32, + port: u16, listener: Option, pub router: Option, is_active: bool, @@ -108,7 +108,7 @@ impl Actor { ) } - pub async fn start(mut self) -> Handle<()> { + pub fn start(mut self) -> Handle<()> { self.context.spawn_ref()(self.run()) } diff --git a/chain/src/actors/net/ingress.rs b/chain/src/actors/net/ingress.rs index 00793e57..9b23a395 100644 --- a/chain/src/actors/net/ingress.rs +++ b/chain/src/actors/net/ingress.rs @@ -49,7 +49,7 @@ pub struct Mailbox { } impl Mailbox { - pub(super) fn new(sender: mpsc::Sender) -> Self { + pub fn new(sender: mpsc::Sender) -> Self { Self { sender } } diff --git a/chain/src/actors/net/mod.rs b/chain/src/actors/net/mod.rs index 981d6e7c..bc917379 100644 --- a/chain/src/actors/net/mod.rs +++ b/chain/src/actors/net/mod.rs @@ -1,3 +1,6 @@ +pub use ingress::{Mailbox, Message}; +pub use actor::{Actor, Config}; + pub mod actor; pub mod ingress; diff --git a/chain/src/actors/syncer/actor.rs b/chain/src/actors/syncer/actor.rs index 6ae3feab..b51f66ab 100644 --- a/chain/src/actors/syncer/actor.rs +++ b/chain/src/actors/syncer/actor.rs @@ -7,10 +7,10 @@ use super::{ Config, }; use crate::{ - actors::syncer::{ + actors::{net, syncer::{ handler, key::{self, MultiIndex, Value}, - }, + }}, Indexer, }; use alto_types::{Block, Finalization, Finalized, Notarized}; @@ -236,8 +236,9 @@ impl, I: Index impl Sender, impl Receiver, ), + mut net: net::Mailbox, ) -> Handle<()> { - self.context.spawn_ref()(self.run(broadcast_network, backfill_network)) + self.context.spawn_ref()(self.run(broadcast_network, backfill_network, net)) } /// Run the application actor. @@ -251,6 +252,7 @@ impl, I: Index impl Sender, impl Receiver, ), + mut net: net::Mailbox, ) { // Initialize resolver let coordinator = Coordinator::new(self.participants.clone()); @@ -321,6 +323,7 @@ impl, I: Index // In an application that maintains state, you would compute the state transition function here. + // Cancel any outstanding requests (by height and by digest) resolver .cancel(MultiIndex::new(Value::Finalized(next))) @@ -331,6 +334,9 @@ impl, I: Index .cancel(MultiIndex::new(Value::Digest(block.digest()))) .await; + // send block to net actor and try to broadcast block to any subscribers + net.broadcast_block(block).await; + // Update the latest indexed self.contiguous_height.set(next as i64); last_indexed = next; diff --git a/chain/src/bin/validator.rs b/chain/src/bin/validator.rs index e9566626..874c6318 100644 --- a/chain/src/bin/validator.rs +++ b/chain/src/bin/validator.rs @@ -1,4 +1,7 @@ -use alto_chain::{actors::mempool::{self, mempool::Mempool}, engine, Config}; +use alto_chain::{actors::{ + mempool::{self, mempool::Mempool}, + net +}, engine, Config}; use alto_client::Client; use alto_types::P2P_NAMESPACE; use axum::{routing::get, serve, Extension, Router}; @@ -30,6 +33,7 @@ use sysinfo::{Disks, System}; use tracing::{error, info, Level}; const SYSTEM_METRICS_REFRESH: Duration = Duration::from_secs(5); +const RPC_PORT: u16 = 7890; // const METRICS_PORT: u16 = 9090; const VOTER_CHANNEL: u32 = 0; @@ -276,8 +280,17 @@ fn main() { let broadcast_engine = broadcast_engine.start(mempool_broadcaster, mempool_ack_broadcaster); + let broadcaster_mempool_mailbox = mempool_mailbox.clone(); let mempool_handler = mempool.start(mempool_batch_broadcaster, mempool_backfill_broadcaster, broadcast_coordinator, mempool_app_mailbox); - let mempool_broadcast_app_handler = context.with_label("mempool_app").spawn(|_| mempool_application.run(broadcast_mailbox, mempool_mailbox)); + let mempool_broadcast_app_handler = context.with_label("mempool_app").spawn(|_| mempool_application.run(broadcast_mailbox, broadcaster_mempool_mailbox)); + + // Create net + let (net, net_mailbox) = net::Actor::new(context.with_label("net"), net::Config { + port: RPC_PORT, + mempool: mempool_mailbox.clone() + }); + + let net_handler = net.start(); // Create engine let config = engine::Config { @@ -304,7 +317,7 @@ fn main() { let engine = engine::Engine::new(context.with_label("engine"), config).await; // Start engine - let engine = engine.start(voter, resolver, broadcaster, backfiller); + let engine = engine.start(voter, resolver, broadcaster, backfiller, net_mailbox); // Start system metrics collector let system = context.with_label("system").spawn(|context| async move { @@ -377,7 +390,7 @@ fn main() { }); // Wait for any task to error - if let Err(e) = try_join_all(vec![p2p, engine, broadcast_engine, system, metrics, mempool_handler, mempool_broadcast_app_handler]).await { + if let Err(e) = try_join_all(vec![p2p, engine, broadcast_engine, system, metrics, mempool_handler, mempool_broadcast_app_handler, net_handler]).await { error!(?e, "task failed"); } }); diff --git a/chain/src/engine.rs b/chain/src/engine.rs index 10c52135..eae0ee45 100644 --- a/chain/src/engine.rs +++ b/chain/src/engine.rs @@ -1,14 +1,14 @@ use crate::{ - actors::{application, syncer}, + actors::{application, net, syncer}, Indexer, }; use alto_types::NAMESPACE; use commonware_consensus::threshold_simplex::{self, Engine as Consensus, Prover}; use commonware_cryptography::{ - bls12381::primitives::{group, poly::public, poly::Poly}, + bls12381::primitives::{group, poly::{public, Poly}}, ed25519::PublicKey, sha256::Digest, - Ed25519, Scheme, + Ed25519, Scheme, Sha256, }; use commonware_p2p::{Receiver, Sender}; use commonware_runtime::{Blob, Clock, Handle, Metrics, Spawner, Storage}; @@ -164,6 +164,7 @@ impl + Metri impl Sender, impl Receiver, ), + net: net::Mailbox, ) -> Handle<()> { self.context.clone().spawn(|_| { self.run( @@ -171,6 +172,7 @@ impl + Metri resolver_network, broadcast_network, backfill_network, + net ) }) } @@ -193,12 +195,13 @@ impl + Metri impl Sender, impl Receiver, ), + net: net::Mailbox, ) { // Start the application let application_handle = self.application.start(self.syncer_mailbox); // Start the syncer - let syncer_handle = self.syncer.start(broadcast_network, backfill_network); + let syncer_handle = self.syncer.start(broadcast_network, backfill_network, net); // Start consensus let consensus_handle = self.consensus.start(voter_network, resolver_network); diff --git a/chain/src/lib.rs b/chain/src/lib.rs index 02d3a20b..3f575aac 100644 --- a/chain/src/lib.rs +++ b/chain/src/lib.rs @@ -83,20 +83,23 @@ pub struct Config { #[cfg(test)] mod tests { + use crate::actors::{mempool::mempool, net}; + use super::*; use alto_types::{Finalized, Notarized, Seed}; use bls12381::primitives::poly; - use commonware_cryptography::{bls12381::dkg::ops, ed25519::PublicKey, Ed25519, Scheme}; + use commonware_cryptography::{bls12381::dkg::ops, ed25519::PublicKey, Ed25519, Scheme, Sha256}; use commonware_macros::test_traced; use commonware_p2p::simulated::{self, Link, Network, Oracle, Receiver, Sender}; use commonware_runtime::{ - deterministic::{self, Executor}, + deterministic::{self, Context, Executor}, Clock, Metrics, Runner, Spawner, }; use commonware_utils::quorum; use engine::{Config, Engine}; use governor::Quota; use rand::{rngs::StdRng, Rng, SeedableRng}; + use futures::{channel::mpsc, StreamExt}; use std::{ collections::{HashMap, HashSet}, num::NonZeroU32, @@ -185,6 +188,23 @@ mod tests { registrations } + async fn spawn_net( + context: Context, + ) -> net::Mailbox { + let (net_sender, mut net_receiver) = mpsc::channel(1024); + context.with_label("mock_net").spawn(async move |_| { + while let Some(msg) = net_receiver.next().await { + match msg { + net::Message::PublishBlock { block } => { + info!(?block, "received block from syncer") + } + } + } + }); + + net::Mailbox::new(net_sender) + } + /// Links (or unlinks) validators using the oracle. /// /// The `action` parameter determines the action (e.g. link, unlink) to take. @@ -296,8 +316,11 @@ mod tests { let (voter, resolver, broadcast, backfill) = registrations.remove(&public_key).unwrap(); + // Spawn mock net actor + let net = spawn_net(context.with_label("net")).await; + // Start engine - engine.start(voter, resolver, broadcast, backfill); + engine.start(voter, resolver, broadcast, backfill, net); } // Poll metrics @@ -455,8 +478,11 @@ mod tests { let (voter, resolver, broadcast, backfill) = registrations.remove(&public_key).unwrap(); + // Spawn mock net actor + let net = spawn_net(context.with_label("net")).await; + // Start engine - engine.start(voter, resolver, broadcast, backfill); + engine.start(voter, resolver, broadcast, backfill, net); } // Poll metrics @@ -538,8 +564,11 @@ mod tests { // Get networking let (voter, resolver, broadcast, backfill) = registrations.remove(&public_key).unwrap(); + // Spawn mock net actor + let net = spawn_net(context.with_label("net")).await; + // Start engine - engine.start(voter, resolver, broadcast, backfill); + engine.start(voter, resolver, broadcast, backfill, net); // Poll metrics loop { @@ -673,8 +702,10 @@ mod tests { let (voter, resolver, broadcast, backfill) = registrations.remove(&public_key).unwrap(); + // Spawn mock net actor + let net = spawn_net(context.with_label("net")).await; // Start engine - engine.start(voter, resolver, broadcast, backfill); + engine.start(voter, resolver, broadcast, backfill, net); } // Poll metrics @@ -818,8 +849,10 @@ mod tests { let (voter, resolver, broadcast, backfill) = registrations.remove(&public_key).unwrap(); + // Spawn mock net actor + let net = spawn_net(context.with_label("net")).await; // Start engine - engine.start(voter, resolver, broadcast, backfill); + engine.start(voter, resolver, broadcast, backfill, net); } // Poll metrics diff --git a/types/src/batch.rs b/types/src/batch.rs index c021eed8..b4966a6a 100644 --- a/types/src/batch.rs +++ b/types/src/batch.rs @@ -88,6 +88,6 @@ impl Batch { } pub fn tx(&self, digest: &H::Digest) -> Option> { - self.txs.iter().find(|tx| &tx.hash() == digest).map_or(None, |tx| Some(tx.clone())) + self.txs.iter().find(|tx| &tx.digest == digest).map_or(None, |tx| Some(tx.clone())) } } \ No newline at end of file diff --git a/types/src/lib.rs b/types/src/lib.rs index 172c4185..874e874f 100644 --- a/types/src/lib.rs +++ b/types/src/lib.rs @@ -37,6 +37,7 @@ type PublicKey = commonware_cryptography::ed25519::PublicKey; type PrivateKey = commonware_cryptography::ed25519::PrivateKey; type Signature = commonware_cryptography::ed25519::Signature; + pub fn create_test_keypair() -> (PublicKey, PrivateKey) { let mut rng = OsRng; // generates keypair using random number generator diff --git a/types/src/signed_tx.rs b/types/src/signed_tx.rs index a87c3ad4..7b214d75 100644 --- a/types/src/signed_tx.rs +++ b/types/src/signed_tx.rs @@ -44,7 +44,7 @@ impl SignedTx { self.cached_payload.get().unwrap().to_vec() } - pub fn size(&mut self) -> usize { + pub fn size(&self) -> usize { self.payload().len() } From 172a73454740156d53b2278e9f3e93f3c6039bee Mon Sep 17 00:00:00 2001 From: bianyuanop Date: Fri, 4 Apr 2025 13:27:14 -0400 Subject: [PATCH 34/38] unit registry & optimizations --- types/src/signed_tx.rs | 14 +- types/src/tx.rs | 292 ++++++++++++------------------------ types/src/units/mod.rs | 149 +++++++++++++++++- types/src/units/msg.rs | 79 +++++----- types/src/units/transfer.rs | 85 ++++++----- 5 files changed, 337 insertions(+), 282 deletions(-) diff --git a/types/src/signed_tx.rs b/types/src/signed_tx.rs index 7b214d75..00e4f37f 100644 --- a/types/src/signed_tx.rs +++ b/types/src/signed_tx.rs @@ -193,8 +193,8 @@ mod tests { use std::hash::Hash; use super::*; - use crate::tx::Unit; use crate::units::transfer::Transfer; + use crate::units::Unit; use crate::{create_test_keypair, curr_timestamp}; use commonware_cryptography::sha256::{self, Digest}; use commonware_cryptography::Sha256; @@ -206,23 +206,21 @@ mod tests { let max_fee = 100; let priority_fee = 75; let chain_id = 45205; - let transfer = Transfer::new(); + let transfer = Transfer::default(); let units: Vec> = vec![Box::new(transfer)]; let digest: [u8; 32] = [0; 32]; let id = Digest::from(digest.clone()); let (pk, sk) = create_test_keypair(); // TODO: the .encode call on next line gave error and said origin_msg needed to be mut? but why? // shouldn't encode be able to encode without changing the msg? - let tx = Tx:: { + let tx = Tx::::new( timestamp, max_fee, priority_fee, chain_id, - units: units.clone(), - id, - digest: OnceCell::from(digest.to_vec()), - actor: Address::empty(), - }; + Address::empty(), + units, + ); let digest = sha256::hash(&[0; 32]); let mut origin_msg = SignedTx { tx, diff --git a/types/src/tx.rs b/types/src/tx.rs index 8fecd5a4..220b9c92 100644 --- a/types/src/tx.rs +++ b/types/src/tx.rs @@ -1,79 +1,21 @@ +use bytes::{Buf, BufMut}; use commonware_cryptography::{hash, sha256, Hasher}; use commonware_cryptography::sha256::Digest; use core::hash; use std::any::Any; use std::cell::OnceCell; +use std::error::Error; use std::fmt::Debug; +use std::ops::Add; use crate::address::Address; use crate::signed_tx::SignedTx; use crate::state_view::StateView; -use crate::units; +use crate::units::{self, decode_units, encode_units, transfer, Unit, UnitType}; use crate::wallet::Wallet; use commonware_utils::SystemTimeExt; use std::time::SystemTime; -#[derive(Debug)] -pub enum UnitType { - Transfer, - SequencerMsg, -} - -impl TryFrom for UnitType { - type Error = String; - - fn try_from(value: u8) -> Result { - match value { - 0 => Ok(UnitType::Transfer), - 1 => Ok(UnitType::SequencerMsg), - _ => Err(format!("unknown unit type: {}", value)), - } - } -} - -pub struct UnitContext { - // timestamp of the tx. - pub timestamp: u64, - // chain id of the tx. - pub chain_id: u64, - // sender of the tx. - pub sender: Address, -} - -pub trait UnitClone { - fn clone_box(&self) -> Box; -} - -impl UnitClone for T -where - T: 'static + Unit + Clone, -{ - fn clone_box(&self) -> Box { - Box::new(self.clone()) - } -} - -// unit need to be simple and easy to be packed in the tx and executed by the vm. -pub trait Unit: UnitClone + Send + Sync + std::fmt::Debug { - fn unit_type(&self) -> UnitType; - fn encode(&self) -> Vec; - fn decode(&mut self, bytes: &[u8]); - - fn apply( - &self, - context: &UnitContext, - state: &mut Box<&mut dyn StateView>, - ) -> Result>, Box>; - - fn as_any(&self) -> &dyn Any; -} - -impl Clone for Box { - fn clone(&self) -> Box { - self.clone_box() - } -} - // TODO: add a commonware_cryptography::Hasher trait for Tx, and the digest should be labeled as H::Digest #[derive(Clone)] pub struct Tx { @@ -96,7 +38,7 @@ pub struct Tx { /// id is the transaction id. It is the hash of digest. pub id: H::Digest, /// digest is encoded tx. - pub digest: OnceCell>, + payload: OnceCell>, /// address of the tx sender. wrap this in a better way. pub actor: Address, @@ -113,26 +55,36 @@ impl Debug for Tx { .field("chain_id", &self.chain_id) .field("units", &self.units) .field("id", &self.id) - .field("digest", &self.digest) .field("actor", &self.actor) .finish() } } impl Tx { + fn compute_digest(&self) -> H::Digest { + if self.payload.get().is_none() { + let _ = self.serialize(); + } + + let payload = self.payload.get().expect("payload nil"); + let mut hasher = H::new(); + hasher.update(&payload); + hasher.finalize() + } + pub fn digest(&mut self) -> H::Digest { - self.encode() + self.id } pub fn validate(&self) -> bool { todo!() } - pub fn payload(&mut self) -> Vec { + pub fn payload(&self) -> Vec { self.encode() } - pub fn serialize(&mut self) -> Vec { + pub fn serialize(&self) -> Vec { self.encode() } @@ -149,13 +101,29 @@ impl Tx { todo!() } - fn new(units: Vec>, chain_id: u64) -> Self { - let mut tx = Self::default(); - tx.timestamp = SystemTime::now().epoch_millis(); - tx.units = units; - tx.chain_id = chain_id; + pub fn new( + timestamp: u64, + max_fee: u64, + priority_fee: u64, + chain_id: u64, + actor: Address, + units: Vec>, + ) -> Self { + let mut hasher = H::new(); + hasher.update(&[0; 32]); + + let mut tx = Self { + id: hasher.finalize(), + timestamp, + max_fee, + priority_fee, + chain_id, + units, + actor, + payload: OnceCell::new(), + }; + tx.id = tx.compute_digest(); - // do not encode and generate tx_id as Tx::new doesnot yet have priority fee and max fee. tx } @@ -188,55 +156,47 @@ impl Tx { } pub fn encode(&self) -> Vec { - if self.digest.get().is_some() { - return self.digest.get().unwrap().to_vec(); + if let Some(payload) = self.payload.get() { + // TODO: use ref counter instead of copying? + return payload.to_vec(); } - let mut digest: Vec = Vec::new(); + let mut payload: Vec = Vec::new(); // pack tx timestamp. - digest.extend(self.timestamp.to_be_bytes()); + payload.put_u64(self.timestamp); // pack max fee - digest.extend(self.max_fee.to_be_bytes()); + payload.put_u64(self.max_fee); // pack priority fee - digest.extend(self.priority_fee.to_be_bytes()); + payload.put_u64(self.priority_fee); // pack chain id - digest.extend(self.chain_id.to_be_bytes()); + payload.put_u64(self.chain_id); // pack # of units. - digest.extend((self.units.len() as u64).to_be_bytes()); - // pack individual units - self.units.iter().for_each(|unit| { - let unit_bytes = unit.encode(); - // pack the unit type info. - digest.extend((unit.unit_type() as u8).to_be_bytes()); - // pack len of individual unit. - digest.extend((unit_bytes.len() as u64).to_be_bytes()); - // pack individual unit. - digest.extend_from_slice(&unit_bytes); - }); - self.digest.set(digest).expect("cannot set digest"); - // return encoded digest. - self.digest.get().unwrap().to_vec() + let units_raw = encode_units(&self.units); + payload.extend_from_slice(&units_raw); + + // cache the payload + self.payload.set(payload).expect("cannot set payload"); + self.payload.get().expect("unable to get payload").to_vec() } - pub fn decode(bytes: &[u8]) -> Result { + pub fn decode(mut bytes: &[u8]) -> Result { if bytes.is_empty() { - return Err("Empty bytes".to_string()); + return Err("Empty bytes".into()); } + // Store the payload the compute digest let mut tx = Self::default(); - tx.digest = OnceCell::from(bytes.to_vec()); // @todo ?? - tx.timestamp = u64::from_be_bytes(bytes[0..8].try_into().unwrap()); - tx.max_fee = u64::from_be_bytes(bytes[8..16].try_into().unwrap()); - tx.priority_fee = u64::from_be_bytes(bytes[16..24].try_into().unwrap()); - tx.chain_id = u64::from_be_bytes(bytes[24..32].try_into().unwrap()); - let units = unpack_units(&bytes[32..]); - if units.is_err() { - return Err(format!("Failed to unpack units: {}", units.unwrap_err())); - } - tx.units = units?; - // generate tx id. - let mut hasher = H::new(); - hasher.update(&tx.digest()); - tx.id = hasher.finalize(); - // return transaction. + tx.payload = OnceCell::from(bytes.to_vec()); + let digest = tx.compute_digest(); + tx.id = digest; + + + tx.timestamp = bytes.get_u64(); + tx.max_fee = bytes.get_u64(); + tx.priority_fee = bytes.get_u64(); + tx.chain_id = bytes.get_u64(); + + let units = decode_units(bytes).unwrap().into(); + tx.units = units; + Ok(tx) } @@ -261,85 +221,22 @@ impl Default for Tx { priority_fee: 0, chain_id: 19517, id: hasher.finalize(), - digest: OnceCell::new(), + payload: OnceCell::new(), actor: Address::empty(), } } } -fn unpack_units(digest: &[u8]) -> Result>, String> { - let mut offset = 0; - - fn read_u8(input: &[u8], offset: &mut usize) -> Result { - if input.len() < *offset + 1 { - return Err("Unexpected end of input when reading u8".into()); - } - let val = input[*offset]; - *offset += 1; - Ok(val) - } - - fn read_u64(input: &[u8], offset: &mut usize) -> Result { - if input.len() < *offset + 8 { - return Err("Unexpected end of input when reading u64".into()); - } - let val = u64::from_be_bytes(input[*offset..*offset + 8].try_into().unwrap()); - *offset += 8; - Ok(val) - } - - fn read_bytes<'a>( - input: &'a [u8], - offset: &'a mut usize, - len: usize, - ) -> Result<&'a [u8], String> { - if input.len() < *offset + len { - return Err("Unexpected end of input when reading bytes".into()); - } - let bytes = &input[*offset..*offset + len]; - *offset += len; - Ok(bytes) - } - - let unit_count = read_u64(digest, &mut offset)?; - - let mut units: Vec> = Vec::with_capacity(unit_count as usize); - - for _ in 0..unit_count { - let unit_type = read_u8(digest, &mut offset)?; - let unit_len = read_u64(digest, &mut offset)?; - let unit_bytes = read_bytes(digest, &mut offset, unit_len as usize)?.to_vec(); - let unit_type = UnitType::try_from(unit_type); - if unit_type.is_err() { - return Err(format!("Invalid unit type: {}", unit_type.unwrap_err())); - } - let unit_type = unit_type?; - let unit: Box = match unit_type { - UnitType::Transfer => { - let mut transfer = units::transfer::Transfer::default(); - transfer.decode(&unit_bytes); - Box::new(transfer) - } - UnitType::SequencerMsg => { - let mut msg = units::msg::SequencerMsg::default(); - msg.decode(&unit_bytes); - Box::new(msg) - } - }; - units.push(unit); - } - - Ok(units) -} - #[cfg(test)] mod tests { use super::*; use crate::curr_timestamp; + use crate::units::msg::SequencerMsg; use crate::units::transfer::Transfer; use commonware_cryptography::Sha256; use more_asserts::assert_gt; use std::error::Error; + use std::vec; #[test] fn test_encode_decode() -> Result<(), Box> { @@ -347,26 +244,27 @@ mod tests { let max_fee = 100; let priority_fee = 75; let chain_id = 45205; - let transfer = Transfer::new(); - let units: Vec> = vec![Box::new(transfer)]; - let digest: [u8; 32] = [0; 32]; - let id = Digest::from(digest.clone()); + let transfer = Transfer::new(Address::empty(), 100, vec![34,10,43]); + let msg = SequencerMsg::new(10, Address::empty(), vec![1, 2, 3]); + let units: Vec> = vec![Box::new(transfer), Box::new(msg)]; // TODO: the .encode call on next line gave error and said origin_msg needed to be mut? but why? // shouldn't encode be able to encode without changing the msg? - let mut origin_msg = Tx:: { + let mut tx = Tx::::new( timestamp, max_fee, priority_fee, chain_id, - units: units.clone(), - id, - actor: Address::empty(), - digest: OnceCell::from(digest.to_vec()), - }; - let encoded_bytes = origin_msg.encode(); - assert_gt!(encoded_bytes.len(), 0); + Address::empty(), + units.clone(), + ); + let encoded_bytes = tx.encode(); + print!("encoded tx length: {}\n", encoded_bytes.len()); + let decoded_msg = Tx::::decode(&encoded_bytes)?; - let origin_transfer = origin_msg.units[0] + + assert_eq!(decoded_msg.payload().len(), encoded_bytes.len()); + + let origin_transfer = tx.units[0] .as_ref() .as_any() .downcast_ref::() @@ -378,15 +276,15 @@ mod tests { .downcast_ref::() .expect("Failed to downcast to Transfer"); - assert_eq!(origin_msg.timestamp, decoded_msg.timestamp); - assert_eq!(origin_msg.max_fee, decoded_msg.max_fee); - assert_eq!(origin_msg.priority_fee, decoded_msg.priority_fee); - assert_eq!(origin_msg.chain_id, decoded_msg.chain_id); - assert_eq!(origin_msg.id, decoded_msg.id); - assert_eq!(origin_msg.digest, decoded_msg.digest); + assert_eq!(tx.timestamp, decoded_msg.timestamp); + assert_eq!(tx.max_fee, decoded_msg.max_fee); + assert_eq!(tx.priority_fee, decoded_msg.priority_fee); + assert_eq!(tx.chain_id, decoded_msg.chain_id); + assert_eq!(tx.id, decoded_msg.id); + assert_eq!(tx.payload, decoded_msg.payload); // units - assert_eq!(origin_transfer.to_address, decode_transfer.to_address); + assert_eq!(origin_transfer.to, decode_transfer.to); assert_eq!(origin_transfer.value, decode_transfer.value); assert_eq!(origin_transfer.memo, decode_transfer.memo); Ok(()) diff --git a/types/src/units/mod.rs b/types/src/units/mod.rs index ab28bbd8..f7d1bda4 100644 --- a/types/src/units/mod.rs +++ b/types/src/units/mod.rs @@ -1,2 +1,149 @@ pub(crate) mod msg; -pub(crate) mod transfer; \ No newline at end of file +pub(crate) mod transfer; + +use std::any::Any; +use std::collections::HashMap; +use std::error::Error; +use std::sync::OnceLock; + +use bytes::{Buf, BufMut}; +use msg::SequencerMsg; +use transfer::Transfer; + +use crate::address::Address; +use crate::state_view::StateView; + +// A registry mapping each UnitType to its decode function. +static UNIT_DECODERS: OnceLock Result, Box>>> = OnceLock::new(); + +fn init_registry() -> HashMap Result, Box>> { + let mut m: HashMap Result, Box>> = HashMap::new(); + m.insert(UnitType::Transfer, Transfer::decode_box); + m.insert(UnitType::SequencerMsg, SequencerMsg::decode_box); + // register additional units as needed... + m +} + +pub fn decode_unit(unit_type: UnitType, data: &[u8]) -> Result, Box> { + let registry = UNIT_DECODERS.get_or_init(init_registry); + let Some(decode_fn) = registry.get(&unit_type) else { + return Err(format!("unsupported unit type {:?}", unit_type).into()) + }; + + decode_fn(data) +} + +pub fn encode_units(units: &Vec>) -> Vec { + let mut raw = Vec::new(); + raw.put_u64(units.len() as u64); + for unit in units.iter() { + // put unit length + unit type + unit data + let unit_raw = unit.encode(); + raw.put_u8(unit.unit_type() as u8); + raw.put_u64(unit_raw.len() as u64); + raw.extend_from_slice(&unit_raw); + } + + raw +} + +pub fn decode_units(mut raw: T) -> Result>, Box> { + if raw.remaining() < size_of::() { + return Err(format!("invalid raw units size: {}", raw.remaining()).into()) + } + + let num_units = raw.get_u64(); + let mut units = Vec::with_capacity(num_units as usize); + for _ in 0..num_units { + let unit_type = raw.get_u8(); + let raw_len = raw.get_u64(); + let unit_raw = raw.copy_to_bytes(raw_len as usize); + let unit = decode_unit(unit_type.try_into()?, &unit_raw)?; + units.push(unit); + } + + Ok(units) +} + +#[derive(Debug, PartialEq, Eq, Hash)] +pub enum UnitType { + Transfer = 1, + SequencerMsg, +} + +impl TryFrom for UnitType { + type Error = String; + + fn try_from(value: u8) -> Result { + match value { + 1 => Ok(UnitType::Transfer), + 2 => Ok(UnitType::SequencerMsg), + _ => Err(format!("unknown unit type: {}", value)), + } + } +} + +pub struct UnitContext { + // timestamp of the tx. + pub timestamp: u64, + // chain id of the tx. + pub chain_id: u64, + // sender of the tx. + pub sender: Address, +} + +pub trait UnitClone { + fn clone_box(&self) -> Box; +} + +impl UnitClone for T +where + T: 'static + Unit + Clone, +{ + fn clone_box(&self) -> Box { + Box::new(self.clone()) + } +} + +// unit need to be simple and easy to be packed in the tx and executed by the vm. +pub trait Unit: UnitClone + Send + Sync + std::fmt::Debug { + fn unit_type(&self) -> UnitType; + fn encode(&self) -> Vec; + + fn apply( + &self, + context: &UnitContext, + state: &mut Box<&mut dyn StateView>, + ) -> Result>, Box>; + + fn as_any(&self) -> &dyn Any; +} + +impl Clone for Box { + fn clone(&self) -> Box { + self.clone_box() + } +} + +#[cfg(test)] +mod tests { + use crate::address::Address; + + use super::{decode_units, encode_units, msg::SequencerMsg, transfer::Transfer, Unit}; + + #[test] + fn test_encode_decode() { + let mut units: Vec> = Vec::new(); + + let transfer = Transfer::new(Address::empty(), 100, vec![0, 1, 2, 3]); + let msg = SequencerMsg::new(0, Address::empty(), vec![3, 4, 5, 6]); + + units.push(Box::new(transfer)); + units.push(Box::new(msg)); + + let units_raw = encode_units(&units); + let decoded_untis = decode_units(units_raw.as_slice()).unwrap(); + + assert_eq!(units.len(), decoded_untis.len()) + } +} \ No newline at end of file diff --git a/types/src/units/msg.rs b/types/src/units/msg.rs index ebca636c..5097a782 100644 --- a/types/src/units/msg.rs +++ b/types/src/units/msg.rs @@ -1,27 +1,54 @@ +use bytes::{Buf, BufMut}; + use crate::{ address::Address, - state_view::StateView, - tx::{Unit, UnitContext, UnitType}, + state_view::StateView, ADDRESSLEN, }; -use std::any::Any; +use std::{any::Any, ops::Add}; use std::error::Error; +use super::{Unit, UnitContext, UnitType}; + // @todo couple SequencerMsg with DA. // and skip execution no-op. #[derive(Clone, Debug)] pub struct SequencerMsg { pub chain_id: u64, + pub from: Address, pub data: Vec, - pub from_address: Address, } impl SequencerMsg { - pub fn new() -> SequencerMsg { + pub fn new(chain_id: u64, from: Address, data: Vec) -> SequencerMsg { Self { - chain_id: 0, - data: Vec::new(), - from_address: Address::empty(), + chain_id, + data, + from, + } + } + + // @todo introduce syntactic checks. + pub fn decode(mut bytes: &[u8]) -> Result> { + // ChainID + DataLen + AddressLen + + let expected_size = size_of::()*2 + size_of::
(); + if bytes.len() < expected_size { + return Err(format!("Not enough data to decode sequencer message, wanted: >{}, actual: {}", expected_size, bytes.len()).into()); + } + + let chain_id = bytes.get_u64(); + let from= Address::from_bytes(&bytes.copy_to_bytes(ADDRESSLEN))?; + let data_len = bytes.get_u64() as usize; + if bytes.remaining() != data_len { + return Err(format!("Incorrect data length, wanted: {}, actual: {}", data_len, bytes.remaining()).into()); } + let data = bytes.copy_to_bytes(data_len).to_vec(); + + Ok(SequencerMsg { chain_id, data, from }) + } + + pub fn decode_box(mut bytes: &[u8]) -> Result, Box> { + let msg = Self::decode(bytes)?; + Ok(Box::new(msg)) } } @@ -32,36 +59,18 @@ impl Unit for SequencerMsg { fn encode(&self) -> Vec { let mut bytes: Vec = Vec::new(); - // data length is 8 bytes. - let data_len = self.data.len() as u64; // chain id length is 8 bytes.n store chain id. - bytes.extend(&self.chain_id.to_be_bytes()); + bytes.put_u64(self.chain_id); // address length is 32. store address. - bytes.extend_from_slice(self.from_address.as_slice()); + bytes.extend_from_slice(self.from.as_slice()); // store data length. - bytes.extend(data_len.to_be_bytes()); + bytes.put_u64(self.data.len() as u64); // store data. bytes.extend_from_slice(&self.data); bytes } - // @todo introduce syntactic checks. - fn decode(&mut self, bytes: &[u8]) -> Result<(), Box>{ - let expected_size = size_of::() * 2 + size_of::
(); - if expected_size < bytes.len() { - return Err("Not enough data to decode sequencer message".into()); - } - self.chain_id = u64::from_be_bytes(bytes[0..8].try_into()?); - self.from_address = Address::from_bytes(&bytes[8..40])?; - let data_len:usize = u64::from_be_bytes(bytes[40..48].try_into()?) as usize; - if (expected_size + data_len) != bytes.len() { - return Err("Incorrect sequencer message length".into()); - } - self.data = bytes[48..(48 + data_len as usize)].to_vec(); - Ok(()) - } - fn apply( &self, _: &UnitContext, @@ -80,7 +89,7 @@ impl Default for SequencerMsg { Self { chain_id: 0, data: vec![], - from_address: Address::empty(), + from: Address::empty(), } } } @@ -95,21 +104,19 @@ mod tests { fn test_encode_decode() -> Result<(), Box> { let chain_id = 4502; let data = vec![0xDE, 0xAD, 0xBE, 0xEF]; - let from_address = Address::create_random_address(); - let relayer_id = 1; + let from = Address::create_random_address(); let origin_msg = SequencerMsg { chain_id, data, - from_address, + from, }; let encoded_bytes = origin_msg.encode(); assert_gt!(encoded_bytes.len(), 0); - let mut decoded_msg = SequencerMsg::new(); - decoded_msg.decode(&encoded_bytes); + let decoded_msg = SequencerMsg::decode(&encoded_bytes)?; assert_eq!(origin_msg.chain_id, decoded_msg.chain_id); assert_eq!(origin_msg.data.len(), decoded_msg.data.len()); assert_eq!(origin_msg.data, decoded_msg.data); - assert_eq!(origin_msg.from_address, decoded_msg.from_address); + assert_eq!(origin_msg.from, decoded_msg.from); Ok(()) } } \ No newline at end of file diff --git a/types/src/units/transfer.rs b/types/src/units/transfer.rs index dc4df2c1..9ef01f53 100644 --- a/types/src/units/transfer.rs +++ b/types/src/units/transfer.rs @@ -1,24 +1,51 @@ -use crate::address::Address; +use bytes::{Buf, BufMut}; + +use crate::{address::Address, ADDRESSLEN}; use crate::state_view::StateView; -use crate::tx::{Unit, UnitContext, UnitType}; +use std::ops::Add; use std::{any::Any, error::Error, fmt::Display}; +use super::{Unit, UnitContext, UnitType}; + const MAX_MEMO_SIZE: usize = 256; #[derive(Debug, Clone)] pub struct Transfer { - pub to_address: Address, + pub to: Address, pub value: u64, pub memo: Vec, } impl Transfer { - pub fn new() -> Transfer { + pub fn new(to: Address, value: u64, memo: Vec) -> Transfer { Self { - to_address: Address::empty(), - value: 0, - memo: Vec::new(), + to, + value, + memo, + } + } + + pub fn decode(mut bytes: &[u8]) -> Result> { + // Value + MemoLen + AddressLen + + let expected_size = size_of::() * 2 + size_of::
(); + if bytes.len() < expected_size { + return Err("Not enough data to decode sequencer message".into()); } + + let to = Address::from_bytes(&bytes.copy_to_bytes(ADDRESSLEN))?; + let value = bytes.get_u64(); + let memo_len = bytes.get_u64() as usize; + if bytes.remaining() != memo_len { + return Err(format!("Incorrect memo length, wanted: {}, actual: {}", memo_len, bytes.remaining()).into()); + } + let memo = bytes.copy_to_bytes(memo_len).to_vec(); + Ok( Self { to, value, memo }) + } + + // @todo introduce syntactic checks. + pub fn decode_box(mut bytes: &[u8]) -> Result, Box> { + let transfer = Self::decode(bytes)?; + Ok(Box::new(transfer)) } } @@ -50,35 +77,14 @@ impl Unit for Transfer { fn encode(&self) -> Vec { let mut bytes = Vec::new(); - let memo_len = self.memo.len() as u64; - bytes.extend_from_slice(self.to_address.as_slice()); - bytes.extend(self.value.to_be_bytes()); - bytes.extend(memo_len.to_be_bytes()); - if memo_len > 0 { - bytes.extend_from_slice(&self.memo); - } + bytes.extend_from_slice(self.to.as_slice()); + bytes.extend(self.value.to_be_bytes()); + bytes.put_u64(self.memo.len() as u64); + bytes.extend_from_slice(&self.memo); bytes } - // @todo introduce syntactic checks. - fn decode(&mut self, bytes: &[u8]) -> Result<(), Box> { - let expected_size = size_of::() * 2 + size_of::
(); - if expected_size < bytes.len() { - return Err("Not enough data to decode sequencer message".into()); - } - self.to_address = Address::from_bytes(&bytes[0..32])?; - self.value = u64::from_be_bytes(bytes[32..40].try_into()?); - let memo_len :usize = u64::from_be_bytes(bytes[40..48].try_into()?) as usize; - if (expected_size + memo_len) != bytes.len() { - return Err("Incorrect sequencer message length".into()); - } - if memo_len > 0 { - self.memo = bytes[48..(48 + memo_len as usize)].to_vec(); - } - Ok(()) - } - fn apply( &self, context: &UnitContext, @@ -92,10 +98,10 @@ impl Unit for Transfer { if bal < self.value { return Err(TransferError::InsufficientFunds.into()); } - let receiver_bal = state.get_balance(&self.to_address).unwrap_or(0); + let receiver_bal = state.get_balance(&self.to).unwrap_or(0); if !state.set_balance(&context.sender, bal - self.value) - || !state.set_balance(&self.to_address, receiver_bal + self.value) + || !state.set_balance(&self.to, receiver_bal + self.value) { return Err(TransferError::StorageError.into()); } @@ -114,7 +120,7 @@ impl Unit for Transfer { impl Default for Transfer { fn default() -> Self { Self { - to_address: Address::empty(), + to: Address::empty(), value: 0, memo: vec![], } @@ -129,19 +135,18 @@ mod tests { #[test] fn test_encode_decode() -> Result<(), Box> { - let to_address = Address::create_random_address(); + let to = Address::create_random_address(); let value = 5; let memo = vec![0xDE, 0xAD, 0xBE, 0xEF]; let origin_msg = Transfer { - to_address, + to, value, memo, }; let encoded_bytes = origin_msg.encode(); assert_gt!(encoded_bytes.len(), 0); - let mut decoded_msg = Transfer::new(); - decoded_msg.decode(&encoded_bytes); - assert_eq!(origin_msg.to_address, decoded_msg.to_address); + let decoded_msg = Transfer::decode(&encoded_bytes)?; + assert_eq!(origin_msg.to, decoded_msg.to); assert_eq!(origin_msg.value, decoded_msg.value); assert_eq!(origin_msg.memo, decoded_msg.memo); Ok(()) From f6f2b73151d48ea7456ea1a894f1760ebc4b4342 Mon Sep 17 00:00:00 2001 From: rikoeldon <106416799+rikoeldon@users.noreply.github.com> Date: Fri, 4 Apr 2025 16:16:42 -0400 Subject: [PATCH 35/38] fixed conflicts and worked on validate/random in txs. --- types/src/lib.rs | 14 ++++++++++ types/src/signed_tx.rs | 59 +++++++++++++++++++++++++--------------- types/src/tx.rs | 61 +++++++++++++++++++++++++++--------------- types/src/wallet.rs | 4 +-- 4 files changed, 93 insertions(+), 45 deletions(-) diff --git a/types/src/lib.rs b/types/src/lib.rs index 874e874f..48fc720f 100644 --- a/types/src/lib.rs +++ b/types/src/lib.rs @@ -21,6 +21,7 @@ pub mod wallet; pub mod wasm; use rand::rngs::OsRng; +use crate::address::Address; // We don't use functions here to guard against silent changes. pub const NAMESPACE: &[u8] = b"_ALTO"; @@ -49,10 +50,23 @@ pub fn create_test_keypair() -> (PublicKey, PrivateKey) { (public_key, private_key) } +pub fn empty_pub_key() -> PublicKey { + PublicKey::try_from(&[0; 33]).unwrap() +} + pub fn curr_timestamp() -> u64 { SystemTime::now().epoch_millis() } +pub fn empty_signature() -> Signature { + Signature::try_from("").unwrap() +} + +pub fn random_signature() -> Signature { + let addr = Address::create_random_address(); + Signature::try_from(addr).unwrap() +} + #[cfg(test)] mod tests { use super::*; diff --git a/types/src/signed_tx.rs b/types/src/signed_tx.rs index 00e4f37f..bbeaed32 100644 --- a/types/src/signed_tx.rs +++ b/types/src/signed_tx.rs @@ -4,10 +4,13 @@ use commonware_codec::Codec; use crate::address::Address; use crate::tx::{Tx}; use crate::wallet::{Wallet, WalletMethods}; -use crate::{PublicKey, Signature, TX_NAMESPACE}; -use commonware_cryptography::{Ed25519, Hasher, Scheme}; -use std::cell::{Cell, OnceCell, RefCell}; -use std::iter::Once; +use crate::{create_test_keypair, curr_timestamp, PublicKey, Signature, TX_NAMESPACE}; +use commonware_cryptography::{Ed25519, Hasher, Scheme, Sha256}; +use std::cell::OnceCell; +use rand::rngs::OsRng; +use crate::units::msg::SequencerMsg; +use crate::units::transfer::Transfer; +use crate::units::Unit; // this is sent by the user to the validators. #[derive(Clone)] @@ -15,9 +18,8 @@ pub struct SignedTx { pub tx: Tx, pub digest: H::Digest, - pub_key: PublicKey, - signature: Vec, + signature: Signature, // cached is encode of SignedTx // todo use OnceCell since payload encode is set once or use RefCell for mutable access? cached_payload: OnceCell>, @@ -48,7 +50,7 @@ impl SignedTx { self.payload().len() } - pub fn serialize(&mut self) -> Vec { + pub fn serialize(&self) -> Vec { self.payload() } @@ -57,11 +59,24 @@ impl SignedTx { } pub fn validate(&self) -> bool { - todo!() + let tx_data = self.tx.encode(); + let signature = self.signature.clone(); + if signature.is_empty() { + return false; + } + let sender_pk = self.pub_key.clone(); + Ed25519::verify(Some(TX_NAMESPACE), &tx_data, &sender_pk, &signature) } - pub fn random() -> Self { - todo!() + pub fn random(&self) -> Self { + // create a tx + let tx = Tx::random(); + // generate keypair for sk usage + let (_, sk) = create_test_keypair(); + // create wallet + let wallet = Wallet::load(sk.as_ref()); + // sign tx to create a signed tx + Self::sign(tx, wallet) } fn new(tx: Tx, pub_key: PublicKey, signature: Vec) -> Self { @@ -70,28 +85,27 @@ impl SignedTx { Self { tx, pub_key: pub_key.clone(), - signature: signature.clone(), + signature: Signature::try_from(signature.clone()).unwrap(), cached_payload: OnceCell::new(), digest } } - fn verify(&mut self) -> bool { + fn verify(&self) -> bool { let tx_data = self.tx.encode(); - let signature = Signature::try_from(self.signature.as_slice()); - if signature.is_err() { + let signature = self.signature.clone(); + if signature.is_empty() { return false; } - let signature = signature.unwrap(); Ed25519::verify(Some(TX_NAMESPACE), &tx_data, &self.pub_key, &signature) } - fn signature(&self) -> Vec { + fn signature(&self) -> Signature { self.signature.clone() } - fn public_key(&self) -> Vec { - self.pub_key.to_vec() + fn public_key(&self) -> PublicKey { + self.pub_key.clone() } // @todo add syntactic checks. @@ -132,7 +146,7 @@ impl SignedTx { Ok(SignedTx { tx: tx?, pub_key: public_key.clone(), - signature: signature.to_vec(), + signature: Signature::try_from(signature.to_vec()).unwrap(), digest, cached_payload: OnceCell::new(), }) @@ -147,7 +161,7 @@ impl SignedTx { SignedTx { tx: tx.clone(), - signature: wallet.sign(&tx_data), + signature: Signature::try_from(wallet.sign(&tx_data)).unwrap(), pub_key: wallet.public_key(), digest, cached_payload: OnceCell::new(), @@ -195,7 +209,7 @@ mod tests { use super::*; use crate::units::transfer::Transfer; use crate::units::Unit; - use crate::{create_test_keypair, curr_timestamp}; + use crate::{create_test_keypair, curr_timestamp, random_signature}; use commonware_cryptography::sha256::{self, Digest}; use commonware_cryptography::Sha256; use more_asserts::assert_gt; @@ -210,6 +224,7 @@ mod tests { let units: Vec> = vec![Box::new(transfer)]; let digest: [u8; 32] = [0; 32]; let id = Digest::from(digest.clone()); + let sig = random_signature(); let (pk, sk) = create_test_keypair(); // TODO: the .encode call on next line gave error and said origin_msg needed to be mut? but why? // shouldn't encode be able to encode without changing the msg? @@ -225,7 +240,7 @@ mod tests { let mut origin_msg = SignedTx { tx, pub_key: pk, - signature: vec![], + signature: sig.clone(), digest, cached_payload: OnceCell::new(), }; diff --git a/types/src/tx.rs b/types/src/tx.rs index 220b9c92..5f3a9aaf 100644 --- a/types/src/tx.rs +++ b/types/src/tx.rs @@ -1,7 +1,6 @@ use bytes::{Buf, BufMut}; -use commonware_cryptography::{hash, sha256, Hasher}; +use commonware_cryptography::{hash, sha256, Hasher, Scheme}; use commonware_cryptography::sha256::Digest; -use core::hash; use std::any::Any; use std::cell::OnceCell; use std::error::Error; @@ -15,6 +14,10 @@ use crate::units::{self, decode_units, encode_units, transfer, Unit, UnitType}; use crate::wallet::Wallet; use commonware_utils::SystemTimeExt; use std::time::SystemTime; +use commonware_cryptography::ed25519::PublicKey; +use crate::curr_timestamp; +use crate::units::msg::SequencerMsg; +use crate::units::transfer::Transfer; // TODO: add a commonware_cryptography::Hasher trait for Tx, and the digest should be labeled as H::Digest #[derive(Clone)] @@ -34,13 +37,13 @@ pub struct Tx { /// units are fundamental unit of a tx. similar to actions. pub units: Vec>, - // TODO: the digest and the id should be the same thing, which is the hash of the tx payload - /// id is the transaction id. It is the hash of digest. + // TODO: the payload and the id should be the same thing, which is the hash of the tx payload + /// id is the transaction id. It is the hash of payload. pub id: H::Digest, - /// digest is encoded tx. - payload: OnceCell>, - /// address of the tx sender. wrap this in a better way. - pub actor: Address, + /// payload is encoded tx. + pub payload: OnceCell>, + /// address of the tx sender. + pub sender: Address, // TODO: add a payload referenced by OnceCell here possibly to avoid repeated serialization/deserialization } @@ -55,7 +58,8 @@ impl Debug for Tx { .field("chain_id", &self.chain_id) .field("units", &self.units) .field("id", &self.id) - .field("actor", &self.actor) + .field("payload", &self.payload) + .field("sender", &self.sender) .finish() } } @@ -78,7 +82,7 @@ impl Tx { pub fn validate(&self) -> bool { todo!() - } + } pub fn payload(&self) -> Vec { self.encode() @@ -98,7 +102,23 @@ impl Tx { } pub fn random() -> Self { - todo!() + // create a tx + let timestamp = curr_timestamp(); + let max_fee = 100; + let priority_fee = 75; + let chain_id = 45205; + let transfer = Transfer::new(Address::empty(), 100, vec![34,10,43]); + let msg = SequencerMsg::new(10, Address::empty(), vec![1, 2, 3]); + let units: Vec> = vec![Box::new(transfer), Box::new(msg)]; + + Tx::new( + timestamp, + max_fee, + priority_fee, + chain_id, + Address::empty(), + units.clone(), + ) } pub fn new( @@ -106,7 +126,7 @@ impl Tx { max_fee: u64, priority_fee: u64, chain_id: u64, - actor: Address, + sender: Address, units: Vec>, ) -> Self { let mut hasher = H::new(); @@ -119,7 +139,7 @@ impl Tx { priority_fee, chain_id, units, - actor, + sender, payload: OnceCell::new(), }; tx.id = tx.compute_digest(); @@ -142,7 +162,7 @@ impl Tx { priority_fee: u64, max_fee: u64, chain_id: u64, - actor: Address, + sender: Address, ) -> Self { let mut tx = Self::default(); tx.timestamp = timestamp; @@ -150,7 +170,7 @@ impl Tx { tx.max_fee = max_fee; tx.priority_fee = priority_fee; tx.chain_id = chain_id; - tx.actor = actor; + tx.sender = sender; tx.encode(); tx } @@ -200,12 +220,12 @@ impl Tx { Ok(tx) } - fn set_actor(&mut self, actor: Address) { - self.actor = actor; + fn set_sender(&mut self, sender: Address) { + self.sender = sender; } - fn actor(&self) -> Address { - self.actor.clone() + fn sender(&self) -> Address { + self.sender.clone() } } @@ -222,7 +242,7 @@ impl Default for Tx { chain_id: 19517, id: hasher.finalize(), payload: OnceCell::new(), - actor: Address::empty(), + sender: Address::empty(), } } } @@ -259,7 +279,6 @@ mod tests { ); let encoded_bytes = tx.encode(); print!("encoded tx length: {}\n", encoded_bytes.len()); - let decoded_msg = Tx::::decode(&encoded_bytes)?; assert_eq!(decoded_msg.payload().len(), encoded_bytes.len()); diff --git a/types/src/wallet.rs b/types/src/wallet.rs index beace065..0aba3844 100644 --- a/types/src/wallet.rs +++ b/types/src/wallet.rs @@ -40,7 +40,7 @@ pub trait WalletMethods { // create a new wallet using the given randomness. fn generate(r: &mut R) -> Self; // load signer from bytes rep of a private key and initialize the wallet. - fn load(&self, priv_key: &[u8]) -> Self; + fn load(priv_key: &[u8]) -> Self; // sign the given arbitary data with the private key of the wallet. fn sign(&mut self, data: &[u8]) -> Vec; // verify the signature of the given data with the public key of the wallet. @@ -71,7 +71,7 @@ impl WalletMethods for Wallet { } } - fn load(&self, priv_key: &[u8]) -> Self { + fn load(priv_key: &[u8]) -> Self { let private_key = PrivateKey::try_from(priv_key).expect("Invalid private key"); let signer = ::from(private_key).unwrap(); Self { From 76526d72e1d5381fed1b438492fde3978b011fe2 Mon Sep 17 00:00:00 2001 From: rikoeldon <106416799+rikoeldon@users.noreply.github.com> Date: Fri, 4 Apr 2025 16:31:00 -0400 Subject: [PATCH 36/38] removing sender from Tx --- types/src/tx.rs | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/types/src/tx.rs b/types/src/tx.rs index 5f3a9aaf..08c21d36 100644 --- a/types/src/tx.rs +++ b/types/src/tx.rs @@ -37,13 +37,10 @@ pub struct Tx { /// units are fundamental unit of a tx. similar to actions. pub units: Vec>, - // TODO: the payload and the id should be the same thing, which is the hash of the tx payload /// id is the transaction id. It is the hash of payload. pub id: H::Digest, /// payload is encoded tx. pub payload: OnceCell>, - /// address of the tx sender. - pub sender: Address, // TODO: add a payload referenced by OnceCell here possibly to avoid repeated serialization/deserialization } @@ -59,7 +56,6 @@ impl Debug for Tx { .field("units", &self.units) .field("id", &self.id) .field("payload", &self.payload) - .field("sender", &self.sender) .finish() } } @@ -139,7 +135,6 @@ impl Tx { priority_fee, chain_id, units, - sender, payload: OnceCell::new(), }; tx.id = tx.compute_digest(); @@ -170,7 +165,6 @@ impl Tx { tx.max_fee = max_fee; tx.priority_fee = priority_fee; tx.chain_id = chain_id; - tx.sender = sender; tx.encode(); tx } @@ -219,14 +213,6 @@ impl Tx { Ok(tx) } - - fn set_sender(&mut self, sender: Address) { - self.sender = sender; - } - - fn sender(&self) -> Address { - self.sender.clone() - } } impl Default for Tx { @@ -242,7 +228,6 @@ impl Default for Tx { chain_id: 19517, id: hasher.finalize(), payload: OnceCell::new(), - sender: Address::empty(), } } } From a8b8c049e3a3154198de299b2270b78e618c3b81 Mon Sep 17 00:00:00 2001 From: bianyuanop Date: Mon, 7 Apr 2025 08:55:53 -0400 Subject: [PATCH 37/38] test fixes --- chain/src/actors/net/actor.rs | 4 ++-- chain/src/actors/net/mod.rs | 19 ++++++++++--------- types/src/lib.rs | 32 ++++++++++++++++---------------- types/src/signed_tx.rs | 34 ++++++++++++++-------------------- types/src/tx.rs | 15 +++++++-------- 5 files changed, 49 insertions(+), 55 deletions(-) diff --git a/chain/src/actors/net/actor.rs b/chain/src/actors/net/actor.rs index ed08ac8c..9205a5fa 100644 --- a/chain/src/actors/net/actor.rs +++ b/chain/src/actors/net/actor.rs @@ -127,7 +127,7 @@ impl Actor { /// handles messages from other services within a node such as block messages async fn handle_message(state: SharedState, msg: Message) { - println!("handling msg {:?}", msg); + debug!("handling msg {:?}", msg); let block_listeners = state.read().unwrap().block_listeners.clone(); let clients = state.read().unwrap().clients.clone(); @@ -242,7 +242,7 @@ impl Actor { }; match SignedTx::::deserialize(&payload) { Ok(tx) => { - let success = mempool.submit_txs(vec![]).await[0]; + let success = mempool.submit_txs(vec![tx]).await[0]; if success { "submitted".to_string() } else { diff --git a/chain/src/actors/net/mod.rs b/chain/src/actors/net/mod.rs index bc917379..c6532869 100644 --- a/chain/src/actors/net/mod.rs +++ b/chain/src/actors/net/mod.rs @@ -8,7 +8,7 @@ pub mod ingress; mod tests { use core::panic; use std::time::Duration; - use alto_types::{Block}; + use alto_types::{signed_tx::SignedTx, Block}; use axum::{ body::Body, http::{Request, StatusCode} @@ -30,11 +30,11 @@ mod tests { #[test_traced] fn test_submit_tx() { - let (runner, mut context) = Executor::init(tokio::Config::default()); + let (runner, context) = Executor::init(tokio::Config::default()); runner.start(async move { let (mempool_sender, mut mempool_receiver) = mpsc::channel(1024); let mempool_mailbox: mempool::Mailbox = mempool::Mailbox::new(mempool_sender); - let (actor, mailbox) = Actor::new(context.with_label("router"), actor::Config { + let (actor, _) = Actor::new(context.with_label("router"), actor::Config { port: 7890, mempool: mempool_mailbox }); @@ -60,7 +60,8 @@ mod tests { // Note: the handler expects a payload (a String). Since GET requests normally have no body, // you might decide to pass the payload as a query parameter or in the body if that's what you intend. // Here, we'll assume the payload is extracted from the request body. - let payload = "test-tx"; + let tx = SignedTx::::random(); + let payload = tx.payload(); let request = Request::builder() .method("GET") .uri("/mempool/submit") @@ -87,10 +88,10 @@ mod tests { mempool: mempool_mailbox }); - debug!("starting router"); - let app_handler = actor.start().await; + println!("starting router"); + let app_handler = actor.start(); - debug!("launching ws client"); + println!("launching ws client"); // instantiate websocket client listening block let url = format!("ws://127.0.0.1:7890/ws"); let (ws_stream, response) = connect_async(url).await.expect("Failed to connect"); @@ -108,7 +109,7 @@ mod tests { let msg = Message::deserialize(&bin).unwrap(); match msg { Message::PublishBlock { block } => { - print!("received a block from server: {:?}", block); + println!("received a block from server: {:?}", block); return; } } @@ -121,7 +122,7 @@ mod tests { }); // send a dummy block - debug!("mock sending dummy block from another service"); + println!("mock sending dummy block from another service"); let parent_digest = sha256::hash(&[0; 32]); let height = 0; let timestamp = 1; diff --git a/types/src/lib.rs b/types/src/lib.rs index 48fc720f..928e44b5 100644 --- a/types/src/lib.rs +++ b/types/src/lib.rs @@ -50,22 +50,22 @@ pub fn create_test_keypair() -> (PublicKey, PrivateKey) { (public_key, private_key) } -pub fn empty_pub_key() -> PublicKey { - PublicKey::try_from(&[0; 33]).unwrap() -} - -pub fn curr_timestamp() -> u64 { - SystemTime::now().epoch_millis() -} - -pub fn empty_signature() -> Signature { - Signature::try_from("").unwrap() -} - -pub fn random_signature() -> Signature { - let addr = Address::create_random_address(); - Signature::try_from(addr).unwrap() -} +// pub fn empty_pub_key() -> PublicKey { +// PublicKey::try_from(&[0; 33]).unwrap() +// } + +// pub fn curr_timestamp() -> u64 { +// SystemTime::now().epoch_millis() +// } + +// pub fn empty_signature() -> Signature { +// Signature::try_from("").unwrap() +// } + +// pub fn random_signature() -> Signature { +// let addr = Address::create_random_address(); +// Signature::try_from(addr).unwrap() +// } #[cfg(test)] mod tests { diff --git a/types/src/signed_tx.rs b/types/src/signed_tx.rs index bbeaed32..592d0d3b 100644 --- a/types/src/signed_tx.rs +++ b/types/src/signed_tx.rs @@ -1,10 +1,11 @@ use std::fmt::Debug; use std::hash::Hash; +use std::sync::OnceLock; use commonware_codec::Codec; use crate::address::Address; use crate::tx::{Tx}; use crate::wallet::{Wallet, WalletMethods}; -use crate::{create_test_keypair, curr_timestamp, PublicKey, Signature, TX_NAMESPACE}; +use crate::{create_test_keypair, PublicKey, Signature, TX_NAMESPACE}; use commonware_cryptography::{Ed25519, Hasher, Scheme, Sha256}; use std::cell::OnceCell; use rand::rngs::OsRng; @@ -22,7 +23,7 @@ pub struct SignedTx { signature: Signature, // cached is encode of SignedTx // todo use OnceCell since payload encode is set once or use RefCell for mutable access? - cached_payload: OnceCell>, + cached_payload: OnceLock>, } impl Debug for SignedTx { @@ -68,7 +69,7 @@ impl SignedTx { Ed25519::verify(Some(TX_NAMESPACE), &tx_data, &sender_pk, &signature) } - pub fn random(&self) -> Self { + pub fn random() -> Self { // create a tx let tx = Tx::random(); // generate keypair for sk usage @@ -86,7 +87,7 @@ impl SignedTx { tx, pub_key: pub_key.clone(), signature: Signature::try_from(signature.clone()).unwrap(), - cached_payload: OnceCell::new(), + cached_payload: OnceLock::new(), digest } } @@ -124,6 +125,7 @@ impl SignedTx { // @todo add syntactic checks and use methods consume. fn decode(bytes: &[u8]) -> Result { // @todo this method seems untidy. + // TODO: size check let raw_tx_len = u64::from_be_bytes(bytes[0..8].try_into().unwrap()); let raw_tx = &bytes[8..8 + raw_tx_len as usize]; @@ -148,7 +150,7 @@ impl SignedTx { pub_key: public_key.clone(), signature: Signature::try_from(signature.to_vec()).unwrap(), digest, - cached_payload: OnceCell::new(), + cached_payload: OnceLock::new(), }) } @@ -164,7 +166,7 @@ impl SignedTx { signature: Signature::try_from(wallet.sign(&tx_data)).unwrap(), pub_key: wallet.public_key(), digest, - cached_payload: OnceCell::new(), + cached_payload: OnceLock::new(), } } } @@ -205,27 +207,26 @@ mod tests { use std::default; use std::error::Error; use std::hash::Hash; + use std::time::SystemTime; use super::*; use crate::units::transfer::Transfer; use crate::units::Unit; - use crate::{create_test_keypair, curr_timestamp, random_signature}; + use crate::{create_test_keypair}; use commonware_cryptography::sha256::{self, Digest}; use commonware_cryptography::Sha256; + use commonware_utils::SystemTimeExt; use more_asserts::assert_gt; #[test] fn test_encode_decode() -> Result<(), Box> { - let timestamp = curr_timestamp(); + let timestamp = SystemTime::now().epoch_millis(); let max_fee = 100; let priority_fee = 75; let chain_id = 45205; let transfer = Transfer::default(); let units: Vec> = vec![Box::new(transfer)]; - let digest: [u8; 32] = [0; 32]; - let id = Digest::from(digest.clone()); - let sig = random_signature(); - let (pk, sk) = create_test_keypair(); + let (_, sk) = create_test_keypair(); // TODO: the .encode call on next line gave error and said origin_msg needed to be mut? but why? // shouldn't encode be able to encode without changing the msg? let tx = Tx::::new( @@ -236,14 +237,7 @@ mod tests { Address::empty(), units, ); - let digest = sha256::hash(&[0; 32]); - let mut origin_msg = SignedTx { - tx, - pub_key: pk, - signature: sig.clone(), - digest, - cached_payload: OnceCell::new(), - }; + let origin_msg = SignedTx::sign(tx, Wallet::load(&sk)); let encoded_bytes = origin_msg.encode(); assert_gt!(encoded_bytes.len(), 0); let decoded_msg = SignedTx::::decode(&encoded_bytes)?; diff --git a/types/src/tx.rs b/types/src/tx.rs index 08c21d36..c01c0d56 100644 --- a/types/src/tx.rs +++ b/types/src/tx.rs @@ -6,6 +6,7 @@ use std::cell::OnceCell; use std::error::Error; use std::fmt::Debug; use std::ops::Add; +use std::sync::OnceLock; use crate::address::Address; use crate::signed_tx::SignedTx; @@ -15,7 +16,6 @@ use crate::wallet::Wallet; use commonware_utils::SystemTimeExt; use std::time::SystemTime; use commonware_cryptography::ed25519::PublicKey; -use crate::curr_timestamp; use crate::units::msg::SequencerMsg; use crate::units::transfer::Transfer; @@ -40,7 +40,7 @@ pub struct Tx { /// id is the transaction id. It is the hash of payload. pub id: H::Digest, /// payload is encoded tx. - pub payload: OnceCell>, + pub payload: OnceLock>, // TODO: add a payload referenced by OnceCell here possibly to avoid repeated serialization/deserialization } @@ -99,7 +99,7 @@ impl Tx { pub fn random() -> Self { // create a tx - let timestamp = curr_timestamp(); + let timestamp = SystemTime::now().epoch_millis(); let max_fee = 100; let priority_fee = 75; let chain_id = 45205; @@ -135,7 +135,7 @@ impl Tx { priority_fee, chain_id, units, - payload: OnceCell::new(), + payload: OnceLock::new(), }; tx.id = tx.compute_digest(); @@ -198,7 +198,7 @@ impl Tx { } // Store the payload the compute digest let mut tx = Self::default(); - tx.payload = OnceCell::from(bytes.to_vec()); + tx.payload = OnceLock::from(bytes.to_vec()); let digest = tx.compute_digest(); tx.id = digest; @@ -227,7 +227,7 @@ impl Default for Tx { priority_fee: 0, chain_id: 19517, id: hasher.finalize(), - payload: OnceCell::new(), + payload: OnceLock::new(), } } } @@ -235,7 +235,6 @@ impl Default for Tx { #[cfg(test)] mod tests { use super::*; - use crate::curr_timestamp; use crate::units::msg::SequencerMsg; use crate::units::transfer::Transfer; use commonware_cryptography::Sha256; @@ -245,7 +244,7 @@ mod tests { #[test] fn test_encode_decode() -> Result<(), Box> { - let timestamp = curr_timestamp(); + let timestamp = SystemTime::now().epoch_millis(); let max_fee = 100; let priority_fee = 75; let chain_id = 45205; From 2ffc4fb279f0f3643e38204c3a00f66cf27eb5ec Mon Sep 17 00:00:00 2001 From: bianyuanop Date: Mon, 7 Apr 2025 11:12:48 -0400 Subject: [PATCH 38/38] decoding fix & many unit tests --- chain/src/actors/net/actor.rs | 1 - chain/src/actors/net/mod.rs | 97 ++++++++++++++------- types/src/batch.rs | 60 +++++++++++-- types/src/block.rs | 26 +++--- types/src/lib.rs | 31 +++++++ types/src/signed_tx.rs | 153 ++++++++++++++++++++++++++++------ types/src/tx.rs | 104 +++++++++++++++++++++-- types/src/units/mod.rs | 69 +++++++++++++++ types/src/units/msg.rs | 10 ++- types/src/units/transfer.rs | 10 ++- 10 files changed, 477 insertions(+), 84 deletions(-) diff --git a/chain/src/actors/net/actor.rs b/chain/src/actors/net/actor.rs index 9205a5fa..a2abcf6f 100644 --- a/chain/src/actors/net/actor.rs +++ b/chain/src/actors/net/actor.rs @@ -19,7 +19,6 @@ use rand::Rng; use serde::Deserialize; use tokio::net::TcpListener; use tracing::{debug, event, Level, error}; -use tracing_subscriber::fmt::format; use crate::actors::mempool::mempool; diff --git a/chain/src/actors/net/mod.rs b/chain/src/actors/net/mod.rs index c6532869..9dd0919c 100644 --- a/chain/src/actors/net/mod.rs +++ b/chain/src/actors/net/mod.rs @@ -7,19 +7,18 @@ pub mod ingress; #[cfg(test)] mod tests { use core::panic; - use std::time::Duration; + use std::{ops::Deref, str, time::Duration}; use alto_types::{signed_tx::SignedTx, Block}; use axum::{ - body::Body, - http::{Request, StatusCode} + body::{to_bytes, Body}, + http::{Request, StatusCode}, Router }; use commonware_cryptography::{sha256, Sha256}; use commonware_macros::{test_async, test_traced}; - use commonware_runtime::{tokio::{self, Context, Executor}, Clock, Metrics, Runner, Spawner}; + use commonware_runtime::{tokio::{self, Context, Executor}, Clock, Handle, Metrics, Runner, Spawner}; use futures::{channel::mpsc, future::{join_all, try_join_all}, SinkExt, StreamExt}; use tokio_tungstenite::{connect_async, tungstenite::{client, Message as WsClientMessage}}; - use tower::ServiceExt; - use tracing_subscriber::{field::debug, fmt::format}; + use tower::{ServiceExt}; use alto_client::client_types::{WebsocketClientMessage}; use crate::actors::{mempool::mempool}; @@ -28,34 +27,39 @@ mod tests { use super::{actor::Actor, ingress::Message, actor::{self}}; use tracing::debug; + fn spawn_mempool(context: Context) -> (Handle<()>, Router) { + let (mempool_sender, mut mempool_receiver) = mpsc::channel(1024); + let mempool_mailbox: mempool::Mailbox = mempool::Mailbox::new(mempool_sender); + let (actor, _) = Actor::new(context.with_label("router"), actor::Config { + port: 7890, + mempool: mempool_mailbox + }); + + let Some(router) = actor.router else { + panic!("router not initalized"); + }; + + let handler = context.with_label("mock_mempool").spawn(async move |_| { + while let Some(msg) = mempool_receiver.next().await { + match msg { + mempool::Message::SubmitTxs { payload, response } => { + print!("received txs from rpc: {:?}", payload); + let _ = response.send(vec![true; payload.len()]); + return; + }, + _ => unreachable!() + } + } + }); + + (handler, router) + } + #[test_traced] fn test_submit_tx() { let (runner, context) = Executor::init(tokio::Config::default()); runner.start(async move { - let (mempool_sender, mut mempool_receiver) = mpsc::channel(1024); - let mempool_mailbox: mempool::Mailbox = mempool::Mailbox::new(mempool_sender); - let (actor, _) = Actor::new(context.with_label("router"), actor::Config { - port: 7890, - mempool: mempool_mailbox - }); - - let Some(router) = actor.router else { - panic!("router not initalized"); - }; - - let mempool_handler = context.with_label("mock_mempool").spawn(async move |_| { - while let Some(msg) = mempool_receiver.next().await { - match msg { - mempool::Message::SubmitTxs { payload, response } => { - print!("received txs from rpc: {:?}", payload); - let _ = response.send(vec![true; payload.len()]); - return; - }, - _ => unreachable!() - } - } - }); - + let (mempool_handler, router) = spawn_mempool(context); // Construct a GET request. // Note: the handler expects a payload (a String). Since GET requests normally have no body, // you might decide to pass the payload as a query parameter or in the body if that's what you intend. @@ -76,6 +80,39 @@ mod tests { let _ = try_join_all(vec![mempool_handler]).await; }) } + + #[test_traced] + fn test_submit_tx_wrong_format() { + let (runner, context) = Executor::init(tokio::Config::default()); + runner.start(async move { + let (mempool_handler, router) = spawn_mempool(context); + // Construct a GET request. + // Note: the handler expects a payload (a String). Since GET requests normally have no body, + // you might decide to pass the payload as a query parameter or in the body if that's what you intend. + // Here, we'll assume the payload is extracted from the request body. + let tx = b"test-tx"; + let request = Request::builder() + .method("GET") + .uri("/mempool/submit") + .body(Body::from(tx.to_vec())) + .unwrap(); + + // Send the request to the app. + let response = router.oneshot(request).await.unwrap(); + + // Check that the response status is OK. + assert_eq!(response.status(), StatusCode::OK); + let body = response.into_body(); + let body = to_bytes(body, 2*1024*1024).await.unwrap(); + let result = String::from_utf8(body.to_vec()).unwrap(); + print!("submission result {}\n", result); + + assert!(result.contains("failed to submit tx")); + + let _ = try_join_all(vec![mempool_handler]).await; + }) + } + #[test_traced] fn test_ws() { diff --git a/types/src/batch.rs b/types/src/batch.rs index b4966a6a..08e0db95 100644 --- a/types/src/batch.rs +++ b/types/src/batch.rs @@ -3,7 +3,8 @@ use std::time::{Duration, SystemTime}; use bytes::BufMut; use commonware_cryptography::Hasher; -use commonware_utils::SystemTimeExt; +use commonware_utils::{SizedSerialize, SystemTimeExt}; +use bytes::Buf; use crate::signed_tx::SignedTx; @@ -15,12 +16,16 @@ pub struct Batch { pub digest: H::Digest, } +impl SizedSerialize for Batch { + const SERIALIZED_LEN: usize = size_of::()*2; +} + impl Batch { fn compute_digest(txs: &Vec>) -> H::Digest { let mut hasher = H::new(); for tx in txs.iter() { - hasher.update(&tx.payload()); + hasher.update(&tx.digest()); } hasher.finalize() @@ -48,9 +53,7 @@ impl Batch { } pub fn deserialize(mut bytes: &[u8]) -> Result { - use bytes::Buf; - // We expect at least 18 bytes for the header - if bytes.remaining() < 18 { + if bytes.remaining() < Self::SERIALIZED_LEN { return Err(format!("not enough bytes for header")); } let timestamp = bytes.get_u64(); @@ -60,7 +63,7 @@ impl Batch { let mut txs = Vec::with_capacity(tx_count as usize); for _ in 0..tx_count { // For each transaction, first read the size (u64). - if bytes.remaining() < 8 { + if bytes.remaining() < size_of::() { return Err("not enough bytes for tx size".to_string()); } let tx_size = bytes.get_u64() as usize; @@ -72,6 +75,10 @@ impl Batch { let tx_bytes = bytes.copy_to_bytes(tx_size); txs.push(SignedTx::deserialize(&tx_bytes)?); } + if bytes.remaining() != 0 { + return Err(format!("left residue after decoding all the txs: {}", bytes.remaining())); + } + // Compute the digest from the transactions. let digest = Self::compute_digest(&txs); // Since serialize did not include accepted and timestamp, we set accepted to false @@ -90,4 +97,45 @@ impl Batch { pub fn tx(&self, digest: &H::Digest) -> Option> { self.txs.iter().find(|tx| &tx.digest == digest).map_or(None, |tx| Some(tx.clone())) } +} + +#[cfg(test)] +mod tests { + use std::time::SystemTime; + + use commonware_cryptography::Sha256; + use commonware_utils::SystemTimeExt; + + use crate::signed_tx::SignedTx; + + use super::Batch; + + #[test] + fn test_encode_decode() { + let tx = SignedTx::::random(); + let batch = Batch::new(vec![tx], SystemTime::now()); + let payload = batch.serialize(); + + let batch_recover = Batch::::deserialize(&payload).unwrap(); + + assert_eq!(batch.timestamp.epoch_millis(), batch_recover.timestamp.epoch_millis()); + assert_eq!(batch.txs.len(), batch_recover.txs.len()); + assert_eq!(batch.txs[0].digest, batch_recover.txs[0].digest); + assert_eq!(batch.txs[0].payload(), batch_recover.txs[0].payload()); + assert_eq!(batch.digest, batch_recover.digest); + } + + #[test] + fn test_residue() { + let tx = SignedTx::::random(); + let batch = Batch::new(vec![tx], SystemTime::now()); + let mut payload = batch.serialize(); + payload.push(10); + + let decode_result = Batch::::deserialize(&payload); + + let err_str = decode_result.map_err(|e| e.to_string()).err().unwrap(); + print!("{}\n", err_str); + assert!(err_str.contains("left residue after decoding all the txs")); + } } \ No newline at end of file diff --git a/types/src/block.rs b/types/src/block.rs index bf57b984..1c082dea 100644 --- a/types/src/block.rs +++ b/types/src/block.rs @@ -30,6 +30,12 @@ pub struct Block { digest: Sha256Digest, } +impl SizedSerialize for Block { + // parent + height + timestamp + state_root + len(batches) + const SERIALIZED_LEN: usize = + Sha256Digest::SERIALIZED_LEN + u64::SERIALIZED_LEN + u64::SERIALIZED_LEN + Sha256Digest::SERIALIZED_LEN + u64::SERIALIZED_LEN; +} + impl Block { fn compute_digest( parent: &Sha256Digest, @@ -87,9 +93,9 @@ impl Block { pub fn deserialize(mut bytes: &[u8]) -> Option { // Parse the block - // if bytes.len() != Self::SERIALIZED_LEN { - // return None; - // } + if bytes.len() < Self::SERIALIZED_LEN { + return None; + } let parent = Sha256Digest::read_from(&mut bytes).ok()?; let height = bytes.get_u64(); let timestamp = bytes.get_u64(); @@ -97,10 +103,17 @@ impl Block { let num_batches = bytes.get_u64(); let mut batch_digests = Vec::with_capacity(num_batches as usize); for _ in 0..num_batches { + if bytes.remaining() < Sha256Digest::SERIALIZED_LEN { + return None; + } let batch_digest = Sha256Digest::read_from(&mut bytes).ok()?; batch_digests.push(batch_digest); } + if bytes.remaining() != 0 { + return None; + } + let digest = Self::compute_digest(&parent, height, timestamp, &batch_digests, &state_root); // Return block @@ -120,13 +133,6 @@ impl Block { } } -// TODO: this should be an estimate of the size of one block since batches size can be variable -impl SizedSerialize for Block { - // there is an assumed factor `5` multiply by Sha256Digest::SERIALIZED_LEN, which is an estimate how average many batches will be included in one block - const SERIALIZED_LEN: usize = - Sha256Digest::SERIALIZED_LEN + u64::SERIALIZED_LEN + u64::SERIALIZED_LEN + Sha256Digest::SERIALIZED_LEN + 5 * Sha256Digest::SERIALIZED_LEN; -} - pub struct Notarized { pub proof: Notarization, pub block: Block, diff --git a/types/src/lib.rs b/types/src/lib.rs index 928e44b5..e4014763 100644 --- a/types/src/lib.rs +++ b/types/src/lib.rs @@ -71,6 +71,7 @@ pub fn create_test_keypair() -> (PublicKey, PrivateKey) { mod tests { use super::*; use commonware_cryptography::{hash, Bls12381, Scheme}; + use commonware_utils::SizedSerialize; use rand::{rngs::StdRng, SeedableRng}; #[test] @@ -260,4 +261,34 @@ mod tests { let result = Finalization::deserialize(None, &serialized); assert!(result.is_some()); } + + #[test] + fn test_block_residue() { + // Create network key + let mut rng = StdRng::seed_from_u64(0); + // Create block + let parent_digest = hash(&[0; 32]); + let height = 0; + let timestamp = 1; + let block = Block::new(parent_digest, height, timestamp, Vec::new(), [0; 32].into()); + let mut payload = block.serialize(); + payload.push(10); + + let block_recover = Block::deserialize(&payload); + assert!(block_recover.is_none()); + } + + #[test] + fn test_block_below_serialize_len() { + // Create block + let parent_digest = hash(&[0; 32]); + let height = 0; + let timestamp = 1; + let block = Block::new(parent_digest, height, timestamp, Vec::new(), [0; 32].into()); + let mut payload = block.serialize(); + payload.push(10); + + let block_recover = Block::deserialize(&payload[0..Block::SERIALIZED_LEN-1]); + assert!(block_recover.is_none()); + } } \ No newline at end of file diff --git a/types/src/signed_tx.rs b/types/src/signed_tx.rs index 592d0d3b..f7298b3d 100644 --- a/types/src/signed_tx.rs +++ b/types/src/signed_tx.rs @@ -1,7 +1,10 @@ use std::fmt::Debug; use std::hash::Hash; +use std::ops::Deref; use std::sync::OnceLock; +use bytes::{Buf, BufMut}; use commonware_codec::Codec; +use commonware_utils::SizedSerialize; use crate::address::Address; use crate::tx::{Tx}; use crate::wallet::{Wallet, WalletMethods}; @@ -16,16 +19,21 @@ use crate::units::Unit; // this is sent by the user to the validators. #[derive(Clone)] pub struct SignedTx { + pub_key: PublicKey, + signature: Signature, pub tx: Tx, pub digest: H::Digest, - pub_key: PublicKey, - signature: Signature, // cached is encode of SignedTx // todo use OnceCell since payload encode is set once or use RefCell for mutable access? cached_payload: OnceLock>, } +impl SizedSerialize for SignedTx { + // Pubkey + Sig + TxLen + Sizeof(Tx) + const SERIALIZED_LEN: usize = PublicKey::SERIALIZED_LEN + Signature::SERIALIZED_LEN + size_of::(); +} + impl Debug for SignedTx { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { // todo! do any of these need to be hex encoded? @@ -40,6 +48,10 @@ impl Debug for SignedTx { impl SignedTx { + pub fn digest(&self) -> H::Digest { + self.digest + } + pub fn payload(&self) -> Vec { if self.cached_payload.get().is_none() { self.cached_payload.set(self.encode()).expect("could not set cache payload"); @@ -113,44 +125,44 @@ impl SignedTx { pub fn encode(&self) -> Vec { let mut bytes = Vec::new(); - let raw_tx = self.tx.encode(); - let raw_tx_len = raw_tx.len() as u64; - bytes.extend(raw_tx_len.to_be_bytes()); - bytes.extend_from_slice(&raw_tx); bytes.extend_from_slice(&self.pub_key); bytes.extend_from_slice(&self.signature); + + let raw_tx = self.tx.payload(); + bytes.put_u64(raw_tx.len() as u64); + bytes.extend_from_slice(&raw_tx); bytes } // @todo add syntactic checks and use methods consume. - fn decode(bytes: &[u8]) -> Result { - // @todo this method seems untidy. - // TODO: size check - - let raw_tx_len = u64::from_be_bytes(bytes[0..8].try_into().unwrap()); - let raw_tx = &bytes[8..8 + raw_tx_len as usize]; - let pub_key = &bytes[8 + raw_tx_len as usize..8 + raw_tx_len as usize + 32]; - let signature = &bytes[8 + raw_tx_len as usize + 32..]; - let public_key = PublicKey::try_from(pub_key); - if public_key.is_err() { - return Err(public_key.unwrap_err().to_string()); + fn decode(mut bytes: &[u8]) -> Result { + let payload = bytes.to_vec(); + + if bytes.len() < Self::SERIALIZED_LEN { + return Err(format!("bytes len: {} below min size: {}", bytes.len(), Self::SERIALIZED_LEN)) } - let public_key = public_key.unwrap(); - let tx = Tx::decode(raw_tx); - if tx.is_err() { - return Err(tx.unwrap_err()); + + let pub_key = PublicKey::try_from(bytes.copy_to_bytes(PublicKey::SERIALIZED_LEN).deref()).map_err(|e| stringify!(e))?; + let signature = Signature::try_from(bytes.copy_to_bytes(Signature::SERIALIZED_LEN).deref()).map_err(|e| stringify!(e))?; + + let raw_tx_len = bytes.get_u64(); + if bytes.remaining() != raw_tx_len as usize { + return Err(format!("remaining bytes length not equal to tx len, wanted: {}, actual: {}", raw_tx_len, bytes.remaining())) } + let raw_tx = bytes.copy_to_bytes(raw_tx_len as usize); + let tx = Tx::decode(&raw_tx)?; + let mut hasher = H::new(); - hasher.update(raw_tx); + hasher.update(&raw_tx); let digest = hasher.finalize(); Ok(SignedTx { - tx: tx?, - pub_key: public_key.clone(), - signature: Signature::try_from(signature.to_vec()).unwrap(), + tx, + pub_key, + signature, digest, - cached_payload: OnceLock::new(), + cached_payload: OnceLock::from(payload), }) } @@ -246,4 +258,93 @@ mod tests { // @todo make helper to compare fields in tx and units. same issue when testing in tx.rs file. Ok(()) } + + #[test] + fn test_insufficient_bytes() { + let timestamp = SystemTime::now().epoch_millis(); + let max_fee = 100; + let priority_fee = 75; + let chain_id = 45205; + let transfer = Transfer::default(); + let units: Vec> = vec![Box::new(transfer)]; + let (_, sk) = create_test_keypair(); + // TODO: the .encode call on next line gave error and said origin_msg needed to be mut? but why? + // shouldn't encode be able to encode without changing the msg? + let tx = Tx::::new( + timestamp, + max_fee, + priority_fee, + chain_id, + Address::empty(), + units, + ); + let origin_msg = SignedTx::sign(tx, Wallet::load(&sk)); + let encoded_bytes = origin_msg.encode(); + assert_gt!(encoded_bytes.len(), 0); + let decode_result = SignedTx::::decode(&encoded_bytes[0..encoded_bytes.len()-10]); + + let err_str = decode_result.map_err(|e| e.to_string()).err().unwrap(); + print!("{}\n", err_str); + assert!(err_str.contains("remaining bytes length not equal to tx len")); + } + + #[test] + fn test_below_serialize_len() { + let timestamp = SystemTime::now().epoch_millis(); + let max_fee = 100; + let priority_fee = 75; + let chain_id = 45205; + let transfer = Transfer::default(); + let units: Vec> = vec![Box::new(transfer)]; + let (_, sk) = create_test_keypair(); + // TODO: the .encode call on next line gave error and said origin_msg needed to be mut? but why? + // shouldn't encode be able to encode without changing the msg? + let tx = Tx::::new( + timestamp, + max_fee, + priority_fee, + chain_id, + Address::empty(), + units, + ); + let origin_msg = SignedTx::sign(tx, Wallet::load(&sk)); + let encoded_bytes = origin_msg.encode(); + assert_gt!(encoded_bytes.len(), 0); + let decode_result = SignedTx::::decode(&encoded_bytes[0..SignedTx::::SERIALIZED_LEN-1]); + + let err_str = decode_result.map_err(|e| e.to_string()).err().unwrap(); + print!("{}\n", err_str); + assert!(err_str.contains("below min size")); + } + + #[test] + fn test_residue() { + let timestamp = SystemTime::now().epoch_millis(); + let max_fee = 100; + let priority_fee = 75; + let chain_id = 45205; + let transfer = Transfer::default(); + let units: Vec> = vec![Box::new(transfer)]; + let (_, sk) = create_test_keypair(); + // TODO: the .encode call on next line gave error and said origin_msg needed to be mut? but why? + // shouldn't encode be able to encode without changing the msg? + let tx = Tx::::new( + timestamp, + max_fee, + priority_fee, + chain_id, + Address::empty(), + units, + ); + let origin_msg = SignedTx::sign(tx, Wallet::load(&sk)); + let mut encoded_bytes = origin_msg.encode(); + encoded_bytes.push(10); + + assert_gt!(encoded_bytes.len(), 0); + let decode_result = SignedTx::::decode(&encoded_bytes); + + let err_str = decode_result.map_err(|e| e.to_string()).err().unwrap(); + print!("{}\n", err_str); + assert!(err_str.contains("remaining bytes length not equal to tx len")); + } } \ No newline at end of file diff --git a/types/src/tx.rs b/types/src/tx.rs index c01c0d56..f4151dd0 100644 --- a/types/src/tx.rs +++ b/types/src/tx.rs @@ -13,7 +13,7 @@ use crate::signed_tx::SignedTx; use crate::state_view::StateView; use crate::units::{self, decode_units, encode_units, transfer, Unit, UnitType}; use crate::wallet::Wallet; -use commonware_utils::SystemTimeExt; +use commonware_utils::{SizedSerialize, SystemTimeExt}; use std::time::SystemTime; use commonware_cryptography::ed25519::PublicKey; use crate::units::msg::SequencerMsg; @@ -45,6 +45,12 @@ pub struct Tx { // TODO: add a payload referenced by OnceCell here possibly to avoid repeated serialization/deserialization } +/// The minimal length for a tx +impl SizedSerialize for Tx { + // timestamp + max_fee + priority_fee + chain_id + len(units) + sizeof(Units) + const SERIALIZED_LEN: usize = size_of::()*5; +} + impl Debug for Tx { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { // todo! do any of these need to be hex encoded? @@ -193,22 +199,22 @@ impl Tx { } pub fn decode(mut bytes: &[u8]) -> Result { - if bytes.is_empty() { - return Err("Empty bytes".into()); + if bytes.len() < Self::SERIALIZED_LEN { + return Err(format!("bytes length: {} below min: {}", bytes.len(), Self::SERIALIZED_LEN).into()); } + // Store the payload the compute digest let mut tx = Self::default(); tx.payload = OnceLock::from(bytes.to_vec()); let digest = tx.compute_digest(); tx.id = digest; - tx.timestamp = bytes.get_u64(); tx.max_fee = bytes.get_u64(); tx.priority_fee = bytes.get_u64(); tx.chain_id = bytes.get_u64(); - let units = decode_units(bytes).unwrap().into(); + let units = decode_units(bytes).map_err(|e| e.to_string())?; tx.units = units; Ok(tx) @@ -292,4 +298,92 @@ mod tests { assert_eq!(origin_transfer.memo, decode_transfer.memo); Ok(()) } + + #[test] + fn test_insufficient_bytes() { + let timestamp = SystemTime::now().epoch_millis(); + let max_fee = 100; + let priority_fee = 75; + let chain_id = 45205; + let transfer = Transfer::new(Address::empty(), 100, vec![34,10,43]); + let msg = SequencerMsg::new(10, Address::empty(), vec![1, 2, 3]); + let units: Vec> = vec![Box::new(transfer), Box::new(msg)]; + // TODO: the .encode call on next line gave error and said origin_msg needed to be mut? but why? + // shouldn't encode be able to encode without changing the msg? + let mut tx = Tx::::new( + timestamp, + max_fee, + priority_fee, + chain_id, + Address::empty(), + units.clone(), + ); + let encoded_bytes = tx.encode(); + print!("encoded tx length: {}\n", encoded_bytes.len()); + let decode_result = Tx::::decode(&encoded_bytes[0..&encoded_bytes.len() - 10]); + + let err_str = decode_result.map_err(|e| e.to_string()).err().unwrap(); + print!("{}\n", err_str); + assert!(err_str.contains("remaining bytes invalid to decode a unit")); + } + + #[test] + fn test_excessive_bytes() { + let timestamp = SystemTime::now().epoch_millis(); + let max_fee = 100; + let priority_fee = 75; + let chain_id = 45205; + let transfer = Transfer::new(Address::empty(), 100, vec![34,10,43]); + let msg = SequencerMsg::new(10, Address::empty(), vec![1, 2, 3]); + let units: Vec> = vec![Box::new(transfer), Box::new(msg)]; + // TODO: the .encode call on next line gave error and said origin_msg needed to be mut? but why? + // shouldn't encode be able to encode without changing the msg? + let mut tx = Tx::::new( + timestamp, + max_fee, + priority_fee, + chain_id, + Address::empty(), + units.clone(), + ); + let mut encoded_bytes = tx.encode(); + encoded_bytes.push(10); + print!("encoded tx length: {}\n", encoded_bytes.len()); + let decode_result = Tx::::decode(&encoded_bytes); + + let err_str = decode_result.map_err(|e| e.to_string()).err().unwrap(); + print!("{}\n", err_str); + assert!(err_str.contains("left residue after decoding all the units")); + } + + + #[test] + fn test_below_serialize_len() { + let timestamp = SystemTime::now().epoch_millis(); + let max_fee = 100; + let priority_fee = 75; + let chain_id = 45205; + let transfer = Transfer::new(Address::empty(), 100, vec![34,10,43]); + let msg = SequencerMsg::new(10, Address::empty(), vec![1, 2, 3]); + let units: Vec> = vec![Box::new(transfer), Box::new(msg)]; + // TODO: the .encode call on next line gave error and said origin_msg needed to be mut? but why? + // shouldn't encode be able to encode without changing the msg? + let mut tx = Tx::::new( + timestamp, + max_fee, + priority_fee, + chain_id, + Address::empty(), + units.clone(), + ); + let mut encoded_bytes = tx.encode(); + encoded_bytes.push(10); + print!("encoded tx length: {}\n", encoded_bytes.len()); + let decode_result = Tx::::decode(&encoded_bytes[0..Tx::::SERIALIZED_LEN-1]); + + let err_str = decode_result.map_err(|e| e.to_string()).err().unwrap(); + print!("{}\n", err_str); + assert!(err_str.contains("below min")); + } + } \ No newline at end of file diff --git a/types/src/units/mod.rs b/types/src/units/mod.rs index f7d1bda4..154cd443 100644 --- a/types/src/units/mod.rs +++ b/types/src/units/mod.rs @@ -55,13 +55,24 @@ pub fn decode_units(mut raw: T) -> Result>, Box() + size_of::() { + return Err(format!("remaining bytes invalid to form a unit header: {}", raw.remaining()).into()) + } + let unit_type = raw.get_u8(); let raw_len = raw.get_u64(); + if raw.remaining() < raw_len as usize { + return Err(format!("remaining bytes invalid to decode a unit, wanted: {}, actual: {}", raw_len, raw.remaining()).into()) + } let unit_raw = raw.copy_to_bytes(raw_len as usize); let unit = decode_unit(unit_type.try_into()?, &unit_raw)?; units.push(unit); } + if raw.remaining() != 0 { + return Err(format!("left residue after decoding all the units: {} ", raw.remaining()).into()) + } + Ok(units) } @@ -127,6 +138,8 @@ impl Clone for Box { #[cfg(test)] mod tests { + use commonware_utils::SizedSerialize; + use crate::address::Address; use super::{decode_units, encode_units, msg::SequencerMsg, transfer::Transfer, Unit}; @@ -146,4 +159,60 @@ mod tests { assert_eq!(units.len(), decoded_untis.len()) } + + #[test] + fn test_insufficient_bytes() { + let mut units: Vec> = Vec::new(); + + let transfer = Transfer::new(Address::empty(), 100, vec![0, 1, 2, 3]); + let msg = SequencerMsg::new(0, Address::empty(), vec![3, 4, 5, 6]); + + units.push(Box::new(transfer)); + units.push(Box::new(msg)); + + let mut units_raw = encode_units(&units); + + let decode_result = decode_units(&units_raw[0..units_raw.len() - 10]); + + let err_str = decode_result.map_err(|e| e.to_string()).err().unwrap(); + assert!(err_str.contains("remaining bytes invalid to decode a unit")); + } + + + #[test] + fn test_excessive_bytes() { + let mut units: Vec> = Vec::new(); + + let transfer = Transfer::new(Address::empty(), 100, vec![0, 1, 2, 3]); + let msg = SequencerMsg::new(0, Address::empty(), vec![3, 4, 5, 6]); + + units.push(Box::new(transfer)); + units.push(Box::new(msg)); + + let mut units_raw = encode_units(&units); + units_raw.append(&mut [0 as u8; 32].to_vec()); + + let decode_result = decode_units(units_raw.as_slice()); + + let err_str = decode_result.map_err(|e| e.to_string()).err().unwrap(); + assert!(err_str.contains("left residue after decoding all the units")); + } + + #[test] + fn test_unit_header_truncated() { + let mut units: Vec> = Vec::new(); + + let transfer = Transfer::new(Address::empty(), 100, vec![0, 1, 2, 3]); + let msg = SequencerMsg::new(0, Address::empty(), vec![3, 4, 5, 6]); + + units.push(Box::new(transfer)); + units.push(Box::new(msg)); + + let units_raw = encode_units(&units); + let decode_result = decode_units(&units_raw[0..units_raw.len()-Transfer::SERIALIZED_LEN-5]); + + let err_str = decode_result.map_err(|e| e.to_string()).err().unwrap(); + print!("{}\n", err_str); + assert!(err_str.contains("remaining bytes invalid to form a unit header")); + } } \ No newline at end of file diff --git a/types/src/units/msg.rs b/types/src/units/msg.rs index 5097a782..4e99e491 100644 --- a/types/src/units/msg.rs +++ b/types/src/units/msg.rs @@ -1,4 +1,5 @@ use bytes::{Buf, BufMut}; +use commonware_utils::SizedSerialize; use crate::{ address::Address, @@ -18,6 +19,10 @@ pub struct SequencerMsg { pub data: Vec, } +impl SizedSerialize for SequencerMsg { + const SERIALIZED_LEN: usize = ADDRESSLEN + size_of::() * 2; +} + impl SequencerMsg { pub fn new(chain_id: u64, from: Address, data: Vec) -> SequencerMsg { Self { @@ -30,9 +35,8 @@ impl SequencerMsg { // @todo introduce syntactic checks. pub fn decode(mut bytes: &[u8]) -> Result> { // ChainID + DataLen + AddressLen + - let expected_size = size_of::()*2 + size_of::
(); - if bytes.len() < expected_size { - return Err(format!("Not enough data to decode sequencer message, wanted: >{}, actual: {}", expected_size, bytes.len()).into()); + if bytes.len() < Self::SERIALIZED_LEN { + return Err(format!("Not enough data to decode sequencer message, wanted: >{}, actual: {}", Self::SERIALIZED_LEN, bytes.len()).into()); } let chain_id = bytes.get_u64(); diff --git a/types/src/units/transfer.rs b/types/src/units/transfer.rs index 9ef01f53..dd441530 100644 --- a/types/src/units/transfer.rs +++ b/types/src/units/transfer.rs @@ -1,4 +1,5 @@ use bytes::{Buf, BufMut}; +use commonware_utils::SizedSerialize; use crate::{address::Address, ADDRESSLEN}; use crate::state_view::StateView; @@ -16,6 +17,10 @@ pub struct Transfer { pub memo: Vec, } +impl SizedSerialize for Transfer { + const SERIALIZED_LEN: usize = ADDRESSLEN + size_of::() * 2; +} + impl Transfer { pub fn new(to: Address, value: u64, memo: Vec) -> Transfer { Self { @@ -27,9 +32,8 @@ impl Transfer { pub fn decode(mut bytes: &[u8]) -> Result> { // Value + MemoLen + AddressLen + - let expected_size = size_of::() * 2 + size_of::
(); - if bytes.len() < expected_size { - return Err("Not enough data to decode sequencer message".into()); + if bytes.len() < Self::SERIALIZED_LEN { + return Err("Not enough data to decode transfer".into()); } let to = Address::from_bytes(&bytes.copy_to_bytes(ADDRESSLEN))?;