From d61d5fad1acb3c601eca6e2e25b0360322c95df7 Mon Sep 17 00:00:00 2001 From: rikoeldon <106416799+rikoeldon@users.noreply.github.com> Date: Wed, 19 Mar 2025 14:58:56 -0400 Subject: [PATCH 01/43] git ignore update --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 18dfef8f..9c5ef416 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,5 @@ /target *.vscode /chain/assets -/types/pkg \ No newline at end of file +/types/pkg +.idea \ No newline at end of file From 83b2456bd26da8bdde3b16ad31ae089316e36a3d Mon Sep 17 00:00:00 2001 From: rikoeldon <106416799+rikoeldon@users.noreply.github.com> Date: Thu, 20 Mar 2025 11:22:25 -0400 Subject: [PATCH 02/43] setting up actions --- actions/msg.rs | 21 +++++++++++++++++++++ actions/transfer.rs | 0 types/src/codec.rs | 11 +++++++++++ types/src/lib.rs | 1 + 4 files changed, 33 insertions(+) create mode 100644 actions/msg.rs create mode 100644 actions/transfer.rs create mode 100644 types/src/codec.rs diff --git a/actions/msg.rs b/actions/msg.rs new file mode 100644 index 00000000..30dcccbe --- /dev/null +++ b/actions/msg.rs @@ -0,0 +1,21 @@ +use types::Address; +pub struct SequencerMsg { + pub chain_id: Vec, + pub data: Vec, + pub from_address: Address::new(), + pub relayer_id: u64, +} + +impl SequencerMsg { + pub fn new() -> Self { + Self { + chain_id: vec![], + data: vec![], + from_address: Address::new(), + relayer_id: 0, + } + } + pub fn get_type_id(&self) -> u8 { + 0 + } +} \ No newline at end of file diff --git a/actions/transfer.rs b/actions/transfer.rs new file mode 100644 index 00000000..e69de29b diff --git a/types/src/codec.rs b/types/src/codec.rs new file mode 100644 index 00000000..f1a6b529 --- /dev/null +++ b/types/src/codec.rs @@ -0,0 +1,11 @@ +pub enum Codec { + +} +const ADDRESSLEN: usize = 33; +struct Address([u8;ADDRESSLEN]); + +impl Address { + pub fn new() -> Self { + Self([0;ADDRESSLEN]) + } +} \ No newline at end of file diff --git a/types/src/lib.rs b/types/src/lib.rs index 9ffac637..75a601a8 100644 --- a/types/src/lib.rs +++ b/types/src/lib.rs @@ -5,6 +5,7 @@ pub use block::{Block, Finalized, Notarized}; mod consensus; pub use consensus::{leader_index, Finalization, Kind, Notarization, Nullification, Seed}; pub mod wasm; +mod codec; // We don't use functions here to guard against silent changes. pub const NAMESPACE: &[u8] = b"_ALTO"; From 1da828e2a1b13f6adc3e0f337b6014390ee42f20 Mon Sep 17 00:00:00 2001 From: NateAtNodeKit Date: Fri, 21 Mar 2025 15:34:26 -0400 Subject: [PATCH 03/43] prototyping --- Cargo.lock | 11 ++++++ Cargo.toml | 2 ++ actions/transfer.rs | 16 +++++++++ storage/Cargo.toml | 7 ++++ storage/src/account.rs | 15 ++++++++ storage/src/database.rs | 9 +++++ storage/src/hashmap_db.rs | 74 +++++++++++++++++++++++++++++++++++++++ storage/src/lib.rs | 3 ++ types/src/codec.rs | 4 ++- types/src/lib.rs | 11 ++++++ vm/Cargo.toml | 6 ++++ vm/src/lib.rs | 1 + vm/src/vm.rs | 17 +++++++++ 13 files changed, 175 insertions(+), 1 deletion(-) create mode 100644 storage/Cargo.toml create mode 100644 storage/src/account.rs create mode 100644 storage/src/database.rs create mode 100644 storage/src/hashmap_db.rs create mode 100644 storage/src/lib.rs create mode 100644 vm/Cargo.toml create mode 100644 vm/src/lib.rs create mode 100644 vm/src/vm.rs diff --git a/Cargo.lock b/Cargo.lock index a3d67ca7..afa95dcc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -102,6 +102,13 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "alto-storage" +version = "0.1.0" +dependencies = [ + "alto-types", +] + [[package]] name = "alto-types" version = "0.0.4" @@ -117,6 +124,10 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "alto-vm" +version = "0.1.0" + [[package]] name = "anstream" version = "0.6.18" diff --git a/Cargo.toml b/Cargo.toml index bfa926c9..2e211b94 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,7 +3,9 @@ members = [ "chain", "client", "inspector", + "storage", "types", + "vm", ] resolver = "2" diff --git a/actions/transfer.rs b/actions/transfer.rs index e69de29b..799f84dc 100644 --- a/actions/transfer.rs +++ b/actions/transfer.rs @@ -0,0 +1,16 @@ + +pub struct Transfer { + pub from_address: Address::new(), + pub to_address: Address::new(), + pub value: u64, +} + +impl Transfer { + pub fn new(from: Address, to: Address, value: u64) -> Transfer { + Self { + from_address: from, + to_address: to, + value: value, + } + } +} \ No newline at end of file diff --git a/storage/Cargo.toml b/storage/Cargo.toml new file mode 100644 index 00000000..12168468 --- /dev/null +++ b/storage/Cargo.toml @@ -0,0 +1,7 @@ +[package] +name = "alto-storage" +version = "0.1.0" +edition = "2024" + +[dependencies] +alto-types = { workspace = true } diff --git a/storage/src/account.rs b/storage/src/account.rs new file mode 100644 index 00000000..c4694b5d --- /dev/null +++ b/storage/src/account.rs @@ -0,0 +1,15 @@ + +pub(crate) type Balance = u64; + +#[derive(Clone, Debug)] +pub struct Account { + pub balance: Balance, +} + +impl Account { + pub fn new(initial_balance: u64) -> Self { + Self { + balance: initial_balance, + } + } +} \ No newline at end of file diff --git a/storage/src/database.rs b/storage/src/database.rs new file mode 100644 index 00000000..90afc817 --- /dev/null +++ b/storage/src/database.rs @@ -0,0 +1,9 @@ +use alto_types::Address; +use crate::account::Balance; + +// Define database interface that will be used for all impls +pub trait Database { + fn get_balance(&self, address: Address) -> Option; + fn add_balance(&mut self, address: Address, amt: u64) -> Option; + fn sub_balance(&mut self, address: Address, amt: u64) -> Option; +} \ No newline at end of file diff --git a/storage/src/hashmap_db.rs b/storage/src/hashmap_db.rs new file mode 100644 index 00000000..ae0881b8 --- /dev/null +++ b/storage/src/hashmap_db.rs @@ -0,0 +1,74 @@ +use std::collections::HashMap; +use alto_types::Address; +use crate::account::{Account, Balance}; +use crate::database::Database; + +pub struct HashmapDb { + accts: HashMap, +} + +impl HashmapDb { + pub fn new() -> Self { + Self { + accts: HashMap::new(), + } + } +} + +impl Database for HashmapDb { + fn get_balance(&self, address: Address) -> Option { + match self.accts.get(&address) { + Some(acct) => Some(acct.balance), + None => None, + } + } + + fn add_balance(&mut self, address: Address, amt: u64) -> Option { + let curr_balance = self.accts.get(&address); + match curr_balance { + Some(account) => { + let updated_account = account.clone(); + updated_account.balance.checked_add(amt)?; + self.accts.insert(address, updated_account.clone()); + Some(updated_account.balance) + }, + None => { + let new_account = Account::new(amt); + self.accts.insert(address, new_account.clone()); + Some(new_account.balance) + }, + } + } + + fn sub_balance(&mut self, address: Address, amt: u64) -> Option { + let curr_balance = self.accts.get(&address); + match curr_balance { + Some(account) => { + let updated_account = account.clone(); + updated_account.balance.checked_add(amt)?; + self.accts.insert(address, updated_account.clone()); + Some(updated_account.balance) + }, + None => { + let new_account = Account::new(amt); + self.accts.insert(address, new_account.clone()); + Some(new_account.balance) + }, + } + } +} + +#[cfg(test)] // Compile these only when running tests +mod tests { + use super::*; // Import items from the parent module + + #[test] // Mark this function as a test + fn test_addition() { + let a = 2; + let b = 3; + let result = a + b; + + // Use assertions to validate + assert_eq!(result, 6); // Test passes if result == 5 + } +} diff --git a/storage/src/lib.rs b/storage/src/lib.rs new file mode 100644 index 00000000..0b99d3b6 --- /dev/null +++ b/storage/src/lib.rs @@ -0,0 +1,3 @@ +pub mod account; +pub mod database; +pub mod hashmap_db; \ No newline at end of file diff --git a/types/src/codec.rs b/types/src/codec.rs index f1a6b529..583acc66 100644 --- a/types/src/codec.rs +++ b/types/src/codec.rs @@ -1,6 +1,7 @@ pub enum Codec { } +/* const ADDRESSLEN: usize = 33; struct Address([u8;ADDRESSLEN]); @@ -8,4 +9,5 @@ impl Address { pub fn new() -> Self { Self([0;ADDRESSLEN]) } -} \ No newline at end of file +} + */ \ No newline at end of file diff --git a/types/src/lib.rs b/types/src/lib.rs index 75a601a8..1d6d80eb 100644 --- a/types/src/lib.rs +++ b/types/src/lib.rs @@ -15,6 +15,17 @@ pub const NOTARIZE_NAMESPACE: &[u8] = b"_ALTO_NOTARIZE"; pub const NULLIFY_NAMESPACE: &[u8] = b"_ALTO_NULLIFY"; pub const FINALIZE_NAMESPACE: &[u8] = b"_ALTO_FINALIZE"; +const ADDRESSLEN: usize = 33; + +#[derive(Hash, Eq, PartialEq)] +pub struct Address([u8;ADDRESSLEN]); + +impl Address { + pub fn new() -> Self { + Self([0;ADDRESSLEN]) + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/vm/Cargo.toml b/vm/Cargo.toml new file mode 100644 index 00000000..5d1e5c51 --- /dev/null +++ b/vm/Cargo.toml @@ -0,0 +1,6 @@ +[package] +name = "alto-vm" +version = "0.1.0" +edition = "2024" + +[dependencies] diff --git a/vm/src/lib.rs b/vm/src/lib.rs new file mode 100644 index 00000000..77e9423e --- /dev/null +++ b/vm/src/lib.rs @@ -0,0 +1 @@ +pub mod vm; \ No newline at end of file diff --git a/vm/src/vm.rs b/vm/src/vm.rs new file mode 100644 index 00000000..3390c221 --- /dev/null +++ b/vm/src/vm.rs @@ -0,0 +1,17 @@ +use std::collections::HashMap; + +struct VM { + state_db: Box +} + +impl VM { + pub fn new() { + Self { + state_db: Box::new(HashMapDb::new()) + } + } + + pub fn execute(msg: Transfer) { + // process the transfer + } +} \ No newline at end of file From acd89076237c021bd2d6c2db296b22a15fe9e37e Mon Sep 17 00:00:00 2001 From: NateAtNodeKit Date: Fri, 21 Mar 2025 17:23:19 -0400 Subject: [PATCH 04/43] fix project settings --- Cargo.lock | 11 +++++++++++ Cargo.toml | 5 ++++- actions/Cargo.toml | 7 +++++++ actions/src/lib.rs | 2 ++ actions/{ => src}/msg.rs | 5 +++-- actions/{ => src}/transfer.rs | 5 +++-- vm/Cargo.toml | 4 ++++ vm/src/vm.rs | 8 +++++--- 8 files changed, 39 insertions(+), 8 deletions(-) create mode 100644 actions/Cargo.toml create mode 100644 actions/src/lib.rs rename actions/{ => src}/msg.rs (86%) rename actions/{ => src}/transfer.rs (74%) diff --git a/Cargo.lock b/Cargo.lock index afa95dcc..8e1578fb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -36,6 +36,13 @@ dependencies = [ "memchr", ] +[[package]] +name = "alto-actions" +version = "0.1.0" +dependencies = [ + "alto-types", +] + [[package]] name = "alto-chain" version = "0.0.4" @@ -127,6 +134,10 @@ dependencies = [ [[package]] name = "alto-vm" version = "0.1.0" +dependencies = [ + "alto-actions", + "alto-storage", +] [[package]] name = "anstream" diff --git a/Cargo.toml b/Cargo.toml index 2e211b94..d270f159 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,5 @@ [workspace] -members = [ +members = [ "actions", "chain", "client", "inspector", @@ -12,6 +12,9 @@ resolver = "2" [workspace.dependencies] alto-client = { version = "0.0.4", path = "client" } alto-types = { version = "0.0.4", path = "types" } +alto-actions = { version = "0.1.0", path = "actions"} +alto-storage = { version = "0.1.0", path = "storage"} +alto-vm = { version = "0.1.0", path = "vm"} commonware-consensus = { version = "0.0.40" } commonware-cryptography = { version = "0.0.40" } commonware-deployer = { version = "0.0.40" } diff --git a/actions/Cargo.toml b/actions/Cargo.toml new file mode 100644 index 00000000..189380eb --- /dev/null +++ b/actions/Cargo.toml @@ -0,0 +1,7 @@ +[package] +name = "alto-actions" +version = "0.1.0" +edition = "2024" + +[dependencies] +alto-types = { workspace = true } diff --git a/actions/src/lib.rs b/actions/src/lib.rs new file mode 100644 index 00000000..6b3f5403 --- /dev/null +++ b/actions/src/lib.rs @@ -0,0 +1,2 @@ +pub mod msg; +pub mod transfer; \ No newline at end of file diff --git a/actions/msg.rs b/actions/src/msg.rs similarity index 86% rename from actions/msg.rs rename to actions/src/msg.rs index 30dcccbe..b71c7065 100644 --- a/actions/msg.rs +++ b/actions/src/msg.rs @@ -1,8 +1,9 @@ -use types::Address; +use alto_types::Address; + pub struct SequencerMsg { pub chain_id: Vec, pub data: Vec, - pub from_address: Address::new(), + pub from_address: Address, pub relayer_id: u64, } diff --git a/actions/transfer.rs b/actions/src/transfer.rs similarity index 74% rename from actions/transfer.rs rename to actions/src/transfer.rs index 799f84dc..be863b50 100644 --- a/actions/transfer.rs +++ b/actions/src/transfer.rs @@ -1,7 +1,8 @@ +use alto_types::Address; pub struct Transfer { - pub from_address: Address::new(), - pub to_address: Address::new(), + pub from_address: Address, + pub to_address: Address, pub value: u64, } diff --git a/vm/Cargo.toml b/vm/Cargo.toml index 5d1e5c51..1725c336 100644 --- a/vm/Cargo.toml +++ b/vm/Cargo.toml @@ -4,3 +4,7 @@ version = "0.1.0" edition = "2024" [dependencies] +alto-storage = { workspace = true } +alto-actions = { workspace = true } +#alto-storage = { path = "../storage" } +#alto-actions = { path = "../actions" } diff --git a/vm/src/vm.rs b/vm/src/vm.rs index 3390c221..2616fd4e 100644 --- a/vm/src/vm.rs +++ b/vm/src/vm.rs @@ -1,4 +1,6 @@ -use std::collections::HashMap; +use alto_storage::database::Database; +use alto_storage::hashmap_db::HashmapDb; +use alto_actions::transfer::Transfer; struct VM { state_db: Box @@ -7,8 +9,8 @@ struct VM { impl VM { pub fn new() { Self { - state_db: Box::new(HashMapDb::new()) - } + state_db: Box::new(HashmapDb::new()) + }; } pub fn execute(msg: Transfer) { From 384a3d2cabb4e7dc8278179c29292185c11dfb41 Mon Sep 17 00:00:00 2001 From: rikoeldon <106416799+rikoeldon@users.noreply.github.com> Date: Mon, 24 Mar 2025 15:02:31 -0400 Subject: [PATCH 05/43] defined base accounts, balance, address, and working on VM. --- Cargo.lock | 9 +++++++++ Cargo.toml | 1 + actions/src/transfer.rs | 39 ++++++++++++++++++++++++++++++++----- storage/Cargo.toml | 2 ++ storage/src/account.rs | 20 +++++++++++++++++-- types/Cargo.toml | 1 + types/src/lib.rs | 17 +++++++++++++--- vm/src/vm.rs | 43 ++++++++++++++++++++++++++++++++++++----- 8 files changed, 117 insertions(+), 15 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8e1578fb..f5571977 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -114,6 +114,8 @@ name = "alto-storage" version = "0.1.0" dependencies = [ "alto-types", + "commonware-cryptography", + "rand", ] [[package]] @@ -124,6 +126,7 @@ dependencies = [ "commonware-cryptography", "commonware-utils", "getrandom 0.2.15", + "more-asserts", "rand", "serde", "serde-wasm-bindgen", @@ -2093,6 +2096,12 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "more-asserts" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fafa6961cabd9c63bcd77a45d7e3b7f3b552b70417831fb0f56db717e72407e" + [[package]] name = "multimap" version = "0.10.0" diff --git a/Cargo.toml b/Cargo.toml index d270f159..660e31d0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -37,6 +37,7 @@ tracing-subscriber = "0.3.19" governor = "0.6.3" prometheus-client = "0.22.3" clap = "4.5.18" +more-asserts = "0.3.1" [profile.bench] # Because we enable overflow checks in "release," we should benchmark with them. diff --git a/actions/src/transfer.rs b/actions/src/transfer.rs index be863b50..6cdd0ef7 100644 --- a/actions/src/transfer.rs +++ b/actions/src/transfer.rs @@ -1,17 +1,46 @@ use alto_types::Address; +const MAX_MEMO_SIZE: usize = 256; pub struct Transfer { pub from_address: Address, pub to_address: Address, pub value: u64, + pub memo: Vec, +} + +pub enum TransferError { + DuplicateAddress, + InvalidToAddress, + InvalidFromAddress, + + InsufficientFunds, + InvalidMemoSize, } impl Transfer { - pub fn new(from: Address, to: Address, value: u64) -> Transfer { - Self { - from_address: from, - to_address: to, - value: value, + pub fn new(from: Address, to: Address, value: u64, memo: Vec) -> Result { + if value == 0 { + Err(TransferError::InsufficientFunds) + } + else if memo.len() > MAX_MEMO_SIZE { + Err(TransferError::InvalidMemoSize) + } + else if from.is_empty() { + Err(TransferError::InvalidFromAddress) + } + else if to.is_empty() { + Err(TransferError::InvalidToAddress) + } + else if from == to { + Err(TransferError::DuplicateAddress) + } + else { + Ok(Self { + from_address: from, + to_address: to, + value, + memo, + }) } } } \ No newline at end of file diff --git a/storage/Cargo.toml b/storage/Cargo.toml index 12168468..1d99074f 100644 --- a/storage/Cargo.toml +++ b/storage/Cargo.toml @@ -5,3 +5,5 @@ edition = "2024" [dependencies] alto-types = { workspace = true } +commonware-cryptography = { workspace = true } +rand = "0.8.5" diff --git a/storage/src/account.rs b/storage/src/account.rs index c4694b5d..a50712a6 100644 --- a/storage/src/account.rs +++ b/storage/src/account.rs @@ -1,15 +1,31 @@ +use commonware_cryptography::{ed25519::PublicKey, Ed25519, Scheme}; +use alto_types::Address; +use rand::rngs::OsRng; pub(crate) type Balance = u64; #[derive(Clone, Debug)] pub struct Account { + pub address: Address, + pub public_key: PublicKey, pub balance: Balance, } impl Account { - pub fn new(initial_balance: u64) -> Self { + pub fn new() -> Self { + let mut rng = OsRng; + // generates keypair using random number generator + let keypair = Ed25519::new(&mut rng); + + let public_key = keypair.public_key(); + + // takes addr from pk (truncating or hashing if necessary) + let address = Address::new(&public_key.as_ref()[..32]); + Self { - balance: initial_balance, + address, + public_key, + balance: 0, } } } \ No newline at end of file diff --git a/types/Cargo.toml b/types/Cargo.toml index 34cc41ed..d45e2773 100644 --- a/types/Cargo.toml +++ b/types/Cargo.toml @@ -22,6 +22,7 @@ thiserror = { workspace = true } wasm-bindgen = "0.2.100" serde = { version = "1.0.219", features = ["derive"] } serde-wasm-bindgen = "0.6.5" +more-asserts = "0.3.1" # Enable "js" feature when WASM is target [target.'cfg(target_arch = "wasm32")'.dependencies.getrandom] diff --git a/types/src/lib.rs b/types/src/lib.rs index 1d6d80eb..05465228 100644 --- a/types/src/lib.rs +++ b/types/src/lib.rs @@ -6,6 +6,8 @@ mod consensus; pub use consensus::{leader_index, Finalization, Kind, Notarization, Nullification, Seed}; pub mod wasm; mod codec; +use more_asserts; +use more_asserts::assert_le; // We don't use functions here to guard against silent changes. pub const NAMESPACE: &[u8] = b"_ALTO"; @@ -15,15 +17,24 @@ pub const NOTARIZE_NAMESPACE: &[u8] = b"_ALTO_NOTARIZE"; pub const NULLIFY_NAMESPACE: &[u8] = b"_ALTO_NULLIFY"; pub const FINALIZE_NAMESPACE: &[u8] = b"_ALTO_FINALIZE"; -const ADDRESSLEN: usize = 33; +const ADDRESSLEN: usize = 32; -#[derive(Hash, Eq, PartialEq)] +#[derive(Hash, Eq, PartialEq, Clone, Debug)] pub struct Address([u8;ADDRESSLEN]); impl Address { - pub fn new() -> Self { + pub fn new(slice: &[u8]) -> Self { + assert_le!(slice.len(), ADDRESSLEN); + let mut address = Self::empty(); + address.0.copy_from_slice(slice); + address + } + pub fn empty() -> Self { Self([0;ADDRESSLEN]) } + pub fn is_empty(&self) -> bool { + self.0 == Self::empty().0 + } } #[cfg(test)] diff --git a/vm/src/vm.rs b/vm/src/vm.rs index 2616fd4e..291ca75e 100644 --- a/vm/src/vm.rs +++ b/vm/src/vm.rs @@ -1,19 +1,52 @@ +use alto_actions::transfer::{Transfer, TransferError}; +use alto_storage::account::Account; use alto_storage::database::Database; use alto_storage::hashmap_db::HashmapDb; -use alto_actions::transfer::Transfer; struct VM { - state_db: Box + state_db: Box, } impl VM { pub fn new() { Self { - state_db: Box::new(HashmapDb::new()) + state_db: Box::new(HashmapDb::new()), }; } - pub fn execute(msg: Transfer) { + pub fn execute(&mut self, msg: Transfer) -> Result<(), TransferError> { // process the transfer + + // add balance to account + let from_balance = self.state_db.get_balance(msg.from_address); + if from_balance.is_none() { + return Err(TransferError::InvalidFromAddress); + } + let to_balance = self.state_db.get_balance(msg.to_address); + if to_balance.is_none() { + return Err(TransferError::InvalidToAddress); + } + // TODO: + // make check for sending funds > 0 so reject any negative values or balances. + // make sure address isn't a duplicate of self. + // need from balance to be greater than or equal to the transfer amount + // need the to_balance to not overflow + + let updated_from_balance = from_balance.unwrap().checked_sub(msg.value); + let updated_to_balance = to_balance.unwrap().checked_add(msg.value); + + match curr_balance { + Some(account) => { + let updated_account = account.clone(); + updated_account.balance.checked_add(amt)?; + self.accts.insert(address, updated_account.clone()); + Some(updated_account.balance) + } + None => { + let new_account = Account::new(amt); + self.accts.insert(address, new_account.clone()); + Some(new_account.balance) + } + } } -} \ No newline at end of file +} From e31d470d8e9a81886e732f2d02e915779b255f78 Mon Sep 17 00:00:00 2001 From: NateAtNodeKit Date: Mon, 24 Mar 2025 17:24:11 -0400 Subject: [PATCH 06/43] transfer logic --- Cargo.lock | 1 + actions/src/msg.rs | 2 +- actions/src/transfer.rs | 13 ++++- storage/src/account.rs | 2 +- storage/src/database.rs | 5 +- storage/src/hashmap_db.rs | 56 +++--------------- types/src/lib.rs | 10 ++-- vm/Cargo.toml | 3 +- vm/src/vm.rs | 119 +++++++++++++++++++++++++++++--------- 9 files changed, 124 insertions(+), 87 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f5571977..951dbe1e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -140,6 +140,7 @@ version = "0.1.0" dependencies = [ "alto-actions", "alto-storage", + "alto-types", ] [[package]] diff --git a/actions/src/msg.rs b/actions/src/msg.rs index b71c7065..e8e1d093 100644 --- a/actions/src/msg.rs +++ b/actions/src/msg.rs @@ -12,7 +12,7 @@ impl SequencerMsg { Self { chain_id: vec![], data: vec![], - from_address: Address::new(), + from_address: Address::empty(), relayer_id: 0, } } diff --git a/actions/src/transfer.rs b/actions/src/transfer.rs index 6cdd0ef7..84a16129 100644 --- a/actions/src/transfer.rs +++ b/actions/src/transfer.rs @@ -1,6 +1,8 @@ use alto_types::Address; const MAX_MEMO_SIZE: usize = 256; + +#[derive(Debug)] pub struct Transfer { pub from_address: Address, pub to_address: Address, @@ -8,17 +10,24 @@ pub struct Transfer { pub memo: Vec, } +#[derive(Debug)] pub enum TransferError { DuplicateAddress, InvalidToAddress, InvalidFromAddress, - InsufficientFunds, + TooMuchFunds, InvalidMemoSize, + StorageError, } impl Transfer { - pub fn new(from: Address, to: Address, value: u64, memo: Vec) -> Result { + pub fn new(from_address: Address, to_address: Address, value: u64) -> Result { + let empty_memo = b"".to_vec(); + Self::new_with_memo(from_address, to_address, value, empty_memo) + } + + pub fn new_with_memo(from: Address, to: Address, value: u64, memo: Vec) -> Result { if value == 0 { Err(TransferError::InsufficientFunds) } diff --git a/storage/src/account.rs b/storage/src/account.rs index a50712a6..53e31da5 100644 --- a/storage/src/account.rs +++ b/storage/src/account.rs @@ -2,7 +2,7 @@ use commonware_cryptography::{ed25519::PublicKey, Ed25519, Scheme}; use alto_types::Address; use rand::rngs::OsRng; -pub(crate) type Balance = u64; +pub type Balance = u64; #[derive(Clone, Debug)] pub struct Account { diff --git a/storage/src/database.rs b/storage/src/database.rs index 90afc817..1486787b 100644 --- a/storage/src/database.rs +++ b/storage/src/database.rs @@ -3,7 +3,6 @@ use crate::account::Balance; // Define database interface that will be used for all impls pub trait Database { - fn get_balance(&self, address: Address) -> Option; - fn add_balance(&mut self, address: Address, amt: u64) -> Option; - fn sub_balance(&mut self, address: Address, amt: u64) -> Option; + fn get_balance(&self, address: &Address) -> Option; + fn set_balance(&mut self, address: &Address, amt: u64) -> bool; } \ No newline at end of file diff --git a/storage/src/hashmap_db.rs b/storage/src/hashmap_db.rs index ae0881b8..a845e657 100644 --- a/storage/src/hashmap_db.rs +++ b/storage/src/hashmap_db.rs @@ -16,59 +16,19 @@ impl HashmapDb { } impl Database for HashmapDb { - fn get_balance(&self, address: Address) -> Option { + fn get_balance(&self, address: &Address) -> Option { match self.accts.get(&address) { Some(acct) => Some(acct.balance), None => None, } } - fn add_balance(&mut self, address: Address, amt: u64) -> Option { - let curr_balance = self.accts.get(&address); - match curr_balance { - Some(account) => { - let updated_account = account.clone(); - updated_account.balance.checked_add(amt)?; - self.accts.insert(address, updated_account.clone()); - Some(updated_account.balance) - }, - None => { - let new_account = Account::new(amt); - self.accts.insert(address, new_account.clone()); - Some(new_account.balance) - }, - } - } - - fn sub_balance(&mut self, address: Address, amt: u64) -> Option { - let curr_balance = self.accts.get(&address); - match curr_balance { - Some(account) => { - let updated_account = account.clone(); - updated_account.balance.checked_add(amt)?; - self.accts.insert(address, updated_account.clone()); - Some(updated_account.balance) - }, - None => { - let new_account = Account::new(amt); - self.accts.insert(address, new_account.clone()); - Some(new_account.balance) - }, - } - } -} - -#[cfg(test)] // Compile these only when running tests -mod tests { - use super::*; // Import items from the parent module - - #[test] // Mark this function as a test - fn test_addition() { - let a = 2; - let b = 3; - let result = a + b; - - // Use assertions to validate - assert_eq!(result, 6); // Test passes if result == 5 + fn set_balance(&mut self, address: &Address, amt: Balance) -> bool { + // TODO: Add in public key update later + let mut updated_account = Account::new(); + updated_account.address = address.clone(); + updated_account.balance = amt; + self.accts.insert(address.clone(), updated_account); + true } } diff --git a/types/src/lib.rs b/types/src/lib.rs index 05465228..be687dae 100644 --- a/types/src/lib.rs +++ b/types/src/lib.rs @@ -24,14 +24,16 @@ pub struct Address([u8;ADDRESSLEN]); impl Address { pub fn new(slice: &[u8]) -> Self { - assert_le!(slice.len(), ADDRESSLEN); - let mut address = Self::empty(); - address.0.copy_from_slice(slice); - address + assert_le!(slice.len(), ADDRESSLEN, "address slice is too large"); + let mut arr = [0u8; ADDRESSLEN]; + arr[..slice.len()].copy_from_slice(slice); + Address(arr) } + pub fn empty() -> Self { Self([0;ADDRESSLEN]) } + pub fn is_empty(&self) -> bool { self.0 == Self::empty().0 } diff --git a/vm/Cargo.toml b/vm/Cargo.toml index 1725c336..8e773002 100644 --- a/vm/Cargo.toml +++ b/vm/Cargo.toml @@ -6,5 +6,4 @@ edition = "2024" [dependencies] alto-storage = { workspace = true } alto-actions = { workspace = true } -#alto-storage = { path = "../storage" } -#alto-actions = { path = "../actions" } +alto-types = { workspace = true } diff --git a/vm/src/vm.rs b/vm/src/vm.rs index 291ca75e..06a4c417 100644 --- a/vm/src/vm.rs +++ b/vm/src/vm.rs @@ -1,52 +1,119 @@ use alto_actions::transfer::{Transfer, TransferError}; -use alto_storage::account::Account; use alto_storage::database::Database; use alto_storage::hashmap_db::HashmapDb; +use alto_storage::account::Balance; +use alto_types::Address; + +const TEST_FAUCET_ADDRESS: &[u8] = b"0x0123456789abcdef0123456789abcd"; +const TEST_FAUCET_BALANCE: Balance = 10_000_000; struct VM { state_db: Box, } impl VM { - pub fn new() { + pub fn new() -> Self { Self { state_db: Box::new(HashmapDb::new()), - }; + } } - pub fn execute(&mut self, msg: Transfer) -> Result<(), TransferError> { - // process the transfer + pub fn new_test_vm() -> Self { + let mut state_db = Box::new(HashmapDb::new()); + state_db.set_balance(&Self::test_faucet_address(), TEST_FAUCET_BALANCE); + + Self { + state_db + } + } - // add balance to account - let from_balance = self.state_db.get_balance(msg.from_address); + // TODO: + // make check for sending funds > 0 so reject any negative values or balances. + // make sure address isn't a duplicate of self. + // need from balance to be greater than or equal to the transfer amount + // need the to_balance to not overflow + pub fn execute(&mut self, msg: Transfer) -> Result<(), TransferError> { + let from_balance = self.state_db.get_balance(&msg.from_address); if from_balance.is_none() { return Err(TransferError::InvalidFromAddress); } - let to_balance = self.state_db.get_balance(msg.to_address); + + + let mut to_balance = self.state_db.get_balance(&msg.to_address); if to_balance.is_none() { - return Err(TransferError::InvalidToAddress); + to_balance = Some(0) } - // TODO: - // make check for sending funds > 0 so reject any negative values or balances. - // make sure address isn't a duplicate of self. - // need from balance to be greater than or equal to the transfer amount - // need the to_balance to not overflow let updated_from_balance = from_balance.unwrap().checked_sub(msg.value); + if updated_from_balance.is_none() { + return Err(TransferError::InsufficientFunds); + } + let updated_to_balance = to_balance.unwrap().checked_add(msg.value); + if updated_to_balance.is_none() { + return Err(TransferError::TooMuchFunds); + } - match curr_balance { - Some(account) => { - let updated_account = account.clone(); - updated_account.balance.checked_add(amt)?; - self.accts.insert(address, updated_account.clone()); - Some(updated_account.balance) - } - None => { - let new_account = Account::new(amt); - self.accts.insert(address, new_account.clone()); - Some(new_account.balance) - } + // TODO: Below doesn't rollback cases. Fix later. + if !self.state_db.set_balance(&msg.from_address, updated_from_balance.unwrap()) { + return Err(TransferError::StorageError); + } + if !self.state_db.set_balance(&msg.to_address, updated_to_balance.unwrap()) { + return Err(TransferError::StorageError); } + + Ok(()) + } + + pub fn query_balance(&self, address: &Address) -> Option { + self.state_db.get_balance(address) + } + + pub fn test_faucet_address() -> Address { + Address::new(TEST_FAUCET_ADDRESS) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_basic() -> Result<(), String> { + let from_address = Address::new(b"0x10000"); + let to_address = Address::new(b"0x20000"); + let faucet_address = VM::test_faucet_address(); + + let mut test_vm = VM::new_test_vm(); + + // should be empty + let from_balance1 = test_vm.query_balance(&from_address); + assert!(from_balance1.is_none()); + + let faucet_balance = test_vm.query_balance(&VM::test_faucet_address()); + assert_eq!(faucet_balance, Some(TEST_FAUCET_BALANCE)); + + // process faucet transfers to addresses + let transfer1 = Transfer::new(faucet_address.clone(), from_address.clone(), 100); + test_vm.execute(transfer1.unwrap()).unwrap(); + let transfer2 = Transfer::new(faucet_address.clone(), to_address.clone(), 200); + test_vm.execute(transfer2.unwrap()).unwrap(); + + // check balances have been updated from faucet transfers + let from_balance2 = test_vm.query_balance(&from_address); + assert_eq!(from_balance2, Some(100)); + let from_balance3 = test_vm.query_balance(&to_address); + assert_eq!(from_balance3, Some(200)); + + let transfer3 = Transfer::new(from_address.clone(), to_address.clone(), 50); + test_vm.execute(transfer3.unwrap()).unwrap(); + + // check updated balances + let from_balance2 = test_vm.query_balance(&from_address); + assert_eq!(from_balance2, Some(50)); + let from_balance3 = test_vm.query_balance(&to_address); + assert_eq!(from_balance3, Some(250)); + + Ok(()) } } From bb418a45f41ab5f8378392a14388acc1a174dfdb Mon Sep 17 00:00:00 2001 From: NateAtNodeKit Date: Tue, 25 Mar 2025 14:28:40 -0400 Subject: [PATCH 07/43] rocksdb update --- Cargo.lock | 148 +++++++++++++++++++++++++++++++++++++- storage/Cargo.toml | 1 + storage/src/database.rs | 5 ++ storage/src/hashmap_db.rs | 24 +++++-- storage/src/lib.rs | 3 +- storage/src/rocks_db.rs | 51 +++++++++++++ vm/src/vm.rs | 6 +- 7 files changed, 225 insertions(+), 13 deletions(-) create mode 100644 storage/src/rocks_db.rs diff --git a/Cargo.lock b/Cargo.lock index 951dbe1e..f2567793 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -116,6 +116,7 @@ dependencies = [ "alto-types", "commonware-cryptography", "rand", + "rocksdb", ] [[package]] @@ -670,6 +671,26 @@ version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "230c5f1ca6a325a32553f8640d31ac9b49f2411e901e427570154868b46da4f7" +[[package]] +name = "bindgen" +version = "0.69.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "271383c67ccabffb7381723dea0672a673f292304fcb45c01cc648c7a8d58088" +dependencies = [ + "bitflags", + "cexpr", + "clang-sys", + "itertools 0.12.1", + "lazy_static", + "lazycell", + "proc-macro2", + "quote", + "regex", + "rustc-hash", + "shlex", + "syn 2.0.99", +] + [[package]] name = "bitflags" version = "2.9.0" @@ -746,6 +767,16 @@ dependencies = [ "either", ] +[[package]] +name = "bzip2-sys" +version = "0.1.13+1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "225bff33b2141874fe80d71e07d6eec4f85c5c216453dd96388240f96e1acc14" +dependencies = [ + "cc", + "pkg-config", +] + [[package]] name = "cc" version = "1.2.16" @@ -757,6 +788,15 @@ dependencies = [ "shlex", ] +[[package]] +name = "cexpr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" +dependencies = [ + "nom", +] + [[package]] name = "cfg-if" version = "1.0.0" @@ -798,6 +838,17 @@ dependencies = [ "zeroize", ] +[[package]] +name = "clang-sys" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" +dependencies = [ + "glob", + "libc", + "libloading", +] + [[package]] name = "clap" version = "4.5.31" @@ -1979,6 +2030,15 @@ version = "1.70.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" +[[package]] +name = "itertools" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" +dependencies = [ + "either", +] + [[package]] name = "itertools" version = "0.14.0" @@ -2019,18 +2079,60 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +[[package]] +name = "lazycell" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" + [[package]] name = "libc" version = "0.2.170" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "875b3680cb2f8f71bdcf9a30f38d48282f5d3c95cbf9b3fa57269bb5d5c06828" +[[package]] +name = "libloading" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc2f4eb4bc735547cfed7c0a4922cbd04a4655978c09b54f1f7b228750664c34" +dependencies = [ + "cfg-if", + "windows-targets", +] + [[package]] name = "libm" version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8355be11b20d696c8f18f6cc018c4e372165b1fa8126cef092399c9951984ffa" +[[package]] +name = "librocksdb-sys" +version = "0.17.1+9.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b7869a512ae9982f4d46ba482c2a304f1efd80c6412a3d4bf57bb79a619679f" +dependencies = [ + "bindgen", + "bzip2-sys", + "cc", + "libc", + "libz-sys", + "lz4-sys", + "zstd-sys", +] + +[[package]] +name = "libz-sys" +version = "1.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b70e7a7df205e92a1a4cd9aaae7898dac0aa555503cc0a649494d0d60e7651d" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + [[package]] name = "linux-raw-sys" version = "0.9.2" @@ -2059,6 +2161,16 @@ version = "0.4.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "30bde2b3dc3671ae49d8e2e9f044c7c005836e7a023ee57cffa25ab82764bb9e" +[[package]] +name = "lz4-sys" +version = "1.11.1+lz4-1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bd8c0d6c6ed0cd30b3652886bb8711dc4bb01d637a68105a3d5158039b418e6" +dependencies = [ + "cc", + "libc", +] + [[package]] name = "matchit" version = "0.8.4" @@ -2077,6 +2189,12 @@ version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + [[package]] name = "miniz_oxide" version = "0.8.5" @@ -2132,6 +2250,16 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b93853da6d84c2e3c7d730d6473e8817692dd89be387eb01b94d7f108ecb5b8c" +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + [[package]] name = "nonzero_ext" version = "0.3.0" @@ -2457,7 +2585,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" dependencies = [ "heck", - "itertools", + "itertools 0.14.0", "log", "multimap", "once_cell", @@ -2477,7 +2605,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" dependencies = [ "anyhow", - "itertools", + "itertools 0.14.0", "proc-macro2", "quote", "syn 2.0.99", @@ -2709,12 +2837,28 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rocksdb" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26ec73b20525cb235bad420f911473b69f9fe27cc856c5461bccd7e4af037f43" +dependencies = [ + "libc", + "librocksdb-sys", +] + [[package]] name = "rustc-demangle" version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + [[package]] name = "rustc_version" version = "0.4.1" diff --git a/storage/Cargo.toml b/storage/Cargo.toml index 1d99074f..02c6487f 100644 --- a/storage/Cargo.toml +++ b/storage/Cargo.toml @@ -7,3 +7,4 @@ edition = "2024" alto-types = { workspace = true } commonware-cryptography = { workspace = true } rand = "0.8.5" +rocksdb = "0.23.0" diff --git a/storage/src/database.rs b/storage/src/database.rs index 1486787b..08c88ab9 100644 --- a/storage/src/database.rs +++ b/storage/src/database.rs @@ -4,5 +4,10 @@ use crate::account::Balance; // Define database interface that will be used for all impls pub trait Database { fn get_balance(&self, address: &Address) -> Option; + fn set_balance(&mut self, address: &Address, amt: u64) -> bool; + + fn put(&mut self, key: &[u8], value: &[u8]) -> Result<(), Box>; + + fn get(&self, key: &[u8]) -> Result>, Box>; } \ No newline at end of file diff --git a/storage/src/hashmap_db.rs b/storage/src/hashmap_db.rs index a845e657..6a7bb979 100644 --- a/storage/src/hashmap_db.rs +++ b/storage/src/hashmap_db.rs @@ -3,21 +3,23 @@ use alto_types::Address; use crate::account::{Account, Balance}; use crate::database::Database; -pub struct HashmapDb { - accts: HashMap, +pub struct HashmapDatabase { + accounts: HashMap, + data: HashMap, } -impl HashmapDb { +impl HashmapDatabase { pub fn new() -> Self { Self { - accts: HashMap::new(), + accounts: HashMap::new(), + data: HashMap::new(), } } } -impl Database for HashmapDb { +impl Database for HashmapDatabase { fn get_balance(&self, address: &Address) -> Option { - match self.accts.get(&address) { + match self.accounts.get(&address) { Some(acct) => Some(acct.balance), None => None, } @@ -28,7 +30,15 @@ impl Database for HashmapDb { let mut updated_account = Account::new(); updated_account.address = address.clone(); updated_account.balance = amt; - self.accts.insert(address.clone(), updated_account); + self.accounts.insert(address.clone(), updated_account); true } + + fn put(&mut self, key: &[u8], value: &[u8]) -> Result<(), Box> { + self.data.insert(String::from_utf8(key.into())?, value.into()); + } + + fn get(&self, key: &[u8]) -> Result>, Box> { + + } } diff --git a/storage/src/lib.rs b/storage/src/lib.rs index 0b99d3b6..53447579 100644 --- a/storage/src/lib.rs +++ b/storage/src/lib.rs @@ -1,3 +1,4 @@ pub mod account; pub mod database; -pub mod hashmap_db; \ No newline at end of file +pub mod hashmap_db; +mod rocks_db; \ No newline at end of file diff --git a/storage/src/rocks_db.rs b/storage/src/rocks_db.rs new file mode 100644 index 00000000..276e7a94 --- /dev/null +++ b/storage/src/rocks_db.rs @@ -0,0 +1,51 @@ +use alto_types::Address; +use rocksdb::{DB, Options}; +use crate::account::{Account, Balance}; +use crate::database::Database; +use std::path::Path; + +const SAL_ROCKS_DB_PATH: &str = "rocksdb"; + +pub struct RocksDbDatabase { + db: DB, +} + +impl RocksDbDatabase { + pub fn new() -> Result> { + let mut opts = Options::default(); + opts.create_if_missing(true); + + let db_path = Path::new(SAL_ROCKS_DB_PATH); + let db = DB::open(&opts, &db_path)?; + Ok(RocksDbDatabase { db }) + } + + pub fn put(&self, key: &[u8], value: &[u8]) -> Result<(), Box> { + self.db.put(key, value)?; + Ok(()) + } + + pub fn get(&self, key: &[u8]) -> Result>, Box> { + let result = self.db.get(key)?; + Ok(result) + } +} + +impl Database for RocksDbDatabase { + fn get_balance(&self, address: &Address) -> Option { + match self.accts.get(&address) { + Some(acct) => Some(acct.balance), + None => None, + } + } + + fn set_balance(&mut self, address: &Address, amt: Balance) -> bool { + // TODO: Add in public key update later + let mut updated_account = Account::new(); + updated_account.address = address.clone(); + updated_account.balance = amt; + self.accts.insert(address.clone(), updated_account); + true + } +} +*/ \ No newline at end of file diff --git a/vm/src/vm.rs b/vm/src/vm.rs index 06a4c417..8859df3a 100644 --- a/vm/src/vm.rs +++ b/vm/src/vm.rs @@ -1,6 +1,6 @@ use alto_actions::transfer::{Transfer, TransferError}; use alto_storage::database::Database; -use alto_storage::hashmap_db::HashmapDb; +use alto_storage::hashmap_db::HashmapDatabase; use alto_storage::account::Balance; use alto_types::Address; @@ -14,12 +14,12 @@ struct VM { impl VM { pub fn new() -> Self { Self { - state_db: Box::new(HashmapDb::new()), + state_db: Box::new(HashmapDatabase::new()), } } pub fn new_test_vm() -> Self { - let mut state_db = Box::new(HashmapDb::new()); + let mut state_db = Box::new(HashmapDatabase::new()); state_db.set_balance(&Self::test_faucet_address(), TEST_FAUCET_BALANCE); Self { From 8813a1630dcda1c79b2ed4004257d0f833a46c3d Mon Sep 17 00:00:00 2001 From: rikoeldon <106416799+rikoeldon@users.noreply.github.com> Date: Tue, 25 Mar 2025 16:40:58 -0400 Subject: [PATCH 08/43] added to rocksDB --- Cargo.lock | 20 ++++++++++ storage/Cargo.toml | 2 + storage/src/account.rs | 38 +++++++++++++++++++ storage/src/database.rs | 11 ++++-- storage/src/rocks_db.rs | 84 ++++++++++++++++++++++++++++++++--------- types/Cargo.toml | 1 + types/src/codec.rs | 21 +++++------ types/src/lib.rs | 8 ++++ 8 files changed, 153 insertions(+), 32 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f2567793..f109dbcf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -114,6 +114,8 @@ name = "alto-storage" version = "0.1.0" dependencies = [ "alto-types", + "bytes", + "commonware-codec", "commonware-cryptography", "rand", "rocksdb", @@ -124,6 +126,7 @@ name = "alto-types" version = "0.0.4" dependencies = [ "bytes", + "commonware-codec", "commonware-cryptography", "commonware-utils", "getrandom 0.2.15", @@ -882,6 +885,17 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" +[[package]] +name = "commonware-codec" +version = "0.0.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65ebb5fc2fbf7d3fbdaf8d3dd3afdb80f03ee02a7d5a9fefc979235a2c06569a" +dependencies = [ + "bytes", + "paste", + "thiserror 2.0.12", +] + [[package]] name = "commonware-consensus" version = "0.0.40" @@ -2432,6 +2446,12 @@ dependencies = [ "windows-targets", ] +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + [[package]] name = "pem-rfc7468" version = "0.7.0" diff --git a/storage/Cargo.toml b/storage/Cargo.toml index 02c6487f..b5ea0322 100644 --- a/storage/Cargo.toml +++ b/storage/Cargo.toml @@ -8,3 +8,5 @@ alto-types = { workspace = true } commonware-cryptography = { workspace = true } rand = "0.8.5" rocksdb = "0.23.0" +commonware-codec = { version = "0.0.43" } +bytes = "1.7.1" \ No newline at end of file diff --git a/storage/src/account.rs b/storage/src/account.rs index 53e31da5..c313959f 100644 --- a/storage/src/account.rs +++ b/storage/src/account.rs @@ -1,4 +1,6 @@ +use commonware_codec::{Codec, Error, Reader, Writer}; use commonware_cryptography::{ed25519::PublicKey, Ed25519, Scheme}; +use commonware_cryptography::ed25519::PrivateKey; use alto_types::Address; use rand::rngs::OsRng; @@ -8,6 +10,7 @@ pub type Balance = u64; pub struct Account { pub address: Address, pub public_key: PublicKey, + pub private_key: PrivateKey, pub balance: Balance, } @@ -19,13 +22,48 @@ impl Account { let public_key = keypair.public_key(); + let private_key = keypair.private_key(); + // takes addr from pk (truncating or hashing if necessary) let address = Address::new(&public_key.as_ref()[..32]); Self { address, public_key, + private_key, balance: 0, } } +} +impl Codec for Account { + fn write(&self, writer: &mut impl Writer) { + let _ = self.address.write(writer); + self.balance.write(writer); + let len: u32 = self.public_key.as_ref().len() as u32; + len.write(writer); + let _ = self.public_key.as_ref().write(writer); + let priv_len: u32 = self.private_key.as_ref().len() as u32; + priv_len.write(writer); + let _ = self.private_key.as_ref().write(writer); + } + + fn read(reader: &mut impl Reader) -> Result { + let addr = <[u8; 32]>::read(reader)?; + let balance = ::read(reader)?; + // setting up public key + let pk_len = u32::read(reader)?; + let pk_bytes = reader.read_n_bytes(pk_len as usize)?; + let public_key = PublicKey::try_from(pk_bytes).unwrap(); + // setting up private key + let priv_key_len = u32::read(reader)?; + let priv_key_bytes = reader.read_n_bytes(priv_key_len as usize)?; + let private_key = PrivateKey::try_from(priv_key_bytes).unwrap(); + let address = Address::try_from(addr).unwrap(); + + Ok(Self{ address, public_key, private_key, balance}) + } + + fn len_encoded(&self) -> usize { + todo!() + } } \ No newline at end of file diff --git a/storage/src/database.rs b/storage/src/database.rs index 08c88ab9..2023e797 100644 --- a/storage/src/database.rs +++ b/storage/src/database.rs @@ -1,13 +1,18 @@ use alto_types::Address; -use crate::account::Balance; +use crate::account::{Balance, Account}; +use std::error::Error; // Define database interface that will be used for all impls pub trait Database { + fn get_account(&self, address: &Address) -> Option; + + fn set_account(&mut self, acc: &Account) -> Result<(), Box>; + fn get_balance(&self, address: &Address) -> Option; fn set_balance(&mut self, address: &Address, amt: u64) -> bool; - fn put(&mut self, key: &[u8], value: &[u8]) -> Result<(), Box>; + fn put(&mut self, key: &[u8], value: &[u8]) -> Result<(), Box>; - fn get(&self, key: &[u8]) -> Result>, Box>; + fn get(&self, key: &[u8]) -> Result>, Box>; } \ No newline at end of file diff --git a/storage/src/rocks_db.rs b/storage/src/rocks_db.rs index 276e7a94..5e922aa3 100644 --- a/storage/src/rocks_db.rs +++ b/storage/src/rocks_db.rs @@ -1,17 +1,24 @@ +use std::error::Error; use alto_types::Address; use rocksdb::{DB, Options}; +use commonware_codec::{Codec, ReadBuffer, WriteBuffer}; use crate::account::{Account, Balance}; use crate::database::Database; use std::path::Path; +use bytes::{BufMut, Bytes}; + const SAL_ROCKS_DB_PATH: &str = "rocksdb"; +const ACCOUNTS_PREFIX: &[u8] = b"sal_accounts"; + +const WRITE_BUFFER_CAPACITY: u64 = 500; pub struct RocksDbDatabase { db: DB, } impl RocksDbDatabase { - pub fn new() -> Result> { + pub fn new() -> Result> { let mut opts = Options::default(); opts.create_if_missing(true); @@ -20,32 +27,75 @@ impl RocksDbDatabase { Ok(RocksDbDatabase { db }) } - pub fn put(&self, key: &[u8], value: &[u8]) -> Result<(), Box> { - self.db.put(key, value)?; - Ok(()) + pub fn key_accounts(addr: &Address) -> Vec { + Self::make_multi_key(ACCOUNTS_PREFIX, addr.as_slice()) } - pub fn get(&self, key: &[u8]) -> Result>, Box> { - let result = self.db.get(key)?; - Ok(result) + fn make_multi_key(prefix: &[u8], sub_id: &[u8]) -> Vec { + let mut key = Vec::with_capacity(prefix.len() + sub_id.len() + 1); + key.extend_from_slice(prefix); + key.push(b':'); + key.extend_from_slice(sub_id); + key } } impl Database for RocksDbDatabase { + + fn get_account(&self, address: &Address) -> Result, Box> { + let key = Self::key_accounts(address); + let result = self.get(key.as_slice())?; + match result { + None => Ok(None), + Some(value) => { + let bytes = Bytes::copy_from_slice(&value); + let mut read_buf = ReadBuffer::new(bytes); + let acc = Account::read(&mut read_buf)?; + Ok(Some(acc)) + } + } + } + + fn set_account(&mut self, acc: &Account) -> Result<(), Box> { + let key = Self::key_accounts(&acc.address); + let mut write_buf = WriteBuffer::new(WRITE_BUFFER_CAPACITY as usize); + acc.write(&mut write_buf); + self.put(&key, write_buf.as_ref())?; + Ok(()) + } + fn get_balance(&self, address: &Address) -> Option { - match self.accts.get(&address) { - Some(acct) => Some(acct.balance), + let result = self.get_account(address).and_then(|acc| { + match acc { + Some(acc) => Ok(acc.balance), + None => Ok(0), + } + }); + match result { + Some(balance) => Some(balance), None => None, } } fn set_balance(&mut self, address: &Address, amt: Balance) -> bool { - // TODO: Add in public key update later - let mut updated_account = Account::new(); - updated_account.address = address.clone(); - updated_account.balance = amt; - self.accts.insert(address.clone(), updated_account); - true + let result = self.get_account(address); + match result { + Some(mut acc) => { + acc.balance = amt; + let result = self.set_account(&acc); + result.is_ok() + }, + None => false, + } } -} -*/ \ No newline at end of file + + fn put(&self, key: &[u8], value: &[u8]) -> Result<(), Box> { + self.db.put(key, value)?; + Ok(()) + } + + fn get(&self, key: &[u8]) -> Result>, Box> { + let result = self.db.get(key)?; + Ok(result) + } +} \ No newline at end of file diff --git a/types/Cargo.toml b/types/Cargo.toml index d45e2773..b6b721b3 100644 --- a/types/Cargo.toml +++ b/types/Cargo.toml @@ -23,6 +23,7 @@ wasm-bindgen = "0.2.100" serde = { version = "1.0.219", features = ["derive"] } serde-wasm-bindgen = "0.6.5" more-asserts = "0.3.1" +commonware-codec = { version = "0.0.43" } # Enable "js" feature when WASM is target [target.'cfg(target_arch = "wasm32")'.dependencies.getrandom] diff --git a/types/src/codec.rs b/types/src/codec.rs index 583acc66..79120e97 100644 --- a/types/src/codec.rs +++ b/types/src/codec.rs @@ -1,13 +1,10 @@ -pub enum Codec { +use commonware_codec::{Codec, Error, Reader, Writer}; +use commonware_cryptography::{ed25519::PublicKey, Ed25519, Scheme}; +use commonware_cryptography::ed25519::PrivateKey; +use serde::Serialize; -} -/* -const ADDRESSLEN: usize = 33; -struct Address([u8;ADDRESSLEN]); - -impl Address { - pub fn new() -> Self { - Self([0;ADDRESSLEN]) - } -} - */ \ No newline at end of file +pub fn serialize_pk(pk: &PublicKey, writer: &mut impl Writer) { + // let slice: &[u8] = pk.as_ref(); + // let length = slice.len(); + // length.serialize(&mut *writer).unwrap(); +} \ No newline at end of file diff --git a/types/src/lib.rs b/types/src/lib.rs index be687dae..6c608813 100644 --- a/types/src/lib.rs +++ b/types/src/lib.rs @@ -37,6 +37,14 @@ impl Address { pub fn is_empty(&self) -> bool { self.0 == Self::empty().0 } + + pub fn as_slice(&self) -> &[u8] { + &self.0 + } + + pub fn as_bytes(&self) -> &[u8;ADDRESSLEN] { + &self.0 + } } #[cfg(test)] From 70d07bb58a8c5161790f7a674524cfbb74311bb7 Mon Sep 17 00:00:00 2001 From: NateAtNodeKit Date: Wed, 26 Mar 2025 13:17:19 -0400 Subject: [PATCH 09/43] db refactoring --- storage/src/account.rs | 47 +++++++------------------- storage/src/database.rs | 69 +++++++++++++++++++++++++++++++++++++++ storage/src/hashmap_db.rs | 41 +++++++++++++---------- storage/src/rocks_db.rs | 11 ++++--- types/src/lib.rs | 10 +++++- 5 files changed, 119 insertions(+), 59 deletions(-) diff --git a/storage/src/account.rs b/storage/src/account.rs index c313959f..dca506a2 100644 --- a/storage/src/account.rs +++ b/storage/src/account.rs @@ -1,4 +1,4 @@ -use commonware_codec::{Codec, Error, Reader, Writer}; +use commonware_codec::{Codec, Error, Reader, SizedCodec, Writer}; use commonware_cryptography::{ed25519::PublicKey, Ed25519, Scheme}; use commonware_cryptography::ed25519::PrivateKey; use alto_types::Address; @@ -9,61 +9,38 @@ pub type Balance = u64; #[derive(Clone, Debug)] pub struct Account { pub address: Address, - pub public_key: PublicKey, - pub private_key: PrivateKey, pub balance: Balance, } impl Account { pub fn new() -> Self { - let mut rng = OsRng; - // generates keypair using random number generator - let keypair = Ed25519::new(&mut rng); - - let public_key = keypair.public_key(); - - let private_key = keypair.private_key(); - - // takes addr from pk (truncating or hashing if necessary) - let address = Address::new(&public_key.as_ref()[..32]); + Self { + address: Address::empty(), + balance: 0, + } + } + pub fn from_address(address: Address) -> Self { Self { address, - public_key, - private_key, balance: 0, } } } impl Codec for Account { fn write(&self, writer: &mut impl Writer) { - let _ = self.address.write(writer); + writer.write_bytes(self.address.0.as_slice()); self.balance.write(writer); - let len: u32 = self.public_key.as_ref().len() as u32; - len.write(writer); - let _ = self.public_key.as_ref().write(writer); - let priv_len: u32 = self.private_key.as_ref().len() as u32; - priv_len.write(writer); - let _ = self.private_key.as_ref().write(writer); } fn read(reader: &mut impl Reader) -> Result { - let addr = <[u8; 32]>::read(reader)?; + let addr_bytes = <[u8; 32]>::read(reader)?; + let address = Address::from_bytes(&addr_bytes).unwrap(); let balance = ::read(reader)?; - // setting up public key - let pk_len = u32::read(reader)?; - let pk_bytes = reader.read_n_bytes(pk_len as usize)?; - let public_key = PublicKey::try_from(pk_bytes).unwrap(); - // setting up private key - let priv_key_len = u32::read(reader)?; - let priv_key_bytes = reader.read_n_bytes(priv_key_len as usize)?; - let private_key = PrivateKey::try_from(priv_key_bytes).unwrap(); - let address = Address::try_from(addr).unwrap(); - - Ok(Self{ address, public_key, private_key, balance}) + Ok(Self{address, balance}) } fn len_encoded(&self) -> usize { - todo!() + Codec::len_encoded(&self.address.0) + Codec::len_encoded(&self.balance) } } \ No newline at end of file diff --git a/storage/src/database.rs b/storage/src/database.rs index 2023e797..68744de3 100644 --- a/storage/src/database.rs +++ b/storage/src/database.rs @@ -1,9 +1,16 @@ use alto_types::Address; use crate::account::{Balance, Account}; use std::error::Error; +use bytes::Bytes; +use commonware_codec::{Codec, ReadBuffer, WriteBuffer}; + +const ACCOUNTS_PREFIX: &[u8] = b"sal_accounts"; + +const DB_WRITE_BUFFER_CAPACITY: usize = 500; // Define database interface that will be used for all impls pub trait Database { + /* fn get_account(&self, address: &Address) -> Option; fn set_account(&mut self, acc: &Account) -> Result<(), Box>; @@ -12,6 +19,68 @@ pub trait Database { fn set_balance(&mut self, address: &Address, amt: u64) -> bool; + */ + + + fn get_account(&self, address: &Address) -> Result, Box> { + let key = Self::key_accounts(address); + let result = self.get(key.as_slice())?; + match result { + None => Ok(None), + Some(value) => { + let bytes = Bytes::copy_from_slice(&value); + let mut read_buf = ReadBuffer::new(bytes); + let acc = Account::read(&mut read_buf)?; + Ok(Some(acc)) + } + } + } + + fn set_account(&mut self, acc: &Account) -> Result<(), Box> { + let key = Self::key_accounts(&acc.address); + let mut write_buf = WriteBuffer::new(DB_WRITE_BUFFER_CAPACITY); + acc.write(&mut write_buf); + self.put(&key, write_buf.as_ref())?; + Ok(()) + } + + fn get_balance(&self, address: &Address) -> Option { + let result = self.get_account(address).and_then(|acc| { + match acc { + Some(acc) => Ok(acc.balance), + None => Ok(0), + } + }); + match result { + Ok(balance) => Some(balance), + _ => None, + } + } + + fn set_balance(&mut self, address: &Address, amt: Balance) -> bool { + let result = self.get_account(address); + match result { + Ok(Some(mut acc)) => { + acc.balance = amt; + let result = self.set_account(&acc); + result.is_ok() + }, + _ => false, + } + } + + fn key_accounts(addr: &Address) -> Vec { + Self::make_multi_key(ACCOUNTS_PREFIX, addr.as_slice()) + } + + fn make_multi_key(prefix: &[u8], sub_id: &[u8]) -> Vec { + let mut key = Vec::with_capacity(prefix.len() + sub_id.len() + 1); + key.extend_from_slice(prefix); + key.push(b':'); + key.extend_from_slice(sub_id); + key + } + fn put(&mut self, key: &[u8], value: &[u8]) -> Result<(), Box>; fn get(&self, key: &[u8]) -> Result>, Box>; diff --git a/storage/src/hashmap_db.rs b/storage/src/hashmap_db.rs index 6a7bb979..1684e89a 100644 --- a/storage/src/hashmap_db.rs +++ b/storage/src/hashmap_db.rs @@ -4,41 +4,46 @@ use crate::account::{Account, Balance}; use crate::database::Database; pub struct HashmapDatabase { - accounts: HashMap, data: HashMap, } impl HashmapDatabase { pub fn new() -> Self { Self { - accounts: HashMap::new(), data: HashMap::new(), } } } impl Database for HashmapDatabase { - fn get_balance(&self, address: &Address) -> Option { - match self.accounts.get(&address) { - Some(acct) => Some(acct.balance), - None => None, + fn put(&mut self, key: &[u8], value: &[u8]) -> Result<(), Box> { + //self.data.insert(String::from_utf8(key.into())?, value.into()) + let str_value: String = String::from_utf8(value.into()).unwrap(); + match self.data.insert(String::from_utf8(key.into())?, str_value) { + Some(_) => Ok(()), + None => Err("hashmap db insert failed".into()), } } - fn set_balance(&mut self, address: &Address, amt: Balance) -> bool { - // TODO: Add in public key update later - let mut updated_account = Account::new(); - updated_account.address = address.clone(); - updated_account.balance = amt; - self.accounts.insert(address.clone(), updated_account); - true - } - - fn put(&mut self, key: &[u8], value: &[u8]) -> Result<(), Box> { - self.data.insert(String::from_utf8(key.into())?, value.into()); + fn get(&self, key: &[u8]) -> Result>, Box> { + let str_key: String = String::from_utf8(key.into()).unwrap(); + self.data.get(&str_key).map_or( + Ok(None), + |v| Ok(Some(v.clone().into()))) } +} - fn get(&self, key: &[u8]) -> Result>, Box> { +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn test_hashmap_db() { + let mut db = HashmapDatabase::new(); + let key = b"key1"; + let value = b"value1"; + db.put(key, value).unwrap(); + let retrieved = db.get(key).unwrap().unwrap(); + assert_eq!(retrieved.as_slice(), value); } } diff --git a/storage/src/rocks_db.rs b/storage/src/rocks_db.rs index 5e922aa3..a8af5a63 100644 --- a/storage/src/rocks_db.rs +++ b/storage/src/rocks_db.rs @@ -9,9 +9,6 @@ use bytes::{BufMut, Bytes}; const SAL_ROCKS_DB_PATH: &str = "rocksdb"; -const ACCOUNTS_PREFIX: &[u8] = b"sal_accounts"; - -const WRITE_BUFFER_CAPACITY: u64 = 500; pub struct RocksDbDatabase { db: DB, @@ -27,6 +24,7 @@ impl RocksDbDatabase { Ok(RocksDbDatabase { db }) } + /* pub fn key_accounts(addr: &Address) -> Vec { Self::make_multi_key(ACCOUNTS_PREFIX, addr.as_slice()) } @@ -38,10 +36,11 @@ impl RocksDbDatabase { key.extend_from_slice(sub_id); key } + */ } impl Database for RocksDbDatabase { - +/* fn get_account(&self, address: &Address) -> Result, Box> { let key = Self::key_accounts(address); let result = self.get(key.as_slice())?; @@ -89,7 +88,9 @@ impl Database for RocksDbDatabase { } } - fn put(&self, key: &[u8], value: &[u8]) -> Result<(), Box> { + */ + + fn put(&mut self, key: &[u8], value: &[u8]) -> Result<(), Box> { self.db.put(key, value)?; Ok(()) } diff --git a/types/src/lib.rs b/types/src/lib.rs index 6c608813..a03ed207 100644 --- a/types/src/lib.rs +++ b/types/src/lib.rs @@ -20,7 +20,7 @@ pub const FINALIZE_NAMESPACE: &[u8] = b"_ALTO_FINALIZE"; const ADDRESSLEN: usize = 32; #[derive(Hash, Eq, PartialEq, Clone, Debug)] -pub struct Address([u8;ADDRESSLEN]); +pub struct Address(pub [u8;ADDRESSLEN]); impl Address { pub fn new(slice: &[u8]) -> Self { @@ -30,6 +30,14 @@ impl Address { Address(arr) } + pub fn from_bytes(bytes: &[u8]) -> Result { + if bytes.len() != 32 { + return Err("Address must be 32 bytes."); + } + + Ok(Address(<[u8; 32]>::try_from(bytes.clone()).unwrap())) + } + pub fn empty() -> Self { Self([0;ADDRESSLEN]) } From 701b2c5de5fe12c8be77a22c53a328d70447e056 Mon Sep 17 00:00:00 2001 From: NateAtNodeKit Date: Wed, 26 Mar 2025 17:45:14 -0400 Subject: [PATCH 10/43] fixes for db --- .gitignore | 3 +- Cargo.lock | 1 + storage/Cargo.toml | 3 +- storage/src/account.rs | 4 +- storage/src/hashmap_db.rs | 11 ++-- storage/src/rocks_db.rs | 118 ++++++++++++++++---------------------- types/src/lib.rs | 15 +++++ 7 files changed, 78 insertions(+), 77 deletions(-) diff --git a/.gitignore b/.gitignore index 9c5ef416..2ae10116 100644 --- a/.gitignore +++ b/.gitignore @@ -4,4 +4,5 @@ *.vscode /chain/assets /types/pkg -.idea \ No newline at end of file +/storage/rocksdb +.idea diff --git a/Cargo.lock b/Cargo.lock index f109dbcf..fabca603 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -119,6 +119,7 @@ dependencies = [ "commonware-cryptography", "rand", "rocksdb", + "tempfile", ] [[package]] diff --git a/storage/Cargo.toml b/storage/Cargo.toml index b5ea0322..7685351c 100644 --- a/storage/Cargo.toml +++ b/storage/Cargo.toml @@ -9,4 +9,5 @@ commonware-cryptography = { workspace = true } rand = "0.8.5" rocksdb = "0.23.0" commonware-codec = { version = "0.0.43" } -bytes = "1.7.1" \ No newline at end of file +bytes = "1.7.1" +tempfile = "3.18.0" \ No newline at end of file diff --git a/storage/src/account.rs b/storage/src/account.rs index dca506a2..a6d46ab6 100644 --- a/storage/src/account.rs +++ b/storage/src/account.rs @@ -34,8 +34,8 @@ impl Codec for Account { } fn read(reader: &mut impl Reader) -> Result { - let addr_bytes = <[u8; 32]>::read(reader)?; - let address = Address::from_bytes(&addr_bytes).unwrap(); + let addr_bytes = <[u8; 33]>::read(reader)?; + let address = Address::from_bytes(&addr_bytes[1..]).unwrap(); let balance = ::read(reader)?; Ok(Self{address, balance}) } diff --git a/storage/src/hashmap_db.rs b/storage/src/hashmap_db.rs index 1684e89a..6b2ce71b 100644 --- a/storage/src/hashmap_db.rs +++ b/storage/src/hashmap_db.rs @@ -17,12 +17,11 @@ impl HashmapDatabase { impl Database for HashmapDatabase { fn put(&mut self, key: &[u8], value: &[u8]) -> Result<(), Box> { - //self.data.insert(String::from_utf8(key.into())?, value.into()) - let str_value: String = String::from_utf8(value.into()).unwrap(); - match self.data.insert(String::from_utf8(key.into())?, str_value) { - Some(_) => Ok(()), - None => Err("hashmap db insert failed".into()), - } + let str_value: String = String::from_utf8(value.into())?; + let key_value: String = String::from_utf8(key.into())?; + + self.data.insert(key_value, str_value); + Ok(()) } fn get(&self, key: &[u8]) -> Result>, Box> { diff --git a/storage/src/rocks_db.rs b/storage/src/rocks_db.rs index a8af5a63..68c00fc0 100644 --- a/storage/src/rocks_db.rs +++ b/storage/src/rocks_db.rs @@ -1,12 +1,10 @@ use std::error::Error; -use alto_types::Address; use rocksdb::{DB, Options}; -use commonware_codec::{Codec, ReadBuffer, WriteBuffer}; -use crate::account::{Account, Balance}; +use commonware_codec::{Codec}; use crate::database::Database; use std::path::Path; -use bytes::{BufMut, Bytes}; - +use bytes::{BufMut}; +use tempfile::TempDir; const SAL_ROCKS_DB_PATH: &str = "rocksdb"; @@ -16,87 +14,73 @@ pub struct RocksDbDatabase { impl RocksDbDatabase { pub fn new() -> Result> { + Self::new_with_path(SAL_ROCKS_DB_PATH) + } + + pub fn new_with_path(path: &str) -> Result> { let mut opts = Options::default(); opts.create_if_missing(true); - let db_path = Path::new(SAL_ROCKS_DB_PATH); + let db_path = Path::new(path); let db = DB::open(&opts, &db_path)?; Ok(RocksDbDatabase { db }) } - /* - pub fn key_accounts(addr: &Address) -> Vec { - Self::make_multi_key(ACCOUNTS_PREFIX, addr.as_slice()) + pub fn new_tmp_db() -> Result> { + let temp_dir = TempDir::new()?; + let db_path = temp_dir.path().join("testdb"); + Self::new_with_path(db_path.to_str().unwrap()) } - fn make_multi_key(prefix: &[u8], sub_id: &[u8]) -> Vec { - let mut key = Vec::with_capacity(prefix.len() + sub_id.len() + 1); - key.extend_from_slice(prefix); - key.push(b':'); - key.extend_from_slice(sub_id); - key - } - */ } impl Database for RocksDbDatabase { -/* - fn get_account(&self, address: &Address) -> Result, Box> { - let key = Self::key_accounts(address); - let result = self.get(key.as_slice())?; - match result { - None => Ok(None), - Some(value) => { - let bytes = Bytes::copy_from_slice(&value); - let mut read_buf = ReadBuffer::new(bytes); - let acc = Account::read(&mut read_buf)?; - Ok(Some(acc)) - } - } - } - - fn set_account(&mut self, acc: &Account) -> Result<(), Box> { - let key = Self::key_accounts(&acc.address); - let mut write_buf = WriteBuffer::new(WRITE_BUFFER_CAPACITY as usize); - acc.write(&mut write_buf); - self.put(&key, write_buf.as_ref())?; + fn put(&mut self, key: &[u8], value: &[u8]) -> Result<(), Box> { + self.db.put(key, value)?; Ok(()) } - fn get_balance(&self, address: &Address) -> Option { - let result = self.get_account(address).and_then(|acc| { - match acc { - Some(acc) => Ok(acc.balance), - None => Ok(0), - } - }); - match result { - Some(balance) => Some(balance), - None => None, - } + fn get(&self, key: &[u8]) -> Result>, Box> { + let result = self.db.get(key)?; + Ok(result) } +} - fn set_balance(&mut self, address: &Address, amt: Balance) -> bool { - let result = self.get_account(address); - match result { - Some(mut acc) => { - acc.balance = amt; - let result = self.set_account(&acc); - result.is_ok() - }, - None => false, - } +#[cfg(test)] +mod tests { + use alto_types::Address; + use crate::account::Account; + use super::*; + #[test] + fn test_rocks_db_basic() { + let mut db = RocksDbDatabase::new().expect("db could not be created"); + let key = b"key1"; + let value = b"value1"; + db.put(key, value).unwrap(); + let retrieved = db.get(key).unwrap().unwrap(); + assert_eq!(retrieved.as_slice(), value); } - */ + #[test] + fn test_rocks_db_accounts() { + let mut db = RocksDbDatabase::new_tmp_db().expect("db could not be created"); - fn put(&mut self, key: &[u8], value: &[u8]) -> Result<(), Box> { - self.db.put(key, value)?; - Ok(()) - } + let mut account = Account::new(); + let test_address = Address::new(b"0xBEEF"); + account.address = test_address.clone(); + account.balance = 100; - fn get(&self, key: &[u8]) -> Result>, Box> { - let result = self.db.get(key)?; - Ok(result) + // get account for test address is empty + let empty_result = db.get_account(&test_address); + empty_result.unwrap().is_none(); + + // set account + db.set_account(&account).unwrap(); + + let acct_result = db.get_account(&test_address).unwrap(); + assert!(acct_result.is_some()); + let account = acct_result.unwrap(); + assert_eq!(account.address, test_address); + assert_eq!(account.balance, 100); } -} \ No newline at end of file +} diff --git a/types/src/lib.rs b/types/src/lib.rs index a03ed207..b9beda3c 100644 --- a/types/src/lib.rs +++ b/types/src/lib.rs @@ -1,6 +1,9 @@ //! Common types used throughout `alto`. mod block; + +use commonware_cryptography::{Ed25519, Scheme}; +use commonware_cryptography::ed25519::{PrivateKey, PublicKey}; pub use block::{Block, Finalized, Notarized}; mod consensus; pub use consensus::{leader_index, Finalization, Kind, Notarization, Nullification, Seed}; @@ -8,6 +11,7 @@ pub mod wasm; mod codec; use more_asserts; use more_asserts::assert_le; +use rand::rngs::OsRng; // We don't use functions here to guard against silent changes. pub const NAMESPACE: &[u8] = b"_ALTO"; @@ -55,6 +59,17 @@ impl Address { } } +pub fn create_test_keypair() -> (PublicKey, PrivateKey) { + let mut rng = OsRng; + // generates keypair using random number generator + let keypair = Ed25519::new(&mut rng); + + let public_key = keypair.public_key(); + let private_key = keypair.private_key(); + + (public_key, private_key) +} + #[cfg(test)] mod tests { use super::*; From 71a7172ad4ba5b6abb6a816653e7be00d3ef262f Mon Sep 17 00:00:00 2001 From: NateAtNodeKit Date: Thu, 27 Mar 2025 14:24:08 -0400 Subject: [PATCH 11/43] router scaffold --- Cargo.lock | 1 + chain/Cargo.toml | 1 + chain/src/actors/application/actor.rs | 1 + chain/src/actors/application/mod.rs | 2 + chain/src/actors/application/router.rs | 105 +++++++++++++++++++++++++ chain/src/engine.rs | 3 + 6 files changed, 113 insertions(+) create mode 100644 chain/src/actors/application/router.rs diff --git a/Cargo.lock b/Cargo.lock index fabca603..26ed7941 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -70,6 +70,7 @@ dependencies = [ "serde_yaml", "sysinfo", "thiserror 2.0.12", + "tokio", "tracing", "tracing-subscriber", "uuid", diff --git a/chain/Cargo.toml b/chain/Cargo.toml index 81ee85ea..e59b1946 100644 --- a/chain/Cargo.toml +++ b/chain/Cargo.toml @@ -37,6 +37,7 @@ axum = "0.8.1" uuid = "1.15.1" serde = { version = "1.0.218", features = ["derive"] } serde_yaml = "0.9.34" +tokio = "1.44.0" [[bin]] name = "validator" diff --git a/chain/src/actors/application/actor.rs b/chain/src/actors/application/actor.rs index 89094f93..c70cf1b3 100644 --- a/chain/src/actors/application/actor.rs +++ b/chain/src/actors/application/actor.rs @@ -89,6 +89,7 @@ impl Actor { let genesis_digest = genesis.digest(); let built: Option = None; let built = Arc::new(Mutex::new(built)); + while let Some(message) = self.mailbox.next().await { match message { Message::Genesis { response } => { diff --git a/chain/src/actors/application/mod.rs b/chain/src/actors/application/mod.rs index 70f2dcbc..72bb076a 100644 --- a/chain/src/actors/application/mod.rs +++ b/chain/src/actors/application/mod.rs @@ -10,6 +10,8 @@ pub use actor::Actor; mod ingress; pub use ingress::Mailbox; mod supervisor; +mod router; + pub use supervisor::Supervisor; /// Configuration for the application. diff --git a/chain/src/actors/application/router.rs b/chain/src/actors/application/router.rs new file mode 100644 index 00000000..c6975526 --- /dev/null +++ b/chain/src/actors/application/router.rs @@ -0,0 +1,105 @@ +use std::io; +use axum::response::IntoResponse; +use axum::routing::get; +use commonware_runtime::{Clock, Handle, Metrics, Spawner}; +use rand::Rng; +use tokio::net::TcpListener; +use tracing::{event, Level}; +use crate::actors::syncer; + +pub struct RouterConfig { + port: i32, +} + +impl RouterConfig { + pub fn default_config() -> RouterConfig { + RouterConfig { + port: 7844, + } + } +} + +pub struct Router { + context: R, + cfg: RouterConfig, + listener: Option, + router: Option, + is_active: bool, +} + +impl Router { + const PATH_SUBMIT_BLOCK: &'static str = "/builder/submit"; + + pub fn new(context: R, cfg: RouterConfig) -> Self { + if cfg.port == 0 { + panic!("Invalid port number"); + } + + Router { + context, + cfg, + listener: None, + router: None, + is_active: false, + } + } + + pub async fn start(mut self) -> Handle<()> { + self.context.spawn_ref()(self.run()) + } + + pub fn stop(&self) { + if !self.is_active { + return + } + + event!(Level::INFO, "stopped router service"); + } + + async fn init_listener(&mut self) -> io::Result { + let listener = TcpListener::bind(format!("127.0.0.1:{}", self.cfg.port)).await?; + Ok(listener) + } + + async fn handle_default() -> impl IntoResponse { + "Hello world!" + } + + async fn handle_submit_block() -> impl IntoResponse { + "Submit block" + } + + fn init_router(&mut self) { + let router = axum::Router::new() + .route("/", get(Router::handle_default)) + .route(Router::PATH_SUBMIT_BLOCK, get(Router::handle_submit_block())); + + self.router = Some(router) + } + + async fn serve(&mut self) -> Result<(), Box> { + let listener = self.listener.take().ok_or("serve failed because listener is None"); + let router = self.router.take().ok_or("serve failed because router is None"); + axum::serve(listener.unwrap(), router.unwrap()).await?; + Ok(()) + } + + async fn run(mut self) { + event!(Level::INFO, "starting router service"); + + let listener_res = self.init_listener(); + match listener_res.await { + Ok(value) => self.listener = Some(value), + Err(error) => { + println!("Error during listener init: {}", error); + return + }, + } + + self.init_router(); + self.serve().await.unwrap(); + self.is_active = true; + + event!(Level::INFO, "finished starting router service"); + } +} diff --git a/chain/src/engine.rs b/chain/src/engine.rs index 2bb8dabc..096065a1 100644 --- a/chain/src/engine.rs +++ b/chain/src/engine.rs @@ -201,6 +201,9 @@ impl + Metri // Start consensus let consensus_handle = self.consensus.start(voter_network, resolver_network); + // Start the router + let router_config = RouterConfig::default_config(); + // Wait for any actor to finish if let Err(e) = try_join_all(vec![application_handle, syncer_handle, consensus_handle]).await From ab18e5412aaeb244ef491923162700df0e73f194 Mon Sep 17 00:00:00 2001 From: rikoeldon <106416799+rikoeldon@users.noreply.github.com> Date: Thu, 27 Mar 2025 15:04:52 -0400 Subject: [PATCH 12/43] init types for SAL --- chain/src/engine.rs | 2 +- storage/src/database.rs | 12 ------ storage/src/lib.rs | 3 +- storage/src/tx_state_view.rs | 37 +++++++++++++++++ types/src/lib.rs | 5 +++ types/src/signed_tx.rs | 26 ++++++++++++ types/src/state_db.rs | 11 +++++ types/src/tx.rs | 78 ++++++++++++++++++++++++++++++++++++ types/src/wallet.rs | 37 +++++++++++++++++ 9 files changed, 197 insertions(+), 14 deletions(-) create mode 100644 storage/src/tx_state_view.rs create mode 100644 types/src/signed_tx.rs create mode 100644 types/src/state_db.rs create mode 100644 types/src/tx.rs create mode 100644 types/src/wallet.rs diff --git a/chain/src/engine.rs b/chain/src/engine.rs index 096065a1..8536fbd6 100644 --- a/chain/src/engine.rs +++ b/chain/src/engine.rs @@ -202,7 +202,7 @@ impl + Metri let consensus_handle = self.consensus.start(voter_network, resolver_network); // Start the router - let router_config = RouterConfig::default_config(); + // let router_config = RouterConfig::default_config(); // Wait for any actor to finish if let Err(e) = diff --git a/storage/src/database.rs b/storage/src/database.rs index 68744de3..e600426b 100644 --- a/storage/src/database.rs +++ b/storage/src/database.rs @@ -10,18 +10,6 @@ const DB_WRITE_BUFFER_CAPACITY: usize = 500; // Define database interface that will be used for all impls pub trait Database { - /* - fn get_account(&self, address: &Address) -> Option; - - fn set_account(&mut self, acc: &Account) -> Result<(), Box>; - - fn get_balance(&self, address: &Address) -> Option; - - fn set_balance(&mut self, address: &Address, amt: u64) -> bool; - - */ - - fn get_account(&self, address: &Address) -> Result, Box> { let key = Self::key_accounts(address); let result = self.get(key.as_slice())?; diff --git a/storage/src/lib.rs b/storage/src/lib.rs index 53447579..6b6c512a 100644 --- a/storage/src/lib.rs +++ b/storage/src/lib.rs @@ -1,4 +1,5 @@ pub mod account; pub mod database; pub mod hashmap_db; -mod rocks_db; \ No newline at end of file +mod rocks_db; +mod tx_state_view; \ No newline at end of file diff --git a/storage/src/tx_state_view.rs b/storage/src/tx_state_view.rs new file mode 100644 index 00000000..0d2ef71e --- /dev/null +++ b/storage/src/tx_state_view.rs @@ -0,0 +1,37 @@ +use std::collections::HashMap; +use crate::database::Database; +use std::error::Error; + +type Key<'a> = &'a [u8;33]; // 1st byte denotes the type of key. 0b for account key, 1b for others. + +pub enum OpAction { + Read, // key was read + Create, // key was created + Update, // key was updated + Delete, // key got deleted +} + +pub struct Op<'a>{ + pub action: OpAction, + pub key: Key<'a>, + pub value: Vec, +} + +pub struct TxStateView<'a, DB: Database> { + pub cache: HashMap, Vec>, // key-value, state view cache before tx execution. This is not an exhaustive list of all state keys read/write during tx. If cache misses occur, the state view will read from the underlying storage. + pub ops: Vec>, // list of state ops applied. + pub touched: HashMap, Vec>, // key-value pairs that were changed during tx execution. This is a subset of the cache. + pub db: DB, // underlying state storage, to use when cache misses occur. +} + + +pub trait TxStateViewTrait { + fn new(db: DB) -> Self; // create a new instance of TxStateView. + fn init_cache(&mut self, cache: HashMap>); // initialize the cache with an already available hashmap of key-value pairs. + fn get(&mut self, key: Key) -> Result>, Box>; // get a key from the state view. If the key is not in the touched and cache, it will read from the underlying storage. add the key-value pair to `touched` and append op to `ops`. + fn get_from_cache(&self, key: Key) -> Result>, Box>; // get a key from the cache. If the key is not in the cache, it will return an error. + fn get_from_state(&self, key: Key) -> Result>, Box>; // get a key from the underlying storage. If the key is not in the storage, it will return an error. + fn update(&mut self, key: Key, value: Vec) -> Result<(), Box>; // update a key in the state view. add the key-value pair to `touched` and append op to `ops`. + fn delete(&mut self, key: Key) -> Result<(), Box>; // delete a key in the state view. add the key-value pair to `touched` and append op to `ops`. + fn rollback(&mut self); // rollback the state view to the state before the tx execution. +} diff --git a/types/src/lib.rs b/types/src/lib.rs index b9beda3c..4b675f12 100644 --- a/types/src/lib.rs +++ b/types/src/lib.rs @@ -9,6 +9,11 @@ mod consensus; pub use consensus::{leader_index, Finalization, Kind, Notarization, Nullification, Seed}; pub mod wasm; mod codec; +mod wallet; +mod tx; +mod signed_tx; +mod state_db; + use more_asserts; use more_asserts::assert_le; use rand::rngs::OsRng; diff --git a/types/src/signed_tx.rs b/types/src/signed_tx.rs new file mode 100644 index 00000000..eb0c9b0b --- /dev/null +++ b/types/src/signed_tx.rs @@ -0,0 +1,26 @@ +use crate::Address; +use crate::wallet::{AuthTypes, Wallet}; +use crate::tx::{Tx, Unit}; + +// this is sent by the user to the validators. +pub struct SignedTx<'a> { + pub tx: Tx<'a>, + pub auth_type: AuthTypes, + + + pub_key: Vec, + address: Address, + signature: Vec, +} + + +pub trait SignedTxChars { + fn new(tx: Tx, auth_type: AuthTypes) -> Self; + fn sign(&self, wallet: Wallet) -> SignedTx; + fn verify(&self) -> bool; + fn signature(&self) -> Vec; + fn public_key(&self) -> Vec; + fn address(&self) -> Address; + fn encode(&self) -> Vec; + fn decode(&self, bytes: &[u8]) -> Self; +} diff --git a/types/src/state_db.rs b/types/src/state_db.rs new file mode 100644 index 00000000..6ec7e9bd --- /dev/null +++ b/types/src/state_db.rs @@ -0,0 +1,11 @@ +use std::error::Error; + +type Key<'a> = &'a [u8;33]; // 1st byte denotes the type of key. 0b for account key, 1b for others. +pub trait StateDB { + fn get(&self, key: Key) -> Result>, Box>; + fn get_multi_key(&self, key: Vec) -> Result>, Box>; + fn update(&mut self, key: Key, value: Vec) -> Result<(), Box>; + fn delete(&mut self, key: Key) -> Result<(), Box>; + fn commit(&mut self) -> Result<(), Box>; + fn rollback(&mut self) -> Result<(), Box>; +} \ No newline at end of file diff --git a/types/src/tx.rs b/types/src/tx.rs new file mode 100644 index 00000000..ba09ae2d --- /dev/null +++ b/types/src/tx.rs @@ -0,0 +1,78 @@ +use std::error::Error; +use crate::Address; +use crate::state_db::StateDB; +use crate::wallet::Wallet; +use crate::signed_tx::SignedTx; + +pub enum UnitType { + Transfer, + SequencerMsg, +} + + +pub struct UnitContext { + pub timestamp: u64, // timestamp of the tx. + pub chain_id: u64, // chain id of the tx. + pub sender: Address, // sender of the tx. +} + + +// unit need to be simple and easy to be packed in the tx and executed by the vm. +pub trait Unit { + // type DB: Database; + // type T: TxStateViewTrait; + + + fn unit_type(&self) -> UnitType; // return the unit type. + fn encode(&self) -> Vec; // encode the unit. + fn decode(bytes: &[u8]) -> Self; // decode the unit. + fn apply(&self, context: &UnitContext, state: &mut dyn StateDB) -> Result>, Box >; // apply the unit to the state view. +} + + +// pub struct Tx<'a, U: Unit> { +pub struct Tx<'a> { + // timestamp of the tx creation. set by the user. + // will be verified if the tx is in the valid window once received by validators. + // if the timestamp is not in the valid window, the tx will be rejected. + // if tx is in a valid window it is added to mempool. + // timestamp is used to prevent replay attacks. and counter infinite spam attacks as Tx does not have nonce. + pub timestamp: u64, + // units are fundamental unit of a tx. similar to actions. + pub units: Vec, + // max fee is the maximum fee the user is willing to pay for the tx. + pub max_fee: u64, + // priority fee is the fee the user is willing to pay for the tx to be included in the next block. + pub priority_fee: u64, + // chain id is the id of the chain the tx is intended for. + pub chain_id: u64, + + + // id is the transaction id. It is the hash of digest. + id: &'a [u8;32], + // digest is encoded tx. + digest: Vec, +} + + +pub trait TxChars { + // init is used to create a new instance of Tx. + fn init() -> Self; + // new is used to create a new instance of Tx with given units and chain id. + fn new(units: Vec, chain_id: u64) -> Self; + // set_fee is used to set the max fee and priority fee of the tx. + fn set_fee(&mut self, max_fee: u64, priority_fee: u64); + // sign is used to sign the tx with the given wallet. + fn sign(&self, wallet: Wallet) -> SignedTx; + + + // returns tx id. + fn id(&self) -> &[u8;32]; + // returns digest of the tx. + fn digest(&self) -> Vec; + // encodes the tx, writes to digest and returns the digest. + fn encode(&mut self) -> Vec; + + + fn decode(bytes: &[u8]) -> Self; +} diff --git a/types/src/wallet.rs b/types/src/wallet.rs new file mode 100644 index 00000000..2e93bfef --- /dev/null +++ b/types/src/wallet.rs @@ -0,0 +1,37 @@ +use crate::Address; +pub enum AuthTypes { + ED25519, +} + + +// auth should have a method to verify signatures. +// also batch signature verification. +pub trait Auth { + fn public_key(&self) -> Vec; // return the public key of the signer. + fn address(&self) -> Address; // return the account address of the signer. + fn verify(&self, data: &[u8], signature: &[u8]) -> bool; // verify a signature. + fn batch_verify(&self, data: &[u8], signatures: Vec<&[u8]>) -> bool; // batch verify signatures. returns error if batch verification fails. +} + + +pub struct Wallet { + auth: AuthTypes, // auth type + p_key: Vec, // private key + pub_key: Vec, // public key + address: Address, // account address +} + + +// wallet generation, management, and signing should be functions of the wallet. +pub trait ImplWallet { + fn generate(auth_type: AuthTypes) -> Self; // create a new wallet of given auth type. + fn load(&self, auth_type: AuthTypes, p_key: &[u8]) -> Self; // load a new wallet from private key. + fn sign(&self, data: &[u8]) -> Vec; // sign data with the private key. + fn verify(&self,data: &[u8], signature: &[u8]) -> bool; // verify a signature with the public key. + fn address(&self) -> Address; // return the account address. + fn public_key(&self) -> Vec; // return the public key. + fn auth_type(&self) -> AuthTypes; // return the auth type. + fn private_key(&self) -> Vec; // returns the private key. + fn store_private_key(&self); // store the private key. + fn new_address(&mut self); // generate a new address. +} From 6883900d4b67f311df9cc533ad13b48f0ab78edd Mon Sep 17 00:00:00 2001 From: rikoeldon <106416799+rikoeldon@users.noreply.github.com> Date: Thu, 27 Mar 2025 16:22:26 -0400 Subject: [PATCH 13/43] state & DB setup --- storage/src/database.rs | 68 ++------------------------- storage/src/hashmap_db.rs | 7 ++- storage/src/lib.rs | 4 +- storage/src/rocks_db.rs | 3 +- storage/src/state_db.rs | 75 +++++++++++++++++++++++++++++ storage/src/tx_state_view.rs | 78 ++++++++++++++++++++++++------- {storage => types}/src/account.rs | 7 +-- types/src/address.rs | 38 +++++++++++++++ types/src/lib.rs | 51 ++++---------------- types/src/signed_tx.rs | 8 ++-- types/src/state.rs | 11 +++++ types/src/state_db.rs | 11 ----- types/src/tx.rs | 4 +- types/src/wallet.rs | 10 ++-- vm/src/vm.rs | 2 +- 15 files changed, 222 insertions(+), 155 deletions(-) create mode 100644 storage/src/state_db.rs rename {storage => types}/src/account.rs (80%) create mode 100644 types/src/address.rs create mode 100644 types/src/state.rs delete mode 100644 types/src/state_db.rs diff --git a/storage/src/database.rs b/storage/src/database.rs index e600426b..87425cda 100644 --- a/storage/src/database.rs +++ b/storage/src/database.rs @@ -1,75 +1,13 @@ -use alto_types::Address; -use crate::account::{Balance, Account}; +use alto_types::account::{Balance, Account}; use std::error::Error; use bytes::Bytes; use commonware_codec::{Codec, ReadBuffer, WriteBuffer}; - -const ACCOUNTS_PREFIX: &[u8] = b"sal_accounts"; - -const DB_WRITE_BUFFER_CAPACITY: usize = 500; +use alto_types::address::Address; // Define database interface that will be used for all impls pub trait Database { - fn get_account(&self, address: &Address) -> Result, Box> { - let key = Self::key_accounts(address); - let result = self.get(key.as_slice())?; - match result { - None => Ok(None), - Some(value) => { - let bytes = Bytes::copy_from_slice(&value); - let mut read_buf = ReadBuffer::new(bytes); - let acc = Account::read(&mut read_buf)?; - Ok(Some(acc)) - } - } - } - - fn set_account(&mut self, acc: &Account) -> Result<(), Box> { - let key = Self::key_accounts(&acc.address); - let mut write_buf = WriteBuffer::new(DB_WRITE_BUFFER_CAPACITY); - acc.write(&mut write_buf); - self.put(&key, write_buf.as_ref())?; - Ok(()) - } - - fn get_balance(&self, address: &Address) -> Option { - let result = self.get_account(address).and_then(|acc| { - match acc { - Some(acc) => Ok(acc.balance), - None => Ok(0), - } - }); - match result { - Ok(balance) => Some(balance), - _ => None, - } - } - - fn set_balance(&mut self, address: &Address, amt: Balance) -> bool { - let result = self.get_account(address); - match result { - Ok(Some(mut acc)) => { - acc.balance = amt; - let result = self.set_account(&acc); - result.is_ok() - }, - _ => false, - } - } - - fn key_accounts(addr: &Address) -> Vec { - Self::make_multi_key(ACCOUNTS_PREFIX, addr.as_slice()) - } - - fn make_multi_key(prefix: &[u8], sub_id: &[u8]) -> Vec { - let mut key = Vec::with_capacity(prefix.len() + sub_id.len() + 1); - key.extend_from_slice(prefix); - key.push(b':'); - key.extend_from_slice(sub_id); - key - } - fn put(&mut self, key: &[u8], value: &[u8]) -> Result<(), Box>; fn get(&self, key: &[u8]) -> Result>, Box>; + fn delete(&mut self, key: &[u8]) -> Result<(), Box>; } \ No newline at end of file diff --git a/storage/src/hashmap_db.rs b/storage/src/hashmap_db.rs index 6b2ce71b..23f256f8 100644 --- a/storage/src/hashmap_db.rs +++ b/storage/src/hashmap_db.rs @@ -1,6 +1,5 @@ use std::collections::HashMap; -use alto_types::Address; -use crate::account::{Account, Balance}; +use std::error::Error; use crate::database::Database; pub struct HashmapDatabase { @@ -30,6 +29,10 @@ impl Database for HashmapDatabase { Ok(None), |v| Ok(Some(v.clone().into()))) } + + fn delete(&mut self, key: &[u8]) -> Result<(), Box> { + todo!() + } } diff --git a/storage/src/lib.rs b/storage/src/lib.rs index 6b6c512a..ae0cda68 100644 --- a/storage/src/lib.rs +++ b/storage/src/lib.rs @@ -1,5 +1,5 @@ -pub mod account; pub mod database; pub mod hashmap_db; mod rocks_db; -mod tx_state_view; \ No newline at end of file +mod tx_state_view; +mod state_db; \ No newline at end of file diff --git a/storage/src/rocks_db.rs b/storage/src/rocks_db.rs index 68c00fc0..f559289f 100644 --- a/storage/src/rocks_db.rs +++ b/storage/src/rocks_db.rs @@ -49,7 +49,8 @@ impl Database for RocksDbDatabase { #[cfg(test)] mod tests { use alto_types::Address; - use crate::account::Account; + use alto_types::account::Account; + use alto_types::address::Address; use super::*; #[test] fn test_rocks_db_basic() { diff --git a/storage/src/state_db.rs b/storage/src/state_db.rs new file mode 100644 index 00000000..e5c124f8 --- /dev/null +++ b/storage/src/state_db.rs @@ -0,0 +1,75 @@ +use crate::database::Database; +use alto_types::account::{Account, Balance}; +use alto_types::address::Address; +use bytes::Bytes; +use commonware_codec::{Codec, ReadBuffer, WriteBuffer}; +use std::error::Error; + +const ACCOUNTS_PREFIX: &[u8] = b"sal_accounts"; +const DB_WRITE_BUFFER_CAPACITY: usize = 500; + +pub struct StateDb { + db: Box, +} +// can use like a redis from Arcadia like get and set for diff types? +impl StateDb { + pub fn new(db: Box) -> StateDb { + StateDb { db } + } + + pub fn get_account(&self, address: &Address) -> Result, Box> { + let key = Self::key_accounts(address); + let result = self.db.get(key.as_slice())?; + match result { + None => Ok(None), + Some(value) => { + let bytes = Bytes::copy_from_slice(&value); + let mut read_buf = ReadBuffer::new(bytes); + let acc = Account::read(&mut read_buf)?; + Ok(Some(acc)) + } + } + } + + pub fn set_account(&mut self, acc: &Account) -> Result<(), Box> { + let key = Self::key_accounts(&acc.address); + let mut write_buf = WriteBuffer::new(DB_WRITE_BUFFER_CAPACITY); + acc.write(&mut write_buf); + self.db.put(&key, write_buf.as_ref()).expect("TODO: panic message"); + Ok(()) + } + + pub fn get_balance(&self, address: &Address) -> Option { + let result = self.get_account(address).and_then(|acc| match acc { + Some(acc) => Ok(acc.balance), + None => Ok(0), + }); + match result { + Ok(balance) => Some(balance), + _ => None, + } + } + + pub fn set_balance(&mut self, address: &Address, amt: Balance) -> bool { + let result = self.get_account(address); + match result { + Ok(Some(mut acc)) => { + acc.balance = amt; + let result = self.set_account(&acc); + result.is_ok() + } + _ => false, + } + } + + fn key_accounts(addr: &Address) -> Vec { + Self::make_multi_key(ACCOUNTS_PREFIX, addr.as_slice()) + } + fn make_multi_key(prefix: &[u8], sub_id: &[u8]) -> Vec { + let mut key = Vec::with_capacity(prefix.len() + sub_id.len() + 1); + key.extend_from_slice(prefix); + key.push(b':'); + key.extend_from_slice(sub_id); + key + } +} diff --git a/storage/src/tx_state_view.rs b/storage/src/tx_state_view.rs index 0d2ef71e..29ef5af8 100644 --- a/storage/src/tx_state_view.rs +++ b/storage/src/tx_state_view.rs @@ -1,8 +1,12 @@ use std::collections::HashMap; use crate::database::Database; use std::error::Error; +use rocksdb::LogLevel::Error; +use alto_types::state::{State}; +use crate::state_db::StateDb; -type Key<'a> = &'a [u8;33]; // 1st byte denotes the type of key. 0b for account key, 1b for others. +const ACCOUNT_KEY_TYPE: u8 = 0; +type UnitKey<'a> = alto_types::state::UnitKey<'a>; // 1st byte denotes the type of key. 0b for account key, 1b for others. pub enum OpAction { Read, // key was read @@ -13,25 +17,67 @@ pub enum OpAction { pub struct Op<'a>{ pub action: OpAction, - pub key: Key<'a>, + pub key: UnitKey<'a>, pub value: Vec, } -pub struct TxStateView<'a, DB: Database> { - pub cache: HashMap, Vec>, // key-value, state view cache before tx execution. This is not an exhaustive list of all state keys read/write during tx. If cache misses occur, the state view will read from the underlying storage. +pub trait TxStateViewTrait: State { + fn init_cache(&mut self, cache: HashMap>); // initialize the cache with an already available hashmap of key-value pairs. + fn get_from_cache(&self, key: UnitKey) -> Result>, Box>; // get a key from the cache. If the key is not in the cache, it will return an error. + fn get_from_state(&self, key: UnitKey) -> Result>, Box>; // get a key from the underlying storage. If the key is not in the storage, it will return an error. +} + +pub struct TxStateView<'a> { + pub cache: HashMap, Vec>, // key-value, state view cache before tx execution. This is not an exhaustive list of all state keys read/write during tx. If cache misses occur, the state view will read from the underlying storage. pub ops: Vec>, // list of state ops applied. - pub touched: HashMap, Vec>, // key-value pairs that were changed during tx execution. This is a subset of the cache. - pub db: DB, // underlying state storage, to use when cache misses occur. + pub touched: HashMap, Vec>, // key-value pairs that were changed during tx execution. This is a subset of the cache. + pub state_db: StateDb, // underlying state storage, to use when cache misses occur. } +impl<'a> TxStateView<'a> { + pub fn new(state_db: StateDb) -> Self { + Self{ + cache: HashMap::new(), + ops: Vec::new(), + touched: HashMap::new(), + state_db, + } + } + // initialize the cache with an already available hashmap of key-value pairs. + pub fn init_cache(&mut self, cache: HashMap>) { + self.cache = cache; + } + pub fn get_from_cache(&self, key: UnitKey) -> Result>, Box> { + self.cache.get(&key).map_or( + Ok(None), + |v| Ok(Some(v.clone().into()))) + } -pub trait TxStateViewTrait { - fn new(db: DB) -> Self; // create a new instance of TxStateView. - fn init_cache(&mut self, cache: HashMap>); // initialize the cache with an already available hashmap of key-value pairs. - fn get(&mut self, key: Key) -> Result>, Box>; // get a key from the state view. If the key is not in the touched and cache, it will read from the underlying storage. add the key-value pair to `touched` and append op to `ops`. - fn get_from_cache(&self, key: Key) -> Result>, Box>; // get a key from the cache. If the key is not in the cache, it will return an error. - fn get_from_state(&self, key: Key) -> Result>, Box>; // get a key from the underlying storage. If the key is not in the storage, it will return an error. - fn update(&mut self, key: Key, value: Vec) -> Result<(), Box>; // update a key in the state view. add the key-value pair to `touched` and append op to `ops`. - fn delete(&mut self, key: Key) -> Result<(), Box>; // delete a key in the state view. add the key-value pair to `touched` and append op to `ops`. - fn rollback(&mut self); // rollback the state view to the state before the tx execution. -} + pub fn get_from_state(&self, key: UnitKey) -> Result>, Box> { + match key[0] { + ACCOUNT_KEY_TYPE => { + self.get_from_state(key) + }, + _ => Err(format!("invalid state key {:?}", key[0]).into()) + } + } + + pub fn get(&self, key: alto_types::state::UnitKey) -> Result>, Box>{ + todo!() + } + pub fn get_multi_key(&self, key: Vec) -> Result>, Box> { + todo!() + } + pub fn update(&mut self, key: alto_types::state::UnitKey, value: Vec) -> Result<(), Box>{ + todo!() + } + pub fn delete(&mut self, key: alto_types::state::UnitKey) -> Result<(), Box> { + todo!() + } + pub fn commit(&mut self) -> Result<(), Box> { + todo!() + } + pub fn rollback(&mut self) -> Result<(), Box> { + todo!() + } +} \ No newline at end of file diff --git a/storage/src/account.rs b/types/src/account.rs similarity index 80% rename from storage/src/account.rs rename to types/src/account.rs index a6d46ab6..fb4faea5 100644 --- a/storage/src/account.rs +++ b/types/src/account.rs @@ -1,8 +1,5 @@ -use commonware_codec::{Codec, Error, Reader, SizedCodec, Writer}; -use commonware_cryptography::{ed25519::PublicKey, Ed25519, Scheme}; -use commonware_cryptography::ed25519::PrivateKey; -use alto_types::Address; -use rand::rngs::OsRng; +use commonware_codec::{Codec, Error, Reader, Writer}; +use crate::address::Address; pub type Balance = u64; diff --git a/types/src/address.rs b/types/src/address.rs new file mode 100644 index 00000000..53aff458 --- /dev/null +++ b/types/src/address.rs @@ -0,0 +1,38 @@ +use more_asserts::assert_le; +use crate::ADDRESSLEN; + +#[derive(Hash, Eq, PartialEq, Clone, Debug)] +pub struct Address(pub [u8;ADDRESSLEN]); + +impl Address { + pub fn new(slice: &[u8]) -> Self { + assert_le!(slice.len(), ADDRESSLEN, "address slice is too large"); + let mut arr = [0u8; ADDRESSLEN]; + arr[..slice.len()].copy_from_slice(slice); + Address(arr) + } + + pub fn from_bytes(bytes: &[u8]) -> Result { + if bytes.len() != 32 { + return Err("Address must be 32 bytes."); + } + + Ok(Address(<[u8; 32]>::try_from(bytes.clone()).unwrap())) + } + + pub fn empty() -> Self { + Self([0;ADDRESSLEN]) + } + + pub fn is_empty(&self) -> bool { + self.0 == Self::empty().0 + } + + pub fn as_slice(&self) -> &[u8] { + &self.0 + } + + pub fn as_bytes(&self) -> &[u8;ADDRESSLEN] { + &self.0 + } +} \ No newline at end of file diff --git a/types/src/lib.rs b/types/src/lib.rs index 4b675f12..5dd5075e 100644 --- a/types/src/lib.rs +++ b/types/src/lib.rs @@ -3,19 +3,19 @@ mod block; use commonware_cryptography::{Ed25519, Scheme}; -use commonware_cryptography::ed25519::{PrivateKey, PublicKey}; pub use block::{Block, Finalized, Notarized}; mod consensus; pub use consensus::{leader_index, Finalization, Kind, Notarization, Nullification, Seed}; pub mod wasm; -mod codec; -mod wallet; -mod tx; -mod signed_tx; -mod state_db; +pub mod codec; +pub mod wallet; +pub mod tx; +pub mod signed_tx; +pub mod state; +pub mod address; +pub mod account; use more_asserts; -use more_asserts::assert_le; use rand::rngs::OsRng; // We don't use functions here to guard against silent changes. @@ -28,41 +28,8 @@ pub const FINALIZE_NAMESPACE: &[u8] = b"_ALTO_FINALIZE"; const ADDRESSLEN: usize = 32; -#[derive(Hash, Eq, PartialEq, Clone, Debug)] -pub struct Address(pub [u8;ADDRESSLEN]); - -impl Address { - pub fn new(slice: &[u8]) -> Self { - assert_le!(slice.len(), ADDRESSLEN, "address slice is too large"); - let mut arr = [0u8; ADDRESSLEN]; - arr[..slice.len()].copy_from_slice(slice); - Address(arr) - } - - pub fn from_bytes(bytes: &[u8]) -> Result { - if bytes.len() != 32 { - return Err("Address must be 32 bytes."); - } - - Ok(Address(<[u8; 32]>::try_from(bytes.clone()).unwrap())) - } - - pub fn empty() -> Self { - Self([0;ADDRESSLEN]) - } - - pub fn is_empty(&self) -> bool { - self.0 == Self::empty().0 - } - - pub fn as_slice(&self) -> &[u8] { - &self.0 - } - - pub fn as_bytes(&self) -> &[u8;ADDRESSLEN] { - &self.0 - } -} +type PublicKey = commonware_cryptography::ed25519::PublicKey; +type PrivateKey = commonware_cryptography::ed25519::PrivateKey; pub fn create_test_keypair() -> (PublicKey, PrivateKey) { let mut rng = OsRng; diff --git a/types/src/signed_tx.rs b/types/src/signed_tx.rs index eb0c9b0b..1aec2917 100644 --- a/types/src/signed_tx.rs +++ b/types/src/signed_tx.rs @@ -1,14 +1,14 @@ -use crate::Address; +use crate::address::Address; +use crate::PublicKey; use crate::wallet::{AuthTypes, Wallet}; -use crate::tx::{Tx, Unit}; +use crate::tx::Tx; // this is sent by the user to the validators. pub struct SignedTx<'a> { pub tx: Tx<'a>, pub auth_type: AuthTypes, - - pub_key: Vec, + pub_key: PublicKey, address: Address, signature: Vec, } diff --git a/types/src/state.rs b/types/src/state.rs new file mode 100644 index 00000000..cdd3e5f9 --- /dev/null +++ b/types/src/state.rs @@ -0,0 +1,11 @@ +use std::error::Error; + +pub type UnitKey<'a> = &'a [u8;33]; // 1st byte denotes the type of key. 0b for account key, 1b for others. +pub trait State { + fn get(&self, key: UnitKey) -> Result>, Box>; + fn get_multi_key(&self, key: Vec) -> Result>, Box>; + fn update(&mut self, key: UnitKey, value: Vec) -> Result<(), Box>; + fn delete(&mut self, key: UnitKey) -> Result<(), Box>; + fn commit(&mut self) -> Result<(), Box>; + fn rollback(&mut self) -> Result<(), Box>; +} \ No newline at end of file diff --git a/types/src/state_db.rs b/types/src/state_db.rs deleted file mode 100644 index 6ec7e9bd..00000000 --- a/types/src/state_db.rs +++ /dev/null @@ -1,11 +0,0 @@ -use std::error::Error; - -type Key<'a> = &'a [u8;33]; // 1st byte denotes the type of key. 0b for account key, 1b for others. -pub trait StateDB { - fn get(&self, key: Key) -> Result>, Box>; - fn get_multi_key(&self, key: Vec) -> Result>, Box>; - fn update(&mut self, key: Key, value: Vec) -> Result<(), Box>; - fn delete(&mut self, key: Key) -> Result<(), Box>; - fn commit(&mut self) -> Result<(), Box>; - fn rollback(&mut self) -> Result<(), Box>; -} \ No newline at end of file diff --git a/types/src/tx.rs b/types/src/tx.rs index ba09ae2d..53a251fb 100644 --- a/types/src/tx.rs +++ b/types/src/tx.rs @@ -1,6 +1,6 @@ use std::error::Error; -use crate::Address; -use crate::state_db::StateDB; +use crate::address::Address; +use crate::state::StateDB; use crate::wallet::Wallet; use crate::signed_tx::SignedTx; diff --git a/types/src/wallet.rs b/types/src/wallet.rs index 2e93bfef..106686da 100644 --- a/types/src/wallet.rs +++ b/types/src/wallet.rs @@ -1,4 +1,6 @@ -use crate::Address; +use crate::address::Address; +use crate::{PrivateKey, PublicKey}; + pub enum AuthTypes { ED25519, } @@ -7,7 +9,7 @@ pub enum AuthTypes { // auth should have a method to verify signatures. // also batch signature verification. pub trait Auth { - fn public_key(&self) -> Vec; // return the public key of the signer. + fn public_key(&self) -> PublicKey; // return the public key of the signer. fn address(&self) -> Address; // return the account address of the signer. fn verify(&self, data: &[u8], signature: &[u8]) -> bool; // verify a signature. fn batch_verify(&self, data: &[u8], signatures: Vec<&[u8]>) -> bool; // batch verify signatures. returns error if batch verification fails. @@ -16,8 +18,8 @@ pub trait Auth { pub struct Wallet { auth: AuthTypes, // auth type - p_key: Vec, // private key - pub_key: Vec, // public key + p_key: PrivateKey, // private key + pub_key: PublicKey, // public key address: Address, // account address } diff --git a/vm/src/vm.rs b/vm/src/vm.rs index 8859df3a..e6d65bb1 100644 --- a/vm/src/vm.rs +++ b/vm/src/vm.rs @@ -1,7 +1,7 @@ use alto_actions::transfer::{Transfer, TransferError}; use alto_storage::database::Database; use alto_storage::hashmap_db::HashmapDatabase; -use alto_storage::account::Balance; +use alto_types::account::Balance; use alto_types::Address; const TEST_FAUCET_ADDRESS: &[u8] = b"0x0123456789abcdef0123456789abcd"; From 24b820baae75a217451fbe36da48290618b080fd Mon Sep 17 00:00:00 2001 From: NateAtNodeKit Date: Thu, 27 Mar 2025 18:02:21 -0400 Subject: [PATCH 14/43] state update --- storage/src/database.rs | 3 ++- storage/src/hashmap_db.rs | 6 +++-- storage/src/rocks_db.rs | 31 ++++-------------------- storage/src/state_db.rs | 47 ++++++++++++++++++++++++++++++++++++ storage/src/tx_state_view.rs | 41 +++++++++++++++++++++++++------ types/src/tx.rs | 25 ++++++++++--------- 6 files changed, 105 insertions(+), 48 deletions(-) diff --git a/storage/src/database.rs b/storage/src/database.rs index 87425cda..032938f4 100644 --- a/storage/src/database.rs +++ b/storage/src/database.rs @@ -8,6 +8,7 @@ use alto_types::address::Address; pub trait Database { fn put(&mut self, key: &[u8], value: &[u8]) -> Result<(), Box>; - fn get(&self, key: &[u8]) -> Result>, Box>; + fn get(&self, key: &[u8]) -> Result, Box>; + fn delete(&mut self, key: &[u8]) -> Result<(), Box>; } \ No newline at end of file diff --git a/storage/src/hashmap_db.rs b/storage/src/hashmap_db.rs index 23f256f8..d3396498 100644 --- a/storage/src/hashmap_db.rs +++ b/storage/src/hashmap_db.rs @@ -16,8 +16,8 @@ impl HashmapDatabase { impl Database for HashmapDatabase { fn put(&mut self, key: &[u8], value: &[u8]) -> Result<(), Box> { - let str_value: String = String::from_utf8(value.into())?; let key_value: String = String::from_utf8(key.into())?; + let str_value: String = String::from_utf8(value.into())?; self.data.insert(key_value, str_value); Ok(()) @@ -31,7 +31,9 @@ impl Database for HashmapDatabase { } fn delete(&mut self, key: &[u8]) -> Result<(), Box> { - todo!() + let key_value: String = String::from_utf8(key.into())?; + self.data.remove(&key_value); + Ok(()) } } diff --git a/storage/src/rocks_db.rs b/storage/src/rocks_db.rs index f559289f..5e39d5d7 100644 --- a/storage/src/rocks_db.rs +++ b/storage/src/rocks_db.rs @@ -44,13 +44,15 @@ impl Database for RocksDbDatabase { let result = self.db.get(key)?; Ok(result) } + + fn delete(&mut self, key: &[u8]) -> Result<(), Box> { + self.db.delete(key)?; + Ok(()) + } } #[cfg(test)] mod tests { - use alto_types::Address; - use alto_types::account::Account; - use alto_types::address::Address; use super::*; #[test] fn test_rocks_db_basic() { @@ -61,27 +63,4 @@ mod tests { let retrieved = db.get(key).unwrap().unwrap(); assert_eq!(retrieved.as_slice(), value); } - - #[test] - fn test_rocks_db_accounts() { - let mut db = RocksDbDatabase::new_tmp_db().expect("db could not be created"); - - let mut account = Account::new(); - let test_address = Address::new(b"0xBEEF"); - account.address = test_address.clone(); - account.balance = 100; - - // get account for test address is empty - let empty_result = db.get_account(&test_address); - empty_result.unwrap().is_none(); - - // set account - db.set_account(&account).unwrap(); - - let acct_result = db.get_account(&test_address).unwrap(); - assert!(acct_result.is_some()); - let account = acct_result.unwrap(); - assert_eq!(account.address, test_address); - assert_eq!(account.balance, 100); - } } diff --git a/storage/src/state_db.rs b/storage/src/state_db.rs index e5c124f8..ac72a58a 100644 --- a/storage/src/state_db.rs +++ b/storage/src/state_db.rs @@ -4,6 +4,7 @@ use alto_types::address::Address; use bytes::Bytes; use commonware_codec::{Codec, ReadBuffer, WriteBuffer}; use std::error::Error; +use crate::rocks_db::RocksDbDatabase; const ACCOUNTS_PREFIX: &[u8] = b"sal_accounts"; const DB_WRITE_BUFFER_CAPACITY: usize = 500; @@ -73,3 +74,49 @@ impl StateDb { key } } + +impl Database for StateDb { + fn put(&mut self, key: &[u8], value: &[u8]) -> Result<(), Box> { + self.db.put(key, value) + } + + fn get(&self, key: &[u8]) -> Result>, Box> { + self.db.get(key) + } + + fn delete(&mut self, key: &[u8]) -> Result<(), Box> { + self.db.delete(key) + } +} + +#[cfg(test)] +mod tests { + use alto_types::address::Address; + use alto_types::account::Account; + use alto_types::address::Address; + use super::*; + + #[test] + fn test_rocks_db_accounts() { + let db = RocksDbDatabase::new_tmp_db().expect("db could not be created"); + let mut state_db = StateDb::new(Box::new(db)); + + let mut account = Account::new(); + let test_address = Address::new(b"0xBEEF"); + account.address = test_address.clone(); + account.balance = 100; + + // get account for test address is empty + let empty_result = state_db.get_account(&test_address); + empty_result.unwrap().is_none(); + + // set account + state_db.set_account(&account).unwrap(); + + let acct_result = state_db.get_account(&test_address).unwrap(); + assert!(acct_result.is_some()); + let account = acct_result.unwrap(); + assert_eq!(account.address, test_address); + assert_eq!(account.balance, 100); + } +} \ No newline at end of file diff --git a/storage/src/tx_state_view.rs b/storage/src/tx_state_view.rs index 29ef5af8..6a171111 100644 --- a/storage/src/tx_state_view.rs +++ b/storage/src/tx_state_view.rs @@ -1,13 +1,20 @@ use std::collections::HashMap; -use crate::database::Database; use std::error::Error; -use rocksdb::LogLevel::Error; +use alto_types::address::Address; +use crate::database::Database; use alto_types::state::{State}; use crate::state_db::StateDb; const ACCOUNT_KEY_TYPE: u8 = 0; + type UnitKey<'a> = alto_types::state::UnitKey<'a>; // 1st byte denotes the type of key. 0b for account key, 1b for others. +pub fn decode_unit_key(key: UnitKey) -> (u8, Address) { + let key_type: u8 = key[0]; + let address_bytes: &[u8] = &key[1..]; + (key_type, address_bytes.into()) +} + pub enum OpAction { Read, // key was read Create, // key was created @@ -47,6 +54,7 @@ impl<'a> TxStateView<'a> { pub fn init_cache(&mut self, cache: HashMap>) { self.cache = cache; } + pub fn get_from_cache(&self, key: UnitKey) -> Result>, Box> { self.cache.get(&key).map_or( Ok(None), @@ -54,6 +62,7 @@ impl<'a> TxStateView<'a> { } pub fn get_from_state(&self, key: UnitKey) -> Result>, Box> { + let (key_type, address) match key[0] { ACCOUNT_KEY_TYPE => { self.get_from_state(key) @@ -62,16 +71,16 @@ impl<'a> TxStateView<'a> { } } - pub fn get(&self, key: alto_types::state::UnitKey) -> Result>, Box>{ - todo!() + pub fn get(&self, key: UnitKey) -> Result>, Box>{ } - pub fn get_multi_key(&self, key: Vec) -> Result>, Box> { + + pub fn get_multi_key(&self, key: UnitKey) -> Result>, Box> { todo!() } - pub fn update(&mut self, key: alto_types::state::UnitKey, value: Vec) -> Result<(), Box>{ + pub fn update(&mut self, key: UnitKey, value: Vec) -> Result<(), Box>{ todo!() } - pub fn delete(&mut self, key: alto_types::state::UnitKey) -> Result<(), Box> { + pub fn delete(&mut self, key: UnitKey) -> Result<(), Box> { todo!() } pub fn commit(&mut self) -> Result<(), Box> { @@ -80,4 +89,22 @@ impl<'a> TxStateView<'a> { pub fn rollback(&mut self) -> Result<(), Box> { todo!() } + + pub fn process_get_action(&mut self, cmd_type: u8, key: &UnitKey) -> Result>, Box> { + match cmd_type { + ACCOUNT_KEY_TYPE => { + self.state_db.get(key) + } + _ => Err(format!("invalid state key {:?}", key).into()) + } + } + + pub fn process_put_action(&mut self, cmd_type: u8, key: &UnitKey) -> Result>, Box> { + match cmd_type { + ACCOUNT_KEY_TYPE => { + self.state_db.get(key) + } + _ => Err(format!("invalid state key {:?}", key).into()) + } + } } \ No newline at end of file diff --git a/types/src/tx.rs b/types/src/tx.rs index 53a251fb..05bf2021 100644 --- a/types/src/tx.rs +++ b/types/src/tx.rs @@ -1,8 +1,8 @@ use std::error::Error; use crate::address::Address; -use crate::state::StateDB; use crate::wallet::Wallet; use crate::signed_tx::SignedTx; +use crate::state::State; pub enum UnitType { Transfer, @@ -18,15 +18,16 @@ pub struct UnitContext { // unit need to be simple and easy to be packed in the tx and executed by the vm. -pub trait Unit { - // type DB: Database; - // type T: TxStateViewTrait; - - - fn unit_type(&self) -> UnitType; // return the unit type. - fn encode(&self) -> Vec; // encode the unit. - fn decode(bytes: &[u8]) -> Self; // decode the unit. - fn apply(&self, context: &UnitContext, state: &mut dyn StateDB) -> Result>, Box >; // apply the unit to the state view. +pub trait Unit : Send + Sync { + fn unit_type(&self) -> UnitType; + fn encode(&self) -> Vec; + fn decode(&mut self, bytes: &[u8]); + + fn apply( + &self, + context: &UnitContext, + state: &mut Box, + ) -> Result>, Box>; } @@ -39,7 +40,7 @@ pub struct Tx<'a> { // timestamp is used to prevent replay attacks. and counter infinite spam attacks as Tx does not have nonce. pub timestamp: u64, // units are fundamental unit of a tx. similar to actions. - pub units: Vec, + pub units: Vec>, // max fee is the maximum fee the user is willing to pay for the tx. pub max_fee: u64, // priority fee is the fee the user is willing to pay for the tx to be included in the next block. @@ -59,7 +60,7 @@ pub trait TxChars { // init is used to create a new instance of Tx. fn init() -> Self; // new is used to create a new instance of Tx with given units and chain id. - fn new(units: Vec, chain_id: u64) -> Self; + fn new(units: Vec>, chain_id: u64) -> Self; // set_fee is used to set the max fee and priority fee of the tx. fn set_fee(&mut self, max_fee: u64, priority_fee: u64); // sign is used to sign the tx with the given wallet. From 69135e51f4b67ca00aac675360739fe4f2c7304f Mon Sep 17 00:00:00 2001 From: manojkgorle Date: Fri, 28 Mar 2025 04:05:19 +0000 Subject: [PATCH 15/43] edition2024 to edition2021 --- actions/Cargo.toml | 2 +- storage/Cargo.toml | 2 +- vm/Cargo.toml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/actions/Cargo.toml b/actions/Cargo.toml index 189380eb..60a2bf23 100644 --- a/actions/Cargo.toml +++ b/actions/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "alto-actions" version = "0.1.0" -edition = "2024" +edition = "2021" [dependencies] alto-types = { workspace = true } diff --git a/storage/Cargo.toml b/storage/Cargo.toml index 7685351c..cba09b57 100644 --- a/storage/Cargo.toml +++ b/storage/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "alto-storage" version = "0.1.0" -edition = "2024" +edition = "2021" [dependencies] alto-types = { workspace = true } diff --git a/vm/Cargo.toml b/vm/Cargo.toml index 8e773002..3ef612cc 100644 --- a/vm/Cargo.toml +++ b/vm/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "alto-vm" version = "0.1.0" -edition = "2024" +edition = "2021" [dependencies] alto-storage = { workspace = true } From 13f1a7f4e9738b5239df3ad02f3a25b5a911375f Mon Sep 17 00:00:00 2001 From: manojkgorle Date: Fri, 28 Mar 2025 04:18:42 +0000 Subject: [PATCH 16/43] clippy --- types/src/account.rs | 6 ++++++ types/src/address.rs | 2 +- types/src/codec.rs | 6 ++---- types/src/lib.rs | 1 - types/src/tx.rs | 1 - 5 files changed, 9 insertions(+), 7 deletions(-) diff --git a/types/src/account.rs b/types/src/account.rs index fb4faea5..061b7ca4 100644 --- a/types/src/account.rs +++ b/types/src/account.rs @@ -9,6 +9,12 @@ pub struct Account { pub balance: Balance, } +impl Default for Account { + fn default() -> Self { + Self::new() + } +} + impl Account { pub fn new() -> Self { Self { diff --git a/types/src/address.rs b/types/src/address.rs index 53aff458..594e4d65 100644 --- a/types/src/address.rs +++ b/types/src/address.rs @@ -17,7 +17,7 @@ impl Address { return Err("Address must be 32 bytes."); } - Ok(Address(<[u8; 32]>::try_from(bytes.clone()).unwrap())) + Ok(Address(<[u8; 32]>::try_from(bytes).unwrap())) } pub fn empty() -> Self { diff --git a/types/src/codec.rs b/types/src/codec.rs index 79120e97..d8688a2a 100644 --- a/types/src/codec.rs +++ b/types/src/codec.rs @@ -1,7 +1,5 @@ -use commonware_codec::{Codec, Error, Reader, Writer}; -use commonware_cryptography::{ed25519::PublicKey, Ed25519, Scheme}; -use commonware_cryptography::ed25519::PrivateKey; -use serde::Serialize; +use commonware_codec::Writer; +use commonware_cryptography::ed25519::PublicKey; pub fn serialize_pk(pk: &PublicKey, writer: &mut impl Writer) { // let slice: &[u8] = pk.as_ref(); diff --git a/types/src/lib.rs b/types/src/lib.rs index 5dd5075e..b9e06500 100644 --- a/types/src/lib.rs +++ b/types/src/lib.rs @@ -15,7 +15,6 @@ pub mod state; pub mod address; pub mod account; -use more_asserts; use rand::rngs::OsRng; // We don't use functions here to guard against silent changes. diff --git a/types/src/tx.rs b/types/src/tx.rs index 05bf2021..fb5400a9 100644 --- a/types/src/tx.rs +++ b/types/src/tx.rs @@ -1,4 +1,3 @@ -use std::error::Error; use crate::address::Address; use crate::wallet::Wallet; use crate::signed_tx::SignedTx; From c0bb662ad48da21c34bfa71330285bfe80e644de Mon Sep 17 00:00:00 2001 From: NateAtNodeKit Date: Fri, 28 Mar 2025 13:30:26 -0400 Subject: [PATCH 17/43] transact db --- storage/src/database.rs | 2 +- storage/src/lib.rs | 3 +- storage/src/state_db.rs | 1 - storage/src/transactional_db.rs | 176 ++++++++++++++++++++++++++++++++ storage/src/tx_state_view.rs | 4 + 5 files changed, 183 insertions(+), 3 deletions(-) create mode 100644 storage/src/transactional_db.rs diff --git a/storage/src/database.rs b/storage/src/database.rs index 032938f4..5ea85c67 100644 --- a/storage/src/database.rs +++ b/storage/src/database.rs @@ -8,7 +8,7 @@ use alto_types::address::Address; pub trait Database { fn put(&mut self, key: &[u8], value: &[u8]) -> Result<(), Box>; - fn get(&self, key: &[u8]) -> Result, Box>; + fn get(&self, key: &[u8]) -> Result>, Box>; fn delete(&mut self, key: &[u8]) -> Result<(), Box>; } \ No newline at end of file diff --git a/storage/src/lib.rs b/storage/src/lib.rs index ae0cda68..0f0cd0c6 100644 --- a/storage/src/lib.rs +++ b/storage/src/lib.rs @@ -2,4 +2,5 @@ pub mod database; pub mod hashmap_db; mod rocks_db; mod tx_state_view; -mod state_db; \ No newline at end of file +mod state_db; +mod transactional_db; \ No newline at end of file diff --git a/storage/src/state_db.rs b/storage/src/state_db.rs index ac72a58a..1ce84f9f 100644 --- a/storage/src/state_db.rs +++ b/storage/src/state_db.rs @@ -93,7 +93,6 @@ impl Database for StateDb { mod tests { use alto_types::address::Address; use alto_types::account::Account; - use alto_types::address::Address; use super::*; #[test] diff --git a/storage/src/transactional_db.rs b/storage/src/transactional_db.rs new file mode 100644 index 00000000..7fbf4a92 --- /dev/null +++ b/storage/src/transactional_db.rs @@ -0,0 +1,176 @@ +use std::collections::HashMap; +use std::error::Error; +use bytes::Bytes; +use alto_types::address::Address; +use alto_types::state::State; +use crate::database::Database; + +const ACCOUNT_KEY_TYPE: u8 = 0; + +type Key<'a> = &'a[u8]; + +pub fn decode_unit_key(key: UnitKey) -> (u8, Address) { + let key_type: u8 = key[0]; + let address_bytes: &[u8] = &key[1..]; + (key_type, address_bytes.into()) +} + +pub enum OpAction { + Read, // key was read + Create, // key was created + Update, // key was updated + Delete, // key got deleted +} + +pub struct Op<'a>{ + pub action: crate::tx_state_view::OpAction, + pub key: Key<'a>, + pub value: Vec, +} + +pub trait TransactionalDb : Database { + fn init_cache(&mut self, cache: HashMap>); // initialize the cache with an already available hashmap of key-value pairs. + fn get_from_cache(&self, key: Key) -> Result>, Box>; // get a key from the cache. If the key is not in the cache, it will return an error. + fn get_from_db(&self, key: Key) -> Result>, Box>; // get a key from the underlying storage. If the key is not in the storage, it will return an error. + fn commit(&mut self) -> Result<(), Box>; + fn rollback(&mut self) -> Result<(), Box>; +} + +pub struct InMemoryCachingTransactionalDb<'a> { + pub cache: HashMap, Vec>, // key-value, state view cache before tx execution. This is not an exhaustive list of all state keys read/write during tx. If cache misses occur, the state view will read from the underlying storage. + //pub ops: Vec>, // list of state ops applied. + pub touched: HashMap, Vec>, // key-value pairs that were changed during tx execution. This is a subset of the cache. + pub db: Box, // underlying state storage, to use when cache misses occur. +} + +impl<'a> TransactionalDb for InMemoryCachingTransactionalDb<'a> { + fn init_cache(&mut self, cache: HashMap>) { + self.cache = cache; + } + + // get a key from the cache. If the key is not in the cache, it will return an error. + fn get_from_cache(&self, key: Key) -> Result>, Box> { + self.cache.get(&key).map_or( + Ok(None), + |v| Ok(Some(v.clone().into()))) + } + + // get a key from the underlying storage. If the key is not in the storage, it will return an error. + fn get_from_db(&self, key: Key) -> Result>, Box> { + self.db.get(key) + } + + fn commit(&mut self) -> Result<(), Box> { + for (key, value) in self.touched.iter() {} + } + + fn rollback(&mut self) -> Result<(), Box> { + self.touched.clear(); + Ok(()) + } +} + +impl<'a> Database for InMemoryCachingTransactionalDb<'a> { + fn put(&mut self, key: &[u8], value: &[u8]) -> Result<(), Box> { + self.cache.insert(key, value.clone().to_vec()); + self.touched.insert(key, value.to_vec()); + Ok(()) + } + + fn get(&self, key: &[u8]) -> Result>, Box> { + match self.get_from_cache(key) { + Ok(Some(value)) => Ok(Some(value.into())), + _ => self.db.get(key), + } + } + + // TODO: The below deletes in both cache and underlying db. Change later such that deletes must be committed. + fn delete(&mut self, key: &[u8]) -> Result<(), Box> { + self.cache.remove(key); + self.db.delete(key) + } +} + +/* +pub trait : State { + fn init_cache(&mut self, cache: HashMap>); // initialize the cache with an already available hashmap of key-value pairs. + fn get_from_cache(&self, key: UnitKey) -> Result>, Box>; // get a key from the cache. If the key is not in the cache, it will return an error. + fn get_from_state(&self, key: UnitKey) -> Result>, Box>; // get a key from the underlying storage. If the key is not in the storage, it will return an error. +} + +pub struct TxStateView<'a> { + pub cache: HashMap, Vec>, // key-value, state view cache before tx execution. This is not an exhaustive list of all state keys read/write during tx. If cache misses occur, the state view will read from the underlying storage. + pub ops: Vec>, // list of state ops applied. + pub touched: HashMap, Vec>, // key-value pairs that were changed during tx execution. This is a subset of the cache. + pub state_db: StateDb, // underlying state storage, to use when cache misses occur. +} + +impl<'a> crate::tx_state_view::TxStateView<'a> { + pub fn new(state_db: StateDb) -> Self { + Self{ + cache: HashMap::new(), + ops: Vec::new(), + touched: HashMap::new(), + state_db, + } + } + // initialize the cache with an already available hashmap of key-value pairs. + pub fn init_cache(&mut self, cache: HashMap>) { + self.cache = cache; + } + + pub fn get_from_cache(&self, key: UnitKey) -> Result>, Box> { + self.cache.get(&key).map_or( + Ok(None), + |v| Ok(Some(v.clone().into()))) + } + + pub fn get_from_state(&self, key: UnitKey) -> Result>, Box> { + let (key_type, address) + match key[0] { + ACCOUNT_KEY_TYPE => { + self.get_from_state(key) + }, + _ => Err(format!("invalid state key {:?}", key[0]).into()) + } + } + + pub fn get(&self, key: UnitKey) -> Result>, Box>{ + } + + pub fn get_multi_key(&self, key: UnitKey) -> Result>, Box> { + todo!() + } + pub fn update(&mut self, key: UnitKey, value: Vec) -> Result<(), Box>{ + todo!() + } + pub fn delete(&mut self, key: UnitKey) -> Result<(), Box> { + todo!() + } + pub fn commit(&mut self) -> Result<(), Box> { + todo!() + } + pub fn rollback(&mut self) -> Result<(), Box> { + todo!() + } + + pub fn process_get_action(&mut self, cmd_type: u8, key: &UnitKey) -> Result>, Box> { + match cmd_type { + ACCOUNT_KEY_TYPE => { + self.state_db.get(key) + } + _ => Err(format!("invalid state key {:?}", key).into()) + } + } + + pub fn process_put_action(&mut self, cmd_type: u8, key: &UnitKey) -> Result>, Box> { + match cmd_type { + ACCOUNT_KEY_TYPE => { + self.state_db.get(key) + } + _ => Err(format!("invalid state key {:?}", key).into()) + } + } +} + + */ diff --git a/storage/src/tx_state_view.rs b/storage/src/tx_state_view.rs index 6a171111..b93e48bf 100644 --- a/storage/src/tx_state_view.rs +++ b/storage/src/tx_state_view.rs @@ -62,6 +62,7 @@ impl<'a> TxStateView<'a> { } pub fn get_from_state(&self, key: UnitKey) -> Result>, Box> { + /* let (key_type, address) match key[0] { ACCOUNT_KEY_TYPE => { @@ -69,6 +70,9 @@ impl<'a> TxStateView<'a> { }, _ => Err(format!("invalid state key {:?}", key[0]).into()) } + + */ + todo!() } pub fn get(&self, key: UnitKey) -> Result>, Box>{ From ed2fad598c8472080c3891866352642ceb2a1d90 Mon Sep 17 00:00:00 2001 From: rikoeldon <106416799+rikoeldon@users.noreply.github.com> Date: Fri, 28 Mar 2025 15:44:24 -0400 Subject: [PATCH 18/43] DB changes & lifetime errors to be resolved.. --- storage/src/database.rs | 10 +- storage/src/hashmap_db.rs | 6 +- storage/src/rocks_db.rs | 2 +- storage/src/state_db.rs | 13 +- storage/src/transactional_db.rs | 215 ++++++++++++++------------------ storage/src/tx_state_view.rs | 205 +++++++++++++++--------------- 6 files changed, 211 insertions(+), 240 deletions(-) diff --git a/storage/src/database.rs b/storage/src/database.rs index 5ea85c67..d7b902cc 100644 --- a/storage/src/database.rs +++ b/storage/src/database.rs @@ -1,14 +1,10 @@ -use alto_types::account::{Balance, Account}; use std::error::Error; -use bytes::Bytes; -use commonware_codec::{Codec, ReadBuffer, WriteBuffer}; -use alto_types::address::Address; // Define database interface that will be used for all impls pub trait Database { - fn put(&mut self, key: &[u8], value: &[u8]) -> Result<(), Box>; + fn put<'a: 'b>(&mut self, key: &'a [u8], value: &[u8]) -> Result<(), Box>; - fn get(&self, key: &[u8]) -> Result>, Box>; + fn get<'a: 'b>(&mut self, key: &'a [u8]) -> Result>, Box>; - fn delete(&mut self, key: &[u8]) -> Result<(), Box>; + fn delete<'a: 'b>(&mut self, key: &'a [u8]) -> Result<(), Box>; } \ No newline at end of file diff --git a/storage/src/hashmap_db.rs b/storage/src/hashmap_db.rs index d3396498..88b9d505 100644 --- a/storage/src/hashmap_db.rs +++ b/storage/src/hashmap_db.rs @@ -15,7 +15,7 @@ impl HashmapDatabase { } impl Database for HashmapDatabase { - fn put(&mut self, key: &[u8], value: &[u8]) -> Result<(), Box> { + fn put<'a>(&mut self, key: &'a [u8], value: &[u8]) -> Result<(), Box> { let key_value: String = String::from_utf8(key.into())?; let str_value: String = String::from_utf8(value.into())?; @@ -23,14 +23,14 @@ impl Database for HashmapDatabase { Ok(()) } - fn get(&self, key: &[u8]) -> Result>, Box> { + fn get<'a>(&mut self, key: &'a [u8]) -> Result>, Box> { let str_key: String = String::from_utf8(key.into()).unwrap(); self.data.get(&str_key).map_or( Ok(None), |v| Ok(Some(v.clone().into()))) } - fn delete(&mut self, key: &[u8]) -> Result<(), Box> { + fn delete<'a>(&mut self, key: &'a [u8]) -> Result<(), Box> { let key_value: String = String::from_utf8(key.into())?; self.data.remove(&key_value); Ok(()) diff --git a/storage/src/rocks_db.rs b/storage/src/rocks_db.rs index 5e39d5d7..16f6f200 100644 --- a/storage/src/rocks_db.rs +++ b/storage/src/rocks_db.rs @@ -40,7 +40,7 @@ impl Database for RocksDbDatabase { Ok(()) } - fn get(&self, key: &[u8]) -> Result>, Box> { + fn get(&mut self, key: &[u8]) -> Result>, Box> { let result = self.db.get(key)?; Ok(result) } diff --git a/storage/src/state_db.rs b/storage/src/state_db.rs index 1ce84f9f..0a25beb8 100644 --- a/storage/src/state_db.rs +++ b/storage/src/state_db.rs @@ -18,8 +18,9 @@ impl StateDb { StateDb { db } } - pub fn get_account(&self, address: &Address) -> Result, Box> { + pub fn get_account(&mut self, address: &Address) -> Result, Box> { let key = Self::key_accounts(address); + let result = self.db.get(key.as_slice())?; match result { None => Ok(None), @@ -40,7 +41,7 @@ impl StateDb { Ok(()) } - pub fn get_balance(&self, address: &Address) -> Option { + pub fn get_balance(&mut self, address: &Address) -> Option { let result = self.get_account(address).and_then(|acc| match acc { Some(acc) => Ok(acc.balance), None => Ok(0), @@ -75,16 +76,16 @@ impl StateDb { } } -impl Database for StateDb { - fn put(&mut self, key: &[u8], value: &[u8]) -> Result<(), Box> { +impl<'b> Database for StateDb { + fn put<'a>(&mut self, key: &'a [u8], value: &[u8]) -> Result<(), Box> { self.db.put(key, value) } - fn get(&self, key: &[u8]) -> Result>, Box> { + fn get<'a>(&mut self, key: &'a [u8]) -> Result>, Box> { self.db.get(key) } - fn delete(&mut self, key: &[u8]) -> Result<(), Box> { + fn delete<'a>(&mut self, key: &'a [u8]) -> Result<(), Box> { self.db.delete(key) } } diff --git a/storage/src/transactional_db.rs b/storage/src/transactional_db.rs index 7fbf4a92..669f115e 100644 --- a/storage/src/transactional_db.rs +++ b/storage/src/transactional_db.rs @@ -9,168 +9,141 @@ const ACCOUNT_KEY_TYPE: u8 = 0; type Key<'a> = &'a[u8]; -pub fn decode_unit_key(key: UnitKey) -> (u8, Address) { - let key_type: u8 = key[0]; - let address_bytes: &[u8] = &key[1..]; - (key_type, address_bytes.into()) -} - -pub enum OpAction { - Read, // key was read - Create, // key was created - Update, // key was updated - Delete, // key got deleted -} - -pub struct Op<'a>{ - pub action: crate::tx_state_view::OpAction, - pub key: Key<'a>, - pub value: Vec, -} - -pub trait TransactionalDb : Database { - fn init_cache(&mut self, cache: HashMap>); // initialize the cache with an already available hashmap of key-value pairs. - fn get_from_cache(&self, key: Key) -> Result>, Box>; // get a key from the cache. If the key is not in the cache, it will return an error. - fn get_from_db(&self, key: Key) -> Result>, Box>; // get a key from the underlying storage. If the key is not in the storage, it will return an error. +// pub enum OpAction { +// Read, // key was read +// Create, // key was created +// Update, // key was updated +// Delete, // key got deleted +// } + +// pub struct Op<'a>{ +// pub action: crate::tx_state_view::OpAction, +// pub key: Key<'a>, +// pub value: Vec, +// } + +pub trait TransactionalDb<'b> : Database { + fn init_cache<'a: 'b>(&mut self, cache: HashMap, Vec>); // initialize the cache with an already available hashmap of key-value pairs. + fn get_from_cache<'a: 'b>(&self, key: Key) -> Result>, Box>; // get a key from the cache. If the key is not in the cache, it will return an error. + fn get_from_db(&mut self, key: Key) -> Result>, Box>; // get a key from the underlying storage. If the key is not in the storage, it will return an error. fn commit(&mut self) -> Result<(), Box>; fn rollback(&mut self) -> Result<(), Box>; } pub struct InMemoryCachingTransactionalDb<'a> { pub cache: HashMap, Vec>, // key-value, state view cache before tx execution. This is not an exhaustive list of all state keys read/write during tx. If cache misses occur, the state view will read from the underlying storage. - //pub ops: Vec>, // list of state ops applied. + // pub ops: Vec>, // list of state ops applied. pub touched: HashMap, Vec>, // key-value pairs that were changed during tx execution. This is a subset of the cache. pub db: Box, // underlying state storage, to use when cache misses occur. } +impl<'a> InMemoryCachingTransactionalDb<'a> { + pub fn new(db: Box) -> InMemoryCachingTransactionalDb<'a> { + Self{ + cache: HashMap::new(), + touched: HashMap::new(), + db, + } + } + fn get_from_touched<'b>(&self, key: Key) -> Result>, Box> { + self.touched.get(&key).map_or( + Ok(None), + |v| Ok(Some(v.clone().into()))) + } +} -impl<'a> TransactionalDb for InMemoryCachingTransactionalDb<'a> { - fn init_cache(&mut self, cache: HashMap>) { +impl<'b> TransactionalDb<'b> for InMemoryCachingTransactionalDb<'b> { + fn init_cache<'a: 'b>(&mut self, cache: HashMap>) { self.cache = cache; } // get a key from the cache. If the key is not in the cache, it will return an error. - fn get_from_cache(&self, key: Key) -> Result>, Box> { - self.cache.get(&key).map_or( - Ok(None), - |v| Ok(Some(v.clone().into()))) + fn get_from_cache<'a: 'b>(&self, key: Key) -> Result>, Box> { + // searches touched + if let Some(v) = self.get_from_touched(&key)? { + return Ok(Some(v.clone())); + } + // searches cache + match self.cache.get(&key) { + Some(cached_value) => { + Ok(Some(cached_value.clone().into())) + } + // not found in either cache or db + None => Ok(None), + } } // get a key from the underlying storage. If the key is not in the storage, it will return an error. - fn get_from_db(&self, key: Key) -> Result>, Box> { + fn get_from_db<'a>(&mut self, key: Key) -> Result>, Box> { self.db.get(key) } - fn commit(&mut self) -> Result<(), Box> { - for (key, value) in self.touched.iter() {} + fn commit<'a>(&mut self) -> Result<(), Box> { + for (key, value) in self.touched.iter() { + self.cache.insert(key.clone(), value.clone()); + //TODO: what to do if an intermediary operation fails maybe use rocks db transact? + // ex: what if we go through half of touched and it fails halfway? rare but possible. + self.db.put(key.clone(), value)?; + } + self.touched.clear(); + Ok(()) } - fn rollback(&mut self) -> Result<(), Box> { + fn rollback<'a>(&mut self) -> Result<(), Box> { self.touched.clear(); Ok(()) } } -impl<'a> Database for InMemoryCachingTransactionalDb<'a> { - fn put(&mut self, key: &[u8], value: &[u8]) -> Result<(), Box> { - self.cache.insert(key, value.clone().to_vec()); +impl<'b> Database for InMemoryCachingTransactionalDb<'b> { + fn put<'a: 'b>(&mut self, key: &'a [u8], value: &[u8]) -> Result<(), Box> { self.touched.insert(key, value.to_vec()); Ok(()) } - fn get(&self, key: &[u8]) -> Result>, Box> { + fn get<'a: 'b>(&mut self, key: &'a [u8]) -> Result>, Box> { match self.get_from_cache(key) { - Ok(Some(value)) => Ok(Some(value.into())), - _ => self.db.get(key), + Ok(Some(value)) => { + Ok(Some(value.into())) + }, + Ok(None) => { + match self.db.get(key) { + Ok(Some(value)) => { + self.cache.insert(key, value.clone()); + Ok(Some(value.into())) + } + Ok(None) => { + Ok(None) + }, + Err(e) => { + Err(e) + } + } + }, + Err(e) => { + Err(e) + } } } // TODO: The below deletes in both cache and underlying db. Change later such that deletes must be committed. - fn delete(&mut self, key: &[u8]) -> Result<(), Box> { + fn delete<'a>(&mut self, key: &'a [u8]) -> Result<(), Box> { + self.touched.remove(key); self.cache.remove(key); self.db.delete(key) } } -/* -pub trait : State { - fn init_cache(&mut self, cache: HashMap>); // initialize the cache with an already available hashmap of key-value pairs. - fn get_from_cache(&self, key: UnitKey) -> Result>, Box>; // get a key from the cache. If the key is not in the cache, it will return an error. - fn get_from_state(&self, key: UnitKey) -> Result>, Box>; // get a key from the underlying storage. If the key is not in the storage, it will return an error. -} - -pub struct TxStateView<'a> { - pub cache: HashMap, Vec>, // key-value, state view cache before tx execution. This is not an exhaustive list of all state keys read/write during tx. If cache misses occur, the state view will read from the underlying storage. - pub ops: Vec>, // list of state ops applied. - pub touched: HashMap, Vec>, // key-value pairs that were changed during tx execution. This is a subset of the cache. - pub state_db: StateDb, // underlying state storage, to use when cache misses occur. -} - -impl<'a> crate::tx_state_view::TxStateView<'a> { - pub fn new(state_db: StateDb) -> Self { - Self{ - cache: HashMap::new(), - ops: Vec::new(), - touched: HashMap::new(), - state_db, - } - } - // initialize the cache with an already available hashmap of key-value pairs. - pub fn init_cache(&mut self, cache: HashMap>) { - self.cache = cache; - } - - pub fn get_from_cache(&self, key: UnitKey) -> Result>, Box> { - self.cache.get(&key).map_or( - Ok(None), - |v| Ok(Some(v.clone().into()))) - } - - pub fn get_from_state(&self, key: UnitKey) -> Result>, Box> { - let (key_type, address) - match key[0] { - ACCOUNT_KEY_TYPE => { - self.get_from_state(key) - }, - _ => Err(format!("invalid state key {:?}", key[0]).into()) - } - } - - pub fn get(&self, key: UnitKey) -> Result>, Box>{ - } - - pub fn get_multi_key(&self, key: UnitKey) -> Result>, Box> { - todo!() - } - pub fn update(&mut self, key: UnitKey, value: Vec) -> Result<(), Box>{ - todo!() - } - pub fn delete(&mut self, key: UnitKey) -> Result<(), Box> { - todo!() - } - pub fn commit(&mut self) -> Result<(), Box> { - todo!() - } - pub fn rollback(&mut self) -> Result<(), Box> { - todo!() - } - - pub fn process_get_action(&mut self, cmd_type: u8, key: &UnitKey) -> Result>, Box> { - match cmd_type { - ACCOUNT_KEY_TYPE => { - self.state_db.get(key) - } - _ => Err(format!("invalid state key {:?}", key).into()) - } - } - - pub fn process_put_action(&mut self, cmd_type: u8, key: &UnitKey) -> Result>, Box> { - match cmd_type { - ACCOUNT_KEY_TYPE => { - self.state_db.get(key) - } - _ => Err(format!("invalid state key {:?}", key).into()) - } +#[cfg(test)] +mod tests { + use crate::hashmap_db::HashmapDatabase; + use super::*; + #[test] + fn test_transactional_db_basic() { + let mut db = HashmapDatabase::new(); + let mut tx_db = InMemoryCachingTransactionalDb::new(Box::new(db)); + let test_key = b"test_key".to_vec(); + let test_value = b"test_value".to_vec(); + tx_db.put(&test_key, &test_value).unwrap(); } } - */ diff --git a/storage/src/tx_state_view.rs b/storage/src/tx_state_view.rs index b93e48bf..60ede1f3 100644 --- a/storage/src/tx_state_view.rs +++ b/storage/src/tx_state_view.rs @@ -9,106 +9,107 @@ const ACCOUNT_KEY_TYPE: u8 = 0; type UnitKey<'a> = alto_types::state::UnitKey<'a>; // 1st byte denotes the type of key. 0b for account key, 1b for others. -pub fn decode_unit_key(key: UnitKey) -> (u8, Address) { - let key_type: u8 = key[0]; - let address_bytes: &[u8] = &key[1..]; - (key_type, address_bytes.into()) -} +// pub fn decode_unit_key(key: UnitKey) -> (u8, Address) { +// let key_type: u8 = key[0]; +// let address_bytes: &[u8] = &key[1..]; +// (key_type, address_bytes.into()) +// } -pub enum OpAction { - Read, // key was read - Create, // key was created - Update, // key was updated - Delete, // key got deleted -} - -pub struct Op<'a>{ - pub action: OpAction, - pub key: UnitKey<'a>, - pub value: Vec, -} - -pub trait TxStateViewTrait: State { - fn init_cache(&mut self, cache: HashMap>); // initialize the cache with an already available hashmap of key-value pairs. - fn get_from_cache(&self, key: UnitKey) -> Result>, Box>; // get a key from the cache. If the key is not in the cache, it will return an error. - fn get_from_state(&self, key: UnitKey) -> Result>, Box>; // get a key from the underlying storage. If the key is not in the storage, it will return an error. -} - -pub struct TxStateView<'a> { - pub cache: HashMap, Vec>, // key-value, state view cache before tx execution. This is not an exhaustive list of all state keys read/write during tx. If cache misses occur, the state view will read from the underlying storage. - pub ops: Vec>, // list of state ops applied. - pub touched: HashMap, Vec>, // key-value pairs that were changed during tx execution. This is a subset of the cache. - pub state_db: StateDb, // underlying state storage, to use when cache misses occur. -} - -impl<'a> TxStateView<'a> { - pub fn new(state_db: StateDb) -> Self { - Self{ - cache: HashMap::new(), - ops: Vec::new(), - touched: HashMap::new(), - state_db, - } - } - // initialize the cache with an already available hashmap of key-value pairs. - pub fn init_cache(&mut self, cache: HashMap>) { - self.cache = cache; - } - - pub fn get_from_cache(&self, key: UnitKey) -> Result>, Box> { - self.cache.get(&key).map_or( - Ok(None), - |v| Ok(Some(v.clone().into()))) - } - - pub fn get_from_state(&self, key: UnitKey) -> Result>, Box> { - /* - let (key_type, address) - match key[0] { - ACCOUNT_KEY_TYPE => { - self.get_from_state(key) - }, - _ => Err(format!("invalid state key {:?}", key[0]).into()) - } - - */ - todo!() - } - - pub fn get(&self, key: UnitKey) -> Result>, Box>{ - } - - pub fn get_multi_key(&self, key: UnitKey) -> Result>, Box> { - todo!() - } - pub fn update(&mut self, key: UnitKey, value: Vec) -> Result<(), Box>{ - todo!() - } - pub fn delete(&mut self, key: UnitKey) -> Result<(), Box> { - todo!() - } - pub fn commit(&mut self) -> Result<(), Box> { - todo!() - } - pub fn rollback(&mut self) -> Result<(), Box> { - todo!() - } - - pub fn process_get_action(&mut self, cmd_type: u8, key: &UnitKey) -> Result>, Box> { - match cmd_type { - ACCOUNT_KEY_TYPE => { - self.state_db.get(key) - } - _ => Err(format!("invalid state key {:?}", key).into()) - } - } - - pub fn process_put_action(&mut self, cmd_type: u8, key: &UnitKey) -> Result>, Box> { - match cmd_type { - ACCOUNT_KEY_TYPE => { - self.state_db.get(key) - } - _ => Err(format!("invalid state key {:?}", key).into()) - } - } -} \ No newline at end of file +// pub enum OpAction { +// Read, // key was read +// Create, // key was created +// Update, // key was updated +// Delete, // key got deleted +// } +// +// pub struct Op<'a>{ +// pub action: OpAction, +// pub key: UnitKey<'a>, +// pub value: Vec, +// } +// +// pub trait TxStateViewTrait: State { +// fn init_cache(&mut self, cache: HashMap>); // initialize the cache with an already available hashmap of key-value pairs. +// fn get_from_cache(&self, key: UnitKey) -> Result>, Box>; // get a key from the cache. If the key is not in the cache, it will return an error. +// fn get_from_state(&self, key: UnitKey) -> Result>, Box>; // get a key from the underlying storage. If the key is not in the storage, it will return an error. +// } +// +// pub struct TxStateView<'a> { +// pub cache: HashMap, Vec>, // key-value, state view cache before tx execution. This is not an exhaustive list of all state keys read/write during tx. If cache misses occur, the state view will read from the underlying storage. +// pub ops: Vec>, // list of state ops applied. +// pub touched: HashMap, Vec>, // key-value pairs that were changed during tx execution. This is a subset of the cache. +// pub state_db: StateDb, // underlying state storage, to use when cache misses occur. +// } +// +// impl<'a> TxStateView<'a> { +// pub fn new(state_db: StateDb) -> Self { +// Self{ +// cache: HashMap::new(), +// ops: Vec::new(), +// touched: HashMap::new(), +// state_db, +// } +// } +// // initialize the cache with an already available hashmap of key-value pairs. +// pub fn init_cache(&mut self, cache: HashMap>) { +// self.cache = cache; +// } +// +// pub fn get_from_cache(&self, key: UnitKey) -> Result>, Box> { +// self.cache.get(&key).map_or( +// Ok(None), +// |v| Ok(Some(v.clone().into()))) +// } +// +// pub fn get_from_state(&self, key: UnitKey) -> Result>, Box> { +// /* +// let (key_type, address) +// match key[0] { +// ACCOUNT_KEY_TYPE => { +// self.get_from_state(key) +// }, +// _ => Err(format!("invalid state key {:?}", key[0]).into()) +// } +// +// */ +// todo!() +// } +// +// pub fn get(&self, key: UnitKey) -> Result>, Box>{ +// todo!() +// } +// +// pub fn get_multi_key(&self, key: UnitKey) -> Result>, Box> { +// todo!() +// } +// pub fn update(&mut self, key: UnitKey, value: Vec) -> Result<(), Box>{ +// todo!() +// } +// pub fn delete(&mut self, key: UnitKey) -> Result<(), Box> { +// todo!() +// } +// pub fn commit(&mut self) -> Result<(), Box> { +// todo!() +// } +// pub fn rollback(&mut self) -> Result<(), Box> { +// todo!() +// } +// +// pub fn process_get_action(&mut self, cmd_type: u8, key: &UnitKey) -> Result>, Box> { +// match cmd_type { +// ACCOUNT_KEY_TYPE => { +// self.state_db.get(key) +// } +// _ => Err(format!("invalid state key {:?}", key).into()) +// } +// } +// +// pub fn process_put_action(&mut self, cmd_type: u8, key: &UnitKey) -> Result>, Box> { +// match cmd_type { +// ACCOUNT_KEY_TYPE => { +// self.state_db.get(key) +// } +// _ => Err(format!("invalid state key {:?}", key).into()) +// } +// } +// } \ No newline at end of file From f7cfc95013744a0951a52a91ef439e67f80daec3 Mon Sep 17 00:00:00 2001 From: NateAtNodeKit Date: Fri, 28 Mar 2025 16:28:26 -0400 Subject: [PATCH 19/43] transact db fixes --- storage/src/database.rs | 8 ++-- storage/src/hashmap_db.rs | 2 +- storage/src/rocks_db.rs | 2 +- storage/src/state_db.rs | 6 ++- storage/src/transactional_db.rs | 76 ++++++++++++++++++++++++--------- storage/src/tx_state_view.rs | 6 +-- 6 files changed, 69 insertions(+), 31 deletions(-) diff --git a/storage/src/database.rs b/storage/src/database.rs index d7b902cc..ad98eb1d 100644 --- a/storage/src/database.rs +++ b/storage/src/database.rs @@ -1,10 +1,10 @@ use std::error::Error; // Define database interface that will be used for all impls -pub trait Database { - fn put<'a: 'b>(&mut self, key: &'a [u8], value: &[u8]) -> Result<(), Box>; +pub trait Database<'a> { + fn put(&mut self, key: &'a [u8], value: &[u8]) -> Result<(), Box>; - fn get<'a: 'b>(&mut self, key: &'a [u8]) -> Result>, Box>; + fn get(&mut self, key: &'a [u8]) -> Result>, Box>; - fn delete<'a: 'b>(&mut self, key: &'a [u8]) -> Result<(), Box>; + fn delete(&mut self, key: &'a [u8]) -> Result<(), Box>; } \ No newline at end of file diff --git a/storage/src/hashmap_db.rs b/storage/src/hashmap_db.rs index 88b9d505..b457f3f3 100644 --- a/storage/src/hashmap_db.rs +++ b/storage/src/hashmap_db.rs @@ -14,7 +14,7 @@ impl HashmapDatabase { } } -impl Database for HashmapDatabase { +impl Database<'_> for HashmapDatabase { fn put<'a>(&mut self, key: &'a [u8], value: &[u8]) -> Result<(), Box> { let key_value: String = String::from_utf8(key.into())?; let str_value: String = String::from_utf8(value.into())?; diff --git a/storage/src/rocks_db.rs b/storage/src/rocks_db.rs index 16f6f200..23352a51 100644 --- a/storage/src/rocks_db.rs +++ b/storage/src/rocks_db.rs @@ -34,7 +34,7 @@ impl RocksDbDatabase { } -impl Database for RocksDbDatabase { +impl Database<'_> for RocksDbDatabase { fn put(&mut self, key: &[u8], value: &[u8]) -> Result<(), Box> { self.db.put(key, value)?; Ok(()) diff --git a/storage/src/state_db.rs b/storage/src/state_db.rs index 0a25beb8..2cc47c3d 100644 --- a/storage/src/state_db.rs +++ b/storage/src/state_db.rs @@ -1,3 +1,4 @@ +/* use crate::database::Database; use alto_types::account::{Account, Balance}; use alto_types::address::Address; @@ -9,7 +10,7 @@ use crate::rocks_db::RocksDbDatabase; const ACCOUNTS_PREFIX: &[u8] = b"sal_accounts"; const DB_WRITE_BUFFER_CAPACITY: usize = 500; -pub struct StateDb { +pub struct StateDb<'a> { db: Box, } // can use like a redis from Arcadia like get and set for diff types? @@ -119,4 +120,5 @@ mod tests { assert_eq!(account.address, test_address); assert_eq!(account.balance, 100); } -} \ No newline at end of file +} + */ \ No newline at end of file diff --git a/storage/src/transactional_db.rs b/storage/src/transactional_db.rs index 669f115e..60bea025 100644 --- a/storage/src/transactional_db.rs +++ b/storage/src/transactional_db.rs @@ -22,10 +22,10 @@ type Key<'a> = &'a[u8]; // pub value: Vec, // } -pub trait TransactionalDb<'b> : Database { - fn init_cache<'a: 'b>(&mut self, cache: HashMap, Vec>); // initialize the cache with an already available hashmap of key-value pairs. - fn get_from_cache<'a: 'b>(&self, key: Key) -> Result>, Box>; // get a key from the cache. If the key is not in the cache, it will return an error. - fn get_from_db(&mut self, key: Key) -> Result>, Box>; // get a key from the underlying storage. If the key is not in the storage, it will return an error. +pub trait TransactionalDb<'b> : Database<'b> { + fn init_cache(&mut self, cache: HashMap, Vec>); // initialize the cache with an already available hashmap of key-value pairs. + fn get_from_cache(&self, key: Key<'b>) -> Result>, Box>; // get a key from the cache. If the key is not in the cache, it will return an error. + fn get_from_db(&mut self, key: Key<'b>) -> Result>, Box>; // get a key from the underlying storage. If the key is not in the storage, it will return an error. fn commit(&mut self) -> Result<(), Box>; fn rollback(&mut self) -> Result<(), Box>; } @@ -34,10 +34,10 @@ pub struct InMemoryCachingTransactionalDb<'a> { pub cache: HashMap, Vec>, // key-value, state view cache before tx execution. This is not an exhaustive list of all state keys read/write during tx. If cache misses occur, the state view will read from the underlying storage. // pub ops: Vec>, // list of state ops applied. pub touched: HashMap, Vec>, // key-value pairs that were changed during tx execution. This is a subset of the cache. - pub db: Box, // underlying state storage, to use when cache misses occur. + pub db: Box>, // underlying state storage, to use when cache misses occur. } impl<'a> InMemoryCachingTransactionalDb<'a> { - pub fn new(db: Box) -> InMemoryCachingTransactionalDb<'a> { + pub fn new(db: Box>) -> InMemoryCachingTransactionalDb<'a> { Self{ cache: HashMap::new(), touched: HashMap::new(), @@ -52,12 +52,12 @@ impl<'a> InMemoryCachingTransactionalDb<'a> { } impl<'b> TransactionalDb<'b> for InMemoryCachingTransactionalDb<'b> { - fn init_cache<'a: 'b>(&mut self, cache: HashMap>) { + fn init_cache(&mut self, cache: HashMap, Vec>) { self.cache = cache; } // get a key from the cache. If the key is not in the cache, it will return an error. - fn get_from_cache<'a: 'b>(&self, key: Key) -> Result>, Box> { + fn get_from_cache(&self, key: Key<'b>) -> Result>, Box> { // searches touched if let Some(v) = self.get_from_touched(&key)? { return Ok(Some(v.clone())); @@ -73,11 +73,11 @@ impl<'b> TransactionalDb<'b> for InMemoryCachingTransactionalDb<'b> { } // get a key from the underlying storage. If the key is not in the storage, it will return an error. - fn get_from_db<'a>(&mut self, key: Key) -> Result>, Box> { + fn get_from_db(&mut self, key: Key<'b>) -> Result>, Box> { self.db.get(key) } - fn commit<'a>(&mut self) -> Result<(), Box> { + fn commit(&mut self) -> Result<(), Box> { for (key, value) in self.touched.iter() { self.cache.insert(key.clone(), value.clone()); //TODO: what to do if an intermediary operation fails maybe use rocks db transact? @@ -88,19 +88,19 @@ impl<'b> TransactionalDb<'b> for InMemoryCachingTransactionalDb<'b> { Ok(()) } - fn rollback<'a>(&mut self) -> Result<(), Box> { + fn rollback(&mut self) -> Result<(), Box> { self.touched.clear(); Ok(()) } } -impl<'b> Database for InMemoryCachingTransactionalDb<'b> { - fn put<'a: 'b>(&mut self, key: &'a [u8], value: &[u8]) -> Result<(), Box> { +impl<'b> Database<'b> for InMemoryCachingTransactionalDb<'b> { + fn put(&mut self, key: &'b [u8], value: &[u8]) -> Result<(), Box> { self.touched.insert(key, value.to_vec()); Ok(()) } - fn get<'a: 'b>(&mut self, key: &'a [u8]) -> Result>, Box> { + fn get(&mut self, key: &'b [u8]) -> Result>, Box> { match self.get_from_cache(key) { Ok(Some(value)) => { Ok(Some(value.into())) @@ -126,7 +126,7 @@ impl<'b> Database for InMemoryCachingTransactionalDb<'b> { } // TODO: The below deletes in both cache and underlying db. Change later such that deletes must be committed. - fn delete<'a>(&mut self, key: &'a [u8]) -> Result<(), Box> { + fn delete(&mut self, key: &'b [u8]) -> Result<(), Box> { self.touched.remove(key); self.cache.remove(key); self.db.delete(key) @@ -137,13 +137,49 @@ impl<'b> Database for InMemoryCachingTransactionalDb<'b> { mod tests { use crate::hashmap_db::HashmapDatabase; use super::*; + #[test] fn test_transactional_db_basic() { - let mut db = HashmapDatabase::new(); - let mut tx_db = InMemoryCachingTransactionalDb::new(Box::new(db)); - let test_key = b"test_key".to_vec(); - let test_value = b"test_value".to_vec(); - tx_db.put(&test_key, &test_value).unwrap(); + let mut hash_db = HashmapDatabase::new(); // underlying store + + let test_key1 = b"test_key1".to_vec(); + let test_value1 = b"test_value1".to_vec(); + let test_key2 = b"test_key2".to_vec(); + let test_value2 = b"test_value2".to_vec(); + let test_key3 = b"test_key3".to_vec(); + let test_value3 = b"test_value3".to_vec(); + + { + let mut tx_db = InMemoryCachingTransactionalDb::new(Box::new(hash_db)); + + // should add to cache but not underlying store db + tx_db.put(&test_key1, &test_value1).unwrap(); + assert!(tx_db.get_from_db(&test_key1).unwrap().is_none()); + assert_eq!(tx_db.get_from_cache(&test_key1).unwrap().is_some(), true); + assert_eq!(tx_db.get_from_cache(&test_key1).unwrap().unwrap(), test_value1); + assert_eq!(tx_db.get(&test_key1).unwrap().unwrap(), test_value1); + + // commit the change. should be visible in underlying store. + tx_db.commit().unwrap(); + assert!(tx_db.get_from_db(&test_key1).unwrap().is_some()); + assert_eq!(tx_db.get_from_db(&test_key1).unwrap().unwrap(), test_value1); + assert_eq!(tx_db.get_from_cache(&test_key1).unwrap().unwrap(), test_value1); + assert_eq!(tx_db.get(&test_key1).unwrap().unwrap(), test_value1); + + // add 2nd key-value. do not commit yet. + tx_db.put(&test_key2, &test_value2).unwrap(); + assert!(tx_db.get_from_db(&test_key2).unwrap().is_none()); + assert_eq!(tx_db.get_from_cache(&test_key2).unwrap().is_some(), true); + assert_eq!(tx_db.get_from_cache(&test_key2).unwrap().unwrap(), test_value2); + assert_eq!(tx_db.get(&test_key2).unwrap().unwrap(), test_value2); + + // rollback and commit should make no changes to underlying. + tx_db.rollback().unwrap(); + tx_db.commit().unwrap(); + assert!(tx_db.get_from_db(&test_key2).unwrap().is_none()); + assert!(tx_db.get_from_cache(&test_key2).unwrap().is_none()); + assert!(tx_db.get(&test_key2).unwrap().is_none()); + } } } diff --git a/storage/src/tx_state_view.rs b/storage/src/tx_state_view.rs index 60ede1f3..16e74a27 100644 --- a/storage/src/tx_state_view.rs +++ b/storage/src/tx_state_view.rs @@ -3,11 +3,11 @@ use std::error::Error; use alto_types::address::Address; use crate::database::Database; use alto_types::state::{State}; -use crate::state_db::StateDb; +//use crate::state_db::StateDb; -const ACCOUNT_KEY_TYPE: u8 = 0; +//const ACCOUNT_KEY_TYPE: u8 = 0; -type UnitKey<'a> = alto_types::state::UnitKey<'a>; // 1st byte denotes the type of key. 0b for account key, 1b for others. +//type UnitKey<'a> = alto_types::state::UnitKey<'a>; // 1st byte denotes the type of key. 0b for account key, 1b for others. // pub fn decode_unit_key(key: UnitKey) -> (u8, Address) { // let key_type: u8 = key[0]; From cc08b08d54df6b20b0b25dc60b5520a5e2fe343d Mon Sep 17 00:00:00 2001 From: rikoeldon <106416799+rikoeldon@users.noreply.github.com> Date: Sat, 29 Mar 2025 12:36:31 -0400 Subject: [PATCH 20/43] removed mit/apache --- LICENSE-APACHE | 178 ------------------------------------------------- LICENSE-MIT | 21 ------ 2 files changed, 199 deletions(-) delete mode 100644 LICENSE-APACHE delete mode 100644 LICENSE-MIT diff --git a/LICENSE-APACHE b/LICENSE-APACHE deleted file mode 100644 index e5268c58..00000000 --- a/LICENSE-APACHE +++ /dev/null @@ -1,178 +0,0 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - Copyright (c) 2025 Commonware, Inc. - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS \ No newline at end of file diff --git a/LICENSE-MIT b/LICENSE-MIT deleted file mode 100644 index 21092130..00000000 --- a/LICENSE-MIT +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) 2025 Commonware, Inc. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. \ No newline at end of file From 6e1c6d1cd77e57fe5ea074656c73ced97a2787ac Mon Sep 17 00:00:00 2001 From: rikoeldon <106416799+rikoeldon@users.noreply.github.com> Date: Sat, 29 Mar 2025 12:37:34 -0400 Subject: [PATCH 21/43] update readme --- README.md | 4 ---- 1 file changed, 4 deletions(-) diff --git a/README.md b/README.md index 007664d6..f8bda3bc 100644 --- a/README.md +++ b/README.md @@ -14,10 +14,6 @@ _Components are designed for deployment in adversarial environments. If you find * [inspector](./inspector/README.md): Inspect `alto` activity. * [types](./types/README.md): Common types used throughout `alto`. -## Licensing - -This repository is dual-licensed under both the [Apache 2.0](./LICENSE-APACHE) and [MIT](./LICENSE-MIT) licenses. You may choose either license when employing this code. - ## Support If you have any questions about `alto`, we encourage you to post in [GitHub Discussions](https://github.com/commonwarexyz/monorepo/discussions). We're happy to help! \ No newline at end of file From 117c22b33af12f3b6b758f4f05a5864807911721 Mon Sep 17 00:00:00 2001 From: rikoeldon <106416799+rikoeldon@users.noreply.github.com> Date: Sun, 30 Mar 2025 18:38:40 -0400 Subject: [PATCH 22/43] no lifetimes & updated DBs --- actions/src/msg.rs | 2 +- actions/src/transfer.rs | 2 +- chain/src/actors/application/router.rs | 14 +-- storage/src/database.rs | 8 +- storage/src/hashmap_db.rs | 8 +- storage/src/lib.rs | 4 +- storage/src/rocks_db.rs | 2 +- storage/src/state_db.rs | 114 ++++++++++++++----------- storage/src/transactional_db.rs | 70 +++++++-------- vm/src/vm.rs | 9 +- 10 files changed, 124 insertions(+), 109 deletions(-) diff --git a/actions/src/msg.rs b/actions/src/msg.rs index e8e1d093..ac211393 100644 --- a/actions/src/msg.rs +++ b/actions/src/msg.rs @@ -1,4 +1,4 @@ -use alto_types::Address; +use alto_types::address::Address; pub struct SequencerMsg { pub chain_id: Vec, diff --git a/actions/src/transfer.rs b/actions/src/transfer.rs index 84a16129..9d026c33 100644 --- a/actions/src/transfer.rs +++ b/actions/src/transfer.rs @@ -1,4 +1,4 @@ -use alto_types::Address; +use alto_types::address::Address; const MAX_MEMO_SIZE: usize = 256; diff --git a/chain/src/actors/application/router.rs b/chain/src/actors/application/router.rs index c6975526..65a7fff5 100644 --- a/chain/src/actors/application/router.rs +++ b/chain/src/actors/application/router.rs @@ -69,13 +69,13 @@ impl Router { "Submit block" } - fn init_router(&mut self) { - let router = axum::Router::new() - .route("/", get(Router::handle_default)) - .route(Router::PATH_SUBMIT_BLOCK, get(Router::handle_submit_block())); - - self.router = Some(router) - } + // fn init_router(&mut self) { + // let router = axum::Router::new() + // .route("/", get(Router::handle_default)) + // .route(Router::PATH_SUBMIT_BLOCK, get(Router::handle_submit_block())); + // + // self.router = Some(router) + // } async fn serve(&mut self) -> Result<(), Box> { let listener = self.listener.take().ok_or("serve failed because listener is None"); diff --git a/storage/src/database.rs b/storage/src/database.rs index ad98eb1d..f1708685 100644 --- a/storage/src/database.rs +++ b/storage/src/database.rs @@ -1,10 +1,10 @@ use std::error::Error; // Define database interface that will be used for all impls -pub trait Database<'a> { - fn put(&mut self, key: &'a [u8], value: &[u8]) -> Result<(), Box>; +pub trait Database { + fn put(&mut self, key: &[u8], value: &[u8]) -> Result<(), Box>; - fn get(&mut self, key: &'a [u8]) -> Result>, Box>; + fn get(&mut self, key: &[u8]) -> Result>, Box>; - fn delete(&mut self, key: &'a [u8]) -> Result<(), Box>; + fn delete(&mut self, key: &[u8]) -> Result<(), Box>; } \ No newline at end of file diff --git a/storage/src/hashmap_db.rs b/storage/src/hashmap_db.rs index b457f3f3..7abe6420 100644 --- a/storage/src/hashmap_db.rs +++ b/storage/src/hashmap_db.rs @@ -14,8 +14,8 @@ impl HashmapDatabase { } } -impl Database<'_> for HashmapDatabase { - fn put<'a>(&mut self, key: &'a [u8], value: &[u8]) -> Result<(), Box> { +impl Database for HashmapDatabase { + fn put(&mut self, key: &[u8], value: &[u8]) -> Result<(), Box> { let key_value: String = String::from_utf8(key.into())?; let str_value: String = String::from_utf8(value.into())?; @@ -23,14 +23,14 @@ impl Database<'_> for HashmapDatabase { Ok(()) } - fn get<'a>(&mut self, key: &'a [u8]) -> Result>, Box> { + fn get(&mut self, key: &[u8]) -> Result>, Box> { let str_key: String = String::from_utf8(key.into()).unwrap(); self.data.get(&str_key).map_or( Ok(None), |v| Ok(Some(v.clone().into()))) } - fn delete<'a>(&mut self, key: &'a [u8]) -> Result<(), Box> { + fn delete(&mut self, key: &[u8]) -> Result<(), Box> { let key_value: String = String::from_utf8(key.into())?; self.data.remove(&key_value); Ok(()) diff --git a/storage/src/lib.rs b/storage/src/lib.rs index 0f0cd0c6..a4802311 100644 --- a/storage/src/lib.rs +++ b/storage/src/lib.rs @@ -2,5 +2,5 @@ pub mod database; pub mod hashmap_db; mod rocks_db; mod tx_state_view; -mod state_db; -mod transactional_db; \ No newline at end of file +pub mod state_db; +pub mod transactional_db; \ No newline at end of file diff --git a/storage/src/rocks_db.rs b/storage/src/rocks_db.rs index 23352a51..16f6f200 100644 --- a/storage/src/rocks_db.rs +++ b/storage/src/rocks_db.rs @@ -34,7 +34,7 @@ impl RocksDbDatabase { } -impl Database<'_> for RocksDbDatabase { +impl Database for RocksDbDatabase { fn put(&mut self, key: &[u8], value: &[u8]) -> Result<(), Box> { self.db.put(key, value)?; Ok(()) diff --git a/storage/src/state_db.rs b/storage/src/state_db.rs index 2cc47c3d..092987f5 100644 --- a/storage/src/state_db.rs +++ b/storage/src/state_db.rs @@ -1,5 +1,5 @@ -/* use crate::database::Database; +use crate::transactional_db::{TransactionalDb,InMemoryCachingTransactionalDb}; use alto_types::account::{Account, Balance}; use alto_types::address::Address; use bytes::Bytes; @@ -7,86 +7,81 @@ use commonware_codec::{Codec, ReadBuffer, WriteBuffer}; use std::error::Error; use crate::rocks_db::RocksDbDatabase; -const ACCOUNTS_PREFIX: &[u8] = b"sal_accounts"; +const ACCOUNTS_PREFIX: u8 = 0x0; const DB_WRITE_BUFFER_CAPACITY: usize = 500; -pub struct StateDb<'a> { - db: Box, +pub struct StateDb { + db: Box, } // can use like a redis from Arcadia like get and set for diff types? impl StateDb { - pub fn new(db: Box) -> StateDb { + pub fn new(db: Box) -> StateDb { StateDb { db } } pub fn get_account(&mut self, address: &Address) -> Result, Box> { let key = Self::key_accounts(address); - - let result = self.db.get(key.as_slice())?; - match result { - None => Ok(None), - Some(value) => { - let bytes = Bytes::copy_from_slice(&value); - let mut read_buf = ReadBuffer::new(bytes); - let acc = Account::read(&mut read_buf)?; - Ok(Some(acc)) - } + // try cache first + if let Some(value) = self.db.get_from_cache(&key)? + .or_else(|| self.db.get(&key).ok().flatten()) // falls to DB if cache miss + { + let bytes = Bytes::from(value); + let mut read_buf = ReadBuffer::new(bytes); + let acc = Account::read(&mut read_buf)?; + return Ok(Some(acc)); } + Ok(None) } pub fn set_account(&mut self, acc: &Account) -> Result<(), Box> { let key = Self::key_accounts(&acc.address); let mut write_buf = WriteBuffer::new(DB_WRITE_BUFFER_CAPACITY); acc.write(&mut write_buf); - self.db.put(&key, write_buf.as_ref()).expect("TODO: panic message"); + self.db.put(&key, write_buf.as_ref())?; Ok(()) } pub fn get_balance(&mut self, address: &Address) -> Option { - let result = self.get_account(address).and_then(|acc| match acc { - Some(acc) => Ok(acc.balance), - None => Ok(0), - }); - match result { - Ok(balance) => Some(balance), - _ => None, + match self.get_account(address) { + Ok(Some(acc)) => Some(acc.balance), // return balance if account exists + Ok(None) => Some(0), // return 0 if no account + Err(_) => None, // return none if an err occurred } } pub fn set_balance(&mut self, address: &Address, amt: Balance) -> bool { - let result = self.get_account(address); - match result { + match self.get_account(address) { Ok(Some(mut acc)) => { acc.balance = amt; - let result = self.set_account(&acc); - result.is_ok() + self.set_account(&acc).is_ok() } _ => false, } } - fn key_accounts(addr: &Address) -> Vec { + fn key_accounts(addr: &Address) -> [u8; 33] { Self::make_multi_key(ACCOUNTS_PREFIX, addr.as_slice()) } - fn make_multi_key(prefix: &[u8], sub_id: &[u8]) -> Vec { - let mut key = Vec::with_capacity(prefix.len() + sub_id.len() + 1); - key.extend_from_slice(prefix); - key.push(b':'); - key.extend_from_slice(sub_id); + fn make_multi_key(prefix: u8, sub_id: &[u8]) -> [u8; 33] { + assert_eq!(sub_id.len(), 32, "Sub_id must be exactly 32 bytes"); + + let mut key = [0u8; 33]; + key[0] = prefix; + key[1..33].copy_from_slice(sub_id); key } } -impl<'b> Database for StateDb { - fn put<'a>(&mut self, key: &'a [u8], value: &[u8]) -> Result<(), Box> { +impl Database for StateDb { + fn put(&mut self, key: &[u8], value: &[u8]) -> Result<(), Box> { self.db.put(key, value) } - fn get<'a>(&mut self, key: &'a [u8]) -> Result>, Box> { + fn get(&mut self, key: &[u8]) -> Result>, Box> { self.db.get(key) } - fn delete<'a>(&mut self, key: &'a [u8]) -> Result<(), Box> { + fn delete(&mut self, key: &[u8]) -> Result<(), Box> { self.db.delete(key) } } @@ -97,28 +92,45 @@ mod tests { use alto_types::account::Account; use super::*; + fn setup_state_db() -> StateDb { + let db = InMemoryCachingTransactionalDb::new(Box::new( + RocksDbDatabase::new_tmp_db().expect("db could not be created"), + )); + StateDb::new(Box::new(db)) + } #[test] fn test_rocks_db_accounts() { - let db = RocksDbDatabase::new_tmp_db().expect("db could not be created"); - let mut state_db = StateDb::new(Box::new(db)); + let mut state_db = setup_state_db(); let mut account = Account::new(); - let test_address = Address::new(b"0xBEEF"); + let test_address_bytes = [0u8; 32]; + let test_address = Address::new(&test_address_bytes); account.address = test_address.clone(); account.balance = 100; - // get account for test address is empty - let empty_result = state_db.get_account(&test_address); - empty_result.unwrap().is_none(); + // make sure account does not exist (base case) + assert!(state_db.get_account(&test_address).unwrap().is_none()); - // set account + // create account and check retrieval state_db.set_account(&account).unwrap(); + let retrieved_account = state_db.get_account(&test_address).unwrap().expect("Account not found"); + assert_eq!(retrieved_account.address, test_address); + assert_eq!(retrieved_account.balance, 100); - let acct_result = state_db.get_account(&test_address).unwrap(); - assert!(acct_result.is_some()); - let account = acct_result.unwrap(); - assert_eq!(account.address, test_address); - assert_eq!(account.balance, 100); + // update account balance and check retrieval + assert!(state_db.set_balance(&test_address, 200)); + assert_eq!(state_db.get_balance(&test_address), Some(200)); + + // test if updating balance is persistent + assert!(state_db.set_balance(&test_address, 300)); + let updated_account = state_db.get_account(&test_address).unwrap().expect("Account not found"); + assert_eq!(updated_account.balance, 300); + + // test retrieval of balance directly + assert_eq!(state_db.get_balance(&test_address), Some(300)); + + // check a non-existent account returns None + let non_existent_address = Address::new(b"0xDEAD"); + assert!(state_db.get_account(&non_existent_address).unwrap().is_none()); } -} - */ \ No newline at end of file +} \ No newline at end of file diff --git a/storage/src/transactional_db.rs b/storage/src/transactional_db.rs index 60bea025..ba861414 100644 --- a/storage/src/transactional_db.rs +++ b/storage/src/transactional_db.rs @@ -1,13 +1,11 @@ use std::collections::HashMap; use std::error::Error; -use bytes::Bytes; -use alto_types::address::Address; use alto_types::state::State; use crate::database::Database; const ACCOUNT_KEY_TYPE: u8 = 0; -type Key<'a> = &'a[u8]; +type Key = [u8; 33]; // pub enum OpAction { // Read, // key was read @@ -22,50 +20,50 @@ type Key<'a> = &'a[u8]; // pub value: Vec, // } -pub trait TransactionalDb<'b> : Database<'b> { - fn init_cache(&mut self, cache: HashMap, Vec>); // initialize the cache with an already available hashmap of key-value pairs. - fn get_from_cache(&self, key: Key<'b>) -> Result>, Box>; // get a key from the cache. If the key is not in the cache, it will return an error. - fn get_from_db(&mut self, key: Key<'b>) -> Result>, Box>; // get a key from the underlying storage. If the key is not in the storage, it will return an error. +pub trait TransactionalDb : Database { + fn init_cache(&mut self, cache: HashMap>); // initialize the cache with an already available hashmap of key-value pairs. + fn get_from_cache(&self, key: &Key) -> Result>, Box>; // get a key from the cache. If the key is not in the cache, it will return an error. + fn get_from_db(&mut self, key: &Key) -> Result>, Box>; // get a key from the underlying storage. If the key is not in the storage, it will return an error. fn commit(&mut self) -> Result<(), Box>; fn rollback(&mut self) -> Result<(), Box>; } -pub struct InMemoryCachingTransactionalDb<'a> { - pub cache: HashMap, Vec>, // key-value, state view cache before tx execution. This is not an exhaustive list of all state keys read/write during tx. If cache misses occur, the state view will read from the underlying storage. +pub struct InMemoryCachingTransactionalDb { + pub cache: HashMap>, // key-value, state view cache before tx execution. This is not an exhaustive list of all state keys read/write during tx. If cache misses occur, the state view will read from the underlying storage. // pub ops: Vec>, // list of state ops applied. - pub touched: HashMap, Vec>, // key-value pairs that were changed during tx execution. This is a subset of the cache. - pub db: Box>, // underlying state storage, to use when cache misses occur. + pub touched: HashMap>, // key-value pairs that were changed during tx execution. This is a subset of the cache. + pub db: Box, // underlying state storage, to use when cache misses occur. } -impl<'a> InMemoryCachingTransactionalDb<'a> { - pub fn new(db: Box>) -> InMemoryCachingTransactionalDb<'a> { +impl InMemoryCachingTransactionalDb { + pub fn new(db: Box) -> InMemoryCachingTransactionalDb { Self{ cache: HashMap::new(), touched: HashMap::new(), db, } } - fn get_from_touched<'b>(&self, key: Key) -> Result>, Box> { - self.touched.get(&key).map_or( + fn get_from_touched(&self, key: &Key) -> Result>, Box> { + self.touched.get(key).map_or( Ok(None), - |v| Ok(Some(v.clone().into()))) + |v| Ok(Some(v.clone()))) } } -impl<'b> TransactionalDb<'b> for InMemoryCachingTransactionalDb<'b> { - fn init_cache(&mut self, cache: HashMap, Vec>) { +impl TransactionalDb for InMemoryCachingTransactionalDb { + fn init_cache(&mut self, cache: HashMap>) { self.cache = cache; } // get a key from the cache. If the key is not in the cache, it will return an error. - fn get_from_cache(&self, key: Key<'b>) -> Result>, Box> { + fn get_from_cache(&self, key: &Key) -> Result>, Box> { // searches touched - if let Some(v) = self.get_from_touched(&key)? { + if let Some(v) = self.get_from_touched(key)? { return Ok(Some(v.clone())); } // searches cache - match self.cache.get(&key) { + match self.cache.get(key) { Some(cached_value) => { - Ok(Some(cached_value.clone().into())) + Ok(Some(cached_value.clone())) } // not found in either cache or db None => Ok(None), @@ -73,7 +71,7 @@ impl<'b> TransactionalDb<'b> for InMemoryCachingTransactionalDb<'b> { } // get a key from the underlying storage. If the key is not in the storage, it will return an error. - fn get_from_db(&mut self, key: Key<'b>) -> Result>, Box> { + fn get_from_db(&mut self, key: &Key) -> Result>, Box> { self.db.get(key) } @@ -82,7 +80,7 @@ impl<'b> TransactionalDb<'b> for InMemoryCachingTransactionalDb<'b> { self.cache.insert(key.clone(), value.clone()); //TODO: what to do if an intermediary operation fails maybe use rocks db transact? // ex: what if we go through half of touched and it fails halfway? rare but possible. - self.db.put(key.clone(), value)?; + self.db.put(&key[..], value)?; } self.touched.clear(); Ok(()) @@ -94,21 +92,21 @@ impl<'b> TransactionalDb<'b> for InMemoryCachingTransactionalDb<'b> { } } -impl<'b> Database<'b> for InMemoryCachingTransactionalDb<'b> { - fn put(&mut self, key: &'b [u8], value: &[u8]) -> Result<(), Box> { - self.touched.insert(key, value.to_vec()); +impl Database for InMemoryCachingTransactionalDb { + fn put(&mut self, key: &[u8], value: &[u8]) -> Result<(), Box> { + self.touched.insert(Key::try_from(key).unwrap(), value.to_vec()); Ok(()) } - fn get(&mut self, key: &'b [u8]) -> Result>, Box> { - match self.get_from_cache(key) { + fn get(&mut self, key: &[u8]) -> Result>, Box> { + match self.get_from_cache(&Key::try_from(key).unwrap()) { Ok(Some(value)) => { Ok(Some(value.into())) }, Ok(None) => { match self.db.get(key) { Ok(Some(value)) => { - self.cache.insert(key, value.clone()); + self.cache.insert(Key::try_from(key).unwrap(), value.clone()); Ok(Some(value.into())) } Ok(None) => { @@ -126,7 +124,7 @@ impl<'b> Database<'b> for InMemoryCachingTransactionalDb<'b> { } // TODO: The below deletes in both cache and underlying db. Change later such that deletes must be committed. - fn delete(&mut self, key: &'b [u8]) -> Result<(), Box> { + fn delete(&mut self, key: &[u8]) -> Result<(), Box> { self.touched.remove(key); self.cache.remove(key); self.db.delete(key) @@ -140,13 +138,15 @@ mod tests { #[test] fn test_transactional_db_basic() { - let mut hash_db = HashmapDatabase::new(); // underlying store + let hash_db = HashmapDatabase::new(); // underlying store - let test_key1 = b"test_key1".to_vec(); + let test_key1: [u8; 33] = [0; 33]; let test_value1 = b"test_value1".to_vec(); - let test_key2 = b"test_key2".to_vec(); + + let test_key2: [u8; 33] = [1; 33]; let test_value2 = b"test_value2".to_vec(); - let test_key3 = b"test_key3".to_vec(); + + let test_key3: [u8; 33] = [2; 33]; let test_value3 = b"test_value3".to_vec(); { diff --git a/vm/src/vm.rs b/vm/src/vm.rs index e6d65bb1..c9ae06ac 100644 --- a/vm/src/vm.rs +++ b/vm/src/vm.rs @@ -1,20 +1,23 @@ use alto_actions::transfer::{Transfer, TransferError}; use alto_storage::database::Database; use alto_storage::hashmap_db::HashmapDatabase; +use alto_storage::transactional_db::TransactionalDb; +use alto_storage::state_db::StateDb; use alto_types::account::Balance; -use alto_types::Address; +use alto_types::address::Address; const TEST_FAUCET_ADDRESS: &[u8] = b"0x0123456789abcdef0123456789abcd"; const TEST_FAUCET_BALANCE: Balance = 10_000_000; +// TODO: figure out vm file struct VM { - state_db: Box, + state_db: Box, } impl VM { pub fn new() -> Self { Self { - state_db: Box::new(HashmapDatabase::new()), + state_db: Box::new(StateDb::new(Box::new(()))), } } From ecd5121c1cba4ef991388c982314789cdc25888c Mon Sep 17 00:00:00 2001 From: manojkgorle Date: Mon, 31 Mar 2025 14:44:17 +0530 Subject: [PATCH 23/43] checkpoint --- Cargo.lock | 15 ++- Cargo.toml | 1 + chain/src/actors/application/actor.rs | 23 ++++- types/Cargo.toml | 2 +- types/src/address.rs | 10 +- types/src/block.rs | 43 +++++++-- types/src/lib.rs | 8 +- types/src/signed_tx.rs | 74 ++++++++++++--- types/src/tx.rs | 67 +++++++++++-- types/src/wallet.rs | 129 +++++++++++++++++++++----- 10 files changed, 310 insertions(+), 62 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 26ed7941..216dbe90 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -116,7 +116,7 @@ version = "0.1.0" dependencies = [ "alto-types", "bytes", - "commonware-codec", + "commonware-codec 0.0.43", "commonware-cryptography", "rand", "rocksdb", @@ -128,7 +128,7 @@ name = "alto-types" version = "0.0.4" dependencies = [ "bytes", - "commonware-codec", + "commonware-codec 0.0.40", "commonware-cryptography", "commonware-utils", "getrandom 0.2.15", @@ -887,6 +887,17 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" +[[package]] +name = "commonware-codec" +version = "0.0.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ea2a8daeca3c1346a30a4c92af7791b1c37e669fad9e1907203e2cc6f92eec7" +dependencies = [ + "bytes", + "paste", + "thiserror 2.0.12", +] + [[package]] name = "commonware-codec" version = "0.0.43" diff --git a/Cargo.toml b/Cargo.toml index 660e31d0..43d48c28 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,6 +25,7 @@ commonware-runtime = { version = "0.0.40" } commonware-storage = { version = "0.0.40" } commonware-stream = { version = "0.0.40" } commonware-utils = { version = "0.0.40" } +commonware-codec = { version = "0.0.40"} thiserror = "2.0.12" bytes = "1.7.1" rand = "0.8.5" diff --git a/chain/src/actors/application/actor.rs b/chain/src/actors/application/actor.rs index c70cf1b3..058a295d 100644 --- a/chain/src/actors/application/actor.rs +++ b/chain/src/actors/application/actor.rs @@ -85,18 +85,28 @@ impl Actor { // Compute genesis digest self.hasher.update(GENESIS); let genesis_parent = self.hasher.finalize(); - let genesis = Block::new(genesis_parent, 0, 0); + let genesis_state_root = [0u8;32]; + let genesis = Block::new(genesis_parent, 0, 0, Vec::new(), genesis_state_root.into()); let genesis_digest = genesis.digest(); + // --> prepared the genesis digest. + + // there are no blocks built, while genesis. let built: Option = None; let built = Arc::new(Mutex::new(built)); - + // @todo initiate fee manager here. + // @todo get the state view. + // @todo init the database. + // @todo commit to database. while let Some(message) = self.mailbox.next().await { match message { + // return the genesis digest Message::Genesis { response } => { // Use the digest of the genesis message as the initial // payload. let _ = response.send(genesis_digest.clone()); } + // its this validators turn to propose the block. + // So, it should check for available blocks and propose a block. Message::Propose { view, parent, @@ -125,7 +135,12 @@ impl Actor { if current <= parent.timestamp { current = parent.timestamp + 1; } - let block = Block::new(parent.digest(), parent.height+1, current); + // fetch transactions from mempool. + // serialize the transactions fetched from mempool into a vec. + // execute the transactions and get the result? + let txs = Vec::new(); + let dummy_state_root = [0u8;32]; + let block = Block::new(parent.digest(), parent.height+1, current, txs, dummy_state_root.into()); let digest = block.digest(); { let mut built = built.lock().unwrap(); @@ -199,6 +214,7 @@ impl Actor { let _ = response.send(false); return; } + //@todo unmarshall txs, execute txs, generate state root, collect fees, build state root, verify state root. // Persist the verified block syncer.verified(view, block).await; @@ -231,6 +247,7 @@ impl Actor { let finalization = Finalization::new(view, parent, payload, signature.into()); let seed = Seed::new(view, seed.into()); + // @todo syncer does the heavy lifting of post finalization processing. // Send the finalization to the syncer syncer.finalized(finalization, seed).await; } diff --git a/types/Cargo.toml b/types/Cargo.toml index b6b721b3..a67dd5d0 100644 --- a/types/Cargo.toml +++ b/types/Cargo.toml @@ -16,6 +16,7 @@ crate-type = ["rlib", "cdylib"] [dependencies] commonware-cryptography = { workspace = true } commonware-utils = { workspace = true } +commonware-codec = { workspace = true} bytes = { workspace = true } rand = { workspace = true } thiserror = { workspace = true } @@ -23,7 +24,6 @@ wasm-bindgen = "0.2.100" serde = { version = "1.0.219", features = ["derive"] } serde-wasm-bindgen = "0.6.5" more-asserts = "0.3.1" -commonware-codec = { version = "0.0.43" } # Enable "js" feature when WASM is target [target.'cfg(target_arch = "wasm32")'.dependencies.getrandom] diff --git a/types/src/address.rs b/types/src/address.rs index 594e4d65..a671ff45 100644 --- a/types/src/address.rs +++ b/types/src/address.rs @@ -1,5 +1,5 @@ use more_asserts::assert_le; -use crate::ADDRESSLEN; +use crate::{PublicKey, ADDRESSLEN}; #[derive(Hash, Eq, PartialEq, Clone, Debug)] pub struct Address(pub [u8;ADDRESSLEN]); @@ -12,6 +12,14 @@ impl Address { Address(arr) } + pub fn from_pub_key(pub_key: &PublicKey) -> Self { + // @todo implement a hasher to derive the address from the public key. + assert_le!(pub_key.len(), ADDRESSLEN, "public key is too large"); + let mut arr = [0u8; ADDRESSLEN]; + arr[..pub_key.len()].copy_from_slice(pub_key.as_ref()); + Address(arr) + } + pub fn from_bytes(bytes: &[u8]) -> Result { if bytes.len() != 32 { return Err("Address must be 32 bytes."); diff --git a/types/src/block.rs b/types/src/block.rs index f86afe30..a17c3243 100644 --- a/types/src/block.rs +++ b/types/src/block.rs @@ -1,9 +1,14 @@ +use std::os::macos::raw::{self, stat}; + use crate::{Finalization, Notarization}; +use crate::signed_tx::{SignedTx, SignedTxChars}; use bytes::{Buf, BufMut}; use commonware_cryptography::{bls12381::PublicKey, sha256::Digest, Hasher, Sha256}; use commonware_utils::{Array, SizedSerialize}; -#[derive(Clone, Debug, PartialEq, Eq)] +// @todo add state root, fee manager and results to the block struct. +// what method of state root generation should be used? +#[derive(Clone, Debug)] pub struct Block { /// The parent block's digest. pub parent: Digest, @@ -14,25 +19,38 @@ pub struct Block { /// The timestamp of the block (in milliseconds since the Unix epoch). pub timestamp: u64, + /// The raw transactions in the block. + pub raw_txs: Vec, + + /// The state root of the block. + pub state_root: Digest, + + txs: Vec, /// Pre-computed digest of the block. digest: Digest, } impl Block { - fn compute_digest(parent: &Digest, height: u64, timestamp: u64) -> Digest { + fn compute_digest(parent: &Digest, height: u64, timestamp: u64, raw_txs: Vec, state_root: &Digest) -> Digest { let mut hasher = Sha256::new(); hasher.update(parent); hasher.update(&height.to_be_bytes()); hasher.update(×tamp.to_be_bytes()); + hasher.update(&raw_txs); + hasher.update(state_root); hasher.finalize() } - pub fn new(parent: Digest, height: u64, timestamp: u64) -> Self { - let digest = Self::compute_digest(&parent, height, timestamp); + pub fn new(parent: Digest, height: u64, timestamp: u64, txs: Vec, state_root: Digest) -> Self { + let raw_txs = txs.iter().flat_map(|tx| tx.encode()).collect::>(); + let digest = Self::compute_digest(&parent, height, timestamp, raw_txs.clone(), &state_root); Self { parent, height, timestamp, + raw_txs, + state_root, + txs, digest, } } @@ -42,24 +60,31 @@ impl Block { bytes.extend_from_slice(&self.parent); bytes.put_u64(self.height); bytes.put_u64(self.timestamp); + bytes.extend_from_slice(&self.state_root); + bytes.extend_from_slice(self.raw_txs.as_slice()); bytes } pub fn deserialize(mut bytes: &[u8]) -> Option { // Parse the block - if bytes.len() != Self::SERIALIZED_LEN { - return None; - } + // if bytes.len() != Self::SERIALIZED_LEN { + // return None; + // } let parent = Digest::read_from(&mut bytes).ok()?; let height = bytes.get_u64(); let timestamp = bytes.get_u64(); - + let state_root = Digest::read_from(&mut bytes).ok()?; + let raw_txs = bytes.to_vec(); // Return block - let digest = Self::compute_digest(&parent, height, timestamp); + let digest = Self::compute_digest(&parent, height, timestamp, raw_txs.clone(), &state_root); + let txs = Vec::new(); // @todo deserialze txs from raw_txs. Some(Self { parent, height, timestamp, + raw_txs, + state_root, + txs, digest, }) } diff --git a/types/src/lib.rs b/types/src/lib.rs index b9e06500..b7345718 100644 --- a/types/src/lib.rs +++ b/types/src/lib.rs @@ -19,6 +19,7 @@ use rand::rngs::OsRng; // We don't use functions here to guard against silent changes. pub const NAMESPACE: &[u8] = b"_ALTO"; +pub const TX_NAMESPACE:&[u8] = b"_tx_namespace_"; pub const P2P_NAMESPACE: &[u8] = b"_ALTO_P2P"; pub const SEED_NAMESPACE: &[u8] = b"_ALTO_SEED"; pub const NOTARIZE_NAMESPACE: &[u8] = b"_ALTO_NOTARIZE"; @@ -29,6 +30,8 @@ const ADDRESSLEN: usize = 32; type PublicKey = commonware_cryptography::ed25519::PublicKey; type PrivateKey = commonware_cryptography::ed25519::PrivateKey; +type Signature = commonware_cryptography::ed25519::Signature; + pub fn create_test_keypair() -> (PublicKey, PrivateKey) { let mut rng = OsRng; @@ -144,7 +147,7 @@ mod tests { let parent_digest = hash(&[0; 32]); let height = 0; let timestamp = 1; - let block = Block::new(parent_digest, height, timestamp); + let block = Block::new(parent_digest, height, timestamp, Vec::new(), [0; 32].into()); let block_digest = block.digest(); // Check block serialization @@ -154,6 +157,7 @@ mod tests { assert_eq!(block.parent, deserialized.parent); assert_eq!(block.height, deserialized.height); assert_eq!(block.timestamp, deserialized.timestamp); + // @todo add deserialization checks for signed transactions. // Create notarization let view = 0; @@ -195,7 +199,7 @@ mod tests { let parent_digest = hash(&[0; 32]); let height = 0; let timestamp = 1; - let block = Block::new(parent_digest, height, timestamp); + let block = Block::new(parent_digest, height, timestamp, Vec::new(), [0;32].into()); // Create notarization let view = 0; diff --git a/types/src/signed_tx.rs b/types/src/signed_tx.rs index 1aec2917..84f9edc8 100644 --- a/types/src/signed_tx.rs +++ b/types/src/signed_tx.rs @@ -1,26 +1,78 @@ use crate::address::Address; -use crate::PublicKey; -use crate::wallet::{AuthTypes, Wallet}; -use crate::tx::Tx; - +use crate::{PublicKey, TX_NAMESPACE, Signature}; +use crate::wallet::{Wallet, WalletMethods}; +use crate::tx::{Tx, TxMethods}; +use commonware_cryptography::{Ed25519, Scheme}; // this is sent by the user to the validators. -pub struct SignedTx<'a> { - pub tx: Tx<'a>, - pub auth_type: AuthTypes, +#[derive(Clone, Debug)] +pub struct SignedTx { + pub tx: Tx, pub_key: PublicKey, address: Address, signature: Vec, } - +// function names are self explanatory. pub trait SignedTxChars { - fn new(tx: Tx, auth_type: AuthTypes) -> Self; - fn sign(&self, wallet: Wallet) -> SignedTx; - fn verify(&self) -> bool; + fn new(tx: Tx, pub_key: PublicKey, signature: Vec) -> Self; + fn sign(&mut self, wallet: Wallet) -> SignedTx; + fn verify(&mut self) -> bool; fn signature(&self) -> Vec; fn public_key(&self) -> Vec; fn address(&self) -> Address; fn encode(&self) -> Vec; fn decode(&self, bytes: &[u8]) -> Self; } + +impl SignedTxChars for SignedTx { + // @todo either have all fields initialized or none. + fn new(tx: Tx, pub_key: PublicKey, signature: Vec) -> Self { + Self { + tx, + pub_key: pub_key.clone(), + address: Address::from_pub_key(&pub_key), + signature: signature, + } + } + + fn sign(&mut self, mut wallet: Wallet) -> SignedTx { + let tx_data = self.tx.encode(); + SignedTx { + tx: self.tx.clone(), + signature: wallet.sign(&tx_data), + address: wallet.address(), + pub_key: wallet.public_key(), + } + } + + fn verify(&mut self) -> bool { + let tx_data = self.tx.encode(); + let signature = Signature::try_from(self.signature.as_slice()); + if signature.is_err() { + return false; + } + let signature = signature.unwrap(); + Ed25519::verify(Some(TX_NAMESPACE), &tx_data, &self.pub_key, &signature) + } + + fn signature(&self) -> Vec { + self.signature.clone() + } + + fn public_key(&self) -> Vec { + self.pub_key.as_ref().to_vec() + } + + fn address(&self) -> Address { + self.address.clone() + } + + fn encode(&self) -> Vec { + todo!() + } + + fn decode(&self, bytes: &[u8]) -> Self { + todo!() + } +} \ No newline at end of file diff --git a/types/src/tx.rs b/types/src/tx.rs index fb5400a9..08cc8262 100644 --- a/types/src/tx.rs +++ b/types/src/tx.rs @@ -1,23 +1,36 @@ +use commonware_cryptography::sha256::Digest; + use crate::address::Address; use crate::wallet::Wallet; use crate::signed_tx::SignedTx; use crate::state::State; - +// use std::fmt::Debug; pub enum UnitType { Transfer, SequencerMsg, } - pub struct UnitContext { pub timestamp: u64, // timestamp of the tx. pub chain_id: u64, // chain id of the tx. pub sender: Address, // sender of the tx. } +pub trait UnitClone { + fn clone_box(&self) -> Box; +} + +impl UnitClone for T +where + T: 'static + Unit + Clone, +{ + fn clone_box(&self) -> Box { + Box::new(self.clone()) + } +} // unit need to be simple and easy to be packed in the tx and executed by the vm. -pub trait Unit : Send + Sync { +pub trait Unit : UnitClone + Send + Sync + std::fmt::Debug { fn unit_type(&self) -> UnitType; fn encode(&self) -> Vec; fn decode(&mut self, bytes: &[u8]); @@ -29,9 +42,14 @@ pub trait Unit : Send + Sync { ) -> Result>, Box>; } +impl Clone for Box { + fn clone(&self) -> Box { + self.clone_box() + } +} -// pub struct Tx<'a, U: Unit> { -pub struct Tx<'a> { +#[derive(Clone, Debug)] +pub struct Tx { // timestamp of the tx creation. set by the user. // will be verified if the tx is in the valid window once received by validators. // if the timestamp is not in the valid window, the tx will be rejected. @@ -49,13 +67,12 @@ pub struct Tx<'a> { // id is the transaction id. It is the hash of digest. - id: &'a [u8;32], + id: Digest, // digest is encoded tx. digest: Vec, } - -pub trait TxChars { +pub trait TxMethods { // init is used to create a new instance of Tx. fn init() -> Self; // new is used to create a new instance of Tx with given units and chain id. @@ -76,3 +93,37 @@ pub trait TxChars { fn decode(bytes: &[u8]) -> Self; } + +impl TxMethods for Tx { + fn init() -> Self { + todo!() + } + + fn new(units: Vec>, chain_id: u64) -> Self { + todo!() + } + + fn set_fee(&mut self, max_fee: u64, priority_fee: u64) { + todo!() + } + + fn sign(&self, wallet: Wallet) -> SignedTx { + todo!() + } + + fn id(&self) -> &[u8;32] { + todo!() + } + + fn digest(&self) -> Vec { + todo!() + } + + fn encode(&mut self) -> Vec { + todo!() + } + + fn decode(bytes: &[u8]) -> Self { + todo!() + } +} \ No newline at end of file diff --git a/types/src/wallet.rs b/types/src/wallet.rs index 106686da..a2a0aff0 100644 --- a/types/src/wallet.rs +++ b/types/src/wallet.rs @@ -1,39 +1,118 @@ use crate::address::Address; -use crate::{PrivateKey, PublicKey}; +use crate::{PrivateKey, PublicKey,Signature, TX_NAMESPACE}; +use commonware_cryptography::ed25519::Ed25519; +use commonware_cryptography::Scheme; +use rand::{CryptoRng, Rng}; +use std::fmt::Error; +use std::path; +#[derive(Clone, Debug)] pub enum AuthTypes { ED25519, } - -// auth should have a method to verify signatures. -// also batch signature verification. +/// auth should have a method to verify signatures. +/// also batch signature verification. pub trait Auth { - fn public_key(&self) -> PublicKey; // return the public key of the signer. - fn address(&self) -> Address; // return the account address of the signer. - fn verify(&self, data: &[u8], signature: &[u8]) -> bool; // verify a signature. - fn batch_verify(&self, data: &[u8], signatures: Vec<&[u8]>) -> bool; // batch verify signatures. returns error if batch verification fails. + // returns the public key of the signer. + fn public_key(&self) -> PublicKey; + // returns the account address of the signer. + fn address(&self) -> Address; + // verifys the signature. + fn verify(&self, data: &[u8], signature: &[u8]) -> bool; + // batch verify signatures. returns false if batch verification fails. + fn batch_verify(&self, data: &[u8], signatures: Vec<&[u8]>) -> bool; } - +/// Wallet is the module used by the user to sign transactions. Wallet uses Ed25519 signature scheme. pub struct Wallet { - auth: AuthTypes, // auth type - p_key: PrivateKey, // private key - pub_key: PublicKey, // public key - address: Address, // account address + // Private key + priv_key: PrivateKey, + // Public key + pub_key: PublicKey, + // Account Address, is derived from the public key. + address: Address, + // Signer + signer: Ed25519, } - // wallet generation, management, and signing should be functions of the wallet. -pub trait ImplWallet { - fn generate(auth_type: AuthTypes) -> Self; // create a new wallet of given auth type. - fn load(&self, auth_type: AuthTypes, p_key: &[u8]) -> Self; // load a new wallet from private key. - fn sign(&self, data: &[u8]) -> Vec; // sign data with the private key. - fn verify(&self,data: &[u8], signature: &[u8]) -> bool; // verify a signature with the public key. - fn address(&self) -> Address; // return the account address. - fn public_key(&self) -> Vec; // return the public key. - fn auth_type(&self) -> AuthTypes; // return the auth type. - fn private_key(&self) -> Vec; // returns the private key. - fn store_private_key(&self); // store the private key. - fn new_address(&mut self); // generate a new address. +pub trait WalletMethods { + // create a new wallet using the given randomness. + fn generate(r: &mut R) -> Self; + // load signer from bytes rep of a private key and initialize the wallet. + fn load(&self, priv_key: &[u8]) -> Self; + // sign the given arbitary data with the private key of the wallet. + fn sign(&mut self, data: &[u8]) -> Vec; + // verify the signature of the given data with the public key of the wallet. + fn verify(&self,data: &[u8], signature: &[u8]) -> Result; + // return corresponding wallet's address. + fn address(&self) -> Address; + // return corresponding wallet's public key. + fn public_key(&self) -> PublicKey; + // return corresponding wallet's private key. + fn private_key(&self) -> Vec; + // store the private key at the given path. + fn store_private_key(&self, path: &str) -> Result<(), Error>; + // @todo remove this? + fn init_address(&mut self); } + +impl WalletMethods for Wallet { + fn generate(r: &mut R) -> Self { + let signer = Ed25519::new(r); + let pub_key = signer.public_key(); + let address = Address::from_pub_key(&pub_key); + Self { + priv_key: signer.private_key(), + pub_key: signer.public_key(), + address: address, + signer: signer, + } + } + + fn load(&self, priv_key: &[u8]) -> Self { + let private_key = PrivateKey::try_from(priv_key).expect("Invalid private key"); + let signer = ::from(private_key).unwrap(); + Self { + priv_key: signer.private_key(), + pub_key: signer.public_key(), + address: Address::from_pub_key(&signer.public_key()), + signer: signer, + } + } + + fn sign(&mut self, data: &[u8]) -> Vec { + self.signer.sign(Some(TX_NAMESPACE), data).as_ref().to_vec() + } + + fn verify(&self, data: &[u8], signature: &[u8]) -> Result { + let signature = Signature::try_from(signature); + if signature.is_err() { + return Err(signature.unwrap_err()); + } + let signature = signature.unwrap(); + let pub_key = self.signer.public_key(); + Ok(Ed25519::verify(Some(TX_NAMESPACE), data, &pub_key, &signature)) + } + + fn address(&self) -> Address { + self.address.clone() + } + + fn public_key(&self) -> PublicKey { + self.pub_key.clone() + } + + fn private_key(&self) -> Vec { + self.priv_key.as_ref().to_vec() + } + + fn store_private_key(&self, path: &str) -> Result<(), Error> { + todo!() + } + + fn init_address(&mut self) { + todo!() + } +} \ No newline at end of file From 28391ef5f0cdf8f266129de0b6fbc74ce8761384 Mon Sep 17 00:00:00 2001 From: manojkgorle Date: Mon, 31 Mar 2025 16:48:37 +0530 Subject: [PATCH 24/43] barebones --- Cargo.lock | 8 - Cargo.toml | 3 +- actions/Cargo.toml | 7 - actions/src/msg.rs | 22 -- actions/src/transfer.rs | 55 ----- .../src/lib.rs => types/src/actions/mod.rs | 0 types/src/actions/msg.rs | 42 ++++ types/src/actions/transfer.rs | 56 +++++ types/src/block.rs | 1 - types/src/lib.rs | 1 + types/src/signed_tx.rs | 26 ++- types/src/tx.rs | 205 +++++++++++++++--- types/src/wallet.rs | 12 +- vm/Cargo.toml | 1 - 14 files changed, 298 insertions(+), 141 deletions(-) delete mode 100644 actions/Cargo.toml delete mode 100644 actions/src/msg.rs delete mode 100644 actions/src/transfer.rs rename actions/src/lib.rs => types/src/actions/mod.rs (100%) create mode 100644 types/src/actions/msg.rs create mode 100644 types/src/actions/transfer.rs diff --git a/Cargo.lock b/Cargo.lock index 216dbe90..9ba29a2c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -36,13 +36,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "alto-actions" -version = "0.1.0" -dependencies = [ - "alto-types", -] - [[package]] name = "alto-chain" version = "0.0.4" @@ -144,7 +137,6 @@ dependencies = [ name = "alto-vm" version = "0.1.0" dependencies = [ - "alto-actions", "alto-storage", "alto-types", ] diff --git a/Cargo.toml b/Cargo.toml index 43d48c28..b05eb5ba 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,5 @@ [workspace] -members = [ "actions", +members = [ "chain", "client", "inspector", @@ -12,7 +12,6 @@ resolver = "2" [workspace.dependencies] alto-client = { version = "0.0.4", path = "client" } alto-types = { version = "0.0.4", path = "types" } -alto-actions = { version = "0.1.0", path = "actions"} alto-storage = { version = "0.1.0", path = "storage"} alto-vm = { version = "0.1.0", path = "vm"} commonware-consensus = { version = "0.0.40" } diff --git a/actions/Cargo.toml b/actions/Cargo.toml deleted file mode 100644 index 60a2bf23..00000000 --- a/actions/Cargo.toml +++ /dev/null @@ -1,7 +0,0 @@ -[package] -name = "alto-actions" -version = "0.1.0" -edition = "2021" - -[dependencies] -alto-types = { workspace = true } diff --git a/actions/src/msg.rs b/actions/src/msg.rs deleted file mode 100644 index ac211393..00000000 --- a/actions/src/msg.rs +++ /dev/null @@ -1,22 +0,0 @@ -use alto_types::address::Address; - -pub struct SequencerMsg { - pub chain_id: Vec, - pub data: Vec, - pub from_address: Address, - pub relayer_id: u64, -} - -impl SequencerMsg { - pub fn new() -> Self { - Self { - chain_id: vec![], - data: vec![], - from_address: Address::empty(), - relayer_id: 0, - } - } - pub fn get_type_id(&self) -> u8 { - 0 - } -} \ No newline at end of file diff --git a/actions/src/transfer.rs b/actions/src/transfer.rs deleted file mode 100644 index 9d026c33..00000000 --- a/actions/src/transfer.rs +++ /dev/null @@ -1,55 +0,0 @@ -use alto_types::address::Address; - -const MAX_MEMO_SIZE: usize = 256; - -#[derive(Debug)] -pub struct Transfer { - pub from_address: Address, - pub to_address: Address, - pub value: u64, - pub memo: Vec, -} - -#[derive(Debug)] -pub enum TransferError { - DuplicateAddress, - InvalidToAddress, - InvalidFromAddress, - InsufficientFunds, - TooMuchFunds, - InvalidMemoSize, - StorageError, -} - -impl Transfer { - pub fn new(from_address: Address, to_address: Address, value: u64) -> Result { - let empty_memo = b"".to_vec(); - Self::new_with_memo(from_address, to_address, value, empty_memo) - } - - pub fn new_with_memo(from: Address, to: Address, value: u64, memo: Vec) -> Result { - if value == 0 { - Err(TransferError::InsufficientFunds) - } - else if memo.len() > MAX_MEMO_SIZE { - Err(TransferError::InvalidMemoSize) - } - else if from.is_empty() { - Err(TransferError::InvalidFromAddress) - } - else if to.is_empty() { - Err(TransferError::InvalidToAddress) - } - else if from == to { - Err(TransferError::DuplicateAddress) - } - else { - Ok(Self { - from_address: from, - to_address: to, - value, - memo, - }) - } - } -} \ No newline at end of file diff --git a/actions/src/lib.rs b/types/src/actions/mod.rs similarity index 100% rename from actions/src/lib.rs rename to types/src/actions/mod.rs diff --git a/types/src/actions/msg.rs b/types/src/actions/msg.rs new file mode 100644 index 00000000..cac444dd --- /dev/null +++ b/types/src/actions/msg.rs @@ -0,0 +1,42 @@ +use crate::{address::Address, tx::{Unit, UnitType, UnitContext}, state::State}; + +#[derive(Clone, Debug)] +pub struct SequencerMsg { + pub chain_id: Vec, + pub data: Vec, + pub from_address: Address, + pub relayer_id: u64, +} + +impl Unit for SequencerMsg { + fn unit_type(&self) -> UnitType { + todo!() + } + + fn encode(&self) -> Vec { + todo!() + } + + fn decode(&mut self, bytes: &[u8]) { + todo!() + } + + fn apply( + &self, + context: &UnitContext, + state: &mut Box, + ) -> Result>, Box> { + todo!() + } +} + +impl Default for SequencerMsg { + fn default() -> Self { + Self { + chain_id: vec![], + data: vec![], + from_address: Address::empty(), + relayer_id: 0, + } + } +} \ No newline at end of file diff --git a/types/src/actions/transfer.rs b/types/src/actions/transfer.rs new file mode 100644 index 00000000..b0e15ee5 --- /dev/null +++ b/types/src/actions/transfer.rs @@ -0,0 +1,56 @@ +use crate::address::Address; +use crate::state::State; +use crate::tx::{Unit, UnitType, UnitContext}; +const MAX_MEMO_SIZE: usize = 256; + +#[derive(Debug, Clone)] +pub struct Transfer { + pub from_address: Address, + pub to_address: Address, + pub value: u64, + pub memo: Vec, +} + +#[derive(Debug)] +pub enum TransferError { + DuplicateAddress, + InvalidToAddress, + InvalidFromAddress, + InsufficientFunds, + TooMuchFunds, + InvalidMemoSize, + StorageError, +} + +impl Unit for Transfer { + fn unit_type(&self) -> UnitType { + todo!() + } + + fn encode(&self) -> Vec { + todo!() + } + + fn decode(&mut self, bytes: &[u8]) { + todo!() + } + + fn apply( + &self, + context: &UnitContext, + state: &mut Box, + ) -> Result>, Box> { + todo!() + } +} + +impl Default for Transfer { + fn default() -> Self { + Self { + from_address: Address::empty(), + to_address: Address::empty(), + value: 0, + memo: vec![], + } + } +} \ No newline at end of file diff --git a/types/src/block.rs b/types/src/block.rs index a17c3243..6495b038 100644 --- a/types/src/block.rs +++ b/types/src/block.rs @@ -1,4 +1,3 @@ -use std::os::macos::raw::{self, stat}; use crate::{Finalization, Notarization}; use crate::signed_tx::{SignedTx, SignedTxChars}; diff --git a/types/src/lib.rs b/types/src/lib.rs index b7345718..f9751121 100644 --- a/types/src/lib.rs +++ b/types/src/lib.rs @@ -14,6 +14,7 @@ pub mod signed_tx; pub mod state; pub mod address; pub mod account; +pub mod actions; use rand::rngs::OsRng; diff --git a/types/src/signed_tx.rs b/types/src/signed_tx.rs index 84f9edc8..5e1d803c 100644 --- a/types/src/signed_tx.rs +++ b/types/src/signed_tx.rs @@ -16,7 +16,7 @@ pub struct SignedTx { // function names are self explanatory. pub trait SignedTxChars { fn new(tx: Tx, pub_key: PublicKey, signature: Vec) -> Self; - fn sign(&mut self, wallet: Wallet) -> SignedTx; + // fn sign(&mut self, wallet: Wallet) -> SignedTx; fn verify(&mut self) -> bool; fn signature(&self) -> Vec; fn public_key(&self) -> Vec; @@ -32,17 +32,7 @@ impl SignedTxChars for SignedTx { tx, pub_key: pub_key.clone(), address: Address::from_pub_key(&pub_key), - signature: signature, - } - } - - fn sign(&mut self, mut wallet: Wallet) -> SignedTx { - let tx_data = self.tx.encode(); - SignedTx { - tx: self.tx.clone(), - signature: wallet.sign(&tx_data), - address: wallet.address(), - pub_key: wallet.public_key(), + signature: signature.clone(), } } @@ -75,4 +65,16 @@ impl SignedTxChars for SignedTx { fn decode(&self, bytes: &[u8]) -> Self { todo!() } +} + +impl SignedTx { + pub fn sign(mut tx: Tx, mut wallet: Wallet) -> SignedTx { + let tx_data = tx.encode(); + SignedTx { + tx: tx.clone(), + signature: wallet.sign(&tx_data), + address: wallet.address(), + pub_key: wallet.public_key(), + } + } } \ No newline at end of file diff --git a/types/src/tx.rs b/types/src/tx.rs index 08cc8262..47556f58 100644 --- a/types/src/tx.rs +++ b/types/src/tx.rs @@ -1,19 +1,39 @@ +use commonware_cryptography::sha256; use commonware_cryptography::sha256::Digest; +use crate::actions; use crate::address::Address; use crate::wallet::Wallet; use crate::signed_tx::SignedTx; use crate::state::State; -// use std::fmt::Debug; +use commonware_utils::SystemTimeExt; +use std::time::SystemTime; + +#[derive(Debug)] pub enum UnitType { Transfer, SequencerMsg, } +impl TryFrom for UnitType { + type Error = String; + + fn try_from(value: u8) -> Result { + match value { + 0 => Ok(UnitType::Transfer), + 1 => Ok(UnitType::SequencerMsg), + _ => Err(format!("unknown unit type: {}", value)), + } + } +} + pub struct UnitContext { - pub timestamp: u64, // timestamp of the tx. - pub chain_id: u64, // chain id of the tx. - pub sender: Address, // sender of the tx. + // timestamp of the tx. + pub timestamp: u64, + // chain id of the tx. + pub chain_id: u64, + // sender of the tx. + pub sender: Address, } pub trait UnitClone { @@ -56,14 +76,14 @@ pub struct Tx { // if tx is in a valid window it is added to mempool. // timestamp is used to prevent replay attacks. and counter infinite spam attacks as Tx does not have nonce. pub timestamp: u64, - // units are fundamental unit of a tx. similar to actions. - pub units: Vec>, // max fee is the maximum fee the user is willing to pay for the tx. pub max_fee: u64, // priority fee is the fee the user is willing to pay for the tx to be included in the next block. pub priority_fee: u64, // chain id is the id of the chain the tx is intended for. pub chain_id: u64, + // units are fundamental unit of a tx. similar to actions. + pub units: Vec>, // id is the transaction id. It is the hash of digest. @@ -72,58 +92,189 @@ pub struct Tx { digest: Vec, } -pub trait TxMethods { - // init is used to create a new instance of Tx. - fn init() -> Self; +pub trait TxMethods:Sized { // new is used to create a new instance of Tx with given units and chain id. fn new(units: Vec>, chain_id: u64) -> Self; // set_fee is used to set the max fee and priority fee of the tx. fn set_fee(&mut self, max_fee: u64, priority_fee: u64); // sign is used to sign the tx with the given wallet. - fn sign(&self, wallet: Wallet) -> SignedTx; - + fn sign(&mut self, wallet: Wallet) -> SignedTx; + fn from(timestamp: u64, units: Vec>, priority_fee: u64, max_fee: u64, chain_id: u64) -> Self; // returns tx id. - fn id(&self) -> &[u8;32]; + fn id(&mut self) -> Digest; // returns digest of the tx. fn digest(&self) -> Vec; // encodes the tx, writes to digest and returns the digest. fn encode(&mut self) -> Vec; - fn decode(bytes: &[u8]) -> Self; + fn decode(bytes: &[u8]) -> Result; } -impl TxMethods for Tx { - fn init() -> Self { - todo!() +impl Default for Tx { + fn default() -> Self { + Self { + timestamp: 0, + units: vec![], + max_fee: 0, + priority_fee: 0, + chain_id: 19517, + id: [0; 32].into(), + digest: vec![], + } } +} +impl TxMethods for Tx { fn new(units: Vec>, chain_id: u64) -> Self { - todo!() + let mut tx = Self::default(); + tx.timestamp = SystemTime::now().epoch_millis(); + tx.units = units; + tx.chain_id = chain_id; + + // do not encode and generate tx_id as Tx::new doesnot yet have priority fee and max fee. + tx } fn set_fee(&mut self, max_fee: u64, priority_fee: u64) { - todo!() + self.max_fee = max_fee; + self.priority_fee = priority_fee; + } + + fn sign(&mut self, wallet: Wallet) -> SignedTx { + SignedTx::sign(self.clone(), wallet) } - fn sign(&self, wallet: Wallet) -> SignedTx { - todo!() + fn from(timestamp: u64, units: Vec>, priority_fee: u64, max_fee: u64, chain_id: u64) -> Self { + let mut tx = Self::default(); + tx.timestamp = timestamp; + tx.units = units; + tx.max_fee = max_fee; + tx.priority_fee = priority_fee; + tx.chain_id = chain_id; + tx.encode(); + tx } - fn id(&self) -> &[u8;32] { - todo!() + fn id(&mut self) -> Digest { + if self.digest.len() == 0 { + self.encode(); + } + self.id.clone() } fn digest(&self) -> Vec { - todo!() + self.digest.clone() } fn encode(&mut self) -> Vec { - todo!() + if self.digest.len() > 0 { + return self.digest.clone(); + } + // pack tx timestamp. + self.digest.extend(self.timestamp.to_be_bytes()); + // pack max fee + self.digest.extend(self.max_fee.to_be_bytes()); + // pack priority fee + self.digest.extend(self.priority_fee.to_be_bytes()); + // pack chain id + self.digest.extend(self.chain_id.to_be_bytes()); + // pack # of units. + self.digest.extend((self.units.len() as u64).to_be_bytes()); + // pack individual units + self.units.iter().for_each(|unit| { + let unit_bytes = unit.encode(); + // pack the unit type info. + self.digest.extend((unit.unit_type() as u8).to_be_bytes()); + // pack len of inidividual unit. + self.digest.extend((unit_bytes.len() as u64).to_be_bytes()); + // pack individual unit. + self.digest.extend_from_slice(&unit_bytes); + }); + + // generate tx id. + self.id = sha256::hash(&self.digest); + + // return encoded digest. + self.digest.clone() + } + + fn decode(bytes: &[u8]) -> Result { + if bytes.len() == 0 { + return Err("Empty bytes".to_string()); + } + let mut tx = Self::default(); + tx.digest = bytes.to_vec(); // @todo ?? + tx.timestamp = u64::from_be_bytes(bytes[0..8].try_into().unwrap()); + tx.max_fee = u64::from_be_bytes(bytes[8..16].try_into().unwrap()); + tx.priority_fee = u64::from_be_bytes(bytes[16..24].try_into().unwrap()); + tx.chain_id = u64::from_be_bytes(bytes[24..32].try_into().unwrap()); + unpack_units(&bytes[32..]); + Ok(tx) + } +} + +fn unpack_units(digest: &[u8]) -> Result>, String> { + let mut offset = 0; + + fn read_u8(input: &[u8], offset: &mut usize) -> Result { + if input.len() < *offset + 1 { + return Err("Unexpected end of input when reading u8".into()); + } + let val = input[*offset]; + *offset += 1; + Ok(val) + } + + fn read_u64(input: &[u8], offset: &mut usize) -> Result { + if input.len() < *offset + 8 { + return Err("Unexpected end of input when reading u64".into()); + } + let val = u64::from_be_bytes(input[*offset..*offset + 8].try_into().unwrap()); + *offset += 8; + Ok(val) } - fn decode(bytes: &[u8]) -> Self { - todo!() + fn read_bytes<'a>(input: &'a [u8], offset: &'a mut usize, len: usize) -> Result<&'a [u8], String> { + if input.len() < *offset + len { + return Err("Unexpected end of input when reading bytes".into()); + } + let bytes = &input[*offset..*offset + len]; + *offset += len; + Ok(bytes) } -} \ No newline at end of file + + let unit_count = read_u64(digest, &mut offset)?; + + let mut units:Vec> = Vec::with_capacity(unit_count as usize); + + for _ in 0..unit_count { + let unit_type = read_u8(digest, &mut offset)?; + let unit_len = read_u64(digest, &mut offset)?; + let unit_bytes = read_bytes(digest, &mut offset, unit_len as usize)?.to_vec(); + let unit_type = UnitType::try_from(unit_type); + if unit_type.is_err() { + return Err(format!("Invalid unit type: {}", unit_type.unwrap_err())); + } + let unit_type = unit_type.unwrap(); + let unit:Box = match unit_type { + UnitType::Transfer => { + let mut transfer = actions::transfer::Transfer::default(); + transfer.decode(&unit_bytes); + Box::new(transfer) + } + UnitType::SequencerMsg => { + let mut msg = actions::msg::SequencerMsg::default(); + msg.decode(&unit_bytes); + Box::new(msg) + } + }; + units.push(unit); + } + + Ok(units) +} + + +// @todo implement tests for encoding and decoding of tx. \ No newline at end of file diff --git a/types/src/wallet.rs b/types/src/wallet.rs index a2a0aff0..fa1ff922 100644 --- a/types/src/wallet.rs +++ b/types/src/wallet.rs @@ -4,7 +4,6 @@ use commonware_cryptography::ed25519::Ed25519; use commonware_cryptography::Scheme; use rand::{CryptoRng, Rng}; use std::fmt::Error; -use std::path; #[derive(Clone, Debug)] pub enum AuthTypes { @@ -66,8 +65,8 @@ impl WalletMethods for Wallet { Self { priv_key: signer.private_key(), pub_key: signer.public_key(), - address: address, - signer: signer, + address, + signer, } } @@ -78,7 +77,7 @@ impl WalletMethods for Wallet { priv_key: signer.private_key(), pub_key: signer.public_key(), address: Address::from_pub_key(&signer.public_key()), - signer: signer, + signer, } } @@ -88,9 +87,10 @@ impl WalletMethods for Wallet { fn verify(&self, data: &[u8], signature: &[u8]) -> Result { let signature = Signature::try_from(signature); - if signature.is_err() { - return Err(signature.unwrap_err()); + if let Err(e) = signature { + return Err(e); } + let signature = signature.unwrap(); let pub_key = self.signer.public_key(); Ok(Ed25519::verify(Some(TX_NAMESPACE), data, &pub_key, &signature)) diff --git a/vm/Cargo.toml b/vm/Cargo.toml index 3ef612cc..838d16df 100644 --- a/vm/Cargo.toml +++ b/vm/Cargo.toml @@ -5,5 +5,4 @@ edition = "2021" [dependencies] alto-storage = { workspace = true } -alto-actions = { workspace = true } alto-types = { workspace = true } From 70c8f223cc77befd3bb14f42173a8891b95aebb4 Mon Sep 17 00:00:00 2001 From: manojkgorle Date: Mon, 31 Mar 2025 17:13:52 +0530 Subject: [PATCH 25/43] its nits units --- types/src/actions/msg.rs | 42 ---------------- types/src/lib.rs | 2 +- types/src/tx.rs | 15 ++++-- types/src/{actions => units}/mod.rs | 0 types/src/units/msg.rs | 61 ++++++++++++++++++++++++ types/src/{actions => units}/transfer.rs | 25 ++++++++-- 6 files changed, 95 insertions(+), 50 deletions(-) delete mode 100644 types/src/actions/msg.rs rename types/src/{actions => units}/mod.rs (100%) create mode 100644 types/src/units/msg.rs rename types/src/{actions => units}/transfer.rs (53%) diff --git a/types/src/actions/msg.rs b/types/src/actions/msg.rs deleted file mode 100644 index cac444dd..00000000 --- a/types/src/actions/msg.rs +++ /dev/null @@ -1,42 +0,0 @@ -use crate::{address::Address, tx::{Unit, UnitType, UnitContext}, state::State}; - -#[derive(Clone, Debug)] -pub struct SequencerMsg { - pub chain_id: Vec, - pub data: Vec, - pub from_address: Address, - pub relayer_id: u64, -} - -impl Unit for SequencerMsg { - fn unit_type(&self) -> UnitType { - todo!() - } - - fn encode(&self) -> Vec { - todo!() - } - - fn decode(&mut self, bytes: &[u8]) { - todo!() - } - - fn apply( - &self, - context: &UnitContext, - state: &mut Box, - ) -> Result>, Box> { - todo!() - } -} - -impl Default for SequencerMsg { - fn default() -> Self { - Self { - chain_id: vec![], - data: vec![], - from_address: Address::empty(), - relayer_id: 0, - } - } -} \ No newline at end of file diff --git a/types/src/lib.rs b/types/src/lib.rs index f9751121..e8acc08f 100644 --- a/types/src/lib.rs +++ b/types/src/lib.rs @@ -14,7 +14,7 @@ pub mod signed_tx; pub mod state; pub mod address; pub mod account; -pub mod actions; +pub mod units; use rand::rngs::OsRng; diff --git a/types/src/tx.rs b/types/src/tx.rs index 47556f58..996935ef 100644 --- a/types/src/tx.rs +++ b/types/src/tx.rs @@ -1,7 +1,7 @@ use commonware_cryptography::sha256; use commonware_cryptography::sha256::Digest; -use crate::actions; +use crate::units; use crate::address::Address; use crate::wallet::Wallet; use crate::signed_tx::SignedTx; @@ -210,7 +210,14 @@ impl TxMethods for Tx { tx.max_fee = u64::from_be_bytes(bytes[8..16].try_into().unwrap()); tx.priority_fee = u64::from_be_bytes(bytes[16..24].try_into().unwrap()); tx.chain_id = u64::from_be_bytes(bytes[24..32].try_into().unwrap()); - unpack_units(&bytes[32..]); + let units = unpack_units(&bytes[32..]); + if units.is_err() { + return Err(format!("Failed to unpack units: {}", units.unwrap_err())); + } + tx.units = units.unwrap(); + // generate tx id. + tx.id = sha256::hash(&tx.digest); + // return transaction. Ok(tx) } } @@ -260,12 +267,12 @@ fn unpack_units(digest: &[u8]) -> Result>, String> { let unit_type = unit_type.unwrap(); let unit:Box = match unit_type { UnitType::Transfer => { - let mut transfer = actions::transfer::Transfer::default(); + let mut transfer = units::transfer::Transfer::default(); transfer.decode(&unit_bytes); Box::new(transfer) } UnitType::SequencerMsg => { - let mut msg = actions::msg::SequencerMsg::default(); + let mut msg = units::msg::SequencerMsg::default(); msg.decode(&unit_bytes); Box::new(msg) } diff --git a/types/src/actions/mod.rs b/types/src/units/mod.rs similarity index 100% rename from types/src/actions/mod.rs rename to types/src/units/mod.rs diff --git a/types/src/units/msg.rs b/types/src/units/msg.rs new file mode 100644 index 00000000..3023c73a --- /dev/null +++ b/types/src/units/msg.rs @@ -0,0 +1,61 @@ +use crate::{address::Address, tx::{Unit, UnitType, UnitContext}, state::State}; + +#[derive(Clone, Debug)] +pub struct SequencerMsg { + pub chain_id: u64, + pub data: Vec, + pub from_address: Address, + pub relayer_id: u64, +} + +impl Unit for SequencerMsg { + fn unit_type(&self) -> UnitType { + UnitType::SequencerMsg + } + + fn encode(&self) -> Vec { + let mut bytes:Vec = Vec::new(); + // data length is 8 bytes. + let data_len = self.data.len() as u64; + // chain id length is 8 bytes.n store chain id. + bytes.extend(&self.chain_id.to_be_bytes()); + // address length is 32. store address. + bytes.extend_from_slice(self.from_address.as_slice()); + // relayer id length is 8 bytes. store relayer id. + bytes.extend(self.relayer_id.to_be_bytes()); + // store data length. + bytes.extend(data_len.to_be_bytes()); + // store data. + bytes.extend_from_slice(&self.data); + + bytes + } + + // @todo introduce syntactic checks. + fn decode(&mut self, bytes: &[u8]) { + self.chain_id = u64::from_be_bytes(bytes[0..8].try_into().unwrap()); + self.from_address = Address::from_bytes(&bytes[8..40]).unwrap(); + self.relayer_id = u64::from_be_bytes(bytes[40..48].try_into().unwrap()); + let data_len = u64::from_be_bytes(bytes[48..56].try_into().unwrap()); + self.data = bytes[56..(56 + data_len as usize)].to_vec(); + } + + fn apply( + &self, + context: &UnitContext, + state: &mut Box, + ) -> Result>, Box> { + todo!() + } +} + +impl Default for SequencerMsg { + fn default() -> Self { + Self { + chain_id: 0, + data: vec![], + from_address: Address::empty(), + relayer_id: 0, + } + } +} \ No newline at end of file diff --git a/types/src/actions/transfer.rs b/types/src/units/transfer.rs similarity index 53% rename from types/src/actions/transfer.rs rename to types/src/units/transfer.rs index b0e15ee5..75d0cfb6 100644 --- a/types/src/actions/transfer.rs +++ b/types/src/units/transfer.rs @@ -1,6 +1,8 @@ use crate::address::Address; use crate::state::State; use crate::tx::{Unit, UnitType, UnitContext}; + +use super::msg::SequencerMsg; const MAX_MEMO_SIZE: usize = 256; #[derive(Debug, Clone)] @@ -24,15 +26,32 @@ pub enum TransferError { impl Unit for Transfer { fn unit_type(&self) -> UnitType { - todo!() + UnitType::Transfer } fn encode(&self) -> Vec { - todo!() + let mut bytes = Vec::new(); + let memo_len = self.memo.len() as u64; + bytes.extend_from_slice(self.from_address.as_slice()); + bytes.extend_from_slice(self.to_address.as_slice()); + bytes.extend(self.value.to_be_bytes()); + bytes.extend(memo_len.to_be_bytes()); + if memo_len > 0 { + bytes.extend_from_slice(&self.memo); + } + + bytes } + // @todo introduce syntactic checks. fn decode(&mut self, bytes: &[u8]) { - todo!() + self.from_address = Address::from_bytes(&bytes[0..32]).unwrap(); + self.to_address = Address::from_bytes(&bytes[32..64]).unwrap(); + self.value = u64::from_be_bytes(bytes[64..72].try_into().unwrap()); + let memo_len = u64::from_be_bytes(bytes[72..80].try_into().unwrap()); + if memo_len > 0 { + self.memo = bytes[80..(80 + memo_len as usize)].to_vec(); + } } fn apply( From 135a93520d1c2d9500e5b127c1c3e32d8d95c0e8 Mon Sep 17 00:00:00 2001 From: manojkgorle Date: Mon, 31 Mar 2025 18:17:24 +0530 Subject: [PATCH 26/43] some more work --- types/src/block.rs | 12 ++++--- types/src/codec.rs | 8 ----- types/src/lib.rs | 2 -- types/src/signed_tx.rs | 72 +++++++++++++++++++++++++++++++++++++----- types/src/tx.rs | 6 ++-- 5 files changed, 75 insertions(+), 25 deletions(-) delete mode 100644 types/src/codec.rs diff --git a/types/src/block.rs b/types/src/block.rs index 6495b038..009e6f28 100644 --- a/types/src/block.rs +++ b/types/src/block.rs @@ -1,6 +1,7 @@ +use std::os::macos::raw::{self, stat}; use crate::{Finalization, Notarization}; -use crate::signed_tx::{SignedTx, SignedTxChars}; +use crate::signed_tx::{SignedTx, pack_signed_txs, unpack_signed_txs}; use bytes::{Buf, BufMut}; use commonware_cryptography::{bls12381::PublicKey, sha256::Digest, Hasher, Sha256}; use commonware_utils::{Array, SizedSerialize}; @@ -41,7 +42,9 @@ impl Block { } pub fn new(parent: Digest, height: u64, timestamp: u64, txs: Vec, state_root: Digest) -> Self { - let raw_txs = txs.iter().flat_map(|tx| tx.encode()).collect::>(); + // let mut txs = txs; + // @todo this is packing txs in a block. + let raw_txs = pack_signed_txs(txs.clone()); let digest = Self::compute_digest(&parent, height, timestamp, raw_txs.clone(), &state_root); Self { parent, @@ -74,9 +77,10 @@ impl Block { let timestamp = bytes.get_u64(); let state_root = Digest::read_from(&mut bytes).ok()?; let raw_txs = bytes.to_vec(); - // Return block let digest = Self::compute_digest(&parent, height, timestamp, raw_txs.clone(), &state_root); - let txs = Vec::new(); // @todo deserialze txs from raw_txs. + let txs = unpack_signed_txs(raw_txs.clone()); + + // Return block Some(Self { parent, height, diff --git a/types/src/codec.rs b/types/src/codec.rs deleted file mode 100644 index d8688a2a..00000000 --- a/types/src/codec.rs +++ /dev/null @@ -1,8 +0,0 @@ -use commonware_codec::Writer; -use commonware_cryptography::ed25519::PublicKey; - -pub fn serialize_pk(pk: &PublicKey, writer: &mut impl Writer) { - // let slice: &[u8] = pk.as_ref(); - // let length = slice.len(); - // length.serialize(&mut *writer).unwrap(); -} \ No newline at end of file diff --git a/types/src/lib.rs b/types/src/lib.rs index e8acc08f..d31cdb81 100644 --- a/types/src/lib.rs +++ b/types/src/lib.rs @@ -7,7 +7,6 @@ pub use block::{Block, Finalized, Notarized}; mod consensus; pub use consensus::{leader_index, Finalization, Kind, Notarization, Nullification, Seed}; pub mod wasm; -pub mod codec; pub mod wallet; pub mod tx; pub mod signed_tx; @@ -33,7 +32,6 @@ type PublicKey = commonware_cryptography::ed25519::PublicKey; type PrivateKey = commonware_cryptography::ed25519::PrivateKey; type Signature = commonware_cryptography::ed25519::Signature; - pub fn create_test_keypair() -> (PublicKey, PrivateKey) { let mut rng = OsRng; // generates keypair using random number generator diff --git a/types/src/signed_tx.rs b/types/src/signed_tx.rs index 5e1d803c..d6e1f08c 100644 --- a/types/src/signed_tx.rs +++ b/types/src/signed_tx.rs @@ -14,15 +14,15 @@ pub struct SignedTx { } // function names are self explanatory. -pub trait SignedTxChars { +pub trait SignedTxChars:Sized { fn new(tx: Tx, pub_key: PublicKey, signature: Vec) -> Self; // fn sign(&mut self, wallet: Wallet) -> SignedTx; fn verify(&mut self) -> bool; fn signature(&self) -> Vec; fn public_key(&self) -> Vec; fn address(&self) -> Address; - fn encode(&self) -> Vec; - fn decode(&self, bytes: &[u8]) -> Self; + fn encode(&mut self) -> Vec; + fn decode(bytes: &[u8]) -> Result; } impl SignedTxChars for SignedTx { @@ -51,19 +51,44 @@ impl SignedTxChars for SignedTx { } fn public_key(&self) -> Vec { - self.pub_key.as_ref().to_vec() + self.pub_key.to_vec() } fn address(&self) -> Address { self.address.clone() } - fn encode(&self) -> Vec { - todo!() + // @todo add syntactic checks. + fn encode(&mut self) -> Vec { + let mut bytes = Vec::new(); + + let raw_tx = self.tx.encode(); + let raw_tx_len = raw_tx.len() as u64; + bytes.extend(raw_tx_len.to_be_bytes()); + bytes.extend_from_slice(&raw_tx); + bytes.extend_from_slice(&self.pub_key); + bytes.extend_from_slice(&self.signature); + bytes } - fn decode(&self, bytes: &[u8]) -> Self { - todo!() + // @todo add syntactic checks and use methods consume. + fn decode(bytes: &[u8]) -> Result { // @todo this method seems untidy. + + let raw_tx_len = u64::from_be_bytes(bytes[0..8].try_into().unwrap()); + let raw_tx = &bytes[8..8+raw_tx_len as usize]; + let pub_key = &bytes[8+raw_tx_len as usize..8+raw_tx_len as usize+32]; + let signature = &bytes[8+raw_tx_len as usize+32..]; + let public_key = PublicKey::try_from(pub_key); + if public_key.is_err() { + return Err(public_key.unwrap_err().to_string()); + } + let public_key = public_key.unwrap(); + let tx = Tx::decode(raw_tx); + if tx.is_err(){ + return Err(tx.unwrap_err()); + } + + Ok(SignedTx { tx: tx.unwrap(), pub_key: public_key.clone(), address: Address::from_pub_key(&public_key), signature: signature.to_vec() }) } } @@ -77,4 +102,35 @@ impl SignedTx { pub_key: wallet.public_key(), } } +} + +pub fn pack_signed_txs(signed_txs: Vec) -> Vec { + let mut bytes = Vec::new(); + bytes.extend((signed_txs.len() as u64).to_be_bytes()); + for signed_tx in signed_txs { + // @todo improvise + let mut signed_tx = signed_tx; + let signed_tx_bytes = signed_tx.encode(); + bytes.extend((signed_tx_bytes.len() as u64).to_be_bytes()); + bytes.extend_from_slice(&signed_tx_bytes); + } + bytes +} + +pub fn unpack_signed_txs(bytes: Vec) -> Vec { + let signed_txs_len = u64::from_be_bytes(bytes[0..8].try_into().unwrap()); + let mut signed_txs = Vec::with_capacity(signed_txs_len as usize); + let mut offset = 8; + for i in 0..signed_txs_len { + let signed_tx_len = u64::from_be_bytes(bytes[offset..offset + 8].try_into().unwrap()); + offset += 8; + let signed_tx_bytes = &bytes[offset..offset + signed_tx_len as usize]; + offset += signed_tx_len as usize; + let signed_tx = SignedTx::decode(signed_tx_bytes); + if signed_tx.is_err() { + panic!("Failed to unpack signed tx: {}", signed_tx.unwrap_err()); + } + signed_txs.push(signed_tx.unwrap()); + } + signed_txs } \ No newline at end of file diff --git a/types/src/tx.rs b/types/src/tx.rs index 996935ef..6dda43fc 100644 --- a/types/src/tx.rs +++ b/types/src/tx.rs @@ -106,9 +106,9 @@ pub trait TxMethods:Sized { // returns digest of the tx. fn digest(&self) -> Vec; // encodes the tx, writes to digest and returns the digest. + // ensure all fields are properly set before calling this function. fn encode(&mut self) -> Vec; - fn decode(bytes: &[u8]) -> Result; } @@ -169,7 +169,7 @@ impl TxMethods for Tx { } fn encode(&mut self) -> Vec { - if self.digest.len() > 0 { + if self.digest.is_empty() { return self.digest.clone(); } // pack tx timestamp. @@ -201,7 +201,7 @@ impl TxMethods for Tx { } fn decode(bytes: &[u8]) -> Result { - if bytes.len() == 0 { + if bytes.is_empty() { return Err("Empty bytes".to_string()); } let mut tx = Self::default(); From 52bd3115faec23fd27b6905d20df74e39b92ca91 Mon Sep 17 00:00:00 2001 From: manojkgorle Date: Mon, 31 Mar 2025 18:52:33 +0530 Subject: [PATCH 27/43] more progress --- Cargo.lock | 15 ++------------- chain/src/actors/application/router.rs | 6 ++---- storage/Cargo.toml | 2 +- storage/src/hashmap_db.rs | 6 ++++++ storage/src/rocks_db.rs | 4 +--- storage/src/state_db.rs | 6 ++++-- storage/src/transactional_db.rs | 7 +++---- storage/src/tx_state_view.rs | 5 ----- types/src/account.rs | 1 + types/src/block.rs | 2 -- types/src/signed_tx.rs | 2 +- types/src/units/transfer.rs | 1 - types/src/wallet.rs | 2 +- 13 files changed, 22 insertions(+), 37 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9ba29a2c..a078abaa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -109,7 +109,7 @@ version = "0.1.0" dependencies = [ "alto-types", "bytes", - "commonware-codec 0.0.43", + "commonware-codec", "commonware-cryptography", "rand", "rocksdb", @@ -121,7 +121,7 @@ name = "alto-types" version = "0.0.4" dependencies = [ "bytes", - "commonware-codec 0.0.40", + "commonware-codec", "commonware-cryptography", "commonware-utils", "getrandom 0.2.15", @@ -890,17 +890,6 @@ dependencies = [ "thiserror 2.0.12", ] -[[package]] -name = "commonware-codec" -version = "0.0.43" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65ebb5fc2fbf7d3fbdaf8d3dd3afdb80f03ee02a7d5a9fefc979235a2c06569a" -dependencies = [ - "bytes", - "paste", - "thiserror 2.0.12", -] - [[package]] name = "commonware-consensus" version = "0.0.40" diff --git a/chain/src/actors/application/router.rs b/chain/src/actors/application/router.rs index 65a7fff5..efb6135a 100644 --- a/chain/src/actors/application/router.rs +++ b/chain/src/actors/application/router.rs @@ -1,11 +1,9 @@ use std::io; use axum::response::IntoResponse; -use axum::routing::get; use commonware_runtime::{Clock, Handle, Metrics, Spawner}; use rand::Rng; use tokio::net::TcpListener; use tracing::{event, Level}; -use crate::actors::syncer; pub struct RouterConfig { port: i32, @@ -69,13 +67,13 @@ impl Router { "Submit block" } - // fn init_router(&mut self) { + fn init_router(&mut self) { // let router = axum::Router::new() // .route("/", get(Router::handle_default)) // .route(Router::PATH_SUBMIT_BLOCK, get(Router::handle_submit_block())); // // self.router = Some(router) - // } + } async fn serve(&mut self) -> Result<(), Box> { let listener = self.listener.take().ok_or("serve failed because listener is None"); diff --git a/storage/Cargo.toml b/storage/Cargo.toml index cba09b57..aded60a6 100644 --- a/storage/Cargo.toml +++ b/storage/Cargo.toml @@ -8,6 +8,6 @@ alto-types = { workspace = true } commonware-cryptography = { workspace = true } rand = "0.8.5" rocksdb = "0.23.0" -commonware-codec = { version = "0.0.43" } +commonware-codec = { workspace = true} bytes = "1.7.1" tempfile = "3.18.0" \ No newline at end of file diff --git a/storage/src/hashmap_db.rs b/storage/src/hashmap_db.rs index 7abe6420..cdf4155e 100644 --- a/storage/src/hashmap_db.rs +++ b/storage/src/hashmap_db.rs @@ -6,6 +6,12 @@ pub struct HashmapDatabase { data: HashMap, } +impl Default for HashmapDatabase { + fn default() -> Self { + Self::new() + } +} + impl HashmapDatabase { pub fn new() -> Self { Self { diff --git a/storage/src/rocks_db.rs b/storage/src/rocks_db.rs index 16f6f200..a779c257 100644 --- a/storage/src/rocks_db.rs +++ b/storage/src/rocks_db.rs @@ -1,9 +1,7 @@ use std::error::Error; use rocksdb::{DB, Options}; -use commonware_codec::{Codec}; use crate::database::Database; use std::path::Path; -use bytes::{BufMut}; use tempfile::TempDir; const SAL_ROCKS_DB_PATH: &str = "rocksdb"; @@ -22,7 +20,7 @@ impl RocksDbDatabase { opts.create_if_missing(true); let db_path = Path::new(path); - let db = DB::open(&opts, &db_path)?; + let db = DB::open(&opts, db_path)?; Ok(RocksDbDatabase { db }) } diff --git a/storage/src/state_db.rs b/storage/src/state_db.rs index 092987f5..60b7096a 100644 --- a/storage/src/state_db.rs +++ b/storage/src/state_db.rs @@ -1,11 +1,10 @@ use crate::database::Database; -use crate::transactional_db::{TransactionalDb,InMemoryCachingTransactionalDb}; +use crate::transactional_db::TransactionalDb; use alto_types::account::{Account, Balance}; use alto_types::address::Address; use bytes::Bytes; use commonware_codec::{Codec, ReadBuffer, WriteBuffer}; use std::error::Error; -use crate::rocks_db::RocksDbDatabase; const ACCOUNTS_PREFIX: u8 = 0x0; const DB_WRITE_BUFFER_CAPACITY: usize = 500; @@ -90,6 +89,9 @@ impl Database for StateDb { mod tests { use alto_types::address::Address; use alto_types::account::Account; + use crate::rocks_db::RocksDbDatabase; + use crate::transactional_db::InMemoryCachingTransactionalDb; + use super::*; fn setup_state_db() -> StateDb { diff --git a/storage/src/transactional_db.rs b/storage/src/transactional_db.rs index ba861414..fc302a19 100644 --- a/storage/src/transactional_db.rs +++ b/storage/src/transactional_db.rs @@ -1,6 +1,5 @@ use std::collections::HashMap; use std::error::Error; -use alto_types::state::State; use crate::database::Database; const ACCOUNT_KEY_TYPE: u8 = 0; @@ -77,7 +76,7 @@ impl TransactionalDb for InMemoryCachingTransactionalDb { fn commit(&mut self) -> Result<(), Box> { for (key, value) in self.touched.iter() { - self.cache.insert(key.clone(), value.clone()); + self.cache.insert(*key, value.clone()); //TODO: what to do if an intermediary operation fails maybe use rocks db transact? // ex: what if we go through half of touched and it fails halfway? rare but possible. self.db.put(&key[..], value)?; @@ -101,13 +100,13 @@ impl Database for InMemoryCachingTransactionalDb { fn get(&mut self, key: &[u8]) -> Result>, Box> { match self.get_from_cache(&Key::try_from(key).unwrap()) { Ok(Some(value)) => { - Ok(Some(value.into())) + Ok(Some(value)) }, Ok(None) => { match self.db.get(key) { Ok(Some(value)) => { self.cache.insert(Key::try_from(key).unwrap(), value.clone()); - Ok(Some(value.into())) + Ok(Some(value)) } Ok(None) => { Ok(None) diff --git a/storage/src/tx_state_view.rs b/storage/src/tx_state_view.rs index 16e74a27..907c637f 100644 --- a/storage/src/tx_state_view.rs +++ b/storage/src/tx_state_view.rs @@ -1,8 +1,3 @@ -use std::collections::HashMap; -use std::error::Error; -use alto_types::address::Address; -use crate::database::Database; -use alto_types::state::{State}; //use crate::state_db::StateDb; //const ACCOUNT_KEY_TYPE: u8 = 0; diff --git a/types/src/account.rs b/types/src/account.rs index 061b7ca4..955980e2 100644 --- a/types/src/account.rs +++ b/types/src/account.rs @@ -30,6 +30,7 @@ impl Account { } } } + impl Codec for Account { fn write(&self, writer: &mut impl Writer) { writer.write_bytes(self.address.0.as_slice()); diff --git a/types/src/block.rs b/types/src/block.rs index 009e6f28..8052ab74 100644 --- a/types/src/block.rs +++ b/types/src/block.rs @@ -1,5 +1,3 @@ -use std::os::macos::raw::{self, stat}; - use crate::{Finalization, Notarization}; use crate::signed_tx::{SignedTx, pack_signed_txs, unpack_signed_txs}; use bytes::{Buf, BufMut}; diff --git a/types/src/signed_tx.rs b/types/src/signed_tx.rs index d6e1f08c..cf5860e1 100644 --- a/types/src/signed_tx.rs +++ b/types/src/signed_tx.rs @@ -121,7 +121,7 @@ pub fn unpack_signed_txs(bytes: Vec) -> Vec { let signed_txs_len = u64::from_be_bytes(bytes[0..8].try_into().unwrap()); let mut signed_txs = Vec::with_capacity(signed_txs_len as usize); let mut offset = 8; - for i in 0..signed_txs_len { + for _ in 0..signed_txs_len { let signed_tx_len = u64::from_be_bytes(bytes[offset..offset + 8].try_into().unwrap()); offset += 8; let signed_tx_bytes = &bytes[offset..offset + signed_tx_len as usize]; diff --git a/types/src/units/transfer.rs b/types/src/units/transfer.rs index 75d0cfb6..fe99ef9e 100644 --- a/types/src/units/transfer.rs +++ b/types/src/units/transfer.rs @@ -2,7 +2,6 @@ use crate::address::Address; use crate::state::State; use crate::tx::{Unit, UnitType, UnitContext}; -use super::msg::SequencerMsg; const MAX_MEMO_SIZE: usize = 256; #[derive(Debug, Clone)] diff --git a/types/src/wallet.rs b/types/src/wallet.rs index fa1ff922..f61b46a3 100644 --- a/types/src/wallet.rs +++ b/types/src/wallet.rs @@ -108,7 +108,7 @@ impl WalletMethods for Wallet { self.priv_key.as_ref().to_vec() } - fn store_private_key(&self, path: &str) -> Result<(), Error> { + fn store_private_key(&self, _path: &str) -> Result<(), Error> { todo!() } From 35a6b23afadb3e86ad58b5701e1e330bfad04550 Mon Sep 17 00:00:00 2001 From: rikoeldon <106416799+rikoeldon@users.noreply.github.com> Date: Mon, 31 Mar 2025 15:57:47 -0400 Subject: [PATCH 28/43] added tests for encode/decode --- storage/src/state_db.rs | 2 +- types/src/address.rs | 7 ++++ types/src/block.rs | 21 +++++++++-- types/src/lib.rs | 6 ++++ types/src/signed_tx.rs | 53 ++++++++++++++++++++++++++- types/src/tx.rs | 72 ++++++++++++++++++++++++++++++++++--- types/src/units/msg.rs | 47 ++++++++++++++++++++++++ types/src/units/transfer.rs | 48 +++++++++++++++++++++++++ 8 files changed, 247 insertions(+), 9 deletions(-) diff --git a/storage/src/state_db.rs b/storage/src/state_db.rs index 60b7096a..4c4056f9 100644 --- a/storage/src/state_db.rs +++ b/storage/src/state_db.rs @@ -101,7 +101,7 @@ mod tests { StateDb::new(Box::new(db)) } #[test] - fn test_rocks_db_accounts() { + fn test_statedb_accounts() { let mut state_db = setup_state_db(); let mut account = Account::new(); diff --git a/types/src/address.rs b/types/src/address.rs index a671ff45..c14b7b0f 100644 --- a/types/src/address.rs +++ b/types/src/address.rs @@ -1,4 +1,5 @@ use more_asserts::assert_le; +use rand::Rng; use crate::{PublicKey, ADDRESSLEN}; #[derive(Hash, Eq, PartialEq, Clone, Debug)] @@ -12,6 +13,12 @@ impl Address { Address(arr) } + pub fn create_random_address() -> Self { + let mut arr = [0u8; ADDRESSLEN]; + rand::thread_rng().fill(&mut arr); + Address(arr) + } + pub fn from_pub_key(pub_key: &PublicKey) -> Self { // @todo implement a hasher to derive the address from the public key. assert_le!(pub_key.len(), ADDRESSLEN, "public key is too large"); diff --git a/types/src/block.rs b/types/src/block.rs index 8052ab74..142d19f5 100644 --- a/types/src/block.rs +++ b/types/src/block.rs @@ -1,7 +1,7 @@ use crate::{Finalization, Notarization}; -use crate::signed_tx::{SignedTx, pack_signed_txs, unpack_signed_txs}; +use crate::signed_tx::{SignedTx, pack_signed_txs, unpack_signed_txs, SignedTxChars}; use bytes::{Buf, BufMut}; -use commonware_cryptography::{bls12381::PublicKey, sha256::Digest, Hasher, Sha256}; +use commonware_cryptography::{bls12381::PublicKey, sha256, sha256::Digest, Hasher, Sha256}; use commonware_utils::{Array, SizedSerialize}; // @todo add state root, fee manager and results to the block struct. @@ -93,6 +93,23 @@ impl Block { pub fn digest(&self) -> Digest { self.digest.clone() } + //todo check logic below + pub fn encode(&mut self) -> Vec { + let mut bytes: Vec = Vec::new(); + bytes.extend_from_slice(&self.parent); + bytes.extend_from_slice(&(self.height.to_be_bytes())); + bytes.extend_from_slice(&(self.timestamp.to_be_bytes())); + bytes.extend_from_slice(self.raw_txs.as_slice()); + bytes.extend_from_slice(&self.state_root); + // encoding signed txs + for tx in self.txs.iter_mut() { + bytes.extend_from_slice(&tx.encode()); + } + + // return encoded digest. + self.digest = sha256::hash(&bytes); + bytes + } } impl SizedSerialize for Block { diff --git a/types/src/lib.rs b/types/src/lib.rs index d31cdb81..a0f1508c 100644 --- a/types/src/lib.rs +++ b/types/src/lib.rs @@ -2,7 +2,9 @@ mod block; +use std::time::SystemTime; use commonware_cryptography::{Ed25519, Scheme}; +use commonware_utils::SystemTimeExt; pub use block::{Block, Finalized, Notarized}; mod consensus; pub use consensus::{leader_index, Finalization, Kind, Notarization, Nullification, Seed}; @@ -43,6 +45,10 @@ pub fn create_test_keypair() -> (PublicKey, PrivateKey) { (public_key, private_key) } +pub fn curr_timestamp() -> u64 { + SystemTime::now().epoch_millis() +} + #[cfg(test)] mod tests { use super::*; diff --git a/types/src/signed_tx.rs b/types/src/signed_tx.rs index cf5860e1..7331fe8e 100644 --- a/types/src/signed_tx.rs +++ b/types/src/signed_tx.rs @@ -133,4 +133,55 @@ pub fn unpack_signed_txs(bytes: Vec) -> Vec { signed_txs.push(signed_tx.unwrap()); } signed_txs -} \ No newline at end of file +} + +#[cfg(test)] +mod tests { + use std::error::Error; + use commonware_cryptography::bls12381::primitives::ops::keypair; + use commonware_cryptography::sha256::Digest; + use more_asserts::assert_gt; + use crate::{create_test_keypair, curr_timestamp}; + use crate::tx::Unit; + use crate::units::transfer::Transfer; + use super::*; + + #[test] + fn test_encode_decode() -> Result<(), Box> { + let timestamp = curr_timestamp(); + let max_fee = 100; + let priority_fee = 75; + let chain_id = 45205; + let transfer = Transfer::new(); + let units : Vec> = vec![Box::new(transfer)]; + let mut digest: [u8; 32] = [0; 32]; + let id = Digest::from(digest.clone()); + let (pk, sk) = create_test_keypair(); + // TODO: the .encode call on next line gave error and said origin_msg needed to be mut? but why? + // shouldn't encode be able to encode without changing the msg? + let mut tx = Tx { + timestamp, + max_fee, + priority_fee, + chain_id, + units: units.clone(), + id, + digest: digest.to_vec(), + }; + let mut origin_msg = SignedTx { + tx, + pub_key: pk, + address: Address::create_random_address(), + signature: vec![], + }; + let encoded_bytes = origin_msg.encode(); + assert_gt!(encoded_bytes.len(), 0); + let mut decoded_msg = SignedTx::decode(&encoded_bytes)?; + assert_eq!(origin_msg.pub_key, decoded_msg.pub_key); + assert_eq!(origin_msg.address, decoded_msg.address); + assert_eq!(origin_msg.signature, decoded_msg.signature); + // @todo make helper to compare fields in tx and units. same issue when testing in tx.rs file. + Ok(()) + } +} + diff --git a/types/src/tx.rs b/types/src/tx.rs index 6dda43fc..c610c685 100644 --- a/types/src/tx.rs +++ b/types/src/tx.rs @@ -1,3 +1,4 @@ +use std::any::Any; use commonware_cryptography::sha256; use commonware_cryptography::sha256::Digest; @@ -60,6 +61,8 @@ pub trait Unit : UnitClone + Send + Sync + std::fmt::Debug { context: &UnitContext, state: &mut Box, ) -> Result>, Box>; + + fn as_any(&self) -> &dyn Any; } impl Clone for Box { @@ -87,9 +90,9 @@ pub struct Tx { // id is the transaction id. It is the hash of digest. - id: Digest, + pub id: Digest, // digest is encoded tx. - digest: Vec, + pub digest: Vec, } pub trait TxMethods:Sized { @@ -214,7 +217,7 @@ impl TxMethods for Tx { if units.is_err() { return Err(format!("Failed to unpack units: {}", units.unwrap_err())); } - tx.units = units.unwrap(); + tx.units = units?; // generate tx id. tx.id = sha256::hash(&tx.digest); // return transaction. @@ -264,7 +267,7 @@ fn unpack_units(digest: &[u8]) -> Result>, String> { if unit_type.is_err() { return Err(format!("Invalid unit type: {}", unit_type.unwrap_err())); } - let unit_type = unit_type.unwrap(); + let unit_type = unit_type?; let unit:Box = match unit_type { UnitType::Transfer => { let mut transfer = units::transfer::Transfer::default(); @@ -284,4 +287,63 @@ fn unpack_units(digest: &[u8]) -> Result>, String> { } -// @todo implement tests for encoding and decoding of tx. \ No newline at end of file +// @todo implement tests for encoding and decoding of tx. +#[cfg(test)] +mod tests { + use std::error::Error; + use more_asserts::assert_gt; + use crate::curr_timestamp; + use crate::units::transfer::Transfer; + use super::*; + + #[test] + fn test_encode_decode() -> Result<(), Box> { + let timestamp = curr_timestamp(); + let max_fee = 100; + let priority_fee = 75; + let chain_id = 45205; + let transfer = Transfer::new(); + let units : Vec> = vec![Box::new(transfer)]; + let mut digest: [u8; 32] = [0; 32]; + let id = Digest::from(digest.clone()); + // TODO: the .encode call on next line gave error and said origin_msg needed to be mut? but why? + // shouldn't encode be able to encode without changing the msg? + let mut origin_msg = Tx { + timestamp, + max_fee, + priority_fee, + chain_id, + units: units.clone(), + id, + digest: digest.to_vec(), + }; + let encoded_bytes = origin_msg.encode(); + assert_gt!(encoded_bytes.len(), 0); + let mut decoded_msg = Tx::decode(&encoded_bytes)?; + let origin_transfer = origin_msg.units[0] + .as_ref() + .as_any() + .downcast_ref::() + .expect("Failed to downcast to Transfer"); + + let decode_transfer = decoded_msg.units[0] + .as_ref() + .as_any() + .downcast_ref::() + .expect("Failed to downcast to Transfer"); + + assert_eq!(origin_msg.timestamp, decoded_msg.timestamp); + assert_eq!(origin_msg.max_fee, decoded_msg.max_fee); + assert_eq!(origin_msg.priority_fee, decoded_msg.priority_fee); + assert_eq!(origin_msg.chain_id, decoded_msg.chain_id); + assert_eq!(origin_msg.id, decoded_msg.id); + assert_eq!(origin_msg.digest, decoded_msg.digest); + + // units + assert_eq!(origin_transfer.to_address, decode_transfer.to_address); + assert_eq!(origin_transfer.from_address, decode_transfer.from_address); + assert_eq!(origin_transfer.value, decode_transfer.value); + assert_eq!(origin_transfer.memo, decode_transfer.memo); + Ok(()) + } +} \ No newline at end of file diff --git a/types/src/units/msg.rs b/types/src/units/msg.rs index 3023c73a..84dc7a34 100644 --- a/types/src/units/msg.rs +++ b/types/src/units/msg.rs @@ -1,3 +1,4 @@ +use std::any::Any; use crate::{address::Address, tx::{Unit, UnitType, UnitContext}, state::State}; #[derive(Clone, Debug)] @@ -8,6 +9,17 @@ pub struct SequencerMsg { pub relayer_id: u64, } +impl SequencerMsg { + pub fn new() -> SequencerMsg { + Self { + chain_id: 0, + data: Vec::new(), + from_address: Address::empty(), + relayer_id: 0, + } + } +} + impl Unit for SequencerMsg { fn unit_type(&self) -> UnitType { UnitType::SequencerMsg @@ -47,6 +59,10 @@ impl Unit for SequencerMsg { ) -> Result>, Box> { todo!() } + + fn as_any(&self) -> &dyn Any { + self + } } impl Default for SequencerMsg { @@ -58,4 +74,35 @@ impl Default for SequencerMsg { relayer_id: 0, } } +} + +#[cfg(test)] +mod tests { + use std::error::Error; + use more_asserts::assert_gt; + use super::*; + + #[test] + fn test_encode_decode() -> Result<(), Box> { + let chain_id = 4502; + let data = vec!(0xDE, 0xAD, 0xBE, 0xEF); + let from_address = Address::create_random_address(); + let relayer_id = 1; + let origin_msg = SequencerMsg { + chain_id, + data, + from_address, + relayer_id, + }; + let encoded_bytes = origin_msg.encode(); + assert_gt!(encoded_bytes.len(), 0); + let mut decoded_msg = SequencerMsg::new(); + decoded_msg.decode(&encoded_bytes); + assert_eq!(origin_msg.chain_id, decoded_msg.chain_id); + assert_eq!(origin_msg.data.len(), decoded_msg.data.len()); + assert_eq!(origin_msg.data, decoded_msg.data); + assert_eq!(origin_msg.from_address, decoded_msg.from_address); + assert_eq!(origin_msg.relayer_id, decoded_msg.relayer_id); + Ok(()) + } } \ No newline at end of file diff --git a/types/src/units/transfer.rs b/types/src/units/transfer.rs index fe99ef9e..d00e27d6 100644 --- a/types/src/units/transfer.rs +++ b/types/src/units/transfer.rs @@ -1,3 +1,4 @@ +use std::any::Any; use crate::address::Address; use crate::state::State; use crate::tx::{Unit, UnitType, UnitContext}; @@ -12,6 +13,17 @@ pub struct Transfer { pub memo: Vec, } +impl Transfer { + pub fn new() -> Transfer { + Self { + from_address: Address::empty(), + to_address: Address::empty(), + value: 0, + memo: Vec::new(), + } + } +} + #[derive(Debug)] pub enum TransferError { DuplicateAddress, @@ -60,6 +72,10 @@ impl Unit for Transfer { ) -> Result>, Box> { todo!() } + + fn as_any(&self) -> &dyn Any { + self + } } impl Default for Transfer { @@ -71,4 +87,36 @@ impl Default for Transfer { memo: vec![], } } +} + + +#[cfg(test)] +mod tests { + use std::error::Error; + use more_asserts::assert_gt; + use super::*; + + #[test] + fn test_encode_decode() -> Result<(), Box> { + let from_address = Address::create_random_address(); + let to_address = Address::create_random_address(); + let value = 5; + let memo = vec!(0xDE, 0xAD, 0xBE, 0xEF); + let relayer_id = 1; + let origin_msg = Transfer { + from_address, + to_address, + value, + memo, + }; + let encoded_bytes = origin_msg.encode(); + assert_gt!(encoded_bytes.len(), 0); + let mut decoded_msg = Transfer::new(); + decoded_msg.decode(&encoded_bytes); + assert_eq!(origin_msg.from_address, decoded_msg.from_address); + assert_eq!(origin_msg.to_address, decoded_msg.to_address); + assert_eq!(origin_msg.value, decoded_msg.value); + assert_eq!(origin_msg.memo, decoded_msg.memo); + Ok(()) + } } \ No newline at end of file From 1e28cf68f5358352ad8e9c2e1b40fa82d4f95c1f Mon Sep 17 00:00:00 2001 From: manojkgorle Date: Wed, 2 Apr 2025 11:04:51 +0530 Subject: [PATCH 29/43] some progress on transactional_db --- chain/src/actors/syncer/actor.rs | 2 +- storage/src/database.rs | 2 +- storage/src/state_db.rs | 5 +- storage/src/transactional_db.rs | 410 +++++++++++++++++++++---------- vm/src/vm.rs | 108 -------- 5 files changed, 287 insertions(+), 240 deletions(-) diff --git a/chain/src/actors/syncer/actor.rs b/chain/src/actors/syncer/actor.rs index 0b2d85f5..897b05a3 100644 --- a/chain/src/actors/syncer/actor.rs +++ b/chain/src/actors/syncer/actor.rs @@ -830,7 +830,7 @@ impl, I: Index debug!(height, "received finalization"); let _ = response.send(true); - // Persist the finalization + // Persist the finalization @todo finalized .put(height, finalization.block.digest(), finalization.proof.serialize().into()) .await diff --git a/storage/src/database.rs b/storage/src/database.rs index f1708685..ee73b320 100644 --- a/storage/src/database.rs +++ b/storage/src/database.rs @@ -1,7 +1,7 @@ use std::error::Error; // Define database interface that will be used for all impls -pub trait Database { +pub trait Database{ fn put(&mut self, key: &[u8], value: &[u8]) -> Result<(), Box>; fn get(&mut self, key: &[u8]) -> Result>, Box>; diff --git a/storage/src/state_db.rs b/storage/src/state_db.rs index 4c4056f9..d17e4c4c 100644 --- a/storage/src/state_db.rs +++ b/storage/src/state_db.rs @@ -9,10 +9,13 @@ use std::error::Error; const ACCOUNTS_PREFIX: u8 = 0x0; const DB_WRITE_BUFFER_CAPACITY: usize = 500; +// @todo state db will be a wrapper around a db implementing trait TransactionalDb. +// state db interface will be provided for every transaction. +// db implementing TransactionalDb will be changed for every block and has an ability to rollback over reverted transactions. pub struct StateDb { db: Box, } -// can use like a redis from Arcadia like get and set for diff types? + impl StateDb { pub fn new(db: Box) -> StateDb { StateDb { db } diff --git a/storage/src/transactional_db.rs b/storage/src/transactional_db.rs index fc302a19..e67ec525 100644 --- a/storage/src/transactional_db.rs +++ b/storage/src/transactional_db.rs @@ -1,183 +1,335 @@ use std::collections::HashMap; use std::error::Error; -use crate::database::Database; -const ACCOUNT_KEY_TYPE: u8 = 0; +use crate::database::Database; +use std::sync::Arc; type Key = [u8; 33]; +// @todo rewrite InMemoryCachingTransactionalDb, to implement rollback and tracking state changes properly. +// carrying forward the cached transactions among others. + +// i. should track every operation, that a tx does. +// ii. should be able to rollback if a tx reverts. +// iii. should be able to rollback if a block forks. +// iv. should be able to commit to all the state changes once a block has been accepted. + +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum OpAction { + // key is read + Read, + // key is created + Create, + // key is updated + Update, + // key got deleted + Delete, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Op{ + pub action: OpAction, + pub value: Vec, +} -// pub enum OpAction { -// Read, // key was read -// Create, // key was created -// Update, // key was updated -// Delete, // key got deleted -// } - -// pub struct Op<'a>{ -// pub action: crate::tx_state_view::OpAction, -// pub key: Key<'a>, -// pub value: Vec, -// } - -pub trait TransactionalDb : Database { - fn init_cache(&mut self, cache: HashMap>); // initialize the cache with an already available hashmap of key-value pairs. - fn get_from_cache(&self, key: &Key) -> Result>, Box>; // get a key from the cache. If the key is not in the cache, it will return an error. - fn get_from_db(&mut self, key: &Key) -> Result>, Box>; // get a key from the underlying storage. If the key is not in the storage, it will return an error. +/// Implements finalization to database out of TransactionalDb trait. +pub trait TransactionalDb<'a> { + // initialize the cache with an already available hashmap of key-value pairs. + fn init_cache(&mut self, cache: & 'a mut HashMap); + // create a next instance. + // this is called only after calling commit_last_tx. + // else, some of the key value pair changes get lost. + // fn next_instance(&mut self) -> Self; + /// get the value corresponding to the key. + /// use this method for querying state. + fn get(&mut self, key: &Key) -> Result>, Box>; + // insert a key-pair. could be a create or delete action. + // underlying struct should handle the OpAction part. + fn insert(&mut self, key: &Key, value: Vec) -> Result<(), Box>; + // delete a key-pair + fn delete(&mut self, key: &Key) -> Result<(), Box>; + // get a key from cache. do not call this method directly, instead call get. + fn get_from_cache(&self, key: &Key) -> Result>, Box>; + // get a key from the underlying storage. do not call this method directly, instead call get. If the key is not in the storage, it will return an error. + fn get_from_db(&mut self, key: &Key) -> Result>, Box>; + // commit last tx changes within the cache + fn commit_last_tx(&mut self) -> Result<(), Box>; + // commit changes to the unfinalized map. fn commit(&mut self) -> Result<(), Box>; + // rollback last tx changes within the cache. + fn rollback_last_tx(&mut self) -> Result<(), Box>; + // rollback entirely. fn rollback(&mut self) -> Result<(), Box>; } -pub struct InMemoryCachingTransactionalDb { - pub cache: HashMap>, // key-value, state view cache before tx execution. This is not an exhaustive list of all state keys read/write during tx. If cache misses occur, the state view will read from the underlying storage. - // pub ops: Vec>, // list of state ops applied. - pub touched: HashMap>, // key-value pairs that were changed during tx execution. This is a subset of the cache. - pub db: Box, // underlying state storage, to use when cache misses occur. +pub struct InMemoryCachingTransactionalDb<'a> { + // cache is init'ed at the start. + pub cache: & 'a mut HashMap, + // unfinalized changes from previous block(s). + pub unfinalized: & 'a mut HashMap, + // set of all key value changes from last init. + pub touched: HashMap, + // set of all key value changes from last commit_last_tx + pub touched_tx: HashMap, + // underlying database. + pub db: Arc>, } -impl InMemoryCachingTransactionalDb { - pub fn new(db: Box) -> InMemoryCachingTransactionalDb { + +impl<'a> InMemoryCachingTransactionalDb<'a> { + pub fn new(cache: & 'a mut HashMap, unfinalized: & 'a mut HashMap, db: Arc>) -> InMemoryCachingTransactionalDb<'a> { Self{ - cache: HashMap::new(), + cache: cache, + unfinalized: unfinalized, touched: HashMap::new(), + touched_tx: HashMap::new(), db, } } - fn get_from_touched(&self, key: &Key) -> Result>, Box> { - self.touched.get(key).map_or( - Ok(None), - |v| Ok(Some(v.clone()))) - } } -impl TransactionalDb for InMemoryCachingTransactionalDb { - fn init_cache(&mut self, cache: HashMap>) { +impl<'a> TransactionalDb<'a> for InMemoryCachingTransactionalDb<'a> { + fn init_cache(&mut self, cache: & 'a mut HashMap) { self.cache = cache; } - // get a key from the cache. If the key is not in the cache, it will return an error. - fn get_from_cache(&self, key: &Key) -> Result>, Box> { - // searches touched - if let Some(v) = self.get_from_touched(key)? { - return Ok(Some(v.clone())); - } - // searches cache - match self.cache.get(key) { - Some(cached_value) => { - Ok(Some(cached_value.clone())) + fn get(&mut self, key: &Key) -> Result>, Box> { + match self.touched_tx.get(key) { + // the key is used in the current transaction. + Some(op) => { + // No need to update the op.action as create, insert or delete are superior to read. + Ok(Some(op.value.clone())) + } + // key is not used in the current transaction. + None => { + match self.touched.get(key) { + // key is used in the current block. + Some(t_op) => { + let v = Op { + action: OpAction::Read, + value:t_op.value.clone() + }; + self.touched_tx.insert(*key, v); + Ok(Some(t_op.value.clone())) + } + // key is not used in the current block. + None => { + match self.unfinalized.get(key) { + // the key is used in the previous block(s). but the blocks did nt finalze yet. + Some(u_op) => { + let v = Op { + action: OpAction::Read, + value:u_op.value.clone() + }; + self.touched_tx.insert(*key, v); + return Ok(Some(u_op.value.clone())); + } + // the key is not used in the previous block(s). + None => { + // check if the key is in the cache. + // if it is, return the value. + // if it is not, check if the key is in the underlying db. + // if it is, return the value. + // if it is not, return an error. + if let Some(f_c) = self.get_from_cache(key) + .ok() + .flatten() + .or_else(|| self.db.lock().ok()?.get(key).ok().flatten()) { + let v = Op { + action: OpAction::Read, + value: f_c.clone() + }; + self.touched_tx.insert(*key, v); + return Ok(Some(f_c)); + } else { + return Err("Key does not exist.".into()); + } + } + } + } + } } - // not found in either cache or db - None => Ok(None), } } - // get a key from the underlying storage. If the key is not in the storage, it will return an error. + fn insert(&mut self, key: &Key, value: Vec) -> Result<(), Box> { + let op = Op { + action: OpAction::Update, // @todo change this to OpAction::Update or OpAction::Create? based on if the key-pair actually exists. + value: value, + }; + self.touched_tx.insert(*key, op); + Ok(()) + } + + fn delete(&mut self, key: &Key) -> Result<(), Box> { + let op = Op{ + action: OpAction::Delete, + value: vec![], + }; + self.touched_tx.insert(*key, op); + Ok(()) + } + + fn get_from_cache(&self, key: &Key) -> Result>, Box> { + self.cache.get(key) + .map(|op| Some(op.value.clone())) + .ok_or_else(|| "Key not found in cache.".into()) + } + fn get_from_db(&mut self, key: &Key) -> Result>, Box> { - self.db.get(key) + self.db.lock().unwrap().get(key) + .map(|v| Some(v.clone()))? + .ok_or_else(|| "Key not found in db.".into()) + } + + fn commit_last_tx(&mut self) -> Result<(), Box> { + merge_maps(&mut self.touched, &self.touched_tx); + self.touched_tx.clear(); + Ok(()) } fn commit(&mut self) -> Result<(), Box> { - for (key, value) in self.touched.iter() { - self.cache.insert(*key, value.clone()); - //TODO: what to do if an intermediary operation fails maybe use rocks db transact? - // ex: what if we go through half of touched and it fails halfway? rare but possible. - self.db.put(&key[..], value)?; - } + merge_maps(&mut self.unfinalized, &self.touched); self.touched.clear(); Ok(()) } - fn rollback(&mut self) -> Result<(), Box> { - self.touched.clear(); + fn rollback_last_tx(&mut self) -> Result<(), Box> { + self.touched_tx.clear(); Ok(()) } -} -impl Database for InMemoryCachingTransactionalDb { - fn put(&mut self, key: &[u8], value: &[u8]) -> Result<(), Box> { - self.touched.insert(Key::try_from(key).unwrap(), value.to_vec()); + fn rollback(&mut self) -> Result<(), Box> { + self.touched.clear(); Ok(()) } +} - fn get(&mut self, key: &[u8]) -> Result>, Box> { - match self.get_from_cache(&Key::try_from(key).unwrap()) { - Ok(Some(value)) => { - Ok(Some(value)) - }, - Ok(None) => { - match self.db.get(key) { - Ok(Some(value)) => { - self.cache.insert(Key::try_from(key).unwrap(), value.clone()); - Ok(Some(value)) - } - Ok(None) => { - Ok(None) - }, - Err(e) => { - Err(e) - } - } - }, - Err(e) => { - Err(e) +pub fn merge_maps<'a, 'b>(map1: & 'a mut HashMap, map2: & 'b HashMap) -> & 'a mut HashMap { + for (key, op) in map2.iter() { + if let Some(existing_op) = map1.get(key) { + // there is a key existing in both maps. + if op.action == OpAction::Delete { + map1.insert(*key, op.clone()); + + }else if op.action == OpAction::Update { + // if op.action is update, then update the map1. as update superseeds everything. + map1.insert(*key, op.clone()); + }else if op.action == OpAction::Read && (existing_op.action == OpAction::Update || existing_op.action == OpAction::Create) { + // reading on a delete will return a nill value with existing op set to delete. this should not be an issue. + let new_op = Op { + action: existing_op.action, + value: op.value.clone(), + }; + map1.insert(*key, new_op.clone()); } + } else { + // there is a key not existing in the first map. + map1.insert(*key, op.clone()); } } - - // TODO: The below deletes in both cache and underlying db. Change later such that deletes must be committed. - fn delete(&mut self, key: &[u8]) -> Result<(), Box> { - self.touched.remove(key); - self.cache.remove(key); - self.db.delete(key) - } + map1 } #[cfg(test)] mod tests { use crate::hashmap_db::HashmapDatabase; use super::*; - + use std::sync::Mutex; #[test] - fn test_transactional_db_basic() { - let hash_db = HashmapDatabase::new(); // underlying store - - let test_key1: [u8; 33] = [0; 33]; - let test_value1 = b"test_value1".to_vec(); - - let test_key2: [u8; 33] = [1; 33]; - let test_value2 = b"test_value2".to_vec(); - - let test_key3: [u8; 33] = [2; 33]; - let test_value3 = b"test_value3".to_vec(); + fn test_it_works() { + let mut cache: HashMap = HashMap::new(); + let mut unfinalized: HashMap = HashMap::new(); + let db = Arc::new(Mutex::new(HashmapDatabase::new())); + let key1 = [1; 33]; + let value1 = [1; 33]; + let key2 = [2; 33]; + let value2 = [2; 33]; + db.lock().unwrap().put(&key1, &value1).unwrap(); + { + let mut in_mem_db = InMemoryCachingTransactionalDb::new(&mut cache, &mut unfinalized, db.clone()); + // start a tx + assert_eq!(in_mem_db.get(&key1).unwrap(), Some(value1.to_vec())); + // end a tx + assert_eq!(in_mem_db.touched_tx.len(), 1); + assert_eq!(in_mem_db.touched.len(),0); + assert_eq!(in_mem_db.cache.len(), 0); + assert_eq!(in_mem_db.touched_tx.get(&key1).unwrap(), &Op{ + action: OpAction::Read, + value: value1.to_vec(), + }); + // commit the tx + let _ = in_mem_db.commit_last_tx(); + assert_eq!(in_mem_db.touched_tx.len(), 0); + assert_eq!(in_mem_db.touched.len(),1); + assert_eq!(in_mem_db.cache.len(), 0); + assert_eq!(in_mem_db.touched.get(&key1).unwrap(), &Op{ + action: OpAction::Read, + value: value1.to_vec(), + }); + // start a new tx + in_mem_db.insert(&key2, value2.to_vec()).unwrap(); + assert_eq!(in_mem_db.touched_tx.len(), 1); + assert_eq!(in_mem_db.touched.len(), 1); + assert_eq!(in_mem_db.cache.len(), 0); + assert_eq!(in_mem_db.touched_tx.get(&key2).unwrap(), &Op{ + action: OpAction::Update, + value: value2.to_vec(), + }); + assert_eq!(in_mem_db.touched.get(&key1).unwrap(), &Op{ + action: OpAction::Read, + value: value1.to_vec(), + }); + // end tx + assert_eq!(in_mem_db.get(&key2).unwrap(), Some(value2.to_vec())); + assert_eq!(in_mem_db.touched_tx.len(), 1); + assert_eq!(in_mem_db.touched.len(), 1); + assert_eq!(in_mem_db.cache.len(), 0); + // commit tx + let _ = in_mem_db.commit_last_tx().unwrap(); + assert_eq!(in_mem_db.touched_tx.len(), 0); + assert_eq!(in_mem_db.touched.len(), 2); + assert_eq!(in_mem_db.cache.len(), 0); + // commit block + let _ = in_mem_db.commit().unwrap(); + assert_eq!(in_mem_db.touched_tx.len(), 0); + assert_eq!(in_mem_db.touched.len(), 0); + assert_eq!(in_mem_db.cache.len(), 0); + assert_eq!(in_mem_db.unfinalized.len(), 2); + } + // implement finalize out of db trait. + assert_eq!(unfinalized.len(), 2); + assert_eq!(cache.len(), 0); + for (key, op) in unfinalized.iter() { + if op.action == OpAction::Delete { + db.lock().unwrap().delete(key).unwrap(); + } else if op.action == OpAction::Update { + db.lock().unwrap().put(key, &op.value).unwrap(); + } + } + merge_maps(&mut cache, &unfinalized); + assert_eq!(cache.len(), 2); + unfinalized.clear(); + assert_eq!(unfinalized.len(), 0); + // try fetching key2 from db. this should pass. + assert_eq!(db.lock().unwrap().get(&key2).unwrap(), Some(value2.to_vec())); + // try fetching key1 from db. this should pass. + assert_eq!(db.lock().unwrap().get(&key1).unwrap(), Some(value1.to_vec())); + // new block { - let mut tx_db = InMemoryCachingTransactionalDb::new(Box::new(hash_db)); - - // should add to cache but not underlying store db - tx_db.put(&test_key1, &test_value1).unwrap(); - assert!(tx_db.get_from_db(&test_key1).unwrap().is_none()); - assert_eq!(tx_db.get_from_cache(&test_key1).unwrap().is_some(), true); - assert_eq!(tx_db.get_from_cache(&test_key1).unwrap().unwrap(), test_value1); - assert_eq!(tx_db.get(&test_key1).unwrap().unwrap(), test_value1); - - // commit the change. should be visible in underlying store. - tx_db.commit().unwrap(); - assert!(tx_db.get_from_db(&test_key1).unwrap().is_some()); - assert_eq!(tx_db.get_from_db(&test_key1).unwrap().unwrap(), test_value1); - assert_eq!(tx_db.get_from_cache(&test_key1).unwrap().unwrap(), test_value1); - assert_eq!(tx_db.get(&test_key1).unwrap().unwrap(), test_value1); - - // add 2nd key-value. do not commit yet. - tx_db.put(&test_key2, &test_value2).unwrap(); - assert!(tx_db.get_from_db(&test_key2).unwrap().is_none()); - assert_eq!(tx_db.get_from_cache(&test_key2).unwrap().is_some(), true); - assert_eq!(tx_db.get_from_cache(&test_key2).unwrap().unwrap(), test_value2); - assert_eq!(tx_db.get(&test_key2).unwrap().unwrap(), test_value2); - - // rollback and commit should make no changes to underlying. - tx_db.rollback().unwrap(); - tx_db.commit().unwrap(); - assert!(tx_db.get_from_db(&test_key2).unwrap().is_none()); - assert!(tx_db.get_from_cache(&test_key2).unwrap().is_none()); - assert!(tx_db.get(&test_key2).unwrap().is_none()); + let mut in_mem_db = InMemoryCachingTransactionalDb::new(&mut cache, &mut unfinalized, db.clone()); + assert_eq!(db.lock().unwrap().get(&key1).unwrap(), Some(value1.to_vec())); + assert_eq!(in_mem_db.get(&key1).unwrap(), Some(value1.to_vec())); + assert_eq!(in_mem_db.get(&key2).unwrap(), Some(value2.to_vec())); + // delete key1 + in_mem_db.delete(&key1).unwrap(); + assert_eq!(in_mem_db.touched_tx.len(), 2); + assert_eq!(in_mem_db.touched.len(), 0); + assert_eq!(in_mem_db.cache.len(), 2); + assert_eq!(in_mem_db.unfinalized.len(), 0); + assert_eq!(in_mem_db.touched_tx.get(&key1).unwrap(), &Op{ + action: OpAction::Delete, + value: vec![], + }); } } } diff --git a/vm/src/vm.rs b/vm/src/vm.rs index c9ae06ac..d9894b49 100644 --- a/vm/src/vm.rs +++ b/vm/src/vm.rs @@ -1,122 +1,14 @@ -use alto_actions::transfer::{Transfer, TransferError}; -use alto_storage::database::Database; -use alto_storage::hashmap_db::HashmapDatabase; use alto_storage::transactional_db::TransactionalDb; -use alto_storage::state_db::StateDb; -use alto_types::account::Balance; -use alto_types::address::Address; -const TEST_FAUCET_ADDRESS: &[u8] = b"0x0123456789abcdef0123456789abcd"; -const TEST_FAUCET_BALANCE: Balance = 10_000_000; - -// TODO: figure out vm file struct VM { state_db: Box, } impl VM { - pub fn new() -> Self { - Self { - state_db: Box::new(StateDb::new(Box::new(()))), - } - } - - pub fn new_test_vm() -> Self { - let mut state_db = Box::new(HashmapDatabase::new()); - state_db.set_balance(&Self::test_faucet_address(), TEST_FAUCET_BALANCE); - - Self { - state_db - } - } - - // TODO: - // make check for sending funds > 0 so reject any negative values or balances. - // make sure address isn't a duplicate of self. - // need from balance to be greater than or equal to the transfer amount - // need the to_balance to not overflow - pub fn execute(&mut self, msg: Transfer) -> Result<(), TransferError> { - let from_balance = self.state_db.get_balance(&msg.from_address); - if from_balance.is_none() { - return Err(TransferError::InvalidFromAddress); - } - - - let mut to_balance = self.state_db.get_balance(&msg.to_address); - if to_balance.is_none() { - to_balance = Some(0) - } - - let updated_from_balance = from_balance.unwrap().checked_sub(msg.value); - if updated_from_balance.is_none() { - return Err(TransferError::InsufficientFunds); - } - - let updated_to_balance = to_balance.unwrap().checked_add(msg.value); - if updated_to_balance.is_none() { - return Err(TransferError::TooMuchFunds); - } - - // TODO: Below doesn't rollback cases. Fix later. - if !self.state_db.set_balance(&msg.from_address, updated_from_balance.unwrap()) { - return Err(TransferError::StorageError); - } - if !self.state_db.set_balance(&msg.to_address, updated_to_balance.unwrap()) { - return Err(TransferError::StorageError); - } - - Ok(()) - } - - pub fn query_balance(&self, address: &Address) -> Option { - self.state_db.get_balance(address) - } - - pub fn test_faucet_address() -> Address { - Address::new(TEST_FAUCET_ADDRESS) - } } #[cfg(test)] mod tests { use super::*; - #[test] - fn test_basic() -> Result<(), String> { - let from_address = Address::new(b"0x10000"); - let to_address = Address::new(b"0x20000"); - let faucet_address = VM::test_faucet_address(); - - let mut test_vm = VM::new_test_vm(); - - // should be empty - let from_balance1 = test_vm.query_balance(&from_address); - assert!(from_balance1.is_none()); - - let faucet_balance = test_vm.query_balance(&VM::test_faucet_address()); - assert_eq!(faucet_balance, Some(TEST_FAUCET_BALANCE)); - - // process faucet transfers to addresses - let transfer1 = Transfer::new(faucet_address.clone(), from_address.clone(), 100); - test_vm.execute(transfer1.unwrap()).unwrap(); - let transfer2 = Transfer::new(faucet_address.clone(), to_address.clone(), 200); - test_vm.execute(transfer2.unwrap()).unwrap(); - - // check balances have been updated from faucet transfers - let from_balance2 = test_vm.query_balance(&from_address); - assert_eq!(from_balance2, Some(100)); - let from_balance3 = test_vm.query_balance(&to_address); - assert_eq!(from_balance3, Some(200)); - - let transfer3 = Transfer::new(from_address.clone(), to_address.clone(), 50); - test_vm.execute(transfer3.unwrap()).unwrap(); - - // check updated balances - let from_balance2 = test_vm.query_balance(&from_address); - assert_eq!(from_balance2, Some(50)); - let from_balance3 = test_vm.query_balance(&to_address); - assert_eq!(from_balance3, Some(250)); - - Ok(()) - } } From 47592d8c4a0edd01f593e73ece8b6ae9cebb85a8 Mon Sep 17 00:00:00 2001 From: manojkgorle Date: Wed, 2 Apr 2025 12:34:31 +0530 Subject: [PATCH 30/43] Aha, these lifetimes are a keeper --- storage/src/state_db.rs | 114 +++++++++++--------------------- storage/src/transactional_db.rs | 47 ++++++------- types/src/account.rs | 2 + 3 files changed, 61 insertions(+), 102 deletions(-) diff --git a/storage/src/state_db.rs b/storage/src/state_db.rs index d17e4c4c..1d3d3498 100644 --- a/storage/src/state_db.rs +++ b/storage/src/state_db.rs @@ -1,5 +1,4 @@ -use crate::database::Database; -use crate::transactional_db::TransactionalDb; +use crate::transactional_db::{InMemoryCachingTransactionalDb, TransactionalDb}; use alto_types::account::{Account, Balance}; use alto_types::address::Address; use bytes::Bytes; @@ -9,38 +8,36 @@ use std::error::Error; const ACCOUNTS_PREFIX: u8 = 0x0; const DB_WRITE_BUFFER_CAPACITY: usize = 500; -// @todo state db will be a wrapper around a db implementing trait TransactionalDb. -// state db interface will be provided for every transaction. -// db implementing TransactionalDb will be changed for every block and has an ability to rollback over reverted transactions. -pub struct StateDb { - db: Box, +// StateDb is a wrapper around TransactionalDb that provides StateViews for block execution. +// StateDb simplifies the interactions with state by providing methods that abstract away the underlying database operations. +// It allows for easy retrieval and modification of account states, such as balances. +pub struct StateDb<'a> { + db: &'a mut dyn TransactionalDb<'a>, } -impl StateDb { - pub fn new(db: Box) -> StateDb { +impl<'a> StateDb<'a> { + pub fn new(db: &'a mut dyn TransactionalDb<'a>) -> Self { StateDb { db } } pub fn get_account(&mut self, address: &Address) -> Result, Box> { let key = Self::key_accounts(address); - // try cache first - if let Some(value) = self.db.get_from_cache(&key)? - .or_else(|| self.db.get(&key).ok().flatten()) // falls to DB if cache miss - { - let bytes = Bytes::from(value); - let mut read_buf = ReadBuffer::new(bytes); - let acc = Account::read(&mut read_buf)?; - return Ok(Some(acc)); - } - Ok(None) + self.db.get(&key).and_then(|v| { + if let Some(value) = v { + let bytes = Bytes::from(value); + let mut read_buf = ReadBuffer::new(bytes); + Account::read(&mut read_buf).map(Some).map_err(|e| Box::new(e) as Box) + } else { + Err("Account not found".into()) + } + }) } pub fn set_account(&mut self, acc: &Account) -> Result<(), Box> { let key = Self::key_accounts(&acc.address); let mut write_buf = WriteBuffer::new(DB_WRITE_BUFFER_CAPACITY); acc.write(&mut write_buf); - self.db.put(&key, write_buf.as_ref())?; - Ok(()) + self.db.insert(&key, write_buf.as_ref().to_vec()) } pub fn get_balance(&mut self, address: &Address) -> Option { @@ -74,68 +71,31 @@ impl StateDb { } } -impl Database for StateDb { - fn put(&mut self, key: &[u8], value: &[u8]) -> Result<(), Box> { - self.db.put(key, value) - } - - fn get(&mut self, key: &[u8]) -> Result>, Box> { - self.db.get(key) - } - - fn delete(&mut self, key: &[u8]) -> Result<(), Box> { - self.db.delete(key) - } -} - #[cfg(test)] mod tests { use alto_types::address::Address; - use alto_types::account::Account; - use crate::rocks_db::RocksDbDatabase; - use crate::transactional_db::InMemoryCachingTransactionalDb; - + use crate::hashmap_db::HashmapDatabase; + use crate::transactional_db::{InMemoryCachingTransactionalDb, Op, Key}; use super::*; + use std::collections::HashMap; + use std::sync::{Arc, Mutex}; - fn setup_state_db() -> StateDb { - let db = InMemoryCachingTransactionalDb::new(Box::new( - RocksDbDatabase::new_tmp_db().expect("db could not be created"), - )); - StateDb::new(Box::new(db)) - } #[test] fn test_statedb_accounts() { - let mut state_db = setup_state_db(); - - let mut account = Account::new(); - let test_address_bytes = [0u8; 32]; - let test_address = Address::new(&test_address_bytes); - account.address = test_address.clone(); - account.balance = 100; - - // make sure account does not exist (base case) - assert!(state_db.get_account(&test_address).unwrap().is_none()); - - // create account and check retrieval - state_db.set_account(&account).unwrap(); - let retrieved_account = state_db.get_account(&test_address).unwrap().expect("Account not found"); - assert_eq!(retrieved_account.address, test_address); - assert_eq!(retrieved_account.balance, 100); - - // update account balance and check retrieval - assert!(state_db.set_balance(&test_address, 200)); - assert_eq!(state_db.get_balance(&test_address), Some(200)); - - // test if updating balance is persistent - assert!(state_db.set_balance(&test_address, 300)); - let updated_account = state_db.get_account(&test_address).unwrap().expect("Account not found"); - assert_eq!(updated_account.balance, 300); - - // test retrieval of balance directly - assert_eq!(state_db.get_balance(&test_address), Some(300)); - - // check a non-existent account returns None - let non_existent_address = Address::new(b"0xDEAD"); - assert!(state_db.get_account(&non_existent_address).unwrap().is_none()); + // setup state db + let mut cache: HashMap = HashMap::new(); + let mut unfinalized: HashMap = HashMap::new(); + let db = Arc::new(Mutex::new(HashmapDatabase::new())); + + let mut in_mem = InMemoryCachingTransactionalDb::new(&mut cache, &mut unfinalized, db); + let address = Address::create_random_address(); + { + let mut state_db = StateDb::new(&mut in_mem); + // use state_db + let _ = state_db.get_account(&address); // sample call + } // <- state_db dropped here + + // ✅ Continue using `in_mem` freely + // let _ = in_mem.commit(); } } \ No newline at end of file diff --git a/storage/src/transactional_db.rs b/storage/src/transactional_db.rs index e67ec525..f5b08864 100644 --- a/storage/src/transactional_db.rs +++ b/storage/src/transactional_db.rs @@ -4,9 +4,7 @@ use std::error::Error; use crate::database::Database; use std::sync::Arc; -type Key = [u8; 33]; -// @todo rewrite InMemoryCachingTransactionalDb, to implement rollback and tracking state changes properly. -// carrying forward the cached transactions among others. +pub type Key = [u8; 33]; // i. should track every operation, that a tx does. // ii. should be able to rollback if a tx reverts. @@ -25,57 +23,56 @@ pub enum OpAction { Delete, } +/// Op contains action performed and value stored over a key. #[derive(Clone, Debug, PartialEq, Eq)] pub struct Op{ + // Action performed pub action: OpAction, + // Resulting value after perfromed action. pub value: Vec, } /// Implements finalization to database out of TransactionalDb trait. pub trait TransactionalDb<'a> { - // initialize the cache with an already available hashmap of key-value pairs. + /// initialize the cache with an already available hashmap of key-value pairs. fn init_cache(&mut self, cache: & 'a mut HashMap); - // create a next instance. - // this is called only after calling commit_last_tx. - // else, some of the key value pair changes get lost. - // fn next_instance(&mut self) -> Self; /// get the value corresponding to the key. /// use this method for querying state. fn get(&mut self, key: &Key) -> Result>, Box>; - // insert a key-pair. could be a create or delete action. - // underlying struct should handle the OpAction part. + /// insert a key-pair. could be a create or delete action. + /// underlying struct should handle the OpAction part. fn insert(&mut self, key: &Key, value: Vec) -> Result<(), Box>; - // delete a key-pair + /// delete a key-pair fn delete(&mut self, key: &Key) -> Result<(), Box>; - // get a key from cache. do not call this method directly, instead call get. + /// get a key from cache. do not call this method directly, instead call get. fn get_from_cache(&self, key: &Key) -> Result>, Box>; - // get a key from the underlying storage. do not call this method directly, instead call get. If the key is not in the storage, it will return an error. + /// get a key from the underlying storage. do not call this method directly, instead call get. If the key is not in the storage, it will return an error. fn get_from_db(&mut self, key: &Key) -> Result>, Box>; - // commit last tx changes within the cache + /// commit last tx changes within the cache fn commit_last_tx(&mut self) -> Result<(), Box>; - // commit changes to the unfinalized map. + /// commit changes to the unfinalized map. fn commit(&mut self) -> Result<(), Box>; - // rollback last tx changes within the cache. + /// rollback last tx changes within the cache. fn rollback_last_tx(&mut self) -> Result<(), Box>; - // rollback entirely. + /// rollback entirely. fn rollback(&mut self) -> Result<(), Box>; } pub struct InMemoryCachingTransactionalDb<'a> { - // cache is init'ed at the start. - pub cache: & 'a mut HashMap, - // unfinalized changes from previous block(s). - pub unfinalized: & 'a mut HashMap, - // set of all key value changes from last init. + /// cache is init'ed at the start. + pub cache: &'a mut HashMap, + /// unfinalized changes from previous block(s). + pub unfinalized: &'a mut HashMap, + /// set of all key value changes from last init. pub touched: HashMap, - // set of all key value changes from last commit_last_tx + /// set of all key value changes from last commit_last_tx pub touched_tx: HashMap, - // underlying database. + /// underlying database. pub db: Arc>, } impl<'a> InMemoryCachingTransactionalDb<'a> { - pub fn new(cache: & 'a mut HashMap, unfinalized: & 'a mut HashMap, db: Arc>) -> InMemoryCachingTransactionalDb<'a> { + pub fn new(cache: &'a mut HashMap, unfinalized: &'a mut HashMap, db: Arc>) -> Self { Self{ cache: cache, unfinalized: unfinalized, diff --git a/types/src/account.rs b/types/src/account.rs index 955980e2..3ec4da60 100644 --- a/types/src/account.rs +++ b/types/src/account.rs @@ -33,6 +33,8 @@ impl Account { impl Codec for Account { fn write(&self, writer: &mut impl Writer) { + // @rikoeldon I think we don't need to write account address into the state. + // account address is part of the key. writer.write_bytes(self.address.0.as_slice()); self.balance.write(writer); } From eaf1214d281b94a818ebb6ea29cfd21b9baeb250 Mon Sep 17 00:00:00 2001 From: manojkgorle Date: Wed, 2 Apr 2025 13:36:16 +0530 Subject: [PATCH 31/43] some progress --- Cargo.lock | 1 + chain/Cargo.toml | 1 + chain/src/bin/setup.rs | 1 + chain/src/bin/validator.rs | 8 +++ chain/src/engine.rs | 21 +++++- chain/src/lib.rs | 21 +++++- storage/src/lib.rs | 3 +- storage/src/transactional_db.rs | 1 - storage/src/tx_state_view.rs | 110 -------------------------------- vm/src/vm.rs | 2 +- 10 files changed, 51 insertions(+), 118 deletions(-) delete mode 100644 storage/src/tx_state_view.rs diff --git a/Cargo.lock b/Cargo.lock index a078abaa..8e89a203 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -41,6 +41,7 @@ name = "alto-chain" version = "0.0.4" dependencies = [ "alto-client", + "alto-storage", "alto-types", "axum", "bytes", diff --git a/chain/Cargo.toml b/chain/Cargo.toml index e59b1946..61a6b98b 100644 --- a/chain/Cargo.toml +++ b/chain/Cargo.toml @@ -13,6 +13,7 @@ documentation = "https://docs.rs/alto-chain" [dependencies] alto-types = { workspace = true } alto-client = { workspace = true } +alto-storage = { workspace = true} commonware-consensus = { workspace = true } commonware-cryptography = { workspace = true } commonware-deployer = { workspace = true } diff --git a/chain/src/bin/setup.rs b/chain/src/bin/setup.rs index a68f4629..4218d620 100644 --- a/chain/src/bin/setup.rs +++ b/chain/src/bin/setup.rs @@ -244,6 +244,7 @@ fn generate(sub_matches: &ArgMatches) { message_backlog, mailbox_size, + state_db_directory: "/home/ubuntu/alto/state".to_string(), indexer: None, }; peer_configs.push((peer_config_file.clone(), peer_config)); diff --git a/chain/src/bin/validator.rs b/chain/src/bin/validator.rs index 8ed27842..ca4120ae 100644 --- a/chain/src/bin/validator.rs +++ b/chain/src/bin/validator.rs @@ -1,6 +1,7 @@ use alto_chain::{engine, Config}; use alto_client::Client; use alto_types::P2P_NAMESPACE; +use alto_storage::rocks_db::RocksDbDatabase; use axum::{routing::get, serve, Extension, Router}; use clap::{Arg, Command}; use commonware_cryptography::{ @@ -29,6 +30,7 @@ use std::{ }; use sysinfo::{Disks, System}; use tracing::{error, info, Level}; +use std::sync::{Arc, Mutex}; const SYSTEM_METRICS_REFRESH: Duration = Duration::from_secs(5); const METRICS_PORT: u16 = 9090; @@ -185,6 +187,11 @@ fn main() { indexer = Some(Client::new(&uri, identity_public.into())); } + // create state db: rocks db for now. + let state_db = RocksDbDatabase::new_with_path(&config.state_db_directory) + .expect("Could not create state db"); + let wrapped_state_db = Arc::new(Mutex::new(state_db)); + // Create engine let config = engine::Config { partition_prefix: "engine".to_string(), @@ -204,6 +211,7 @@ fn main() { fetch_concurrent: FETCH_CONCURRENT, fetch_rate_per_peer: resolver_limit, indexer, + state_db: wrapped_state_db, }; let engine = engine::Engine::new(context.with_label("engine"), config).await; diff --git a/chain/src/engine.rs b/chain/src/engine.rs index 8536fbd6..96d53393 100644 --- a/chain/src/engine.rs +++ b/chain/src/engine.rs @@ -18,6 +18,9 @@ use governor::clock::Clock as GClock; use governor::Quota; use rand::{CryptoRng, Rng}; use std::time::Duration; +use std::sync::{Arc, Mutex}; +use std::collections::HashMap; +use alto_storage::{transactional_db::{Key, Op}, database::Database}; use tracing::{error, warn}; pub struct Config { @@ -40,12 +43,14 @@ pub struct Config { pub fetch_rate_per_peer: Quota, pub indexer: Option, + + pub state_db: Arc>, } pub struct Engine< B: Blob, E: Clock + GClock + Rng + CryptoRng + Spawner + Storage + Metrics, - I: Indexer, + I: Indexer, > { context: E, @@ -62,12 +67,22 @@ pub struct Engine< application::Mailbox, application::Supervisor, >, + + // state + state_cache: Arc>>, + unfinalized_state: Arc>>, + state_db: Arc>, } impl + Metrics, I: Indexer> Engine { pub async fn new(context: E, cfg: Config) -> Self { + // @todo initalizing state cache and unfinalized state. + // if it is necessary pass state_cache, unfinalized_state and state_db to both application and syncer. + let state_cache:Arc>> = Arc::new(Mutex::new(HashMap::new())); + let unfinalized_state:Arc>> = Arc::new(Mutex::new(HashMap::new())); + // Create the application let public = public(&cfg.identity); let (application, supervisor, application_mailbox) = application::Actor::new( @@ -129,7 +144,6 @@ impl + Metri fetch_rate_per_peer: cfg.fetch_rate_per_peer, }, ); - // Return the engine Self { context, @@ -138,6 +152,9 @@ impl + Metri syncer, syncer_mailbox, consensus, + state_cache, + unfinalized_state, + state_db: cfg.state_db, } } diff --git a/chain/src/lib.rs b/chain/src/lib.rs index dd73bb76..a097e40e 100644 --- a/chain/src/lib.rs +++ b/chain/src/lib.rs @@ -75,12 +75,15 @@ pub struct Config { pub mailbox_size: usize, pub indexer: Option, + + pub state_db_directory: String, } #[cfg(test)] mod tests { use super::*; use alto_types::{Finalized, Notarized, Seed}; + use alto_storage::rocks_db::RocksDbDatabase; use bls12381::primitives::poly; use commonware_cryptography::{bls12381::dkg::ops, ed25519::PublicKey, Ed25519, Scheme}; use commonware_macros::test_traced; @@ -93,6 +96,7 @@ mod tests { use engine::{Config, Engine}; use governor::Quota; use rand::{rngs::StdRng, Rng, SeedableRng}; + use tracing_subscriber::fmt::format; use std::{ collections::{HashMap, HashSet}, num::NonZeroU32, @@ -263,7 +267,8 @@ mod tests { // Create scheme context let public_key = scheme.public_key(); public_keys.insert(public_key.clone()); - + let state_db = RocksDbDatabase::new_with_path(format!("/home/ubuntu/state_db/{}", idx).as_str()).expect("Could not create state db"); + let wrapped_state_db = Arc::new(Mutex::new(state_db)); // Configure engine let uid = format!("validator-{}", public_key); let config: Config = engine::Config { @@ -284,6 +289,7 @@ mod tests { fetch_concurrent: 10, fetch_rate_per_peer: Quota::per_second(NonZeroU32::new(10).unwrap()), indexer: None, + state_db: wrapped_state_db, }; let engine = Engine::new(context.with_label(&uid), config).await; @@ -424,6 +430,8 @@ mod tests { // Configure engine let public_key = scheme.public_key(); let uid = format!("validator-{}", public_key); + let state_db = RocksDbDatabase::new_with_path(format!("/home/ubuntu/state_db/{}", idx).as_str()).expect("Could not create state db"); + let wrapped_state_db = Arc::new(Mutex::new(state_db)); let config: Config = engine::Config { partition_prefix: uid.clone(), signer: scheme.clone(), @@ -442,6 +450,7 @@ mod tests { fetch_concurrent: 10, fetch_rate_per_peer: Quota::per_second(NonZeroU32::new(10).unwrap()), indexer: None, + state_db: wrapped_state_db, }; let engine = Engine::new(context.with_label(&uid), config).await; @@ -507,6 +516,8 @@ mod tests { let share = shares[0]; let public_key = scheme.public_key(); let uid = format!("validator-{}", public_key); + let state_db = RocksDbDatabase::new_with_path(format!("/home/ubuntu/state_db/{}", uid).as_str()).expect("Could not create state db"); + let wrapped_state_db = Arc::new(Mutex::new(state_db)); let config: Config = engine::Config { partition_prefix: uid.clone(), signer: scheme.clone(), @@ -525,6 +536,7 @@ mod tests { fetch_concurrent: 10, fetch_rate_per_peer: Quota::per_second(NonZeroU32::new(10).unwrap()), indexer: None, + state_db: wrapped_state_db, }; let engine = Engine::new(context.with_label(&uid), config).await; @@ -637,7 +649,8 @@ mod tests { // Create scheme context let public_key = scheme.public_key(); public_keys.insert(public_key.clone()); - + let state_db = RocksDbDatabase::new_with_path(format!("/home/ubuntu/state_db/{}", idx).as_str()).expect("Could not create state db"); + let wrapped_state_db = Arc::new(Mutex::new(state_db)); // Configure engine let uid = format!("validator-{}", public_key); let config: Config = engine::Config { @@ -658,6 +671,7 @@ mod tests { fetch_concurrent: 10, fetch_rate_per_peer: Quota::per_second(NonZeroU32::new(10).unwrap()), indexer: None, + state_db: wrapped_state_db, }; let engine = Engine::new(context.with_label(&uid), config).await; @@ -783,6 +797,8 @@ mod tests { // Configure engine let uid = format!("validator-{}", public_key); + let state_db = RocksDbDatabase::new_with_path(format!("/home/ubuntu/state_db/{}", idx).as_str()).expect("Could not create state db"); + let wrapped_state_db = Arc::new(Mutex::new(state_db)); let config: Config = engine::Config { partition_prefix: uid.clone(), signer: scheme, @@ -801,6 +817,7 @@ mod tests { fetch_concurrent: 10, fetch_rate_per_peer: Quota::per_second(NonZeroU32::new(10).unwrap()), indexer: Some(indexer.clone()), + state_db: wrapped_state_db, }; let engine = Engine::new(context.with_label(&uid), config).await; diff --git a/storage/src/lib.rs b/storage/src/lib.rs index a4802311..14018bda 100644 --- a/storage/src/lib.rs +++ b/storage/src/lib.rs @@ -1,6 +1,5 @@ pub mod database; pub mod hashmap_db; -mod rocks_db; -mod tx_state_view; +pub mod rocks_db; pub mod state_db; pub mod transactional_db; \ No newline at end of file diff --git a/storage/src/transactional_db.rs b/storage/src/transactional_db.rs index f5b08864..387ceee2 100644 --- a/storage/src/transactional_db.rs +++ b/storage/src/transactional_db.rs @@ -10,7 +10,6 @@ pub type Key = [u8; 33]; // ii. should be able to rollback if a tx reverts. // iii. should be able to rollback if a block forks. // iv. should be able to commit to all the state changes once a block has been accepted. - #[derive(Clone, Copy, PartialEq, Eq, Debug)] pub enum OpAction { // key is read diff --git a/storage/src/tx_state_view.rs b/storage/src/tx_state_view.rs deleted file mode 100644 index 907c637f..00000000 --- a/storage/src/tx_state_view.rs +++ /dev/null @@ -1,110 +0,0 @@ -//use crate::state_db::StateDb; - -//const ACCOUNT_KEY_TYPE: u8 = 0; - -//type UnitKey<'a> = alto_types::state::UnitKey<'a>; // 1st byte denotes the type of key. 0b for account key, 1b for others. - -// pub fn decode_unit_key(key: UnitKey) -> (u8, Address) { -// let key_type: u8 = key[0]; -// let address_bytes: &[u8] = &key[1..]; -// (key_type, address_bytes.into()) -// } - -// pub enum OpAction { -// Read, // key was read -// Create, // key was created -// Update, // key was updated -// Delete, // key got deleted -// } -// -// pub struct Op<'a>{ -// pub action: OpAction, -// pub key: UnitKey<'a>, -// pub value: Vec, -// } -// -// pub trait TxStateViewTrait: State { -// fn init_cache(&mut self, cache: HashMap>); // initialize the cache with an already available hashmap of key-value pairs. -// fn get_from_cache(&self, key: UnitKey) -> Result>, Box>; // get a key from the cache. If the key is not in the cache, it will return an error. -// fn get_from_state(&self, key: UnitKey) -> Result>, Box>; // get a key from the underlying storage. If the key is not in the storage, it will return an error. -// } -// -// pub struct TxStateView<'a> { -// pub cache: HashMap, Vec>, // key-value, state view cache before tx execution. This is not an exhaustive list of all state keys read/write during tx. If cache misses occur, the state view will read from the underlying storage. -// pub ops: Vec>, // list of state ops applied. -// pub touched: HashMap, Vec>, // key-value pairs that were changed during tx execution. This is a subset of the cache. -// pub state_db: StateDb, // underlying state storage, to use when cache misses occur. -// } -// -// impl<'a> TxStateView<'a> { -// pub fn new(state_db: StateDb) -> Self { -// Self{ -// cache: HashMap::new(), -// ops: Vec::new(), -// touched: HashMap::new(), -// state_db, -// } -// } -// // initialize the cache with an already available hashmap of key-value pairs. -// pub fn init_cache(&mut self, cache: HashMap>) { -// self.cache = cache; -// } -// -// pub fn get_from_cache(&self, key: UnitKey) -> Result>, Box> { -// self.cache.get(&key).map_or( -// Ok(None), -// |v| Ok(Some(v.clone().into()))) -// } -// -// pub fn get_from_state(&self, key: UnitKey) -> Result>, Box> { -// /* -// let (key_type, address) -// match key[0] { -// ACCOUNT_KEY_TYPE => { -// self.get_from_state(key) -// }, -// _ => Err(format!("invalid state key {:?}", key[0]).into()) -// } -// -// */ -// todo!() -// } -// -// pub fn get(&self, key: UnitKey) -> Result>, Box>{ -// todo!() -// } -// -// pub fn get_multi_key(&self, key: UnitKey) -> Result>, Box> { -// todo!() -// } -// pub fn update(&mut self, key: UnitKey, value: Vec) -> Result<(), Box>{ -// todo!() -// } -// pub fn delete(&mut self, key: UnitKey) -> Result<(), Box> { -// todo!() -// } -// pub fn commit(&mut self) -> Result<(), Box> { -// todo!() -// } -// pub fn rollback(&mut self) -> Result<(), Box> { -// todo!() -// } -// -// pub fn process_get_action(&mut self, cmd_type: u8, key: &UnitKey) -> Result>, Box> { -// match cmd_type { -// ACCOUNT_KEY_TYPE => { -// self.state_db.get(key) -// } -// _ => Err(format!("invalid state key {:?}", key).into()) -// } -// } -// -// pub fn process_put_action(&mut self, cmd_type: u8, key: &UnitKey) -> Result>, Box> { -// match cmd_type { -// ACCOUNT_KEY_TYPE => { -// self.state_db.get(key) -// } -// _ => Err(format!("invalid state key {:?}", key).into()) -// } -// } -// } \ No newline at end of file diff --git a/vm/src/vm.rs b/vm/src/vm.rs index d9894b49..508c3148 100644 --- a/vm/src/vm.rs +++ b/vm/src/vm.rs @@ -1,7 +1,7 @@ use alto_storage::transactional_db::TransactionalDb; struct VM { - state_db: Box, + // state_db: Box, } impl VM { From d63043c322329bb33d027a9d2c70f7f6809c5085 Mon Sep 17 00:00:00 2001 From: manojkgorle Date: Wed, 2 Apr 2025 15:16:56 +0530 Subject: [PATCH 32/43] switch to mutex --- chain/src/actors/application/actor.rs | 11 ++- chain/src/actors/application/mod.rs | 7 ++ chain/src/actors/syncer/mod.rs | 10 ++ chain/src/engine.rs | 6 ++ storage/src/transactional_db.rs | 127 ++++++++++++++++++-------- types/src/units/msg.rs | 12 +-- vm/src/vm.rs | 39 +++++++- 7 files changed, 161 insertions(+), 51 deletions(-) diff --git a/chain/src/actors/application/actor.rs b/chain/src/actors/application/actor.rs index 058a295d..d5906db2 100644 --- a/chain/src/actors/application/actor.rs +++ b/chain/src/actors/application/actor.rs @@ -5,6 +5,7 @@ use super::{ }; use crate::actors::syncer; use alto_types::{Block, Finalization, Notarization, Seed}; +use alto_storage::{database::Database, transactional_db::{Key, Op}}; use commonware_consensus::threshold_simplex::Prover; use commonware_cryptography::{sha256::Digest, Hasher, Sha256}; use commonware_macros::select; @@ -19,8 +20,7 @@ use futures::{ }; use rand::Rng; use std::{ - pin::Pin, - sync::{Arc, Mutex}, + collections::HashMap, pin::Pin, sync::{Arc, Mutex} }; use tracing::{info, warn}; @@ -58,6 +58,10 @@ pub struct Actor { prover: Prover, hasher: Sha256, mailbox: mpsc::Receiver, + state_cache: Arc>>, + unfinalized_state: Arc>>, + staete_db: Arc>, + } impl Actor { @@ -70,6 +74,9 @@ impl Actor { prover: config.prover, hasher: Sha256::new(), mailbox, + state_cache: config.state_cache, + unfinalized_state: config.unfinalized_state, + staete_db: config.state_db, }, Supervisor::new(config.identity, config.participants, config.share), Mailbox::new(sender), diff --git a/chain/src/actors/application/mod.rs b/chain/src/actors/application/mod.rs index 72bb076a..d852928c 100644 --- a/chain/src/actors/application/mod.rs +++ b/chain/src/actors/application/mod.rs @@ -4,6 +4,8 @@ use commonware_cryptography::{ ed25519::PublicKey, sha256::Digest, }; +use std::{collections::HashMap, sync::{Arc, Mutex}}; +use alto_storage::{database::Database, transactional_db::{Key, Op}}; mod actor; pub use actor::Actor; @@ -29,4 +31,9 @@ pub struct Config { /// Number of messages from consensus to hold in our backlog /// before blocking. pub mailbox_size: usize, + + /// State + pub state_cache: Arc>>, + pub unfinalized_state: Arc>>, + pub state_db: Arc>, } diff --git a/chain/src/actors/syncer/mod.rs b/chain/src/actors/syncer/mod.rs index 6e568253..53236310 100644 --- a/chain/src/actors/syncer/mod.rs +++ b/chain/src/actors/syncer/mod.rs @@ -1,5 +1,10 @@ use commonware_cryptography::{bls12381::primitives::group, ed25519::PublicKey}; use governor::Quota; +use std::{ + collections::HashMap, + sync::{Arc, Mutex}, +}; +use alto_storage::{database::Database, transactional_db::{Key, Op}}; mod actor; mod archive; @@ -33,4 +38,9 @@ pub struct Config { pub activity_timeout: u64, pub indexer: Option, + + // State + pub state_cache: Arc>>, + pub unfinalized_state: Arc>>, + pub state_db: Arc>, } diff --git a/chain/src/engine.rs b/chain/src/engine.rs index 96d53393..1ee2bada 100644 --- a/chain/src/engine.rs +++ b/chain/src/engine.rs @@ -93,6 +93,9 @@ impl + Metri identity: cfg.identity.clone(), share: cfg.share, mailbox_size: cfg.mailbox_size, + state_cache: Arc::clone(&state_cache), + unfinalized_state: Arc::clone(&unfinalized_state), + state_db: Arc::clone(&cfg.state_db), }, ); @@ -108,6 +111,9 @@ impl + Metri backfill_quota: cfg.backfill_quota, activity_timeout: cfg.activity_timeout, indexer: cfg.indexer, + state_cache: Arc::clone(&state_cache), + unfinalized_state: Arc::clone(&unfinalized_state), + state_db: Arc::clone(&cfg.state_db), }, ) .await; diff --git a/storage/src/transactional_db.rs b/storage/src/transactional_db.rs index 387ceee2..2e09b078 100644 --- a/storage/src/transactional_db.rs +++ b/storage/src/transactional_db.rs @@ -2,7 +2,7 @@ use std::collections::HashMap; use std::error::Error; use crate::database::Database; -use std::sync::Arc; +use std::sync::{Arc, Mutex}; pub type Key = [u8; 33]; @@ -32,9 +32,9 @@ pub struct Op{ } /// Implements finalization to database out of TransactionalDb trait. -pub trait TransactionalDb<'a> { +pub trait TransactionalDb { /// initialize the cache with an already available hashmap of key-value pairs. - fn init_cache(&mut self, cache: & 'a mut HashMap); + fn init_cache(&mut self, cache: Arc>>); /// get the value corresponding to the key. /// use this method for querying state. fn get(&mut self, key: &Key) -> Result>, Box>; @@ -57,21 +57,21 @@ pub trait TransactionalDb<'a> { fn rollback(&mut self) -> Result<(), Box>; } -pub struct InMemoryCachingTransactionalDb<'a> { +pub struct InMemoryCachingTransactionalDb { /// cache is init'ed at the start. - pub cache: &'a mut HashMap, + pub cache:Arc>>, /// unfinalized changes from previous block(s). - pub unfinalized: &'a mut HashMap, + pub unfinalized:Arc>>, /// set of all key value changes from last init. pub touched: HashMap, /// set of all key value changes from last commit_last_tx pub touched_tx: HashMap, /// underlying database. - pub db: Arc>, + pub db: Arc>, } -impl<'a> InMemoryCachingTransactionalDb<'a> { - pub fn new(cache: &'a mut HashMap, unfinalized: &'a mut HashMap, db: Arc>) -> Self { +impl InMemoryCachingTransactionalDb { + pub fn new(cache: Arc>>, unfinalized: Arc>>, db: Arc>) -> Self { Self{ cache: cache, unfinalized: unfinalized, @@ -82,8 +82,8 @@ impl<'a> InMemoryCachingTransactionalDb<'a> { } } -impl<'a> TransactionalDb<'a> for InMemoryCachingTransactionalDb<'a> { - fn init_cache(&mut self, cache: & 'a mut HashMap) { +impl TransactionalDb for InMemoryCachingTransactionalDb { + fn init_cache(&mut self, cache: Arc>>) { self.cache = cache; } @@ -108,7 +108,7 @@ impl<'a> TransactionalDb<'a> for InMemoryCachingTransactionalDb<'a> { } // key is not used in the current block. None => { - match self.unfinalized.get(key) { + match self.unfinalized.lock().unwrap().get(key) { // the key is used in the previous block(s). but the blocks did nt finalze yet. Some(u_op) => { let v = Op { @@ -165,7 +165,7 @@ impl<'a> TransactionalDb<'a> for InMemoryCachingTransactionalDb<'a> { } fn get_from_cache(&self, key: &Key) -> Result>, Box> { - self.cache.get(key) + self.cache.lock().unwrap().get(key) .map(|op| Some(op.value.clone())) .ok_or_else(|| "Key not found in cache.".into()) } @@ -177,13 +177,13 @@ impl<'a> TransactionalDb<'a> for InMemoryCachingTransactionalDb<'a> { } fn commit_last_tx(&mut self) -> Result<(), Box> { - merge_maps(&mut self.touched, &self.touched_tx); + merge_maps_nl(&mut self.touched, &self.touched_tx); self.touched_tx.clear(); Ok(()) } fn commit(&mut self) -> Result<(), Box> { - merge_maps(&mut self.unfinalized, &self.touched); + merge_maps(Arc::clone(&self.unfinalized), &mut self.touched); self.touched.clear(); Ok(()) } @@ -199,7 +199,62 @@ impl<'a> TransactionalDb<'a> for InMemoryCachingTransactionalDb<'a> { } } -pub fn merge_maps<'a, 'b>(map1: & 'a mut HashMap, map2: & 'b HashMap) -> & 'a mut HashMap { +pub fn merge_maps_l(map1: Arc>>, map2: Arc>>) -> Arc>> { + let mut map1_g = map1.lock().unwrap(); + let map2_g = map2.lock().unwrap(); + for (key, op) in map2_g.iter() { + if let Some(existing_op) = map1_g.get(key) { + // there is a key existing in both maps. + if op.action == OpAction::Delete { + map1_g.insert(*key, op.clone()); + + }else if op.action == OpAction::Update { + // if op.action is update, then update the map1. as update superseeds everything. + map1_g.insert(*key, op.clone()); + }else if op.action == OpAction::Read && (existing_op.action == OpAction::Update || existing_op.action == OpAction::Create) { + // reading on a delete will return a nill value with existing op set to delete. this should not be an issue. + let new_op = Op { + action: existing_op.action, + value: op.value.clone(), + }; + map1_g.insert(*key, new_op.clone()); + } + } else { + // there is a key not existing in the first map. + map1_g.insert(*key, op.clone()); + } + } + Arc::clone(&map1) +} + +pub fn merge_maps<'a>(map1: Arc>>, map2: &'a mut HashMap) -> Arc>> { + let mut map1_g = map1.lock().unwrap(); + for (key, op) in map2.iter() { + if let Some(existing_op) = map1_g.get(key) { + // there is a key existing in both maps. + if op.action == OpAction::Delete { + map1_g.insert(*key, op.clone()); + + }else if op.action == OpAction::Update { + // if op.action is update, then update the map1. as update superseeds everything. + map1_g.insert(*key, op.clone()); + }else if op.action == OpAction::Read && (existing_op.action == OpAction::Update || existing_op.action == OpAction::Create) { + // reading on a delete will return a nill value with existing op set to delete. this should not be an issue. + let new_op = Op { + action: existing_op.action, + value: op.value.clone(), + }; + map1_g.insert(*key, new_op.clone()); + } + } else { + // there is a key not existing in the first map. + map1_g.insert(*key, op.clone()); + } + } + Arc::clone(&map1) +} + +pub fn merge_maps_nl<'a, 'b>(map1: & 'a mut HashMap, map2: & 'b HashMap) -> & 'a mut HashMap { for (key, op) in map2.iter() { if let Some(existing_op) = map1.get(key) { // there is a key existing in both maps. @@ -232,8 +287,8 @@ mod tests { use std::sync::Mutex; #[test] fn test_it_works() { - let mut cache: HashMap = HashMap::new(); - let mut unfinalized: HashMap = HashMap::new(); + let mut cache: Arc>> = Arc::new(Mutex::new(HashMap::new())); + let mut unfinalized: Arc>> = Arc::new(Mutex::new(HashMap::new())); let db = Arc::new(Mutex::new(HashmapDatabase::new())); let key1 = [1; 33]; let value1 = [1; 33]; @@ -241,14 +296,14 @@ mod tests { let value2 = [2; 33]; db.lock().unwrap().put(&key1, &value1).unwrap(); { - let mut in_mem_db = InMemoryCachingTransactionalDb::new(&mut cache, &mut unfinalized, db.clone()); + let mut in_mem_db = InMemoryCachingTransactionalDb::new(Arc::clone(&cache), Arc::clone(&unfinalized), db.clone()); // start a tx assert_eq!(in_mem_db.get(&key1).unwrap(), Some(value1.to_vec())); // end a tx assert_eq!(in_mem_db.touched_tx.len(), 1); assert_eq!(in_mem_db.touched.len(),0); - assert_eq!(in_mem_db.cache.len(), 0); + assert_eq!(in_mem_db.cache.lock().unwrap().len(), 0); assert_eq!(in_mem_db.touched_tx.get(&key1).unwrap(), &Op{ action: OpAction::Read, value: value1.to_vec(), @@ -257,7 +312,7 @@ mod tests { let _ = in_mem_db.commit_last_tx(); assert_eq!(in_mem_db.touched_tx.len(), 0); assert_eq!(in_mem_db.touched.len(),1); - assert_eq!(in_mem_db.cache.len(), 0); + assert_eq!(in_mem_db.cache.lock().unwrap().len(), 0); assert_eq!(in_mem_db.touched.get(&key1).unwrap(), &Op{ action: OpAction::Read, value: value1.to_vec(), @@ -266,7 +321,7 @@ mod tests { in_mem_db.insert(&key2, value2.to_vec()).unwrap(); assert_eq!(in_mem_db.touched_tx.len(), 1); assert_eq!(in_mem_db.touched.len(), 1); - assert_eq!(in_mem_db.cache.len(), 0); + assert_eq!(in_mem_db.cache.lock().unwrap().len(), 0); assert_eq!(in_mem_db.touched_tx.get(&key2).unwrap(), &Op{ action: OpAction::Update, value: value2.to_vec(), @@ -279,40 +334,40 @@ mod tests { assert_eq!(in_mem_db.get(&key2).unwrap(), Some(value2.to_vec())); assert_eq!(in_mem_db.touched_tx.len(), 1); assert_eq!(in_mem_db.touched.len(), 1); - assert_eq!(in_mem_db.cache.len(), 0); + assert_eq!(in_mem_db.cache.lock().unwrap().len(), 0); // commit tx let _ = in_mem_db.commit_last_tx().unwrap(); assert_eq!(in_mem_db.touched_tx.len(), 0); assert_eq!(in_mem_db.touched.len(), 2); - assert_eq!(in_mem_db.cache.len(), 0); + assert_eq!(in_mem_db.cache.lock().unwrap().len(), 0); // commit block let _ = in_mem_db.commit().unwrap(); assert_eq!(in_mem_db.touched_tx.len(), 0); assert_eq!(in_mem_db.touched.len(), 0); - assert_eq!(in_mem_db.cache.len(), 0); - assert_eq!(in_mem_db.unfinalized.len(), 2); + assert_eq!(in_mem_db.cache.lock().unwrap().len(), 0); + assert_eq!(in_mem_db.unfinalized.lock().unwrap().len(), 2); } // implement finalize out of db trait. - assert_eq!(unfinalized.len(), 2); - assert_eq!(cache.len(), 0); - for (key, op) in unfinalized.iter() { + assert_eq!(unfinalized.lock().unwrap().len(), 2); + assert_eq!(cache.lock().unwrap().len(), 0); + for (key, op) in unfinalized.lock().unwrap().iter() { if op.action == OpAction::Delete { db.lock().unwrap().delete(key).unwrap(); } else if op.action == OpAction::Update { db.lock().unwrap().put(key, &op.value).unwrap(); } } - merge_maps(&mut cache, &unfinalized); - assert_eq!(cache.len(), 2); - unfinalized.clear(); - assert_eq!(unfinalized.len(), 0); + merge_maps_l(Arc::clone(&cache), Arc::clone(&unfinalized)); + assert_eq!(cache.lock().unwrap().len(), 2); + unfinalized.lock().unwrap().clear(); + assert_eq!(unfinalized.lock().unwrap().len(), 0); // try fetching key2 from db. this should pass. assert_eq!(db.lock().unwrap().get(&key2).unwrap(), Some(value2.to_vec())); // try fetching key1 from db. this should pass. assert_eq!(db.lock().unwrap().get(&key1).unwrap(), Some(value1.to_vec())); // new block { - let mut in_mem_db = InMemoryCachingTransactionalDb::new(&mut cache, &mut unfinalized, db.clone()); + let mut in_mem_db = InMemoryCachingTransactionalDb::new(Arc::clone(&cache), Arc::clone(&unfinalized), db.clone()); assert_eq!(db.lock().unwrap().get(&key1).unwrap(), Some(value1.to_vec())); assert_eq!(in_mem_db.get(&key1).unwrap(), Some(value1.to_vec())); assert_eq!(in_mem_db.get(&key2).unwrap(), Some(value2.to_vec())); @@ -320,8 +375,8 @@ mod tests { in_mem_db.delete(&key1).unwrap(); assert_eq!(in_mem_db.touched_tx.len(), 2); assert_eq!(in_mem_db.touched.len(), 0); - assert_eq!(in_mem_db.cache.len(), 2); - assert_eq!(in_mem_db.unfinalized.len(), 0); + assert_eq!(in_mem_db.cache.lock().unwrap().len(), 2); + assert_eq!(in_mem_db.unfinalized.lock().unwrap().len(), 0); assert_eq!(in_mem_db.touched_tx.get(&key1).unwrap(), &Op{ action: OpAction::Delete, value: vec![], diff --git a/types/src/units/msg.rs b/types/src/units/msg.rs index 84dc7a34..63db0d77 100644 --- a/types/src/units/msg.rs +++ b/types/src/units/msg.rs @@ -6,7 +6,6 @@ pub struct SequencerMsg { pub chain_id: u64, pub data: Vec, pub from_address: Address, - pub relayer_id: u64, } impl SequencerMsg { @@ -15,7 +14,6 @@ impl SequencerMsg { chain_id: 0, data: Vec::new(), from_address: Address::empty(), - relayer_id: 0, } } } @@ -33,8 +31,6 @@ impl Unit for SequencerMsg { bytes.extend(&self.chain_id.to_be_bytes()); // address length is 32. store address. bytes.extend_from_slice(self.from_address.as_slice()); - // relayer id length is 8 bytes. store relayer id. - bytes.extend(self.relayer_id.to_be_bytes()); // store data length. bytes.extend(data_len.to_be_bytes()); // store data. @@ -47,9 +43,8 @@ impl Unit for SequencerMsg { fn decode(&mut self, bytes: &[u8]) { self.chain_id = u64::from_be_bytes(bytes[0..8].try_into().unwrap()); self.from_address = Address::from_bytes(&bytes[8..40]).unwrap(); - self.relayer_id = u64::from_be_bytes(bytes[40..48].try_into().unwrap()); - let data_len = u64::from_be_bytes(bytes[48..56].try_into().unwrap()); - self.data = bytes[56..(56 + data_len as usize)].to_vec(); + let data_len = u64::from_be_bytes(bytes[40..48].try_into().unwrap()); + self.data = bytes[48..(48 + data_len as usize)].to_vec(); } fn apply( @@ -71,7 +66,6 @@ impl Default for SequencerMsg { chain_id: 0, data: vec![], from_address: Address::empty(), - relayer_id: 0, } } } @@ -92,7 +86,6 @@ mod tests { chain_id, data, from_address, - relayer_id, }; let encoded_bytes = origin_msg.encode(); assert_gt!(encoded_bytes.len(), 0); @@ -102,7 +95,6 @@ mod tests { assert_eq!(origin_msg.data.len(), decoded_msg.data.len()); assert_eq!(origin_msg.data, decoded_msg.data); assert_eq!(origin_msg.from_address, decoded_msg.from_address); - assert_eq!(origin_msg.relayer_id, decoded_msg.relayer_id); Ok(()) } } \ No newline at end of file diff --git a/vm/src/vm.rs b/vm/src/vm.rs index 508c3148..7695ccbd 100644 --- a/vm/src/vm.rs +++ b/vm/src/vm.rs @@ -1,10 +1,43 @@ -use alto_storage::transactional_db::TransactionalDb; +use std::error::Error; +use std::sync::{Arc, Mutex}; +use std::collections::HashMap; -struct VM { - // state_db: Box, +use alto_storage::transactional_db::InMemoryCachingTransactionalDb; +use alto_storage::{transactional_db::{TransactionalDb, Key, Op}, database::Database}; +use alto_types::tx::Tx; + +pub struct VM { + pub block_number: u64, + pub state_cache: Arc>>, + pub unfinalized_state: Arc>>, + pub state_db: Arc>, } impl VM { + pub fn new( + block_number: u64, + state_cache: Arc>>, + unfinalized_state: Arc>>, + state_db: Arc>, + ) -> Self { + Self { + block_number, + state_cache, + unfinalized_state, + state_db, + } + } + + // applies new set of txs on the given state. + pub fn apply(&mut self, txs: Vec) -> Result<(), Box> { + // let mut in_mem_db = InMemoryCachingTransactionalDb::new(Arc::clone(self.state_cache), Arc::clone(self.unfinalized_state), self.state_db); + Ok(()) + } + + // applies a single tx on the given state. + fn apply_tx(&mut self, tx: Vec) -> Result<(), Box> { + Ok(()) + } } #[cfg(test)] From f91641cec9024ab2dcfcb768a48444895896e851 Mon Sep 17 00:00:00 2001 From: manojkgorle Date: Wed, 2 Apr 2025 19:17:00 +0530 Subject: [PATCH 33/43] database --- storage/src/state_db.rs | 47 ++++++++++++++++++++++----------- storage/src/transactional_db.rs | 4 +-- types/src/account.rs | 2 +- 3 files changed, 35 insertions(+), 18 deletions(-) diff --git a/storage/src/state_db.rs b/storage/src/state_db.rs index 1d3d3498..143667f7 100644 --- a/storage/src/state_db.rs +++ b/storage/src/state_db.rs @@ -12,11 +12,11 @@ const DB_WRITE_BUFFER_CAPACITY: usize = 500; // StateDb simplifies the interactions with state by providing methods that abstract away the underlying database operations. // It allows for easy retrieval and modification of account states, such as balances. pub struct StateDb<'a> { - db: &'a mut dyn TransactionalDb<'a>, + db: &'a mut dyn TransactionalDb, } impl<'a> StateDb<'a> { - pub fn new(db: &'a mut dyn TransactionalDb<'a>) -> Self { + pub fn new(db: &'a mut dyn TransactionalDb) -> Self { StateDb { db } } @@ -81,21 +81,38 @@ mod tests { use std::sync::{Arc, Mutex}; #[test] - fn test_statedb_accounts() { + fn test_it_works() { // setup state db - let mut cache: HashMap = HashMap::new(); - let mut unfinalized: HashMap = HashMap::new(); + let cache: Arc>>= Arc::new(Mutex::new(HashMap::new())); + let unfinalized: Arc>>= Arc::new(Mutex::new(HashMap::new())); let db = Arc::new(Mutex::new(HashmapDatabase::new())); - - let mut in_mem = InMemoryCachingTransactionalDb::new(&mut cache, &mut unfinalized, db); + let mut in_mem = InMemoryCachingTransactionalDb::new(Arc::clone(&cache),Arc::clone(&unfinalized), db); + + let address = Address::create_random_address(); + let account = Account{address: address.clone(), balance: 1000}; + let mut state_db = StateDb::new(&mut in_mem); + let _ = state_db.set_account(&account).unwrap(); + let _ = in_mem.commit_last_tx(); + let _ = in_mem.commit(); + assert_eq!(unfinalized.lock().unwrap().len(), 1); + let mut state_db2 = StateDb::new(&mut in_mem); + let retrieved = state_db2.get_account(&address).unwrap().unwrap(); + assert_eq!(retrieved, account); + } + + #[test] + #[should_panic] + fn test_no_account_earlier() { + // setup state db + let cache: Arc>>= Arc::new(Mutex::new(HashMap::new())); + let unfinalized: Arc>>= Arc::new(Mutex::new(HashMap::new())); + let db = Arc::new(Mutex::new(HashmapDatabase::new())); + let mut in_mem = InMemoryCachingTransactionalDb::new(Arc::clone(&cache),Arc::clone(&unfinalized), db); + let address = Address::create_random_address(); - { - let mut state_db = StateDb::new(&mut in_mem); - // use state_db - let _ = state_db.get_account(&address); // sample call - } // <- state_db dropped here - - // ✅ Continue using `in_mem` freely - // let _ = in_mem.commit(); + + let mut state_db = StateDb::new(&mut in_mem); + + let _ = state_db.get_account(&address).unwrap(); } } \ No newline at end of file diff --git a/storage/src/transactional_db.rs b/storage/src/transactional_db.rs index 2e09b078..0d108814 100644 --- a/storage/src/transactional_db.rs +++ b/storage/src/transactional_db.rs @@ -287,8 +287,8 @@ mod tests { use std::sync::Mutex; #[test] fn test_it_works() { - let mut cache: Arc>> = Arc::new(Mutex::new(HashMap::new())); - let mut unfinalized: Arc>> = Arc::new(Mutex::new(HashMap::new())); + let cache: Arc>> = Arc::new(Mutex::new(HashMap::new())); + let unfinalized: Arc>> = Arc::new(Mutex::new(HashMap::new())); let db = Arc::new(Mutex::new(HashmapDatabase::new())); let key1 = [1; 33]; let value1 = [1; 33]; diff --git a/types/src/account.rs b/types/src/account.rs index 3ec4da60..e947d67d 100644 --- a/types/src/account.rs +++ b/types/src/account.rs @@ -3,7 +3,7 @@ use crate::address::Address; pub type Balance = u64; -#[derive(Clone, Debug)] +#[derive(Clone, Debug, PartialEq, Eq)] pub struct Account { pub address: Address, pub balance: Balance, From c3b740a4c97e308808d194e69e5a0e5fd827d254 Mon Sep 17 00:00:00 2001 From: manojkgorle Date: Wed, 2 Apr 2025 19:22:15 +0530 Subject: [PATCH 34/43] cargo fmt --- chain/src/actors/application/actor.rs | 20 ++- chain/src/actors/application/mod.rs | 14 +- chain/src/actors/application/router.rs | 32 ++-- chain/src/actors/syncer/mod.rs | 5 +- chain/src/bin/validator.rs | 6 +- chain/src/engine.rs | 17 +- chain/src/lib.rs | 28 +++- storage/src/database.rs | 4 +- storage/src/hashmap_db.rs | 9 +- storage/src/lib.rs | 2 +- storage/src/rocks_db.rs | 5 +- storage/src/state_db.rs | 41 +++-- storage/src/transactional_db.rs | 220 ++++++++++++++++--------- types/src/account.rs | 8 +- types/src/address.rs | 10 +- types/src/block.rs | 26 ++- types/src/lib.rs | 18 +- types/src/signed_tx.rs | 47 +++--- types/src/state.rs | 4 +- types/src/tx.rs | 58 ++++--- types/src/units/mod.rs | 2 +- types/src/units/msg.rs | 16 +- types/src/units/transfer.rs | 15 +- types/src/wallet.rs | 42 +++-- vm/src/lib.rs | 2 +- vm/src/vm.rs | 13 +- 26 files changed, 402 insertions(+), 262 deletions(-) diff --git a/chain/src/actors/application/actor.rs b/chain/src/actors/application/actor.rs index d5906db2..18f9313a 100644 --- a/chain/src/actors/application/actor.rs +++ b/chain/src/actors/application/actor.rs @@ -4,8 +4,11 @@ use super::{ Config, }; use crate::actors::syncer; +use alto_storage::{ + database::Database, + transactional_db::{Key, Op}, +}; use alto_types::{Block, Finalization, Notarization, Seed}; -use alto_storage::{database::Database, transactional_db::{Key, Op}}; use commonware_consensus::threshold_simplex::Prover; use commonware_cryptography::{sha256::Digest, Hasher, Sha256}; use commonware_macros::select; @@ -20,7 +23,9 @@ use futures::{ }; use rand::Rng; use std::{ - collections::HashMap, pin::Pin, sync::{Arc, Mutex} + collections::HashMap, + pin::Pin, + sync::{Arc, Mutex}, }; use tracing::{info, warn}; @@ -61,7 +66,6 @@ pub struct Actor { state_cache: Arc>>, unfinalized_state: Arc>>, staete_db: Arc>, - } impl Actor { @@ -92,16 +96,16 @@ impl Actor { // Compute genesis digest self.hasher.update(GENESIS); let genesis_parent = self.hasher.finalize(); - let genesis_state_root = [0u8;32]; + let genesis_state_root = [0u8; 32]; let genesis = Block::new(genesis_parent, 0, 0, Vec::new(), genesis_state_root.into()); let genesis_digest = genesis.digest(); - // --> prepared the genesis digest. + // --> prepared the genesis digest. // there are no blocks built, while genesis. let built: Option = None; let built = Arc::new(Mutex::new(built)); // @todo initiate fee manager here. - // @todo get the state view. + // @todo get the state view. // @todo init the database. // @todo commit to database. while let Some(message) = self.mailbox.next().await { @@ -112,7 +116,7 @@ impl Actor { // payload. let _ = response.send(genesis_digest.clone()); } - // its this validators turn to propose the block. + // its this validators turn to propose the block. // So, it should check for available blocks and propose a block. Message::Propose { view, @@ -221,7 +225,7 @@ impl Actor { let _ = response.send(false); return; } - //@todo unmarshall txs, execute txs, generate state root, collect fees, build state root, verify state root. + //@todo unmarshall txs, execute txs, generate state root, collect fees, build state root, verify state root. // Persist the verified block syncer.verified(view, block).await; diff --git a/chain/src/actors/application/mod.rs b/chain/src/actors/application/mod.rs index d852928c..916dafd5 100644 --- a/chain/src/actors/application/mod.rs +++ b/chain/src/actors/application/mod.rs @@ -1,18 +1,24 @@ +use alto_storage::{ + database::Database, + transactional_db::{Key, Op}, +}; use commonware_consensus::threshold_simplex::Prover; use commonware_cryptography::{ bls12381::primitives::{group, poly::Poly}, ed25519::PublicKey, sha256::Digest, }; -use std::{collections::HashMap, sync::{Arc, Mutex}}; -use alto_storage::{database::Database, transactional_db::{Key, Op}}; +use std::{ + collections::HashMap, + sync::{Arc, Mutex}, +}; mod actor; pub use actor::Actor; mod ingress; pub use ingress::Mailbox; -mod supervisor; mod router; +mod supervisor; pub use supervisor::Supervisor; @@ -32,7 +38,7 @@ pub struct Config { /// before blocking. pub mailbox_size: usize, - /// State + /// State pub state_cache: Arc>>, pub unfinalized_state: Arc>>, pub state_db: Arc>, diff --git a/chain/src/actors/application/router.rs b/chain/src/actors/application/router.rs index efb6135a..28e7bc4a 100644 --- a/chain/src/actors/application/router.rs +++ b/chain/src/actors/application/router.rs @@ -1,7 +1,7 @@ -use std::io; use axum::response::IntoResponse; use commonware_runtime::{Clock, Handle, Metrics, Spawner}; use rand::Rng; +use std::io; use tokio::net::TcpListener; use tracing::{event, Level}; @@ -11,9 +11,7 @@ pub struct RouterConfig { impl RouterConfig { pub fn default_config() -> RouterConfig { - RouterConfig { - port: 7844, - } + RouterConfig { port: 7844 } } } @@ -48,7 +46,7 @@ impl Router { pub fn stop(&self) { if !self.is_active { - return + return; } event!(Level::INFO, "stopped router service"); @@ -68,16 +66,22 @@ impl Router { } fn init_router(&mut self) { - // let router = axum::Router::new() - // .route("/", get(Router::handle_default)) - // .route(Router::PATH_SUBMIT_BLOCK, get(Router::handle_submit_block())); - // - // self.router = Some(router) + // let router = axum::Router::new() + // .route("/", get(Router::handle_default)) + // .route(Router::PATH_SUBMIT_BLOCK, get(Router::handle_submit_block())); + // + // self.router = Some(router) } async fn serve(&mut self) -> Result<(), Box> { - let listener = self.listener.take().ok_or("serve failed because listener is None"); - let router = self.router.take().ok_or("serve failed because router is None"); + let listener = self + .listener + .take() + .ok_or("serve failed because listener is None"); + let router = self + .router + .take() + .ok_or("serve failed because router is None"); axum::serve(listener.unwrap(), router.unwrap()).await?; Ok(()) } @@ -90,8 +94,8 @@ impl Router { Ok(value) => self.listener = Some(value), Err(error) => { println!("Error during listener init: {}", error); - return - }, + return; + } } self.init_router(); diff --git a/chain/src/actors/syncer/mod.rs b/chain/src/actors/syncer/mod.rs index 53236310..727d6c03 100644 --- a/chain/src/actors/syncer/mod.rs +++ b/chain/src/actors/syncer/mod.rs @@ -1,10 +1,13 @@ +use alto_storage::{ + database::Database, + transactional_db::{Key, Op}, +}; use commonware_cryptography::{bls12381::primitives::group, ed25519::PublicKey}; use governor::Quota; use std::{ collections::HashMap, sync::{Arc, Mutex}, }; -use alto_storage::{database::Database, transactional_db::{Key, Op}}; mod actor; mod archive; diff --git a/chain/src/bin/validator.rs b/chain/src/bin/validator.rs index ca4120ae..6d9517cc 100644 --- a/chain/src/bin/validator.rs +++ b/chain/src/bin/validator.rs @@ -1,7 +1,7 @@ use alto_chain::{engine, Config}; use alto_client::Client; -use alto_types::P2P_NAMESPACE; use alto_storage::rocks_db::RocksDbDatabase; +use alto_types::P2P_NAMESPACE; use axum::{routing::get, serve, Extension, Router}; use clap::{Arg, Command}; use commonware_cryptography::{ @@ -19,6 +19,7 @@ use commonware_utils::{from_hex_formatted, hex, quorum}; use futures::future::try_join_all; use governor::Quota; use prometheus_client::metrics::gauge::Gauge; +use std::sync::{Arc, Mutex}; use std::{ collections::HashMap, net::{IpAddr, Ipv4Addr, SocketAddr}, @@ -30,7 +31,6 @@ use std::{ }; use sysinfo::{Disks, System}; use tracing::{error, info, Level}; -use std::sync::{Arc, Mutex}; const SYSTEM_METRICS_REFRESH: Duration = Duration::from_secs(5); const METRICS_PORT: u16 = 9090; @@ -191,7 +191,7 @@ fn main() { let state_db = RocksDbDatabase::new_with_path(&config.state_db_directory) .expect("Could not create state db"); let wrapped_state_db = Arc::new(Mutex::new(state_db)); - + // Create engine let config = engine::Config { partition_prefix: "engine".to_string(), diff --git a/chain/src/engine.rs b/chain/src/engine.rs index 1ee2bada..2bba2344 100644 --- a/chain/src/engine.rs +++ b/chain/src/engine.rs @@ -2,6 +2,10 @@ use crate::{ actors::{application, syncer}, Indexer, }; +use alto_storage::{ + database::Database, + transactional_db::{Key, Op}, +}; use alto_types::NAMESPACE; use commonware_consensus::threshold_simplex::{self, Engine as Consensus, Prover}; use commonware_cryptography::{ @@ -17,10 +21,9 @@ use futures::future::try_join_all; use governor::clock::Clock as GClock; use governor::Quota; use rand::{CryptoRng, Rng}; -use std::time::Duration; -use std::sync::{Arc, Mutex}; use std::collections::HashMap; -use alto_storage::{transactional_db::{Key, Op}, database::Database}; +use std::sync::{Arc, Mutex}; +use std::time::Duration; use tracing::{error, warn}; pub struct Config { @@ -50,7 +53,7 @@ pub struct Config { pub struct Engine< B: Blob, E: Clock + GClock + Rng + CryptoRng + Spawner + Storage + Metrics, - I: Indexer, + I: Indexer, > { context: E, @@ -71,7 +74,7 @@ pub struct Engine< // state state_cache: Arc>>, unfinalized_state: Arc>>, - state_db: Arc>, + state_db: Arc>, } impl + Metrics, I: Indexer> @@ -80,8 +83,8 @@ impl + Metri pub async fn new(context: E, cfg: Config) -> Self { // @todo initalizing state cache and unfinalized state. // if it is necessary pass state_cache, unfinalized_state and state_db to both application and syncer. - let state_cache:Arc>> = Arc::new(Mutex::new(HashMap::new())); - let unfinalized_state:Arc>> = Arc::new(Mutex::new(HashMap::new())); + let state_cache: Arc>> = Arc::new(Mutex::new(HashMap::new())); + let unfinalized_state: Arc>> = Arc::new(Mutex::new(HashMap::new())); // Create the application let public = public(&cfg.identity); diff --git a/chain/src/lib.rs b/chain/src/lib.rs index a097e40e..e1fb66cf 100644 --- a/chain/src/lib.rs +++ b/chain/src/lib.rs @@ -82,8 +82,8 @@ pub struct Config { #[cfg(test)] mod tests { use super::*; - use alto_types::{Finalized, Notarized, Seed}; use alto_storage::rocks_db::RocksDbDatabase; + use alto_types::{Finalized, Notarized, Seed}; use bls12381::primitives::poly; use commonware_cryptography::{bls12381::dkg::ops, ed25519::PublicKey, Ed25519, Scheme}; use commonware_macros::test_traced; @@ -96,7 +96,7 @@ mod tests { use engine::{Config, Engine}; use governor::Quota; use rand::{rngs::StdRng, Rng, SeedableRng}; - use tracing_subscriber::fmt::format; + use std::{ collections::{HashMap, HashSet}, num::NonZeroU32, @@ -267,7 +267,10 @@ mod tests { // Create scheme context let public_key = scheme.public_key(); public_keys.insert(public_key.clone()); - let state_db = RocksDbDatabase::new_with_path(format!("/home/ubuntu/state_db/{}", idx).as_str()).expect("Could not create state db"); + let state_db = RocksDbDatabase::new_with_path( + format!("/home/ubuntu/state_db/{}", idx).as_str(), + ) + .expect("Could not create state db"); let wrapped_state_db = Arc::new(Mutex::new(state_db)); // Configure engine let uid = format!("validator-{}", public_key); @@ -430,7 +433,10 @@ mod tests { // Configure engine let public_key = scheme.public_key(); let uid = format!("validator-{}", public_key); - let state_db = RocksDbDatabase::new_with_path(format!("/home/ubuntu/state_db/{}", idx).as_str()).expect("Could not create state db"); + let state_db = RocksDbDatabase::new_with_path( + format!("/home/ubuntu/state_db/{}", idx).as_str(), + ) + .expect("Could not create state db"); let wrapped_state_db = Arc::new(Mutex::new(state_db)); let config: Config = engine::Config { partition_prefix: uid.clone(), @@ -516,7 +522,9 @@ mod tests { let share = shares[0]; let public_key = scheme.public_key(); let uid = format!("validator-{}", public_key); - let state_db = RocksDbDatabase::new_with_path(format!("/home/ubuntu/state_db/{}", uid).as_str()).expect("Could not create state db"); + let state_db = + RocksDbDatabase::new_with_path(format!("/home/ubuntu/state_db/{}", uid).as_str()) + .expect("Could not create state db"); let wrapped_state_db = Arc::new(Mutex::new(state_db)); let config: Config = engine::Config { partition_prefix: uid.clone(), @@ -649,7 +657,10 @@ mod tests { // Create scheme context let public_key = scheme.public_key(); public_keys.insert(public_key.clone()); - let state_db = RocksDbDatabase::new_with_path(format!("/home/ubuntu/state_db/{}", idx).as_str()).expect("Could not create state db"); + let state_db = RocksDbDatabase::new_with_path( + format!("/home/ubuntu/state_db/{}", idx).as_str(), + ) + .expect("Could not create state db"); let wrapped_state_db = Arc::new(Mutex::new(state_db)); // Configure engine let uid = format!("validator-{}", public_key); @@ -797,7 +808,10 @@ mod tests { // Configure engine let uid = format!("validator-{}", public_key); - let state_db = RocksDbDatabase::new_with_path(format!("/home/ubuntu/state_db/{}", idx).as_str()).expect("Could not create state db"); + let state_db = RocksDbDatabase::new_with_path( + format!("/home/ubuntu/state_db/{}", idx).as_str(), + ) + .expect("Could not create state db"); let wrapped_state_db = Arc::new(Mutex::new(state_db)); let config: Config = engine::Config { partition_prefix: uid.clone(), diff --git a/storage/src/database.rs b/storage/src/database.rs index ee73b320..0964eef3 100644 --- a/storage/src/database.rs +++ b/storage/src/database.rs @@ -1,10 +1,10 @@ use std::error::Error; // Define database interface that will be used for all impls -pub trait Database{ +pub trait Database { fn put(&mut self, key: &[u8], value: &[u8]) -> Result<(), Box>; fn get(&mut self, key: &[u8]) -> Result>, Box>; fn delete(&mut self, key: &[u8]) -> Result<(), Box>; -} \ No newline at end of file +} diff --git a/storage/src/hashmap_db.rs b/storage/src/hashmap_db.rs index cdf4155e..9fc9582a 100644 --- a/storage/src/hashmap_db.rs +++ b/storage/src/hashmap_db.rs @@ -1,6 +1,6 @@ +use crate::database::Database; use std::collections::HashMap; use std::error::Error; -use crate::database::Database; pub struct HashmapDatabase { data: HashMap, @@ -31,9 +31,9 @@ impl Database for HashmapDatabase { fn get(&mut self, key: &[u8]) -> Result>, Box> { let str_key: String = String::from_utf8(key.into()).unwrap(); - self.data.get(&str_key).map_or( - Ok(None), - |v| Ok(Some(v.clone().into()))) + self.data + .get(&str_key) + .map_or(Ok(None), |v| Ok(Some(v.clone().into()))) } fn delete(&mut self, key: &[u8]) -> Result<(), Box> { @@ -43,7 +43,6 @@ impl Database for HashmapDatabase { } } - #[cfg(test)] mod tests { use super::*; diff --git a/storage/src/lib.rs b/storage/src/lib.rs index 14018bda..0734abad 100644 --- a/storage/src/lib.rs +++ b/storage/src/lib.rs @@ -2,4 +2,4 @@ pub mod database; pub mod hashmap_db; pub mod rocks_db; pub mod state_db; -pub mod transactional_db; \ No newline at end of file +pub mod transactional_db; diff --git a/storage/src/rocks_db.rs b/storage/src/rocks_db.rs index a779c257..67af4397 100644 --- a/storage/src/rocks_db.rs +++ b/storage/src/rocks_db.rs @@ -1,6 +1,6 @@ -use std::error::Error; -use rocksdb::{DB, Options}; use crate::database::Database; +use rocksdb::{Options, DB}; +use std::error::Error; use std::path::Path; use tempfile::TempDir; @@ -29,7 +29,6 @@ impl RocksDbDatabase { let db_path = temp_dir.path().join("testdb"); Self::new_with_path(db_path.to_str().unwrap()) } - } impl Database for RocksDbDatabase { diff --git a/storage/src/state_db.rs b/storage/src/state_db.rs index 143667f7..65bcea40 100644 --- a/storage/src/state_db.rs +++ b/storage/src/state_db.rs @@ -1,4 +1,4 @@ -use crate::transactional_db::{InMemoryCachingTransactionalDb, TransactionalDb}; +use crate::transactional_db::TransactionalDb; use alto_types::account::{Account, Balance}; use alto_types::address::Address; use bytes::Bytes; @@ -26,7 +26,9 @@ impl<'a> StateDb<'a> { if let Some(value) = v { let bytes = Bytes::from(value); let mut read_buf = ReadBuffer::new(bytes); - Account::read(&mut read_buf).map(Some).map_err(|e| Box::new(e) as Box) + Account::read(&mut read_buf) + .map(Some) + .map_err(|e| Box::new(e) as Box) } else { Err("Account not found".into()) } @@ -42,9 +44,9 @@ impl<'a> StateDb<'a> { pub fn get_balance(&mut self, address: &Address) -> Option { match self.get_account(address) { - Ok(Some(acc)) => Some(acc.balance), // return balance if account exists - Ok(None) => Some(0), // return 0 if no account - Err(_) => None, // return none if an err occurred + Ok(Some(acc)) => Some(acc.balance), // return balance if account exists + Ok(None) => Some(0), // return 0 if no account + Err(_) => None, // return none if an err occurred } } @@ -73,25 +75,29 @@ impl<'a> StateDb<'a> { #[cfg(test)] mod tests { - use alto_types::address::Address; - use crate::hashmap_db::HashmapDatabase; - use crate::transactional_db::{InMemoryCachingTransactionalDb, Op, Key}; use super::*; + use crate::hashmap_db::HashmapDatabase; + use crate::transactional_db::{InMemoryCachingTransactionalDb, Key, Op}; + use alto_types::address::Address; use std::collections::HashMap; use std::sync::{Arc, Mutex}; #[test] fn test_it_works() { // setup state db - let cache: Arc>>= Arc::new(Mutex::new(HashMap::new())); - let unfinalized: Arc>>= Arc::new(Mutex::new(HashMap::new())); + let cache: Arc>> = Arc::new(Mutex::new(HashMap::new())); + let unfinalized: Arc>> = Arc::new(Mutex::new(HashMap::new())); let db = Arc::new(Mutex::new(HashmapDatabase::new())); - let mut in_mem = InMemoryCachingTransactionalDb::new(Arc::clone(&cache),Arc::clone(&unfinalized), db); + let mut in_mem = + InMemoryCachingTransactionalDb::new(Arc::clone(&cache), Arc::clone(&unfinalized), db); let address = Address::create_random_address(); - let account = Account{address: address.clone(), balance: 1000}; + let account = Account { + address: address.clone(), + balance: 1000, + }; let mut state_db = StateDb::new(&mut in_mem); - let _ = state_db.set_account(&account).unwrap(); + state_db.set_account(&account).unwrap(); let _ = in_mem.commit_last_tx(); let _ = in_mem.commit(); assert_eq!(unfinalized.lock().unwrap().len(), 1); @@ -104,10 +110,11 @@ mod tests { #[should_panic] fn test_no_account_earlier() { // setup state db - let cache: Arc>>= Arc::new(Mutex::new(HashMap::new())); - let unfinalized: Arc>>= Arc::new(Mutex::new(HashMap::new())); + let cache: Arc>> = Arc::new(Mutex::new(HashMap::new())); + let unfinalized: Arc>> = Arc::new(Mutex::new(HashMap::new())); let db = Arc::new(Mutex::new(HashmapDatabase::new())); - let mut in_mem = InMemoryCachingTransactionalDb::new(Arc::clone(&cache),Arc::clone(&unfinalized), db); + let mut in_mem = + InMemoryCachingTransactionalDb::new(Arc::clone(&cache), Arc::clone(&unfinalized), db); let address = Address::create_random_address(); @@ -115,4 +122,4 @@ mod tests { let _ = state_db.get_account(&address).unwrap(); } -} \ No newline at end of file +} diff --git a/storage/src/transactional_db.rs b/storage/src/transactional_db.rs index 0d108814..0214c5d2 100644 --- a/storage/src/transactional_db.rs +++ b/storage/src/transactional_db.rs @@ -13,18 +13,18 @@ pub type Key = [u8; 33]; #[derive(Clone, Copy, PartialEq, Eq, Debug)] pub enum OpAction { // key is read - Read, + Read, // key is created - Create, + Create, // key is updated - Update, + Update, // key got deleted - Delete, + Delete, } /// Op contains action performed and value stored over a key. #[derive(Clone, Debug, PartialEq, Eq)] -pub struct Op{ +pub struct Op { // Action performed pub action: OpAction, // Resulting value after perfromed action. @@ -35,10 +35,10 @@ pub struct Op{ pub trait TransactionalDb { /// initialize the cache with an already available hashmap of key-value pairs. fn init_cache(&mut self, cache: Arc>>); - /// get the value corresponding to the key. + /// get the value corresponding to the key. /// use this method for querying state. fn get(&mut self, key: &Key) -> Result>, Box>; - /// insert a key-pair. could be a create or delete action. + /// insert a key-pair. could be a create or delete action. /// underlying struct should handle the OpAction part. fn insert(&mut self, key: &Key, value: Vec) -> Result<(), Box>; /// delete a key-pair @@ -46,7 +46,7 @@ pub trait TransactionalDb { /// get a key from cache. do not call this method directly, instead call get. fn get_from_cache(&self, key: &Key) -> Result>, Box>; /// get a key from the underlying storage. do not call this method directly, instead call get. If the key is not in the storage, it will return an error. - fn get_from_db(&mut self, key: &Key) -> Result>, Box>; + fn get_from_db(&mut self, key: &Key) -> Result>, Box>; /// commit last tx changes within the cache fn commit_last_tx(&mut self) -> Result<(), Box>; /// commit changes to the unfinalized map. @@ -59,22 +59,26 @@ pub trait TransactionalDb { pub struct InMemoryCachingTransactionalDb { /// cache is init'ed at the start. - pub cache:Arc>>, + pub cache: Arc>>, /// unfinalized changes from previous block(s). - pub unfinalized:Arc>>, - /// set of all key value changes from last init. - pub touched: HashMap, + pub unfinalized: Arc>>, + /// set of all key value changes from last init. + pub touched: HashMap, /// set of all key value changes from last commit_last_tx pub touched_tx: HashMap, /// underlying database. - pub db: Arc>, + pub db: Arc>, } impl InMemoryCachingTransactionalDb { - pub fn new(cache: Arc>>, unfinalized: Arc>>, db: Arc>) -> Self { - Self{ - cache: cache, - unfinalized: unfinalized, + pub fn new( + cache: Arc>>, + unfinalized: Arc>>, + db: Arc>, + ) -> Self { + Self { + cache, + unfinalized, touched: HashMap::new(), touched_tx: HashMap::new(), db, @@ -99,9 +103,9 @@ impl TransactionalDb for InMemoryCachingTransactionalDb { match self.touched.get(key) { // key is used in the current block. Some(t_op) => { - let v = Op { - action: OpAction::Read, - value:t_op.value.clone() + let v = Op { + action: OpAction::Read, + value: t_op.value.clone(), }; self.touched_tx.insert(*key, v); Ok(Some(t_op.value.clone())) @@ -111,12 +115,12 @@ impl TransactionalDb for InMemoryCachingTransactionalDb { match self.unfinalized.lock().unwrap().get(key) { // the key is used in the previous block(s). but the blocks did nt finalze yet. Some(u_op) => { - let v = Op { - action: OpAction::Read, - value:u_op.value.clone() + let v = Op { + action: OpAction::Read, + value: u_op.value.clone(), }; self.touched_tx.insert(*key, v); - return Ok(Some(u_op.value.clone())); + Ok(Some(u_op.value.clone())) } // the key is not used in the previous block(s). None => { @@ -125,18 +129,20 @@ impl TransactionalDb for InMemoryCachingTransactionalDb { // if it is not, check if the key is in the underlying db. // if it is, return the value. // if it is not, return an error. - if let Some(f_c) = self.get_from_cache(key) - .ok() - .flatten() - .or_else(|| self.db.lock().ok()?.get(key).ok().flatten()) { - let v = Op { - action: OpAction::Read, - value: f_c.clone() + if let Some(f_c) = self + .get_from_cache(key) + .ok() + .flatten() + .or_else(|| self.db.lock().ok()?.get(key).ok().flatten()) + { + let v = Op { + action: OpAction::Read, + value: f_c.clone(), }; self.touched_tx.insert(*key, v); - return Ok(Some(f_c)); + Ok(Some(f_c)) } else { - return Err("Key does not exist.".into()); + Err("Key does not exist.".into()) } } } @@ -149,14 +155,14 @@ impl TransactionalDb for InMemoryCachingTransactionalDb { fn insert(&mut self, key: &Key, value: Vec) -> Result<(), Box> { let op = Op { action: OpAction::Update, // @todo change this to OpAction::Update or OpAction::Create? based on if the key-pair actually exists. - value: value, + value, }; self.touched_tx.insert(*key, op); Ok(()) } fn delete(&mut self, key: &Key) -> Result<(), Box> { - let op = Op{ + let op = Op { action: OpAction::Delete, value: vec![], }; @@ -165,13 +171,19 @@ impl TransactionalDb for InMemoryCachingTransactionalDb { } fn get_from_cache(&self, key: &Key) -> Result>, Box> { - self.cache.lock().unwrap().get(key) + self.cache + .lock() + .unwrap() + .get(key) .map(|op| Some(op.value.clone())) .ok_or_else(|| "Key not found in cache.".into()) } fn get_from_db(&mut self, key: &Key) -> Result>, Box> { - self.db.lock().unwrap().get(key) + self.db + .lock() + .unwrap() + .get(key) .map(|v| Some(v.clone()))? .ok_or_else(|| "Key not found in db.".into()) } @@ -199,7 +211,10 @@ impl TransactionalDb for InMemoryCachingTransactionalDb { } } -pub fn merge_maps_l(map1: Arc>>, map2: Arc>>) -> Arc>> { +pub fn merge_maps_l( + map1: Arc>>, + map2: Arc>>, +) -> Arc>> { let mut map1_g = map1.lock().unwrap(); let map2_g = map2.lock().unwrap(); for (key, op) in map2_g.iter() { @@ -207,11 +222,13 @@ pub fn merge_maps_l(map1: Arc>>, map2: Arc>>, map2: Arc(map1: Arc>>, map2: &'a mut HashMap) -> Arc>> { +pub fn merge_maps( + map1: Arc>>, + map2: &mut HashMap, +) -> Arc>> { let mut map1_g = map1.lock().unwrap(); for (key, op) in map2.iter() { if let Some(existing_op) = map1_g.get(key) { // there is a key existing in both maps. if op.action == OpAction::Delete { map1_g.insert(*key, op.clone()); - - }else if op.action == OpAction::Update { + } else if op.action == OpAction::Update { // if op.action is update, then update the map1. as update superseeds everything. map1_g.insert(*key, op.clone()); - }else if op.action == OpAction::Read && (existing_op.action == OpAction::Update || existing_op.action == OpAction::Create) { + } else if op.action == OpAction::Read + && (existing_op.action == OpAction::Update + || existing_op.action == OpAction::Create) + { // reading on a delete will return a nill value with existing op set to delete. this should not be an issue. let new_op = Op { action: existing_op.action, @@ -254,17 +276,22 @@ pub fn merge_maps<'a>(map1: Arc>>, map2: &'a mut HashMap< Arc::clone(&map1) } -pub fn merge_maps_nl<'a, 'b>(map1: & 'a mut HashMap, map2: & 'b HashMap) -> & 'a mut HashMap { +pub fn merge_maps_nl<'a>( + map1: &'a mut HashMap, + map2: &HashMap, +) -> &'a mut HashMap { for (key, op) in map2.iter() { if let Some(existing_op) = map1.get(key) { // there is a key existing in both maps. if op.action == OpAction::Delete { map1.insert(*key, op.clone()); - - }else if op.action == OpAction::Update { + } else if op.action == OpAction::Update { // if op.action is update, then update the map1. as update superseeds everything. map1.insert(*key, op.clone()); - }else if op.action == OpAction::Read && (existing_op.action == OpAction::Update || existing_op.action == OpAction::Create) { + } else if op.action == OpAction::Read + && (existing_op.action == OpAction::Update + || existing_op.action == OpAction::Create) + { // reading on a delete will return a nill value with existing op set to delete. this should not be an issue. let new_op = Op { action: existing_op.action, @@ -282,8 +309,8 @@ pub fn merge_maps_nl<'a, 'b>(map1: & 'a mut HashMap, map2: & 'b HashMap #[cfg(test)] mod tests { - use crate::hashmap_db::HashmapDatabase; use super::*; + use crate::hashmap_db::HashmapDatabase; use std::sync::Mutex; #[test] fn test_it_works() { @@ -296,52 +323,68 @@ mod tests { let value2 = [2; 33]; db.lock().unwrap().put(&key1, &value1).unwrap(); { - let mut in_mem_db = InMemoryCachingTransactionalDb::new(Arc::clone(&cache), Arc::clone(&unfinalized), db.clone()); + let mut in_mem_db = InMemoryCachingTransactionalDb::new( + Arc::clone(&cache), + Arc::clone(&unfinalized), + db.clone(), + ); // start a tx assert_eq!(in_mem_db.get(&key1).unwrap(), Some(value1.to_vec())); // end a tx assert_eq!(in_mem_db.touched_tx.len(), 1); - assert_eq!(in_mem_db.touched.len(),0); + assert_eq!(in_mem_db.touched.len(), 0); assert_eq!(in_mem_db.cache.lock().unwrap().len(), 0); - assert_eq!(in_mem_db.touched_tx.get(&key1).unwrap(), &Op{ - action: OpAction::Read, - value: value1.to_vec(), - }); + assert_eq!( + in_mem_db.touched_tx.get(&key1).unwrap(), + &Op { + action: OpAction::Read, + value: value1.to_vec(), + } + ); // commit the tx let _ = in_mem_db.commit_last_tx(); assert_eq!(in_mem_db.touched_tx.len(), 0); - assert_eq!(in_mem_db.touched.len(),1); + assert_eq!(in_mem_db.touched.len(), 1); assert_eq!(in_mem_db.cache.lock().unwrap().len(), 0); - assert_eq!(in_mem_db.touched.get(&key1).unwrap(), &Op{ - action: OpAction::Read, - value: value1.to_vec(), - }); + assert_eq!( + in_mem_db.touched.get(&key1).unwrap(), + &Op { + action: OpAction::Read, + value: value1.to_vec(), + } + ); // start a new tx in_mem_db.insert(&key2, value2.to_vec()).unwrap(); assert_eq!(in_mem_db.touched_tx.len(), 1); assert_eq!(in_mem_db.touched.len(), 1); assert_eq!(in_mem_db.cache.lock().unwrap().len(), 0); - assert_eq!(in_mem_db.touched_tx.get(&key2).unwrap(), &Op{ - action: OpAction::Update, - value: value2.to_vec(), - }); - assert_eq!(in_mem_db.touched.get(&key1).unwrap(), &Op{ - action: OpAction::Read, - value: value1.to_vec(), - }); + assert_eq!( + in_mem_db.touched_tx.get(&key2).unwrap(), + &Op { + action: OpAction::Update, + value: value2.to_vec(), + } + ); + assert_eq!( + in_mem_db.touched.get(&key1).unwrap(), + &Op { + action: OpAction::Read, + value: value1.to_vec(), + } + ); // end tx assert_eq!(in_mem_db.get(&key2).unwrap(), Some(value2.to_vec())); assert_eq!(in_mem_db.touched_tx.len(), 1); assert_eq!(in_mem_db.touched.len(), 1); assert_eq!(in_mem_db.cache.lock().unwrap().len(), 0); // commit tx - let _ = in_mem_db.commit_last_tx().unwrap(); + in_mem_db.commit_last_tx().unwrap(); assert_eq!(in_mem_db.touched_tx.len(), 0); assert_eq!(in_mem_db.touched.len(), 2); assert_eq!(in_mem_db.cache.lock().unwrap().len(), 0); // commit block - let _ = in_mem_db.commit().unwrap(); + in_mem_db.commit().unwrap(); assert_eq!(in_mem_db.touched_tx.len(), 0); assert_eq!(in_mem_db.touched.len(), 0); assert_eq!(in_mem_db.cache.lock().unwrap().len(), 0); @@ -355,20 +398,33 @@ mod tests { db.lock().unwrap().delete(key).unwrap(); } else if op.action == OpAction::Update { db.lock().unwrap().put(key, &op.value).unwrap(); - } + } } merge_maps_l(Arc::clone(&cache), Arc::clone(&unfinalized)); assert_eq!(cache.lock().unwrap().len(), 2); unfinalized.lock().unwrap().clear(); assert_eq!(unfinalized.lock().unwrap().len(), 0); // try fetching key2 from db. this should pass. - assert_eq!(db.lock().unwrap().get(&key2).unwrap(), Some(value2.to_vec())); + assert_eq!( + db.lock().unwrap().get(&key2).unwrap(), + Some(value2.to_vec()) + ); // try fetching key1 from db. this should pass. - assert_eq!(db.lock().unwrap().get(&key1).unwrap(), Some(value1.to_vec())); + assert_eq!( + db.lock().unwrap().get(&key1).unwrap(), + Some(value1.to_vec()) + ); // new block { - let mut in_mem_db = InMemoryCachingTransactionalDb::new(Arc::clone(&cache), Arc::clone(&unfinalized), db.clone()); - assert_eq!(db.lock().unwrap().get(&key1).unwrap(), Some(value1.to_vec())); + let mut in_mem_db = InMemoryCachingTransactionalDb::new( + Arc::clone(&cache), + Arc::clone(&unfinalized), + db.clone(), + ); + assert_eq!( + db.lock().unwrap().get(&key1).unwrap(), + Some(value1.to_vec()) + ); assert_eq!(in_mem_db.get(&key1).unwrap(), Some(value1.to_vec())); assert_eq!(in_mem_db.get(&key2).unwrap(), Some(value2.to_vec())); // delete key1 @@ -377,11 +433,13 @@ mod tests { assert_eq!(in_mem_db.touched.len(), 0); assert_eq!(in_mem_db.cache.lock().unwrap().len(), 2); assert_eq!(in_mem_db.unfinalized.lock().unwrap().len(), 0); - assert_eq!(in_mem_db.touched_tx.get(&key1).unwrap(), &Op{ - action: OpAction::Delete, - value: vec![], - }); + assert_eq!( + in_mem_db.touched_tx.get(&key1).unwrap(), + &Op { + action: OpAction::Delete, + value: vec![], + } + ); } } } - diff --git a/types/src/account.rs b/types/src/account.rs index e947d67d..3be32107 100644 --- a/types/src/account.rs +++ b/types/src/account.rs @@ -1,5 +1,5 @@ -use commonware_codec::{Codec, Error, Reader, Writer}; use crate::address::Address; +use commonware_codec::{Codec, Error, Reader, Writer}; pub type Balance = u64; @@ -33,7 +33,7 @@ impl Account { impl Codec for Account { fn write(&self, writer: &mut impl Writer) { - // @rikoeldon I think we don't need to write account address into the state. + // @rikoeldon I think we don't need to write account address into the state. // account address is part of the key. writer.write_bytes(self.address.0.as_slice()); self.balance.write(writer); @@ -43,10 +43,10 @@ impl Codec for Account { let addr_bytes = <[u8; 33]>::read(reader)?; let address = Address::from_bytes(&addr_bytes[1..]).unwrap(); let balance = ::read(reader)?; - Ok(Self{address, balance}) + Ok(Self { address, balance }) } fn len_encoded(&self) -> usize { Codec::len_encoded(&self.address.0) + Codec::len_encoded(&self.balance) } -} \ No newline at end of file +} diff --git a/types/src/address.rs b/types/src/address.rs index c14b7b0f..79a7bb8b 100644 --- a/types/src/address.rs +++ b/types/src/address.rs @@ -1,9 +1,9 @@ +use crate::{PublicKey, ADDRESSLEN}; use more_asserts::assert_le; use rand::Rng; -use crate::{PublicKey, ADDRESSLEN}; #[derive(Hash, Eq, PartialEq, Clone, Debug)] -pub struct Address(pub [u8;ADDRESSLEN]); +pub struct Address(pub [u8; ADDRESSLEN]); impl Address { pub fn new(slice: &[u8]) -> Self { @@ -36,7 +36,7 @@ impl Address { } pub fn empty() -> Self { - Self([0;ADDRESSLEN]) + Self([0; ADDRESSLEN]) } pub fn is_empty(&self) -> bool { @@ -47,7 +47,7 @@ impl Address { &self.0 } - pub fn as_bytes(&self) -> &[u8;ADDRESSLEN] { + pub fn as_bytes(&self) -> &[u8; ADDRESSLEN] { &self.0 } -} \ No newline at end of file +} diff --git a/types/src/block.rs b/types/src/block.rs index 142d19f5..49ff4fe1 100644 --- a/types/src/block.rs +++ b/types/src/block.rs @@ -1,11 +1,11 @@ +use crate::signed_tx::{pack_signed_txs, unpack_signed_txs, SignedTx, SignedTxChars}; use crate::{Finalization, Notarization}; -use crate::signed_tx::{SignedTx, pack_signed_txs, unpack_signed_txs, SignedTxChars}; use bytes::{Buf, BufMut}; use commonware_cryptography::{bls12381::PublicKey, sha256, sha256::Digest, Hasher, Sha256}; use commonware_utils::{Array, SizedSerialize}; -// @todo add state root, fee manager and results to the block struct. -// what method of state root generation should be used? +// @todo add state root, fee manager and results to the block struct. +// what method of state root generation should be used? #[derive(Clone, Debug)] pub struct Block { /// The parent block's digest. @@ -18,7 +18,7 @@ pub struct Block { pub timestamp: u64, /// The raw transactions in the block. - pub raw_txs: Vec, + pub raw_txs: Vec, /// The state root of the block. pub state_root: Digest, @@ -29,7 +29,13 @@ pub struct Block { } impl Block { - fn compute_digest(parent: &Digest, height: u64, timestamp: u64, raw_txs: Vec, state_root: &Digest) -> Digest { + fn compute_digest( + parent: &Digest, + height: u64, + timestamp: u64, + raw_txs: Vec, + state_root: &Digest, + ) -> Digest { let mut hasher = Sha256::new(); hasher.update(parent); hasher.update(&height.to_be_bytes()); @@ -39,7 +45,13 @@ impl Block { hasher.finalize() } - pub fn new(parent: Digest, height: u64, timestamp: u64, txs: Vec, state_root: Digest) -> Self { + pub fn new( + parent: Digest, + height: u64, + timestamp: u64, + txs: Vec, + state_root: Digest, + ) -> Self { // let mut txs = txs; // @todo this is packing txs in a block. let raw_txs = pack_signed_txs(txs.clone()); @@ -77,7 +89,7 @@ impl Block { let raw_txs = bytes.to_vec(); let digest = Self::compute_digest(&parent, height, timestamp, raw_txs.clone(), &state_root); let txs = unpack_signed_txs(raw_txs.clone()); - + // Return block Some(Self { parent, diff --git a/types/src/lib.rs b/types/src/lib.rs index a0f1508c..741a1a24 100644 --- a/types/src/lib.rs +++ b/types/src/lib.rs @@ -2,26 +2,26 @@ mod block; -use std::time::SystemTime; +pub use block::{Block, Finalized, Notarized}; use commonware_cryptography::{Ed25519, Scheme}; use commonware_utils::SystemTimeExt; -pub use block::{Block, Finalized, Notarized}; +use std::time::SystemTime; mod consensus; pub use consensus::{leader_index, Finalization, Kind, Notarization, Nullification, Seed}; -pub mod wasm; -pub mod wallet; -pub mod tx; +pub mod account; +pub mod address; pub mod signed_tx; pub mod state; -pub mod address; -pub mod account; +pub mod tx; pub mod units; +pub mod wallet; +pub mod wasm; use rand::rngs::OsRng; // We don't use functions here to guard against silent changes. pub const NAMESPACE: &[u8] = b"_ALTO"; -pub const TX_NAMESPACE:&[u8] = b"_tx_namespace_"; +pub const TX_NAMESPACE: &[u8] = b"_tx_namespace_"; pub const P2P_NAMESPACE: &[u8] = b"_ALTO_P2P"; pub const SEED_NAMESPACE: &[u8] = b"_ALTO_SEED"; pub const NOTARIZE_NAMESPACE: &[u8] = b"_ALTO_NOTARIZE"; @@ -204,7 +204,7 @@ mod tests { let parent_digest = hash(&[0; 32]); let height = 0; let timestamp = 1; - let block = Block::new(parent_digest, height, timestamp, Vec::new(), [0;32].into()); + let block = Block::new(parent_digest, height, timestamp, Vec::new(), [0; 32].into()); // Create notarization let view = 0; diff --git a/types/src/signed_tx.rs b/types/src/signed_tx.rs index 7331fe8e..610d1f95 100644 --- a/types/src/signed_tx.rs +++ b/types/src/signed_tx.rs @@ -1,7 +1,7 @@ use crate::address::Address; -use crate::{PublicKey, TX_NAMESPACE, Signature}; -use crate::wallet::{Wallet, WalletMethods}; use crate::tx::{Tx, TxMethods}; +use crate::wallet::{Wallet, WalletMethods}; +use crate::{PublicKey, Signature, TX_NAMESPACE}; use commonware_cryptography::{Ed25519, Scheme}; // this is sent by the user to the validators. #[derive(Clone, Debug)] @@ -14,7 +14,7 @@ pub struct SignedTx { } // function names are self explanatory. -pub trait SignedTxChars:Sized { +pub trait SignedTxChars: Sized { fn new(tx: Tx, pub_key: PublicKey, signature: Vec) -> Self; // fn sign(&mut self, wallet: Wallet) -> SignedTx; fn verify(&mut self) -> bool; @@ -22,7 +22,7 @@ pub trait SignedTxChars:Sized { fn public_key(&self) -> Vec; fn address(&self) -> Address; fn encode(&mut self) -> Vec; - fn decode(bytes: &[u8]) -> Result; + fn decode(bytes: &[u8]) -> Result; } impl SignedTxChars for SignedTx { @@ -72,23 +72,29 @@ impl SignedTxChars for SignedTx { } // @todo add syntactic checks and use methods consume. - fn decode(bytes: &[u8]) -> Result { // @todo this method seems untidy. + fn decode(bytes: &[u8]) -> Result { + // @todo this method seems untidy. let raw_tx_len = u64::from_be_bytes(bytes[0..8].try_into().unwrap()); - let raw_tx = &bytes[8..8+raw_tx_len as usize]; - let pub_key = &bytes[8+raw_tx_len as usize..8+raw_tx_len as usize+32]; - let signature = &bytes[8+raw_tx_len as usize+32..]; + let raw_tx = &bytes[8..8 + raw_tx_len as usize]; + let pub_key = &bytes[8 + raw_tx_len as usize..8 + raw_tx_len as usize + 32]; + let signature = &bytes[8 + raw_tx_len as usize + 32..]; let public_key = PublicKey::try_from(pub_key); if public_key.is_err() { return Err(public_key.unwrap_err().to_string()); } let public_key = public_key.unwrap(); let tx = Tx::decode(raw_tx); - if tx.is_err(){ - return Err(tx.unwrap_err()); + if tx.is_err() { + return Err(tx.unwrap_err()); } - Ok(SignedTx { tx: tx.unwrap(), pub_key: public_key.clone(), address: Address::from_pub_key(&public_key), signature: signature.to_vec() }) + Ok(SignedTx { + tx: tx.unwrap(), + pub_key: public_key.clone(), + address: Address::from_pub_key(&public_key), + signature: signature.to_vec(), + }) } } @@ -138,13 +144,13 @@ pub fn unpack_signed_txs(bytes: Vec) -> Vec { #[cfg(test)] mod tests { use std::error::Error; - use commonware_cryptography::bls12381::primitives::ops::keypair; - use commonware_cryptography::sha256::Digest; - use more_asserts::assert_gt; - use crate::{create_test_keypair, curr_timestamp}; + + use super::*; use crate::tx::Unit; use crate::units::transfer::Transfer; - use super::*; + use crate::{create_test_keypair, curr_timestamp}; + use commonware_cryptography::sha256::Digest; + use more_asserts::assert_gt; #[test] fn test_encode_decode() -> Result<(), Box> { @@ -153,13 +159,13 @@ mod tests { let priority_fee = 75; let chain_id = 45205; let transfer = Transfer::new(); - let units : Vec> = vec![Box::new(transfer)]; - let mut digest: [u8; 32] = [0; 32]; + let units: Vec> = vec![Box::new(transfer)]; + let digest: [u8; 32] = [0; 32]; let id = Digest::from(digest.clone()); let (pk, sk) = create_test_keypair(); // TODO: the .encode call on next line gave error and said origin_msg needed to be mut? but why? // shouldn't encode be able to encode without changing the msg? - let mut tx = Tx { + let tx = Tx { timestamp, max_fee, priority_fee, @@ -176,7 +182,7 @@ mod tests { }; let encoded_bytes = origin_msg.encode(); assert_gt!(encoded_bytes.len(), 0); - let mut decoded_msg = SignedTx::decode(&encoded_bytes)?; + let decoded_msg = SignedTx::decode(&encoded_bytes)?; assert_eq!(origin_msg.pub_key, decoded_msg.pub_key); assert_eq!(origin_msg.address, decoded_msg.address); assert_eq!(origin_msg.signature, decoded_msg.signature); @@ -184,4 +190,3 @@ mod tests { Ok(()) } } - diff --git a/types/src/state.rs b/types/src/state.rs index cdd3e5f9..93a4e2ff 100644 --- a/types/src/state.rs +++ b/types/src/state.rs @@ -1,6 +1,6 @@ use std::error::Error; -pub type UnitKey<'a> = &'a [u8;33]; // 1st byte denotes the type of key. 0b for account key, 1b for others. +pub type UnitKey<'a> = &'a [u8; 33]; // 1st byte denotes the type of key. 0b for account key, 1b for others. pub trait State { fn get(&self, key: UnitKey) -> Result>, Box>; fn get_multi_key(&self, key: Vec) -> Result>, Box>; @@ -8,4 +8,4 @@ pub trait State { fn delete(&mut self, key: UnitKey) -> Result<(), Box>; fn commit(&mut self) -> Result<(), Box>; fn rollback(&mut self) -> Result<(), Box>; -} \ No newline at end of file +} diff --git a/types/src/tx.rs b/types/src/tx.rs index c610c685..26e1e440 100644 --- a/types/src/tx.rs +++ b/types/src/tx.rs @@ -1,12 +1,12 @@ -use std::any::Any; use commonware_cryptography::sha256; use commonware_cryptography::sha256::Digest; +use std::any::Any; -use crate::units; use crate::address::Address; -use crate::wallet::Wallet; use crate::signed_tx::SignedTx; use crate::state::State; +use crate::units; +use crate::wallet::Wallet; use commonware_utils::SystemTimeExt; use std::time::SystemTime; @@ -32,9 +32,9 @@ pub struct UnitContext { // timestamp of the tx. pub timestamp: u64, // chain id of the tx. - pub chain_id: u64, + pub chain_id: u64, // sender of the tx. - pub sender: Address, + pub sender: Address, } pub trait UnitClone { @@ -51,7 +51,7 @@ where } // unit need to be simple and easy to be packed in the tx and executed by the vm. -pub trait Unit : UnitClone + Send + Sync + std::fmt::Debug { +pub trait Unit: UnitClone + Send + Sync + std::fmt::Debug { fn unit_type(&self) -> UnitType; fn encode(&self) -> Vec; fn decode(&mut self, bytes: &[u8]); @@ -88,21 +88,26 @@ pub struct Tx { // units are fundamental unit of a tx. similar to actions. pub units: Vec>, - // id is the transaction id. It is the hash of digest. pub id: Digest, // digest is encoded tx. pub digest: Vec, } -pub trait TxMethods:Sized { +pub trait TxMethods: Sized { // new is used to create a new instance of Tx with given units and chain id. fn new(units: Vec>, chain_id: u64) -> Self; // set_fee is used to set the max fee and priority fee of the tx. fn set_fee(&mut self, max_fee: u64, priority_fee: u64); // sign is used to sign the tx with the given wallet. fn sign(&mut self, wallet: Wallet) -> SignedTx; - fn from(timestamp: u64, units: Vec>, priority_fee: u64, max_fee: u64, chain_id: u64) -> Self; + fn from( + timestamp: u64, + units: Vec>, + priority_fee: u64, + max_fee: u64, + chain_id: u64, + ) -> Self; // returns tx id. fn id(&mut self) -> Digest; @@ -149,7 +154,13 @@ impl TxMethods for Tx { SignedTx::sign(self.clone(), wallet) } - fn from(timestamp: u64, units: Vec>, priority_fee: u64, max_fee: u64, chain_id: u64) -> Self { + fn from( + timestamp: u64, + units: Vec>, + priority_fee: u64, + max_fee: u64, + chain_id: u64, + ) -> Self { let mut tx = Self::default(); tx.timestamp = timestamp; tx.units = units; @@ -210,7 +221,7 @@ impl TxMethods for Tx { let mut tx = Self::default(); tx.digest = bytes.to_vec(); // @todo ?? tx.timestamp = u64::from_be_bytes(bytes[0..8].try_into().unwrap()); - tx.max_fee = u64::from_be_bytes(bytes[8..16].try_into().unwrap()); + tx.max_fee = u64::from_be_bytes(bytes[8..16].try_into().unwrap()); tx.priority_fee = u64::from_be_bytes(bytes[16..24].try_into().unwrap()); tx.chain_id = u64::from_be_bytes(bytes[24..32].try_into().unwrap()); let units = unpack_units(&bytes[32..]); @@ -246,7 +257,11 @@ fn unpack_units(digest: &[u8]) -> Result>, String> { Ok(val) } - fn read_bytes<'a>(input: &'a [u8], offset: &'a mut usize, len: usize) -> Result<&'a [u8], String> { + fn read_bytes<'a>( + input: &'a [u8], + offset: &'a mut usize, + len: usize, + ) -> Result<&'a [u8], String> { if input.len() < *offset + len { return Err("Unexpected end of input when reading bytes".into()); } @@ -257,7 +272,7 @@ fn unpack_units(digest: &[u8]) -> Result>, String> { let unit_count = read_u64(digest, &mut offset)?; - let mut units:Vec> = Vec::with_capacity(unit_count as usize); + let mut units: Vec> = Vec::with_capacity(unit_count as usize); for _ in 0..unit_count { let unit_type = read_u8(digest, &mut offset)?; @@ -268,7 +283,7 @@ fn unpack_units(digest: &[u8]) -> Result>, String> { return Err(format!("Invalid unit type: {}", unit_type.unwrap_err())); } let unit_type = unit_type?; - let unit:Box = match unit_type { + let unit: Box = match unit_type { UnitType::Transfer => { let mut transfer = units::transfer::Transfer::default(); transfer.decode(&unit_bytes); @@ -286,15 +301,14 @@ fn unpack_units(digest: &[u8]) -> Result>, String> { Ok(units) } - // @todo implement tests for encoding and decoding of tx. #[cfg(test)] mod tests { - use std::error::Error; - use more_asserts::assert_gt; + use super::*; use crate::curr_timestamp; use crate::units::transfer::Transfer; - use super::*; + use more_asserts::assert_gt; + use std::error::Error; #[test] fn test_encode_decode() -> Result<(), Box> { @@ -303,8 +317,8 @@ mod tests { let priority_fee = 75; let chain_id = 45205; let transfer = Transfer::new(); - let units : Vec> = vec![Box::new(transfer)]; - let mut digest: [u8; 32] = [0; 32]; + let units: Vec> = vec![Box::new(transfer)]; + let digest: [u8; 32] = [0; 32]; let id = Digest::from(digest.clone()); // TODO: the .encode call on next line gave error and said origin_msg needed to be mut? but why? // shouldn't encode be able to encode without changing the msg? @@ -319,7 +333,7 @@ mod tests { }; let encoded_bytes = origin_msg.encode(); assert_gt!(encoded_bytes.len(), 0); - let mut decoded_msg = Tx::decode(&encoded_bytes)?; + let decoded_msg = Tx::decode(&encoded_bytes)?; let origin_transfer = origin_msg.units[0] .as_ref() .as_any() @@ -346,4 +360,4 @@ mod tests { assert_eq!(origin_transfer.memo, decode_transfer.memo); Ok(()) } -} \ No newline at end of file +} diff --git a/types/src/units/mod.rs b/types/src/units/mod.rs index 6b3f5403..faef4436 100644 --- a/types/src/units/mod.rs +++ b/types/src/units/mod.rs @@ -1,2 +1,2 @@ pub mod msg; -pub mod transfer; \ No newline at end of file +pub mod transfer; diff --git a/types/src/units/msg.rs b/types/src/units/msg.rs index 63db0d77..107c93cf 100644 --- a/types/src/units/msg.rs +++ b/types/src/units/msg.rs @@ -1,5 +1,9 @@ +use crate::{ + address::Address, + state::State, + tx::{Unit, UnitContext, UnitType}, +}; use std::any::Any; -use crate::{address::Address, tx::{Unit, UnitType, UnitContext}, state::State}; #[derive(Clone, Debug)] pub struct SequencerMsg { @@ -24,7 +28,7 @@ impl Unit for SequencerMsg { } fn encode(&self) -> Vec { - let mut bytes:Vec = Vec::new(); + let mut bytes: Vec = Vec::new(); // data length is 8 bytes. let data_len = self.data.len() as u64; // chain id length is 8 bytes.n store chain id. @@ -72,14 +76,14 @@ impl Default for SequencerMsg { #[cfg(test)] mod tests { - use std::error::Error; - use more_asserts::assert_gt; use super::*; + use more_asserts::assert_gt; + use std::error::Error; #[test] fn test_encode_decode() -> Result<(), Box> { let chain_id = 4502; - let data = vec!(0xDE, 0xAD, 0xBE, 0xEF); + let data = vec![0xDE, 0xAD, 0xBE, 0xEF]; let from_address = Address::create_random_address(); let relayer_id = 1; let origin_msg = SequencerMsg { @@ -97,4 +101,4 @@ mod tests { assert_eq!(origin_msg.from_address, decoded_msg.from_address); Ok(()) } -} \ No newline at end of file +} diff --git a/types/src/units/transfer.rs b/types/src/units/transfer.rs index d00e27d6..b646cf54 100644 --- a/types/src/units/transfer.rs +++ b/types/src/units/transfer.rs @@ -1,7 +1,7 @@ -use std::any::Any; use crate::address::Address; use crate::state::State; -use crate::tx::{Unit, UnitType, UnitContext}; +use crate::tx::{Unit, UnitContext, UnitType}; +use std::any::Any; const MAX_MEMO_SIZE: usize = 256; @@ -50,7 +50,7 @@ impl Unit for Transfer { if memo_len > 0 { bytes.extend_from_slice(&self.memo); } - + bytes } @@ -89,19 +89,18 @@ impl Default for Transfer { } } - #[cfg(test)] mod tests { - use std::error::Error; - use more_asserts::assert_gt; use super::*; + use more_asserts::assert_gt; + use std::error::Error; #[test] fn test_encode_decode() -> Result<(), Box> { let from_address = Address::create_random_address(); let to_address = Address::create_random_address(); let value = 5; - let memo = vec!(0xDE, 0xAD, 0xBE, 0xEF); + let memo = vec![0xDE, 0xAD, 0xBE, 0xEF]; let relayer_id = 1; let origin_msg = Transfer { from_address, @@ -119,4 +118,4 @@ mod tests { assert_eq!(origin_msg.memo, decoded_msg.memo); Ok(()) } -} \ No newline at end of file +} diff --git a/types/src/wallet.rs b/types/src/wallet.rs index f61b46a3..6cd03d2e 100644 --- a/types/src/wallet.rs +++ b/types/src/wallet.rs @@ -1,5 +1,5 @@ use crate::address::Address; -use crate::{PrivateKey, PublicKey,Signature, TX_NAMESPACE}; +use crate::{PrivateKey, PublicKey, Signature, TX_NAMESPACE}; use commonware_cryptography::ed25519::Ed25519; use commonware_cryptography::Scheme; use rand::{CryptoRng, Rng}; @@ -18,7 +18,7 @@ pub trait Auth { // returns the account address of the signer. fn address(&self) -> Address; // verifys the signature. - fn verify(&self, data: &[u8], signature: &[u8]) -> bool; + fn verify(&self, data: &[u8], signature: &[u8]) -> bool; // batch verify signatures. returns false if batch verification fails. fn batch_verify(&self, data: &[u8], signatures: Vec<&[u8]>) -> bool; } @@ -32,29 +32,30 @@ pub struct Wallet { // Account Address, is derived from the public key. address: Address, // Signer - signer: Ed25519, + signer: Ed25519, } // wallet generation, management, and signing should be functions of the wallet. pub trait WalletMethods { // create a new wallet using the given randomness. - fn generate(r: &mut R) -> Self; + fn generate(r: &mut R) -> Self; // load signer from bytes rep of a private key and initialize the wallet. fn load(&self, priv_key: &[u8]) -> Self; // sign the given arbitary data with the private key of the wallet. fn sign(&mut self, data: &[u8]) -> Vec; // verify the signature of the given data with the public key of the wallet. - fn verify(&self,data: &[u8], signature: &[u8]) -> Result; + fn verify(&self, data: &[u8], signature: &[u8]) + -> Result; // return corresponding wallet's address. fn address(&self) -> Address; // return corresponding wallet's public key. fn public_key(&self) -> PublicKey; - // return corresponding wallet's private key. - fn private_key(&self) -> Vec; + // return corresponding wallet's private key. + fn private_key(&self) -> Vec; // store the private key at the given path. fn store_private_key(&self, path: &str) -> Result<(), Error>; // @todo remove this? - fn init_address(&mut self); + fn init_address(&mut self); } impl WalletMethods for Wallet { @@ -62,10 +63,10 @@ impl WalletMethods for Wallet { let signer = Ed25519::new(r); let pub_key = signer.public_key(); let address = Address::from_pub_key(&pub_key); - Self { - priv_key: signer.private_key(), - pub_key: signer.public_key(), - address, + Self { + priv_key: signer.private_key(), + pub_key: signer.public_key(), + address, signer, } } @@ -85,15 +86,24 @@ impl WalletMethods for Wallet { self.signer.sign(Some(TX_NAMESPACE), data).as_ref().to_vec() } - fn verify(&self, data: &[u8], signature: &[u8]) -> Result { + fn verify( + &self, + data: &[u8], + signature: &[u8], + ) -> Result { let signature = Signature::try_from(signature); if let Err(e) = signature { return Err(e); } - + let signature = signature.unwrap(); let pub_key = self.signer.public_key(); - Ok(Ed25519::verify(Some(TX_NAMESPACE), data, &pub_key, &signature)) + Ok(Ed25519::verify( + Some(TX_NAMESPACE), + data, + &pub_key, + &signature, + )) } fn address(&self) -> Address { @@ -115,4 +125,4 @@ impl WalletMethods for Wallet { fn init_address(&mut self) { todo!() } -} \ No newline at end of file +} diff --git a/vm/src/lib.rs b/vm/src/lib.rs index 77e9423e..e44013b9 100644 --- a/vm/src/lib.rs +++ b/vm/src/lib.rs @@ -1 +1 @@ -pub mod vm; \ No newline at end of file +pub mod vm; diff --git a/vm/src/vm.rs b/vm/src/vm.rs index 7695ccbd..7219bb58 100644 --- a/vm/src/vm.rs +++ b/vm/src/vm.rs @@ -1,9 +1,11 @@ +use std::collections::HashMap; use std::error::Error; use std::sync::{Arc, Mutex}; -use std::collections::HashMap; -use alto_storage::transactional_db::InMemoryCachingTransactionalDb; -use alto_storage::{transactional_db::{TransactionalDb, Key, Op}, database::Database}; +use alto_storage::{ + database::Database, + transactional_db::{Key, Op}, +}; use alto_types::tx::Tx; pub struct VM { @@ -41,7 +43,4 @@ impl VM { } #[cfg(test)] -mod tests { - use super::*; - -} +mod tests {} From e0d027c7e1a78881622cc0563ab44fbf3250bd37 Mon Sep 17 00:00:00 2001 From: manojkgorle Date: Wed, 2 Apr 2025 20:39:54 +0530 Subject: [PATCH 35/43] mod unit apply --- storage/src/state_db.rs | 30 +++++++++-------- types/src/lib.rs | 2 +- types/src/signed_tx.rs | 2 ++ types/src/state.rs | 11 ------- types/src/state_view.rs | 10 ++++++ types/src/tx.rs | 60 ++++++++++++++++++++++------------ types/src/units/msg.rs | 12 +++---- types/src/units/transfer.rs | 64 +++++++++++++++++++++++++------------ vm/src/vm.rs | 34 +++++++++++++++++--- 9 files changed, 148 insertions(+), 77 deletions(-) delete mode 100644 types/src/state.rs create mode 100644 types/src/state_view.rs diff --git a/storage/src/state_db.rs b/storage/src/state_db.rs index 65bcea40..68831975 100644 --- a/storage/src/state_db.rs +++ b/storage/src/state_db.rs @@ -4,23 +4,20 @@ use alto_types::address::Address; use bytes::Bytes; use commonware_codec::{Codec, ReadBuffer, WriteBuffer}; use std::error::Error; - +use alto_types::state_view::StateView; const ACCOUNTS_PREFIX: u8 = 0x0; const DB_WRITE_BUFFER_CAPACITY: usize = 500; -// StateDb is a wrapper around TransactionalDb that provides StateViews for block execution. -// StateDb simplifies the interactions with state by providing methods that abstract away the underlying database operations. -// It allows for easy retrieval and modification of account states, such as balances. +// @todo rename StateDb to StateView. +/// StateDb is a wrapper around TransactionalDb that provides StateViews for block execution. +/// StateDb simplifies the interactions with state by providing methods that abstract away the underlying database operations. +/// It allows for easy retrieval and modification of account states, such as balances. pub struct StateDb<'a> { db: &'a mut dyn TransactionalDb, } -impl<'a> StateDb<'a> { - pub fn new(db: &'a mut dyn TransactionalDb) -> Self { - StateDb { db } - } - - pub fn get_account(&mut self, address: &Address) -> Result, Box> { +impl<'a> StateView for StateDb<'a> { + fn get_account(&mut self, address: &Address) -> Result, Box> { let key = Self::key_accounts(address); self.db.get(&key).and_then(|v| { if let Some(value) = v { @@ -35,14 +32,14 @@ impl<'a> StateDb<'a> { }) } - pub fn set_account(&mut self, acc: &Account) -> Result<(), Box> { + fn set_account(&mut self, acc: &Account) -> Result<(), Box> { let key = Self::key_accounts(&acc.address); let mut write_buf = WriteBuffer::new(DB_WRITE_BUFFER_CAPACITY); acc.write(&mut write_buf); self.db.insert(&key, write_buf.as_ref().to_vec()) } - pub fn get_balance(&mut self, address: &Address) -> Option { + fn get_balance(&mut self, address: &Address) -> Option { match self.get_account(address) { Ok(Some(acc)) => Some(acc.balance), // return balance if account exists Ok(None) => Some(0), // return 0 if no account @@ -50,7 +47,7 @@ impl<'a> StateDb<'a> { } } - pub fn set_balance(&mut self, address: &Address, amt: Balance) -> bool { + fn set_balance(&mut self, address: &Address, amt: Balance) -> bool { match self.get_account(address) { Ok(Some(mut acc)) => { acc.balance = amt; @@ -59,10 +56,17 @@ impl<'a> StateDb<'a> { _ => false, } } +} + +impl<'a> StateDb <'a> { + pub fn new(db: &'a mut dyn TransactionalDb) -> Self { + StateDb { db } + } fn key_accounts(addr: &Address) -> [u8; 33] { Self::make_multi_key(ACCOUNTS_PREFIX, addr.as_slice()) } + fn make_multi_key(prefix: u8, sub_id: &[u8]) -> [u8; 33] { assert_eq!(sub_id.len(), 32, "Sub_id must be exactly 32 bytes"); diff --git a/types/src/lib.rs b/types/src/lib.rs index 741a1a24..afa9549f 100644 --- a/types/src/lib.rs +++ b/types/src/lib.rs @@ -11,11 +11,11 @@ pub use consensus::{leader_index, Finalization, Kind, Notarization, Nullificatio pub mod account; pub mod address; pub mod signed_tx; -pub mod state; pub mod tx; pub mod units; pub mod wallet; pub mod wasm; +pub mod state_view; use rand::rngs::OsRng; diff --git a/types/src/signed_tx.rs b/types/src/signed_tx.rs index 610d1f95..35d9aeca 100644 --- a/types/src/signed_tx.rs +++ b/types/src/signed_tx.rs @@ -143,6 +143,7 @@ pub fn unpack_signed_txs(bytes: Vec) -> Vec { #[cfg(test)] mod tests { + use std::default; use std::error::Error; use super::*; @@ -173,6 +174,7 @@ mod tests { units: units.clone(), id, digest: digest.to_vec(), + actor: Address::empty(), }; let mut origin_msg = SignedTx { tx, diff --git a/types/src/state.rs b/types/src/state.rs deleted file mode 100644 index 93a4e2ff..00000000 --- a/types/src/state.rs +++ /dev/null @@ -1,11 +0,0 @@ -use std::error::Error; - -pub type UnitKey<'a> = &'a [u8; 33]; // 1st byte denotes the type of key. 0b for account key, 1b for others. -pub trait State { - fn get(&self, key: UnitKey) -> Result>, Box>; - fn get_multi_key(&self, key: Vec) -> Result>, Box>; - fn update(&mut self, key: UnitKey, value: Vec) -> Result<(), Box>; - fn delete(&mut self, key: UnitKey) -> Result<(), Box>; - fn commit(&mut self) -> Result<(), Box>; - fn rollback(&mut self) -> Result<(), Box>; -} diff --git a/types/src/state_view.rs b/types/src/state_view.rs new file mode 100644 index 00000000..7b3c13e0 --- /dev/null +++ b/types/src/state_view.rs @@ -0,0 +1,10 @@ +use crate::address::Address; +use crate::account::{Account, Balance}; +use std::error::Error; + +pub trait StateView { + fn get_account(&mut self, address: &Address) -> Result, Box>; + fn set_account(&mut self, acc: &Account) -> Result<(), Box>; + fn get_balance(&mut self, address: &Address) -> Option; + fn set_balance(&mut self, address: &Address, amt: Balance) -> bool; +} \ No newline at end of file diff --git a/types/src/tx.rs b/types/src/tx.rs index 26e1e440..bcdf49ca 100644 --- a/types/src/tx.rs +++ b/types/src/tx.rs @@ -4,7 +4,7 @@ use std::any::Any; use crate::address::Address; use crate::signed_tx::SignedTx; -use crate::state::State; +use crate::state_view::StateView; use crate::units; use crate::wallet::Wallet; use commonware_utils::SystemTimeExt; @@ -59,7 +59,7 @@ pub trait Unit: UnitClone + Send + Sync + std::fmt::Debug { fn apply( &self, context: &UnitContext, - state: &mut Box, + state: &mut Box<&mut dyn StateView>, ) -> Result>, Box>; fn as_any(&self) -> &dyn Any; @@ -73,33 +73,35 @@ impl Clone for Box { #[derive(Clone, Debug)] pub struct Tx { - // timestamp of the tx creation. set by the user. - // will be verified if the tx is in the valid window once received by validators. - // if the timestamp is not in the valid window, the tx will be rejected. - // if tx is in a valid window it is added to mempool. - // timestamp is used to prevent replay attacks. and counter infinite spam attacks as Tx does not have nonce. + /// timestamp of the tx creation. set by the user. + /// will be verified if the tx is in the valid window once received by validators. + /// if the timestamp is not in the valid window, the tx will be rejected. + /// if tx is in a valid window it is added to mempool. + /// timestamp is used to prevent replay attacks. and counter infinite spam attacks as Tx does not have nonce. pub timestamp: u64, - // max fee is the maximum fee the user is willing to pay for the tx. + /// max fee is the maximum fee the user is willing to pay for the tx. pub max_fee: u64, - // priority fee is the fee the user is willing to pay for the tx to be included in the next block. + /// priority fee is the fee the user is willing to pay for the tx to be included in the next block. pub priority_fee: u64, - // chain id is the id of the chain the tx is intended for. + /// chain id is the id of the chain the tx is intended for. pub chain_id: u64, - // units are fundamental unit of a tx. similar to actions. + /// units are fundamental unit of a tx. similar to actions. pub units: Vec>, - // id is the transaction id. It is the hash of digest. + /// id is the transaction id. It is the hash of digest. pub id: Digest, - // digest is encoded tx. + /// digest is encoded tx. pub digest: Vec, + /// address of the tx sender. wrap this in a better way. + pub actor: Address, } pub trait TxMethods: Sized { - // new is used to create a new instance of Tx with given units and chain id. + /// new is used to create a new instance of Tx with given units and chain id. fn new(units: Vec>, chain_id: u64) -> Self; - // set_fee is used to set the max fee and priority fee of the tx. + /// set_fee is used to set the max fee and priority fee of the tx. fn set_fee(&mut self, max_fee: u64, priority_fee: u64); - // sign is used to sign the tx with the given wallet. + /// sign is used to sign the tx with the given wallet. fn sign(&mut self, wallet: Wallet) -> SignedTx; fn from( timestamp: u64, @@ -107,17 +109,22 @@ pub trait TxMethods: Sized { priority_fee: u64, max_fee: u64, chain_id: u64, + actor: Address, ) -> Self; - // returns tx id. + /// returns tx id. fn id(&mut self) -> Digest; - // returns digest of the tx. + /// returns digest of the tx. fn digest(&self) -> Vec; - // encodes the tx, writes to digest and returns the digest. - // ensure all fields are properly set before calling this function. + /// encodes the tx, writes to digest and returns the digest. + /// ensure all fields are properly set before calling this function. fn encode(&mut self) -> Vec; fn decode(bytes: &[u8]) -> Result; + + fn set_actor(&mut self, actor: Address); + + fn actor(&self) -> Address; } impl Default for Tx { @@ -130,6 +137,7 @@ impl Default for Tx { chain_id: 19517, id: [0; 32].into(), digest: vec![], + actor: Address::empty(), } } } @@ -160,6 +168,7 @@ impl TxMethods for Tx { priority_fee: u64, max_fee: u64, chain_id: u64, + actor: Address, ) -> Self { let mut tx = Self::default(); tx.timestamp = timestamp; @@ -167,6 +176,7 @@ impl TxMethods for Tx { tx.max_fee = max_fee; tx.priority_fee = priority_fee; tx.chain_id = chain_id; + tx.actor = actor; tx.encode(); tx } @@ -234,6 +244,14 @@ impl TxMethods for Tx { // return transaction. Ok(tx) } + + fn set_actor(&mut self, actor: Address) { + self.actor = actor; + } + + fn actor(&self) -> Address { + self.actor.clone() + } } fn unpack_units(digest: &[u8]) -> Result>, String> { @@ -329,6 +347,7 @@ mod tests { chain_id, units: units.clone(), id, + actor: Address::empty(), digest: digest.to_vec(), }; let encoded_bytes = origin_msg.encode(); @@ -355,7 +374,6 @@ mod tests { // units assert_eq!(origin_transfer.to_address, decode_transfer.to_address); - assert_eq!(origin_transfer.from_address, decode_transfer.from_address); assert_eq!(origin_transfer.value, decode_transfer.value); assert_eq!(origin_transfer.memo, decode_transfer.memo); Ok(()) diff --git a/types/src/units/msg.rs b/types/src/units/msg.rs index 107c93cf..c6f3fee9 100644 --- a/types/src/units/msg.rs +++ b/types/src/units/msg.rs @@ -1,10 +1,10 @@ use crate::{ - address::Address, - state::State, - tx::{Unit, UnitContext, UnitType}, + address::Address, state_view::StateView, tx::{Unit, UnitContext, UnitType} }; use std::any::Any; +// @todo couple SequencerMsg with DA. +// and skip execution no-op. #[derive(Clone, Debug)] pub struct SequencerMsg { pub chain_id: u64, @@ -53,10 +53,10 @@ impl Unit for SequencerMsg { fn apply( &self, - context: &UnitContext, - state: &mut Box, + _: &UnitContext, + _: &mut Box<&mut dyn StateView>, ) -> Result>, Box> { - todo!() + Ok(None) } fn as_any(&self) -> &dyn Any { diff --git a/types/src/units/transfer.rs b/types/src/units/transfer.rs index b646cf54..d2228097 100644 --- a/types/src/units/transfer.rs +++ b/types/src/units/transfer.rs @@ -1,13 +1,12 @@ use crate::address::Address; -use crate::state::State; +use crate::state_view::StateView; use crate::tx::{Unit, UnitContext, UnitType}; -use std::any::Any; +use std::{any::Any, error::Error, fmt::Display}; const MAX_MEMO_SIZE: usize = 256; #[derive(Debug, Clone)] pub struct Transfer { - pub from_address: Address, pub to_address: Address, pub value: u64, pub memo: Vec, @@ -16,7 +15,6 @@ pub struct Transfer { impl Transfer { pub fn new() -> Transfer { Self { - from_address: Address::empty(), to_address: Address::empty(), value: 0, memo: Vec::new(), @@ -26,15 +24,25 @@ impl Transfer { #[derive(Debug)] pub enum TransferError { - DuplicateAddress, - InvalidToAddress, - InvalidFromAddress, + SenderAccountNotFound, InsufficientFunds, - TooMuchFunds, InvalidMemoSize, StorageError, } +impl Display for TransferError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + TransferError::SenderAccountNotFound => write!(f, "Sender account not found"), + TransferError::InsufficientFunds => write!(f, "Insufficient funds"), + TransferError::InvalidMemoSize => write!(f, "Invalid memo size"), + TransferError::StorageError => write!(f, "Storage error"), + } + } +} + +impl Error for TransferError {} + impl Unit for Transfer { fn unit_type(&self) -> UnitType { UnitType::Transfer @@ -43,7 +51,6 @@ impl Unit for Transfer { fn encode(&self) -> Vec { let mut bytes = Vec::new(); let memo_len = self.memo.len() as u64; - bytes.extend_from_slice(self.from_address.as_slice()); bytes.extend_from_slice(self.to_address.as_slice()); bytes.extend(self.value.to_be_bytes()); bytes.extend(memo_len.to_be_bytes()); @@ -56,21 +63,41 @@ impl Unit for Transfer { // @todo introduce syntactic checks. fn decode(&mut self, bytes: &[u8]) { - self.from_address = Address::from_bytes(&bytes[0..32]).unwrap(); - self.to_address = Address::from_bytes(&bytes[32..64]).unwrap(); - self.value = u64::from_be_bytes(bytes[64..72].try_into().unwrap()); - let memo_len = u64::from_be_bytes(bytes[72..80].try_into().unwrap()); + self.to_address = Address::from_bytes(&bytes[0..32]).unwrap(); + self.value = u64::from_be_bytes(bytes[32..40].try_into().unwrap()); + let memo_len = u64::from_be_bytes(bytes[40..48].try_into().unwrap()); if memo_len > 0 { - self.memo = bytes[80..(80 + memo_len as usize)].to_vec(); + self.memo = bytes[48..(48 + memo_len as usize)].to_vec(); } } fn apply( &self, context: &UnitContext, - state: &mut Box, + state: &mut Box<&mut dyn StateView>, ) -> Result>, Box> { - todo!() + + if self.memo.len() > MAX_MEMO_SIZE { + return Err(TransferError::InvalidMemoSize.into()); + } + + if let Some(bal) = state.get_balance(&context.sender) { + + if bal < self.value { + return Err(TransferError::InsufficientFunds.into()); + } + + let receiver_bal = state.get_balance(&self.to_address).unwrap_or(0); + + if !state.set_balance(&context.sender, bal - self.value) || !state.set_balance(&self.to_address, receiver_bal + self.value){ + return Err(TransferError::StorageError.into()); + } + + } else { + return Err(TransferError::SenderAccountNotFound.into()); + } + + Ok(None) } fn as_any(&self) -> &dyn Any { @@ -81,7 +108,6 @@ impl Unit for Transfer { impl Default for Transfer { fn default() -> Self { Self { - from_address: Address::empty(), to_address: Address::empty(), value: 0, memo: vec![], @@ -97,13 +123,10 @@ mod tests { #[test] fn test_encode_decode() -> Result<(), Box> { - let from_address = Address::create_random_address(); let to_address = Address::create_random_address(); let value = 5; let memo = vec![0xDE, 0xAD, 0xBE, 0xEF]; - let relayer_id = 1; let origin_msg = Transfer { - from_address, to_address, value, memo, @@ -112,7 +135,6 @@ mod tests { assert_gt!(encoded_bytes.len(), 0); let mut decoded_msg = Transfer::new(); decoded_msg.decode(&encoded_bytes); - assert_eq!(origin_msg.from_address, decoded_msg.from_address); assert_eq!(origin_msg.to_address, decoded_msg.to_address); assert_eq!(origin_msg.value, decoded_msg.value); assert_eq!(origin_msg.memo, decoded_msg.memo); diff --git a/vm/src/vm.rs b/vm/src/vm.rs index 7219bb58..756ae8bd 100644 --- a/vm/src/vm.rs +++ b/vm/src/vm.rs @@ -2,14 +2,18 @@ use std::collections::HashMap; use std::error::Error; use std::sync::{Arc, Mutex}; +use alto_storage::state_db::StateDb; use alto_storage::{ database::Database, - transactional_db::{Key, Op}, + transactional_db::{Key, Op, InMemoryCachingTransactionalDb}, }; -use alto_types::tx::Tx; +use alto_types::tx::{Tx, Unit, UnitContext, TxMethods}; +use alto_types::state_view::StateView; pub struct VM { pub block_number: u64, + pub timestamp: u64, + pub chain_id: u64, pub state_cache: Arc>>, pub unfinalized_state: Arc>>, pub state_db: Arc>, @@ -18,12 +22,16 @@ pub struct VM { impl VM { pub fn new( block_number: u64, + timestamp: u64, + chain_id: u64, state_cache: Arc>>, unfinalized_state: Arc>>, state_db: Arc>, ) -> Self { Self { block_number, + timestamp, + chain_id, state_cache, unfinalized_state, state_db, @@ -32,12 +40,30 @@ impl VM { // applies new set of txs on the given state. pub fn apply(&mut self, txs: Vec) -> Result<(), Box> { - // let mut in_mem_db = InMemoryCachingTransactionalDb::new(Arc::clone(self.state_cache), Arc::clone(self.unfinalized_state), self.state_db); + let mut in_mem_db = InMemoryCachingTransactionalDb::new(Arc::clone(&self.state_cache), Arc::clone(&self.unfinalized_state), Arc::clone(&self.state_db)); + let mut state_view = StateDb::new(&mut in_mem_db); + for tx in txs { + self.apply_tx(tx.clone(), &mut state_view)?; + } Ok(()) } // applies a single tx on the given state. - fn apply_tx(&mut self, tx: Vec) -> Result<(), Box> { + fn apply_tx(&mut self, tx: Tx, state_view: &mut T) -> Result<(), Box> { + let tx_context = UnitContext{ + timestamp: self.timestamp, + chain_id: self.chain_id, + sender: tx.actor(), + }; + let mut sv_boxed:Box<&mut dyn StateView> = Box::new(state_view); + // apply units one by one. + // stop and revert if any unit fails. + // rollback the state changes if unit fails. + for unit in tx.units { + // success status. + // revertion handling. + unit.apply(&tx_context, &mut sv_boxed)?; + } Ok(()) } } From d5bddb6438fb390707c6631e3aef9015594eab7b Mon Sep 17 00:00:00 2001 From: manojkgorle Date: Wed, 2 Apr 2025 22:18:32 +0530 Subject: [PATCH 36/43] vm works --- Cargo.lock | 3 + storage/src/hashmap_db.rs | 13 +--- storage/src/state_db.rs | 35 +++++---- types/src/lib.rs | 3 +- types/src/null_error.rs | 12 +++ types/src/state_view.rs | 4 +- types/src/tx.rs | 2 +- types/src/units/msg.rs | 6 +- types/src/units/transfer.rs | 10 +-- vm/Cargo.toml | 3 + vm/src/capture_logs.rs | 47 ++++++++++++ vm/src/lib.rs | 1 + vm/src/vm.rs | 141 +++++++++++++++++++++++++++++++----- 13 files changed, 228 insertions(+), 52 deletions(-) create mode 100644 types/src/null_error.rs create mode 100644 vm/src/capture_logs.rs diff --git a/Cargo.lock b/Cargo.lock index 8e89a203..8d326fb0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -140,6 +140,9 @@ version = "0.1.0" dependencies = [ "alto-storage", "alto-types", + "commonware-codec", + "tracing", + "tracing-subscriber", ] [[package]] diff --git a/storage/src/hashmap_db.rs b/storage/src/hashmap_db.rs index 9fc9582a..f5e7d8e2 100644 --- a/storage/src/hashmap_db.rs +++ b/storage/src/hashmap_db.rs @@ -3,7 +3,7 @@ use std::collections::HashMap; use std::error::Error; pub struct HashmapDatabase { - data: HashMap, + data: HashMap, Vec>, } impl Default for HashmapDatabase { @@ -22,23 +22,18 @@ impl HashmapDatabase { impl Database for HashmapDatabase { fn put(&mut self, key: &[u8], value: &[u8]) -> Result<(), Box> { - let key_value: String = String::from_utf8(key.into())?; - let str_value: String = String::from_utf8(value.into())?; - - self.data.insert(key_value, str_value); + self.data.insert(key.into(), value.into()); Ok(()) } fn get(&mut self, key: &[u8]) -> Result>, Box> { - let str_key: String = String::from_utf8(key.into()).unwrap(); self.data - .get(&str_key) + .get(key) .map_or(Ok(None), |v| Ok(Some(v.clone().into()))) } fn delete(&mut self, key: &[u8]) -> Result<(), Box> { - let key_value: String = String::from_utf8(key.into())?; - self.data.remove(&key_value); + self.data.remove(key); Ok(()) } } diff --git a/storage/src/state_db.rs b/storage/src/state_db.rs index 68831975..7e7806b0 100644 --- a/storage/src/state_db.rs +++ b/storage/src/state_db.rs @@ -1,22 +1,22 @@ use crate::transactional_db::TransactionalDb; use alto_types::account::{Account, Balance}; use alto_types::address::Address; +use alto_types::state_view::StateView; use bytes::Bytes; use commonware_codec::{Codec, ReadBuffer, WriteBuffer}; use std::error::Error; -use alto_types::state_view::StateView; const ACCOUNTS_PREFIX: u8 = 0x0; const DB_WRITE_BUFFER_CAPACITY: usize = 500; -// @todo rename StateDb to StateView. -/// StateDb is a wrapper around TransactionalDb that provides StateViews for block execution. -/// StateDb simplifies the interactions with state by providing methods that abstract away the underlying database operations. +// @todo rename StateViewDb to StateView. +/// StateViewDb is a wrapper around TransactionalDb that provides StateViews for block execution. +/// StateViewDb simplifies the interactions with state by providing methods that abstract away the underlying database operations. /// It allows for easy retrieval and modification of account states, such as balances. -pub struct StateDb<'a> { +pub struct StateViewDb<'a> { db: &'a mut dyn TransactionalDb, } -impl<'a> StateView for StateDb<'a> { +impl<'a> StateView for StateViewDb<'a> { fn get_account(&mut self, address: &Address) -> Result, Box> { let key = Self::key_accounts(address); self.db.get(&key).and_then(|v| { @@ -52,18 +52,25 @@ impl<'a> StateView for StateDb<'a> { Ok(Some(mut acc)) => { acc.balance = amt; self.set_account(&acc).is_ok() - } - _ => false, + }, + Err(e) => { + let acc = Account { + address: address.clone(), + balance: amt, + }; + self.set_account(&acc).is_ok() + }, + _ => false, } } } -impl<'a> StateDb <'a> { +impl<'a> StateViewDb<'a> { pub fn new(db: &'a mut dyn TransactionalDb) -> Self { - StateDb { db } + StateViewDb { db } } - fn key_accounts(addr: &Address) -> [u8; 33] { + pub fn key_accounts(addr: &Address) -> [u8; 33] { Self::make_multi_key(ACCOUNTS_PREFIX, addr.as_slice()) } @@ -100,12 +107,12 @@ mod tests { address: address.clone(), balance: 1000, }; - let mut state_db = StateDb::new(&mut in_mem); + let mut state_db = StateViewDb::new(&mut in_mem); state_db.set_account(&account).unwrap(); let _ = in_mem.commit_last_tx(); let _ = in_mem.commit(); assert_eq!(unfinalized.lock().unwrap().len(), 1); - let mut state_db2 = StateDb::new(&mut in_mem); + let mut state_db2 = StateViewDb::new(&mut in_mem); let retrieved = state_db2.get_account(&address).unwrap().unwrap(); assert_eq!(retrieved, account); } @@ -122,7 +129,7 @@ mod tests { let address = Address::create_random_address(); - let mut state_db = StateDb::new(&mut in_mem); + let mut state_db = StateViewDb::new(&mut in_mem); let _ = state_db.get_account(&address).unwrap(); } diff --git a/types/src/lib.rs b/types/src/lib.rs index afa9549f..3075bbce 100644 --- a/types/src/lib.rs +++ b/types/src/lib.rs @@ -11,11 +11,12 @@ pub use consensus::{leader_index, Finalization, Kind, Notarization, Nullificatio pub mod account; pub mod address; pub mod signed_tx; +pub mod state_view; +pub mod null_error; pub mod tx; pub mod units; pub mod wallet; pub mod wasm; -pub mod state_view; use rand::rngs::OsRng; diff --git a/types/src/null_error.rs b/types/src/null_error.rs new file mode 100644 index 00000000..129e84f6 --- /dev/null +++ b/types/src/null_error.rs @@ -0,0 +1,12 @@ +use std::{fmt::{Display, Formatter, Result}, error::Error}; + +#[derive(Debug)] +pub struct NullError; + +impl Display for NullError { + fn fmt(&self, f: &mut Formatter<'_>) -> Result { + write!(f, "NoError") + } +} + +impl Error for NullError {} \ No newline at end of file diff --git a/types/src/state_view.rs b/types/src/state_view.rs index 7b3c13e0..e61fed16 100644 --- a/types/src/state_view.rs +++ b/types/src/state_view.rs @@ -1,5 +1,5 @@ -use crate::address::Address; use crate::account::{Account, Balance}; +use crate::address::Address; use std::error::Error; pub trait StateView { @@ -7,4 +7,4 @@ pub trait StateView { fn set_account(&mut self, acc: &Account) -> Result<(), Box>; fn get_balance(&mut self, address: &Address) -> Option; fn set_balance(&mut self, address: &Address, amt: Balance) -> bool; -} \ No newline at end of file +} diff --git a/types/src/tx.rs b/types/src/tx.rs index bcdf49ca..0d5632e1 100644 --- a/types/src/tx.rs +++ b/types/src/tx.rs @@ -124,7 +124,7 @@ pub trait TxMethods: Sized { fn set_actor(&mut self, actor: Address); - fn actor(&self) -> Address; + fn actor(&self) -> Address; } impl Default for Tx { diff --git a/types/src/units/msg.rs b/types/src/units/msg.rs index c6f3fee9..b31083dc 100644 --- a/types/src/units/msg.rs +++ b/types/src/units/msg.rs @@ -1,9 +1,11 @@ use crate::{ - address::Address, state_view::StateView, tx::{Unit, UnitContext, UnitType} + address::Address, + state_view::StateView, + tx::{Unit, UnitContext, UnitType}, }; use std::any::Any; -// @todo couple SequencerMsg with DA. +// @todo couple SequencerMsg with DA. // and skip execution no-op. #[derive(Clone, Debug)] pub struct SequencerMsg { diff --git a/types/src/units/transfer.rs b/types/src/units/transfer.rs index d2228097..2bd61667 100644 --- a/types/src/units/transfer.rs +++ b/types/src/units/transfer.rs @@ -76,27 +76,25 @@ impl Unit for Transfer { context: &UnitContext, state: &mut Box<&mut dyn StateView>, ) -> Result>, Box> { - if self.memo.len() > MAX_MEMO_SIZE { return Err(TransferError::InvalidMemoSize.into()); } if let Some(bal) = state.get_balance(&context.sender) { - if bal < self.value { return Err(TransferError::InsufficientFunds.into()); } - let receiver_bal = state.get_balance(&self.to_address).unwrap_or(0); - if !state.set_balance(&context.sender, bal - self.value) || !state.set_balance(&self.to_address, receiver_bal + self.value){ + if !state.set_balance(&context.sender, bal - self.value) + || !state.set_balance(&self.to_address, receiver_bal + self.value) + { return Err(TransferError::StorageError.into()); } - } else { return Err(TransferError::SenderAccountNotFound.into()); } - + Ok(None) } diff --git a/vm/Cargo.toml b/vm/Cargo.toml index 838d16df..0884eda3 100644 --- a/vm/Cargo.toml +++ b/vm/Cargo.toml @@ -6,3 +6,6 @@ edition = "2021" [dependencies] alto-storage = { workspace = true } alto-types = { workspace = true } +tracing ={ workspace = true} +tracing-subscriber = { workspace = true } +commonware-codec = {workspace = true} \ No newline at end of file diff --git a/vm/src/capture_logs.rs b/vm/src/capture_logs.rs new file mode 100644 index 00000000..2204cec1 --- /dev/null +++ b/vm/src/capture_logs.rs @@ -0,0 +1,47 @@ +use tracing_subscriber::{fmt, layer::SubscriberExt, registry::Registry}; +use tracing::dispatcher::with_default; + +use std::io::{self, Write}; +use std::sync::{Arc, Mutex}; + +struct SharedWriter { + buffer: Arc>>, +} + +impl Write for SharedWriter { + fn write(&mut self, buf: &[u8]) -> io::Result { + self.buffer.lock().unwrap().extend_from_slice(buf); + Ok(buf.len()) + } + + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} + +/// Captures all logs emitted inside the given closure and returns them as a string +pub fn capture_logs(func: F) -> (R, String) +where + F: FnOnce() -> R, +{ + let buffer = Arc::new(Mutex::new(Vec::new())); + let writer = { + let buf = Arc::clone(&buffer); + move || SharedWriter { buffer: Arc::clone(&buf) } + }; + + let layer = fmt::layer() + .with_writer(writer) + .with_ansi(false) + .without_time(); + + let subscriber = Registry::default().with(layer); + + let dispatch = tracing::Dispatch::new(subscriber); + let result = with_default(&dispatch, || func()); + + let logs = buffer.lock().unwrap(); + let log_str = String::from_utf8(logs.clone()).unwrap_or_default(); + + (result, log_str) +} \ No newline at end of file diff --git a/vm/src/lib.rs b/vm/src/lib.rs index e44013b9..267a6df8 100644 --- a/vm/src/lib.rs +++ b/vm/src/lib.rs @@ -1 +1,2 @@ pub mod vm; +pub mod capture_logs; \ No newline at end of file diff --git a/vm/src/vm.rs b/vm/src/vm.rs index 756ae8bd..3548e64e 100644 --- a/vm/src/vm.rs +++ b/vm/src/vm.rs @@ -1,14 +1,17 @@ use std::collections::HashMap; use std::error::Error; use std::sync::{Arc, Mutex}; +use std::vec; -use alto_storage::state_db::StateDb; +use alto_storage::state_db::StateViewDb; +use alto_storage::transactional_db::TransactionalDb; use alto_storage::{ database::Database, - transactional_db::{Key, Op, InMemoryCachingTransactionalDb}, + transactional_db::{InMemoryCachingTransactionalDb, Key, Op}, }; -use alto_types::tx::{Tx, Unit, UnitContext, TxMethods}; use alto_types::state_view::StateView; +use alto_types::tx::{Tx, TxMethods, Unit, UnitContext}; +use alto_types::null_error::NullError; pub struct VM { pub block_number: u64, @@ -39,34 +42,138 @@ impl VM { } // applies new set of txs on the given state. - pub fn apply(&mut self, txs: Vec) -> Result<(), Box> { - let mut in_mem_db = InMemoryCachingTransactionalDb::new(Arc::clone(&self.state_cache), Arc::clone(&self.unfinalized_state), Arc::clone(&self.state_db)); - let mut state_view = StateDb::new(&mut in_mem_db); + pub fn apply(&mut self, txs: Vec) -> (Vec>>, Vec>) { + let mut in_mem_db = InMemoryCachingTransactionalDb::new( + Arc::clone(&self.state_cache), + Arc::clone(&self.unfinalized_state), + Arc::clone(&self.state_db), + ); + let mut outputs: Vec>> = Vec::new(); + let mut errors: Vec> = Vec::new(); for tx in txs { - self.apply_tx(tx.clone(), &mut state_view)?; + let mut state_view = StateViewDb::new(&mut in_mem_db); + let result = self.apply_tx(tx.clone(), &mut state_view); + match result { + Ok(output) => { + // tx executed successfully. + // commit the state changes made by the tx. + let _ = in_mem_db.commit_last_tx(); + // push the output of the tx to the outputs. + outputs.push(output); + // push null error + errors.push(Box::new(NullError)); + } + Err(e) => { + // tx execution failed. + // rollback the transaction. + let _ = in_mem_db.rollback_last_tx(); + // push empty vec to the outputs. + outputs.push(vec![]); + // push the error to the errors. + errors.push(e); + } + } } - Ok(()) + (outputs, errors) } // applies a single tx on the given state. - fn apply_tx(&mut self, tx: Tx, state_view: &mut T) -> Result<(), Box> { - let tx_context = UnitContext{ + fn apply_tx(&mut self, tx: Tx, state_view: &mut T) -> Result>, Box> { + let tx_context = UnitContext { timestamp: self.timestamp, chain_id: self.chain_id, sender: tx.actor(), }; - let mut sv_boxed:Box<&mut dyn StateView> = Box::new(state_view); + let mut sv_boxed: Box<&mut dyn StateView> = Box::new(state_view); + let mut outputs:Vec> = Vec::new(); // apply units one by one. // stop and revert if any unit fails. - // rollback the state changes if unit fails. for unit in tx.units { - // success status. - // revertion handling. - unit.apply(&tx_context, &mut sv_boxed)?; + let result = unit.apply(&tx_context, &mut sv_boxed); + match result { + Ok(output) => { + if let Some(output) = output { + outputs.push(output); + }else{ + // if output is None, unit execution does not return anything. + // push empty vec. + outputs.push(vec![]); + } + } + Err(e) => { + // return the error. + return Err(e); + } + } } - Ok(()) + Ok(outputs) } } #[cfg(test)] -mod tests {} +mod tests { + use alto_storage::database::Database; + use alto_storage::hashmap_db::HashmapDatabase; + use alto_storage::state_db::StateViewDb; + use alto_types::account::Account; + use alto_types::address::Address; + use alto_types::curr_timestamp; + use alto_types::units::msg::SequencerMsg; + use alto_types::units::transfer::Transfer; + use std::sync::{Arc,Mutex}; + use std::collections::HashMap; + use alto_storage::transactional_db::{Key, Op}; + use alto_types::tx::{Tx,TxMethods, Unit}; + use commonware_codec::{WriteBuffer, Codec}; + + use super::VM; + + const DB_WRITE_BUFFER_CAPACITY: usize = 500; + + #[test] + fn test_single_tx(){ + let state_db = Arc::new(Mutex::new(HashmapDatabase::new())); + let cache: Arc>> = Arc::new(Mutex::new(HashMap::new())); + let unfinalized: Arc>> = Arc::new(Mutex::new(HashMap::new())); + + let address = Address::create_random_address(); + let account = Account{ + address: address.clone(), + balance: 1000, + }; + + let key = StateViewDb::key_accounts(&account.address); + let mut write_buf = WriteBuffer::new(DB_WRITE_BUFFER_CAPACITY); + account.write(&mut write_buf); + state_db.lock().unwrap().put(&key, write_buf.as_ref()).unwrap(); + + let block_number = 10; + let timestamp = curr_timestamp(); + let chain_id = 1; + let mut vm = VM::new( + block_number, + timestamp, + chain_id, + cache, + unfinalized, + state_db, + ); + let tfer_unit = Transfer{ + to_address: Address::create_random_address(), + value: 100, + memo: vec![], + }; + let msg_unit = SequencerMsg{ + chain_id: 10, + data: vec![0,0,0,0], + from_address: Address::create_random_address(), + }; + let units: Vec> = vec![Box::new(tfer_unit), Box::new(msg_unit)]; + let tx = ::from(timestamp, units, 10, 5, 1, address); + let (outputs, errors) = vm.apply(vec![tx]); + assert_eq!(outputs.len(), 1); + assert_eq!(outputs[0].len(), 2); + assert_eq!(errors.len(), 1); + assert_eq!(errors[0].to_string(), "NoError"); + } +} From 9ef58d3fdabb22ce537d6a8ebbb3814c61097d1b Mon Sep 17 00:00:00 2001 From: manojkgorle Date: Wed, 2 Apr 2025 22:20:29 +0530 Subject: [PATCH 37/43] cargo fmt --- storage/src/state_db.rs | 15 +++++++++------ types/src/lib.rs | 2 +- types/src/null_error.rs | 7 +++++-- vm/src/capture_logs.rs | 8 +++++--- vm/src/lib.rs | 2 +- vm/src/vm.rs | 38 +++++++++++++++++++++++--------------- 6 files changed, 44 insertions(+), 28 deletions(-) diff --git a/storage/src/state_db.rs b/storage/src/state_db.rs index 7e7806b0..12059d62 100644 --- a/storage/src/state_db.rs +++ b/storage/src/state_db.rs @@ -41,9 +41,12 @@ impl<'a> StateView for StateViewDb<'a> { fn get_balance(&mut self, address: &Address) -> Option { match self.get_account(address) { - Ok(Some(acc)) => Some(acc.balance), // return balance if account exists - Ok(None) => Some(0), // return 0 if no account - Err(_) => None, // return none if an err occurred + // return balance if account exists + Ok(Some(acc)) => Some(acc.balance), + // return 0 if no account + Ok(None) => Some(0), + // return none if an err occurred + Err(_) => None, } } @@ -52,15 +55,15 @@ impl<'a> StateView for StateViewDb<'a> { Ok(Some(mut acc)) => { acc.balance = amt; self.set_account(&acc).is_ok() - }, + } Err(e) => { let acc = Account { address: address.clone(), balance: amt, }; self.set_account(&acc).is_ok() - }, - _ => false, + } + _ => false, } } } diff --git a/types/src/lib.rs b/types/src/lib.rs index 3075bbce..fa7ae9de 100644 --- a/types/src/lib.rs +++ b/types/src/lib.rs @@ -10,9 +10,9 @@ mod consensus; pub use consensus::{leader_index, Finalization, Kind, Notarization, Nullification, Seed}; pub mod account; pub mod address; +pub mod null_error; pub mod signed_tx; pub mod state_view; -pub mod null_error; pub mod tx; pub mod units; pub mod wallet; diff --git a/types/src/null_error.rs b/types/src/null_error.rs index 129e84f6..15ed0238 100644 --- a/types/src/null_error.rs +++ b/types/src/null_error.rs @@ -1,4 +1,7 @@ -use std::{fmt::{Display, Formatter, Result}, error::Error}; +use std::{ + error::Error, + fmt::{Display, Formatter, Result}, +}; #[derive(Debug)] pub struct NullError; @@ -9,4 +12,4 @@ impl Display for NullError { } } -impl Error for NullError {} \ No newline at end of file +impl Error for NullError {} diff --git a/vm/src/capture_logs.rs b/vm/src/capture_logs.rs index 2204cec1..bcbabdda 100644 --- a/vm/src/capture_logs.rs +++ b/vm/src/capture_logs.rs @@ -1,5 +1,5 @@ -use tracing_subscriber::{fmt, layer::SubscriberExt, registry::Registry}; use tracing::dispatcher::with_default; +use tracing_subscriber::{fmt, layer::SubscriberExt, registry::Registry}; use std::io::{self, Write}; use std::sync::{Arc, Mutex}; @@ -27,7 +27,9 @@ where let buffer = Arc::new(Mutex::new(Vec::new())); let writer = { let buf = Arc::clone(&buffer); - move || SharedWriter { buffer: Arc::clone(&buf) } + move || SharedWriter { + buffer: Arc::clone(&buf), + } }; let layer = fmt::layer() @@ -44,4 +46,4 @@ where let log_str = String::from_utf8(logs.clone()).unwrap_or_default(); (result, log_str) -} \ No newline at end of file +} diff --git a/vm/src/lib.rs b/vm/src/lib.rs index 267a6df8..6264646b 100644 --- a/vm/src/lib.rs +++ b/vm/src/lib.rs @@ -1,2 +1,2 @@ +pub mod capture_logs; pub mod vm; -pub mod capture_logs; \ No newline at end of file diff --git a/vm/src/vm.rs b/vm/src/vm.rs index 3548e64e..248bf9b0 100644 --- a/vm/src/vm.rs +++ b/vm/src/vm.rs @@ -9,9 +9,9 @@ use alto_storage::{ database::Database, transactional_db::{InMemoryCachingTransactionalDb, Key, Op}, }; +use alto_types::null_error::NullError; use alto_types::state_view::StateView; use alto_types::tx::{Tx, TxMethods, Unit, UnitContext}; -use alto_types::null_error::NullError; pub struct VM { pub block_number: u64, @@ -55,7 +55,7 @@ impl VM { let result = self.apply_tx(tx.clone(), &mut state_view); match result { Ok(output) => { - // tx executed successfully. + // tx executed successfully. // commit the state changes made by the tx. let _ = in_mem_db.commit_last_tx(); // push the output of the tx to the outputs. @@ -78,14 +78,18 @@ impl VM { } // applies a single tx on the given state. - fn apply_tx(&mut self, tx: Tx, state_view: &mut T) -> Result>, Box> { + fn apply_tx( + &mut self, + tx: Tx, + state_view: &mut T, + ) -> Result>, Box> { let tx_context = UnitContext { timestamp: self.timestamp, chain_id: self.chain_id, sender: tx.actor(), }; let mut sv_boxed: Box<&mut dyn StateView> = Box::new(state_view); - let mut outputs:Vec> = Vec::new(); + let mut outputs: Vec> = Vec::new(); // apply units one by one. // stop and revert if any unit fails. for unit in tx.units { @@ -94,7 +98,7 @@ impl VM { Ok(output) => { if let Some(output) = output { outputs.push(output); - }else{ + } else { // if output is None, unit execution does not return anything. // push empty vec. outputs.push(vec![]); @@ -115,29 +119,29 @@ mod tests { use alto_storage::database::Database; use alto_storage::hashmap_db::HashmapDatabase; use alto_storage::state_db::StateViewDb; + use alto_storage::transactional_db::{Key, Op}; use alto_types::account::Account; use alto_types::address::Address; use alto_types::curr_timestamp; + use alto_types::tx::{Tx, TxMethods, Unit}; use alto_types::units::msg::SequencerMsg; use alto_types::units::transfer::Transfer; - use std::sync::{Arc,Mutex}; + use commonware_codec::{Codec, WriteBuffer}; use std::collections::HashMap; - use alto_storage::transactional_db::{Key, Op}; - use alto_types::tx::{Tx,TxMethods, Unit}; - use commonware_codec::{WriteBuffer, Codec}; + use std::sync::{Arc, Mutex}; use super::VM; const DB_WRITE_BUFFER_CAPACITY: usize = 500; #[test] - fn test_single_tx(){ + fn test_single_tx() { let state_db = Arc::new(Mutex::new(HashmapDatabase::new())); let cache: Arc>> = Arc::new(Mutex::new(HashMap::new())); let unfinalized: Arc>> = Arc::new(Mutex::new(HashMap::new())); let address = Address::create_random_address(); - let account = Account{ + let account = Account { address: address.clone(), balance: 1000, }; @@ -145,7 +149,11 @@ mod tests { let key = StateViewDb::key_accounts(&account.address); let mut write_buf = WriteBuffer::new(DB_WRITE_BUFFER_CAPACITY); account.write(&mut write_buf); - state_db.lock().unwrap().put(&key, write_buf.as_ref()).unwrap(); + state_db + .lock() + .unwrap() + .put(&key, write_buf.as_ref()) + .unwrap(); let block_number = 10; let timestamp = curr_timestamp(); @@ -158,14 +166,14 @@ mod tests { unfinalized, state_db, ); - let tfer_unit = Transfer{ + let tfer_unit = Transfer { to_address: Address::create_random_address(), value: 100, memo: vec![], }; - let msg_unit = SequencerMsg{ + let msg_unit = SequencerMsg { chain_id: 10, - data: vec![0,0,0,0], + data: vec![0, 0, 0, 0], from_address: Address::create_random_address(), }; let units: Vec> = vec![Box::new(tfer_unit), Box::new(msg_unit)]; From d5e7ea1db565b9946a046502bc7f289f45ddbc38 Mon Sep 17 00:00:00 2001 From: manojkgorle Date: Thu, 3 Apr 2025 11:15:51 +0530 Subject: [PATCH 38/43] rule 1: code goes no where --- Cargo.lock | 2 + chain/Cargo.toml | 1 + chain/src/actors/application/actor.rs | 36 ++++-- chain/src/actors/application/mod.rs | 3 +- chain/src/actors/syncer/actor.rs | 13 +++ chain/src/actors/syncer/mod.rs | 2 +- chain/src/bin/validator.rs | 1 + chain/src/engine.rs | 7 +- chain/src/lib.rs | 5 + storage/Cargo.toml | 1 + storage/src/hashmap_db.rs | 4 +- storage/src/state_db.rs | 6 +- types/src/address.rs | 9 ++ types/src/signed_tx.rs | 3 +- types/src/tx.rs | 11 +- types/src/units/msg.rs | 1 - vm/src/capture_logs.rs | 2 +- vm/src/vm.rs | 152 +++++++++++++++++--------- 18 files changed, 185 insertions(+), 74 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8d326fb0..4ad1a85e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -43,6 +43,7 @@ dependencies = [ "alto-client", "alto-storage", "alto-types", + "alto-vm", "axum", "bytes", "clap", @@ -115,6 +116,7 @@ dependencies = [ "rand", "rocksdb", "tempfile", + "tracing", ] [[package]] diff --git a/chain/Cargo.toml b/chain/Cargo.toml index 61a6b98b..5a54ed77 100644 --- a/chain/Cargo.toml +++ b/chain/Cargo.toml @@ -14,6 +14,7 @@ documentation = "https://docs.rs/alto-chain" alto-types = { workspace = true } alto-client = { workspace = true } alto-storage = { workspace = true} +alto-vm = { workspace = true} commonware-consensus = { workspace = true } commonware-cryptography = { workspace = true } commonware-deployer = { workspace = true } diff --git a/chain/src/actors/application/actor.rs b/chain/src/actors/application/actor.rs index 18f9313a..70c5dec8 100644 --- a/chain/src/actors/application/actor.rs +++ b/chain/src/actors/application/actor.rs @@ -9,6 +9,7 @@ use alto_storage::{ transactional_db::{Key, Op}, }; use alto_types::{Block, Finalization, Notarization, Seed}; +use alto_vm::vm::VM; use commonware_consensus::threshold_simplex::Prover; use commonware_cryptography::{sha256::Digest, Hasher, Sha256}; use commonware_macros::select; @@ -63,9 +64,15 @@ pub struct Actor { prover: Prover, hasher: Sha256, mailbox: mpsc::Receiver, + // chain id. + chain_id: u64, + // State chache. state_cache: Arc>>, - unfinalized_state: Arc>>, - staete_db: Arc>, + // Unfinalized State. + // hashmap of block number -> touched keys. + unfinalized_state: Arc>>>, + // State database. + state_db: Arc>, } impl Actor { @@ -78,9 +85,10 @@ impl Actor { prover: config.prover, hasher: Sha256::new(), mailbox, + chain_id: config.chain_id, state_cache: config.state_cache, unfinalized_state: config.unfinalized_state, - staete_db: config.state_db, + state_db: config.state_db, }, Supervisor::new(config.identity, config.participants, config.share), Mailbox::new(sender), @@ -105,8 +113,6 @@ impl Actor { let built: Option = None; let built = Arc::new(Mutex::new(built)); // @todo initiate fee manager here. - // @todo get the state view. - // @todo init the database. // @todo commit to database. while let Some(message) = self.mailbox.next().await { match message { @@ -114,6 +120,7 @@ impl Actor { Message::Genesis { response } => { // Use the digest of the genesis message as the initial // payload. + // @todo make genesis allocations. let _ = response.send(genesis_digest.clone()); } // its this validators turn to propose the block. @@ -134,6 +141,9 @@ impl Actor { // continue processing other messages) self.context.with_label("propose").spawn({ let built = built.clone(); + let state_cache = Arc::clone(&self.state_cache); + let unfinalized_state = Arc::clone(&self.unfinalized_state); + let state_db = Arc::clone(&self.state_db); move |context| async move { let response_closed = oneshot_closed_future(&mut response); select! { @@ -149,8 +159,20 @@ impl Actor { // fetch transactions from mempool. // serialize the transactions fetched from mempool into a vec. // execute the transactions and get the result? - let txs = Vec::new(); - let dummy_state_root = [0u8;32]; + let txs = Vec::new(); // @todo + let dummy_state_root = [0u8;32]; //@todo + // all the touched keys by the block will be added to unfinalized state. + let mut executor_vm = VM::new( + parent.height + 1, + current, + self.chain_id, + state_cache, + unfinalized_state, + state_db + ); + // let (outputs, errors) = executor_vm.apply(txs.clone()); + // @todo we need to keep track of touched keys per block. + // when a block gets finalised, those keys should be removed from unfinalized and moved to cache. let block = Block::new(parent.digest(), parent.height+1, current, txs, dummy_state_root.into()); let digest = block.digest(); { diff --git a/chain/src/actors/application/mod.rs b/chain/src/actors/application/mod.rs index 916dafd5..d78b6c4a 100644 --- a/chain/src/actors/application/mod.rs +++ b/chain/src/actors/application/mod.rs @@ -38,8 +38,9 @@ pub struct Config { /// before blocking. pub mailbox_size: usize, + pub chain_id: u64, /// State pub state_cache: Arc>>, - pub unfinalized_state: Arc>>, + pub unfinalized_state: Arc>>>, pub state_db: Arc>, } diff --git a/chain/src/actors/syncer/actor.rs b/chain/src/actors/syncer/actor.rs index 897b05a3..7a78b467 100644 --- a/chain/src/actors/syncer/actor.rs +++ b/chain/src/actors/syncer/actor.rs @@ -13,6 +13,7 @@ use crate::{ }, Indexer, }; +use alto_storage::{database::Database, transactional_db::{Key, Op}}; use alto_types::{Block, Finalization, Finalized, Notarized}; use bytes::Bytes; use commonware_cryptography::{bls12381, ed25519::PublicKey, sha256::Digest}; @@ -76,6 +77,14 @@ pub struct Actor>>, + // Unfinalized State. + // hashmap of block number -> touched keys. + unfinalized_state: Arc>>>, + // State database. + state_db: Arc>, } impl, I: Indexer> Actor { @@ -221,6 +230,10 @@ impl, I: Index finalized_height, contiguous_height, + + state_cache: config.state_cache, + unfinalized_state: config.unfinalized_state, + state_db: config.state_db, }, Mailbox::new(sender), ) diff --git a/chain/src/actors/syncer/mod.rs b/chain/src/actors/syncer/mod.rs index 727d6c03..868cb963 100644 --- a/chain/src/actors/syncer/mod.rs +++ b/chain/src/actors/syncer/mod.rs @@ -44,6 +44,6 @@ pub struct Config { // State pub state_cache: Arc>>, - pub unfinalized_state: Arc>>, + pub unfinalized_state: Arc>>>, pub state_db: Arc>, } diff --git a/chain/src/bin/validator.rs b/chain/src/bin/validator.rs index 6d9517cc..bb7cbdd3 100644 --- a/chain/src/bin/validator.rs +++ b/chain/src/bin/validator.rs @@ -211,6 +211,7 @@ fn main() { fetch_concurrent: FETCH_CONCURRENT, fetch_rate_per_peer: resolver_limit, indexer, + chain_id: 10, state_db: wrapped_state_db, }; let engine = engine::Engine::new(context.with_label("engine"), config).await; diff --git a/chain/src/engine.rs b/chain/src/engine.rs index 2bba2344..204c5824 100644 --- a/chain/src/engine.rs +++ b/chain/src/engine.rs @@ -47,6 +47,7 @@ pub struct Config { pub indexer: Option, + pub chain_id: u64, pub state_db: Arc>, } @@ -73,7 +74,7 @@ pub struct Engine< // state state_cache: Arc>>, - unfinalized_state: Arc>>, + unfinalized_state: Arc>>>, state_db: Arc>, } @@ -84,7 +85,8 @@ impl + Metri // @todo initalizing state cache and unfinalized state. // if it is necessary pass state_cache, unfinalized_state and state_db to both application and syncer. let state_cache: Arc>> = Arc::new(Mutex::new(HashMap::new())); - let unfinalized_state: Arc>> = Arc::new(Mutex::new(HashMap::new())); + let unfinalized_state: Arc>>> = + Arc::new(Mutex::new(HashMap::new())); // Create the application let public = public(&cfg.identity); @@ -96,6 +98,7 @@ impl + Metri identity: cfg.identity.clone(), share: cfg.share, mailbox_size: cfg.mailbox_size, + chain_id: cfg.chain_id, state_cache: Arc::clone(&state_cache), unfinalized_state: Arc::clone(&unfinalized_state), state_db: Arc::clone(&cfg.state_db), diff --git a/chain/src/lib.rs b/chain/src/lib.rs index e1fb66cf..7ba48f14 100644 --- a/chain/src/lib.rs +++ b/chain/src/lib.rs @@ -292,6 +292,7 @@ mod tests { fetch_concurrent: 10, fetch_rate_per_peer: Quota::per_second(NonZeroU32::new(10).unwrap()), indexer: None, + chain_id: 10, state_db: wrapped_state_db, }; let engine = Engine::new(context.with_label(&uid), config).await; @@ -456,6 +457,7 @@ mod tests { fetch_concurrent: 10, fetch_rate_per_peer: Quota::per_second(NonZeroU32::new(10).unwrap()), indexer: None, + chain_id: 10, state_db: wrapped_state_db, }; let engine = Engine::new(context.with_label(&uid), config).await; @@ -544,6 +546,7 @@ mod tests { fetch_concurrent: 10, fetch_rate_per_peer: Quota::per_second(NonZeroU32::new(10).unwrap()), indexer: None, + chain_id: 10, state_db: wrapped_state_db, }; let engine = Engine::new(context.with_label(&uid), config).await; @@ -682,6 +685,7 @@ mod tests { fetch_concurrent: 10, fetch_rate_per_peer: Quota::per_second(NonZeroU32::new(10).unwrap()), indexer: None, + chain_id: 10, state_db: wrapped_state_db, }; let engine = Engine::new(context.with_label(&uid), config).await; @@ -831,6 +835,7 @@ mod tests { fetch_concurrent: 10, fetch_rate_per_peer: Quota::per_second(NonZeroU32::new(10).unwrap()), indexer: Some(indexer.clone()), + chain_id: 10, state_db: wrapped_state_db, }; let engine = Engine::new(context.with_label(&uid), config).await; diff --git a/storage/Cargo.toml b/storage/Cargo.toml index aded60a6..5b20b6f7 100644 --- a/storage/Cargo.toml +++ b/storage/Cargo.toml @@ -6,6 +6,7 @@ edition = "2021" [dependencies] alto-types = { workspace = true } commonware-cryptography = { workspace = true } +tracing = {workspace = true} rand = "0.8.5" rocksdb = "0.23.0" commonware-codec = { workspace = true} diff --git a/storage/src/hashmap_db.rs b/storage/src/hashmap_db.rs index f5e7d8e2..89195625 100644 --- a/storage/src/hashmap_db.rs +++ b/storage/src/hashmap_db.rs @@ -27,9 +27,7 @@ impl Database for HashmapDatabase { } fn get(&mut self, key: &[u8]) -> Result>, Box> { - self.data - .get(key) - .map_or(Ok(None), |v| Ok(Some(v.clone().into()))) + self.data.get(key).map_or(Ok(None), |v| Ok(Some(v.clone()))) } fn delete(&mut self, key: &[u8]) -> Result<(), Box> { diff --git a/storage/src/state_db.rs b/storage/src/state_db.rs index 12059d62..bd235e40 100644 --- a/storage/src/state_db.rs +++ b/storage/src/state_db.rs @@ -5,10 +5,10 @@ use alto_types::state_view::StateView; use bytes::Bytes; use commonware_codec::{Codec, ReadBuffer, WriteBuffer}; use std::error::Error; +use tracing::{info, warn}; const ACCOUNTS_PREFIX: u8 = 0x0; const DB_WRITE_BUFFER_CAPACITY: usize = 500; -// @todo rename StateViewDb to StateView. /// StateViewDb is a wrapper around TransactionalDb that provides StateViews for block execution. /// StateViewDb simplifies the interactions with state by providing methods that abstract away the underlying database operations. /// It allows for easy retrieval and modification of account states, such as balances. @@ -16,7 +16,7 @@ pub struct StateViewDb<'a> { db: &'a mut dyn TransactionalDb, } -impl<'a> StateView for StateViewDb<'a> { +impl StateView for StateViewDb<'_> { fn get_account(&mut self, address: &Address) -> Result, Box> { let key = Self::key_accounts(address); self.db.get(&key).and_then(|v| { @@ -51,12 +51,14 @@ impl<'a> StateView for StateViewDb<'a> { } fn set_balance(&mut self, address: &Address, amt: Balance) -> bool { + info!("Setting balance for address: {}", address); match self.get_account(address) { Ok(Some(mut acc)) => { acc.balance = amt; self.set_account(&acc).is_ok() } Err(e) => { + warn!("Error getting account: {}", e); let acc = Account { address: address.clone(), balance: amt, diff --git a/types/src/address.rs b/types/src/address.rs index 79a7bb8b..9fb8da51 100644 --- a/types/src/address.rs +++ b/types/src/address.rs @@ -1,4 +1,7 @@ +use std::fmt::{Display, Formatter}; + use crate::{PublicKey, ADDRESSLEN}; +use commonware_utils::hex; use more_asserts::assert_le; use rand::Rng; @@ -51,3 +54,9 @@ impl Address { &self.0 } } + +impl Display for Address { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", hex(&self.0)) + } +} diff --git a/types/src/signed_tx.rs b/types/src/signed_tx.rs index 35d9aeca..e0a70613 100644 --- a/types/src/signed_tx.rs +++ b/types/src/signed_tx.rs @@ -143,7 +143,6 @@ pub fn unpack_signed_txs(bytes: Vec) -> Vec { #[cfg(test)] mod tests { - use std::default; use std::error::Error; use super::*; @@ -163,7 +162,7 @@ mod tests { let units: Vec> = vec![Box::new(transfer)]; let digest: [u8; 32] = [0; 32]; let id = Digest::from(digest.clone()); - let (pk, sk) = create_test_keypair(); + let (pk, _sk) = create_test_keypair(); // TODO: the .encode call on next line gave error and said origin_msg needed to be mut? but why? // shouldn't encode be able to encode without changing the msg? let tx = Tx { diff --git a/types/src/tx.rs b/types/src/tx.rs index 0d5632e1..cba8bbc7 100644 --- a/types/src/tx.rs +++ b/types/src/tx.rs @@ -8,7 +8,7 @@ use crate::state_view::StateView; use crate::units; use crate::wallet::Wallet; use commonware_utils::SystemTimeExt; -use std::time::SystemTime; +use std::{error::Error, time::SystemTime}; #[derive(Debug)] pub enum UnitType { @@ -96,6 +96,14 @@ pub struct Tx { pub actor: Address, } +#[derive(Debug)] +pub struct TxResult { + pub status: bool, + pub error: Box, + pub output: Vec>, + pub exec_logs: String, +} + pub trait TxMethods: Sized { /// new is used to create a new instance of Tx with given units and chain id. fn new(units: Vec>, chain_id: u64) -> Self; @@ -319,7 +327,6 @@ fn unpack_units(digest: &[u8]) -> Result>, String> { Ok(units) } -// @todo implement tests for encoding and decoding of tx. #[cfg(test)] mod tests { use super::*; diff --git a/types/src/units/msg.rs b/types/src/units/msg.rs index b31083dc..aa071286 100644 --- a/types/src/units/msg.rs +++ b/types/src/units/msg.rs @@ -87,7 +87,6 @@ mod tests { let chain_id = 4502; let data = vec![0xDE, 0xAD, 0xBE, 0xEF]; let from_address = Address::create_random_address(); - let relayer_id = 1; let origin_msg = SequencerMsg { chain_id, data, diff --git a/vm/src/capture_logs.rs b/vm/src/capture_logs.rs index bcbabdda..15969e72 100644 --- a/vm/src/capture_logs.rs +++ b/vm/src/capture_logs.rs @@ -40,7 +40,7 @@ where let subscriber = Registry::default().with(layer); let dispatch = tracing::Dispatch::new(subscriber); - let result = with_default(&dispatch, || func()); + let result = with_default(&dispatch, func); let logs = buffer.lock().unwrap(); let log_str = String::from_utf8(logs.clone()).unwrap_or_default(); diff --git a/vm/src/vm.rs b/vm/src/vm.rs index 248bf9b0..942bf71e 100644 --- a/vm/src/vm.rs +++ b/vm/src/vm.rs @@ -3,6 +3,7 @@ use std::error::Error; use std::sync::{Arc, Mutex}; use std::vec; +use crate::capture_logs::capture_logs; use alto_storage::state_db::StateViewDb; use alto_storage::transactional_db::TransactionalDb; use alto_storage::{ @@ -11,14 +12,17 @@ use alto_storage::{ }; use alto_types::null_error::NullError; use alto_types::state_view::StateView; -use alto_types::tx::{Tx, TxMethods, Unit, UnitContext}; +use alto_types::{ + signed_tx::SignedTx, + tx::{Tx, TxMethods, UnitContext, TxResult}, +}; pub struct VM { pub block_number: u64, pub timestamp: u64, pub chain_id: u64, pub state_cache: Arc>>, - pub unfinalized_state: Arc>>, + pub unfinalized_state: Arc>>>, pub state_db: Arc>, } @@ -28,7 +32,7 @@ impl VM { timestamp: u64, chain_id: u64, state_cache: Arc>>, - unfinalized_state: Arc>>, + unfinalized_state: Arc>>>, state_db: Arc>, ) -> Self { Self { @@ -42,47 +46,44 @@ impl VM { } // applies new set of txs on the given state. - pub fn apply(&mut self, txs: Vec) -> (Vec>>, Vec>) { + // apply assumes apply is equivalent to executing all the txs in a block. + // and moves all the touched state by the txs into unfinalized state. + pub fn apply(&mut self, stxs: Vec) -> Vec{ + let unfinalized_state_for_in_mem = + merge_maps(self.unfinalized_state.lock().unwrap().clone()); let mut in_mem_db = InMemoryCachingTransactionalDb::new( Arc::clone(&self.state_cache), - Arc::clone(&self.unfinalized_state), + Arc::new(Mutex::new(unfinalized_state_for_in_mem)), Arc::clone(&self.state_db), ); - let mut outputs: Vec>> = Vec::new(); - let mut errors: Vec> = Vec::new(); - for tx in txs { + let mut results = Vec::new(); + for stx in stxs { let mut state_view = StateViewDb::new(&mut in_mem_db); - let result = self.apply_tx(tx.clone(), &mut state_view); - match result { - Ok(output) => { - // tx executed successfully. - // commit the state changes made by the tx. - let _ = in_mem_db.commit_last_tx(); - // push the output of the tx to the outputs. - outputs.push(output); - // push null error - errors.push(Box::new(NullError)); - } - Err(e) => { - // tx execution failed. - // rollback the transaction. - let _ = in_mem_db.rollback_last_tx(); - // push empty vec to the outputs. - outputs.push(vec![]); - // push the error to the errors. - errors.push(e); - } + let tx = stx.tx; + let result = self.apply_tx(tx, &mut state_view); + if result.status { + let _ = in_mem_db.commit_last_tx(); + }else{ + let _ = in_mem_db.rollback_last_tx(); } + results.push(result); } - (outputs, errors) + self.unfinalized_state + .lock() + .unwrap() + .insert(self.block_number, in_mem_db.touched); + // let _ = in_mem_db.commit(); + // @todo we did not call commit, but moved all the touched state into unfinalized state. + results } // applies a single tx on the given state. - fn apply_tx( + fn apply_tx<'a, T: StateView>( &mut self, tx: Tx, state_view: &mut T, - ) -> Result>, Box> { + // exec_logs: &'a mut Vec, + ) -> TxResult { let tx_context = UnitContext { timestamp: self.timestamp, chain_id: self.chain_id, @@ -92,28 +93,67 @@ impl VM { let mut outputs: Vec> = Vec::new(); // apply units one by one. // stop and revert if any unit fails. - for unit in tx.units { - let result = unit.apply(&tx_context, &mut sv_boxed); - match result { - Ok(output) => { - if let Some(output) = output { - outputs.push(output); - } else { - // if output is None, unit execution does not return anything. - // push empty vec. - outputs.push(vec![]); + let (result, log) = capture_logs(||{ + for unit in tx.units { + let res = unit.apply(&tx_context, &mut sv_boxed); + match res { + Ok(output) => { + if let Some(output) = output { + outputs.push(output); + } else { + // if output is None, unit execution does not return anything. + // push empty vec. + outputs.push(vec![]); + } + } + Err(e) => { + // return the error. + return Err(e); } } - Err(e) => { - // return the error. - return Err(e); - } + } + Ok(()) + }); + if result.is_err() { + TxResult{ + status: false, + error: result.err().unwrap(), + exec_logs: log, + output: outputs, + } + }else{ + TxResult{ + status: true, + error: Box::new(NullError), + exec_logs: log, + output: outputs, } } - Ok(outputs) + } } +fn merge_maps( + input: HashMap>, +) -> HashMap { + let mut merged = HashMap::new(); + + // Sort the keys in ascending order so we can override with higher keys last + let mut keys: Vec<_> = input.keys().cloned().collect(); + keys.sort(); + + for k in keys { + if let Some(inner_map) = input.get(&k) { + for (inner_key, val) in inner_map { + // Insert or override + merged.insert(inner_key.clone(), val.clone()); + } + } + } + + merged +} + #[cfg(test)] mod tests { use alto_storage::database::Database; @@ -122,13 +162,16 @@ mod tests { use alto_storage::transactional_db::{Key, Op}; use alto_types::account::Account; use alto_types::address::Address; + use alto_types::create_test_keypair; use alto_types::curr_timestamp; + use alto_types::signed_tx::{SignedTx, SignedTxChars}; use alto_types::tx::{Tx, TxMethods, Unit}; use alto_types::units::msg::SequencerMsg; use alto_types::units::transfer::Transfer; use commonware_codec::{Codec, WriteBuffer}; use std::collections::HashMap; use std::sync::{Arc, Mutex}; + use std::vec; use super::VM; @@ -138,7 +181,8 @@ mod tests { fn test_single_tx() { let state_db = Arc::new(Mutex::new(HashmapDatabase::new())); let cache: Arc>> = Arc::new(Mutex::new(HashMap::new())); - let unfinalized: Arc>> = Arc::new(Mutex::new(HashMap::new())); + let unfinalized: Arc>>> = + Arc::new(Mutex::new(HashMap::new())); let address = Address::create_random_address(); let account = Account { @@ -178,10 +222,14 @@ mod tests { }; let units: Vec> = vec![Box::new(tfer_unit), Box::new(msg_unit)]; let tx = ::from(timestamp, units, 10, 5, 1, address); - let (outputs, errors) = vm.apply(vec![tx]); - assert_eq!(outputs.len(), 1); - assert_eq!(outputs[0].len(), 2); - assert_eq!(errors.len(), 1); - assert_eq!(errors[0].to_string(), "NoError"); + let (pk, _sk) = create_test_keypair(); + let stx = SignedTx::new(tx, pk, vec![]); + let results = vm.apply(vec![stx]); + assert_eq!(results.len(), 1); + assert_eq!(results[0].status, true); + assert_eq!(results[0].output.len(), 2); + assert_eq!(results[0].output[0].len(), 0); + assert_eq!(results[0].error.to_string(), "NoError"); + println!("{}", results[0].exec_logs); } } From 4b694ab091e356e61979bff65400b010761939be Mon Sep 17 00:00:00 2001 From: manojkgorle Date: Thu, 3 Apr 2025 13:52:28 +0530 Subject: [PATCH 39/43] dummy genesis --- Cargo.lock | 1 + chain/Cargo.toml | 1 + chain/src/actors/application/actor.rs | 48 ++++++++++++++++++++++----- chain/src/actors/application/mod.rs | 4 ++- chain/src/actors/syncer/actor.rs | 12 +++++-- chain/src/actors/syncer/mod.rs | 5 ++- chain/src/bin/setup.rs | 15 ++++++++- chain/src/bin/validator.rs | 10 ++++-- chain/src/engine.rs | 6 +++- chain/src/lib.rs | 37 ++++++++++++++++++++- storage/src/state_db.rs | 2 +- types/src/address.rs | 18 ++++++---- types/src/tx.rs | 2 +- vm/src/vm.rs | 20 +++++------ 14 files changed, 143 insertions(+), 38 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4ad1a85e..61c10ade 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -47,6 +47,7 @@ dependencies = [ "axum", "bytes", "clap", + "commonware-codec", "commonware-consensus", "commonware-cryptography", "commonware-deployer", diff --git a/chain/Cargo.toml b/chain/Cargo.toml index 5a54ed77..d91af01e 100644 --- a/chain/Cargo.toml +++ b/chain/Cargo.toml @@ -16,6 +16,7 @@ alto-client = { workspace = true } alto-storage = { workspace = true} alto-vm = { workspace = true} commonware-consensus = { workspace = true } +commonware-codec = {workspace = true} commonware-cryptography = { workspace = true } commonware-deployer = { workspace = true } commonware-macros = { workspace = true } diff --git a/chain/src/actors/application/actor.rs b/chain/src/actors/application/actor.rs index 70c5dec8..924a0e37 100644 --- a/chain/src/actors/application/actor.rs +++ b/chain/src/actors/application/actor.rs @@ -3,13 +3,16 @@ use super::{ supervisor::Supervisor, Config, }; -use crate::actors::syncer; +use crate::{actors::syncer, GenesisAllocations}; +use alto_storage::state_db::DB_WRITE_BUFFER_CAPACITY; use alto_storage::{ database::Database, + state_db::StateViewDb, transactional_db::{Key, Op}, }; -use alto_types::{Block, Finalization, Notarization, Seed}; +use alto_types::{account::Account, address::Address, Block, Finalization, Notarization, Seed}; use alto_vm::vm::VM; +use commonware_codec::{Codec, WriteBuffer}; use commonware_consensus::threshold_simplex::Prover; use commonware_cryptography::{sha256::Digest, Hasher, Sha256}; use commonware_macros::select; @@ -66,7 +69,9 @@ pub struct Actor { mailbox: mpsc::Receiver, // chain id. chain_id: u64, - // State chache. + // genesis allocations. + genesis: GenesisAllocations, + // State cache. state_cache: Arc>>, // Unfinalized State. // hashmap of block number -> touched keys. @@ -86,6 +91,7 @@ impl Actor { hasher: Sha256::new(), mailbox, chain_id: config.chain_id, + genesis: config.genesis, state_cache: config.state_cache, unfinalized_state: config.unfinalized_state, state_db: config.state_db, @@ -120,7 +126,30 @@ impl Actor { Message::Genesis { response } => { // Use the digest of the genesis message as the initial // payload. - // @todo make genesis allocations. + + // @todo check formating of genesis in validator.rs + let address = self.genesis.address.clone(); + let allocations = self.genesis.value.clone(); + // lock state database. + let mut s_db = self.state_db.lock().unwrap(); + // zip address with allocation. + address + .iter() + .zip(allocations.iter()) + .for_each(|(addr, allo)| { + // create key from address + let adrs = Address(*addr); + let key = StateViewDb::key_accounts(&adrs); + let mut write_buf = WriteBuffer::new(DB_WRITE_BUFFER_CAPACITY); + let acc = Account { + address: adrs, + balance: allo.clone(), + }; + // write the account to the buffer. + acc.write(&mut write_buf); + // store the account in the state database. + let _ = s_db.put(&key, write_buf.as_ref()); + }); let _ = response.send(genesis_digest.clone()); } // its this validators turn to propose the block. @@ -159,8 +188,7 @@ impl Actor { // fetch transactions from mempool. // serialize the transactions fetched from mempool into a vec. // execute the transactions and get the result? - let txs = Vec::new(); // @todo - let dummy_state_root = [0u8;32]; //@todo + let txs = Vec::new(); // @todo get txs from mempool. // all the touched keys by the block will be added to unfinalized state. let mut executor_vm = VM::new( parent.height + 1, @@ -170,9 +198,11 @@ impl Actor { unfinalized_state, state_db ); - // let (outputs, errors) = executor_vm.apply(txs.clone()); - // @todo we need to keep track of touched keys per block. - // when a block gets finalised, those keys should be removed from unfinalized and moved to cache. + let results = executor_vm.apply(txs.clone()); // @todo store results seperately. + let dummy_state_root = [0u8;32]; //@todo + // touched keys per block are stored in unfinalized state. + // when a block finalises, touched keys are removed from unfinalized state and moved to cache afte writing to db. + // @todo post finalisation moving to cache and db, removing unfinalized state. let block = Block::new(parent.digest(), parent.height+1, current, txs, dummy_state_root.into()); let digest = block.digest(); { diff --git a/chain/src/actors/application/mod.rs b/chain/src/actors/application/mod.rs index d78b6c4a..5eef80c5 100644 --- a/chain/src/actors/application/mod.rs +++ b/chain/src/actors/application/mod.rs @@ -19,9 +19,10 @@ mod ingress; pub use ingress::Mailbox; mod router; mod supervisor; - pub use supervisor::Supervisor; +use crate::GenesisAllocations; + /// Configuration for the application. pub struct Config { /// Prover used to decode opaque proofs from consensus. @@ -43,4 +44,5 @@ pub struct Config { pub state_cache: Arc>>, pub unfinalized_state: Arc>>>, pub state_db: Arc>, + pub genesis: GenesisAllocations, } diff --git a/chain/src/actors/syncer/actor.rs b/chain/src/actors/syncer/actor.rs index 7a78b467..7ce84939 100644 --- a/chain/src/actors/syncer/actor.rs +++ b/chain/src/actors/syncer/actor.rs @@ -13,8 +13,11 @@ use crate::{ }, Indexer, }; -use alto_storage::{database::Database, transactional_db::{Key, Op}}; -use alto_types::{Block, Finalization, Finalized, Notarized}; +use alto_storage::{ + database::Database, + transactional_db::{Key, Op}, +}; +use alto_types::{tx::TxResult, Block, Finalization, Finalized, Notarized}; use bytes::Bytes; use commonware_cryptography::{bls12381, ed25519::PublicKey, sha256::Digest}; use commonware_macros::select; @@ -58,6 +61,9 @@ pub struct Actor, + // @todo this should be improvised. results should be streamed as soon as the block is finalised. + // and should be stored in a seperate db and maintained by rpc services. + results: Arc>>>, // Blocks verified stored by view<>digest verified: Archive, // Blocks notarized stored by view<>digest @@ -220,6 +226,8 @@ impl, I: Index activity_timeout: config.activity_timeout, indexer: config.indexer, + results: config.results, + verified: verified_archive, notarized: notarized_archive, diff --git a/chain/src/actors/syncer/mod.rs b/chain/src/actors/syncer/mod.rs index 868cb963..8e804d6b 100644 --- a/chain/src/actors/syncer/mod.rs +++ b/chain/src/actors/syncer/mod.rs @@ -2,7 +2,8 @@ use alto_storage::{ database::Database, transactional_db::{Key, Op}, }; -use commonware_cryptography::{bls12381::primitives::group, ed25519::PublicKey}; +use alto_types::tx::TxResult; +use commonware_cryptography::{bls12381::primitives::group, ed25519::PublicKey, sha256::Digest}; use governor::Quota; use std::{ collections::HashMap, @@ -42,6 +43,8 @@ pub struct Config { pub indexer: Option, + pub results: Arc>>>, + // State pub state_cache: Arc>>, pub unfinalized_state: Arc>>>, diff --git a/chain/src/bin/setup.rs b/chain/src/bin/setup.rs index 4218d620..97bee6e1 100644 --- a/chain/src/bin/setup.rs +++ b/chain/src/bin/setup.rs @@ -1,4 +1,5 @@ -use alto_chain::Config; +use alto_chain::{Config, GenesisAllocations}; +use alto_types::address::Address; use clap::{value_parser, Arg, ArgMatches, Command}; use commonware_cryptography::{ bls12381::{ @@ -298,6 +299,18 @@ fn generate(sub_matches: &ArgMatches) { let file = fs::File::create(&path).unwrap(); serde_yaml::to_writer(file, &config).unwrap(); info!(path = "config.yaml", "wrote configuration file"); + + // genesis @todo this is a simple genesis implementation. + // addr should be considered along with the values. + let addr1 = Address::create_random_address(); + let genesis = GenesisAllocations { + address: vec![*addr1.as_bytes()], + value: vec![1000000000000], + }; + let genesis_path = format!("{}/genesis.yaml", output); + let file = fs::File::create(&genesis_path).unwrap(); + serde_yaml::to_writer(file, &genesis).unwrap(); + info!(path = "genesis.yaml", "wrote genesis file"); } fn indexer(sub_matches: &ArgMatches) { diff --git a/chain/src/bin/validator.rs b/chain/src/bin/validator.rs index bb7cbdd3..7fcd1377 100644 --- a/chain/src/bin/validator.rs +++ b/chain/src/bin/validator.rs @@ -1,4 +1,4 @@ -use alto_chain::{engine, Config}; +use alto_chain::{engine, Config, GenesisAllocations}; use alto_client::Client; use alto_storage::rocks_db::RocksDbDatabase; use alto_types::P2P_NAMESPACE; @@ -56,6 +56,7 @@ fn main() { .about("Validator for an alto chain.") .arg(Arg::new("peers").long("peers").required(true)) .arg(Arg::new("config").long("config").required(true)) + .arg(Arg::new("genesis").long("genesis").required(true)) .get_matches(); // Create logger @@ -81,7 +82,11 @@ fn main() { .collect(); info!(peers = peers.len(), "loaded peers"); let peers_u32 = peers.len() as u32; - + // @todo load genesis and use to initiate the application. + let genesis_file = matches.get_one::("genesis").unwrap(); + let genesis_file = std::fs::read_to_string(genesis_file).expect("Could not read genesis file"); + let genesis: GenesisAllocations = + serde_yaml::from_str(&genesis_file).expect("Could not parse genesis file"); // Load config let config_file = matches.get_one::("config").unwrap(); let config_file = std::fs::read_to_string(config_file).expect("Could not read config file"); @@ -213,6 +218,7 @@ fn main() { indexer, chain_id: 10, state_db: wrapped_state_db, + genesis: genesis, }; let engine = engine::Engine::new(context.with_label("engine"), config).await; diff --git a/chain/src/engine.rs b/chain/src/engine.rs index 204c5824..99de46ed 100644 --- a/chain/src/engine.rs +++ b/chain/src/engine.rs @@ -1,6 +1,6 @@ use crate::{ actors::{application, syncer}, - Indexer, + GenesisAllocations, Indexer, }; use alto_storage::{ database::Database, @@ -49,6 +49,7 @@ pub struct Config { pub chain_id: u64, pub state_db: Arc>, + pub genesis: GenesisAllocations, } pub struct Engine< @@ -84,6 +85,7 @@ impl + Metri pub async fn new(context: E, cfg: Config) -> Self { // @todo initalizing state cache and unfinalized state. // if it is necessary pass state_cache, unfinalized_state and state_db to both application and syncer. + let results = Arc::new(Mutex::new(HashMap::new())); let state_cache: Arc>> = Arc::new(Mutex::new(HashMap::new())); let unfinalized_state: Arc>>> = Arc::new(Mutex::new(HashMap::new())); @@ -102,6 +104,7 @@ impl + Metri state_cache: Arc::clone(&state_cache), unfinalized_state: Arc::clone(&unfinalized_state), state_db: Arc::clone(&cfg.state_db), + genesis: cfg.genesis.clone(), }, ); @@ -117,6 +120,7 @@ impl + Metri backfill_quota: cfg.backfill_quota, activity_timeout: cfg.activity_timeout, indexer: cfg.indexer, + results: Arc::clone(&results), state_cache: Arc::clone(&state_cache), unfinalized_state: Arc::clone(&unfinalized_state), state_db: Arc::clone(&cfg.state_db), diff --git a/chain/src/lib.rs b/chain/src/lib.rs index 7ba48f14..0a73518e 100644 --- a/chain/src/lib.rs +++ b/chain/src/lib.rs @@ -79,11 +79,17 @@ pub struct Config { pub state_db_directory: String, } +#[derive(Deserialize, Serialize, Debug, Clone)] +pub struct GenesisAllocations { + pub address: Vec<[u8; 32]>, + pub value: Vec, +} + #[cfg(test)] mod tests { use super::*; use alto_storage::rocks_db::RocksDbDatabase; - use alto_types::{Finalized, Notarized, Seed}; + use alto_types::{address::Address, Finalized, Notarized, Seed}; use bls12381::primitives::poly; use commonware_cryptography::{bls12381::dkg::ops, ed25519::PublicKey, Ed25519, Scheme}; use commonware_macros::test_traced; @@ -231,6 +237,12 @@ mod tests { }; let (executor, mut context, auditor) = Executor::init(cfg); executor.start(async move { + let addr1 = Address::create_random_address(); + let addr2 = Address::create_random_address(); + let genesis = GenesisAllocations { + address: vec![*addr1.as_bytes(), *addr2.as_bytes()], + value: vec![1000000000000, 1000000000000], + }; // Create simulated network let (network, mut oracle) = Network::new( context.with_label("network"), @@ -294,6 +306,7 @@ mod tests { indexer: None, chain_id: 10, state_db: wrapped_state_db, + genesis: genesis.clone(), }; let engine = Engine::new(context.with_label(&uid), config).await; @@ -383,6 +396,12 @@ mod tests { let final_container_required = 20; let (executor, mut context, _) = Executor::timed(Duration::from_secs(30)); executor.start(async move { + let addr1 = Address::create_random_address(); + let addr2 = Address::create_random_address(); + let genesis = GenesisAllocations { + address: vec![*addr1.as_bytes(), *addr2.as_bytes()], + value: vec![1000000000000, 1000000000000], + }; // Create simulated network let (network, mut oracle) = Network::new( context.with_label("network"), @@ -459,6 +478,7 @@ mod tests { indexer: None, chain_id: 10, state_db: wrapped_state_db, + genesis: genesis.clone(), }; let engine = Engine::new(context.with_label(&uid), config).await; @@ -548,6 +568,7 @@ mod tests { indexer: None, chain_id: 10, state_db: wrapped_state_db, + genesis: genesis.clone(), }; let engine = Engine::new(context.with_label(&uid), config).await; @@ -617,6 +638,12 @@ mod tests { while !*done.lock().unwrap() { runs += 1; executor.start({ + let addr1 = Address::create_random_address(); + let addr2 = Address::create_random_address(); + let genesis = GenesisAllocations { + address: vec![*addr1.as_bytes(), *addr2.as_bytes()], + value: vec![1000000000000, 1000000000000], + }; let mut context = context.clone(); let public = public.clone(); let shares = shares.clone(); @@ -687,6 +714,7 @@ mod tests { indexer: None, chain_id: 10, state_db: wrapped_state_db, + genesis: genesis.clone(), }; let engine = Engine::new(context.with_label(&uid), config).await; @@ -765,6 +793,12 @@ mod tests { let required_container = 10; let (executor, mut context, _) = Executor::timed(Duration::from_secs(30)); executor.start(async move { + let addr1 = Address::create_random_address(); + let addr2 = Address::create_random_address(); + let genesis = GenesisAllocations { + address: vec![*addr1.as_bytes(), *addr2.as_bytes()], + value: vec![1000000000000, 1000000000000], + }; // Create simulated network let (network, mut oracle) = Network::new( context.with_label("network"), @@ -837,6 +871,7 @@ mod tests { indexer: Some(indexer.clone()), chain_id: 10, state_db: wrapped_state_db, + genesis: genesis.clone(), }; let engine = Engine::new(context.with_label(&uid), config).await; diff --git a/storage/src/state_db.rs b/storage/src/state_db.rs index bd235e40..47699d23 100644 --- a/storage/src/state_db.rs +++ b/storage/src/state_db.rs @@ -7,7 +7,7 @@ use commonware_codec::{Codec, ReadBuffer, WriteBuffer}; use std::error::Error; use tracing::{info, warn}; const ACCOUNTS_PREFIX: u8 = 0x0; -const DB_WRITE_BUFFER_CAPACITY: usize = 500; +pub const DB_WRITE_BUFFER_CAPACITY: usize = 500; /// StateViewDb is a wrapper around TransactionalDb that provides StateViews for block execution. /// StateViewDb simplifies the interactions with state by providing methods that abstract away the underlying database operations. diff --git a/types/src/address.rs b/types/src/address.rs index 9fb8da51..75ba38a2 100644 --- a/types/src/address.rs +++ b/types/src/address.rs @@ -1,10 +1,13 @@ -use std::fmt::{Display, Formatter}; +use std::{ + fmt::{Display, Formatter}, + io::copy, +}; use crate::{PublicKey, ADDRESSLEN}; +use commonware_cryptography::sha256::hash; use commonware_utils::hex; use more_asserts::assert_le; use rand::Rng; - #[derive(Hash, Eq, PartialEq, Clone, Debug)] pub struct Address(pub [u8; ADDRESSLEN]); @@ -23,11 +26,10 @@ impl Address { } pub fn from_pub_key(pub_key: &PublicKey) -> Self { - // @todo implement a hasher to derive the address from the public key. - assert_le!(pub_key.len(), ADDRESSLEN, "public key is too large"); - let mut arr = [0u8; ADDRESSLEN]; - arr[..pub_key.len()].copy_from_slice(pub_key.as_ref()); - Address(arr) + let hashed = hash(pub_key.as_ref()); + let mut addr = Address::empty(); + addr.0.copy_from_slice(&hashed); + addr } pub fn from_bytes(bytes: &[u8]) -> Result { @@ -60,3 +62,5 @@ impl Display for Address { write!(f, "{}", hex(&self.0)) } } + +// @todo write tests for all the methods. diff --git a/types/src/tx.rs b/types/src/tx.rs index cba8bbc7..e34e51af 100644 --- a/types/src/tx.rs +++ b/types/src/tx.rs @@ -99,7 +99,7 @@ pub struct Tx { #[derive(Debug)] pub struct TxResult { pub status: bool, - pub error: Box, + pub error: String, pub output: Vec>, pub exec_logs: String, } diff --git a/vm/src/vm.rs b/vm/src/vm.rs index 942bf71e..8fd6553d 100644 --- a/vm/src/vm.rs +++ b/vm/src/vm.rs @@ -1,5 +1,4 @@ use std::collections::HashMap; -use std::error::Error; use std::sync::{Arc, Mutex}; use std::vec; @@ -14,7 +13,7 @@ use alto_types::null_error::NullError; use alto_types::state_view::StateView; use alto_types::{ signed_tx::SignedTx, - tx::{Tx, TxMethods, UnitContext, TxResult}, + tx::{Tx, TxMethods, TxResult, UnitContext}, }; pub struct VM { @@ -48,7 +47,7 @@ impl VM { // applies new set of txs on the given state. // apply assumes apply is equivalent to executing all the txs in a block. // and moves all the touched state by the txs into unfinalized state. - pub fn apply(&mut self, stxs: Vec) -> Vec{ + pub fn apply(&mut self, stxs: Vec) -> Vec { let unfinalized_state_for_in_mem = merge_maps(self.unfinalized_state.lock().unwrap().clone()); let mut in_mem_db = InMemoryCachingTransactionalDb::new( @@ -63,7 +62,7 @@ impl VM { let result = self.apply_tx(tx, &mut state_view); if result.status { let _ = in_mem_db.commit_last_tx(); - }else{ + } else { let _ = in_mem_db.rollback_last_tx(); } results.push(result); @@ -93,7 +92,7 @@ impl VM { let mut outputs: Vec> = Vec::new(); // apply units one by one. // stop and revert if any unit fails. - let (result, log) = capture_logs(||{ + let (result, log) = capture_logs(|| { for unit in tx.units { let res = unit.apply(&tx_context, &mut sv_boxed); match res { @@ -115,21 +114,20 @@ impl VM { Ok(()) }); if result.is_err() { - TxResult{ + TxResult { status: false, - error: result.err().unwrap(), + error: result.err().unwrap().to_string(), exec_logs: log, output: outputs, } - }else{ - TxResult{ + } else { + TxResult { status: true, - error: Box::new(NullError), + error: NullError.to_string(), exec_logs: log, output: outputs, } } - } } From 94e3f78d86113240dd553ccebbad86ef9d618bfe Mon Sep 17 00:00:00 2001 From: manojkgorle Date: Thu, 3 Apr 2025 18:53:53 +0530 Subject: [PATCH 40/43] project x --- Cargo.lock | 1 + chain/src/actors/application/actor.rs | 71 +++++++++++++++++++++++---- chain/src/actors/syncer/actor.rs | 7 +++ chain/src/actors/syncer/ingress.rs | 15 +++++- storage/Cargo.toml | 1 + storage/src/transactional_db.rs | 2 +- 6 files changed, 85 insertions(+), 12 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 61c10ade..c40e282d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -114,6 +114,7 @@ dependencies = [ "bytes", "commonware-codec", "commonware-cryptography", + "futures", "rand", "rocksdb", "tempfile", diff --git a/chain/src/actors/application/actor.rs b/chain/src/actors/application/actor.rs index 924a0e37..b670faab 100644 --- a/chain/src/actors/application/actor.rs +++ b/chain/src/actors/application/actor.rs @@ -8,9 +8,9 @@ use alto_storage::state_db::DB_WRITE_BUFFER_CAPACITY; use alto_storage::{ database::Database, state_db::StateViewDb, - transactional_db::{Key, Op}, + transactional_db::{Key, Op, OpAction}, }; -use alto_types::{account::Account, address::Address, Block, Finalization, Notarization, Seed}; +use alto_types::{account::Account, address::Address, Block, Finalization, Notarization, Seed, signed_tx::unpack_signed_txs}; use alto_vm::vm::VM; use commonware_codec::{Codec, WriteBuffer}; use commonware_consensus::threshold_simplex::Prover; @@ -27,9 +27,7 @@ use futures::{ }; use rand::Rng; use std::{ - collections::HashMap, - pin::Pin, - sync::{Arc, Mutex}, + collections::HashMap, pin::Pin, sync::{Arc, Mutex} }; use tracing::{info, warn}; @@ -173,6 +171,7 @@ impl Actor { let state_cache = Arc::clone(&self.state_cache); let unfinalized_state = Arc::clone(&self.unfinalized_state); let state_db = Arc::clone(&self.state_db); + let mut syncer_clone = syncer.clone(); move |context| async move { let response_closed = oneshot_closed_future(&mut response); select! { @@ -209,10 +208,12 @@ impl Actor { let mut built = built.lock().unwrap(); *built = Some(block); } - + // Send the digest to the consensus let result = response.send(digest.clone()); info!(view, ?digest, success=result.is_ok(), "proposed new block"); + // send the result to syncer. + syncer_clone.store_results(digest, results).await; }, _ = response_closed => { // The response was cancelled @@ -245,14 +246,16 @@ impl Actor { } else { Either::Right(syncer.get(Some(parent.0), parent.1).await) }; - + let state_cache = Arc::clone(&self.state_cache); + let unfinalized_state = Arc::clone(&self.unfinalized_state); + let state_db = Arc::clone(&self.state_db); // Wait for the blocks to be available or the request to be cancelled in a separate task (to // continue processing other messages) self.context.with_label("verify").spawn({ let mut syncer = syncer.clone(); move |context| async move { let requester = - try_join(parent_request, syncer.get(None, payload).await); + try_join(parent_request, syncer.get(None, payload.clone()).await); let response_closed = oneshot_closed_future(&mut response); select! { result = requester => { @@ -277,11 +280,32 @@ impl Actor { let _ = response.send(false); return; } - //@todo unmarshall txs, execute txs, generate state root, collect fees, build state root, verify state root. + // State transition checks. + + // unpack transactions from the block. + let stxs = unpack_signed_txs(block.raw_txs.clone()); + // create a new vm instance. + let mut executor_vm = VM::new( + block.height, + block.timestamp, + self.chain_id, + state_cache, + unfinalized_state, + state_db + ); + // apply transactions. + let results = executor_vm.apply(stxs.clone()); + // state root generation. + let dummy_state_root = [0u8;32]; //@todo + // verify state root equivalence. + if block.state_root != dummy_state_root.into() { + let _ = response.send(false); + return; + } // Persist the verified block syncer.verified(view, block).await; - + syncer.store_results(payload, results).await; // Send the verification result to the consensus let _ = response.send(true); }, @@ -304,6 +328,7 @@ impl Actor { syncer.notarized(notarization, seed).await; } Message::Finalized { proof, payload } => { + // @todo let state changes be restricted to application. // Parse the proof let (view, parent, _, signature, seed) = self.prover.deserialize_finalization(proof.clone()).unwrap(); @@ -311,6 +336,32 @@ impl Actor { let seed = Seed::new(view, seed.into()); // @todo syncer does the heavy lifting of post finalization processing. + if let Some(at_view_touched) = self.unfinalized_state.lock().unwrap().remove(&view){ + if at_view_touched.is_empty(){ + info!(view, "finalized block with no touched keys"); + }else{ + // lock state database. + let mut s_db = self.state_db.lock().unwrap(); + // iterate over the touched keys and write to the state database. + for (key, op) in at_view_touched.iter(){ + match op.action { + OpAction::Update => { + let _ = s_db.put(key, &op.value); + }, + OpAction::Delete => { + let _ = s_db.delete(key); + }, + _ =>{ + /*nothing to do with the database. */ + } + } + } + info!(view, "finalized block with touched keys"); + } + }else{ + + } + // Send the finalization to the syncer syncer.finalized(finalization, seed).await; } diff --git a/chain/src/actors/syncer/actor.rs b/chain/src/actors/syncer/actor.rs index 7ce84939..b9c9dc0b 100644 --- a/chain/src/actors/syncer/actor.rs +++ b/chain/src/actors/syncer/actor.rs @@ -297,12 +297,14 @@ impl, I: Index ); resolver_engine.start(backfill_network); + // @todo block syncing process is happening here. // Process all finalized blocks in order (fetching any that are missing) let last_view_processed = Arc::new(Mutex::new(0)); let verified = Wrapped::new(self.verified); let notarized = Wrapped::new(self.notarized); let finalized = Wrapped::new(self.finalized); let blocks = Wrapped::new(self.blocks); + let results = Arc::clone(&self.results); let (mut finalizer_sender, mut finalizer_receiver) = mpsc::channel::<()>(1); self.context.with_label("finalizer").spawn({ let mut resolver = resolver.clone(); @@ -485,6 +487,9 @@ impl, I: Index mailbox_message = self.mailbox.next() => { let message = mailbox_message.expect("Mailbox closed"); match message { + Message::StoreResults {payload, result} => { + results.lock().unwrap().insert(payload, result); + } Message::Broadcast { payload } => { broadcast_network .0 @@ -612,6 +617,7 @@ impl, I: Index let view = proof.view; let digest = proof.payload.clone(); let height = block.height; + // if we have block either in verified or notarized, it means we have the block verified and unfinalized state map. finalized .put(height, proof.payload.clone(), proof.serialize().into()) .await @@ -714,6 +720,7 @@ impl, I: Index debug!(view, ?payload, "registering waiter"); waiters.entry(payload).or_default().push(response); } + } }, // Handle incoming broadcasts diff --git a/chain/src/actors/syncer/ingress.rs b/chain/src/actors/syncer/ingress.rs index 7cc0cc24..8ef4ab16 100644 --- a/chain/src/actors/syncer/ingress.rs +++ b/chain/src/actors/syncer/ingress.rs @@ -1,4 +1,4 @@ -use alto_types::{Block, Finalization, Notarization, Seed}; +use alto_types::{tx::TxResult, Block, Finalization, Notarization, Seed}; use commonware_cryptography::sha256::Digest; use futures::{ channel::{mpsc, oneshot}, @@ -27,6 +27,10 @@ pub enum Message { proof: Finalization, seed: Seed, }, + StoreResults { + payload: Digest, + result: Vec, + }, } /// Mailbox for the application. @@ -83,4 +87,13 @@ impl Mailbox { .await .expect("Failed to send lock"); } + + /// store results, stores block execution results in in mem cache. + /// @todo find a better way to store results. + pub async fn store_results(&mut self, payload: Digest, result: Vec) { + self.sender + .send(Message::StoreResults { payload, result }) + .await + .expect("Failed to send store results"); + } } diff --git a/storage/Cargo.toml b/storage/Cargo.toml index 5b20b6f7..a78dbb1f 100644 --- a/storage/Cargo.toml +++ b/storage/Cargo.toml @@ -7,6 +7,7 @@ edition = "2021" alto-types = { workspace = true } commonware-cryptography = { workspace = true } tracing = {workspace = true} +futures = {workspace = true} rand = "0.8.5" rocksdb = "0.23.0" commonware-codec = { workspace = true} diff --git a/storage/src/transactional_db.rs b/storage/src/transactional_db.rs index 0214c5d2..dfd4e926 100644 --- a/storage/src/transactional_db.rs +++ b/storage/src/transactional_db.rs @@ -74,7 +74,7 @@ impl InMemoryCachingTransactionalDb { pub fn new( cache: Arc>>, unfinalized: Arc>>, - db: Arc>, + db: Arc>, ) -> Self { Self { cache, From ecaa5a864bb3ce085d146cd3b64449061a0b6c9a Mon Sep 17 00:00:00 2001 From: rikoeldon <106416799+rikoeldon@users.noreply.github.com> Date: Thu, 3 Apr 2025 14:48:00 -0400 Subject: [PATCH 41/43] added hash function for pk to addr --- Cargo.lock | 20 ++++++++++++++++++++ types/Cargo.toml | 1 + types/src/account.rs | 6 ++++++ types/src/address.rs | 7 ++----- types/src/lib.rs | 14 ++++++++++++++ 5 files changed, 43 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c40e282d..995cdfe8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -134,6 +134,7 @@ dependencies = [ "rand", "serde", "serde-wasm-bindgen", + "sha3", "thiserror 2.0.12", "wasm-bindgen", ] @@ -2089,6 +2090,15 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "keccak" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecc2af9a1119c51f12a14607e783cb977bde58bc069ff0c3da1095e635d70654" +dependencies = [ + "cpufeatures", +] + [[package]] name = "lazy_static" version = "1.5.0" @@ -3178,6 +3188,16 @@ dependencies = [ "digest 0.10.7", ] +[[package]] +name = "sha3" +version = "0.10.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75872d278a8f37ef87fa0ddbda7802605cb18344497949862c0d4dcb291eba60" +dependencies = [ + "digest 0.10.7", + "keccak", +] + [[package]] name = "sharded-slab" version = "0.1.7" diff --git a/types/Cargo.toml b/types/Cargo.toml index a67dd5d0..8d7f10fe 100644 --- a/types/Cargo.toml +++ b/types/Cargo.toml @@ -24,6 +24,7 @@ wasm-bindgen = "0.2.100" serde = { version = "1.0.219", features = ["derive"] } serde-wasm-bindgen = "0.6.5" more-asserts = "0.3.1" +sha3 = "0.10" # Enable "js" feature when WASM is target [target.'cfg(target_arch = "wasm32")'.dependencies.getrandom] diff --git a/types/src/account.rs b/types/src/account.rs index 3be32107..cc6fe3e9 100644 --- a/types/src/account.rs +++ b/types/src/account.rs @@ -1,5 +1,7 @@ use crate::address::Address; use commonware_codec::{Codec, Error, Reader, Writer}; +use commonware_cryptography::ed25519::PublicKey; +use crate::pub_key_to_address; pub type Balance = u64; @@ -23,6 +25,10 @@ impl Account { } } + pub fn from_pubkey(pk: PublicKey) -> Self { + Self::from_address(pub_key_to_address(&pk)) + } + pub fn from_address(address: Address) -> Self { Self { address, diff --git a/types/src/address.rs b/types/src/address.rs index 75ba38a2..df6391c1 100644 --- a/types/src/address.rs +++ b/types/src/address.rs @@ -3,7 +3,7 @@ use std::{ io::copy, }; -use crate::{PublicKey, ADDRESSLEN}; +use crate::{pub_key_to_address, PublicKey, ADDRESSLEN}; use commonware_cryptography::sha256::hash; use commonware_utils::hex; use more_asserts::assert_le; @@ -26,10 +26,7 @@ impl Address { } pub fn from_pub_key(pub_key: &PublicKey) -> Self { - let hashed = hash(pub_key.as_ref()); - let mut addr = Address::empty(); - addr.0.copy_from_slice(&hashed); - addr + pub_key_to_address(pub_key) } pub fn from_bytes(bytes: &[u8]) -> Result { diff --git a/types/src/lib.rs b/types/src/lib.rs index fa7ae9de..4aef6305 100644 --- a/types/src/lib.rs +++ b/types/src/lib.rs @@ -6,6 +6,8 @@ pub use block::{Block, Finalized, Notarized}; use commonware_cryptography::{Ed25519, Scheme}; use commonware_utils::SystemTimeExt; use std::time::SystemTime; +use sha3::{Digest, Keccak256}; + mod consensus; pub use consensus::{leader_index, Finalization, Kind, Notarization, Nullification, Seed}; pub mod account; @@ -19,6 +21,7 @@ pub mod wallet; pub mod wasm; use rand::rngs::OsRng; +use crate::address::Address; // We don't use functions here to guard against silent changes. pub const NAMESPACE: &[u8] = b"_ALTO"; @@ -50,6 +53,17 @@ pub fn curr_timestamp() -> u64 { SystemTime::now().epoch_millis() } +// returns an address from pubkey provided +fn pub_key_to_address(pk: &PublicKey) -> Address { + let pk_hash = Keccak256::digest(pk.as_bytes()); + + // Return the full 32-byte Keccak256 hash as the address + let mut address = [0u8; 32]; + address.copy_from_slice(&pk_hash); + Address(address) +} + + #[cfg(test)] mod tests { use super::*; From 9a9043f3af4b46e8a6a51fc9e6f914cde8d93834 Mon Sep 17 00:00:00 2001 From: manojkgorle Date: Fri, 4 Apr 2025 00:51:14 +0530 Subject: [PATCH 42/43] some changes --- chain/src/actors/application/actor.rs | 31 +++++++++++++----------- types/src/account.rs | 2 +- types/src/lib.rs | 7 +++--- types/src/signed_tx.rs | 35 ++++++++++----------------- types/src/tx.rs | 11 +++++---- 5 files changed, 40 insertions(+), 46 deletions(-) diff --git a/chain/src/actors/application/actor.rs b/chain/src/actors/application/actor.rs index b670faab..4336154e 100644 --- a/chain/src/actors/application/actor.rs +++ b/chain/src/actors/application/actor.rs @@ -10,7 +10,10 @@ use alto_storage::{ state_db::StateViewDb, transactional_db::{Key, Op, OpAction}, }; -use alto_types::{account::Account, address::Address, Block, Finalization, Notarization, Seed, signed_tx::unpack_signed_txs}; +use alto_types::{ + account::Account, address::Address, signed_tx::unpack_signed_txs, Block, Finalization, + Notarization, Seed, +}; use alto_vm::vm::VM; use commonware_codec::{Codec, WriteBuffer}; use commonware_consensus::threshold_simplex::Prover; @@ -27,7 +30,9 @@ use futures::{ }; use rand::Rng; use std::{ - collections::HashMap, pin::Pin, sync::{Arc, Mutex} + collections::HashMap, + pin::Pin, + sync::{Arc, Mutex}, }; use tracing::{info, warn}; @@ -208,7 +213,6 @@ impl Actor { let mut built = built.lock().unwrap(); *built = Some(block); } - // Send the digest to the consensus let result = response.send(digest.clone()); info!(view, ?digest, success=result.is_ok(), "proposed new block"); @@ -297,7 +301,7 @@ impl Actor { // apply transactions. let results = executor_vm.apply(stxs.clone()); // state root generation. - let dummy_state_root = [0u8;32]; //@todo + let dummy_state_root = [0u8;32]; //@todo // verify state root equivalence. if block.state_root != dummy_state_root.into() { let _ = response.send(false); @@ -336,30 +340,29 @@ impl Actor { let seed = Seed::new(view, seed.into()); // @todo syncer does the heavy lifting of post finalization processing. - if let Some(at_view_touched) = self.unfinalized_state.lock().unwrap().remove(&view){ - if at_view_touched.is_empty(){ + if let Some(at_view_touched) = + self.unfinalized_state.lock().unwrap().remove(&view) + { + if at_view_touched.is_empty() { info!(view, "finalized block with no touched keys"); - }else{ + } else { // lock state database. let mut s_db = self.state_db.lock().unwrap(); // iterate over the touched keys and write to the state database. - for (key, op) in at_view_touched.iter(){ + for (key, op) in at_view_touched.iter() { match op.action { OpAction::Update => { let _ = s_db.put(key, &op.value); - }, + } OpAction::Delete => { let _ = s_db.delete(key); - }, - _ =>{ - /*nothing to do with the database. */ } + _ => { /*nothing to do with the database. */ } } } info!(view, "finalized block with touched keys"); } - }else{ - + } else { } // Send the finalization to the syncer diff --git a/types/src/account.rs b/types/src/account.rs index cc6fe3e9..a69d2be4 100644 --- a/types/src/account.rs +++ b/types/src/account.rs @@ -1,7 +1,7 @@ use crate::address::Address; +use crate::pub_key_to_address; use commonware_codec::{Codec, Error, Reader, Writer}; use commonware_cryptography::ed25519::PublicKey; -use crate::pub_key_to_address; pub type Balance = u64; diff --git a/types/src/lib.rs b/types/src/lib.rs index 4aef6305..d290dcd0 100644 --- a/types/src/lib.rs +++ b/types/src/lib.rs @@ -5,8 +5,8 @@ mod block; pub use block::{Block, Finalized, Notarized}; use commonware_cryptography::{Ed25519, Scheme}; use commonware_utils::SystemTimeExt; -use std::time::SystemTime; use sha3::{Digest, Keccak256}; +use std::time::SystemTime; mod consensus; pub use consensus::{leader_index, Finalization, Kind, Notarization, Nullification, Seed}; @@ -20,8 +20,8 @@ pub mod units; pub mod wallet; pub mod wasm; -use rand::rngs::OsRng; use crate::address::Address; +use rand::rngs::OsRng; // We don't use functions here to guard against silent changes. pub const NAMESPACE: &[u8] = b"_ALTO"; @@ -55,7 +55,7 @@ pub fn curr_timestamp() -> u64 { // returns an address from pubkey provided fn pub_key_to_address(pk: &PublicKey) -> Address { - let pk_hash = Keccak256::digest(pk.as_bytes()); + let pk_hash = Keccak256::digest(pk.as_ref()); // Return the full 32-byte Keccak256 hash as the address let mut address = [0u8; 32]; @@ -63,7 +63,6 @@ fn pub_key_to_address(pk: &PublicKey) -> Address { Address(address) } - #[cfg(test)] mod tests { use super::*; diff --git a/types/src/signed_tx.rs b/types/src/signed_tx.rs index e0a70613..89d945ac 100644 --- a/types/src/signed_tx.rs +++ b/types/src/signed_tx.rs @@ -16,7 +16,6 @@ pub struct SignedTx { // function names are self explanatory. pub trait SignedTxChars: Sized { fn new(tx: Tx, pub_key: PublicKey, signature: Vec) -> Self; - // fn sign(&mut self, wallet: Wallet) -> SignedTx; fn verify(&mut self) -> bool; fn signature(&self) -> Vec; fn public_key(&self) -> Vec; @@ -73,8 +72,6 @@ impl SignedTxChars for SignedTx { // @todo add syntactic checks and use methods consume. fn decode(bytes: &[u8]) -> Result { - // @todo this method seems untidy. - let raw_tx_len = u64::from_be_bytes(bytes[0..8].try_into().unwrap()); let raw_tx = &bytes[8..8 + raw_tx_len as usize]; let pub_key = &bytes[8 + raw_tx_len as usize..8 + raw_tx_len as usize + 32]; @@ -143,17 +140,17 @@ pub fn unpack_signed_txs(bytes: Vec) -> Vec { #[cfg(test)] mod tests { - use std::error::Error; use super::*; + use crate::curr_timestamp; use crate::tx::Unit; use crate::units::transfer::Transfer; - use crate::{create_test_keypair, curr_timestamp}; + use crate::wallet::Wallet; use commonware_cryptography::sha256::Digest; use more_asserts::assert_gt; #[test] - fn test_encode_decode() -> Result<(), Box> { + fn test_encode_decode() { let timestamp = curr_timestamp(); let max_fee = 100; let priority_fee = 75; @@ -162,9 +159,7 @@ mod tests { let units: Vec> = vec![Box::new(transfer)]; let digest: [u8; 32] = [0; 32]; let id = Digest::from(digest.clone()); - let (pk, _sk) = create_test_keypair(); - // TODO: the .encode call on next line gave error and said origin_msg needed to be mut? but why? - // shouldn't encode be able to encode without changing the msg? + let tx = Tx { timestamp, max_fee, @@ -172,22 +167,18 @@ mod tests { chain_id, units: units.clone(), id, - digest: digest.to_vec(), + digest: vec![], actor: Address::empty(), }; - let mut origin_msg = SignedTx { - tx, - pub_key: pk, - address: Address::create_random_address(), - signature: vec![], - }; - let encoded_bytes = origin_msg.encode(); + let mut rng = rand::rngs::OsRng; + let wallet = Wallet::generate(&mut rng); + let mut signed_tx = SignedTx::sign(tx, wallet); + let encoded_bytes = signed_tx.encode(); assert_gt!(encoded_bytes.len(), 0); - let decoded_msg = SignedTx::decode(&encoded_bytes)?; - assert_eq!(origin_msg.pub_key, decoded_msg.pub_key); - assert_eq!(origin_msg.address, decoded_msg.address); - assert_eq!(origin_msg.signature, decoded_msg.signature); + let decoded_msg = SignedTx::decode(&encoded_bytes).unwrap(); + assert_eq!(signed_tx.pub_key, decoded_msg.pub_key); + assert_eq!(signed_tx.address, decoded_msg.address); + assert_eq!(signed_tx.signature, decoded_msg.signature); // @todo make helper to compare fields in tx and units. same issue when testing in tx.rs file. - Ok(()) } } diff --git a/types/src/tx.rs b/types/src/tx.rs index e34e51af..b75ac57d 100644 --- a/types/src/tx.rs +++ b/types/src/tx.rs @@ -201,7 +201,7 @@ impl TxMethods for Tx { } fn encode(&mut self) -> Vec { - if self.digest.is_empty() { + if !self.digest.is_empty() { return self.digest.clone(); } // pack tx timestamp. @@ -222,7 +222,7 @@ impl TxMethods for Tx { // pack len of inidividual unit. self.digest.extend((unit_bytes.len() as u64).to_be_bytes()); // pack individual unit. - self.digest.extend_from_slice(&unit_bytes); + self.digest.extend(&unit_bytes); }); // generate tx id. @@ -237,7 +237,6 @@ impl TxMethods for Tx { return Err("Empty bytes".to_string()); } let mut tx = Self::default(); - tx.digest = bytes.to_vec(); // @todo ?? tx.timestamp = u64::from_be_bytes(bytes[0..8].try_into().unwrap()); tx.max_fee = u64::from_be_bytes(bytes[8..16].try_into().unwrap()); tx.priority_fee = u64::from_be_bytes(bytes[16..24].try_into().unwrap()); @@ -248,7 +247,8 @@ impl TxMethods for Tx { } tx.units = units?; // generate tx id. - tx.id = sha256::hash(&tx.digest); + tx.id = sha256::hash(bytes); + tx.digest = bytes.to_vec(); // return transaction. Ok(tx) } @@ -334,6 +334,7 @@ mod tests { use crate::units::transfer::Transfer; use more_asserts::assert_gt; use std::error::Error; + use std::vec; #[test] fn test_encode_decode() -> Result<(), Box> { @@ -355,7 +356,7 @@ mod tests { units: units.clone(), id, actor: Address::empty(), - digest: digest.to_vec(), + digest: vec![], }; let encoded_bytes = origin_msg.encode(); assert_gt!(encoded_bytes.len(), 0); From 473385327c1110cd644d87c1257ed62a0cc51617 Mon Sep 17 00:00:00 2001 From: manojkgorle Date: Fri, 4 Apr 2025 01:33:46 +0530 Subject: [PATCH 43/43] fixes --- chain/src/actors/application/actor.rs | 5 ++--- chain/src/bin/validator.rs | 2 +- storage/src/state_db.rs | 11 +++++++++-- types/src/wallet.rs | 1 + vm/src/vm.rs | 5 +++-- 5 files changed, 16 insertions(+), 8 deletions(-) diff --git a/chain/src/actors/application/actor.rs b/chain/src/actors/application/actor.rs index 4336154e..d0f0053b 100644 --- a/chain/src/actors/application/actor.rs +++ b/chain/src/actors/application/actor.rs @@ -146,7 +146,7 @@ impl Actor { let mut write_buf = WriteBuffer::new(DB_WRITE_BUFFER_CAPACITY); let acc = Account { address: adrs, - balance: allo.clone(), + balance: *allo, }; // write the account to the buffer. acc.write(&mut write_buf); @@ -362,8 +362,7 @@ impl Actor { } info!(view, "finalized block with touched keys"); } - } else { - } + } // Send the finalization to the syncer syncer.finalized(finalization, seed).await; diff --git a/chain/src/bin/validator.rs b/chain/src/bin/validator.rs index 7fcd1377..4821521e 100644 --- a/chain/src/bin/validator.rs +++ b/chain/src/bin/validator.rs @@ -218,7 +218,7 @@ fn main() { indexer, chain_id: 10, state_db: wrapped_state_db, - genesis: genesis, + genesis, }; let engine = engine::Engine::new(context.with_label("engine"), config).await; diff --git a/storage/src/state_db.rs b/storage/src/state_db.rs index 47699d23..676dd407 100644 --- a/storage/src/state_db.rs +++ b/storage/src/state_db.rs @@ -40,13 +40,20 @@ impl StateView for StateViewDb<'_> { } fn get_balance(&mut self, address: &Address) -> Option { + info!("Getting balance for address: {}", address); match self.get_account(address) { // return balance if account exists Ok(Some(acc)) => Some(acc.balance), // return 0 if no account - Ok(None) => Some(0), + Ok(None) => { + info!("Account not found, returning 0 balance"); + Some(0) + } // return none if an err occurred - Err(_) => None, + Err(e) => { + warn!("Error getting account: {}", e); + None + } } } diff --git a/types/src/wallet.rs b/types/src/wallet.rs index 6cd03d2e..e9e17908 100644 --- a/types/src/wallet.rs +++ b/types/src/wallet.rs @@ -24,6 +24,7 @@ pub trait Auth { } /// Wallet is the module used by the user to sign transactions. Wallet uses Ed25519 signature scheme. +#[derive(Clone)] pub struct Wallet { // Private key priv_key: PrivateKey, diff --git a/vm/src/vm.rs b/vm/src/vm.rs index 8fd6553d..5ce8af70 100644 --- a/vm/src/vm.rs +++ b/vm/src/vm.rs @@ -12,7 +12,7 @@ use alto_storage::{ use alto_types::null_error::NullError; use alto_types::state_view::StateView; use alto_types::{ - signed_tx::SignedTx, + signed_tx::{SignedTx, SignedTxChars}, tx::{Tx, TxMethods, TxResult, UnitContext}, }; @@ -58,7 +58,8 @@ impl VM { let mut results = Vec::new(); for stx in stxs { let mut state_view = StateViewDb::new(&mut in_mem_db); - let tx = stx.tx; + let mut tx = stx.tx.clone(); + tx.set_actor(stx.address()); let result = self.apply_tx(tx, &mut state_view); if result.status { let _ = in_mem_db.commit_last_tx();