Change account layout (#451)

Modify account layout in accounts database:
1. reuse slack space which is available in block allocation if possible,
   this avoids reallocation of accounts if the data field grows
   insignificantly.
2. track account owner change to avoid index checks in happy path

depends on #419 
closes #443
closes #441  

<!-- greptile_comment -->

## Greptile Summary

Major refactoring of AccountsDB storage and indexing system to optimize
account data storage and improve performance through block space reuse
and owner tracking.

- Consolidated multiple LMDB environments into a single environment and
introduced new `Table` abstraction in
`magicblock-accounts-db/src/index/table.rs` for better database
operation encapsulation
- Modified account serialization in `magicblock-accounts-db/src/lib.rs`
to reuse slack space in block allocations, reducing unnecessary
reallocations for small data growth
- Added owner change tracking to optimize index checks by skipping them
in the happy path when owner hasn't changed
- Improved LMDB iteration safety and efficiency in
`magicblock-accounts-db/src/index/iterator.rs` by removing manual cursor
management
- Changed program account iteration behavior to return empty iterators
instead of NotFound errors for better API consistency



<!-- /greptile_comment -->

Author: bmuddha

Cargo.lock

@@ -142,7 +142,7 @@ serde = "1.0.217"
 serde_derive = "1.0"
 serde_json = "1.0"
 sha3 = "0.10.8"
-solana-account = { git = "https://github.com/magicblock-labs/solana-account.git", rev = "7bdfefc" }
+solana-account = { git = "https://github.com/magicblock-labs/solana-account.git", rev = "5b50fad" }
 solana-accounts-db = { version = "2.2" }
 solana-account-decoder = { version = "2.2" }
 solana-address-lookup-table-program = { version = "2.2" }
@@ -203,6 +203,6 @@ vergen = "8.3.1"
 # some solana dependencies have solana-storage-proto as dependency
 # we need to patch them with our version, because they use protobuf-src v1.1.0
 # and we use protobuf-src v2.1.1. Otherwise compilation fails
-solana-account = { git = "https://github.com/magicblock-labs/solana-account.git", rev = "7bdfefc" }
+solana-account = { git = "https://github.com/magicblock-labs/solana-account.git", rev = "5b50fad" }
 solana-storage-proto = { path = "./storage-proto" }
 solana-svm = { git = "https://github.com/magicblock-labs/magicblock-svm.git" }

Cargo.toml

@@ -106,6 +106,10 @@ impl AccountsDb {
                 // For borrowed variants everything is already written and we just increment the
                 // atomic counter. New readers will see the latest update.
                 acc.commit();
+                // check whether the account's owner has changed
+                if !acc.owner_changed {
+                    return;
+                }
                 // and perform some index bookkeeping to ensure correct owner
                 let _ = self
                     .index
@@ -116,11 +120,11 @@ impl AccountsDb {
                     ));
             }
             AccountSharedData::Owned(acc) => {
-                let datalen = account.data().len();
-                // we multiply by 2 for shadow buffer and add extra space for metadata
-                let size = AccountSharedData::serialized_size_aligned(datalen)
-                    * 2
-                    + AccountSharedData::SERIALIZED_META_SIZE;
+                let datalen = account.data().len() as u32;
+                let block_size = self.storage.block_size() as u32;
+                let size = AccountSharedData::serialized_size_aligned(
+                    datalen, block_size,
+                ) as usize;
 
                 let blocks = self.storage.get_block_count(size);
                 // TODO(bmuddha) perf optimization: use reallocs sparringly
@@ -143,13 +147,14 @@ impl AccountsDb {
                 };
 
                 // SAFETY:
-                // Allocation object is obtained by obtaining valid offset from storage, which
+                // Allocation object is constructed by obtaining a valid offset from storage, which
                 // is unoccupied by other accounts, points to valid memory within mmap and is
                 // properly aligned to 8 bytes, so the contract of serialize_to_mmap is satisfied
                 unsafe {
                     AccountSharedData::serialize_to_mmap(
                         acc,
                         allocation.storage.as_ptr(),
+                        block_size * allocation.blocks,
                     )
                 };
                 // update accounts index

magicblock-accounts-db/src/lib.rs

@@ -273,7 +273,7 @@ impl AccountsStorage {
             + METADATA_STORAGE_SIZE as u64
     }
 
-    fn block_size(&self) -> usize {
+    pub(crate) fn block_size(&self) -> usize {
         self.meta.block_size as usize
     }
 

magicblock-accounts-db/src/storage.rs

@@ -76,7 +76,7 @@ fn test_modify_account() {
 #[test]
 fn test_account_resize() {
     let tenv = init_test_env();
-    let huge_data = [42; SPACE * 2];
+    let huge_data = [42; SPACE * 4];
     let AccountWithPubkey {
         pubkey,
         account: mut uncommitted,
@@ -89,7 +89,7 @@ fn test_account_resize() {
     );
     assert_eq!(
         uncommitted.data().len(),
-        SPACE * 2,
+        SPACE * 4,
         "account should have been resized to double of SPACE"
     );
 
@@ -123,7 +123,7 @@ fn test_alloc_reuse() {
         pubkey,
         account: mut acc1,
     } = tenv.account();
-    let huge_data = [42; SPACE * 2];
+    let huge_data = [42; SPACE * 4];
 
     let old_addr = acc1.data().as_ptr();
 

magicblock-accounts-db/src/tests.rs

@@ -51,7 +51,7 @@ program-schedulecommit-security = { path = "programs/schedulecommit-security" }
 rayon = "1.10.0"
 schedulecommit-client = { path = "schedulecommit/client" }
 serde = "1.0.217"
-solana-account = { git = "https://github.com/magicblock-labs/solana-account.git", rev = "7bdfefc" }
+solana-account = { git = "https://github.com/magicblock-labs/solana-account.git", rev = "5b50fad" }
 solana-program = "2.2"
 solana-program-test = "2.2"
 solana-pubkey = { version = "2.2" }
@@ -73,4 +73,4 @@ tokio = "1.0"
 # and we use protobuf-src v2.1.1. Otherwise compilation fails
 solana-storage-proto = { path = "../storage-proto" }
 # same reason as above
-solana-account = { git = "https://github.com/magicblock-labs/solana-account.git", rev = "7bdfefc" }
+solana-account = { git = "https://github.com/magicblock-labs/solana-account.git", rev = "5b50fad" }