chore: improve/fix ledger tooling (#294)

## Summary

Fix: prevent account file deletion during ledger replay and ledger tool
access
Feat: genx supports many more accounts loaded into test-validator than
before

## Details

### Fix

When an `AppendVec` is dropped the underlying file is removed. This
results in an account file
being deleted each time we replay the ledger or access the accounts via
a ledger tool, i.e.
`ledger-stats` or `genx`.

After attempting to fix this by setting a flag to prevent the file from
being removed, I
resorted to just copying the account file before using it as underlying
storage for the
`AppendVec`.
This copy is cleaned up when the tool or ledger replay completes

### Feat

Instead of passing accounts to be loaded from chain as arguments when
starting the validator,
_genx_ now downloads the accounts into a directory (100 in parallel
which is the limit) and
provides that directory to the validator in the generated script
instead.


<!-- greptile_comment -->

## Greptile Summary

This PR improves ledger tooling by addressing account file persistence
and enhancing account loading capabilities in the test validator
environment.

- Added file copying mechanism in `load_most_recent_store` to prevent
account file deletion when `AppendVec` is dropped
- Implemented batch downloading of accounts (100 at a time) in
`tools/genx/src/test_validator.rs` to work within RPC limits
- Added JSON serialization support for account data storage with new
dependencies in `tools/genx/Cargo.toml`
- Modified test validator to use `--account-dir` instead of
`--maybe-clone` for more efficient account loading
- Introduced temporary directory management for storing downloaded
account data during validator operation



<!-- /greptile_comment -->

Author: thlorenz

Cargo.lock

@@ -316,6 +316,17 @@ impl AccountsPersister {
             return Ok(None);
         };
 
+        // When we drop the AppendVec the underlying file is removed from the
+        // filesystem. There is no way to configure this via public methods.
+        // Thus we copy the file before using it for the AppendVec. This way
+        // we prevent account files being removed when we point a tool at the ledger
+        // or replay it.
+        let file = {
+            let copy = file.with_extension("copy");
+            fs::copy(&file, &copy)?;
+            copy
+        };
+
         // Create a AccountStorageEntry from the file
         let file_size = fs::metadata(&file)?.len() as usize;
         let (append_vec, num_accounts) =

magicblock-accounts-db/src/persist/accounts_persister.rs

@@ -8,8 +8,11 @@ license.workspace = true
 edition.workspace = true
 
 [dependencies]
+base64 = { workspace = true }
 clap = { version = "4.5.23", features = ["derive"] }
 ledger-stats = { workspace = true }
 magicblock-accounts-db = { workspace = true }
+serde_json = { workspace = true }
+solana-rpc-client = { workspace = true }
 solana-sdk = { workspace = true }
 tempfile = { workspace = true }

test-integration/Cargo.lock

@@ -1,8 +1,14 @@
-use std::{fs, os::unix::fs::PermissionsExt, path::PathBuf};
+use serde_json::{json, Value};
+use solana_rpc_client::rpc_client::RpcClient;
+use std::{
+    fs,
+    os::unix::fs::PermissionsExt,
+    path::{Path, PathBuf},
+};
 
 use ledger_stats::{accounts_storage_from_ledger, open_ledger};
 use magicblock_accounts_db::utils::all_accounts;
-use solana_sdk::pubkey::Pubkey;
+use solana_sdk::{commitment_config::CommitmentConfig, pubkey::Pubkey};
 use tempfile::tempdir;
 
 pub struct TestValidatorConfig {
@@ -16,6 +22,9 @@ pub(crate) fn gen_test_validator_start_script(
 ) {
     let temp_dir = tempdir().expect("Failed to create temporary directory");
     let temp_dir_path = temp_dir.into_path();
+    let accounts_dir = temp_dir_path.join("accounts");
+    fs::create_dir(&accounts_dir).expect("Failed to create accounts directory");
+
     let file_path = temp_dir_path.join("run-validator.sh");
     let accounts: Vec<Pubkey> = if let Some(ledger_path) = ledger_path {
         let ledger = open_ledger(ledger_path);
@@ -39,16 +48,14 @@ pub(crate) fn gen_test_validator_start_script(
         "10000".to_string(),
     ];
 
-    for pubkey in accounts {
-        // NOTE: we may need to treat executables differently if just cloning
-        // at startup is not sufficient even though we also will clone the
-        // executable data account.
-        // For now we don't run programs in the test validator, but just
-        // want to make sure they can be cloned by the ephemeral, so it is not
-        // yet important.
-        args.push("--maybe-clone".to_string());
-        args.push(pubkey.to_string());
-    }
+    download_accounts_into_from_url_into_dir(
+        &accounts,
+        config.url.clone(),
+        &accounts_dir,
+    );
+
+    args.push("--account-dir".into());
+    args.push(accounts_dir.to_string_lossy().to_string());
 
     args.push("--url".into());
     args.push(config.url);
@@ -71,3 +78,54 @@ solana-test-validator  \\\n  {}",
         file_path.display()
     );
 }
+
+fn download_accounts_into_from_url_into_dir(
+    pubkeys: &[Pubkey],
+    url: String,
+    dir: &Path,
+) {
+    // Derived from error from helius RPC: Failed to download accounts: Error { request: Some(GetMultipleAccounts), kind: RpcError(RpcResponseError { code: -32602, message: "Too many inputs provided; max 100", data: Empty }) }
+    const MAX_ACCOUNTS: usize = 100;
+    let rpc_client =
+        RpcClient::new_with_commitment(url, CommitmentConfig::confirmed());
+    let total_len = pubkeys.len();
+    for (idx, pubkeys) in pubkeys.chunks(MAX_ACCOUNTS).enumerate() {
+        let start = idx * MAX_ACCOUNTS;
+        let end = start + pubkeys.len();
+        eprintln!("Downloading {}..{}/{} accounts", start, end, total_len);
+        match rpc_client.get_multiple_accounts(pubkeys) {
+            Ok(accs) => accs
+                .into_iter()
+                .zip(pubkeys)
+                .filter_map(|(acc, pubkey)| acc.map(|x| (x, pubkey)))
+                .for_each(|(acc, pubkey)| {
+                    let path = dir.join(format!("{pubkey}.json"));
+                    let pk = pubkey.to_string();
+                    let lamports = acc.lamports;
+                    let data = [
+                        #[allow(deprecated)] // this is just a dev tool
+                        base64::encode(&acc.data),
+                        "base64".to_string(),
+                    ];
+                    let owner = acc.owner.to_string();
+                    let executable = acc.executable;
+                    let rent_epoch = acc.rent_epoch;
+                    let space = acc.data.len();
+                    let json: Value = json! {{
+                        "pubkey": pk,
+                        "account": {
+                            "lamports": lamports,
+                            "data": data,
+                            "owner": owner,
+                            "executable": executable,
+                            "space": space,
+                            "rentEpoch": rent_epoch
+                        },
+                    }};
+                    fs::write(&path, format!("{:#}", json))
+                        .expect("Failed to write account");
+                }),
+            Err(err) => eprintln!("Failed to download accounts: {:?}", err),
+        }
+    }
+}