feat: set default ledger path (#521)

Closes #520 

Replaying a ledger without providing a path created a new, empty ledger
without a validator and faucet keypairs, which caused errors.

This PR sets a default ledger path and creates the corresponding key
pairs if they are missing.

Author: Dodecahedr0x

.gitignore

@@ -18,6 +18,7 @@ target/
 
 # Ledger
 test-ledger/
+test-ledger-magicblock/
 
 # Mac
 **/.DS_Store

magicblock-api/src/fund_account.rs

@@ -1,7 +1,6 @@
 use std::path::Path;
 
 use magicblock_bank::bank::Bank;
-use magicblock_config::LedgerResumeStrategy;
 use magicblock_core::magic_program;
 use solana_sdk::{
     account::Account, clock::Epoch, pubkey::Pubkey, signature::Keypair,
@@ -57,14 +56,14 @@ pub(crate) fn fund_validator_identity(bank: &Bank, validator_id: &Pubkey) {
 pub(crate) fn funded_faucet(
     bank: &Bank,
     ledger_path: &Path,
-    resume_strategy: &LedgerResumeStrategy,
 ) -> ApiResult<Keypair> {
-    let faucet_keypair = if resume_strategy.is_removing_ledger() {
-        let faucet_keypair = Keypair::new();
-        write_faucet_keypair_to_ledger(ledger_path, &faucet_keypair)?;
-        faucet_keypair
-    } else {
-        read_faucet_keypair_from_ledger(ledger_path)?
+    let faucet_keypair = match read_faucet_keypair_from_ledger(ledger_path) {
+        Ok(faucet_keypair) => faucet_keypair,
+        Err(_) => {
+            let faucet_keypair = Keypair::new();
+            write_faucet_keypair_to_ledger(ledger_path, &faucet_keypair)?;
+            faucet_keypair
+        }
     };
 
     fund_account(bank, &faucet_keypair.pubkey(), u64::MAX / 2);

magicblock-api/src/ledger.rs

@@ -17,38 +17,33 @@ use crate::errors::{ApiError, ApiResult};
 // Init
 // -----------------
 pub(crate) fn init(
-    ledger_path: PathBuf,
+    ledger_path: &Path,
     ledger_config: &LedgerConfig,
 ) -> ApiResult<(Ledger, Slot)> {
     let resume_strategy = &ledger_config.resume_strategy();
     let starting_slot = match resume_strategy {
         LedgerResumeStrategy::Resume { .. } => {
-            let previous_ledger = Ledger::open(ledger_path.as_path())?;
+            let previous_ledger = Ledger::open(ledger_path)?;
             previous_ledger.get_max_blockhash().map(|(slot, _)| slot)?
         }
         LedgerResumeStrategy::Reset { slot, .. } => *slot,
     };
 
     if resume_strategy.is_removing_ledger() {
-        remove_ledger_directory_if_exists(
-            ledger_path.as_path(),
-            resume_strategy,
-        )
-        .map_err(|err| {
-            error!(
-                "Error: Unable to remove {}: {}",
-                ledger_path.display(),
-                err
-            );
-            ApiError::UnableToCleanLedgerDirectory(
-                ledger_path.display().to_string(),
-            )
-        })?;
+        remove_ledger_directory_if_exists(ledger_path, resume_strategy)
+            .map_err(|err| {
+                error!(
+                    "Error: Unable to remove {}: {}",
+                    ledger_path.display(),
+                    err
+                );
+                ApiError::UnableToCleanLedgerDirectory(
+                    ledger_path.display().to_string(),
+                )
+            })?;
     }
 
-    fs::create_dir_all(&ledger_path)?;
-
-    Ok((Ledger::open(ledger_path.as_path())?, starting_slot))
+    Ok((Ledger::open(ledger_path)?, starting_slot))
 }
 
 // -----------------

magicblock-api/src/magic_validator.rs

@@ -1,6 +1,6 @@
 use std::{
     net::SocketAddr,
-    path::{Path, PathBuf},
+    path::Path,
     process,
     sync::{
         atomic::{AtomicBool, Ordering},
@@ -89,7 +89,6 @@ use solana_sdk::{
     signature::Keypair,
     signer::Signer,
 };
-use tempfile::TempDir;
 use tokio_util::sync::CancellationToken;
 
 use crate::{
@@ -210,13 +209,12 @@ impl MagicValidator {
             Self::init_ledger(&config.validator_config.ledger)?;
         info!("Starting slot: {}", starting_slot);
 
-        if !config.validator_config.ledger.skip_keypair_match_check {
-            Self::sync_validator_keypair_with_ledger(
-                ledger.ledger_path(),
-                &identity_keypair,
-                ledger_resume_strategy,
-            )?;
-        }
+        Self::sync_validator_keypair_with_ledger(
+            ledger.ledger_path(),
+            &identity_keypair,
+            ledger_resume_strategy,
+            config.validator_config.ledger.skip_keypair_match_check,
+        )?;
 
         // SAFETY:
         // this code will never panic as the ledger_path always appends the
@@ -249,11 +247,8 @@ impl MagicValidator {
 
         fund_validator_identity(&bank, &validator_pubkey);
         fund_magic_context(&bank);
-        let faucet_keypair = funded_faucet(
-            &bank,
-            ledger.ledger_path().as_path(),
-            ledger_resume_strategy,
-        )?;
+        let faucet_keypair =
+            funded_faucet(&bank, ledger.ledger_path().as_path())?;
 
         load_programs_into_bank(
             &bank,
@@ -554,13 +549,7 @@ impl MagicValidator {
     fn init_ledger(
         ledger_config: &LedgerConfig,
     ) -> ApiResult<(Arc<Ledger>, Slot)> {
-        let ledger_path = match &ledger_config.path {
-            Some(ledger_path) => PathBuf::from(ledger_path),
-            None => {
-                let ledger_path = TempDir::new()?;
-                ledger_path.path().to_path_buf()
-            }
-        };
+        let ledger_path = Path::new(&ledger_config.path);
         let (ledger, last_slot) = ledger::init(ledger_path, ledger_config)?;
         let ledger_shared = Arc::new(ledger);
         init_persister(ledger_shared.clone());
@@ -571,21 +560,27 @@ impl MagicValidator {
         ledger_path: &Path,
         validator_keypair: &Keypair,
         resume_strategy: &LedgerResumeStrategy,
+        skip_keypair_match_check: bool,
     ) -> ApiResult<()> {
         if resume_strategy.is_removing_ledger() {
             write_validator_keypair_to_ledger(ledger_path, validator_keypair)?;
-        } else {
-            let ledger_validator_keypair =
-                read_validator_keypair_from_ledger(ledger_path)?;
-            if ledger_validator_keypair.ne(validator_keypair) {
+        } else if let Ok(ledger_validator_keypair) =
+            read_validator_keypair_from_ledger(ledger_path)
+        {
+            if ledger_validator_keypair.ne(validator_keypair)
+                && !skip_keypair_match_check
+            {
                 return Err(
                     ApiError::LedgerValidatorKeypairNotMatchingProvidedKeypair(
                         ledger_path.display().to_string(),
                         ledger_validator_keypair.pubkey().to_string(),
                     ),
                 );
             }
+        } else {
+            write_validator_keypair_to_ledger(ledger_path, validator_keypair)?;
         }
+
         Ok(())
     }
 

magicblock-config/src/ledger.rs

@@ -34,12 +34,11 @@ pub struct LedgerConfig {
     /// The file system path onto which the ledger should be written at
     /// If left empty it will be auto-generated to a temporary folder
     #[derive_env_var]
-    #[clap_from_serde_skip]
     #[arg(
         help = "The file system path onto which the ledger should be written at."
     )]
-    #[serde(default)]
-    pub path: Option<String>,
+    #[serde(default = "default_ledger_path")]
+    pub path: String,
     /// The size under which it's desired to keep ledger in bytes.
     #[derive_env_var]
     #[arg(help = "The size under which it's desired to keep ledger in bytes.")]
@@ -75,7 +74,7 @@ impl Default for LedgerConfig {
         Self {
             resume_strategy_config: LedgerResumeStrategyConfig::default(),
             skip_keypair_match_check: false,
-            path: Default::default(),
+            path: default_ledger_path(),
             size: DEFAULT_LEDGER_SIZE_BYTES,
         }
     }
@@ -226,6 +225,10 @@ const fn default_cloning_concurrency() -> usize {
     10
 }
 
+fn default_ledger_path() -> String {
+    "test-ledger-magicblock".to_string()
+}
+
 #[cfg(test)]
 mod tests {
     use magicblock_config_helpers::Merge;
@@ -284,7 +287,7 @@ mod tests {
                 account_hydration_concurrency: 20,
             },
             skip_keypair_match_check: true,
-            path: Some("ledger.example.com".to_string()),
+            path: "ledger.example.com".to_string(),
             size: 1000000000,
         };
         let original_config = config.clone();
@@ -306,7 +309,7 @@ mod tests {
                 account_hydration_concurrency: 20,
             },
             skip_keypair_match_check: true,
-            path: Some("ledger.example.com".to_string()),
+            path: "ledger.example.com".to_string(),
             size: 1000000000,
         };
 
@@ -325,7 +328,7 @@ mod tests {
                 account_hydration_concurrency: 20,
             },
             skip_keypair_match_check: true,
-            path: Some("ledger.example.com".to_string()),
+            path: "ledger.example.com".to_string(),
             size: 1000000000,
         };
         let original_config = config.clone();
@@ -337,7 +340,7 @@ mod tests {
                 account_hydration_concurrency: 150,
             },
             skip_keypair_match_check: true,
-            path: Some("ledger2.example.com".to_string()),
+            path: "ledger2.example.com".to_string(),
             size: 10000,
         };
 
@@ -368,7 +371,7 @@ size = 1000000000
                     ),
                 },
                 skip_keypair_match_check: true,
-                path: Some("ledger.example.com".to_string()),
+                path: "ledger.example.com".to_string(),
                 size: 1000000000,
             }
         );
@@ -391,7 +394,7 @@ size = 1000000000
                     ),
                 },
                 skip_keypair_match_check: false,
-                path: None,
+                path: default_ledger_path(),
                 size: 1000000000,
             }
         );
@@ -414,7 +417,7 @@ size = 1000000000
                     ),
                 },
                 skip_keypair_match_check: false,
-                path: None,
+                path: "test-ledger-magicblock".to_string(),
                 size: 1000000000,
             }
         );

magicblock-config/src/lib.rs

@@ -257,7 +257,7 @@ mod tests {
                     account_hydration_concurrency: 20,
                 },
                 skip_keypair_match_check: true,
-                path: Some("ledger.example.com".to_string()),
+                path: "ledger.example.com".to_string(),
                 size: 1000000000,
             },
             programs: vec![ProgramConfig {
@@ -346,7 +346,7 @@ mod tests {
                     account_hydration_concurrency: 20,
                 },
                 skip_keypair_match_check: true,
-                path: Some("ledger.example.com".to_string()),
+                path: "ledger.example.com".to_string(),
                 size: 1000000000,
             },
             programs: vec![ProgramConfig {
@@ -432,7 +432,7 @@ mod tests {
                     account_hydration_concurrency: 20,
                 },
                 skip_keypair_match_check: true,
-                path: Some("ledger2.example.com".to_string()),
+                path: "ledger2.example.com".to_string(),
                 size: 100000,
             },
             programs: vec![ProgramConfig {
@@ -511,7 +511,7 @@ mod tests {
                     account_hydration_concurrency: 20,
                 },
                 skip_keypair_match_check: true,
-                path: Some("ledger.example.com".to_string()),
+                path: "ledger.example.com".to_string(),
                 size: 1000000000,
             },
             programs: vec![ProgramConfig {
@@ -569,7 +569,7 @@ mod tests {
                     account_hydration_concurrency: 20,
                 },
                 skip_keypair_match_check: true,
-                path: Some("ledger.example.com".to_string()),
+                path: "ledger.example.com".to_string(),
                 size: 1000000000,
             },
             programs: vec![],

magicblock-config/tests/parse_config.rs

@@ -274,7 +274,7 @@ fn test_everything_defined() {
                     account_hydration_concurrency: 20,
                 },
                 skip_keypair_match_check: true,
-                path: Some("ledger.example.com".to_string()),
+                path: "ledger.example.com".to_string(),
                 size: 1_000_000_000,
             },
             programs: vec![ProgramConfig {

magicblock-config/tests/read_config.rs

@@ -199,7 +199,7 @@ fn test_load_local_dev_with_programs_toml_envs_override() {
                     account_hydration_concurrency: 20,
                 },
                 skip_keypair_match_check: true,
-                path: Some("/hello/world".to_string()),
+                path: "/hello/world".to_string(),
                 size: 123123,
             },
             metrics: MetricsConfig {

test-integration/test-ledger-restore/src/lib.rs

@@ -61,7 +61,7 @@ pub fn setup_offline_validator(
         ledger: LedgerConfig {
             resume_strategy_config: resume_strategy.into(),
             skip_keypair_match_check,
-            path: Some(ledger_path.display().to_string()),
+            path: ledger_path.display().to_string(),
             size: DEFAULT_LEDGER_SIZE_BYTES,
         },
         accounts: accounts_config.clone(),
@@ -121,7 +121,7 @@ pub fn setup_validator_with_local_remote(
         ledger: LedgerConfig {
             resume_strategy_config,
             skip_keypair_match_check,
-            path: Some(ledger_path.display().to_string()),
+            path: ledger_path.display().to_string(),
             size: DEFAULT_LEDGER_SIZE_BYTES,
         },
         accounts: accounts_config.clone(),