From 359db852546cd259ed12a52cd2c2cd47bf1e8e0a Mon Sep 17 00:00:00 2001 From: marta-lokhova Date: Tue, 16 Dec 2025 11:22:56 -0800 Subject: [PATCH 1/3] Simplify and speed up SorobanLoadGeneration --- src/FSLibrary/MissionLoadGeneration.fs | 2 +- src/FSLibrary/MissionSorobanLoadGeneration.fs | 38 +++++++------------ 2 files changed, 15 insertions(+), 25 deletions(-) diff --git a/src/FSLibrary/MissionLoadGeneration.fs b/src/FSLibrary/MissionLoadGeneration.fs index 996f6927..e7a7d3e3 100644 --- a/src/FSLibrary/MissionLoadGeneration.fs +++ b/src/FSLibrary/MissionLoadGeneration.fs @@ -16,7 +16,7 @@ let loadGeneration (context: MissionContext) = MakeLiveCoreSet "core" { CoreSetOptions.GetDefault context.image with - invariantChecks = AllInvariantsExceptBucketConsistencyChecksAndEvents + invariantChecks = AllInvariantsExceptEvents dumpDatabase = false } let context = diff --git a/src/FSLibrary/MissionSorobanLoadGeneration.fs b/src/FSLibrary/MissionSorobanLoadGeneration.fs index 374f47b5..665ec235 100644 --- a/src/FSLibrary/MissionSorobanLoadGeneration.fs +++ b/src/FSLibrary/MissionSorobanLoadGeneration.fs @@ -8,7 +8,6 @@ open StellarCoreSet open StellarMissionContext open StellarFormation open StellarStatefulSets -open StellarNetworkData open StellarSupercluster open StellarCoreHTTP @@ -16,41 +15,32 @@ open StellarCoreHTTP let sorobanLoadGeneration (context: MissionContext) = let rate = 5 + let coreSet = + MakeLiveCoreSet + "core" + { CoreSetOptions.GetDefault context.image with + invariantChecks = AllInvariantsExceptEvents + dumpDatabase = false } + let context = { context.WithSmallLoadgenOptions with - coreResources = SimulatePubnetTier1PerfResources - installNetworkDelay = Some(context.installNetworkDelay |> Option.defaultValue true) txRate = rate numAccounts = 10000 - numTxs = rate * 2000 + numTxs = rate * 200 skipLowFeeTxs = true maxFeeRate = Some 100000000 - enableTailLogging = false updateSorobanCosts = Some(true) genesisTestAccountCount = Some 10000 } - let fullCoreSet = FullPubnetCoreSets context true true - - let sdf = - List.find (fun (cs: CoreSet) -> cs.name.StringName = "stellar" || cs.name.StringName = "sdf") fullCoreSet - - let tier1 = List.filter (fun (cs: CoreSet) -> cs.options.tier1 = Some true) fullCoreSet - context.Execute - fullCoreSet + [ coreSet ] None (fun (formation: StellarFormation) -> - // Setup overlay connections first before manually closing - // ledger, which kick off consensus - formation.WaitUntilConnected fullCoreSet - formation.ManualClose tier1 - // Wait until the whole network is synced before proceeding, // to fail asap in case of a misconfiguration - formation.WaitUntilSynced fullCoreSet - formation.UpgradeProtocolToLatest tier1 - formation.UpgradeMaxTxSetSize tier1 1000000 + formation.WaitUntilSynced [ coreSet ] + formation.UpgradeProtocolToLatest [ coreSet ] + formation.UpgradeMaxTxSetSize [ coreSet ] 1000000 - formation.UpgradeSorobanLedgerLimitsWithMultiplier tier1 100 - formation.RunLoadgen sdf context.GenerateSorobanUploadLoad - formation.EnsureAllNodesInSync fullCoreSet) + formation.UpgradeSorobanLedgerLimitsWithMultiplier [ coreSet ] 100 + formation.RunLoadgen coreSet context.GenerateSorobanUploadLoad) From e2fc2d994d973e61d501de68e0d536a82b6524dc Mon Sep 17 00:00:00 2001 From: marta-lokhova Date: Tue, 16 Dec 2025 11:27:26 -0800 Subject: [PATCH 2/3] Strengthen invariants across all missions except pubnet/testnet --- src/FSLibrary/MissionLoadGenerationWithSpikes.fs | 2 +- src/FSLibrary/MissionLoadGenerationWithTxSetLimit.fs | 2 +- src/FSLibrary/MissionMixedImageLoadGeneration.fs | 4 ++-- src/FSLibrary/MissionMixedNominationLeaderElection.fs | 4 ++-- src/FSLibrary/MissionProtocolUpgradeWithLoad.fs | 2 +- src/FSLibrary/MissionSorobanCatchupWithPrevAndCurr.fs | 2 +- src/FSLibrary/MissionSorobanConfigUpgrades.fs | 2 +- src/FSLibrary/MissionSorobanInvokeHostLoad.fs | 2 +- src/FSLibrary/MissionUpgradeTxClusters.fs | 2 +- 9 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/FSLibrary/MissionLoadGenerationWithSpikes.fs b/src/FSLibrary/MissionLoadGenerationWithSpikes.fs index 031c65dd..090e7853 100644 --- a/src/FSLibrary/MissionLoadGenerationWithSpikes.fs +++ b/src/FSLibrary/MissionLoadGenerationWithSpikes.fs @@ -16,7 +16,7 @@ let loadGenerationWithSpikes (context: MissionContext) = MakeLiveCoreSet "core" { CoreSetOptions.GetDefault context.image with - invariantChecks = AllInvariantsExceptBucketConsistencyChecksAndEvents + invariantChecks = AllInvariantsExceptEvents dumpDatabase = false } let context = diff --git a/src/FSLibrary/MissionLoadGenerationWithTxSetLimit.fs b/src/FSLibrary/MissionLoadGenerationWithTxSetLimit.fs index 098348ae..4cae0d86 100644 --- a/src/FSLibrary/MissionLoadGenerationWithTxSetLimit.fs +++ b/src/FSLibrary/MissionLoadGenerationWithTxSetLimit.fs @@ -16,7 +16,7 @@ let loadGenerationWithTxSetLimit (context: MissionContext) = MakeLiveCoreSet "core" { CoreSetOptions.GetDefault context.image with - invariantChecks = AllInvariantsExceptBucketConsistencyChecksAndEvents + invariantChecks = AllInvariantsExceptEvents dumpDatabase = false updateSorobanCosts = Some(true) } diff --git a/src/FSLibrary/MissionMixedImageLoadGeneration.fs b/src/FSLibrary/MissionMixedImageLoadGeneration.fs index 6cbe7b7c..8ed62b68 100644 --- a/src/FSLibrary/MissionMixedImageLoadGeneration.fs +++ b/src/FSLibrary/MissionMixedImageLoadGeneration.fs @@ -43,7 +43,7 @@ let mixedImageLoadGeneration (oldImageNodeCount: int) (context: MissionContext) oldName { CoreSetOptions.GetDefault oldImage with nodeCount = oldNodeCount - invariantChecks = AllInvariantsExceptBucketConsistencyChecksAndEvents + invariantChecks = AllInvariantsExceptEvents accelerateTime = false dumpDatabase = false quorumSet = qSet @@ -57,7 +57,7 @@ let mixedImageLoadGeneration (oldImageNodeCount: int) (context: MissionContext) newName { CoreSetOptions.GetDefault newImage with nodeCount = newNodeCount - invariantChecks = AllInvariantsExceptBucketConsistencyChecksAndEvents + invariantChecks = AllInvariantsExceptEvents accelerateTime = false dumpDatabase = false quorumSet = qSet } diff --git a/src/FSLibrary/MissionMixedNominationLeaderElection.fs b/src/FSLibrary/MissionMixedNominationLeaderElection.fs index 43b4f16c..9d8ecb6d 100644 --- a/src/FSLibrary/MissionMixedNominationLeaderElection.fs +++ b/src/FSLibrary/MissionMixedNominationLeaderElection.fs @@ -27,7 +27,7 @@ let mixedNominationAlgorithm (oldCount: int) (context: MissionContext) = oldName { CoreSetOptions.GetDefault context.image with nodeCount = oldNodeCount - invariantChecks = AllInvariantsExceptBucketConsistencyChecksAndEvents + invariantChecks = AllInvariantsExceptEvents accelerateTime = false dumpDatabase = false forceOldStyleLeaderElection = true } @@ -37,7 +37,7 @@ let mixedNominationAlgorithm (oldCount: int) (context: MissionContext) = newName { CoreSetOptions.GetDefault context.image with nodeCount = newNodeCount - invariantChecks = AllInvariantsExceptBucketConsistencyChecksAndEvents + invariantChecks = AllInvariantsExceptEvents accelerateTime = false dumpDatabase = false quorumSetConfigType = RequireAutoQset } diff --git a/src/FSLibrary/MissionProtocolUpgradeWithLoad.fs b/src/FSLibrary/MissionProtocolUpgradeWithLoad.fs index e26cec2d..dc13ce1c 100644 --- a/src/FSLibrary/MissionProtocolUpgradeWithLoad.fs +++ b/src/FSLibrary/MissionProtocolUpgradeWithLoad.fs @@ -17,7 +17,7 @@ let protocolUpgradeWithLoad (context: MissionContext) = MakeLiveCoreSet "core" { CoreSetOptions.GetDefault context.image with - invariantChecks = AllInvariantsExceptBucketConsistencyChecksAndEvents + invariantChecks = AllInvariantsExceptEvents dumpDatabase = false updateSorobanCosts = Some(true) // Set `quorumSetConfigType` to `RequireAutoQset` as an extra diff --git a/src/FSLibrary/MissionSorobanCatchupWithPrevAndCurr.fs b/src/FSLibrary/MissionSorobanCatchupWithPrevAndCurr.fs index 25d889aa..8ffae9c2 100644 --- a/src/FSLibrary/MissionSorobanCatchupWithPrevAndCurr.fs +++ b/src/FSLibrary/MissionSorobanCatchupWithPrevAndCurr.fs @@ -18,7 +18,7 @@ let sorobanCatchupWithPrevAndCurr (context: MissionContext) = MakeLiveCoreSet "core" { CoreSetOptions.GetDefault context.image with - invariantChecks = AllInvariantsExceptBucketConsistencyChecksAndEvents + invariantChecks = AllInvariantsExceptEvents emptyDirType = DiskBackedEmptyDir updateSorobanCosts = Some(true) } diff --git a/src/FSLibrary/MissionSorobanConfigUpgrades.fs b/src/FSLibrary/MissionSorobanConfigUpgrades.fs index 8314efdf..f2d5508d 100644 --- a/src/FSLibrary/MissionSorobanConfigUpgrades.fs +++ b/src/FSLibrary/MissionSorobanConfigUpgrades.fs @@ -23,7 +23,7 @@ let sorobanConfigUpgrades (context: MissionContext) = MakeLiveCoreSet "core" { CoreSetOptions.GetDefault context.image with - invariantChecks = AllInvariantsExceptBucketConsistencyChecksAndEvents + invariantChecks = AllInvariantsExceptEvents emptyDirType = DiskBackedEmptyDir quorumSet = quorumSet updateSorobanCosts = Some(true) diff --git a/src/FSLibrary/MissionSorobanInvokeHostLoad.fs b/src/FSLibrary/MissionSorobanInvokeHostLoad.fs index 8710fafe..660c4ce5 100644 --- a/src/FSLibrary/MissionSorobanInvokeHostLoad.fs +++ b/src/FSLibrary/MissionSorobanInvokeHostLoad.fs @@ -16,7 +16,7 @@ let sorobanInvokeHostLoad (context: MissionContext) = MakeLiveCoreSet "core" { CoreSetOptions.GetDefault context.image with - invariantChecks = AllInvariantsExceptBucketConsistencyChecksAndEvents + invariantChecks = AllInvariantsExceptEvents emptyDirType = DiskBackedEmptyDir updateSorobanCosts = Some(true) } diff --git a/src/FSLibrary/MissionUpgradeTxClusters.fs b/src/FSLibrary/MissionUpgradeTxClusters.fs index d3093900..21b08b99 100644 --- a/src/FSLibrary/MissionUpgradeTxClusters.fs +++ b/src/FSLibrary/MissionUpgradeTxClusters.fs @@ -19,7 +19,7 @@ let upgradeTxClusters (context: MissionContext) = MakeLiveCoreSet "core" { CoreSetOptions.GetDefault context.image with - invariantChecks = AllInvariantsExceptBucketConsistencyChecksAndEvents + invariantChecks = AllInvariantsExceptEvents emptyDirType = DiskBackedEmptyDir updateSorobanCosts = Some(true) } From 3aae9effa6fcf84c63c2b50edec22b06806feae3 Mon Sep 17 00:00:00 2001 From: marta-lokhova Date: Tue, 16 Dec 2025 11:54:31 -0800 Subject: [PATCH 3/3] Renaming: in-memory core mode does not exist anymore --- doc/missions.md | 4 ++-- src/FSLibrary/FSLibrary.fsproj | 2 +- .../{MissionInMemoryMode.fs => MissionEmitMeta.fs} | 6 +++--- src/FSLibrary/StellarCoreCfg.fs | 8 ++++---- src/FSLibrary/StellarCoreSet.fs | 4 ++-- src/FSLibrary/StellarKubeSpecs.fs | 6 ------ src/FSLibrary/StellarMission.fs | 4 ++-- 7 files changed, 14 insertions(+), 20 deletions(-) rename src/FSLibrary/{MissionInMemoryMode.fs => MissionEmitMeta.fs} (94%) diff --git a/doc/missions.md b/doc/missions.md index 8ccf13cd..dad337e2 100644 --- a/doc/missions.md +++ b/doc/missions.md @@ -8,9 +8,9 @@ Basic test to complete a payment on a small network of nodes. Complete basic payment load generation on an asymmetric network topology. -## MissionInMemoryMode +## MissionEmitMeta -Complete basic payment and Soroban load generation on a network with in-memory watcher nodes. +Complete basic payment and Soroban load generation on a network with watcher nodes that emit metadata. ## MissionLoadGeneration diff --git a/src/FSLibrary/FSLibrary.fsproj b/src/FSLibrary/FSLibrary.fsproj index c569b3f0..57a17bcd 100644 --- a/src/FSLibrary/FSLibrary.fsproj +++ b/src/FSLibrary/FSLibrary.fsproj @@ -38,7 +38,7 @@ - + diff --git a/src/FSLibrary/MissionInMemoryMode.fs b/src/FSLibrary/MissionEmitMeta.fs similarity index 94% rename from src/FSLibrary/MissionInMemoryMode.fs rename to src/FSLibrary/MissionEmitMeta.fs index a1844ced..18c1d54c 100644 --- a/src/FSLibrary/MissionInMemoryMode.fs +++ b/src/FSLibrary/MissionEmitMeta.fs @@ -2,7 +2,7 @@ // under the Apache License, Version 2.0. See the COPYING file at the root // of this distribution or at http://www.apache.org/licenses/LICENSE-2.0 -module MissionInMemoryMode +module MissionEmitMeta open StellarCoreSet open StellarMissionContext @@ -11,7 +11,7 @@ open StellarSupercluster open StellarStatefulSets open StellarCoreHTTP -let runInMemoryMode (context: MissionContext) = +let runEmitMeta (context: MissionContext) = let coreSet = MakeLiveCoreSet "core" @@ -25,7 +25,7 @@ let runInMemoryMode (context: MissionContext) = { CoreSetOptions.GetDefault context.image with invariantChecks = AllInvariantsExceptBucketConsistencyChecksAndEvents nodeCount = 1 - inMemoryMode = true + emitMeta = true validate = false localHistory = false quorumSet = CoreSetQuorum(CoreSetName "core") diff --git a/src/FSLibrary/StellarCoreCfg.fs b/src/FSLibrary/StellarCoreCfg.fs index 6ad9abc1..e8e0dd23 100644 --- a/src/FSLibrary/StellarCoreCfg.fs +++ b/src/FSLibrary/StellarCoreCfg.fs @@ -164,7 +164,7 @@ type StellarCoreCfg = localHistory: bool maxSlotsToRemember: int maxBatchWriteCount: int - inMemoryMode: bool + emitMeta: bool addArtificialDelayUsec: int option // optional delay for testing in microseconds surveyPhaseDuration: int option containerType: CoreContainerType @@ -306,7 +306,7 @@ type StellarCoreCfg = t.Add("OUTBOUND_TX_QUEUE_BYTE_LIMIT", self.network.missionContext.outboundByteLimit.Value) |> ignore - if self.inMemoryMode then + if self.emitMeta then t.Add("METADATA_OUTPUT_STREAM", CfgVal.metaStreamPath) |> ignore match self.network.missionContext.simulateApplyWeight, self.network.missionContext.simulateApplyDuration with @@ -633,7 +633,7 @@ type NetworkCfg with localHistory = opts.localHistory maxSlotsToRemember = opts.maxSlotsToRemember maxBatchWriteCount = opts.maxBatchWriteCount - inMemoryMode = opts.inMemoryMode + emitMeta = opts.emitMeta addArtificialDelayUsec = opts.addArtificialDelayUsec surveyPhaseDuration = opts.surveyPhaseDuration containerType = MainCoreContainer @@ -674,7 +674,7 @@ type NetworkCfg with localHistory = c.options.localHistory maxSlotsToRemember = c.options.maxSlotsToRemember maxBatchWriteCount = c.options.maxBatchWriteCount - inMemoryMode = c.options.inMemoryMode + emitMeta = c.options.emitMeta addArtificialDelayUsec = c.options.addArtificialDelayUsec surveyPhaseDuration = c.options.surveyPhaseDuration containerType = ctype diff --git a/src/FSLibrary/StellarCoreSet.fs b/src/FSLibrary/StellarCoreSet.fs index 3aae4747..34a2d84f 100644 --- a/src/FSLibrary/StellarCoreSet.fs +++ b/src/FSLibrary/StellarCoreSet.fs @@ -218,7 +218,7 @@ type CoreSetOptions = dumpDatabase: bool maxSlotsToRemember: int maxBatchWriteCount: int - inMemoryMode: bool + emitMeta: bool addArtificialDelayUsec: int option surveyPhaseDuration: int option updateSorobanCosts: bool option @@ -259,7 +259,7 @@ type CoreSetOptions = dumpDatabase = true maxSlotsToRemember = 12 maxBatchWriteCount = 1024 - inMemoryMode = false + emitMeta = false addArtificialDelayUsec = None surveyPhaseDuration = None updateSorobanCosts = None diff --git a/src/FSLibrary/StellarKubeSpecs.fs b/src/FSLibrary/StellarKubeSpecs.fs index 7079e674..36ceb877 100644 --- a/src/FSLibrary/StellarKubeSpecs.fs +++ b/src/FSLibrary/StellarKubeSpecs.fs @@ -736,12 +736,6 @@ type NetworkCfg with else runCmd - let runCmd = - if coreSet.options.inMemoryMode then - Array.append runCmd [| "--in-memory" |] - else - runCmd - let usePostgres = (coreSet.options.dbType = Postgres || self.missionContext.runForMaxTps.IsSome) let exportToPrometheus = self.missionContext.exportToPrometheus diff --git a/src/FSLibrary/StellarMission.fs b/src/FSLibrary/StellarMission.fs index d5a9ae53..128bf80d 100644 --- a/src/FSLibrary/StellarMission.fs +++ b/src/FSLibrary/StellarMission.fs @@ -7,7 +7,7 @@ module StellarMission open MissionBootAndSync open MissionSimplePayment open MissionComplexTopology -open MissionInMemoryMode +open MissionEmitMeta open MissionLoadGeneration open MissionLoadGenerationWithSpikes open MissionLoadGenerationWithTxSetLimit @@ -59,7 +59,7 @@ let allMissions : Map = ("LoadGenerationWithTxSetLimit", loadGenerationWithTxSetLimit) ("MixedImageLoadGenerationWithOldImageMajority", mixedImageLoadGenerationWithOldImageMajority) ("MixedImageLoadGenerationWithNewImageMajority", mixedImageLoadGenerationWithNewImageMajority) - ("InMemoryMode", runInMemoryMode) + ("EmitMeta", runEmitMeta) ("HistoryGenerateAndCatchup", historyGenerateAndCatchup) ("HistoryPubnetMinimumCatchup", historyPubnetMinimumCatchup) ("HistoryPubnetRecentCatchup", historyPubnetRecentCatchup)