Simplify soroban load generation and other improvements

doc/missions.md

@@ -8,9 +8,9 @@ Basic test to complete a payment on a small network of nodes.
 
 Complete basic payment load generation on an asymmetric network topology.
 
-## MissionInMemoryMode
+## MissionEmitMeta
 
-Complete basic payment and Soroban load generation on a network with in-memory watcher nodes.
+Complete basic payment and Soroban load generation on a network with watcher nodes that emit metadata.
 
 ## MissionLoadGeneration
 

src/FSLibrary/FSLibrary.fsproj

@@ -38,7 +38,7 @@
     <Compile Include="MissionSimplePayment.fs" />
     <Compile Include="MissionComplexTopology.fs" />
     <Compile Include="MissionLoadGeneration.fs" />
-    <Compile Include="MissionInMemoryMode.fs" />
+    <Compile Include="MissionEmitMeta.fs" />
     <Compile Include="MissionLoadGenerationWithSpikes.fs" />
     <Compile Include="MissionLoadGenerationWithTxSetLimit.fs" />
     <Compile Include="MissionHistoryGenerateAndCatchup.fs" />

src/FSLibrary/MissionInMemoryMode.fs → src/FSLibrary/MissionEmitMeta.fs

@@ -2,7 +2,7 @@
 // under the Apache License, Version 2.0. See the COPYING file at the root
 // of this distribution or at http://www.apache.org/licenses/LICENSE-2.0
 
-module MissionInMemoryMode
+module MissionEmitMeta
 
 open StellarCoreSet
 open StellarMissionContext
@@ -11,7 +11,7 @@ open StellarSupercluster
 open StellarStatefulSets
 open StellarCoreHTTP
 
-let runInMemoryMode (context: MissionContext) =
+let runEmitMeta (context: MissionContext) =
     let coreSet =
         MakeLiveCoreSet
             "core"
@@ -25,7 +25,7 @@ let runInMemoryMode (context: MissionContext) =
             { CoreSetOptions.GetDefault context.image with
                   invariantChecks = AllInvariantsExceptBucketConsistencyChecksAndEvents
                   nodeCount = 1
-                  inMemoryMode = true
+                  emitMeta = true
                   validate = false
                   localHistory = false
                   quorumSet = CoreSetQuorum(CoreSetName "core")

src/FSLibrary/MissionLoadGeneration.fs

@@ -16,7 +16,7 @@ let loadGeneration (context: MissionContext) =
         MakeLiveCoreSet
             "core"
             { CoreSetOptions.GetDefault context.image with
-                  invariantChecks = AllInvariantsExceptBucketConsistencyChecksAndEvents
+                  invariantChecks = AllInvariantsExceptEvents
                   dumpDatabase = false }
 
     let context =

src/FSLibrary/MissionLoadGenerationWithSpikes.fs

@@ -16,7 +16,7 @@ let loadGenerationWithSpikes (context: MissionContext) =
         MakeLiveCoreSet
             "core"
             { CoreSetOptions.GetDefault context.image with
-                  invariantChecks = AllInvariantsExceptBucketConsistencyChecksAndEvents
+                  invariantChecks = AllInvariantsExceptEvents
                   dumpDatabase = false }
 
     let context =

src/FSLibrary/MissionLoadGenerationWithTxSetLimit.fs

@@ -16,7 +16,7 @@ let loadGenerationWithTxSetLimit (context: MissionContext) =
         MakeLiveCoreSet
             "core"
             { CoreSetOptions.GetDefault context.image with
-                  invariantChecks = AllInvariantsExceptBucketConsistencyChecksAndEvents
+                  invariantChecks = AllInvariantsExceptEvents
                   dumpDatabase = false
                   updateSorobanCosts = Some(true) }
 

src/FSLibrary/MissionMixedImageLoadGeneration.fs

@@ -43,7 +43,7 @@ let mixedImageLoadGeneration (oldImageNodeCount: int) (context: MissionContext)
             oldName
             { CoreSetOptions.GetDefault oldImage with
                   nodeCount = oldNodeCount
-                  invariantChecks = AllInvariantsExceptBucketConsistencyChecksAndEvents
+                  invariantChecks = AllInvariantsExceptEvents
                   accelerateTime = false
                   dumpDatabase = false
                   quorumSet = qSet
@@ -57,7 +57,7 @@ let mixedImageLoadGeneration (oldImageNodeCount: int) (context: MissionContext)
             newName
             { CoreSetOptions.GetDefault newImage with
                   nodeCount = newNodeCount
-                  invariantChecks = AllInvariantsExceptBucketConsistencyChecksAndEvents
+                  invariantChecks = AllInvariantsExceptEvents
                   accelerateTime = false
                   dumpDatabase = false
                   quorumSet = qSet }

src/FSLibrary/MissionMixedNominationLeaderElection.fs

@@ -27,7 +27,7 @@ let mixedNominationAlgorithm (oldCount: int) (context: MissionContext) =
             oldName
             { CoreSetOptions.GetDefault context.image with
                   nodeCount = oldNodeCount
-                  invariantChecks = AllInvariantsExceptBucketConsistencyChecksAndEvents
+                  invariantChecks = AllInvariantsExceptEvents
                   accelerateTime = false
                   dumpDatabase = false
                   forceOldStyleLeaderElection = true }
@@ -37,7 +37,7 @@ let mixedNominationAlgorithm (oldCount: int) (context: MissionContext) =
             newName
             { CoreSetOptions.GetDefault context.image with
                   nodeCount = newNodeCount
-                  invariantChecks = AllInvariantsExceptBucketConsistencyChecksAndEvents
+                  invariantChecks = AllInvariantsExceptEvents
                   accelerateTime = false
                   dumpDatabase = false
                   quorumSetConfigType = RequireAutoQset }

src/FSLibrary/MissionProtocolUpgradeWithLoad.fs

@@ -17,7 +17,7 @@ let protocolUpgradeWithLoad (context: MissionContext) =
         MakeLiveCoreSet
             "core"
             { CoreSetOptions.GetDefault context.image with
-                  invariantChecks = AllInvariantsExceptBucketConsistencyChecksAndEvents
+                  invariantChecks = AllInvariantsExceptEvents
                   dumpDatabase = false
                   updateSorobanCosts = Some(true)
                   // Set `quorumSetConfigType` to `RequireAutoQset` as an extra

src/FSLibrary/MissionSorobanCatchupWithPrevAndCurr.fs

@@ -18,7 +18,7 @@ let sorobanCatchupWithPrevAndCurr (context: MissionContext) =
         MakeLiveCoreSet
             "core"
             { CoreSetOptions.GetDefault context.image with
-                  invariantChecks = AllInvariantsExceptBucketConsistencyChecksAndEvents
+                  invariantChecks = AllInvariantsExceptEvents
                   emptyDirType = DiskBackedEmptyDir
                   updateSorobanCosts = Some(true) }
 

src/FSLibrary/MissionSorobanConfigUpgrades.fs

@@ -23,7 +23,7 @@ let sorobanConfigUpgrades (context: MissionContext) =
         MakeLiveCoreSet
             "core"
             { CoreSetOptions.GetDefault context.image with
-                  invariantChecks = AllInvariantsExceptBucketConsistencyChecksAndEvents
+                  invariantChecks = AllInvariantsExceptEvents
                   emptyDirType = DiskBackedEmptyDir
                   quorumSet = quorumSet
                   updateSorobanCosts = Some(true)

src/FSLibrary/MissionSorobanInvokeHostLoad.fs

@@ -16,7 +16,7 @@ let sorobanInvokeHostLoad (context: MissionContext) =
         MakeLiveCoreSet
             "core"
             { CoreSetOptions.GetDefault context.image with
-                  invariantChecks = AllInvariantsExceptBucketConsistencyChecksAndEvents
+                  invariantChecks = AllInvariantsExceptEvents
                   emptyDirType = DiskBackedEmptyDir
                   updateSorobanCosts = Some(true) }
 

src/FSLibrary/MissionSorobanLoadGeneration.fs

@@ -8,49 +8,39 @@ open StellarCoreSet
 open StellarMissionContext
 open StellarFormation
 open StellarStatefulSets
-open StellarNetworkData
 open StellarSupercluster
 open StellarCoreHTTP
 
 
 let sorobanLoadGeneration (context: MissionContext) =
     let rate = 5
 
+    let coreSet =
+        MakeLiveCoreSet
+            "core"
+            { CoreSetOptions.GetDefault context.image with
+                  invariantChecks = AllInvariantsExceptEvents
+                  dumpDatabase = false }
+
     let context =
         { context.WithSmallLoadgenOptions with
-              coreResources = SimulatePubnetTier1PerfResources
-              installNetworkDelay = Some(context.installNetworkDelay |> Option.defaultValue true)
               txRate = rate
               numAccounts = 10000
-              numTxs = rate * 2000
+              numTxs = rate * 200
               skipLowFeeTxs = true
               maxFeeRate = Some 100000000
-              enableTailLogging = false
               updateSorobanCosts = Some(true)
               genesisTestAccountCount = Some 10000 }
 
-    let fullCoreSet = FullPubnetCoreSets context true true
-
-    let sdf =
-        List.find (fun (cs: CoreSet) -> cs.name.StringName = "stellar" || cs.name.StringName = "sdf") fullCoreSet
-
-    let tier1 = List.filter (fun (cs: CoreSet) -> cs.options.tier1 = Some true) fullCoreSet
-
     context.Execute
-        fullCoreSet
+        [ coreSet ]
         None
         (fun (formation: StellarFormation) ->
-            // Setup overlay connections first before manually closing
-            // ledger, which kick off consensus
-            formation.WaitUntilConnected fullCoreSet
-            formation.ManualClose tier1
-
             // Wait until the whole network is synced before proceeding,
             // to fail asap in case of a misconfiguration
-            formation.WaitUntilSynced fullCoreSet
-            formation.UpgradeProtocolToLatest tier1
-            formation.UpgradeMaxTxSetSize tier1 1000000
+            formation.WaitUntilSynced [ coreSet ]
+            formation.UpgradeProtocolToLatest [ coreSet ]
+            formation.UpgradeMaxTxSetSize [ coreSet ] 1000000
 
-            formation.UpgradeSorobanLedgerLimitsWithMultiplier tier1 100
-            formation.RunLoadgen sdf context.GenerateSorobanUploadLoad
-            formation.EnsureAllNodesInSync fullCoreSet)
+            formation.UpgradeSorobanLedgerLimitsWithMultiplier [ coreSet ] 100
+            formation.RunLoadgen coreSet context.GenerateSorobanUploadLoad)

src/FSLibrary/MissionUpgradeTxClusters.fs

@@ -19,7 +19,7 @@ let upgradeTxClusters (context: MissionContext) =
         MakeLiveCoreSet
             "core"
             { CoreSetOptions.GetDefault context.image with
-                  invariantChecks = AllInvariantsExceptBucketConsistencyChecksAndEvents
+                  invariantChecks = AllInvariantsExceptEvents
                   emptyDirType = DiskBackedEmptyDir
                   updateSorobanCosts = Some(true) }
 

src/FSLibrary/StellarCoreCfg.fs

@@ -164,7 +164,7 @@ type StellarCoreCfg =
       localHistory: bool
       maxSlotsToRemember: int
       maxBatchWriteCount: int
-      inMemoryMode: bool
+      emitMeta: bool
       addArtificialDelayUsec: int option // optional delay for testing in microseconds
       surveyPhaseDuration: int option
       containerType: CoreContainerType
@@ -306,7 +306,7 @@ type StellarCoreCfg =
             t.Add("OUTBOUND_TX_QUEUE_BYTE_LIMIT", self.network.missionContext.outboundByteLimit.Value)
             |> ignore
 
-        if self.inMemoryMode then
+        if self.emitMeta then
             t.Add("METADATA_OUTPUT_STREAM", CfgVal.metaStreamPath) |> ignore
 
         match self.network.missionContext.simulateApplyWeight, self.network.missionContext.simulateApplyDuration with
@@ -633,7 +633,7 @@ type NetworkCfg with
           localHistory = opts.localHistory
           maxSlotsToRemember = opts.maxSlotsToRemember
           maxBatchWriteCount = opts.maxBatchWriteCount
-          inMemoryMode = opts.inMemoryMode
+          emitMeta = opts.emitMeta
           addArtificialDelayUsec = opts.addArtificialDelayUsec
           surveyPhaseDuration = opts.surveyPhaseDuration
           containerType = MainCoreContainer
@@ -674,7 +674,7 @@ type NetworkCfg with
           localHistory = c.options.localHistory
           maxSlotsToRemember = c.options.maxSlotsToRemember
           maxBatchWriteCount = c.options.maxBatchWriteCount
-          inMemoryMode = c.options.inMemoryMode
+          emitMeta = c.options.emitMeta
           addArtificialDelayUsec = c.options.addArtificialDelayUsec
           surveyPhaseDuration = c.options.surveyPhaseDuration
           containerType = ctype

src/FSLibrary/StellarCoreSet.fs

@@ -218,7 +218,7 @@ type CoreSetOptions =
       dumpDatabase: bool
       maxSlotsToRemember: int
       maxBatchWriteCount: int
-      inMemoryMode: bool
+      emitMeta: bool
       addArtificialDelayUsec: int option
       surveyPhaseDuration: int option
       updateSorobanCosts: bool option
@@ -259,7 +259,7 @@ type CoreSetOptions =
           dumpDatabase = true
           maxSlotsToRemember = 12
           maxBatchWriteCount = 1024
-          inMemoryMode = false
+          emitMeta = false
           addArtificialDelayUsec = None
           surveyPhaseDuration = None
           updateSorobanCosts = None

src/FSLibrary/StellarKubeSpecs.fs

@@ -736,12 +736,6 @@ type NetworkCfg with
             else
                 runCmd
 
-        let runCmd =
-            if coreSet.options.inMemoryMode then
-                Array.append runCmd [| "--in-memory" |]
-            else
-                runCmd
-
         let usePostgres = (coreSet.options.dbType = Postgres || self.missionContext.runForMaxTps.IsSome)
         let exportToPrometheus = self.missionContext.exportToPrometheus
 

src/FSLibrary/StellarMission.fs

@@ -7,7 +7,7 @@ module StellarMission
 open MissionBootAndSync
 open MissionSimplePayment
 open MissionComplexTopology
-open MissionInMemoryMode
+open MissionEmitMeta
 open MissionLoadGeneration
 open MissionLoadGenerationWithSpikes
 open MissionLoadGenerationWithTxSetLimit
@@ -59,7 +59,7 @@ let allMissions : Map<string, Mission> =
                  ("LoadGenerationWithTxSetLimit", loadGenerationWithTxSetLimit)
                  ("MixedImageLoadGenerationWithOldImageMajority", mixedImageLoadGenerationWithOldImageMajority)
                  ("MixedImageLoadGenerationWithNewImageMajority", mixedImageLoadGenerationWithNewImageMajority)
-                 ("InMemoryMode", runInMemoryMode)
+                 ("EmitMeta", runEmitMeta)
                  ("HistoryGenerateAndCatchup", historyGenerateAndCatchup)
                  ("HistoryPubnetMinimumCatchup", historyPubnetMinimumCatchup)
                  ("HistoryPubnetRecentCatchup", historyPubnetRecentCatchup)