LSM trees initial integration (#1572)

# Description

Integrate the LSM trees as a new backend for the LedgerDB.

Author: jasagredo

cabal.project

@@ -14,9 +14,9 @@ repository cardano-haskell-packages
 -- update either of these.
 index-state:
   -- Bump this if you need newer packages from Hackage
-  , hackage.haskell.org 2025-09-11T01:58:40Z
+  , hackage.haskell.org 2025-09-26T20:57:57Z
   -- Bump this if you need newer packages from CHaP
-  , cardano-haskell-packages 2025-09-24T15:29:30Z
+  , cardano-haskell-packages 2025-10-01T14:54:25Z
 
 packages:
   ouroboros-consensus
@@ -49,3 +49,38 @@ if impl (ghc >= 9.12)
   allow-newer:
     -- https://github.com/kapralVV/Unique/issues/11
     , Unique:hashable
+
+if impl (ghc >= 9.10)
+  allow-newer:
+    -- https://github.com/phadej/regression-simple/pull/14
+    , regression-simple:base
+
+source-repository-package
+  type: git
+  location: https://github.com/IntersectMBO/cardano-ledger
+  tag: fb09078fa55015c881303a2ddb609c024cec258f
+  --sha256: sha256-9Y9CRiyMn0AWD+C4aNVMaJgrj3FDAYfCX4VrLvtoMaI=
+  subdir:
+    eras/allegra/impl
+    eras/alonzo/impl
+    eras/alonzo/test-suite
+    eras/babbage/impl
+    eras/conway/impl
+    eras/dijkstra/impl
+    eras/mary/impl
+    eras/shelley/impl
+    eras/shelley/test-suite
+    eras/shelley-ma/test-suite
+    libs/cardano-ledger-api
+    libs/cardano-ledger-core
+    libs/cardano-ledger-binary
+    libs/cardano-protocol-tpraos
+    libs/non-integral
+    libs/small-steps
+    libs/cardano-data
+    libs/set-algebra
+    libs/vector-map
+    eras/byron/chain/executable-spec
+    eras/byron/ledger/executable-spec
+    eras/byron/ledger/impl
+    eras/byron/crypto

flake.lock

@@ -89,9 +89,13 @@ in
     nativeBuildInputs = [
       final.fd
       final.cabal-docspec
-      (hsPkgs.ghcWithPackages
-        (ps: [ ps.latex-svg-image ] ++ lib.filter (p: p ? components.library)
-          (lib.attrValues (haskell-nix.haskellLib.selectProjectPackages ps))))
+      (hsPkgs.shellFor {
+        withHoogle = false;
+        exactDeps = true;
+        packages = _: [ ];
+        additional = (ps: [ ps.latex-svg-image ] ++ lib.filter (p: p ? components.library)
+          (lib.attrValues (haskell-nix.haskellLib.selectProjectPackages ps)));
+      }).ghc
       final.texliveFull
     ];
 

nix/haskell.nix

@@ -46,7 +46,7 @@ parseDBAnalyserConfig =
       [ flag' V1InMem $
           mconcat
             [ long "v1-in-mem"
-            , help "use v1 in-memory backing store"
+            , help "use v1 in-memory backing store [deprecated]"
             ]
       , flag' V1LMDB $
           mconcat
@@ -55,9 +55,14 @@ parseDBAnalyserConfig =
             ]
       , flag' V2InMem $
           mconcat
-            [ long "v2-in-mem"
+            [ long "in-mem"
             , help "use v2 in-memory backend"
             ]
+      , flag' V2LSM $
+          mconcat
+            [ long "lsm"
+            , help "use v2 LSM backend"
+            ]
       ]
 
 parseSelectDB :: Parser SelectDB

ouroboros-consensus-cardano/app/DBAnalyser/Parsers.hs

@@ -38,6 +38,7 @@ import qualified Ouroboros.Consensus.Storage.LedgerDB.V1.BackingStore.Impl.LMDB
 import qualified Ouroboros.Consensus.Storage.LedgerDB.V1.DbChangelog as V1
 import qualified Ouroboros.Consensus.Storage.LedgerDB.V1.Lock as V1
 import qualified Ouroboros.Consensus.Storage.LedgerDB.V1.Snapshots as V1
+import qualified Ouroboros.Consensus.Storage.LedgerDB.V2.InMemory as InMemory
 import qualified Ouroboros.Consensus.Storage.LedgerDB.V2.InMemory as V2
 import qualified Ouroboros.Consensus.Storage.LedgerDB.V2.LedgerSeq as V2
 import Ouroboros.Consensus.Util.CRC
@@ -199,7 +200,7 @@ load config@Config{inpath = pathToDiskSnapshot -> Just (fs@(SomeHasFS hasFS), pa
       checkSnapshotFileStructure Mem path fs
       (ls, _) <- withExceptT SnapshotError $ V2.loadSnapshot nullTracer rr ccfg fs ds
       let h = V2.currentHandle ls
-      (V2.state h,) <$> Trans.lift (V2.readAll (V2.tables h))
+      (V2.state h,) <$> Trans.lift (V2.readAll (V2.tables h) (V2.state h))
     LMDB -> do
       checkSnapshotFileStructure LMDB path fs
       ((dbch, k, bstore), _) <-
@@ -240,7 +241,7 @@ store config@Config{outpath = pathToDiskSnapshot -> Just (fs@(SomeHasFS hasFS),
     Mem -> do
       lseq <- V2.empty state tbs $ V2.newInMemoryLedgerTablesHandle nullTracer fs
       let h = V2.currentHandle lseq
-      Monad.void $ V2.implTakeSnapshot ccfg nullTracer fs suffix h
+      Monad.void $ InMemory.implTakeSnapshot ccfg nullTracer fs suffix h
     LMDB -> do
       chlog <- newTVarIO (V1.empty state)
       lock <- V1.mkLedgerDBLock

ouroboros-consensus-cardano/app/snapshot-converter.hs

@@ -0,0 +1,25 @@
+<!--
+A new scriv changelog fragment.
+
+Uncomment the section that is right (remove the HTML comment wrapper).
+For top level release notes, leave all the headers commented out.
+-->
+
+<!--
+### Patch
+
+- A bullet item for the Patch category.
+
+-->
+<!--
+### Non-Breaking
+
+- A bullet item for the Non-Breaking category.
+
+-->
+<!--
+### Breaking
+
+- A bullet item for the Breaking category.
+
+-->

ouroboros-consensus-cardano/changelog.d/20250729_124022_javier.sagredo_lsm1.md

@@ -0,0 +1,29 @@
+<!--
+A new scriv changelog fragment.
+
+Uncomment the section that is right (remove the HTML comment wrapper).
+For top level release notes, leave all the headers commented out.
+-->
+
+### Patch
+
+- Bump ledger dependencies:
+  - `cardano-ledger-allegra` 1.8 → 1.9
+  - `cardano-ledger-alonzo` 1.14 → 1.15
+  - `cardano-ledger-api` 1.12 → 1.13
+  - `cardano-ledger-conway` 1.20 → 1.21
+  - `cardano-ledger-core` 1.18 → 1.19
+  - `cardano-ledger-dijkstra` 0.1 → 0.2
+
+<!--
+### Non-Breaking
+
+- A bullet item for the Non-Breaking category.
+
+-->
+<!--
+### Breaking
+
+- A bullet item for the Breaking category.
+
+-->

ouroboros-consensus-cardano/changelog.d/20251002_112515_javier.sagredo_lsm1.md

@@ -139,15 +139,15 @@ library
     cardano-crypto,
     cardano-crypto-class ^>=2.2,
     cardano-crypto-wrapper,
-    cardano-ledger-allegra ^>=1.8,
-    cardano-ledger-alonzo ^>=1.14,
-    cardano-ledger-api ^>=1.12,
+    cardano-ledger-allegra ^>=1.9,
+    cardano-ledger-alonzo ^>=1.15,
+    cardano-ledger-api ^>=1.13,
     cardano-ledger-babbage ^>=1.12,
     cardano-ledger-binary ^>=1.7,
     cardano-ledger-byron ^>=1.2,
-    cardano-ledger-conway ^>=1.20,
-    cardano-ledger-core ^>=1.18,
-    cardano-ledger-dijkstra ^>=0.1,
+    cardano-ledger-conway ^>=1.21,
+    cardano-ledger-core ^>=1.19,
+    cardano-ledger-dijkstra ^>=0.2,
     cardano-ledger-mary ^>=1.9,
     cardano-ledger-shelley ^>=1.17,
     cardano-prelude,
@@ -593,6 +593,7 @@ library unstable-cardano-tools
     ouroboros-network-api,
     ouroboros-network-framework ^>=0.19,
     ouroboros-network-protocols,
+    random,
     resource-registry,
     singletons,
     sop-core,

ouroboros-consensus-cardano/ouroboros-consensus-cardano.cabal

@@ -165,7 +165,7 @@ mkShelleyBlock ::
 mkShelleyBlock raw =
   ShelleyBlock
     { shelleyBlockRaw = raw
-    , shelleyBlockHeaderHash = pHeaderHash $ SL.bheader raw
+    , shelleyBlockHeaderHash = pHeaderHash $ SL.blockHeader raw
     }
 
 class
@@ -200,7 +200,7 @@ instance
 instance ShelleyCompatible proto era => GetHeader (ShelleyBlock proto era) where
   getHeader (ShelleyBlock rawBlk hdrHash) =
     ShelleyHeader
-      { shelleyHeaderRaw = SL.bheader rawBlk
+      { shelleyHeaderRaw = SL.blockHeader rawBlk
       , shelleyHeaderHash = hdrHash
       }
 

ouroboros-consensus-cardano/src/shelley/Ouroboros/Consensus/Shelley/Ledger/Block.hs

@@ -629,8 +629,8 @@ applyHelper f cfg blk stBefore = do
       globals
       tickedShelleyLedgerState
       ( let b = shelleyBlockRaw blk
-            h' = mkHeaderView (SL.bheader b)
-         in SL.Block h' (SL.bbody b)
+            h' = mkHeaderView (SL.blockHeader b)
+         in SL.Block h' (SL.blockBody b)
       )
 
   let track ::