From 28607e25ed31c4dbd59c9310c11cee436aedc676 Mon Sep 17 00:00:00 2001 From: Felipe Gonzalez Date: Thu, 16 Oct 2025 10:08:43 -0300 Subject: [PATCH] feat: Add compression to archive --- Cargo.lock | 28 ++++++++++++++++++++-- Cargo.toml | 3 ++- crates/redb3/Cargo.toml | 1 + crates/redb3/src/archive/mod.rs | 6 +++++ crates/redb3/src/archive/tables.rs | 37 ++++++++++++++++++++++++------ 5 files changed, 65 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c1a7052f..3230317c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1291,6 +1291,7 @@ dependencies = [ "bincode 1.3.3", "dolos-core", "dolos-testing", + "flate2", "hex", "itertools 0.13.0", "pallas 1.0.0-alpha.2 (git+https://github.com/txpipe/pallas.git?rev=38546da)", @@ -1508,11 +1509,12 @@ checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" [[package]] name = "flate2" -version = "1.1.1" +version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ced92e76e966ca2fd84c8f7aa01a4aea65b0eb6648d72f7c8f3e2764a67fece" +checksum = "dc5a4e564e38c699f2880d3fda590bedc2e69f3f84cd48b457bd892ce61d0aa9" dependencies = [ "crc32fast", + "libz-rs-sys", "miniz_oxide", ] @@ -2511,6 +2513,15 @@ dependencies = [ "redox_syscall", ] +[[package]] +name = "libz-rs-sys" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "840db8cf39d9ec4dd794376f38acc40d0fc65eec2a8f484f7fd375b84602becd" +dependencies = [ + "zlib-rs", +] + [[package]] name = "linux-raw-sys" version = "0.4.15" @@ -2673,6 +2684,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3be647b768db090acb35d5ec5db2b0e1f1de11133ca123b9eacf5137868f892a" dependencies = [ "adler2", + "simd-adler32", ] [[package]] @@ -4608,6 +4620,12 @@ dependencies = [ "rand_core 0.6.4", ] +[[package]] +name = "simd-adler32" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe" + [[package]] name = "siphasher" version = "1.0.1" @@ -6321,6 +6339,12 @@ dependencies = [ "syn", ] +[[package]] +name = "zlib-rs" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f06ae92f42f5e5c42443fd094f245eb656abf56dd7cce9b8b263236565e00f2" + [[package]] name = "zstd" version = "0.13.3" diff --git a/Cargo.toml b/Cargo.toml index 23d33f74..a51adae9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,6 +15,7 @@ base64.workspace = true bech32.workspace = true bincode.workspace = true chrono.workspace = true +flate2.workspace = true futures-core.workspace = true futures-util.workspace = true hex.workspace = true @@ -61,7 +62,6 @@ comfy-table = { version = "7.1.1", optional = true } inquire = { version = "0.7.5", optional = true } toml = { version = "0.8.13", optional = true } console-subscriber = { version = "0.3.0", optional = true } -flate2 = "1.0.34" tar = "0.4.41" paste = "1.0.15" @@ -147,6 +147,7 @@ hex = "0.4.3" tracing = "0.1.37" itertools = "0.13.0" bincode = "1.3.3" +flate2 = { version = "1.1.4", features = ["zlib-rs"]} futures-core = "0.3.31" futures-util = "0.3.28" trait-variant = "0.1.2" diff --git a/crates/redb3/Cargo.toml b/crates/redb3/Cargo.toml index 20147e8f..951c370b 100644 --- a/crates/redb3/Cargo.toml +++ b/crates/redb3/Cargo.toml @@ -15,6 +15,7 @@ bincode.workspace = true hex.workspace = true thiserror.workspace = true xxhash-rust.workspace = true +flate2.workspace = true dolos-core = { path = "../core" } diff --git a/crates/redb3/src/archive/mod.rs b/crates/redb3/src/archive/mod.rs index 9f6f151f..1b493b5b 100644 --- a/crates/redb3/src/archive/mod.rs +++ b/crates/redb3/src/archive/mod.rs @@ -81,6 +81,12 @@ impl From<::redb::TransactionError> for RedbArchiveError { } } +impl From for RedbArchiveError { + fn from(value: std::io::Error) -> Self { + Self(ArchiveError::InternalError(value.to_string())) + } +} + const DEFAULT_CACHE_SIZE_MB: usize = 500; #[derive(Clone)] diff --git a/crates/redb3/src/archive/tables.rs b/crates/redb3/src/archive/tables.rs index e43f83f0..8a698713 100644 --- a/crates/redb3/src/archive/tables.rs +++ b/crates/redb3/src/archive/tables.rs @@ -1,14 +1,21 @@ +use std::io::{Read, Write}; + +use pallas::ledger::primitives::byron::Block; use redb::{Range, ReadTransaction, ReadableTable as _, TableDefinition, WriteTransaction}; +use flate2::{read::ZlibDecoder, write::ZlibEncoder}; +use flate2::Compression; use tracing::trace; use dolos_core::{BlockBody, BlockSlot, ChainPoint, RawBlock}; type Error = super::RedbArchiveError; +type CompressedBlockBody = Vec; + pub struct BlocksTable; impl BlocksTable { - pub const DEF: TableDefinition<'static, BlockSlot, BlockBody> = TableDefinition::new("blocks"); + pub const DEF: TableDefinition<'static, BlockSlot, CompressedBlockBody> = TableDefinition::new("blocks"); pub fn initialize(wx: &WriteTransaction) -> Result<(), Error> { wx.open_table(Self::DEF)?; @@ -16,18 +23,34 @@ impl BlocksTable { Ok(()) } + fn compress(block: BlockBody) -> std::io::Result { + let mut e = ZlibEncoder::new(Vec::new(), Compression::best()); + e.write_all(block.as_slice())?; + e.finish() + } + + fn decompress(compressed: CompressedBlockBody) -> std::io::Result { + let mut z = ZlibDecoder::new(compressed.as_slice()); + let mut result = Vec::new(); + z.read_to_end(&mut result)?; + Ok(result) + } + pub fn get_tip(rx: &ReadTransaction) -> Result, Error> { let table = rx.open_table(Self::DEF)?; let result = table .last()? - .map(|(slot, raw)| (slot.value(), raw.value().clone())); - Ok(result) + .map(|(slot, raw)| + Ok((slot.value(), BlocksTable::decompress(raw.value().clone())?))); + result.transpose() } pub fn get_by_slot(rx: &ReadTransaction, slot: BlockSlot) -> Result, Error> { let table = rx.open_table(Self::DEF)?; match table.get(slot)? { - Some(value) => Ok(Some(value.value().clone())), + Some(value) => { + Ok(Some(BlocksTable::decompress(value.value().clone())?)) + }, None => Ok(None), } } @@ -36,7 +59,7 @@ impl BlocksTable { let mut table = wx.open_table(Self::DEF)?; let slot = point.slot(); - table.insert(slot, block.clone())?; + table.insert(slot, BlocksTable::compress(block.to_vec())?)?; Ok(()) } @@ -66,8 +89,8 @@ impl BlocksTable { let table = rx.open_table(Self::DEF)?; let result = table .first()? - .map(|(slot, raw)| (slot.value(), raw.value().clone())); - Ok(result) + .map(|(slot, raw)| Ok((slot.value(), BlocksTable::decompress(raw.value().clone())?))); + result.transpose() } pub fn last(rx: &ReadTransaction) -> Result, Error> {