From a77ec64ea1cfe561c9d5aa9db11c2a1603ab1aa1 Mon Sep 17 00:00:00 2001 From: UdjinM6 Date: Mon, 19 Jan 2026 21:42:57 +0300 Subject: [PATCH 01/14] feat: migrate TimestampIndex from synchronous to async operation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Migrates TimestampIndex to follow the BaseIndex async pattern (similar to TxIndex), enabling background syncing without blocking validation. Key changes: - Move data structures from src/timestampindex.h to src/index/timestampindex.h - Implement async TimestampIndex class inheriting from BaseIndex - Create separate database at indexes/timestampindex/ (previously in blocks/index/) - Remove synchronous writes from validation (ConnectBlock, DisconnectBlock, RollforwardBlock) - Update getblockhashes RPC to use async index with BlockUntilSyncedToCurrentChain() - Remove global fTimestampIndex flag and ERROR_TIMEIDX_NEEDS_REINDEX - Add cache allocation in node/caches.{h,cpp} - Update initialization/shutdown in init.cpp Benefits: - Index can be toggled on/off without requiring -reindex - Background syncing doesn't block block validation - Works with pruned nodes (only stores block metadata) - Consistent with other async indexes (txindex) 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Sonnet 4.5 --- src/Makefile.am | 4 +- src/index/timestampindex.cpp | 114 ++++++++++++++++++++++++++++++ src/index/timestampindex.h | 71 +++++++++++++++++++ src/index/timestampindex_types.h | 74 +++++++++++++++++++ src/init.cpp | 28 +++++--- src/node/blockstorage.cpp | 3 - src/node/blockstorage.h | 3 - src/node/caches.cpp | 3 + src/node/caches.h | 1 + src/node/chainstate.cpp | 6 -- src/node/chainstate.h | 2 - src/rpc/blockchain.cpp | 10 ++- src/rpc/index_util.cpp | 12 ---- src/rpc/index_util.h | 5 -- src/test/util/setup_common.cpp | 2 - src/timestampindex.h | 81 --------------------- src/txdb.cpp | 32 --------- src/txdb.h | 5 -- src/validation.cpp | 22 ------ test/util/data/non-backported.txt | 1 + 20 files changed, 293 insertions(+), 186 deletions(-) create mode 100644 src/index/timestampindex.cpp create mode 100644 src/index/timestampindex.h create mode 100644 src/index/timestampindex_types.h delete mode 100644 src/timestampindex.h diff --git a/src/Makefile.am b/src/Makefile.am index 47f3c6bd55fc..224672ecd6e6 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -251,6 +251,8 @@ BITCOIN_CORE_H = \ index/blockfilterindex.h \ index/coinstatsindex.h \ index/disktxpos.h \ + index/timestampindex.h \ + index/timestampindex_types.h \ index/txindex.h \ indirectmap.h \ init.h \ @@ -379,7 +381,6 @@ BITCOIN_CORE_H = \ support/events.h \ support/lockedpool.h \ sync.h \ - timestampindex.h \ threadsafety.h \ timedata.h \ torcontrol.h \ @@ -539,6 +540,7 @@ libbitcoin_node_a_SOURCES = \ index/base.cpp \ index/blockfilterindex.cpp \ index/coinstatsindex.cpp \ + index/timestampindex.cpp \ index/txindex.cpp \ init.cpp \ instantsend/db.cpp \ diff --git a/src/index/timestampindex.cpp b/src/index/timestampindex.cpp new file mode 100644 index 000000000000..97a7b5aebc81 --- /dev/null +++ b/src/index/timestampindex.cpp @@ -0,0 +1,114 @@ +// Copyright (c) 2026 The Dash Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include + +#include +#include +#include + +constexpr uint8_t DB_TIMESTAMPINDEX{'s'}; + +std::unique_ptr g_timestampindex; + +TimestampIndex::DB::DB(size_t n_cache_size, bool f_memory, bool f_wipe) : + BaseIndex::DB(gArgs.GetDataDirNet() / "indexes" / "timestampindex", n_cache_size, f_memory, f_wipe) +{ +} + +bool TimestampIndex::DB::Write(const CTimestampIndexKey& key) +{ + return CDBWrapper::Write(std::make_pair(DB_TIMESTAMPINDEX, key), true); +} + +bool TimestampIndex::DB::ReadRange(uint32_t high, uint32_t low, std::vector& hashes) +{ + std::unique_ptr pcursor(NewIterator()); + + // Seek to the starting timestamp + pcursor->Seek(std::make_pair(DB_TIMESTAMPINDEX, CTimestampIndexIteratorKey(low))); + + // Iterate through all entries in the timestamp range + while (pcursor->Valid()) { + std::pair key; + if (pcursor->GetKey(key) && key.first == DB_TIMESTAMPINDEX && key.second.m_block_time <= high) { + hashes.push_back(key.second.m_block_hash); + pcursor->Next(); + } else { + break; + } + } + + return true; +} + +bool TimestampIndex::DB::EraseTimestampIndex(const CTimestampIndexKey& key) +{ + return CDBWrapper::Erase(std::make_pair(DB_TIMESTAMPINDEX, key)); +} + +TimestampIndex::TimestampIndex(size_t n_cache_size, bool f_memory, bool f_wipe) + : m_db(std::make_unique(n_cache_size, f_memory, f_wipe)) +{ +} + +TimestampIndex::~TimestampIndex() = default; + +bool TimestampIndex::WriteBlock(const CBlock& block, const CBlockIndex* pindex) +{ + // Skip genesis block + if (pindex->nHeight == 0) return true; + + // Create timestamp index key from block metadata + CTimestampIndexKey key(pindex->nTime, pindex->GetBlockHash()); + + // Write to database + return m_db->Write(key); +} + +bool TimestampIndex::Rewind(const CBlockIndex* current_tip, const CBlockIndex* new_tip) +{ + assert(current_tip->GetAncestor(new_tip->nHeight) == new_tip); + + // Erase timestamp index entries for blocks being rewound + for (const CBlockIndex* pindex = current_tip; pindex != new_tip; pindex = pindex->pprev) { + // Skip genesis block + if (pindex->nHeight == 0) continue; + + CTimestampIndexKey key(pindex->nTime, pindex->GetBlockHash()); + if (!m_db->EraseTimestampIndex(key)) { + return error("%s: Failed to erase timestamp index for block %s during rewind", + __func__, pindex->GetBlockHash().ToString()); + } + } + + // Call base class Rewind to update the best block pointer + return BaseIndex::Rewind(current_tip, new_tip); +} + +void TimestampIndex::BlockDisconnected(const std::shared_ptr& block, const CBlockIndex* pindex) +{ + // When a block is disconnected (e.g., via invalidateblock), we need to rewind the index + // to remove this block's data + const CBlockIndex* best_block_index = CurrentIndex(); + + // Only rewind if we have this block indexed + if (best_block_index && best_block_index->nHeight >= pindex->nHeight) { + if (!Rewind(best_block_index, pindex->pprev)) { + error("%s: Failed to rewind %s to previous block after disconnect", + __func__, GetName()); + } + } +} + +BaseIndex::DB& TimestampIndex::GetDB() const { return *m_db; } + +bool TimestampIndex::GetBlockHashes(uint32_t high, uint32_t low, std::vector& hashes) const +{ + if (!BlockUntilSyncedToCurrentChain()) { + return false; + } + + return m_db->ReadRange(high, low, hashes); +} diff --git a/src/index/timestampindex.h b/src/index/timestampindex.h new file mode 100644 index 000000000000..fc717e6c0d71 --- /dev/null +++ b/src/index/timestampindex.h @@ -0,0 +1,71 @@ +// Copyright (c) 2026 The Dash Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_INDEX_TIMESTAMPINDEX_H +#define BITCOIN_INDEX_TIMESTAMPINDEX_H + +#include +#include + +static constexpr bool DEFAULT_TIMESTAMPINDEX{false}; + +/** + * TimestampIndex is used to map block timestamps to block hashes. + * This allows efficient querying of blocks within a timestamp range. + * + * The index maintains a separate LevelDB database at /indexes/timestampindex/ + */ +class TimestampIndex final : public BaseIndex +{ +protected: + class DB; + +private: + const std::unique_ptr m_db; + +protected: + class DB : public BaseIndex::DB + { + public: + explicit DB(size_t n_cache_size, bool f_memory = false, bool f_wipe = false); + + /// Write a timestamp index entry to the database + bool Write(const CTimestampIndexKey& key); + + /// Read timestamp index entries within the given range + bool ReadRange(uint32_t high, uint32_t low, std::vector& hashes); + + /// Erase timestamp index entry + bool EraseTimestampIndex(const CTimestampIndexKey& key); + }; + + bool WriteBlock(const CBlock& block, const CBlockIndex* pindex) override; + + /// Custom rewind to handle timestamp index cleanup + bool Rewind(const CBlockIndex* current_tip, const CBlockIndex* new_tip) override; + + /// Handle block disconnections (e.g., from invalidateblock) + void BlockDisconnected(const std::shared_ptr& block, const CBlockIndex* pindex) override; + + BaseIndex::DB& GetDB() const override; + const char* GetName() const override { return "timestampindex"; } + + /// TimestampIndex works with pruned nodes since it only stores block metadata + bool AllowPrune() const override { return true; } + +public: + /// Constructs a new TimestampIndex. + explicit TimestampIndex(size_t n_cache_size, bool f_memory = false, bool f_wipe = false); + + /// Destructor + virtual ~TimestampIndex() override; + + /// Retrieve block hashes within the given timestamp range [low, high] + bool GetBlockHashes(uint32_t high, uint32_t low, std::vector& hashes) const; +}; + +/// Global TimestampIndex instance +extern std::unique_ptr g_timestampindex; + +#endif // BITCOIN_INDEX_TIMESTAMPINDEX_H diff --git a/src/index/timestampindex_types.h b/src/index/timestampindex_types.h new file mode 100644 index 000000000000..95ecc4bebf95 --- /dev/null +++ b/src/index/timestampindex_types.h @@ -0,0 +1,74 @@ +// Copyright (c) 2009-2010 Satoshi Nakamoto +// Copyright (c) 2009-2015 The Bitcoin Core developers +// Copyright (c) 2016 BitPay Inc. +// Copyright (c) 2023-2026 The Dash Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_INDEX_TIMESTAMPINDEX_TYPES_H +#define BITCOIN_INDEX_TIMESTAMPINDEX_TYPES_H + +#include +#include + +struct CTimestampIndexIteratorKey { + uint32_t m_time{0}; + + CTimestampIndexIteratorKey() = default; + explicit CTimestampIndexIteratorKey(uint32_t time) : + m_time{time} + { + } + + void SetNull() { m_time = 0; } + + size_t GetSerializeSize(int nType, int nVersion) const { return 4; } + + template + void Serialize(Stream& s) const + { + ser_writedata32be(s, m_time); + } + + template + void Unserialize(Stream& s) + { + m_time = ser_readdata32be(s); + } +}; + +struct CTimestampIndexKey { + uint32_t m_block_time{0}; + uint256 m_block_hash; + + CTimestampIndexKey() = default; + CTimestampIndexKey(uint32_t block_time, uint256 block_hash) : + m_block_time{block_time}, + m_block_hash{block_hash} + { + } + + void SetNull() + { + m_block_time = 0; + m_block_hash.SetNull(); + } + + size_t GetSerializeSize(int nType, int nVersion) const { return 36; } + + template + void Serialize(Stream& s) const + { + ser_writedata32be(s, m_block_time); + m_block_hash.Serialize(s); + } + + template + void Unserialize(Stream& s) + { + m_block_time = ser_readdata32be(s); + m_block_hash.Unserialize(s); + } +}; + +#endif // BITCOIN_INDEX_TIMESTAMPINDEX_TYPES_H diff --git a/src/init.cpp b/src/init.cpp index f97d98f8a766..5ec6e8283556 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -27,6 +27,7 @@ #include #include #include +#include #include #include #include @@ -150,7 +151,6 @@ using node::DEFAULT_ADDRESSINDEX; using node::DEFAULT_PRINTPRIORITY; using node::DEFAULT_SPENTINDEX; using node::DEFAULT_STOPAFTERBLOCKIMPORT; -using node::DEFAULT_TIMESTAMPINDEX; using node::LoadChainstate; using node::NodeContext; using node::ThreadImport; @@ -159,7 +159,6 @@ using node::fAddressIndex; using node::fPruneMode; using node::fReindex; using node::fSpentIndex; -using node::fTimestampIndex; using node::nPruneTarget; #ifdef ENABLE_WALLET using wallet::DEFAULT_DISABLE_WALLET; @@ -264,6 +263,9 @@ void Interrupt(NodeContext& node) if (g_txindex) { g_txindex->Interrupt(); } + if (g_timestampindex) { + g_timestampindex->Interrupt(); + } ForEachBlockFilterIndex([](BlockFilterIndex& index) { index.Interrupt(); }); if (g_coin_stats_index) { g_coin_stats_index->Interrupt(); @@ -380,6 +382,10 @@ void PrepareShutdown(NodeContext& node) g_txindex->Stop(); g_txindex.reset(); } + if (g_timestampindex) { + g_timestampindex->Stop(); + g_timestampindex.reset(); + } if (g_coin_stats_index) { g_coin_stats_index->Stop(); g_coin_stats_index.reset(); @@ -1033,8 +1039,7 @@ void InitParameterInteraction(ArgsManager& args) // (we must reconnect blocks whenever we disconnect them for these indexes to work) bool fAdditionalIndexes = args.GetBoolArg("-addressindex", DEFAULT_ADDRESSINDEX) || - args.GetBoolArg("-spentindex", DEFAULT_SPENTINDEX) || - args.GetBoolArg("-timestampindex", DEFAULT_TIMESTAMPINDEX); + args.GetBoolArg("-spentindex", DEFAULT_SPENTINDEX); if (fAdditionalIndexes && args.GetIntArg("-checklevel", DEFAULT_CHECKLEVEL) < 4) { args.ForceSetArg("-checklevel", "4"); @@ -1945,6 +1950,9 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) if (args.GetBoolArg("-txindex", DEFAULT_TXINDEX)) { LogPrintf("* Using %.1f MiB for transaction index database\n", cache_sizes.tx_index * (1.0 / 1024 / 1024)); } + if (args.GetBoolArg("-timestampindex", DEFAULT_TIMESTAMPINDEX)) { + LogPrintf("* Using %.1f MiB for timestamp index database\n", cache_sizes.timestamp_index * (1.0 / 1024 / 1024)); + } for (BlockFilterType filter_type : g_enabled_filter_types) { LogPrintf("* Using %.1f MiB for %s block filter index database\n", cache_sizes.filter_index * (1.0 / 1024 / 1024), BlockFilterTypeName(filter_type)); @@ -2000,7 +2008,6 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) fPruneMode, args.GetBoolArg("-addressindex", DEFAULT_ADDRESSINDEX), args.GetBoolArg("-spentindex", DEFAULT_SPENTINDEX), - args.GetBoolArg("-timestampindex", DEFAULT_TIMESTAMPINDEX), chainparams.GetConsensus(), fReindexChainState, cache_sizes.block_tree_db, @@ -2047,9 +2054,6 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) case ChainstateLoadingError::ERROR_SPENTIDX_NEEDS_REINDEX: strLoadError = _("You need to rebuild the database using -reindex to enable -spentindex"); break; - case ChainstateLoadingError::ERROR_TIMEIDX_NEEDS_REINDEX: - strLoadError = _("You need to rebuild the database using -reindex to enable -timestampindex"); - break; case ChainstateLoadingError::ERROR_PRUNED_NEEDS_REINDEX: strLoadError = _("You need to rebuild the database using -reindex to go back to unpruned mode. This will redownload the entire blockchain"); break; @@ -2083,7 +2087,6 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) } } else { LogPrintf("%s: address index %s\n", __func__, fAddressIndex ? "enabled" : "disabled"); - LogPrintf("%s: timestamp index %s\n", __func__, fTimestampIndex ? "enabled" : "disabled"); LogPrintf("%s: spent index %s\n", __func__, fSpentIndex ? "enabled" : "disabled"); std::optional maybe_verify_error; @@ -2262,6 +2265,13 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) } } + if (args.GetBoolArg("-timestampindex", DEFAULT_TIMESTAMPINDEX)) { + g_timestampindex = std::make_unique(cache_sizes.timestamp_index, false, fReindex); + if (!g_timestampindex->Start(chainman.ActiveChainstate())) { + return false; + } + } + for (const auto& filter_type : g_enabled_filter_types) { InitBlockFilterIndex(filter_type, cache_sizes.filter_index, false, fReindex); if (!GetBlockFilterIndex(filter_type)->Start(chainman.ActiveChainstate())) { diff --git a/src/node/blockstorage.cpp b/src/node/blockstorage.cpp index 76fd09843ac7..1f535d694d05 100644 --- a/src/node/blockstorage.cpp +++ b/src/node/blockstorage.cpp @@ -30,7 +30,6 @@ bool fPruneMode = false; uint64_t nPruneTarget = 0; bool fAddressIndex = DEFAULT_ADDRESSINDEX; -bool fTimestampIndex = DEFAULT_TIMESTAMPINDEX; bool fSpentIndex = DEFAULT_SPENTINDEX; bool CBlockIndexWorkComparator::operator()(const CBlockIndex* pa, const CBlockIndex* pb) const @@ -397,8 +396,6 @@ bool BlockManager::LoadBlockIndexDB() // Check whether we have an address index m_block_tree_db->ReadFlag("addressindex", fAddressIndex); - // Check whether we have a timestamp index - m_block_tree_db->ReadFlag("timestampindex", fTimestampIndex); // Check whether we have a spent index m_block_tree_db->ReadFlag("spentindex", fSpentIndex); return true; diff --git a/src/node/blockstorage.h b/src/node/blockstorage.h index 2cfae856c2cc..67ee1a6acee9 100644 --- a/src/node/blockstorage.h +++ b/src/node/blockstorage.h @@ -36,7 +36,6 @@ namespace node { static constexpr bool DEFAULT_ADDRESSINDEX{false}; static constexpr bool DEFAULT_SPENTINDEX{false}; static constexpr bool DEFAULT_STOPAFTERBLOCKIMPORT{false}; -static constexpr bool DEFAULT_TIMESTAMPINDEX{false}; /** The pre-allocation chunk size for blk?????.dat files (since 0.8) */ static const unsigned int BLOCKFILE_CHUNK_SIZE = 0x1000000; // 16 MiB @@ -58,8 +57,6 @@ extern uint64_t nPruneTarget; /** True if we're running in -addressindex mode. */ extern bool fAddressIndex; -/** True if we're running in -timestampindex mode. */ -extern bool fTimestampIndex; /** True if we're running in -spentindex mode. */ extern bool fSpentIndex; diff --git a/src/node/caches.cpp b/src/node/caches.cpp index a39ad7aeb661..0acca3c0b9b1 100644 --- a/src/node/caches.cpp +++ b/src/node/caches.cpp @@ -4,6 +4,7 @@ #include +#include #include #include #include @@ -19,6 +20,8 @@ CacheSizes CalculateCacheSizes(const ArgsManager& args, size_t n_indexes) nTotalCache -= sizes.block_tree_db; sizes.tx_index = std::min(nTotalCache / 8, args.GetBoolArg("-txindex", DEFAULT_TXINDEX) ? nMaxTxIndexCache << 20 : 0); nTotalCache -= sizes.tx_index; + sizes.timestamp_index = std::min(nTotalCache / 64, args.GetBoolArg("-timestampindex", DEFAULT_TIMESTAMPINDEX) ? int64_t(8 << 20) : int64_t(0)); + nTotalCache -= sizes.timestamp_index; sizes.filter_index = 0; if (n_indexes > 0) { int64_t max_cache = std::min(nTotalCache / 8, max_filter_index_cache << 20); diff --git a/src/node/caches.h b/src/node/caches.h index 67388b91fd26..6f91d3218349 100644 --- a/src/node/caches.h +++ b/src/node/caches.h @@ -17,6 +17,7 @@ struct CacheSizes { int64_t coins; int64_t tx_index; int64_t filter_index; + int64_t timestamp_index; }; CacheSizes CalculateCacheSizes(const ArgsManager& args, size_t n_indexes = 0); } // namespace node diff --git a/src/node/chainstate.cpp b/src/node/chainstate.cpp index e60eaace5630..d76623974856 100644 --- a/src/node/chainstate.cpp +++ b/src/node/chainstate.cpp @@ -51,7 +51,6 @@ std::optional LoadChainstate(bool fReset, bool fPruneMode, bool is_addrindex_enabled, bool is_spentindex_enabled, - bool is_timeindex_enabled, const Consensus::Params& consensus_params, bool fReindexChainState, int64_t nBlockTreeDBCache, @@ -125,11 +124,6 @@ std::optional LoadChainstate(bool fReset, return ChainstateLoadingError::ERROR_ADDRIDX_NEEDS_REINDEX; } - // Check for changed -timestampindex state - if (!fTimestampIndex && fTimestampIndex != is_timeindex_enabled) { - return ChainstateLoadingError::ERROR_TIMEIDX_NEEDS_REINDEX; - } - // Check for changed -spentindex state if (!fSpentIndex && fSpentIndex != is_spentindex_enabled) { return ChainstateLoadingError::ERROR_SPENTIDX_NEEDS_REINDEX; diff --git a/src/node/chainstate.h b/src/node/chainstate.h index 0022b7637d87..dbf390e27261 100644 --- a/src/node/chainstate.h +++ b/src/node/chainstate.h @@ -39,7 +39,6 @@ enum class ChainstateLoadingError { ERROR_BAD_DEVNET_GENESIS_BLOCK, ERROR_ADDRIDX_NEEDS_REINDEX, ERROR_SPENTIDX_NEEDS_REINDEX, - ERROR_TIMEIDX_NEEDS_REINDEX, ERROR_PRUNED_NEEDS_REINDEX, ERROR_LOAD_GENESIS_BLOCK_FAILED, ERROR_CHAINSTATE_UPGRADE_FAILED, @@ -95,7 +94,6 @@ std::optional LoadChainstate(bool fReset, bool fPruneMode, bool is_addrindex_enabled, bool is_spentindex_enabled, - bool is_timeindex_enabled, const Consensus::Params& consensus_params, bool fReindexChainState, int64_t nBlockTreeDBCache, diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index 203e20da3ba2..a0ec01610e58 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -19,6 +19,7 @@ #include #include #include +#include #include #include #include @@ -552,13 +553,16 @@ static RPCHelpMan getblockhashes() }, [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { + if (!g_timestampindex) { + throw JSONRPCError(RPC_MISC_ERROR, "Timestamp index is not enabled. Start with -timestampindex to enable."); + } + unsigned int high = request.params[0].getInt(); unsigned int low = request.params[1].getInt(); std::vector blockHashes; - ChainstateManager& chainman = EnsureAnyChainman(request.context); - if (LOCK(::cs_main); !GetTimestampIndex(*chainman.m_blockman.m_block_tree_db, high, low, blockHashes)) { - throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "No information available for block hashes"); + if (!g_timestampindex->GetBlockHashes(high, low, blockHashes)) { + throw JSONRPCError(RPC_MISC_ERROR, "Timestamp index is still syncing. Try again later."); } UniValue result(UniValue::VARR); diff --git a/src/rpc/index_util.cpp b/src/rpc/index_util.cpp index 743eb39d6940..938730afd73a 100644 --- a/src/rpc/index_util.cpp +++ b/src/rpc/index_util.cpp @@ -82,15 +82,3 @@ bool GetSpentIndex(CBlockTreeDB& block_tree_db, const CTxMemPool& mempool, const return block_tree_db.ReadSpentIndex(key, value); } - -bool GetTimestampIndex(CBlockTreeDB& block_tree_db, const uint32_t high, const uint32_t low, - std::vector& hashes) -{ - AssertLockHeld(::cs_main); - - if (!node::fTimestampIndex) { - throw JSONRPCError(RPC_INVALID_REQUEST, "Timestamp index is disabled. You should run Dash Core with -timestampindex (requires reindex)"); - } - - return block_tree_db.ReadTimestampIndex(high, low, hashes); -} diff --git a/src/rpc/index_util.h b/src/rpc/index_util.h index 1b480f82f53a..eda9aba9fe61 100644 --- a/src/rpc/index_util.h +++ b/src/rpc/index_util.h @@ -44,9 +44,4 @@ bool GetSpentIndex(CBlockTreeDB& block_tree_db, const CTxMemPool& mempool, const CSpentIndexValue& value) EXCLUSIVE_LOCKS_REQUIRED(::cs_main); -//! throws JSONRPCError if timestamp index is unavailable -bool GetTimestampIndex(CBlockTreeDB& block_tree_db, const uint32_t high, const uint32_t low, - std::vector& hashes) - EXCLUSIVE_LOCKS_REQUIRED(::cs_main); - #endif // BITCOIN_RPC_INDEX_UTIL_H diff --git a/src/test/util/setup_common.cpp b/src/test/util/setup_common.cpp index 2d758f1afc5f..9e24135275cb 100644 --- a/src/test/util/setup_common.cpp +++ b/src/test/util/setup_common.cpp @@ -87,7 +87,6 @@ using node::DashChainstateSetup; using node::DashChainstateSetupClose; using node::DEFAULT_ADDRESSINDEX; using node::DEFAULT_SPENTINDEX; -using node::DEFAULT_TIMESTAMPINDEX; using node::LoadChainstate; using node::NodeContext; using node::VerifyLoadedChainstate; @@ -334,7 +333,6 @@ TestingSetup::TestingSetup(const std::string& chainName, const std::vector -#include - -struct CTimestampIndexIteratorKey { -public: - uint32_t m_time; - -public: - CTimestampIndexIteratorKey() { - SetNull(); - } - - CTimestampIndexIteratorKey(uint32_t time) : m_time{time} {}; - - void SetNull() { - m_time = 0; - } - -public: - size_t GetSerializeSize(int nType, int nVersion) const { - return 4; - } - - template - void Serialize(Stream& s) const { - ser_writedata32be(s, m_time); - } - - template - void Unserialize(Stream& s) { - m_time = ser_readdata32be(s); - } -}; - -struct CTimestampIndexKey { -public: - uint32_t m_block_time; - uint256 m_block_hash; - -public: - CTimestampIndexKey() { - SetNull(); - } - - CTimestampIndexKey(uint32_t block_time, uint256 block_hash) : - m_block_time{block_time}, m_block_hash{block_hash} {}; - - void SetNull() { - m_block_time = 0; - m_block_hash.SetNull(); - } - -public: - size_t GetSerializeSize(int nType, int nVersion) const { - return 36; - } - - template - void Serialize(Stream& s) const { - ser_writedata32be(s, m_block_time); - m_block_hash.Serialize(s); - } - - template - void Unserialize(Stream& s) { - m_block_time = ser_readdata32be(s); - m_block_hash.Unserialize(s); - } -}; - -#endif // BITCOIN_TIMESTAMPINDEX_H diff --git a/src/txdb.cpp b/src/txdb.cpp index 73827a9a3e5f..f96e94a474ad 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -23,7 +23,6 @@ static constexpr uint8_t DB_COIN{'C'}; static constexpr uint8_t DB_BLOCK_FILES{'f'}; static constexpr uint8_t DB_ADDRESSINDEX{'a'}; static constexpr uint8_t DB_ADDRESSUNSPENTINDEX{'u'}; -static constexpr uint8_t DB_TIMESTAMPINDEX{'s'}; static constexpr uint8_t DB_SPENTINDEX{'p'}; static constexpr uint8_t DB_BLOCK_INDEX{'b'}; @@ -378,37 +377,6 @@ bool CBlockTreeDB::ReadAddressIndex(const uint160& addressHash, const AddressTyp return true; } -bool CBlockTreeDB::WriteTimestampIndex(const CTimestampIndexKey& timestampIndex) { - CDBBatch batch(*this); - batch.Write(std::make_pair(DB_TIMESTAMPINDEX, timestampIndex), 0); - return WriteBatch(batch); -} - -bool CBlockTreeDB::EraseTimestampIndex(const CTimestampIndexKey& timestampIndex) -{ - CDBBatch batch(*this); - batch.Erase(std::make_pair(DB_TIMESTAMPINDEX, timestampIndex)); - return WriteBatch(batch); -} - -bool CBlockTreeDB::ReadTimestampIndex(const uint32_t high, const uint32_t low, std::vector& hashes) { - std::unique_ptr pcursor(NewIterator()); - - pcursor->Seek(std::make_pair(DB_TIMESTAMPINDEX, CTimestampIndexIteratorKey(low))); - - while (pcursor->Valid()) { - std::pair key; - if (pcursor->GetKey(key) && key.first == DB_TIMESTAMPINDEX && key.second.m_block_time <= high) { - hashes.push_back(key.second.m_block_hash); - pcursor->Next(); - } else { - break; - } - } - - return true; -} - bool CBlockTreeDB::WriteFlag(const std::string &name, bool fValue) { return Write(std::make_pair(DB_FLAG, name), fValue ? uint8_t{'1'} : uint8_t{'0'}); } diff --git a/src/txdb.h b/src/txdb.h index 886071586514..752522860cfb 100644 --- a/src/txdb.h +++ b/src/txdb.h @@ -10,7 +10,6 @@ #include #include #include -#include #include #include @@ -102,10 +101,6 @@ class CBlockTreeDB : public CDBWrapper std::vector& addressIndex, const int32_t start = 0, const int32_t end = 0); - bool WriteTimestampIndex(const CTimestampIndexKey& timestampIndex); - bool EraseTimestampIndex(const CTimestampIndexKey& timestampIndex); - bool ReadTimestampIndex(const uint32_t high, const uint32_t low, std::vector& hashes); - bool WriteFlag(const std::string &name, bool fValue); bool ReadFlag(const std::string &name, bool &fValue); bool LoadBlockIndexGuts(const Consensus::Params& consensusParams, std::function insertBlockIndex) diff --git a/src/validation.cpp b/src/validation.cpp index 95629b462cf1..b8186117fa35 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -76,13 +76,11 @@ using node::CBlockIndexHeightOnlyComparator; using node::CBlockIndexWorkComparator; using node::DEFAULT_ADDRESSINDEX; using node::DEFAULT_SPENTINDEX; -using node::DEFAULT_TIMESTAMPINDEX; using node::fAddressIndex; using node::fImporting; using node::fPruneMode; using node::fReindex; using node::fSpentIndex; -using node::fTimestampIndex; using node::ReadBlockFromDisk; using node::SnapshotMetadata; using node::UndoReadFromDisk; @@ -2140,13 +2138,6 @@ DisconnectResult CChainState::DisconnectBlock(const CBlock& block, const CBlockI } } - if (fTimestampIndex) { - if (!m_blockman.m_block_tree_db->EraseTimestampIndex(CTimestampIndexKey(pindex->nTime, pindex->GetBlockHash()))) { - AbortNode("Failed to delete timestamp index"); - return DISCONNECT_FAILED; - } - } - // move best block pointer to prevout block view.SetBestBlock(pindex->pprev->GetBlockHash()); m_evoDb.WriteBestBlock(pindex->pprev->GetBlockHash()); @@ -2692,10 +2683,6 @@ bool CChainState::ConnectBlock(const CBlock& block, BlockValidationState& state, if (!m_blockman.m_block_tree_db->UpdateSpentIndex(spentIndex)) return AbortNode(state, "Failed to write transaction index"); - if (fTimestampIndex) - if (!m_blockman.m_block_tree_db->WriteTimestampIndex(CTimestampIndexKey(pindex->nTime, pindex->GetBlockHash()))) - return AbortNode(state, "Failed to write timestamp index"); - int64_t nTime8 = GetTimeMicros(); nTimeIndexWrite += nTime8 - nTime7; LogPrint(BCLog::BENCHMARK, " - Index writing: %.2fms [%.2fs (%.2fms/blk)]\n", MILLI * (nTime8 - nTime7), nTimeIndexWrite * MICRO, nTimeIndexWrite * MILLI / nBlocksTotal); @@ -4821,11 +4808,6 @@ bool CChainState::RollforwardBlock(const CBlockIndex* pindex, CCoinsViewCache& i return error("RollforwardBlock(DASH): Failed to write transaction index"); } - if (fTimestampIndex) { - if (!m_blockman.m_block_tree_db->WriteTimestampIndex(CTimestampIndexKey(pindex->nTime, pindex->GetBlockHash()))) - return error("RollforwardBlock(DASH): Failed to write timestamp index"); - } - return true; } @@ -5011,10 +4993,6 @@ void ChainstateManager::InitAdditionalIndexes() fAddressIndex = gArgs.GetBoolArg("-addressindex", DEFAULT_ADDRESSINDEX); m_blockman.m_block_tree_db->WriteFlag("addressindex", fAddressIndex); - // Use the provided setting for -timestampindex in the new database - fTimestampIndex = gArgs.GetBoolArg("-timestampindex", DEFAULT_TIMESTAMPINDEX); - m_blockman.m_block_tree_db->WriteFlag("timestampindex", fTimestampIndex); - // Use the provided setting for -spentindex in the new database fSpentIndex = gArgs.GetBoolArg("-spentindex", DEFAULT_SPENTINDEX); m_blockman.m_block_tree_db->WriteFlag("spentindex", fSpentIndex); diff --git a/test/util/data/non-backported.txt b/test/util/data/non-backported.txt index a33f92fa6a74..b648a52b84d9 100644 --- a/test/util/data/non-backported.txt +++ b/test/util/data/non-backported.txt @@ -18,6 +18,7 @@ src/evo/*.cpp src/evo/*.h src/governance/*.cpp src/governance/*.h +src/index/timestamp*.h src/instantsend/*.cpp src/instantsend/*.h src/llmq/*.cpp From 931d03a51a7564ba354a5a086a32c698e0b8ea02 Mon Sep 17 00:00:00 2001 From: UdjinM6 Date: Mon, 19 Jan 2026 21:43:01 +0300 Subject: [PATCH 02/14] test: update timestampindex functional test for async behavior MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Updates feature_timestampindex.py to validate async index capabilities: - Test that index can be disabled without -reindex - Test that index can be re-enabled without -reindex (syncs in background) - Use bump_mocktime() to ensure distinct timestamps for range queries - Verify correct RPC behavior after toggling These tests validate key advantages of the async migration over the old synchronous implementation, where changing the index required a full blockchain reindex. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Sonnet 4.5 --- test/functional/feature_timestampindex.py | 53 +++++++++++++++-------- 1 file changed, 35 insertions(+), 18 deletions(-) diff --git a/test/functional/feature_timestampindex.py b/test/functional/feature_timestampindex.py index 76b188b8cd1f..14111a7e4719 100755 --- a/test/functional/feature_timestampindex.py +++ b/test/functional/feature_timestampindex.py @@ -8,7 +8,6 @@ # from test_framework.test_framework import BitcoinTestFramework -from test_framework.test_node import ErrorMatch from test_framework.util import assert_equal @@ -16,43 +15,61 @@ class TimestampIndexTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True - self.num_nodes = 4 + self.num_nodes = 2 def setup_network(self): self.add_nodes(self.num_nodes) - # Nodes 0/1 are "wallet" nodes self.start_node(0) self.start_node(1, ["-timestampindex"]) - # Nodes 2/3 are used for testing - self.start_node(2) - self.start_node(3, ["-timestampindex"]) self.connect_nodes(0, 1) - self.connect_nodes(0, 2) - self.connect_nodes(0, 3) - - self.sync_all() def run_test(self): self.log.info("Test that settings can be disabled without -reindex...") - self.restart_node(1, ["-timestampindex=0"]) + self.generate(self.nodes[0], 1) + self.stop_node(1) + self.generate(self.nodes[0], 1, sync_fun=self.no_op) + self.start_node(1, ["-timestampindex=0"]) self.connect_nodes(0, 1) self.sync_all() - self.log.info("Test that settings can't be enabled without -reindex...") + + self.log.info("Test that settings can be enabled without -reindex...") self.stop_node(1) - self.nodes[1].assert_start_raises_init_error(["-timestampindex"], "You need to rebuild the database using -reindex to enable -timestampindex", match=ErrorMatch.PARTIAL_REGEX) - self.start_node(1, ["-timestampindex", "-reindex"]) + self.generate(self.nodes[0], 1, sync_fun=self.no_op) + self.start_node(1, ["-timestampindex"]) self.connect_nodes(0, 1) self.sync_all() - self.log.info("Mining 5 blocks...") - blockhashes = self.generate(self.nodes[0], 5) + self.log.info("Check timestamp index via getblockhashes rpc") + blockhashes = [] + for _ in range(5): + self.bump_mocktime(1) + blockhashes += self.generate(self.nodes[0], 1) low = self.nodes[0].getblock(blockhashes[0])["time"] high = self.nodes[0].getblock(blockhashes[4])["time"] - self.log.info("Checking timestamp index...") + self.bump_mocktime(1) hashes = self.nodes[1].getblockhashes(high, low) assert_equal(len(hashes), 5) assert_equal(sorted(blockhashes), sorted(hashes)) - self.log.info("Passed") + + self.log.info("Testing reorg handling...") + # Invalidate the last 2 blocks on both nodes + self.nodes[0].invalidateblock(blockhashes[3]) + self.nodes[1].invalidateblock(blockhashes[3]) + + # Verify that invalidated blocks are no longer in the index + hashes_after_invalidate = self.nodes[1].getblockhashes(high, low) + assert_equal(len(hashes_after_invalidate), 3) + assert_equal(sorted(hashes_after_invalidate), sorted(blockhashes[:3])) + + # Reconsider the blocks on both nodes + self.nodes[0].reconsiderblock(blockhashes[3]) + self.nodes[1].reconsiderblock(blockhashes[3]) + self.sync_all() + + # Verify all blocks are back in the index + hashes_after_reconsider = self.nodes[1].getblockhashes(high, low) + assert_equal(len(hashes_after_reconsider), 5) + assert_equal(sorted(hashes_after_reconsider), sorted(blockhashes)) if __name__ == '__main__': From f9e86c217cb529d4bf05e30d29a9ca461cde97c5 Mon Sep 17 00:00:00 2001 From: UdjinM6 Date: Mon, 19 Jan 2026 21:43:01 +0300 Subject: [PATCH 03/14] feat: migrate SpentIndex from synchronous to async operation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This commit migrates SpentIndex to use the async BaseIndex pattern, similar to the TimestampIndex migration. Key changes: - Create src/index/spentindex.{h,cpp} with async SpentIndex class - Use CBlockUndo to access historical spent output data - Incompatible with pruned nodes (requires undo data) - Database location: indexes/spentindex/ (128MB cache) - Remove synchronous writes from validation.cpp - Remove database methods from txdb.{h,cpp} - Remove global fSpentIndex flag from blockstorage.{h,cpp} - Remove validation check from chainstate.{h,cpp} - Update RPC handlers (getspentinfo, getrawtransaction) - Move CAddressUnspent* structures to addressindex.h - Fix locking: query async index before acquiring cs_main in TxToJSON Implementation details: - Uses undo data via UndoReadFromDisk() for historical UTXO access - Cache size: 128MB (~1.2M entries, proportional to TxIndex) - Properly handles both mempool and confirmed transactions - Background syncing with BlockUntilSyncedToCurrentChain() 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Sonnet 4.5 --- src/Makefile.am | 4 +- src/addressindex.h | 82 ++++++++++++- src/core_write.cpp | 2 +- src/index/spentindex.cpp | 171 +++++++++++++++++++++++++++ src/index/spentindex.h | 82 +++++++++++++ src/index/spentindex_types.h | 91 +++++++++++++++ src/init.cpp | 32 ++++-- src/node/blockstorage.cpp | 3 - src/node/blockstorage.h | 3 - src/node/caches.cpp | 3 + src/node/caches.h | 1 + src/node/chainstate.cpp | 6 - src/node/chainstate.h | 2 - src/rpc/index_util.cpp | 15 --- src/rpc/index_util.h | 6 - src/rpc/node.cpp | 27 ++++- src/rpc/rawtransaction.cpp | 51 ++++++-- src/spentindex.h | 185 ------------------------------ src/test/util/setup_common.cpp | 2 - src/txdb.cpp | 17 --- src/txdb.h | 5 +- src/txmempool.cpp | 7 ++ src/txmempool.h | 2 +- src/validation.cpp | 53 +-------- test/util/data/non-backported.txt | 1 + 25 files changed, 536 insertions(+), 317 deletions(-) create mode 100644 src/index/spentindex.cpp create mode 100644 src/index/spentindex.h create mode 100644 src/index/spentindex_types.h delete mode 100644 src/spentindex.h diff --git a/src/Makefile.am b/src/Makefile.am index 224672ecd6e6..708da993f1a2 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -158,7 +158,6 @@ BITCOIN_CORE_H = \ active/quorums.h \ addrdb.h \ addressindex.h \ - spentindex.h \ addrman.h \ addrman_impl.h \ attributes.h \ @@ -251,6 +250,8 @@ BITCOIN_CORE_H = \ index/blockfilterindex.h \ index/coinstatsindex.h \ index/disktxpos.h \ + index/spentindex.h \ + index/spentindex_types.h \ index/timestampindex.h \ index/timestampindex_types.h \ index/txindex.h \ @@ -540,6 +541,7 @@ libbitcoin_node_a_SOURCES = \ index/base.cpp \ index/blockfilterindex.cpp \ index/coinstatsindex.cpp \ + index/spentindex.cpp \ index/timestampindex.cpp \ index/txindex.cpp \ init.cpp \ diff --git a/src/addressindex.h b/src/addressindex.h index 55dbc8284a58..7152e9dc28ff 100644 --- a/src/addressindex.h +++ b/src/addressindex.h @@ -9,14 +9,13 @@ #define BITCOIN_ADDRESSINDEX_H #include +#include