From 3153829255a3385894f658464e8547b9ae30dfb6 Mon Sep 17 00:00:00 2001 From: barrystyle Date: Mon, 21 Feb 2022 09:42:11 +0800 Subject: [PATCH 1/8] Implement a persistent powhash cache, to speed up powhashing operations. Signed-off-by: barrystyle --- src/Makefile.am | 14 +++++++++++-- src/logging.cpp | 1 + src/logging.h | 1 + src/powcache.cpp | 24 ++++++++++++++++++++++ src/powcache.h | 44 ++++++++++++++++++++++++++++++++++++++++ src/primitives/block.cpp | 30 +++++++++++++++++++++++++++ src/primitives/block.h | 2 ++ 7 files changed, 114 insertions(+), 2 deletions(-) create mode 100644 src/powcache.cpp create mode 100644 src/powcache.h diff --git a/src/Makefile.am b/src/Makefile.am index 328b14a62f..296f1d5da7 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -434,6 +434,8 @@ libbitcoin_consensus_a_SOURCES = \ consensus/params.h \ consensus/tx_check.cpp \ consensus/validation.h \ + dbwrapper.cpp \ + dbwrapper.h \ hash.cpp \ hash.h \ prevector.h \ @@ -441,6 +443,8 @@ libbitcoin_consensus_a_SOURCES = \ primitives/block.h \ primitives/transaction.cpp \ primitives/transaction.h \ + powcache.cpp \ + powcache.h \ pubkey.cpp \ pubkey.h \ script/bitcoinconsensus.cpp \ @@ -610,11 +614,17 @@ bitcoin_tx_SOURCES += bitcoin-tx-res.rc endif bitcoin_tx_LDADD = \ - $(LIBUNIVALUE) \ + $(LIBBITCOIN_SERVER) \ + $(LIBBITCOIN_WALLET) \ $(LIBBITCOIN_COMMON) \ + $(LIBUNIVALUE) \ $(LIBBITCOIN_UTIL) \ + $(LIBBITCOIN_ZMQ) \ $(LIBBITCOIN_CONSENSUS) \ $(LIBBITCOIN_CRYPTO) \ + $(LIBLEVELDB) \ + $(LIBLEVELDB_SSE42) \ + $(LIBMEMENV) \ $(LIBSECP256K1) bitcoin_tx_LDADD += $(BOOST_LIBS) @@ -658,7 +668,7 @@ endif libbitcoinconsensus_la_LDFLAGS = $(AM_LDFLAGS) -no-undefined $(RELDFLAGS) libbitcoinconsensus_la_LIBADD = $(LIBSECP256K1) -libbitcoinconsensus_la_CPPFLAGS = $(AM_CPPFLAGS) -I$(builddir)/obj -I$(srcdir)/secp256k1/include -DBUILD_BITCOIN_INTERNAL $(BOOST_CPPFLAGS) +libbitcoinconsensus_la_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -I$(builddir)/obj -I$(srcdir)/secp256k1/include -DBUILD_BITCOIN_INTERNAL $(BOOST_CPPFLAGS) libbitcoinconsensus_la_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) endif diff --git a/src/logging.cpp b/src/logging.cpp index 6fd916b603..ffb62b5936 100644 --- a/src/logging.cpp +++ b/src/logging.cpp @@ -163,6 +163,7 @@ const CLogCategoryDesc LogCategories[] = {BCLog::QT, "qt"}, {BCLog::LEVELDB, "leveldb"}, {BCLog::VALIDATION, "validation"}, + {BCLog::POWCACHE, "powcache"}, {BCLog::ALL, "1"}, {BCLog::ALL, "all"}, }; diff --git a/src/logging.h b/src/logging.h index b2fde1b9ea..07ff235024 100644 --- a/src/logging.h +++ b/src/logging.h @@ -55,6 +55,7 @@ namespace BCLog { QT = (1 << 19), LEVELDB = (1 << 20), VALIDATION = (1 << 21), + POWCACHE = (1 << 22), ALL = ~(uint32_t)0, }; diff --git a/src/powcache.cpp b/src/powcache.cpp new file mode 100644 index 0000000000..b70eaa9216 --- /dev/null +++ b/src/powcache.cpp @@ -0,0 +1,24 @@ +// Copyright (c) 2022 barrystyle +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include + +PowCacheDB pow_cache(GetDataDir(false) / "powcache", 4194304, false, false); + +PowCacheDB::PowCacheDB(fs::path ldb_path, size_t nCacheSize, bool fMemory, bool fWipe) : + db(ldb_path, nCacheSize, fMemory, fWipe, true) +{} + +bool PowCacheDB::HaveCacheEntry(const uint256& lookupHash) const { + return db.Exists(lookupHash); +} + +bool PowCacheDB::GetCacheEntry(const uint256& lookupHash, uint256& powHash) const { + return db.Read(lookupHash, powHash); +} + +bool PowCacheDB::WriteCacheEntry(const uint256& lookupHash, uint256& powHash) { + return db.Write(lookupHash, powHash); +} + diff --git a/src/powcache.h b/src/powcache.h new file mode 100644 index 0000000000..0a475811dd --- /dev/null +++ b/src/powcache.h @@ -0,0 +1,44 @@ +// Copyright (c) 2022 barrystyle +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_POWCACHE_H +#define BITCOIN_POWCACHE_H + +#include +#include + +/** Implements a persistent hash lookup cache, using SHA1 hash as the key + * allowing instant retrieval of the more cpu-expensive pow hash if known. + * The hash is not tied to a height, preventing invalid hashes from potentially + * being returned in the instance of a block reorganisation etc. + */ +class PowCacheDB +{ +protected: + CDBWrapper db; + +private: + int hit_log{0}; + int miss_log{0}; + +public: + explicit PowCacheDB(fs::path ldb_path, size_t nCacheSize, bool fMemory, bool fWipe); + + int hit(bool inc = false) { + if (inc) ++hit_log; + return hit_log; + } + int miss(bool inc = false) { + if (inc) ++miss_log; + return miss_log; + } + + bool HaveCacheEntry(const uint256& lookupHash) const; + bool GetCacheEntry(const uint256& lookupHash, uint256& powHash) const; + bool WriteCacheEntry(const uint256& lookupHash, uint256& powHash); +}; + +extern PowCacheDB pow_cache; + +#endif // BITCOIN_POWCACHE_H diff --git a/src/primitives/block.cpp b/src/primitives/block.cpp index d673a3d6bf..b35b5e883e 100644 --- a/src/primitives/block.cpp +++ b/src/primitives/block.cpp @@ -1,9 +1,13 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto // Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2022 barrystyle // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include +#include #include +#include #include #include @@ -12,7 +16,33 @@ uint256 CBlockHeader::GetHash() const return GetPoWHash(); } +uint256 CBlockHeader::GetLightHash() const +{ + uint256 hash; + CSHA1().Write((const unsigned char*)this, 80).Finalize((unsigned char*)&hash); + return hash; +} + uint256 CBlockHeader::GetPoWHash() const +{ + //! light sha1 hash for lookup + const uint256 lookupHash = GetLightHash(); + bool cacheEntry = pow_cache.HaveCacheEntry(lookupHash); + if (cacheEntry) { + uint256 powHash; + pow_cache.GetCacheEntry(lookupHash, powHash); + LogPrint(BCLog::POWCACHE, "%s - cachehit %6d cachemiss %6d (%s)\n", __func__, pow_cache.hit(true), pow_cache.miss(), powHash.ToString()); + return powHash; + } + + //! store for later usage + uint256 powHash = GetHeavyHash(); + pow_cache.WriteCacheEntry(lookupHash, powHash); + LogPrint(BCLog::POWCACHE, "%s - cachehit %6d cachemiss %6d (%s)\n", __func__, pow_cache.hit(), pow_cache.miss(true), powHash.ToString()); + return powHash; +} + +uint256 CBlockHeader::GetHeavyHash() const { uint256 seed; CSHA3_256().Write(hashPrevBlock.begin(), 32).Finalize(seed.begin()); diff --git a/src/primitives/block.h b/src/primitives/block.h index cecd755da8..730501d3e7 100644 --- a/src/primitives/block.h +++ b/src/primitives/block.h @@ -62,6 +62,8 @@ class CBlockHeader uint256 GetHash() const; uint256 GetPoWHash() const; + uint256 GetLightHash() const; + uint256 GetHeavyHash() const; int64_t GetBlockTime() const { From abe7f854b0532145ee926acf0778071089b826ea Mon Sep 17 00:00:00 2001 From: Anton Kovalov Date: Wed, 13 Jul 2022 11:16:32 +0200 Subject: [PATCH 2/8] Fixes dependecy hell introduced by powcache Rationel: Convience libraries built by bitcoin contain logically separated code, which allows for modularity. Introducing powcache -> dbwrapper dependecy breaks this modularity by making consensus dependant on server/wallet functionality. Apart from that it breaks builds for libconsesnsus shared library, and bruteforce fixing by satisfying every symbol is incorrect. It is designed to be stand-alone and lightweight for a reason. Thus this commit relinks powcache.* files to server library, and implements cache lookup outside of primitives. --- src/Makefile.am | 16 ++++------------ src/primitives/block.cpp | 23 +---------------------- src/primitives/block.h | 3 +-- 3 files changed, 6 insertions(+), 36 deletions(-) diff --git a/src/Makefile.am b/src/Makefile.am index 296f1d5da7..1fa6290ffb 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -173,6 +173,7 @@ BITCOIN_CORE_H = \ policy/rbf.h \ policy/settings.h \ pow.h \ + powcache.h \ protocol.h \ psbt.h \ random.h \ @@ -300,6 +301,7 @@ libbitcoin_server_a_SOURCES = \ policy/rbf.cpp \ policy/settings.cpp \ pow.cpp \ + powcache.cpp \ rest.cpp \ rpc/blockchain.cpp \ rpc/mining.cpp \ @@ -434,8 +436,6 @@ libbitcoin_consensus_a_SOURCES = \ consensus/params.h \ consensus/tx_check.cpp \ consensus/validation.h \ - dbwrapper.cpp \ - dbwrapper.h \ hash.cpp \ hash.h \ prevector.h \ @@ -443,8 +443,6 @@ libbitcoin_consensus_a_SOURCES = \ primitives/block.h \ primitives/transaction.cpp \ primitives/transaction.h \ - powcache.cpp \ - powcache.h \ pubkey.cpp \ pubkey.h \ script/bitcoinconsensus.cpp \ @@ -614,17 +612,11 @@ bitcoin_tx_SOURCES += bitcoin-tx-res.rc endif bitcoin_tx_LDADD = \ - $(LIBBITCOIN_SERVER) \ - $(LIBBITCOIN_WALLET) \ - $(LIBBITCOIN_COMMON) \ $(LIBUNIVALUE) \ + $(LIBBITCOIN_COMMON) \ $(LIBBITCOIN_UTIL) \ - $(LIBBITCOIN_ZMQ) \ $(LIBBITCOIN_CONSENSUS) \ $(LIBBITCOIN_CRYPTO) \ - $(LIBLEVELDB) \ - $(LIBLEVELDB_SSE42) \ - $(LIBMEMENV) \ $(LIBSECP256K1) bitcoin_tx_LDADD += $(BOOST_LIBS) @@ -668,7 +660,7 @@ endif libbitcoinconsensus_la_LDFLAGS = $(AM_LDFLAGS) -no-undefined $(RELDFLAGS) libbitcoinconsensus_la_LIBADD = $(LIBSECP256K1) -libbitcoinconsensus_la_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -I$(builddir)/obj -I$(srcdir)/secp256k1/include -DBUILD_BITCOIN_INTERNAL $(BOOST_CPPFLAGS) +libbitcoinconsensus_la_CPPFLAGS = $(AM_CPPFLAGS) -I$(builddir)/obj -I$(srcdir)/secp256k1/include -DBUILD_BITCOIN_INTERNAL $(BOOST_CPPFLAGS) libbitcoinconsensus_la_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) endif diff --git a/src/primitives/block.cpp b/src/primitives/block.cpp index b35b5e883e..b1000ca43f 100644 --- a/src/primitives/block.cpp +++ b/src/primitives/block.cpp @@ -5,9 +5,7 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include -#include #include -#include #include #include @@ -16,7 +14,7 @@ uint256 CBlockHeader::GetHash() const return GetPoWHash(); } -uint256 CBlockHeader::GetLightHash() const +uint256 CBlockHeader::GetCacheHash() const { uint256 hash; CSHA1().Write((const unsigned char*)this, 80).Finalize((unsigned char*)&hash); @@ -24,25 +22,6 @@ uint256 CBlockHeader::GetLightHash() const } uint256 CBlockHeader::GetPoWHash() const -{ - //! light sha1 hash for lookup - const uint256 lookupHash = GetLightHash(); - bool cacheEntry = pow_cache.HaveCacheEntry(lookupHash); - if (cacheEntry) { - uint256 powHash; - pow_cache.GetCacheEntry(lookupHash, powHash); - LogPrint(BCLog::POWCACHE, "%s - cachehit %6d cachemiss %6d (%s)\n", __func__, pow_cache.hit(true), pow_cache.miss(), powHash.ToString()); - return powHash; - } - - //! store for later usage - uint256 powHash = GetHeavyHash(); - pow_cache.WriteCacheEntry(lookupHash, powHash); - LogPrint(BCLog::POWCACHE, "%s - cachehit %6d cachemiss %6d (%s)\n", __func__, pow_cache.hit(), pow_cache.miss(true), powHash.ToString()); - return powHash; -} - -uint256 CBlockHeader::GetHeavyHash() const { uint256 seed; CSHA3_256().Write(hashPrevBlock.begin(), 32).Finalize(seed.begin()); diff --git a/src/primitives/block.h b/src/primitives/block.h index 730501d3e7..7f47594b00 100644 --- a/src/primitives/block.h +++ b/src/primitives/block.h @@ -62,8 +62,7 @@ class CBlockHeader uint256 GetHash() const; uint256 GetPoWHash() const; - uint256 GetLightHash() const; - uint256 GetHeavyHash() const; + uint256 GetCacheHash() const; int64_t GetBlockTime() const { From 1dcb71c1fe2aa59bfa1d49eacfd289d2fef32566 Mon Sep 17 00:00:00 2001 From: Anton Kovalov Date: Wed, 13 Jul 2022 14:28:34 +0200 Subject: [PATCH 3/8] Refactors powcache implementation 1. Adds a wrapper around PoW cache database, which allows for easy substitution of any heavy block.GetHash() encounter. 2. Removes redundant calls. Consideration: Performing a database lookup with 160 bit long buffer instead of 256 bit blob. Due to the fact that SHA1 output is 160 bits long 256 - 160 = 96 bits are zeros simply taking up space. It is not a big issue, but may result in extra 1 MB of data every week. --- src/powcache.cpp | 44 +++++++++++++++++++++++++++++++++++--------- src/powcache.h | 47 ++++++++++++++++++++++++++--------------------- 2 files changed, 61 insertions(+), 30 deletions(-) diff --git a/src/powcache.cpp b/src/powcache.cpp index b70eaa9216..8f22e20694 100644 --- a/src/powcache.cpp +++ b/src/powcache.cpp @@ -4,21 +4,47 @@ #include -PowCacheDB pow_cache(GetDataDir(false) / "powcache", 4194304, false, false); - -PowCacheDB::PowCacheDB(fs::path ldb_path, size_t nCacheSize, bool fMemory, bool fWipe) : +CPowCacheDB::CPowCacheDB(fs::path ldb_path, size_t nCacheSize, bool fMemory, bool fWipe) : db(ldb_path, nCacheSize, fMemory, fWipe, true) {} -bool PowCacheDB::HaveCacheEntry(const uint256& lookupHash) const { - return db.Exists(lookupHash); -} +Optional CPowCacheDB::GetCacheEntry(const uint256& lookupHash) const { + uint256 blockHash; + if (db.Read(lookupHash, blockHash)) return blockHash; -bool PowCacheDB::GetCacheEntry(const uint256& lookupHash, uint256& powHash) const { - return db.Read(lookupHash, powHash); + return nullopt; } -bool PowCacheDB::WriteCacheEntry(const uint256& lookupHash, uint256& powHash) { +bool CPowCacheDB::WriteCacheEntry(const uint256& lookupHash, const uint256& powHash) { return db.Write(lookupHash, powHash); } +void CPowHashProxy::Init(size_t nCacheSize) +{ + pow_cachedb.reset(); + pow_cachedb.reset(new CPowCacheDB(GetDataDir(false) / "powcache", nCacheSize, false, false)); +}; + +void CPowHashProxy::Stop() +{ + pow_cachedb.reset(); +}; + +uint256 CPowHashProxy::GetHash(CBlockHeader& block) +{ + if(pow_cachedb) + { + uint256 cacheHash = block.GetCacheHash(); + + if (Optional optionalBlockHash = pow_cachedb->GetCacheEntry(cacheHash)) + return optionalBlockHash.get(); + + uint256 blockHash = block.GetHash(); + pow_cachedb->WriteCacheEntry(cacheHash, blockHash); + return blockHash; + } + + return block.GetHash(); +}; + +CPowHashProxy powHashProxy; diff --git a/src/powcache.h b/src/powcache.h index 0a475811dd..81dbd78842 100644 --- a/src/powcache.h +++ b/src/powcache.h @@ -6,39 +6,44 @@ #define BITCOIN_POWCACHE_H #include +#include +#include #include -/** Implements a persistent hash lookup cache, using SHA1 hash as the key - * allowing instant retrieval of the more cpu-expensive pow hash if known. +/** Implements a persistent hash lookup cache, using SHA1 hash as the key. + * If hash was previosuly encountered and stored it allows for instant retrieval of the more cpu-expensive pow HeavyHash. * The hash is not tied to a height, preventing invalid hashes from potentially * being returned in the instance of a block reorganisation etc. */ -class PowCacheDB + +//! Max memory allocated for HeavyHash cache (4 MiB ~ 4.2 MB) +static const int64_t nMaxHeavyHashCache = 4; + +class CPowCacheDB { protected: CDBWrapper db; -private: - int hit_log{0}; - int miss_log{0}; +public: + explicit CPowCacheDB(fs::path ldb_path, size_t nCacheSize, bool fMemory, bool fWipe); + + // Read method already checks for record existence + Optional GetCacheEntry(const uint256& lookupHash) const; + bool WriteCacheEntry(const uint256& lookupHash, const uint256& powHash); +}; + +class CPowHashProxy +{ +protected: + std::unique_ptr pow_cachedb; public: - explicit PowCacheDB(fs::path ldb_path, size_t nCacheSize, bool fMemory, bool fWipe); - - int hit(bool inc = false) { - if (inc) ++hit_log; - return hit_log; - } - int miss(bool inc = false) { - if (inc) ++miss_log; - return miss_log; - } - - bool HaveCacheEntry(const uint256& lookupHash) const; - bool GetCacheEntry(const uint256& lookupHash, uint256& powHash) const; - bool WriteCacheEntry(const uint256& lookupHash, uint256& powHash); + void Init(size_t nCacheSize); + void Stop(); + uint256 GetHash(CBlockHeader& block); }; -extern PowCacheDB pow_cache; +// Global wrapper around CPoWCacheDB object. It is responsible for its existence, and based on this it decides how to retrieve block's hash +extern CPowHashProxy powHashProxy; #endif // BITCOIN_POWCACHE_H From 466eb66b116fd86fdc2592161766c476efefccf4 Mon Sep 17 00:00:00 2001 From: Anton Kovalov Date: Wed, 13 Jul 2022 14:34:17 +0200 Subject: [PATCH 4/8] Initializes PoW cache database + logs info --- src/init.cpp | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/init.cpp b/src/init.cpp index 48a2838c1b..36c50d4519 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -34,6 +34,7 @@ #include #include #include +#include #include #include #include @@ -278,6 +279,7 @@ void Shutdown(NodeContext& node) g_chainstate->ResetCoinsViews(); } pblocktree.reset(); + powHashProxy.Stop(); } for (const auto& client : node.chain_clients) { client->stop(); @@ -1508,6 +1510,8 @@ bool AppInitMain(NodeContext& node) int64_t nCoinDBCache = std::min(nTotalCache / 2, (nTotalCache / 4) + (1 << 23)); // use 25%-50% of the remainder for disk cache nCoinDBCache = std::min(nCoinDBCache, nMaxCoinsDBCache << 20); // cap total coins db cache nTotalCache -= nCoinDBCache; + int64_t nHeavyHashCache = std::min(nTotalCache / 32, nMaxHeavyHashCache << 20); + nTotalCache -= nHeavyHashCache; nCoinCacheUsage = nTotalCache; // the rest goes to in-memory cache int64_t nMempoolSizeMax = gArgs.GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000; LogPrintf("Cache configuration:\n"); @@ -1520,6 +1524,7 @@ bool AppInitMain(NodeContext& node) filter_index_cache * (1.0 / 1024 / 1024), BlockFilterTypeName(filter_type)); } LogPrintf("* Using %.1f MiB for chain state database\n", nCoinDBCache * (1.0 / 1024 / 1024)); + LogPrintf("* Using %.1f MiB for heavy hash database\n", nHeavyHashCache * (1.0 / 1024 / 1024)); LogPrintf("* Using %.1f MiB for in-memory UTXO set (plus up to %.1f MiB of unused mempool space)\n", nCoinCacheUsage * (1.0 / 1024 / 1024), nMempoolSizeMax * (1.0 / 1024 / 1024)); bool fLoaded = false; @@ -1538,8 +1543,11 @@ bool AppInitMain(NodeContext& node) g_chainstate = MakeUnique(); UnloadBlockIndex(); + // Initialize cache for PoW HeavyHash[es] + powHashProxy.Init(nHeavyHashCache); + // new CBlockTreeDB tries to delete the existing file, which - // fails if it's still open from the previous loop. Close it first: + // fails if it's still open from the previous loop. Close it first: pblocktree.reset(); pblocktree.reset(new CBlockTreeDB(nBlockTreeDBCache, false, fReset)); From 66847875985d429ceea796dc631c7919c8ae42bf Mon Sep 17 00:00:00 2001 From: Anton Kovalov Date: Wed, 13 Jul 2022 14:35:17 +0200 Subject: [PATCH 5/8] Adds hash lookup in performance critical places --- src/chain.h | 3 ++- src/validation.cpp | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/chain.h b/src/chain.h index 64c016a1d6..f44aa26a8b 100644 --- a/src/chain.h +++ b/src/chain.h @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -360,7 +361,7 @@ class CDiskBlockIndex : public CBlockIndex block.nTime = nTime; block.nBits = nBits; block.nNonce = nNonce; - return block.GetHash(); + return powHashProxy.GetHash(block); } diff --git a/src/validation.cpp b/src/validation.cpp index c4dc6fd338..71f4f9456b 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -1148,7 +1148,7 @@ bool ReadBlockFromDisk(CBlock& block, const FlatFilePos& pos, const Consensus::P } // Check the header - if (!CheckProofOfWork(block.GetPoWHash(), block.nBits, consensusParams)) + if (!CheckProofOfWork(powHashProxy.GetHash(block), block.nBits, consensusParams)) return error("ReadBlockFromDisk: Errors in block header at %s", pos.ToString()); return true; @@ -1164,7 +1164,7 @@ bool ReadBlockFromDisk(CBlock& block, const CBlockIndex* pindex, const Consensus if (!ReadBlockFromDisk(block, blockPos, consensusParams)) return false; - if (block.GetHash() != pindex->GetBlockHash()) + if (powHashProxy.GetHash(block) != pindex->GetBlockHash()) return error("ReadBlockFromDisk(CBlock&, CBlockIndex*): GetHash() doesn't match index for %s at %s", pindex->ToString(), pindex->GetBlockPos().ToString()); return true; From 462075a154ac90fa987c04ced3fdfd0566ef9ade Mon Sep 17 00:00:00 2001 From: Anton Kovalov Date: Mon, 18 Jul 2022 18:53:28 +0200 Subject: [PATCH 6/8] Changes powcache data dir This takes into account regtest and testnet3 sub directories --- src/powcache.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/powcache.cpp b/src/powcache.cpp index 8f22e20694..7a67da2ef4 100644 --- a/src/powcache.cpp +++ b/src/powcache.cpp @@ -22,7 +22,7 @@ bool CPowCacheDB::WriteCacheEntry(const uint256& lookupHash, const uint256& powH void CPowHashProxy::Init(size_t nCacheSize) { pow_cachedb.reset(); - pow_cachedb.reset(new CPowCacheDB(GetDataDir(false) / "powcache", nCacheSize, false, false)); + pow_cachedb.reset(new CPowCacheDB(GetDataDir() / "powcache", nCacheSize, false, false)); }; void CPowHashProxy::Stop() From 1613031623aed7536ad0302862760b5c6c864248 Mon Sep 17 00:00:00 2001 From: Anton Kovalov Date: Mon, 18 Jul 2022 19:13:37 +0200 Subject: [PATCH 7/8] Changes cache hash from 256 to 160 bits GetCacheHash() uses SHA1 for a unique key. Its output size is 20 bytes -> 160 bits, however initial implementation uses 256 bit key, thus 96 bits are simply 0, which uses up extra memory. --- src/powcache.cpp | 6 +++--- src/powcache.h | 4 ++-- src/primitives/block.cpp | 4 ++-- src/primitives/block.h | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/powcache.cpp b/src/powcache.cpp index 7a67da2ef4..149fdacefb 100644 --- a/src/powcache.cpp +++ b/src/powcache.cpp @@ -8,14 +8,14 @@ CPowCacheDB::CPowCacheDB(fs::path ldb_path, size_t nCacheSize, bool fMemory, boo db(ldb_path, nCacheSize, fMemory, fWipe, true) {} -Optional CPowCacheDB::GetCacheEntry(const uint256& lookupHash) const { +Optional CPowCacheDB::GetCacheEntry(const uint160& lookupHash) const { uint256 blockHash; if (db.Read(lookupHash, blockHash)) return blockHash; return nullopt; } -bool CPowCacheDB::WriteCacheEntry(const uint256& lookupHash, const uint256& powHash) { +bool CPowCacheDB::WriteCacheEntry(const uint160& lookupHash, const uint256& powHash) { return db.Write(lookupHash, powHash); } @@ -34,7 +34,7 @@ uint256 CPowHashProxy::GetHash(CBlockHeader& block) { if(pow_cachedb) { - uint256 cacheHash = block.GetCacheHash(); + uint160 cacheHash = block.GetCacheHash(); if (Optional optionalBlockHash = pow_cachedb->GetCacheEntry(cacheHash)) return optionalBlockHash.get(); diff --git a/src/powcache.h b/src/powcache.h index 81dbd78842..c58f0b3039 100644 --- a/src/powcache.h +++ b/src/powcache.h @@ -28,8 +28,8 @@ class CPowCacheDB explicit CPowCacheDB(fs::path ldb_path, size_t nCacheSize, bool fMemory, bool fWipe); // Read method already checks for record existence - Optional GetCacheEntry(const uint256& lookupHash) const; - bool WriteCacheEntry(const uint256& lookupHash, const uint256& powHash); + Optional GetCacheEntry(const uint160& lookupHash) const; + bool WriteCacheEntry(const uint160& lookupHash, const uint256& powHash); }; class CPowHashProxy diff --git a/src/primitives/block.cpp b/src/primitives/block.cpp index b1000ca43f..15373137d9 100644 --- a/src/primitives/block.cpp +++ b/src/primitives/block.cpp @@ -14,9 +14,9 @@ uint256 CBlockHeader::GetHash() const return GetPoWHash(); } -uint256 CBlockHeader::GetCacheHash() const +uint160 CBlockHeader::GetCacheHash() const { - uint256 hash; + uint160 hash; CSHA1().Write((const unsigned char*)this, 80).Finalize((unsigned char*)&hash); return hash; } diff --git a/src/primitives/block.h b/src/primitives/block.h index 7f47594b00..30909bd06f 100644 --- a/src/primitives/block.h +++ b/src/primitives/block.h @@ -62,7 +62,7 @@ class CBlockHeader uint256 GetHash() const; uint256 GetPoWHash() const; - uint256 GetCacheHash() const; + uint160 GetCacheHash() const; int64_t GetBlockTime() const { From 026748eb0a094bbca9d141cc2918b4c702a21ec2 Mon Sep 17 00:00:00 2001 From: Anton Kovalov Date: Mon, 18 Jul 2022 22:09:07 +0200 Subject: [PATCH 8/8] Adds docs information about powcache --- doc/files.md | 1 + doc/release-notes/release-notes-obtc-0.3.0.md | 26 +++++++++++++++++++ 2 files changed, 27 insertions(+) create mode 100644 doc/release-notes/release-notes-obtc-0.3.0.md diff --git a/doc/files.md b/doc/files.md index e7b459921f..61e62316c4 100644 --- a/doc/files.md +++ b/doc/files.md @@ -45,6 +45,7 @@ Subdirectory | File(s) | Description `blocks/` | `blkNNNNN.dat`[\[2\]](#note2) | Actual Bitcoin blocks (in network format, dumped in raw on disk, 128 MiB per file) `blocks/` | `revNNNNN.dat`[\[2\]](#note2) | Block undo data (custom format) `chainstate/` | LevelDB database | Blockchain state (a compact representation of all currently unspent transaction outputs and some metadata about the transactions they are from) +`powcache/` | LevelDB database | Heavy hash store. Keeps heavy hashes on disk, and uses a light SHA1 as a key for a lookup `indexes/txindex/` | LevelDB database | Transaction index; *optional*, used if `-txindex=1` `indexes/blockfilter/basic/db/` | LevelDB database | Blockfilter index LevelDB database for the basic filtertype; *optional*, used if `-blockfilterindex=basic` `indexes/blockfilter/basic/` | `fltrNNNNN.dat`[\[2\]](#note2) | Blockfilter index filters for the basic filtertype; *optional*, used if `-blockfilterindex=basic` diff --git a/doc/release-notes/release-notes-obtc-0.3.0.md b/doc/release-notes/release-notes-obtc-0.3.0.md new file mode 100644 index 0000000000..e08fbc22c4 --- /dev/null +++ b/doc/release-notes/release-notes-obtc-0.3.0.md @@ -0,0 +1,26 @@ +Version 0.3.0 +=============================== + +Summary +=============================== + +This version contains an enhancement related to long block indexation and wallet rescaning, which requires chain marching. +A new leveldb database is added, which is stored in `powcache` file. +Due to the fact that computing block's heavy hash is very expensive, it is stored in the `powcache` database. + +Key -> Value: SHA1 -> Heavy Hash + +Compatibility +=============================== + +It is fully compatile with the previous versions, and has no forking potential. + +Updated and new sources +=============================== + +- powcache.* +- block.* +- logging.* +- init.cpp +- txdb.cpp +- validation.cpp \ No newline at end of file