diff --git a/doc/files.md b/doc/files.md index e7b459921f..61e62316c4 100644 --- a/doc/files.md +++ b/doc/files.md @@ -45,6 +45,7 @@ Subdirectory | File(s) | Description `blocks/` | `blkNNNNN.dat`[\[2\]](#note2) | Actual Bitcoin blocks (in network format, dumped in raw on disk, 128 MiB per file) `blocks/` | `revNNNNN.dat`[\[2\]](#note2) | Block undo data (custom format) `chainstate/` | LevelDB database | Blockchain state (a compact representation of all currently unspent transaction outputs and some metadata about the transactions they are from) +`powcache/` | LevelDB database | Heavy hash store. Keeps heavy hashes on disk, and uses a light SHA1 as a key for a lookup `indexes/txindex/` | LevelDB database | Transaction index; *optional*, used if `-txindex=1` `indexes/blockfilter/basic/db/` | LevelDB database | Blockfilter index LevelDB database for the basic filtertype; *optional*, used if `-blockfilterindex=basic` `indexes/blockfilter/basic/` | `fltrNNNNN.dat`[\[2\]](#note2) | Blockfilter index filters for the basic filtertype; *optional*, used if `-blockfilterindex=basic` diff --git a/doc/release-notes/release-notes-obtc-0.3.0.md b/doc/release-notes/release-notes-obtc-0.3.0.md new file mode 100644 index 0000000000..e08fbc22c4 --- /dev/null +++ b/doc/release-notes/release-notes-obtc-0.3.0.md @@ -0,0 +1,26 @@ +Version 0.3.0 +=============================== + +Summary +=============================== + +This version contains an enhancement related to long block indexation and wallet rescaning, which requires chain marching. +A new leveldb database is added, which is stored in `powcache` file. +Due to the fact that computing block's heavy hash is very expensive, it is stored in the `powcache` database. + +Key -> Value: SHA1 -> Heavy Hash + +Compatibility +=============================== + +It is fully compatile with the previous versions, and has no forking potential. + +Updated and new sources +=============================== + +- powcache.* +- block.* +- logging.* +- init.cpp +- txdb.cpp +- validation.cpp \ No newline at end of file diff --git a/src/Makefile.am b/src/Makefile.am index 328b14a62f..1fa6290ffb 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -173,6 +173,7 @@ BITCOIN_CORE_H = \ policy/rbf.h \ policy/settings.h \ pow.h \ + powcache.h \ protocol.h \ psbt.h \ random.h \ @@ -300,6 +301,7 @@ libbitcoin_server_a_SOURCES = \ policy/rbf.cpp \ policy/settings.cpp \ pow.cpp \ + powcache.cpp \ rest.cpp \ rpc/blockchain.cpp \ rpc/mining.cpp \ diff --git a/src/chain.h b/src/chain.h index 64c016a1d6..f44aa26a8b 100644 --- a/src/chain.h +++ b/src/chain.h @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -360,7 +361,7 @@ class CDiskBlockIndex : public CBlockIndex block.nTime = nTime; block.nBits = nBits; block.nNonce = nNonce; - return block.GetHash(); + return powHashProxy.GetHash(block); } diff --git a/src/init.cpp b/src/init.cpp index 48a2838c1b..36c50d4519 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -34,6 +34,7 @@ #include #include #include +#include #include #include #include @@ -278,6 +279,7 @@ void Shutdown(NodeContext& node) g_chainstate->ResetCoinsViews(); } pblocktree.reset(); + powHashProxy.Stop(); } for (const auto& client : node.chain_clients) { client->stop(); @@ -1508,6 +1510,8 @@ bool AppInitMain(NodeContext& node) int64_t nCoinDBCache = std::min(nTotalCache / 2, (nTotalCache / 4) + (1 << 23)); // use 25%-50% of the remainder for disk cache nCoinDBCache = std::min(nCoinDBCache, nMaxCoinsDBCache << 20); // cap total coins db cache nTotalCache -= nCoinDBCache; + int64_t nHeavyHashCache = std::min(nTotalCache / 32, nMaxHeavyHashCache << 20); + nTotalCache -= nHeavyHashCache; nCoinCacheUsage = nTotalCache; // the rest goes to in-memory cache int64_t nMempoolSizeMax = gArgs.GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000; LogPrintf("Cache configuration:\n"); @@ -1520,6 +1524,7 @@ bool AppInitMain(NodeContext& node) filter_index_cache * (1.0 / 1024 / 1024), BlockFilterTypeName(filter_type)); } LogPrintf("* Using %.1f MiB for chain state database\n", nCoinDBCache * (1.0 / 1024 / 1024)); + LogPrintf("* Using %.1f MiB for heavy hash database\n", nHeavyHashCache * (1.0 / 1024 / 1024)); LogPrintf("* Using %.1f MiB for in-memory UTXO set (plus up to %.1f MiB of unused mempool space)\n", nCoinCacheUsage * (1.0 / 1024 / 1024), nMempoolSizeMax * (1.0 / 1024 / 1024)); bool fLoaded = false; @@ -1538,8 +1543,11 @@ bool AppInitMain(NodeContext& node) g_chainstate = MakeUnique(); UnloadBlockIndex(); + // Initialize cache for PoW HeavyHash[es] + powHashProxy.Init(nHeavyHashCache); + // new CBlockTreeDB tries to delete the existing file, which - // fails if it's still open from the previous loop. Close it first: + // fails if it's still open from the previous loop. Close it first: pblocktree.reset(); pblocktree.reset(new CBlockTreeDB(nBlockTreeDBCache, false, fReset)); diff --git a/src/logging.cpp b/src/logging.cpp index 6fd916b603..ffb62b5936 100644 --- a/src/logging.cpp +++ b/src/logging.cpp @@ -163,6 +163,7 @@ const CLogCategoryDesc LogCategories[] = {BCLog::QT, "qt"}, {BCLog::LEVELDB, "leveldb"}, {BCLog::VALIDATION, "validation"}, + {BCLog::POWCACHE, "powcache"}, {BCLog::ALL, "1"}, {BCLog::ALL, "all"}, }; diff --git a/src/logging.h b/src/logging.h index b2fde1b9ea..07ff235024 100644 --- a/src/logging.h +++ b/src/logging.h @@ -55,6 +55,7 @@ namespace BCLog { QT = (1 << 19), LEVELDB = (1 << 20), VALIDATION = (1 << 21), + POWCACHE = (1 << 22), ALL = ~(uint32_t)0, }; diff --git a/src/powcache.cpp b/src/powcache.cpp new file mode 100644 index 0000000000..149fdacefb --- /dev/null +++ b/src/powcache.cpp @@ -0,0 +1,50 @@ +// Copyright (c) 2022 barrystyle +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include + +CPowCacheDB::CPowCacheDB(fs::path ldb_path, size_t nCacheSize, bool fMemory, bool fWipe) : + db(ldb_path, nCacheSize, fMemory, fWipe, true) +{} + +Optional CPowCacheDB::GetCacheEntry(const uint160& lookupHash) const { + uint256 blockHash; + if (db.Read(lookupHash, blockHash)) return blockHash; + + return nullopt; +} + +bool CPowCacheDB::WriteCacheEntry(const uint160& lookupHash, const uint256& powHash) { + return db.Write(lookupHash, powHash); +} + +void CPowHashProxy::Init(size_t nCacheSize) +{ + pow_cachedb.reset(); + pow_cachedb.reset(new CPowCacheDB(GetDataDir() / "powcache", nCacheSize, false, false)); +}; + +void CPowHashProxy::Stop() +{ + pow_cachedb.reset(); +}; + +uint256 CPowHashProxy::GetHash(CBlockHeader& block) +{ + if(pow_cachedb) + { + uint160 cacheHash = block.GetCacheHash(); + + if (Optional optionalBlockHash = pow_cachedb->GetCacheEntry(cacheHash)) + return optionalBlockHash.get(); + + uint256 blockHash = block.GetHash(); + pow_cachedb->WriteCacheEntry(cacheHash, blockHash); + return blockHash; + } + + return block.GetHash(); +}; + +CPowHashProxy powHashProxy; diff --git a/src/powcache.h b/src/powcache.h new file mode 100644 index 0000000000..c58f0b3039 --- /dev/null +++ b/src/powcache.h @@ -0,0 +1,49 @@ +// Copyright (c) 2022 barrystyle +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_POWCACHE_H +#define BITCOIN_POWCACHE_H + +#include +#include +#include +#include + +/** Implements a persistent hash lookup cache, using SHA1 hash as the key. + * If hash was previosuly encountered and stored it allows for instant retrieval of the more cpu-expensive pow HeavyHash. + * The hash is not tied to a height, preventing invalid hashes from potentially + * being returned in the instance of a block reorganisation etc. + */ + +//! Max memory allocated for HeavyHash cache (4 MiB ~ 4.2 MB) +static const int64_t nMaxHeavyHashCache = 4; + +class CPowCacheDB +{ +protected: + CDBWrapper db; + +public: + explicit CPowCacheDB(fs::path ldb_path, size_t nCacheSize, bool fMemory, bool fWipe); + + // Read method already checks for record existence + Optional GetCacheEntry(const uint160& lookupHash) const; + bool WriteCacheEntry(const uint160& lookupHash, const uint256& powHash); +}; + +class CPowHashProxy +{ +protected: + std::unique_ptr pow_cachedb; + +public: + void Init(size_t nCacheSize); + void Stop(); + uint256 GetHash(CBlockHeader& block); +}; + +// Global wrapper around CPoWCacheDB object. It is responsible for its existence, and based on this it decides how to retrieve block's hash +extern CPowHashProxy powHashProxy; + +#endif // BITCOIN_POWCACHE_H diff --git a/src/primitives/block.cpp b/src/primitives/block.cpp index d673a3d6bf..15373137d9 100644 --- a/src/primitives/block.cpp +++ b/src/primitives/block.cpp @@ -1,8 +1,10 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto // Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2022 barrystyle // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include #include #include #include @@ -12,6 +14,13 @@ uint256 CBlockHeader::GetHash() const return GetPoWHash(); } +uint160 CBlockHeader::GetCacheHash() const +{ + uint160 hash; + CSHA1().Write((const unsigned char*)this, 80).Finalize((unsigned char*)&hash); + return hash; +} + uint256 CBlockHeader::GetPoWHash() const { uint256 seed; diff --git a/src/primitives/block.h b/src/primitives/block.h index cecd755da8..30909bd06f 100644 --- a/src/primitives/block.h +++ b/src/primitives/block.h @@ -62,6 +62,7 @@ class CBlockHeader uint256 GetHash() const; uint256 GetPoWHash() const; + uint160 GetCacheHash() const; int64_t GetBlockTime() const { diff --git a/src/validation.cpp b/src/validation.cpp index c4dc6fd338..71f4f9456b 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -1148,7 +1148,7 @@ bool ReadBlockFromDisk(CBlock& block, const FlatFilePos& pos, const Consensus::P } // Check the header - if (!CheckProofOfWork(block.GetPoWHash(), block.nBits, consensusParams)) + if (!CheckProofOfWork(powHashProxy.GetHash(block), block.nBits, consensusParams)) return error("ReadBlockFromDisk: Errors in block header at %s", pos.ToString()); return true; @@ -1164,7 +1164,7 @@ bool ReadBlockFromDisk(CBlock& block, const CBlockIndex* pindex, const Consensus if (!ReadBlockFromDisk(block, blockPos, consensusParams)) return false; - if (block.GetHash() != pindex->GetBlockHash()) + if (powHashProxy.GetHash(block) != pindex->GetBlockHash()) return error("ReadBlockFromDisk(CBlock&, CBlockIndex*): GetHash() doesn't match index for %s at %s", pindex->ToString(), pindex->GetBlockPos().ToString()); return true;