Heavy Hash cache

doc/files.md

@@ -45,6 +45,7 @@ Subdirectory       | File(s)               | Description
 `blocks/`          | `blkNNNNN.dat`<sup>[\[2\]](#note2)</sup> | Actual Bitcoin blocks (in network format, dumped in raw on disk, 128 MiB per file)
 `blocks/`          | `revNNNNN.dat`<sup>[\[2\]](#note2)</sup> | Block undo data (custom format)
 `chainstate/`      | LevelDB database      | Blockchain state (a compact representation of all currently unspent transaction outputs and some metadata about the transactions they are from)
+`powcache/`        | LevelDB database      | Heavy hash store. Keeps heavy hashes on disk, and uses a light SHA1 as a key for a lookup
 `indexes/txindex/` | LevelDB database      | Transaction index; *optional*, used if `-txindex=1`
 `indexes/blockfilter/basic/db/` | LevelDB database      | Blockfilter index LevelDB database for the basic filtertype; *optional*, used if `-blockfilterindex=basic`
 `indexes/blockfilter/basic/`    | `fltrNNNNN.dat`<sup>[\[2\]](#note2)</sup> | Blockfilter index filters for the basic filtertype; *optional*, used if `-blockfilterindex=basic`

doc/release-notes/release-notes-obtc-0.3.0.md

@@ -0,0 +1,26 @@
+Version 0.3.0
+===============================
+
+Summary
+===============================
+
+This version contains an enhancement related to long block indexation and wallet rescaning, which requires chain marching.
+A new leveldb database is added, which is stored in `powcache` file.
+Due to the fact that computing block's heavy hash is very expensive, it is stored in the `powcache` database.
+
+Key -> Value: SHA1 -> Heavy Hash
+
+Compatibility
+===============================
+
+It is fully compatile with the previous versions, and has no forking potential.
+
+Updated and new sources
+===============================
+
+- powcache.*
+- block.*
+- logging.*
+- init.cpp
+- txdb.cpp
+- validation.cpp

src/Makefile.am

@@ -173,6 +173,7 @@ BITCOIN_CORE_H = \
   policy/rbf.h \
   policy/settings.h \
   pow.h \
+  powcache.h \
   protocol.h \
   psbt.h \
   random.h \
@@ -300,6 +301,7 @@ libbitcoin_server_a_SOURCES = \
   policy/rbf.cpp \
   policy/settings.cpp \
   pow.cpp \
+  powcache.cpp \
   rest.cpp \
   rpc/blockchain.cpp \
   rpc/mining.cpp \

src/chain.h

@@ -9,6 +9,7 @@
 #include <arith_uint256.h>
 #include <consensus/params.h>
 #include <flatfile.h>
+#include <powcache.h>
 #include <primitives/block.h>
 #include <tinyformat.h>
 #include <uint256.h>
@@ -360,7 +361,7 @@ class CDiskBlockIndex : public CBlockIndex
         block.nTime           = nTime;
         block.nBits           = nBits;
         block.nNonce          = nNonce;
-        return block.GetHash();
+        return powHashProxy.GetHash(block);
     }
 
 

src/init.cpp

@@ -34,6 +34,7 @@
 #include <policy/fees.h>
 #include <policy/policy.h>
 #include <policy/settings.h>
+#include <powcache.h>
 #include <rpc/blockchain.h>
 #include <rpc/register.h>
 #include <rpc/server.h>
@@ -278,6 +279,7 @@ void Shutdown(NodeContext& node)
             g_chainstate->ResetCoinsViews();
         }
         pblocktree.reset();
+        powHashProxy.Stop();
     }
     for (const auto& client : node.chain_clients) {
         client->stop();
@@ -1508,6 +1510,8 @@ bool AppInitMain(NodeContext& node)
     int64_t nCoinDBCache = std::min(nTotalCache / 2, (nTotalCache / 4) + (1 << 23)); // use 25%-50% of the remainder for disk cache
     nCoinDBCache = std::min(nCoinDBCache, nMaxCoinsDBCache << 20); // cap total coins db cache
     nTotalCache -= nCoinDBCache;
+    int64_t nHeavyHashCache = std::min(nTotalCache / 32, nMaxHeavyHashCache << 20);
+    nTotalCache -= nHeavyHashCache;
     nCoinCacheUsage = nTotalCache; // the rest goes to in-memory cache
     int64_t nMempoolSizeMax = gArgs.GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000;
     LogPrintf("Cache configuration:\n");
@@ -1520,6 +1524,7 @@ bool AppInitMain(NodeContext& node)
                   filter_index_cache * (1.0 / 1024 / 1024), BlockFilterTypeName(filter_type));
     }
     LogPrintf("* Using %.1f MiB for chain state database\n", nCoinDBCache * (1.0 / 1024 / 1024));
+    LogPrintf("* Using %.1f MiB for heavy hash database\n", nHeavyHashCache * (1.0 / 1024 / 1024));
     LogPrintf("* Using %.1f MiB for in-memory UTXO set (plus up to %.1f MiB of unused mempool space)\n", nCoinCacheUsage * (1.0 / 1024 / 1024), nMempoolSizeMax * (1.0 / 1024 / 1024));
 
     bool fLoaded = false;
@@ -1538,8 +1543,11 @@ bool AppInitMain(NodeContext& node)
                 g_chainstate = MakeUnique<CChainState>();
                 UnloadBlockIndex();
 
+                // Initialize cache for PoW HeavyHash[es]
+                powHashProxy.Init(nHeavyHashCache);
+
                 // new CBlockTreeDB tries to delete the existing file, which
-                // fails if it's still open from the previous loop. Close it first:
+                // fails if it's still open from the previous loop. Close it first:                
                 pblocktree.reset();
                 pblocktree.reset(new CBlockTreeDB(nBlockTreeDBCache, false, fReset));
 

src/logging.cpp

@@ -163,6 +163,7 @@ const CLogCategoryDesc LogCategories[] =
     {BCLog::QT, "qt"},
     {BCLog::LEVELDB, "leveldb"},
     {BCLog::VALIDATION, "validation"},
+    {BCLog::POWCACHE, "powcache"},
     {BCLog::ALL, "1"},
     {BCLog::ALL, "all"},
 };

src/logging.h

@@ -55,6 +55,7 @@ namespace BCLog {
         QT          = (1 << 19),
         LEVELDB     = (1 << 20),
         VALIDATION  = (1 << 21),
+        POWCACHE    = (1 << 22),
         ALL         = ~(uint32_t)0,
     };
 

src/powcache.cpp

@@ -0,0 +1,50 @@
+// Copyright (c) 2022 barrystyle
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#include <powcache.h>
+
+CPowCacheDB::CPowCacheDB(fs::path ldb_path, size_t nCacheSize, bool fMemory, bool fWipe) :
+    db(ldb_path, nCacheSize, fMemory, fWipe, true)
+{}
+
+Optional<uint256> CPowCacheDB::GetCacheEntry(const uint160& lookupHash) const {
+    uint256 blockHash;
+    if (db.Read(lookupHash, blockHash)) return blockHash;
+
+    return nullopt;
+}
+
+bool CPowCacheDB::WriteCacheEntry(const uint160& lookupHash, const uint256& powHash) {
+    return db.Write(lookupHash, powHash);
+}
+
+void CPowHashProxy::Init(size_t nCacheSize)
+{
+    pow_cachedb.reset();
+    pow_cachedb.reset(new CPowCacheDB(GetDataDir() / "powcache", nCacheSize, false, false));
+};
+
+void CPowHashProxy::Stop()
+{
+    pow_cachedb.reset();
+};
+
+uint256 CPowHashProxy::GetHash(CBlockHeader& block)
+{
+    if(pow_cachedb)
+    {
+        uint160 cacheHash = block.GetCacheHash();
+
+        if (Optional<uint256> optionalBlockHash = pow_cachedb->GetCacheEntry(cacheHash))
+            return optionalBlockHash.get();
+
+        uint256 blockHash = block.GetHash();
+        pow_cachedb->WriteCacheEntry(cacheHash, blockHash);
+        return blockHash;
+    }
+
+    return block.GetHash();
+};
+
+CPowHashProxy powHashProxy;

src/powcache.h

@@ -0,0 +1,49 @@
+// Copyright (c) 2022 barrystyle
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#ifndef BITCOIN_POWCACHE_H
+#define BITCOIN_POWCACHE_H
+
+#include <dbwrapper.h>
+#include <optional.h>
+#include <primitives/block.h>
+#include <uint256.h>
+
+/** Implements a persistent hash lookup cache, using SHA1 hash as the key.
+ * If hash was previosuly encountered and stored it allows for instant retrieval of the more cpu-expensive pow HeavyHash.
+ * The hash is not tied to a height, preventing invalid hashes from potentially
+ * being returned in the instance of a block reorganisation etc.
+ */
+
+//! Max memory allocated for HeavyHash cache (4 MiB ~ 4.2 MB)
+static const int64_t nMaxHeavyHashCache = 4;
+
+class CPowCacheDB
+{
+protected:
+    CDBWrapper db;
+
+public:
+    explicit CPowCacheDB(fs::path ldb_path, size_t nCacheSize, bool fMemory, bool fWipe);
+
+    // Read method already checks for record existence
+    Optional<uint256> GetCacheEntry(const uint160& lookupHash) const;
+    bool WriteCacheEntry(const uint160& lookupHash, const uint256& powHash);
+};
+
+class CPowHashProxy
+{
+protected:
+    std::unique_ptr<CPowCacheDB> pow_cachedb;
+
+public:
+    void Init(size_t nCacheSize);
+    void Stop();
+    uint256 GetHash(CBlockHeader& block);
+};
+
+// Global wrapper around CPoWCacheDB object. It is responsible for its existence, and based on this it decides how to retrieve block's hash
+extern CPowHashProxy powHashProxy;
+
+#endif // BITCOIN_POWCACHE_H

src/primitives/block.cpp

@@ -1,8 +1,10 @@
 // Copyright (c) 2009-2010 Satoshi Nakamoto
 // Copyright (c) 2009-2019 The Bitcoin Core developers
+// Copyright (c) 2022 barrystyle
 // Distributed under the MIT software license, see the accompanying
 // file COPYING or http://www.opensource.org/licenses/mit-license.php.
 
+#include <crypto/sha1.h>
 #include <primitives/block.h>
 #include <hash.h>
 #include <tinyformat.h>
@@ -12,6 +14,13 @@ uint256 CBlockHeader::GetHash() const
     return GetPoWHash();
 }
 
+uint160 CBlockHeader::GetCacheHash() const
+{
+    uint160 hash;
+    CSHA1().Write((const unsigned char*)this, 80).Finalize((unsigned char*)&hash);
+    return hash;
+}
+
 uint256 CBlockHeader::GetPoWHash() const
 {
     uint256 seed;

src/primitives/block.h

@@ -62,6 +62,7 @@ class CBlockHeader
 
     uint256 GetHash() const;
     uint256 GetPoWHash() const;
+    uint160 GetCacheHash() const;
 
     int64_t GetBlockTime() const
     {

src/validation.cpp

@@ -1148,7 +1148,7 @@ bool ReadBlockFromDisk(CBlock& block, const FlatFilePos& pos, const Consensus::P
     }
 
     // Check the header
-    if (!CheckProofOfWork(block.GetPoWHash(), block.nBits, consensusParams))
+    if (!CheckProofOfWork(powHashProxy.GetHash(block), block.nBits, consensusParams))
         return error("ReadBlockFromDisk: Errors in block header at %s", pos.ToString());
 
     return true;
@@ -1164,7 +1164,7 @@ bool ReadBlockFromDisk(CBlock& block, const CBlockIndex* pindex, const Consensus
 
     if (!ReadBlockFromDisk(block, blockPos, consensusParams))
         return false;
-    if (block.GetHash() != pindex->GetBlockHash())
+    if (powHashProxy.GetHash(block) != pindex->GetBlockHash())
         return error("ReadBlockFromDisk(CBlock&, CBlockIndex*): GetHash() doesn't match index for %s at %s",
                 pindex->ToString(), pindex->GetBlockPos().ToString());
     return true;