mining: add getMemoryLoad() and track template non-mempool memory footprint

src/interfaces/mining.h

@@ -92,6 +92,12 @@ class BlockTemplate
     virtual void interruptWait() = 0;
 };
 
+//! Tracks memory usage for template-bound transactions.
+struct MemoryLoad
+{
+    uint64_t usage{0};
+};
+
 //! Interface giving clients (RPC, Stratum v2 Template Provider in the future)
 //! ability to create block templates.
 class Mining
@@ -147,6 +153,12 @@ class Mining
      */
     virtual bool checkBlock(const CBlock& block, const node::BlockCheckOptions& options, std::string& reason, std::string& debug) = 0;
 
+    /**
+     * Returns the current memory load for template transactions outside the
+     * mempool.
+     */
+    virtual MemoryLoad getMemoryLoad() = 0;
+
     //! Get internal node context. Useful for RPC and testing,
     //! but not accessible across processes.
     virtual node::NodeContext* context() { return nullptr; }

src/ipc/capnp/mining.capnp

@@ -19,6 +19,7 @@ interface Mining $Proxy.wrap("interfaces::Mining") {
     waitTipChanged @3 (context :Proxy.Context, currentTip: Data, timeout: Float64) -> (result: Common.BlockRef);
     createNewBlock @4 (options: BlockCreateOptions) -> (result: BlockTemplate);
     checkBlock @5 (block: Data, options: BlockCheckOptions) -> (reason: Text, debug: Text, result: Bool);
+    getMemoryLoad @6 (context :Proxy.Context) -> (result: MemoryLoad);
 }
 
 interface BlockTemplate $Proxy.wrap("interfaces::BlockTemplate") {
@@ -42,6 +43,10 @@ struct BlockCreateOptions $Proxy.wrap("node::BlockCreateOptions") {
     coinbaseOutputMaxAdditionalSigops @2 :UInt64 $Proxy.name("coinbase_output_max_additional_sigops");
 }
 
+struct MemoryLoad $Proxy.wrap("interfaces::MemoryLoad") {
+    usage @0 :UInt64;
+}
+
 struct BlockWaitOptions $Proxy.wrap("node::BlockWaitOptions") {
     timeout @0 : Float64 $Proxy.name("timeout");
     feeThreshold @1 : Int64 $Proxy.name("fee_threshold");

src/node/context.h

@@ -5,6 +5,8 @@
 #ifndef BITCOIN_NODE_CONTEXT_H
 #define BITCOIN_NODE_CONTEXT_H
 
+#include <node/types.h>
+
 #include <atomic>
 #include <cstdlib>
 #include <functional>
@@ -26,6 +28,7 @@ class ECC_Context;
 class NetGroupManager;
 class PeerManager;
 namespace interfaces {
+class BlockTemplate;
 class Chain;
 class ChainClient;
 class Mining;
@@ -66,6 +69,12 @@ struct NodeContext {
     std::unique_ptr<AddrMan> addrman;
     std::unique_ptr<CConnman> connman;
     std::unique_ptr<CTxMemPool> mempool;
+    //! Track how many templates (which we hold on to on behalf of connected IPC
+    //! clients) are referencing each transaction.
+    TxTemplateMap template_tx_refs;
+    //! Cache latest getblocktemplate result for BIP 22 long polling. Must be cleared
+    //! before template_tx_refs.
+    std::unique_ptr<interfaces::BlockTemplate> gbt_template;
     std::unique_ptr<const NetGroupManager> netgroupman;
     std::unique_ptr<CBlockPolicyEstimator> fee_estimator;
     std::unique_ptr<PeerManager> peerman;

src/node/interfaces.cpp

@@ -67,6 +67,7 @@
 #include <any>
 #include <memory>
 #include <optional>
+#include <ranges>
 #include <utility>
 
 #include <boost/signals2/signal.hpp>
@@ -78,6 +79,7 @@ using interfaces::Chain;
 using interfaces::FoundBlock;
 using interfaces::Handler;
 using interfaces::MakeSignalHandler;
+using interfaces::MemoryLoad;
 using interfaces::Mining;
 using interfaces::Node;
 using interfaces::WalletLoader;
@@ -862,7 +864,22 @@ class BlockTemplateImpl : public BlockTemplate
                                                     m_block_template(std::move(block_template)),
                                                     m_node(node)
     {
-        assert(m_block_template);
+        // Don't track the dummy coinbase, because it can be modified in-place
+        // by submitSolution()
+        for (const CTransactionRef& tx : Assert(m_block_template)->block.vtx | std::views::drop(1)) {
+            m_node.template_tx_refs[tx]++;
+        }
+    }
+
+    ~BlockTemplateImpl()
+    {
+        for (const CTransactionRef& tx : m_block_template->block.vtx | std::views::drop(1)) {
+            auto ref_count{m_node.template_tx_refs.find(tx)};
+            if (!Assume(ref_count != m_node.template_tx_refs.end())) break;
+            if (--ref_count->second == 0) {
+                m_node.template_tx_refs.erase(ref_count);
+            }
+        }
     }
 
     CBlockHeader getBlockHeader() override
@@ -977,6 +994,12 @@ class MinerImpl : public Mining
         return state.IsValid();
     }
 
+    MemoryLoad getMemoryLoad() override
+    {
+        CTxMemPool& mempool{*Assert(m_node.mempool)};
+        return {.usage = GetTemplateMemoryUsage(mempool, m_node.template_tx_refs)};
+    }
+
     NodeContext* context() override { return &m_node; }
     ChainstateManager& chainman() { return *Assert(m_node.chainman); }
     KernelNotifications& notifications() { return *Assert(m_node.notifications); }

src/node/miner.cpp

@@ -16,12 +16,15 @@
 #include <consensus/validation.h>
 #include <deploymentstatus.h>
 #include <logging.h>
+#include <logging/timer.h>
+#include <memusage.h>
 #include <node/context.h>
 #include <node/kernel_notifications.h>
 #include <policy/feerate.h>
 #include <policy/policy.h>
 #include <pow.h>
 #include <primitives/transaction.h>
+#include <util/check.h>
 #include <util/moneystr.h>
 #include <util/signalinterrupt.h>
 #include <util/time.h>
@@ -440,4 +443,25 @@ std::optional<BlockRef> WaitTipChanged(ChainstateManager& chainman, KernelNotifi
     // avoid deadlocks.
     return GetTip(chainman);
 }
+
+size_t GetTemplateMemoryUsage(const CTxMemPool& mempool, const TxTemplateMap& tx_refs)
+{
+    size_t usage_bytes{0};
+    {
+        LOG_TIME_MILLIS_WITH_CATEGORY("Calculate template transaction reference memory footprint", BCLog::BENCH);
+        for (const auto& [tx_ref, _] : tx_refs) {
+            if (!Assume(tx_ref)) continue;
+            // mempool.exists() locks mempool.cs each time, which slows down
+            // our calculation. This is preferable to potentially blocking
+            // the node from processing new transactions while this
+            // (non-urgent) calculation is in progress.
+            //
+            // As a side-effect the result is not an accurate snapshot, because
+            // the mempool may change during the loop.
+            if (mempool.exists(tx_ref->GetWitnessHash())) continue;
+            usage_bytes += RecursiveDynamicUsage(*tx_ref);
+        }
+    }
+    return usage_bytes;
+}
 } // namespace node

src/node/miner.h

@@ -14,6 +14,7 @@
 #include <util/feefrac.h>
 
 #include <cstdint>
+#include <map>
 #include <memory>
 #include <optional>
 
@@ -157,6 +158,15 @@ std::optional<BlockRef> GetTip(ChainstateManager& chainman);
 /* Waits for the connected tip to change until timeout has elapsed. During node initialization, this will wait until the tip is connected (regardless of `timeout`).
  * Returns the current tip, or nullopt if the node is shutting down. */
 std::optional<BlockRef> WaitTipChanged(ChainstateManager& chainman, KernelNotifications& kernel_notifications, const uint256& current_tip, MillisecondsDouble& timeout);
+
+/**
+ * Estimate non-mempool memory footprint of transaction data referenced by block
+ * templates.
+ *
+ * Result is not guaranteed to be an accurate snapshot, because it does not
+ * lock mempool.cs while iterating over transaction references.
+ */
+size_t GetTemplateMemoryUsage(const CTxMemPool& mempool, const TxTemplateMap& tx_refs);
 } // namespace node
 
 #endif // BITCOIN_NODE_MINER_H

src/node/types.h

@@ -16,7 +16,9 @@
 #include <consensus/amount.h>
 #include <cstddef>
 #include <cstdint>
+#include <map>
 #include <policy/policy.h>
+#include <primitives/transaction.h>
 #include <script/script.h>
 #include <uint256.h>
 #include <util/time.h>
@@ -110,6 +112,11 @@ enum class TxBroadcast : uint8_t {
     MEMPOOL_NO_BROADCAST,
 };
 
+/*
+ * Map how many templates refer to each transaction reference.
+ */
+using TxTemplateMap = std::map<CTransactionRef, size_t>;
+
 } // namespace node
 
 #endif // BITCOIN_NODE_TYPES_H

src/rpc/mining.cpp

@@ -858,7 +858,7 @@ static RPCHelpMan getblocktemplate()
     // Update block
     static CBlockIndex* pindexPrev;
     static int64_t time_start;
-    static std::unique_ptr<BlockTemplate> block_template;
+    std::unique_ptr<BlockTemplate>& block_template{node.gbt_template};
     if (!pindexPrev || pindexPrev->GetBlockHash() != tip ||
         (mempool.GetTransactionsUpdated() != nTransactionsUpdatedLast && GetTime() - time_start > 5))
     {