backport: bitcoin#23006, #23212, #23500, #23521, #23686, #23927, #24367, #26358

contrib/devtools/circular-dependencies.py

@@ -5,7 +5,7 @@
 
 import sys
 import re
-from multiprocess import Pool
+from multiprocess import Pool # type: ignore[import]
 from typing import Dict, List, Set
 
 MAPPING = {

contrib/devtools/optimize-pngs.py

@@ -10,7 +10,8 @@
 import sys
 import subprocess
 import hashlib
-from PIL import Image  # pip3 install Pillow
+# pip3 install Pillow
+from PIL import Image # type: ignore[import]
 
 def file_hash(filename):
     '''Return hash of raw file contents'''

doc/design/libraries.md

@@ -37,7 +37,7 @@ Note: libbitcoin_kernel is a subject to be backported & dashified.
 
 ```mermaid
 
-%%{ init : { "flowchart" : { "curve" : "linear" }}}%%
+%%{ init : { "flowchart" : { "curve" : "basis" }}}%%
 
 graph TD;
 
@@ -53,18 +53,19 @@ dash-qt[dash-qt]-->libbitcoin_wallet;
 dash-wallet[dash-wallet]-->libbitcoin_wallet;
 dash-wallet[dash-wallet]-->libbitcoin_wallet_tool;
 
-libbitcoin_cli-->libbitcoin_common;
 libbitcoin_cli-->libbitcoin_util;
+libbitcoin_cli-->libbitcoin_common;
 
-libbitcoin_common-->libbitcoin_util;
 libbitcoin_common-->libdash_consensus;
+libbitcoin_common-->libbitcoin_util;
 
 libbitcoin_kernel-->libdash_consensus;
 libbitcoin_kernel-->libbitcoin_util;
 
-libbitcoin_node-->libbitcoin_common;
 libbitcoin_node-->libdash_consensus;
+libbitcoin_node-->libbitcoin_common;
 libbitcoin_node-->libbitcoin_kernel;
+libbitcoin_node-->libbitcoin_common;
 libbitcoin_node-->libbitcoin_util;
 
 libbitcoinqt-->libbitcoin_common;
@@ -73,8 +74,8 @@ libbitcoinqt-->libbitcoin_util;
 libbitcoin_wallet-->libbitcoin_common;
 libbitcoin_wallet-->libbitcoin_util;
 
-libbitcoin_wallet_tool-->libbitcoin_util;
 libbitcoin_wallet_tool-->libbitcoin_wallet;
+libbitcoin_wallet_tool-->libbitcoin_util;
 
 classDef bold stroke-width:2px, font-weight:bold, font-size: smaller;
 class dash-qt,dashd,dash-cli,dash-wallet bold

src/Makefile.am

@@ -1090,6 +1090,7 @@ dash_tx_LDADD += $(BACKTRACE_LIBS) $(GMP_LIBS)
 
 # dash-wallet binary #
 dash_wallet_SOURCES = bitcoin-wallet.cpp
+dash_wallet_SOURCES += init/bitcoin-wallet.cpp
 dash_wallet_CPPFLAGS = $(bitcoin_bin_cppflags)
 dash_wallet_CXXFLAGS = $(bitcoin_bin_cxxflags)
 dash_wallet_LDFLAGS = $(bitcoin_bin_ldflags)

src/Makefile.qt.include

@@ -424,15 +424,15 @@ bitcoin_qt_libtoolflags = $(AM_LIBTOOLFLAGS) --tag CXX
 
 qt_dash_qt_CPPFLAGS = $(bitcoin_qt_cppflags)
 qt_dash_qt_CXXFLAGS = $(bitcoin_qt_cxxflags)
-qt_dash_qt_SOURCES = $(bitcoin_qt_sources) init/bitcoind.cpp
+qt_dash_qt_SOURCES = $(bitcoin_qt_sources) init/bitcoin-qt.cpp
 qt_dash_qt_LDADD = $(bitcoin_qt_ldadd)
 qt_dash_qt_LDFLAGS = $(bitcoin_qt_ldflags)
 qt_dash_qt_LIBTOOLFLAGS = $(bitcoin_qt_libtoolflags)
 
 dash_gui_CPPFLAGS = $(bitcoin_qt_cppflags)
 dash_gui_CXXFLAGS = $(bitcoin_qt_cxxflags)
-dash_gui_SOURCES = $(bitcoin_qt_sources) init/bitcoind.cpp
-dash_gui_LDADD = $(bitcoin_qt_ldadd)
+dash_gui_SOURCES = $(bitcoin_qt_sources) init/bitcoin-gui.cpp
+dash_gui_LDADD = $(bitcoin_qt_ldadd) $(LIBBITCOIN_IPC) $(LIBBITCOIN_UTIL) $(LIBMULTIPROCESS_LIBS)
 dash_gui_LDFLAGS = $(bitcoin_qt_ldflags)
 dash_gui_LIBTOOLFLAGS = $(bitcoin_qt_libtoolflags)
 

src/Makefile.qttest.include

@@ -33,7 +33,7 @@ qt_test_test_dash_qt_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(BITCOIN_QT_
   $(QT_INCLUDES) $(QT_TEST_INCLUDES) $(BOOST_CPPFLAGS)
 
 qt_test_test_dash_qt_SOURCES = \
-  init/bitcoind.cpp \
+  init/bitcoin-qt.cpp \
   qt/test/apptests.cpp \
   qt/test/optiontests.cpp \
   qt/test/rpcnestedtests.cpp \

src/bitcoin-wallet.cpp

@@ -9,6 +9,7 @@
 #include <chainparams.h>
 #include <chainparamsbase.h>
 #include <compat/compat.h>
+#include <interfaces/init.h>
 #include <logging.h>
 #include <util/strencodings.h>
 #include <clientversion.h>
@@ -100,6 +101,13 @@ MAIN_FUNCTION
     util::WinCmdLineArgs winArgs;
     std::tie(argc, argv) = winArgs.get();
 #endif
+
+    int exit_status;
+    std::unique_ptr<interfaces::Init> init = interfaces::MakeWalletInit(argc, argv, exit_status);
+    if (!init) {
+        return exit_status;
+    }
+
     SetupEnvironment();
     RandomInit();
     try {

src/init/bitcoin-gui.cpp

@@ -0,0 +1,52 @@
+// Copyright (c) 2021 The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#include <interfaces/chain.h>
+#include <interfaces/coinjoin.h>
+#include <interfaces/echo.h>
+#include <interfaces/init.h>
+#include <interfaces/ipc.h>
+#include <interfaces/node.h>
+#include <interfaces/wallet.h>
+#include <node/context.h>
+#include <util/system.h>
+
+#include <memory>
+
+namespace init {
+namespace {
+const char* EXE_NAME = "dash-gui";
+
+class BitcoinGuiInit : public interfaces::Init
+{
+public:
+    BitcoinGuiInit(const char* arg0) : m_ipc(interfaces::MakeIpc(EXE_NAME, arg0, *this))
+    {
+        m_node.args = &gArgs;
+        m_node.init = this;
+    }
+    std::unique_ptr<interfaces::Node> makeNode() override { return interfaces::MakeNode(m_node); }
+    std::unique_ptr<interfaces::Chain> makeChain() override { return interfaces::MakeChain(m_node); }
+    std::unique_ptr<interfaces::CoinJoin::Loader> makeCoinJoinLoader() override
+    {
+        return interfaces::MakeCoinJoinLoader(m_node);
+    }
+    std::unique_ptr<interfaces::WalletLoader> makeWalletLoader(interfaces::Chain& chain, interfaces::CoinJoin::Loader& coinjoin_loader) override
+    {
+        return MakeWalletLoader(chain, *Assert(m_node.args), m_node, coinjoin_loader);
+    }
+    std::unique_ptr<interfaces::Echo> makeEcho() override { return interfaces::MakeEcho(); }
+    interfaces::Ipc* ipc() override { return m_ipc.get(); }
+    node::NodeContext m_node;
+    std::unique_ptr<interfaces::Ipc> m_ipc;
+};
+} // namespace
+} // namespace init
+
+namespace interfaces {
+std::unique_ptr<Init> MakeGuiInit(int argc, char* argv[])
+{
+    return std::make_unique<init::BitcoinGuiInit>(argc > 0 ? argv[0] : "");
+}
+} // namespace interfaces

src/init/bitcoin-qt.cpp

@@ -0,0 +1,47 @@
+// Copyright (c) 2021 The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#include <interfaces/chain.h>
+#include <interfaces/coinjoin.h>
+#include <interfaces/echo.h>
+#include <interfaces/init.h>
+#include <interfaces/node.h>
+#include <interfaces/wallet.h>
+#include <node/context.h>
+#include <util/system.h>
+
+#include <memory>
+
+namespace init {
+namespace {
+class BitcoinQtInit : public interfaces::Init
+{
+public:
+    BitcoinQtInit()
+    {
+        m_node.args = &gArgs;
+        m_node.init = this;
+    }
+    std::unique_ptr<interfaces::Node> makeNode() override { return interfaces::MakeNode(m_node); }
+    std::unique_ptr<interfaces::Chain> makeChain() override { return interfaces::MakeChain(m_node); }
+    std::unique_ptr<interfaces::CoinJoin::Loader> makeCoinJoinLoader() override
+    {
+        return interfaces::MakeCoinJoinLoader(m_node);
+    }
+    std::unique_ptr<interfaces::WalletLoader> makeWalletLoader(interfaces::Chain& chain, interfaces::CoinJoin::Loader& coinjoin_loader) override
+    {
+        return MakeWalletLoader(chain, *Assert(m_node.args), m_node, coinjoin_loader);
+    }
+    std::unique_ptr<interfaces::Echo> makeEcho() override { return interfaces::MakeEcho(); }
+    node::NodeContext m_node;
+};
+} // namespace
+} // namespace init
+
+namespace interfaces {
+std::unique_ptr<Init> MakeGuiInit(int argc, char* argv[])
+{
+    return std::make_unique<init::BitcoinQtInit>();
+}
+} // namespace interfaces

src/init/bitcoin-wallet.cpp

@@ -0,0 +1,12 @@
+// Copyright (c) 2021 The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#include <interfaces/init.h>
+
+namespace interfaces {
+std::unique_ptr<Init> MakeWalletInit(int argc, char* argv[], int& exit_status)
+{
+    return std::make_unique<Init>();
+}
+} // namespace interfaces

src/qt/bitcoin.cpp

@@ -16,7 +16,6 @@
 #include <interfaces/init.h>
 #include <interfaces/node.h>
 #include <net.h>
-#include <node/context.h>
 #include <node/interface_ui.h>
 #include <noui.h>
 #include <qt/bitcoingui.h>
@@ -81,8 +80,6 @@ Q_DECLARE_METATYPE(CAmount)
 Q_DECLARE_METATYPE(SynchronizationState)
 Q_DECLARE_METATYPE(uint256)
 
-using node::NodeContext;
-
 static void RegisterMetaTypes()
 {
     // Register meta types used for QMetaObject::invokeMethod and Qt::QueuedConnection
@@ -511,9 +508,7 @@ int GuiMain(int argc, char* argv[])
     std::tie(argc, argv) = winArgs.get();
 #endif
 
-    NodeContext node_context;
-    int unused_exit_status;
-    std::unique_ptr<interfaces::Init> init = interfaces::MakeNodeInit(node_context, argc, argv, unused_exit_status);
+    std::unique_ptr<interfaces::Init> init = interfaces::MakeGuiInit(argc, argv);
 
     SetupEnvironment();
     util::ThreadSetInternalName("main");

src/qt/test/test_main.cpp

@@ -45,8 +45,6 @@ Q_IMPORT_PLUGIN(QAndroidPlatformIntegrationPlugin)
 #endif
 #endif
 
-using node::NodeContext;
-
 const std::function<void(const std::string&)> G_TEST_LOG_FUN{};
 
 const std::function<std::vector<const char*>()> G_TEST_COMMAND_LINE_ARGUMENTS{};
@@ -64,9 +62,7 @@ int main(int argc, char* argv[])
         BasicTestingSetup dummy{CBaseChainParams::REGTEST};
     }
 
-    NodeContext node_context;
-    int unused_exit_status;
-    std::unique_ptr<interfaces::Init> init = interfaces::MakeNodeInit(node_context, argc, argv, unused_exit_status);
+    std::unique_ptr<interfaces::Init> init = interfaces::MakeGuiInit(argc, argv);
     gArgs.ForceSetArg("-listen", "0");
     gArgs.ForceSetArg("-listenonion", "0");
     gArgs.ForceSetArg("-discover", "0");

src/rpc/blockchain.cpp

@@ -517,6 +517,12 @@ static RPCHelpMan getblockfrompeer()
         throw JSONRPCError(RPC_MISC_ERROR, "Block header missing");
     }
 
+    // Fetching blocks before the node has syncing past their height can prevent block files from
+    // being pruned, so we avoid it if the node is in prune mode.
+    if (index->nHeight > chainman.ActiveChain().Tip()->nHeight && node::fPruneMode) {
+        throw JSONRPCError(RPC_MISC_ERROR, "In prune mode, only blocks that the node has already synced previously can be fetched from a peer");
+    }
+
     const bool block_has_data = WITH_LOCK(::cs_main, return index->nStatus & BLOCK_HAVE_DATA);
     if (block_has_data) {
         throw JSONRPCError(RPC_MISC_ERROR, "Block already downloaded");

src/wallet/test/fuzz/notifications.cpp

@@ -95,7 +95,7 @@ FUZZ_TARGET(wallet_notifications, .init = initialize_setup)
     using Coins = std::set<std::tuple<CAmount, COutPoint>>;
     std::vector<std::tuple<Coins, CBlock>> chain;
     {
-        // Add the inital entry
+        // Add the initial entry
         chain.emplace_back();
         auto& [coins, block]{chain.back()};
         coins.emplace(total_amount, COutPoint{uint256::ONE, 1});

test/functional/feature_assumevalid.py

@@ -126,10 +126,8 @@ def run_test(self):
         tx.vout.append(CTxOut(49 * 100000000, CScript([OP_TRUE])))
         tx.calc_sha256()
 
-        block102 = create_block(self.tip, create_coinbase(height), self.block_time)
+        block102 = create_block(self.tip, create_coinbase(height), self.block_time, txlist=[tx])
         self.block_time += 1
-        block102.vtx.extend([tx])
-        block102.hashMerkleRoot = block102.calc_merkle_root()
         block102.solve()
         self.blocks.append(block102)
         self.tip = block102.sha256
@@ -139,7 +137,6 @@ def run_test(self):
         # Bury the assumed valid block 8400 deep (Dash needs 4x as much blocks to allow -assumevalid to work)
         for _ in range(8400):
             block = create_block(self.tip, create_coinbase(height), self.block_time)
-            block.nVersion = 4
             block.solve()
             self.blocks.append(block)
             self.tip = block.sha256

test/functional/feature_bip68_sequence.py

@@ -385,8 +385,7 @@ def test_bip68_not_consensus(self):
         assert_raises_rpc_error(-26, NOT_FINAL_ERROR, self.wallet.sendrawtransaction, from_node=self.nodes[0], tx_hex=tx3.serialize().hex())
 
         # make a block that violates bip68; ensure that the tip updates
-        block = create_block(tmpl=self.nodes[0].getblocktemplate(NORMAL_GBT_REQUEST_PARAMS))
-        block.vtx.extend([tx1, tx2, tx3])
+        block = create_block(tmpl=self.nodes[0].getblocktemplate(NORMAL_GBT_REQUEST_PARAMS), txlist=[tx1, tx2, tx3])
         block.hashMerkleRoot = block.calc_merkle_root()
         block.solve()
 

test/functional/feature_block.py

@@ -1380,11 +1380,10 @@ def next_block(self, number, spend=None, additional_coinbase_value=0, script=CSc
         else:
             coinbase.vout[0].nValue += spend.vout[0].nValue - 1  # all but one satoshi to fees
             coinbase.rehash()
-            block = create_block(base_block_hash, coinbase, block_time, version=version)
             tx = self.create_tx(spend, 0, 1, script)  # spend 1 satoshi
             self.sign_tx(tx, spend)
-            self.add_transactions_to_block(block, [tx])
-            block.hashMerkleRoot = block.calc_merkle_root()
+            tx.rehash()
+            block = create_block(base_block_hash, coinbase, block_time, version=version, txlist=[tx])
         # Block is created. Find a valid nonce.
         block.solve()
         self.tip = block

test/functional/feature_cltv.py

@@ -128,10 +128,7 @@ def run_test(self):
 
         tip = self.nodes[0].getbestblockhash()
         block_time = self.nodes[0].getblockheader(tip)['mediantime'] + 1
-        block = create_block(int(tip, 16), create_coinbase(CLTV_HEIGHT - 1), block_time)
-        block.nVersion = 3
-        block.vtx.extend(invalid_cltv_txs)
-        block.hashMerkleRoot = block.calc_merkle_root()
+        block = create_block(int(tip, 16), create_coinbase(CLTV_HEIGHT - 1), block_time, version=3, txlist=invalid_cltv_txs)
         block.solve()
 
         self.test_cltv_info(is_active=False)  # Not active as of current tip and next block does not need to obey rules
@@ -142,8 +139,7 @@ def run_test(self):
         self.log.info("Test that blocks must now be at least version 4")
         tip = block.sha256
         block_time += 1
-        block = create_block(tip, create_coinbase(CLTV_HEIGHT), block_time)
-        block.nVersion = 3
+        block = create_block(tip, create_coinbase(CLTV_HEIGHT), block_time, version=3)
         block.solve()
 
         with self.nodes[0].assert_debug_log(expected_msgs=[f'{block.hash}, bad-version(0x00000003)']):

test/functional/feature_csv_activation.py

@@ -182,10 +182,7 @@ def generate_blocks(self, number):
         return test_blocks
 
     def create_test_block(self, txs):
-        block = create_block(self.tip, create_coinbase(self.tipheight + 1), self.last_block_time + 600)
-        block.nVersion = 4
-        block.vtx.extend(txs)
-        block.hashMerkleRoot = block.calc_merkle_root()
+        block = create_block(self.tip, create_coinbase(self.tipheight + 1), self.last_block_time + 600, txlist=txs)
         block.solve()
         return block