Merge #79: Vendor 0.21.2

12d7ec3 Fix Bitcoin Core 0.21.2 upgrade (Tobin C. Harding)
a4f30ef Vendor Bitcoin Core v0.21.2 (Tobin C. Harding)
8b3d23f Update README about vendoring (Tobin C. Harding)
3ca86b9 Remove the depend directory (Tobin C. Harding)
619611e Add Bitcoin Core vendor script (Tobin C. Harding)

Pull request description:

  This is #77 without the version bump patch.

  Upgrade to a new way of vendoring Core and vendor version 0.21.2 - note please this will be directly followed by an upgrade to 0.21-final. Done separately to proof out the workflow of such an upgrade.

ACKs for top commit:
  apoelstra:
    ACK 12d7ec3

Tree-SHA512: f708b4bda1a209afe7552ec0e691ed104847d4a4a5fae6cd13f2da8f3a205c0a652ced8883900b9ce312cb8f4f39f77f0abbdf3372016c5af7aeddc3e33984ba

Author: apoelstra

README.md

@@ -26,18 +26,10 @@ For example, if we upgrade the Bitcoin Core code by a `Patch` version we also bu
 One side effect of this is that `crates.io` shows our release versions in yellow as if they were pre-release versions, this is due to us using a `-` which, in semantic versioning, implies a pre-release version.
 
 
-## Bitcoin Core subtree
+## Vendor Bitcoin Core
 
-We use a git subtree to vendor the Bitcoin Core code. This can be seen from the following commits that were created using `git subtree add --prefix='depend/bitcoin' git@github.com:bitcoin/bitcoin.git v0.19.2 --squash`.
-```
-f751613e62 Squashed 'depend/bitcoin/' content from commit 204cc0f575
-264282188a Merge commit 'f751613e6203770fa94143b9aba1d116512f0ce7' as 'depend/bitcoin'
-```
-
-To use a later version of Bitcoin Core, for example, v0.20.2
-```
-git subtree pull --prefix='depend/bitcoin' git@github.com:bitcoin/bitcoin.git v0.20.2 --squash
-```
+We use a script to vendor the Bitcoin Core code, the script takes the
+Bitcoin Core version number to vendor: `./vendor-bitcoin-core.sh 0.21.1`
 
 
 ## MSRV

build.rs

@@ -21,16 +21,22 @@ fn main() {
     let is_big_endian = env::var("CARGO_CFG_TARGET_ENDIAN").expect("No endian is set") == "big";
     let mut base_config = cc::Build::new();
     base_config
-        .cpp(true)
-        .include("depend/bitcoin/src")
         .include("depend/bitcoin/src/secp256k1/include")
         .define("__STDC_FORMAT_MACROS", None)
         .flag_if_supported("-Wno-implicit-fallthrough");
 
+    if target.contains("windows") {
+        base_config.define("WIN32", "1");
+    }
+
+    let mut secp_config = base_config.clone();
+    let mut consensus_config = base_config;
+
     // **Secp256k1**
     if !cfg!(feature = "external-secp") {
-        base_config
+        secp_config
             .include("depend/bitcoin/src/secp256k1")
+            .include("depend/bitcoin/src/secp256k1/src")
             .flag_if_supported("-Wno-unused-function") // some ecmult stuff is defined but not used upstream
             .define("SECP256K1_BUILD", "1")
             // Bitcoin core defines libsecp to *not* use libgmp.
@@ -39,34 +45,40 @@ fn main() {
             .define("USE_SCALAR_INV_BUILTIN", "1")
             // Technically libconsensus doesn't require the recovery feautre, but `pubkey.cpp` does.
             .define("ENABLE_MODULE_RECOVERY", "1")
+            .define("ECMULT_WINDOW_SIZE", "15")
+            .define("ECMULT_GEN_PREC_BITS", "4")
+            .define("ENABLE_MODULE_SCHNORRSIG", "1")
+            .define("ENABLE_MODULE_EXTRAKEYS", "1")
             // The actual libsecp256k1 C code.
             .file("depend/bitcoin/src/secp256k1/src/secp256k1.c");
 
         if is_big_endian {
-            base_config.define("WORDS_BIGENDIAN", "1");
+            secp_config.define("WORDS_BIGENDIAN", "1");
         }
 
         if use_64bit_compilation {
-            base_config
+            secp_config
                 .define("USE_FIELD_5X52", "1")
                 .define("USE_SCALAR_4X64", "1")
                 .define("HAVE___INT128", "1");
         } else {
-            base_config.define("USE_FIELD_10X26", "1").define("USE_SCALAR_8X32", "1");
+            secp_config.define("USE_FIELD_10X26", "1").define("USE_SCALAR_8X32", "1");
         }
+
+        secp_config.compile("libsecp256k1.a");
     }
 
-    let tool = base_config.get_compiler();
+    let tool = consensus_config.get_compiler();
     if tool.is_like_msvc() {
-        base_config.flag("/std:c++14").flag("/wd4100");
+        consensus_config.flag("/std:c++17").flag("/wd4100");
     } else if tool.is_like_clang() || tool.is_like_gnu() {
-        base_config.flag("-std=c++11").flag("-Wno-unused-parameter");
+        consensus_config.flag("-std=c++17").flag("-Wno-unused-parameter");
     }
 
-    if target.contains("windows") {
-        base_config.define("WIN32", "1");
-    }
-    base_config
+    consensus_config
+        .cpp(true)
+        .include("depend/bitcoin/src")
+        .include("depend/bitcoin/src/secp256k1/include")
         .file("depend/bitcoin/src/util/strencodings.cpp")
         .file("depend/bitcoin/src/uint256.cpp")
         .file("depend/bitcoin/src/pubkey.cpp")

contrib/vendor-bitcoin-core.sh

@@ -0,0 +1,119 @@
+#!/usr/bin/env bash
+set -e
+
+function usage() {
+    echo
+    echo "Usage: script OPTIONS [bitcoin-core-version]"
+    echo
+    echo "OPTIONS:"
+    echo
+    echo " -f    Vendor even if there are local changes to the rust-bitcoinconsensus git index"
+    echo " -h    Print this help an exit"
+    echo
+    echo "Example:"
+    echo
+    echo "    vendor-bitcoin-core v0.21.2"
+    echo
+
+    exit 0
+}
+
+if (($# < 1)) || [ "$1" == '-h' ]; then
+   usage
+fi
+
+# Set default variables
+
+if [ -z "$CORE_VENDOR_GIT_ROOT" ]; then
+    CORE_VENDOR_GIT_ROOT="$(git rev-parse --show-toplevel)"
+else
+    CORE_VENDOR_GIT_ROOT="$(realpath "$CORE_VENDOR_GIT_ROOT")"
+fi
+
+DEFAULT_DEPEND_DIR="depend"
+DEFAULT_CORE_REPO=https://github.com/bitcoin/bitcoin.git
+
+: "${CORE_VENDOR_DEPEND_DIR:=$DEFAULT_DEPEND_DIR}"
+: "${CORE_VENDOR_REPO:=$DEFAULT_CORE_REPO}"
+
+# CP_NOT_CLONE lets us just copy a directory rather than git cloning.
+# This is usually a bad idea, since it will bring in build artifacts or any other
+# junk from the source directory, but may be useful during development or CI.
+: "${CORE_VENDOR_CP_NOT_CLONE:=no}"
+
+echo "Using depend directory $CORE_VENDOR_DEPEND_DIR. Set CORE_VENDOR_DEPEND_DIR to override."
+echo "Using bitcoin repository $CORE_VENDOR_REPO. Set CORE_VENDOR_REPO to override."
+
+# Parse command-line options
+CORE_REV=""
+FORCE=no
+while (( "$#" )); do
+    case "$1" in
+    -h)
+        echo ""
+        usage
+        ;;
+    -f)
+        FORCE=yes
+        ;;
+    *)
+        if [ -z "$CORE_REV" ]; then
+            CORE_REV="$1"
+        else
+            echo "WARNING: ignoring unknown command-line argument $1"
+        fi
+        ;;
+    esac
+    shift
+done
+
+echo
+if [ "$CORE_REV" ]; then
+    echo "Vendoring Bitcoin Core version: $CORE_REV"
+else
+    echo "WARNING: No Bitcoin Core revision specified. Will use whatever we find at the git repo."
+fi
+echo
+
+# Check if we will do anything destructive.
+
+if [ "$FORCE" == "no" ]; then
+    if ! git diff --quiet -- "*.rs"; then
+        echo "ERROR: There appear to be modified source files. Check these in or pass -f (some source files will be modified to have symbols renamed)."
+        exit 2
+    fi
+    if ! git diff --quiet -- "$CORE_VENDOR_DEPEND_DIR"; then
+        echo "ERROR: The depend directory appears to be modified. Check it in or pass -f (this directory will be deleted)."
+        exit 2
+    fi
+fi
+
+DIR=./bitcoin
+
+pushd "$CORE_VENDOR_DEPEND_DIR" > /dev/null
+rm -rf "$DIR" || true
+
+# Clone the repo. As a special case, if the repo is a local path and we have
+# not specified a revision, just copy the directory rather than using 'git clone'.
+# This lets us use non-git repos or dirty source trees as secp sources.
+if [ "$CORE_VENDOR_CP_NOT_CLONE" == "yes" ]; then
+    cp -r "$CORE_VENDOR_REPO" "$DIR"
+    chmod -R +w "$DIR" # cp preserves write perms, which if missing will cause patch to fail
+else
+    git clone "$CORE_VENDOR_REPO" "$DIR"
+fi
+
+# Check out specified revision
+pushd "$DIR" > /dev/null
+if [ -n "$CORE_REV" ]; then
+    git checkout "$CORE_REV"
+fi
+SOURCE_REV=$(git rev-parse HEAD || echo "[unknown revision from $CORE_VENDOR_REPO]")
+rm -rf .git/ || true
+popd
+
+# Record revision
+echo "# This file was automatically created by $(basename "$0")" > ./bitcoin-HEAD-revision.txt
+echo "$SOURCE_REV" >> ./bitcoin-HEAD-revision.txt
+
+popd > /dev/null

depend/bitcoin-HEAD-revision.txt

@@ -0,0 +1,2 @@
+# This file was automatically created by vendor-bitcoin-core.sh
+af591f2068d0363c92d9756ca39c43db85e5804c

depend/bitcoin/.appveyor.yml

@@ -7,29 +7,24 @@ clone_depth: 5
 environment:
   PATH: 'C:\Python37-x64;C:\Python37-x64\Scripts;%PATH%'
   PYTHONUTF8: 1
-  QT_DOWNLOAD_URL: 'https://github.com/sipsorcery/qt_win_binary/releases/download/v1.6/Qt5.9.8_x64_static_vs2019.zip'
-  QT_DOWNLOAD_HASH: '9a8c6eb20967873785057fdcd329a657c7f922b0af08c5fde105cc597dd37e21'
+  QT_DOWNLOAD_URL: 'https://github.com/sipsorcery/qt_win_binary/releases/download/qt598x64_vs2019_v1681/qt598_x64_vs2019_1681.zip'
+  QT_DOWNLOAD_HASH: '00cf7327818c07d74e0b1a4464ffe987c2728b00d49d4bf333065892af0515c3'
   QT_LOCAL_PATH: 'C:\Qt5.9.8_x64_static_vs2019'
-  VCPKG_INSTALL_PATH: 'C:\tools\vcpkg\installed'
-  VCPKG_COMMIT_ID: 'f3f329a048eaff759c1992c458f2e12351486bc7'
+  VCPKG_TAG: '75522bb1f2e7d863078bcd06322348f053a9e33f'
 install:
 # Disable zmq test for now since python zmq library on Windows would cause Access violation sometimes.
 # - cmd: pip install zmq
-# Powershell block below is to install the c++ dependencies via vcpkg. The pseudo code is:
+# The powershell block below is to set up vcpkg to install the c++ dependencies. The pseudo code is:
 #    a. Checkout the vcpkg source (including port files) for the specific checkout and build the vcpkg binary,
-#    b. Install the missing packages.
+#    b. Append a setting to the vcpkg cmake config file to only do release builds of dependencies (skipping deubg builds saves ~5 mins).
+# Note originally this block also installed the dependencies using 'vcpkg install'. Dependencies are now installed
+# as part of the msbuild command using vcpkg mainfests.
 - ps: |
-      $env:PACKAGES = Get-Content -Path build_msvc\vcpkg-packages.txt
-      Write-Host "vcpkg installing packages: $env:PACKAGES"
       cd c:\tools\vcpkg
       $env:GIT_REDIRECT_STDERR = '2>&1' # git is writing non-errors to STDERR when doing git pull. Send to STDOUT instead.
-      git pull origin master > $null
-      git -c advice.detachedHead=false checkout $env:VCPKG_COMMIT_ID
+      git -c advice.detachedHead=false checkout $env:VCPKG_TAG
       .\bootstrap-vcpkg.bat > $null
       Add-Content "C:\tools\vcpkg\triplets\$env:PLATFORM-windows-static.cmake" "set(VCPKG_BUILD_TYPE release)"
-      .\vcpkg install --triplet $env:PLATFORM-windows-static $env:PACKAGES.split() > $null
-      Write-Host "vcpkg packages installed successfully."
-      .\vcpkg integrate install
       cd "$env:APPVEYOR_BUILD_FOLDER"
 before_build:
 # Powershell block below is to download and extract the Qt static libraries. The pseudo code is:
@@ -55,11 +50,14 @@ after_build:
 #- 7z a bitcoin-%APPVEYOR_BUILD_VERSION%.zip %APPVEYOR_BUILD_FOLDER%\build_msvc\%platform%\%configuration%\*.exe
 test_script:
 - cmd: src\test_bitcoin.exe -l test_suite
-- cmd: src\bench_bitcoin.exe -evals=1 -scaling=0 > NUL
+- cmd: src\bench_bitcoin.exe > NUL
 - ps:  python test\util\bitcoin-util-test.py
 - cmd: python test\util\rpcauth-test.py
 # Fee estimation test failing on appveyor with: WinError 10048] Only one usage of each socket address (protocol/network address/port) is normally permitted.
-- cmd: python test\functional\test_runner.py --ci --quiet --combinedlogslen=4000 --failfast --exclude feature_fee_estimation
+# functional tests disabled for now. See
+# https://github.com/bitcoin/bitcoin/pull/18626#issuecomment-613396202
+# https://github.com/bitcoin/bitcoin/issues/18623
+# - cmd: python test\functional\test_runner.py --ci --quiet --combinedlogslen=4000 --failfast --exclude feature_fee_estimation
 artifacts:
 #- path: bitcoin-%APPVEYOR_BUILD_VERSION%.zip
 deploy: off