Tests

.appveyor.yml

@@ -5,86 +5,62 @@ configuration: Release
 platform: x64
 clone_depth: 5
 environment:
-  APPVEYOR_SAVE_CACHE_ON_ERROR: true
-  CLCACHE_SERVER: 1
   PATH: 'C:\Python37-x64;C:\Python37-x64\Scripts;%PATH%'
   PYTHONUTF8: 1
-  QT_DOWNLOAD_URL: 'https://github.com/sipsorcery/qt_win_binary/releases/download/v1.6/Qt5.9.8_x64_static_vs2019.zip'
-  QT_DOWNLOAD_HASH: '9a8c6eb20967873785057fdcd329a657c7f922b0af08c5fde105cc597dd37e21'
+  QT_DOWNLOAD_URL: 'https://github.com/sipsorcery/qt_win_binary/releases/download/qt598x64_vs2019_v1681/qt598_x64_vs2019_1681.zip'
+  QT_DOWNLOAD_HASH: '00cf7327818c07d74e0b1a4464ffe987c2728b00d49d4bf333065892af0515c3'
   QT_LOCAL_PATH: 'C:\Qt5.9.8_x64_static_vs2019'
-  VCPKG_INSTALL_PATH: 'C:\tools\vcpkg\installed'
-cache:
-- C:\tools\vcpkg\installed -> build_msvc\vcpkg-packages.txt
-- C:\Users\appveyor\clcache -> .appveyor.yml, build_msvc\**, **\Makefile.am, **\*.vcxproj.in
-- C:\Qt5.9.8_x64_static_vs2019
+  VCPKG_TAG: '2020.11-1'
 install:
-- cmd: pip install --quiet git+https://github.com/frerich/clcache.git@v4.2.0
 # Disable zmq test for now since python zmq library on Windows would cause Access violation sometimes.
 # - cmd: pip install zmq
-# Powershell block below is to install the c++ dependencies via vcpkg. The pseudo code is:
-# 1. Check whether the vcpkg install directory exists (note that updating the vcpkg-packages.txt file
-#    will cause the appveyor cache rules to invalidate the directory)
-# 2. If the directory is missing:
-#    a. Update the vcpkg source (including port files) and build the vcpkg binary,
-#    b. Install the missing packages.
+# The powershell block below is to set up vcpkg to install the c++ dependencies. The pseudo code is:
+#    a. Checkout the vcpkg source (including port files) for the specific checkout and build the vcpkg binary,
+#    b. Append a setting to the vcpkg cmake config file to only do release builds of dependencies (skipping deubg builds saves ~5 mins).
+# Note originally this block also installed the dependencies using 'vcpkg install'. Dependencies are now installed
+# as part of the msbuild command using vcpkg mainfests.
 - ps: |
-      $env:PACKAGES = Get-Content -Path build_msvc\vcpkg-packages.txt
-      Write-Host "vcpkg list: $env:PACKAGES"
-      if(!(Test-Path -Path ($env:VCPKG_INSTALL_PATH))) {
-          cd c:\tools\vcpkg
-          $env:GIT_REDIRECT_STDERR = '2>&1' # git is writing non-errors to STDERR when doing git pull. Send to STDOUT instead.
-          git pull origin master
-          .\bootstrap-vcpkg.bat
-          Add-Content "C:\tools\vcpkg\triplets\$env:PLATFORM-windows-static.cmake" "set(VCPKG_BUILD_TYPE release)"
-          .\vcpkg install --triplet $env:PLATFORM-windows-static $env:PACKAGES.split() > $null
-          cd "$env:APPVEYOR_BUILD_FOLDER"
-      }
-      else {
-        Write-Host "required vcpkg packages already installed."
-      }
-      c:\tools\vcpkg\vcpkg integrate install
+      cd c:\tools\vcpkg
+      $env:GIT_REDIRECT_STDERR = '2>&1' # git is writing non-errors to STDERR when doing git pull. Send to STDOUT instead.
+      git -c advice.detachedHead=false checkout $env:VCPKG_TAG
+      git pull origin $env:VCPKG_TAG
+      .\bootstrap-vcpkg.bat > $null
+      Add-Content "C:\tools\vcpkg\triplets\$env:PLATFORM-windows-static.cmake" "set(VCPKG_BUILD_TYPE release)"
+      cd "$env:APPVEYOR_BUILD_FOLDER"
 before_build:
-- ps:  clcache -M 536870912
 # Powershell block below is to download and extract the Qt static libraries. The pseudo code is:
-# 1. If the Qt destination directory exists assume it is correct and do nothing. To
-#    force a fresh install of the packages delete the job's appveyor cache.
-# 2. Otherwise:
 #    a. Download the zip file with the prebuilt Qt static libraries.
 #    b. Check that the downloaded file matches the expected hash.
 #    c. Extract the zip file to the specific destination path expected by the msbuild projects.
 - ps: |
-      if(!(Test-Path -Path ($env:QT_LOCAL_PATH))) {
-        Write-Host "Downloading Qt binaries.";
-        Invoke-WebRequest -Uri $env:QT_DOWNLOAD_URL -Out qtdownload.zip;
-        Write-Host "Qt binaries successfully downloaded, checking hash against $env:QT_DOWNLOAD_HASH...";
-        if((Get-FileHash qtdownload.zip).Hash -eq $env:QT_DOWNLOAD_HASH) {
-          Expand-Archive qtdownload.zip -DestinationPath $env:QT_LOCAL_PATH;
-          Write-Host "Qt binary download matched the expected hash.";
-        }
-        else {
-          Write-Host "ERROR: Qt binary download did not match the expected hash.";
-          Exit-AppveyorBuild;
-        }
+      Write-Host "Downloading Qt binaries.";
+      Invoke-WebRequest -Uri $env:QT_DOWNLOAD_URL -Out qtdownload.zip;
+      Write-Host "Qt binaries successfully downloaded, checking hash against $env:QT_DOWNLOAD_HASH...";
+      if((Get-FileHash qtdownload.zip).Hash -eq $env:QT_DOWNLOAD_HASH) {
+        Expand-Archive qtdownload.zip -DestinationPath $env:QT_LOCAL_PATH;
+        Write-Host "Qt binary download matched the expected hash.";
       }
       else {
-         Write-Host "Qt binaries already present.";
+        Write-Host "ERROR: Qt binary download did not match the expected hash.";
+        Exit-AppveyorBuild;
       }
 - cmd: python build_msvc\msvc-autogen.py
-- ps:  Start-Process clcache-server
-- ps:  fsutil behavior set disablelastaccess 0 # Enable Access time feature on Windows (for clcache)
 build_script:
 - cmd: msbuild /p:TrackFileAccess=false /p:CLToolExe=clcache.exe build_msvc\BGL.sln /m /v:q /nologo
 after_build:
 - ps:  fsutil behavior set disablelastaccess 1 # Disable Access time feature on Windows (better performance)
 - ps:  clcache -z
 #- 7z a BGL-%APPVEYOR_BUILD_VERSION%.zip %APPVEYOR_BUILD_FOLDER%\build_msvc\%platform%\%configuration%\*.exe
 test_script:
-- cmd: src\test_BGL.exe -k stdout -e stdout 2> NUL
-- cmd: src\bench_BGL.exe -evals=1 -scaling=0 > NUL
+- cmd: src\test_BGL.exe -l test_suite
+- cmd: src\bench_BGL.exe > NUL
 - ps:  python test\util\BGL-util-test.py
 - cmd: python test\util\rpcauth-test.py
 # Fee estimation test failing on appveyor with: WinError 10048] Only one usage of each socket address (protocol/network address/port) is normally permitted.
-- cmd: python test\functional\test_runner.py --ci --quiet --combinedlogslen=4000 --failfast --exclude feature_fee_estimation
+# functional tests disabled for now. See
+# https://github.com/BGL/BGL/pull/18626#issuecomment-613396202
+# https://github.com/BGL/BGL/issues/18623
+# - cmd: python test\functional\test_runner.py --ci --quiet --combinedlogslen=4000 --failfast --exclude feature_fee_estimation
 artifacts:
 #- path: BGL-%APPVEYOR_BUILD_VERSION%.zip
 deploy: off

.cirrus.yml

@@ -1,53 +1,161 @@
-task:
-  name: "FreeBsd 12.0 amd64  [GOAL: install]  [no depends, only system libs]"
-  freebsd_instance:
-    image: freebsd-12-0-release-amd64
-    cpu: 8
-    memory: 8G
-  timeout_in: 60m
-  env:
-    MAKEJOBS: "-j9"
-    CONFIGURE_OPTS: "--disable-dependency-tracking"
-    GOAL: "install"
-    TEST_RUNNER_PORT_MIN: "14000"  # Must be larger than 12321, which is used for the http cache. See https://cirrus-ci.org/guide/writing-tasks/#http-cache
-    CCACHE_SIZE: "200M"
-    CCACHE_COMPRESS: 1
-    CCACHE_DIR: "/tmp/ccache_dir"
-  ccache_cache:
-    folder: "/tmp/ccache_dir"
-  install_script:
-    - pkg install -y autoconf automake boost-libs git gmake libevent libtool pkgconf python3 ccache
-    - ./contrib/install_db4.sh $(pwd)
-    - ccache --max-size=${CCACHE_SIZE}
-  configure_script:
-    - ./autogen.sh
-    - ./configure ${CONFIGURE_OPTS} BDB_LIBS="-L$(pwd)/db4/lib -ldb_cxx-4.8" BDB_CFLAGS="-I$(pwd)/db4/include" || ( cat config.log && false)
-  make_script:
-    - gmake ${MAKEJOBS} ${GOAL} || ( echo "Build failure. Verbose build follows." && gmake ${GOAL} V=1 ; false )
-  check_script:
-    - gmake check ${MAKEJOBS} VERBOSE=1
-  functional_test_script:
-    - ./test/functional/test_runner.py --jobs 9 --ci --extended --exclude feature_dbcrash --combinedlogslen=1000 --quiet --failfast
-task:
-  name: "x86_64 Linux  [GOAL: install]  [bionic]  [Using ./ci/ system]"
-  container:
-    image: ubuntu:18.04
-    cpu: 8
-    memory: 8G
-  timeout_in: 60m
-  env:
-    MAKEJOBS: "-j9"
-    RUN_CI_ON_HOST: "1"
-    TEST_RUNNER_PORT_MIN: "14000"  # Must be larger than 12321, which is used for the http cache. See https://cirrus-ci.org/guide/writing-tasks/#http-cache
-    CCACHE_SIZE: "200M"
-    CCACHE_DIR: "/tmp/ccache_dir"
+### Global defaults
+
+timeout_in: 120m  # https://cirrus-ci.org/faq/#instance-timed-out
+container:
+  # https://cirrus-ci.org/faq/#are-there-any-limits
+  # Each project has 16 CPU in total, assign 2 to each container, so that 8 tasks run in parallel
+  cpu: 2
+  memory: 8G  # Set to 8GB to avoid OOM. https://cirrus-ci.org/guide/linux/#linux-containers
+  kvm: true  # Use kvm to avoid spurious CI failures in the default virtualization cluster, see https://github.com/bitcoin/bitcoin/issues/20093
+env:
+  PACKAGE_MANAGER_INSTALL: "apt-get update && apt-get install -y"
+  MAKEJOBS: "-j4"
+  DANGER_RUN_CI_ON_HOST: "1"  # Containers will be discarded after the run, so there is no risk that the ci scripts modify the system
+  TEST_RUNNER_PORT_MIN: "14000"  # Must be larger than 12321, which is used for the http cache. See https://cirrus-ci.org/guide/writing-tasks/#http-cache
+  CCACHE_SIZE: "200M"
+  CCACHE_DIR: "/tmp/ccache_dir"
+
+### Global task template
+
+# https://cirrus-ci.org/guide/tips-and-tricks/#sharing-configuration-between-tasks
+global_task_template: &GLOBAL_TASK_TEMPLATE
+  skip: $CIRRUS_REPO_FULL_NAME == "bitcoin-core/gui" && $CIRRUS_PR == ""  # No need to run on the read-only mirror, unless it is a PR. https://cirrus-ci.org/guide/writing-tasks/#conditional-task-execution
   ccache_cache:
     folder: "/tmp/ccache_dir"
   depends_built_cache:
     folder: "/tmp/cirrus-ci-build/depends/built"
-  install_script:
-    - apt-get update
-    - apt-get -y install git bash ccache
-    - ccache --max-size=${CCACHE_SIZE}
+  depends_sdk_cache:
+    folder: "/tmp/cirrus-ci-build/depends/sdk-sources"
+  depends_releases_cache:
+    folder: "/tmp/cirrus-ci-build/releases"
+  merge_base_script:
+    - if [ "$CIRRUS_PR" = "" ]; then exit 0; fi
+    - bash -c "$PACKAGE_MANAGER_INSTALL git"
+    - git fetch $CIRRUS_REPO_CLONE_URL $CIRRUS_BASE_BRANCH
+    - git config --global user.email "ci@ci.ci"
+    - git config --global user.name "ci"
+    - git merge FETCH_HEAD  # Merge base to detect silent merge conflicts
   ci_script:
     - ./ci/test_run_all.sh
+
+#task:
+#  name: "Windows"
+#  windows_container:
+#    image: cirrusci/windowsservercore:2019
+#  env:
+#    CIRRUS_SHELL: powershell
+#    PATH: 'C:\Python37;C:\Python37\Scripts;%PATH%'
+#    PYTHONUTF8: 1
+#    QT_DOWNLOAD_URL: 'https://github.com/sipsorcery/qt_win_binary/releases/download/v1.6/Qt5.9.8_x64_static_vs2019.zip'
+#    QT_DOWNLOAD_HASH: '9a8c6eb20967873785057fdcd329a657c7f922b0af08c5fde105cc597dd37e21'
+#    QT_LOCAL_PATH: 'C:\Qt5.9.8_x64_static_vs2019'
+#    VCPKG_INSTALL_PATH: 'C:\tools\vcpkg\installed'
+#    VCPKG_COMMIT_ID: 'ed0df8ecc4ed7e755ea03e18aaf285fd9b4b4a74'
+#  install_script:
+#    - choco install python --version=3.7.7 -y
+
+task:
+  name: 'ARM [unit tests, no functional tests] [buster]'
+  << : *GLOBAL_TASK_TEMPLATE
+  container:
+    image: debian:buster
+  env:
+    FILE_ENV: "./ci/test/00_setup_env_arm.sh"
+
+task:
+  name: 'Win64 [unit tests, no gui tests, no boost::process, no functional tests] [bionic]'
+  << : *GLOBAL_TASK_TEMPLATE
+  container:
+    image: ubuntu:bionic
+  env:
+    FILE_ENV: "./ci/test/00_setup_env_win64.sh"
+
+task:
+  name: '32-bit + dash [gui] [CentOS 8]'
+  << : *GLOBAL_TASK_TEMPLATE
+  container:
+    image: centos:8
+  env:
+    PACKAGE_MANAGER_INSTALL: "yum install -y"
+    FILE_ENV: "./ci/test/00_setup_env_i686_centos.sh"
+
+task:
+  name: '[previous releases, uses qt5 dev package and some depends packages] [unsigned char] [bionic]'
+  << : *GLOBAL_TASK_TEMPLATE
+  container:
+    image: ubuntu:bionic
+  env:
+    FILE_ENV: "./ci/test/00_setup_env_native_qt5.sh"
+
+task:
+  name: '[depends, sanitizers: thread (TSan), no gui] [focal]'
+  << : *GLOBAL_TASK_TEMPLATE
+  container:
+    image: ubuntu:focal
+    cpu: 4  # Double CPU and increase Memory to avoid timeout
+    memory: 24G
+  env:
+    MAKEJOBS: "-j8"
+    FILE_ENV: "./ci/test/00_setup_env_native_tsan.sh"
+
+task:
+  name: '[depends, sanitizers: memory (MSan)] [focal]'
+  << : *GLOBAL_TASK_TEMPLATE
+  container:
+    image: ubuntu:focal
+  env:
+    FILE_ENV: "./ci/test/00_setup_env_native_msan.sh"
+
+task:
+  name: '[no depends, only system libs, sanitizers: address/leak (ASan + LSan) + undefined (UBSan) + integer] [focal]'
+  << : *GLOBAL_TASK_TEMPLATE
+  container:
+    image: ubuntu:focal
+  env:
+    FILE_ENV: "./ci/test/00_setup_env_native_asan.sh"
+
+task:
+  name: '[no depends, only system libs, sanitizers: fuzzer,address,undefined] [focal]'
+  << : *GLOBAL_TASK_TEMPLATE
+  container:
+    image: ubuntu:focal
+  env:
+    FILE_ENV: "./ci/test/00_setup_env_native_fuzz.sh"
+
+task:
+  name: '[multiprocess] [focal]'
+  << : *GLOBAL_TASK_TEMPLATE
+  container:
+    image: ubuntu:focal
+  env:
+    FILE_ENV: "./ci/test/00_setup_env_native_multiprocess.sh"
+
+task:
+  name: '[no wallet] [bionic]'
+  << : *GLOBAL_TASK_TEMPLATE
+  container:
+    image: ubuntu:bionic
+  env:
+    FILE_ENV: "./ci/test/00_setup_env_native_nowallet.sh"
+
+task:
+  name: 'macOS 10.14 [gui, no tests] [bionic]'
+  << : *GLOBAL_TASK_TEMPLATE
+  container:
+    image: ubuntu:bionic
+  env:
+    FILE_ENV: "./ci/test/00_setup_env_mac.sh"
+
+task:
+  name: 'macOS 10.15 native [gui] [no depends]'
+  macos_brew_addon_script:
+    - brew install boost libevent berkeley-db4 qt miniupnpc ccache zeromq qrencode sqlite libtool automake pkg-config gnu-getopt
+  << : *GLOBAL_TASK_TEMPLATE
+  osx_instance:
+    # Use latest image, but hardcode version to avoid silent upgrades (and breaks)
+    image: catalina-xcode-12.1  # https://cirrus-ci.org/guide/macOS
+  env:
+    DANGER_RUN_CI_ON_HOST: "true"
+    CI_USE_APT_INSTALL: "no"
+    PACKAGE_MANAGER_INSTALL: "echo"  # Nothing to do
+    FILE_ENV: "./ci/test/00_setup_env_mac_host.sh"

.fuzzbuzz.yml

@@ -0,0 +1,16 @@
+base: ubuntu:16.04
+language: c++
+engine: libFuzzer
+environment:
+  - CXXFLAGS=-fcoverage-mapping -fno-omit-frame-pointer -fprofile-instr-generate -gline-tables-only -O1
+setup:
+  - sudo apt-get update
+  - sudo apt-get install -y autoconf bsdmainutils clang git libboost-all-dev libboost-program-options-dev libc++1 libc++abi1 libc++abi-dev libc++-dev libclang1 libclang-dev libdb5.3++ libevent-dev libllvm-ocaml-dev libomp5 libomp-dev libprotobuf-dev libqt5core5a libqt5dbus5 libqt5gui5 libssl-dev libtool llvm llvm-dev llvm-runtime pkg-config protobuf-compiler qttools5-dev qttools5-dev-tools software-properties-common
+  - ./autogen.sh
+  - CC=clang CXX=clang++ ./configure --enable-fuzz --with-sanitizers=address,fuzzer,undefined
+  - make
+  - git clone https://github.com/bitcoin-core/qa-assets
+auto_targets:
+  find_targets_command: find src/test/fuzz/ -executable -type f ! -name "*.cpp" ! -name "*.h"
+  base_corpus_dir: qa-assets/fuzz_seed_corpus/
+  memory_limit: none

.github/ISSUE_TEMPLATE/gui_issue.md

@@ -0,0 +1,11 @@
+---
+name: An issue or feature request related to the GUI
+about: Any report, issue or feature request related to the GUI should be reported at https://github.com/bitcoin-core/gui/issues/
+title: Any report, issue or feature request related to the GUI should be reported at https://github.com/bitcoin-core/gui/issues/
+labels: GUI
+assignees: ''
+
+---
+
+Any report, issue or feature request related to the GUI should be reported at
+https://github.com/bitcoin-core/gui/issues/

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,8 +1,13 @@
 ### Description
 what is the goal of the pull request
 
-### Items done
-item list of changes or additions
+Pull requests without a rationale and clear improvement may be closed
+immediately.
+
+GUI-related pull requests should be opened against
+https://github.com/BGL-core/gui
+first. See CONTRIBUTING.md
+-->
 
 ### Notes
 implementation details, hints for reviewers