diff --git a/.gitmodules b/.gitmodules index 91bddb0a3f8..6e2339fb942 100644 --- a/.gitmodules +++ b/.gitmodules @@ -9,3 +9,6 @@ [submodule "external/rapidjson"] path = external/rapidjson url = https://github.com/Tencent/rapidjson +[submodule "external/trezor-common"] + path = external/trezor-common + url = https://github.com/trezor/trezor-common.git diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 00000000000..d83675869e5 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,62 @@ +sudo: required +dist: trusty +os: linux +language: minimal +cache: + directories: + - contrib/depends/built + - contrib/depends/sdk-sources + - $HOME/.ccache +env: + global: + - MAKEJOBS=-j3 + - CCACHE_SIZE=100M + - CCACHE_TEMPDIR=/tmp/.ccache-temp + - CCACHE_COMPRESS=1 + - CCACHE_DIR=$HOME/.ccache + - BASE_OUTDIR=$TRAVIS_BUILD_DIR/out + - SDK_URL=https://bitcoincore.org/depends-sources/sdks + - DOCKER_PACKAGES="build-essential libtool cmake autotools-dev automake pkg-config bsdmainutils curl git ca-certificates ccache" + matrix: +# ARM v7 + - HOST=arm-linux-gnueabihf PACKAGES="python3 gperf g++-arm-linux-gnueabihf" +# ARM v8 + - HOST=aarch64-linux-gnu PACKAGES="python3 gperf g++-aarch64-linux-gnu" +# i686 Win + - HOST=i686-w64-mingw32 DEP_OPTS="NO_QT=1" PACKAGES="python3 g++-mingw-w64-i686 qttools5-dev-tools" +# i686 Linux + - HOST=i686-pc-linux-gnu PACKAGES="gperf cmake g++-multilib python3-zmq" +# Win64 + - HOST=x86_64-w64-mingw32 DEP_OPTS="NO_QT=1" PACKAGES="cmake python3 g++-mingw-w64-x86-64 qttools5-dev-tools" +# x86_64 Linux + - HOST=x86_64-unknown-linux-gnu PACKAGES="gperf cmake python3-zmq libdbus-1-dev libharfbuzz-dev" +# Cross-Mac + - HOST=x86_64-apple-darwin11 PACKAGES="cmake imagemagick libcap-dev librsvg2-bin libz-dev libbz2-dev libtiff-tools python-dev python3-setuptools-git" OSX_SDK=10.11 + +before_install: + - export PATH=$(echo $PATH | tr ':' "\n" | sed '/\/opt\/python/d' | tr "\n" ":" | sed "s|::|:|g") +install: + - env | grep -E '^(CCACHE_|DISPLAY|CONFIG_SHELL)' | tee /tmp/env + - if [[ $HOST = *-mingw32 ]]; then DOCKER_ADMIN="--cap-add SYS_ADMIN"; fi + - DOCKER_ID=$(docker run $DOCKER_ADMIN -idt --mount type=bind,src=$TRAVIS_BUILD_DIR,dst=$TRAVIS_BUILD_DIR --mount type=bind,src=$CCACHE_DIR,dst=$CCACHE_DIR -w $TRAVIS_BUILD_DIR --env-file /tmp/env ubuntu:18.04) + - DOCKER_EXEC="docker exec $DOCKER_ID" + - if [ -n "$DPKG_ADD_ARCH" ]; then $DOCKER_EXEC dpkg --add-architecture "$DPKG_ADD_ARCH" ; fi + - travis_retry $DOCKER_EXEC apt-get update + - travis_retry $DOCKER_EXEC apt-get install --no-install-recommends --no-upgrade -qq $PACKAGES $DOCKER_PACKAGES +before_script: + - mkdir -p contrib/depends/SDKs contrib/depends/sdk-sources + - if [ -n "$OSX_SDK" -a ! -f contrib/depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz ]; then curl --location --fail $SDK_URL/MacOSX${OSX_SDK}.sdk.tar.gz -o contrib/depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz; fi + - if [ -n "$OSX_SDK" -a -f contrib/depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz ]; then tar -C contrib/depends/SDKs -xf contrib/depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz; fi + - if [[ $HOST = *-mingw32 ]]; then $DOCKER_EXEC bash -c "update-alternatives --set $HOST-g++ \$(which $HOST-g++-posix)"; fi + - if [[ $HOST = *-mingw32 ]]; then $DOCKER_EXEC bash -c "update-alternatives --set $HOST-gcc \$(which $HOST-gcc-posix)"; fi + - if [ -z "$NO_DEPENDS" ]; then $DOCKER_EXEC bash -c "CONFIG_SHELL= make $MAKEJOBS -C contrib/depends HOST=$HOST $DEP_OPTS"; fi +script: + - git submodule init && git submodule update + - export TRAVIS_COMMIT_LOG=`git log --format=fuller -1` + - OUTDIR=$BASE_OUTDIR/$TRAVIS_PULL_REQUEST/$TRAVIS_JOB_NUMBER-$HOST + - if [ -z "$NO_DEPENDS" ]; then $DOCKER_EXEC ccache --max-size=$CCACHE_SIZE; fi + - $DOCKER_EXEC bash -c "mkdir build && cd build && cmake -DCMAKE_TOOLCHAIN_FILE=$TRAVIS_BUILD_DIR/contrib/depends/$HOST/share/toolchain.cmake .. && make $MAKEJOBS" + - export LD_LIBRARY_PATH=$TRAVIS_BUILD_DIR/contrib/depends/$HOST/lib +after_script: + - echo $TRAVIS_COMMIT_RANGE + - echo $TRAVIS_COMMIT_LOG diff --git a/ANONYMITY_NETWORKS.md b/ANONYMITY_NETWORKS.md new file mode 100644 index 00000000000..feb8528da46 --- /dev/null +++ b/ANONYMITY_NETWORKS.md @@ -0,0 +1,210 @@ +# Anonymity Networks with Monero + +Currently only Tor and I2P have been integrated into Monero. The usage of +these networks is still considered experimental - there are a few pessimistic +cases where privacy is leaked. The design is intended to maximize privacy of +the source of a transaction by broadcasting it over an anonymity network, while +relying on IPv4 for the remainder of messages to make surrounding node attacks +(via sybil) more difficult. + + +## Behavior + +If _any_ anonymity network is enabled, transactions being broadcast that lack +a valid "context" (i.e. the transaction did not come from a p2p connection), +will only be sent to peers on anonymity networks. If an anonymity network is +enabled but no peers over an anonymity network are available, an error is +logged and the transaction is kept for future broadcasting over an anonymity +network. The transaction will not be broadcast unless an anonymity connection +is made or until `monerod` is shutdown and restarted with only public +connections enabled. + +Anonymity networks can also be used with `monero-wallet-cli` and +`monero-wallet-rpc` - the wallets will connect to a daemon through a proxy. The +daemon must provide a hidden service for the RPC itself, which is separate from +the hidden service for P2P connections. + + +## P2P Commands + +Only handshakes, peer timed syncs and transaction broadcast messages are +supported over anonymity networks. If one `--add-exclusive-node` p2p address +is specified, then no syncing will take place and only transaction broadcasting +can occur. It is therefore recommended that `--add-exclusive-node` be combined +with additional exclusive IPv4 address(es). + + +## Usage + +Anonymity networks have no seed nodes (the feature is still considered +experimental), so a user must specify an address. If configured properly, +additional peers can be found through typical p2p peerlist sharing. + +### Outbound Connections + +Connecting to an anonymous address requires the command line option +`--proxy` which tells `monerod` the ip/port of a socks proxy provided by a +separate process. On most systems the configuration will look like: + +> `--proxy tor,127.0.0.1:9050,10` +> `--proxy i2p,127.0.0.1:9000` + +which tells `monerod` that ".onion" p2p addresses can be forwarded to a socks +proxy at IP 127.0.0.1 port 9050 with a max of 10 outgoing connections and +".b32.i2p" p2p addresses can be forwarded to a socks proxy at IP 127.0.0.1 port +9000 with the default max outgoing connections. Since there are no seed nodes +for anonymity connections, peers must be manually specified: + +> `--add-exclusive-node rveahdfho7wo4b2m.onion:28083` +> `--add-peer rveahdfho7wo4b2m.onion:28083` + +Either option can be listed multiple times, and can specify any mix of Tor, +I2P, and IPv4 addresses. Using `--add-exclusive-node` will prevent the usage of +seed nodes on ALL networks, which will typically be undesireable. + +### Inbound Connections + +Receiving anonymity connections is done through the option +`--anonymous-inbound`. This option tells `monerod` the inbound address, network +type, and max connections: + +> `--anonymous-inbound rveahdfho7wo4b2m.onion:28083,127.0.0.1:28083,25` +> `--anonymous-inbound cmeua5767mz2q5jsaelk2rxhf67agrwuetaso5dzbenyzwlbkg2q.b32.i2p:5000,127.0.0.1:30000` + +which tells `monerod` that a max of 25 inbound Tor connections are being +received at address "rveahdfho7wo4b2m.onion:28083" and forwarded to `monerod` +localhost port 28083, and a default max I2P connections are being received at +address "cmeua5767mz2q5jsaelk2rxhf67agrwuetaso5dzbenyzwlbkg2q.b32.i2p:5000" and +forwarded to `monerod` localhost port 30000. +These addresses will be shared with outgoing peers, over the same network type, +otherwise the peer will not be notified of the peer address by the proxy. + +### Wallet RPC + +An anonymity network can be configured to forward incoming connections to a +`monerod` RPC port - which is independent from the configuration for incoming +P2P anonymity connections. The anonymity network (Tor/i2p) is +[configured in the same manner](#configuration), except the localhost port +must be the RPC port (typically 18081 for mainnet) instead of the p2p port: + +> HiddenServiceDir /var/lib/tor/data/monero +> HiddenServicePort 18081 127.0.0.1:18081 + +Then the wallet will be configured to use a Tor/i2p address: +> `--proxy 127.0.0.1:9050` +> `--daemon-address rveahdfho7wo4b2m.onion` + +The proxy must match the address type - a Tor proxy will not work properly with +i2p addresses, etc. + +i2p and onion addresses provide the information necessary to authenticate and +encrypt the connection from end-to-end. If desired, SSL can also be applied to +the connection with `--daemon-address https://rveahdfho7wo4b2m.onion` which +requires a server certificate that is signed by a "root" certificate on the +machine running the wallet. Alternatively, `--daemon-cert-file` can be used to +specify a certificate to authenticate the server. + +Proxies can also be used to connect to "clearnet" (ipv4 addresses or ICANN +domains), but `--daemon-cert-file` _must_ be used for authentication and +encryption. + +### Network Types + +#### Tor & I2P + +Options `--add-exclusive-node` and `--add-peer` recognize ".onion" and +".b32.i2p" addresses, and will properly forward those addresses to the proxy +provided with `--proxy tor,...` or `--proxy i2p,...`. + +Option `--anonymous-inbound` also recognizes ".onion" and ".b32.i2p" addresses, +and will automatically be sent out to outgoing Tor/I2P connections so the peer +can distribute the address to its other peers. + +##### Configuration + +Tor must be configured for hidden services. An example configuration ("torrc") +might look like: + +> HiddenServiceDir /var/lib/tor/data/monero +> HiddenServicePort 28083 127.0.0.1:28083 + +This will store key information in `/var/lib/tor/data/monero` and will forward +"Tor port" 28083 to port 28083 of ip 127.0.0.1. The file +`/usr/lib/tor/data/monero/hostname` will contain the ".onion" address for use +with `--anonymous-inbound`. + +I2P must be configured with a standard server tunnel. Configuration differs by +I2P implementation. + +## Privacy Limitations + +There are currently some techniques that could be used to _possibly_ identify +the machine that broadcast a transaction over an anonymity network. + +### Timestamps + +The peer timed sync command sends the current time in the message. This value +can be used to link an onion address to an IPv4/IPv6 address. If a peer first +sees a transaction over Tor, it could _assume_ (possibly incorrectly) that the +transaction originated from the peer. If both the Tor connection and an +IPv4/IPv6 connection have timestamps that are approximately close in value they +could be used to link the two connections. This is less likely to happen if the +system clock is fairly accurate - many peers on the Monero network should have +similar timestamps. + +#### Mitigation + +Keep the system clock accurate so that fingerprinting is more difficult. In +the future a random offset might be applied to anonymity networks so that if +the system clock is noticeably off (and therefore more fingerprintable), +linking the public IPv4/IPv6 connections with the anonymity networks will be +more difficult. + +### Bandwidth Usage + +An ISP can passively monitor `monerod` connections from a node and observe when +a transaction is sent over a Tor/I2P connection via timing analysis + size of +data sent during that timeframe. I2P should provide better protection against +this attack - its connections are not circuit based. However, if a node is +only using I2P for broadcasting Monero transactions, the total aggregate of +I2P data would also leak information. + +#### Mitigation + +There is no current mitigation for the user right now. This attack is fairly +sophisticated, and likely requires support from the internet host of a Monero +user. + +In the near future, "whitening" the amount of data sent over anonymity network +connections will be performed. An attempt will be made to make a transaction +broadcast indistinguishable from a peer timed sync command. + +### Intermittent Monero Syncing + +If a user only runs `monerod` to send a transaction then quit, this can also +be used by an ISP to link a user to a transaction. + +#### Mitigation + +Run `monerod` as often as possible to conceal when transactions are being sent. +Future versions will also have peers that first receive a transaction over an +anonymity network delay the broadcast to public peers by a randomized amount. +This will not completetely mitigate a user who syncs up sends then quits, in +part because this rule is not enforceable, so this mitigation strategy is +simply a best effort attempt. + +### Active Bandwidth Shaping + +An attacker could attempt to bandwidth shape traffic in an attempt to determine +the source of a Tor/I2P connection. There isn't great mitigation against +this, but I2P should provide better protection against this attack since +the connections are not circuit based. + +#### Mitigation + +The best mitigiation is to use I2P instead of Tor. However, I2P +has a smaller set of users (less cover traffic) and academic reviews, so there +is a tradeoff in potential isses. Also, anyone attempting this strategy really +wants to uncover a user, it seems unlikely that this would be performed against +every Tor/I2P user. + diff --git a/CMakeLists.txt b/CMakeLists.txt index e3cfaa24b7a..6a22478e9ae 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2014-2018, The Monero Project +# Copyright (c) 2014-2019, The Monero Project # # All rights reserved. # @@ -40,9 +40,12 @@ if (IOS) endif() cmake_minimum_required(VERSION 2.8.7) +message(STATUS "CMake version ${CMAKE_VERSION}") project(monero) +enable_language(C ASM) + function (die msg) if (NOT WIN32) string(ASCII 27 Esc) @@ -112,6 +115,9 @@ string(TOLOWER ${CMAKE_BUILD_TYPE} CMAKE_BUILD_TYPE_LOWER) # to identify the target architecture, to direct logic in this cmake script. # Since ARCH is a cached variable, it will not be set on first cmake invocation. if (NOT ARCH OR ARCH STREQUAL "" OR ARCH STREQUAL "native" OR ARCH STREQUAL "default") + if(CMAKE_SYSTEM_PROCESSOR STREQUAL "") + set(CMAKE_SYSTEM_PROCESSOR ${CMAKE_HOST_SYSTEM_PROCESSOR}) + endif() set(ARCH_ID "${CMAKE_SYSTEM_PROCESSOR}") else() set(ARCH_ID "${ARCH}") @@ -137,9 +143,27 @@ endif() if(ARCH_ID STREQUAL "ppc64le") set(PPC64LE 1) + set(PPC64 0) + set(PPC 0) +endif() + +if(ARCH_ID STREQUAL "powerpc64" OR ARCH_ID STREQUAL "ppc64") + set(PPC64LE 0) + set(PPC64 1) + set(PPC 0) endif() -if(WIN32 OR ARM) +if(ARCH_ID STREQUAL "powerpc" OR ARCH_ID STREQUAL "ppc") + set(PPC64LE 0) + set(PPC64 0) + set(PPC 1) +endif() + +if(ARCH_ID STREQUAL "s390x") + set(S390X 1) +endif() + +if(WIN32 OR ARM OR PPC64LE OR PPC64 OR PPC) set(OPT_FLAGS_RELEASE "-O2") else() set(OPT_FLAGS_RELEASE "-Ofast") @@ -153,6 +177,28 @@ else() message(STATUS "Building without build tag") endif() +if(NOT MANUAL_SUBMODULES) + find_package(Git) + if(GIT_FOUND) + function (check_submodule relative_path) + execute_process(COMMAND git rev-parse "HEAD" WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/${relative_path} OUTPUT_VARIABLE localHead) + execute_process(COMMAND git rev-parse "HEAD:${relative_path}" WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} OUTPUT_VARIABLE checkedHead) + string(COMPARE EQUAL "${localHead}" "${checkedHead}" upToDate) + if (upToDate) + message(STATUS "Submodule '${relative_path}' is up-to-date") + else() + message(FATAL_ERROR "Submodule '${relative_path}' is not up-to-date. Please update with\ngit submodule update --init --force ${relative_path}\nor run cmake with -DMANUAL_SUBMODULES=1") + endif() + endfunction () + + message(STATUS "Checking submodules") + check_submodule(external/miniupnp) + check_submodule(external/unbound) + check_submodule(external/rapidjson) + check_submodule(external/trezor-common) + endif() +endif() + set(CMAKE_C_FLAGS_RELEASE "-DNDEBUG ${OPT_FLAGS_RELEASE}") set(CMAKE_CXX_FLAGS_RELEASE "-DNDEBUG ${OPT_FLAGS_RELEASE}") @@ -161,6 +207,9 @@ set(PER_BLOCK_CHECKPOINT 1) if(PER_BLOCK_CHECKPOINT) add_definitions("-DPER_BLOCK_CHECKPOINT") + set(Blocks "blocks") +else() + set(Blocks "") endif() list(INSERT CMAKE_MODULE_PATH 0 @@ -357,6 +406,9 @@ add_definitions("-DBLOCKCHAIN_DB=${BLOCKCHAIN_DB}") if (APPLE) set(DEFAULT_STACK_TRACE OFF) set(LIBUNWIND_LIBRARIES "") +elseif (DEPENDS AND NOT LINUX) + set(DEFAULT_STACK_TRACE OFF) + set(LIBUNWIND_LIBRARIES "") elseif(CMAKE_C_COMPILER_ID STREQUAL "GNU" AND NOT MINGW) set(DEFAULT_STACK_TRACE ON) set(STACK_TRACE_LIB "easylogging++") # for diag output only @@ -387,6 +439,8 @@ if (UNIX AND NOT APPLE) # Note that at the time of this writing the -Wstrict-prototypes flag added below will make this fail set(THREADS_PREFER_PTHREAD_FLAG ON) find_package(Threads) + add_c_flag_if_supported(-pthread CMAKE_C_FLAGS) + add_cxx_flag_if_supported(-pthread CMAKE_CXX_FLAGS) endif() # Handle OpenSSL, used for sha256sum on binary updates and light wallet ssl http @@ -395,6 +449,7 @@ if (CMAKE_SYSTEM_NAME MATCHES "(SunOS|Solaris)") endif () if (APPLE AND NOT IOS) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=x86-64 -fvisibility=default") if (NOT OpenSSL_DIR) EXECUTE_PROCESS(COMMAND brew --prefix openssl OUTPUT_VARIABLE OPENSSL_ROOT_DIR @@ -412,7 +467,7 @@ if(STATIC AND NOT IOS) endif() endif() -find_package(PCSC) +find_package(HIDAPI) add_definition_if_library_exists(c memset_s "string.h" HAVE_MEMSET_S) add_definition_if_library_exists(c explicit_bzero "strings.h" HAVE_EXPLICIT_BZERO) @@ -421,13 +476,16 @@ add_definition_if_function_found(strptime HAVE_STRPTIME) add_definitions(-DAUTO_INITIALIZE_EASYLOGGINGPP) # Generate header for embedded translations +# Generate header for embedded translations, use target toolchain if depends, otherwise use the +# lrelease and lupdate binaries from the host include(ExternalProject) ExternalProject_Add(generate_translations_header SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/translations" BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}/translations" + STAMP_DIR ${LRELEASE_PATH} + CMAKE_ARGS -DLRELEASE_PATH=${LRELEASE_PATH} INSTALL_COMMAND cmake -E echo "") include_directories("${CMAKE_CURRENT_BINARY_DIR}/translations") - add_subdirectory(external) # Final setup for libunbound @@ -450,13 +508,19 @@ endif() include_directories(${LIBUNWIND_INCLUDE}) link_directories(${LIBUNWIND_LIBRARY_DIRS}) -# Final setup for libpcsc -if (PCSC_FOUND) - add_definitions(-DHAVE_PCSC) - include_directories(${PCSC_INCLUDE_DIR}) - link_directories(${LIBPCSC_LIBRARY_DIRS}) +# Final setup for hid +if (HIDAPI_FOUND) + message(STATUS "Using HIDAPI include dir at ${HIDAPI_INCLUDE_DIR}") + add_definitions(-DHAVE_HIDAPI) + include_directories(${HIDAPI_INCLUDE_DIR}) + link_directories(${LIBHIDAPI_LIBRARY_DIRS}) +else (HIDAPI_FOUND) + message(STATUS "Could not find HIDAPI") endif() +# Trezor support check +include(CheckTrezor) + if(MSVC) add_definitions("/bigobj /MP /W3 /GS- /D_CRT_SECURE_NO_WARNINGS /wd4996 /wd4345 /D_WIN32_WINNT=0x0600 /DWIN32_LEAN_AND_MEAN /DGTEST_HAS_TR1_TUPLE=0 /FIinline_c.h /D__SSE4_1__") # set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /Dinline=__inline") @@ -469,22 +533,70 @@ if(MSVC) include_directories(SYSTEM src/platform/msc) else() include(TestCXXAcceptsFlag) - set(ARCH native CACHE STRING "CPU to build for: -march value or 'default' to not pass -march at all") + if (NOT ARCH) + set(ARCH native CACHE STRING "CPU to build for: -march value or 'default' to not pass -march at all") + endif() message(STATUS "Building on ${CMAKE_SYSTEM_PROCESSOR} for ${ARCH}") if(ARCH STREQUAL "default") set(ARCH_FLAG "") elseif(PPC64LE) - set(ARCH_FLAG "-mcpu=${ARCH}") + set(ARCH_FLAG "-mcpu=power8") + elseif(PPC64) + set(ARCH_FLAG "-mcpu=970") + elseif(PPC) + set(ARCH_FLAG "-mcpu=7400") elseif(IOS AND ARCH STREQUAL "arm64") message(STATUS "IOS: Changing arch from arm64 to armv8") set(ARCH_FLAG "-march=armv8") else() set(ARCH_FLAG "-march=${ARCH}") + if(ARCH STREQUAL "native") + check_c_compiler_flag(-march=native CC_SUPPORTS_MARCH_NATIVE) + if (NOT CC_SUPPORTS_MARCH_NATIVE) + check_c_compiler_flag(-mtune=native CC_SUPPORTS_MTUNE_NATIVE) + if (CC_SUPPORTS_MTUNE_NATIVE) + set(ARCH_FLAG "-mtune=${ARCH}") + else() + set(ARCH_FLAG "") + endif() + endif() + endif() endif() - set(WARNINGS "-Wall -Wextra -Wpointer-arith -Wundef -Wvla -Wwrite-strings -Wno-error=extra -Wno-error=deprecated-declarations -Wno-unused-parameter -Wno-unused-variable -Wno-error=unused-variable -Wno-error=undef -Wno-error=uninitialized") - if(NOT MINGW) - set(WARNINGS_AS_ERRORS_FLAG "-Werror") + + option(NO_AES "Explicitly disable AES support" ${NO_AES}) + + if(NO_AES) + message(STATUS "AES support explicitly disabled") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DNO_AES") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DNO_AES") + elseif(NOT ARM AND NOT PPC64LE AND NOT PPC64 AND NOT PPC AND NOT S390X) + message(STATUS "AES support enabled") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -maes") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -maes") + elseif(PPC64LE OR PPC64 OR PPC) + message(STATUS "AES support not available on POWER") + elseif(S390X) + message(STATUS "AES support not available on s390x") + elseif(ARM6) + message(STATUS "AES support not available on ARMv6") + elseif(ARM7) + message(STATUS "AES support not available on ARMv7") + elseif(ARM8) + CHECK_CXX_ACCEPTS_FLAG("-march=${ARCH}+crypto" ARCH_PLUS_CRYPTO) + if(ARCH_PLUS_CRYPTO) + message(STATUS "Crypto extensions enabled for ARMv8") + set(ARCH_FLAG "-march=${ARCH}+crypto") + else() + message(STATUS "Crypto extensions unavailable on your ARMv8 device") + endif() + else() + message(STATUS "AES support disabled") endif() + + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${ARCH_FLAG}") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${ARCH_FLAG}") + + set(WARNINGS "-Wall -Wextra -Wpointer-arith -Wundef -Wvla -Wwrite-strings -Wno-error=extra -Wno-error=deprecated-declarations -Wno-unused-parameter -Wno-unused-variable -Wno-error=unused-variable -Wno-error=undef -Wno-error=uninitialized") if(CMAKE_C_COMPILER_ID STREQUAL "Clang") if(ARM) set(WARNINGS "${WARNINGS} -Wno-error=inline-asm") @@ -545,10 +657,26 @@ else() add_cxx_flag_if_supported(-Wformat-security CXX_SECURITY_FLAGS) # -fstack-protector - add_c_flag_if_supported(-fstack-protector C_SECURITY_FLAGS) - add_cxx_flag_if_supported(-fstack-protector CXX_SECURITY_FLAGS) - add_c_flag_if_supported(-fstack-protector-strong C_SECURITY_FLAGS) - add_cxx_flag_if_supported(-fstack-protector-strong CXX_SECURITY_FLAGS) + if (NOT WIN32) + add_c_flag_if_supported(-fstack-protector C_SECURITY_FLAGS) + add_cxx_flag_if_supported(-fstack-protector CXX_SECURITY_FLAGS) + add_c_flag_if_supported(-fstack-protector-strong C_SECURITY_FLAGS) + add_cxx_flag_if_supported(-fstack-protector-strong CXX_SECURITY_FLAGS) + endif() + + # New in GCC 8.2 + if (NOT WIN32) + add_c_flag_if_supported(-fcf-protection=full C_SECURITY_FLAGS) + add_cxx_flag_if_supported(-fcf-protection=full CXX_SECURITY_FLAGS) + add_c_flag_if_supported(-fstack-clash-protection C_SECURITY_FLAGS) + add_cxx_flag_if_supported(-fstack-clash-protection CXX_SECURITY_FLAGS) + endif() + + # Removed in GCC 9.1 (or before ?), but still accepted, so spams the output + if (NOT (CMAKE_C_COMPILER_ID STREQUAL "GNU" AND NOT CMAKE_C_COMPILER_VERSION VERSION_LESS 9.1)) + add_c_flag_if_supported(-mmitigate-rop C_SECURITY_FLAGS) + add_cxx_flag_if_supported(-mmitigate-rop CXX_SECURITY_FLAGS) + endif() # linker if (NOT WIN32) @@ -560,12 +688,16 @@ else() add_linker_flag_if_supported(-Wl,-z,noexecstack noexecstack_SUPPORTED) if (noexecstack_SUPPORTED) set(LD_SECURITY_FLAGS "${LD_SECURITY_FLAGS} -Wl,-z,noexecstack") - set(LD_RAW_FLAGS ${LD_RAW_FLAGS} -z noexecstack) endif() add_linker_flag_if_supported(-Wl,-z,noexecheap noexecheap_SUPPORTED) if (noexecheap_SUPPORTED) set(LD_SECURITY_FLAGS "${LD_SECURITY_FLAGS} -Wl,-z,noexecheap") - set(LD_RAW_FLAGS ${LD_RAW_FLAGS} -z noexecheap) + endif() + + if(BACKCOMPAT) + add_linker_flag_if_supported(-Wl,--wrap=__divmoddi4 LD_BACKCOMPAT_FLAGS) + add_linker_flag_if_supported(-Wl,--wrap=glob LD_BACKCOMPAT_FLAGS) + message(STATUS "Using Lib C back compat flags: ${LD_BACKCOMPAT_FLAGS}") endif() # some windows linker bits @@ -578,37 +710,9 @@ else() message(STATUS "Using C++ security hardening flags: ${CXX_SECURITY_FLAGS}") message(STATUS "Using linker security hardening flags: ${LD_SECURITY_FLAGS}") - option(NO_AES "Explicitly disable AES support" ${NO_AES}) - - if(NO_AES) - message(STATUS "AES support explicitly disabled") - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DNO_AES") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DNO_AES") - elseif(NOT ARM AND NOT PPC64LE) - message(STATUS "AES support enabled") - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -maes") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -maes") - elseif(PPC64LE) - message(STATUS "AES support not available on ppc64le") - elseif(ARM6) - message(STATUS "AES support not available on ARMv6") - elseif(ARM7) - message(STATUS "AES support not available on ARMv7") - elseif(ARM8) - CHECK_CXX_ACCEPTS_FLAG("-march=${ARCH}+crypto" ARCH_PLUS_CRYPTO) - if(ARCH_PLUS_CRYPTO) - message(STATUS "Crypto extensions enabled for ARMv8") - set(ARCH_FLAG "-march=${ARCH}+crypto") - else() - message(STATUS "Crypto extensions unavailable on your ARMv8 device") - endif() - else() - message(STATUS "AES support disabled") - endif() - - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c11 -D_GNU_SOURCE ${MINGW_FLAG} ${STATIC_ASSERT_FLAG} ${WARNINGS} ${C_WARNINGS} ${ARCH_FLAG} ${COVERAGE_FLAGS} ${PIC_FLAG} ${C_SECURITY_FLAGS}") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -D_GNU_SOURCE ${MINGW_FLAG} ${STATIC_ASSERT_CPP_FLAG} ${WARNINGS} ${CXX_WARNINGS} ${ARCH_FLAG} ${COVERAGE_FLAGS} ${PIC_FLAG} ${CXX_SECURITY_FLAGS}") - set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${LD_SECURITY_FLAGS}") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c11 -D_GNU_SOURCE ${MINGW_FLAG} ${STATIC_ASSERT_FLAG} ${WARNINGS} ${C_WARNINGS} ${COVERAGE_FLAGS} ${PIC_FLAG} ${C_SECURITY_FLAGS}") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -D_GNU_SOURCE ${MINGW_FLAG} ${STATIC_ASSERT_CPP_FLAG} ${WARNINGS} ${CXX_WARNINGS} ${COVERAGE_FLAGS} ${PIC_FLAG} ${CXX_SECURITY_FLAGS}") + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${LD_SECURITY_FLAGS} ${LD_BACKCOMPAT_FLAGS}") # With GCC 6.1.1 the compiled binary malfunctions due to aliasing. Until that # is fixed in the code (Issue #847), force compiler to be conservative. @@ -648,6 +752,10 @@ else() message(STATUS "Selecting VFP for ARMv6") set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mfpu=vfp") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mfpu=vfp") + if(DEPENDS) + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -marm") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -marm") + endif() endif(ARM6) if(ARM7) @@ -702,7 +810,7 @@ else() endif() if(APPLE) - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DGTEST_HAS_TR1_TUPLE=0") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fvisibility=default -DGTEST_HAS_TR1_TUPLE=0") endif() set(DEBUG_FLAGS "-g3") @@ -712,6 +820,9 @@ else() set(DEBUG_FLAGS "${DEBUG_FLAGS} -O0 ") endif() + # At least some CLANGs default to not enough for monero + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -ftemplate-depth=900") + if(NOT DEFINED USE_LTO_DEFAULT) set(USE_LTO_DEFAULT false) endif() @@ -770,6 +881,7 @@ set(OLD_LIB_SUFFIXES ${CMAKE_FIND_LIBRARY_SUFFIXES}) if(STATIC) if(MINGW) set(CMAKE_FIND_LIBRARY_SUFFIXES .a) + set(Boost_NO_BOOST_CMAKE ON) endif() set(Boost_USE_STATIC_LIBS ON) @@ -779,20 +891,31 @@ find_package(Boost 1.58 QUIET REQUIRED COMPONENTS system filesystem thread date_ set(CMAKE_FIND_LIBRARY_SUFFIXES ${OLD_LIB_SUFFIXES}) if(NOT Boost_FOUND) - die("Could not find Boost libraries, please make sure you have installed Boost or libboost-all-dev (1.58) or the equivalent") + die("Could not find Boost libraries, please make sure you have installed Boost or libboost-all-dev (>=1.58) or the equivalent") elseif(Boost_FOUND) message(STATUS "Found Boost Version: ${Boost_VERSION}") - if (Boost_VERSION VERSION_LESS 106200 AND NOT (OPENSSL_VERSION VERSION_LESS 1.1)) - message(FATAL_ERROR "Boost older than 1.62 is too old to link with OpenSSL 1.1 or newer. " + if (Boost_VERSION VERSION_LESS 10 AND Boost_VERSION VERSION_LESS 1.62.0 AND NOT (OPENSSL_VERSION VERSION_LESS 1.1)) + set(BOOST_BEFORE_1_62 true) + endif() + if (NOT Boost_VERSION VERSION_LESS 10 AND Boost_VERSION VERSION_LESS 106200 AND NOT (OPENSSL_VERSION VERSION_LESS 1.1)) + set(BOOST_BEFORE_1_62 true) + endif() + if (BOOST_BEFORE_1_62) + message(FATAL_ERROR "Boost ${Boost_VERSION} (older than 1.62) is too old to link with OpenSSL ${OPENSSL_VERSION} (1.1 or newer) found at ${OPENSSL_INCLUDE_DIR} and ${OPENSSL_LIBRARIES}. " "Update Boost or install OpenSSL 1.0 and set path to it when running cmake: " - "cmake -DOPENSSL_ROOT_DIR='/usr/include/openssl-1.0;/usr/lib/openssl-1.0'") + "cmake -DOPENSSL_ROOT_DIR='/usr/include/openssl-1.0'") endif() endif() include_directories(SYSTEM ${Boost_INCLUDE_DIRS}) if(MINGW) - set(EXTRA_LIBRARIES mswsock;ws2_32;iphlpapi) - set(ICU_LIBRARIES ${Boost_LOCALE_LIBRARY} icuio icuin icuuc icudt icutu iconv) + set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -Wa,-mbig-obj") + set(EXTRA_LIBRARIES mswsock;ws2_32;iphlpapi;crypt32;bcrypt) + if(DEPENDS) + set(ICU_LIBRARIES icuio icui18n icuuc icudata icutu iconv) + else() + set(ICU_LIBRARIES icuio icuin icuuc icudt icutu iconv) + endif() elseif(APPLE OR OPENBSD OR ANDROID) set(EXTRA_LIBRARIES "") elseif(FREEBSD) @@ -802,13 +925,29 @@ elseif(DRAGONFLY) set(EXTRA_LIBRARIES execinfo ${COMPAT}) elseif(CMAKE_SYSTEM_NAME MATCHES "(SunOS|Solaris)") set(EXTRA_LIBRARIES socket nsl resolv) -elseif(NOT MSVC) +elseif(NOT MSVC AND NOT DEPENDS) find_library(RT rt) set(EXTRA_LIBRARIES ${RT}) endif() list(APPEND EXTRA_LIBRARIES ${CMAKE_DL_LIBS}) +if (HIDAPI_FOUND OR LibUSB_COMPILE_TEST_PASSED) + if (APPLE) + if(DEPENDS) + list(APPEND EXTRA_LIBRARIES "-framework Foundation -framework IOKit") + else() + find_library(COREFOUNDATION CoreFoundation) + find_library(IOKIT IOKit) + list(APPEND EXTRA_LIBRARIES ${IOKIT}) + list(APPEND EXTRA_LIBRARIES ${COREFOUNDATION}) + endif() + endif() + if (WIN32) + list(APPEND EXTRA_LIBRARIES setupapi) + endif() +endif() + option(USE_READLINE "Build with GNU readline support." ON) if(USE_READLINE) find_package(Readline) @@ -824,10 +963,13 @@ endif() if(ANDROID) set(ATOMIC libatomic.a) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-error=user-defined-warnings") endif() if(CMAKE_C_COMPILER_ID STREQUAL "Clang" AND ARCH_WIDTH EQUAL "32" AND NOT IOS AND NOT FREEBSD) find_library(ATOMIC atomic) - list(APPEND EXTRA_LIBRARIES ${ATOMIC}) + if (ATOMIC_FOUND) + list(APPEND EXTRA_LIBRARIES ${ATOMIC}) + endif() endif() find_path(ZMQ_INCLUDE_PATH zmq.hpp) @@ -890,3 +1032,5 @@ option(INSTALL_VENDORED_LIBUNBOUND "Install libunbound binary built from source CHECK_C_COMPILER_FLAG(-std=c11 HAVE_C11) + +find_package(PythonInterp) diff --git a/CMakeLists_IOS.txt b/CMakeLists_IOS.txt index 9273af19f6e..3b7eacc8a45 100644 --- a/CMakeLists_IOS.txt +++ b/CMakeLists_IOS.txt @@ -1,4 +1,4 @@ -# Portions Copyright (c) 2017-2018, The Monero Project +# Portions Copyright (c) 2017-2019, The Monero Project # This file is based off of the https://code.google.com/archive/p/ios-cmake/ # It has been altered for Monero iOS development # diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f794c0d5e58..7b184c00ac3 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,7 +1,7 @@ # Contributing to Monero A good way to help is to test, and report bugs. See -[How to Report Bugs Effectively (by Simon Tatham)](http://www.chiark.greenend.org.uk/~sgtatham/bugs.html) +[How to Report Bugs Effectively (by Simon Tatham)](https://www.chiark.greenend.org.uk/~sgtatham/bugs.html) if you want to help that way. Testing is invaluable in making a piece of software solid and usable. @@ -19,7 +19,8 @@ posted to #monero-dev on irc.freenode.net). Patches should be self contained. A good rule of thumb is to have one patch per separate issue, feature, or logical change. Also, no -other changes, such as random whitespace changes or reindentation. +other changes, such as random whitespace changes, reindentation, +or fixing typoes, spelling, or wording, unless user visible. Following the code style of the particular chunk of code you're modifying is encouraged. Proper squashing should be done (eg, if you're making a buggy patch, then a later patch to fix the bug, @@ -87,26 +88,26 @@ C4 is meant to provide a reusable optimal collaboration model for open source so ### Preliminaries -- The project SHALL use the git distributed revision control system. -- The project SHALL be hosted on github.com or equivalent, herein called the "Platform". -- The project SHALL use the Platform issue tracker. +- The project MUST use the git distributed revision control system. +- The project MUST be hosted on github.com or equivalent, herein called the "Platform". +- The project MUST use the Platform issue tracker. - Non-GitHub example: - "Platform" could be a vanilla git repo and Trac hosted on the same machine/network. - The Platform issue tracker would be Trac. - The project SHOULD have clearly documented guidelines for code style. - A "Contributor" is a person who wishes to provide a patch, being a set of commits that solve some clearly identified problem. - A "Maintainer" is a person who merges patches to the project. Maintainers are not developers; their job is to enforce process. -- Contributors SHALL NOT have commit access to the repository unless they are also Maintainers. -- Maintainers SHALL have commit access to the repository. -- Everyone, without distinction or discrimination, SHALL have an equal right to become a Contributor under the terms of this contract. +- Contributors MUST NOT have commit access to the repository unless they are also Maintainers. +- Maintainers MUST have commit access to the repository. +- Everyone, without distinction or discrimination, MUST have an equal right to become a Contributor under the terms of this contract. ### Licensing and ownership -- The project SHALL use a share-alike license, such as BSD-3, the GPLv3 or a variant thereof (LGPL, AGPL), or the MPLv2. -- All contributions to the project source code ("patches") SHALL use the same license as the project. -- All patches are owned by their authors. There SHALL NOT be any copyright assignment process. -- The copyrights in the project SHALL be owned collectively by all its Contributors. -- Each Contributor SHALL be responsible for identifying themselves in the project Contributor list. +- The project MUST use a share-alike license, such as BSD-3, the GPLv3 or a variant thereof (LGPL, AGPL), or the MPLv2. +- All contributions to the project source code ("patches") MUST use the same license as the project. +- All patches are owned by their authors. There MUST NOT be any copyright assignment process. +- The copyrights in the project MUST be owned collectively by all its Contributors. +- Each Contributor MUST be responsible for identifying themselves in the project Contributor list. ### Patch requirements @@ -115,27 +116,27 @@ C4 is meant to provide a reusable optimal collaboration model for open source so - A patch SHOULD be a minimal and accurate answer to exactly one identified and agreed problem. - A patch MUST adhere to the code style guidelines of the project if these are defined. - A patch MUST adhere to the "Evolution of Public Contracts" guidelines defined below. -- A patch SHALL NOT include non-trivial code from other projects unless the Contributor is the original author of that code. +- A patch MUST NOT include non-trivial code from other projects unless the Contributor is the original author of that code. - A patch MUST compile cleanly and pass project self-tests on at least the principle target platform. - A patch commit message SHOULD consist of a single short (less than 50 character) line summarizing the change, optionally followed by a blank line and then a more thorough description. - A "Correct Patch" is one that satisfies the above requirements. ### Development process -- Change on the project SHALL be governed by the pattern of accurately identifying problems and applying minimal, accurate solutions to these problems. +- Change on the project MUST be governed by the pattern of accurately identifying problems and applying minimal, accurate solutions to these problems. - To request changes, a user SHOULD log an issue on the project Platform issue tracker. - The user or Contributor SHOULD write the issue by describing the problem they face or observe. - The user or Contributor SHOULD seek consensus on the accuracy of their observation, and the value of solving the problem. -- Users SHALL NOT log feature requests, ideas, or suggestions unrelated to Monero code or Monero's dependency code or Monero's potential/future dependency code or research which successfully implements Monero. -- Users SHALL NOT log any solutions to problems (verifiable or hypothetical) of which are not explicitly documented and/or not provable and/or cannot be reasonably proven. -- Thus, the release history of the project SHALL be a list of meaningful issues logged and solved. -- To work on an issue, a Contributor SHALL fork the project repository and then work on their forked repository. -- To submit a patch, a Contributor SHALL create a Platform pull request back to the project. -- A Contributor SHALL NOT commit changes directly to the project. +- Users MUST NOT log feature requests, ideas, or suggestions unrelated to Monero code or Monero's dependency code or Monero's potential/future dependency code or research which successfully implements Monero. +- Users MUST NOT log any solutions to problems (verifiable or hypothetical) of which are not explicitly documented and/or not provable and/or cannot be reasonably proven. +- Thus, the release history of the project MUST be a list of meaningful issues logged and solved. +- To work on an issue, a Contributor MUST fork the project repository and then work on their forked repository. +- To submit a patch, a Contributor MUST create a Platform pull request back to the project. +- A Contributor MUST NOT commit changes directly to the project. - To discuss a patch, people MAY comment on the Platform pull request, on the commit, or elsewhere. -- To accept or reject a patch, a Maintainer SHALL use the Platform interface. +- To accept or reject a patch, a Maintainer MUST use the Platform interface. - Maintainers SHOULD NOT merge their own patches except in exceptional cases, such as non-responsiveness from other Maintainers for an extended period (more than 30 days) or unless urgent as defined by the Monero Maintainers Team. -- Maintainers SHALL NOT make value judgments on correct patches unless the Maintainer (as may happen in rare circumstances) is a core code developer. +- Maintainers MUST NOT make value judgments on correct patches unless the Maintainer (as may happen in rare circumstances) is a core code developer. - Maintainers MUST NOT merge pull requests in less than 168 hours (1 week) unless deemed urgent by at least 2 people from the Monero Maintainer Team. - The Contributor MAY tag an issue as "Ready" after making a pull request for the issue. - The user who created an issue SHOULD close the issue after checking the patch is successful. @@ -145,27 +146,27 @@ C4 is meant to provide a reusable optimal collaboration model for open source so ### Creating stable releases -- The project SHALL have one branch ("master") that always holds the latest in-progress version and SHOULD always build. -- The project SHALL NOT use topic branches for any reason. Personal forks MAY use topic branches. -- To make a stable release someone SHALL fork the repository by copying it and thus become maintainer of this repository. +- The project MUST have one branch ("master") that always holds the latest in-progress version and SHOULD always build. +- The project MUST NOT use topic branches for any reason. Personal forks MAY use topic branches. +- To make a stable release someone MUST fork the repository by copying it and thus become maintainer of this repository. - Forking a project for stabilization MAY be done unilaterally and without agreement of project maintainers. -- A patch to a stabilization project declared "stable" SHALL be accompanied by a reproducible test case. +- A patch to a stabilization project declared "stable" MUST be accompanied by a reproducible test case. ### Evolution of public contracts -- All Public Contracts (APIs or protocols) SHALL be documented. +- All Public Contracts (APIs or protocols) MUST be documented. - All Public Contracts SHOULD have space for extensibility and experimentation. - A patch that modifies a stable Public Contract SHOULD not break existing applications unless there is overriding consensus on the value of doing this. - A patch that introduces new features to a Public Contract SHOULD do so using new names. - Old names SHOULD be deprecated in a systematic fashion by marking new names as "experimental" until they are stable, then marking the old names as "deprecated". - When sufficient time has passed, old deprecated names SHOULD be marked "legacy" and eventually removed. -- Old names SHALL NOT be reused by new features. +- Old names MUST NOT be reused by new features. - When old names are removed, their implementations MUST provoke an exception (assertion) if used by applications. ### Project administration -- The project founders SHALL act as Administrators to manage the set of project Maintainers. -- The Administrators SHALL ensure their own succession over time by promoting the most effective Maintainers. -- A new Contributor who makes a correct patch SHALL be invited to become a Maintainer. +- The project founders MUST act as Administrators to manage the set of project Maintainers. +- The Administrators MUST ensure their own succession over time by promoting the most effective Maintainers. +- A new Contributor who makes a correct patch MUST be invited to become a Maintainer. - Administrators MAY remove Maintainers who are inactive for an extended period of time, or who repeatedly fail to apply this process accurately. - Administrators SHOULD block or ban "bad actors" who cause stress and pain to others in the project. This should be done after public discussion, with a chance for all parties to speak. A bad actor is someone who repeatedly ignores the rules and culture of the project, who is needlessly argumentative or hostile, or who is offensive, and who is unable to self-correct their behavior when asked to do so by others. diff --git a/Dockerfile b/Dockerfile index e36e2082652..999290c952c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,87 +17,161 @@ RUN set -ex && \ curl \ libtool-bin \ autoconf \ - automake + automake \ + bzip2 \ + xsltproc \ + gperf \ + unzip WORKDIR /usr/local +ENV CFLAGS='-fPIC' +ENV CXXFLAGS='-fPIC' + +#Cmake +ARG CMAKE_VERSION=3.14.0 +ARG CMAKE_VERSION_DOT=v3.14 +ARG CMAKE_HASH=aa76ba67b3c2af1946701f847073f4652af5cbd9f141f221c97af99127e75502 +RUN set -ex \ + && curl -s -O https://cmake.org/files/${CMAKE_VERSION_DOT}/cmake-${CMAKE_VERSION}.tar.gz \ + && echo "${CMAKE_HASH} cmake-${CMAKE_VERSION}.tar.gz" | sha256sum -c \ + && tar -xzf cmake-${CMAKE_VERSION}.tar.gz \ + && cd cmake-${CMAKE_VERSION} \ + && ./configure \ + && make \ + && make install + ## Boost -ARG BOOST_VERSION=1_66_0 -ARG BOOST_VERSION_DOT=1.66.0 -ARG BOOST_HASH=5721818253e6a0989583192f96782c4a98eb6204965316df9f5ad75819225ca9 +ARG BOOST_VERSION=1_69_0 +ARG BOOST_VERSION_DOT=1.69.0 +ARG BOOST_HASH=8f32d4617390d1c2d16f26a27ab60d97807b35440d45891fa340fc2648b04406 RUN set -ex \ && curl -s -L -o boost_${BOOST_VERSION}.tar.bz2 https://dl.bintray.com/boostorg/release/${BOOST_VERSION_DOT}/source/boost_${BOOST_VERSION}.tar.bz2 \ && echo "${BOOST_HASH} boost_${BOOST_VERSION}.tar.bz2" | sha256sum -c \ && tar -xvf boost_${BOOST_VERSION}.tar.bz2 \ && cd boost_${BOOST_VERSION} \ && ./bootstrap.sh \ - && ./b2 --build-type=minimal link=static runtime-link=static --with-chrono --with-date_time --with-filesystem --with-program_options --with-regex --with-serialization --with-system --with-thread --with-locale threading=multi threadapi=pthread cflags="-fPIC" cxxflags="-fPIC" stage + && ./b2 --build-type=minimal link=static runtime-link=static --with-chrono --with-date_time --with-filesystem --with-program_options --with-regex --with-serialization --with-system --with-thread --with-locale threading=multi threadapi=pthread cflags="$CFLAGS" cxxflags="$CXXFLAGS" stage ENV BOOST_ROOT /usr/local/boost_${BOOST_VERSION} # OpenSSL -ARG OPENSSL_VERSION=1.0.2n -ARG OPENSSL_HASH=370babb75f278c39e0c50e8c4e7493bc0f18db6867478341a832a982fd15a8fe +ARG OPENSSL_VERSION=1.1.1b +ARG OPENSSL_HASH=5c557b023230413dfb0756f3137a13e6d726838ccd1430888ad15bfb2b43ea4b RUN set -ex \ && curl -s -O https://www.openssl.org/source/openssl-${OPENSSL_VERSION}.tar.gz \ && echo "${OPENSSL_HASH} openssl-${OPENSSL_VERSION}.tar.gz" | sha256sum -c \ && tar -xzf openssl-${OPENSSL_VERSION}.tar.gz \ && cd openssl-${OPENSSL_VERSION} \ - && ./Configure linux-x86_64 no-shared --static -fPIC \ - && make build_crypto build_ssl \ + && ./Configure linux-x86_64 no-shared --static "$CFLAGS" \ + && make build_generated \ + && make libcrypto.a \ && make install ENV OPENSSL_ROOT_DIR=/usr/local/openssl-${OPENSSL_VERSION} # ZMQ -ARG ZMQ_VERSION=v4.2.3 -ARG ZMQ_HASH=3226b8ebddd9c6c738ba42986822c26418a49afb +ARG ZMQ_VERSION=v4.3.1 +ARG ZMQ_HASH=2cb1240db64ce1ea299e00474c646a2453a8435b RUN set -ex \ && git clone https://github.com/zeromq/libzmq.git -b ${ZMQ_VERSION} \ && cd libzmq \ && test `git rev-parse HEAD` = ${ZMQ_HASH} || exit 1 \ && ./autogen.sh \ - && CFLAGS="-fPIC" CXXFLAGS="-fPIC" ./configure --enable-static --disable-shared \ + && ./configure --enable-static --disable-shared \ && make \ && make install \ && ldconfig # zmq.hpp -ARG CPPZMQ_HASH=6aa3ab686e916cb0e62df7fa7d12e0b13ae9fae6 +ARG CPPZMQ_VERSION=v4.3.0 +ARG CPPZMQ_HASH=213da0b04ae3b4d846c9abc46bab87f86bfb9cf4 RUN set -ex \ - && git clone https://github.com/zeromq/cppzmq.git -b ${ZMQ_VERSION} \ + && git clone https://github.com/zeromq/cppzmq.git -b ${CPPZMQ_VERSION} \ && cd cppzmq \ && test `git rev-parse HEAD` = ${CPPZMQ_HASH} || exit 1 \ && mv *.hpp /usr/local/include # Readline -ARG READLINE_VERSION=7.0 -ARG READLINE_HASH=750d437185286f40a369e1e4f4764eda932b9459b5ec9a731628393dd3d32334 +ARG READLINE_VERSION=8.0 +ARG READLINE_HASH=e339f51971478d369f8a053a330a190781acb9864cf4c541060f12078948e461 RUN set -ex \ && curl -s -O https://ftp.gnu.org/gnu/readline/readline-${READLINE_VERSION}.tar.gz \ && echo "${READLINE_HASH} readline-${READLINE_VERSION}.tar.gz" | sha256sum -c \ && tar -xzf readline-${READLINE_VERSION}.tar.gz \ && cd readline-${READLINE_VERSION} \ - && CFLAGS="-fPIC" CXXFLAGS="-fPIC" ./configure \ + && ./configure \ && make \ && make install # Sodium -ARG SODIUM_VERSION=1.0.16 -ARG SODIUM_HASH=675149b9b8b66ff44152553fb3ebf9858128363d +ARG SODIUM_VERSION=1.0.17 +ARG SODIUM_HASH=b732443c442239c2e0184820e9b23cca0de0828c RUN set -ex \ && git clone https://github.com/jedisct1/libsodium.git -b ${SODIUM_VERSION} \ && cd libsodium \ && test `git rev-parse HEAD` = ${SODIUM_HASH} || exit 1 \ && ./autogen.sh \ - && CFLAGS="-fPIC" CXXFLAGS="-fPIC" ./configure \ + && ./configure \ && make \ && make check \ && make install +# Udev +ARG UDEV_VERSION=v3.2.7 +ARG UDEV_HASH=4758e346a14126fc3a964de5831e411c27ebe487 +RUN set -ex \ + && git clone https://github.com/gentoo/eudev -b ${UDEV_VERSION} \ + && cd eudev \ + && test `git rev-parse HEAD` = ${UDEV_HASH} || exit 1 \ + && ./autogen.sh \ + && ./configure --disable-gudev --disable-introspection --disable-hwdb --disable-manpages --disable-shared \ + && make \ + && make install + +# Libusb +ARG USB_VERSION=v1.0.22 +ARG USB_HASH=0034b2afdcdb1614e78edaa2a9e22d5936aeae5d +RUN set -ex \ + && git clone https://github.com/libusb/libusb.git -b ${USB_VERSION} \ + && cd libusb \ + && test `git rev-parse HEAD` = ${USB_HASH} || exit 1 \ + && ./autogen.sh \ + && ./configure --disable-shared \ + && make \ + && make install + +# Hidapi +ARG HIDAPI_VERSION=hidapi-0.8.0-rc1 +ARG HIDAPI_HASH=40cf516139b5b61e30d9403a48db23d8f915f52c +RUN set -ex \ + && git clone https://github.com/signal11/hidapi -b ${HIDAPI_VERSION} \ + && cd hidapi \ + && test `git rev-parse HEAD` = ${HIDAPI_HASH} || exit 1 \ + && ./bootstrap \ + && ./configure --enable-static --disable-shared \ + && make \ + && make install + +# Protobuf +ARG PROTOBUF_VERSION=v3.7.0 +ARG PROTOBUF_HASH=582743bf40c5d3639a70f98f183914a2c0cd0680 +RUN set -ex \ + && git clone https://github.com/protocolbuffers/protobuf -b ${PROTOBUF_VERSION} \ + && cd protobuf \ + && test `git rev-parse HEAD` = ${PROTOBUF_HASH} || exit 1 \ + && git submodule update --init --recursive \ + && ./autogen.sh \ + && ./configure --enable-static --disable-shared \ + && make \ + && make install \ + && ldconfig + WORKDIR /src COPY . . +ENV USE_SINGLE_BUILDDIR=1 ARG NPROC RUN set -ex && \ + git submodule init && git submodule update && \ rm -rf build && \ if [ -z "$NPROC" ] ; \ then make -j$(nproc) release-static ; \ @@ -112,11 +186,16 @@ RUN set -ex && \ apt-get --no-install-recommends --yes install ca-certificates && \ apt-get clean && \ rm -rf /var/lib/apt +COPY --from=builder /src/build/release/bin /usr/local/bin/ -COPY --from=builder /src/build/release/bin/* /usr/local/bin/ +# Create monero user +RUN adduser --system --group --disabled-password monero && \ + mkdir -p /wallet /home/monero/.bitmonero && \ + chown -R monero:monero /home/monero/.bitmonero && \ + chown -R monero:monero /wallet # Contains the blockchain -VOLUME /root/.bitmonero +VOLUME /home/monero/.bitmonero # Generate your wallet via accessing the container and run: # cd /wallet @@ -126,4 +205,8 @@ VOLUME /wallet EXPOSE 18080 EXPOSE 18081 +# switch to user monero +USER monero + ENTRYPOINT ["monerod", "--p2p-bind-ip=0.0.0.0", "--p2p-bind-port=18080", "--rpc-bind-ip=0.0.0.0", "--rpc-bind-port=18081", "--non-interactive", "--confirm-external-bind"] + diff --git a/Doxyfile b/Doxyfile index f622c618d2c..8e08ad3ec79 100644 --- a/Doxyfile +++ b/Doxyfile @@ -20,7 +20,7 @@ # This tag specifies the encoding used for all characters in the config file # that follow. The default is UTF-8 which is also the encoding used for all text # before the first occurrence of this tag. Doxygen uses libiconv (or the iconv -# built into libc) for the transcoding. See http://www.gnu.org/software/libiconv +# built into libc) for the transcoding. See https://www.gnu.org/software/libiconv # for the list of possible encodings. # The default value is: UTF-8. @@ -285,7 +285,7 @@ EXTENSION_MAPPING = # If the MARKDOWN_SUPPORT tag is enabled then doxygen pre-processes all comments # according to the Markdown format, which allows for more readable -# documentation. See http://daringfireball.net/projects/markdown/ for details. +# documentation. See https://daringfireball.net/projects/markdown/ for details. # The output of markdown processing is further processed by doxygen, so you can # mix doxygen, HTML, and XML commands with Markdown formatting. Disable only in # case of backward compatibilities issues. @@ -318,7 +318,7 @@ BUILTIN_STL_SUPPORT = NO CPP_CLI_SUPPORT = NO # Set the SIP_SUPPORT tag to YES if your project consists of sip (see: -# http://www.riverbankcomputing.co.uk/software/sip/intro) sources only. Doxygen +# https://www.riverbankcomputing.co.uk/software/sip/intro) sources only. Doxygen # will parse them like normal C++ but will assume all classes use public instead # of private inheritance when no explicit protection keyword is present. # The default value is: NO. @@ -677,7 +677,7 @@ LAYOUT_FILE = # The CITE_BIB_FILES tag can be used to specify one or more bib files containing # the reference definitions. This must be a list of .bib files. The .bib # extension is automatically appended if omitted. This requires the bibtex tool -# to be installed. See also http://en.wikipedia.org/wiki/BibTeX for more info. +# to be installed. See also https://en.wikipedia.org/wiki/BibTeX for more info. # For LaTeX the style of the bibliography can be controlled using # LATEX_BIB_STYLE. To use this feature you need bibtex and perl available in the # search path. Do not use file names with spaces, bibtex cannot handle them. See @@ -759,7 +759,7 @@ INPUT = src # This tag can be used to specify the character encoding of the source files # that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses # libiconv (or the iconv built into libc) for the transcoding. See the libiconv -# documentation (see: http://www.gnu.org/software/libiconv) for the list of +# documentation (see: https://www.gnu.org/software/libiconv) for the list of # possible encodings. # The default value is: UTF-8. @@ -951,7 +951,7 @@ SOURCE_TOOLTIPS = YES # If the USE_HTAGS tag is set to YES then the references to source code will # point to the HTML generated by the htags(1) tool instead of doxygen built-in # source browser. The htags tool is part of GNU's global source tagging system -# (see http://www.gnu.org/software/global/global.html). You will need version +# (see https://www.gnu.org/software/global/global.html). You will need version # 4.8.6 or higher. # # To use it do the following: @@ -1094,7 +1094,7 @@ HTML_EXTRA_FILES = # The HTML_COLORSTYLE_HUE tag controls the color of the HTML output. Doxygen # will adjust the colors in the stylesheet and background images according to # this color. Hue is specified as an angle on a colorwheel, see -# http://en.wikipedia.org/wiki/Hue for more information. For instance the value +# https://en.wikipedia.org/wiki/Hue for more information. For instance the value # 0 represents red, 60 is yellow, 120 is green, 180 is cyan, 240 is blue, 300 # purple, and 360 is red again. # Minimum value: 0, maximum value: 359, default value: 220. @@ -1152,12 +1152,12 @@ HTML_INDEX_NUM_ENTRIES = 100 # If the GENERATE_DOCSET tag is set to YES, additional index files will be # generated that can be used as input for Apple's Xcode 3 integrated development -# environment (see: http://developer.apple.com/tools/xcode/), introduced with +# environment (see: https://developer.apple.com/tools/xcode/), introduced with # OSX 10.5 (Leopard). To create a documentation set, doxygen will generate a # Makefile in the HTML output directory. Running make will produce the docset in # that directory and running make install will install the docset in # ~/Library/Developer/Shared/Documentation/DocSets so that Xcode will find it at -# startup. See http://developer.apple.com/tools/creatingdocsetswithdoxygen.html +# startup. See https://developer.apple.com/tools/creatingdocsetswithdoxygen.html # for more information. # The default value is: NO. # This tag requires that the tag GENERATE_HTML is set to YES. @@ -1197,7 +1197,7 @@ DOCSET_PUBLISHER_NAME = Publisher # If the GENERATE_HTMLHELP tag is set to YES then doxygen generates three # additional HTML index files: index.hhp, index.hhc, and index.hhk. The # index.hhp is a project file that can be read by Microsoft's HTML Help Workshop -# (see: http://www.microsoft.com/en-us/download/details.aspx?id=21138) on +# (see: https://www.microsoft.com/en-us/download/details.aspx?id=21138) on # Windows. # # The HTML Help Workshop contains a compiler that can convert all HTML output @@ -1273,7 +1273,7 @@ QCH_FILE = # The QHP_NAMESPACE tag specifies the namespace to use when generating Qt Help # Project output. For more information please see Qt Help Project / Namespace -# (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#namespace). +# (see: https://doc.qt.io/archives/qt-4.8/qthelpproject.html#namespace). # The default value is: org.doxygen.Project. # This tag requires that the tag GENERATE_QHP is set to YES. @@ -1281,7 +1281,7 @@ QHP_NAMESPACE = org.doxygen.Project # The QHP_VIRTUAL_FOLDER tag specifies the namespace to use when generating Qt # Help Project output. For more information please see Qt Help Project / Virtual -# Folders (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#virtual- +# Folders (see: https://doc.qt.io/archives/qt-4.8/qthelpproject.html#virtual- # folders). # The default value is: doc. # This tag requires that the tag GENERATE_QHP is set to YES. @@ -1290,7 +1290,7 @@ QHP_VIRTUAL_FOLDER = doc # If the QHP_CUST_FILTER_NAME tag is set, it specifies the name of a custom # filter to add. For more information please see Qt Help Project / Custom -# Filters (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#custom- +# Filters (see: https://doc.qt.io/archives/qt-4.8/qthelpproject.html#custom- # filters). # This tag requires that the tag GENERATE_QHP is set to YES. @@ -1298,7 +1298,7 @@ QHP_CUST_FILTER_NAME = # The QHP_CUST_FILTER_ATTRS tag specifies the list of the attributes of the # custom filter to add. For more information please see Qt Help Project / Custom -# Filters (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#custom- +# Filters (see: https://doc.qt.io/archives/qt-4.8/qthelpproject.html#custom- # filters). # This tag requires that the tag GENERATE_QHP is set to YES. @@ -1306,7 +1306,7 @@ QHP_CUST_FILTER_ATTRS = # The QHP_SECT_FILTER_ATTRS tag specifies the list of the attributes this # project's filter section matches. Qt Help Project / Filter Attributes (see: -# http://qt-project.org/doc/qt-4.8/qthelpproject.html#filter-attributes). +# https://doc.qt.io/archives/qt-4.8/qthelpproject.html#filter-attributes). # This tag requires that the tag GENERATE_QHP is set to YES. QHP_SECT_FILTER_ATTRS = @@ -1411,7 +1411,7 @@ FORMULA_FONTSIZE = 10 FORMULA_TRANSPARENT = YES # Enable the USE_MATHJAX option to render LaTeX formulas using MathJax (see -# http://www.mathjax.org) which uses client side Javascript for the rendering +# https://www.mathjax.org) which uses client side Javascript for the rendering # instead of using prerendered bitmaps. Use this if you do not have LaTeX # installed or if you want to formulas look prettier in the HTML output. When # enabled you may also need to install MathJax separately and configure the path @@ -1423,7 +1423,7 @@ USE_MATHJAX = NO # When MathJax is enabled you can set the default output format to be used for # the MathJax output. See the MathJax site (see: -# http://docs.mathjax.org/en/latest/output.html) for more details. +# https://docs.mathjax.org/en/latest/output.html) for more details. # Possible values are: HTML-CSS (which is slower, but has the best # compatibility), NativeMML (i.e. MathML) and SVG. # The default value is: HTML-CSS. @@ -1438,11 +1438,11 @@ MATHJAX_FORMAT = HTML-CSS # MATHJAX_RELPATH should be ../mathjax. The default value points to the MathJax # Content Delivery Network so you can quickly see the result without installing # MathJax. However, it is strongly recommended to install a local copy of -# MathJax from http://www.mathjax.org before deployment. -# The default value is: http://cdn.mathjax.org/mathjax/latest. +# MathJax from https://www.mathjax.org before deployment. +# The default value is: https://cdn.mathjax.org/mathjax/latest. # This tag requires that the tag USE_MATHJAX is set to YES. -MATHJAX_RELPATH = http://cdn.mathjax.org/mathjax/latest +MATHJAX_RELPATH = https://cdn.mathjax.org/mathjax/latest # The MATHJAX_EXTENSIONS tag can be used to specify one or more MathJax # extension names that should be enabled during MathJax rendering. For example @@ -1453,7 +1453,7 @@ MATHJAX_EXTENSIONS = # The MATHJAX_CODEFILE tag can be used to specify a file with javascript pieces # of code that will be used on startup of the MathJax code. See the MathJax site -# (see: http://docs.mathjax.org/en/latest/output.html) for more details. For an +# (see: https://docs.mathjax.org/en/latest/output.html) for more details. For an # example see the documentation. # This tag requires that the tag USE_MATHJAX is set to YES. @@ -1500,7 +1500,7 @@ SERVER_BASED_SEARCH = NO # # Doxygen ships with an example indexer ( doxyindexer) and search engine # (doxysearch.cgi) which are based on the open source search engine library -# Xapian (see: http://xapian.org/). +# Xapian (see: https://xapian.org/). # # See the section "External Indexing and Searching" for details. # The default value is: NO. @@ -1513,7 +1513,7 @@ EXTERNAL_SEARCH = NO # # Doxygen ships with an example indexer ( doxyindexer) and search engine # (doxysearch.cgi) which are based on the open source search engine library -# Xapian (see: http://xapian.org/). See the section "External Indexing and +# Xapian (see: https://xapian.org/). See the section "External Indexing and # Searching" for details. # This tag requires that the tag SEARCHENGINE is set to YES. @@ -1684,7 +1684,7 @@ LATEX_SOURCE_CODE = NO # The LATEX_BIB_STYLE tag can be used to specify the style to use for the # bibliography, e.g. plainnat, or ieeetr. See -# http://en.wikipedia.org/wiki/BibTeX and \cite for more info. +# https://en.wikipedia.org/wiki/BibTeX and \cite for more info. # The default value is: plain. # This tag requires that the tag GENERATE_LATEX is set to YES. @@ -2051,7 +2051,7 @@ HIDE_UNDOC_RELATIONS = YES # If you set the HAVE_DOT tag to YES then doxygen will assume the dot tool is # available from the path. This tool is part of Graphviz (see: -# http://www.graphviz.org/), a graph visualization toolkit from AT&T and Lucent +# https://www.graphviz.org/), a graph visualization toolkit from AT&T and Lucent # Bell Labs. The other options in this section have no effect if this option is # set to NO # The default value is: NO. diff --git a/LICENSE b/LICENSE index 733f09f2328..7035c2324ac 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright (c) 2014-2018, The Monero Project +Copyright (c) 2014-2019, The Monero Project All rights reserved. @@ -29,3 +29,13 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. Parts of the project are originally copyright (c) 2012-2013 The Cryptonote developers + +Parts of the project are originally copyright (c) 2014 The Boolberry +developers, distributed under the MIT licence: + + Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + diff --git a/Makefile b/Makefile index d0be9bcd679..1fd3109151a 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -# Copyright (c) 2014-2018, The Monero Project +# Copyright (c) 2014-2019, The Monero Project # # All rights reserved. # @@ -26,103 +26,149 @@ # STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF # THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +ANDROID_STANDALONE_TOOLCHAIN_PATH ?= /usr/local/toolchain + +dotgit=$(shell ls -d .git/config) +ifneq ($(dotgit), .git/config) + USE_SINGLE_BUILDDIR=1 +endif + +subbuilddir:=$(shell echo `uname | sed -e 's|[:/\\ \(\)]|_|g'`/`git branch | grep '\* ' | cut -f2- -d' '| sed -e 's|[:/\\ \(\)]|_|g'`) +ifeq ($(USE_SINGLE_BUILDDIR),) + builddir := build/"$(subbuilddir)" + topdir := ../../../.. + deldirs := $(builddir) +else + builddir := build + topdir := ../.. + deldirs := $(builddir)/debug $(builddir)/release $(builddir)/fuzz +endif + all: release-all +depends: + cd contrib/depends && $(MAKE) HOST=$(target) && cd ../.. && mkdir -p build/$(target)/release + cd build/$(target)/release && cmake -DCMAKE_TOOLCHAIN_FILE=$(CURDIR)/contrib/depends/$(target)/share/toolchain.cmake ../../.. && $(MAKE) + cmake-debug: - mkdir -p build/debug - cd build/debug && cmake -D CMAKE_BUILD_TYPE=Debug ../.. + mkdir -p $(builddir)/debug + cd $(builddir)/debug && cmake -D CMAKE_BUILD_TYPE=Debug $(topdir) debug: cmake-debug - cd build/debug && $(MAKE) + cd $(builddir)/debug && $(MAKE) # Temporarily disable some tests: # * libwallet_api_tests fail (Issue #895) debug-test: - mkdir -p build/debug - cd build/debug && cmake -D BUILD_TESTS=ON -D CMAKE_BUILD_TYPE=Debug ../.. && $(MAKE) && $(MAKE) ARGS="-E libwallet_api_tests" test + mkdir -p $(builddir)/debug + cd $(builddir)/debug && cmake -D BUILD_TESTS=ON -D CMAKE_BUILD_TYPE=Debug $(topdir) && $(MAKE) && $(MAKE) ARGS="-E libwallet_api_tests" test + +debug-test-trezor: + mkdir -p $(builddir)/debug + cd $(builddir)/debug && cmake -D BUILD_TESTS=ON -D TREZOR_DEBUG=ON -D CMAKE_BUILD_TYPE=Debug $(topdir) && $(MAKE) && $(MAKE) ARGS="-E libwallet_api_tests" test debug-all: - mkdir -p build/debug - cd build/debug && cmake -D BUILD_TESTS=ON -D BUILD_SHARED_LIBS=OFF -D CMAKE_BUILD_TYPE=Debug ../.. && $(MAKE) + mkdir -p $(builddir)/debug + cd $(builddir)/debug && cmake -D BUILD_TESTS=ON -D BUILD_SHARED_LIBS=OFF -D CMAKE_BUILD_TYPE=Debug $(topdir) && $(MAKE) debug-static-all: - mkdir -p build/debug - cd build/debug && cmake -D BUILD_TESTS=ON -D STATIC=ON -D CMAKE_BUILD_TYPE=Debug ../.. && $(MAKE) - + mkdir -p $(builddir)/debug + cd $(builddir)/debug && cmake -D BUILD_TESTS=ON -D STATIC=ON -D CMAKE_BUILD_TYPE=Debug $(topdir) && $(MAKE) + +debug-static-win64: + mkdir -p $(builddir)/debug + cd $(builddir)/debug && cmake -G "MSYS Makefiles" -D STATIC=ON -D ARCH="x86-64" -D BUILD_64=ON -D CMAKE_BUILD_TYPE=Debug -D BUILD_TAG="win-x64" -D CMAKE_TOOLCHAIN_FILE=$(topdir)/cmake/64-bit-toolchain.cmake -D MSYS2_FOLDER=c:/msys64 $(topdir) && $(MAKE) + +debug-static-win32: + mkdir -p $(builddir)/debug + cd $(builddir)/debug && cmake -G "MSYS Makefiles" -D STATIC=ON -D ARCH="i686" -D BUILD_64=OFF -D CMAKE_BUILD_TYPE=Debug -D BUILD_TAG="win-x32" -D CMAKE_TOOLCHAIN_FILE=$(topdir)/cmake/32-bit-toolchain.cmake -D MSYS2_FOLDER=c:/msys32 $(topdir) && $(MAKE) + cmake-release: - mkdir -p build/release - cd build/release && cmake -D CMAKE_BUILD_TYPE=Release ../.. + mkdir -p $(builddir)/release + cd $(builddir)/release && cmake -D CMAKE_BUILD_TYPE=Release $(topdir) release: cmake-release - cd build/release && $(MAKE) + cd $(builddir)/release && $(MAKE) release-test: - mkdir -p build/release - cd build/release && cmake -D BUILD_TESTS=ON -D CMAKE_BUILD_TYPE=release ../.. && $(MAKE) && $(MAKE) test + mkdir -p $(builddir)/release + cd $(builddir)/release && cmake -D BUILD_TESTS=ON -D CMAKE_BUILD_TYPE=release $(topdir) && $(MAKE) && $(MAKE) test release-all: - mkdir -p build/release - cd build/release && cmake -D BUILD_TESTS=ON -D CMAKE_BUILD_TYPE=release ../.. && $(MAKE) + mkdir -p $(builddir)/release + cd $(builddir)/release && cmake -D BUILD_TESTS=ON -D CMAKE_BUILD_TYPE=release $(topdir) && $(MAKE) release-static: - mkdir -p build/release - cd build/release && cmake -D STATIC=ON -D ARCH="x86-64" -D BUILD_64=ON -D CMAKE_BUILD_TYPE=release ../.. && $(MAKE) + mkdir -p $(builddir)/release + cd $(builddir)/release && cmake -D STATIC=ON -D ARCH="x86-64" -D BUILD_64=ON -D CMAKE_BUILD_TYPE=release $(topdir) && $(MAKE) coverage: - mkdir -p build/debug - cd build/debug && cmake -D BUILD_TESTS=ON -D CMAKE_BUILD_TYPE=Debug -D COVERAGE=ON ../.. && $(MAKE) && $(MAKE) test + mkdir -p $(builddir)/debug + cd $(builddir)/debug && cmake -D BUILD_TESTS=ON -D CMAKE_BUILD_TYPE=Debug -D COVERAGE=ON $(topdir) && $(MAKE) && $(MAKE) test # Targets for specific prebuilt builds which will be advertised for updates by their build tag release-static-linux-armv6: - mkdir -p build/release - cd build/release && cmake -D BUILD_TESTS=OFF -D ARCH="armv6zk" -D STATIC=ON -D BUILD_64=OFF -D CMAKE_BUILD_TYPE=release -D BUILD_TAG="linux-armv6" ../.. && $(MAKE) + mkdir -p $(builddir)/release + cd $(builddir)/release && cmake -D BUILD_TESTS=OFF -D ARCH="armv6zk" -D STATIC=ON -D BUILD_64=OFF -D CMAKE_BUILD_TYPE=release -D BUILD_TAG="linux-armv6" $(topdir) && $(MAKE) release-static-linux-armv7: - mkdir -p build/release - cd build/release && cmake -D BUILD_TESTS=OFF -D ARCH="armv7-a" -D STATIC=ON -D BUILD_64=OFF -D CMAKE_BUILD_TYPE=release -D BUILD_TAG="linux-armv7" ../.. && $(MAKE) + mkdir -p $(builddir)/release + cd $(builddir)/release && cmake -D BUILD_TESTS=OFF -D ARCH="armv7-a" -D STATIC=ON -D BUILD_64=OFF -D CMAKE_BUILD_TYPE=release -D BUILD_TAG="linux-armv7" $(topdir) && $(MAKE) + +release-static-android-armv7: + mkdir -p $(builddir)/release/translations + cd $(builddir)/release/translations && cmake ../../../translations && $(MAKE) + cd $(builddir)/release && CC=arm-linux-androideabi-clang CXX=arm-linux-androideabi-clang++ cmake -D BUILD_TESTS=OFF -D ARCH="armv7-a" -D STATIC=ON -D BUILD_64=OFF -D CMAKE_BUILD_TYPE=release -D ANDROID=true -D INSTALL_VENDORED_LIBUNBOUND=ON -D BUILD_TAG="android-armv7" -D CMAKE_SYSTEM_NAME="Android" -D CMAKE_ANDROID_STANDALONE_TOOLCHAIN="${ANDROID_STANDALONE_TOOLCHAIN_PATH}" -D CMAKE_ANDROID_ARM_MODE=ON -D CMAKE_ANDROID_ARCH_ABI="armeabi-v7a" ../.. && $(MAKE) -release-static-android: - mkdir -p build/release - cd build/release && cmake -D BUILD_TESTS=OFF -D ARCH="armv7-a" -D STATIC=ON -D BUILD_64=OFF -D CMAKE_BUILD_TYPE=release -D ANDROID=true -D INSTALL_VENDORED_LIBUNBOUND=ON -D BUILD_TAG="android" ../.. && $(MAKE) +release-static-android-armv8: + mkdir -p $(builddir)/release/translations + cd $(builddir)/release/translations && cmake ../../../translations && $(MAKE) + cd $(builddir)/release && CC=aarch64-linux-android-clang CXX=aarch64-linux-android-clang++ cmake -D BUILD_TESTS=OFF -D ARCH="armv8-a" -D STATIC=ON -D BUILD_64=ON -D CMAKE_BUILD_TYPE=release -D ANDROID=true -D INSTALL_VENDORED_LIBUNBOUND=ON -D BUILD_TAG="android-armv8" -D CMAKE_SYSTEM_NAME="Android" -D CMAKE_ANDROID_STANDALONE_TOOLCHAIN="${ANDROID_STANDALONE_TOOLCHAIN_PATH}" -D CMAKE_ANDROID_ARCH_ABI="arm64-v8a" ../.. && $(MAKE) release-static-linux-armv8: - mkdir -p build/release - cd build/release && cmake -D BUILD_TESTS=OFF -D ARCH="armv8-a" -D STATIC=ON -D BUILD_64=ON -D CMAKE_BUILD_TYPE=release -D BUILD_TAG="linux-armv8" ../.. && $(MAKE) + mkdir -p $(builddir)/release + cd $(builddir)/release && cmake -D BUILD_TESTS=OFF -D ARCH="armv8-a" -D STATIC=ON -D BUILD_64=ON -D CMAKE_BUILD_TYPE=release -D BUILD_TAG="linux-armv8" $(topdir) && $(MAKE) release-static-linux-x86_64: - mkdir -p build/release - cd build/release && cmake -D STATIC=ON -D ARCH="x86-64" -D BUILD_64=ON -D CMAKE_BUILD_TYPE=release -D BUILD_TAG="linux-x64" ../.. && $(MAKE) + mkdir -p $(builddir)/release + cd $(builddir)/release && cmake -D STATIC=ON -D ARCH="x86-64" -D BUILD_64=ON -D CMAKE_BUILD_TYPE=release -D BUILD_TAG="linux-x64" $(topdir) && $(MAKE) release-static-freebsd-x86_64: - mkdir -p build/release - cd build/release && cmake -D STATIC=ON -D ARCH="x86-64" -D BUILD_64=ON -D CMAKE_BUILD_TYPE=release -D BUILD_TAG="freebsd-x64" ../.. && $(MAKE) + mkdir -p $(builddir)/release + cd $(builddir)/release && cmake -D STATIC=ON -D ARCH="x86-64" -D BUILD_64=ON -D CMAKE_BUILD_TYPE=release -D BUILD_TAG="freebsd-x64" $(topdir) && $(MAKE) release-static-mac-x86_64: - mkdir -p build/release - cd build/release && cmake -D STATIC=ON -D ARCH="x86-64" -D BUILD_64=ON -D CMAKE_BUILD_TYPE=release -D BUILD_TAG="mac-x64" ../.. && $(MAKE) + mkdir -p $(builddir)/release + cd $(builddir)/release && cmake -D STATIC=ON -D ARCH="x86-64" -D BUILD_64=ON -D CMAKE_BUILD_TYPE=release -D BUILD_TAG="mac-x64" $(topdir) && $(MAKE) release-static-linux-i686: - mkdir -p build/release - cd build/release && cmake -D STATIC=ON -D ARCH="i686" -D BUILD_64=OFF -D CMAKE_BUILD_TYPE=release -D BUILD_TAG="linux-x86" ../.. && $(MAKE) + mkdir -p $(builddir)/release + cd $(builddir)/release && cmake -D STATIC=ON -D ARCH="i686" -D BUILD_64=OFF -D CMAKE_BUILD_TYPE=release -D BUILD_TAG="linux-x86" $(topdir) && $(MAKE) release-static-win64: - mkdir -p build/release - cd build/release && cmake -G "MSYS Makefiles" -D STATIC=ON -D ARCH="x86-64" -D BUILD_64=ON -D CMAKE_BUILD_TYPE=Release -D BUILD_TAG="win-x64" -D CMAKE_TOOLCHAIN_FILE=../../cmake/64-bit-toolchain.cmake -D MSYS2_FOLDER=c:/msys64 ../.. && $(MAKE) + mkdir -p $(builddir)/release + cd $(builddir)/release && cmake -G "MSYS Makefiles" -D STATIC=ON -D ARCH="x86-64" -D BUILD_64=ON -D CMAKE_BUILD_TYPE=Release -D BUILD_TAG="win-x64" -D CMAKE_TOOLCHAIN_FILE=$(topdir)/cmake/64-bit-toolchain.cmake -D MSYS2_FOLDER=c:/msys64 $(topdir) && $(MAKE) release-static-win32: - mkdir -p build/release - cd build/release && cmake -G "MSYS Makefiles" -D STATIC=ON -D ARCH="i686" -D BUILD_64=OFF -D CMAKE_BUILD_TYPE=Release -D BUILD_TAG="win-x32" -D CMAKE_TOOLCHAIN_FILE=../../cmake/32-bit-toolchain.cmake -D MSYS2_FOLDER=c:/msys32 ../.. && $(MAKE) + mkdir -p $(builddir)/release + cd $(builddir)/release && cmake -G "MSYS Makefiles" -D STATIC=ON -D ARCH="i686" -D BUILD_64=OFF -D CMAKE_BUILD_TYPE=Release -D BUILD_TAG="win-x32" -D CMAKE_TOOLCHAIN_FILE=$(topdir)/cmake/32-bit-toolchain.cmake -D MSYS2_FOLDER=c:/msys32 $(topdir) && $(MAKE) fuzz: - mkdir -p build/fuzz - cd build/fuzz && cmake -D STATIC=ON -D SANITIZE=ON -D BUILD_TESTS=ON -D USE_LTO=OFF -D CMAKE_C_COMPILER=afl-gcc -D CMAKE_CXX_COMPILER=afl-g++ -D ARCH="x86-64" -D CMAKE_BUILD_TYPE=fuzz -D BUILD_TAG="linux-x64" ../.. && $(MAKE) + mkdir -p $(builddir)/fuzz + cd $(builddir)/fuzz && cmake -D STATIC=ON -D SANITIZE=ON -D BUILD_TESTS=ON -D USE_LTO=OFF -D CMAKE_C_COMPILER=afl-gcc -D CMAKE_CXX_COMPILER=afl-g++ -D ARCH="x86-64" -D CMAKE_BUILD_TYPE=fuzz -D BUILD_TAG="linux-x64" $(topdir) && $(MAKE) clean: + @echo "WARNING: Back-up your wallet if it exists within ./"$(deldirs)"!" ; \ + read -r -p "This will destroy the build directory, continue (y/N)?: " CONTINUE; \ + [ $$CONTINUE = "y" ] || [ $$CONTINUE = "Y" ] || (echo "Exiting."; exit 1;) + rm -rf $(deldirs) + +clean-all: @echo "WARNING: Back-up your wallet if it exists within ./build!" ; \ - read -r -p "This will destroy the build directory, continue (y/N)?: " CONTINUE; \ + read -r -p "This will destroy all build directories, continue (y/N)?: " CONTINUE; \ [ $$CONTINUE = "y" ] || [ $$CONTINUE = "Y" ] || (echo "Exiting."; exit 1;) - rm -rf build + rm -rf ./build tags: ctags -R --sort=1 --c++-kinds=+p --fields=+iaS --extra=+q --language-force=C++ src contrib tests/gtest diff --git a/README.i18n.md b/README.i18n.md index b0379bfb3a3..5df27762454 100644 --- a/README.i18n.md +++ b/README.i18n.md @@ -3,11 +3,11 @@ Monero daemon internationalization The Monero command line tools can be translated in various languages. If you wish to contribute and need help/support, contact the [Monero Localization Workgroup on Taiga](https://taiga.getmonero.org/project/erciccione-monero-localization/) or come chat on `#monero-translations` (Freenode/IRC, riot/matrix, MatterMost) -In order to use the same translation workflow as the [Monero Core GUI](https://github.com/monero-project/monero-core), they use Qt Linguist translation files. However, to avoid the dependencies on Qt this normally implies, they use a custom loader to read those files at runtime. +In order to use the same translation workflow as the [Monero Core GUI](https://github.com/monero-project/monero-gui), they use Qt Linguist translation files. However, to avoid the dependencies on Qt this normally implies, they use a custom loader to read those files at runtime. ### Tools for translators -In order to create, update or build translations files, you need to have Qt tools installed. For translating, you need either the **Qt Linguist GUI** ([part of QT Creator](https://www.qt.io/download-open-source/#allDownloadsDiv-9) or a [3rd-party standalone version](https://github.com/lelegard/qtlinguist-installers/releases)), or another tool that supports Qt ts files, such as Transifex. The files are XML, so they can be edited in any plain text editor if needed. +In order to create, update or build translations files, you need to have Qt tools installed. For translating, you need either the **Qt Linguist GUI** ([part of Qt Creator](https://www.qt.io/download) or a [3rd-party standalone version](https://github.com/lelegard/qtlinguist-installers/releases)), or another tool that supports Qt ts files, such as Transifex. The files are XML, so they can be edited in any plain text editor if needed. ### Creating / modifying translations @@ -15,23 +15,33 @@ You do not need anything from Qt in order to use the final translations. To update ts files after changing source code: - ./utils/translations/update-translations.sh +```bash +./utils/translations/update-translations.sh +``` To add a new language, eg Spanish (ISO code es): - cp translations/monero.ts translations/monero_es.ts +```bash +cp translations/monero.ts translations/monero_es.ts +``` To edit translations for Spanish: - linguist translations/monero_es.ts +```bash +linguist translations/monero_es.ts +``` To build translations after modifying them: - ./utils/translations/build-translations.sh +```bash +./utils/translations/build-translations.sh +``` To test a translation: - LANG=es ./build/release/bin/monero-wallet-cli +```bash +LANG=es ./build/release/bin/monero-wallet-cli +``` To add new translatable strings in the source code: @@ -39,6 +49,8 @@ Use the `tr(string)` function if possible. If the code is in a class, and this c If you're getting messages of the form: - Class 'cryptonote::simple_wallet' lacks Q_OBJECT macro +``` +Class 'cryptonote::simple_wallet' lacks Q_OBJECT macro +``` all is fine, we don't actually need that here. diff --git a/README.md b/README.md index 4c3e9799a5f..10c0daa2805 100644 --- a/README.md +++ b/README.md @@ -1,36 +1,75 @@ # Monero -Copyright (c) 2014-2018 The Monero Project. +Copyright (c) 2014-2019 The Monero Project. Portions Copyright (c) 2012-2013 The Cryptonote developers. +## Table of Contents + + - [Development resources](#development-resources) + - [Vulnerability response](#vulnerability-response) + - [Research](#research) + - [Announcements](#announcements) + - [Translations](#translations) + - [Build](#build) + - [IMPORTANT](#important) + - [Coverage](#coverage) + - [Introduction](#introduction) + - [About this project](#about-this-project) + - [Supporting the project](#supporting-the-project) + - [License](#license) + - [Contributing](#contributing) + - [Scheduled software upgrades](#scheduled-software-upgrades) + - [Release staging schedule and protocol](#release-staging-schedule-and-protocol) + - [Compiling Monero from source](#compiling-monero-from-source) + - [Dependencies](#dependencies) + ## Development resources - Web: [getmonero.org](https://getmonero.org) - Forum: [forum.getmonero.org](https://forum.getmonero.org) - Mail: [dev@getmonero.org](mailto:dev@getmonero.org) - GitHub: [https://github.com/monero-project/monero](https://github.com/monero-project/monero) -- IRC: [#monero-dev on Freenode](http://webchat.freenode.net/?randomnick=1&channels=%23monero-dev&prompt=1&uio=d4) +- IRC: [#monero-dev on Freenode](https://webchat.freenode.net/?randomnick=1&channels=%23monero-dev&prompt=1&uio=d4) ## Vulnerability response - Our [Vulnerability Response Process](https://github.com/monero-project/meta/blob/master/VULNERABILITY_RESPONSE_PROCESS.md) encourages responsible disclosure - We are also available via [HackerOne](https://hackerone.com/monero) +## Research + +The [Monero Research Lab](https://src.getmonero.org/resources/research-lab/) is an open forum where the community coordinates research into Monero cryptography, protocols, fungibility, analysis, and more. We welcome collaboration and contributions from outside researchers! Because not all Lab work and publications are distributed as traditional preprints or articles, they may be easy to miss if you are conducting literature reviews for your own Monero research. You are encouraged to get in touch with our researchers if you have questions, wish to collaborate, or would like guidance to help avoid unnecessarily duplicating earlier or known work. + +Our researchers are available on IRC in [#monero-research-lab on Freenode](https://webchat.freenode.net/?randomnick=1&channels=%23monero-research-lab&prompt=1&uio=d4) or by email: + +- Sarang Noether, Ph.D.: [sarang@getmonero.org](mailto:sarang@getmonero.org) or [sarang.noether@protonmail.com](mailto:sarang.noether@protonmail.com); [research repository](https://github.com/SarangNoether/research-lab) +- Surae Noether (Brandon Goodell), Ph.D.: [surae@getmonero.org](mailto:surae@getmonero.org) or [surae.noether@protonmail.com](mailto:surae.noether@protonmail.com); [research repository](https://github.com/b-g-goodell/research-lab) + ## Announcements - You can subscribe to an [announcement listserv](https://lists.getmonero.org) to get critical announcements from the Monero core team. The announcement list can be very helpful for knowing when software updates are needed. +## Translations +The CLI wallet is available in different languages. If you want to help translate it, see our self-hosted localization platform, Pootle, on [translate.getmonero.org](https://translate.getmonero.org/projects/CLI/). Every translation *must* be uploaded on the platform, pull requests directly editing the code in this repository will be closed. If you need help with Pootle, you can find a guide with screenshots [here](https://github.com/monero-ecosystem/monero-translations/blob/master/pootle.md). +  + +If you need help/support/info about translations, contact the localization workgroup. You can find the complete list of contacts on the repository of the workgroup: [monero-translations](https://github.com/monero-ecosystem/monero-translations#contacts). + ## Build +### IMPORTANT + +These builds are of the master branch, which is used for active development and can be either unstable or incompatible with release software. Please compile release branches. + | Operating System | Processor | Status | | --------------------- | -------- |--------| | Ubuntu 16.04 | i686 | [![Ubuntu 16.04 i686](https://build.getmonero.org/png?builder=monero-static-ubuntu-i686)](https://build.getmonero.org/builders/monero-static-ubuntu-i686) | Ubuntu 16.04 | amd64 | [![Ubuntu 16.04 amd64](https://build.getmonero.org/png?builder=monero-static-ubuntu-amd64)](https://build.getmonero.org/builders/monero-static-ubuntu-amd64) | Ubuntu 16.04 | armv7 | [![Ubuntu 16.04 armv7](https://build.getmonero.org/png?builder=monero-static-ubuntu-arm7)](https://build.getmonero.org/builders/monero-static-ubuntu-arm7) | Debian Stable | armv8 | [![Debian armv8](https://build.getmonero.org/png?builder=monero-static-debian-armv8)](https://build.getmonero.org/builders/monero-static-debian-armv8) -| OSX 10.10 | amd64 | [![OSX 10.10 amd64](https://build.getmonero.org/png?builder=monero-static-osx-10.10)](https://build.getmonero.org/builders/monero-static-osx-10.10) -| OSX 10.11 | amd64 | [![OSX 10.11 amd64](https://build.getmonero.org/png?builder=monero-static-osx-10.11)](https://build.getmonero.org/builders/monero-static-osx-10.11) -| OSX 10.12 | amd64 | [![OSX 10.12 amd64](https://build.getmonero.org/png?builder=monero-static-osx-10.12)](https://build.getmonero.org/builders/monero-static-osx-10.12) +| macOS 10.11 | amd64 | [![macOS 10.11 amd64](https://build.getmonero.org/png?builder=monero-static-osx-10.11)](https://build.getmonero.org/builders/monero-static-osx-10.11) +| macOS 10.12 | amd64 | [![macOS 10.12 amd64](https://build.getmonero.org/png?builder=monero-static-osx-10.12)](https://build.getmonero.org/builders/monero-static-osx-10.12) +| macOS 10.13 | amd64 | [![macOS 10.13 amd64](https://build.getmonero.org/png?builder=monero-static-osx-10.13)](https://build.getmonero.org/builders/monero-static-osx-10.13) | FreeBSD 11 | amd64 | [![FreeBSD 11 amd64](https://build.getmonero.org/png?builder=monero-static-freebsd64)](https://build.getmonero.org/builders/monero-static-freebsd64) | DragonFly BSD 4.6 | amd64 | [![DragonFly BSD amd64](https://build.getmonero.org/png?builder=monero-static-dragonflybsd-amd64)](https://build.getmonero.org/builders/monero-static-dragonflybsd-amd64) | Windows (MSYS2/MinGW) | i686 | [![Windows (MSYS2/MinGW) i686](https://build.getmonero.org/png?builder=monero-static-win32)](https://build.getmonero.org/builders/monero-static-win32) @@ -54,6 +93,8 @@ Monero is a private, secure, untraceable, decentralised digital currency. You ar **Untraceability:** By taking advantage of ring signatures, a special property of a certain type of cryptography, Monero is able to ensure that transactions are not only untraceable, but have an optional measure of ambiguity that ensures that transactions cannot easily be tied back to an individual user or computer. +**Decentralization:** The utility of monero depends on its decentralised peer-to-peer consensus network - anyone should be able to run the monero software, validate the integrity of the blockchain, and participate in all aspects of the monero network using consumer-grade commodity hardware. Decentralization of the monero network is maintained by software development that minimizes the costs of running the monero software and inhibits the proliferation of specialized, non-commodity hardware. + ## About this project This is the core implementation of Monero. It is open source and completely free to use without restrictions, except for those specified in the license agreement below. There are no restrictions on anyone creating an alternative implementation of Monero that uses the protocol and network in a compatible manner. @@ -73,12 +114,12 @@ The Bitcoin donation address is: `1KTexdemPdxSBcG55heUuTjDRYqbC5ZL8H` Core development funding and/or some supporting services are also graciously provided by sponsors: [](https://mymonero.com) -[](http://kitware.com) -[](http://dome9.com) -[](http://araxis.com) -[](http://www.jetbrains.com/) -[](http://www.navicat.com/) -[](http://www.symas.com/) +[](https://kitware.com) +[](https://dome9.com) +[](https://araxis.com) +[](https://www.jetbrains.com/) +[](https://www.navicat.com/) +[](https://www.symas.com/) There are also several mining pools that kindly donate a portion of their fees, [a list of them can be found on our Bitcointalk post](https://bitcointalk.org/index.php?topic=583449.0). @@ -93,67 +134,28 @@ If you want to help out, see [CONTRIBUTING](CONTRIBUTING.md) for a set of guidel ## Scheduled software upgrades Monero uses a fixed-schedule software upgrade (hard fork) mechanism to implement new features. This means that users of Monero (end users and service providers) should run current versions and upgrade their software on a regular schedule. Software upgrades occur during the months of April and October. The required software for these upgrades will be available prior to the scheduled date. Please check the repository prior to this date for the proper Monero software version. Below is the historical schedule and the projected schedule for the next upgrade. -Dates are provided in the format YYYY-MM-DD. +Dates are provided in the format YYYY-MM-DD. -| Software upgrade block height | Date | Fork version | Minimum Monero version | Recommended Monero version | Details | +| Software upgrade block height | Date | Fork version | Minimum Monero version | Recommended Monero version | Details | | ------------------------------ | -----------| ----------------- | ---------------------- | -------------------------- | ---------------------------------------------------------------------------------- | | 1009827 | 2016-03-22 | v2 | v0.9.4 | v0.9.4 | Allow only >= ringsize 3, blocktime = 120 seconds, fee-free blocksize 60 kb | | 1141317 | 2016-09-21 | v3 | v0.9.4 | v0.10.0 | Splits coinbase into denominations | | 1220516 | 2017-01-05 | v4 | v0.10.1 | v0.10.2.1 | Allow normal and RingCT transactions | | 1288616 | 2017-04-15 | v5 | v0.10.3.0 | v0.10.3.1 | Adjusted minimum blocksize and fee algorithm | | 1400000 | 2017-09-16 | v6 | v0.11.0.0 | v0.11.0.0 | Allow only RingCT transactions, allow only >= ringsize 5 | -| 1546000 | 2018-04-06 | v7 | v0.12.0.0 | v0.12.0.0 | Cryptonight variant 1, ringsize >= 7, sorted inputs -| XXXXXXX | 2018-10-XX | XX | XXXXXXXXX | XXXXXXXXX | X +| 1546000 | 2018-04-06 | v7 | v0.12.0.0 | v0.12.3.0 | Cryptonight variant 1, ringsize >= 7, sorted inputs +| 1685555 | 2018-10-18 | v8 | v0.13.0.0 | v0.13.0.4 | max transaction size at half the penalty free block size, bulletproofs enabled, cryptonight variant 2, fixed ringsize [11](https://youtu.be/KOO5S4vxi0o) +| 1686275 | 2018-10-19 | v9 | v0.13.0.0 | v0.13.0.4 | bulletproofs required +| 1788000 | 2019-03-09 | v10 | v0.14.0.0 | v0.14.1.2 | New PoW based on Cryptonight-R, new block weight algorithm, slightly more efficient RingCT format +| 1788720 | 2019-03-10 | v11 | v0.14.0.0 | v0.14.1.2 | forbid old RingCT transaction format +| XXXXXXX | 2019-10-XX | XX | XXXXXXXXX | XXXXXXXXX | X X's indicate that these details have not been determined as of commit date. ## Release staging schedule and protocol -Approximately three months prior to a scheduled software upgrade, a branch from Master will be created with the new release version tag. Pull requests that address bugs should then be made to both Master and the new release branch. Pull requests that require extensive review and testing (generally, optimizations and new features) should *not* be made to the release branch. - -## Installing Monero from a package - -Packages are available for - -* Ubuntu and [snap supported](https://snapcraft.io/docs/core/install) systems, via a community contributed build. - - snap install monero --beta - -Installing a snap is very quick. Snaps are secure. They are isolated with all of their dependencies. Snaps also auto update when a new version is released. - -* Arch Linux (via [AUR](https://aur.archlinux.org/)): - - Stable release: [`monero`](https://aur.archlinux.org/packages/monero) - - Bleeding edge: [`monero-git`](https://aur.archlinux.org/packages/monero-git) - -* Void Linux: - - xbps-install -S monero - -* GuixSD - - guix package -i monero - -* OS X via [Homebrew](http://brew.sh) - - brew tap sammy007/cryptonight - brew install monero --build-from-source - -* Docker - - # Build using all available cores - docker build -t monero . - - # or build using a specific number of cores (reduce RAM requirement) - docker build --build-arg NPROC=1 -t monero . - - # either run in foreground - docker run -it -v /monero/chain:/root/.bitmonero -v /monero/wallet:/wallet -p 18080:18080 monero - - # or in background - docker run -it -d -v /monero/chain:/root/.bitmonero -v /monero/wallet:/wallet -p 18080:18080 monero - -Packaging for your favorite distribution would be a welcome contribution! +Approximately three months prior to a scheduled software upgrade, a branch from Master will be created with the new release version tag. Pull requests that address bugs should then be made to both Master and the new release branch. Pull requests that require extensive review and testing (generally, optimizations and new features) should *not* be made to the release branch. ## Compiling Monero from source @@ -171,26 +173,38 @@ library archives (`.a`). | Dep | Min. version | Vendored | Debian/Ubuntu pkg | Arch pkg | Fedora | Optional | Purpose | | ------------ | ------------- | -------- | ------------------ | ------------ | ----------------- | -------- | -------------- | | GCC | 4.7.3 | NO | `build-essential` | `base-devel` | `gcc` | NO | | -| CMake | 3.0.0 | NO | `cmake` | `cmake` | `cmake` | NO | | +| CMake | 3.5 | NO | `cmake` | `cmake` | `cmake` | NO | | | pkg-config | any | NO | `pkg-config` | `base-devel` | `pkgconf` | NO | | | Boost | 1.58 | NO | `libboost-all-dev` | `boost` | `boost-devel` | NO | C++ libraries | | OpenSSL | basically any | NO | `libssl-dev` | `openssl` | `openssl-devel` | NO | sha256 sum | | libzmq | 3.0.0 | NO | `libzmq3-dev` | `zeromq` | `cppzmq-devel` | NO | ZeroMQ library | +| OpenPGM | ? | NO | `libpgm-dev` | `libpgm` | `openpgm-devel` | NO | For ZeroMQ | +| libnorm[2] | ? | NO | `libnorm-dev` | | ` | YES | For ZeroMQ | | libunbound | 1.4.16 | YES | `libunbound-dev` | `unbound` | `unbound-devel` | NO | DNS resolver | -| libsodium | ? | NO | `libsodium-dev` | ? | `libsodium-devel` | NO | libsodium | -| libminiupnpc | 2.0 | YES | `libminiupnpc-dev` | `miniupnpc` | `miniupnpc-devel` | YES | NAT punching | +| libsodium | ? | NO | `libsodium-dev` | `libsodium` | `libsodium-devel` | NO | cryptography | | libunwind | any | NO | `libunwind8-dev` | `libunwind` | `libunwind-devel` | YES | Stack traces | | liblzma | any | NO | `liblzma-dev` | `xz` | `xz-devel` | YES | For libunwind | | libreadline | 6.3.0 | NO | `libreadline6-dev` | `readline` | `readline-devel` | YES | Input editing | | ldns | 1.6.17 | NO | `libldns-dev` | `ldns` | `ldns-devel` | YES | SSL toolkit | | expat | 1.1 | NO | `libexpat1-dev` | `expat` | `expat-devel` | YES | XML parsing | -| GTest | 1.5 | YES | `libgtest-dev`^ | `gtest` | `gtest-devel` | YES | Test suite | +| GTest | 1.5 | YES | `libgtest-dev`[1] | `gtest` | `gtest-devel` | YES | Test suite | | Doxygen | any | NO | `doxygen` | `doxygen` | `doxygen` | YES | Documentation | | Graphviz | any | NO | `graphviz` | `graphviz` | `graphviz` | YES | Documentation | -[^] On Debian/Ubuntu `libgtest-dev` only includes sources and headers. You must +[1] On Debian/Ubuntu `libgtest-dev` only includes sources and headers. You must build the library binary manually. This can be done with the following command ```sudo apt-get install libgtest-dev && cd /usr/src/gtest && sudo cmake . && sudo make && sudo mv libg* /usr/lib/ ``` +[2] libnorm-dev is needed if your zmq library was built with libnorm, and not needed otherwise + +Install all dependencies at once on Debian/Ubuntu: + +``` sudo apt update && sudo apt install build-essential cmake pkg-config libboost-all-dev libssl-dev libzmq3-dev libunbound-dev libsodium-dev libunwind8-dev liblzma-dev libreadline6-dev libldns-dev libexpat1-dev doxygen graphviz libpgm-dev``` + +Install all dependencies at once on macOS with the provided Brewfile: +``` brew update && brew bundle --file=contrib/brew/Brewfile ``` + +FreeBSD one liner for required to build dependencies +```pkg install git gmake cmake pkgconf boost-libs cppzmq libsodium``` ### Cloning the repository @@ -207,22 +221,31 @@ If you already have a repo cloned, initialize and update: Monero uses the CMake build system and a top-level [Makefile](Makefile) that invokes cmake commands as needed. -#### On Linux and OS X +#### On Linux and macOS * Install the dependencies -* Change to the root of the source code directory and build: +* Change to the root of the source code directory, change to the most recent release branch, and build: - cd monero - make + ```bash + cd monero + git checkout release-v0.14 + make + ``` *Optional*: If your machine has several cores and enough memory, enable parallel build by running `make -j` instead of `make`. For this to be worthwhile, the machine should have one core and about 2GB of RAM available per thread. - *Note*: If cmake can not find zmq.hpp file on OS X, installing `zmq.hpp` from + *Note*: If cmake can not find zmq.hpp file on macOS, installing `zmq.hpp` from https://github.com/zeromq/cppzmq to `/usr/local/include` should fix that error. + *Note*: The instructions above will compile the most stable release of the + Monero software. If you would like to use and test the most recent software, + use ```git checkout master```. The master branch may contain updates that are + both unstable and incompatible with release software, though testing is always + encouraged. + * The resulting executables can be found in `build/release/bin` * Add `PATH="$PATH:$HOME/monero/build/release/bin"` to `.profile` @@ -231,49 +254,65 @@ invokes cmake commands as needed. * **Optional**: build and run the test suite to verify the binaries: - make release-test + ```bash + make release-test + ``` *NOTE*: `core_tests` test may take a few hours to complete. * **Optional**: to build binaries suitable for debugging: - make debug + ```bash + make debug + ``` * **Optional**: to build statically-linked binaries: - make release-static + ```bash + make release-static + ``` Dependencies need to be built with -fPIC. Static libraries usually aren't, so you may have to build them yourself with -fPIC. Refer to their documentation for how to build them. * **Optional**: build documentation in `doc/html` (omit `HAVE_DOT=YES` if `graphviz` is not installed): - HAVE_DOT=YES doxygen Doxyfile + ```bash + HAVE_DOT=YES doxygen Doxyfile + ``` #### On the Raspberry Pi -Tested on a Raspberry Pi Zero with a clean install of minimal Raspbian Stretch (2017-09-07 or later) from https://www.raspberrypi.org/downloads/raspbian/. If you are using Raspian Jessie, [please see note in the following section](#note-for-raspbian-jessie-users). +Tested on a Raspberry Pi Zero with a clean install of minimal Raspbian Stretch (2017-09-07 or later) from https://www.raspberrypi.org/downloads/raspbian/. If you are using Raspian Jessie, [please see note in the following section](#note-for-raspbian-jessie-users). * `apt-get update && apt-get upgrade` to install all of the latest software * Install the dependencies for Monero from the 'Debian' column in the table above. * Increase the system swap size: -``` - sudo /etc/init.d/dphys-swapfile stop - sudo nano /etc/dphys-swapfile - CONF_SWAPSIZE=1024 - sudo /etc/init.d/dphys-swapfile start -``` -* Clone monero and checkout most recent release version: -``` - git clone https://github.com/monero-project/monero.git - cd monero - git checkout tags/v0.11.1.0 -``` + + ```bash + sudo /etc/init.d/dphys-swapfile stop + sudo nano /etc/dphys-swapfile + CONF_SWAPSIZE=2048 + sudo /etc/init.d/dphys-swapfile start + ``` + +* If using an external hard disk without an external power supply, ensure it gets enough power to avoid hardware issues when syncing, by adding the line "max_usb_current=1" to /boot/config.txt + +* Clone monero and checkout the most recent release version: + + ```bash + git clone https://github.com/monero-project/monero.git + cd monero + git checkout tags/v0.14.1.2 + ``` + * Build: -``` - make release -``` + + ```bash + make release + ``` + * Wait 4-6 hours * The resulting executables can be found in `build/release/bin` @@ -290,28 +329,33 @@ If you are using the older Raspbian Jessie image, compiling Monero is a bit more * As before, `apt-get update && apt-get upgrade` to install all of the latest software, and increase the system swap size -``` - sudo /etc/init.d/dphys-swapfile stop - sudo nano /etc/dphys-swapfile - CONF_SWAPSIZE=1024 - sudo /etc/init.d/dphys-swapfile start -``` + ```bash + sudo /etc/init.d/dphys-swapfile stop + sudo nano /etc/dphys-swapfile + CONF_SWAPSIZE=2048 + sudo /etc/init.d/dphys-swapfile start + ``` + * Then, install the dependencies for Monero except `libunwind` and `libboost-all-dev` * Install the latest version of boost (this may first require invoking `apt-get remove --purge libboost*` to remove a previous version if you're not using a clean install): -``` - cd - wget https://sourceforge.net/projects/boost/files/boost/1.64.0/boost_1_64_0.tar.bz2 - tar xvfo boost_1_64_0.tar.bz2 - cd boost_1_64_0 - ./bootstrap.sh - sudo ./b2 -``` + + ```bash + cd + wget https://sourceforge.net/projects/boost/files/boost/1.64.0/boost_1_64_0.tar.bz2 + tar xvfo boost_1_64_0.tar.bz2 + cd boost_1_64_0 + ./bootstrap.sh + sudo ./b2 + ``` + * Wait ~8 hours -``` - sudo ./bjam install -``` + + ```bash + sudo ./bjam cxxflags=-fPIC cflags=-fPIC -a install + ``` + * Wait ~4 hours * From here, follow the [general Raspberry Pi instructions](#on-the-raspberry-pi) from the "Clone monero and checkout most recent release version" step. @@ -319,55 +363,101 @@ If you are using the older Raspbian Jessie image, compiling Monero is a bit more #### On Windows: Binaries for Windows are built on Windows using the MinGW toolchain within -[MSYS2 environment](http://msys2.github.io). The MSYS2 environment emulates a +[MSYS2 environment](https://www.msys2.org). The MSYS2 environment emulates a POSIX system. The toolchain runs within the environment and *cross-compiles* binaries that can run outside of the environment as a regular Windows application. **Preparing the build environment** -* Download and install the [MSYS2 installer](http://msys2.github.io), either the 64-bit or the 32-bit package, depending on your system. +* Download and install the [MSYS2 installer](https://www.msys2.org), either the 64-bit or the 32-bit package, depending on your system. * Open the MSYS shell via the `MSYS2 Shell` shortcut * Update packages using pacman: - pacman -Syuu + ```bash + pacman -Syu + ``` * Exit the MSYS shell using Alt+F4 * Edit the properties for the `MSYS2 Shell` shortcut changing "msys2_shell.bat" to "msys2_shell.cmd -mingw64" for 64-bit builds or "msys2_shell.cmd -mingw32" for 32-bit builds * Restart MSYS shell via modified shortcut and update packages again using pacman: - pacman -Syuu + ```bash + pacman -Syu + ``` * Install dependencies: To build for 64-bit Windows: - pacman -S mingw-w64-x86_64-toolchain make mingw-w64-x86_64-cmake mingw-w64-x86_64-boost mingw-w64-x86_64-openssl mingw-w64-x86_64-zeromq mingw-w64-x86_64-libsodium + ```bash + pacman -S mingw-w64-x86_64-toolchain make mingw-w64-x86_64-cmake mingw-w64-x86_64-boost mingw-w64-x86_64-openssl mingw-w64-x86_64-zeromq mingw-w64-x86_64-libsodium mingw-w64-x86_64-hidapi + ``` To build for 32-bit Windows: - - pacman -S mingw-w64-i686-toolchain make mingw-w64-i686-cmake mingw-w64-i686-boost mingw-w64-i686-openssl mingw-w64-i686-zeromq mingw-w64-i686-libsodium + + ```bash + pacman -S mingw-w64-i686-toolchain make mingw-w64-i686-cmake mingw-w64-i686-boost mingw-w64-i686-openssl mingw-w64-i686-zeromq mingw-w64-i686-libsodium mingw-w64-i686-hidapi + ``` * Open the MingW shell via `MinGW-w64-Win64 Shell` shortcut on 64-bit Windows or `MinGW-w64-Win64 Shell` shortcut on 32-bit Windows. Note that if you are running 64-bit Windows, you will have both 64-bit and 32-bit MinGW shells. +**Cloning** + +* To git clone, run: + + ```bash + git clone --recursive https://github.com/monero-project/monero.git + ``` + **Building** +* Change to the cloned directory, run: + + ```bash + cd monero + ``` + +* If you would like a specific [version/tag](https://github.com/monero-project/monero/tags), do a git checkout for that version. eg. 'v0.14.1.2'. If you don't care about the version and just want binaries from master, skip this step: + + ```bash + git checkout v0.14.1.2 + ``` + * If you are on a 64-bit system, run: - make release-static-win64 + ```bash + make release-static-win64 + ``` * If you are on a 32-bit system, run: - make release-static-win32 + ```bash + make release-static-win32 + ``` * The resulting executables can be found in `build/release/bin` +* **Optional**: to build Windows binaries suitable for debugging on a 64-bit system, run: + + ```bash + make debug-static-win64 + ``` + +* **Optional**: to build Windows binaries suitable for debugging on a 32-bit system, run: + + ```bash + make debug-static-win32 + ``` + +* The resulting executables can be found in `build/debug/bin` + ### On FreeBSD: -The project can be built from scratch by following instructions for Linux above. If you are running monero in a jail you need to add the flag: `allow.sysvipc=1` to your jail configuration, otherwise lmdb will throw the error message: `Failed to open lmdb environment: Function not implemented`. +The project can be built from scratch by following instructions for Linux above(but use `gmake` instead of `make`). If you are running monero in a jail you need to add the flag: `allow.sysvipc=1` to your jail configuration, otherwise lmdb will throw the error message: `Failed to open lmdb environment: Function not implemented`. We expect to add Monero into the ports tree in the near future, which will aid in managing installations using ports or packages. @@ -377,7 +467,7 @@ We expect to add Monero into the ports tree in the near future, which will aid i This has been tested on OpenBSD 5.8. -You will need to add a few packages to your system. `pkg_add db cmake gcc gcc-libs g++ miniupnpc gtest`. +You will need to add a few packages to your system. `pkg_add db cmake gcc gcc-libs g++ gtest`. The doxygen and graphviz packages are optional and require the xbase set. @@ -388,9 +478,9 @@ You will have to add the serialization, date_time, and regex modules to Boost wh To build: `env CC=egcc CXX=eg++ CPP=ecpp DEVELOPER_LOCAL_TOOLS=1 BOOST_ROOT=/path/to/the/boost/you/built make release-static-64` -#### OpenBSD >= 6.2 +#### OpenBSD 6.2 and 6.3 -You will need to add a few packages to your system. `pkg_add cmake miniupnpc zeromq libiconv`. +You will need to add a few packages to your system. `pkg_add cmake zeromq libiconv`. The doxygen and graphviz packages are optional and require the xbase set. @@ -401,13 +491,13 @@ We assume you are compiling with a non-root user and you have `doas` enabled. Note: do not use the boost package provided by OpenBSD, as we are installing boost to `/usr/local`. -``` +```bash # Create boost building directory mkdir ~/boost cd ~/boost # Fetch boost source -ftp -o boost_1_64_0.tar.bz2 https://netcologne.dl.sourceforge.net/project/boost/boost/1.64.0/boost_1_64_0.tar.bz2 +ftp -o boost_1_64_0.tar.bz2 https://netcologne.dl.sourceforge.net/project/boost/boost/1.64.0/boost_1_64_0.tar.bz2 # MUST output: (SHA256) boost_1_64_0.tar.bz2: OK echo "7bcc5caace97baa948931d712ea5f37038dbb1c5d89b43ad4def4ed7cb683332 boost_1_64_0.tar.bz2" | sha256 -c @@ -433,13 +523,11 @@ echo 'using clang : : c++ : "-fvisibility=hidden -fPIC" "" doas ./b2 -d0 runtime-link=shared threadapi=pthread threading=multi link=static variant=release --layout=tagged --build-type=complete --user-config=user-config.jam -sNO_BZIP2=1 -sICONV_PATH=/usr/local --prefix=/usr/local install ``` -Build cppzmq - Build the cppzmq bindings. We assume you are compiling with a non-root user and you have `doas` enabled. -``` +```bash # Create cppzmq building directory mkdir ~/cppzmq cd ~/cppzmq @@ -459,27 +547,53 @@ cmake .. doas make install ``` -Build monero: `env DEVELOPER_LOCAL_TOOLS=1 BOOST_ROOT=/usr/local make release-static` +Build monero: +```bash +env DEVELOPER_LOCAL_TOOLS=1 BOOST_ROOT=/usr/local make release-static +``` + +#### OpenBSD >= 6.4 + +You will need to add a few packages to your system. `pkg_add cmake gmake zeromq cppzmq libiconv boost`. + +The doxygen and graphviz packages are optional and require the xbase set. + +Build monero: `env DEVELOPER_LOCAL_TOOLS=1 BOOST_ROOT=/usr/local gmake release-static` + +Note: you may encounter the following error, when compiling the latest version of monero as a normal user: + +``` +LLVM ERROR: out of memory +c++: error: unable to execute command: Abort trap (core dumped) +``` + +Then you need to increase the data ulimit size to 2GB and try again: `ulimit -d 2000000` ### On Solaris: The default Solaris linker can't be used, you have to install GNU ld, then run cmake manually with the path to your copy of GNU ld: - mkdir -p build/release - cd build/release - cmake -DCMAKE_LINKER=/path/to/ld -D CMAKE_BUILD_TYPE=Release ../.. - cd ../.. +```bash +mkdir -p build/release +cd build/release +cmake -DCMAKE_LINKER=/path/to/ld -D CMAKE_BUILD_TYPE=Release ../.. +cd ../.. +``` Then you can run make as usual. ### On Linux for Android (using docker): - # Build image (select android64.Dockerfile for aarch64) - cd utils/build_scripts/ && docker build -f android32.Dockerfile -t monero-android . - # Create container - docker create -it --name monero-android monero-android bash - # Get binaries - docker cp monero-android:/opt/android/monero/build/release/bin . +```bash +# Build image (for ARM 32-bit) +docker build -f utils/build_scripts/android32.Dockerfile -t monero-android . +# Build image (for ARM 64-bit) +docker build -f utils/build_scripts/android64.Dockerfile -t monero-android . +# Create container +docker create -it --name monero-android monero-android bash +# Get binaries +docker cp monero-android:/src/build/release/bin . +``` ### Building portable statically linked binaries @@ -493,13 +607,90 @@ By default, in either dynamically or statically linked builds, binaries target t * ```make release-static-win64``` builds binaries on 64-bit Windows portable across 64-bit Windows systems * ```make release-static-win32``` builds binaries on 64-bit or 32-bit Windows portable across 32-bit Windows systems +### Cross Compiling + +You can also cross-compile static binaries on Linux for Windows and macOS with the `depends` system. + +* ```make depends target=x86_64-linux-gnu``` for 64-bit linux binaries. +* ```make depends target=x86_64-w64-mingw32``` for 64-bit windows binaries. + * Requires: `python3 g++-mingw-w64-x86-64 wine1.6 bc` +* ```make depends target=x86_64-apple-darwin11``` for macOS binaries. + * Requires: `cmake imagemagick libcap-dev librsvg2-bin libz-dev libbz2-dev libtiff-tools python-dev` +* ```make depends target=i686-linux-gnu``` for 32-bit linux binaries. + * Requires: `g++-multilib bc` +* ```make depends target=i686-w64-mingw32``` for 32-bit windows binaries. + * Requires: `python3 g++-mingw-w64-i686` +* ```make depends target=arm-linux-gnueabihf``` for armv7 binaries. + * Requires: `g++-arm-linux-gnueabihf` +* ```make depends target=aarch64-linux-gnu``` for armv8 binaries. + * Requires: `g++-aarch64-linux-gnu` + +The required packages are the names for each toolchain on apt. Depending on your distro, they may have different names. + +Using `depends` might also be easier to compile Monero on Windows than using MSYS. Activate Windows Subsystem for Linux (WSL) with a distro (for example Ubuntu), install the apt build-essentials and follow the `depends` steps as depicted above. + +The produced binaries still link libc dynamically. If the binary is compiled on a current distribution, it might not run on an older distribution with an older installation of libc. Passing `-DBACKCOMPAT=ON` to cmake will make sure that the binary will run on systems having at least libc version 2.17. + +## Installing Monero from a package + +**DISCLAIMER: These packages are not part of this repository or maintained by this project's contributors, and as such, do not go through the same review process to ensure their trustworthiness and security.** + +Packages are available for + +* Ubuntu and [snap supported](https://snapcraft.io/docs/core/install) systems, via a community contributed build. + + ```bash + snap install monero --beta + ``` + +Installing a snap is very quick. Snaps are secure. They are isolated with all of their dependencies. Snaps also auto update when a new version is released. + +* Arch Linux (via [AUR](https://aur.archlinux.org/)): + - Stable release: [`monero`](https://aur.archlinux.org/packages/monero) + - Bleeding edge: [`monero-git`](https://aur.archlinux.org/packages/monero-git) + +* Void Linux: + + ```bash + xbps-install -S monero + ``` + +* GuixSD + + ```bash + guix package -i monero + ``` + +* Docker + + ```bash + # Build using all available cores + docker build -t monero . + + # or build using a specific number of cores (reduce RAM requirement) + docker build --build-arg NPROC=1 -t monero . + + # either run in foreground + docker run -it -v /monero/chain:/root/.bitmonero -v /monero/wallet:/wallet -p 18080:18080 monero + + # or in background + docker run -it -d -v /monero/chain:/root/.bitmonero -v /monero/wallet:/wallet -p 18080:18080 monero + ``` + +* The build needs 3 GB space. +* Wait one hour or more + +Packaging for your favorite distribution would be a welcome contribution! + ## Running monerod The build places the binary in `bin/` sub-directory within the build directory from which cmake was invoked (repository root by default). To run in foreground: - ./bin/monerod +```bash +./bin/monerod +``` To list all available options, run `./bin/monerod --help`. Options can be specified either on the command line or in a configuration file passed by the @@ -509,7 +700,9 @@ of the argument without the leading dashes, for example `log-level=1`. To run in background: - ./bin/monerod --log-file monerod.log --detach +```bash +./bin/monerod --log-file monerod.log --detach +``` To run as a systemd service, copy [monerod.service](utils/systemd/monerod.service) to `/etc/systemd/system/` and @@ -527,6 +720,12 @@ See [README.i18n.md](README.i18n.md). ## Using Tor +> There is a new, still experimental, [integration with Tor](ANONYMITY_NETWORKS.md). The +> feature allows connecting over IPv4 and Tor simulatenously - IPv4 is used for +> relaying blocks and relaying transactions received by peers whereas Tor is +> used solely for relaying transactions received over local RPC. This provides +> privacy and better protection against surrounding node (sybil) attacks. + While Monero isn't made to integrate with Tor, it can be used wrapped with torsocks, by setting the following configuration parameters and environment variables: @@ -546,10 +745,14 @@ setting the following configuration parameters and environment variables: as well. * Do NOT pass `--detach` when running through torsocks with systemd, (see [utils/systemd/monerod.service](utils/systemd/monerod.service) for details). +* If you use the wallet with a Tor daemon via the loopback IP (eg, 127.0.0.1:9050), + then use `--untrusted-daemon` unless it is your own hidden service. Example command line to start monerod through Tor: - DNS_PUBLIC=tcp torsocks monerod --p2p-bind-ip 127.0.0.1 --no-igd +```bash +DNS_PUBLIC=tcp torsocks monerod --p2p-bind-ip 127.0.0.1 --no-igd +``` ### Using Tor on Tails @@ -557,26 +760,28 @@ TAILS ships with a very restrictive set of firewall rules. Therefore, you need to add a rule to allow this connection too, in addition to telling torsocks to allow inbound connections. Full example: - sudo iptables -I OUTPUT 2 -p tcp -d 127.0.0.1 -m tcp --dport 18081 -j ACCEPT - DNS_PUBLIC=tcp torsocks ./monerod --p2p-bind-ip 127.0.0.1 --no-igd --rpc-bind-ip 127.0.0.1 \ - --data-dir /home/amnesia/Persistent/your/directory/to/the/blockchain +```bash +sudo iptables -I OUTPUT 2 -p tcp -d 127.0.0.1 -m tcp --dport 18081 -j ACCEPT +DNS_PUBLIC=tcp torsocks ./monerod --p2p-bind-ip 127.0.0.1 --no-igd --rpc-bind-ip 127.0.0.1 \ + --data-dir /home/amnesia/Persistent/your/directory/to/the/blockchain +``` ## Debugging -This section contains general instructions for debugging failed installs or problems encountered with Monero. First ensure you are running the latest version built from the Github repo. +This section contains general instructions for debugging failed installs or problems encountered with Monero. First, ensure you are running the latest version built from the Github repo. ### Obtaining stack traces and core dumps on Unix systems We generally use the tool `gdb` (GNU debugger) to provide stack trace functionality, and `ulimit` to provide core dumps in builds which crash or segfault. -* To use gdb in order to obtain a stack trace for a build that has stalled: +* To use `gdb` in order to obtain a stack trace for a build that has stalled: Run the build. Once it stalls, enter the following command: -``` -gdb /path/to/monerod `pidof monerod` +```bash +gdb /path/to/monerod `pidof monerod` ``` Type `thread apply all bt` within gdb in order to obtain the stack trace @@ -593,11 +798,13 @@ When it terminates with an output along the lines of "Segmentation fault (core d You can now analyse this core dump with `gdb` as follows: -`gdb /path/to/monerod /path/to/dumpfile` +```bash +gdb /path/to/monerod /path/to/dumpfile` +``` Print the stack trace with `bt` -* To run monero within gdb: +#### To run monero within gdb: Type `gdb /path/to/monerod` @@ -607,9 +814,21 @@ Type `run` to run monerod ### Analysing memory corruption -We use the tool `valgrind` for this. +There are two tools available: + +#### ASAN + +Configure Monero with the -D SANITIZE=ON cmake flag, eg: + +```bash +cd build/debug && cmake -D SANITIZE=ON -D CMAKE_BUILD_TYPE=Debug ../.. +``` + +You can then run the monero tools normally. Performance will typically halve. -Run with `valgrind /path/to/monerod`. It will be slow. +#### valgrind + +Install valgrind and run as `valgrind /path/to/monerod`. It will be very slow. ### LMDB @@ -617,7 +836,9 @@ Instructions for debugging suspected blockchain corruption as per @HYC There is an `mdb_stat` command in the LMDB source that can print statistics about the database but it's not routinely built. This can be built with the following command: -`cd ~/monero/external/db_drivers/liblmdb && make` +```bash +cd ~/monero/external/db_drivers/liblmdb && make +``` The output of `mdb_stat -ea ` will indicate inconsistencies in the blocks, block_heights and block_info table. diff --git a/cmake/32-bit-toolchain.cmake b/cmake/32-bit-toolchain.cmake index fe9aa483f5c..2d53adf06d5 100644 --- a/cmake/32-bit-toolchain.cmake +++ b/cmake/32-bit-toolchain.cmake @@ -1,4 +1,4 @@ -# Copyright (c) 2014-2018, The Monero Project +# Copyright (c) 2014-2019, The Monero Project # # All rights reserved. # @@ -26,7 +26,9 @@ # STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF # THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -set (CMAKE_SYSTEM_NAME Windows) +if (NOT CMAKE_HOST_WIN32) + set (CMAKE_SYSTEM_NAME Windows) +endif() set (GCC_PREFIX i686-w64-mingw32) set (CMAKE_C_COMPILER ${GCC_PREFIX}-gcc) diff --git a/cmake/64-bit-toolchain.cmake b/cmake/64-bit-toolchain.cmake index 20b568c71c8..7c56eef955a 100644 --- a/cmake/64-bit-toolchain.cmake +++ b/cmake/64-bit-toolchain.cmake @@ -1,4 +1,4 @@ -# Copyright (c) 2014-2018, The Monero Project +# Copyright (c) 2014-2019, The Monero Project # # All rights reserved. # @@ -26,7 +26,9 @@ # STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF # THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -set (CMAKE_SYSTEM_NAME Windows) +if (NOT CMAKE_HOST_WIN32) + set (CMAKE_SYSTEM_NAME Windows) +endif() set (GCC_PREFIX x86_64-w64-mingw32) set (CMAKE_C_COMPILER ${GCC_PREFIX}-gcc) diff --git a/cmake/CheckLinkerFlag.cmake b/cmake/CheckLinkerFlag.cmake index a3879d0befa..2b507ab7144 100644 --- a/cmake/CheckLinkerFlag.cmake +++ b/cmake/CheckLinkerFlag.cmake @@ -39,7 +39,7 @@ macro(CHECK_LINKER_FLAG flag VARIABLE) endif() set(${VARIABLE} "" CACHE INTERNAL "Have linker flag ${flag}") file(APPEND ${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeError.log - "Determining if the ${flag} linker flag is suppored " + "Determining if the ${flag} linker flag is supported " "failed with the following output:\n" "${OUTPUT}\n\n") endif() diff --git a/cmake/CheckTrezor.cmake b/cmake/CheckTrezor.cmake new file mode 100644 index 00000000000..6aabdda36ac --- /dev/null +++ b/cmake/CheckTrezor.cmake @@ -0,0 +1,181 @@ +OPTION(USE_DEVICE_TREZOR "Trezor support compilation" ON) +OPTION(USE_DEVICE_TREZOR_LIBUSB "Trezor LibUSB compilation" ON) +OPTION(USE_DEVICE_TREZOR_UDP_RELEASE "Trezor UdpTransport in release mode" OFF) +OPTION(USE_DEVICE_TREZOR_DEBUG "Trezor Debugging enabled" OFF) +OPTION(TREZOR_DEBUG "Main trezor debugging switch" OFF) + +# Helper function to fix cmake < 3.6.0 FindProtobuf variables +function(_trezor_protobuf_fix_vars) + if(${CMAKE_VERSION} VERSION_LESS "3.6.0") + foreach(UPPER + PROTOBUF_SRC_ROOT_FOLDER + PROTOBUF_IMPORT_DIRS + PROTOBUF_DEBUG + PROTOBUF_LIBRARY + PROTOBUF_PROTOC_LIBRARY + PROTOBUF_INCLUDE_DIR + PROTOBUF_PROTOC_EXECUTABLE + PROTOBUF_LIBRARY_DEBUG + PROTOBUF_PROTOC_LIBRARY_DEBUG + PROTOBUF_LITE_LIBRARY + PROTOBUF_LITE_LIBRARY_DEBUG + ) + if (DEFINED ${UPPER}) + string(REPLACE "PROTOBUF_" "Protobuf_" Camel ${UPPER}) + if (NOT DEFINED ${Camel}) + set(${Camel} ${${UPPER}} PARENT_SCOPE) + endif() + endif() + endforeach() + endif() +endfunction() + +# Use Trezor master switch +if (USE_DEVICE_TREZOR) + # Protobuf is required to build protobuf messages for Trezor + include(FindProtobuf OPTIONAL) + find_package(Protobuf) + _trezor_protobuf_fix_vars() + + # Protobuf handling the cache variables set in docker. + if(NOT Protobuf_FOUND AND NOT Protobuf_LIBRARY AND NOT Protobuf_PROTOC_EXECUTABLE AND NOT Protobuf_INCLUDE_DIR) + message(STATUS "Could not find Protobuf") + elseif(NOT Protobuf_LIBRARY OR NOT EXISTS "${Protobuf_LIBRARY}") + message(STATUS "Protobuf library not found: ${Protobuf_LIBRARY}") + unset(Protobuf_FOUND) + elseif(NOT Protobuf_PROTOC_EXECUTABLE OR NOT EXISTS "${Protobuf_PROTOC_EXECUTABLE}") + message(STATUS "Protobuf executable not found: ${Protobuf_PROTOC_EXECUTABLE}") + unset(Protobuf_FOUND) + elseif(NOT Protobuf_INCLUDE_DIR OR NOT EXISTS "${Protobuf_INCLUDE_DIR}") + message(STATUS "Protobuf include dir not found: ${Protobuf_INCLUDE_DIR}") + unset(Protobuf_FOUND) + else() + message(STATUS "Protobuf lib: ${Protobuf_LIBRARY}, inc: ${Protobuf_INCLUDE_DIR}, protoc: ${Protobuf_PROTOC_EXECUTABLE}") + set(Protobuf_INCLUDE_DIRS ${Protobuf_INCLUDE_DIR}) + set(Protobuf_FOUND 1) # override found if all rquired info was provided by variables + endif() + + if(TREZOR_DEBUG) + set(USE_DEVICE_TREZOR_DEBUG 1) + endif() + + # Compile debugging support (for tests) + if (USE_DEVICE_TREZOR_DEBUG) + add_definitions(-DWITH_TREZOR_DEBUGGING=1) + endif() +else() + message(STATUS "Trezor support disabled by USE_DEVICE_TREZOR") +endif() + +if(Protobuf_FOUND AND USE_DEVICE_TREZOR) + if (NOT "$ENV{TREZOR_PYTHON}" STREQUAL "") + set(TREZOR_PYTHON "$ENV{TREZOR_PYTHON}" CACHE INTERNAL "Copied from environment variable TREZOR_PYTHON") + else() + find_package(Python QUIET COMPONENTS Interpreter) # cmake 3.12+ + if(Python_Interpreter_FOUND) + set(TREZOR_PYTHON "${Python_EXECUTABLE}") + endif() + endif() + + if(NOT TREZOR_PYTHON) + find_package(PythonInterp) + if(PYTHONINTERP_FOUND AND PYTHON_EXECUTABLE) + set(TREZOR_PYTHON "${PYTHON_EXECUTABLE}") + endif() + endif() + + if(NOT TREZOR_PYTHON) + message(STATUS "Trezor: Python not found") + endif() +endif() + +# Protobuf compilation test +if(Protobuf_FOUND AND USE_DEVICE_TREZOR AND TREZOR_PYTHON) + execute_process(COMMAND ${Protobuf_PROTOC_EXECUTABLE} -I "${CMAKE_SOURCE_DIR}/cmake" -I "${Protobuf_INCLUDE_DIR}" "${CMAKE_SOURCE_DIR}/cmake/test-protobuf.proto" --cpp_out ${CMAKE_BINARY_DIR} RESULT_VARIABLE RET OUTPUT_VARIABLE OUT ERROR_VARIABLE ERR) + if(RET) + message(STATUS "Protobuf test generation failed: ${OUT} ${ERR}") + endif() + + try_compile(Protobuf_COMPILE_TEST_PASSED + "${CMAKE_BINARY_DIR}" + SOURCES + "${CMAKE_BINARY_DIR}/test-protobuf.pb.cc" + "${CMAKE_SOURCE_DIR}/cmake/test-protobuf.cpp" + CMAKE_FLAGS + "-DINCLUDE_DIRECTORIES=${Protobuf_INCLUDE_DIR};${CMAKE_BINARY_DIR}" + "-DCMAKE_CXX_STANDARD=11" + LINK_LIBRARIES ${Protobuf_LIBRARY} + OUTPUT_VARIABLE OUTPUT + ) + if(NOT Protobuf_COMPILE_TEST_PASSED) + message(STATUS "Protobuf Compilation test failed: ${OUTPUT}.") + endif() +endif() + +# Try to build protobuf messages +if(Protobuf_FOUND AND USE_DEVICE_TREZOR AND TREZOR_PYTHON AND Protobuf_COMPILE_TEST_PASSED) + set(ENV{PROTOBUF_INCLUDE_DIRS} "${Protobuf_INCLUDE_DIR}") + set(ENV{PROTOBUF_PROTOC_EXECUTABLE} "${Protobuf_PROTOC_EXECUTABLE}") + set(TREZOR_PROTOBUF_PARAMS "") + if (USE_DEVICE_TREZOR_DEBUG) + set(TREZOR_PROTOBUF_PARAMS "--debug") + endif() + + execute_process(COMMAND ${TREZOR_PYTHON} tools/build_protob.py ${TREZOR_PROTOBUF_PARAMS} WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR}/../src/device_trezor/trezor RESULT_VARIABLE RET OUTPUT_VARIABLE OUT ERROR_VARIABLE ERR) + if(RET) + message(WARNING "Trezor protobuf messages could not be regenerated (err=${RET}, python ${PYTHON})." + "OUT: ${OUT}, ERR: ${ERR}." + "Please read src/device_trezor/trezor/tools/README.md") + else() + message(STATUS "Trezor protobuf messages regenerated out: \"${OUT}.\"") + set(DEVICE_TREZOR_READY 1) + add_definitions(-DDEVICE_TREZOR_READY=1) + add_definitions(-DPROTOBUF_INLINE_NOT_IN_HEADERS=0) + + if(CMAKE_BUILD_TYPE STREQUAL "Debug") + add_definitions(-DTREZOR_DEBUG=1) + endif() + + if(USE_DEVICE_TREZOR_UDP_RELEASE) + add_definitions(-DUSE_DEVICE_TREZOR_UDP_RELEASE=1) + endif() + + if (Protobuf_INCLUDE_DIR) + include_directories(${Protobuf_INCLUDE_DIR}) + endif() + + # LibUSB support, check for particular version + # Include support only if compilation test passes + if (USE_DEVICE_TREZOR_LIBUSB) + find_package(LibUSB) + endif() + + if (LibUSB_COMPILE_TEST_PASSED) + add_definitions(-DHAVE_TREZOR_LIBUSB=1) + if(LibUSB_INCLUDE_DIRS) + include_directories(${LibUSB_INCLUDE_DIRS}) + endif() + endif() + + set(TREZOR_LIBUSB_LIBRARIES "") + if(LibUSB_COMPILE_TEST_PASSED) + list(APPEND TREZOR_LIBUSB_LIBRARIES ${LibUSB_LIBRARIES}) + message(STATUS "Trezor compatible LibUSB found at: ${LibUSB_INCLUDE_DIRS}") + endif() + + if (BUILD_GUI_DEPS) + set(TREZOR_DEP_LIBS "") + set(TREZOR_DEP_LINKER "") + + if (Protobuf_LIBRARY) + list(APPEND TREZOR_DEP_LIBS ${Protobuf_LIBRARY}) + string(APPEND TREZOR_DEP_LINKER " -lprotobuf") + endif() + + if (TREZOR_LIBUSB_LIBRARIES) + list(APPEND TREZOR_DEP_LIBS ${TREZOR_LIBUSB_LIBRARIES}) + string(APPEND TREZOR_DEP_LINKER " -lusb-1.0") + endif() + endif() + endif() +endif() diff --git a/cmake/Doxyfile.in b/cmake/Doxyfile.in index 35a4911f8d8..81199d5e448 100644 --- a/cmake/Doxyfile.in +++ b/cmake/Doxyfile.in @@ -18,7 +18,7 @@ # that follow. The default is UTF-8 which is also the encoding used for all # text before the first occurrence of this tag. Doxygen uses libiconv (or the # iconv built into libc) for the transcoding. See -# http://www.gnu.org/software/libiconv for the list of possible encodings. +# https://www.gnu.org/software/libiconv for the list of possible encodings. DOXYFILE_ENCODING = UTF-8 @@ -242,7 +242,7 @@ EXTENSION_MAPPING = # If MARKDOWN_SUPPORT is enabled (the default) then doxygen pre-processes all # comments according to the Markdown format, which allows for more readable -# documentation. See http://daringfireball.net/projects/markdown/ for details. +# documentation. See https://daringfireball.net/projects/markdown/ for details. # The output of markdown processing is further processed by doxygen, so you # can mix doxygen, HTML, and XML commands with Markdown formatting. # Disable only in case of backward compatibilities issues. @@ -589,7 +589,7 @@ LAYOUT_FILE = # containing the references data. This must be a list of .bib files. The # .bib extension is automatically appended if omitted. Using this command # requires the bibtex tool to be installed. See also -# http://en.wikipedia.org/wiki/BibTeX for more info. For LaTeX the style +# https://en.wikipedia.org/wiki/BibTeX for more info. For LaTeX the style # of the bibliography can be controlled using LATEX_BIB_STYLE. To use this # feature you need bibtex and perl available in the search path. @@ -665,7 +665,7 @@ INPUT = ../README.md \ # This tag can be used to specify the character encoding of the source files # that doxygen parses. Internally doxygen uses the UTF-8 encoding, which is # also the default input encoding. Doxygen uses libiconv (or the iconv built -# into libc) for the transcoding. See http://www.gnu.org/software/libiconv for +# into libc) for the transcoding. See https://www.gnu.org/software/libiconv for # the list of possible encodings. INPUT_ENCODING = UTF-8 @@ -833,7 +833,7 @@ REFERENCES_LINK_SOURCE = YES # If the USE_HTAGS tag is set to YES then the references to source code # will point to the HTML generated by the htags(1) tool instead of doxygen # built-in source browser. The htags tool is part of GNU's global source -# tagging system (see http://www.gnu.org/software/global/global.html). You +# tagging system (see https://www.gnu.org/software/global/global.html). You # will need version 4.8.6 or higher. USE_HTAGS = NO @@ -928,7 +928,7 @@ HTML_EXTRA_FILES = # The HTML_COLORSTYLE_HUE tag controls the color of the HTML output. # Doxygen will adjust the colors in the style sheet and background images # according to this color. Hue is specified as an angle on a colorwheel, -# see http://en.wikipedia.org/wiki/Hue for more information. +# see https://en.wikipedia.org/wiki/Hue for more information. # For instance the value 0 represents red, 60 is yellow, 120 is green, # 180 is cyan, 240 is blue, 300 purple, and 360 is red again. # The allowed range is 0 to 359. @@ -981,7 +981,7 @@ HTML_INDEX_NUM_ENTRIES = 100 # directory and running "make install" will install the docset in # ~/Library/Developer/Shared/Documentation/DocSets so that Xcode will find # it at startup. -# See http://developer.apple.com/tools/creatingdocsetswithdoxygen.html +# See https://developer.apple.com/tools/creatingdocsetswithdoxygen.html # for more information. GENERATE_DOCSET = NO @@ -1179,7 +1179,7 @@ FORMULA_FONTSIZE = 10 FORMULA_TRANSPARENT = YES # Enable the USE_MATHJAX option to render LaTeX formulas using MathJax -# (see http://www.mathjax.org) which uses client side Javascript for the +# (see https://www.mathjax.org) which uses client side Javascript for the # rendering instead of using prerendered bitmaps. Use this if you do not # have LaTeX installed or if you want to formulas look prettier in the HTML # output. When enabled you may also need to install MathJax separately and @@ -1194,9 +1194,9 @@ USE_MATHJAX = NO # MATHJAX_RELPATH should be ../mathjax. The default value points to # the MathJax Content Delivery Network so you can quickly see the result without # installing MathJax. However, it is strongly recommended to install a local -# copy of MathJax from http://www.mathjax.org before deployment. +# copy of MathJax from https://www.mathjax.org before deployment. -MATHJAX_RELPATH = http://cdn.mathjax.org/mathjax/latest +MATHJAX_RELPATH = https://cdn.mathjax.org/mathjax/latest # The MATHJAX_EXTENSIONS tag can be used to specify one or MathJax extension # names that should be enabled during MathJax rendering. @@ -1318,7 +1318,7 @@ LATEX_SOURCE_CODE = NO # The LATEX_BIB_STYLE tag can be used to specify the style to use for the # bibliography, e.g. plainnat, or ieeetr. The default style is "plain". See -# http://en.wikipedia.org/wiki/BibTeX for more info. +# https://en.wikipedia.org/wiki/BibTeX for more info. LATEX_BIB_STYLE = plain diff --git a/cmake/FindBacktrace.cmake b/cmake/FindBacktrace.cmake new file mode 100644 index 00000000000..8137125ab98 --- /dev/null +++ b/cmake/FindBacktrace.cmake @@ -0,0 +1,98 @@ +# Distributed under the OSI-approved BSD 3-Clause License. See accompanying +# file Copyright.txt or https://cmake.org/licensing for details. + +#.rst: +# FindBacktrace +# ------------- +# +# Find provider for backtrace(3). +# +# Checks if OS supports backtrace(3) via either libc or custom library. +# This module defines the following variables: +# +# ``Backtrace_HEADER`` +# The header file needed for backtrace(3). Cached. +# Could be forcibly set by user. +# ``Backtrace_INCLUDE_DIRS`` +# The include directories needed to use backtrace(3) header. +# ``Backtrace_LIBRARIES`` +# The libraries (linker flags) needed to use backtrace(3), if any. +# ``Backtrace_FOUND`` +# Is set if and only if backtrace(3) support detected. +# +# The following cache variables are also available to set or use: +# +# ``Backtrace_LIBRARY`` +# The external library providing backtrace, if any. +# ``Backtrace_INCLUDE_DIR`` +# The directory holding the backtrace(3) header. +# +# Typical usage is to generate of header file using configure_file() with the +# contents like the following:: +# +# #cmakedefine01 Backtrace_FOUND +# #if Backtrace_FOUND +# # include <${Backtrace_HEADER}> +# #endif +# +# And then reference that generated header file in actual source. + +include(CMakePushCheckState) +include(CheckSymbolExists) +include(FindPackageHandleStandardArgs) + +# List of variables to be provided to find_package_handle_standard_args() +set(_Backtrace_STD_ARGS Backtrace_INCLUDE_DIR) + +if(Backtrace_HEADER) + set(_Backtrace_HEADER_TRY "${Backtrace_HEADER}") +else(Backtrace_HEADER) + set(_Backtrace_HEADER_TRY "execinfo.h") +endif(Backtrace_HEADER) + +find_path(Backtrace_INCLUDE_DIR "${_Backtrace_HEADER_TRY}") +set(Backtrace_INCLUDE_DIRS ${Backtrace_INCLUDE_DIR}) + +if (NOT DEFINED Backtrace_LIBRARY) + # First, check if we already have backtrace(), e.g., in libc + cmake_push_check_state(RESET) + set(CMAKE_REQUIRED_INCLUDES ${Backtrace_INCLUDE_DIRS}) + set(CMAKE_REQUIRED_QUIET ${Backtrace_FIND_QUIETLY}) + check_symbol_exists("backtrace" "${_Backtrace_HEADER_TRY}" _Backtrace_SYM_FOUND) + cmake_pop_check_state() +endif() + +if(_Backtrace_SYM_FOUND) + # Avoid repeating the message() call below each time CMake is run. + if(NOT Backtrace_FIND_QUIETLY AND NOT DEFINED Backtrace_LIBRARY) + message(STATUS "backtrace facility detected in default set of libraries") + endif() + set(Backtrace_LIBRARY "" CACHE FILEPATH "Library providing backtrace(3), empty for default set of libraries") +else() + # Check for external library, for non-glibc systems + if(Backtrace_INCLUDE_DIR) + # OpenBSD has libbacktrace renamed to libexecinfo + find_library(Backtrace_LIBRARY "execinfo") + elseif() # respect user wishes + set(_Backtrace_HEADER_TRY "backtrace.h") + find_path(Backtrace_INCLUDE_DIR ${_Backtrace_HEADER_TRY}) + find_library(Backtrace_LIBRARY "backtrace") + endif() + + # Prepend list with library path as it's more common practice + set(_Backtrace_STD_ARGS Backtrace_LIBRARY ${_Backtrace_STD_ARGS}) +endif() + +message(STATUS "Backtrace_LIBRARY: ${Backtrace_LIBRARY}") +if(Backtrace_LIBRARY STREQUAL "NOTFOUND") + set(Backtrace_LIBRARY "") +endif() +if(Backtrace_LIBRARY STREQUAL "Backtrace_LIBRARY-NOTFOUND") + set(Backtrace_LIBRARY "") +endif() + +set(Backtrace_LIBRARIES ${Backtrace_LIBRARY}) +set(Backtrace_HEADER "${_Backtrace_HEADER_TRY}" CACHE STRING "Header providing backtrace(3) facility") + +find_package_handle_standard_args(Backtrace FOUND_VAR Backtrace_FOUND REQUIRED_VARS ${_Backtrace_STD_ARGS}) +mark_as_advanced(Backtrace_HEADER Backtrace_INCLUDE_DIR Backtrace_LIBRARY) diff --git a/cmake/FindHIDAPI.cmake b/cmake/FindHIDAPI.cmake new file mode 100644 index 00000000000..9b45dcc4c10 --- /dev/null +++ b/cmake/FindHIDAPI.cmake @@ -0,0 +1,60 @@ +# - try to find HIDAPI library +# from http://www.signal11.us/oss/hidapi/ +# +# Cache Variables: (probably not for direct use in your scripts) +# HIDAPI_INCLUDE_DIR +# HIDAPI_LIBRARY +# +# Non-cache variables you might use in your CMakeLists.txt: +# HIDAPI_FOUND +# HIDAPI_INCLUDE_DIRS +# HIDAPI_LIBRARIES +# +# Requires these CMake modules: +# FindPackageHandleStandardArgs (known included with CMake >=2.6.2) +# +# Original Author: +# 2009-2010 Ryan Pavlik +# http://academic.cleardefinition.com +# Iowa State University HCI Graduate Program/VRAC +# +# Copyright Iowa State University 2009-2010. +# Distributed under the Boost Software License, Version 1.0. +# (See accompanying file LICENSE_1_0.txt or copy at +# http://www.boost.org/LICENSE_1_0.txt) + +find_library(HIDAPI_LIBRARY + NAMES hidapi hidapi-libusb) + +find_path(HIDAPI_INCLUDE_DIR + NAMES hidapi.h + PATH_SUFFIXES + hidapi) + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(HIDAPI + DEFAULT_MSG + HIDAPI_LIBRARY + HIDAPI_INCLUDE_DIR) + +if(HIDAPI_FOUND) + set(HIDAPI_LIBRARIES "${HIDAPI_LIBRARY}") + if((STATIC AND UNIX AND NOT APPLE) OR (DEPENDS AND CMAKE_SYSTEM_NAME STREQUAL "Linux")) + find_library(LIBUSB-1.0_LIBRARY usb-1.0) + find_library(LIBUDEV_LIBRARY udev) + if(LIBUSB-1.0_LIBRARY) + set(HIDAPI_LIBRARIES "${HIDAPI_LIBRARIES};${LIBUSB-1.0_LIBRARY}") + if(LIBUDEV_LIBRARY) + set(HIDAPI_LIBRARIES "${HIDAPI_LIBRARIES};${LIBUDEV_LIBRARY}") + else() + message(WARNING "libudev library not found, binaries may fail to link.") + endif() + else() + message(WARNING "libusb-1.0 library not found, binaries may fail to link.") + endif() + endif() + + set(HIDAPI_INCLUDE_DIRS "${HIDAPI_INCLUDE_DIR}") +endif() + +mark_as_advanced(HIDAPI_INCLUDE_DIR HIDAPI_LIBRARY) diff --git a/cmake/FindLibUSB.cmake b/cmake/FindLibUSB.cmake new file mode 100644 index 00000000000..c7e09d4c05f --- /dev/null +++ b/cmake/FindLibUSB.cmake @@ -0,0 +1,149 @@ +# - Find libusb for portable USB support +# This module will find libusb as published by +# http://libusb.sf.net and +# http://libusb-win32.sf.net +# +# It will use PkgConfig if present and supported, else search +# it on its own. If the LibUSB_ROOT_DIR environment variable +# is defined, it will be used as base path. +# The following standard variables get defined: +# LibUSB_FOUND: true if LibUSB was found +# LibUSB_HEADER_FILE: the location of the C header file +# LibUSB_INCLUDE_DIRS: the directory that contains the include file +# LibUSB_LIBRARIES: the library +# source: https://github.com/IntelRealSense/librealsense + +include ( CheckLibraryExists ) +include ( CheckIncludeFile ) + +find_package ( PkgConfig ) +if ( PKG_CONFIG_FOUND ) + pkg_check_modules ( PKGCONFIG_LIBUSB libusb-1.0 ) + if ( NOT PKGCONFIG_LIBUSB_FOUND ) + pkg_check_modules ( PKGCONFIG_LIBUSB libusb ) + endif ( NOT PKGCONFIG_LIBUSB_FOUND ) +endif ( PKG_CONFIG_FOUND ) + +if ( PKGCONFIG_LIBUSB_FOUND ) + set ( LibUSB_INCLUDE_DIRS ${PKGCONFIG_LIBUSB_INCLUDE_DIRS} ) + foreach ( i ${PKGCONFIG_LIBUSB_LIBRARIES} ) + string ( REGEX MATCH "[^-]*" ibase "${i}" ) + find_library ( ${ibase}_LIBRARY + NAMES ${i} + PATHS ${PKGCONFIG_LIBUSB_LIBRARY_DIRS} + ) + if ( ${ibase}_LIBRARY ) + list ( APPEND LibUSB_LIBRARIES ${${ibase}_LIBRARY} ) + endif ( ${ibase}_LIBRARY ) + mark_as_advanced ( ${ibase}_LIBRARY ) + endforeach ( i ) + +else ( PKGCONFIG_LIBUSB_FOUND ) + find_file ( LibUSB_HEADER_FILE + NAMES + libusb.h usb.h + PATHS + $ENV{ProgramFiles}/LibUSB-Win32 + $ENV{LibUSB_ROOT_DIR} + PATH_SUFFIXES + include + libusb-1.0 + include/libusb-1.0 + ) + mark_as_advanced ( LibUSB_HEADER_FILE ) + get_filename_component ( LibUSB_INCLUDE_DIRS "${LibUSB_HEADER_FILE}" PATH ) + + if ( ${CMAKE_SYSTEM_NAME} STREQUAL "Windows" ) + # LibUSB-Win32 binary distribution contains several libs. + # Use the lib that got compiled with the same compiler. + if ( MSVC ) + if ( WIN32 ) + set ( LibUSB_LIBRARY_PATH_SUFFIX lib/msvc ) + else ( WIN32 ) + set ( LibUSB_LIBRARY_PATH_SUFFIX lib/msvc_x64 ) + endif ( WIN32 ) + elseif ( BORLAND ) + set ( LibUSB_LIBRARY_PATH_SUFFIX lib/bcc ) + elseif ( CMAKE_COMPILER_IS_GNUCC ) + set ( LibUSB_LIBRARY_PATH_SUFFIX lib/gcc ) + endif ( MSVC ) + endif ( ${CMAKE_SYSTEM_NAME} STREQUAL "Windows" ) + + find_library ( usb_LIBRARY + NAMES + usb-1.0 libusb usb + PATHS + $ENV{ProgramFiles}/LibUSB-Win32 + $ENV{LibUSB_ROOT_DIR} + PATH_SUFFIXES + ${LibUSB_LIBRARY_PATH_SUFFIX} + ) + mark_as_advanced ( usb_LIBRARY ) + if ( usb_LIBRARY ) + set ( LibUSB_LIBRARIES ${usb_LIBRARY} ) + endif ( usb_LIBRARY ) + +endif ( PKGCONFIG_LIBUSB_FOUND ) + +if ( LibUSB_INCLUDE_DIRS AND LibUSB_LIBRARIES ) + set ( LibUSB_FOUND true ) +endif ( LibUSB_INCLUDE_DIRS AND LibUSB_LIBRARIES ) + +if ( LibUSB_FOUND ) + set ( CMAKE_REQUIRED_INCLUDES "${LibUSB_INCLUDE_DIRS}" ) + check_include_file ( "${LibUSB_HEADER_FILE}" LibUSB_FOUND ) +endif ( LibUSB_FOUND ) + +if ( LibUSB_FOUND ) + check_library_exists ( "${LibUSB_LIBRARIES}" usb_open "" LibUSB_FOUND ) + check_library_exists ( "${LibUSB_LIBRARIES}" libusb_get_device_list "" LibUSB_VERSION_1.0 ) + check_library_exists ( "${LibUSB_LIBRARIES}" libusb_get_port_numbers "" LibUSB_VERSION_1.0.16 ) + + if((STATIC AND UNIX AND NOT APPLE) OR (DEPENDS AND CMAKE_SYSTEM_NAME STREQUAL "Linux")) + find_library(LIBUDEV_LIBRARY udev) + if(LIBUDEV_LIBRARY) + set(LibUSB_LIBRARIES "${LibUSB_LIBRARIES};${LIBUDEV_LIBRARY}") + else() + message(WARNING "libudev library not found, binaries may fail to link.") + endif() + endif() + + # Library 1.0.16+ compilation test. + # The check_library_exists does not work well on Apple with shared libs. + if (APPLE OR LibUSB_VERSION_1.0.16 OR STATIC) + if (APPLE) + if(DEPENDS) + list(APPEND TEST_COMPILE_EXTRA_LIBRARIES "-framework Foundation -framework IOKit") + else() + find_library(COREFOUNDATION CoreFoundation) + find_library(IOKIT IOKit) + list(APPEND TEST_COMPILE_EXTRA_LIBRARIES ${IOKIT}) + list(APPEND TEST_COMPILE_EXTRA_LIBRARIES ${COREFOUNDATION}) + endif() + endif() + if (WIN32) + list(APPEND TEST_COMPILE_EXTRA_LIBRARIES setupapi) + endif() + list(APPEND TEST_COMPILE_EXTRA_LIBRARIES ${LibUSB_LIBRARIES}) + + try_compile(LibUSB_COMPILE_TEST_PASSED + ${CMAKE_BINARY_DIR} + "${CMAKE_SOURCE_DIR}/cmake/test-libusb-version.c" + CMAKE_FLAGS + "-DINCLUDE_DIRECTORIES=${LibUSB_INCLUDE_DIRS}" + "-DLINK_DIRECTORIES=${LibUSB_LIBRARIES}" + LINK_LIBRARIES ${TEST_COMPILE_EXTRA_LIBRARIES} + OUTPUT_VARIABLE OUTPUT) + unset(TEST_COMPILE_EXTRA_LIBRARIES) + message(STATUS "LibUSB Compilation test: ${LibUSB_COMPILE_TEST_PASSED}") + endif() +endif ( LibUSB_FOUND ) + +if ( NOT LibUSB_FOUND ) + if ( NOT LibUSB_FIND_QUIETLY ) + message ( STATUS "LibUSB not found, try setting LibUSB_ROOT_DIR environment variable." ) + endif ( NOT LibUSB_FIND_QUIETLY ) + if ( LibUSB_FIND_REQUIRED ) + message ( FATAL_ERROR "" ) + endif ( LibUSB_FIND_REQUIRED ) +endif ( NOT LibUSB_FOUND ) diff --git a/cmake/FindPCSC.cmake b/cmake/FindPCSC.cmake deleted file mode 100644 index 8dd9d0e7640..00000000000 --- a/cmake/FindPCSC.cmake +++ /dev/null @@ -1,44 +0,0 @@ -# - Find PCSC -# Find the native PCSC includes and library -# -# PCSC_INCLUDE_DIR - where to find winscard.h, wintypes.h, etc. -# PCSC_LIBRARIES - List of libraries when using PCSC. -# PCSC_FOUND - True if PCSC found. - - -IF (PCSC_INCLUDE_DIR AND PCSC_LIBRARIES) - # Already in cache, be silent - SET(PCSC_FIND_QUIETLY TRUE) -ENDIF (PCSC_INCLUDE_DIR AND PCSC_LIBRARIES) - -IF (NOT WIN32) - FIND_PACKAGE(PkgConfig) - PKG_CHECK_MODULES(PC_PCSC libpcsclite) -ENDIF (NOT WIN32) - -FIND_PATH(PCSC_INCLUDE_DIR winscard.h - HINTS - /usr/include/PCSC - ${PC_PCSC_INCLUDEDIR} - ${PC_PCSC_INCLUDE_DIRS} - PATH_SUFFIXES PCSC - ) - -FIND_LIBRARY(PCSC_LIBRARY NAMES pcsclite libpcsclite WinSCard PCSC - HINTS - ${PC_PCSC_LIBDIR} - ${PC_PCSC_LIBRARY_DIRS} - ) - -# handle the QUIETLY and REQUIRED arguments and set PCSC_FOUND to TRUE if -# all listed variables are TRUE -INCLUDE(FindPackageHandleStandardArgs) -FIND_PACKAGE_HANDLE_STANDARD_ARGS(PCSC DEFAULT_MSG PCSC_LIBRARY PCSC_INCLUDE_DIR) - -IF(PCSC_FOUND) - SET( PCSC_LIBRARIES ${PCSC_LIBRARY} ) -ELSE(PCSC_FOUND) - SET( PCSC_LIBRARIES ) -ENDIF(PCSC_FOUND) - -MARK_AS_ADVANCED( PCSC_LIBRARY PCSC_INCLUDE_DIR ) diff --git a/cmake/FindReadline.cmake b/cmake/FindReadline.cmake index 87f8ccace0a..f26911b2612 100644 --- a/cmake/FindReadline.cmake +++ b/cmake/FindReadline.cmake @@ -23,7 +23,7 @@ find_path(Readline_ROOT_DIR NAMES include/readline/readline.h - PATHS /opt/local/ /usr/local/ /usr/ + PATHS /usr/local/opt/readline/ /opt/local/ /usr/local/ /usr/ NO_DEFAULT_PATH ) @@ -66,7 +66,9 @@ check_function_exists(rl_copy_text HAVE_COPY_TEXT) check_function_exists(rl_filename_completion_function HAVE_COMPLETION_FUNCTION) if(NOT HAVE_COMPLETION_FUNCTION) - set(CMAKE_REQUIRED_LIBRARIES ${Readline_LIBRARY} ${Termcap_LIBRARY}) + if (Readline_LIBRARY) + set(CMAKE_REQUIRED_LIBRARIES ${Readline_LIBRARY} ${Termcap_LIBRARY}) + endif(Readline_LIBRARY) check_function_exists(rl_copy_text HAVE_COPY_TEXT_TC) check_function_exists(rl_filename_completion_function HAVE_COMPLETION_FUNCTION_TC) set(HAVE_COMPLETION_FUNCTION ${HAVE_COMPLETION_FUNCTION_TC}) diff --git a/cmake/FindUnbound.cmake b/cmake/FindUnbound.cmake index a392d59cac9..e3eb3d67eae 100644 --- a/cmake/FindUnbound.cmake +++ b/cmake/FindUnbound.cmake @@ -1,4 +1,4 @@ -# Copyright (c) 2014-2018, The Monero Project +# Copyright (c) 2014-2019, The Monero Project # All rights reserved. # # Redistribution and use in source and binary forms, with or without modification, are diff --git a/cmake/GenVersion.cmake b/cmake/GenVersion.cmake index 2891151753f..b2ccfbc3447 100644 --- a/cmake/GenVersion.cmake +++ b/cmake/GenVersion.cmake @@ -1,4 +1,4 @@ -# Copyright (c) 2014-2018, The Monero Project +# Copyright (c) 2014-2019, The Monero Project # # All rights reserved. # @@ -29,7 +29,7 @@ # Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers # Check what commit we're on -execute_process(COMMAND "${GIT}" rev-parse --short HEAD RESULT_VARIABLE RET OUTPUT_VARIABLE COMMIT OUTPUT_STRIP_TRAILING_WHITESPACE) +execute_process(COMMAND "${GIT}" rev-parse --short=9 HEAD RESULT_VARIABLE RET OUTPUT_VARIABLE COMMIT OUTPUT_STRIP_TRAILING_WHITESPACE) if(RET) # Something went wrong, set the version tag to -unknown @@ -38,6 +38,7 @@ if(RET) set(VERSIONTAG "unknown") configure_file("src/version.cpp.in" "${TO}") else() + string(SUBSTRING ${COMMIT} 0 9 COMMIT) message(STATUS "You are currently on commit ${COMMIT}") # Get all the tags diff --git a/cmake/Version.cmake b/cmake/Version.cmake index 3677e80d7c4..632c1431c95 100644 --- a/cmake/Version.cmake +++ b/cmake/Version.cmake @@ -1,4 +1,4 @@ -# Copyright (c) 2014-2018, The Monero Project +# Copyright (c) 2014-2019, The Monero Project # # All rights reserved. # diff --git a/cmake/test-libusb-version.c b/cmake/test-libusb-version.c new file mode 100644 index 00000000000..b80191b58db --- /dev/null +++ b/cmake/test-libusb-version.c @@ -0,0 +1,52 @@ +// Copyright (c) 2014-2019, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include + +#define UNUSED(expr) (void)(expr) + +int main(int argc, char *argv[]) { + libusb_device **devs; + libusb_context *ctx = NULL; + + int r = libusb_init(&ctx); UNUSED(r); + ssize_t cnt = libusb_get_device_list(ctx, &devs); UNUSED(cnt); + + struct libusb_device_descriptor desc; + r = libusb_get_device_descriptor(devs[0], &desc); UNUSED(r); + uint8_t bus_id = libusb_get_bus_number(devs[0]); UNUSED(bus_id); + uint8_t addr = libusb_get_device_address(devs[0]); UNUSED(addr); + + uint8_t tmp_path[16]; + r = libusb_get_port_numbers(devs[0], tmp_path, sizeof(tmp_path)); + UNUSED(r); + UNUSED(tmp_path); + + libusb_free_device_list(devs, 1); + libusb_exit(ctx); +} diff --git a/cmake/test-protobuf.cpp b/cmake/test-protobuf.cpp new file mode 100644 index 00000000000..0a1ca82befc --- /dev/null +++ b/cmake/test-protobuf.cpp @@ -0,0 +1,43 @@ +// Copyright (c) 2014-2019, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include +#include +#include +#include +#include "test-protobuf.pb.h" + +int main(int argc, char *argv[]) { + google::protobuf::UnknownFieldSet ufs; + ufs.ClearAndFreeMemory(); + + Success sc; + sc.set_message("test"); + sc.SerializeToOstream(&std::cerr); + return 0; +} diff --git a/cmake/test-protobuf.proto b/cmake/test-protobuf.proto new file mode 100644 index 00000000000..5300aea3569 --- /dev/null +++ b/cmake/test-protobuf.proto @@ -0,0 +1,7 @@ +syntax = "proto2"; + +import "google/protobuf/descriptor.proto"; + +message Success { + optional string message = 1; +} diff --git a/cmake/test-static-assert.c b/cmake/test-static-assert.c index 28a80a97253..2e6c8483939 100644 --- a/cmake/test-static-assert.c +++ b/cmake/test-static-assert.c @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/cmake/test-static-assert.cpp b/cmake/test-static-assert.cpp index 28a80a97253..2e6c8483939 100644 --- a/cmake/test-static-assert.cpp +++ b/cmake/test-static-assert.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 990a05c0879..0db645ce914 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2014-2018, The Monero Project +# Copyright (c) 2014-2019, The Monero Project # # All rights reserved. # @@ -26,10 +26,5 @@ # STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF # THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# warnings are cleared only for GCC on Linux -if (NOT (MINGW OR APPLE OR FREEBSD OR OPENBSD OR DRAGONFLY)) - add_compile_options("${WARNINGS_AS_ERRORS_FLAG}") # applies only to targets that follow -endif() - add_subdirectory(epee) diff --git a/contrib/brew/Brewfile b/contrib/brew/Brewfile new file mode 100644 index 00000000000..1fdf45cfe53 --- /dev/null +++ b/contrib/brew/Brewfile @@ -0,0 +1,34 @@ +# Brewfile for Monero +# A homebrew Brewfile installs all required dependencies in one shot +# see https://coderwall.com/p/afmnbq/homebrew-s-new-feature-brewfiles +# https://github.com/Homebrew/homebrew-bundle +# execute brew bundle in the directory containing the Brewfile + +tap "homebrew/bundle" +tap "homebrew/cask" +tap "homebrew/cask-versions" +tap "homebrew/core" + +brew "autoconf" +brew "autogen" +brew "automake" +brew "binutils" +brew "coreutils" +brew "cmake" +brew "pkg-config" +brew "boost" +brew "openssl" +brew "hidapi" +brew "zmq" +brew "libpgm" +brew "unbound" +brew "libsodium" +brew "miniupnpc" +brew "readline" +brew "ldns" +brew "expat" +brew "doxygen" +brew "graphviz" +brew "libunwind-headers" +brew "xz" +brew "protobuf" diff --git a/contrib/codefresh/codefresh.yml b/contrib/codefresh/codefresh.yml new file mode 100644 index 00000000000..a22debfa1ac --- /dev/null +++ b/contrib/codefresh/codefresh.yml @@ -0,0 +1,18 @@ +version: '1.0' +steps: + init_submodules: + title: Init Submodules + commands: + - git submodule update --init --recursive + image: codefreshio/git-image:latest + working_directory: ${{main_clone}} + + BuildingDockerImage: + title: Building Docker Image + type: build + image_name: monero + working_directory: ./ + tag: '${{CF_BRANCH_TAG_NORMALIZED}}' + dockerfile: Dockerfile + build_arguments: + - NPROC=1 diff --git a/contrib/depends/.gitignore b/contrib/depends/.gitignore new file mode 100644 index 00000000000..3cb4b9ac155 --- /dev/null +++ b/contrib/depends/.gitignore @@ -0,0 +1,10 @@ +SDKs/ +work/ +built/ +sources/ +config.site +x86_64* +i686* +mips* +arm* +aarch64* diff --git a/contrib/depends/Makefile b/contrib/depends/Makefile new file mode 100644 index 00000000000..ec0e4cfae0f --- /dev/null +++ b/contrib/depends/Makefile @@ -0,0 +1,222 @@ +.NOTPARALLEL : + +SOURCES_PATH ?= $(BASEDIR)/sources +BASE_CACHE ?= $(BASEDIR)/built +SDK_PATH ?= $(BASEDIR)/SDKs +FALLBACK_DOWNLOAD_PATH ?= https://bitcoincore.org/depends-sources + +BUILD = $(shell ./config.guess) +HOST ?= $(BUILD) +PATCHES_PATH = $(BASEDIR)/patches +BASEDIR = $(CURDIR) +HASH_LENGTH:=11 +DOWNLOAD_CONNECT_TIMEOUT:=10 +DOWNLOAD_RETRIES:=3 +HOST_ID_SALT ?= salt +BUILD_ID_SALT ?= salt + +host:=$(BUILD) +ifneq ($(HOST),) +host:=$(HOST) +host_toolchain:=$(HOST)- +endif + +ifneq ($(DEBUG),) +release_type=Debug +else +release_type=Release +endif + +ifneq ($(TESTS),) +build_tests=ON +release_type=Debug +else +build_tests=OFF +endif + +base_build_dir=$(BASEDIR)/work/build +base_staging_dir=$(BASEDIR)/work/staging +base_download_dir=$(BASEDIR)/work/download +canonical_host:=$(shell ./config.sub $(HOST)) +build:=$(shell ./config.sub $(BUILD)) + +build_arch =$(firstword $(subst -, ,$(build))) +build_vendor=$(word 2,$(subst -, ,$(build))) +full_build_os:=$(subst $(build_arch)-$(build_vendor)-,,$(build)) +build_os:=$(findstring linux,$(full_build_os)) +build_os+=$(findstring darwin,$(full_build_os)) +build_os:=$(strip $(build_os)) +ifeq ($(build_os),) +build_os=$(full_build_os) +endif + +host_arch=$(firstword $(subst -, ,$(canonical_host))) +host_vendor=$(word 2,$(subst -, ,$(canonical_host))) +full_host_os:=$(subst $(host_arch)-$(host_vendor)-,,$(canonical_host)) +host_os:=$(findstring linux,$(full_host_os)) +host_os+=$(findstring darwin,$(full_host_os)) +host_os+=$(findstring mingw32,$(full_host_os)) +host_os:=$(strip $(host_os)) +ifeq ($(host_os),) +host_os=$(full_host_os) +endif + +$(host_arch)_$(host_os)_prefix=$(BASEDIR)/$(host) +$(host_arch)_$(host_os)_host=$(host) +host_prefix=$($(host_arch)_$(host_os)_prefix) +build_prefix=$(host_prefix)/native +ifeq ($(host_os),mingw32) +host_cmake=Windows +endif +ifeq ($(host_os),linux) +host_cmake=Linux +endif +ifeq ($(host_os),darwin) +host_cmake=Darwin +endif + +AT_$(V):= +AT_:=@ +AT:=$(AT_$(V)) + +all: install + +include hosts/$(host_os).mk +include hosts/default.mk +include builders/$(build_os).mk +include builders/default.mk +include packages/packages.mk + +build_id_string:=$(BUILD_ID_SALT) +build_id_string+=$(shell $(build_CC) --version 2>/dev/null) +build_id_string+=$(shell $(build_AR) --version 2>/dev/null) +build_id_string+=$(shell $(build_CXX) --version 2>/dev/null) +build_id_string+=$(shell $(build_RANLIB) --version 2>/dev/null) +build_id_string+=$(shell $(build_STRIP) --version 2>/dev/null) + +$(host_arch)_$(host_os)_id_string:=$(HOST_ID_SALT) +$(host_arch)_$(host_os)_id_string+=$(shell $(host_CC) --version 2>/dev/null) +$(host_arch)_$(host_os)_id_string+=$(shell $(host_AR) --version 2>/dev/null) +$(host_arch)_$(host_os)_id_string+=$(shell $(host_CXX) --version 2>/dev/null) +$(host_arch)_$(host_os)_id_string+=$(shell $(host_RANLIB) --version 2>/dev/null) +$(host_arch)_$(host_os)_id_string+=$(shell $(host_STRIP) --version 2>/dev/null) + +qt_packages_$(NO_QT) = $(qt_packages) +packages += $($(host_arch)_$(host_os)_packages) $($(host_os)_packages) $(qt_packages_) +native_packages += $($(host_arch)_$(host_os)_native_packages) $($(host_os)_native_packages) + +all_packages = $(packages) $(native_packages) + +meta_depends = Makefile funcs.mk builders/default.mk hosts/default.mk hosts/$(host_os).mk builders/$(build_os).mk + +$(host_arch)_$(host_os)_native_toolchain?=$($(host_os)_native_toolchain) + +include funcs.mk + +CONF_PKGS := cmake-conf mxe-conf + +build-only-$(1)_$(3): CMAKE_RUNRESULT_FILE = $(PREFIX)/share/cmake/modules/TryRunResults.cmake +build-only-$(1)_$(3): CMAKE_TOOLCHAIN_FILE = $(PREFIX)/$(3)/share/cmake/mxe-conf.cmake +build-only-$(1)_$(3): CMAKE_TOOLCHAIN_DIR = $(PREFIX)/$(3)/share/cmake/mxe-conf.d +build-only-$(1)_$(3): CMAKE_STATIC_BOOL = $(if $(findstring shared,$(3)),OFF,ON) +build-only-$(1)_$(3): CMAKE_SHARED_BOOL = $(if $(findstring shared,$(3)),ON,OFF) + + +toolchain_path=$($($(host_arch)_$(host_os)_native_toolchain)_prefixbin) +final_build_id_long+=$(shell $(build_SHA256SUM) config.site.in) +final_build_id_long+=$(shell $(build_SHA256SUM) toolchain.cmake.in) +final_build_id+=$(shell echo -n "$(final_build_id_long)" | $(build_SHA256SUM) | cut -c-$(HASH_LENGTH)) +$(host_prefix)/.stamp_$(final_build_id): $(native_packages) $(packages) + $(AT)rm -rf $(@D) + $(AT)mkdir -p $(@D) + $(AT)echo copying packages: $^ + $(AT)echo to: $(@D) + $(AT)cd $(@D); $(foreach package,$^, tar xf $($(package)_cached); ) + $(AT)touch $@ + +$(host_prefix)/share/config.site : config.site.in $(host_prefix)/.stamp_$(final_build_id) + $(AT)@mkdir -p $(@D) + $(AT)sed -e 's|@HOST@|$(host)|' \ + -e 's|@CC@|$(toolchain_path)$(host_CC)|' \ + -e 's|@CXX@|$(toolchain_path)$(host_CXX)|' \ + -e 's|@AR@|$(toolchain_path)$(host_AR)|' \ + -e 's|@RANLIB@|$(toolchain_path)$(host_RANLIB)|' \ + -e 's|@NM@|$(toolchain_path)$(host_NM)|' \ + -e 's|@STRIP@|$(toolchain_path)$(host_STRIP)|' \ + -e 's|@build_os@|$(build_os)|' \ + -e 's|@host_os@|$(host_os)|' \ + -e 's|@CFLAGS@|$(strip $(host_CFLAGS) $(host_$(release_type)_CFLAGS))|' \ + -e 's|@CXXFLAGS@|$(strip $(host_CXXFLAGS) $(host_$(release_type)_CXXFLAGS))|' \ + -e 's|@CPPFLAGS@|$(strip $(host_CPPFLAGS) $(host_$(release_type)_CPPFLAGS))|' \ + -e 's|@LDFLAGS@|$(strip $(host_LDFLAGS) $(host_$(release_type)_LDFLAGS))|' \ + -e 's|@allow_host_packages@|$(ALLOW_HOST_PACKAGES)|' \ + -e 's|@debug@|$(DEBUG)|' \ + $< > $@ + $(AT)touch $@ + +$(host_prefix)/share/toolchain.cmake : toolchain.cmake.in $(host_prefix)/.stamp_$(final_build_id) + $(AT)@mkdir -p $(@D) + $(AT)sed -e 's|@HOST@|$(host)|' \ + -e 's|@CC@|$(toolchain_path)$(host_CC)|' \ + -e 's|@CXX@|$(toolchain_path)$(host_CXX)|' \ + -e 's|@AR@|$(toolchain_path)$(host_AR)|' \ + -e 's|@RANLIB@|$(toolchain_path)$(host_RANLIB)|' \ + -e 's|@NM@|$(toolchain_path)$(host_NM)|' \ + -e 's|@STRIP@|$(toolchain_path)$(host_STRIP)|' \ + -e 's|@build_os@|$(build_os)|' \ + -e 's|@host_os@|$(host_os)|' \ + -e 's|@CFLAGS@|$(strip $(host_CFLAGS) $(host_$(release_type)_CFLAGS))|' \ + -e 's|@CXXFLAGS@|$(strip $(host_CXXFLAGS) $(host_$(release_type)_CXXFLAGS))|' \ + -e 's|@CPPFLAGS@|$(strip $(host_CPPFLAGS) $(host_$(release_type)_CPPFLAGS))|' \ + -e 's|@LDFLAGS@|$(strip $(host_LDFLAGS) $(host_$(release_type)_LDFLAGS))|' \ + -e 's|@allow_host_packages@|$(ALLOW_HOST_PACKAGES)|' \ + -e 's|@debug@|$(DEBUG)|' \ + -e 's|@release_type@|$(release_type)|' \ + -e 's|@build_tests@|$(build_tests)|' \ + -e 's|@depends@|$(host_cmake)|' \ + -e 's|@prefix@|$($(host_arch)_$(host_os)_prefix)|'\ + -e 's|@sdk@|$(SDK_PATH)|'\ + -e 's|@arch@|$(host_arch)|'\ + $< > $@ + $(AT)touch $@ + +define check_or_remove_cached + mkdir -p $(BASE_CACHE)/$(host)/$(package) && cd $(BASE_CACHE)/$(host)/$(package); \ + $(build_SHA256SUM) -c $($(package)_cached_checksum) >/dev/null 2>/dev/null || \ + ( rm -f $($(package)_cached_checksum); \ + if test -f "$($(package)_cached)"; then echo "Checksum mismatch for $(package). Forcing rebuild.."; rm -f $($(package)_cached_checksum) $($(package)_cached); fi ) +endef + +define check_or_remove_sources + mkdir -p $($(package)_source_dir); cd $($(package)_source_dir); \ + test -f $($(package)_fetched) && ( $(build_SHA256SUM) -c $($(package)_fetched) >/dev/null 2>/dev/null || \ + ( echo "Checksum missing or mismatched for $(package) source. Forcing re-download."; \ + rm -f $($(package)_all_sources) $($(1)_fetched))) || true +endef + +check-packages: + @$(foreach package,$(all_packages),$(call check_or_remove_cached,$(package));) +check-sources: + @$(foreach package,$(all_packages),$(call check_or_remove_sources,$(package));) + +$(host_prefix)/share/config.site: check-packages +$(host_prefix)/share/toolchain.cmake: check-packages + +check-packages: check-sources + +install: check-packages $(host_prefix)/share/config.site +install: check-packages $(host_prefix)/share/toolchain.cmake + +download-one: check-sources $(all_sources) + +download-osx: + @$(MAKE) -s HOST=x86_64-apple-darwin11 download-one +download-linux: + @$(MAKE) -s HOST=x86_64-unknown-linux-gnu download-one +download-win: + @$(MAKE) -s HOST=x86_64-w64-mingw32 download-one +download: download-osx download-linux download-win + + $(foreach package,$(all_packages),$(eval $(call ext_add_stages,$(package)))) + +.PHONY: install cached download-one download-osx download-linux download-win download check-packages check-sources diff --git a/contrib/depends/README.md b/contrib/depends/README.md new file mode 100644 index 00000000000..6a15055b48a --- /dev/null +++ b/contrib/depends/README.md @@ -0,0 +1,82 @@ +### Usage + +To build dependencies for the current arch+OS: + +```bash +make +``` + +To build for another arch/OS: + +```bash +make HOST=host-platform-triplet +``` + +For example: + +```bash +make HOST=x86_64-w64-mingw32 -j4 +``` + +A toolchain will be generated that's suitable for plugging into Monero's +cmake. In the above example, a dir named x86_64-w64-mingw32 will be +created. To use it for Monero: + +```bash +cmake -DCMAKE_TOOLCHAIN=`pwd`/contrib/depends/x86_64-w64-mingw32 +``` + +Common `host-platform-triplets` for cross compilation are: + +- `i686-w64-mingw32` for Win32 +- `x86_64-w64-mingw32` for Win64 +- `x86_64-apple-darwin11` for MacOSX +- `arm-linux-gnueabihf` for Linux ARM 32 bit +- `aarch64-linux-gnu` for Linux ARM 64 bit + +No other options are needed, the paths are automatically configured. + +Dependency Options: +The following can be set when running make: make FOO=bar + +``` +SOURCES_PATH: downloaded sources will be placed here +BASE_CACHE: built packages will be placed here +SDK_PATH: Path where sdk's can be found (used by OSX) +FALLBACK_DOWNLOAD_PATH: If a source file can't be fetched, try here before giving up +DEBUG: disable some optimizations and enable more runtime checking +HOST_ID_SALT: Optional salt to use when generating host package ids +BUILD_ID_SALT: Optional salt to use when generating build package ids +``` + +Additional targets: + +``` +download: run 'make download' to fetch all sources without building them +download-osx: run 'make download-osx' to fetch all sources needed for osx builds +download-win: run 'make download-win' to fetch all sources needed for win builds +download-linux: run 'make download-linux' to fetch all sources needed for linux builds +``` + +#Darwin (macos) builds: + +To build with the x86_64-apple-darwin11 you require the mac os developer tools in MacOSX10.11.sdk. +Download it from apple, or search for it on github. Create a new directoty called SDKs in this +directory and place the entire MacOSX10.11.sdk folder in it. The depends build will then pick it up automatically +(without requiring SDK_PATH). + + +#Mingw builds + +Building for 32/64bit mingw requires switching alternatives to a posix mode + +```bash +update-alternatives --set x86_64-w64-mingw32-g++ x86_64-w64-mingw32-g++-posix +update-alternatives --set x86_64-w64-mingw32-gcc x86_64-w64-mingw32-gcc-posix +``` + +### Other documentation + +- [description.md](description.md): General description of the depends system +- [packages.md](packages.md): Steps for adding packages + diff --git a/contrib/depends/builders/darwin.mk b/contrib/depends/builders/darwin.mk new file mode 100644 index 00000000000..27f550ab036 --- /dev/null +++ b/contrib/depends/builders/darwin.mk @@ -0,0 +1,22 @@ +build_darwin_CC: = $(shell xcrun -f clang) +build_darwin_CXX: = $(shell xcrun -f clang++) +build_darwin_AR: = $(shell xcrun -f ar) +build_darwin_RANLIB: = $(shell xcrun -f ranlib) +build_darwin_STRIP: = $(shell xcrun -f strip) +build_darwin_OTOOL: = $(shell xcrun -f otool) +build_darwin_NM: = $(shell xcrun -f nm) +build_darwin_INSTALL_NAME_TOOL:=$(shell xcrun -f install_name_tool) +build_darwin_SHA256SUM = shasum -a 256 +build_darwin_DOWNLOAD = curl --location --fail --connect-timeout $(DOWNLOAD_CONNECT_TIMEOUT) --retry $(DOWNLOAD_RETRIES) -o + +#darwin host on darwin builder. overrides darwin host preferences. +darwin_CC=$(shell xcrun -f clang) -mmacosx-version-min=$(OSX_MIN_VERSION) +darwin_CXX:=$(shell xcrun -f clang++) -mmacosx-version-min=$(OSX_MIN_VERSION) -stdlib=libc++ +darwin_AR:=$(shell xcrun -f ar) +darwin_RANLIB:=$(shell xcrun -f ranlib) +darwin_STRIP:=$(shell xcrun -f strip) +darwin_LIBTOOL:=$(shell xcrun -f libtool) +darwin_OTOOL:=$(shell xcrun -f otool) +darwin_NM:=$(shell xcrun -f nm) +darwin_INSTALL_NAME_TOOL:=$(shell xcrun -f install_name_tool) +darwin_native_toolchain= diff --git a/contrib/depends/builders/default.mk b/contrib/depends/builders/default.mk new file mode 100644 index 00000000000..f097db65d60 --- /dev/null +++ b/contrib/depends/builders/default.mk @@ -0,0 +1,20 @@ +default_build_CC = gcc +default_build_CXX = g++ +default_build_AR = ar +default_build_RANLIB = ranlib +default_build_STRIP = strip +default_build_NM = nm +default_build_OTOOL = otool +default_build_INSTALL_NAME_TOOL = install_name_tool + +define add_build_tool_func +build_$(build_os)_$1 ?= $$(default_build_$1) +build_$(build_arch)_$(build_os)_$1 ?= $$(build_$(build_os)_$1) +build_$1=$$(build_$(build_arch)_$(build_os)_$1) +endef +$(foreach var,CC CXX AR RANLIB NM STRIP SHA256SUM DOWNLOAD OTOOL INSTALL_NAME_TOOL,$(eval $(call add_build_tool_func,$(var)))) +define add_build_flags_func +build_$(build_arch)_$(build_os)_$1 += $(build_$(build_os)_$1) +build_$1=$$(build_$(build_arch)_$(build_os)_$1) +endef +$(foreach flags, CFLAGS CXXFLAGS LDFLAGS, $(eval $(call add_build_flags_func,$(flags)))) diff --git a/contrib/depends/builders/linux.mk b/contrib/depends/builders/linux.mk new file mode 100644 index 00000000000..b03f4240104 --- /dev/null +++ b/contrib/depends/builders/linux.mk @@ -0,0 +1,2 @@ +build_linux_SHA256SUM = sha256sum +build_linux_DOWNLOAD = curl --location --fail --connect-timeout $(DOWNLOAD_CONNECT_TIMEOUT) --retry $(DOWNLOAD_RETRIES) -o diff --git a/contrib/depends/config.guess b/contrib/depends/config.guess new file mode 100755 index 00000000000..69ed3e573bb --- /dev/null +++ b/contrib/depends/config.guess @@ -0,0 +1,1466 @@ +#! /bin/sh +# Attempt to guess a canonical system name. +# Copyright 1992-2017 Free Software Foundation, Inc. + +timestamp='2017-03-05' + +# This file is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, see . +# +# As a special exception to the GNU General Public License, if you +# distribute this file as part of a program that contains a +# configuration script generated by Autoconf, you may include it under +# the same distribution terms that you use for the rest of that +# program. This Exception is an additional permission under section 7 +# of the GNU General Public License, version 3 ("GPLv3"). +# +# Originally written by Per Bothner; maintained since 2000 by Ben Elliston. +# +# You can get the latest version of this script from: +# http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess +# +# Please send patches to . + + +me=`echo "$0" | sed -e 's,.*/,,'` + +usage="\ +Usage: $0 [OPTION] + +Output the configuration name of the system \`$me' is run on. + +Operation modes: + -h, --help print this help, then exit + -t, --time-stamp print date of last modification, then exit + -v, --version print version number, then exit + +Report bugs and patches to ." + +version="\ +GNU config.guess ($timestamp) + +Originally written by Per Bothner. +Copyright 1992-2017 Free Software Foundation, Inc. + +This is free software; see the source for copying conditions. There is NO +warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE." + +help=" +Try \`$me --help' for more information." + +# Parse command line +while test $# -gt 0 ; do + case $1 in + --time-stamp | --time* | -t ) + echo "$timestamp" ; exit ;; + --version | -v ) + echo "$version" ; exit ;; + --help | --h* | -h ) + echo "$usage"; exit ;; + -- ) # Stop option processing + shift; break ;; + - ) # Use stdin as input. + break ;; + -* ) + echo "$me: invalid option $1$help" >&2 + exit 1 ;; + * ) + break ;; + esac +done + +if test $# != 0; then + echo "$me: too many arguments$help" >&2 + exit 1 +fi + +trap 'exit 1' 1 2 15 + +# CC_FOR_BUILD -- compiler used by this script. Note that the use of a +# compiler to aid in system detection is discouraged as it requires +# temporary files to be created and, as you can see below, it is a +# headache to deal with in a portable fashion. + +# Historically, `CC_FOR_BUILD' used to be named `HOST_CC'. We still +# use `HOST_CC' if defined, but it is deprecated. + +# Portable tmp directory creation inspired by the Autoconf team. + +set_cc_for_build=' +trap "exitcode=\$?; (rm -f \$tmpfiles 2>/dev/null; rmdir \$tmp 2>/dev/null) && exit \$exitcode" 0 ; +trap "rm -f \$tmpfiles 2>/dev/null; rmdir \$tmp 2>/dev/null; exit 1" 1 2 13 15 ; +: ${TMPDIR=/tmp} ; + { tmp=`(umask 077 && mktemp -d "$TMPDIR/cgXXXXXX") 2>/dev/null` && test -n "$tmp" && test -d "$tmp" ; } || + { test -n "$RANDOM" && tmp=$TMPDIR/cg$$-$RANDOM && (umask 077 && mkdir $tmp) ; } || + { tmp=$TMPDIR/cg-$$ && (umask 077 && mkdir $tmp) && echo "Warning: creating insecure temp directory" >&2 ; } || + { echo "$me: cannot create a temporary directory in $TMPDIR" >&2 ; exit 1 ; } ; +dummy=$tmp/dummy ; +tmpfiles="$dummy.c $dummy.o $dummy.rel $dummy" ; +case $CC_FOR_BUILD,$HOST_CC,$CC in + ,,) echo "int x;" > $dummy.c ; + for c in cc gcc c89 c99 ; do + if ($c -c -o $dummy.o $dummy.c) >/dev/null 2>&1 ; then + CC_FOR_BUILD="$c"; break ; + fi ; + done ; + if test x"$CC_FOR_BUILD" = x ; then + CC_FOR_BUILD=no_compiler_found ; + fi + ;; + ,,*) CC_FOR_BUILD=$CC ;; + ,*,*) CC_FOR_BUILD=$HOST_CC ;; +esac ; set_cc_for_build= ;' + +# This is needed to find uname on a Pyramid OSx when run in the BSD universe. +# (ghazi@noc.rutgers.edu 1994-08-24) +if (test -f /.attbin/uname) >/dev/null 2>&1 ; then + PATH=$PATH:/.attbin ; export PATH +fi + +UNAME_MACHINE=`(uname -m) 2>/dev/null` || UNAME_MACHINE=unknown +UNAME_RELEASE=`(uname -r) 2>/dev/null` || UNAME_RELEASE=unknown +UNAME_SYSTEM=`(uname -s) 2>/dev/null` || UNAME_SYSTEM=unknown +UNAME_VERSION=`(uname -v) 2>/dev/null` || UNAME_VERSION=unknown + +case "${UNAME_SYSTEM}" in +Linux|GNU|GNU/*) + # If the system lacks a compiler, then just pick glibc. + # We could probably try harder. + LIBC=gnu + + eval $set_cc_for_build + cat <<-EOF > $dummy.c + #include + #if defined(__UCLIBC__) + LIBC=uclibc + #elif defined(__dietlibc__) + LIBC=dietlibc + #else + LIBC=gnu + #endif + EOF + eval `$CC_FOR_BUILD -E $dummy.c 2>/dev/null | grep '^LIBC' | sed 's, ,,g'` + ;; +esac + +# Note: order is significant - the case branches are not exclusive. + +case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in + *:NetBSD:*:*) + # NetBSD (nbsd) targets should (where applicable) match one or + # more of the tuples: *-*-netbsdelf*, *-*-netbsdaout*, + # *-*-netbsdecoff* and *-*-netbsd*. For targets that recently + # switched to ELF, *-*-netbsd* would select the old + # object file format. This provides both forward + # compatibility and a consistent mechanism for selecting the + # object file format. + # + # Note: NetBSD doesn't particularly care about the vendor + # portion of the name. We always set it to "unknown". + sysctl="sysctl -n hw.machine_arch" + UNAME_MACHINE_ARCH=`(uname -p 2>/dev/null || \ + /sbin/$sysctl 2>/dev/null || \ + /usr/sbin/$sysctl 2>/dev/null || \ + echo unknown)` + case "${UNAME_MACHINE_ARCH}" in + armeb) machine=armeb-unknown ;; + arm*) machine=arm-unknown ;; + sh3el) machine=shl-unknown ;; + sh3eb) machine=sh-unknown ;; + sh5el) machine=sh5le-unknown ;; + earmv*) + arch=`echo ${UNAME_MACHINE_ARCH} | sed -e 's,^e\(armv[0-9]\).*$,\1,'` + endian=`echo ${UNAME_MACHINE_ARCH} | sed -ne 's,^.*\(eb\)$,\1,p'` + machine=${arch}${endian}-unknown + ;; + *) machine=${UNAME_MACHINE_ARCH}-unknown ;; + esac + # The Operating System including object format, if it has switched + # to ELF recently (or will in the future) and ABI. + case "${UNAME_MACHINE_ARCH}" in + earm*) + os=netbsdelf + ;; + arm*|i386|m68k|ns32k|sh3*|sparc|vax) + eval $set_cc_for_build + if echo __ELF__ | $CC_FOR_BUILD -E - 2>/dev/null \ + | grep -q __ELF__ + then + # Once all utilities can be ECOFF (netbsdecoff) or a.out (netbsdaout). + # Return netbsd for either. FIX? + os=netbsd + else + os=netbsdelf + fi + ;; + *) + os=netbsd + ;; + esac + # Determine ABI tags. + case "${UNAME_MACHINE_ARCH}" in + earm*) + expr='s/^earmv[0-9]/-eabi/;s/eb$//' + abi=`echo ${UNAME_MACHINE_ARCH} | sed -e "$expr"` + ;; + esac + # The OS release + # Debian GNU/NetBSD machines have a different userland, and + # thus, need a distinct triplet. However, they do not need + # kernel version information, so it can be replaced with a + # suitable tag, in the style of linux-gnu. + case "${UNAME_VERSION}" in + Debian*) + release='-gnu' + ;; + *) + release=`echo ${UNAME_RELEASE} | sed -e 's/[-_].*//' | cut -d. -f1,2` + ;; + esac + # Since CPU_TYPE-MANUFACTURER-KERNEL-OPERATING_SYSTEM: + # contains redundant information, the shorter form: + # CPU_TYPE-MANUFACTURER-OPERATING_SYSTEM is used. + echo "${machine}-${os}${release}${abi}" + exit ;; + *:Bitrig:*:*) + UNAME_MACHINE_ARCH=`arch | sed 's/Bitrig.//'` + echo ${UNAME_MACHINE_ARCH}-unknown-bitrig${UNAME_RELEASE} + exit ;; + *:OpenBSD:*:*) + UNAME_MACHINE_ARCH=`arch | sed 's/OpenBSD.//'` + echo ${UNAME_MACHINE_ARCH}-unknown-openbsd${UNAME_RELEASE} + exit ;; + *:LibertyBSD:*:*) + UNAME_MACHINE_ARCH=`arch | sed 's/^.*BSD\.//'` + echo ${UNAME_MACHINE_ARCH}-unknown-libertybsd${UNAME_RELEASE} + exit ;; + *:ekkoBSD:*:*) + echo ${UNAME_MACHINE}-unknown-ekkobsd${UNAME_RELEASE} + exit ;; + *:SolidBSD:*:*) + echo ${UNAME_MACHINE}-unknown-solidbsd${UNAME_RELEASE} + exit ;; + macppc:MirBSD:*:*) + echo powerpc-unknown-mirbsd${UNAME_RELEASE} + exit ;; + *:MirBSD:*:*) + echo ${UNAME_MACHINE}-unknown-mirbsd${UNAME_RELEASE} + exit ;; + *:Sortix:*:*) + echo ${UNAME_MACHINE}-unknown-sortix + exit ;; + alpha:OSF1:*:*) + case $UNAME_RELEASE in + *4.0) + UNAME_RELEASE=`/usr/sbin/sizer -v | awk '{print $3}'` + ;; + *5.*) + UNAME_RELEASE=`/usr/sbin/sizer -v | awk '{print $4}'` + ;; + esac + # According to Compaq, /usr/sbin/psrinfo has been available on + # OSF/1 and Tru64 systems produced since 1995. I hope that + # covers most systems running today. This code pipes the CPU + # types through head -n 1, so we only detect the type of CPU 0. + ALPHA_CPU_TYPE=`/usr/sbin/psrinfo -v | sed -n -e 's/^ The alpha \(.*\) processor.*$/\1/p' | head -n 1` + case "$ALPHA_CPU_TYPE" in + "EV4 (21064)") + UNAME_MACHINE=alpha ;; + "EV4.5 (21064)") + UNAME_MACHINE=alpha ;; + "LCA4 (21066/21068)") + UNAME_MACHINE=alpha ;; + "EV5 (21164)") + UNAME_MACHINE=alphaev5 ;; + "EV5.6 (21164A)") + UNAME_MACHINE=alphaev56 ;; + "EV5.6 (21164PC)") + UNAME_MACHINE=alphapca56 ;; + "EV5.7 (21164PC)") + UNAME_MACHINE=alphapca57 ;; + "EV6 (21264)") + UNAME_MACHINE=alphaev6 ;; + "EV6.7 (21264A)") + UNAME_MACHINE=alphaev67 ;; + "EV6.8CB (21264C)") + UNAME_MACHINE=alphaev68 ;; + "EV6.8AL (21264B)") + UNAME_MACHINE=alphaev68 ;; + "EV6.8CX (21264D)") + UNAME_MACHINE=alphaev68 ;; + "EV6.9A (21264/EV69A)") + UNAME_MACHINE=alphaev69 ;; + "EV7 (21364)") + UNAME_MACHINE=alphaev7 ;; + "EV7.9 (21364A)") + UNAME_MACHINE=alphaev79 ;; + esac + # A Pn.n version is a patched version. + # A Vn.n version is a released version. + # A Tn.n version is a released field test version. + # A Xn.n version is an unreleased experimental baselevel. + # 1.2 uses "1.2" for uname -r. + echo ${UNAME_MACHINE}-dec-osf`echo ${UNAME_RELEASE} | sed -e 's/^[PVTX]//' | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz` + # Reset EXIT trap before exiting to avoid spurious non-zero exit code. + exitcode=$? + trap '' 0 + exit $exitcode ;; + Alpha\ *:Windows_NT*:*) + # How do we know it's Interix rather than the generic POSIX subsystem? + # Should we change UNAME_MACHINE based on the output of uname instead + # of the specific Alpha model? + echo alpha-pc-interix + exit ;; + 21064:Windows_NT:50:3) + echo alpha-dec-winnt3.5 + exit ;; + Amiga*:UNIX_System_V:4.0:*) + echo m68k-unknown-sysv4 + exit ;; + *:[Aa]miga[Oo][Ss]:*:*) + echo ${UNAME_MACHINE}-unknown-amigaos + exit ;; + *:[Mm]orph[Oo][Ss]:*:*) + echo ${UNAME_MACHINE}-unknown-morphos + exit ;; + *:OS/390:*:*) + echo i370-ibm-openedition + exit ;; + *:z/VM:*:*) + echo s390-ibm-zvmoe + exit ;; + *:OS400:*:*) + echo powerpc-ibm-os400 + exit ;; + arm:RISC*:1.[012]*:*|arm:riscix:1.[012]*:*) + echo arm-acorn-riscix${UNAME_RELEASE} + exit ;; + arm*:riscos:*:*|arm*:RISCOS:*:*) + echo arm-unknown-riscos + exit ;; + SR2?01:HI-UX/MPP:*:* | SR8000:HI-UX/MPP:*:*) + echo hppa1.1-hitachi-hiuxmpp + exit ;; + Pyramid*:OSx*:*:* | MIS*:OSx*:*:* | MIS*:SMP_DC-OSx*:*:*) + # akee@wpdis03.wpafb.af.mil (Earle F. Ake) contributed MIS and NILE. + if test "`(/bin/universe) 2>/dev/null`" = att ; then + echo pyramid-pyramid-sysv3 + else + echo pyramid-pyramid-bsd + fi + exit ;; + NILE*:*:*:dcosx) + echo pyramid-pyramid-svr4 + exit ;; + DRS?6000:unix:4.0:6*) + echo sparc-icl-nx6 + exit ;; + DRS?6000:UNIX_SV:4.2*:7* | DRS?6000:isis:4.2*:7*) + case `/usr/bin/uname -p` in + sparc) echo sparc-icl-nx7; exit ;; + esac ;; + s390x:SunOS:*:*) + echo ${UNAME_MACHINE}-ibm-solaris2`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'` + exit ;; + sun4H:SunOS:5.*:*) + echo sparc-hal-solaris2`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'` + exit ;; + sun4*:SunOS:5.*:* | tadpole*:SunOS:5.*:*) + echo sparc-sun-solaris2`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'` + exit ;; + i86pc:AuroraUX:5.*:* | i86xen:AuroraUX:5.*:*) + echo i386-pc-auroraux${UNAME_RELEASE} + exit ;; + i86pc:SunOS:5.*:* | i86xen:SunOS:5.*:*) + eval $set_cc_for_build + SUN_ARCH=i386 + # If there is a compiler, see if it is configured for 64-bit objects. + # Note that the Sun cc does not turn __LP64__ into 1 like gcc does. + # This test works for both compilers. + if [ "$CC_FOR_BUILD" != no_compiler_found ]; then + if (echo '#ifdef __amd64'; echo IS_64BIT_ARCH; echo '#endif') | \ + (CCOPTS="" $CC_FOR_BUILD -E - 2>/dev/null) | \ + grep IS_64BIT_ARCH >/dev/null + then + SUN_ARCH=x86_64 + fi + fi + echo ${SUN_ARCH}-pc-solaris2`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'` + exit ;; + sun4*:SunOS:6*:*) + # According to config.sub, this is the proper way to canonicalize + # SunOS6. Hard to guess exactly what SunOS6 will be like, but + # it's likely to be more like Solaris than SunOS4. + echo sparc-sun-solaris3`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'` + exit ;; + sun4*:SunOS:*:*) + case "`/usr/bin/arch -k`" in + Series*|S4*) + UNAME_RELEASE=`uname -v` + ;; + esac + # Japanese Language versions have a version number like `4.1.3-JL'. + echo sparc-sun-sunos`echo ${UNAME_RELEASE}|sed -e 's/-/_/'` + exit ;; + sun3*:SunOS:*:*) + echo m68k-sun-sunos${UNAME_RELEASE} + exit ;; + sun*:*:4.2BSD:*) + UNAME_RELEASE=`(sed 1q /etc/motd | awk '{print substr($5,1,3)}') 2>/dev/null` + test "x${UNAME_RELEASE}" = x && UNAME_RELEASE=3 + case "`/bin/arch`" in + sun3) + echo m68k-sun-sunos${UNAME_RELEASE} + ;; + sun4) + echo sparc-sun-sunos${UNAME_RELEASE} + ;; + esac + exit ;; + aushp:SunOS:*:*) + echo sparc-auspex-sunos${UNAME_RELEASE} + exit ;; + # The situation for MiNT is a little confusing. The machine name + # can be virtually everything (everything which is not + # "atarist" or "atariste" at least should have a processor + # > m68000). The system name ranges from "MiNT" over "FreeMiNT" + # to the lowercase version "mint" (or "freemint"). Finally + # the system name "TOS" denotes a system which is actually not + # MiNT. But MiNT is downward compatible to TOS, so this should + # be no problem. + atarist[e]:*MiNT:*:* | atarist[e]:*mint:*:* | atarist[e]:*TOS:*:*) + echo m68k-atari-mint${UNAME_RELEASE} + exit ;; + atari*:*MiNT:*:* | atari*:*mint:*:* | atarist[e]:*TOS:*:*) + echo m68k-atari-mint${UNAME_RELEASE} + exit ;; + *falcon*:*MiNT:*:* | *falcon*:*mint:*:* | *falcon*:*TOS:*:*) + echo m68k-atari-mint${UNAME_RELEASE} + exit ;; + milan*:*MiNT:*:* | milan*:*mint:*:* | *milan*:*TOS:*:*) + echo m68k-milan-mint${UNAME_RELEASE} + exit ;; + hades*:*MiNT:*:* | hades*:*mint:*:* | *hades*:*TOS:*:*) + echo m68k-hades-mint${UNAME_RELEASE} + exit ;; + *:*MiNT:*:* | *:*mint:*:* | *:*TOS:*:*) + echo m68k-unknown-mint${UNAME_RELEASE} + exit ;; + m68k:machten:*:*) + echo m68k-apple-machten${UNAME_RELEASE} + exit ;; + powerpc:machten:*:*) + echo powerpc-apple-machten${UNAME_RELEASE} + exit ;; + RISC*:Mach:*:*) + echo mips-dec-mach_bsd4.3 + exit ;; + RISC*:ULTRIX:*:*) + echo mips-dec-ultrix${UNAME_RELEASE} + exit ;; + VAX*:ULTRIX*:*:*) + echo vax-dec-ultrix${UNAME_RELEASE} + exit ;; + 2020:CLIX:*:* | 2430:CLIX:*:*) + echo clipper-intergraph-clix${UNAME_RELEASE} + exit ;; + mips:*:*:UMIPS | mips:*:*:RISCos) + eval $set_cc_for_build + sed 's/^ //' << EOF >$dummy.c +#ifdef __cplusplus +#include /* for printf() prototype */ + int main (int argc, char *argv[]) { +#else + int main (argc, argv) int argc; char *argv[]; { +#endif + #if defined (host_mips) && defined (MIPSEB) + #if defined (SYSTYPE_SYSV) + printf ("mips-mips-riscos%ssysv\n", argv[1]); exit (0); + #endif + #if defined (SYSTYPE_SVR4) + printf ("mips-mips-riscos%ssvr4\n", argv[1]); exit (0); + #endif + #if defined (SYSTYPE_BSD43) || defined(SYSTYPE_BSD) + printf ("mips-mips-riscos%sbsd\n", argv[1]); exit (0); + #endif + #endif + exit (-1); + } +EOF + $CC_FOR_BUILD -o $dummy $dummy.c && + dummyarg=`echo "${UNAME_RELEASE}" | sed -n 's/\([0-9]*\).*/\1/p'` && + SYSTEM_NAME=`$dummy $dummyarg` && + { echo "$SYSTEM_NAME"; exit; } + echo mips-mips-riscos${UNAME_RELEASE} + exit ;; + Motorola:PowerMAX_OS:*:*) + echo powerpc-motorola-powermax + exit ;; + Motorola:*:4.3:PL8-*) + echo powerpc-harris-powermax + exit ;; + Night_Hawk:*:*:PowerMAX_OS | Synergy:PowerMAX_OS:*:*) + echo powerpc-harris-powermax + exit ;; + Night_Hawk:Power_UNIX:*:*) + echo powerpc-harris-powerunix + exit ;; + m88k:CX/UX:7*:*) + echo m88k-harris-cxux7 + exit ;; + m88k:*:4*:R4*) + echo m88k-motorola-sysv4 + exit ;; + m88k:*:3*:R3*) + echo m88k-motorola-sysv3 + exit ;; + AViiON:dgux:*:*) + # DG/UX returns AViiON for all architectures + UNAME_PROCESSOR=`/usr/bin/uname -p` + if [ $UNAME_PROCESSOR = mc88100 ] || [ $UNAME_PROCESSOR = mc88110 ] + then + if [ ${TARGET_BINARY_INTERFACE}x = m88kdguxelfx ] || \ + [ ${TARGET_BINARY_INTERFACE}x = x ] + then + echo m88k-dg-dgux${UNAME_RELEASE} + else + echo m88k-dg-dguxbcs${UNAME_RELEASE} + fi + else + echo i586-dg-dgux${UNAME_RELEASE} + fi + exit ;; + M88*:DolphinOS:*:*) # DolphinOS (SVR3) + echo m88k-dolphin-sysv3 + exit ;; + M88*:*:R3*:*) + # Delta 88k system running SVR3 + echo m88k-motorola-sysv3 + exit ;; + XD88*:*:*:*) # Tektronix XD88 system running UTekV (SVR3) + echo m88k-tektronix-sysv3 + exit ;; + Tek43[0-9][0-9]:UTek:*:*) # Tektronix 4300 system running UTek (BSD) + echo m68k-tektronix-bsd + exit ;; + *:IRIX*:*:*) + echo mips-sgi-irix`echo ${UNAME_RELEASE}|sed -e 's/-/_/g'` + exit ;; + ????????:AIX?:[12].1:2) # AIX 2.2.1 or AIX 2.1.1 is RT/PC AIX. + echo romp-ibm-aix # uname -m gives an 8 hex-code CPU id + exit ;; # Note that: echo "'`uname -s`'" gives 'AIX ' + i*86:AIX:*:*) + echo i386-ibm-aix + exit ;; + ia64:AIX:*:*) + if [ -x /usr/bin/oslevel ] ; then + IBM_REV=`/usr/bin/oslevel` + else + IBM_REV=${UNAME_VERSION}.${UNAME_RELEASE} + fi + echo ${UNAME_MACHINE}-ibm-aix${IBM_REV} + exit ;; + *:AIX:2:3) + if grep bos325 /usr/include/stdio.h >/dev/null 2>&1; then + eval $set_cc_for_build + sed 's/^ //' << EOF >$dummy.c + #include + + main() + { + if (!__power_pc()) + exit(1); + puts("powerpc-ibm-aix3.2.5"); + exit(0); + } +EOF + if $CC_FOR_BUILD -o $dummy $dummy.c && SYSTEM_NAME=`$dummy` + then + echo "$SYSTEM_NAME" + else + echo rs6000-ibm-aix3.2.5 + fi + elif grep bos324 /usr/include/stdio.h >/dev/null 2>&1; then + echo rs6000-ibm-aix3.2.4 + else + echo rs6000-ibm-aix3.2 + fi + exit ;; + *:AIX:*:[4567]) + IBM_CPU_ID=`/usr/sbin/lsdev -C -c processor -S available | sed 1q | awk '{ print $1 }'` + if /usr/sbin/lsattr -El ${IBM_CPU_ID} | grep ' POWER' >/dev/null 2>&1; then + IBM_ARCH=rs6000 + else + IBM_ARCH=powerpc + fi + if [ -x /usr/bin/lslpp ] ; then + IBM_REV=`/usr/bin/lslpp -Lqc bos.rte.libc | + awk -F: '{ print $3 }' | sed s/[0-9]*$/0/` + else + IBM_REV=${UNAME_VERSION}.${UNAME_RELEASE} + fi + echo ${IBM_ARCH}-ibm-aix${IBM_REV} + exit ;; + *:AIX:*:*) + echo rs6000-ibm-aix + exit ;; + ibmrt:4.4BSD:*|romp-ibm:BSD:*) + echo romp-ibm-bsd4.4 + exit ;; + ibmrt:*BSD:*|romp-ibm:BSD:*) # covers RT/PC BSD and + echo romp-ibm-bsd${UNAME_RELEASE} # 4.3 with uname added to + exit ;; # report: romp-ibm BSD 4.3 + *:BOSX:*:*) + echo rs6000-bull-bosx + exit ;; + DPX/2?00:B.O.S.:*:*) + echo m68k-bull-sysv3 + exit ;; + 9000/[34]??:4.3bsd:1.*:*) + echo m68k-hp-bsd + exit ;; + hp300:4.4BSD:*:* | 9000/[34]??:4.3bsd:2.*:*) + echo m68k-hp-bsd4.4 + exit ;; + 9000/[34678]??:HP-UX:*:*) + HPUX_REV=`echo ${UNAME_RELEASE}|sed -e 's/[^.]*.[0B]*//'` + case "${UNAME_MACHINE}" in + 9000/31? ) HP_ARCH=m68000 ;; + 9000/[34]?? ) HP_ARCH=m68k ;; + 9000/[678][0-9][0-9]) + if [ -x /usr/bin/getconf ]; then + sc_cpu_version=`/usr/bin/getconf SC_CPU_VERSION 2>/dev/null` + sc_kernel_bits=`/usr/bin/getconf SC_KERNEL_BITS 2>/dev/null` + case "${sc_cpu_version}" in + 523) HP_ARCH=hppa1.0 ;; # CPU_PA_RISC1_0 + 528) HP_ARCH=hppa1.1 ;; # CPU_PA_RISC1_1 + 532) # CPU_PA_RISC2_0 + case "${sc_kernel_bits}" in + 32) HP_ARCH=hppa2.0n ;; + 64) HP_ARCH=hppa2.0w ;; + '') HP_ARCH=hppa2.0 ;; # HP-UX 10.20 + esac ;; + esac + fi + if [ "${HP_ARCH}" = "" ]; then + eval $set_cc_for_build + sed 's/^ //' << EOF >$dummy.c + + #define _HPUX_SOURCE + #include + #include + + int main () + { + #if defined(_SC_KERNEL_BITS) + long bits = sysconf(_SC_KERNEL_BITS); + #endif + long cpu = sysconf (_SC_CPU_VERSION); + + switch (cpu) + { + case CPU_PA_RISC1_0: puts ("hppa1.0"); break; + case CPU_PA_RISC1_1: puts ("hppa1.1"); break; + case CPU_PA_RISC2_0: + #if defined(_SC_KERNEL_BITS) + switch (bits) + { + case 64: puts ("hppa2.0w"); break; + case 32: puts ("hppa2.0n"); break; + default: puts ("hppa2.0"); break; + } break; + #else /* !defined(_SC_KERNEL_BITS) */ + puts ("hppa2.0"); break; + #endif + default: puts ("hppa1.0"); break; + } + exit (0); + } +EOF + (CCOPTS="" $CC_FOR_BUILD -o $dummy $dummy.c 2>/dev/null) && HP_ARCH=`$dummy` + test -z "$HP_ARCH" && HP_ARCH=hppa + fi ;; + esac + if [ ${HP_ARCH} = hppa2.0w ] + then + eval $set_cc_for_build + + # hppa2.0w-hp-hpux* has a 64-bit kernel and a compiler generating + # 32-bit code. hppa64-hp-hpux* has the same kernel and a compiler + # generating 64-bit code. GNU and HP use different nomenclature: + # + # $ CC_FOR_BUILD=cc ./config.guess + # => hppa2.0w-hp-hpux11.23 + # $ CC_FOR_BUILD="cc +DA2.0w" ./config.guess + # => hppa64-hp-hpux11.23 + + if echo __LP64__ | (CCOPTS="" $CC_FOR_BUILD -E - 2>/dev/null) | + grep -q __LP64__ + then + HP_ARCH=hppa2.0w + else + HP_ARCH=hppa64 + fi + fi + echo ${HP_ARCH}-hp-hpux${HPUX_REV} + exit ;; + ia64:HP-UX:*:*) + HPUX_REV=`echo ${UNAME_RELEASE}|sed -e 's/[^.]*.[0B]*//'` + echo ia64-hp-hpux${HPUX_REV} + exit ;; + 3050*:HI-UX:*:*) + eval $set_cc_for_build + sed 's/^ //' << EOF >$dummy.c + #include + int + main () + { + long cpu = sysconf (_SC_CPU_VERSION); + /* The order matters, because CPU_IS_HP_MC68K erroneously returns + true for CPU_PA_RISC1_0. CPU_IS_PA_RISC returns correct + results, however. */ + if (CPU_IS_PA_RISC (cpu)) + { + switch (cpu) + { + case CPU_PA_RISC1_0: puts ("hppa1.0-hitachi-hiuxwe2"); break; + case CPU_PA_RISC1_1: puts ("hppa1.1-hitachi-hiuxwe2"); break; + case CPU_PA_RISC2_0: puts ("hppa2.0-hitachi-hiuxwe2"); break; + default: puts ("hppa-hitachi-hiuxwe2"); break; + } + } + else if (CPU_IS_HP_MC68K (cpu)) + puts ("m68k-hitachi-hiuxwe2"); + else puts ("unknown-hitachi-hiuxwe2"); + exit (0); + } +EOF + $CC_FOR_BUILD -o $dummy $dummy.c && SYSTEM_NAME=`$dummy` && + { echo "$SYSTEM_NAME"; exit; } + echo unknown-hitachi-hiuxwe2 + exit ;; + 9000/7??:4.3bsd:*:* | 9000/8?[79]:4.3bsd:*:* ) + echo hppa1.1-hp-bsd + exit ;; + 9000/8??:4.3bsd:*:*) + echo hppa1.0-hp-bsd + exit ;; + *9??*:MPE/iX:*:* | *3000*:MPE/iX:*:*) + echo hppa1.0-hp-mpeix + exit ;; + hp7??:OSF1:*:* | hp8?[79]:OSF1:*:* ) + echo hppa1.1-hp-osf + exit ;; + hp8??:OSF1:*:*) + echo hppa1.0-hp-osf + exit ;; + i*86:OSF1:*:*) + if [ -x /usr/sbin/sysversion ] ; then + echo ${UNAME_MACHINE}-unknown-osf1mk + else + echo ${UNAME_MACHINE}-unknown-osf1 + fi + exit ;; + parisc*:Lites*:*:*) + echo hppa1.1-hp-lites + exit ;; + C1*:ConvexOS:*:* | convex:ConvexOS:C1*:*) + echo c1-convex-bsd + exit ;; + C2*:ConvexOS:*:* | convex:ConvexOS:C2*:*) + if getsysinfo -f scalar_acc + then echo c32-convex-bsd + else echo c2-convex-bsd + fi + exit ;; + C34*:ConvexOS:*:* | convex:ConvexOS:C34*:*) + echo c34-convex-bsd + exit ;; + C38*:ConvexOS:*:* | convex:ConvexOS:C38*:*) + echo c38-convex-bsd + exit ;; + C4*:ConvexOS:*:* | convex:ConvexOS:C4*:*) + echo c4-convex-bsd + exit ;; + CRAY*Y-MP:*:*:*) + echo ymp-cray-unicos${UNAME_RELEASE} | sed -e 's/\.[^.]*$/.X/' + exit ;; + CRAY*[A-Z]90:*:*:*) + echo ${UNAME_MACHINE}-cray-unicos${UNAME_RELEASE} \ + | sed -e 's/CRAY.*\([A-Z]90\)/\1/' \ + -e y/ABCDEFGHIJKLMNOPQRSTUVWXYZ/abcdefghijklmnopqrstuvwxyz/ \ + -e 's/\.[^.]*$/.X/' + exit ;; + CRAY*TS:*:*:*) + echo t90-cray-unicos${UNAME_RELEASE} | sed -e 's/\.[^.]*$/.X/' + exit ;; + CRAY*T3E:*:*:*) + echo alphaev5-cray-unicosmk${UNAME_RELEASE} | sed -e 's/\.[^.]*$/.X/' + exit ;; + CRAY*SV1:*:*:*) + echo sv1-cray-unicos${UNAME_RELEASE} | sed -e 's/\.[^.]*$/.X/' + exit ;; + *:UNICOS/mp:*:*) + echo craynv-cray-unicosmp${UNAME_RELEASE} | sed -e 's/\.[^.]*$/.X/' + exit ;; + F30[01]:UNIX_System_V:*:* | F700:UNIX_System_V:*:*) + FUJITSU_PROC=`uname -m | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz` + FUJITSU_SYS=`uname -p | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz | sed -e 's/\///'` + FUJITSU_REL=`echo ${UNAME_RELEASE} | sed -e 's/ /_/'` + echo "${FUJITSU_PROC}-fujitsu-${FUJITSU_SYS}${FUJITSU_REL}" + exit ;; + 5000:UNIX_System_V:4.*:*) + FUJITSU_SYS=`uname -p | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz | sed -e 's/\///'` + FUJITSU_REL=`echo ${UNAME_RELEASE} | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz | sed -e 's/ /_/'` + echo "sparc-fujitsu-${FUJITSU_SYS}${FUJITSU_REL}" + exit ;; + i*86:BSD/386:*:* | i*86:BSD/OS:*:* | *:Ascend\ Embedded/OS:*:*) + echo ${UNAME_MACHINE}-pc-bsdi${UNAME_RELEASE} + exit ;; + sparc*:BSD/OS:*:*) + echo sparc-unknown-bsdi${UNAME_RELEASE} + exit ;; + *:BSD/OS:*:*) + echo ${UNAME_MACHINE}-unknown-bsdi${UNAME_RELEASE} + exit ;; + *:FreeBSD:*:*) + UNAME_PROCESSOR=`/usr/bin/uname -p` + case ${UNAME_PROCESSOR} in + amd64) + UNAME_PROCESSOR=x86_64 ;; + i386) + UNAME_PROCESSOR=i586 ;; + esac + echo ${UNAME_PROCESSOR}-unknown-freebsd`echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'` + exit ;; + i*:CYGWIN*:*) + echo ${UNAME_MACHINE}-pc-cygwin + exit ;; + *:MINGW64*:*) + echo ${UNAME_MACHINE}-pc-mingw64 + exit ;; + *:MINGW*:*) + echo ${UNAME_MACHINE}-pc-mingw32 + exit ;; + *:MSYS*:*) + echo ${UNAME_MACHINE}-pc-msys + exit ;; + i*:windows32*:*) + # uname -m includes "-pc" on this system. + echo ${UNAME_MACHINE}-mingw32 + exit ;; + i*:PW*:*) + echo ${UNAME_MACHINE}-pc-pw32 + exit ;; + *:Interix*:*) + case ${UNAME_MACHINE} in + x86) + echo i586-pc-interix${UNAME_RELEASE} + exit ;; + authenticamd | genuineintel | EM64T) + echo x86_64-unknown-interix${UNAME_RELEASE} + exit ;; + IA64) + echo ia64-unknown-interix${UNAME_RELEASE} + exit ;; + esac ;; + [345]86:Windows_95:* | [345]86:Windows_98:* | [345]86:Windows_NT:*) + echo i${UNAME_MACHINE}-pc-mks + exit ;; + 8664:Windows_NT:*) + echo x86_64-pc-mks + exit ;; + i*:Windows_NT*:* | Pentium*:Windows_NT*:*) + # How do we know it's Interix rather than the generic POSIX subsystem? + # It also conflicts with pre-2.0 versions of AT&T UWIN. Should we + # UNAME_MACHINE based on the output of uname instead of i386? + echo i586-pc-interix + exit ;; + i*:UWIN*:*) + echo ${UNAME_MACHINE}-pc-uwin + exit ;; + amd64:CYGWIN*:*:* | x86_64:CYGWIN*:*:*) + echo x86_64-unknown-cygwin + exit ;; + p*:CYGWIN*:*) + echo powerpcle-unknown-cygwin + exit ;; + prep*:SunOS:5.*:*) + echo powerpcle-unknown-solaris2`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'` + exit ;; + *:GNU:*:*) + # the GNU system + echo `echo ${UNAME_MACHINE}|sed -e 's,[-/].*$,,'`-unknown-${LIBC}`echo ${UNAME_RELEASE}|sed -e 's,/.*$,,'` + exit ;; + *:GNU/*:*:*) + # other systems with GNU libc and userland + echo ${UNAME_MACHINE}-unknown-`echo ${UNAME_SYSTEM} | sed 's,^[^/]*/,,' | tr "[:upper:]" "[:lower:]"``echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'`-${LIBC} + exit ;; + i*86:Minix:*:*) + echo ${UNAME_MACHINE}-pc-minix + exit ;; + aarch64:Linux:*:*) + echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + exit ;; + aarch64_be:Linux:*:*) + UNAME_MACHINE=aarch64_be + echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + exit ;; + alpha:Linux:*:*) + case `sed -n '/^cpu model/s/^.*: \(.*\)/\1/p' < /proc/cpuinfo` in + EV5) UNAME_MACHINE=alphaev5 ;; + EV56) UNAME_MACHINE=alphaev56 ;; + PCA56) UNAME_MACHINE=alphapca56 ;; + PCA57) UNAME_MACHINE=alphapca56 ;; + EV6) UNAME_MACHINE=alphaev6 ;; + EV67) UNAME_MACHINE=alphaev67 ;; + EV68*) UNAME_MACHINE=alphaev68 ;; + esac + objdump --private-headers /bin/sh | grep -q ld.so.1 + if test "$?" = 0 ; then LIBC=gnulibc1 ; fi + echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + exit ;; + arc:Linux:*:* | arceb:Linux:*:*) + echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + exit ;; + arm*:Linux:*:*) + eval $set_cc_for_build + if echo __ARM_EABI__ | $CC_FOR_BUILD -E - 2>/dev/null \ + | grep -q __ARM_EABI__ + then + echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + else + if echo __ARM_PCS_VFP | $CC_FOR_BUILD -E - 2>/dev/null \ + | grep -q __ARM_PCS_VFP + then + echo ${UNAME_MACHINE}-unknown-linux-${LIBC}eabi + else + echo ${UNAME_MACHINE}-unknown-linux-${LIBC}eabihf + fi + fi + exit ;; + avr32*:Linux:*:*) + echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + exit ;; + cris:Linux:*:*) + echo ${UNAME_MACHINE}-axis-linux-${LIBC} + exit ;; + crisv32:Linux:*:*) + echo ${UNAME_MACHINE}-axis-linux-${LIBC} + exit ;; + e2k:Linux:*:*) + echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + exit ;; + frv:Linux:*:*) + echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + exit ;; + hexagon:Linux:*:*) + echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + exit ;; + i*86:Linux:*:*) + echo ${UNAME_MACHINE}-pc-linux-${LIBC} + exit ;; + ia64:Linux:*:*) + echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + exit ;; + k1om:Linux:*:*) + echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + exit ;; + m32r*:Linux:*:*) + echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + exit ;; + m68*:Linux:*:*) + echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + exit ;; + mips:Linux:*:* | mips64:Linux:*:*) + eval $set_cc_for_build + sed 's/^ //' << EOF >$dummy.c + #undef CPU + #undef ${UNAME_MACHINE} + #undef ${UNAME_MACHINE}el + #if defined(__MIPSEL__) || defined(__MIPSEL) || defined(_MIPSEL) || defined(MIPSEL) + CPU=${UNAME_MACHINE}el + #else + #if defined(__MIPSEB__) || defined(__MIPSEB) || defined(_MIPSEB) || defined(MIPSEB) + CPU=${UNAME_MACHINE} + #else + CPU= + #endif + #endif +EOF + eval `$CC_FOR_BUILD -E $dummy.c 2>/dev/null | grep '^CPU'` + test x"${CPU}" != x && { echo "${CPU}-unknown-linux-${LIBC}"; exit; } + ;; + mips64el:Linux:*:*) + echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + exit ;; + openrisc*:Linux:*:*) + echo or1k-unknown-linux-${LIBC} + exit ;; + or32:Linux:*:* | or1k*:Linux:*:*) + echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + exit ;; + padre:Linux:*:*) + echo sparc-unknown-linux-${LIBC} + exit ;; + parisc64:Linux:*:* | hppa64:Linux:*:*) + echo hppa64-unknown-linux-${LIBC} + exit ;; + parisc:Linux:*:* | hppa:Linux:*:*) + # Look for CPU level + case `grep '^cpu[^a-z]*:' /proc/cpuinfo 2>/dev/null | cut -d' ' -f2` in + PA7*) echo hppa1.1-unknown-linux-${LIBC} ;; + PA8*) echo hppa2.0-unknown-linux-${LIBC} ;; + *) echo hppa-unknown-linux-${LIBC} ;; + esac + exit ;; + ppc64:Linux:*:*) + echo powerpc64-unknown-linux-${LIBC} + exit ;; + ppc:Linux:*:*) + echo powerpc-unknown-linux-${LIBC} + exit ;; + ppc64le:Linux:*:*) + echo powerpc64le-unknown-linux-${LIBC} + exit ;; + ppcle:Linux:*:*) + echo powerpcle-unknown-linux-${LIBC} + exit ;; + riscv32:Linux:*:* | riscv64:Linux:*:*) + echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + exit ;; + s390:Linux:*:* | s390x:Linux:*:*) + echo ${UNAME_MACHINE}-ibm-linux-${LIBC} + exit ;; + sh64*:Linux:*:*) + echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + exit ;; + sh*:Linux:*:*) + echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + exit ;; + sparc:Linux:*:* | sparc64:Linux:*:*) + echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + exit ;; + tile*:Linux:*:*) + echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + exit ;; + vax:Linux:*:*) + echo ${UNAME_MACHINE}-dec-linux-${LIBC} + exit ;; + x86_64:Linux:*:*) + echo ${UNAME_MACHINE}-pc-linux-${LIBC} + exit ;; + xtensa*:Linux:*:*) + echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + exit ;; + i*86:DYNIX/ptx:4*:*) + # ptx 4.0 does uname -s correctly, with DYNIX/ptx in there. + # earlier versions are messed up and put the nodename in both + # sysname and nodename. + echo i386-sequent-sysv4 + exit ;; + i*86:UNIX_SV:4.2MP:2.*) + # Unixware is an offshoot of SVR4, but it has its own version + # number series starting with 2... + # I am not positive that other SVR4 systems won't match this, + # I just have to hope. -- rms. + # Use sysv4.2uw... so that sysv4* matches it. + echo ${UNAME_MACHINE}-pc-sysv4.2uw${UNAME_VERSION} + exit ;; + i*86:OS/2:*:*) + # If we were able to find `uname', then EMX Unix compatibility + # is probably installed. + echo ${UNAME_MACHINE}-pc-os2-emx + exit ;; + i*86:XTS-300:*:STOP) + echo ${UNAME_MACHINE}-unknown-stop + exit ;; + i*86:atheos:*:*) + echo ${UNAME_MACHINE}-unknown-atheos + exit ;; + i*86:syllable:*:*) + echo ${UNAME_MACHINE}-pc-syllable + exit ;; + i*86:LynxOS:2.*:* | i*86:LynxOS:3.[01]*:* | i*86:LynxOS:4.[02]*:*) + echo i386-unknown-lynxos${UNAME_RELEASE} + exit ;; + i*86:*DOS:*:*) + echo ${UNAME_MACHINE}-pc-msdosdjgpp + exit ;; + i*86:*:4.*:* | i*86:SYSTEM_V:4.*:*) + UNAME_REL=`echo ${UNAME_RELEASE} | sed 's/\/MP$//'` + if grep Novell /usr/include/link.h >/dev/null 2>/dev/null; then + echo ${UNAME_MACHINE}-univel-sysv${UNAME_REL} + else + echo ${UNAME_MACHINE}-pc-sysv${UNAME_REL} + fi + exit ;; + i*86:*:5:[678]*) + # UnixWare 7.x, OpenUNIX and OpenServer 6. + case `/bin/uname -X | grep "^Machine"` in + *486*) UNAME_MACHINE=i486 ;; + *Pentium) UNAME_MACHINE=i586 ;; + *Pent*|*Celeron) UNAME_MACHINE=i686 ;; + esac + echo ${UNAME_MACHINE}-unknown-sysv${UNAME_RELEASE}${UNAME_SYSTEM}${UNAME_VERSION} + exit ;; + i*86:*:3.2:*) + if test -f /usr/options/cb.name; then + UNAME_REL=`sed -n 's/.*Version //p' /dev/null >/dev/null ; then + UNAME_REL=`(/bin/uname -X|grep Release|sed -e 's/.*= //')` + (/bin/uname -X|grep i80486 >/dev/null) && UNAME_MACHINE=i486 + (/bin/uname -X|grep '^Machine.*Pentium' >/dev/null) \ + && UNAME_MACHINE=i586 + (/bin/uname -X|grep '^Machine.*Pent *II' >/dev/null) \ + && UNAME_MACHINE=i686 + (/bin/uname -X|grep '^Machine.*Pentium Pro' >/dev/null) \ + && UNAME_MACHINE=i686 + echo ${UNAME_MACHINE}-pc-sco$UNAME_REL + else + echo ${UNAME_MACHINE}-pc-sysv32 + fi + exit ;; + pc:*:*:*) + # Left here for compatibility: + # uname -m prints for DJGPP always 'pc', but it prints nothing about + # the processor, so we play safe by assuming i586. + # Note: whatever this is, it MUST be the same as what config.sub + # prints for the "djgpp" host, or else GDB configure will decide that + # this is a cross-build. + echo i586-pc-msdosdjgpp + exit ;; + Intel:Mach:3*:*) + echo i386-pc-mach3 + exit ;; + paragon:*:*:*) + echo i860-intel-osf1 + exit ;; + i860:*:4.*:*) # i860-SVR4 + if grep Stardent /usr/include/sys/uadmin.h >/dev/null 2>&1 ; then + echo i860-stardent-sysv${UNAME_RELEASE} # Stardent Vistra i860-SVR4 + else # Add other i860-SVR4 vendors below as they are discovered. + echo i860-unknown-sysv${UNAME_RELEASE} # Unknown i860-SVR4 + fi + exit ;; + mini*:CTIX:SYS*5:*) + # "miniframe" + echo m68010-convergent-sysv + exit ;; + mc68k:UNIX:SYSTEM5:3.51m) + echo m68k-convergent-sysv + exit ;; + M680?0:D-NIX:5.3:*) + echo m68k-diab-dnix + exit ;; + M68*:*:R3V[5678]*:*) + test -r /sysV68 && { echo 'm68k-motorola-sysv'; exit; } ;; + 3[345]??:*:4.0:3.0 | 3[34]??A:*:4.0:3.0 | 3[34]??,*:*:4.0:3.0 | 3[34]??/*:*:4.0:3.0 | 4400:*:4.0:3.0 | 4850:*:4.0:3.0 | SKA40:*:4.0:3.0 | SDS2:*:4.0:3.0 | SHG2:*:4.0:3.0 | S7501*:*:4.0:3.0) + OS_REL='' + test -r /etc/.relid \ + && OS_REL=.`sed -n 's/[^ ]* [^ ]* \([0-9][0-9]\).*/\1/p' < /etc/.relid` + /bin/uname -p 2>/dev/null | grep 86 >/dev/null \ + && { echo i486-ncr-sysv4.3${OS_REL}; exit; } + /bin/uname -p 2>/dev/null | /bin/grep entium >/dev/null \ + && { echo i586-ncr-sysv4.3${OS_REL}; exit; } ;; + 3[34]??:*:4.0:* | 3[34]??,*:*:4.0:*) + /bin/uname -p 2>/dev/null | grep 86 >/dev/null \ + && { echo i486-ncr-sysv4; exit; } ;; + NCR*:*:4.2:* | MPRAS*:*:4.2:*) + OS_REL='.3' + test -r /etc/.relid \ + && OS_REL=.`sed -n 's/[^ ]* [^ ]* \([0-9][0-9]\).*/\1/p' < /etc/.relid` + /bin/uname -p 2>/dev/null | grep 86 >/dev/null \ + && { echo i486-ncr-sysv4.3${OS_REL}; exit; } + /bin/uname -p 2>/dev/null | /bin/grep entium >/dev/null \ + && { echo i586-ncr-sysv4.3${OS_REL}; exit; } + /bin/uname -p 2>/dev/null | /bin/grep pteron >/dev/null \ + && { echo i586-ncr-sysv4.3${OS_REL}; exit; } ;; + m68*:LynxOS:2.*:* | m68*:LynxOS:3.0*:*) + echo m68k-unknown-lynxos${UNAME_RELEASE} + exit ;; + mc68030:UNIX_System_V:4.*:*) + echo m68k-atari-sysv4 + exit ;; + TSUNAMI:LynxOS:2.*:*) + echo sparc-unknown-lynxos${UNAME_RELEASE} + exit ;; + rs6000:LynxOS:2.*:*) + echo rs6000-unknown-lynxos${UNAME_RELEASE} + exit ;; + PowerPC:LynxOS:2.*:* | PowerPC:LynxOS:3.[01]*:* | PowerPC:LynxOS:4.[02]*:*) + echo powerpc-unknown-lynxos${UNAME_RELEASE} + exit ;; + SM[BE]S:UNIX_SV:*:*) + echo mips-dde-sysv${UNAME_RELEASE} + exit ;; + RM*:ReliantUNIX-*:*:*) + echo mips-sni-sysv4 + exit ;; + RM*:SINIX-*:*:*) + echo mips-sni-sysv4 + exit ;; + *:SINIX-*:*:*) + if uname -p 2>/dev/null >/dev/null ; then + UNAME_MACHINE=`(uname -p) 2>/dev/null` + echo ${UNAME_MACHINE}-sni-sysv4 + else + echo ns32k-sni-sysv + fi + exit ;; + PENTIUM:*:4.0*:*) # Unisys `ClearPath HMP IX 4000' SVR4/MP effort + # says + echo i586-unisys-sysv4 + exit ;; + *:UNIX_System_V:4*:FTX*) + # From Gerald Hewes . + # How about differentiating between stratus architectures? -djm + echo hppa1.1-stratus-sysv4 + exit ;; + *:*:*:FTX*) + # From seanf@swdc.stratus.com. + echo i860-stratus-sysv4 + exit ;; + i*86:VOS:*:*) + # From Paul.Green@stratus.com. + echo ${UNAME_MACHINE}-stratus-vos + exit ;; + *:VOS:*:*) + # From Paul.Green@stratus.com. + echo hppa1.1-stratus-vos + exit ;; + mc68*:A/UX:*:*) + echo m68k-apple-aux${UNAME_RELEASE} + exit ;; + news*:NEWS-OS:6*:*) + echo mips-sony-newsos6 + exit ;; + R[34]000:*System_V*:*:* | R4000:UNIX_SYSV:*:* | R*000:UNIX_SV:*:*) + if [ -d /usr/nec ]; then + echo mips-nec-sysv${UNAME_RELEASE} + else + echo mips-unknown-sysv${UNAME_RELEASE} + fi + exit ;; + BeBox:BeOS:*:*) # BeOS running on hardware made by Be, PPC only. + echo powerpc-be-beos + exit ;; + BeMac:BeOS:*:*) # BeOS running on Mac or Mac clone, PPC only. + echo powerpc-apple-beos + exit ;; + BePC:BeOS:*:*) # BeOS running on Intel PC compatible. + echo i586-pc-beos + exit ;; + BePC:Haiku:*:*) # Haiku running on Intel PC compatible. + echo i586-pc-haiku + exit ;; + x86_64:Haiku:*:*) + echo x86_64-unknown-haiku + exit ;; + SX-4:SUPER-UX:*:*) + echo sx4-nec-superux${UNAME_RELEASE} + exit ;; + SX-5:SUPER-UX:*:*) + echo sx5-nec-superux${UNAME_RELEASE} + exit ;; + SX-6:SUPER-UX:*:*) + echo sx6-nec-superux${UNAME_RELEASE} + exit ;; + SX-7:SUPER-UX:*:*) + echo sx7-nec-superux${UNAME_RELEASE} + exit ;; + SX-8:SUPER-UX:*:*) + echo sx8-nec-superux${UNAME_RELEASE} + exit ;; + SX-8R:SUPER-UX:*:*) + echo sx8r-nec-superux${UNAME_RELEASE} + exit ;; + SX-ACE:SUPER-UX:*:*) + echo sxace-nec-superux${UNAME_RELEASE} + exit ;; + Power*:Rhapsody:*:*) + echo powerpc-apple-rhapsody${UNAME_RELEASE} + exit ;; + *:Rhapsody:*:*) + echo ${UNAME_MACHINE}-apple-rhapsody${UNAME_RELEASE} + exit ;; + *:Darwin:*:*) + UNAME_PROCESSOR=`uname -p` || UNAME_PROCESSOR=unknown + eval $set_cc_for_build + if test "$UNAME_PROCESSOR" = unknown ; then + UNAME_PROCESSOR=powerpc + fi + if test `echo "$UNAME_RELEASE" | sed -e 's/\..*//'` -le 10 ; then + if [ "$CC_FOR_BUILD" != no_compiler_found ]; then + if (echo '#ifdef __LP64__'; echo IS_64BIT_ARCH; echo '#endif') | \ + (CCOPTS="" $CC_FOR_BUILD -E - 2>/dev/null) | \ + grep IS_64BIT_ARCH >/dev/null + then + case $UNAME_PROCESSOR in + i386) UNAME_PROCESSOR=x86_64 ;; + powerpc) UNAME_PROCESSOR=powerpc64 ;; + esac + fi + fi + elif test "$UNAME_PROCESSOR" = i386 ; then + # Avoid executing cc on OS X 10.9, as it ships with a stub + # that puts up a graphical alert prompting to install + # developer tools. Any system running Mac OS X 10.7 or + # later (Darwin 11 and later) is required to have a 64-bit + # processor. This is not true of the ARM version of Darwin + # that Apple uses in portable devices. + UNAME_PROCESSOR=x86_64 + fi + echo ${UNAME_PROCESSOR}-apple-darwin${UNAME_RELEASE} + exit ;; + *:procnto*:*:* | *:QNX:[0123456789]*:*) + UNAME_PROCESSOR=`uname -p` + if test "$UNAME_PROCESSOR" = x86; then + UNAME_PROCESSOR=i386 + UNAME_MACHINE=pc + fi + echo ${UNAME_PROCESSOR}-${UNAME_MACHINE}-nto-qnx${UNAME_RELEASE} + exit ;; + *:QNX:*:4*) + echo i386-pc-qnx + exit ;; + NEO-?:NONSTOP_KERNEL:*:*) + echo neo-tandem-nsk${UNAME_RELEASE} + exit ;; + NSE-*:NONSTOP_KERNEL:*:*) + echo nse-tandem-nsk${UNAME_RELEASE} + exit ;; + NSR-?:NONSTOP_KERNEL:*:*) + echo nsr-tandem-nsk${UNAME_RELEASE} + exit ;; + NSX-?:NONSTOP_KERNEL:*:*) + echo nsx-tandem-nsk${UNAME_RELEASE} + exit ;; + *:NonStop-UX:*:*) + echo mips-compaq-nonstopux + exit ;; + BS2000:POSIX*:*:*) + echo bs2000-siemens-sysv + exit ;; + DS/*:UNIX_System_V:*:*) + echo ${UNAME_MACHINE}-${UNAME_SYSTEM}-${UNAME_RELEASE} + exit ;; + *:Plan9:*:*) + # "uname -m" is not consistent, so use $cputype instead. 386 + # is converted to i386 for consistency with other x86 + # operating systems. + if test "$cputype" = 386; then + UNAME_MACHINE=i386 + else + UNAME_MACHINE="$cputype" + fi + echo ${UNAME_MACHINE}-unknown-plan9 + exit ;; + *:TOPS-10:*:*) + echo pdp10-unknown-tops10 + exit ;; + *:TENEX:*:*) + echo pdp10-unknown-tenex + exit ;; + KS10:TOPS-20:*:* | KL10:TOPS-20:*:* | TYPE4:TOPS-20:*:*) + echo pdp10-dec-tops20 + exit ;; + XKL-1:TOPS-20:*:* | TYPE5:TOPS-20:*:*) + echo pdp10-xkl-tops20 + exit ;; + *:TOPS-20:*:*) + echo pdp10-unknown-tops20 + exit ;; + *:ITS:*:*) + echo pdp10-unknown-its + exit ;; + SEI:*:*:SEIUX) + echo mips-sei-seiux${UNAME_RELEASE} + exit ;; + *:DragonFly:*:*) + echo ${UNAME_MACHINE}-unknown-dragonfly`echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'` + exit ;; + *:*VMS:*:*) + UNAME_MACHINE=`(uname -p) 2>/dev/null` + case "${UNAME_MACHINE}" in + A*) echo alpha-dec-vms ; exit ;; + I*) echo ia64-dec-vms ; exit ;; + V*) echo vax-dec-vms ; exit ;; + esac ;; + *:XENIX:*:SysV) + echo i386-pc-xenix + exit ;; + i*86:skyos:*:*) + echo ${UNAME_MACHINE}-pc-skyos`echo ${UNAME_RELEASE} | sed -e 's/ .*$//'` + exit ;; + i*86:rdos:*:*) + echo ${UNAME_MACHINE}-pc-rdos + exit ;; + i*86:AROS:*:*) + echo ${UNAME_MACHINE}-pc-aros + exit ;; + x86_64:VMkernel:*:*) + echo ${UNAME_MACHINE}-unknown-esx + exit ;; + amd64:Isilon\ OneFS:*:*) + echo x86_64-unknown-onefs + exit ;; +esac + +cat >&2 </dev/null || echo unknown` +uname -r = `(uname -r) 2>/dev/null || echo unknown` +uname -s = `(uname -s) 2>/dev/null || echo unknown` +uname -v = `(uname -v) 2>/dev/null || echo unknown` + +/usr/bin/uname -p = `(/usr/bin/uname -p) 2>/dev/null` +/bin/uname -X = `(/bin/uname -X) 2>/dev/null` + +hostinfo = `(hostinfo) 2>/dev/null` +/bin/universe = `(/bin/universe) 2>/dev/null` +/usr/bin/arch -k = `(/usr/bin/arch -k) 2>/dev/null` +/bin/arch = `(/bin/arch) 2>/dev/null` +/usr/bin/oslevel = `(/usr/bin/oslevel) 2>/dev/null` +/usr/convex/getsysinfo = `(/usr/convex/getsysinfo) 2>/dev/null` + +UNAME_MACHINE = ${UNAME_MACHINE} +UNAME_RELEASE = ${UNAME_RELEASE} +UNAME_SYSTEM = ${UNAME_SYSTEM} +UNAME_VERSION = ${UNAME_VERSION} +EOF + +exit 1 + +# Local variables: +# eval: (add-hook 'write-file-hooks 'time-stamp) +# time-stamp-start: "timestamp='" +# time-stamp-format: "%:y-%02m-%02d" +# time-stamp-end: "'" +# End: diff --git a/contrib/depends/config.site.in b/contrib/depends/config.site.in new file mode 100644 index 00000000000..dd91bcb2aa4 --- /dev/null +++ b/contrib/depends/config.site.in @@ -0,0 +1,81 @@ +depends_prefix="`dirname ${ac_site_file}`/.." + +cross_compiling=maybe +host_alias=@HOST@ +ac_tool_prefix=${host_alias}- + +if test -z $with_boost; then + with_boost=$depends_prefix +fi +if test -z $with_qt_plugindir; then + with_qt_plugindir=$depends_prefix/plugins +fi +if test -z $with_qt_translationdir; then + with_qt_translationdir=$depends_prefix/translations +fi + +if test x@host_os@ = xdarwin; then + BREW=no + PORT=no +fi + +if test x@host_os@ = xmingw32; then + if test -z $with_qt_incdir; then + with_qt_incdir=$depends_prefix/include + fi + if test -z $with_qt_libdir; then + with_qt_libdir=$depends_prefix/lib + fi +fi + +PATH=$depends_prefix/native/bin:$PATH +PKG_CONFIG="`which pkg-config` --static" + +# These two need to remain exported because pkg-config does not see them +# otherwise. That means they must be unexported at the end of configure.ac to +# avoid ruining the cache. Sigh. +export PKG_CONFIG_PATH=$depends_prefix/share/pkgconfig:$depends_prefix/lib/pkgconfig +if test -z "@allow_host_packages@"; then + export PKGCONFIG_LIBDIR= +fi + +CPPFLAGS="-I$depends_prefix/include/ $CPPFLAGS" +LDFLAGS="-L$depends_prefix/lib $LDFLAGS" + +CC="@CC@" +CXX="@CXX@" +OBJC="${CC}" +CCACHE=$depends_prefix/native/bin/ccache +PYTHONPATH=$depends_prefix/native/lib/python/dist-packages:$PYTHONPATH + +if test -n "@AR@"; then + AR=@AR@ + ac_cv_path_ac_pt_AR=${AR} +fi + +if test -n "@RANLIB@"; then + RANLIB=@RANLIB@ + ac_cv_path_ac_pt_RANLIB=${RANLIB} +fi + +if test -n "@NM@"; then + NM=@NM@ + ac_cv_path_ac_pt_NM=${NM} +fi + +if test -n "@debug@"; then + enable_reduce_exports=no +fi + +if test -n "@CFLAGS@"; then + CFLAGS="@CFLAGS@ $CFLAGS" +fi +if test -n "@CXXFLAGS@"; then + CXXFLAGS="@CXXFLAGS@ $CXXFLAGS" +fi +if test -n "@CPPFLAGS@"; then + CPPFLAGS="@CPPFLAGS@ $CPPFLAGS" +fi +if test -n "@LDFLAGS@"; then + LDFLAGS="@LDFLAGS@ $LDFLAGS" +fi diff --git a/contrib/depends/config.sub b/contrib/depends/config.sub new file mode 100755 index 00000000000..40ea5dfe115 --- /dev/null +++ b/contrib/depends/config.sub @@ -0,0 +1,1836 @@ +#! /bin/sh +# Configuration validation subroutine script. +# Copyright 1992-2017 Free Software Foundation, Inc. + +timestamp='2017-04-02' + +# This file is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, see . +# +# As a special exception to the GNU General Public License, if you +# distribute this file as part of a program that contains a +# configuration script generated by Autoconf, you may include it under +# the same distribution terms that you use for the rest of that +# program. This Exception is an additional permission under section 7 +# of the GNU General Public License, version 3 ("GPLv3"). + + +# Please send patches to . +# +# Configuration subroutine to validate and canonicalize a configuration type. +# Supply the specified configuration type as an argument. +# If it is invalid, we print an error message on stderr and exit with code 1. +# Otherwise, we print the canonical config type on stdout and succeed. + +# You can get the latest version of this script from: +# http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.sub + +# This file is supposed to be the same for all GNU packages +# and recognize all the CPU types, system types and aliases +# that are meaningful with *any* GNU software. +# Each package is responsible for reporting which valid configurations +# it does not support. The user should be able to distinguish +# a failure to support a valid configuration from a meaningless +# configuration. + +# The goal of this file is to map all the various variations of a given +# machine specification into a single specification in the form: +# CPU_TYPE-MANUFACTURER-OPERATING_SYSTEM +# or in some cases, the newer four-part form: +# CPU_TYPE-MANUFACTURER-KERNEL-OPERATING_SYSTEM +# It is wrong to echo any other type of specification. + +me=`echo "$0" | sed -e 's,.*/,,'` + +usage="\ +Usage: $0 [OPTION] CPU-MFR-OPSYS or ALIAS + +Canonicalize a configuration name. + +Operation modes: + -h, --help print this help, then exit + -t, --time-stamp print date of last modification, then exit + -v, --version print version number, then exit + +Report bugs and patches to ." + +version="\ +GNU config.sub ($timestamp) + +Copyright 1992-2017 Free Software Foundation, Inc. + +This is free software; see the source for copying conditions. There is NO +warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE." + +help=" +Try \`$me --help' for more information." + +# Parse command line +while test $# -gt 0 ; do + case $1 in + --time-stamp | --time* | -t ) + echo "$timestamp" ; exit ;; + --version | -v ) + echo "$version" ; exit ;; + --help | --h* | -h ) + echo "$usage"; exit ;; + -- ) # Stop option processing + shift; break ;; + - ) # Use stdin as input. + break ;; + -* ) + echo "$me: invalid option $1$help" + exit 1 ;; + + *local*) + # First pass through any local machine types. + echo $1 + exit ;; + + * ) + break ;; + esac +done + +case $# in + 0) echo "$me: missing argument$help" >&2 + exit 1;; + 1) ;; + *) echo "$me: too many arguments$help" >&2 + exit 1;; +esac + +# Separate what the user gave into CPU-COMPANY and OS or KERNEL-OS (if any). +# Here we must recognize all the valid KERNEL-OS combinations. +maybe_os=`echo $1 | sed 's/^\(.*\)-\([^-]*-[^-]*\)$/\2/'` +case $maybe_os in + nto-qnx* | linux-gnu* | linux-android* | linux-dietlibc | linux-newlib* | \ + linux-musl* | linux-uclibc* | uclinux-uclibc* | uclinux-gnu* | kfreebsd*-gnu* | \ + knetbsd*-gnu* | netbsd*-gnu* | netbsd*-eabi* | \ + kopensolaris*-gnu* | cloudabi*-eabi* | \ + storm-chaos* | os2-emx* | rtmk-nova*) + os=-$maybe_os + basic_machine=`echo $1 | sed 's/^\(.*\)-\([^-]*-[^-]*\)$/\1/'` + ;; + android-linux) + os=-linux-android + basic_machine=`echo $1 | sed 's/^\(.*\)-\([^-]*-[^-]*\)$/\1/'`-unknown + ;; + *) + basic_machine=`echo $1 | sed 's/-[^-]*$//'` + if [ $basic_machine != $1 ] + then os=`echo $1 | sed 's/.*-/-/'` + else os=; fi + ;; +esac + +### Let's recognize common machines as not being operating systems so +### that things like config.sub decstation-3100 work. We also +### recognize some manufacturers as not being operating systems, so we +### can provide default operating systems below. +case $os in + -sun*os*) + # Prevent following clause from handling this invalid input. + ;; + -dec* | -mips* | -sequent* | -encore* | -pc532* | -sgi* | -sony* | \ + -att* | -7300* | -3300* | -delta* | -motorola* | -sun[234]* | \ + -unicom* | -ibm* | -next | -hp | -isi* | -apollo | -altos* | \ + -convergent* | -ncr* | -news | -32* | -3600* | -3100* | -hitachi* |\ + -c[123]* | -convex* | -sun | -crds | -omron* | -dg | -ultra | -tti* | \ + -harris | -dolphin | -highlevel | -gould | -cbm | -ns | -masscomp | \ + -apple | -axis | -knuth | -cray | -microblaze*) + os= + basic_machine=$1 + ;; + -bluegene*) + os=-cnk + ;; + -sim | -cisco | -oki | -wec | -winbond) + os= + basic_machine=$1 + ;; + -scout) + ;; + -wrs) + os=-vxworks + basic_machine=$1 + ;; + -chorusos*) + os=-chorusos + basic_machine=$1 + ;; + -chorusrdb) + os=-chorusrdb + basic_machine=$1 + ;; + -hiux*) + os=-hiuxwe2 + ;; + -sco6) + os=-sco5v6 + basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'` + ;; + -sco5) + os=-sco3.2v5 + basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'` + ;; + -sco4) + os=-sco3.2v4 + basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'` + ;; + -sco3.2.[4-9]*) + os=`echo $os | sed -e 's/sco3.2./sco3.2v/'` + basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'` + ;; + -sco3.2v[4-9]*) + # Don't forget version if it is 3.2v4 or newer. + basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'` + ;; + -sco5v6*) + # Don't forget version if it is 3.2v4 or newer. + basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'` + ;; + -sco*) + os=-sco3.2v2 + basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'` + ;; + -udk*) + basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'` + ;; + -isc) + os=-isc2.2 + basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'` + ;; + -clix*) + basic_machine=clipper-intergraph + ;; + -isc*) + basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'` + ;; + -lynx*178) + os=-lynxos178 + ;; + -lynx*5) + os=-lynxos5 + ;; + -lynx*) + os=-lynxos + ;; + -ptx*) + basic_machine=`echo $1 | sed -e 's/86-.*/86-sequent/'` + ;; + -windowsnt*) + os=`echo $os | sed -e 's/windowsnt/winnt/'` + ;; + -psos*) + os=-psos + ;; + -mint | -mint[0-9]*) + basic_machine=m68k-atari + os=-mint + ;; +esac + +# Decode aliases for certain CPU-COMPANY combinations. +case $basic_machine in + # Recognize the basic CPU types without company name. + # Some are omitted here because they have special meanings below. + 1750a | 580 \ + | a29k \ + | aarch64 | aarch64_be \ + | alpha | alphaev[4-8] | alphaev56 | alphaev6[78] | alphapca5[67] \ + | alpha64 | alpha64ev[4-8] | alpha64ev56 | alpha64ev6[78] | alpha64pca5[67] \ + | am33_2.0 \ + | arc | arceb \ + | arm | arm[bl]e | arme[lb] | armv[2-8] | armv[3-8][lb] | armv7[arm] \ + | avr | avr32 \ + | ba \ + | be32 | be64 \ + | bfin \ + | c4x | c8051 | clipper \ + | d10v | d30v | dlx | dsp16xx \ + | e2k | epiphany \ + | fido | fr30 | frv | ft32 \ + | h8300 | h8500 | hppa | hppa1.[01] | hppa2.0 | hppa2.0[nw] | hppa64 \ + | hexagon \ + | i370 | i860 | i960 | ia16 | ia64 \ + | ip2k | iq2000 \ + | k1om \ + | le32 | le64 \ + | lm32 \ + | m32c | m32r | m32rle | m68000 | m68k | m88k \ + | maxq | mb | microblaze | microblazeel | mcore | mep | metag \ + | mips | mipsbe | mipseb | mipsel | mipsle \ + | mips16 \ + | mips64 | mips64el \ + | mips64octeon | mips64octeonel \ + | mips64orion | mips64orionel \ + | mips64r5900 | mips64r5900el \ + | mips64vr | mips64vrel \ + | mips64vr4100 | mips64vr4100el \ + | mips64vr4300 | mips64vr4300el \ + | mips64vr5000 | mips64vr5000el \ + | mips64vr5900 | mips64vr5900el \ + | mipsisa32 | mipsisa32el \ + | mipsisa32r2 | mipsisa32r2el \ + | mipsisa32r6 | mipsisa32r6el \ + | mipsisa64 | mipsisa64el \ + | mipsisa64r2 | mipsisa64r2el \ + | mipsisa64r6 | mipsisa64r6el \ + | mipsisa64sb1 | mipsisa64sb1el \ + | mipsisa64sr71k | mipsisa64sr71kel \ + | mipsr5900 | mipsr5900el \ + | mipstx39 | mipstx39el \ + | mn10200 | mn10300 \ + | moxie \ + | mt \ + | msp430 \ + | nds32 | nds32le | nds32be \ + | nios | nios2 | nios2eb | nios2el \ + | ns16k | ns32k \ + | open8 | or1k | or1knd | or32 \ + | pdp10 | pdp11 | pj | pjl \ + | powerpc | powerpc64 | powerpc64le | powerpcle \ + | pru \ + | pyramid \ + | riscv32 | riscv64 \ + | rl78 | rx \ + | score \ + | sh | sh[1234] | sh[24]a | sh[24]aeb | sh[23]e | sh[234]eb | sheb | shbe | shle | sh[1234]le | sh3ele \ + | sh64 | sh64le \ + | sparc | sparc64 | sparc64b | sparc64v | sparc86x | sparclet | sparclite \ + | sparcv8 | sparcv9 | sparcv9b | sparcv9v \ + | spu \ + | tahoe | tic4x | tic54x | tic55x | tic6x | tic80 | tron \ + | ubicom32 \ + | v850 | v850e | v850e1 | v850e2 | v850es | v850e2v3 \ + | visium \ + | wasm32 \ + | we32k \ + | x86 | xc16x | xstormy16 | xtensa \ + | z8k | z80) + basic_machine=$basic_machine-unknown + ;; + c54x) + basic_machine=tic54x-unknown + ;; + c55x) + basic_machine=tic55x-unknown + ;; + c6x) + basic_machine=tic6x-unknown + ;; + leon|leon[3-9]) + basic_machine=sparc-$basic_machine + ;; + m6811 | m68hc11 | m6812 | m68hc12 | m68hcs12x | nvptx | picochip) + basic_machine=$basic_machine-unknown + os=-none + ;; + m88110 | m680[12346]0 | m683?2 | m68360 | m5200 | v70 | w65 | z8k) + ;; + ms1) + basic_machine=mt-unknown + ;; + + strongarm | thumb | xscale) + basic_machine=arm-unknown + ;; + xgate) + basic_machine=$basic_machine-unknown + os=-none + ;; + xscaleeb) + basic_machine=armeb-unknown + ;; + + xscaleel) + basic_machine=armel-unknown + ;; + + # We use `pc' rather than `unknown' + # because (1) that's what they normally are, and + # (2) the word "unknown" tends to confuse beginning users. + i*86 | x86_64) + basic_machine=$basic_machine-pc + ;; + # Object if more than one company name word. + *-*-*) + echo Invalid configuration \`$1\': machine \`$basic_machine\' not recognized 1>&2 + exit 1 + ;; + # Recognize the basic CPU types with company name. + 580-* \ + | a29k-* \ + | aarch64-* | aarch64_be-* \ + | alpha-* | alphaev[4-8]-* | alphaev56-* | alphaev6[78]-* \ + | alpha64-* | alpha64ev[4-8]-* | alpha64ev56-* | alpha64ev6[78]-* \ + | alphapca5[67]-* | alpha64pca5[67]-* | arc-* | arceb-* \ + | arm-* | armbe-* | armle-* | armeb-* | armv*-* \ + | avr-* | avr32-* \ + | ba-* \ + | be32-* | be64-* \ + | bfin-* | bs2000-* \ + | c[123]* | c30-* | [cjt]90-* | c4x-* \ + | c8051-* | clipper-* | craynv-* | cydra-* \ + | d10v-* | d30v-* | dlx-* \ + | e2k-* | elxsi-* \ + | f30[01]-* | f700-* | fido-* | fr30-* | frv-* | fx80-* \ + | h8300-* | h8500-* \ + | hppa-* | hppa1.[01]-* | hppa2.0-* | hppa2.0[nw]-* | hppa64-* \ + | hexagon-* \ + | i*86-* | i860-* | i960-* | ia16-* | ia64-* \ + | ip2k-* | iq2000-* \ + | k1om-* \ + | le32-* | le64-* \ + | lm32-* \ + | m32c-* | m32r-* | m32rle-* \ + | m68000-* | m680[012346]0-* | m68360-* | m683?2-* | m68k-* \ + | m88110-* | m88k-* | maxq-* | mcore-* | metag-* \ + | microblaze-* | microblazeel-* \ + | mips-* | mipsbe-* | mipseb-* | mipsel-* | mipsle-* \ + | mips16-* \ + | mips64-* | mips64el-* \ + | mips64octeon-* | mips64octeonel-* \ + | mips64orion-* | mips64orionel-* \ + | mips64r5900-* | mips64r5900el-* \ + | mips64vr-* | mips64vrel-* \ + | mips64vr4100-* | mips64vr4100el-* \ + | mips64vr4300-* | mips64vr4300el-* \ + | mips64vr5000-* | mips64vr5000el-* \ + | mips64vr5900-* | mips64vr5900el-* \ + | mipsisa32-* | mipsisa32el-* \ + | mipsisa32r2-* | mipsisa32r2el-* \ + | mipsisa32r6-* | mipsisa32r6el-* \ + | mipsisa64-* | mipsisa64el-* \ + | mipsisa64r2-* | mipsisa64r2el-* \ + | mipsisa64r6-* | mipsisa64r6el-* \ + | mipsisa64sb1-* | mipsisa64sb1el-* \ + | mipsisa64sr71k-* | mipsisa64sr71kel-* \ + | mipsr5900-* | mipsr5900el-* \ + | mipstx39-* | mipstx39el-* \ + | mmix-* \ + | mt-* \ + | msp430-* \ + | nds32-* | nds32le-* | nds32be-* \ + | nios-* | nios2-* | nios2eb-* | nios2el-* \ + | none-* | np1-* | ns16k-* | ns32k-* \ + | open8-* \ + | or1k*-* \ + | orion-* \ + | pdp10-* | pdp11-* | pj-* | pjl-* | pn-* | power-* \ + | powerpc-* | powerpc64-* | powerpc64le-* | powerpcle-* \ + | pru-* \ + | pyramid-* \ + | riscv32-* | riscv64-* \ + | rl78-* | romp-* | rs6000-* | rx-* \ + | sh-* | sh[1234]-* | sh[24]a-* | sh[24]aeb-* | sh[23]e-* | sh[34]eb-* | sheb-* | shbe-* \ + | shle-* | sh[1234]le-* | sh3ele-* | sh64-* | sh64le-* \ + | sparc-* | sparc64-* | sparc64b-* | sparc64v-* | sparc86x-* | sparclet-* \ + | sparclite-* \ + | sparcv8-* | sparcv9-* | sparcv9b-* | sparcv9v-* | sv1-* | sx*-* \ + | tahoe-* \ + | tic30-* | tic4x-* | tic54x-* | tic55x-* | tic6x-* | tic80-* \ + | tile*-* \ + | tron-* \ + | ubicom32-* \ + | v850-* | v850e-* | v850e1-* | v850es-* | v850e2-* | v850e2v3-* \ + | vax-* \ + | visium-* \ + | wasm32-* \ + | we32k-* \ + | x86-* | x86_64-* | xc16x-* | xps100-* \ + | xstormy16-* | xtensa*-* \ + | ymp-* \ + | z8k-* | z80-*) + ;; + # Recognize the basic CPU types without company name, with glob match. + xtensa*) + basic_machine=$basic_machine-unknown + ;; + # Recognize the various machine names and aliases which stand + # for a CPU type and a company and sometimes even an OS. + 386bsd) + basic_machine=i386-unknown + os=-bsd + ;; + 3b1 | 7300 | 7300-att | att-7300 | pc7300 | safari | unixpc) + basic_machine=m68000-att + ;; + 3b*) + basic_machine=we32k-att + ;; + a29khif) + basic_machine=a29k-amd + os=-udi + ;; + abacus) + basic_machine=abacus-unknown + ;; + adobe68k) + basic_machine=m68010-adobe + os=-scout + ;; + alliant | fx80) + basic_machine=fx80-alliant + ;; + altos | altos3068) + basic_machine=m68k-altos + ;; + am29k) + basic_machine=a29k-none + os=-bsd + ;; + amd64) + basic_machine=x86_64-pc + ;; + amd64-*) + basic_machine=x86_64-`echo $basic_machine | sed 's/^[^-]*-//'` + ;; + amdahl) + basic_machine=580-amdahl + os=-sysv + ;; + amiga | amiga-*) + basic_machine=m68k-unknown + ;; + amigaos | amigados) + basic_machine=m68k-unknown + os=-amigaos + ;; + amigaunix | amix) + basic_machine=m68k-unknown + os=-sysv4 + ;; + apollo68) + basic_machine=m68k-apollo + os=-sysv + ;; + apollo68bsd) + basic_machine=m68k-apollo + os=-bsd + ;; + aros) + basic_machine=i386-pc + os=-aros + ;; + asmjs) + basic_machine=asmjs-unknown + ;; + aux) + basic_machine=m68k-apple + os=-aux + ;; + balance) + basic_machine=ns32k-sequent + os=-dynix + ;; + blackfin) + basic_machine=bfin-unknown + os=-linux + ;; + blackfin-*) + basic_machine=bfin-`echo $basic_machine | sed 's/^[^-]*-//'` + os=-linux + ;; + bluegene*) + basic_machine=powerpc-ibm + os=-cnk + ;; + c54x-*) + basic_machine=tic54x-`echo $basic_machine | sed 's/^[^-]*-//'` + ;; + c55x-*) + basic_machine=tic55x-`echo $basic_machine | sed 's/^[^-]*-//'` + ;; + c6x-*) + basic_machine=tic6x-`echo $basic_machine | sed 's/^[^-]*-//'` + ;; + c90) + basic_machine=c90-cray + os=-unicos + ;; + cegcc) + basic_machine=arm-unknown + os=-cegcc + ;; + convex-c1) + basic_machine=c1-convex + os=-bsd + ;; + convex-c2) + basic_machine=c2-convex + os=-bsd + ;; + convex-c32) + basic_machine=c32-convex + os=-bsd + ;; + convex-c34) + basic_machine=c34-convex + os=-bsd + ;; + convex-c38) + basic_machine=c38-convex + os=-bsd + ;; + cray | j90) + basic_machine=j90-cray + os=-unicos + ;; + craynv) + basic_machine=craynv-cray + os=-unicosmp + ;; + cr16 | cr16-*) + basic_machine=cr16-unknown + os=-elf + ;; + crds | unos) + basic_machine=m68k-crds + ;; + crisv32 | crisv32-* | etraxfs*) + basic_machine=crisv32-axis + ;; + cris | cris-* | etrax*) + basic_machine=cris-axis + ;; + crx) + basic_machine=crx-unknown + os=-elf + ;; + da30 | da30-*) + basic_machine=m68k-da30 + ;; + decstation | decstation-3100 | pmax | pmax-* | pmin | dec3100 | decstatn) + basic_machine=mips-dec + ;; + decsystem10* | dec10*) + basic_machine=pdp10-dec + os=-tops10 + ;; + decsystem20* | dec20*) + basic_machine=pdp10-dec + os=-tops20 + ;; + delta | 3300 | motorola-3300 | motorola-delta \ + | 3300-motorola | delta-motorola) + basic_machine=m68k-motorola + ;; + delta88) + basic_machine=m88k-motorola + os=-sysv3 + ;; + dicos) + basic_machine=i686-pc + os=-dicos + ;; + djgpp) + basic_machine=i586-pc + os=-msdosdjgpp + ;; + dpx20 | dpx20-*) + basic_machine=rs6000-bull + os=-bosx + ;; + dpx2* | dpx2*-bull) + basic_machine=m68k-bull + os=-sysv3 + ;; + e500v[12]) + basic_machine=powerpc-unknown + os=$os"spe" + ;; + e500v[12]-*) + basic_machine=powerpc-`echo $basic_machine | sed 's/^[^-]*-//'` + os=$os"spe" + ;; + ebmon29k) + basic_machine=a29k-amd + os=-ebmon + ;; + elxsi) + basic_machine=elxsi-elxsi + os=-bsd + ;; + encore | umax | mmax) + basic_machine=ns32k-encore + ;; + es1800 | OSE68k | ose68k | ose | OSE) + basic_machine=m68k-ericsson + os=-ose + ;; + fx2800) + basic_machine=i860-alliant + ;; + genix) + basic_machine=ns32k-ns + ;; + gmicro) + basic_machine=tron-gmicro + os=-sysv + ;; + go32) + basic_machine=i386-pc + os=-go32 + ;; + h3050r* | hiux*) + basic_machine=hppa1.1-hitachi + os=-hiuxwe2 + ;; + h8300hms) + basic_machine=h8300-hitachi + os=-hms + ;; + h8300xray) + basic_machine=h8300-hitachi + os=-xray + ;; + h8500hms) + basic_machine=h8500-hitachi + os=-hms + ;; + harris) + basic_machine=m88k-harris + os=-sysv3 + ;; + hp300-*) + basic_machine=m68k-hp + ;; + hp300bsd) + basic_machine=m68k-hp + os=-bsd + ;; + hp300hpux) + basic_machine=m68k-hp + os=-hpux + ;; + hp3k9[0-9][0-9] | hp9[0-9][0-9]) + basic_machine=hppa1.0-hp + ;; + hp9k2[0-9][0-9] | hp9k31[0-9]) + basic_machine=m68000-hp + ;; + hp9k3[2-9][0-9]) + basic_machine=m68k-hp + ;; + hp9k6[0-9][0-9] | hp6[0-9][0-9]) + basic_machine=hppa1.0-hp + ;; + hp9k7[0-79][0-9] | hp7[0-79][0-9]) + basic_machine=hppa1.1-hp + ;; + hp9k78[0-9] | hp78[0-9]) + # FIXME: really hppa2.0-hp + basic_machine=hppa1.1-hp + ;; + hp9k8[67]1 | hp8[67]1 | hp9k80[24] | hp80[24] | hp9k8[78]9 | hp8[78]9 | hp9k893 | hp893) + # FIXME: really hppa2.0-hp + basic_machine=hppa1.1-hp + ;; + hp9k8[0-9][13679] | hp8[0-9][13679]) + basic_machine=hppa1.1-hp + ;; + hp9k8[0-9][0-9] | hp8[0-9][0-9]) + basic_machine=hppa1.0-hp + ;; + hppa-next) + os=-nextstep3 + ;; + hppaosf) + basic_machine=hppa1.1-hp + os=-osf + ;; + hppro) + basic_machine=hppa1.1-hp + os=-proelf + ;; + i370-ibm* | ibm*) + basic_machine=i370-ibm + ;; + i*86v32) + basic_machine=`echo $1 | sed -e 's/86.*/86-pc/'` + os=-sysv32 + ;; + i*86v4*) + basic_machine=`echo $1 | sed -e 's/86.*/86-pc/'` + os=-sysv4 + ;; + i*86v) + basic_machine=`echo $1 | sed -e 's/86.*/86-pc/'` + os=-sysv + ;; + i*86sol2) + basic_machine=`echo $1 | sed -e 's/86.*/86-pc/'` + os=-solaris2 + ;; + i386mach) + basic_machine=i386-mach + os=-mach + ;; + i386-vsta | vsta) + basic_machine=i386-unknown + os=-vsta + ;; + iris | iris4d) + basic_machine=mips-sgi + case $os in + -irix*) + ;; + *) + os=-irix4 + ;; + esac + ;; + isi68 | isi) + basic_machine=m68k-isi + os=-sysv + ;; + leon-*|leon[3-9]-*) + basic_machine=sparc-`echo $basic_machine | sed 's/-.*//'` + ;; + m68knommu) + basic_machine=m68k-unknown + os=-linux + ;; + m68knommu-*) + basic_machine=m68k-`echo $basic_machine | sed 's/^[^-]*-//'` + os=-linux + ;; + m88k-omron*) + basic_machine=m88k-omron + ;; + magnum | m3230) + basic_machine=mips-mips + os=-sysv + ;; + merlin) + basic_machine=ns32k-utek + os=-sysv + ;; + microblaze*) + basic_machine=microblaze-xilinx + ;; + mingw64) + basic_machine=x86_64-pc + os=-mingw64 + ;; + mingw32) + basic_machine=i686-pc + os=-mingw32 + ;; + mingw32ce) + basic_machine=arm-unknown + os=-mingw32ce + ;; + miniframe) + basic_machine=m68000-convergent + ;; + *mint | -mint[0-9]* | *MiNT | *MiNT[0-9]*) + basic_machine=m68k-atari + os=-mint + ;; + mips3*-*) + basic_machine=`echo $basic_machine | sed -e 's/mips3/mips64/'` + ;; + mips3*) + basic_machine=`echo $basic_machine | sed -e 's/mips3/mips64/'`-unknown + ;; + monitor) + basic_machine=m68k-rom68k + os=-coff + ;; + morphos) + basic_machine=powerpc-unknown + os=-morphos + ;; + moxiebox) + basic_machine=moxie-unknown + os=-moxiebox + ;; + msdos) + basic_machine=i386-pc + os=-msdos + ;; + ms1-*) + basic_machine=`echo $basic_machine | sed -e 's/ms1-/mt-/'` + ;; + msys) + basic_machine=i686-pc + os=-msys + ;; + mvs) + basic_machine=i370-ibm + os=-mvs + ;; + nacl) + basic_machine=le32-unknown + os=-nacl + ;; + ncr3000) + basic_machine=i486-ncr + os=-sysv4 + ;; + netbsd386) + basic_machine=i386-unknown + os=-netbsd + ;; + netwinder) + basic_machine=armv4l-rebel + os=-linux + ;; + news | news700 | news800 | news900) + basic_machine=m68k-sony + os=-newsos + ;; + news1000) + basic_machine=m68030-sony + os=-newsos + ;; + news-3600 | risc-news) + basic_machine=mips-sony + os=-newsos + ;; + necv70) + basic_machine=v70-nec + os=-sysv + ;; + next | m*-next ) + basic_machine=m68k-next + case $os in + -nextstep* ) + ;; + -ns2*) + os=-nextstep2 + ;; + *) + os=-nextstep3 + ;; + esac + ;; + nh3000) + basic_machine=m68k-harris + os=-cxux + ;; + nh[45]000) + basic_machine=m88k-harris + os=-cxux + ;; + nindy960) + basic_machine=i960-intel + os=-nindy + ;; + mon960) + basic_machine=i960-intel + os=-mon960 + ;; + nonstopux) + basic_machine=mips-compaq + os=-nonstopux + ;; + np1) + basic_machine=np1-gould + ;; + neo-tandem) + basic_machine=neo-tandem + ;; + nse-tandem) + basic_machine=nse-tandem + ;; + nsr-tandem) + basic_machine=nsr-tandem + ;; + nsx-tandem) + basic_machine=nsx-tandem + ;; + op50n-* | op60c-*) + basic_machine=hppa1.1-oki + os=-proelf + ;; + openrisc | openrisc-*) + basic_machine=or32-unknown + ;; + os400) + basic_machine=powerpc-ibm + os=-os400 + ;; + OSE68000 | ose68000) + basic_machine=m68000-ericsson + os=-ose + ;; + os68k) + basic_machine=m68k-none + os=-os68k + ;; + pa-hitachi) + basic_machine=hppa1.1-hitachi + os=-hiuxwe2 + ;; + paragon) + basic_machine=i860-intel + os=-osf + ;; + parisc) + basic_machine=hppa-unknown + os=-linux + ;; + parisc-*) + basic_machine=hppa-`echo $basic_machine | sed 's/^[^-]*-//'` + os=-linux + ;; + pbd) + basic_machine=sparc-tti + ;; + pbb) + basic_machine=m68k-tti + ;; + pc532 | pc532-*) + basic_machine=ns32k-pc532 + ;; + pc98) + basic_machine=i386-pc + ;; + pc98-*) + basic_machine=i386-`echo $basic_machine | sed 's/^[^-]*-//'` + ;; + pentium | p5 | k5 | k6 | nexgen | viac3) + basic_machine=i586-pc + ;; + pentiumpro | p6 | 6x86 | athlon | athlon_*) + basic_machine=i686-pc + ;; + pentiumii | pentium2 | pentiumiii | pentium3) + basic_machine=i686-pc + ;; + pentium4) + basic_machine=i786-pc + ;; + pentium-* | p5-* | k5-* | k6-* | nexgen-* | viac3-*) + basic_machine=i586-`echo $basic_machine | sed 's/^[^-]*-//'` + ;; + pentiumpro-* | p6-* | 6x86-* | athlon-*) + basic_machine=i686-`echo $basic_machine | sed 's/^[^-]*-//'` + ;; + pentiumii-* | pentium2-* | pentiumiii-* | pentium3-*) + basic_machine=i686-`echo $basic_machine | sed 's/^[^-]*-//'` + ;; + pentium4-*) + basic_machine=i786-`echo $basic_machine | sed 's/^[^-]*-//'` + ;; + pn) + basic_machine=pn-gould + ;; + power) basic_machine=power-ibm + ;; + ppc | ppcbe) basic_machine=powerpc-unknown + ;; + ppc-* | ppcbe-*) + basic_machine=powerpc-`echo $basic_machine | sed 's/^[^-]*-//'` + ;; + ppcle | powerpclittle) + basic_machine=powerpcle-unknown + ;; + ppcle-* | powerpclittle-*) + basic_machine=powerpcle-`echo $basic_machine | sed 's/^[^-]*-//'` + ;; + ppc64) basic_machine=powerpc64-unknown + ;; + ppc64-*) basic_machine=powerpc64-`echo $basic_machine | sed 's/^[^-]*-//'` + ;; + ppc64le | powerpc64little) + basic_machine=powerpc64le-unknown + ;; + ppc64le-* | powerpc64little-*) + basic_machine=powerpc64le-`echo $basic_machine | sed 's/^[^-]*-//'` + ;; + ps2) + basic_machine=i386-ibm + ;; + pw32) + basic_machine=i586-unknown + os=-pw32 + ;; + rdos | rdos64) + basic_machine=x86_64-pc + os=-rdos + ;; + rdos32) + basic_machine=i386-pc + os=-rdos + ;; + rom68k) + basic_machine=m68k-rom68k + os=-coff + ;; + rm[46]00) + basic_machine=mips-siemens + ;; + rtpc | rtpc-*) + basic_machine=romp-ibm + ;; + s390 | s390-*) + basic_machine=s390-ibm + ;; + s390x | s390x-*) + basic_machine=s390x-ibm + ;; + sa29200) + basic_machine=a29k-amd + os=-udi + ;; + sb1) + basic_machine=mipsisa64sb1-unknown + ;; + sb1el) + basic_machine=mipsisa64sb1el-unknown + ;; + sde) + basic_machine=mipsisa32-sde + os=-elf + ;; + sei) + basic_machine=mips-sei + os=-seiux + ;; + sequent) + basic_machine=i386-sequent + ;; + sh) + basic_machine=sh-hitachi + os=-hms + ;; + sh5el) + basic_machine=sh5le-unknown + ;; + sh64) + basic_machine=sh64-unknown + ;; + sparclite-wrs | simso-wrs) + basic_machine=sparclite-wrs + os=-vxworks + ;; + sps7) + basic_machine=m68k-bull + os=-sysv2 + ;; + spur) + basic_machine=spur-unknown + ;; + st2000) + basic_machine=m68k-tandem + ;; + stratus) + basic_machine=i860-stratus + os=-sysv4 + ;; + strongarm-* | thumb-*) + basic_machine=arm-`echo $basic_machine | sed 's/^[^-]*-//'` + ;; + sun2) + basic_machine=m68000-sun + ;; + sun2os3) + basic_machine=m68000-sun + os=-sunos3 + ;; + sun2os4) + basic_machine=m68000-sun + os=-sunos4 + ;; + sun3os3) + basic_machine=m68k-sun + os=-sunos3 + ;; + sun3os4) + basic_machine=m68k-sun + os=-sunos4 + ;; + sun4os3) + basic_machine=sparc-sun + os=-sunos3 + ;; + sun4os4) + basic_machine=sparc-sun + os=-sunos4 + ;; + sun4sol2) + basic_machine=sparc-sun + os=-solaris2 + ;; + sun3 | sun3-*) + basic_machine=m68k-sun + ;; + sun4) + basic_machine=sparc-sun + ;; + sun386 | sun386i | roadrunner) + basic_machine=i386-sun + ;; + sv1) + basic_machine=sv1-cray + os=-unicos + ;; + symmetry) + basic_machine=i386-sequent + os=-dynix + ;; + t3e) + basic_machine=alphaev5-cray + os=-unicos + ;; + t90) + basic_machine=t90-cray + os=-unicos + ;; + tile*) + basic_machine=$basic_machine-unknown + os=-linux-gnu + ;; + tx39) + basic_machine=mipstx39-unknown + ;; + tx39el) + basic_machine=mipstx39el-unknown + ;; + toad1) + basic_machine=pdp10-xkl + os=-tops20 + ;; + tower | tower-32) + basic_machine=m68k-ncr + ;; + tpf) + basic_machine=s390x-ibm + os=-tpf + ;; + udi29k) + basic_machine=a29k-amd + os=-udi + ;; + ultra3) + basic_machine=a29k-nyu + os=-sym1 + ;; + v810 | necv810) + basic_machine=v810-nec + os=-none + ;; + vaxv) + basic_machine=vax-dec + os=-sysv + ;; + vms) + basic_machine=vax-dec + os=-vms + ;; + vpp*|vx|vx-*) + basic_machine=f301-fujitsu + ;; + vxworks960) + basic_machine=i960-wrs + os=-vxworks + ;; + vxworks68) + basic_machine=m68k-wrs + os=-vxworks + ;; + vxworks29k) + basic_machine=a29k-wrs + os=-vxworks + ;; + wasm32) + basic_machine=wasm32-unknown + ;; + w65*) + basic_machine=w65-wdc + os=-none + ;; + w89k-*) + basic_machine=hppa1.1-winbond + os=-proelf + ;; + xbox) + basic_machine=i686-pc + os=-mingw32 + ;; + xps | xps100) + basic_machine=xps100-honeywell + ;; + xscale-* | xscalee[bl]-*) + basic_machine=`echo $basic_machine | sed 's/^xscale/arm/'` + ;; + ymp) + basic_machine=ymp-cray + os=-unicos + ;; + z8k-*-coff) + basic_machine=z8k-unknown + os=-sim + ;; + z80-*-coff) + basic_machine=z80-unknown + os=-sim + ;; + none) + basic_machine=none-none + os=-none + ;; + +# Here we handle the default manufacturer of certain CPU types. It is in +# some cases the only manufacturer, in others, it is the most popular. + w89k) + basic_machine=hppa1.1-winbond + ;; + op50n) + basic_machine=hppa1.1-oki + ;; + op60c) + basic_machine=hppa1.1-oki + ;; + romp) + basic_machine=romp-ibm + ;; + mmix) + basic_machine=mmix-knuth + ;; + rs6000) + basic_machine=rs6000-ibm + ;; + vax) + basic_machine=vax-dec + ;; + pdp10) + # there are many clones, so DEC is not a safe bet + basic_machine=pdp10-unknown + ;; + pdp11) + basic_machine=pdp11-dec + ;; + we32k) + basic_machine=we32k-att + ;; + sh[1234] | sh[24]a | sh[24]aeb | sh[34]eb | sh[1234]le | sh[23]ele) + basic_machine=sh-unknown + ;; + sparc | sparcv8 | sparcv9 | sparcv9b | sparcv9v) + basic_machine=sparc-sun + ;; + cydra) + basic_machine=cydra-cydrome + ;; + orion) + basic_machine=orion-highlevel + ;; + orion105) + basic_machine=clipper-highlevel + ;; + mac | mpw | mac-mpw) + basic_machine=m68k-apple + ;; + pmac | pmac-mpw) + basic_machine=powerpc-apple + ;; + *-unknown) + # Make sure to match an already-canonicalized machine name. + ;; + *) + echo Invalid configuration \`$1\': machine \`$basic_machine\' not recognized 1>&2 + exit 1 + ;; +esac + +# Here we canonicalize certain aliases for manufacturers. +case $basic_machine in + *-digital*) + basic_machine=`echo $basic_machine | sed 's/digital.*/dec/'` + ;; + *-commodore*) + basic_machine=`echo $basic_machine | sed 's/commodore.*/cbm/'` + ;; + *) + ;; +esac + +# Decode manufacturer-specific aliases for certain operating systems. + +if [ x"$os" != x"" ] +then +case $os in + # First match some system type aliases + # that might get confused with valid system types. + # -solaris* is a basic system type, with this one exception. + -auroraux) + os=-auroraux + ;; + -solaris1 | -solaris1.*) + os=`echo $os | sed -e 's|solaris1|sunos4|'` + ;; + -solaris) + os=-solaris2 + ;; + -svr4*) + os=-sysv4 + ;; + -unixware*) + os=-sysv4.2uw + ;; + -gnu/linux*) + os=`echo $os | sed -e 's|gnu/linux|linux-gnu|'` + ;; + # First accept the basic system types. + # The portable systems comes first. + # Each alternative MUST END IN A *, to match a version number. + # -sysv* is not here because it comes later, after sysvr4. + -gnu* | -bsd* | -mach* | -minix* | -genix* | -ultrix* | -irix* \ + | -*vms* | -sco* | -esix* | -isc* | -aix* | -cnk* | -sunos | -sunos[34]*\ + | -hpux* | -unos* | -osf* | -luna* | -dgux* | -auroraux* | -solaris* \ + | -sym* | -kopensolaris* | -plan9* \ + | -amigaos* | -amigados* | -msdos* | -newsos* | -unicos* | -aof* \ + | -aos* | -aros* | -cloudabi* | -sortix* \ + | -nindy* | -vxsim* | -vxworks* | -ebmon* | -hms* | -mvs* \ + | -clix* | -riscos* | -uniplus* | -iris* | -rtu* | -xenix* \ + | -hiux* | -386bsd* | -knetbsd* | -mirbsd* | -netbsd* \ + | -bitrig* | -openbsd* | -solidbsd* | -libertybsd* \ + | -ekkobsd* | -kfreebsd* | -freebsd* | -riscix* | -lynxos* \ + | -bosx* | -nextstep* | -cxux* | -aout* | -elf* | -oabi* \ + | -ptx* | -coff* | -ecoff* | -winnt* | -domain* | -vsta* \ + | -udi* | -eabi* | -lites* | -ieee* | -go32* | -aux* \ + | -chorusos* | -chorusrdb* | -cegcc* | -glidix* \ + | -cygwin* | -msys* | -pe* | -psos* | -moss* | -proelf* | -rtems* \ + | -midipix* | -mingw32* | -mingw64* | -linux-gnu* | -linux-android* \ + | -linux-newlib* | -linux-musl* | -linux-uclibc* \ + | -uxpv* | -beos* | -mpeix* | -udk* | -moxiebox* \ + | -interix* | -uwin* | -mks* | -rhapsody* | -darwin* | -opened* \ + | -openstep* | -oskit* | -conix* | -pw32* | -nonstopux* \ + | -storm-chaos* | -tops10* | -tenex* | -tops20* | -its* \ + | -os2* | -vos* | -palmos* | -uclinux* | -nucleus* \ + | -morphos* | -superux* | -rtmk* | -rtmk-nova* | -windiss* \ + | -powermax* | -dnix* | -nx6 | -nx7 | -sei* | -dragonfly* \ + | -skyos* | -haiku* | -rdos* | -toppers* | -drops* | -es* \ + | -onefs* | -tirtos* | -phoenix* | -fuchsia* | -redox*) + # Remember, each alternative MUST END IN *, to match a version number. + ;; + -qnx*) + case $basic_machine in + x86-* | i*86-*) + ;; + *) + os=-nto$os + ;; + esac + ;; + -nto-qnx*) + ;; + -nto*) + os=`echo $os | sed -e 's|nto|nto-qnx|'` + ;; + -sim | -es1800* | -hms* | -xray | -os68k* | -none* | -v88r* \ + | -windows* | -osx | -abug | -netware* | -os9* | -beos* | -haiku* \ + | -macos* | -mpw* | -magic* | -mmixware* | -mon960* | -lnews*) + ;; + -mac*) + os=`echo $os | sed -e 's|mac|macos|'` + ;; + -linux-dietlibc) + os=-linux-dietlibc + ;; + -linux*) + os=`echo $os | sed -e 's|linux|linux-gnu|'` + ;; + -sunos5*) + os=`echo $os | sed -e 's|sunos5|solaris2|'` + ;; + -sunos6*) + os=`echo $os | sed -e 's|sunos6|solaris3|'` + ;; + -opened*) + os=-openedition + ;; + -os400*) + os=-os400 + ;; + -wince*) + os=-wince + ;; + -osfrose*) + os=-osfrose + ;; + -osf*) + os=-osf + ;; + -utek*) + os=-bsd + ;; + -dynix*) + os=-bsd + ;; + -acis*) + os=-aos + ;; + -atheos*) + os=-atheos + ;; + -syllable*) + os=-syllable + ;; + -386bsd) + os=-bsd + ;; + -ctix* | -uts*) + os=-sysv + ;; + -nova*) + os=-rtmk-nova + ;; + -ns2 ) + os=-nextstep2 + ;; + -nsk*) + os=-nsk + ;; + # Preserve the version number of sinix5. + -sinix5.*) + os=`echo $os | sed -e 's|sinix|sysv|'` + ;; + -sinix*) + os=-sysv4 + ;; + -tpf*) + os=-tpf + ;; + -triton*) + os=-sysv3 + ;; + -oss*) + os=-sysv3 + ;; + -svr4) + os=-sysv4 + ;; + -svr3) + os=-sysv3 + ;; + -sysvr4) + os=-sysv4 + ;; + # This must come after -sysvr4. + -sysv*) + ;; + -ose*) + os=-ose + ;; + -es1800*) + os=-ose + ;; + -xenix) + os=-xenix + ;; + -*mint | -mint[0-9]* | -*MiNT | -MiNT[0-9]*) + os=-mint + ;; + -aros*) + os=-aros + ;; + -zvmoe) + os=-zvmoe + ;; + -dicos*) + os=-dicos + ;; + -nacl*) + ;; + -ios) + ;; + -none) + ;; + *) + # Get rid of the `-' at the beginning of $os. + os=`echo $os | sed 's/[^-]*-//'` + echo Invalid configuration \`$1\': system \`$os\' not recognized 1>&2 + exit 1 + ;; +esac +else + +# Here we handle the default operating systems that come with various machines. +# The value should be what the vendor currently ships out the door with their +# machine or put another way, the most popular os provided with the machine. + +# Note that if you're going to try to match "-MANUFACTURER" here (say, +# "-sun"), then you have to tell the case statement up towards the top +# that MANUFACTURER isn't an operating system. Otherwise, code above +# will signal an error saying that MANUFACTURER isn't an operating +# system, and we'll never get to this point. + +case $basic_machine in + score-*) + os=-elf + ;; + spu-*) + os=-elf + ;; + *-acorn) + os=-riscix1.2 + ;; + arm*-rebel) + os=-linux + ;; + arm*-semi) + os=-aout + ;; + c4x-* | tic4x-*) + os=-coff + ;; + c8051-*) + os=-elf + ;; + hexagon-*) + os=-elf + ;; + tic54x-*) + os=-coff + ;; + tic55x-*) + os=-coff + ;; + tic6x-*) + os=-coff + ;; + # This must come before the *-dec entry. + pdp10-*) + os=-tops20 + ;; + pdp11-*) + os=-none + ;; + *-dec | vax-*) + os=-ultrix4.2 + ;; + m68*-apollo) + os=-domain + ;; + i386-sun) + os=-sunos4.0.2 + ;; + m68000-sun) + os=-sunos3 + ;; + m68*-cisco) + os=-aout + ;; + mep-*) + os=-elf + ;; + mips*-cisco) + os=-elf + ;; + mips*-*) + os=-elf + ;; + or32-*) + os=-coff + ;; + *-tti) # must be before sparc entry or we get the wrong os. + os=-sysv3 + ;; + sparc-* | *-sun) + os=-sunos4.1.1 + ;; + pru-*) + os=-elf + ;; + *-be) + os=-beos + ;; + *-haiku) + os=-haiku + ;; + *-ibm) + os=-aix + ;; + *-knuth) + os=-mmixware + ;; + *-wec) + os=-proelf + ;; + *-winbond) + os=-proelf + ;; + *-oki) + os=-proelf + ;; + *-hp) + os=-hpux + ;; + *-hitachi) + os=-hiux + ;; + i860-* | *-att | *-ncr | *-altos | *-motorola | *-convergent) + os=-sysv + ;; + *-cbm) + os=-amigaos + ;; + *-dg) + os=-dgux + ;; + *-dolphin) + os=-sysv3 + ;; + m68k-ccur) + os=-rtu + ;; + m88k-omron*) + os=-luna + ;; + *-next ) + os=-nextstep + ;; + *-sequent) + os=-ptx + ;; + *-crds) + os=-unos + ;; + *-ns) + os=-genix + ;; + i370-*) + os=-mvs + ;; + *-next) + os=-nextstep3 + ;; + *-gould) + os=-sysv + ;; + *-highlevel) + os=-bsd + ;; + *-encore) + os=-bsd + ;; + *-sgi) + os=-irix + ;; + *-siemens) + os=-sysv4 + ;; + *-masscomp) + os=-rtu + ;; + f30[01]-fujitsu | f700-fujitsu) + os=-uxpv + ;; + *-rom68k) + os=-coff + ;; + *-*bug) + os=-coff + ;; + *-apple) + os=-macos + ;; + *-atari*) + os=-mint + ;; + *) + os=-none + ;; +esac +fi + +# Here we handle the case where we know the os, and the CPU type, but not the +# manufacturer. We pick the logical manufacturer. +vendor=unknown +case $basic_machine in + *-unknown) + case $os in + -riscix*) + vendor=acorn + ;; + -sunos*) + vendor=sun + ;; + -cnk*|-aix*) + vendor=ibm + ;; + -beos*) + vendor=be + ;; + -hpux*) + vendor=hp + ;; + -mpeix*) + vendor=hp + ;; + -hiux*) + vendor=hitachi + ;; + -unos*) + vendor=crds + ;; + -dgux*) + vendor=dg + ;; + -luna*) + vendor=omron + ;; + -genix*) + vendor=ns + ;; + -mvs* | -opened*) + vendor=ibm + ;; + -os400*) + vendor=ibm + ;; + -ptx*) + vendor=sequent + ;; + -tpf*) + vendor=ibm + ;; + -vxsim* | -vxworks* | -windiss*) + vendor=wrs + ;; + -aux*) + vendor=apple + ;; + -hms*) + vendor=hitachi + ;; + -mpw* | -macos*) + vendor=apple + ;; + -*mint | -mint[0-9]* | -*MiNT | -MiNT[0-9]*) + vendor=atari + ;; + -vos*) + vendor=stratus + ;; + esac + basic_machine=`echo $basic_machine | sed "s/unknown/$vendor/"` + ;; +esac + +echo $basic_machine$os +exit + +# Local variables: +# eval: (add-hook 'write-file-hooks 'time-stamp) +# time-stamp-start: "timestamp='" +# time-stamp-format: "%:y-%02m-%02d" +# time-stamp-end: "'" +# End: diff --git a/contrib/depends/description.md b/contrib/depends/description.md new file mode 100644 index 00000000000..74f9ef3f205 --- /dev/null +++ b/contrib/depends/description.md @@ -0,0 +1,53 @@ +This is a system of building and caching dependencies necessary for building Bitcoin. +There are several features that make it different from most similar systems: + +### It is designed to be builder and host agnostic + +In theory, binaries for any target OS/architecture can be created, from a +builder running any OS/architecture. In practice, build-side tools must be +specified when the defaults don't fit, and packages must be amended to work +on new hosts. For now, a build architecture of x86_64 is assumed, either on +Linux or OSX. + +### No reliance on timestamps + +File presence is used to determine what needs to be built. This makes the +results distributable and easily digestable by automated builders. + +### Each build only has its specified dependencies available at build-time. + +For each build, the sysroot is wiped and the (recursive) dependencies are +installed. This makes each build deterministic, since there will never be any +unknown files available to cause side-effects. + +### Each package is cached and only rebuilt as needed. + +Before building, a unique build-id is generated for each package. This id +consists of a hash of all files used to build the package (Makefiles, packages, +etc), and as well as a hash of the same data for each recursive dependency. If +any portion of a package's build recipe changes, it will be rebuilt as well as +any other package that depends on it. If any of the main makefiles (Makefile, +funcs.mk, etc) are changed, all packages will be rebuilt. After building, the +results are cached into a tarball that can be re-used and distributed. + +### Package build results are (relatively) deterministic. + +Each package is configured and patched so that it will yield the same +build-results with each consequent build, within a reasonable set of +constraints. Some things like timestamp insertion are unavoidable, and are +beyond the scope of this system. Additionally, the toolchain itself must be +capable of deterministic results. When revisions are properly bumped, a cached +build should represent an exact single payload. + +### Sources are fetched and verified automatically + +Each package must define its source location and checksum. The build will fail +if the fetched source does not match. Sources may be pre-seeded and/or cached +as desired. + +### Self-cleaning + +Build and staging dirs are wiped after use, and any previous version of a +cached result is removed following a successful build. Automated builders +should be able to build each revision and store the results with no further +intervention. diff --git a/contrib/depends/funcs.mk b/contrib/depends/funcs.mk new file mode 100644 index 00000000000..46914436142 --- /dev/null +++ b/contrib/depends/funcs.mk @@ -0,0 +1,253 @@ +define int_vars +#Set defaults for vars which may be overridden per-package +$(1)_cc=$($($(1)_type)_CC) +$(1)_cxx=$($($(1)_type)_CXX) +$(1)_objc=$($($(1)_type)_OBJC) +$(1)_objcxx=$($($(1)_type)_OBJCXX) +$(1)_ar=$($($(1)_type)_AR) +$(1)_ranlib=$($($(1)_type)_RANLIB) +$(1)_libtool=$($($(1)_type)_LIBTOOL) +$(1)_nm=$($($(1)_type)_NM) +$(1)_cflags=$($($(1)_type)_CFLAGS) $($($(1)_type)_$(release_type)_CFLAGS) +$(1)_cxxflags=$($($(1)_type)_CXXFLAGS) $($($(1)_type)_$(release_type)_CXXFLAGS) +$(1)_ldflags=$($($(1)_type)_LDFLAGS) $($($(1)_type)_$(release_type)_LDFLAGS) -L$($($(1)_type)_prefix)/lib +$(1)_cppflags=$($($(1)_type)_CPPFLAGS) $($($(1)_type)_$(release_type)_CPPFLAGS) -I$($($(1)_type)_prefix)/include +$(1)_recipe_hash:= +endef + +define int_get_all_dependencies +$(sort $(foreach dep,$(2),$(2) $(call int_get_all_dependencies,$(1),$($(dep)_dependencies)))) +endef + +define fetch_file_inner + ( mkdir -p $$($(1)_download_dir) && echo Fetching $(3) from $(2) && \ + $(build_DOWNLOAD) "$$($(1)_download_dir)/$(4).temp" "$(2)/$(3)" && \ + echo "$(5) $$($(1)_download_dir)/$(4).temp" > $$($(1)_download_dir)/.$(4).hash && \ + $(build_SHA256SUM) -c $$($(1)_download_dir)/.$(4).hash && \ + mv $$($(1)_download_dir)/$(4).temp $$($(1)_source_dir)/$(4) && \ + rm -rf $$($(1)_download_dir) ) +endef + +define fetch_file + ( test -f $$($(1)_source_dir)/$(4) || \ + ( $(call fetch_file_inner,$(1),$(2),$(3),$(4),$(5)) || \ + $(call fetch_file_inner,$(1),$(FALLBACK_DOWNLOAD_PATH),$(3),$(4),$(5)))) +endef + +define int_get_build_recipe_hash +$(eval $(1)_all_file_checksums:=$(shell $(build_SHA256SUM) $(meta_depends) packages/$(1).mk $(addprefix $(PATCHES_PATH)/$(1)/,$($(1)_patches)) | cut -d" " -f1)) +$(eval $(1)_recipe_hash:=$(shell echo -n "$($(1)_all_file_checksums)" | $(build_SHA256SUM) | cut -d" " -f1)) +endef + +define int_get_build_id +$(eval $(1)_dependencies += $($(1)_$(host_arch)_$(host_os)_dependencies) $($(1)_$(host_os)_dependencies)) +$(eval $(1)_all_dependencies:=$(call int_get_all_dependencies,$(1),$($($(1)_type)_native_toolchain) $($(1)_dependencies))) +$(foreach dep,$($(1)_all_dependencies),$(eval $(1)_build_id_deps+=$(dep)-$($(dep)_version)-$($(dep)_recipe_hash))) +$(eval $(1)_build_id_long:=$(1)-$($(1)_version)-$($(1)_recipe_hash)-$(release_type) $($(1)_build_id_deps) $($($(1)_type)_id_string)) +$(eval $(1)_build_id:=$(shell echo -n "$($(1)_build_id_long)" | $(build_SHA256SUM) | cut -c-$(HASH_LENGTH))) +final_build_id_long+=$($(package)_build_id_long) + +#compute package-specific paths +$(1)_build_subdir?=. +$(1)_download_file?=$($(1)_file_name) +$(1)_source_dir:=$(SOURCES_PATH) +$(1)_source:=$$($(1)_source_dir)/$($(1)_file_name) +$(1)_staging_dir=$(base_staging_dir)/$(host)/$(1)/$($(1)_version)-$($(1)_build_id) +$(1)_staging_prefix_dir:=$$($(1)_staging_dir)$($($(1)_type)_prefix) +$(1)_extract_dir:=$(base_build_dir)/$(host)/$(1)/$($(1)_version)-$($(1)_build_id) +$(1)_download_dir:=$(base_download_dir)/$(1)-$($(1)_version) +$(1)_build_dir:=$$($(1)_extract_dir)/$$($(1)_build_subdir) +$(1)_cached_checksum:=$(BASE_CACHE)/$(host)/$(1)/$(1)-$($(1)_version)-$($(1)_build_id).tar.gz.hash +$(1)_patch_dir:=$(base_build_dir)/$(host)/$(1)/$($(1)_version)-$($(1)_build_id)/.patches-$($(1)_build_id) +$(1)_prefixbin:=$($($(1)_type)_prefix)/bin/ +$(1)_cached:=$(BASE_CACHE)/$(host)/$(1)/$(1)-$($(1)_version)-$($(1)_build_id).tar.gz +$(1)_all_sources=$($(1)_file_name) $($(1)_extra_sources) + +#stamps +$(1)_fetched=$(SOURCES_PATH)/download-stamps/.stamp_fetched-$(1)-$($(1)_file_name).hash +$(1)_extracted=$$($(1)_extract_dir)/.stamp_extracted +$(1)_preprocessed=$$($(1)_extract_dir)/.stamp_preprocessed +$(1)_cleaned=$$($(1)_extract_dir)/.stamp_cleaned +$(1)_built=$$($(1)_build_dir)/.stamp_built +$(1)_configured=$$($(1)_build_dir)/.stamp_configured +$(1)_staged=$$($(1)_staging_dir)/.stamp_staged +$(1)_postprocessed=$$($(1)_staging_prefix_dir)/.stamp_postprocessed +$(1)_download_path_fixed=$(subst :,\:,$$($(1)_download_path)) + + +#default commands +$(1)_fetch_cmds ?= $(call fetch_file,$(1),$(subst \:,:,$$($(1)_download_path_fixed)),$$($(1)_download_file),$($(1)_file_name),$($(1)_sha256_hash)) +$(1)_extract_cmds ?= mkdir -p $$($(1)_extract_dir) && echo "$$($(1)_sha256_hash) $$($(1)_source)" > $$($(1)_extract_dir)/.$$($(1)_file_name).hash && $(build_SHA256SUM) -c $$($(1)_extract_dir)/.$$($(1)_file_name).hash && tar --strip-components=1 -xf $$($(1)_source) +$(1)_preprocess_cmds ?= +$(1)_build_cmds ?= +$(1)_config_cmds ?= +$(1)_stage_cmds ?= +$(1)_set_vars ?= + + +all_sources+=$$($(1)_fetched) +endef +#$(foreach dep_target,$($(1)_all_dependencies),$(eval $(1)_dependency_targets=$($(dep_target)_cached))) + + +define int_config_attach_build_config +$(eval $(call $(1)_set_vars,$(1))) +$(1)_cflags+=$($(1)_cflags_$(release_type)) +$(1)_cflags+=$($(1)_cflags_$(host_arch)) $($(1)_cflags_$(host_arch)_$(release_type)) +$(1)_cflags+=$($(1)_cflags_$(host_os)) $($(1)_cflags_$(host_os)_$(release_type)) +$(1)_cflags+=$($(1)_cflags_$(host_arch)_$(host_os)) $($(1)_cflags_$(host_arch)_$(host_os)_$(release_type)) + +$(1)_cxxflags+=$($(1)_cxxflags_$(release_type)) +$(1)_cxxflags+=$($(1)_cxxflags_$(host_arch)) $($(1)_cxxflags_$(host_arch)_$(release_type)) +$(1)_cxxflags+=$($(1)_cxxflags_$(host_os)) $($(1)_cxxflags_$(host_os)_$(release_type)) +$(1)_cxxflags+=$($(1)_cxxflags_$(host_arch)_$(host_os)) $($(1)_cxxflags_$(host_arch)_$(host_os)_$(release_type)) + +$(1)_cppflags+=$($(1)_cppflags_$(release_type)) +$(1)_cppflags+=$($(1)_cppflags_$(host_arch)) $($(1)_cppflags_$(host_arch)_$(release_type)) +$(1)_cppflags+=$($(1)_cppflags_$(host_os)) $($(1)_cppflags_$(host_os)_$(release_type)) +$(1)_cppflags+=$($(1)_cppflags_$(host_arch)_$(host_os)) $($(1)_cppflags_$(host_arch)_$(host_os)_$(release_type)) + +$(1)_ldflags+=$($(1)_ldflags_$(release_type)) +$(1)_ldflags+=$($(1)_ldflags_$(host_arch)) $($(1)_ldflags_$(host_arch)_$(release_type)) +$(1)_ldflags+=$($(1)_ldflags_$(host_os)) $($(1)_ldflags_$(host_os)_$(release_type)) +$(1)_ldflags+=$($(1)_ldflags_$(host_arch)_$(host_os)) $($(1)_ldflags_$(host_arch)_$(host_os)_$(release_type)) + +$(1)_build_opts+=$$($(1)_build_opts_$(release_type)) +$(1)_build_opts+=$$($(1)_build_opts_$(host_arch)) $$($(1)_build_opts_$(host_arch)_$(release_type)) +$(1)_build_opts+=$$($(1)_build_opts_$(host_os)) $$($(1)_build_opts_$(host_os)_$(release_type)) +$(1)_build_opts+=$$($(1)_build_opts_$(host_arch)_$(host_os)) $$($(1)_build_opts_$(host_arch)_$(host_os)_$(release_type)) + +$(1)_config_opts+=$$($(1)_config_opts_$(release_type)) +$(1)_config_opts+=$$($(1)_config_opts_$(host_arch)) $$($(1)_config_opts_$(host_arch)_$(release_type)) +$(1)_config_opts+=$$($(1)_config_opts_$(host_os)) $$($(1)_config_opts_$(host_os)_$(release_type)) +$(1)_config_opts+=$$($(1)_config_opts_$(host_arch)_$(host_os)) $$($(1)_config_opts_$(host_arch)_$(host_os)_$(release_type)) + +$(1)_config_env+=$$($(1)_config_env_$(release_type)) +$(1)_config_env+=$($(1)_config_env_$(host_arch)) $($(1)_config_env_$(host_arch)_$(release_type)) +$(1)_config_env+=$($(1)_config_env_$(host_os)) $($(1)_config_env_$(host_os)_$(release_type)) +$(1)_config_env+=$($(1)_config_env_$(host_arch)_$(host_os)) $($(1)_config_env_$(host_arch)_$(host_os)_$(release_type)) + +$(1)_config_env+=PKG_CONFIG_LIBDIR=$($($(1)_type)_prefix)/lib/pkgconfig +$(1)_config_env+=PKG_CONFIG_PATH=$($($(1)_type)_prefix)/share/pkgconfig +$(1)_config_env+=PATH=$(build_prefix)/bin:$(PATH) +$(1)_build_env+=PATH=$(build_prefix)/bin:$(PATH) +$(1)_stage_env+=PATH=$(build_prefix)/bin:$(PATH) +$(1)_autoconf=./configure --host=$($($(1)_type)_host) --disable-dependency-tracking --prefix=$($($(1)_type)_prefix) $$($(1)_config_opts) CC="$$($(1)_cc)" CXX="$$($(1)_cxx)" + +ifneq ($($(1)_nm),) +$(1)_autoconf += NM="$$($(1)_nm)" +endif +ifneq ($($(1)_ranlib),) +$(1)_autoconf += RANLIB="$$($(1)_ranlib)" +endif +ifneq ($($(1)_ar),) +$(1)_autoconf += AR="$$($(1)_ar)" +endif +ifneq ($($(1)_cflags),) +$(1)_autoconf += CFLAGS="$$($(1)_cflags)" +endif +ifneq ($($(1)_cxxflags),) +$(1)_autoconf += CXXFLAGS="$$($(1)_cxxflags)" +endif +ifneq ($($(1)_cppflags),) +$(1)_autoconf += CPPFLAGS="$$($(1)_cppflags)" +endif +ifneq ($($(1)_ldflags),) +$(1)_autoconf += LDFLAGS="$$($(1)_ldflags)" +endif +endef + +define int_add_cmds +$($(1)_fetched): + $(AT)mkdir -p $$(@D) $(SOURCES_PATH) + $(AT)rm -f $$@ + $(AT)touch $$@ + $(AT)cd $$(@D); $(call $(1)_fetch_cmds,$(1)) + $(AT)cd $($(1)_source_dir); $(foreach source,$($(1)_all_sources),$(build_SHA256SUM) $(source) >> $$(@);) + $(AT)touch $$@ +$($(1)_extracted): | $($(1)_fetched) + $(AT)echo Extracting $(1)... + $(AT)mkdir -p $$(@D) + $(AT)cd $$(@D); $(call $(1)_extract_cmds,$(1)) + $(AT)touch $$@ +$($(1)_preprocessed): | $($(1)_dependencies) $($(1)_extracted) + $(AT)echo Preprocessing $(1)... + $(AT)mkdir -p $$(@D) $($(1)_patch_dir) + $(AT)$(foreach patch,$($(1)_patches),cd $(PATCHES_PATH)/$(1); cp $(patch) $($(1)_patch_dir) ;) + $(AT)cd $$(@D); $(call $(1)_preprocess_cmds, $(1)) + $(AT)touch $$@ +$($(1)_configured): | $($(1)_preprocessed) + $(AT)echo Configuring $(1)... + $(AT)rm -rf $(host_prefix); mkdir -p $(host_prefix)/lib; cd $(host_prefix); $(foreach package,$($(1)_all_dependencies), tar xf $($(package)_cached); ) + $(AT)mkdir -p $$(@D) + $(AT)+cd $$(@D); $($(1)_config_env) $(call $(1)_config_cmds, $(1)) + $(AT)touch $$@ +$($(1)_built): | $($(1)_configured) + $(AT)echo Building $(1)... + $(AT)mkdir -p $$(@D) + $(AT)+cd $$(@D); $($(1)_build_env) $(call $(1)_build_cmds, $(1)) + $(AT)touch $$@ +$($(1)_staged): | $($(1)_built) + $(AT)echo Staging $(1)... + $(AT)mkdir -p $($(1)_staging_dir)/$(host_prefix) + $(AT)cd $($(1)_build_dir); $($(1)_stage_env) $(call $(1)_stage_cmds, $(1)) + $(AT)rm -rf $($(1)_extract_dir) + $(AT)touch $$@ +$($(1)_postprocessed): | $($(1)_staged) + $(AT)echo Postprocessing $(1)... + $(AT)cd $($(1)_staging_prefix_dir); $(call $(1)_postprocess_cmds) + $(AT)touch $$@ +$($(1)_cached): | $($(1)_dependencies) $($(1)_postprocessed) + $(AT)echo Caching $(1)... + $(AT)cd $$($(1)_staging_dir)/$(host_prefix); find . | sort | tar --no-recursion -czf $$($(1)_staging_dir)/$$(@F) -T - + $(AT)mkdir -p $$(@D) + $(AT)rm -rf $$(@D) && mkdir -p $$(@D) + $(AT)mv $$($(1)_staging_dir)/$$(@F) $$(@) + $(AT)rm -rf $($(1)_staging_dir) +$($(1)_cached_checksum): $($(1)_cached) + $(AT)cd $$(@D); $(build_SHA256SUM) $$( $$(@) + +.PHONY: $(1) +$(1): | $($(1)_cached_checksum) +.SECONDARY: $($(1)_cached) $($(1)_postprocessed) $($(1)_staged) $($(1)_built) $($(1)_configured) $($(1)_preprocessed) $($(1)_extracted) $($(1)_fetched) + +endef + +stages = fetched extracted preprocessed configured built staged postprocessed cached cached_checksum + +define ext_add_stages +$(foreach stage,$(stages), + $(1)_$(stage): $($(1)_$(stage)) + .PHONY: $(1)_$(stage)) +endef + +# These functions create the build targets for each package. They must be +# broken down into small steps so that each part is done for all packages +# before moving on to the next step. Otherwise, a package's info +# (build-id for example) would only be available to another package if it +# happened to be computed already. + +#set the type for host/build packages. +$(foreach native_package,$(native_packages),$(eval $(native_package)_type=build)) +$(foreach package,$(packages),$(eval $(package)_type=$(host_arch)_$(host_os))) + +#set overridable defaults +$(foreach package,$(all_packages),$(eval $(call int_vars,$(package)))) + +#include package files +$(foreach package,$(all_packages),$(eval include packages/$(package).mk)) + +#compute a hash of all files that comprise this package's build recipe +$(foreach package,$(all_packages),$(eval $(call int_get_build_recipe_hash,$(package)))) + +#generate a unique id for this package, incorporating its dependencies as well +$(foreach package,$(all_packages),$(eval $(call int_get_build_id,$(package)))) + +#compute final vars after reading package vars +$(foreach package,$(all_packages),$(eval $(call int_config_attach_build_config,$(package)))) + +#create build targets +$(foreach package,$(all_packages),$(eval $(call int_add_cmds,$(package)))) + +#special exception: if a toolchain package exists, all non-native packages depend on it +$(foreach package,$(packages),$(eval $($(package)_unpacked): |$($($(host_arch)_$(host_os)_native_toolchain)_cached) )) diff --git a/contrib/depends/hosts/darwin.mk b/contrib/depends/hosts/darwin.mk new file mode 100644 index 00000000000..6f8f2ffe225 --- /dev/null +++ b/contrib/depends/hosts/darwin.mk @@ -0,0 +1,17 @@ +OSX_MIN_VERSION=10.8 +OSX_SDK_VERSION=10.11 +OSX_SDK=$(SDK_PATH)/MacOSX$(OSX_SDK_VERSION).sdk +LD64_VERSION=253.9 +darwin_CC=clang -target $(host) -mmacosx-version-min=$(OSX_MIN_VERSION) --sysroot $(OSX_SDK) -mlinker-version=$(LD64_VERSION) -B $(host_prefix)/native/bin +darwin_CXX=clang++ -target $(host) -mmacosx-version-min=$(OSX_MIN_VERSION) --sysroot $(OSX_SDK) -mlinker-version=$(LD64_VERSION) -stdlib=libc++ -B $(host_prefix)/native/bin + +darwin_CFLAGS=-pipe +darwin_CXXFLAGS=$(darwin_CFLAGS) + +darwin_release_CFLAGS=-O1 +darwin_release_CXXFLAGS=$(darwin_release_CFLAGS) + +darwin_debug_CFLAGS=-O1 +darwin_debug_CXXFLAGS=$(darwin_debug_CFLAGS) + +darwin_native_toolchain=native_cctools diff --git a/contrib/depends/hosts/default.mk b/contrib/depends/hosts/default.mk new file mode 100644 index 00000000000..6f60d6b3fd0 --- /dev/null +++ b/contrib/depends/hosts/default.mk @@ -0,0 +1,26 @@ +default_host_CC = $(host_toolchain)gcc +default_host_CXX = $(host_toolchain)g++ +default_host_AR = $(host_toolchain)ar +default_host_RANLIB = $(host_toolchain)ranlib +default_host_STRIP = $(host_toolchain)strip +default_host_LIBTOOL = $(host_toolchain)libtool +default_host_INSTALL_NAME_TOOL = $(host_toolchain)install_name_tool +default_host_OTOOL = $(host_toolchain)otool +default_host_NM = $(host_toolchain)nm + +define add_host_tool_func +$(host_os)_$1?=$$(default_host_$1) +$(host_arch)_$(host_os)_$1?=$$($(host_os)_$1) +$(host_arch)_$(host_os)_$(release_type)_$1?=$$($(host_os)_$1) +host_$1=$$($(host_arch)_$(host_os)_$1) +endef + +define add_host_flags_func +$(host_arch)_$(host_os)_$1 += $($(host_os)_$1) +$(host_arch)_$(host_os)_$(release_type)_$1 += $($(host_os)_$(release_type)_$1) +host_$1 = $$($(host_arch)_$(host_os)_$1) +host_$(release_type)_$1 = $$($(host_arch)_$(host_os)_$(release_type)_$1) +endef + +$(foreach tool,CC CXX AR RANLIB STRIP NM LIBTOOL OTOOL INSTALL_NAME_TOOL,$(eval $(call add_host_tool_func,$(tool)))) +$(foreach flags,CFLAGS CXXFLAGS CPPFLAGS LDFLAGS, $(eval $(call add_host_flags_func,$(flags)))) diff --git a/contrib/depends/hosts/linux.mk b/contrib/depends/hosts/linux.mk new file mode 100644 index 00000000000..b13a0f1ad71 --- /dev/null +++ b/contrib/depends/hosts/linux.mk @@ -0,0 +1,31 @@ +linux_CFLAGS=-pipe +linux_CXXFLAGS=$(linux_CFLAGS) + +linux_release_CFLAGS=-O2 +linux_release_CXXFLAGS=$(linux_release_CFLAGS) + +linux_debug_CFLAGS=-O1 +linux_debug_CXXFLAGS=$(linux_debug_CFLAGS) + +linux_debug_CPPFLAGS=-D_GLIBCXX_DEBUG -D_GLIBCXX_DEBUG_PEDANTIC + +ifeq (86,$(findstring 86,$(build_arch))) +i686_linux_CC=gcc -m32 +i686_linux_CXX=g++ -m32 +i686_linux_AR=ar +i686_linux_RANLIB=ranlib +i686_linux_NM=nm +i686_linux_STRIP=strip + +x86_64_linux_CC=gcc -m64 +x86_64_linux_CXX=g++ -m64 +x86_64_linux_AR=ar +x86_64_linux_RANLIB=ranlib +x86_64_linux_NM=nm +x86_64_linux_STRIP=strip +else +i686_linux_CC=$(default_host_CC) -m32 +i686_linux_CXX=$(default_host_CXX) -m32 +x86_64_linux_CC=$(default_host_CC) -m64 +x86_64_linux_CXX=$(default_host_CXX) -m64 +endif diff --git a/contrib/depends/hosts/mingw32.mk b/contrib/depends/hosts/mingw32.mk new file mode 100644 index 00000000000..dbfb62fdcf9 --- /dev/null +++ b/contrib/depends/hosts/mingw32.mk @@ -0,0 +1,10 @@ +mingw32_CFLAGS=-pipe +mingw32_CXXFLAGS=$(mingw32_CFLAGS) + +mingw32_release_CFLAGS=-O2 +mingw32_release_CXXFLAGS=$(mingw32_release_CFLAGS) + +mingw32_debug_CFLAGS=-O1 +mingw32_debug_CXXFLAGS=$(mingw32_debug_CFLAGS) + +mingw32_debug_CPPFLAGS=-D_GLIBCXX_DEBUG -D_GLIBCXX_DEBUG_PEDANTIC diff --git a/contrib/depends/packages.md b/contrib/depends/packages.md new file mode 100644 index 00000000000..ae5b47327db --- /dev/null +++ b/contrib/depends/packages.md @@ -0,0 +1,165 @@ +Each recipe consists of 3 main parts: defining identifiers, setting build +variables, and defining build commands. + +The package "mylib" will be used here as an example + +General tips: +- mylib_foo is written as $(package)_foo in order to make recipes more similar. + +## Identifiers +Each package is required to define at least these variables: + +``` +$(package)_version: +Version of the upstream library or program. If there is no version, a +placeholder such as 1.0 can be used. + +$(package)_download_path: +Location of the upstream source, without the file-name. Usually http or +ftp. + +$(package)_file_name: +The upstream source filename available at the download path. + +$(package)_sha256_hash: +The sha256 hash of the upstream file +``` + +These variables are optional: + +``` +$(package)_build_subdir: +cd to this dir before running configure/build/stage commands. + +$(package)_download_file: +The file-name of the upstream source if it differs from how it should be +stored locally. This can be used to avoid storing file-names with strange +characters. + +$(package)_dependencies: +Names of any other packages that this one depends on. + +$(package)_patches: +Filenames of any patches needed to build the package + +$(package)_extra_sources: +Any extra files that will be fetched via $(package)_fetch_cmds. These are +specified so that they can be fetched and verified via 'make download'. +``` + + +## Build Variables: +After defining the main identifiers, build variables may be added or customized +before running the build commands. They should be added to a function called +$(package)_set_vars. For example: + +``` +define $(package)_set_vars +... +endef +``` + +Most variables can be prefixed with the host, architecture, or both, to make +the modifications specific to that case. For example: + +``` +Universal: $(package)_cc=gcc +Linux only: $(package)_linux_cc=gcc +x86_64 only: $(package)_x86_64_cc = gcc +x86_64 linux only: $(package)_x86_64_linux_cc = gcc +``` + +These variables may be set to override or append their default values. + +``` +$(package)_cc +$(package)_cxx +$(package)_objc +$(package)_objcxx +$(package)_ar +$(package)_ranlib +$(package)_libtool +$(package)_nm +$(package)_cflags +$(package)_cxxflags +$(package)_ldflags +$(package)_cppflags +$(package)_config_env +$(package)_build_env +$(package)_stage_env +$(package)_build_opts +$(package)_config_opts +``` + +The `*_env` variables are used to add environment variables to the respective +commands. + +Many variables respect a debug/release suffix as well, in order to use them for +only the appropriate build config. For example: + +``` +$(package)_cflags_release = -O3 +$(package)_cflags_i686_debug = -g +$(package)_config_opts_release = --disable-debug +``` + +These will be used in addition to the options that do not specify +debug/release. All builds are considered to be release unless DEBUG=1 is set by +the user. Other variables may be defined as needed. + +## Build commands: + +For each build, a unique build dir and staging dir are created. For example, +`work/build/mylib/1.0-1adac830f6e` and `work/staging/mylib/1.0-1adac830f6e`. + +The following build commands are available for each recipe: + +``` +$(package)_fetch_cmds: +Runs from: build dir +Fetch the source file. If undefined, it will be fetched and verified +against its hash. + +$(package)_extract_cmds: +Runs from: build dir +Verify the source file against its hash and extract it. If undefined, the +source is assumed to be a tarball. + +$(package)_preprocess_cmds: +Runs from: build dir/$(package)_build_subdir +Preprocess the source as necessary. If undefined, does nothing. + +$(package)_config_cmds: +Runs from: build dir/$(package)_build_subdir +Configure the source. If undefined, does nothing. + +$(package)_build_cmds: +Runs from: build dir/$(package)_build_subdir +Build the source. If undefined, does nothing. + +$(package)_stage_cmds: +Runs from: build dir/$(package)_build_subdir +Stage the build results. If undefined, does nothing. +``` + +The following variables are available for each recipe: + +``` +$(1)_staging_dir: package's destination sysroot path +$(1)_staging_prefix_dir: prefix path inside of the package's staging dir +$(1)_extract_dir: path to the package's extracted sources +$(1)_build_dir: path where configure/build/stage commands will be run +$(1)_patch_dir: path where the package's patches (if any) are found +``` + +Notes on build commands: + +For packages built with autotools, `$($(package)_autoconf)` can be used in the +configure step to (usually) correctly configure automatically. Any +`$($(package)_config_opts`) will be appended. + +Most autotools projects can be properly staged using: + +```bash +$(MAKE) DESTDIR=$($(package)_staging_dir) install +``` diff --git a/contrib/depends/packages/boost.mk b/contrib/depends/packages/boost.mk new file mode 100644 index 00000000000..e60a1c67773 --- /dev/null +++ b/contrib/depends/packages/boost.mk @@ -0,0 +1,41 @@ +package=boost +$(package)_version=1_64_0 +$(package)_download_path=https://dl.bintray.com/boostorg/release/1.64.0/source/ +$(package)_file_name=$(package)_$($(package)_version).tar.bz2 +$(package)_sha256_hash=7bcc5caace97baa948931d712ea5f37038dbb1c5d89b43ad4def4ed7cb683332 + +define $(package)_set_vars +$(package)_config_opts_release=variant=release +$(package)_config_opts_debug=variant=debug +$(package)_config_opts=--layout=tagged --build-type=complete --user-config=user-config.jam +$(package)_config_opts+=threading=multi link=static -sNO_BZIP2=1 -sNO_ZLIB=1 +$(package)_config_opts_linux=threadapi=pthread runtime-link=shared +$(package)_config_opts_darwin=--toolset=darwin-4.2.1 runtime-link=shared +$(package)_config_opts_mingw32=binary-format=pe target-os=windows threadapi=win32 runtime-link=static +$(package)_config_opts_x86_64_mingw32=address-model=64 +$(package)_config_opts_i686_mingw32=address-model=32 +$(package)_config_opts_i686_linux=address-model=32 architecture=x86 +$(package)_toolset_$(host_os)=gcc +$(package)_archiver_$(host_os)=$($(package)_ar) +$(package)_toolset_darwin=darwin +$(package)_archiver_darwin=$($(package)_libtool) +$(package)_config_libraries=chrono,filesystem,program_options,system,thread,test,date_time,regex,serialization,locale +$(package)_cxxflags=-std=c++11 +$(package)_cxxflags_linux=-fPIC +endef + +define $(package)_preprocess_cmds + echo "using $(boost_toolset_$(host_os)) : : $($(package)_cxx) : \"$($(package)_cxxflags) $($(package)_cppflags)\" \"$($(package)_ldflags)\" \"$(boost_archiver_$(host_os))\" \"$(host_STR IP)\" \"$(host_RANLIB)\" \"$(host_WINDRES)\" : ;" > user-config.jam +endef + +define $(package)_config_cmds + ./bootstrap.sh --without-icu --with-libraries=$(boost_config_libraries) +endef + +define $(package)_build_cmds + ./b2 -d2 -j2 -d1 --prefix=$($(package)_staging_prefix_dir) $($(package)_config_opts) stage +endef + +define $(package)_stage_cmds + ./b2 -d0 -j4 --prefix=$($(package)_staging_prefix_dir) $($(package)_config_opts) install +endef diff --git a/contrib/depends/packages/cppzmq.mk b/contrib/depends/packages/cppzmq.mk new file mode 100644 index 00000000000..6c0faf08432 --- /dev/null +++ b/contrib/depends/packages/cppzmq.mk @@ -0,0 +1,15 @@ +package=cppzmq +$(package)_version=4.2.3 +$(package)_download_path=https://github.com/zeromq/cppzmq/archive/ +$(package)_file_name=v$($(package)_version).tar.gz +$(package)_sha256_hash=3e6b57bf49115f4ae893b1ff7848ead7267013087dc7be1ab27636a97144d373 +$(package)_dependencies=zeromq + +define $(package)_stage_cmds + mkdir $($(package)_staging_prefix_dir)/include &&\ + cp zmq.hpp $($(package)_staging_prefix_dir)/include +endef + +define $(package)_postprocess_cmds + rm -rf bin share +endef diff --git a/contrib/depends/packages/eudev.mk b/contrib/depends/packages/eudev.mk new file mode 100644 index 00000000000..a7795b7773e --- /dev/null +++ b/contrib/depends/packages/eudev.mk @@ -0,0 +1,29 @@ +package=eudev +$(package)_version=v3.2.6 +$(package)_download_path=https://github.com/gentoo/eudev/archive/ +$(package)_file_name=$($(package)_version).tar.gz +$(package)_sha256_hash=a96ecb8637667897b8bd4dee4c22c7c5f08b327be45186e912ce6bc768385852 + +define $(package)_set_vars + $(package)_config_opts=--disable-gudev --disable-introspection --disable-hwdb --disable-manpages --disable-shared +endef + +define $(package)_config_cmds + $($(package)_autoconf) +endef + +define $(package)_build_cmd + $(MAKE) +endef + +define $(package)_preprocess_cmds + cd $($(package)_build_subdir); autoreconf -f -i +endef + +define $(package)_stage_cmds + $(MAKE) DESTDIR=$($(package)_staging_dir) install +endef + +define $(package)_postprocess_cmds + rm lib/*.la +endef diff --git a/contrib/depends/packages/expat.mk b/contrib/depends/packages/expat.mk new file mode 100644 index 00000000000..ef81636a2fa --- /dev/null +++ b/contrib/depends/packages/expat.mk @@ -0,0 +1,28 @@ +package=expat +$(package)_version=2.2.4 +$(package)_download_path=https://downloads.sourceforge.net/project/expat/expat/$($(package)_version) +$(package)_file_name=$(package)-$($(package)_version).tar.bz2 +$(package)_sha256_hash=03ad85db965f8ab2d27328abcf0bc5571af6ec0a414874b2066ee3fdd372019e + +define $(package)_set_vars +$(package)_config_opts=--enable-static +$(package)_config_opts=--disable-shared +$(package)_config_opts+=--prefix=$(host_prefix) +endef + +define $(package)_config_cmds + $($(package)_autoconf) $($(package)_config_opts) +endef + +define $(package)_build_cmds + $(MAKE) +endef + +define $(package)_stage_cmds + $(MAKE) DESTDIR=$($(package)_staging_dir) install +endef + +define $(package)_postprocess_cmds + rm lib/*.la +endef + diff --git a/contrib/depends/packages/graphviz.mk b/contrib/depends/packages/graphviz.mk new file mode 100644 index 00000000000..1c4bc1b7160 --- /dev/null +++ b/contrib/depends/packages/graphviz.mk @@ -0,0 +1,30 @@ +package=graphviz +$(package)_version=2.40.1 +$(package)_download_path=www.graphviz.org/pub/graphviz/stable/SOURCES/ +$(package)_file_name=$(package)-$($(package)_version).tar.gz +$(package)_sha256_hash=ca5218fade0204d59947126c38439f432853543b0818d9d728c589dfe7f3a421 + +define $(package)_preprocess_cmds + ./autogen.sh +endef + +define $(package)_set_vars + $(package)_config_opts=--disable-shared --enable-multibye --without-purify --without-curses + $(package)_config_opts_release=--disable-debug-mode + $(package)_config_opts_linux=--with-pic +endef + +define $(package)_config_cmds + $($(package)_autoconf) +endef + +define $(package)_build_cmds + $(MAKE) +endef + +define $(package)_stage_cmds + $(MAKE) DESTDIR=$($(package)_staging_dir) install +endef + +define $(package)_postprocess_cmds +endef diff --git a/contrib/depends/packages/gtest.mk b/contrib/depends/packages/gtest.mk new file mode 100644 index 00000000000..5df07a32eaf --- /dev/null +++ b/contrib/depends/packages/gtest.mk @@ -0,0 +1,38 @@ +package=gtest +$(package)_version=1.8.1 +$(package)_download_path=https://github.com/google/googletest/archive/ +$(package)_file_name=release-$($(package)_version).tar.gz +$(package)_sha256_hash=9bf1fe5182a604b4135edc1a425ae356c9ad15e9b23f9f12a02e80184c3a249c +$(package)_cxxflags=-std=c++11 +$(package)_cxxflags_linux=-fPIC + +define $(package)_config_cmds + cd googletest && \ + CC="$(host_prefix)/native/bin/$($(package)_cc)" \ + CXX="$(host_prefix)/native/bin/$($(package)_cxx)" \ + AR="$(host_prefix)/native/bin/$($(package)_ar)" \ + RANLIB="$(host_prefix)/native/bin/$($(package)_ranlib)" \ + LIBTOOL="$(host_prefix)/native/bin/$($(package)_libtool)" \ + CXXFLAGS="$($(package)_cxxflags)" \ + CCFLAGS="$($(package)_ccflags)" \ + CPPFLAGS="$($(package)_cppflags)" \ + CFLAGS="$($(package)_cflags) $($(package)_cppflags)" \ + LDLAGS="$($(package)_ldflags)" \ + cmake -DCMAKE_INSTALL_PREFIX=$(build_prefix) \ + -DTOOLCHAIN_PREFIX=$(host_toolchain) \ + -DCMAKE_AR="$(host_prefix)/native/bin/$($(package)_ar)" \ + -DCMAKE_RANLIB="$(host_prefix)/native/bin/$($(package)_ranlib)" \ + -DCMAKE_CXX_FLAGS_DEBUG=ON +endef +# -DCMAKE_TOOLCHAIN_FILE=$(HOST)/share/toolchain.cmake + +define $(package)_build_cmds + cd googletest && CC="$(host_prefix)/native/bin/$($(package)_cc)" $(MAKE) +endef + +define $(package)_stage_cmds + mkdir $($(package)_staging_prefix_dir)/lib $($(package)_staging_prefix_dir)/include &&\ + cp googletest/libgtest.a $($(package)_staging_prefix_dir)/lib/ &&\ + cp googletest/libgtest_main.a $($(package)_staging_prefix_dir)/lib/ &&\ + cp -a googletest/include/* $($(package)_staging_prefix_dir)/include/ +endef diff --git a/contrib/depends/packages/hidapi.mk b/contrib/depends/packages/hidapi.mk new file mode 100644 index 00000000000..a27df04fac9 --- /dev/null +++ b/contrib/depends/packages/hidapi.mk @@ -0,0 +1,35 @@ +package=hidapi +$(package)_version=0.9.0 +$(package)_download_path=https://github.com/libusb/hidapi/archive +$(package)_file_name=$(package)-$($(package)_version).tar.gz +$(package)_sha256_hash=630ee1834bdd5c5761ab079fd04f463a89585df8fcae51a7bfe4229b1e02a652 +$(package)_linux_dependencies=libusb eudev + +define $(package)_set_vars +$(package)_config_opts=--enable-static --disable-shared +$(package)_config_opts+=--prefix=$(host_prefix) +$(package)_config_opts_darwin+=RANLIB="$(host_prefix)/native/bin/x86_64-apple-darwin11-ranlib" AR="$(host_prefix)/native/bin/x86_64-apple-darwin11-ar" CC="$(host_prefix)/native/bin/$($(package)_cc)" +$(package)_config_opts_linux+=libudev_LIBS="-L$(host_prefix)/lib -ludev" +$(package)_config_opts_linux+=libudev_CFLAGS=-I$(host_prefix)/include +$(package)_config_opts_linux+=libusb_LIBS="-L$(host_prefix)/lib -lusb-1.0" +$(package)_config_opts_linux+=libusb_CFLAGS=-I$(host_prefix)/include/libusb-1.0 +$(package)_config_opts_linux+=--with-pic +endef + +define $(package)_config_cmds + ./bootstrap &&\ + $($(package)_autoconf) $($(package)_config_opts) +endef + +define $(package)_build_cmds + $(MAKE) +endef + +define $(package)_stage_cmds + $(MAKE) DESTDIR=$($(package)_staging_dir) install +endef + +define $(package)_postprocess_cmds + rm lib/*.la +endef + diff --git a/contrib/depends/packages/icu4c.mk b/contrib/depends/packages/icu4c.mk new file mode 100644 index 00000000000..dc089bbbea1 --- /dev/null +++ b/contrib/depends/packages/icu4c.mk @@ -0,0 +1,27 @@ +package=icu4c +$(package)_version=55.1 +$(package)_download_path=https://github.com/TheCharlatan/icu4c/archive +$(package)_file_name=55.1.tar.gz +$(package)_sha256_hash=1f912c54035533fb4268809701d65c7468d00e292efbc31e6444908450cc46ef +$(package)_patches=icu-001-dont-build-static-dynamic-twice.patch + +define $(package)_set_vars + $(package)_build_opts=CFLAGS="$($(package)_cflags) $($(package)_cppflags) -DU_USING_ICU_NAMESPACE=0 -DU_STATIC_IMPLEMENTATION -DU_COMBINED_IMPLEMENTATION -fPIC -DENABLE_STATIC=YES -DPGKDATA_MODE=static" +endef + +define $(package)_config_cmds + patch -p1 < $($(package)_patch_dir)/icu-001-dont-build-static-dynamic-twice.patch &&\ + mkdir builda &&\ + mkdir buildb &&\ + cd builda &&\ + sh ../source/runConfigureICU Linux &&\ + make &&\ + cd ../buildb &&\ + sh ../source/runConfigureICU MinGW --enable-static=yes --disable-shared --disable-layout --disable-layoutex --disable-tests --disable-samples --prefix=$(host_prefix) --with-cross-build=`pwd`/../builda &&\ + $(MAKE) $($(package)_build_opts) +endef + +define $(package)_stage_cmds + cd buildb &&\ + $(MAKE) $($(package)_build_opts) DESTDIR=$($(package)_staging_dir) install lib/* +endef diff --git a/contrib/depends/packages/ldns.mk b/contrib/depends/packages/ldns.mk new file mode 100644 index 00000000000..ea49021705e --- /dev/null +++ b/contrib/depends/packages/ldns.mk @@ -0,0 +1,30 @@ +package=ldns +$(package)_version=1.6.17 +$(package)_download_path=https://www.nlnetlabs.nl/downloads/ldns/ +$(package)_file_name=$(package)-$($(package)_version).tar.gz +$(package)_sha256_hash=8b88e059452118e8949a2752a55ce59bc71fa5bc414103e17f5b6b06f9bcc8cd +$(package)_dependencies=openssl + +define $(package)_set_vars + $(package)_config_opts=--disable-shared --enable-static --with-drill + $(package)_config_opts+=--with-ssl=$(host_prefix) + $(package)_config_opts_release=--disable-debug-mode + $(package)_config_opts_linux=--with-pic +endef + +define $(package)_config_cmds + $($(package)_autoconf) +endef + +define $(package)_build_cmds + $(MAKE) +endef + +define $(package)_stage_cmds + $(MAKE) DESTDIR=$($(package)_staging_dir) install-h install-lib +endef + +define $(package)_postprocess_cmds + rm lib/*.la +endef + diff --git a/contrib/depends/packages/libICE.mk b/contrib/depends/packages/libICE.mk new file mode 100644 index 00000000000..a897d9aed77 --- /dev/null +++ b/contrib/depends/packages/libICE.mk @@ -0,0 +1,23 @@ +package=libICE +$(package)_version=1.0.9 +$(package)_download_path=https://xorg.freedesktop.org/releases/individual/lib/ +$(package)_file_name=$(package)-$($(package)_version).tar.bz2 +$(package)_sha256_hash=8f7032f2c1c64352b5423f6b48a8ebdc339cc63064af34d66a6c9aa79759e202 +$(package)_dependencies=xtrans xproto + +define $(package)_set_vars + $(package)_config_opts=--disable-static --disable-docs --disable-specs --without-xsltproc + $(package)_config_opts_linux=--with-pic +endef + +define $(package)_config_cmds + $($(package)_autoconf) +endef + +define $(package)_build_cmds + $(MAKE) +endef + +define $(package)_stage_cmds + $(MAKE) DESTDIR=$($(package)_staging_dir) install +endef diff --git a/contrib/depends/packages/libSM.mk b/contrib/depends/packages/libSM.mk new file mode 100644 index 00000000000..83fcd4cdb56 --- /dev/null +++ b/contrib/depends/packages/libSM.mk @@ -0,0 +1,23 @@ +package=libSM +$(package)_version=1.2.2 +$(package)_download_path=https://xorg.freedesktop.org/releases/individual/lib/ +$(package)_file_name=$(package)-$($(package)_version).tar.bz2 +$(package)_sha256_hash=0baca8c9f5d934450a70896c4ad38d06475521255ca63b717a6510fdb6e287bd +$(package)_dependencies=xtrans xproto libICE + +define $(package)_set_vars + $(package)_config_opts=--without-libuuid --without-xsltproc --disable-docs --disable-static + $(package)_config_opts_linux=--with-pic +endef + +define $(package)_config_cmds + $($(package)_autoconf) +endef + +define $(package)_build_cmds + $(MAKE) +endef + +define $(package)_stage_cmds + $(MAKE) DESTDIR=$($(package)_staging_dir) install +endef diff --git a/contrib/depends/packages/libiconv.mk b/contrib/depends/packages/libiconv.mk new file mode 100644 index 00000000000..d4995c1b7f6 --- /dev/null +++ b/contrib/depends/packages/libiconv.mk @@ -0,0 +1,34 @@ +package=libiconv +$(package)_version=1.15 +$(package)_download_path=https://ftp.gnu.org/gnu/libiconv +$(package)_file_name=libiconv-$($(package)_version).tar.gz +$(package)_sha256_hash=ccf536620a45458d26ba83887a983b96827001e92a13847b45e4925cc8913178 +$(package)_patches=fix-whitespace.patch + +define $(package)_set_vars + $(package)_config_opts=--disable-nls + $(package)_config_opts=--enable-static + $(package)_config_opts=--disable-shared + $(package)_config_opts_linux=--with-pic +endef + +define $(package)_preprocess_cmds + cp -f $(BASEDIR)/config.guess $(BASEDIR)/config.sub build-aux/ &&\ + patch -p1 < $($(package)_patch_dir)/fix-whitespace.patch +endef + +define $(package)_config_cmds + $($(package)_autoconf) +endef + +define $(package)_build_cmds + $(MAKE) +endef + +define $(package)_stage_cmds + $(MAKE) DESTDIR=$($(package)_staging_dir) install +endef + +define $(package)_postprocess_cmds + rm lib/*.la +endef diff --git a/contrib/depends/packages/libusb.mk b/contrib/depends/packages/libusb.mk new file mode 100644 index 00000000000..d865d2a17dc --- /dev/null +++ b/contrib/depends/packages/libusb.mk @@ -0,0 +1,39 @@ +package=libusb +$(package)_version=1.0.22 +$(package)_download_path=https://sourceforge.net/projects/libusb/files/libusb-1.0/libusb-$($(package)_version)/ +$(package)_file_name=$(package)-$($(package)_version).tar.bz2 +$(package)_sha256_hash=75aeb9d59a4fdb800d329a545c2e6799f732362193b465ea198f2aa275518157 + +define $(package)_preprocess_cmds + autoreconf -i +endef + +define $(package)_set_vars + $(package)_config_opts=--disable-shared + $(package)_config_opts_linux=--with-pic --disable-udev + $(package)_config_opts_mingw32=--disable-udev + $(package)_config_opts_darwin=--disable-udev +endef + +ifneq ($(host_os),darwin) + define $(package)_config_cmds + cp -f $(BASEDIR)/config.guess config.guess &&\ + cp -f $(BASEDIR)/config.sub config.sub &&\ + $($(package)_autoconf) + endef +else + define $(package)_config_cmds + $($(package)_autoconf) + endef +endif + +define $(package)_build_cmd + $(MAKE) +endef + +define $(package)_stage_cmds + $(MAKE) DESTDIR=$($(package)_staging_dir) install +endef + +define $(package)_postprocess_cmds cp -f lib/libusb-1.0.a lib/libusb.a +endef diff --git a/contrib/depends/packages/native_biplist.mk b/contrib/depends/packages/native_biplist.mk new file mode 100644 index 00000000000..3c6e8900f66 --- /dev/null +++ b/contrib/depends/packages/native_biplist.mk @@ -0,0 +1,20 @@ +package=native_biplist +$(package)_version=0.9 +$(package)_download_path=https://pypi.python.org/packages/source/b/biplist +$(package)_file_name=biplist-$($(package)_version).tar.gz +$(package)_sha256_hash=b57cadfd26e4754efdf89e9e37de87885f9b5c847b2615688ca04adfaf6ca604 +$(package)_install_libdir=$(build_prefix)/lib/python/dist-packages +$(package)_patches=sorted_list.patch + +define $(package)_preprocess_cmds + patch -p1 < $($(package)_patch_dir)/sorted_list.patch +endef + +define $(package)_build_cmds + python setup.py build +endef + +define $(package)_stage_cmds + mkdir -p $($(package)_install_libdir) && \ + python setup.py install --root=$($(package)_staging_dir) --prefix=$(build_prefix) --install-lib=$($(package)_install_libdir) +endef diff --git a/contrib/depends/packages/native_ccache.mk b/contrib/depends/packages/native_ccache.mk new file mode 100644 index 00000000000..966804ce8bb --- /dev/null +++ b/contrib/depends/packages/native_ccache.mk @@ -0,0 +1,25 @@ +package=native_ccache +$(package)_version=3.3.4 +$(package)_download_path=https://samba.org/ftp/ccache +$(package)_file_name=ccache-$($(package)_version).tar.bz2 +$(package)_sha256_hash=fa9d7f38367431bc86b19ad107d709ca7ecf1574fdacca01698bdf0a47cd8567 + +define $(package)_set_vars +$(package)_config_opts= +endef + +define $(package)_config_cmds + $($(package)_autoconf) +endef + +define $(package)_build_cmds + $(MAKE) +endef + +define $(package)_stage_cmds + $(MAKE) DESTDIR=$($(package)_staging_dir) install +endef + +define $(package)_postprocess_cmds + rm -rf lib include +endef diff --git a/contrib/depends/packages/native_cctools.mk b/contrib/depends/packages/native_cctools.mk new file mode 100644 index 00000000000..bcfe1af6b37 --- /dev/null +++ b/contrib/depends/packages/native_cctools.mk @@ -0,0 +1,66 @@ +package=native_cctools +$(package)_version=807d6fd1be5d2224872e381870c0a75387fe05e6 +$(package)_download_path=https://github.com/theuni/cctools-port/archive +$(package)_file_name=$($(package)_version).tar.gz +$(package)_sha256_hash=a09c9ba4684670a0375e42d9d67e7f12c1f62581a27f28f7c825d6d7032ccc6a +$(package)_build_subdir=cctools +$(package)_clang_version=3.7.1 +$(package)_clang_download_path=http://llvm.org/releases/$($(package)_clang_version) +$(package)_clang_download_file=clang+llvm-$($(package)_clang_version)-x86_64-linux-gnu-ubuntu-14.04.tar.xz +$(package)_clang_file_name=clang-llvm-$($(package)_clang_version)-x86_64-linux-gnu-ubuntu-14.04.tar.xz +$(package)_clang_sha256_hash=99b28a6b48e793705228a390471991386daa33a9717cd9ca007fcdde69608fd9 +$(package)_extra_sources=$($(package)_clang_file_name) + +define $(package)_fetch_cmds +$(call fetch_file,$(package),$($(package)_download_path),$($(package)_download_file),$($(package)_file_name),$($(package)_sha256_hash)) && \ +$(call fetch_file,$(package),$($(package)_clang_download_path),$($(package)_clang_download_file),$($(package)_clang_file_name),$($(package)_clang_sha256_hash)) +endef + +define $(package)_extract_cmds + mkdir -p $($(package)_extract_dir) && \ + echo "$($(package)_sha256_hash) $($(package)_source)" > $($(package)_extract_dir)/.$($(package)_file_name).hash && \ + echo "$($(package)_clang_sha256_hash) $($(package)_source_dir)/$($(package)_clang_file_name)" >> $($(package)_extract_dir)/.$($(package)_file_name).hash && \ + $(build_SHA256SUM) -c $($(package)_extract_dir)/.$($(package)_file_name).hash && \ + mkdir -p toolchain/bin toolchain/lib/clang/3.5/include && \ + tar --strip-components=1 -C toolchain -xf $($(package)_source_dir)/$($(package)_clang_file_name) && \ + rm -f toolchain/lib/libc++abi.so* && \ + echo "#!/bin/sh" > toolchain/bin/$(host)-dsymutil && \ + echo "exit 0" >> toolchain/bin/$(host)-dsymutil && \ + chmod +x toolchain/bin/$(host)-dsymutil && \ + tar --strip-components=1 -xf $($(package)_source) +endef + +define $(package)_set_vars +$(package)_config_opts=--target=$(host) --disable-lto-support +$(package)_ldflags+=-Wl,-rpath=\\$$$$$$$$\$$$$$$$$ORIGIN/../lib +$(package)_cc=$($(package)_extract_dir)/toolchain/bin/clang +$(package)_cxx=$($(package)_extract_dir)/toolchain/bin/clang++ +endef + +define $(package)_preprocess_cmds + cd $($(package)_build_subdir); ./autogen.sh && \ + sed -i.old "/define HAVE_PTHREADS/d" ld64/src/ld/InputFiles.h +endef + +define $(package)_config_cmds + $($(package)_autoconf) +endef + +define $(package)_build_cmds + $(MAKE) +endef + +define $(package)_stage_cmds + $(MAKE) DESTDIR=$($(package)_staging_dir) install && \ + cp $($(package)_extract_dir)/cctools/misc/install_name_tool $($(package)_staging_prefix_dir)/bin/ &&\ + cd $($(package)_extract_dir)/toolchain && \ + mkdir -p $($(package)_staging_prefix_dir)/lib/clang/$($(package)_clang_version)/include && \ + mkdir -p $($(package)_staging_prefix_dir)/bin $($(package)_staging_prefix_dir)/include && \ + cp bin/clang $($(package)_staging_prefix_dir)/bin/ &&\ + cp -P bin/clang++ $($(package)_staging_prefix_dir)/bin/ &&\ + cp lib/libLTO.so $($(package)_staging_prefix_dir)/lib/ && \ + cp -rf lib/clang/$($(package)_clang_version)/include/* $($(package)_staging_prefix_dir)/lib/clang/$($(package)_clang_version)/include/ && \ + cp bin/llvm-dsymutil $($(package)_staging_prefix_dir)/bin/$(host)-dsymutil && \ + if `test -d include/c++/`; then cp -rf include/c++/ $($(package)_staging_prefix_dir)/include/; fi && \ + if `test -d lib/c++/`; then cp -rf lib/c++/ $($(package)_staging_prefix_dir)/lib/; fi +endef diff --git a/contrib/depends/packages/native_cdrkit.mk b/contrib/depends/packages/native_cdrkit.mk new file mode 100644 index 00000000000..8243458ec85 --- /dev/null +++ b/contrib/depends/packages/native_cdrkit.mk @@ -0,0 +1,26 @@ +package=native_cdrkit +$(package)_version=1.1.11 +$(package)_download_path=https://distro.ibiblio.org/fatdog/source/600/c +$(package)_file_name=cdrkit-$($(package)_version).tar.bz2 +$(package)_sha256_hash=b50d64c214a65b1a79afe3a964c691931a4233e2ba605d793eb85d0ac3652564 +$(package)_patches=cdrkit-deterministic.patch + +define $(package)_preprocess_cmds + patch -p1 < $($(package)_patch_dir)/cdrkit-deterministic.patch +endef + +define $(package)_config_cmds + cmake -DCMAKE_INSTALL_PREFIX=$(build_prefix) +endef + +define $(package)_build_cmds + $(MAKE) genisoimage +endef + +define $(package)_stage_cmds + $(MAKE) DESTDIR=$($(package)_staging_dir) -C genisoimage install +endef + +define $(package)_postprocess_cmds + rm bin/isovfy bin/isoinfo bin/isodump bin/isodebug bin/devdump +endef diff --git a/contrib/depends/packages/native_cmake-unused.mk b/contrib/depends/packages/native_cmake-unused.mk new file mode 100644 index 00000000000..c9ab7571141 --- /dev/null +++ b/contrib/depends/packages/native_cmake-unused.mk @@ -0,0 +1,23 @@ +package=native_cmake +$(package)_version=3.14.0 +$(package)_version_dot=v3.14 +$(package)_download_path=https://cmake.org/files/$($(package)_version_dot)/ +$(package)_file_name=cmake-$($(package)_version).tar.gz +$(package)_sha256_hash=aa76ba67b3c2af1946701f847073f4652af5cbd9f141f221c97af99127e75502 + +define $(package)_set_vars +$(package)_config_opts= +endef + +define $(package)_config_cmds + ./bootstrap &&\ + ./configure $($(package)_config_opts) +endef + +define $(package)_build_cmd + $(MAKE) +endef + +define $(package)_stage_cmds + $(MAKE) DESTDIR=$($(package)_staging_dir) install +endef diff --git a/contrib/depends/packages/native_ds_store.mk b/contrib/depends/packages/native_ds_store.mk new file mode 100644 index 00000000000..49f5829ac1c --- /dev/null +++ b/contrib/depends/packages/native_ds_store.mk @@ -0,0 +1,17 @@ +package=native_ds_store +$(package)_version=1.1.0 +$(package)_download_path=https://bitbucket.org/al45tair/ds_store/get +$(package)_download_file=v$($(package)_version).tar.bz2 +$(package)_file_name=$(package)-$($(package)_version).tar.bz2 +$(package)_sha256_hash=921596764d71d1bbd3297a90ef6d286f718794d667e4f81d91d14053525d64c1 +$(package)_install_libdir=$(build_prefix)/lib/python/dist-packages +$(package)_dependencies=native_biplist + +define $(package)_build_cmds + python setup.py build +endef + +define $(package)_stage_cmds + mkdir -p $($(package)_install_libdir) && \ + python setup.py install --root=$($(package)_staging_dir) --prefix=$(build_prefix) --install-lib=$($(package)_install_libdir) +endef diff --git a/contrib/depends/packages/native_libdmg-hfsplus.mk b/contrib/depends/packages/native_libdmg-hfsplus.mk new file mode 100644 index 00000000000..a4ffb6046cc --- /dev/null +++ b/contrib/depends/packages/native_libdmg-hfsplus.mk @@ -0,0 +1,22 @@ +package=native_libdmg-hfsplus +$(package)_version=0.1 +$(package)_download_path=https://github.com/theuni/libdmg-hfsplus/archive +$(package)_file_name=libdmg-hfsplus-v$($(package)_version).tar.gz +$(package)_sha256_hash=6569a02eb31c2827080d7d59001869ea14484c281efab0ae7f2b86af5c3120b3 +$(package)_build_subdir=build + +define $(package)_preprocess_cmds + mkdir build +endef + +define $(package)_config_cmds + cmake -DCMAKE_INSTALL_PREFIX:PATH=$(build_prefix)/bin .. +endef + +define $(package)_build_cmds + $(MAKE) -C dmg +endef + +define $(package)_stage_cmds + $(MAKE) DESTDIR=$($(package)_staging_dir) -C dmg install +endef diff --git a/contrib/depends/packages/native_mac_alias.mk b/contrib/depends/packages/native_mac_alias.mk new file mode 100644 index 00000000000..85a8a402bf9 --- /dev/null +++ b/contrib/depends/packages/native_mac_alias.mk @@ -0,0 +1,21 @@ +package=native_mac_alias +$(package)_version=1.1.0 +$(package)_download_path=https://bitbucket.org/al45tair/mac_alias/get +$(package)_download_file=v$($(package)_version).tar.bz2 +$(package)_file_name=$(package)-$($(package)_version).tar.bz2 +$(package)_sha256_hash=87ad827e66790028361e43fc754f68ed041a9bdb214cca03c853f079b04fb120 +$(package)_install_libdir=$(build_prefix)/lib/python/dist-packages +$(package)_patches=python3.patch + +define $(package)_preprocess_cmds + patch -p1 < $($(package)_patch_dir)/python3.patch +endef + +define $(package)_build_cmds + python setup.py build +endef + +define $(package)_stage_cmds + mkdir -p $($(package)_install_libdir) && \ + python setup.py install --root=$($(package)_staging_dir) --prefix=$(build_prefix) --install-lib=$($(package)_install_libdir) +endef diff --git a/contrib/depends/packages/native_protobuf.mk b/contrib/depends/packages/native_protobuf.mk new file mode 100644 index 00000000000..35f648b9aca --- /dev/null +++ b/contrib/depends/packages/native_protobuf.mk @@ -0,0 +1,27 @@ +package=protobuf3 +$(package)_version=3.6.1 +$(package)_download_path=https://github.com/protocolbuffers/protobuf/releases/download/v$($(package)_version)/ +$(package)_file_name=protobuf-cpp-$($(package)_version).tar.gz +$(package)_sha256_hash=b3732e471a9bb7950f090fd0457ebd2536a9ba0891b7f3785919c654fe2a2529 +$(package)_cxxflags=-std=c++11 + +define $(package)_set_vars + $(package)_config_opts=--disable-shared --prefix=$(build_prefix) + $(package)_config_opts_linux=--with-pic +endef + +define $(package)_config_cmds + $($(package)_autoconf) +endef + +define $(package)_build_cmds + $(MAKE) -C src +endef + +define $(package)_stage_cmds + $(MAKE) DESTDIR=$($(package)_staging_dir) -C src install +endef + +define $(package)_postprocess_cmds + rm lib/libprotoc.a +endef diff --git a/contrib/depends/packages/ncurses.mk b/contrib/depends/packages/ncurses.mk new file mode 100644 index 00000000000..4e06c00d901 --- /dev/null +++ b/contrib/depends/packages/ncurses.mk @@ -0,0 +1,58 @@ +package=ncurses +$(package)_version=6.1 +$(package)_download_path=https://ftp.gnu.org/gnu/ncurses +$(package)_file_name=$(package)-$($(package)_version).tar.gz +$(package)_sha256_hash=aa057eeeb4a14d470101eff4597d5833dcef5965331be3528c08d99cebaa0d17 + +define $(package)_set_vars + $(package)_build_opts=CC="$($(package)_cc)" + $(package)_config_env=AR="$($(package)_ar)" RANLIB="$($(package)_ranlib)" CC="$($(package)_cc)" + $(package)_config_env_darwin=RANLIB="$(host_prefix)/native/bin/x86_64-apple-darwin11-ranlib" AR="$(host_prefix)/native/bin/x86_64-apple-darwin11-ar" CC="$(host_prefix)/native/bin/$($(package)_cc)" + $(package)_config_opts=--prefix=$(host_prefix) + $(package)_config_opts+=--disable-shared + $(package)_config_opts+=--with-build-cc=gcc + $(package)_config_opts+=--without-debug + $(package)_config_opts+=--without-ada + $(package)_config_opts+=--without-cxx-binding + $(package)_config_opts+=--without-cxx + $(package)_config_opts+=--without-ticlib + $(package)_config_opts+=--without-tic + $(package)_config_opts+=--without-progs + $(package)_config_opts+=--without-tests + $(package)_config_opts+=--without-tack + $(package)_config_opts+=--without-manpages + $(package)_config_opts+=--disable-tic-depends + $(package)_config_opts+=--disable-big-strings + $(package)_config_opts+=--disable-ext-colors + $(package)_config_opts+=--enable-pc-files + $(package)_config_opts+=--host=$(HOST) + $(pacakge)_config_opts+=--without-shared + $(pacakge)_config_opts+=--without-pthread + $(pacakge)_config_opts+=--disable-rpath + $(pacakge)_config_opts+=--disable-colorfgbg + $(pacakge)_config_opts+=--disable-ext-colors + $(pacakge)_config_opts+=--disable-ext-mouse + $(pacakge)_config_opts+=--disable-symlinks + $(pacakge)_config_opts+=--enable-warnings + $(pacakge)_config_opts+=--enable-assertions + $(pacakge)_config_opts+=--disable-home-terminfo + $(pacakge)_config_opts+=--enable-database + $(pacakge)_config_opts+=--enable-sp-funcs + $(pacakge)_config_opts+=--enable-term-driver + $(pacakge)_config_opts+=--enable-interop + $(pacakge)_config_opts+=--enable-widec + $(package)_build_opts=CFLAGS="$($(package)_cflags) $($(package)_cppflags) -fPIC" +endef + +define $(package)_config_cmds + ./configure $($(package)_config_opts) +endef + +define $(package)_build_cmds + $(MAKE) $($(package)_build_opts) V=1 +endef + +define $(package)_stage_cmds + $(MAKE) install DESTDIR=$($(package)_staging_dir) +endef + diff --git a/contrib/depends/packages/openssl.mk b/contrib/depends/packages/openssl.mk new file mode 100644 index 00000000000..e39dc1d0454 --- /dev/null +++ b/contrib/depends/packages/openssl.mk @@ -0,0 +1,70 @@ +package=openssl +$(package)_version=1.0.2r +$(package)_download_path=https://www.openssl.org/source +$(package)_file_name=$(package)-$($(package)_version).tar.gz +$(package)_sha256_hash=ae51d08bba8a83958e894946f15303ff894d75c2b8bbd44a852b64e3fe11d0d6 + +define $(package)_set_vars +$(package)_config_env=AR="$($(package)_ar)" RANLIB="$($(package)_ranlib)" CC="$($(package)_cc)" +$(package)_config_opts=--prefix=$(host_prefix) --openssldir=$(host_prefix)/etc/openssl +$(package)_config_opts+=no-capieng +$(package)_config_opts+=no-dso +$(package)_config_opts+=no-dtls1 +$(package)_config_opts+=no-ec_nistp_64_gcc_128 +$(package)_config_opts+=no-gost +$(package)_config_opts+=no-gmp +$(package)_config_opts+=no-heartbeats +$(package)_config_opts+=no-jpake +$(package)_config_opts+=no-krb5 +$(package)_config_opts+=no-libunbound +$(package)_config_opts+=no-md2 +$(package)_config_opts+=no-rc5 +$(package)_config_opts+=no-rdrand +$(package)_config_opts+=no-rfc3779 +$(package)_config_opts+=no-rsax +$(package)_config_opts+=no-sctp +$(package)_config_opts+=no-sha0 +$(package)_config_opts+=no-shared +$(package)_config_opts+=no-ssl-trace +$(package)_config_opts+=no-ssl2 +$(package)_config_opts+=no-ssl3 +$(package)_config_opts+=no-static_engine +$(package)_config_opts+=no-store +$(package)_config_opts+=no-unit-test +$(package)_config_opts+=no-weak-ssl-ciphers +$(package)_config_opts+=no-zlib +$(package)_config_opts+=no-zlib-dynamic +$(package)_config_opts+=$($(package)_cflags) $($(package)_cppflags) +$(package)_config_opts_linux=-fPIC -Wa,--noexecstack +$(package)_config_opts_x86_64_linux=linux-x86_64 +$(package)_config_opts_i686_linux=linux-generic32 +$(package)_config_opts_arm_linux=linux-generic32 +$(package)_config_opts_aarch64_linux=linux-generic64 +$(package)_config_opts_mipsel_linux=linux-generic32 +$(package)_config_opts_mips_linux=linux-generic32 +$(package)_config_opts_powerpc_linux=linux-generic32 +$(package)_config_opts_x86_64_darwin=darwin64-x86_64-cc +$(package)_config_opts_x86_64_mingw32=mingw64 +$(package)_config_opts_i686_mingw32=mingw +endef + +define $(package)_preprocess_cmds + sed -i.old "/define DATE/d" util/mkbuildinf.pl && \ + sed -i.old "s|engines apps test|engines|" Makefile.org +endef + +define $(package)_config_cmds + ./Configure $($(package)_config_opts) +endef + +define $(package)_build_cmds + $(MAKE) -j1 build_libs libcrypto.pc libssl.pc openssl.pc +endef + +define $(package)_stage_cmds + $(MAKE) INSTALL_PREFIX=$($(package)_staging_dir) -j1 install_sw +endef + +define $(package)_postprocess_cmds + rm -rf share bin etc +endef diff --git a/contrib/depends/packages/packages.mk b/contrib/depends/packages/packages.mk new file mode 100644 index 00000000000..562f4f7d6bd --- /dev/null +++ b/contrib/depends/packages/packages.mk @@ -0,0 +1,22 @@ +packages:=boost openssl zeromq cppzmq expat ldns libiconv hidapi protobuf libusb +native_packages := native_ccache native_protobuf + +darwin_native_packages = native_biplist native_ds_store native_mac_alias +darwin_packages = sodium-darwin ncurses readline + +linux_packages = eudev ncurses readline unwind sodium +qt_packages = qt + +ifeq ($(build_tests),ON) +packages += gtest +endif + +ifeq ($(host_os),mingw32) +packages += icu4c +packages += sodium +endif + +ifneq ($(build_os),darwin) +darwin_native_packages += native_cctools native_cdrkit native_libdmg-hfsplus +endif + diff --git a/contrib/depends/packages/protobuf.mk b/contrib/depends/packages/protobuf.mk new file mode 100644 index 00000000000..81fa78a3fb4 --- /dev/null +++ b/contrib/depends/packages/protobuf.mk @@ -0,0 +1,31 @@ +package=protobuf +$(package)_version=$(native_$(package)_version) +$(package)_download_path=$(native_$(package)_download_path) +$(package)_file_name=$(native_$(package)_file_name) +$(package)_sha256_hash=$(native_$(package)_sha256_hash) +$(package)_dependencies=native_$(package) +$(package)_cxxflags=-std=c++11 + +define $(package)_set_vars + $(package)_config_opts=--disable-shared --with-protoc=$(build_prefix)/bin/protoc + $(package)_config_opts_linux=--with-pic +endef + +define $(package)_config_cmds + $($(package)_autoconf) +endef + +define $(package)_build_cmds + $(MAKE) -C src libprotobuf.la +endef + +define $(package)_stage_cmds + $(MAKE) DESTDIR=$($(package)_staging_dir) -C src install-libLTLIBRARIES install-nobase_includeHEADERS &&\ + $(MAKE) DESTDIR=$($(package)_staging_dir) install-pkgconfigDATA +endef + +define $(package)_postprocess_cmds + rm lib/libprotoc.a &&\ + rm lib/*.la +endef + diff --git a/contrib/depends/packages/qt.mk b/contrib/depends/packages/qt.mk new file mode 100644 index 00000000000..6a6f0e0f709 --- /dev/null +++ b/contrib/depends/packages/qt.mk @@ -0,0 +1,148 @@ +PACKAGE=qt +$(package)_version=5.7.1 +$(package)_download_path=https://download.qt.io/archive/qt/5.7/5.7.1/submodules +$(package)_suffix=opensource-src-$($(package)_version).tar.gz +$(package)_file_name=qtbase-$($(package)_suffix) +$(package)_sha256_hash=95f83e532d23b3ddbde7973f380ecae1bac13230340557276f75f2e37984e410 +$(package)_build_subdir=qtbase +$(package)_qt_libs=corelib +$(package)_patches=pidlist_absolute.patch fix_qt_pkgconfig.patch qfixed-coretext.patch + +$(package)_qttranslations_file_name=qttranslations-$($(package)_suffix) +$(package)_qttranslations_sha256_hash=3a15aebd523c6d89fb97b2d3df866c94149653a26d27a00aac9b6d3020bc5a1d + +$(package)_qttools_file_name=qttools-$($(package)_suffix) +$(package)_qttools_sha256_hash=22d67de915cb8cd93e16fdd38fa006224ad9170bd217c2be1e53045a8dd02f0f + +$(package)_extra_sources = $($(package)_qttranslations_file_name) +$(package)_extra_sources += $($(package)_qttools_file_name) + +define $(package)_set_vars +$(package)_config_opts_release = -release +$(package)_config_opts_debug = -debug +$(package)_config_opts += -bindir $(build_prefix)/bin +$(package)_config_opts += -c++std c++11 +$(package)_config_opts += -confirm-license +$(package)_config_opts += -dbus-runtime +$(package)_config_opts += -no-alsa +$(package)_config_opts += -no-audio-backend +$(package)_config_opts += -no-cups +$(package)_config_opts += -no-egl +$(package)_config_opts += -no-eglfs +$(package)_config_opts += -no-feature-style-windowsmobile +$(package)_config_opts += -no-feature-style-windowsce +$(package)_config_opts += -no-freetype +$(package)_config_opts += -no-gif +$(package)_config_opts += -no-glib +$(package)_config_opts += -no-gstreamer +$(package)_config_opts += -no-icu +$(package)_config_opts += -no-iconv +$(package)_config_opts += -no-kms +$(package)_config_opts += -no-linuxfb +$(package)_config_opts += -no-libudev +$(package)_config_opts += -no-mitshm +$(package)_config_opts += -no-mtdev +$(package)_config_opts += -no-pulseaudio +$(package)_config_opts += -no-openvg +$(package)_config_opts += -no-reduce-relocations +$(package)_config_opts += -no-qml-debug +$(package)_config_opts += -no-sql-db2 +$(package)_config_opts += -no-sql-ibase +$(package)_config_opts += -no-sql-oci +$(package)_config_opts += -no-sql-tds +$(package)_config_opts += -no-sql-mysql +$(package)_config_opts += -no-sql-odbc +$(package)_config_opts += -no-sql-psql +$(package)_config_opts += -no-sql-sqlite +$(package)_config_opts += -no-sql-sqlite2 +$(package)_config_opts += -no-use-gold-linker +$(package)_config_opts += -no-xinput2 +$(package)_config_opts += -no-xrender +$(package)_config_opts += -nomake examples +$(package)_config_opts += -nomake tests +$(package)_config_opts += -opensource +$(package)_config_opts += -no-openssl +$(package)_config_opts += -optimized-qmake +$(package)_config_opts += -pch +$(package)_config_opts += -pkg-config +$(package)_config_opts += -no-libpng +$(package)_config_opts += -no-libjpeg +$(package)_config_opts += -qt-pcre +$(package)_config_opts += -no-zlib +$(package)_config_opts += -reduce-exports +$(package)_config_opts += -static +$(package)_config_opts += -silent +$(package)_config_opts += -v +$(package)_config_opts += -no-feature-printer +$(package)_config_opts += -no-feature-printdialog +$(package)_config_opts += -no-gui +$(package)_config_opts += -no-freetype +$(package)_config_opts += -no-sm +$(package)_config_opts += -no-fontconfig +$(package)_config_opts += -no-opengl +$(package)_config_opts += -no-xkb +$(package)_config_opts += -no-xcb +$(package)_config_opts += -no-xshape +$(package)_build_env = QT_RCC_TEST=1 +endef + +define $(package)_fetch_cmds +$(call fetch_file,$(package),$($(package)_download_path),$($(package)_download_file),$($(package)_file_name),$($(package)_sha256_hash)) && \ +$(call fetch_file,$(package),$($(package)_download_path),$($(package)_qttranslations_file_name),$($(package)_qttranslations_file_name),$($(package)_qttranslations_sha256_hash)) && \ +$(call fetch_file,$(package),$($(package)_download_path),$($(package)_qttools_file_name),$($(package)_qttools_file_name),$($(package)_qttools_sha256_hash)) +endef + +define $(package)_extract_cmds + mkdir -p $($(package)_extract_dir) && \ + echo "$($(package)_sha256_hash) $($(package)_source)" > $($(package)_extract_dir)/.$($(package)_file_name).hash && \ + echo "$($(package)_qttranslations_sha256_hash) $($(package)_source_dir)/$($(package)_qttranslations_file_name)" >> $($(package)_extract_dir)/.$($(package)_file_name).hash && \ + echo "$($(package)_qttools_sha256_hash) $($(package)_source_dir)/$($(package)_qttools_file_name)" >> $($(package)_extract_dir)/.$($(package)_file_name).hash && \ + $(build_SHA256SUM) -c $($(package)_extract_dir)/.$($(package)_file_name).hash && \ + mkdir qtbase && \ + tar --strip-components=1 -xf $($(package)_source) -C qtbase && \ + mkdir qttranslations && \ + tar --strip-components=1 -xf $($(package)_source_dir)/$($(package)_qttranslations_file_name) -C qttranslations && \ + mkdir qttools && \ + tar --strip-components=1 -xf $($(package)_source_dir)/$($(package)_qttools_file_name) -C qttools +endef + + +define $(package)_preprocess_cmds + sed -i.old "s|updateqm.commands = \$$$$\$$$$LRELEASE|updateqm.commands = $($(package)_extract_dir)/qttools/bin/lrelease|" qttranslations/translations/translations.pro && \ + sed -i.old "/updateqm.depends =/d" qttranslations/translations/translations.pro && \ + patch -p1 < $($(package)_patch_dir)/pidlist_absolute.patch && \ + patch -p1 < $($(package)_patch_dir)/fix_qt_pkgconfig.patch && \ + patch -p1 < $($(package)_patch_dir)/qfixed-coretext.patch && \ + echo "!host_build: QMAKE_CFLAGS += $($(package)_cflags) $($(package)_cppflags)" >> qtbase/mkspecs/common/gcc-base.conf && \ + echo "!host_build: QMAKE_CXXFLAGS += $($(package)_cxxflags) $($(package)_cppflags)" >> qtbase/mkspecs/common/gcc-base.conf && \ + echo "!host_build: QMAKE_LFLAGS += $($(package)_ldflags)" >> qtbase/mkspecs/common/gcc-base.conf +endef + +define $(package)_config_cmds + export PKG_CONFIG_SYSROOT_DIR=/ && \ + export PKG_CONFIG_LIBDIR=$(host_prefix)/lib/pkgconfig && \ + export PKG_CONFIG_PATH=$(host_prefix)/share/pkgconfig && \ + ./configure $($(package)_config_opts) && \ + echo "CONFIG += force_bootstrap" >> mkspecs/qconfig.pri && \ + $(MAKE) sub-src-clean && \ + cd ../qttranslations && ../qtbase/bin/qmake qttranslations.pro -o Makefile && \ + cd translations && ../../qtbase/bin/qmake translations.pro -o Makefile && cd ../.. &&\ + cd qttools/src/linguist/lrelease/ && ../../../../qtbase/bin/qmake lrelease.pro -o Makefile +endef + +define $(package)_build_cmds + $(MAKE) -C src $(addprefix sub-,$($(package)_qt_libs)) && \ + $(MAKE) -C ../qttools/src/linguist/lrelease && \ + $(MAKE) -C ../qttranslations +endef + +define $(package)_stage_cmds + $(MAKE) -C src INSTALL_ROOT=$($(package)_staging_dir) $(addsuffix -install_subtargets,$(addprefix sub-,$($(package)_qt_libs))) && cd .. &&\ + $(MAKE) -C qttools/src/linguist/lrelease INSTALL_ROOT=$($(package)_staging_dir) install_target && \ + $(MAKE) -C qttranslations INSTALL_ROOT=$($(package)_staging_dir) install_subtargets +endef + +define $(package)_postprocess_cmds + rm -rf native/mkspecs/ native/lib/ lib/cmake/ && \ + rm -f lib/lib*.la lib/*.prl plugins/*/*.prl +endef diff --git a/contrib/depends/packages/readline.mk b/contrib/depends/packages/readline.mk new file mode 100644 index 00000000000..0e21007491a --- /dev/null +++ b/contrib/depends/packages/readline.mk @@ -0,0 +1,29 @@ +package=readline +$(package)_version=8.0 +$(package)_download_path=https://ftp.gnu.org/gnu/readline +$(package)_file_name=$(package)-$($(package)_version).tar.gz +$(package)_sha256_hash=e339f51971478d369f8a053a330a190781acb9864cf4c541060f12078948e461 +$(package)_dependencies=ncurses + +define $(package)_set_vars + $(package)_config_opts=--prefix=$(host_prefix) + $(package)_config_opts+=--exec-prefix=$(host_prefix) + $(package)_config_opts+=--host=$(HOST) + $(package)_config_opts+=--disable-shared --with-curses + $(package)_config_opts_release=--disable-debug-mode + $(package)_config_opts_darwin+=RANLIB="$(host_prefix)/native/bin/x86_64-apple-darwin11-ranlib" AR="$(host_prefix)/native/bin/x86_64-apple-darwin11-ar" CC="$(host_prefix)/native/bin/$($(package)_cc)" + $(package)_build_opts=CFLAGS="$($(package)_cflags) $($(package)_cppflags) -fPIC" +endef + +define $(package)_config_cmds + ./configure $($(package)_config_opts) +endef + +define $(package)_build_cmds + $(MAKE) $($(package)_build_opts) +endef + +define $(package)_stage_cmds + $(MAKE) install DESTDIR=$($(package)_staging_dir) prefix=$(host_prefix) exec-prefix=$(host_prefix) +endef + diff --git a/contrib/depends/packages/sodium-darwin.mk b/contrib/depends/packages/sodium-darwin.mk new file mode 100644 index 00000000000..9f11a942602 --- /dev/null +++ b/contrib/depends/packages/sodium-darwin.mk @@ -0,0 +1,30 @@ +package=sodium-darwin +$(package)_version=1.0.16 +$(package)_download_path=https://download.libsodium.org/libsodium/releases/ +$(package)_file_name=libsodium-$($(package)_version).tar.gz +$(package)_sha256_hash=eeadc7e1e1bcef09680fb4837d448fbdf57224978f865ac1c16745868fbd0533 + +define $(package)_set_vars +$(package)_build_opts_darwin=OS=Darwin LIBTOOL="$($(package)_libtool)" +$(package)_config_opts=--enable-static --disable-shared +$(package)_config_opts+=--prefix=$(host_prefix) +endef + +define $(package)_config_cmds + ./autogen.sh &&\ + $($(package)_autoconf) $($(package)_config_opts) RANLIB="$(host_prefix)/native/bin/x86_64-apple-darwin11-ranlib" AR="$(host_prefix)/native/bin/x86_64-apple-darwin11-ar" CC="$(host_prefix)/native/bin/$($(package)_cc)" +endef + +define $(package)_build_cmds + echo "path is problematic here" &&\ + make +endef + +define $(package)_stage_cmds + $(MAKE) DESTDIR=$($(package)_staging_dir) install +endef + +define $(package)_postprocess_cmds + rm lib/*.la +endef + diff --git a/contrib/depends/packages/sodium.mk b/contrib/depends/packages/sodium.mk new file mode 100644 index 00000000000..b71f4383e0b --- /dev/null +++ b/contrib/depends/packages/sodium.mk @@ -0,0 +1,30 @@ +package=sodium +$(package)_version=1.0.16 +$(package)_download_path=https://download.libsodium.org/libsodium/releases/ +$(package)_file_name=libsodium-$($(package)_version).tar.gz +$(package)_sha256_hash=eeadc7e1e1bcef09680fb4837d448fbdf57224978f865ac1c16745868fbd0533 +$(package)_patches=fix-whitespace.patch + +define $(package)_set_vars +$(package)_config_opts=--enable-static --disable-shared +$(package)_config_opts+=--prefix=$(host_prefix) +endef + +define $(package)_config_cmds + ./autogen.sh &&\ + patch -p1 < $($(package)_patch_dir)/fix-whitespace.patch &&\ + $($(package)_autoconf) $($(package)_config_opts) +endef + +define $(package)_build_cmds + $(MAKE) +endef + +define $(package)_stage_cmds + $(MAKE) DESTDIR=$($(package)_staging_dir) install +endef + +define $(package)_postprocess_cmds + rm lib/*.la +endef + diff --git a/contrib/depends/packages/unbound.mk b/contrib/depends/packages/unbound.mk new file mode 100644 index 00000000000..733a7f2326b --- /dev/null +++ b/contrib/depends/packages/unbound.mk @@ -0,0 +1,28 @@ +package=unbound +$(package)_version=1.6.8 +$(package)_download_path=https://www.unbound.net/downloads/ +$(package)_file_name=$(package)-$($(package)_version).tar.gz +$(package)_sha256_hash=e3b428e33f56a45417107448418865fe08d58e0e7fea199b855515f60884dd49 +$(package)_dependencies=openssl expat ldns + +define $(package)_set_vars + $(package)_config_opts=--disable-shared --enable-static --without-pyunbound --prefix=$(host_prefix) --with-libexpat=$(host_prefix) --with-ssl=$(host_prefix) --with-libevent=no --without-pythonmodule --disable-flto --with-pthreads + $(package)_config_opts_linux=--with-pic + $(package)_config_opts_w64=--enable-static-exe --sysconfdir=/etc --prefix=$(host_prefix) --target=$(host_prefix) + $(package)_build_opts_mingw32=LDFLAGS="$($(package)_ldflags) -lpthread" +endef + +define $(package)_config_cmds + $($(package)_autoconf) $($(package)_config_opts) +endef + +define $(package)_build_cmds + $(MAKE) $($(package)_build_opts) +endef + +define $(package)_stage_cmds + $(MAKE) DESTDIR=$($(package)_staging_dir) install +endef + +define $(package)_postprocess_cmds +endef diff --git a/contrib/depends/packages/unwind.mk b/contrib/depends/packages/unwind.mk new file mode 100644 index 00000000000..fddbd056114 --- /dev/null +++ b/contrib/depends/packages/unwind.mk @@ -0,0 +1,24 @@ +package=unwind +$(package)_version=1.2 +$(package)_download_path=https://download.savannah.nongnu.org/releases/libunwind +$(package)_file_name=lib$(package)-$($(package)_version).tar.gz +$(package)_sha256_hash=1de38ffbdc88bd694d10081865871cd2bfbb02ad8ef9e1606aee18d65532b992 + +define $(package)_config_cmds + cp -f $(BASEDIR)/config.guess config/config.guess &&\ + cp -f $(BASEDIR)/config.sub config/config.sub &&\ + $($(package)_autoconf) --disable-shared --enable-static +endef + +define $(package)_build_cmds + $(MAKE) +endef + +define $(package)_stage_cmds + $(MAKE) DESTDIR=$($(package)_staging_dir) install +endef + +define $(package)_postprocess_cmds + rm lib/*.la +endef + diff --git a/contrib/depends/packages/xproto.mk b/contrib/depends/packages/xproto.mk new file mode 100644 index 00000000000..52fe253c77c --- /dev/null +++ b/contrib/depends/packages/xproto.mk @@ -0,0 +1,21 @@ +package=xproto +$(package)_version=7.0.26 +$(package)_download_path=https://xorg.freedesktop.org/releases/individual/proto +$(package)_file_name=$(package)-$($(package)_version).tar.bz2 +$(package)_sha256_hash=636162c1759805a5a0114a369dffdeccb8af8c859ef6e1445f26a4e6e046514f + +define $(package)_set_vars +$(package)_config_opts=--disable-shared +endef + +define $(package)_config_cmds + $($(package)_autoconf) +endef + +define $(package)_build_cmds + $(MAKE) +endef + +define $(package)_stage_cmds + $(MAKE) DESTDIR=$($(package)_staging_dir) install +endef diff --git a/contrib/depends/packages/zeromq.mk b/contrib/depends/packages/zeromq.mk new file mode 100644 index 00000000000..f17dbeebe4c --- /dev/null +++ b/contrib/depends/packages/zeromq.mk @@ -0,0 +1,36 @@ +package=zeromq +$(package)_version=4.1.5 +$(package)_download_path=https://github.com/zeromq/zeromq4-1/releases/download/v$($(package)_version)/ +$(package)_file_name=$(package)-$($(package)_version).tar.gz +$(package)_sha256_hash=04aac57f081ffa3a2ee5ed04887be9e205df3a7ddade0027460b8042432bdbcf +$(package)_patches=9114d3957725acd34aa8b8d011585812f3369411.patch 9e6745c12e0b100cd38acecc16ce7db02905e27c.patch + +define $(package)_set_vars + $(package)_config_opts=--without-documentation --disable-shared --without-libsodium --disable-curve + $(package)_config_opts_linux=--with-pic + $(package)_cxxflags=-std=c++11 +endef + +define $(package)_preprocess_cmds + patch -p1 < $($(package)_patch_dir)/9114d3957725acd34aa8b8d011585812f3369411.patch && \ + patch -p1 < $($(package)_patch_dir)/9e6745c12e0b100cd38acecc16ce7db02905e27c.patch && \ + ./autogen.sh +endef + +define $(package)_config_cmds + $($(package)_autoconf) +endef + +define $(package)_build_cmds + $(MAKE) libzmq.la +endef + +define $(package)_stage_cmds + $(MAKE) DESTDIR=$($(package)_staging_dir) install-libLTLIBRARIES install-includeHEADERS install-pkgconfigDATA +endef + +define $(package)_postprocess_cmds + rm -rf bin share &&\ + rm lib/*.la +endef + diff --git a/contrib/depends/patches/cmake/cmake-1-fixes.patch b/contrib/depends/patches/cmake/cmake-1-fixes.patch new file mode 100644 index 00000000000..062c067676d --- /dev/null +++ b/contrib/depends/patches/cmake/cmake-1-fixes.patch @@ -0,0 +1,67 @@ +This file is part of MXE. See LICENSE.md for licensing information. + +Contains ad hoc patches for cross building. + +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: Tony Theodore +Date: Fri, 12 Aug 2016 02:01:20 +1000 +Subject: [PATCH 1/3] fix windres invocation options + +windres doesn't recognise various gcc flags like -mms-bitfields, +-fopenmp, -mthreads etc. (basically not `-D` or `-I`) + +diff --git a/Modules/Platform/Windows-windres.cmake b/Modules/Platform/Windows-windres.cmake +index 1111111..2222222 100644 +--- a/Modules/Platform/Windows-windres.cmake ++++ b/Modules/Platform/Windows-windres.cmake +@@ -1 +1 @@ +-set(CMAKE_RC_COMPILE_OBJECT " -O coff ") ++set(CMAKE_RC_COMPILE_OBJECT " -O coff ") + +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: Tony Theodore +Date: Tue, 25 Jul 2017 20:34:56 +1000 +Subject: [PATCH 2/3] add option to disable -isystem + +taken from (not accepted): +https://gitlab.kitware.com/cmake/cmake/merge_requests/895 + +see also: +https://gitlab.kitware.com/cmake/cmake/issues/16291 +https://gitlab.kitware.com/cmake/cmake/issues/16919 + +diff --git a/Modules/Compiler/GNU.cmake b/Modules/Compiler/GNU.cmake +index 1111111..2222222 100644 +--- a/Modules/Compiler/GNU.cmake ++++ b/Modules/Compiler/GNU.cmake +@@ -42,7 +42,7 @@ macro(__compiler_gnu lang) + string(APPEND CMAKE_${lang}_FLAGS_RELWITHDEBINFO_INIT " -O2 -g -DNDEBUG") + set(CMAKE_${lang}_CREATE_PREPROCESSED_SOURCE " -E > ") + set(CMAKE_${lang}_CREATE_ASSEMBLY_SOURCE " -S -o ") +- if(NOT APPLE OR NOT CMAKE_${lang}_COMPILER_VERSION VERSION_LESS 4) # work around #4462 ++ if(NOT APPLE OR NOT CMAKE_${lang}_COMPILER_VERSION VERSION_LESS 4 AND (NOT MXE_DISABLE_INCLUDE_SYSTEM_FLAG)) # work around #4462 + set(CMAKE_INCLUDE_SYSTEM_FLAG_${lang} "-isystem ") + endif() + endmacro() + +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: Tony Theodore +Date: Tue, 15 Aug 2017 15:25:06 +1000 +Subject: [PATCH 3/3] add CPACK_NSIS_EXECUTABLE variable + + +diff --git a/Source/CPack/cmCPackNSISGenerator.cxx b/Source/CPack/cmCPackNSISGenerator.cxx +index 1111111..2222222 100644 +--- a/Source/CPack/cmCPackNSISGenerator.cxx ++++ b/Source/CPack/cmCPackNSISGenerator.cxx +@@ -384,7 +384,9 @@ int cmCPackNSISGenerator::InitializeInternal() + } + #endif + +- nsisPath = cmSystemTools::FindProgram("makensis", path, false); ++ this->SetOptionIfNotSet("CPACK_NSIS_EXECUTABLE", "makensis"); ++ nsisPath = cmSystemTools::FindProgram( ++ this->GetOption("CPACK_NSIS_EXECUTABLE"), path, false); + + if (nsisPath.empty()) { + cmCPackLogger( diff --git a/contrib/depends/patches/icu4c/icu-001-dont-build-static-dynamic-twice.patch b/contrib/depends/patches/icu4c/icu-001-dont-build-static-dynamic-twice.patch new file mode 100644 index 00000000000..bbd4e99e7f3 --- /dev/null +++ b/contrib/depends/patches/icu4c/icu-001-dont-build-static-dynamic-twice.patch @@ -0,0 +1,37 @@ +Don't build object files twice + +When passed --enable-static and --enable-shared, icu will generate +both a shared and a static version of its libraries. + +However, in order to do so, it builds each and every object file +twice: once with -fPIC (for the shared library), and once without +-fPIC (for the static library). While admittedly building -fPIC for a +static library generates a slightly suboptimal code, this is what all +the autotools-based project are doing. They build each object file +once, and they use it for both the static and shared libraries. + +icu builds the object files for the shared library as .o files, and +the object files for static library as .ao files. By simply changing +the suffix of object files used for static libraries to ".o", we tell +icu to use the ones built for the shared library (i.e, with -fPIC), +and avoid the double build of icu. + +On a fast build server, this brings the target icu build from +3m41.302s down to 1m43.926s (approximate numbers: some other builds +are running on the system at the same time). + +Signed-off-by: Thomas Petazzoni + +Index: b/source/config/mh-linux +=================================================================== +--- a/source/config/mh-linux ++++ b/source/config/mh-linux +@@ -38,7 +38,7 @@ + ## Shared object suffix + SO = so + ## Non-shared intermediate object suffix +-STATIC_O = ao ++STATIC_O = o + + ## Compilation rules + %.$(STATIC_O): $(srcdir)/%.c diff --git a/contrib/depends/patches/libiconv/fix-whitespace.patch b/contrib/depends/patches/libiconv/fix-whitespace.patch new file mode 100644 index 00000000000..531364b45f2 --- /dev/null +++ b/contrib/depends/patches/libiconv/fix-whitespace.patch @@ -0,0 +1,13 @@ +diff --git a/preload/configure b/preload/configure +index aab5c77..e20b8f0 100755 +--- a/preload/configure ++++ b/preload/configure +@@ -588,7 +588,7 @@ MAKEFLAGS= + PACKAGE_NAME='libiconv' + PACKAGE_TARNAME='libiconv' + PACKAGE_VERSION='0' +-PACKAGE_STRING='libiconv 0' ++PACKAGE_STRING='libiconv0' + PACKAGE_BUGREPORT='' + PACKAGE_URL='' + diff --git a/contrib/depends/patches/native_biplist/sorted_list.patch b/contrib/depends/patches/native_biplist/sorted_list.patch new file mode 100644 index 00000000000..89abdb1b71f --- /dev/null +++ b/contrib/depends/patches/native_biplist/sorted_list.patch @@ -0,0 +1,29 @@ +--- a/biplist/__init__.py 2014-10-26 19:03:11.000000000 +0000 ++++ b/biplist/__init__.py 2016-07-19 19:30:17.663521999 +0000 +@@ -541,7 +541,7 @@ + return HashableWrapper(n) + elif isinstance(root, dict): + n = {} +- for key, value in iteritems(root): ++ for key, value in sorted(iteritems(root)): + n[self.wrapRoot(key)] = self.wrapRoot(value) + return HashableWrapper(n) + elif isinstance(root, list): +@@ -616,7 +616,7 @@ + elif isinstance(obj, dict): + size = proc_size(len(obj)) + self.incrementByteCount('dictBytes', incr=1+size) +- for key, value in iteritems(obj): ++ for key, value in sorted(iteritems(obj)): + check_key(key) + self.computeOffsets(key, asReference=True) + self.computeOffsets(value, asReference=True) +@@ -714,7 +714,7 @@ + keys = [] + values = [] + objectsToWrite = [] +- for key, value in iteritems(obj): ++ for key, value in sorted(iteritems(obj)): + keys.append(key) + values.append(value) + for key in keys: diff --git a/contrib/depends/patches/native_cdrkit/cdrkit-deterministic.patch b/contrib/depends/patches/native_cdrkit/cdrkit-deterministic.patch new file mode 100644 index 00000000000..8ab0993dc4d --- /dev/null +++ b/contrib/depends/patches/native_cdrkit/cdrkit-deterministic.patch @@ -0,0 +1,86 @@ +--- cdrkit-1.1.11.old/genisoimage/tree.c 2008-10-21 19:57:47.000000000 -0400 ++++ cdrkit-1.1.11/genisoimage/tree.c 2013-12-06 00:23:18.489622668 -0500 +@@ -1139,8 +1139,9 @@ + scan_directory_tree(struct directory *this_dir, char *path, + struct directory_entry *de) + { +- DIR *current_dir; ++ int current_file; + char whole_path[PATH_MAX]; ++ struct dirent **d_list; + struct dirent *d_entry; + struct directory *parent; + int dflag; +@@ -1164,7 +1165,8 @@ + this_dir->dir_flags |= DIR_WAS_SCANNED; + + errno = 0; /* Paranoia */ +- current_dir = opendir(path); ++ //current_dir = opendir(path); ++ current_file = scandir(path, &d_list, NULL, alphasort); + d_entry = NULL; + + /* +@@ -1173,12 +1175,12 @@ + */ + old_path = path; + +- if (current_dir) { ++ if (current_file >= 0) { + errno = 0; +- d_entry = readdir(current_dir); ++ d_entry = d_list[0]; + } + +- if (!current_dir || !d_entry) { ++ if (current_file < 0 || !d_entry) { + int ret = 1; + + #ifdef USE_LIBSCHILY +@@ -1191,8 +1193,8 @@ + de->isorec.flags[0] &= ~ISO_DIRECTORY; + ret = 0; + } +- if (current_dir) +- closedir(current_dir); ++ if(d_list) ++ free(d_list); + return (ret); + } + #ifdef ABORT_DEEP_ISO_ONLY +@@ -1208,7 +1210,7 @@ + errmsgno(EX_BAD, "use Rock Ridge extensions via -R or -r,\n"); + errmsgno(EX_BAD, "or allow deep ISO9660 directory nesting via -D.\n"); + } +- closedir(current_dir); ++ free(d_list); + return (1); + } + #endif +@@ -1250,13 +1252,13 @@ + * The first time through, skip this, since we already asked + * for the first entry when we opened the directory. + */ +- if (dflag) +- d_entry = readdir(current_dir); ++ if (dflag && current_file >= 0) ++ d_entry = d_list[current_file]; + dflag++; + +- if (!d_entry) ++ if (current_file < 0) + break; +- ++ current_file--; + /* OK, got a valid entry */ + + /* If we do not want all files, then pitch the backups. */ +@@ -1348,7 +1350,7 @@ + insert_file_entry(this_dir, whole_path, d_entry->d_name); + #endif /* APPLE_HYB */ + } +- closedir(current_dir); ++ free(d_list); + + #ifdef APPLE_HYB + /* diff --git a/contrib/depends/patches/native_mac_alias/python3.patch b/contrib/depends/patches/native_mac_alias/python3.patch new file mode 100644 index 00000000000..1a32340be5b --- /dev/null +++ b/contrib/depends/patches/native_mac_alias/python3.patch @@ -0,0 +1,72 @@ +diff -dur a/mac_alias/alias.py b/mac_alias/alias.py +--- a/mac_alias/alias.py 2015-10-19 12:12:48.000000000 +0200 ++++ b/mac_alias/alias.py 2016-04-03 12:13:12.037159417 +0200 +@@ -243,10 +243,10 @@ + alias = Alias() + alias.appinfo = appinfo + +- alias.volume = VolumeInfo (volname.replace('/',':'), ++ alias.volume = VolumeInfo (volname.decode().replace('/',':'), + voldate, fstype, disktype, + volattrs, volfsid) +- alias.target = TargetInfo (kind, filename.replace('/',':'), ++ alias.target = TargetInfo (kind, filename.decode().replace('/',':'), + folder_cnid, cnid, + crdate, creator_code, type_code) + alias.target.levels_from = levels_from +@@ -261,9 +261,9 @@ + b.read(1) + + if tag == TAG_CARBON_FOLDER_NAME: +- alias.target.folder_name = value.replace('/',':') ++ alias.target.folder_name = value.decode().replace('/',':') + elif tag == TAG_CNID_PATH: +- alias.target.cnid_path = struct.unpack(b'>%uI' % (length // 4), ++ alias.target.cnid_path = struct.unpack('>%uI' % (length // 4), + value) + elif tag == TAG_CARBON_PATH: + alias.target.carbon_path = value +@@ -298,9 +298,9 @@ + alias.target.creation_date \ + = mac_epoch + datetime.timedelta(seconds=seconds) + elif tag == TAG_POSIX_PATH: +- alias.target.posix_path = value ++ alias.target.posix_path = value.decode() + elif tag == TAG_POSIX_PATH_TO_MOUNTPOINT: +- alias.volume.posix_path = value ++ alias.volume.posix_path = value.decode() + elif tag == TAG_RECURSIVE_ALIAS_OF_DISK_IMAGE: + alias.volume.disk_image_alias = Alias.from_bytes(value) + elif tag == TAG_USER_HOME_LENGTH_PREFIX: +@@ -422,13 +422,13 @@ + # (so doing so is ridiculous, and nothing could rely on it). + b.write(struct.pack(b'>h28pI2shI64pII4s4shhI2s10s', + self.target.kind, +- carbon_volname, voldate, ++ carbon_volname, int(voldate), + self.volume.fs_type, + self.volume.disk_type, + self.target.folder_cnid, + carbon_filename, + self.target.cnid, +- crdate, ++ int(crdate), + self.target.creator_code, + self.target.type_code, + self.target.levels_from, +@@ -449,12 +449,12 @@ + + b.write(struct.pack(b'>hhQhhQ', + TAG_HIGH_RES_VOLUME_CREATION_DATE, +- 8, long(voldate * 65536), ++ 8, int(voldate * 65536), + TAG_HIGH_RES_CREATION_DATE, +- 8, long(crdate * 65536))) ++ 8, int(crdate * 65536))) + + if self.target.cnid_path: +- cnid_path = struct.pack(b'>%uI' % len(self.target.cnid_path), ++ cnid_path = struct.pack('>%uI' % len(self.target.cnid_path), + *self.target.cnid_path) + b.write(struct.pack(b'>hh', TAG_CNID_PATH, + len(cnid_path))) diff --git a/contrib/depends/patches/qt/fix_qt_pkgconfig.patch b/contrib/depends/patches/qt/fix_qt_pkgconfig.patch new file mode 100644 index 00000000000..34302a9f2d2 --- /dev/null +++ b/contrib/depends/patches/qt/fix_qt_pkgconfig.patch @@ -0,0 +1,11 @@ +--- old/qtbase/mkspecs/features/qt_module.prf ++++ new/qtbase/mkspecs/features/qt_module.prf +@@ -245,7 +245,7 @@ + load(qt_targets) + + # this builds on top of qt_common +-!internal_module:!lib_bundle:if(unix|mingw) { ++unix|mingw { + CONFIG += create_pc + QMAKE_PKGCONFIG_DESTDIR = pkgconfig + host_build: \ diff --git a/contrib/depends/patches/qt/pidlist_absolute.patch b/contrib/depends/patches/qt/pidlist_absolute.patch new file mode 100644 index 00000000000..c7928241790 --- /dev/null +++ b/contrib/depends/patches/qt/pidlist_absolute.patch @@ -0,0 +1,37 @@ +diff -dur old/qtbase/src/plugins/platforms/windows/qwindowscontext.h new/qtbase/src/plugins/platforms/windows/qwindowscontext.h +--- old/qtbase/src/plugins/platforms/windows/qwindowscontext.h ++++ new/qtbase/src/plugins/platforms/windows/qwindowscontext.h +@@ -136,10 +136,18 @@ + inline void init(); + + typedef HRESULT (WINAPI *SHCreateItemFromParsingName)(PCWSTR, IBindCtx *, const GUID&, void **); ++#if defined(Q_CC_MINGW) && (!defined(__MINGW64_VERSION_MAJOR) || __MINGW64_VERSION_MAJOR < 3) ++ typedef HRESULT (WINAPI *SHGetKnownFolderIDList)(const GUID &, DWORD, HANDLE, ITEMIDLIST **); ++#else + typedef HRESULT (WINAPI *SHGetKnownFolderIDList)(const GUID &, DWORD, HANDLE, PIDLIST_ABSOLUTE *); ++#endif + typedef HRESULT (WINAPI *SHGetStockIconInfo)(int , int , _SHSTOCKICONINFO *); + typedef HRESULT (WINAPI *SHGetImageList)(int, REFIID , void **); ++#if defined(Q_CC_MINGW) && (!defined(__MINGW64_VERSION_MAJOR) || __MINGW64_VERSION_MAJOR < 3) ++ typedef HRESULT (WINAPI *SHCreateItemFromIDList)(const ITEMIDLIST *, REFIID, void **); ++#else + typedef HRESULT (WINAPI *SHCreateItemFromIDList)(PCIDLIST_ABSOLUTE, REFIID, void **); ++#endif + + SHCreateItemFromParsingName sHCreateItemFromParsingName; + SHGetKnownFolderIDList sHGetKnownFolderIDList; +diff -dur old/qtbase/src/plugins/platforms/windows/qwindowsdialoghelpers.cpp new/qtbase/src/plugins/platforms/windows/qwindowsdialoghelpers.cpp +--- old/qtbase/src/plugins/platforms/windows/qwindowsdialoghelpers.cpp ++++ new/qtbase/src/plugins/platforms/windows/qwindowsdialoghelpers.cpp +@@ -1016,7 +1016,11 @@ + qWarning() << __FUNCTION__ << ": Invalid CLSID: " << url.path(); + return Q_NULLPTR; + } ++#if defined(Q_CC_MINGW) && (!defined(__MINGW64_VERSION_MAJOR) || __MINGW64_VERSION_MAJOR < 3) ++ ITEMIDLIST *idList; ++#else + PIDLIST_ABSOLUTE idList; ++#endif + HRESULT hr = QWindowsContext::shell32dll.sHGetKnownFolderIDList(uuid, 0, 0, &idList); + if (FAILED(hr)) { + qErrnoWarning("%s: SHGetKnownFolderIDList(%s)) failed", __FUNCTION__, qPrintable(url.toString())); diff --git a/contrib/depends/patches/qt/qfixed-coretext.patch b/contrib/depends/patches/qt/qfixed-coretext.patch new file mode 100644 index 00000000000..aa56f1e1de5 --- /dev/null +++ b/contrib/depends/patches/qt/qfixed-coretext.patch @@ -0,0 +1,34 @@ +From dbdd5f0ffbce52c8b789ed09f1aa3f1da6c02e23 Mon Sep 17 00:00:00 2001 +From: Gabriel de Dietrich +Date: Fri, 30 Mar 2018 11:58:16 -0700 +Subject: [PATCH] QCoreTextFontEngine: Fix build with Xcode 9.3 + +Apple LLVM version 9.1.0 (clang-902.0.39.1) + +Error message: + +.../qfontengine_coretext.mm:827:20: error: qualified reference to + 'QFixed' is a constructor name rather than a type in this context + return QFixed::QFixed(int(CTFontGetUnitsPerEm(ctfont))); + +Change-Id: Iebe26b3b087a16b10664208fc8851cbddb47f043 +Reviewed-by: Konstantin Ritt +--- + src/platformsupport/fontdatabases/mac/qfontengine_coretext.mm | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git old/qtbase/src/platformsupport/fontdatabases/mac/qfontengine_coretext.mm new/qtbase/src/platformsupport/fontdatabases/mac/qfontengine_coretext.mm +index 25ff69d877d..98b753eff96 100644 +--- old/qtbase/src/platformsupport/fontdatabases/mac/qfontengine_coretext.mm ++++ new/qtbase/src/platformsupport/fontdatabases/mac/qfontengine_coretext.mm +@@ -824,7 +824,7 @@ void QCoreTextFontEngine::getUnscaledGlyph(glyph_t glyph, QPainterPath *path, gl + + QFixed QCoreTextFontEngine::emSquareSize() const + { +- return QFixed::QFixed(int(CTFontGetUnitsPerEm(ctfont))); ++ return QFixed(int(CTFontGetUnitsPerEm(ctfont))); + } + + QFontEngine *QCoreTextFontEngine::cloneWithSize(qreal pixelSize) const +-- +2.16.3 \ No newline at end of file diff --git a/contrib/depends/patches/sodium/fix-whitespace.patch b/contrib/depends/patches/sodium/fix-whitespace.patch new file mode 100644 index 00000000000..efbfe4e83c4 --- /dev/null +++ b/contrib/depends/patches/sodium/fix-whitespace.patch @@ -0,0 +1,13 @@ +diff --git a/configure b/configure +index b29f769..ca008ae 100755 +--- a/configure ++++ b/configure +@@ -591,7 +591,7 @@ MAKEFLAGS= + PACKAGE_NAME='libsodium' + PACKAGE_TARNAME='libsodium' + PACKAGE_VERSION='1.0.16' +-PACKAGE_STRING='libsodium 1.0.16' ++PACKAGE_STRING='libsodium' + PACKAGE_BUGREPORT='https://github.com/jedisct1/libsodium/issues' + PACKAGE_URL='https://github.com/jedisct1/libsodium' + diff --git a/contrib/depends/patches/zeromq/9114d3957725acd34aa8b8d011585812f3369411.patch b/contrib/depends/patches/zeromq/9114d3957725acd34aa8b8d011585812f3369411.patch new file mode 100644 index 00000000000..f704b3d94fd --- /dev/null +++ b/contrib/depends/patches/zeromq/9114d3957725acd34aa8b8d011585812f3369411.patch @@ -0,0 +1,22 @@ +From 9114d3957725acd34aa8b8d011585812f3369411 Mon Sep 17 00:00:00 2001 +From: Jeroen Ooms +Date: Tue, 20 Oct 2015 13:10:38 +0200 +Subject: [PATCH] enable static libraries on mingw + +--- + configure.ac | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/configure.ac b/configure.ac +index 393505b..e92131a 100644 +--- a/configure.ac ++++ b/configure.ac +@@ -265,7 +265,7 @@ case "${host_os}" in + libzmq_dso_visibility="no" + + if test "x$enable_static" = "xyes"; then +- AC_MSG_ERROR([Building static libraries is not supported under MinGW32]) ++ CPPFLAGS="-DZMQ_STATIC" + fi + + # Set FD_SETSIZE to 1024 \ No newline at end of file diff --git a/contrib/depends/patches/zeromq/9e6745c12e0b100cd38acecc16ce7db02905e27c.patch b/contrib/depends/patches/zeromq/9e6745c12e0b100cd38acecc16ce7db02905e27c.patch new file mode 100644 index 00000000000..9aff2c179a1 --- /dev/null +++ b/contrib/depends/patches/zeromq/9e6745c12e0b100cd38acecc16ce7db02905e27c.patch @@ -0,0 +1,22 @@ +From 9e6745c12e0b100cd38acecc16ce7db02905e27c Mon Sep 17 00:00:00 2001 +From: David Millard +Date: Tue, 10 May 2016 13:53:53 -0700 +Subject: [PATCH] Fix autotools for static MinGW builds + +--- + configure.ac | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/configure.ac b/configure.ac +index 5a0fa14..def6ea7 100644 +--- a/configure.ac ++++ b/configure.ac +@@ -259,7 +259,7 @@ case "${host_os}" in + libzmq_dso_visibility="no" + + if test "x$enable_static" = "xyes"; then +- CPPFLAGS="-DZMQ_STATIC" ++ CPPFLAGS="-DZMQ_STATIC $CPPFLAGS" + fi + + # Set FD_SETSIZE to 1024 \ No newline at end of file diff --git a/contrib/depends/protobuf.mk b/contrib/depends/protobuf.mk new file mode 100644 index 00000000000..54d3fd92451 --- /dev/null +++ b/contrib/depends/protobuf.mk @@ -0,0 +1,29 @@ +package=protobuf +$(package)_version=$(native_$(package)_version) +$(package)_download_path=$(native_$(package)_download_path) +$(package)_file_name=$(native_$(package)_file_name) +$(package)_sha256_hash=$(native_$(package)_sha256_hash) +$(package)_dependencies=native_$(package) +$(package)_cxxflags=-std=c++11 + +define $(package)_set_vars + $(package)_config_opts=--disable-shared --with-protoc=$(build_prefix)/bin/protoc + $(package)_config_opts_linux=--with-pic +endef + +define $(package)_config_cmds + $($(package)_autoconf) +endef + +define $(package)_build_cmds + $(MAKE) -C src libprotobuf.la +endef + +define $(package)_stage_cmds + $(MAKE) DESTDIR=$($(package)_staging_dir) -C src install-libLTLIBRARIES install-nobase_includeHEADERS &&\ + $(MAKE) DESTDIR=$($(package)_staging_dir) install-pkgconfigDATA +endef + +define $(package)_postprocess_cmds + rm lib/libprotoc.a +endef diff --git a/contrib/depends/toolchain.cmake.in b/contrib/depends/toolchain.cmake.in new file mode 100644 index 00000000000..ee0407a5e9e --- /dev/null +++ b/contrib/depends/toolchain.cmake.in @@ -0,0 +1,111 @@ +# Set the system name, either Darwin, Linux, or Windows +SET(CMAKE_SYSTEM_NAME @depends@) +SET(CMAKE_BUILD_TYPE @release_type@) + +OPTION(STATIC "Link libraries statically" ON) +OPTION(TREZOR_DEBUG "Main trezor debugging switch" OFF) +OPTION(BUILD_TESTS "Build tests." OFF) + +SET(STATIC ON) +SET(UNBOUND_STATIC ON) +SET(ARCH "default") + +SET(BUILD_TESTS @build_tests@) +SET(TREZOR_DEBUG @build_tests@) + +# where is the target environment +SET(CMAKE_FIND_ROOT_PATH @prefix@ /usr) + +SET(ENV{PKG_CONFIG_PATH} @prefix@/lib/pkgconfig) + +SET(LRELEASE_PATH @prefix@/native/bin CACHE FILEPATH "path to lrelease" FORCE) + +SET(Readline_ROOT_DIR @prefix@) +SET(Readline_INCLUDE_DIR @prefix@/include) +SET(Termcap_LIBRARY @prefix@/lib/libncurses.a) + +SET(LIBUNWIND_INCLUDE_DIR @prefix@/include) +SET(LIBUNWIND_LIBRARIES @prefix@/lib/libunwind.a) +SET(LIBUNWIND_LIBRARY_DIRS @prefix@/lib) + +SET(LIBUSB-1.0_LIBRARY @prefix@/lib/libusb-1.0.a) +SET(LIBUDEV_LIBRARY @prefix@/lib/libudev.a) + +SET(Protobuf_FOUND 1) +SET(Protobuf_PROTOC_EXECUTABLE @prefix@/native/bin/protoc CACHE FILEPATH "Path to the native protoc") +SET(Protobuf_INCLUDE_DIR @prefix@/include CACHE PATH "Protobuf include dir") +SET(Protobuf_INCLUDE_DIRS @prefix@/include CACHE PATH "Protobuf include dir") +SET(Protobuf_LIBRARY @prefix@/lib/libprotobuf.a CACHE FILEPATH "Protobuf library") + +SET(ZMQ_INCLUDE_PATH @prefix@/include) +SET(ZMQ_LIB @prefix@/lib/libzmq.a) + +SET(BOOST_IGNORE_SYSTEM_PATHS_DEFAULT ON) +SET(BOOST_IGNORE_SYSTEM_PATH ON) +SET(BOOST_ROOT @prefix@) +SET(BOOST_LIBRARYDIR @prefix@/lib) +SET(BOOST_IGNORE_SYSTEM_PATHS_DEFAULT OFF) +SET(BOOST_NO_SYSTEM_PATHS TRUE) +SET(BOOST_USE_STATIC_LIBS TRUE) +SET(BOOST_USE_STATIC_RUNTIME TRUE) + +SET(OpenSSL_DIR @prefix@/lib) +SET(ARCHITECTURE @arch@) + +# for libraries and headers in the target directories +set (CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER) # Find programs on host +set (CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY) # Find libs in target +set (CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY) # Find includes in target + +set(CMAKE_SYSTEM_PROCESSOR ${CMAKE_HOST_SYSTEM_PROCESSOR} CACHE STRING "" FORCE) + +# specify the cross compiler to be used. Darwin uses clang provided by the SDK. +if(CMAKE_SYSTEM_NAME STREQUAL "Darwin") + SET(CMAKE_C_COMPILER @prefix@/native/bin/clang) + SET(CMAKE_C_COMPILER_TARGET x86_64-apple-darwin11) + SET(CMAKE_CXX_COMPILER @prefix@/native/bin/clang++ -stdlib=libc++) + SET(CMAKE_CXX_COMPILER_TARGET x86_64-apple-darwin11) + SET(_CMAKE_TOOLCHAIN_PREFIX x86_64-apple-darwin11-) + SET(APPLE True) + SET(BUILD_TAG "mac-x64") + SET(BUILD_64 ON) + SET(BREW OFF) + SET(PORT OFF) + SET(CMAKE_OSX_SYSROOT "@sdk@/MacOSX10.11.sdk/") + SET(CMAKE_OSX_DEPLOYMENT_TARGET "10.08") + SET(CMAKE_CXX_STANDARD 11) + SET(CMAKE_OSX_ARCHITECTURES "x86_64") + SET(LLVM_ENABLE_PIC OFF) + SET(LLVM_ENABLE_PIE OFF) +else() + SET(CMAKE_C_COMPILER @CC@) + SET(CMAKE_CXX_COMPILER @CXX@) +endif() + +if(ARCHITECTURE STREQUAL "arm") + set(ARCH "armv7-a") + set(ARM ON) + set(ARM_ID "armv7-a") + set(BUILD_64 OFF) + set(CMAKE_BUILD_TYPE release) + set(BUILD_TAG "linux-armv7") + set(ARM7) +elseif(ARCHITECTURE STREQUAL "aarch64") + set(ARCH "armv8-a") + set(ARM ON) + set(ARM_ID "armv8-a") + set(BUILD_TAG "linux-armv8") + set(BUILD_64 ON) +endif() + +if(ARCHITECTURE STREQUAL "i686" AND CMAKE_SYSTEM_NAME STREQUAL "Linux") + SET(LINUX_32 ON) + SET(ARCH_ID "i386") +endif() + +if(ARCHITECTURE STREQUAL "x86_64" AND CMAKE_SYSTEM_NAME STREQUAL "Linux") + SET(ARCH_ID "x86_64") +endif() + +#Create a new global cmake flag that indicates building with depends +set (DEPENDS true) diff --git a/contrib/epee/CMakeLists.txt b/contrib/epee/CMakeLists.txt index 035b24b36e0..a2c63630452 100644 --- a/contrib/epee/CMakeLists.txt +++ b/contrib/epee/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2014-2018, The Monero Project +# Copyright (c) 2014-2019, The Monero Project # # All rights reserved. # diff --git a/contrib/epee/demo/iface/transport_defs.h b/contrib/epee/demo/iface/transport_defs.h index 746694aa3a4..8638b5db9c1 100644 --- a/contrib/epee/demo/iface/transport_defs.h +++ b/contrib/epee/demo/iface/transport_defs.h @@ -81,7 +81,7 @@ namespace demo { const static int ID = 1000; - struct request + struct request_t { std::string example_string_data; some_test_data sub; @@ -91,9 +91,10 @@ namespace demo KV_SERIALIZE(sub) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init request; - struct response + struct response_t { bool m_success; std::list subs; @@ -104,6 +105,7 @@ namespace demo END_KV_SERIALIZE_MAP() }; }; + typedef epee::misc_utils::struct_init response; @@ -111,7 +113,7 @@ namespace demo { const static int ID = 1001; - struct request + struct request_t { std::string example_string_data2; @@ -119,8 +121,9 @@ namespace demo KV_SERIALIZE(example_string_data2) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init request; - struct response + struct response_t { bool m_success; @@ -129,6 +132,7 @@ namespace demo KV_SERIALIZE(m_success) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init response; }; diff --git a/contrib/epee/include/console_handler.h b/contrib/epee/include/console_handler.h index 0e22a971c27..e07e16d91ad 100644 --- a/contrib/epee/include/console_handler.h +++ b/contrib/epee/include/console_handler.h @@ -63,7 +63,8 @@ namespace epee ~async_stdin_reader() { - stop(); + try { stop(); } + catch (...) { /* ignore */ } } #ifdef HAVE_READLINE @@ -351,8 +352,12 @@ namespace epee std::string command; bool get_line_ret = m_stdin_reader.get_line(command); - if (!m_running || m_stdin_reader.eos()) + if (!m_running) + break; + if (m_stdin_reader.eos()) { + MGINFO("EOF on stdin, exiting"); + std::cout << std::endl; break; } if (!get_line_ret) diff --git a/contrib/epee/include/file_io_utils.h b/contrib/epee/include/file_io_utils.h index 1966106743f..25f8c648b69 100644 --- a/contrib/epee/include/file_io_utils.h +++ b/contrib/epee/include/file_io_utils.h @@ -28,11 +28,12 @@ #ifndef _FILE_IO_UTILS_H_ #define _FILE_IO_UTILS_H_ -#include +#include #include #include #ifdef WIN32 #include +#include "string_tools.h" #endif // On Windows there is a problem with non-ASCII characters in path and file names @@ -72,11 +73,9 @@ namespace file_io_utils bool save_string_to_file(const std::string& path_to_file, const std::string& str) { #ifdef WIN32 - WCHAR wide_path[1000]; - int chars = MultiByteToWideChar(CP_UTF8, 0, path_to_file.c_str(), path_to_file.size() + 1, wide_path, 1000); - if (chars == 0) - return false; - HANDLE file_handle = CreateFileW(wide_path, GENERIC_WRITE, 0, NULL, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL); + std::wstring wide_path; + try { wide_path = string_tools::utf8_to_utf16(path_to_file); } catch (...) { return false; } + HANDLE file_handle = CreateFileW(wide_path.c_str(), GENERIC_WRITE, 0, NULL, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL); if (file_handle == INVALID_HANDLE_VALUE) return false; DWORD bytes_written; @@ -128,18 +127,16 @@ namespace file_io_utils inline - bool load_file_to_string(const std::string& path_to_file, std::string& target_str) + bool load_file_to_string(const std::string& path_to_file, std::string& target_str, size_t max_size = 1000000000) { #ifdef WIN32 - WCHAR wide_path[1000]; - int chars = MultiByteToWideChar(CP_UTF8, 0, path_to_file.c_str(), path_to_file.size() + 1, wide_path, 1000); - if (chars == 0) - return false; - HANDLE file_handle = CreateFileW(wide_path, GENERIC_READ, 0, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL); + std::wstring wide_path; + try { wide_path = string_tools::utf8_to_utf16(path_to_file); } catch (...) { return false; } + HANDLE file_handle = CreateFileW(wide_path.c_str(), GENERIC_READ, 0, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL); if (file_handle == INVALID_HANDLE_VALUE) return false; DWORD file_size = GetFileSize(file_handle, NULL); - if ((file_size == INVALID_FILE_SIZE) || (file_size > 1000000000)) { + if ((file_size == INVALID_FILE_SIZE) || (uint64_t)file_size > (uint64_t)max_size) { CloseHandle(file_handle); return false; } @@ -159,7 +156,7 @@ namespace file_io_utils std::ifstream::pos_type file_size = fstream.tellg(); - if(file_size > 1000000000) + if((uint64_t)file_size > (uint64_t)max_size) // ensure a large domain for comparison, and negative -> too large return false;//don't go crazy size_t file_size_t = static_cast(file_size); @@ -202,11 +199,9 @@ namespace file_io_utils bool get_file_size(const std::string& path_to_file, uint64_t &size) { #ifdef WIN32 - WCHAR wide_path[1000]; - int chars = MultiByteToWideChar(CP_UTF8, 0, path_to_file.c_str(), path_to_file.size() + 1, wide_path, 1000); - if (chars == 0) - return false; - HANDLE file_handle = CreateFileW(wide_path, GENERIC_READ, 0, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL); + std::wstring wide_path; + try { wide_path = string_tools::utf8_to_utf16(path_to_file); } catch (...) { return false; } + HANDLE file_handle = CreateFileW(wide_path.c_str(), GENERIC_READ, 0, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL); if (file_handle == INVALID_HANDLE_VALUE) return false; LARGE_INTEGER file_size; diff --git a/contrib/epee/include/fnv1.h b/contrib/epee/include/fnv1.h new file mode 100644 index 00000000000..c04389bca5c --- /dev/null +++ b/contrib/epee/include/fnv1.h @@ -0,0 +1,45 @@ +// Copyright (c) 2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +namespace epee +{ + +namespace fnv +{ + inline uint64_t FNV1a(const char *ptr, size_t sz) + { + uint64_t h = 0xcbf29ce484222325; + for (size_t i = 0; i < sz; ++i) + h = (h ^ *(const uint8_t*)ptr++) * 0x100000001b3; + return h; + } +} + +} diff --git a/contrib/epee/include/hex.h b/contrib/epee/include/hex.h index e960da1d2c4..6e720f128b6 100644 --- a/contrib/epee/include/hex.h +++ b/contrib/epee/include/hex.h @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2018, The Monero Project +// Copyright (c) 2017-2019, The Monero Project // // All rights reserved. // @@ -32,7 +32,9 @@ #include #include #include +#include +#include "wipeable_string.h" #include "span.h" namespace epee @@ -41,6 +43,9 @@ namespace epee { //! \return A std::string containing hex of `src`. static std::string string(const span src); + //! \return A epee::wipeable_string containing hex of `src`. + static epee::wipeable_string wipeable_string(const span src); + template static epee::wipeable_string wipeable_string(const T &pod) { return wipeable_string(span((const uint8_t*)&pod, sizeof(pod))); } //! \return An array containing hex of `src`. template @@ -59,7 +64,15 @@ namespace epee static void formatted(std::ostream& out, const span src); private: + template T static convert(const span src); + //! Write `src` bytes as hex to `out`. `out` must be twice the length static void buffer_unchecked(char* out, const span src) noexcept; }; + + struct from_hex + { + //! \return An std::vector of unsigned integers from the `src` + static std::vector vector(boost::string_ref src); + }; } diff --git a/src/common/int-util.h b/contrib/epee/include/int-util.h similarity index 99% rename from src/common/int-util.h rename to contrib/epee/include/int-util.h index 3bcc085e2ba..0ed6505ffa4 100644 --- a/src/common/int-util.h +++ b/contrib/epee/include/int-util.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/contrib/epee/include/math_helper.h b/contrib/epee/include/math_helper.h index ef839f609a6..ddc1f7f4b8e 100644 --- a/contrib/epee/include/math_helper.h +++ b/contrib/epee/include/math_helper.h @@ -32,7 +32,7 @@ #include #include -#include +#include #include #include @@ -230,35 +230,57 @@ namespace math_helper } } - template - class once_a_time_seconds + template + class once_a_time { + uint64_t get_time() const + { +#ifdef _WIN32 + FILETIME fileTime; + GetSystemTimeAsFileTime(&fileTime); + unsigned __int64 present = 0; + present |= fileTime.dwHighDateTime; + present = present << 32; + present |= fileTime.dwLowDateTime; + present /= 10; // mic-sec + return present; +#else + struct timeval tv; + gettimeofday(&tv, NULL); + return tv.tv_sec * 1000000 + tv.tv_usec; +#endif + } + public: - once_a_time_seconds():m_interval(default_interval) + once_a_time():m_interval(default_interval * scale) { m_last_worked_time = 0; if(!start_immediate) - time(&m_last_worked_time); + m_last_worked_time = get_time(); } template bool do_call(functor_t functr) { - time_t current_time = 0; - time(¤t_time); + uint64_t current_time = get_time(); if(current_time - m_last_worked_time > m_interval) { bool res = functr(); - time(&m_last_worked_time); + m_last_worked_time = get_time(); return res; } return true; } private: - time_t m_last_worked_time; - time_t m_interval; + uint64_t m_last_worked_time; + uint64_t m_interval; }; + + template + class once_a_time_seconds: public once_a_time<1000000, default_interval, start_immediate> {}; + template + class once_a_time_milliseconds: public once_a_time<1000, default_interval, start_immediate> {}; } } diff --git a/contrib/epee/include/memwipe.h b/contrib/epee/include/memwipe.h index 0d8f491b751..d586608cbfa 100644 --- a/contrib/epee/include/memwipe.h +++ b/contrib/epee/include/memwipe.h @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2018, The Monero Project +// Copyright (c) 2017-2019, The Monero Project // // All rights reserved. // diff --git a/contrib/epee/include/misc_language.h b/contrib/epee/include/misc_language.h index d5157365c8f..5f72021508a 100644 --- a/contrib/epee/include/misc_language.h +++ b/contrib/epee/include/misc_language.h @@ -147,7 +147,8 @@ namespace misc_utils {} ~call_befor_die() { - m_func(); + try { m_func(); } + catch (...) { /* ignore */ } } }; @@ -158,5 +159,10 @@ namespace misc_utils return slc; } + template struct struct_init: T + { + struct_init(): T{} {} + }; + } } diff --git a/contrib/epee/include/misc_log_ex.h b/contrib/epee/include/misc_log_ex.h index 35ec0950bc8..602b6a3711b 100644 --- a/contrib/epee/include/misc_log_ex.h +++ b/contrib/epee/include/misc_log_ex.h @@ -32,17 +32,27 @@ #include "easylogging++.h" +#undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "default" + #define MAX_LOG_FILE_SIZE 104850000 // 100 MB - 7600 bytes +#define MAX_LOG_FILES 50 + +#define MCLOG_TYPE(level, cat, type, x) do { \ + if (ELPP->vRegistry()->allowed(level, cat)) { \ + el::base::Writer(level, __FILE__, __LINE__, ELPP_FUNC, type).construct(cat) << x; \ + } \ + } while (0) + +#define MCLOG(level, cat, x) MCLOG_TYPE(level, cat, el::base::DispatchAction::NormalLog, x) +#define MCLOG_FILE(level, cat, x) MCLOG_TYPE(level, cat, el::base::DispatchAction::FileOnlyLog, x) -#define MCFATAL(cat,x) CLOG(FATAL,cat) << x -#define MCERROR(cat,x) CLOG(ERROR,cat) << x -#define MCWARNING(cat,x) CLOG(WARNING,cat) << x -#define MCINFO(cat,x) CLOG(INFO,cat) << x -#define MCDEBUG(cat,x) CLOG(DEBUG,cat) << x -#define MCTRACE(cat,x) CLOG(TRACE,cat) << x -#define MCLOG(level,cat,x) ELPP_WRITE_LOG(el::base::Writer, level, el::base::DispatchAction::NormalLog, cat) << x -#define MCLOG_FILE(level,cat,x) ELPP_WRITE_LOG(el::base::Writer, level, el::base::DispatchAction::FileOnlyLog, cat) << x +#define MCFATAL(cat,x) MCLOG(el::Level::Fatal,cat, x) +#define MCERROR(cat,x) MCLOG(el::Level::Error,cat, x) +#define MCWARNING(cat,x) MCLOG(el::Level::Warning,cat, x) +#define MCINFO(cat,x) MCLOG(el::Level::Info,cat, x) +#define MCDEBUG(cat,x) MCLOG(el::Level::Debug,cat, x) +#define MCTRACE(cat,x) MCLOG(el::Level::Trace,cat, x) #define MCLOG_COLOR(level,cat,color,x) MCLOG(level,cat,"\033[1;" color "m" << x << "\033[0m") #define MCLOG_RED(level,cat,x) MCLOG_COLOR(level,cat,"31",x) @@ -75,6 +85,16 @@ #define MGINFO_MAGENTA(x) MCLOG_MAGENTA(el::Level::Info, "global",x) #define MGINFO_CYAN(x) MCLOG_CYAN(el::Level::Info, "global",x) +#define IFLOG(level, cat, type, init, x) \ + do { \ + if (ELPP->vRegistry()->allowed(level, cat)) { \ + init; \ + el::base::Writer(level, __FILE__, __LINE__, ELPP_FUNC, type).construct(cat) << x; \ + } \ + } while(0) +#define MIDEBUG(init, x) IFLOG(el::Level::Debug, MONERO_DEFAULT_LOG_CATEGORY, el::base::DispatchAction::NormalLog, init, x) + + #define LOG_ERROR(x) MERROR(x) #define LOG_PRINT_L0(x) MWARNING(x) #define LOG_PRINT_L1(x) MINFO(x) @@ -105,7 +125,7 @@ #endif std::string mlog_get_default_log_path(const char *default_filename); -void mlog_configure(const std::string &filename_base, bool console, const std::size_t max_log_file_size = MAX_LOG_FILE_SIZE); +void mlog_configure(const std::string &filename_base, bool console, const std::size_t max_log_file_size = MAX_LOG_FILE_SIZE, const std::size_t max_log_files = MAX_LOG_FILES); void mlog_set_categories(const char *categories); std::string mlog_get_categories(); void mlog_set_log_level(int level); diff --git a/contrib/epee/include/misc_os_dependent.h b/contrib/epee/include/misc_os_dependent.h index ffe575501eb..5fffde8d5a0 100644 --- a/contrib/epee/include/misc_os_dependent.h +++ b/contrib/epee/include/misc_os_dependent.h @@ -24,7 +24,7 @@ // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // #ifdef _WIN32 -#include +#include #endif #ifdef WIN32 @@ -122,6 +122,15 @@ namespace misc_utils return boost::lexical_cast(GetCurrentThreadId()); #elif defined(__GNUC__) return boost::lexical_cast(pthread_self()); +#endif + } + + inline bool get_gmt_time(time_t t, struct tm &tm) + { +#ifdef _WIN32 + return gmtime_s(&tm, &t); +#else + return gmtime_r(&t, &tm); #endif } } diff --git a/contrib/epee/include/mlocker.h b/contrib/epee/include/mlocker.h new file mode 100644 index 00000000000..a6d94b3d289 --- /dev/null +++ b/contrib/epee/include/mlocker.h @@ -0,0 +1,87 @@ +// Copyright (c) 2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +#include +#include + +namespace epee +{ + class mlocker + { + public: + mlocker(void *ptr, size_t len); + ~mlocker(); + + static size_t get_page_size(); + static size_t get_num_locked_pages(); + static size_t get_num_locked_objects(); + + static void lock(void *ptr, size_t len); + static void unlock(void *ptr, size_t len); + + private: + static size_t page_size; + static size_t num_locked_objects; + + static boost::mutex &mutex(); + static std::map &map(); + static void lock_page(size_t page); + static void unlock_page(size_t page); + + void *ptr; + size_t len; + }; + + /// Locks memory while in scope + /// + /// Primarily useful for making sure that private keys don't get swapped out + // to disk + template + struct mlocked : public T { + using type = T; + + mlocked(): T() { mlocker::lock(this, sizeof(T)); } + mlocked(const T &t): T(t) { mlocker::lock(this, sizeof(T)); } + mlocked(const mlocked &mt): T(mt) { mlocker::lock(this, sizeof(T)); } + mlocked(const T &&t): T(t) { mlocker::lock(this, sizeof(T)); } + mlocked(const mlocked &&mt): T(mt) { mlocker::lock(this, sizeof(T)); } + mlocked &operator=(const mlocked &mt) { T::operator=(mt); return *this; } + ~mlocked() { try { mlocker::unlock(this, sizeof(T)); } catch (...) { /* do not propagate */ } } + }; + + template + T& unwrap(mlocked& src) { return src; } + + template + const T& unwrap(mlocked const& src) { return src; } + + template + using mlocked_arr = mlocked>; +} diff --git a/contrib/epee/include/net/abstract_tcp_server2.h b/contrib/epee/include/net/abstract_tcp_server2.h index 2f7325be549..374a28a2e3f 100644 --- a/contrib/epee/include/net/abstract_tcp_server2.h +++ b/contrib/epee/include/net/abstract_tcp_server2.h @@ -36,16 +36,17 @@ #define _ABSTRACT_TCP_SERVER2_H_ -#include #include #include #include #include #include +#include #include #include #include +#include #include #include #include @@ -88,22 +89,39 @@ namespace net_utils { public: typedef typename t_protocol_handler::connection_context t_connection_context; + + struct shared_state : connection_basic_shared_state + { + shared_state() + : connection_basic_shared_state(), pfilter(nullptr), config(), stop_signal_sent(false) + {} + + i_connection_filter* pfilter; + typename t_protocol_handler::config_type config; + bool stop_signal_sent; + }; + /// Construct a connection with the given io_service. - explicit connection( boost::asio::io_service& io_service, - typename t_protocol_handler::config_type& config, - std::atomic &ref_sock_count, // the ++/-- counter - std::atomic &sock_number, // the only increasing ++ number generator - i_connection_filter * &pfilter - ,t_connection_type connection_type); + boost::shared_ptr state, + t_connection_type connection_type, + epee::net_utils::ssl_support_t ssl_support); + + explicit connection( boost::asio::ip::tcp::socket&& sock, + boost::shared_ptr state, + t_connection_type connection_type, + epee::net_utils::ssl_support_t ssl_support); + + virtual ~connection() noexcept(false); - /// Get the socket associated with the connection. - boost::asio::ip::tcp::socket& socket(); /// Start the first asynchronous operation for the connection. bool start(bool is_income, bool is_multithreaded); + // `real_remote` is the actual endpoint (if connection is to proxy, etc.) + bool start(bool is_income, bool is_multithreaded, network_address real_remote); + void get_context(t_connection_context& context_){context_ = context;} void call_back_starter(); @@ -119,6 +137,7 @@ namespace net_utils //----------------- i_service_endpoint --------------------- virtual bool do_send(const void* ptr, size_t cb); ///< (see do_send from i_service_endpoint) virtual bool do_send_chunk(const void* ptr, size_t cb); ///< will send (or queue) a part of data + virtual bool send_done(); virtual bool close(); virtual bool call_run_once_service_io(); virtual bool request_callback(); @@ -128,6 +147,10 @@ namespace net_utils //------------------------------------------------------ boost::shared_ptr > safe_shared_from_this(); bool shutdown(); + /// Handle completion of a receive operation. + void handle_receive(const boost::system::error_code& e, + std::size_t bytes_transferred); + /// Handle completion of a read operation. void handle_read(const boost::system::error_code& e, std::size_t bytes_transferred); @@ -137,23 +160,27 @@ namespace net_utils /// reset connection timeout timer and callback void reset_timer(boost::posix_time::milliseconds ms, bool add); - boost::posix_time::milliseconds get_default_time() const; - boost::posix_time::milliseconds get_timeout_from_bytes_read(size_t bytes) const; + boost::posix_time::milliseconds get_default_timeout(); + boost::posix_time::milliseconds get_timeout_from_bytes_read(size_t bytes); + + /// host connection count tracking + unsigned int host_count(const std::string &host, int delta = 0); /// Buffer for incoming data. boost::array buffer_; - //boost::array buffer_; + size_t buffer_ssl_init_fill; t_connection_context context; - i_connection_filter* &m_pfilter; // TODO what do they mean about wait on destructor?? --rfree : //this should be the last one, because it could be wait on destructor, while other activities possible on other threads t_protocol_handler m_protocol_handler; //typename t_protocol_handler::config_type m_dummy_config; - std::list > > m_self_refs; // add_ref/release support + size_t m_reference_count = 0; // reference count managed through add_ref/release support + boost::shared_ptr > m_self_ref; // the reference to hold critical_section m_self_refs_lock; critical_section m_chunking_lock; // held while we add small chunks of the big do_send() to small do_send_chunk() + critical_section m_shutdown_lock; // held while shutting down t_connection_type m_connection_type; @@ -165,6 +192,8 @@ namespace net_utils boost::asio::deadline_timer m_timer; bool m_local; + bool m_ready_to_close; + std::string m_host; public: void setRpcStation(); @@ -178,6 +207,13 @@ namespace net_utils class boosted_tcp_server : private boost::noncopyable { + enum try_connect_result_t + { + CONNECT_SUCCESS, + CONNECT_FAILURE, + CONNECT_NO_SSL, + }; + public: typedef boost::shared_ptr > connection_ptr; typedef typename t_protocol_handler::connection_context t_connection_context; @@ -191,8 +227,8 @@ namespace net_utils std::map server_type_map; void create_server_type_map(); - bool init_server(uint32_t port, const std::string address = "0.0.0.0"); - bool init_server(const std::string port, const std::string& address = "0.0.0.0"); + bool init_server(uint32_t port, const std::string address = "0.0.0.0", ssl_options_t ssl_options = ssl_support_t::e_ssl_support_autodetect); + bool init_server(const std::string port, const std::string& address = "0.0.0.0", ssl_options_t ssl_options = ssl_support_t::e_ssl_support_autodetect); /// Run the server's io_service loop. bool run_server(size_t threads_count, bool wait = true, const boost::thread::attributes& attrs = boost::thread::attributes()); @@ -203,7 +239,9 @@ namespace net_utils /// Stop the server. void send_stop_signal(); - bool is_stop_signal_sent(); + bool is_stop_signal_sent() const noexcept { return m_stop_signal_sent; }; + + const std::atomic& get_stop_signal() const noexcept { return m_stop_signal_sent; } void set_threads_prefix(const std::string& prefix_name); @@ -213,17 +251,29 @@ namespace net_utils void set_connection_filter(i_connection_filter* pfilter); - bool connect(const std::string& adr, const std::string& port, uint32_t conn_timeot, t_connection_context& cn, const std::string& bind_ip = "0.0.0.0"); + void set_default_remote(epee::net_utils::network_address remote) + { + default_remote = std::move(remote); + } + + bool add_connection(t_connection_context& out, boost::asio::ip::tcp::socket&& sock, network_address real_remote, epee::net_utils::ssl_support_t ssl_support = epee::net_utils::ssl_support_t::e_ssl_support_autodetect); + try_connect_result_t try_connect(connection_ptr new_connection_l, const std::string& adr, const std::string& port, boost::asio::ip::tcp::socket &sock_, const boost::asio::ip::tcp::endpoint &remote_endpoint, const std::string &bind_ip, uint32_t conn_timeout, epee::net_utils::ssl_support_t ssl_support); + bool connect(const std::string& adr, const std::string& port, uint32_t conn_timeot, t_connection_context& cn, const std::string& bind_ip = "0.0.0.0", epee::net_utils::ssl_support_t ssl_support = epee::net_utils::ssl_support_t::e_ssl_support_autodetect); template - bool connect_async(const std::string& adr, const std::string& port, uint32_t conn_timeot, const t_callback &cb, const std::string& bind_ip = "0.0.0.0"); + bool connect_async(const std::string& adr, const std::string& port, uint32_t conn_timeot, const t_callback &cb, const std::string& bind_ip = "0.0.0.0", epee::net_utils::ssl_support_t ssl_support = epee::net_utils::ssl_support_t::e_ssl_support_autodetect); - typename t_protocol_handler::config_type& get_config_object(){return m_config;} + typename t_protocol_handler::config_type& get_config_object() + { + assert(m_state != nullptr); // always set in constructor + return m_state->config; + } int get_binded_port(){return m_port;} long get_connections_count() const { - auto connections_count = (m_sock_count > 0) ? (m_sock_count - 1) : 0; // Socket count minus listening socket + assert(m_state != nullptr); // always set in constructor + auto connections_count = m_state->sock_count > 0 ? (m_state->sock_count - 1) : 0; // Socket count minus listening socket return connections_count; } @@ -239,7 +289,6 @@ namespace net_utils m_timer(io_serice) {} boost::asio::deadline_timer m_timer; - uint64_t m_period; }; template @@ -255,25 +304,27 @@ namespace net_utils { return m_handler(); } + uint64_t m_period; }; template bool add_idle_handler(t_handler t_callback, uint64_t timeout_ms) { - boost::shared_ptr ptr(new idle_callback_conext(io_service_, t_callback, timeout_ms)); + boost::shared_ptr> ptr(new idle_callback_conext(io_service_, t_callback, timeout_ms)); //needed call handler here ?... ptr->m_timer.expires_from_now(boost::posix_time::milliseconds(ptr->m_period)); - ptr->m_timer.async_wait(boost::bind(&boosted_tcp_server::global_timer_handler, this, ptr)); + ptr->m_timer.async_wait(boost::bind(&boosted_tcp_server::global_timer_handler, this, ptr)); return true; } - bool global_timer_handler(/*const boost::system::error_code& err, */boost::shared_ptr ptr) + template + bool global_timer_handler(/*const boost::system::error_code& err, */boost::shared_ptr> ptr) { //if handler return false - he don't want to be called anymore if(!ptr->call_handler()) return true; ptr->m_timer.expires_from_now(boost::posix_time::milliseconds(ptr->m_period)); - ptr->m_timer.async_wait(boost::bind(&boosted_tcp_server::global_timer_handler, this, ptr)); + ptr->m_timer.async_wait(boost::bind(&boosted_tcp_server::global_timer_handler, this, ptr)); return true; } @@ -284,9 +335,6 @@ namespace net_utils return true; } - protected: - typename t_protocol_handler::config_type m_config; - private: /// Run the server's io_service loop. bool worker_thread(); @@ -295,21 +343,30 @@ namespace net_utils bool is_thread_worker(); + const boost::shared_ptr::shared_state> m_state; + /// The io_service used to perform asynchronous operations. - std::unique_ptr m_io_service_local_instance; + struct worker + { + worker() + : io_service(), work(io_service) + {} + + boost::asio::io_service io_service; + boost::asio::io_service::work work; + }; + std::unique_ptr m_io_service_local_instance; boost::asio::io_service& io_service_; /// Acceptor used to listen for incoming connections. boost::asio::ip::tcp::acceptor acceptor_; + epee::net_utils::network_address default_remote; std::atomic m_stop_signal_sent; uint32_t m_port; - std::atomic m_sock_count; - std::atomic m_sock_number; std::string m_address; std::string m_thread_name_prefix; //TODO: change to enum server_type, now used size_t m_threads_count; - i_connection_filter* m_pfilter; std::vector > m_threads; boost::thread::id m_main_thread_id; critical_section m_threads_lock; @@ -322,7 +379,6 @@ namespace net_utils boost::mutex connections_mutex; std::set connections_; - }; // class <>boosted_tcp_server diff --git a/contrib/epee/include/net/abstract_tcp_server2.inl b/contrib/epee/include/net/abstract_tcp_server2.inl index 91a94c21ebf..fa5858b9f2f 100644 --- a/contrib/epee/include/net/abstract_tcp_server2.inl +++ b/contrib/epee/include/net/abstract_tcp_server2.inl @@ -32,17 +32,17 @@ -//#include "net_utils_base.h" -#include +#include #include -#include #include #include #include #include #include // TODO -#include // TODO #include // TODO +#include +#include "warnings.h" +#include "string_tools.h" #include "misc_language.h" #include "net/local_ip.h" #include "pragma_comp_defs.h" @@ -51,20 +51,29 @@ #include #include -#include "../../../../src/cryptonote_core/cryptonote_core.h" // e.g. for the send_stop_signal() - #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "net" -#define DEFAULT_TIMEOUT_MS_LOCAL boost::posix_time::milliseconds(120000) // 2 minutes -#define DEFAULT_TIMEOUT_MS_REMOTE boost::posix_time::milliseconds(10000) // 10 seconds +#define AGGRESSIVE_TIMEOUT_THRESHOLD 120 // sockets +#define NEW_CONNECTION_TIMEOUT_LOCAL 1200000 // 2 minutes +#define NEW_CONNECTION_TIMEOUT_REMOTE 10000 // 10 seconds +#define DEFAULT_TIMEOUT_MS_LOCAL 1800000 // 30 minutes +#define DEFAULT_TIMEOUT_MS_REMOTE 300000 // 5 minutes #define TIMEOUT_EXTRA_MS_PER_BYTE 0.2 + PRAGMA_WARNING_PUSH namespace epee { namespace net_utils { + template + T& check_and_get(boost::shared_ptr& ptr) + { + CHECK_AND_ASSERT_THROW_MES(bool(ptr), "shared_state cannot be null"); + return *ptr; + } + /************************************************************************/ /* */ /************************************************************************/ @@ -72,24 +81,34 @@ PRAGMA_WARNING_DISABLE_VS(4355) template connection::connection( boost::asio::io_service& io_service, - typename t_protocol_handler::config_type& config, - std::atomic &ref_sock_count, // the ++/-- counter - std::atomic &sock_number, // the only increasing ++ number generator - i_connection_filter* &pfilter - ,t_connection_type connection_type + boost::shared_ptr state, + t_connection_type connection_type, + ssl_support_t ssl_support + ) + : connection(boost::asio::ip::tcp::socket{io_service}, std::move(state), connection_type, ssl_support) + { + } + + template + connection::connection( boost::asio::ip::tcp::socket&& sock, + boost::shared_ptr state, + t_connection_type connection_type, + ssl_support_t ssl_support ) : - connection_basic(io_service, ref_sock_count, sock_number), - m_protocol_handler(this, config, context), - m_pfilter( pfilter ), + connection_basic(std::move(sock), state, ssl_support), + m_protocol_handler(this, check_and_get(state).config, context), + buffer_ssl_init_fill(0), m_connection_type( connection_type ), m_throttle_speed_in("speed_in", "throttle_speed_in"), m_throttle_speed_out("speed_out", "throttle_speed_out"), - m_timer(io_service), - m_local(false) + m_timer(GET_IO_SERVICE(socket_)), + m_local(false), + m_ready_to_close(false) { MDEBUG("test, connection constructor set m_connection_type="< @@ -97,17 +116,11 @@ PRAGMA_WARNING_DISABLE_VS(4355) { if(!m_was_shutdown) { - _dbg3("[sock " << socket_.native_handle() << "] Socket destroyed without shutdown."); + _dbg3("[sock " << socket().native_handle() << "] Socket destroyed without shutdown."); shutdown(); } - _dbg3("[sock " << socket_.native_handle() << "] Socket destroyed"); - } - //--------------------------------------------------------------------------------- - template - boost::asio::ip::tcp::socket& connection::socket() - { - return socket_; + _dbg3("[sock " << socket().native_handle() << "] Socket destroyed"); } //--------------------------------------------------------------------------------- template @@ -129,51 +142,73 @@ PRAGMA_WARNING_DISABLE_VS(4355) { TRY_ENTRY(); + boost::system::error_code ec; + auto remote_ep = socket().remote_endpoint(ec); + CHECK_AND_NO_ASSERT_MES(!ec, false, "Failed to get remote endpoint: " << ec.message() << ':' << ec.value()); + CHECK_AND_NO_ASSERT_MES(remote_ep.address().is_v4(), false, "IPv6 not supported here"); + + const unsigned long ip_{boost::asio::detail::socket_ops::host_to_network_long(remote_ep.address().to_v4().to_ulong())}; + return start(is_income, is_multithreaded, ipv4_network_address{uint32_t(ip_), remote_ep.port()}); + CATCH_ENTRY_L0("connection::start()", false); + } + //--------------------------------------------------------------------------------- + template + bool connection::start(bool is_income, bool is_multithreaded, network_address real_remote) + { + TRY_ENTRY(); + // Use safe_shared_from_this, because of this is public method and it can be called on the object being deleted auto self = safe_shared_from_this(); if(!self) return false; m_is_multithreaded = is_multithreaded; + m_local = real_remote.is_loopback() || real_remote.is_local(); - boost::system::error_code ec; - auto remote_ep = socket_.remote_endpoint(ec); - CHECK_AND_NO_ASSERT_MES(!ec, false, "Failed to get remote endpoint: " << ec.message() << ':' << ec.value()); - CHECK_AND_NO_ASSERT_MES(remote_ep.address().is_v4(), false, "IPv6 not supported here"); - - auto local_ep = socket_.local_endpoint(ec); - CHECK_AND_NO_ASSERT_MES(!ec, false, "Failed to get local endpoint: " << ec.message() << ':' << ec.value()); + // create a random uuid, we don't need crypto strength here + const boost::uuids::uuid random_uuid = boost::uuids::random_generator()(); - context = boost::value_initialized(); - const unsigned long ip_{boost::asio::detail::socket_ops::host_to_network_long(remote_ep.address().to_v4().to_ulong())}; - m_local = epee::net_utils::is_ip_loopback(ip_); + context = t_connection_context{}; + bool ssl = m_ssl_support == epee::net_utils::ssl_support_t::e_ssl_support_enabled; + context.set_details(random_uuid, std::move(real_remote), is_income, ssl); - // create a random uuid - boost::uuids::uuid random_uuid; - // that stuff turns out to be included, even though it's from src... Taking advantage - random_uuid = crypto::rand(); + boost::system::error_code ec; + auto local_ep = socket().local_endpoint(ec); + CHECK_AND_NO_ASSERT_MES(!ec, false, "Failed to get local endpoint: " << ec.message() << ':' << ec.value()); - context.set_details(random_uuid, epee::net_utils::ipv4_network_address(ip_, remote_ep.port()), is_income); _dbg3("[sock " << socket_.native_handle() << "] new connection from " << print_connection_context_short(context) << " to " << local_ep.address().to_string() << ':' << local_ep.port() << - ", total sockets objects " << m_ref_sock_count); + ", total sockets objects " << get_state().sock_count); - if(m_pfilter && !m_pfilter->is_remote_host_allowed(context.m_remote_address)) + if(static_cast(get_state()).pfilter && !static_cast(get_state()).pfilter->is_remote_host_allowed(context.m_remote_address)) { - _dbg2("[sock " << socket_.native_handle() << "] host denied " << context.m_remote_address.host_str() << ", shutdowning connection"); + _dbg2("[sock " << socket().native_handle() << "] host denied " << context.m_remote_address.host_str() << ", shutdowning connection"); close(); return false; } - m_protocol_handler.after_init_connection(); + m_host = context.m_remote_address.host_str(); + try { host_count(m_host, 1); } catch(...) { /* ignore */ } - reset_timer(get_default_time(), false); + m_protocol_handler.after_init_connection(); - socket_.async_read_some(boost::asio::buffer(buffer_), - strand_.wrap( - boost::bind(&connection::handle_read, self, - boost::asio::placeholders::error, - boost::asio::placeholders::bytes_transferred))); + reset_timer(boost::posix_time::milliseconds(m_local ? NEW_CONNECTION_TIMEOUT_LOCAL : NEW_CONNECTION_TIMEOUT_REMOTE), false); + + // first read on the raw socket to detect SSL for the server + buffer_ssl_init_fill = 0; + if (is_income && m_ssl_support != epee::net_utils::ssl_support_t::e_ssl_support_disabled) + socket().async_receive(boost::asio::buffer(buffer_), + boost::asio::socket_base::message_peek, + strand_.wrap( + boost::bind(&connection::handle_receive, self, + boost::asio::placeholders::error, + boost::asio::placeholders::bytes_transferred))); + else + async_read_some(boost::asio::buffer(buffer_), + strand_.wrap( + boost::bind(&connection::handle_read, self, + boost::asio::placeholders::error, + boost::asio::placeholders::bytes_transferred))); #if !defined(_WIN32) || !defined(__i686) // not supported before Windows7, too lazy for runtime check // Just exclude for 32bit windows builds @@ -181,12 +216,12 @@ PRAGMA_WARNING_DISABLE_VS(4355) int tos = get_tos_flag(); boost::asio::detail::socket_option::integer< IPPROTO_IP, IP_TOS > optionTos( tos ); - socket_.set_option( optionTos ); + socket().set_option( optionTos ); //_dbg1("Set ToS flag to " << tos); #endif boost::asio::ip::tcp::no_delay noDelayOption(false); - socket_.set_option(noDelayOption); + socket().set_option(noDelayOption); return true; @@ -211,7 +246,7 @@ PRAGMA_WARNING_DISABLE_VS(4355) template boost::asio::io_service& connection::get_io_service() { - return socket_.get_io_service(); + return GET_IO_SERVICE(socket()); } //--------------------------------------------------------------------------------- template @@ -223,12 +258,13 @@ PRAGMA_WARNING_DISABLE_VS(4355) auto self = safe_shared_from_this(); if(!self) return false; - //_dbg3("[sock " << socket_.native_handle() << "] add_ref, m_peer_number=" << mI->m_peer_number); + //_dbg3("[sock " << socket().native_handle() << "] add_ref, m_peer_number=" << mI->m_peer_number); CRITICAL_REGION_LOCAL(self->m_self_refs_lock); - //_dbg3("[sock " << socket_.native_handle() << "] add_ref 2, m_peer_number=" << mI->m_peer_number); + //_dbg3("[sock " << socket().native_handle() << "] add_ref 2, m_peer_number=" << mI->m_peer_number); if(m_was_shutdown) return false; - m_self_refs.push_back(self); + ++m_reference_count; + m_self_ref = std::move(self); return true; CATCH_ENTRY_L0("connection::add_ref()", false); } @@ -238,12 +274,14 @@ PRAGMA_WARNING_DISABLE_VS(4355) { TRY_ENTRY(); boost::shared_ptr > back_connection_copy; - LOG_TRACE_CC(context, "[sock " << socket_.native_handle() << "] release"); + LOG_TRACE_CC(context, "[sock " << socket().native_handle() << "] release"); CRITICAL_REGION_BEGIN(m_self_refs_lock); - CHECK_AND_ASSERT_MES(m_self_refs.size(), false, "[sock " << socket_.native_handle() << "] m_self_refs empty at connection::release() call"); - //erasing from container without additional copy can cause start deleting object, including m_self_refs - back_connection_copy = m_self_refs.back(); - m_self_refs.pop_back(); + CHECK_AND_ASSERT_MES(m_reference_count, false, "[sock " << socket().native_handle() << "] m_reference_count already at 0 at connection::release() call"); + // is this the last reference? + if (--m_reference_count == 0) { + // move the held reference to a local variable, keeping the object alive until the function terminates + std::swap(back_connection_copy, m_self_ref); + } CRITICAL_REGION_END(); return true; CATCH_ENTRY_L0("connection::release()", false); @@ -264,7 +302,7 @@ PRAGMA_WARNING_DISABLE_VS(4355) std::string address, port; boost::system::error_code e; - boost::asio::ip::tcp::endpoint endpoint = socket_.remote_endpoint(e); + boost::asio::ip::tcp::endpoint endpoint = socket().remote_endpoint(e); if (e) { address = ""; @@ -276,8 +314,8 @@ PRAGMA_WARNING_DISABLE_VS(4355) port = boost::lexical_cast(endpoint.port()); } MDEBUG(" connection type " << to_string( m_connection_type ) << " " - << socket_.local_endpoint().address().to_string() << ":" << socket_.local_endpoint().port() - << " <--> " << address << ":" << port); + << socket().local_endpoint().address().to_string() << ":" << socket().local_endpoint().port() + << " <--> " << context.m_remote_address.str() << " (via " << address << ":" << port << ")"); } //--------------------------------------------------------------------------------- template @@ -285,7 +323,7 @@ PRAGMA_WARNING_DISABLE_VS(4355) std::size_t bytes_transferred) { TRY_ENTRY(); - //_info("[sock " << socket_.native_handle() << "] Async read calledback."); + //_info("[sock " << socket().native_handle() << "] Async read calledback."); if (!e) { @@ -293,6 +331,7 @@ PRAGMA_WARNING_DISABLE_VS(4355) CRITICAL_REGION_LOCAL(m_throttle_speed_in_mutex); m_throttle_speed_in.handle_trafic_exact(bytes_transferred); context.m_current_speed_down = m_throttle_speed_in.get_current_speed(); + context.m_max_speed_down = std::max(context.m_max_speed_down, context.m_current_speed_down); } { @@ -320,14 +359,15 @@ PRAGMA_WARNING_DISABLE_VS(4355) } while(delay > 0); } // any form of sleeping - //_info("[sock " << socket_.native_handle() << "] RECV " << bytes_transferred); + //_info("[sock " << socket().native_handle() << "] RECV " << bytes_transferred); logger_handle_net_read(bytes_transferred); context.m_last_recv = time(NULL); context.m_recv_cnt += bytes_transferred; + m_ready_to_close = false; bool recv_res = m_protocol_handler.handle_recv(buffer_.data(), bytes_transferred); if(!recv_res) { - //_info("[sock " << socket_.native_handle() << "] protocol_want_close"); + //_info("[sock " << socket().native_handle() << "] protocol_want_close"); //some error in protocol, protocol handler ask to close connection boost::interprocess::ipcdetail::atomic_write32(&m_want_close_connection, 1); @@ -341,21 +381,28 @@ PRAGMA_WARNING_DISABLE_VS(4355) }else { reset_timer(get_timeout_from_bytes_read(bytes_transferred), false); - socket_.async_read_some(boost::asio::buffer(buffer_), + async_read_some(boost::asio::buffer(buffer_), strand_.wrap( boost::bind(&connection::handle_read, connection::shared_from_this(), boost::asio::placeholders::error, boost::asio::placeholders::bytes_transferred))); - //_info("[sock " << socket_.native_handle() << "]Async read requested."); + //_info("[sock " << socket().native_handle() << "]Async read requested."); } }else { - _dbg3("[sock " << socket_.native_handle() << "] Some not success at read: " << e.message() << ':' << e.value()); + _dbg3("[sock " << socket().native_handle() << "] Some not success at read: " << e.message() << ':' << e.value()); if(e.value() != 2) { - _dbg3("[sock " << socket_.native_handle() << "] Some problems at read: " << e.message() << ':' << e.value()); + _dbg3("[sock " << socket().native_handle() << "] Some problems at read: " << e.message() << ':' << e.value()); shutdown(); } + else + { + _dbg3("[sock " << socket().native_handle() << "] peer closed connection"); + if (m_ready_to_close) + shutdown(); + } + m_ready_to_close = true; } // If an error occurs then no new asynchronous operations are started. This // means that all shared_ptr references to the connection object will @@ -365,13 +412,85 @@ PRAGMA_WARNING_DISABLE_VS(4355) } //--------------------------------------------------------------------------------- template + void connection::handle_receive(const boost::system::error_code& e, + std::size_t bytes_transferred) + { + TRY_ENTRY(); + if (e) + { + // offload the error case + handle_read(e, bytes_transferred); + return; + } + + reset_timer(get_timeout_from_bytes_read(bytes_transferred), false); + + buffer_ssl_init_fill += bytes_transferred; + if (buffer_ssl_init_fill <= get_ssl_magic_size()) + { + socket().async_receive(boost::asio::buffer(buffer_.data() + buffer_ssl_init_fill, buffer_.size() - buffer_ssl_init_fill), + boost::asio::socket_base::message_peek, + strand_.wrap( + boost::bind(&connection::handle_receive, connection::shared_from_this(), + boost::asio::placeholders::error, + boost::asio::placeholders::bytes_transferred))); + return; + } + + // detect SSL + if (m_ssl_support == epee::net_utils::ssl_support_t::e_ssl_support_autodetect) + { + if (is_ssl((const unsigned char*)buffer_.data(), buffer_ssl_init_fill)) + { + MDEBUG("That looks like SSL"); + m_ssl_support = epee::net_utils::ssl_support_t::e_ssl_support_enabled; // read/write to the SSL socket + } + else + { + MDEBUG("That does not look like SSL"); + m_ssl_support = epee::net_utils::ssl_support_t::e_ssl_support_disabled; // read/write to the raw socket + } + } + + if (m_ssl_support == epee::net_utils::ssl_support_t::e_ssl_support_enabled) + { + // Handshake + if (!handshake(boost::asio::ssl::stream_base::server)) + { + MERROR("SSL handshake failed"); + boost::interprocess::ipcdetail::atomic_write32(&m_want_close_connection, 1); + bool do_shutdown = false; + CRITICAL_REGION_BEGIN(m_send_que_lock); + if(!m_send_que.size()) + do_shutdown = true; + CRITICAL_REGION_END(); + if(do_shutdown) + shutdown(); + return; + } + } + + async_read_some(boost::asio::buffer(buffer_), + strand_.wrap( + boost::bind(&connection::handle_read, connection::shared_from_this(), + boost::asio::placeholders::error, + boost::asio::placeholders::bytes_transferred))); + + // If an error occurs then no new asynchronous operations are started. This + // means that all shared_ptr references to the connection object will + // disappear and the object will be destroyed automatically after this + // handler returns. The connection class's destructor closes the socket. + CATCH_ENTRY_L0("connection::handle_receive", void()); + } + //--------------------------------------------------------------------------------- + template bool connection::call_run_once_service_io() { TRY_ENTRY(); if(!m_is_multithreaded) { //single thread model, we can wait in blocked call - size_t cnt = socket_.get_io_service().run_one(); + size_t cnt = GET_IO_SERVICE(socket()).run_one(); if(!cnt)//service is going to quit return false; }else @@ -381,9 +500,9 @@ PRAGMA_WARNING_DISABLE_VS(4355) //if no handlers were called //TODO: Maybe we need to have have critical section + event + callback to upper protocol to //ask it inside(!) critical region if we still able to go in event wait... - size_t cnt = socket_.get_io_service().poll_one(); + size_t cnt = GET_IO_SERVICE(socket()).poll_one(); if(!cnt) - misc_utils::sleep_no_w(0); + misc_utils::sleep_no_w(1); } return true; @@ -487,9 +606,10 @@ PRAGMA_WARNING_DISABLE_VS(4355) CRITICAL_REGION_LOCAL(m_throttle_speed_out_mutex); m_throttle_speed_out.handle_trafic_exact(cb); context.m_current_speed_up = m_throttle_speed_out.get_current_speed(); + context.m_max_speed_up = std::max(context.m_max_speed_up, context.m_current_speed_up); } - //_info("[sock " << socket_.native_handle() << "] SEND " << cb); + //_info("[sock " << socket().native_handle() << "] SEND " << cb); context.m_last_send = time(NULL); context.m_send_cnt += cb; //some data should be wrote to stream @@ -531,10 +651,10 @@ PRAGMA_WARNING_DISABLE_VS(4355) if(m_send_que.size() > 1) { // active operation should be in progress, nothing to do, just wait last operation callback auto size_now = cb; - MDEBUG("do_send() NOW just queues: packet="<::handle_write, self, _1, _2) - //) + ) ); //_dbg3("(chunk): " << size_now); //logger_handle_net_write(size_now); - //_info("[sock " << socket_.native_handle() << "] Async send requested " << m_send_que.front().size()); + //_info("[sock " << socket().native_handle() << "] Async send requested " << m_send_que.front().size()); } //do_send_handler_stop( ptr , cb ); // empty function return true; - CATCH_ENTRY_L0("connection::do_send", false); + CATCH_ENTRY_L0("connection::do_send_chunk", false); } // do_send_chunk //--------------------------------------------------------------------------------- template - boost::posix_time::milliseconds connection::get_default_time() const + boost::posix_time::milliseconds connection::get_default_timeout() { + unsigned count; + try { count = host_count(m_host); } catch (...) { count = 0; } + const unsigned shift = get_state().sock_count > AGGRESSIVE_TIMEOUT_THRESHOLD ? std::min(std::max(count, 1u) - 1, 8u) : 0; + boost::posix_time::milliseconds timeout(0); if (m_local) - return DEFAULT_TIMEOUT_MS_LOCAL; + timeout = boost::posix_time::milliseconds(DEFAULT_TIMEOUT_MS_LOCAL >> shift); else - return DEFAULT_TIMEOUT_MS_REMOTE; + timeout = boost::posix_time::milliseconds(DEFAULT_TIMEOUT_MS_REMOTE >> shift); + return timeout; } //--------------------------------------------------------------------------------- template - boost::posix_time::milliseconds connection::get_timeout_from_bytes_read(size_t bytes) const + boost::posix_time::milliseconds connection::get_timeout_from_bytes_read(size_t bytes) { boost::posix_time::milliseconds ms = (boost::posix_time::milliseconds)(unsigned)(bytes * TIMEOUT_EXTRA_MS_PER_BYTE); ms += m_timer.expires_from_now(); - if (ms > get_default_time()) - ms = get_default_time(); + if (ms > get_default_timeout()) + ms = get_default_timeout(); return ms; } //--------------------------------------------------------------------------------- template + unsigned int connection::host_count(const std::string &host, int delta) + { + static boost::mutex hosts_mutex; + CRITICAL_REGION_LOCAL(hosts_mutex); + static std::map hosts; + unsigned int &val = hosts[host]; + if (delta > 0) + MTRACE("New connection from host " << host << ": " << val); + else if (delta < 0) + MTRACE("Closed connection from host " << host << ": " << val); + CHECK_AND_ASSERT_THROW_MES(delta >= 0 || val >= (unsigned)-delta, "Count would go negative"); + CHECK_AND_ASSERT_THROW_MES(delta <= 0 || val <= std::numeric_limits::max() - (unsigned)delta, "Count would wrap"); + val += delta; + return val; + } + //--------------------------------------------------------------------------------- + template void connection::reset_timer(boost::posix_time::milliseconds ms, bool add) { - if (m_connection_type != e_connection_type_RPC) - return; MTRACE("Setting " << ms << " expiry"); auto self = safe_shared_from_this(); if(!self) @@ -615,11 +755,26 @@ PRAGMA_WARNING_DISABLE_VS(4355) template bool connection::shutdown() { + CRITICAL_REGION_BEGIN(m_shutdown_lock); + if (m_was_shutdown) + return true; + m_was_shutdown = true; // Initiate graceful connection closure. m_timer.cancel(); boost::system::error_code ignored_ec; - socket_.shutdown(boost::asio::ip::tcp::socket::shutdown_both, ignored_ec); - m_was_shutdown = true; + if (m_ssl_support == epee::net_utils::ssl_support_t::e_ssl_support_enabled) + { + const shared_state &state = static_cast(get_state()); + if (!state.stop_signal_sent) + socket_.shutdown(ignored_ec); + } + socket().shutdown(boost::asio::ip::tcp::socket::shutdown_both, ignored_ec); + if (!m_host.empty()) + { + try { host_count(m_host, -1); } catch (...) { /* ignore */ } + m_host = ""; + } + CRITICAL_REGION_END(); m_protocol_handler.release_protocol(); return true; } @@ -628,7 +783,10 @@ PRAGMA_WARNING_DISABLE_VS(4355) bool connection::close() { TRY_ENTRY(); - //_info("[sock " << socket_.native_handle() << "] Que Shutdown called."); + auto self = safe_shared_from_this(); + if(!self) + return false; + //_info("[sock " << socket().native_handle() << "] Que Shutdown called."); m_timer.cancel(); size_t send_que_size = 0; CRITICAL_REGION_BEGIN(m_send_que_lock); @@ -645,6 +803,15 @@ PRAGMA_WARNING_DISABLE_VS(4355) } //--------------------------------------------------------------------------------- template + bool connection::send_done() + { + if (m_ready_to_close) + return close(); + m_ready_to_close = true; + return true; + } + //--------------------------------------------------------------------------------- + template bool connection::cancel() { return close(); @@ -654,11 +821,11 @@ PRAGMA_WARNING_DISABLE_VS(4355) void connection::handle_write(const boost::system::error_code& e, size_t cb) { TRY_ENTRY(); - LOG_TRACE_CC(context, "[sock " << socket_.native_handle() << "] Async send calledback " << cb); + LOG_TRACE_CC(context, "[sock " << socket().native_handle() << "] Async send calledback " << cb); if (e) { - _dbg1("[sock " << socket_.native_handle() << "] Some problems at write: " << e.message() << ':' << e.value()); + _dbg1("[sock " << socket().native_handle() << "] Some problems at write: " << e.message() << ':' << e.value()); shutdown(); return; } @@ -673,7 +840,7 @@ PRAGMA_WARNING_DISABLE_VS(4355) CRITICAL_REGION_BEGIN(m_send_que_lock); if(m_send_que.empty()) { - _erro("[sock " << socket_.native_handle() << "] m_send_que.size() == 0 at handle_write!"); + _erro("[sock " << socket().native_handle() << "] m_send_que.size() == 0 at handle_write!"); return; } @@ -687,17 +854,17 @@ PRAGMA_WARNING_DISABLE_VS(4355) }else { //have more data to send - reset_timer(get_default_time(), false); + reset_timer(get_default_timeout(), false); auto size_now = m_send_que.front().size(); MDEBUG("handle_write() NOW SENDS: packet="<::handle_write, connection::shared_from_this(), _1, _2) - // ) - ); + async_write(boost::asio::buffer(m_send_que.front().data(), size_now) , + strand_.wrap( + boost::bind(&connection::handle_write, connection::shared_from_this(), _1, _2) + ) + ); //_dbg3("(normal)" << size_now); } CRITICAL_REGION_END(); @@ -729,12 +896,14 @@ PRAGMA_WARNING_DISABLE_VS(4355) template boosted_tcp_server::boosted_tcp_server( t_connection_type connection_type ) : - m_io_service_local_instance(new boost::asio::io_service()), - io_service_(*m_io_service_local_instance.get()), + m_state(boost::make_shared::shared_state>()), + m_io_service_local_instance(new worker()), + io_service_(m_io_service_local_instance->io_service), acceptor_(io_service_), + default_remote(), m_stop_signal_sent(false), m_port(0), - m_sock_count(0), m_sock_number(0), m_threads_count(0), - m_pfilter(NULL), m_thread_index(0), + m_threads_count(0), + m_thread_index(0), m_connection_type( connection_type ), new_connection_() { @@ -744,11 +913,13 @@ PRAGMA_WARNING_DISABLE_VS(4355) template boosted_tcp_server::boosted_tcp_server(boost::asio::io_service& extarnal_io_service, t_connection_type connection_type) : + m_state(boost::make_shared::shared_state>()), io_service_(extarnal_io_service), acceptor_(io_service_), - m_stop_signal_sent(false), m_port(0), - m_sock_count(0), m_sock_number(0), m_threads_count(0), - m_pfilter(NULL), m_thread_index(0), + default_remote(), + m_stop_signal_sent(false), m_port(0), + m_threads_count(0), + m_thread_index(0), m_connection_type(connection_type), new_connection_() { @@ -772,12 +943,14 @@ PRAGMA_WARNING_DISABLE_VS(4355) } //--------------------------------------------------------------------------------- template - bool boosted_tcp_server::init_server(uint32_t port, const std::string address) + bool boosted_tcp_server::init_server(uint32_t port, const std::string address, ssl_options_t ssl_options) { TRY_ENTRY(); m_stop_signal_sent = false; m_port = port; m_address = address; + if (ssl_options) + m_state->configure_ssl(std::move(ssl_options)); // Open the acceptor with the option to reuse the address (i.e. SO_REUSEADDR). boost::asio::ip::tcp::resolver resolver(io_service_); boost::asio::ip::tcp::resolver::query query(address, boost::lexical_cast(port), boost::asio::ip::tcp::resolver::query::canonical_name); @@ -789,7 +962,7 @@ PRAGMA_WARNING_DISABLE_VS(4355) boost::asio::ip::tcp::endpoint binded_endpoint = acceptor_.local_endpoint(); m_port = binded_endpoint.port(); MDEBUG("start accept"); - new_connection_.reset(new connection(io_service_, m_config, m_sock_count, m_sock_number, m_pfilter, m_connection_type)); + new_connection_.reset(new connection(io_service_, m_state, m_connection_type, m_state->ssl_options().support)); acceptor_.async_accept(new_connection_->socket(), boost::bind(&boosted_tcp_server::handle_accept, this, boost::asio::placeholders::error)); @@ -811,7 +984,7 @@ PRAGMA_WARNING_DISABLE_VS(4355) PUSH_WARNINGS DISABLE_GCC_WARNING(maybe-uninitialized) template - bool boosted_tcp_server::init_server(const std::string port, const std::string& address) + bool boosted_tcp_server::init_server(const std::string port, const std::string& address, ssl_options_t ssl_options) { uint32_t p = 0; @@ -819,7 +992,7 @@ DISABLE_GCC_WARNING(maybe-uninitialized) MERROR("Failed to convert port no = " << port); return false; } - return this->init_server(p, address); + return this->init_server(p, address, std::move(ssl_options)); } POP_WARNINGS //--------------------------------------------------------------------------------- @@ -837,6 +1010,7 @@ POP_WARNINGS try { io_service_.run(); + return true; } catch(const std::exception& ex) { @@ -865,7 +1039,8 @@ POP_WARNINGS template void boosted_tcp_server::set_connection_filter(i_connection_filter* pfilter) { - m_pfilter = pfilter; + assert(m_state != nullptr); // always set in constructor + m_state->pfilter = pfilter; } //--------------------------------------------------------------------------------- template @@ -960,6 +1135,8 @@ POP_WARNINGS void boosted_tcp_server::send_stop_signal() { m_stop_signal_sent = true; + typename connection::shared_state *state = static_cast::shared_state*>(m_state.get()); + state->stop_signal_sent = true; TRY_ENTRY(); connections_mutex.lock(); for (auto &c: connections_) @@ -973,24 +1150,26 @@ POP_WARNINGS } //--------------------------------------------------------------------------------- template - bool boosted_tcp_server::is_stop_signal_sent() - { - return m_stop_signal_sent; - } - //--------------------------------------------------------------------------------- - template void boosted_tcp_server::handle_accept(const boost::system::error_code& e) { MDEBUG("handle_accept"); - TRY_ENTRY(); + try + { if (!e) { if (m_connection_type == e_connection_type_RPC) { - MDEBUG("New server for RPC connections"); + const char *ssl_message = "unknown"; + switch (new_connection_->get_ssl_support()) + { + case epee::net_utils::ssl_support_t::e_ssl_support_disabled: ssl_message = "disabled"; break; + case epee::net_utils::ssl_support_t::e_ssl_support_enabled: ssl_message = "enabled"; break; + case epee::net_utils::ssl_support_t::e_ssl_support_autodetect: ssl_message = "autodetection"; break; + } + MDEBUG("New server for RPC connections, SSL " << ssl_message); new_connection_->setRpcStation(); // hopefully this is not needed actually } connection_ptr conn(std::move(new_connection_)); - new_connection_.reset(new connection(io_service_, m_config, m_sock_count, m_sock_number, m_pfilter, m_connection_type)); + new_connection_.reset(new connection(io_service_, m_state, m_connection_type, conn->get_ssl_support())); acceptor_.async_accept(new_connection_->socket(), boost::bind(&boosted_tcp_server::handle_accept, this, boost::asio::placeholders::error)); @@ -998,49 +1177,77 @@ POP_WARNINGS boost::asio::socket_base::keep_alive opt(true); conn->socket().set_option(opt); - conn->start(true, 1 < m_threads_count); + bool res; + if (default_remote.get_type_id() == net_utils::address_type::invalid) + res = conn->start(true, 1 < m_threads_count); + else + res = conn->start(true, 1 < m_threads_count, default_remote); + if (!res) + { + conn->cancel(); + return; + } conn->save_dbg_log(); - }else + return; + } + else { - _erro("Some problems at accept: " << e.message() << ", connections_count = " << m_sock_count); + MERROR("Error in boosted_tcp_server::handle_accept: " << e); + } } - CATCH_ENTRY_L0("boosted_tcp_server::handle_accept", void()); + catch (const std::exception &e) + { + MERROR("Exception in boosted_tcp_server::handle_accept: " << e.what()); + } + + // error path, if e or exception + assert(m_state != nullptr); // always set in constructor + _erro("Some problems at accept: " << e.message() << ", connections_count = " << m_state->sock_count); + misc_utils::sleep_no_w(100); + new_connection_.reset(new connection(io_service_, m_state, m_connection_type, new_connection_->get_ssl_support())); + acceptor_.async_accept(new_connection_->socket(), + boost::bind(&boosted_tcp_server::handle_accept, this, + boost::asio::placeholders::error)); } //--------------------------------------------------------------------------------- template - bool boosted_tcp_server::connect(const std::string& adr, const std::string& port, uint32_t conn_timeout, t_connection_context& conn_context, const std::string& bind_ip) + bool boosted_tcp_server::add_connection(t_connection_context& out, boost::asio::ip::tcp::socket&& sock, network_address real_remote, epee::net_utils::ssl_support_t ssl_support) { - TRY_ENTRY(); - - connection_ptr new_connection_l(new connection(io_service_, m_config, m_sock_count, m_sock_number, m_pfilter, m_connection_type) ); - connections_mutex.lock(); - connections_.insert(new_connection_l); - MDEBUG("connections_ size now " << connections_.size()); - connections_mutex.unlock(); - epee::misc_utils::auto_scope_leave_caller scope_exit_handler = epee::misc_utils::create_scope_leave_handler([&](){ CRITICAL_REGION_LOCAL(connections_mutex); connections_.erase(new_connection_l); }); - boost::asio::ip::tcp::socket& sock_ = new_connection_l->socket(); - - ////////////////////////////////////////////////////////////////////////// - boost::asio::ip::tcp::resolver resolver(io_service_); - boost::asio::ip::tcp::resolver::query query(boost::asio::ip::tcp::v4(), adr, port, boost::asio::ip::tcp::resolver::query::canonical_name); - boost::asio::ip::tcp::resolver::iterator iterator = resolver.resolve(query); - boost::asio::ip::tcp::resolver::iterator end; - if(iterator == end) + if(std::addressof(get_io_service()) == std::addressof(GET_IO_SERVICE(sock))) { - _erro("Failed to resolve " << adr); - return false; + connection_ptr conn(new connection(std::move(sock), m_state, m_connection_type, ssl_support)); + if(conn->start(false, 1 < m_threads_count, std::move(real_remote))) + { + conn->get_context(out); + conn->save_dbg_log(); + return true; + } } - ////////////////////////////////////////////////////////////////////////// - + else + { + MWARNING(out << " was not added, socket/io_service mismatch"); + } + return false; + } + //--------------------------------------------------------------------------------- + template + typename boosted_tcp_server::try_connect_result_t boosted_tcp_server::try_connect(connection_ptr new_connection_l, const std::string& adr, const std::string& port, boost::asio::ip::tcp::socket &sock_, const boost::asio::ip::tcp::endpoint &remote_endpoint, const std::string &bind_ip, uint32_t conn_timeout, epee::net_utils::ssl_support_t ssl_support) + { + TRY_ENTRY(); - //boost::asio::ip::tcp::endpoint remote_endpoint(boost::asio::ip::address::from_string(addr.c_str()), port); - boost::asio::ip::tcp::endpoint remote_endpoint(*iterator); - sock_.open(remote_endpoint.protocol()); if(bind_ip != "0.0.0.0" && bind_ip != "0" && bind_ip != "" ) { - boost::asio::ip::tcp::endpoint local_endpoint(boost::asio::ip::address::from_string(adr.c_str()), 0); - sock_.bind(local_endpoint); + boost::asio::ip::tcp::endpoint local_endpoint(boost::asio::ip::address::from_string(bind_ip.c_str()), 0); + boost::system::error_code ec; + sock_.bind(local_endpoint, ec); + if (ec) + { + MERROR("Error binding to " << bind_ip << ": " << ec.message()); + if (sock_.is_open()) + sock_.close(); + return CONNECT_FAILURE; + } } /* @@ -1073,14 +1280,14 @@ POP_WARNINGS { if (sock_.is_open()) sock_.close(); - return false; + return CONNECT_FAILURE; } if(local_shared_context->ec == boost::asio::error::would_block && !r) { //timeout sock_.close(); _dbg3("Failed to connect to " << adr << ":" << port << ", because of timeout (" << conn_timeout << ")"); - return false; + return CONNECT_FAILURE; } } ec = local_shared_context->ec; @@ -1090,11 +1297,79 @@ POP_WARNINGS _dbg3("Some problems at connect, message: " << ec.message()); if (sock_.is_open()) sock_.close(); - return false; + return CONNECT_FAILURE; } _dbg3("Connected success to " << adr << ':' << port); + const ssl_support_t ssl_support = new_connection_l->get_ssl_support(); + if (ssl_support == epee::net_utils::ssl_support_t::e_ssl_support_enabled || ssl_support == epee::net_utils::ssl_support_t::e_ssl_support_autodetect) + { + // Handshake + MDEBUG("Handshaking SSL..."); + if (!new_connection_l->handshake(boost::asio::ssl::stream_base::client)) + { + if (ssl_support == epee::net_utils::ssl_support_t::e_ssl_support_autodetect) + { + boost::system::error_code ignored_ec; + sock_.shutdown(boost::asio::ip::tcp::socket::shutdown_both, ignored_ec); + sock_.close(); + return CONNECT_NO_SSL; + } + MERROR("SSL handshake failed"); + if (sock_.is_open()) + sock_.close(); + return CONNECT_FAILURE; + } + } + + return CONNECT_SUCCESS; + + CATCH_ENTRY_L0("boosted_tcp_server::try_connect", CONNECT_FAILURE); + } + //--------------------------------------------------------------------------------- + template + bool boosted_tcp_server::connect(const std::string& adr, const std::string& port, uint32_t conn_timeout, t_connection_context& conn_context, const std::string& bind_ip, epee::net_utils::ssl_support_t ssl_support) + { + TRY_ENTRY(); + + connection_ptr new_connection_l(new connection(io_service_, m_state, m_connection_type, ssl_support) ); + connections_mutex.lock(); + connections_.insert(new_connection_l); + MDEBUG("connections_ size now " << connections_.size()); + connections_mutex.unlock(); + epee::misc_utils::auto_scope_leave_caller scope_exit_handler = epee::misc_utils::create_scope_leave_handler([&](){ CRITICAL_REGION_LOCAL(connections_mutex); connections_.erase(new_connection_l); }); + boost::asio::ip::tcp::socket& sock_ = new_connection_l->socket(); + + ////////////////////////////////////////////////////////////////////////// + boost::asio::ip::tcp::resolver resolver(io_service_); + boost::asio::ip::tcp::resolver::query query(boost::asio::ip::tcp::v4(), adr, port, boost::asio::ip::tcp::resolver::query::canonical_name); + boost::asio::ip::tcp::resolver::iterator iterator = resolver.resolve(query); + boost::asio::ip::tcp::resolver::iterator end; + if(iterator == end) + { + _erro("Failed to resolve " << adr); + return false; + } + ////////////////////////////////////////////////////////////////////////// + + + //boost::asio::ip::tcp::endpoint remote_endpoint(boost::asio::ip::address::from_string(addr.c_str()), port); + boost::asio::ip::tcp::endpoint remote_endpoint(*iterator); + + auto try_connect_result = try_connect(new_connection_l, adr, port, sock_, remote_endpoint, bind_ip, conn_timeout, ssl_support); + if (try_connect_result == CONNECT_FAILURE) + return false; + if (ssl_support == epee::net_utils::ssl_support_t::e_ssl_support_autodetect && try_connect_result == CONNECT_NO_SSL) + { + // we connected, but could not connect with SSL, try without + MERROR("SSL handshake failed on an autodetect connection, reconnecting without SSL"); + new_connection_l->disable_ssl(); + try_connect_result = try_connect(new_connection_l, adr, port, sock_, remote_endpoint, bind_ip, conn_timeout, epee::net_utils::ssl_support_t::e_ssl_support_disabled); + if (try_connect_result != CONNECT_SUCCESS) + return false; + } + // start adds the connection to the config object's list, so we don't need to have it locally anymore connections_mutex.lock(); connections_.erase(new_connection_l); @@ -1107,7 +1382,8 @@ POP_WARNINGS } else { - _erro("[sock " << new_connection_l->socket().native_handle() << "] Failed to start connection, connections_count = " << m_sock_count); + assert(m_state != nullptr); // always set in constructor + _erro("[sock " << new_connection_l->socket().native_handle() << "] Failed to start connection, connections_count = " << m_state->sock_count); } new_connection_l->save_dbg_log(); @@ -1118,10 +1394,10 @@ POP_WARNINGS } //--------------------------------------------------------------------------------- template template - bool boosted_tcp_server::connect_async(const std::string& adr, const std::string& port, uint32_t conn_timeout, const t_callback &cb, const std::string& bind_ip) + bool boosted_tcp_server::connect_async(const std::string& adr, const std::string& port, uint32_t conn_timeout, const t_callback &cb, const std::string& bind_ip, epee::net_utils::ssl_support_t ssl_support) { TRY_ENTRY(); - connection_ptr new_connection_l(new connection(io_service_, m_config, m_sock_count, m_sock_number, m_pfilter, m_connection_type) ); + connection_ptr new_connection_l(new connection(io_service_, m_state, m_connection_type, ssl_support) ); connections_mutex.lock(); connections_.insert(new_connection_l); MDEBUG("connections_ size now " << connections_.size()); @@ -1145,8 +1421,16 @@ POP_WARNINGS sock_.open(remote_endpoint.protocol()); if(bind_ip != "0.0.0.0" && bind_ip != "0" && bind_ip != "" ) { - boost::asio::ip::tcp::endpoint local_endpoint(boost::asio::ip::address::from_string(adr.c_str()), 0); - sock_.bind(local_endpoint); + boost::asio::ip::tcp::endpoint local_endpoint(boost::asio::ip::address::from_string(bind_ip.c_str()), 0); + boost::system::error_code ec; + sock_.bind(local_endpoint, ec); + if (ec) + { + MERROR("Error binding to " << bind_ip << ": " << ec.message()); + if (sock_.is_open()) + sock_.close(); + return false; + } } boost::shared_ptr sh_deadline(new boost::asio::deadline_timer(io_service_)); diff --git a/contrib/epee/include/net/buffer.h b/contrib/epee/include/net/buffer.h new file mode 100644 index 00000000000..56c67f6bff2 --- /dev/null +++ b/contrib/epee/include/net/buffer.h @@ -0,0 +1,62 @@ +// Copyright (c) 2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +#include +#include "misc_log_ex.h" +#include "span.h" + +#undef MONERO_DEFAULT_LOG_CATEGORY +#define MONERO_DEFAULT_LOG_CATEGORY "net.buffer" + +//#define NET_BUFFER_LOG(x) MDEBUG(x) +#define NET_BUFFER_LOG(x) ((void)0) + +namespace epee +{ +namespace net_utils +{ +class buffer +{ +public: + buffer(size_t reserve = 0): offset(0) { storage.reserve(reserve); } + + void append(const void *data, size_t sz); + void erase(size_t sz) { NET_BUFFER_LOG("erasing " << sz << "/" << size()); CHECK_AND_ASSERT_THROW_MES(offset + sz <= storage.size(), "erase: sz too large"); offset += sz; if (offset == storage.size()) { storage.resize(0); offset = 0; } } + epee::span span(size_t sz) const { CHECK_AND_ASSERT_THROW_MES(sz <= size(), "span is too large"); return epee::span(storage.data() + offset, sz); } + // carve must keep the data in scope till next call, other API calls (such as append, erase) can invalidate the carved buffer + epee::span carve(size_t sz) { CHECK_AND_ASSERT_THROW_MES(sz <= size(), "span is too large"); offset += sz; return epee::span(storage.data() + offset - sz, sz); } + size_t size() const { return storage.size() - offset; } + +private: + std::vector storage; + size_t offset; +}; +} +} diff --git a/contrib/epee/include/net/connection_basic.hpp b/contrib/epee/include/net/connection_basic.hpp index 095e747a5ab..d3f5f4f242d 100644 --- a/contrib/epee/include/net/connection_basic.hpp +++ b/contrib/epee/include/net/connection_basic.hpp @@ -8,7 +8,7 @@ // ! (how ever if in some wonderful juristdictions that is not the case, then why not make another sub-class withat that members and licence it as epee part) // ! Working on above premise, IF this is valid in your juristdictions, then consider this code as released as: -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -42,24 +42,15 @@ #define INCLUDED_p2p_connection_basic_hpp -#include #include -#include -#include -#include #include +#include #include -#include -#include -#include -#include -#include -#include - -#include +#include #include "net/net_utils_base.h" +#include "net/net_ssl.h" #include "syncobj.h" namespace epee @@ -67,6 +58,30 @@ namespace epee namespace net_utils { + class connection_basic_shared_state + { + ssl_options_t ssl_options_; + public: + boost::asio::ssl::context ssl_context; + std::atomic sock_count; + std::atomic sock_number; + + connection_basic_shared_state() + : ssl_options_(ssl_support_t::e_ssl_support_disabled), + ssl_context(boost::asio::ssl::context::tlsv12), + sock_count(0), + sock_number(0) + {} + + void configure_ssl(ssl_options_t src) + { + ssl_options_ = std::move(src); + ssl_context = ssl_options_.create_context(); + } + + const ssl_options_t& ssl_options() const noexcept { return ssl_options_; } + }; + /************************************************************************/ /* */ /************************************************************************/ @@ -83,7 +98,10 @@ class connection_basic_pimpl; // PIMPL for this class std::string to_string(t_connection_type type); class connection_basic { // not-templated base class for rapid developmet of some code parts + // beware of removing const, net_utils::connection is sketchily doing a cast to prevent storing ptr twice + const boost::shared_ptr m_state; public: + std::unique_ptr< connection_basic_pimpl > mI; // my Implementation // moved here from orginal connecton<> - common member variables that do not depend on template in connection<> @@ -92,19 +110,60 @@ class connection_basic { // not-templated base class for rapid developmet of som critical_section m_send_que_lock; std::list m_send_que; volatile bool m_is_multithreaded; - double m_start_time; /// Strand to ensure the connection's handlers are not called concurrently. boost::asio::io_service::strand strand_; /// Socket for the connection. - boost::asio::ip::tcp::socket socket_; + boost::asio::ssl::stream socket_; + ssl_support_t m_ssl_support; - std::atomic &m_ref_sock_count; // reference to external counter of existing sockets that we will ++/-- public: // first counter is the ++/-- count of current sockets, the other socket_number is only-increasing ++ number generator - connection_basic(boost::asio::io_service& io_service, std::atomic &ref_sock_count, std::atomic &sock_number); + connection_basic(boost::asio::ip::tcp::socket&& socket, boost::shared_ptr state, ssl_support_t ssl_support); + connection_basic(boost::asio::io_service &io_service, boost::shared_ptr state, ssl_support_t ssl_support); virtual ~connection_basic() noexcept(false); + //! \return `shared_state` object passed in construction (ptr never changes). + connection_basic_shared_state& get_state() noexcept { return *m_state; /* verified in constructor */ } + connection_basic(boost::asio::io_service& io_service, std::atomic &ref_sock_count, std::atomic &sock_number, ssl_support_t ssl); + + boost::asio::ip::tcp::socket& socket() { return socket_.next_layer(); } + ssl_support_t get_ssl_support() const { return m_ssl_support; } + void disable_ssl() { m_ssl_support = epee::net_utils::ssl_support_t::e_ssl_support_disabled; } + + bool handshake(boost::asio::ssl::stream_base::handshake_type type) + { + //m_state != nullptr verified in constructor + return m_state->ssl_options().handshake(socket_, type); + } + + template + void async_read_some(const MutableBufferSequence &buffers, ReadHandler &&handler) + { + if (m_ssl_support == epee::net_utils::ssl_support_t::e_ssl_support_enabled) + socket_.async_read_some(buffers, std::forward(handler)); + else + socket().async_read_some(buffers, std::forward(handler)); + } + + template + void async_write_some(const ConstBufferSequence &buffers, WriteHandler &&handler) + { + if (m_ssl_support == epee::net_utils::ssl_support_t::e_ssl_support_enabled) + socket_.async_write_some(buffers, std::forward(handler)); + else + socket().async_write_some(buffers, std::forward(handler)); + } + + template + void async_write(const ConstBufferSequence &buffers, WriteHandler &&handler) + { + if (m_ssl_support == epee::net_utils::ssl_support_t::e_ssl_support_enabled) + boost::asio::async_write(socket_, buffers, std::forward(handler)); + else + boost::asio::async_write(socket(), buffers, std::forward(handler)); + } + // various handlers to be called from connection class: void do_send_handler_write(const void * ptr , size_t cb); void do_send_handler_write_from_queue(const boost::system::error_code& e, size_t cb , int q_len); // from handle_write, sending next part @@ -112,8 +171,6 @@ class connection_basic { // not-templated base class for rapid developmet of som void logger_handle_net_write(size_t size); // network data written void logger_handle_net_read(size_t size); // network data read - void set_start_time(); - // config for rate limit static void set_rate_up_limit(uint64_t limit); diff --git a/contrib/epee/include/net/enums.h b/contrib/epee/include/net/enums.h new file mode 100644 index 00000000000..078a4b27463 --- /dev/null +++ b/contrib/epee/include/net/enums.h @@ -0,0 +1,65 @@ +// Copyright (c) 2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +#include +#include + +namespace epee +{ +namespace net_utils +{ + enum class address_type : std::uint8_t + { + // Do not change values, this will break serialization + invalid = 0, + ipv4 = 1, + ipv6 = 2, + i2p = 3, + tor = 4 + }; + + enum class zone : std::uint8_t + { + invalid = 0, + public_ = 1, // public is keyword + i2p = 2, + tor = 3 + }; + + // implementations in src/net_utils_base.cpp + + //! \return String name of zone or "invalid" on error. + const char* zone_to_string(zone value) noexcept; + + //! \return `zone` enum of `value` or `zone::invalid` on error. + zone zone_from_string(boost::string_ref value) noexcept; +} // net_utils +} // epee + diff --git a/contrib/epee/include/net/http_auth.h b/contrib/epee/include/net/http_auth.h index 4324c41fd9b..00b9750b7ef 100644 --- a/contrib/epee/include/net/http_auth.h +++ b/contrib/epee/include/net/http_auth.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/contrib/epee/include/net/http_client.h b/contrib/epee/include/net/http_client.h index e0169179472..588d5f0e368 100644 --- a/contrib/epee/include/net/http_client.h +++ b/contrib/epee/include/net/http_client.h @@ -274,8 +274,8 @@ namespace net_utils reciev_machine_state m_state; chunked_state m_chunked_state; std::string m_chunked_cache; + bool m_auto_connect; critical_section m_lock; - bool m_ssl; public: explicit http_simple_client_template() @@ -292,36 +292,48 @@ namespace net_utils , m_state() , m_chunked_state() , m_chunked_cache() + , m_auto_connect(true) , m_lock() - , m_ssl(false) {} const std::string &get_host() const { return m_host_buff; }; const std::string &get_port() const { return m_port; }; - bool set_server(const std::string& address, boost::optional user, bool ssl = false) + bool set_server(const std::string& address, boost::optional user, ssl_options_t ssl_options = ssl_support_t::e_ssl_support_autodetect) { http::url_content parsed{}; const bool r = parse_url(address, parsed); CHECK_AND_ASSERT_MES(r, false, "failed to parse url: " << address); - set_server(std::move(parsed.host), std::to_string(parsed.port), std::move(user), ssl); + set_server(std::move(parsed.host), std::to_string(parsed.port), std::move(user), std::move(ssl_options)); return true; } - void set_server(std::string host, std::string port, boost::optional user, bool ssl = false) + void set_server(std::string host, std::string port, boost::optional user, ssl_options_t ssl_options = ssl_support_t::e_ssl_support_autodetect) { CRITICAL_REGION_LOCAL(m_lock); disconnect(); m_host_buff = std::move(host); m_port = std::move(port); m_auth = user ? http_client_auth{std::move(*user)} : http_client_auth{}; - m_ssl = ssl; + m_net_client.set_ssl(std::move(ssl_options)); + } + + void set_auto_connect(bool auto_connect) + { + m_auto_connect = auto_connect; + } + + template + void set_connector(F connector) + { + CRITICAL_REGION_LOCAL(m_lock); + m_net_client.set_connector(std::move(connector)); } bool connect(std::chrono::milliseconds timeout) { CRITICAL_REGION_LOCAL(m_lock); - return m_net_client.connect(m_host_buff, m_port, timeout, m_ssl); + return m_net_client.connect(m_host_buff, m_port, timeout); } //--------------------------------------------------------------------------- bool disconnect() @@ -330,10 +342,10 @@ namespace net_utils return m_net_client.disconnect(); } //--------------------------------------------------------------------------- - bool is_connected() + bool is_connected(bool *ssl = NULL) { CRITICAL_REGION_LOCAL(m_lock); - return m_net_client.is_connected(); + return m_net_client.is_connected(ssl); } //--------------------------------------------------------------------------- virtual bool handle_target_data(std::string& piece_of_transfer) @@ -362,6 +374,11 @@ namespace net_utils CRITICAL_REGION_LOCAL(m_lock); if(!is_connected()) { + if (!m_auto_connect) + { + MWARNING("Auto connect attempt to " << m_host_buff << ":" << m_port << " disabled"); + return false; + } MDEBUG("Reconnecting..."); if(!connect(timeout)) { @@ -439,6 +456,16 @@ namespace net_utils return handle_reciev(timeout); } //--------------------------------------------------------------------------- + uint64_t get_bytes_sent() const + { + return m_net_client.get_bytes_sent(); + } + //--------------------------------------------------------------------------- + uint64_t get_bytes_received() const + { + return m_net_client.get_bytes_received(); + } + //--------------------------------------------------------------------------- private: //--------------------------------------------------------------------------- inline bool handle_reciev(std::chrono::milliseconds timeout) @@ -825,21 +852,21 @@ namespace net_utils const char *ptr = m_header_cache.c_str(); CHECK_AND_ASSERT_MES(!memcmp(ptr, "HTTP/", 5), false, "Invalid first response line: " + m_header_cache); ptr += 5; - CHECK_AND_ASSERT_MES(isdigit(*ptr), false, "Invalid first response line: " + m_header_cache); + CHECK_AND_ASSERT_MES(epee::misc_utils::parse::isdigit(*ptr), false, "Invalid first response line: " + m_header_cache); unsigned long ul; char *end; ul = strtoul(ptr, &end, 10); CHECK_AND_ASSERT_MES(ul <= INT_MAX && *end =='.', false, "Invalid first response line: " + m_header_cache); m_response_info.m_http_ver_hi = ul; ptr = end + 1; - CHECK_AND_ASSERT_MES(isdigit(*ptr), false, "Invalid first response line: " + m_header_cache + ", ptr: " << ptr); + CHECK_AND_ASSERT_MES(epee::misc_utils::parse::isdigit(*ptr), false, "Invalid first response line: " + m_header_cache + ", ptr: " << ptr); ul = strtoul(ptr, &end, 10); CHECK_AND_ASSERT_MES(ul <= INT_MAX && isblank(*end), false, "Invalid first response line: " + m_header_cache + ", ptr: " << ptr); m_response_info.m_http_ver_lo = ul; ptr = end + 1; while (isblank(*ptr)) ++ptr; - CHECK_AND_ASSERT_MES(isdigit(*ptr), false, "Invalid first response line: " + m_header_cache); + CHECK_AND_ASSERT_MES(epee::misc_utils::parse::isdigit(*ptr), false, "Invalid first response line: " + m_header_cache); ul = strtoul(ptr, &end, 10); CHECK_AND_ASSERT_MES(ul >= 100 && ul <= 999 && isspace(*end), false, "Invalid first response line: " + m_header_cache); m_response_info.m_response_code = ul; diff --git a/contrib/epee/include/net/http_protocol_handler.h b/contrib/epee/include/net/http_protocol_handler.h index e602fac2b6b..1780f23935c 100644 --- a/contrib/epee/include/net/http_protocol_handler.h +++ b/contrib/epee/include/net/http_protocol_handler.h @@ -69,7 +69,7 @@ namespace net_utils typedef t_connection_context connection_context;//t_connection_context net_utils::connection_context_base connection_context; typedef http_server_config config_type; - simple_http_connection_handler(i_service_endpoint* psnd_hndlr, config_type& config); + simple_http_connection_handler(i_service_endpoint* psnd_hndlr, config_type& config, t_connection_context& conn_context); virtual ~simple_http_connection_handler(){} bool release_protocol() @@ -144,6 +144,7 @@ namespace net_utils size_t m_newlines; protected: i_service_endpoint* m_psnd_hndlr; + t_connection_context& m_conn_context; }; template @@ -175,9 +176,8 @@ namespace net_utils typedef custum_handler_config config_type; http_custom_handler(i_service_endpoint* psnd_hndlr, config_type& config, t_connection_context& conn_context) - : simple_http_connection_handler(psnd_hndlr, config), + : simple_http_connection_handler(psnd_hndlr, config, conn_context), m_config(config), - m_conn_context(conn_context), m_auth(m_config.m_user ? http_server_auth{*m_config.m_user, config.rng} : http_server_auth{}) {} inline bool handle_request(const http_request_info& query_info, http_response_info& response) @@ -197,7 +197,7 @@ namespace net_utils response.m_response_comment = "OK"; response.m_body.clear(); - return m_config.m_phandler->handle_http_request(query_info, response, m_conn_context); + return m_config.m_phandler->handle_http_request(query_info, response, this->m_conn_context); } virtual bool thread_init() @@ -219,7 +219,6 @@ namespace net_utils private: //simple_http_connection_handler::config_type m_stub_config; config_type& m_config; - t_connection_context& m_conn_context; http_server_auth m_auth; }; } diff --git a/contrib/epee/include/net/http_protocol_handler.inl b/contrib/epee/include/net/http_protocol_handler.inl index f1da5067a9e..ae8e43477ca 100644 --- a/contrib/epee/include/net/http_protocol_handler.inl +++ b/contrib/epee/include/net/http_protocol_handler.inl @@ -32,6 +32,7 @@ #include "string_tools.h" #include "file_io_utils.h" #include "net_parse_helpers.h" +#include "time_helper.h" #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "net.http" @@ -196,16 +197,17 @@ namespace net_utils //-------------------------------------------------------------------------------------------- template - simple_http_connection_handler::simple_http_connection_handler(i_service_endpoint* psnd_hndlr, config_type& config): + simple_http_connection_handler::simple_http_connection_handler(i_service_endpoint* psnd_hndlr, config_type& config, t_connection_context& conn_context): m_state(http_state_retriving_comand_line), m_body_transfer_type(http_body_transfer_undefined), - m_is_stop_handling(false), + m_is_stop_handling(false), m_len_summary(0), m_len_remain(0), - m_config(config), + m_config(config), m_want_close(false), m_newlines(0), - m_psnd_hndlr(psnd_hndlr) + m_psnd_hndlr(psnd_hndlr), + m_conn_context(conn_context) { } @@ -281,7 +283,7 @@ namespace net_utils m_is_stop_handling = true; if(m_cache.size() > HTTP_MAX_URI_LEN) { - LOG_ERROR("simple_http_connection_handler::handle_buff_out: Too long URI line"); + LOG_ERROR_CC(m_conn_context, "simple_http_connection_handler::handle_buff_out: Too long URI line"); m_state = http_state_error; return false; } @@ -295,7 +297,7 @@ namespace net_utils m_is_stop_handling = true; if(m_cache.size() > HTTP_MAX_HEADER_LEN) { - LOG_ERROR("simple_http_connection_handler::handle_buff_in: Too long header area"); + LOG_ERROR_CC(m_conn_context, "simple_http_connection_handler::handle_buff_in: Too long header area"); m_state = http_state_error; return false; } @@ -310,10 +312,10 @@ namespace net_utils case http_state_connection_close: return false; default: - LOG_ERROR("simple_http_connection_handler::handle_char_out: Wrong state: " << m_state); + LOG_ERROR_CC(m_conn_context, "simple_http_connection_handler::handle_char_out: Wrong state: " << m_state); return false; case http_state_error: - LOG_ERROR("simple_http_connection_handler::handle_char_out: Error state!!!"); + LOG_ERROR_CC(m_conn_context, "simple_http_connection_handler::handle_char_out: Error state!!!"); return false; } @@ -327,8 +329,10 @@ namespace net_utils inline bool analize_http_method(const boost::smatch& result, http::http_method& method, int& http_ver_major, int& http_ver_minor) { CHECK_AND_ASSERT_MES(result[0].matched, false, "simple_http_connection_handler::analize_http_method() assert failed..."); - http_ver_major = boost::lexical_cast(result[11]); - http_ver_minor = boost::lexical_cast(result[12]); + if (!boost::conversion::try_lexical_convert(result[11], http_ver_major)) + return false; + if (!boost::conversion::try_lexical_convert(result[12], http_ver_minor)) + return false; if(result[3].matched) method = http::http_method_options; @@ -350,13 +354,18 @@ namespace net_utils template bool simple_http_connection_handler::handle_invoke_query_line() { - STATIC_REGEXP_EXPR_1(rexp_match_command_line, "^(((OPTIONS)|(GET)|(HEAD)|(POST)|(PUT)|(DELETE)|(TRACE)) (\\S+) HTTP/(\\d+).(\\d+))\r?\n", boost::regex::icase | boost::regex::normal); + STATIC_REGEXP_EXPR_1(rexp_match_command_line, "^(((OPTIONS)|(GET)|(HEAD)|(POST)|(PUT)|(DELETE)|(TRACE)) (\\S+) HTTP/(\\d+)\\.(\\d+))\r?\n", boost::regex::icase | boost::regex::normal); // 123 4 5 6 7 8 9 10 11 12 //size_t match_len = 0; boost::smatch result; if(boost::regex_search(m_cache, result, rexp_match_command_line, boost::match_default) && result[0].matched) { - analize_http_method(result, m_query_info.m_http_method, m_query_info.m_http_ver_hi, m_query_info.m_http_ver_hi); + if (!analize_http_method(result, m_query_info.m_http_method, m_query_info.m_http_ver_hi, m_query_info.m_http_ver_hi)) + { + m_state = http_state_error; + MERROR("Failed to analyze method"); + return false; + } m_query_info.m_URI = result[10]; if (!parse_uri(m_query_info.m_URI, m_query_info.m_uri_content)) { @@ -375,7 +384,7 @@ namespace net_utils }else { m_state = http_state_error; - LOG_ERROR("simple_http_connection_handler::handle_invoke_query_line(): Failed to match first line: " << m_cache); + LOG_ERROR_CC(m_conn_context, "simple_http_connection_handler::handle_invoke_query_line(): Failed to match first line: " << m_cache); return false; } @@ -399,14 +408,14 @@ namespace net_utils template bool simple_http_connection_handler::analize_cached_request_header_and_invoke_state(size_t pos) { - //LOG_PRINT_L4("HTTP HEAD:\r\n" << m_cache.substr(0, pos)); + LOG_PRINT_L3("HTTP HEAD:\r\n" << m_cache.substr(0, pos)); m_query_info.m_full_request_buf_size = pos; m_query_info.m_request_head.assign(m_cache.begin(), m_cache.begin()+pos); if(!parse_cached_header(m_query_info.m_header_info, m_cache, pos)) { - LOG_ERROR("simple_http_connection_handler::analize_cached_request_header_and_invoke_state(): failed to anilize request header: " << m_cache); + LOG_ERROR_CC(m_conn_context, "simple_http_connection_handler::analize_cached_request_header_and_invoke_state(): failed to anilize request header: " << m_cache); m_state = http_state_error; return false; } @@ -422,7 +431,7 @@ namespace net_utils m_body_transfer_type = http_body_transfer_measure; if(!get_len_from_content_lenght(m_query_info.m_header_info.m_content_length, m_len_summary)) { - LOG_ERROR("simple_http_connection_handler::analize_cached_request_header_and_invoke_state(): Failed to get_len_from_content_lenght();, m_query_info.m_content_length="<::analize_cached_request_header_and_invoke_state(): Failed to get_len_from_content_lenght();, m_query_info.m_content_length="<::handle_retriving_query_body(): Unexpected m_body_query_type state:" << m_body_transfer_type); + LOG_ERROR_CC(m_conn_context, "simple_http_connection_handler::handle_retriving_query_body(): Unexpected m_body_query_type state:" << m_body_transfer_type); m_state = http_state_error; return false; } @@ -536,7 +545,7 @@ namespace net_utils body_info.m_etc_fields.push_back(std::pair(result[field_etc_name], result[field_val])); else { - LOG_ERROR("simple_http_connection_handler::parse_cached_header() not matched last entry in:"<::parse_cached_header() not matched last entry in:" << m_cache_to_process); } it_current_bound = result[(int)result.size()-1]. first; @@ -553,7 +562,8 @@ namespace net_utils if(!(boost::regex_search( str, result, rexp_mach_field, boost::match_default) && result[0].matched)) return false; - len = boost::lexical_cast(result[0]); + try { len = boost::lexical_cast(result[0]); } + catch(...) { return false; } return true; } //----------------------------------------------------------------------------------- @@ -582,6 +592,7 @@ namespace net_utils m_psnd_hndlr->do_send((void*)response_data.data(), response_data.size()); if ((response.m_body.size() && (query_info.m_http_method != http::http_method_head)) || (query_info.m_http_method == http::http_method_options)) m_psnd_hndlr->do_send((void*)response.m_body.data(), response.m_body.size()); + m_psnd_hndlr->send_done(); return res; } //----------------------------------------------------------------------------------- @@ -666,7 +677,7 @@ namespace net_utils //add additional fields, if it is for(fields_list::const_iterator it = response.m_additional_fields.begin(); it!=response.m_additional_fields.end(); it++) - buf += it->first + ":" + it->second + "\r\n"; + buf += it->first + ": " + it->second + "\r\n"; buf+="\r\n"; diff --git a/contrib/epee/include/net/http_server_handlers_map2.h b/contrib/epee/include/net/http_server_handlers_map2.h index 429e3e1afdc..4b20530918d 100644 --- a/contrib/epee/include/net/http_server_handlers_map2.h +++ b/contrib/epee/include/net/http_server_handlers_map2.h @@ -39,7 +39,7 @@ epee::net_utils::http::http_response_info& response, \ context_type& m_conn_context) \ {\ - LOG_PRINT_L2("HTTP [" << m_conn_context.m_remote_address.host_str() << "] " << query_info.m_http_method_str << " " << query_info.m_URI); \ + MINFO("HTTP [" << m_conn_context.m_remote_address.host_str() << "] " << query_info.m_http_method_str << " " << query_info.m_URI); \ response.m_response_code = 200; \ response.m_response_comment = "Ok"; \ if(!handle_http_request_map(query_info, response, m_conn_context)) \ @@ -54,7 +54,7 @@ bool handled = false; \ if(false) return true; //just a stub to have "else if" -#define MAP_URI2(pattern, callback) else if(std::string::npos != query_info.m_URI.find(pattern)) return callback(query_info, response_info, m_conn_context); +#define MAP_URI2(pattern, callback) else if(std::string::npos != query_info.m_URI.find(pattern)) return callback(query_info, response_info, &m_conn_context); #define MAP_URI_AUTO_XML2(s_pattern, callback_f, command_type) //TODO: don't think i ever again will use xml - ambiguous and "overtagged" format @@ -68,7 +68,8 @@ CHECK_AND_ASSERT_MES(parse_res, false, "Failed to parse json: \r\n" << query_info.m_body); \ uint64_t ticks1 = epee::misc_utils::get_tick_count(); \ boost::value_initialized resp;\ - if(!callback_f(static_cast(req), static_cast(resp))) \ + MINFO(m_conn_context << "calling " << s_pattern); \ + if(!callback_f(static_cast(req), static_cast(resp), &m_conn_context)) \ { \ LOG_ERROR("Failed to " << #callback_f << "()"); \ response_info.m_response_code = 500; \ @@ -91,11 +92,12 @@ handled = true; \ uint64_t ticks = misc_utils::get_tick_count(); \ boost::value_initialized req; \ - bool parse_res = epee::serialization::load_t_from_binary(static_cast(req), query_info.m_body); \ + bool parse_res = epee::serialization::load_t_from_binary(static_cast(req), epee::strspan(query_info.m_body)); \ CHECK_AND_ASSERT_MES(parse_res, false, "Failed to parse bin body data, body size=" << query_info.m_body.size()); \ uint64_t ticks1 = misc_utils::get_tick_count(); \ boost::value_initialized resp;\ - if(!callback_f(static_cast(req), static_cast(resp))) \ + MINFO(m_conn_context << "calling " << s_pattern); \ + if(!callback_f(static_cast(req), static_cast(resp), &m_conn_context)) \ { \ LOG_ERROR("Failed to " << #callback_f << "()"); \ response_info.m_response_code = 500; \ @@ -122,6 +124,7 @@ if(!ps.load_from_json(query_info.m_body)) \ { \ boost::value_initialized rsp; \ + static_cast(rsp).jsonrpc = "2.0"; \ static_cast(rsp).error.code = -32700; \ static_cast(rsp).error.message = "Parse error"; \ epee::serialization::store_t_to_json(static_cast(rsp), response_info.m_body); \ @@ -178,7 +181,8 @@ epee::json_rpc::error_response fail_resp = AUTO_VAL_INIT(fail_resp); \ fail_resp.jsonrpc = "2.0"; \ fail_resp.id = req.id; \ - if(!callback_f(req.params, resp.result, fail_resp.error)) \ + MINFO(m_conn_context << "Calling RPC method " << method_name); \ + if(!callback_f(req.params, resp.result, fail_resp.error, &m_conn_context)) \ { \ epee::serialization::store_t_to_json(static_cast(fail_resp), response_info.m_body); \ return true; \ @@ -196,7 +200,8 @@ epee::json_rpc::error_response fail_resp = AUTO_VAL_INIT(fail_resp); \ fail_resp.jsonrpc = "2.0"; \ fail_resp.id = req.id; \ - if(!callback_f(req.params, resp.result, fail_resp.error, m_conn_context, response_info)) \ + MINFO(m_conn_context << "calling RPC method " << method_name); \ + if(!callback_f(req.params, resp.result, fail_resp.error, response_info, &m_conn_context)) \ { \ epee::serialization::store_t_to_json(static_cast(fail_resp), response_info.m_body); \ return true; \ @@ -209,7 +214,8 @@ else if(callback_name == method_name) \ { \ PREPARE_OBJECTS_FROM_JSON(command_type) \ - if(!callback_f(req.params, resp.result)) \ + MINFO(m_conn_context << "calling RPC method " << method_name); \ + if(!callback_f(req.params, resp.result, &m_conn_context)) \ { \ epee::json_rpc::error_response fail_resp = AUTO_VAL_INIT(fail_resp); \ fail_resp.jsonrpc = "2.0"; \ diff --git a/contrib/epee/include/net/http_server_impl_base.h b/contrib/epee/include/net/http_server_impl_base.h index 1a97e610a61..fc2dcbf67c7 100644 --- a/contrib/epee/include/net/http_server_impl_base.h +++ b/contrib/epee/include/net/http_server_impl_base.h @@ -33,7 +33,8 @@ #include #include -#include "net/http_server_cp2.h" +#include "net/abstract_tcp_server2.h" +#include "http_protocol_handler.h" #include "net/http_server_handlers_map2.h" #undef MONERO_DEFAULT_LOG_CATEGORY @@ -57,7 +58,8 @@ namespace epee bool init(std::function rng, const std::string& bind_port = "0", const std::string& bind_ip = "0.0.0.0", std::vector access_control_origins = std::vector(), - boost::optional user = boost::none) + boost::optional user = boost::none, + net_utils::ssl_options_t ssl_options = net_utils::ssl_support_t::e_ssl_support_autodetect) { //set self as callback handler @@ -74,7 +76,7 @@ namespace epee m_net_server.get_config_object().m_user = std::move(user); MGINFO("Binding on " << bind_ip << ":" << bind_port); - bool res = m_net_server.init_server(bind_port, bind_ip); + bool res = m_net_server.init_server(bind_port, bind_ip, std::move(ssl_options)); if(!res) { LOG_ERROR("Failed to bind server"); diff --git a/contrib/epee/include/net/jsonrpc_structs.h b/contrib/epee/include/net/jsonrpc_structs.h index 9df9e259612..b61c369f283 100644 --- a/contrib/epee/include/net/jsonrpc_structs.h +++ b/contrib/epee/include/net/jsonrpc_structs.h @@ -18,6 +18,8 @@ namespace epee epee::serialization::storage_entry id; t_param params; + request(): id{}, params{} {} + BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(jsonrpc) KV_SERIALIZE(id) @@ -30,6 +32,9 @@ namespace epee { int64_t code; std::string message; + + error(): code(0) {} + BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(code) KV_SERIALIZE(message) @@ -55,6 +60,9 @@ namespace epee t_param result; epee::serialization::storage_entry id; t_error error; + + response(): result{}, id(), error{} {} + BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(jsonrpc) KV_SERIALIZE(id) @@ -69,6 +77,9 @@ namespace epee std::string jsonrpc; t_param result; epee::serialization::storage_entry id; + + response(): result{}, id{} {} + BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(jsonrpc) KV_SERIALIZE(id) @@ -82,6 +93,9 @@ namespace epee std::string jsonrpc; t_error error; epee::serialization::storage_entry id; + + response(): error{}, id{} {} + BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(jsonrpc) KV_SERIALIZE(id) diff --git a/contrib/epee/include/net/levin_base.h b/contrib/epee/include/net/levin_base.h index 7d060f5ef78..a88a1eb4958 100644 --- a/contrib/epee/include/net/levin_base.h +++ b/contrib/epee/include/net/levin_base.h @@ -80,8 +80,8 @@ namespace levin template struct levin_commands_handler { - virtual int invoke(int command, const std::string& in_buff, std::string& buff_out, t_connection_context& context)=0; - virtual int notify(int command, const std::string& in_buff, t_connection_context& context)=0; + virtual int invoke(int command, const epee::span in_buff, std::string& buff_out, t_connection_context& context)=0; + virtual int notify(int command, const epee::span in_buff, t_connection_context& context)=0; virtual void callback(t_connection_context& context){}; virtual void on_connection_new(t_connection_context& context){}; diff --git a/contrib/epee/include/net/levin_client.h b/contrib/epee/include/net/levin_client.h index 335f6ba02f7..76d52823449 100644 --- a/contrib/epee/include/net/levin_client.h +++ b/contrib/epee/include/net/levin_client.h @@ -57,7 +57,7 @@ namespace levin bool is_connected(); bool disconnect(); - virtual int invoke(int command, const std::string& in_buff, std::string& buff_out); + virtual int invoke(int command, const epee::span in_buff, std::string& buff_out); virtual int notify(int command, const std::string& in_buff); protected: @@ -72,7 +72,7 @@ namespace levin { public: - int invoke(int command, const std::string& in_buff, std::string& buff_out); + int invoke(int command, const epee::span in_buff, std::string& buff_out); int notify(int command, const std::string& in_buff); }; diff --git a/contrib/epee/include/net/levin_client.inl b/contrib/epee/include/net/levin_client.inl index ab7c32c32cb..2f048b02738 100644 --- a/contrib/epee/include/net/levin_client.inl +++ b/contrib/epee/include/net/levin_client.inl @@ -74,16 +74,16 @@ levin_client_impl::~levin_client_impl() } //------------------------------------------------------------------------------ inline -int levin_client_impl::invoke(int command, const std::string& in_buff, std::string& buff_out) +int levin_client_impl::invoke(int command, const epee::span in_buff, std::string& buff_out) { if(!is_connected()) return -1; bucket_head head = {0}; - head.m_signature = LEVIN_SIGNATURE; - head.m_cb = in_buff.size(); + head.m_signature = SWAP64LE(LEVIN_SIGNATURE); + head.m_cb = SWAP64LE(in_buff.size()); head.m_have_to_return_data = true; - head.m_command = command; + head.m_command = SWAP32LE(command); if(!m_transport.send(&head, sizeof(head))) return -1; @@ -97,7 +97,7 @@ int levin_client_impl::invoke(int command, const std::string& in_buff, std::stri head = *(bucket_head*)local_buff.data(); - if(head.m_signature!=LEVIN_SIGNATURE) + if(head.m_signature!=SWAP64LE(LEVIN_SIGNATURE)) { LOG_PRINT_L1("Signature mismatch in response"); return -1; @@ -116,10 +116,10 @@ int levin_client_impl::notify(int command, const std::string& in_buff) return -1; bucket_head head = {0}; - head.m_signature = LEVIN_SIGNATURE; - head.m_cb = in_buff.size(); + head.m_signature = SWAP64LE(LEVIN_SIGNATURE); + head.m_cb = SWAP64LE(in_buff.size()); head.m_have_to_return_data = false; - head.m_command = command; + head.m_command = SWAP32LE(command); if(!m_transport.send((const char*)&head, sizeof(head))) return -1; @@ -133,18 +133,19 @@ int levin_client_impl::notify(int command, const std::string& in_buff) //------------------------------------------------------------------------------ //------------------------------------------------------------------------------ inline - int levin_client_impl2::invoke(int command, const std::string& in_buff, std::string& buff_out) + int levin_client_impl2::invoke(int command, epee::spanstring& in_buff, std::string& buff_out) { if(!is_connected()) return -1; bucket_head2 head = {0}; - head.m_signature = LEVIN_SIGNATURE; - head.m_cb = in_buff.size(); + head.m_signature = SWAP64LE(LEVIN_SIGNATURE); + head.m_cb = SWAP64LE(in_buff.size()); head.m_have_to_return_data = true; - head.m_command = command; - head.m_protocol_version = LEVIN_PROTOCOL_VER_1; - head.m_flags = LEVIN_PACKET_REQUEST; + head.m_command = SWAP32LE(command); + head.m_return_code = SWAP32LE(0); + head.m_flags = SWAP32LE(LEVIN_PACKET_REQUEST); + head.m_protocol_version = SWAP32LE(LEVIN_PROTOCOL_VER_1); if(!m_transport.send(&head, sizeof(head))) return -1; @@ -157,14 +158,13 @@ inline head = *(bucket_head2*)local_buff.data(); - - if(head.m_signature!=LEVIN_SIGNATURE) + if(head.m_signature != SWAP64LE(LEVIN_SIGNATURE)) { LOG_PRINT_L1("Signature mismatch in response"); return -1; } - if(!m_transport.recv_n(buff_out, head.m_cb)) + if(!m_transport.recv_n(buff_out, SWAP64LE(head.m_cb))) return -1; return head.m_return_code; @@ -177,12 +177,13 @@ inline return -1; bucket_head2 head = {0}; - head.m_signature = LEVIN_SIGNATURE; - head.m_cb = in_buff.size(); + head.m_signature = SWAP64LE(LEVIN_SIGNATURE); + head.m_cb = SWAP64LE(in_buff.size()); head.m_have_to_return_data = false; - head.m_command = command; - head.m_protocol_version = LEVIN_PROTOCOL_VER_1; - head.m_flags = LEVIN_PACKET_REQUEST; + head.m_command = SWAP32LE(command); + head.m_return_code = SWAP32LE(0); + head.m_flags = SWAP32LE(LEVIN_PACKET_REQUEST); + head.m_protocol_version = SWAP32LE(LEVIN_PROTOCOL_VER_1); if(!m_transport.send((const char*)&head, sizeof(head))) return -1; diff --git a/contrib/epee/include/net/levin_client_async.h b/contrib/epee/include/net/levin_client_async.h index b3a46465bf0..ed92f4b95d9 100644 --- a/contrib/epee/include/net/levin_client_async.h +++ b/contrib/epee/include/net/levin_client_async.h @@ -431,7 +431,7 @@ namespace levin } CRITICAL_REGION_END(); - LOG_PRINT_L4("LEVIN_PACKET_RECIEVED. [len=" << head.m_cb << ", flags=" << head.m_flags << ", is_cmd=" << head.m_have_to_return_data <<", cmd_id = " << head.m_command << ", pr_v=" << head.m_protocol_version << ", uid=" << string_tools::get_str_from_guid_a(head.m_id) << "]"); + LOG_PRINT_L4("LEVIN_PACKET_RECEIVED. [len=" << head.m_cb << ", flags=" << head.m_flags << ", is_cmd=" << head.m_have_to_return_data <<", cmd_id = " << head.m_command << ", pr_v=" << head.m_protocol_version << ", uid=" << string_tools::get_str_from_guid_a(head.m_id) << "]"); if(is_request) { diff --git a/contrib/epee/include/net/levin_helper.h b/contrib/epee/include/net/levin_helper.h index 05560dd9080..da926a91410 100644 --- a/contrib/epee/include/net/levin_helper.h +++ b/contrib/epee/include/net/levin_helper.h @@ -30,6 +30,7 @@ #include "levin_base.h" #include "serializeble_struct_helper.h" +#include "int-util.h" #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "net" @@ -43,11 +44,11 @@ namespace levin { buff.resize(sizeof(levin::bucket_head)); levin::bucket_head& head = *(levin::bucket_head*)(&buff[0]); - head.m_signature = LEVIN_SIGNATURE; + head.m_signature = SWAP64LE(LEVIN_SIGNATURE); head.m_cb = 0; head.m_have_to_return_data = true; - head.m_command = command_id; - head.m_return_code = 1; + head.m_command = SWAP32LE(command_id); + head.m_return_code = SWAP32LE(1); head.m_reservedA = rand(); //probably some flags in future head.m_reservedB = rand(); //probably some check summ in future @@ -55,7 +56,7 @@ namespace levin if(!StorageNamed::save_struct_as_storage_to_buff_t(t, buff_strg)) return false; - head.m_cb = buff_strg.size(); + head.m_cb = SWAP64LE(buff_strg.size()); buff.append(buff_strg); return true; } @@ -65,15 +66,15 @@ namespace levin { buff.resize(sizeof(levin::bucket_head)); levin::bucket_head& head = *(levin::bucket_head*)(&buff[0]); - head.m_signature = LEVIN_SIGNATURE; + head.m_signature = SWAP64LE(LEVIN_SIGNATURE); head.m_cb = 0; head.m_have_to_return_data = true; - head.m_command = command_id; - head.m_return_code = 1; + head.m_command = SWAP32LE(command_id); + head.m_return_code = SWAP32LE(1); head.m_reservedA = rand(); //probably some flags in future head.m_reservedB = rand(); //probably some check summ in future - head.m_cb = data.size(); + head.m_cb = SWAP64LE(data.size()); buff.append(data); return true; } @@ -86,7 +87,17 @@ namespace levin return false; } - levin::bucket_head& head = *(levin::bucket_head*)(&buff[0]); +#if BYTE_ORDER == LITTLE_ENDIAN + levin::bucket_head &head = *(levin::bucket_head*)(&buff[0]); +#else + levin::bucket_head head = *(levin::bucket_head*)(&buff[0]); + head.m_signature = SWAP64LE(head.m_signature); + head.m_cb = SWAP64LE(head.m_cb); + head.m_command = SWAP32LE(head.m_command); + head.m_return_code = SWAP32LE(head.m_return_code); + head.m_reservedA = SWAP32LE(head.m_reservedA); + head.m_reservedB = SWAP32LE(head.m_reservedB); +#endif if(head.m_signature != LEVIN_SIGNATURE) { LOG_PRINT_L3("Failed to read signature in levin message, at load_struct_from_levin_message"); @@ -113,7 +124,17 @@ namespace levin return false; } - levin::bucket_head& head = *(levin::bucket_head*)(&buff[0]); +#if BYTE_ORDER == LITTLE_ENDIAN + levin::bucket_head &head = *(levin::bucket_head*)(&buff[0]); +#else + levin::bucket_head head = *(levin::bucket_head*)(&buff[0]); + head.m_signature = SWAP64LE(head.m_signature); + head.m_cb = SWAP64LE(head.m_cb); + head.m_command = SWAP32LE(head.m_command); + head.m_return_code = SWAP32LE(head.m_return_code); + head.m_reservedA = SWAP32LE(head.m_reservedA); + head.m_reservedB = SWAP32LE(head.m_reservedB); +#endif if(head.m_signature != LEVIN_SIGNATURE) { LOG_ERROR("Failed to read signature in levin message, at load_struct_from_levin_message"); diff --git a/contrib/epee/include/net/levin_protocol_handler.h b/contrib/epee/include/net/levin_protocol_handler.h index b3a75bedc55..791766762a9 100644 --- a/contrib/epee/include/net/levin_protocol_handler.h +++ b/contrib/epee/include/net/levin_protocol_handler.h @@ -31,6 +31,7 @@ #include #include "levin_base.h" +#include "int-util.h" #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "net" @@ -103,7 +104,7 @@ namespace levin case conn_state_reading_head: if(m_cach_in_buffer.size() < sizeof(bucket_head)) { - if(m_cach_in_buffer.size() >= sizeof(uint64_t) && *((uint64_t*)m_cach_in_buffer.data()) != LEVIN_SIGNATURE) + if(m_cach_in_buffer.size() >= sizeof(uint64_t) && *((uint64_t*)m_cach_in_buffer.data()) != SWAP64LE(LEVIN_SIGNATURE)) { LOG_ERROR_CC(m_conn_context, "Signature mismatch on accepted connection"); return false; @@ -112,13 +113,23 @@ namespace levin break; } { - bucket_head* phead = (bucket_head*)m_cach_in_buffer.data(); - if(LEVIN_SIGNATURE != phead->m_signature) +#if BYTE_ORDER == LITTLE_ENDIAN + bucket_head &phead = *(bucket_head*)m_cach_in_buffer.data(); +#else + bucket_head phead = *(bucket_head*)m_cach_in_buffer.data(); + phead.m_signature = SWAP64LE(phead.m_signature); + phead.m_cb = SWAP64LE(phead.m_cb); + phead.m_command = SWAP32LE(phead.m_command); + phead.m_return_code = SWAP32LE(phead.m_return_code); + phead.m_reservedA = SWAP32LE(phead.m_reservedA); + phead.m_reservedB = SWAP32LE(phead.m_reservedB); +#endif + if(LEVIN_SIGNATURE != phead.m_signature) { LOG_ERROR_CC(m_conn_context, "Signature mismatch on accepted connection"); return false; } - m_current_head = *phead; + m_current_head = phead; } m_cach_in_buffer.erase(0, sizeof(bucket_head)); m_state = conn_state_reading_body; diff --git a/contrib/epee/include/net/levin_protocol_handler_async.h b/contrib/epee/include/net/levin_protocol_handler_async.h index 0b1fe05fa1d..116b3ace152 100644 --- a/contrib/epee/include/net/levin_protocol_handler_async.h +++ b/contrib/epee/include/net/levin_protocol_handler_async.h @@ -34,9 +34,11 @@ #include #include "levin_base.h" +#include "buffer.h" #include "misc_language.h" #include "syncobj.h" #include "misc_os_dependent.h" +#include "int-util.h" #include #include @@ -84,11 +86,11 @@ class async_protocol_handler_config uint64_t m_max_packet_size; uint64_t m_invoke_timeout; - int invoke(int command, const std::string& in_buff, std::string& buff_out, boost::uuids::uuid connection_id); + int invoke(int command, const epee::span in_buff, std::string& buff_out, boost::uuids::uuid connection_id); template - int invoke_async(int command, const std::string& in_buff, boost::uuids::uuid connection_id, const callback_t &cb, size_t timeout = LEVIN_DEFAULT_TIMEOUT_PRECONFIGURED); + int invoke_async(int command, const epee::span in_buff, boost::uuids::uuid connection_id, const callback_t &cb, size_t timeout = LEVIN_DEFAULT_TIMEOUT_PRECONFIGURED); - int notify(int command, const std::string& in_buff, boost::uuids::uuid connection_id); + int notify(int command, const epee::span in_buff, boost::uuids::uuid connection_id); bool close(boost::uuids::uuid connection_id); bool update_connection_context(const t_connection_context& contxt); bool request_callback(boost::uuids::uuid connection_id); @@ -99,7 +101,7 @@ class async_protocol_handler_config size_t get_connections_count(); void set_handler(levin_commands_handler* handler, void (*destroy)(levin_commands_handler*) = NULL); - async_protocol_handler_config():m_pcommands_handler(NULL), m_pcommands_handler_destroy(NULL), m_max_packet_size(LEVIN_DEFAULT_MAX_PACKET_SIZE) + async_protocol_handler_config():m_pcommands_handler(NULL), m_pcommands_handler_destroy(NULL), m_max_packet_size(LEVIN_DEFAULT_MAX_PACKET_SIZE), m_invoke_timeout(LEVIN_DEFAULT_TIMEOUT_PRECONFIGURED) {} ~async_protocol_handler_config() { set_handler(NULL, NULL); } void del_out_connections(size_t count); @@ -142,7 +144,7 @@ class async_protocol_handler config_type& m_config; t_connection_context& m_connection_context; - std::string m_cache_in_buffer; + net_utils::buffer m_cache_in_buffer; stream_state m_state; int32_t m_oponent_protocol_ver; @@ -150,7 +152,7 @@ class async_protocol_handler struct invoke_response_handler_base { - virtual bool handle(int res, const std::string& buff, connection_context& context)=0; + virtual bool handle(int res, const epee::span buff, connection_context& context)=0; virtual bool is_timer_started() const=0; virtual void cancel()=0; virtual bool cancel_timer()=0; @@ -172,7 +174,7 @@ class async_protocol_handler if(ec == boost::asio::error::operation_aborted) return; MINFO(con.get_context_ref() << "Timeout on invoke operation happened, command: " << command << " timeout: " << timeout); - std::string fake; + epee::span fake; cb(LEVIN_ERROR_CONNECTION_TIMEDOUT, fake, con.get_context_ref()); con.close(); con.finish_outer_call(); @@ -190,7 +192,7 @@ class async_protocol_handler bool m_timer_cancelled; uint64_t m_timeout; int m_command; - virtual bool handle(int res, const std::string& buff, typename async_protocol_handler::connection_context& context) + virtual bool handle(int res, const epee::span buff, typename async_protocol_handler::connection_context& context) { if(!cancel_timer()) return false; @@ -206,7 +208,7 @@ class async_protocol_handler { if(cancel_timer()) { - std::string fake; + epee::span fake; m_cb(LEVIN_ERROR_CONNECTION_DESTROYED, fake, m_con.get_context_ref()); m_con.finish_outer_call(); } @@ -236,7 +238,7 @@ class async_protocol_handler if(ec == boost::asio::error::operation_aborted) return; MINFO(con.get_context_ref() << "Timeout on invoke operation happened, command: " << command << " timeout: " << timeout); - std::string fake; + epee::span fake; cb(LEVIN_ERROR_CONNECTION_TIMEDOUT, fake, con.get_context_ref()); con.close(); con.finish_outer_call(); @@ -264,6 +266,7 @@ class async_protocol_handler m_pservice_endpoint(psnd_hndlr), m_config(config), m_connection_context(conn_context), + m_cache_in_buffer(4 * 1024), m_state(stream_state_head) { m_close_called = 0; @@ -272,9 +275,14 @@ class async_protocol_handler m_wait_count = 0; m_oponent_protocol_ver = 0; m_connection_initialized = false; + m_invoke_buf_ready = 0; + m_invoke_result_code = LEVIN_ERROR_CONNECTION; } virtual ~async_protocol_handler() { + try + { + m_deletion_initiated = true; if(m_connection_initialized) { @@ -288,6 +296,9 @@ class async_protocol_handler CHECK_AND_ASSERT_MES_NO_RET(0 == boost::interprocess::ipcdetail::atomic_read32(&m_wait_count), "Failed to wait for operation completion. m_wait_count = " << m_wait_count); MTRACE(m_connection_context << "~async_protocol_handler()"); + + } + catch (...) { /* ignore */ } } bool start_outer_call() @@ -396,18 +407,11 @@ class async_protocol_handler break; } { - std::string buff_to_invoke; - if(m_cache_in_buffer.size() == m_current_head.m_cb) - buff_to_invoke.swap(m_cache_in_buffer); - else - { - buff_to_invoke.assign(m_cache_in_buffer, 0, (std::string::size_type)m_current_head.m_cb); - m_cache_in_buffer.erase(0, (std::string::size_type)m_current_head.m_cb); - } + epee::span buff_to_invoke = m_cache_in_buffer.carve((std::string::size_type)m_current_head.m_cb); bool is_response = (m_oponent_protocol_ver == LEVIN_PROTOCOL_VER_1 && m_current_head.m_flags&LEVIN_PACKET_RESPONSE); - MDEBUG(m_connection_context << "LEVIN_PACKET_RECIEVED. [len=" << m_current_head.m_cb + MDEBUG(m_connection_context << "LEVIN_PACKET_RECEIVED. [len=" << m_current_head.m_cb << ", flags" << m_current_head.m_flags << ", r?=" << m_current_head.m_have_to_return_data <<", cmd = " << m_current_head.m_command @@ -440,8 +444,8 @@ class async_protocol_handler }else { CRITICAL_REGION_BEGIN(m_local_inv_buff_lock); - buff_to_invoke.swap(m_local_inv_buff); - buff_to_invoke.clear(); + m_local_inv_buff = std::string((const char*)buff_to_invoke.data(), buff_to_invoke.size()); + buff_to_invoke = epee::span((const uint8_t*)NULL, 0); m_invoke_result_code = m_current_head.m_return_code; CRITICAL_REGION_END(); boost::interprocess::ipcdetail::atomic_write32(&m_invoke_buf_ready, 1); @@ -461,7 +465,18 @@ class async_protocol_handler m_current_head.m_have_to_return_data = false; m_current_head.m_protocol_version = LEVIN_PROTOCOL_VER_1; m_current_head.m_flags = LEVIN_PACKET_RESPONSE; +#if BYTE_ORDER == LITTLE_ENDIAN std::string send_buff((const char*)&m_current_head, sizeof(m_current_head)); +#else + bucket_head2 head = m_current_head; + head.m_signature = SWAP64LE(head.m_signature); + head.m_cb = SWAP64LE(head.m_cb); + head.m_command = SWAP32LE(head.m_command); + head.m_return_code = SWAP32LE(head.m_return_code); + head.m_flags = SWAP32LE(head.m_flags); + head.m_protocol_version = SWAP32LE(head.m_protocol_version); + std::string send_buff((const char*)&head, sizeof(head)); +#endif send_buff += return_buff; CRITICAL_REGION_BEGIN(m_send_lock); if(!m_pservice_endpoint->do_send(send_buff.data(), send_buff.size())) @@ -483,7 +498,7 @@ class async_protocol_handler { if(m_cache_in_buffer.size() < sizeof(bucket_head2)) { - if(m_cache_in_buffer.size() >= sizeof(uint64_t) && *((uint64_t*)m_cache_in_buffer.data()) != LEVIN_SIGNATURE) + if(m_cache_in_buffer.size() >= sizeof(uint64_t) && *((uint64_t*)m_cache_in_buffer.span(8).data()) != SWAP64LE(LEVIN_SIGNATURE)) { MWARNING(m_connection_context << "Signature mismatch, connection will be closed"); return false; @@ -492,15 +507,25 @@ class async_protocol_handler break; } - bucket_head2* phead = (bucket_head2*)m_cache_in_buffer.data(); - if(LEVIN_SIGNATURE != phead->m_signature) +#if BYTE_ORDER == LITTLE_ENDIAN + bucket_head2& phead = *(bucket_head2*)m_cache_in_buffer.span(sizeof(bucket_head2)).data(); +#else + bucket_head2 phead = *(bucket_head2*)m_cache_in_buffer.span(sizeof(bucket_head2)).data(); + phead.m_signature = SWAP64LE(phead.m_signature); + phead.m_cb = SWAP64LE(phead.m_cb); + phead.m_command = SWAP32LE(phead.m_command); + phead.m_return_code = SWAP32LE(phead.m_return_code); + phead.m_flags = SWAP32LE(phead.m_flags); + phead.m_protocol_version = SWAP32LE(phead.m_protocol_version); +#endif + if(LEVIN_SIGNATURE != phead.m_signature) { LOG_ERROR_CC(m_connection_context, "Signature mismatch, connection will be closed"); return false; } - m_current_head = *phead; + m_current_head = phead; - m_cache_in_buffer.erase(0, sizeof(bucket_head2)); + m_cache_in_buffer.erase(sizeof(bucket_head2)); m_state = stream_state_body; m_oponent_protocol_ver = m_current_head.m_protocol_version; if(m_current_head.m_cb > m_config.m_max_packet_size) @@ -532,7 +557,7 @@ class async_protocol_handler } template - bool async_invoke(int command, const std::string& in_buff, const callback_t &cb, size_t timeout = LEVIN_DEFAULT_TIMEOUT_PRECONFIGURED) + bool async_invoke(int command, const epee::span in_buff, const callback_t &cb, size_t timeout = LEVIN_DEFAULT_TIMEOUT_PRECONFIGURED) { misc_utils::auto_scope_leave_caller scope_exit_handler = misc_utils::create_scope_leave_handler( boost::bind(&async_protocol_handler::finish_outer_call, this)); @@ -558,13 +583,13 @@ class async_protocol_handler } bucket_head2 head = {0}; - head.m_signature = LEVIN_SIGNATURE; - head.m_cb = in_buff.size(); + head.m_signature = SWAP64LE(LEVIN_SIGNATURE); + head.m_cb = SWAP64LE(in_buff.size()); head.m_have_to_return_data = true; - head.m_flags = LEVIN_PACKET_REQUEST; - head.m_command = command; - head.m_protocol_version = LEVIN_PROTOCOL_VER_1; + head.m_flags = SWAP32LE(LEVIN_PACKET_REQUEST); + head.m_command = SWAP32LE(command); + head.m_protocol_version = SWAP32LE(LEVIN_PROTOCOL_VER_1); boost::interprocess::ipcdetail::atomic_write32(&m_invoke_buf_ready, 0); CRITICAL_REGION_BEGIN(m_send_lock); @@ -576,7 +601,7 @@ class async_protocol_handler break; } - if(!m_pservice_endpoint->do_send(in_buff.data(), (int)in_buff.size())) + if(!m_pservice_endpoint->do_send(in_buff.data(), in_buff.size())) { LOG_ERROR_CC(m_connection_context, "Failed to do_send"); err_code = LEVIN_ERROR_CONNECTION; @@ -593,7 +618,7 @@ class async_protocol_handler if (LEVIN_OK != err_code) { - std::string stub_buff; + epee::span stub_buff{(const uint8_t*)"", 0}; // Never call callback inside critical section, that can cause deadlock cb(err_code, stub_buff, m_connection_context); return false; @@ -602,7 +627,7 @@ class async_protocol_handler return true; } - int invoke(int command, const std::string& in_buff, std::string& buff_out) + int invoke(int command, const epee::span in_buff, std::string& buff_out) { misc_utils::auto_scope_leave_caller scope_exit_handler = misc_utils::create_scope_leave_handler( boost::bind(&async_protocol_handler::finish_outer_call, this)); @@ -616,13 +641,13 @@ class async_protocol_handler return LEVIN_ERROR_CONNECTION_DESTROYED; bucket_head2 head = {0}; - head.m_signature = LEVIN_SIGNATURE; - head.m_cb = in_buff.size(); + head.m_signature = SWAP64LE(LEVIN_SIGNATURE); + head.m_cb = SWAP64LE(in_buff.size()); head.m_have_to_return_data = true; - head.m_flags = LEVIN_PACKET_REQUEST; - head.m_command = command; - head.m_protocol_version = LEVIN_PROTOCOL_VER_1; + head.m_flags = SWAP32LE(LEVIN_PACKET_REQUEST); + head.m_command = SWAP32LE(command); + head.m_protocol_version = SWAP32LE(LEVIN_PROTOCOL_VER_1); boost::interprocess::ipcdetail::atomic_write32(&m_invoke_buf_ready, 0); CRITICAL_REGION_BEGIN(m_send_lock); @@ -632,7 +657,7 @@ class async_protocol_handler return LEVIN_ERROR_CONNECTION; } - if(!m_pservice_endpoint->do_send(in_buff.data(), (int)in_buff.size())) + if(!m_pservice_endpoint->do_send(in_buff.data(), in_buff.size())) { LOG_ERROR_CC(m_connection_context, "Failed to do_send"); return LEVIN_ERROR_CONNECTION; @@ -676,7 +701,7 @@ class async_protocol_handler return m_invoke_result_code; } - int notify(int command, const std::string& in_buff) + int notify(int command, const epee::span in_buff) { misc_utils::auto_scope_leave_caller scope_exit_handler = misc_utils::create_scope_leave_handler( boost::bind(&async_protocol_handler::finish_outer_call, this)); @@ -690,13 +715,13 @@ class async_protocol_handler return LEVIN_ERROR_CONNECTION_DESTROYED; bucket_head2 head = {0}; - head.m_signature = LEVIN_SIGNATURE; + head.m_signature = SWAP64LE(LEVIN_SIGNATURE); head.m_have_to_return_data = false; - head.m_cb = in_buff.size(); + head.m_cb = SWAP64LE(in_buff.size()); - head.m_command = command; - head.m_protocol_version = LEVIN_PROTOCOL_VER_1; - head.m_flags = LEVIN_PACKET_REQUEST; + head.m_command = SWAP32LE(command); + head.m_protocol_version = SWAP32LE(LEVIN_PROTOCOL_VER_1); + head.m_flags = SWAP32LE(LEVIN_PACKET_REQUEST); CRITICAL_REGION_BEGIN(m_send_lock); if(!m_pservice_endpoint->do_send(&head, sizeof(head))) { @@ -704,7 +729,7 @@ class async_protocol_handler return -1; } - if(!m_pservice_endpoint->do_send(in_buff.data(), (int)in_buff.size())) + if(!m_pservice_endpoint->do_send(in_buff.data(), in_buff.size())) { LOG_ERROR_CC(m_connection_context, "Failed to do_send()"); return -1; @@ -809,7 +834,7 @@ int async_protocol_handler_config::find_and_lock_connectio } //------------------------------------------------------------------------------------------ template -int async_protocol_handler_config::invoke(int command, const std::string& in_buff, std::string& buff_out, boost::uuids::uuid connection_id) +int async_protocol_handler_config::invoke(int command, const epee::span in_buff, std::string& buff_out, boost::uuids::uuid connection_id) { async_protocol_handler* aph; int r = find_and_lock_connection(connection_id, aph); @@ -817,7 +842,7 @@ int async_protocol_handler_config::invoke(int command, con } //------------------------------------------------------------------------------------------ template template -int async_protocol_handler_config::invoke_async(int command, const std::string& in_buff, boost::uuids::uuid connection_id, const callback_t &cb, size_t timeout) +int async_protocol_handler_config::invoke_async(int command, const epee::span in_buff, boost::uuids::uuid connection_id, const callback_t &cb, size_t timeout) { async_protocol_handler* aph; int r = find_and_lock_connection(connection_id, aph); @@ -866,7 +891,7 @@ void async_protocol_handler_config::set_handler(levin_comm } //------------------------------------------------------------------------------------------ template -int async_protocol_handler_config::notify(int command, const std::string& in_buff, boost::uuids::uuid connection_id) +int async_protocol_handler_config::notify(int command, const epee::span in_buff, boost::uuids::uuid connection_id) { async_protocol_handler* aph; int r = find_and_lock_connection(connection_id, aph); diff --git a/contrib/epee/include/net/local_ip.h b/contrib/epee/include/net/local_ip.h index 0d458963c62..52c5855b9bf 100644 --- a/contrib/epee/include/net/local_ip.h +++ b/contrib/epee/include/net/local_ip.h @@ -48,7 +48,7 @@ namespace epee if( (ip | 0xffffff00) == 0xffffffac) { - uint32_t second_num = (ip << 8) & 0xff000000; + uint32_t second_num = (ip >> 8) & 0xff; if(second_num >= 16 && second_num <= 31 ) return true; } diff --git a/contrib/epee/include/net/net_fwd.h b/contrib/epee/include/net/net_fwd.h new file mode 100644 index 00000000000..ba4fe62592e --- /dev/null +++ b/contrib/epee/include/net/net_fwd.h @@ -0,0 +1,38 @@ +// Copyright (c) 2019, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +namespace epee +{ + namespace net_utils + { + struct ssl_authentication_t; + class ssl_options_t; + } +} diff --git a/contrib/epee/include/net/net_helper.h b/contrib/epee/include/net/net_helper.h index 2c2efcd82df..e315555fc89 100644 --- a/contrib/epee/include/net/net_helper.h +++ b/contrib/epee/include/net/net_helper.h @@ -33,13 +33,19 @@ //#include #include #include -#include +#include +#include +#include #include #include +#include #include #include #include +#include +#include #include "net/net_utils_base.h" +#include "net/net_ssl.h" #include "misc_language.h" #undef MONERO_DEFAULT_LOG_CATEGORY @@ -54,9 +60,22 @@ namespace epee { namespace net_utils { + struct direct_connect + { + boost::unique_future + operator()(const std::string& addr, const std::string& port, boost::asio::steady_timer&) const; + }; + class blocked_mode_client { + enum try_connect_result_t + { + CONNECT_SUCCESS, + CONNECT_FAILURE, + CONNECT_NO_SSL, + }; + struct handler_obj @@ -77,118 +96,140 @@ namespace net_utils ref_bytes_transferred = bytes_transferred; } }; - + public: inline - blocked_mode_client():m_initialized(false), - m_connected(false), - m_deadline(m_io_service), - m_shutdowned(0), - m_ssl(false), - m_ctx(boost::asio::ssl::context::sslv23), - m_ssl_socket(m_io_service,m_ctx) + blocked_mode_client() : + m_io_service(), + m_ctx(boost::asio::ssl::context::tlsv12), + m_connector(direct_connect{}), + m_ssl_socket(new boost::asio::ssl::stream(m_io_service, m_ctx)), + m_ssl_options(epee::net_utils::ssl_support_t::e_ssl_support_autodetect), + m_initialized(true), + m_connected(false), + m_deadline(m_io_service), + m_shutdowned(0), + m_bytes_sent(0), + m_bytes_received(0) { - - - m_initialized = true; + } + /*! The first/second parameters are host/port respectively. The third + parameter is for setting the timeout callback - the timer is + already set by the caller, the callee only needs to set the + behavior. - // No deadline is required until the first socket operation is started. We - // set the deadline to positive infinity so that the actor takes no action - // until a specific deadline is set. - m_deadline.expires_at(std::chrono::steady_clock::time_point::max()); + Additional asynchronous operations should be queued using the + `io_service` from the timer. The implementation should assume + multi-threaded I/O processing. - // Start the persistent actor that checks for deadline expiry. - check_deadline(); + If the callee cannot start an asynchronous operation, an exception + should be thrown to signal an immediate failure. + + The return value is a future to a connected socket. Asynchronous + failures should use the `set_exception` method. */ + using connect_func = boost::unique_future(const std::string&, const std::string&, boost::asio::steady_timer&); - } inline ~blocked_mode_client() { //profile_tools::local_coast lc("~blocked_mode_client()", 3); - shutdown(); + try { shutdown(); } + catch(...) { /* ignore */ } + } + + inline void set_ssl(ssl_options_t ssl_options) + { + if (ssl_options) + m_ctx = ssl_options.create_context(); + else + m_ctx = boost::asio::ssl::context(boost::asio::ssl::context::tlsv12); + m_ssl_options = std::move(ssl_options); } inline - bool connect(const std::string& addr, int port, std::chrono::milliseconds timeout, bool ssl = false, const std::string& bind_ip = "0.0.0.0") + bool connect(const std::string& addr, int port, std::chrono::milliseconds timeout) { - return connect(addr, std::to_string(port), timeout, ssl, bind_ip); + return connect(addr, std::to_string(port), timeout); } inline - bool connect(const std::string& addr, const std::string& port, std::chrono::milliseconds timeout, bool ssl = false, const std::string& bind_ip = "0.0.0.0") + try_connect_result_t try_connect(const std::string& addr, const std::string& port, std::chrono::milliseconds timeout, epee::net_utils::ssl_support_t ssl_support) { - m_connected = false; - m_ssl = ssl; - try - { - m_ssl_socket.next_layer().close(); - - // Set SSL options - // disable sslv2 - m_ctx.set_options(boost::asio::ssl::context::default_workarounds | boost::asio::ssl::context::no_sslv2); - m_ctx.set_default_verify_paths(); - - // Get a list of endpoints corresponding to the server name. - - - ////////////////////////////////////////////////////////////////////////// - - boost::asio::ip::tcp::resolver resolver(m_io_service); - boost::asio::ip::tcp::resolver::query query(boost::asio::ip::tcp::v4(), addr, port, boost::asio::ip::tcp::resolver::query::canonical_name); - boost::asio::ip::tcp::resolver::iterator iterator = resolver.resolve(query); - boost::asio::ip::tcp::resolver::iterator end; - if(iterator == end) + m_deadline.expires_from_now(timeout); + boost::unique_future connection = m_connector(addr, port, m_deadline); + for (;;) { - LOG_ERROR("Failed to resolve " << addr); - return false; - } - - ////////////////////////////////////////////////////////////////////////// - - - //boost::asio::ip::tcp::endpoint remote_endpoint(boost::asio::ip::address::from_string(addr.c_str()), port); - boost::asio::ip::tcp::endpoint remote_endpoint(*iterator); - + m_io_service.reset(); + m_io_service.run_one(); - m_ssl_socket.next_layer().open(remote_endpoint.protocol()); - if(bind_ip != "0.0.0.0" && bind_ip != "0" && bind_ip != "" ) - { - boost::asio::ip::tcp::endpoint local_endpoint(boost::asio::ip::address::from_string(addr.c_str()), 0); - m_ssl_socket.next_layer().bind(local_endpoint); + if (connection.is_ready()) + break; } - - m_deadline.expires_from_now(timeout); - - - boost::system::error_code ec = boost::asio::error::would_block; - - m_ssl_socket.next_layer().async_connect(remote_endpoint, boost::lambda::var(ec) = boost::lambda::_1); - while (ec == boost::asio::error::would_block) - { - m_io_service.run_one(); - } - - if (!ec && m_ssl_socket.next_layer().is_open()) + m_ssl_socket->next_layer() = connection.get(); + m_deadline.cancel(); + if (m_ssl_socket->next_layer().is_open()) { m_connected = true; m_deadline.expires_at(std::chrono::steady_clock::time_point::max()); // SSL Options - if(m_ssl) { - // Disable verification of host certificate - m_ssl_socket.set_verify_mode(boost::asio::ssl::verify_peer); - // Handshake - m_ssl_socket.next_layer().set_option(boost::asio::ip::tcp::no_delay(true)); - m_ssl_socket.handshake(boost::asio::ssl::stream_base::client); + if (ssl_support == epee::net_utils::ssl_support_t::e_ssl_support_enabled || ssl_support == epee::net_utils::ssl_support_t::e_ssl_support_autodetect) + { + if (!m_ssl_options.handshake(*m_ssl_socket, boost::asio::ssl::stream_base::client, addr)) + { + if (ssl_support == epee::net_utils::ssl_support_t::e_ssl_support_autodetect) + { + boost::system::error_code ignored_ec; + m_ssl_socket->next_layer().shutdown(boost::asio::ip::tcp::socket::shutdown_both, ignored_ec); + m_ssl_socket->next_layer().close(); + m_connected = false; + return CONNECT_NO_SSL; + } + else + { + MWARNING("Failed to establish SSL connection"); + m_connected = false; + return CONNECT_FAILURE; + } + } } - return true; + return CONNECT_SUCCESS; }else { - MWARNING("Some problems at connect, message: " << ec.message()); - return false; + MWARNING("Some problems at connect, expected open socket"); + return CONNECT_FAILURE; } + } + + inline + bool connect(const std::string& addr, const std::string& port, std::chrono::milliseconds timeout) + { + m_connected = false; + try + { + m_ssl_socket->next_layer().close(); + + // Set SSL options + // disable sslv2 + m_ssl_socket.reset(new boost::asio::ssl::stream(m_io_service, m_ctx)); + + // Get a list of endpoints corresponding to the server name. + + try_connect_result_t try_connect_result = try_connect(addr, port, timeout, m_ssl_options.support); + if (try_connect_result == CONNECT_FAILURE) + return false; + if (m_ssl_options.support == epee::net_utils::ssl_support_t::e_ssl_support_autodetect) + { + if (try_connect_result == CONNECT_NO_SSL) + { + MERROR("SSL handshake failed on an autodetect connection, reconnecting without SSL"); + m_ssl_options.support = epee::net_utils::ssl_support_t::e_ssl_support_disabled; + if (try_connect(addr, port, timeout, m_ssl_options.support) != CONNECT_SUCCESS) + return false; + } + } } catch(const boost::system::system_error& er) { @@ -203,6 +244,11 @@ namespace net_utils return true; } + //! Change the connection routine (proxy, etc.) + void set_connector(std::function connector) + { + m_connector = std::move(connector); + } inline bool disconnect() @@ -212,12 +258,11 @@ namespace net_utils if(m_connected) { m_connected = false; - if(m_ssl) + if(m_ssl_options) shutdown_ssl(); - m_ssl_socket.next_layer().shutdown(boost::asio::ip::tcp::socket::shutdown_both); + m_ssl_socket->next_layer().shutdown(boost::asio::ip::tcp::socket::shutdown_both); } } - catch(const boost::system::system_error& /*er*/) { //LOG_ERROR("Some problems at disconnect, message: " << er.what()); @@ -256,6 +301,7 @@ namespace net_utils // Block until the asynchronous operation has completed. while (ec == boost::asio::error::would_block) { + m_io_service.reset(); m_io_service.run_one(); } @@ -267,6 +313,7 @@ namespace net_utils }else { m_deadline.expires_at(std::chrono::steady_clock::time_point::max()); + m_bytes_sent += buff.size(); } } @@ -323,6 +370,7 @@ namespace net_utils }else { m_deadline.expires_at(std::chrono::steady_clock::time_point::max()); + m_bytes_sent += sz; } } @@ -341,9 +389,13 @@ namespace net_utils return true; } - bool is_connected() + bool is_connected(bool *ssl = NULL) { - return m_connected && m_ssl_socket.next_layer().is_open(); + if (!m_connected || !m_ssl_socket->next_layer().is_open()) + return false; + if (ssl) + *ssl = m_ssl_options.support != ssl_support_t::e_ssl_support_disabled; + return true; } inline @@ -374,13 +426,15 @@ namespace net_utils handler_obj hndlr(ec, bytes_transfered); - char local_buff[10000] = {0}; + static const size_t max_size = 16384; + buff.resize(max_size); - async_read(local_buff, sizeof(local_buff), boost::asio::transfer_at_least(1), hndlr); + async_read(&buff[0], max_size, boost::asio::transfer_at_least(1), hndlr); // Block until the asynchronous operation has completed. while (ec == boost::asio::error::would_block && !boost::interprocess::ipcdetail::atomic_read32(&m_shutdowned)) { + m_io_service.reset(); m_io_service.run_one(); } @@ -392,6 +446,7 @@ namespace net_utils { MTRACE("Connection err_code eof."); //connection closed there, empty + buff.clear(); return true; } @@ -407,7 +462,8 @@ namespace net_utils /*if(!bytes_transfered) return false;*/ - buff.assign(local_buff, bytes_transfered); + m_bytes_received += bytes_transfered; + buff.resize(bytes_transfered); return true; } @@ -475,6 +531,7 @@ namespace net_utils m_deadline.expires_at(std::chrono::steady_clock::time_point::max()); } + m_bytes_received += bytes_transfered; if(bytes_transfered != buff.size()) { LOG_ERROR("Transferred mismatch with transfer_at_least value: m_bytes_transferred=" << bytes_transfered << " at_least value=" << buff.size()); @@ -505,15 +562,15 @@ namespace net_utils { m_deadline.cancel(); boost::system::error_code ec; - if(m_ssl) + if(m_ssl_options.support != ssl_support_t::e_ssl_support_disabled) shutdown_ssl(); - m_ssl_socket.next_layer().cancel(ec); + m_ssl_socket->next_layer().cancel(ec); if(ec) MDEBUG("Problems at cancel: " << ec.message()); - m_ssl_socket.next_layer().shutdown(boost::asio::ip::tcp::socket::shutdown_both, ec); + m_ssl_socket->next_layer().shutdown(boost::asio::ip::tcp::socket::shutdown_both, ec); if(ec) MDEBUG("Problems at shutdown: " << ec.message()); - m_ssl_socket.next_layer().close(ec); + m_ssl_socket->next_layer().close(ec); if(ec) MDEBUG("Problems at close: " << ec.message()); boost::interprocess::ipcdetail::atomic_write32(&m_shutdowned, 1); @@ -521,10 +578,6 @@ namespace net_utils return true; } - void set_connected(bool connected) - { - m_connected = connected; - } boost::asio::io_service& get_io_service() { return m_io_service; @@ -532,9 +585,19 @@ namespace net_utils boost::asio::ip::tcp::socket& get_socket() { - return m_ssl_socket.next_layer(); + return m_ssl_socket->next_layer(); } - + + uint64_t get_bytes_sent() const + { + return m_bytes_sent; + } + + uint64_t get_bytes_received() const + { + return m_bytes_received; + } + private: void check_deadline() @@ -549,7 +612,7 @@ namespace net_utils // connect(), read_line() or write_line() functions to return. LOG_PRINT_L3("Timed out socket"); m_connected = false; - m_ssl_socket.next_layer().close(); + m_ssl_socket->next_layer().close(); // There is no longer an active deadline. The expiry is set to positive // infinity so that the actor takes no action until a new deadline is set. @@ -564,9 +627,10 @@ namespace net_utils // ssl socket shutdown blocks if server doesn't respond. We close after 2 secs boost::system::error_code ec = boost::asio::error::would_block; m_deadline.expires_from_now(std::chrono::milliseconds(2000)); - m_ssl_socket.async_shutdown(boost::lambda::var(ec) = boost::lambda::_1); + m_ssl_socket->async_shutdown(boost::lambda::var(ec) = boost::lambda::_1); while (ec == boost::asio::error::would_block) { + m_io_service.reset(); m_io_service.run_one(); } // Ignore "short read" error @@ -585,39 +649,42 @@ namespace net_utils bool write(const void* data, size_t sz, boost::system::error_code& ec) { bool success; - if(m_ssl) - success = boost::asio::write(m_ssl_socket, boost::asio::buffer(data, sz), ec); + if(m_ssl_options.support != ssl_support_t::e_ssl_support_disabled) + success = boost::asio::write(*m_ssl_socket, boost::asio::buffer(data, sz), ec); else - success = boost::asio::write(m_ssl_socket.next_layer(), boost::asio::buffer(data, sz), ec); + success = boost::asio::write(m_ssl_socket->next_layer(), boost::asio::buffer(data, sz), ec); return success; } void async_write(const void* data, size_t sz, boost::system::error_code& ec) { - if(m_ssl) - boost::asio::async_write(m_ssl_socket, boost::asio::buffer(data, sz), boost::lambda::var(ec) = boost::lambda::_1); + if(m_ssl_options.support != ssl_support_t::e_ssl_support_disabled) + boost::asio::async_write(*m_ssl_socket, boost::asio::buffer(data, sz), boost::lambda::var(ec) = boost::lambda::_1); else - boost::asio::async_write(m_ssl_socket.next_layer(), boost::asio::buffer(data, sz), boost::lambda::var(ec) = boost::lambda::_1); + boost::asio::async_write(m_ssl_socket->next_layer(), boost::asio::buffer(data, sz), boost::lambda::var(ec) = boost::lambda::_1); } void async_read(char* buff, size_t sz, boost::asio::detail::transfer_at_least_t transfer_at_least, handler_obj& hndlr) { - if(!m_ssl) - boost::asio::async_read(m_ssl_socket.next_layer(), boost::asio::buffer(buff, sz), transfer_at_least, hndlr); + if(m_ssl_options.support == ssl_support_t::e_ssl_support_disabled) + boost::asio::async_read(m_ssl_socket->next_layer(), boost::asio::buffer(buff, sz), transfer_at_least, hndlr); else - boost::asio::async_read(m_ssl_socket, boost::asio::buffer(buff, sz), transfer_at_least, hndlr); + boost::asio::async_read(*m_ssl_socket, boost::asio::buffer(buff, sz), transfer_at_least, hndlr); } protected: boost::asio::io_service m_io_service; boost::asio::ssl::context m_ctx; - boost::asio::ssl::stream m_ssl_socket; - bool m_ssl; + std::shared_ptr> m_ssl_socket; + std::function m_connector; + ssl_options_t m_ssl_options; bool m_initialized; bool m_connected; boost::asio::steady_timer m_deadline; volatile uint32_t m_shutdowned; + std::atomic m_bytes_sent; + std::atomic m_bytes_received; }; @@ -721,7 +788,7 @@ namespace net_utils // asynchronous operations are cancelled. This allows the blocked // connect(), read_line() or write_line() functions to return. LOG_PRINT_L3("Timed out socket"); - m_ssl_socket.next_layer().close(); + m_ssl_socket->next_layer().close(); // There is no longer an active deadline. The expiry is set to positive // infinity so that the actor takes no action until a new deadline is set. @@ -734,3 +801,4 @@ namespace net_utils }; } } + diff --git a/contrib/epee/include/net/net_ssl.h b/contrib/epee/include/net/net_ssl.h new file mode 100644 index 00000000000..3a97dfdaf97 --- /dev/null +++ b/contrib/epee/include/net/net_ssl.h @@ -0,0 +1,144 @@ +// Copyright (c) 2006-2013, Andrey N. Sabelnikov, www.sabelnikov.net +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of the Andrey N. Sabelnikov nor the +// names of its contributors may be used to endorse or promote products +// derived from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER BE LIABLE FOR ANY +// DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + + + +#ifndef _NET_SSL_H +#define _NET_SSL_H + +#include +#include +#include +#include +#include +#include +#include + +#define SSL_FINGERPRINT_SIZE 32 + +namespace epee +{ +namespace net_utils +{ + enum class ssl_support_t: uint8_t { + e_ssl_support_disabled, + e_ssl_support_enabled, + e_ssl_support_autodetect, + }; + + enum class ssl_verification_t : uint8_t + { + none = 0, //!< Do not verify peer. + system_ca, //!< Verify peer via system ca only (do not inspect user certificates) + user_certificates,//!< Verify peer via specific (non-chain) certificate(s) only. + user_ca //!< Verify peer via specific (possibly chain) certificate(s) only. + }; + + struct ssl_authentication_t + { + std::string private_key_path; //!< Private key used for authentication + std::string certificate_path; //!< Certificate used for authentication to peer. + + //! Load `private_key_path` and `certificate_path` into `ssl_context`. + void use_ssl_certificate(boost::asio::ssl::context &ssl_context) const; + }; + + /*! + \note `verification != disabled && support == disabled` is currently + "allowed" via public interface but obviously invalid configuation. + */ + class ssl_options_t + { + // force sorted behavior in private + std::vector> fingerprints_; + + public: + std::string ca_path; + ssl_authentication_t auth; + ssl_support_t support; + ssl_verification_t verification; + + //! Verification is set to system ca unless SSL is disabled. + ssl_options_t(ssl_support_t support) + : fingerprints_(), + ca_path(), + auth(), + support(support), + verification(support == ssl_support_t::e_ssl_support_disabled ? ssl_verification_t::none : ssl_verification_t::system_ca) + {} + + //! Provide user fingerprints and/or ca path. Enables SSL and user_certificate verification + ssl_options_t(std::vector> fingerprints, std::string ca_path); + + ssl_options_t(const ssl_options_t&) = default; + ssl_options_t(ssl_options_t&&) = default; + + ssl_options_t& operator=(const ssl_options_t&) = default; + ssl_options_t& operator=(ssl_options_t&&) = default; + + //! \return False iff ssl is disabled, otherwise true. + explicit operator bool() const noexcept { return support != ssl_support_t::e_ssl_support_disabled; } + + //! \retrurn True if `host` can be verified using `this` configuration WITHOUT system "root" CAs. + bool has_strong_verification(boost::string_ref host) const noexcept; + + //! Search against internal fingerprints. Always false if `behavior() != user_certificate_check`. + bool has_fingerprint(boost::asio::ssl::verify_context &ctx) const; + + boost::asio::ssl::context create_context() const; + + /*! \note If `this->support == autodetect && this->verification != none`, + then the handshake will not fail when peer verification fails. The + assumption is that a re-connect will be attempted, so a warning is + logged instead of failure. + + \note It is strongly encouraged that clients using `system_ca` + verification provide a non-empty `host` for rfc2818 verification. + + \param socket Used in SSL handshake and verification + \param type Client or server + \param host This parameter is only used when + `type == client && !host.empty()`. The value is sent to the server for + situations where multiple hostnames are being handled by a server. If + `verification == system_ca` the client also does a rfc2818 check to + ensure that the server certificate is to the provided hostname. + + \return True if the SSL handshake completes with peer verification + settings. */ + bool handshake(boost::asio::ssl::stream &socket, boost::asio::ssl::stream_base::handshake_type type, const std::string& host = {}) const; + }; + + // https://security.stackexchange.com/questions/34780/checking-client-hello-for-https-classification + constexpr size_t get_ssl_magic_size() { return 9; } + bool is_ssl(const unsigned char *data, size_t len); + bool ssl_support_from_string(ssl_support_t &ssl, boost::string_ref s); + + bool create_ec_ssl_certificate(EVP_PKEY *&pkey, X509 *&cert); + bool create_rsa_ssl_certificate(EVP_PKEY *&pkey, X509 *&cert); +} +} + +#endif //_NET_SSL_H diff --git a/contrib/epee/include/net/net_utils_base.h b/contrib/epee/include/net/net_utils_base.h index 7615786bea6..83e6b5ab8f9 100644 --- a/contrib/epee/include/net/net_utils_base.h +++ b/contrib/epee/include/net/net_utils_base.h @@ -33,6 +33,7 @@ #include #include #include +#include "enums.h" #include "serialization/keyvalue_serialization.h" #include "misc_log_ex.h" @@ -43,6 +44,18 @@ #define MAKE_IP( a1, a2, a3, a4 ) (a1|(a2<<8)|(a3<<16)|(a4<<24)) #endif +#if BOOST_VERSION >= 107000 +#define GET_IO_SERVICE(s) ((boost::asio::io_context&)(s).get_executor().context()) +#else +#define GET_IO_SERVICE(s) ((s).get_io_service()) +#endif + +namespace net +{ + class tor_address; + class i2p_address; +} + namespace epee { namespace net_utils @@ -53,6 +66,10 @@ namespace net_utils uint16_t m_port; public: + constexpr ipv4_network_address() noexcept + : ipv4_network_address(0, 0) + {} + constexpr ipv4_network_address(uint32_t ip, uint16_t port) noexcept : m_ip(ip), m_port(port) {} @@ -67,9 +84,10 @@ namespace net_utils std::string host_str() const; bool is_loopback() const; bool is_local() const; - static constexpr uint8_t get_type_id() noexcept { return ID; } + static constexpr address_type get_type_id() noexcept { return address_type::ipv4; } + static constexpr zone get_zone() noexcept { return zone::public_; } + static constexpr bool is_blockable() noexcept { return true; } - static const uint8_t ID = 1; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(m_ip) KV_SERIALIZE(m_port) @@ -103,7 +121,9 @@ namespace net_utils virtual std::string host_str() const = 0; virtual bool is_loopback() const = 0; virtual bool is_local() const = 0; - virtual uint8_t get_type_id() const = 0; + virtual address_type get_type_id() const = 0; + virtual zone get_zone() const = 0; + virtual bool is_blockable() const = 0; }; template @@ -131,7 +151,9 @@ namespace net_utils virtual std::string host_str() const override { return value.host_str(); } virtual bool is_loopback() const override { return value.is_loopback(); } virtual bool is_local() const override { return value.is_local(); } - virtual uint8_t get_type_id() const override { return value.get_type_id(); } + virtual address_type get_type_id() const override { return value.get_type_id(); } + virtual zone get_zone() const override { return value.get_zone(); } + virtual bool is_blockable() const override { return value.is_blockable(); } }; std::shared_ptr self; @@ -146,6 +168,23 @@ namespace net_utils throw std::bad_cast{}; return static_cast*>(self_)->value; } + + template + bool serialize_addr(std::false_type, t_storage& stg, typename t_storage::hsection hparent) + { + T addr{}; + if (!epee::serialization::selector::serialize(addr, stg, hparent, "addr")) + return false; + *this = std::move(addr); + return true; + } + + template + bool serialize_addr(std::true_type, t_storage& stg, typename t_storage::hsection hparent) const + { + return epee::serialization::selector::serialize(as(), stg, hparent, "addr"); + } + public: network_address() : self(nullptr) {} template @@ -158,43 +197,34 @@ namespace net_utils std::string host_str() const { return self ? self->host_str() : ""; } bool is_loopback() const { return self ? self->is_loopback() : false; } bool is_local() const { return self ? self->is_local() : false; } - uint8_t get_type_id() const { return self ? self->get_type_id() : 0; } + address_type get_type_id() const { return self ? self->get_type_id() : address_type::invalid; } + zone get_zone() const { return self ? self->get_zone() : zone::invalid; } + bool is_blockable() const { return self ? self->is_blockable() : false; } template const Type &as() const { return as_mutable(); } BEGIN_KV_SERIALIZE_MAP() - uint8_t type = is_store ? this_ref.get_type_id() : 0; + // need to `#include "net/[i2p|tor]_address.h"` when serializing `network_address` + static constexpr std::integral_constant is_store_{}; + + std::uint8_t type = std::uint8_t(is_store ? this_ref.get_type_id() : address_type::invalid); if (!epee::serialization::selector::serialize(type, stg, hparent_section, "type")) return false; - switch (type) + + switch (address_type(type)) { - case ipv4_network_address::ID: - { - if (!is_store) - { - const_cast(this_ref) = ipv4_network_address{0, 0}; - auto &addr = this_ref.template as_mutable(); - if (epee::serialization::selector::serialize(addr, stg, hparent_section, "addr")) - MDEBUG("Found as addr: " << this_ref.str()); - else if (epee::serialization::selector::serialize(addr, stg, hparent_section, "template as()")) - MDEBUG("Found as template as(): " << this_ref.str()); - else if (epee::serialization::selector::serialize(addr, stg, hparent_section, "template as_mutable()")) - MDEBUG("Found as template as_mutable(): " << this_ref.str()); - else - { - MWARNING("Address not found"); - return false; - } - } - else - { - auto &addr = this_ref.template as_mutable(); - if (!epee::serialization::selector::serialize(addr, stg, hparent_section, "addr")) - return false; - } + case address_type::ipv4: + return this_ref.template serialize_addr(is_store_, stg, hparent_section); + case address_type::tor: + return this_ref.template serialize_addr(is_store_, stg, hparent_section); + case address_type::i2p: + return this_ref.template serialize_addr(is_store_, stg, hparent_section); + case address_type::invalid: + default: break; - } - default: MERROR("Unsupported network address type: " << (unsigned)type); return false; } + + MERROR("Unsupported network address type: " << (unsigned)type); + return false; END_KV_SERIALIZE_MAP() }; @@ -211,8 +241,6 @@ namespace net_utils inline bool operator>=(const network_address& lhs, const network_address& rhs) { return !lhs.less(rhs); } - bool create_network_address(network_address &address, const std::string &string, uint16_t default_port = 0); - /************************************************************************/ /* */ /************************************************************************/ @@ -222,54 +250,68 @@ namespace net_utils const network_address m_remote_address; const bool m_is_income; const time_t m_started; + const time_t m_ssl; time_t m_last_recv; time_t m_last_send; uint64_t m_recv_cnt; uint64_t m_send_cnt; double m_current_speed_down; double m_current_speed_up; + double m_max_speed_down; + double m_max_speed_up; connection_context_base(boost::uuids::uuid connection_id, - const network_address &remote_address, bool is_income, + const network_address &remote_address, bool is_income, bool ssl, time_t last_recv = 0, time_t last_send = 0, uint64_t recv_cnt = 0, uint64_t send_cnt = 0): m_connection_id(connection_id), m_remote_address(remote_address), m_is_income(is_income), m_started(time(NULL)), + m_ssl(ssl), m_last_recv(last_recv), m_last_send(last_send), m_recv_cnt(recv_cnt), m_send_cnt(send_cnt), m_current_speed_down(0), - m_current_speed_up(0) + m_current_speed_up(0), + m_max_speed_down(0), + m_max_speed_up(0) {} connection_context_base(): m_connection_id(), - m_remote_address(ipv4_network_address{0,0}), + m_remote_address(), m_is_income(false), m_started(time(NULL)), + m_ssl(false), m_last_recv(0), m_last_send(0), m_recv_cnt(0), m_send_cnt(0), m_current_speed_down(0), - m_current_speed_up(0) + m_current_speed_up(0), + m_max_speed_down(0), + m_max_speed_up(0) {} + connection_context_base(const connection_context_base& a): connection_context_base() + { + set_details(a.m_connection_id, a.m_remote_address, a.m_is_income, a.m_ssl); + } + connection_context_base& operator=(const connection_context_base& a) { - set_details(a.m_connection_id, a.m_remote_address, a.m_is_income); + set_details(a.m_connection_id, a.m_remote_address, a.m_is_income, a.m_ssl); return *this; } private: template friend class connection; - void set_details(boost::uuids::uuid connection_id, const network_address &remote_address, bool is_income) + void set_details(boost::uuids::uuid connection_id, const network_address &remote_address, bool is_income, bool ssl) { this->~connection_context_base(); - new(this) connection_context_base(connection_id, remote_address, is_income); + new(this) connection_context_base(connection_id, remote_address, is_income, ssl); } }; @@ -281,6 +323,7 @@ namespace net_utils { virtual bool do_send(const void* ptr, size_t cb)=0; virtual bool close()=0; + virtual bool send_done()=0; virtual bool call_run_once_service_io()=0; virtual bool request_callback()=0; virtual boost::asio::io_service& get_io_service()=0; diff --git a/contrib/epee/include/net/network_throttle-detail.hpp b/contrib/epee/include/net/network_throttle-detail.hpp index 955668d62d9..353ae0c0c38 100644 --- a/contrib/epee/include/net/network_throttle-detail.hpp +++ b/contrib/epee/include/net/network_throttle-detail.hpp @@ -2,7 +2,7 @@ /// @author rfree (current maintainer in monero.cc project) /// @brief implementaion for throttling of connection (count and rate-limit speed etc) -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -36,6 +36,7 @@ #ifndef INCLUDED_throttle_detail_hpp #define INCLUDED_throttle_detail_hpp +#include #include "network_throttle.hpp" namespace epee @@ -61,10 +62,12 @@ class network_throttle : public i_network_throttle { network_time_seconds m_slot_size; // the size of one slot. TODO: now hardcoded for 1 second e.g. in time_to_slot() // TODO for big window size, for performance better the substract on change of m_last_sample_time instead of recalculating average of eg >100 elements - std::vector< packet_info > m_history; // the history of bw usage + boost::circular_buffer< packet_info > m_history; // the history of bw usage network_time_seconds m_last_sample_time; // time of last history[0] - so we know when to rotate the buffer network_time_seconds m_start_time; // when we were created bool m_any_packet_yet; // did we yet got any packet to count + uint64_t m_total_packets; + uint64_t m_total_bytes; std::string m_name; // my name for debug and logs std::string m_nameshort; // my name for debug and logs (used in log file name) @@ -94,6 +97,7 @@ class network_throttle : public i_network_throttle { virtual size_t get_recommended_size_of_planned_transport() const; ///< what should be the size (bytes) of next data block to be transported virtual size_t get_recommended_size_of_planned_transport_window(double force_window) const; ///< ditto, but for given windows time frame virtual double get_current_speed() const; + virtual void get_stats(uint64_t &total_packets, uint64_t &total_bytes) const; private: virtual network_time_seconds time_to_slot(network_time_seconds t) const { return std::floor( t ); } // convert exact time eg 13.7 to rounded time for slot number in history 13 diff --git a/contrib/epee/include/net/network_throttle.hpp b/contrib/epee/include/net/network_throttle.hpp index 7df79a90833..02a28632630 100644 --- a/contrib/epee/include/net/network_throttle.hpp +++ b/contrib/epee/include/net/network_throttle.hpp @@ -2,7 +2,7 @@ /// @author rfree (current maintainer in monero.cc project) /// @brief interface for throttling of connection (count and rate-limit speed etc) -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -152,7 +152,8 @@ class i_network_throttle { virtual size_t get_recommended_size_of_planned_transport() const =0; // what should be the recommended limit of data size that we can transport over current network_throttle in near future virtual double get_time_seconds() const =0; // a timer - virtual void logger_handle_net(const std::string &filename, double time, size_t size)=0; + virtual void logger_handle_net(const std::string &filename, double time, size_t size)=0; + virtual void get_stats(uint64_t &total_packets, uint64_t &total_bytes) const =0; }; diff --git a/contrib/epee/include/readline_buffer.h b/contrib/epee/include/readline_buffer.h index 87c8826cbd8..5968d243d5d 100644 --- a/contrib/epee/include/readline_buffer.h +++ b/contrib/epee/include/readline_buffer.h @@ -27,6 +27,7 @@ namespace rdln private: std::streambuf* m_cout_buf; + size_t m_prompt_length; static std::vector& completion_commands(); }; diff --git a/contrib/epee/include/rolling_median.h b/contrib/epee/include/rolling_median.h new file mode 100644 index 00000000000..8b5a82a8442 --- /dev/null +++ b/contrib/epee/include/rolling_median.h @@ -0,0 +1,236 @@ +// Copyright (c) 2019, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Adapted from source by AShelly: +// Copyright (c) 2011 ashelly.myopenid.com, licenced under the MIT licence +// https://stackoverflow.com/questions/5527437/rolling-median-in-c-turlach-implementation +// https://stackoverflow.com/questions/1309263/rolling-median-algorithm-in-c +// https://ideone.com/XPbl6 + +#pragma once + +#include +#include + +namespace epee +{ +namespace misc_utils +{ + +template +struct rolling_median_t +{ +private: + Item* data; //circular queue of values + int* pos; //index into `heap` for each value + int* heap; //max/median/min heap holding indexes into `data`. + int N; //allocated size. + int idx; //position in circular queue + int minCt; //count of items in min heap + int maxCt; //count of items in max heap + int sz; //count of items in heap + +private: + + //returns true if heap[i] < heap[j] + bool mmless(int i, int j) const + { + return data[heap[i]] < data[heap[j]]; + } + + //swaps items i&j in heap, maintains indexes + bool mmexchange(int i, int j) + { + const int t = heap[i]; + heap[i] = heap[j]; + heap[j] = t; + pos[heap[i]] = i; + pos[heap[j]] = j; + return 1; + } + + //swaps items i&j if i= -maxCt; i *= 2) + { + if (i > -maxCt && mmless(i, i - 1)) + --i; + if (!mmCmpExch(i / 2, i)) + break; + } + } + + //maintains minheap property for all items above i, including median + //returns true if median changed + bool minSortUp(int i) + { + while (i > 0 && mmCmpExch(i, i / 2)) + i /= 2; + return i == 0; + } + + //maintains maxheap property for all items above i, including median + //returns true if median changed + bool maxSortUp(int i) + { + while (i < 0 && mmCmpExch(i / 2, i)) + i /= 2; + return i == 0; + } + +protected: + rolling_median_t &operator=(const rolling_median_t&) = delete; + rolling_median_t(const rolling_median_t&) = delete; + +public: + //creates new rolling_median_t: to calculate `nItems` running median. + rolling_median_t(size_t N): N(N) + { + int size = N * (sizeof(Item) + sizeof(int) * 2); + data = (Item*)malloc(size); + pos = (int*) (data + N); + heap = pos + N + (N / 2); //points to middle of storage. + clear(); + } + + rolling_median_t(rolling_median_t &&m) + { + free(data); + memcpy(this, &m, sizeof(rolling_median_t)); + m.data = NULL; + } + rolling_median_t &operator=(rolling_median_t &&m) + { + free(data); + memcpy(this, &m, sizeof(rolling_median_t)); + m.data = NULL; + return *this; + } + + ~rolling_median_t() + { + free(data); + } + + void clear() + { + idx = 0; + minCt = 0; + maxCt = 0; + sz = 0; + int nItems = N; + while (nItems--) //set up initial heap fill pattern: median,max,min,max,... + { + pos[nItems] = ((nItems + 1) / 2) * ((nItems & 1) ? -1 : 1); + heap[pos[nItems]] = nItems; + } + } + + int size() const + { + return sz; + } + + //Inserts item, maintains median in O(lg nItems) + void insert(Item v) + { + int p = pos[idx]; + Item old = data[idx]; + data[idx] = v; + idx = (idx + 1) % N; + sz = std::min(sz + 1, N); + if (p > 0) //new item is in minHeap + { + if (minCt < (N - 1) / 2) + { + ++minCt; + } + else if (v > old) + { + minSortDown(p); + return; + } + if (minSortUp(p) && mmCmpExch(0, -1)) + maxSortDown(-1); + } + else if (p < 0) //new item is in maxheap + { + if (maxCt < N / 2) + { + ++maxCt; + } + else if (v < old) + { + maxSortDown(p); + return; + } + if (maxSortUp(p) && minCt && mmCmpExch(1, 0)) + minSortDown(1); + } + else //new item is at median + { + if (maxCt && maxSortUp(-1)) + maxSortDown(-1); + if (minCt && minSortUp(1)) + minSortDown(1); + } + } + + //returns median item (or average of 2 when item count is even) + Item median() const + { + Item v = data[heap[0]]; + if (minCt < maxCt) + { + v = (v + data[heap[-1]]) / 2; + } + return v; + } +}; + +} +} diff --git a/contrib/epee/include/serialization/keyvalue_serialization.h b/contrib/epee/include/serialization/keyvalue_serialization.h index 5791e199828..5459c840950 100644 --- a/contrib/epee/include/serialization/keyvalue_serialization.h +++ b/contrib/epee/include/serialization/keyvalue_serialization.h @@ -32,6 +32,9 @@ #include "enableable.h" #include "keyvalue_serialization_overloads.h" +#undef MONERO_DEFAULT_LOG_CATEGORY +#define MONERO_DEFAULT_LOG_CATEGORY "serialization" + namespace epee { /************************************************************************/ @@ -85,6 +88,14 @@ public: \ static_assert(std::is_pod::value, "t_type must be a POD type."); \ KV_SERIALIZE_VAL_POD_AS_BLOB_FORCE_N(varialble, val_name) +#define KV_SERIALIZE_VAL_POD_AS_BLOB_OPT_N(varialble, val_name, default_value) \ + do { \ + static_assert(std::is_pod::value, "t_type must be a POD type."); \ + bool ret = KV_SERIALIZE_VAL_POD_AS_BLOB_FORCE_N(varialble, val_name); \ + if (!ret) \ + epee::serialize_default(this_ref.varialble, default_value); \ + } while(0); + #define KV_SERIALIZE_CONTAINER_POD_AS_BLOB_N(varialble, val_name) \ epee::serialization::selector::serialize_stl_container_pod_val_as_blob(this_ref.varialble, stg, hparent_section, val_name); @@ -92,6 +103,7 @@ public: \ #define KV_SERIALIZE(varialble) KV_SERIALIZE_N(varialble, #varialble) #define KV_SERIALIZE_VAL_POD_AS_BLOB(varialble) KV_SERIALIZE_VAL_POD_AS_BLOB_N(varialble, #varialble) +#define KV_SERIALIZE_VAL_POD_AS_BLOB_OPT(varialble, def) KV_SERIALIZE_VAL_POD_AS_BLOB_OPT_N(varialble, #varialble, def) #define KV_SERIALIZE_VAL_POD_AS_BLOB_FORCE(varialble) KV_SERIALIZE_VAL_POD_AS_BLOB_FORCE_N(varialble, #varialble) //skip is_pod compile time check #define KV_SERIALIZE_CONTAINER_POD_AS_BLOB(varialble) KV_SERIALIZE_CONTAINER_POD_AS_BLOB_N(varialble, #varialble) #define KV_SERIALIZE_OPT(variable,default_value) KV_SERIALIZE_OPT_N(variable, #variable, default_value) diff --git a/contrib/epee/include/serialization/keyvalue_serialization_overloads.h b/contrib/epee/include/serialization/keyvalue_serialization_overloads.h index 7087136ccfb..fc8b90a2cff 100644 --- a/contrib/epee/include/serialization/keyvalue_serialization_overloads.h +++ b/contrib/epee/include/serialization/keyvalue_serialization_overloads.h @@ -33,8 +33,16 @@ #include #include +#undef MONERO_DEFAULT_LOG_CATEGORY +#define MONERO_DEFAULT_LOG_CATEGORY "serialization" + namespace epee { + namespace + { + template void hint_resize(C &container, size_t size) {} + template void hint_resize(std::vector &container, size_t size) { container.reserve(size); } + } namespace serialization { @@ -156,8 +164,9 @@ namespace epee typename stl_container::value_type* pelem = (typename stl_container::value_type*)buff.data(); CHECK_AND_ASSERT_MES(!(loaded_size%sizeof(typename stl_container::value_type)), false, - "size in blob " << loaded_size << " not have not zero modulo for sizeof(value_type) = " << sizeof(typename stl_container::value_type)); + "size in blob " << loaded_size << " not have not zero modulo for sizeof(value_type) = " << sizeof(typename stl_container::value_type) << ", type " << typeid(typename stl_container::value_type).name()); size_t count = (loaded_size/sizeof(typename stl_container::value_type)); + hint_resize(container, count); for(size_t i = 0; i < count; i++) container.insert(container.end(), *(pelem++)); } diff --git a/contrib/epee/include/span.h b/contrib/epee/include/span.h index 452cc088fae..e100452ca0a 100644 --- a/contrib/epee/include/span.h +++ b/contrib/epee/include/span.h @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2018, The Monero Project +// Copyright (c) 2017-2019, The Monero Project // // All rights reserved. // @@ -28,8 +28,10 @@ #pragma once +#include #include #include +#include #include namespace epee @@ -52,11 +54,15 @@ namespace epee template class span { - /* Supporting class types is tricky - the {ptr,len} constructor will allow - derived-to-base conversions. This is NOT desireable because an array of - derived types is not an array of base types. It is possible to handle - this case, implement when/if needed. */ - static_assert(!std::is_class(), "no class types are currently allowed"); + template + static constexpr bool safe_conversion() noexcept + { + // Allow exact matches or `T*` -> `const T*`. + using with_const = typename std::add_const::type; + return std::is_same() || + (std::is_const() && std::is_same()); + } + public: using value_type = T; using size_type = std::size_t; @@ -71,7 +77,9 @@ namespace epee constexpr span() noexcept : ptr(nullptr), len(0) {} constexpr span(std::nullptr_t) noexcept : span() {} - constexpr span(T* const src_ptr, const std::size_t count) noexcept + //! Prevent derived-to-base conversions; invalid in this context. + template()>::type> + constexpr span(U* const src_ptr, const std::size_t count) noexcept : ptr(src_ptr), len(count) {} //! Conversion from C-array. Prevents common bugs with sizeof + arrays. @@ -81,6 +89,16 @@ namespace epee constexpr span(const span&) noexcept = default; span& operator=(const span&) noexcept = default; + /*! Try to remove `amount` elements from beginning of span. + \return Number of elements removed. */ + std::size_t remove_prefix(std::size_t amount) noexcept + { + amount = std::min(len, amount); + ptr += amount; + len -= amount; + return amount; + } + constexpr iterator begin() const noexcept { return ptr; } constexpr const_iterator cbegin() const noexcept { return ptr; } @@ -92,6 +110,8 @@ namespace epee constexpr std::size_t size() const noexcept { return len; } constexpr std::size_t size_bytes() const noexcept { return size() * sizeof(value_type); } + const T &operator[](size_t idx) const { return ptr[idx]; } + private: T* ptr; std::size_t len; @@ -105,6 +125,14 @@ namespace epee return {src.data(), src.size()}; } + //! \return `span` from a STL compatible `src`. + template + constexpr span to_mut_span(T& src) + { + // compiler provides diagnostic if size() is not size_t. + return {src.data(), src.size()}; + } + template constexpr bool has_padding() noexcept { @@ -127,4 +155,21 @@ namespace epee static_assert(!has_padding(), "source type may have padding"); return {reinterpret_cast(std::addressof(src)), sizeof(T)}; } + + //! \return `span` which represents the bytes at `&src`. + template + span as_mut_byte_span(T& src) noexcept + { + static_assert(!std::is_empty(), "empty types will not work -> sizeof == 1"); + static_assert(!has_padding(), "source type may have padding"); + return {reinterpret_cast(std::addressof(src)), sizeof(T)}; + } + + //! make a span from a std::string + template + span strspan(const std::string &s) noexcept + { + static_assert(std::is_same() || std::is_same() || std::is_same() || std::is_same(), "Unexpected type"); + return {reinterpret_cast(s.data()), s.size()}; + } } diff --git a/contrib/epee/include/stats.h b/contrib/epee/include/stats.h new file mode 100644 index 00000000000..1cf9c68fb58 --- /dev/null +++ b/contrib/epee/include/stats.h @@ -0,0 +1,58 @@ +#pragma once + +#include + +template +class Stats +{ +public: + Stats(const std::vector &v): values(v), cached(0) {} + ~Stats() {} + + size_t get_size() const; + Tpod get_min() const; + Tpod get_max() const; + Tpod get_median() const; + double get_mean() const; + double get_confidence_interval_95() const; + double get_confidence_interval_99() const; + double get_standard_deviation() const; + double get_standard_error() const; + double get_variance() const; + double get_kurtosis() const; + double get_non_parametric_skew() const; + double get_t_test(T t) const; + double get_t_test(size_t npoints, double mean, double stddev) const; + double get_t_test(const Stats &other) const; + double get_z_test(const Stats &other) const; + double get_test(const Stats &other) const; + std::vector get_quantiles(unsigned int quantiles) const; + std::vector get_bins(unsigned int bins) const; + bool is_same_distribution_95(size_t npoints, double mean, double stddev) const; + bool is_same_distribution_95(const Stats &other) const; + bool is_same_distribution_99(size_t npoints, double mean, double stddev) const; + bool is_same_distribution_99(const Stats &other) const; + + double get_cdf95(size_t df) const; + double get_cdf95(const Stats &other) const; + double get_cdf99(size_t df) const; + double get_cdf99(const Stats &other) const; + +private: + inline bool is_cached(int bit) const; + inline void set_cached(int bit) const; + + const std::vector &values; + + mutable uint64_t cached; + mutable Tpod min; + mutable Tpod max; + mutable Tpod median; + mutable double mean; + mutable double standard_deviation; + mutable double standard_error; + mutable double variance; + mutable double kurtosis; +}; + +#include "stats.inl" diff --git a/contrib/epee/include/stats.inl b/contrib/epee/include/stats.inl new file mode 100644 index 00000000000..5a5cd0b939f --- /dev/null +++ b/contrib/epee/include/stats.inl @@ -0,0 +1,359 @@ +#include +#include +#include +#include "stats.h" + +enum +{ + bit_min = 0, + bit_max, + bit_median, + bit_mean, + bit_standard_deviation, + bit_standard_error, + bit_variance, + bit_kurtosis, +}; + +static inline double square(double x) +{ + return x * x; +} + +template +static inline double interpolate(T v, T v0, double i0, T v1, double i1) +{ + return i0 + (i1 - i0) * (v - v0) / (v1 - v0); +} + +template +inline bool Stats::is_cached(int bit) const +{ + return cached & (1< +inline void Stats::set_cached(int bit) const +{ + cached |= 1< +size_t Stats::get_size() const +{ + return values.size(); +} + +template +Tpod Stats::get_min() const +{ + if (!is_cached(bit_min)) + { + min = std::numeric_limits::max(); + for (const T &v: values) + min = std::min(min, v); + set_cached(bit_min); + } + return min; +} + +template +Tpod Stats::get_max() const +{ + if (!is_cached(bit_max)) + { + max = std::numeric_limits::min(); + for (const T &v: values) + max = std::max(max, v); + set_cached(bit_max); + } + return max; +} + +template +Tpod Stats::get_median() const +{ + if (!is_cached(bit_median)) + { + std::vector sorted; + sorted.reserve(values.size()); + for (const T &v: values) + sorted.push_back(v); + std::sort(sorted.begin(), sorted.end()); + if (sorted.size() & 1) + { + median = sorted[sorted.size() / 2]; + } + else + { + median = (sorted[(sorted.size() - 1) / 2] + sorted[sorted.size() / 2]) / 2; + } + set_cached(bit_median); + } + return median; +} + +template +double Stats::get_mean() const +{ + if (values.empty()) + return 0.0; + if (!is_cached(bit_mean)) + { + mean = 0.0; + for (const T &v: values) + mean += v; + mean /= values.size(); + set_cached(bit_mean); + } + return mean; +} + +template +double Stats::get_cdf95(size_t df) const +{ + static const double p[101] = { + -1, 12.706, 4.3027, 3.1824, 2.7765, 2.5706, 2.4469, 2.3646, 2.3060, 2.2622, 2.2281, 2.2010, 2.1788, 2.1604, 2.1448, 2.1315, + 2.1199, 2.1098, 2.1009, 2.0930, 2.0860, 2.0796, 2.0739, 2.0687, 2.0639, 2.0595, 2.0555, 2.0518, 2.0484, 2.0452, 2.0423, 2.0395, + 2.0369, 2.0345, 2.0322, 2.0301, 2.0281, 2.0262, 2.0244, 2.0227, 2.0211, 2.0195, 2.0181, 2.0167, 2.0154, 2.0141, 2.0129, 2.0117, + 2.0106, 2.0096, 2.0086, 2.0076, 2.0066, 2.0057, 2.0049, 2.0040, 2.0032, 2.0025, 2.0017, 2.0010, 2.0003, 1.9996, 1.9990, 1.9983, + 1.9977, 1.9971, 1.9966, 1.9960, 1.9955, 1.9949, 1.9944, 1.9939, 1.9935, 1.9930, 1.9925, 1.9921, 1.9917, 1.9913, 1.9908, 1.9905, + 1.9901, 1.9897, 1.9893, 1.9890, 1.9886, 1.9883, 1.9879, 1.9876, 1.9873, 1.9870, 1.9867, 1.9864, 1.9861, 1.9858, 1.9855, 1.9852, + 1.9850, 1.9847, 1.9845, 1.9842, 1.9840, + }; + if (df <= 100) + return p[df]; + if (df <= 120) + return interpolate(df, 100, 1.9840, 120, 1.98); + return 1.96; +} + +template +double Stats::get_cdf95(const Stats &other) const +{ + return get_cdf95(get_size() + other.get_size() - 2); +} + +template +double Stats::get_cdf99(size_t df) const +{ + static const double p[101] = { + -1, 9.9250, 5.8408, 4.6041, 4.0321, 3.7074, 3.4995, 3.3554, 3.2498, 3.1693, 3.1058, 3.0545, 3.0123, 2.9768, 2.9467, 2.9208, 2.8982, + 2.8784, 2.8609, 2.8453, 2.8314, 2.8188, 2.8073, 2.7970, 2.7874, 2.7787, 2.7707, 2.7633, 2.7564, 2.7500, 2.7440, 2.7385, 2.7333, + 2.7284, 2.7238, 2.7195, 2.7154, 2.7116, 2.7079, 2.7045, 2.7012, 2.6981, 2.6951, 2.6923, 2.6896, 2.6870, 2.6846, 2.6822, 2.6800, + 2.6778, 2.6757, 2.6737, 2.6718, 2.6700, 2.6682, 2.6665, 2.6649, 2.6633, 2.6618, 2.6603, 2.6589, 2.6575, 2.6561, 2.6549, 2.6536, + 2.6524, 2.6512, 2.6501, 2.6490, 2.6479, 2.6469, 2.6458, 2.6449, 2.6439, 2.6430, 2.6421, 2.6412, 2.6403, 2.6395, 2.6387, 2.6379, + 2.6371, 2.6364, 2.6356, 2.6349, 2.6342, 2.6335, 2.6329, 2.6322, 2.6316, 2.6309, 2.6303, 2.6297, 2.6291, 2.6286, 2.6280, 2.6275, + 2.6269, 2.6264, 2.6259, + }; + if (df <= 100) + return p[df]; + if (df <= 120) + return interpolate(df, 100, 2.6529, 120, 2.617); + return 2.576; +} + +template +double Stats::get_cdf99(const Stats &other) const +{ + return get_cdf99(get_size() + other.get_size() - 2); +} + +template +double Stats::get_confidence_interval_95() const +{ + const size_t df = get_size() - 1; + return get_standard_error() * get_cdf95(df); +} + +template +double Stats::get_confidence_interval_99() const +{ + const size_t df = get_size() - 1; + return get_standard_error() * get_cdf99(df); +} + +template +bool Stats::is_same_distribution_95(size_t npoints, double mean, double stddev) const +{ + return fabs(get_t_test(npoints, mean, stddev)) < get_cdf95(get_size() + npoints - 2); +} + +template +bool Stats::is_same_distribution_95(const Stats &other) const +{ + return fabs(get_t_test(other)) < get_cdf95(other); +} + +template +bool Stats::is_same_distribution_99(size_t npoints, double mean, double stddev) const +{ + return fabs(get_t_test(npoints, mean, stddev)) < get_cdf99(get_size() + npoints - 2); +} + +template +bool Stats::is_same_distribution_99(const Stats &other) const +{ + return fabs(get_t_test(other)) < get_cdf99(other); +} + +template +double Stats::get_standard_deviation() const +{ + if (values.size() <= 1) + return 0.0; + if (!is_cached(bit_standard_deviation)) + { + Tpod m = get_mean(), t = 0; + for (const T &v: values) + t += ((T)v - m) * ((T)v - m); + standard_deviation = sqrt(t / ((double)values.size() - 1)); + set_cached(bit_standard_deviation); + } + return standard_deviation; +} + +template +double Stats::get_standard_error() const +{ + if (!is_cached(bit_standard_error)) + { + standard_error = get_standard_deviation() / sqrt(get_size()); + set_cached(bit_standard_error); + } + return standard_error; +} + +template +double Stats::get_variance() const +{ + if (!is_cached(bit_variance)) + { + double stddev = get_standard_deviation(); + variance = stddev * stddev; + set_cached(bit_variance); + } + return variance; +} + +template +double Stats::get_kurtosis() const +{ + if (values.empty()) + return 0.0; + if (!is_cached(bit_kurtosis)) + { + double m = get_mean(); + double n = 0, d = 0; + for (const T &v: values) + { + T p2 = (v - m) * (v - m); + T p4 = p2 * p2; + n += p4; + d += p2; + } + n /= values.size(); + d /= values.size(); + d *= d; + kurtosis = n / d; + set_cached(bit_kurtosis); + } + return kurtosis; +} + +template +double Stats::get_non_parametric_skew() const +{ + return (get_mean() - get_median()) / get_standard_deviation(); +} + +template +double Stats::get_t_test(T t) const +{ + const double n = get_mean() - t; + const double d = get_standard_deviation() / sqrt(get_size()); + return n / d; +} + +template +double Stats::get_t_test(size_t npoints, double mean, double stddev) const +{ + const double n = get_mean() - mean; + const double d = sqrt(get_variance() / get_size() + square(stddev) / npoints); + return n / d; +} + +template +double Stats::get_t_test(const Stats &other) const +{ + const double n = get_mean() - other.get_mean(); + const double d = sqrt(get_variance() / get_size() + other.get_variance() / other.get_size()); + return n / d; +} + +template +double Stats::get_z_test(const Stats &other) const +{ + const double m0 = get_mean(); + const double m1 = other.get_mean(); + const double sd0 = get_standard_deviation(); + const double sd1 = other.get_standard_deviation(); + const size_t s0 = get_size(); + const size_t s1 = other.get_size(); + + const double n = m0 - m1; + const double d = sqrt(square(sd0 / sqrt(s0)) + square(sd1 / sqrt(s1))); + + return n / d; +} + +template +double Stats::get_test(const Stats &other) const +{ + if (get_size() >= 30 && other.get_size() >= 30) + return get_z_test(other); + else + return get_t_test(other); +} + +template +std::vector Stats::get_quantiles(unsigned int quantiles) const +{ + std::vector sorted; + sorted.reserve(values.size()); + for (const T &v: values) + sorted.push_back(v); + std::sort(sorted.begin(), sorted.end()); + std::vector q(quantiles + 1, 0); + for (unsigned int i = 1; i <= quantiles; ++i) + { + unsigned idx = (unsigned)ceil(values.size() * i / (double)quantiles); + q[i] = sorted[idx - 1]; + } + if (!is_cached(bit_min)) + { + min = sorted.front(); + set_cached(bit_min); + } + q[0] = min; + if (!is_cached(bit_max)) + { + max = sorted.back(); + set_cached(bit_max); + } + return q; +} + +template +std::vector Stats::get_bins(unsigned int bins) const +{ + std::vector b(bins, 0); + const double scale = 1.0 / (get_max() - get_min()); + const T base = get_min(); + for (const T &v: values) + { + unsigned int idx = (v - base) * scale; + ++b[idx]; + } + return b; +} diff --git a/contrib/epee/include/storages/http_abstract_invoke.h b/contrib/epee/include/storages/http_abstract_invoke.h index d93084ab0a3..18b7f10c177 100644 --- a/contrib/epee/include/storages/http_abstract_invoke.h +++ b/contrib/epee/include/storages/http_abstract_invoke.h @@ -97,7 +97,7 @@ namespace epee return false; } - return serialization::load_t_from_binary(result_struct, pri->m_body); + return serialization::load_t_from_binary(result_struct, epee::strspan(pri->m_body)); } template diff --git a/contrib/epee/include/storages/levin_abstract_invoke2.h b/contrib/epee/include/storages/levin_abstract_invoke2.h index d77e7a1f869..06eb9bdaf8a 100644 --- a/contrib/epee/include/storages/levin_abstract_invoke2.h +++ b/contrib/epee/include/storages/levin_abstract_invoke2.h @@ -28,6 +28,7 @@ #include "portable_storage_template_helper.h" #include +#include "span.h" #include "net/levin_base.h" #undef MONERO_DEFAULT_LOG_CATEGORY @@ -114,7 +115,7 @@ namespace epee const_cast(out_struct).store(stg);//TODO: add true const support to searilzation std::string buff_to_send; stg.store_to_binary(buff_to_send); - int res = transport.invoke_async(command, buff_to_send, conn_id, [cb, command](int code, const std::string& buff, typename t_transport::connection_context& context)->bool + int res = transport.invoke_async(command, epee::strspan(buff_to_send), conn_id, [cb, command](int code, const epee::span buff, typename t_transport::connection_context& context)->bool { t_result result_struct = AUTO_VAL_INIT(result_struct); if( code <=0 ) @@ -156,7 +157,7 @@ namespace epee std::string buff_to_send; stg.store_to_binary(buff_to_send); - int res = transport.notify(command, buff_to_send, conn_id); + int res = transport.notify(command, epee::strspan(buff_to_send), conn_id); if(res <=0 ) { MERROR("Failed to notify command " << command << " return code " << res); @@ -167,7 +168,7 @@ namespace epee //---------------------------------------------------------------------------------------------------- //---------------------------------------------------------------------------------------------------- template - int buff_to_t_adapter(int command, const std::string& in_buff, std::string& buff_out, callback_t cb, t_context& context ) + int buff_to_t_adapter(int command, const epee::span in_buff, std::string& buff_out, callback_t cb, t_context& context ) { serialization::portable_storage strg; if(!strg.load_from_binary(in_buff)) @@ -197,7 +198,7 @@ namespace epee } template - int buff_to_t_adapter(t_owner* powner, int command, const std::string& in_buff, callback_t cb, t_context& context) + int buff_to_t_adapter(t_owner* powner, int command, const epee::span in_buff, callback_t cb, t_context& context) { serialization::portable_storage strg; if(!strg.load_from_binary(in_buff)) @@ -215,14 +216,14 @@ namespace epee } #define CHAIN_LEVIN_INVOKE_MAP2(context_type) \ - int invoke(int command, const std::string& in_buff, std::string& buff_out, context_type& context) \ + int invoke(int command, const epee::span in_buff, std::string& buff_out, context_type& context) \ { \ bool handled = false; \ return handle_invoke_map(false, command, in_buff, buff_out, context, handled); \ } #define CHAIN_LEVIN_NOTIFY_MAP2(context_type) \ - int notify(int command, const std::string& in_buff, context_type& context) \ + int notify(int command, const epee::span in_buff, context_type& context) \ { \ bool handled = false; std::string fake_str;\ return handle_invoke_map(true, command, in_buff, fake_str, context, handled); \ @@ -230,27 +231,27 @@ namespace epee #define CHAIN_LEVIN_INVOKE_MAP() \ - int invoke(int command, const std::string& in_buff, std::string& buff_out, epee::net_utils::connection_context_base& context) \ + int invoke(int command, const epee::span in_buff, std::string& buff_out, epee::net_utils::connection_context_base& context) \ { \ bool handled = false; \ return handle_invoke_map(false, command, in_buff, buff_out, context, handled); \ } #define CHAIN_LEVIN_NOTIFY_MAP() \ - int notify(int command, const std::string& in_buff, epee::net_utils::connection_context_base& context) \ + int notify(int command, const epee::span in_buff, epee::net_utils::connection_context_base& context) \ { \ bool handled = false; std::string fake_str;\ return handle_invoke_map(true, command, in_buff, fake_str, context, handled); \ } #define CHAIN_LEVIN_NOTIFY_STUB() \ - int notify(int command, const std::string& in_buff, epee::net_utils::connection_context_base& context) \ + int notify(int command, const epee::span in_buff, epee::net_utils::connection_context_base& context) \ { \ return -1; \ } #define BEGIN_INVOKE_MAP2(owner_type) \ - template int handle_invoke_map(bool is_notify, int command, const std::string& in_buff, std::string& buff_out, t_context& context, bool& handled) \ + template int handle_invoke_map(bool is_notify, int command, const epee::span in_buff, std::string& buff_out, t_context& context, bool& handled) \ { \ typedef owner_type internal_owner_type_name; diff --git a/contrib/epee/include/storages/parserse_base_utils.h b/contrib/epee/include/storages/parserse_base_utils.h index c809392f47b..b5c4138c5d3 100644 --- a/contrib/epee/include/storages/parserse_base_utils.h +++ b/contrib/epee/include/storages/parserse_base_utils.h @@ -28,17 +28,61 @@ #pragma once +#include +#include + namespace epee { namespace misc_utils { namespace parse { + // 1: digit + // 2: .eE (floating point) + // 4: alpha + // 8: whitespace + // 16: allowed in float but doesn't necessarily mean it's a float + // 32: \ and " (end of verbatim string) + static const constexpr uint8_t lut[256]={ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 8, 8, 8, 8, 0, 0, // 16 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 32 + 8, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 16, 18, 0, // 48 + 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 0, 0, 0, 0, 0, 0, // 64 + 0, 4, 4, 4, 4, 22, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, // 80 + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 32, 0, 0, 0, // 96 + 0, 4, 4, 4, 4, 22, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, // 112 + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, // 128 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + }; + + inline bool isspace(char c) + { + return lut[(uint8_t)c] & 8; + } + + inline bool isdigit(char c) + { + return lut[(uint8_t)c] & 1; + } + inline std::string transform_to_escape_sequence(const std::string& src) { - //std::stringstream res; + static const char escaped[] = "\b\f\n\r\t\v\"\\/"; + std::string::const_iterator it = std::find_first_of(src.begin(), src.end(), escaped, escaped + sizeof(escaped)); + if (it == src.end()) + return src; + std::string res; - for(std::string::const_iterator it = src.begin(); it!=src.end(); ++it) + res.reserve(2 * src.size()); + res.assign(src.begin(), it); + for(; it!=src.end(); ++it) { switch(*it) { @@ -83,10 +127,15 @@ namespace misc_utils */ inline void match_string2(std::string::const_iterator& star_end_string, std::string::const_iterator buf_end, std::string& val) { - val.clear(); bool escape_mode = false; std::string::const_iterator it = star_end_string; ++it; + std::string::const_iterator fi = it; + while (fi != buf_end && ((lut[(uint8_t)*fi] & 32)) == 0) + ++fi; + val.assign(it, fi); + val.reserve(std::distance(star_end_string, buf_end)); + it = fi; for(;it != buf_end;it++) { if(escape_mode/*prev_ch == '\\'*/) @@ -146,25 +195,34 @@ namespace misc_utils return false; } } - inline void match_number2(std::string::const_iterator& star_end_string, std::string::const_iterator buf_end, std::string& val, bool& is_float_val, bool& is_signed_val) + inline void match_number2(std::string::const_iterator& star_end_string, std::string::const_iterator buf_end, boost::string_ref& val, bool& is_float_val, bool& is_signed_val) { val.clear(); - is_float_val = false; - for(std::string::const_iterator it = star_end_string;it != buf_end;it++) + uint8_t float_flag = 0; + is_signed_val = false; + size_t chars = 0; + std::string::const_iterator it = star_end_string; + if (it != buf_end && *it == '-') { - if(isdigit(*it) || (it == star_end_string && *it == '-') || (val.size() && *it == '.' ) || (is_float_val && (*it == 'e' || *it == 'E' || *it == '-' || *it == '+' )) ) + is_signed_val = true; + ++chars; + ++it; + } + for(;it != buf_end;it++) + { + const uint8_t flags = lut[(uint8_t)*it]; + if (flags & 16) { - if(!val.size() && *it == '-') - is_signed_val = true; - if(*it == '.' ) - is_float_val = true; - val.push_back(*it); + float_flag |= flags; + ++chars; } else { + val = boost::string_ref(&*star_end_string, chars); if(val.size()) { star_end_string = --it; + is_float_val = !!(float_flag & 2); return; } else @@ -173,7 +231,7 @@ namespace misc_utils } ASSERT_MES_AND_THROW("wrong number in json entry: " << std::string(star_end_string, buf_end)); } - inline bool match_number(std::string::const_iterator& star_end_string, std::string::const_iterator buf_end, std::string& val) + inline bool match_number(std::string::const_iterator& star_end_string, std::string::const_iterator buf_end, boost::string_ref& val) { try { @@ -186,15 +244,15 @@ namespace misc_utils return false; } } - inline void match_word2(std::string::const_iterator& star_end_string, std::string::const_iterator buf_end, std::string& val) + inline void match_word2(std::string::const_iterator& star_end_string, std::string::const_iterator buf_end, boost::string_ref& val) { val.clear(); for(std::string::const_iterator it = star_end_string;it != buf_end;it++) { - if(!isalpha(*it)) + if (!(lut[(uint8_t)*it] & 4)) { - val.assign(star_end_string, it); + val = boost::string_ref(&*star_end_string, std::distance(star_end_string, it)); if(val.size()) { star_end_string = --it; @@ -205,7 +263,7 @@ namespace misc_utils } ASSERT_MES_AND_THROW("failed to match word number in json entry: " << std::string(star_end_string, buf_end)); } - inline bool match_word(std::string::const_iterator& star_end_string, std::string::const_iterator buf_end, std::string& val) + inline bool match_word(std::string::const_iterator& star_end_string, std::string::const_iterator buf_end, boost::string_ref& val) { try { diff --git a/contrib/epee/include/storages/portable_storage.h b/contrib/epee/include/storages/portable_storage.h index 2023e2f2aa2..d0e40d606e6 100644 --- a/contrib/epee/include/storages/portable_storage.h +++ b/contrib/epee/include/storages/portable_storage.h @@ -35,6 +35,8 @@ #include "portable_storage_to_json.h" #include "portable_storage_from_json.h" #include "portable_storage_val_converters.h" +#include "span.h" +#include "int-util.h" namespace epee { @@ -80,7 +82,8 @@ namespace epee //------------------------------------------------------------------------------- bool store_to_binary(binarybuffer& target); - bool load_from_binary(const binarybuffer& target); + bool load_from_binary(const epee::span target); + bool load_from_binary(const std::string& target) { return load_from_binary(epee::strspan(target)); } template bool dump_as_xml(std::string& targetObj, const std::string& root_name = ""); bool dump_as_json(std::string& targetObj, size_t indent = 0, bool insert_newlines = true); @@ -135,8 +138,8 @@ namespace epee TRY_ENTRY(); std::stringstream ss; storage_block_header sbh = AUTO_VAL_INIT(sbh); - sbh.m_signature_a = PORTABLE_STORAGE_SIGNATUREA; - sbh.m_signature_b = PORTABLE_STORAGE_SIGNATUREB; + sbh.m_signature_a = SWAP32LE(PORTABLE_STORAGE_SIGNATUREA); + sbh.m_signature_b = SWAP32LE(PORTABLE_STORAGE_SIGNATUREB); sbh.m_ver = PORTABLE_STORAGE_FORMAT_VER; ss.write((const char*)&sbh, sizeof(storage_block_header)); pack_entry_to_buff(ss, m_root); @@ -145,7 +148,7 @@ namespace epee CATCH_ENTRY("portable_storage::store_to_binary", false) } inline - bool portable_storage::load_from_binary(const binarybuffer& source) + bool portable_storage::load_from_binary(const epee::span source) { m_root.m_entries.clear(); if(source.size() < sizeof(storage_block_header)) @@ -154,8 +157,8 @@ namespace epee return false; } storage_block_header* pbuff = (storage_block_header*)source.data(); - if(pbuff->m_signature_a != PORTABLE_STORAGE_SIGNATUREA || - pbuff->m_signature_b != PORTABLE_STORAGE_SIGNATUREB + if(pbuff->m_signature_a != SWAP32LE(PORTABLE_STORAGE_SIGNATUREA) || + pbuff->m_signature_b != SWAP32LE(PORTABLE_STORAGE_SIGNATUREB) ) { LOG_ERROR("portable_storage: wrong binary format - signature mismatch"); diff --git a/contrib/epee/include/storages/portable_storage_base.h b/contrib/epee/include/storages/portable_storage_base.h index 93132548b13..ca7c81ddc13 100644 --- a/contrib/epee/include/storages/portable_storage_base.h +++ b/contrib/epee/include/storages/portable_storage_base.h @@ -31,7 +31,8 @@ #include #include #include -#include +#include +#include #define PORTABLE_STORAGE_SIGNATUREA 0x01011101 #define PORTABLE_STORAGE_SIGNATUREB 0x01020101 // bender's nightmare @@ -71,6 +72,9 @@ namespace epee { struct section; + template struct entry_container { typedef std::vector type; static void reserve(type &t, size_t n) { t.reserve(n); } }; + template<> struct entry_container { typedef std::deque type; static void reserve(type &t, size_t n) {} }; + /************************************************************************/ /* */ /************************************************************************/ @@ -78,6 +82,7 @@ namespace epee struct array_entry_t { array_entry_t():m_it(m_array.end()){} + array_entry_t(const array_entry_t& other):m_array(other.m_array), m_it(m_array.end()){} const t_entry_type* get_first_val() const { @@ -119,8 +124,13 @@ namespace epee return m_array.back(); } - std::list m_array; - mutable typename std::list::const_iterator m_it; + void reserve(size_t n) + { + entry_container::reserve(m_array, n); + } + + typename entry_container::type m_array; + mutable typename entry_container::type::const_iterator m_it; }; diff --git a/contrib/epee/include/storages/portable_storage_from_bin.h b/contrib/epee/include/storages/portable_storage_from_bin.h index 44a80cb2175..e0a32b3ca0f 100644 --- a/contrib/epee/include/storages/portable_storage_from_bin.h +++ b/contrib/epee/include/storages/portable_storage_from_bin.h @@ -59,6 +59,7 @@ namespace epee storage_entry load_storage_entry(); void read(section& sec); void read(std::string& str); + void read(array_entry &ae); private: struct recursuion_limitation_guard { @@ -114,6 +115,7 @@ namespace epee void throwable_buffer_reader::read(t_pod_type& pod_val) { RECURSION_LIMITATION(); + static_assert(std::is_pod::value, "POD type expected"); read(&pod_val, sizeof(pod_val)); } @@ -134,6 +136,8 @@ namespace epee //for pod types array_entry_t sa; size_t size = read_varint(); + CHECK_AND_ASSERT_THROW_MES(size <= m_count, "Size sanity check failed"); + sa.reserve(size); //TODO: add some optimization here later while(size--) sa.m_array.push_back(read()); @@ -277,5 +281,11 @@ namespace epee m_ptr+=len; m_count -= len; } + inline + void throwable_buffer_reader::read(array_entry &ae) + { + RECURSION_LIMITATION(); + CHECK_AND_ASSERT_THROW_MES(false, "Reading array entry is not supported"); + } } } diff --git a/contrib/epee/include/storages/portable_storage_from_json.h b/contrib/epee/include/storages/portable_storage_from_json.h index 727f3655251..3e305254190 100644 --- a/contrib/epee/include/storages/portable_storage_from_json.h +++ b/contrib/epee/include/storages/portable_storage_from_json.h @@ -30,6 +30,8 @@ #include "parserse_base_utils.h" #include "file_io_utils.h" +#define EPEE_JSON_RECURSION_LIMIT_INTERNAL 100 + namespace epee { using namespace misc_utils::parse; @@ -37,15 +39,16 @@ namespace epee { namespace json { -#define CHECK_ISSPACE() if(!isspace(*it)){ ASSERT_MES_AND_THROW("Wrong JSON character at: " << std::string(it, buf_end));} +#define CHECK_ISSPACE() if(!epee::misc_utils::parse::isspace(*it)){ ASSERT_MES_AND_THROW("Wrong JSON character at: " << std::string(it, buf_end));} /*inline void parse_error() { ASSERT_MES_AND_THROW("json parse error"); }*/ template - inline void run_handler(typename t_storage::hsection current_section, std::string::const_iterator& sec_buf_begin, std::string::const_iterator buf_end, t_storage& stg) + inline void run_handler(typename t_storage::hsection current_section, std::string::const_iterator& sec_buf_begin, std::string::const_iterator buf_end, t_storage& stg, unsigned int recursion) { + CHECK_AND_ASSERT_THROW_MES(recursion < EPEE_JSON_RECURSION_LIMIT_INTERNAL, "Wrong JSON data: recursion limitation (" << EPEE_JSON_RECURSION_LIMIT_INTERNAL << ") exceeded"); std::string::const_iterator sub_element_start; std::string name; @@ -111,33 +114,39 @@ namespace epee std::string val; match_string2(it, buf_end, val); //insert text value - stg.set_value(name, val, current_section); + stg.set_value(name, std::move(val), current_section); state = match_state_wonder_after_value; - }else if (isdigit(*it) || *it == '-') + }else if (epee::misc_utils::parse::isdigit(*it) || *it == '-') {//just a named number value started - std::string val; + boost::string_ref val; bool is_v_float = false;bool is_signed = false; match_number2(it, buf_end, val, is_v_float, is_signed); if(!is_v_float) { if(is_signed) { - int64_t nval = boost::lexical_cast(val); + errno = 0; + int64_t nval = strtoll(val.data(), NULL, 10); + if (errno) throw std::runtime_error("Invalid number: " + std::string(val)); stg.set_value(name, nval, current_section); }else { - uint64_t nval = boost::lexical_cast(val); + errno = 0; + uint64_t nval = strtoull(val.data(), NULL, 10); + if (errno) throw std::runtime_error("Invalid number: " + std::string(val)); stg.set_value(name, nval, current_section); } }else { - double nval = boost::lexical_cast(val); + errno = 0; + double nval = strtod(val.data(), NULL); + if (errno) throw std::runtime_error("Invalid number: " + std::string(val)); stg.set_value(name, nval, current_section); } state = match_state_wonder_after_value; }else if(isalpha(*it) ) {// could be null, true or false - std::string word; + boost::string_ref word; match_word2(it, buf_end, word); if(boost::iequals(word, "null")) { @@ -157,7 +166,7 @@ namespace epee //sub section here typename t_storage::hsection new_sec = stg.open_section(name, current_section, true); CHECK_AND_ASSERT_THROW_MES(new_sec, "Failed to insert new section in json: " << std::string(it, buf_end)); - run_handler(new_sec, it, buf_end, stg); + run_handler(new_sec, it, buf_end, stg, recursion + 1); state = match_state_wonder_after_value; }else if(*it == '[') {//array of something @@ -186,7 +195,7 @@ namespace epee typename t_storage::hsection new_sec = nullptr; h_array = stg.insert_first_section(name, new_sec, current_section); CHECK_AND_ASSERT_THROW_MES(h_array&&new_sec, "failed to create new section"); - run_handler(new_sec, it, buf_end, stg); + run_handler(new_sec, it, buf_end, stg, recursion + 1); state = match_state_array_after_value; array_md = array_mode_sections; }else if(*it == '"') @@ -194,23 +203,36 @@ namespace epee //mean array of strings std::string val; match_string2(it, buf_end, val); - h_array = stg.insert_first_value(name, val, current_section); + h_array = stg.insert_first_value(name, std::move(val), current_section); CHECK_AND_ASSERT_THROW_MES(h_array, " failed to insert values entry"); state = match_state_array_after_value; array_md = array_mode_string; - }else if (isdigit(*it) || *it == '-') + }else if (epee::misc_utils::parse::isdigit(*it) || *it == '-') {//array of numbers value started - std::string val; + boost::string_ref val; bool is_v_float = false;bool is_signed_val = false; match_number2(it, buf_end, val, is_v_float, is_signed_val); if(!is_v_float) { - int64_t nval = boost::lexical_cast(val);//bool res = string_tools::string_to_num_fast(val, nval); - h_array = stg.insert_first_value(name, nval, current_section); + if (is_signed_val) + { + errno = 0; + int64_t nval = strtoll(val.data(), NULL, 10); + if (errno) throw std::runtime_error("Invalid number: " + std::string(val)); + h_array = stg.insert_first_value(name, nval, current_section); + }else + { + errno = 0; + uint64_t nval = strtoull(val.data(), NULL, 10); + if (errno) throw std::runtime_error("Invalid number: " + std::string(val)); + h_array = stg.insert_first_value(name, nval, current_section); + } CHECK_AND_ASSERT_THROW_MES(h_array, " failed to insert values section entry"); }else { - double nval = boost::lexical_cast(val);//bool res = string_tools::string_to_num_fast(val, nval); + errno = 0; + double nval = strtod(val.data(), NULL); + if (errno) throw std::runtime_error("Invalid number: " + std::string(val)); h_array = stg.insert_first_value(name, nval, current_section); CHECK_AND_ASSERT_THROW_MES(h_array, " failed to insert values section entry"); } @@ -223,7 +245,7 @@ namespace epee state = match_state_wonder_after_value; }else if(isalpha(*it) ) {// array of booleans - std::string word; + boost::string_ref word; match_word2(it, buf_end, word); if(boost::iequals(word, "true")) { @@ -260,7 +282,7 @@ namespace epee typename t_storage::hsection new_sec = NULL; bool res = stg.insert_next_section(h_array, new_sec); CHECK_AND_ASSERT_THROW_MES(res&&new_sec, "failed to insert next section"); - run_handler(new_sec, it, buf_end, stg); + run_handler(new_sec, it, buf_end, stg, recursion + 1); state = match_state_array_after_value; }else CHECK_ISSPACE(); break; @@ -269,27 +291,38 @@ namespace epee { std::string val; match_string2(it, buf_end, val); - bool res = stg.insert_next_value(h_array, val); + bool res = stg.insert_next_value(h_array, std::move(val)); CHECK_AND_ASSERT_THROW_MES(res, "failed to insert values"); state = match_state_array_after_value; }else CHECK_ISSPACE(); break; case array_mode_numbers: - if (isdigit(*it) || *it == '-') + if (epee::misc_utils::parse::isdigit(*it) || *it == '-') {//array of numbers value started - std::string val; + boost::string_ref val; bool is_v_float = false;bool is_signed_val = false; match_number2(it, buf_end, val, is_v_float, is_signed_val); bool insert_res = false; if(!is_v_float) { - int64_t nval = boost::lexical_cast(val); //bool res = string_tools::string_to_num_fast(val, nval); - insert_res = stg.insert_next_value(h_array, nval); - + if (is_signed_val) + { + errno = 0; + int64_t nval = strtoll(val.data(), NULL, 10); + if (errno) throw std::runtime_error("Invalid number: " + std::string(val)); + insert_res = stg.insert_next_value(h_array, nval); + }else + { + errno = 0; + uint64_t nval = strtoull(val.data(), NULL, 10); + if (errno) throw std::runtime_error("Invalid number: " + std::string(val)); + insert_res = stg.insert_next_value(h_array, nval); + } }else { - //TODO: optimize here if need - double nval = boost::lexical_cast(val); //string_tools::string_to_num_fast(val, nval); + errno = 0; + double nval = strtod(val.data(), NULL); + if (errno) throw std::runtime_error("Invalid number: " + std::string(val)); insert_res = stg.insert_next_value(h_array, nval); } CHECK_AND_ASSERT_THROW_MES(insert_res, "Failed to insert next value"); @@ -300,7 +333,7 @@ namespace epee case array_mode_booleans: if(isalpha(*it) ) {// array of booleans - std::string word; + boost::string_ref word; match_word2(it, buf_end, word); if(boost::iequals(word, "true")) { @@ -362,7 +395,7 @@ namespace epee std::string::const_iterator sec_buf_begin = buff_json.begin(); try { - run_handler(nullptr, sec_buf_begin, buff_json.end(), stg); + run_handler(nullptr, sec_buf_begin, buff_json.end(), stg, 0); return true; } catch(const std::exception& ex) diff --git a/contrib/epee/include/storages/portable_storage_template_helper.h b/contrib/epee/include/storages/portable_storage_template_helper.h index bbd8413fc4c..13c870d4490 100644 --- a/contrib/epee/include/storages/portable_storage_template_helper.h +++ b/contrib/epee/include/storages/portable_storage_template_helper.h @@ -84,7 +84,7 @@ namespace epee } //----------------------------------------------------------------------------------------------------------- template - bool load_t_from_binary(t_struct& out, const std::string& binary_buff) + bool load_t_from_binary(t_struct& out, const epee::span binary_buff) { portable_storage ps; bool rs = ps.load_from_binary(binary_buff); @@ -95,6 +95,12 @@ namespace epee } //----------------------------------------------------------------------------------------------------------- template + bool load_t_from_binary(t_struct& out, const std::string& binary_buff) + { + return load_t_from_binary(out, epee::strspan(binary_buff)); + } + //----------------------------------------------------------------------------------------------------------- + template bool load_t_from_binary_file(t_struct& out, const std::string& binary_file) { std::string f_buff; diff --git a/contrib/epee/include/storages/portable_storage_val_converters.h b/contrib/epee/include/storages/portable_storage_val_converters.h index 36bb286275c..e54cda8284a 100644 --- a/contrib/epee/include/storages/portable_storage_val_converters.h +++ b/contrib/epee/include/storages/portable_storage_val_converters.h @@ -144,7 +144,7 @@ POP_WARNINGS { MTRACE("Converting std::string to uint64_t. Source: " << from); // String only contains digits - if(std::all_of(from.begin(), from.end(), ::isdigit)) + if(std::all_of(from.begin(), from.end(), epee::misc_utils::parse::isdigit)) to = boost::lexical_cast(from); // MyMonero ISO 8061 timestamp (2017-05-06T16:27:06Z) else if (boost::regex_match (from, boost::regex("\\d{4}-[01]\\d-[0-3]\\dT[0-2]\\d:[0-5]\\d:[0-5]\\dZ"))) diff --git a/contrib/epee/include/string_tools.h b/contrib/epee/include/string_tools.h index 63705e401b4..da47b7d5546 100644 --- a/contrib/epee/include/string_tools.h +++ b/contrib/epee/include/string_tools.h @@ -40,12 +40,13 @@ #include #include #include -#include -#include #include #include +#include "misc_log_ex.h" +#include "storages/parserse_base_utils.h" #include "hex.h" #include "memwipe.h" +#include "mlocker.h" #include "span.h" #include "warnings.h" @@ -58,89 +59,64 @@ #pragma comment (lib, "Rpcrt4.lib") #endif +static const constexpr unsigned char isx[256] = +{ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 10, 11, 12, 13, 14, 15, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 10, 11, 12, 13, 14, 15, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, +}; + namespace epee { namespace string_tools { - //---------------------------------------------------------------------------- - inline std::string get_str_from_guid_a(const boost::uuids::uuid& rid) - { - return boost::lexical_cast(rid); - } - //---------------------------------------------------------------------------- - inline bool get_guid_from_string(OUT boost::uuids::uuid& inetifer, const std::string& str_id) - { - std::string local_str_id = str_id; - if(local_str_id.size() < 36) - return false; - - if('{' == *local_str_id.begin()) - local_str_id.erase(0, 1); - - if('}' == *(--local_str_id.end())) - local_str_id.erase(--local_str_id.end()); - - try - { - inetifer = boost::lexical_cast(local_str_id); - return true; - } - catch(...) - { - return false; - } - } //---------------------------------------------------------------------------- inline std::string buff_to_hex_nodelimer(const std::string& src) { return to_hex::string(to_byte_span(to_span(src))); } //---------------------------------------------------------------------------- - template - bool parse_hexstr_to_binbuff(const std::basic_string& s, std::basic_string& res, bool allow_partial_byte = false) + inline bool parse_hexstr_to_binbuff(const epee::span s, epee::span& res) { - res.clear(); - if (!allow_partial_byte && (s.size() & 1)) - return false; - try - { - long v = 0; - for(size_t i = 0; i < (s.size() + 1) / 2; i++) + if (s.size() != res.size() * 2) + return false; + + unsigned char *dst = (unsigned char *)&res[0]; + const unsigned char *src = (const unsigned char *)s.data(); + for(size_t i = 0; i < s.size(); i += 2) { - CharT byte_str[3]; - size_t copied = s.copy(byte_str, 2, 2 * i); - byte_str[copied] = CharT(0); - CharT* endptr; - v = strtoul(byte_str, &endptr, 16); - if (v < 0 || 0xFF < v || endptr != byte_str + copied) - { - return false; - } - res.push_back(static_cast(v)); + int tmp = *src++; + tmp = isx[tmp]; + if (tmp == 0xff) return false; + int t2 = *src++; + t2 = isx[t2]; + if (t2 == 0xff) return false; + *dst++ = (tmp << 4) | t2; } return true; - }catch(...) - { - return false; - } } //---------------------------------------------------------------------------- - template - bool parse_tpod_from_hex_string(const std::string& str_hash, t_pod_type& t_pod) + inline bool parse_hexstr_to_binbuff(const std::string& s, std::string& res) { - static_assert(std::is_pod::value, "expected pod type"); - std::string buf; - bool res = epee::string_tools::parse_hexstr_to_binbuff(str_hash, buf); - if (!res || buf.size() != sizeof(t_pod_type)) - { + if (s.size() & 1) return false; - } - else - { - buf.copy(reinterpret_cast(&t_pod), sizeof(t_pod_type)); - return true; - } + res.resize(s.size() / 2); + epee::span rspan((char*)&res[0], res.size()); + return parse_hexstr_to_binbuff(epee::to_span(s), rspan); } //---------------------------------------------------------------------------- PUSH_WARNINGS @@ -152,7 +128,7 @@ DISABLE_GCC_WARNING(maybe-uninitialized) { for (char c : str_id) { - if (!std::isdigit(c)) + if (!epee::misc_utils::parse::isdigit(c)) return false; } } @@ -232,6 +208,15 @@ POP_WARNINGS return boost::lexical_cast(val); } //---------------------------------------------------------------------------- + inline std::string to_string_hex(uint32_t val) + { + std::stringstream ss; + ss << std::hex << val; + std::string s; + ss >> s; + return s; + } + //---------------------------------------------------------------------------- inline bool compare_no_case(const std::string& str1, const std::string& str2) { @@ -339,17 +324,10 @@ POP_WARNINGS bool hex_to_pod(const std::string& hex_str, t_pod_type& s) { static_assert(std::is_pod::value, "expected pod type"); - std::string hex_str_tr = trim(hex_str); if(sizeof(s)*2 != hex_str.size()) return false; - std::string bin_buff; - if(!parse_hexstr_to_binbuff(hex_str_tr, bin_buff)) - return false; - if(bin_buff.size()!=sizeof(s)) - return false; - - s = *(t_pod_type*)bin_buff.data(); - return true; + epee::span rspan((char*)&s, sizeof(s)); + return parse_hexstr_to_binbuff(epee::to_span(hex_str), rspan); } //---------------------------------------------------------------------------- template @@ -358,6 +336,12 @@ POP_WARNINGS return hex_to_pod(hex_str, unwrap(s)); } //---------------------------------------------------------------------------- + template + bool hex_to_pod(const std::string& hex_str, epee::mlocked& s) + { + return hex_to_pod(hex_str, unwrap(s)); + } + //---------------------------------------------------------------------------- bool validate_hex(uint64_t length, const std::string& str); //---------------------------------------------------------------------------- inline std::string get_extension(const std::string& str) @@ -381,6 +365,41 @@ POP_WARNINGS res = str.substr(0, pos); return res; } + //---------------------------------------------------------------------------- +#ifdef _WIN32 + inline std::wstring utf8_to_utf16(const std::string& str) + { + if (str.empty()) + return {}; + int wstr_size = MultiByteToWideChar(CP_UTF8, 0, &str[0], str.size(), NULL, 0); + if (wstr_size == 0) + { + throw std::runtime_error(std::error_code(GetLastError(), std::system_category()).message()); + } + std::wstring wstr(wstr_size, wchar_t{}); + if (!MultiByteToWideChar(CP_UTF8, 0, &str[0], str.size(), &wstr[0], wstr_size)) + { + throw std::runtime_error(std::error_code(GetLastError(), std::system_category()).message()); + } + return wstr; + } + inline std::string utf16_to_utf8(const std::wstring& wstr) + { + if (wstr.empty()) + return {}; + int str_size = WideCharToMultiByte(CP_UTF8, 0, &wstr[0], wstr.size(), NULL, 0, NULL, NULL); + if (str_size == 0) + { + throw std::runtime_error(std::error_code(GetLastError(), std::system_category()).message()); + } + std::string str(str_size, char{}); + if (!WideCharToMultiByte(CP_UTF8, 0, &wstr[0], wstr.size(), &str[0], str_size, NULL, NULL)) + { + throw std::runtime_error(std::error_code(GetLastError(), std::system_category()).message()); + } + return str; + } +#endif } } #endif //_STRING_TOOLS_H_ diff --git a/contrib/epee/include/wipeable_string.h b/contrib/epee/include/wipeable_string.h index 70d1a958642..f0e526b9204 100644 --- a/contrib/epee/include/wipeable_string.h +++ b/contrib/epee/include/wipeable_string.h @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2018, The Monero Project +// Copyright (c) 2017-2019, The Monero Project // // All rights reserved. // @@ -28,28 +28,46 @@ #pragma once +#include #include #include #include +#include "memwipe.h" +#include "fnv1.h" namespace epee { class wipeable_string { public: + typedef char value_type; + wipeable_string() {} wipeable_string(const wipeable_string &other); wipeable_string(wipeable_string &&other); wipeable_string(const std::string &other); wipeable_string(std::string &&other); wipeable_string(const char *s); + wipeable_string(const char *s, size_t len); ~wipeable_string(); void wipe(); void push_back(char c); - void pop_back(); + void operator+=(char c); + void operator+=(const std::string &s); + void operator+=(const epee::wipeable_string &s); + void operator+=(const char *s); + void append(const char *ptr, size_t len); + char pop_back(); const char *data() const noexcept { return buffer.data(); } + char *data() noexcept { return buffer.data(); } size_t size() const noexcept { return buffer.size(); } + size_t length() const noexcept { return buffer.size(); } bool empty() const noexcept { return buffer.empty(); } + void trim(); + void split(std::vector &fields) const; + boost::optional parse_hexstr() const; + template inline bool hex_to_pod(T &pod) const; + template inline bool hex_to_pod(tools::scrubbed &pod) const { return hex_to_pod(unwrap(pod)); } void resize(size_t sz); void reserve(size_t sz); void clear(); @@ -64,4 +82,29 @@ namespace epee private: std::vector buffer; }; + + template inline bool wipeable_string::hex_to_pod(T &pod) const + { + static_assert(std::is_pod::value, "expected pod type"); + if (size() != sizeof(T) * 2) + return false; + boost::optional blob = parse_hexstr(); + if (!blob) + return false; + if (blob->size() != sizeof(T)) + return false; + pod = *(const T*)blob->data(); + return true; + } +} + +namespace std +{ + template<> struct hash + { + size_t operator()(const epee::wipeable_string &s) const + { + return epee::fnv::FNV1a(s.data(), s.size()); + } + }; } diff --git a/contrib/epee/src/CMakeLists.txt b/contrib/epee/src/CMakeLists.txt index c4750cea037..2465afebbf2 100644 --- a/contrib/epee/src/CMakeLists.txt +++ b/contrib/epee/src/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2014-2018, The Monero Project +# Copyright (c) 2014-2019, The Monero Project # # All rights reserved. # @@ -26,8 +26,9 @@ # STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF # THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -add_library(epee STATIC hex.cpp http_auth.cpp mlog.cpp net_utils_base.cpp string_tools.cpp wipeable_string.cpp memwipe.c - connection_basic.cpp network_throttle.cpp network_throttle-detail.cpp) +add_library(epee STATIC hex.cpp http_auth.cpp mlog.cpp net_helper.cpp net_utils_base.cpp string_tools.cpp wipeable_string.cpp memwipe.c + connection_basic.cpp network_throttle.cpp network_throttle-detail.cpp mlocker.cpp buffer.cpp net_ssl.cpp) + if (USE_READLINE AND GNU_READLINE_FOUND) add_library(epee_readline STATIC readline_buffer.cpp) endif() @@ -54,7 +55,9 @@ endif() target_link_libraries(epee PUBLIC easylogging + ${Boost_CHRONO_LIBRARY} ${Boost_FILESYSTEM_LIBRARY} + ${Boost_THREAD_LIBRARY} PRIVATE ${OPENSSL_LIBRARIES} ${EXTRA_LIBRARIES}) diff --git a/contrib/epee/src/buffer.cpp b/contrib/epee/src/buffer.cpp new file mode 100644 index 00000000000..10ea6de56af --- /dev/null +++ b/contrib/epee/src/buffer.cpp @@ -0,0 +1,98 @@ +// Copyright (c) 2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include +#include "net/buffer.h" + +#undef MONERO_DEFAULT_LOG_CATEGORY +#define MONERO_DEFAULT_LOG_CATEGORY "net.buffer" + +namespace epee +{ +namespace net_utils +{ + +void buffer::append(const void *data, size_t sz) +{ + const size_t capacity = storage.capacity(); + const size_t avail = capacity - storage.size(); + + CHECK_AND_ASSERT_THROW_MES(storage.size() < std::numeric_limits::max() - sz, "Too much data to append"); + + // decide when to move + if (sz > avail) + { + // we have to reallocate or move + const bool move = size() + sz <= capacity; + if (move) + { + const size_t bytes = storage.size() - offset; + NET_BUFFER_LOG("appending " << sz << " from " << size() << " by moving " << bytes << " from offset " << offset << " first (forced)"); + memmove(storage.data(), storage.data() + offset, bytes); + storage.resize(bytes); + offset = 0; + } + else + { + NET_BUFFER_LOG("appending " << sz << " from " << size() << " by reallocating"); + std::vector new_storage; + size_t reserve = (((size() + sz) * 3 / 2) + 4095) & ~4095; + new_storage.reserve(reserve); + new_storage.resize(size()); + if (size() > 0) + memcpy(new_storage.data(), storage.data() + offset, storage.size() - offset); + offset = 0; + std::swap(storage, new_storage); + } + } + else + { + // we have space already + if (size() <= 4096 && offset > 4096 * 16 && offset >= capacity / 2) + { + // we have little to move, and we're far enough into the buffer that it's probably a win to move anyway + const size_t pos = storage.size() - offset; + NET_BUFFER_LOG("appending " << sz << " from " << size() << " by moving " << pos << " from offset " << offset << " first (unforced)"); + memmove(storage.data(), storage.data() + offset, storage.size() - offset); + storage.resize(pos); + offset = 0; + } + else + { + NET_BUFFER_LOG("appending " << sz << " from " << size() << " by writing to existing capacity"); + } + } + + // add the new data + storage.insert(storage.end(), (const uint8_t*)data, (const uint8_t*)data + sz); + + NET_BUFFER_LOG("storage now " << offset << "/" << storage.size() << "/" << storage.capacity()); +} + +} +} diff --git a/contrib/epee/src/connection_basic.cpp b/contrib/epee/src/connection_basic.cpp index dea1928a7e7..19f2c7b023c 100644 --- a/contrib/epee/src/connection_basic.cpp +++ b/contrib/epee/src/connection_basic.cpp @@ -2,7 +2,7 @@ /// @author rfree (current maintainer in monero.cc project) /// @brief base for connection, contains e.g. the ratelimit hooks -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -34,53 +34,27 @@ #include "net/connection_basic.hpp" -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include - -#include - -#include "syncobj.h" - #include "net/net_utils_base.h" #include "misc_log_ex.h" -#include -#include -#include -#include -#include -#include #include #include -#include #include "misc_language.h" #include "pragma_comp_defs.h" -#include -#include #include -#include -#include #include -#include -#include "net/abstract_tcp_server2.h" // TODO: #include "net/network_throttle-detail.hpp" +#if BOOST_VERSION >= 107000 +#define GET_IO_SERVICE(s) ((boost::asio::io_context&)(s).get_executor().context()) +#else +#define GET_IO_SERVICE(s) ((s).get_io_service()) +#endif + #undef MONERO_DEFAULT_LOG_CATEGORY -#define MONERO_DEFAULT_LOG_CATEGORY "net.p2p" +#define MONERO_DEFAULT_LOG_CATEGORY "net.conn" // ################################################################################################ // local (TU local) headers @@ -91,6 +65,15 @@ namespace epee namespace net_utils { +namespace +{ + boost::asio::ssl::context& get_context(connection_basic_shared_state* state) + { + CHECK_AND_ASSERT_THROW_MES(state != nullptr, "state shared_ptr cannot be null"); + return state->ssl_context; + } +} + std::string to_string(t_connection_type type) { if (type == e_connection_type_NET) @@ -135,7 +118,7 @@ namespace net_utils // connection_basic_pimpl // ================================================================================================ -connection_basic_pimpl::connection_basic_pimpl(const std::string &name) : m_throttle(name) { } +connection_basic_pimpl::connection_basic_pimpl(const std::string &name) : m_throttle(name), m_peer_number(0) { } // ================================================================================================ // connection_basic @@ -145,30 +128,58 @@ connection_basic_pimpl::connection_basic_pimpl(const std::string &name) : m_thro int connection_basic_pimpl::m_default_tos; // methods: -connection_basic::connection_basic(boost::asio::io_service& io_service, std::atomic &ref_sock_count, std::atomic &sock_number) - : +connection_basic::connection_basic(boost::asio::ip::tcp::socket&& sock, boost::shared_ptr state, ssl_support_t ssl_support) + : + m_state(std::move(state)), + mI( new connection_basic_pimpl("peer") ), + strand_(GET_IO_SERVICE(sock)), + socket_(GET_IO_SERVICE(sock), get_context(m_state.get())), + m_want_close_connection(false), + m_was_shutdown(false), + m_ssl_support(ssl_support) +{ + // add nullptr checks if removed + assert(m_state != nullptr); // release runtime check in get_context + + socket_.next_layer() = std::move(sock); + + ++(m_state->sock_count); // increase the global counter + mI->m_peer_number = m_state->sock_number.fetch_add(1); // use, and increase the generated number + + std::string remote_addr_str = "?"; + try { boost::system::error_code e; remote_addr_str = socket().remote_endpoint(e).address().to_string(); } catch(...){} ; + + _note("Spawned connection #"<m_peer_number<<" to " << remote_addr_str << " currently we have sockets count:" << m_state->sock_count); +} + +connection_basic::connection_basic(boost::asio::io_service &io_service, boost::shared_ptr state, ssl_support_t ssl_support) + : + m_state(std::move(state)), mI( new connection_basic_pimpl("peer") ), strand_(io_service), - socket_(io_service), - m_want_close_connection(false), + socket_(io_service, get_context(m_state.get())), + m_want_close_connection(false), m_was_shutdown(false), - m_ref_sock_count(ref_sock_count) -{ - ++ref_sock_count; // increase the global counter - mI->m_peer_number = sock_number.fetch_add(1); // use, and increase the generated number + m_ssl_support(ssl_support) +{ + // add nullptr checks if removed + assert(m_state != nullptr); // release runtime check in get_context + + ++(m_state->sock_count); // increase the global counter + mI->m_peer_number = m_state->sock_number.fetch_add(1); // use, and increase the generated number std::string remote_addr_str = "?"; - try { boost::system::error_code e; remote_addr_str = socket_.remote_endpoint(e).address().to_string(); } catch(...){} ; + try { boost::system::error_code e; remote_addr_str = socket().remote_endpoint(e).address().to_string(); } catch(...){} ; - _note("Spawned connection p2p#"<m_peer_number<<" to " << remote_addr_str << " currently we have sockets count:" << m_ref_sock_count); - //boost::filesystem::create_directories("log/dr-monero/net/"); + _note("Spawned connection #"<m_peer_number<<" to " << remote_addr_str << " currently we have sockets count:" << m_state->sock_count); } connection_basic::~connection_basic() noexcept(false) { + --(m_state->sock_count); + std::string remote_addr_str = "?"; - m_ref_sock_count--; - try { boost::system::error_code e; remote_addr_str = socket_.remote_endpoint(e).address().to_string(); } catch(...){} ; - _note("Destructing connection p2p#"<m_peer_number << " to " << remote_addr_str); + try { boost::system::error_code e; remote_addr_str = socket().remote_endpoint(e).address().to_string(); } catch(...){} ; + _note("Destructing connection #"<m_peer_number << " to " << remote_addr_str); } void connection_basic::set_rate_up_limit(uint64_t limit) { @@ -250,22 +261,15 @@ void connection_basic::sleep_before_packet(size_t packet_size, int phase, int q } } -void connection_basic::set_start_time() { - CRITICAL_REGION_LOCAL( network_throttle_manager::m_lock_get_global_throttle_out ); - m_start_time = network_throttle_manager::get_global_throttle_out().get_time_seconds(); -} void connection_basic::do_send_handler_write(const void* ptr , size_t cb ) { // No sleeping here; sleeping is done once and for all in connection::handle_write MTRACE("handler_write (direct) - before ASIO write, for packet="<::handle_write MTRACE("handler_write (after write, from queue="< src) + template + T to_hex::convert(const span src) { if (std::numeric_limits::max() / 2 < src.size()) throw std::range_error("hex_view::to_string exceeded maximum size"); - std::string out{}; + T out{}; out.resize(src.size() * 2); - buffer_unchecked(std::addressof(out[0]), src); + to_hex::buffer_unchecked((char*)out.data(), src); // can't see the non const version in wipeable_string?? return out; } + std::string to_hex::string(const span src) { return convert(src); } + epee::wipeable_string to_hex::wipeable_string(const span src) { return convert(src); } + void to_hex::buffer(std::ostream& out, const span src) { write_hex(std::ostreambuf_iterator{out}, src); @@ -79,4 +83,67 @@ namespace epee { return write_hex(out, src); } + + std::vector from_hex::vector(boost::string_ref src) + { + // should we include a specific character + auto include = [](char input) { + // we ignore spaces and colons + return !std::isspace(input) && input != ':'; + }; + + // the number of relevant characters to decode + auto count = std::count_if(src.begin(), src.end(), include); + + // this must be a multiple of two, otherwise we have a truncated input + if (count % 2) { + throw std::length_error{ "Invalid hexadecimal input length" }; + } + + std::vector result; + result.reserve(count / 2); + + // the data to work with (std::string is always null-terminated) + auto data = src.data(); + + // convert a single hex character to an unsigned integer + auto char_to_int = [](const char *input) { + switch (std::tolower(*input)) { + case '0': return 0; + case '1': return 1; + case '2': return 2; + case '3': return 3; + case '4': return 4; + case '5': return 5; + case '6': return 6; + case '7': return 7; + case '8': return 8; + case '9': return 9; + case 'a': return 10; + case 'b': return 11; + case 'c': return 12; + case 'd': return 13; + case 'e': return 14; + case 'f': return 15; + default: throw std::range_error{ "Invalid hexadecimal input" }; + } + }; + + // keep going until we reach the end + while (data[0] != '\0') { + // skip unwanted characters + if (!include(data[0])) { + ++data; + continue; + } + + // convert two matching characters to int + auto high = char_to_int(data++); + auto low = char_to_int(data++); + + result.push_back(high << 4 | low); + } + + return result; + } } diff --git a/contrib/epee/src/http_auth.cpp b/contrib/epee/src/http_auth.cpp index dc968d9716d..289069daaef 100644 --- a/contrib/epee/src/http_auth.cpp +++ b/contrib/epee/src/http_auth.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/contrib/epee/src/memwipe.c b/contrib/epee/src/memwipe.c index e3a2f76c8fd..ad1ef510dea 100644 --- a/contrib/epee/src/memwipe.c +++ b/contrib/epee/src/memwipe.c @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2018, The Monero Project +// Copyright (c) 2017-2019, The Monero Project // // All rights reserved. // @@ -50,7 +50,7 @@ void *memwipe(void *ptr, size_t n) { - if (memset_s(ptr, n, 0, n)) + if (n > 0 && memset_s(ptr, n, 0, n)) { #ifdef NDEBUG fprintf(stderr, "Error: memset_s failed\n"); @@ -67,7 +67,8 @@ void *memwipe(void *ptr, size_t n) void *memwipe(void *ptr, size_t n) { - explicit_bzero(ptr, n); + if (n > 0) + explicit_bzero(ptr, n); SCARECROW return ptr; } @@ -105,7 +106,8 @@ static void memory_cleanse(void *ptr, size_t len) void *memwipe(void *ptr, size_t n) { - memory_cleanse(ptr, n); + if (n > 0) + memory_cleanse(ptr, n); SCARECROW return ptr; } diff --git a/contrib/epee/src/mlocker.cpp b/contrib/epee/src/mlocker.cpp new file mode 100644 index 00000000000..446fa3315b4 --- /dev/null +++ b/contrib/epee/src/mlocker.cpp @@ -0,0 +1,202 @@ +// Copyright (c) 2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#if defined __GNUC__ && !defined _WIN32 +#define HAVE_MLOCK 1 +#endif + +#include +#if defined HAVE_MLOCK +#include +#endif +#include "misc_log_ex.h" +#include "syncobj.h" +#include "mlocker.h" + +#include + +#undef MONERO_DEFAULT_LOG_CATEGORY +#define MONERO_DEFAULT_LOG_CATEGORY "mlocker" + +// did an mlock operation previously fail? we only +// want to log an error once and be done with it +static std::atomic previously_failed{ false }; + +static size_t query_page_size() +{ +#if defined HAVE_MLOCK + long ret = sysconf(_SC_PAGESIZE); + if (ret <= 0) + { + MERROR("Failed to determine page size"); + return 0; + } + return ret; +#else +#warning Missing query_page_size implementation +#endif + return 0; +} + +static void do_lock(void *ptr, size_t len) +{ +#if defined HAVE_MLOCK + int ret = mlock(ptr, len); + if (ret < 0 && !previously_failed.exchange(true)) + MERROR("Error locking page at " << ptr << ": " << strerror(errno) << ", subsequent mlock errors will be silenced"); +#else +#warning Missing do_lock implementation +#endif +} + +static void do_unlock(void *ptr, size_t len) +{ +#if defined HAVE_MLOCK + int ret = munlock(ptr, len); + // check whether we previously failed, but don't set it, this is just + // to pacify the errors of mlock()ing failed, in which case unlocking + // is also not going to work of course + if (ret < 0 && !previously_failed.load()) + MERROR("Error unlocking page at " << ptr << ": " << strerror(errno)); +#else +#warning Missing implementation of page size detection +#endif +} + +namespace epee +{ + size_t mlocker::page_size = 0; + size_t mlocker::num_locked_objects = 0; + + boost::mutex &mlocker::mutex() + { + static boost::mutex *vmutex = new boost::mutex(); + return *vmutex; + } + std::map &mlocker::map() + { + static std::map *vmap = new std::map(); + return *vmap; + } + + size_t mlocker::get_page_size() + { + CRITICAL_REGION_LOCAL(mutex()); + if (page_size == 0) + page_size = query_page_size(); + return page_size; + } + + mlocker::mlocker(void *ptr, size_t len): ptr(ptr), len(len) + { + lock(ptr, len); + } + + mlocker::~mlocker() + { + try { unlock(ptr, len); } + catch (...) { /* ignore and do not propagate through the dtor */ } + } + + void mlocker::lock(void *ptr, size_t len) + { + TRY_ENTRY(); + + size_t page_size = get_page_size(); + if (page_size == 0) + return; + + CRITICAL_REGION_LOCAL(mutex()); + const size_t first = ((uintptr_t)ptr) / page_size; + const size_t last = (((uintptr_t)ptr) + len - 1) / page_size; + for (size_t page = first; page <= last; ++page) + lock_page(page); + ++num_locked_objects; + + CATCH_ENTRY_L1("mlocker::lock", void()); + } + + void mlocker::unlock(void *ptr, size_t len) + { + TRY_ENTRY(); + + size_t page_size = get_page_size(); + if (page_size == 0) + return; + CRITICAL_REGION_LOCAL(mutex()); + const size_t first = ((uintptr_t)ptr) / page_size; + const size_t last = (((uintptr_t)ptr) + len - 1) / page_size; + for (size_t page = first; page <= last; ++page) + unlock_page(page); + --num_locked_objects; + + CATCH_ENTRY_L1("mlocker::lock", void()); + } + + size_t mlocker::get_num_locked_pages() + { + CRITICAL_REGION_LOCAL(mutex()); + return map().size(); + } + + size_t mlocker::get_num_locked_objects() + { + CRITICAL_REGION_LOCAL(mutex()); + return num_locked_objects; + } + + void mlocker::lock_page(size_t page) + { + std::pair::iterator, bool> p = map().insert(std::make_pair(page, 1)); + if (p.second) + { + do_lock((void*)(page * page_size), page_size); + } + else + { + ++p.first->second; + } + } + + void mlocker::unlock_page(size_t page) + { + std::map::iterator i = map().find(page); + if (i == map().end()) + { + MERROR("Attempt to unlock unlocked page at " << (void*)(page * page_size)); + } + else + { + if (!--i->second) + { + map().erase(i); + do_unlock((void*)(page * page_size), page_size); + } + } + } +} diff --git a/contrib/epee/src/mlog.cpp b/contrib/epee/src/mlog.cpp index fb0b4ac2bb9..4c6ad5516be 100644 --- a/contrib/epee/src/mlog.cpp +++ b/contrib/epee/src/mlog.cpp @@ -28,11 +28,19 @@ #ifndef _MLOG_H_ #define _MLOG_H_ +#ifdef _WIN32 +#include +#ifndef ENABLE_VIRTUAL_TERMINAL_PROCESSING +#define ENABLE_VIRTUAL_TERMINAL_PROCESSING 0x0004 +#endif +#endif + #include #include #include #include #include "string_tools.h" +#include "misc_os_dependent.h" #include "misc_log_ex.h" #undef MONERO_DEFAULT_LOG_CATEGORY @@ -47,16 +55,12 @@ using namespace epee; static std::string generate_log_filename(const char *base) { std::string filename(base); + static unsigned int fallback_counter = 0; char tmp[200]; struct tm tm; time_t now = time(NULL); - if -#ifdef WIN32 - (!gmtime_s(&tm, &now)) -#else - (!gmtime_r(&now, &tm)) -#endif - strcpy(tmp, "unknown"); + if (!epee::misc_utils::get_gmt_time(now, tm)) + snprintf(tmp, sizeof(tmp), "part-%u", ++fallback_counter); else strftime(tmp, sizeof(tmp), "%Y-%m-%d-%H-%M-%S", &tm); tmp[sizeof(tmp) - 1] = 0; @@ -96,10 +100,10 @@ static const char *get_default_categories(int level) switch (level) { case 0: - categories = "*:WARNING,net:FATAL,net.p2p:FATAL,net.cn:FATAL,global:INFO,verify:FATAL,stacktrace:INFO,logging:INFO,msgwriter:INFO"; + categories = "*:WARNING,net:FATAL,net.http:FATAL,net.ssl:FATAL,net.p2p:FATAL,net.cn:FATAL,global:INFO,verify:FATAL,serialization:FATAL,stacktrace:INFO,logging:INFO,msgwriter:INFO"; break; case 1: - categories = "*:INFO,global:INFO,stacktrace:INFO,logging:INFO,msgwriter:INFO"; + categories = "*:INFO,global:INFO,stacktrace:INFO,logging:INFO,msgwriter:INFO,perf.*:DEBUG"; break; case 2: categories = "*:DEBUG"; @@ -116,7 +120,32 @@ static const char *get_default_categories(int level) return categories; } -void mlog_configure(const std::string &filename_base, bool console, const std::size_t max_log_file_size) +#ifdef WIN32 +bool EnableVTMode() +{ + // Set output mode to handle virtual terminal sequences + HANDLE hOut = GetStdHandle(STD_OUTPUT_HANDLE); + if (hOut == INVALID_HANDLE_VALUE) + { + return false; + } + + DWORD dwMode = 0; + if (!GetConsoleMode(hOut, &dwMode)) + { + return false; + } + + dwMode |= ENABLE_VIRTUAL_TERMINAL_PROCESSING; + if (!SetConsoleMode(hOut, dwMode)) + { + return false; + } + return true; +} +#endif + +void mlog_configure(const std::string &filename_base, bool console, const std::size_t max_log_file_size, const std::size_t max_log_files) { el::Configurations c; c.setGlobally(el::ConfigurationType::Filename, filename_base); @@ -134,9 +163,65 @@ void mlog_configure(const std::string &filename_base, bool console, const std::s el::Loggers::addFlag(el::LoggingFlag::DisableApplicationAbortOnFatalLog); el::Loggers::addFlag(el::LoggingFlag::ColoredTerminalOutput); el::Loggers::addFlag(el::LoggingFlag::StrictLogFileSizeCheck); - el::Helpers::installPreRollOutCallback([filename_base](const char *name, size_t){ + el::Helpers::installPreRollOutCallback([filename_base, max_log_files](const char *name, size_t){ std::string rname = generate_log_filename(filename_base.c_str()); - rename(name, rname.c_str()); + int ret = rename(name, rname.c_str()); + if (ret < 0) + { + // can't log a failure, but don't do the file removal below + return; + } + if (max_log_files != 0) + { + std::vector found_files; + const boost::filesystem::directory_iterator end_itr; + const boost::filesystem::path filename_base_path(filename_base); + const boost::filesystem::path parent_path = filename_base_path.has_parent_path() ? filename_base_path.parent_path() : "."; + for (boost::filesystem::directory_iterator iter(parent_path); iter != end_itr; ++iter) + { + const std::string filename = iter->path().string(); + if (filename.size() >= filename_base.size() && std::memcmp(filename.data(), filename_base.data(), filename_base.size()) == 0) + { + found_files.push_back(iter->path()); + } + } + if (found_files.size() >= max_log_files) + { + std::sort(found_files.begin(), found_files.end(), [](const boost::filesystem::path &a, const boost::filesystem::path &b) { + boost::system::error_code ec; + std::time_t ta = boost::filesystem::last_write_time(boost::filesystem::path(a), ec); + if (ec) + { + MERROR("Failed to get timestamp from " << a << ": " << ec); + ta = std::time(nullptr); + } + std::time_t tb = boost::filesystem::last_write_time(boost::filesystem::path(b), ec); + if (ec) + { + MERROR("Failed to get timestamp from " << b << ": " << ec); + tb = std::time(nullptr); + } + static_assert(std::is_integral(), "bad time_t"); + return ta < tb; + }); + for (size_t i = 0; i <= found_files.size() - max_log_files; ++i) + { + try + { + boost::system::error_code ec; + boost::filesystem::remove(found_files[i], ec); + if (ec) + { + MERROR("Failed to remove " << found_files[i] << ": " << ec); + } + } + catch (const std::exception &e) + { + MERROR("Failed to remove " << found_files[i] << ": " << e.what()); + } + } + } + } }); mlog_set_common_prefix(); const char *monero_log = getenv("MONERO_LOGS"); @@ -145,6 +230,9 @@ void mlog_configure(const std::string &filename_base, bool console, const std::s monero_log = get_default_categories(0); } mlog_set_log(monero_log); +#ifdef WIN32 + EnableVTMode(); +#endif } void mlog_set_categories(const char *categories) diff --git a/contrib/epee/src/net_helper.cpp b/contrib/epee/src/net_helper.cpp new file mode 100644 index 00000000000..3543f571645 --- /dev/null +++ b/contrib/epee/src/net_helper.cpp @@ -0,0 +1,54 @@ +#include "net/net_helper.h" + +namespace epee +{ +namespace net_utils +{ + boost::unique_future + direct_connect::operator()(const std::string& addr, const std::string& port, boost::asio::steady_timer& timeout) const + { + // Get a list of endpoints corresponding to the server name. + ////////////////////////////////////////////////////////////////////////// + boost::asio::ip::tcp::resolver resolver(GET_IO_SERVICE(timeout)); + boost::asio::ip::tcp::resolver::query query(boost::asio::ip::tcp::v4(), addr, port, boost::asio::ip::tcp::resolver::query::canonical_name); + boost::asio::ip::tcp::resolver::iterator iterator = resolver.resolve(query); + boost::asio::ip::tcp::resolver::iterator end; + if(iterator == end) // Documentation states that successful call is guaranteed to be non-empty + throw boost::system::system_error{boost::asio::error::fault, "Failed to resolve " + addr}; + + ////////////////////////////////////////////////////////////////////////// + + struct new_connection + { + boost::promise result_; + boost::asio::ip::tcp::socket socket_; + + explicit new_connection(boost::asio::io_service& io_service) + : result_(), socket_(io_service) + {} + }; + + const auto shared = std::make_shared(GET_IO_SERVICE(timeout)); + timeout.async_wait([shared] (boost::system::error_code error) + { + if (error != boost::system::errc::operation_canceled && shared && shared->socket_.is_open()) + { + shared->socket_.shutdown(boost::asio::ip::tcp::socket::shutdown_both); + shared->socket_.close(); + } + }); + shared->socket_.async_connect(*iterator, [shared] (boost::system::error_code error) + { + if (shared) + { + if (error) + shared->result_.set_exception(boost::system::system_error{error}); + else + shared->result_.set_value(std::move(shared->socket_)); + } + }); + return shared->result_.get_future(); + } +} +} + diff --git a/contrib/epee/src/net_ssl.cpp b/contrib/epee/src/net_ssl.cpp new file mode 100644 index 00000000000..7d48d2a6423 --- /dev/null +++ b/contrib/epee/src/net_ssl.cpp @@ -0,0 +1,531 @@ +// Copyright (c) 2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include +#include +#include +#include +#include "misc_log_ex.h" +#include "net/net_ssl.h" + +#undef MONERO_DEFAULT_LOG_CATEGORY +#define MONERO_DEFAULT_LOG_CATEGORY "net.ssl" + +// openssl genrsa -out /tmp/KEY 4096 +// openssl req -new -key /tmp/KEY -out /tmp/REQ +// openssl x509 -req -days 999999 -sha256 -in /tmp/REQ -signkey /tmp/KEY -out /tmp/CERT + +namespace +{ + struct openssl_bio_free + { + void operator()(BIO* ptr) const noexcept + { + BIO_free(ptr); + } + }; + using openssl_bio = std::unique_ptr; + + struct openssl_pkey_free + { + void operator()(EVP_PKEY* ptr) const noexcept + { + EVP_PKEY_free(ptr); + } + }; + using openssl_pkey = std::unique_ptr; + + struct openssl_rsa_free + { + void operator()(RSA* ptr) const noexcept + { + RSA_free(ptr); + } + }; + using openssl_rsa = std::unique_ptr; + + struct openssl_bignum_free + { + void operator()(BIGNUM* ptr) const noexcept + { + BN_free(ptr); + } + }; + using openssl_bignum = std::unique_ptr; + + struct openssl_ec_key_free + { + void operator()(EC_KEY* ptr) const noexcept + { + EC_KEY_free(ptr); + } + }; + using openssl_ec_key = std::unique_ptr; + + struct openssl_group_free + { + void operator()(EC_GROUP* ptr) const noexcept + { + EC_GROUP_free(ptr); + } + }; + using openssl_group = std::unique_ptr; + + boost::system::error_code load_ca_file(boost::asio::ssl::context& ctx, const std::string& path) + { + SSL_CTX* const ssl_ctx = ctx.native_handle(); // could be moved from context + if (ssl_ctx == nullptr) + return {boost::asio::error::invalid_argument}; + + if (!SSL_CTX_load_verify_locations(ssl_ctx, path.c_str(), nullptr)) + { + return boost::system::error_code{ + int(::ERR_get_error()), boost::asio::error::get_ssl_category() + }; + } + return boost::system::error_code{}; + } +} + +namespace epee +{ +namespace net_utils +{ + + +// https://stackoverflow.com/questions/256405/programmatically-create-x509-certificate-using-openssl +bool create_rsa_ssl_certificate(EVP_PKEY *&pkey, X509 *&cert) +{ + MGINFO("Generating SSL certificate"); + pkey = EVP_PKEY_new(); + if (!pkey) + { + MERROR("Failed to create new private key"); + return false; + } + + openssl_pkey pkey_deleter{pkey}; + openssl_rsa rsa{RSA_new()}; + if (!rsa) + { + MERROR("Error allocating RSA private key"); + return false; + } + + openssl_bignum exponent{BN_new()}; + if (!exponent) + { + MERROR("Error allocating exponent"); + return false; + } + + BN_set_word(exponent.get(), RSA_F4); + + if (RSA_generate_key_ex(rsa.get(), 4096, exponent.get(), nullptr) != 1) + { + MERROR("Error generating RSA private key"); + return false; + } + + if (EVP_PKEY_assign_RSA(pkey, rsa.get()) <= 0) + { + MERROR("Error assigning RSA private key"); + return false; + } + + // the RSA key is now managed by the EVP_PKEY structure + (void)rsa.release(); + + cert = X509_new(); + if (!cert) + { + MERROR("Failed to create new X509 certificate"); + return false; + } + ASN1_INTEGER_set(X509_get_serialNumber(cert), 1); + X509_gmtime_adj(X509_get_notBefore(cert), 0); + X509_gmtime_adj(X509_get_notAfter(cert), 3600 * 24 * 182); // half a year + if (!X509_set_pubkey(cert, pkey)) + { + MERROR("Error setting pubkey on certificate"); + X509_free(cert); + return false; + } + X509_NAME *name = X509_get_subject_name(cert); + X509_set_issuer_name(cert, name); + + if (X509_sign(cert, pkey, EVP_sha256()) == 0) + { + MERROR("Error signing certificate"); + X509_free(cert); + return false; + } + (void)pkey_deleter.release(); + return true; +} + +bool create_ec_ssl_certificate(EVP_PKEY *&pkey, X509 *&cert, int type) +{ + MGINFO("Generating SSL certificate"); + pkey = EVP_PKEY_new(); + if (!pkey) + { + MERROR("Failed to create new private key"); + return false; + } + + openssl_pkey pkey_deleter{pkey}; + openssl_ec_key ec_key{EC_KEY_new()}; + if (!ec_key) + { + MERROR("Error allocating EC private key"); + return false; + } + + EC_GROUP *group = EC_GROUP_new_by_curve_name(type); + if (!group) + { + MERROR("Error getting EC group " << type); + return false; + } + openssl_group group_deleter{group}; + + EC_GROUP_set_asn1_flag(group, OPENSSL_EC_NAMED_CURVE); + EC_GROUP_set_point_conversion_form(group, POINT_CONVERSION_UNCOMPRESSED); + + if (!EC_GROUP_check(group, NULL)) + { + MERROR("Group failed check: " << ERR_reason_error_string(ERR_get_error())); + return false; + } + if (EC_KEY_set_group(ec_key.get(), group) != 1) + { + MERROR("Error setting EC group"); + return false; + } + if (EC_KEY_generate_key(ec_key.get()) != 1) + { + MERROR("Error generating EC private key"); + return false; + } + if (EVP_PKEY_assign_EC_KEY(pkey, ec_key.get()) <= 0) + { + MERROR("Error assigning EC private key"); + return false; + } + + // the key is now managed by the EVP_PKEY structure + (void)ec_key.release(); + + cert = X509_new(); + if (!cert) + { + MERROR("Failed to create new X509 certificate"); + return false; + } + ASN1_INTEGER_set(X509_get_serialNumber(cert), 1); + X509_gmtime_adj(X509_get_notBefore(cert), 0); + X509_gmtime_adj(X509_get_notAfter(cert), 3600 * 24 * 182); // half a year + if (!X509_set_pubkey(cert, pkey)) + { + MERROR("Error setting pubkey on certificate"); + X509_free(cert); + return false; + } + X509_NAME *name = X509_get_subject_name(cert); + X509_set_issuer_name(cert, name); + + if (X509_sign(cert, pkey, EVP_sha256()) == 0) + { + MERROR("Error signing certificate"); + X509_free(cert); + return false; + } + (void)pkey_deleter.release(); + return true; +} + +ssl_options_t::ssl_options_t(std::vector> fingerprints, std::string ca_path) + : fingerprints_(std::move(fingerprints)), + ca_path(std::move(ca_path)), + auth(), + support(ssl_support_t::e_ssl_support_enabled), + verification(ssl_verification_t::user_certificates) +{ + std::sort(fingerprints_.begin(), fingerprints_.end()); +} + +boost::asio::ssl::context ssl_options_t::create_context() const +{ + boost::asio::ssl::context ssl_context{boost::asio::ssl::context::tlsv12}; + if (!bool(*this)) + return ssl_context; + + // only allow tls v1.2 and up + ssl_context.set_options(boost::asio::ssl::context::default_workarounds); + ssl_context.set_options(boost::asio::ssl::context::no_sslv2); + ssl_context.set_options(boost::asio::ssl::context::no_sslv3); + ssl_context.set_options(boost::asio::ssl::context::no_tlsv1); + ssl_context.set_options(boost::asio::ssl::context::no_tlsv1_1); + + // only allow a select handful of tls v1.3 and v1.2 ciphers to be used + SSL_CTX_set_cipher_list(ssl_context.native_handle(), "ECDHE-ECDSA-CHACHA20-POLY1305-SHA256:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-GCM-SHA256"); + + // set options on the SSL context for added security + SSL_CTX *ctx = ssl_context.native_handle(); + CHECK_AND_ASSERT_THROW_MES(ctx, "Failed to get SSL context"); + SSL_CTX_clear_options(ctx, SSL_OP_LEGACY_SERVER_CONNECT); // SSL_CTX_SET_OPTIONS(3) + SSL_CTX_set_session_cache_mode(ctx, SSL_SESS_CACHE_OFF); // https://stackoverflow.com/questions/22378442 +#ifdef SSL_OP_NO_TICKET + SSL_CTX_set_options(ctx, SSL_OP_NO_TICKET); // https://stackoverflow.com/questions/22378442 +#endif +#ifdef SSL_OP_NO_RENEGOTIATION + SSL_CTX_set_options(ctx, SSL_OP_NO_RENEGOTIATION); +#endif +#ifdef SSL_OP_NO_SESSION_RESUMPTION_ON_RENEGOTIATION + SSL_CTX_set_options(ctx, SSL_OP_NO_SESSION_RESUMPTION_ON_RENEGOTIATION); +#endif +#ifdef SSL_OP_NO_COMPRESSION + SSL_CTX_set_options(ctx, SSL_OP_NO_COMPRESSION); +#endif +#ifdef SSL_OP_CIPHER_SERVER_PREFERENCE + SSL_CTX_set_options(ctx, SSL_OP_CIPHER_SERVER_PREFERENCE); +#endif + SSL_CTX_set_ecdh_auto(ctx, 1); + + switch (verification) + { + case ssl_verification_t::system_ca: + ssl_context.set_default_verify_paths(); + break; + case ssl_verification_t::user_certificates: + ssl_context.set_verify_depth(0); + /* fallthrough */ + case ssl_verification_t::user_ca: + if (!ca_path.empty()) + { + const boost::system::error_code err = load_ca_file(ssl_context, ca_path); + if (err) + throw boost::system::system_error{err, "Failed to load user CA file at " + ca_path}; + } + break; + default: + break; + } + + CHECK_AND_ASSERT_THROW_MES(auth.private_key_path.empty() == auth.certificate_path.empty(), "private key and certificate must be either both given or both empty"); + if (auth.private_key_path.empty()) + { + EVP_PKEY *pkey; + X509 *cert; + bool ok = false; + +#ifdef USE_EXTRA_EC_CERT + CHECK_AND_ASSERT_THROW_MES(create_ec_ssl_certificate(pkey, cert, NID_secp256k1), "Failed to create certificate"); + CHECK_AND_ASSERT_THROW_MES(SSL_CTX_use_certificate(ctx, cert), "Failed to use generated certificate"); + if (!SSL_CTX_use_PrivateKey(ctx, pkey)) + MERROR("Failed to use generated EC private key for " << NID_secp256k1); + else + ok = true; + X509_free(cert); + EVP_PKEY_free(pkey); +#endif + + CHECK_AND_ASSERT_THROW_MES(create_rsa_ssl_certificate(pkey, cert), "Failed to create certificate"); + CHECK_AND_ASSERT_THROW_MES(SSL_CTX_use_certificate(ctx, cert), "Failed to use generated certificate"); + if (!SSL_CTX_use_PrivateKey(ctx, pkey)) + MERROR("Failed to use generated RSA private key for RSA"); + else + ok = true; + X509_free(cert); + EVP_PKEY_free(pkey); + + CHECK_AND_ASSERT_THROW_MES(ok, "Failed to use any generated certificate"); + } + else + auth.use_ssl_certificate(ssl_context); + + return ssl_context; +} + +void ssl_authentication_t::use_ssl_certificate(boost::asio::ssl::context &ssl_context) const +{ + ssl_context.use_private_key_file(private_key_path, boost::asio::ssl::context::pem); + ssl_context.use_certificate_chain_file(certificate_path); +} + +bool is_ssl(const unsigned char *data, size_t len) +{ + if (len < get_ssl_magic_size()) + return false; + + // https://security.stackexchange.com/questions/34780/checking-client-hello-for-https-classification + MDEBUG("SSL detection buffer, " << len << " bytes: " + << (unsigned)(unsigned char)data[0] << " " << (unsigned)(unsigned char)data[1] << " " + << (unsigned)(unsigned char)data[2] << " " << (unsigned)(unsigned char)data[3] << " " + << (unsigned)(unsigned char)data[4] << " " << (unsigned)(unsigned char)data[5] << " " + << (unsigned)(unsigned char)data[6] << " " << (unsigned)(unsigned char)data[7] << " " + << (unsigned)(unsigned char)data[8]); + if (data[0] == 0x16) // record + if (data[1] == 3) // major version + if (data[5] == 1) // ClientHello + if (data[6] == 0 && data[3]*256 + data[4] == data[7]*256 + data[8] + 4) // length check + return true; + return false; +} + +bool ssl_options_t::has_strong_verification(boost::string_ref host) const noexcept +{ + // onion and i2p addresses contain information about the server cert + // which both authenticates and encrypts + if (host.ends_with(".onion") || host.ends_with(".i2p")) + return true; + switch (verification) + { + default: + case ssl_verification_t::none: + case ssl_verification_t::system_ca: + return false; + case ssl_verification_t::user_certificates: + case ssl_verification_t::user_ca: + break; + } + return true; +} + +bool ssl_options_t::has_fingerprint(boost::asio::ssl::verify_context &ctx) const +{ + // can we check the certificate against a list of fingerprints? + if (!fingerprints_.empty()) { + X509_STORE_CTX *sctx = ctx.native_handle(); + if (!sctx) + { + MERROR("Error getting verify_context handle"); + return false; + } + + X509* cert = nullptr; + const STACK_OF(X509)* chain = X509_STORE_CTX_get_chain(sctx); + if (!chain || sk_X509_num(chain) < 1 || !(cert = sk_X509_value(chain, 0))) + { + MERROR("No certificate found in verify_context"); + return false; + } + + // buffer for the certificate digest and the size of the result + std::vector digest(EVP_MAX_MD_SIZE); + unsigned int size{ 0 }; + + // create the digest from the certificate + if (!X509_digest(cert, EVP_sha256(), digest.data(), &size)) { + MERROR("Failed to create certificate fingerprint"); + return false; + } + + // strip unnecessary bytes from the digest + digest.resize(size); + + return std::binary_search(fingerprints_.begin(), fingerprints_.end(), digest); + } + + return false; +} + +bool ssl_options_t::handshake(boost::asio::ssl::stream &socket, boost::asio::ssl::stream_base::handshake_type type, const std::string& host) const +{ + socket.next_layer().set_option(boost::asio::ip::tcp::no_delay(true)); + + /* Using system-wide CA store for client verification is funky - there is + no expected hostname for server to verify against. If server doesn't have + specific whitelisted certificates for client, don't require client to + send certificate at all. */ + const bool no_verification = verification == ssl_verification_t::none || + (type == boost::asio::ssl::stream_base::server && fingerprints_.empty() && ca_path.empty()); + + /* According to OpenSSL documentation (and SSL specifications), server must + always send certificate unless "anonymous" cipher mode is used which are + disabled by default. Either way, the certificate is never inspected. */ + if (no_verification) + socket.set_verify_mode(boost::asio::ssl::verify_none); + else + { + socket.set_verify_mode(boost::asio::ssl::verify_peer | boost::asio::ssl::verify_fail_if_no_peer_cert); + + // in case server is doing "virtual" domains, set hostname + SSL* const ssl_ctx = socket.native_handle(); + if (type == boost::asio::ssl::stream_base::client && !host.empty() && ssl_ctx) + SSL_set_tlsext_host_name(ssl_ctx, host.c_str()); + + socket.set_verify_callback([&](const bool preverified, boost::asio::ssl::verify_context &ctx) + { + // preverified means it passed system or user CA check. System CA is never loaded + // when fingerprints are whitelisted. + const bool verified = preverified && + (verification != ssl_verification_t::system_ca || host.empty() || boost::asio::ssl::rfc2818_verification(host)(preverified, ctx)); + + if (!verified && !has_fingerprint(ctx)) + { + // autodetect will reconnect without SSL - warn and keep connection encrypted + if (support != ssl_support_t::e_ssl_support_autodetect) + { + MERROR("SSL certificate is not in the allowed list, connection droppped"); + return false; + } + MWARNING("SSL peer has not been verified"); + } + return true; + }); + } + + boost::system::error_code ec; + socket.handshake(type, ec); + if (ec) + { + MERROR("SSL handshake failed, connection dropped: " << ec.message()); + return false; + } + MDEBUG("SSL handshake success"); + return true; +} + +bool ssl_support_from_string(ssl_support_t &ssl, boost::string_ref s) +{ + if (s == "enabled") + ssl = epee::net_utils::ssl_support_t::e_ssl_support_enabled; + else if (s == "disabled") + ssl = epee::net_utils::ssl_support_t::e_ssl_support_disabled; + else if (s == "autodetect") + ssl = epee::net_utils::ssl_support_t::e_ssl_support_autodetect; + else + return false; + return true; +} + +} // namespace +} // namespace + diff --git a/contrib/epee/src/net_utils_base.cpp b/contrib/epee/src/net_utils_base.cpp index 2f4015e814d..9b781027ec0 100644 --- a/contrib/epee/src/net_utils_base.cpp +++ b/contrib/epee/src/net_utils_base.cpp @@ -1,15 +1,13 @@ #include "net/net_utils_base.h" -#include "string_tools.h" -#include -#include +#include + +#include "string_tools.h" #include "net/local_ip.h" namespace epee { namespace net_utils { - const uint8_t ipv4_network_address::ID; - bool ipv4_network_address::equal(const ipv4_network_address& other) const noexcept { return is_same_host(other) && port() == other.port(); } @@ -58,24 +56,10 @@ namespace epee { namespace net_utils return self_->is_same_host(*other_self); } - bool create_network_address(network_address &address, const std::string &string, uint16_t default_port) - { - uint32_t ip; - uint16_t port; - if (epee::string_tools::parse_peer_from_string(ip, port, string)) - { - if (default_port && !port) - port = default_port; - address = ipv4_network_address{ip, port}; - return true; - } - return false; - } - std::string print_connection_context(const connection_context_base& ctx) { std::stringstream ss; - ss << ctx.m_remote_address.str() << " " << epee::string_tools::get_str_from_guid_a(ctx.m_connection_id) << (ctx.m_is_income ? " INC":" OUT"); + ss << ctx.m_remote_address.str() << " " << ctx.m_connection_id << (ctx.m_is_income ? " INC":" OUT"); return ss.str(); } @@ -86,5 +70,31 @@ namespace epee { namespace net_utils return ss.str(); } + const char* zone_to_string(zone value) noexcept + { + switch (value) + { + case zone::public_: + return "public"; + case zone::i2p: + return "i2p"; + case zone::tor: + return "tor"; + default: + break; + } + return "invalid"; + } + + zone zone_from_string(const boost::string_ref value) noexcept + { + if (value == "public") + return zone::public_; + if (value == "i2p") + return zone::i2p; + if (value == "tor") + return zone::tor; + return zone::invalid; + } }} diff --git a/contrib/epee/src/network_throttle-detail.cpp b/contrib/epee/src/network_throttle-detail.cpp index 7eeade3a190..72544cbf65c 100644 --- a/contrib/epee/src/network_throttle-detail.cpp +++ b/contrib/epee/src/network_throttle-detail.cpp @@ -2,7 +2,7 @@ /// @author rfree (current maintainer in monero.cc project) /// @brief implementaion for throttling of connection (count and rate-limit speed etc) -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -32,20 +32,11 @@ /* rfree: implementation for throttle details */ -#include #include #include -#include -#include #include #include -#include -#include -#include -#include -#include -#include #include @@ -53,14 +44,7 @@ #include "net/net_utils_base.h" #include "misc_log_ex.h" -#include -#include -#include #include -#include -#include -#include -#include #include "misc_language.h" #include "pragma_comp_defs.h" #include @@ -146,9 +130,14 @@ network_throttle::network_throttle(const std::string &nameshort, const std::stri m_network_add_cost = 128; m_network_minimal_segment = 256; m_network_max_segment = 1024*1024; + m_start_time = 0; m_any_packet_yet = false; m_slot_size = 1.0; // hard coded in few places m_target_speed = 16 * 1024; // other defaults are probably defined in the command-line parsing code when this class is used e.g. as main global throttle + m_last_sample_time = 0; + m_history.resize(m_window_size); + m_total_packets = 0; + m_total_bytes = 0; } void network_throttle::set_name(const std::string &name) @@ -182,8 +171,7 @@ void network_throttle::tick() { _dbg3("Moving counter buffer by 1 second " << last_sample_time_slot << " < " << current_sample_time_slot << " (last time " << m_last_sample_time<<")"); // rotate buffer - for (size_t i=m_history.size()-1; i>=1; --i) m_history[i] = m_history[i-1]; - m_history[0] = packet_info(); + m_history.push_front(packet_info()); if (! m_any_packet_yet) { m_last_sample_time = time_now; @@ -205,7 +193,9 @@ void network_throttle::_handle_trafic_exact(size_t packet_size, size_t orginal_s calculate_times_struct cts ; calculate_times(packet_size, cts , false, -1); calculate_times_struct cts2; calculate_times(packet_size, cts2, false, 5); - m_history[0].m_size += packet_size; + m_history.front().m_size += packet_size; + m_total_packets++; + m_total_bytes += packet_size; std::ostringstream oss; oss << "["; for (auto sample: m_history) oss << sample.m_size << " "; oss << "]" << std::ends; std::string history_str = oss.str(); @@ -366,6 +356,12 @@ double network_throttle::get_current_speed() const { return bytes_transferred / ((m_history.size() - 1) * m_slot_size); } +void network_throttle::get_stats(uint64_t &total_packets, uint64_t &total_bytes) const { + total_packets = m_total_packets; + total_bytes = m_total_bytes; +} + + } // namespace } // namespace diff --git a/contrib/epee/src/network_throttle.cpp b/contrib/epee/src/network_throttle.cpp index 167738855bd..f4f0b2c4632 100644 --- a/contrib/epee/src/network_throttle.cpp +++ b/contrib/epee/src/network_throttle.cpp @@ -26,7 +26,7 @@ Throttling work by: */ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/contrib/epee/src/readline_buffer.cpp b/contrib/epee/src/readline_buffer.cpp index 076a63612ce..39369c43ffa 100644 --- a/contrib/epee/src/readline_buffer.cpp +++ b/contrib/epee/src/readline_buffer.cpp @@ -1,9 +1,9 @@ #include "readline_buffer.h" #include #include -#include #include -#include +#include +#include #include static void install_line_handler(); @@ -44,7 +44,7 @@ std::vector& rdln::readline_buffer::completion_commands() } rdln::readline_buffer::readline_buffer() -: std::stringbuf(), m_cout_buf(NULL) +: std::stringbuf(), m_cout_buf(NULL), m_prompt_length(0) { current = this; } @@ -86,8 +86,11 @@ void rdln::readline_buffer::set_prompt(const std::string& prompt) if(m_cout_buf == NULL) return; boost::lock_guard lock(sync_mutex); + rl_set_prompt(std::string(m_prompt_length, ' ').c_str()); + rl_redisplay(); rl_set_prompt(prompt.c_str()); rl_redisplay(); + m_prompt_length = prompt.size(); } void rdln::readline_buffer::add_completion(const std::string& command) @@ -111,7 +114,7 @@ int rdln::readline_buffer::sync() int end = 0, point = 0; #endif - if (rl_end || *rl_prompt) + if (rl_end || (rl_prompt && *rl_prompt)) { #if RL_READLINE_VERSION >= 0x0700 rl_clear_visible_line(); @@ -134,7 +137,7 @@ int rdln::readline_buffer::sync() while ( this->snextc() != EOF ); #if RL_READLINE_VERSION < 0x0700 - if (end || *rl_prompt) + if (end || (rl_prompt && *rl_prompt)) { rl_restore_prompt(); rl_line_buffer = line; diff --git a/contrib/epee/src/wipeable_string.cpp b/contrib/epee/src/wipeable_string.cpp index 6ed4ee8a222..4209b71bf18 100644 --- a/contrib/epee/src/wipeable_string.cpp +++ b/contrib/epee/src/wipeable_string.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2018, The Monero Project +// Copyright (c) 2017-2019, The Monero Project // // All rights reserved. // @@ -26,11 +26,24 @@ // STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF // THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#include #include #include "memwipe.h" #include "misc_log_ex.h" #include "wipeable_string.h" +static constexpr const char hex[] = u8"0123456789abcdef"; + +namespace +{ + int atolower(int c) + { + if (c >= 'A' && c <= 'Z') + c |= 32; + return c; + } +} + namespace epee { @@ -49,13 +62,15 @@ wipeable_string::wipeable_string(wipeable_string &&other) wipeable_string::wipeable_string(const std::string &other) { grow(other.size()); - memcpy(buffer.data(), other.c_str(), size()); + if (size() > 0) + memcpy(buffer.data(), other.c_str(), size()); } wipeable_string::wipeable_string(std::string &&other) { grow(other.size()); - memcpy(buffer.data(), other.c_str(), size()); + if (size() > 0) + memcpy(buffer.data(), other.c_str(), size()); if (!other.empty()) { memwipe(&other[0], other.size()); // we're kinda left with this again aren't we @@ -66,7 +81,14 @@ wipeable_string::wipeable_string(std::string &&other) wipeable_string::wipeable_string(const char *s) { grow(strlen(s)); - memcpy(buffer.data(), s, size()); + if (size() > 0) + memcpy(buffer.data(), s, size()); +} + +wipeable_string::wipeable_string(const char *s, size_t len) +{ + grow(len); + memcpy(buffer.data(), s, len); } wipeable_string::~wipeable_string() @@ -93,14 +115,18 @@ void wipeable_string::grow(size_t sz, size_t reserved) } size_t old_sz = buffer.size(); std::unique_ptr tmp{new char[old_sz]}; - memcpy(tmp.get(), buffer.data(), old_sz * sizeof(char)); if (old_sz > 0) + { + memcpy(tmp.get(), buffer.data(), old_sz * sizeof(char)); memwipe(buffer.data(), old_sz * sizeof(char)); + } buffer.reserve(reserved); buffer.resize(sz); - memcpy(buffer.data(), tmp.get(), old_sz * sizeof(char)); if (old_sz > 0) + { + memcpy(buffer.data(), tmp.get(), old_sz * sizeof(char)); memwipe(tmp.get(), old_sz * sizeof(char)); + } } void wipeable_string::push_back(char c) @@ -109,9 +135,99 @@ void wipeable_string::push_back(char c) buffer.back() = c; } -void wipeable_string::pop_back() +void wipeable_string::operator+=(char c) +{ + push_back(c); +} + +void wipeable_string::append(const char *ptr, size_t len) +{ + const size_t orgsz = size(); + CHECK_AND_ASSERT_THROW_MES(orgsz < std::numeric_limits::max() - len, "Appended data too large"); + grow(orgsz + len); + if (len > 0) + memcpy(data() + orgsz, ptr, len); +} + +void wipeable_string::operator+=(const char *s) +{ + append(s, strlen(s)); +} + +void wipeable_string::operator+=(const epee::wipeable_string &s) +{ + append(s.data(), s.size()); +} + +void wipeable_string::operator+=(const std::string &s) +{ + append(s.c_str(), s.size()); +} + +void wipeable_string::trim() +{ + size_t prefix = 0; + while (prefix < size() && data()[prefix] == ' ') + ++prefix; + if (prefix > 0) + memmove(buffer.data(), buffer.data() + prefix, size() - prefix); + + size_t suffix = 0; + while (suffix < size()-prefix && data()[size() - 1 - prefix - suffix] == ' ') + ++suffix; + + resize(size() - prefix - suffix); +} + +void wipeable_string::split(std::vector &fields) const +{ + fields.clear(); + size_t len = size(); + const char *ptr = data(); + bool space = true; + while (len--) + { + const char c = *ptr++; + if (c != ' ') + { + if (space) + fields.push_back({}); + fields.back().push_back(c); + } + space = c == ' '; + } +} + +boost::optional wipeable_string::parse_hexstr() const +{ + if (size() % 2 != 0) + return boost::none; + boost::optional res = epee::wipeable_string(""); + const size_t len = size(); + const char *d = data(); + res->grow(0, len / 2); + for (size_t i = 0; i < len; i += 2) + { + char c = atolower(d[i]); + const char *ptr0 = strchr(hex, c); + if (!ptr0) + return boost::none; + c = atolower(d[i+1]); + const char *ptr1 = strchr(hex, c); + if (!ptr1) + return boost::none; + res->push_back(((ptr0-hex)<<4) | (ptr1-hex)); + } + return res; +} + +char wipeable_string::pop_back() { - resize(size() - 1); + const size_t sz = size(); + CHECK_AND_ASSERT_THROW_MES(sz > 0, "Popping from an empty string"); + const char c = buffer.back(); + resize(sz - 1); + return c; } void wipeable_string::resize(size_t sz) diff --git a/contrib/epee/tests/src/CMakeLists.txt b/contrib/epee/tests/src/CMakeLists.txt index c7d31735baf..4807fa7ead8 100644 --- a/contrib/epee/tests/src/CMakeLists.txt +++ b/contrib/epee/tests/src/CMakeLists.txt @@ -14,8 +14,8 @@ IF (MSVC) include_directories(SYSTEM platform/msvc) ELSE() # set stuff for other systems - SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c11 -Wall -Werror") - SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -Wall -Werror -Wno-reorder") + SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c11 -Wall") + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -Wall -Wno-reorder") ENDIF() diff --git a/contrib/epee/tests/src/net/test_net.h b/contrib/epee/tests/src/net/test_net.h index 04fef089ce6..f99639afc20 100644 --- a/contrib/epee/tests/src/net/test_net.h +++ b/contrib/epee/tests/src/net/test_net.h @@ -29,7 +29,9 @@ #include #include -#include "net/levin_server_cp2.h" +#include "net/abstract_tcp_server2.h" +#include "net/levin_protocol_handler.h" +#include "net/levin_protocol_handler_async.h" #include "storages/abstract_invoke.h" namespace epee @@ -60,7 +62,7 @@ namespace tests { const static int ID = 1000; - struct request + struct request_t { std::string example_string_data; @@ -73,9 +75,9 @@ namespace tests SERIALIZE_T(sub) END_NAMED_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init request; - - struct response + struct response_t { bool m_success; uint64_t example_id_data; @@ -87,13 +89,14 @@ namespace tests SERIALIZE_STL_CONTAINER_T(subs) END_NAMED_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init response; }; struct COMMAND_EXAMPLE_2 { const static int ID = 1001; - struct request + struct request_t { std::string example_string_data2; uint64_t example_id_data; @@ -103,8 +106,9 @@ namespace tests SERIALIZE_STL_ANSI_STRING(example_string_data2) END_NAMED_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init request; - struct response + struct response_t { bool m_success; uint64_t example_id_data; @@ -114,6 +118,7 @@ namespace tests SERIALIZE_POD(m_success) END_NAMED_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init response; }; typedef boost::uuids::uuid uuid; diff --git a/contrib/fuzz_testing/fuzz.sh b/contrib/fuzz_testing/fuzz.sh index f1c4ff2025b..efd43c23146 100755 --- a/contrib/fuzz_testing/fuzz.sh +++ b/contrib/fuzz_testing/fuzz.sh @@ -10,12 +10,12 @@ fi type="$1" if test -z "$type" then - echo "usage: $0 block|transaction|signature|cold-outputs|cold-transaction|load-from-binary|load-from-json|base58|parse-url|http-client|levin" + echo "usage: $0 block|transaction|signature|cold-outputs|cold-transaction|load-from-binary|load-from-json|base58|parse-url|http-client|levin|bulletproof" exit 1 fi case "$type" in - block|transaction|signature|cold-outputs|cold-transaction|load-from-binary|load-from-json|base58|parse-url|http-client|levin) ;; - *) echo "usage: $0 block|transaction|signature|cold-outputs|cold-transaction|load-from-binary|load-from-json|base58|parse-url|http-client|levin"; exit 1 ;; + block|transaction|signature|cold-outputs|cold-transaction|load-from-binary|load-from-json|base58|parse-url|http-client|levin|bulletproof) ;; + *) echo "usage: $0 block|transaction|signature|cold-outputs|cold-transaction|load-from-binary|load-from-json|base58|parse-url|http-client|levin|bulletproof"; exit 1 ;; esac if test -d "fuzz-out/$type" diff --git a/contrib/gitian/README.md b/contrib/gitian/README.md new file mode 100644 index 00000000000..1efc87e0ac7 --- /dev/null +++ b/contrib/gitian/README.md @@ -0,0 +1,179 @@ +Gitian building +================ + +*Setup instructions for a Gitian build of Monero.* + +Gitian is the deterministic build process that is used to build the Monero CLI +executables. It provides a way to be reasonably sure that the +executables are really built from the git source. It also makes sure that +the same, tested dependencies are used and statically built into the executable. + +Multiple developers build the source code by following a specific descriptor +("recipe"), cryptographically sign the result, and upload the resulting signature. +These results are compared and only if they match, the build is accepted and provided +for download. + +Gitian runs compilation steps in an isolated container. It is flexible and gives you full +control over the build environment, while still ensuring reproducibility and consistent output +formats. + +More independent Gitian builders are needed, which is why this guide exists. +It is preferred you follow these steps yourself instead of using someone else's +VM image to avoid 'contaminating' the build. + +Preparing the Gitian builder host +--------------------------------- + +The first step is to prepare the host environment that will be used to perform the Gitian builds. +This guide explains how to set up the environment, and how to start the builds. +Gitian offers to build with either `kvm`, `docker` or `lxc`. The default build +path chosen is `lxc`, but its setup is more complicated. You need to be logged in as the `gitianuser`. +If this user does not exist yet on your system, create it. Gitian can use +either kvm, lxc or docker as a host environment. This documentation will show +how to build with lxc and docker. While the docker setup is easy, the lxc setup +is more involved. + +LXC +--- + +LXC builds should be run on Ubuntu 18.04 "Bionic Beaver". + +Note that a version of `lxc-execute` higher or equal to 2.1.1 is required. +You can check the version with `lxc-execute --version`. + +First we need to set up dependencies. Type/paste the following in the terminal: + +```bash +sudo apt-get install git ruby apt-cacher-ng qemu-utils debootstrap lxc python-cheetah parted kpartx bridge-utils make ubuntu-archive-keyring curl firewalld +``` + +Then set up LXC and the rest with the following, which is a complex jumble of settings and workarounds: + +```bash +sudo -s +# the version of lxc-start in Debian needs to run as root, so make sure +# that the build script can execute it without providing a password +echo "%sudo ALL=NOPASSWD: /usr/bin/lxc-start" > /etc/sudoers.d/gitian-lxc +echo "%sudo ALL=NOPASSWD: /usr/bin/lxc-execute" >> /etc/sudoers.d/gitian-lxc +# make /etc/rc.local script that sets up bridge between guest and host +echo '#!/bin/sh -e' > /etc/rc.local +echo 'brctl addbr br0' >> /etc/rc.local +echo 'ip addr add 10.0.3.1/24 broadcast 10.0.3.255 dev br0' >> /etc/rc.local +echo 'ip link set br0 up' >> /etc/rc.local +echo 'firewall-cmd --zone=trusted --add-interface=br0' >> /etc/rc.local +echo 'exit 0' >> /etc/rc.local +chmod +x /etc/rc.local +# make sure that USE_LXC is always set when logging in as gitianuser, +# and configure LXC IP addresses +echo 'export USE_LXC=1' >> /home/gitianuser/.profile +echo 'export GITIAN_HOST_IP=10.0.3.1' >> /home/gitianuser/.profile +echo 'export LXC_GUEST_IP=10.0.3.5' >> /home/gitianuser/.profile +reboot +``` + +This setup is required to enable networking in the container. + +Docker +------ + +Building in docker does not require much setup. Install docker on your host, then type the following: + +```bash +sudo apt-get install git make curl +sudo usermod -aG docker gitianuser +``` + + +Manual and Building +------------------- + +The instructions below use the automated script [gitian-build.py](gitian-build.py) which only works in Ubuntu. +It calls all available .yml descriptors, which in turn pass the build configurations for different platforms to gitian. +Help for the build steps taken can be accessed with `./gitian-build.py --help`. + +Initial Gitian Setup +-------------------- + +The `gitian-build.py` script will checkout different release tags, so it's best to copy it to the top level directory: + +```bash +cp monero/contrib/gitian/gitian-build.py . +``` + +Setup the required environment, you only need to do this once: + +```bash +./gitian-build.py --setup fluffypony v0.14.0 +``` + +Where `fluffypony` is your Github name and `v0.14.0` is the version tag you want to build. +If you are using docker, run it with: + +```bash +./gitian-build.py --setup --docker fluffypony v0.14.0 +``` + +While gitian and this build script does provide a way for you to sign the build directly, it is recommended to sign in a seperate step. +This script is only there for convenience. Seperate steps for building can still be taken. +In order to sign gitian builds on your host machine, which has your PGP key, +fork the gitian.sigs repository and clone it on your host machine, +or pass the signed assert file back to your build machine. + +```bash +git clone git@github.com:monero-project/gitian.sigs.git +git remote add fluffypony git@github.com:fluffypony/gitian.sigs.git +``` + +Build Binaries +----------------------------- +To build the most recent tag (pass in `--docker` after setting up with docker): + +```bash +./gitian-build.py --detach-sign --no-commit -b fluffypony v0.14.0 +``` + +To speed up the build, use `-j 5 -m 5000` as the first arguments, where `5` is the number of CPU's you allocated to the VM plus one, and 5000 is a little bit less than then the MB's of RAM you allocated. If there is memory corruption on your machine, try to tweak these values. + +If all went well, this produces a number of (uncommited) `.assert` files in the gitian.sigs repository. + +If you do detached, offline signing, you need to copy these uncommited changes to your host machine, where you can sign them. For example: + +```bash +export NAME=fluffypony +export VERSION=v0.14.0 +gpg --output $VERSION-linux/$NAME/monero-linux-$VERSION-build.assert.sig --detach-sign $VERSION-linux/$NAME/monero-linux-$VERSION-build.assert +gpg --output $VERSION-osx-unsigned/$NAME/monero-osx-$VERSION-build.assert.sig --detach-sign $VERSION-osx-unsigned/$NAME/monero-osx-$VERSION-build.assert +gpg --output $VERSION-win-unsigned/$NAME/monero-win-$VERSION-build.assert.sig --detach-sign $VERSION-win-unsigned/$NAME/monero-win-$VERSION-build.assert +``` + +Make a pull request (both the `.assert` and `.assert.sig` files) to the +[monero-project/gitian.sigs](https://github.com/monero-project/gitian.sigs/) repository: + +```bash +git checkout -b v0.14.0 +git commit -S -a -m "Add $NAME v0.14.0" +git push --set-upstream $NAME v0.14.0 +``` + +```bash +gpg --detach-sign ${VERSION}-linux/${SIGNER}/monero-linux-*-build.assert +gpg --detach-sign ${VERSION}-win-unsigned/${SIGNER}/monero-win-*-build.assert +gpg --detach-sign ${VERSION}-osx-unsigned/${SIGNER}/monero-osx-*-build.assert +``` + +More Build Options +------------------ + +You can choose your own remote and commit hash by running for example: +```bash +./gitian-build.py --detach-sign --no-commit --url https://github.com/moneromooo-monero/bitmonero -b moneromooo 1f5680c8db8f4cc7acc04a04c724b832003440fd +``` + +Note that you won't be able to build commits authored before the gitian scripts +were added. Gitian clones the source files from the given url, be sure to push +to the remote you provide before building. +To get all build options run: +```bash +./gitian-build.py --help +``` + diff --git a/contrib/gitian/gitian-build.py b/contrib/gitian/gitian-build.py new file mode 100755 index 00000000000..cd88ecb20b7 --- /dev/null +++ b/contrib/gitian/gitian-build.py @@ -0,0 +1,192 @@ +#!/usr/bin/env python3 + +import argparse +import os +import subprocess +import sys + +def setup(): + global args, workdir + programs = ['ruby', 'git', 'apt-cacher-ng', 'make', 'wget'] + if args.kvm: + programs += ['python-vm-builder', 'qemu-kvm', 'qemu-utils'] + elif args.docker: + dockers = ['docker.io', 'docker-ce'] + for i in dockers: + return_code = subprocess.call(['sudo', 'apt-get', 'install', '-qq', i]) + if return_code == 0: + break + if return_code != 0: + print('Cannot find any way to install docker', file=sys.stderr) + exit(1) + else: + programs += ['lxc', 'debootstrap'] + subprocess.check_call(['sudo', 'apt-get', 'install', '-qq'] + programs) + if not os.path.isdir('gitian.sigs'): + subprocess.check_call(['git', 'clone', 'https://github.com/monero-project/gitian.sigs.git']) + if not os.path.isdir('gitian-builder'): + subprocess.check_call(['git', 'clone', 'https://github.com/devrandom/gitian-builder.git']) + if not os.path.isdir('monero'): + subprocess.check_call(['git', 'clone', 'https://github.com/monero-project/monero.git']) + os.chdir('gitian-builder') + subprocess.check_call(['git', 'checkout', '963322de8420c50502c4cc33d4d7c0d84437b576']) + make_image_prog = ['bin/make-base-vm', '--suite', 'bionic', '--arch', 'amd64'] + if args.docker: + make_image_prog += ['--docker'] + elif not args.kvm: + make_image_prog += ['--lxc'] + subprocess.check_call(make_image_prog) + os.chdir(workdir) + if args.is_bionic and not args.kvm and not args.docker: + subprocess.check_call(['sudo', 'sed', '-i', 's/lxcbr0/br0/', '/etc/default/lxc-net']) + print('Reboot is required') + exit(0) + +def build(): + global args, workdir + + os.makedirs('monero-binaries/' + args.version, exist_ok=True) + print('\nBuilding Dependencies\n') + os.chdir('gitian-builder') + os.makedirs('inputs', exist_ok=True) + + subprocess.check_call(['wget', '-N', '-P', 'inputs', 'https://downloads.sourceforge.net/project/osslsigncode/osslsigncode/osslsigncode-1.7.1.tar.gz']) + subprocess.check_call(['wget', '-N', '-P', 'inputs', 'https://bitcoincore.org/cfields/osslsigncode-Backports-to-1.7.1.patch']) + subprocess.check_output(["echo 'a8c4e9cafba922f89de0df1f2152e7be286aba73f78505169bc351a7938dd911 inputs/osslsigncode-Backports-to-1.7.1.patch' | sha256sum -c"], shell=True) + subprocess.check_output(["echo 'f9a8cdb38b9c309326764ebc937cba1523a3a751a7ab05df3ecc99d18ae466c9 inputs/osslsigncode-1.7.1.tar.gz' | sha256sum -c"], shell=True) + subprocess.check_call(['make', '-C', '../monero/contrib/depends', 'download', 'SOURCES_PATH=' + os.getcwd() + '/cache/common']) + + if args.linux: + print('\nCompiling ' + args.version + ' Linux') + subprocess.check_call(['bin/gbuild', '-j', args.jobs, '-m', args.memory, '--commit', 'monero='+args.commit, '--url', 'monero='+args.url, '../monero/contrib/gitian/gitian-linux.yml']) + subprocess.check_call(['bin/gsign', '-p', args.sign_prog, '--signer', args.signer, '--release', args.version+'-linux', '--destination', '../gitian.sigs/', '../monero/contrib/gitian/gitian-linux.yml']) + subprocess.check_call('mv build/out/monero-*.tar.gz ../monero-binaries/'+args.version, shell=True) + + if args.windows: + print('\nCompiling ' + args.version + ' Windows') + subprocess.check_call(['bin/gbuild', '-j', args.jobs, '-m', args.memory, '--commit', 'monero='+args.commit, '--url', 'monero='+args.url, '../monero/contrib/gitian/gitian-win.yml']) + subprocess.check_call(['bin/gsign', '-p', args.sign_prog, '--signer', args.signer, '--release', args.version+'-win', '--destination', '../gitian.sigs/', '../monero/contrib/gitian/gitian-win.yml']) + subprocess.check_call('mv build/out/monero*.zip ../monero-binaries/'+args.version, shell=True) + + if args.macos: + print('\nCompiling ' + args.version + ' MacOS') + subprocess.check_call(['bin/gbuild', '-j', args.jobs, '-m', args.memory, '--commit', 'monero='+args.commit, '--url', 'monero'+args.url, '../monero/contrib/gitian/gitian-osx.yml']) + subprocess.check_call(['bin/gsign', '-p', args.sign_prog, '--signer', args.signer, '--release', args.version+'-osx', '--destination', '../gitian.sigs/', '../monero/contrib/gitian/gitian-osx.yml']) + subprocess.check_call('mv build/out/monero*.tar.gz ../monero-binaries/'+args.version, shell=True) + + os.chdir(workdir) + + if args.commit_files: + print('\nCommitting '+args.version+' Unsigned Sigs\n') + os.chdir('gitian.sigs') + subprocess.check_call(['git', 'add', args.version+'-linux/'+args.signer]) + subprocess.check_call(['git', 'add', args.version+'-win/'+args.signer]) + subprocess.check_call(['git', 'add', args.version+'-osx/'+args.signer]) + subprocess.check_call(['git', 'commit', '-m', 'Add '+args.version+' unsigned sigs for '+args.signer]) + os.chdir(workdir) + +def verify(): + global args, workdir + os.chdir('gitian-builder') + + print('\nVerifying v'+args.version+' Linux\n') + subprocess.check_call(['bin/gverify', '-v', '-d', '../gitian.sigs/', '-r', args.version+'-linux', '../monero/contrib/gitian/gitian-linux.yml']) + print('\nVerifying v'+args.version+' Windows\n') + subprocess.check_call(['bin/gverify', '-v', '-d', '../gitian.sigs/', '-r', args.version+'-win', '../monero/contrib/gitian/gitian-win.yml']) + print('\nVerifying v'+args.version+' MacOS\n') + subprocess.check_call(['bin/gverify', '-v', '-d', '../gitian.sigs/', '-r', args.version+'-osx', '../monero/contrib/gitian/gitian-osx.yml']) + os.chdir(workdir) + +def main(): + global args, workdir + + parser = argparse.ArgumentParser(usage='%(prog)s [options] signer version') + parser.add_argument('-c', '--commit', action='store_true', dest='commit', help='Indicate that the version argument is for a commit or branch') + parser.add_argument('-p', '--pull', action='store_true', dest='pull', help='Indicate that the version argument is the number of a github repository pull request') + parser.add_argument('-u', '--url', dest='url', default='https://github.com/monero-project/monero', help='Specify the URL of the repository. Default is %(default)s') + parser.add_argument('-v', '--verify', action='store_true', dest='verify', help='Verify the Gitian build') + parser.add_argument('-b', '--build', action='store_true', dest='build', help='Do a Gitian build') + parser.add_argument('-B', '--buildsign', action='store_true', dest='buildsign', help='Build both signed and unsigned binaries') + parser.add_argument('-o', '--os', dest='os', default='lwm', help='Specify which Operating Systems the build is for. Default is %(default)s. l for Linux, w for Windows, m for MacOS') + parser.add_argument('-j', '--jobs', dest='jobs', default='2', help='Number of processes to use. Default %(default)s') + parser.add_argument('-m', '--memory', dest='memory', default='2000', help='Memory to allocate in MiB. Default %(default)s') + parser.add_argument('-k', '--kvm', action='store_true', dest='kvm', help='Use KVM instead of LXC') + parser.add_argument('-d', '--docker', action='store_true', dest='docker', help='Use Docker instead of LXC') + parser.add_argument('-S', '--setup', action='store_true', dest='setup', help='Set up the Gitian building environment. Uses LXC. If you want to use KVM, use the --kvm option. Only works on Debian-based systems (Ubuntu, Debian)') + parser.add_argument('-D', '--detach-sign', action='store_true', dest='detach_sign', help='Create the assert file for detached signing. Will not commit anything.') + parser.add_argument('-n', '--no-commit', action='store_false', dest='commit_files', help='Do not commit anything to git') + parser.add_argument('signer', help='GPG signer to sign each build assert file') + parser.add_argument('version', help='Version number, commit, or branch to build.') + + args = parser.parse_args() + workdir = os.getcwd() + + args.linux = 'l' in args.os + args.windows = 'w' in args.os + args.macos = 'm' in args.os + + args.is_bionic = b'bionic' in subprocess.check_output(['lsb_release', '-cs']) + + if args.buildsign: + args.build=True + args.sign=True + + if args.kvm and args.docker: + raise Exception('Error: cannot have both kvm and docker') + + args.sign_prog = 'true' if args.detach_sign else 'gpg --detach-sign' + + # Set enviroment variable USE_LXC or USE_DOCKER, let gitian-builder know that we use lxc or docker + if args.docker: + os.environ['USE_DOCKER'] = '1' + elif not args.kvm: + os.environ['USE_LXC'] = '1' + if not 'GITIAN_HOST_IP' in os.environ.keys(): + os.environ['GITIAN_HOST_IP'] = '10.0.3.1' + if not 'LXC_GUEST_IP' in os.environ.keys(): + os.environ['LXC_GUEST_IP'] = '10.0.3.5' + + # Disable for MacOS if no SDK found + if args.macos and not os.path.isfile('gitian-builder/inputs/MacOSX10.11.sdk.tar.gz'): + print('Cannot build for MacOS, SDK does not exist. Will build for other OSes') + args.macos = False + + script_name = os.path.basename(sys.argv[0]) + # Signer and version shouldn't be empty + if args.signer == '': + print(script_name+': Missing signer.') + print('Try '+script_name+' --help for more information') + exit(1) + if args.version == '': + print(script_name+': Missing version.') + print('Try '+script_name+' --help for more information') + exit(1) + + # Add leading 'v' for tags + if args.commit and args.pull: + raise Exception('Cannot have both commit and pull') + args.commit = args.commit if args.commit else args.version + + if args.setup: + setup() + + os.chdir('monero') + if args.pull: + subprocess.check_call(['git', 'fetch', args.url, 'refs/pull/'+args.version+'/merge']) + os.chdir('../gitian-builder/inputs/monero') + subprocess.check_call(['git', 'fetch', args.url, 'refs/pull/'+args.version+'/merge']) + args.commit = subprocess.check_output(['git', 'show', '-s', '--format=%H', 'FETCH_HEAD'], universal_newlines=True).strip() + args.version = 'pull-' + args.version + print(args.commit) + subprocess.check_call(['git', 'fetch']) + subprocess.check_call(['git', 'checkout', args.commit]) + os.chdir(workdir) + + if args.build: + build() + + if args.verify: + verify() + +if __name__ == '__main__': + main() diff --git a/contrib/gitian/gitian-linux.yml b/contrib/gitian/gitian-linux.yml new file mode 100644 index 00000000000..6e3ad108ee9 --- /dev/null +++ b/contrib/gitian/gitian-linux.yml @@ -0,0 +1,163 @@ +--- +name: "monero-linux-0.14" +enable_cache: true +suites: +- "bionic" +architectures: +- "amd64" +packages: +- "curl" +- "gperf" +- "gcc-7" +- "g++-7" +- "gcc" +- "g++" +- "gcc-7-aarch64-linux-gnu" +- "g++-7-aarch64-linux-gnu" +- "gcc-aarch64-linux-gnu" +- "g++-aarch64-linux-gnu" +- "binutils-aarch64-linux-gnu" +- "gcc-7-arm-linux-gnueabihf" +- "g++-7-arm-linux-gnueabihf" +- "gcc-arm-linux-gnueabihf" +- "g++-arm-linux-gnueabihf" +- "g++-7-multilib" +- "gcc-7-multilib" +- "binutils-arm-linux-gnueabihf" +- "binutils-gold" +- "git" +- "pkg-config" +- "build-essential" +- "autoconf" +- "libtool" +- "automake" +- "faketime" +- "bsdmainutils" +- "ca-certificates" +- "python" +- "cmake" +- "ccache" +- "protobuf-compiler" +- "libdbus-1-dev" +- "libharfbuzz-dev" +- "libprotobuf-dev" +- "python3-zmq" +remotes: +- "url": "https://github.com/monero-project/monero.git" + "dir": "monero" +files: [] +script: | + + WRAP_DIR=$HOME/wrapped + HOSTS="x86_64-linux-gnu arm-linux-gnueabihf aarch64-linux-gnu i686-linux-gnu" + FAKETIME_HOST_PROGS="gcc g++" + FAKETIME_PROGS="date ar ranlib nm" + HOST_CFLAGS="-O2 -g" + HOST_CXXFLAGS="-O2 -g" + HOST_LDFLAGS=-static-libstdc++ + + export GZIP="-9n" + export TAR_OPTIONS="--mtime="$REFERENCE_DATE\\\ $REFERENCE_TIME"" + export TZ="UTC" + export BUILD_DIR=`pwd` + mkdir -p ${WRAP_DIR} + if test -n "$GBUILD_CACHE_ENABLED"; then + export SOURCES_PATH=${GBUILD_COMMON_CACHE} + export BASE_CACHE=${GBUILD_PACKAGE_CACHE} + mkdir -p ${BASE_CACHE} ${SOURCES_PATH} + fi + + function create_global_faketime_wrappers { + for prog in ${FAKETIME_PROGS}; do + echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${prog} + echo "REAL=\`which -a ${prog} | grep -v ${WRAP_DIR}/${prog} | head -1\`" >> ${WRAP_DIR}/${prog} + echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${prog} + echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${prog} + echo "\$REAL \$@" >> $WRAP_DIR/${prog} + chmod +x ${WRAP_DIR}/${prog} + done + } + + function create_per-host_faketime_wrappers { + for i in $HOSTS; do + for prog in ${FAKETIME_HOST_PROGS}; do + if which ${i}-${prog}-7 + then + echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${i}-${prog} + echo "REAL=\`which -a ${i}-${prog}-7 | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAP_DIR}/${i}-${prog} + echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${i}-${prog} + echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${i}-${prog} + echo "\$REAL \$@" >> $WRAP_DIR/${i}-${prog} + chmod +x ${WRAP_DIR}/${i}-${prog} + fi + done + done + } + + # Faketime for depends so intermediate results are comparable + export PATH_orig=${PATH} + create_global_faketime_wrappers "2000-01-01 12:00:00" + create_per-host_faketime_wrappers "2000-01-01 12:00:00" + export PATH=${WRAP_DIR}:${PATH} + + EXTRA_INCLUDES_BASE=$WRAP_DIR/extra_includes + mkdir -p $EXTRA_INCLUDES_BASE + + # x86 needs /usr/include/i386-linux-gnu/asm pointed to /usr/include/x86_64-linux-gnu/asm, + # but we can't write there. Instead, create a link here and force it to be included in the + # search paths by wrapping gcc/g++. + + mkdir -p $EXTRA_INCLUDES_BASE/i686-pc-linux-gnu + rm -f $WRAP_DIR/extra_includes/i686-pc-linux-gnu/asm + ln -s /usr/include/x86_64-linux-gnu/asm $EXTRA_INCLUDES_BASE/i686-pc-linux-gnu/asm + + for prog in gcc g++; do + rm -f ${WRAP_DIR}/${prog} + cat << EOF > ${WRAP_DIR}/${prog} + #!/usr/bin/env bash + REAL="`which -a ${prog}-7 | grep -v ${WRAP_DIR}/${prog} | head -1`" + for var in "\$@" + do + if [ "\$var" = "-m32" ]; then + export C_INCLUDE_PATH="$EXTRA_INCLUDES_BASE/i686-pc-linux-gnu" + export CPLUS_INCLUDE_PATH="$EXTRA_INCLUDES_BASE/i686-pc-linux-gnu" + break + fi + done + \$REAL \$@ + EOF + chmod +x ${WRAP_DIR}/${prog} + done + + git config --global core.abbrev 9 + cd monero + BASEPREFIX=`pwd`/contrib/depends + # Build dependencies for each host + for i in $HOSTS; do + EXTRA_INCLUDES="$EXTRA_INCLUDES_BASE/$i" + if [ -d "$EXTRA_INCLUDES" ]; then + export HOST_ID_SALT="$EXTRA_INCLUDES" + fi + make ${MAKEOPTS} -C ${BASEPREFIX} HOST="${i}" V=1 + unset HOST_ID_SALT + done + + # Faketime for binaries + export PATH=${PATH_orig} + create_global_faketime_wrappers "${REFERENCE_DATETIME}" + create_per-host_faketime_wrappers "${REFERENCE_DATETIME}" + export PATH=${WRAP_DIR}:${PATH} + + ORIGPATH="$PATH" + # Build in a new dir for each host + for i in ${HOSTS}; do + export PATH=${BASEPREFIX}/${i}/native/bin:${ORIGPATH} + mkdir build && cd build + cmake .. -DCMAKE_TOOLCHAIN_FILE=${BASEPREFIX}/${i}/share/toolchain.cmake -DBACKCOMPAT=ON + make ${MAKEOPTS} + DISTNAME=monero-${i} + mv bin ${DISTNAME} + find ${DISTNAME}/ | sort | tar --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}.tar.gz + cd .. + rm -rf build + done diff --git a/contrib/gitian/gitian-osx.yml b/contrib/gitian/gitian-osx.yml new file mode 100644 index 00000000000..1cdb8527314 --- /dev/null +++ b/contrib/gitian/gitian-osx.yml @@ -0,0 +1,111 @@ +--- +name: "monero-osx-0.14" +enable_cache: true +suites: +- "bionic" +architectures: +- "amd64" +packages: +- "ca-certificates" +- "curl" +- "g++" +- "git" +- "pkg-config" +- "autoconf" +- "libtool" +- "automake" +- "faketime" +- "bsdmainutils" +- "cmake" +- "libcap-dev" +- "libz-dev" +- "libbz2-dev" +- "python" +- "python-dev" +- "python-setuptools" +remotes: +- "url": "https://github.com/monero-project/monero.git" + "dir": "monero" +files: +- "MacOSX10.11.sdk.tar.gz" +script: | + WRAP_DIR=$HOME/wrapped + HOSTS="x86_64-apple-darwin11" + FAKETIME_HOST_PROGS="" + FAKETIME_PROGS="ar ranlib date dmg genisoimage" + + export GZIP="-9n" + export TAR_OPTIONS="--mtime="$REFERENCE_DATE\\\ $REFERENCE_TIME"" + export TZ="UTC" + export BUILD_DIR=`pwd` + mkdir -p ${WRAP_DIR} + if test -n "$GBUILD_CACHE_ENABLED"; then + export SOURCES_PATH=${GBUILD_COMMON_CACHE} + export BASE_CACHE=${GBUILD_PACKAGE_CACHE} + mkdir -p ${BASE_CACHE} ${SOURCES_PATH} + fi + + export ZERO_AR_DATE=1 + + function create_global_faketime_wrappers { + for prog in ${FAKETIME_PROGS}; do + echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${prog} + echo "REAL=\`which -a ${prog} | grep -v ${WRAP_DIR}/${prog} | head -1\`" >> ${WRAP_DIR}/${prog} + echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${prog} + echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${prog} + echo "\$REAL \$@" >> $WRAP_DIR/${prog} + chmod +x ${WRAP_DIR}/${prog} + done + } + + function create_per-host_faketime_wrappers { + for i in $HOSTS; do + for prog in ${FAKETIME_HOST_PROGS}; do + echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${i}-${prog} + echo "REAL=\`which -a ${i}-${prog} | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAP_DIR}/${i}-${prog} + echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${i}-${prog} + echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${i}-${prog} + echo "\$REAL \$@" >> $WRAP_DIR/${i}-${prog} + chmod +x ${WRAP_DIR}/${i}-${prog} + done + done + } + + # Faketime for depends so intermediate results are comparable + export PATH_orig=${PATH} + create_global_faketime_wrappers "2000-01-01 12:00:00" + create_per-host_faketime_wrappers "2000-01-01 12:00:00" + export PATH=${WRAP_DIR}:${PATH} + + git config --global core.abbrev 9 + cd monero + BASEPREFIX=`pwd`/contrib/depends + + mkdir -p ${BASEPREFIX}/SDKs + tar -C ${BASEPREFIX}/SDKs -xf ${BUILD_DIR}/MacOSX10.11.sdk.tar.gz + + # Build dependencies for each host + for i in $HOSTS; do + make ${MAKEOPTS} -C ${BASEPREFIX} HOST="${i}" + done + + # Faketime for binaries + export PATH=${PATH_orig} + create_global_faketime_wrappers "${REFERENCE_DATETIME}" + create_per-host_faketime_wrappers "${REFERENCE_DATETIME}" + export PATH=${WRAP_DIR}:${PATH} + + ORIGPATH="$PATH" + # Build in a new dir for each host + for i in ${HOSTS}; do + export PATH=${BASEPREFIX}/${i}/native/bin:${ORIGPATH} + mkdir build && cd build + cmake .. -DCMAKE_TOOLCHAIN_FILE=${BASEPREFIX}/${i}/share/toolchain.cmake + make ${MAKEOPTS} + DISTNAME=monero-${i} + mv bin ${DISTNAME} + find ${DISTNAME}/ | sort | tar --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}.tar.gz + cd .. + rm -rf build + done + diff --git a/contrib/gitian/gitian-win.yml b/contrib/gitian/gitian-win.yml new file mode 100644 index 00000000000..d15383905e5 --- /dev/null +++ b/contrib/gitian/gitian-win.yml @@ -0,0 +1,135 @@ +--- +name: "monero-win-0.14" +enable_cache: true +suites: +- "bionic" +architectures: +- "amd64" +packages: +- "curl" +- "g++" +- "git" +- "pkg-config" +- "autoconf" +- "libtool" +- "automake" +- "faketime" +- "bsdmainutils" +- "mingw-w64" +- "g++-mingw-w64" +- "zip" +- "ca-certificates" +- "python" +- "rename" +- "cmake" +remotes: +- "url": "https://github.com/monero-project/monero.git" + "dir": "monero" +files: [] +script: | + WRAP_DIR=$HOME/wrapped + HOSTS="i686-w64-mingw32 x86_64-w64-mingw32" + FAKETIME_HOST_PROGS="ar ranlib nm windres strip objcopy" + FAKETIME_PROGS="date zip" + HOST_CFLAGS="-O2 -g" + HOST_CXXFLAGS="-O2 -g" + + export GZIP="-9n" + export TAR_OPTIONS="--mtime="$REFERENCE_DATE\\\ $REFERENCE_TIME"" + export TZ="UTC" + export BUILD_DIR=`pwd` + mkdir -p ${WRAP_DIR} + if test -n "$GBUILD_CACHE_ENABLED"; then + export SOURCES_PATH=${GBUILD_COMMON_CACHE} + export BASE_CACHE=${GBUILD_PACKAGE_CACHE} + mkdir -p ${BASE_CACHE} ${SOURCES_PATH} + fi + + function create_global_faketime_wrappers { + for prog in ${FAKETIME_PROGS}; do + echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${prog} + echo "REAL=\`which -a ${prog} | grep -v ${WRAP_DIR}/${prog} | head -1\`" >> ${WRAP_DIR}/${prog} + echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${prog} + echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${prog} + echo "\$REAL \$@" >> $WRAP_DIR/${prog} + chmod +x ${WRAP_DIR}/${prog} + done + } + + function create_per-host_faketime_wrappers { + for i in $HOSTS; do + for prog in ${FAKETIME_HOST_PROGS}; do + echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${i}-${prog} + echo "REAL=\`which -a ${i}-${prog} | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAP_DIR}/${i}-${prog} + echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${i}-${prog} + echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${i}-${prog} + echo "\$REAL \$@" >> $WRAP_DIR/${i}-${prog} + chmod +x ${WRAP_DIR}/${i}-${prog} + done + done + } + + function create_per-host_linker_wrapper { + # This is only needed for trusty, as the mingw linker leaks a few bytes of + # heap, causing non-determinism. See discussion in https://github.com/bitcoin/bitcoin/pull/6900 + for i in $HOSTS; do + mkdir -p ${WRAP_DIR}/${i} + for prog in collect2; do + echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${i}/${prog} + REAL=$(${i}-gcc -print-prog-name=${prog}) + echo "export MALLOC_PERTURB_=255" >> ${WRAP_DIR}/${i}/${prog} + echo "${REAL} \$@" >> $WRAP_DIR/${i}/${prog} + chmod +x ${WRAP_DIR}/${i}/${prog} + done + for prog in gcc g++; do + echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${i}-${prog} + echo "REAL=\`which -a ${i}-${prog}-posix | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAP_DIR}/${i}-${prog} + echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${i}-${prog} + echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${i}-${prog} + echo "export COMPILER_PATH=${WRAP_DIR}/${i}" >> ${WRAP_DIR}/${i}-${prog} + echo "\$REAL \$@" >> $WRAP_DIR/${i}-${prog} + chmod +x ${WRAP_DIR}/${i}-${prog} + done + done + } + + # Faketime for depends so intermediate results are comparable + export PATH_orig=${PATH} + create_global_faketime_wrappers "2000-01-01 12:00:00" + create_per-host_faketime_wrappers "2000-01-01 12:00:00" + create_per-host_linker_wrapper "2000-01-01 12:00:00" + export PATH=${WRAP_DIR}:${PATH} + + git config --global core.abbrev 9 + cd monero + BASEPREFIX=`pwd`/contrib/depends + # Build dependencies for each host + for i in $HOSTS; do + EXTRA_INCLUDES="$EXTRA_INCLUDES_BASE/$i" + if [ -d "$EXTRA_INCLUDES" ]; then + export HOST_ID_SALT="$EXTRA_INCLUDES" + fi + make ${MAKEOPTS} -C ${BASEPREFIX} HOST="${i}" V=1 + unset HOST_ID_SALT + done + + # Faketime for binaries + export PATH=${PATH_orig} + create_global_faketime_wrappers "${REFERENCE_DATETIME}" + create_per-host_faketime_wrappers "${REFERENCE_DATETIME}" + export PATH=${WRAP_DIR}:${PATH} + + ORIGPATH="$PATH" + # Run cmake and make, for each create a new build/ directory, + # compile from there, archive, export and delete the archive again + for i in ${HOSTS}; do + export PATH=${BASEPREFIX}/${i}/native/bin:${ORIGPATH} + mkdir build && cd build + cmake .. -DCMAKE_TOOLCHAIN_FILE=${BASEPREFIX}/${i}/share/toolchain.cmake + make ${MAKEOPTS} + DISTNAME=monero-${i} + mv bin ${DISTNAME} + find ${DISTNAME}/ | sort | zip -X@ ${OUTDIR}/${DISTNAME}.zip + cd .. && rm -rf build + done + diff --git a/contrib/snap/monerod.conf b/contrib/snap/monerod.conf index 9b3d308ede0..1078e73b2da 100644 --- a/contrib/snap/monerod.conf +++ b/contrib/snap/monerod.conf @@ -1,8 +1,9 @@ # Configuration for monerod # Syntax: any command line option may be specified as 'clioptionname=value'. +# Boolean options such as 'no-igd' are specified as 'no-igd=1'. # See 'monerod --help' for all available options. -# Overrided by snap: +# Overridden by snap: # data-dir=/var/lib/monero # log-file=/var/log/monero/monero.log diff --git a/contrib/snap/setup/gui/icon.png b/contrib/snap/setup/gui/icon.png index 17b8bd47b2f..b7e82127049 100644 Binary files a/contrib/snap/setup/gui/icon.png and b/contrib/snap/setup/gui/icon.png differ diff --git a/contrib/valgrind/monero.supp b/contrib/valgrind/monero.supp index 16e34e82feb..015b05a1c17 100644 --- a/contrib/valgrind/monero.supp +++ b/contrib/valgrind/monero.supp @@ -17,3 +17,12 @@ fun:maybe_unlock_and_signal_one > ... } + +{ + we leak the logger, for performance reasons in on-the-fly init + Memcheck:Leak + match-leak-kinds: definite + fun:_Znwm + fun:_ZN2el4base7Storage7getELPPEv + ... +} diff --git a/external/CMakeLists.txt b/external/CMakeLists.txt index 1fc4d64c1cb..bc4344b3483 100644 --- a/external/CMakeLists.txt +++ b/external/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2014-2018, The Monero Project +# Copyright (c) 2014-2019, The Monero Project # # All rights reserved. # @@ -38,15 +38,16 @@ find_package(Miniupnpc REQUIRED) message(STATUS "Using in-tree miniupnpc") - add_subdirectory(miniupnp/miniupnpc) - set_property(TARGET libminiupnpc-static PROPERTY FOLDER "external") if(MSVC) set_property(TARGET libminiupnpc-static APPEND_STRING PROPERTY COMPILE_FLAGS " -wd4244 -wd4267") elseif(NOT MSVC) set_property(TARGET libminiupnpc-static APPEND_STRING PROPERTY COMPILE_FLAGS " -Wno-undef -Wno-unused-result -Wno-unused-value") endif() +if(CMAKE_SYSTEM_NAME MATCHES "NetBSD") + set_property(TARGET libminiupnpc-static APPEND_STRING PROPERTY COMPILE_FLAGS " -D_NETBSD_SOURCE") +endif() set(UPNP_LIBRARIES "libminiupnpc-static" PARENT_SCOPE) diff --git a/external/db_drivers/CMakeLists.txt b/external/db_drivers/CMakeLists.txt index 13f70aa40ef..04776c475e3 100644 --- a/external/db_drivers/CMakeLists.txt +++ b/external/db_drivers/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2014-2018, The Monero Project +# Copyright (c) 2014-2019, The Monero Project # # All rights reserved. # @@ -32,3 +32,6 @@ message(STATUS "Using ${ARCH_WIDTH}-bit LMDB from source tree") add_subdirectory(liblmdb) set(LMDB_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/liblmdb" CACHE STRING "LMDB Include path") set(LMDB_LIBRARY "lmdb" CACHE STRING "LMDB Library name") +if(CMAKE_SYSTEM_NAME MATCHES "NetBSD") + set_property(TARGET lmdb APPEND_STRING PROPERTY COMPILE_FLAGS " -D_SEM_SEMUN_UNDEFINED") +endif() diff --git a/external/db_drivers/liblmdb/.gitignore b/external/db_drivers/liblmdb/.gitignore index d5102a87c07..80b6d811445 100644 --- a/external/db_drivers/liblmdb/.gitignore +++ b/external/db_drivers/liblmdb/.gitignore @@ -5,6 +5,7 @@ mdb_copy mdb_stat mdb_dump mdb_load +mdb_drop *.lo *.[ao] *.so diff --git a/external/db_drivers/liblmdb/CMakeLists.txt b/external/db_drivers/liblmdb/CMakeLists.txt index 2e8822f54a4..562ebb1eb14 100644 --- a/external/db_drivers/liblmdb/CMakeLists.txt +++ b/external/db_drivers/liblmdb/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2014-2018, The Monero Project +# Copyright (c) 2014-2019, The Monero Project # # All rights reserved. # diff --git a/external/db_drivers/liblmdb/COPYRIGHT b/external/db_drivers/liblmdb/COPYRIGHT index 722d1a51545..f076556eb50 100644 --- a/external/db_drivers/liblmdb/COPYRIGHT +++ b/external/db_drivers/liblmdb/COPYRIGHT @@ -1,4 +1,4 @@ -Copyright 2011-2015 Howard Chu, Symas Corp. +Copyright 2011-2019 Howard Chu, Symas Corp. All rights reserved. Redistribution and use in source and binary forms, with or without diff --git a/external/db_drivers/liblmdb/Doxyfile b/external/db_drivers/liblmdb/Doxyfile index 5047c0bb1fc..5ca2cfe8f6f 100644 --- a/external/db_drivers/liblmdb/Doxyfile +++ b/external/db_drivers/liblmdb/Doxyfile @@ -253,7 +253,7 @@ IDL_PROPERTY_SUPPORT = YES # member in the group (if any) for the other members of the group. By default # all members of a group must be documented explicitly. -DISTRIBUTE_GROUP_DOC = NO +DISTRIBUTE_GROUP_DOC = YES # Set the SUBGROUPING tag to YES (the default) to allow class member groups of # the same type (for instance a group of public functions) to be put as a diff --git a/external/db_drivers/liblmdb/Makefile b/external/db_drivers/liblmdb/Makefile index 186168038c5..942d0dbe4c8 100644 --- a/external/db_drivers/liblmdb/Makefile +++ b/external/db_drivers/liblmdb/Makefile @@ -26,6 +26,7 @@ OPT = -O2 -g CFLAGS = $(THREADS) $(OPT) $(W) $(XCFLAGS) LDLIBS = SOLIBS = +SOEXT = .so prefix = /usr/local exec_prefix = $(prefix) bindir = $(exec_prefix)/bin @@ -37,7 +38,7 @@ mandir = $(datarootdir)/man ######################################################################## IHDRS = lmdb.h -ILIBS = liblmdb.a liblmdb.so +ILIBS = liblmdb.a liblmdb$(SOEXT) IPROGS = mdb_stat mdb_copy mdb_dump mdb_load mdb_drop IDOCS = mdb_stat.1 mdb_copy.1 mdb_dump.1 mdb_load.1 mdb_drop.1 PROGS = $(IPROGS) mtest mtest2 mtest3 mtest4 mtest5 @@ -63,7 +64,7 @@ test: all liblmdb.a: mdb.o midl.o $(AR) rs $@ mdb.o midl.o -liblmdb.so: mdb.lo midl.lo +liblmdb$(SOEXT): mdb.lo midl.lo # $(CC) $(LDFLAGS) -pthread -shared -Wl,-Bsymbolic -o $@ mdb.o midl.o $(SOLIBS) $(CC) $(LDFLAGS) -pthread -shared -o $@ mdb.lo midl.lo $(SOLIBS) diff --git a/external/db_drivers/liblmdb/intro.doc b/external/db_drivers/liblmdb/intro.doc index 870c7bb8e72..64dfcaad848 100644 --- a/external/db_drivers/liblmdb/intro.doc +++ b/external/db_drivers/liblmdb/intro.doc @@ -1,5 +1,5 @@ /* - * Copyright 2015 Howard Chu, Symas Corp. + * Copyright 2015-2018 Howard Chu, Symas Corp. * All rights reserved. * * Redistribution and use in source and binary forms, with or without diff --git a/external/db_drivers/liblmdb/lmdb.h b/external/db_drivers/liblmdb/lmdb.h index 0bca3eb7481..1f4736ce2be 100644 --- a/external/db_drivers/liblmdb/lmdb.h +++ b/external/db_drivers/liblmdb/lmdb.h @@ -97,11 +97,12 @@ * transactions. Each transaction belongs to one thread. See below. * The #MDB_NOTLS flag changes this for read-only transactions. * - * - Use an MDB_env* in the process which opened it, without fork()ing. + * - Use an MDB_env* in the process which opened it, not after fork(). * * - Do not have open an LMDB database twice in the same process at * the same time. Not even from a plain open() call - close()ing it - * breaks flock() advisory locking. + * breaks fcntl() advisory locking. (It is OK to reopen it after + * fork() - exec*(), since the lockfile has FD_CLOEXEC set.) * * - Avoid long-lived transactions. Read transactions prevent * reuse of pages freed by newer write transactions, thus the @@ -135,7 +136,7 @@ * * @author Howard Chu, Symas Corporation. * - * @copyright Copyright 2011-2016 Howard Chu, Symas Corp. All rights reserved. + * @copyright Copyright 2011-2019 Howard Chu, Symas Corp. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted only as authorized by the OpenLDAP @@ -167,6 +168,7 @@ #include #include +#include #ifdef __cplusplus extern "C" { @@ -179,11 +181,30 @@ typedef int mdb_mode_t; typedef mode_t mdb_mode_t; #endif -#ifdef MDB_VL32 -typedef uint64_t mdb_size_t; -#define mdb_env_create mdb_env_create_vl32 /**< Prevent mixing with non-VL32 builds */ +#ifdef _WIN32 +# define MDB_FMT_Z "I" #else +# define MDB_FMT_Z "z" /**< printf/scanf format modifier for size_t */ +#endif + +#ifndef MDB_VL32 +/** Unsigned type used for mapsize, entry counts and page/transaction IDs. + * + * It is normally size_t, hence the name. Defining MDB_VL32 makes it + * uint64_t, but do not try this unless you know what you are doing. + */ typedef size_t mdb_size_t; +# define MDB_SIZE_MAX SIZE_MAX /**< max #mdb_size_t */ +/** #mdb_size_t printf formats, \b t = one of [diouxX] without quotes */ +# define MDB_PRIy(t) MDB_FMT_Z #t +/** #mdb_size_t scanf formats, \b t = one of [dioux] without quotes */ +# define MDB_SCNy(t) MDB_FMT_Z #t +#else +typedef uint64_t mdb_size_t; +# define MDB_SIZE_MAX UINT64_MAX +# define MDB_PRIy(t) PRI##t##64 +# define MDB_SCNy(t) SCN##t##64 +# define mdb_env_create mdb_env_create_vl32 /**< Prevent mixing with non-VL32 builds */ #endif /** An abstraction for a file handle. @@ -322,7 +343,8 @@ typedef void (MDB_rel_func)(MDB_val *item, void *oldptr, void *newptr, void *rel #define MDB_REVERSEKEY 0x02 /** use sorted duplicates */ #define MDB_DUPSORT 0x04 - /** numeric keys in native byte order: either unsigned int or size_t. + /** numeric keys in native byte order, either unsigned int or #mdb_size_t. + * (lmdb expects 32-bit int <= size_t <= 32/64-bit mdb_size_t.) * The keys must all be of the same size. */ #define MDB_INTEGERKEY 0x08 /** with #MDB_DUPSORT, sorted dup items have fixed size */ @@ -380,7 +402,7 @@ typedef enum MDB_cursor_op { MDB_GET_BOTH, /**< Position at key/data pair. Only for #MDB_DUPSORT */ MDB_GET_BOTH_RANGE, /**< position at key, nearest data. Only for #MDB_DUPSORT */ MDB_GET_CURRENT, /**< Return key/data at current cursor position */ - MDB_GET_MULTIPLE, /**< Return key and up to a page of duplicate data items + MDB_GET_MULTIPLE, /**< Return up to a page of duplicate data items from current cursor position. Move cursor to prepare for #MDB_NEXT_MULTIPLE. Only for #MDB_DUPFIXED */ MDB_LAST, /**< Position at last key/data item */ @@ -389,7 +411,7 @@ typedef enum MDB_cursor_op { MDB_NEXT, /**< Position at next data item */ MDB_NEXT_DUP, /**< Position at next data item of current key. Only for #MDB_DUPSORT */ - MDB_NEXT_MULTIPLE, /**< Return key and up to a page of duplicate data items + MDB_NEXT_MULTIPLE, /**< Return up to a page of duplicate data items from next cursor position. Move cursor to prepare for #MDB_NEXT_MULTIPLE. Only for #MDB_DUPFIXED */ MDB_NEXT_NODUP, /**< Position at first data item of next key */ @@ -400,7 +422,7 @@ typedef enum MDB_cursor_op { MDB_SET, /**< Position at specified key */ MDB_SET_KEY, /**< Position at specified key, return key + data */ MDB_SET_RANGE, /**< Position at first key greater than or equal to specified key. */ - MDB_PREV_MULTIPLE /**< Position at previous page and return key and up to + MDB_PREV_MULTIPLE /**< Position at previous page and return up to a page of duplicate data items. Only for #MDB_DUPFIXED */ } MDB_cursor_op; @@ -458,8 +480,10 @@ typedef enum MDB_cursor_op { #define MDB_BAD_VALSIZE (-30781) /** The specified DBI was changed unexpectedly */ #define MDB_BAD_DBI (-30780) + /** Unexpected problem - txn should abort */ +#define MDB_PROBLEM (-30779) /** The last defined error code */ -#define MDB_LAST_ERRCODE MDB_BAD_DBI +#define MDB_LAST_ERRCODE MDB_PROBLEM /** @} */ /** @brief Statistics for a database in the environment */ @@ -696,6 +720,7 @@ int mdb_env_copyfd(MDB_env *env, mdb_filehandle_t fd); *
  • #MDB_CP_COMPACT - Perform compaction while copying: omit free * pages and sequentially renumber all pages in output. This option * consumes more CPU and runs more slowly than the default. + * Currently it fails if the environment has suffered a page leak. * * @return A non-zero error value on failure and 0 on success. */ @@ -810,6 +835,10 @@ int mdb_env_get_flags(MDB_env *env, unsigned int *flags); int mdb_env_get_path(MDB_env *env, const char **path); /** @brief Return the filedescriptor for the given environment. + * + * This function may be called after fork(), so the descriptor can be + * closed before exec*(). Other LMDB file descriptors have FD_CLOEXEC. + * (Until LMDB 0.9.18, only the lockfile had that.) * * @param[in] env An environment handle returned by #mdb_env_create() * @param[out] fd Address of a mdb_filehandle_t to contain the descriptor. @@ -1112,14 +1141,16 @@ int mdb_txn_renew(MDB_txn *txn); * keys must be unique and may have only a single data item. *
  • #MDB_INTEGERKEY * Keys are binary integers in native byte order, either unsigned int - * or size_t, and will be sorted as such. + * or #mdb_size_t, and will be sorted as such. + * (lmdb expects 32-bit int <= size_t <= 32/64-bit mdb_size_t.) * The keys must all be of the same size. *
  • #MDB_DUPFIXED * This flag may only be used in combination with #MDB_DUPSORT. This option * tells the library that the data items for this database are all the same * size, which allows further optimizations in storage and retrieval. When - * all data items are the same size, the #MDB_GET_MULTIPLE and #MDB_NEXT_MULTIPLE - * cursor operations may be used to retrieve multiple items at once. + * all data items are the same size, the #MDB_GET_MULTIPLE, #MDB_NEXT_MULTIPLE + * and #MDB_PREV_MULTIPLE cursor operations may be used to retrieve multiple + * items at once. *
  • #MDB_INTEGERDUP * This option specifies that duplicate data items are binary integers, * similar to #MDB_INTEGERKEY keys. @@ -1524,6 +1555,10 @@ int mdb_cursor_put(MDB_cursor *cursor, MDB_val *key, MDB_val *data, /** @brief Delete current key/data pair * * This function deletes the key/data pair to which the cursor refers. + * This does not invalidate the cursor, so operations such as MDB_NEXT + * can still be used on it. + * Both MDB_NEXT and MDB_GET_CURRENT will return the same record after + * this operation. * @param[in] cursor A cursor handle returned by #mdb_cursor_open() * @param[in] flags Options for this operation. This parameter * must be set to 0 or one of the values described here. diff --git a/external/db_drivers/liblmdb/mdb.c b/external/db_drivers/liblmdb/mdb.c index 8e2b1eca195..ba131540125 100644 --- a/external/db_drivers/liblmdb/mdb.c +++ b/external/db_drivers/liblmdb/mdb.c @@ -5,7 +5,7 @@ * BerkeleyDB API, but much simplified. */ /* - * Copyright 2011-2016 Howard Chu, Symas Corp. + * Copyright 2011-2019 Howard Chu, Symas Corp. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -41,6 +41,7 @@ #ifdef _WIN32 #include #include +#include /* get wcscpy() */ /* We use native NT APIs to setup the memory map, so that we can * let the DB file grow incrementally instead of always preallocating @@ -51,7 +52,7 @@ * NTDLL.DLL at runtime, to avoid buildtime dependencies on any * NTDLL import libraries. */ -typedef NTSTATUS WINAPI (NtCreateSectionFunc) +typedef NTSTATUS (WINAPI NtCreateSectionFunc) (OUT PHANDLE sh, IN ACCESS_MASK acc, IN void * oa OPTIONAL, IN PLARGE_INTEGER ms OPTIONAL, @@ -64,7 +65,7 @@ typedef enum _SECTION_INHERIT { ViewUnmap = 2 } SECTION_INHERIT; -typedef NTSTATUS WINAPI (NtMapViewOfSectionFunc) +typedef NTSTATUS (WINAPI NtMapViewOfSectionFunc) (IN PHANDLE sh, IN HANDLE ph, IN OUT PVOID *addr, IN ULONG_PTR zbits, IN SIZE_T cs, IN OUT PLARGE_INTEGER off OPTIONAL, @@ -73,7 +74,7 @@ typedef NTSTATUS WINAPI (NtMapViewOfSectionFunc) static NtMapViewOfSectionFunc *NtMapViewOfSection; -typedef NTSTATUS WINAPI (NtCloseFunc)(HANDLE h); +typedef NTSTATUS (WINAPI NtCloseFunc)(HANDLE h); static NtCloseFunc *NtClose; @@ -147,6 +148,10 @@ typedef SSIZE_T ssize_t; /* Most platforms have posix_memalign, older may only have memalign */ #define HAVE_MEMALIGN 1 #include +/* On Solaris, we need the POSIX sigwait function */ +#if defined (__sun) +# define _POSIX_PTHREAD_SEMANTICS 1 +#endif #endif #if !(defined(BYTE_ORDER) || defined(__BYTE_ORDER)) @@ -154,7 +159,7 @@ typedef SSIZE_T ssize_t; #include /* defines BYTE_ORDER on HPUX and Solaris */ #endif -#if defined(__APPLE__) || defined (BSD) +#if defined(__APPLE__) || defined (BSD) || defined(__FreeBSD_kernel__) # if !(defined(MDB_USE_POSIX_MUTEX) || defined(MDB_USE_POSIX_SEM)) # define MDB_USE_SYSV_SEM 1 # endif @@ -165,6 +170,7 @@ typedef SSIZE_T ssize_t; #ifndef _WIN32 #include +#include #ifdef MDB_USE_POSIX_SEM # define MDB_USE_HASH 1 #include @@ -234,7 +240,7 @@ union semun { #if (BYTE_ORDER == LITTLE_ENDIAN) == (BYTE_ORDER == BIG_ENDIAN) # error "Unknown or unsupported endianness (BYTE_ORDER)" -#elif (-6 & 5) || CHAR_BIT != 8 || UINT_MAX < 0xffffffff || ULONG_MAX % 0xFFFF +#elif (-6 & 5) || CHAR_BIT!=8 || UINT_MAX!=0xffffffff || MDB_SIZE_MAX%UINT_MAX # error "Two's complement, reasonably sized integer types, please" #endif @@ -309,6 +315,10 @@ union semun { # define MDB_USE_ROBUST 0 # else # define MDB_USE_ROBUST 1 +# endif +#endif /* !MDB_USE_ROBUST */ + +#if defined(MDB_USE_POSIX_MUTEX) && (MDB_USE_ROBUST) /* glibc < 2.12 only provided _np API */ # if (defined(__GLIBC__) && GLIBC_VER < 0x02000c) || \ (defined(PTHREAD_MUTEX_ROBUST_NP) && !defined(PTHREAD_MUTEX_ROBUST)) @@ -316,10 +326,9 @@ union semun { # define pthread_mutexattr_setrobust(attr, flag) pthread_mutexattr_setrobust_np(attr, flag) # define pthread_mutex_consistent(mutex) pthread_mutex_consistent_np(mutex) # endif -# endif -#endif /* MDB_USE_ROBUST */ +#endif /* MDB_USE_POSIX_MUTEX && MDB_USE_ROBUST */ -#if defined(MDB_OWNERDEAD) && MDB_USE_ROBUST +#if defined(MDB_OWNERDEAD) && (MDB_USE_ROBUST) #define MDB_ROBUST_SUPPORTED 1 #endif @@ -342,8 +351,10 @@ typedef HANDLE mdb_mutex_t, mdb_mutexref_t; #define pthread_mutex_lock(x) WaitForSingleObject(*x, INFINITE) #define pthread_cond_signal(x) SetEvent(*x) #define pthread_cond_wait(cond,mutex) do{SignalObjectAndWait(*mutex, *cond, INFINITE, FALSE); WaitForSingleObject(*mutex, INFINITE);}while(0) -#define THREAD_CREATE(thr,start,arg) thr=CreateThread(NULL,0,start,arg,0,NULL) -#define THREAD_FINISH(thr) WaitForSingleObject(thr, INFINITE) +#define THREAD_CREATE(thr,start,arg) \ + (((thr) = CreateThread(NULL, 0, start, arg, 0, NULL)) ? 0 : ErrCode()) +#define THREAD_FINISH(thr) \ + (WaitForSingleObject(thr, INFINITE) ? ErrCode() : 0) #define LOCK_MUTEX0(mutex) WaitForSingleObject(mutex, INFINITE) #define UNLOCK_MUTEX(mutex) ReleaseMutex(mutex) #define mdb_mutex_consistent(mutex) 0 @@ -359,12 +370,10 @@ typedef HANDLE mdb_mutex_t, mdb_mutexref_t; #else #define MDB_PROCESS_QUERY_LIMITED_INFORMATION 0x1000 #endif -#define Z "I" #else #define THREAD_RET void * #define THREAD_CREATE(thr,start,arg) pthread_create(&thr,NULL,start,arg) #define THREAD_FINISH(thr) pthread_join(thr,NULL) -#define Z "z" /**< printf format modifier for size_t */ /** For MDB_LOCK_FORMAT: True if readers take a pid lock in the lockfile */ #define MDB_PIDLOCK 1 @@ -418,15 +427,15 @@ mdb_sem_wait(mdb_mutexref_t sem) #define mdb_mutex_consistent(mutex) 0 #else /* MDB_USE_POSIX_MUTEX: */ - /** Shared mutex/semaphore as it is stored (mdb_mutex_t), and as - * local variables keep it (mdb_mutexref_t). + /** Shared mutex/semaphore as the original is stored. * - * An mdb_mutex_t can be assigned to an mdb_mutexref_t. They can - * be the same, or an array[size 1] and a pointer. - * @{ + * Not for copies. Instead it can be assigned to an #mdb_mutexref_t. + * When mdb_mutexref_t is a pointer and mdb_mutex_t is not, then it + * is array[size 1] so it can be assigned to the pointer. */ -typedef pthread_mutex_t mdb_mutex_t[1], *mdb_mutexref_t; - /* @} */ +typedef pthread_mutex_t mdb_mutex_t[1]; + /** Reference to an #mdb_mutex_t */ +typedef pthread_mutex_t *mdb_mutexref_t; /** Lock the reader or writer mutex. * Returns 0 or a code to give #mdb_mutex_failed(), as in #LOCK_MUTEX(). */ @@ -462,28 +471,23 @@ typedef pthread_mutex_t mdb_mutex_t[1], *mdb_mutexref_t; #define GET_PAGESIZE(x) ((x) = sysconf(_SC_PAGE_SIZE)) #endif -#ifdef MDB_VL32 -#ifdef _WIN32 -#define Y "I64" -#else -#define Y "ll" -#endif -#else -#define Y Z -#endif +#define Z MDB_FMT_Z /**< printf/scanf format modifier for size_t */ +#define Yu MDB_PRIy(u) /**< printf format for #mdb_size_t */ +#define Yd MDB_PRIy(d) /**< printf format for 'signed #mdb_size_t' */ -#if defined(_WIN32) || defined(MDB_USE_POSIX_SEM) -#define MNAME_LEN 32 -#elif defined(MDB_USE_SYSV_SEM) +#ifdef MDB_USE_SYSV_SEM #define MNAME_LEN (sizeof(int)) #else #define MNAME_LEN (sizeof(pthread_mutex_t)) #endif -#ifdef MDB_USE_SYSV_SEM -#define SYSV_SEM_FLAG 1 /**< SysV sems in lockfile format */ -#else -#define SYSV_SEM_FLAG 0 +/** Initial part of #MDB_env.me_mutexname[]. + * Changes to this code must be reflected in #MDB_LOCK_FORMAT. + */ +#ifdef _WIN32 +#define MUTEXNAME_PREFIX "Global\\MDB" +#elif defined MDB_USE_POSIX_SEM +#define MUTEXNAME_PREFIX "/MDB" #endif /** @} */ @@ -627,7 +631,11 @@ static txnid_t mdb_debug_start; /** The version number for a database's datafile format. */ #define MDB_DATA_VERSION ((MDB_DEVEL) ? 999 : 1) /** The version number for a database's lockfile format. */ -#define MDB_LOCK_VERSION ((MDB_DEVEL) ? 999 : 1) +#define MDB_LOCK_VERSION ((MDB_DEVEL) ? 999 : 2) + /** Number of bits representing #MDB_LOCK_VERSION in #MDB_LOCK_FORMAT. + * The remaining bits must leave room for #MDB_lock_desc. + */ +#define MDB_LOCK_VERSION_BITS 12 /** @brief The max size of a key we can write, or 0 for computed max. * @@ -693,12 +701,27 @@ static txnid_t mdb_debug_start; /** Round \b n up to an even number. */ #define EVEN(n) (((n) + 1U) & -2) /* sign-extending -2 to match n+1U */ + /** Least significant 1-bit of \b n. n must be of an unsigned type. */ +#define LOW_BIT(n) ((n) & (-(n))) + + /** (log2(\b p2) % \b n), for p2 = power of 2 and 0 < n < 8. */ +#define LOG2_MOD(p2, n) (7 - 86 / ((p2) % ((1U<<(n))-1) + 11)) + /* Explanation: Let p2 = 2**(n*y + x), x> (CACHELINE>64), 5)) + + 6 * (sizeof(MDB_PID_T)/4 % 3) /* legacy(2) to word(4/8)? */ + + 18 * (sizeof(pthread_t)/4 % 5) /* can be struct{id, active data} */ + + 90 * (sizeof(MDB_txbody) / CACHELINE % 3) + + 270 * (MDB_LOCK_TYPE % 120) + /* The above is < 270*120 < 2**15 */ + + ((sizeof(txnid_t) == 8) << 15) /* 32bit/64bit */ + + ((sizeof(MDB_reader) > CACHELINE) << 16) + /* Not really needed - implied by MDB_LOCK_TYPE != (_WIN32 locking) */ + + (((MDB_PIDLOCK) != 0) << 17) + /* 18 bits total: Must be <= (32 - MDB_LOCK_VERSION_BITS). */ +}; /** @} */ -/** Common header for all page types. - * Overflow records occupy a number of contiguous pages with no - * headers on any page after the first. +/** Common header for all page types. The page type depends on #mp_flags. + * + * #P_BRANCH and #P_LEAF pages have unsorted '#MDB_node's at the end, with + * sorted #mp_ptrs[] entries referring to them. Exception: #P_LEAF2 pages + * omit mp_ptrs and pack sorted #MDB_DUPFIXED values after the page header. + * + * #P_OVERFLOW records occupy one or more contiguous pages where only the + * first has a page header. They hold the real data of #F_BIGDATA nodes. + * + * #P_SUBP sub-pages are small leaf "pages" with duplicate data. + * A node with flag #F_DUPDATA but not #F_SUBDATA contains a sub-page. + * (Duplicate data can also go in sub-databases, which use normal pages.) + * + * #P_META pages contain #MDB_meta, the start point of an LMDB snapshot. + * + * Each non-metapage up to #MDB_meta.%mm_last_pg is reachable exactly once + * in the snapshot: Either used by a database or listed in a freeDB record. */ typedef struct MDB_page { #define mp_pgno mp_p.p_pgno @@ -891,7 +971,7 @@ typedef struct MDB_page { pgno_t p_pgno; /**< page number */ struct MDB_page *p_next; /**< for in-memory list of freed pages */ } mp_p; - uint16_t mp_pad; + uint16_t mp_pad; /**< key size if this is a LEAF2 page */ /** @defgroup mdb_page Page Flags * @ingroup internal * Flags for the page headers. @@ -958,25 +1038,34 @@ typedef struct MDB_page { /** The number of overflow pages needed to store the given size. */ #define OVPAGES(size, psize) ((PAGEHDRSZ-1 + (size)) / (psize) + 1) - /** Link in #MDB_txn.%mt_loose_pgs list */ + /** Link in #MDB_txn.%mt_loose_pgs list. + * Kept outside the page header, which is needed when reusing the page. + */ #define NEXT_LOOSE_PAGE(p) (*(MDB_page **)((p) + 2)) /** Header for a single key/data pair within a page. * Used in pages of type #P_BRANCH and #P_LEAF without #P_LEAF2. * We guarantee 2-byte alignment for 'MDB_node's. + * + * #mn_lo and #mn_hi are used for data size on leaf nodes, and for child + * pgno on branch nodes. On 64 bit platforms, #mn_flags is also used + * for pgno. (Branch nodes have no flags). Lo and hi are in host byte + * order in case some accesses can be optimized to 32-bit word access. + * + * Leaf node flags describe node contents. #F_BIGDATA says the node's + * data part is the page number of an overflow page with actual data. + * #F_DUPDATA and #F_SUBDATA can be combined giving duplicate data in + * a sub-page/sub-database, and named databases (just #F_SUBDATA). */ typedef struct MDB_node { - /** lo and hi are used for data size on leaf nodes and for - * child pgno on branch nodes. On 64 bit platforms, flags - * is also used for pgno. (Branch nodes have no flags). - * They are in host byte order in case that lets some - * accesses be optimized into a 32-bit word access. - */ + /** part of data size or pgno + * @{ */ #if BYTE_ORDER == LITTLE_ENDIAN - unsigned short mn_lo, mn_hi; /**< part of data size or pgno */ + unsigned short mn_lo, mn_hi; #else unsigned short mn_hi, mn_lo; #endif + /** @} */ /** @defgroup mdb_node Node Flags * @ingroup internal * Flags for node headers. @@ -1041,7 +1130,7 @@ typedef struct MDB_node { #ifdef MISALIGNED_OK #define COPY_PGNO(dst,src) dst = src #else -#if SIZE_MAX > 4294967295UL +#if MDB_SIZE_MAX > 0xffffffffU #define COPY_PGNO(dst,src) do { \ unsigned short *s, *d; \ s = (unsigned short *)&(src); \ @@ -1086,9 +1175,9 @@ typedef struct MDB_db { pgno_t md_root; /**< the root page of this tree */ } MDB_db; - /** mdb_dbi_open flags */ #define MDB_VALID 0x8000 /**< DB handle is valid, for me_dbflags */ #define PERSISTENT_FLAGS (0xffff & ~(MDB_VALID)) + /** #mdb_dbi_open() flags */ #define VALID_FLAGS (MDB_REVERSEKEY|MDB_DUPSORT|MDB_INTEGERKEY|MDB_DUPFIXED|\ MDB_INTEGERDUP|MDB_REVERSEDUP|MDB_CREATE) @@ -1121,13 +1210,16 @@ typedef struct MDB_meta { #else void *mm_address; /**< address for fixed mapping */ #endif - pgno_t mm_mapsize; /**< size of mmap region */ + mdb_size_t mm_mapsize; /**< size of mmap region */ MDB_db mm_dbs[CORE_DBS]; /**< first is free space, 2nd is main db */ /** The size of pages used in this DB */ #define mm_psize mm_dbs[FREE_DBI].md_pad /** Any persistent environment flags. @ref mdb_env */ #define mm_flags mm_dbs[FREE_DBI].md_flags - pgno_t mm_last_pg; /**< last used page in file */ + /** Last used page in the datafile. + * Actually the file may be shorter if the freeDB lists the final pages. + */ + pgno_t mm_last_pg; volatile txnid_t mm_txnid; /**< txnid that committed this page */ } MDB_meta; @@ -1180,7 +1272,7 @@ struct MDB_txn { * in this transaction, linked through #NEXT_LOOSE_PAGE(page). */ MDB_page *mt_loose_pgs; - /* #Number of loose pages (#mt_loose_pgs) */ + /** Number of loose pages (#mt_loose_pgs) */ int mt_loose_count; /** The sorted list of dirty pages we temporarily wrote to disk * because the dirty list was full. page numbers in here are @@ -1203,11 +1295,12 @@ struct MDB_txn { * @ingroup internal * @{ */ -#define DB_DIRTY 0x01 /**< DB was modified or is DUPSORT data */ +#define DB_DIRTY 0x01 /**< DB was written in this txn */ #define DB_STALE 0x02 /**< Named-DB record is older than txnID */ #define DB_NEW 0x04 /**< Named-DB handle opened in this txn */ #define DB_VALID 0x08 /**< DB handle is valid, see also #MDB_VALID */ #define DB_USRVALID 0x10 /**< As #DB_VALID, but not set for #FREE_DBI */ +#define DB_DUPDATA 0x20 /**< DB is #MDB_DUPSORT data */ /** @} */ /** In write txns, array of cursors for each DB */ MDB_cursor **mt_cursors; @@ -1317,6 +1410,11 @@ struct MDB_cursor { indx_t mc_ki[CURSOR_STACK]; /**< stack of page indices */ #ifdef MDB_VL32 MDB_page *mc_ovpg; /**< a referenced overflow page */ +# define MC_OVPG(mc) ((mc)->mc_ovpg) +# define MC_SET_OVPG(mc, pg) ((mc)->mc_ovpg = (pg)) +#else +# define MC_OVPG(mc) ((MDB_page *)0) +# define MC_SET_OVPG(mc, pg) ((void)0) #endif }; @@ -1336,6 +1434,23 @@ typedef struct MDB_xcursor { unsigned char mx_dbflag; } MDB_xcursor; + /** Check if there is an inited xcursor */ +#define XCURSOR_INITED(mc) \ + ((mc)->mc_xcursor && ((mc)->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED)) + + /** Update the xcursor's sub-page pointer, if any, in \b mc. Needed + * when the node which contains the sub-page may have moved. Called + * with leaf page \b mp = mc->mc_pg[\b top]. + */ +#define XCURSOR_REFRESH(mc, top, mp) do { \ + MDB_page *xr_pg = (mp); \ + MDB_node *xr_node; \ + if (!XCURSOR_INITED(mc) || (mc)->mc_ki[top] >= NUMKEYS(xr_pg)) break; \ + xr_node = NODEPTR(xr_pg, (mc)->mc_ki[top]); \ + if ((xr_node->mn_flags & (F_DUPDATA|F_SUBDATA)) == F_DUPDATA) \ + (mc)->mc_xcursor->mx_cursor.mc_pg[0] = NODEDATA(xr_node); \ +} while (0) + /** State of FreeDB old pages, stored in the MDB_env */ typedef struct MDB_pgstate { pgno_t *mf_pghead; /**< Reclaimed freeDB pages, or NULL before use */ @@ -1346,7 +1461,7 @@ typedef struct MDB_pgstate { struct MDB_env { HANDLE me_fd; /**< The main data file */ HANDLE me_lfd; /**< The lock file */ - HANDLE me_mfd; /**< just for writing the meta pages */ + HANDLE me_mfd; /**< For writing and syncing the meta pages */ #if defined(MDB_VL32) && defined(_WIN32) HANDLE me_fmh; /**< File Mapping handle */ #endif @@ -1407,6 +1522,10 @@ struct MDB_env { #else mdb_mutex_t me_rmutex; mdb_mutex_t me_wmutex; +# if defined(_WIN32) || defined(MDB_USE_POSIX_SEM) + /** Half-initialized name of mutexes, to be completed by #MUTEXNAME() */ + char me_mutexname[sizeof(MUTEXNAME_PREFIX) + 11]; +# endif #endif #ifdef MDB_VL32 MDB_ID3L me_rpages; /**< like #mt_rpages, but global to env */ @@ -1522,19 +1641,25 @@ static int mdb_reader_check0(MDB_env *env, int rlocked, int *dead); static MDB_cmp_func mdb_cmp_memn, mdb_cmp_memnr, mdb_cmp_int, mdb_cmp_cint, mdb_cmp_long; /** @endcond */ -/** Compare two items pointing at size_t's of unknown alignment. */ +/** Compare two items pointing at '#mdb_size_t's of unknown alignment. */ #ifdef MISALIGNED_OK # define mdb_cmp_clong mdb_cmp_long #else # define mdb_cmp_clong mdb_cmp_cint #endif +/** True if we need #mdb_cmp_clong() instead of \b cmp for #MDB_INTEGERDUP */ +#define NEED_CMP_CLONG(cmp, ksize) \ + (UINT_MAX < MDB_SIZE_MAX && \ + (cmp) == mdb_cmp_int && (ksize) == sizeof(mdb_size_t)) + #ifdef _WIN32 static SECURITY_DESCRIPTOR mdb_null_sd; static SECURITY_ATTRIBUTES mdb_all_sa; static int mdb_sec_inited; -static int utf8_to_utf16(const char *src, int srcsize, wchar_t **dst, int *dstsize); +struct MDB_name; +static int utf8_to_utf16(const char *src, struct MDB_name *dst, int xtra); #endif /** Return the library version info. */ @@ -1569,6 +1694,7 @@ static char *const mdb_errstr[] = { "MDB_BAD_TXN: Transaction must abort, has a child, or is invalid", "MDB_BAD_VALSIZE: Unsupported size of key/DB name/data, or wrong DUPFIXED size", "MDB_BAD_DBI: The specified DBI handle was closed/changed unexpectedly", + "MDB_PROBLEM: Unexpected problem - txn should abort", }; char * @@ -1616,6 +1742,8 @@ mdb_strerror(int err) NULL, err, 0, ptr, MSGSIZE, (va_list *)buf+MSGSIZE); return ptr; #else + if (err < 0) + return "Invalid error code"; return strerror(err); #endif } @@ -1641,11 +1769,7 @@ mdb_assert_fail(MDB_env *env, const char *expr_txt, if (env->me_assert_func) env->me_assert_func(env, buf); fprintf(stderr, "%s\n", buf); -#ifdef NDEBUG - _exit(); -#else abort(); -#endif } #else # define mdb_assert0(env, expr, expr_txt) ((void) 0) @@ -1717,20 +1841,20 @@ mdb_page_list(MDB_page *mp) case P_LEAF|P_LEAF2: type = "LEAF2 page"; break; case P_LEAF|P_LEAF2|P_SUBP: type = "LEAF2 sub-page"; break; case P_OVERFLOW: - fprintf(stderr, "Overflow page %"Y"u pages %u%s\n", + fprintf(stderr, "Overflow page %"Yu" pages %u%s\n", pgno, mp->mp_pages, state); return; case P_META: - fprintf(stderr, "Meta-page %"Y"u txnid %"Y"u\n", + fprintf(stderr, "Meta-page %"Yu" txnid %"Yu"\n", pgno, ((MDB_meta *)METADATA(mp))->mm_txnid); return; default: - fprintf(stderr, "Bad page %"Y"u flags 0x%u\n", pgno, mp->mp_flags); + fprintf(stderr, "Bad page %"Yu" flags 0x%X\n", pgno, mp->mp_flags); return; } nkeys = NUMKEYS(mp); - fprintf(stderr, "%s %"Y"u numkeys %d%s\n", type, pgno, nkeys, state); + fprintf(stderr, "%s %"Yu" numkeys %d%s\n", type, pgno, nkeys, state); for (i=0; imn_data; nsize = NODESIZE + key.mv_size; if (IS_BRANCH(mp)) { - fprintf(stderr, "key %d: page %"Y"u, %s\n", i, NODEPGNO(node), + fprintf(stderr, "key %d: page %"Yu", %s\n", i, NODEPGNO(node), DKEY(&key)); total += nsize; } else { @@ -1780,7 +1904,7 @@ mdb_cursor_chk(MDB_cursor *mc) } if (mc->mc_ki[i] >= NUMKEYS(mc->mc_pg[i])) printf("ack!\n"); - if (mc->mc_xcursor && (mc->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED)) { + if (XCURSOR_INITED(mc)) { node = NODEPTR(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]); if (((node->mn_flags & (F_DUPDATA|F_SUBDATA)) == F_DUPDATA) && mc->mc_xcursor->mx_cursor.mc_pg[0] != NODEDATA(node)) { @@ -1841,7 +1965,7 @@ static void mdb_audit(MDB_txn *txn) } } if (freecount + count + NUM_METAS != txn->mt_next_pgno) { - fprintf(stderr, "audit: %"Y"u freecount: %"Y"u count: %"Y"u total: %"Y"u next_pgno: %"Y"u\n", + fprintf(stderr, "audit: %"Yu" freecount: %"Yu" count: %"Yu" total: %"Yu" next_pgno: %"Yu"\n", txn->mt_txnid, freecount, count+NUM_METAS, freecount+count+NUM_METAS, txn->mt_next_pgno); } @@ -1858,15 +1982,14 @@ int mdb_dcmp(MDB_txn *txn, MDB_dbi dbi, const MDB_val *a, const MDB_val *b) { MDB_cmp_func *dcmp = txn->mt_dbxs[dbi].md_dcmp; -#if UINT_MAX < SIZE_MAX || defined(MDB_VL32) - if (dcmp == mdb_cmp_int && a->mv_size == sizeof(mdb_size_t)) + if (NEED_CMP_CLONG(dcmp, a->mv_size)) dcmp = mdb_cmp_clong; -#endif return dcmp(a, b); } /** Allocate memory for a page. * Re-use old malloc'd pages first for singletons, otherwise just malloc. + * Set #MDB_TXN_ERROR on failure. */ static MDB_page * mdb_page_malloc(MDB_txn *txn, unsigned num) @@ -1978,8 +2101,14 @@ mdb_cursor_unref(MDB_cursor *mc) mc->mc_pg[0] = NULL; mc->mc_flags &= ~C_INITIALIZED; } +#define MDB_CURSOR_UNREF(mc, force) \ + (((force) || ((mc)->mc_flags & C_INITIALIZED)) \ + ? mdb_cursor_unref(mc) \ + : (void)0) + #else #define MDB_PAGE_UNREF(txn, mp) +#define MDB_CURSOR_UNREF(mc, force) ((void)0) #endif /* MDB_VL32 */ /** Loosen or free a single page. @@ -2011,7 +2140,7 @@ mdb_page_loose(MDB_cursor *mc, MDB_page *mp) if (mp != dl[x].mptr) { /* bad cursor? */ mc->mc_flags &= ~(C_INITIALIZED|C_EOF); txn->mt_flags |= MDB_TXN_ERROR; - return MDB_CORRUPTED; + return MDB_PROBLEM; } /* ok, it's ours */ loose = 1; @@ -2023,8 +2152,7 @@ mdb_page_loose(MDB_cursor *mc, MDB_page *mp) } } if (loose) { - DPRINTF(("loosen db %d page %"Y"u", DDBI(mc), - mp->mp_pgno)); + DPRINTF(("loosen db %d page %"Yu, DDBI(mc), mp->mp_pgno)); NEXT_LOOSE_PAGE(mp) = txn->mt_loose_pgs; txn->mt_loose_pgs = mp; txn->mt_loose_count++; @@ -2057,13 +2185,9 @@ mdb_pages_xkeep(MDB_cursor *mc, unsigned pflags, int all) unsigned i, j; int rc = MDB_SUCCESS, level; - /* Mark pages seen by cursors */ - if (mc->mc_flags & C_UNTRACK) - mc = NULL; /* will find mc in mt_cursors */ - for (i = txn->mt_numdbs;; mc = txn->mt_cursors[--i]) { - for (; mc; mc=mc->mc_next) { - if (!(mc->mc_flags & C_INITIALIZED)) - continue; + /* Mark pages seen by cursors: First m0, then tracked cursors */ + for (i = txn->mt_numdbs;; ) { + if (mc->mc_flags & C_INITIALIZED) { for (m3 = mc;; m3 = &mx->mx_cursor) { mp = NULL; for (j=0; jmc_snum; j++) { @@ -2082,10 +2206,13 @@ mdb_pages_xkeep(MDB_cursor *mc, unsigned pflags, int all) break; } } - if (i == 0) - break; + mc = mc->mc_next; + for (; !mc || mc == m0; mc = txn->mt_cursors[--i]) + if (i == 0) + goto mark_done; } +mark_done: if (all) { /* Mark dirty root pages */ for (i=0; imt_numdbs; i++) { @@ -2275,7 +2402,7 @@ mdb_page_dirty(MDB_txn *txn, MDB_page *mp) } /** Allocate page numbers and memory for writing. Maintain me_pglast, - * me_pghead and mt_next_pgno. + * me_pghead and mt_next_pgno. Set #MDB_TXN_ERROR on failure. * * If there are free pages available from older transactions, they * are re-used first. Otherwise allocate a new page at mt_next_pgno. @@ -2321,8 +2448,7 @@ mdb_page_alloc(MDB_cursor *mc, int num, MDB_page **mp) np = txn->mt_loose_pgs; txn->mt_loose_pgs = NEXT_LOOSE_PAGE(np); txn->mt_loose_count--; - DPRINTF(("db %d use loose page %"Y"u", DDBI(mc), - np->mp_pgno)); + DPRINTF(("db %d use loose page %"Yu, DDBI(mc), np->mp_pgno)); *mp = np; return MDB_SUCCESS; } @@ -2408,7 +2534,7 @@ mdb_page_alloc(MDB_cursor *mc, int num, MDB_page **mp) np = m2.mc_pg[m2.mc_top]; leaf = NODEPTR(np, m2.mc_ki[m2.mc_top]); if ((rc = mdb_node_read(&m2, leaf, &data)) != MDB_SUCCESS) - return rc; + goto fail; idl = (MDB_ID *) data.mv_data; i = idl[0]; @@ -2424,10 +2550,10 @@ mdb_page_alloc(MDB_cursor *mc, int num, MDB_page **mp) } env->me_pglast = last; #if (MDB_DEBUG) > 1 - DPRINTF(("IDL read txn %"Y"u root %"Y"u num %u", + DPRINTF(("IDL read txn %"Yu" root %"Yu" num %u", last, txn->mt_dbs[FREE_DBI].md_root, i)); for (j = i; j; j--) - DPRINTF(("IDL %"Y"u", idl[j])); + DPRINTF(("IDL %"Yu, idl[j])); #endif /* Merge in descending sorted order */ mdb_midl_xmerge(mop, idl); @@ -2572,6 +2698,7 @@ mdb_page_unspill(MDB_txn *txn, MDB_page *mp, MDB_page **ret) } /** Touch a page: make it dirty and re-insert into tree with updated pgno. + * Set #MDB_TXN_ERROR on failure. * @param[in] mc cursor pointing to the page to be touched * @return 0 on success, non-zero on failure. */ @@ -2597,7 +2724,7 @@ mdb_page_touch(MDB_cursor *mc) (rc = mdb_page_alloc(mc, 1, &np))) goto fail; pgno = np->mp_pgno; - DPRINTF(("touched db %d page %"Y"u -> %"Y"u", DDBI(mc), + DPRINTF(("touched db %d page %"Yu" -> %"Yu, DDBI(mc), mp->mp_pgno, pgno)); mdb_cassert(mc, mp->mp_pgno != pgno); mdb_midl_xappend(txn->mt_free_pgs, mp->mp_pgno); @@ -2621,7 +2748,7 @@ mdb_page_touch(MDB_cursor *mc) if (mp != dl[x].mptr) { /* bad cursor? */ mc->mc_flags &= ~(C_INITIALIZED|C_EOF); txn->mt_flags |= MDB_TXN_ERROR; - return MDB_CORRUPTED; + return MDB_PROBLEM; } return 0; } @@ -2660,14 +2787,8 @@ mdb_page_touch(MDB_cursor *mc) if (m2 == mc) continue; if (m2->mc_pg[mc->mc_top] == mp) { m2->mc_pg[mc->mc_top] = np; - if ((mc->mc_db->md_flags & MDB_DUPSORT) && - IS_LEAF(np) && - (m2->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED)) - { - MDB_node *leaf = NODEPTR(np, m2->mc_ki[mc->mc_top]); - if ((leaf->mn_flags & (F_DUPDATA|F_SUBDATA)) == F_DUPDATA) - m2->mc_xcursor->mx_cursor.mc_pg[0] = NODEDATA(leaf); - } + if (IS_LEAF(np)) + XCURSOR_REFRESH(m2, mc->mc_top, np); } } } @@ -2998,7 +3119,7 @@ mdb_txn_renew(MDB_txn *txn) rc = mdb_txn_renew0(txn); if (rc == MDB_SUCCESS) { - DPRINTF(("renew txn %"Y"u%c %p on mdbenv %p, root page %"Y"u", + DPRINTF(("renew txn %"Yu"%c %p on mdbenv %p, root page %"Yu, txn->mt_txnid, (txn->mt_flags & MDB_TXN_RDONLY) ? 'r' : 'w', (void *)txn, (void *)txn->mt_env, txn->mt_dbs[MAIN_DBI].md_root)); } @@ -3116,7 +3237,7 @@ mdb_txn_begin(MDB_env *env, MDB_txn *parent, unsigned int flags, MDB_txn **ret) } else { txn->mt_flags |= flags; /* could not change txn=me_txn0 earlier */ *ret = txn; - DPRINTF(("begin txn %"Y"u%c %p on mdbenv %p, root page %"Y"u", + DPRINTF(("begin txn %"Yu"%c %p on mdbenv %p, root page %"Yu, txn->mt_txnid, (flags & MDB_RDONLY) ? 'r' : 'w', (void *) txn, (void *) env, txn->mt_dbs[MAIN_DBI].md_root)); } @@ -3183,7 +3304,7 @@ mdb_txn_end(MDB_txn *txn, unsigned mode) /* Export or close DBI handles opened in this txn */ mdb_dbis_update(txn, mode & MDB_END_UPDATE); - DPRINTF(("%s txn %"Y"u%c %p on mdbenv %p, root page %"Y"u", + DPRINTF(("%s txn %"Yu"%c %p on mdbenv %p, root page %"Yu, names[mode & MDB_END_OPMASK], txn->mt_txnid, (txn->mt_flags & MDB_TXN_RDONLY) ? 'r' : 'w', (void *) txn, (void *)env, txn->mt_dbs[MAIN_DBI].md_root)); @@ -3325,10 +3446,41 @@ mdb_freelist_save(MDB_txn *txn) * we may be unable to return them to me_pghead. */ MDB_page *mp = txn->mt_loose_pgs; + MDB_ID2 *dl = txn->mt_u.dirty_list; + unsigned x; if ((rc = mdb_midl_need(&txn->mt_free_pgs, txn->mt_loose_count)) != 0) return rc; - for (; mp; mp = NEXT_LOOSE_PAGE(mp)) + for (; mp; mp = NEXT_LOOSE_PAGE(mp)) { mdb_midl_xappend(txn->mt_free_pgs, mp->mp_pgno); + /* must also remove from dirty list */ + if (txn->mt_flags & MDB_TXN_WRITEMAP) { + for (x=1; x<=dl[0].mid; x++) + if (dl[x].mid == mp->mp_pgno) + break; + mdb_tassert(txn, x <= dl[0].mid); + } else { + x = mdb_mid2l_search(dl, mp->mp_pgno); + mdb_tassert(txn, dl[x].mid == mp->mp_pgno); + } + dl[x].mptr = NULL; + mdb_dpage_free(env, mp); + } + { + /* squash freed slots out of the dirty list */ + unsigned y; + for (y=1; dl[y].mptr && y <= dl[0].mid; y++); + if (y <= dl[0].mid) { + for(x=y, y++;;) { + while (!dl[y].mptr && y <= dl[0].mid) y++; + if (y > dl[0].mid) break; + dl[x++] = dl[y++]; + } + dl[0].mid = x-1; + } else { + /* all slots freed */ + dl[0].mid = 0; + } + } txn->mt_loose_pgs = NULL; txn->mt_loose_count = 0; } @@ -3384,10 +3536,10 @@ mdb_freelist_save(MDB_txn *txn) #if (MDB_DEBUG) > 1 { unsigned int i = free_pgs[0]; - DPRINTF(("IDL write txn %"Y"u root %"Y"u num %u", + DPRINTF(("IDL write txn %"Yu" root %"Yu" num %u", txn->mt_txnid, txn->mt_dbs[FREE_DBI].md_root, i)); for (; i; i--) - DPRINTF(("IDL %"Y"u", free_pgs[i])); + DPRINTF(("IDL %"Yu, free_pgs[i])); } #endif continue; @@ -3555,7 +3707,7 @@ mdb_page_flush(MDB_txn *txn, int keep) * the write offset, to at least save the overhead of a Seek * system call. */ - DPRINTF(("committing page %"Z"u", pgno)); + DPRINTF(("committing page %"Yu, pgno)); memset(&ov, 0, sizeof(ov)); ov.Offset = pos & 0xffffffff; ov.OffsetHigh = pos >> 16 >> 16; @@ -3606,7 +3758,7 @@ mdb_page_flush(MDB_txn *txn, int keep) wpos = pos; wsize = 0; } - DPRINTF(("committing page %"Y"u", pgno)); + DPRINTF(("committing page %"Yu, pgno)); next_pos = pos + size; iov[n].iov_len = size; iov[n].iov_base = (char *)dp; @@ -3821,7 +3973,7 @@ mdb_txn_commit(MDB_txn *txn) !(txn->mt_flags & (MDB_TXN_DIRTY|MDB_TXN_SPILLS))) goto done; - DPRINTF(("committing txn %"Y"u %p on mdbenv %p, root page %"Y"u", + DPRINTF(("committing txn %"Yu" %p on mdbenv %p, root page %"Yu, txn->mt_txnid, (void*)txn, (void*)env, txn->mt_dbs[MAIN_DBI].md_root)); /* Update DB root pointers */ @@ -3929,7 +4081,7 @@ mdb_env_read_header(MDB_env *env, int prev, MDB_meta *meta) p = (MDB_page *)&pbuf; if (!F_ISSET(p->mp_flags, P_META)) { - DPRINTF(("page %"Y"u not a meta page", p->mp_pgno)); + DPRINTF(("page %"Yu" not a meta page", p->mp_pgno)); return MDB_INVALID; } @@ -4041,7 +4193,7 @@ mdb_env_write_meta(MDB_txn *txn) #endif toggle = txn->mt_txnid & 1; - DPRINTF(("writing meta page %d for root page %"Y"u", + DPRINTF(("writing meta page %d for root page %"Yu, toggle, txn->mt_dbs[MAIN_DBI].md_root)); env = txn->mt_env; @@ -4093,7 +4245,10 @@ mdb_env_write_meta(MDB_txn *txn) len = sizeof(MDB_meta) - off; off += (char *)mp - env->me_map; - /* Write to the SYNC fd */ + /* Write to the SYNC fd unless MDB_NOSYNC/MDB_NOMETASYNC. + * (me_mfd goes to the same file as me_fd, but writing to it + * also syncs to disk. Avoids a separate fdatasync() call.) + */ mfd = (flags & (MDB_NOSYNC|MDB_NOMETASYNC)) ? env->me_fd : env->me_mfd; #ifdef _WIN32 { @@ -4382,6 +4537,189 @@ mdb_fsize(HANDLE fd, mdb_size_t *size) return MDB_SUCCESS; } + +#ifdef _WIN32 +typedef wchar_t mdb_nchar_t; +# define MDB_NAME(str) L##str +# define mdb_name_cpy wcscpy +#else +/** Character type for file names: char on Unix, wchar_t on Windows */ +typedef char mdb_nchar_t; +# define MDB_NAME(str) str /**< #mdb_nchar_t[] string literal */ +# define mdb_name_cpy strcpy /**< Copy name (#mdb_nchar_t string) */ +#endif + +/** Filename - string of #mdb_nchar_t[] */ +typedef struct MDB_name { + int mn_len; /**< Length */ + int mn_alloced; /**< True if #mn_val was malloced */ + mdb_nchar_t *mn_val; /**< Contents */ +} MDB_name; + +/** Filename suffixes [datafile,lockfile][without,with MDB_NOSUBDIR] */ +static const mdb_nchar_t *const mdb_suffixes[2][2] = { + { MDB_NAME("/data.mdb"), MDB_NAME("") }, + { MDB_NAME("/lock.mdb"), MDB_NAME("-lock") } +}; + +#define MDB_SUFFLEN 9 /**< Max string length in #mdb_suffixes[] */ + +/** Set up filename + scratch area for filename suffix, for opening files. + * It should be freed with #mdb_fname_destroy(). + * On Windows, paths are converted from char *UTF-8 to wchar_t *UTF-16. + * + * @param[in] path Pathname for #mdb_env_open(). + * @param[in] envflags Whether a subdir and/or lockfile will be used. + * @param[out] fname Resulting filename, with room for a suffix if necessary. + */ +static int ESECT +mdb_fname_init(const char *path, unsigned envflags, MDB_name *fname) +{ + int no_suffix = F_ISSET(envflags, MDB_NOSUBDIR|MDB_NOLOCK); + fname->mn_alloced = 0; +#ifdef _WIN32 + return utf8_to_utf16(path, fname, no_suffix ? 0 : MDB_SUFFLEN); +#else + fname->mn_len = strlen(path); + if (no_suffix) + fname->mn_val = (char *) path; + else if ((fname->mn_val = malloc(fname->mn_len + MDB_SUFFLEN+1)) != NULL) { + fname->mn_alloced = 1; + strcpy(fname->mn_val, path); + } + else + return ENOMEM; + return MDB_SUCCESS; +#endif +} + +/** Destroy \b fname from #mdb_fname_init() */ +#define mdb_fname_destroy(fname) \ + do { if ((fname).mn_alloced) free((fname).mn_val); } while (0) + +#ifdef O_CLOEXEC /* POSIX.1-2008: Set FD_CLOEXEC atomically at open() */ +# define MDB_CLOEXEC O_CLOEXEC +#else +# define MDB_CLOEXEC 0 +#endif + +/** File type, access mode etc. for #mdb_fopen() */ +enum mdb_fopen_type { +#ifdef _WIN32 + MDB_O_RDONLY, MDB_O_RDWR, MDB_O_META, MDB_O_COPY, MDB_O_LOCKS +#else + /* A comment in mdb_fopen() explains some O_* flag choices. */ + MDB_O_RDONLY= O_RDONLY, /**< for RDONLY me_fd */ + MDB_O_RDWR = O_RDWR |O_CREAT, /**< for me_fd */ + MDB_O_META = O_WRONLY|MDB_DSYNC |MDB_CLOEXEC, /**< for me_mfd */ + MDB_O_COPY = O_WRONLY|O_CREAT|O_EXCL|MDB_CLOEXEC, /**< for #mdb_env_copy() */ + /** Bitmask for open() flags in enum #mdb_fopen_type. The other bits + * distinguish otherwise-equal MDB_O_* constants from each other. + */ + MDB_O_MASK = MDB_O_RDWR|MDB_CLOEXEC | MDB_O_RDONLY|MDB_O_META|MDB_O_COPY, + MDB_O_LOCKS = MDB_O_RDWR|MDB_CLOEXEC | ((MDB_O_MASK+1) & ~MDB_O_MASK) /**< for me_lfd */ +#endif +}; + +/** Open an LMDB file. + * @param[in] env The LMDB environment. + * @param[in,out] fname Path from from #mdb_fname_init(). A suffix is + * appended if necessary to create the filename, without changing mn_len. + * @param[in] which Determines file type, access mode, etc. + * @param[in] mode The Unix permissions for the file, if we create it. + * @param[out] res Resulting file handle. + * @return 0 on success, non-zero on failure. + */ +static int ESECT +mdb_fopen(const MDB_env *env, MDB_name *fname, + enum mdb_fopen_type which, mdb_mode_t mode, + HANDLE *res) +{ + int rc = MDB_SUCCESS; + HANDLE fd; +#ifdef _WIN32 + DWORD acc, share, disp, attrs; +#else + int flags; +#endif + + if (fname->mn_alloced) /* modifiable copy */ + mdb_name_cpy(fname->mn_val + fname->mn_len, + mdb_suffixes[which==MDB_O_LOCKS][F_ISSET(env->me_flags, MDB_NOSUBDIR)]); + + /* The directory must already exist. Usually the file need not. + * MDB_O_META requires the file because we already created it using + * MDB_O_RDWR. MDB_O_COPY must not overwrite an existing file. + * + * With MDB_O_COPY we do not want the OS to cache the writes, since + * the source data is already in the OS cache. + * + * The lockfile needs FD_CLOEXEC (close file descriptor on exec*()) + * to avoid the flock() issues noted under Caveats in lmdb.h. + * Also set it for other filehandles which the user cannot get at + * and close himself, which he may need after fork(). I.e. all but + * me_fd, which programs do use via mdb_env_get_fd(). + */ + +#ifdef _WIN32 + acc = GENERIC_READ|GENERIC_WRITE; + share = FILE_SHARE_READ|FILE_SHARE_WRITE; + disp = OPEN_ALWAYS; + attrs = FILE_ATTRIBUTE_NORMAL; + switch (which) { + case MDB_O_RDONLY: /* read-only datafile */ + acc = GENERIC_READ; + disp = OPEN_EXISTING; + break; + case MDB_O_META: /* for writing metapages */ + acc = GENERIC_WRITE; + disp = OPEN_EXISTING; + attrs = FILE_ATTRIBUTE_NORMAL|FILE_FLAG_WRITE_THROUGH; + break; + case MDB_O_COPY: /* mdb_env_copy() & co */ + acc = GENERIC_WRITE; + share = 0; + disp = CREATE_NEW; + attrs = FILE_FLAG_NO_BUFFERING|FILE_FLAG_WRITE_THROUGH; + break; + default: break; /* silence gcc -Wswitch (not all enum values handled) */ + } + fd = CreateFileW(fname->mn_val, acc, share, NULL, disp, attrs, NULL); +#else + fd = open(fname->mn_val, which & MDB_O_MASK, mode); +#endif + + if (fd == INVALID_HANDLE_VALUE) + rc = ErrCode(); +#ifndef _WIN32 + else { + if (which != MDB_O_RDONLY && which != MDB_O_RDWR) { + /* Set CLOEXEC if we could not pass it to open() */ + if (!MDB_CLOEXEC && (flags = fcntl(fd, F_GETFD)) != -1) + (void) fcntl(fd, F_SETFD, flags | FD_CLOEXEC); + } + if (which == MDB_O_COPY && env->me_psize >= env->me_os_psize) { + /* This may require buffer alignment. There is no portable + * way to ask how much, so we require OS pagesize alignment. + */ +# ifdef F_NOCACHE /* __APPLE__ */ + (void) fcntl(fd, F_NOCACHE, 1); +# elif defined O_DIRECT + /* open(...O_DIRECT...) would break on filesystems without + * O_DIRECT support (ITS#7682). Try to set it here instead. + */ + if ((flags = fcntl(fd, F_GETFL)) != -1) + (void) fcntl(fd, F_SETFL, flags | O_DIRECT); +# endif + } + } +#endif /* !_WIN32 */ + + *res = fd; + return rc; +} + + #ifdef BROKEN_FDATASYNC #include #include @@ -4407,16 +4745,16 @@ mdb_env_open2(MDB_env *env, int prev) if (!NtCreateSection) { HMODULE h = GetModuleHandle("NTDLL.DLL"); if (!h) - return MDB_PANIC; + return MDB_PROBLEM; NtClose = (NtCloseFunc *)GetProcAddress(h, "NtClose"); if (!NtClose) - return MDB_PANIC; + return MDB_PROBLEM; NtMapViewOfSection = (NtMapViewOfSectionFunc *)GetProcAddress(h, "NtMapViewOfSection"); if (!NtMapViewOfSection) - return MDB_PANIC; + return MDB_PROBLEM; NtCreateSection = (NtCreateSectionFunc *)GetProcAddress(h, "NtCreateSection"); if (!NtCreateSection) - return MDB_PANIC; + return MDB_PROBLEM; } #endif /* _WIN32 */ @@ -4554,13 +4892,13 @@ mdb_env_open2(MDB_env *env, int prev) DPRINTF(("opened database version %u, pagesize %u", meta->mm_version, env->me_psize)); - DPRINTF(("using meta page %d", (int) (meta->mm_txnid & 1))); - DPRINTF(("depth: %u", db->md_depth)); - DPRINTF(("entries: %"Y"u", db->md_entries)); - DPRINTF(("branch pages: %"Y"u", db->md_branch_pages)); - DPRINTF(("leaf pages: %"Y"u", db->md_leaf_pages)); - DPRINTF(("overflow pages: %"Y"u", db->md_overflow_pages)); - DPRINTF(("root: %"Y"u", db->md_root)); + DPRINTF(("using meta page %d", (int) (meta->mm_txnid & 1))); + DPRINTF(("depth: %u", db->md_depth)); + DPRINTF(("entries: %"Yu, db->md_entries)); + DPRINTF(("branch pages: %"Yu, db->md_branch_pages)); + DPRINTF(("leaf pages: %"Yu, db->md_leaf_pages)); + DPRINTF(("overflow pages: %"Yu, db->md_overflow_pages)); + DPRINTF(("root: %"Yu, db->md_root)); } #endif @@ -4577,7 +4915,11 @@ mdb_env_reader_dest(void *ptr) { MDB_reader *reader = ptr; - reader->mr_pid = 0; +#ifndef _WIN32 + if (reader->mr_pid == getpid()) /* catch pthread_exit() in child process */ +#endif + /* We omit the mutex, so do this atomically (i.e. skip mr_txnid) */ + reader->mr_pid = 0; } #ifdef _WIN32 @@ -4749,32 +5091,21 @@ mdb_env_excl_lock(MDB_env *env, int *excl) * Share and Enjoy! :-) */ -typedef unsigned long long mdb_hash_t; -#define MDB_HASH_INIT ((mdb_hash_t)0xcbf29ce484222325ULL) - /** perform a 64 bit Fowler/Noll/Vo FNV-1a hash on a buffer * @param[in] val value to hash - * @param[in] hval initial value for hash + * @param[in] len length of value * @return 64 bit hash - * - * NOTE: To use the recommended 64 bit FNV-1a hash, use MDB_HASH_INIT as the - * hval arg on the first call. */ static mdb_hash_t -mdb_hash_val(MDB_val *val, mdb_hash_t hval) +mdb_hash(const void *val, size_t len) { - unsigned char *s = (unsigned char *)val->mv_data; /* unsigned string */ - unsigned char *end = s + val->mv_size; + const unsigned char *s = (const unsigned char *) val, *end = s + len; + mdb_hash_t hval = 0xcbf29ce484222325ULL; /* - * FNV-1a hash each octet of the string + * FNV-1a hash each octet of the buffer */ while (s < end) { - /* xor the bottom with the current octet */ - hval ^= (mdb_hash_t)*s++; - - /* multiply by the 64 bit FNV magic prime mod 2^64 */ - hval += (hval << 1) + (hval << 4) + (hval << 5) + - (hval << 7) + (hval << 8) + (hval << 40); + hval = (hval ^ *s++) * 0x100000001b3ULL; } /* return our new hash value */ return hval; @@ -4791,47 +5122,49 @@ mdb_hash_val(MDB_val *val, mdb_hash_t hval) static const char mdb_a85[]= "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz!#$%&()*+-;<=>?@^_`{|}~"; static void ESECT -mdb_pack85(unsigned long l, char *out) +mdb_pack85(unsigned long long l, char *out) { int i; - for (i=0; i<5; i++) { + for (i=0; i<10 && l; i++) { *out++ = mdb_a85[l % 85]; l /= 85; } + *out = '\0'; } +/** Init #MDB_env.me_mutexname[] except the char which #MUTEXNAME() will set. + * Changes to this code must be reflected in #MDB_LOCK_FORMAT. + */ static void ESECT -mdb_hash_enc(MDB_val *val, char *encbuf) +mdb_env_mname_init(MDB_env *env) { - mdb_hash_t h = mdb_hash_val(val, MDB_HASH_INIT); - - mdb_pack85(h, encbuf); - mdb_pack85(h>>32, encbuf+5); - encbuf[10] = '\0'; + char *nm = env->me_mutexname; + strcpy(nm, MUTEXNAME_PREFIX); + mdb_pack85(env->me_txns->mti_mutexid, nm + sizeof(MUTEXNAME_PREFIX)); } + +/** Return env->me_mutexname after filling in ch ('r'/'w') for convenience */ +#define MUTEXNAME(env, ch) ( \ + (void) ((env)->me_mutexname[sizeof(MUTEXNAME_PREFIX)-1] = (ch)), \ + (env)->me_mutexname) + #endif /** Open and/or initialize the lock region for the environment. * @param[in] env The LMDB environment. - * @param[in] lpath The pathname of the file used for the lock region. + * @param[in] fname Filename + scratch area, from #mdb_fname_init(). * @param[in] mode The Unix permissions for the file, if we create it. * @param[in,out] excl In -1, out lock type: -1 none, 0 shared, 1 exclusive * @return 0 on success, non-zero on failure. */ static int ESECT -mdb_env_setup_locks(MDB_env *env, char *lpath, int mode, int *excl) +mdb_env_setup_locks(MDB_env *env, MDB_name *fname, int mode, int *excl) { #ifdef _WIN32 # define MDB_ERRCODE_ROFS ERROR_WRITE_PROTECT #else # define MDB_ERRCODE_ROFS EROFS -#ifdef O_CLOEXEC /* Linux: Open file and set FD_CLOEXEC atomically */ -# define MDB_CLOEXEC O_CLOEXEC -#else - int fdflags; -# define MDB_CLOEXEC 0 -#endif #endif #ifdef MDB_USE_SYSV_SEM int semid; @@ -4840,30 +5173,14 @@ mdb_env_setup_locks(MDB_env *env, char *lpath, int mode, int *excl) int rc; off_t size, rsize; -#ifdef _WIN32 - wchar_t *wlpath; - rc = utf8_to_utf16(lpath, -1, &wlpath, NULL); - if (rc) - return rc; - env->me_lfd = CreateFileW(wlpath, GENERIC_READ|GENERIC_WRITE, - FILE_SHARE_READ|FILE_SHARE_WRITE, NULL, OPEN_ALWAYS, - FILE_ATTRIBUTE_NORMAL, NULL); - free(wlpath); -#else - env->me_lfd = open(lpath, O_RDWR|O_CREAT|MDB_CLOEXEC, mode); -#endif - if (env->me_lfd == INVALID_HANDLE_VALUE) { - rc = ErrCode(); + rc = mdb_fopen(env, fname, MDB_O_LOCKS, mode, &env->me_lfd); + if (rc) { + /* Omit lockfile if read-only env on read-only filesystem */ if (rc == MDB_ERRCODE_ROFS && (env->me_flags & MDB_RDONLY)) { return MDB_SUCCESS; } - goto fail_errno; + goto fail; } -#if ! ((MDB_CLOEXEC) || defined(_WIN32)) - /* Lose record locks when exec*() */ - if ((fdflags = fcntl(env->me_lfd, F_GETFD) | FD_CLOEXEC) >= 0) - fcntl(env->me_lfd, F_SETFD, fdflags); -#endif if (!(env->me_flags & MDB_NOTLS)) { rc = pthread_key_create(&env->me_txkey, mdb_env_reader_dest); @@ -4929,8 +5246,6 @@ mdb_env_setup_locks(MDB_env *env, char *lpath, int mode, int *excl) DWORD nhigh; DWORD nlow; } idbuf; - MDB_val val; - char encbuf[11]; if (!mdb_sec_inited) { InitializeSecurityDescriptor(&mdb_null_sd, @@ -4945,14 +5260,11 @@ mdb_env_setup_locks(MDB_env *env, char *lpath, int mode, int *excl) idbuf.volume = stbuf.dwVolumeSerialNumber; idbuf.nhigh = stbuf.nFileIndexHigh; idbuf.nlow = stbuf.nFileIndexLow; - val.mv_data = &idbuf; - val.mv_size = sizeof(idbuf); - mdb_hash_enc(&val, encbuf); - sprintf(env->me_txns->mti_rmname, "Global\\MDBr%s", encbuf); - sprintf(env->me_txns->mti_wmname, "Global\\MDBw%s", encbuf); - env->me_rmutex = CreateMutexA(&mdb_all_sa, FALSE, env->me_txns->mti_rmname); + env->me_txns->mti_mutexid = mdb_hash(&idbuf, sizeof(idbuf)); + mdb_env_mname_init(env); + env->me_rmutex = CreateMutexA(&mdb_all_sa, FALSE, MUTEXNAME(env, 'r')); if (!env->me_rmutex) goto fail_errno; - env->me_wmutex = CreateMutexA(&mdb_all_sa, FALSE, env->me_txns->mti_wmname); + env->me_wmutex = CreateMutexA(&mdb_all_sa, FALSE, MUTEXNAME(env, 'w')); if (!env->me_wmutex) goto fail_errno; #elif defined(MDB_USE_POSIX_SEM) struct stat stbuf; @@ -4960,37 +5272,35 @@ mdb_env_setup_locks(MDB_env *env, char *lpath, int mode, int *excl) dev_t dev; ino_t ino; } idbuf; - MDB_val val; - char encbuf[11]; #if defined(__NetBSD__) #define MDB_SHORT_SEMNAMES 1 /* limited to 14 chars */ #endif if (fstat(env->me_lfd, &stbuf)) goto fail_errno; + memset(&idbuf, 0, sizeof(idbuf)); idbuf.dev = stbuf.st_dev; idbuf.ino = stbuf.st_ino; - val.mv_data = &idbuf; - val.mv_size = sizeof(idbuf); - mdb_hash_enc(&val, encbuf); + env->me_txns->mti_mutexid = mdb_hash(&idbuf, sizeof(idbuf)) #ifdef MDB_SHORT_SEMNAMES - encbuf[9] = '\0'; /* drop name from 15 chars to 14 chars */ + /* Max 9 base85-digits. We truncate here instead of in + * mdb_env_mname_init() to keep the latter portable. + */ + % ((mdb_hash_t)85*85*85*85*85*85*85*85*85) #endif - sprintf(env->me_txns->mti_rmname, "/MDBr%s", encbuf); - sprintf(env->me_txns->mti_wmname, "/MDBw%s", encbuf); + ; + mdb_env_mname_init(env); /* Clean up after a previous run, if needed: Try to * remove both semaphores before doing anything else. */ - sem_unlink(env->me_txns->mti_rmname); - sem_unlink(env->me_txns->mti_wmname); - env->me_rmutex = sem_open(env->me_txns->mti_rmname, - O_CREAT|O_EXCL, mode, 1); + sem_unlink(MUTEXNAME(env, 'r')); + sem_unlink(MUTEXNAME(env, 'w')); + env->me_rmutex = sem_open(MUTEXNAME(env, 'r'), O_CREAT|O_EXCL, mode, 1); if (env->me_rmutex == SEM_FAILED) goto fail_errno; - env->me_wmutex = sem_open(env->me_txns->mti_wmname, - O_CREAT|O_EXCL, mode, 1); + env->me_wmutex = sem_open(MUTEXNAME(env, 'w'), O_CREAT|O_EXCL, mode, 1); if (env->me_wmutex == SEM_FAILED) goto fail_errno; #elif defined(MDB_USE_SYSV_SEM) unsigned short vals[2] = {1, 1}; - key_t key = ftok(lpath, 'M'); + key_t key = ftok(fname->mn_val, 'M'); /* fname is lockfile path now */ if (key == -1) goto fail_errno; semid = semget(key, 2, (mode & 0777) | IPC_CREAT); @@ -5000,6 +5310,8 @@ mdb_env_setup_locks(MDB_env *env, char *lpath, int mode, int *excl) if (semctl(semid, 0, SETALL, semu) < 0) goto fail_errno; env->me_txns->mti_semid = semid; + env->me_txns->mti_rlocked = 0; + env->me_txns->mti_wlocked = 0; #else /* MDB_USE_POSIX_MUTEX: */ pthread_mutexattr_t mattr; @@ -5010,15 +5322,17 @@ mdb_env_setup_locks(MDB_env *env, char *lpath, int mode, int *excl) memset(env->me_txns->mti_rmutex, 0, sizeof(*env->me_txns->mti_rmutex)); memset(env->me_txns->mti_wmutex, 0, sizeof(*env->me_txns->mti_wmutex)); - if ((rc = pthread_mutexattr_init(&mattr)) - || (rc = pthread_mutexattr_setpshared(&mattr, PTHREAD_PROCESS_SHARED)) + if ((rc = pthread_mutexattr_init(&mattr)) != 0) + goto fail; + rc = pthread_mutexattr_setpshared(&mattr, PTHREAD_PROCESS_SHARED); #ifdef MDB_ROBUST_SUPPORTED - || (rc = pthread_mutexattr_setrobust(&mattr, PTHREAD_MUTEX_ROBUST)) + if (!rc) rc = pthread_mutexattr_setrobust(&mattr, PTHREAD_MUTEX_ROBUST); #endif - || (rc = pthread_mutex_init(env->me_txns->mti_rmutex, &mattr)) - || (rc = pthread_mutex_init(env->me_txns->mti_wmutex, &mattr))) - goto fail; + if (!rc) rc = pthread_mutex_init(env->me_txns->mti_rmutex, &mattr); + if (!rc) rc = pthread_mutex_init(env->me_txns->mti_wmutex, &mattr); pthread_mutexattr_destroy(&mattr); + if (rc) + goto fail; #endif /* _WIN32 || ... */ env->me_txns->mti_magic = MDB_MAGIC; @@ -5046,14 +5360,16 @@ mdb_env_setup_locks(MDB_env *env, char *lpath, int mode, int *excl) goto fail; } #ifdef _WIN32 - env->me_rmutex = OpenMutexA(SYNCHRONIZE, FALSE, env->me_txns->mti_rmname); + mdb_env_mname_init(env); + env->me_rmutex = OpenMutexA(SYNCHRONIZE, FALSE, MUTEXNAME(env, 'r')); if (!env->me_rmutex) goto fail_errno; - env->me_wmutex = OpenMutexA(SYNCHRONIZE, FALSE, env->me_txns->mti_wmname); + env->me_wmutex = OpenMutexA(SYNCHRONIZE, FALSE, MUTEXNAME(env, 'w')); if (!env->me_wmutex) goto fail_errno; #elif defined(MDB_USE_POSIX_SEM) - env->me_rmutex = sem_open(env->me_txns->mti_rmname, 0); + mdb_env_mname_init(env); + env->me_rmutex = sem_open(MUTEXNAME(env, 'r'), 0); if (env->me_rmutex == SEM_FAILED) goto fail_errno; - env->me_wmutex = sem_open(env->me_txns->mti_wmname, 0); + env->me_wmutex = sem_open(MUTEXNAME(env, 'w'), 0); if (env->me_wmutex == SEM_FAILED) goto fail_errno; #elif defined(MDB_USE_SYSV_SEM) semid = env->me_txns->mti_semid; @@ -5073,13 +5389,6 @@ mdb_env_setup_locks(MDB_env *env, char *lpath, int mode, int *excl) env->me_wmutex->semnum = 1; env->me_rmutex->locked = &env->me_txns->mti_rlocked; env->me_wmutex->locked = &env->me_txns->mti_wlocked; -#endif -#ifdef MDB_VL32 -#ifdef _WIN32 - env->me_rpmutex = CreateMutex(NULL, FALSE, NULL); -#else - pthread_mutex_init(&env->me_rpmutex, NULL); -#endif #endif return MDB_SUCCESS; @@ -5090,12 +5399,6 @@ mdb_env_setup_locks(MDB_env *env, char *lpath, int mode, int *excl) return rc; } - /** The name of the lock file in the DB environment */ -#define LOCKNAME "/lock.mdb" - /** The name of the data file in the DB environment */ -#define DATANAME "/data.mdb" - /** The suffix of the lock file when no subdir is used */ -#define LOCKSUFF "-lock" /** Only a subset of the @ref mdb_env flags can be changed * at runtime. Changing other flags requires closing the * environment and re-opening it with the new flags. @@ -5111,11 +5414,8 @@ mdb_env_setup_locks(MDB_env *env, char *lpath, int mode, int *excl) int ESECT mdb_env_open(MDB_env *env, const char *path, unsigned int flags, mdb_mode_t mode) { - int oflags, rc, len, excl = -1; - char *lpath, *dpath; -#ifdef _WIN32 - wchar_t *wpath; -#endif + int rc, excl = -1; + MDB_name fname; if (env->me_fd!=INVALID_HANDLE_VALUE || (flags & ~(CHANGEABLE|CHANGELESS))) return EINVAL; @@ -5130,28 +5430,27 @@ mdb_env_open(MDB_env *env, const char *path, unsigned int flags, mdb_mode_t mode return EINVAL; } #endif + flags |= env->me_flags; - len = strlen(path); - if (flags & MDB_NOSUBDIR) { - rc = len + sizeof(LOCKSUFF) + len + 1; - } else { - rc = len + sizeof(LOCKNAME) + len + sizeof(DATANAME); - } - lpath = malloc(rc); - if (!lpath) - return ENOMEM; - if (flags & MDB_NOSUBDIR) { - dpath = lpath + len + sizeof(LOCKSUFF); - sprintf(lpath, "%s" LOCKSUFF, path); - strcpy(dpath, path); - } else { - dpath = lpath + len + sizeof(LOCKNAME); - sprintf(lpath, "%s" LOCKNAME, path); - sprintf(dpath, "%s" DATANAME, path); + rc = mdb_fname_init(path, flags, &fname); + if (rc) + return rc; + +#ifdef MDB_VL32 +#ifdef _WIN32 + env->me_rpmutex = CreateMutex(NULL, FALSE, NULL); + if (!env->me_rpmutex) { + rc = ErrCode(); + goto leave; } +#else + rc = pthread_mutex_init(&env->me_rpmutex, NULL); + if (rc) + goto leave; +#endif +#endif + flags |= MDB_ENV_ACTIVE; /* tell mdb_env_close0() to clean up */ - rc = MDB_SUCCESS; - flags |= env->me_flags; if (flags & MDB_RDONLY) { /* silently ignore WRITEMAP when we're only getting read access */ flags &= ~MDB_WRITEMAP; @@ -5160,8 +5459,13 @@ mdb_env_open(MDB_env *env, const char *path, unsigned int flags, mdb_mode_t mode (env->me_dirty_list = calloc(MDB_IDL_UM_SIZE, sizeof(MDB_ID2))))) rc = ENOMEM; } + + env->me_flags = flags; + if (rc) + goto leave; + #ifdef MDB_VL32 - if (!rc) { + { env->me_rpages = malloc(MDB_ERPAGE_SIZE * sizeof(MDB_ID3)); if (!env->me_rpages) { rc = ENOMEM; @@ -5171,9 +5475,6 @@ mdb_env_open(MDB_env *env, const char *path, unsigned int flags, mdb_mode_t mode env->me_rpcheck = MDB_ERPAGE_SIZE/2; } #endif - env->me_flags = flags |= MDB_ENV_ACTIVE; - if (rc) - goto leave; env->me_path = strdup(path); env->me_dbxs = calloc(env->me_maxdbs, sizeof(MDB_dbx)); @@ -5187,73 +5488,35 @@ mdb_env_open(MDB_env *env, const char *path, unsigned int flags, mdb_mode_t mode /* For RDONLY, get lockfile after we know datafile exists */ if (!(flags & (MDB_RDONLY|MDB_NOLOCK))) { - rc = mdb_env_setup_locks(env, lpath, mode, &excl); + rc = mdb_env_setup_locks(env, &fname, mode, &excl); if (rc) goto leave; + if ((flags & MDB_PREVSNAPSHOT) && !excl) { + rc = EAGAIN; + goto leave; + } } -#ifdef _WIN32 - if (F_ISSET(flags, MDB_RDONLY)) { - oflags = GENERIC_READ; - len = OPEN_EXISTING; - } else { - oflags = GENERIC_READ|GENERIC_WRITE; - len = OPEN_ALWAYS; - } - mode = FILE_ATTRIBUTE_NORMAL; - rc = utf8_to_utf16(dpath, -1, &wpath, NULL); + rc = mdb_fopen(env, &fname, + (flags & MDB_RDONLY) ? MDB_O_RDONLY : MDB_O_RDWR, + mode, &env->me_fd); if (rc) goto leave; - env->me_fd = CreateFileW(wpath, oflags, FILE_SHARE_READ|FILE_SHARE_WRITE, - NULL, len, mode, NULL); - free(wpath); -#else - if (F_ISSET(flags, MDB_RDONLY)) - oflags = O_RDONLY; - else - oflags = O_RDWR | O_CREAT; - - env->me_fd = open(dpath, oflags, mode); -#endif - if (env->me_fd == INVALID_HANDLE_VALUE) { - rc = ErrCode(); - goto leave; - } if ((flags & (MDB_RDONLY|MDB_NOLOCK)) == MDB_RDONLY) { - rc = mdb_env_setup_locks(env, lpath, mode, &excl); + rc = mdb_env_setup_locks(env, &fname, mode, &excl); if (rc) goto leave; - if ((flags & MDB_PREVSNAPSHOT) && !excl) { - rc = EAGAIN; - goto leave; - } } if ((rc = mdb_env_open2(env, flags & MDB_PREVSNAPSHOT)) == MDB_SUCCESS) { - if (flags & (MDB_RDONLY|MDB_WRITEMAP)) { - env->me_mfd = env->me_fd; - } else { + if (!(flags & (MDB_RDONLY|MDB_WRITEMAP))) { /* Synchronous fd for meta writes. Needed even with * MDB_NOSYNC/MDB_NOMETASYNC, in case these get reset. */ -#ifdef _WIN32 - len = OPEN_EXISTING; - rc = utf8_to_utf16(dpath, -1, &wpath, NULL); + rc = mdb_fopen(env, &fname, MDB_O_META, mode, &env->me_mfd); if (rc) goto leave; - env->me_mfd = CreateFileW(wpath, oflags, - FILE_SHARE_READ|FILE_SHARE_WRITE, NULL, len, - mode | FILE_FLAG_WRITE_THROUGH, NULL); - free(wpath); -#else - oflags &= ~O_CREAT; - env->me_mfd = open(dpath, oflags | MDB_DSYNC, mode); -#endif - if (env->me_mfd == INVALID_HANDLE_VALUE) { - rc = ErrCode(); - goto leave; - } } DPRINTF(("opened dbenv %p", (void *) env)); if (excl > 0 && !(flags & MDB_PREVSNAPSHOT)) { @@ -5296,7 +5559,7 @@ mdb_env_open(MDB_env *env, const char *path, unsigned int flags, mdb_mode_t mode if (rc) { mdb_env_close0(env, excl); } - free(lpath); + mdb_fname_destroy(fname); return rc; } @@ -5324,11 +5587,13 @@ mdb_env_close0(MDB_env *env, int excl) #ifdef MDB_VL32 if (env->me_txn0 && env->me_txn0->mt_rpages) free(env->me_txn0->mt_rpages); - { unsigned int x; - for (x=1; x<=env->me_rpages[0].mid; x++) - munmap(env->me_rpages[x].mptr, env->me_rpages[x].mcnt * env->me_psize); + if (env->me_rpages) { + MDB_ID3L el = env->me_rpages; + unsigned int x; + for (x=1; x<=el[0].mid; x++) + munmap(el[x].mptr, el[x].mcnt * env->me_psize); + free(el); } - free(env->me_rpages); #endif free(env->me_txn0); mdb_midl_free(env->me_free_pgs); @@ -5353,12 +5618,12 @@ mdb_env_close0(MDB_env *env, int excl) munmap(env->me_map, env->me_mapsize); #endif } - if (env->me_mfd != env->me_fd && env->me_mfd != INVALID_HANDLE_VALUE) + if (env->me_mfd != INVALID_HANDLE_VALUE) (void) close(env->me_mfd); if (env->me_fd != INVALID_HANDLE_VALUE) (void) close(env->me_fd); if (env->me_txns) { - MDB_PID_T pid = env->me_pid; + MDB_PID_T pid = getpid(); /* Clearing readers is done in this function because * me_txkey with its destructor must be disabled first. * @@ -5388,8 +5653,8 @@ mdb_env_close0(MDB_env *env, int excl) if (excl == 0) mdb_env_excl_lock(env, &excl); if (excl > 0) { - sem_unlink(env->me_txns->mti_rmname); - sem_unlink(env->me_txns->mti_wmname); + sem_unlink(MUTEXNAME(env, 'r')); + sem_unlink(MUTEXNAME(env, 'w')); } } #elif defined(MDB_USE_SYSV_SEM) @@ -5447,7 +5712,7 @@ mdb_env_close(MDB_env *env) free(env); } -/** Compare two items pointing at aligned mdb_size_t's */ +/** Compare two items pointing at aligned #mdb_size_t's */ static int mdb_cmp_long(const MDB_val *a, const MDB_val *b) { @@ -5458,7 +5723,7 @@ mdb_cmp_long(const MDB_val *a, const MDB_val *b) /** Compare two items pointing at aligned unsigned int's. * * This is also set as #MDB_INTEGERDUP|#MDB_DUPFIXED's #MDB_dbx.%md_dcmp, - * but #mdb_cmp_clong() is called instead if the data type is mdb_size_t. + * but #mdb_cmp_clong() is called instead if the data type is #mdb_size_t. */ static int mdb_cmp_int(const MDB_val *a, const MDB_val *b) @@ -5563,7 +5828,7 @@ mdb_node_search(MDB_cursor *mc, MDB_val *key, int *exactp) nkeys = NUMKEYS(mp); - DPRINTF(("searching %u keys in %s %spage %"Y"u", + DPRINTF(("searching %u keys in %s %spage %"Yu, nkeys, IS_LEAF(mp) ? "leaf" : "branch", IS_SUBP(mp) ? "sub-" : "", mdb_dbg_pgno(mp))); @@ -5611,7 +5876,7 @@ mdb_node_search(MDB_cursor *mc, MDB_val *key, int *exactp) DPRINTF(("found leaf index %u [%s], rc = %i", i, DKEY(&nodekey), rc)); else - DPRINTF(("found branch index %u [%s -> %"Y"u], rc = %i", + DPRINTF(("found branch index %u [%s -> %"Yu"], rc = %i", i, DKEY(&nodekey), NODEPGNO(node), rc)); #endif if (rc == 0) @@ -5659,7 +5924,7 @@ static void mdb_cursor_pop(MDB_cursor *mc) { if (mc->mc_snum) { - DPRINTF(("popping page %"Y"u off db %d cursor %p", + DPRINTF(("popping page %"Yu" off db %d cursor %p", mc->mc_pg[mc->mc_top]->mp_pgno, DDBI(mc), (void *) mc)); mc->mc_snum--; @@ -5671,11 +5936,13 @@ mdb_cursor_pop(MDB_cursor *mc) } } -/** Push a page onto the top of the cursor's stack. */ +/** Push a page onto the top of the cursor's stack. + * Set #MDB_TXN_ERROR on failure. + */ static int mdb_cursor_push(MDB_cursor *mc, MDB_page *mp) { - DPRINTF(("pushing page %"Y"u on db %d cursor %p", mp->mp_pgno, + DPRINTF(("pushing page %"Yu" on db %d cursor %p", mp->mp_pgno, DDBI(mc), (void *) mc)); if (mc->mc_snum >= CURSOR_STACK) { @@ -5979,6 +6246,7 @@ mdb_rpage_get(MDB_txn *txn, pgno_t pg0, MDB_page **ret) #endif /** Find the address of the page corresponding to a given page number. + * Set #MDB_TXN_ERROR on failure. * @param[in] mc the cursor accessing the page. * @param[in] pgno the page number for the page to retrieve. * @param[out] ret address of a pointer where the page's address will be stored. @@ -5989,9 +6257,6 @@ static int mdb_page_get(MDB_cursor *mc, pgno_t pgno, MDB_page **ret, int *lvl) { MDB_txn *txn = mc->mc_txn; -#ifndef MDB_VL32 - MDB_env *env = txn->mt_env; -#endif MDB_page *p = NULL; int level; @@ -6010,14 +6275,7 @@ mdb_page_get(MDB_cursor *mc, pgno_t pgno, MDB_page **ret, int *lvl) MDB_ID pn = pgno << 1; x = mdb_midl_search(tx2->mt_spill_pgs, pn); if (x <= tx2->mt_spill_pgs[0] && tx2->mt_spill_pgs[x] == pn) { -#ifdef MDB_VL32 - int rc = mdb_rpage_get(txn, pgno, &p); - if (rc) - return rc; -#else - p = (MDB_page *)(env->me_map + env->me_psize * pgno); -#endif - goto done; + goto mapped; } } if (dl[0].mid) { @@ -6031,21 +6289,26 @@ mdb_page_get(MDB_cursor *mc, pgno_t pgno, MDB_page **ret, int *lvl) } while ((tx2 = tx2->mt_parent) != NULL); } - if (pgno < txn->mt_next_pgno) { - level = 0; + if (pgno >= txn->mt_next_pgno) { + DPRINTF(("page %"Yu" not found", pgno)); + txn->mt_flags |= MDB_TXN_ERROR; + return MDB_PAGE_NOTFOUND; + } + + level = 0; + +mapped: + { #ifdef MDB_VL32 - { - int rc = mdb_rpage_get(txn, pgno, &p); - if (rc) - return rc; + int rc = mdb_rpage_get(txn, pgno, &p); + if (rc) { + txn->mt_flags |= MDB_TXN_ERROR; + return rc; } #else + MDB_env *env = txn->mt_env; p = (MDB_page *)(env->me_map + env->me_psize * pgno); #endif - } else { - DPRINTF(("page %"Y"u not found", pgno)); - txn->mt_flags |= MDB_TXN_ERROR; - return MDB_PAGE_NOTFOUND; } done: @@ -6069,18 +6332,27 @@ mdb_page_search_root(MDB_cursor *mc, MDB_val *key, int flags) MDB_node *node; indx_t i; - DPRINTF(("branch page %"Y"u has %u keys", mp->mp_pgno, NUMKEYS(mp))); + DPRINTF(("branch page %"Yu" has %u keys", mp->mp_pgno, NUMKEYS(mp))); /* Don't assert on branch pages in the FreeDB. We can get here * while in the process of rebalancing a FreeDB branch page; we must * let that proceed. ITS#8336 */ mdb_cassert(mc, !mc->mc_dbi || NUMKEYS(mp) > 1); - DPRINTF(("found index 0 to page %"Y"u", NODEPGNO(NODEPTR(mp, 0)))); + DPRINTF(("found index 0 to page %"Yu, NODEPGNO(NODEPTR(mp, 0)))); if (flags & (MDB_PS_FIRST|MDB_PS_LAST)) { i = 0; - if (flags & MDB_PS_LAST) + if (flags & MDB_PS_LAST) { i = NUMKEYS(mp) - 1; + /* if already init'd, see if we're already in right place */ + if (mc->mc_flags & C_INITIALIZED) { + if (mc->mc_ki[mc->mc_top] == i) { + mc->mc_top = mc->mc_snum++; + mp = mc->mc_pg[mc->mc_top]; + goto ready; + } + } + } } else { int exact; node = mdb_node_search(mc, key, &exact); @@ -6106,6 +6378,7 @@ mdb_page_search_root(MDB_cursor *mc, MDB_val *key, int flags) if ((rc = mdb_cursor_push(mc, mp))) return rc; +ready: if (flags & MDB_PS_MODIFY) { if ((rc = mdb_page_touch(mc)) != 0) return rc; @@ -6120,7 +6393,7 @@ mdb_page_search_root(MDB_cursor *mc, MDB_val *key, int flags) return MDB_CORRUPTED; } - DPRINTF(("found leaf page %"Y"u for key [%s]", mp->mp_pgno, + DPRINTF(("found leaf page %"Yu" for key [%s]", mp->mp_pgno, key ? DKEY(key) : "null")); mc->mc_flags |= C_INITIALIZED; mc->mc_flags &= ~C_EOF; @@ -6235,7 +6508,7 @@ mdb_page_search(MDB_cursor *mc, MDB_val *key, int flags) mc->mc_snum = 1; mc->mc_top = 0; - DPRINTF(("db %d root page %"Y"u has flags 0x%X", + DPRINTF(("db %d root page %"Yu" has flags 0x%X", DDBI(mc), root, mc->mc_pg[0]->mp_flags)); if (flags & MDB_PS_MODIFY) { @@ -6260,7 +6533,7 @@ mdb_ovpage_free(MDB_cursor *mc, MDB_page *mp) MDB_ID pn = pg << 1; int rc; - DPRINTF(("free ov page %"Y"u (%d)", pg, ovpages)); + DPRINTF(("free ov page %"Yu" (%d)", pg, ovpages)); /* If the page is dirty or on the spill list we just acquired it, * so we should give it back to our current free list, if any. * Otherwise put it onto the list of pages we freed in this txn. @@ -6301,7 +6574,7 @@ mdb_ovpage_free(MDB_cursor *mc, MDB_page *mp) j = ++(dl[0].mid); dl[j] = ix; /* Unsorted. OK when MDB_TXN_ERROR. */ txn->mt_flags |= MDB_TXN_ERROR; - return MDB_CORRUPTED; + return MDB_PROBLEM; } } txn->mt_dirty_room++; @@ -6342,12 +6615,10 @@ mdb_node_read(MDB_cursor *mc, MDB_node *leaf, MDB_val *data) pgno_t pgno; int rc; -#ifdef MDB_VL32 - if (mc->mc_ovpg) { - MDB_PAGE_UNREF(mc->mc_txn, mc->mc_ovpg); - mc->mc_ovpg = 0; + if (MC_OVPG(mc)) { + MDB_PAGE_UNREF(mc->mc_txn, MC_OVPG(mc)); + MC_SET_OVPG(mc, NULL); } -#endif if (!F_ISSET(leaf->mn_flags, F_BIGDATA)) { data->mv_size = NODEDSZ(leaf); data->mv_data = NODEDATA(leaf); @@ -6359,13 +6630,11 @@ mdb_node_read(MDB_cursor *mc, MDB_node *leaf, MDB_val *data) data->mv_size = NODEDSZ(leaf); memcpy(&pgno, NODEDATA(leaf), sizeof(pgno)); if ((rc = mdb_page_get(mc, pgno, &omp, NULL)) != 0) { - DPRINTF(("read overflow page %"Y"u failed", pgno)); + DPRINTF(("read overflow page %"Yu" failed", pgno)); return rc; } data->mv_data = METADATA(omp); -#ifdef MDB_VL32 - mc->mc_ovpg = omp; -#endif + MC_SET_OVPG(mc, omp); return MDB_SUCCESS; } @@ -6389,14 +6658,10 @@ mdb_get(MDB_txn *txn, MDB_dbi dbi, mdb_cursor_init(&mc, txn, dbi, &mx); rc = mdb_cursor_set(&mc, key, data, MDB_SET, &exact); -#ifdef MDB_VL32 - { - /* unref all the pages - caller must copy the data - * before doing anything else - */ - mdb_cursor_unref(&mc); - } -#endif + /* unref all the pages when MDB_VL32 - caller must copy the data + * before doing anything else + */ + MDB_CURSOR_UNREF(&mc, 1); return rc; } @@ -6426,7 +6691,7 @@ mdb_cursor_sibling(MDB_cursor *mc, int move_right) op = mc->mc_pg[mc->mc_top]; #endif mdb_cursor_pop(mc); - DPRINTF(("parent page is page %"Y"u, index %u", + DPRINTF(("parent page is page %"Yu", index %u", mc->mc_pg[mc->mc_top]->mp_pgno, mc->mc_ki[mc->mc_top])); if (move_right ? (mc->mc_ki[mc->mc_top] + 1u >= NUMKEYS(mc->mc_pg[mc->mc_top])) @@ -6473,14 +6738,20 @@ mdb_cursor_next(MDB_cursor *mc, MDB_val *key, MDB_val *data, MDB_cursor_op op) MDB_node *leaf; int rc; - if (mc->mc_flags & C_EOF) { + if ((mc->mc_flags & C_DEL && op == MDB_NEXT_DUP)) return MDB_NOTFOUND; - } - mdb_cassert(mc, mc->mc_flags & C_INITIALIZED); + if (!(mc->mc_flags & C_INITIALIZED)) + return mdb_cursor_first(mc, key, data); mp = mc->mc_pg[mc->mc_top]; + if (mc->mc_flags & C_EOF) { + if (mc->mc_ki[mc->mc_top] >= NUMKEYS(mp)-1) + return MDB_NOTFOUND; + mc->mc_flags ^= C_EOF; + } + if (mc->mc_db->md_flags & MDB_DUPSORT) { leaf = NODEPTR(mp, mc->mc_ki[mc->mc_top]); if (F_ISSET(leaf->mn_flags, F_DUPDATA)) { @@ -6492,13 +6763,9 @@ mdb_cursor_next(MDB_cursor *mc, MDB_val *key, MDB_val *data, MDB_cursor_op op) return rc; } } -#ifdef MDB_VL32 else { - if (mc->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED) { - mdb_cursor_unref(&mc->mc_xcursor->mx_cursor); - } + MDB_CURSOR_UNREF(&mc->mc_xcursor->mx_cursor, 0); } -#endif } else { mc->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED|C_EOF); if (op == MDB_NEXT_DUP) @@ -6506,7 +6773,7 @@ mdb_cursor_next(MDB_cursor *mc, MDB_val *key, MDB_val *data, MDB_cursor_op op) } } - DPRINTF(("cursor_next: top page is %"Y"u in cursor %p", + DPRINTF(("cursor_next: top page is %"Yu" in cursor %p", mdb_dbg_pgno(mp), (void *) mc)); if (mc->mc_flags & C_DEL) { mc->mc_flags ^= C_DEL; @@ -6520,12 +6787,12 @@ mdb_cursor_next(MDB_cursor *mc, MDB_val *key, MDB_val *data, MDB_cursor_op op) return rc; } mp = mc->mc_pg[mc->mc_top]; - DPRINTF(("next page is %"Y"u, key index %u", mp->mp_pgno, mc->mc_ki[mc->mc_top])); + DPRINTF(("next page is %"Yu", key index %u", mp->mp_pgno, mc->mc_ki[mc->mc_top])); } else mc->mc_ki[mc->mc_top]++; skip: - DPRINTF(("==> cursor points to page %"Y"u with %u keys, key index %u", + DPRINTF(("==> cursor points to page %"Yu" with %u keys, key index %u", mdb_dbg_pgno(mp), NUMKEYS(mp), mc->mc_ki[mc->mc_top])); if (IS_LEAF2(mp)) { @@ -6563,7 +6830,12 @@ mdb_cursor_prev(MDB_cursor *mc, MDB_val *key, MDB_val *data, MDB_cursor_op op) MDB_node *leaf; int rc; - mdb_cassert(mc, mc->mc_flags & C_INITIALIZED); + if (!(mc->mc_flags & C_INITIALIZED)) { + rc = mdb_cursor_last(mc, key, data); + if (rc) + return rc; + mc->mc_ki[mc->mc_top]++; + } mp = mc->mc_pg[mc->mc_top]; @@ -6580,13 +6852,9 @@ mdb_cursor_prev(MDB_cursor *mc, MDB_val *key, MDB_val *data, MDB_cursor_op op) return rc; } } -#ifdef MDB_VL32 else { - if (mc->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED) { - mdb_cursor_unref(&mc->mc_xcursor->mx_cursor); - } + MDB_CURSOR_UNREF(&mc->mc_xcursor->mx_cursor, 0); } -#endif } else { mc->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED|C_EOF); if (op == MDB_PREV_DUP) @@ -6594,7 +6862,7 @@ mdb_cursor_prev(MDB_cursor *mc, MDB_val *key, MDB_val *data, MDB_cursor_op op) } } - DPRINTF(("cursor_prev: top page is %"Y"u in cursor %p", + DPRINTF(("cursor_prev: top page is %"Yu" in cursor %p", mdb_dbg_pgno(mp), (void *) mc)); mc->mc_flags &= ~(C_EOF|C_DEL); @@ -6606,13 +6874,11 @@ mdb_cursor_prev(MDB_cursor *mc, MDB_val *key, MDB_val *data, MDB_cursor_op op) } mp = mc->mc_pg[mc->mc_top]; mc->mc_ki[mc->mc_top] = NUMKEYS(mp) - 1; - DPRINTF(("prev page is %"Y"u, key index %u", mp->mp_pgno, mc->mc_ki[mc->mc_top])); + DPRINTF(("prev page is %"Yu", key index %u", mp->mp_pgno, mc->mc_ki[mc->mc_top])); } else mc->mc_ki[mc->mc_top]--; - mc->mc_flags &= ~C_EOF; - - DPRINTF(("==> cursor points to page %"Y"u with %u keys, key index %u", + DPRINTF(("==> cursor points to page %"Yu" with %u keys, key index %u", mdb_dbg_pgno(mp), NUMKEYS(mp), mc->mc_ki[mc->mc_top])); if (IS_LEAF2(mp)) { @@ -6656,11 +6922,7 @@ mdb_cursor_set(MDB_cursor *mc, MDB_val *key, MDB_val *data, return MDB_BAD_VALSIZE; if (mc->mc_xcursor) { -#ifdef MDB_VL32 - if (mc->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED) { - mdb_cursor_unref(&mc->mc_xcursor->mx_cursor); - } -#endif + MDB_CURSOR_UNREF(&mc->mc_xcursor->mx_cursor, 0); mc->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED|C_EOF); } @@ -6728,6 +6990,7 @@ mdb_cursor_set(MDB_cursor *mc, MDB_val *key, MDB_val *data, } } rc = 0; + mc->mc_flags &= ~C_EOF; goto set2; } } @@ -6819,10 +7082,8 @@ mdb_cursor_set(MDB_cursor *mc, MDB_val *key, MDB_val *data, if ((rc = mdb_node_read(mc, leaf, &olddata)) != MDB_SUCCESS) return rc; dcmp = mc->mc_dbx->md_dcmp; -#if UINT_MAX < SIZE_MAX || defined(MDB_VL32) - if (dcmp == mdb_cmp_int && olddata.mv_size == sizeof(mdb_size_t)) + if (NEED_CMP_CLONG(dcmp, olddata.mv_size)) dcmp = mdb_cmp_clong; -#endif rc = dcmp(data, &olddata); if (rc) { if (op == MDB_GET_BOTH || rc > 0) @@ -6855,11 +7116,7 @@ mdb_cursor_first(MDB_cursor *mc, MDB_val *key, MDB_val *data) MDB_node *leaf; if (mc->mc_xcursor) { -#ifdef MDB_VL32 - if (mc->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED) { - mdb_cursor_unref(&mc->mc_xcursor->mx_cursor); - } -#endif + MDB_CURSOR_UNREF(&mc->mc_xcursor->mx_cursor, 0); mc->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED|C_EOF); } @@ -6905,24 +7162,17 @@ mdb_cursor_last(MDB_cursor *mc, MDB_val *key, MDB_val *data) MDB_node *leaf; if (mc->mc_xcursor) { -#ifdef MDB_VL32 - if (mc->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED) { - mdb_cursor_unref(&mc->mc_xcursor->mx_cursor); - } -#endif + MDB_CURSOR_UNREF(&mc->mc_xcursor->mx_cursor, 0); mc->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED|C_EOF); } - if (!(mc->mc_flags & C_EOF)) { - - if (!(mc->mc_flags & C_INITIALIZED) || mc->mc_top) { - rc = mdb_page_search(mc, NULL, MDB_PS_LAST); - if (rc != MDB_SUCCESS) - return rc; - } - mdb_cassert(mc, IS_LEAF(mc->mc_pg[mc->mc_top])); - + if (!(mc->mc_flags & C_INITIALIZED) || mc->mc_top) { + rc = mdb_page_search(mc, NULL, MDB_PS_LAST); + if (rc != MDB_SUCCESS) + return rc; } + mdb_cassert(mc, IS_LEAF(mc->mc_pg[mc->mc_top])); + mc->mc_ki[mc->mc_top] = NUMKEYS(mc->mc_pg[mc->mc_top]) - 1; mc->mc_flags |= C_INITIALIZED|C_EOF; leaf = NODEPTR(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]); @@ -7036,10 +7286,7 @@ mdb_cursor_get(MDB_cursor *mc, MDB_val *key, MDB_val *data, rc = MDB_INCOMPATIBLE; break; } - if (!(mc->mc_flags & C_INITIALIZED)) - rc = mdb_cursor_first(mc, key, data); - else - rc = mdb_cursor_next(mc, key, data, MDB_NEXT_DUP); + rc = mdb_cursor_next(mc, key, data, MDB_NEXT_DUP); if (rc == MDB_SUCCESS) { if (mc->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED) { MDB_cursor *mx; @@ -7081,21 +7328,11 @@ mdb_cursor_get(MDB_cursor *mc, MDB_val *key, MDB_val *data, case MDB_NEXT: case MDB_NEXT_DUP: case MDB_NEXT_NODUP: - if (!(mc->mc_flags & C_INITIALIZED)) - rc = mdb_cursor_first(mc, key, data); - else - rc = mdb_cursor_next(mc, key, data, op); + rc = mdb_cursor_next(mc, key, data, op); break; case MDB_PREV: case MDB_PREV_DUP: case MDB_PREV_NODUP: - if (!(mc->mc_flags & C_INITIALIZED)) { - rc = mdb_cursor_last(mc, key, data); - if (rc) - break; - mc->mc_flags |= C_INITIALIZED; - mc->mc_ki[mc->mc_top]++; - } rc = mdb_cursor_prev(mc, key, data, op); break; case MDB_FIRST: @@ -7112,6 +7349,11 @@ mdb_cursor_get(MDB_cursor *mc, MDB_val *key, MDB_val *data, rc = MDB_INCOMPATIBLE; break; } + if (mc->mc_ki[mc->mc_top] >= NUMKEYS(mc->mc_pg[mc->mc_top])) { + mc->mc_ki[mc->mc_top] = NUMKEYS(mc->mc_pg[mc->mc_top]); + rc = MDB_NOTFOUND; + break; + } { MDB_node *leaf = NODEPTR(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]); if (!F_ISSET(leaf->mn_flags, F_DUPDATA)) { @@ -7153,7 +7395,8 @@ mdb_cursor_touch(MDB_cursor *mc) { int rc = MDB_SUCCESS; - if (mc->mc_dbi >= CORE_DBS && !(*mc->mc_dbflag & DB_DIRTY)) { + if (mc->mc_dbi >= CORE_DBS && !(*mc->mc_dbflag & (DB_DIRTY|DB_DUPDATA))) { + /* Touch DB record of named DB */ MDB_cursor mc2; MDB_xcursor mcx; if (TXN_DBI_CHANGED(mc->mc_txn, mc->mc_dbi)) @@ -7378,10 +7621,8 @@ mdb_cursor_put(MDB_cursor *mc, MDB_val *key, MDB_val *data, if (flags == MDB_CURRENT) goto current; dcmp = mc->mc_dbx->md_dcmp; -#if UINT_MAX < SIZE_MAX || defined(MDB_VL32) - if (dcmp == mdb_cmp_int && olddata.mv_size == sizeof(mdb_size_t)) + if (NEED_CMP_CLONG(dcmp, olddata.mv_size)) dcmp = mdb_cmp_clong; -#endif /* does data match? */ if (!dcmp(data, &olddata)) { if (flags & (MDB_NODUPDATA|MDB_APPENDDUP)) @@ -7477,8 +7718,9 @@ mdb_cursor_put(MDB_cursor *mc, MDB_val *key, MDB_val *data, } else { memcpy((char *)mp + mp->mp_upper + PAGEBASE, (char *)fp + fp->mp_upper + PAGEBASE, olddata.mv_size - fp->mp_upper - PAGEBASE); + memcpy((char *)(&mp->mp_ptrs), (char *)(&fp->mp_ptrs), NUMKEYS(fp) * sizeof(mp->mp_ptrs[0])); for (i=0; imp_ptrs[i] = fp->mp_ptrs[i] + offset; + mp->mp_ptrs[i] += offset; } } @@ -7532,8 +7774,13 @@ mdb_cursor_put(MDB_cursor *mc, MDB_val *key, MDB_val *data, /* Note - this page is already counted in parent's dirty_room */ rc2 = mdb_mid2l_insert(mc->mc_txn->mt_u.dirty_list, &id2); mdb_cassert(mc, rc2 == 0); + /* Currently we make the page look as with put() in the + * parent txn, in case the user peeks at MDB_RESERVEd + * or unused parts. Some users treat ovpages specially. + */ if (!(flags & MDB_RESERVE)) { - /* Copy end of page, adjusting alignment so + /* Skip the part where LMDB will put *data. + * Copy end of page, adjusting alignment so * compiler may copy words instead of bytes. */ off = (PAGEHDRSZ + data->mv_size) & -sizeof(size_t); @@ -7602,11 +7849,7 @@ mdb_cursor_put(MDB_cursor *mc, MDB_val *key, MDB_val *data, if (m3->mc_ki[i] >= mc->mc_ki[i] && insert_key) { m3->mc_ki[i]++; } - if (m3->mc_xcursor && (m3->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED)) { - MDB_node *n2 = NODEPTR(mp, m3->mc_ki[i]); - if ((n2->mn_flags & (F_SUBDATA|F_DUPDATA)) == F_DUPDATA) - m3->mc_xcursor->mx_cursor.mc_pg[0] = NODEDATA(n2); - } + XCURSOR_REFRESH(m3, i, mp); } } } @@ -7648,7 +7891,6 @@ mdb_cursor_put(MDB_cursor *mc, MDB_val *key, MDB_val *data, MDB_xcursor *mx = mc->mc_xcursor; unsigned i = mc->mc_top; MDB_page *mp = mc->mc_pg[i]; - int nkeys = NUMKEYS(mp); for (m2 = mc->mc_txn->mt_cursors[mc->mc_dbi]; m2; m2=m2->mc_next) { if (m2 == mc || m2->mc_snum < mc->mc_snum) continue; @@ -7656,10 +7898,8 @@ mdb_cursor_put(MDB_cursor *mc, MDB_val *key, MDB_val *data, if (m2->mc_pg[i] == mp) { if (m2->mc_ki[i] == mc->mc_ki[i]) { mdb_xcursor_init2(m2, mx, new_dupdata); - } else if (!insert_key && m2->mc_ki[i] < nkeys) { - MDB_node *n2 = NODEPTR(mp, m2->mc_ki[i]); - if ((n2->mn_flags & (F_SUBDATA|F_DUPDATA)) == F_DUPDATA) - m2->mc_xcursor->mx_cursor.mc_pg[0] = NODEDATA(n2); + } else if (!insert_key) { + XCURSOR_REFRESH(m2, i, mp); } } } @@ -7701,7 +7941,7 @@ mdb_cursor_put(MDB_cursor *mc, MDB_val *key, MDB_val *data, return rc; bad_sub: if (rc == MDB_KEYEXIST) /* should not happen, we deleted that item */ - rc = MDB_CORRUPTED; + rc = MDB_PROBLEM; } mc->mc_txn->mt_flags |= MDB_TXN_ERROR; return rc; @@ -7764,13 +8004,7 @@ mdb_cursor_del(MDB_cursor *mc, unsigned int flags) if (m2 == mc || m2->mc_snum < mc->mc_snum) continue; if (!(m2->mc_flags & C_INITIALIZED)) continue; if (m2->mc_pg[mc->mc_top] == mp) { - if (m2->mc_ki[mc->mc_top] == mc->mc_ki[mc->mc_top]) { - m2->mc_xcursor->mx_cursor.mc_pg[0] = NODEDATA(leaf); - } else { - MDB_node *n2 = NODEPTR(mp, m2->mc_ki[mc->mc_top]); - if (!(n2->mn_flags & F_SUBDATA)) - m2->mc_xcursor->mx_cursor.mc_pg[0] = NODEDATA(n2); - } + XCURSOR_REFRESH(m2, mc->mc_top, mp); } } } @@ -7815,6 +8049,7 @@ mdb_cursor_del(MDB_cursor *mc, unsigned int flags) } /** Allocate and initialize new pages for a database. + * Set #MDB_TXN_ERROR on failure. * @param[in] mc a cursor on the database being added to. * @param[in] flags flags defining what type of page is being allocated. * @param[in] num the number of pages to allocate. This is usually 1, @@ -7830,7 +8065,7 @@ mdb_page_new(MDB_cursor *mc, uint32_t flags, int num, MDB_page **mp) if ((rc = mdb_page_alloc(mc, num, &np))) return rc; - DPRINTF(("allocated new mpage %"Y"u, page size %u", + DPRINTF(("allocated new mpage %"Yu", page size %u", np->mp_pgno, mc->mc_txn->mt_env->me_psize)); np->mp_flags = flags | P_DIRTY; np->mp_lower = (PAGEHDRSZ-PAGEBASE); @@ -7900,6 +8135,7 @@ mdb_branch_size(MDB_env *env, MDB_val *key) } /** Add a node to the page pointed to by the cursor. + * Set #MDB_TXN_ERROR on failure. * @param[in] mc The cursor for this operation. * @param[in] indx The index on the page where the new node should be added. * @param[in] key The key for the new node. @@ -7930,7 +8166,7 @@ mdb_node_add(MDB_cursor *mc, indx_t indx, mdb_cassert(mc, mp->mp_upper >= mp->mp_lower); - DPRINTF(("add to %s %spage %"Y"u index %i, data size %"Z"u key size %"Z"u [%s]", + DPRINTF(("add to %s %spage %"Yu" index %i, data size %"Z"u key size %"Z"u [%s]", IS_LEAF(mp) ? "leaf" : "branch", IS_SUBP(mp) ? "sub-" : "", mdb_dbg_pgno(mp), indx, data ? data->mv_size : 0, @@ -7971,7 +8207,7 @@ mdb_node_add(MDB_cursor *mc, indx_t indx, goto full; if ((rc = mdb_page_new(mc, P_OVERFLOW, ovpages, &ofp))) return rc; - DPRINTF(("allocated overflow page %"Y"u", ofp->mp_pgno)); + DPRINTF(("allocated overflow page %"Yu, ofp->mp_pgno)); flags |= F_BIGDATA; goto update; } else { @@ -8028,7 +8264,7 @@ mdb_node_add(MDB_cursor *mc, indx_t indx, return MDB_SUCCESS; full: - DPRINTF(("not enough room in page %"Y"u, got %u ptrs", + DPRINTF(("not enough room in page %"Yu", got %u ptrs", mdb_dbg_pgno(mp), NUMKEYS(mp))); DPRINTF(("upper-lower = %u - %u = %"Z"d", mp->mp_upper,mp->mp_lower,room)); DPRINTF(("node size = %"Z"u", node_size)); @@ -8051,7 +8287,7 @@ mdb_node_del(MDB_cursor *mc, int ksize) MDB_node *node; char *base; - DPRINTF(("delete node %u on %s page %"Y"u", indx, + DPRINTF(("delete node %u on %s page %"Yu, indx, IS_LEAF(mp) ? "leaf" : "branch", mdb_dbg_pgno(mp))); numkeys = NUMKEYS(mp); mdb_cassert(mc, indx < numkeys); @@ -8160,9 +8396,7 @@ mdb_xcursor_init0(MDB_cursor *mc) mx->mx_cursor.mc_dbflag = &mx->mx_dbflag; mx->mx_cursor.mc_snum = 0; mx->mx_cursor.mc_top = 0; -#ifdef MDB_VL32 - mx->mx_cursor.mc_ovpg = 0; -#endif + MC_SET_OVPG(&mx->mx_cursor, NULL); mx->mx_cursor.mc_flags = C_SUB | (mc->mc_flags & (C_ORIG_RDONLY|C_WRITEMAP)); mx->mx_dbx.md_name.mv_size = 0; mx->mx_dbx.md_name.mv_data = NULL; @@ -8210,13 +8444,11 @@ mdb_xcursor_init1(MDB_cursor *mc, MDB_node *node) mx->mx_db.md_flags |= MDB_INTEGERKEY; } } - DPRINTF(("Sub-db -%u root page %"Y"u", mx->mx_cursor.mc_dbi, + DPRINTF(("Sub-db -%u root page %"Yu, mx->mx_cursor.mc_dbi, mx->mx_db.md_root)); - mx->mx_dbflag = DB_VALID|DB_USRVALID|DB_DIRTY; /* DB_DIRTY guides mdb_cursor_touch */ -#if UINT_MAX < SIZE_MAX || defined(MDB_VL32) - if (mx->mx_dbx.md_cmp == mdb_cmp_int && mx->mx_db.md_pad == sizeof(mdb_size_t)) + mx->mx_dbflag = DB_VALID|DB_USRVALID|DB_DUPDATA; + if (NEED_CMP_CLONG(mx->mx_dbx.md_cmp, mx->mx_db.md_pad)) mx->mx_dbx.md_cmp = mdb_cmp_clong; -#endif } @@ -8238,8 +8470,8 @@ mdb_xcursor_init2(MDB_cursor *mc, MDB_xcursor *src_mx, int new_dupdata) mx->mx_cursor.mc_top = 0; mx->mx_cursor.mc_flags |= C_INITIALIZED; mx->mx_cursor.mc_ki[0] = 0; - mx->mx_dbflag = DB_VALID|DB_USRVALID|DB_DIRTY; /* DB_DIRTY guides mdb_cursor_touch */ -#if UINT_MAX < SIZE_MAX + mx->mx_dbflag = DB_VALID|DB_USRVALID|DB_DUPDATA; +#if UINT_MAX < MDB_SIZE_MAX /* matches mdb_xcursor_init1:NEED_CMP_CLONG() */ mx->mx_dbx.md_cmp = src_mx->mx_dbx.md_cmp; #endif } else if (!(mx->mx_cursor.mc_flags & C_INITIALIZED)) { @@ -8247,7 +8479,7 @@ mdb_xcursor_init2(MDB_cursor *mc, MDB_xcursor *src_mx, int new_dupdata) } mx->mx_db = src_mx->mx_db; mx->mx_cursor.mc_pg[0] = src_mx->mx_cursor.mc_pg[0]; - DPRINTF(("Sub-db -%u root page %"Y"u", mx->mx_cursor.mc_dbi, + DPRINTF(("Sub-db -%u root page %"Yu, mx->mx_cursor.mc_dbi, mx->mx_db.md_root)); } @@ -8266,9 +8498,7 @@ mdb_cursor_init(MDB_cursor *mc, MDB_txn *txn, MDB_dbi dbi, MDB_xcursor *mx) mc->mc_top = 0; mc->mc_pg[0] = 0; mc->mc_ki[0] = 0; -#ifdef MDB_VL32 - mc->mc_ovpg = 0; -#endif + MC_SET_OVPG(mc, NULL); mc->mc_flags = txn->mt_flags & (C_ORIG_RDONLY|C_WRITEMAP); if (txn->mt_dbs[dbi].md_flags & MDB_DUPSORT) { mdb_tassert(txn, mx != NULL); @@ -8350,9 +8580,15 @@ mdb_cursor_count(MDB_cursor *mc, mdb_size_t *countp) if (!(mc->mc_flags & C_INITIALIZED)) return EINVAL; - if (!mc->mc_snum || (mc->mc_flags & C_EOF)) + if (!mc->mc_snum) return MDB_NOTFOUND; + if (mc->mc_flags & C_EOF) { + if (mc->mc_ki[mc->mc_top] >= NUMKEYS(mc->mc_pg[mc->mc_top])) + return MDB_NOTFOUND; + mc->mc_flags ^= C_EOF; + } + leaf = NODEPTR(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]); if (!F_ISSET(leaf->mn_flags, F_DUPDATA)) { *countp = 1; @@ -8368,13 +8604,14 @@ mdb_cursor_count(MDB_cursor *mc, mdb_size_t *countp) void mdb_cursor_close(MDB_cursor *mc) { -#ifdef MDB_VL32 if (mc) { - mdb_cursor_unref(mc); + MDB_CURSOR_UNREF(mc, 0); } -#endif if (mc && !mc->mc_backup) { - /* remove from txn, if tracked */ + /* Remove from txn, if tracked. + * A read-only txn (!C_UNTRACK) may have been freed already, + * so do not peek inside it. Only write txns track cursors. + */ if ((mc->mc_flags & C_UNTRACK) && mc->mc_txn->mt_cursors) { MDB_cursor **prev = &mc->mc_txn->mt_cursors[mc->mc_dbi]; while (*prev && *prev != mc) prev = &(*prev)->mc_next; @@ -8399,6 +8636,7 @@ mdb_cursor_dbi(MDB_cursor *mc) } /** Replace the key for a branch node with a new key. + * Set #MDB_TXN_ERROR on failure. * @param[in] mc Cursor pointing to the node to operate on. * @param[in] key The new key to use. * @return 0 on success, non-zero on failure. @@ -8424,7 +8662,7 @@ mdb_update_key(MDB_cursor *mc, MDB_val *key) char kbuf2[DKBUF_MAXKEYSIZE*2+1]; k2.mv_data = NODEKEY(node); k2.mv_size = node->mn_ksize; - DPRINTF(("update key %u (ofs %u) [%s] to [%s] on page %"Y"u", + DPRINTF(("update key %u (ofs %u) [%s] to [%s] on page %"Yu, indx, ptr, mdb_dkey(&k2, kbuf2), DKEY(key), @@ -8572,7 +8810,7 @@ mdb_node_move(MDB_cursor *csrc, MDB_cursor *cdst, int fromleft) return rc; } - DPRINTF(("moving %s node %u [%s] on page %"Y"u to node %u on page %"Y"u", + DPRINTF(("moving %s node %u [%s] on page %"Yu" to node %u on page %"Yu, IS_LEAF(csrc->mc_pg[csrc->mc_top]) ? "leaf" : "branch", csrc->mc_ki[csrc->mc_top], DKEY(&key), @@ -8618,12 +8856,8 @@ mdb_node_move(MDB_cursor *csrc, MDB_cursor *cdst, int fromleft) m3->mc_ki[csrc->mc_top] = cdst->mc_ki[cdst->mc_top]; m3->mc_ki[csrc->mc_top-1]++; } - if (m3->mc_xcursor && (m3->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED) && - IS_LEAF(mps)) { - MDB_node *node = NODEPTR(m3->mc_pg[csrc->mc_top], m3->mc_ki[csrc->mc_top]); - if ((node->mn_flags & (F_DUPDATA|F_SUBDATA)) == F_DUPDATA) - m3->mc_xcursor->mx_cursor.mc_pg[0] = NODEDATA(node); - } + if (IS_LEAF(mps)) + XCURSOR_REFRESH(m3, csrc->mc_top, m3->mc_pg[csrc->mc_top]); } } else /* Adding on the right, bump others down */ @@ -8644,12 +8878,8 @@ mdb_node_move(MDB_cursor *csrc, MDB_cursor *cdst, int fromleft) } else { m3->mc_ki[csrc->mc_top]--; } - if (m3->mc_xcursor && (m3->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED) && - IS_LEAF(mps)) { - MDB_node *node = NODEPTR(m3->mc_pg[csrc->mc_top], m3->mc_ki[csrc->mc_top]); - if ((node->mn_flags & (F_DUPDATA|F_SUBDATA)) == F_DUPDATA) - m3->mc_xcursor->mx_cursor.mc_pg[0] = NODEDATA(node); - } + if (IS_LEAF(mps)) + XCURSOR_REFRESH(m3, csrc->mc_top, m3->mc_pg[csrc->mc_top]); } } } @@ -8666,7 +8896,7 @@ mdb_node_move(MDB_cursor *csrc, MDB_cursor *cdst, int fromleft) key.mv_size = NODEKSZ(srcnode); key.mv_data = NODEKEY(srcnode); } - DPRINTF(("update separator for source page %"Y"u to [%s]", + DPRINTF(("update separator for source page %"Yu" to [%s]", csrc->mc_pg[csrc->mc_top]->mp_pgno, DKEY(&key))); mdb_cursor_copy(csrc, &mn); mn.mc_snum--; @@ -8697,7 +8927,7 @@ mdb_node_move(MDB_cursor *csrc, MDB_cursor *cdst, int fromleft) key.mv_size = NODEKSZ(srcnode); key.mv_data = NODEKEY(srcnode); } - DPRINTF(("update separator for destination page %"Y"u to [%s]", + DPRINTF(("update separator for destination page %"Yu" to [%s]", cdst->mc_pg[cdst->mc_top]->mp_pgno, DKEY(&key))); mdb_cursor_copy(cdst, &mn); mn.mc_snum--; @@ -8743,7 +8973,7 @@ mdb_page_merge(MDB_cursor *csrc, MDB_cursor *cdst) psrc = csrc->mc_pg[csrc->mc_top]; pdst = cdst->mc_pg[cdst->mc_top]; - DPRINTF(("merging page %"Y"u into %"Y"u", psrc->mp_pgno, pdst->mp_pgno)); + DPRINTF(("merging page %"Yu" into %"Yu, psrc->mp_pgno, pdst->mp_pgno)); mdb_cassert(csrc, csrc->mc_snum > 1); /* can't merge root page */ mdb_cassert(csrc, cdst->mc_snum > 1); @@ -8800,7 +9030,7 @@ mdb_page_merge(MDB_cursor *csrc, MDB_cursor *cdst) } } - DPRINTF(("dst page %"Y"u now has %u keys (%.1f%% filled)", + DPRINTF(("dst page %"Yu" now has %u keys (%.1f%% filled)", pdst->mp_pgno, NUMKEYS(pdst), (float)PAGEFILL(cdst->mc_txn->mt_env, pdst) / 10)); @@ -8850,12 +9080,8 @@ mdb_page_merge(MDB_cursor *csrc, MDB_cursor *cdst) m3->mc_ki[top-1] > csrc->mc_ki[top-1]) { m3->mc_ki[top-1]--; } - if (m3->mc_xcursor && (m3->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED) && - IS_LEAF(psrc)) { - MDB_node *node = NODEPTR(m3->mc_pg[top], m3->mc_ki[top]); - if ((node->mn_flags & (F_DUPDATA|F_SUBDATA)) == F_DUPDATA) - m3->mc_xcursor->mx_cursor.mc_pg[0] = NODEDATA(node); - } + if (IS_LEAF(psrc)) + XCURSOR_REFRESH(m3, top, m3->mc_pg[top]); } } { @@ -8888,9 +9114,7 @@ mdb_cursor_copy(const MDB_cursor *csrc, MDB_cursor *cdst) cdst->mc_snum = csrc->mc_snum; cdst->mc_top = csrc->mc_top; cdst->mc_flags = csrc->mc_flags; -#ifdef MDB_VL32 - cdst->mc_ovpg = csrc->mc_ovpg; -#endif + MC_SET_OVPG(cdst, MC_OVPG(csrc)); for (i=0; imc_snum; i++) { cdst->mc_pg[i] = csrc->mc_pg[i]; @@ -8919,14 +9143,14 @@ mdb_rebalance(MDB_cursor *mc) minkeys = 1; thresh = FILL_THRESHOLD; } - DPRINTF(("rebalancing %s page %"Y"u (has %u keys, %.1f%% full)", + DPRINTF(("rebalancing %s page %"Yu" (has %u keys, %.1f%% full)", IS_LEAF(mc->mc_pg[mc->mc_top]) ? "leaf" : "branch", mdb_dbg_pgno(mc->mc_pg[mc->mc_top]), NUMKEYS(mc->mc_pg[mc->mc_top]), (float)PAGEFILL(mc->mc_txn->mt_env, mc->mc_pg[mc->mc_top]) / 10)); if (PAGEFILL(mc->mc_txn->mt_env, mc->mc_pg[mc->mc_top]) >= thresh && NUMKEYS(mc->mc_pg[mc->mc_top]) >= minkeys) { - DPRINTF(("no need to rebalance page %"Y"u, above fill threshold", + DPRINTF(("no need to rebalance page %"Yu", above fill threshold", mdb_dbg_pgno(mc->mc_pg[mc->mc_top]))); return MDB_SUCCESS; } @@ -9055,7 +9279,7 @@ mdb_rebalance(MDB_cursor *mc) fromleft = 1; } - DPRINTF(("found neighbor page %"Y"u (%u keys, %.1f%% full)", + DPRINTF(("found neighbor page %"Yu" (%u keys, %.1f%% full)", mn.mc_pg[mn.mc_top]->mp_pgno, NUMKEYS(mn.mc_pg[mn.mc_top]), (float)PAGEFILL(mc->mc_txn->mt_env, mn.mc_pg[mn.mc_top]) / 10)); @@ -9112,16 +9336,15 @@ mdb_cursor_del0(MDB_cursor *mc) if (m3->mc_pg[mc->mc_top] == mp) { if (m3->mc_ki[mc->mc_top] == ki) { m3->mc_flags |= C_DEL; - if (mc->mc_db->md_flags & MDB_DUPSORT) - m3->mc_xcursor->mx_cursor.mc_flags &= ~C_INITIALIZED; + if (mc->mc_db->md_flags & MDB_DUPSORT) { + /* Sub-cursor referred into dataset which is gone */ + m3->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED|C_EOF); + } + continue; } else if (m3->mc_ki[mc->mc_top] > ki) { m3->mc_ki[mc->mc_top]--; } - if (m3->mc_xcursor && (m3->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED)) { - MDB_node *node = NODEPTR(m3->mc_pg[mc->mc_top], m3->mc_ki[mc->mc_top]); - if ((node->mn_flags & (F_DUPDATA|F_SUBDATA)) == F_DUPDATA) - m3->mc_xcursor->mx_cursor.mc_pg[0] = NODEDATA(node); - } + XCURSOR_REFRESH(m3, mc->mc_top, mp); } } } @@ -9147,11 +9370,32 @@ mdb_cursor_del0(MDB_cursor *mc) continue; if (m3->mc_pg[mc->mc_top] == mp) { /* if m3 points past last node in page, find next sibling */ - if (m3->mc_ki[mc->mc_top] >= nkeys) { - rc = mdb_cursor_sibling(m3, 1); - if (rc == MDB_NOTFOUND) { - m3->mc_flags |= C_EOF; - rc = MDB_SUCCESS; + if (m3->mc_ki[mc->mc_top] >= mc->mc_ki[mc->mc_top]) { + if (m3->mc_ki[mc->mc_top] >= nkeys) { + rc = mdb_cursor_sibling(m3, 1); + if (rc == MDB_NOTFOUND) { + m3->mc_flags |= C_EOF; + rc = MDB_SUCCESS; + continue; + } + } + if (mc->mc_db->md_flags & MDB_DUPSORT) { + MDB_node *node = NODEPTR(m3->mc_pg[m3->mc_top], m3->mc_ki[m3->mc_top]); + /* If this node has dupdata, it may need to be reinited + * because its data has moved. + * If the xcursor was not initd it must be reinited. + * Else if node points to a subDB, nothing is needed. + * Else (xcursor was initd, not a subDB) needs mc_pg[0] reset. + */ + if (node->mn_flags & F_DUPDATA) { + if (m3->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED) { + if (!(node->mn_flags & F_SUBDATA)) + m3->mc_xcursor->mx_cursor.mc_pg[0] = NODEDATA(node); + } else { + mdb_xcursor_init1(m3, node); + m3->mc_xcursor->mx_cursor.mc_flags |= C_DEL; + } + } } } } @@ -9216,7 +9460,6 @@ mdb_del0(MDB_txn *txn, MDB_dbi dbi, * run out of space, triggering a split. We need this * cursor to be consistent until the end of the rebalance. */ - mc.mc_flags |= C_UNTRACK; mc.mc_next = txn->mt_cursors[dbi]; txn->mt_cursors[dbi] = &mc; rc = mdb_cursor_del(&mc, flags); @@ -9226,6 +9469,7 @@ mdb_del0(MDB_txn *txn, MDB_dbi dbi, } /** Split a page and insert a new node. + * Set #MDB_TXN_ERROR on failure. * @param[in,out] mc Cursor pointing to the page and desired insertion index. * The cursor will be updated to point to the actual page and index where * the node got inserted after the split. @@ -9257,7 +9501,7 @@ mdb_page_split(MDB_cursor *mc, MDB_val *newkey, MDB_val *newdata, pgno_t newpgno newindx = mc->mc_ki[mc->mc_top]; nkeys = NUMKEYS(mp); - DPRINTF(("-----> splitting %s page %"Y"u and adding [%s] at index %i/%i", + DPRINTF(("-----> splitting %s page %"Yu" and adding [%s] at index %i/%i", IS_LEAF(mp) ? "leaf" : "branch", mp->mp_pgno, DKEY(newkey), mc->mc_ki[mc->mc_top], nkeys)); @@ -9265,7 +9509,7 @@ mdb_page_split(MDB_cursor *mc, MDB_val *newkey, MDB_val *newdata, pgno_t newpgno if ((rc = mdb_page_new(mc, mp->mp_flags, 1, &rp))) return rc; rp->mp_pad = mp->mp_pad; - DPRINTF(("new right sibling: page %"Y"u", rp->mp_pgno)); + DPRINTF(("new right sibling: page %"Yu, rp->mp_pgno)); /* Usually when splitting the root page, the cursor * height is 1. But when called from mdb_update_key, @@ -9283,7 +9527,7 @@ mdb_page_split(MDB_cursor *mc, MDB_val *newkey, MDB_val *newdata, pgno_t newpgno mc->mc_pg[0] = pp; mc->mc_ki[0] = 0; mc->mc_db->md_root = pp->mp_pgno; - DPRINTF(("root split! new root = %"Y"u", pp->mp_pgno)); + DPRINTF(("root split! new root = %"Yu, pp->mp_pgno)); new_root = mc->mc_db->md_depth++; /* Add left (implicit) pointer. */ @@ -9300,7 +9544,7 @@ mdb_page_split(MDB_cursor *mc, MDB_val *newkey, MDB_val *newdata, pgno_t newpgno ptop = 0; } else { ptop = mc->mc_top-1; - DPRINTF(("parent branch page is %"Y"u", mc->mc_pg[ptop]->mp_pgno)); + DPRINTF(("parent branch page is %"Yu, mc->mc_pg[ptop]->mp_pgno)); } mdb_cursor_copy(mc, &mn); @@ -9399,7 +9643,7 @@ mdb_page_split(MDB_cursor *mc, MDB_val *newkey, MDB_val *newdata, pgno_t newpgno * the split so the new page is emptier than the old page. * This yields better packing during sequential inserts. */ - if (nkeys < 20 || nsize > pmax/16 || newindx >= nkeys) { + if (nkeys < 32 || nsize > pmax/16 || newindx >= nkeys) { /* Find split point */ psize = 0; if (newindx <= split_indx || newindx >= nkeys) { @@ -9475,7 +9719,7 @@ mdb_page_split(MDB_cursor *mc, MDB_val *newkey, MDB_val *newdata, pgno_t newpgno } else { /* find right page's left sibling */ mc->mc_ki[ptop] = mn.mc_ki[ptop]; - mdb_cursor_sibling(mc, 0); + rc = mdb_cursor_sibling(mc, 0); } } } else { @@ -9484,6 +9728,8 @@ mdb_page_split(MDB_cursor *mc, MDB_val *newkey, MDB_val *newdata, pgno_t newpgno mn.mc_top++; } if (rc != MDB_SUCCESS) { + if (rc == MDB_NOTFOUND) /* improper mdb_cursor_sibling() result */ + rc = MDB_PROBLEM; goto done; } if (nflags & MDB_APPEND) { @@ -9635,12 +9881,8 @@ mdb_page_split(MDB_cursor *mc, MDB_val *newkey, MDB_val *newdata, pgno_t newpgno m3->mc_ki[ptop] >= mc->mc_ki[ptop]) { m3->mc_ki[ptop]++; } - if (m3->mc_xcursor && (m3->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED) && - IS_LEAF(mp)) { - MDB_node *node = NODEPTR(m3->mc_pg[mc->mc_top], m3->mc_ki[mc->mc_top]); - if ((node->mn_flags & (F_DUPDATA|F_SUBDATA)) == F_DUPDATA) - m3->mc_xcursor->mx_cursor.mc_pg[0] = NODEDATA(node); - } + if (IS_LEAF(mp)) + XCURSOR_REFRESH(m3, mc->mc_top, m3->mc_pg[mc->mc_top]); } } DPRINTF(("mp left: %d, rp left: %d", SIZELEFT(mp), SIZELEFT(rp))); @@ -9681,23 +9923,26 @@ mdb_put(MDB_txn *txn, MDB_dbi dbi, #ifndef MDB_WBUF #define MDB_WBUF (1024*1024) #endif +#define MDB_EOF 0x10 /**< #mdb_env_copyfd1() is done reading */ - /** State needed for a compacting copy. */ + /** State needed for a double-buffering compacting copy. */ typedef struct mdb_copy { + MDB_env *mc_env; + MDB_txn *mc_txn; pthread_mutex_t mc_mutex; - pthread_cond_t mc_cond; + pthread_cond_t mc_cond; /**< Condition variable for #mc_new */ char *mc_wbuf[2]; char *mc_over[2]; - MDB_env *mc_env; - MDB_txn *mc_txn; int mc_wlen[2]; int mc_olen[2]; pgno_t mc_next_pgno; HANDLE mc_fd; - int mc_status; - volatile int mc_new; - int mc_toggle; - + int mc_toggle; /**< Buffer number in provider */ + int mc_new; /**< (0-2 buffers to write) | (#MDB_EOF at end) */ + /** Error code. Never cleared if set. Both threads can set nonzero + * to fail the copy. Not mutex-protected, LMDB expects atomic int. + */ + volatile int mc_error; } mdb_copy; /** Dedicated writer thread for compacting copy. */ @@ -9713,26 +9958,38 @@ mdb_env_copythr(void *arg) #else int len; #define DO_WRITE(rc, fd, ptr, w2, len) len = write(fd, ptr, w2); rc = (len >= 0) +#ifdef SIGPIPE + sigset_t set; + sigemptyset(&set); + sigaddset(&set, SIGPIPE); + if ((rc = pthread_sigmask(SIG_BLOCK, &set, NULL)) != 0) + my->mc_error = rc; +#endif #endif pthread_mutex_lock(&my->mc_mutex); - my->mc_new = 0; - pthread_cond_signal(&my->mc_cond); for(;;) { while (!my->mc_new) pthread_cond_wait(&my->mc_cond, &my->mc_mutex); - if (my->mc_new < 0) { - my->mc_new = 0; + if (my->mc_new == 0 + MDB_EOF) /* 0 buffers, just EOF */ break; - } - my->mc_new = 0; wsize = my->mc_wlen[toggle]; ptr = my->mc_wbuf[toggle]; again: - while (wsize > 0) { + rc = MDB_SUCCESS; + while (wsize > 0 && !my->mc_error) { DO_WRITE(rc, my->mc_fd, ptr, wsize, len); if (!rc) { rc = ErrCode(); +#if defined(SIGPIPE) && !defined(_WIN32) + if (rc == EPIPE) { + /* Collect the pending SIGPIPE, otherwise at least OS X + * gives it to the process on thread-exit (ITS#8504). + */ + int tmp; + sigwait(&set, &tmp); + } +#endif break; } else if (len > 0) { rc = MDB_SUCCESS; @@ -9745,8 +10002,7 @@ mdb_env_copythr(void *arg) } } if (rc) { - my->mc_status = rc; - break; + my->mc_error = rc; } /* If there's an overflow page tail, write it too */ if (my->mc_olen[toggle]) { @@ -9757,38 +10013,45 @@ mdb_env_copythr(void *arg) } my->mc_wlen[toggle] = 0; toggle ^= 1; + /* Return the empty buffer to provider */ + my->mc_new--; pthread_cond_signal(&my->mc_cond); } - pthread_cond_signal(&my->mc_cond); pthread_mutex_unlock(&my->mc_mutex); return (THREAD_RET)0; #undef DO_WRITE } - /** Tell the writer thread there's a buffer ready to write */ + /** Give buffer and/or #MDB_EOF to writer thread, await unused buffer. + * + * @param[in] my control structure. + * @param[in] adjust (1 to hand off 1 buffer) | (MDB_EOF when ending). + */ static int ESECT -mdb_env_cthr_toggle(mdb_copy *my, int st) +mdb_env_cthr_toggle(mdb_copy *my, int adjust) { - int toggle = my->mc_toggle ^ 1; pthread_mutex_lock(&my->mc_mutex); - if (my->mc_status) { - pthread_mutex_unlock(&my->mc_mutex); - return my->mc_status; - } - while (my->mc_new == 1) - pthread_cond_wait(&my->mc_cond, &my->mc_mutex); - my->mc_new = st; - my->mc_toggle = toggle; + my->mc_new += adjust; pthread_cond_signal(&my->mc_cond); + while (my->mc_new & 2) /* both buffers in use */ + pthread_cond_wait(&my->mc_cond, &my->mc_mutex); pthread_mutex_unlock(&my->mc_mutex); - return 0; + + my->mc_toggle ^= (adjust & 1); + /* Both threads reset mc_wlen, to be safe from threading errors */ + my->mc_wlen[my->mc_toggle] = 0; + return my->mc_error; } - /** Depth-first tree traversal for compacting copy. */ + /** Depth-first tree traversal for compacting copy. + * @param[in] my control structure. + * @param[in,out] pg database root. + * @param[in] flags includes #F_DUPDATA if it is a sorted-duplicate sub-DB. + */ static int ESECT mdb_env_cwalk(mdb_copy *my, pgno_t *pg, int flags) { - MDB_cursor mc; + MDB_cursor mc = {0}; MDB_node *ni; MDB_page *mo, *mp, *leaf; char *buf, *ptr; @@ -9800,8 +10063,8 @@ mdb_env_cwalk(mdb_copy *my, pgno_t *pg, int flags) return MDB_SUCCESS; mc.mc_snum = 1; - mc.mc_top = 0; mc.mc_txn = my->mc_txn; + mc.mc_flags = my->mc_txn->mt_flags & (C_ORIG_RDONLY|C_WRITEMAP); rc = mdb_page_get(&mc, *pg, &mc.mc_pg[0], NULL); if (rc) @@ -9847,6 +10110,7 @@ mdb_env_cwalk(mdb_copy *my, pgno_t *pg, int flags) } memcpy(&pg, NODEDATA(ni), sizeof(pg)); + memcpy(NODEDATA(ni), &my->mc_next_pgno, sizeof(pgno_t)); rc = mdb_page_get(&mc, pg, &omp, NULL); if (rc) goto done; @@ -9869,7 +10133,6 @@ mdb_env_cwalk(mdb_copy *my, pgno_t *pg, int flags) goto done; toggle = my->mc_toggle; } - memcpy(NODEDATA(ni), &mo->mp_pgno, sizeof(pgno_t)); } else if (ni->mn_flags & F_SUBDATA) { MDB_db db; @@ -9947,47 +10210,56 @@ mdb_env_copyfd1(MDB_env *env, HANDLE fd) { MDB_meta *mm; MDB_page *mp; - mdb_copy my; + mdb_copy my = {0}; MDB_txn *txn = NULL; pthread_t thr; - int rc; + pgno_t root, new_root; + int rc = MDB_SUCCESS; #ifdef _WIN32 - my.mc_mutex = CreateMutex(NULL, FALSE, NULL); - my.mc_cond = CreateEvent(NULL, FALSE, FALSE, NULL); + if (!(my.mc_mutex = CreateMutex(NULL, FALSE, NULL)) || + !(my.mc_cond = CreateEvent(NULL, FALSE, FALSE, NULL))) { + rc = ErrCode(); + goto done; + } my.mc_wbuf[0] = _aligned_malloc(MDB_WBUF*2, env->me_os_psize); - if (my.mc_wbuf[0] == NULL) - return errno; + if (my.mc_wbuf[0] == NULL) { + /* _aligned_malloc() sets errno, but we use Windows error codes */ + rc = ERROR_NOT_ENOUGH_MEMORY; + goto done; + } #else - pthread_mutex_init(&my.mc_mutex, NULL); - pthread_cond_init(&my.mc_cond, NULL); + if ((rc = pthread_mutex_init(&my.mc_mutex, NULL)) != 0) + return rc; + if ((rc = pthread_cond_init(&my.mc_cond, NULL)) != 0) + goto done2; #ifdef HAVE_MEMALIGN my.mc_wbuf[0] = memalign(env->me_os_psize, MDB_WBUF*2); - if (my.mc_wbuf[0] == NULL) - return errno; + if (my.mc_wbuf[0] == NULL) { + rc = errno; + goto done; + } #else - rc = posix_memalign((void **)&my.mc_wbuf[0], env->me_os_psize, MDB_WBUF*2); - if (rc) - return rc; + { + void *p; + if ((rc = posix_memalign(&p, env->me_os_psize, MDB_WBUF*2)) != 0) + goto done; + my.mc_wbuf[0] = p; + } #endif #endif memset(my.mc_wbuf[0], 0, MDB_WBUF*2); my.mc_wbuf[1] = my.mc_wbuf[0] + MDB_WBUF; - my.mc_wlen[0] = 0; - my.mc_wlen[1] = 0; - my.mc_olen[0] = 0; - my.mc_olen[1] = 0; my.mc_next_pgno = NUM_METAS; - my.mc_status = 0; - my.mc_new = 1; - my.mc_toggle = 0; my.mc_env = env; my.mc_fd = fd; - THREAD_CREATE(thr, mdb_env_copythr, &my); + rc = THREAD_CREATE(thr, mdb_env_copythr, &my); + if (rc) + goto done; rc = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn); if (rc) - return rc; + goto finish; mp = (MDB_page *)my.mc_wbuf[0]; memset(mp, 0, NUM_METAS * env->me_psize); @@ -10003,57 +10275,64 @@ mdb_env_copyfd1(MDB_env *env, HANDLE fd) *(MDB_meta *)METADATA(mp) = *mm; mm = (MDB_meta *)METADATA(mp); - /* Count the number of free pages, subtract from lastpg to find - * number of active pages - */ - { + /* Set metapage 1 with current main DB */ + root = new_root = txn->mt_dbs[MAIN_DBI].md_root; + if (root != P_INVALID) { + /* Count free pages + freeDB pages. Subtract from last_pg + * to find the new last_pg, which also becomes the new root. + */ MDB_ID freecount = 0; MDB_cursor mc; MDB_val key, data; mdb_cursor_init(&mc, txn, FREE_DBI, NULL); while ((rc = mdb_cursor_get(&mc, &key, &data, MDB_NEXT)) == 0) freecount += *(MDB_ID *)data.mv_data; + if (rc != MDB_NOTFOUND) + goto finish; freecount += txn->mt_dbs[FREE_DBI].md_branch_pages + txn->mt_dbs[FREE_DBI].md_leaf_pages + txn->mt_dbs[FREE_DBI].md_overflow_pages; - /* Set metapage 1 */ - mm->mm_last_pg = txn->mt_next_pgno - freecount - 1; + new_root = txn->mt_next_pgno - 1 - freecount; + mm->mm_last_pg = new_root; mm->mm_dbs[MAIN_DBI] = txn->mt_dbs[MAIN_DBI]; - if (mm->mm_last_pg > NUM_METAS-1) { - mm->mm_dbs[MAIN_DBI].md_root = mm->mm_last_pg; - mm->mm_txnid = 1; - } else { - mm->mm_dbs[MAIN_DBI].md_root = P_INVALID; - } + mm->mm_dbs[MAIN_DBI].md_root = new_root; + } else { + /* When the DB is empty, handle it specially to + * fix any breakage like page leaks from ITS#8174. + */ + mm->mm_dbs[MAIN_DBI].md_flags = txn->mt_dbs[MAIN_DBI].md_flags; + } + if (root != P_INVALID || mm->mm_dbs[MAIN_DBI].md_flags) { + mm->mm_txnid = 1; /* use metapage 1 */ } + my.mc_wlen[0] = env->me_psize * NUM_METAS; my.mc_txn = txn; - pthread_mutex_lock(&my.mc_mutex); - while(my.mc_new) - pthread_cond_wait(&my.mc_cond, &my.mc_mutex); - pthread_mutex_unlock(&my.mc_mutex); - rc = mdb_env_cwalk(&my, &txn->mt_dbs[MAIN_DBI].md_root, 0); - if (rc == MDB_SUCCESS && my.mc_wlen[my.mc_toggle]) - rc = mdb_env_cthr_toggle(&my, 1); - mdb_env_cthr_toggle(&my, -1); - pthread_mutex_lock(&my.mc_mutex); - while(my.mc_new) - pthread_cond_wait(&my.mc_cond, &my.mc_mutex); - pthread_mutex_unlock(&my.mc_mutex); - THREAD_FINISH(thr); + rc = mdb_env_cwalk(&my, &root, 0); + if (rc == MDB_SUCCESS && root != new_root) { + rc = MDB_INCOMPATIBLE; /* page leak or corrupt DB */ + } +finish: + if (rc) + my.mc_error = rc; + mdb_env_cthr_toggle(&my, 1 | MDB_EOF); + rc = THREAD_FINISH(thr); mdb_txn_abort(txn); + +done: #ifdef _WIN32 - CloseHandle(my.mc_cond); - CloseHandle(my.mc_mutex); - _aligned_free(my.mc_wbuf[0]); + if (my.mc_wbuf[0]) _aligned_free(my.mc_wbuf[0]); + if (my.mc_cond) CloseHandle(my.mc_cond); + if (my.mc_mutex) CloseHandle(my.mc_mutex); #else + free(my.mc_wbuf[0]); pthread_cond_destroy(&my.mc_cond); +done2: pthread_mutex_destroy(&my.mc_mutex); - free(my.mc_wbuf[0]); #endif - return rc; + return rc ? rc : my.mc_error; } /** Copy environment as-is. */ @@ -10174,67 +10453,20 @@ mdb_env_copyfd(MDB_env *env, HANDLE fd) int ESECT mdb_env_copy2(MDB_env *env, const char *path, unsigned int flags) { - int rc, len; - char *lpath; + int rc; + MDB_name fname; HANDLE newfd = INVALID_HANDLE_VALUE; -#ifdef _WIN32 - wchar_t *wpath; -#endif - - if (env->me_flags & MDB_NOSUBDIR) { - lpath = (char *)path; - } else { - len = strlen(path); - len += sizeof(DATANAME); - lpath = malloc(len); - if (!lpath) - return ENOMEM; - sprintf(lpath, "%s" DATANAME, path); - } - /* The destination path must exist, but the destination file must not. - * We don't want the OS to cache the writes, since the source data is - * already in the OS cache. - */ -#ifdef _WIN32 - rc = utf8_to_utf16(lpath, -1, &wpath, NULL); - if (rc) - goto leave; - newfd = CreateFileW(wpath, GENERIC_WRITE, 0, NULL, CREATE_NEW, - FILE_FLAG_NO_BUFFERING|FILE_FLAG_WRITE_THROUGH, NULL); - free(wpath); -#else - newfd = open(lpath, O_WRONLY|O_CREAT|O_EXCL, 0666); -#endif - if (newfd == INVALID_HANDLE_VALUE) { - rc = ErrCode(); - goto leave; - } - - if (env->me_psize >= env->me_os_psize) { -#ifdef O_DIRECT - /* Set O_DIRECT if the file system supports it */ - if ((rc = fcntl(newfd, F_GETFL)) != -1) - (void) fcntl(newfd, F_SETFL, rc | O_DIRECT); -#endif -#ifdef F_NOCACHE /* __APPLE__ */ - rc = fcntl(newfd, F_NOCACHE, 1); - if (rc) { - rc = ErrCode(); - goto leave; - } -#endif + rc = mdb_fname_init(path, env->me_flags | MDB_NOLOCK, &fname); + if (rc == MDB_SUCCESS) { + rc = mdb_fopen(env, &fname, MDB_O_COPY, 0666, &newfd); + mdb_fname_destroy(fname); } - - rc = mdb_env_copyfd2(env, newfd, flags); - -leave: - if (!(env->me_flags & MDB_NOSUBDIR)) - free(lpath); - if (newfd != INVALID_HANDLE_VALUE) + if (rc == MDB_SUCCESS) { + rc = mdb_env_copyfd2(env, newfd, flags); if (close(newfd) < 0 && rc == MDB_SUCCESS) rc = ErrCode(); - + } return rc; } @@ -10456,8 +10688,11 @@ int mdb_dbi_open(MDB_txn *txn, const char *name, unsigned int flags, MDB_dbi *db MDB_node *node = NODEPTR(mc.mc_pg[mc.mc_top], mc.mc_ki[mc.mc_top]); if ((node->mn_flags & (F_DUPDATA|F_SUBDATA)) != F_SUBDATA) return MDB_INCOMPATIBLE; - } else if (! (rc == MDB_NOTFOUND && (flags & MDB_CREATE))) { - return rc; + } else { + if (rc != MDB_NOTFOUND || !(flags & MDB_CREATE)) + return rc; + if (F_ISSET(txn->mt_flags, MDB_TXN_RDONLY)) + return EACCES; } /* Done here so we cannot fail after creating a new DB */ @@ -10471,7 +10706,8 @@ int mdb_dbi_open(MDB_txn *txn, const char *name, unsigned int flags, MDB_dbi *db memset(&dummy, 0, sizeof(dummy)); dummy.md_root = P_INVALID; dummy.md_flags = flags & PERSISTENT_FLAGS; - rc = mdb_cursor_put(&mc, &key, &data, F_SUBDATA); + WITH_CURSOR_TRACKING(mc, + rc = mdb_cursor_put(&mc, &key, &data, F_SUBDATA)); dbflag |= DB_DIRTY; } @@ -10641,10 +10877,8 @@ mdb_drop0(MDB_cursor *mc, int subs) done: if (rc) txn->mt_flags |= MDB_TXN_ERROR; -#ifdef MDB_VL32 /* drop refcount for mx's pages */ - mdb_cursor_unref(&mx); -#endif + MDB_CURSOR_UNREF(&mx, 0); } else if (rc == MDB_NOTFOUND) { rc = MDB_SUCCESS; } @@ -10764,7 +10998,7 @@ mdb_reader_list(MDB_env *env, MDB_msg_func *func, void *ctx) if (mr[i].mr_pid) { txnid_t txnid = mr[i].mr_txnid; sprintf(buf, txnid == (txnid_t)-1 ? - "%10d %"Z"x -\n" : "%10d %"Z"x %"Y"u\n", + "%10d %"Z"x -\n" : "%10d %"Z"x %"Yu"\n", (int)mr[i].mr_pid, (size_t)mr[i].mr_tid, txnid); if (first) { first = 0; @@ -10833,7 +11067,7 @@ mdb_reader_check(MDB_env *env, int *dead) return env->me_txns ? mdb_reader_check0(env, 0, dead) : MDB_SUCCESS; } -/** As #mdb_reader_check(). rlocked = . */ +/** As #mdb_reader_check(). \b rlocked is set if caller locked #me_rmutex. */ static int ESECT mdb_reader_check0(MDB_env *env, int rlocked, int *dead) { @@ -10869,7 +11103,7 @@ mdb_reader_check0(MDB_env *env, int rlocked, int *dead) } for (; jmn_alloced = 1; + dst->mn_len = need - 1; + dst->mn_val = result; + return MDB_SUCCESS; + } } #endif /* defined(_WIN32) */ +/** @} */ diff --git a/external/db_drivers/liblmdb/mdb_copy.1 b/external/db_drivers/liblmdb/mdb_copy.1 index 401e47abd19..30e4754f137 100644 --- a/external/db_drivers/liblmdb/mdb_copy.1 +++ b/external/db_drivers/liblmdb/mdb_copy.1 @@ -1,5 +1,5 @@ -.TH MDB_COPY 1 "2014/06/20" "LMDB 0.9.14" -.\" Copyright 2012-2015 Howard Chu, Symas Corp. All Rights Reserved. +.TH MDB_COPY 1 "2017/07/31" "LMDB 0.9.70" +.\" Copyright 2012-2019 Howard Chu, Symas Corp. All Rights Reserved. .\" Copying restrictions apply. See COPYRIGHT/LICENSE. .SH NAME mdb_copy \- LMDB environment copy tool @@ -38,6 +38,7 @@ Write the library version number to the standard output, and exit. Compact while copying. Only current data pages will be copied; freed or unused pages will be omitted from the copy. This option will slow down the backup process as it is more CPU-intensive. +Currently it fails if the environment has suffered a page leak. .TP .BR \-n Open LDMB environment(s) which do not use subdirectories. diff --git a/external/db_drivers/liblmdb/mdb_copy.c b/external/db_drivers/liblmdb/mdb_copy.c index 95a6e7130de..23be8150621 100644 --- a/external/db_drivers/liblmdb/mdb_copy.c +++ b/external/db_drivers/liblmdb/mdb_copy.c @@ -1,6 +1,6 @@ /* mdb_copy.c - memory-mapped database backup tool */ /* - * Copyright 2012-2015 Howard Chu, Symas Corp. + * Copyright 2012-2018 Howard Chu, Symas Corp. * All rights reserved. * * Redistribution and use in source and binary forms, with or without diff --git a/external/db_drivers/liblmdb/mdb_drop.1 b/external/db_drivers/liblmdb/mdb_drop.1 index 997e2911820..10666e52041 100644 --- a/external/db_drivers/liblmdb/mdb_drop.1 +++ b/external/db_drivers/liblmdb/mdb_drop.1 @@ -1,5 +1,5 @@ .TH MDB_DROP 1 "2017/11/19" "LMDB 0.9.70" -.\" Copyright 2014-2017 Howard Chu, Symas Corp. All Rights Reserved. +.\" Copyright 2014-2018 Howard Chu, Symas Corp. All Rights Reserved. .\" Copying restrictions apply. See COPYRIGHT/LICENSE. .SH NAME mdb_drop \- LMDB database delete tool diff --git a/external/db_drivers/liblmdb/mdb_drop.c b/external/db_drivers/liblmdb/mdb_drop.c index 725891685ac..7d2af4b62eb 100644 --- a/external/db_drivers/liblmdb/mdb_drop.c +++ b/external/db_drivers/liblmdb/mdb_drop.c @@ -1,6 +1,6 @@ /* mdb_drop.c - memory-mapped database delete tool */ /* - * Copyright 2016-2017 Howard Chu, Symas Corp. + * Copyright 2016-2018 Howard Chu, Symas Corp. * All rights reserved. * * Redistribution and use in source and binary forms, with or without diff --git a/external/db_drivers/liblmdb/mdb_dump.1 b/external/db_drivers/liblmdb/mdb_dump.1 index a25fb92e97b..424ad9321cf 100644 --- a/external/db_drivers/liblmdb/mdb_dump.1 +++ b/external/db_drivers/liblmdb/mdb_dump.1 @@ -1,5 +1,5 @@ -.TH MDB_DUMP 1 "2014/06/20" "LMDB 0.9.14" -.\" Copyright 2014-2015 Howard Chu, Symas Corp. All Rights Reserved. +.TH MDB_DUMP 1 "2017/07/31" "LMDB 0.9.70" +.\" Copyright 2014-2017 Howard Chu, Symas Corp. All Rights Reserved. .\" Copying restrictions apply. See COPYRIGHT/LICENSE. .SH NAME mdb_dump \- LMDB environment export tool diff --git a/external/db_drivers/liblmdb/mdb_dump.c b/external/db_drivers/liblmdb/mdb_dump.c index 7a42bc0b699..b7737f12de1 100644 --- a/external/db_drivers/liblmdb/mdb_dump.c +++ b/external/db_drivers/liblmdb/mdb_dump.c @@ -1,6 +1,6 @@ /* mdb_dump.c - memory-mapped database dump tool */ /* - * Copyright 2011-2015 Howard Chu, Symas Corp. + * Copyright 2011-2018 Howard Chu, Symas Corp. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -20,20 +20,7 @@ #include #include "lmdb.h" -#ifdef _WIN32 -#define Z "I" -#else -#define Z "z" -#endif -#ifdef MDB_VL32 -#ifdef _WIN32 -#define Y "I64" -#else -#define Y "ll" -#endif -#else -#define Y Z -#endif +#define Yu MDB_PRIy(u) #define PRINT 1 static int mode; @@ -124,7 +111,7 @@ static int dumpit(MDB_txn *txn, MDB_dbi dbi, char *name) if (name) printf("database=%s\n", name); printf("type=btree\n"); - printf("mapsize=%" Y "u\n", info.me_mapsize); + printf("mapsize=%"Yu"\n", info.me_mapsize); if (info.me_mapaddr) printf("mapaddr=%p\n", info.me_mapaddr); printf("maxreaders=%u\n", info.me_maxreaders); diff --git a/external/db_drivers/liblmdb/mdb_load.1 b/external/db_drivers/liblmdb/mdb_load.1 index ede3702d928..a2f8373c575 100644 --- a/external/db_drivers/liblmdb/mdb_load.1 +++ b/external/db_drivers/liblmdb/mdb_load.1 @@ -1,5 +1,5 @@ -.TH MDB_LOAD 1 "2014/06/20" "LMDB 0.9.14" -.\" Copyright 2014-2015 Howard Chu, Symas Corp. All Rights Reserved. +.TH MDB_LOAD 1 "2015/09/30" "LMDB 0.9.17" +.\" Copyright 2014-2018 Howard Chu, Symas Corp. All Rights Reserved. .\" Copying restrictions apply. See COPYRIGHT/LICENSE. .SH NAME mdb_load \- LMDB environment import tool diff --git a/external/db_drivers/liblmdb/mdb_load.c b/external/db_drivers/liblmdb/mdb_load.c index c74ce81c94e..ad911c088cf 100644 --- a/external/db_drivers/liblmdb/mdb_load.c +++ b/external/db_drivers/liblmdb/mdb_load.c @@ -1,6 +1,6 @@ /* mdb_load.c - memory-mapped database load tool */ /* - * Copyright 2011-2015 Howard Chu, Symas Corp. + * Copyright 2011-2018 Howard Chu, Symas Corp. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -25,7 +25,7 @@ static int mode; static char *subname = NULL; -static size_t lineno; +static mdb_size_t lineno; static int version; static int flags; @@ -39,20 +39,7 @@ static MDB_envinfo info; static MDB_val kbuf, dbuf; static MDB_val k0buf; -#ifdef _WIN32 -#define Z "I" -#else -#define Z "z" -#endif -#ifdef MDB_VL32 -#ifdef _WIN32 -#define Y "I64" -#else -#define Y "ll" -#endif -#else -#define Y Z -#endif +#define Yu MDB_PRIy(u) #define STRLENOF(s) (sizeof(s)-1) @@ -84,7 +71,7 @@ static void readhdr(void) if (!strncmp(dbuf.mv_data, "VERSION=", STRLENOF("VERSION="))) { version=atoi((char *)dbuf.mv_data+STRLENOF("VERSION=")); if (version > 3) { - fprintf(stderr, "%s: line %" Z "d: unsupported VERSION %d\n", + fprintf(stderr, "%s: line %"Yu": unsupported VERSION %d\n", prog, lineno, version); exit(EXIT_FAILURE); } @@ -94,7 +81,7 @@ static void readhdr(void) if (!strncmp((char *)dbuf.mv_data+STRLENOF("FORMAT="), "print", STRLENOF("print"))) mode |= PRINT; else if (strncmp((char *)dbuf.mv_data+STRLENOF("FORMAT="), "bytevalue", STRLENOF("bytevalue"))) { - fprintf(stderr, "%s: line %" Z "d: unsupported FORMAT %s\n", + fprintf(stderr, "%s: line %"Yu": unsupported FORMAT %s\n", prog, lineno, (char *)dbuf.mv_data+STRLENOF("FORMAT=")); exit(EXIT_FAILURE); } @@ -105,7 +92,7 @@ static void readhdr(void) subname = strdup((char *)dbuf.mv_data+STRLENOF("database=")); } else if (!strncmp(dbuf.mv_data, "type=", STRLENOF("type="))) { if (strncmp((char *)dbuf.mv_data+STRLENOF("type="), "btree", STRLENOF("btree"))) { - fprintf(stderr, "%s: line %" Z "d: unsupported type %s\n", + fprintf(stderr, "%s: line %"Yu": unsupported type %s\n", prog, lineno, (char *)dbuf.mv_data+STRLENOF("type=")); exit(EXIT_FAILURE); } @@ -115,7 +102,7 @@ static void readhdr(void) if (ptr) *ptr = '\0'; i = sscanf((char *)dbuf.mv_data+STRLENOF("mapaddr="), "%p", &info.me_mapaddr); if (i != 1) { - fprintf(stderr, "%s: line %" Z "d: invalid mapaddr %s\n", + fprintf(stderr, "%s: line %"Yu": invalid mapaddr %s\n", prog, lineno, (char *)dbuf.mv_data+STRLENOF("mapaddr=")); exit(EXIT_FAILURE); } @@ -123,9 +110,10 @@ static void readhdr(void) int i; ptr = memchr(dbuf.mv_data, '\n', dbuf.mv_size); if (ptr) *ptr = '\0'; - i = sscanf((char *)dbuf.mv_data+STRLENOF("mapsize="), "%" Y "u", &info.me_mapsize); + i = sscanf((char *)dbuf.mv_data+STRLENOF("mapsize="), + "%" MDB_SCNy(u), &info.me_mapsize); if (i != 1) { - fprintf(stderr, "%s: line %" Z "d: invalid mapsize %s\n", + fprintf(stderr, "%s: line %"Yu": invalid mapsize %s\n", prog, lineno, (char *)dbuf.mv_data+STRLENOF("mapsize=")); exit(EXIT_FAILURE); } @@ -135,7 +123,7 @@ static void readhdr(void) if (ptr) *ptr = '\0'; i = sscanf((char *)dbuf.mv_data+STRLENOF("maxreaders="), "%u", &info.me_maxreaders); if (i != 1) { - fprintf(stderr, "%s: line %" Z "d: invalid maxreaders %s\n", + fprintf(stderr, "%s: line %"Yu": invalid maxreaders %s\n", prog, lineno, (char *)dbuf.mv_data+STRLENOF("maxreaders=")); exit(EXIT_FAILURE); } @@ -151,12 +139,12 @@ static void readhdr(void) if (!dbflags[i].bit) { ptr = memchr(dbuf.mv_data, '=', dbuf.mv_size); if (!ptr) { - fprintf(stderr, "%s: line %" Z "d: unexpected format\n", + fprintf(stderr, "%s: line %"Yu": unexpected format\n", prog, lineno); exit(EXIT_FAILURE); } else { *ptr = '\0'; - fprintf(stderr, "%s: line %" Z "d: unrecognized keyword ignored: %s\n", + fprintf(stderr, "%s: line %"Yu": unrecognized keyword ignored: %s\n", prog, lineno, (char *)dbuf.mv_data); } } @@ -166,7 +154,7 @@ static void readhdr(void) static void badend(void) { - fprintf(stderr, "%s: line %" Z "d: unexpected end of input\n", + fprintf(stderr, "%s: line %"Yu": unexpected end of input\n", prog, lineno); } @@ -224,7 +212,7 @@ static int readline(MDB_val *out, MDB_val *buf) buf->mv_data = realloc(buf->mv_data, buf->mv_size*2); if (!buf->mv_data) { Eof = 1; - fprintf(stderr, "%s: line %" Z "d: out of memory, line too long\n", + fprintf(stderr, "%s: line %"Yu": out of memory, line too long\n", prog, lineno); return EOF; } @@ -259,7 +247,8 @@ static int readline(MDB_val *out, MDB_val *buf) c2 += 2; } } else { - c1++; c2++; + /* copies are redundant when no escapes were used */ + *c1++ = *c2++; } } } else { @@ -435,7 +424,7 @@ int main(int argc, char *argv[]) rc = readline(&data, &dbuf); if (rc) { - fprintf(stderr, "%s: line %" Z "d: failed to read key value\n", prog, lineno); + fprintf(stderr, "%s: line %"Yu": failed to read key value\n", prog, lineno); goto txn_abort; } @@ -463,7 +452,7 @@ int main(int argc, char *argv[]) if (batch == 100) { rc = mdb_txn_commit(txn); if (rc) { - fprintf(stderr, "%s: line %" Z "d: txn_commit: %s\n", + fprintf(stderr, "%s: line %"Yu": txn_commit: %s\n", prog, lineno, mdb_strerror(rc)); goto env_close; } @@ -487,7 +476,7 @@ int main(int argc, char *argv[]) rc = mdb_txn_commit(txn); txn = NULL; if (rc) { - fprintf(stderr, "%s: line %" Z "d: txn_commit: %s\n", + fprintf(stderr, "%s: line %"Yu": txn_commit: %s\n", prog, lineno, mdb_strerror(rc)); goto env_close; } diff --git a/external/db_drivers/liblmdb/mdb_stat.1 b/external/db_drivers/liblmdb/mdb_stat.1 index bf49bd3bd21..83988a0fa02 100644 --- a/external/db_drivers/liblmdb/mdb_stat.1 +++ b/external/db_drivers/liblmdb/mdb_stat.1 @@ -1,5 +1,5 @@ -.TH MDB_STAT 1 "2014/06/20" "LMDB 0.9.14" -.\" Copyright 2012-2015 Howard Chu, Symas Corp. All Rights Reserved. +.TH MDB_STAT 1 "2017/07/31" "LMDB 0.9.70" +.\" Copyright 2012-2019 Howard Chu, Symas Corp. All Rights Reserved. .\" Copying restrictions apply. See COPYRIGHT/LICENSE. .SH NAME mdb_stat \- LMDB environment status tool diff --git a/external/db_drivers/liblmdb/mdb_stat.c b/external/db_drivers/liblmdb/mdb_stat.c index 30ec81fea3c..08f30befd66 100644 --- a/external/db_drivers/liblmdb/mdb_stat.c +++ b/external/db_drivers/liblmdb/mdb_stat.c @@ -1,6 +1,6 @@ /* mdb_stat.c - memory-mapped database status tool */ /* - * Copyright 2011-2015 Howard Chu, Symas Corp. + * Copyright 2011-2018 Howard Chu, Symas Corp. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -17,20 +17,8 @@ #include #include "lmdb.h" -#ifdef _WIN32 -#define Z "I" -#else -#define Z "z" -#endif -#ifdef MDB_VL32 -#ifdef _WIN32 -#define Y "I64" -#else -#define Y "ll" -#endif -#else -#define Y Z -#endif +#define Z MDB_FMT_Z +#define Yu MDB_PRIy(u) static void prstat(MDB_stat *ms) { @@ -38,10 +26,10 @@ static void prstat(MDB_stat *ms) printf(" Page size: %u\n", ms->ms_psize); #endif printf(" Tree depth: %u\n", ms->ms_depth); - printf(" Branch pages: %"Y"u\n", ms->ms_branch_pages); - printf(" Leaf pages: %"Y"u\n", ms->ms_leaf_pages); - printf(" Overflow pages: %"Y"u\n", ms->ms_overflow_pages); - printf(" Entries: %"Y"u\n", ms->ms_entries); + printf(" Branch pages: %"Yu"\n", ms->ms_branch_pages); + printf(" Leaf pages: %"Yu"\n", ms->ms_leaf_pages); + printf(" Overflow pages: %"Yu"\n", ms->ms_overflow_pages); + printf(" Entries: %"Yu"\n", ms->ms_entries); } static void usage(char *prog) @@ -138,11 +126,11 @@ int main(int argc, char *argv[]) (void)mdb_env_info(env, &mei); printf("Environment Info\n"); printf(" Map address: %p\n", mei.me_mapaddr); - printf(" Map size: %"Y"u\n", mei.me_mapsize); + printf(" Map size: %"Yu"\n", mei.me_mapsize); printf(" Page size: %u\n", mst.ms_psize); - printf(" Max pages: %"Y"u\n", mei.me_mapsize / mst.ms_psize); - printf(" Number of pages used: %"Y"u\n", mei.me_last_pgno+1); - printf(" Last transaction ID: %"Y"u\n", mei.me_last_txnid); + printf(" Max pages: %"Yu"\n", mei.me_mapsize / mst.ms_psize); + printf(" Number of pages used: %"Yu"\n", mei.me_last_pgno+1); + printf(" Last transaction ID: %"Yu"\n", mei.me_last_txnid); printf(" Max readers: %u\n", mei.me_maxreaders); printf(" Number of readers used: %u\n", mei.me_numreaders); } @@ -169,7 +157,7 @@ int main(int argc, char *argv[]) if (freinfo) { MDB_cursor *cursor; MDB_val key, data; - size_t pages = 0, *iptr; + mdb_size_t pages = 0, *iptr; printf("Freelist Status\n"); dbi = 0; @@ -189,7 +177,7 @@ int main(int argc, char *argv[]) pages += *iptr; if (freinfo > 1) { char *bad = ""; - size_t pg, prev; + mdb_size_t pg, prev; ssize_t i, j, span = 0; j = *iptr++; for (i = j, prev = 1; --i >= 0; ) { @@ -200,20 +188,20 @@ int main(int argc, char *argv[]) pg += span; for (; i >= span && iptr[i-span] == pg; span++, pg++) ; } - printf(" Transaction %"Z"u, %"Z"d pages, maxspan %"Z"d%s\n", - *(size_t *)key.mv_data, j, span, bad); + printf(" Transaction %"Yu", %"Z"d pages, maxspan %"Z"d%s\n", + *(mdb_size_t *)key.mv_data, j, span, bad); if (freinfo > 2) { for (--j; j >= 0; ) { pg = iptr[j]; for (span=1; --j >= 0 && iptr[j] == pg+span; span++) ; - printf(span>1 ? " %9"Z"u[%"Z"d]\n" : " %9"Z"u\n", + printf(span>1 ? " %9"Yu"[%"Z"d]\n" : " %9"Yu"\n", pg, span); } } } } mdb_cursor_close(cursor); - printf(" Free pages: %"Z"u\n", pages); + printf(" Free pages: %"Yu"\n", pages); } rc = mdb_open(txn, subname, 0, &dbi); diff --git a/external/db_drivers/liblmdb/midl.c b/external/db_drivers/liblmdb/midl.c index 152a1ec0dda..ee89822e394 100644 --- a/external/db_drivers/liblmdb/midl.c +++ b/external/db_drivers/liblmdb/midl.c @@ -3,7 +3,8 @@ /* $OpenLDAP$ */ /* This work is part of OpenLDAP Software . * - * Copyright 2000-2015 The OpenLDAP Foundation. + * Copyright 2000-2019 The OpenLDAP Foundation. + * Portions Copyright 2001-2018 Howard Chu, Symas Corp. * All rights reserved. * * Redistribution and use in source and binary forms, with or without diff --git a/external/db_drivers/liblmdb/midl.h b/external/db_drivers/liblmdb/midl.h index 1555ecb1950..0f826771a1c 100644 --- a/external/db_drivers/liblmdb/midl.h +++ b/external/db_drivers/liblmdb/midl.h @@ -11,7 +11,8 @@ /* $OpenLDAP$ */ /* This work is part of OpenLDAP Software . * - * Copyright 2000-2015 The OpenLDAP Foundation. + * Copyright 2000-2019 The OpenLDAP Foundation. + * Portions Copyright 2001-2019 Howard Chu, Symas Corp. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -26,8 +27,7 @@ #ifndef _MDB_MIDL_H_ #define _MDB_MIDL_H_ -#include -#include +#include "lmdb.h" #ifdef __cplusplus extern "C" { @@ -43,11 +43,7 @@ extern "C" { /** A generic unsigned ID number. These were entryIDs in back-bdb. * Preferably it should have the same size as a pointer. */ -#ifdef MDB_VL32 -typedef uint64_t MDB_ID; -#else -typedef size_t MDB_ID; -#endif +typedef mdb_size_t MDB_ID; /** An IDL is an ID List, a sorted array of IDs. The first * element of the array is a counter for how many actual diff --git a/external/db_drivers/liblmdb/mtest.c b/external/db_drivers/liblmdb/mtest.c index 9d15088b0c2..6fc5840c3f8 100644 --- a/external/db_drivers/liblmdb/mtest.c +++ b/external/db_drivers/liblmdb/mtest.c @@ -1,6 +1,6 @@ /* mtest.c - memory-mapped database tester/toy */ /* - * Copyright 2011-2015 Howard Chu, Symas Corp. + * Copyright 2011-2018 Howard Chu, Symas Corp. * All rights reserved. * * Redistribution and use in source and binary forms, with or without diff --git a/external/db_drivers/liblmdb/mtest2.c b/external/db_drivers/liblmdb/mtest2.c index eacbe59d53b..64b742aa467 100644 --- a/external/db_drivers/liblmdb/mtest2.c +++ b/external/db_drivers/liblmdb/mtest2.c @@ -1,6 +1,6 @@ /* mtest2.c - memory-mapped database tester/toy */ /* - * Copyright 2011-2015 Howard Chu, Symas Corp. + * Copyright 2011-2018 Howard Chu, Symas Corp. * All rights reserved. * * Redistribution and use in source and binary forms, with or without diff --git a/external/db_drivers/liblmdb/mtest3.c b/external/db_drivers/liblmdb/mtest3.c index 9db79e625d2..81e4bbf9b05 100644 --- a/external/db_drivers/liblmdb/mtest3.c +++ b/external/db_drivers/liblmdb/mtest3.c @@ -1,6 +1,6 @@ /* mtest3.c - memory-mapped database tester/toy */ /* - * Copyright 2011-2015 Howard Chu, Symas Corp. + * Copyright 2011-2018 Howard Chu, Symas Corp. * All rights reserved. * * Redistribution and use in source and binary forms, with or without diff --git a/external/db_drivers/liblmdb/mtest4.c b/external/db_drivers/liblmdb/mtest4.c index 6df890e2d70..c355cf105a9 100644 --- a/external/db_drivers/liblmdb/mtest4.c +++ b/external/db_drivers/liblmdb/mtest4.c @@ -1,6 +1,6 @@ /* mtest4.c - memory-mapped database tester/toy */ /* - * Copyright 2011-2015 Howard Chu, Symas Corp. + * Copyright 2011-2018 Howard Chu, Symas Corp. * All rights reserved. * * Redistribution and use in source and binary forms, with or without diff --git a/external/db_drivers/liblmdb/mtest5.c b/external/db_drivers/liblmdb/mtest5.c index 14e3c0da4a9..95793ec1fb7 100644 --- a/external/db_drivers/liblmdb/mtest5.c +++ b/external/db_drivers/liblmdb/mtest5.c @@ -1,6 +1,6 @@ /* mtest5.c - memory-mapped database tester/toy */ /* - * Copyright 2011-2015 Howard Chu, Symas Corp. + * Copyright 2011-2018 Howard Chu, Symas Corp. * All rights reserved. * * Redistribution and use in source and binary forms, with or without diff --git a/external/db_drivers/liblmdb/mtest6.c b/external/db_drivers/liblmdb/mtest6.c index ae3c7f264ca..cb0d4d73c9a 100644 --- a/external/db_drivers/liblmdb/mtest6.c +++ b/external/db_drivers/liblmdb/mtest6.c @@ -1,6 +1,6 @@ /* mtest6.c - memory-mapped database tester/toy */ /* - * Copyright 2011-2015 Howard Chu, Symas Corp. + * Copyright 2011-2018 Howard Chu, Symas Corp. * All rights reserved. * * Redistribution and use in source and binary forms, with or without diff --git a/external/db_drivers/liblmdb/sample-bdb.txt b/external/db_drivers/liblmdb/sample-bdb.txt index 563807a2b0b..97220f0ed22 100644 --- a/external/db_drivers/liblmdb/sample-bdb.txt +++ b/external/db_drivers/liblmdb/sample-bdb.txt @@ -3,7 +3,7 @@ * Do a line-by-line comparison of this and sample-mdb.txt */ /* - * Copyright 2012-2015 Howard Chu, Symas Corp. + * Copyright 2012-2018 Howard Chu, Symas Corp. * All rights reserved. * * Redistribution and use in source and binary forms, with or without diff --git a/external/db_drivers/liblmdb/sample-mdb.txt b/external/db_drivers/liblmdb/sample-mdb.txt index 10a256870ba..1d20ed3d02b 100644 --- a/external/db_drivers/liblmdb/sample-mdb.txt +++ b/external/db_drivers/liblmdb/sample-mdb.txt @@ -3,7 +3,7 @@ * Do a line-by-line comparison of this and sample-bdb.txt */ /* - * Copyright 2012-2015 Howard Chu, Symas Corp. + * Copyright 2012-2018 Howard Chu, Symas Corp. * All rights reserved. * * Redistribution and use in source and binary forms, with or without diff --git a/external/easylogging++/CMakeLists.txt b/external/easylogging++/CMakeLists.txt index 78795d54bf7..35fb86552b3 100644 --- a/external/easylogging++/CMakeLists.txt +++ b/external/easylogging++/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2014-2018, The Monero Project +# Copyright (c) 2014-2019, The Monero Project # # All rights reserved. # @@ -33,6 +33,7 @@ project(easylogging CXX) SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11") find_package(Threads) +find_package(Backtrace) add_library(easylogging easylogging++.cc) @@ -41,7 +42,8 @@ include_directories("${CMAKE_CURRENT_SOURCE_DIR}") include_directories("${CMAKE_CURRENT_BINARY_DIR}") target_link_libraries(easylogging PRIVATE - ${CMAKE_THREAD_LIBS_INIT}) + ${CMAKE_THREAD_LIBS_INIT} + ${Backtrace_LIBRARIES}) # GUI/libwallet install target if (BUILD_GUI_DEPS) diff --git a/external/easylogging++/ea_config.h b/external/easylogging++/ea_config.h index 4c74925d389..4fb48ce3edd 100644 --- a/external/easylogging++/ea_config.h +++ b/external/easylogging++/ea_config.h @@ -6,6 +6,7 @@ #define ELPP_NO_CHECK_MACROS #define ELPP_WINSOCK2 #define ELPP_NO_DEBUG_MACROS +#define ELPP_UTC_DATETIME #ifdef EASYLOGGING_CC #if !(!defined __GNUC__ || defined __MINGW32__ || defined __MINGW64__ || defined __ANDROID__) diff --git a/external/easylogging++/easylogging++.cc b/external/easylogging++/easylogging++.cc index 212a1822dd1..5a806dc695d 100644 --- a/external/easylogging++/easylogging++.cc +++ b/external/easylogging++/easylogging++.cc @@ -1,16 +1,16 @@ // // Bismillah ar-Rahmaan ar-Raheem // -// Easylogging++ v9.94.1 +// Easylogging++ v9.96.7 // Cross-platform logging library for C++ applications // -// Copyright (c) 2017 muflihun.com +// Copyright (c) 2012-2018 Zuhd Web Services +// Copyright (c) 2012-2018 @abumusamq // // This library is released under the MIT Licence. -// http://labs.muflihun.com/easyloggingpp/licence.php +// https://github.com/zuhd-org/easyloggingpp/blob/master/LICENSE // -// https://github.com/muflihun/easyloggingpp -// https://muflihun.github.io/easyloggingpp +// https://zuhd.org // http://muflihun.com // @@ -25,8 +25,95 @@ INITIALIZE_EASYLOGGINGPP namespace el { -// el::base::utils +// el::base namespace base { +// el::base::consts +namespace consts { + +// Level log values - These are values that are replaced in place of %level format specifier +// Extra spaces after format specifiers are only for readability purposes in log files +static const base::type::char_t* kInfoLevelLogValue = ELPP_LITERAL("INFO"); +static const base::type::char_t* kDebugLevelLogValue = ELPP_LITERAL("DEBUG"); +static const base::type::char_t* kWarningLevelLogValue = ELPP_LITERAL("WARNING"); +static const base::type::char_t* kErrorLevelLogValue = ELPP_LITERAL("ERROR"); +static const base::type::char_t* kFatalLevelLogValue = ELPP_LITERAL("FATAL"); +static const base::type::char_t* kVerboseLevelLogValue = + ELPP_LITERAL("VERBOSE"); // will become VERBOSE-x where x = verbose level +static const base::type::char_t* kTraceLevelLogValue = ELPP_LITERAL("TRACE"); +static const base::type::char_t* kInfoLevelShortLogValue = ELPP_LITERAL("I"); +static const base::type::char_t* kDebugLevelShortLogValue = ELPP_LITERAL("D"); +static const base::type::char_t* kWarningLevelShortLogValue = ELPP_LITERAL("W"); +static const base::type::char_t* kErrorLevelShortLogValue = ELPP_LITERAL("E"); +static const base::type::char_t* kFatalLevelShortLogValue = ELPP_LITERAL("F"); +static const base::type::char_t* kVerboseLevelShortLogValue = ELPP_LITERAL("V"); +static const base::type::char_t* kTraceLevelShortLogValue = ELPP_LITERAL("T"); +// Format specifiers - These are used to define log format +static const base::type::char_t* kAppNameFormatSpecifier = ELPP_LITERAL("%app"); +static const base::type::char_t* kLoggerIdFormatSpecifier = ELPP_LITERAL("%logger"); +static const base::type::char_t* kThreadIdFormatSpecifier = ELPP_LITERAL("%thread"); +static const base::type::char_t* kSeverityLevelFormatSpecifier = ELPP_LITERAL("%level"); +static const base::type::char_t* kSeverityLevelShortFormatSpecifier = ELPP_LITERAL("%levshort"); +static const base::type::char_t* kDateTimeFormatSpecifier = ELPP_LITERAL("%datetime"); +static const base::type::char_t* kLogFileFormatSpecifier = ELPP_LITERAL("%file"); +static const base::type::char_t* kLogFileBaseFormatSpecifier = ELPP_LITERAL("%fbase"); +static const base::type::char_t* kLogLineFormatSpecifier = ELPP_LITERAL("%line"); +static const base::type::char_t* kLogLocationFormatSpecifier = ELPP_LITERAL("%loc"); +static const base::type::char_t* kLogFunctionFormatSpecifier = ELPP_LITERAL("%func"); +static const base::type::char_t* kCurrentUserFormatSpecifier = ELPP_LITERAL("%user"); +static const base::type::char_t* kCurrentHostFormatSpecifier = ELPP_LITERAL("%host"); +static const base::type::char_t* kMessageFormatSpecifier = ELPP_LITERAL("%msg"); +static const base::type::char_t* kVerboseLevelFormatSpecifier = ELPP_LITERAL("%vlevel"); +static const char* kDateTimeFormatSpecifierForFilename = "%datetime"; +// Date/time +static const char* kDays[7] = { "Sunday", "Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday" }; +static const char* kDaysAbbrev[7] = { "Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat" }; +static const char* kMonths[12] = { "January", "February", "March", "Apri", "May", "June", "July", "August", + "September", "October", "November", "December" + }; +static const char* kMonthsAbbrev[12] = { "Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec" }; +static const char* kDefaultDateTimeFormat = "%Y-%M-%d %H:%m:%s,%g"; +static const char* kDefaultDateTimeFormatInFilename = "%Y-%M-%d_%H-%m"; +static const int kYearBase = 1900; +static const char* kAm = "AM"; +static const char* kPm = "PM"; +// Miscellaneous constants + +static const char* kNullPointer = "nullptr"; +#if ELPP_VARIADIC_TEMPLATES_SUPPORTED +#endif // ELPP_VARIADIC_TEMPLATES_SUPPORTED +static const base::type::VerboseLevel kMaxVerboseLevel = 9; +static const char* kUnknownUser = "user"; +static const char* kUnknownHost = "unknown-host"; + + +//---------------- DEFAULT LOG FILE ----------------------- + +#if defined(ELPP_NO_DEFAULT_LOG_FILE) +# if ELPP_OS_UNIX +static const char* kDefaultLogFile = "/dev/null"; +# elif ELPP_OS_WINDOWS +static const char* kDefaultLogFile = "nul"; +# endif // ELPP_OS_UNIX +#elif defined(ELPP_DEFAULT_LOG_FILE) +static const char* kDefaultLogFile = ELPP_DEFAULT_LOG_FILE; +#else +static const char* kDefaultLogFile = "myeasylog.log"; +#endif // defined(ELPP_NO_DEFAULT_LOG_FILE) + + +#if !defined(ELPP_DISABLE_LOG_FILE_FROM_ARG) +static const char* kDefaultLogFileParam = "--default-log-file"; +#endif // !defined(ELPP_DISABLE_LOG_FILE_FROM_ARG) +#if defined(ELPP_LOGGING_FLAGS_FROM_ARG) +static const char* kLoggingFlagsParam = "--logging-flags"; +#endif // defined(ELPP_LOGGING_FLAGS_FROM_ARG) +static const char* kValidLoggerIdSymbols = + "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-._"; +static const char* kConfigurationComment = "##"; +static const char* kConfigurationLevel = "*"; +static const char* kConfigurationLoggerId = "--"; +} +// el::base::utils namespace utils { /// @brief Aborts application due with user-defined status @@ -303,11 +390,7 @@ void Configurations::set(Configuration* conf) { void Configurations::setToDefault(void) { setGlobally(ConfigurationType::Enabled, std::string("true"), true); -#if !defined(ELPP_NO_DEFAULT_LOG_FILE) setGlobally(ConfigurationType::Filename, std::string(base::consts::kDefaultLogFile), true); -#else - ELPP_UNUSED(base::consts::kDefaultLogFile); -#endif // !defined(ELPP_NO_DEFAULT_LOG_FILE) #if defined(ELPP_NO_LOG_TO_FILE) setGlobally(ConfigurationType::ToFile, std::string("false"), true); #else @@ -336,9 +419,7 @@ void Configurations::setRemainingToDefault(void) { #else unsafeSetIfNotExist(Level::Global, ConfigurationType::Enabled, std::string("true")); #endif // defined(ELPP_NO_LOG_TO_FILE) -#if !defined(ELPP_NO_DEFAULT_LOG_FILE) unsafeSetIfNotExist(Level::Global, ConfigurationType::Filename, std::string(base::consts::kDefaultLogFile)); -#endif // !defined(ELPP_NO_DEFAULT_LOG_FILE) unsafeSetIfNotExist(Level::Global, ConfigurationType::ToStandardOutput, std::string("true")); unsafeSetIfNotExist(Level::Global, ConfigurationType::SubsecondPrecision, std::string("3")); unsafeSetIfNotExist(Level::Global, ConfigurationType::PerformanceTracking, std::string("true")); @@ -596,7 +677,6 @@ void Logger::configure(const Configurations& configurations) { if (m_typedConfigurations != nullptr) { Configurations* c = const_cast(m_typedConfigurations->configurations()); if (c->hasConfiguration(Level::Global, ConfigurationType::Filename)) { - // This check is definitely needed for cases like ELPP_NO_DEFAULT_LOG_FILE flush(); } } @@ -640,7 +720,11 @@ void Logger::flush(Level level, base::type::fstream_t* fs) { } if (fs != nullptr) { fs->flush(); - m_unflushedCount.find(level)->second = 0; + std::unordered_map::iterator iter = m_unflushedCount.find(level); + if (iter != m_unflushedCount.end()) { + iter->second = 0; + } + Helpers::validateFileRolling(this, level); } } @@ -699,10 +783,9 @@ std::size_t File::getSizeOfFile(base::type::fstream_t* fs) { if (fs == nullptr) { return 0; } - std::streampos currPos = fs->tellg(); - fs->seekg(0, fs->end); + // Since the file stream is appended to or truncated, the current + // offset is the file size. std::size_t size = static_cast(fs->tellg()); - fs->seekg(currPos); return size; } @@ -878,7 +961,7 @@ void Str::replaceFirstWithEscape(base::type::string_t& str, const base::type::st std::size_t foundAt = base::type::string_t::npos; while ((foundAt = str.find(replaceWhat, foundAt + 1)) != base::type::string_t::npos) { if (foundAt > 0 && str[foundAt - 1] == base::consts::kFormatSpecifierChar) { - str.erase(foundAt > 0 ? foundAt - 1 : 0, 1); + str.erase(foundAt - 1, 1); ++foundAt; } else { str.replace(foundAt, replaceWhat.length(), replaceWith); @@ -894,7 +977,10 @@ void Str::replaceFirstWithEscape(base::type::string_t& str, const base::type::st #endif // defined(ELPP_UNICODE) std::string& Str::toUpper(std::string& str) { - std::transform(str.begin(), str.end(), str.begin(), ::toupper); + std::transform(str.begin(), str.end(), str.begin(), + [](char c) { + return static_cast(::toupper(c)); + }); return str; } @@ -1022,11 +1108,13 @@ const std::string OS::getBashOutput(const char* command) { char hBuff[4096]; if (fgets(hBuff, sizeof(hBuff), proc) != nullptr) { pclose(proc); - const size_t len = strlen(hBuff); - if (len > 0 && hBuff[len - 1] == '\n') { - hBuff[len- 1] = '\0'; + const std::size_t buffLen = strlen(hBuff); + if (buffLen > 0 && hBuff[buffLen - 1] == '\n') { + hBuff[buffLen- 1] = '\0'; } return std::string(hBuff); + } else { + pclose(proc); } return std::string(); #else @@ -1172,19 +1260,23 @@ unsigned long long DateTime::getTimeDifference(const struct timeval& endTime, co struct ::tm* DateTime::buildTimeInfo(struct timeval* currTime, struct ::tm* timeInfo) { #if ELPP_OS_UNIX time_t rawTime = currTime->tv_sec; - ::gmtime_r(&rawTime, timeInfo); + ::elpptime_r(&rawTime, timeInfo); return timeInfo; #else # if ELPP_COMPILER_MSVC ELPP_UNUSED(currTime); time_t t; +# if defined(_USE_32BIT_TIME_T) + _time32(&t); +# else _time64(&t); - gmtime_s(timeInfo, &t); +# endif + elpptime_s(timeInfo, &t); return timeInfo; # else // For any other compilers that don't have CRT warnings issue e.g, MinGW or TDM GCC- we use different method time_t rawTime = currTime->tv_sec; - struct tm* tmInf = gmtime(&rawTime); + struct tm* tmInf = elpptime(&rawTime); *timeInfo = *tmInf; return timeInfo; # endif // ELPP_COMPILER_MSVC @@ -1292,7 +1384,8 @@ bool CommandLineArgs::hasParamWithValue(const char* paramKey) const { } const char* CommandLineArgs::getParamValue(const char* paramKey) const { - return m_paramsWithValue.find(std::string(paramKey))->second.c_str(); + std::unordered_map::const_iterator iter = m_paramsWithValue.find(std::string(paramKey)); + return iter != m_paramsWithValue.end() ? iter->second.c_str() : ""; } bool CommandLineArgs::hasParam(const char* paramKey) const { @@ -1437,7 +1530,7 @@ void LogFormat::parseFromFormat(const base::type::string_t& userFormat) { if (hasFlag(flag)) { // If we already have flag we remove the escape chars so that '%%' is turned to '%' // even after specifier resolution - this is because we only replaceFirst specifier - formatCopy.erase(foundAt > 0 ? foundAt - 1 : 0, 1); + formatCopy.erase(foundAt - 1, 1); ++foundAt; } } else { @@ -1641,10 +1734,11 @@ void TypedConfigurations::build(Configurations* configurations) { } else if (conf->configurationType() == ConfigurationType::PerformanceTracking) { setValue(Level::Global, getBool(conf->value()), &m_performanceTrackingMap); } else if (conf->configurationType() == ConfigurationType::MaxLogFileSize) { - setValue(conf->level(), static_cast(getULong(conf->value())), &m_maxLogFileSizeMap); -#if !defined(ELPP_NO_DEFAULT_LOG_FILE) - withFileSizeLimit.push_back(conf); -#endif // !defined(ELPP_NO_DEFAULT_LOG_FILE) + auto v = getULong(conf->value()); + setValue(conf->level(), static_cast(v), &m_maxLogFileSizeMap); + if (v != 0) { + withFileSizeLimit.push_back(conf); + } } else if (conf->configurationType() == ConfigurationType::LogFlushThreshold) { setValue(conf->level(), static_cast(getULong(conf->value())), &m_logFlushThresholdMap); } @@ -1718,12 +1812,6 @@ std::string TypedConfigurations::resolveFilename(const std::string& filename) { } void TypedConfigurations::insertFile(Level level, const std::string& fullFilename) { -#if defined(ELPP_NO_LOG_TO_FILE) - setValue(level, false, &m_toFileMap); - ELPP_UNUSED(fullFilename); - m_fileStreamMap.insert(std::make_pair(level, base::FileStreamPtr(nullptr))); - return; -#endif if (fullFilename.empty()) return; std::string resolvedFilename = resolveFilename(fullFilename); @@ -1861,8 +1949,10 @@ bool RegisteredLoggers::remove(const std::string& id) { if (id == base::consts::kDefaultLoggerId) { return false; } + // get has internal lock Logger* logger = base::utils::Registry::get(id); if (logger != nullptr) { + // unregister has internal lock unregister(logger); } return true; @@ -1967,17 +2057,25 @@ void VRegistry::setCategories(const char* categories, bool clear) { base::threading::ScopedLock scopedLock(lock()); auto insert = [&](std::stringstream& ss, Level level) { m_categories.push_back(std::make_pair(ss.str(), level)); + m_cached_allowed_categories.clear(); }; if (clear) { m_categories.clear(); + m_cached_allowed_categories.clear(); m_categoriesString.clear(); } + if (!categories) + return; if (!m_categoriesString.empty()) m_categoriesString += ","; m_categoriesString += categories; - if (!categories) - return; + + size_t n_fields = m_categories.size() + 1; + for (const char *ptr = categories; *ptr; ++ptr) + if (*ptr == ',') + ++n_fields; + m_categories.reserve(n_fields); bool isCat = true; bool isLevel = false; @@ -2031,17 +2129,23 @@ static int priority(Level level) { return 7; } -bool VRegistry::allowed(Level level, const char* category) { +bool VRegistry::allowed(Level level, const std::string &category) { base::threading::ScopedLock scopedLock(lock()); - if (m_categories.empty() || category == nullptr) { + const std::map::const_iterator it = m_cached_allowed_categories.find(category); + if (it != m_cached_allowed_categories.end()) + return priority(level) <= it->second; + if (m_categories.empty()) { return false; } else { - std::deque>::const_reverse_iterator it = m_categories.rbegin(); + std::vector>::const_reverse_iterator it = m_categories.rbegin(); for (; it != m_categories.rend(); ++it) { - if (base::utils::Str::wildCardMatch(category, it->first.c_str())) { - return priority(level) <= priority(it->second); + if (base::utils::Str::wildCardMatch(category.c_str(), it->first.c_str())) { + const int p = priority(it->second); + m_cached_allowed_categories.insert(std::make_pair(category, p)); + return priority(level) <= p; } } + m_cached_allowed_categories.insert(std::make_pair(category, -1)); return false; } } @@ -2051,9 +2155,11 @@ bool VRegistry::allowed(base::type::VerboseLevel vlevel, const char* file) { if (m_modules.empty() || file == nullptr) { return vlevel <= m_level; } else { - std::map::iterator it = m_modules.begin(); + char baseFilename[base::consts::kSourceFilenameMaxLength] = ""; + base::utils::File::buildBaseFilename(file, baseFilename); + std::unordered_map::iterator it = m_modules.begin(); for (; it != m_modules.end(); ++it) { - if (base::utils::Str::wildCardMatch(file, it->first.c_str())) { + if (base::utils::Str::wildCardMatch(baseFilename, it->first.c_str())) { return vlevel <= it->second; } } @@ -2083,6 +2189,13 @@ void VRegistry::setFromArgs(const base::utils::CommandLineArgs* commandLineArgs) # define ELPP_DEFAULT_LOGGING_FLAGS 0x0 #endif // !defined(ELPP_DEFAULT_LOGGING_FLAGS) // Storage +el::base::type::StoragePointer &el::base::Storage::getELPP() +{ + if (!el::base::elStorage) + el::base::elStorage = new el::base::Storage(el::LogBuilderPtr(new el::base::DefaultLogBuilder())); + return el::base::elStorage; +} +static struct EnsureELPP { EnsureELPP() { el::base::Storage::getELPP(); } } ensureELPP; #if ELPP_ASYNC_LOGGING Storage::Storage(const LogBuilderPtr& defaultLogBuilder, base::IWorker* asyncDispatchWorker) : #else @@ -2092,20 +2205,26 @@ Storage::Storage(const LogBuilderPtr& defaultLogBuilder) : m_registeredLoggers(new base::RegisteredLoggers(defaultLogBuilder)), m_flags(ELPP_DEFAULT_LOGGING_FLAGS), m_vRegistry(new base::VRegistry(0, &m_flags)), + #if ELPP_ASYNC_LOGGING m_asyncLogQueue(new base::AsyncLogQueue()), m_asyncDispatchWorker(asyncDispatchWorker), #endif // ELPP_ASYNC_LOGGING + m_preRollOutCallback(base::defaultPreRollOutCallback) { // Register default logger m_registeredLoggers->get(std::string(base::consts::kDefaultLoggerId)); // We register default logger anyway (worse case it's not going to register) just in case m_registeredLoggers->get("default"); + +#if defined(ELPP_FEATURE_ALL) || defined(ELPP_FEATURE_PERFORMANCE_TRACKING) // Register performance logger and reconfigure format Logger* performanceLogger = m_registeredLoggers->get(std::string(base::consts::kPerformanceLoggerId)); m_registeredLoggers->get("performance"); performanceLogger->configurations()->setGlobally(ConfigurationType::Format, std::string("%datetime %level %msg")); performanceLogger->reconfigure(); +#endif // defined(ELPP_FEATURE_ALL) || defined(ELPP_FEATURE_PERFORMANCE_TRACKING) + #if defined(ELPP_SYSLOG) // Register syslog logger and reconfigure format Logger* sysLogLogger = m_registeredLoggers->get(std::string(base::consts::kSysLogLoggerId)); @@ -2149,7 +2268,7 @@ Storage::~Storage(void) { } bool Storage::hasCustomFormatSpecifier(const char* formatSpecifier) { - base::threading::ScopedLock scopedLock(lock()); + base::threading::ScopedLock scopedLock(customFormatSpecifiersLock()); return std::find(m_customFormatSpecifiers.begin(), m_customFormatSpecifiers.end(), formatSpecifier) != m_customFormatSpecifiers.end(); } @@ -2158,12 +2277,12 @@ void Storage::installCustomFormatSpecifier(const CustomFormatSpecifier& customFo if (hasCustomFormatSpecifier(customFormatSpecifier.formatSpecifier())) { return; } - base::threading::ScopedLock scopedLock(lock()); + base::threading::ScopedLock scopedLock(customFormatSpecifiersLock()); m_customFormatSpecifiers.push_back(customFormatSpecifier); } bool Storage::uninstallCustomFormatSpecifier(const char* formatSpecifier) { - base::threading::ScopedLock scopedLock(lock()); + base::threading::ScopedLock scopedLock(customFormatSpecifiersLock()); std::vector::iterator it = std::find(m_customFormatSpecifiers.begin(), m_customFormatSpecifiers.end(), formatSpecifier); if (it != m_customFormatSpecifiers.end() && strcmp(formatSpecifier, it->formatSpecifier()) == 0) { @@ -2201,15 +2320,58 @@ void Storage::setApplicationArguments(int argc, char** argv) { #endif // defined(ELPP_LOGGING_FLAGS_FROM_ARG) } +} // namespace base + +// LogDispatchCallback +void LogDispatchCallback::handle(const LogDispatchData* data) { +#if defined(ELPP_THREAD_SAFE) + base::threading::ScopedLock scopedLock(m_fileLocksMapLock); + std::string filename = data->logMessage()->logger()->typedConfigurations()->filename(data->logMessage()->level()); + auto lock = m_fileLocks.find(filename); + if (lock == m_fileLocks.end()) { + m_fileLocks.emplace(std::make_pair(filename, std::unique_ptr(new base::threading::Mutex))); + } +#endif +} + +base::threading::Mutex& LogDispatchCallback::fileHandle(const LogDispatchData* data) { + auto it = m_fileLocks.find(data->logMessage()->logger()->typedConfigurations()->filename(data->logMessage()->level())); + return *(it->second.get()); +} + +namespace base { // DefaultLogDispatchCallback +const char* convertToChar(Level level) { + // Do not use switch over strongly typed enums because Intel C++ compilers dont support them yet. + if (level == Level::Global) return "G"; + if (level == Level::Debug) return "D"; + if (level == Level::Info) return "I"; + if (level == Level::Warning) return "W"; + if (level == Level::Error) return "E"; + if (level == Level::Fatal) return "F"; + if (level == Level::Verbose) return "V"; + if (level == Level::Trace) return "T"; + return "?"; +} + void DefaultLogDispatchCallback::handle(const LogDispatchData* data) { +#if defined(ELPP_THREAD_SAFE) +#if 0 + LogDispatchCallback::handle(data); + base::threading::ScopedLock scopedLock(fileHandle(data)); +#endif +#endif m_data = data; - dispatch(m_data->logMessage()->logger()->logBuilder()->build(m_data->logMessage(), + base::TypedConfigurations* tc = m_data->logMessage()->logger()->typedConfigurations(); + const base::LogFormat* logFormat = &tc->logFormat(m_data->logMessage()->level()); + dispatch(base::utils::DateTime::getDateTime(logFormat->dateTimeFormat().c_str(), &tc->subsecondPrecision(m_data->logMessage()->level())) + + "\t" + convertToChar(m_data->logMessage()->level()) + " " + m_data->logMessage()->message() + "\n", + m_data->logMessage()->logger()->logBuilder()->build(m_data->logMessage(), m_data->dispatchAction() == base::DispatchAction::NormalLog || m_data->dispatchAction() == base::DispatchAction::FileOnlyLog)); } -void DefaultLogDispatchCallback::dispatch(base::type::string_t&& logLine) { +void DefaultLogDispatchCallback::dispatch(base::type::string_t&& rawLine, base::type::string_t&& logLine) { if (m_data->dispatchAction() == base::DispatchAction::NormalLog || m_data->dispatchAction() == base::DispatchAction::FileOnlyLog) { if (m_data->logMessage()->logger()->m_typedConfigurations->toFile(m_data->logMessage()->level())) { base::type::fstream_t* fs = m_data->logMessage()->logger()->m_typedConfigurations->fileStream( @@ -2236,8 +2398,8 @@ void DefaultLogDispatchCallback::dispatch(base::type::string_t&& logLine) { if (m_data->dispatchAction() != base::DispatchAction::FileOnlyLog) { if (m_data->logMessage()->logger()->m_typedConfigurations->toStandardOutput(m_data->logMessage()->level())) { if (ELPP->hasFlag(LoggingFlag::ColoredTerminalOutput)) - m_data->logMessage()->logger()->logBuilder()->convertToColoredOutput(&logLine, m_data->logMessage()->level()); - ELPP_COUT << ELPP_COUT_LINE(logLine); + m_data->logMessage()->logger()->logBuilder()->convertToColoredOutput(&rawLine, m_data->logMessage()->level()); + ELPP_COUT << ELPP_COUT_LINE(rawLine); } } } @@ -2454,6 +2616,8 @@ base::type::string_t DefaultLogBuilder::build(const LogMessage* logMessage, bool base::utils::Str::replaceFirstWithEscape(logLine, base::consts::kMessageFormatSpecifier, logMessage->message()); } #if !defined(ELPP_DISABLE_CUSTOM_FORMAT_SPECIFIERS) + el::base::threading::ScopedLock lock_(ELPP->customFormatSpecifiersLock()); + ELPP_UNUSED(lock_); for (std::vector::const_iterator it = ELPP->customFormatSpecifiers()->begin(); it != ELPP->customFormatSpecifiers()->end(); ++it) { std::string fs(it->formatSpecifier()); @@ -2474,10 +2638,15 @@ void LogDispatcher::dispatch(void) { if (!m_proceed) { return; } +#ifndef ELPP_NO_GLOBAL_LOCK + // see https://github.com/muflihun/easyloggingpp/issues/580 + // global lock is turned off by default unless + // ELPP_NO_GLOBAL_LOCK is defined base::threading::ScopedLock scopedLock(ELPP->lock()); - base::TypedConfigurations* tc = m_logMessage.logger()->m_typedConfigurations; +#endif + base::TypedConfigurations* tc = m_logMessage->logger()->m_typedConfigurations; if (ELPP->hasFlag(LoggingFlag::StrictLogFileSizeCheck)) { - tc->validateFileRolling(m_logMessage.level(), ELPP->preRollOutCallback()); + tc->validateFileRolling(m_logMessage->level(), ELPP->preRollOutCallback()); } LogDispatchCallback* callback = nullptr; LogDispatchData data; @@ -2485,7 +2654,7 @@ void LogDispatcher::dispatch(void) { : ELPP->m_logDispatchCallbacks) { callback = h.second.get(); if (callback != nullptr && callback->enabled()) { - data.setLogMessage(&m_logMessage); + data.setLogMessage(m_logMessage); data.setDispatchAction(m_dispatchAction); callback->handle(&data); } @@ -2532,6 +2701,7 @@ Writer& Writer::construct(int count, const char* loggerIds, ...) { va_list loggersList; va_start(loggersList, loggerIds); const char* id = loggerIds; + m_loggerIds.reserve(count); for (int i = 0; i < count; ++i) { m_loggerIds.push_back(std::string(id)); id = va_arg(loggersList, const char*); @@ -2545,17 +2715,23 @@ Writer& Writer::construct(int count, const char* loggerIds, ...) { return *this; } +Writer& Writer::construct(const char *loggerId) { + initializeLogger(ELPP->registeredLoggers()->get(loggerId, ELPP->hasFlag(LoggingFlag::CreateLoggerAutomatically))); + m_messageBuilder.initialize(m_logger); + return *this; +} + void Writer::initializeLogger(const std::string& loggerId, bool lookup, bool needLock) { if (lookup) { m_logger = ELPP->registeredLoggers()->get(loggerId, ELPP->hasFlag(LoggingFlag::CreateLoggerAutomatically)); } if (m_logger == nullptr) { - ELPP->acquireLock(); - if (!ELPP->registeredLoggers()->has(std::string(base::consts::kDefaultLoggerId))) { - // Somehow default logger has been unregistered. Not good! Register again - ELPP->registeredLoggers()->get(std::string(base::consts::kDefaultLoggerId)); + { + if (!ELPP->registeredLoggers()->has(std::string(base::consts::kDefaultLoggerId))) { + // Somehow default logger has been unregistered. Not good! Register again + ELPP->registeredLoggers()->get(std::string(base::consts::kDefaultLoggerId)); + } } - ELPP->releaseLock(); // Need to unlock it for next writer Writer(Level::Debug, m_file, m_line, m_func).construct(1, base::consts::kDefaultLoggerId) << "Logger [" << loggerId << "] is not registered yet!"; m_proceed = false; @@ -2566,13 +2742,26 @@ void Writer::initializeLogger(const std::string& loggerId, bool lookup, bool nee } if (ELPP->hasFlag(LoggingFlag::HierarchicalLogging)) { m_proceed = m_level == Level::Verbose ? m_logger->enabled(m_level) : - ELPP->vRegistry()->allowed(m_level, loggerId.c_str()); + ELPP->vRegistry()->allowed(m_level, loggerId); } else { m_proceed = m_logger->enabled(m_level); } } } +void Writer::initializeLogger(Logger *logger, bool needLock) { + m_logger = logger; + if (m_logger == nullptr) { + m_proceed = false; + } else { + if (needLock) { + m_logger->acquireLock(); // This should not be unlocked by checking m_proceed because + // m_proceed can be changed by lines below + } + m_proceed = true; + } +} + void Writer::processDispatch() { #if ELPP_LOGGING_ENABLED if (ELPP->hasFlag(LoggingFlag::MultiLoggerSupport)) { @@ -2616,8 +2805,13 @@ void Writer::processDispatch() { void Writer::triggerDispatch(void) { if (m_proceed) { - base::LogDispatcher(m_proceed, LogMessage(m_level, m_file, m_line, m_func, m_verboseLevel, - m_logger), m_dispatchAction).dispatch(); + if (m_msg == nullptr) { + LogMessage msg(m_level, m_file, m_line, m_func, m_verboseLevel, + m_logger); + base::LogDispatcher(m_proceed, &msg, m_dispatchAction).dispatch(); + } else { + base::LogDispatcher(m_proceed, m_msg, m_dispatchAction).dispatch(); + } } if (m_logger != nullptr) { m_logger->stream().str(ELPP_LITERAL("")); @@ -2630,7 +2824,7 @@ void Writer::triggerDispatch(void) { std::stringstream reasonStream; reasonStream << "Fatal log at [" << m_file << ":" << m_line << "]" << " If you wish to disable 'abort on fatal log' please use " - << "el::Helpers::addFlag(el::LoggingFlag::DisableApplicationAbortOnFatalLog)"; + << "el::Loggers::addFlag(el::LoggingFlag::DisableApplicationAbortOnFatalLog)"; base::utils::abort(1, reasonStream.str()); } m_proceed = false; @@ -2746,18 +2940,19 @@ namespace debug { // StackTrace -StackTrace::StackTraceEntry::StackTraceEntry(std::size_t index, const char* loc, const char* demang, const char* hex, - const char* addr) { - m_index = index; - m_location = std::string(loc); - m_demangled = std::string(demang); - m_hex = std::string(hex); - m_addr = std::string(addr); +StackTrace::StackTraceEntry::StackTraceEntry(std::size_t index, const std::string& loc, const std::string& demang, + const std::string& hex, + const std::string& addr) : + m_index(index), + m_location(loc), + m_demangled(demang), + m_hex(hex), + m_addr(addr) { } std::ostream& operator<<(std::ostream& ss, const StackTrace::StackTraceEntry& si) { - ss << "[" << si.m_index << "] " << si.m_location << (si.m_demangled.empty() ? "" : ":") << si.m_demangled - << (si.m_hex.empty() ? "" : "+") << si.m_hex << si.m_addr; + ss << "[" << si.m_index << "] " << si.m_location << (si.m_hex.empty() ? "" : "+") << si.m_hex << " " << si.m_addr << + (si.m_demangled.empty() ? "" : ":") << si.m_demangled; return ss; } @@ -2777,44 +2972,40 @@ void StackTrace::generateNew(void) { char** strings = backtrace_symbols(stack, size); if (size > kStackStart) { // Skip StackTrace c'tor and generateNew for (std::size_t i = kStackStart; i < size; ++i) { - char* mangName = nullptr; - char* hex = nullptr; - char* addr = nullptr; - for (char* c = strings[i]; *c; ++c) { - switch (*c) { - case '(': - mangName = c; - break; - case '+': - hex = c; - break; - case ')': - addr = c; - break; - default: - break; - } + std::string mangName; + std::string location; + std::string hex; + std::string addr; + + // entry: 2 crash.cpp.bin 0x0000000101552be5 _ZN2el4base5debug10StackTraceC1Ev + 21 + const std::string line(strings[i]); + auto p = line.find("_"); + if (p != std::string::npos) { + mangName = line.substr(p); + mangName = mangName.substr(0, mangName.find(" +")); + } + p = line.find("0x"); + if (p != std::string::npos) { + addr = line.substr(p); + addr = addr.substr(0, addr.find("_")); } // Perform demangling if parsed properly - if (mangName != nullptr && hex != nullptr && addr != nullptr && mangName < hex) { - *mangName++ = '\0'; - *hex++ = '\0'; - *addr++ = '\0'; + if (!mangName.empty()) { int status = 0; - char* demangName = abi::__cxa_demangle(mangName, 0, 0, &status); + char* demangName = abi::__cxa_demangle(mangName.data(), 0, 0, &status); // if demangling is successful, output the demangled function name if (status == 0) { // Success (see http://gcc.gnu.org/onlinedocs/libstdc++/libstdc++-html-USERS-4.3/a01696.html) - StackTraceEntry entry(i - 1, strings[i], demangName, hex, addr); + StackTraceEntry entry(i - 1, location, demangName, hex, addr); m_stack.push_back(entry); } else { // Not successful - we will use mangled name - StackTraceEntry entry(i - 1, strings[i], mangName, hex, addr); + StackTraceEntry entry(i - 1, location, mangName, hex, addr); m_stack.push_back(entry); } free(demangName); } else { - StackTraceEntry entry(i - 1, strings[i]); + StackTraceEntry entry(i - 1, line); m_stack.push_back(entry); } } @@ -2848,6 +3039,9 @@ static std::string crashReason(int sig) { } /// @brief Logs reason of crash from sig static void logCrashReason(int sig, bool stackTraceIfAvailable, Level level, const char* logger) { + if (sig == SIGINT && ELPP->hasFlag(el::LoggingFlag::IgnoreSigInt)) { + return; + } std::stringstream ss; ss << "CRASH HANDLED; "; ss << crashReason(sig); @@ -2926,7 +3120,6 @@ void Helpers::logCrashReason(int sig, bool stackTraceIfAvailable, Level level, c // Loggers Logger* Loggers::getLogger(const std::string& identity, bool registerIfNotAvailable) { - base::threading::ScopedLock scopedLock(ELPP->lock()); return ELPP->registeredLoggers()->get(identity, registerIfNotAvailable); } @@ -2935,12 +3128,10 @@ void Loggers::setDefaultLogBuilder(el::LogBuilderPtr& logBuilderPtr) { } bool Loggers::unregisterLogger(const std::string& identity) { - base::threading::ScopedLock scopedLock(ELPP->lock()); return ELPP->registeredLoggers()->remove(identity); } bool Loggers::hasLogger(const std::string& identity) { - base::threading::ScopedLock scopedLock(ELPP->lock()); return ELPP->registeredLoggers()->has(identity); } @@ -3110,11 +3301,11 @@ const std::string &Loggers::getFilenameCommonPrefix() { // VersionInfo const std::string VersionInfo::version(void) { - return std::string("9.94.1"); + return std::string("9.96.7"); } /// @brief Release date of current version const std::string VersionInfo::releaseDate(void) { - return std::string("25-02-2017 0813hrs"); + return std::string("24-11-2018 0728hrs"); } } // namespace el diff --git a/external/easylogging++/easylogging++.h b/external/easylogging++/easylogging++.h index 3270bd6074a..f0d8d5df7bf 100644 --- a/external/easylogging++/easylogging++.h +++ b/external/easylogging++/easylogging++.h @@ -1,18 +1,19 @@ // // Bismillah ar-Rahmaan ar-Raheem // -// Easylogging++ v9.94.1 +// Easylogging++ v9.96.7 // Single-header only, cross-platform logging library for C++ applications // -// Copyright (c) 2017 muflihun.com +// Copyright (c) 2012-2018 Zuhd Web Services +// Copyright (c) 2012-2018 @abumusamq // // This library is released under the MIT Licence. -// http://labs.muflihun.com/easyloggingpp/licence.php +// https://github.com/zuhd-org/easyloggingpp/blob/master/LICENSE // -// https://github.com/muflihun/easyloggingpp -// https://muflihun.github.io/easyloggingpp +// https://zuhd.org // http://muflihun.com // + #ifndef EASYLOGGINGPP_H #define EASYLOGGINGPP_H #include "ea_config.h" @@ -94,7 +95,7 @@ #else # define ELPP_OS_MAC 0 #endif -#if (defined(__FreeBSD__)) +#if (defined(__FreeBSD__) || defined(__FreeBSD_kernel__)) # define ELPP_OS_FREEBSD 1 #else # define ELPP_OS_FREEBSD 0 @@ -104,18 +105,33 @@ #else # define ELPP_OS_OPENBSD 0 #endif +#if (defined(__NetBSD__)) +# define ELPP_OS_NETBSD 1 +#else +# define ELPP_OS_NETBSD 0 +#endif #if (defined(__sun)) # define ELPP_OS_SOLARIS 1 #else # define ELPP_OS_SOLARIS 0 #endif +#if (defined(_AIX)) +# define ELPP_OS_AIX 1 +#else +# define ELPP_OS_AIX 0 +#endif +#if (defined(__NetBSD__)) +# define ELPP_OS_NETBSD 1 +#else +# define ELPP_OS_NETBSD 0 +#endif #if (defined(__DragonFly__)) # define ELPP_OS_DRAGONFLY 1 #else # define ELPP_OS_DRAGONFLY 0 #endif // Unix -#if ((ELPP_OS_LINUX || ELPP_OS_MAC || ELPP_OS_FREEBSD || ELPP_OS_SOLARIS || ELPP_OS_DRAGONFLY || ELPP_OS_OPENBSD) && (!ELPP_OS_WINDOWS)) +#if ((ELPP_OS_LINUX || ELPP_OS_MAC || ELPP_OS_FREEBSD || ELPP_OS_NETBSD || ELPP_OS_SOLARIS || ELPP_OS_AIX || ELPP_OS_DRAGONFLY || ELPP_OS_OPENBSD) && (!ELPP_OS_WINDOWS)) # define ELPP_OS_UNIX 1 #else # define ELPP_OS_UNIX 0 @@ -200,7 +216,7 @@ ELPP_INTERNAL_DEBUGGING_OUT_INFO << ELPP_INTERNAL_DEBUGGING_MSG(internalInfoStre # define ELPP_INTERNAL_INFO(lvl, msg) #endif // (defined(ELPP_DEBUG_INFO)) #if (defined(ELPP_FEATURE_ALL)) || (defined(ELPP_FEATURE_CRASH_LOG)) -# if (ELPP_COMPILER_GCC && !ELPP_MINGW && !ELPP_OS_OPENBSD) +# if (ELPP_COMPILER_GCC && !ELPP_MINGW && !ELPP_OS_OPENBSD && !ELPP_OS_NETBSD && !ELPP_OS_ANDROID) # define ELPP_STACKTRACE 1 # else # define ELPP_STACKTRACE 0 @@ -381,7 +397,7 @@ ELPP_INTERNAL_DEBUGGING_OUT_INFO << ELPP_INTERNAL_DEBUGGING_MSG(internalInfoStre #include #include #include -#include +#include #include #include #include @@ -419,9 +435,6 @@ ELPP_INTERNAL_DEBUGGING_OUT_INFO << ELPP_INTERNAL_DEBUGGING_MSG(internalInfoStre # if defined(ELPP_LOG_STD_ARRAY) # include # endif // defined(ELPP_LOG_STD_ARRAY) -# if defined(ELPP_LOG_UNORDERED_MAP) -# include -# endif // defined(ELPP_LOG_UNORDERED_MAP) # if defined(ELPP_LOG_UNORDERED_SET) # include # endif // defined(ELPP_UNORDERED_SET) @@ -456,6 +469,15 @@ ELPP_INTERNAL_DEBUGGING_OUT_INFO << ELPP_INTERNAL_DEBUGGING_MSG(internalInfoStre // For logging wxWidgets based classes & templates # include #endif // defined(ELPP_WXWIDGETS_LOGGING) +#if defined(ELPP_UTC_DATETIME) +# define elpptime_r gmtime_r +# define elpptime_s gmtime_s +# define elpptime gmtime +#else +# define elpptime_r localtime_r +# define elpptime_s localtime_s +# define elpptime localtime +#endif // defined(ELPP_UTC_DATETIME) // Forward declarations namespace el { class Logger; @@ -529,7 +551,7 @@ typedef std::ostream ostream_t; typedef unsigned int EnumType; typedef unsigned short VerboseLevel; typedef unsigned long int LineNumber; -typedef std::shared_ptr StoragePointer; +typedef base::Storage *StoragePointer; typedef std::shared_ptr LogDispatchCallbackPtr; typedef std::shared_ptr PerformanceTrackingCallbackPtr; typedef std::shared_ptr LoggerRegistrationCallbackPtr; @@ -580,6 +602,16 @@ enum class Level : base::type::EnumType { /// @brief Represents unknown level Unknown = 1010 }; +} // namespace el +namespace std { +template<> struct hash { + public: + std::size_t operator()(const el::Level& l) const { + return hash {}(static_cast(l)); + } +}; +} +namespace el { /// @brief Static class that contains helper functions for el::Level class LevelHelper : base::StaticClass { public: @@ -706,113 +738,43 @@ enum class LoggingFlag : base::type::EnumType { /// @brief Adds spaces b/w logs that separated by left-shift operator AutoSpacing = 8192, /// @brief Preserves time format and does not convert it to sec, hour etc (performance tracking only) - FixedTimeFormat = 16384 + FixedTimeFormat = 16384, + // @brief Ignore SIGINT or crash + IgnoreSigInt = 32768, }; namespace base { /// @brief Namespace containing constants used internally. namespace consts { -#if defined(__GNUC__) && !defined(__clang__) -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wunused-variable" -#endif -// Level log values - These are values that are replaced in place of %level format specifier -static const base::type::char_t* kInfoLevelLogValue = ELPP_LITERAL("INFO "); -static const base::type::char_t* kDebugLevelLogValue = ELPP_LITERAL("DEBUG"); -static const base::type::char_t* kWarningLevelLogValue = ELPP_LITERAL("WARN "); -static const base::type::char_t* kErrorLevelLogValue = ELPP_LITERAL("ERROR"); -static const base::type::char_t* kFatalLevelLogValue = ELPP_LITERAL("FATAL"); -static const base::type::char_t* kVerboseLevelLogValue = ELPP_LITERAL("VER"); -static const base::type::char_t* kTraceLevelLogValue = ELPP_LITERAL("TRACE"); -static const base::type::char_t* kInfoLevelShortLogValue = ELPP_LITERAL("I"); -static const base::type::char_t* kDebugLevelShortLogValue = ELPP_LITERAL("D"); -static const base::type::char_t* kWarningLevelShortLogValue = ELPP_LITERAL("W"); -static const base::type::char_t* kErrorLevelShortLogValue = ELPP_LITERAL("E"); -static const base::type::char_t* kFatalLevelShortLogValue = ELPP_LITERAL("F"); -static const base::type::char_t* kVerboseLevelShortLogValue = ELPP_LITERAL("V"); -static const base::type::char_t* kTraceLevelShortLogValue = ELPP_LITERAL("T"); -// Format specifiers - These are used to define log format -static const base::type::char_t* kAppNameFormatSpecifier = ELPP_LITERAL("%app"); -static const base::type::char_t* kLoggerIdFormatSpecifier = ELPP_LITERAL("%logger"); -static const base::type::char_t* kThreadIdFormatSpecifier = ELPP_LITERAL("%thread"); -static const base::type::char_t* kSeverityLevelFormatSpecifier = ELPP_LITERAL("%level"); -static const base::type::char_t* kSeverityLevelShortFormatSpecifier = ELPP_LITERAL("%levshort"); -static const base::type::char_t* kDateTimeFormatSpecifier = ELPP_LITERAL("%datetime"); -static const base::type::char_t* kLogFileFormatSpecifier = ELPP_LITERAL("%file"); -static const base::type::char_t* kLogFileBaseFormatSpecifier = ELPP_LITERAL("%fbase"); -static const base::type::char_t* kLogLineFormatSpecifier = ELPP_LITERAL("%line"); -static const base::type::char_t* kLogLocationFormatSpecifier = ELPP_LITERAL("%loc"); -static const base::type::char_t* kLogFunctionFormatSpecifier = ELPP_LITERAL("%func"); -static const base::type::char_t* kCurrentUserFormatSpecifier = ELPP_LITERAL("%user"); -static const base::type::char_t* kCurrentHostFormatSpecifier = ELPP_LITERAL("%host"); -static const base::type::char_t* kMessageFormatSpecifier = ELPP_LITERAL("%msg"); -static const base::type::char_t* kVerboseLevelFormatSpecifier = ELPP_LITERAL("%vlevel"); -static const char* kDateTimeFormatSpecifierForFilename = "%datetime"; -// Date/time -static const char* kDays[7] = { "Sunday", "Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday" }; -static const char* kDaysAbbrev[7] = { "Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat" }; -static const char* kMonths[12] = { "January", "February", "March", "Apri", "May", "June", "July", "August", - "September", "October", "November", "December" - }; -static const char* kMonthsAbbrev[12] = { "Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec" }; -static const char* kDefaultDateTimeFormat = "%Y-%M-%d %H:%m:%s,%g"; -static const char* kDefaultDateTimeFormatInFilename = "%Y-%M-%d_%H-%m"; -static const int kYearBase = 1900; -static const char* kAm = "AM"; -static const char* kPm = "PM"; -// Miscellaneous constants +static const char kFormatSpecifierCharValue = 'v'; +static const char kFormatSpecifierChar = '%'; +static const unsigned int kMaxLogPerCounter = 100000; +static const unsigned int kMaxLogPerContainer = 100; +static const unsigned int kDefaultSubsecondPrecision = 3; + #ifdef ELPP_DEFAULT_LOGGER static const char* kDefaultLoggerId = ELPP_DEFAULT_LOGGER; #else static const char* kDefaultLoggerId = "default"; #endif + +#if defined(ELPP_FEATURE_ALL) || defined(ELPP_FEATURE_PERFORMANCE_TRACKING) #ifdef ELPP_DEFAULT_PERFORMANCE_LOGGER static const char* kPerformanceLoggerId = ELPP_DEFAULT_PERFORMANCE_LOGGER; #else static const char* kPerformanceLoggerId = "performance"; +#endif // ELPP_DEFAULT_PERFORMANCE_LOGGER #endif + #if defined(ELPP_SYSLOG) static const char* kSysLogLoggerId = "syslog"; #endif // defined(ELPP_SYSLOG) -static const char* kNullPointer = "nullptr"; -static const char kFormatSpecifierChar = '%'; -#if ELPP_VARIADIC_TEMPLATES_SUPPORTED -static const char kFormatSpecifierCharValue = 'v'; -#endif // ELPP_VARIADIC_TEMPLATES_SUPPORTED -static const unsigned int kMaxLogPerContainer = 100; -static const unsigned int kMaxLogPerCounter = 100000; -static const unsigned int kDefaultSubsecondPrecision = 3; -static const base::type::VerboseLevel kMaxVerboseLevel = 9; -static const char* kUnknownUser = "user"; -static const char* kUnknownHost = "unknown-host"; -#if defined(ELPP_DEFAULT_LOG_FILE) -static const char* kDefaultLogFile = ELPP_DEFAULT_LOG_FILE; -#else -# if ELPP_OS_UNIX -# if ELPP_OS_ANDROID -static const char* kDefaultLogFile = "logs/myeasylog.log"; -# else -static const char* kDefaultLogFile = "logs/myeasylog.log"; -# endif // ELPP_OS_ANDROID -# elif ELPP_OS_WINDOWS -static const char* kDefaultLogFile = "logs\\myeasylog.log"; -# endif // ELPP_OS_UNIX -#endif // defined(ELPP_DEFAULT_LOG_FILE) -#if !defined(ELPP_DISABLE_LOG_FILE_FROM_ARG) -static const char* kDefaultLogFileParam = "--default-log-file"; -#endif // !defined(ELPP_DISABLE_LOG_FILE_FROM_ARG) -#if defined(ELPP_LOGGING_FLAGS_FROM_ARG) -static const char* kLoggingFlagsParam = "--logging-flags"; -#endif // defined(ELPP_LOGGING_FLAGS_FROM_ARG) + #if ELPP_OS_WINDOWS static const char* kFilePathSeperator = "\\"; #else static const char* kFilePathSeperator = "/"; #endif // ELPP_OS_WINDOWS -static const char* kValidLoggerIdSymbols = - "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-._"; -static const char* kConfigurationComment = "##"; -static const char* kConfigurationLevel = "*"; -static const char* kConfigurationLoggerId = "--"; + static const std::size_t kSourceFilenameMaxLength = 100; static const std::size_t kSourceLineMaxLength = 10; static const Level kPerformanceTrackerDefaultLevel = Level::Info; @@ -857,9 +819,6 @@ const struct { }, }; static const int kCrashSignalsCount = sizeof(kCrashSignals) / sizeof(kCrashSignals[0]); -#if defined(__GNUC__) && !defined(__clang__) -#pragma GCC diagnostic pop -#endif } // namespace consts } // namespace base typedef std::function PreRollOutCallback; @@ -1266,8 +1225,8 @@ class DateTime : base::StaticClass { base::TimestampUnit timestampUnit); - private: static struct ::tm* buildTimeInfo(struct timeval* currTime, struct ::tm* timeInfo); + private: static char* parseFormat(char* buf, std::size_t bufSz, const char* format, const struct tm* tInfo, std::size_t msec, const base::SubsecondPrecision* ssPrec); }; @@ -1306,7 +1265,7 @@ class CommandLineArgs { private: int m_argc; char** m_argv; - std::map m_paramsWithValue; + std::unordered_map m_paramsWithValue; std::vector m_params; }; /// @brief Abstract registry (aka repository) that provides basic interface for pointer repository specified by T_Ptr type. @@ -1431,7 +1390,7 @@ class AbstractRegistry : public base::threading::ThreadSafe { /// of AbstractRegistry. Any implementation of this class should be /// explicitly (by using lock functions) template -class Registry : public AbstractRegistry> { +class Registry : public AbstractRegistry> { public: typedef typename Registry::iterator iterator; typedef typename Registry::const_iterator const_iterator; @@ -1481,8 +1440,8 @@ class Registry : public AbstractRegistry> { void unregister(const T_Key& uniqKey) { T_Ptr* existing = get(uniqKey); if (existing != nullptr) { - base::utils::safeDelete(existing); this->list().erase(uniqKey); + base::utils::safeDelete(existing); } } @@ -1495,7 +1454,7 @@ class Registry : public AbstractRegistry> { } private: - virtual void deepCopy(const AbstractRegistry>& sr) ELPP_FINAL { + virtual void deepCopy(const AbstractRegistry>& sr) ELPP_FINAL { for (const_iterator it = sr.cbegin(); it != sr.cend(); ++it) { registerNew(it->first, new T_Ptr(*it->second)); } @@ -1595,7 +1554,7 @@ class RegistryWithPred : public AbstractRegistry> { class Utils { public: template - static bool installCallback(const std::string& id, std::map* mapT) { + static bool installCallback(const std::string& id, std::unordered_map* mapT) { if (mapT->find(id) == mapT->end()) { mapT->insert(std::make_pair(id, TPtr(new T()))); return true; @@ -1604,15 +1563,15 @@ class Utils { } template - static void uninstallCallback(const std::string& id, std::map* mapT) { + static void uninstallCallback(const std::string& id, std::unordered_map* mapT) { if (mapT->find(id) != mapT->end()) { mapT->erase(id); } } template - static T* callback(const std::string& id, std::map* mapT) { - typename std::map::iterator iter = mapT->find(id); + static T* callback(const std::string& id, std::unordered_map* mapT) { + typename std::unordered_map::iterator iter = mapT->find(id); if (iter != mapT->end()) { return static_cast(iter->second.get()); } @@ -1957,7 +1916,7 @@ class Configurations : public base::utils::RegistryWithPred FileStreamPtr; -typedef std::map LogStreamsReferenceMap; +typedef std::unordered_map LogStreamsReferenceMap; /// @brief Configurations with data types. /// /// @detail el::Configurations have string based values. This is whats used internally in order to read correct configurations. @@ -1994,16 +1953,16 @@ class TypedConfigurations : public base::threading::ThreadSafe { private: Configurations* m_configurations; - std::map m_enabledMap; - std::map m_toFileMap; - std::map m_filenameMap; - std::map m_toStandardOutputMap; - std::map m_logFormatMap; - std::map m_subsecondPrecisionMap; - std::map m_performanceTrackingMap; - std::map m_fileStreamMap; - std::map m_maxLogFileSizeMap; - std::map m_logFlushThresholdMap; + std::unordered_map m_enabledMap; + std::unordered_map m_toFileMap; + std::unordered_map m_filenameMap; + std::unordered_map m_toStandardOutputMap; + std::unordered_map m_logFormatMap; + std::unordered_map m_subsecondPrecisionMap; + std::unordered_map m_performanceTrackingMap; + std::unordered_map m_fileStreamMap; + std::unordered_map m_maxLogFileSizeMap; + std::unordered_map m_logFlushThresholdMap; base::LogStreamsReferenceMap* m_logStreamsReference; friend class el::Helpers; @@ -2013,21 +1972,21 @@ class TypedConfigurations : public base::threading::ThreadSafe { friend class el::base::LogDispatcher; template - inline Conf_T getConfigByVal(Level level, const std::map* confMap, const char* confName) { + inline Conf_T getConfigByVal(Level level, const std::unordered_map* confMap, const char* confName) { base::threading::ScopedLock scopedLock(lock()); return unsafeGetConfigByVal(level, confMap, confName); // This is not unsafe anymore - mutex locked in scope } template - inline Conf_T& getConfigByRef(Level level, std::map* confMap, const char* confName) { + inline Conf_T& getConfigByRef(Level level, std::unordered_map* confMap, const char* confName) { base::threading::ScopedLock scopedLock(lock()); return unsafeGetConfigByRef(level, confMap, confName); // This is not unsafe anymore - mutex locked in scope } template - Conf_T unsafeGetConfigByVal(Level level, const std::map* confMap, const char* confName) { + Conf_T unsafeGetConfigByVal(Level level, const std::unordered_map* confMap, const char* confName) { ELPP_UNUSED(confName); - typename std::map::const_iterator it = confMap->find(level); + typename std::unordered_map::const_iterator it = confMap->find(level); if (it == confMap->end()) { try { return confMap->at(Level::Global); @@ -2042,9 +2001,9 @@ class TypedConfigurations : public base::threading::ThreadSafe { } template - Conf_T& unsafeGetConfigByRef(Level level, std::map* confMap, const char* confName) { + Conf_T& unsafeGetConfigByRef(Level level, std::unordered_map* confMap, const char* confName) { ELPP_UNUSED(confName); - typename std::map::iterator it = confMap->find(level); + typename std::unordered_map::iterator it = confMap->find(level); if (it == confMap->end()) { try { return confMap->at(Level::Global); @@ -2058,14 +2017,15 @@ class TypedConfigurations : public base::threading::ThreadSafe { } template - void setValue(Level level, const Conf_T& value, std::map* confMap, bool includeGlobalLevel = true) { + void setValue(Level level, const Conf_T& value, std::unordered_map* confMap, + bool includeGlobalLevel = true) { // If map is empty and we are allowed to add into generic level (Level::Global), do it! if (confMap->empty() && includeGlobalLevel) { confMap->insert(std::make_pair(Level::Global, value)); return; } // If same value exist in generic level already, dont add it to explicit level - typename std::map::iterator it = confMap->find(Level::Global); + typename std::unordered_map::iterator it = confMap->find(Level::Global); if (it != confMap->end() && it->second == value) { return; } @@ -2227,21 +2187,26 @@ class LogDispatchData { inline base::DispatchAction dispatchAction(void) const { return m_dispatchAction; } - private: - LogMessage* m_logMessage; - base::DispatchAction m_dispatchAction; - friend class base::LogDispatcher; - inline void setLogMessage(LogMessage* logMessage) { m_logMessage = logMessage; } inline void setDispatchAction(base::DispatchAction dispatchAction) { m_dispatchAction = dispatchAction; } + private: + LogMessage* m_logMessage; + base::DispatchAction m_dispatchAction; + friend class base::LogDispatcher; + }; class LogDispatchCallback : public Callback { + protected: + virtual void handle(const LogDispatchData* data); + base::threading::Mutex& fileHandle(const LogDispatchData* data); private: friend class base::LogDispatcher; + std::unordered_map> m_fileLocks; + base::threading::Mutex m_fileLocksMapLock; }; class PerformanceTrackingCallback : public Callback { private: @@ -2359,7 +2324,7 @@ inline void FUNCTION_NAME(const T&); std::string m_parentApplicationName; bool m_isConfigured; Configurations m_configurations; - std::map m_unflushedCount; + std::unordered_map m_unflushedCount; base::LogStreamsReferenceMap* m_logStreamsReference; LogBuilderPtr m_logBuilder; @@ -2465,7 +2430,7 @@ class RegisteredLoggers : public base::utils::Registry { LogBuilderPtr m_defaultLogBuilder; Configurations m_defaultConfigurations; base::LogStreamsReferenceMap m_logStreamsReference; - std::map m_loggerRegistrationCallbacks; + std::unordered_map m_loggerRegistrationCallbacks; friend class el::base::Storage; void unsafeFlushAll(void); @@ -2485,6 +2450,7 @@ class VRegistry : base::NoCopy, public base::threading::ThreadSafe { inline void clearCategories(void) { base::threading::ScopedLock scopedLock(lock()); m_categories.clear(); + m_cached_allowed_categories.clear(); } inline void clearModules(void) { @@ -2498,11 +2464,11 @@ class VRegistry : base::NoCopy, public base::threading::ThreadSafe { void setModules(const char* modules); - bool allowed(Level level, const char* category); + bool allowed(Level level, const std::string &category); bool allowed(base::type::VerboseLevel vlevel, const char* file); - inline const std::map& modules(void) const { + inline const std::unordered_map& modules(void) const { return m_modules; } @@ -2524,8 +2490,9 @@ class VRegistry : base::NoCopy, public base::threading::ThreadSafe { private: base::type::VerboseLevel m_level; base::type::EnumType* m_pFlags; - std::map m_modules; - std::deque> m_categories; + std::unordered_map m_modules; + std::vector> m_categories; + std::map m_cached_allowed_categories; std::string m_categoriesString; std::string m_filenameCommonPrefix; }; @@ -2711,6 +2678,10 @@ class Storage : base::NoCopy, public base::threading::ThreadSafe { return &m_customFormatSpecifiers; } + base::threading::Mutex& customFormatSpecifiersLock() { + return m_customFormatSpecifiersLock; + } + inline void setLoggingLevel(Level level) { m_loggingLevel = level; } @@ -2751,19 +2722,21 @@ class Storage : base::NoCopy, public base::threading::ThreadSafe { /// @brief Sets thread name for current thread. Requires std::thread inline void setThreadName(const std::string& name) { if (name.empty()) return; - base::threading::ScopedLock scopedLock(lock()); + base::threading::ScopedLock scopedLock(m_threadNamesLock); m_threadNames[base::threading::getCurrentThreadId()] = name; } inline std::string getThreadName(const std::string& threadId) { - base::threading::ScopedLock scopedLock(lock()); - std::map::const_iterator it = m_threadNames.find(threadId); + base::threading::ScopedLock scopedLock(m_threadNamesLock); + std::unordered_map::const_iterator it = m_threadNames.find(threadId); if (it == m_threadNames.end()) { return threadId; } return it->second; } + static el::base::type::StoragePointer &getELPP(); + private: base::RegisteredHitCounters* m_registeredHitCounters; base::RegisteredLoggers* m_registeredLoggers; @@ -2775,10 +2748,12 @@ class Storage : base::NoCopy, public base::threading::ThreadSafe { #endif // ELPP_ASYNC_LOGGING base::utils::CommandLineArgs m_commandLineArgs; PreRollOutCallback m_preRollOutCallback; - std::map m_logDispatchCallbacks; - std::map m_performanceTrackingCallbacks; - std::map m_threadNames; + std::unordered_map m_logDispatchCallbacks; + std::unordered_map m_performanceTrackingCallbacks; + std::unordered_map m_threadNames; std::vector m_customFormatSpecifiers; + base::threading::Mutex m_customFormatSpecifiersLock; + base::threading::Mutex m_threadNamesLock; Level m_loggingLevel; friend class el::Helpers; @@ -2796,13 +2771,13 @@ class Storage : base::NoCopy, public base::threading::ThreadSafe { } }; extern ELPP_EXPORT base::type::StoragePointer elStorage; -#define ELPP el::base::elStorage +#define ELPP el::base::Storage::getELPP() class DefaultLogDispatchCallback : public LogDispatchCallback { protected: void handle(const LogDispatchData* data); private: const LogDispatchData* m_data; - void dispatch(base::type::string_t&& logLine); + void dispatch(base::type::string_t&& rawLine, base::type::string_t&& logLine); }; #if ELPP_ASYNC_LOGGING class AsyncLogDispatchCallback : public LogDispatchCallback { @@ -2821,7 +2796,7 @@ class AsyncDispatchWorker : public base::IWorker, public base::threading::Thread void run(void); void setContinueRunning(bool value) { - base::threading::ScopedLock scopedLock(m_continueRunningMutex); + base::threading::ScopedLock scopedLock(m_continueRunningLock); m_continueRunning = value; } @@ -2831,7 +2806,7 @@ class AsyncDispatchWorker : public base::IWorker, public base::threading::Thread private: std::condition_variable cv; bool m_continueRunning; - base::threading::Mutex m_continueRunningMutex; + base::threading::Mutex m_continueRunningLock; }; #endif // ELPP_ASYNC_LOGGING } // namespace base @@ -2843,9 +2818,9 @@ class DefaultLogBuilder : public LogBuilder { /// @brief Dispatches log messages class LogDispatcher : base::NoCopy { public: - LogDispatcher(bool proceed, LogMessage&& logMessage, base::DispatchAction dispatchAction) : + LogDispatcher(bool proceed, LogMessage* logMessage, base::DispatchAction dispatchAction) : m_proceed(proceed), - m_logMessage(std::move(logMessage)), + m_logMessage(logMessage), m_dispatchAction(std::move(dispatchAction)) { } @@ -2853,7 +2828,7 @@ class LogDispatcher : base::NoCopy { private: bool m_proceed; - LogMessage m_logMessage; + LogMessage* m_logMessage; base::DispatchAction m_dispatchAction; }; #if defined(ELPP_STL_LOGGING) @@ -3266,10 +3241,15 @@ class Writer : base::NoCopy { Writer(Level level, const char* file, base::type::LineNumber line, const char* func, base::DispatchAction dispatchAction = base::DispatchAction::NormalLog, base::type::VerboseLevel verboseLevel = 0) : - m_level(level), m_file(file), m_line(line), m_func(func), m_verboseLevel(verboseLevel), + m_msg(nullptr), m_level(level), m_file(file), m_line(line), m_func(func), m_verboseLevel(verboseLevel), m_logger(nullptr), m_proceed(false), m_dispatchAction(dispatchAction) { } + Writer(LogMessage* msg, base::DispatchAction dispatchAction = base::DispatchAction::NormalLog) : + m_msg(msg), m_level(msg != nullptr ? msg->level() : Level::Unknown), + m_line(0), m_logger(nullptr), m_proceed(false), m_dispatchAction(dispatchAction) { + } + virtual ~Writer(void) { processDispatch(); } @@ -3311,7 +3291,9 @@ class Writer : base::NoCopy { Writer& construct(Logger* logger, bool needLock = true); Writer& construct(int count, const char* loggerIds, ...); + Writer& construct(const char *loggerId); protected: + LogMessage* m_msg; Level m_level; const char* m_file; const base::type::LineNumber m_line; @@ -3325,6 +3307,7 @@ class Writer : base::NoCopy { friend class el::Helpers; void initializeLogger(const std::string& loggerId, bool lookup = true, bool needLock = true); + void initializeLogger(Logger *logger, bool needLock = true); void processDispatch(); void triggerDispatch(void); }; @@ -3370,6 +3353,7 @@ void Logger::log_(Level level, int vlevel, const T& log) { base::DispatchAction::NormalLog, vlevel).construct(this, false) << log; } else { stream().str(ELPP_LITERAL("")); + releaseLock(); } } else { base::Writer(level, "FILE", 0, "FUNCTION").construct(this, false) << log; @@ -3377,23 +3361,23 @@ void Logger::log_(Level level, int vlevel, const T& log) { } template inline void Logger::log(Level level, const char* s, const T& value, const Args&... args) { - base::threading::ScopedLock scopedLock(lock()); + acquireLock(); // released in Writer! log_(level, 0, s, value, args...); } template inline void Logger::log(Level level, const T& log) { - base::threading::ScopedLock scopedLock(lock()); + acquireLock(); // released in Writer! log_(level, 0, log); } # if ELPP_VERBOSE_LOG template inline void Logger::verbose(int vlevel, const char* s, const T& value, const Args&... args) { - base::threading::ScopedLock scopedLock(lock()); + acquireLock(); // released in Writer! log_(el::Level::Verbose, vlevel, s, value, args...); } template inline void Logger::verbose(int vlevel, const T& log) { - base::threading::ScopedLock scopedLock(lock()); + acquireLock(); // released in Writer! log_(el::Level::Verbose, vlevel, log); } # else @@ -3648,8 +3632,9 @@ class StackTrace : base::NoCopy { static const unsigned int kStackStart = 2; // We want to skip c'tor and StackTrace::generateNew() class StackTraceEntry { public: - StackTraceEntry(std::size_t index, const char* loc, const char* demang, const char* hex, const char* addr); - StackTraceEntry(std::size_t index, char* loc) : + StackTraceEntry(std::size_t index, const std::string& loc, const std::string& demang, const std::string& hex, + const std::string& addr); + StackTraceEntry(std::size_t index, const std::string& loc) : m_index(index), m_location(loc) { } @@ -3834,6 +3819,11 @@ class Helpers : base::StaticClass { static inline const el::base::utils::CommandLineArgs* commandLineArgs(void) { return ELPP->commandLineArgs(); } + /// @brief Reserve space for custom format specifiers for performance + /// @see std::vector::reserve + static inline void reserveCustomFormatSpecifiers(std::size_t size) { + ELPP->m_customFormatSpecifiers.reserve(size); + } /// @brief Installs user defined format specifier and handler static inline void installCustomFormatSpecifier(const CustomFormatSpecifier& customFormatSpecifier) { ELPP->installCustomFormatSpecifier(customFormatSpecifier); @@ -3847,7 +3837,7 @@ class Helpers : base::StaticClass { return ELPP->hasCustomFormatSpecifier(formatSpecifier); } static inline void validateFileRolling(Logger* logger, Level level) { - if (logger == nullptr) return; + if (ELPP == nullptr || logger == nullptr) return; logger->m_typedConfigurations->validateFileRolling(level, ELPP->preRollOutCallback()); } }; diff --git a/external/miniupnp b/external/miniupnp index 6a63f995495..4c700e09526 160000 --- a/external/miniupnp +++ b/external/miniupnp @@ -1 +1 @@ -Subproject commit 6a63f9954959119568fbc4af57d7b491b9428d87 +Subproject commit 4c700e09526a7d546394e85628c57e9490feefa0 diff --git a/external/rapidjson b/external/rapidjson index af223d44f4e..129d19ba7f4 160000 --- a/external/rapidjson +++ b/external/rapidjson @@ -1 +1 @@ -Subproject commit af223d44f4e8d3772cb1ac0ce8bc2a132b51717f +Subproject commit 129d19ba7f496df5e33658527a7158c79b99c21c diff --git a/external/trezor-common b/external/trezor-common new file mode 160000 index 00000000000..cb238cb1f13 --- /dev/null +++ b/external/trezor-common @@ -0,0 +1 @@ +Subproject commit cb238cb1f134accc4200217d9511115a8f61c6cb diff --git a/external/unbound b/external/unbound index 193bdc4ee3f..0f6c0579d66 160000 --- a/external/unbound +++ b/external/unbound @@ -1 +1 @@ -Subproject commit 193bdc4ee3fe2b0d17e547e86512528c2614483a +Subproject commit 0f6c0579d66b65f86066e30e7876105ba2775ef4 diff --git a/include/INode.h b/include/INode.h index 21be0b2f3d3..0d896ca69d5 100644 --- a/include/INode.h +++ b/include/INode.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/include/IWallet.h b/include/IWallet.h index 98110a9d666..98362895da3 100644 --- a/include/IWallet.h +++ b/include/IWallet.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 3b71c38cd22..da6d76d9732 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2014-2018, The Monero Project +# Copyright (c) 2014-2019, The Monero Project # # All rights reserved. # @@ -34,11 +34,6 @@ if (WIN32 OR STATIC) add_definitions(-DMINIUPNP_STATICLIB) endif () -# warnings are cleared only for GCC on Linux -if (NOT (MINGW OR APPLE OR FREEBSD OR OPENBSD OR DRAGONFLY)) - add_compile_options("${WARNINGS_AS_ERRORS_FLAG}") # applies only to targets that follow -endif() - function (monero_private_headers group) source_group("${group}\\Private" FILES @@ -114,7 +109,9 @@ add_subdirectory(ringct) add_subdirectory(checkpoints) add_subdirectory(cryptonote_basic) add_subdirectory(cryptonote_core) +add_subdirectory(lmdb) add_subdirectory(multisig) +add_subdirectory(net) if(NOT IOS) add_subdirectory(blockchain_db) endif() @@ -145,3 +142,4 @@ if(PER_BLOCK_CHECKPOINT) endif() add_subdirectory(device) +add_subdirectory(device_trezor) diff --git a/src/blockchain_db/CMakeLists.txt b/src/blockchain_db/CMakeLists.txt index 277f4458ef7..db161fa5e6e 100644 --- a/src/blockchain_db/CMakeLists.txt +++ b/src/blockchain_db/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2014-2018, The Monero Project +# Copyright (c) 2014-2019, The Monero Project # # All rights reserved. # diff --git a/src/blockchain_db/berkeleydb/db_bdb.cpp b/src/blockchain_db/berkeleydb/db_bdb.cpp index e1b76ec1ef7..d138a1e7edf 100644 --- a/src/blockchain_db/berkeleydb/db_bdb.cpp +++ b/src/blockchain_db/berkeleydb/db_bdb.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // All rights reserved. // // Redistribution and use in source and binary forms, with or without modification, are @@ -224,7 +224,7 @@ struct Dbt_safe : public Dbt namespace cryptonote { -void BlockchainBDB::add_block(const block& blk, const size_t& block_size, const difficulty_type& cumulative_difficulty, const uint64_t& coins_generated, const crypto::hash& blk_hash) +void BlockchainBDB::add_block(const block& blk, size_t block_weight, const difficulty_type& cumulative_difficulty, const uint64_t& coins_generated, const crypto::hash& blk_hash) { LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); @@ -255,7 +255,7 @@ void BlockchainBDB::add_block(const block& blk, const size_t& block_size, const if (res) throw0(DB_ERROR("Failed to add block blob to db transaction.")); - Dbt_copy sz(block_size); + Dbt_copy sz(block_weight); if (m_block_sizes->put(DB_DEFAULT_TX, &key, &sz, 0)) throw0(DB_ERROR("Failed to add block size to db transaction.")); @@ -714,29 +714,6 @@ bool BlockchainBDB::for_all_outputs(std::function ba(ss); - tx_out o; - - if (!(::serialization::serialize(ba, o))) - throw1(DB_ERROR("Error deserializing tx output blob")); - - return o; -} - uint64_t BlockchainBDB::get_output_global_index(const uint64_t& amount, const uint64_t& index) { LOG_PRINT_L3("BlockchainBDB::" << __func__); @@ -1213,6 +1190,11 @@ std::vector BlockchainBDB::get_filenames() const return full_paths; } +bool BlockchainBDB::remove_data_file(const std::string& folder) +{ + return true; +} + std::string BlockchainBDB::get_db_name() const { LOG_PRINT_L3("BlockchainBDB::" << __func__); @@ -1348,7 +1330,7 @@ uint64_t BlockchainBDB::get_top_block_timestamp() const return get_block_timestamp(m_height - 1); } -size_t BlockchainBDB::get_block_size(const uint64_t& height) const +size_t BlockchainBDB::get_block_weight(const uint64_t& height) const { LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); @@ -1650,7 +1632,7 @@ output_data_t BlockchainBDB::get_output_key(const uint64_t& global_index) const return v; } -output_data_t BlockchainBDB::get_output_key(const uint64_t& amount, const uint64_t& index) +output_data_t BlockchainBDB::get_output_key(const uint64_t& amount, const uint64_t& index) const { LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); @@ -1659,7 +1641,7 @@ output_data_t BlockchainBDB::get_output_key(const uint64_t& amount, const uint64 return get_output_key(glob_index); } -tx_out_index BlockchainBDB::get_output_tx_and_index(const uint64_t& amount, const uint64_t& index) +tx_out_index BlockchainBDB::get_output_tx_and_index(const uint64_t& amount, const uint64_t& index) const { LOG_PRINT_L3("BlockchainBDB::" << __func__); std::vector < uint64_t > offsets; @@ -1672,36 +1654,6 @@ tx_out_index BlockchainBDB::get_output_tx_and_index(const uint64_t& amount, cons return indices[0]; } -std::vector BlockchainBDB::get_tx_output_indices(const crypto::hash& h) const -{ - LOG_PRINT_L3("BlockchainBDB::" << __func__); - check_open(); - std::vector index_vec; - - bdb_cur cur(DB_DEFAULT_TX, m_tx_outputs); - - Dbt_copy k(h); - Dbt_copy v; - auto result = cur->get(&k, &v, DB_SET); - if (result == DB_NOTFOUND) - throw1(OUTPUT_DNE("Attempting to get an output by tx hash and tx index, but output not found")); - else if (result) - throw0(DB_ERROR("DB error attempting to get an output")); - - db_recno_t num_elems = 0; - cur->count(&num_elems, 0); - - for (uint64_t i = 0; i < num_elems; ++i) - { - index_vec.push_back(v); - cur->get(&k, &v, DB_NEXT_DUP); - } - - cur.close(); - - return index_vec; -} - std::vector BlockchainBDB::get_tx_amount_output_indices(const crypto::hash& h) const { LOG_PRINT_L3("BlockchainBDB::" << __func__); @@ -1856,7 +1808,7 @@ void BlockchainBDB::block_txn_abort() // TODO } -uint64_t BlockchainBDB::add_block(const block& blk, const size_t& block_size, const difficulty_type& cumulative_difficulty, const uint64_t& coins_generated, const std::vector& txs) +uint64_t BlockchainBDB::add_block(const block& blk, size_t block_weight, const difficulty_type& cumulative_difficulty, const uint64_t& coins_generated, const std::vector& txs) { LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); @@ -1869,7 +1821,7 @@ uint64_t BlockchainBDB::add_block(const block& blk, const size_t& block_size, co uint64_t num_outputs = m_num_outputs; try { - BlockchainDB::add_block(blk, block_size, cumulative_difficulty, coins_generated, txs); + BlockchainDB::add_block(blk, block_weight, cumulative_difficulty, coins_generated, txs); m_write_txn = NULL; TIME_MEASURE_START(time1); diff --git a/src/blockchain_db/berkeleydb/db_bdb.h b/src/blockchain_db/berkeleydb/db_bdb.h index cecbba28f80..3ae90efe1f0 100644 --- a/src/blockchain_db/berkeleydb/db_bdb.h +++ b/src/blockchain_db/berkeleydb/db_bdb.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // All rights reserved. // // Redistribution and use in source and binary forms, with or without modification, are @@ -244,6 +244,8 @@ class BlockchainBDB : public BlockchainDB virtual std::vector get_filenames() const; + virtual bool remove_data_file(const std::string& folder); + virtual std::string get_db_name() const; virtual bool lock(); @@ -264,7 +266,7 @@ class BlockchainBDB : public BlockchainDB virtual uint64_t get_top_block_timestamp() const; - virtual size_t get_block_size(const uint64_t& height) const; + virtual size_t get_block_weight(const uint64_t& height) const; virtual difficulty_type get_block_cumulative_difficulty(const uint64_t& height) const; @@ -301,7 +303,6 @@ class BlockchainBDB : public BlockchainDB virtual uint64_t get_indexing_base() const { return 1; } virtual output_data_t get_output_key(const uint64_t& amount, const uint64_t& index); - virtual output_data_t get_output_key(const uint64_t& global_index) const; virtual void get_output_key(const uint64_t &amount, const std::vector &offsets, std::vector &outputs); virtual tx_out_index get_output_tx_and_index_from_global(const uint64_t& index) const; @@ -311,13 +312,12 @@ class BlockchainBDB : public BlockchainDB virtual tx_out_index get_output_tx_and_index(const uint64_t& amount, const uint64_t& index); virtual void get_output_tx_and_index(const uint64_t& amount, const std::vector &offsets, std::vector &indices); - virtual std::vector get_tx_output_indices(const crypto::hash& h) const; virtual std::vector get_tx_amount_output_indices(const crypto::hash& h) const; virtual bool has_key_image(const crypto::key_image& img) const; virtual uint64_t add_block( const block& blk - , const size_t& block_size + , size_t block_weight , const difficulty_type& cumulative_difficulty , const uint64_t& coins_generated , const std::vector& txs @@ -352,7 +352,7 @@ class BlockchainBDB : public BlockchainDB private: virtual void add_block( const block& blk - , const size_t& block_size + , size_t block_weight , const difficulty_type& cumulative_difficulty , const uint64_t& coins_generated , const crypto::hash& block_hash @@ -390,24 +390,6 @@ class BlockchainBDB : public BlockchainDB virtual void check_hard_fork_info(); virtual void drop_hard_fork_info(); - /** - * @brief convert a tx output to a blob for storage - * - * @param output the output to convert - * - * @return the resultant blob - */ - blobdata output_to_blob(const tx_out& output) const; - - /** - * @brief convert a tx output blob to a tx output - * - * @param blob the blob to convert - * - * @return the resultant tx output - */ - tx_out output_from_blob(const blobdata& blob) const; - /** * @brief get the global index of the index-th output of the given amount * @@ -417,6 +399,7 @@ class BlockchainBDB : public BlockchainDB * @return the global index of the desired output */ uint64_t get_output_global_index(const uint64_t& amount, const uint64_t& index); + output_data_t get_output_key(const uint64_t& global_index) const; void checkpoint_worker() const; void check_open() const; diff --git a/src/blockchain_db/blockchain_db.cpp b/src/blockchain_db/blockchain_db.cpp index 88ac342557b..2b039f55702 100644 --- a/src/blockchain_db/blockchain_db.cpp +++ b/src/blockchain_db/blockchain_db.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -87,8 +87,8 @@ const command_line::arg_descriptor arg_db_type = { }; const command_line::arg_descriptor arg_db_sync_mode = { "db-sync-mode" -, "Specify sync option, using format [safe|fast|fastest]:[sync|async]:[nblocks_per_sync]." -, "fast:async:1000" +, "Specify sync option, using format [safe|fast|fastest]:[sync|async]:[[blocks]|[bytes]]." +, "fast:async:250000000bytes" }; const command_line::arg_descriptor arg_db_salvage = { "db-salvage" @@ -121,8 +121,10 @@ void BlockchainDB::pop_block() pop_block(blk, txs); } -void BlockchainDB::add_transaction(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash* tx_hash_ptr, const crypto::hash* tx_prunable_hash_ptr) +void BlockchainDB::add_transaction(const crypto::hash& blk_hash, const std::pair& txp, const crypto::hash* tx_hash_ptr, const crypto::hash* tx_prunable_hash_ptr) { + const transaction &tx = txp.first; + bool miner_tx = false; crypto::hash tx_hash, tx_prunable_hash; if (!tx_hash_ptr) @@ -138,7 +140,7 @@ void BlockchainDB::add_transaction(const crypto::hash& blk_hash, const transacti if (tx.version >= 2) { if (!tx_prunable_hash_ptr) - tx_prunable_hash = get_transaction_prunable_hash(tx); + tx_prunable_hash = get_transaction_prunable_hash(tx, &txp.second); else tx_prunable_hash = *tx_prunable_hash_ptr; } @@ -168,9 +170,9 @@ void BlockchainDB::add_transaction(const crypto::hash& blk_hash, const transacti } } - uint64_t tx_id = add_transaction_data(blk_hash, tx, tx_hash, tx_prunable_hash); + uint64_t tx_id = add_transaction_data(blk_hash, txp, tx_hash, tx_prunable_hash); - std::vector amount_output_indices; + std::vector amount_output_indices(tx.vout.size()); // iterate tx.vout using indices instead of C++11 foreach syntax because // we need the index @@ -183,31 +185,32 @@ void BlockchainDB::add_transaction(const crypto::hash& blk_hash, const transacti cryptonote::tx_out vout = tx.vout[i]; rct::key commitment = rct::zeroCommit(vout.amount); vout.amount = 0; - amount_output_indices.push_back(add_output(tx_hash, vout, i, tx.unlock_time, - &commitment)); + amount_output_indices[i] = add_output(tx_hash, vout, i, tx.unlock_time, + &commitment); } else { - amount_output_indices.push_back(add_output(tx_hash, tx.vout[i], i, tx.unlock_time, - tx.version > 1 ? &tx.rct_signatures.outPk[i].mask : NULL)); + amount_output_indices[i] = add_output(tx_hash, tx.vout[i], i, tx.unlock_time, + tx.version > 1 ? &tx.rct_signatures.outPk[i].mask : NULL); } } add_tx_amount_output_indices(tx_id, amount_output_indices); } -uint64_t BlockchainDB::add_block( const block& blk - , const size_t& block_size +uint64_t BlockchainDB::add_block( const std::pair& blck + , size_t block_weight + , uint64_t long_term_block_weight , const difficulty_type& cumulative_difficulty , const uint64_t& coins_generated - , const std::vector& txs + , const std::vector>& txs ) { + const block &blk = blck.first; + // sanity if (blk.tx_hashes.size() != txs.size()) throw std::runtime_error("Inconsistent tx/hashes sizes"); - block_txn_start(false); - TIME_MEASURE_START(time1); crypto::hash blk_hash = get_block_hash(blk); TIME_MEASURE_FINISH(time1); @@ -218,13 +221,22 @@ uint64_t BlockchainDB::add_block( const block& blk // call out to add the transactions time1 = epee::misc_utils::get_tick_count(); - add_transaction(blk_hash, blk.miner_tx); + + uint64_t num_rct_outs = 0; + add_transaction(blk_hash, std::make_pair(blk.miner_tx, tx_to_blob(blk.miner_tx))); + if (blk.miner_tx.version == 2) + num_rct_outs += blk.miner_tx.vout.size(); int tx_i = 0; crypto::hash tx_hash = crypto::null_hash; - for (const transaction& tx : txs) + for (const std::pair& tx : txs) { tx_hash = blk.tx_hashes[tx_i]; add_transaction(blk_hash, tx, &tx_hash); + for (const auto &vout: tx.first.vout) + { + if (vout.amount == 0) + ++num_rct_outs; + } ++tx_i; } TIME_MEASURE_FINISH(time1); @@ -232,14 +244,12 @@ uint64_t BlockchainDB::add_block( const block& blk // call out to subclass implementation to add the block & metadata time1 = epee::misc_utils::get_tick_count(); - add_block(blk, block_size, cumulative_difficulty, coins_generated, blk_hash); + add_block(blk, block_weight, long_term_block_weight, cumulative_difficulty, coins_generated, num_rct_outs, blk_hash); TIME_MEASURE_FINISH(time1); time_add_block1 += time1; m_hardfork->add(blk, prev_height); - block_txn_stop(); - ++num_calls; return prev_height; @@ -258,7 +268,10 @@ void BlockchainDB::pop_block(block& blk, std::vector& txs) for (const auto& h : boost::adaptors::reverse(blk.tx_hashes)) { - txs.push_back(get_tx(h)); + cryptonote::transaction tx; + if (!get_tx(h, tx) && !get_pruned_tx(h, tx)) + throw DB_ERROR("Failed to get pruned or unpruned transaction from the db"); + txs.push_back(std::move(tx)); remove_transaction(h); } remove_transaction(get_transaction_hash(blk.miner_tx)); @@ -271,7 +284,7 @@ bool BlockchainDB::is_open() const void BlockchainDB::remove_transaction(const crypto::hash& tx_hash) { - transaction tx = get_tx(tx_hash); + transaction tx = get_pruned_tx(tx_hash); for (const txin_v& tx_input : tx.vin) { @@ -316,6 +329,17 @@ bool BlockchainDB::get_tx(const crypto::hash& h, cryptonote::transaction &tx) co return true; } +bool BlockchainDB::get_pruned_tx(const crypto::hash& h, cryptonote::transaction &tx) const +{ + blobdata bd; + if (!get_pruned_tx_blob(h, bd)) + return false; + if (!parse_and_validate_tx_base_from_blob(bd, tx)) + throw DB_ERROR("Failed to parse transaction base from blob retrieved from the db"); + + return true; +} + transaction BlockchainDB::get_tx(const crypto::hash& h) const { transaction tx; @@ -324,6 +348,14 @@ transaction BlockchainDB::get_tx(const crypto::hash& h) const return tx; } +transaction BlockchainDB::get_pruned_tx(const crypto::hash& h) const +{ + transaction tx; + if (!get_pruned_tx(h, tx)) + throw TX_DNE(std::string("pruned tx with hash ").append(epee::string_tools::pod_to_hex(h)).append(" not found in db").c_str()); + return tx; +} + void BlockchainDB::reset_stats() { num_calls = 0; diff --git a/src/blockchain_db/blockchain_db.h b/src/blockchain_db/blockchain_db.h index 19ba3234085..b6b8c6c3e35 100644 --- a/src/blockchain_db/blockchain_db.h +++ b/src/blockchain_db/blockchain_db.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -30,7 +30,6 @@ #pragma once -#include #include #include #include @@ -137,7 +136,7 @@ struct txpool_tx_meta_t { crypto::hash max_used_block_id; crypto::hash last_failed_id; - uint64_t blob_size; + uint64_t weight; uint64_t fee; uint64_t max_used_block_height; uint64_t last_failed_height; @@ -148,6 +147,7 @@ struct txpool_tx_meta_t uint8_t relayed; uint8_t do_not_relay; uint8_t double_spend_seen: 1; + uint8_t bf_padding: 7; uint8_t padding[76]; // till 192 bytes }; @@ -357,15 +357,18 @@ class BlockchainDB * subclass of DB_EXCEPTION * * @param blk the block to be added - * @param block_size the size of the block (transactions and all) + * @param block_weight the weight of the block (transactions and all) + * @param long_term_block_weight the long term block weight of the block (transactions and all) * @param cumulative_difficulty the accumulated difficulty after this block * @param coins_generated the number of coins generated total after this block * @param blk_hash the hash of the block */ virtual void add_block( const block& blk - , const size_t& block_size + , size_t block_weight + , uint64_t long_term_block_weight , const difficulty_type& cumulative_difficulty , const uint64_t& coins_generated + , uint64_t num_rct_outs , const crypto::hash& blk_hash ) = 0; @@ -374,7 +377,7 @@ class BlockchainDB * * The subclass implementing this will remove the block data from the top * block in the chain. The data to be removed is that which was added in - * BlockchainDB::add_block(const block& blk, const size_t& block_size, const difficulty_type& cumulative_difficulty, const uint64_t& coins_generated, const crypto::hash& blk_hash) + * BlockchainDB::add_block(const block& blk, size_t block_weight, uint64_t long_term_block_weight, const difficulty_type& cumulative_difficulty, const uint64_t& coins_generated, const crypto::hash& blk_hash) * * If any of this cannot be done, the subclass should throw the corresponding * subclass of DB_EXCEPTION @@ -401,7 +404,7 @@ class BlockchainDB * @param tx_prunable_hash the hash of the prunable part of the transaction * @return the transaction ID */ - virtual uint64_t add_transaction_data(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash& tx_hash, const crypto::hash& tx_prunable_hash) = 0; + virtual uint64_t add_transaction_data(const crypto::hash& blk_hash, const std::pair& tx, const crypto::hash& tx_hash, const crypto::hash& tx_prunable_hash) = 0; /** * @brief remove data about a transaction @@ -529,7 +532,7 @@ class BlockchainDB * @param tx_hash_ptr the hash of the transaction, if already calculated * @param tx_prunable_hash_ptr the hash of the prunable part of the transaction, if already calculated */ - void add_transaction(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash* tx_hash_ptr = NULL, const crypto::hash* tx_prunable_hash_ptr = NULL); + void add_transaction(const crypto::hash& blk_hash, const std::pair& tx, const crypto::hash* tx_hash_ptr = NULL, const crypto::hash* tx_prunable_hash_ptr = NULL); mutable uint64_t time_tx_exists = 0; //!< a performance metric uint64_t time_commit1 = 0; //!< a performance metric @@ -542,7 +545,7 @@ class BlockchainDB /** * @brief An empty constructor. */ - BlockchainDB(): m_open(false) { } + BlockchainDB(): m_hardfork(NULL), m_open(false) { } /** * @brief An empty destructor. @@ -654,6 +657,20 @@ class BlockchainDB */ virtual std::vector get_filenames() const = 0; + /** + * @brief remove file(s) storing the database + * + * This function is for resetting the database (for core tests, functional tests, etc). + * The function reset() is not usable because it needs to open the database file first + * which can fail if the existing database file is in an incompatible format. + * As such, this function needs to be called before calling open(). + * + * @param folder The path of the folder containing the database file(s) which must not end with slash '/'. + * + * @return true if the operation is succesfull + */ + virtual bool remove_data_file(const std::string& folder) const = 0; + // return the name of the folder the db's file(s) should reside in /** * @brief gets the name of the folder the BlockchainDB's file(s) should be in @@ -736,6 +753,21 @@ class BlockchainDB */ virtual void batch_stop() = 0; + /** + * @brief aborts a batch transaction + * + * If the subclass implements batching, this function should abort the + * batch it is currently on. + * + * If no batch is in-progress, this function should throw a DB_ERROR. + * This exception may change in the future if it is deemed necessary to + * have a more granular exception type for this scenario. + * + * If any of this cannot be done, the subclass should throw the corresponding + * subclass of DB_EXCEPTION + */ + virtual void batch_abort() = 0; + /** * @brief sets whether or not to batch transactions * @@ -753,9 +785,12 @@ class BlockchainDB */ virtual void set_batch_transactions(bool) = 0; - virtual void block_txn_start(bool readonly=false) = 0; - virtual void block_txn_stop() = 0; - virtual void block_txn_abort() = 0; + virtual void block_wtxn_start() = 0; + virtual void block_wtxn_stop() = 0; + virtual void block_wtxn_abort() = 0; + virtual bool block_rtxn_start() const = 0; + virtual void block_rtxn_stop() const = 0; + virtual void block_rtxn_abort() const = 0; virtual void set_hard_fork(HardFork* hf); @@ -773,18 +808,20 @@ class BlockchainDB * subclass of DB_EXCEPTION * * @param blk the block to be added - * @param block_size the size of the block (transactions and all) + * @param block_weight the size of the block (transactions and all) + * @param long_term_block_weight the long term weight of the block (transactions and all) * @param cumulative_difficulty the accumulated difficulty after this block * @param coins_generated the number of coins generated total after this block * @param txs the transactions in the block * * @return the height of the chain post-addition */ - virtual uint64_t add_block( const block& blk - , const size_t& block_size + virtual uint64_t add_block( const std::pair& blk + , size_t block_weight + , uint64_t long_term_block_weight , const difficulty_type& cumulative_difficulty , const uint64_t& coins_generated - , const std::vector& txs + , const std::vector>& txs ); /** @@ -890,6 +927,20 @@ class BlockchainDB */ virtual uint64_t get_block_timestamp(const uint64_t& height) const = 0; + /** + * @brief fetch a block's cumulative number of rct outputs + * + * The subclass should return the numer of rct outputs in the blockchain + * up to the block with the given height (inclusive). + * + * If the block does not exist, the subclass should throw BLOCK_DNE + * + * @param height the height requested + * + * @return the cumulative number of rct outputs + */ + virtual std::vector get_block_cumulative_rct_outputs(const std::vector &heights) const = 0; + /** * @brief fetch the top block's timestamp * @@ -900,18 +951,29 @@ class BlockchainDB virtual uint64_t get_top_block_timestamp() const = 0; /** - * @brief fetch a block's size + * @brief fetch a block's weight * - * The subclass should return the size of the block with the + * The subclass should return the weight of the block with the * given height. * * If the block does not exist, the subclass should throw BLOCK_DNE * * @param height the height requested * - * @return the size + * @return the weight + */ + virtual size_t get_block_weight(const uint64_t& height) const = 0; + + /** + * @brief fetch the last N blocks' weights + * + * If there are fewer than N blocks, the returned array will be smaller than N + * + * @param count the number of blocks requested + * + * @return the weights */ - virtual size_t get_block_size(const uint64_t& height) const = 0; + virtual std::vector get_block_weights(uint64_t start_height, size_t count) const = 0; /** * @brief fetch a block's cumulative difficulty @@ -955,6 +1017,28 @@ class BlockchainDB */ virtual uint64_t get_block_already_generated_coins(const uint64_t& height) const = 0; + /** + * @brief fetch a block's long term weight + * + * If the block does not exist, the subclass should throw BLOCK_DNE + * + * @param height the height requested + * + * @return the long term weight + */ + virtual uint64_t get_block_long_term_weight(const uint64_t& height) const = 0; + + /** + * @brief fetch the last N blocks' long term weights + * + * If there are fewer than N blocks, the returned array will be smaller than N + * + * @param count the number of blocks requested + * + * @return the weights + */ + virtual std::vector get_long_term_block_weights(uint64_t start_height, size_t count) const = 0; + /** * @brief fetch a block's hash * @@ -1008,9 +1092,11 @@ class BlockchainDB * * The subclass should return the hash of the most recent block * + * @param block_height if non NULL, returns the height of that block (ie, the blockchain height minus 1) + * * @return the top block's hash */ - virtual crypto::hash top_block_hash() const = 0; + virtual crypto::hash top_block_hash(uint64_t *block_height = NULL) const = 0; /** * @brief fetch the top block @@ -1096,6 +1182,17 @@ class BlockchainDB */ virtual transaction get_tx(const crypto::hash& h) const; + /** + * @brief fetches the transaction base with the given hash + * + * If the transaction does not exist, the subclass should throw TX_DNE. + * + * @param h the hash to look for + * + * @return the transaction with the given hash + */ + virtual transaction get_pruned_tx(const crypto::hash& h) const; + /** * @brief fetches the transaction with the given hash * @@ -1107,6 +1204,17 @@ class BlockchainDB */ virtual bool get_tx(const crypto::hash& h, transaction &tx) const; + /** + * @brief fetches the transaction base with the given hash + * + * If the transaction does not exist, the subclass should return false. + * + * @param h the hash to look for + * + * @return true iff the transaction was found + */ + virtual bool get_pruned_tx(const crypto::hash& h, transaction &tx) const; + /** * @brief fetches the transaction blob with the given hash * @@ -1135,6 +1243,21 @@ class BlockchainDB */ virtual bool get_pruned_tx_blob(const crypto::hash& h, cryptonote::blobdata &tx) const = 0; + /** + * @brief fetches the prunable transaction blob with the given hash + * + * The subclass should return the prunable transaction stored which has the given + * hash. + * + * If the transaction does not exist, or if we do not have that prunable data, + * the subclass should return false. + * + * @param h the hash to look for + * + * @return true iff the transaction was found and we have its prunable data + */ + virtual bool get_prunable_tx_blob(const crypto::hash& h, cryptonote::blobdata &tx) const = 0; + /** * @brief fetches the prunable transaction hash * @@ -1229,24 +1352,7 @@ class BlockchainDB * * @return the requested output data */ - virtual output_data_t get_output_key(const uint64_t& amount, const uint64_t& index) = 0; - - /** - * @brief get some of an output's data - * - * The subclass should return the public key, unlock time, and block height - * for the output with the given global index, collected in a struct. - * - * If the output cannot be found, the subclass should throw OUTPUT_DNE. - * - * If any of these parts cannot be found, but some are, the subclass - * should throw DB_ERROR with a message stating as much. - * - * @param global_index the output's index (global) - * - * @return the requested output data - */ - virtual output_data_t get_output_key(const uint64_t& global_index) const = 0; + virtual output_data_t get_output_key(const uint64_t& amount, const uint64_t& index, bool include_commitmemt = true) const = 0; /** * @brief gets an output's tx hash and index @@ -1294,11 +1400,11 @@ class BlockchainDB * get_output_data(const uint64_t& amount, const uint64_t& index) * but for a list of outputs rather than just one. * - * @param amount an output amount + * @param amounts an output amount, or as many as offsets * @param offsets a list of amount-specific output indices * @param outputs return-by-reference a list of outputs' metadata */ - virtual void get_output_key(const uint64_t &amount, const std::vector &offsets, std::vector &outputs, bool allow_partial = false) = 0; + virtual void get_output_key(const epee::span &amounts, const std::vector &offsets, std::vector &outputs, bool allow_partial = false) const = 0; /* * FIXME: Need to check with git blame and ask what this does to @@ -1317,10 +1423,11 @@ class BlockchainDB * If an output cannot be found, the subclass should throw OUTPUT_DNE. * * @param tx_id a transaction ID + * @param n_txes how many txes to get data for, starting with tx_id * * @return a list of amount-specific output indices */ - virtual std::vector get_tx_amount_output_indices(const uint64_t tx_id) const = 0; + virtual std::vector> get_tx_amount_output_indices(const uint64_t tx_id, size_t n_txes = 1) const = 0; /** * @brief check if a key image is stored as spent @@ -1336,7 +1443,7 @@ class BlockchainDB * * @param details the details of the transaction to add */ - virtual void add_txpool_tx(const transaction &tx, const txpool_tx_meta_t& details) = 0; + virtual void add_txpool_tx(const crypto::hash &txid, const cryptonote::blobdata &blob, const txpool_tx_meta_t& details) = 0; /** * @brief update a txpool transaction's metadata @@ -1392,6 +1499,52 @@ class BlockchainDB */ virtual cryptonote::blobdata get_txpool_tx_blob(const crypto::hash& txid) const = 0; + /** + * @brief prune output data for the given amount + * + * @param amount the amount for which to prune data + */ + virtual void prune_outputs(uint64_t amount) = 0; + + /** + * @brief get the blockchain pruning seed + * @return the blockchain pruning seed + */ + virtual uint32_t get_blockchain_pruning_seed() const = 0; + + /** + * @brief prunes the blockchain + * @param pruning_seed the seed to use, 0 for default (highly recommended) + * @return success iff true + */ + virtual bool prune_blockchain(uint32_t pruning_seed = 0) = 0; + + /** + * @brief prunes recent blockchain changes as needed, iff pruning is enabled + * @return success iff true + */ + virtual bool update_pruning() = 0; + + /** + * @brief checks pruning was done correctly, iff enabled + * @return success iff true + */ + virtual bool check_pruning() = 0; + + /** + * @brief get the max block size + */ + virtual uint64_t get_max_block_size() = 0; + + /** + * @brief add a new max block size + * + * The max block size will be the maximum of sz and the current block size + * + * @param: sz the block size + */ + + virtual void add_max_block_size(uint64_t sz) = 0; /** * @brief runs a function over all txpool transactions * @@ -1535,6 +1688,13 @@ class BlockchainDB */ virtual bool is_read_only() const = 0; + /** + * @brief get disk space requirements + * + * @return the size required + */ + virtual uint64_t get_database_size() const = 0; + // TODO: this should perhaps be (or call) a series of functions which // progressively update through version updates /** @@ -1557,6 +1717,52 @@ class BlockchainDB }; // class BlockchainDB +class db_txn_guard +{ +public: + db_txn_guard(BlockchainDB *db, bool readonly): db(db), readonly(readonly), active(false) + { + if (readonly) + { + active = db->block_rtxn_start(); + } + else + { + db->block_wtxn_start(); + active = true; + } + } + virtual ~db_txn_guard() + { + if (active) + stop(); + } + void stop() + { + if (readonly) + db->block_rtxn_stop(); + else + db->block_wtxn_stop(); + active = false; + } + void abort() + { + if (readonly) + db->block_rtxn_abort(); + else + db->block_wtxn_abort(); + active = false; + } + +private: + BlockchainDB *db; + bool readonly; + bool active; +}; + +class db_rtxn_guard: public db_txn_guard { public: db_rtxn_guard(BlockchainDB *db): db_txn_guard(db, true) {} }; +class db_wtxn_guard: public db_txn_guard { public: db_wtxn_guard(BlockchainDB *db): db_txn_guard(db, false) {} }; + BlockchainDB *new_db(const std::string& db_type); } // namespace cryptonote diff --git a/src/blockchain_db/db_types.h b/src/blockchain_db/db_types.h index b8c7fa3e336..04cadbb10ce 100644 --- a/src/blockchain_db/db_types.h +++ b/src/blockchain_db/db_types.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp index 300fb6d2f6c..d0fd787db00 100644 --- a/src/blockchain_db/lmdb/db_lmdb.cpp +++ b/src/blockchain_db/lmdb/db_lmdb.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // All rights reserved. // // Redistribution and use in source and binary forms, with or without modification, are @@ -29,13 +29,14 @@ #include #include -#include +#include #include // std::unique_ptr #include // memcpy -#include #include "string_tools.h" +#include "file_io_utils.h" #include "common/util.h" +#include "common/pruning.h" #include "cryptonote_basic/cryptonote_format_utils.h" #include "crypto/crypto.h" #include "profile_tools.h" @@ -53,7 +54,7 @@ using epee::string_tools::pod_to_hex; using namespace crypto; // Increase when the DB structure changes -#define VERSION 2 +#define VERSION 5 namespace { @@ -84,6 +85,10 @@ inline void throw1(const T &e) #define MDB_val_set(var, val) MDB_val var = {sizeof(val), (void *)&val} +#define MDB_val_sized(var, val) MDB_val var = {val.size(), (void *)val.data()} + +#define MDB_val_str(var, val) MDB_val var = {strlen(val) + 1, (void *)val} + template struct MDB_val_copy: public MDB_val { @@ -127,14 +132,20 @@ struct MDB_val_copy: public MDB_val std::unique_ptr data; }; -int compare_uint64(const MDB_val *a, const MDB_val *b) +} + +namespace cryptonote +{ + +int BlockchainLMDB::compare_uint64(const MDB_val *a, const MDB_val *b) { - const uint64_t va = *(const uint64_t *)a->mv_data; - const uint64_t vb = *(const uint64_t *)b->mv_data; + uint64_t va, vb; + memcpy(&va, a->mv_data, sizeof(va)); + memcpy(&vb, b->mv_data, sizeof(vb)); return (va < vb) ? -1 : va > vb; } -int compare_hash32(const MDB_val *a, const MDB_val *b) +int BlockchainLMDB::compare_hash32(const MDB_val *a, const MDB_val *b) { uint32_t *va = (uint32_t*) a->mv_data; uint32_t *vb = (uint32_t*) b->mv_data; @@ -148,13 +159,18 @@ int compare_hash32(const MDB_val *a, const MDB_val *b) return 0; } -int compare_string(const MDB_val *a, const MDB_val *b) +int BlockchainLMDB::compare_string(const MDB_val *a, const MDB_val *b) { const char *va = (const char*) a->mv_data; const char *vb = (const char*) b->mv_data; return strcmp(va, vb); } +} + +namespace +{ + /* DB schema: * * Table Key Data @@ -166,6 +182,7 @@ int compare_string(const MDB_val *a, const MDB_val *b) * txs_pruned txn ID pruned txn blob * txs_prunable txn ID prunable txn blob * txs_prunable_hash txn ID prunable txn hash + * txs_prunable_tip txn ID height * tx_indices txn hash {txn ID, metadata} * tx_outputs txn ID [txn amount output indices] * @@ -193,6 +210,7 @@ const char* const LMDB_TXS = "txs"; const char* const LMDB_TXS_PRUNED = "txs_pruned"; const char* const LMDB_TXS_PRUNABLE = "txs_prunable"; const char* const LMDB_TXS_PRUNABLE_HASH = "txs_prunable_hash"; +const char* const LMDB_TXS_PRUNABLE_TIP = "txs_prunable_tip"; const char* const LMDB_TX_INDICES = "tx_indices"; const char* const LMDB_TX_OUTPUTS = "tx_outputs"; @@ -250,26 +268,59 @@ inline void lmdb_db_open(MDB_txn* txn, const char* name, int flags, MDB_dbi& dbi namespace cryptonote { -typedef struct mdb_block_info +typedef struct mdb_block_info_1 +{ + uint64_t bi_height; + uint64_t bi_timestamp; + uint64_t bi_coins; + uint64_t bi_weight; // a size_t really but we need 32-bit compat + uint64_t bi_diff; + crypto::hash bi_hash; +} mdb_block_info_1; + +typedef struct mdb_block_info_2 +{ + uint64_t bi_height; + uint64_t bi_timestamp; + uint64_t bi_coins; + uint64_t bi_weight; // a size_t really but we need 32-bit compat + uint64_t bi_diff; + crypto::hash bi_hash; + uint64_t bi_cum_rct; +} mdb_block_info_2; + +typedef struct mdb_block_info_3 +{ + uint64_t bi_height; + uint64_t bi_timestamp; + uint64_t bi_coins; + uint64_t bi_weight; // a size_t really but we need 32-bit compat + uint64_t bi_diff; + crypto::hash bi_hash; + uint64_t bi_cum_rct; + uint64_t bi_long_term_block_weight; +} mdb_block_info_3; + +typedef struct mdb_block_info_4 { uint64_t bi_height; uint64_t bi_timestamp; uint64_t bi_coins; - uint64_t bi_size; // a size_t really but we need 32-bit compat - difficulty_type bi_diff; + uint64_t bi_weight; // a size_t really but we need 32-bit compat + uint64_t bi_diff_lo; + uint64_t bi_diff_hi; crypto::hash bi_hash; -} mdb_block_info; + uint64_t bi_cum_rct; + uint64_t bi_long_term_block_weight; +} mdb_block_info_4; + +typedef mdb_block_info_4 mdb_block_info; typedef struct blk_height { crypto::hash bh_hash; uint64_t bh_height; } blk_height; -typedef struct txindex { - crypto::hash key; - tx_data_t data; -} txindex; - typedef struct pre_rct_outkey { uint64_t amount_index; uint64_t output_id; @@ -442,6 +493,12 @@ inline int lmdb_txn_renew(MDB_txn *txn) return res; } +inline void BlockchainLMDB::check_open() const +{ + if (!m_open) + throw0(DB_ERROR("DB operation attempted on a not-open DB instance")); +} + void BlockchainLMDB::do_resize(uint64_t increase_size) { LOG_PRINT_L3("BlockchainLMDB::" << __func__); @@ -475,7 +532,7 @@ void BlockchainLMDB::do_resize(uint64_t increase_size) mdb_env_stat(m_env, &mst); // add 1Gb per resize, instead of doing a percentage increase - uint64_t new_mapsize = (double) mei.me_mapsize + add_size; + uint64_t new_mapsize = (uint64_t) mei.me_mapsize + add_size; // If given, use increase_size instead of above way of resizing. // This is currently used for increasing by an estimated size at start of new @@ -529,31 +586,27 @@ bool BlockchainLMDB::need_resize(uint64_t threshold_size) const // additional size needed. uint64_t size_used = mst.ms_psize * mei.me_last_pgno; - LOG_PRINT_L1("DB map size: " << mei.me_mapsize); - LOG_PRINT_L1("Space used: " << size_used); - LOG_PRINT_L1("Space remaining: " << mei.me_mapsize - size_used); - LOG_PRINT_L1("Size threshold: " << threshold_size); - float resize_percent_old = RESIZE_PERCENT; - LOG_PRINT_L1(boost::format("Percent used: %.04f Percent threshold: %.04f") % ((double)size_used/mei.me_mapsize) % resize_percent_old); + MDEBUG("DB map size: " << mei.me_mapsize); + MDEBUG("Space used: " << size_used); + MDEBUG("Space remaining: " << mei.me_mapsize - size_used); + MDEBUG("Size threshold: " << threshold_size); + float resize_percent = RESIZE_PERCENT; + MDEBUG(boost::format("Percent used: %.04f Percent threshold: %.04f") % ((double)size_used/mei.me_mapsize) % resize_percent); if (threshold_size > 0) { if (mei.me_mapsize - size_used < threshold_size) { - LOG_PRINT_L1("Threshold met (size-based)"); + MINFO("Threshold met (size-based)"); return true; } else return false; } - std::mt19937 engine(std::random_device{}()); - std::uniform_real_distribution fdis(0.6, 0.9); - double resize_percent = fdis(engine); - if ((double)size_used / mei.me_mapsize > resize_percent) { - LOG_PRINT_L1("Threshold met (percent-based)"); + MINFO("Threshold met (percent-based)"); return true; } return false; @@ -565,7 +618,7 @@ bool BlockchainLMDB::need_resize(uint64_t threshold_size) const void BlockchainLMDB::check_and_resize_for_batch(uint64_t batch_num_blocks, uint64_t batch_bytes) { LOG_PRINT_L3("BlockchainLMDB::" << __func__); - LOG_PRINT_L1("[" << __func__ << "] " << "checking DB size"); + MTRACE("[" << __func__ << "] " << "checking DB size"); const uint64_t min_increase_size = 512 * (1 << 20); uint64_t threshold_size = 0; uint64_t increase_size = 0; @@ -641,16 +694,19 @@ uint64_t BlockchainLMDB::get_estimated_batch_size(uint64_t batch_num_blocks, uin { MDB_txn *rtxn; mdb_txn_cursors *rcurs; - block_rtxn_start(&rtxn, &rcurs); + bool my_rtxn = block_rtxn_start(&rtxn, &rcurs); for (uint64_t block_num = block_start; block_num <= block_stop; ++block_num) { - uint32_t block_size = get_block_size(block_num); - total_block_size += block_size; + // we have access to block weight, which will be greater or equal to block size, + // so use this as a proxy. If it's too much off, we might have to check actual size, + // which involves reading more data, so is not really wanted + size_t block_weight = get_block_weight(block_num); + total_block_size += block_weight; // Track number of blocks being totalled here instead of assuming, in case // some blocks were to be skipped for being outliers. ++num_blocks_used; } - block_rtxn_stop(); + if (my_rtxn) block_rtxn_stop(); avg_block_size = total_block_size / num_blocks_used; MDEBUG("average block size across recent " << num_blocks_used << " blocks: " << avg_block_size); } @@ -666,8 +722,8 @@ uint64_t BlockchainLMDB::get_estimated_batch_size(uint64_t batch_num_blocks, uin return threshold_size; } -void BlockchainLMDB::add_block(const block& blk, const size_t& block_size, const difficulty_type& cumulative_difficulty, const uint64_t& coins_generated, - const crypto::hash& blk_hash) +void BlockchainLMDB::add_block(const block& blk, size_t block_weight, uint64_t long_term_block_weight, const difficulty_type& cumulative_difficulty, const uint64_t& coins_generated, + uint64_t num_rct_outs, const crypto::hash& blk_hash) { LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); @@ -703,7 +759,8 @@ void BlockchainLMDB::add_block(const block& blk, const size_t& block_size, const CURSOR(block_info) // this call to mdb_cursor_put will change height() - MDB_val_copy blob(block_to_blob(blk)); + cryptonote::blobdata block_blob(block_to_blob(blk)); + MDB_val_sized(blob, block_blob); result = mdb_cursor_put(m_cur_blocks, &key, &blob, MDB_APPEND); if (result) throw0(DB_ERROR(lmdb_error("Failed to add block blob to db transaction: ", result).c_str())); @@ -712,9 +769,21 @@ void BlockchainLMDB::add_block(const block& blk, const size_t& block_size, const bi.bi_height = m_height; bi.bi_timestamp = blk.timestamp; bi.bi_coins = coins_generated; - bi.bi_size = block_size; - bi.bi_diff = cumulative_difficulty; + bi.bi_weight = block_weight; + bi.bi_diff_hi = ((cumulative_difficulty >> 64) & 0xffffffffffffffff).convert_to(); + bi.bi_diff_lo = (cumulative_difficulty & 0xffffffffffffffff).convert_to(); bi.bi_hash = blk_hash; + bi.bi_cum_rct = num_rct_outs; + if (blk.major_version >= 4) + { + uint64_t last_height = m_height-1; + MDB_val_set(h, last_height); + if ((result = mdb_cursor_get(m_cur_block_info, (MDB_val *)&zerokval, &h, MDB_GET_BOTH))) + throw1(BLOCK_DNE(lmdb_error("Failed to get block info: ", result).c_str())); + const mdb_block_info *bi_prev = (const mdb_block_info*)h.mv_data; + bi.bi_cum_rct += bi_prev->bi_cum_rct; + } + bi.bi_long_term_block_weight = long_term_block_weight; MDB_val_set(val, bi); result = mdb_cursor_put(m_cur_block_info, (MDB_val *)&zerokval, &val, MDB_APPENDDUP); @@ -725,7 +794,9 @@ void BlockchainLMDB::add_block(const block& blk, const size_t& block_size, const if (result) throw0(DB_ERROR(lmdb_error("Failed to add block height by hash to db transaction: ", result).c_str())); - m_cum_size += block_size; + // we use weight as a proxy for size, since we don't have size but weight is >= size + // and often actually equal + m_cum_size += block_weight; m_cum_count++; } @@ -759,8 +830,6 @@ void BlockchainLMDB::remove_block() if ((result = mdb_cursor_del(m_cur_block_heights, 0))) throw1(DB_ERROR(lmdb_error("Failed to add removal of block height by hash to db transaction: ", result).c_str())); - if ((result = mdb_cursor_get(m_cur_blocks, &k, NULL, MDB_SET))) - throw1(DB_ERROR(lmdb_error("Failed to locate block for removal: ", result).c_str())); if ((result = mdb_cursor_del(m_cur_blocks, 0))) throw1(DB_ERROR(lmdb_error("Failed to add removal of block to db transaction: ", result).c_str())); @@ -768,7 +837,7 @@ void BlockchainLMDB::remove_block() throw1(DB_ERROR(lmdb_error("Failed to add removal of block info to db transaction: ", result).c_str())); } -uint64_t BlockchainLMDB::add_transaction_data(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash& tx_hash, const crypto::hash& tx_prunable_hash) +uint64_t BlockchainLMDB::add_transaction_data(const crypto::hash& blk_hash, const std::pair& txp, const crypto::hash& tx_hash, const crypto::hash& tx_prunable_hash) { LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); @@ -781,6 +850,7 @@ uint64_t BlockchainLMDB::add_transaction_data(const crypto::hash& blk_hash, cons CURSOR(txs_pruned) CURSOR(txs_prunable) CURSOR(txs_prunable_hash) + CURSOR(txs_prunable_tip) CURSOR(tx_indices) MDB_val_set(val_tx_id, tx_id); @@ -793,6 +863,7 @@ uint64_t BlockchainLMDB::add_transaction_data(const crypto::hash& blk_hash, cons throw1(DB_ERROR(lmdb_error(std::string("Error checking if tx index exists for tx hash ") + epee::string_tools::pod_to_hex(tx_hash) + ": ", result).c_str())); } + const cryptonote::transaction &tx = txp.first; txindex ti; ti.key = tx_hash; ti.data.tx_id = tx_id; @@ -806,28 +877,41 @@ uint64_t BlockchainLMDB::add_transaction_data(const crypto::hash& blk_hash, cons if (result) throw0(DB_ERROR(lmdb_error("Failed to add tx data to db transaction: ", result).c_str())); - cryptonote::blobdata blob = tx_to_blob(tx); - MDB_val_copy blobval(blob); + const cryptonote::blobdata &blob = txp.second; + MDB_val_sized(blobval, blob); - std::stringstream ss; - binary_archive ba(ss); - bool r = const_cast(tx).serialize_base(ba); - if (!r) - throw0(DB_ERROR("Failed to serialize pruned tx")); - std::string pruned = ss.str(); - MDB_val_copy pruned_blob(pruned); + unsigned int unprunable_size = tx.unprunable_size; + if (unprunable_size == 0) + { + std::stringstream ss; + binary_archive ba(ss); + bool r = const_cast(tx).serialize_base(ba); + if (!r) + throw0(DB_ERROR("Failed to serialize pruned tx")); + unprunable_size = ss.str().size(); + } + + if (unprunable_size > blob.size()) + throw0(DB_ERROR("pruned tx size is larger than tx size")); + + MDB_val pruned_blob = {unprunable_size, (void*)blob.data()}; result = mdb_cursor_put(m_cur_txs_pruned, &val_tx_id, &pruned_blob, MDB_APPEND); if (result) throw0(DB_ERROR(lmdb_error("Failed to add pruned tx blob to db transaction: ", result).c_str())); - if (pruned.size() > blob.size()) - throw0(DB_ERROR("pruned tx size is larger than tx size")); - cryptonote::blobdata prunable(blob.data() + pruned.size(), blob.size() - pruned.size()); - MDB_val_copy prunable_blob(prunable); + MDB_val prunable_blob = {blob.size() - unprunable_size, (void*)(blob.data() + unprunable_size)}; result = mdb_cursor_put(m_cur_txs_prunable, &val_tx_id, &prunable_blob, MDB_APPEND); if (result) throw0(DB_ERROR(lmdb_error("Failed to add prunable tx blob to db transaction: ", result).c_str())); + if (get_blockchain_pruning_seed()) + { + MDB_val_set(val_height, m_height); + result = mdb_cursor_put(m_cur_txs_prunable_tip, &val_tx_id, &val_height, 0); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to add prunable tx id to db transaction: ", result).c_str())); + } + if (tx.version > 1) { MDB_val_set(val_prunable_hash, tx_prunable_hash); @@ -853,6 +937,7 @@ void BlockchainLMDB::remove_transaction_data(const crypto::hash& tx_hash, const CURSOR(txs_pruned) CURSOR(txs_prunable) CURSOR(txs_prunable_hash) + CURSOR(txs_prunable_tip) CURSOR(tx_outputs) MDB_val_set(val_h, tx_hash); @@ -868,11 +953,25 @@ void BlockchainLMDB::remove_transaction_data(const crypto::hash& tx_hash, const if (result) throw1(DB_ERROR(lmdb_error("Failed to add removal of pruned tx to db transaction: ", result).c_str())); - if ((result = mdb_cursor_get(m_cur_txs_prunable, &val_tx_id, NULL, MDB_SET))) + result = mdb_cursor_get(m_cur_txs_prunable, &val_tx_id, NULL, MDB_SET); + if (result == 0) + { + result = mdb_cursor_del(m_cur_txs_prunable, 0); + if (result) + throw1(DB_ERROR(lmdb_error("Failed to add removal of prunable tx to db transaction: ", result).c_str())); + } + else if (result != MDB_NOTFOUND) throw1(DB_ERROR(lmdb_error("Failed to locate prunable tx for removal: ", result).c_str())); - result = mdb_cursor_del(m_cur_txs_prunable, 0); - if (result) - throw1(DB_ERROR(lmdb_error("Failed to add removal of prunable tx to db transaction: ", result).c_str())); + + result = mdb_cursor_get(m_cur_txs_prunable_tip, &val_tx_id, NULL, MDB_SET); + if (result && result != MDB_NOTFOUND) + throw1(DB_ERROR(lmdb_error("Failed to locate tx id for removal: ", result).c_str())); + if (result == 0) + { + result = mdb_cursor_del(m_cur_txs_prunable_tip, 0); + if (result) + throw1(DB_ERROR(lmdb_error("Error adding removal of tx id to db transaction", result).c_str())); + } if (tx.version > 1) { @@ -978,11 +1077,11 @@ void BlockchainLMDB::add_tx_amount_output_indices(const uint64_t tx_id, int result = 0; - int num_outputs = amount_output_indices.size(); + size_t num_outputs = amount_output_indices.size(); MDB_val_set(k_tx_id, tx_id); MDB_val v; - v.mv_data = (void *)amount_output_indices.data(); + v.mv_data = num_outputs ? (void *)amount_output_indices.data() : (void*)""; v.mv_size = sizeof(uint64_t) * num_outputs; // LOG_PRINT_L1("tx_outputs[tx_hash] size: " << v.mv_size); @@ -995,7 +1094,8 @@ void BlockchainLMDB::remove_tx_outputs(const uint64_t tx_id, const transaction& { LOG_PRINT_L3("BlockchainLMDB::" << __func__); - std::vector amount_output_indices = get_tx_amount_output_indices(tx_id); + std::vector> amount_output_indices_set = get_tx_amount_output_indices(tx_id, 1); + const std::vector &amount_output_indices = amount_output_indices_set.front(); if (amount_output_indices.empty()) { @@ -1051,6 +1151,60 @@ void BlockchainLMDB::remove_output(const uint64_t amount, const uint64_t& out_in throw0(DB_ERROR(lmdb_error(std::string("Error deleting amount for output index ").append(boost::lexical_cast(out_index).append(": ")).c_str(), result).c_str())); } +void BlockchainLMDB::prune_outputs(uint64_t amount) +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + mdb_txn_cursors *m_cursors = &m_wcursors; + CURSOR(output_amounts); + CURSOR(output_txs); + + MINFO("Pruning outputs for amount " << amount); + + MDB_val v; + MDB_val_set(k, amount); + int result = mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_SET); + if (result == MDB_NOTFOUND) + return; + if (result) + throw0(DB_ERROR(lmdb_error("Error looking up outputs: ", result).c_str())); + + // gather output ids + mdb_size_t num_elems; + mdb_cursor_count(m_cur_output_amounts, &num_elems); + MINFO(num_elems << " outputs found"); + std::vector output_ids; + output_ids.reserve(num_elems); + while (1) + { + const pre_rct_outkey *okp = (const pre_rct_outkey *)v.mv_data; + output_ids.push_back(okp->output_id); + MDEBUG("output id " << okp->output_id); + result = mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_NEXT_DUP); + if (result == MDB_NOTFOUND) + break; + if (result) + throw0(DB_ERROR(lmdb_error("Error counting outputs: ", result).c_str())); + } + if (output_ids.size() != num_elems) + throw0(DB_ERROR("Unexpected number of outputs")); + + result = mdb_cursor_del(m_cur_output_amounts, MDB_NODUPDATA); + if (result) + throw0(DB_ERROR(lmdb_error("Error deleting outputs: ", result).c_str())); + + for (uint64_t output_id: output_ids) + { + MDB_val_set(v, output_id); + result = mdb_cursor_get(m_cur_output_txs, (MDB_val *)&zerokval, &v, MDB_GET_BOTH); + if (result) + throw0(DB_ERROR(lmdb_error("Error looking up output: ", result).c_str())); + result = mdb_cursor_del(m_cur_output_txs, 0); + if (result) + throw0(DB_ERROR(lmdb_error("Error deleting output: ", result).c_str())); + } +} + void BlockchainLMDB::add_spent_key(const crypto::key_image& k_image) { LOG_PRINT_L3("BlockchainLMDB::" << __func__); @@ -1088,43 +1242,16 @@ void BlockchainLMDB::remove_spent_key(const crypto::key_image& k_image) } } -blobdata BlockchainLMDB::output_to_blob(const tx_out& output) const -{ - LOG_PRINT_L3("BlockchainLMDB::" << __func__); - blobdata b; - if (!t_serializable_object_to_blob(output, b)) - throw1(DB_ERROR("Error serializing output to blob")); - return b; -} - -tx_out BlockchainLMDB::output_from_blob(const blobdata& blob) const -{ - LOG_PRINT_L3("BlockchainLMDB::" << __func__); - std::stringstream ss; - ss << blob; - binary_archive ba(ss); - tx_out o; - - if (!(::serialization::serialize(ba, o))) - throw1(DB_ERROR("Error deserializing tx output blob")); - - return o; -} - -void BlockchainLMDB::check_open() const -{ -// LOG_PRINT_L3("BlockchainLMDB::" << __func__); - if (!m_open) - throw0(DB_ERROR("DB operation attempted on a not-open DB instance")); -} - BlockchainLMDB::~BlockchainLMDB() { LOG_PRINT_L3("BlockchainLMDB::" << __func__); // batch transaction shouldn't be active at this point. If it is, consider it aborted. if (m_batch_active) - batch_abort(); + { + try { batch_abort(); } + catch (...) { /* ignore */ } + } if (m_open) close(); } @@ -1143,6 +1270,8 @@ BlockchainLMDB::BlockchainLMDB(bool batch_transactions): BlockchainDB() m_cum_size = 0; m_cum_count = 0; + // reset may also need changing when initialize things here + m_hardfork = nullptr; } @@ -1178,6 +1307,13 @@ void BlockchainLMDB::open(const std::string& filename, const int db_flags) throw DB_ERROR("Database could not be opened"); } + boost::optional is_hdd_result = tools::is_hdd(filename.c_str()); + if (is_hdd_result) + { + if (is_hdd_result.value()) + MCLOG_RED(el::Level::Warning, "global", "The blockchain is on a rotating drive: this will be very slow, use an SSD if possible"); + } + m_folder = filename; #ifdef __OpenBSD__ @@ -1213,14 +1349,14 @@ void BlockchainLMDB::open(const std::string& filename, const int db_flags) MDB_envinfo mei; mdb_env_info(m_env, &mei); - uint64_t cur_mapsize = (double)mei.me_mapsize; + uint64_t cur_mapsize = (uint64_t)mei.me_mapsize; if (cur_mapsize < mapsize) { if (auto result = mdb_env_set_mapsize(m_env, mapsize)) throw0(DB_ERROR(lmdb_error("Failed to set max memory map size: ", result).c_str())); mdb_env_info(m_env, &mei); - cur_mapsize = (double)mei.me_mapsize; + cur_mapsize = (uint64_t)mei.me_mapsize; LOG_PRINT_L1("LMDB memory map size: " << cur_mapsize); } @@ -1241,6 +1377,7 @@ void BlockchainLMDB::open(const std::string& filename, const int db_flags) // open necessary databases, and set properties as needed // uses macros to avoid having to change things too many places + // also change blockchain_prune.cpp to match lmdb_db_open(txn, LMDB_BLOCKS, MDB_INTEGERKEY | MDB_CREATE, m_blocks, "Failed to open db handle for m_blocks"); lmdb_db_open(txn, LMDB_BLOCK_INFO, MDB_INTEGERKEY | MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED, m_block_info, "Failed to open db handle for m_block_info"); @@ -1249,7 +1386,9 @@ void BlockchainLMDB::open(const std::string& filename, const int db_flags) lmdb_db_open(txn, LMDB_TXS, MDB_INTEGERKEY | MDB_CREATE, m_txs, "Failed to open db handle for m_txs"); lmdb_db_open(txn, LMDB_TXS_PRUNED, MDB_INTEGERKEY | MDB_CREATE, m_txs_pruned, "Failed to open db handle for m_txs_pruned"); lmdb_db_open(txn, LMDB_TXS_PRUNABLE, MDB_INTEGERKEY | MDB_CREATE, m_txs_prunable, "Failed to open db handle for m_txs_prunable"); - lmdb_db_open(txn, LMDB_TXS_PRUNABLE_HASH, MDB_INTEGERKEY | MDB_CREATE, m_txs_prunable_hash, "Failed to open db handle for m_txs_prunable_hash"); + lmdb_db_open(txn, LMDB_TXS_PRUNABLE_HASH, MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED | MDB_CREATE, m_txs_prunable_hash, "Failed to open db handle for m_txs_prunable_hash"); + if (!(mdb_flags & MDB_RDONLY)) + lmdb_db_open(txn, LMDB_TXS_PRUNABLE_TIP, MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED | MDB_CREATE, m_txs_prunable_tip, "Failed to open db handle for m_txs_prunable_tip"); lmdb_db_open(txn, LMDB_TX_INDICES, MDB_INTEGERKEY | MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED, m_tx_indices, "Failed to open db handle for m_tx_indices"); lmdb_db_open(txn, LMDB_TX_OUTPUTS, MDB_INTEGERKEY | MDB_CREATE, m_tx_outputs, "Failed to open db handle for m_tx_outputs"); @@ -1277,6 +1416,10 @@ void BlockchainLMDB::open(const std::string& filename, const int db_flags) mdb_set_dupsort(txn, m_output_amounts, compare_uint64); mdb_set_dupsort(txn, m_output_txs, compare_uint64); mdb_set_dupsort(txn, m_block_info, compare_uint64); + if (!(mdb_flags & MDB_RDONLY)) + mdb_set_dupsort(txn, m_txs_prunable_tip, compare_uint64); + mdb_set_compare(txn, m_txs_prunable, compare_uint64); + mdb_set_dupsort(txn, m_txs_prunable_hash, compare_uint64); mdb_set_compare(txn, m_txpool_meta, compare_hash32); mdb_set_compare(txn, m_txpool_blob, compare_hash32); @@ -1298,25 +1441,35 @@ void BlockchainLMDB::open(const std::string& filename, const int db_flags) bool compatible = true; - MDB_val_copy k("version"); + MDB_val_str(k, "version"); MDB_val v; auto get_result = mdb_get(txn, m_properties, &k, &v); if(get_result == MDB_SUCCESS) { - if (*(const uint32_t*)v.mv_data > VERSION) + const uint32_t db_version = *(const uint32_t*)v.mv_data; + if (db_version > VERSION) { - MWARNING("Existing lmdb database was made by a later version. We don't know how it will change yet."); + MWARNING("Existing lmdb database was made by a later version (" << db_version << "). We don't know how it will change yet."); compatible = false; } #if VERSION > 0 - else if (*(const uint32_t*)v.mv_data < VERSION) + else if (db_version < VERSION) { + if (mdb_flags & MDB_RDONLY) + { + txn.abort(); + mdb_env_close(m_env); + m_open = false; + MFATAL("Existing lmdb database needs to be converted, which cannot be done on a read-only database."); + MFATAL("Please run monerod once to convert the database."); + return; + } // Note that there was a schema change within version 0 as well. // See commit e5d2680094ee15889934fe28901e4e133cda56f2 2015/07/10 // We don't handle the old format previous to that commit. txn.commit(); m_open = true; - migrate(*(const uint32_t *)v.mv_data); + migrate(db_version); return; } #endif @@ -1345,7 +1498,7 @@ void BlockchainLMDB::open(const std::string& filename, const int db_flags) // only write version on an empty DB if (m_height == 0) { - MDB_val_copy k("version"); + MDB_val_str(k, "version"); MDB_val_copy v(VERSION); auto put_result = mdb_put(txn, m_properties, &k, &v, 0); if (put_result != MDB_SUCCESS) @@ -1387,6 +1540,9 @@ void BlockchainLMDB::sync() LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); + if (is_read_only()) + return; + // Does nothing unless LMDB environment was opened with MDB_NOSYNC or in part // MDB_NOMETASYNC. Force flush to be synchronous. if (auto result = mdb_env_sync(m_env, true)) @@ -1422,6 +1578,8 @@ void BlockchainLMDB::reset() throw0(DB_ERROR(lmdb_error("Failed to drop m_txs_prunable: ", result).c_str())); if (auto result = mdb_drop(txn, m_txs_prunable_hash, 0)) throw0(DB_ERROR(lmdb_error("Failed to drop m_txs_prunable_hash: ", result).c_str())); + if (auto result = mdb_drop(txn, m_txs_prunable_tip, 0)) + throw0(DB_ERROR(lmdb_error("Failed to drop m_txs_prunable_tip: ", result).c_str())); if (auto result = mdb_drop(txn, m_tx_indices, 0)) throw0(DB_ERROR(lmdb_error("Failed to drop m_tx_indices: ", result).c_str())); if (auto result = mdb_drop(txn, m_tx_outputs, 0)) @@ -1439,7 +1597,7 @@ void BlockchainLMDB::reset() throw0(DB_ERROR(lmdb_error("Failed to drop m_properties: ", result).c_str())); // init with current version - MDB_val_copy k("version"); + MDB_val_str(k, "version"); MDB_val_copy v(VERSION); if (auto result = mdb_put(txn, m_properties, &k, &v, 0)) throw0(DB_ERROR(lmdb_error("Failed to write version to database: ", result).c_str())); @@ -1465,6 +1623,21 @@ std::vector BlockchainLMDB::get_filenames() const return filenames; } +bool BlockchainLMDB::remove_data_file(const std::string& folder) const +{ + const std::string filename = folder + "/data.mdb"; + try + { + boost::filesystem::remove(filename); + } + catch (const std::exception &e) + { + MERROR("Failed to remove " << filename << ": " << e.what()); + return false; + } + return true; +} + std::string BlockchainLMDB::get_db_name() const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); @@ -1539,7 +1712,7 @@ void BlockchainLMDB::unlock() auto_txn.commit(); \ } while(0) -void BlockchainLMDB::add_txpool_tx(const transaction &tx, const txpool_tx_meta_t &meta) +void BlockchainLMDB::add_txpool_tx(const crypto::hash &txid, const cryptonote::blobdata &blob, const txpool_tx_meta_t &meta) { LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); @@ -1548,8 +1721,6 @@ void BlockchainLMDB::add_txpool_tx(const transaction &tx, const txpool_tx_meta_t CURSOR(txpool_meta) CURSOR(txpool_blob) - const crypto::hash txid = get_transaction_hash(tx); - MDB_val k = {sizeof(txid), (void *)&txid}; MDB_val v = {sizeof(meta), (void *)&meta}; if (auto result = mdb_cursor_put(m_cur_txpool_meta, &k, &v, MDB_NODUPDATA)) { @@ -1558,7 +1729,7 @@ void BlockchainLMDB::add_txpool_tx(const transaction &tx, const txpool_tx_meta_t else throw1(DB_ERROR(lmdb_error("Error adding txpool tx metadata to db transaction: ", result).c_str())); } - MDB_val_copy blob_val(tx_to_blob(tx)); + MDB_val_sized(blob_val, blob); if (auto result = mdb_cursor_put(m_cur_txpool_blob, &k, &blob_val, MDB_NODUPDATA)) { if (result == MDB_KEYEXIST) throw1(DB_ERROR("Attempting to add txpool tx blob that's already in the db")); @@ -1734,111 +1905,444 @@ cryptonote::blobdata BlockchainLMDB::get_txpool_tx_blob(const crypto::hash& txid return bd; } -bool BlockchainLMDB::for_all_txpool_txes(std::function f, bool include_blob, bool include_unrelayed_txes) const +uint32_t BlockchainLMDB::get_blockchain_pruning_seed() const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); TXN_PREFIX_RDONLY(); - RCURSOR(txpool_meta); - RCURSOR(txpool_blob); - - MDB_val k; + RCURSOR(properties) + MDB_val_str(k, "pruning_seed"); MDB_val v; - bool ret = true; - - MDB_cursor_op op = MDB_FIRST; - while (1) - { - int result = mdb_cursor_get(m_cur_txpool_meta, &k, &v, op); - op = MDB_NEXT; - if (result == MDB_NOTFOUND) - break; - if (result) - throw0(DB_ERROR(lmdb_error("Failed to enumerate txpool tx metadata: ", result).c_str())); - const crypto::hash txid = *(const crypto::hash*)k.mv_data; - const txpool_tx_meta_t &meta = *(const txpool_tx_meta_t*)v.mv_data; - if (!include_unrelayed_txes && meta.do_not_relay) - // Skipping that tx - continue; - const cryptonote::blobdata *passed_bd = NULL; - cryptonote::blobdata bd; - if (include_blob) - { - MDB_val b; - result = mdb_cursor_get(m_cur_txpool_blob, &k, &b, MDB_SET); - if (result == MDB_NOTFOUND) - throw0(DB_ERROR("Failed to find txpool tx blob to match metadata")); - if (result) - throw0(DB_ERROR(lmdb_error("Failed to enumerate txpool tx blob: ", result).c_str())); - bd.assign(reinterpret_cast(b.mv_data), b.mv_size); - passed_bd = &bd; - } - - if (!f(txid, meta, passed_bd)) { - ret = false; - break; - } - } - + int result = mdb_cursor_get(m_cur_properties, &k, &v, MDB_SET); + if (result == MDB_NOTFOUND) + return 0; + if (result) + throw0(DB_ERROR(lmdb_error("Failed to retrieve pruning seed: ", result).c_str())); + if (v.mv_size != sizeof(uint32_t)) + throw0(DB_ERROR("Failed to retrieve or create pruning seed: unexpected value size")); + uint32_t pruning_seed; + memcpy(&pruning_seed, v.mv_data, sizeof(pruning_seed)); TXN_POSTFIX_RDONLY(); + return pruning_seed; +} - return ret; +static bool is_v1_tx(MDB_cursor *c_txs_pruned, MDB_val *tx_id) +{ + MDB_val v; + int ret = mdb_cursor_get(c_txs_pruned, tx_id, &v, MDB_SET); + if (ret) + throw0(DB_ERROR(lmdb_error("Failed to find transaction pruned data: ", ret).c_str())); + if (v.mv_size == 0) + throw0(DB_ERROR("Invalid transaction pruned data")); + return cryptonote::is_v1_tx(cryptonote::blobdata_ref{(const char*)v.mv_data, v.mv_size}); } -bool BlockchainLMDB::block_exists(const crypto::hash& h, uint64_t *height) const +enum { prune_mode_prune, prune_mode_update, prune_mode_check }; + +bool BlockchainLMDB::prune_worker(int mode, uint32_t pruning_seed) { LOG_PRINT_L3("BlockchainLMDB::" << __func__); + const uint32_t log_stripes = tools::get_pruning_log_stripes(pruning_seed); + if (log_stripes && log_stripes != CRYPTONOTE_PRUNING_LOG_STRIPES) + throw0(DB_ERROR("Pruning seed not in range")); + pruning_seed = tools::get_pruning_stripe(pruning_seed);; + if (pruning_seed > (1ul << CRYPTONOTE_PRUNING_LOG_STRIPES)) + throw0(DB_ERROR("Pruning seed not in range")); check_open(); - TXN_PREFIX_RDONLY(); - RCURSOR(block_heights); + TIME_MEASURE_START(t); - bool ret = false; - MDB_val_set(key, h); - auto get_result = mdb_cursor_get(m_cur_block_heights, (MDB_val *)&zerokval, &key, MDB_GET_BOTH); - if (get_result == MDB_NOTFOUND) + size_t n_total_records = 0, n_prunable_records = 0, n_pruned_records = 0, commit_counter = 0; + uint64_t n_bytes = 0; + + mdb_txn_safe txn; + auto result = mdb_txn_begin(m_env, NULL, 0, txn); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); + + MDB_stat db_stats; + if ((result = mdb_stat(txn, m_txs_prunable, &db_stats))) + throw0(DB_ERROR(lmdb_error("Failed to query m_txs_prunable: ", result).c_str())); + const size_t pages0 = db_stats.ms_branch_pages + db_stats.ms_leaf_pages + db_stats.ms_overflow_pages; + + MDB_val_str(k, "pruning_seed"); + MDB_val v; + result = mdb_get(txn, m_properties, &k, &v); + bool prune_tip_table = false; + if (result == MDB_NOTFOUND) { - LOG_PRINT_L3("Block with hash " << epee::string_tools::pod_to_hex(h) << " not found in db"); + // not pruned yet + if (mode != prune_mode_prune) + { + txn.abort(); + TIME_MEASURE_FINISH(t); + MDEBUG("Pruning not enabled, nothing to do"); + return true; + } + if (pruning_seed == 0) + pruning_seed = tools::get_random_stripe(); + pruning_seed = tools::make_pruning_seed(pruning_seed, CRYPTONOTE_PRUNING_LOG_STRIPES); + v.mv_data = &pruning_seed; + v.mv_size = sizeof(pruning_seed); + result = mdb_put(txn, m_properties, &k, &v, 0); + if (result) + throw0(DB_ERROR("Failed to save pruning seed")); + prune_tip_table = false; + } + else if (result == 0) + { + // pruned already + if (v.mv_size != sizeof(uint32_t)) + throw0(DB_ERROR("Failed to retrieve or create pruning seed: unexpected value size")); + const uint32_t data = *(const uint32_t*)v.mv_data; + if (pruning_seed == 0) + pruning_seed = tools::get_pruning_stripe(data); + if (tools::get_pruning_stripe(data) != pruning_seed) + throw0(DB_ERROR("Blockchain already pruned with different seed")); + if (tools::get_pruning_log_stripes(data) != CRYPTONOTE_PRUNING_LOG_STRIPES) + throw0(DB_ERROR("Blockchain already pruned with different base")); + pruning_seed = tools::make_pruning_seed(pruning_seed, CRYPTONOTE_PRUNING_LOG_STRIPES); + prune_tip_table = (mode == prune_mode_update); } - else if (get_result) - throw0(DB_ERROR(lmdb_error("DB error attempting to fetch block index from hash", get_result).c_str())); else { - if (height) - { - const blk_height *bhp = (const blk_height *)key.mv_data; - *height = bhp->bh_height; - } - ret = true; + throw0(DB_ERROR(lmdb_error("Failed to retrieve or create pruning seed: ", result).c_str())); } - TXN_POSTFIX_RDONLY(); - return ret; -} - -cryptonote::blobdata BlockchainLMDB::get_block_blob(const crypto::hash& h) const -{ - LOG_PRINT_L3("BlockchainLMDB::" << __func__); - check_open(); - - return get_block_blob_from_height(get_block_height(h)); -} + if (mode == prune_mode_check) + MINFO("Checking blockchain pruning..."); + else + MINFO("Pruning blockchain..."); -uint64_t BlockchainLMDB::get_block_height(const crypto::hash& h) const -{ - LOG_PRINT_L3("BlockchainLMDB::" << __func__); - check_open(); + MDB_cursor *c_txs_pruned, *c_txs_prunable, *c_txs_prunable_tip; + result = mdb_cursor_open(txn, m_txs_pruned, &c_txs_pruned); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for txs_pruned: ", result).c_str())); + result = mdb_cursor_open(txn, m_txs_prunable, &c_txs_prunable); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for txs_prunable: ", result).c_str())); + result = mdb_cursor_open(txn, m_txs_prunable_tip, &c_txs_prunable_tip); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for txs_prunable_tip: ", result).c_str())); + const uint64_t blockchain_height = height(); - TXN_PREFIX_RDONLY(); - RCURSOR(block_heights); + if (prune_tip_table) + { + MDB_cursor_op op = MDB_FIRST; + while (1) + { + int ret = mdb_cursor_get(c_txs_prunable_tip, &k, &v, op); + op = MDB_NEXT; + if (ret == MDB_NOTFOUND) + break; + if (ret) + throw0(DB_ERROR(lmdb_error("Failed to enumerate transactions: ", ret).c_str())); - MDB_val_set(key, h); - auto get_result = mdb_cursor_get(m_cur_block_heights, (MDB_val *)&zerokval, &key, MDB_GET_BOTH); - if (get_result == MDB_NOTFOUND) - throw1(BLOCK_DNE("Attempted to retrieve non-existent block height")); - else if (get_result) - throw0(DB_ERROR("Error attempting to retrieve a block height from the db")); + uint64_t block_height; + memcpy(&block_height, v.mv_data, sizeof(block_height)); + if (block_height + CRYPTONOTE_PRUNING_TIP_BLOCKS < blockchain_height) + { + ++n_total_records; + if (!tools::has_unpruned_block(block_height, blockchain_height, pruning_seed) && !is_v1_tx(c_txs_pruned, &k)) + { + ++n_prunable_records; + result = mdb_cursor_get(c_txs_prunable, &k, &v, MDB_SET); + if (result == MDB_NOTFOUND) + MWARNING("Already pruned at height " << block_height << "/" << blockchain_height); + else if (result) + throw0(DB_ERROR(lmdb_error("Failed to find transaction prunable data: ", result).c_str())); + else + { + MDEBUG("Pruning at height " << block_height << "/" << blockchain_height); + ++n_pruned_records; + ++commit_counter; + n_bytes += k.mv_size + v.mv_size; + result = mdb_cursor_del(c_txs_prunable, 0); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to delete transaction prunable data: ", result).c_str())); + } + } + result = mdb_cursor_del(c_txs_prunable_tip, 0); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to delete transaction tip data: ", result).c_str())); + + if (mode != prune_mode_check && commit_counter >= 4096) + { + MDEBUG("Committing txn at checkpoint..."); + txn.commit(); + result = mdb_txn_begin(m_env, NULL, 0, txn); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); + result = mdb_cursor_open(txn, m_txs_pruned, &c_txs_pruned); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for txs_pruned: ", result).c_str())); + result = mdb_cursor_open(txn, m_txs_prunable, &c_txs_prunable); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for txs_prunable: ", result).c_str())); + result = mdb_cursor_open(txn, m_txs_prunable_tip, &c_txs_prunable_tip); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for txs_prunable_tip: ", result).c_str())); + commit_counter = 0; + } + } + } + } + else + { + MDB_cursor *c_tx_indices; + result = mdb_cursor_open(txn, m_tx_indices, &c_tx_indices); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for tx_indices: ", result).c_str())); + MDB_cursor_op op = MDB_FIRST; + while (1) + { + int ret = mdb_cursor_get(c_tx_indices, &k, &v, op); + op = MDB_NEXT; + if (ret == MDB_NOTFOUND) + break; + if (ret) + throw0(DB_ERROR(lmdb_error("Failed to enumerate transactions: ", ret).c_str())); + + ++n_total_records; + //const txindex *ti = (const txindex *)v.mv_data; + txindex ti; + memcpy(&ti, v.mv_data, sizeof(ti)); + const uint64_t block_height = ti.data.block_id; + if (block_height + CRYPTONOTE_PRUNING_TIP_BLOCKS >= blockchain_height) + { + MDB_val_set(kp, ti.data.tx_id); + MDB_val_set(vp, block_height); + if (mode == prune_mode_check) + { + result = mdb_cursor_get(c_txs_prunable_tip, &kp, &vp, MDB_SET); + if (result && result != MDB_NOTFOUND) + throw0(DB_ERROR(lmdb_error("Error looking for transaction prunable data: ", result).c_str())); + if (result == MDB_NOTFOUND) + MERROR("Transaction not found in prunable tip table for height " << block_height << "/" << blockchain_height << + ", seed " << epee::string_tools::to_string_hex(pruning_seed)); + } + else + { + result = mdb_cursor_put(c_txs_prunable_tip, &kp, &vp, 0); + if (result && result != MDB_NOTFOUND) + throw0(DB_ERROR(lmdb_error("Error looking for transaction prunable data: ", result).c_str())); + } + } + MDB_val_set(kp, ti.data.tx_id); + if (!tools::has_unpruned_block(block_height, blockchain_height, pruning_seed) && !is_v1_tx(c_txs_pruned, &kp)) + { + result = mdb_cursor_get(c_txs_prunable, &kp, &v, MDB_SET); + if (result && result != MDB_NOTFOUND) + throw0(DB_ERROR(lmdb_error("Error looking for transaction prunable data: ", result).c_str())); + if (mode == prune_mode_check) + { + if (result != MDB_NOTFOUND) + MERROR("Prunable data found for pruned height " << block_height << "/" << blockchain_height << + ", seed " << epee::string_tools::to_string_hex(pruning_seed)); + } + else + { + ++n_prunable_records; + if (result == MDB_NOTFOUND) + MWARNING("Already pruned at height " << block_height << "/" << blockchain_height); + else + { + MDEBUG("Pruning at height " << block_height << "/" << blockchain_height); + ++n_pruned_records; + n_bytes += kp.mv_size + v.mv_size; + result = mdb_cursor_del(c_txs_prunable, 0); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to delete transaction prunable data: ", result).c_str())); + ++commit_counter; + } + } + } + else + { + if (mode == prune_mode_check) + { + MDB_val_set(kp, ti.data.tx_id); + result = mdb_cursor_get(c_txs_prunable, &kp, &v, MDB_SET); + if (result && result != MDB_NOTFOUND) + throw0(DB_ERROR(lmdb_error("Error looking for transaction prunable data: ", result).c_str())); + if (result == MDB_NOTFOUND) + MERROR("Prunable data not found for unpruned height " << block_height << "/" << blockchain_height << + ", seed " << epee::string_tools::to_string_hex(pruning_seed)); + } + } + + if (mode != prune_mode_check && commit_counter >= 4096) + { + MDEBUG("Committing txn at checkpoint..."); + txn.commit(); + result = mdb_txn_begin(m_env, NULL, 0, txn); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); + result = mdb_cursor_open(txn, m_txs_pruned, &c_txs_pruned); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for txs_pruned: ", result).c_str())); + result = mdb_cursor_open(txn, m_txs_prunable, &c_txs_prunable); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for txs_prunable: ", result).c_str())); + result = mdb_cursor_open(txn, m_txs_prunable_tip, &c_txs_prunable_tip); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for txs_prunable_tip: ", result).c_str())); + result = mdb_cursor_open(txn, m_tx_indices, &c_tx_indices); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for tx_indices: ", result).c_str())); + MDB_val val; + val.mv_size = sizeof(ti); + val.mv_data = (void *)&ti; + result = mdb_cursor_get(c_tx_indices, (MDB_val*)&zerokval, &val, MDB_GET_BOTH); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to restore cursor for tx_indices: ", result).c_str())); + commit_counter = 0; + } + } + mdb_cursor_close(c_tx_indices); + } + + if ((result = mdb_stat(txn, m_txs_prunable, &db_stats))) + throw0(DB_ERROR(lmdb_error("Failed to query m_txs_prunable: ", result).c_str())); + const size_t pages1 = db_stats.ms_branch_pages + db_stats.ms_leaf_pages + db_stats.ms_overflow_pages; + const size_t db_bytes = (pages0 - pages1) * db_stats.ms_psize; + + mdb_cursor_close(c_txs_prunable_tip); + mdb_cursor_close(c_txs_prunable); + mdb_cursor_close(c_txs_pruned); + + txn.commit(); + + TIME_MEASURE_FINISH(t); + + MINFO((mode == prune_mode_check ? "Checked" : "Pruned") << " blockchain in " << + t << " ms: " << (n_bytes/1024.0f/1024.0f) << " MB (" << db_bytes/1024.0f/1024.0f << " MB) pruned in " << + n_pruned_records << " records (" << pages0 - pages1 << "/" << pages0 << " " << db_stats.ms_psize << " byte pages), " << + n_prunable_records << "/" << n_total_records << " pruned records"); + return true; +} + +bool BlockchainLMDB::prune_blockchain(uint32_t pruning_seed) +{ + return prune_worker(prune_mode_prune, pruning_seed); +} + +bool BlockchainLMDB::update_pruning() +{ + return prune_worker(prune_mode_update, 0); +} + +bool BlockchainLMDB::check_pruning() +{ + return prune_worker(prune_mode_check, 0); +} + +bool BlockchainLMDB::for_all_txpool_txes(std::function f, bool include_blob, bool include_unrelayed_txes) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + TXN_PREFIX_RDONLY(); + RCURSOR(txpool_meta); + RCURSOR(txpool_blob); + + MDB_val k; + MDB_val v; + bool ret = true; + + MDB_cursor_op op = MDB_FIRST; + while (1) + { + int result = mdb_cursor_get(m_cur_txpool_meta, &k, &v, op); + op = MDB_NEXT; + if (result == MDB_NOTFOUND) + break; + if (result) + throw0(DB_ERROR(lmdb_error("Failed to enumerate txpool tx metadata: ", result).c_str())); + const crypto::hash txid = *(const crypto::hash*)k.mv_data; + const txpool_tx_meta_t &meta = *(const txpool_tx_meta_t*)v.mv_data; + if (!include_unrelayed_txes && meta.do_not_relay) + // Skipping that tx + continue; + const cryptonote::blobdata *passed_bd = NULL; + cryptonote::blobdata bd; + if (include_blob) + { + MDB_val b; + result = mdb_cursor_get(m_cur_txpool_blob, &k, &b, MDB_SET); + if (result == MDB_NOTFOUND) + throw0(DB_ERROR("Failed to find txpool tx blob to match metadata")); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to enumerate txpool tx blob: ", result).c_str())); + bd.assign(reinterpret_cast(b.mv_data), b.mv_size); + passed_bd = &bd; + } + + if (!f(txid, meta, passed_bd)) { + ret = false; + break; + } + } + + TXN_POSTFIX_RDONLY(); + + return ret; +} + +bool BlockchainLMDB::block_exists(const crypto::hash& h, uint64_t *height) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + TXN_PREFIX_RDONLY(); + RCURSOR(block_heights); + + bool ret = false; + MDB_val_set(key, h); + auto get_result = mdb_cursor_get(m_cur_block_heights, (MDB_val *)&zerokval, &key, MDB_GET_BOTH); + if (get_result == MDB_NOTFOUND) + { + LOG_PRINT_L3("Block with hash " << epee::string_tools::pod_to_hex(h) << " not found in db"); + } + else if (get_result) + throw0(DB_ERROR(lmdb_error("DB error attempting to fetch block index from hash", get_result).c_str())); + else + { + if (height) + { + const blk_height *bhp = (const blk_height *)key.mv_data; + *height = bhp->bh_height; + } + ret = true; + } + + TXN_POSTFIX_RDONLY(); + return ret; +} + +cryptonote::blobdata BlockchainLMDB::get_block_blob(const crypto::hash& h) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + return get_block_blob_from_height(get_block_height(h)); +} + +uint64_t BlockchainLMDB::get_block_height(const crypto::hash& h) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + TXN_PREFIX_RDONLY(); + RCURSOR(block_heights); + + MDB_val_set(key, h); + auto get_result = mdb_cursor_get(m_cur_block_heights, (MDB_val *)&zerokval, &key, MDB_GET_BOTH); + if (get_result == MDB_NOTFOUND) + throw1(BLOCK_DNE("Attempted to retrieve non-existent block height")); + else if (get_result) + throw0(DB_ERROR("Error attempting to retrieve a block height from the db")); blk_height *bhp = (blk_height *)key.mv_data; uint64_t ret = bhp->bh_height; @@ -1904,6 +2408,68 @@ uint64_t BlockchainLMDB::get_block_timestamp(const uint64_t& height) const return ret; } +std::vector BlockchainLMDB::get_block_cumulative_rct_outputs(const std::vector &heights) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + std::vector res; + int result; + + if (heights.empty()) + return {}; + res.reserve(heights.size()); + + TXN_PREFIX_RDONLY(); + RCURSOR(block_info); + + MDB_stat db_stats; + if ((result = mdb_stat(m_txn, m_blocks, &db_stats))) + throw0(DB_ERROR(lmdb_error("Failed to query m_blocks: ", result).c_str())); + for (size_t i = 0; i < heights.size(); ++i) + if (heights[i] >= db_stats.ms_entries) + throw0(BLOCK_DNE(std::string("Attempt to get rct distribution from height " + std::to_string(heights[i]) + " failed -- block size not in db").c_str())); + + MDB_val v; + + uint64_t prev_height = heights[0]; + uint64_t range_begin = 0, range_end = 0; + for (uint64_t height: heights) + { + if (height >= range_begin && height < range_end) + { + // nohting to do + } + else + { + if (height == prev_height + 1) + { + MDB_val k2; + result = mdb_cursor_get(m_cur_block_info, &k2, &v, MDB_NEXT_MULTIPLE); + range_begin = ((const mdb_block_info*)v.mv_data)->bi_height; + range_end = range_begin + v.mv_size / sizeof(mdb_block_info); // whole records please + if (height < range_begin || height >= range_end) + throw0(DB_ERROR(("Height " + std::to_string(height) + " not included in multuple record range: " + std::to_string(range_begin) + "-" + std::to_string(range_end)).c_str())); + } + else + { + v.mv_size = sizeof(uint64_t); + v.mv_data = (void*)&height; + result = mdb_cursor_get(m_cur_block_info, (MDB_val *)&zerokval, &v, MDB_GET_BOTH); + range_begin = height; + range_end = range_begin + 1; + } + if (result) + throw0(DB_ERROR(lmdb_error("Error attempting to retrieve rct distribution from the db: ", result).c_str())); + } + const mdb_block_info *bi = ((const mdb_block_info *)v.mv_data) + (height - range_begin); + res.push_back(bi->bi_cum_rct); + prev_height = height; + } + + TXN_POSTFIX_RDONLY(); + return res; +} + uint64_t BlockchainLMDB::get_top_block_timestamp() const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); @@ -1919,7 +2485,7 @@ uint64_t BlockchainLMDB::get_top_block_timestamp() const return get_block_timestamp(m_height - 1); } -size_t BlockchainLMDB::get_block_size(const uint64_t& height) const +size_t BlockchainLMDB::get_block_weight(const uint64_t& height) const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); @@ -1937,66 +2503,184 @@ size_t BlockchainLMDB::get_block_size(const uint64_t& height) const throw0(DB_ERROR("Error attempting to retrieve a block size from the db")); mdb_block_info *bi = (mdb_block_info *)result.mv_data; - size_t ret = bi->bi_size; + size_t ret = bi->bi_weight; TXN_POSTFIX_RDONLY(); return ret; } -difficulty_type BlockchainLMDB::get_block_cumulative_difficulty(const uint64_t& height) const +std::vector BlockchainLMDB::get_block_info_64bit_fields(uint64_t start_height, size_t count, off_t offset) const { - LOG_PRINT_L3("BlockchainLMDB::" << __func__ << " height: " << height); + LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); TXN_PREFIX_RDONLY(); RCURSOR(block_info); - MDB_val_set(result, height); - auto get_result = mdb_cursor_get(m_cur_block_info, (MDB_val *)&zerokval, &result, MDB_GET_BOTH); - if (get_result == MDB_NOTFOUND) + const uint64_t h = height(); + if (start_height >= h) + throw0(DB_ERROR(("Height " + std::to_string(start_height) + " not in blockchain").c_str())); + + std::vector ret; + ret.reserve(count); + + MDB_val v; + uint64_t range_begin = 0, range_end = 0; + for (uint64_t height = start_height; height < h && count--; ++height) { - throw0(BLOCK_DNE(std::string("Attempt to get cumulative difficulty from height ").append(boost::lexical_cast(height)).append(" failed -- difficulty not in db").c_str())); + if (height >= range_begin && height < range_end) + { + // nothing to do + } + else + { + int result = 0; + if (range_end > 0) + { + MDB_val k2; + result = mdb_cursor_get(m_cur_block_info, &k2, &v, MDB_NEXT_MULTIPLE); + range_begin = ((const mdb_block_info*)v.mv_data)->bi_height; + range_end = range_begin + v.mv_size / sizeof(mdb_block_info); // whole records please + if (height < range_begin || height >= range_end) + throw0(DB_ERROR(("Height " + std::to_string(height) + " not included in multiple record range: " + std::to_string(range_begin) + "-" + std::to_string(range_end)).c_str())); + } + else + { + v.mv_size = sizeof(uint64_t); + v.mv_data = (void*)&height; + result = mdb_cursor_get(m_cur_block_info, (MDB_val *)&zerokval, &v, MDB_GET_BOTH); + range_begin = height; + range_end = range_begin + 1; + } + if (result) + throw0(DB_ERROR(lmdb_error("Error attempting to retrieve block_info from the db: ", result).c_str())); + } + const mdb_block_info *bi = ((const mdb_block_info *)v.mv_data) + (height - range_begin); + ret.push_back(*(const uint64_t*)(((const char*)bi) + offset)); } - else if (get_result) - throw0(DB_ERROR("Error attempting to retrieve a cumulative difficulty from the db")); - mdb_block_info *bi = (mdb_block_info *)result.mv_data; - difficulty_type ret = bi->bi_diff; TXN_POSTFIX_RDONLY(); return ret; } -difficulty_type BlockchainLMDB::get_block_difficulty(const uint64_t& height) const +uint64_t BlockchainLMDB::get_max_block_size() { LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); - difficulty_type diff1 = 0; - difficulty_type diff2 = 0; - - diff1 = get_block_cumulative_difficulty(height); - if (height != 0) - { - diff2 = get_block_cumulative_difficulty(height - 1); - } - - return diff1 - diff2; + TXN_PREFIX_RDONLY(); + RCURSOR(properties) + MDB_val_str(k, "max_block_size"); + MDB_val v; + int result = mdb_cursor_get(m_cur_properties, &k, &v, MDB_SET); + if (result == MDB_NOTFOUND) + return std::numeric_limits::max(); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to retrieve max block size: ", result).c_str())); + if (v.mv_size != sizeof(uint64_t)) + throw0(DB_ERROR("Failed to retrieve or create max block size: unexpected value size")); + uint64_t max_block_size; + memcpy(&max_block_size, v.mv_data, sizeof(max_block_size)); + TXN_POSTFIX_RDONLY(); + return max_block_size; } -uint64_t BlockchainLMDB::get_block_already_generated_coins(const uint64_t& height) const +void BlockchainLMDB::add_max_block_size(uint64_t sz) { LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); + mdb_txn_cursors *m_cursors = &m_wcursors; - TXN_PREFIX_RDONLY(); - RCURSOR(block_info); + CURSOR(properties) - MDB_val_set(result, height); - auto get_result = mdb_cursor_get(m_cur_block_info, (MDB_val *)&zerokval, &result, MDB_GET_BOTH); - if (get_result == MDB_NOTFOUND) + MDB_val_str(k, "max_block_size"); + MDB_val v; + int result = mdb_cursor_get(m_cur_properties, &k, &v, MDB_SET); + if (result && result != MDB_NOTFOUND) + throw0(DB_ERROR(lmdb_error("Failed to retrieve max block size: ", result).c_str())); + uint64_t max_block_size = 0; + if (result == 0) { - throw0(BLOCK_DNE(std::string("Attempt to get generated coins from height ").append(boost::lexical_cast(height)).append(" failed -- block size not in db").c_str())); + if (v.mv_size != sizeof(uint64_t)) + throw0(DB_ERROR("Failed to retrieve or create max block size: unexpected value size")); + memcpy(&max_block_size, v.mv_data, sizeof(max_block_size)); } - else if (get_result) + if (sz > max_block_size) + max_block_size = sz; + v.mv_data = (void*)&max_block_size; + v.mv_size = sizeof(max_block_size); + result = mdb_cursor_put(m_cur_properties, &k, &v, 0); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to set max_block_size: ", result).c_str())); +} + + +std::vector BlockchainLMDB::get_block_weights(uint64_t start_height, size_t count) const +{ + return get_block_info_64bit_fields(start_height, count, offsetof(mdb_block_info, bi_weight)); +} + +std::vector BlockchainLMDB::get_long_term_block_weights(uint64_t start_height, size_t count) const +{ + return get_block_info_64bit_fields(start_height, count, offsetof(mdb_block_info, bi_long_term_block_weight)); +} + +difficulty_type BlockchainLMDB::get_block_cumulative_difficulty(const uint64_t& height) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__ << " height: " << height); + check_open(); + + TXN_PREFIX_RDONLY(); + RCURSOR(block_info); + + MDB_val_set(result, height); + auto get_result = mdb_cursor_get(m_cur_block_info, (MDB_val *)&zerokval, &result, MDB_GET_BOTH); + if (get_result == MDB_NOTFOUND) + { + throw0(BLOCK_DNE(std::string("Attempt to get cumulative difficulty from height ").append(boost::lexical_cast(height)).append(" failed -- difficulty not in db").c_str())); + } + else if (get_result) + throw0(DB_ERROR("Error attempting to retrieve a cumulative difficulty from the db")); + + mdb_block_info *bi = (mdb_block_info *)result.mv_data; + difficulty_type ret = bi->bi_diff_hi; + ret <<= 64; + ret |= bi->bi_diff_lo; + TXN_POSTFIX_RDONLY(); + return ret; +} + +difficulty_type BlockchainLMDB::get_block_difficulty(const uint64_t& height) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + difficulty_type diff1 = 0; + difficulty_type diff2 = 0; + + diff1 = get_block_cumulative_difficulty(height); + if (height != 0) + { + diff2 = get_block_cumulative_difficulty(height - 1); + } + + return diff1 - diff2; +} + +uint64_t BlockchainLMDB::get_block_already_generated_coins(const uint64_t& height) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + TXN_PREFIX_RDONLY(); + RCURSOR(block_info); + + MDB_val_set(result, height); + auto get_result = mdb_cursor_get(m_cur_block_info, (MDB_val *)&zerokval, &result, MDB_GET_BOTH); + if (get_result == MDB_NOTFOUND) + { + throw0(BLOCK_DNE(std::string("Attempt to get generated coins from height ").append(boost::lexical_cast(height)).append(" failed -- block size not in db").c_str())); + } + else if (get_result) throw0(DB_ERROR("Error attempting to retrieve a total generated coins from the db")); mdb_block_info *bi = (mdb_block_info *)result.mv_data; @@ -2005,6 +2689,29 @@ uint64_t BlockchainLMDB::get_block_already_generated_coins(const uint64_t& heigh return ret; } +uint64_t BlockchainLMDB::get_block_long_term_weight(const uint64_t& height) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + TXN_PREFIX_RDONLY(); + RCURSOR(block_info); + + MDB_val_set(result, height); + auto get_result = mdb_cursor_get(m_cur_block_info, (MDB_val *)&zerokval, &result, MDB_GET_BOTH); + if (get_result == MDB_NOTFOUND) + { + throw0(BLOCK_DNE(std::string("Attempt to get block long term weight from height ").append(boost::lexical_cast(height)).append(" failed -- block info not in db").c_str())); + } + else if (get_result) + throw0(DB_ERROR("Error attempting to retrieve a long term block weight from the db")); + + mdb_block_info *bi = (mdb_block_info *)result.mv_data; + uint64_t ret = bi->bi_long_term_block_weight; + TXN_POSTFIX_RDONLY(); + return ret; +} + crypto::hash BlockchainLMDB::get_block_hash_from_height(const uint64_t& height) const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); @@ -2056,11 +2763,13 @@ std::vector BlockchainLMDB::get_hashes_range(const uint64_t& h1, c return v; } -crypto::hash BlockchainLMDB::top_block_hash() const +crypto::hash BlockchainLMDB::top_block_hash(uint64_t *block_height) const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); uint64_t m_height = height(); + if (block_height) + *block_height = m_height - 1; if (m_height != 0) { return get_block_hash_from_height(m_height - 1); @@ -2105,11 +2814,19 @@ uint64_t BlockchainLMDB::num_outputs() const TXN_PREFIX_RDONLY(); int result; - // get current height - MDB_stat db_stats; - if ((result = mdb_stat(m_txn, m_output_txs, &db_stats))) + RCURSOR(output_txs) + + uint64_t num = 0; + MDB_val k, v; + result = mdb_cursor_get(m_cur_output_txs, &k, &v, MDB_LAST); + if (result == MDB_NOTFOUND) + num = 0; + else if (result == 0) + num = 1 + ((const outtx*)v.mv_data)->output_id; + else throw0(DB_ERROR(lmdb_error("Failed to query m_output_txs: ", result).c_str())); - return db_stats.ms_entries; + + return num; } bool BlockchainLMDB::tx_exists(const crypto::hash& h) const @@ -2265,6 +2982,36 @@ bool BlockchainLMDB::get_pruned_tx_blob(const crypto::hash& h, cryptonote::blobd return true; } +bool BlockchainLMDB::get_prunable_tx_blob(const crypto::hash& h, cryptonote::blobdata &bd) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + TXN_PREFIX_RDONLY(); + RCURSOR(tx_indices); + RCURSOR(txs_prunable); + + MDB_val_set(v, h); + MDB_val result; + auto get_result = mdb_cursor_get(m_cur_tx_indices, (MDB_val *)&zerokval, &v, MDB_GET_BOTH); + if (get_result == 0) + { + const txindex *tip = (const txindex *)v.mv_data; + MDB_val_set(val_tx_id, tip->data.tx_id); + get_result = mdb_cursor_get(m_cur_txs_prunable, &val_tx_id, &result, MDB_SET); + } + if (get_result == MDB_NOTFOUND) + return false; + else if (get_result) + throw0(DB_ERROR(lmdb_error("DB error attempting to fetch tx from hash", get_result).c_str())); + + bd.assign(reinterpret_cast(result.mv_data), result.mv_size); + + TXN_POSTFIX_RDONLY(); + + return true; +} + bool BlockchainLMDB::get_prunable_tx_hash(const crypto::hash& tx_hash, crypto::hash &prunable_hash) const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); @@ -2373,56 +3120,7 @@ uint64_t BlockchainLMDB::get_num_outputs(const uint64_t& amount) const return num_elems; } -// This is a lot harder now that we've removed the output_keys index -output_data_t BlockchainLMDB::get_output_key(const uint64_t &global_index) const -{ - LOG_PRINT_L3("BlockchainLMDB::" << __func__ << " (unused version - does nothing)"); - check_open(); - TXN_PREFIX_RDONLY(); - RCURSOR(output_txs); - RCURSOR(tx_indices); - - output_data_t od; - MDB_val_set(v, global_index); - auto get_result = mdb_cursor_get(m_cur_output_txs, (MDB_val *)&zerokval, &v, MDB_GET_BOTH); - if (get_result == MDB_NOTFOUND) - throw1(OUTPUT_DNE("output with given index not in db")); - else if (get_result) - throw0(DB_ERROR("DB error attempting to fetch output tx hash")); - - outtx *ot = (outtx *)v.mv_data; - - MDB_val_set(val_h, ot->tx_hash); - get_result = mdb_cursor_get(m_cur_tx_indices, (MDB_val *)&zerokval, &val_h, MDB_GET_BOTH); - if (get_result) - throw0(DB_ERROR(lmdb_error(std::string("DB error attempting to fetch transaction index from hash ") + epee::string_tools::pod_to_hex(ot->tx_hash) + ": ", get_result).c_str())); - - txindex *tip = (txindex *)val_h.mv_data; - MDB_val_set(val_tx_id, tip->data.tx_id); - MDB_val result; - get_result = mdb_cursor_get(m_cur_txs_pruned, &val_tx_id, &result, MDB_SET); - if (get_result == MDB_NOTFOUND) - throw1(TX_DNE(std::string("tx with hash ").append(epee::string_tools::pod_to_hex(ot->tx_hash)).append(" not found in db").c_str())); - else if (get_result) - throw0(DB_ERROR(lmdb_error("DB error attempting to fetch tx from hash", get_result).c_str())); - - blobdata bd; - bd.assign(reinterpret_cast(result.mv_data), result.mv_size); - - transaction tx; - if (!parse_and_validate_tx_base_from_blob(bd, tx)) - throw0(DB_ERROR("Failed to parse tx from blob retrieved from the db")); - - const tx_out tx_output = tx.vout[ot->local_index]; - od.unlock_time = tip->data.unlock_time; - od.height = tip->data.block_id; - od.pubkey = boost::get(tx_output.target).key; - - TXN_POSTFIX_RDONLY(); - return od; -} - -output_data_t BlockchainLMDB::get_output_key(const uint64_t& amount, const uint64_t& index) +output_data_t BlockchainLMDB::get_output_key(const uint64_t& amount, const uint64_t& index, bool include_commitmemt) const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); @@ -2434,7 +3132,8 @@ output_data_t BlockchainLMDB::get_output_key(const uint64_t& amount, const uint6 MDB_val_set(v, index); auto get_result = mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_GET_BOTH); if (get_result == MDB_NOTFOUND) - throw1(OUTPUT_DNE("Attempting to get output pubkey by index, but key does not exist")); + throw1(OUTPUT_DNE(std::string("Attempting to get output pubkey by index, but key does not exist: amount " + + std::to_string(amount) + ", index " + std::to_string(index)).c_str())); else if (get_result) throw0(DB_ERROR("Error attempting to retrieve an output pubkey from the db")); @@ -2448,7 +3147,8 @@ output_data_t BlockchainLMDB::get_output_key(const uint64_t& amount, const uint6 { const pre_rct_outkey *okp = (const pre_rct_outkey *)v.mv_data; memcpy(&ret, &okp->data, sizeof(pre_rct_output_data_t));; - ret.commitment = rct::zeroCommit(amount); + if (include_commitmemt) + ret.commitment = rct::zeroCommit(amount); } TXN_POSTFIX_RDONLY(); return ret; @@ -2490,7 +3190,7 @@ tx_out_index BlockchainLMDB::get_output_tx_and_index(const uint64_t& amount, con return indices[0]; } -std::vector BlockchainLMDB::get_tx_amount_output_indices(const uint64_t tx_id) const +std::vector> BlockchainLMDB::get_tx_amount_output_indices(uint64_t tx_id, size_t n_txes) const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); @@ -2499,35 +3199,40 @@ std::vector BlockchainLMDB::get_tx_amount_output_indices(const uint64_ TXN_PREFIX_RDONLY(); RCURSOR(tx_outputs); - int result = 0; MDB_val_set(k_tx_id, tx_id); MDB_val v; - std::vector amount_output_indices; + std::vector> amount_output_indices_set; + amount_output_indices_set.reserve(n_txes); - result = mdb_cursor_get(m_cur_tx_outputs, &k_tx_id, &v, MDB_SET); - if (result == MDB_NOTFOUND) - LOG_PRINT_L0("WARNING: Unexpected: tx has no amount indices stored in " - "tx_outputs, but it should have an empty entry even if it's a tx without " - "outputs"); - else if (result) - throw0(DB_ERROR(lmdb_error("DB error attempting to get data for tx_outputs[tx_index]", result).c_str())); + MDB_cursor_op op = MDB_SET; + while (n_txes-- > 0) + { + int result = mdb_cursor_get(m_cur_tx_outputs, &k_tx_id, &v, op); + if (result == MDB_NOTFOUND) + LOG_PRINT_L0("WARNING: Unexpected: tx has no amount indices stored in " + "tx_outputs, but it should have an empty entry even if it's a tx without " + "outputs"); + else if (result) + throw0(DB_ERROR(lmdb_error("DB error attempting to get data for tx_outputs[tx_index]", result).c_str())); - const uint64_t* indices = (const uint64_t*)v.mv_data; - int num_outputs = v.mv_size / sizeof(uint64_t); + op = MDB_NEXT; - amount_output_indices.reserve(num_outputs); - for (int i = 0; i < num_outputs; ++i) - { - // LOG_PRINT_L0("amount output index[" << 2*i << "]" << ": " << paired_indices[2*i] << " global output index: " << paired_indices[2*i+1]); - amount_output_indices.push_back(indices[i]); + const uint64_t* indices = (const uint64_t*)v.mv_data; + size_t num_outputs = v.mv_size / sizeof(uint64_t); + + amount_output_indices_set.resize(amount_output_indices_set.size() + 1); + std::vector &amount_output_indices = amount_output_indices_set.back(); + amount_output_indices.reserve(num_outputs); + for (size_t i = 0; i < num_outputs; ++i) + { + amount_output_indices.push_back(indices[i]); + } } - indices = nullptr; TXN_POSTFIX_RDONLY(); - return amount_output_indices; + return amount_output_indices_set; } - bool BlockchainLMDB::has_key_image(const crypto::key_image& img) const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); @@ -2904,10 +3609,10 @@ void BlockchainLMDB::set_batch_transactions(bool batch_transactions) LOG_PRINT_L3("BlockchainLMDB::" << __func__); if ((batch_transactions) && (m_batch_transactions)) { - LOG_PRINT_L0("WARNING: batch transaction mode already enabled, but asked to enable batch mode"); + MINFO("batch transaction mode already enabled, but asked to enable batch mode"); } m_batch_transactions = batch_transactions; - LOG_PRINT_L3("batch transactions " << (m_batch_transactions ? "enabled" : "disabled")); + MINFO("batch transactions " << (m_batch_transactions ? "enabled" : "disabled")); } // return true if we started the txn, false if already started @@ -2955,16 +3660,15 @@ void BlockchainLMDB::block_rtxn_stop() const memset(&m_tinfo->m_ti_rflags, 0, sizeof(m_tinfo->m_ti_rflags)); } -void BlockchainLMDB::block_txn_start(bool readonly) +bool BlockchainLMDB::block_rtxn_start() const { - if (readonly) - { - MDB_txn *mtxn; - mdb_txn_cursors *mcur; - block_rtxn_start(&mtxn, &mcur); - return; - } + MDB_txn *mtxn; + mdb_txn_cursors *mcur; + return block_rtxn_start(&mtxn, &mcur); +} +void BlockchainLMDB::block_wtxn_start() +{ LOG_PRINT_L3("BlockchainLMDB::" << __func__); // Distinguish the exceptions here from exceptions that would be thrown while // using the txn and committing it. @@ -2996,10 +3700,13 @@ void BlockchainLMDB::block_txn_start(bool readonly) throw0(DB_ERROR_TXN_START((std::string("Attempted to start new write txn when batch txn already exists in ")+__FUNCTION__).c_str())); } -void BlockchainLMDB::block_txn_stop() +void BlockchainLMDB::block_wtxn_stop() { LOG_PRINT_L3("BlockchainLMDB::" << __func__); - if (m_write_txn && m_writer == boost::this_thread::get_id()) + if (!m_write_txn) + throw0(DB_ERROR_TXN_START((std::string("Attempted to stop write txn when no such txn exists in ")+__FUNCTION__).c_str())); + if (m_writer != boost::this_thread::get_id()) + throw0(DB_ERROR_TXN_START((std::string("Attempted to stop write txn from the wrong thread in ")+__FUNCTION__).c_str())); { if (! m_batch_active) { @@ -3013,48 +3720,39 @@ void BlockchainLMDB::block_txn_stop() memset(&m_wcursors, 0, sizeof(m_wcursors)); } } - else if (m_tinfo->m_ti_rtxn) - { - mdb_txn_reset(m_tinfo->m_ti_rtxn); - memset(&m_tinfo->m_ti_rflags, 0, sizeof(m_tinfo->m_ti_rflags)); - } } -void BlockchainLMDB::block_txn_abort() +void BlockchainLMDB::block_wtxn_abort() { LOG_PRINT_L3("BlockchainLMDB::" << __func__); - if (m_write_txn && m_writer == boost::this_thread::get_id()) - { - if (! m_batch_active) - { - delete m_write_txn; - m_write_txn = nullptr; - memset(&m_wcursors, 0, sizeof(m_wcursors)); - } - } - else if (m_tinfo->m_ti_rtxn) - { - mdb_txn_reset(m_tinfo->m_ti_rtxn); - memset(&m_tinfo->m_ti_rflags, 0, sizeof(m_tinfo->m_ti_rflags)); - } - else + if (!m_write_txn) + throw0(DB_ERROR_TXN_START((std::string("Attempted to abort write txn when no such txn exists in ")+__FUNCTION__).c_str())); + if (m_writer != boost::this_thread::get_id()) + throw0(DB_ERROR_TXN_START((std::string("Attempted to abort write txn from the wrong thread in ")+__FUNCTION__).c_str())); + + if (! m_batch_active) { - // This would probably mean an earlier exception was caught, but then we - // proceeded further than we should have. - throw0(DB_ERROR((std::string("BlockchainLMDB::") + __func__ + - std::string(": block-level DB transaction abort called when write txn doesn't exist") - ).c_str())); + delete m_write_txn; + m_write_txn = nullptr; + memset(&m_wcursors, 0, sizeof(m_wcursors)); } } -uint64_t BlockchainLMDB::add_block(const block& blk, const size_t& block_size, const difficulty_type& cumulative_difficulty, const uint64_t& coins_generated, - const std::vector& txs) +void BlockchainLMDB::block_rtxn_abort() const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + mdb_txn_reset(m_tinfo->m_ti_rtxn); + memset(&m_tinfo->m_ti_rflags, 0, sizeof(m_tinfo->m_ti_rflags)); +} + +uint64_t BlockchainLMDB::add_block(const std::pair& blk, size_t block_weight, uint64_t long_term_block_weight, const difficulty_type& cumulative_difficulty, const uint64_t& coins_generated, + const std::vector>& txs) { LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); uint64_t m_height = height(); - if (m_height % 1000 == 0) + if (m_height % 1024 == 0) { // for batch mode, DB resize check is done at start of batch transaction if (! m_batch_active && need_resize()) @@ -3066,17 +3764,12 @@ uint64_t BlockchainLMDB::add_block(const block& blk, const size_t& block_size, c try { - BlockchainDB::add_block(blk, block_size, cumulative_difficulty, coins_generated, txs); + BlockchainDB::add_block(blk, block_weight, long_term_block_weight, cumulative_difficulty, coins_generated, txs); } catch (const DB_ERROR_TXN_START& e) { throw; } - catch (...) - { - block_txn_abort(); - throw; - } return ++m_height; } @@ -3086,16 +3779,16 @@ void BlockchainLMDB::pop_block(block& blk, std::vector& txs) LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); - block_txn_start(false); + block_wtxn_start(); try { BlockchainDB::pop_block(blk, txs); - block_txn_stop(); + block_wtxn_stop(); } catch (...) { - block_txn_abort(); + block_wtxn_abort(); throw; } } @@ -3106,6 +3799,7 @@ void BlockchainLMDB::get_output_tx_and_index_from_global(const std::vectortx_hash, ot->local_index); - tx_out_indices.push_back(result); + const outtx *ot = (const outtx *)v.mv_data; + tx_out_indices.push_back(tx_out_index(ot->tx_hash, ot->local_index)); } TXN_POSTFIX_RDONLY(); } -void BlockchainLMDB::get_output_key(const uint64_t &amount, const std::vector &offsets, std::vector &outputs, bool allow_partial) +void BlockchainLMDB::get_output_key(const epee::span &amounts, const std::vector &offsets, std::vector &outputs, bool allow_partial) const { + if (amounts.size() != 1 && amounts.size() != offsets.size()) + throw0(DB_ERROR("Invalid sizes of amounts and offets")); + LOG_PRINT_L3("BlockchainLMDB::" << __func__); TIME_MEASURE_START(db3); check_open(); outputs.clear(); + outputs.reserve(offsets.size()); TXN_PREFIX_RDONLY(); RCURSOR(output_amounts); - MDB_val_set(k, amount); - for (const uint64_t &index : offsets) + for (size_t i = 0; i < offsets.size(); ++i) { - MDB_val_set(v, index); + const uint64_t amount = amounts.size() == 1 ? amounts[0] : amounts[i]; + MDB_val_set(k, amount); + MDB_val_set(v, offsets[i]); auto get_result = mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_GET_BOTH); if (get_result == MDB_NOTFOUND) @@ -3152,24 +3850,24 @@ void BlockchainLMDB::get_output_key(const uint64_t &amount, const std::vector(amount) + ", index " + boost::lexical_cast(index) + ", count " + boost::lexical_cast(get_num_outputs(amount)) + "), but key does not exist (current height " + boost::lexical_cast(height()) + ")").c_str())); + throw1(OUTPUT_DNE((std::string("Attempting to get output pubkey by global index (amount ") + boost::lexical_cast(amount) + ", index " + boost::lexical_cast(offsets[i]) + ", count " + boost::lexical_cast(get_num_outputs(amount)) + "), but key does not exist (current height " + boost::lexical_cast(height()) + ")").c_str())); } else if (get_result) throw0(DB_ERROR(lmdb_error("Error attempting to retrieve an output pubkey from the db", get_result).c_str())); - output_data_t data; if (amount == 0) { const outkey *okp = (const outkey *)v.mv_data; - data = okp->data; + outputs.push_back(okp->data); } else { const pre_rct_outkey *okp = (const pre_rct_outkey *)v.mv_data; + outputs.resize(outputs.size() + 1); + output_data_t &data = outputs.back(); memcpy(&data, &okp->data, sizeof(pre_rct_output_data_t)); data.commitment = rct::zeroCommit(amount); } - outputs.push_back(data); } TXN_POSTFIX_RDONLY(); @@ -3185,6 +3883,7 @@ void BlockchainLMDB::get_output_tx_and_index(const uint64_t& amount, const std:: indices.clear(); std::vector tx_indices; + tx_indices.reserve(offsets.size()); TXN_PREFIX_RDONLY(); RCURSOR(output_amounts); @@ -3324,6 +4023,7 @@ bool BlockchainLMDB::get_output_distribution(uint64_t amount, uint64_t from_heig MDB_val_set(k, amount); MDB_val v; MDB_cursor_op op = MDB_SET; + base = 0; while (1) { int ret = mdb_cursor_get(m_cur_output_amounts, &k, &v, op); @@ -3342,6 +4042,11 @@ bool BlockchainLMDB::get_output_distribution(uint64_t amount, uint64_t from_heig break; } + distribution[0] += base; + for (size_t n = 1; n < distribution.size(); ++n) + distribution[n] += distribution[n - 1]; + base = 0; + TXN_POSTFIX_RDONLY(); return true; @@ -3419,6 +4124,16 @@ bool BlockchainLMDB::is_read_only() const return false; } +uint64_t BlockchainLMDB::get_database_size() const +{ + uint64_t size = 0; + boost::filesystem::path datafile(m_folder); + datafile /= CRYPTONOTE_BLOCKCHAINDATA_FILENAME; + if (!epee::file_io_utils::get_file_size(datafile.string(), size)) + size = 0; + return size; +} + void BlockchainLMDB::fixup() { LOG_PRINT_L3("BlockchainLMDB::" << __func__); @@ -3426,7 +4141,17 @@ void BlockchainLMDB::fixup() BlockchainDB::fixup(); } -#define RENAME_DB(name) \ +#define RENAME_DB(name) do { \ + char n2[] = name; \ + MDB_dbi tdbi; \ + n2[sizeof(n2)-2]--; \ + /* play some games to put (name) on a writable page */ \ + result = mdb_dbi_open(txn, n2, MDB_CREATE, &tdbi); \ + if (result) \ + throw0(DB_ERROR(lmdb_error("Failed to create " + std::string(n2) + ": ", result).c_str())); \ + result = mdb_drop(txn, tdbi, 1); \ + if (result) \ + throw0(DB_ERROR(lmdb_error("Failed to delete " + std::string(n2) + ": ", result).c_str())); \ k.mv_data = (void *)name; \ k.mv_size = sizeof(name)-1; \ result = mdb_cursor_open(txn, 1, &c_cur); \ @@ -3436,7 +4161,7 @@ void BlockchainLMDB::fixup() if (result) \ throw0(DB_ERROR(lmdb_error("Failed to get DB record for " name ": ", result).c_str())); \ ptr = (char *)k.mv_data; \ - ptr[sizeof(name)-2] = 's' + ptr[sizeof(name)-2]++; } while(0) #define LOGIF(y) if (ELPP->vRegistry()->allowed(y, "global")) @@ -3515,7 +4240,9 @@ void BlockchainLMDB::migrate_0_1() throw0(DB_ERROR(lmdb_error("Failed to open a cursor for block_heights: ", result).c_str())); if (!i) { MDB_stat ms; - mdb_stat(txn, m_block_heights, &ms); + result = mdb_stat(txn, m_block_heights, &ms); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to query block_heights table: ", result).c_str())); i = ms.ms_entries; } } @@ -3576,7 +4303,7 @@ void BlockchainLMDB::migrate_0_1() break; } MDB_dbi diffs, hashes, sizes, timestamps; - mdb_block_info bi; + mdb_block_info_1 bi; MDB_val_set(nv, bi); lmdb_db_open(txn, "block_diffs", 0, diffs, "Failed to open db handle for block_diffs"); @@ -3618,7 +4345,9 @@ void BlockchainLMDB::migrate_0_1() throw0(DB_ERROR(lmdb_error("Failed to open a cursor for block_timestamps: ", result).c_str())); if (!i) { MDB_stat ms; - mdb_stat(txn, m_block_info, &ms); + result = mdb_stat(txn, m_block_info, &ms); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to query block_info table: ", result).c_str())); i = ms.ms_entries; } } @@ -3641,9 +4370,9 @@ void BlockchainLMDB::migrate_0_1() if (result) throw0(DB_ERROR(lmdb_error("Failed to get a record from block_sizes: ", result).c_str())); if (v.mv_size == sizeof(uint32_t)) - bi.bi_size = *(uint32_t *)v.mv_data; + bi.bi_weight = *(uint32_t *)v.mv_data; else - bi.bi_size = *(uint64_t *)v.mv_data; // this is a 32/64 compat bug in version 0 + bi.bi_weight = *(uint64_t *)v.mv_data; // this is a 32/64 compat bug in version 0 result = mdb_cursor_get(c_timestamps, &k, &v, MDB_NEXT); if (result) throw0(DB_ERROR(lmdb_error("Failed to get a record from block_timestamps: ", result).c_str())); @@ -3738,7 +4467,9 @@ void BlockchainLMDB::migrate_0_1() throw0(DB_ERROR(lmdb_error("Failed to open a cursor for spent_keys: ", result).c_str())); if (!i) { MDB_stat ms; - mdb_stat(txn, m_hf_versions, &ms); + result = mdb_stat(txn, m_hf_versions, &ms); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to query hf_versions table: ", result).c_str())); i = ms.ms_entries; } } @@ -3893,7 +4624,9 @@ void BlockchainLMDB::migrate_0_1() throw0(DB_ERROR(lmdb_error("Failed to open a cursor for txs: ", result).c_str())); if (!i) { MDB_stat ms; - mdb_stat(txn, m_txs, &ms); + result = mdb_stat(txn, m_txs, &ms); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to query txs table: ", result).c_str())); i = ms.ms_entries; if (i) { MDB_val_set(pk, "txblk"); @@ -3927,7 +4660,7 @@ void BlockchainLMDB::migrate_0_1() if (!parse_and_validate_block_from_blob(bd, b)) throw0(DB_ERROR("Failed to parse block from blob retrieved from the db")); - add_transaction(null_hash, b.miner_tx); + add_transaction(null_hash, std::make_pair(b.miner_tx, tx_to_blob(b.miner_tx))); for (unsigned int j = 0; j(v.mv_data), v.mv_size); if (!parse_and_validate_tx_from_blob(bd, tx)) throw0(DB_ERROR("Failed to parse tx from blob retrieved from the db")); - add_transaction(null_hash, tx, &b.tx_hashes[j]); + add_transaction(null_hash, std::make_pair(std::move(tx), bd), &b.tx_hashes[j]); result = mdb_cursor_del(c_txs, 0); if (result) throw0(DB_ERROR(lmdb_error("Failed to get record from txs: ", result).c_str())); @@ -3964,7 +4697,7 @@ void BlockchainLMDB::migrate_0_1() uint32_t version = 1; v.mv_data = (void *)&version; v.mv_size = sizeof(version); - MDB_val_copy vk("version"); + MDB_val_str(vk, "version"); result = mdb_txn_begin(m_env, NULL, 0, txn); if (result) throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); @@ -4106,7 +4839,430 @@ void BlockchainLMDB::migrate_1_2() uint32_t version = 2; v.mv_data = (void *)&version; v.mv_size = sizeof(version); - MDB_val_copy vk("version"); + MDB_val_str(vk, "version"); + result = mdb_txn_begin(m_env, NULL, 0, txn); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); + result = mdb_put(txn, m_properties, &vk, &v, 0); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to update version for the db: ", result).c_str())); + txn.commit(); +} + +void BlockchainLMDB::migrate_2_3() +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + uint64_t i; + int result; + mdb_txn_safe txn(false); + MDB_val k, v; + char *ptr; + + MGINFO_YELLOW("Migrating blockchain from DB version 2 to 3 - this may take a while:"); + + do { + LOG_PRINT_L1("migrating block info:"); + + result = mdb_txn_begin(m_env, NULL, 0, txn); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); + + MDB_stat db_stats; + if ((result = mdb_stat(txn, m_blocks, &db_stats))) + throw0(DB_ERROR(lmdb_error("Failed to query m_blocks: ", result).c_str())); + const uint64_t blockchain_height = db_stats.ms_entries; + + MDEBUG("enumerating rct outputs..."); + std::vector distribution(blockchain_height, 0); + bool r = for_all_outputs(0, [&](uint64_t height) { + if (height >= blockchain_height) + { + MERROR("Output found claiming height >= blockchain height"); + return false; + } + distribution[height]++; + return true; + }); + if (!r) + throw0(DB_ERROR("Failed to build rct output distribution")); + for (size_t i = 1; i < distribution.size(); ++i) + distribution[i] += distribution[i - 1]; + + /* the block_info table name is the same but the old version and new version + * have incompatible data. Create a new table. We want the name to be similar + * to the old name so that it will occupy the same location in the DB. + */ + MDB_dbi o_block_info = m_block_info; + lmdb_db_open(txn, "block_infn", MDB_INTEGERKEY | MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED, m_block_info, "Failed to open db handle for block_infn"); + mdb_set_dupsort(txn, m_block_info, compare_uint64); + + MDB_cursor *c_old, *c_cur; + i = 0; + while(1) { + if (!(i % 1000)) { + if (i) { + LOGIF(el::Level::Info) { + std::cout << i << " / " << blockchain_height << " \r" << std::flush; + } + txn.commit(); + result = mdb_txn_begin(m_env, NULL, 0, txn); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); + } + result = mdb_cursor_open(txn, m_block_info, &c_cur); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for block_infn: ", result).c_str())); + result = mdb_cursor_open(txn, o_block_info, &c_old); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for block_info: ", result).c_str())); + if (!i) { + MDB_stat db_stat; + result = mdb_stat(txn, m_block_info, &db_stats); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to query m_block_info: ", result).c_str())); + i = db_stats.ms_entries; + } + } + result = mdb_cursor_get(c_old, &k, &v, MDB_NEXT); + if (result == MDB_NOTFOUND) { + txn.commit(); + break; + } + else if (result) + throw0(DB_ERROR(lmdb_error("Failed to get a record from block_info: ", result).c_str())); + const mdb_block_info_1 *bi_old = (const mdb_block_info_1*)v.mv_data; + mdb_block_info_2 bi; + bi.bi_height = bi_old->bi_height; + bi.bi_timestamp = bi_old->bi_timestamp; + bi.bi_coins = bi_old->bi_coins; + bi.bi_weight = bi_old->bi_weight; + bi.bi_diff = bi_old->bi_diff; + bi.bi_hash = bi_old->bi_hash; + if (bi_old->bi_height >= distribution.size()) + throw0(DB_ERROR("Bad height in block_info record")); + bi.bi_cum_rct = distribution[bi_old->bi_height]; + MDB_val_set(nv, bi); + result = mdb_cursor_put(c_cur, (MDB_val *)&zerokval, &nv, MDB_APPENDDUP); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to put a record into block_infn: ", result).c_str())); + /* we delete the old records immediately, so the overall DB and mapsize should not grow. + * This is a little slower than just letting mdb_drop() delete it all at the end, but + * it saves a significant amount of disk space. + */ + result = mdb_cursor_del(c_old, 0); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to delete a record from block_info: ", result).c_str())); + i++; + } + + result = mdb_txn_begin(m_env, NULL, 0, txn); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); + /* Delete the old table */ + result = mdb_drop(txn, o_block_info, 1); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to delete old block_info table: ", result).c_str())); + + RENAME_DB("block_infn"); + mdb_dbi_close(m_env, m_block_info); + + lmdb_db_open(txn, "block_info", MDB_INTEGERKEY | MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED, m_block_info, "Failed to open db handle for block_infn"); + mdb_set_dupsort(txn, m_block_info, compare_uint64); + + txn.commit(); + } while(0); + + uint32_t version = 3; + v.mv_data = (void *)&version; + v.mv_size = sizeof(version); + MDB_val_str(vk, "version"); + result = mdb_txn_begin(m_env, NULL, 0, txn); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); + result = mdb_put(txn, m_properties, &vk, &v, 0); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to update version for the db: ", result).c_str())); + txn.commit(); +} + +void BlockchainLMDB::migrate_3_4() +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + uint64_t i; + int result; + mdb_txn_safe txn(false); + MDB_val k, v; + char *ptr; + bool past_long_term_weight = false; + + MGINFO_YELLOW("Migrating blockchain from DB version 3 to 4 - this may take a while:"); + + do { + LOG_PRINT_L1("migrating block info:"); + + result = mdb_txn_begin(m_env, NULL, 0, txn); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); + + MDB_stat db_stats; + if ((result = mdb_stat(txn, m_blocks, &db_stats))) + throw0(DB_ERROR(lmdb_error("Failed to query m_blocks: ", result).c_str())); + const uint64_t blockchain_height = db_stats.ms_entries; + + boost::circular_buffer long_term_block_weights(CRYPTONOTE_LONG_TERM_BLOCK_WEIGHT_WINDOW_SIZE); + + /* the block_info table name is the same but the old version and new version + * have incompatible data. Create a new table. We want the name to be similar + * to the old name so that it will occupy the same location in the DB. + */ + MDB_dbi o_block_info = m_block_info; + lmdb_db_open(txn, "block_infn", MDB_INTEGERKEY | MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED, m_block_info, "Failed to open db handle for block_infn"); + mdb_set_dupsort(txn, m_block_info, compare_uint64); + + + MDB_cursor *c_blocks; + result = mdb_cursor_open(txn, m_blocks, &c_blocks); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for blocks: ", result).c_str())); + + MDB_cursor *c_old, *c_cur; + i = 0; + while(1) { + if (!(i % 1000)) { + if (i) { + LOGIF(el::Level::Info) { + std::cout << i << " / " << blockchain_height << " \r" << std::flush; + } + txn.commit(); + result = mdb_txn_begin(m_env, NULL, 0, txn); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); + } + result = mdb_cursor_open(txn, m_block_info, &c_cur); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for block_infn: ", result).c_str())); + result = mdb_cursor_open(txn, o_block_info, &c_old); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for block_info: ", result).c_str())); + result = mdb_cursor_open(txn, m_blocks, &c_blocks); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for blocks: ", result).c_str())); + if (!i) { + MDB_stat db_stat; + result = mdb_stat(txn, m_block_info, &db_stats); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to query m_block_info: ", result).c_str())); + i = db_stats.ms_entries; + } + } + result = mdb_cursor_get(c_old, &k, &v, MDB_NEXT); + if (result == MDB_NOTFOUND) { + txn.commit(); + break; + } + else if (result) + throw0(DB_ERROR(lmdb_error("Failed to get a record from block_info: ", result).c_str())); + const mdb_block_info_2 *bi_old = (const mdb_block_info_2*)v.mv_data; + mdb_block_info_3 bi; + bi.bi_height = bi_old->bi_height; + bi.bi_timestamp = bi_old->bi_timestamp; + bi.bi_coins = bi_old->bi_coins; + bi.bi_weight = bi_old->bi_weight; + bi.bi_diff = bi_old->bi_diff; + bi.bi_hash = bi_old->bi_hash; + bi.bi_cum_rct = bi_old->bi_cum_rct; + + // get block major version to determine which rule is in place + if (!past_long_term_weight) + { + MDB_val_copy kb(bi.bi_height); + MDB_val vb; + result = mdb_cursor_get(c_blocks, &kb, &vb, MDB_SET); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to query m_blocks: ", result).c_str())); + if (vb.mv_size == 0) + throw0(DB_ERROR("Invalid data from m_blocks")); + const uint8_t block_major_version = *((const uint8_t*)vb.mv_data); + if (block_major_version >= HF_VERSION_LONG_TERM_BLOCK_WEIGHT) + past_long_term_weight = true; + } + + uint64_t long_term_block_weight; + if (past_long_term_weight) + { + std::vector weights(long_term_block_weights.begin(), long_term_block_weights.end()); + uint64_t long_term_effective_block_median_weight = std::max(CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V5, epee::misc_utils::median(weights)); + long_term_block_weight = std::min(bi.bi_weight, long_term_effective_block_median_weight + long_term_effective_block_median_weight * 2 / 5); + } + else + { + long_term_block_weight = bi.bi_weight; + } + long_term_block_weights.push_back(long_term_block_weight); + bi.bi_long_term_block_weight = long_term_block_weight; + + MDB_val_set(nv, bi); + result = mdb_cursor_put(c_cur, (MDB_val *)&zerokval, &nv, MDB_APPENDDUP); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to put a record into block_infn: ", result).c_str())); + /* we delete the old records immediately, so the overall DB and mapsize should not grow. + * This is a little slower than just letting mdb_drop() delete it all at the end, but + * it saves a significant amount of disk space. + */ + result = mdb_cursor_del(c_old, 0); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to delete a record from block_info: ", result).c_str())); + i++; + } + + result = mdb_txn_begin(m_env, NULL, 0, txn); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); + /* Delete the old table */ + result = mdb_drop(txn, o_block_info, 1); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to delete old block_info table: ", result).c_str())); + + RENAME_DB("block_infn"); + mdb_dbi_close(m_env, m_block_info); + + lmdb_db_open(txn, "block_info", MDB_INTEGERKEY | MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED, m_block_info, "Failed to open db handle for block_infn"); + mdb_set_dupsort(txn, m_block_info, compare_uint64); + + txn.commit(); + } while(0); + + uint32_t version = 4; + v.mv_data = (void *)&version; + v.mv_size = sizeof(version); + MDB_val_str(vk, "version"); + result = mdb_txn_begin(m_env, NULL, 0, txn); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); + result = mdb_put(txn, m_properties, &vk, &v, 0); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to update version for the db: ", result).c_str())); + txn.commit(); +} + +void BlockchainLMDB::migrate_4_5() +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + uint64_t i; + int result; + mdb_txn_safe txn(false); + MDB_val k, v; + char *ptr; + + MGINFO_YELLOW("Migrating blockchain from DB version 4 to 5 - this may take a while:"); + + do { + LOG_PRINT_L1("migrating block info:"); + + result = mdb_txn_begin(m_env, NULL, 0, txn); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); + + MDB_stat db_stats; + if ((result = mdb_stat(txn, m_blocks, &db_stats))) + throw0(DB_ERROR(lmdb_error("Failed to query m_blocks: ", result).c_str())); + const uint64_t blockchain_height = db_stats.ms_entries; + + /* the block_info table name is the same but the old version and new version + * have incompatible data. Create a new table. We want the name to be similar + * to the old name so that it will occupy the same location in the DB. + */ + MDB_dbi o_block_info = m_block_info; + lmdb_db_open(txn, "block_infn", MDB_INTEGERKEY | MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED, m_block_info, "Failed to open db handle for block_infn"); + mdb_set_dupsort(txn, m_block_info, compare_uint64); + + + MDB_cursor *c_blocks; + result = mdb_cursor_open(txn, m_blocks, &c_blocks); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for blocks: ", result).c_str())); + + MDB_cursor *c_old, *c_cur; + i = 0; + while(1) { + if (!(i % 1000)) { + if (i) { + LOGIF(el::Level::Info) { + std::cout << i << " / " << blockchain_height << " \r" << std::flush; + } + txn.commit(); + result = mdb_txn_begin(m_env, NULL, 0, txn); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); + } + result = mdb_cursor_open(txn, m_block_info, &c_cur); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for block_infn: ", result).c_str())); + result = mdb_cursor_open(txn, o_block_info, &c_old); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for block_info: ", result).c_str())); + if (!i) { + MDB_stat db_stat; + result = mdb_stat(txn, m_block_info, &db_stats); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to query m_block_info: ", result).c_str())); + i = db_stats.ms_entries; + } + } + result = mdb_cursor_get(c_old, &k, &v, MDB_NEXT); + if (result == MDB_NOTFOUND) { + txn.commit(); + break; + } + else if (result) + throw0(DB_ERROR(lmdb_error("Failed to get a record from block_info: ", result).c_str())); + const mdb_block_info_3 *bi_old = (const mdb_block_info_3*)v.mv_data; + mdb_block_info_4 bi; + bi.bi_height = bi_old->bi_height; + bi.bi_timestamp = bi_old->bi_timestamp; + bi.bi_coins = bi_old->bi_coins; + bi.bi_weight = bi_old->bi_weight; + bi.bi_diff_lo = bi_old->bi_diff; + bi.bi_diff_hi = 0; + bi.bi_hash = bi_old->bi_hash; + bi.bi_cum_rct = bi_old->bi_cum_rct; + bi.bi_long_term_block_weight = bi_old->bi_long_term_block_weight; + + MDB_val_set(nv, bi); + result = mdb_cursor_put(c_cur, (MDB_val *)&zerokval, &nv, MDB_APPENDDUP); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to put a record into block_infn: ", result).c_str())); + /* we delete the old records immediately, so the overall DB and mapsize should not grow. + * This is a little slower than just letting mdb_drop() delete it all at the end, but + * it saves a significant amount of disk space. + */ + result = mdb_cursor_del(c_old, 0); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to delete a record from block_info: ", result).c_str())); + i++; + } + + result = mdb_txn_begin(m_env, NULL, 0, txn); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); + /* Delete the old table */ + result = mdb_drop(txn, o_block_info, 1); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to delete old block_info table: ", result).c_str())); + + RENAME_DB("block_infn"); + mdb_dbi_close(m_env, m_block_info); + + lmdb_db_open(txn, "block_info", MDB_INTEGERKEY | MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED, m_block_info, "Failed to open db handle for block_infn"); + mdb_set_dupsort(txn, m_block_info, compare_uint64); + + txn.commit(); + } while(0); + + uint32_t version = 5; + v.mv_data = (void *)&version; + v.mv_size = sizeof(version); + MDB_val_str(vk, "version"); result = mdb_txn_begin(m_env, NULL, 0, txn); if (result) throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); @@ -4118,14 +5274,16 @@ void BlockchainLMDB::migrate_1_2() void BlockchainLMDB::migrate(const uint32_t oldversion) { - switch(oldversion) { - case 0: - migrate_0_1(); /* FALLTHRU */ - case 1: - migrate_1_2(); /* FALLTHRU */ - default: - ; - } + if (oldversion < 1) + migrate_0_1(); + if (oldversion < 2) + migrate_1_2(); + if (oldversion < 3) + migrate_2_3(); + if (oldversion < 4) + migrate_3_4(); + if (oldversion < 5) + migrate_4_5(); } } // namespace cryptonote diff --git a/src/blockchain_db/lmdb/db_lmdb.h b/src/blockchain_db/lmdb/db_lmdb.h index cc1b06ca02c..4b46f081e23 100644 --- a/src/blockchain_db/lmdb/db_lmdb.h +++ b/src/blockchain_db/lmdb/db_lmdb.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // All rights reserved. // // Redistribution and use in source and binary forms, with or without modification, are @@ -40,6 +40,11 @@ namespace cryptonote { +typedef struct txindex { + crypto::hash key; + tx_data_t data; +} txindex; + typedef struct mdb_txn_cursors { MDB_cursor *m_txc_blocks; @@ -53,6 +58,7 @@ typedef struct mdb_txn_cursors MDB_cursor *m_txc_txs_pruned; MDB_cursor *m_txc_txs_prunable; MDB_cursor *m_txc_txs_prunable_hash; + MDB_cursor *m_txc_txs_prunable_tip; MDB_cursor *m_txc_tx_indices; MDB_cursor *m_txc_tx_outputs; @@ -62,6 +68,8 @@ typedef struct mdb_txn_cursors MDB_cursor *m_txc_txpool_blob; MDB_cursor *m_txc_hf_versions; + + MDB_cursor *m_txc_properties; } mdb_txn_cursors; #define m_cur_blocks m_cursors->m_txc_blocks @@ -73,12 +81,14 @@ typedef struct mdb_txn_cursors #define m_cur_txs_pruned m_cursors->m_txc_txs_pruned #define m_cur_txs_prunable m_cursors->m_txc_txs_prunable #define m_cur_txs_prunable_hash m_cursors->m_txc_txs_prunable_hash +#define m_cur_txs_prunable_tip m_cursors->m_txc_txs_prunable_tip #define m_cur_tx_indices m_cursors->m_txc_tx_indices #define m_cur_tx_outputs m_cursors->m_txc_tx_outputs #define m_cur_spent_keys m_cursors->m_txc_spent_keys #define m_cur_txpool_meta m_cursors->m_txc_txpool_meta #define m_cur_txpool_blob m_cursors->m_txc_txpool_blob #define m_cur_hf_versions m_cursors->m_txc_hf_versions +#define m_cur_properties m_cursors->m_txc_properties typedef struct mdb_rflags { @@ -92,12 +102,14 @@ typedef struct mdb_rflags bool m_rf_txs_pruned; bool m_rf_txs_prunable; bool m_rf_txs_prunable_hash; + bool m_rf_txs_prunable_tip; bool m_rf_tx_indices; bool m_rf_tx_outputs; bool m_rf_spent_keys; bool m_rf_txpool_meta; bool m_rf_txpool_blob; bool m_rf_hf_versions; + bool m_rf_properties; } mdb_rflags; typedef struct mdb_threadinfo @@ -166,7 +178,7 @@ struct mdb_txn_safe class BlockchainLMDB : public BlockchainDB { public: - BlockchainLMDB(bool batch_transactions=false); + BlockchainLMDB(bool batch_transactions=true); ~BlockchainLMDB(); virtual void open(const std::string& filename, const int mdb_flags=0); @@ -181,6 +193,8 @@ class BlockchainLMDB : public BlockchainDB virtual std::vector get_filenames() const; + virtual bool remove_data_file(const std::string& folder) const; + virtual std::string get_db_name() const; virtual bool lock(); @@ -197,11 +211,15 @@ class BlockchainLMDB : public BlockchainDB virtual cryptonote::blobdata get_block_blob_from_height(const uint64_t& height) const; + virtual std::vector get_block_cumulative_rct_outputs(const std::vector &heights) const; + virtual uint64_t get_block_timestamp(const uint64_t& height) const; virtual uint64_t get_top_block_timestamp() const; - virtual size_t get_block_size(const uint64_t& height) const; + virtual size_t get_block_weight(const uint64_t& height) const; + + virtual std::vector get_block_weights(uint64_t start_height, size_t count) const; virtual difficulty_type get_block_cumulative_difficulty(const uint64_t& height) const; @@ -209,13 +227,17 @@ class BlockchainLMDB : public BlockchainDB virtual uint64_t get_block_already_generated_coins(const uint64_t& height) const; + virtual uint64_t get_block_long_term_weight(const uint64_t& height) const; + + virtual std::vector get_long_term_block_weights(uint64_t start_height, size_t count) const; + virtual crypto::hash get_block_hash_from_height(const uint64_t& height) const; virtual std::vector get_blocks_range(const uint64_t& h1, const uint64_t& h2) const; virtual std::vector get_hashes_range(const uint64_t& h1, const uint64_t& h2) const; - virtual crypto::hash top_block_hash() const; + virtual crypto::hash top_block_hash(uint64_t *block_height = NULL) const; virtual block get_top_block() const; @@ -228,6 +250,7 @@ class BlockchainLMDB : public BlockchainDB virtual bool get_tx_blob(const crypto::hash& h, cryptonote::blobdata &tx) const; virtual bool get_pruned_tx_blob(const crypto::hash& h, cryptonote::blobdata &tx) const; + virtual bool get_prunable_tx_blob(const crypto::hash& h, cryptonote::blobdata &tx) const; virtual bool get_prunable_tx_hash(const crypto::hash& tx_hash, crypto::hash &prunable_hash) const; virtual uint64_t get_tx_count() const; @@ -238,9 +261,8 @@ class BlockchainLMDB : public BlockchainDB virtual uint64_t get_num_outputs(const uint64_t& amount) const; - virtual output_data_t get_output_key(const uint64_t& amount, const uint64_t& index); - virtual output_data_t get_output_key(const uint64_t& global_index) const; - virtual void get_output_key(const uint64_t &amount, const std::vector &offsets, std::vector &outputs, bool allow_partial = false); + virtual output_data_t get_output_key(const uint64_t& amount, const uint64_t& index, bool include_commitmemt) const; + virtual void get_output_key(const epee::span &amounts, const std::vector &offsets, std::vector &outputs, bool allow_partial = false) const; virtual tx_out_index get_output_tx_and_index_from_global(const uint64_t& index) const; virtual void get_output_tx_and_index_from_global(const std::vector &global_indices, @@ -249,11 +271,11 @@ class BlockchainLMDB : public BlockchainDB virtual tx_out_index get_output_tx_and_index(const uint64_t& amount, const uint64_t& index) const; virtual void get_output_tx_and_index(const uint64_t& amount, const std::vector &offsets, std::vector &indices) const; - virtual std::vector get_tx_amount_output_indices(const uint64_t tx_id) const; + virtual std::vector> get_tx_amount_output_indices(const uint64_t tx_id, size_t n_txes) const; virtual bool has_key_image(const crypto::key_image& img) const; - virtual void add_txpool_tx(const transaction &tx, const txpool_tx_meta_t& meta); + virtual void add_txpool_tx(const crypto::hash &txid, const cryptonote::blobdata &blob, const txpool_tx_meta_t& meta); virtual void update_txpool_tx(const crypto::hash &txid, const txpool_tx_meta_t& meta); virtual uint64_t get_txpool_tx_count(bool include_unrelayed_txes = true) const; virtual bool txpool_has_tx(const crypto::hash &txid) const; @@ -261,6 +283,11 @@ class BlockchainLMDB : public BlockchainDB virtual bool get_txpool_tx_meta(const crypto::hash& txid, txpool_tx_meta_t &meta) const; virtual bool get_txpool_tx_blob(const crypto::hash& txid, cryptonote::blobdata &bd) const; virtual cryptonote::blobdata get_txpool_tx_blob(const crypto::hash& txid) const; + virtual uint32_t get_blockchain_pruning_seed() const; + virtual bool prune_blockchain(uint32_t pruning_seed = 0); + virtual bool update_pruning(); + virtual bool check_pruning(); + virtual bool for_all_txpool_txes(std::function f, bool include_blob = false, bool include_unrelayed_txes = true) const; virtual bool for_all_key_images(std::function) const; @@ -269,11 +296,12 @@ class BlockchainLMDB : public BlockchainDB virtual bool for_all_outputs(std::function f) const; virtual bool for_all_outputs(uint64_t amount, const std::function &f) const; - virtual uint64_t add_block( const block& blk - , const size_t& block_size + virtual uint64_t add_block( const std::pair& blk + , size_t block_weight + , uint64_t long_term_block_weight , const difficulty_type& cumulative_difficulty , const uint64_t& coins_generated - , const std::vector& txs + , const std::vector>& txs ); virtual void set_batch_transactions(bool batch_transactions); @@ -282,11 +310,14 @@ class BlockchainLMDB : public BlockchainDB virtual void batch_stop(); virtual void batch_abort(); - virtual void block_txn_start(bool readonly); - virtual void block_txn_stop(); - virtual void block_txn_abort(); - virtual bool block_rtxn_start(MDB_txn **mtxn, mdb_txn_cursors **mcur) const; + virtual void block_wtxn_start(); + virtual void block_wtxn_stop(); + virtual void block_wtxn_abort(); + virtual bool block_rtxn_start() const; virtual void block_rtxn_stop() const; + virtual void block_rtxn_abort() const; + + bool block_rtxn_start(MDB_txn **mtxn, mdb_txn_cursors **mcur) const; virtual void pop_block(block& blk, std::vector& txs); @@ -306,6 +337,11 @@ class BlockchainLMDB : public BlockchainDB bool get_output_distribution(uint64_t amount, uint64_t from_height, uint64_t to_height, std::vector &distribution, uint64_t &base) const; + // helper functions + static int compare_uint64(const MDB_val *a, const MDB_val *b); + static int compare_hash32(const MDB_val *a, const MDB_val *b); + static int compare_string(const MDB_val *a, const MDB_val *b); + private: void do_resize(uint64_t size_increase=0); @@ -314,15 +350,17 @@ class BlockchainLMDB : public BlockchainDB uint64_t get_estimated_batch_size(uint64_t batch_num_blocks, uint64_t batch_bytes) const; virtual void add_block( const block& blk - , const size_t& block_size + , size_t block_weight + , uint64_t long_term_block_weight , const difficulty_type& cumulative_difficulty , const uint64_t& coins_generated + , uint64_t num_rct_outs , const crypto::hash& block_hash ); virtual void remove_block(); - virtual uint64_t add_transaction_data(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash& tx_hash, const crypto::hash& tx_prunable_hash); + virtual uint64_t add_transaction_data(const crypto::hash& blk_hash, const std::pair& tx, const crypto::hash& tx_hash, const crypto::hash& tx_prunable_hash); virtual void remove_transaction_data(const crypto::hash& tx_hash, const transaction& tx); @@ -341,6 +379,8 @@ class BlockchainLMDB : public BlockchainDB void remove_output(const uint64_t amount, const uint64_t& out_index); + virtual void prune_outputs(uint64_t amount); + virtual void add_spent_key(const crypto::key_image& k_image); virtual void remove_spent_key(const crypto::key_image& k_image); @@ -353,28 +393,19 @@ class BlockchainLMDB : public BlockchainDB virtual void check_hard_fork_info(); virtual void drop_hard_fork_info(); - /** - * @brief convert a tx output to a blob for storage - * - * @param output the output to convert - * - * @return the resultant blob - */ - blobdata output_to_blob(const tx_out& output) const; - - /** - * @brief convert a tx output blob to a tx output - * - * @param blob the blob to convert - * - * @return the resultant tx output - */ - tx_out output_from_blob(const blobdata& blob) const; + inline void check_open() const; - void check_open() const; + bool prune_worker(int mode, uint32_t pruning_seed); virtual bool is_read_only() const; + virtual uint64_t get_database_size() const; + + std::vector get_block_info_64bit_fields(uint64_t start_height, size_t count, off_t offset) const; + + uint64_t get_max_block_size(); + void add_max_block_size(uint64_t sz); + // fix up anything that may be wrong due to past bugs virtual void fixup(); @@ -387,6 +418,15 @@ class BlockchainLMDB : public BlockchainDB // migrate from DB version 1 to 2 void migrate_1_2(); + // migrate from DB version 2 to 3 + void migrate_2_3(); + + // migrate from DB version 3 to 4 + void migrate_3_4(); + + // migrate from DB version 4 to 5 + void migrate_4_5(); + void cleanup_batch(); private: @@ -400,6 +440,7 @@ class BlockchainLMDB : public BlockchainDB MDB_dbi m_txs_pruned; MDB_dbi m_txs_prunable; MDB_dbi m_txs_prunable_hash; + MDB_dbi m_txs_prunable_tip; MDB_dbi m_tx_indices; MDB_dbi m_tx_outputs; @@ -440,7 +481,7 @@ class BlockchainLMDB : public BlockchainDB #endif #endif - constexpr static float RESIZE_PERCENT = 0.8f; + constexpr static float RESIZE_PERCENT = 0.9f; }; } // namespace cryptonote diff --git a/src/blockchain_db/testdb.h b/src/blockchain_db/testdb.h new file mode 100644 index 00000000000..34e63589955 --- /dev/null +++ b/src/blockchain_db/testdb.h @@ -0,0 +1,161 @@ +// Copyright (c) 2014-2019, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers + +#pragma once + +#include +#include +#include + +#include "blockchain_db.h" + +namespace cryptonote +{ + +class BaseTestDB: public cryptonote::BlockchainDB { +public: + BaseTestDB() {} + virtual void open(const std::string& filename, const int db_flags = 0) override { } + virtual void close() override {} + virtual void sync() override {} + virtual void safesyncmode(const bool onoff) override {} + virtual void reset() override {} + virtual std::vector get_filenames() const override { return std::vector(); } + virtual bool remove_data_file(const std::string& folder) const override { return true; } + virtual std::string get_db_name() const override { return std::string(); } + virtual bool lock() override { return true; } + virtual void unlock() override { } + virtual bool batch_start(uint64_t batch_num_blocks=0, uint64_t batch_bytes=0) override { return true; } + virtual void batch_stop() override {} + virtual void batch_abort() override {} + virtual void set_batch_transactions(bool) override {} + virtual void block_wtxn_start() override {} + virtual void block_wtxn_stop() override {} + virtual void block_wtxn_abort() override {} + virtual bool block_rtxn_start() const override { return true; } + virtual void block_rtxn_stop() const override {} + virtual void block_rtxn_abort() const override {} + + virtual void drop_hard_fork_info() override {} + virtual bool block_exists(const crypto::hash& h, uint64_t *height) const override { return false; } + virtual cryptonote::blobdata get_block_blob_from_height(const uint64_t& height) const override { return cryptonote::t_serializable_object_to_blob(get_block_from_height(height)); } + virtual cryptonote::blobdata get_block_blob(const crypto::hash& h) const override { return cryptonote::blobdata(); } + virtual bool get_tx_blob(const crypto::hash& h, cryptonote::blobdata &tx) const override { return false; } + virtual bool get_pruned_tx_blob(const crypto::hash& h, cryptonote::blobdata &tx) const override { return false; } + virtual bool get_prunable_tx_blob(const crypto::hash& h, cryptonote::blobdata &tx) const override { return false; } + virtual bool get_prunable_tx_hash(const crypto::hash& tx_hash, crypto::hash &prunable_hash) const override { return false; } + virtual uint64_t get_block_height(const crypto::hash& h) const override { return 0; } + virtual cryptonote::block_header get_block_header(const crypto::hash& h) const override { return cryptonote::block_header(); } + virtual uint64_t get_block_timestamp(const uint64_t& height) const override { return 0; } + virtual std::vector get_block_cumulative_rct_outputs(const std::vector &heights) const override { return {}; } + virtual uint64_t get_top_block_timestamp() const override { return 0; } + virtual size_t get_block_weight(const uint64_t& height) const override { return 128; } + virtual std::vector get_block_weights(uint64_t start_height, size_t count) const override { return {}; } + virtual cryptonote::difficulty_type get_block_cumulative_difficulty(const uint64_t& height) const override { return 10; } + virtual cryptonote::difficulty_type get_block_difficulty(const uint64_t& height) const override { return 0; } + virtual uint64_t get_block_already_generated_coins(const uint64_t& height) const override { return 10000000000; } + virtual uint64_t get_block_long_term_weight(const uint64_t& height) const override { return 128; } + virtual std::vector get_long_term_block_weights(uint64_t start_height, size_t count) const override { return {}; } + virtual crypto::hash get_block_hash_from_height(const uint64_t& height) const override { return crypto::hash(); } + virtual std::vector get_blocks_range(const uint64_t& h1, const uint64_t& h2) const override { return std::vector(); } + virtual std::vector get_hashes_range(const uint64_t& h1, const uint64_t& h2) const override { return std::vector(); } + virtual crypto::hash top_block_hash(uint64_t *block_height = NULL) const override { if (block_height) *block_height = 0; return crypto::hash(); } + virtual cryptonote::block get_top_block() const override { return cryptonote::block(); } + virtual uint64_t height() const override { return 1; } + virtual bool tx_exists(const crypto::hash& h) const override { return false; } + virtual bool tx_exists(const crypto::hash& h, uint64_t& tx_index) const override { return false; } + virtual uint64_t get_tx_unlock_time(const crypto::hash& h) const override { return 0; } + virtual cryptonote::transaction get_tx(const crypto::hash& h) const override { return cryptonote::transaction(); } + virtual bool get_tx(const crypto::hash& h, cryptonote::transaction &tx) const override { return false; } + virtual uint64_t get_tx_count() const override { return 0; } + virtual std::vector get_tx_list(const std::vector& hlist) const override { return std::vector(); } + virtual uint64_t get_tx_block_height(const crypto::hash& h) const override { return 0; } + virtual uint64_t get_num_outputs(const uint64_t& amount) const override { return 1; } + virtual uint64_t get_indexing_base() const override { return 0; } + virtual cryptonote::output_data_t get_output_key(const uint64_t& amount, const uint64_t& index, bool include_commitmemt) const override { return cryptonote::output_data_t(); } + virtual cryptonote::tx_out_index get_output_tx_and_index_from_global(const uint64_t& index) const override { return cryptonote::tx_out_index(); } + virtual cryptonote::tx_out_index get_output_tx_and_index(const uint64_t& amount, const uint64_t& index) const override { return cryptonote::tx_out_index(); } + virtual void get_output_tx_and_index(const uint64_t& amount, const std::vector &offsets, std::vector &indices) const override {} + virtual void get_output_key(const epee::span &amounts, const std::vector &offsets, std::vector &outputs, bool allow_partial = false) const override {} + virtual bool can_thread_bulk_indices() const override { return false; } + virtual std::vector> get_tx_amount_output_indices(const uint64_t tx_index, size_t n_txes) const override { return std::vector>(); } + virtual bool has_key_image(const crypto::key_image& img) const override { return false; } + virtual void remove_block() override { } + virtual uint64_t add_transaction_data(const crypto::hash& blk_hash, const std::pair& tx, const crypto::hash& tx_hash, const crypto::hash& tx_prunable_hash) override {return 0;} + virtual void remove_transaction_data(const crypto::hash& tx_hash, const cryptonote::transaction& tx) override {} + virtual uint64_t add_output(const crypto::hash& tx_hash, const cryptonote::tx_out& tx_output, const uint64_t& local_index, const uint64_t unlock_time, const rct::key *commitment) override {return 0;} + virtual void add_tx_amount_output_indices(const uint64_t tx_index, const std::vector& amount_output_indices) override {} + virtual void add_spent_key(const crypto::key_image& k_image) override {} + virtual void remove_spent_key(const crypto::key_image& k_image) override {} + + virtual bool for_all_key_images(std::function) const override { return true; } + virtual bool for_blocks_range(const uint64_t&, const uint64_t&, std::function) const override { return true; } + virtual bool for_all_transactions(std::function, bool pruned) const override { return true; } + virtual bool for_all_outputs(std::function f) const override { return true; } + virtual bool for_all_outputs(uint64_t amount, const std::function &f) const override { return true; } + virtual bool is_read_only() const override { return false; } + virtual std::map> get_output_histogram(const std::vector &amounts, bool unlocked, uint64_t recent_cutoff, uint64_t min_count) const override { return std::map>(); } + virtual bool get_output_distribution(uint64_t amount, uint64_t from_height, uint64_t to_height, std::vector &distribution, uint64_t &base) const override { return false; } + + virtual void add_txpool_tx(const crypto::hash &txid, const cryptonote::blobdata &blob, const cryptonote::txpool_tx_meta_t& details) override {} + virtual void update_txpool_tx(const crypto::hash &txid, const cryptonote::txpool_tx_meta_t& details) override {} + virtual uint64_t get_txpool_tx_count(bool include_unrelayed_txes = true) const override { return 0; } + virtual bool txpool_has_tx(const crypto::hash &txid) const override { return false; } + virtual void remove_txpool_tx(const crypto::hash& txid) override {} + virtual bool get_txpool_tx_meta(const crypto::hash& txid, cryptonote::txpool_tx_meta_t &meta) const override { return false; } + virtual bool get_txpool_tx_blob(const crypto::hash& txid, cryptonote::blobdata &bd) const override { return false; } + virtual uint64_t get_database_size() const override { return 0; } + virtual cryptonote::blobdata get_txpool_tx_blob(const crypto::hash& txid) const override { return ""; } + virtual bool for_all_txpool_txes(std::function, bool include_blob = false, bool include_unrelayed_txes = false) const override { return false; } + + virtual void add_block( const cryptonote::block& blk + , size_t block_weight + , uint64_t long_term_block_weight + , const cryptonote::difficulty_type& cumulative_difficulty + , const uint64_t& coins_generated + , uint64_t num_rct_outs + , const crypto::hash& blk_hash + ) override { } + virtual cryptonote::block get_block_from_height(const uint64_t& height) const override { return cryptonote::block(); } + virtual void set_hard_fork_version(uint64_t height, uint8_t version) override {} + virtual uint8_t get_hard_fork_version(uint64_t height) const override { return 0; } + virtual void check_hard_fork_info() override {} + + virtual uint32_t get_blockchain_pruning_seed() const override { return 0; } + virtual bool prune_blockchain(uint32_t pruning_seed = 0) override { return true; } + virtual bool update_pruning() override { return true; } + virtual bool check_pruning() override { return true; } + virtual void prune_outputs(uint64_t amount) override {} + + virtual uint64_t get_max_block_size() override { return 100000000; } + virtual void add_max_block_size(uint64_t sz) override { } +}; + +} diff --git a/src/blockchain_utilities/CMakeLists.txt b/src/blockchain_utilities/CMakeLists.txt index a5dd6955674..0ba7ee86c91 100644 --- a/src/blockchain_utilities/CMakeLists.txt +++ b/src/blockchain_utilities/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2014-2018, The Monero Project +# Copyright (c) 2014-2019, The Monero Project # # All rights reserved. # @@ -26,16 +26,6 @@ # STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF # THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -set(blocksdat "") -if(PER_BLOCK_CHECKPOINT) - if(APPLE) - add_custom_command(OUTPUT blocksdat.o MAIN_DEPENDENCY ../blocks/checkpoints.dat COMMAND cd ${CMAKE_CURRENT_SOURCE_DIR} && touch stub.c && ${CMAKE_C_COMPILER} -o stub.o -c stub.c COMMAND cd ${CMAKE_CURRENT_SOURCE_DIR} && ${CMAKE_LINKER} ${LD_RAW_FLAGS} -r -sectcreate __DATA __blocks_dat ../blocks/checkpoints.dat -o ${CMAKE_CURRENT_BINARY_DIR}/blocksdat.o stub.o && rm -f stub.*) - else() - add_custom_command(OUTPUT blocksdat.o MAIN_DEPENDENCY ../blocks/checkpoints.dat COMMAND cd ${CMAKE_CURRENT_SOURCE_DIR} && cp ../blocks/checkpoints.dat blocks.dat && ${CMAKE_LINKER} ${LD_RAW_FLAGS} -r -b binary -o ${CMAKE_CURRENT_BINARY_DIR}/blocksdat.o blocks.dat && rm -f blocks.dat) - endif() - set(blocksdat "blocksdat.o") -endif() - set(blockchain_import_sources blockchain_import.cpp bootstrap_file.cpp @@ -91,23 +81,74 @@ monero_private_headers(blockchain_usage +set(blockchain_prune_known_spent_data_sources + blockchain_prune_known_spent_data.cpp + ) + +set(blockchain_prune_known_spent_data_private_headers) + +monero_private_headers(blockchain_prune_known_spent_data + ${blockchain_prune_known_spent_data_private_headers}) + + + +set(blockchain_prune_sources + blockchain_prune.cpp + ) + +set(blockchain_prune_private_headers) + +monero_private_headers(blockchain_prune + ${blockchain_prune_private_headers}) + + + +set(blockchain_ancestry_sources + blockchain_ancestry.cpp + ) + +set(blockchain_ancestry_private_headers) + +monero_private_headers(blockchain_ancestry + ${blockchain_ancestry_private_headers}) + + + +set(blockchain_depth_sources + blockchain_depth.cpp + ) + +set(blockchain_depth_private_headers) + +monero_private_headers(blockchain_depth + ${blockchain_depth_private_headers}) + +set(blockchain_stats_sources + blockchain_stats.cpp + ) + +set(blockchain_stats_private_headers) + +monero_private_headers(blockchain_stats + ${blockchain_stats_private_headers}) + + monero_add_executable(blockchain_import ${blockchain_import_sources} - ${blockchain_import_private_headers} - ${blocksdat}) + ${blockchain_import_private_headers}) target_link_libraries(blockchain_import PRIVATE cryptonote_core blockchain_db - p2p version epee ${Boost_FILESYSTEM_LIBRARY} ${Boost_SYSTEM_LIBRARY} ${Boost_THREAD_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} - ${EXTRA_LIBRARIES}) + ${EXTRA_LIBRARIES} + ${Blocks}) if(ARCH_WIDTH) target_compile_definitions(blockchain_import @@ -127,7 +168,6 @@ target_link_libraries(blockchain_export PRIVATE cryptonote_core blockchain_db - p2p version epee ${Boost_FILESYSTEM_LIBRARY} @@ -150,7 +190,6 @@ target_link_libraries(blockchain_blackball wallet cryptonote_core blockchain_db - p2p version epee ${Boost_FILESYSTEM_LIBRARY} @@ -161,7 +200,7 @@ target_link_libraries(blockchain_blackball set_property(TARGET blockchain_blackball PROPERTY - OUTPUT_NAME "monero-blockchain-blackball") + OUTPUT_NAME "monero-blockchain-mark-spent-outputs") install(TARGETS blockchain_blackball DESTINATION bin) @@ -173,7 +212,6 @@ target_link_libraries(blockchain_usage PRIVATE cryptonote_core blockchain_db - p2p version epee ${Boost_FILESYSTEM_LIBRARY} @@ -187,3 +225,109 @@ set_property(TARGET blockchain_usage OUTPUT_NAME "monero-blockchain-usage") install(TARGETS blockchain_usage DESTINATION bin) +monero_add_executable(blockchain_ancestry + ${blockchain_ancestry_sources} + ${blockchain_ancestry_private_headers}) + +target_link_libraries(blockchain_ancestry + PRIVATE + cryptonote_core + blockchain_db + version + epee + ${Boost_FILESYSTEM_LIBRARY} + ${Boost_SYSTEM_LIBRARY} + ${Boost_THREAD_LIBRARY} + ${CMAKE_THREAD_LIBS_INIT} + ${EXTRA_LIBRARIES}) + +set_property(TARGET blockchain_ancestry + PROPERTY + OUTPUT_NAME "monero-blockchain-ancestry") +install(TARGETS blockchain_ancestry DESTINATION bin) + +monero_add_executable(blockchain_depth + ${blockchain_depth_sources} + ${blockchain_depth_private_headers}) + +target_link_libraries(blockchain_depth + PRIVATE + cryptonote_core + blockchain_db + version + epee + ${Boost_FILESYSTEM_LIBRARY} + ${Boost_SYSTEM_LIBRARY} + ${Boost_THREAD_LIBRARY} + ${CMAKE_THREAD_LIBS_INIT} + ${EXTRA_LIBRARIES}) + +set_property(TARGET blockchain_depth + PROPERTY + OUTPUT_NAME "monero-blockchain-depth") +install(TARGETS blockchain_depth DESTINATION bin) + +monero_add_executable(blockchain_stats + ${blockchain_stats_sources} + ${blockchain_stats_private_headers}) + +target_link_libraries(blockchain_stats + PRIVATE + cryptonote_core + blockchain_db + version + epee + ${Boost_FILESYSTEM_LIBRARY} + ${Boost_SYSTEM_LIBRARY} + ${Boost_THREAD_LIBRARY} + ${CMAKE_THREAD_LIBS_INIT} + ${EXTRA_LIBRARIES}) + +set_property(TARGET blockchain_stats + PROPERTY + OUTPUT_NAME "monero-blockchain-stats") +install(TARGETS blockchain_stats DESTINATION bin) + +monero_add_executable(blockchain_prune_known_spent_data + ${blockchain_prune_known_spent_data_sources} + ${blockchain_prune_known_spent_data_private_headers}) + +target_link_libraries(blockchain_prune_known_spent_data + PRIVATE + cryptonote_core + blockchain_db + p2p + version + epee + ${Boost_FILESYSTEM_LIBRARY} + ${Boost_SYSTEM_LIBRARY} + ${Boost_THREAD_LIBRARY} + ${CMAKE_THREAD_LIBS_INIT} + ${EXTRA_LIBRARIES}) + +set_property(TARGET blockchain_prune_known_spent_data + PROPERTY + OUTPUT_NAME "monero-blockchain-prune-known-spent-data") +install(TARGETS blockchain_prune_known_spent_data DESTINATION bin) + +monero_add_executable(blockchain_prune + ${blockchain_prune_sources} + ${blockchain_prune_private_headers}) + +set_property(TARGET blockchain_prune + PROPERTY + OUTPUT_NAME "monero-blockchain-prune") +install(TARGETS blockchain_prune DESTINATION bin) + +target_link_libraries(blockchain_prune + PRIVATE + cryptonote_core + blockchain_db + p2p + version + epee + ${Boost_FILESYSTEM_LIBRARY} + ${Boost_SYSTEM_LIBRARY} + ${Boost_THREAD_LIBRARY} + ${CMAKE_THREAD_LIBS_INIT} + ${EXTRA_LIBRARIES}) diff --git a/src/blockchain_utilities/README.md b/src/blockchain_utilities/README.md index 5d968cd755c..1462e31867d 100644 --- a/src/blockchain_utilities/README.md +++ b/src/blockchain_utilities/README.md @@ -1,6 +1,6 @@ # Monero Blockchain Utilities -Copyright (c) 2014-2018, The Monero Project +Copyright (c) 2014-2019, The Monero Project ## Introduction @@ -79,7 +79,7 @@ LMDB flags (more than one may be specified): ## Examples: -``` +```bash $ monero-blockchain-import --database lmdb#fastest $ monero-blockchain-import --database lmdb#nosync diff --git a/src/blockchain_utilities/blockchain_ancestry.cpp b/src/blockchain_utilities/blockchain_ancestry.cpp new file mode 100644 index 00000000000..a6ee0573f97 --- /dev/null +++ b/src/blockchain_utilities/blockchain_ancestry.cpp @@ -0,0 +1,748 @@ +// Copyright (c) 2014-2019, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include +#include +#include +#include +#include +#include +#include "common/unordered_containers_boost_serialization.h" +#include "common/command_line.h" +#include "common/varint.h" +#include "cryptonote_basic/cryptonote_boost_serialization.h" +#include "cryptonote_core/tx_pool.h" +#include "cryptonote_core/cryptonote_core.h" +#include "cryptonote_core/blockchain.h" +#include "blockchain_db/blockchain_db.h" +#include "blockchain_db/db_types.h" +#include "version.h" + +#undef MONERO_DEFAULT_LOG_CATEGORY +#define MONERO_DEFAULT_LOG_CATEGORY "bcutil" + +namespace po = boost::program_options; +using namespace epee; +using namespace cryptonote; + +static bool stop_requested = false; +static uint64_t cached_txes = 0, cached_blocks = 0, cached_outputs = 0, total_txes = 0, total_blocks = 0, total_outputs = 0; +static bool opt_cache_outputs = false, opt_cache_txes = false, opt_cache_blocks = false; + +struct ancestor +{ + uint64_t amount; + uint64_t offset; + + bool operator==(const ancestor &other) const { return amount == other.amount && offset == other.offset; } + + template void serialize(t_archive &a, const unsigned int ver) + { + a & amount; + a & offset; + } +}; +BOOST_CLASS_VERSION(ancestor, 0) + +namespace std +{ + template<> struct hash + { + size_t operator()(const ancestor &a) const + { + return a.amount ^ a.offset; // not that bad, since amount almost always have a high bit set, and offset doesn't + } + }; +} + +struct tx_data_t +{ + std::vector>> vin; + std::vector vout; + bool coinbase; + + tx_data_t(): coinbase(false) {} + tx_data_t(const cryptonote::transaction &tx) + { + coinbase = tx.vin.size() == 1 && tx.vin[0].type() == typeid(cryptonote::txin_gen); + if (!coinbase) + { + vin.reserve(tx.vin.size()); + for (size_t ring = 0; ring < tx.vin.size(); ++ring) + { + if (tx.vin[ring].type() == typeid(cryptonote::txin_to_key)) + { + const cryptonote::txin_to_key &txin = boost::get(tx.vin[ring]); + vin.push_back(std::make_pair(txin.amount, cryptonote::relative_output_offsets_to_absolute(txin.key_offsets))); + } + else + { + LOG_PRINT_L0("Bad vin type in txid " << get_transaction_hash(tx)); + throw std::runtime_error("Bad vin type"); + } + } + } + vout.reserve(tx.vout.size()); + for (size_t out = 0; out < tx.vout.size(); ++out) + { + if (tx.vout[out].target.type() == typeid(cryptonote::txout_to_key)) + { + const auto &txout = boost::get(tx.vout[out].target); + vout.push_back(txout.key); + } + else + { + LOG_PRINT_L0("Bad vout type in txid " << get_transaction_hash(tx)); + throw std::runtime_error("Bad vout type"); + } + } + } + + template void serialize(t_archive &a, const unsigned int ver) + { + a & coinbase; + a & vin; + a & vout; + } +}; + +struct ancestry_state_t +{ + uint64_t height; + std::unordered_map> ancestry; + std::unordered_map output_cache; + std::unordered_map tx_cache; + std::vector block_cache; + + ancestry_state_t(): height(0) {} + + template void serialize(t_archive &a, const unsigned int ver) + { + a & height; + a & ancestry; + a & output_cache; + if (ver < 1) + { + std::unordered_map old_tx_cache; + a & old_tx_cache; + for (const auto i: old_tx_cache) + tx_cache.insert(std::make_pair(i.first, ::tx_data_t(i.second))); + } + else + { + a & tx_cache; + } + if (ver < 2) + { + std::unordered_map old_block_cache; + a & old_block_cache; + block_cache.resize(old_block_cache.size()); + for (const auto i: old_block_cache) + block_cache[i.first] = i.second; + } + else + { + a & block_cache; + } + } +}; +BOOST_CLASS_VERSION(ancestry_state_t, 2) + +static void add_ancestor(std::unordered_map &ancestry, uint64_t amount, uint64_t offset) +{ + std::pair::iterator, bool> p = ancestry.insert(std::make_pair(ancestor{amount, offset}, 1)); + if (!p.second) + { + ++p.first->second; + } +} + +static size_t get_full_ancestry(const std::unordered_map &ancestry) +{ + size_t count = 0; + for (const auto &i: ancestry) + count += i.second; + return count; +} + +static size_t get_deduplicated_ancestry(const std::unordered_map &ancestry) +{ + return ancestry.size(); +} + +static void add_ancestry(std::unordered_map> &ancestry, const crypto::hash &txid, const std::unordered_set &ancestors) +{ + std::pair>::iterator, bool> p = ancestry.insert(std::make_pair(txid, ancestors)); + if (!p.second) + { + for (const auto &e: ancestors) + p.first->second.insert(e); + } +} + +static void add_ancestry(std::unordered_map> &ancestry, const crypto::hash &txid, const ancestor &new_ancestor) +{ + std::pair>::iterator, bool> p = ancestry.insert(std::make_pair(txid, std::unordered_set())); + p.first->second.insert(new_ancestor); +} + +static std::unordered_set get_ancestry(const std::unordered_map> &ancestry, const crypto::hash &txid) +{ + std::unordered_map>::const_iterator i = ancestry.find(txid); + if (i == ancestry.end()) + { + //MERROR("txid ancestry not found: " << txid); + //throw std::runtime_error("txid ancestry not found"); + return std::unordered_set(); + } + return i->second; +} + +static bool get_block_from_height(ancestry_state_t &state, BlockchainDB *db, uint64_t height, cryptonote::block &b) +{ + ++total_blocks; + if (state.block_cache.size() > height && !state.block_cache[height].miner_tx.vin.empty()) + { + ++cached_blocks; + b = state.block_cache[height]; + return true; + } + cryptonote::blobdata bd = db->get_block_blob_from_height(height); + if (!cryptonote::parse_and_validate_block_from_blob(bd, b)) + { + LOG_PRINT_L0("Bad block from db"); + return false; + } + if (opt_cache_blocks) + { + state.block_cache.resize(height + 1); + state.block_cache[height] = b; + } + return true; +} + +static bool get_transaction(ancestry_state_t &state, BlockchainDB *db, const crypto::hash &txid, ::tx_data_t &tx_data) +{ + std::unordered_map::const_iterator i = state.tx_cache.find(txid); + ++total_txes; + if (i != state.tx_cache.end()) + { + ++cached_txes; + tx_data = i->second; + return true; + } + + cryptonote::blobdata bd; + if (!db->get_pruned_tx_blob(txid, bd)) + { + LOG_PRINT_L0("Failed to get txid " << txid << " from db"); + return false; + } + cryptonote::transaction tx; + if (!cryptonote::parse_and_validate_tx_base_from_blob(bd, tx)) + { + LOG_PRINT_L0("Bad tx: " << txid); + return false; + } + tx_data = ::tx_data_t(tx); + if (opt_cache_txes) + state.tx_cache.insert(std::make_pair(txid, tx_data)); + return true; +} + +static bool get_output_txid(ancestry_state_t &state, BlockchainDB *db, uint64_t amount, uint64_t offset, crypto::hash &txid) +{ + ++total_outputs; + std::unordered_map::const_iterator i = state.output_cache.find({amount, offset}); + if (i != state.output_cache.end()) + { + ++cached_outputs; + txid = i->second; + return true; + } + + const output_data_t od = db->get_output_key(amount, offset, false); + cryptonote::block b; + if (!get_block_from_height(state, db, od.height, b)) + return false; + + for (size_t out = 0; out < b.miner_tx.vout.size(); ++out) + { + if (b.miner_tx.vout[out].target.type() == typeid(cryptonote::txout_to_key)) + { + const auto &txout = boost::get(b.miner_tx.vout[out].target); + if (txout.key == od.pubkey) + { + txid = cryptonote::get_transaction_hash(b.miner_tx); + if (opt_cache_outputs) + state.output_cache.insert(std::make_pair(ancestor{amount, offset}, txid)); + return true; + } + } + else + { + LOG_PRINT_L0("Bad vout type in txid " << cryptonote::get_transaction_hash(b.miner_tx)); + return false; + } + } + for (const crypto::hash &block_txid: b.tx_hashes) + { + ::tx_data_t tx_data3; + if (!get_transaction(state, db, block_txid, tx_data3)) + return false; + + for (size_t out = 0; out < tx_data3.vout.size(); ++out) + { + if (tx_data3.vout[out] == od.pubkey) + { + txid = block_txid; + if (opt_cache_outputs) + state.output_cache.insert(std::make_pair(ancestor{amount, offset}, txid)); + return true; + } + } + } + return false; +} + +int main(int argc, char* argv[]) +{ + TRY_ENTRY(); + + epee::string_tools::set_module_name_and_folder(argv[0]); + + std::string default_db_type = "lmdb"; + + std::string available_dbs = cryptonote::blockchain_db_types(", "); + available_dbs = "available: " + available_dbs; + + uint32_t log_level = 0; + + tools::on_startup(); + + boost::filesystem::path output_file_path; + + po::options_description desc_cmd_only("Command line options"); + po::options_description desc_cmd_sett("Command line options and settings options"); + const command_line::arg_descriptor arg_log_level = {"log-level", "0-4 or categories", ""}; + const command_line::arg_descriptor arg_database = { + "database", available_dbs.c_str(), default_db_type + }; + const command_line::arg_descriptor arg_txid = {"txid", "Get ancestry for this txid", ""}; + const command_line::arg_descriptor arg_output = {"output", "Get ancestry for this output (amount/offset format)", ""}; + const command_line::arg_descriptor arg_height = {"height", "Get ancestry for all txes at this height", 0}; + const command_line::arg_descriptor arg_refresh = {"refresh", "Refresh the whole chain first", false}; + const command_line::arg_descriptor arg_cache_outputs = {"cache-outputs", "Cache outputs (memory hungry)", false}; + const command_line::arg_descriptor arg_cache_txes = {"cache-txes", "Cache txes (memory hungry)", false}; + const command_line::arg_descriptor arg_cache_blocks = {"cache-blocks", "Cache blocks (memory hungry)", false}; + const command_line::arg_descriptor arg_include_coinbase = {"include-coinbase", "Including coinbase tx in per height average", false}; + const command_line::arg_descriptor arg_show_cache_stats = {"show-cache-stats", "Show cache statistics", false}; + + command_line::add_arg(desc_cmd_sett, cryptonote::arg_data_dir); + command_line::add_arg(desc_cmd_sett, cryptonote::arg_testnet_on); + command_line::add_arg(desc_cmd_sett, cryptonote::arg_stagenet_on); + command_line::add_arg(desc_cmd_sett, arg_log_level); + command_line::add_arg(desc_cmd_sett, arg_database); + command_line::add_arg(desc_cmd_sett, arg_txid); + command_line::add_arg(desc_cmd_sett, arg_output); + command_line::add_arg(desc_cmd_sett, arg_height); + command_line::add_arg(desc_cmd_sett, arg_refresh); + command_line::add_arg(desc_cmd_sett, arg_cache_outputs); + command_line::add_arg(desc_cmd_sett, arg_cache_txes); + command_line::add_arg(desc_cmd_sett, arg_cache_blocks); + command_line::add_arg(desc_cmd_sett, arg_include_coinbase); + command_line::add_arg(desc_cmd_sett, arg_show_cache_stats); + command_line::add_arg(desc_cmd_only, command_line::arg_help); + + po::options_description desc_options("Allowed options"); + desc_options.add(desc_cmd_only).add(desc_cmd_sett); + + po::variables_map vm; + bool r = command_line::handle_error_helper(desc_options, [&]() + { + auto parser = po::command_line_parser(argc, argv).options(desc_options); + po::store(parser.run(), vm); + po::notify(vm); + return true; + }); + if (! r) + return 1; + + if (command_line::get_arg(vm, command_line::arg_help)) + { + std::cout << "Monero '" << MONERO_RELEASE_NAME << "' (v" << MONERO_VERSION_FULL << ")" << ENDL << ENDL; + std::cout << desc_options << std::endl; + return 1; + } + + mlog_configure(mlog_get_default_log_path("monero-blockchain-ancestry.log"), true); + if (!command_line::is_arg_defaulted(vm, arg_log_level)) + mlog_set_log(command_line::get_arg(vm, arg_log_level).c_str()); + else + mlog_set_log(std::string(std::to_string(log_level) + ",bcutil:INFO").c_str()); + + LOG_PRINT_L0("Starting..."); + + std::string opt_data_dir = command_line::get_arg(vm, cryptonote::arg_data_dir); + bool opt_testnet = command_line::get_arg(vm, cryptonote::arg_testnet_on); + bool opt_stagenet = command_line::get_arg(vm, cryptonote::arg_stagenet_on); + network_type net_type = opt_testnet ? TESTNET : opt_stagenet ? STAGENET : MAINNET; + std::string opt_txid_string = command_line::get_arg(vm, arg_txid); + std::string opt_output_string = command_line::get_arg(vm, arg_output); + uint64_t opt_height = command_line::get_arg(vm, arg_height); + bool opt_refresh = command_line::get_arg(vm, arg_refresh); + opt_cache_outputs = command_line::get_arg(vm, arg_cache_outputs); + opt_cache_txes = command_line::get_arg(vm, arg_cache_txes); + opt_cache_blocks = command_line::get_arg(vm, arg_cache_blocks); + bool opt_include_coinbase = command_line::get_arg(vm, arg_include_coinbase); + bool opt_show_cache_stats = command_line::get_arg(vm, arg_show_cache_stats); + + if ((!opt_txid_string.empty()) + !!opt_height + !opt_output_string.empty() > 1) + { + std::cerr << "Only one of --txid, --height, --output can be given" << std::endl; + return 1; + } + crypto::hash opt_txid = crypto::null_hash; + uint64_t output_amount = 0, output_offset = 0; + if (!opt_txid_string.empty()) + { + if (!epee::string_tools::hex_to_pod(opt_txid_string, opt_txid)) + { + std::cerr << "Invalid txid" << std::endl; + return 1; + } + } + else if (!opt_output_string.empty()) + { + if (sscanf(opt_output_string.c_str(), "%" SCNu64 "/%" SCNu64, &output_amount, &output_offset) != 2) + { + std::cerr << "Invalid output" << std::endl; + return 1; + } + } + + std::string db_type = command_line::get_arg(vm, arg_database); + if (!cryptonote::blockchain_valid_db_type(db_type)) + { + std::cerr << "Invalid database type: " << db_type << std::endl; + return 1; + } + + // If we wanted to use the memory pool, we would set up a fake_core. + + // Use Blockchain instead of lower-level BlockchainDB for two reasons: + // 1. Blockchain has the init() method for easy setup + // 2. exporter needs to use get_current_blockchain_height(), get_block_id_by_height(), get_block_by_hash() + // + // cannot match blockchain_storage setup above with just one line, + // e.g. + // Blockchain* core_storage = new Blockchain(NULL); + // because unlike blockchain_storage constructor, which takes a pointer to + // tx_memory_pool, Blockchain's constructor takes tx_memory_pool object. + LOG_PRINT_L0("Initializing source blockchain (BlockchainDB)"); + std::unique_ptr core_storage; + tx_memory_pool m_mempool(*core_storage); + core_storage.reset(new Blockchain(m_mempool)); + BlockchainDB *db = new_db(db_type); + if (db == NULL) + { + LOG_ERROR("Attempted to use non-existent database type: " << db_type); + throw std::runtime_error("Attempting to use non-existent database type"); + } + LOG_PRINT_L0("database: " << db_type); + + const std::string filename = (boost::filesystem::path(opt_data_dir) / db->get_db_name()).string(); + LOG_PRINT_L0("Loading blockchain from folder " << filename << " ..."); + + try + { + db->open(filename, DBF_RDONLY); + } + catch (const std::exception& e) + { + LOG_PRINT_L0("Error opening database: " << e.what()); + return 1; + } + r = core_storage->init(db, net_type); + + CHECK_AND_ASSERT_MES(r, 1, "Failed to initialize source blockchain storage"); + LOG_PRINT_L0("Source blockchain storage initialized OK"); + + std::vector start_txids; + + ancestry_state_t state; + + const std::string state_file_path = (boost::filesystem::path(opt_data_dir) / "ancestry-state.bin").string(); + LOG_PRINT_L0("Loading state data from " << state_file_path); + std::ifstream state_data_in; + state_data_in.open(state_file_path, std::ios_base::binary | std::ios_base::in); + if (!state_data_in.fail()) + { + try + { + boost::archive::portable_binary_iarchive a(state_data_in); + a >> state; + } + catch (const std::exception &e) + { + MERROR("Failed to load state data from " << state_file_path << ", restarting from scratch"); + state = ancestry_state_t(); + } + state_data_in.close(); + } + + tools::signal_handler::install([](int type) { + stop_requested = true; + }); + + // forward method + const uint64_t db_height = db->height(); + if (opt_refresh) + { + MINFO("Starting from height " << state.height); + state.block_cache.reserve(db_height); + for (uint64_t h = state.height; h < db_height; ++h) + { + size_t block_ancestry_size = 0; + const cryptonote::blobdata bd = db->get_block_blob_from_height(h); + ++total_blocks; + cryptonote::block b; + if (!cryptonote::parse_and_validate_block_from_blob(bd, b)) + { + LOG_PRINT_L0("Bad block from db"); + return 1; + } + if (opt_cache_blocks) + { + state.block_cache.resize(h + 1); + state.block_cache[h] = b; + } + std::vector txids; + txids.reserve(1 + b.tx_hashes.size()); + if (opt_include_coinbase) + txids.push_back(cryptonote::get_transaction_hash(b.miner_tx)); + for (const auto &h: b.tx_hashes) + txids.push_back(h); + for (const crypto::hash &txid: txids) + { + printf("%lu/%lu \r", (unsigned long)h, (unsigned long)db_height); + fflush(stdout); + ::tx_data_t tx_data; + std::unordered_map::const_iterator i = state.tx_cache.find(txid); + ++total_txes; + if (i != state.tx_cache.end()) + { + ++cached_txes; + tx_data = i->second; + } + else + { + cryptonote::blobdata bd; + if (!db->get_pruned_tx_blob(txid, bd)) + { + LOG_PRINT_L0("Failed to get txid " << txid << " from db"); + return 1; + } + cryptonote::transaction tx; + if (!cryptonote::parse_and_validate_tx_base_from_blob(bd, tx)) + { + LOG_PRINT_L0("Bad tx: " << txid); + return 1; + } + tx_data = ::tx_data_t(tx); + if (opt_cache_txes) + state.tx_cache.insert(std::make_pair(txid, tx_data)); + } + if (tx_data.coinbase) + { + add_ancestry(state.ancestry, txid, std::unordered_set()); + } + else + { + for (size_t ring = 0; ring < tx_data.vin.size(); ++ring) + { + const uint64_t amount = tx_data.vin[ring].first; + const std::vector &absolute_offsets = tx_data.vin[ring].second; + for (uint64_t offset: absolute_offsets) + { + add_ancestry(state.ancestry, txid, ancestor{amount, offset}); + // find the tx which created this output + bool found = false; + crypto::hash output_txid; + if (!get_output_txid(state, db, amount, offset, output_txid)) + { + LOG_PRINT_L0("Output originating transaction not found"); + return 1; + } + add_ancestry(state.ancestry, txid, get_ancestry(state.ancestry, output_txid)); + } + } + } + const size_t ancestry_size = get_ancestry(state.ancestry, txid).size(); + block_ancestry_size += ancestry_size; + MINFO(txid << ": " << ancestry_size); + } + if (!txids.empty()) + { + std::string stats_msg; + MINFO("Height " << h << ": " << (block_ancestry_size / txids.size()) << " average over " << txids.size() << stats_msg); + } + state.height = h; + if (stop_requested) + break; + } + + LOG_PRINT_L0("Saving state data to " << state_file_path); + std::ofstream state_data_out; + state_data_out.open(state_file_path, std::ios_base::binary | std::ios_base::out | std::ios::trunc); + if (!state_data_out.fail()) + { + try + { + boost::archive::portable_binary_oarchive a(state_data_out); + a << state; + } + catch (const std::exception &e) + { + MERROR("Failed to save state data to " << state_file_path); + } + state_data_out.close(); + } + } + else + { + if (state.height < db_height) + { + MWARNING("The state file is only built up to height " << state.height << ", but the blockchain reached height " << db_height); + MWARNING("You may want to run with --refresh if you want to get ancestry for newer data"); + } + } + + if (!opt_txid_string.empty()) + { + start_txids.push_back(opt_txid); + } + else if (!opt_output_string.empty()) + { + crypto::hash txid; + if (!get_output_txid(state, db, output_amount, output_offset, txid)) + { + LOG_PRINT_L0("Output not found in db"); + return 1; + } + start_txids.push_back(txid); + } + else + { + const cryptonote::blobdata bd = db->get_block_blob_from_height(opt_height); + cryptonote::block b; + if (!cryptonote::parse_and_validate_block_from_blob(bd, b)) + { + LOG_PRINT_L0("Bad block from db"); + return 1; + } + for (const crypto::hash &txid: b.tx_hashes) + start_txids.push_back(txid); + } + + if (start_txids.empty()) + { + LOG_PRINT_L0("No transaction(s) to check"); + return 1; + } + + for (const crypto::hash &start_txid: start_txids) + { + LOG_PRINT_L0("Checking ancestry for txid " << start_txid); + + std::unordered_map ancestry; + + std::list txids; + txids.push_back(start_txid); + while (!txids.empty()) + { + const crypto::hash txid = txids.front(); + txids.pop_front(); + + if (stop_requested) + goto done; + + ::tx_data_t tx_data2; + if (!get_transaction(state, db, txid, tx_data2)) + return 1; + + const bool coinbase = tx_data2.coinbase; + if (coinbase) + continue; + + for (size_t ring = 0; ring < tx_data2.vin.size(); ++ring) + { + { + const uint64_t amount = tx_data2.vin[ring].first; + auto absolute_offsets = tx_data2.vin[ring].second; + for (uint64_t offset: absolute_offsets) + { + add_ancestor(ancestry, amount, offset); + + // find the tx which created this output + bool found = false; + crypto::hash output_txid; + if (!get_output_txid(state, db, amount, offset, output_txid)) + { + LOG_PRINT_L0("Output originating transaction not found"); + return 1; + } + + add_ancestry(state.ancestry, txid, get_ancestry(state.ancestry, output_txid)); + txids.push_back(output_txid); + MDEBUG("adding txid: " << output_txid); + } + } + } + } + + MINFO("Ancestry for " << start_txid << ": " << get_deduplicated_ancestry(ancestry) << " / " << get_full_ancestry(ancestry)); + for (const auto &i: ancestry) + { + MINFO(cryptonote::print_money(i.first.amount) << "/" << i.first.offset << ": " << i.second); + } + } + +done: + core_storage->deinit(); + + if (opt_show_cache_stats) + MINFO("cache: txes " << std::to_string(cached_txes*100./total_txes) + << "%, blocks " << std::to_string(cached_blocks*100./total_blocks) + << "%, outputs " << std::to_string(cached_outputs*100./total_outputs) + << "%"); + + return 0; + + CATCH_ENTRY("Depth query error", 1); +} diff --git a/src/blockchain_utilities/blockchain_blackball.cpp b/src/blockchain_utilities/blockchain_blackball.cpp index 1243822bb2a..6ff1840416a 100644 --- a/src/blockchain_utilities/blockchain_blackball.cpp +++ b/src/blockchain_utilities/blockchain_blackball.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -28,8 +28,13 @@ #include #include +#include +#include +#include "common/unordered_containers_boost_serialization.h" #include "common/command_line.h" #include "common/varint.h" +#include "serialization/crypto.h" +#include "cryptonote_basic/cryptonote_boost_serialization.h" #include "cryptonote_core/tx_pool.h" #include "cryptonote_core/cryptonote_core.h" #include "cryptonote_core/blockchain.h" @@ -45,25 +50,86 @@ namespace po = boost::program_options; using namespace epee; using namespace cryptonote; +static const char zerokey[8] = {0}; +static const MDB_val zerokval = { sizeof(zerokey), (void *)zerokey }; + +static uint64_t records_per_sync = 200; +static uint64_t db_flags = 0; +static MDB_dbi dbi_relative_rings; +static MDB_dbi dbi_outputs; +static MDB_dbi dbi_processed_txidx; +static MDB_dbi dbi_spent; +static MDB_dbi dbi_per_amount; +static MDB_dbi dbi_ring_instances; +static MDB_dbi dbi_stats; +static MDB_env *env = NULL; + struct output_data { uint64_t amount; - uint64_t index; - output_data(uint64_t a, uint64_t i): amount(a), index(i) {} - bool operator==(const output_data &other) const { return other.amount == amount && other.index == index; } + uint64_t offset; + output_data(): amount(0), offset(0) {} + output_data(uint64_t a, uint64_t i): amount(a), offset(i) {} + bool operator==(const output_data &other) const { return other.amount == amount && other.offset == offset; } }; -namespace std + +// +// relative_rings: key_image -> vector +// outputs: 128 bits -> set of key images +// processed_txidx: string -> uint64_t +// spent: amount -> offset +// ring_instances: vector -> uint64_t +// stats: string -> arbitrary +// + +static bool parse_db_sync_mode(std::string db_sync_mode) { - template<> struct hash + std::vector options; + boost::trim(db_sync_mode); + boost::split(options, db_sync_mode, boost::is_any_of(" :")); + + for(const auto &option : options) + MDEBUG("option: " << option); + + // default to fast:async:1 + uint64_t DEFAULT_FLAGS = DBF_FAST; + + if(options.size() == 0) { - size_t operator()(const output_data &od) const + // default to fast:async:1 + db_flags = DEFAULT_FLAGS; + } + + bool safemode = false; + if(options.size() >= 1) + { + if(options[0] == "safe") { - const uint64_t data[2] = {od.amount, od.index}; - crypto::hash h; - crypto::cn_fast_hash(data, 2 * sizeof(uint64_t), h); - return reinterpret_cast(h); + safemode = true; + db_flags = DBF_SAFE; } - }; + else if(options[0] == "fast") + { + db_flags = DBF_FAST; + } + else if(options[0] == "fastest") + { + db_flags = DBF_FASTEST; + records_per_sync = 1000; // default to fastest:async:1000 + } + else + db_flags = DEFAULT_FLAGS; + } + + if(options.size() >= 2 && !safemode) + { + char *endptr; + uint64_t bps = strtoull(options[1].c_str(), &endptr, 0); + if (*endptr == '\0') + records_per_sync = bps; + } + + return true; } static std::string get_default_db_path() @@ -75,7 +141,200 @@ static std::string get_default_db_path() return dir.string(); } -static bool for_all_transactions(const std::string &filename, const std::function &f) +static std::string get_cache_filename(boost::filesystem::path filename) +{ + if (!boost::filesystem::is_directory(filename)) + filename.remove_filename(); + return filename.string(); +} + +static int compare_hash32(const MDB_val *a, const MDB_val *b) +{ + const uint32_t *va = (const uint32_t*) a->mv_data; + const uint32_t *vb = (const uint32_t*) b->mv_data; + for (int n = 7; n >= 0; n--) + { + if (va[n] == vb[n]) + continue; + return va[n] < vb[n] ? -1 : 1; + } + + return 0; +} + +int compare_uint64(const MDB_val *a, const MDB_val *b) +{ + const uint64_t va = *(const uint64_t *)a->mv_data; + const uint64_t vb = *(const uint64_t *)b->mv_data; + return (va < vb) ? -1 : va > vb; +} + +static int compare_double64(const MDB_val *a, const MDB_val *b) +{ + const uint64_t va = *(const uint64_t*) a->mv_data; + const uint64_t vb = *(const uint64_t*) b->mv_data; + if (va == vb) + { + const uint64_t va = ((const uint64_t*) a->mv_data)[1]; + const uint64_t vb = ((const uint64_t*) b->mv_data)[1]; + return va < vb ? -1 : va > vb; + } + return va < vb ? -1 : va > vb; +} + +static int resize_env(const char *db_path) +{ + MDB_envinfo mei; + MDB_stat mst; + int ret; + + size_t needed = 1000ul * 1024 * 1024; // at least 1000 MB + + ret = mdb_env_info(env, &mei); + if (ret) + return ret; + ret = mdb_env_stat(env, &mst); + if (ret) + return ret; + uint64_t size_used = mst.ms_psize * mei.me_last_pgno; + uint64_t mapsize = mei.me_mapsize; + if (size_used + needed > mei.me_mapsize) + { + try + { + boost::filesystem::path path(db_path); + boost::filesystem::space_info si = boost::filesystem::space(path); + if(si.available < needed) + { + MERROR("!! WARNING: Insufficient free space to extend database !!: " << (si.available >> 20L) << " MB available"); + return ENOSPC; + } + } + catch(...) + { + // print something but proceed. + MWARNING("Unable to query free disk space."); + } + + mapsize += needed; + } + return mdb_env_set_mapsize(env, mapsize); +} + +static void init(std::string cache_filename) +{ + MDB_txn *txn; + bool tx_active = false; + int dbr; + + MINFO("Creating spent output cache in " << cache_filename); + + tools::create_directories_if_necessary(cache_filename); + + int flags = 0; + if (db_flags & DBF_FAST) + flags |= MDB_NOSYNC; + if (db_flags & DBF_FASTEST) + flags |= MDB_NOSYNC | MDB_WRITEMAP | MDB_MAPASYNC; + + dbr = mdb_env_create(&env); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to create LDMB environment: " + std::string(mdb_strerror(dbr))); + dbr = mdb_env_set_maxdbs(env, 7); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to set max env dbs: " + std::string(mdb_strerror(dbr))); + const std::string actual_filename = get_cache_filename(cache_filename); + dbr = mdb_env_open(env, actual_filename.c_str(), flags, 0664); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open rings database file '" + + actual_filename + "': " + std::string(mdb_strerror(dbr))); + + dbr = mdb_txn_begin(env, NULL, 0, &txn); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); + epee::misc_utils::auto_scope_leave_caller txn_dtor = epee::misc_utils::create_scope_leave_handler([&](){if (tx_active) mdb_txn_abort(txn);}); + tx_active = true; + + dbr = mdb_dbi_open(txn, "relative_rings", MDB_CREATE | MDB_INTEGERKEY, &dbi_relative_rings); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); + mdb_set_compare(txn, dbi_relative_rings, compare_hash32); + + dbr = mdb_dbi_open(txn, "outputs", MDB_CREATE | MDB_INTEGERKEY, &dbi_outputs); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); + mdb_set_compare(txn, dbi_outputs, compare_double64); + + dbr = mdb_dbi_open(txn, "processed_txidx", MDB_CREATE, &dbi_processed_txidx); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); + + dbr = mdb_dbi_open(txn, "spent", MDB_CREATE | MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, &dbi_spent); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); + mdb_set_dupsort(txn, dbi_spent, compare_uint64); + + dbr = mdb_dbi_open(txn, "per_amount", MDB_CREATE | MDB_INTEGERKEY, &dbi_per_amount); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); + mdb_set_compare(txn, dbi_per_amount, compare_uint64); + + dbr = mdb_dbi_open(txn, "ring_instances", MDB_CREATE, &dbi_ring_instances); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); + + dbr = mdb_dbi_open(txn, "stats", MDB_CREATE, &dbi_stats); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); + + dbr = mdb_txn_commit(txn); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to commit txn creating/opening database: " + std::string(mdb_strerror(dbr))); + tx_active = false; +} + +static void close() +{ + if (env) + { + mdb_dbi_close(env, dbi_relative_rings); + mdb_dbi_close(env, dbi_outputs); + mdb_dbi_close(env, dbi_processed_txidx); + mdb_dbi_close(env, dbi_per_amount); + mdb_dbi_close(env, dbi_spent); + mdb_dbi_close(env, dbi_ring_instances); + mdb_dbi_close(env, dbi_stats); + mdb_env_close(env); + env = NULL; + } +} + +static std::string compress_ring(const std::vector &ring, std::string s = "") +{ + const size_t sz = s.size(); + s.resize(s.size() + 12 * ring.size()); + char *ptr = (char*)s.data() + sz; + for (uint64_t out: ring) + tools::write_varint(ptr, out); + if (ptr > s.data() + sz + 12 * ring.size()) + throw std::runtime_error("varint output overflow"); + s.resize(ptr - s.data()); + return s; +} + +static std::string compress_ring(uint64_t amount, const std::vector &ring) +{ + char s[12], *ptr = s; + tools::write_varint(ptr, amount); + if (ptr > s + sizeof(s)) + throw std::runtime_error("varint output overflow"); + return compress_ring(ring, std::string(s, ptr-s)); +} + +static std::vector decompress_ring(const std::string &s) +{ + std::vector ring; + int read = 0; + for (std::string::const_iterator i = s.begin(); i != s.cend(); std::advance(i, read)) + { + uint64_t out; + std::string tmp(i, s.cend()); + read = tools::read_varint(tmp.begin(), tmp.end(), out); + CHECK_AND_ASSERT_THROW_MES(read > 0 && read <= 256, "Internal error decompressing ring"); + ring.push_back(out); + } + return ring; +} + +static bool for_all_transactions(const std::string &filename, uint64_t &start_idx, uint64_t &n_txes, const std::function &f) { MDB_env *env; MDB_dbi dbi; @@ -83,6 +342,8 @@ static bool for_all_transactions(const std::string &filename, const std::functio MDB_cursor *cur; int dbr; bool tx_active = false; + MDB_val k; + MDB_val v; dbr = mdb_env_create(&env); if (dbr) throw std::runtime_error("Failed to create LDMB environment: " + std::string(mdb_strerror(dbr))); @@ -93,7 +354,7 @@ static bool for_all_transactions(const std::string &filename, const std::functio if (dbr) throw std::runtime_error("Failed to open rings database file '" + actual_filename + "': " + std::string(mdb_strerror(dbr))); - dbr = mdb_txn_begin(env, NULL, 0, &txn); + dbr = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn); if (dbr) throw std::runtime_error("Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); epee::misc_utils::auto_scope_leave_caller txn_dtor = epee::misc_utils::create_scope_leave_handler([&](){if (tx_active) mdb_txn_abort(txn);}); tx_active = true; @@ -104,12 +365,16 @@ static bool for_all_transactions(const std::string &filename, const std::functio if (dbr) throw std::runtime_error("Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); dbr = mdb_cursor_open(txn, dbi, &cur); if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr))); + MDB_stat stat; + dbr = mdb_stat(txn, dbi, &stat); + if (dbr) throw std::runtime_error("Failed to query m_block_info: " + std::string(mdb_strerror(dbr))); + n_txes = stat.ms_entries; - MDB_val k; - MDB_val v; bool fret = true; - MDB_cursor_op op = MDB_FIRST; + k.mv_size = sizeof(uint64_t); + k.mv_data = &start_idx; + MDB_cursor_op op = MDB_SET; while (1) { int ret = mdb_cursor_get(cur, &k, &v, op); @@ -119,6 +384,12 @@ static bool for_all_transactions(const std::string &filename, const std::functio if (ret) throw std::runtime_error("Failed to enumerate transactions: " + std::string(mdb_strerror(ret))); + if (k.mv_size != sizeof(uint64_t)) + throw std::runtime_error("Bad key size"); + const uint64_t idx = *(uint64_t*)k.mv_data; + if (idx < start_idx) + continue; + cryptonote::transaction_prefix tx; blobdata bd; bd.assign(reinterpret_cast(v.mv_data), v.mv_size); @@ -128,6 +399,7 @@ static bool for_all_transactions(const std::string &filename, const std::functio bool r = do_serialize(ba, tx); CHECK_AND_ASSERT_MES(r, false, "Failed to parse transaction from blob"); + start_idx = *(uint64_t*)k.mv_data; if (!f(tx)) { fret = false; break; @@ -135,13 +407,653 @@ static bool for_all_transactions(const std::string &filename, const std::functio } mdb_cursor_close(cur); - mdb_txn_commit(txn); + dbr = mdb_txn_commit(txn); + if (dbr) throw std::runtime_error("Failed to commit db transaction: " + std::string(mdb_strerror(dbr))); tx_active = false; mdb_dbi_close(env, dbi); mdb_env_close(env); return fret; } +static uint64_t find_first_diverging_transaction(const std::string &first_filename, const std::string &second_filename) +{ + MDB_env *env[2]; + MDB_dbi dbi[2]; + MDB_txn *txn[2]; + MDB_cursor *cur[2]; + int dbr; + bool tx_active[2] = { false, false }; + uint64_t n_txes[2]; + MDB_val k; + MDB_val v[2]; + + epee::misc_utils::auto_scope_leave_caller txn_dtor[2]; + for (int i = 0; i < 2; ++i) + { + dbr = mdb_env_create(&env[i]); + if (dbr) throw std::runtime_error("Failed to create LDMB environment: " + std::string(mdb_strerror(dbr))); + dbr = mdb_env_set_maxdbs(env[i], 2); + if (dbr) throw std::runtime_error("Failed to set max env dbs: " + std::string(mdb_strerror(dbr))); + const std::string actual_filename = i ? second_filename : first_filename; + dbr = mdb_env_open(env[i], actual_filename.c_str(), 0, 0664); + if (dbr) throw std::runtime_error("Failed to open rings database file '" + + actual_filename + "': " + std::string(mdb_strerror(dbr))); + + dbr = mdb_txn_begin(env[i], NULL, MDB_RDONLY, &txn[i]); + if (dbr) throw std::runtime_error("Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); + txn_dtor[i] = epee::misc_utils::create_scope_leave_handler([&, i](){if (tx_active[i]) mdb_txn_abort(txn[i]);}); + tx_active[i] = true; + + dbr = mdb_dbi_open(txn[i], "txs_pruned", MDB_INTEGERKEY, &dbi[i]); + if (dbr) + dbr = mdb_dbi_open(txn[i], "txs", MDB_INTEGERKEY, &dbi[i]); + if (dbr) throw std::runtime_error("Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); + dbr = mdb_cursor_open(txn[i], dbi[i], &cur[i]); + if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr))); + MDB_stat stat; + dbr = mdb_stat(txn[i], dbi[i], &stat); + if (dbr) throw std::runtime_error("Failed to query m_block_info: " + std::string(mdb_strerror(dbr))); + n_txes[i] = stat.ms_entries; + } + + if (n_txes[0] == 0 || n_txes[1] == 0) + throw std::runtime_error("No transaction in the database"); + uint64_t lo = 0, hi = std::min(n_txes[0], n_txes[1]) - 1; + while (lo <= hi) + { + uint64_t mid = (lo + hi) / 2; + + k.mv_size = sizeof(uint64_t); + k.mv_data = (void*)∣ + dbr = mdb_cursor_get(cur[0], &k, &v[0], MDB_SET); + if (dbr) throw std::runtime_error("Failed to query transaction: " + std::string(mdb_strerror(dbr))); + dbr = mdb_cursor_get(cur[1], &k, &v[1], MDB_SET); + if (dbr) throw std::runtime_error("Failed to query transaction: " + std::string(mdb_strerror(dbr))); + if (v[0].mv_size == v[1].mv_size && !memcmp(v[0].mv_data, v[1].mv_data, v[0].mv_size)) + lo = mid + 1; + else + hi = mid - 1; + } + + for (int i = 0; i < 2; ++i) + { + mdb_cursor_close(cur[i]); + dbr = mdb_txn_commit(txn[i]); + if (dbr) throw std::runtime_error("Failed to query transaction: " + std::string(mdb_strerror(dbr))); + tx_active[i] = false; + mdb_dbi_close(env[i], dbi[i]); + mdb_env_close(env[i]); + } + return hi; +} + +static std::vector canonicalize(const std::vector &v) +{ + std::vector c; + c.reserve(v.size()); + c.push_back(v[0]); + for (size_t n = 1; n < v.size(); ++n) + { + if (v[n] != 0) + c.push_back(v[n]); + } + if (c.size() < v.size()) + { + MINFO("Ring has duplicate member(s): " << + boost::join(v | boost::adaptors::transformed([](uint64_t out){return std::to_string(out);}), " ")); + } + return c; +} + +static uint64_t get_num_spent_outputs() +{ + MDB_txn *txn; + bool tx_active = false; + + int dbr = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); + epee::misc_utils::auto_scope_leave_caller txn_dtor = epee::misc_utils::create_scope_leave_handler([&](){if (tx_active) mdb_txn_abort(txn);}); + tx_active = true; + + MDB_cursor *cur; + dbr = mdb_cursor_open(txn, dbi_spent, &cur); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open cursor for spent outputs: " + std::string(mdb_strerror(dbr))); + MDB_val k, v; + mdb_size_t count = 0, tmp; + + MDB_cursor_op op = MDB_FIRST; + while (1) + { + dbr = mdb_cursor_get(cur, &k, &v, op); + op = MDB_NEXT_NODUP; + if (dbr == MDB_NOTFOUND) + break; + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to get first/next spent output: " + std::string(mdb_strerror(dbr))); + dbr = mdb_cursor_count(cur, &tmp); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to count entries: " + std::string(mdb_strerror(dbr))); + count += tmp; + } + + mdb_cursor_close(cur); + dbr = mdb_txn_commit(txn); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to commit txn: " + std::string(mdb_strerror(dbr))); + tx_active = false; + + return count; +} + +static bool add_spent_output(MDB_cursor *cur, const output_data &od) +{ + MDB_val k = {sizeof(od.amount), (void*)&od.amount}; + MDB_val v = {sizeof(od.offset), (void*)&od.offset}; + int dbr = mdb_cursor_put(cur, &k, &v, MDB_NODUPDATA); + if (dbr == MDB_KEYEXIST) + return false; + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to add spent output: " + std::string(mdb_strerror(dbr))); + return true; +} + +static bool is_output_spent(MDB_cursor *cur, const output_data &od) +{ + MDB_val k = {sizeof(od.amount), (void*)&od.amount}; + MDB_val v = {sizeof(od.offset), (void*)&od.offset}; + int dbr = mdb_cursor_get(cur, &k, &v, MDB_GET_BOTH); + CHECK_AND_ASSERT_THROW_MES(!dbr || dbr == MDB_NOTFOUND, "Failed to get spent output: " + std::string(mdb_strerror(dbr))); + bool spent = dbr == 0; + return spent; +} + +static std::vector get_spent_outputs(MDB_txn *txn) +{ + MDB_cursor *cur; + int dbr = mdb_cursor_open(txn, dbi_spent, &cur); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open cursor for spent outputs: " + std::string(mdb_strerror(dbr))); + MDB_val k, v; + mdb_size_t count = 0; + dbr = mdb_cursor_get(cur, &k, &v, MDB_FIRST); + if (dbr != MDB_NOTFOUND) + { + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to get first spent output: " + std::string(mdb_strerror(dbr))); + dbr = mdb_cursor_count(cur, &count); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to count entries: " + std::string(mdb_strerror(dbr))); + } + std::vector outs; + outs.reserve(count); + while (1) + { + outs.push_back({*(const uint64_t*)k.mv_data, *(const uint64_t*)v.mv_data}); + dbr = mdb_cursor_get(cur, &k, &v, MDB_NEXT); + if (dbr == MDB_NOTFOUND) + break; + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to get next spent output: " + std::string(mdb_strerror(dbr))); + } + mdb_cursor_close(cur); + return outs; +} + +static void get_per_amount_outputs(MDB_txn *txn, uint64_t amount, uint64_t &total, uint64_t &spent) +{ + MDB_cursor *cur; + int dbr = mdb_cursor_open(txn, dbi_per_amount, &cur); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open cursor for per amount outputs: " + std::string(mdb_strerror(dbr))); + MDB_val k, v; + mdb_size_t count = 0; + k.mv_size = sizeof(uint64_t); + k.mv_data = (void*)&amount; + dbr = mdb_cursor_get(cur, &k, &v, MDB_SET); + if (dbr == MDB_NOTFOUND) + { + total = spent = 0; + } + else + { + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to get per amount outputs: " + std::string(mdb_strerror(dbr))); + total = ((const uint64_t*)v.mv_data)[0]; + spent = ((const uint64_t*)v.mv_data)[1]; + } + mdb_cursor_close(cur); +} + +static void inc_per_amount_outputs(MDB_txn *txn, uint64_t amount, uint64_t total, uint64_t spent) +{ + MDB_cursor *cur; + int dbr = mdb_cursor_open(txn, dbi_per_amount, &cur); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open cursor for per amount outputs: " + std::string(mdb_strerror(dbr))); + MDB_val k, v; + mdb_size_t count = 0; + k.mv_size = sizeof(uint64_t); + k.mv_data = (void*)&amount; + dbr = mdb_cursor_get(cur, &k, &v, MDB_SET); + if (dbr == 0) + { + total += ((const uint64_t*)v.mv_data)[0]; + spent += ((const uint64_t*)v.mv_data)[1]; + } + else + { + CHECK_AND_ASSERT_THROW_MES(dbr == MDB_NOTFOUND, "Failed to get per amount outputs: " + std::string(mdb_strerror(dbr))); + } + uint64_t data[2] = {total, spent}; + v.mv_size = 2 * sizeof(uint64_t); + v.mv_data = (void*)data; + dbr = mdb_cursor_put(cur, &k, &v, 0); + mdb_cursor_close(cur); +} + +static uint64_t get_processed_txidx(const std::string &name) +{ + MDB_txn *txn; + bool tx_active = false; + + int dbr = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); + epee::misc_utils::auto_scope_leave_caller txn_dtor = epee::misc_utils::create_scope_leave_handler([&](){if (tx_active) mdb_txn_abort(txn);}); + tx_active = true; + + uint64_t height = 0; + MDB_val k, v; + k.mv_data = (void*)name.c_str(); + k.mv_size = name.size(); + dbr = mdb_get(txn, dbi_processed_txidx, &k, &v); + if (dbr != MDB_NOTFOUND) + { + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to get processed height: " + std::string(mdb_strerror(dbr))); + height = *(const uint64_t*)v.mv_data; + } + + dbr = mdb_txn_commit(txn); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to commit txn: " + std::string(mdb_strerror(dbr))); + tx_active = false; + + return height; +} + +static void set_processed_txidx(MDB_txn *txn, const std::string &name, uint64_t height) +{ + MDB_val k, v; + k.mv_data = (void*)name.c_str(); + k.mv_size = name.size(); + v.mv_data = (void*)&height; + v.mv_size = sizeof(height); + int dbr = mdb_put(txn, dbi_processed_txidx, &k, &v, 0); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to set processed height: " + std::string(mdb_strerror(dbr))); +} + +static bool get_relative_ring(MDB_txn *txn, const crypto::key_image &ki, std::vector &ring) +{ + MDB_val k, v; + k.mv_data = (void*)&ki; + k.mv_size = sizeof(ki); + int dbr = mdb_get(txn, dbi_relative_rings, &k, &v); + if (dbr == MDB_NOTFOUND) + return false; + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to get relative ring: " + std::string(mdb_strerror(dbr))); + ring = decompress_ring(std::string((const char*)v.mv_data, v.mv_size)); + return true; +} + +static void set_relative_ring(MDB_txn *txn, const crypto::key_image &ki, const std::vector &ring) +{ + const std::string sring = compress_ring(ring); + MDB_val k, v; + k.mv_data = (void*)&ki; + k.mv_size = sizeof(ki); + v.mv_data = (void*)sring.c_str(); + v.mv_size = sring.size(); + int dbr = mdb_put(txn, dbi_relative_rings, &k, &v, 0); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to set relative ring: " + std::string(mdb_strerror(dbr))); +} + +static std::string keep_under_511(const std::string &s) +{ + if (s.size() <= 511) + return s; + crypto::hash hash; + crypto::cn_fast_hash(s.data(), s.size(), hash); + return std::string((const char*)&hash, 32); +} + +static uint64_t get_ring_instances(MDB_txn *txn, uint64_t amount, const std::vector &ring) +{ + const std::string sring = keep_under_511(compress_ring(amount, ring)); + MDB_val k, v; + k.mv_data = (void*)sring.data(); + k.mv_size = sring.size(); + int dbr = mdb_get(txn, dbi_ring_instances, &k, &v); + if (dbr == MDB_NOTFOUND) + return 0; + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to get ring instances: " + std::string(mdb_strerror(dbr))); + return *(const uint64_t*)v.mv_data; +} + +static uint64_t get_ring_subset_instances(MDB_txn *txn, uint64_t amount, const std::vector &ring) +{ + uint64_t instances = get_ring_instances(txn, amount, ring); + if (ring.size() > 11) + return instances; + + uint64_t extra = 0; + std::vector subset; + subset.reserve(ring.size()); + for (uint64_t mask = 1; mask < (((uint64_t)1) << ring.size()) - 1; ++mask) + { + subset.resize(0); + for (size_t i = 0; i < ring.size(); ++i) + if ((mask >> i) & 1) + subset.push_back(ring[i]); + extra += get_ring_instances(txn, amount, subset); + } + return instances + extra; +} + +static uint64_t inc_ring_instances(MDB_txn *txn, uint64_t amount, const std::vector &ring) +{ + const std::string sring = keep_under_511(compress_ring(amount, ring)); + MDB_val k, v; + k.mv_data = (void*)sring.data(); + k.mv_size = sring.size(); + + int dbr = mdb_get(txn, dbi_ring_instances, &k, &v); + CHECK_AND_ASSERT_THROW_MES(!dbr || dbr == MDB_NOTFOUND, "Failed to get ring instances: " + std::string(mdb_strerror(dbr))); + + uint64_t count; + if (dbr == MDB_NOTFOUND) + count = 1; + else + count = 1 + *(const uint64_t*)v.mv_data; + + v.mv_data = &count; + v.mv_size = sizeof(count); + dbr = mdb_put(txn, dbi_ring_instances, &k, &v, 0); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to set ring instances: " + std::string(mdb_strerror(dbr))); + + return count; +} + +static std::vector get_key_images(MDB_txn *txn, const output_data &od) +{ + MDB_val k, v; + k.mv_data = (void*)&od; + k.mv_size = sizeof(od); + int dbr = mdb_get(txn, dbi_outputs, &k, &v); + CHECK_AND_ASSERT_THROW_MES(!dbr || dbr == MDB_NOTFOUND, "Failed to get output: " + std::string(mdb_strerror(dbr))); + if (dbr == MDB_NOTFOUND) + return {}; + CHECK_AND_ASSERT_THROW_MES(v.mv_size % 32 == 0, "Unexpected record size"); + std::vector key_images; + key_images.reserve(v.mv_size / 32); + const crypto::key_image *ki = (const crypto::key_image*)v.mv_data; + for (size_t n = 0; n < v.mv_size / 32; ++n) + key_images.push_back(*ki++); + return key_images; +} + +static void add_key_image(MDB_txn *txn, const output_data &od, const crypto::key_image &ki) +{ + MDB_val k, v; + k.mv_data = (void*)&od; + k.mv_size = sizeof(od); + int dbr = mdb_get(txn, dbi_outputs, &k, &v); + CHECK_AND_ASSERT_THROW_MES(!dbr || dbr == MDB_NOTFOUND, "Failed to get output"); + std::string data; + if (!dbr) + { + CHECK_AND_ASSERT_THROW_MES(v.mv_size % 32 == 0, "Unexpected record size"); + data = std::string((const char*)v.mv_data, v.mv_size); + } + data += std::string((const char*)&ki, sizeof(ki)); + + v.mv_data = (void*)data.data(); + v.mv_size = data.size(); + dbr = mdb_put(txn, dbi_outputs, &k, &v, 0); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to set outputs: " + std::string(mdb_strerror(dbr))); +} + +static bool get_stat(MDB_txn *txn, const char *key, uint64_t &data) +{ + MDB_val k, v; + k.mv_data = (void*)key; + k.mv_size = strlen(key); + int dbr = mdb_get(txn, dbi_stats, &k, &v); + if (dbr == MDB_NOTFOUND) + return false; + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to get stat record"); + CHECK_AND_ASSERT_THROW_MES(v.mv_size == sizeof(uint64_t), "Unexpected record size"); + data = *(const uint64_t*)v.mv_data; + return true; +} + +static void set_stat(MDB_txn *txn, const char *key, uint64_t data) +{ + MDB_val k, v; + k.mv_data = (void*)key; + k.mv_size = strlen(key); + v.mv_data = (void*)&data; + v.mv_size = sizeof(uint64_t); + int dbr = mdb_put(txn, dbi_stats, &k, &v, 0); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to set stat record"); +} + +static void inc_stat(MDB_txn *txn, const char *key) +{ + uint64_t data; + if (!get_stat(txn, key, data)) + data = 0; + ++data; + set_stat(txn, key, data); +} + +static void open_db(const std::string &filename, MDB_env **env, MDB_txn **txn, MDB_cursor **cur, MDB_dbi *dbi) +{ + tools::create_directories_if_necessary(filename); + + int flags = MDB_RDONLY; + if (db_flags & DBF_FAST) + flags |= MDB_NOSYNC; + if (db_flags & DBF_FASTEST) + flags |= MDB_NOSYNC | MDB_WRITEMAP | MDB_MAPASYNC; + + int dbr = mdb_env_create(env); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to create LDMB environment: " + std::string(mdb_strerror(dbr))); + dbr = mdb_env_set_maxdbs(*env, 1); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to set max env dbs: " + std::string(mdb_strerror(dbr))); + const std::string actual_filename = filename; + MINFO("Opening monero blockchain at " << actual_filename); + dbr = mdb_env_open(*env, actual_filename.c_str(), flags, 0664); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open rings database file '" + + actual_filename + "': " + std::string(mdb_strerror(dbr))); + + dbr = mdb_txn_begin(*env, NULL, MDB_RDONLY, txn); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); + + dbr = mdb_dbi_open(*txn, "output_amounts", MDB_CREATE | MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, dbi); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); + mdb_set_dupsort(*txn, *dbi, compare_uint64); + + dbr = mdb_cursor_open(*txn, *dbi, cur); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr))); +} + +static void close_db(MDB_env *env, MDB_txn *txn, MDB_cursor *cur, MDB_dbi dbi) +{ + mdb_txn_abort(txn); + mdb_cursor_close(cur); + mdb_dbi_close(env, dbi); + mdb_env_close(env); +} + +static void get_num_outputs(MDB_txn *txn, MDB_cursor *cur, MDB_dbi dbi, uint64_t &pre_rct, uint64_t &rct) +{ + uint64_t amount = 0; + MDB_val k = { sizeof(amount), (void*)&amount }, v; + int dbr = mdb_cursor_get(cur, &k, &v, MDB_SET); + if (dbr == MDB_NOTFOUND) + { + rct = 0; + } + else + { + if (dbr) throw std::runtime_error("Record 0 not found: " + std::string(mdb_strerror(dbr))); + mdb_size_t count = 0; + dbr = mdb_cursor_count(cur, &count); + if (dbr) throw std::runtime_error("Failed to count records: " + std::string(mdb_strerror(dbr))); + rct = count; + } + MDB_stat s; + dbr = mdb_stat(txn, dbi, &s); + if (dbr) throw std::runtime_error("Failed to count records: " + std::string(mdb_strerror(dbr))); + if (s.ms_entries < rct) throw std::runtime_error("Inconsistent records: " + std::string(mdb_strerror(dbr))); + pre_rct = s.ms_entries - rct; +} + +static crypto::hash get_genesis_block_hash(const std::string &filename) +{ + MDB_env *env; + MDB_dbi dbi; + MDB_txn *txn; + int dbr; + bool tx_active = false; + + dbr = mdb_env_create(&env); + if (dbr) throw std::runtime_error("Failed to create LDMB environment: " + std::string(mdb_strerror(dbr))); + dbr = mdb_env_set_maxdbs(env, 1); + if (dbr) throw std::runtime_error("Failed to set max env dbs: " + std::string(mdb_strerror(dbr))); + const std::string actual_filename = filename; + dbr = mdb_env_open(env, actual_filename.c_str(), 0, 0664); + if (dbr) throw std::runtime_error("Failed to open rings database file '" + + actual_filename + "': " + std::string(mdb_strerror(dbr))); + + dbr = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn); + if (dbr) throw std::runtime_error("Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); + epee::misc_utils::auto_scope_leave_caller txn_dtor = epee::misc_utils::create_scope_leave_handler([&](){if (tx_active) mdb_txn_abort(txn);}); + tx_active = true; + + dbr = mdb_dbi_open(txn, "block_info", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, &dbi); + mdb_set_dupsort(txn, dbi, compare_uint64); + if (dbr) throw std::runtime_error("Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); + uint64_t zero = 0; + MDB_val k = { sizeof(uint64_t), (void*)&zero}, v; + dbr = mdb_get(txn, dbi, &k, &v); + if (dbr) throw std::runtime_error("Failed to retrieve genesis block: " + std::string(mdb_strerror(dbr))); + crypto::hash genesis_block_hash = *(const crypto::hash*)(((const uint64_t*)v.mv_data) + 5); + mdb_dbi_close(env, dbi); + mdb_txn_abort(txn); + mdb_env_close(env); + tx_active = false; + return genesis_block_hash; +} + +static std::vector> load_outputs(const std::string &filename) +{ + std::vector> outputs; + uint64_t amount = std::numeric_limits::max(); + FILE *f; + + f = fopen(filename.c_str(), "r"); + if (!f) + { + MERROR("Failed to load outputs from " << filename << ": " << strerror(errno)); + return {}; + } + while (1) + { + char s[256]; + if (!fgets(s, sizeof(s), f)) + { + MERROR("Error reading from " << filename << ": " << strerror(errno)); + break; + } + if (feof(f)) + break; + const size_t len = strlen(s); + if (len > 0 && s[len - 1] == '\n') + s[len - 1] = 0; + if (!s[0]) + continue; + std::pair output; + uint64_t offset, num_offsets; + if (sscanf(s, "@%" PRIu64, &amount) == 1) + { + continue; + } + if (amount == std::numeric_limits::max()) + { + MERROR("Bad format in " << filename); + continue; + } + if (sscanf(s, "%" PRIu64 "*%" PRIu64, &offset, &num_offsets) == 2 && num_offsets < std::numeric_limits::max() - offset) + { + while (num_offsets-- > 0) + outputs.push_back(std::make_pair(amount, offset++)); + } + else if (sscanf(s, "%" PRIu64, &offset) == 1) + { + outputs.push_back(std::make_pair(amount, offset)); + } + else + { + MERROR("Bad format in " << filename); + continue; + } + } + fclose(f); + return outputs; +} + +static bool export_spent_outputs(MDB_cursor *cur, const std::string &filename) +{ + FILE *f = fopen(filename.c_str(), "w"); + if (!f) + { + MERROR("Failed to open " << filename << ": " << strerror(errno)); + return false; + } + + uint64_t pending_amount = std::numeric_limits::max(); + std::vector pending_offsets; + MDB_val k, v; + MDB_cursor_op op = MDB_FIRST; + while (1) + { + int dbr = mdb_cursor_get(cur, &k, &v, op); + if (dbr == MDB_NOTFOUND) + break; + op = MDB_NEXT; + if (dbr) + { + fclose(f); + MERROR("Failed to enumerate spent outputs: " << mdb_strerror(dbr)); + return false; + } + const uint64_t amount = *(const uint64_t*)k.mv_data; + const uint64_t offset = *(const uint64_t*)v.mv_data; + if (!pending_offsets.empty() && (amount != pending_amount || pending_offsets.back()+1 != offset)) + { + if (pending_offsets.size() == 1) + fprintf(f, "%" PRIu64 "\n", pending_offsets.front()); + else + fprintf(f, "%" PRIu64 "*%zu\n", pending_offsets.front(), pending_offsets.size()); + pending_offsets.clear(); + } + if (pending_amount != amount) + { + fprintf(f, "@%" PRIu64 "\n", amount); + pending_amount = amount; + } + pending_offsets.push_back(offset); + } + if (!pending_offsets.empty()) + { + if (pending_offsets.size() == 1) + fprintf(f, "%" PRIu64 "\n", pending_offsets.front()); + else + fprintf(f, "%" PRIu64 "*%zu\n", pending_offsets.front(), pending_offsets.size()); + pending_offsets.clear(); + } + fclose(f); + return true; +} + int main(int argc, char* argv[]) { TRY_ENTRY(); @@ -161,31 +1073,37 @@ int main(int argc, char* argv[]) po::options_description desc_cmd_only("Command line options"); po::options_description desc_cmd_sett("Command line options and settings options"); - const command_line::arg_descriptor arg_blackball_db_dir = { - "blackball-db-dir", "Specify blackball database directory", + const command_line::arg_descriptor arg_blackball_db_dir = { + "spent-output-db-dir", "Specify spent output database directory", get_default_db_path(), - {{ &arg_testnet_on, &arg_stagenet_on }}, - [](std::array testnet_stagenet, bool defaulted, std::string val) { - if (testnet_stagenet[0]) - return (boost::filesystem::path(val) / "testnet").string(); - else if (testnet_stagenet[1]) - return (boost::filesystem::path(val) / "stagenet").string(); - return val; - } }; const command_line::arg_descriptor arg_log_level = {"log-level", "0-4 or categories", ""}; const command_line::arg_descriptor arg_database = { "database", available_dbs.c_str(), default_db_type }; const command_line::arg_descriptor arg_rct_only = {"rct-only", "Only work on ringCT outputs", false}; + const command_line::arg_descriptor arg_check_subsets = {"check-subsets", "Check ring subsets (very expensive)", false}; + const command_line::arg_descriptor arg_verbose = {"verbose", "Verbose output)", false}; const command_line::arg_descriptor > arg_inputs = {"inputs", "Path to Monero DB, and path to any fork DBs"}; + const command_line::arg_descriptor arg_db_sync_mode = { + "db-sync-mode" + , "Specify sync option, using format [safe|fast|fastest]:[nrecords_per_sync]." + , "fast:1000" + }; + const command_line::arg_descriptor arg_extra_spent_list = {"extra-spent-list", "Optional list of known spent outputs",""}; + const command_line::arg_descriptor arg_export = {"export", "Filename to export the backball list to"}; + const command_line::arg_descriptor arg_force_chain_reaction_pass = {"force-chain-reaction-pass", "Run the chain reaction pass even if no new blockchain data was processed"}; command_line::add_arg(desc_cmd_sett, arg_blackball_db_dir); - command_line::add_arg(desc_cmd_sett, cryptonote::arg_testnet_on); - command_line::add_arg(desc_cmd_sett, cryptonote::arg_stagenet_on); command_line::add_arg(desc_cmd_sett, arg_log_level); command_line::add_arg(desc_cmd_sett, arg_database); command_line::add_arg(desc_cmd_sett, arg_rct_only); + command_line::add_arg(desc_cmd_sett, arg_check_subsets); + command_line::add_arg(desc_cmd_sett, arg_verbose); + command_line::add_arg(desc_cmd_sett, arg_db_sync_mode); + command_line::add_arg(desc_cmd_sett, arg_extra_spent_list); + command_line::add_arg(desc_cmd_sett, arg_export); + command_line::add_arg(desc_cmd_sett, arg_force_chain_reaction_pass); command_line::add_arg(desc_cmd_sett, arg_inputs); command_line::add_arg(desc_cmd_only, command_line::arg_help); @@ -213,7 +1131,7 @@ int main(int argc, char* argv[]) return 1; } - mlog_configure(mlog_get_default_log_path("monero-blockchain-blackball.log"), true); + mlog_configure(mlog_get_default_log_path("monero-blockchain-mark-spent-outputs.log"), true); if (!command_line::is_arg_defaulted(vm, arg_log_level)) mlog_set_log(command_line::get_arg(vm, arg_log_level).c_str()); else @@ -221,11 +1139,14 @@ int main(int argc, char* argv[]) LOG_PRINT_L0("Starting..."); - bool opt_testnet = command_line::get_arg(vm, cryptonote::arg_testnet_on); - bool opt_stagenet = command_line::get_arg(vm, cryptonote::arg_stagenet_on); - network_type net_type = opt_testnet ? TESTNET : opt_stagenet ? STAGENET : MAINNET; output_file_path = command_line::get_arg(vm, arg_blackball_db_dir); bool opt_rct_only = command_line::get_arg(vm, arg_rct_only); + bool opt_check_subsets = command_line::get_arg(vm, arg_check_subsets); + bool opt_verbose = command_line::get_arg(vm, arg_verbose); + bool opt_force_chain_reaction_pass = command_line::get_arg(vm, arg_force_chain_reaction_pass); + std::string opt_export = command_line::get_arg(vm, arg_export); + std::string extra_spent_list = command_line::get_arg(vm, arg_extra_spent_list); + std::vector> extra_spent_outputs = extra_spent_list.empty() ? std::vector>() : load_outputs(extra_spent_list); std::string db_type = command_line::get_arg(vm, arg_database); if (!cryptonote::blockchain_valid_db_type(db_type)) @@ -234,92 +1155,100 @@ int main(int argc, char* argv[]) return 1; } - // If we wanted to use the memory pool, we would set up a fake_core. + std::string db_sync_mode = command_line::get_arg(vm, arg_db_sync_mode); + if (!parse_db_sync_mode(db_sync_mode)) + { + MERROR("Invalid db sync mode: " << db_sync_mode); + return 1; + } - // Use Blockchain instead of lower-level BlockchainDB for two reasons: - // 1. Blockchain has the init() method for easy setup - // 2. exporter needs to use get_current_blockchain_height(), get_block_id_by_height(), get_block_by_hash() - // - // cannot match blockchain_storage setup above with just one line, - // e.g. - // Blockchain* core_storage = new Blockchain(NULL); - // because unlike blockchain_storage constructor, which takes a pointer to - // tx_memory_pool, Blockchain's constructor takes tx_memory_pool object. - LOG_PRINT_L0("Initializing source blockchain (BlockchainDB)"); const std::vector inputs = command_line::get_arg(vm, arg_inputs); if (inputs.empty()) { LOG_PRINT_L0("No inputs given"); return 1; } - std::vector> core_storage(inputs.size()); - Blockchain *blockchain = NULL; - tx_memory_pool m_mempool(*blockchain); - for (size_t n = 0; n < inputs.size(); ++n) - { - core_storage[n].reset(new Blockchain(m_mempool)); - BlockchainDB* db = new_db(db_type); - if (db == NULL) - { - LOG_ERROR("Attempted to use non-existent database type: " << db_type); - throw std::runtime_error("Attempting to use non-existent database type"); - } - LOG_PRINT_L0("database: " << db_type); + const std::string cache_dir = (output_file_path / "spent-outputs-cache").string(); + init(cache_dir); - std::string filename = inputs[n]; - while (boost::ends_with(filename, "/") || boost::ends_with(filename, "\\")) - filename.pop_back(); - LOG_PRINT_L0("Loading blockchain from folder " << filename << " ..."); + LOG_PRINT_L0("Scanning for spent outputs..."); - try - { - db->open(filename, DBF_RDONLY); - } - catch (const std::exception& e) - { - LOG_PRINT_L0("Error opening database: " << e.what()); - return 1; - } - r = core_storage[n]->init(db, net_type); + size_t done = 0; - CHECK_AND_ASSERT_MES(r, 1, "Failed to initialize source blockchain storage"); - LOG_PRINT_L0("Source blockchain storage initialized OK"); - } + const uint64_t start_blackballed_outputs = get_num_spent_outputs(); + + tools::ringdb ringdb(output_file_path.string(), epee::string_tools::pod_to_hex(get_genesis_block_hash(inputs[0]))); + + bool stop_requested = false; + tools::signal_handler::install([&stop_requested](int type) { + stop_requested = true; + }); + + int dbr = resize_env(cache_dir.c_str()); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to resize LMDB database: " + std::string(mdb_strerror(dbr))); - boost::filesystem::path direc(output_file_path.string()); - if (boost::filesystem::exists(direc)) + // open first db + MDB_env *env0; + MDB_txn *txn0; + MDB_dbi dbi0; + MDB_cursor *cur0; + open_db(inputs[0], &env0, &txn0, &cur0, &dbi0); + + if (!extra_spent_outputs.empty()) { - if (!boost::filesystem::is_directory(direc)) + MINFO("Adding " << extra_spent_outputs.size() << " extra spent outputs"); + MDB_txn *txn; + int dbr = mdb_txn_begin(env, NULL, 0, &txn); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); + MDB_cursor *cur; + dbr = mdb_cursor_open(txn, dbi_spent, &cur); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open LMDB cursor: " + std::string(mdb_strerror(dbr))); + + std::vector> blackballs; + for (const std::pair &output: extra_spent_outputs) { - MERROR("LMDB needs a directory path, but a file was passed: " << output_file_path.string()); - return 1; + if (!is_output_spent(cur, output_data(output.first, output.second))) + { + blackballs.push_back(output); + if (add_spent_output(cur, output_data(output.first, output.second))) + inc_stat(txn, output.first ? "pre-rct-extra" : "rct-ring-extra"); + } } - } - else - { - if (!boost::filesystem::create_directories(direc)) + if (!blackballs.empty()) { - MERROR("Failed to create directory: " << output_file_path.string()); - return 1; + ringdb.blackball(blackballs); + blackballs.clear(); } + mdb_cursor_close(cur); + dbr = mdb_txn_commit(txn); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to commit txn creating/opening database: " + std::string(mdb_strerror(dbr))); } - LOG_PRINT_L0("Scanning for blackballable outputs..."); - - size_t done = 0; - std::unordered_map> relative_rings; - std::unordered_map> outputs; - std::unordered_set spent, newly_spent; - - cryptonote::block b = core_storage[0]->get_db().get_block_from_height(0); - tools::ringdb ringdb(output_file_path.string(), epee::string_tools::pod_to_hex(get_block_hash(b))); - for (size_t n = 0; n < inputs.size(); ++n) { - LOG_PRINT_L0("Reading blockchain from " << inputs[n]); - for_all_transactions(inputs[n], [&](const cryptonote::transaction_prefix &tx)->bool + const std::string canonical = boost::filesystem::canonical(inputs[n]).string(); + uint64_t start_idx = get_processed_txidx(canonical); + if (n > 0 && start_idx == 0) { + start_idx = find_first_diverging_transaction(inputs[0], inputs[n]); + LOG_PRINT_L0("First diverging transaction at " << start_idx); + } + LOG_PRINT_L0("Reading blockchain from " << inputs[n] << " from " << start_idx); + MDB_txn *txn; + int dbr = mdb_txn_begin(env, NULL, 0, &txn); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); + MDB_cursor *cur; + dbr = mdb_cursor_open(txn, dbi_spent, &cur); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open LMDB cursor: " + std::string(mdb_strerror(dbr))); + size_t records = 0; + const std::string filename = inputs[n]; + std::vector> blackballs; + uint64_t n_txes; + for_all_transactions(filename, start_idx, n_txes, [&](const cryptonote::transaction_prefix &tx)->bool + { + std::cout << "\r" << start_idx << "/" << n_txes << " \r" << std::flush; + const bool miner_tx = tx.vin.size() == 1 && tx.vin[0].type() == typeid(txin_gen); for (const auto &in: tx.vin) { if (in.type() != typeid(txin_to_key)) @@ -331,27 +1260,83 @@ int main(int argc, char* argv[]) const std::vector absolute = cryptonote::relative_output_offsets_to_absolute(txin.key_offsets); if (n == 0) for (uint64_t out: absolute) - outputs[output_data(txin.amount, out)].insert(txin.k_image); + add_key_image(txn, output_data(txin.amount, out), txin.k_image); - std::vector new_ring = txin.key_offsets; + std::vector relative_ring; + std::vector new_ring = canonicalize(txin.key_offsets); const uint32_t ring_size = txin.key_offsets.size(); - if (ring_size == 1) + const uint64_t instances = inc_ring_instances(txn, txin.amount, new_ring); + uint64_t pa_total = 0, pa_spent = 0; + if (!opt_rct_only) + get_per_amount_outputs(txn, txin.amount, pa_total, pa_spent); + if (n == 0 && ring_size == 1) { - const crypto::public_key pkey = core_storage[n]->get_output_key(txin.amount, txin.key_offsets[0]); - MINFO("Blackballing output " << pkey << ", due to being used in a 1-ring"); - ringdb.blackball(pkey); - newly_spent.insert(output_data(txin.amount, txin.key_offsets[0])); - spent.insert(output_data(txin.amount, txin.key_offsets[0])); + const std::pair output = std::make_pair(txin.amount, absolute[0]); + if (opt_verbose) + { + MINFO("Marking output " << output.first << "/" << output.second << " as spent, due to being used in a 1-ring"); + std::cout << "\r" << start_idx << "/" << n_txes << " \r" << std::flush; + } + blackballs.push_back(output); + if (add_spent_output(cur, output_data(txin.amount, absolute[0]))) + inc_stat(txn, txin.amount ? "pre-rct-ring-size-1" : "rct-ring-size-1"); } - else if (relative_rings.find(txin.k_image) != relative_rings.end()) + else if (n == 0 && instances == new_ring.size()) { - MINFO("Key image " << txin.k_image << " already seen: rings " << - boost::join(relative_rings[txin.k_image] | boost::adaptors::transformed([](uint64_t out){return std::to_string(out);}), " ") << + for (size_t o = 0; o < new_ring.size(); ++o) + { + const std::pair output = std::make_pair(txin.amount, absolute[o]); + if (opt_verbose) + { + MINFO("Marking output " << output.first << "/" << output.second << " as spent, due to being used in " << new_ring.size() << " identical " << new_ring.size() << "-rings"); + std::cout << "\r" << start_idx << "/" << n_txes << " \r" << std::flush; + } + blackballs.push_back(output); + if (add_spent_output(cur, output_data(txin.amount, absolute[o]))) + inc_stat(txn, txin.amount ? "pre-rct-duplicate-rings" : "rct-duplicate-rings"); + } + } + else if (n == 0 && !opt_rct_only && pa_spent + 1 == pa_total) + { + for (size_t o = 0; o < pa_total; ++o) + { + const std::pair output = std::make_pair(txin.amount, o); + if (opt_verbose) + { + MINFO("Marking output " << output.first << "/" << output.second << " as spent, due to as many outputs of that amount being spent as exist so far"); + std::cout << "\r" << start_idx << "/" << n_txes << " \r" << std::flush; + } + blackballs.push_back(output); + if (add_spent_output(cur, output_data(txin.amount, o))) + inc_stat(txn, txin.amount ? "pre-rct-full-count" : "rct-full-count"); + } + } + else if (n == 0 && opt_check_subsets && get_ring_subset_instances(txn, txin.amount, new_ring) >= new_ring.size()) + { + for (size_t o = 0; o < new_ring.size(); ++o) + { + const std::pair output = std::make_pair(txin.amount, absolute[o]); + if (opt_verbose) + { + MINFO("Marking output " << output.first << "/" << output.second << " as spent, due to being used in " << new_ring.size() << " subsets of " << new_ring.size() << "-rings"); + std::cout << "\r" << start_idx << "/" << n_txes << " \r" << std::flush; + } + blackballs.push_back(output); + if (add_spent_output(cur, output_data(txin.amount, absolute[o]))) + inc_stat(txn, txin.amount ? "pre-rct-subset-rings" : "rct-subset-rings"); + } + } + else if (n > 0 && get_relative_ring(txn, txin.k_image, relative_ring)) + { + MDEBUG("Key image " << txin.k_image << " already seen: rings " << + boost::join(relative_ring | boost::adaptors::transformed([](uint64_t out){return std::to_string(out);}), " ") << ", " << boost::join(txin.key_offsets | boost::adaptors::transformed([](uint64_t out){return std::to_string(out);}), " ")); - if (relative_rings[txin.k_image] != txin.key_offsets) + std::cout << "\r" << start_idx << "/" << n_txes << " \r" << std::flush; + if (relative_ring != txin.key_offsets) { - MINFO("Rings are different"); - const std::vector r0 = cryptonote::relative_output_offsets_to_absolute(relative_rings[txin.k_image]); + MDEBUG("Rings are different"); + std::cout << "\r" << start_idx << "/" << n_txes << " \r" << std::flush; + const std::vector r0 = cryptonote::relative_output_offsets_to_absolute(relative_ring); const std::vector r1 = cryptonote::relative_output_offsets_to_absolute(txin.key_offsets); std::vector common; for (uint64_t out: r0) @@ -362,18 +1347,24 @@ int main(int argc, char* argv[]) if (common.empty()) { MERROR("Rings for the same key image are disjoint"); + std::cout << "\r" << start_idx << "/" << n_txes << " \r" << std::flush; } else if (common.size() == 1) { - const crypto::public_key pkey = core_storage[n]->get_output_key(txin.amount, common[0]); - MINFO("Blackballing output " << pkey << ", due to being used in rings with a single common element"); - ringdb.blackball(pkey); - newly_spent.insert(output_data(txin.amount, common[0])); - spent.insert(output_data(txin.amount, common[0])); + const std::pair output = std::make_pair(txin.amount, common[0]); + if (opt_verbose) + { + MINFO("Marking output " << output.first << "/" << output.second << " as spent, due to being used in rings with a single common element"); + std::cout << "\r" << start_idx << "/" << n_txes << " \r" << std::flush; + } + blackballs.push_back(output); + if (add_spent_output(cur, output_data(txin.amount, common[0]))) + inc_stat(txn, txin.amount ? "pre-rct-key-image-attack" : "rct-key-image-attack"); } else { - MINFO("The intersection has more than one element, it's still ok"); + MDEBUG("The intersection has more than one element, it's still ok"); + std::cout << "\r" << start_idx << "/" << n_txes << " \r" << std::flush; for (const auto &out: r0) if (std::find(common.begin(), common.end(), out) != common.end()) new_ring.push_back(out); @@ -381,48 +1372,192 @@ int main(int argc, char* argv[]) } } } - relative_rings[txin.k_image] = new_ring; + if (n == 0) + { + set_relative_ring(txn, txin.k_image, new_ring); + if (!opt_rct_only) + inc_per_amount_outputs(txn, txin.amount, 0, 1); + } + } + set_processed_txidx(txn, canonical, start_idx+1); + if (!opt_rct_only) + { + for (const auto &out: tx.vout) + { + uint64_t amount = out.amount; + if (miner_tx && tx.version >= 2) + amount = 0; + + if (opt_rct_only && amount != 0) + continue; + if (out.target.type() != typeid(txout_to_key)) + continue; + inc_per_amount_outputs(txn, amount, 1, 0); + } + } + + ++records; + if (records >= records_per_sync) + { + if (!blackballs.empty()) + { + ringdb.blackball(blackballs); + blackballs.clear(); + } + mdb_cursor_close(cur); + dbr = mdb_txn_commit(txn); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to commit txn creating/opening database: " + std::string(mdb_strerror(dbr))); + int dbr = resize_env(cache_dir.c_str()); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to resize LMDB database: " + std::string(mdb_strerror(dbr))); + dbr = mdb_txn_begin(env, NULL, 0, &txn); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); + dbr = mdb_cursor_open(txn, dbi_spent, &cur); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open LMDB cursor: " + std::string(mdb_strerror(dbr))); + records = 0; + } + + if (stop_requested) + { + MINFO("Stopping scan..."); + return false; } return true; }); + mdb_cursor_close(cur); + dbr = mdb_txn_commit(txn); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to commit txn creating/opening database: " + std::string(mdb_strerror(dbr))); + LOG_PRINT_L0("blockchain from " << inputs[n] << " processed till tx idx " << start_idx); + if (stop_requested) + break; } - while (!newly_spent.empty()) + std::vector work_spent; + + if (stop_requested) + goto skip_secondary_passes; + + if (opt_force_chain_reaction_pass || get_num_spent_outputs() > start_blackballed_outputs) { - LOG_PRINT_L0("Secondary pass due to " << newly_spent.size() << " newly found spent outputs"); - std::unordered_set work_spent = std::move(newly_spent); - newly_spent.clear(); + MDB_txn *txn; + dbr = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); + work_spent = get_spent_outputs(txn); + mdb_txn_abort(txn); + } - for (const output_data &od: work_spent) + while (!work_spent.empty()) + { + LOG_PRINT_L0("Secondary pass on " << work_spent.size() << " spent outputs"); + + int dbr = resize_env(cache_dir.c_str()); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to resize LMDB database: " + std::string(mdb_strerror(dbr))); + + MDB_txn *txn; + dbr = mdb_txn_begin(env, NULL, 0, &txn); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); + MDB_cursor *cur; + dbr = mdb_cursor_open(txn, dbi_spent, &cur); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open LMDB cursor: " + std::string(mdb_strerror(dbr))); + + std::vector> blackballs; + std::vector scan_spent = std::move(work_spent); + work_spent.clear(); + for (const output_data &od: scan_spent) { - for (const crypto::key_image &ki: outputs[od]) + std::vector key_images = get_key_images(txn, od); + for (const crypto::key_image &ki: key_images) { - std::vector absolute = cryptonote::relative_output_offsets_to_absolute(relative_rings[ki]); + std::vector relative_ring; + CHECK_AND_ASSERT_THROW_MES(get_relative_ring(txn, ki, relative_ring), "Relative ring not found"); + std::vector absolute = cryptonote::relative_output_offsets_to_absolute(relative_ring); size_t known = 0; uint64_t last_unknown = 0; for (uint64_t out: absolute) { output_data new_od(od.amount, out); - if (spent.find(new_od) != spent.end()) + if (is_output_spent(cur, new_od)) ++known; else last_unknown = out; } if (known == absolute.size() - 1) { - const crypto::public_key pkey = core_storage[0]->get_output_key(od.amount, last_unknown); - MINFO("Blackballing output " << pkey << ", due to being used in a " << - absolute.size() << "-ring where all other outputs are known to be spent"); - ringdb.blackball(pkey); - newly_spent.insert(output_data(od.amount, last_unknown)); - spent.insert(output_data(od.amount, last_unknown)); + const std::pair output = std::make_pair(od.amount, last_unknown); + if (opt_verbose) + { + MINFO("Marking output " << output.first << "/" << output.second << " as spent, due to being used in a " << + absolute.size() << "-ring where all other outputs are known to be spent"); + } + blackballs.push_back(output); + if (add_spent_output(cur, output_data(od.amount, last_unknown))) + inc_stat(txn, od.amount ? "pre-rct-chain-reaction" : "rct-chain-reaction"); + work_spent.push_back(output_data(od.amount, last_unknown)); } } + + if (stop_requested) + { + MINFO("Stopping secondary passes. Secondary passes are not incremental, they will re-run fully."); + return 0; + } + } + if (!blackballs.empty()) + { + ringdb.blackball(blackballs); + blackballs.clear(); } + mdb_cursor_close(cur); + dbr = mdb_txn_commit(txn); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to commit txn creating/opening database: " + std::string(mdb_strerror(dbr))); + } + +skip_secondary_passes: + uint64_t diff = get_num_spent_outputs() - start_blackballed_outputs; + LOG_PRINT_L0(std::to_string(diff) << " new outputs marked as spent, " << get_num_spent_outputs() << " total outputs marked as spent"); + + MDB_txn *txn; + dbr = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); + uint64_t pre_rct = 0, rct = 0; + get_num_outputs(txn0, cur0, dbi0, pre_rct, rct); + MINFO("Total pre-rct outputs: " << pre_rct); + MINFO("Total rct outputs: " << rct); + static const struct { const char *key; uint64_t base; } stat_keys[] = { + { "pre-rct-ring-size-1", pre_rct }, { "rct-ring-size-1", rct }, + { "pre-rct-duplicate-rings", pre_rct }, { "rct-duplicate-rings", rct }, + { "pre-rct-subset-rings", pre_rct }, { "rct-subset-rings", rct }, + { "pre-rct-full-count", pre_rct }, { "rct-full-count", rct }, + { "pre-rct-key-image-attack", pre_rct }, { "rct-key-image-attack", rct }, + { "pre-rct-extra", pre_rct }, { "rct-ring-extra", rct }, + { "pre-rct-chain-reaction", pre_rct }, { "rct-chain-reaction", rct }, + }; + for (const auto &key: stat_keys) + { + uint64_t data; + if (!get_stat(txn, key.key, data)) + data = 0; + float percent = key.base ? 100.0f * data / key.base : 0.0f; + MINFO(key.key << ": " << data << " (" << percent << "%)"); + } + mdb_txn_abort(txn); + + if (!opt_export.empty()) + { + MDB_txn *txn; + int dbr = mdb_txn_begin(env, NULL, 0, &txn); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); + MDB_cursor *cur; + dbr = mdb_cursor_open(txn, dbi_spent, &cur); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open LMDB cursor: " + std::string(mdb_strerror(dbr))); + export_spent_outputs(cur, opt_export); + mdb_cursor_close(cur); + mdb_txn_abort(txn); } - LOG_PRINT_L0("Blockchain blackball data exported OK"); + LOG_PRINT_L0("Blockchain spent output data exported OK"); + close_db(env0, txn0, cur0, dbi0); + close(); return 0; - CATCH_ENTRY("Export error", 1); + CATCH_ENTRY("Error", 1); } diff --git a/src/blockchain_utilities/blockchain_depth.cpp b/src/blockchain_utilities/blockchain_depth.cpp new file mode 100644 index 00000000000..8be83ee679c --- /dev/null +++ b/src/blockchain_utilities/blockchain_depth.cpp @@ -0,0 +1,350 @@ +// Copyright (c) 2014-2019, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include +#include +#include "common/command_line.h" +#include "common/varint.h" +#include "cryptonote_core/tx_pool.h" +#include "cryptonote_core/cryptonote_core.h" +#include "cryptonote_core/blockchain.h" +#include "blockchain_db/blockchain_db.h" +#include "blockchain_db/db_types.h" +#include "version.h" + +#undef MONERO_DEFAULT_LOG_CATEGORY +#define MONERO_DEFAULT_LOG_CATEGORY "bcutil" + +namespace po = boost::program_options; +using namespace epee; +using namespace cryptonote; + +int main(int argc, char* argv[]) +{ + TRY_ENTRY(); + + epee::string_tools::set_module_name_and_folder(argv[0]); + + std::string default_db_type = "lmdb"; + + std::string available_dbs = cryptonote::blockchain_db_types(", "); + available_dbs = "available: " + available_dbs; + + uint32_t log_level = 0; + + tools::on_startup(); + + boost::filesystem::path output_file_path; + + po::options_description desc_cmd_only("Command line options"); + po::options_description desc_cmd_sett("Command line options and settings options"); + const command_line::arg_descriptor arg_log_level = {"log-level", "0-4 or categories", ""}; + const command_line::arg_descriptor arg_database = { + "database", available_dbs.c_str(), default_db_type + }; + const command_line::arg_descriptor arg_txid = {"txid", "Get min depth for this txid", ""}; + const command_line::arg_descriptor arg_height = {"height", "Get min depth for all txes at this height", 0}; + const command_line::arg_descriptor arg_include_coinbase = {"include-coinbase", "Include coinbase in the average", false}; + + command_line::add_arg(desc_cmd_sett, cryptonote::arg_data_dir); + command_line::add_arg(desc_cmd_sett, cryptonote::arg_testnet_on); + command_line::add_arg(desc_cmd_sett, cryptonote::arg_stagenet_on); + command_line::add_arg(desc_cmd_sett, arg_log_level); + command_line::add_arg(desc_cmd_sett, arg_database); + command_line::add_arg(desc_cmd_sett, arg_txid); + command_line::add_arg(desc_cmd_sett, arg_height); + command_line::add_arg(desc_cmd_sett, arg_include_coinbase); + command_line::add_arg(desc_cmd_only, command_line::arg_help); + + po::options_description desc_options("Allowed options"); + desc_options.add(desc_cmd_only).add(desc_cmd_sett); + + po::variables_map vm; + bool r = command_line::handle_error_helper(desc_options, [&]() + { + auto parser = po::command_line_parser(argc, argv).options(desc_options); + po::store(parser.run(), vm); + po::notify(vm); + return true; + }); + if (! r) + return 1; + + if (command_line::get_arg(vm, command_line::arg_help)) + { + std::cout << "Monero '" << MONERO_RELEASE_NAME << "' (v" << MONERO_VERSION_FULL << ")" << ENDL << ENDL; + std::cout << desc_options << std::endl; + return 1; + } + + mlog_configure(mlog_get_default_log_path("monero-blockchain-depth.log"), true); + if (!command_line::is_arg_defaulted(vm, arg_log_level)) + mlog_set_log(command_line::get_arg(vm, arg_log_level).c_str()); + else + mlog_set_log(std::string(std::to_string(log_level) + ",bcutil:INFO").c_str()); + + LOG_PRINT_L0("Starting..."); + + std::string opt_data_dir = command_line::get_arg(vm, cryptonote::arg_data_dir); + bool opt_testnet = command_line::get_arg(vm, cryptonote::arg_testnet_on); + bool opt_stagenet = command_line::get_arg(vm, cryptonote::arg_stagenet_on); + network_type net_type = opt_testnet ? TESTNET : opt_stagenet ? STAGENET : MAINNET; + std::string opt_txid_string = command_line::get_arg(vm, arg_txid); + uint64_t opt_height = command_line::get_arg(vm, arg_height); + bool opt_include_coinbase = command_line::get_arg(vm, arg_include_coinbase); + + if (!opt_txid_string.empty() && opt_height) + { + std::cerr << "txid and height cannot be given at the same time" << std::endl; + return 1; + } + crypto::hash opt_txid = crypto::null_hash; + if (!opt_txid_string.empty()) + { + if (!epee::string_tools::hex_to_pod(opt_txid_string, opt_txid)) + { + std::cerr << "Invalid txid" << std::endl; + return 1; + } + } + + std::string db_type = command_line::get_arg(vm, arg_database); + if (!cryptonote::blockchain_valid_db_type(db_type)) + { + std::cerr << "Invalid database type: " << db_type << std::endl; + return 1; + } + + // If we wanted to use the memory pool, we would set up a fake_core. + + // Use Blockchain instead of lower-level BlockchainDB for two reasons: + // 1. Blockchain has the init() method for easy setup + // 2. exporter needs to use get_current_blockchain_height(), get_block_id_by_height(), get_block_by_hash() + // + // cannot match blockchain_storage setup above with just one line, + // e.g. + // Blockchain* core_storage = new Blockchain(NULL); + // because unlike blockchain_storage constructor, which takes a pointer to + // tx_memory_pool, Blockchain's constructor takes tx_memory_pool object. + LOG_PRINT_L0("Initializing source blockchain (BlockchainDB)"); + std::unique_ptr core_storage; + tx_memory_pool m_mempool(*core_storage); + core_storage.reset(new Blockchain(m_mempool)); + BlockchainDB *db = new_db(db_type); + if (db == NULL) + { + LOG_ERROR("Attempted to use non-existent database type: " << db_type); + throw std::runtime_error("Attempting to use non-existent database type"); + } + LOG_PRINT_L0("database: " << db_type); + + const std::string filename = (boost::filesystem::path(opt_data_dir) / db->get_db_name()).string(); + LOG_PRINT_L0("Loading blockchain from folder " << filename << " ..."); + + try + { + db->open(filename, DBF_RDONLY); + } + catch (const std::exception& e) + { + LOG_PRINT_L0("Error opening database: " << e.what()); + return 1; + } + r = core_storage->init(db, net_type); + + CHECK_AND_ASSERT_MES(r, 1, "Failed to initialize source blockchain storage"); + LOG_PRINT_L0("Source blockchain storage initialized OK"); + + std::vector start_txids; + if (!opt_txid_string.empty()) + { + start_txids.push_back(opt_txid); + } + else + { + const cryptonote::blobdata bd = db->get_block_blob_from_height(opt_height); + cryptonote::block b; + if (!cryptonote::parse_and_validate_block_from_blob(bd, b)) + { + LOG_PRINT_L0("Bad block from db"); + return 1; + } + for (const crypto::hash &txid: b.tx_hashes) + start_txids.push_back(txid); + if (opt_include_coinbase) + start_txids.push_back(cryptonote::get_transaction_hash(b.miner_tx)); + } + + if (start_txids.empty()) + { + LOG_PRINT_L0("No transaction(s) to check"); + return 1; + } + + std::vector depths; + for (const crypto::hash &start_txid: start_txids) + { + uint64_t depth = 0; + bool coinbase = false; + + LOG_PRINT_L0("Checking depth for txid " << start_txid); + std::vector txids(1, start_txid); + while (!coinbase) + { + LOG_PRINT_L0("Considering "<< txids.size() << " transaction(s) at depth " << depth); + std::vector new_txids; + for (const crypto::hash &txid: txids) + { + cryptonote::blobdata bd; + if (!db->get_pruned_tx_blob(txid, bd)) + { + LOG_PRINT_L0("Failed to get txid " << txid << " from db"); + return 1; + } + cryptonote::transaction tx; + if (!cryptonote::parse_and_validate_tx_base_from_blob(bd, tx)) + { + LOG_PRINT_L0("Bad tx: " << txid); + return 1; + } + for (size_t ring = 0; ring < tx.vin.size(); ++ring) + { + if (tx.vin[ring].type() == typeid(cryptonote::txin_gen)) + { + MDEBUG(txid << " is a coinbase transaction"); + coinbase = true; + goto done; + } + if (tx.vin[ring].type() == typeid(cryptonote::txin_to_key)) + { + const cryptonote::txin_to_key &txin = boost::get(tx.vin[ring]); + const uint64_t amount = txin.amount; + auto absolute_offsets = cryptonote::relative_output_offsets_to_absolute(txin.key_offsets); + for (uint64_t offset: absolute_offsets) + { + const output_data_t od = db->get_output_key(amount, offset); + const crypto::hash block_hash = db->get_block_hash_from_height(od.height); + bd = db->get_block_blob(block_hash); + cryptonote::block b; + if (!cryptonote::parse_and_validate_block_from_blob(bd, b)) + { + LOG_PRINT_L0("Bad block from db"); + return 1; + } + // find the tx which created this output + bool found = false; + for (size_t out = 0; out < b.miner_tx.vout.size(); ++out) + { + if (b.miner_tx.vout[out].target.type() == typeid(cryptonote::txout_to_key)) + { + const auto &txout = boost::get(b.miner_tx.vout[out].target); + if (txout.key == od.pubkey) + { + found = true; + new_txids.push_back(cryptonote::get_transaction_hash(b.miner_tx)); + MDEBUG("adding txid: " << cryptonote::get_transaction_hash(b.miner_tx)); + break; + } + } + else + { + LOG_PRINT_L0("Bad vout type in txid " << cryptonote::get_transaction_hash(b.miner_tx)); + return 1; + } + } + for (const crypto::hash &block_txid: b.tx_hashes) + { + if (found) + break; + if (!db->get_pruned_tx_blob(block_txid, bd)) + { + LOG_PRINT_L0("Failed to get txid " << block_txid << " from db"); + return 1; + } + cryptonote::transaction tx2; + if (!cryptonote::parse_and_validate_tx_base_from_blob(bd, tx2)) + { + LOG_PRINT_L0("Bad tx: " << block_txid); + return 1; + } + for (size_t out = 0; out < tx2.vout.size(); ++out) + { + if (tx2.vout[out].target.type() == typeid(cryptonote::txout_to_key)) + { + const auto &txout = boost::get(tx2.vout[out].target); + if (txout.key == od.pubkey) + { + found = true; + new_txids.push_back(block_txid); + MDEBUG("adding txid: " << block_txid); + break; + } + } + else + { + LOG_PRINT_L0("Bad vout type in txid " << block_txid); + return 1; + } + } + } + if (!found) + { + LOG_PRINT_L0("Output originating transaction not found"); + return 1; + } + } + } + else + { + LOG_PRINT_L0("Bad vin type in txid " << txid); + return 1; + } + } + } + if (!coinbase) + { + std::swap(txids, new_txids); + ++depth; + } + } +done: + LOG_PRINT_L0("Min depth for txid " << start_txid << ": " << depth); + depths.push_back(depth); + } + + uint64_t cumulative_depth = 0; + for (uint64_t depth: depths) + cumulative_depth += depth; + LOG_PRINT_L0("Average min depth for " << start_txids.size() << " transaction(s): " << cumulative_depth/(float)depths.size()); + LOG_PRINT_L0("Median min depth for " << start_txids.size() << " transaction(s): " << epee::misc_utils::median(depths)); + + core_storage->deinit(); + return 0; + + CATCH_ENTRY("Depth query error", 1); +} diff --git a/src/blockchain_utilities/blockchain_export.cpp b/src/blockchain_utilities/blockchain_export.cpp index 5a49f34781e..fa1243c1f9d 100644 --- a/src/blockchain_utilities/blockchain_export.cpp +++ b/src/blockchain_utilities/blockchain_export.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -177,6 +177,12 @@ int main(int argc, char* argv[]) } r = core_storage->init(db, opt_testnet ? cryptonote::TESTNET : opt_stagenet ? cryptonote::STAGENET : cryptonote::MAINNET); + if (core_storage->get_blockchain_pruning_seed()) + { + LOG_PRINT_L0("Blockchain is pruned, cannot export"); + return 1; + } + CHECK_AND_ASSERT_MES(r, 1, "Failed to initialize source blockchain storage"); LOG_PRINT_L0("Source blockchain storage initialized OK"); LOG_PRINT_L0("Exporting blockchain raw data..."); diff --git a/src/blockchain_utilities/blockchain_import.cpp b/src/blockchain_utilities/blockchain_import.cpp index caa549c1327..cb9154f2932 100644 --- a/src/blockchain_utilities/blockchain_import.cpp +++ b/src/blockchain_utilities/blockchain_import.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -33,9 +33,11 @@ #include #include +#include #include "misc_log_ex.h" #include "bootstrap_file.h" #include "bootstrap_serialization.h" +#include "blocks/blocks.h" #include "cryptonote_basic/cryptonote_format_utils.h" #include "serialization/binary_utils.h" // dump_binary(), parse_binary() #include "serialization/json_utils.h" // dump_json() @@ -164,7 +166,7 @@ int pop_blocks(cryptonote::core& core, int num_blocks) return num_blocks; } -int check_flush(cryptonote::core &core, std::list &blocks, bool force) +int check_flush(cryptonote::core &core, std::vector &blocks, bool force) { if (blocks.empty()) return 0; @@ -176,7 +178,7 @@ int check_flush(cryptonote::core &core, std::list &blocks, if (!force && new_height % HASH_OF_HASHES_STEP) return 0; - std::list hashes; + std::vector hashes; for (const auto &b: blocks) { cryptonote::block block; @@ -191,8 +193,20 @@ int check_flush(cryptonote::core &core, std::list &blocks, } core.prevalidate_block_hashes(core.get_blockchain_storage().get_db().height(), hashes); - core.prepare_handle_incoming_blocks(blocks); + std::vector pblocks; + if (!core.prepare_handle_incoming_blocks(blocks, pblocks)) + { + MERROR("Failed to prepare to add blocks"); + return 1; + } + if (!pblocks.empty() && pblocks.size() != blocks.size()) + { + MERROR("Unexpected parsed blocks size"); + core.cleanup_handle_incoming_blocks(); + return 1; + } + size_t blockidx = 0; for(const block_complete_entry& block_entry: blocks) { // process transactions @@ -213,7 +227,7 @@ int check_flush(cryptonote::core &core, std::list &blocks, block_verification_context bvc = boost::value_initialized(); - core.handle_incoming_block(block_entry.block, bvc, false); // <--- process block + core.handle_incoming_block(block_entry.block, pblocks.empty() ? NULL : &pblocks[blockidx++], bvc, false); // <--- process block if(bvc.m_verifivation_failed) { @@ -284,7 +298,8 @@ int import_from_file(cryptonote::core& core, const std::string& import_file_path } // 4 byte magic + (currently) 1024 byte header structures - bootstrap.seek_to_first_chunk(import_file); + uint8_t major_version, minor_version; + bootstrap.seek_to_first_chunk(import_file, major_version, minor_version); std::string str1; char buffer1[1024]; @@ -312,7 +327,7 @@ int import_from_file(cryptonote::core& core, const std::string& import_file_path MINFO("Reading blockchain from bootstrap file..."); std::cout << ENDL; - std::list blocks; + std::vector blocks; // Skip to start_height before we start adding. { @@ -394,7 +409,7 @@ int import_from_file(cryptonote::core& core, const std::string& import_file_path { std::cout << refresh_string << "block " << h-1 << " / " << block_stop - << std::flush; + << "\r" << std::flush; std::cout << ENDL << ENDL; MINFO("Specified block number reached - stopping. block: " << h-1 << " total blocks: " << h); quit = 1; @@ -405,7 +420,23 @@ int import_from_file(cryptonote::core& core, const std::string& import_file_path { str1.assign(buffer_block, chunk_size); bootstrap::block_package bp; - if (! ::serialization::parse_binary(str1, bp)) + bool res; + if (major_version == 0) + { + bootstrap::block_package_1 bp1; + res = ::serialization::parse_binary(str1, bp1); + if (res) + { + bp.block = std::move(bp1.block); + bp.txs = std::move(bp1.txs); + bp.block_weight = bp1.block_weight; + bp.cumulative_difficulty = bp1.cumulative_difficulty; + bp.coins_generated = bp1.coins_generated; + } + } + else + res = ::serialization::parse_binary(str1, bp); + if (!res) throw std::runtime_error("Error in deserialization of chunk"); int display_interval = 1000; @@ -430,14 +461,14 @@ int import_from_file(cryptonote::core& core, const std::string& import_file_path { std::cout << refresh_string << "block " << h-1 << " / " << block_stop - << std::flush; + << "\r" << std::flush; } if (opt_verify) { cryptonote::blobdata block; cryptonote::block_to_blob(bp.block, block); - std::list txs; + std::vector txs; for (const auto &tx: bp.txs) { txs.push_back(cryptonote::blobdata()); @@ -453,7 +484,7 @@ int import_from_file(cryptonote::core& core, const std::string& import_file_path } else { - std::vector txs; + std::vector> txs; std::vector archived_txs; archived_txs = bp.txs; @@ -470,20 +501,21 @@ int import_from_file(cryptonote::core& core, const std::string& import_file_path // because add_block() calls // add_transaction(blk_hash, blk.miner_tx) first, and // then a for loop for the transactions in txs. - txs.push_back(tx); + txs.push_back(std::make_pair(tx, tx_to_blob(tx))); } - size_t block_size; + size_t block_weight; difficulty_type cumulative_difficulty; uint64_t coins_generated; - block_size = bp.block_size; + block_weight = bp.block_weight; cumulative_difficulty = bp.cumulative_difficulty; coins_generated = bp.coins_generated; try { - core.get_blockchain_storage().get_db().add_block(b, block_size, cumulative_difficulty, coins_generated, txs); + uint64_t long_term_block_weight = core.get_blockchain_storage().get_next_long_term_block_weight(block_weight); + core.get_blockchain_storage().get_db().add_block(std::make_pair(b, block_to_blob(b)), block_weight, long_term_block_weight, cumulative_difficulty, coins_generated, txs); } catch (const std::exception& e) { @@ -593,8 +625,8 @@ int main(int argc, char* argv[]) const command_line::arg_descriptor arg_database = { "database", available_dbs.c_str(), default_db_type }; - const command_line::arg_descriptor arg_verify = {"guard-against-pwnage", - "Verify blocks and transactions during import (only disable if you exported the file yourself)", true}; + const command_line::arg_descriptor arg_noverify = {"dangerous-unverified-import", + "Blindly trust the import file and use potentially malicious blocks and transactions during import (only enable if you exported the file yourself)", false}; const command_line::arg_descriptor arg_batch = {"batch", "Batch transactions for faster import", true}; const command_line::arg_descriptor arg_resume = {"resume", @@ -614,7 +646,7 @@ int main(int argc, char* argv[]) // call add_options() directly for these arguments since // command_line helpers support only boolean switch, not boolean argument desc_cmd_sett.add_options() - (arg_verify.name, make_semantic(arg_verify), arg_verify.description) + (arg_noverify.name, make_semantic(arg_noverify), arg_noverify.description) (arg_batch.name, make_semantic(arg_batch), arg_batch.description) (arg_resume.name, make_semantic(arg_resume), arg_resume.description) ; @@ -633,7 +665,7 @@ int main(int argc, char* argv[]) if (! r) return 1; - opt_verify = command_line::get_arg(vm, arg_verify); + opt_verify = !command_line::get_arg(vm, arg_noverify); opt_batch = command_line::get_arg(vm, arg_batch); opt_resume = command_line::get_arg(vm, arg_resume); block_stop = command_line::get_arg(vm, arg_block_stop); @@ -738,6 +770,18 @@ int main(int argc, char* argv[]) MINFO("bootstrap file path: " << import_file_path); MINFO("database path: " << m_config_folder); + if (!opt_verify) + { + MCLOG_RED(el::Level::Warning, "global", "\n" + "Import is set to proceed WITHOUT VERIFICATION.\n" + "This is a DANGEROUS operation: if the file was tampered with in transit, or obtained from a malicious source,\n" + "you could end up with a compromised database. It is recommended to NOT use " << arg_noverify.name << ".\n" + "*****************************************************************************************\n" + "You have 90 seconds to press ^C or terminate this program before unverified import starts\n" + "*****************************************************************************************"); + sleep(90); + } + cryptonote::cryptonote_protocol_stub pr; //TODO: stub only for this kind of test, make real validation of relayed objects cryptonote::core core(&pr); @@ -745,7 +789,12 @@ int main(int argc, char* argv[]) { core.disable_dns_checkpoints(true); - if (!core.init(vm, NULL)) +#if defined(PER_BLOCK_CHECKPOINT) + const GetCheckpointsCallback& get_checkpoints = blocks::GetCheckpointsData; +#else + const GetCheckpointsCallback& get_checkpoints = nullptr; +#endif + if (!core.init(vm, nullptr, get_checkpoints)) { std::cerr << "Failed to initialize core" << ENDL; return 1; diff --git a/src/blockchain_utilities/blockchain_prune.cpp b/src/blockchain_utilities/blockchain_prune.cpp new file mode 100644 index 00000000000..8e13f2c04bd --- /dev/null +++ b/src/blockchain_utilities/blockchain_prune.cpp @@ -0,0 +1,663 @@ +// Copyright (c) 2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include +#include +#include +#include "common/command_line.h" +#include "common/pruning.h" +#include "cryptonote_core/cryptonote_core.h" +#include "cryptonote_core/blockchain.h" +#include "blockchain_db/blockchain_db.h" +#include "blockchain_db/lmdb/db_lmdb.h" +#include "blockchain_db/db_types.h" +#include "version.h" + +#undef MONERO_DEFAULT_LOG_CATEGORY +#define MONERO_DEFAULT_LOG_CATEGORY "bcutil" + +#define MDB_val_set(var, val) MDB_val var = {sizeof(val), (void *)&val} + +namespace po = boost::program_options; +using namespace epee; +using namespace cryptonote; + +static std::string db_path; + +// default to fast:1 +static uint64_t records_per_sync = 128; +static const size_t slack = 512 * 1024 * 1024; + +static std::error_code replace_file(const boost::filesystem::path& replacement_name, const boost::filesystem::path& replaced_name) +{ + std::error_code ec = tools::replace_file(replacement_name.string(), replaced_name.string()); + if (ec) + MERROR("Error renaming " << replacement_name << " to " << replaced_name << ": " << ec.message()); + return ec; +} + +static void open(MDB_env *&env, const boost::filesystem::path &path, uint64_t db_flags, bool readonly) +{ + int dbr; + int flags = 0; + + if (db_flags & DBF_FAST) + flags |= MDB_NOSYNC; + if (db_flags & DBF_FASTEST) + flags |= MDB_NOSYNC | MDB_WRITEMAP | MDB_MAPASYNC; + if (readonly) + flags |= MDB_RDONLY; + + dbr = mdb_env_create(&env); + if (dbr) throw std::runtime_error("Failed to create LDMB environment: " + std::string(mdb_strerror(dbr))); + dbr = mdb_env_set_maxdbs(env, 32); + if (dbr) throw std::runtime_error("Failed to set max env dbs: " + std::string(mdb_strerror(dbr))); + dbr = mdb_env_open(env, path.string().c_str(), flags, 0664); + if (dbr) throw std::runtime_error("Failed to open database file '" + + path.string() + "': " + std::string(mdb_strerror(dbr))); +} + +static void close(MDB_env *env) +{ + mdb_env_close(env); +} + +static void add_size(MDB_env *env, uint64_t bytes) +{ + try + { + boost::filesystem::path path(db_path); + boost::filesystem::space_info si = boost::filesystem::space(path); + if(si.available < bytes) + { + MERROR("!! WARNING: Insufficient free space to extend database !!: " << + (si.available >> 20L) << " MB available, " << (bytes >> 20L) << " MB needed"); + return; + } + } + catch(...) + { + // print something but proceed. + MWARNING("Unable to query free disk space."); + } + + MDB_envinfo mei; + mdb_env_info(env, &mei); + MDB_stat mst; + mdb_env_stat(env, &mst); + + uint64_t new_mapsize = (uint64_t)mei.me_mapsize + bytes; + new_mapsize += (new_mapsize % mst.ms_psize); + + int result = mdb_env_set_mapsize(env, new_mapsize); + if (result) + throw std::runtime_error("Failed to set new mapsize to " + std::to_string(new_mapsize) + ": " + std::string(mdb_strerror(result))); + + MGINFO("LMDB Mapsize increased." << " Old: " << mei.me_mapsize / (1024 * 1024) << "MiB" << ", New: " << new_mapsize / (1024 * 1024) << "MiB"); +} + +static void check_resize(MDB_env *env, size_t bytes) +{ + MDB_envinfo mei; + MDB_stat mst; + + mdb_env_info(env, &mei); + mdb_env_stat(env, &mst); + + uint64_t size_used = mst.ms_psize * mei.me_last_pgno; + if (size_used + bytes + slack >= mei.me_mapsize) + add_size(env, size_used + bytes + 2 * slack - mei.me_mapsize); +} + +static bool resize_point(size_t nrecords, MDB_env *env, MDB_txn **txn, size_t &bytes) +{ + if (nrecords % records_per_sync && bytes <= slack / 2) + return false; + int dbr = mdb_txn_commit(*txn); + if (dbr) throw std::runtime_error("Failed to commit txn: " + std::string(mdb_strerror(dbr))); + check_resize(env, bytes); + dbr = mdb_txn_begin(env, NULL, 0, txn); + if (dbr) throw std::runtime_error("Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); + bytes = 0; + return true; +} + +static void copy_table(MDB_env *env0, MDB_env *env1, const char *table, unsigned int flags, unsigned int putflags, int (*cmp)(const MDB_val*, const MDB_val*)=0) +{ + MDB_dbi dbi0, dbi1; + MDB_txn *txn0, *txn1; + MDB_cursor *cur0, *cur1; + bool tx_active0 = false, tx_active1 = false; + int dbr; + + MINFO("Copying " << table); + + epee::misc_utils::auto_scope_leave_caller txn_dtor = epee::misc_utils::create_scope_leave_handler([&](){ + if (tx_active1) mdb_txn_abort(txn1); + if (tx_active0) mdb_txn_abort(txn0); + }); + + dbr = mdb_txn_begin(env0, NULL, MDB_RDONLY, &txn0); + if (dbr) throw std::runtime_error("Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); + tx_active0 = true; + dbr = mdb_txn_begin(env1, NULL, 0, &txn1); + if (dbr) throw std::runtime_error("Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); + tx_active1 = true; + + dbr = mdb_dbi_open(txn0, table, flags, &dbi0); + if (dbr) throw std::runtime_error("Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); + if (cmp) + ((flags & MDB_DUPSORT) ? mdb_set_dupsort : mdb_set_compare)(txn0, dbi0, cmp); + + dbr = mdb_dbi_open(txn1, table, flags, &dbi1); + if (dbr) throw std::runtime_error("Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); + if (cmp) + ((flags & MDB_DUPSORT) ? mdb_set_dupsort : mdb_set_compare)(txn1, dbi1, cmp); + + dbr = mdb_txn_commit(txn1); + if (dbr) throw std::runtime_error("Failed to commit txn: " + std::string(mdb_strerror(dbr))); + tx_active1 = false; + MDB_stat stats; + dbr = mdb_env_stat(env0, &stats); + if (dbr) throw std::runtime_error("Failed to stat " + std::string(table) + " LMDB table: " + std::string(mdb_strerror(dbr))); + check_resize(env1, (stats.ms_branch_pages + stats.ms_overflow_pages + stats.ms_leaf_pages) * stats.ms_psize); + dbr = mdb_txn_begin(env1, NULL, 0, &txn1); + if (dbr) throw std::runtime_error("Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); + tx_active1 = true; + + dbr = mdb_drop(txn1, dbi1, 0); + if (dbr) throw std::runtime_error("Failed to empty " + std::string(table) + " LMDB table: " + std::string(mdb_strerror(dbr))); + + dbr = mdb_cursor_open(txn0, dbi0, &cur0); + if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr))); + dbr = mdb_cursor_open(txn1, dbi1, &cur1); + if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr))); + + MDB_val k; + MDB_val v; + MDB_cursor_op op = MDB_FIRST; + size_t nrecords = 0, bytes = 0; + while (1) + { + int ret = mdb_cursor_get(cur0, &k, &v, op); + op = MDB_NEXT; + if (ret == MDB_NOTFOUND) + break; + if (ret) + throw std::runtime_error("Failed to enumerate " + std::string(table) + " records: " + std::string(mdb_strerror(ret))); + + bytes += k.mv_size + v.mv_size; + if (resize_point(++nrecords, env1, &txn1, bytes)) + { + dbr = mdb_cursor_open(txn1, dbi1, &cur1); + if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr))); + } + + ret = mdb_cursor_put(cur1, &k, &v, putflags); + if (ret) + throw std::runtime_error("Failed to write " + std::string(table) + " record: " + std::string(mdb_strerror(ret))); + } + + mdb_cursor_close(cur1); + mdb_cursor_close(cur0); + mdb_txn_commit(txn1); + tx_active1 = false; + mdb_txn_commit(txn0); + tx_active0 = false; + mdb_dbi_close(env1, dbi1); + mdb_dbi_close(env0, dbi0); +} + +static bool is_v1_tx(MDB_cursor *c_txs_pruned, MDB_val *tx_id) +{ + MDB_val v; + int ret = mdb_cursor_get(c_txs_pruned, tx_id, &v, MDB_SET); + if (ret) + throw std::runtime_error("Failed to find transaction pruned data: " + std::string(mdb_strerror(ret))); + if (v.mv_size == 0) + throw std::runtime_error("Invalid transaction pruned data"); + return cryptonote::is_v1_tx(cryptonote::blobdata_ref{(const char*)v.mv_data, v.mv_size}); +} + +static void prune(MDB_env *env0, MDB_env *env1) +{ + MDB_dbi dbi0_blocks, dbi0_txs_pruned, dbi0_txs_prunable, dbi0_tx_indices, dbi1_txs_prunable, dbi1_txs_prunable_tip, dbi1_properties; + MDB_txn *txn0, *txn1; + MDB_cursor *cur0_txs_pruned, *cur0_txs_prunable, *cur0_tx_indices, *cur1_txs_prunable, *cur1_txs_prunable_tip; + bool tx_active0 = false, tx_active1 = false; + int dbr; + + MGINFO("Creating pruned txs_prunable"); + + epee::misc_utils::auto_scope_leave_caller txn_dtor = epee::misc_utils::create_scope_leave_handler([&](){ + if (tx_active1) mdb_txn_abort(txn1); + if (tx_active0) mdb_txn_abort(txn0); + }); + + dbr = mdb_txn_begin(env0, NULL, MDB_RDONLY, &txn0); + if (dbr) throw std::runtime_error("Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); + tx_active0 = true; + dbr = mdb_txn_begin(env1, NULL, 0, &txn1); + if (dbr) throw std::runtime_error("Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); + tx_active1 = true; + + dbr = mdb_dbi_open(txn0, "txs_pruned", MDB_INTEGERKEY, &dbi0_txs_pruned); + if (dbr) throw std::runtime_error("Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); + mdb_set_compare(txn0, dbi0_txs_pruned, BlockchainLMDB::compare_uint64); + dbr = mdb_cursor_open(txn0, dbi0_txs_pruned, &cur0_txs_pruned); + if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr))); + + dbr = mdb_dbi_open(txn0, "txs_prunable", MDB_INTEGERKEY, &dbi0_txs_prunable); + if (dbr) throw std::runtime_error("Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); + mdb_set_compare(txn0, dbi0_txs_prunable, BlockchainLMDB::compare_uint64); + dbr = mdb_cursor_open(txn0, dbi0_txs_prunable, &cur0_txs_prunable); + if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr))); + + dbr = mdb_dbi_open(txn0, "tx_indices", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, &dbi0_tx_indices); + if (dbr) throw std::runtime_error("Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); + mdb_set_dupsort(txn0, dbi0_tx_indices, BlockchainLMDB::compare_hash32); + dbr = mdb_cursor_open(txn0, dbi0_tx_indices, &cur0_tx_indices); + if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr))); + + dbr = mdb_dbi_open(txn1, "txs_prunable", MDB_INTEGERKEY, &dbi1_txs_prunable); + if (dbr) throw std::runtime_error("Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); + mdb_set_compare(txn1, dbi1_txs_prunable, BlockchainLMDB::compare_uint64); + dbr = mdb_cursor_open(txn1, dbi1_txs_prunable, &cur1_txs_prunable); + if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr))); + + dbr = mdb_dbi_open(txn1, "txs_prunable_tip", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, &dbi1_txs_prunable_tip); + if (dbr) throw std::runtime_error("Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); + mdb_set_dupsort(txn1, dbi1_txs_prunable_tip, BlockchainLMDB::compare_uint64); + dbr = mdb_cursor_open(txn1, dbi1_txs_prunable_tip, &cur1_txs_prunable_tip); + if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr))); + + dbr = mdb_drop(txn1, dbi1_txs_prunable, 0); + if (dbr) throw std::runtime_error("Failed to empty LMDB table: " + std::string(mdb_strerror(dbr))); + dbr = mdb_drop(txn1, dbi1_txs_prunable_tip, 0); + if (dbr) throw std::runtime_error("Failed to empty LMDB table: " + std::string(mdb_strerror(dbr))); + + dbr = mdb_dbi_open(txn1, "properties", 0, &dbi1_properties); + if (dbr) throw std::runtime_error("Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); + + MDB_val k, v; + uint32_t pruning_seed = tools::make_pruning_seed(tools::get_random_stripe(), CRYPTONOTE_PRUNING_LOG_STRIPES); + static char pruning_seed_key[] = "pruning_seed"; + k.mv_data = pruning_seed_key; + k.mv_size = strlen("pruning_seed") + 1; + v.mv_data = (void*)&pruning_seed; + v.mv_size = sizeof(pruning_seed); + dbr = mdb_put(txn1, dbi1_properties, &k, &v, 0); + if (dbr) throw std::runtime_error("Failed to save pruning seed: " + std::string(mdb_strerror(dbr))); + + MDB_stat stats; + dbr = mdb_dbi_open(txn0, "blocks", 0, &dbi0_blocks); + if (dbr) throw std::runtime_error("Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); + dbr = mdb_stat(txn0, dbi0_blocks, &stats); + if (dbr) throw std::runtime_error("Failed to query size of blocks: " + std::string(mdb_strerror(dbr))); + mdb_dbi_close(env0, dbi0_blocks); + const uint64_t blockchain_height = stats.ms_entries; + size_t nrecords = 0, bytes = 0; + + MDB_cursor_op op = MDB_FIRST; + while (1) + { + int ret = mdb_cursor_get(cur0_tx_indices, &k, &v, op); + op = MDB_NEXT; + if (ret == MDB_NOTFOUND) + break; + if (ret) throw std::runtime_error("Failed to enumerate records: " + std::string(mdb_strerror(ret))); + + const txindex *ti = (const txindex*)v.mv_data; + const uint64_t block_height = ti->data.block_id; + MDB_val_set(kk, ti->data.tx_id); + if (block_height + CRYPTONOTE_PRUNING_TIP_BLOCKS >= blockchain_height) + { + MDEBUG(block_height << "/" << blockchain_height << " is in tip"); + MDB_val_set(vv, block_height); + dbr = mdb_cursor_put(cur1_txs_prunable_tip, &kk, &vv, 0); + if (dbr) throw std::runtime_error("Failed to write prunable tx tip data: " + std::string(mdb_strerror(dbr))); + bytes += kk.mv_size + vv.mv_size; + } + if (tools::has_unpruned_block(block_height, blockchain_height, pruning_seed) || is_v1_tx(cur0_txs_pruned, &kk)) + { + MDB_val vv; + dbr = mdb_cursor_get(cur0_txs_prunable, &kk, &vv, MDB_SET); + if (dbr) throw std::runtime_error("Failed to read prunable tx data: " + std::string(mdb_strerror(dbr))); + bytes += kk.mv_size + vv.mv_size; + if (resize_point(++nrecords, env1, &txn1, bytes)) + { + dbr = mdb_cursor_open(txn1, dbi1_txs_prunable, &cur1_txs_prunable); + if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr))); + dbr = mdb_cursor_open(txn1, dbi1_txs_prunable_tip, &cur1_txs_prunable_tip); + if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr))); + } + dbr = mdb_cursor_put(cur1_txs_prunable, &kk, &vv, 0); + if (dbr) throw std::runtime_error("Failed to write prunable tx data: " + std::string(mdb_strerror(dbr))); + } + else + { + MDEBUG("" << block_height << "/" << blockchain_height << " should be pruned, dropping"); + } + } + + mdb_cursor_close(cur1_txs_prunable_tip); + mdb_cursor_close(cur1_txs_prunable); + mdb_cursor_close(cur0_txs_prunable); + mdb_cursor_close(cur0_txs_pruned); + mdb_cursor_close(cur0_tx_indices); + mdb_txn_commit(txn1); + tx_active1 = false; + mdb_txn_commit(txn0); + tx_active0 = false; + mdb_dbi_close(env1, dbi1_properties); + mdb_dbi_close(env1, dbi1_txs_prunable_tip); + mdb_dbi_close(env1, dbi1_txs_prunable); + mdb_dbi_close(env0, dbi0_txs_prunable); + mdb_dbi_close(env0, dbi0_txs_pruned); + mdb_dbi_close(env0, dbi0_tx_indices); +} + +static bool parse_db_sync_mode(std::string db_sync_mode, uint64_t &db_flags) +{ + std::vector options; + boost::trim(db_sync_mode); + boost::split(options, db_sync_mode, boost::is_any_of(" :")); + + for(const auto &option : options) + MDEBUG("option: " << option); + + // default to fast:async:1 + uint64_t DEFAULT_FLAGS = DBF_FAST; + + db_flags = 0; + + if(options.size() == 0) + { + // default to fast:async:1 + db_flags = DEFAULT_FLAGS; + } + + bool safemode = false; + if(options.size() >= 1) + { + if(options[0] == "safe") + { + safemode = true; + db_flags = DBF_SAFE; + } + else if(options[0] == "fast") + { + db_flags = DBF_FAST; + } + else if(options[0] == "fastest") + { + db_flags = DBF_FASTEST; + records_per_sync = 1000; // default to fastest:async:1000 + } + else + return false; + } + + if(options.size() >= 2 && !safemode) + { + char *endptr; + uint64_t bps = strtoull(options[1].c_str(), &endptr, 0); + if (*endptr != '\0') + return false; + records_per_sync = bps; + } + + return true; +} + +int main(int argc, char* argv[]) +{ + TRY_ENTRY(); + + epee::string_tools::set_module_name_and_folder(argv[0]); + + std::string default_db_type = "lmdb"; + + std::string available_dbs = cryptonote::blockchain_db_types(", "); + available_dbs = "available: " + available_dbs; + + uint32_t log_level = 0; + + tools::on_startup(); + + boost::filesystem::path output_file_path; + + po::options_description desc_cmd_only("Command line options"); + po::options_description desc_cmd_sett("Command line options and settings options"); + const command_line::arg_descriptor arg_log_level = {"log-level", "0-4 or categories", ""}; + const command_line::arg_descriptor arg_database = { + "database", available_dbs.c_str(), default_db_type + }; + const command_line::arg_descriptor arg_db_sync_mode = { + "db-sync-mode" + , "Specify sync option, using format [safe|fast|fastest]:[nrecords_per_sync]." + , "fast:1000" + }; + const command_line::arg_descriptor arg_copy_pruned_database = {"copy-pruned-database", "Copy database anyway if already pruned"}; + + command_line::add_arg(desc_cmd_sett, cryptonote::arg_data_dir); + command_line::add_arg(desc_cmd_sett, cryptonote::arg_testnet_on); + command_line::add_arg(desc_cmd_sett, cryptonote::arg_stagenet_on); + command_line::add_arg(desc_cmd_sett, arg_log_level); + command_line::add_arg(desc_cmd_sett, arg_database); + command_line::add_arg(desc_cmd_sett, arg_db_sync_mode); + command_line::add_arg(desc_cmd_sett, arg_copy_pruned_database); + command_line::add_arg(desc_cmd_only, command_line::arg_help); + + po::options_description desc_options("Allowed options"); + desc_options.add(desc_cmd_only).add(desc_cmd_sett); + + po::variables_map vm; + bool r = command_line::handle_error_helper(desc_options, [&]() + { + auto parser = po::command_line_parser(argc, argv).options(desc_options); + po::store(parser.run(), vm); + po::notify(vm); + return true; + }); + if (! r) + return 1; + + if (command_line::get_arg(vm, command_line::arg_help)) + { + std::cout << "Monero '" << MONERO_RELEASE_NAME << "' (v" << MONERO_VERSION_FULL << ")" << ENDL << ENDL; + std::cout << desc_options << std::endl; + return 1; + } + + mlog_configure(mlog_get_default_log_path("monero-blockchain-prune.log"), true); + if (!command_line::is_arg_defaulted(vm, arg_log_level)) + mlog_set_log(command_line::get_arg(vm, arg_log_level).c_str()); + else + mlog_set_log(std::string(std::to_string(log_level) + ",bcutil:INFO").c_str()); + + MINFO("Starting..."); + + bool opt_testnet = command_line::get_arg(vm, cryptonote::arg_testnet_on); + bool opt_stagenet = command_line::get_arg(vm, cryptonote::arg_stagenet_on); + network_type net_type = opt_testnet ? TESTNET : opt_stagenet ? STAGENET : MAINNET; + bool opt_copy_pruned_database = command_line::get_arg(vm, arg_copy_pruned_database); + std::string data_dir = command_line::get_arg(vm, cryptonote::arg_data_dir); + while (boost::ends_with(data_dir, "/") || boost::ends_with(data_dir, "\\")) + data_dir.pop_back(); + + std::string db_type = command_line::get_arg(vm, arg_database); + if (!cryptonote::blockchain_valid_db_type(db_type)) + { + MERROR("Invalid database type: " << db_type); + return 1; + } + if (db_type != "lmdb") + { + MERROR("Unsupported database type: " << db_type << ". Only lmdb is supported"); + return 1; + } + + std::string db_sync_mode = command_line::get_arg(vm, arg_db_sync_mode); + uint64_t db_flags = 0; + if (!parse_db_sync_mode(db_sync_mode, db_flags)) + { + MERROR("Invalid db sync mode: " << db_sync_mode); + return 1; + } + + // If we wanted to use the memory pool, we would set up a fake_core. + + // Use Blockchain instead of lower-level BlockchainDB for two reasons: + // 1. Blockchain has the init() method for easy setup + // 2. exporter needs to use get_current_blockchain_height(), get_block_id_by_height(), get_block_by_hash() + // + // cannot match blockchain_storage setup above with just one line, + // e.g. + // Blockchain* core_storage = new Blockchain(NULL); + // because unlike blockchain_storage constructor, which takes a pointer to + // tx_memory_pool, Blockchain's constructor takes tx_memory_pool object. + MINFO("Initializing source blockchain (BlockchainDB)"); + std::array, 2> core_storage; + Blockchain *blockchain = NULL; + tx_memory_pool m_mempool(*blockchain); + boost::filesystem::path paths[2]; + bool already_pruned = false; + for (size_t n = 0; n < core_storage.size(); ++n) + { + core_storage[n].reset(new Blockchain(m_mempool)); + + BlockchainDB* db = new_db(db_type); + if (db == NULL) + { + MERROR("Attempted to use non-existent database type: " << db_type); + throw std::runtime_error("Attempting to use non-existent database type"); + } + MDEBUG("database: " << db_type); + + if (n == 1) + { + paths[1] = boost::filesystem::path(data_dir) / (db->get_db_name() + "-pruned"); + if (boost::filesystem::exists(paths[1])) + { + if (!boost::filesystem::is_directory(paths[1])) + { + MERROR("LMDB needs a directory path, but a file was passed: " << paths[1].string()); + return 1; + } + } + else + { + if (!boost::filesystem::create_directories(paths[1])) + { + MERROR("Failed to create directory: " << paths[1].string()); + return 1; + } + } + db_path = paths[1].string(); + } + else + { + paths[0] = boost::filesystem::path(data_dir) / db->get_db_name(); + } + + MINFO("Loading blockchain from folder " << paths[n] << " ..."); + + try + { + db->open(paths[n].string(), n == 0 ? DBF_RDONLY : 0); + } + catch (const std::exception& e) + { + MERROR("Error opening database: " << e.what()); + return 1; + } + r = core_storage[n]->init(db, net_type); + + std::string source_dest = n == 0 ? "source" : "pruned"; + CHECK_AND_ASSERT_MES(r, 1, "Failed to initialize " << source_dest << " blockchain storage"); + MINFO(source_dest << " blockchain storage initialized OK"); + if (n == 0 && core_storage[0]->get_blockchain_pruning_seed()) + { + if (!opt_copy_pruned_database) + { + MERROR("Blockchain is already pruned, use --" << arg_copy_pruned_database.name << " to copy it anyway"); + return 1; + } + already_pruned = true; + } + } + core_storage[0]->deinit(); + core_storage[0].reset(NULL); + core_storage[1]->deinit(); + core_storage[1].reset(NULL); + + MINFO("Pruning..."); + MDB_env *env0 = NULL, *env1 = NULL; + open(env0, paths[0], db_flags, true); + open(env1, paths[1], db_flags, false); + copy_table(env0, env1, "blocks", MDB_INTEGERKEY, MDB_APPEND); + copy_table(env0, env1, "block_info", MDB_INTEGERKEY | MDB_DUPSORT| MDB_DUPFIXED, MDB_APPENDDUP, BlockchainLMDB::compare_uint64); + copy_table(env0, env1, "block_heights", MDB_INTEGERKEY | MDB_DUPSORT| MDB_DUPFIXED, 0, BlockchainLMDB::compare_hash32); + //copy_table(env0, env1, "txs", MDB_INTEGERKEY); + copy_table(env0, env1, "txs_pruned", MDB_INTEGERKEY, MDB_APPEND); + copy_table(env0, env1, "txs_prunable_hash", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, MDB_APPEND); + // not copied: prunable, prunable_tip + copy_table(env0, env1, "tx_indices", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, 0, BlockchainLMDB::compare_hash32); + copy_table(env0, env1, "tx_outputs", MDB_INTEGERKEY, MDB_APPEND); + copy_table(env0, env1, "output_txs", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, MDB_APPENDDUP, BlockchainLMDB::compare_uint64); + copy_table(env0, env1, "output_amounts", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, MDB_APPENDDUP, BlockchainLMDB::compare_uint64); + copy_table(env0, env1, "spent_keys", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, MDB_NODUPDATA, BlockchainLMDB::compare_hash32); + copy_table(env0, env1, "txpool_meta", 0, MDB_NODUPDATA, BlockchainLMDB::compare_hash32); + copy_table(env0, env1, "txpool_blob", 0, MDB_NODUPDATA, BlockchainLMDB::compare_hash32); + copy_table(env0, env1, "hf_versions", MDB_INTEGERKEY, MDB_APPEND); + copy_table(env0, env1, "properties", 0, 0, BlockchainLMDB::compare_string); + if (already_pruned) + { + copy_table(env0, env1, "txs_prunable", MDB_INTEGERKEY, MDB_APPEND, BlockchainLMDB::compare_uint64); + copy_table(env0, env1, "txs_prunable_tip", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, MDB_NODUPDATA, BlockchainLMDB::compare_uint64); + } + else + { + prune(env0, env1); + } + close(env1); + close(env0); + + MINFO("Swapping databases, pre-pruning blockchain will be left in " << paths[0].string() + "-old and can be removed if desired"); + if (replace_file(paths[0].string(), paths[0].string() + "-old") || replace_file(paths[1].string(), paths[0].string())) + { + MERROR("Blockchain pruned OK, but renaming failed"); + return 1; + } + + MINFO("Blockchain pruned OK"); + return 0; + + CATCH_ENTRY("Pruning error", 1); +} diff --git a/src/blockchain_utilities/blockchain_prune_known_spent_data.cpp b/src/blockchain_utilities/blockchain_prune_known_spent_data.cpp new file mode 100644 index 00000000000..2d49b6ecd56 --- /dev/null +++ b/src/blockchain_utilities/blockchain_prune_known_spent_data.cpp @@ -0,0 +1,305 @@ +// Copyright (c) 2014-2019, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include +#include "common/command_line.h" +#include "serialization/crypto.h" +#include "cryptonote_core/tx_pool.h" +#include "cryptonote_core/cryptonote_core.h" +#include "cryptonote_core/blockchain.h" +#include "blockchain_db/blockchain_db.h" +#include "blockchain_db/db_types.h" +#include "version.h" + +#undef MONERO_DEFAULT_LOG_CATEGORY +#define MONERO_DEFAULT_LOG_CATEGORY "bcutil" + +namespace po = boost::program_options; +using namespace epee; +using namespace cryptonote; + +static std::map load_outputs(const std::string &filename) +{ + std::map outputs; + uint64_t amount = std::numeric_limits::max(); + FILE *f; + + f = fopen(filename.c_str(), "r"); + if (!f) + { + MERROR("Failed to load outputs from " << filename << ": " << strerror(errno)); + return {}; + } + while (1) + { + char s[256]; + if (!fgets(s, sizeof(s), f)) + break; + if (feof(f)) + break; + const size_t len = strlen(s); + if (len > 0 && s[len - 1] == '\n') + s[len - 1] = 0; + if (!s[0]) + continue; + std::pair output; + uint64_t offset, num_offsets; + if (sscanf(s, "@%" PRIu64, &amount) == 1) + { + continue; + } + if (amount == std::numeric_limits::max()) + { + MERROR("Bad format in " << filename); + continue; + } + if (sscanf(s, "%" PRIu64 "*%" PRIu64, &offset, &num_offsets) == 2 && num_offsets < std::numeric_limits::max() - offset) + { + outputs[amount] += num_offsets; + } + else if (sscanf(s, "%" PRIu64, &offset) == 1) + { + outputs[amount] += 1; + } + else + { + MERROR("Bad format in " << filename); + continue; + } + } + fclose(f); + return outputs; +} + +int main(int argc, char* argv[]) +{ + TRY_ENTRY(); + + epee::string_tools::set_module_name_and_folder(argv[0]); + + std::string default_db_type = "lmdb"; + + std::string available_dbs = cryptonote::blockchain_db_types(", "); + available_dbs = "available: " + available_dbs; + + uint32_t log_level = 0; + + tools::on_startup(); + + po::options_description desc_cmd_only("Command line options"); + po::options_description desc_cmd_sett("Command line options and settings options"); + const command_line::arg_descriptor arg_log_level = {"log-level", "0-4 or categories", ""}; + const command_line::arg_descriptor arg_database = { + "database", available_dbs.c_str(), default_db_type + }; + const command_line::arg_descriptor arg_verbose = {"verbose", "Verbose output", false}; + const command_line::arg_descriptor arg_dry_run = {"dry-run", "Do not actually prune", false}; + const command_line::arg_descriptor arg_input = {"input", "Path to the known spent outputs file"}; + + command_line::add_arg(desc_cmd_sett, cryptonote::arg_data_dir); + command_line::add_arg(desc_cmd_sett, cryptonote::arg_testnet_on); + command_line::add_arg(desc_cmd_sett, cryptonote::arg_stagenet_on); + command_line::add_arg(desc_cmd_sett, arg_log_level); + command_line::add_arg(desc_cmd_sett, arg_database); + command_line::add_arg(desc_cmd_sett, arg_verbose); + command_line::add_arg(desc_cmd_sett, arg_dry_run); + command_line::add_arg(desc_cmd_sett, arg_input); + command_line::add_arg(desc_cmd_only, command_line::arg_help); + + po::options_description desc_options("Allowed options"); + desc_options.add(desc_cmd_only).add(desc_cmd_sett); + + po::variables_map vm; + bool r = command_line::handle_error_helper(desc_options, [&]() + { + auto parser = po::command_line_parser(argc, argv).options(desc_options); + po::store(parser.run(), vm); + po::notify(vm); + return true; + }); + if (! r) + return 1; + + if (command_line::get_arg(vm, command_line::arg_help)) + { + std::cout << "Monero '" << MONERO_RELEASE_NAME << "' (v" << MONERO_VERSION_FULL << ")" << ENDL << ENDL; + std::cout << desc_options << std::endl; + return 1; + } + + mlog_configure(mlog_get_default_log_path("monero-blockchain-prune-known-spent-data.log"), true); + if (!command_line::is_arg_defaulted(vm, arg_log_level)) + mlog_set_log(command_line::get_arg(vm, arg_log_level).c_str()); + else + mlog_set_log(std::string(std::to_string(log_level) + ",bcutil:INFO").c_str()); + + LOG_PRINT_L0("Starting..."); + + std::string opt_data_dir = command_line::get_arg(vm, cryptonote::arg_data_dir); + bool opt_testnet = command_line::get_arg(vm, cryptonote::arg_testnet_on); + bool opt_stagenet = command_line::get_arg(vm, cryptonote::arg_stagenet_on); + network_type net_type = opt_testnet ? TESTNET : opt_stagenet ? STAGENET : MAINNET; + bool opt_verbose = command_line::get_arg(vm, arg_verbose); + bool opt_dry_run = command_line::get_arg(vm, arg_dry_run); + + std::string db_type = command_line::get_arg(vm, arg_database); + if (!cryptonote::blockchain_valid_db_type(db_type)) + { + std::cerr << "Invalid database type: " << db_type << std::endl; + return 1; + } + + const std::string input = command_line::get_arg(vm, arg_input); + + LOG_PRINT_L0("Initializing source blockchain (BlockchainDB)"); + std::unique_ptr core_storage; + tx_memory_pool m_mempool(*core_storage); + core_storage.reset(new Blockchain(m_mempool)); + BlockchainDB *db = new_db(db_type); + if (db == NULL) + { + LOG_ERROR("Attempted to use non-existent database type: " << db_type); + throw std::runtime_error("Attempting to use non-existent database type"); + } + LOG_PRINT_L0("database: " << db_type); + + const std::string filename = (boost::filesystem::path(opt_data_dir) / db->get_db_name()).string(); + LOG_PRINT_L0("Loading blockchain from folder " << filename << " ..."); + + try + { + db->open(filename, 0); + } + catch (const std::exception& e) + { + LOG_PRINT_L0("Error opening database: " << e.what()); + return 1; + } + r = core_storage->init(db, net_type); + + CHECK_AND_ASSERT_MES(r, 1, "Failed to initialize source blockchain storage"); + LOG_PRINT_L0("Source blockchain storage initialized OK"); + + std::map known_spent_outputs; + if (input.empty()) + { + std::map> outputs; + + LOG_PRINT_L0("Scanning for known spent data..."); + db->for_all_transactions([&](const crypto::hash &txid, const cryptonote::transaction &tx){ + const bool miner_tx = tx.vin.size() == 1 && tx.vin[0].type() == typeid(txin_gen); + for (const auto &in: tx.vin) + { + if (in.type() != typeid(txin_to_key)) + continue; + const auto &txin = boost::get(in); + if (txin.amount == 0) + continue; + + outputs[txin.amount].second++; + } + + for (const auto &out: tx.vout) + { + uint64_t amount = out.amount; + if (miner_tx && tx.version >= 2) + amount = 0; + if (amount == 0) + continue; + if (out.target.type() != typeid(txout_to_key)) + continue; + + outputs[amount].first++; + } + return true; + }, true); + + for (const auto &i: outputs) + { + known_spent_outputs[i.first] = i.second.second; + } + } + else + { + LOG_PRINT_L0("Loading known spent data..."); + known_spent_outputs = load_outputs(input); + } + + LOG_PRINT_L0("Pruning known spent data..."); + + bool stop_requested = false; + tools::signal_handler::install([&stop_requested](int type) { + stop_requested = true; + }); + + db->batch_start(); + + size_t num_total_outputs = 0, num_prunable_outputs = 0, num_known_spent_outputs = 0, num_eligible_outputs = 0, num_eligible_known_spent_outputs = 0; + for (auto i = known_spent_outputs.begin(); i != known_spent_outputs.end(); ++i) + { + uint64_t num_outputs = db->get_num_outputs(i->first); + num_total_outputs += num_outputs; + num_known_spent_outputs += i->second; + if (i->first == 0 || is_valid_decomposed_amount(i->first)) + { + if (opt_verbose) + MINFO("Ignoring output value " << i->first << ", with " << num_outputs << " outputs"); + continue; + } + num_eligible_outputs += num_outputs; + num_eligible_known_spent_outputs += i->second; + if (opt_verbose) + MINFO(i->first << ": " << i->second << "/" << num_outputs); + if (num_outputs > i->second) + continue; + if (num_outputs && num_outputs < i->second) + { + MERROR("More outputs are spent than known for amount " << i->first << ", not touching"); + continue; + } + if (opt_verbose) + MINFO("Pruning data for " << num_outputs << " outputs"); + if (!opt_dry_run) + db->prune_outputs(i->first); + num_prunable_outputs += i->second; + } + + db->batch_stop(); + + MINFO("Total outputs: " << num_total_outputs); + MINFO("Known spent outputs: " << num_known_spent_outputs); + MINFO("Eligible outputs: " << num_eligible_outputs); + MINFO("Eligible known spent outputs: " << num_eligible_known_spent_outputs); + MINFO("Prunable outputs: " << num_prunable_outputs); + + LOG_PRINT_L0("Blockchain known spent data pruned OK"); + core_storage->deinit(); + return 0; + + CATCH_ENTRY("Error", 1); +} diff --git a/src/blockchain_utilities/blockchain_stats.cpp b/src/blockchain_utilities/blockchain_stats.cpp new file mode 100644 index 00000000000..33c26277e96 --- /dev/null +++ b/src/blockchain_utilities/blockchain_stats.cpp @@ -0,0 +1,337 @@ +// Copyright (c) 2014-2019, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include +#include "common/command_line.h" +#include "common/varint.h" +#include "cryptonote_basic/cryptonote_boost_serialization.h" +#include "cryptonote_core/tx_pool.h" +#include "cryptonote_core/cryptonote_core.h" +#include "cryptonote_core/blockchain.h" +#include "blockchain_db/blockchain_db.h" +#include "blockchain_db/db_types.h" +#include "version.h" + +#undef MONERO_DEFAULT_LOG_CATEGORY +#define MONERO_DEFAULT_LOG_CATEGORY "bcutil" + +namespace po = boost::program_options; +using namespace epee; +using namespace cryptonote; + +static bool stop_requested = false; + +int main(int argc, char* argv[]) +{ + TRY_ENTRY(); + + epee::string_tools::set_module_name_and_folder(argv[0]); + + std::string default_db_type = "lmdb"; + + std::string available_dbs = cryptonote::blockchain_db_types(", "); + available_dbs = "available: " + available_dbs; + + uint32_t log_level = 0; + uint64_t block_start = 0; + uint64_t block_stop = 0; + + tools::on_startup(); + + boost::filesystem::path output_file_path; + + po::options_description desc_cmd_only("Command line options"); + po::options_description desc_cmd_sett("Command line options and settings options"); + const command_line::arg_descriptor arg_log_level = {"log-level", "0-4 or categories", ""}; + const command_line::arg_descriptor arg_database = { + "database", available_dbs.c_str(), default_db_type + }; + const command_line::arg_descriptor arg_block_start = {"block-start", "start at block number", block_start}; + const command_line::arg_descriptor arg_block_stop = {"block-stop", "Stop at block number", block_stop}; + const command_line::arg_descriptor arg_inputs = {"with-inputs", "with input stats", false}; + const command_line::arg_descriptor arg_outputs = {"with-outputs", "with output stats", false}; + const command_line::arg_descriptor arg_ringsize = {"with-ringsize", "with ringsize stats", false}; + const command_line::arg_descriptor arg_hours = {"with-hours", "with txns per hour", false}; + + command_line::add_arg(desc_cmd_sett, cryptonote::arg_data_dir); + command_line::add_arg(desc_cmd_sett, cryptonote::arg_testnet_on); + command_line::add_arg(desc_cmd_sett, cryptonote::arg_stagenet_on); + command_line::add_arg(desc_cmd_sett, arg_log_level); + command_line::add_arg(desc_cmd_sett, arg_database); + command_line::add_arg(desc_cmd_sett, arg_block_start); + command_line::add_arg(desc_cmd_sett, arg_block_stop); + command_line::add_arg(desc_cmd_sett, arg_inputs); + command_line::add_arg(desc_cmd_sett, arg_outputs); + command_line::add_arg(desc_cmd_sett, arg_ringsize); + command_line::add_arg(desc_cmd_sett, arg_hours); + command_line::add_arg(desc_cmd_only, command_line::arg_help); + + po::options_description desc_options("Allowed options"); + desc_options.add(desc_cmd_only).add(desc_cmd_sett); + + po::variables_map vm; + bool r = command_line::handle_error_helper(desc_options, [&]() + { + auto parser = po::command_line_parser(argc, argv).options(desc_options); + po::store(parser.run(), vm); + po::notify(vm); + return true; + }); + if (! r) + return 1; + + if (command_line::get_arg(vm, command_line::arg_help)) + { + std::cout << "Monero '" << MONERO_RELEASE_NAME << "' (v" << MONERO_VERSION_FULL << ")" << ENDL << ENDL; + std::cout << desc_options << std::endl; + return 1; + } + + mlog_configure(mlog_get_default_log_path("monero-blockchain-stats.log"), true); + if (!command_line::is_arg_defaulted(vm, arg_log_level)) + mlog_set_log(command_line::get_arg(vm, arg_log_level).c_str()); + else + mlog_set_log(std::string(std::to_string(log_level) + ",bcutil:INFO").c_str()); + + LOG_PRINT_L0("Starting..."); + + std::string opt_data_dir = command_line::get_arg(vm, cryptonote::arg_data_dir); + bool opt_testnet = command_line::get_arg(vm, cryptonote::arg_testnet_on); + bool opt_stagenet = command_line::get_arg(vm, cryptonote::arg_stagenet_on); + network_type net_type = opt_testnet ? TESTNET : opt_stagenet ? STAGENET : MAINNET; + block_start = command_line::get_arg(vm, arg_block_start); + block_stop = command_line::get_arg(vm, arg_block_stop); + bool do_inputs = command_line::get_arg(vm, arg_inputs); + bool do_outputs = command_line::get_arg(vm, arg_outputs); + bool do_ringsize = command_line::get_arg(vm, arg_ringsize); + bool do_hours = command_line::get_arg(vm, arg_hours); + + std::string db_type = command_line::get_arg(vm, arg_database); + if (!cryptonote::blockchain_valid_db_type(db_type)) + { + std::cerr << "Invalid database type: " << db_type << std::endl; + return 1; + } + + LOG_PRINT_L0("Initializing source blockchain (BlockchainDB)"); + std::unique_ptr core_storage; + tx_memory_pool m_mempool(*core_storage); + core_storage.reset(new Blockchain(m_mempool)); + BlockchainDB *db = new_db(db_type); + if (db == NULL) + { + LOG_ERROR("Attempted to use non-existent database type: " << db_type); + throw std::runtime_error("Attempting to use non-existent database type"); + } + LOG_PRINT_L0("database: " << db_type); + + const std::string filename = (boost::filesystem::path(opt_data_dir) / db->get_db_name()).string(); + LOG_PRINT_L0("Loading blockchain from folder " << filename << " ..."); + + try + { + db->open(filename, DBF_RDONLY); + } + catch (const std::exception& e) + { + LOG_PRINT_L0("Error opening database: " << e.what()); + return 1; + } + r = core_storage->init(db, net_type); + + CHECK_AND_ASSERT_MES(r, 1, "Failed to initialize source blockchain storage"); + LOG_PRINT_L0("Source blockchain storage initialized OK"); + + tools::signal_handler::install([](int type) { + stop_requested = true; + }); + + const uint64_t db_height = db->height(); + if (!block_stop) + block_stop = db_height; + MINFO("Starting from height " << block_start << ", stopping at height " << block_stop); + +/* + * The default output can be plotted with GnuPlot using these commands: +set key autotitle columnhead +set title "Monero Blockchain Growth" +set timefmt "%Y-%m-%d" +set xdata time +set xrange ["2014-04-17":*] +set format x "%Y-%m-%d" +set yrange [0:*] +set y2range [0:*] +set ylabel "Txs/Day" +set y2label "Bytes" +set y2tics nomirror +plot 'stats.csv' index "DATA" using (timecolumn(1,"%Y-%m-%d")):4 with lines, '' using (timecolumn(1,"%Y-%m-%d")):7 axes x1y2 with lines + */ + + // spit out a comment that GnuPlot can use as an index + std::cout << ENDL << "# DATA" << ENDL; + std::cout << "Date\tBlocks/day\tBlocks\tTxs/Day\tTxs\tBytes/Day\tBytes"; + if (do_inputs) + std::cout << "\tInMin\tInMax\tInAvg"; + if (do_outputs) + std::cout << "\tOutMin\tOutMax\tOutAvg"; + if (do_ringsize) + std::cout << "\tRingMin\tRingMax\tRingAvg"; + if (do_hours) { + char buf[8]; + unsigned int i; + for (i=0; i<24; i++) { + sprintf(buf, "\t%02u:00", i); + std::cout << buf; + } + } + std::cout << ENDL; + + struct tm prevtm = {0}, currtm; + uint64_t prevsz = 0, currsz = 0; + uint64_t prevtxs = 0, currtxs = 0; + uint64_t currblks = 0; + uint64_t totins = 0, totouts = 0, totrings = 0; + uint32_t minins = 10, maxins = 0; + uint32_t minouts = 10, maxouts = 0; + uint32_t minrings = 50, maxrings = 0; + uint32_t io, tottxs = 0; + uint32_t txhr[24] = {0}; + unsigned int i; + + for (uint64_t h = block_start; h < block_stop; ++h) + { + cryptonote::blobdata bd = db->get_block_blob_from_height(h); + cryptonote::block blk; + if (!cryptonote::parse_and_validate_block_from_blob(bd, blk)) + { + LOG_PRINT_L0("Bad block from db"); + return 1; + } + time_t tt = blk.timestamp; + char timebuf[64]; + epee::misc_utils::get_gmt_time(tt, currtm); + if (!prevtm.tm_year) + prevtm = currtm; + // catch change of day + if (currtm.tm_mday > prevtm.tm_mday || (currtm.tm_mday == 1 && prevtm.tm_mday > 27)) + { + // check for timestamp fudging around month ends + if (prevtm.tm_mday == 1 && currtm.tm_mday > 27) + goto skip; + strftime(timebuf, sizeof(timebuf), "%Y-%m-%d", &prevtm); + prevtm = currtm; + std::cout << timebuf << "\t" << currblks << "\t" << h << "\t" << currtxs << "\t" << prevtxs + currtxs << "\t" << currsz << "\t" << prevsz + currsz; + prevsz += currsz; + currsz = 0; + currblks = 0; + prevtxs += currtxs; + currtxs = 0; + if (!tottxs) + tottxs = 1; + if (do_inputs) { + std::cout << "\t" << (maxins ? minins : 0) << "\t" << maxins << "\t" << totins / tottxs; + minins = 10; maxins = 0; totins = 0; + } + if (do_outputs) { + std::cout << "\t" << (maxouts ? minouts : 0) << "\t" << maxouts << "\t" << totouts / tottxs; + minouts = 10; maxouts = 0; totouts = 0; + } + if (do_ringsize) { + std::cout << "\t" << (maxrings ? minrings : 0) << "\t" << maxrings << "\t" << totrings / tottxs; + minrings = 50; maxrings = 0; totrings = 0; + } + tottxs = 0; + if (do_hours) { + for (i=0; i<24; i++) { + std::cout << "\t" << txhr[i]; + txhr[i] = 0; + } + } + std::cout << ENDL; + } +skip: + currsz += bd.size(); + for (const auto& tx_id : blk.tx_hashes) + { + if (tx_id == crypto::null_hash) + { + throw std::runtime_error("Aborting: tx == null_hash"); + } + if (!db->get_tx_blob(tx_id, bd)) + { + throw std::runtime_error("Aborting: tx not found"); + } + transaction tx; + if (!parse_and_validate_tx_from_blob(bd, tx)) + { + LOG_PRINT_L0("Bad txn from db"); + return 1; + } + currsz += bd.size(); + currtxs++; + if (do_hours) + txhr[currtm.tm_hour]++; + if (do_inputs) { + io = tx.vin.size(); + if (io < minins) + minins = io; + else if (io > maxins) + maxins = io; + totins += io; + } + if (do_ringsize) { + const cryptonote::txin_to_key& tx_in_to_key + = boost::get(tx.vin[0]); + io = tx_in_to_key.key_offsets.size(); + if (io < minrings) + minrings = io; + else if (io > maxrings) + maxrings = io; + totrings += io; + } + if (do_outputs) { + io = tx.vout.size(); + if (io < minouts) + minouts = io; + else if (io > maxouts) + maxouts = io; + totouts += io; + } + tottxs++; + } + currblks++; + + if (stop_requested) + break; + } + + core_storage->deinit(); + return 0; + + CATCH_ENTRY("Stats reporting error", 1); +} diff --git a/src/blockchain_utilities/blockchain_usage.cpp b/src/blockchain_utilities/blockchain_usage.cpp index 38a0b26480b..bd73350b38e 100644 --- a/src/blockchain_utilities/blockchain_usage.cpp +++ b/src/blockchain_utilities/blockchain_usage.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/blockchain_utilities/blockchain_utilities.h b/src/blockchain_utilities/blockchain_utilities.h index e690305c4fb..78487b9953c 100644 --- a/src/blockchain_utilities/blockchain_utilities.h +++ b/src/blockchain_utilities/blockchain_utilities.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/blockchain_utilities/blocksdat_file.cpp b/src/blockchain_utilities/blocksdat_file.cpp index 45ef33acb05..f56ff5f949d 100644 --- a/src/blockchain_utilities/blocksdat_file.cpp +++ b/src/blockchain_utilities/blocksdat_file.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/blockchain_utilities/blocksdat_file.h b/src/blockchain_utilities/blocksdat_file.h index 70a1f30a7ad..315713424de 100644 --- a/src/blockchain_utilities/blocksdat_file.h +++ b/src/blockchain_utilities/blocksdat_file.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/blockchain_utilities/bootstrap_file.cpp b/src/blockchain_utilities/bootstrap_file.cpp index aa85e5e5312..252c7977622 100644 --- a/src/blockchain_utilities/bootstrap_file.cpp +++ b/src/blockchain_utilities/bootstrap_file.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -124,8 +124,8 @@ bool BootstrapFile::initialize_file() *m_raw_data_file << blob; bootstrap::file_info bfi; - bfi.major_version = 0; - bfi.minor_version = 1; + bfi.major_version = 1; + bfi.minor_version = 0; bfi.header_size = header_size; bootstrap::blocks_info bbi; @@ -236,11 +236,11 @@ void BootstrapFile::write_block(block& block) bool include_extra_block_data = true; if (include_extra_block_data) { - size_t block_size = m_blockchain_storage->get_db().get_block_size(block_height); + size_t block_weight = m_blockchain_storage->get_db().get_block_weight(block_height); difficulty_type cumulative_difficulty = m_blockchain_storage->get_db().get_block_cumulative_difficulty(block_height); uint64_t coins_generated = m_blockchain_storage->get_db().get_block_already_generated_coins(block_height); - bp.block_size = block_size; + bp.block_weight = block_weight; bp.cumulative_difficulty = cumulative_difficulty; bp.coins_generated = coins_generated; } @@ -304,7 +304,7 @@ bool BootstrapFile::store_blockchain_raw(Blockchain* _blockchain_storage, tx_mem } if (m_cur_height % progress_interval == 0) { std::cout << refresh_string; - std::cout << "block " << m_cur_height << "/" << block_stop << std::flush; + std::cout << "block " << m_cur_height << "/" << block_stop << "\r" << std::flush; } } // NOTE: use of NUM_BLOCKS_PER_CHUNK is a placeholder in case multi-block chunks are later supported. @@ -323,7 +323,7 @@ bool BootstrapFile::store_blockchain_raw(Blockchain* _blockchain_storage, tx_mem return BootstrapFile::close(); } -uint64_t BootstrapFile::seek_to_first_chunk(std::ifstream& import_file) +uint64_t BootstrapFile::seek_to_first_chunk(std::ifstream& import_file, uint8_t &major_version, uint8_t &minor_version) { uint32_t file_magic; @@ -371,6 +371,8 @@ uint64_t BootstrapFile::seek_to_first_chunk(std::ifstream& import_file) uint64_t full_header_size = sizeof(file_magic) + bfi.header_size; import_file.seekg(full_header_size); + major_version = bfi.major_version; + minor_version = bfi.minor_version; return full_header_size; } @@ -400,18 +402,18 @@ uint64_t BootstrapFile::count_bytes(std::ifstream& import_file, uint64_t blocks, { std::cout << refresh_string; MWARNING("WARNING: chunk_size " << chunk_size << " > BUFFER_SIZE " << BUFFER_SIZE - << " height: " << h-1); + << " height: " << h-1 << ", offset " << bytes_read); throw std::runtime_error("Aborting: chunk size exceeds buffer size"); } if (chunk_size > CHUNK_SIZE_WARNING_THRESHOLD) { std::cout << refresh_string; MDEBUG("NOTE: chunk_size " << chunk_size << " > " << CHUNK_SIZE_WARNING_THRESHOLD << " << height: " - << h-1); + << h-1 << ", offset " << bytes_read); } else if (chunk_size <= 0) { std::cout << refresh_string; - MDEBUG("ERROR: chunk_size " << chunk_size << " <= 0" << " height: " << h-1); + MDEBUG("ERROR: chunk_size " << chunk_size << " <= 0" << " height: " << h-1 << ", offset " << bytes_read); throw std::runtime_error("Aborting"); } // skip to next expected block size value @@ -461,7 +463,8 @@ uint64_t BootstrapFile::count_blocks(const std::string& import_file_path, std::s } uint64_t full_header_size; // 4 byte magic + length of header structures - full_header_size = seek_to_first_chunk(import_file); + uint8_t major_version, minor_version; + full_header_size = seek_to_first_chunk(import_file, major_version, minor_version); MINFO("Scanning blockchain from bootstrap file..."); bool quit = false; @@ -479,7 +482,7 @@ uint64_t BootstrapFile::count_blocks(const std::string& import_file_path, std::s bytes_read += count_bytes(import_file, progress_interval, blocks, quit); h += blocks; std::cout << "\r" << "block height: " << h-1 << - " " << + " \r" << std::flush; // std::cout << refresh_string; diff --git a/src/blockchain_utilities/bootstrap_file.h b/src/blockchain_utilities/bootstrap_file.h index 187db0938f0..1e6ef5d819c 100644 --- a/src/blockchain_utilities/bootstrap_file.h +++ b/src/blockchain_utilities/bootstrap_file.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -60,7 +60,7 @@ class BootstrapFile uint64_t count_bytes(std::ifstream& import_file, uint64_t blocks, uint64_t& h, bool& quit); uint64_t count_blocks(const std::string& dir_path, std::streampos& start_pos, uint64_t& seek_height); uint64_t count_blocks(const std::string& dir_path); - uint64_t seek_to_first_chunk(std::ifstream& import_file); + uint64_t seek_to_first_chunk(std::ifstream& import_file, uint8_t &major_version, uint8_t &minor_version); bool store_blockchain_raw(cryptonote::Blockchain* cs, cryptonote::tx_memory_pool* txp, boost::filesystem::path& output_file, uint64_t use_block_height=0); diff --git a/src/blockchain_utilities/bootstrap_serialization.h b/src/blockchain_utilities/bootstrap_serialization.h index 8755d3fe3e2..70b3eea7ed9 100644 --- a/src/blockchain_utilities/bootstrap_serialization.h +++ b/src/blockchain_utilities/bootstrap_serialization.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -29,7 +29,7 @@ #pragma once #include "cryptonote_basic/cryptonote_boost_serialization.h" -#include "cryptonote_basic/difficulty.h" +#include "serialization/difficulty_type.h" namespace cryptonote @@ -66,19 +66,36 @@ namespace cryptonote END_SERIALIZE() }; + struct block_package_1 + { + cryptonote::block block; + std::vector txs; + size_t block_weight; + uint64_t cumulative_difficulty; + uint64_t coins_generated; + + BEGIN_SERIALIZE() + FIELD(block) + FIELD(txs) + VARINT_FIELD(block_weight) + VARINT_FIELD(cumulative_difficulty) + VARINT_FIELD(coins_generated) + END_SERIALIZE() + }; + struct block_package { cryptonote::block block; std::vector txs; - size_t block_size; + size_t block_weight; difficulty_type cumulative_difficulty; uint64_t coins_generated; BEGIN_SERIALIZE() FIELD(block) FIELD(txs) - VARINT_FIELD(block_size) - VARINT_FIELD(cumulative_difficulty) + VARINT_FIELD(block_weight) + FIELD(cumulative_difficulty) VARINT_FIELD(coins_generated) END_SERIALIZE() }; diff --git a/src/blocks/CMakeLists.txt b/src/blocks/CMakeLists.txt index cf3f0b35480..0b7b02fee36 100644 --- a/src/blocks/CMakeLists.txt +++ b/src/blocks/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2014-2018, The Monero Project +# Copyright (c) 2014-2019, The Monero Project # # All rights reserved. # @@ -26,14 +26,23 @@ # STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF # THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -if(APPLE) - add_library(blocks STATIC blockexports.c) - set_target_properties(blocks PROPERTIES LINKER_LANGUAGE C) -else() - add_custom_command(OUTPUT blocks.o MAIN_DEPENDENCY blocks.dat COMMAND cd ${CMAKE_CURRENT_SOURCE_DIR} && ${CMAKE_LINKER} ${LD_RAW_FLAGS} -r -b binary -o ${CMAKE_CURRENT_BINARY_DIR}/blocks.o blocks.dat) - add_custom_command(OUTPUT testnet_blocks.o MAIN_DEPENDENCY testnet_blocks.dat COMMAND cd ${CMAKE_CURRENT_SOURCE_DIR} && ${CMAKE_LINKER} ${LD_RAW_FLAGS} -r -b binary -o ${CMAKE_CURRENT_BINARY_DIR}/testnet_blocks.o testnet_blocks.dat) - add_custom_command(OUTPUT stagenet_blocks.o MAIN_DEPENDENCY stagenet_blocks.dat COMMAND cd ${CMAKE_CURRENT_SOURCE_DIR} && ${CMAKE_LINKER} ${LD_RAW_FLAGS} -r -b binary -o ${CMAKE_CURRENT_BINARY_DIR}/stagenet_blocks.o stagenet_blocks.dat) - add_library(blocks STATIC blocks.o testnet_blocks.o stagenet_blocks.o blockexports.c) - set_target_properties(blocks PROPERTIES LINKER_LANGUAGE C) -endif() +set(GENERATED_SOURCES "") +foreach(BLOB_NAME checkpoints testnet_blocks stagenet_blocks) + set(OUTPUT_C_SOURCE "generated_${BLOB_NAME}.c") + list(APPEND GENERATED_SOURCES ${OUTPUT_C_SOURCE}) + set(INPUT_DAT_FILE "${BLOB_NAME}.dat") + add_custom_command( + OUTPUT ${OUTPUT_C_SOURCE} + MAIN_DEPENDENCY ${INPUT_DAT_FILE} + COMMAND + cd ${CMAKE_CURRENT_BINARY_DIR} && + echo "'#include\t'" > ${OUTPUT_C_SOURCE} && + echo "'const\tunsigned\tchar\t${BLOB_NAME}[]={'" >> ${OUTPUT_C_SOURCE} && + od -v -An -tx1 ${CMAKE_CURRENT_SOURCE_DIR}/${INPUT_DAT_FILE} | sed -e "'s/[0-9a-fA-F]\\{1,\\}/0x&,/g'" -e "'$$s/.$$//'" >> ${OUTPUT_C_SOURCE} && + echo "'};'" >> ${OUTPUT_C_SOURCE} && + echo "'const\tsize_t\t${BLOB_NAME}_len\t=\tsizeof(${BLOB_NAME});'" >> ${OUTPUT_C_SOURCE} + ) +endforeach() + +add_library(blocks STATIC blocks.cpp ${GENERATED_SOURCES}) diff --git a/src/blocks/blockexports.c b/src/blocks/blockexports.c deleted file mode 100644 index 0154b041351..00000000000 --- a/src/blocks/blockexports.c +++ /dev/null @@ -1,87 +0,0 @@ -#include - -#if defined(__APPLE__) -#include -#ifdef BUILD_SHARED_LIBS -#if !defined(__LP64__) -const struct mach_header _mh_execute_header; -#else -const struct mach_header_64 _mh_execute_header; -#endif -#else -#if !defined(__LP64__) -extern const struct mach_header _mh_execute_header; -#else -extern const struct mach_header_64 _mh_execute_header; -#endif -#endif - -const unsigned char *get_blocks_dat_start(int testnet, int stagenet) -{ - size_t size; - if (testnet) - return getsectiondata(&_mh_execute_header, "__DATA", "__testnet_blocks_dat", &size); - else if (stagenet) - return getsectiondata(&_mh_execute_header, "__DATA", "__stagenet_blocks_dat", &size); - else - return getsectiondata(&_mh_execute_header, "__DATA", "__blocks_dat", &size); -} - -size_t get_blocks_dat_size(int testnet, int stagenet) -{ - size_t size; - if (testnet) - getsectiondata(&_mh_execute_header, "__DATA", "__testnet_blocks_dat", &size); - else if (stagenet) - getsectiondata(&_mh_execute_header, "__DATA", "__stagenet_blocks_dat", &size); - else - getsectiondata(&_mh_execute_header, "__DATA", "__blocks_dat", &size); - return size; -} - -#else - -#if defined(_WIN32) && !defined(_WIN64) -#define _binary_blocks_start binary_blocks_dat_start -#define _binary_blocks_end binary_blocks_dat_end -#define _binary_testnet_blocks_start binary_testnet_blocks_dat_start -#define _binary_testnet_blocks_end binary_testnet_blocks_dat_end -#define _binary_stagenet_blocks_start binary_stagenet_blocks_dat_start -#define _binary_stagenet_blocks_end binary_stagenet_blocks_dat_end -#else -#define _binary_blocks_start _binary_blocks_dat_start -#define _binary_blocks_end _binary_blocks_dat_end -#define _binary_testnet_blocks_start _binary_testnet_blocks_dat_start -#define _binary_testnet_blocks_end _binary_testnet_blocks_dat_end -#define _binary_stagenet_blocks_start _binary_stagenet_blocks_dat_start -#define _binary_stagenet_blocks_end _binary_stagenet_blocks_dat_end -#endif - -extern const unsigned char _binary_blocks_start[]; -extern const unsigned char _binary_blocks_end[]; -extern const unsigned char _binary_testnet_blocks_start[]; -extern const unsigned char _binary_testnet_blocks_end[]; -extern const unsigned char _binary_stagenet_blocks_start[]; -extern const unsigned char _binary_stagenet_blocks_end[]; - -const unsigned char *get_blocks_dat_start(int testnet, int stagenet) -{ - if (testnet) - return _binary_testnet_blocks_start; - else if (stagenet) - return _binary_stagenet_blocks_start; - else - return _binary_blocks_start; -} - -size_t get_blocks_dat_size(int testnet, int stagenet) -{ - if (testnet) - return (size_t) (_binary_testnet_blocks_end - _binary_testnet_blocks_start); - else if (stagenet) - return (size_t) (_binary_stagenet_blocks_end - _binary_stagenet_blocks_start); - else - return (size_t) (_binary_blocks_end - _binary_blocks_start); -} - -#endif diff --git a/src/blocks/blocks.cpp b/src/blocks/blocks.cpp new file mode 100644 index 00000000000..0661f8448bd --- /dev/null +++ b/src/blocks/blocks.cpp @@ -0,0 +1,31 @@ +#include "blocks.h" + +#include + +extern const unsigned char checkpoints[]; +extern const size_t checkpoints_len; +extern const unsigned char stagenet_blocks[]; +extern const size_t stagenet_blocks_len; +extern const unsigned char testnet_blocks[]; +extern const size_t testnet_blocks_len; + +namespace blocks +{ + + const std::unordered_map, std::hash> CheckpointsByNetwork = { + {cryptonote::network_type::MAINNET, {checkpoints, checkpoints_len}}, + {cryptonote::network_type::STAGENET, {stagenet_blocks, stagenet_blocks_len}}, + {cryptonote::network_type::TESTNET, {testnet_blocks, testnet_blocks_len}} + }; + + const epee::span GetCheckpointsData(cryptonote::network_type network) + { + const auto it = CheckpointsByNetwork.find(network); + if (it != CheckpointsByNetwork.end()) + { + return it->second; + } + return nullptr; + } + +} diff --git a/src/blocks/blocks.h b/src/blocks/blocks.h index ec683f47ef6..14e391319f6 100644 --- a/src/blocks/blocks.h +++ b/src/blocks/blocks.h @@ -1,16 +1,12 @@ #ifndef SRC_BLOCKS_BLOCKS_H_ #define SRC_BLOCKS_BLOCKS_H_ -#ifdef __cplusplus -extern "C" { -#endif +#include "cryptonote_config.h" +#include "span.h" -const unsigned char *get_blocks_dat_start(int testnet, int stagenet); -size_t get_blocks_dat_size(int testnet, int stagenet); - -#ifdef __cplusplus +namespace blocks +{ + const epee::span GetCheckpointsData(cryptonote::network_type network); } -#endif - #endif /* SRC_BLOCKS_BLOCKS_H_ */ diff --git a/src/blocks/checkpoints.dat b/src/blocks/checkpoints.dat index 501a55673eb..a7d3097535d 100644 Binary files a/src/blocks/checkpoints.dat and b/src/blocks/checkpoints.dat differ diff --git a/src/checkpoints/CMakeLists.txt b/src/checkpoints/CMakeLists.txt index 02bb2891a32..d324f518e02 100644 --- a/src/checkpoints/CMakeLists.txt +++ b/src/checkpoints/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2014-2018, The Monero Project +# Copyright (c) 2014-2019, The Monero Project # # All rights reserved. # @@ -27,9 +27,13 @@ # THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. if(APPLE) - find_library(IOKIT_LIBRARY IOKit) - mark_as_advanced(IOKIT_LIBRARY) - list(APPEND EXTRA_LIBRARIES ${IOKIT_LIBRARY}) + if(DEPENDS) + list(APPEND EXTRA_LIBRARIES "-framework Foundation -framework ApplicationServices -framework AppKit -framework IOKit") + else() + find_library(IOKIT_LIBRARY IOKit) + mark_as_advanced(IOKIT_LIBRARY) + list(APPEND EXTRA_LIBRARIES ${IOKIT_LIBRARY}) + endif() endif() set(checkpoints_sources diff --git a/src/checkpoints/checkpoints.cpp b/src/checkpoints/checkpoints.cpp index ef1ee171d18..11bbe2e24e3 100644 --- a/src/checkpoints/checkpoints.cpp +++ b/src/checkpoints/checkpoints.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -28,17 +28,15 @@ // // Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers -#include "include_base_utils.h" - -using namespace epee; - #include "checkpoints.h" #include "common/dns_utils.h" -#include "include_base_utils.h" #include "string_tools.h" #include "storages/portable_storage_template_helper.h" // epee json include #include "serialization/keyvalue_serialization.h" +#include + +using namespace epee; #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "checkpoints" @@ -76,7 +74,7 @@ namespace cryptonote bool checkpoints::add_checkpoint(uint64_t height, const std::string& hash_str) { crypto::hash h = crypto::null_hash; - bool r = epee::string_tools::parse_tpod_from_hex_string(hash_str, h); + bool r = epee::string_tools::hex_to_pod(hash_str, h); CHECK_AND_ASSERT_MES(r, false, "Failed to parse checkpoint hash string into binary representation!"); // return false if adding at a height we already have AND the hash is different @@ -165,6 +163,7 @@ namespace cryptonote { ADD_CHECKPOINT(0, "48ca7cd3c8de5b6a4d53d2861fbdaedca141553559f9be9520068053cda8430b"); ADD_CHECKPOINT(1000000, "46b690b710a07ea051bc4a6b6842ac37be691089c0f7758cfeec4d5fc0b4a258"); + ADD_CHECKPOINT(1058600, "12904f6b4d9e60fd875674e07147d2c83d6716253f046af7b894c3e81da7e1bd"); return true; } if (nettype == STAGENET) @@ -208,7 +207,9 @@ namespace cryptonote ADD_CHECKPOINT(1450000, "ac94e8860093bc7c83e4e91215cba1d663421ecf4067a0ae609c3a8b52bcfac2"); ADD_CHECKPOINT(1530000, "01759bce497ec38e63c78b1038892169203bb78f87e488172f6b854fcd63ba7e"); ADD_CHECKPOINT(1579000, "7d0d7a2346373afd41ed1e744a939fc5d474a7dbaa257be5c6fff4009e789241"); - + ADD_CHECKPOINT(1668900, "ac2dcaf3d2f58ffcf8391639f0f1ebafcb8eac43c49479c7c37f611868d07568"); + ADD_CHECKPOINT(1775600, "1c6e01c661dc22cab939e79ec6a5272190624ce8356d2f7b958e4f9a57fdb05e"); + ADD_CHECKPOINT(1856000, "9b57f17f29c71a3acd8a7904b93c41fa6eb8d2b7c73936ce4f1702d14880ba29"); return true; } @@ -293,7 +294,7 @@ namespace cryptonote // parse the second part as crypto::hash, // if this fails move on to the next record std::string hashStr = record.substr(pos + 1); - if (!epee::string_tools::parse_tpod_from_hex_string(hashStr, hash)) + if (!epee::string_tools::hex_to_pod(hashStr, hash)) { continue; } diff --git a/src/checkpoints/checkpoints.h b/src/checkpoints/checkpoints.h index 61be2c27acc..a55b94bf089 100644 --- a/src/checkpoints/checkpoints.h +++ b/src/checkpoints/checkpoints.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -30,7 +30,6 @@ #pragma once #include -#include #include "misc_log_ex.h" #include "crypto/hash.h" #include "cryptonote_config.h" diff --git a/src/common/CMakeLists.txt b/src/common/CMakeLists.txt index 808ef76306c..f06737b31b6 100644 --- a/src/common/CMakeLists.txt +++ b/src/common/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2014-2018, The Monero Project +# Copyright (c) 2014-2019, The Monero Project # # All rights reserved. # @@ -33,17 +33,29 @@ set(common_sources command_line.cpp dns_utils.cpp download.cpp + error.cpp + expect.cpp util.cpp i18n.cpp + notify.cpp password.cpp perf_timer.cpp + pruning.cpp + spawn.cpp threadpool.cpp - updates.cpp) + updates.cpp + aligned.c + timings.cc + combinator.cpp) if (STACK_TRACE) list(APPEND common_sources stack_trace.cpp) endif() +if (BACKCOMPAT) + list(APPEND common_sources compat/glibc_compat.cpp) +endif() + set(common_headers) set(common_private_headers @@ -54,9 +66,12 @@ set(common_private_headers common_fwd.h dns_utils.h download.h + error.h + expect.h http_connection.h - int-util.h + notify.h pod-class.h + pruning.h rpc_client.h scoped_message_writer.h unordered_containers_boost_serialization.h @@ -65,9 +80,13 @@ set(common_private_headers i18n.h password.h perf_timer.h + spawn.h stack_trace.h threadpool.h - updates.h) + updates.h + aligned.h + timings.h + combinator.h) monero_private_headers(common ${common_private_headers}) @@ -78,7 +97,6 @@ monero_add_library(common DEPENDS generate_translations_header) target_link_libraries(common PUBLIC - epee cncrypto ${UNBOUND_LIBRARY} ${LIBUNWIND_LIBRARIES} diff --git a/src/common/aligned.c b/src/common/aligned.c new file mode 100644 index 00000000000..6982409f7df --- /dev/null +++ b/src/common/aligned.c @@ -0,0 +1,139 @@ +// Copyright (c) 2017-2019, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include +#include +#include +#include +#include +#include "aligned.h" + +static inline int is_power_of_2(size_t n) { return n && (n & (n-1)) == 0; } + +#define MAGIC 0xaa0817161500ff81 +#define MAGIC_FREED 0xaa0817161500ff82 + +static void local_abort(const char *msg) +{ + fprintf(stderr, "%s\n", msg); +#ifdef NDEBUG + _exit(1); +#else + abort(); +#endif +} + +typedef struct +{ + uint64_t magic; + void *raw; + size_t bytes; + size_t align; +} control; + +void *aligned_malloc(size_t bytes, size_t align) +{ + void *raw, *ptr; + control *ctrl; + + if (!is_power_of_2(align)) + return NULL; + if (bytes > (size_t)-1 - align) + return NULL; + if (bytes + align > (size_t)-1 - sizeof(control)) + return NULL; + + raw = malloc(bytes + sizeof(control) + align); + if (!raw) + return NULL; + ptr = (void*)(((uintptr_t)raw + align + sizeof(control) - 1) & ~(align-1)); + ctrl = ((control*)ptr) - 1; + ctrl->magic = MAGIC; + ctrl->raw = raw; + ctrl->bytes = bytes; + ctrl->align = align; + return ptr; +} + +void *aligned_realloc(void *ptr, size_t bytes, size_t align) +{ + void *raw, *ptr2; + control *ctrl, *ctrl2; + + if (!ptr) + return aligned_malloc(bytes, align); + if (!bytes) + { + aligned_free(ptr); + return NULL; + } + if (!is_power_of_2(align)) + return NULL; + + ctrl = ((control*)ptr) - 1; + if (ctrl->magic == MAGIC_FREED) + local_abort("Double free detected"); + if (ctrl->magic != MAGIC) + local_abort("Freeing unallocated memory"); + if (ctrl->align != align) + return NULL; + if (ctrl->bytes >= bytes) + return ptr; + + if (ctrl->bytes > (size_t)-1 - ctrl->align) + return NULL; + if (ctrl->bytes + ctrl->align > (size_t)-1 - sizeof(control)) + return NULL; + + raw = malloc(bytes + sizeof(control) + ctrl->align); + if (!raw) + return NULL; + ptr2 = (void*)(((uintptr_t)raw + ctrl->align + sizeof(control) - 1) & ~(ctrl->align-1)); + memcpy(ptr2, ptr, ctrl->bytes); + ctrl2 = ((control*)ptr2) - 1; + ctrl2->magic = MAGIC; + ctrl2->raw = raw; + ctrl2->bytes = bytes; + ctrl2->align = ctrl->align; + ctrl->magic = MAGIC_FREED; + free(ctrl->raw); + return ptr2; +} + +void aligned_free(void *ptr) +{ + if (!ptr) + return; + control *ctrl = ((control*)ptr) - 1; + if (ctrl->magic == MAGIC_FREED) + local_abort("Double free detected"); + if (ctrl->magic != MAGIC) + local_abort("Freeing unallocated memory"); + ctrl->magic = MAGIC_FREED; + free(ctrl->raw); +} diff --git a/src/common/aligned.h b/src/common/aligned.h new file mode 100644 index 00000000000..fed3ccb36ac --- /dev/null +++ b/src/common/aligned.h @@ -0,0 +1,41 @@ +// Copyright (c) 2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +#ifdef __cplusplus +extern "C" { +#endif + +void *aligned_malloc(size_t bytes, size_t align); +void *aligned_realloc(void *ptr, size_t bytes, size_t align); +void aligned_free(void *ptr); + +#ifdef __cplusplus +} +#endif diff --git a/src/common/apply_permutation.h b/src/common/apply_permutation.h index ff346bab1fe..a4b2c8b78df 100644 --- a/src/common/apply_permutation.h +++ b/src/common/apply_permutation.h @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2018, The Monero Project +// Copyright (c) 2017-2019, The Monero Project // // All rights reserved. // diff --git a/src/common/base58.cpp b/src/common/base58.cpp index 75556cad94c..ac1bf4b2975 100644 --- a/src/common/base58.cpp +++ b/src/common/base58.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -36,7 +36,6 @@ #include "crypto/hash.h" #include "int-util.h" -#include "util.h" #include "varint.h" namespace tools @@ -109,20 +108,8 @@ namespace tools assert(1 <= size && size <= sizeof(uint64_t)); uint64_t res = 0; - switch (9 - size) - { - case 1: res |= *data++; /* FALLTHRU */ - case 2: res <<= 8; res |= *data++; /* FALLTHRU */ - case 3: res <<= 8; res |= *data++; /* FALLTHRU */ - case 4: res <<= 8; res |= *data++; /* FALLTHRU */ - case 5: res <<= 8; res |= *data++; /* FALLTHRU */ - case 6: res <<= 8; res |= *data++; /* FALLTHRU */ - case 7: res <<= 8; res |= *data++; /* FALLTHRU */ - case 8: res <<= 8; res |= *data; break; - default: assert(false); - } - - return res; + memcpy(reinterpret_cast(&res) + sizeof(uint64_t) - size, data, size); + return SWAP64BE(res); } void uint_64_to_8be(uint64_t num, size_t size, uint8_t* data) @@ -247,7 +234,7 @@ namespace tools return encode(buf); } - bool decode_addr(std::string addr, uint64_t& tag, std::string& data) + bool decode_addr(const std::string &addr, uint64_t& tag, std::string& data) { std::string addr_data; bool r = decode(addr, addr_data); diff --git a/src/common/base58.h b/src/common/base58.h index 02ca969564d..6bf2c3bb7c3 100644 --- a/src/common/base58.h +++ b/src/common/base58.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -41,6 +41,6 @@ namespace tools bool decode(const std::string& enc, std::string& data); std::string encode_addr(uint64_t tag, const std::string& data); - bool decode_addr(std::string addr, uint64_t& tag, std::string& data); + bool decode_addr(const std::string &addr, uint64_t& tag, std::string& data); } } diff --git a/src/common/boost_serialization_helper.h b/src/common/boost_serialization_helper.h index 3f5c623f8b0..2280f3312e6 100644 --- a/src/common/boost_serialization_helper.h +++ b/src/common/boost_serialization_helper.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/common/combinator.cpp b/src/common/combinator.cpp new file mode 100644 index 00000000000..cb4fbc908f3 --- /dev/null +++ b/src/common/combinator.cpp @@ -0,0 +1,50 @@ +// Copyright (c) 2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers + +#include "combinator.h" + +namespace tools { + +uint64_t combinations_count(uint32_t k, uint32_t n) +{ + if (k > n) { + throw std::runtime_error("k must not be greater than n"); + } + + uint64_t c = 1; + for (uint64_t i = 1; i <= k; ++i) { + c *= n--; + c /= i; + } + + return c; +} + +} diff --git a/src/common/combinator.h b/src/common/combinator.h new file mode 100644 index 00000000000..ba851bd8149 --- /dev/null +++ b/src/common/combinator.h @@ -0,0 +1,97 @@ +// Copyright (c) 2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers + +#pragma once + +#include +#include +#include + +namespace tools { + +uint64_t combinations_count(uint32_t k, uint32_t n); + +template +class Combinator { +public: + Combinator(const std::vector& v) : origin(v) { } + + std::vector> combine(size_t k); + +private: + void doCombine(size_t from, size_t k); + + std::vector origin; + std::vector> combinations; + std::vector current; +}; + +template +std::vector> Combinator::combine(size_t k) +{ + if (k > origin.size()) + { + throw std::runtime_error("k must be smaller than elements number"); + } + + if (k == 0) + { + throw std::runtime_error("k must be greater than zero"); + } + + combinations.clear(); + doCombine(0, k); + return combinations; +} + +template +void Combinator::doCombine(size_t from, size_t k) +{ + current.push_back(0); + + for (size_t i = from; i <= origin.size() - k; ++i) + { + current.back() = i; + + if (k > 1) { + doCombine(i + 1, k - 1); + } else { + std::vector comb; + for (auto ind: current) { + comb.push_back(origin[ind]); + } + combinations.push_back(comb); + } + } + + current.pop_back(); +} + +} //namespace tools diff --git a/src/common/command_line.cpp b/src/common/command_line.cpp index 7980b381fee..cae744ea50d 100644 --- a/src/common/command_line.cpp +++ b/src/common/command_line.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -31,10 +31,7 @@ #include "command_line.h" #include #include -#include #include "common/i18n.h" -#include "cryptonote_config.h" -#include "string_tools.h" namespace command_line { diff --git a/src/common/command_line.h b/src/common/command_line.h index 3a869bb26ec..b5e3d94a702 100644 --- a/src/common/command_line.h +++ b/src/common/command_line.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/common/common_fwd.h b/src/common/common_fwd.h index 2924d9cbe58..7eaa6cdee6c 100644 --- a/src/common/common_fwd.h +++ b/src/common/common_fwd.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/common/compat/glibc_compat.cpp b/src/common/compat/glibc_compat.cpp new file mode 100644 index 00000000000..435e7930fca --- /dev/null +++ b/src/common/compat/glibc_compat.cpp @@ -0,0 +1,98 @@ +#include +#include +#include +#include +#include +#include +#include + +#if defined(HAVE_SYS_SELECT_H) +#include +#endif + +// Prior to GLIBC_2.14, memcpy was aliased to memmove. +extern "C" void* memmove(void* a, const void* b, size_t c); +//extern "C" void* memset(void* a, int b, long unsigned int c); +extern "C" void* memcpy(void* a, const void* b, size_t c) +{ + return memmove(a, b, c); +} + +extern "C" void __chk_fail(void) __attribute__((__noreturn__)); + +#if defined(__i386__) || defined(__arm__) + +extern "C" int64_t __udivmoddi4(uint64_t u, uint64_t v, uint64_t* rp); + +extern "C" int64_t __wrap___divmoddi4(int64_t u, int64_t v, int64_t* rp) +{ + int32_t c1 = 0, c2 = 0; + int64_t uu = u, vv = v; + int64_t w; + int64_t r; + + if (uu < 0) { + c1 = ~c1, c2 = ~c2, uu = -uu; + } + if (vv < 0) { + c1 = ~c1, vv = -vv; + } + + w = __udivmoddi4(uu, vv, (uint64_t*)&r); + if (c1) + w = -w; + if (c2) + r = -r; + + *rp = r; + return w; +} +#endif + +/* glibc-internal users use __explicit_bzero_chk, and explicit_bzero + redirects to that. */ +#undef explicit_bzero +/* Set LEN bytes of S to 0. The compiler will not delete a call to + this function, even if S is dead after the call. */ +void +explicit_bzero (void *s, size_t len) +{ + memset (s, '\0', len); + /* Compiler barrier. */ + asm volatile ("" ::: "memory"); +} + +// Redefine explicit_bzero_chk +void +__explicit_bzero_chk (void *dst, size_t len, size_t dstlen) +{ + /* Inline __memset_chk to avoid a PLT reference to __memset_chk. */ + if (__glibc_unlikely (dstlen < len)) + __chk_fail (); + memset (dst, '\0', len); + /* Compiler barrier. */ + asm volatile ("" ::: "memory"); +} +/* libc-internal references use the hidden + __explicit_bzero_chk_internal symbol. This is necessary if + __explicit_bzero_chk is implemented as an IFUNC because some + targets do not support hidden references to IFUNC symbols. */ +#define strong_alias (__explicit_bzero_chk, __explicit_bzero_chk_internal) + +#undef glob +extern "C" int glob_old(const char * pattern, int flags, int (*errfunc) (const char *epath, int eerrno), glob_t *pglob); +#ifdef __i386__ +__asm__(".symver glob_old,glob@GLIBC_2.0"); +#elif defined(__amd64__) +__asm__(".symver glob_old,glob@GLIBC_2.2.5"); +#elif defined(__arm__) +__asm(".symver glob_old,glob@GLIBC_2.4"); +#elif defined(__aarch64__) +__asm__(".symver glob_old,glob@GLIBC_2.17"); +#endif + +extern "C" int __wrap_glob(const char * pattern, int flags, int (*errfunc) (const char *epath, int eerrno), glob_t *pglob) +{ + return glob_old(pattern, flags, errfunc, pglob); +} + diff --git a/src/common/dns_utils.cpp b/src/common/dns_utils.cpp index 33f60bc3c80..dc1f335a77b 100644 --- a/src/common/dns_utils.cpp +++ b/src/common/dns_utils.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -32,13 +32,12 @@ #include #include "include_base_utils.h" -#include -#include +#include "common/threadpool.h" +#include "crypto/crypto.h" #include -#include #include +#include using namespace epee; -namespace bf = boost::filesystem; #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "net.dns" @@ -46,10 +45,10 @@ namespace bf = boost::filesystem; static const char *DEFAULT_DNS_PUBLIC_ADDR[] = { "194.150.168.168", // CCC (Germany) - "81.3.27.54", // Lightning Wire Labs (Germany) - "31.3.135.232", // OpenNIC (Switzerland) "80.67.169.40", // FDN (France) - "209.58.179.186", // Cyberghost (Singapore) + "89.233.43.71", // http://censurfridns.dk (Denmark) + "109.69.8.51", // punCAT (Spain) + "193.58.251.251", // SkyDNS (Russia) }; static boost::mutex instance_lock; @@ -97,11 +96,16 @@ get_builtin_cert(void) */ /** return the built in root DS trust anchor */ -static const char* +static const char* const* get_builtin_ds(void) { - return -". IN DS 19036 8 2 49AAC11D7B6F6446702E54A1607371607A1A41855200FD2CE1CDDE32F24E8FB5\n"; + static const char * const ds[] = + { + ". IN DS 19036 8 2 49AAC11D7B6F6446702E54A1607371607A1A41855200FD2CE1CDDE32F24E8FB5\n", + ". IN DS 20326 8 2 E06D44B80B8F1D39A95C0B0D7C65D08458E880409BBC683457104237C7F8EC8D\n", + NULL + }; + return ds; } /************************************************************ @@ -113,10 +117,25 @@ get_builtin_ds(void) namespace tools { +static const char *get_record_name(int record_type) +{ + switch (record_type) + { + case DNS_TYPE_A: return "A"; + case DNS_TYPE_TXT: return "TXT"; + case DNS_TYPE_AAAA: return "AAAA"; + default: return "unknown"; + } +} + // fuck it, I'm tired of dealing with getnameinfo()/inet_ntop/etc -std::string ipv4_to_string(const char* src, size_t len) +boost::optional ipv4_to_string(const char* src, size_t len) { - assert(len >= 4); + if (len < 4) + { + MERROR("Invalid IPv4 address: " << std::string(src, len)); + return boost::none; + } std::stringstream ss; unsigned int bytes[4]; @@ -134,9 +153,13 @@ std::string ipv4_to_string(const char* src, size_t len) // this obviously will need to change, but is here to reflect the above // stop-gap measure and to make the tests pass at least... -std::string ipv6_to_string(const char* src, size_t len) +boost::optional ipv6_to_string(const char* src, size_t len) { - assert(len >= 8); + if (len < 8) + { + MERROR("Invalid IPv4 address: " << std::string(src, len)); + return boost::none; + } std::stringstream ss; unsigned int bytes[8]; @@ -156,8 +179,10 @@ std::string ipv6_to_string(const char* src, size_t len) return ss.str(); } -std::string txt_to_string(const char* src, size_t len) +boost::optional txt_to_string(const char* src, size_t len) { + if (len == 0) + return boost::none; return std::string(src+1, len-1); } @@ -206,13 +231,24 @@ class string_copy { char *str; }; +static void add_anchors(ub_ctx *ctx) +{ + const char * const *ds = ::get_builtin_ds(); + while (*ds) + { + MINFO("adding trust anchor: " << *ds); + ub_ctx_add_ta(ctx, string_copy(*ds++)); + } +} + DNSResolver::DNSResolver() : m_data(new DNSResolverData()) { int use_dns_public = 0; std::vector dns_public_addr; - if (auto res = getenv("DNS_PUBLIC")) + const char *DNS_PUBLIC = getenv("DNS_PUBLIC"); + if (DNS_PUBLIC) { - dns_public_addr = tools::dns_utils::parse_dns_public(res); + dns_public_addr = tools::dns_utils::parse_dns_public(DNS_PUBLIC); if (!dns_public_addr.empty()) { MGINFO("Using public DNS server(s): " << boost::join(dns_public_addr, ", ") << " (TCP)"); @@ -240,7 +276,28 @@ DNSResolver::DNSResolver() : m_data(new DNSResolverData()) ub_ctx_hosts(m_data->m_ub_context, NULL); } - ub_ctx_add_ta(m_data->m_ub_context, string_copy(::get_builtin_ds())); + add_anchors(m_data->m_ub_context); + + if (!DNS_PUBLIC) + { + // if no DNS_PUBLIC specified, we try a lookup to what we know + // should be a valid DNSSEC record, and switch to known good + // DNSSEC resolvers if verification fails + bool available, valid; + static const char *probe_hostname = "updates.moneropulse.org"; + auto records = get_txt_record(probe_hostname, available, valid); + if (!valid) + { + MINFO("Failed to verify DNSSEC record from " << probe_hostname << ", falling back to TCP with well known DNSSEC resolvers"); + ub_ctx_delete(m_data->m_ub_context); + m_data->m_ub_context = ub_ctx_create(); + add_anchors(m_data->m_ub_context); + for (const auto &ip: DEFAULT_DNS_PUBLIC_ADDR) + ub_ctx_set_fwd(m_data->m_ub_context, string_copy(ip)); + ub_ctx_set_option(m_data->m_ub_context, string_copy("do-udp:"), string_copy("no")); + ub_ctx_set_option(m_data->m_ub_context, string_copy("do-tcp:"), string_copy("yes")); + } + } } DNSResolver::~DNSResolver() @@ -255,7 +312,7 @@ DNSResolver::~DNSResolver() } } -std::vector DNSResolver::get_record(const std::string& url, int record_type, std::string (*reader)(const char *,size_t), bool& dnssec_available, bool& dnssec_valid) +std::vector DNSResolver::get_record(const std::string& url, int record_type, boost::optional (*reader)(const char *,size_t), bool& dnssec_available, bool& dnssec_valid) { std::vector addresses; dnssec_available = false; @@ -272,13 +329,18 @@ std::vector DNSResolver::get_record(const std::string& url, int rec // call DNS resolver, blocking. if return value not zero, something went wrong if (!ub_resolve(m_data->m_ub_context, string_copy(url.c_str()), record_type, DNS_CLASS_IN, &result)) { - dnssec_available = (result->secure || (!result->secure && result->bogus)); + dnssec_available = (result->secure || result->bogus); dnssec_valid = result->secure && !result->bogus; if (result->havedata) { for (size_t i=0; result->data[i] != NULL; i++) { - addresses.push_back((*reader)(result->data[i], result->len[i])); + boost::optional res = (*reader)(result->data[i], result->len[i]); + if (res) + { + MINFO("Found \"" << *res << "\" in " << get_record_name(record_type) << " record for " << url); + addresses.push_back(*res); + } } } } @@ -454,22 +516,19 @@ bool load_txt_records_from_dns(std::vector &good_records, const std std::vector > records; records.resize(dns_urls.size()); - std::random_device rd; - std::mt19937 gen(rd()); - std::uniform_int_distribution dis(0, dns_urls.size() - 1); - size_t first_index = dis(gen); + size_t first_index = crypto::rand_idx(dns_urls.size()); // send all requests in parallel - std::vector threads(dns_urls.size()); std::deque avail(dns_urls.size(), false), valid(dns_urls.size(), false); + tools::threadpool& tpool = tools::threadpool::getInstance(); + tools::threadpool::waiter waiter; for (size_t n = 0; n < dns_urls.size(); ++n) { - threads[n] = boost::thread([n, dns_urls, &records, &avail, &valid](){ + tpool.submit(&waiter,[n, dns_urls, &records, &avail, &valid](){ records[n] = tools::DNSResolver::instance().get_txt_record(dns_urls[n], avail[n], valid[n]); }); } - for (size_t n = 0; n < dns_urls.size(); ++n) - threads[n].join(); + waiter.wait(&tpool); size_t cur_index = first_index; do @@ -478,12 +537,12 @@ bool load_txt_records_from_dns(std::vector &good_records, const std if (!avail[cur_index]) { records[cur_index].clear(); - LOG_PRINT_L2("DNSSEC not available for checkpoint update at URL: " << url << ", skipping."); + LOG_PRINT_L2("DNSSEC not available for hostname: " << url << ", skipping."); } if (!valid[cur_index]) { records[cur_index].clear(); - LOG_PRINT_L2("DNSSEC validation failed for checkpoint update at URL: " << url << ", skipping."); + LOG_PRINT_L2("DNSSEC validation failed for hostname: " << url << ", skipping."); } cur_index++; @@ -505,7 +564,7 @@ bool load_txt_records_from_dns(std::vector &good_records, const std if (num_valid_records < 2) { - LOG_PRINT_L0("WARNING: no two valid MoneroPulse DNS checkpoint records were received"); + LOG_PRINT_L0("WARNING: no two valid DNS TXT records were received"); return false; } @@ -527,7 +586,7 @@ bool load_txt_records_from_dns(std::vector &good_records, const std if (good_records_index < 0) { - LOG_PRINT_L0("WARNING: no two MoneroPulse DNS checkpoint records matched"); + LOG_PRINT_L0("WARNING: no two DNS TXT records matched"); return false; } diff --git a/src/common/dns_utils.h b/src/common/dns_utils.h index f46bca3dd02..a6bc7463a5a 100644 --- a/src/common/dns_utils.h +++ b/src/common/dns_utils.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -30,6 +30,7 @@ #include #include #include +#include namespace tools { @@ -143,7 +144,7 @@ class DNSResolver * @return A vector of strings containing the requested record; or an empty vector */ // TODO: modify this to accommodate DNSSEC - std::vector get_record(const std::string& url, int record_type, std::string (*reader)(const char *,size_t), bool& dnssec_available, bool& dnssec_valid); + std::vector get_record(const std::string& url, int record_type, boost::optional (*reader)(const char *,size_t), bool& dnssec_available, bool& dnssec_valid); /** * @brief Checks a string to see if it looks like a URL diff --git a/src/common/download.cpp b/src/common/download.cpp index 6698a5abfe0..f07d6798d7d 100644 --- a/src/common/download.cpp +++ b/src/common/download.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2018, The Monero Project +// Copyright (c) 2017-2019, The Monero Project // // All rights reserved. // @@ -29,10 +29,7 @@ #include #include #include -#include #include -#include "cryptonote_config.h" -#include "include_base_utils.h" #include "file_io_utils.h" #include "net/http_client.h" #include "download.h" @@ -182,8 +179,8 @@ namespace tools lock.unlock(); - bool ssl = u_c.schema == "https"; - uint16_t port = u_c.port ? u_c.port : ssl ? 443 : 80; + epee::net_utils::ssl_support_t ssl = u_c.schema == "https" ? epee::net_utils::ssl_support_t::e_ssl_support_enabled : epee::net_utils::ssl_support_t::e_ssl_support_disabled; + uint16_t port = u_c.port ? u_c.port : ssl == epee::net_utils::ssl_support_t::e_ssl_support_enabled ? 443 : 80; MDEBUG("Connecting to " << u_c.host << ":" << port); client.set_server(u_c.host, std::to_string(port), boost::none, ssl); if (!client.connect(std::chrono::seconds(30))) diff --git a/src/common/download.h b/src/common/download.h index 3097394bcb8..f8656a59c43 100644 --- a/src/common/download.h +++ b/src/common/download.h @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2018, The Monero Project +// Copyright (c) 2017-2019, The Monero Project // // All rights reserved. // diff --git a/src/common/error.cpp b/src/common/error.cpp new file mode 100644 index 00000000000..e091e447813 --- /dev/null +++ b/src/common/error.cpp @@ -0,0 +1,75 @@ +// Copyright (c) 2018, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#include "error.h" + +#include + +namespace +{ + struct category final : std::error_category + { + virtual const char* name() const noexcept override final + { + return "common_category()"; + } + + virtual std::string message(int value) const override final + { + switch (common_error(value)) + { + case common_error::kInvalidArgument: + return make_error_code(std::errc::invalid_argument).message(); + case common_error::kInvalidErrorCode: + return "expect was given an error value of zero"; + default: + break; + } + return "Unknown basic_category() value"; + } + + virtual std::error_condition default_error_condition(int value) const noexcept override final + { + // maps specific errors to generic `std::errc` cases. + switch (common_error(value)) + { + case common_error::kInvalidArgument: + case common_error::kInvalidErrorCode: + return std::errc::invalid_argument; + default: + break; + } + return std::error_condition{value, *this}; + } + }; +} + +std::error_category const& common_category() noexcept +{ + static const category instance{}; + return instance; +} + diff --git a/src/common/error.h b/src/common/error.h new file mode 100644 index 00000000000..6fef3eb4b3c --- /dev/null +++ b/src/common/error.h @@ -0,0 +1,52 @@ +// Copyright (c) 2018, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#pragma once + +#include +#include + +enum class common_error : int +{ + // 0 is reserved for no error, as per expect + kInvalidArgument = 1, //!< A function argument is invalid + kInvalidErrorCode //!< Default `std::error_code` given to `expect` +}; + +std::error_category const& common_category() noexcept; + +inline std::error_code make_error_code(::common_error value) noexcept +{ + return std::error_code{int(value), common_category()}; +} + +namespace std +{ + template<> + struct is_error_code_enum<::common_error> + : true_type + {}; +} diff --git a/src/common/expect.cpp b/src/common/expect.cpp new file mode 100644 index 00000000000..c86e23e9514 --- /dev/null +++ b/src/common/expect.cpp @@ -0,0 +1,70 @@ +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "expect.h" + +#include +#include + +namespace detail +{ + namespace + { + std::string generate_error(const char* msg, const char* file, unsigned line) + { + std::string error_msg{}; + if (msg) + { + error_msg.append(msg); + if (file) + error_msg.append(" ("); + } + if (file) + { + error_msg.append("thrown at "); + + // remove path, get just filename + extension + char buff[256] = {0}; + el::base::utils::File::buildBaseFilename(file, buff, sizeof(buff) - 1); + error_msg.append(buff); + + error_msg.push_back(':'); + error_msg.append(std::to_string(line)); + } + if (msg && file) + error_msg.push_back(')'); + return error_msg; + } + } + + void expect::throw_(std::error_code ec, const char* msg, const char* file, unsigned line) + { + if (msg || file) + throw std::system_error{ec, generate_error(msg, file, line)}; + throw std::system_error{ec}; + } +} // detail diff --git a/src/common/expect.h b/src/common/expect.h new file mode 100644 index 00000000000..72e4060a7a9 --- /dev/null +++ b/src/common/expect.h @@ -0,0 +1,449 @@ +// Copyright (c) 2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +#include +#include +#include +#include + +#include "common/error.h" + +//! If precondition fails, return `::error::kInvalidArgument` in current scope. +#define MONERO_PRECOND(...) \ + do \ + { \ + if (!( __VA_ARGS__ )) \ + return {::common_error::kInvalidArgument}; \ + } while (0) + +//! Check `expect` and return errors in current scope. +#define MONERO_CHECK(...) \ + do \ + { \ + const ::expect result = __VA_ARGS__ ; \ + if (!result) \ + return result.error(); \ + } while (0) + +/*! Get `T` from `expect` by `std::move` as-if by function call. + `expect` returns nothing. + + \throw std::system_error with `expect::error()`, filename and line + number when `expect::has_error() == true`.*/ +#define MONERO_UNWRAP(...) \ + ::detail::expect::unwrap( __VA_ARGS__ , nullptr, __FILE__ , __LINE__ ) + +/* \throw std::system_error with `code` and `msg` as part of the details. The +filename and line number will automatically be injected into the explanation +string. `code` can be any enum convertible to `std::error_code`. */ +#define MONERO_THROW(code, msg) \ + ::detail::expect::throw_( code , msg , __FILE__ , __LINE__ ) + + +template class expect; + +namespace detail +{ + // Shortens the characters in the places that `enable_if` is used below. + template + using enable_if = typename std::enable_if::type; + + struct expect + { + //! \throw std::system_error with `ec`, optional `msg` and/or optional `file` + `line`. + static void throw_(std::error_code ec, const char* msg, const char* file, unsigned line); + + //! If `result.has_error()` call `throw_`. Otherwise, \return `*result` by move. + template + static T unwrap(::expect&& result, const char* error_msg, const char* file, unsigned line) + { + if (!result) + throw_(result.error(), error_msg, file, line); + return std::move(*result); + } + + //! If `result.has_error()` call `throw_`. + static void unwrap(::expect&& result, const char* error_msg, const char* file, unsigned line); + }; +} + +/*! + `expect` is a value or error implementation, similar to Rust std::result + or various C++ proposals (boost::expected, boost::outcome). This + implementation currently has a strict error type, `std::error_code`, and a + templated value type `T`. `expect` is implicitly convertible from `T` + or `std::error_code`, and one `expect` object type is implicitly + convertible to another `expect` object iff the destination value type + can be implicitly constructed from the source value type (i.e. + `struct U { ... U(T src) { ...} ... };`). + + `operator==` and `operator!=` are the only comparison operators provided; + comparison between different value types is allowed provided the two values + types have a `operator==` defined between them (i.e. + `assert(expect{100} == expect{100});`). Comparisons can also be + done against `std::error_code` objects or error code enums directly (i.e. + `assert(expect{make_error_code(common_error::kInvalidArgument)} == error::kInvalidArgument)`). + Comparison of default constructed `std::error_code` will always fail. + "Generic" comparisons can be done with `std::error_condition` via the `matches` + method only (i.e. + `assert(expect{make_error_code{common_error::kInvalidErrorCode}.matches(std::errc::invalid_argument))`), + `operator==` and `operator!=` will not work with `std::errc` or + `std::error_condition`. A comparison with `matches` is more expensive + because an equivalency between error categories is computed, but is + recommended when an error can be one of several categories (this is going + to be the case in nearly every situation when calling a function from + another C++ struct/class). + + `expect` is a special case with no stored value. It is used by + functions that can fail, but otherwise would return `void`. It is useful + for consistency; all macros, standalone functions, and comparison operators + work with `expect`. + + \note See `src/common/error.h` for creating a custom error enum. + */ +template +class expect +{ + static_assert(std::is_nothrow_destructible(), "T must have a nothrow destructor"); + + template + static constexpr bool is_convertible() noexcept + { + return std::is_constructible() && + std::is_convertible(); + } + + // MEMBERS + std::error_code code_; + typename std::aligned_storage::type storage_; + // MEMBERS + + T& get() noexcept + { + assert(has_value()); + return *reinterpret_cast(std::addressof(storage_)); + } + + T const& get() const noexcept + { + assert(has_value()); + return *reinterpret_cast(std::addressof(storage_)); + } + + template + void store(U&& value) noexcept(std::is_nothrow_constructible()) + { + new (std::addressof(storage_)) T{std::forward(value)}; + code_ = std::error_code{}; + } + + void maybe_throw() const + { + if (has_error()) + ::detail::expect::throw_(error(), nullptr, nullptr, 0); + } + +public: + using value_type = T; + using error_type = std::error_code; + + expect() = delete; + + /*! Store an error, `code`, in the `expect` object. If `code` creates a + `std::error_code` object whose `.value() == 0`, then `error()` will be set + to `::common_error::kInvalidErrorCode`. */ + expect(std::error_code const& code) noexcept + : code_(code), storage_() + { + if (!has_error()) + code_ = ::common_error::kInvalidErrorCode; + } + + //! Store a value, `val`, in the `expect` object. + expect(T val) noexcept(std::is_nothrow_move_constructible()) + : code_(), storage_() + { + store(std::move(val)); + } + + expect(expect const& src) noexcept(std::is_nothrow_copy_constructible()) + : code_(src.error()), storage_() + { + if (src.has_value()) + store(src.get()); + } + + //! Copy conversion from `U` to `T`. + template()>> + expect(expect const& src) noexcept(std::is_nothrow_constructible()) + : code_(src.error()), storage_() + { + if (src.has_value()) + store(*src); + } + + expect(expect&& src) noexcept(std::is_nothrow_move_constructible()) + : code_(src.error()), storage_() + { + if (src.has_value()) + store(std::move(src.get())); + } + + //! Move conversion from `U` to `T`. + template()>> + expect(expect&& src) noexcept(std::is_nothrow_constructible()) + : code_(src.error()), storage_() + { + if (src.has_value()) + store(std::move(*src)); + } + + ~expect() noexcept + { + if (has_value()) + get().~T(); + } + + expect& operator=(expect const& src) noexcept(std::is_nothrow_copy_constructible() && std::is_nothrow_copy_assignable()) + { + if (this != std::addressof(src)) + { + if (has_value() && src.has_value()) + get() = src.get(); + else if (has_value()) + get().~T(); + else if (src.has_value()) + store(src.get()); + code_ = src.error(); + } + return *this; + } + + /*! Move `src` into `this`. If `src.has_value() && addressof(src) != this` + then `src.value() will be in a "moved from state". */ + expect& operator=(expect&& src) noexcept(std::is_nothrow_move_constructible() && std::is_nothrow_move_assignable()) + { + if (this != std::addressof(src)) + { + if (has_value() && src.has_value()) + get() = std::move(src.get()); + else if (has_value()) + get().~T(); + else if (src.has_value()) + store(std::move(src.get())); + code_ = src.error(); + } + return *this; + } + + //! \return True if `this` is storing a value instead of an error. + explicit operator bool() const noexcept { return has_value(); } + + //! \return True if `this` is storing an error instead of a value. + bool has_error() const noexcept { return bool(code_); } + + //! \return True if `this` is storing a value instead of an error. + bool has_value() const noexcept { return !has_error(); } + + //! \return Error - always safe to call. Empty when `!has_error()`. + std::error_code error() const noexcept { return code_; } + + //! \return Value if `has_value()` otherwise \throw `std::system_error{error()}`. + T& value() & + { + maybe_throw(); + return get(); + } + + //! \return Value if `has_value()` otherwise \throw `std::system_error{error()}`. + T const& value() const & + { + maybe_throw(); + return get(); + } + + /*! Same as other overloads, but expressions such as `foo(bar().value())` + will automatically perform moves with no copies. */ + T&& value() && + { + maybe_throw(); + return std::move(get()); + } + + //! \return Value, \pre `has_value()`. + T* operator->() noexcept { return std::addressof(get()); } + //! \return Value, \pre `has_value()`. + T const* operator->() const noexcept { return std::addressof(get()); } + //! \return Value, \pre `has_value()`. + T& operator*() noexcept { return get(); } + //! \return Value, \pre `has_value()`. + T const& operator*() const noexcept { return get(); } + + /*! + \note This function is `noexcept` when `U == T` is `noexcept`. + \return True if `has_value() == rhs.has_value()` and if values or errors are equal. + */ + template + bool equal(expect const& rhs) const noexcept(noexcept(*std::declval>() == *rhs)) + { + return has_value() && rhs.has_value() ? + get() == *rhs : error() == rhs.error(); + } + + //! \return False if `has_value()`, otherwise `error() == rhs`. + bool equal(std::error_code const& rhs) const noexcept + { + return has_error() && error() == rhs; + } + + /*! + \note This function is `noexcept` when `U == T` is `noexcept`. + \return False if `has_error()`, otherwise `value() == rhs`. + */ + template::value>> + bool equal(U const& rhs) const noexcept(noexcept(*std::declval>() == rhs)) + { + return has_value() && get() == rhs; + } + + //! \return False if `has_value()`, otherwise `error() == rhs`. + bool matches(std::error_condition const& rhs) const noexcept + { + return has_error() && error() == rhs; + } +}; + +template<> +class expect +{ + std::error_code code_; + +public: + using value_type = void; + using error_type = std::error_code; + + //! Create a successful object. + expect() noexcept + : code_() + {} + + expect(std::error_code const& code) noexcept + : code_(code) + { + if (!has_error()) + code_ = ::common_error::kInvalidErrorCode; + } + + expect(expect const&) = default; + ~expect() = default; + expect& operator=(expect const&) = default; + + //! \return True if `this` is storing a value instead of an error. + explicit operator bool() const noexcept { return !has_error(); } + + //! \return True if `this` is storing an error instead of a value. + bool has_error() const noexcept { return bool(code_); } + + //! \return Error - alway + std::error_code error() const noexcept { return code_; } + + //! \return `error() == rhs.error()`. + bool equal(expect const& rhs) const noexcept + { + return error() == rhs.error(); + } + + //! \return `has_error() && error() == rhs`. + bool equal(std::error_code const& rhs) const noexcept + { + return has_error() && error() == rhs; + } + + //! \return False if `has_value()`, otherwise `error() == rhs`. + bool matches(std::error_condition const& rhs) const noexcept + { + return has_error() && error() == rhs; + } +}; + +//! \return An `expect` object with `!has_error()`. +inline expect success() noexcept { return expect{}; } + +template +inline +bool operator==(expect const& lhs, expect const& rhs) noexcept(noexcept(lhs.equal(rhs))) +{ + return lhs.equal(rhs); +} + +template +inline +bool operator==(expect const& lhs, U const& rhs) noexcept(noexcept(lhs.equal(rhs))) +{ + return lhs.equal(rhs); +} + +template +inline +bool operator==(T const& lhs, expect const& rhs) noexcept(noexcept(rhs.equal(lhs))) +{ + return rhs.equal(lhs); +} + +template +inline +bool operator!=(expect const& lhs, expect const& rhs) noexcept(noexcept(lhs.equal(rhs))) +{ + return !lhs.equal(rhs); +} + +template +inline +bool operator!=(expect const& lhs, U const& rhs) noexcept(noexcept(lhs.equal(rhs))) +{ + return !lhs.equal(rhs); +} + +template +inline +bool operator!=(T const& lhs, expect const& rhs) noexcept(noexcept(rhs.equal(lhs))) +{ + return !rhs.equal(lhs); +} + +namespace detail +{ + inline void expect::unwrap(::expect&& result, const char* error_msg, const char* file, unsigned line) + { + if (!result) + throw_(result.error(), error_msg, file, line); + } +} + diff --git a/src/common/http_connection.h b/src/common/http_connection.h index 9fc6be261d4..6b429480226 100644 --- a/src/common/http_connection.h +++ b/src/common/http_connection.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -55,7 +55,8 @@ class t_http_connection { { if (m_ok) { - mp_http_client->disconnect(); + try { mp_http_client->disconnect(); } + catch (...) { /* do not propagate through dtor */ } } } diff --git a/src/common/i18n.cpp b/src/common/i18n.cpp index 4a89876fbc2..9ac3472637f 100644 --- a/src/common/i18n.cpp +++ b/src/common/i18n.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -31,15 +31,15 @@ #include #include #include -#include "include_base_utils.h" #include "file_io_utils.h" -#include "common/util.h" #include "common/i18n.h" #include "translation_files.h" #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "i18n" +#define MAX_LANGUAGE_SIZE 16 + static const unsigned char qm_magic[16] = {0x3c, 0xb8, 0x64, 0x18, 0xca, 0xef, 0x9c, 0x95, 0xcd, 0x21, 0x1c, 0xbf, 0x60, 0xa1, 0xbd, 0xdd}; static std::map i18n_entries; @@ -64,7 +64,19 @@ std::string i18n_get_language() std::string language = e; language = language.substr(0, language.find(".")); + language = language.substr(0, language.find("@")); + + // check valid values + for (char c: language) + if (!strchr("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ_-.@", c)) + return "en"; + std::transform(language.begin(), language.end(), language.begin(), tolower); + if (language.size() > MAX_LANGUAGE_SIZE) + { + i18n_log("Language from LANG/LC_ALL suspiciously long, defaulting to en"); + return "en"; + } return language; } diff --git a/src/common/i18n.h b/src/common/i18n.h index d21d0027501..82a07410d49 100644 --- a/src/common/i18n.h +++ b/src/common/i18n.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/common/json_util.h b/src/common/json_util.h index 661022a6f0b..96f4b90e664 100644 --- a/src/common/json_util.h +++ b/src/common/json_util.h @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2018, The Monero Project +// Copyright (c) 2016-2019, The Monero Project // // All rights reserved. // @@ -29,14 +29,14 @@ #pragma once #define GET_FIELD_FROM_JSON_RETURN_ON_ERROR(json, name, type, jtype, mandatory, def) \ - type field_##name = def; \ + type field_##name = static_cast(def); \ bool field_##name##_found = false; \ (void)field_##name##_found; \ do if (json.HasMember(#name)) \ { \ if (json[#name].Is##jtype()) \ { \ - field_##name = json[#name].Get##jtype(); \ + field_##name = static_cast(json[#name].Get##jtype()); \ field_##name##_found = true; \ } \ else \ diff --git a/src/common/notify.cpp b/src/common/notify.cpp new file mode 100644 index 00000000000..e2df5096dfe --- /dev/null +++ b/src/common/notify.cpp @@ -0,0 +1,83 @@ +// Copyright (c) 2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include +#include +#include "misc_log_ex.h" +#include "file_io_utils.h" +#include "spawn.h" +#include "notify.h" + +#undef MONERO_DEFAULT_LOG_CATEGORY +#define MONERO_DEFAULT_LOG_CATEGORY "notify" + +namespace tools +{ + +/* + TODO: + - Improve tokenization to handle paths containing whitespaces, quotes, etc. + - Windows unicode support (implies implementing unicode command line parsing code) +*/ +Notify::Notify(const char *spec) +{ + CHECK_AND_ASSERT_THROW_MES(spec, "Null spec"); + + boost::split(args, spec, boost::is_any_of(" \t"), boost::token_compress_on); + CHECK_AND_ASSERT_THROW_MES(args.size() > 0, "Failed to parse spec"); + if (strchr(spec, '\'') || strchr(spec, '\"') || strchr(spec, '\\')) + MWARNING("A notification spec contains a quote or backslash: note that these are handled verbatim, which may not be the intent"); + filename = args[0]; + CHECK_AND_ASSERT_THROW_MES(epee::file_io_utils::is_file_exist(filename), "File not found: " << filename); +} + +static void replace(std::vector &v, const char *tag, const char *s) +{ + for (std::string &str: v) + boost::replace_all(str, tag, s); +} + +int Notify::notify(const char *tag, const char *s, ...) +{ + std::vector margs = args; + + replace(margs, tag, s); + + va_list ap; + va_start(ap, s); + while ((tag = va_arg(ap, const char*))) + { + s = va_arg(ap, const char*); + replace(margs, tag, s); + } + va_end(ap); + + return tools::spawn(filename.c_str(), margs, false); +} + +} diff --git a/src/common/notify.h b/src/common/notify.h new file mode 100644 index 00000000000..f813e8deff8 --- /dev/null +++ b/src/common/notify.h @@ -0,0 +1,49 @@ +// Copyright (c) 2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +#include +#include + +namespace tools +{ + +class Notify +{ +public: + Notify(const char *spec); + + int notify(const char *tag, const char *s, ...); + +private: + std::string filename; + std::vector args; +}; + +} diff --git a/src/common/password.cpp b/src/common/password.cpp index 9336a14fc83..33e1f48fdbd 100644 --- a/src/common/password.cpp +++ b/src/common/password.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -31,7 +31,6 @@ #include "password.h" #include -#include #include #if defined(_WIN32) @@ -42,8 +41,6 @@ #include #endif -#include "memwipe.h" - #define EOT 0x4 namespace @@ -54,45 +51,57 @@ namespace return 0 != _isatty(_fileno(stdin)); } - bool read_from_tty(epee::wipeable_string& pass) + bool read_from_tty(epee::wipeable_string& pass, bool hide_input) { - static constexpr const char BACKSPACE = 8; - HANDLE h_cin = ::GetStdHandle(STD_INPUT_HANDLE); DWORD mode_old; ::GetConsoleMode(h_cin, &mode_old); - DWORD mode_new = mode_old & ~(ENABLE_ECHO_INPUT | ENABLE_LINE_INPUT); + DWORD mode_new = mode_old & ~((hide_input ? ENABLE_ECHO_INPUT : 0) | ENABLE_LINE_INPUT); ::SetConsoleMode(h_cin, mode_new); bool r = true; pass.reserve(tools::password_container::max_password_size); + std::vector chlen; + chlen.reserve(tools::password_container::max_password_size); while (pass.size() < tools::password_container::max_password_size) { DWORD read; - char ch; - r = (TRUE == ::ReadConsoleA(h_cin, &ch, 1, &read, NULL)); + wchar_t ucs2_ch; + r = (TRUE == ::ReadConsoleW(h_cin, &ucs2_ch, 1, &read, NULL)); r &= (1 == read); + if (!r) { break; } - else if (ch == '\n' || ch == '\r') + else if (ucs2_ch == L'\r') { std::cout << std::endl; break; } - else if (ch == BACKSPACE) + else if (ucs2_ch == L'\b') { if (!pass.empty()) { - pass.pop_back(); + int len = chlen.back(); + chlen.pop_back(); + while(len-- > 0) + pass.pop_back(); } + continue; } - else - { - pass.push_back(ch); - } + + char utf8_ch[8] = {0}; + int len; + if((len = WideCharToMultiByte(CP_UTF8, 0, &ucs2_ch, 1, utf8_ch, sizeof(utf8_ch), NULL, NULL)) <= 0) + break; + + if(pass.size() + len >= tools::password_container::max_password_size) + break; + + chlen.push_back(len); + pass += utf8_ch; } ::SetConsoleMode(h_cin, mode_old); @@ -107,14 +116,14 @@ namespace return 0 != isatty(fileno(stdin)); } - int getch() noexcept + int getch(bool hide_input) noexcept { struct termios tty_old; tcgetattr(STDIN_FILENO, &tty_old); struct termios tty_new; tty_new = tty_old; - tty_new.c_lflag &= ~(ICANON | ECHO); + tty_new.c_lflag &= ~(ICANON | (hide_input ? ECHO : 0)); tcsetattr(STDIN_FILENO, TCSANOW, &tty_new); int ch = getchar(); @@ -124,14 +133,14 @@ namespace return ch; } - bool read_from_tty(epee::wipeable_string& aPass) + bool read_from_tty(epee::wipeable_string& aPass, bool hide_input) { static constexpr const char BACKSPACE = 127; aPass.reserve(tools::password_container::max_password_size); while (aPass.size() < tools::password_container::max_password_size) { - int ch = getch(); + int ch = getch(hide_input); if (EOF == ch || ch == EOT) { return false; @@ -146,6 +155,13 @@ namespace if (!aPass.empty()) { aPass.pop_back(); + if (!hide_input) + std::cout << "\b\b\b \b\b\b" << std::flush; + } + else + { + if (!hide_input) + std::cout << "\b\b \b\b" << std::flush; } } else @@ -159,18 +175,18 @@ namespace #endif // end !WIN32 - bool read_from_tty(const bool verify, const char *message, epee::wipeable_string& pass1, epee::wipeable_string& pass2) + bool read_from_tty(const bool verify, const char *message, bool hide_input, epee::wipeable_string& pass1, epee::wipeable_string& pass2) { while (true) { if (message) - std::cout << message <<": "; - if (!read_from_tty(pass1)) + std::cout << message <<": " << std::flush; + if (!read_from_tty(pass1, hide_input)) return false; if (verify) { std::cout << "Confirm password: "; - if (!read_from_tty(pass2)) + if (!read_from_tty(pass2, hide_input)) return false; if(pass1!=pass2) { @@ -221,6 +237,10 @@ namespace tools : m_password(std::move(password)) { } + password_container::password_container(const epee::wipeable_string& password) noexcept + : m_password(password) + { + } password_container::~password_container() noexcept { @@ -229,12 +249,12 @@ namespace tools std::atomic password_container::is_prompting(false); - boost::optional password_container::prompt(const bool verify, const char *message) + boost::optional password_container::prompt(const bool verify, const char *message, bool hide_input) { is_prompting = true; password_container pass1{}; password_container pass2{}; - if (is_cin_tty() ? read_from_tty(verify, message, pass1.m_password, pass2.m_password) : read_from_file(pass1.m_password)) + if (is_cin_tty() ? read_from_tty(verify, message, hide_input, pass1.m_password, pass2.m_password) : read_from_file(pass1.m_password)) { is_prompting = false; return {std::move(pass1)}; diff --git a/src/common/password.h b/src/common/password.h index 61937b93a75..2837c70f378 100644 --- a/src/common/password.h +++ b/src/common/password.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -47,9 +47,10 @@ namespace tools //! `password` is used as password password_container(std::string&& password) noexcept; + password_container(const epee::wipeable_string& password) noexcept; //! \return A password from stdin TTY prompt or `std::cin` pipe. - static boost::optional prompt(bool verify, const char *mesage = "Password"); + static boost::optional prompt(bool verify, const char *mesage = "Password", bool hide_input = true); static std::atomic is_prompting; password_container(const password_container&) = delete; diff --git a/src/common/perf_timer.cpp b/src/common/perf_timer.cpp index 16abdfd991c..189eb85eb72 100644 --- a/src/common/perf_timer.cpp +++ b/src/common/perf_timer.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2018, The Monero Project +// Copyright (c) 2016-2019, The Monero Project // // All rights reserved. // @@ -33,7 +33,14 @@ #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "perf" -namespace +#define PERF_LOG_ALWAYS(level, cat, x) \ + el::base::Writer(level, __FILE__, __LINE__, ELPP_FUNC, el::base::DispatchAction::FileOnlyLog).construct(cat) << x +#define PERF_LOG(level, cat, x) \ + do { \ + if (ELPP->vRegistry()->allowed(level, cat)) PERF_LOG_ALWAYS(level, cat, x); \ + } while(0) + +namespace tools { uint64_t get_tick_count() { @@ -81,36 +88,49 @@ namespace namespace tools { -el::Level performance_timer_log_level = el::Level::Debug; +el::Level performance_timer_log_level = el::Level::Info; -static __thread std::vector *performance_timers = NULL; +static __thread std::vector *performance_timers = NULL; void set_performance_timer_log_level(el::Level level) { if (level != el::Level::Debug && level != el::Level::Trace && level != el::Level::Info && level != el::Level::Warning && level != el::Level::Error && level != el::Level::Fatal) { - MERROR("Wrong log level: " << el::LevelHelper::convertToString(level) << ", using Debug"); - level = el::Level::Debug; + MERROR("Wrong log level: " << el::LevelHelper::convertToString(level) << ", using Info"); + level = el::Level::Info; } performance_timer_log_level = level; } -PerformanceTimer::PerformanceTimer(const std::string &s, uint64_t unit, el::Level l): name(s), unit(unit), level(l), started(false), paused(false) +PerformanceTimer::PerformanceTimer(bool paused): started(true), paused(paused) +{ + if (paused) + ticks = 0; + else + ticks = get_tick_count(); +} + +LoggingPerformanceTimer::LoggingPerformanceTimer(const std::string &s, const std::string &cat, uint64_t unit, el::Level l): PerformanceTimer(), name(s), cat(cat), unit(unit), level(l) { - ticks = get_tick_count(); + const bool log = ELPP->vRegistry()->allowed(level, cat.c_str()); if (!performance_timers) { - MLOG(level, "PERF ----------"); - performance_timers = new std::vector(); + if (log) + PERF_LOG_ALWAYS(level, cat.c_str(), "PERF ----------"); + performance_timers = new std::vector(); + performance_timers->reserve(16); // how deep before realloc } else { - PerformanceTimer *pt = performance_timers->back(); + LoggingPerformanceTimer *pt = performance_timers->back(); if (!pt->started && !pt->paused) { - size_t size = 0; for (const auto *tmp: *performance_timers) if (!tmp->paused) ++size; - MLOG(pt->level, "PERF " << std::string((size-1) * 2, ' ') << " " << pt->name); + if (log) + { + size_t size = 0; for (const auto *tmp: *performance_timers) if (!tmp->paused) ++size; + PERF_LOG_ALWAYS(pt->level, cat.c_str(), "PERF " << std::string((size-1) * 2, ' ') << " " << pt->name); + } pt->started = true; } } @@ -119,13 +139,22 @@ PerformanceTimer::PerformanceTimer(const std::string &s, uint64_t unit, el::Leve PerformanceTimer::~PerformanceTimer() { - performance_timers->pop_back(); if (!paused) ticks = get_tick_count() - ticks; - char s[12]; - snprintf(s, sizeof(s), "%8llu ", (unsigned long long)(ticks_to_ns(ticks) / (1000000000 / unit))); - size_t size = 0; for (const auto *tmp: *performance_timers) if (!tmp->paused || tmp==this) ++size; - MLOG(level, "PERF " << s << std::string(size * 2, ' ') << " " << name); +} + +LoggingPerformanceTimer::~LoggingPerformanceTimer() +{ + pause(); + performance_timers->pop_back(); + const bool log = ELPP->vRegistry()->allowed(level, cat.c_str()); + if (log) + { + char s[12]; + snprintf(s, sizeof(s), "%8llu ", (unsigned long long)(ticks_to_ns(ticks) / (1000000000 / unit))); + size_t size = 0; for (const auto *tmp: *performance_timers) if (!tmp->paused || tmp==this) ++size; + PERF_LOG_ALWAYS(level, cat.c_str(), "PERF " << s << std::string(size * 2, ' ') << " " << name); + } if (performance_timers->empty()) { delete performance_timers; @@ -149,4 +178,20 @@ void PerformanceTimer::resume() paused = false; } +void PerformanceTimer::reset() +{ + if (paused) + ticks = 0; + else + ticks = get_tick_count(); +} + +uint64_t PerformanceTimer::value() const +{ + uint64_t v = ticks; + if (!paused) + v = get_tick_count() - v; + return ticks_to_ns(v); +} + } diff --git a/src/common/perf_timer.h b/src/common/perf_timer.h index 0e910caf9f2..71739162379 100644 --- a/src/common/perf_timer.h +++ b/src/common/perf_timer.h @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2018, The Monero Project +// Copyright (c) 2016-2019, The Monero Project // // All rights reserved. // @@ -33,9 +33,6 @@ #include #include "misc_log_ex.h" -#undef MONERO_DEFAULT_LOG_CATEGORY -#define MONERO_DEFAULT_LOG_CATEGORY "perf" - namespace tools { @@ -43,31 +40,48 @@ class PerformanceTimer; extern el::Level performance_timer_log_level; +uint64_t get_tick_count(); +uint64_t get_ticks_per_ns(); +uint64_t ticks_to_ns(uint64_t ticks); + class PerformanceTimer { public: - PerformanceTimer(const std::string &s, uint64_t unit, el::Level l = el::Level::Debug); + PerformanceTimer(bool paused = false); ~PerformanceTimer(); void pause(); void resume(); + void reset(); + uint64_t value() const; + operator uint64_t() const { return value(); } + +protected: + uint64_t ticks; + bool started; + bool paused; +}; + +class LoggingPerformanceTimer: public PerformanceTimer +{ +public: + LoggingPerformanceTimer(const std::string &s, const std::string &cat, uint64_t unit, el::Level l = el::Level::Info); + ~LoggingPerformanceTimer(); private: std::string name; + std::string cat; uint64_t unit; el::Level level; - uint64_t ticks; - bool started; - bool paused; }; void set_performance_timer_log_level(el::Level level); -#define PERF_TIMER_UNIT(name, unit) tools::PerformanceTimer pt_##name(#name, unit, tools::performance_timer_log_level) -#define PERF_TIMER_UNIT_L(name, unit, l) tools::PerformanceTimer pt_##name(#name, unit, l) -#define PERF_TIMER(name) PERF_TIMER_UNIT(name, 1000) -#define PERF_TIMER_L(name, l) PERF_TIMER_UNIT_L(name, 1000, l) -#define PERF_TIMER_START_UNIT(name, unit) std::unique_ptr pt_##name(new tools::PerformanceTimer(#name, unit, el::Level::Info)) -#define PERF_TIMER_START(name) PERF_TIMER_START_UNIT(name, 1000) +#define PERF_TIMER_UNIT(name, unit) tools::LoggingPerformanceTimer pt_##name(#name, "perf." MONERO_DEFAULT_LOG_CATEGORY, unit, tools::performance_timer_log_level) +#define PERF_TIMER_UNIT_L(name, unit, l) tools::LoggingPerformanceTimer pt_##name(#name, "perf." MONERO_DEFAULT_LOG_CATEGORY, unit, l) +#define PERF_TIMER(name) PERF_TIMER_UNIT(name, 1000000) +#define PERF_TIMER_L(name, l) PERF_TIMER_UNIT_L(name, 1000000, l) +#define PERF_TIMER_START_UNIT(name, unit) std::unique_ptr pt_##name(new tools::LoggingPerformanceTimer(#name, "perf." MONERO_DEFAULT_LOG_CATEGORY, unit, el::Level::Info)) +#define PERF_TIMER_START(name) PERF_TIMER_START_UNIT(name, 1000000) #define PERF_TIMER_STOP(name) do { pt_##name.reset(NULL); } while(0) #define PERF_TIMER_PAUSE(name) pt_##name->pause() #define PERF_TIMER_RESUME(name) pt_##name->resume() diff --git a/src/common/pod-class.h b/src/common/pod-class.h index 5f6709eefb2..200647590f6 100644 --- a/src/common/pod-class.h +++ b/src/common/pod-class.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/common/pruning.cpp b/src/common/pruning.cpp new file mode 100644 index 00000000000..442b24e4ef5 --- /dev/null +++ b/src/common/pruning.cpp @@ -0,0 +1,116 @@ +// Copyright (c) 2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "cryptonote_config.h" +#include "misc_log_ex.h" +#include "crypto/crypto.h" +#include "pruning.h" + +namespace tools +{ + +uint32_t make_pruning_seed(uint32_t stripe, uint32_t log_stripes) +{ + CHECK_AND_ASSERT_THROW_MES(log_stripes <= PRUNING_SEED_LOG_STRIPES_MASK, "log_stripes out of range"); + CHECK_AND_ASSERT_THROW_MES(stripe > 0 && stripe <= (1ul << log_stripes), "stripe out of range"); + return (log_stripes << PRUNING_SEED_LOG_STRIPES_SHIFT) | ((stripe - 1) << PRUNING_SEED_STRIPE_SHIFT); +} + +bool has_unpruned_block(uint64_t block_height, uint64_t blockchain_height, uint32_t pruning_seed) +{ + const uint32_t stripe = get_pruning_stripe(pruning_seed); + if (stripe == 0) + return true; + const uint32_t log_stripes = get_pruning_log_stripes(pruning_seed); + uint32_t block_stripe = get_pruning_stripe(block_height, blockchain_height, log_stripes); + return block_stripe == 0 || block_stripe == stripe; +} + +uint32_t get_pruning_stripe(uint64_t block_height, uint64_t blockchain_height, uint32_t log_stripes) +{ + if (block_height + CRYPTONOTE_PRUNING_TIP_BLOCKS >= blockchain_height) + return 0; + return ((block_height / CRYPTONOTE_PRUNING_STRIPE_SIZE) & (uint64_t)((1ul << log_stripes) - 1)) + 1; +} + +uint32_t get_pruning_seed(uint64_t block_height, uint64_t blockchain_height, uint32_t log_stripes) +{ + const uint32_t stripe = get_pruning_stripe(block_height, blockchain_height, log_stripes); + if (stripe == 0) + return 0; + return make_pruning_seed(stripe, log_stripes); +} + +uint64_t get_next_unpruned_block_height(uint64_t block_height, uint64_t blockchain_height, uint32_t pruning_seed) +{ + CHECK_AND_ASSERT_MES(block_height <= CRYPTONOTE_MAX_BLOCK_NUMBER+1, block_height, "block_height too large"); + CHECK_AND_ASSERT_MES(blockchain_height <= CRYPTONOTE_MAX_BLOCK_NUMBER+1, block_height, "blockchain_height too large"); + const uint32_t stripe = get_pruning_stripe(pruning_seed); + if (stripe == 0) + return block_height; + if (block_height + CRYPTONOTE_PRUNING_TIP_BLOCKS >= blockchain_height) + return block_height; + const uint32_t seed_log_stripes = get_pruning_log_stripes(pruning_seed); + const uint64_t log_stripes = seed_log_stripes ? seed_log_stripes : CRYPTONOTE_PRUNING_LOG_STRIPES; + const uint64_t mask = (1ul << log_stripes) - 1; + const uint32_t block_pruning_stripe = ((block_height / CRYPTONOTE_PRUNING_STRIPE_SIZE) & mask) + 1; + if (block_pruning_stripe == stripe) + return block_height; + const uint64_t cycles = ((block_height / CRYPTONOTE_PRUNING_STRIPE_SIZE) >> log_stripes); + const uint64_t cycle_start = cycles + ((stripe > block_pruning_stripe) ? 0 : 1); + const uint64_t h = cycle_start * (CRYPTONOTE_PRUNING_STRIPE_SIZE << log_stripes) + (stripe - 1) * CRYPTONOTE_PRUNING_STRIPE_SIZE; + if (h + CRYPTONOTE_PRUNING_TIP_BLOCKS > blockchain_height) + return blockchain_height < CRYPTONOTE_PRUNING_TIP_BLOCKS ? 0 : blockchain_height - CRYPTONOTE_PRUNING_TIP_BLOCKS; + CHECK_AND_ASSERT_MES(h >= block_height, block_height, "h < block_height, unexpected"); + return h; +} + +uint64_t get_next_pruned_block_height(uint64_t block_height, uint64_t blockchain_height, uint32_t pruning_seed) +{ + const uint32_t stripe = get_pruning_stripe(pruning_seed); + if (stripe == 0) + return blockchain_height; + if (block_height + CRYPTONOTE_PRUNING_TIP_BLOCKS >= blockchain_height) + return blockchain_height; + const uint32_t seed_log_stripes = get_pruning_log_stripes(pruning_seed); + const uint64_t log_stripes = seed_log_stripes ? seed_log_stripes : CRYPTONOTE_PRUNING_LOG_STRIPES; + const uint64_t mask = (1ul << log_stripes) - 1; + const uint32_t block_pruning_seed = ((block_height / CRYPTONOTE_PRUNING_STRIPE_SIZE) & mask) + 1; + if (block_pruning_seed != stripe) + return block_height; + const uint32_t next_stripe = 1 + (block_pruning_seed & mask); + return get_next_unpruned_block_height(block_height, blockchain_height, tools::make_pruning_seed(next_stripe, log_stripes)); +} + +uint32_t get_random_stripe() +{ + return 1 + crypto::rand() % (1ul << CRYPTONOTE_PRUNING_LOG_STRIPES); +} + +} + diff --git a/src/common/pruning.h b/src/common/pruning.h new file mode 100644 index 00000000000..3fac3c0fa30 --- /dev/null +++ b/src/common/pruning.h @@ -0,0 +1,52 @@ +// Copyright (c) 2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +#include + +namespace tools +{ + static constexpr uint32_t PRUNING_SEED_LOG_STRIPES_SHIFT = 7; + static constexpr uint32_t PRUNING_SEED_LOG_STRIPES_MASK = 0x7; + static constexpr uint32_t PRUNING_SEED_STRIPE_SHIFT = 0; + static constexpr uint32_t PRUNING_SEED_STRIPE_MASK = 0x7f; + + constexpr inline uint32_t get_pruning_log_stripes(uint32_t pruning_seed) { return (pruning_seed >> PRUNING_SEED_LOG_STRIPES_SHIFT) & PRUNING_SEED_LOG_STRIPES_MASK; } + inline uint32_t get_pruning_stripe(uint32_t pruning_seed) { if (pruning_seed == 0) return 0; return 1 + ((pruning_seed >> PRUNING_SEED_STRIPE_SHIFT) & PRUNING_SEED_STRIPE_MASK); } + + uint32_t make_pruning_seed(uint32_t stripe, uint32_t log_stripes); + + bool has_unpruned_block(uint64_t block_height, uint64_t blockchain_height, uint32_t pruning_seed); + uint32_t get_pruning_stripe(uint64_t block_height, uint64_t blockchain_height, uint32_t log_stripes); + uint32_t get_pruning_seed(uint64_t block_height, uint64_t blockchain_height, uint32_t log_stripes); + uint64_t get_next_unpruned_block_height(uint64_t block_height, uint64_t blockchain_height, uint32_t pruning_seed); + uint64_t get_next_pruned_block_height(uint64_t block_height, uint64_t blockchain_height, uint32_t pruning_seed); + uint32_t get_random_stripe(); +} + diff --git a/src/common/rpc_client.h b/src/common/rpc_client.h index 9665966ae36..dab3e562de5 100644 --- a/src/common/rpc_client.h +++ b/src/common/rpc_client.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -36,6 +36,7 @@ #include "storages/http_abstract_invoke.h" #include "net/http_auth.h" #include "net/http_client.h" +#include "net/net_ssl.h" #include "string_tools.h" namespace tools @@ -49,11 +50,12 @@ namespace tools uint32_t ip , uint16_t port , boost::optional user + , epee::net_utils::ssl_options_t ssl_options ) : m_http_client{} { m_http_client.set_server( - epee::string_tools::get_ip_string_from_int32(ip), std::to_string(port), std::move(user) + epee::string_tools::get_ip_string_from_int32(ip), std::to_string(port), std::move(user), std::move(ssl_options) ); } diff --git a/src/common/scoped_message_writer.h b/src/common/scoped_message_writer.h index d7517babb48..546377392c2 100644 --- a/src/common/scoped_message_writer.h +++ b/src/common/scoped_message_writer.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -73,7 +73,7 @@ class scoped_message_writer #if defined(_MSC_VER) , m_oss(std::move(rhs.m_oss)) #else - // GCC bug: http://gcc.gnu.org/bugzilla/show_bug.cgi?id=54316 + // GCC bug: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=54316 , m_oss(rhs.m_oss.str(), std::ios_base::out | std::ios_base::ate) #endif , m_color(std::move(rhs.m_color)) @@ -101,13 +101,13 @@ class scoped_message_writer MCLOG_FILE(m_log_level, "msgwriter", m_oss.str()); + PAUSE_READLINE(); if (epee::console_color_default == m_color) { std::cout << m_oss.str(); } else { - PAUSE_READLINE(); set_console_color(m_color, m_bright); std::cout << m_oss.str(); epee::reset_console_color(); diff --git a/src/common/sfinae_helpers.h b/src/common/sfinae_helpers.h index fa5052a2e5b..e9a98bb6399 100644 --- a/src/common/sfinae_helpers.h +++ b/src/common/sfinae_helpers.h @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2018, The Monero Project +// Copyright (c) 2016-2019, The Monero Project // // All rights reserved. // diff --git a/src/common/spawn.cpp b/src/common/spawn.cpp new file mode 100644 index 00000000000..9a7e75d4195 --- /dev/null +++ b/src/common/spawn.cpp @@ -0,0 +1,151 @@ +// Copyright (c) 2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include +#include +#include +#ifdef _WIN32 +#include +#include +#include +#else +#include +#include +#endif + +#include "misc_log_ex.h" +#include "util.h" +#include "spawn.h" + +#undef MONERO_DEFAULT_LOG_CATEGORY +#define MONERO_DEFAULT_LOG_CATEGORY "spawn" + +namespace tools +{ + +int spawn(const char *filename, const std::vector& args, bool wait) +{ +#ifdef _WIN32 + std::string joined = boost::algorithm::join(args, " "); + char *commandLine = !joined.empty() ? &joined[0] : nullptr; + STARTUPINFOA si = {}; + si.cb = sizeof(si); + PROCESS_INFORMATION pi; + if (!CreateProcessA(filename, commandLine, nullptr, nullptr, false, 0, nullptr, nullptr, &si, &pi)) + { + MERROR("CreateProcess failed. Error code " << GetLastError()); + return -1; + } + + BOOST_SCOPE_EXIT(&pi) + { + CloseHandle(pi.hThread); + CloseHandle(pi.hProcess); + } + BOOST_SCOPE_EXIT_END + + if (!wait) + { + return 0; + } + + DWORD result = WaitForSingleObject(pi.hProcess, INFINITE); + if (result != WAIT_OBJECT_0) + { + MERROR("WaitForSingleObject failed. Result " << result << ", error code " << GetLastError()); + return -1; + } + + DWORD exitCode; + if (!GetExitCodeProcess(pi.hProcess, &exitCode)) + { + MERROR("GetExitCodeProcess failed. Error code " << GetLastError()); + return -1; + } + + MINFO("Child exited with " << exitCode); + return static_cast(exitCode); +#else + std::vector argv(args.size() + 1); + for (size_t n = 0; n < args.size(); ++n) + argv[n] = (char*)args[n].c_str(); + argv[args.size()] = NULL; + + pid_t pid = fork(); + if (pid < 0) + { + MERROR("Error forking: " << strerror(errno)); + return -1; + } + + // child + if (pid == 0) + { + tools::closefrom(3); + close(0); + char *envp[] = {NULL}; + execve(filename, argv.data(), envp); + MERROR("Failed to execve: " << strerror(errno)); + return -1; + } + + // parent + if (pid > 0) + { + if (!wait) + { + signal(SIGCHLD, SIG_IGN); + return 0; + } + + while (1) + { + int wstatus = 0; + pid_t w = waitpid(pid, &wstatus, WUNTRACED | WCONTINUED); + if (w < 0) { + MERROR("Error waiting for child: " << strerror(errno)); + return -1; + } + if (WIFEXITED(wstatus)) + { + MINFO("Child exited with " << WEXITSTATUS(wstatus)); + return WEXITSTATUS(wstatus); + } + if (WIFSIGNALED(wstatus)) + { + MINFO("Child killed by " << WEXITSTATUS(wstatus)); + return WEXITSTATUS(wstatus); + } + } + } + MERROR("Secret passage found"); + return -1; +#endif +} + +} diff --git a/src/common/spawn.h b/src/common/spawn.h new file mode 100644 index 00000000000..c90a0f790b3 --- /dev/null +++ b/src/common/spawn.h @@ -0,0 +1,36 @@ +// Copyright (c) 2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +namespace tools +{ + +int spawn(const char *filename, const std::vector& args, bool wait); + +} diff --git a/src/common/stack_trace.cpp b/src/common/stack_trace.cpp index 9c2bf4b5379..8d4f8c6f18c 100644 --- a/src/common/stack_trace.cpp +++ b/src/common/stack_trace.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2018, The Monero Project +// Copyright (c) 2016-2019, The Monero Project // // All rights reserved. // @@ -49,9 +49,18 @@ #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "stacktrace" -#define ST_LOG(x) CINFO(el::base::Writer,el::base::DispatchAction::FileOnlyLog,MONERO_DEFAULT_LOG_CATEGORY) << x - -// from http://stackoverflow.com/questions/11665829/how-can-i-print-stack-trace-for-caught-exceptions-in-c-code-injection-in-c +#define ST_LOG(x) \ + do { \ + auto elpp = ELPP; \ + if (elpp) { \ + CINFO(el::base::Writer,el::base::DispatchAction::FileOnlyLog,MONERO_DEFAULT_LOG_CATEGORY) << x; \ + } \ + else { \ + std::cout << x << std::endl; \ + } \ + } while(0) + +// from https://stackoverflow.com/questions/11665829/how-can-i-print-stack-trace-for-caught-exceptions-in-c-code-injection-in-c // The decl of __cxa_throw in /usr/include/.../cxxabi.h uses // 'std::type_info *', but GCC's built-in protype uses 'void *'. diff --git a/src/common/stack_trace.h b/src/common/stack_trace.h index 272fb89ae14..ae657388599 100644 --- a/src/common/stack_trace.h +++ b/src/common/stack_trace.h @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2018, The Monero Project +// Copyright (c) 2016-2019, The Monero Project // // All rights reserved. // diff --git a/src/common/threadpool.cpp b/src/common/threadpool.cpp index 51e0715772d..2748c798c20 100644 --- a/src/common/threadpool.cpp +++ b/src/common/threadpool.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2018, The Monero Project +// Copyright (c) 2017-2019, The Monero Project // // All rights reserved. // @@ -28,70 +28,83 @@ #include "misc_log_ex.h" #include "common/threadpool.h" -#include -#include -#include - #include "cryptonote_config.h" #include "common/util.h" static __thread int depth = 0; +static __thread bool is_leaf = false; namespace tools { -threadpool::threadpool() : running(true), active(0) { +threadpool::threadpool(unsigned int max_threads) : running(true), active(0) { boost::thread::attributes attrs; attrs.set_stack_size(THREAD_STACK_SIZE); - max = tools::get_max_concurrency(); - size_t i = max; + max = max_threads ? max_threads : tools::get_max_concurrency(); + size_t i = max ? max - 1 : 0; while(i--) { - threads.push_back(boost::thread(attrs, boost::bind(&threadpool::run, this))); + threads.push_back(boost::thread(attrs, boost::bind(&threadpool::run, this, false))); } } threadpool::~threadpool() { + try { const boost::unique_lock lock(mutex); running = false; has_work.notify_all(); } + catch (...) + { + // if the lock throws, we're just do it without a lock and hope, + // since the alternative is terminate + running = false; + has_work.notify_all(); + } for (size_t i = 0; i f) { - entry e = {obj, f}; +void threadpool::submit(waiter *obj, std::function f, bool leaf) { + CHECK_AND_ASSERT_THROW_MES(!is_leaf, "A leaf routine is using a thread pool"); boost::unique_lock lock(mutex); - if ((active == max && !queue.empty()) || depth > 0) { + if (!leaf && ((active == max && !queue.empty()) || depth > 0)) { // if all available threads are already running // and there's work waiting, just run in current thread lock.unlock(); ++depth; + is_leaf = leaf; f(); --depth; + is_leaf = false; } else { if (obj) obj->inc(); - queue.push_back(e); + if (leaf) + queue.push_front({obj, f, leaf}); + else + queue.push_back({obj, f, leaf}); has_work.notify_one(); } } -int threadpool::get_max_concurrency() { +unsigned int threadpool::get_max_concurrency() const { return max; } threadpool::waiter::~waiter() { + try { boost::unique_lock lock(mt); if (num) MERROR("wait should have been called before waiter dtor - waiting now"); } + catch (...) { /* ignore */ } try { - wait(); + wait(NULL); } catch (const std::exception &e) { @@ -99,9 +112,12 @@ threadpool::waiter::~waiter() } } -void threadpool::waiter::wait() { +void threadpool::waiter::wait(threadpool *tpool) { + if (tpool) + tpool->run(true); boost::unique_lock lock(mt); - while(num) cv.wait(lock); + while(num) + cv.wait(lock); } void threadpool::waiter::inc() { @@ -113,15 +129,19 @@ void threadpool::waiter::dec() { const boost::unique_lock lock(mt); num--; if (!num) - cv.notify_one(); + cv.notify_all(); } -void threadpool::run() { +void threadpool::run(bool flush) { boost::unique_lock lock(mutex); while (running) { entry e; while(queue.empty() && running) + { + if (flush) + return; has_work.wait(lock); + } if (!running) break; active++; @@ -129,8 +149,10 @@ void threadpool::run() { queue.pop_front(); lock.unlock(); ++depth; + is_leaf = e.leaf; e.f(); --depth; + is_leaf = false; if (e.wo) e.wo->dec(); diff --git a/src/common/threadpool.h b/src/common/threadpool.h index 34152541cb5..5e490ee7d2c 100644 --- a/src/common/threadpool.h +++ b/src/common/threadpool.h @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2018, The Monero Project +// Copyright (c) 2017-2019, The Monero Project // // All rights reserved. // @@ -46,6 +46,9 @@ class threadpool static threadpool instance; return instance; } + static threadpool *getNewForUnitTests(unsigned max_threads = 0) { + return new threadpool(max_threads); + } // The waiter lets the caller know when all of its // tasks are completed. @@ -56,7 +59,7 @@ class threadpool public: void inc(); void dec(); - void wait(); //! Wait for a set of tasks to finish. + void wait(threadpool *tpool); //! Wait for a set of tasks to finish. waiter() : num(0){} ~waiter(); }; @@ -64,25 +67,27 @@ class threadpool // Submit a task to the pool. The waiter pointer may be // NULL if the caller doesn't care to wait for the // task to finish. - void submit(waiter *waiter, std::function f); + void submit(waiter *waiter, std::function f, bool leaf = false); + + unsigned int get_max_concurrency() const; - int get_max_concurrency(); + ~threadpool(); private: - threadpool(); - ~threadpool(); + threadpool(unsigned int max_threads = 0); typedef struct entry { waiter *wo; std::function f; + bool leaf; } entry; std::deque queue; boost::condition_variable has_work; boost::mutex mutex; std::vector threads; - int active; - int max; + unsigned int active; + unsigned int max; bool running; - void run(); + void run(bool flush = false); }; } diff --git a/src/common/timings.cc b/src/common/timings.cc new file mode 100644 index 00000000000..612ac2cc671 --- /dev/null +++ b/src/common/timings.cc @@ -0,0 +1,125 @@ +#include +#include +#include +#include +#include +#include "misc_log_ex.h" +#include "timings.h" + +#define N_EXPECTED_FIELDS (8+11) + +TimingsDatabase::TimingsDatabase() +{ +} + +TimingsDatabase::TimingsDatabase(const std::string &filename): + filename(filename) +{ + load(); +} + +TimingsDatabase::~TimingsDatabase() +{ + save(); +} + +bool TimingsDatabase::load() +{ + instances.clear(); + + if (filename.empty()) + return true; + + FILE *f = fopen(filename.c_str(), "r"); + if (!f) + { + MDEBUG("Failed to load timings file " << filename << ": " << strerror(errno)); + return false; + } + while (1) + { + char s[4096]; + if (!fgets(s, sizeof(s), f)) + break; + char *tab = strchr(s, '\t'); + if (!tab) + { + MWARNING("Bad format: no tab found"); + continue; + } + const std::string name = std::string(s, tab - s); + std::vector fields; + char *ptr = tab + 1; + boost::split(fields, ptr, boost::is_any_of(" ")); + if (fields.size() != N_EXPECTED_FIELDS) + { + MERROR("Bad format: wrong number of fields: got " << fields.size() << " expected " << N_EXPECTED_FIELDS); + continue; + } + + instance i; + + unsigned int idx = 0; + i.t = atoi(fields[idx++].c_str()); + i.npoints = atoi(fields[idx++].c_str()); + i.min = atof(fields[idx++].c_str()); + i.max = atof(fields[idx++].c_str()); + i.mean = atof(fields[idx++].c_str()); + i.median = atof(fields[idx++].c_str()); + i.stddev = atof(fields[idx++].c_str()); + i.npskew = atof(fields[idx++].c_str()); + i.deciles.reserve(11); + for (int n = 0; n < 11; ++n) + { + i.deciles.push_back(atoi(fields[idx++].c_str())); + } + instances.insert(std::make_pair(name, i)); + } + fclose(f); + return true; +} + +bool TimingsDatabase::save() +{ + if (filename.empty()) + return true; + + FILE *f = fopen(filename.c_str(), "w"); + if (!f) + { + MERROR("Failed to write to file " << filename << ": " << strerror(errno)); + return false; + } + for (const auto &i: instances) + { + fprintf(f, "%s", i.first.c_str()); + fprintf(f, "\t%lu", (unsigned long)i.second.t); + fprintf(f, " %zu", i.second.npoints); + fprintf(f, " %f", i.second.min); + fprintf(f, " %f", i.second.max); + fprintf(f, " %f", i.second.mean); + fprintf(f, " %f", i.second.median); + fprintf(f, " %f", i.second.stddev); + fprintf(f, " %f", i.second.npskew); + for (uint64_t v: i.second.deciles) + fprintf(f, " %lu", (unsigned long)v); + fputc('\n', f); + } + fclose(f); + return true; +} + +std::vector TimingsDatabase::get(const char *name) const +{ + std::vector ret; + auto range = instances.equal_range(name); + for (auto i = range.first; i != range.second; ++i) + ret.push_back(i->second); + std::sort(ret.begin(), ret.end(), [](const instance &e0, const instance &e1){ return e0.t < e1.t; }); + return ret; +} + +void TimingsDatabase::add(const char *name, const instance &i) +{ + instances.insert(std::make_pair(name, i)); +} diff --git a/src/common/timings.h b/src/common/timings.h new file mode 100644 index 00000000000..fb905611f8d --- /dev/null +++ b/src/common/timings.h @@ -0,0 +1,34 @@ +#pragma once + +#include +#include +#include +#include + +class TimingsDatabase +{ +public: + struct instance + { + time_t t; + size_t npoints; + double min, max, mean, median, stddev, npskew; + std::vector deciles; + }; + +public: + TimingsDatabase(); + TimingsDatabase(const std::string &filename); + ~TimingsDatabase(); + + std::vector get(const char *name) const; + void add(const char *name, const instance &data); + +private: + bool load(); + bool save(); + +private: + std::string filename; + std::multimap instances; +}; diff --git a/src/common/unordered_containers_boost_serialization.h b/src/common/unordered_containers_boost_serialization.h index d78dc6a30fc..74e2c3f81ba 100644 --- a/src/common/unordered_containers_boost_serialization.h +++ b/src/common/unordered_containers_boost_serialization.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/common/updates.cpp b/src/common/updates.cpp index 9eb402e0b5c..0bc6ff63c11 100644 --- a/src/common/updates.cpp +++ b/src/common/updates.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2018, The Monero Project +// Copyright (c) 2017-2019, The Monero Project // // All rights reserved. // @@ -69,12 +69,12 @@ namespace tools continue; bool alnum = true; - for (auto c: hash) + for (auto c: fields[3]) if (!isalnum(c)) alnum = false; - if (hash.size() != 64 && !alnum) + if (fields[3].size() != 64 && !alnum) { - MWARNING("Invalid hash: " << hash); + MWARNING("Invalid hash: " << fields[3]); continue; } diff --git a/src/common/updates.h b/src/common/updates.h index 6ec22f18395..8fda6d20722 100644 --- a/src/common/updates.h +++ b/src/common/updates.h @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2018, The Monero Project +// Copyright (c) 2017-2019, The Monero Project // // All rights reserved. // diff --git a/src/common/util.cpp b/src/common/util.cpp index 7e77e19b167..db5aa3052bc 100644 --- a/src/common/util.cpp +++ b/src/common/util.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -28,17 +28,37 @@ // // Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers +#include #include #ifdef __GLIBC__ #include #endif +#ifdef __GLIBC__ +#include +#include +#include +#include +#include +#include +#include +#include +#endif + +//tools::is_hdd +#ifdef __GLIBC__ + #include + #include + #include +#endif + #include "unbound.h" #include "include_base_utils.h" #include "file_io_utils.h" #include "wipeable_string.h" +#include "misc_os_dependent.h" using namespace epee; #include "crypto/crypto.h" @@ -49,6 +69,9 @@ using namespace epee; #include "net/http_client.h" // epee::net_utils::... #ifdef WIN32 +#ifndef STRSAFE_NO_DEPRECATE +#define STRSAFE_NO_DEPRECATE +#endif #include #include #include @@ -60,8 +83,35 @@ using namespace epee; #include #include #include +#include #include +#undef MONERO_DEFAULT_LOG_CATEGORY +#define MONERO_DEFAULT_LOG_CATEGORY "util" + +namespace +{ + +#ifndef _WIN32 +static int flock_exnb(int fd) +{ + struct flock fl; + int ret; + + memset(&fl, 0, sizeof(fl)); + fl.l_type = F_WRLCK; + fl.l_whence = SEEK_SET; + fl.l_start = 0; + fl.l_len = 0; + ret = fcntl(fd, F_SETLK, &fl); + if (ret < 0) + MERROR("Error locking fd " << fd << ": " << errno << " (" << strerror(errno) << ")"); + return ret; +} +#endif + +} + namespace tools { std::function signal_handler::m_handler; @@ -162,7 +212,7 @@ namespace tools struct stat wstats = {}; if (fstat(fdw, std::addressof(wstats)) == 0 && rstats.st_dev == wstats.st_dev && rstats.st_ino == wstats.st_ino && - flock(fdw, (LOCK_EX | LOCK_NB)) == 0 && ftruncate(fdw, 0) == 0) + flock_exnb(fdw) == 0 && ftruncate(fdw, 0) == 0) { std::FILE* file = fdopen(fdw, "w"); if (file) return {file, std::move(name)}; @@ -184,6 +234,73 @@ namespace tools catch (...) {} } + file_locker::file_locker(const std::string &filename) + { +#ifdef WIN32 + m_fd = INVALID_HANDLE_VALUE; + std::wstring filename_wide; + try + { + filename_wide = string_tools::utf8_to_utf16(filename); + } + catch (const std::exception &e) + { + MERROR("Failed to convert path \"" << filename << "\" to UTF-16: " << e.what()); + return; + } + m_fd = CreateFileW(filename_wide.c_str(), GENERIC_READ, 0, NULL, OPEN_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL); + if (m_fd != INVALID_HANDLE_VALUE) + { + OVERLAPPED ov; + memset(&ov, 0, sizeof(ov)); + if (!LockFileEx(m_fd, LOCKFILE_FAIL_IMMEDIATELY | LOCKFILE_EXCLUSIVE_LOCK, 0, 1, 0, &ov)) + { + MERROR("Failed to lock " << filename << ": " << std::error_code(GetLastError(), std::system_category())); + CloseHandle(m_fd); + m_fd = INVALID_HANDLE_VALUE; + } + } + else + { + MERROR("Failed to open " << filename << ": " << std::error_code(GetLastError(), std::system_category())); + } +#else + m_fd = open(filename.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0666); + if (m_fd != -1) + { + if (flock_exnb(m_fd) == -1) + { + MERROR("Failed to lock " << filename << ": " << std::strerror(errno)); + close(m_fd); + m_fd = -1; + } + } + else + { + MERROR("Failed to open " << filename << ": " << std::strerror(errno)); + } +#endif + } + file_locker::~file_locker() + { + if (locked()) + { +#ifdef WIN32 + CloseHandle(m_fd); +#else + close(m_fd); +#endif + } + } + bool file_locker::locked() const + { +#ifdef WIN32 + return m_fd != INVALID_HANDLE_VALUE; +#else + return m_fd != -1; +#endif + } + #ifdef WIN32 std::string get_windows_version_display_string() { @@ -222,10 +339,19 @@ namespace tools StringCchCopy(pszOS, BUFSIZE, TEXT("Microsoft ")); // Test for the specific product. + if ( osvi.dwMajorVersion == 10 ) + { + if ( osvi.dwMinorVersion == 0 ) + { + if( osvi.wProductType == VER_NT_WORKSTATION ) + StringCchCat(pszOS, BUFSIZE, TEXT("Windows 10 ")); + else StringCchCat(pszOS, BUFSIZE, TEXT("Windows Server 2016 " )); + } + } if ( osvi.dwMajorVersion == 6 ) { - if( osvi.dwMinorVersion == 0 ) + if ( osvi.dwMinorVersion == 0 ) { if( osvi.wProductType == VER_NT_WORKSTATION ) StringCchCat(pszOS, BUFSIZE, TEXT("Windows Vista ")); @@ -239,6 +365,20 @@ namespace tools else StringCchCat(pszOS, BUFSIZE, TEXT("Windows Server 2008 R2 " )); } + if ( osvi.dwMinorVersion == 2 ) + { + if( osvi.wProductType == VER_NT_WORKSTATION ) + StringCchCat(pszOS, BUFSIZE, TEXT("Windows 8 ")); + else StringCchCat(pszOS, BUFSIZE, TEXT("Windows Server 2012 " )); + } + + if ( osvi.dwMinorVersion == 3 ) + { + if( osvi.wProductType == VER_NT_WORKSTATION ) + StringCchCat(pszOS, BUFSIZE, TEXT("Windows 8.1 ")); + else StringCchCat(pszOS, BUFSIZE, TEXT("Windows Server 2012 R2 " )); + } + pGPI = (PGPI) GetProcAddress( GetModuleHandle(TEXT("kernel32.dll")), "GetProductInfo"); @@ -440,10 +580,15 @@ std::string get_nix_version_display_string() if (SHGetSpecialFolderPathW(NULL, psz_path, nfolder, iscreate)) { - int size_needed = WideCharToMultiByte(CP_UTF8, 0, psz_path, wcslen(psz_path), NULL, 0, NULL, NULL); - std::string folder_name(size_needed, 0); - WideCharToMultiByte(CP_UTF8, 0, psz_path, wcslen(psz_path), &folder_name[0], size_needed, NULL, NULL); - return folder_name; + try + { + return string_tools::utf16_to_utf8(psz_path); + } + catch (const std::exception &e) + { + MERROR("utf16_to_utf8 failed: " << e.what()); + return ""; + } } LOG_ERROR("SHGetSpecialFolderPathW() failed, could not obtain requested path."); @@ -499,26 +644,28 @@ std::string get_nix_version_display_string() return res; } - std::error_code replace_file(const std::string& replacement_name, const std::string& replaced_name) + std::error_code replace_file(const std::string& old_name, const std::string& new_name) { int code; #if defined(WIN32) // Maximizing chances for success - WCHAR wide_replacement_name[1000]; - MultiByteToWideChar(CP_UTF8, 0, replacement_name.c_str(), replacement_name.size() + 1, wide_replacement_name, 1000); - WCHAR wide_replaced_name[1000]; - MultiByteToWideChar(CP_UTF8, 0, replaced_name.c_str(), replaced_name.size() + 1, wide_replaced_name, 1000); - - DWORD attributes = ::GetFileAttributesW(wide_replaced_name); + std::wstring wide_replacement_name; + try { wide_replacement_name = string_tools::utf8_to_utf16(old_name); } + catch (...) { return std::error_code(GetLastError(), std::system_category()); } + std::wstring wide_replaced_name; + try { wide_replaced_name = string_tools::utf8_to_utf16(new_name); } + catch (...) { return std::error_code(GetLastError(), std::system_category()); } + + DWORD attributes = ::GetFileAttributesW(wide_replaced_name.c_str()); if (INVALID_FILE_ATTRIBUTES != attributes) { - ::SetFileAttributesW(wide_replaced_name, attributes & (~FILE_ATTRIBUTE_READONLY)); + ::SetFileAttributesW(wide_replaced_name.c_str(), attributes & (~FILE_ATTRIBUTE_READONLY)); } - bool ok = 0 != ::MoveFileExW(wide_replacement_name, wide_replaced_name, MOVEFILE_REPLACE_EXISTING); + bool ok = 0 != ::MoveFileExW(wide_replacement_name.c_str(), wide_replaced_name.c_str(), MOVEFILE_REPLACE_EXISTING); code = ok ? 0 : static_cast(::GetLastError()); #else - bool ok = 0 == std::rename(replacement_name.c_str(), replaced_name.c_str()); + bool ok = 0 == std::rename(old_name.c_str(), new_name.c_str()); code = ok ? 0 : errno; #endif return std::error_code(code, std::system_category()); @@ -597,6 +744,36 @@ std::string get_nix_version_display_string() static void setup_crash_dump() {} #endif + bool disable_core_dumps() + { +#ifdef __GLIBC__ + // disable core dumps in release mode + struct rlimit rlimit; + rlimit.rlim_cur = rlimit.rlim_max = 0; + if (setrlimit(RLIMIT_CORE, &rlimit)) + { + MWARNING("Failed to disable core dumps"); + return false; + } +#endif + return true; + } + + ssize_t get_lockable_memory() + { +#ifdef __GLIBC__ + struct rlimit rlim; + if (getrlimit(RLIMIT_MEMLOCK, &rlim) < 0) + { + MERROR("Failed to determine the lockable memory limit"); + return -1; + } + return rlim.rlim_cur; +#else + return -1; +#endif + } + bool on_startup() { mlog_configure("", true); @@ -632,6 +809,44 @@ std::string get_nix_version_display_string() #endif } + boost::optional is_hdd(const char *file_path) + { +#ifdef __GLIBC__ + struct stat st; + std::string prefix; + if(stat(file_path, &st) == 0) + { + std::ostringstream s; + s << "/sys/dev/block/" << major(st.st_dev) << ":" << minor(st.st_dev); + prefix = s.str(); + } + else + { + return boost::none; + } + std::string attr_path = prefix + "/queue/rotational"; + std::ifstream f(attr_path, std::ios_base::in); + if(not f.is_open()) + { + attr_path = prefix + "/../queue/rotational"; + f.open(attr_path, std::ios_base::in); + if(not f.is_open()) + { + return boost::none; + } + } + unsigned short val = 0xdead; + f >> val; + if(not f.fail()) + { + return (val == 1); + } + return boost::none; +#else + return boost::none; +#endif + } + namespace { boost::mutex max_concurrency_lock; @@ -657,6 +872,13 @@ std::string get_nix_version_display_string() bool is_local_address(const std::string &address) { + // always assume Tor/I2P addresses to be untrusted by default + if (boost::ends_with(address, ".onion") || boost::ends_with(address, ".i2p")) + { + MDEBUG("Address '" << address << "' is Tor/I2P, non local"); + return false; + } + // extract host epee::net_utils::http::url_content u_c; if (!epee::net_utils::parse_url(address, u_c)) @@ -750,4 +972,134 @@ std::string get_nix_version_display_string() return false; return true; } + + boost::optional> parse_subaddress_lookahead(const std::string& str) + { + auto pos = str.find(":"); + bool r = pos != std::string::npos; + uint32_t major; + r = r && epee::string_tools::get_xtype_from_string(major, str.substr(0, pos)); + uint32_t minor; + r = r && epee::string_tools::get_xtype_from_string(minor, str.substr(pos + 1)); + if (r) + { + return std::make_pair(major, minor); + } + else + { + return {}; + } + } + + std::string glob_to_regex(const std::string &val) + { + std::string newval; + + bool escape = false; + for (char c: val) + { + if (c == '*') + newval += escape ? "*" : ".*"; + else if (c == '?') + newval += escape ? "?" : "."; + else if (c == '\\') + newval += '\\', escape = !escape; + else + newval += c; + } + return newval; + } + +#ifdef _WIN32 + std::string input_line_win() + { + HANDLE hConIn = CreateFileW(L"CONIN$", GENERIC_READ | GENERIC_WRITE, FILE_SHARE_READ | FILE_SHARE_WRITE, nullptr, OPEN_EXISTING, 0, nullptr); + DWORD oldMode; + + FlushConsoleInputBuffer(hConIn); + GetConsoleMode(hConIn, &oldMode); + SetConsoleMode(hConIn, ENABLE_LINE_INPUT | ENABLE_ECHO_INPUT | ENABLE_PROCESSED_INPUT); + + wchar_t buffer[1024]; + DWORD read; + + ReadConsoleW(hConIn, buffer, sizeof(buffer)/sizeof(wchar_t)-1, &read, nullptr); + buffer[read] = 0; + + SetConsoleMode(hConIn, oldMode); + CloseHandle(hConIn); + + int size_needed = WideCharToMultiByte(CP_UTF8, 0, buffer, -1, NULL, 0, NULL, NULL); + std::string buf(size_needed, '\0'); + WideCharToMultiByte(CP_UTF8, 0, buffer, -1, &buf[0], size_needed, NULL, NULL); + buf.pop_back(); //size_needed includes null that we needed to have space for + return buf; + } +#endif + + void closefrom(int fd) + { +#if defined __FreeBSD__ || defined __OpenBSD__ || defined __NetBSD__ || defined __DragonFly__ + ::closefrom(fd); +#else +#if defined __GLIBC__ + const int sc_open_max = sysconf(_SC_OPEN_MAX); + const int MAX_FDS = std::min(65536, sc_open_max); +#else + const int MAX_FDS = 65536; +#endif + while (fd < MAX_FDS) + { + close(fd); + ++fd; + } +#endif + } + + std::string get_human_readable_timestamp(uint64_t ts) + { + char buffer[64]; + if (ts < 1234567890) + return ""; + time_t tt = ts; + struct tm tm; + misc_utils::get_gmt_time(tt, tm); + strftime(buffer, sizeof(buffer), "%Y-%m-%d %H:%M:%S", &tm); + return std::string(buffer); + } + + std::string get_human_readable_bytes(uint64_t bytes) + { + // Use 1024 for "kilo", 1024*1024 for "mega" and so on instead of the more modern and standard-conforming + // 1000, 1000*1000 and so on, to be consistent with other Monero code that also uses base 2 units + struct byte_map + { + const char* const format; + const std::uint64_t bytes; + }; + + static constexpr const byte_map sizes[] = + { + {"%.0f B", 1024}, + {"%.2f KB", 1024 * 1024}, + {"%.2f MB", std::uint64_t(1024) * 1024 * 1024}, + {"%.2f GB", std::uint64_t(1024) * 1024 * 1024 * 1024}, + {"%.2f TB", std::uint64_t(1024) * 1024 * 1024 * 1024 * 1024} + }; + + struct bytes_less + { + bool operator()(const byte_map& lhs, const byte_map& rhs) const noexcept + { + return lhs.bytes < rhs.bytes; + } + }; + + const auto size = std::upper_bound( + std::begin(sizes), std::end(sizes) - 1, byte_map{"", bytes}, bytes_less{} + ); + const std::uint64_t divisor = size->bytes / 1024; + return (boost::format(size->format) % (double(bytes) / divisor)).str(); + } + } diff --git a/src/common/util.h b/src/common/util.h index d3ba47a4fc5..f6d5c9b1f19 100644 --- a/src/common/util.h +++ b/src/common/util.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -32,6 +32,7 @@ #include #include +#include #include #include #include @@ -90,6 +91,20 @@ namespace tools const std::string& filename() const noexcept { return m_filename; } }; + class file_locker + { + public: + file_locker(const std::string &filename); + ~file_locker(); + bool locked() const; + private: +#ifdef WIN32 + HANDLE m_fd; +#else + int m_fd; +#endif + }; + /*! \brief Returns the default data directory. * * \details Windows < Vista: C:\\Documents and Settings\\Username\\Application Data\\CRYPTONOTE_NAME @@ -130,10 +145,12 @@ namespace tools bool create_directories_if_necessary(const std::string& path); /*! \brief std::rename wrapper for nix and something strange for windows. */ - std::error_code replace_file(const std::string& replacement_name, const std::string& replaced_name); + std::error_code replace_file(const std::string& old_name, const std::string& new_name); bool sanitize_locale(); + bool disable_core_dumps(); + bool on_startup(); /*! \brief Defines a signal handler for win32 and *nix @@ -204,6 +221,8 @@ namespace tools void set_strict_default_file_permissions(bool strict); + ssize_t get_lockable_memory(); + void set_max_concurrency(unsigned n); unsigned get_max_concurrency(); @@ -212,4 +231,19 @@ namespace tools bool sha256sum(const uint8_t *data, size_t len, crypto::hash &hash); bool sha256sum(const std::string &filename, crypto::hash &hash); + + boost::optional is_hdd(const char *path); + + boost::optional> parse_subaddress_lookahead(const std::string& str); + + std::string glob_to_regex(const std::string &val); +#ifdef _WIN32 + std::string input_line_win(); +#endif + + void closefrom(int fd); + + std::string get_human_readable_timestamp(uint64_t ts); + + std::string get_human_readable_bytes(uint64_t bytes); } diff --git a/src/common/varint.h b/src/common/varint.h index 151d13dbfb7..a0d79be28a8 100644 --- a/src/common/varint.h +++ b/src/common/varint.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -123,6 +123,6 @@ namespace tools { */ template int read_varint(InputIt &&first, InputIt &&last, T &i) { - return read_varint::digits, InputIt, T>(std::move(first), std::move(last), i); + return read_varint::digits>(std::forward(first), std::forward(last), i); } } diff --git a/src/crypto/CMakeLists.txt b/src/crypto/CMakeLists.txt index 71dcedcab32..3a4d09fd8d0 100644 --- a/src/crypto/CMakeLists.txt +++ b/src/crypto/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2014-2018, The Monero Project +# Copyright (c) 2014-2019, The Monero Project # # All rights reserved. # @@ -39,14 +39,20 @@ set(crypto_sources hash-extra-jh.c hash-extra-skein.c hash.c + hmac-keccak.c jh.c keccak.c oaes_lib.c random.c skein.c slow-hash.c + CryptonightR_JIT.c tree-hash.c) +if(ARCH_ID STREQUAL "i386" OR ARCH_ID STREQUAL "x86_64" OR ARCH_ID STREQUAL "x86-64" OR ARCH_ID STREQUAL "amd64") +list(APPEND crypto_sources CryptonightR_template.S) +endif() + set(crypto_headers) set(crypto_private_headers @@ -59,6 +65,7 @@ set(crypto_private_headers groestl_tables.h hash-ops.h hash.h + hmac-keccak.h initializer.h jh.h keccak.h @@ -66,7 +73,9 @@ set(crypto_private_headers oaes_lib.h random.h skein.h - skein_port.h) + skein_port.h + CryptonightR_JIT.h + CryptonightR_template.h) monero_private_headers(cncrypto ${crypto_private_headers}) @@ -78,6 +87,7 @@ target_link_libraries(cncrypto PUBLIC epee ${Boost_SYSTEM_LIBRARY} + ${SODIUM_LIBRARY} PRIVATE ${EXTRA_LIBRARIES}) @@ -100,4 +110,5 @@ if (ANDROID OR IOS) endif() endif() - +# cheat because cmake and ccache hate each other +set_property(SOURCE CryptonightR_template.S PROPERTY LANGUAGE C) diff --git a/src/crypto/CryptonightR_JIT.c b/src/crypto/CryptonightR_JIT.c new file mode 100644 index 00000000000..ee8f3f36f47 --- /dev/null +++ b/src/crypto/CryptonightR_JIT.c @@ -0,0 +1,123 @@ +#include +#include +#include +#include +#include +#include +#if !(defined(_MSC_VER) || defined(__MINGW32__)) +#include +#endif + +#include "int-util.h" +#include "hash-ops.h" +#include "variant4_random_math.h" +#include "CryptonightR_JIT.h" +#include "CryptonightR_template.h" + +static const uint8_t prologue[] = { +#if defined __i386 || defined __x86_64__ + 0x4C, 0x8B, 0xD7, // mov r10, rdi + 0x53, // push rbx + 0x55, // push rbp + 0x41, 0x57, // push r15 + 0x4C, 0x8B, 0xDC, // mov r11, rsp + 0x41, 0x8B, 0x1A, // mov ebx, DWORD PTR [r10] + 0x41, 0x8B, 0x72, 0x04, // mov esi, DWORD PTR [r10+4] + 0x41, 0x8B, 0x7A, 0x08, // mov edi, DWORD PTR [r10+8] + 0x41, 0x8B, 0x6A, 0x0C, // mov ebp, DWORD PTR [r10+12] + 0x41, 0x8B, 0x62, 0x10, // mov esp, DWORD PTR [r10+16] + 0x45, 0x8B, 0x7A, 0x14, // mov r15d, DWORD PTR [r10+20] + 0x41, 0x8B, 0x42, 0x18, // mov eax, DWORD PTR [r10+24] + 0x41, 0x8B, 0x52, 0x1C, // mov edx, DWORD PTR [r10+28] + 0x45, 0x8B, 0x4A, 0x20, // mov r9d, DWORD PTR [r10+32] +#endif +}; + +static const uint8_t epilogue[] = { +#if defined __i386 || defined __x86_64__ + 0x49, 0x8B, 0xE3, // mov rsp, r11 + 0x41, 0x89, 0x1A, // mov DWORD PTR [r10], ebx + 0x41, 0x89, 0x72, 0x04, // mov DWORD PTR [r10+4], esi + 0x41, 0x89, 0x7A, 0x08, // mov DWORD PTR [r10+8], edi + 0x41, 0x89, 0x6A, 0x0C, // mov DWORD PTR [r10+12], ebp + 0x41, 0x5F, // pop r15 + 0x5D, // pop rbp + 0x5B, // pop rbx + 0xC3, // ret +#endif +}; + +#define APPEND_CODE(src, size) \ + do { \ + if (JIT_code + (size) > JIT_code_end) \ + return -1; \ + memcpy(JIT_code, (src), (size)); \ + JIT_code += (size); \ + } while (0) + +int v4_generate_JIT_code(const struct V4_Instruction* code, v4_random_math_JIT_func buf, const size_t buf_size) +{ +#if defined __i386 || defined __x86_64__ + uint8_t* JIT_code = (uint8_t*) buf; + const uint8_t* JIT_code_end = JIT_code + buf_size; + +#if !(defined(_MSC_VER) || defined(__MINGW32__)) + if (mprotect((void*)buf, buf_size, PROT_READ | PROT_WRITE)) + return 1; +#endif + + APPEND_CODE(prologue, sizeof(prologue)); + + uint32_t prev_rot_src = 0xFFFFFFFFU; + + for (int i = 0;; ++i) + { + const struct V4_Instruction inst = code[i]; + if (inst.opcode == RET) + break; + + const uint8_t opcode = (inst.opcode == MUL) ? inst.opcode : (inst.opcode + 2); + + const uint32_t a = inst.dst_index; + const uint32_t b = inst.src_index; + const uint8_t c = opcode | (inst.dst_index << V4_OPCODE_BITS) | (((inst.src_index == 8) ? inst.dst_index : inst.src_index) << (V4_OPCODE_BITS + V4_DST_INDEX_BITS)); + + switch (inst.opcode) + { + case ROR: + case ROL: + if (b != prev_rot_src) + { + prev_rot_src = b; + const uint8_t* p1 = (const uint8_t*) instructions_mov[c]; + const uint8_t* p2 = (const uint8_t*) instructions_mov[c + 1]; + APPEND_CODE(p1, p2 - p1); + } + break; + } + + if (a == prev_rot_src) + prev_rot_src = 0xFFFFFFFFU; + + const uint8_t* p1 = (const uint8_t*) instructions[c]; + const uint8_t* p2 = (const uint8_t*) instructions[c + 1]; + APPEND_CODE(p1, p2 - p1); + + if (inst.opcode == ADD) + *(uint32_t*)(JIT_code - 4) = inst.C; + } + + APPEND_CODE(epilogue, sizeof(epilogue)); + +#if !(defined(_MSC_VER) || defined(__MINGW32__)) + if (mprotect((void*)buf, buf_size, PROT_READ | PROT_EXEC)) + return 1; +#endif + + __builtin___clear_cache((char*)buf, (char*)JIT_code); + + return 0; +#else + return 1; +#endif +} diff --git a/src/crypto/CryptonightR_JIT.h b/src/crypto/CryptonightR_JIT.h new file mode 100644 index 00000000000..cb32c3a7990 --- /dev/null +++ b/src/crypto/CryptonightR_JIT.h @@ -0,0 +1,22 @@ +#ifndef CRYPTONIGHTR_JIT_H +#define CRYPTONIGHTR_JIT_H + +// Minimalistic JIT code generator for random math sequence in CryptonightR +// +// Usage: +// - Allocate writable and executable memory +// - Call v4_generate_JIT_code with "buf" pointed to memory allocated on previous step +// - Call the generated code instead of "v4_random_math(code, r)", omit the "code" parameter + +typedef void (*v4_random_math_JIT_func)(uint32_t* r) +#if defined __i386 || defined __x86_64__ +__attribute__((sysv_abi)) +#endif +; + +// Given the random math sequence, generates machine code (x86-64) for it +// Returns 0 if code was generated successfully +// Returns -1 if provided buffer was too small +int v4_generate_JIT_code(const struct V4_Instruction* code, v4_random_math_JIT_func buf, const size_t buf_size); + +#endif // CRYPTONIGHTR_JIT_H diff --git a/src/crypto/CryptonightR_template.S b/src/crypto/CryptonightR_template.S new file mode 100644 index 00000000000..068de22ec73 --- /dev/null +++ b/src/crypto/CryptonightR_template.S @@ -0,0 +1,1590 @@ +#ifdef __APPLE__ +# define ALIGN(x) .align 6 +#else +# define ALIGN(x) .align 64 +#endif +.intel_syntax noprefix +#ifdef __APPLE__ +# define FN_PREFIX(fn) _ ## fn +.text +#else +# define FN_PREFIX(fn) fn +.section .text +#endif + +#define PUBLIC .global + +PUBLIC FN_PREFIX(CryptonightR_instruction0) +PUBLIC FN_PREFIX(CryptonightR_instruction1) +PUBLIC FN_PREFIX(CryptonightR_instruction2) +PUBLIC FN_PREFIX(CryptonightR_instruction3) +PUBLIC FN_PREFIX(CryptonightR_instruction4) +PUBLIC FN_PREFIX(CryptonightR_instruction5) +PUBLIC FN_PREFIX(CryptonightR_instruction6) +PUBLIC FN_PREFIX(CryptonightR_instruction7) +PUBLIC FN_PREFIX(CryptonightR_instruction8) +PUBLIC FN_PREFIX(CryptonightR_instruction9) +PUBLIC FN_PREFIX(CryptonightR_instruction10) +PUBLIC FN_PREFIX(CryptonightR_instruction11) +PUBLIC FN_PREFIX(CryptonightR_instruction12) +PUBLIC FN_PREFIX(CryptonightR_instruction13) +PUBLIC FN_PREFIX(CryptonightR_instruction14) +PUBLIC FN_PREFIX(CryptonightR_instruction15) +PUBLIC FN_PREFIX(CryptonightR_instruction16) +PUBLIC FN_PREFIX(CryptonightR_instruction17) +PUBLIC FN_PREFIX(CryptonightR_instruction18) +PUBLIC FN_PREFIX(CryptonightR_instruction19) +PUBLIC FN_PREFIX(CryptonightR_instruction20) +PUBLIC FN_PREFIX(CryptonightR_instruction21) +PUBLIC FN_PREFIX(CryptonightR_instruction22) +PUBLIC FN_PREFIX(CryptonightR_instruction23) +PUBLIC FN_PREFIX(CryptonightR_instruction24) +PUBLIC FN_PREFIX(CryptonightR_instruction25) +PUBLIC FN_PREFIX(CryptonightR_instruction26) +PUBLIC FN_PREFIX(CryptonightR_instruction27) +PUBLIC FN_PREFIX(CryptonightR_instruction28) +PUBLIC FN_PREFIX(CryptonightR_instruction29) +PUBLIC FN_PREFIX(CryptonightR_instruction30) +PUBLIC FN_PREFIX(CryptonightR_instruction31) +PUBLIC FN_PREFIX(CryptonightR_instruction32) +PUBLIC FN_PREFIX(CryptonightR_instruction33) +PUBLIC FN_PREFIX(CryptonightR_instruction34) +PUBLIC FN_PREFIX(CryptonightR_instruction35) +PUBLIC FN_PREFIX(CryptonightR_instruction36) +PUBLIC FN_PREFIX(CryptonightR_instruction37) +PUBLIC FN_PREFIX(CryptonightR_instruction38) +PUBLIC FN_PREFIX(CryptonightR_instruction39) +PUBLIC FN_PREFIX(CryptonightR_instruction40) +PUBLIC FN_PREFIX(CryptonightR_instruction41) +PUBLIC FN_PREFIX(CryptonightR_instruction42) +PUBLIC FN_PREFIX(CryptonightR_instruction43) +PUBLIC FN_PREFIX(CryptonightR_instruction44) +PUBLIC FN_PREFIX(CryptonightR_instruction45) +PUBLIC FN_PREFIX(CryptonightR_instruction46) +PUBLIC FN_PREFIX(CryptonightR_instruction47) +PUBLIC FN_PREFIX(CryptonightR_instruction48) +PUBLIC FN_PREFIX(CryptonightR_instruction49) +PUBLIC FN_PREFIX(CryptonightR_instruction50) +PUBLIC FN_PREFIX(CryptonightR_instruction51) +PUBLIC FN_PREFIX(CryptonightR_instruction52) +PUBLIC FN_PREFIX(CryptonightR_instruction53) +PUBLIC FN_PREFIX(CryptonightR_instruction54) +PUBLIC FN_PREFIX(CryptonightR_instruction55) +PUBLIC FN_PREFIX(CryptonightR_instruction56) +PUBLIC FN_PREFIX(CryptonightR_instruction57) +PUBLIC FN_PREFIX(CryptonightR_instruction58) +PUBLIC FN_PREFIX(CryptonightR_instruction59) +PUBLIC FN_PREFIX(CryptonightR_instruction60) +PUBLIC FN_PREFIX(CryptonightR_instruction61) +PUBLIC FN_PREFIX(CryptonightR_instruction62) +PUBLIC FN_PREFIX(CryptonightR_instruction63) +PUBLIC FN_PREFIX(CryptonightR_instruction64) +PUBLIC FN_PREFIX(CryptonightR_instruction65) +PUBLIC FN_PREFIX(CryptonightR_instruction66) +PUBLIC FN_PREFIX(CryptonightR_instruction67) +PUBLIC FN_PREFIX(CryptonightR_instruction68) +PUBLIC FN_PREFIX(CryptonightR_instruction69) +PUBLIC FN_PREFIX(CryptonightR_instruction70) +PUBLIC FN_PREFIX(CryptonightR_instruction71) +PUBLIC FN_PREFIX(CryptonightR_instruction72) +PUBLIC FN_PREFIX(CryptonightR_instruction73) +PUBLIC FN_PREFIX(CryptonightR_instruction74) +PUBLIC FN_PREFIX(CryptonightR_instruction75) +PUBLIC FN_PREFIX(CryptonightR_instruction76) +PUBLIC FN_PREFIX(CryptonightR_instruction77) +PUBLIC FN_PREFIX(CryptonightR_instruction78) +PUBLIC FN_PREFIX(CryptonightR_instruction79) +PUBLIC FN_PREFIX(CryptonightR_instruction80) +PUBLIC FN_PREFIX(CryptonightR_instruction81) +PUBLIC FN_PREFIX(CryptonightR_instruction82) +PUBLIC FN_PREFIX(CryptonightR_instruction83) +PUBLIC FN_PREFIX(CryptonightR_instruction84) +PUBLIC FN_PREFIX(CryptonightR_instruction85) +PUBLIC FN_PREFIX(CryptonightR_instruction86) +PUBLIC FN_PREFIX(CryptonightR_instruction87) +PUBLIC FN_PREFIX(CryptonightR_instruction88) +PUBLIC FN_PREFIX(CryptonightR_instruction89) +PUBLIC FN_PREFIX(CryptonightR_instruction90) +PUBLIC FN_PREFIX(CryptonightR_instruction91) +PUBLIC FN_PREFIX(CryptonightR_instruction92) +PUBLIC FN_PREFIX(CryptonightR_instruction93) +PUBLIC FN_PREFIX(CryptonightR_instruction94) +PUBLIC FN_PREFIX(CryptonightR_instruction95) +PUBLIC FN_PREFIX(CryptonightR_instruction96) +PUBLIC FN_PREFIX(CryptonightR_instruction97) +PUBLIC FN_PREFIX(CryptonightR_instruction98) +PUBLIC FN_PREFIX(CryptonightR_instruction99) +PUBLIC FN_PREFIX(CryptonightR_instruction100) +PUBLIC FN_PREFIX(CryptonightR_instruction101) +PUBLIC FN_PREFIX(CryptonightR_instruction102) +PUBLIC FN_PREFIX(CryptonightR_instruction103) +PUBLIC FN_PREFIX(CryptonightR_instruction104) +PUBLIC FN_PREFIX(CryptonightR_instruction105) +PUBLIC FN_PREFIX(CryptonightR_instruction106) +PUBLIC FN_PREFIX(CryptonightR_instruction107) +PUBLIC FN_PREFIX(CryptonightR_instruction108) +PUBLIC FN_PREFIX(CryptonightR_instruction109) +PUBLIC FN_PREFIX(CryptonightR_instruction110) +PUBLIC FN_PREFIX(CryptonightR_instruction111) +PUBLIC FN_PREFIX(CryptonightR_instruction112) +PUBLIC FN_PREFIX(CryptonightR_instruction113) +PUBLIC FN_PREFIX(CryptonightR_instruction114) +PUBLIC FN_PREFIX(CryptonightR_instruction115) +PUBLIC FN_PREFIX(CryptonightR_instruction116) +PUBLIC FN_PREFIX(CryptonightR_instruction117) +PUBLIC FN_PREFIX(CryptonightR_instruction118) +PUBLIC FN_PREFIX(CryptonightR_instruction119) +PUBLIC FN_PREFIX(CryptonightR_instruction120) +PUBLIC FN_PREFIX(CryptonightR_instruction121) +PUBLIC FN_PREFIX(CryptonightR_instruction122) +PUBLIC FN_PREFIX(CryptonightR_instruction123) +PUBLIC FN_PREFIX(CryptonightR_instruction124) +PUBLIC FN_PREFIX(CryptonightR_instruction125) +PUBLIC FN_PREFIX(CryptonightR_instruction126) +PUBLIC FN_PREFIX(CryptonightR_instruction127) +PUBLIC FN_PREFIX(CryptonightR_instruction128) +PUBLIC FN_PREFIX(CryptonightR_instruction129) +PUBLIC FN_PREFIX(CryptonightR_instruction130) +PUBLIC FN_PREFIX(CryptonightR_instruction131) +PUBLIC FN_PREFIX(CryptonightR_instruction132) +PUBLIC FN_PREFIX(CryptonightR_instruction133) +PUBLIC FN_PREFIX(CryptonightR_instruction134) +PUBLIC FN_PREFIX(CryptonightR_instruction135) +PUBLIC FN_PREFIX(CryptonightR_instruction136) +PUBLIC FN_PREFIX(CryptonightR_instruction137) +PUBLIC FN_PREFIX(CryptonightR_instruction138) +PUBLIC FN_PREFIX(CryptonightR_instruction139) +PUBLIC FN_PREFIX(CryptonightR_instruction140) +PUBLIC FN_PREFIX(CryptonightR_instruction141) +PUBLIC FN_PREFIX(CryptonightR_instruction142) +PUBLIC FN_PREFIX(CryptonightR_instruction143) +PUBLIC FN_PREFIX(CryptonightR_instruction144) +PUBLIC FN_PREFIX(CryptonightR_instruction145) +PUBLIC FN_PREFIX(CryptonightR_instruction146) +PUBLIC FN_PREFIX(CryptonightR_instruction147) +PUBLIC FN_PREFIX(CryptonightR_instruction148) +PUBLIC FN_PREFIX(CryptonightR_instruction149) +PUBLIC FN_PREFIX(CryptonightR_instruction150) +PUBLIC FN_PREFIX(CryptonightR_instruction151) +PUBLIC FN_PREFIX(CryptonightR_instruction152) +PUBLIC FN_PREFIX(CryptonightR_instruction153) +PUBLIC FN_PREFIX(CryptonightR_instruction154) +PUBLIC FN_PREFIX(CryptonightR_instruction155) +PUBLIC FN_PREFIX(CryptonightR_instruction156) +PUBLIC FN_PREFIX(CryptonightR_instruction157) +PUBLIC FN_PREFIX(CryptonightR_instruction158) +PUBLIC FN_PREFIX(CryptonightR_instruction159) +PUBLIC FN_PREFIX(CryptonightR_instruction160) +PUBLIC FN_PREFIX(CryptonightR_instruction161) +PUBLIC FN_PREFIX(CryptonightR_instruction162) +PUBLIC FN_PREFIX(CryptonightR_instruction163) +PUBLIC FN_PREFIX(CryptonightR_instruction164) +PUBLIC FN_PREFIX(CryptonightR_instruction165) +PUBLIC FN_PREFIX(CryptonightR_instruction166) +PUBLIC FN_PREFIX(CryptonightR_instruction167) +PUBLIC FN_PREFIX(CryptonightR_instruction168) +PUBLIC FN_PREFIX(CryptonightR_instruction169) +PUBLIC FN_PREFIX(CryptonightR_instruction170) +PUBLIC FN_PREFIX(CryptonightR_instruction171) +PUBLIC FN_PREFIX(CryptonightR_instruction172) +PUBLIC FN_PREFIX(CryptonightR_instruction173) +PUBLIC FN_PREFIX(CryptonightR_instruction174) +PUBLIC FN_PREFIX(CryptonightR_instruction175) +PUBLIC FN_PREFIX(CryptonightR_instruction176) +PUBLIC FN_PREFIX(CryptonightR_instruction177) +PUBLIC FN_PREFIX(CryptonightR_instruction178) +PUBLIC FN_PREFIX(CryptonightR_instruction179) +PUBLIC FN_PREFIX(CryptonightR_instruction180) +PUBLIC FN_PREFIX(CryptonightR_instruction181) +PUBLIC FN_PREFIX(CryptonightR_instruction182) +PUBLIC FN_PREFIX(CryptonightR_instruction183) +PUBLIC FN_PREFIX(CryptonightR_instruction184) +PUBLIC FN_PREFIX(CryptonightR_instruction185) +PUBLIC FN_PREFIX(CryptonightR_instruction186) +PUBLIC FN_PREFIX(CryptonightR_instruction187) +PUBLIC FN_PREFIX(CryptonightR_instruction188) +PUBLIC FN_PREFIX(CryptonightR_instruction189) +PUBLIC FN_PREFIX(CryptonightR_instruction190) +PUBLIC FN_PREFIX(CryptonightR_instruction191) +PUBLIC FN_PREFIX(CryptonightR_instruction192) +PUBLIC FN_PREFIX(CryptonightR_instruction193) +PUBLIC FN_PREFIX(CryptonightR_instruction194) +PUBLIC FN_PREFIX(CryptonightR_instruction195) +PUBLIC FN_PREFIX(CryptonightR_instruction196) +PUBLIC FN_PREFIX(CryptonightR_instruction197) +PUBLIC FN_PREFIX(CryptonightR_instruction198) +PUBLIC FN_PREFIX(CryptonightR_instruction199) +PUBLIC FN_PREFIX(CryptonightR_instruction200) +PUBLIC FN_PREFIX(CryptonightR_instruction201) +PUBLIC FN_PREFIX(CryptonightR_instruction202) +PUBLIC FN_PREFIX(CryptonightR_instruction203) +PUBLIC FN_PREFIX(CryptonightR_instruction204) +PUBLIC FN_PREFIX(CryptonightR_instruction205) +PUBLIC FN_PREFIX(CryptonightR_instruction206) +PUBLIC FN_PREFIX(CryptonightR_instruction207) +PUBLIC FN_PREFIX(CryptonightR_instruction208) +PUBLIC FN_PREFIX(CryptonightR_instruction209) +PUBLIC FN_PREFIX(CryptonightR_instruction210) +PUBLIC FN_PREFIX(CryptonightR_instruction211) +PUBLIC FN_PREFIX(CryptonightR_instruction212) +PUBLIC FN_PREFIX(CryptonightR_instruction213) +PUBLIC FN_PREFIX(CryptonightR_instruction214) +PUBLIC FN_PREFIX(CryptonightR_instruction215) +PUBLIC FN_PREFIX(CryptonightR_instruction216) +PUBLIC FN_PREFIX(CryptonightR_instruction217) +PUBLIC FN_PREFIX(CryptonightR_instruction218) +PUBLIC FN_PREFIX(CryptonightR_instruction219) +PUBLIC FN_PREFIX(CryptonightR_instruction220) +PUBLIC FN_PREFIX(CryptonightR_instruction221) +PUBLIC FN_PREFIX(CryptonightR_instruction222) +PUBLIC FN_PREFIX(CryptonightR_instruction223) +PUBLIC FN_PREFIX(CryptonightR_instruction224) +PUBLIC FN_PREFIX(CryptonightR_instruction225) +PUBLIC FN_PREFIX(CryptonightR_instruction226) +PUBLIC FN_PREFIX(CryptonightR_instruction227) +PUBLIC FN_PREFIX(CryptonightR_instruction228) +PUBLIC FN_PREFIX(CryptonightR_instruction229) +PUBLIC FN_PREFIX(CryptonightR_instruction230) +PUBLIC FN_PREFIX(CryptonightR_instruction231) +PUBLIC FN_PREFIX(CryptonightR_instruction232) +PUBLIC FN_PREFIX(CryptonightR_instruction233) +PUBLIC FN_PREFIX(CryptonightR_instruction234) +PUBLIC FN_PREFIX(CryptonightR_instruction235) +PUBLIC FN_PREFIX(CryptonightR_instruction236) +PUBLIC FN_PREFIX(CryptonightR_instruction237) +PUBLIC FN_PREFIX(CryptonightR_instruction238) +PUBLIC FN_PREFIX(CryptonightR_instruction239) +PUBLIC FN_PREFIX(CryptonightR_instruction240) +PUBLIC FN_PREFIX(CryptonightR_instruction241) +PUBLIC FN_PREFIX(CryptonightR_instruction242) +PUBLIC FN_PREFIX(CryptonightR_instruction243) +PUBLIC FN_PREFIX(CryptonightR_instruction244) +PUBLIC FN_PREFIX(CryptonightR_instruction245) +PUBLIC FN_PREFIX(CryptonightR_instruction246) +PUBLIC FN_PREFIX(CryptonightR_instruction247) +PUBLIC FN_PREFIX(CryptonightR_instruction248) +PUBLIC FN_PREFIX(CryptonightR_instruction249) +PUBLIC FN_PREFIX(CryptonightR_instruction250) +PUBLIC FN_PREFIX(CryptonightR_instruction251) +PUBLIC FN_PREFIX(CryptonightR_instruction252) +PUBLIC FN_PREFIX(CryptonightR_instruction253) +PUBLIC FN_PREFIX(CryptonightR_instruction254) +PUBLIC FN_PREFIX(CryptonightR_instruction255) +PUBLIC FN_PREFIX(CryptonightR_instruction256) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov0) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov1) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov2) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov3) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov4) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov5) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov6) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov7) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov8) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov9) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov10) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov11) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov12) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov13) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov14) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov15) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov16) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov17) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov18) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov19) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov20) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov21) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov22) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov23) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov24) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov25) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov26) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov27) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov28) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov29) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov30) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov31) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov32) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov33) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov34) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov35) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov36) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov37) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov38) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov39) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov40) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov41) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov42) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov43) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov44) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov45) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov46) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov47) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov48) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov49) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov50) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov51) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov52) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov53) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov54) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov55) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov56) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov57) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov58) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov59) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov60) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov61) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov62) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov63) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov64) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov65) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov66) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov67) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov68) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov69) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov70) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov71) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov72) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov73) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov74) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov75) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov76) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov77) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov78) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov79) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov80) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov81) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov82) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov83) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov84) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov85) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov86) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov87) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov88) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov89) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov90) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov91) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov92) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov93) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov94) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov95) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov96) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov97) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov98) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov99) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov100) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov101) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov102) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov103) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov104) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov105) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov106) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov107) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov108) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov109) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov110) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov111) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov112) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov113) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov114) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov115) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov116) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov117) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov118) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov119) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov120) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov121) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov122) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov123) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov124) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov125) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov126) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov127) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov128) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov129) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov130) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov131) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov132) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov133) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov134) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov135) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov136) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov137) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov138) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov139) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov140) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov141) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov142) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov143) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov144) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov145) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov146) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov147) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov148) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov149) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov150) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov151) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov152) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov153) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov154) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov155) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov156) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov157) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov158) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov159) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov160) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov161) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov162) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov163) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov164) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov165) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov166) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov167) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov168) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov169) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov170) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov171) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov172) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov173) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov174) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov175) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov176) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov177) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov178) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov179) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov180) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov181) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov182) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov183) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov184) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov185) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov186) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov187) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov188) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov189) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov190) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov191) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov192) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov193) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov194) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov195) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov196) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov197) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov198) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov199) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov200) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov201) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov202) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov203) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov204) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov205) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov206) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov207) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov208) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov209) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov210) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov211) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov212) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov213) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov214) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov215) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov216) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov217) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov218) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov219) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov220) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov221) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov222) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov223) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov224) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov225) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov226) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov227) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov228) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov229) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov230) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov231) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov232) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov233) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov234) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov235) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov236) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov237) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov238) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov239) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov240) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov241) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov242) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov243) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov244) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov245) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov246) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov247) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov248) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov249) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov250) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov251) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov252) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov253) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov254) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov255) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov256) + +FN_PREFIX(CryptonightR_instruction0): + imul ebx, ebx +FN_PREFIX(CryptonightR_instruction1): + imul ebx, ebx +FN_PREFIX(CryptonightR_instruction2): + imul ebx, ebx +FN_PREFIX(CryptonightR_instruction3): + add ebx, r9d + add ebx, 2147483647 +FN_PREFIX(CryptonightR_instruction4): + sub ebx, r9d +FN_PREFIX(CryptonightR_instruction5): + ror ebx, cl +FN_PREFIX(CryptonightR_instruction6): + rol ebx, cl +FN_PREFIX(CryptonightR_instruction7): + xor ebx, r9d +FN_PREFIX(CryptonightR_instruction8): + imul esi, ebx +FN_PREFIX(CryptonightR_instruction9): + imul esi, ebx +FN_PREFIX(CryptonightR_instruction10): + imul esi, ebx +FN_PREFIX(CryptonightR_instruction11): + add esi, ebx + add esi, 2147483647 +FN_PREFIX(CryptonightR_instruction12): + sub esi, ebx +FN_PREFIX(CryptonightR_instruction13): + ror esi, cl +FN_PREFIX(CryptonightR_instruction14): + rol esi, cl +FN_PREFIX(CryptonightR_instruction15): + xor esi, ebx +FN_PREFIX(CryptonightR_instruction16): + imul edi, ebx +FN_PREFIX(CryptonightR_instruction17): + imul edi, ebx +FN_PREFIX(CryptonightR_instruction18): + imul edi, ebx +FN_PREFIX(CryptonightR_instruction19): + add edi, ebx + add edi, 2147483647 +FN_PREFIX(CryptonightR_instruction20): + sub edi, ebx +FN_PREFIX(CryptonightR_instruction21): + ror edi, cl +FN_PREFIX(CryptonightR_instruction22): + rol edi, cl +FN_PREFIX(CryptonightR_instruction23): + xor edi, ebx +FN_PREFIX(CryptonightR_instruction24): + imul ebp, ebx +FN_PREFIX(CryptonightR_instruction25): + imul ebp, ebx +FN_PREFIX(CryptonightR_instruction26): + imul ebp, ebx +FN_PREFIX(CryptonightR_instruction27): + add ebp, ebx + add ebp, 2147483647 +FN_PREFIX(CryptonightR_instruction28): + sub ebp, ebx +FN_PREFIX(CryptonightR_instruction29): + ror ebp, cl +FN_PREFIX(CryptonightR_instruction30): + rol ebp, cl +FN_PREFIX(CryptonightR_instruction31): + xor ebp, ebx +FN_PREFIX(CryptonightR_instruction32): + imul ebx, esi +FN_PREFIX(CryptonightR_instruction33): + imul ebx, esi +FN_PREFIX(CryptonightR_instruction34): + imul ebx, esi +FN_PREFIX(CryptonightR_instruction35): + add ebx, esi + add ebx, 2147483647 +FN_PREFIX(CryptonightR_instruction36): + sub ebx, esi +FN_PREFIX(CryptonightR_instruction37): + ror ebx, cl +FN_PREFIX(CryptonightR_instruction38): + rol ebx, cl +FN_PREFIX(CryptonightR_instruction39): + xor ebx, esi +FN_PREFIX(CryptonightR_instruction40): + imul esi, esi +FN_PREFIX(CryptonightR_instruction41): + imul esi, esi +FN_PREFIX(CryptonightR_instruction42): + imul esi, esi +FN_PREFIX(CryptonightR_instruction43): + add esi, r9d + add esi, 2147483647 +FN_PREFIX(CryptonightR_instruction44): + sub esi, r9d +FN_PREFIX(CryptonightR_instruction45): + ror esi, cl +FN_PREFIX(CryptonightR_instruction46): + rol esi, cl +FN_PREFIX(CryptonightR_instruction47): + xor esi, r9d +FN_PREFIX(CryptonightR_instruction48): + imul edi, esi +FN_PREFIX(CryptonightR_instruction49): + imul edi, esi +FN_PREFIX(CryptonightR_instruction50): + imul edi, esi +FN_PREFIX(CryptonightR_instruction51): + add edi, esi + add edi, 2147483647 +FN_PREFIX(CryptonightR_instruction52): + sub edi, esi +FN_PREFIX(CryptonightR_instruction53): + ror edi, cl +FN_PREFIX(CryptonightR_instruction54): + rol edi, cl +FN_PREFIX(CryptonightR_instruction55): + xor edi, esi +FN_PREFIX(CryptonightR_instruction56): + imul ebp, esi +FN_PREFIX(CryptonightR_instruction57): + imul ebp, esi +FN_PREFIX(CryptonightR_instruction58): + imul ebp, esi +FN_PREFIX(CryptonightR_instruction59): + add ebp, esi + add ebp, 2147483647 +FN_PREFIX(CryptonightR_instruction60): + sub ebp, esi +FN_PREFIX(CryptonightR_instruction61): + ror ebp, cl +FN_PREFIX(CryptonightR_instruction62): + rol ebp, cl +FN_PREFIX(CryptonightR_instruction63): + xor ebp, esi +FN_PREFIX(CryptonightR_instruction64): + imul ebx, edi +FN_PREFIX(CryptonightR_instruction65): + imul ebx, edi +FN_PREFIX(CryptonightR_instruction66): + imul ebx, edi +FN_PREFIX(CryptonightR_instruction67): + add ebx, edi + add ebx, 2147483647 +FN_PREFIX(CryptonightR_instruction68): + sub ebx, edi +FN_PREFIX(CryptonightR_instruction69): + ror ebx, cl +FN_PREFIX(CryptonightR_instruction70): + rol ebx, cl +FN_PREFIX(CryptonightR_instruction71): + xor ebx, edi +FN_PREFIX(CryptonightR_instruction72): + imul esi, edi +FN_PREFIX(CryptonightR_instruction73): + imul esi, edi +FN_PREFIX(CryptonightR_instruction74): + imul esi, edi +FN_PREFIX(CryptonightR_instruction75): + add esi, edi + add esi, 2147483647 +FN_PREFIX(CryptonightR_instruction76): + sub esi, edi +FN_PREFIX(CryptonightR_instruction77): + ror esi, cl +FN_PREFIX(CryptonightR_instruction78): + rol esi, cl +FN_PREFIX(CryptonightR_instruction79): + xor esi, edi +FN_PREFIX(CryptonightR_instruction80): + imul edi, edi +FN_PREFIX(CryptonightR_instruction81): + imul edi, edi +FN_PREFIX(CryptonightR_instruction82): + imul edi, edi +FN_PREFIX(CryptonightR_instruction83): + add edi, r9d + add edi, 2147483647 +FN_PREFIX(CryptonightR_instruction84): + sub edi, r9d +FN_PREFIX(CryptonightR_instruction85): + ror edi, cl +FN_PREFIX(CryptonightR_instruction86): + rol edi, cl +FN_PREFIX(CryptonightR_instruction87): + xor edi, r9d +FN_PREFIX(CryptonightR_instruction88): + imul ebp, edi +FN_PREFIX(CryptonightR_instruction89): + imul ebp, edi +FN_PREFIX(CryptonightR_instruction90): + imul ebp, edi +FN_PREFIX(CryptonightR_instruction91): + add ebp, edi + add ebp, 2147483647 +FN_PREFIX(CryptonightR_instruction92): + sub ebp, edi +FN_PREFIX(CryptonightR_instruction93): + ror ebp, cl +FN_PREFIX(CryptonightR_instruction94): + rol ebp, cl +FN_PREFIX(CryptonightR_instruction95): + xor ebp, edi +FN_PREFIX(CryptonightR_instruction96): + imul ebx, ebp +FN_PREFIX(CryptonightR_instruction97): + imul ebx, ebp +FN_PREFIX(CryptonightR_instruction98): + imul ebx, ebp +FN_PREFIX(CryptonightR_instruction99): + add ebx, ebp + add ebx, 2147483647 +FN_PREFIX(CryptonightR_instruction100): + sub ebx, ebp +FN_PREFIX(CryptonightR_instruction101): + ror ebx, cl +FN_PREFIX(CryptonightR_instruction102): + rol ebx, cl +FN_PREFIX(CryptonightR_instruction103): + xor ebx, ebp +FN_PREFIX(CryptonightR_instruction104): + imul esi, ebp +FN_PREFIX(CryptonightR_instruction105): + imul esi, ebp +FN_PREFIX(CryptonightR_instruction106): + imul esi, ebp +FN_PREFIX(CryptonightR_instruction107): + add esi, ebp + add esi, 2147483647 +FN_PREFIX(CryptonightR_instruction108): + sub esi, ebp +FN_PREFIX(CryptonightR_instruction109): + ror esi, cl +FN_PREFIX(CryptonightR_instruction110): + rol esi, cl +FN_PREFIX(CryptonightR_instruction111): + xor esi, ebp +FN_PREFIX(CryptonightR_instruction112): + imul edi, ebp +FN_PREFIX(CryptonightR_instruction113): + imul edi, ebp +FN_PREFIX(CryptonightR_instruction114): + imul edi, ebp +FN_PREFIX(CryptonightR_instruction115): + add edi, ebp + add edi, 2147483647 +FN_PREFIX(CryptonightR_instruction116): + sub edi, ebp +FN_PREFIX(CryptonightR_instruction117): + ror edi, cl +FN_PREFIX(CryptonightR_instruction118): + rol edi, cl +FN_PREFIX(CryptonightR_instruction119): + xor edi, ebp +FN_PREFIX(CryptonightR_instruction120): + imul ebp, ebp +FN_PREFIX(CryptonightR_instruction121): + imul ebp, ebp +FN_PREFIX(CryptonightR_instruction122): + imul ebp, ebp +FN_PREFIX(CryptonightR_instruction123): + add ebp, r9d + add ebp, 2147483647 +FN_PREFIX(CryptonightR_instruction124): + sub ebp, r9d +FN_PREFIX(CryptonightR_instruction125): + ror ebp, cl +FN_PREFIX(CryptonightR_instruction126): + rol ebp, cl +FN_PREFIX(CryptonightR_instruction127): + xor ebp, r9d +FN_PREFIX(CryptonightR_instruction128): + imul ebx, esp +FN_PREFIX(CryptonightR_instruction129): + imul ebx, esp +FN_PREFIX(CryptonightR_instruction130): + imul ebx, esp +FN_PREFIX(CryptonightR_instruction131): + add ebx, esp + add ebx, 2147483647 +FN_PREFIX(CryptonightR_instruction132): + sub ebx, esp +FN_PREFIX(CryptonightR_instruction133): + ror ebx, cl +FN_PREFIX(CryptonightR_instruction134): + rol ebx, cl +FN_PREFIX(CryptonightR_instruction135): + xor ebx, esp +FN_PREFIX(CryptonightR_instruction136): + imul esi, esp +FN_PREFIX(CryptonightR_instruction137): + imul esi, esp +FN_PREFIX(CryptonightR_instruction138): + imul esi, esp +FN_PREFIX(CryptonightR_instruction139): + add esi, esp + add esi, 2147483647 +FN_PREFIX(CryptonightR_instruction140): + sub esi, esp +FN_PREFIX(CryptonightR_instruction141): + ror esi, cl +FN_PREFIX(CryptonightR_instruction142): + rol esi, cl +FN_PREFIX(CryptonightR_instruction143): + xor esi, esp +FN_PREFIX(CryptonightR_instruction144): + imul edi, esp +FN_PREFIX(CryptonightR_instruction145): + imul edi, esp +FN_PREFIX(CryptonightR_instruction146): + imul edi, esp +FN_PREFIX(CryptonightR_instruction147): + add edi, esp + add edi, 2147483647 +FN_PREFIX(CryptonightR_instruction148): + sub edi, esp +FN_PREFIX(CryptonightR_instruction149): + ror edi, cl +FN_PREFIX(CryptonightR_instruction150): + rol edi, cl +FN_PREFIX(CryptonightR_instruction151): + xor edi, esp +FN_PREFIX(CryptonightR_instruction152): + imul ebp, esp +FN_PREFIX(CryptonightR_instruction153): + imul ebp, esp +FN_PREFIX(CryptonightR_instruction154): + imul ebp, esp +FN_PREFIX(CryptonightR_instruction155): + add ebp, esp + add ebp, 2147483647 +FN_PREFIX(CryptonightR_instruction156): + sub ebp, esp +FN_PREFIX(CryptonightR_instruction157): + ror ebp, cl +FN_PREFIX(CryptonightR_instruction158): + rol ebp, cl +FN_PREFIX(CryptonightR_instruction159): + xor ebp, esp +FN_PREFIX(CryptonightR_instruction160): + imul ebx, r15d +FN_PREFIX(CryptonightR_instruction161): + imul ebx, r15d +FN_PREFIX(CryptonightR_instruction162): + imul ebx, r15d +FN_PREFIX(CryptonightR_instruction163): + add ebx, r15d + add ebx, 2147483647 +FN_PREFIX(CryptonightR_instruction164): + sub ebx, r15d +FN_PREFIX(CryptonightR_instruction165): + ror ebx, cl +FN_PREFIX(CryptonightR_instruction166): + rol ebx, cl +FN_PREFIX(CryptonightR_instruction167): + xor ebx, r15d +FN_PREFIX(CryptonightR_instruction168): + imul esi, r15d +FN_PREFIX(CryptonightR_instruction169): + imul esi, r15d +FN_PREFIX(CryptonightR_instruction170): + imul esi, r15d +FN_PREFIX(CryptonightR_instruction171): + add esi, r15d + add esi, 2147483647 +FN_PREFIX(CryptonightR_instruction172): + sub esi, r15d +FN_PREFIX(CryptonightR_instruction173): + ror esi, cl +FN_PREFIX(CryptonightR_instruction174): + rol esi, cl +FN_PREFIX(CryptonightR_instruction175): + xor esi, r15d +FN_PREFIX(CryptonightR_instruction176): + imul edi, r15d +FN_PREFIX(CryptonightR_instruction177): + imul edi, r15d +FN_PREFIX(CryptonightR_instruction178): + imul edi, r15d +FN_PREFIX(CryptonightR_instruction179): + add edi, r15d + add edi, 2147483647 +FN_PREFIX(CryptonightR_instruction180): + sub edi, r15d +FN_PREFIX(CryptonightR_instruction181): + ror edi, cl +FN_PREFIX(CryptonightR_instruction182): + rol edi, cl +FN_PREFIX(CryptonightR_instruction183): + xor edi, r15d +FN_PREFIX(CryptonightR_instruction184): + imul ebp, r15d +FN_PREFIX(CryptonightR_instruction185): + imul ebp, r15d +FN_PREFIX(CryptonightR_instruction186): + imul ebp, r15d +FN_PREFIX(CryptonightR_instruction187): + add ebp, r15d + add ebp, 2147483647 +FN_PREFIX(CryptonightR_instruction188): + sub ebp, r15d +FN_PREFIX(CryptonightR_instruction189): + ror ebp, cl +FN_PREFIX(CryptonightR_instruction190): + rol ebp, cl +FN_PREFIX(CryptonightR_instruction191): + xor ebp, r15d +FN_PREFIX(CryptonightR_instruction192): + imul ebx, eax +FN_PREFIX(CryptonightR_instruction193): + imul ebx, eax +FN_PREFIX(CryptonightR_instruction194): + imul ebx, eax +FN_PREFIX(CryptonightR_instruction195): + add ebx, eax + add ebx, 2147483647 +FN_PREFIX(CryptonightR_instruction196): + sub ebx, eax +FN_PREFIX(CryptonightR_instruction197): + ror ebx, cl +FN_PREFIX(CryptonightR_instruction198): + rol ebx, cl +FN_PREFIX(CryptonightR_instruction199): + xor ebx, eax +FN_PREFIX(CryptonightR_instruction200): + imul esi, eax +FN_PREFIX(CryptonightR_instruction201): + imul esi, eax +FN_PREFIX(CryptonightR_instruction202): + imul esi, eax +FN_PREFIX(CryptonightR_instruction203): + add esi, eax + add esi, 2147483647 +FN_PREFIX(CryptonightR_instruction204): + sub esi, eax +FN_PREFIX(CryptonightR_instruction205): + ror esi, cl +FN_PREFIX(CryptonightR_instruction206): + rol esi, cl +FN_PREFIX(CryptonightR_instruction207): + xor esi, eax +FN_PREFIX(CryptonightR_instruction208): + imul edi, eax +FN_PREFIX(CryptonightR_instruction209): + imul edi, eax +FN_PREFIX(CryptonightR_instruction210): + imul edi, eax +FN_PREFIX(CryptonightR_instruction211): + add edi, eax + add edi, 2147483647 +FN_PREFIX(CryptonightR_instruction212): + sub edi, eax +FN_PREFIX(CryptonightR_instruction213): + ror edi, cl +FN_PREFIX(CryptonightR_instruction214): + rol edi, cl +FN_PREFIX(CryptonightR_instruction215): + xor edi, eax +FN_PREFIX(CryptonightR_instruction216): + imul ebp, eax +FN_PREFIX(CryptonightR_instruction217): + imul ebp, eax +FN_PREFIX(CryptonightR_instruction218): + imul ebp, eax +FN_PREFIX(CryptonightR_instruction219): + add ebp, eax + add ebp, 2147483647 +FN_PREFIX(CryptonightR_instruction220): + sub ebp, eax +FN_PREFIX(CryptonightR_instruction221): + ror ebp, cl +FN_PREFIX(CryptonightR_instruction222): + rol ebp, cl +FN_PREFIX(CryptonightR_instruction223): + xor ebp, eax +FN_PREFIX(CryptonightR_instruction224): + imul ebx, edx +FN_PREFIX(CryptonightR_instruction225): + imul ebx, edx +FN_PREFIX(CryptonightR_instruction226): + imul ebx, edx +FN_PREFIX(CryptonightR_instruction227): + add ebx, edx + add ebx, 2147483647 +FN_PREFIX(CryptonightR_instruction228): + sub ebx, edx +FN_PREFIX(CryptonightR_instruction229): + ror ebx, cl +FN_PREFIX(CryptonightR_instruction230): + rol ebx, cl +FN_PREFIX(CryptonightR_instruction231): + xor ebx, edx +FN_PREFIX(CryptonightR_instruction232): + imul esi, edx +FN_PREFIX(CryptonightR_instruction233): + imul esi, edx +FN_PREFIX(CryptonightR_instruction234): + imul esi, edx +FN_PREFIX(CryptonightR_instruction235): + add esi, edx + add esi, 2147483647 +FN_PREFIX(CryptonightR_instruction236): + sub esi, edx +FN_PREFIX(CryptonightR_instruction237): + ror esi, cl +FN_PREFIX(CryptonightR_instruction238): + rol esi, cl +FN_PREFIX(CryptonightR_instruction239): + xor esi, edx +FN_PREFIX(CryptonightR_instruction240): + imul edi, edx +FN_PREFIX(CryptonightR_instruction241): + imul edi, edx +FN_PREFIX(CryptonightR_instruction242): + imul edi, edx +FN_PREFIX(CryptonightR_instruction243): + add edi, edx + add edi, 2147483647 +FN_PREFIX(CryptonightR_instruction244): + sub edi, edx +FN_PREFIX(CryptonightR_instruction245): + ror edi, cl +FN_PREFIX(CryptonightR_instruction246): + rol edi, cl +FN_PREFIX(CryptonightR_instruction247): + xor edi, edx +FN_PREFIX(CryptonightR_instruction248): + imul ebp, edx +FN_PREFIX(CryptonightR_instruction249): + imul ebp, edx +FN_PREFIX(CryptonightR_instruction250): + imul ebp, edx +FN_PREFIX(CryptonightR_instruction251): + add ebp, edx + add ebp, 2147483647 +FN_PREFIX(CryptonightR_instruction252): + sub ebp, edx +FN_PREFIX(CryptonightR_instruction253): + ror ebp, cl +FN_PREFIX(CryptonightR_instruction254): + rol ebp, cl +FN_PREFIX(CryptonightR_instruction255): + xor ebp, edx +FN_PREFIX(CryptonightR_instruction256): + imul ebx, ebx +FN_PREFIX(CryptonightR_instruction_mov0): + +FN_PREFIX(CryptonightR_instruction_mov1): + +FN_PREFIX(CryptonightR_instruction_mov2): + +FN_PREFIX(CryptonightR_instruction_mov3): + +FN_PREFIX(CryptonightR_instruction_mov4): + +FN_PREFIX(CryptonightR_instruction_mov5): + mov ecx, ebx +FN_PREFIX(CryptonightR_instruction_mov6): + mov ecx, ebx +FN_PREFIX(CryptonightR_instruction_mov7): + +FN_PREFIX(CryptonightR_instruction_mov8): + +FN_PREFIX(CryptonightR_instruction_mov9): + +FN_PREFIX(CryptonightR_instruction_mov10): + +FN_PREFIX(CryptonightR_instruction_mov11): + +FN_PREFIX(CryptonightR_instruction_mov12): + +FN_PREFIX(CryptonightR_instruction_mov13): + mov ecx, ebx +FN_PREFIX(CryptonightR_instruction_mov14): + mov ecx, ebx +FN_PREFIX(CryptonightR_instruction_mov15): + +FN_PREFIX(CryptonightR_instruction_mov16): + +FN_PREFIX(CryptonightR_instruction_mov17): + +FN_PREFIX(CryptonightR_instruction_mov18): + +FN_PREFIX(CryptonightR_instruction_mov19): + +FN_PREFIX(CryptonightR_instruction_mov20): + +FN_PREFIX(CryptonightR_instruction_mov21): + mov ecx, ebx +FN_PREFIX(CryptonightR_instruction_mov22): + mov ecx, ebx +FN_PREFIX(CryptonightR_instruction_mov23): + +FN_PREFIX(CryptonightR_instruction_mov24): + +FN_PREFIX(CryptonightR_instruction_mov25): + +FN_PREFIX(CryptonightR_instruction_mov26): + +FN_PREFIX(CryptonightR_instruction_mov27): + +FN_PREFIX(CryptonightR_instruction_mov28): + +FN_PREFIX(CryptonightR_instruction_mov29): + mov ecx, ebx +FN_PREFIX(CryptonightR_instruction_mov30): + mov ecx, ebx +FN_PREFIX(CryptonightR_instruction_mov31): + +FN_PREFIX(CryptonightR_instruction_mov32): + +FN_PREFIX(CryptonightR_instruction_mov33): + +FN_PREFIX(CryptonightR_instruction_mov34): + +FN_PREFIX(CryptonightR_instruction_mov35): + +FN_PREFIX(CryptonightR_instruction_mov36): + +FN_PREFIX(CryptonightR_instruction_mov37): + mov ecx, esi +FN_PREFIX(CryptonightR_instruction_mov38): + mov ecx, esi +FN_PREFIX(CryptonightR_instruction_mov39): + +FN_PREFIX(CryptonightR_instruction_mov40): + +FN_PREFIX(CryptonightR_instruction_mov41): + +FN_PREFIX(CryptonightR_instruction_mov42): + +FN_PREFIX(CryptonightR_instruction_mov43): + +FN_PREFIX(CryptonightR_instruction_mov44): + +FN_PREFIX(CryptonightR_instruction_mov45): + mov ecx, esi +FN_PREFIX(CryptonightR_instruction_mov46): + mov ecx, esi +FN_PREFIX(CryptonightR_instruction_mov47): + +FN_PREFIX(CryptonightR_instruction_mov48): + +FN_PREFIX(CryptonightR_instruction_mov49): + +FN_PREFIX(CryptonightR_instruction_mov50): + +FN_PREFIX(CryptonightR_instruction_mov51): + +FN_PREFIX(CryptonightR_instruction_mov52): + +FN_PREFIX(CryptonightR_instruction_mov53): + mov ecx, esi +FN_PREFIX(CryptonightR_instruction_mov54): + mov ecx, esi +FN_PREFIX(CryptonightR_instruction_mov55): + +FN_PREFIX(CryptonightR_instruction_mov56): + +FN_PREFIX(CryptonightR_instruction_mov57): + +FN_PREFIX(CryptonightR_instruction_mov58): + +FN_PREFIX(CryptonightR_instruction_mov59): + +FN_PREFIX(CryptonightR_instruction_mov60): + +FN_PREFIX(CryptonightR_instruction_mov61): + mov ecx, esi +FN_PREFIX(CryptonightR_instruction_mov62): + mov ecx, esi +FN_PREFIX(CryptonightR_instruction_mov63): + +FN_PREFIX(CryptonightR_instruction_mov64): + +FN_PREFIX(CryptonightR_instruction_mov65): + +FN_PREFIX(CryptonightR_instruction_mov66): + +FN_PREFIX(CryptonightR_instruction_mov67): + +FN_PREFIX(CryptonightR_instruction_mov68): + +FN_PREFIX(CryptonightR_instruction_mov69): + mov ecx, edi +FN_PREFIX(CryptonightR_instruction_mov70): + mov ecx, edi +FN_PREFIX(CryptonightR_instruction_mov71): + +FN_PREFIX(CryptonightR_instruction_mov72): + +FN_PREFIX(CryptonightR_instruction_mov73): + +FN_PREFIX(CryptonightR_instruction_mov74): + +FN_PREFIX(CryptonightR_instruction_mov75): + +FN_PREFIX(CryptonightR_instruction_mov76): + +FN_PREFIX(CryptonightR_instruction_mov77): + mov ecx, edi +FN_PREFIX(CryptonightR_instruction_mov78): + mov ecx, edi +FN_PREFIX(CryptonightR_instruction_mov79): + +FN_PREFIX(CryptonightR_instruction_mov80): + +FN_PREFIX(CryptonightR_instruction_mov81): + +FN_PREFIX(CryptonightR_instruction_mov82): + +FN_PREFIX(CryptonightR_instruction_mov83): + +FN_PREFIX(CryptonightR_instruction_mov84): + +FN_PREFIX(CryptonightR_instruction_mov85): + mov ecx, edi +FN_PREFIX(CryptonightR_instruction_mov86): + mov ecx, edi +FN_PREFIX(CryptonightR_instruction_mov87): + +FN_PREFIX(CryptonightR_instruction_mov88): + +FN_PREFIX(CryptonightR_instruction_mov89): + +FN_PREFIX(CryptonightR_instruction_mov90): + +FN_PREFIX(CryptonightR_instruction_mov91): + +FN_PREFIX(CryptonightR_instruction_mov92): + +FN_PREFIX(CryptonightR_instruction_mov93): + mov ecx, edi +FN_PREFIX(CryptonightR_instruction_mov94): + mov ecx, edi +FN_PREFIX(CryptonightR_instruction_mov95): + +FN_PREFIX(CryptonightR_instruction_mov96): + +FN_PREFIX(CryptonightR_instruction_mov97): + +FN_PREFIX(CryptonightR_instruction_mov98): + +FN_PREFIX(CryptonightR_instruction_mov99): + +FN_PREFIX(CryptonightR_instruction_mov100): + +FN_PREFIX(CryptonightR_instruction_mov101): + mov ecx, ebp +FN_PREFIX(CryptonightR_instruction_mov102): + mov ecx, ebp +FN_PREFIX(CryptonightR_instruction_mov103): + +FN_PREFIX(CryptonightR_instruction_mov104): + +FN_PREFIX(CryptonightR_instruction_mov105): + +FN_PREFIX(CryptonightR_instruction_mov106): + +FN_PREFIX(CryptonightR_instruction_mov107): + +FN_PREFIX(CryptonightR_instruction_mov108): + +FN_PREFIX(CryptonightR_instruction_mov109): + mov ecx, ebp +FN_PREFIX(CryptonightR_instruction_mov110): + mov ecx, ebp +FN_PREFIX(CryptonightR_instruction_mov111): + +FN_PREFIX(CryptonightR_instruction_mov112): + +FN_PREFIX(CryptonightR_instruction_mov113): + +FN_PREFIX(CryptonightR_instruction_mov114): + +FN_PREFIX(CryptonightR_instruction_mov115): + +FN_PREFIX(CryptonightR_instruction_mov116): + +FN_PREFIX(CryptonightR_instruction_mov117): + mov ecx, ebp +FN_PREFIX(CryptonightR_instruction_mov118): + mov ecx, ebp +FN_PREFIX(CryptonightR_instruction_mov119): + +FN_PREFIX(CryptonightR_instruction_mov120): + +FN_PREFIX(CryptonightR_instruction_mov121): + +FN_PREFIX(CryptonightR_instruction_mov122): + +FN_PREFIX(CryptonightR_instruction_mov123): + +FN_PREFIX(CryptonightR_instruction_mov124): + +FN_PREFIX(CryptonightR_instruction_mov125): + mov ecx, ebp +FN_PREFIX(CryptonightR_instruction_mov126): + mov ecx, ebp +FN_PREFIX(CryptonightR_instruction_mov127): + +FN_PREFIX(CryptonightR_instruction_mov128): + +FN_PREFIX(CryptonightR_instruction_mov129): + +FN_PREFIX(CryptonightR_instruction_mov130): + +FN_PREFIX(CryptonightR_instruction_mov131): + +FN_PREFIX(CryptonightR_instruction_mov132): + +FN_PREFIX(CryptonightR_instruction_mov133): + mov ecx, esp +FN_PREFIX(CryptonightR_instruction_mov134): + mov ecx, esp +FN_PREFIX(CryptonightR_instruction_mov135): + +FN_PREFIX(CryptonightR_instruction_mov136): + +FN_PREFIX(CryptonightR_instruction_mov137): + +FN_PREFIX(CryptonightR_instruction_mov138): + +FN_PREFIX(CryptonightR_instruction_mov139): + +FN_PREFIX(CryptonightR_instruction_mov140): + +FN_PREFIX(CryptonightR_instruction_mov141): + mov ecx, esp +FN_PREFIX(CryptonightR_instruction_mov142): + mov ecx, esp +FN_PREFIX(CryptonightR_instruction_mov143): + +FN_PREFIX(CryptonightR_instruction_mov144): + +FN_PREFIX(CryptonightR_instruction_mov145): + +FN_PREFIX(CryptonightR_instruction_mov146): + +FN_PREFIX(CryptonightR_instruction_mov147): + +FN_PREFIX(CryptonightR_instruction_mov148): + +FN_PREFIX(CryptonightR_instruction_mov149): + mov ecx, esp +FN_PREFIX(CryptonightR_instruction_mov150): + mov ecx, esp +FN_PREFIX(CryptonightR_instruction_mov151): + +FN_PREFIX(CryptonightR_instruction_mov152): + +FN_PREFIX(CryptonightR_instruction_mov153): + +FN_PREFIX(CryptonightR_instruction_mov154): + +FN_PREFIX(CryptonightR_instruction_mov155): + +FN_PREFIX(CryptonightR_instruction_mov156): + +FN_PREFIX(CryptonightR_instruction_mov157): + mov ecx, esp +FN_PREFIX(CryptonightR_instruction_mov158): + mov ecx, esp +FN_PREFIX(CryptonightR_instruction_mov159): + +FN_PREFIX(CryptonightR_instruction_mov160): + +FN_PREFIX(CryptonightR_instruction_mov161): + +FN_PREFIX(CryptonightR_instruction_mov162): + +FN_PREFIX(CryptonightR_instruction_mov163): + +FN_PREFIX(CryptonightR_instruction_mov164): + +FN_PREFIX(CryptonightR_instruction_mov165): + mov ecx, r15d +FN_PREFIX(CryptonightR_instruction_mov166): + mov ecx, r15d +FN_PREFIX(CryptonightR_instruction_mov167): + +FN_PREFIX(CryptonightR_instruction_mov168): + +FN_PREFIX(CryptonightR_instruction_mov169): + +FN_PREFIX(CryptonightR_instruction_mov170): + +FN_PREFIX(CryptonightR_instruction_mov171): + +FN_PREFIX(CryptonightR_instruction_mov172): + +FN_PREFIX(CryptonightR_instruction_mov173): + mov ecx, r15d +FN_PREFIX(CryptonightR_instruction_mov174): + mov ecx, r15d +FN_PREFIX(CryptonightR_instruction_mov175): + +FN_PREFIX(CryptonightR_instruction_mov176): + +FN_PREFIX(CryptonightR_instruction_mov177): + +FN_PREFIX(CryptonightR_instruction_mov178): + +FN_PREFIX(CryptonightR_instruction_mov179): + +FN_PREFIX(CryptonightR_instruction_mov180): + +FN_PREFIX(CryptonightR_instruction_mov181): + mov ecx, r15d +FN_PREFIX(CryptonightR_instruction_mov182): + mov ecx, r15d +FN_PREFIX(CryptonightR_instruction_mov183): + +FN_PREFIX(CryptonightR_instruction_mov184): + +FN_PREFIX(CryptonightR_instruction_mov185): + +FN_PREFIX(CryptonightR_instruction_mov186): + +FN_PREFIX(CryptonightR_instruction_mov187): + +FN_PREFIX(CryptonightR_instruction_mov188): + +FN_PREFIX(CryptonightR_instruction_mov189): + mov ecx, r15d +FN_PREFIX(CryptonightR_instruction_mov190): + mov ecx, r15d +FN_PREFIX(CryptonightR_instruction_mov191): + +FN_PREFIX(CryptonightR_instruction_mov192): + +FN_PREFIX(CryptonightR_instruction_mov193): + +FN_PREFIX(CryptonightR_instruction_mov194): + +FN_PREFIX(CryptonightR_instruction_mov195): + +FN_PREFIX(CryptonightR_instruction_mov196): + +FN_PREFIX(CryptonightR_instruction_mov197): + mov ecx, eax +FN_PREFIX(CryptonightR_instruction_mov198): + mov ecx, eax +FN_PREFIX(CryptonightR_instruction_mov199): + +FN_PREFIX(CryptonightR_instruction_mov200): + +FN_PREFIX(CryptonightR_instruction_mov201): + +FN_PREFIX(CryptonightR_instruction_mov202): + +FN_PREFIX(CryptonightR_instruction_mov203): + +FN_PREFIX(CryptonightR_instruction_mov204): + +FN_PREFIX(CryptonightR_instruction_mov205): + mov ecx, eax +FN_PREFIX(CryptonightR_instruction_mov206): + mov ecx, eax +FN_PREFIX(CryptonightR_instruction_mov207): + +FN_PREFIX(CryptonightR_instruction_mov208): + +FN_PREFIX(CryptonightR_instruction_mov209): + +FN_PREFIX(CryptonightR_instruction_mov210): + +FN_PREFIX(CryptonightR_instruction_mov211): + +FN_PREFIX(CryptonightR_instruction_mov212): + +FN_PREFIX(CryptonightR_instruction_mov213): + mov ecx, eax +FN_PREFIX(CryptonightR_instruction_mov214): + mov ecx, eax +FN_PREFIX(CryptonightR_instruction_mov215): + +FN_PREFIX(CryptonightR_instruction_mov216): + +FN_PREFIX(CryptonightR_instruction_mov217): + +FN_PREFIX(CryptonightR_instruction_mov218): + +FN_PREFIX(CryptonightR_instruction_mov219): + +FN_PREFIX(CryptonightR_instruction_mov220): + +FN_PREFIX(CryptonightR_instruction_mov221): + mov ecx, eax +FN_PREFIX(CryptonightR_instruction_mov222): + mov ecx, eax +FN_PREFIX(CryptonightR_instruction_mov223): + +FN_PREFIX(CryptonightR_instruction_mov224): + +FN_PREFIX(CryptonightR_instruction_mov225): + +FN_PREFIX(CryptonightR_instruction_mov226): + +FN_PREFIX(CryptonightR_instruction_mov227): + +FN_PREFIX(CryptonightR_instruction_mov228): + +FN_PREFIX(CryptonightR_instruction_mov229): + mov ecx, edx +FN_PREFIX(CryptonightR_instruction_mov230): + mov ecx, edx +FN_PREFIX(CryptonightR_instruction_mov231): + +FN_PREFIX(CryptonightR_instruction_mov232): + +FN_PREFIX(CryptonightR_instruction_mov233): + +FN_PREFIX(CryptonightR_instruction_mov234): + +FN_PREFIX(CryptonightR_instruction_mov235): + +FN_PREFIX(CryptonightR_instruction_mov236): + +FN_PREFIX(CryptonightR_instruction_mov237): + mov ecx, edx +FN_PREFIX(CryptonightR_instruction_mov238): + mov ecx, edx +FN_PREFIX(CryptonightR_instruction_mov239): + +FN_PREFIX(CryptonightR_instruction_mov240): + +FN_PREFIX(CryptonightR_instruction_mov241): + +FN_PREFIX(CryptonightR_instruction_mov242): + +FN_PREFIX(CryptonightR_instruction_mov243): + +FN_PREFIX(CryptonightR_instruction_mov244): + +FN_PREFIX(CryptonightR_instruction_mov245): + mov ecx, edx +FN_PREFIX(CryptonightR_instruction_mov246): + mov ecx, edx +FN_PREFIX(CryptonightR_instruction_mov247): + +FN_PREFIX(CryptonightR_instruction_mov248): + +FN_PREFIX(CryptonightR_instruction_mov249): + +FN_PREFIX(CryptonightR_instruction_mov250): + +FN_PREFIX(CryptonightR_instruction_mov251): + +FN_PREFIX(CryptonightR_instruction_mov252): + +FN_PREFIX(CryptonightR_instruction_mov253): + mov ecx, edx +FN_PREFIX(CryptonightR_instruction_mov254): + mov ecx, edx +FN_PREFIX(CryptonightR_instruction_mov255): + +FN_PREFIX(CryptonightR_instruction_mov256): diff --git a/src/crypto/CryptonightR_template.h b/src/crypto/CryptonightR_template.h new file mode 100644 index 00000000000..dab1bd75023 --- /dev/null +++ b/src/crypto/CryptonightR_template.h @@ -0,0 +1,1043 @@ +#ifndef CRYPTONIGHTR_TEMPLATE_H +#define CRYPTONIGHTR_TEMPLATE_H + +#if defined __i386 || defined __x86_64__ + +void CryptonightR_instruction0(void); +void CryptonightR_instruction1(void); +void CryptonightR_instruction2(void); +void CryptonightR_instruction3(void); +void CryptonightR_instruction4(void); +void CryptonightR_instruction5(void); +void CryptonightR_instruction6(void); +void CryptonightR_instruction7(void); +void CryptonightR_instruction8(void); +void CryptonightR_instruction9(void); +void CryptonightR_instruction10(void); +void CryptonightR_instruction11(void); +void CryptonightR_instruction12(void); +void CryptonightR_instruction13(void); +void CryptonightR_instruction14(void); +void CryptonightR_instruction15(void); +void CryptonightR_instruction16(void); +void CryptonightR_instruction17(void); +void CryptonightR_instruction18(void); +void CryptonightR_instruction19(void); +void CryptonightR_instruction20(void); +void CryptonightR_instruction21(void); +void CryptonightR_instruction22(void); +void CryptonightR_instruction23(void); +void CryptonightR_instruction24(void); +void CryptonightR_instruction25(void); +void CryptonightR_instruction26(void); +void CryptonightR_instruction27(void); +void CryptonightR_instruction28(void); +void CryptonightR_instruction29(void); +void CryptonightR_instruction30(void); +void CryptonightR_instruction31(void); +void CryptonightR_instruction32(void); +void CryptonightR_instruction33(void); +void CryptonightR_instruction34(void); +void CryptonightR_instruction35(void); +void CryptonightR_instruction36(void); +void CryptonightR_instruction37(void); +void CryptonightR_instruction38(void); +void CryptonightR_instruction39(void); +void CryptonightR_instruction40(void); +void CryptonightR_instruction41(void); +void CryptonightR_instruction42(void); +void CryptonightR_instruction43(void); +void CryptonightR_instruction44(void); +void CryptonightR_instruction45(void); +void CryptonightR_instruction46(void); +void CryptonightR_instruction47(void); +void CryptonightR_instruction48(void); +void CryptonightR_instruction49(void); +void CryptonightR_instruction50(void); +void CryptonightR_instruction51(void); +void CryptonightR_instruction52(void); +void CryptonightR_instruction53(void); +void CryptonightR_instruction54(void); +void CryptonightR_instruction55(void); +void CryptonightR_instruction56(void); +void CryptonightR_instruction57(void); +void CryptonightR_instruction58(void); +void CryptonightR_instruction59(void); +void CryptonightR_instruction60(void); +void CryptonightR_instruction61(void); +void CryptonightR_instruction62(void); +void CryptonightR_instruction63(void); +void CryptonightR_instruction64(void); +void CryptonightR_instruction65(void); +void CryptonightR_instruction66(void); +void CryptonightR_instruction67(void); +void CryptonightR_instruction68(void); +void CryptonightR_instruction69(void); +void CryptonightR_instruction70(void); +void CryptonightR_instruction71(void); +void CryptonightR_instruction72(void); +void CryptonightR_instruction73(void); +void CryptonightR_instruction74(void); +void CryptonightR_instruction75(void); +void CryptonightR_instruction76(void); +void CryptonightR_instruction77(void); +void CryptonightR_instruction78(void); +void CryptonightR_instruction79(void); +void CryptonightR_instruction80(void); +void CryptonightR_instruction81(void); +void CryptonightR_instruction82(void); +void CryptonightR_instruction83(void); +void CryptonightR_instruction84(void); +void CryptonightR_instruction85(void); +void CryptonightR_instruction86(void); +void CryptonightR_instruction87(void); +void CryptonightR_instruction88(void); +void CryptonightR_instruction89(void); +void CryptonightR_instruction90(void); +void CryptonightR_instruction91(void); +void CryptonightR_instruction92(void); +void CryptonightR_instruction93(void); +void CryptonightR_instruction94(void); +void CryptonightR_instruction95(void); +void CryptonightR_instruction96(void); +void CryptonightR_instruction97(void); +void CryptonightR_instruction98(void); +void CryptonightR_instruction99(void); +void CryptonightR_instruction100(void); +void CryptonightR_instruction101(void); +void CryptonightR_instruction102(void); +void CryptonightR_instruction103(void); +void CryptonightR_instruction104(void); +void CryptonightR_instruction105(void); +void CryptonightR_instruction106(void); +void CryptonightR_instruction107(void); +void CryptonightR_instruction108(void); +void CryptonightR_instruction109(void); +void CryptonightR_instruction110(void); +void CryptonightR_instruction111(void); +void CryptonightR_instruction112(void); +void CryptonightR_instruction113(void); +void CryptonightR_instruction114(void); +void CryptonightR_instruction115(void); +void CryptonightR_instruction116(void); +void CryptonightR_instruction117(void); +void CryptonightR_instruction118(void); +void CryptonightR_instruction119(void); +void CryptonightR_instruction120(void); +void CryptonightR_instruction121(void); +void CryptonightR_instruction122(void); +void CryptonightR_instruction123(void); +void CryptonightR_instruction124(void); +void CryptonightR_instruction125(void); +void CryptonightR_instruction126(void); +void CryptonightR_instruction127(void); +void CryptonightR_instruction128(void); +void CryptonightR_instruction129(void); +void CryptonightR_instruction130(void); +void CryptonightR_instruction131(void); +void CryptonightR_instruction132(void); +void CryptonightR_instruction133(void); +void CryptonightR_instruction134(void); +void CryptonightR_instruction135(void); +void CryptonightR_instruction136(void); +void CryptonightR_instruction137(void); +void CryptonightR_instruction138(void); +void CryptonightR_instruction139(void); +void CryptonightR_instruction140(void); +void CryptonightR_instruction141(void); +void CryptonightR_instruction142(void); +void CryptonightR_instruction143(void); +void CryptonightR_instruction144(void); +void CryptonightR_instruction145(void); +void CryptonightR_instruction146(void); +void CryptonightR_instruction147(void); +void CryptonightR_instruction148(void); +void CryptonightR_instruction149(void); +void CryptonightR_instruction150(void); +void CryptonightR_instruction151(void); +void CryptonightR_instruction152(void); +void CryptonightR_instruction153(void); +void CryptonightR_instruction154(void); +void CryptonightR_instruction155(void); +void CryptonightR_instruction156(void); +void CryptonightR_instruction157(void); +void CryptonightR_instruction158(void); +void CryptonightR_instruction159(void); +void CryptonightR_instruction160(void); +void CryptonightR_instruction161(void); +void CryptonightR_instruction162(void); +void CryptonightR_instruction163(void); +void CryptonightR_instruction164(void); +void CryptonightR_instruction165(void); +void CryptonightR_instruction166(void); +void CryptonightR_instruction167(void); +void CryptonightR_instruction168(void); +void CryptonightR_instruction169(void); +void CryptonightR_instruction170(void); +void CryptonightR_instruction171(void); +void CryptonightR_instruction172(void); +void CryptonightR_instruction173(void); +void CryptonightR_instruction174(void); +void CryptonightR_instruction175(void); +void CryptonightR_instruction176(void); +void CryptonightR_instruction177(void); +void CryptonightR_instruction178(void); +void CryptonightR_instruction179(void); +void CryptonightR_instruction180(void); +void CryptonightR_instruction181(void); +void CryptonightR_instruction182(void); +void CryptonightR_instruction183(void); +void CryptonightR_instruction184(void); +void CryptonightR_instruction185(void); +void CryptonightR_instruction186(void); +void CryptonightR_instruction187(void); +void CryptonightR_instruction188(void); +void CryptonightR_instruction189(void); +void CryptonightR_instruction190(void); +void CryptonightR_instruction191(void); +void CryptonightR_instruction192(void); +void CryptonightR_instruction193(void); +void CryptonightR_instruction194(void); +void CryptonightR_instruction195(void); +void CryptonightR_instruction196(void); +void CryptonightR_instruction197(void); +void CryptonightR_instruction198(void); +void CryptonightR_instruction199(void); +void CryptonightR_instruction200(void); +void CryptonightR_instruction201(void); +void CryptonightR_instruction202(void); +void CryptonightR_instruction203(void); +void CryptonightR_instruction204(void); +void CryptonightR_instruction205(void); +void CryptonightR_instruction206(void); +void CryptonightR_instruction207(void); +void CryptonightR_instruction208(void); +void CryptonightR_instruction209(void); +void CryptonightR_instruction210(void); +void CryptonightR_instruction211(void); +void CryptonightR_instruction212(void); +void CryptonightR_instruction213(void); +void CryptonightR_instruction214(void); +void CryptonightR_instruction215(void); +void CryptonightR_instruction216(void); +void CryptonightR_instruction217(void); +void CryptonightR_instruction218(void); +void CryptonightR_instruction219(void); +void CryptonightR_instruction220(void); +void CryptonightR_instruction221(void); +void CryptonightR_instruction222(void); +void CryptonightR_instruction223(void); +void CryptonightR_instruction224(void); +void CryptonightR_instruction225(void); +void CryptonightR_instruction226(void); +void CryptonightR_instruction227(void); +void CryptonightR_instruction228(void); +void CryptonightR_instruction229(void); +void CryptonightR_instruction230(void); +void CryptonightR_instruction231(void); +void CryptonightR_instruction232(void); +void CryptonightR_instruction233(void); +void CryptonightR_instruction234(void); +void CryptonightR_instruction235(void); +void CryptonightR_instruction236(void); +void CryptonightR_instruction237(void); +void CryptonightR_instruction238(void); +void CryptonightR_instruction239(void); +void CryptonightR_instruction240(void); +void CryptonightR_instruction241(void); +void CryptonightR_instruction242(void); +void CryptonightR_instruction243(void); +void CryptonightR_instruction244(void); +void CryptonightR_instruction245(void); +void CryptonightR_instruction246(void); +void CryptonightR_instruction247(void); +void CryptonightR_instruction248(void); +void CryptonightR_instruction249(void); +void CryptonightR_instruction250(void); +void CryptonightR_instruction251(void); +void CryptonightR_instruction252(void); +void CryptonightR_instruction253(void); +void CryptonightR_instruction254(void); +void CryptonightR_instruction255(void); +void CryptonightR_instruction256(void); +void CryptonightR_instruction_mov0(void); +void CryptonightR_instruction_mov1(void); +void CryptonightR_instruction_mov2(void); +void CryptonightR_instruction_mov3(void); +void CryptonightR_instruction_mov4(void); +void CryptonightR_instruction_mov5(void); +void CryptonightR_instruction_mov6(void); +void CryptonightR_instruction_mov7(void); +void CryptonightR_instruction_mov8(void); +void CryptonightR_instruction_mov9(void); +void CryptonightR_instruction_mov10(void); +void CryptonightR_instruction_mov11(void); +void CryptonightR_instruction_mov12(void); +void CryptonightR_instruction_mov13(void); +void CryptonightR_instruction_mov14(void); +void CryptonightR_instruction_mov15(void); +void CryptonightR_instruction_mov16(void); +void CryptonightR_instruction_mov17(void); +void CryptonightR_instruction_mov18(void); +void CryptonightR_instruction_mov19(void); +void CryptonightR_instruction_mov20(void); +void CryptonightR_instruction_mov21(void); +void CryptonightR_instruction_mov22(void); +void CryptonightR_instruction_mov23(void); +void CryptonightR_instruction_mov24(void); +void CryptonightR_instruction_mov25(void); +void CryptonightR_instruction_mov26(void); +void CryptonightR_instruction_mov27(void); +void CryptonightR_instruction_mov28(void); +void CryptonightR_instruction_mov29(void); +void CryptonightR_instruction_mov30(void); +void CryptonightR_instruction_mov31(void); +void CryptonightR_instruction_mov32(void); +void CryptonightR_instruction_mov33(void); +void CryptonightR_instruction_mov34(void); +void CryptonightR_instruction_mov35(void); +void CryptonightR_instruction_mov36(void); +void CryptonightR_instruction_mov37(void); +void CryptonightR_instruction_mov38(void); +void CryptonightR_instruction_mov39(void); +void CryptonightR_instruction_mov40(void); +void CryptonightR_instruction_mov41(void); +void CryptonightR_instruction_mov42(void); +void CryptonightR_instruction_mov43(void); +void CryptonightR_instruction_mov44(void); +void CryptonightR_instruction_mov45(void); +void CryptonightR_instruction_mov46(void); +void CryptonightR_instruction_mov47(void); +void CryptonightR_instruction_mov48(void); +void CryptonightR_instruction_mov49(void); +void CryptonightR_instruction_mov50(void); +void CryptonightR_instruction_mov51(void); +void CryptonightR_instruction_mov52(void); +void CryptonightR_instruction_mov53(void); +void CryptonightR_instruction_mov54(void); +void CryptonightR_instruction_mov55(void); +void CryptonightR_instruction_mov56(void); +void CryptonightR_instruction_mov57(void); +void CryptonightR_instruction_mov58(void); +void CryptonightR_instruction_mov59(void); +void CryptonightR_instruction_mov60(void); +void CryptonightR_instruction_mov61(void); +void CryptonightR_instruction_mov62(void); +void CryptonightR_instruction_mov63(void); +void CryptonightR_instruction_mov64(void); +void CryptonightR_instruction_mov65(void); +void CryptonightR_instruction_mov66(void); +void CryptonightR_instruction_mov67(void); +void CryptonightR_instruction_mov68(void); +void CryptonightR_instruction_mov69(void); +void CryptonightR_instruction_mov70(void); +void CryptonightR_instruction_mov71(void); +void CryptonightR_instruction_mov72(void); +void CryptonightR_instruction_mov73(void); +void CryptonightR_instruction_mov74(void); +void CryptonightR_instruction_mov75(void); +void CryptonightR_instruction_mov76(void); +void CryptonightR_instruction_mov77(void); +void CryptonightR_instruction_mov78(void); +void CryptonightR_instruction_mov79(void); +void CryptonightR_instruction_mov80(void); +void CryptonightR_instruction_mov81(void); +void CryptonightR_instruction_mov82(void); +void CryptonightR_instruction_mov83(void); +void CryptonightR_instruction_mov84(void); +void CryptonightR_instruction_mov85(void); +void CryptonightR_instruction_mov86(void); +void CryptonightR_instruction_mov87(void); +void CryptonightR_instruction_mov88(void); +void CryptonightR_instruction_mov89(void); +void CryptonightR_instruction_mov90(void); +void CryptonightR_instruction_mov91(void); +void CryptonightR_instruction_mov92(void); +void CryptonightR_instruction_mov93(void); +void CryptonightR_instruction_mov94(void); +void CryptonightR_instruction_mov95(void); +void CryptonightR_instruction_mov96(void); +void CryptonightR_instruction_mov97(void); +void CryptonightR_instruction_mov98(void); +void CryptonightR_instruction_mov99(void); +void CryptonightR_instruction_mov100(void); +void CryptonightR_instruction_mov101(void); +void CryptonightR_instruction_mov102(void); +void CryptonightR_instruction_mov103(void); +void CryptonightR_instruction_mov104(void); +void CryptonightR_instruction_mov105(void); +void CryptonightR_instruction_mov106(void); +void CryptonightR_instruction_mov107(void); +void CryptonightR_instruction_mov108(void); +void CryptonightR_instruction_mov109(void); +void CryptonightR_instruction_mov110(void); +void CryptonightR_instruction_mov111(void); +void CryptonightR_instruction_mov112(void); +void CryptonightR_instruction_mov113(void); +void CryptonightR_instruction_mov114(void); +void CryptonightR_instruction_mov115(void); +void CryptonightR_instruction_mov116(void); +void CryptonightR_instruction_mov117(void); +void CryptonightR_instruction_mov118(void); +void CryptonightR_instruction_mov119(void); +void CryptonightR_instruction_mov120(void); +void CryptonightR_instruction_mov121(void); +void CryptonightR_instruction_mov122(void); +void CryptonightR_instruction_mov123(void); +void CryptonightR_instruction_mov124(void); +void CryptonightR_instruction_mov125(void); +void CryptonightR_instruction_mov126(void); +void CryptonightR_instruction_mov127(void); +void CryptonightR_instruction_mov128(void); +void CryptonightR_instruction_mov129(void); +void CryptonightR_instruction_mov130(void); +void CryptonightR_instruction_mov131(void); +void CryptonightR_instruction_mov132(void); +void CryptonightR_instruction_mov133(void); +void CryptonightR_instruction_mov134(void); +void CryptonightR_instruction_mov135(void); +void CryptonightR_instruction_mov136(void); +void CryptonightR_instruction_mov137(void); +void CryptonightR_instruction_mov138(void); +void CryptonightR_instruction_mov139(void); +void CryptonightR_instruction_mov140(void); +void CryptonightR_instruction_mov141(void); +void CryptonightR_instruction_mov142(void); +void CryptonightR_instruction_mov143(void); +void CryptonightR_instruction_mov144(void); +void CryptonightR_instruction_mov145(void); +void CryptonightR_instruction_mov146(void); +void CryptonightR_instruction_mov147(void); +void CryptonightR_instruction_mov148(void); +void CryptonightR_instruction_mov149(void); +void CryptonightR_instruction_mov150(void); +void CryptonightR_instruction_mov151(void); +void CryptonightR_instruction_mov152(void); +void CryptonightR_instruction_mov153(void); +void CryptonightR_instruction_mov154(void); +void CryptonightR_instruction_mov155(void); +void CryptonightR_instruction_mov156(void); +void CryptonightR_instruction_mov157(void); +void CryptonightR_instruction_mov158(void); +void CryptonightR_instruction_mov159(void); +void CryptonightR_instruction_mov160(void); +void CryptonightR_instruction_mov161(void); +void CryptonightR_instruction_mov162(void); +void CryptonightR_instruction_mov163(void); +void CryptonightR_instruction_mov164(void); +void CryptonightR_instruction_mov165(void); +void CryptonightR_instruction_mov166(void); +void CryptonightR_instruction_mov167(void); +void CryptonightR_instruction_mov168(void); +void CryptonightR_instruction_mov169(void); +void CryptonightR_instruction_mov170(void); +void CryptonightR_instruction_mov171(void); +void CryptonightR_instruction_mov172(void); +void CryptonightR_instruction_mov173(void); +void CryptonightR_instruction_mov174(void); +void CryptonightR_instruction_mov175(void); +void CryptonightR_instruction_mov176(void); +void CryptonightR_instruction_mov177(void); +void CryptonightR_instruction_mov178(void); +void CryptonightR_instruction_mov179(void); +void CryptonightR_instruction_mov180(void); +void CryptonightR_instruction_mov181(void); +void CryptonightR_instruction_mov182(void); +void CryptonightR_instruction_mov183(void); +void CryptonightR_instruction_mov184(void); +void CryptonightR_instruction_mov185(void); +void CryptonightR_instruction_mov186(void); +void CryptonightR_instruction_mov187(void); +void CryptonightR_instruction_mov188(void); +void CryptonightR_instruction_mov189(void); +void CryptonightR_instruction_mov190(void); +void CryptonightR_instruction_mov191(void); +void CryptonightR_instruction_mov192(void); +void CryptonightR_instruction_mov193(void); +void CryptonightR_instruction_mov194(void); +void CryptonightR_instruction_mov195(void); +void CryptonightR_instruction_mov196(void); +void CryptonightR_instruction_mov197(void); +void CryptonightR_instruction_mov198(void); +void CryptonightR_instruction_mov199(void); +void CryptonightR_instruction_mov200(void); +void CryptonightR_instruction_mov201(void); +void CryptonightR_instruction_mov202(void); +void CryptonightR_instruction_mov203(void); +void CryptonightR_instruction_mov204(void); +void CryptonightR_instruction_mov205(void); +void CryptonightR_instruction_mov206(void); +void CryptonightR_instruction_mov207(void); +void CryptonightR_instruction_mov208(void); +void CryptonightR_instruction_mov209(void); +void CryptonightR_instruction_mov210(void); +void CryptonightR_instruction_mov211(void); +void CryptonightR_instruction_mov212(void); +void CryptonightR_instruction_mov213(void); +void CryptonightR_instruction_mov214(void); +void CryptonightR_instruction_mov215(void); +void CryptonightR_instruction_mov216(void); +void CryptonightR_instruction_mov217(void); +void CryptonightR_instruction_mov218(void); +void CryptonightR_instruction_mov219(void); +void CryptonightR_instruction_mov220(void); +void CryptonightR_instruction_mov221(void); +void CryptonightR_instruction_mov222(void); +void CryptonightR_instruction_mov223(void); +void CryptonightR_instruction_mov224(void); +void CryptonightR_instruction_mov225(void); +void CryptonightR_instruction_mov226(void); +void CryptonightR_instruction_mov227(void); +void CryptonightR_instruction_mov228(void); +void CryptonightR_instruction_mov229(void); +void CryptonightR_instruction_mov230(void); +void CryptonightR_instruction_mov231(void); +void CryptonightR_instruction_mov232(void); +void CryptonightR_instruction_mov233(void); +void CryptonightR_instruction_mov234(void); +void CryptonightR_instruction_mov235(void); +void CryptonightR_instruction_mov236(void); +void CryptonightR_instruction_mov237(void); +void CryptonightR_instruction_mov238(void); +void CryptonightR_instruction_mov239(void); +void CryptonightR_instruction_mov240(void); +void CryptonightR_instruction_mov241(void); +void CryptonightR_instruction_mov242(void); +void CryptonightR_instruction_mov243(void); +void CryptonightR_instruction_mov244(void); +void CryptonightR_instruction_mov245(void); +void CryptonightR_instruction_mov246(void); +void CryptonightR_instruction_mov247(void); +void CryptonightR_instruction_mov248(void); +void CryptonightR_instruction_mov249(void); +void CryptonightR_instruction_mov250(void); +void CryptonightR_instruction_mov251(void); +void CryptonightR_instruction_mov252(void); +void CryptonightR_instruction_mov253(void); +void CryptonightR_instruction_mov254(void); +void CryptonightR_instruction_mov255(void); +void CryptonightR_instruction_mov256(void); + +const void* instructions[257] = { + CryptonightR_instruction0, + CryptonightR_instruction1, + CryptonightR_instruction2, + CryptonightR_instruction3, + CryptonightR_instruction4, + CryptonightR_instruction5, + CryptonightR_instruction6, + CryptonightR_instruction7, + CryptonightR_instruction8, + CryptonightR_instruction9, + CryptonightR_instruction10, + CryptonightR_instruction11, + CryptonightR_instruction12, + CryptonightR_instruction13, + CryptonightR_instruction14, + CryptonightR_instruction15, + CryptonightR_instruction16, + CryptonightR_instruction17, + CryptonightR_instruction18, + CryptonightR_instruction19, + CryptonightR_instruction20, + CryptonightR_instruction21, + CryptonightR_instruction22, + CryptonightR_instruction23, + CryptonightR_instruction24, + CryptonightR_instruction25, + CryptonightR_instruction26, + CryptonightR_instruction27, + CryptonightR_instruction28, + CryptonightR_instruction29, + CryptonightR_instruction30, + CryptonightR_instruction31, + CryptonightR_instruction32, + CryptonightR_instruction33, + CryptonightR_instruction34, + CryptonightR_instruction35, + CryptonightR_instruction36, + CryptonightR_instruction37, + CryptonightR_instruction38, + CryptonightR_instruction39, + CryptonightR_instruction40, + CryptonightR_instruction41, + CryptonightR_instruction42, + CryptonightR_instruction43, + CryptonightR_instruction44, + CryptonightR_instruction45, + CryptonightR_instruction46, + CryptonightR_instruction47, + CryptonightR_instruction48, + CryptonightR_instruction49, + CryptonightR_instruction50, + CryptonightR_instruction51, + CryptonightR_instruction52, + CryptonightR_instruction53, + CryptonightR_instruction54, + CryptonightR_instruction55, + CryptonightR_instruction56, + CryptonightR_instruction57, + CryptonightR_instruction58, + CryptonightR_instruction59, + CryptonightR_instruction60, + CryptonightR_instruction61, + CryptonightR_instruction62, + CryptonightR_instruction63, + CryptonightR_instruction64, + CryptonightR_instruction65, + CryptonightR_instruction66, + CryptonightR_instruction67, + CryptonightR_instruction68, + CryptonightR_instruction69, + CryptonightR_instruction70, + CryptonightR_instruction71, + CryptonightR_instruction72, + CryptonightR_instruction73, + CryptonightR_instruction74, + CryptonightR_instruction75, + CryptonightR_instruction76, + CryptonightR_instruction77, + CryptonightR_instruction78, + CryptonightR_instruction79, + CryptonightR_instruction80, + CryptonightR_instruction81, + CryptonightR_instruction82, + CryptonightR_instruction83, + CryptonightR_instruction84, + CryptonightR_instruction85, + CryptonightR_instruction86, + CryptonightR_instruction87, + CryptonightR_instruction88, + CryptonightR_instruction89, + CryptonightR_instruction90, + CryptonightR_instruction91, + CryptonightR_instruction92, + CryptonightR_instruction93, + CryptonightR_instruction94, + CryptonightR_instruction95, + CryptonightR_instruction96, + CryptonightR_instruction97, + CryptonightR_instruction98, + CryptonightR_instruction99, + CryptonightR_instruction100, + CryptonightR_instruction101, + CryptonightR_instruction102, + CryptonightR_instruction103, + CryptonightR_instruction104, + CryptonightR_instruction105, + CryptonightR_instruction106, + CryptonightR_instruction107, + CryptonightR_instruction108, + CryptonightR_instruction109, + CryptonightR_instruction110, + CryptonightR_instruction111, + CryptonightR_instruction112, + CryptonightR_instruction113, + CryptonightR_instruction114, + CryptonightR_instruction115, + CryptonightR_instruction116, + CryptonightR_instruction117, + CryptonightR_instruction118, + CryptonightR_instruction119, + CryptonightR_instruction120, + CryptonightR_instruction121, + CryptonightR_instruction122, + CryptonightR_instruction123, + CryptonightR_instruction124, + CryptonightR_instruction125, + CryptonightR_instruction126, + CryptonightR_instruction127, + CryptonightR_instruction128, + CryptonightR_instruction129, + CryptonightR_instruction130, + CryptonightR_instruction131, + CryptonightR_instruction132, + CryptonightR_instruction133, + CryptonightR_instruction134, + CryptonightR_instruction135, + CryptonightR_instruction136, + CryptonightR_instruction137, + CryptonightR_instruction138, + CryptonightR_instruction139, + CryptonightR_instruction140, + CryptonightR_instruction141, + CryptonightR_instruction142, + CryptonightR_instruction143, + CryptonightR_instruction144, + CryptonightR_instruction145, + CryptonightR_instruction146, + CryptonightR_instruction147, + CryptonightR_instruction148, + CryptonightR_instruction149, + CryptonightR_instruction150, + CryptonightR_instruction151, + CryptonightR_instruction152, + CryptonightR_instruction153, + CryptonightR_instruction154, + CryptonightR_instruction155, + CryptonightR_instruction156, + CryptonightR_instruction157, + CryptonightR_instruction158, + CryptonightR_instruction159, + CryptonightR_instruction160, + CryptonightR_instruction161, + CryptonightR_instruction162, + CryptonightR_instruction163, + CryptonightR_instruction164, + CryptonightR_instruction165, + CryptonightR_instruction166, + CryptonightR_instruction167, + CryptonightR_instruction168, + CryptonightR_instruction169, + CryptonightR_instruction170, + CryptonightR_instruction171, + CryptonightR_instruction172, + CryptonightR_instruction173, + CryptonightR_instruction174, + CryptonightR_instruction175, + CryptonightR_instruction176, + CryptonightR_instruction177, + CryptonightR_instruction178, + CryptonightR_instruction179, + CryptonightR_instruction180, + CryptonightR_instruction181, + CryptonightR_instruction182, + CryptonightR_instruction183, + CryptonightR_instruction184, + CryptonightR_instruction185, + CryptonightR_instruction186, + CryptonightR_instruction187, + CryptonightR_instruction188, + CryptonightR_instruction189, + CryptonightR_instruction190, + CryptonightR_instruction191, + CryptonightR_instruction192, + CryptonightR_instruction193, + CryptonightR_instruction194, + CryptonightR_instruction195, + CryptonightR_instruction196, + CryptonightR_instruction197, + CryptonightR_instruction198, + CryptonightR_instruction199, + CryptonightR_instruction200, + CryptonightR_instruction201, + CryptonightR_instruction202, + CryptonightR_instruction203, + CryptonightR_instruction204, + CryptonightR_instruction205, + CryptonightR_instruction206, + CryptonightR_instruction207, + CryptonightR_instruction208, + CryptonightR_instruction209, + CryptonightR_instruction210, + CryptonightR_instruction211, + CryptonightR_instruction212, + CryptonightR_instruction213, + CryptonightR_instruction214, + CryptonightR_instruction215, + CryptonightR_instruction216, + CryptonightR_instruction217, + CryptonightR_instruction218, + CryptonightR_instruction219, + CryptonightR_instruction220, + CryptonightR_instruction221, + CryptonightR_instruction222, + CryptonightR_instruction223, + CryptonightR_instruction224, + CryptonightR_instruction225, + CryptonightR_instruction226, + CryptonightR_instruction227, + CryptonightR_instruction228, + CryptonightR_instruction229, + CryptonightR_instruction230, + CryptonightR_instruction231, + CryptonightR_instruction232, + CryptonightR_instruction233, + CryptonightR_instruction234, + CryptonightR_instruction235, + CryptonightR_instruction236, + CryptonightR_instruction237, + CryptonightR_instruction238, + CryptonightR_instruction239, + CryptonightR_instruction240, + CryptonightR_instruction241, + CryptonightR_instruction242, + CryptonightR_instruction243, + CryptonightR_instruction244, + CryptonightR_instruction245, + CryptonightR_instruction246, + CryptonightR_instruction247, + CryptonightR_instruction248, + CryptonightR_instruction249, + CryptonightR_instruction250, + CryptonightR_instruction251, + CryptonightR_instruction252, + CryptonightR_instruction253, + CryptonightR_instruction254, + CryptonightR_instruction255, + CryptonightR_instruction256, +}; + +const void* instructions_mov[257] = { + CryptonightR_instruction_mov0, + CryptonightR_instruction_mov1, + CryptonightR_instruction_mov2, + CryptonightR_instruction_mov3, + CryptonightR_instruction_mov4, + CryptonightR_instruction_mov5, + CryptonightR_instruction_mov6, + CryptonightR_instruction_mov7, + CryptonightR_instruction_mov8, + CryptonightR_instruction_mov9, + CryptonightR_instruction_mov10, + CryptonightR_instruction_mov11, + CryptonightR_instruction_mov12, + CryptonightR_instruction_mov13, + CryptonightR_instruction_mov14, + CryptonightR_instruction_mov15, + CryptonightR_instruction_mov16, + CryptonightR_instruction_mov17, + CryptonightR_instruction_mov18, + CryptonightR_instruction_mov19, + CryptonightR_instruction_mov20, + CryptonightR_instruction_mov21, + CryptonightR_instruction_mov22, + CryptonightR_instruction_mov23, + CryptonightR_instruction_mov24, + CryptonightR_instruction_mov25, + CryptonightR_instruction_mov26, + CryptonightR_instruction_mov27, + CryptonightR_instruction_mov28, + CryptonightR_instruction_mov29, + CryptonightR_instruction_mov30, + CryptonightR_instruction_mov31, + CryptonightR_instruction_mov32, + CryptonightR_instruction_mov33, + CryptonightR_instruction_mov34, + CryptonightR_instruction_mov35, + CryptonightR_instruction_mov36, + CryptonightR_instruction_mov37, + CryptonightR_instruction_mov38, + CryptonightR_instruction_mov39, + CryptonightR_instruction_mov40, + CryptonightR_instruction_mov41, + CryptonightR_instruction_mov42, + CryptonightR_instruction_mov43, + CryptonightR_instruction_mov44, + CryptonightR_instruction_mov45, + CryptonightR_instruction_mov46, + CryptonightR_instruction_mov47, + CryptonightR_instruction_mov48, + CryptonightR_instruction_mov49, + CryptonightR_instruction_mov50, + CryptonightR_instruction_mov51, + CryptonightR_instruction_mov52, + CryptonightR_instruction_mov53, + CryptonightR_instruction_mov54, + CryptonightR_instruction_mov55, + CryptonightR_instruction_mov56, + CryptonightR_instruction_mov57, + CryptonightR_instruction_mov58, + CryptonightR_instruction_mov59, + CryptonightR_instruction_mov60, + CryptonightR_instruction_mov61, + CryptonightR_instruction_mov62, + CryptonightR_instruction_mov63, + CryptonightR_instruction_mov64, + CryptonightR_instruction_mov65, + CryptonightR_instruction_mov66, + CryptonightR_instruction_mov67, + CryptonightR_instruction_mov68, + CryptonightR_instruction_mov69, + CryptonightR_instruction_mov70, + CryptonightR_instruction_mov71, + CryptonightR_instruction_mov72, + CryptonightR_instruction_mov73, + CryptonightR_instruction_mov74, + CryptonightR_instruction_mov75, + CryptonightR_instruction_mov76, + CryptonightR_instruction_mov77, + CryptonightR_instruction_mov78, + CryptonightR_instruction_mov79, + CryptonightR_instruction_mov80, + CryptonightR_instruction_mov81, + CryptonightR_instruction_mov82, + CryptonightR_instruction_mov83, + CryptonightR_instruction_mov84, + CryptonightR_instruction_mov85, + CryptonightR_instruction_mov86, + CryptonightR_instruction_mov87, + CryptonightR_instruction_mov88, + CryptonightR_instruction_mov89, + CryptonightR_instruction_mov90, + CryptonightR_instruction_mov91, + CryptonightR_instruction_mov92, + CryptonightR_instruction_mov93, + CryptonightR_instruction_mov94, + CryptonightR_instruction_mov95, + CryptonightR_instruction_mov96, + CryptonightR_instruction_mov97, + CryptonightR_instruction_mov98, + CryptonightR_instruction_mov99, + CryptonightR_instruction_mov100, + CryptonightR_instruction_mov101, + CryptonightR_instruction_mov102, + CryptonightR_instruction_mov103, + CryptonightR_instruction_mov104, + CryptonightR_instruction_mov105, + CryptonightR_instruction_mov106, + CryptonightR_instruction_mov107, + CryptonightR_instruction_mov108, + CryptonightR_instruction_mov109, + CryptonightR_instruction_mov110, + CryptonightR_instruction_mov111, + CryptonightR_instruction_mov112, + CryptonightR_instruction_mov113, + CryptonightR_instruction_mov114, + CryptonightR_instruction_mov115, + CryptonightR_instruction_mov116, + CryptonightR_instruction_mov117, + CryptonightR_instruction_mov118, + CryptonightR_instruction_mov119, + CryptonightR_instruction_mov120, + CryptonightR_instruction_mov121, + CryptonightR_instruction_mov122, + CryptonightR_instruction_mov123, + CryptonightR_instruction_mov124, + CryptonightR_instruction_mov125, + CryptonightR_instruction_mov126, + CryptonightR_instruction_mov127, + CryptonightR_instruction_mov128, + CryptonightR_instruction_mov129, + CryptonightR_instruction_mov130, + CryptonightR_instruction_mov131, + CryptonightR_instruction_mov132, + CryptonightR_instruction_mov133, + CryptonightR_instruction_mov134, + CryptonightR_instruction_mov135, + CryptonightR_instruction_mov136, + CryptonightR_instruction_mov137, + CryptonightR_instruction_mov138, + CryptonightR_instruction_mov139, + CryptonightR_instruction_mov140, + CryptonightR_instruction_mov141, + CryptonightR_instruction_mov142, + CryptonightR_instruction_mov143, + CryptonightR_instruction_mov144, + CryptonightR_instruction_mov145, + CryptonightR_instruction_mov146, + CryptonightR_instruction_mov147, + CryptonightR_instruction_mov148, + CryptonightR_instruction_mov149, + CryptonightR_instruction_mov150, + CryptonightR_instruction_mov151, + CryptonightR_instruction_mov152, + CryptonightR_instruction_mov153, + CryptonightR_instruction_mov154, + CryptonightR_instruction_mov155, + CryptonightR_instruction_mov156, + CryptonightR_instruction_mov157, + CryptonightR_instruction_mov158, + CryptonightR_instruction_mov159, + CryptonightR_instruction_mov160, + CryptonightR_instruction_mov161, + CryptonightR_instruction_mov162, + CryptonightR_instruction_mov163, + CryptonightR_instruction_mov164, + CryptonightR_instruction_mov165, + CryptonightR_instruction_mov166, + CryptonightR_instruction_mov167, + CryptonightR_instruction_mov168, + CryptonightR_instruction_mov169, + CryptonightR_instruction_mov170, + CryptonightR_instruction_mov171, + CryptonightR_instruction_mov172, + CryptonightR_instruction_mov173, + CryptonightR_instruction_mov174, + CryptonightR_instruction_mov175, + CryptonightR_instruction_mov176, + CryptonightR_instruction_mov177, + CryptonightR_instruction_mov178, + CryptonightR_instruction_mov179, + CryptonightR_instruction_mov180, + CryptonightR_instruction_mov181, + CryptonightR_instruction_mov182, + CryptonightR_instruction_mov183, + CryptonightR_instruction_mov184, + CryptonightR_instruction_mov185, + CryptonightR_instruction_mov186, + CryptonightR_instruction_mov187, + CryptonightR_instruction_mov188, + CryptonightR_instruction_mov189, + CryptonightR_instruction_mov190, + CryptonightR_instruction_mov191, + CryptonightR_instruction_mov192, + CryptonightR_instruction_mov193, + CryptonightR_instruction_mov194, + CryptonightR_instruction_mov195, + CryptonightR_instruction_mov196, + CryptonightR_instruction_mov197, + CryptonightR_instruction_mov198, + CryptonightR_instruction_mov199, + CryptonightR_instruction_mov200, + CryptonightR_instruction_mov201, + CryptonightR_instruction_mov202, + CryptonightR_instruction_mov203, + CryptonightR_instruction_mov204, + CryptonightR_instruction_mov205, + CryptonightR_instruction_mov206, + CryptonightR_instruction_mov207, + CryptonightR_instruction_mov208, + CryptonightR_instruction_mov209, + CryptonightR_instruction_mov210, + CryptonightR_instruction_mov211, + CryptonightR_instruction_mov212, + CryptonightR_instruction_mov213, + CryptonightR_instruction_mov214, + CryptonightR_instruction_mov215, + CryptonightR_instruction_mov216, + CryptonightR_instruction_mov217, + CryptonightR_instruction_mov218, + CryptonightR_instruction_mov219, + CryptonightR_instruction_mov220, + CryptonightR_instruction_mov221, + CryptonightR_instruction_mov222, + CryptonightR_instruction_mov223, + CryptonightR_instruction_mov224, + CryptonightR_instruction_mov225, + CryptonightR_instruction_mov226, + CryptonightR_instruction_mov227, + CryptonightR_instruction_mov228, + CryptonightR_instruction_mov229, + CryptonightR_instruction_mov230, + CryptonightR_instruction_mov231, + CryptonightR_instruction_mov232, + CryptonightR_instruction_mov233, + CryptonightR_instruction_mov234, + CryptonightR_instruction_mov235, + CryptonightR_instruction_mov236, + CryptonightR_instruction_mov237, + CryptonightR_instruction_mov238, + CryptonightR_instruction_mov239, + CryptonightR_instruction_mov240, + CryptonightR_instruction_mov241, + CryptonightR_instruction_mov242, + CryptonightR_instruction_mov243, + CryptonightR_instruction_mov244, + CryptonightR_instruction_mov245, + CryptonightR_instruction_mov246, + CryptonightR_instruction_mov247, + CryptonightR_instruction_mov248, + CryptonightR_instruction_mov249, + CryptonightR_instruction_mov250, + CryptonightR_instruction_mov251, + CryptonightR_instruction_mov252, + CryptonightR_instruction_mov253, + CryptonightR_instruction_mov254, + CryptonightR_instruction_mov255, + CryptonightR_instruction_mov256, +}; + +#endif + +#endif // CRYPTONIGHTR_TEMPLATE_H diff --git a/src/crypto/aesb.c b/src/crypto/aesb.c index 5d57b8af4bb..6d4905ad5e2 100644 --- a/src/crypto/aesb.c +++ b/src/crypto/aesb.c @@ -19,6 +19,7 @@ Issue Date: 20/12/2007 */ #include +#include "int-util.h" #if defined(__cplusplus) extern "C" @@ -32,11 +33,11 @@ extern "C" #define RC_LENGTH (5 * (AES_BLOCK_SIZE / 4 - 2)) #if defined(_MSC_VER) -#define ALIGN __declspec(align(TABLE_ALIGN)) +#define LOCAL_ALIGN __declspec(align(TABLE_ALIGN)) #elif defined(__GNUC__) -#define ALIGN __attribute__ ((aligned(16))) +#define LOCAL_ALIGN __attribute__ ((aligned(16))) #else -#define ALIGN +#define LOCAL_ALIGN #endif #define rf1(r,c) (r) @@ -50,7 +51,7 @@ extern "C" #define state_out(y,x) so(y,x,0); so(y,x,1); so(y,x,2); so(y,x,3) #define round(rm,y,x,k) rm(y,x,k,0); rm(y,x,k,1); rm(y,x,k,2); rm(y,x,k,3) #define to_byte(x) ((x) & 0xff) -#define bval(x,n) to_byte((x) >> (8 * (n))) +#define bval(x,n) to_byte(SWAP32LE(x) >> (8 * (n))) #define fwd_var(x,r,c)\ ( r == 0 ? ( c == 0 ? s(x,0) : c == 1 ? s(x,1) : c == 2 ? s(x,2) : s(x,3))\ @@ -58,7 +59,7 @@ extern "C" : r == 2 ? ( c == 0 ? s(x,2) : c == 1 ? s(x,3) : c == 2 ? s(x,0) : s(x,1))\ : ( c == 0 ? s(x,3) : c == 1 ? s(x,0) : c == 2 ? s(x,1) : s(x,2))) -#define fwd_rnd(y,x,k,c) (s(y,c) = (k)[c] ^ four_tables(x,t_use(f,n),fwd_var,rf1,c)) +#define fwd_rnd(y,x,k,c) (s(y,c) = (k)[c] ^ SWAP32LE(four_tables(x,t_use(f,n),fwd_var,rf1,c))) #define sb_data(w) {\ w(0x63), w(0x7c), w(0x77), w(0x7b), w(0xf2), w(0x6b), w(0x6f), w(0xc5),\ @@ -130,7 +131,7 @@ extern "C" #define t_set(m,n) t_##m##n #define t_use(m,n) t_##m##n -#define d_4(t,n,b,e,f,g,h) ALIGN const t n[4][256] = { b(e), b(f), b(g), b(h) } +#define d_4(t,n,b,e,f,g,h) LOCAL_ALIGN const t n[4][256] = { b(e), b(f), b(g), b(h) } #define four_tables(x,tab,vf,rf,c) \ (tab[0][bval(vf(x,0,c),rf(0,c))] \ diff --git a/src/crypto/blake256.c b/src/crypto/blake256.c index d503c47e0bf..1e305b3a621 100644 --- a/src/crypto/blake256.c +++ b/src/crypto/blake256.c @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -31,7 +31,7 @@ * The blake256_* and blake224_* functions are largely copied from * blake256_light.c and blake224_light.c from the BLAKE website: * - * http://131002.net/blake/ + * https://131002.net/blake/ * * The hmac_* functions implement HMAC-BLAKE-256 and HMAC-BLAKE-224. * HMAC is specified by RFC 2104. diff --git a/src/crypto/blake256.h b/src/crypto/blake256.h index 0737722898f..309dbf3ec4d 100644 --- a/src/crypto/blake256.h +++ b/src/crypto/blake256.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/crypto/chacha.c b/src/crypto/chacha.c index 5d3edb98d83..d734e8b1b25 100644 --- a/src/crypto/chacha.c +++ b/src/crypto/chacha.c @@ -11,7 +11,7 @@ Public domain. #endif #include "chacha.h" -#include "common/int-util.h" +#include "int-util.h" #include "warnings.h" /* diff --git a/src/crypto/chacha.h b/src/crypto/chacha.h index 2b3ed804391..a39823e5ae5 100644 --- a/src/crypto/chacha.h +++ b/src/crypto/chacha.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -40,6 +40,7 @@ #include #include "memwipe.h" +#include "mlocker.h" #include "hash.h" namespace crypto { @@ -50,7 +51,7 @@ namespace crypto { #if defined(__cplusplus) } - using chacha_key = tools::scrubbed_arr; + using chacha_key = epee::mlocked>; #pragma pack(push, 1) // MS VC 2012 doesn't interpret `class chacha_iv` as POD in spite of [9.0.10], so it is a struct @@ -69,22 +70,26 @@ namespace crypto { chacha20(data, length, key.data(), reinterpret_cast(&iv), cipher); } - inline void generate_chacha_key(const void *data, size_t size, chacha_key& key) { + inline void generate_chacha_key(const void *data, size_t size, chacha_key& key, uint64_t kdf_rounds) { static_assert(sizeof(chacha_key) <= sizeof(hash), "Size of hash must be at least that of chacha_key"); - tools::scrubbed_arr pwd_hash; - crypto::cn_slow_hash(data, size, pwd_hash.data(), 0/*variant*/, 0/*prehashed*/); - memcpy(&unwrap(key), pwd_hash.data(), sizeof(key)); + epee::mlocked> pwd_hash; + crypto::cn_slow_hash(data, size, pwd_hash.data(), 0/*variant*/, 0/*prehashed*/, 0/*height*/); + for (uint64_t n = 1; n < kdf_rounds; ++n) + crypto::cn_slow_hash(pwd_hash.data(), pwd_hash.size(), pwd_hash.data(), 0/*variant*/, 0/*prehashed*/, 0/*height*/); + memcpy(&unwrap(unwrap(key)), pwd_hash.data(), sizeof(key)); } - inline void generate_chacha_key_prehashed(const void *data, size_t size, chacha_key& key) { + inline void generate_chacha_key_prehashed(const void *data, size_t size, chacha_key& key, uint64_t kdf_rounds) { static_assert(sizeof(chacha_key) <= sizeof(hash), "Size of hash must be at least that of chacha_key"); - tools::scrubbed_arr pwd_hash; - crypto::cn_slow_hash(data, size, pwd_hash.data(), 0/*variant*/, 1/*prehashed*/); - memcpy(&unwrap(key), pwd_hash.data(), sizeof(key)); + epee::mlocked> pwd_hash; + crypto::cn_slow_hash(data, size, pwd_hash.data(), 0/*variant*/, 1/*prehashed*/, 0/*height*/); + for (uint64_t n = 1; n < kdf_rounds; ++n) + crypto::cn_slow_hash(pwd_hash.data(), pwd_hash.size(), pwd_hash.data(), 0/*variant*/, 0/*prehashed*/, 0/*height*/); + memcpy(&unwrap(unwrap(key)), pwd_hash.data(), sizeof(key)); } - inline void generate_chacha_key(std::string password, chacha_key& key) { - return generate_chacha_key(password.data(), password.size(), key); + inline void generate_chacha_key(std::string password, chacha_key& key, uint64_t kdf_rounds) { + return generate_chacha_key(password.data(), password.size(), key, kdf_rounds); } } diff --git a/src/crypto/crypto-ops-data.c b/src/crypto/crypto-ops-data.c index 4ff4310deec..c9530bb2a6b 100644 --- a/src/crypto/crypto-ops-data.c +++ b/src/crypto/crypto-ops-data.c @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -871,3 +871,9 @@ const fe fe_fffb2 = {8166131, -6741800, -17040804, 3154616, 21461005, 1466302, - const fe fe_fffb3 = {-13620103, 14639558, 4532995, 7679154, 16815101, -15883539, -22863840, -14813421, 13716513, -6477756}; /* sqrt(-sqrt(-1) * A * (A + 2)) */ const fe fe_fffb4 = {-21786234, -12173074, 21573800, 4524538, -4645904, 16204591, 8012863, -8444712, 3212926, 6885324}; /* sqrt(sqrt(-1) * A * (A + 2)) */ const ge_p3 ge_p3_identity = { {0}, {1, 0}, {1, 0}, {0} }; +const ge_p3 ge_p3_H = { + {7329926, -15101362, 31411471, 7614783, 27996851, -3197071, -11157635, -6878293, 466949, -7986503}, + {5858699, 5096796, 21321203, -7536921, -5553480, -11439507, -5627669, 15045946, 19977121, 5275251}, + {1, 0, 0, 0, 0, 0, 0, 0, 0, 0}, + {23443568, -5110398, -8776029, -4345135, 6889568, -14710814, 7474843, 3279062, 14550766, -7453428} +}; diff --git a/src/crypto/crypto-ops.c b/src/crypto/crypto-ops.c index 45d412ac690..5a3d994a630 100644 --- a/src/crypto/crypto-ops.c +++ b/src/crypto/crypto-ops.c @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -3707,9 +3707,8 @@ void sc_muladd(unsigned char *s, const unsigned char *a, const unsigned char *b, s[31] = s11 >> 17; } -/* Assumes that a != INT64_MIN */ static int64_t signum(int64_t a) { - return (a >> 63) - ((-a) >> 63); + return a > 0 ? 1 : a < 0 ? -1 : 0; } int sc_check(const unsigned char *s) { @@ -3730,3 +3729,16 @@ int sc_isnonzero(const unsigned char *s) { s[18] | s[19] | s[20] | s[21] | s[22] | s[23] | s[24] | s[25] | s[26] | s[27] | s[28] | s[29] | s[30] | s[31]) - 1) >> 8) + 1; } + +int ge_p3_is_point_at_infinity(const ge_p3 *p) { + // X = 0 and Y == Z + int n; + for (n = 0; n < 10; ++n) + { + if (p->X[n] | p->T[n]) + return 0; + if (p->Y[n] != p->Z[n]) + return 0; + } + return 1; +} diff --git a/src/crypto/crypto-ops.h b/src/crypto/crypto-ops.h index dc3c60794bd..7137437bc84 100644 --- a/src/crypto/crypto-ops.h +++ b/src/crypto/crypto-ops.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -140,6 +140,7 @@ extern const fe fe_fffb2; extern const fe fe_fffb3; extern const fe fe_fffb4; extern const ge_p3 ge_p3_identity; +extern const ge_p3 ge_p3_H; void ge_fromfe_frombytes_vartime(ge_p2 *, const unsigned char *); void sc_0(unsigned char *); void sc_reduce32(unsigned char *); @@ -158,3 +159,5 @@ void ge_sub(ge_p1p1 *r, const ge_p3 *p, const ge_cached *q); void fe_add(fe h, const fe f, const fe g); void fe_tobytes(unsigned char *, const fe); void fe_invert(fe out, const fe z); + +int ge_p3_is_point_at_infinity(const ge_p3 *p); diff --git a/src/crypto/crypto.cpp b/src/crypto/crypto.cpp index f4ef751d388..3f06c4f3ff8 100644 --- a/src/crypto/crypto.cpp +++ b/src/crypto/crypto.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -34,7 +34,6 @@ #include #include #include -#include #include #include #include @@ -70,6 +69,9 @@ namespace crypto { #include "random.h" } + const crypto::public_key null_pkey = crypto::public_key{}; + const crypto::secret_key null_skey = crypto::secret_key{}; + static inline unsigned char *operator &(ec_point &point) { return &reinterpret_cast(point); } @@ -93,18 +95,32 @@ namespace crypto { generate_random_bytes_not_thread_safe(N, bytes); } - /* generate a random 32-byte (256-bit) integer and copy it to res */ - static inline void random_scalar_not_thread_safe(ec_scalar &res) { - unsigned char tmp[64]; - generate_random_bytes_not_thread_safe(64, tmp); - sc_reduce(tmp); - memcpy(&res, tmp, 32); + static inline bool less32(const unsigned char *k0, const unsigned char *k1) + { + for (int n = 31; n >= 0; --n) + { + if (k0[n] < k1[n]) + return true; + if (k0[n] > k1[n]) + return false; + } + return false; } + + void random32_unbiased(unsigned char *bytes) + { + // l = 2^252 + 27742317777372353535851937790883648493. + // it fits 15 in 32 bytes + static const unsigned char limit[32] = { 0xe3, 0x6a, 0x67, 0x72, 0x8b, 0xce, 0x13, 0x29, 0x8f, 0x30, 0x82, 0x8c, 0x0b, 0xa4, 0x10, 0x39, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0 }; + do + { + generate_random_bytes_thread_safe(32, bytes); + } while (!sc_isnonzero(bytes) && !less32(bytes, limit)); // should be good about 15/16 of the time + sc_reduce32(bytes); + } + /* generate a random 32-byte (256-bit) integer and copy it to res */ static inline void random_scalar(ec_scalar &res) { - unsigned char tmp[64]; - generate_random_bytes_thread_safe(64, tmp); - sc_reduce(tmp); - memcpy(&res, tmp, 32); + random32_unbiased((unsigned char*)res.data); } void hash_to_scalar(const void *data, size_t length, ec_scalar &res) { @@ -257,11 +273,18 @@ namespace crypto { #endif buf.h = prefix_hash; buf.key = pub; + try_again: random_scalar(k); + if (((const uint32_t*)(&k))[7] == 0) // we don't want tiny numbers here + goto try_again; ge_scalarmult_base(&tmp3, &k); ge_p3_tobytes(&buf.comm, &tmp3); hash_to_scalar(&buf, sizeof(s_comm), sig.c); + if (!sc_isnonzero((const unsigned char*)sig.c.data)) + goto try_again; sc_mulsub(&sig.r, &sig.c, &unwrap(sec), &k); + if (!sc_isnonzero((const unsigned char*)sig.r.data)) + goto try_again; } bool crypto_ops::check_signature(const hash &prefix_hash, const public_key &pub, const signature &sig) { @@ -275,11 +298,14 @@ namespace crypto { if (ge_frombytes_vartime(&tmp3, &pub) != 0) { return false; } - if (sc_check(&sig.c) != 0 || sc_check(&sig.r) != 0) { + if (sc_check(&sig.c) != 0 || sc_check(&sig.r) != 0 || !sc_isnonzero(&sig.c)) { return false; } ge_double_scalarmult_base_vartime(&tmp2, &sig.c, &tmp3, &sig.r); ge_tobytes(&buf.comm, &tmp2); + static const ec_point infinity = {{ 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}; + if (memcmp(&buf.comm, &infinity, 32) == 0) + return false; hash_to_scalar(&buf, sizeof(s_comm), c); sc_sub(&c, &c, &sig.c); return sc_isnonzero(&c) == 0; diff --git a/src/crypto/crypto.h b/src/crypto/crypto.h index 9ea0f2ec0cf..bac456f6006 100644 --- a/src/crypto/crypto.h +++ b/src/crypto/crypto.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -32,16 +32,14 @@ #include #include -#include -#include -#include #include #include #include +#include #include "common/pod-class.h" -#include "common/util.h" #include "memwipe.h" +#include "mlocker.h" #include "generic-ops.h" #include "hex.h" #include "span.h" @@ -66,7 +64,7 @@ namespace crypto { friend class crypto_ops; }; - using secret_key = tools::scrubbed; + using secret_key = epee::mlocked>; POD_CLASS public_keyV { std::vector keys; @@ -99,6 +97,7 @@ namespace crypto { #pragma pack(pop) void hash_to_scalar(const void *data, size_t length, ec_scalar &res); + void random32_unbiased(unsigned char *bytes); static_assert(sizeof(ec_point) == 32 && sizeof(ec_scalar) == 32 && sizeof(public_key) == 32 && sizeof(secret_key) == 32 && @@ -164,6 +163,32 @@ namespace crypto { return res; } + /* UniformRandomBitGenerator using crypto::rand() + */ + struct random_device + { + typedef uint64_t result_type; + static constexpr result_type min() { return 0; } + static constexpr result_type max() { return result_type(-1); } + result_type operator()() const { return crypto::rand(); } + }; + + /* Generate a random value between range_min and range_max + */ + template + typename std::enable_if::value, T>::type rand_range(T range_min, T range_max) { + crypto::random_device rd; + std::uniform_int_distribution dis(range_min, range_max); + return dis(rd); + } + + /* Generate a random index between 0 and sz-1 + */ + template + typename std::enable_if::value, T>::type rand_idx(T sz) { + return crypto::rand_range(0, sz-1); + } + /* Generate a new key pair */ inline secret_key generate_keys(public_key &pub, secret_key &sec, const secret_key& recovery_key = secret_key(), bool recover = false) { @@ -277,11 +302,11 @@ namespace crypto { epee::to_hex::formatted(o, epee::as_byte_span(v)); return o; } - const static crypto::public_key null_pkey = boost::value_initialized(); - const static crypto::secret_key null_skey = boost::value_initialized(); + const extern crypto::public_key null_pkey; + const extern crypto::secret_key null_skey; } CRYPTO_MAKE_HASHABLE(public_key) -CRYPTO_MAKE_HASHABLE(secret_key) +CRYPTO_MAKE_HASHABLE_CONSTANT_TIME(secret_key) CRYPTO_MAKE_HASHABLE(key_image) CRYPTO_MAKE_COMPARABLE(signature) diff --git a/src/crypto/crypto_ops_builder/README.md b/src/crypto/crypto_ops_builder/README.md index 326d2ca6efa..4bb95cc4a0b 100644 --- a/src/crypto/crypto_ops_builder/README.md +++ b/src/crypto/crypto_ops_builder/README.md @@ -1,6 +1,6 @@ # Monero -Copyright (c) 2014-2018, The Monero Project +Copyright (c) 2014-2019, The Monero Project ## Crypto Ops Builder diff --git a/src/crypto/crypto_ops_builder/crypto-ops-data.c b/src/crypto/crypto_ops_builder/crypto-ops-data.c index 127e3e17b56..45e9923b1c9 100644 --- a/src/crypto/crypto_ops_builder/crypto-ops-data.c +++ b/src/crypto/crypto_ops_builder/crypto-ops-data.c @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/crypto/crypto_ops_builder/crypto-ops-old.c b/src/crypto/crypto_ops_builder/crypto-ops-old.c index 9097bf95be8..89c2ced6ed0 100644 --- a/src/crypto/crypto_ops_builder/crypto-ops-old.c +++ b/src/crypto/crypto_ops_builder/crypto-ops-old.c @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/crypto/crypto_ops_builder/crypto-ops.h b/src/crypto/crypto_ops_builder/crypto-ops.h index 9337b56b7a2..b4fcfca9c2f 100644 --- a/src/crypto/crypto_ops_builder/crypto-ops.h +++ b/src/crypto/crypto_ops_builder/crypto-ops.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/crypto/crypto_ops_builder/ietf.txt b/src/crypto/crypto_ops_builder/ietf.txt index 0736f71ec06..609f5e75a55 100644 --- a/src/crypto/crypto_ops_builder/ietf.txt +++ b/src/crypto/crypto_ops_builder/ietf.txt @@ -1,1402 +1,4 @@ - +https://tools.ietf.org/id/draft-josefsson-eddsa-ed25519-02.txt -[Docs] [txt|pdf] [Tracker] [Email] [Diff1] [Diff2] [Nits] - -Versions: 00 01 02 - -Network Working Group S. Josefsson -Internet-Draft SJD AB -Intended status: Informational N. Moeller -Expires: August 26, 2015 - February 22, 2015 - - - EdDSA and Ed25519 - - draft-josefsson-eddsa-ed25519-02 - - -Abstract - - The elliptic curve signature scheme EdDSA and one instance of it - called Ed25519 is described. An example implementation and test - vectors are provided. - -Status of This Memo - - This Internet-Draft is submitted in full conformance with the - provisions of BCP 78 and BCP 79. - - Internet-Drafts are working documents of the Internet Engineering - Task Force (IETF). Note that other groups may also distribute - working documents as Internet-Drafts. The list of current Internet- - Drafts is at http://datatracker.ietf.org/drafts/current/. - - Internet-Drafts are draft documents valid for a maximum of six months - and may be updated, replaced, or obsoleted by other documents at any - time. It is inappropriate to use Internet-Drafts as reference - material or to cite them other than as "work in progress." - - This Internet-Draft will expire on August 26, 2015. - -Copyright Notice - - Copyright (c) 2015 IETF Trust and the persons identified as the - document authors. All rights reserved. - - This document is subject to BCP 78 and the IETF Trust's Legal - Provisions Relating to IETF Documents - (http://trustee.ietf.org/license-info) in effect on the date of - publication of this document. Please review these documents - carefully, as they describe your rights and restrictions with respect - to this document. Code Components extracted from this document must - include Simplified BSD License text as described in Section 4.e of - the Trust Legal Provisions and are provided without warranty as - described in the Simplified BSD License. - - - - -Josefsson & Moeller Expires August 26, 2015 [Page 1] - - -Internet-Draft EdDSA & Ed25519 February 2015 - - -Table of Contents - - 1. Introduction . . . . . . . . . . . . . . . . . . . . . . . . 2 - 2. Notation . . . . . . . . . . . . . . . . . . . . . . . . . . 3 - 3. Background . . . . . . . . . . . . . . . . . . . . . . . . . 3 - 4. EdDSA . . . . . . . . . . . . . . . . . . . . . . . . . . . . 4 - 4.1. Encoding . . . . . . . . . . . . . . . . . . . . . . . . 4 - 4.2. Keys . . . . . . . . . . . . . . . . . . . . . . . . . . 5 - 4.3. Sign . . . . . . . . . . . . . . . . . . . . . . . . . . 5 - 4.4. Verify . . . . . . . . . . . . . . . . . . . . . . . . . 5 - 5. Ed25519 . . . . . . . . . . . . . . . . . . . . . . . . . . . 5 - 5.1. Modular arithmetic . . . . . . . . . . . . . . . . . . . 6 - 5.2. Encoding . . . . . . . . . . . . . . . . . . . . . . . . 6 - 5.3. Decoding . . . . . . . . . . . . . . . . . . . . . . . . 6 - 5.4. Point addition . . . . . . . . . . . . . . . . . . . . . 7 - 5.5. Key Generation . . . . . . . . . . . . . . . . . . . . . 8 - 5.6. Sign . . . . . . . . . . . . . . . . . . . . . . . . . . 8 - 5.7. Verify . . . . . . . . . . . . . . . . . . . . . . . . . 9 - 5.8. Python illustration . . . . . . . . . . . . . . . . . . . 9 - 6. Test Vectors for Ed25519 . . . . . . . . . . . . . . . . . . 14 - 7. Acknowledgements . . . . . . . . . . . . . . . . . . . . . . 17 - 8. IANA Considerations . . . . . . . . . . . . . . . . . . . . . 18 - 9. Security Considerations . . . . . . . . . . . . . . . . . . . 18 - 9.1. Side-channel leaks . . . . . . . . . . . . . . . . . . . 18 - 10. References . . . . . . . . . . . . . . . . . . . . . . . . . 18 - 10.1. Normative References . . . . . . . . . . . . . . . . . . 18 - 10.2. Informative References . . . . . . . . . . . . . . . . . 18 - Appendix A. Ed25519 Python Library . . . . . . . . . . . . . . . 19 - Appendix B. Library driver . . . . . . . . . . . . . . . . . . . 23 - Authors' Addresses . . . . . . . . . . . . . . . . . . . . . . . 24 - -1. Introduction - - - The Edwards-curve Digital Signature Algorithm (EdDSA) is a variant of - Schnorr's signature system with Twisted Edwards curves. EdDSA needs - to be instantiated with certain parameters and this document describe - Ed25519 - an instantiation of EdDSA in a curve over GF(2^255-19). To - facilitate adoption in the Internet community of Ed25519, this - document describe the signature scheme in an implementation-oriented - way, and we provide sample code and test vectors. - - The advantages with EdDSA and Ed25519 include: - - 1. High-performance on a variety of platforms. - - 2. Does not require the use of a unique random number for each - signature. - - - - -Josefsson & Moeller Expires August 26, 2015 [Page 2] - - -Internet-Draft EdDSA & Ed25519 February 2015 - - - 3. More resilient to side-channel attacks. - - 4. Small public keys (32 bytes) and signatures (64 bytes). - - 5. The formulas are "strongly unified", i.e., they are valid for all - points on the curve, with no exceptions. This obviates the need - for EdDSA to perform expensive point validation on untrusted - public values. - - 6. Collision resilience, meaning that hash-function collisions do - not break this system. - - For further background, see the original EdDSA paper [EDDSA]. - -2. Notation - - - The following notation is used throughout the document: - - GF(p) finite field with p elements - - x^y x multiplied by itself y times - - B generator of the group or subgroup of interest - - n B B added to itself n times. - - h_i the i'th bit of h - - a || b (bit-)string a concatenated with (bit-)string b - -3. Background - - - EdDSA is defined using an elliptic curve over GF(p) of the form - - -x^2 + y^2 = 1 + d x^2 y^2 - - In general, p could be a prime power, but it is usually chosen as a - prime number. It is required that p = 1 modulo 4 (which implies that - -1 is a square modulo p) and that d is a non-square modulo p. For - Ed25519, the curve used is equivalent to Curve25519 [CURVE25519], - under a change of coordinates, which means that the difficulty of the - discrete logarithm problem is the same as for Curve25519. - - Points on this curve form a group under addition, (x3, y3) = (x1, y1) - + (x2, y2), with the formulas - - - - - - -Josefsson & Moeller Expires August 26, 2015 [Page 3] - - -Internet-Draft EdDSA & Ed25519 February 2015 - - - x1 y2 + x2 y1 y1 y2 + x1 x2 - x3 = -------------------, y3 = ------------------- - 1 + d x1 x2 y1 y2 1 - d x1 x2 y1 y2 - - The neutral element in the group is (0, 1). - - Unlike manyy other curves used for cryptographic applications, these - formulas are "strongly unified": they are valid for all points on the - curve, with no exceptions. In particular, the denominators are non- - zero for all input points. - - There are more efficient formulas, which are still strongly unified, - which use homogeneous coordinates to avoid the expensive modulo p - inversions. See [Faster-ECC] and [Edwards-revisited]. - -4. EdDSA - - - EdDSA is a digital signature system with several parameters. The - generic EdDSA digital signature system is normally not implemented - directly, but instead a particular instance of EdDSA (like Ed25519) - is implemented. A precise explanation of the generic EdDSA is thus - not particulary useful for implementers, but for background and - completeness, a succint description of the generic EdDSA algorithm is - given here. - - EdDSA has seven parameters: - - 1. an integer b >= 10. - - 2. a cryptographic hash function H producing 2b-bit outputs. - - 3. a prime power p congruent to 1 modulo 4. - - 4. a (b-1)-bit encoding of elements of the finite field GF(p). - - 5. a non-square element d of GF(p) - - 6. an element B != (0,1) of the set E = { (x,y) is a member of GF(p) - x GF(p) such that -x^2 + y^2 = 1 + dx^2y^2 }. - - 7. a prime q, of size b-3 bits, such that qB = (0, 1), i.e., q is - the order of B or a multiple thereof. - -4.1. Encoding - - - An element (x,y) of E is encoded as a b-bit string called ENC(x,y) - which is the (b-1)-bit encoding of y concatenated with one bit that - is 1 if x is negative and 0 if x is not negative. Negative elements - - - -Josefsson & Moeller Expires August 26, 2015 [Page 4] - - -Internet-Draft EdDSA & Ed25519 February 2015 - - - of GF(q) are those x which the (b-1)-bit encoding of x is - lexicographically larger than the (b-1)-bit encoding of -x. - -4.2. Keys - - - An EdDSA secret key is a b-bit string k. Let the hash H(k) = (h_0, - h_1, ..., h_(2b-1)) determine an integer a which is 2^(b-2) plus the - sum of m = 2^i * h_i for all i equal or larger than 3 and equal to or - less than b-3 such that m is a member of the set { 2^(b-2), 2^(b-2) + - 8, ..., 2^(b-1) - 8 }. The EdDSA public key is ENC(A) = ENC(aB). - The bits h_b, ..., h_(2b-1) is used below during signing. - -4.3. Sign - - - The signature of a message M under a secret key k is the 2b-bit - string ENC(R) || ENC'(S), where ENC'(S) is defined as the b-bit - little-endian encoding of S. R and S are derived as follows. First - define r = H(h_b, ... h_(2b-1)), M) interpreting 2b-bit strings in - little-endian form as integers in {0, 1, ..., 2^(2b)-1}. Let R=rB - and S=(r+H(ENC(R) || ENC(A) || M)a) mod l. - -4.4. Verify - - - To verify a signature ENC(R) || ENC'(S) on a message M under a public - key ENC(A), proceed as follows. Parse the inputs so that A and R is - an element of E, and S is a member of the set {0, 1, ..., l-1 }. - Compute H' = H(ENC(R) || ENC(A) || M) and check the group equation - 8SB = 8R + 8H'A in E. Verification is rejected if parsing fails or - the group equation does not hold. - -5. Ed25519 - - - Theoretically, Ed25519 is EdDSA instantiated with b=256, H being - SHA-512 [RFC4634], p is the prime 2^255-19, the 255-bit encoding of - GF(2^255-19) being the little-endian encoding of {0, 1, ..., - 2^255-20}, q is the prime 2^252 + 0x14def9dea2f79cd65812631a5cf5d3ed, - d = -121665/121666 which is a member of GF(p), and B is the unique - point (x, 4/5) in E for which x is "positive", which with the - encoding used simply means that the least significant bit of x is 0. - The curve p, prime q, d and B follows from [I-D.irtf-cfrg-curves]. - - Written out explicitly, B is the point (15112221349535400772501151409 - 588531511454012693041857206046113283949847762202, 4631683569492647816 - 9428394003475163141307993866256225615783033603165251855960). - - - - - - - -Josefsson & Moeller Expires August 26, 2015 [Page 5] - - -Internet-Draft EdDSA & Ed25519 February 2015 - - -5.1. Modular arithmetic - - - For advise on how to implement arithmetic modulo p = 2^255 - 1 - efficiently and securely, see Curve25519 [CURVE25519]. For inversion - modulo p, it is recommended to use the identity x^-1 = x^(p-2) (mod - p). - - For point decoding or "decompression", square roots modulo p are - needed. They can be computed using the Tonelli-Shanks algorithm, or - the special case for p = 5 (mod 8). To find a square root of a, - first compute the candidate root x = a^((p+3)/8) (mod p). Then there - are three cases: - - x^2 = a (mod p). Then x is a square root. - - x^2 = -a (mod p). Then 2^((p-1)/4) x is a square root. - - a is not a square modulo p. - -5.2. Encoding - - - All values are coded as octet strings, and integers are coded using - little endian convention. I.e., a 32-octet string h h[0],...h[31] - represents the integer h[0] + 2^8 h[1] + ... + 2^248 h[31]. - - A curve point (x,y), with coordiantes in the range 0 <= x,y < p, is - coded as follows. First encode the y-coordinate as a little-endian - string of 32 octets. The most significant bit of the final octet is - always zero. To form the encoding of the point, copy the least - significant bit of the x-coordinate to the most significant bit of - the final octet. - -5.3. Decoding - - - Decoding a point, given as a 32-octet string, is a little more - complicated. - - 1. First interpret the string as an integer in little-endian - representation. Bit 255 of this number is the least significant - bit of the x-coordinate, and denote this value x_0. The - y-coordinate is recovered simply by clearing this bit. If the - resulting value is >= p, decoding fails. - - 2. To recover the x coordinate, the curve equation implies x^2 = - (y^2 - 1) / (d y^2 + 1) (mod p). Since d is a non-square and -1 - is a square, the numerator, (d y^2 + 1), is always invertible - modulo p. Let u = y^2 - 1 and v = d y^2 + 1. To compute the - square root of (u/v), the first step is to compute the candidate - - - -Josefsson & Moeller Expires August 26, 2015 [Page 6] - - -Internet-Draft EdDSA & Ed25519 February 2015 - - - root x = (u/v)^((p+3)/8). This can be done using the following - trick, to use a single modular powering for both the inversion of - v and the square root: - - (p+3)/8 3 (p-5)/8 - x = (u/v) = u v (u v^7) (mod p) - - 3. Again, there are three cases: - - 1. If v x^2 = u (mod p), x is a square root. - - 2. If v x^2 = -u (mod p), set x <-- x 2^((p-1)/4), which is a - square root. - - 3. Otherwise, no square root exists modulo p, and decoding - fails. - - 4. Finally, use the x_0 bit to select the right square root. If x = - 0, and x_0 = 1, decoding fails. Otherwise, if x_0 != x mod 2, - set x <-- p - x. Return the decoded point (x,y). - -5.4. Point addition - - - For point addition, the following method is recommended. A point - (x,y) is represented in extended homogeneous coordinates (X, Y, Z, - T), with x = X/Z, y = Y/Z, x y = T/Z. - - The following formulas for adding two points, (x3,y3) = - (x1,y1)+(x2,y2) are described in [Edwards-revisited], section 3.1. - They are strongly unified, i.e., they work for any pair of valid - input points. - - A = (Y1-X1)*(Y2-X2) - B = (Y1+X1)*(Y2+X2) - C = T1*2*d*T2 - D = Z1*2*Z2 - E = B-A - F = D-C - G = D+C - H = B+A - X3 = E*F - Y3 = G*H - T3 = E*H - Z3 = F*G - - - - - - - -Josefsson & Moeller Expires August 26, 2015 [Page 7] - - -Internet-Draft EdDSA & Ed25519 February 2015 - - -5.5. Key Generation - - - The secret is 32 octets (256 bits, corresponding to b) of - cryptographically-secure random data. See [RFC4086] for a discussion - about randomness. - - The 32-byte public key is generated by the following steps. - - 1. Hash the 32-byte secret using SHA-512, storing the digest in a - 64-octet large buffer, denoted h. Only the lower 32 bytes are - used for generating the public key. - - 2. Prune the buffer. In C terminology: - - h[0] &= ~0x07; - h[31] &= 0x7F; - h[31] |= 0x40; - - 3. Interpret the buffer as the little-endian integer, forming a - secret scalar a. Perform a known-base-point scalar - multiplication a B. - - 4. The public key A is the encoding of the point aB. First encode - the y coordinate (in the range 0 <= y < p) as a little-endian - string of 32 octets. The most significant bit of the final octet - is always zero. To form the encoding of the point aB, copy the - least significant bit of the x coordinate to the most significant - bit of the final octet. The result is the public key. - -5.6. Sign - - - The imputs to the signing procedure is the secret key, a 32-octet - string, and a message M of arbitrary size. - - 1. Hash the secret key, 32-octets, using SHA-512. Let h denote the - resulting digest. Construct the secret scalar a from the first - half of the digest, and the corresponding public key A, as - described in the previous section. Let prefix denote the second - half of the hash digest, h[32],...,h[63]. - - 2. Compute SHA-512(prefix || M), where M is the message to be - signed. Interpret the 64-octet digest as a little-endian integer - r. - - 3. Compute the point rB. For efficiency, do this by first reducing - r modulo q, the group order of B. Let the string R be the - encoding of this point. - - - - -Josefsson & Moeller Expires August 26, 2015 [Page 8] - - -Internet-Draft EdDSA & Ed25519 February 2015 - - - 4. Compute SHA512(R || A || M), and interpret the 64-octet digest as - a little-endian integer k. - - 5. Compute s = (r + k a) mod q. For efficiency, again reduce k - modulo q first. - - 6. Form the signature of the concatenation of R (32 octets) and the - little-endian encoding of s (32 octets, three most significant - bits of the final octets always zero). - -5.7. Verify - - - 1. To verify a signature on a message M, first split the signature - into two 32-octet halves. Decode the first half as a point R, - and the second half as an integer s, in the range 0 <= s < q. If - the decoding fails, the signature is invalid. - - 2. Compute SHA512(R || A || M), and interpret the 64-octet digest as - a little-endian integer k. - - 3. Check the group equation 8s B = 8 R + 8k A. It's sufficient, but - not required, to instead check s B = R + k A. - -5.8. Python illustration - - - The rest of this section describes how Ed25519 can be implemented in - Python (version 3.2 or later) for illustration. See appendix A for - the complete implementation and appendix B for a test-driver to run - it through some test vectors. - - First some preliminaries that will be needed. - - - - - - - - - - - - - - - - - - - - -Josefsson & Moeller Expires August 26, 2015 [Page 9] - - -Internet-Draft EdDSA & Ed25519 February 2015 - - - import hashlib - - def sha512(s): - return hashlib.sha512(s).digest() - - # Base field Z_p - p = 2**255 - 19 - - def modp_inv(x): - return pow(x, p-2, p) - - # Curve constant - d = -121665 * modp_inv(121666) % p - - # Group order - q = 2**252 + 27742317777372353535851937790883648493 - - def sha512_modq(s): - return int.from_bytes(sha512(s), "little") % q - - Then follows functions to perform point operations. - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Josefsson & Moeller Expires August 26, 2015 [Page 10] - - -Internet-Draft EdDSA & Ed25519 February 2015 - - -# Points are represented as tuples (X, Y, Z, T) of extended coordinates, -# with x = X/Z, y = Y/Z, x*y = T/Z - -def point_add(P, Q): - A = (P[1]-P[0])*(Q[1]-Q[0]) % p - B = (P[1]+P[0])*(Q[1]+Q[0]) % p - C = 2 * P[3] * Q[3] * d % p - D = 2 * P[2] * Q[2] % p - E = B-A - F = D-C - G = D+C - H = B+A - return (E*F, G*H, F*G, E*H) - -# Computes Q = s * Q -def point_mul(s, P): - Q = (0, 1, 1, 0) # Neutral element - while s > 0: - # Is there any bit-set predicate? - if s & 1: - Q = point_add(Q, P) - P = point_add(P, P) - s >>= 1 - return Q - -def point_equal(P, Q): - # x1 / z1 == x2 / z2 <==> x1 * z2 == x2 * z1 - if (P[0] * Q[2] - Q[0] * P[2]) % p != 0: - return False - if (P[1] * Q[2] - Q[1] * P[2]) % p != 0: - return False - return True - - Now follows functions for point compression. - - - - - - - - - - - - - - - - - -Josefsson & Moeller Expires August 26, 2015 [Page 11] - - -Internet-Draft EdDSA & Ed25519 February 2015 - - -# Square root of -1 -modp_sqrt_m1 = pow(2, (p-1) // 4, p) - -# Compute corresponding x coordinate, with low bit corresponding to sign, -# or return None on failure -def recover_x(y, sign): - x2 = (y*y-1) * modp_inv(d*y*y+1) - if x2 == 0: - if sign: - return None - else: - return 0 - - # Compute square root of x2 - x = pow(x2, (p+3) // 8, p) - if (x*x - x2) % p != 0: - x = x * modp_sqrt_m1 % p - if (x*x - x2) % p != 0: - return None - - if (x & 1) != sign: - x = p - x - return x - -# Base point -g_y = 4 * modp_inv(5) % p -g_x = recover_x(g_y, 0) -G = (g_x, g_y, 1, g_x * g_y % p) - -def point_compress(P): - zinv = modp_inv(P[2]) - x = P[0] * zinv % p - y = P[1] * zinv % p - return int.to_bytes(y | ((x & 1) << 255), 32, "little") - -def point_decompress(s): - if len(s) != 32: - raise Exception("Invalid input length for decompression") - y = int.from_bytes(s, "little") - sign = y >> 255 - y &= (1 << 255) - 1 - - x = recover_x(y, sign) - if x is None: - return None - else: - return (x, y, 1, x*y % p) - - - - -Josefsson & Moeller Expires August 26, 2015 [Page 12] - - -Internet-Draft EdDSA & Ed25519 February 2015 - - - These are functions for manipulating the secret. - - def secret_expand(secret): - if len(secret) != 32: - raise Exception("Bad size of private key") - h = sha512(secret) - a = int.from_bytes(h[:32], "little") - a &= (1 << 254) - 8 - a |= (1 << 254) - return (a, h[32:]) - - def secret_to_public(secret): - (a, dummy) = secret_expand(secret) - return point_compress(point_mul(a, G)) - - The signature function works as below. - - def sign(secret, msg): - a, prefix = secret_expand(secret) - A = point_compress(point_mul(a, G)) - r = sha512_modq(prefix + msg) - R = point_mul(r, G) - Rs = point_compress(R) - h = sha512_modq(Rs + A + msg) - s = (r + h * a) % q - return Rs + int.to_bytes(s, 32, "little") - - And finally the verification function. - - def verify(public, msg, signature): - if len(public) != 32: - raise Exception("Bad public-key length") - if len(signature) != 64: - Exception("Bad signature length") - A = point_decompress(public) - if not A: - return False - Rs = signature[:32] - R = point_decompress(Rs) - if not R: - return False - s = int.from_bytes(signature[32:], "little") - h = sha512_modq(Rs + public + msg) - sB = point_mul(s, G) - hA = point_mul(h, A) - return point_equal(sB, point_add(R, hA)) - - - - - -Josefsson & Moeller Expires August 26, 2015 [Page 13] - - -Internet-Draft EdDSA & Ed25519 February 2015 - - -6. Test Vectors for Ed25519 - - - Below is a sequence of octets with test vectors for the the Ed25519 - signature algorithm. The octets are hex encoded and whitespace is - inserted for readability. Private keys are 64 bytes, public keys 32 - bytes, message of arbitrary length, and signatures are 64 bytes. The - test vectors are taken from [ED25519-TEST-VECTORS] (but we removed - the public key as a suffix of the secret key, and removed the message - from the signature) and [ED25519-LIBGCRYPT-TEST-VECTORS]. - - -----TEST 1 - SECRET KEY: - 9d61b19deffd5a60ba844af492ec2cc4 - 4449c5697b326919703bac031cae7f60 - - PUBLIC KEY: - d75a980182b10ab7d54bfed3c964073a - 0ee172f3daa62325af021a68f707511a - - MESSAGE (length 0 bytes): - - SIGNATURE: - e5564300c360ac729086e2cc806e828a - 84877f1eb8e5d974d873e06522490155 - 5fb8821590a33bacc61e39701cf9b46b - d25bf5f0595bbe24655141438e7a100b - - -----TEST 2 - SECRET KEY: - 4ccd089b28ff96da9db6c346ec114e0f - 5b8a319f35aba624da8cf6ed4fb8a6fb - - PUBLIC KEY: - 3d4017c3e843895a92b70aa74d1b7ebc - 9c982ccf2ec4968cc0cd55f12af4660c - - MESSAGE (length 1 byte): - 72 - - SIGNATURE: - 92a009a9f0d4cab8720e820b5f642540 - a2b27b5416503f8fb3762223ebdb69da - 085ac1e43e15996e458f3613d0f11d8c - 387b2eaeb4302aeeb00d291612bb0c00 - - -----TEST 3 - SECRET KEY: - c5aa8df43f9f837bedb7442f31dcb7b1 - - - -Josefsson & Moeller Expires August 26, 2015 [Page 14] - - -Internet-Draft EdDSA & Ed25519 February 2015 - - - 66d38535076f094b85ce3a2e0b4458f7 - - PUBLIC KEY: - fc51cd8e6218a1a38da47ed00230f058 - 0816ed13ba3303ac5deb911548908025 - - MESSAGE (length 2 bytes): - af82 - - SIGNATURE: - 6291d657deec24024827e69c3abe01a3 - 0ce548a284743a445e3680d7db5ac3ac - 18ff9b538d16f290ae67f760984dc659 - 4a7c15e9716ed28dc027beceea1ec40a - - -----TEST 1024 - SECRET KEY: - f5e5767cf153319517630f226876b86c - 8160cc583bc013744c6bf255f5cc0ee5 - - PUBLIC KEY: - 278117fc144c72340f67d0f2316e8386 - ceffbf2b2428c9c51fef7c597f1d426e - - MESSAGE: - 08b8b2b733424243760fe426a4b54908 - 632110a66c2f6591eabd3345e3e4eb98 - fa6e264bf09efe12ee50f8f54e9f77b1 - e355f6c50544e23fb1433ddf73be84d8 - 79de7c0046dc4996d9e773f4bc9efe57 - 38829adb26c81b37c93a1b270b20329d - 658675fc6ea534e0810a4432826bf58c - 941efb65d57a338bbd2e26640f89ffbc - 1a858efcb8550ee3a5e1998bd177e93a - 7363c344fe6b199ee5d02e82d522c4fe - ba15452f80288a821a579116ec6dad2b - 3b310da903401aa62100ab5d1a36553e - 06203b33890cc9b832f79ef80560ccb9 - a39ce767967ed628c6ad573cb116dbef - efd75499da96bd68a8a97b928a8bbc10 - 3b6621fcde2beca1231d206be6cd9ec7 - aff6f6c94fcd7204ed3455c68c83f4a4 - 1da4af2b74ef5c53f1d8ac70bdcb7ed1 - 85ce81bd84359d44254d95629e9855a9 - 4a7c1958d1f8ada5d0532ed8a5aa3fb2 - d17ba70eb6248e594e1a2297acbbb39d - 502f1a8c6eb6f1ce22b3de1a1f40cc24 - 554119a831a9aad6079cad88425de6bd - - - -Josefsson & Moeller Expires August 26, 2015 [Page 15] - - -Internet-Draft EdDSA & Ed25519 February 2015 - - - e1a9187ebb6092cf67bf2b13fd65f270 - 88d78b7e883c8759d2c4f5c65adb7553 - 878ad575f9fad878e80a0c9ba63bcbcc - 2732e69485bbc9c90bfbd62481d9089b - eccf80cfe2df16a2cf65bd92dd597b07 - 07e0917af48bbb75fed413d238f5555a - 7a569d80c3414a8d0859dc65a46128ba - b27af87a71314f318c782b23ebfe808b - 82b0ce26401d2e22f04d83d1255dc51a - ddd3b75a2b1ae0784504df543af8969b - e3ea7082ff7fc9888c144da2af58429e - c96031dbcad3dad9af0dcbaaaf268cb8 - fcffead94f3c7ca495e056a9b47acdb7 - 51fb73e666c6c655ade8297297d07ad1 - ba5e43f1bca32301651339e22904cc8c - 42f58c30c04aafdb038dda0847dd988d - cda6f3bfd15c4b4c4525004aa06eeff8 - ca61783aacec57fb3d1f92b0fe2fd1a8 - 5f6724517b65e614ad6808d6f6ee34df - f7310fdc82aebfd904b01e1dc54b2927 - 094b2db68d6f903b68401adebf5a7e08 - d78ff4ef5d63653a65040cf9bfd4aca7 - 984a74d37145986780fc0b16ac451649 - de6188a7dbdf191f64b5fc5e2ab47b57 - f7f7276cd419c17a3ca8e1b939ae49e4 - 88acba6b965610b5480109c8b17b80e1 - b7b750dfc7598d5d5011fd2dcc5600a3 - 2ef5b52a1ecc820e308aa342721aac09 - 43bf6686b64b2579376504ccc493d97e - 6aed3fb0f9cd71a43dd497f01f17c0e2 - cb3797aa2a2f256656168e6c496afc5f - b93246f6b1116398a346f1a641f3b041 - e989f7914f90cc2c7fff357876e506b5 - 0d334ba77c225bc307ba537152f3f161 - 0e4eafe595f6d9d90d11faa933a15ef1 - 369546868a7f3a45a96768d40fd9d034 - 12c091c6315cf4fde7cb68606937380d - b2eaaa707b4c4185c32eddcdd306705e - 4dc1ffc872eeee475a64dfac86aba41c - 0618983f8741c5ef68d3a101e8a3b8ca - c60c905c15fc910840b94c00a0b9d0 - - SIGNATURE: - 0aab4c900501b3e24d7cdf4663326a3a - 87df5e4843b2cbdb67cbf6e460fec350 - aa5371b1508f9f4528ecea23c436d94b - 5e8fcd4f681e30a6ac00a9704a188a03 - - - - -Josefsson & Moeller Expires August 26, 2015 [Page 16] - - -Internet-Draft EdDSA & Ed25519 February 2015 - - - -----TEST 1A - -----An additional test with the data from test 1 but using an - -----uncompressed public key. - SECRET KEY: - 9d61b19deffd5a60ba844af492ec2cc4 - 4449c5697b326919703bac031cae7f60 - - PUBLIC KEY: - 0455d0e09a2b9d34292297e08d60d0f6 - 20c513d47253187c24b12786bd777645 - ce1a5107f7681a02af2523a6daf372e1 - 0e3a0764c9d3fe4bd5b70ab18201985a - d7 - - MSG (length 0 bytes): - - SIGNATURE: - e5564300c360ac729086e2cc806e828a - 84877f1eb8e5d974d873e06522490155 - 5fb8821590a33bacc61e39701cf9b46b - d25bf5f0595bbe24655141438e7a100b - - -----TEST 1B - -----An additional test with the data from test 1 but using an - -----compressed prefix. - SECRET KEY: - 9d61b19deffd5a60ba844af492ec2cc4 - 4449c5697b326919703bac031cae7f60 - - PUBLIC KEY: - 40d75a980182b10ab7d54bfed3c96407 - 3a0ee172f3daa62325af021a68f70751 - 1a - - MESSAGE (length 0 bytes): - - SIGNATURE: - e5564300c360ac729086e2cc806e828a - 84877f1eb8e5d974d873e06522490155 - 5fb8821590a33bacc61e39701cf9b46b - d25bf5f0595bbe24655141438e7a100b - ----- - -7. Acknowledgements - - - Feedback on this document was received from Werner Koch and Damien - Miller. - - - - -Josefsson & Moeller Expires August 26, 2015 [Page 17] - - -Internet-Draft EdDSA & Ed25519 February 2015 - - -8. IANA Considerations - - - None. - -9. Security Considerations - - -9.1. Side-channel leaks - - - For implementations performing signatures, secrecy of the key is - fundamental. It is possible to protect against some side-channel - attacks by ensuring that the implementation executes exactly the same - sequence of instructions and performs exactly the same memory - accesses, for any value of the secret key. - - To make an implementation side-channel silent in this way, the modulo - p arithmetic must not use any data-dependent branches, e.g., related - to carry propagation. Side channel-silent point addition is - straight-forward, thanks to the unified formulas. - - Scalar multiplication, multiplying a point by an integer, needs some - additional effort to implement in a side-channel silent manner. One - simple approach is to implement a side-channel silent conditional - assignment, and use together with the binary algorithm to examine one - bit of the integer at a time. - - Note that the example implementation in this document does not - attempt to be side-channel silent. - -10. References - - -10.1. Normative References - - - [RFC4634] Eastlake, D. and T. Hansen, "US Secure Hash Algorithms - (SHA and HMAC-SHA)", RFC 4634, July 2006. - - [I-D.irtf-cfrg-curves] - Langley, A., Salz, R., and S. Turner, "Elliptic Curves for - Security", draft-irtf-cfrg-curves-01 (work in progress), - January 2015. - -10.2. Informative References - - - [RFC4086] Eastlake, D., Schiller, J., and S. Crocker, "Randomness - Requirements for Security", BCP 106, RFC 4086, June 2005. - - - - - - - -Josefsson & Moeller Expires August 26, 2015 [Page 18] - - -Internet-Draft EdDSA & Ed25519 February 2015 - - - [EDDSA] Bernstein, D., Duif, N., Lange, T., Schwabe, P., and B. - Yang, "High-speed high-security signatures", WWW - http://ed25519.cr.yp.to/ed25519-20110926.pdf, September - 2011. - - [Faster-ECC] - Bernstein, D. and T. Lange, "Faster addition and doubling - on elliptic curves", WWW http://eprint.iacr.org/2007/286, - July 2007. - - [Edwards-revisited] - Hisil, H., Wong, K., Carter, G., and E. Dawson, "Twisted - Edwards Curves Revisited", WWW - http://eprint.iacr.org/2008/522, December 2008. - - [CURVE25519] - Bernstein, D., "Curve25519: new Diffie-Hellman speed - records", WWW http://cr.yp.to/ecdh.html, February 2006. - - [ED25519-TEST-VECTORS] - Bernstein, D., Duif, N., Lange, T., Schwabe, P., and B. - Yang, "Ed25519 test vectors", WWW - http://ed25519.cr.yp.to/python/sign.input, July 2011. - - [ED25519-LIBGCRYPT-TEST-VECTORS] - Koch, W., "Ed25519 Libgcrypt test vectors", WWW - http://git.gnupg.org/cgi- - bin/gitweb.cgi?p=libgcrypt.git;a=blob;f=tests/t-ed25519.in - p;h=e13566f826321eece65e02c593bc7d885b3dbe23;hb=refs/ - heads/master, July 2014. - -Appendix A. Ed25519 Python Library - - - Below is an example implementation of Ed25519 written in Python, - version 3.2 or higher is required. - -# Loosely based on the public domain code at -# http://ed25519.cr.yp.to/software.html -# -# Needs python-3.2 - -import hashlib - - -def sha512(s): - return hashlib.sha512(s).digest() - -# Base field Z_p - - - -Josefsson & Moeller Expires August 26, 2015 [Page 19] - - -Internet-Draft EdDSA & Ed25519 February 2015 - - -p = 2**255 - 19 - - -def modp_inv(x): - return pow(x, p-2, p) - -# Curve constant -d = -121665 * modp_inv(121666) % p - -# Group order -q = 2**252 + 27742317777372353535851937790883648493 - - -def sha512_modq(s): - return int.from_bytes(sha512(s), "little") % q - -# Points are represented as tuples (X, Y, Z, T) of extended coordinates, -# with x = X/Z, y = Y/Z, x*y = T/Z - - -def point_add(P, Q): - A = (P[1]-P[0])*(Q[1]-Q[0]) % p - B = (P[1]+P[0])*(Q[1]+Q[0]) % p - C = 2 * P[3] * Q[3] * d % p - D = 2 * P[2] * Q[2] % p - E = B-A - F = D-C - G = D+C - H = B+A - return (E*F, G*H, F*G, E*H) - - -# Computes Q = s * Q -def point_mul(s, P): - Q = (0, 1, 1, 0) # Neutral element - while s > 0: - # Is there any bit-set predicate? - if s & 1: - Q = point_add(Q, P) - P = point_add(P, P) - s >>= 1 - return Q - - -def point_equal(P, Q): - # x1 / z1 == x2 / z2 <==> x1 * z2 == x2 * z1 - if (P[0] * Q[2] - Q[0] * P[2]) % p != 0: - return False - - - -Josefsson & Moeller Expires August 26, 2015 [Page 20] - - -Internet-Draft EdDSA & Ed25519 February 2015 - - - if (P[1] * Q[2] - Q[1] * P[2]) % p != 0: - return False - return True - -# Square root of -1 -modp_sqrt_m1 = pow(2, (p-1) // 4, p) - - -# Compute corresponding x coordinate, with low bit corresponding to sign, -# or return None on failure -def recover_x(y, sign): - x2 = (y*y-1) * modp_inv(d*y*y+1) - if x2 == 0: - if sign: - return None - else: - return 0 - - # Compute square root of x2 - x = pow(x2, (p+3) // 8, p) - if (x*x - x2) % p != 0: - x = x * modp_sqrt_m1 % p - if (x*x - x2) % p != 0: - return None - - if (x & 1) != sign: - x = p - x - return x - -# Base point -g_y = 4 * modp_inv(5) % p -g_x = recover_x(g_y, 0) -G = (g_x, g_y, 1, g_x * g_y % p) - - -def point_compress(P): - zinv = modp_inv(P[2]) - x = P[0] * zinv % p - y = P[1] * zinv % p - return int.to_bytes(y | ((x & 1) << 255), 32, "little") - - -def point_decompress(s): - if len(s) != 32: - raise Exception("Invalid input length for decompression") - y = int.from_bytes(s, "little") - sign = y >> 255 - y &= (1 << 255) - 1 - - - -Josefsson & Moeller Expires August 26, 2015 [Page 21] - - -Internet-Draft EdDSA & Ed25519 February 2015 - - - x = recover_x(y, sign) - if x is None: - return None - else: - return (x, y, 1, x*y % p) - - -def secret_expand(secret): - if len(secret) != 32: - raise Exception("Bad size of private key") - h = sha512(secret) - a = int.from_bytes(h[:32], "little") - a &= (1 << 254) - 8 - a |= (1 << 254) - return (a, h[32:]) - - -def secret_to_public(secret): - (a, dummy) = secret_expand(secret) - return point_compress(point_mul(a, G)) - - -def sign(secret, msg): - a, prefix = secret_expand(secret) - A = point_compress(point_mul(a, G)) - r = sha512_modq(prefix + msg) - R = point_mul(r, G) - Rs = point_compress(R) - h = sha512_modq(Rs + A + msg) - s = (r + h * a) % q - return Rs + int.to_bytes(s, 32, "little") - - -def verify(public, msg, signature): - if len(public) != 32: - raise Exception("Bad public-key length") - if len(signature) != 64: - Exception("Bad signature length") - A = point_decompress(public) - if not A: - return False - Rs = signature[:32] - R = point_decompress(Rs) - if not R: - return False - s = int.from_bytes(signature[32:], "little") - h = sha512_modq(Rs + public + msg) - sB = point_mul(s, G) - - - -Josefsson & Moeller Expires August 26, 2015 [Page 22] - - -Internet-Draft EdDSA & Ed25519 February 2015 - - - hA = point_mul(h, A) - return point_equal(sB, point_add(R, hA)) - -Appendix B. Library driver - - - Below is a command-line tool that uses the library above to perform - computations, for interactive use or for self-checking. - - import sys - import binascii - - from ed25519 import * - - def point_valid(P): - zinv = modp_inv(P[2]) - x = P[0] * zinv % p - y = P[1] * zinv % p - assert (x*y - P[3]*zinv) % p == 0 - return (-x*x + y*y - 1 - d*x*x*y*y) % p == 0 - - assert point_valid(G) - Z = (0, 1, 1, 0) - assert point_valid(Z) - - assert point_equal(Z, point_add(Z, Z)) - assert point_equal(G, point_add(Z, G)) - assert point_equal(Z, point_mul(0, G)) - assert point_equal(G, point_mul(1, G)) - assert point_equal(point_add(G, G), point_mul(2, G)) - for i in range(0, 100): - assert point_valid(point_mul(i, G)) - assert point_equal(Z, point_mul(q, G)) - - def munge_string(s, pos, change): - return (s[:pos] + - int.to_bytes(s[pos] ^ change, 1, "little") + - s[pos+1:]) - - # Read a file in the format of - # http://ed25519.cr.yp.to/python/sign.input - lineno = 0 - while True: - line = sys.stdin.readline() - if not line: - break - lineno = lineno + 1 - print(lineno) - fields = line.split(":") - - - -Josefsson & Moeller Expires August 26, 2015 [Page 23] - - -Internet-Draft EdDSA & Ed25519 February 2015 - - - secret = (binascii.unhexlify(fields[0]))[:32] - public = binascii.unhexlify(fields[1]) - msg = binascii.unhexlify(fields[2]) - signature = binascii.unhexlify(fields[3])[:64] - - assert public == secret_to_public(secret) - assert signature == sign(secret, msg) - assert verify(public, msg, signature) - if len(msg) == 0: - bad_msg = b"x" - else: - bad_msg = munge_string(msg, len(msg) // 3, 4) - assert not verify(public, bad_msg, signature) - bad_signature = munge_string(signature, 20, 8) - assert not verify(public, msg, bad_signature) - bad_signature = munge_string(signature, 40, 16) - assert not verify(public, msg, bad_signature) - -Authors' Addresses - - Simon Josefsson - SJD AB - - Email: simon@josefsson.org - URI: http://josefsson.org/ - - - Niels Moeller - - Email: nisse@lysator.liu.se - - - - - - - - - - - - - - - - - - - - - -Josefsson & Moeller Expires August 26, 2015 [Page 24] - - - -Html markup produced by rfcmarkup 1.113, available from https://tools.ietf.org/tools/rfcmarkup/ +Note: This draft is now superseded by https://datatracker.ietf.org/doc/rfc8032/ +(review of the differences is left as an exercise for the reader) diff --git a/src/crypto/crypto_ops_builder/ref10/README.md b/src/crypto/crypto_ops_builder/ref10/README.md new file mode 100644 index 00000000000..59193305b95 --- /dev/null +++ b/src/crypto/crypto_ops_builder/ref10/README.md @@ -0,0 +1,4 @@ +This code comes from Daniel J. Bernstein's SUPERCOP source, +released in the public domain. + +[http://ed25519.cr.yp.to/software.html](http://ed25519.cr.yp.to/software.html) diff --git a/src/crypto/crypto_ops_builder/ref10/description b/src/crypto/crypto_ops_builder/ref10/description index cbfcb2cbafd..99f74774705 100644 --- a/src/crypto/crypto_ops_builder/ref10/description +++ b/src/crypto/crypto_ops_builder/ref10/description @@ -1,2 +1,2 @@ EdDSA signatures using Curve25519 -from http://hyperelliptic.org/ebats/supercop-20141124.tar.bz2 +from https://hyperelliptic.org/ebats/supercop-20141124.tar.bz2 diff --git a/src/crypto/crypto_ops_builder/ref10CommentedCombined/MakeCryptoOps.py b/src/crypto/crypto_ops_builder/ref10CommentedCombined/MakeCryptoOps.py index 9b55d260d02..dfba583f796 100644 --- a/src/crypto/crypto_ops_builder/ref10CommentedCombined/MakeCryptoOps.py +++ b/src/crypto/crypto_ops_builder/ref10CommentedCombined/MakeCryptoOps.py @@ -1,5 +1,5 @@ #assumes you have gnu sed, osx sed might need slight syntax changeo -#c.f. http://unix.stackexchange.com/questions/112023/how-can-i-replace-a-string-in-a-files +#c.f. https://unix.stackexchange.com/questions/112023/how-can-i-replace-a-string-in-a-files #written by shen-noether monero research labs @@ -8,14 +8,14 @@ import textwrap #for comments etc print("make sure you have cat and grep installed") -print("also assumes gnu sed syntax, c.f. :http://unix.stackexchange.com/questions/112023/how-can-i-replace-a-string-in-a-files") +print("also assumes gnu sed syntax, c.f. :https://unix.stackexchange.com/questions/112023/how-can-i-replace-a-string-in-a-files") print("I believe osx may have slightly different version of sed") print("maybe someone smart can replace the sed with perl..") a = "" license = textwrap.dedent("""\ - // Copyright (c) 2014-2018, The Monero Project + // Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/crypto/crypto_ops_builder/ref10CommentedCombined/crypto-ops.h b/src/crypto/crypto_ops_builder/ref10CommentedCombined/crypto-ops.h index c06af035ffa..f62ff441d89 100644 --- a/src/crypto/crypto_ops_builder/ref10CommentedCombined/crypto-ops.h +++ b/src/crypto/crypto_ops_builder/ref10CommentedCombined/crypto-ops.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/crypto/crypto_ops_builder/ref10CommentedCombined/description b/src/crypto/crypto_ops_builder/ref10CommentedCombined/description index fadc9f9afba..9407b400aa1 100644 --- a/src/crypto/crypto_ops_builder/ref10CommentedCombined/description +++ b/src/crypto/crypto_ops_builder/ref10CommentedCombined/description @@ -2,6 +2,6 @@ shen_ed25519_ref10 MakeCryptoOps.py makes crypto-ops.c in the Monero source from the ref10 implementation EdDSA signatures using Curve25519 -from http://hyperelliptic.org/ebats/supercop-20141124.tar.bz2 +from https://hyperelliptic.org/ebats/supercop-20141124.tar.bz2 Commented / combined by Shen Noether, Monero Research Lab diff --git a/src/crypto/generic-ops.h b/src/crypto/generic-ops.h index 62bc758c9d7..d067266380e 100644 --- a/src/crypto/generic-ops.h +++ b/src/crypto/generic-ops.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -33,19 +33,30 @@ #include #include #include +#include #define CRYPTO_MAKE_COMPARABLE(type) \ namespace crypto { \ inline bool operator==(const type &_v1, const type &_v2) { \ - return std::memcmp(&_v1, &_v2, sizeof(type)) == 0; \ + return !memcmp(&_v1, &_v2, sizeof(_v1)); \ } \ inline bool operator!=(const type &_v1, const type &_v2) { \ - return std::memcmp(&_v1, &_v2, sizeof(type)) != 0; \ + return !operator==(_v1, _v2); \ } \ } -#define CRYPTO_MAKE_HASHABLE(type) \ -CRYPTO_MAKE_COMPARABLE(type) \ +#define CRYPTO_MAKE_COMPARABLE_CONSTANT_TIME(type) \ +namespace crypto { \ + inline bool operator==(const type &_v1, const type &_v2) { \ + static_assert(sizeof(_v1) == 32, "constant time comparison is only implenmted for 32 bytes"); \ + return crypto_verify_32((const unsigned char*)&_v1, (const unsigned char*)&_v2) == 0; \ + } \ + inline bool operator!=(const type &_v1, const type &_v2) { \ + return !operator==(_v1, _v2); \ + } \ +} + +#define CRYPTO_DEFINE_HASH_FUNCTIONS(type) \ namespace crypto { \ static_assert(sizeof(std::size_t) <= sizeof(type), "Size of " #type " must be at least that of size_t"); \ inline std::size_t hash_value(const type &_v) { \ @@ -60,3 +71,12 @@ namespace std { \ } \ }; \ } + +#define CRYPTO_MAKE_HASHABLE(type) \ +CRYPTO_MAKE_COMPARABLE(type) \ +CRYPTO_DEFINE_HASH_FUNCTIONS(type) + +#define CRYPTO_MAKE_HASHABLE_CONSTANT_TIME(type) \ +CRYPTO_MAKE_COMPARABLE_CONSTANT_TIME(type) \ +CRYPTO_DEFINE_HASH_FUNCTIONS(type) + diff --git a/src/crypto/groestl.c b/src/crypto/groestl.c index c8258add334..d5e2989a885 100644 --- a/src/crypto/groestl.c +++ b/src/crypto/groestl.c @@ -20,9 +20,15 @@ const uint8_t shift_Values[2][8] = {{0,1,2,3,4,5,6,7},{1,3,5,7,0,2,4,6}}; const uint8_t indices_cyclic[15] = {0,1,2,3,4,5,6,7,0,1,2,3,4,5,6}; +#if BYTE_ORDER == LITTLE_ENDIAN #define ROTATE_COLUMN_DOWN(v1, v2, amount_bytes, temp_var) {temp_var = (v1<<(8*amount_bytes))|(v2>>(8*(4-amount_bytes))); \ v2 = (v2<<(8*amount_bytes))|(v1>>(8*(4-amount_bytes))); \ v1 = temp_var;} +#else +#define ROTATE_COLUMN_DOWN(v1, v2, amount_bytes, temp_var) {temp_var = (v1>>(8*amount_bytes))|(v2<<(8*(4-amount_bytes))); \ + v2 = (v2>>(8*amount_bytes))|(v1<<(8*(4-amount_bytes))); \ + v1 = temp_var;} +#endif #define COLUMN(x,y,i,c0,c1,c2,c3,c4,c5,c6,c7,tv1,tv2,tu,tl,t) \ @@ -68,14 +74,14 @@ const uint8_t indices_cyclic[15] = {0,1,2,3,4,5,6,7,0,1,2,3,4,5,6}; static void RND512P(uint8_t *x, uint32_t *y, uint32_t r) { uint32_t temp_v1, temp_v2, temp_upper_value, temp_lower_value, temp; uint32_t* x32 = (uint32_t*)x; - x32[ 0] ^= 0x00000000^r; - x32[ 2] ^= 0x00000010^r; - x32[ 4] ^= 0x00000020^r; - x32[ 6] ^= 0x00000030^r; - x32[ 8] ^= 0x00000040^r; - x32[10] ^= 0x00000050^r; - x32[12] ^= 0x00000060^r; - x32[14] ^= 0x00000070^r; + x32[ 0] ^= SWAP32LE(0x00000000)^r; + x32[ 2] ^= SWAP32LE(0x00000010)^r; + x32[ 4] ^= SWAP32LE(0x00000020)^r; + x32[ 6] ^= SWAP32LE(0x00000030)^r; + x32[ 8] ^= SWAP32LE(0x00000040)^r; + x32[10] ^= SWAP32LE(0x00000050)^r; + x32[12] ^= SWAP32LE(0x00000060)^r; + x32[14] ^= SWAP32LE(0x00000070)^r; COLUMN(x,y, 0, 0, 2, 4, 6, 9, 11, 13, 15, temp_v1, temp_v2, temp_upper_value, temp_lower_value, temp); COLUMN(x,y, 2, 2, 4, 6, 8, 11, 13, 15, 1, temp_v1, temp_v2, temp_upper_value, temp_lower_value, temp); COLUMN(x,y, 4, 4, 6, 8, 10, 13, 15, 1, 3, temp_v1, temp_v2, temp_upper_value, temp_lower_value, temp); @@ -91,21 +97,22 @@ static void RND512Q(uint8_t *x, uint32_t *y, uint32_t r) { uint32_t temp_v1, temp_v2, temp_upper_value, temp_lower_value, temp; uint32_t* x32 = (uint32_t*)x; x32[ 0] = ~x32[ 0]; - x32[ 1] ^= 0xffffffff^r; + x32[ 1] ^= SWAP32LE(0xffffffff)^r; x32[ 2] = ~x32[ 2]; - x32[ 3] ^= 0xefffffff^r; + x32[ 3] ^= SWAP32LE(0xefffffff)^r; x32[ 4] = ~x32[ 4]; - x32[ 5] ^= 0xdfffffff^r; + x32[ 5] ^= SWAP32LE(0xdfffffff)^r; x32[ 6] = ~x32[ 6]; - x32[ 7] ^= 0xcfffffff^r; + x32[ 7] ^= SWAP32LE(0xcfffffff)^r; x32[ 8] = ~x32[ 8]; - x32[ 9] ^= 0xbfffffff^r; + x32[ 9] ^= SWAP32LE(0xbfffffff)^r; x32[10] = ~x32[10]; - x32[11] ^= 0xafffffff^r; + x32[11] ^= SWAP32LE(0xafffffff)^r; x32[12] = ~x32[12]; - x32[13] ^= 0x9fffffff^r; + x32[13] ^= SWAP32LE(0x9fffffff)^r; x32[14] = ~x32[14]; - x32[15] ^= 0x8fffffff^r; + x32[15] ^= SWAP32LE(0x8fffffff)^r; + COLUMN(x,y, 0, 2, 6, 10, 14, 1, 5, 9, 13, temp_v1, temp_v2, temp_upper_value, temp_lower_value, temp); COLUMN(x,y, 2, 4, 8, 12, 0, 3, 7, 11, 15, temp_v1, temp_v2, temp_upper_value, temp_lower_value, temp); COLUMN(x,y, 4, 6, 10, 14, 2, 5, 9, 13, 1, temp_v1, temp_v2, temp_upper_value, temp_lower_value, temp); @@ -130,28 +137,28 @@ static void F512(uint32_t *h, const uint32_t *m) { } /* compute Q(m) */ - RND512Q((uint8_t*)z, y, 0x00000000); - RND512Q((uint8_t*)y, z, 0x01000000); - RND512Q((uint8_t*)z, y, 0x02000000); - RND512Q((uint8_t*)y, z, 0x03000000); - RND512Q((uint8_t*)z, y, 0x04000000); - RND512Q((uint8_t*)y, z, 0x05000000); - RND512Q((uint8_t*)z, y, 0x06000000); - RND512Q((uint8_t*)y, z, 0x07000000); - RND512Q((uint8_t*)z, y, 0x08000000); - RND512Q((uint8_t*)y, Qtmp, 0x09000000); + RND512Q((uint8_t*)z, y, SWAP32LE(0x00000000)); + RND512Q((uint8_t*)y, z, SWAP32LE(0x01000000)); + RND512Q((uint8_t*)z, y, SWAP32LE(0x02000000)); + RND512Q((uint8_t*)y, z, SWAP32LE(0x03000000)); + RND512Q((uint8_t*)z, y, SWAP32LE(0x04000000)); + RND512Q((uint8_t*)y, z, SWAP32LE(0x05000000)); + RND512Q((uint8_t*)z, y, SWAP32LE(0x06000000)); + RND512Q((uint8_t*)y, z, SWAP32LE(0x07000000)); + RND512Q((uint8_t*)z, y, SWAP32LE(0x08000000)); + RND512Q((uint8_t*)y, Qtmp, SWAP32LE(0x09000000)); /* compute P(h+m) */ - RND512P((uint8_t*)Ptmp, y, 0x00000000); - RND512P((uint8_t*)y, z, 0x00000001); - RND512P((uint8_t*)z, y, 0x00000002); - RND512P((uint8_t*)y, z, 0x00000003); - RND512P((uint8_t*)z, y, 0x00000004); - RND512P((uint8_t*)y, z, 0x00000005); - RND512P((uint8_t*)z, y, 0x00000006); - RND512P((uint8_t*)y, z, 0x00000007); - RND512P((uint8_t*)z, y, 0x00000008); - RND512P((uint8_t*)y, Ptmp, 0x00000009); + RND512P((uint8_t*)Ptmp, y, SWAP32LE(0x00000000)); + RND512P((uint8_t*)y, z, SWAP32LE(0x00000001)); + RND512P((uint8_t*)z, y, SWAP32LE(0x00000002)); + RND512P((uint8_t*)y, z, SWAP32LE(0x00000003)); + RND512P((uint8_t*)z, y, SWAP32LE(0x00000004)); + RND512P((uint8_t*)y, z, SWAP32LE(0x00000005)); + RND512P((uint8_t*)z, y, SWAP32LE(0x00000006)); + RND512P((uint8_t*)y, z, SWAP32LE(0x00000007)); + RND512P((uint8_t*)z, y, SWAP32LE(0x00000008)); + RND512P((uint8_t*)y, Ptmp, SWAP32LE(0x00000009)); /* compute P(h+m) + Q(m) + h */ for (i = 0; i < 2*COLS512; i++) { @@ -188,16 +195,16 @@ static void OutputTransformation(hashState *ctx) { for (j = 0; j < 2*COLS512; j++) { temp[j] = ctx->chaining[j]; } - RND512P((uint8_t*)temp, y, 0x00000000); - RND512P((uint8_t*)y, z, 0x00000001); - RND512P((uint8_t*)z, y, 0x00000002); - RND512P((uint8_t*)y, z, 0x00000003); - RND512P((uint8_t*)z, y, 0x00000004); - RND512P((uint8_t*)y, z, 0x00000005); - RND512P((uint8_t*)z, y, 0x00000006); - RND512P((uint8_t*)y, z, 0x00000007); - RND512P((uint8_t*)z, y, 0x00000008); - RND512P((uint8_t*)y, temp, 0x00000009); + RND512P((uint8_t*)temp, y, SWAP32LE(0x00000000)); + RND512P((uint8_t*)y, z, SWAP32LE(0x00000001)); + RND512P((uint8_t*)z, y, SWAP32LE(0x00000002)); + RND512P((uint8_t*)y, z, SWAP32LE(0x00000003)); + RND512P((uint8_t*)z, y, SWAP32LE(0x00000004)); + RND512P((uint8_t*)y, z, SWAP32LE(0x00000005)); + RND512P((uint8_t*)z, y, SWAP32LE(0x00000006)); + RND512P((uint8_t*)y, z, SWAP32LE(0x00000007)); + RND512P((uint8_t*)z, y, SWAP32LE(0x00000008)); + RND512P((uint8_t*)y, temp, SWAP32LE(0x00000009)); for (j = 0; j < 2*COLS512; j++) { ctx->chaining[j] ^= temp[j]; } @@ -213,7 +220,7 @@ static void Init(hashState* ctx) { } /* set initial value */ - ctx->chaining[2*COLS512-1] = u32BIG((uint32_t)HASH_BIT_LEN); + ctx->chaining[2*COLS512-1] = SWAP32LE(u32BIG((uint32_t)HASH_BIT_LEN)); /* set other variables */ ctx->buf_ptr = 0; diff --git a/src/crypto/groestl.h b/src/crypto/groestl.h index 19837f3097c..6628947ddee 100644 --- a/src/crypto/groestl.h +++ b/src/crypto/groestl.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/crypto/groestl_tables.h b/src/crypto/groestl_tables.h index c4b368584e1..ca0c4fca646 100644 --- a/src/crypto/groestl_tables.h +++ b/src/crypto/groestl_tables.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -29,7 +29,10 @@ #ifndef __tables_h #define __tables_h +#include "int-util.h" + +#if BYTE_ORDER == LITTLE_ENDIAN const uint32_t T[512] = {0xa5f432c6, 0xc6a597f4, 0x84976ff8, 0xf884eb97, 0x99b05eee, 0xee99c7b0, 0x8d8c7af6, 0xf68df78c, 0xd17e8ff, 0xff0de517, 0xbddc0ad6, 0xd6bdb7dc, 0xb1c816de, 0xdeb1a7c8, 0x54fc6d91, 0x915439fc , 0x50f09060, 0x6050c0f0, 0x3050702, 0x2030405, 0xa9e02ece, 0xcea987e0, 0x7d87d156, 0x567dac87, 0x192bcce7, 0xe719d52b, 0x62a613b5, 0xb56271a6, 0xe6317c4d, 0x4de69a31, 0x9ab559ec, 0xec9ac3b5 , 0x45cf408f, 0x8f4505cf, 0x9dbca31f, 0x1f9d3ebc, 0x40c04989, 0x894009c0, 0x879268fa, 0xfa87ef92, 0x153fd0ef, 0xef15c53f, 0xeb2694b2, 0xb2eb7f26, 0xc940ce8e, 0x8ec90740, 0xb1de6fb, 0xfb0bed1d @@ -62,5 +65,39 @@ const uint32_t T[512] = {0xa5f432c6, 0xc6a597f4, 0x84976ff8, 0xf884eb97, 0x99b05 , 0xb6c1ec2d, 0x2db65ac1, 0x22665a3c, 0x3c227866, 0x92adb815, 0x15922aad, 0x2060a9c9, 0xc9208960, 0x49db5c87, 0x874915db, 0xff1ab0aa, 0xaaff4f1a, 0x7888d850, 0x5078a088, 0x7a8e2ba5, 0xa57a518e , 0x8f8a8903, 0x38f068a, 0xf8134a59, 0x59f8b213, 0x809b9209, 0x980129b, 0x1739231a, 0x1a173439, 0xda751065, 0x65daca75, 0x315384d7, 0xd731b553, 0xc651d584, 0x84c61351, 0xb8d303d0, 0xd0b8bbd3 , 0xc35edc82, 0x82c31f5e, 0xb0cbe229, 0x29b052cb, 0x7799c35a, 0x5a77b499, 0x11332d1e, 0x1e113c33, 0xcb463d7b, 0x7bcbf646, 0xfc1fb7a8, 0xa8fc4b1f, 0xd6610c6d, 0x6dd6da61, 0x3a4e622c, 0x2c3a584e}; +#else +const uint32_t T[512] = {0xc632f4a5, 0xf497a5c6, 0xf86f9784, 0x97eb84f8, 0xee5eb099, 0xb0c799ee, 0xf67a8c8d, 0x8cf78df6, 0xffe8170d, 0x17e50dff, 0xd60adcbd, 0xdcb7bdd6, 0xde16c8b1, 0xc8a7b1de, 0x916dfc54, 0xfc395491 +, 0x6090f050, 0xf0c05060, 0x02070503, 0x05040302, 0xce2ee0a9, 0xe087a9ce, 0x56d1877d, 0x87ac7d56, 0xe7cc2b19, 0x2bd519e7, 0xb513a662, 0xa67162b5, 0x4d7c31e6, 0x319ae64d, 0xec59b59a, 0xb5c39aec +, 0x8f40cf45, 0xcf05458f, 0x1fa3bc9d, 0xbc3e9d1f, 0x8949c040, 0xc0094089, 0xfa689287, 0x92ef87fa, 0xefd03f15, 0x3fc515ef, 0xb29426eb, 0x267febb2, 0x8ece40c9, 0x4007c98e, 0xfbe61d0b, 0x1ded0bfb +, 0x416e2fec, 0x2f82ec41, 0xb31aa967, 0xa97d67b3, 0x5f431cfd, 0x1cbefd5f, 0x456025ea, 0x258aea45, 0x23f9dabf, 0xda46bf23, 0x535102f7, 0x02a6f753, 0xe445a196, 0xa1d396e4, 0x9b76ed5b, 0xed2d5b9b +, 0x75285dc2, 0x5deac275, 0xe1c5241c, 0x24d91ce1, 0x3dd4e9ae, 0xe97aae3d, 0x4cf2be6a, 0xbe986a4c, 0x6c82ee5a, 0xeed85a6c, 0x7ebdc341, 0xc3fc417e, 0xf5f30602, 0x06f102f5, 0x8352d14f, 0xd11d4f83 +, 0x688ce45c, 0xe4d05c68, 0x515607f4, 0x07a2f451, 0xd18d5c34, 0x5cb934d1, 0xf9e11808, 0x18e908f9, 0xe24cae93, 0xaedf93e2, 0xab3e9573, 0x954d73ab, 0x6297f553, 0xf5c45362, 0x2a6b413f, 0x41543f2a +, 0x081c140c, 0x14100c08, 0x9563f652, 0xf6315295, 0x46e9af65, 0xaf8c6546, 0x9d7fe25e, 0xe2215e9d, 0x30487828, 0x78602830, 0x37cff8a1, 0xf86ea137, 0x0a1b110f, 0x11140f0a, 0x2febc4b5, 0xc45eb52f +, 0x0e151b09, 0x1b1c090e, 0x247e5a36, 0x5a483624, 0x1badb69b, 0xb6369b1b, 0xdf98473d, 0x47a53ddf, 0xcda76a26, 0x6a8126cd, 0x4ef5bb69, 0xbb9c694e, 0x7f334ccd, 0x4cfecd7f, 0xea50ba9f, 0xbacf9fea +, 0x123f2d1b, 0x2d241b12, 0x1da4b99e, 0xb93a9e1d, 0x58c49c74, 0x9cb07458, 0x3446722e, 0x72682e34, 0x3641772d, 0x776c2d36, 0xdc11cdb2, 0xcda3b2dc, 0xb49d29ee, 0x2973eeb4, 0x5b4d16fb, 0x16b6fb5b +, 0xa4a501f6, 0x0153f6a4, 0x76a1d74d, 0xd7ec4d76, 0xb714a361, 0xa37561b7, 0x7d3449ce, 0x49face7d, 0x52df8d7b, 0x8da47b52, 0xdd9f423e, 0x42a13edd, 0x5ecd9371, 0x93bc715e, 0x13b1a297, 0xa2269713 +, 0xa6a204f5, 0x0457f5a6, 0xb901b868, 0xb86968b9, 0x00000000, 0x00000000, 0xc1b5742c, 0x74992cc1, 0x40e0a060, 0xa0806040, 0xe3c2211f, 0x21dd1fe3, 0x793a43c8, 0x43f2c879, 0xb69a2ced, 0x2c77edb6 +, 0xd40dd9be, 0xd9b3bed4, 0x8d47ca46, 0xca01468d, 0x671770d9, 0x70ced967, 0x72afdd4b, 0xdde44b72, 0x94ed79de, 0x7933de94, 0x98ff67d4, 0x672bd498, 0xb09323e8, 0x237be8b0, 0x855bde4a, 0xde114a85 +, 0xbb06bd6b, 0xbd6d6bbb, 0xc5bb7e2a, 0x7e912ac5, 0x4f7b34e5, 0x349ee54f, 0xedd73a16, 0x3ac116ed, 0x86d254c5, 0x5417c586, 0x9af862d7, 0x622fd79a, 0x6699ff55, 0xffcc5566, 0x11b6a794, 0xa7229411 +, 0x8ac04acf, 0x4a0fcf8a, 0xe9d93010, 0x30c910e9, 0x040e0a06, 0x0a080604, 0xfe669881, 0x98e781fe, 0xa0ab0bf0, 0x0b5bf0a0, 0x78b4cc44, 0xccf04478, 0x25f0d5ba, 0xd54aba25, 0x4b753ee3, 0x3e96e34b +, 0xa2ac0ef3, 0x0e5ff3a2, 0x5d4419fe, 0x19bafe5d, 0x80db5bc0, 0x5b1bc080, 0x0580858a, 0x850a8a05, 0x3fd3ecad, 0xec7ead3f, 0x21fedfbc, 0xdf42bc21, 0x70a8d848, 0xd8e04870, 0xf1fd0c04, 0x0cf904f1 +, 0x63197adf, 0x7ac6df63, 0x772f58c1, 0x58eec177, 0xaf309f75, 0x9f4575af, 0x42e7a563, 0xa5846342, 0x20705030, 0x50403020, 0xe5cb2e1a, 0x2ed11ae5, 0xfdef120e, 0x12e10efd, 0xbf08b76d, 0xb7656dbf +, 0x8155d44c, 0xd4194c81, 0x18243c14, 0x3c301418, 0x26795f35, 0x5f4c3526, 0xc3b2712f, 0x719d2fc3, 0xbe8638e1, 0x3867e1be, 0x35c8fda2, 0xfd6aa235, 0x88c74fcc, 0x4f0bcc88, 0x2e654b39, 0x4b5c392e +, 0x936af957, 0xf93d5793, 0x55580df2, 0x0daaf255, 0xfc619d82, 0x9de382fc, 0x7ab3c947, 0xc9f4477a, 0xc827efac, 0xef8bacc8, 0xba8832e7, 0x326fe7ba, 0x324f7d2b, 0x7d642b32, 0xe642a495, 0xa4d795e6 +, 0xc03bfba0, 0xfb9ba0c0, 0x19aab398, 0xb3329819, 0x9ef668d1, 0x6827d19e, 0xa322817f, 0x815d7fa3, 0x44eeaa66, 0xaa886644, 0x54d6827e, 0x82a87e54, 0x3bdde6ab, 0xe676ab3b, 0x0b959e83, 0x9e16830b +, 0x8cc945ca, 0x4503ca8c, 0xc7bc7b29, 0x7b9529c7, 0x6b056ed3, 0x6ed6d36b, 0x286c443c, 0x44503c28, 0xa72c8b79, 0x8b5579a7, 0xbc813de2, 0x3d63e2bc, 0x1631271d, 0x272c1d16, 0xad379a76, 0x9a4176ad +, 0xdb964d3b, 0x4dad3bdb, 0x649efa56, 0xfac85664, 0x74a6d24e, 0xd2e84e74, 0x1436221e, 0x22281e14, 0x92e476db, 0x763fdb92, 0x0c121e0a, 0x1e180a0c, 0x48fcb46c, 0xb4906c48, 0xb88f37e4, 0x376be4b8 +, 0x9f78e75d, 0xe7255d9f, 0xbd0fb26e, 0xb2616ebd, 0x43692aef, 0x2a86ef43, 0xc435f1a6, 0xf193a6c4, 0x39dae3a8, 0xe372a839, 0x31c6f7a4, 0xf762a431, 0xd38a5937, 0x59bd37d3, 0xf274868b, 0x86ff8bf2 +, 0xd5835632, 0x56b132d5, 0x8b4ec543, 0xc50d438b, 0x6e85eb59, 0xebdc596e, 0xda18c2b7, 0xc2afb7da, 0x018e8f8c, 0x8f028c01, 0xb11dac64, 0xac7964b1, 0x9cf16dd2, 0x6d23d29c, 0x49723be0, 0x3b92e049 +, 0xd81fc7b4, 0xc7abb4d8, 0xacb915fa, 0x1543faac, 0xf3fa0907, 0x09fd07f3, 0xcfa06f25, 0x6f8525cf, 0xca20eaaf, 0xea8fafca, 0xf47d898e, 0x89f38ef4, 0x476720e9, 0x208ee947, 0x10382818, 0x28201810 +, 0x6f0b64d5, 0x64ded56f, 0xf0738388, 0x83fb88f0, 0x4afbb16f, 0xb1946f4a, 0x5cca9672, 0x96b8725c, 0x38546c24, 0x6c702438, 0x575f08f1, 0x08aef157, 0x732152c7, 0x52e6c773, 0x9764f351, 0xf3355197 +, 0xcbae6523, 0x658d23cb, 0xa125847c, 0x84597ca1, 0xe857bf9c, 0xbfcb9ce8, 0x3e5d6321, 0x637c213e, 0x96ea7cdd, 0x7c37dd96, 0x611e7fdc, 0x7fc2dc61, 0x0d9c9186, 0x911a860d, 0x0f9b9485, 0x941e850f +, 0xe04bab90, 0xabdb90e0, 0x7cbac642, 0xc6f8427c, 0x712657c4, 0x57e2c471, 0xcc29e5aa, 0xe583aacc, 0x90e373d8, 0x733bd890, 0x06090f05, 0x0f0c0506, 0xf7f40301, 0x03f501f7, 0x1c2a3612, 0x3638121c +, 0xc23cfea3, 0xfe9fa3c2, 0x6a8be15f, 0xe1d45f6a, 0xaebe10f9, 0x1047f9ae, 0x69026bd0, 0x6bd2d069, 0x17bfa891, 0xa82e9117, 0x9971e858, 0xe8295899, 0x3a536927, 0x6974273a, 0x27f7d0b9, 0xd04eb927 +, 0xd9914838, 0x48a938d9, 0xebde3513, 0x35cd13eb, 0x2be5ceb3, 0xce56b32b, 0x22775533, 0x55443322, 0xd204d6bb, 0xd6bfbbd2, 0xa9399070, 0x904970a9, 0x07878089, 0x800e8907, 0x33c1f2a7, 0xf266a733 +, 0x2decc1b6, 0xc15ab62d, 0x3c5a6622, 0x6678223c, 0x15b8ad92, 0xad2a9215, 0xc9a96020, 0x608920c9, 0x875cdb49, 0xdb154987, 0xaab01aff, 0x1a4fffaa, 0x50d88878, 0x88a07850, 0xa52b8e7a, 0x8e517aa5 +, 0x03898a8f, 0x8a068f03, 0x594a13f8, 0x13b2f859, 0x09929b80, 0x9b128009, 0x1a233917, 0x3934171a, 0x651075da, 0x75cada65, 0xd7845331, 0x53b531d7, 0x84d551c6, 0x5113c684, 0xd003d3b8, 0xd3bbb8d0 +, 0x82dc5ec3, 0x5e1fc382, 0x29e2cbb0, 0xcb52b029, 0x5ac39977, 0x99b4775a, 0x1e2d3311, 0x333c111e, 0x7b3d46cb, 0x46f6cb7b, 0xa8b71ffc, 0x1f4bfca8, 0x6d0c61d6, 0x61dad66d, 0x2c624e3a, 0x4e583a2c}; +#endif #endif /* __tables_h */ diff --git a/src/crypto/hash-extra-blake.c b/src/crypto/hash-extra-blake.c index d33103c9764..9bada96f360 100644 --- a/src/crypto/hash-extra-blake.c +++ b/src/crypto/hash-extra-blake.c @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/crypto/hash-extra-groestl.c b/src/crypto/hash-extra-groestl.c index 228853a44e6..57866bf9d36 100644 --- a/src/crypto/hash-extra-groestl.c +++ b/src/crypto/hash-extra-groestl.c @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/crypto/hash-extra-jh.c b/src/crypto/hash-extra-jh.c index e765a18f39c..0dbac4fb58d 100644 --- a/src/crypto/hash-extra-jh.c +++ b/src/crypto/hash-extra-jh.c @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/crypto/hash-extra-skein.c b/src/crypto/hash-extra-skein.c index 06d8f87cc76..78f48609fec 100644 --- a/src/crypto/hash-extra-skein.c +++ b/src/crypto/hash-extra-skein.c @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/crypto/hash-ops.h b/src/crypto/hash-ops.h index d77d55cf30a..859c810bdea 100644 --- a/src/crypto/hash-ops.h +++ b/src/crypto/hash-ops.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -37,7 +37,7 @@ #include #include -#include "common/int-util.h" +#include "int-util.h" #include "warnings.h" static inline void *padd(void *p, size_t i) { @@ -79,7 +79,7 @@ enum { }; void cn_fast_hash(const void *data, size_t length, char *hash); -void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int prehashed); +void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int prehashed, uint64_t height); void hash_extra_blake(const void *data, size_t length, char *hash); void hash_extra_groestl(const void *data, size_t length, char *hash); diff --git a/src/crypto/hash.c b/src/crypto/hash.c index 42f272e3473..b66f3b0104f 100644 --- a/src/crypto/hash.c +++ b/src/crypto/hash.c @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -36,7 +36,14 @@ #include "keccak.h" void hash_permutation(union hash_state *state) { +#if BYTE_ORDER == LITTLE_ENDIAN keccakf((uint64_t*)state, 24); +#else + uint64_t le_state[25]; + memcpy_swap64le(le_state, state, 25); + keccakf(le_state, 24); + memcpy_swap64le(state, le_state, 25); +#endif } void hash_process(union hash_state *state, const uint8_t *buf, size_t count) { diff --git a/src/crypto/hash.h b/src/crypto/hash.h index 995e2294ed0..17071923dc2 100644 --- a/src/crypto/hash.h +++ b/src/crypto/hash.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -71,12 +71,12 @@ namespace crypto { return h; } - inline void cn_slow_hash(const void *data, std::size_t length, hash &hash, int variant = 0) { - cn_slow_hash(data, length, reinterpret_cast(&hash), variant, 0/*prehashed*/); + inline void cn_slow_hash(const void *data, std::size_t length, hash &hash, int variant = 0, uint64_t height = 0) { + cn_slow_hash(data, length, reinterpret_cast(&hash), variant, 0/*prehashed*/, height); } - inline void cn_slow_hash_prehashed(const void *data, std::size_t length, hash &hash, int variant = 0) { - cn_slow_hash(data, length, reinterpret_cast(&hash), variant, 1/*prehashed*/); + inline void cn_slow_hash_prehashed(const void *data, std::size_t length, hash &hash, int variant = 0, uint64_t height = 0) { + cn_slow_hash(data, length, reinterpret_cast(&hash), variant, 1/*prehashed*/, height); } inline void tree_hash(const hash *hashes, std::size_t count, hash &root_hash) { diff --git a/src/crypto/hmac-keccak.c b/src/crypto/hmac-keccak.c new file mode 100644 index 00000000000..edcb2065e5d --- /dev/null +++ b/src/crypto/hmac-keccak.c @@ -0,0 +1,81 @@ +// Copyright (c) 2014-2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "hmac-keccak.h" +#include "memwipe.h" + +#define KECCAK_BLOCKLEN 136 +#define HASH_SIZE 32 + +void hmac_keccak_init(hmac_keccak_state *S, const uint8_t *_key, size_t keylen) { + const uint8_t *key = _key; + uint8_t keyhash[HASH_SIZE]; + uint8_t pad[KECCAK_BLOCKLEN]; + uint64_t i; + + if (keylen > KECCAK_BLOCKLEN) { + keccak(key, keylen, keyhash, HASH_SIZE); + key = keyhash; + keylen = HASH_SIZE; + } + + keccak_init(&S->inner); + memset(pad, 0x36, KECCAK_BLOCKLEN); + for (i = 0; i < keylen; ++i) { + pad[i] ^= key[i]; + } + keccak_update(&S->inner, pad, KECCAK_BLOCKLEN); + + keccak_init(&S->outer); + memset(pad, 0x5c, KECCAK_BLOCKLEN); + for (i = 0; i < keylen; ++i) { + pad[i] ^= key[i]; + } + keccak_update(&S->outer, pad, KECCAK_BLOCKLEN); + + memwipe(keyhash, HASH_SIZE); +} + +void hmac_keccak_update(hmac_keccak_state *S, const uint8_t *data, size_t datalen) { + keccak_update(&S->inner, data, datalen); +} + +void hmac_keccak_finish(hmac_keccak_state *S, uint8_t *digest) { + uint8_t ihash[HASH_SIZE]; + keccak_finish(&S->inner, ihash); + keccak_update(&S->outer, ihash, HASH_SIZE); + keccak_finish(&S->outer, digest); + memwipe(ihash, HASH_SIZE); +} + +void hmac_keccak_hash(uint8_t *out, const uint8_t *key, size_t keylen, const uint8_t *in, size_t inlen) { + hmac_keccak_state S; + hmac_keccak_init(&S, key, keylen); + hmac_keccak_update(&S, in, inlen); + hmac_keccak_finish(&S, out); +} diff --git a/src/crypto/hmac-keccak.h b/src/crypto/hmac-keccak.h new file mode 100644 index 00000000000..c450860d41c --- /dev/null +++ b/src/crypto/hmac-keccak.h @@ -0,0 +1,59 @@ +// Copyright (c) 2014-2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#ifndef HMAC_KECCAK_H +#define HMAC_KECCAK_H + +#include "keccak.h" + +// HMAC RFC 2104 with Keccak-256 base hash function +// +// B = KECCAK_BLOCKLEN = 136 B +// L = HASH_SIZE = 32 B +// +// Note this is not HMAC-SHA3 as SHA3 and Keccak differs in +// the padding constant. + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct { + KECCAK_CTX inner; + KECCAK_CTX outer; +} hmac_keccak_state; + +void hmac_keccak_init(hmac_keccak_state *S, const uint8_t *_key, size_t keylen); +void hmac_keccak_update(hmac_keccak_state *S, const uint8_t *data, size_t datalen); +void hmac_keccak_finish(hmac_keccak_state *S, uint8_t *digest); +void hmac_keccak_hash(uint8_t *out, const uint8_t *key, size_t keylen, const uint8_t *in, size_t inlen); + +#ifdef __cplusplus +} +#endif +#endif //HMAC_KECCAK_H diff --git a/src/crypto/initializer.h b/src/crypto/initializer.h index afbace726b6..75d80f054e4 100644 --- a/src/crypto/initializer.h +++ b/src/crypto/initializer.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -43,8 +43,8 @@ #elif defined(_MSC_VER) #include #include -// http://stackoverflow.com/questions/1113409/attribute-constructor-equivalent-in-vc -// http://msdn.microsoft.com/en-us/library/bb918180.aspx +// https://stackoverflow.com/questions/1113409/attribute-constructor-equivalent-in-vc +// https://msdn.microsoft.com/en-us/library/bb918180.aspx #pragma section(".CRT$XCT", read) #define INITIALIZER(name) \ static void __cdecl name(void); \ diff --git a/src/crypto/keccak.c b/src/crypto/keccak.c index de8e2a5b320..851c70a2531 100644 --- a/src/crypto/keccak.c +++ b/src/crypto/keccak.c @@ -5,6 +5,7 @@ #include #include #include +#include "int-util.h" #include "hash-ops.h" #include "keccak.h" @@ -104,9 +105,12 @@ void keccak(const uint8_t *in, size_t inlen, uint8_t *md, int mdlen) memset(st, 0, sizeof(st)); for ( ; inlen >= rsiz; inlen -= rsiz, in += rsiz) { - for (i = 0; i < rsizw; i++) - st[i] ^= ((uint64_t *) in)[i]; - keccakf(st, KECCAK_ROUNDS); + for (i = 0; i < rsizw; i++) { + uint64_t ina; + memcpy(&ina, in + i * 8, 8); + st[i] ^= swap64le(ina); + } + keccakf(st, KECCAK_ROUNDS); } // last block and padding @@ -115,20 +119,100 @@ void keccak(const uint8_t *in, size_t inlen, uint8_t *md, int mdlen) local_abort("Bad keccak use"); } - memcpy(temp, in, inlen); + if (inlen > 0) + memcpy(temp, in, inlen); temp[inlen++] = 1; memset(temp + inlen, 0, rsiz - inlen); temp[rsiz - 1] |= 0x80; for (i = 0; i < rsizw; i++) - st[i] ^= ((uint64_t *) temp)[i]; + st[i] ^= swap64le(((uint64_t *) temp)[i]); keccakf(st, KECCAK_ROUNDS); - memcpy(md, st, mdlen); + if (((size_t)mdlen % sizeof(uint64_t)) != 0) + { + local_abort("Bad keccak use"); + } + memcpy_swap64le(md, st, mdlen/sizeof(uint64_t)); } void keccak1600(const uint8_t *in, size_t inlen, uint8_t *md) { keccak(in, inlen, md, sizeof(state_t)); } + +#define KECCAK_FINALIZED 0x80000000 +#define KECCAK_BLOCKLEN 136 +#define KECCAK_WORDS 17 +#define KECCAK_DIGESTSIZE 32 +#define IS_ALIGNED_64(p) (0 == (7 & ((const char*)(p) - (const char*)0))) +#define KECCAK_PROCESS_BLOCK(st, block) { \ + for (int i_ = 0; i_ < KECCAK_WORDS; i_++){ \ + ((st))[i_] ^= swap64le(((block))[i_]); \ + }; \ + keccakf(st, KECCAK_ROUNDS); } + + +void keccak_init(KECCAK_CTX * ctx){ + memset(ctx, 0, sizeof(KECCAK_CTX)); +} + +void keccak_update(KECCAK_CTX * ctx, const uint8_t *in, size_t inlen){ + if (ctx->rest & KECCAK_FINALIZED) { + local_abort("Bad keccak use"); + } + + const size_t idx = ctx->rest; + ctx->rest = (ctx->rest + inlen) % KECCAK_BLOCKLEN; + + // fill partial block + if (idx) { + size_t left = KECCAK_BLOCKLEN - idx; + memcpy((char*)ctx->message + idx, in, (inlen < left ? inlen : left)); + if (inlen < left) return; + + KECCAK_PROCESS_BLOCK(ctx->hash, ctx->message); + + in += left; + inlen -= left; + } + + const bool is_aligned = IS_ALIGNED_64(in); + while (inlen >= KECCAK_BLOCKLEN) { + const uint64_t* aligned_message_block; + if (is_aligned) { + aligned_message_block = (uint64_t*)in; + } else { + memcpy(ctx->message, in, KECCAK_BLOCKLEN); + aligned_message_block = ctx->message; + } + + KECCAK_PROCESS_BLOCK(ctx->hash, aligned_message_block); + in += KECCAK_BLOCKLEN; + inlen -= KECCAK_BLOCKLEN; + } + if (inlen) { + memcpy(ctx->message, in, inlen); + } +} + +void keccak_finish(KECCAK_CTX * ctx, uint8_t *md){ + if (!(ctx->rest & KECCAK_FINALIZED)) + { + // clear the rest of the data queue + memset((char*)ctx->message + ctx->rest, 0, KECCAK_BLOCKLEN - ctx->rest); + ((char*)ctx->message)[ctx->rest] |= 0x01; + ((char*)ctx->message)[KECCAK_BLOCKLEN - 1] |= 0x80; + + // process final block + KECCAK_PROCESS_BLOCK(ctx->hash, ctx->message); + ctx->rest = KECCAK_FINALIZED; // mark context as finalized + } + + static_assert(KECCAK_BLOCKLEN > KECCAK_DIGESTSIZE, ""); + static_assert(KECCAK_DIGESTSIZE % sizeof(uint64_t) == 0, ""); + if (md) { + memcpy_swap64le(md, ctx->hash, KECCAK_DIGESTSIZE / sizeof(uint64_t)); + } +} diff --git a/src/crypto/keccak.h b/src/crypto/keccak.h index fb9d8bd0430..9123c7a3b58 100644 --- a/src/crypto/keccak.h +++ b/src/crypto/keccak.h @@ -15,6 +15,17 @@ #define ROTL64(x, y) (((x) << (y)) | ((x) >> (64 - (y)))) #endif +// SHA3 Algorithm context. +typedef struct KECCAK_CTX +{ + // 1600 bits algorithm hashing state + uint64_t hash[25]; + // 1088-bit buffer for leftovers, block size = 136 B for 256-bit keccak + uint64_t message[17]; + // count of bytes in the message[] buffer + size_t rest; +} KECCAK_CTX; + // compute a keccak hash (md) of given byte length from "in" void keccak(const uint8_t *in, size_t inlen, uint8_t *md, int mdlen); @@ -23,4 +34,7 @@ void keccakf(uint64_t st[25], int norounds); void keccak1600(const uint8_t *in, size_t inlen, uint8_t *md); +void keccak_init(KECCAK_CTX * ctx); +void keccak_update(KECCAK_CTX * ctx, const uint8_t *in, size_t inlen); +void keccak_finish(KECCAK_CTX * ctx, uint8_t *md); #endif diff --git a/src/crypto/oaes_lib.c b/src/crypto/oaes_lib.c index 9e31ebf4649..210f5d43a11 100644 --- a/src/crypto/oaes_lib.c +++ b/src/crypto/oaes_lib.c @@ -33,14 +33,15 @@ #include #include -// OS X, FreeBSD, and OpenBSD don't need malloc.h +// OS X, FreeBSD, OpenBSD and NetBSD don't need malloc.h #if !defined(__APPLE__) && !defined(__FreeBSD__) && !defined(__OpenBSD__) \ - && !defined(__DragonFly__) + && !defined(__DragonFly__) && !defined(__NetBSD__) #include #endif -// ANDROID, FreeBSD, and OpenBSD also don't need timeb.h -#if !defined(__FreeBSD__) && !defined(__OpenBSD__) && !defined(__ANDROID__) +// ANDROID, FreeBSD, OpenBSD and NetBSD also don't need timeb.h +#if !defined(__FreeBSD__) && !defined(__OpenBSD__) && !defined(__ANDROID__) \ + && !defined(__NetBSD__) #include #else #include @@ -473,7 +474,7 @@ OAES_RET oaes_sprintf( #ifdef OAES_HAVE_ISAAC static void oaes_get_seed( char buf[RANDSIZ + 1] ) { - #if !defined(__FreeBSD__) && !defined(__OpenBSD__) + #if !defined(__FreeBSD__) && !defined(__OpenBSD__) && !defined(__NetBSD__) struct timeb timer; struct tm *gmTimer; char * _test = NULL; @@ -505,7 +506,7 @@ static void oaes_get_seed( char buf[RANDSIZ + 1] ) #else static uint32_t oaes_get_seed(void) { - #if !defined(__FreeBSD__) && !defined(__OpenBSD__) && !defined(__ANDROID__) + #if !defined(__FreeBSD__) && !defined(__OpenBSD__) && !defined(__ANDROID__) && !defined(__NetBSD__) struct timeb timer; struct tm *gmTimer; char * _test = NULL; diff --git a/src/crypto/random.c b/src/crypto/random.c index 9e1a70a2dcd..74b202661fd 100644 --- a/src/crypto/random.c +++ b/src/crypto/random.c @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/crypto/random.h b/src/crypto/random.h index 6468136cc92..ccb9f4853d4 100644 --- a/src/crypto/random.h +++ b/src/crypto/random.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/crypto/skein_port.h b/src/crypto/skein_port.h index a50a28e6baf..1ec07a4d13b 100644 --- a/src/crypto/skein_port.h +++ b/src/crypto/skein_port.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -114,7 +114,7 @@ typedef uint64_t u64b_t; /* 64-bit unsigned integer */ #ifndef SKEIN_NEED_SWAP /* compile-time "override" for endianness? */ -#include "common/int-util.h" +#include "int-util.h" #define IS_BIG_ENDIAN 4321 /* byte 0 is most significant (mc68k) */ #define IS_LITTLE_ENDIAN 1234 /* byte 0 is least significant (i386) */ diff --git a/src/crypto/slow-hash.c b/src/crypto/slow-hash.c index 35e98f2f580..7f36c9dc3d4 100644 --- a/src/crypto/slow-hash.c +++ b/src/crypto/slow-hash.c @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -35,9 +35,15 @@ #include #include -#include "common/int-util.h" +#include "int-util.h" #include "hash-ops.h" #include "oaes_lib.h" +#include "variant2_int_sqrt.h" +#include "variant4_random_math.h" +#include "CryptonightR_JIT.h" + +#include +#include #define MEMORY (1 << 21) // 2MB scratchpad #define ITER (1 << 20) @@ -46,11 +52,46 @@ #define INIT_SIZE_BLK 8 #define INIT_SIZE_BYTE (INIT_SIZE_BLK * AES_BLOCK_SIZE) -extern int aesb_single_round(const uint8_t *in, uint8_t*out, const uint8_t *expandedKey); -extern int aesb_pseudo_round(const uint8_t *in, uint8_t *out, const uint8_t *expandedKey); +extern void aesb_single_round(const uint8_t *in, uint8_t *out, const uint8_t *expandedKey); +extern void aesb_pseudo_round(const uint8_t *in, uint8_t *out, const uint8_t *expandedKey); + +static void local_abort(const char *msg) +{ + fprintf(stderr, "%s\n", msg); +#ifdef NDEBUG + _exit(1); +#else + abort(); +#endif +} + +volatile int use_v4_jit_flag = -1; + +static inline int use_v4_jit(void) +{ +#if defined(__x86_64__) + + if (use_v4_jit_flag != -1) + return use_v4_jit_flag; + + const char *env = getenv("MONERO_USE_CNV4_JIT"); + if (!env) { + use_v4_jit_flag = 1; + } + else if (!strcmp(env, "0") || !strcmp(env, "no")) { + use_v4_jit_flag = 0; + } + else { + use_v4_jit_flag = 1; + } + return use_v4_jit_flag; +#else + return 0; +#endif +} #define VARIANT1_1(p) \ - do if (variant > 0) \ + do if (variant == 1) \ { \ const uint8_t tmp = ((const uint8_t*)(p))[11]; \ static const uint32_t table = 0x75310; \ @@ -59,7 +100,7 @@ extern int aesb_pseudo_round(const uint8_t *in, uint8_t *out, const uint8_t *exp } while(0) #define VARIANT1_2(p) \ - do if (variant > 0) \ + do if (variant == 1) \ { \ xor64(p, tweak1_2); \ } while(0) @@ -67,7 +108,7 @@ extern int aesb_pseudo_round(const uint8_t *in, uint8_t *out, const uint8_t *exp #define VARIANT1_CHECK() \ do if (length < 43) \ { \ - fprintf(stderr, "Cryptonight variants need at least 43 bytes of data"); \ + fprintf(stderr, "Cryptonight variant 1 needs at least 43 bytes of data"); \ _exit(1); \ } while(0) @@ -75,7 +116,7 @@ extern int aesb_pseudo_round(const uint8_t *in, uint8_t *out, const uint8_t *exp #define VARIANT1_PORTABLE_INIT() \ uint8_t tweak1_2[8]; \ - do if (variant > 0) \ + do if (variant == 1) \ { \ VARIANT1_CHECK(); \ memcpy(&tweak1_2, &state.hs.b[192], sizeof(tweak1_2)); \ @@ -83,11 +124,224 @@ extern int aesb_pseudo_round(const uint8_t *in, uint8_t *out, const uint8_t *exp } while(0) #define VARIANT1_INIT64() \ - if (variant > 0) \ + if (variant == 1) \ { \ VARIANT1_CHECK(); \ } \ - const uint64_t tweak1_2 = variant > 0 ? (state.hs.w[24] ^ (*((const uint64_t*)NONCE_POINTER))) : 0 + const uint64_t tweak1_2 = (variant == 1) ? (state.hs.w[24] ^ (*((const uint64_t*)NONCE_POINTER))) : 0 + +#define VARIANT2_INIT64() \ + uint64_t division_result = 0; \ + uint64_t sqrt_result = 0; \ + do if (variant >= 2) \ + { \ + U64(b)[2] = state.hs.w[8] ^ state.hs.w[10]; \ + U64(b)[3] = state.hs.w[9] ^ state.hs.w[11]; \ + division_result = state.hs.w[12]; \ + sqrt_result = state.hs.w[13]; \ + } while (0) + +#define VARIANT2_PORTABLE_INIT() \ + uint64_t division_result = 0; \ + uint64_t sqrt_result = 0; \ + do if (variant >= 2) \ + { \ + memcpy(b + AES_BLOCK_SIZE, state.hs.b + 64, AES_BLOCK_SIZE); \ + xor64(b + AES_BLOCK_SIZE, state.hs.b + 80); \ + xor64(b + AES_BLOCK_SIZE + 8, state.hs.b + 88); \ + division_result = SWAP64LE(state.hs.w[12]); \ + sqrt_result = SWAP64LE(state.hs.w[13]); \ + } while (0) + +#define VARIANT2_SHUFFLE_ADD_SSE2(base_ptr, offset) \ + do if (variant >= 2) \ + { \ + __m128i chunk1 = _mm_load_si128((__m128i *)((base_ptr) + ((offset) ^ 0x10))); \ + const __m128i chunk2 = _mm_load_si128((__m128i *)((base_ptr) + ((offset) ^ 0x20))); \ + const __m128i chunk3 = _mm_load_si128((__m128i *)((base_ptr) + ((offset) ^ 0x30))); \ + _mm_store_si128((__m128i *)((base_ptr) + ((offset) ^ 0x10)), _mm_add_epi64(chunk3, _b1)); \ + _mm_store_si128((__m128i *)((base_ptr) + ((offset) ^ 0x20)), _mm_add_epi64(chunk1, _b)); \ + _mm_store_si128((__m128i *)((base_ptr) + ((offset) ^ 0x30)), _mm_add_epi64(chunk2, _a)); \ + if (variant >= 4) \ + { \ + chunk1 = _mm_xor_si128(chunk1, chunk2); \ + _c = _mm_xor_si128(_c, chunk3); \ + _c = _mm_xor_si128(_c, chunk1); \ + } \ + } while (0) + +#define VARIANT2_SHUFFLE_ADD_NEON(base_ptr, offset) \ + do if (variant >= 2) \ + { \ + uint64x2_t chunk1 = vld1q_u64(U64((base_ptr) + ((offset) ^ 0x10))); \ + const uint64x2_t chunk2 = vld1q_u64(U64((base_ptr) + ((offset) ^ 0x20))); \ + const uint64x2_t chunk3 = vld1q_u64(U64((base_ptr) + ((offset) ^ 0x30))); \ + vst1q_u64(U64((base_ptr) + ((offset) ^ 0x10)), vaddq_u64(chunk3, vreinterpretq_u64_u8(_b1))); \ + vst1q_u64(U64((base_ptr) + ((offset) ^ 0x20)), vaddq_u64(chunk1, vreinterpretq_u64_u8(_b))); \ + vst1q_u64(U64((base_ptr) + ((offset) ^ 0x30)), vaddq_u64(chunk2, vreinterpretq_u64_u8(_a))); \ + if (variant >= 4) \ + { \ + chunk1 = veorq_u64(chunk1, chunk2); \ + _c = vreinterpretq_u8_u64(veorq_u64(vreinterpretq_u64_u8(_c), chunk3)); \ + _c = vreinterpretq_u8_u64(veorq_u64(vreinterpretq_u64_u8(_c), chunk1)); \ + } \ + } while (0) + +#define VARIANT2_PORTABLE_SHUFFLE_ADD(out, a_, base_ptr, offset) \ + do if (variant >= 2) \ + { \ + uint64_t* chunk1 = U64((base_ptr) + ((offset) ^ 0x10)); \ + uint64_t* chunk2 = U64((base_ptr) + ((offset) ^ 0x20)); \ + uint64_t* chunk3 = U64((base_ptr) + ((offset) ^ 0x30)); \ + \ + uint64_t chunk1_old[2] = { SWAP64LE(chunk1[0]), SWAP64LE(chunk1[1]) }; \ + const uint64_t chunk2_old[2] = { SWAP64LE(chunk2[0]), SWAP64LE(chunk2[1]) }; \ + const uint64_t chunk3_old[2] = { SWAP64LE(chunk3[0]), SWAP64LE(chunk3[1]) }; \ + \ + uint64_t b1[2]; \ + memcpy_swap64le(b1, b + 16, 2); \ + chunk1[0] = SWAP64LE(chunk3_old[0] + b1[0]); \ + chunk1[1] = SWAP64LE(chunk3_old[1] + b1[1]); \ + \ + uint64_t a0[2]; \ + memcpy_swap64le(a0, a_, 2); \ + chunk3[0] = SWAP64LE(chunk2_old[0] + a0[0]); \ + chunk3[1] = SWAP64LE(chunk2_old[1] + a0[1]); \ + \ + uint64_t b0[2]; \ + memcpy_swap64le(b0, b, 2); \ + chunk2[0] = SWAP64LE(chunk1_old[0] + b0[0]); \ + chunk2[1] = SWAP64LE(SWAP64LE(chunk1_old[1]) + b0[1]); \ + if (variant >= 4) \ + { \ + uint64_t out_copy[2]; \ + memcpy_swap64le(out_copy, out, 2); \ + chunk1_old[0] ^= chunk2_old[0]; \ + chunk1_old[1] ^= chunk2_old[1]; \ + out_copy[0] ^= chunk3_old[0]; \ + out_copy[1] ^= chunk3_old[1]; \ + out_copy[0] ^= chunk1_old[0]; \ + out_copy[1] ^= chunk1_old[1]; \ + memcpy_swap64le(out, out_copy, 2); \ + } \ + } while (0) + +#define VARIANT2_INTEGER_MATH_DIVISION_STEP(b, ptr) \ + uint64_t tmpx = division_result ^ (sqrt_result << 32); \ + ((uint64_t*)(b))[0] ^= SWAP64LE(tmpx); \ + { \ + const uint64_t dividend = SWAP64LE(((uint64_t*)(ptr))[1]); \ + const uint32_t divisor = (SWAP64LE(((uint64_t*)(ptr))[0]) + (uint32_t)(sqrt_result << 1)) | 0x80000001UL; \ + division_result = ((uint32_t)(dividend / divisor)) + \ + (((uint64_t)(dividend % divisor)) << 32); \ + } \ + const uint64_t sqrt_input = SWAP64LE(((uint64_t*)(ptr))[0]) + division_result + +#define VARIANT2_INTEGER_MATH_SSE2(b, ptr) \ + do if ((variant == 2) || (variant == 3)) \ + { \ + VARIANT2_INTEGER_MATH_DIVISION_STEP(b, ptr); \ + VARIANT2_INTEGER_MATH_SQRT_STEP_SSE2(); \ + VARIANT2_INTEGER_MATH_SQRT_FIXUP(sqrt_result); \ + } while(0) + +#if defined DBL_MANT_DIG && (DBL_MANT_DIG >= 50) + // double precision floating point type has enough bits of precision on current platform + #define VARIANT2_PORTABLE_INTEGER_MATH(b, ptr) \ + do if ((variant == 2) || (variant == 3)) \ + { \ + VARIANT2_INTEGER_MATH_DIVISION_STEP(b, ptr); \ + VARIANT2_INTEGER_MATH_SQRT_STEP_FP64(); \ + VARIANT2_INTEGER_MATH_SQRT_FIXUP(sqrt_result); \ + } while (0) +#else + // double precision floating point type is not good enough on current platform + // fall back to the reference code (integer only) + #define VARIANT2_PORTABLE_INTEGER_MATH(b, ptr) \ + do if ((variant == 2) || (variant == 3)) \ + { \ + VARIANT2_INTEGER_MATH_DIVISION_STEP(b, ptr); \ + VARIANT2_INTEGER_MATH_SQRT_STEP_REF(); \ + } while (0) +#endif + +#define VARIANT2_2_PORTABLE() \ + if (variant == 2 || variant == 3) { \ + xor_blocks(long_state + (j ^ 0x10), d); \ + xor_blocks(d, long_state + (j ^ 0x20)); \ + } + +#define VARIANT2_2() \ + do if (variant == 2 || variant == 3) \ + { \ + *U64(local_hp_state + (j ^ 0x10)) ^= SWAP64LE(hi); \ + *(U64(local_hp_state + (j ^ 0x10)) + 1) ^= SWAP64LE(lo); \ + hi ^= SWAP64LE(*U64(local_hp_state + (j ^ 0x20))); \ + lo ^= SWAP64LE(*(U64(local_hp_state + (j ^ 0x20)) + 1)); \ + } while (0) + +#define V4_REG_LOAD(dst, src) \ + do { \ + memcpy((dst), (src), sizeof(v4_reg)); \ + if (sizeof(v4_reg) == sizeof(uint32_t)) \ + *(dst) = SWAP32LE(*(dst)); \ + else \ + *(dst) = SWAP64LE(*(dst)); \ + } while (0) + +#define VARIANT4_RANDOM_MATH_INIT() \ + v4_reg r[9]; \ + struct V4_Instruction code[NUM_INSTRUCTIONS_MAX + 1]; \ + int jit = use_v4_jit(); \ + do if (variant >= 4) \ + { \ + for (int i = 0; i < 4; ++i) \ + V4_REG_LOAD(r + i, (uint8_t*)(state.hs.w + 12) + sizeof(v4_reg) * i); \ + v4_random_math_init(code, height); \ + if (jit) \ + { \ + int ret = v4_generate_JIT_code(code, hp_jitfunc, 4096); \ + if (ret < 0) \ + local_abort("Error generating CryptonightR code"); \ + } \ + } while (0) + +#define VARIANT4_RANDOM_MATH(a, b, r, _b, _b1) \ + do if (variant >= 4) \ + { \ + uint64_t t[2]; \ + memcpy(t, b, sizeof(uint64_t)); \ + \ + if (sizeof(v4_reg) == sizeof(uint32_t)) \ + t[0] ^= SWAP64LE((r[0] + r[1]) | ((uint64_t)(r[2] + r[3]) << 32)); \ + else \ + t[0] ^= SWAP64LE((r[0] + r[1]) ^ (r[2] + r[3])); \ + \ + memcpy(b, t, sizeof(uint64_t)); \ + \ + V4_REG_LOAD(r + 4, a); \ + V4_REG_LOAD(r + 5, (uint64_t*)(a) + 1); \ + V4_REG_LOAD(r + 6, _b); \ + V4_REG_LOAD(r + 7, _b1); \ + V4_REG_LOAD(r + 8, (uint64_t*)(_b1) + 1); \ + \ + if (jit) \ + (*hp_jitfunc)(r); \ + else \ + v4_random_math(code, r); \ + \ + memcpy(t, a, sizeof(uint64_t) * 2); \ + \ + if (sizeof(v4_reg) == sizeof(uint32_t)) { \ + t[0] ^= SWAP64LE(r[2] | ((uint64_t)(r[3]) << 32)); \ + t[1] ^= SWAP64LE(r[0] | ((uint64_t)(r[1]) << 32)); \ + } else { \ + t[0] ^= SWAP64LE(r[2] ^ r[3]); \ + t[1] ^= SWAP64LE(r[0] ^ r[1]); \ + } \ + memcpy(a, t, sizeof(uint64_t) * 2); \ + } while (0) + #if !defined NO_AES && (defined(__x86_64__) || (defined(_MSC_VER) && defined(_WIN64))) // Optimised code below, uses x86-specific intrinsics, SSE2, AES-NI @@ -151,7 +405,7 @@ extern int aesb_pseudo_round(const uint8_t *in, uint8_t *out, const uint8_t *exp #define pre_aes() \ j = state_index(a); \ - _c = _mm_load_si128(R128(&hp_state[j])); \ + _c = _mm_load_si128(R128(&local_hp_state[j])); \ _a = _mm_load_si128(R128(a)); \ /* @@ -164,19 +418,24 @@ extern int aesb_pseudo_round(const uint8_t *in, uint8_t *out, const uint8_t *exp * This code is based upon an optimized implementation by dga. */ #define post_aes() \ + VARIANT2_SHUFFLE_ADD_SSE2(local_hp_state, j); \ _mm_store_si128(R128(c), _c); \ - _b = _mm_xor_si128(_b, _c); \ - _mm_store_si128(R128(&hp_state[j]), _b); \ - VARIANT1_1(&hp_state[j]); \ + _mm_store_si128(R128(&local_hp_state[j]), _mm_xor_si128(_b, _c)); \ + VARIANT1_1(&local_hp_state[j]); \ j = state_index(c); \ - p = U64(&hp_state[j]); \ + p = U64(&local_hp_state[j]); \ b[0] = p[0]; b[1] = p[1]; \ + VARIANT2_INTEGER_MATH_SSE2(b, c); \ + VARIANT4_RANDOM_MATH(a, b, r, &_b, &_b1); \ __mul(); \ + VARIANT2_2(); \ + VARIANT2_SHUFFLE_ADD_SSE2(local_hp_state, j); \ a[0] += hi; a[1] += lo; \ - p = U64(&hp_state[j]); \ + p = U64(&local_hp_state[j]); \ p[0] = a[0]; p[1] = a[1]; \ a[0] ^= b[0]; a[1] ^= b[1]; \ VARIANT1_2(p + 1); \ + _b1 = _b; \ _b = _c; \ #if defined(_MSC_VER) @@ -199,6 +458,9 @@ union cn_slow_hash_state THREADV uint8_t *hp_state = NULL; THREADV int hp_allocated = 0; +THREADV v4_random_math_JIT_func hp_jitfunc = NULL; +THREADV uint8_t *hp_jitfunc_memory = NULL; +THREADV int hp_jitfunc_allocated = 0; #if defined(_MSC_VER) #define cpuid(info,x) __cpuidex(info,x,0) @@ -309,7 +571,7 @@ STATIC INLINE void aes_256_assist2(__m128i* t1, __m128i * t3) * CPU AES support. * For more information about these functions, see page 19 of Intel's AES instructions * white paper: - * http://www.intel.com/content/dam/www/public/us/en/documents/white-papers/aes-instructions-set-white-paper.pdf + * https://www.intel.com/content/dam/doc/white-paper/advanced-encryption-standard-new-instructions-set-paper.pdf * * @param key the input 128 bit key * @param expandedKey An output buffer to hold the generated key schedule @@ -492,12 +754,12 @@ void slow_hash_allocate_state(void) MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE); #else #if defined(__APPLE__) || defined(__FreeBSD__) || defined(__OpenBSD__) || \ - defined(__DragonFly__) + defined(__DragonFly__) || defined(__NetBSD__) hp_state = mmap(0, MEMORY, PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANON, 0, 0); + MAP_PRIVATE | MAP_ANON, -1, 0); #else hp_state = mmap(0, MEMORY, PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB, 0, 0); + MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB, -1, 0); #endif if(hp_state == MAP_FAILED) hp_state = NULL; @@ -508,6 +770,35 @@ void slow_hash_allocate_state(void) hp_allocated = 0; hp_state = (uint8_t *) malloc(MEMORY); } + + +#if defined(_MSC_VER) || defined(__MINGW32__) + hp_jitfunc_memory = (uint8_t *) VirtualAlloc(hp_jitfunc_memory, 4096 + 4095, + MEM_COMMIT | MEM_RESERVE, PAGE_EXECUTE_READWRITE); +#else +#if defined(__APPLE__) || defined(__FreeBSD__) || defined(__OpenBSD__) || \ + defined(__DragonFly__) || defined(__NetBSD__) +#ifdef __NetBSD__ +#define RESERVED_FLAGS PROT_MPROTECT(PROT_EXEC) +#else +#define RESERVED_FLAGS 0 +#endif + hp_jitfunc_memory = mmap(0, 4096 + 4096, PROT_READ | PROT_WRITE | RESERVED_FLAGS, + MAP_PRIVATE | MAP_ANON, -1, 0); +#else + hp_jitfunc_memory = mmap(0, 4096 + 4096, PROT_READ | PROT_WRITE | PROT_EXEC, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); +#endif + if(hp_jitfunc_memory == MAP_FAILED) + hp_jitfunc_memory = NULL; +#endif + hp_jitfunc_allocated = 1; + if (hp_jitfunc_memory == NULL) + { + hp_jitfunc_allocated = 0; + hp_jitfunc_memory = malloc(4096 + 4095); + } + hp_jitfunc = (v4_random_math_JIT_func)((size_t)(hp_jitfunc_memory + 4095) & ~4095); } /** @@ -530,8 +821,22 @@ void slow_hash_free_state(void) #endif } + if(!hp_jitfunc_allocated) + free(hp_jitfunc_memory); + else + { +#if defined(_MSC_VER) || defined(__MINGW32__) + VirtualFree(hp_jitfunc_memory, 0, MEM_RELEASE); +#else + munmap(hp_jitfunc_memory, 4096 + 4095); +#endif + } + hp_state = NULL; hp_allocated = 0; + hp_jitfunc = NULL; + hp_jitfunc_memory = NULL; + hp_jitfunc_allocated = 0; } /** @@ -558,22 +863,22 @@ void slow_hash_free_state(void) * AES support on x86 CPUs. * * A diagram of the inner loop of this function can be found at - * http://www.cs.cmu.edu/~dga/crypto/xmr/cryptonight.png + * https://www.cs.cmu.edu/~dga/crypto/xmr/cryptonight.png * * @param data the data to hash * @param length the length in bytes of the data * @param hash a pointer to a buffer in which the final 256 bit hash will be stored */ -void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int prehashed) +void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int prehashed, uint64_t height) { RDATA_ALIGN16 uint8_t expandedKey[240]; /* These buffers are aligned to use later with SSE functions */ uint8_t text[INIT_SIZE_BYTE]; RDATA_ALIGN16 uint64_t a[2]; - RDATA_ALIGN16 uint64_t b[2]; + RDATA_ALIGN16 uint64_t b[4]; RDATA_ALIGN16 uint64_t c[2]; union cn_slow_hash_state state; - __m128i _a, _b, _c; + __m128i _a, _b, _b1, _c; uint64_t hi, lo; size_t i, j; @@ -590,6 +895,10 @@ void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int if(hp_state == NULL) slow_hash_allocate_state(); + // locals to avoid constant TLS dereferencing + uint8_t *local_hp_state = hp_state; + v4_random_math_JIT_func local_hp_jitfunc = hp_jitfunc; + /* CryptoNight Step 1: Use Keccak1600 to initialize the 'state' (and 'text') buffers from the data. */ if (prehashed) { memcpy(&state.hs, data, length); @@ -599,6 +908,8 @@ void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int memcpy(text, state.init, INIT_SIZE_BYTE); VARIANT1_INIT64(); + VARIANT2_INIT64(); + VARIANT4_RANDOM_MATH_INIT(); /* CryptoNight Step 2: Iteratively encrypt the results from Keccak to fill * the 2MB large random access buffer. @@ -610,7 +921,7 @@ void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int for(i = 0; i < MEMORY / INIT_SIZE_BYTE; i++) { aes_pseudo_round(text, text, expandedKey, INIT_SIZE_BLK); - memcpy(&hp_state[i * INIT_SIZE_BYTE], text, INIT_SIZE_BYTE); + memcpy(&local_hp_state[i * INIT_SIZE_BYTE], text, INIT_SIZE_BYTE); } } else @@ -622,7 +933,7 @@ void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int for(j = 0; j < INIT_SIZE_BLK; j++) aesb_pseudo_round(&text[AES_BLOCK_SIZE * j], &text[AES_BLOCK_SIZE * j], aes_ctx->key->exp_data); - memcpy(&hp_state[i * INIT_SIZE_BYTE], text, INIT_SIZE_BYTE); + memcpy(&local_hp_state[i * INIT_SIZE_BYTE], text, INIT_SIZE_BYTE); } } @@ -637,6 +948,7 @@ void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int */ _b = _mm_load_si128(R128(b)); + _b1 = _mm_load_si128(R128(b) + 1); // Two independent versions, one with AES, one without, to ensure that // the useAes test is only performed once, not every iteration. if(useAes) @@ -669,7 +981,7 @@ void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int for(i = 0; i < MEMORY / INIT_SIZE_BYTE; i++) { // add the xor to the pseudo round - aes_pseudo_round_xor(text, text, expandedKey, &hp_state[i * INIT_SIZE_BYTE], INIT_SIZE_BLK); + aes_pseudo_round_xor(text, text, expandedKey, &local_hp_state[i * INIT_SIZE_BYTE], INIT_SIZE_BLK); } } else @@ -679,7 +991,7 @@ void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int { for(j = 0; j < INIT_SIZE_BLK; j++) { - xor_blocks(&text[j * AES_BLOCK_SIZE], &hp_state[i * INIT_SIZE_BYTE + j * AES_BLOCK_SIZE]); + xor_blocks(&text[j * AES_BLOCK_SIZE], &local_hp_state[i * INIT_SIZE_BYTE + j * AES_BLOCK_SIZE]); aesb_pseudo_round(&text[AES_BLOCK_SIZE * j], &text[AES_BLOCK_SIZE * j], aes_ctx->key->exp_data); } } @@ -723,6 +1035,8 @@ void slow_hash_free_state(void) #define U64(x) ((uint64_t *) (x)) +#define hp_jitfunc ((v4_random_math_JIT_func)NULL) + STATIC INLINE void xor64(uint64_t *a, const uint64_t b) { *a ^= b; @@ -757,23 +1071,28 @@ union cn_slow_hash_state #define pre_aes() \ j = state_index(a); \ - _c = vld1q_u8(&hp_state[j]); \ + _c = vld1q_u8(&local_hp_state[j]); \ _a = vld1q_u8((const uint8_t *)a); \ #define post_aes() \ + VARIANT2_SHUFFLE_ADD_NEON(local_hp_state, j); \ vst1q_u8((uint8_t *)c, _c); \ - _b = veorq_u8(_b, _c); \ - vst1q_u8(&hp_state[j], _b); \ - VARIANT1_1(&hp_state[j]); \ + vst1q_u8(&local_hp_state[j], veorq_u8(_b, _c)); \ + VARIANT1_1(&local_hp_state[j]); \ j = state_index(c); \ - p = U64(&hp_state[j]); \ + p = U64(&local_hp_state[j]); \ b[0] = p[0]; b[1] = p[1]; \ + VARIANT2_PORTABLE_INTEGER_MATH(b, c); \ + VARIANT4_RANDOM_MATH(a, b, r, &_b, &_b1); \ __mul(); \ + VARIANT2_2(); \ + VARIANT2_SHUFFLE_ADD_NEON(local_hp_state, j); \ a[0] += hi; a[1] += lo; \ - p = U64(&hp_state[j]); \ + p = U64(&local_hp_state[j]); \ p[0] = a[0]; p[1] = a[1]; \ a[0] ^= b[0]; a[1] ^= b[1]; \ VARIANT1_2(p + 1); \ + _b1 = _b; \ _b = _c; \ @@ -905,17 +1224,44 @@ STATIC INLINE void aes_pseudo_round_xor(const uint8_t *in, uint8_t *out, const u } } -void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int prehashed) +#ifdef FORCE_USE_HEAP +STATIC INLINE void* aligned_malloc(size_t size, size_t align) +{ + void *result; +#ifdef _MSC_VER + result = _aligned_malloc(size, align); +#else + if (posix_memalign(&result, align, size)) result = NULL; +#endif + return result; +} + +STATIC INLINE void aligned_free(void *ptr) +{ +#ifdef _MSC_VER + _aligned_free(ptr); +#else + free(ptr); +#endif +} +#endif /* FORCE_USE_HEAP */ + +void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int prehashed, uint64_t height) { RDATA_ALIGN16 uint8_t expandedKey[240]; - RDATA_ALIGN16 uint8_t hp_state[MEMORY]; + +#ifndef FORCE_USE_HEAP + RDATA_ALIGN16 uint8_t local_hp_state[MEMORY]; +#else + uint8_t *local_hp_state = (uint8_t *)aligned_malloc(MEMORY,16); +#endif uint8_t text[INIT_SIZE_BYTE]; RDATA_ALIGN16 uint64_t a[2]; - RDATA_ALIGN16 uint64_t b[2]; + RDATA_ALIGN16 uint64_t b[4]; RDATA_ALIGN16 uint64_t c[2]; union cn_slow_hash_state state; - uint8x16_t _a, _b, _c, zero = {0}; + uint8x16_t _a, _b, _b1, _c, zero = {0}; uint64_t hi, lo; size_t i, j; @@ -936,6 +1282,8 @@ void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int memcpy(text, state.init, INIT_SIZE_BYTE); VARIANT1_INIT64(); + VARIANT2_INIT64(); + VARIANT4_RANDOM_MATH_INIT(); /* CryptoNight Step 2: Iteratively encrypt the results from Keccak to fill * the 2MB large random access buffer. @@ -945,7 +1293,7 @@ void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int for(i = 0; i < MEMORY / INIT_SIZE_BYTE; i++) { aes_pseudo_round(text, text, expandedKey, INIT_SIZE_BLK); - memcpy(&hp_state[i * INIT_SIZE_BYTE], text, INIT_SIZE_BYTE); + memcpy(&local_hp_state[i * INIT_SIZE_BYTE], text, INIT_SIZE_BYTE); } U64(a)[0] = U64(&state.k[0])[0] ^ U64(&state.k[32])[0]; @@ -959,7 +1307,7 @@ void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int */ _b = vld1q_u8((const uint8_t *)b); - + _b1 = vld1q_u8(((const uint8_t *)b) + AES_BLOCK_SIZE); for(i = 0; i < ITER / 2; i++) { @@ -980,7 +1328,7 @@ void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int for(i = 0; i < MEMORY / INIT_SIZE_BYTE; i++) { // add the xor to the pseudo round - aes_pseudo_round_xor(text, text, expandedKey, &hp_state[i * INIT_SIZE_BYTE], INIT_SIZE_BLK); + aes_pseudo_round_xor(text, text, expandedKey, &local_hp_state[i * INIT_SIZE_BYTE], INIT_SIZE_BLK); } /* CryptoNight Step 5: Apply Keccak to the state again, and then @@ -993,6 +1341,10 @@ void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int memcpy(state.init, text, INIT_SIZE_BYTE); hash_permutation(&state.hs); extra_hashes[state.hs.b[0] & 3](&state, 200, hash); + +#ifdef FORCE_USE_HEAP + aligned_free(local_hp_state); +#endif } #else /* aarch64 && crypto */ @@ -1075,6 +1427,11 @@ __asm__ __volatile__( #endif /* !aarch64 */ #endif // NO_OPTIMIZED_MULTIPLY_ON_ARM +STATIC INLINE void copy_block(uint8_t* dst, const uint8_t* src) +{ + memcpy(dst, src, AES_BLOCK_SIZE); +} + STATIC INLINE void sum_half_blocks(uint8_t* a, const uint8_t* b) { uint64_t a0, a1, b0, b1; @@ -1105,11 +1462,14 @@ STATIC INLINE void xor_blocks(uint8_t* a, const uint8_t* b) U64(a)[1] ^= U64(b)[1]; } -void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int prehashed) +void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int prehashed, uint64_t height) { uint8_t text[INIT_SIZE_BYTE]; uint8_t a[AES_BLOCK_SIZE]; - uint8_t b[AES_BLOCK_SIZE]; + uint8_t a1[AES_BLOCK_SIZE]; + uint8_t b[AES_BLOCK_SIZE * 2]; + uint8_t c[AES_BLOCK_SIZE]; + uint8_t c1[AES_BLOCK_SIZE]; uint8_t d[AES_BLOCK_SIZE]; uint8_t aes_key[AES_KEY_SIZE]; RDATA_ALIGN16 uint8_t expandedKey[256]; @@ -1127,8 +1487,7 @@ void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int #ifndef FORCE_USE_HEAP uint8_t long_state[MEMORY]; #else - uint8_t *long_state = NULL; - long_state = (uint8_t *)malloc(MEMORY); + uint8_t *long_state = (uint8_t *)malloc(MEMORY); #endif if (prehashed) { @@ -1138,11 +1497,13 @@ void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int } memcpy(text, state.init, INIT_SIZE_BYTE); - VARIANT1_INIT64(); - aes_ctx = (oaes_ctx *) oaes_alloc(); oaes_key_import_data(aes_ctx, state.hs.b, AES_KEY_SIZE); + VARIANT1_INIT64(); + VARIANT2_INIT64(); + VARIANT4_RANDOM_MATH_INIT(); + // use aligned data memcpy(expandedKey, aes_ctx->key->exp_data, aes_ctx->key->exp_data_len); for(i = 0; i < MEMORY / INIT_SIZE_BYTE; i++) @@ -1163,23 +1524,37 @@ void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int #define state_index(x) ((*(uint32_t *) x) & MASK) // Iteration 1 - p = &long_state[state_index(a)]; - aesb_single_round(p, p, a); + j = state_index(a); + p = &long_state[j]; + aesb_single_round(p, c1, a); - xor_blocks(b, p); - swap_blocks(b, p); - swap_blocks(a, b); + VARIANT2_PORTABLE_SHUFFLE_ADD(c1, a, long_state, j); + copy_block(p, c1); + xor_blocks(p, b); VARIANT1_1(p); // Iteration 2 - p = &long_state[state_index(a)]; - - mul(a, p, d); - sum_half_blocks(b, d); - swap_blocks(b, p); - xor_blocks(b, p); - swap_blocks(a, b); - VARIANT1_2(U64(p) + 1); + j = state_index(c1); + p = &long_state[j]; + copy_block(c, p); + + copy_block(a1, a); + VARIANT2_PORTABLE_INTEGER_MATH(c, c1); + VARIANT4_RANDOM_MATH(a1, c, r, b, b + AES_BLOCK_SIZE); + mul(c1, c, d); + VARIANT2_2_PORTABLE(); + VARIANT2_PORTABLE_SHUFFLE_ADD(c1, a, long_state, j); + sum_half_blocks(a1, d); + swap_blocks(a1, c); + xor_blocks(a1, c); + VARIANT1_2(U64(c) + 1); + copy_block(p, c); + + if (variant >= 2) { + copy_block(b + AES_BLOCK_SIZE, b); + } + copy_block(b, c1); + copy_block(a, a1); } memcpy(text, state.init, INIT_SIZE_BYTE); @@ -1207,6 +1582,8 @@ void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int #else // Portable implementation as a fallback +#define hp_jitfunc ((v4_random_math_JIT_func)NULL) + void slow_hash_allocate_state(void) { // Do nothing, this is just to maintain compatibility with the upgraded slow-hash.c @@ -1223,10 +1600,7 @@ static void (*const extra_hashes[4])(const void *, size_t, char *) = { hash_extra_blake, hash_extra_groestl, hash_extra_jh, hash_extra_skein }; -extern int aesb_single_round(const uint8_t *in, uint8_t*out, const uint8_t *expandedKey); -extern int aesb_pseudo_round(const uint8_t *in, uint8_t *out, const uint8_t *expandedKey); - -static size_t e2i(const uint8_t* a, size_t count) { return (*((uint64_t*)a) / AES_BLOCK_SIZE) & (count - 1); } +static size_t e2i(const uint8_t* a, size_t count) { return (SWAP64LE(*((uint64_t*)a)) / AES_BLOCK_SIZE) & (count - 1); } static void mul(const uint8_t* a, const uint8_t* b, uint8_t* res) { uint64_t a0, b0; @@ -1293,13 +1667,20 @@ union cn_slow_hash_state { }; #pragma pack(pop) -void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int prehashed) { +void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int prehashed, uint64_t height) { +#ifndef FORCE_USE_HEAP uint8_t long_state[MEMORY]; +#else + uint8_t *long_state = (uint8_t *)malloc(MEMORY); +#endif + union cn_slow_hash_state state; uint8_t text[INIT_SIZE_BYTE]; uint8_t a[AES_BLOCK_SIZE]; - uint8_t b[AES_BLOCK_SIZE]; - uint8_t c[AES_BLOCK_SIZE]; + uint8_t a1[AES_BLOCK_SIZE]; + uint8_t b[AES_BLOCK_SIZE * 2]; + uint8_t c1[AES_BLOCK_SIZE]; + uint8_t c2[AES_BLOCK_SIZE]; uint8_t d[AES_BLOCK_SIZE]; size_t i, j; uint8_t aes_key[AES_KEY_SIZE]; @@ -1315,6 +1696,8 @@ void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int aes_ctx = (oaes_ctx *) oaes_alloc(); VARIANT1_PORTABLE_INIT(); + VARIANT2_PORTABLE_INIT(); + VARIANT4_RANDOM_MATH_INIT(); oaes_key_import_data(aes_ctx, aes_key, AES_KEY_SIZE); for (i = 0; i < MEMORY / INIT_SIZE_BYTE; i++) { @@ -1324,9 +1707,9 @@ void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int memcpy(&long_state[i * INIT_SIZE_BYTE], text, INIT_SIZE_BYTE); } - for (i = 0; i < 16; i++) { - a[i] = state.k[ i] ^ state.k[32 + i]; - b[i] = state.k[16 + i] ^ state.k[48 + i]; + for (i = 0; i < AES_BLOCK_SIZE; i++) { + a[i] = state.k[ i] ^ state.k[AES_BLOCK_SIZE * 2 + i]; + b[i] = state.k[AES_BLOCK_SIZE + i] ^ state.k[AES_BLOCK_SIZE * 3 + i]; } for (i = 0; i < ITER / 2; i++) { @@ -1335,26 +1718,33 @@ void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int * next address <-+ */ /* Iteration 1 */ - j = e2i(a, MEMORY / AES_BLOCK_SIZE); - copy_block(c, &long_state[j * AES_BLOCK_SIZE]); - aesb_single_round(c, c, a); - xor_blocks(b, c); - swap_blocks(b, c); - copy_block(&long_state[j * AES_BLOCK_SIZE], c); - assert(j == e2i(a, MEMORY / AES_BLOCK_SIZE)); - swap_blocks(a, b); - VARIANT1_1(&long_state[j * AES_BLOCK_SIZE]); + j = e2i(a, MEMORY / AES_BLOCK_SIZE) * AES_BLOCK_SIZE; + copy_block(c1, &long_state[j]); + aesb_single_round(c1, c1, a); + VARIANT2_PORTABLE_SHUFFLE_ADD(c1, a, long_state, j); + copy_block(&long_state[j], c1); + xor_blocks(&long_state[j], b); + assert(j == e2i(a, MEMORY / AES_BLOCK_SIZE) * AES_BLOCK_SIZE); + VARIANT1_1(&long_state[j]); /* Iteration 2 */ - j = e2i(a, MEMORY / AES_BLOCK_SIZE); - copy_block(c, &long_state[j * AES_BLOCK_SIZE]); - mul(a, c, d); - sum_half_blocks(b, d); - swap_blocks(b, c); - xor_blocks(b, c); - VARIANT1_2(c + 8); - copy_block(&long_state[j * AES_BLOCK_SIZE], c); - assert(j == e2i(a, MEMORY / AES_BLOCK_SIZE)); - swap_blocks(a, b); + j = e2i(c1, MEMORY / AES_BLOCK_SIZE) * AES_BLOCK_SIZE; + copy_block(c2, &long_state[j]); + copy_block(a1, a); + VARIANT2_PORTABLE_INTEGER_MATH(c2, c1); + VARIANT4_RANDOM_MATH(a1, c2, r, b, b + AES_BLOCK_SIZE); + mul(c1, c2, d); + VARIANT2_2_PORTABLE(); + VARIANT2_PORTABLE_SHUFFLE_ADD(c1, a, long_state, j); + sum_half_blocks(a1, d); + swap_blocks(a1, c2); + xor_blocks(a1, c2); + VARIANT1_2(c2 + 8); + copy_block(&long_state[j], c2); + if (variant >= 2) { + copy_block(b + AES_BLOCK_SIZE, b); + } + copy_block(b, c1); + copy_block(a, a1); } memcpy(text, state.init, INIT_SIZE_BYTE); @@ -1370,6 +1760,10 @@ void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int /*memcpy(hash, &state, 32);*/ extra_hashes[state.hs.b[0] & 3](&state, 200, hash); oaes_free((OAES_CTX **) &aes_ctx); + +#ifdef FORCE_USE_HEAP + free(long_state); +#endif } #endif diff --git a/src/crypto/tree-hash.c b/src/crypto/tree-hash.c index e6d6a267c0c..0a5860f3b25 100644 --- a/src/crypto/tree-hash.c +++ b/src/crypto/tree-hash.c @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -30,18 +30,11 @@ #include #include +#include #include #include "hash-ops.h" -#ifdef _MSC_VER -#include -#elif !defined(__FreeBSD__) && !defined(__OpenBSD__) && !defined(__DragonFly__) - #include -#else - #include -#endif - /*** * Round to power of two, for count>=3 and for count being not too large (as reasonable for tree hash calculations) */ @@ -67,7 +60,7 @@ size_t tree_hash_cnt(size_t count) { } void tree_hash(const char (*hashes)[HASH_SIZE], size_t count, char *root_hash) { -// The blockchain block at height 202612 http://monerochain.info/block/bbd604d2ba11ba27935e006ed39c9bfdd99b76bf4a50654bc1e1e61217962698 +// The blockchain block at height 202612 https://moneroblocks.info/block/202612 // contained 514 transactions, that triggered bad calculation of variable "cnt" in the original version of this function // as from CryptoNote code. // @@ -90,24 +83,24 @@ void tree_hash(const char (*hashes)[HASH_SIZE], size_t count, char *root_hash) { size_t cnt = tree_hash_cnt( count ); - char (*ints)[HASH_SIZE]; - size_t ints_size = cnt * HASH_SIZE; - ints = alloca(ints_size); memset( ints , 0 , ints_size); // allocate, and zero out as extra protection for using uninitialized mem + char *ints = calloc(cnt, HASH_SIZE); // zero out as extra protection for using uninitialized mem + assert(ints); memcpy(ints, hashes, (2 * cnt - count) * HASH_SIZE); for (i = 2 * cnt - count, j = 2 * cnt - count; j < cnt; i += 2, ++j) { - cn_fast_hash(hashes[i], 64, ints[j]); + cn_fast_hash(hashes[i], 64, ints + j * HASH_SIZE); } assert(i == count); while (cnt > 2) { cnt >>= 1; for (i = 0, j = 0; j < cnt; i += 2, ++j) { - cn_fast_hash(ints[i], 64, ints[j]); + cn_fast_hash(ints + i * HASH_SIZE, 64, ints + j * HASH_SIZE); } } - cn_fast_hash(ints[0], 64, root_hash); + cn_fast_hash(ints, 64, root_hash); + free(ints); } } diff --git a/src/crypto/variant2_int_sqrt.h b/src/crypto/variant2_int_sqrt.h new file mode 100644 index 00000000000..b405bb798dd --- /dev/null +++ b/src/crypto/variant2_int_sqrt.h @@ -0,0 +1,163 @@ +#ifndef VARIANT2_INT_SQRT_H +#define VARIANT2_INT_SQRT_H + +#include +#include + +#define VARIANT2_INTEGER_MATH_SQRT_STEP_SSE2() \ + do { \ + const __m128i exp_double_bias = _mm_set_epi64x(0, 1023ULL << 52); \ + __m128d x = _mm_castsi128_pd(_mm_add_epi64(_mm_cvtsi64_si128(sqrt_input >> 12), exp_double_bias)); \ + x = _mm_sqrt_sd(_mm_setzero_pd(), x); \ + sqrt_result = (uint64_t)(_mm_cvtsi128_si64(_mm_sub_epi64(_mm_castpd_si128(x), exp_double_bias))) >> 19; \ + } while(0) + +#define VARIANT2_INTEGER_MATH_SQRT_STEP_FP64() \ + do { \ + sqrt_result = sqrt(sqrt_input + 18446744073709551616.0) * 2.0 - 8589934592.0; \ + } while(0) + +#define VARIANT2_INTEGER_MATH_SQRT_STEP_REF() \ + sqrt_result = integer_square_root_v2(sqrt_input) + +// Reference implementation of the integer square root for Cryptonight variant 2 +// Computes integer part of "sqrt(2^64 + n) * 2 - 2^33" +// +// In other words, given 64-bit unsigned integer n: +// 1) Write it as x = 1.NNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNN000... in binary (1 <= x < 2, all 64 bits of n are used) +// 2) Calculate sqrt(x) = 1.0RRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRR... (1 <= sqrt(x) < sqrt(2), so it will always start with "1.0" in binary) +// 3) Take 32 bits that come after "1.0" and return them as a 32-bit unsigned integer, discard all remaining bits +// +// Some sample inputs and outputs: +// +// Input | Output | Exact value of "sqrt(2^64 + n) * 2 - 2^33" +// -----------------|------------|------------------------------------------- +// 0 | 0 | 0 +// 2^32 | 0 | 0.99999999994179233909330885695244... +// 2^32 + 1 | 1 | 1.0000000001746229827200734316305... +// 2^50 | 262140 | 262140.00012206565608606978175873... +// 2^55 + 20963331 | 8384515 | 8384515.9999999997673963974959744... +// 2^55 + 20963332 | 8384516 | 8384516 +// 2^62 + 26599786 | 1013904242 | 1013904242.9999999999479374853545... +// 2^62 + 26599787 | 1013904243 | 1013904243.0000000001561875439364... +// 2^64 - 1 | 3558067407 | 3558067407.9041987696409179931096... + +// The reference implementation as it is now uses only unsigned int64 arithmetic, so it can't have undefined behavior +// It was tested once for all edge cases and confirmed correct +static inline uint32_t integer_square_root_v2(uint64_t n) +{ + uint64_t r = 1ULL << 63; + + for (uint64_t bit = 1ULL << 60; bit; bit >>= 2) + { + const bool b = (n < r + bit); + const uint64_t n_next = n - (r + bit); + const uint64_t r_next = r + bit * 2; + n = b ? n : n_next; + r = b ? r : r_next; + r >>= 1; + } + + return r * 2 + ((n > r) ? 1 : 0); +} + +/* +VARIANT2_INTEGER_MATH_SQRT_FIXUP checks that "r" is an integer part of "sqrt(2^64 + sqrt_input) * 2 - 2^33" and adds or subtracts 1 if needed +It's hard to understand how it works, so here is a full calculation of formulas used in VARIANT2_INTEGER_MATH_SQRT_FIXUP + +The following inequalities must hold for r if it's an integer part of "sqrt(2^64 + sqrt_input) * 2 - 2^33": +1) r <= sqrt(2^64 + sqrt_input) * 2 - 2^33 +2) r + 1 > sqrt(2^64 + sqrt_input) * 2 - 2^33 + +We need to check them using only unsigned integer arithmetic to avoid rounding errors and undefined behavior + +First inequality: r <= sqrt(2^64 + sqrt_input) * 2 - 2^33 +----------------------------------------------------------------------------------- +r <= sqrt(2^64 + sqrt_input) * 2 - 2^33 +r + 2^33 <= sqrt(2^64 + sqrt_input) * 2 +r/2 + 2^32 <= sqrt(2^64 + sqrt_input) +(r/2 + 2^32)^2 <= 2^64 + sqrt_input + +Rewrite r as r = s * 2 + b (s = trunc(r/2), b is 0 or 1) + +((s*2+b)/2 + 2^32)^2 <= 2^64 + sqrt_input +(s*2+b)^2/4 + 2*2^32*(s*2+b)/2 + 2^64 <= 2^64 + sqrt_input +(s*2+b)^2/4 + 2*2^32*(s*2+b)/2 <= sqrt_input +(s*2+b)^2/4 + 2^32*r <= sqrt_input +(s^2*4+2*s*2*b+b^2)/4 + 2^32*r <= sqrt_input +s^2+s*b+b^2/4 + 2^32*r <= sqrt_input +s*(s+b) + b^2/4 + 2^32*r <= sqrt_input + +Let r2 = s*(s+b) + r*2^32 +r2 + b^2/4 <= sqrt_input + +If this inequality doesn't hold, then we must decrement r: IF "r2 + b^2/4 > sqrt_input" THEN r = r - 1 + +b can be 0 or 1 +If b is 0 then we need to compare "r2 > sqrt_input" +If b is 1 then b^2/4 = 0.25, so we need to compare "r2 + 0.25 > sqrt_input" +Since both r2 and sqrt_input are integers, we can safely replace it with "r2 + 1 > sqrt_input" +----------------------------------------------------------------------------------- +Both cases can be merged to a single expression "r2 + b > sqrt_input" +----------------------------------------------------------------------------------- +There will be no overflow when calculating "r2 + b", so it's safe to compare with sqrt_input: +r2 + b = s*(s+b) + r*2^32 + b +The largest value s, b and r can have is s = 1779033703, b = 1, r = 3558067407 when sqrt_input = 2^64 - 1 +r2 + b <= 1779033703*1779033704 + 3558067407*2^32 + 1 = 18446744068217447385 < 2^64 + +Second inequality: r + 1 > sqrt(2^64 + sqrt_input) * 2 - 2^33 +----------------------------------------------------------------------------------- +r + 1 > sqrt(2^64 + sqrt_input) * 2 - 2^33 +r + 1 + 2^33 > sqrt(2^64 + sqrt_input) * 2 +((r+1)/2 + 2^32)^2 > 2^64 + sqrt_input + +Rewrite r as r = s * 2 + b (s = trunc(r/2), b is 0 or 1) + +((s*2+b+1)/2 + 2^32)^2 > 2^64 + sqrt_input +(s*2+b+1)^2/4 + 2*(s*2+b+1)/2*2^32 + 2^64 > 2^64 + sqrt_input +(s*2+b+1)^2/4 + (s*2+b+1)*2^32 > sqrt_input +(s*2+b+1)^2/4 + (r+1)*2^32 > sqrt_input +(s*2+(b+1))^2/4 + r*2^32 + 2^32 > sqrt_input +(s^2*4+2*s*2*(b+1)+(b+1)^2)/4 + r*2^32 + 2^32 > sqrt_input +s^2+s*(b+1)+(b+1)^2/4 + r*2^32 + 2^32 > sqrt_input +s*(s+b) + s + (b+1)^2/4 + r*2^32 + 2^32 > sqrt_input + +Let r2 = s*(s+b) + r*2^32 + +r2 + s + (b+1)^2/4 + 2^32 > sqrt_input +r2 + 2^32 + (b+1)^2/4 > sqrt_input - s + +If this inequality doesn't hold, then we must decrement r: IF "r2 + 2^32 + (b+1)^2/4 <= sqrt_input - s" THEN r = r - 1 +b can be 0 or 1 +If b is 0 then we need to compare "r2 + 2^32 + 1/4 <= sqrt_input - s" which is equal to "r2 + 2^32 < sqrt_input - s" because all numbers here are integers +If b is 1 then (b+1)^2/4 = 1, so we need to compare "r2 + 2^32 + 1 <= sqrt_input - s" which is also equal to "r2 + 2^32 < sqrt_input - s" +----------------------------------------------------------------------------------- +Both cases can be merged to a single expression "r2 + 2^32 < sqrt_input - s" +----------------------------------------------------------------------------------- +There will be no overflow when calculating "r2 + 2^32": +r2 + 2^32 = s*(s+b) + r*2^32 + 2^32 = s*(s+b) + (r+1)*2^32 +The largest value s, b and r can have is s = 1779033703, b = 1, r = 3558067407 when sqrt_input = 2^64 - 1 +r2 + b <= 1779033703*1779033704 + 3558067408*2^32 = 18446744072512414680 < 2^64 + +There will be no integer overflow when calculating "sqrt_input - s", i.e. "sqrt_input >= s" at all times: +s = trunc(r/2) = trunc(sqrt(2^64 + sqrt_input) - 2^32) < sqrt(2^64 + sqrt_input) - 2^32 + 1 +sqrt_input > sqrt(2^64 + sqrt_input) - 2^32 + 1 +sqrt_input + 2^32 - 1 > sqrt(2^64 + sqrt_input) +(sqrt_input + 2^32 - 1)^2 > sqrt_input + 2^64 +sqrt_input^2 + 2*sqrt_input*(2^32 - 1) + (2^32-1)^2 > sqrt_input + 2^64 +sqrt_input^2 + sqrt_input*(2^33 - 2) + (2^32-1)^2 > sqrt_input + 2^64 +sqrt_input^2 + sqrt_input*(2^33 - 3) + (2^32-1)^2 > 2^64 +sqrt_input^2 + sqrt_input*(2^33 - 3) + 2^64-2^33+1 > 2^64 +sqrt_input^2 + sqrt_input*(2^33 - 3) - 2^33 + 1 > 0 +This inequality is true if sqrt_input > 1 and it's easy to check that s = 0 if sqrt_input is 0 or 1, so there will be no integer overflow +*/ + +#define VARIANT2_INTEGER_MATH_SQRT_FIXUP(r) \ + do { \ + const uint64_t s = r >> 1; \ + const uint64_t b = r & 1; \ + const uint64_t r2 = (uint64_t)(s) * (s + b) + (r << 32); \ + r += ((r2 + b > sqrt_input) ? -1 : 0) + ((r2 + (1ULL << 32) < sqrt_input - s) ? 1 : 0); \ + } while(0) + +#endif diff --git a/src/crypto/variant4_random_math.h b/src/crypto/variant4_random_math.h new file mode 100644 index 00000000000..f3e41a00163 --- /dev/null +++ b/src/crypto/variant4_random_math.h @@ -0,0 +1,441 @@ +#ifndef VARIANT4_RANDOM_MATH_H +#define VARIANT4_RANDOM_MATH_H + +// Register size can be configured to either 32 bit (uint32_t) or 64 bit (uint64_t) +typedef uint32_t v4_reg; + +enum V4_Settings +{ + // Generate code with minimal theoretical latency = 45 cycles, which is equivalent to 15 multiplications + TOTAL_LATENCY = 15 * 3, + + // Always generate at least 60 instructions + NUM_INSTRUCTIONS_MIN = 60, + + // Never generate more than 70 instructions (final RET instruction doesn't count here) + NUM_INSTRUCTIONS_MAX = 70, + + // Available ALUs for MUL + // Modern CPUs typically have only 1 ALU which can do multiplications + ALU_COUNT_MUL = 1, + + // Total available ALUs + // Modern CPUs have 4 ALUs, but we use only 3 because random math executes together with other main loop code + ALU_COUNT = 3, +}; + +enum V4_InstructionList +{ + MUL, // a*b + ADD, // a+b + C, C is an unsigned 32-bit constant + SUB, // a-b + ROR, // rotate right "a" by "b & 31" bits + ROL, // rotate left "a" by "b & 31" bits + XOR, // a^b + RET, // finish execution + V4_INSTRUCTION_COUNT = RET, +}; + +// V4_InstructionDefinition is used to generate code from random data +// Every random sequence of bytes is a valid code +// +// There are 9 registers in total: +// - 4 variable registers +// - 5 constant registers initialized from loop variables +// This is why dst_index is 2 bits +enum V4_InstructionDefinition +{ + V4_OPCODE_BITS = 3, + V4_DST_INDEX_BITS = 2, + V4_SRC_INDEX_BITS = 3, +}; + +struct V4_Instruction +{ + uint8_t opcode; + uint8_t dst_index; + uint8_t src_index; + uint32_t C; +}; + +#ifndef FORCEINLINE +#if defined(__GNUC__) +#define FORCEINLINE __attribute__((always_inline)) inline +#elif defined(_MSC_VER) +#define FORCEINLINE __forceinline +#else +#define FORCEINLINE inline +#endif +#endif + +#ifndef UNREACHABLE_CODE +#if defined(__GNUC__) +#define UNREACHABLE_CODE __builtin_unreachable() +#elif defined(_MSC_VER) +#define UNREACHABLE_CODE __assume(false) +#else +#define UNREACHABLE_CODE +#endif +#endif + +// Random math interpreter's loop is fully unrolled and inlined to achieve 100% branch prediction on CPU: +// every switch-case will point to the same destination on every iteration of Cryptonight main loop +// +// This is about as fast as it can get without using low-level machine code generation +static FORCEINLINE void v4_random_math(const struct V4_Instruction* code, v4_reg* r) +{ + enum + { + REG_BITS = sizeof(v4_reg) * 8, + }; + +#define V4_EXEC(i) \ + { \ + const struct V4_Instruction* op = code + i; \ + const v4_reg src = r[op->src_index]; \ + v4_reg* dst = r + op->dst_index; \ + switch (op->opcode) \ + { \ + case MUL: \ + *dst *= src; \ + break; \ + case ADD: \ + *dst += src + op->C; \ + break; \ + case SUB: \ + *dst -= src; \ + break; \ + case ROR: \ + { \ + const uint32_t shift = src % REG_BITS; \ + *dst = (*dst >> shift) | (*dst << ((REG_BITS - shift) % REG_BITS)); \ + } \ + break; \ + case ROL: \ + { \ + const uint32_t shift = src % REG_BITS; \ + *dst = (*dst << shift) | (*dst >> ((REG_BITS - shift) % REG_BITS)); \ + } \ + break; \ + case XOR: \ + *dst ^= src; \ + break; \ + case RET: \ + return; \ + default: \ + UNREACHABLE_CODE; \ + break; \ + } \ + } + +#define V4_EXEC_10(j) \ + V4_EXEC(j + 0) \ + V4_EXEC(j + 1) \ + V4_EXEC(j + 2) \ + V4_EXEC(j + 3) \ + V4_EXEC(j + 4) \ + V4_EXEC(j + 5) \ + V4_EXEC(j + 6) \ + V4_EXEC(j + 7) \ + V4_EXEC(j + 8) \ + V4_EXEC(j + 9) + + // Generated program can have 60 + a few more (usually 2-3) instructions to achieve required latency + // I've checked all block heights < 10,000,000 and here is the distribution of program sizes: + // + // 60 27960 + // 61 105054 + // 62 2452759 + // 63 5115997 + // 64 1022269 + // 65 1109635 + // 66 153145 + // 67 8550 + // 68 4529 + // 69 102 + + // Unroll 70 instructions here + V4_EXEC_10(0); // instructions 0-9 + V4_EXEC_10(10); // instructions 10-19 + V4_EXEC_10(20); // instructions 20-29 + V4_EXEC_10(30); // instructions 30-39 + V4_EXEC_10(40); // instructions 40-49 + V4_EXEC_10(50); // instructions 50-59 + V4_EXEC_10(60); // instructions 60-69 + +#undef V4_EXEC_10 +#undef V4_EXEC +} + +// If we don't have enough data available, generate more +static FORCEINLINE void check_data(size_t* data_index, const size_t bytes_needed, int8_t* data, const size_t data_size) +{ + if (*data_index + bytes_needed > data_size) + { + hash_extra_blake(data, data_size, (char*) data); + *data_index = 0; + } +} + +// Generates as many random math operations as possible with given latency and ALU restrictions +// "code" array must have space for NUM_INSTRUCTIONS_MAX+1 instructions +static inline int v4_random_math_init(struct V4_Instruction* code, const uint64_t height) +{ + // MUL is 3 cycles, 3-way addition and rotations are 2 cycles, SUB/XOR are 1 cycle + // These latencies match real-life instruction latencies for Intel CPUs starting from Sandy Bridge and up to Skylake/Coffee lake + // + // AMD Ryzen has the same latencies except 1-cycle ROR/ROL, so it'll be a bit faster than Intel Sandy Bridge and newer processors + // Surprisingly, Intel Nehalem also has 1-cycle ROR/ROL, so it'll also be faster than Intel Sandy Bridge and newer processors + // AMD Bulldozer has 4 cycles latency for MUL (slower than Intel) and 1 cycle for ROR/ROL (faster than Intel), so average performance will be the same + // Source: https://www.agner.org/optimize/instruction_tables.pdf + const int op_latency[V4_INSTRUCTION_COUNT] = { 3, 2, 1, 2, 2, 1 }; + + // Instruction latencies for theoretical ASIC implementation + const int asic_op_latency[V4_INSTRUCTION_COUNT] = { 3, 1, 1, 1, 1, 1 }; + + // Available ALUs for each instruction + const int op_ALUs[V4_INSTRUCTION_COUNT] = { ALU_COUNT_MUL, ALU_COUNT, ALU_COUNT, ALU_COUNT, ALU_COUNT, ALU_COUNT }; + + int8_t data[32]; + memset(data, 0, sizeof(data)); + uint64_t tmp = SWAP64LE(height); + memcpy(data, &tmp, sizeof(uint64_t)); + data[20] = -38; // change seed + + // Set data_index past the last byte in data + // to trigger full data update with blake hash + // before we start using it + size_t data_index = sizeof(data); + + int code_size; + + // There is a small chance (1.8%) that register R8 won't be used in the generated program + // So we keep track of it and try again if it's not used + bool r8_used; + do { + int latency[9]; + int asic_latency[9]; + + // Tracks previous instruction and value of the source operand for registers R0-R3 throughout code execution + // byte 0: current value of the destination register + // byte 1: instruction opcode + // byte 2: current value of the source register + // + // Registers R4-R8 are constant and are treated as having the same value because when we do + // the same operation twice with two constant source registers, it can be optimized into a single operation + uint32_t inst_data[9] = { 0, 1, 2, 3, 0xFFFFFF, 0xFFFFFF, 0xFFFFFF, 0xFFFFFF, 0xFFFFFF }; + + bool alu_busy[TOTAL_LATENCY + 1][ALU_COUNT]; + bool is_rotation[V4_INSTRUCTION_COUNT]; + bool rotated[4]; + int rotate_count = 0; + + memset(latency, 0, sizeof(latency)); + memset(asic_latency, 0, sizeof(asic_latency)); + memset(alu_busy, 0, sizeof(alu_busy)); + memset(is_rotation, 0, sizeof(is_rotation)); + memset(rotated, 0, sizeof(rotated)); + is_rotation[ROR] = true; + is_rotation[ROL] = true; + + int num_retries = 0; + code_size = 0; + + int total_iterations = 0; + r8_used = false; + + // Generate random code to achieve minimal required latency for our abstract CPU + // Try to get this latency for all 4 registers + while (((latency[0] < TOTAL_LATENCY) || (latency[1] < TOTAL_LATENCY) || (latency[2] < TOTAL_LATENCY) || (latency[3] < TOTAL_LATENCY)) && (num_retries < 64)) + { + // Fail-safe to guarantee loop termination + ++total_iterations; + if (total_iterations > 256) + break; + + check_data(&data_index, 1, data, sizeof(data)); + + const uint8_t c = ((uint8_t*)data)[data_index++]; + + // MUL = opcodes 0-2 + // ADD = opcode 3 + // SUB = opcode 4 + // ROR/ROL = opcode 5, shift direction is selected randomly + // XOR = opcodes 6-7 + uint8_t opcode = c & ((1 << V4_OPCODE_BITS) - 1); + if (opcode == 5) + { + check_data(&data_index, 1, data, sizeof(data)); + opcode = (data[data_index++] >= 0) ? ROR : ROL; + } + else if (opcode >= 6) + { + opcode = XOR; + } + else + { + opcode = (opcode <= 2) ? MUL : (opcode - 2); + } + + uint8_t dst_index = (c >> V4_OPCODE_BITS) & ((1 << V4_DST_INDEX_BITS) - 1); + uint8_t src_index = (c >> (V4_OPCODE_BITS + V4_DST_INDEX_BITS)) & ((1 << V4_SRC_INDEX_BITS) - 1); + + const int a = dst_index; + int b = src_index; + + // Don't do ADD/SUB/XOR with the same register + if (((opcode == ADD) || (opcode == SUB) || (opcode == XOR)) && (a == b)) + { + // Use register R8 as source instead + b = 8; + src_index = 8; + } + + // Don't do rotation with the same destination twice because it's equal to a single rotation + if (is_rotation[opcode] && rotated[a]) + { + continue; + } + + // Don't do the same instruction (except MUL) with the same source value twice because all other cases can be optimized: + // 2xADD(a, b, C) = ADD(a, b*2, C1+C2), same for SUB and rotations + // 2xXOR(a, b) = NOP + if ((opcode != MUL) && ((inst_data[a] & 0xFFFF00) == (opcode << 8) + ((inst_data[b] & 255) << 16))) + { + continue; + } + + // Find which ALU is available (and when) for this instruction + int next_latency = (latency[a] > latency[b]) ? latency[a] : latency[b]; + int alu_index = -1; + while (next_latency < TOTAL_LATENCY) + { + for (int i = op_ALUs[opcode] - 1; i >= 0; --i) + { + if (!alu_busy[next_latency][i]) + { + // ADD is implemented as two 1-cycle instructions on a real CPU, so do an additional availability check + if ((opcode == ADD) && alu_busy[next_latency + 1][i]) + { + continue; + } + + // Rotation can only start when previous rotation is finished, so do an additional availability check + if (is_rotation[opcode] && (next_latency < rotate_count * op_latency[opcode])) + { + continue; + } + + alu_index = i; + break; + } + } + if (alu_index >= 0) + { + break; + } + ++next_latency; + } + + // Don't generate instructions that leave some register unchanged for more than 7 cycles + if (next_latency > latency[a] + 7) + { + continue; + } + + next_latency += op_latency[opcode]; + + if (next_latency <= TOTAL_LATENCY) + { + if (is_rotation[opcode]) + { + ++rotate_count; + } + + // Mark ALU as busy only for the first cycle when it starts executing the instruction because ALUs are fully pipelined + alu_busy[next_latency - op_latency[opcode]][alu_index] = true; + latency[a] = next_latency; + + // ASIC is supposed to have enough ALUs to run as many independent instructions per cycle as possible, so latency calculation for ASIC is simple + asic_latency[a] = ((asic_latency[a] > asic_latency[b]) ? asic_latency[a] : asic_latency[b]) + asic_op_latency[opcode]; + + rotated[a] = is_rotation[opcode]; + + inst_data[a] = code_size + (opcode << 8) + ((inst_data[b] & 255) << 16); + + code[code_size].opcode = opcode; + code[code_size].dst_index = dst_index; + code[code_size].src_index = src_index; + code[code_size].C = 0; + + if (src_index == 8) + { + r8_used = true; + } + + if (opcode == ADD) + { + // ADD instruction is implemented as two 1-cycle instructions on a real CPU, so mark ALU as busy for the next cycle too + alu_busy[next_latency - op_latency[opcode] + 1][alu_index] = true; + + // ADD instruction requires 4 more random bytes for 32-bit constant "C" in "a = a + b + C" + check_data(&data_index, sizeof(uint32_t), data, sizeof(data)); + uint32_t t; + memcpy(&t, data + data_index, sizeof(uint32_t)); + code[code_size].C = SWAP32LE(t); + data_index += sizeof(uint32_t); + } + + ++code_size; + if (code_size >= NUM_INSTRUCTIONS_MIN) + { + break; + } + } + else + { + ++num_retries; + } + } + + // ASIC has more execution resources and can extract as much parallelism from the code as possible + // We need to add a few more MUL and ROR instructions to achieve minimal required latency for ASIC + // Get this latency for at least 1 of the 4 registers + const int prev_code_size = code_size; + while ((code_size < NUM_INSTRUCTIONS_MAX) && (asic_latency[0] < TOTAL_LATENCY) && (asic_latency[1] < TOTAL_LATENCY) && (asic_latency[2] < TOTAL_LATENCY) && (asic_latency[3] < TOTAL_LATENCY)) + { + int min_idx = 0; + int max_idx = 0; + for (int i = 1; i < 4; ++i) + { + if (asic_latency[i] < asic_latency[min_idx]) min_idx = i; + if (asic_latency[i] > asic_latency[max_idx]) max_idx = i; + } + + const uint8_t pattern[3] = { ROR, MUL, MUL }; + const uint8_t opcode = pattern[(code_size - prev_code_size) % 3]; + latency[min_idx] = latency[max_idx] + op_latency[opcode]; + asic_latency[min_idx] = asic_latency[max_idx] + asic_op_latency[opcode]; + + code[code_size].opcode = opcode; + code[code_size].dst_index = min_idx; + code[code_size].src_index = max_idx; + code[code_size].C = 0; + ++code_size; + } + + // There is ~98.15% chance that loop condition is false, so this loop will execute only 1 iteration most of the time + // It never does more than 4 iterations for all block heights < 10,000,000 + } while (!r8_used || (code_size < NUM_INSTRUCTIONS_MIN) || (code_size > NUM_INSTRUCTIONS_MAX)); + + // It's guaranteed that NUM_INSTRUCTIONS_MIN <= code_size <= NUM_INSTRUCTIONS_MAX here + // Add final instruction to stop the interpreter + code[code_size].opcode = RET; + code[code_size].dst_index = 0; + code[code_size].src_index = 0; + code[code_size].C = 0; + + return code_size; +} + +#endif diff --git a/src/cryptonote_basic/CMakeLists.txt b/src/cryptonote_basic/CMakeLists.txt index d50a9df6790..5bb56e08373 100644 --- a/src/cryptonote_basic/CMakeLists.txt +++ b/src/cryptonote_basic/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2014-2018, The Monero Project +# Copyright (c) 2014-2019, The Monero Project # # All rights reserved. # @@ -27,9 +27,13 @@ # THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. if(APPLE) - find_library(IOKIT_LIBRARY IOKit) - mark_as_advanced(IOKIT_LIBRARY) - list(APPEND EXTRA_LIBRARIES ${IOKIT_LIBRARY}) + if(DEPENDS) + list(APPEND EXTRA_LIBRARIES "-framework Foundation -framework ApplicationServices -framework AppKit -framework IOKit") + else() + find_library(IOKIT_LIBRARY IOKit) + mark_as_advanced(IOKIT_LIBRARY) + list(APPEND EXTRA_LIBRARIES ${IOKIT_LIBRARY}) + endif() endif() set(cryptonote_basic_sources diff --git a/src/cryptonote_basic/account.cpp b/src/cryptonote_basic/account.cpp index bab991d19bf..a9aec163b82 100644 --- a/src/cryptonote_basic/account.cpp +++ b/src/cryptonote_basic/account.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -44,6 +44,9 @@ extern "C" #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "account" +#define KEYS_ENCRYPTION_SALT 'k' + + using namespace std; DISABLE_VS_WARNINGS(4244 4345) @@ -60,7 +63,70 @@ DISABLE_VS_WARNINGS(4244 4345) m_device = &hwdev; MCDEBUG("device", "account_keys::set_device device type: "<> data; + memcpy(data.data(), &base_key, sizeof(base_key)); + data[sizeof(base_key)] = KEYS_ENCRYPTION_SALT; + crypto::generate_chacha_key(data.data(), sizeof(data), key, 1); + } + //----------------------------------------------------------------- + static epee::wipeable_string get_key_stream(const crypto::chacha_key &base_key, const crypto::chacha_iv &iv, size_t bytes) + { + // derive a new key + crypto::chacha_key key; + derive_key(base_key, key); + // chacha + epee::wipeable_string buffer0(std::string(bytes, '\0')); + epee::wipeable_string buffer1 = buffer0; + crypto::chacha20(buffer0.data(), buffer0.size(), key, iv, buffer1.data()); + return buffer1; + } + //----------------------------------------------------------------- + void account_keys::xor_with_key_stream(const crypto::chacha_key &key) + { + // encrypt a large enough byte stream with chacha20 + epee::wipeable_string key_stream = get_key_stream(key, m_encryption_iv, sizeof(crypto::secret_key) * (2 + m_multisig_keys.size())); + const char *ptr = key_stream.data(); + for (size_t i = 0; i < sizeof(crypto::secret_key); ++i) + m_spend_secret_key.data[i] ^= *ptr++; + for (size_t i = 0; i < sizeof(crypto::secret_key); ++i) + m_view_secret_key.data[i] ^= *ptr++; + for (crypto::secret_key &k: m_multisig_keys) + { + for (size_t i = 0; i < sizeof(crypto::secret_key); ++i) + k.data[i] ^= *ptr++; + } + } + //----------------------------------------------------------------- + void account_keys::encrypt(const crypto::chacha_key &key) + { + m_encryption_iv = crypto::rand(); + xor_with_key_stream(key); + } + //----------------------------------------------------------------- + void account_keys::decrypt(const crypto::chacha_key &key) + { + xor_with_key_stream(key); + } + //----------------------------------------------------------------- + void account_keys::encrypt_viewkey(const crypto::chacha_key &key) + { + // encrypt a large enough byte stream with chacha20 + epee::wipeable_string key_stream = get_key_stream(key, m_encryption_iv, sizeof(crypto::secret_key) * 2); + const char *ptr = key_stream.data(); + ptr += sizeof(crypto::secret_key); + for (size_t i = 0; i < sizeof(crypto::secret_key); ++i) + m_view_secret_key.data[i] ^= *ptr++; + } + //----------------------------------------------------------------- + void account_keys::decrypt_viewkey(const crypto::chacha_key &key) + { + encrypt_viewkey(key); + } //----------------------------------------------------------------- account_base::account_base() { @@ -70,6 +136,16 @@ DISABLE_VS_WARNINGS(4244 4345) void account_base::set_null() { m_keys = account_keys(); + m_creation_timestamp = 0; + } + //----------------------------------------------------------------- + void account_base::deinit() + { + try{ + m_keys.get_device().disconnect(); + } catch (const std::exception &e){ + MERROR("Device disconnect exception: " << e.what()); + } } //----------------------------------------------------------------- void account_base::forget_spend_key() @@ -131,15 +207,24 @@ DISABLE_VS_WARNINGS(4244 4345) //----------------------------------------------------------------- void account_base::create_from_device(const std::string &device_name) { - hw::device &hwdev = hw::get_device(device_name); - m_keys.set_device(hwdev); hwdev.set_name(device_name); - MCDEBUG("ledger", "device type: "< m_multisig_keys; hw::device *m_device = &hw::get_device("default"); + crypto::chacha_iv m_encryption_iv; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(m_account_address) KV_SERIALIZE_VAL_POD_AS_BLOB_FORCE(m_spend_secret_key) KV_SERIALIZE_VAL_POD_AS_BLOB_FORCE(m_view_secret_key) KV_SERIALIZE_CONTAINER_POD_AS_BLOB(m_multisig_keys) + const crypto::chacha_iv default_iv{{0, 0, 0, 0, 0, 0, 0, 0}}; + KV_SERIALIZE_VAL_POD_AS_BLOB_OPT(m_encryption_iv, default_iv) END_KV_SERIALIZE_MAP() account_keys& operator=(account_keys const&) = default; + void encrypt(const crypto::chacha_key &key); + void decrypt(const crypto::chacha_key &key); + void encrypt_viewkey(const crypto::chacha_key &key); + void decrypt_viewkey(const crypto::chacha_key &key); + hw::device& get_device() const ; void set_device( hw::device &hwdev) ; + + private: + void xor_with_key_stream(const crypto::chacha_key &key); }; /************************************************************************/ @@ -66,7 +77,8 @@ namespace cryptonote public: account_base(); crypto::secret_key generate(const crypto::secret_key& recovery_key = crypto::secret_key(), bool recover = false, bool two_random = false); - void create_from_device(const std::string &device_name) ; + void create_from_device(const std::string &device_name); + void create_from_device(hw::device &hwdev); void create_from_keys(const cryptonote::account_public_address& address, const crypto::secret_key& spendkey, const crypto::secret_key& viewkey); void create_from_viewkey(const cryptonote::account_public_address& address, const crypto::secret_key& viewkey); bool make_multisig(const crypto::secret_key &view_secret_key, const crypto::secret_key &spend_secret_key, const crypto::public_key &spend_public_key, const std::vector &multisig_keys); @@ -77,6 +89,7 @@ namespace cryptonote hw::device& get_device() const {return m_keys.get_device();} void set_device( hw::device &hwdev) {m_keys.set_device(hwdev);} + void deinit(); uint64_t get_createtime() const { return m_creation_timestamp; } void set_createtime(uint64_t val) { m_creation_timestamp = val; } @@ -87,6 +100,11 @@ namespace cryptonote void forget_spend_key(); const std::vector &get_multisig_keys() const { return m_keys.m_multisig_keys; } + void encrypt_keys(const crypto::chacha_key &key) { m_keys.encrypt(key); } + void decrypt_keys(const crypto::chacha_key &key) { m_keys.decrypt(key); } + void encrypt_viewkey(const crypto::chacha_key &key) { m_keys.encrypt_viewkey(key); } + void decrypt_viewkey(const crypto::chacha_key &key) { m_keys.decrypt_viewkey(key); } + template inline void serialize(t_archive &a, const unsigned int /*ver*/) { diff --git a/src/cryptonote_basic/account_boost_serialization.h b/src/cryptonote_basic/account_boost_serialization.h index 7379d787fc1..320a960dc9c 100644 --- a/src/cryptonote_basic/account_boost_serialization.h +++ b/src/cryptonote_basic/account_boost_serialization.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/cryptonote_basic/blobdatatype.h b/src/cryptonote_basic/blobdatatype.h index 7d6ff018720..20f6b24211c 100644 --- a/src/cryptonote_basic/blobdatatype.h +++ b/src/cryptonote_basic/blobdatatype.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -30,7 +30,11 @@ #pragma once +#include +#include "span.h" + namespace cryptonote { typedef std::string blobdata; + typedef epee::span blobdata_ref; } diff --git a/src/cryptonote_basic/connection_context.h b/src/cryptonote_basic/connection_context.h index 5cd1709ab35..96398a90b1c 100644 --- a/src/cryptonote_basic/connection_context.h +++ b/src/cryptonote_basic/connection_context.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -40,7 +40,8 @@ namespace cryptonote struct cryptonote_connection_context: public epee::net_utils::connection_context_base { cryptonote_connection_context(): m_state(state_before_handshake), m_remote_blockchain_height(0), m_last_response_height(0), - m_last_request_time(boost::posix_time::microsec_clock::universal_time()), m_callback_request_count(0), m_last_known_hash(crypto::null_hash) {} + m_last_request_time(boost::date_time::not_a_date_time), m_callback_request_count(0), + m_last_known_hash(crypto::null_hash), m_pruning_seed(0), m_rpc_port(0), m_anchor(false) {} enum state { @@ -52,13 +53,16 @@ namespace cryptonote }; state m_state; - std::list m_needed_objects; + std::vector m_needed_objects; std::unordered_set m_requested_objects; uint64_t m_remote_blockchain_height; uint64_t m_last_response_height; boost::posix_time::ptime m_last_request_time; epee::copyable_atomic m_callback_request_count; //in debug purpose: problem with double callback rise crypto::hash m_last_known_hash; + uint32_t m_pruning_seed; + uint16_t m_rpc_port; + bool m_anchor; //size_t m_score; TODO: add score calculations }; @@ -67,18 +71,37 @@ namespace cryptonote switch (s) { case cryptonote_connection_context::state_before_handshake: - return "state_before_handshake"; + return "before_handshake"; case cryptonote_connection_context::state_synchronizing: - return "state_synchronizing"; + return "synchronizing"; case cryptonote_connection_context::state_standby: - return "state_standby"; + return "standby"; case cryptonote_connection_context::state_idle: - return "state_idle"; + return "idle"; case cryptonote_connection_context::state_normal: - return "state_normal"; + return "normal"; default: return "unknown"; } } + inline char get_protocol_state_char(cryptonote_connection_context::state s) + { + switch (s) + { + case cryptonote_connection_context::state_before_handshake: + return 'h'; + case cryptonote_connection_context::state_synchronizing: + return 's'; + case cryptonote_connection_context::state_standby: + return 'w'; + case cryptonote_connection_context::state_idle: + return 'i'; + case cryptonote_connection_context::state_normal: + return 'n'; + default: + return 'u'; + } + } + } diff --git a/src/cryptonote_basic/cryptonote_basic.h b/src/cryptonote_basic/cryptonote_basic.h index d4558ef7b04..055c4a22bad 100644 --- a/src/cryptonote_basic/cryptonote_basic.h +++ b/src/cryptonote_basic/cryptonote_basic.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -47,7 +47,6 @@ #include "crypto/crypto.h" #include "crypto/hash.h" #include "misc_language.h" -#include "tx_extra.h" #include "ringct/rctTypes.h" #include "device/device.hpp" @@ -153,6 +152,10 @@ namespace cryptonote }; + template static inline unsigned int getpos(T &ar) { return 0; } + template<> inline unsigned int getpos(binary_archive &ar) { return ar.stream().tellp(); } + template<> inline unsigned int getpos(binary_archive &ar) { return ar.stream().tellg(); } + class transaction_prefix { @@ -176,7 +179,15 @@ namespace cryptonote END_SERIALIZE() public: - transaction_prefix(){} + transaction_prefix(){ set_null(); } + void set_null() + { + version = 1; + unlock_time = 0; + vin.clear(); + vout.clear(); + extra.clear(); + } }; class transaction: public transaction_prefix @@ -194,9 +205,14 @@ namespace cryptonote mutable crypto::hash hash; mutable size_t blob_size; + bool pruned; + + std::atomic unprunable_size; + std::atomic prefix_size; + transaction(); - transaction(const transaction &t): transaction_prefix(t), hash_valid(false), blob_size_valid(false), signatures(t.signatures), rct_signatures(t.rct_signatures) { if (t.is_hash_valid()) { hash = t.hash; set_hash_valid(true); } if (t.is_blob_size_valid()) { blob_size = t.blob_size; set_blob_size_valid(true); } } - transaction &operator=(const transaction &t) { transaction_prefix::operator=(t); set_hash_valid(false); set_blob_size_valid(false); signatures = t.signatures; rct_signatures = t.rct_signatures; if (t.is_hash_valid()) { hash = t.hash; set_hash_valid(true); } if (t.is_blob_size_valid()) { blob_size = t.blob_size; set_blob_size_valid(true); } return *this; } + transaction(const transaction &t): transaction_prefix(t), hash_valid(false), blob_size_valid(false), signatures(t.signatures), rct_signatures(t.rct_signatures), pruned(t.pruned), unprunable_size(t.unprunable_size.load()), prefix_size(t.prefix_size.load()) { if (t.is_hash_valid()) { hash = t.hash; set_hash_valid(true); } if (t.is_blob_size_valid()) { blob_size = t.blob_size; set_blob_size_valid(true); } } + transaction &operator=(const transaction &t) { transaction_prefix::operator=(t); set_hash_valid(false); set_blob_size_valid(false); signatures = t.signatures; rct_signatures = t.rct_signatures; if (t.is_hash_valid()) { hash = t.hash; set_hash_valid(true); } if (t.is_blob_size_valid()) { blob_size = t.blob_size; set_blob_size_valid(true); } pruned = t.pruned; unprunable_size = t.unprunable_size.load(); prefix_size = t.prefix_size.load(); return *this; } virtual ~transaction(); void set_null(); void invalidate_hashes(); @@ -204,6 +220,8 @@ namespace cryptonote void set_hash_valid(bool v) const { hash_valid.store(v,std::memory_order_release); } bool is_blob_size_valid() const { return blob_size_valid.load(std::memory_order_acquire); } void set_blob_size_valid(bool v) const { blob_size_valid.store(v,std::memory_order_release); } + void set_hash(const crypto::hash &h) { hash = h; set_hash_valid(true); } + void set_blob_size(size_t sz) { blob_size = sz; set_blob_size_valid(true); } BEGIN_SERIALIZE_OBJECT() if (!typename Archive::is_saving()) @@ -212,10 +230,18 @@ namespace cryptonote set_blob_size_valid(false); } + const unsigned int start_pos = getpos(ar); + FIELDS(*static_cast(this)) + if (std::is_same, binary_archive>()) + prefix_size = getpos(ar) - start_pos; + if (version == 1) { + if (std::is_same, binary_archive>()) + unprunable_size = getpos(ar) - start_pos; + ar.tag("signatures"); ar.begin_array(); PREPARE_CUSTOM_VECTOR_SERIALIZATION(vin.size(), signatures); @@ -223,7 +249,7 @@ namespace cryptonote if (!signatures_not_expected && vin.size() != signatures.size()) return false; - for (size_t i = 0; i < vin.size(); ++i) + if (!pruned) for (size_t i = 0; i < vin.size(); ++i) { size_t signature_size = get_signature_size(vin[i]); if (signatures_not_expected) @@ -254,7 +280,11 @@ namespace cryptonote bool r = rct_signatures.serialize_rctsig_base(ar, vin.size(), vout.size()); if (!r || !ar.stream().good()) return false; ar.end_object(); - if (rct_signatures.type != rct::RCTTypeNull) + + if (std::is_same, binary_archive>()) + unprunable_size = getpos(ar) - start_pos; + + if (!pruned && rct_signatures.type != rct::RCTTypeNull) { ar.tag("rctsig_prunable"); ar.begin_object(); @@ -265,6 +295,8 @@ namespace cryptonote } } } + if (!typename Archive::is_saving()) + pruned = false; END_SERIALIZE() template class Archive> @@ -286,7 +318,9 @@ namespace cryptonote ar.end_object(); } } - return true; + if (!typename Archive::is_saving()) + pruned = true; + return ar.stream().good(); } private: @@ -303,21 +337,19 @@ namespace cryptonote inline transaction::~transaction() { - //set_null(); } inline void transaction::set_null() { - version = 1; - unlock_time = 0; - vin.clear(); - vout.clear(); - extra.clear(); + transaction_prefix::set_null(); signatures.clear(); rct_signatures.type = rct::RCTTypeNull; set_hash_valid(false); set_blob_size_valid(false); + pruned = false; + unprunable_size = 0; + prefix_size = 0; } inline @@ -390,6 +422,8 @@ namespace cryptonote FIELDS(*static_cast(this)) FIELD(miner_tx) FIELD(tx_hashes) + if (tx_hashes.size() > CRYPTONOTE_MAX_TX_PER_BLOCK) + return false; END_SERIALIZE() }; diff --git a/src/cryptonote_basic/cryptonote_basic_impl.cpp b/src/cryptonote_basic/cryptonote_basic_impl.cpp index 08a95d2e9f4..d8de65b816e 100644 --- a/src/cryptonote_basic/cryptonote_basic_impl.cpp +++ b/src/cryptonote_basic/cryptonote_basic_impl.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -40,7 +40,7 @@ using namespace epee; #include "misc_language.h" #include "common/base58.h" #include "crypto/hash.h" -#include "common/int-util.h" +#include "int-util.h" #include "common/dns_utils.h" #undef MONERO_DEFAULT_LOG_CATEGORY @@ -67,7 +67,7 @@ namespace cryptonote { /* Cryptonote helper functions */ /************************************************************************/ //----------------------------------------------------------------------------------------------- - size_t get_min_block_size(uint8_t version) + size_t get_min_block_weight(uint8_t version) { if (version < 2) return CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V1; @@ -76,17 +76,12 @@ namespace cryptonote { return CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V5; } //----------------------------------------------------------------------------------------------- - size_t get_max_block_size() - { - return CRYPTONOTE_MAX_BLOCK_SIZE; - } - //----------------------------------------------------------------------------------------------- size_t get_max_tx_size() { return CRYPTONOTE_MAX_TX_SIZE; } //----------------------------------------------------------------------------------------------- - bool get_block_reward(size_t median_size, size_t current_block_size, uint64_t already_generated_coins, uint64_t &reward, uint8_t version) { + bool get_block_reward(size_t median_weight, size_t current_block_weight, uint64_t already_generated_coins, uint64_t &reward, uint8_t version) { static_assert(DIFFICULTY_TARGET_V2%60==0&&DIFFICULTY_TARGET_V1%60==0,"difficulty targets must be a multiple of 60"); const int target = version < 2 ? DIFFICULTY_TARGET_V1 : DIFFICULTY_TARGET_V2; const int target_minutes = target / 60; @@ -98,37 +93,37 @@ namespace cryptonote { base_reward = FINAL_SUBSIDY_PER_MINUTE*target_minutes; } - uint64_t full_reward_zone = get_min_block_size(version); + uint64_t full_reward_zone = get_min_block_weight(version); //make it soft - if (median_size < full_reward_zone) { - median_size = full_reward_zone; + if (median_weight < full_reward_zone) { + median_weight = full_reward_zone; } - if (current_block_size <= median_size) { + if (current_block_weight <= median_weight) { reward = base_reward; return true; } - if(current_block_size > 2 * median_size) { - MERROR("Block cumulative size is too big: " << current_block_size << ", expected less than " << 2 * median_size); + if(current_block_weight > 2 * median_weight) { + MERROR("Block cumulative weight is too big: " << current_block_weight << ", expected less than " << 2 * median_weight); return false; } - assert(median_size < std::numeric_limits::max()); - assert(current_block_size < std::numeric_limits::max()); + assert(median_weight < std::numeric_limits::max()); + assert(current_block_weight < std::numeric_limits::max()); uint64_t product_hi; // BUGFIX: 32-bit saturation bug (e.g. ARM7), the result was being // treated as 32-bit by default. - uint64_t multiplicand = 2 * median_size - current_block_size; - multiplicand *= current_block_size; + uint64_t multiplicand = 2 * median_weight - current_block_weight; + multiplicand *= current_block_weight; uint64_t product_lo = mul128(base_reward, multiplicand, &product_hi); uint64_t reward_hi; uint64_t reward_lo; - div128_32(product_hi, product_lo, static_cast(median_size), &reward_hi, &reward_lo); - div128_32(reward_hi, reward_lo, static_cast(median_size), &reward_hi, &reward_lo); + div128_32(product_hi, product_lo, static_cast(median_weight), &reward_hi, &reward_lo); + div128_32(reward_hi, reward_lo, static_cast(median_weight), &reward_hi, &reward_lo); assert(0 == reward_hi); assert(reward_lo < base_reward); @@ -162,10 +157,7 @@ namespace cryptonote { , account_public_address const & adr ) { - uint64_t address_prefix = nettype == TESTNET ? - (subaddress ? config::testnet::CRYPTONOTE_PUBLIC_SUBADDRESS_BASE58_PREFIX : config::testnet::CRYPTONOTE_PUBLIC_ADDRESS_BASE58_PREFIX) : nettype == STAGENET ? - (subaddress ? config::stagenet::CRYPTONOTE_PUBLIC_SUBADDRESS_BASE58_PREFIX : config::stagenet::CRYPTONOTE_PUBLIC_ADDRESS_BASE58_PREFIX) : - (subaddress ? config::CRYPTONOTE_PUBLIC_SUBADDRESS_BASE58_PREFIX : config::CRYPTONOTE_PUBLIC_ADDRESS_BASE58_PREFIX); + uint64_t address_prefix = subaddress ? get_config(nettype).CRYPTONOTE_PUBLIC_SUBADDRESS_BASE58_PREFIX : get_config(nettype).CRYPTONOTE_PUBLIC_ADDRESS_BASE58_PREFIX; return tools::base58::encode_addr(address_prefix, t_serializable_object_to_blob(adr)); } @@ -176,7 +168,7 @@ namespace cryptonote { , crypto::hash8 const & payment_id ) { - uint64_t integrated_address_prefix = nettype == TESTNET ? config::testnet::CRYPTONOTE_PUBLIC_INTEGRATED_ADDRESS_BASE58_PREFIX : nettype == STAGENET ? config::stagenet::CRYPTONOTE_PUBLIC_INTEGRATED_ADDRESS_BASE58_PREFIX : config::CRYPTONOTE_PUBLIC_INTEGRATED_ADDRESS_BASE58_PREFIX; + uint64_t integrated_address_prefix = get_config(nettype).CRYPTONOTE_PUBLIC_INTEGRATED_ADDRESS_BASE58_PREFIX; integrated_address iadr = { adr, payment_id @@ -201,15 +193,9 @@ namespace cryptonote { , std::string const & str ) { - uint64_t address_prefix = nettype == TESTNET ? - config::testnet::CRYPTONOTE_PUBLIC_ADDRESS_BASE58_PREFIX : nettype == STAGENET ? - config::stagenet::CRYPTONOTE_PUBLIC_ADDRESS_BASE58_PREFIX : config::CRYPTONOTE_PUBLIC_ADDRESS_BASE58_PREFIX; - uint64_t integrated_address_prefix = nettype == TESTNET ? - config::testnet::CRYPTONOTE_PUBLIC_INTEGRATED_ADDRESS_BASE58_PREFIX : nettype == STAGENET ? - config::stagenet::CRYPTONOTE_PUBLIC_INTEGRATED_ADDRESS_BASE58_PREFIX : config::CRYPTONOTE_PUBLIC_INTEGRATED_ADDRESS_BASE58_PREFIX; - uint64_t subaddress_prefix = nettype == TESTNET ? - config::testnet::CRYPTONOTE_PUBLIC_SUBADDRESS_BASE58_PREFIX : nettype == STAGENET ? - config::stagenet::CRYPTONOTE_PUBLIC_SUBADDRESS_BASE58_PREFIX : config::CRYPTONOTE_PUBLIC_SUBADDRESS_BASE58_PREFIX; + uint64_t address_prefix = get_config(nettype).CRYPTONOTE_PUBLIC_ADDRESS_BASE58_PREFIX; + uint64_t integrated_address_prefix = get_config(nettype).CRYPTONOTE_PUBLIC_INTEGRATED_ADDRESS_BASE58_PREFIX; + uint64_t subaddress_prefix = get_config(nettype).CRYPTONOTE_PUBLIC_SUBADDRESS_BASE58_PREFIX; if (2 * sizeof(public_address_outer_blob) != str.size()) { @@ -331,13 +317,13 @@ namespace cryptonote { } //-------------------------------------------------------------------------------- -bool parse_hash256(const std::string str_hash, crypto::hash& hash) +bool parse_hash256(const std::string &str_hash, crypto::hash& hash) { std::string buf; bool res = epee::string_tools::parse_hexstr_to_binbuff(str_hash, buf); if (!res || buf.size() != sizeof(crypto::hash)) { - std::cout << "invalid hash format: <" << str_hash << '>' << std::endl; + MERROR("invalid hash format: " << str_hash); return false; } else diff --git a/src/cryptonote_basic/cryptonote_basic_impl.h b/src/cryptonote_basic/cryptonote_basic_impl.h index f597850214a..c7198a16f3b 100644 --- a/src/cryptonote_basic/cryptonote_basic_impl.h +++ b/src/cryptonote_basic/cryptonote_basic_impl.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -86,10 +86,9 @@ namespace cryptonote { /************************************************************************/ /* Cryptonote helper functions */ /************************************************************************/ - size_t get_min_block_size(uint8_t version); - size_t get_max_block_size(); + size_t get_min_block_weight(uint8_t version); size_t get_max_tx_size(); - bool get_block_reward(size_t median_size, size_t current_block_size, uint64_t already_generated_coins, uint64_t &reward, uint8_t version); + bool get_block_reward(size_t median_weight, size_t current_block_weight, uint64_t already_generated_coins, uint64_t &reward, uint8_t version); uint8_t get_account_address_checksum(const public_address_outer_blob& bl); uint8_t get_account_integrated_address_checksum(const public_integrated_address_outer_blob& bl); @@ -124,5 +123,5 @@ namespace cryptonote { bool operator ==(const cryptonote::block& a, const cryptonote::block& b); } -bool parse_hash256(const std::string str_hash, crypto::hash& hash); +bool parse_hash256(const std::string &str_hash, crypto::hash& hash); diff --git a/src/cryptonote_basic/cryptonote_boost_serialization.h b/src/cryptonote_basic/cryptonote_boost_serialization.h index 143133163e1..79ce610a9fb 100644 --- a/src/cryptonote_basic/cryptonote_boost_serialization.h +++ b/src/cryptonote_basic/cryptonote_boost_serialization.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -40,6 +40,7 @@ #include #include #include "cryptonote_basic.h" +#include "difficulty.h" #include "common/unordered_containers_boost_serialization.h" #include "crypto/crypto.h" #include "ringct/rctTypes.h" @@ -249,7 +250,6 @@ namespace boost { a & x.mask; a & x.amount; - // a & x.senderPk; // not serialized, as we do not use it in monero currently } template @@ -295,7 +295,7 @@ namespace boost a & x.type; if (x.type == rct::RCTTypeNull) return; - if (x.type != rct::RCTTypeFull && x.type != rct::RCTTypeFullBulletproof && x.type != rct::RCTTypeSimple && x.type != rct::RCTTypeSimpleBulletproof) + if (x.type != rct::RCTTypeFull && x.type != rct::RCTTypeSimple && x.type != rct::RCTTypeBulletproof && x.type != rct::RCTTypeBulletproof2) throw boost::archive::archive_exception(boost::archive::archive_exception::other_exception, "Unsupported rct type"); // a & x.message; message is not serialized, as it can be reconstructed from the tx data // a & x.mixRing; mixRing is not serialized, as it can be reconstructed from the offsets @@ -323,7 +323,7 @@ namespace boost a & x.type; if (x.type == rct::RCTTypeNull) return; - if (x.type != rct::RCTTypeFull && x.type != rct::RCTTypeFullBulletproof && x.type != rct::RCTTypeSimple && x.type != rct::RCTTypeSimpleBulletproof) + if (x.type != rct::RCTTypeFull && x.type != rct::RCTTypeSimple && x.type != rct::RCTTypeBulletproof && x.type != rct::RCTTypeBulletproof2) throw boost::archive::archive_exception(boost::archive::archive_exception::other_exception, "Unsupported rct type"); // a & x.message; message is not serialized, as it can be reconstructed from the tx data // a & x.mixRing; mixRing is not serialized, as it can be reconstructed from the offsets @@ -337,9 +337,44 @@ namespace boost if (x.p.rangeSigs.empty()) a & x.p.bulletproofs; a & x.p.MGs; - if (x.type == rct::RCTTypeSimpleBulletproof) + if (x.type == rct::RCTTypeBulletproof || x.type == rct::RCTTypeBulletproof2) a & x.p.pseudoOuts; } + + template + inline void serialize(Archive &a, rct::RCTConfig &x, const boost::serialization::version_type ver) + { + a & x.range_proof_type; + a & x.bp_version; + } + + template + inline void serialize(Archive &a, cryptonote::difficulty_type &x, const boost::serialization::version_type ver) + { + if (Archive::is_loading::value) + { + // load high part + uint64_t v = 0; + a & v; + x = v; + // load low part + x = x << 64; + a & v; + x += v; + } + else + { + // store high part + cryptonote::difficulty_type x_ = (x >> 64) & 0xffffffffffffffff; + uint64_t v = x_.convert_to(); + a & v; + // store low part + x_ = x & 0xffffffffffffffff; + v = x_.convert_to(); + a & v; + } + } + } } diff --git a/src/cryptonote_basic/cryptonote_format_utils.cpp b/src/cryptonote_basic/cryptonote_format_utils.cpp index 428be1c9cc1..7d7de416d0b 100644 --- a/src/cryptonote_basic/cryptonote_format_utils.cpp +++ b/src/cryptonote_basic/cryptonote_format_utils.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -28,9 +28,6 @@ // // Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers -#include "include_base_utils.h" -using namespace epee; - #include #include #include "wipeable_string.h" @@ -42,6 +39,8 @@ using namespace epee; #include "crypto/hash.h" #include "ringct/rctSigs.h" +using namespace epee; + #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "cn" @@ -135,23 +134,41 @@ namespace cryptonote return false; } for (size_t n = 0; n < tx.rct_signatures.outPk.size(); ++n) + { + if (tx.vout[n].target.type() != typeid(txout_to_key)) + { + LOG_PRINT_L1("Unsupported output type in tx " << get_transaction_hash(tx)); + return false; + } rv.outPk[n].dest = rct::pk2rct(boost::get(tx.vout[n].target).key); + } if (!base_only) { - const bool bulletproof = rv.type == rct::RCTTypeFullBulletproof || rv.type == rct::RCTTypeSimpleBulletproof; + const bool bulletproof = rct::is_rct_bulletproof(rv.type); if (bulletproof) { - if (rv.p.bulletproofs.size() != tx.vout.size()) + if (rv.p.bulletproofs.size() != 1) { LOG_PRINT_L1("Failed to parse transaction from blob, bad bulletproofs size in tx " << get_transaction_hash(tx)); return false; } - for (size_t n = 0; n < rv.outPk.size(); ++n) + if (rv.p.bulletproofs[0].L.size() < 6) + { + LOG_PRINT_L1("Failed to parse transaction from blob, bad bulletproofs L size in tx " << get_transaction_hash(tx)); + return false; + } + const size_t max_outputs = 1 << (rv.p.bulletproofs[0].L.size() - 6); + if (max_outputs < tx.vout.size()) { - rv.p.bulletproofs[n].V.resize(1); - rv.p.bulletproofs[n].V[0] = rv.outPk[n].mask; + LOG_PRINT_L1("Failed to parse transaction from blob, bad bulletproofs max outputs in tx " << get_transaction_hash(tx)); + return false; } + const size_t n_amounts = tx.vout.size(); + CHECK_AND_ASSERT_MES(n_amounts == rv.outPk.size(), false, "Internal error filling out V"); + rv.p.bulletproofs[0].V.resize(n_amounts); + for (size_t i = 0; i < n_amounts; ++i) + rv.p.bulletproofs[0].V[i] = rct::scalarmultKey(rv.outPk[i].mask, rct::INV_EIGHT); } } } @@ -167,6 +184,7 @@ namespace cryptonote CHECK_AND_ASSERT_MES(r, false, "Failed to parse transaction from blob"); CHECK_AND_ASSERT_MES(expand_transaction_1(tx, false), false, "Failed to expand transaction data"); tx.invalidate_hashes(); + tx.set_blob_size(tx_blob.size()); return true; } //--------------------------------------------------------------- @@ -178,10 +196,21 @@ namespace cryptonote bool r = tx.serialize_base(ba); CHECK_AND_ASSERT_MES(r, false, "Failed to parse transaction from blob"); CHECK_AND_ASSERT_MES(expand_transaction_1(tx, true), false, "Failed to expand transaction data"); + tx.invalidate_hashes(); return true; } //--------------------------------------------------------------- - bool parse_and_validate_tx_from_blob(const blobdata& tx_blob, transaction& tx, crypto::hash& tx_hash, crypto::hash& tx_prefix_hash) + bool parse_and_validate_tx_prefix_from_blob(const blobdata& tx_blob, transaction_prefix& tx) + { + std::stringstream ss; + ss << tx_blob; + binary_archive ba(ss); + bool r = ::serialization::serialize_noeof(ba, tx); + CHECK_AND_ASSERT_MES(r, false, "Failed to parse transaction prefix from blob"); + return true; + } + //--------------------------------------------------------------- + bool parse_and_validate_tx_from_blob(const blobdata& tx_blob, transaction& tx, crypto::hash& tx_hash) { std::stringstream ss; ss << tx_blob; @@ -192,24 +221,56 @@ namespace cryptonote tx.invalidate_hashes(); //TODO: validate tx - get_transaction_hash(tx, tx_hash); + return get_transaction_hash(tx, tx_hash); + } + //--------------------------------------------------------------- + bool parse_and_validate_tx_from_blob(const blobdata& tx_blob, transaction& tx, crypto::hash& tx_hash, crypto::hash& tx_prefix_hash) + { + if (!parse_and_validate_tx_from_blob(tx_blob, tx, tx_hash)) + return false; get_transaction_prefix_hash(tx, tx_prefix_hash); return true; } //--------------------------------------------------------------- + bool is_v1_tx(const blobdata_ref& tx_blob) + { + uint64_t version; + const char* begin = static_cast(tx_blob.data()); + const char* end = begin + tx_blob.size(); + int read = tools::read_varint(begin, end, version); + if (read <= 0) + throw std::runtime_error("Internal error getting transaction version"); + return version <= 1; + } + //--------------------------------------------------------------- + bool is_v1_tx(const blobdata& tx_blob) + { + return is_v1_tx(blobdata_ref{tx_blob.data(), tx_blob.size()}); + } + //--------------------------------------------------------------- bool generate_key_image_helper(const account_keys& ack, const std::unordered_map& subaddresses, const crypto::public_key& out_key, const crypto::public_key& tx_public_key, const std::vector& additional_tx_public_keys, size_t real_output_index, keypair& in_ephemeral, crypto::key_image& ki, hw::device &hwdev) { crypto::key_derivation recv_derivation = AUTO_VAL_INIT(recv_derivation); bool r = hwdev.generate_key_derivation(tx_public_key, ack.m_view_secret_key, recv_derivation); - CHECK_AND_ASSERT_MES(r, false, "key image helper: failed to generate_key_derivation(" << tx_public_key << ", " << ack.m_view_secret_key << ")"); + if (!r) + { + MWARNING("key image helper: failed to generate_key_derivation(" << tx_public_key << ", " << ack.m_view_secret_key << ")"); + memcpy(&recv_derivation, rct::identity().bytes, sizeof(recv_derivation)); + } std::vector additional_recv_derivations; for (size_t i = 0; i < additional_tx_public_keys.size(); ++i) { crypto::key_derivation additional_recv_derivation = AUTO_VAL_INIT(additional_recv_derivation); r = hwdev.generate_key_derivation(additional_tx_public_keys[i], ack.m_view_secret_key, additional_recv_derivation); - CHECK_AND_ASSERT_MES(r, false, "key image helper: failed to generate_key_derivation(" << additional_tx_public_keys[i] << ", " << ack.m_view_secret_key << ")"); - additional_recv_derivations.push_back(additional_recv_derivation); + if (!r) + { + MWARNING("key image helper: failed to generate_key_derivation(" << additional_tx_public_keys[i] << ", " << ack.m_view_secret_key << ")"); + } + else + { + additional_recv_derivations.push_back(additional_recv_derivation); + } } boost::optional subaddr_recv_info = is_out_to_acc_precomp(subaddresses, out_key, recv_derivation, additional_recv_derivations, real_output_index,hwdev); @@ -220,6 +281,11 @@ namespace cryptonote //--------------------------------------------------------------- bool generate_key_image_helper_precomp(const account_keys& ack, const crypto::public_key& out_key, const crypto::key_derivation& recv_derivation, size_t real_output_index, const subaddress_index& received_index, keypair& in_ephemeral, crypto::key_image& ki, hw::device &hwdev) { + if (hwdev.compute_key_image(ack, out_key, recv_derivation, real_output_index, received_index, in_ephemeral, ki)) + { + return true; + } + if (ack.m_spend_secret_key == crypto::null_skey) { // for watch-only wallet, simply copy the known output pubkey @@ -318,6 +384,46 @@ namespace cryptonote return string_tools::get_xtype_from_string(amount, str_amount); } //--------------------------------------------------------------- + uint64_t get_transaction_weight(const transaction &tx, size_t blob_size) + { + if (tx.version < 2) + return blob_size; + const rct::rctSig &rv = tx.rct_signatures; + if (!rct::is_rct_bulletproof(rv.type)) + return blob_size; + const size_t n_outputs = tx.vout.size(); + if (n_outputs <= 2) + return blob_size; + const uint64_t bp_base = 368; + const size_t n_padded_outputs = rct::n_bulletproof_max_amounts(rv.p.bulletproofs); + size_t nlr = 0; + for (const auto &bp: rv.p.bulletproofs) + nlr += bp.L.size() * 2; + const size_t bp_size = 32 * (9 + nlr); + CHECK_AND_ASSERT_THROW_MES_L1(n_outputs <= BULLETPROOF_MAX_OUTPUTS, "maximum number of outputs is " + std::to_string(BULLETPROOF_MAX_OUTPUTS) + " per transaction"); + CHECK_AND_ASSERT_THROW_MES_L1(bp_base * n_padded_outputs >= bp_size, "Invalid bulletproof clawback"); + const uint64_t bp_clawback = (bp_base * n_padded_outputs - bp_size) * 4 / 5; + CHECK_AND_ASSERT_THROW_MES_L1(bp_clawback <= std::numeric_limits::max() - blob_size, "Weight overflow"); + return blob_size + bp_clawback; + } + //--------------------------------------------------------------- + uint64_t get_transaction_weight(const transaction &tx) + { + size_t blob_size; + if (tx.is_blob_size_valid()) + { + blob_size = tx.blob_size; + } + else + { + std::ostringstream s; + binary_archive a(s); + ::serialization::serialize(a, const_cast(tx)); + blob_size = s.str().size(); + } + return get_transaction_weight(tx, blob_size); + } + //--------------------------------------------------------------- bool get_tx_fee(const transaction& tx, uint64_t & fee) { if (tx.version > 1) @@ -376,6 +482,91 @@ namespace cryptonote return true; } //--------------------------------------------------------------- + template + static bool pick(binary_archive &ar, std::vector &fields, uint8_t tag) + { + std::vector::iterator it; + while ((it = std::find_if(fields.begin(), fields.end(), [](const tx_extra_field &f) { return f.type() == typeid(T); })) != fields.end()) + { + bool r = ::do_serialize(ar, tag); + CHECK_AND_NO_ASSERT_MES_L1(r, false, "failed to serialize tx extra field"); + r = ::do_serialize(ar, boost::get(*it)); + CHECK_AND_NO_ASSERT_MES_L1(r, false, "failed to serialize tx extra field"); + fields.erase(it); + } + return true; + } + //--------------------------------------------------------------- + bool sort_tx_extra(const std::vector& tx_extra, std::vector &sorted_tx_extra, bool allow_partial) + { + std::vector tx_extra_fields; + + if(tx_extra.empty()) + { + sorted_tx_extra.clear(); + return true; + } + + std::string extra_str(reinterpret_cast(tx_extra.data()), tx_extra.size()); + std::istringstream iss(extra_str); + binary_archive ar(iss); + + bool eof = false; + size_t processed = 0; + while (!eof) + { + tx_extra_field field; + bool r = ::do_serialize(ar, field); + if (!r) + { + MWARNING("failed to deserialize extra field. extra = " << string_tools::buff_to_hex_nodelimer(std::string(reinterpret_cast(tx_extra.data()), tx_extra.size()))); + if (!allow_partial) + return false; + break; + } + tx_extra_fields.push_back(field); + processed = iss.tellg(); + + std::ios_base::iostate state = iss.rdstate(); + eof = (EOF == iss.peek()); + iss.clear(state); + } + if (!::serialization::check_stream_state(ar)) + { + MWARNING("failed to deserialize extra field. extra = " << string_tools::buff_to_hex_nodelimer(std::string(reinterpret_cast(tx_extra.data()), tx_extra.size()))); + if (!allow_partial) + return false; + } + MTRACE("Sorted " << processed << "/" << tx_extra.size()); + + std::ostringstream oss; + binary_archive nar(oss); + + // sort by: + if (!pick(nar, tx_extra_fields, TX_EXTRA_TAG_PUBKEY)) return false; + if (!pick(nar, tx_extra_fields, TX_EXTRA_TAG_ADDITIONAL_PUBKEYS)) return false; + if (!pick(nar, tx_extra_fields, TX_EXTRA_NONCE)) return false; + if (!pick(nar, tx_extra_fields, TX_EXTRA_MERGE_MINING_TAG)) return false; + if (!pick(nar, tx_extra_fields, TX_EXTRA_MYSTERIOUS_MINERGATE_TAG)) return false; + if (!pick(nar, tx_extra_fields, TX_EXTRA_TAG_PADDING)) return false; + + // if not empty, someone added a new type and did not add a case above + if (!tx_extra_fields.empty()) + { + MERROR("tx_extra_fields not empty after sorting, someone forgot to add a case above"); + return false; + } + + std::string oss_str = oss.str(); + if (allow_partial && processed < tx_extra.size()) + { + MDEBUG("Appending unparsed data"); + oss_str += std::string((const char*)tx_extra.data() + processed, tx_extra.size() - processed); + } + sorted_tx_extra = std::vector(oss_str.begin(), oss_str.end()); + return true; + } + //--------------------------------------------------------------- crypto::public_key get_tx_pub_key_from_extra(const std::vector& tx_extra, size_t pk_index) { std::vector tx_extra_fields; @@ -699,6 +890,11 @@ namespace cryptonote return true; } //--------------------------------------------------------------- + void get_blob_hash(const epee::span& blob, crypto::hash& res) + { + cn_fast_hash(blob.data(), blob.size(), res); + } + //--------------------------------------------------------------- void get_blob_hash(const blobdata& blob, crypto::hash& res) { cn_fast_hash(blob.data(), blob.size(), res); @@ -729,7 +925,7 @@ namespace cryptonote { if (decimal_point == (unsigned int)-1) decimal_point = default_decimal_point; - switch (std::atomic_load(&default_decimal_point)) + switch (decimal_point) { case 12: return "monero"; @@ -742,7 +938,7 @@ namespace cryptonote case 0: return "piconero"; default: - ASSERT_MES_AND_THROW("Invalid decimal point specification: " << default_decimal_point); + ASSERT_MES_AND_THROW("Invalid decimal point specification: " << decimal_point); } } //--------------------------------------------------------------- @@ -767,10 +963,18 @@ namespace cryptonote return h; } //--------------------------------------------------------------- + crypto::hash get_blob_hash(const epee::span& blob) + { + crypto::hash h = null_hash; + get_blob_hash(blob, h); + return h; + } + //--------------------------------------------------------------- crypto::hash get_transaction_hash(const transaction& t) { crypto::hash h = null_hash; get_transaction_hash(t, h, NULL); + CHECK_AND_ASSERT_THROW_MES(get_transaction_hash(t, h, NULL), "Failed to calculate transaction hash"); return h; } //--------------------------------------------------------------- @@ -779,26 +983,35 @@ namespace cryptonote return get_transaction_hash(t, res, NULL); } //--------------------------------------------------------------- - bool calculate_transaction_prunable_hash(const transaction& t, crypto::hash& res) + bool calculate_transaction_prunable_hash(const transaction& t, const cryptonote::blobdata *blob, crypto::hash& res) { if (t.version == 1) return false; - transaction &tt = const_cast(t); - std::stringstream ss; - binary_archive ba(ss); - const size_t inputs = t.vin.size(); - const size_t outputs = t.vout.size(); - const size_t mixin = t.vin.empty() ? 0 : t.vin[0].type() == typeid(txin_to_key) ? boost::get(t.vin[0]).key_offsets.size() - 1 : 0; - bool r = tt.rct_signatures.p.serialize_rctsig_prunable(ba, t.rct_signatures.type, inputs, outputs, mixin); - CHECK_AND_ASSERT_MES(r, false, "Failed to serialize rct signatures prunable"); - cryptonote::get_blob_hash(ss.str(), res); + const unsigned int unprunable_size = t.unprunable_size; + if (blob && unprunable_size) + { + CHECK_AND_ASSERT_MES(unprunable_size <= blob->size(), false, "Inconsistent transaction unprunable and blob sizes"); + cryptonote::get_blob_hash(epee::span(blob->data() + unprunable_size, blob->size() - unprunable_size), res); + } + else + { + transaction &tt = const_cast(t); + std::stringstream ss; + binary_archive ba(ss); + const size_t inputs = t.vin.size(); + const size_t outputs = t.vout.size(); + const size_t mixin = t.vin.empty() ? 0 : t.vin[0].type() == typeid(txin_to_key) ? boost::get(t.vin[0]).key_offsets.size() - 1 : 0; + bool r = tt.rct_signatures.p.serialize_rctsig_prunable(ba, t.rct_signatures.type, inputs, outputs, mixin); + CHECK_AND_ASSERT_MES(r, false, "Failed to serialize rct signatures prunable"); + cryptonote::get_blob_hash(ss.str(), res); + } return true; } //--------------------------------------------------------------- - crypto::hash get_transaction_prunable_hash(const transaction& t) + crypto::hash get_transaction_prunable_hash(const transaction& t, const cryptonote::blobdata *blobdata) { crypto::hash res; - CHECK_AND_ASSERT_THROW_MES(calculate_transaction_prunable_hash(t, res), "Failed to calculate tx prunable hash"); + CHECK_AND_ASSERT_THROW_MES(calculate_transaction_prunable_hash(t, blobdata, res), "Failed to calculate tx prunable hash"); return res; } //--------------------------------------------------------------- @@ -827,7 +1040,10 @@ namespace cryptonote } // prunable rct - hashes[2] = pruned_data_hash; + if (t.rct_signatures.type == rct::RCTTypeNull) + hashes[2] = crypto::null_hash; + else + hashes[2] = pruned_data_hash; // the tx hash is the hash of the 3 hashes crypto::hash res = cn_fast_hash(hashes, sizeof(hashes)); @@ -849,18 +1065,13 @@ namespace cryptonote // prefix get_transaction_prefix_hash(t, hashes[0]); - transaction &tt = const_cast(t); + const blobdata blob = tx_to_blob(t); + const unsigned int unprunable_size = t.unprunable_size; + const unsigned int prefix_size = t.prefix_size; // base rct - { - std::stringstream ss; - binary_archive ba(ss); - const size_t inputs = t.vin.size(); - const size_t outputs = t.vout.size(); - bool r = tt.rct_signatures.serialize_rctsig_base(ba, inputs, outputs); - CHECK_AND_ASSERT_MES(r, false, "Failed to serialize rct signatures base"); - cryptonote::get_blob_hash(ss.str(), hashes[1]); - } + CHECK_AND_ASSERT_MES(prefix_size <= unprunable_size && unprunable_size <= blob.size(), false, "Inconsistent transaction prefix, unprunable and blob sizes"); + cryptonote::get_blob_hash(epee::span(blob.data() + prefix_size, unprunable_size - prefix_size), hashes[1]); // prunable rct if (t.rct_signatures.type == rct::RCTTypeNull) @@ -869,7 +1080,7 @@ namespace cryptonote } else { - CHECK_AND_ASSERT_MES(calculate_transaction_prunable_hash(t, hashes[2]), false, "Failed to get tx prunable hash"); + CHECK_AND_ASSERT_MES(calculate_transaction_prunable_hash(t, &blob, hashes[2]), false, "Failed to get tx prunable hash"); } // the tx hash is the hash of the 3 hashes @@ -877,7 +1088,14 @@ namespace cryptonote // we still need the size if (blob_size) - *blob_size = get_object_blobsize(t); + { + if (!t.is_blob_size_valid()) + { + t.blob_size = blob.size(); + t.set_blob_size_valid(true); + } + *blob_size = t.blob_size; + } return true; } @@ -930,21 +1148,37 @@ namespace cryptonote return blob; } //--------------------------------------------------------------- - bool calculate_block_hash(const block& b, crypto::hash& res) + bool calculate_block_hash(const block& b, crypto::hash& res, const blobdata *blob) { + blobdata bd; + if (!blob) + { + bd = block_to_blob(b); + blob = &bd; + } + + bool hash_result = get_object_hash(get_block_hashing_blob(b), res); + if (!hash_result) + return false; + + if (b.miner_tx.vin.size() == 1 && b.miner_tx.vin[0].type() == typeid(cryptonote::txin_gen)) + { + const cryptonote::txin_gen &txin_gen = boost::get(b.miner_tx.vin[0]); + if (txin_gen.height != 202612) + return true; + } + // EXCEPTION FOR BLOCK 202612 const std::string correct_blob_hash_202612 = "3a8a2b3a29b50fc86ff73dd087ea43c6f0d6b8f936c849194d5c84c737903966"; const std::string existing_block_id_202612 = "bbd604d2ba11ba27935e006ed39c9bfdd99b76bf4a50654bc1e1e61217962698"; - crypto::hash block_blob_hash = get_blob_hash(block_to_blob(b)); + crypto::hash block_blob_hash = get_blob_hash(*blob); if (string_tools::pod_to_hex(block_blob_hash) == correct_blob_hash_202612) { string_tools::hex_to_pod(existing_block_id_202612, res); return true; } - bool hash_result = get_object_hash(get_block_hashing_blob(b), res); - if (hash_result) { // make sure that we aren't looking at a block with the 202612 block id but not the correct blobdata if (string_tools::pod_to_hex(res) == existing_block_id_202612) @@ -987,15 +1221,15 @@ namespace cryptonote bool get_block_longhash(const block& b, crypto::hash& res, uint64_t height) { // block 202612 bug workaround - const std::string longhash_202612 = "84f64766475d51837ac9efbef1926486e58563c95a19fef4aec3254f03000000"; if (height == 202612) { + static const std::string longhash_202612 = "84f64766475d51837ac9efbef1926486e58563c95a19fef4aec3254f03000000"; string_tools::hex_to_pod(longhash_202612, res); return true; } blobdata bd = get_block_hashing_blob(b); const int cn_variant = b.major_version >= 7 ? b.major_version - 6 : 0; - crypto::cn_slow_hash(bd.data(), bd.size(), res, cn_variant); + crypto::cn_slow_hash(bd.data(), bd.size(), res, cn_variant, height); return true; } //--------------------------------------------------------------- @@ -1026,7 +1260,7 @@ namespace cryptonote return p; } //--------------------------------------------------------------- - bool parse_and_validate_block_from_blob(const blobdata& b_blob, block& b) + bool parse_and_validate_block_from_blob(const blobdata& b_blob, block& b, crypto::hash *block_hash) { std::stringstream ss; ss << b_blob; @@ -1035,9 +1269,26 @@ namespace cryptonote CHECK_AND_ASSERT_MES(r, false, "Failed to parse block from blob"); b.invalidate_hashes(); b.miner_tx.invalidate_hashes(); + if (block_hash) + { + calculate_block_hash(b, *block_hash, &b_blob); + ++block_hashes_calculated_count; + b.hash = *block_hash; + b.set_hash_valid(true); + } return true; } //--------------------------------------------------------------- + bool parse_and_validate_block_from_blob(const blobdata& b_blob, block& b) + { + return parse_and_validate_block_from_blob(b_blob, b, NULL); + } + //--------------------------------------------------------------- + bool parse_and_validate_block_from_blob(const blobdata& b_blob, block& b, crypto::hash &block_hash) + { + return parse_and_validate_block_from_blob(b_blob, b, &block_hash); + } + //--------------------------------------------------------------- blobdata block_to_blob(const block& b) { return t_serializable_object_to_blob(b); @@ -1073,9 +1324,10 @@ namespace cryptonote crypto::hash get_tx_tree_hash(const block& b) { std::vector txs_ids; + txs_ids.reserve(1 + b.tx_hashes.size()); crypto::hash h = null_hash; size_t bl_sz = 0; - get_transaction_hash(b.miner_tx, h, bl_sz); + CHECK_AND_ASSERT_THROW_MES(get_transaction_hash(b.miner_tx, h, bl_sz), "Failed to calculate transaction hash"); txs_ids.push_back(h); for(auto& th: b.tx_hashes) txs_ids.push_back(th); diff --git a/src/cryptonote_basic/cryptonote_format_utils.h b/src/cryptonote_basic/cryptonote_format_utils.h index 8a5296d5bc5..c9de2a56e3b 100644 --- a/src/cryptonote_basic/cryptonote_format_utils.h +++ b/src/cryptonote_basic/cryptonote_format_utils.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -31,6 +31,7 @@ #pragma once #include "blobdatatype.h" #include "cryptonote_basic_impl.h" +#include "tx_extra.h" #include "account.h" #include "subaddress_index.h" #include "include_base_utils.h" @@ -48,9 +49,13 @@ namespace cryptonote //--------------------------------------------------------------- void get_transaction_prefix_hash(const transaction_prefix& tx, crypto::hash& h); crypto::hash get_transaction_prefix_hash(const transaction_prefix& tx); + bool parse_and_validate_tx_prefix_from_blob(const blobdata& tx_blob, transaction_prefix& tx); bool parse_and_validate_tx_from_blob(const blobdata& tx_blob, transaction& tx, crypto::hash& tx_hash, crypto::hash& tx_prefix_hash); + bool parse_and_validate_tx_from_blob(const blobdata& tx_blob, transaction& tx, crypto::hash& tx_hash); bool parse_and_validate_tx_from_blob(const blobdata& tx_blob, transaction& tx); bool parse_and_validate_tx_base_from_blob(const blobdata& tx_blob, transaction& tx); + bool is_v1_tx(const blobdata_ref& tx_blob); + bool is_v1_tx(const blobdata& tx_blob); template bool find_tx_extra_field_by_type(const std::vector& tx_extra_fields, T& field, size_t index = 0) @@ -64,6 +69,7 @@ namespace cryptonote } bool parse_tx_extra(const std::vector& tx_extra, std::vector& tx_extra_fields); + bool sort_tx_extra(const std::vector& tx_extra, std::vector &sorted_tx_extra, bool allow_partial = false); crypto::public_key get_tx_pub_key_from_extra(const std::vector& tx_extra, size_t pk_index = 0); crypto::public_key get_tx_pub_key_from_extra(const transaction_prefix& tx, size_t pk_index = 0); crypto::public_key get_tx_pub_key_from_extra(const transaction& tx, size_t pk_index = 0); @@ -93,30 +99,36 @@ namespace cryptonote bool generate_key_image_helper(const account_keys& ack, const std::unordered_map& subaddresses, const crypto::public_key& out_key, const crypto::public_key& tx_public_key, const std::vector& additional_tx_public_keys, size_t real_output_index, keypair& in_ephemeral, crypto::key_image& ki, hw::device &hwdev); bool generate_key_image_helper_precomp(const account_keys& ack, const crypto::public_key& out_key, const crypto::key_derivation& recv_derivation, size_t real_output_index, const subaddress_index& received_index, keypair& in_ephemeral, crypto::key_image& ki, hw::device &hwdev); void get_blob_hash(const blobdata& blob, crypto::hash& res); + void get_blob_hash(const epee::span& blob, crypto::hash& res); crypto::hash get_blob_hash(const blobdata& blob); + crypto::hash get_blob_hash(const epee::span& blob); std::string short_hash_str(const crypto::hash& h); crypto::hash get_transaction_hash(const transaction& t); bool get_transaction_hash(const transaction& t, crypto::hash& res); bool get_transaction_hash(const transaction& t, crypto::hash& res, size_t& blob_size); bool get_transaction_hash(const transaction& t, crypto::hash& res, size_t* blob_size); - bool calculate_transaction_prunable_hash(const transaction& t, crypto::hash& res); - crypto::hash get_transaction_prunable_hash(const transaction& t); + bool calculate_transaction_prunable_hash(const transaction& t, const cryptonote::blobdata *blob, crypto::hash& res); + crypto::hash get_transaction_prunable_hash(const transaction& t, const cryptonote::blobdata *blob = NULL); bool calculate_transaction_hash(const transaction& t, crypto::hash& res, size_t* blob_size); crypto::hash get_pruned_transaction_hash(const transaction& t, const crypto::hash &pruned_data_hash); blobdata get_block_hashing_blob(const block& b); - bool calculate_block_hash(const block& b, crypto::hash& res); + bool calculate_block_hash(const block& b, crypto::hash& res, const blobdata *blob = NULL); bool get_block_hash(const block& b, crypto::hash& res); crypto::hash get_block_hash(const block& b); bool get_block_longhash(const block& b, crypto::hash& res, uint64_t height); crypto::hash get_block_longhash(const block& b, uint64_t height); + bool parse_and_validate_block_from_blob(const blobdata& b_blob, block& b, crypto::hash *block_hash); bool parse_and_validate_block_from_blob(const blobdata& b_blob, block& b); + bool parse_and_validate_block_from_blob(const blobdata& b_blob, block& b, crypto::hash &block_hash); bool get_inputs_money_amount(const transaction& tx, uint64_t& money); uint64_t get_outs_money_amount(const transaction& tx); bool check_inputs_types_supported(const transaction& tx); bool check_outs_valid(const transaction& tx); bool parse_amount(uint64_t& amount, const std::string& str_amount); + uint64_t get_transaction_weight(const transaction &tx); + uint64_t get_transaction_weight(const transaction &tx, size_t blob_size); bool check_money_overflow(const transaction& tx); bool check_outs_overflow(const transaction& tx); @@ -130,6 +142,16 @@ namespace cryptonote std::string print_money(uint64_t amount, unsigned int decimal_point = -1); //--------------------------------------------------------------- template + bool t_serializable_object_from_blob(t_object& to, const blobdata& b_blob) + { + std::stringstream ss; + ss << b_blob; + binary_archive ba(ss); + bool r = ::serialization::serialize(ba, to); + return r; + } + //--------------------------------------------------------------- + template bool t_serializable_object_to_blob(const t_object& to, blobdata& b_blob) { std::stringstream ss; diff --git a/src/cryptonote_basic/cryptonote_stat_info.h b/src/cryptonote_basic/cryptonote_stat_info.h index c0be2144ef7..4cc1bc76426 100644 --- a/src/cryptonote_basic/cryptonote_stat_info.h +++ b/src/cryptonote_basic/cryptonote_stat_info.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -34,7 +34,7 @@ namespace cryptonote { - struct core_stat_info + struct core_stat_info_t { uint64_t tx_pool_size; uint64_t blockchain_height; @@ -50,4 +50,5 @@ namespace cryptonote KV_SERIALIZE(top_block_id_str) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init core_stat_info; } diff --git a/src/cryptonote_basic/difficulty.cpp b/src/cryptonote_basic/difficulty.cpp index cb2a00a1289..859173aa507 100644 --- a/src/cryptonote_basic/difficulty.cpp +++ b/src/cryptonote_basic/difficulty.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -34,7 +34,7 @@ #include #include -#include "common/int-util.h" +#include "int-util.h" #include "crypto/hash.h" #include "cryptonote_config.h" #include "difficulty.h" @@ -102,7 +102,7 @@ namespace cryptonote { return a + b < a || (c && a + b == (uint64_t) -1); } - bool check_hash(const crypto::hash &hash, difficulty_type difficulty) { + bool check_hash_64(const crypto::hash &hash, uint64_t difficulty) { uint64_t low, high, top, cur; // First check the highest word, this will most likely fail for a random hash. mul(swap64le(((const uint64_t *) &hash)[3]), difficulty, top, high); @@ -119,7 +119,7 @@ namespace cryptonote { return !carry; } - difficulty_type next_difficulty(std::vector timestamps, std::vector cumulative_difficulties, size_t target_seconds) { + uint64_t next_difficulty_64(std::vector timestamps, std::vector cumulative_difficulties, size_t target_seconds) { if(timestamps.size() > DIFFICULTY_WINDOW) { @@ -150,7 +150,7 @@ namespace cryptonote { if (time_span == 0) { time_span = 1; } - difficulty_type total_work = cumulative_difficulties[cut_end - 1] - cumulative_difficulties[cut_begin]; + uint64_t total_work = cumulative_difficulties[cut_end - 1] - cumulative_difficulties[cut_begin]; assert(total_work > 0); uint64_t low, high; mul(total_work, target_seconds, low, high); @@ -162,4 +162,96 @@ namespace cryptonote { return (low + time_span - 1) / time_span; } +#if defined(_MSC_VER) +#ifdef max +#undef max +#endif +#endif + + const difficulty_type max64bit(std::numeric_limits::max()); + const boost::multiprecision::uint256_t max128bit(std::numeric_limits::max()); + const boost::multiprecision::uint512_t max256bit(std::numeric_limits::max()); + +#define FORCE_FULL_128_BITS + + bool check_hash_128(const crypto::hash &hash, difficulty_type difficulty) { +#ifndef FORCE_FULL_128_BITS + // fast check + if (difficulty >= max64bit && ((const uint64_t *) &hash)[3] > 0) + return false; +#endif + // usual slow check + boost::multiprecision::uint512_t hashVal = 0; +#ifdef FORCE_FULL_128_BITS + for(int i = 0; i < 4; i++) { // highest word is zero +#else + for(int i = 1; i < 4; i++) { // highest word is zero +#endif + hashVal <<= 64; + hashVal |= swap64le(((const uint64_t *) &hash)[3 - i]); + } + return hashVal * difficulty <= max256bit; + } + + bool check_hash(const crypto::hash &hash, difficulty_type difficulty) { + if (difficulty <= max64bit) // if can convert to small difficulty - do it + return check_hash_64(hash, difficulty.convert_to()); + else + return check_hash_128(hash, difficulty); + } + + difficulty_type next_difficulty(std::vector timestamps, std::vector cumulative_difficulties, size_t target_seconds) { + //cutoff DIFFICULTY_LAG + if(timestamps.size() > DIFFICULTY_WINDOW) + { + timestamps.resize(DIFFICULTY_WINDOW); + cumulative_difficulties.resize(DIFFICULTY_WINDOW); + } + + + size_t length = timestamps.size(); + assert(length == cumulative_difficulties.size()); + if (length <= 1) { + return 1; + } + static_assert(DIFFICULTY_WINDOW >= 2, "Window is too small"); + assert(length <= DIFFICULTY_WINDOW); + sort(timestamps.begin(), timestamps.end()); + size_t cut_begin, cut_end; + static_assert(2 * DIFFICULTY_CUT <= DIFFICULTY_WINDOW - 2, "Cut length is too large"); + if (length <= DIFFICULTY_WINDOW - 2 * DIFFICULTY_CUT) { + cut_begin = 0; + cut_end = length; + } else { + cut_begin = (length - (DIFFICULTY_WINDOW - 2 * DIFFICULTY_CUT) + 1) / 2; + cut_end = cut_begin + (DIFFICULTY_WINDOW - 2 * DIFFICULTY_CUT); + } + assert(/*cut_begin >= 0 &&*/ cut_begin + 2 <= cut_end && cut_end <= length); + uint64_t time_span = timestamps[cut_end - 1] - timestamps[cut_begin]; + if (time_span == 0) { + time_span = 1; + } + difficulty_type total_work = cumulative_difficulties[cut_end - 1] - cumulative_difficulties[cut_begin]; + assert(total_work > 0); + boost::multiprecision::uint256_t res = (boost::multiprecision::uint256_t(total_work) * target_seconds + time_span - 1) / time_span; + if(res > max128bit) + return 0; // to behave like previous implementation, may be better return max128bit? + return res.convert_to(); + } + + std::string hex(difficulty_type v) + { + static const char chars[] = "0123456789abcdef"; + std::string s; + while (v > 0) + { + s.push_back(chars[(v & 0xf).convert_to()]); + v >>= 4; + } + if (s.empty()) + s += "0"; + std::reverse(s.begin(), s.end()); + return "0x" + s; + } + } diff --git a/src/cryptonote_basic/difficulty.h b/src/cryptonote_basic/difficulty.h index b0653846707..02ed89e5a82 100644 --- a/src/cryptonote_basic/difficulty.h +++ b/src/cryptonote_basic/difficulty.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -32,12 +32,14 @@ #include #include +#include +#include #include "crypto/hash.h" namespace cryptonote { - typedef std::uint64_t difficulty_type; + typedef boost::multiprecision::uint128_t difficulty_type; /** * @brief checks if a hash fits the given difficulty @@ -51,6 +53,12 @@ namespace cryptonote * * @return true if valid, else false */ + bool check_hash_64(const crypto::hash &hash, uint64_t difficulty); + uint64_t next_difficulty_64(std::vector timestamps, std::vector cumulative_difficulties, size_t target_seconds); + + bool check_hash_128(const crypto::hash &hash, difficulty_type difficulty); bool check_hash(const crypto::hash &hash, difficulty_type difficulty); difficulty_type next_difficulty(std::vector timestamps, std::vector cumulative_difficulties, size_t target_seconds); + + std::string hex(difficulty_type v); } diff --git a/src/cryptonote_basic/hardfork.cpp b/src/cryptonote_basic/hardfork.cpp index 95f1ecab9d9..98158a51327 100644 --- a/src/cryptonote_basic/hardfork.cpp +++ b/src/cryptonote_basic/hardfork.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -56,12 +56,13 @@ static uint8_t get_block_version(const cryptonote::block &b) HardFork::HardFork(cryptonote::BlockchainDB &db, uint8_t original_version, uint64_t original_version_till_height, time_t forked_time, time_t update_time, uint64_t window_size, uint8_t default_threshold_percent): db(db), - original_version(original_version), - original_version_till_height(original_version_till_height), forked_time(forked_time), update_time(update_time), window_size(window_size), - default_threshold_percent(default_threshold_percent) + default_threshold_percent(default_threshold_percent), + original_version(original_version), + original_version_till_height(original_version_till_height), + current_fork_index(0) { if (window_size == 0) throw "window_size needs to be strictly positive"; @@ -184,26 +185,8 @@ void HardFork::init() else height = 1; - bool populate = false; - try - { - db.get_hard_fork_version(0); - } - catch (...) { populate = true; } - if (populate) { - MINFO("The DB has no hard fork info, reparsing from start"); - height = 1; - } - MDEBUG("reorganizing from " << height); - if (populate) { - reorganize_from_chain_height(height); - // reorg will not touch the genesis block, use this as a flag for populating done - db.set_hard_fork_version(0, original_version); - } - else { - rescan_from_chain_height(height); - } - MDEBUG("reorganization done"); + rescan_from_chain_height(height); + MDEBUG("init done"); } uint8_t HardFork::get_block_version(uint64_t height) const @@ -221,7 +204,6 @@ bool HardFork::reorganize_from_block_height(uint64_t height) if (height >= db.height()) return false; - db.set_batch_transactions(true); bool stop_batch = db.batch_start(); versions.clear(); @@ -266,11 +248,9 @@ bool HardFork::reorganize_from_chain_height(uint64_t height) bool HardFork::rescan_from_block_height(uint64_t height) { CRITICAL_REGION_LOCAL(lock); - db.block_txn_start(true); - if (height >= db.height()) { - db.block_txn_stop(); + db_rtxn_guard rtxn_guard(&db); + if (height >= db.height()) return false; - } versions.clear(); @@ -293,8 +273,6 @@ bool HardFork::rescan_from_block_height(uint64_t height) current_fork_index = voted; } - db.block_txn_stop(); - return true; } @@ -305,11 +283,36 @@ bool HardFork::rescan_from_chain_height(uint64_t height) return rescan_from_block_height(height - 1); } +void HardFork::on_block_popped(uint64_t nblocks) +{ + CHECK_AND_ASSERT_THROW_MES(nblocks > 0, "nblocks must be greater than 0"); + + CRITICAL_REGION_LOCAL(lock); + + const uint64_t new_chain_height = db.height(); + const uint64_t old_chain_height = new_chain_height + nblocks; + uint8_t version; + for (uint64_t height = old_chain_height - 1; height >= new_chain_height; --height) + { + version = versions.back(); + last_versions[version]--; + versions.pop_back(); + version = db.get_hard_fork_version(height); + versions.push_front(version); + last_versions[version]++; + } + + // does not take voting into account + for (current_fork_index = heights.size() - 1; current_fork_index > 0; --current_fork_index) + if (new_chain_height >= heights[current_fork_index].height) + break; +} + int HardFork::get_voted_fork_index(uint64_t height) const { CRITICAL_REGION_LOCAL(lock); uint32_t accumulated_votes = 0; - for (unsigned int n = heights.size() - 1; n > current_fork_index; --n) { + for (int n = heights.size() - 1; n >= 0; --n) { uint8_t v = heights[n].version; accumulated_votes += last_versions[v]; uint32_t threshold = (window_size * heights[n].threshold + 99) / 100; @@ -379,20 +382,24 @@ uint8_t HardFork::get_ideal_version(uint64_t height) const uint64_t HardFork::get_earliest_ideal_height_for_version(uint8_t version) const { - for (unsigned int n = heights.size() - 1; n > 0; --n) { - if (heights[n].version <= version) - return heights[n].height; + uint64_t height = std::numeric_limits::max(); + for (auto i = heights.rbegin(); i != heights.rend(); ++i) { + if (i->version >= version) { + height = i->height; + } else { + break; + } } - return 0; + return height; } uint8_t HardFork::get_next_version() const { CRITICAL_REGION_LOCAL(lock); uint64_t height = db.height(); - for (unsigned int n = heights.size() - 1; n > 0; --n) { - if (height >= heights[n].height) { - return heights[n < heights.size() - 1 ? n + 1 : n].version; + for (auto i = heights.rbegin(); i != heights.rend(); ++i) { + if (height >= i->height) { + return (i == heights.rbegin() ? i : (i - 1))->version; } } return original_version; diff --git a/src/cryptonote_basic/hardfork.h b/src/cryptonote_basic/hardfork.h index ee5ec0596bc..123978b12a0 100644 --- a/src/cryptonote_basic/hardfork.h +++ b/src/cryptonote_basic/hardfork.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -149,6 +149,16 @@ namespace cryptonote bool reorganize_from_block_height(uint64_t height); bool reorganize_from_chain_height(uint64_t height); + /** + * @brief called when one or more blocks are popped from the blockchain + * + * The current fork will be updated by looking up the db, + * which is much cheaper than recomputing everything + * + * @param new_chain_height the height of the chain after popping + */ + void on_block_popped(uint64_t new_chain_height); + /** * @brief returns current state at the given time * @@ -220,6 +230,14 @@ namespace cryptonote */ uint64_t get_window_size() const { return window_size; } + struct Params { + uint8_t version; + uint8_t threshold; + uint64_t height; + time_t time; + Params(uint8_t version, uint64_t height, uint8_t threshold, time_t time): version(version), threshold(threshold), height(height), time(time) {} + }; + private: uint8_t get_block_version(uint64_t height) const; @@ -244,13 +262,6 @@ namespace cryptonote uint8_t original_version; uint64_t original_version_till_height; - struct Params { - uint8_t version; - uint8_t threshold; - uint64_t height; - time_t time; - Params(uint8_t version, uint64_t height, uint8_t threshold, time_t time): version(version), threshold(threshold), height(height), time(time) {} - }; std::vector heights; std::deque versions; /* rolling window of the last N blocks' versions */ diff --git a/src/cryptonote_basic/miner.cpp b/src/cryptonote_basic/miner.cpp index 2c777f5a26d..e594eb0494c 100644 --- a/src/cryptonote_basic/miner.cpp +++ b/src/cryptonote_basic/miner.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -33,8 +33,6 @@ #include #include #include -#include -#include "include_base_utils.h" #include "misc_language.h" #include "syncobj.h" #include "cryptonote_basic_impl.h" @@ -54,24 +52,30 @@ #include #include #include -#endif - -#ifdef __FreeBSD__ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include +#elif defined(__linux__) + #include + #include + #include + #include +#elif defined(__FreeBSD__) + #include + #include + #include + #include + #include + #include + #include + #include + #include + #include #endif #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "miner" +#define AUTODETECT_WINDOW 10 // seconds +#define AUTODETECT_GAIN_THRESHOLD 1.02f // 2% + using namespace epee; #include "miner.h" @@ -102,11 +106,13 @@ namespace cryptonote m_thread_index(0), m_phandler(phandler), m_height(0), + m_threads_active(0), m_pausers_count(0), m_threads_total(0), m_starter_nonce(0), m_last_hr_merge_time(0), m_hashes(0), + m_total_hashes(0), m_do_print_hashrate(false), m_do_mining(false), m_current_hash_rate(0), @@ -114,22 +120,25 @@ namespace cryptonote m_min_idle_seconds(BACKGROUND_MINING_DEFAULT_MIN_IDLE_INTERVAL_IN_SECONDS), m_idle_threshold(BACKGROUND_MINING_DEFAULT_IDLE_THRESHOLD_PERCENTAGE), m_mining_target(BACKGROUND_MINING_DEFAULT_MINING_TARGET_PERCENTAGE), - m_miner_extra_sleep(BACKGROUND_MINING_DEFAULT_MINER_EXTRA_SLEEP_MILLIS) + m_miner_extra_sleep(BACKGROUND_MINING_DEFAULT_MINER_EXTRA_SLEEP_MILLIS), + m_block_reward(0) { - + m_attrs.set_stack_size(THREAD_STACK_SIZE); } //----------------------------------------------------------------------------------------------------- miner::~miner() { - stop(); + try { stop(); } + catch (...) { /* ignore */ } } //----------------------------------------------------------------------------------------------------- - bool miner::set_block_template(const block& bl, const difficulty_type& di, uint64_t height) + bool miner::set_block_template(const block& bl, const difficulty_type& di, uint64_t height, uint64_t block_reward) { CRITICAL_REGION_LOCAL(m_template_lock); m_template = bl; m_diffic = di; m_height = height; + m_block_reward = block_reward; ++m_template_no; m_starter_nonce = crypto::rand(); return true; @@ -145,7 +154,7 @@ namespace cryptonote //----------------------------------------------------------------------------------------------------- bool miner::request_block_template() { - block bl = AUTO_VAL_INIT(bl); + block bl; difficulty_type di = AUTO_VAL_INIT(di); uint64_t height = AUTO_VAL_INIT(height); uint64_t expected_reward; //only used for RPC calls - could possibly be useful here too? @@ -161,7 +170,7 @@ namespace cryptonote LOG_ERROR("Failed to get_block_template(), stopping mining"); return false; } - set_block_template(bl, di, height); + set_block_template(bl, di, height, expected_reward); return true; } //----------------------------------------------------------------------------------------------------- @@ -176,7 +185,12 @@ namespace cryptonote merge_hr(); return true; }); - + + m_autodetect_interval.do_call([&](){ + update_autodetection(); + return true; + }); + return true; } //----------------------------------------------------------------------------------------------------- @@ -198,14 +212,69 @@ namespace cryptonote { uint64_t total_hr = std::accumulate(m_last_hash_rates.begin(), m_last_hash_rates.end(), 0); float hr = static_cast(total_hr)/static_cast(m_last_hash_rates.size()); + const auto flags = std::cout.flags(); const auto precision = std::cout.precision(); - std::cout << "hashrate: " << std::setprecision(4) << std::fixed << hr << precision << ENDL; + std::cout << "hashrate: " << std::setprecision(4) << std::fixed << hr << std::setiosflags(flags) << std::setprecision(precision) << ENDL; } } m_last_hr_merge_time = misc_utils::get_tick_count(); m_hashes = 0; } //----------------------------------------------------------------------------------------------------- + void miner::update_autodetection() + { + if (m_threads_autodetect.empty()) + return; + + uint64_t now = epee::misc_utils::get_ns_count(); + uint64_t dt = now - m_threads_autodetect.back().first; + if (dt < AUTODETECT_WINDOW * 1000000000ull) + return; + + // work out how many more hashes we got + m_threads_autodetect.back().first = dt; + uint64_t dh = m_total_hashes - m_threads_autodetect.back().second; + m_threads_autodetect.back().second = dh; + float hs = dh / (dt / (float)1000000000); + MGINFO("Mining autodetection: " << m_threads_autodetect.size() << " threads: " << hs << " H/s"); + + // when we don't increase by at least 2%, stop, otherwise check next + // if N and N+1 have mostly the same hash rate, we want to "lighter" one + bool found = false; + if (m_threads_autodetect.size() > 1) + { + int previdx = m_threads_autodetect.size() - 2; + float previous_hs = m_threads_autodetect[previdx].second / (m_threads_autodetect[previdx].first / (float)1000000000); + if (previous_hs > 0 && hs / previous_hs < AUTODETECT_GAIN_THRESHOLD) + { + m_threads_total = m_threads_autodetect.size() - 1; + m_threads_autodetect.clear(); + MGINFO("Optimal number of threads seems to be " << m_threads_total); + found = true; + } + } + + if (!found) + { + // setup one more thread + m_threads_autodetect.push_back({now, m_total_hashes}); + m_threads_total = m_threads_autodetect.size(); + } + + // restart all threads + { + CRITICAL_REGION_LOCAL(m_threads_lock); + boost::interprocess::ipcdetail::atomic_write32(&m_stop, 1); + while (m_threads_active > 0) + misc_utils::sleep_no_w(100); + m_threads.clear(); + } + boost::interprocess::ipcdetail::atomic_write32(&m_stop, 0); + boost::interprocess::ipcdetail::atomic_write32(&m_thread_index, 0); + for(size_t i = 0; i != m_threads_total; i++) + m_threads.push_back(boost::thread(m_attrs, boost::bind(&miner::worker_thread, this))); + } + //----------------------------------------------------------------------------------------------------- void miner::init_options(boost::program_options::options_description& desc) { command_line::add_arg(desc, arg_extra_messages); @@ -239,7 +308,8 @@ namespace cryptonote } m_config_folder_path = boost::filesystem::path(command_line::get_arg(vm, arg_extra_messages)).parent_path().string(); m_config = AUTO_VAL_INIT(m_config); - epee::serialization::load_t_from_json_file(m_config, m_config_folder_path + "/" + MINER_CONFIG_FILE_NAME); + const std::string filename = m_config_folder_path + "/" + MINER_CONFIG_FILE_NAME; + CHECK_AND_ASSERT_MES(epee::serialization::load_t_from_json_file(m_config, filename), false, "Failed to load data from " << filename); MINFO("Loaded " << m_extra_messages.size() << " extra messages, current index " << m_config.current_extra_message_index); } @@ -290,10 +360,17 @@ namespace cryptonote return m_threads_total; } //----------------------------------------------------------------------------------------------------- - bool miner::start(const account_public_address& adr, size_t threads_count, const boost::thread::attributes& attrs, bool do_background, bool ignore_battery) + bool miner::start(const account_public_address& adr, size_t threads_count, bool do_background, bool ignore_battery) { + m_block_reward = 0; m_mine_address = adr; m_threads_total = static_cast(threads_count); + if (threads_count == 0) + { + m_threads_autodetect.clear(); + m_threads_autodetect.push_back({epee::misc_utils::get_ns_count(), m_total_hashes}); + m_threads_total = 1; + } m_starter_nonce = crypto::rand(); CRITICAL_REGION_LOCAL(m_threads_lock); if(is_mining()) @@ -315,19 +392,27 @@ namespace cryptonote set_is_background_mining_enabled(do_background); set_ignore_battery(ignore_battery); - for(size_t i = 0; i != threads_count; i++) + for(size_t i = 0; i != m_threads_total; i++) { - m_threads.push_back(boost::thread(attrs, boost::bind(&miner::worker_thread, this))); + m_threads.push_back(boost::thread(m_attrs, boost::bind(&miner::worker_thread, this))); } - LOG_PRINT_L0("Mining has started with " << threads_count << " threads, good luck!" ); + if (threads_count == 0) + MINFO("Mining has started, autodetecting optimal number of threads, good luck!" ); + else + MINFO("Mining has started with " << threads_count << " threads, good luck!" ); if( get_is_background_mining_enabled() ) { - m_background_mining_thread = boost::thread(attrs, boost::bind(&miner::background_worker_thread, this)); + m_background_mining_thread = boost::thread(m_attrs, boost::bind(&miner::background_worker_thread, this)); LOG_PRINT_L0("Background mining controller thread started" ); } + if(get_ignore_battery()) + { + MINFO("Ignoring battery"); + } + return true; } //----------------------------------------------------------------------------------------------------- @@ -350,29 +435,33 @@ namespace cryptonote { MTRACE("Miner has received stop signal"); - if (!is_mining()) + CRITICAL_REGION_LOCAL(m_threads_lock); + bool mining = !m_threads.empty(); + if (!mining) { - MDEBUG("Not mining - nothing to stop" ); + MTRACE("Not mining - nothing to stop" ); return true; } send_stop_signal(); - CRITICAL_REGION_LOCAL(m_threads_lock); // In case background mining was active and the miner threads are waiting // on the background miner to signal start. - m_is_background_mining_started_cond.notify_all(); - - for(boost::thread& th: m_threads) - th.join(); + while (m_threads_active > 0) + { + m_is_background_mining_started_cond.notify_all(); + misc_utils::sleep_no_w(100); + } // The background mining thread could be sleeping for a long time, so we // interrupt it just in case m_background_mining_thread.interrupt(); m_background_mining_thread.join(); + m_is_background_mining_enabled = false; MINFO("Mining has been stopped, " << m_threads.size() << " finished" ); m_threads.clear(); + m_threads_autodetect.clear(); return true; } //----------------------------------------------------------------------------------------------------- @@ -397,10 +486,7 @@ namespace cryptonote { if(m_do_mining) { - boost::thread::attributes attrs; - attrs.set_stack_size(THREAD_STACK_SIZE); - - start(m_mine_address, m_threads_total, attrs, get_is_background_mining_enabled(), get_ignore_battery()); + start(m_mine_address, m_threads_total, get_is_background_mining_enabled(), get_ignore_battery()); } } //----------------------------------------------------------------------------------------------------- @@ -438,6 +524,7 @@ namespace cryptonote uint32_t local_template_ver = 0; block b; slow_hash_allocate_state(); + ++m_threads_active; while(!m_stop) { if(m_pausers_count)//anti split workaround @@ -485,8 +572,9 @@ namespace cryptonote { //we lucky! ++m_config.current_extra_message_index; - MGINFO_GREEN("Found block for difficulty: " << local_diff); - if(!m_phandler->handle_block_found(b)) + MGINFO_GREEN("Found block " << get_block_hash(b) << " at height " << height << " for difficulty: " << local_diff); + cryptonote::block_verification_context bvc; + if(!m_phandler->handle_block_found(b, bvc) || !bvc.m_added_to_main_chain) { --m_config.current_extra_message_index; }else @@ -498,9 +586,11 @@ namespace cryptonote } nonce+=m_threads_total; ++m_hashes; + ++m_total_hashes; } slow_hash_free_state(); MGINFO("Miner thread stopped ["<< th_local_index << "]"); + --m_threads_active; return true; } //----------------------------------------------------------------------------------------------------- @@ -630,7 +720,7 @@ namespace cryptonote boost::tribool battery_powered(on_battery_power()); if(!indeterminate( battery_powered )) { - on_ac_power = !battery_powered; + on_ac_power = !(bool)battery_powered; } } @@ -659,10 +749,10 @@ namespace cryptonote uint8_t idle_percentage = get_percent_of_total(idle_diff, total_diff); uint8_t process_percentage = get_percent_of_total(process_diff, total_diff); - MGINFO("idle percentage is " << unsigned(idle_percentage) << "\%, miner percentage is " << unsigned(process_percentage) << "\%, ac power : " << on_ac_power); + MDEBUG("idle percentage is " << unsigned(idle_percentage) << "\%, miner percentage is " << unsigned(process_percentage) << "\%, ac power : " << on_ac_power); if( idle_percentage + process_percentage < get_idle_threshold() || !on_ac_power ) { - MGINFO("cpu is " << unsigned(idle_percentage) << "% idle, idle threshold is " << unsigned(get_idle_threshold()) << "\%, ac power : " << on_ac_power << ", background mining stopping, thanks for your contribution!"); + MINFO("cpu is " << unsigned(idle_percentage) << "% idle, idle threshold is " << unsigned(get_idle_threshold()) << "\%, ac power : " << on_ac_power << ", background mining stopping, thanks for your contribution!"); m_is_background_mining_started = false; // reset process times @@ -700,10 +790,10 @@ namespace cryptonote uint64_t idle_diff = (current_idle_time - prev_idle_time); uint8_t idle_percentage = get_percent_of_total(idle_diff, total_diff); - MGINFO("idle percentage is " << unsigned(idle_percentage)); + MDEBUG("idle percentage is " << unsigned(idle_percentage)); if( idle_percentage >= get_idle_threshold() && on_ac_power ) { - MGINFO("cpu is " << unsigned(idle_percentage) << "% idle, idle threshold is " << unsigned(get_idle_threshold()) << "\%, ac power : " << on_ac_power << ", background mining started, good luck!"); + MINFO("cpu is " << unsigned(idle_percentage) << "% idle, idle threshold is " << unsigned(get_idle_threshold()) << "\%, ac power : " << on_ac_power << ", background mining started, good luck!"); m_is_background_mining_started = true; m_is_background_mining_started_cond.notify_all(); @@ -961,7 +1051,12 @@ namespace cryptonote if (boost::logic::indeterminate(on_battery)) { - LOG_ERROR("couldn't query power status from " << power_supply_class_path); + static bool error_shown = false; + if (!error_shown) + { + LOG_ERROR("couldn't query power status from " << power_supply_class_path); + error_shown = true; + } } return on_battery; diff --git a/src/cryptonote_basic/miner.h b/src/cryptonote_basic/miner.h index 2bff784c7ed..ac7a0381c58 100644 --- a/src/cryptonote_basic/miner.h +++ b/src/cryptonote_basic/miner.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -34,15 +34,11 @@ #include #include #include "cryptonote_basic.h" +#include "verification_context.h" #include "difficulty.h" #include "math_helper.h" #ifdef _WIN32 #include -#elif defined(__linux__) -#include -#include -#include -#include #endif namespace cryptonote @@ -50,7 +46,7 @@ namespace cryptonote struct i_miner_handler { - virtual bool handle_block_found(block& b) = 0; + virtual bool handle_block_found(block& b, block_verification_context &bvc) = 0; virtual bool get_block_template(block& b, const account_public_address& adr, difficulty_type& diffic, uint64_t& height, uint64_t& expected_reward, const blobdata& ex_nonce) = 0; protected: ~i_miner_handler(){}; @@ -66,9 +62,9 @@ namespace cryptonote ~miner(); bool init(const boost::program_options::variables_map& vm, network_type nettype); static void init_options(boost::program_options::options_description& desc); - bool set_block_template(const block& bl, const difficulty_type& diffic, uint64_t height); + bool set_block_template(const block& bl, const difficulty_type& diffic, uint64_t height, uint64_t block_reward); bool on_block_chain_update(); - bool start(const account_public_address& adr, size_t threads_count, const boost::thread::attributes& attrs, bool do_background = false, bool ignore_battery = false); + bool start(const account_public_address& adr, size_t threads_count, bool do_background = false, bool ignore_battery = false); uint64_t get_speed() const; uint32_t get_threads_count() const; void send_stop_signal(); @@ -90,6 +86,7 @@ namespace cryptonote bool set_idle_threshold(uint8_t idle_threshold); uint8_t get_mining_target() const; bool set_mining_target(uint8_t mining_target); + uint64_t get_block_reward() const { return m_block_reward; } static constexpr uint8_t BACKGROUND_MINING_DEFAULT_IDLE_THRESHOLD_PERCENTAGE = 90; static constexpr uint8_t BACKGROUND_MINING_MIN_IDLE_THRESHOLD_PERCENTAGE = 50; @@ -108,6 +105,7 @@ namespace cryptonote bool worker_thread(); bool request_block_template(); void merge_hr(); + void update_autodetection(); struct miner_config { @@ -128,6 +126,7 @@ namespace cryptonote uint64_t m_height; volatile uint32_t m_thread_index; volatile uint32_t m_threads_total; + std::atomic m_threads_active; std::atomic m_pausers_count; epee::critical_section m_miners_count_lock; @@ -137,16 +136,20 @@ namespace cryptonote account_public_address m_mine_address; epee::math_helper::once_a_time_seconds<5> m_update_block_template_interval; epee::math_helper::once_a_time_seconds<2> m_update_merge_hr_interval; + epee::math_helper::once_a_time_seconds<1> m_autodetect_interval; std::vector m_extra_messages; miner_config m_config; std::string m_config_folder_path; std::atomic m_last_hr_merge_time; std::atomic m_hashes; + std::atomic m_total_hashes; std::atomic m_current_hash_rate; epee::critical_section m_last_hash_rates_lock; std::list m_last_hash_rates; bool m_do_print_hashrate; bool m_do_mining; + std::vector> m_threads_autodetect; + boost::thread::attributes m_attrs; // background mining stuffs .. @@ -169,5 +172,6 @@ namespace cryptonote static bool get_process_time(uint64_t& total_time); static uint8_t get_percent_of_total(uint64_t some_time, uint64_t total_time); static boost::logic::tribool on_battery_power(); + std::atomic m_block_reward; }; } diff --git a/src/cryptonote_basic/subaddress_index.h b/src/cryptonote_basic/subaddress_index.h index 9b71448f963..99933e22915 100644 --- a/src/cryptonote_basic/subaddress_index.h +++ b/src/cryptonote_basic/subaddress_index.h @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2018, The Monero Project +// Copyright (c) 2017-2019, The Monero Project // // All rights reserved. // diff --git a/src/cryptonote_basic/tx_extra.h b/src/cryptonote_basic/tx_extra.h index 009e35ebe99..ecb4c604086 100644 --- a/src/cryptonote_basic/tx_extra.h +++ b/src/cryptonote_basic/tx_extra.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/cryptonote_basic/verification_context.h b/src/cryptonote_basic/verification_context.h index 8d2b633a22d..36b63f254b4 100644 --- a/src/cryptonote_basic/verification_context.h +++ b/src/cryptonote_basic/verification_context.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/cryptonote_config.h b/src/cryptonote_config.h index 3e64073dd46..56b6a63b7ff 100644 --- a/src/cryptonote_config.h +++ b/src/cryptonote_config.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -30,15 +30,16 @@ #pragma once +#include #include #include #define CRYPTONOTE_DNS_TIMEOUT_MS 20000 #define CRYPTONOTE_MAX_BLOCK_NUMBER 500000000 -#define CRYPTONOTE_MAX_BLOCK_SIZE 500000000 // block header blob limit, never used! #define CRYPTONOTE_GETBLOCKTEMPLATE_MAX_BLOCK_SIZE 196608 //size of block (bytes) that is the maximum that miners will produce -#define CRYPTONOTE_MAX_TX_SIZE 1000000000 +#define CRYPTONOTE_MAX_TX_SIZE 1000000 +#define CRYPTONOTE_MAX_TX_PER_BLOCK 0x10000000 #define CRYPTONOTE_PUBLIC_ADDRESS_TEXTBLOB_VER 0 #define CRYPTONOTE_MINED_MONEY_UNLOCK_WINDOW 60 #define CURRENT_TRANSACTION_VERSION 2 @@ -58,6 +59,8 @@ #define CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V2 60000 //size of block (bytes) after which reward for block calculated using block size #define CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V1 20000 //size of block (bytes) after which reward for block calculated using block size - before first fork #define CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V5 300000 //size of block (bytes) after which reward for block calculated using block size - second change, from v5 +#define CRYPTONOTE_LONG_TERM_BLOCK_WEIGHT_WINDOW_SIZE 100000 // size in blocks of the long term block weight median window +#define CRYPTONOTE_SHORT_TERM_BLOCK_WEIGHT_SURGE_FACTOR 50 #define CRYPTONOTE_COINBASE_BLOB_RESERVED_SIZE 600 #define CRYPTONOTE_DISPLAY_DECIMAL_POINT 12 // COIN - number of smallest units in one coin @@ -65,9 +68,11 @@ #define FEE_PER_KB_OLD ((uint64_t)10000000000) // pow(10, 10) #define FEE_PER_KB ((uint64_t)2000000000) // 2 * pow(10, 9) +#define FEE_PER_BYTE ((uint64_t)300000) #define DYNAMIC_FEE_PER_KB_BASE_FEE ((uint64_t)2000000000) // 2 * pow(10,9) #define DYNAMIC_FEE_PER_KB_BASE_BLOCK_REWARD ((uint64_t)10000000000000) // 10 * pow(10,12) #define DYNAMIC_FEE_PER_KB_BASE_FEE_V5 ((uint64_t)2000000000 * (uint64_t)CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V2 / CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V5) +#define DYNAMIC_FEE_REFERENCE_TRANSACTION_WEIGHT ((uint64_t)3000) #define ORPHANED_BLOCKS_MAX_COUNT 100 @@ -105,11 +110,15 @@ #define P2P_DEFAULT_PACKET_MAX_SIZE 50000000 //50000000 bytes maximum packet size #define P2P_DEFAULT_PEERS_IN_HANDSHAKE 250 #define P2P_DEFAULT_CONNECTION_TIMEOUT 5000 //5 seconds +#define P2P_DEFAULT_SOCKS_CONNECT_TIMEOUT 45 // seconds #define P2P_DEFAULT_PING_CONNECTION_TIMEOUT 2000 //2 seconds #define P2P_DEFAULT_INVOKE_TIMEOUT 60*2*1000 //2 minutes #define P2P_DEFAULT_HANDSHAKE_INVOKE_TIMEOUT 5000 //5 seconds #define P2P_DEFAULT_WHITELIST_CONNECTIONS_PERCENT 70 #define P2P_DEFAULT_ANCHOR_CONNECTIONS_COUNT 2 +#define P2P_DEFAULT_SYNC_SEARCH_CONNECTIONS_COUNT 2 +#define P2P_DEFAULT_LIMIT_RATE_UP 2048 // kB/s +#define P2P_DEFAULT_LIMIT_RATE_DOWN 8192 // kB/s #define P2P_FAILED_ADDR_FORGET_SECONDS (60*60) //1 hour #define P2P_IP_BLOCKTIME (60*60*24) //24 hour @@ -133,13 +142,24 @@ #define HF_VERSION_DYNAMIC_FEE 4 #define HF_VERSION_MIN_MIXIN_4 6 #define HF_VERSION_MIN_MIXIN_6 7 +#define HF_VERSION_MIN_MIXIN_10 8 #define HF_VERSION_ENFORCE_RCT 6 +#define HF_VERSION_PER_BYTE_FEE 8 +#define HF_VERSION_SMALLER_BP 10 +#define HF_VERSION_LONG_TERM_BLOCK_WEIGHT 10 #define PER_KB_FEE_QUANTIZATION_DECIMALS 8 #define HASH_OF_HASHES_STEP 256 -#define DEFAULT_TXPOOL_MAX_SIZE 648000000ull // 3 days at 300000, in bytes +#define DEFAULT_TXPOOL_MAX_WEIGHT 648000000ull // 3 days at 300000, in bytes + +#define BULLETPROOF_MAX_OUTPUTS 16 + +#define CRYPTONOTE_PRUNING_STRIPE_SIZE 4096 // the smaller, the smoother the increase +#define CRYPTONOTE_PRUNING_LOG_STRIPES 3 // the higher, the more space saved +#define CRYPTONOTE_PRUNING_TIP_BLOCKS 5500 // the smaller, the more space saved +//#define CRYPTONOTE_PRUNING_DEBUG_SPOOF_SEED // New constants are intended to go here namespace config @@ -203,4 +223,60 @@ namespace cryptonote FAKECHAIN, UNDEFINED = 255 }; + struct config_t + { + uint64_t const CRYPTONOTE_PUBLIC_ADDRESS_BASE58_PREFIX; + uint64_t const CRYPTONOTE_PUBLIC_INTEGRATED_ADDRESS_BASE58_PREFIX; + uint64_t const CRYPTONOTE_PUBLIC_SUBADDRESS_BASE58_PREFIX; + uint16_t const P2P_DEFAULT_PORT; + uint16_t const RPC_DEFAULT_PORT; + uint16_t const ZMQ_RPC_DEFAULT_PORT; + boost::uuids::uuid const NETWORK_ID; + std::string const GENESIS_TX; + uint32_t const GENESIS_NONCE; + }; + inline const config_t& get_config(network_type nettype) + { + static const config_t mainnet = { + ::config::CRYPTONOTE_PUBLIC_ADDRESS_BASE58_PREFIX, + ::config::CRYPTONOTE_PUBLIC_INTEGRATED_ADDRESS_BASE58_PREFIX, + ::config::CRYPTONOTE_PUBLIC_SUBADDRESS_BASE58_PREFIX, + ::config::P2P_DEFAULT_PORT, + ::config::RPC_DEFAULT_PORT, + ::config::ZMQ_RPC_DEFAULT_PORT, + ::config::NETWORK_ID, + ::config::GENESIS_TX, + ::config::GENESIS_NONCE + }; + static const config_t testnet = { + ::config::testnet::CRYPTONOTE_PUBLIC_ADDRESS_BASE58_PREFIX, + ::config::testnet::CRYPTONOTE_PUBLIC_INTEGRATED_ADDRESS_BASE58_PREFIX, + ::config::testnet::CRYPTONOTE_PUBLIC_SUBADDRESS_BASE58_PREFIX, + ::config::testnet::P2P_DEFAULT_PORT, + ::config::testnet::RPC_DEFAULT_PORT, + ::config::testnet::ZMQ_RPC_DEFAULT_PORT, + ::config::testnet::NETWORK_ID, + ::config::testnet::GENESIS_TX, + ::config::testnet::GENESIS_NONCE + }; + static const config_t stagenet = { + ::config::stagenet::CRYPTONOTE_PUBLIC_ADDRESS_BASE58_PREFIX, + ::config::stagenet::CRYPTONOTE_PUBLIC_INTEGRATED_ADDRESS_BASE58_PREFIX, + ::config::stagenet::CRYPTONOTE_PUBLIC_SUBADDRESS_BASE58_PREFIX, + ::config::stagenet::P2P_DEFAULT_PORT, + ::config::stagenet::RPC_DEFAULT_PORT, + ::config::stagenet::ZMQ_RPC_DEFAULT_PORT, + ::config::stagenet::NETWORK_ID, + ::config::stagenet::GENESIS_TX, + ::config::stagenet::GENESIS_NONCE + }; + switch (nettype) + { + case MAINNET: return mainnet; + case TESTNET: return testnet; + case STAGENET: return stagenet; + case FAKECHAIN: return mainnet; + default: throw std::runtime_error("Invalid network type"); + } + }; } diff --git a/src/cryptonote_core/CMakeLists.txt b/src/cryptonote_core/CMakeLists.txt index 72844db66e0..2cbe89b01d3 100644 --- a/src/cryptonote_core/CMakeLists.txt +++ b/src/cryptonote_core/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2014-2018, The Monero Project +# Copyright (c) 2014-2019, The Monero Project # # All rights reserved. # @@ -30,6 +30,7 @@ set(cryptonote_core_sources blockchain.cpp cryptonote_core.cpp tx_pool.cpp + tx_sanity_check.cpp cryptonote_tx_utils.cpp) set(cryptonote_core_headers) @@ -39,14 +40,9 @@ set(cryptonote_core_private_headers blockchain.h cryptonote_core.h tx_pool.h + tx_sanity_check.h cryptonote_tx_utils.h) -if(PER_BLOCK_CHECKPOINT) - set(Blocks "blocks") -else() - set(Blocks "") -endif() - monero_private_headers(cryptonote_core ${cryptonote_core_private_headers}) monero_add_library(cryptonote_core @@ -69,5 +65,4 @@ target_link_libraries(cryptonote_core ${Boost_SYSTEM_LIBRARY} ${Boost_THREAD_LIBRARY} PRIVATE - ${Blocks} ${EXTRA_LIBRARIES}) diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp index 66e7acff8c9..ac2c079137a 100644 --- a/src/cryptonote_core/blockchain.cpp +++ b/src/cryptonote_core/blockchain.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -44,7 +44,7 @@ #include "misc_language.h" #include "profile_tools.h" #include "file_io_utils.h" -#include "common/int-util.h" +#include "int-util.h" #include "common/threadpool.h" #include "common/boost_serialization_helper.h" #include "warnings.h" @@ -52,9 +52,9 @@ #include "cryptonote_core.h" #include "ringct/rctSigs.h" #include "common/perf_timer.h" -#if defined(PER_BLOCK_CHECKPOINT) -#include "blocks/blocks.h" -#endif +#include "common/notify.h" +#include "common/varint.h" +#include "common/pruning.h" #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "blockchain" @@ -109,6 +109,18 @@ static const struct { // version 7 starts from block 1546000, which is on or around the 6th of April, 2018. Fork time finalised on 2018-03-17. { 7, 1546000, 0, 1521303150 }, + + // version 8 starts from block 1685555, which is on or around the 18th of October, 2018. Fork time finalised on 2018-09-02. + { 8, 1685555, 0, 1535889547 }, + + // version 9 starts from block 1686275, which is on or around the 19th of October, 2018. Fork time finalised on 2018-09-02. + { 9, 1686275, 0, 1535889548 }, + + // version 10 starts from block 1788000, which is on or around the 9th of March, 2019. Fork time finalised on 2019-02-10. + { 10, 1788000, 0, 1549792439 }, + + // version 11 starts from block 1788720, which is on or around the 10th of March, 2019. Fork time finalised on 2019-02-15. + { 11, 1788720, 0, 1550225678 }, }; static const uint64_t mainnet_hard_fork_version_1_till = 1009826; @@ -131,7 +143,10 @@ static const struct { { 6, 971400, 0, 1501709789 }, { 7, 1057027, 0, 1512211236 }, - { 8, 1057058, 0, 1515967497 }, + { 8, 1057058, 0, 1533211200 }, + { 9, 1057778, 0, 1533297600 }, + { 10, 1154318, 0, 1550153694 }, + { 11, 1155038, 0, 1550225678 }, }; static const uint64_t testnet_hard_fork_version_1_till = 624633; @@ -151,18 +166,34 @@ static const struct { { 5, 35000, 0, 1521360000 }, { 6, 36000, 0, 1521480000 }, { 7, 37000, 0, 1521600000 }, + { 8, 176456, 0, 1537821770 }, + { 9, 177176, 0, 1537821771 }, + { 10, 269000, 0, 1550153694 }, + { 11, 269720, 0, 1550225678 }, }; //------------------------------------------------------------------ Blockchain::Blockchain(tx_memory_pool& tx_pool) : - m_db(), m_tx_pool(tx_pool), m_hardfork(NULL), m_timestamps_and_difficulties_height(0), m_current_block_cumul_sz_limit(0), m_current_block_cumul_sz_median(0), - m_enforce_dns_checkpoints(false), m_max_prepare_blocks_threads(4), m_db_blocks_per_sync(1), m_db_sync_mode(db_async), m_db_default_sync(false), m_fast_sync(true), m_show_time_stats(false), m_sync_counter(0), m_cancel(false), + m_db(), m_tx_pool(tx_pool), m_hardfork(NULL), m_timestamps_and_difficulties_height(0), m_current_block_cumul_weight_limit(0), m_current_block_cumul_weight_median(0), + m_enforce_dns_checkpoints(false), m_max_prepare_blocks_threads(4), m_db_sync_on_blocks(true), m_db_sync_threshold(1), m_db_sync_mode(db_async), m_db_default_sync(false), m_fast_sync(true), m_show_time_stats(false), m_sync_counter(0), m_bytes_to_sync(0), m_cancel(false), + m_long_term_block_weights_window(CRYPTONOTE_LONG_TERM_BLOCK_WEIGHT_WINDOW_SIZE), + m_long_term_effective_median_block_weight(0), + m_long_term_block_weights_cache_tip_hash(crypto::null_hash), + m_long_term_block_weights_cache_rolling_median(CRYPTONOTE_LONG_TERM_BLOCK_WEIGHT_WINDOW_SIZE), m_difficulty_for_next_block_top_hash(crypto::null_hash), - m_difficulty_for_next_block(1) + m_difficulty_for_next_block(1), + m_btc_valid(false), + m_batch_success(true) { LOG_PRINT_L3("Blockchain::" << __func__); } //------------------------------------------------------------------ +Blockchain::~Blockchain() +{ + try { deinit(); } + catch (const std::exception &e) { /* ignore */ } +} +//------------------------------------------------------------------ bool Blockchain::have_tx(const crypto::hash &id) const { LOG_PRINT_L3("Blockchain::" << __func__); @@ -221,7 +252,7 @@ bool Blockchain::scan_outputkeys_for_indexes(size_t tx_version, const txin_to_ke { try { - m_db->get_output_key(tx_in_to_key.amount, absolute_offsets, outputs, true); + m_db->get_output_key(epee::span(&tx_in_to_key.amount, 1), absolute_offsets, outputs, true); if (absolute_offsets.size() != outputs.size()) { MERROR_VER("Output does not exist! amount = " << tx_in_to_key.amount); @@ -242,11 +273,12 @@ bool Blockchain::scan_outputkeys_for_indexes(size_t tx_version, const txin_to_ke MDEBUG("Additional outputs needed: " << absolute_offsets.size() - outputs.size()); std::vector < uint64_t > add_offsets; std::vector add_outputs; + add_outputs.reserve(absolute_offsets.size() - outputs.size()); for (size_t i = outputs.size(); i < absolute_offsets.size(); i++) add_offsets.push_back(absolute_offsets[i]); try { - m_db->get_output_key(tx_in_to_key.amount, add_offsets, add_outputs, true); + m_db->get_output_key(epee::span(&tx_in_to_key.amount, 1), add_offsets, add_outputs, true); if (add_offsets.size() != add_outputs.size()) { MERROR_VER("Output does not exist! amount = " << tx_in_to_key.amount); @@ -329,9 +361,12 @@ uint64_t Blockchain::get_current_blockchain_height() const //------------------------------------------------------------------ //FIXME: possibly move this into the constructor, to avoid accidentally // dereferencing a null BlockchainDB pointer -bool Blockchain::init(BlockchainDB* db, const network_type nettype, bool offline, const cryptonote::test_options *test_options) +bool Blockchain::init(BlockchainDB* db, const network_type nettype, bool offline, const cryptonote::test_options *test_options, difficulty_type fixed_difficulty, const GetCheckpointsCallback& get_checkpoints/* = nullptr*/) { LOG_PRINT_L3("Blockchain::" << __func__); + + CHECK_AND_ASSERT_MES(nettype != FAKECHAIN || test_options, false, "fake chain network type used without options"); + CRITICAL_REGION_LOCAL(m_tx_pool); CRITICAL_REGION_LOCAL1(m_blockchain_lock); @@ -351,6 +386,7 @@ bool Blockchain::init(BlockchainDB* db, const network_type nettype, bool offline m_nettype = test_options != NULL ? FAKECHAIN : nettype; m_offline = offline; + m_fixed_difficulty = fixed_difficulty; if (m_hardfork == nullptr) { if (m_nettype == FAKECHAIN || m_nettype == STAGENET) @@ -391,20 +427,10 @@ bool Blockchain::init(BlockchainDB* db, const network_type nettype, bool offline if(!m_db->height()) { MINFO("Blockchain not loaded, generating genesis block."); - block bl = boost::value_initialized(); + block bl; block_verification_context bvc = boost::value_initialized(); - if (m_nettype == TESTNET) - { - generate_genesis_block(bl, config::testnet::GENESIS_TX, config::testnet::GENESIS_NONCE); - } - else if (m_nettype == STAGENET) - { - generate_genesis_block(bl, config::stagenet::GENESIS_TX, config::stagenet::GENESIS_NONCE); - } - else - { - generate_genesis_block(bl, config::GENESIS_TX, config::GENESIS_NONCE); - } + generate_genesis_block(bl, get_config(m_nettype).GENESIS_TX, get_config(m_nettype).GENESIS_NONCE); + db_wtxn_guard wtxn_guard(m_db); add_new_block(bl, bvc); CHECK_AND_ASSERT_MES(!bvc.m_verifivation_failed, false, "Failed to add genesis block to blockchain"); } @@ -420,14 +446,15 @@ bool Blockchain::init(BlockchainDB* db, const network_type nettype, bool offline m_db->fixup(); } - m_db->block_txn_start(true); + db_rtxn_guard rtxn_guard(m_db); + // check how far behind we are uint64_t top_block_timestamp = m_db->get_top_block_timestamp(); uint64_t timestamp_diff = time(NULL) - top_block_timestamp; - // genesis block has no timestamp, could probably change it to have timestamp of 1341378000... + // genesis block has no timestamp, could probably change it to have timestamp of 1397818133... if(!top_block_timestamp) - timestamp_diff = time(NULL) - 1341378000; + timestamp_diff = time(NULL) - 1397818133; // create general purpose async service queue @@ -437,17 +464,18 @@ bool Blockchain::init(BlockchainDB* db, const network_type nettype, bool offline #if defined(PER_BLOCK_CHECKPOINT) if (m_nettype != FAKECHAIN) - load_compiled_in_block_hashes(); + load_compiled_in_block_hashes(get_checkpoints); #endif MINFO("Blockchain initialized. last block: " << m_db->height() - 1 << ", " << epee::misc_utils::get_time_interval_string(timestamp_diff) << " time ago, current difficulty: " << get_difficulty_for_next_block()); - m_db->block_txn_stop(); + + rtxn_guard.stop(); uint64_t num_popped_blocks = 0; - while (true) + while (!m_db->is_read_only()) { - const uint64_t top_height = m_db->height() - 1; - const crypto::hash top_id = m_db->top_block_hash(); + uint64_t top_height; + const crypto::hash top_id = m_db->top_block_hash(&top_height); const block top_block = m_db->get_top_block(); const uint8_t ideal_hf_version = get_ideal_hard_fork_version(top_height); if (ideal_hf_version <= 1 || ideal_hf_version == top_block.major_version) @@ -487,10 +515,22 @@ bool Blockchain::init(BlockchainDB* db, const network_type nettype, bool offline { m_timestamps_and_difficulties_height = 0; m_hardfork->reorganize_from_chain_height(get_current_blockchain_height()); - m_tx_pool.on_blockchain_dec(m_db->height()-1, get_tail_id()); + uint64_t top_block_height; + crypto::hash top_block_hash = get_tail_id(top_block_height); + m_tx_pool.on_blockchain_dec(top_block_height, top_block_hash); + } + + if (test_options && test_options->long_term_block_weight_window) + { + m_long_term_block_weights_window = test_options->long_term_block_weight_window; + m_long_term_block_weights_cache_rolling_median = epee::misc_utils::rolling_median_t(m_long_term_block_weights_window); } - update_next_cumulative_size_limit(); + { + db_txn_guard txn_guard(m_db, m_db->is_read_only()); + if (!update_next_cumulative_weight_limit()) + return false; + } return true; } //------------------------------------------------------------------ @@ -548,15 +588,13 @@ bool Blockchain::deinit() // as this should be called if handling a SIGSEGV, need to check // if m_db is a NULL pointer (and thus may have caused the illegal // memory operation), otherwise we may cause a loop. - if (m_db == NULL) - { - throw DB_ERROR("The db pointer is null in Blockchain, the blockchain may be corrupt!"); - } - try { - m_db->close(); - MTRACE("Local blockchain read/write activity stopped successfully"); + if (m_db) + { + m_db->close(); + MTRACE("Local blockchain read/write activity stopped successfully"); + } } catch (const std::exception& e) { @@ -574,6 +612,38 @@ bool Blockchain::deinit() return true; } //------------------------------------------------------------------ +// This function removes blocks from the top of blockchain. +// It starts a batch and calls private method pop_block_from_blockchain(). +void Blockchain::pop_blocks(uint64_t nblocks) +{ + uint64_t i; + CRITICAL_REGION_LOCAL(m_tx_pool); + CRITICAL_REGION_LOCAL1(m_blockchain_lock); + + bool stop_batch = m_db->batch_start(); + + try + { + const uint64_t blockchain_height = m_db->height(); + if (blockchain_height > 0) + nblocks = std::min(nblocks, blockchain_height - 1); + for (i=0; i < nblocks; ++i) + { + pop_block_from_blockchain(); + } + } + catch (const std::exception& e) + { + LOG_ERROR("Error when popping blocks after processing " << i << " blocks: " << e.what()); + if (stop_batch) + m_db->batch_abort(); + return; + } + + if (stop_batch) + m_db->batch_stop(); +} +//------------------------------------------------------------------ // This function tells BlockchainDB to remove the top block from the // blockchain and then returns all transactions (except the miner tx, of course) // from it to the tx_pool @@ -587,6 +657,8 @@ block Blockchain::pop_block_from_blockchain() block popped_block; std::vector popped_txs; + CHECK_AND_ASSERT_THROW_MES(m_db->height() > 1, "Cannot pop the genesis block"); + try { m_db->pop_block(popped_block, popped_txs); @@ -604,9 +676,18 @@ block Blockchain::pop_block_from_blockchain() throw; } + // make sure the hard fork object updates its current version + m_hardfork->on_block_popped(1); + // return transactions from popped block to the tx_pool + size_t pruned = 0; for (transaction& tx : popped_txs) { + if (tx.pruned) + { + ++pruned; + continue; + } if (!is_coinbase(tx)) { cryptonote::tx_verification_context tvc = AUTO_VAL_INIT(tvc); @@ -614,12 +695,7 @@ block Blockchain::pop_block_from_blockchain() // FIXME: HardFork // Besides the below, popping a block should also remove the last entry // in hf_versions. - // - // FIXME: HardFork - // This is not quite correct, as we really want to add the txes - // to the pool based on the version determined after all blocks - // are popped. - uint8_t version = get_current_hard_fork_version(); + uint8_t version = get_ideal_hard_fork_version(m_db->height()); // We assume that if they were in a block, the transactions are already // known to the network as a whole. However, if we had mined that block, @@ -633,14 +709,19 @@ block Blockchain::pop_block_from_blockchain() } } } + if (pruned) + MWARNING(pruned << " pruned txes could not be added back to the txpool"); m_blocks_longhash_table.clear(); m_scan_table.clear(); m_blocks_txs_check.clear(); m_check_txin_table.clear(); - update_next_cumulative_size_limit(); - m_tx_pool.on_blockchain_dec(m_db->height()-1, get_tail_id()); + CHECK_AND_ASSERT_THROW_MES(update_next_cumulative_weight_limit(), "Error updating next cumulative weight limit"); + uint64_t top_block_height; + crypto::hash top_block_hash = get_tail_id(top_block_height); + m_tx_pool.on_blockchain_dec(top_block_height, top_block_hash); + invalidate_block_template_cache(); return popped_block; } @@ -651,12 +732,15 @@ bool Blockchain::reset_and_set_genesis_block(const block& b) CRITICAL_REGION_LOCAL(m_blockchain_lock); m_timestamps_and_difficulties_height = 0; m_alternative_chains.clear(); + invalidate_block_template_cache(); m_db->reset(); m_hardfork->init(); + db_wtxn_guard wtxn_guard(m_db); block_verification_context bvc = boost::value_initialized(); add_new_block(b, bvc); - update_next_cumulative_size_limit(); + if (!update_next_cumulative_weight_limit()) + return false; return bvc.m_added_to_main_chain && !bvc.m_verifivation_failed; } //------------------------------------------------------------------ @@ -664,8 +748,7 @@ crypto::hash Blockchain::get_tail_id(uint64_t& height) const { LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); - height = m_db->height() - 1; - return get_tail_id(); + return m_db->top_block_hash(&height); } //------------------------------------------------------------------ crypto::hash Blockchain::get_tail_id() const @@ -701,7 +784,7 @@ bool Blockchain::get_short_chain_history(std::list& ids) const if(!sz) return true; - m_db->block_txn_start(true); + db_rtxn_guard rtxn_guard(m_db); bool genesis_included = false; uint64_t current_back_offset = 1; while(current_back_offset < sz) @@ -728,7 +811,6 @@ bool Blockchain::get_short_chain_history(std::list& ids) const { ids.push_back(m_db->get_block_hash_from_height(0)); } - m_db->block_txn_stop(); return true; } @@ -805,6 +887,11 @@ bool Blockchain::get_block_by_hash(const crypto::hash &h, block &blk, bool *orph // less blocks than desired if there aren't enough. difficulty_type Blockchain::get_difficulty_for_next_block() { + if (m_fixed_difficulty) + { + return m_db->height() ? m_fixed_difficulty : 1; + } + LOG_PRINT_L3("Blockchain::" << __func__); crypto::hash top_hash = get_tail_id(); @@ -821,13 +908,15 @@ difficulty_type Blockchain::get_difficulty_for_next_block() CRITICAL_REGION_LOCAL(m_blockchain_lock); std::vector timestamps; std::vector difficulties; - auto height = m_db->height(); + uint64_t height; + top_hash = get_tail_id(height); // get it again now that we have the lock + ++height; // top block height to blockchain height // ND: Speedup // 1. Keep a list of the last 735 (or less) blocks that is used to compute difficulty, // then when the next block difficulty is queried, push the latest height data and // pop the oldest one from the list. This only requires 1x read per height instead // of doing 735 (DIFFICULTY_BLOCKS_COUNT). - if (m_timestamps_and_difficulties_height != 0 && ((height - m_timestamps_and_difficulties_height) == 1)) + if (m_timestamps_and_difficulties_height != 0 && ((height - m_timestamps_and_difficulties_height) == 1) && m_timestamps.size() >= DIFFICULTY_BLOCKS_COUNT) { uint64_t index = height - 1; m_timestamps.push_back(m_db->get_block_timestamp(index)); @@ -844,12 +933,17 @@ difficulty_type Blockchain::get_difficulty_for_next_block() } else { - size_t offset = height - std::min < size_t > (height, static_cast(DIFFICULTY_BLOCKS_COUNT)); + uint64_t offset = height - std::min (height, static_cast(DIFFICULTY_BLOCKS_COUNT)); if (offset == 0) ++offset; timestamps.clear(); difficulties.clear(); + if (height > offset) + { + timestamps.reserve(height - offset); + difficulties.reserve(height - offset); + } for (; offset < height; offset++) { timestamps.push_back(m_db->get_block_timestamp(offset)); @@ -869,6 +963,17 @@ difficulty_type Blockchain::get_difficulty_for_next_block() return diff; } //------------------------------------------------------------------ +std::vector Blockchain::get_last_block_timestamps(unsigned int blocks) const +{ + uint64_t height = m_db->height(); + if (blocks > height) + blocks = height; + std::vector timestamps(blocks); + while (blocks--) + timestamps[blocks] = m_db->get_block_timestamp(height - blocks - 1); + return timestamps; +} +//------------------------------------------------------------------ // This function removes blocks from the blockchain until it gets to the // position where the blockchain switch started and then re-adds the blocks // that had been removed. @@ -905,7 +1010,7 @@ bool Blockchain::rollback_blockchain_switching(std::list& original_chain, m_hardfork->reorganize_from_chain_height(rollback_height); MINFO("Rollback to height " << rollback_height << " was successful."); - if (original_chain.size()) + if (!original_chain.empty()) { MINFO("Restoration to previous blockchain successful as well."); } @@ -914,7 +1019,7 @@ bool Blockchain::rollback_blockchain_switching(std::list& original_chain, //------------------------------------------------------------------ // This function attempts to switch to an alternate chain, returning // boolean based on success therein. -bool Blockchain::switch_to_alternative_blockchain(std::list& alt_chain, bool discard_disconnected_chain) +bool Blockchain::switch_to_alternative_blockchain(std::list& alt_chain, bool discard_disconnected_chain) { LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); @@ -979,6 +1084,7 @@ bool Blockchain::switch_to_alternative_blockchain(std::listreorganize_from_chain_height(split_height); + std::shared_ptr reorg_notify = m_reorg_notify; + if (reorg_notify) + reorg_notify->notify("%s", std::to_string(split_height).c_str(), "%h", std::to_string(m_db->height()).c_str(), + "%n", std::to_string(m_db->height() - split_height).c_str(), "%d", std::to_string(discarded_blocks).c_str(), NULL); + MGINFO_GREEN("REORGANIZE SUCCESS! on height: " << split_height << ", new blockchain size: " << m_db->height()); return true; } //------------------------------------------------------------------ // This function calculates the difficulty target for the block being added to // an alternate chain. -difficulty_type Blockchain::get_next_difficulty_for_alternative_chain(const std::list& alt_chain, block_extended_info& bei) const +difficulty_type Blockchain::get_next_difficulty_for_alternative_chain(const std::list& alt_chain, block_extended_info& bei) const { + if (m_fixed_difficulty) + { + return m_db->height() ? m_fixed_difficulty : 1; + } + LOG_PRINT_L3("Blockchain::" << __func__); std::vector timestamps; std::vector cumulative_difficulties; @@ -1104,7 +1220,7 @@ bool Blockchain::prevalidate_miner_transaction(const block& b, uint64_t height) } //------------------------------------------------------------------ // This function validates the miner transaction reward -bool Blockchain::validate_miner_transaction(const block& b, size_t cumulative_block_size, uint64_t fee, uint64_t& base_reward, uint64_t already_generated_coins, bool &partial_block_reward, uint8_t version) +bool Blockchain::validate_miner_transaction(const block& b, size_t cumulative_block_weight, uint64_t fee, uint64_t& base_reward, uint64_t already_generated_coins, bool &partial_block_reward, uint8_t version) { LOG_PRINT_L3("Blockchain::" << __func__); //validate reward @@ -1122,11 +1238,11 @@ bool Blockchain::validate_miner_transaction(const block& b, size_t cumulative_bl } } - std::vector last_blocks_sizes; - get_last_n_blocks_sizes(last_blocks_sizes, CRYPTONOTE_REWARD_BLOCKS_WINDOW); - if (!get_block_reward(epee::misc_utils::median(last_blocks_sizes), cumulative_block_size, already_generated_coins, base_reward, version)) + std::vector last_blocks_weights; + get_last_n_blocks_weights(last_blocks_weights, CRYPTONOTE_REWARD_BLOCKS_WINDOW); + if (!get_block_reward(epee::misc_utils::median(last_blocks_weights), cumulative_block_weight, already_generated_coins, base_reward, version)) { - MERROR_VER("block size " << cumulative_block_size << " is bigger than allowed for this blockchain"); + MERROR_VER("block weight " << cumulative_block_weight << " is bigger than allowed for this blockchain"); return false; } if(base_reward + fee < money_in_use) @@ -1135,7 +1251,7 @@ bool Blockchain::validate_miner_transaction(const block& b, size_t cumulative_bl return false; } // From hard fork 2, we allow a miner to claim less block reward than is allowed, in case a miner wants less dust - if (m_hardfork->get_current_version() < 2) + if (version < 2) { if(base_reward + fee != money_in_use) { @@ -1156,8 +1272,8 @@ bool Blockchain::validate_miner_transaction(const block& b, size_t cumulative_bl return true; } //------------------------------------------------------------------ -// get the block sizes of the last blocks, and return by reference . -void Blockchain::get_last_n_blocks_sizes(std::vector& sz, size_t count) const +// get the block weights of the last blocks, and return by reference . +void Blockchain::get_last_n_blocks_weights(std::vector& weights, size_t count) const { LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); @@ -1167,26 +1283,69 @@ void Blockchain::get_last_n_blocks_sizes(std::vector& sz, size_t count) if(h == 0) return; - m_db->block_txn_start(true); - // add size of last blocks to vector (or less, if blockchain size < count) + // add weight of last blocks to vector (or less, if blockchain size < count) size_t start_offset = h - std::min(h, count); - for(size_t i = start_offset; i < h; i++) + weights = m_db->get_block_weights(start_offset, count); +} +//------------------------------------------------------------------ +uint64_t Blockchain::get_long_term_block_weight_median(uint64_t start_height, size_t count) const +{ + LOG_PRINT_L3("Blockchain::" << __func__); + CRITICAL_REGION_LOCAL(m_blockchain_lock); + + PERF_TIMER(get_long_term_block_weights); + + CHECK_AND_ASSERT_THROW_MES(count > 0, "count == 0"); + + bool cached = false; + uint64_t blockchain_height = m_db->height(); + uint64_t tip_height = start_height + count - 1; + crypto::hash tip_hash = crypto::null_hash; + if (tip_height < blockchain_height && count == (size_t)m_long_term_block_weights_cache_rolling_median.size()) + { + tip_hash = m_db->get_block_hash_from_height(tip_height); + cached = tip_hash == m_long_term_block_weights_cache_tip_hash; + } + + if (cached) { - sz.push_back(m_db->get_block_size(i)); + MTRACE("requesting " << count << " from " << start_height << ", cached"); + return m_long_term_block_weights_cache_rolling_median.median(); + } + + // in the vast majority of uncached cases, most is still cached, + // as we just move the window one block up: + if (tip_height > 0 && count == (size_t)m_long_term_block_weights_cache_rolling_median.size() && tip_height < blockchain_height) + { + crypto::hash old_tip_hash = m_db->get_block_hash_from_height(tip_height - 1); + if (old_tip_hash == m_long_term_block_weights_cache_tip_hash) + { + MTRACE("requesting " << count << " from " << start_height << ", incremental"); + m_long_term_block_weights_cache_tip_hash = tip_hash; + m_long_term_block_weights_cache_rolling_median.insert(m_db->get_block_long_term_weight(tip_height)); + return m_long_term_block_weights_cache_rolling_median.median(); + } } - m_db->block_txn_stop(); + + MTRACE("requesting " << count << " from " << start_height << ", uncached"); + std::vector weights = m_db->get_long_term_block_weights(start_height, count); + m_long_term_block_weights_cache_tip_hash = tip_hash; + m_long_term_block_weights_cache_rolling_median.clear(); + for (uint64_t w: weights) + m_long_term_block_weights_cache_rolling_median.insert(w); + return m_long_term_block_weights_cache_rolling_median.median(); } //------------------------------------------------------------------ -uint64_t Blockchain::get_current_cumulative_blocksize_limit() const +uint64_t Blockchain::get_current_cumulative_block_weight_limit() const { LOG_PRINT_L3("Blockchain::" << __func__); - return m_current_block_cumul_sz_limit; + return m_current_block_cumul_weight_limit; } //------------------------------------------------------------------ -uint64_t Blockchain::get_current_cumulative_blocksize_median() const +uint64_t Blockchain::get_current_cumulative_block_weight_median() const { LOG_PRINT_L3("Blockchain::" << __func__); - return m_current_block_cumul_sz_median; + return m_current_block_cumul_weight_median; } //------------------------------------------------------------------ //TODO: This function only needed minor modification to work with BlockchainDB, @@ -1200,18 +1359,100 @@ uint64_t Blockchain::get_current_cumulative_blocksize_median() const // in a lot of places. That flag is not referenced in any of the code // nor any of the makefiles, howeve. Need to look into whether or not it's // necessary at all. -bool Blockchain::create_block_template(block& b, const account_public_address& miner_address, difficulty_type& diffic, uint64_t& height, uint64_t& expected_reward, const blobdata& ex_nonce) +bool Blockchain::create_block_template(block& b, const crypto::hash *from_block, const account_public_address& miner_address, difficulty_type& diffic, uint64_t& height, uint64_t& expected_reward, const blobdata& ex_nonce) { LOG_PRINT_L3("Blockchain::" << __func__); - size_t median_size; + size_t median_weight; uint64_t already_generated_coins; + uint64_t pool_cookie; - CRITICAL_REGION_BEGIN(m_blockchain_lock); - height = m_db->height(); + m_tx_pool.lock(); + const auto unlock_guard = epee::misc_utils::create_scope_leave_handler([&]() { m_tx_pool.unlock(); }); + CRITICAL_REGION_LOCAL(m_blockchain_lock); + if (m_btc_valid && !from_block) { + // The pool cookie is atomic. The lack of locking is OK, as if it changes + // just as we compare it, we'll just use a slightly old template, but + // this would be the case anyway if we'd lock, and the change happened + // just after the block template was created + if (!memcmp(&miner_address, &m_btc_address, sizeof(cryptonote::account_public_address)) && m_btc_nonce == ex_nonce + && m_btc_pool_cookie == m_tx_pool.cookie() && m_btc.prev_id == get_tail_id()) { + MDEBUG("Using cached template"); + m_btc.timestamp = time(NULL); // update timestamp unconditionally + b = m_btc; + diffic = m_btc_difficulty; + height = m_btc_height; + expected_reward = m_btc_expected_reward; + return true; + } + MDEBUG("Not using cached template: address " << (!memcmp(&miner_address, &m_btc_address, sizeof(cryptonote::account_public_address))) << ", nonce " << (m_btc_nonce == ex_nonce) << ", cookie " << (m_btc_pool_cookie == m_tx_pool.cookie()) << ", from_block " << (!!from_block)); + invalidate_block_template_cache(); + } + + if (from_block) + { + //build alternative subchain, front -> mainchain, back -> alternative head + //block is not related with head of main chain + //first of all - look in alternative chains container + auto it_prev = m_alternative_chains.find(*from_block); + bool parent_in_main = m_db->block_exists(*from_block); + if(it_prev == m_alternative_chains.end() && !parent_in_main) + { + MERROR("Unknown from block"); + return false; + } + + //we have new block in alternative chain + std::list alt_chain; + block_verification_context bvc = boost::value_initialized(); + std::vector timestamps; + if (!build_alt_chain(*from_block, alt_chain, timestamps, bvc)) + return false; + + if (parent_in_main) + { + cryptonote::block prev_block; + CHECK_AND_ASSERT_MES(get_block_by_hash(*from_block, prev_block), false, "From block not found"); // TODO + uint64_t from_block_height = cryptonote::get_block_height(prev_block); + height = from_block_height + 1; + } + else + { + height = alt_chain.back()->second.height + 1; + } + b.major_version = m_hardfork->get_ideal_version(height); + b.minor_version = m_hardfork->get_ideal_version(); + b.prev_id = *from_block; + + // cheat and use the weight of the block we start from, virtually certain to be acceptable + // and use 1.9 times rather than 2 times so we're even more sure + if (parent_in_main) + { + median_weight = m_db->get_block_weight(height - 1); + already_generated_coins = m_db->get_block_already_generated_coins(height - 1); + } + else + { + median_weight = it_prev->second.block_cumulative_weight - it_prev->second.block_cumulative_weight / 20; + already_generated_coins = alt_chain.back()->second.already_generated_coins; + } + + // FIXME: consider moving away from block_extended_info at some point + block_extended_info bei = boost::value_initialized(); + bei.bl = b; + bei.height = alt_chain.size() ? it_prev->second.height + 1 : m_db->get_block_height(*from_block) + 1; - b.major_version = m_hardfork->get_current_version(); - b.minor_version = m_hardfork->get_ideal_version(); - b.prev_id = get_tail_id(); + diffic = get_next_difficulty_for_alternative_chain(alt_chain, bei); + } + else + { + height = m_db->height(); + b.major_version = m_hardfork->get_current_version(); + b.minor_version = m_hardfork->get_ideal_version(); + b.prev_id = get_tail_id(); + median_weight = m_current_block_cumul_weight_limit / 2; + diffic = get_difficulty_for_next_block(); + already_generated_coins = m_db->get_block_already_generated_coins(height - 1); + } b.timestamp = time(NULL); uint64_t median_ts; @@ -1220,24 +1461,18 @@ bool Blockchain::create_block_template(block& b, const account_public_address& m b.timestamp = median_ts; } - diffic = get_difficulty_for_next_block(); CHECK_AND_ASSERT_MES(diffic, false, "difficulty overhead."); - median_size = m_current_block_cumul_sz_limit / 2; - already_generated_coins = m_db->get_block_already_generated_coins(height - 1); - - CRITICAL_REGION_END(); - - size_t txs_size; + size_t txs_weight; uint64_t fee; - if (!m_tx_pool.fill_block_template(b, median_size, already_generated_coins, txs_size, fee, expected_reward, m_hardfork->get_current_version())) + if (!m_tx_pool.fill_block_template(b, median_weight, already_generated_coins, txs_weight, fee, expected_reward, b.major_version)) { return false; } + pool_cookie = m_tx_pool.cookie(); #if defined(DEBUG_CREATE_BLOCK_TEMPLATE) - size_t real_txs_size = 0; + size_t real_txs_weight = 0; uint64_t real_fee = 0; - CRITICAL_REGION_BEGIN(m_tx_pool.m_transactions_lock); for(crypto::hash &cur_hash: b.tx_hashes) { auto cur_res = m_tx_pool.m_transactions.find(cur_hash); @@ -1247,11 +1482,11 @@ bool Blockchain::create_block_template(block& b, const account_public_address& m continue; } tx_memory_pool::tx_details &cur_tx = cur_res->second; - real_txs_size += cur_tx.blob_size; + real_txs_weight += cur_tx.weight; real_fee += cur_tx.fee; - if (cur_tx.blob_size != get_object_blobsize(cur_tx.tx)) + if (cur_tx.weight != get_transaction_weight(cur_tx.tx)) { - LOG_ERROR("Creating block template: error: invalid transaction size"); + LOG_ERROR("Creating block template: error: invalid transaction weight"); } if (cur_tx.tx.version == 1) { @@ -1273,90 +1508,97 @@ bool Blockchain::create_block_template(block& b, const account_public_address& m } } } - if (txs_size != real_txs_size) + if (txs_weight != real_txs_weight) { - LOG_ERROR("Creating block template: error: wrongly calculated transaction size"); + LOG_ERROR("Creating block template: error: wrongly calculated transaction weight"); } if (fee != real_fee) { LOG_ERROR("Creating block template: error: wrongly calculated fee"); } - CRITICAL_REGION_END(); MDEBUG("Creating block template: height " << height << - ", median size " << median_size << + ", median weight " << median_weight << ", already generated coins " << already_generated_coins << - ", transaction size " << txs_size << + ", transaction weight " << txs_weight << ", fee " << fee); #endif /* - two-phase miner transaction generation: we don't know exact block size until we prepare block, but we don't know reward until we know - block size, so first miner transaction generated with fake amount of money, and with phase we know think we know expected block size + two-phase miner transaction generation: we don't know exact block weight until we prepare block, but we don't know reward until we know + block weight, so first miner transaction generated with fake amount of money, and with phase we know think we know expected block weight */ - //make blocks coin-base tx looks close to real coinbase tx to get truthful blob size - uint8_t hf_version = m_hardfork->get_current_version(); + //make blocks coin-base tx looks close to real coinbase tx to get truthful blob weight + uint8_t hf_version = b.major_version; size_t max_outs = hf_version >= 4 ? 1 : 11; - bool r = construct_miner_tx(height, median_size, already_generated_coins, txs_size, fee, miner_address, b.miner_tx, ex_nonce, max_outs, hf_version); + bool r = construct_miner_tx(height, median_weight, already_generated_coins, txs_weight, fee, miner_address, b.miner_tx, ex_nonce, max_outs, hf_version); CHECK_AND_ASSERT_MES(r, false, "Failed to construct miner tx, first chance"); - size_t cumulative_size = txs_size + get_object_blobsize(b.miner_tx); + size_t cumulative_weight = txs_weight + get_transaction_weight(b.miner_tx); #if defined(DEBUG_CREATE_BLOCK_TEMPLATE) - MDEBUG("Creating block template: miner tx size " << get_object_blobsize(b.miner_tx) << - ", cumulative size " << cumulative_size); + MDEBUG("Creating block template: miner tx weight " << get_transaction_weight(b.miner_tx) << + ", cumulative weight " << cumulative_weight); #endif for (size_t try_count = 0; try_count != 10; ++try_count) { - r = construct_miner_tx(height, median_size, already_generated_coins, cumulative_size, fee, miner_address, b.miner_tx, ex_nonce, max_outs, hf_version); + r = construct_miner_tx(height, median_weight, already_generated_coins, cumulative_weight, fee, miner_address, b.miner_tx, ex_nonce, max_outs, hf_version); CHECK_AND_ASSERT_MES(r, false, "Failed to construct miner tx, second chance"); - size_t coinbase_blob_size = get_object_blobsize(b.miner_tx); - if (coinbase_blob_size > cumulative_size - txs_size) + size_t coinbase_weight = get_transaction_weight(b.miner_tx); + if (coinbase_weight > cumulative_weight - txs_weight) { - cumulative_size = txs_size + coinbase_blob_size; + cumulative_weight = txs_weight + coinbase_weight; #if defined(DEBUG_CREATE_BLOCK_TEMPLATE) - MDEBUG("Creating block template: miner tx size " << coinbase_blob_size << - ", cumulative size " << cumulative_size << " is greater than before"); + MDEBUG("Creating block template: miner tx weight " << coinbase_weight << + ", cumulative weight " << cumulative_weight << " is greater than before"); #endif continue; } - if (coinbase_blob_size < cumulative_size - txs_size) + if (coinbase_weight < cumulative_weight - txs_weight) { - size_t delta = cumulative_size - txs_size - coinbase_blob_size; + size_t delta = cumulative_weight - txs_weight - coinbase_weight; #if defined(DEBUG_CREATE_BLOCK_TEMPLATE) - MDEBUG("Creating block template: miner tx size " << coinbase_blob_size << - ", cumulative size " << txs_size + coinbase_blob_size << + MDEBUG("Creating block template: miner tx weight " << coinbase_weight << + ", cumulative weight " << txs_weight + coinbase_weight << " is less than before, adding " << delta << " zero bytes"); #endif b.miner_tx.extra.insert(b.miner_tx.extra.end(), delta, 0); //here could be 1 byte difference, because of extra field counter is varint, and it can become from 1-byte len to 2-bytes len. - if (cumulative_size != txs_size + get_object_blobsize(b.miner_tx)) + if (cumulative_weight != txs_weight + get_transaction_weight(b.miner_tx)) { - CHECK_AND_ASSERT_MES(cumulative_size + 1 == txs_size + get_object_blobsize(b.miner_tx), false, "unexpected case: cumulative_size=" << cumulative_size << " + 1 is not equal txs_cumulative_size=" << txs_size << " + get_object_blobsize(b.miner_tx)=" << get_object_blobsize(b.miner_tx)); + CHECK_AND_ASSERT_MES(cumulative_weight + 1 == txs_weight + get_transaction_weight(b.miner_tx), false, "unexpected case: cumulative_weight=" << cumulative_weight << " + 1 is not equal txs_cumulative_weight=" << txs_weight << " + get_transaction_weight(b.miner_tx)=" << get_transaction_weight(b.miner_tx)); b.miner_tx.extra.resize(b.miner_tx.extra.size() - 1); - if (cumulative_size != txs_size + get_object_blobsize(b.miner_tx)) + if (cumulative_weight != txs_weight + get_transaction_weight(b.miner_tx)) { - //fuck, not lucky, -1 makes varint-counter size smaller, in that case we continue to grow with cumulative_size + //fuck, not lucky, -1 makes varint-counter size smaller, in that case we continue to grow with cumulative_weight MDEBUG("Miner tx creation has no luck with delta_extra size = " << delta << " and " << delta - 1); - cumulative_size += delta - 1; + cumulative_weight += delta - 1; continue; } MDEBUG("Setting extra for block: " << b.miner_tx.extra.size() << ", try_count=" << try_count); } } - CHECK_AND_ASSERT_MES(cumulative_size == txs_size + get_object_blobsize(b.miner_tx), false, "unexpected case: cumulative_size=" << cumulative_size << " is not equal txs_cumulative_size=" << txs_size << " + get_object_blobsize(b.miner_tx)=" << get_object_blobsize(b.miner_tx)); + CHECK_AND_ASSERT_MES(cumulative_weight == txs_weight + get_transaction_weight(b.miner_tx), false, "unexpected case: cumulative_weight=" << cumulative_weight << " is not equal txs_cumulative_weight=" << txs_weight << " + get_transaction_weight(b.miner_tx)=" << get_transaction_weight(b.miner_tx)); #if defined(DEBUG_CREATE_BLOCK_TEMPLATE) - MDEBUG("Creating block template: miner tx size " << coinbase_blob_size << - ", cumulative size " << cumulative_size << " is now good"); + MDEBUG("Creating block template: miner tx weight " << coinbase_weight << + ", cumulative weight " << cumulative_weight << " is now good"); #endif + + if (!from_block) + cache_block_template(b, miner_address, ex_nonce, diffic, height, expected_reward, pool_cookie); return true; } LOG_ERROR("Failed to create_block_template with " << 10 << " tries"); return false; } //------------------------------------------------------------------ +bool Blockchain::create_block_template(block& b, const account_public_address& miner_address, difficulty_type& diffic, uint64_t& height, uint64_t& expected_reward, const blobdata& ex_nonce) +{ + return create_block_template(b, NULL, miner_address, diffic, height, expected_reward, ex_nonce); +} +//------------------------------------------------------------------ // for an alternate chain, get the timestamps from the main chain to complete // the needed number of timestamps for the BLOCKCHAIN_TIMESTAMP_CHECK_WINDOW. -bool Blockchain::complete_timestamps_vector(uint64_t start_top_height, std::vector& timestamps) +bool Blockchain::complete_timestamps_vector(uint64_t start_top_height, std::vector& timestamps) const { LOG_PRINT_L3("Blockchain::" << __func__); @@ -1367,6 +1609,7 @@ bool Blockchain::complete_timestamps_vector(uint64_t start_top_height, std::vect size_t need_elements = BLOCKCHAIN_TIMESTAMP_CHECK_WINDOW - timestamps.size(); CHECK_AND_ASSERT_MES(start_top_height < m_db->height(), false, "internal error: passed start_height not < " << " m_db->height() -- " << start_top_height << " >= " << m_db->height()); size_t stop_offset = start_top_height > need_elements ? start_top_height - need_elements : 0; + timestamps.reserve(timestamps.size() + start_top_height - stop_offset); while (start_top_height != stop_offset) { timestamps.push_back(m_db->get_block_timestamp(start_top_height)); @@ -1375,6 +1618,52 @@ bool Blockchain::complete_timestamps_vector(uint64_t start_top_height, std::vect return true; } //------------------------------------------------------------------ +bool Blockchain::build_alt_chain(const crypto::hash &prev_id, std::list& alt_chain, std::vector ×tamps, block_verification_context& bvc) const +{ + //build alternative subchain, front -> mainchain, back -> alternative head + blocks_ext_by_hash::const_iterator alt_it = m_alternative_chains.find(prev_id); + timestamps.clear(); + while(alt_it != m_alternative_chains.end()) + { + alt_chain.push_front(alt_it); + timestamps.push_back(alt_it->second.bl.timestamp); + alt_it = m_alternative_chains.find(alt_it->second.bl.prev_id); + } + + // if block to be added connects to known blocks that aren't part of the + // main chain -- that is, if we're adding on to an alternate chain + if(!alt_chain.empty()) + { + // make sure alt chain doesn't somehow start past the end of the main chain + CHECK_AND_ASSERT_MES(m_db->height() > alt_chain.front()->second.height, false, "main blockchain wrong height"); + + // make sure that the blockchain contains the block that should connect + // this alternate chain with it. + if (!m_db->block_exists(alt_chain.front()->second.bl.prev_id)) + { + MERROR("alternate chain does not appear to connect to main chain..."); + return false; + } + + // make sure block connects correctly to the main chain + auto h = m_db->get_block_hash_from_height(alt_chain.front()->second.height - 1); + CHECK_AND_ASSERT_MES(h == alt_chain.front()->second.bl.prev_id, false, "alternative chain has wrong connection to main chain"); + complete_timestamps_vector(m_db->get_block_height(alt_chain.front()->second.bl.prev_id), timestamps); + } + // if block not associated with known alternate chain + else + { + // if block parent is not part of main chain or an alternate chain, + // we ignore it + bool parent_in_main = m_db->block_exists(prev_id); + CHECK_AND_ASSERT_MES(parent_in_main, false, "internal error: broken imperative condition: parent_in_main"); + + complete_timestamps_vector(m_db->get_block_height(prev_id), timestamps); + } + + return true; +} +//------------------------------------------------------------------ // If a block is to be added and its parent block is not the current // main chain top block, then we need to see if we know about its parent block. // If its parent block is part of a known forked chain, then we need to see @@ -1419,47 +1708,18 @@ bool Blockchain::handle_alternative_block(const block& b, const crypto::hash& id if(it_prev != m_alternative_chains.end() || parent_in_main) { //we have new block in alternative chain - - //build alternative subchain, front -> mainchain, back -> alternative head - blocks_ext_by_hash::iterator alt_it = it_prev; //m_alternative_chains.find() - std::list alt_chain; + std::list alt_chain; std::vector timestamps; - while(alt_it != m_alternative_chains.end()) - { - alt_chain.push_front(alt_it); - timestamps.push_back(alt_it->second.bl.timestamp); - alt_it = m_alternative_chains.find(alt_it->second.bl.prev_id); - } - - // if block to be added connects to known blocks that aren't part of the - // main chain -- that is, if we're adding on to an alternate chain - if(alt_chain.size()) - { - // make sure alt chain doesn't somehow start past the end of the main chain - CHECK_AND_ASSERT_MES(m_db->height() > alt_chain.front()->second.height, false, "main blockchain wrong height"); - - // make sure that the blockchain contains the block that should connect - // this alternate chain with it. - if (!m_db->block_exists(alt_chain.front()->second.bl.prev_id)) - { - MERROR("alternate chain does not appear to connect to main chain..."); - return false; - } - - // make sure block connects correctly to the main chain - auto h = m_db->get_block_hash_from_height(alt_chain.front()->second.height - 1); - CHECK_AND_ASSERT_MES(h == alt_chain.front()->second.bl.prev_id, false, "alternative chain has wrong connection to main chain"); - complete_timestamps_vector(m_db->get_block_height(alt_chain.front()->second.bl.prev_id), timestamps); - } - // if block not associated with known alternate chain - else - { - // if block parent is not part of main chain or an alternate chain, - // we ignore it - CHECK_AND_ASSERT_MES(parent_in_main, false, "internal error: broken imperative condition: parent_in_main"); + if (!build_alt_chain(b.prev_id, alt_chain, timestamps, bvc)) + return false; - complete_timestamps_vector(m_db->get_block_height(b.prev_id), timestamps); - } + // FIXME: consider moving away from block_extended_info at some point + block_extended_info bei = boost::value_initialized(); + bei.bl = b; + const uint64_t prev_height = alt_chain.size() ? it_prev->second.height : m_db->get_block_height(b.prev_id); + bei.height = prev_height + 1; + uint64_t block_reward = get_outs_money_amount(b.miner_tx); + bei.already_generated_coins = block_reward + (alt_chain.size() ? it_prev->second.already_generated_coins : m_db->get_block_already_generated_coins(prev_height)); // verify that the block's timestamp is within the acceptable range // (not earlier than the median of the last X blocks) @@ -1470,11 +1730,6 @@ bool Blockchain::handle_alternative_block(const block& b, const crypto::hash& id return false; } - // FIXME: consider moving away from block_extended_info at some point - block_extended_info bei = boost::value_initialized(); - bei.bl = b; - bei.height = alt_chain.size() ? it_prev->second.height + 1 : m_db->get_block_height(b.prev_id) + 1; - bool is_a_checkpoint; if(!m_checkpoints.check_block(bei.height, id, is_a_checkpoint)) { @@ -1566,7 +1821,7 @@ bool Blockchain::handle_alternative_block(const block& b, const crypto::hash& id return true; } //------------------------------------------------------------------ -bool Blockchain::get_blocks(uint64_t start_offset, size_t count, std::list>& blocks, std::list& txs) const +bool Blockchain::get_blocks(uint64_t start_offset, size_t count, std::vector>& blocks, std::vector& txs) const { LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); @@ -1580,7 +1835,7 @@ bool Blockchain::get_blocks(uint64_t start_offset, size_t count, std::list missed_ids; + std::vector missed_ids; get_transactions_blobs(blk.second.tx_hashes, txs, missed_ids); CHECK_AND_ASSERT_MES(!missed_ids.size(), false, "has missed transactions in own block in main blockchain"); } @@ -1588,14 +1843,16 @@ bool Blockchain::get_blocks(uint64_t start_offset, size_t count, std::list>& blocks) const +bool Blockchain::get_blocks(uint64_t start_offset, size_t count, std::vector>& blocks) const { LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); - if(start_offset >= m_db->height()) + const uint64_t height = m_db->height(); + if(start_offset >= height) return false; - for(size_t i = start_offset; i < start_offset + count && i < m_db->height();i++) + blocks.reserve(blocks.size() + height - start_offset); + for(size_t i = start_offset; i < start_offset + count && i < height;i++) { blocks.push_back(std::make_pair(m_db->get_block_blob_from_height(i), block())); if (!parse_and_validate_block_from_blob(blocks.back().first, blocks.back().second)) @@ -1618,59 +1875,55 @@ bool Blockchain::handle_get_objects(NOTIFY_REQUEST_GET_OBJECTS::request& arg, NO { LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); - m_db->block_txn_start(true); + db_rtxn_guard rtxn_guard (m_db); rsp.current_blockchain_height = get_current_blockchain_height(); - std::list> blocks; + std::vector> blocks; get_blocks(arg.blocks, blocks, rsp.missed_ids); - for (const auto& bl: blocks) + for (auto& bl: blocks) { - std::list missed_tx_ids; - std::list txs; + std::vector missed_tx_ids; + + rsp.blocks.push_back(block_complete_entry()); + block_complete_entry& e = rsp.blocks.back(); // FIXME: s/rsp.missed_ids/missed_tx_id/ ? Seems like rsp.missed_ids // is for missed blocks, not missed transactions as well. - get_transactions_blobs(bl.second.tx_hashes, txs, missed_tx_ids); + get_transactions_blobs(bl.second.tx_hashes, e.txs, missed_tx_ids); if (missed_tx_ids.size() != 0) { - LOG_ERROR("Error retrieving blocks, missed " << missed_tx_ids.size() - << " transactions for block with hash: " << get_block_hash(bl.second) - << std::endl - ); + // do not display an error if the peer asked for an unpruned block which we are not meant to have + if (tools::has_unpruned_block(get_block_height(bl.second), get_current_blockchain_height(), get_blockchain_pruning_seed())) + { + LOG_ERROR("Error retrieving blocks, missed " << missed_tx_ids.size() + << " transactions for block with hash: " << get_block_hash(bl.second) + << std::endl + ); + } // append missed transaction hashes to response missed_ids field, // as done below if any standalone transactions were requested // and missed. - rsp.missed_ids.splice(rsp.missed_ids.end(), missed_tx_ids); - m_db->block_txn_stop(); + rsp.missed_ids.insert(rsp.missed_ids.end(), missed_tx_ids.begin(), missed_tx_ids.end()); return false; } - rsp.blocks.push_back(block_complete_entry()); - block_complete_entry& e = rsp.blocks.back(); //pack block - e.block = bl.first; - //pack transactions - for (const cryptonote::blobdata& tx: txs) - e.txs.push_back(tx); - } - //get another transactions, if need - std::list txs; - get_transactions_blobs(arg.txs, txs, rsp.missed_ids); - //pack aside transactions - for (const auto& tx: txs) - rsp.txs.push_back(tx); - - m_db->block_txn_stop(); + e.block = std::move(bl.first); + } + //get and pack other transactions, if needed + get_transactions_blobs(arg.txs, rsp.txs, rsp.missed_ids); + return true; } //------------------------------------------------------------------ -bool Blockchain::get_alternative_blocks(std::list& blocks) const +bool Blockchain::get_alternative_blocks(std::vector& blocks) const { LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); + blocks.reserve(m_alternative_chains.size()); for (const auto& alt_bl: m_alternative_chains) { blocks.push_back(alt_bl.second.bl); @@ -1687,271 +1940,30 @@ size_t Blockchain::get_alternative_blocks_count() const //------------------------------------------------------------------ // This function adds the output specified by to the result_outs container // unlocked and other such checks should be done by here. -void Blockchain::add_out_to_get_random_outs(COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::outs_for_amount& result_outs, uint64_t amount, size_t i) const -{ - LOG_PRINT_L3("Blockchain::" << __func__); - CRITICAL_REGION_LOCAL(m_blockchain_lock); - - COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::out_entry& oen = *result_outs.outs.insert(result_outs.outs.end(), COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::out_entry()); - oen.global_amount_index = i; - output_data_t data = m_db->get_output_key(amount, i); - oen.out_key = data.pubkey; -} - uint64_t Blockchain::get_num_mature_outputs(uint64_t amount) const { uint64_t num_outs = m_db->get_num_outputs(amount); // ensure we don't include outputs that aren't yet eligible to be used // outpouts are sorted by height + const uint64_t blockchain_height = m_db->height(); while (num_outs > 0) { - const tx_out_index toi = m_db->get_output_tx_and_index(amount, num_outs - 1); - const uint64_t height = m_db->get_tx_block_height(toi.first); - if (height + CRYPTONOTE_DEFAULT_TX_SPENDABLE_AGE <= m_db->height()) - break; - --num_outs; - } - - return num_outs; -} - -std::vector Blockchain::get_random_outputs(uint64_t amount, uint64_t count) const -{ - uint64_t num_outs = get_num_mature_outputs(amount); - - std::vector indices; - - std::unordered_set seen_indices; - - // if there aren't enough outputs to mix with (or just enough), - // use all of them. Eventually this should become impossible. - if (num_outs <= count) - { - for (uint64_t i = 0; i < num_outs; i++) - { - // get tx_hash, tx_out_index from DB - tx_out_index toi = m_db->get_output_tx_and_index(amount, i); - - // if tx is unlocked, add output to indices - if (is_tx_spendtime_unlocked(m_db->get_tx_unlock_time(toi.first))) - { - indices.push_back(i); - } - } - } - else - { - // while we still need more mixins - while (indices.size() < count) - { - // if we've gone through every possible output, we've gotten all we can - if (seen_indices.size() == num_outs) - { - break; - } - - // get a random output index from the DB. If we've already seen it, - // return to the top of the loop and try again, otherwise add it to the - // list of output indices we've seen. - - // triangular distribution over [a,b) with a=0, mode c=b=up_index_limit - uint64_t r = crypto::rand() % ((uint64_t)1 << 53); - double frac = std::sqrt((double)r / ((uint64_t)1 << 53)); - uint64_t i = (uint64_t)(frac*num_outs); - // just in case rounding up to 1 occurs after sqrt - if (i == num_outs) - --i; - - if (seen_indices.count(i)) - { - continue; - } - seen_indices.emplace(i); - - // get tx_hash, tx_out_index from DB - tx_out_index toi = m_db->get_output_tx_and_index(amount, i); - - // if the output's transaction is unlocked, add the output's index to - // our list. - if (is_tx_spendtime_unlocked(m_db->get_tx_unlock_time(toi.first))) - { - indices.push_back(i); - } - } - } - - return indices; -} - -crypto::public_key Blockchain::get_output_key(uint64_t amount, uint64_t global_index) const -{ - output_data_t data = m_db->get_output_key(amount, global_index); - return data.pubkey; -} - -//------------------------------------------------------------------ -// This function takes an RPC request for mixins and creates an RPC response -// with the requested mixins. -// TODO: figure out why this returns boolean / if we should be returning false -// in some cases -bool Blockchain::get_random_outs_for_amounts(const COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::request& req, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::response& res) const -{ - LOG_PRINT_L3("Blockchain::" << __func__); - CRITICAL_REGION_LOCAL(m_blockchain_lock); - - // for each amount that we need to get mixins for, get random outputs - // from BlockchainDB where is req.outs_count (number of mixins). - for (uint64_t amount : req.amounts) - { - // create outs_for_amount struct and populate amount field - COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::outs_for_amount& result_outs = *res.outs.insert(res.outs.end(), COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::outs_for_amount()); - result_outs.amount = amount; - - std::vector indices = get_random_outputs(amount, req.outs_count); - - for (auto i : indices) - { - COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::out_entry& oe = *result_outs.outs.insert(result_outs.outs.end(), COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::out_entry()); - - oe.global_amount_index = i; - oe.out_key = get_output_key(amount, i); - } - } - return true; -} -//------------------------------------------------------------------ -// This function adds the ringct output at index i to the list -// unlocked and other such checks should be done by here. -void Blockchain::add_out_to_get_rct_random_outs(std::list& outs, uint64_t amount, size_t i) const -{ - LOG_PRINT_L3("Blockchain::" << __func__); - CRITICAL_REGION_LOCAL(m_blockchain_lock); - - COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS::out_entry& oen = *outs.insert(outs.end(), COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS::out_entry()); - oen.amount = amount; - oen.global_amount_index = i; - output_data_t data = m_db->get_output_key(amount, i); - oen.out_key = data.pubkey; - oen.commitment = data.commitment; -} -//------------------------------------------------------------------ -// This function takes an RPC request for mixins and creates an RPC response -// with the requested mixins. -// TODO: figure out why this returns boolean / if we should be returning false -// in some cases -bool Blockchain::get_random_rct_outs(const COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS::request& req, COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS::response& res) const -{ - LOG_PRINT_L3("Blockchain::" << __func__); - CRITICAL_REGION_LOCAL(m_blockchain_lock); - - // for each amount that we need to get mixins for, get random outputs - // from BlockchainDB where is req.outs_count (number of mixins). - auto num_outs = m_db->get_num_outputs(0); - // ensure we don't include outputs that aren't yet eligible to be used - // outpouts are sorted by height - while (num_outs > 0) - { - const tx_out_index toi = m_db->get_output_tx_and_index(0, num_outs - 1); - const uint64_t height = m_db->get_tx_block_height(toi.first); - if (height + CRYPTONOTE_DEFAULT_TX_SPENDABLE_AGE <= m_db->height()) - break; - --num_outs; - } - - std::unordered_set seen_indices; - - // if there aren't enough outputs to mix with (or just enough), - // use all of them. Eventually this should become impossible. - if (num_outs <= req.outs_count) - { - for (uint64_t i = 0; i < num_outs; i++) - { - // get tx_hash, tx_out_index from DB - tx_out_index toi = m_db->get_output_tx_and_index(0, i); - - // if tx is unlocked, add output to result_outs - if (is_tx_spendtime_unlocked(m_db->get_tx_unlock_time(toi.first))) - { - add_out_to_get_rct_random_outs(res.outs, 0, i); - } - } - } - else - { - // while we still need more mixins - while (res.outs.size() < req.outs_count) - { - // if we've gone through every possible output, we've gotten all we can - if (seen_indices.size() == num_outs) - { - break; - } - - // get a random output index from the DB. If we've already seen it, - // return to the top of the loop and try again, otherwise add it to the - // list of output indices we've seen. - - // triangular distribution over [a,b) with a=0, mode c=b=up_index_limit - uint64_t r = crypto::rand() % ((uint64_t)1 << 53); - double frac = std::sqrt((double)r / ((uint64_t)1 << 53)); - uint64_t i = (uint64_t)(frac*num_outs); - // just in case rounding up to 1 occurs after sqrt - if (i == num_outs) - --i; - - if (seen_indices.count(i)) - { - continue; - } - seen_indices.emplace(i); - - // get tx_hash, tx_out_index from DB - tx_out_index toi = m_db->get_output_tx_and_index(0, i); - - // if the output's transaction is unlocked, add the output's index to - // our list. - if (is_tx_spendtime_unlocked(m_db->get_tx_unlock_time(toi.first))) - { - add_out_to_get_rct_random_outs(res.outs, 0, i); - } - } - } - - if (res.outs.size() < req.outs_count) - return false; -#if 0 - // if we do not have enough RCT inputs, we can pick from the non RCT ones - // which will have a zero mask - if (res.outs.size() < req.outs_count) - { - LOG_PRINT_L0("Out of RCT inputs (" << res.outs.size() << "/" << req.outs_count << "), using regular ones"); - - // TODO: arbitrary selection, needs better - COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::request req2 = AUTO_VAL_INIT(req2); - COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::response res2 = AUTO_VAL_INIT(res2); - req2.outs_count = req.outs_count - res.outs.size(); - static const uint64_t amounts[] = {1, 10, 20, 50, 100, 200, 500, 1000, 10000}; - for (uint64_t a: amounts) - req2.amounts.push_back(a); - if (!get_random_outs_for_amounts(req2, res2)) - return false; - - // pick random ones from there - while (res.outs.size() < req.outs_count) - { - int list_idx = rand() % (sizeof(amounts)/sizeof(amounts[0])); - if (!res2.outs[list_idx].outs.empty()) - { - const COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::out_entry oe = res2.outs[list_idx].outs.back(); - res2.outs[list_idx].outs.pop_back(); - add_out_to_get_rct_random_outs(res.outs, res2.outs[list_idx].amount, oe.global_amount_index); - } - } + const tx_out_index toi = m_db->get_output_tx_and_index(amount, num_outs - 1); + const uint64_t height = m_db->get_tx_block_height(toi.first); + if (height + CRYPTONOTE_DEFAULT_TX_SPENDABLE_AGE <= blockchain_height) + break; + --num_outs; } -#endif - return true; + return num_outs; +} + +crypto::public_key Blockchain::get_output_key(uint64_t amount, uint64_t global_index) const +{ + output_data_t data = m_db->get_output_key(amount, global_index); + return data.pubkey; } + //------------------------------------------------------------------ bool Blockchain::get_outs(const COMMAND_RPC_GET_OUTPUTS_BIN::request& req, COMMAND_RPC_GET_OUTPUTS_BIN::response& res) const { @@ -1960,14 +1972,39 @@ bool Blockchain::get_outs(const COMMAND_RPC_GET_OUTPUTS_BIN::request& req, COMMA res.outs.clear(); res.outs.reserve(req.outputs.size()); - for (const auto &i: req.outputs) + + std::vector data; + try { - // get tx_hash, tx_out_index from DB - const output_data_t od = m_db->get_output_key(i.amount, i.index); - tx_out_index toi = m_db->get_output_tx_and_index(i.amount, i.index); - bool unlocked = is_tx_spendtime_unlocked(m_db->get_tx_unlock_time(toi.first)); + std::vector amounts, offsets; + amounts.reserve(req.outputs.size()); + offsets.reserve(req.outputs.size()); + for (const auto &i: req.outputs) + { + amounts.push_back(i.amount); + offsets.push_back(i.index); + } + m_db->get_output_key(epee::span(amounts.data(), amounts.size()), offsets, data); + if (data.size() != req.outputs.size()) + { + MERROR("Unexpected output data size: expected " << req.outputs.size() << ", got " << data.size()); + return false; + } + for (const auto &t: data) + res.outs.push_back({t.pubkey, t.commitment, is_tx_spendtime_unlocked(t.unlock_time), t.height, crypto::null_hash}); - res.outs.push_back({od.pubkey, od.commitment, unlocked, od.height, toi.first}); + if (req.get_txid) + { + for (size_t i = 0; i < req.outputs.size(); ++i) + { + tx_out_index toi = m_db->get_output_tx_and_index(req.outputs[i].amount, req.outputs[i].index); + res.outs[i].txid = toi.first; + } + } + } + catch (const std::exception &e) + { + return false; } return true; } @@ -1983,14 +2020,15 @@ void Blockchain::get_output_key_mask_unlocked(const uint64_t& amount, const uint //------------------------------------------------------------------ bool Blockchain::get_output_distribution(uint64_t amount, uint64_t from_height, uint64_t to_height, uint64_t &start_height, std::vector &distribution, uint64_t &base) const { - // rct outputs don't exist before v3 + // rct outputs don't exist before v4 if (amount == 0) { switch (m_nettype) { - case STAGENET: start_height = stagenet_hard_forks[2].height; break; - case TESTNET: start_height = testnet_hard_forks[2].height; break; - case MAINNET: start_height = mainnet_hard_forks[2].height; break; + case STAGENET: start_height = stagenet_hard_forks[3].height; break; + case TESTNET: start_height = testnet_hard_forks[3].height; break; + case MAINNET: start_height = mainnet_hard_forks[3].height; break; + case FAKECHAIN: start_height = 0; break; default: return false; } } @@ -1998,11 +2036,38 @@ bool Blockchain::get_output_distribution(uint64_t amount, uint64_t from_height, start_height = 0; base = 0; + if (to_height > 0 && to_height < from_height) + return false; + const uint64_t real_start_height = start_height; if (from_height > start_height) start_height = from_height; - return m_db->get_output_distribution(amount, start_height, to_height, distribution, base); + distribution.clear(); + uint64_t db_height = m_db->height(); + if (db_height == 0) + return false; + if (start_height >= db_height || to_height >= db_height) + return false; + if (amount == 0) + { + std::vector heights; + heights.reserve(to_height + 1 - start_height); + uint64_t real_start_height = start_height > 0 ? start_height-1 : start_height; + for (uint64_t h = real_start_height; h <= to_height; ++h) + heights.push_back(h); + distribution = m_db->get_block_cumulative_rct_outputs(heights); + if (start_height > 0) + { + base = distribution[0]; + distribution.erase(distribution.begin()); + } + return true; + } + else + { + return m_db->get_output_distribution(amount, start_height, to_height, distribution, base); + } } //------------------------------------------------------------------ // This function takes a list of block hashes from another node @@ -2015,20 +2080,19 @@ bool Blockchain::find_blockchain_supplement(const std::list& qbloc // make sure the request includes at least the genesis block, otherwise // how can we expect to sync from the client that the block list came from? - if(!qblock_ids.size() /*|| !req.m_total_height*/) + if(qblock_ids.empty()) { - MCERROR("net.p2p", "Client sent wrong NOTIFY_REQUEST_CHAIN: m_block_ids.size()=" << qblock_ids.size() << /*", m_height=" << req.m_total_height <<*/ ", dropping connection"); + MCERROR("net.p2p", "Client sent wrong NOTIFY_REQUEST_CHAIN: m_block_ids.size()=" << qblock_ids.size() << ", dropping connection"); return false; } - m_db->block_txn_start(true); + db_rtxn_guard rtxn_guard(m_db); // make sure that the last block in the request's block list matches // the genesis block auto gen_hash = m_db->get_block_hash_from_height(0); if(qblock_ids.back() != gen_hash) { MCERROR("net.p2p", "Client sent wrong NOTIFY_REQUEST_CHAIN: genesis block mismatch: " << std::endl << "id: " << qblock_ids.back() << ", " << std::endl << "expected: " << gen_hash << "," << std::endl << " dropping connection"); - m_db->block_txn_abort(); return false; } @@ -2046,11 +2110,9 @@ bool Blockchain::find_blockchain_supplement(const std::list& qbloc catch (const std::exception& e) { MWARNING("Non-critical error trying to find block by hash in BlockchainDB, hash: " << *bl_it); - m_db->block_txn_abort(); return false; } } - m_db->block_txn_stop(); // this should be impossible, as we checked that we share the genesis block, // but just in case... @@ -2065,7 +2127,7 @@ bool Blockchain::find_blockchain_supplement(const std::list& qbloc return true; } //------------------------------------------------------------------ -uint64_t Blockchain::block_difficulty(uint64_t i) const +difficulty_type Blockchain::block_difficulty(uint64_t i) const { LOG_PRINT_L3("Blockchain::" << __func__); // WARNING: this function does not take m_blockchain_lock, and thus should only call read only @@ -2083,6 +2145,9 @@ uint64_t Blockchain::block_difficulty(uint64_t i) const return 0; } //------------------------------------------------------------------ +template void reserve_container(std::vector &v, size_t N) { v.reserve(N); } +template void reserve_container(std::list &v, size_t N) { } +//------------------------------------------------------------------ //TODO: return type should be void, throw on exception // alternatively, return true only if no blocks missed template @@ -2091,20 +2156,24 @@ bool Blockchain::get_blocks(const t_ids_container& block_ids, t_blocks_container LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); + reserve_container(blocks, block_ids.size()); for (const auto& block_hash : block_ids) { try { - blocks.push_back(std::make_pair(m_db->get_block_blob(block_hash), block())); - if (!parse_and_validate_block_from_blob(blocks.back().first, blocks.back().second)) + uint64_t height = 0; + if (m_db->block_exists(block_hash, &height)) { - LOG_ERROR("Invalid block"); - return false; + blocks.push_back(std::make_pair(m_db->get_block_blob_from_height(height), block())); + if (!parse_and_validate_block_from_blob(blocks.back().first, blocks.back().second)) + { + LOG_ERROR("Invalid block: " << block_hash); + blocks.pop_back(); + missed_bs.push_back(block_hash); + } } - } - catch (const BLOCK_DNE& e) - { - missed_bs.push_back(block_hash); + else + missed_bs.push_back(block_hash); } catch (const std::exception& e) { @@ -2122,6 +2191,7 @@ bool Blockchain::get_transactions_blobs(const t_ids_container& txs_ids, t_tx_con LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); + reserve_container(txs, txs_ids.size()); for (const auto& tx_hash : txs_ids) { try @@ -2142,12 +2212,58 @@ bool Blockchain::get_transactions_blobs(const t_ids_container& txs_ids, t_tx_con return true; } //------------------------------------------------------------------ +size_t get_transaction_version(const cryptonote::blobdata &bd) +{ + size_t version; + const char* begin = static_cast(bd.data()); + const char* end = begin + bd.size(); + int read = tools::read_varint(begin, end, version); + if (read <= 0) + throw std::runtime_error("Internal error getting transaction version"); + return version; +} +//------------------------------------------------------------------ +template +bool Blockchain::get_split_transactions_blobs(const t_ids_container& txs_ids, t_tx_container& txs, t_missed_container& missed_txs) const +{ + LOG_PRINT_L3("Blockchain::" << __func__); + CRITICAL_REGION_LOCAL(m_blockchain_lock); + + reserve_container(txs, txs_ids.size()); + for (const auto& tx_hash : txs_ids) + { + try + { + cryptonote::blobdata tx; + if (m_db->get_pruned_tx_blob(tx_hash, tx)) + { + txs.push_back(std::make_tuple(tx_hash, std::move(tx), crypto::null_hash, cryptonote::blobdata())); + if (!is_v1_tx(std::get<1>(txs.back())) && !m_db->get_prunable_tx_hash(tx_hash, std::get<2>(txs.back()))) + { + MERROR("Prunable data hash not found for " << tx_hash); + return false; + } + if (!m_db->get_prunable_tx_blob(tx_hash, std::get<3>(txs.back()))) + std::get<3>(txs.back()).clear(); + } + else + missed_txs.push_back(tx_hash); + } + catch (const std::exception& e) + { + return false; + } + } + return true; +} +//------------------------------------------------------------------ template bool Blockchain::get_transactions(const t_ids_container& txs_ids, t_tx_container& txs, t_missed_container& missed_txs) const { LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); + reserve_container(txs, txs_ids.size()); for (const auto& tx_hash : txs_ids) { try @@ -2176,7 +2292,7 @@ bool Blockchain::get_transactions(const t_ids_container& txs_ids, t_tx_container // Find the split point between us and foreign blockchain and return // (by reference) the most recent common block hash along with up to // BLOCKS_IDS_SYNCHRONIZING_DEFAULT_COUNT additional (more recent) hashes. -bool Blockchain::find_blockchain_supplement(const std::list& qblock_ids, std::list& hashes, uint64_t& start_height, uint64_t& current_height) const +bool Blockchain::find_blockchain_supplement(const std::list& qblock_ids, std::vector& hashes, uint64_t& start_height, uint64_t& current_height, bool clip_pruned) const { LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); @@ -2187,15 +2303,22 @@ bool Blockchain::find_blockchain_supplement(const std::list& qbloc return false; } - m_db->block_txn_start(true); + db_rtxn_guard rtxn_guard(m_db); current_height = get_current_blockchain_height(); + uint64_t stop_height = current_height; + if (clip_pruned) + { + const uint32_t pruning_seed = get_blockchain_pruning_seed(); + start_height = tools::get_next_unpruned_block_height(start_height, current_height, pruning_seed); + stop_height = tools::get_next_pruned_block_height(start_height, current_height, pruning_seed); + } size_t count = 0; - for(size_t i = start_height; i < current_height && count < BLOCKS_IDS_SYNCHRONIZING_DEFAULT_COUNT; i++, count++) + hashes.reserve(std::min((size_t)(stop_height - start_height), (size_t)BLOCKS_IDS_SYNCHRONIZING_DEFAULT_COUNT)); + for(size_t i = start_height; i < stop_height && count < BLOCKS_IDS_SYNCHRONIZING_DEFAULT_COUNT; i++, count++) { hashes.push_back(m_db->get_block_hash_from_height(i)); } - m_db->block_txn_stop(); return true; } @@ -2204,8 +2327,13 @@ bool Blockchain::find_blockchain_supplement(const std::list& qbloc LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); - bool result = find_blockchain_supplement(qblock_ids, resp.m_block_ids, resp.start_height, resp.total_height); - resp.cumulative_difficulty = m_db->get_block_cumulative_difficulty(m_db->height() - 1); + bool result = find_blockchain_supplement(qblock_ids, resp.m_block_ids, resp.start_height, resp.total_height, true); + if (result) + { + cryptonote::difficulty_type wide_cumulative_difficulty = m_db->get_block_cumulative_difficulty(resp.total_height - 1); + resp.cumulative_difficulty = (wide_cumulative_difficulty & 0xffffffffffffffff).convert_to(); + resp.cumulative_difficulty_top64 = ((wide_cumulative_difficulty >> 64) & 0xffffffffffffffff).convert_to(); + } return result; } @@ -2214,7 +2342,7 @@ bool Blockchain::find_blockchain_supplement(const std::list& qbloc // find split point between ours and foreign blockchain (or start at // blockchain height ), and return up to max_count FULL // blocks by reference. -bool Blockchain::find_blockchain_supplement(const uint64_t req_start_block, const std::list& qblock_ids, std::list > >& blocks, uint64_t& total_height, uint64_t& start_height, bool pruned, size_t max_count) const +bool Blockchain::find_blockchain_supplement(const uint64_t req_start_block, const std::list& qblock_ids, std::vector, std::vector > > >& blocks, uint64_t& total_height, uint64_t& start_height, bool pruned, bool get_miner_tx_hash, size_t max_count) const { LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); @@ -2237,23 +2365,32 @@ bool Blockchain::find_blockchain_supplement(const uint64_t req_start_block, cons } } - m_db->block_txn_start(true); + db_rtxn_guard rtxn_guard(m_db); total_height = get_current_blockchain_height(); size_t count = 0, size = 0; - for(size_t i = start_height; i < total_height && count < max_count && (size < FIND_BLOCKCHAIN_SUPPLEMENT_MAX_SIZE || count < 3); i++, count++) + blocks.reserve(std::min(std::min(max_count, (size_t)10000), (size_t)(total_height - start_height))); + for(uint64_t i = start_height; i < total_height && count < max_count && (size < FIND_BLOCKCHAIN_SUPPLEMENT_MAX_SIZE || count < 3); i++, count++) { blocks.resize(blocks.size()+1); - blocks.back().first = m_db->get_block_blob_from_height(i); + blocks.back().first.first = m_db->get_block_blob_from_height(i); block b; - CHECK_AND_ASSERT_MES(parse_and_validate_block_from_blob(blocks.back().first, b), false, "internal error, invalid block"); - std::list mis; - get_transactions_blobs(b.tx_hashes, blocks.back().second, mis, pruned); + CHECK_AND_ASSERT_MES(parse_and_validate_block_from_blob(blocks.back().first.first, b), false, "internal error, invalid block"); + blocks.back().first.second = get_miner_tx_hash ? cryptonote::get_transaction_hash(b.miner_tx) : crypto::null_hash; + std::vector mis; + std::vector txs; + get_transactions_blobs(b.tx_hashes, txs, mis, pruned); CHECK_AND_ASSERT_MES(!mis.size(), false, "internal error, transaction from block not found"); - size += blocks.back().first.size(); - for (const auto &t: blocks.back().second) + size += blocks.back().first.first.size(); + for (const auto &t: txs) size += t.size(); + + CHECK_AND_ASSERT_MES(txs.size() == b.tx_hashes.size(), false, "mismatched sizes of b.tx_hashes and txs"); + blocks.back().second.reserve(txs.size()); + for (size_t i = 0; i < txs.size(); ++i) + { + blocks.back().second.push_back(std::make_pair(b.tx_hashes[i], std::move(txs[i]))); + } } - m_db->block_txn_stop(); return true; } //------------------------------------------------------------------ @@ -2282,19 +2419,19 @@ bool Blockchain::have_block(const crypto::hash& id) const if(m_db->block_exists(id)) { - LOG_PRINT_L3("block exists in main chain"); + LOG_PRINT_L2("block " << id << " found in main chain"); return true; } if(m_alternative_chains.count(id)) { - LOG_PRINT_L3("block found in m_alternative_chains"); + LOG_PRINT_L2("block " << id << " found in m_alternative_chains"); return true; } if(m_invalid_blocks.count(id)) { - LOG_PRINT_L3("block found in m_invalid_blocks"); + LOG_PRINT_L2("block " << id << " found in m_invalid_blocks"); return true; } @@ -2384,7 +2521,7 @@ bool Blockchain::check_for_double_spend(const transaction& tx, key_images_contai return true; } //------------------------------------------------------------------ -bool Blockchain::get_tx_outputs_gindexs(const crypto::hash& tx_id, std::vector& indexs) const +bool Blockchain::get_tx_outputs_gindexs(const crypto::hash& tx_id, size_t n_txes, std::vector>& indexs) const { LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); @@ -2394,16 +2531,25 @@ bool Blockchain::get_tx_outputs_gindexs(const crypto::hash& tx_id, std::vectorget_tx_amount_output_indices(tx_index); - if (indexs.empty()) + return true; +} +//------------------------------------------------------------------ +bool Blockchain::get_tx_outputs_gindexs(const crypto::hash& tx_id, std::vector& indexs) const +{ + LOG_PRINT_L3("Blockchain::" << __func__); + CRITICAL_REGION_LOCAL(m_blockchain_lock); + uint64_t tx_index; + if (!m_db->tx_exists(tx_id, tx_index)) { - // empty indexs is only valid if the vout is empty, which is legal but rare - cryptonote::transaction tx = m_db->get_tx(tx_id); - CHECK_AND_ASSERT_MES(tx.vout.empty(), false, "internal error: global indexes for transaction " << tx_id << " is empty, and tx vout is not"); + MERROR_VER("get_tx_outputs_gindexs failed to find transaction with id = " << tx_id); + return false; } - + std::vector> indices = m_db->get_tx_amount_output_indices(tx_index, 1); + CHECK_AND_ASSERT_MES(indices.size() == 1, false, "Wrong indices size"); + indexs = indices.front(); return true; } //------------------------------------------------------------------ @@ -2454,7 +2600,7 @@ bool Blockchain::check_tx_inputs(transaction& tx, uint64_t& max_used_block_heigh if(m_show_time_stats) { size_t ring_size = !tx.vin.empty() && tx.vin[0].type() == typeid(txin_to_key) ? boost::get(tx.vin[0]).key_offsets.size() : 0; - MINFO("HASH: " << get_transaction_hash(tx) << " I/M/O: " << tx.vin.size() << "/" << ring_size << "/" << tx.vout.size() << " H: " << max_used_block_height << " ms: " << a + m_fake_scan_time << " B: " << get_object_blobsize(tx)); + MINFO("HASH: " << get_transaction_hash(tx) << " I/M/O: " << tx.vin.size() << "/" << ring_size << "/" << tx.vout.size() << " H: " << max_used_block_height << " ms: " << a + m_fake_scan_time << " B: " << get_object_blobsize(tx) << " W: " << get_transaction_weight(tx)); } if (!res) return false; @@ -2511,12 +2657,51 @@ bool Blockchain::check_tx_outputs(const transaction& tx, tx_verification_context // from v8, allow bulletproofs if (hf_version < 8) { - const bool bulletproof = tx.rct_signatures.type == rct::RCTTypeFullBulletproof || tx.rct_signatures.type == rct::RCTTypeSimpleBulletproof; - if (bulletproof || !tx.rct_signatures.p.bulletproofs.empty()) - { - MERROR("Bulletproofs are not allowed before v8"); - tvc.m_invalid_output = true; - return false; + if (tx.version >= 2) { + const bool bulletproof = rct::is_rct_bulletproof(tx.rct_signatures.type); + if (bulletproof || !tx.rct_signatures.p.bulletproofs.empty()) + { + MERROR_VER("Bulletproofs are not allowed before v8"); + tvc.m_invalid_output = true; + return false; + } + } + } + + // from v9, forbid borromean range proofs + if (hf_version > 8) { + if (tx.version >= 2) { + const bool borromean = rct::is_rct_borromean(tx.rct_signatures.type); + if (borromean) + { + MERROR_VER("Borromean range proofs are not allowed after v8"); + tvc.m_invalid_output = true; + return false; + } + } + } + + // from v10, allow bulletproofs v2 + if (hf_version < HF_VERSION_SMALLER_BP) { + if (tx.version >= 2) { + if (tx.rct_signatures.type == rct::RCTTypeBulletproof2) + { + MERROR_VER("Ringct type " << (unsigned)rct::RCTTypeBulletproof2 << " is not allowed before v" << HF_VERSION_SMALLER_BP); + tvc.m_invalid_output = true; + return false; + } + } + } + + // from v11, allow only bulletproofs v2 + if (hf_version > HF_VERSION_SMALLER_BP) { + if (tx.version >= 2) { + if (tx.rct_signatures.type == rct::RCTTypeBulletproof) + { + MERROR_VER("Ringct type " << (unsigned)rct::RCTTypeBulletproof << " is not allowed from v" << (HF_VERSION_SMALLER_BP + 1)); + tvc.m_invalid_output = true; + return false; + } } } @@ -2545,7 +2730,7 @@ bool Blockchain::expand_transaction_2(transaction &tx, const crypto::hash &tx_pr rv.message = rct::hash2rct(tx_prefix_hash); // mixRing - full and simple store it in opposite ways - if (rv.type == rct::RCTTypeFull || rv.type == rct::RCTTypeFullBulletproof) + if (rv.type == rct::RCTTypeFull) { CHECK_AND_ASSERT_MES(!pubkeys.empty() && !pubkeys[0].empty(), false, "empty pubkeys"); rv.mixRing.resize(pubkeys[0].size()); @@ -2560,7 +2745,7 @@ bool Blockchain::expand_transaction_2(transaction &tx, const crypto::hash &tx_pr } } } - else if (rv.type == rct::RCTTypeSimple || rv.type == rct::RCTTypeSimpleBulletproof) + else if (rv.type == rct::RCTTypeSimple || rv.type == rct::RCTTypeBulletproof || rv.type == rct::RCTTypeBulletproof2) { CHECK_AND_ASSERT_MES(!pubkeys.empty() && !pubkeys[0].empty(), false, "empty pubkeys"); rv.mixRing.resize(pubkeys.size()); @@ -2579,14 +2764,14 @@ bool Blockchain::expand_transaction_2(transaction &tx, const crypto::hash &tx_pr } // II - if (rv.type == rct::RCTTypeFull || rv.type == rct::RCTTypeFullBulletproof) + if (rv.type == rct::RCTTypeFull) { rv.p.MGs.resize(1); rv.p.MGs[0].II.resize(tx.vin.size()); for (size_t n = 0; n < tx.vin.size(); ++n) rv.p.MGs[0].II[n] = rct::ki2rct(boost::get(tx.vin[n]).k_image); } - else if (rv.type == rct::RCTTypeSimple || rv.type == rct::RCTTypeSimpleBulletproof) + else if (rv.type == rct::RCTTypeSimple || rv.type == rct::RCTTypeBulletproof || rv.type == rct::RCTTypeBulletproof2) { CHECK_AND_ASSERT_MES(rv.p.MGs.size() == tx.vin.size(), false, "Bad MGs size"); for (size_t n = 0; n < tx.vin.size(); ++n) @@ -2628,7 +2813,7 @@ bool Blockchain::check_tx_inputs(transaction& tx, tx_verification_context &tvc, { size_t n_unmixable = 0, n_mixable = 0; size_t mixin = std::numeric_limits::max(); - const size_t min_mixin = hf_version >= HF_VERSION_MIN_MIXIN_6 ? 6 : hf_version >= HF_VERSION_MIN_MIXIN_4 ? 4 : 2; + const size_t min_mixin = hf_version >= HF_VERSION_MIN_MIXIN_10 ? 10 : hf_version >= HF_VERSION_MIN_MIXIN_6 ? 6 : hf_version >= HF_VERSION_MIN_MIXIN_4 ? 4 : 2; for (const auto& txin : tx.vin) { // non txin_to_key inputs will be rejected below @@ -2657,6 +2842,13 @@ bool Blockchain::check_tx_inputs(transaction& tx, tx_verification_context &tvc, } } + if (((hf_version == HF_VERSION_MIN_MIXIN_10 || hf_version == HF_VERSION_MIN_MIXIN_10+1) && mixin != 10) || (hf_version >= HF_VERSION_MIN_MIXIN_10+2 && mixin > 10)) + { + MERROR_VER("Tx " << get_transaction_hash(tx) << " has invalid ring size (" << (mixin + 1) << "), it should be 11"); + tvc.m_low_mixin = true; + return false; + } + if (mixin < min_mixin) { if (n_unmixable == 0) @@ -2723,7 +2915,7 @@ bool Blockchain::check_tx_inputs(transaction& tx, tx_verification_context &tvc, tools::threadpool& tpool = tools::threadpool::getInstance(); tools::threadpool::waiter waiter; - const auto waiter_guard = epee::misc_utils::create_scope_leave_handler([&]() { waiter.wait(); }); + const auto waiter_guard = epee::misc_utils::create_scope_leave_handler([&]() { waiter.wait(&tpool); }); int threads = tpool.get_max_concurrency(); for (const auto& txin : tx.vin) @@ -2785,7 +2977,7 @@ bool Blockchain::check_tx_inputs(transaction& tx, tx_verification_context &tvc, { // ND: Speedup // 1. Thread ring signature verification if possible. - tpool.submit(&waiter, boost::bind(&Blockchain::check_ring_signature, this, std::cref(tx_prefix_hash), std::cref(in_to_key.k_image), std::cref(pubkeys[sig_index]), std::cref(tx.signatures[sig_index]), std::ref(results[sig_index]))); + tpool.submit(&waiter, boost::bind(&Blockchain::check_ring_signature, this, std::cref(tx_prefix_hash), std::cref(in_to_key.k_image), std::cref(pubkeys[sig_index]), std::cref(tx.signatures[sig_index]), std::ref(results[sig_index])), true); } else { @@ -2809,7 +3001,7 @@ bool Blockchain::check_tx_inputs(transaction& tx, tx_verification_context &tvc, sig_index++; } if (tx.version == 1 && threads > 1) - waiter.wait(); + waiter.wait(&tpool); if (tx.version == 1) { @@ -2852,7 +3044,8 @@ bool Blockchain::check_tx_inputs(transaction& tx, tx_verification_context &tvc, return false; } case rct::RCTTypeSimple: - case rct::RCTTypeSimpleBulletproof: + case rct::RCTTypeBulletproof: + case rct::RCTTypeBulletproof2: { // check all this, either reconstructed (so should really pass), or not { @@ -2902,7 +3095,7 @@ bool Blockchain::check_tx_inputs(transaction& tx, tx_verification_context &tvc, } } - if (!rct::verRctSimple(rv, false)) + if (!rct::verRctNonSemanticsSimple(rv)) { MERROR_VER("Failed to check ringct signatures!"); return false; @@ -2910,7 +3103,6 @@ bool Blockchain::check_tx_inputs(transaction& tx, tx_verification_context &tvc, break; } case rct::RCTTypeFull: - case rct::RCTTypeFullBulletproof: { // check all this, either reconstructed (so should really pass), or not { @@ -2973,6 +3165,22 @@ bool Blockchain::check_tx_inputs(transaction& tx, tx_verification_context &tvc, MERROR_VER("Unsupported rct type: " << rv.type); return false; } + + // for bulletproofs, check they're only multi-output after v8 + if (rct::is_rct_bulletproof(rv.type)) + { + if (hf_version < 8) + { + for (const rct::Bulletproof &proof: rv.p.bulletproofs) + { + if (proof.V.size() > 1) + { + MERROR_VER("Multi output bulletproofs are invalid before v8"); + return false; + } + } + } + } } return true; } @@ -2981,6 +3189,7 @@ bool Blockchain::check_tx_inputs(transaction& tx, tx_verification_context &tvc, void Blockchain::check_ring_signature(const crypto::hash &tx_prefix_hash, const crypto::key_image &key_image, const std::vector &pubkeys, const std::vector& sig, uint64_t &result) { std::vector p_output_keys; + p_output_keys.reserve(pubkeys.size()); for (auto &key : pubkeys) { // rct::key and crypto::public_key have the same structure, avoid object ctor/memcpy @@ -2991,7 +3200,7 @@ void Blockchain::check_ring_signature(const crypto::hash &tx_prefix_hash, const } //------------------------------------------------------------------ -static uint64_t get_fee_quantization_mask() +uint64_t Blockchain::get_fee_quantization_mask() { static uint64_t mask = 0; if (mask == 0) @@ -3004,16 +3213,27 @@ static uint64_t get_fee_quantization_mask() } //------------------------------------------------------------------ -uint64_t Blockchain::get_dynamic_per_kb_fee(uint64_t block_reward, size_t median_block_size, uint8_t version) +uint64_t Blockchain::get_dynamic_base_fee(uint64_t block_reward, size_t median_block_weight, uint8_t version) { - const uint64_t min_block_size = get_min_block_size(version); - const uint64_t fee_per_kb_base = version >= 5 ? DYNAMIC_FEE_PER_KB_BASE_FEE_V5 : DYNAMIC_FEE_PER_KB_BASE_FEE; + const uint64_t min_block_weight = get_min_block_weight(version); + if (median_block_weight < min_block_weight) + median_block_weight = min_block_weight; + uint64_t hi, lo; + + if (version >= HF_VERSION_PER_BYTE_FEE) + { + lo = mul128(block_reward, DYNAMIC_FEE_REFERENCE_TRANSACTION_WEIGHT, &hi); + div128_32(hi, lo, min_block_weight, &hi, &lo); + div128_32(hi, lo, median_block_weight, &hi, &lo); + assert(hi == 0); + lo /= 5; + return lo; + } - if (median_block_size < min_block_size) - median_block_size = min_block_size; + const uint64_t fee_base = version >= 5 ? DYNAMIC_FEE_PER_KB_BASE_FEE_V5 : DYNAMIC_FEE_PER_KB_BASE_FEE; - uint64_t unscaled_fee_per_kb = (fee_per_kb_base * min_block_size / median_block_size); - uint64_t hi, lo = mul128(unscaled_fee_per_kb, block_reward, &hi); + uint64_t unscaled_fee_base = (fee_base * min_block_weight / median_block_weight); + lo = mul128(unscaled_fee_base, block_reward, &hi); static_assert(DYNAMIC_FEE_PER_KB_BASE_BLOCK_REWARD % 1000000 == 0, "DYNAMIC_FEE_PER_KB_BASE_BLOCK_REWARD must be divisible by 1000000"); static_assert(DYNAMIC_FEE_PER_KB_BASE_BLOCK_REWARD / 1000000 <= std::numeric_limits::max(), "DYNAMIC_FEE_PER_KB_BASE_BLOCK_REWARD is too large"); @@ -3031,29 +3251,51 @@ uint64_t Blockchain::get_dynamic_per_kb_fee(uint64_t block_reward, size_t median } //------------------------------------------------------------------ -bool Blockchain::check_fee(size_t blob_size, uint64_t fee) const +bool Blockchain::check_fee(size_t tx_weight, uint64_t fee) const { const uint8_t version = get_current_hard_fork_version(); + const uint64_t blockchain_height = m_db->height(); - uint64_t fee_per_kb; - if (version < HF_VERSION_DYNAMIC_FEE) - { - fee_per_kb = FEE_PER_KB; - } - else + uint64_t median = 0; + uint64_t already_generated_coins = 0; + uint64_t base_reward = 0; + if (version >= HF_VERSION_DYNAMIC_FEE) { - uint64_t median = m_current_block_cumul_sz_limit / 2; - uint64_t already_generated_coins = m_db->height() ? m_db->get_block_already_generated_coins(m_db->height() - 1) : 0; - uint64_t base_reward; + median = m_current_block_cumul_weight_limit / 2; + const uint64_t blockchain_height = m_db->height(); + already_generated_coins = blockchain_height ? m_db->get_block_already_generated_coins(blockchain_height - 1) : 0; if (!get_block_reward(median, 1, already_generated_coins, base_reward, version)) return false; - fee_per_kb = get_dynamic_per_kb_fee(base_reward, median, version); } - MDEBUG("Using " << print_money(fee_per_kb) << "/kB fee"); - uint64_t needed_fee = blob_size / 1024; - needed_fee += (blob_size % 1024) ? 1 : 0; - needed_fee *= fee_per_kb; + uint64_t needed_fee; + if (version >= HF_VERSION_PER_BYTE_FEE) + { + const bool use_long_term_median_in_fee = version >= HF_VERSION_LONG_TERM_BLOCK_WEIGHT; + uint64_t fee_per_byte = get_dynamic_base_fee(base_reward, use_long_term_median_in_fee ? m_long_term_effective_median_block_weight : median, version); + MDEBUG("Using " << print_money(fee_per_byte) << "/byte fee"); + needed_fee = tx_weight * fee_per_byte; + // quantize fee up to 8 decimals + const uint64_t mask = get_fee_quantization_mask(); + needed_fee = (needed_fee + mask - 1) / mask * mask; + } + else + { + uint64_t fee_per_kb; + if (version < HF_VERSION_DYNAMIC_FEE) + { + fee_per_kb = FEE_PER_KB; + } + else + { + fee_per_kb = get_dynamic_base_fee(base_reward, median, version); + } + MDEBUG("Using " << print_money(fee_per_kb) << "/kB fee"); + + needed_fee = tx_weight / 1024; + needed_fee += (tx_weight % 1024) ? 1 : 0; + needed_fee *= fee_per_kb; + } if (fee < needed_fee - needed_fee / 50) // keep a little 2% buffer on acceptance - no integer overflow { @@ -3064,9 +3306,10 @@ bool Blockchain::check_fee(size_t blob_size, uint64_t fee) const } //------------------------------------------------------------------ -uint64_t Blockchain::get_dynamic_per_kb_fee_estimate(uint64_t grace_blocks) const +uint64_t Blockchain::get_dynamic_base_fee_estimate(uint64_t grace_blocks) const { const uint8_t version = get_current_hard_fork_version(); + const uint64_t db_height = m_db->height(); if (version < HF_VERSION_DYNAMIC_FEE) return FEE_PER_KB; @@ -3074,17 +3317,18 @@ uint64_t Blockchain::get_dynamic_per_kb_fee_estimate(uint64_t grace_blocks) cons if (grace_blocks >= CRYPTONOTE_REWARD_BLOCKS_WINDOW) grace_blocks = CRYPTONOTE_REWARD_BLOCKS_WINDOW - 1; - const uint64_t min_block_size = get_min_block_size(version); - std::vector sz; - get_last_n_blocks_sizes(sz, CRYPTONOTE_REWARD_BLOCKS_WINDOW - grace_blocks); + const uint64_t min_block_weight = get_min_block_weight(version); + std::vector weights; + get_last_n_blocks_weights(weights, CRYPTONOTE_REWARD_BLOCKS_WINDOW - grace_blocks); + weights.reserve(grace_blocks); for (size_t i = 0; i < grace_blocks; ++i) - sz.push_back(min_block_size); + weights.push_back(min_block_weight); - uint64_t median = epee::misc_utils::median(sz); - if(median <= min_block_size) - median = min_block_size; + uint64_t median = epee::misc_utils::median(weights); + if(median <= min_block_weight) + median = min_block_weight; - uint64_t already_generated_coins = m_db->height() ? m_db->get_block_already_generated_coins(m_db->height() - 1) : 0; + uint64_t already_generated_coins = db_height ? m_db->get_block_already_generated_coins(db_height - 1) : 0; uint64_t base_reward; if (!get_block_reward(median, 1, already_generated_coins, base_reward, version)) { @@ -3092,8 +3336,10 @@ uint64_t Blockchain::get_dynamic_per_kb_fee_estimate(uint64_t grace_blocks) cons base_reward = BLOCK_REWARD_OVERESTIMATE; } - uint64_t fee = get_dynamic_per_kb_fee(base_reward, median, version); - MDEBUG("Estimating " << grace_blocks << "-block fee at " << print_money(fee) << "/kB"); + const bool use_long_term_median_in_fee = version >= HF_VERSION_LONG_TERM_BLOCK_WEIGHT; + uint64_t fee = get_dynamic_base_fee(base_reward, use_long_term_median_in_fee ? m_long_term_effective_median_block_weight : median, version); + const bool per_byte = version < HF_VERSION_PER_BYTE_FEE; + MDEBUG("Estimating " << grace_blocks << "-block fee at " << print_money(fee) << "/" << (per_byte ? "byte" : "kB")); return fee; } @@ -3223,17 +3469,19 @@ bool Blockchain::check_block_timestamp(const block& b, uint64_t& median_ts) cons return false; } + const auto h = m_db->height(); + // if not enough blocks, no proper median yet, return true - if(m_db->height() < BLOCKCHAIN_TIMESTAMP_CHECK_WINDOW) + if(h < BLOCKCHAIN_TIMESTAMP_CHECK_WINDOW) { return true; } std::vector timestamps; - auto h = m_db->height(); // need most recent 60 blocks, get index of first of those size_t offset = h - BLOCKCHAIN_TIMESTAMP_CHECK_WINDOW; + timestamps.reserve(h - offset); for(;offset < h; ++offset) { timestamps.push_back(m_db->get_block_timestamp(offset)); @@ -3242,7 +3490,7 @@ bool Blockchain::check_block_timestamp(const block& b, uint64_t& median_ts) cons return check_block_timestamp(timestamps, b, median_ts); } //------------------------------------------------------------------ -void Blockchain::return_tx_to_pool(std::vector &txs) +void Blockchain::return_tx_to_pool(std::vector> &txs) { uint8_t version = get_current_hard_fork_version(); for (auto& tx : txs) @@ -3253,14 +3501,16 @@ void Blockchain::return_tx_to_pool(std::vector &txs) // that might not be always true. Unlikely though, and always relaying // these again might cause a spike of traffic as many nodes re-relay // all the transactions in a popped block when a reorg happens. - if (!m_tx_pool.add_tx(tx, tvc, true, true, false, version)) + const size_t weight = get_transaction_weight(tx.first, tx.second.size()); + const crypto::hash tx_hash = get_transaction_hash(tx.first); + if (!m_tx_pool.add_tx(tx.first, tx_hash, tx.second, weight, tvc, true, true, false, version)) { - MERROR("Failed to return taken transaction with hash: " << get_transaction_hash(tx) << " to tx_pool"); + MERROR("Failed to return taken transaction with hash: " << get_transaction_hash(tx.first) << " to tx_pool"); } } } //------------------------------------------------------------------ -bool Blockchain::flush_txes_from_pool(const std::list &txids) +bool Blockchain::flush_txes_from_pool(const std::vector &txids) { CRITICAL_REGION_LOCAL(m_tx_pool); @@ -3268,11 +3518,12 @@ bool Blockchain::flush_txes_from_pool(const std::list &txids) for (const auto &txid: txids) { cryptonote::transaction tx; - size_t blob_size; + cryptonote::blobdata txblob; + size_t tx_weight; uint64_t fee; bool relayed, do_not_relay, double_spend_seen; MINFO("Removing txid " << txid << " from the pool"); - if(m_tx_pool.have_tx(txid) && !m_tx_pool.take_tx(txid, tx, blob_size, fee, relayed, do_not_relay, double_spend_seen)) + if(m_tx_pool.have_tx(txid) && !m_tx_pool.take_tx(txid, tx, txblob, tx_weight, fee, relayed, do_not_relay, double_spend_seen)) { MERROR("Failed to remove txid " << txid << " from the pool"); res = false; @@ -3294,12 +3545,15 @@ bool Blockchain::handle_block_to_main_chain(const block& bl, const crypto::hash& static bool seen_future_version = false; - m_db->block_txn_start(true); - if(bl.prev_id != get_tail_id()) + db_rtxn_guard rtxn_guard(m_db); + uint64_t blockchain_height; + const crypto::hash top_hash = get_tail_id(blockchain_height); + ++blockchain_height; // block height to chain height + if(bl.prev_id != top_hash) { - MERROR_VER("Block with id: " << id << std::endl << "has wrong prev_id: " << bl.prev_id << std::endl << "expected: " << get_tail_id()); + MERROR_VER("Block with id: " << id << std::endl << "has wrong prev_id: " << bl.prev_id << std::endl << "expected: " << top_hash); + bvc.m_verifivation_failed = true; leave: - m_db->block_txn_stop(); return false; } @@ -3365,15 +3619,14 @@ bool Blockchain::handle_block_to_main_chain(const block& bl, const crypto::hash& bool precomputed = false; bool fast_check = false; #if defined(PER_BLOCK_CHECKPOINT) - if (m_db->height() < m_blocks_hash_check.size()) + if (blockchain_height < m_blocks_hash_check.size()) { - auto hash = get_block_hash(bl); - const auto &expected_hash = m_blocks_hash_check[m_db->height()]; + const auto &expected_hash = m_blocks_hash_check[blockchain_height]; if (expected_hash != crypto::null_hash) { - if (memcmp(&hash, &expected_hash, sizeof(hash)) != 0) + if (memcmp(&id, &expected_hash, sizeof(hash)) != 0) { - MERROR_VER("Block with id is INVALID: " << id); + MERROR_VER("Block with id is INVALID: " << id << ", expected " << expected_hash); bvc.m_verifivation_failed = true; goto leave; } @@ -3381,7 +3634,7 @@ bool Blockchain::handle_block_to_main_chain(const block& bl, const crypto::hash& } else { - MCINFO("verify", "No pre-validated hash at height " << m_db->height() << ", verifying fully"); + MCINFO("verify", "No pre-validated hash at height " << blockchain_height << ", verifying fully"); } } else @@ -3394,12 +3647,12 @@ bool Blockchain::handle_block_to_main_chain(const block& bl, const crypto::hash& proof_of_work = it->second; } else - proof_of_work = get_block_longhash(bl, m_db->height()); + proof_of_work = get_block_longhash(bl, blockchain_height); // validate proof_of_work versus difficulty target if(!check_hash(proof_of_work, current_diffic)) { - MERROR_VER("Block with id: " << id << std::endl << "does not have enough proof of work: " << proof_of_work << std::endl << "unexpected difficulty: " << current_diffic); + MERROR_VER("Block with id: " << id << std::endl << "does not have enough proof of work: " << proof_of_work << " at height " << blockchain_height << ", unexpected difficulty: " << current_diffic); bvc.m_verifivation_failed = true; goto leave; } @@ -3407,9 +3660,9 @@ bool Blockchain::handle_block_to_main_chain(const block& bl, const crypto::hash& // If we're at a checkpoint, ensure that our hardcoded checkpoint hash // is correct. - if(m_checkpoints.is_in_checkpoint_zone(get_current_blockchain_height())) + if(m_checkpoints.is_in_checkpoint_zone(blockchain_height)) { - if(!m_checkpoints.check_block(get_current_blockchain_height(), id)) + if(!m_checkpoints.check_block(blockchain_height, id)) { LOG_ERROR("CHECKPOINT VALIDATION FAILED"); bvc.m_verifivation_failed = true; @@ -3424,17 +3677,17 @@ bool Blockchain::handle_block_to_main_chain(const block& bl, const crypto::hash& TIME_MEASURE_START(t3); // sanity check basic miner tx properties; - if(!prevalidate_miner_transaction(bl, m_db->height())) + if(!prevalidate_miner_transaction(bl, blockchain_height)) { MERROR_VER("Block with id: " << id << " failed to pass prevalidation"); bvc.m_verifivation_failed = true; goto leave; } - size_t coinbase_blob_size = get_object_blobsize(bl.miner_tx); - size_t cumulative_block_size = coinbase_blob_size; + size_t coinbase_weight = get_transaction_weight(bl.miner_tx); + size_t cumulative_block_weight = coinbase_weight; - std::vector txs; + std::vector> txs; key_images_container keys; uint64_t fee_summary = 0; @@ -3450,10 +3703,12 @@ bool Blockchain::handle_block_to_main_chain(const block& bl, const crypto::hash& // Iterate over the block's transaction hashes, grabbing each // from the tx_pool and validating them. Each is then added // to txs. Keys spent in each are added to by the double spend check. + txs.reserve(bl.tx_hashes.size()); for (const crypto::hash& tx_id : bl.tx_hashes) { - transaction tx; - size_t blob_size = 0; + transaction tx_tmp; + blobdata txblob; + size_t tx_weight = 0; uint64_t fee = 0; bool relayed = false, do_not_relay = false, double_spend_seen = false; TIME_MEASURE_START(aa); @@ -3472,7 +3727,7 @@ bool Blockchain::handle_block_to_main_chain(const block& bl, const crypto::hash& TIME_MEASURE_START(bb); // get transaction with hash from tx_pool - if(!m_tx_pool.take_tx(tx_id, tx, blob_size, fee, relayed, do_not_relay, double_spend_seen)) + if(!m_tx_pool.take_tx(tx_id, tx_tmp, txblob, tx_weight, fee, relayed, do_not_relay, double_spend_seen)) { MERROR_VER("Block with id: " << id << " has at least one unknown transaction with id: " << tx_id); bvc.m_verifivation_failed = true; @@ -3485,7 +3740,8 @@ bool Blockchain::handle_block_to_main_chain(const block& bl, const crypto::hash& // add the transaction to the temp list of transactions, so we can either // store the list of transactions all at once or return the ones we've // taken from the tx_pool back to it if the block fails verification. - txs.push_back(tx); + txs.push_back(std::make_pair(std::move(tx_tmp), std::move(txblob))); + transaction &tx = txs.back().first; TIME_MEASURE_START(dd); // FIXME: the storage should not be responsible for validation. @@ -3518,6 +3774,8 @@ bool Blockchain::handle_block_to_main_chain(const block& bl, const crypto::hash& //TODO: why is this done? make sure that keeping invalid blocks makes sense. add_block_as_invalid(bl, id); MERROR_VER("Block with id " << id << " added as invalid because of wrong inputs in transactions"); + MERROR_VER("tx_index " << tx_index << ", m_blocks_txs_check " << m_blocks_txs_check.size() << ":"); + for (const auto &h: m_blocks_txs_check) MERROR_VER(" " << h); bvc.m_verifivation_failed = true; return_tx_to_pool(txs); goto leave; @@ -3543,15 +3801,15 @@ bool Blockchain::handle_block_to_main_chain(const block& bl, const crypto::hash& TIME_MEASURE_FINISH(cc); t_checktx += cc; fee_summary += fee; - cumulative_block_size += blob_size; + cumulative_block_weight += tx_weight; } m_blocks_txs_check.clear(); TIME_MEASURE_START(vmt); uint64_t base_reward = 0; - uint64_t already_generated_coins = m_db->height() ? m_db->get_block_already_generated_coins(m_db->height() - 1) : 0; - if(!validate_miner_transaction(bl, cumulative_block_size, fee_summary, base_reward, already_generated_coins, bvc.m_partial_block_reward, m_hardfork->get_current_version())) + uint64_t already_generated_coins = blockchain_height ? m_db->get_block_already_generated_coins(blockchain_height - 1) : 0; + if(!validate_miner_transaction(bl, cumulative_block_weight, fee_summary, base_reward, already_generated_coins, bvc.m_partial_block_reward, m_hardfork->get_current_version())) { MERROR_VER("Block with id: " << id << " has incorrect miner transaction"); bvc.m_verifivation_failed = true; @@ -3560,36 +3818,39 @@ bool Blockchain::handle_block_to_main_chain(const block& bl, const crypto::hash& } TIME_MEASURE_FINISH(vmt); - size_t block_size; + size_t block_weight; difficulty_type cumulative_difficulty; // populate various metadata about the block to be stored alongside it. - block_size = cumulative_block_size; + block_weight = cumulative_block_weight; cumulative_difficulty = current_diffic; // In the "tail" state when the minimum subsidy (implemented in get_block_reward) is in effect, the number of // coins will eventually exceed MONEY_SUPPLY and overflow a uint64. To prevent overflow, cap already_generated_coins // at MONEY_SUPPLY. already_generated_coins is only used to compute the block subsidy and MONEY_SUPPLY yields a // subsidy of 0 under the base formula and therefore the minimum subsidy >0 in the tail state. already_generated_coins = base_reward < (MONEY_SUPPLY-already_generated_coins) ? already_generated_coins + base_reward : MONEY_SUPPLY; - if(m_db->height()) - cumulative_difficulty += m_db->get_block_cumulative_difficulty(m_db->height() - 1); + if(blockchain_height) + cumulative_difficulty += m_db->get_block_cumulative_difficulty(blockchain_height - 1); TIME_MEASURE_FINISH(block_processing_time); if(precomputed) block_processing_time += m_fake_pow_calc_time; - m_db->block_txn_stop(); + rtxn_guard.stop(); TIME_MEASURE_START(addblock); uint64_t new_height = 0; if (!bvc.m_verifivation_failed) { try { - new_height = m_db->add_block(bl, block_size, cumulative_difficulty, already_generated_coins, txs); + uint64_t long_term_block_weight = get_next_long_term_block_weight(block_weight); + cryptonote::blobdata bd = cryptonote::block_to_blob(bl); + new_height = m_db->add_block(std::make_pair(std::move(bl), std::move(bd)), block_weight, long_term_block_weight, cumulative_difficulty, already_generated_coins, txs); } catch (const KEY_IMAGE_EXISTS& e) { LOG_ERROR("Error adding block with hash: " << id << " to blockchain, what = " << e.what()); + m_batch_success = false; bvc.m_verifivation_failed = true; return_tx_to_pool(txs); return false; @@ -3598,6 +3859,8 @@ bool Blockchain::handle_block_to_main_chain(const block& bl, const crypto::hash& { //TODO: figure out the best way to deal with this failure LOG_ERROR("Error adding block with hash: " << id << " to blockchain, what = " << e.what()); + m_batch_success = false; + bvc.m_verifivation_failed = true; return_tx_to_pool(txs); return false; } @@ -3609,14 +3872,19 @@ bool Blockchain::handle_block_to_main_chain(const block& bl, const crypto::hash& TIME_MEASURE_FINISH(addblock); - // do this after updating the hard fork state since the size limit may change due to fork - update_next_cumulative_size_limit(); + // do this after updating the hard fork state since the weight limit may change due to fork + if (!update_next_cumulative_weight_limit()) + { + MERROR("Failed to update next cumulative weight limit"); + pop_block_from_blockchain(); + return false; + } - MINFO("+++++ BLOCK SUCCESSFULLY ADDED" << std::endl << "id:\t" << id << std::endl << "PoW:\t" << proof_of_work << std::endl << "HEIGHT " << new_height-1 << ", difficulty:\t" << current_diffic << std::endl << "block reward: " << print_money(fee_summary + base_reward) << "(" << print_money(base_reward) << " + " << print_money(fee_summary) << "), coinbase_blob_size: " << coinbase_blob_size << ", cumulative size: " << cumulative_block_size << ", " << block_processing_time << "(" << target_calculating_time << "/" << longhash_calculating_time << ")ms"); + MINFO("+++++ BLOCK SUCCESSFULLY ADDED" << std::endl << "id:\t" << id << std::endl << "PoW:\t" << proof_of_work << std::endl << "HEIGHT " << new_height-1 << ", difficulty:\t" << current_diffic << std::endl << "block reward: " << print_money(fee_summary + base_reward) << "(" << print_money(base_reward) << " + " << print_money(fee_summary) << "), coinbase_weight: " << coinbase_weight << ", cumulative weight: " << cumulative_block_weight << ", " << block_processing_time << "(" << target_calculating_time << "/" << longhash_calculating_time << ")ms"); if(m_show_time_stats) { - MINFO("Height: " << new_height << " blob: " << coinbase_blob_size << " cumm: " - << cumulative_block_size << " p/t: " << block_processing_time << " (" + MINFO("Height: " << new_height << " coinbase weight: " << coinbase_weight << " cumm: " + << cumulative_block_weight << " p/t: " << block_processing_time << " (" << target_calculating_time << "/" << longhash_calculating_time << "/" << t1 << "/" << t2 << "/" << t3 << "/" << t_exists << "/" << t_pool << "/" << t_checktx << "/" << t_dblspnd << "/" << vmt << "/" << addblock << ")ms"); @@ -3627,41 +3895,150 @@ bool Blockchain::handle_block_to_main_chain(const block& bl, const crypto::hash& // appears to be a NOP *and* is called elsewhere. wat? m_tx_pool.on_blockchain_inc(new_height, id); + get_difficulty_for_next_block(); // just to cache it + invalidate_block_template_cache(); + + std::shared_ptr block_notify = m_block_notify; + if (block_notify) + block_notify->notify("%s", epee::string_tools::pod_to_hex(id).c_str(), NULL); return true; } //------------------------------------------------------------------ -bool Blockchain::update_next_cumulative_size_limit() +bool Blockchain::prune_blockchain(uint32_t pruning_seed) +{ + m_tx_pool.lock(); + epee::misc_utils::auto_scope_leave_caller unlocker = epee::misc_utils::create_scope_leave_handler([&](){m_tx_pool.unlock();}); + CRITICAL_REGION_LOCAL(m_blockchain_lock); + + return m_db->prune_blockchain(pruning_seed); +} +//------------------------------------------------------------------ +bool Blockchain::update_blockchain_pruning() +{ + m_tx_pool.lock(); + epee::misc_utils::auto_scope_leave_caller unlocker = epee::misc_utils::create_scope_leave_handler([&](){m_tx_pool.unlock();}); + CRITICAL_REGION_LOCAL(m_blockchain_lock); + + return m_db->update_pruning(); +} +//------------------------------------------------------------------ +bool Blockchain::check_blockchain_pruning() +{ + m_tx_pool.lock(); + epee::misc_utils::auto_scope_leave_caller unlocker = epee::misc_utils::create_scope_leave_handler([&](){m_tx_pool.unlock();}); + CRITICAL_REGION_LOCAL(m_blockchain_lock); + + return m_db->check_pruning(); +} +//------------------------------------------------------------------ +uint64_t Blockchain::get_next_long_term_block_weight(uint64_t block_weight) const +{ + PERF_TIMER(get_next_long_term_block_weight); + + const uint64_t db_height = m_db->height(); + const uint64_t nblocks = std::min(m_long_term_block_weights_window, db_height); + + const uint8_t hf_version = get_current_hard_fork_version(); + if (hf_version < HF_VERSION_LONG_TERM_BLOCK_WEIGHT) + return block_weight; + + uint64_t long_term_median = get_long_term_block_weight_median(db_height - nblocks, nblocks); + uint64_t long_term_effective_median_block_weight = std::max(CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V5, long_term_median); + + uint64_t short_term_constraint = long_term_effective_median_block_weight + long_term_effective_median_block_weight * 2 / 5; + uint64_t long_term_block_weight = std::min(block_weight, short_term_constraint); + + return long_term_block_weight; +} +//------------------------------------------------------------------ +bool Blockchain::update_next_cumulative_weight_limit(uint64_t *long_term_effective_median_block_weight) { - uint64_t full_reward_zone = get_min_block_size(get_current_hard_fork_version()); + PERF_TIMER(update_next_cumulative_weight_limit); LOG_PRINT_L3("Blockchain::" << __func__); - std::vector sz; - get_last_n_blocks_sizes(sz, CRYPTONOTE_REWARD_BLOCKS_WINDOW); - uint64_t median = epee::misc_utils::median(sz); - m_current_block_cumul_sz_median = median; - if(median <= full_reward_zone) - median = full_reward_zone; + // when we reach this, the last hf version is not yet written to the db + const uint64_t db_height = m_db->height(); + const uint8_t hf_version = get_current_hard_fork_version(); + uint64_t full_reward_zone = get_min_block_weight(hf_version); + uint64_t long_term_block_weight; + + if (hf_version < HF_VERSION_LONG_TERM_BLOCK_WEIGHT) + { + std::vector weights; + get_last_n_blocks_weights(weights, CRYPTONOTE_REWARD_BLOCKS_WINDOW); + m_current_block_cumul_weight_median = epee::misc_utils::median(weights); + long_term_block_weight = weights.back(); + } + else + { + const uint64_t block_weight = m_db->get_block_weight(db_height - 1); + + uint64_t long_term_median; + if (db_height == 1) + { + long_term_median = CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V5; + } + else + { + uint64_t nblocks = std::min(m_long_term_block_weights_window, db_height); + if (nblocks == db_height) + --nblocks; + long_term_median = get_long_term_block_weight_median(db_height - nblocks - 1, nblocks); + } + + m_long_term_effective_median_block_weight = std::max(CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V5, long_term_median); + + uint64_t short_term_constraint = m_long_term_effective_median_block_weight + m_long_term_effective_median_block_weight * 2 / 5; + long_term_block_weight = std::min(block_weight, short_term_constraint); + + if (db_height == 1) + { + long_term_median = long_term_block_weight; + } + else + { + m_long_term_block_weights_cache_tip_hash = m_db->get_block_hash_from_height(db_height - 1); + m_long_term_block_weights_cache_rolling_median.insert(long_term_block_weight); + long_term_median = m_long_term_block_weights_cache_rolling_median.median(); + } + m_long_term_effective_median_block_weight = std::max(CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V5, long_term_median); + + std::vector weights; + get_last_n_blocks_weights(weights, CRYPTONOTE_REWARD_BLOCKS_WINDOW); + + uint64_t short_term_median = epee::misc_utils::median(weights); + uint64_t effective_median_block_weight = std::min(std::max(CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V5, short_term_median), CRYPTONOTE_SHORT_TERM_BLOCK_WEIGHT_SURGE_FACTOR * m_long_term_effective_median_block_weight); + + m_current_block_cumul_weight_median = effective_median_block_weight; + } + + if (m_current_block_cumul_weight_median <= full_reward_zone) + m_current_block_cumul_weight_median = full_reward_zone; + + m_current_block_cumul_weight_limit = m_current_block_cumul_weight_median * 2; + + if (long_term_effective_median_block_weight) + *long_term_effective_median_block_weight = m_long_term_effective_median_block_weight; + + if (!m_db->is_read_only()) + m_db->add_max_block_size(m_current_block_cumul_weight_limit); - m_current_block_cumul_sz_limit = median*2; return true; } //------------------------------------------------------------------ -bool Blockchain::add_new_block(const block& bl_, block_verification_context& bvc) +bool Blockchain::add_new_block(const block& bl, block_verification_context& bvc) { LOG_PRINT_L3("Blockchain::" << __func__); - //copy block here to let modify block.target - block bl = bl_; crypto::hash id = get_block_hash(bl); CRITICAL_REGION_LOCAL(m_tx_pool);//to avoid deadlock lets lock tx_pool for whole add/reorganize process CRITICAL_REGION_LOCAL1(m_blockchain_lock); - m_db->block_txn_start(true); + db_rtxn_guard rtxn_guard(m_db); if(have_block(id)) { LOG_PRINT_L3("block with id = " << id << " already exists"); bvc.m_already_exists = true; - m_db->block_txn_stop(); m_blocks_txs_check.clear(); return false; } @@ -3671,14 +4048,14 @@ bool Blockchain::add_new_block(const block& bl_, block_verification_context& bvc { //chain switching or wrong block bvc.m_added_to_main_chain = false; - m_db->block_txn_stop(); + rtxn_guard.stop(); bool r = handle_alternative_block(bl, id, bvc); m_blocks_txs_check.clear(); return r; //never relay alternative blocks } - m_db->block_txn_stop(); + rtxn_guard.stop(); return handle_block_to_main_chain(bl, id, bvc); } //------------------------------------------------------------------ @@ -3691,10 +4068,11 @@ void Blockchain::check_against_checkpoints(const checkpoints& points, bool enfor CRITICAL_REGION_LOCAL(m_blockchain_lock); stop_batch = m_db->batch_start(); + const uint64_t blockchain_height = m_db->height(); for (const auto& pt : pts) { // if the checkpoint is for a block we don't have yet, move on - if (pt.first >= m_db->height()) + if (pt.first >= blockchain_height) { continue; } @@ -3762,7 +4140,7 @@ void Blockchain::set_enforce_dns_checkpoints(bool enforce_checkpoints) } //------------------------------------------------------------------ -void Blockchain::block_longhash_worker(uint64_t height, const std::vector &blocks, std::unordered_map &map) const +void Blockchain::block_longhash_worker(uint64_t height, const epee::span &blocks, std::unordered_map &map) const { TIME_MEASURE_START(t); slow_hash_allocate_state(); @@ -3791,7 +4169,10 @@ bool Blockchain::cleanup_handle_incoming_blocks(bool force_sync) try { - m_db->batch_stop(); + if (m_batch_success) + m_db->batch_stop(); + else + m_db->batch_abort(); success = true; } catch (const std::exception &e) @@ -3807,11 +4188,13 @@ bool Blockchain::cleanup_handle_incoming_blocks(bool force_sync) store_blockchain(); m_sync_counter = 0; } - else if (m_db_blocks_per_sync && m_sync_counter >= m_db_blocks_per_sync) + else if (m_db_sync_threshold && ((m_db_sync_on_blocks && m_sync_counter >= m_db_sync_threshold) || (!m_db_sync_on_blocks && m_bytes_to_sync >= m_db_sync_threshold))) { + MDEBUG("Sync threshold met, syncing"); if(m_db_sync_mode == db_async) { m_sync_counter = 0; + m_bytes_to_sync = 0; m_async_service.dispatch(boost::bind(&Blockchain::store_blockchain, this)); } else if(m_db_sync_mode == db_sync) @@ -3842,16 +4225,17 @@ bool Blockchain::cleanup_handle_incoming_blocks(bool force_sync) CRITICAL_REGION_END(); m_tx_pool.unlock(); + update_blockchain_pruning(); + return success; } //------------------------------------------------------------------ -//FIXME: unused parameter txs -void Blockchain::output_scan_worker(const uint64_t amount, const std::vector &offsets, std::vector &outputs, std::unordered_map &txs) const +void Blockchain::output_scan_worker(const uint64_t amount, const std::vector &offsets, std::vector &outputs) const { try { - m_db->get_output_key(amount, offsets, outputs, true); + m_db->get_output_key(epee::span(&amount, 1), offsets, outputs, true); } catch (const std::exception& e) { @@ -3863,7 +4247,7 @@ void Blockchain::output_scan_worker(const uint64_t amount, const std::vector &hashes) +uint64_t Blockchain::prevalidate_block_hashes(uint64_t height, const std::vector &hashes) { // new: . . . . . X X X X X . . . . . . // pre: A A A A B B B B C C C C D D D D @@ -3966,12 +4350,14 @@ uint64_t Blockchain::prevalidate_block_hashes(uint64_t height, const std::list &blocks_entry) +bool Blockchain::prepare_handle_incoming_blocks(const std::vector &blocks_entry, std::vector &blocks) { MTRACE("Blockchain::" << __func__); TIME_MEASURE_START(prepare); bool stop_batch; uint64_t bytes = 0; + size_t total_txs = 0; + blocks.clear(); // Order of locking must be: // m_incoming_tx_lock (optional) @@ -4000,7 +4386,9 @@ bool Blockchain::prepare_handle_incoming_blocks(const std::listbatch_start(blocks_entry.size(), bytes))) { m_blockchain_lock.unlock(); m_tx_pool.unlock(); @@ -4008,78 +4396,69 @@ bool Blockchain::prepare_handle_incoming_blocks(const std::listheight() + blocks_entry.size()) < m_blocks_hash_check.size()) + const uint64_t height = m_db->height(); + if ((height + blocks_entry.size()) < m_blocks_hash_check.size()) return true; bool blocks_exist = false; tools::threadpool& tpool = tools::threadpool::getInstance(); - uint64_t threads = tpool.get_max_concurrency(); + unsigned threads = tpool.get_max_concurrency(); + blocks.resize(blocks_entry.size()); - if (blocks_entry.size() > 1 && threads > 1 && m_max_prepare_blocks_threads > 1) + if (1) { // limit threads, default limit = 4 if(threads > m_max_prepare_blocks_threads) threads = m_max_prepare_blocks_threads; - uint64_t height = m_db->height(); - int batches = blocks_entry.size() / threads; - int extra = blocks_entry.size() % threads; + unsigned int batches = blocks_entry.size() / threads; + unsigned int extra = blocks_entry.size() % threads; MDEBUG("block_batches: " << batches); std::vector> maps(threads); - std::vector < std::vector < block >> blocks(threads); auto it = blocks_entry.begin(); + unsigned blockidx = 0; - for (uint64_t i = 0; i < threads; i++) + const crypto::hash tophash = m_db->top_block_hash(); + for (unsigned i = 0; i < threads; i++) { - for (int j = 0; j < batches; j++) + for (unsigned int j = 0; j < batches; j++, ++blockidx) { - block block; + block &block = blocks[blockidx]; + crypto::hash block_hash; - if (!parse_and_validate_block_from_blob(it->block, block)) - { - std::advance(it, 1); - continue; - } + if (!parse_and_validate_block_from_blob(it->block, block, block_hash)) + return false; // check first block and skip all blocks if its not chained properly - if (i == 0 && j == 0) + if (blockidx == 0) { - crypto::hash tophash = m_db->top_block_hash(); if (block.prev_id != tophash) { MDEBUG("Skipping prepare blocks. New blocks don't belong to chain."); + blocks.clear(); return true; } } - if (have_block(get_block_hash(block))) - { + if (have_block(block_hash)) blocks_exist = true; - break; - } - blocks[i].push_back(block); std::advance(it, 1); } } - for (int i = 0; i < extra && !blocks_exist; i++) + for (unsigned i = 0; i < extra && !blocks_exist; i++, blockidx++) { - block block; + block &block = blocks[blockidx]; + crypto::hash block_hash; - if (!parse_and_validate_block_from_blob(it->block, block)) - { - std::advance(it, 1); - continue; - } + if (!parse_and_validate_block_from_blob(it->block, block, block_hash)) + return false; - if (have_block(get_block_hash(block))) - { + if (have_block(block_hash)) blocks_exist = true; - break; - } - blocks[i].push_back(block); std::advance(it, 1); } @@ -4088,13 +4467,16 @@ bool Blockchain::prepare_handle_incoming_blocks(const std::list(&blocks[thread_height - height], nblocks), std::ref(maps[i])), true); + thread_height += nblocks; } - waiter.wait(); + waiter.wait(&tpool); if (m_cancel) return false; @@ -4111,7 +4493,7 @@ bool Blockchain::prepare_handle_incoming_blocks(const std::list> offset_map; // [output] stores all output_data_t for each absolute_offset std::map> tx_map; + std::vector> txes(total_txs); #define SCAN_TABLE_QUIT(m) \ do { \ @@ -4144,6 +4527,7 @@ bool Blockchain::prepare_handle_incoming_blocks(const std::list= txes.size()) + SCAN_TABLE_QUIT("tx_index is out of sync"); + transaction &tx = txes[tx_index].first; + crypto::hash &tx_prefix_hash = txes[tx_index].second; + ++tx_index; - if (!parse_and_validate_tx_from_blob(tx_blob, tx, tx_hash, tx_prefix_hash)) + if (!parse_and_validate_tx_base_from_blob(tx_blob, tx)) SCAN_TABLE_QUIT("Could not parse tx from incoming blocks."); + cryptonote::get_transaction_prefix_hash(tx, tx_prefix_hash); auto its = m_scan_table.find(tx_prefix_hash); if (its != m_scan_table.end()) @@ -4205,6 +4592,7 @@ bool Blockchain::prepare_handle_incoming_blocks(const std::list> transactions(amounts.size()); - + // gather all the output keys threads = tpool.get_max_concurrency(); if (!m_db->can_thread_bulk_indices()) threads = 1; - if (threads > 1) + if (threads > 1 && amounts.size() > 1) { tools::threadpool::waiter waiter; for (size_t i = 0; i < amounts.size(); i++) { uint64_t amount = amounts[i]; - tpool.submit(&waiter, boost::bind(&Blockchain::output_scan_worker, this, amount, std::cref(offset_map[amount]), std::ref(tx_map[amount]), std::ref(transactions[i]))); + tpool.submit(&waiter, boost::bind(&Blockchain::output_scan_worker, this, amount, std::cref(offset_map[amount]), std::ref(tx_map[amount])), true); } - waiter.wait(); + waiter.wait(&tpool); } else { for (size_t i = 0; i < amounts.size(); i++) { uint64_t amount = amounts[i]; - output_scan_worker(amount, offset_map[amount], tx_map[amount], transactions[i]); + output_scan_worker(amount, offset_map[amount], tx_map[amount]); } } - int total_txs = 0; - // now generate a table for each tx_prefix and k_image hashes + tx_index = 0; for (const auto &entry : blocks_entry) { if (m_cancel) @@ -4252,14 +4637,12 @@ bool Blockchain::prepare_handle_incoming_blocks(const std::list= txes.size()) + SCAN_TABLE_QUIT("tx_index is out of sync"); + const transaction &tx = txes[tx_index].first; + const crypto::hash &tx_prefix_hash = txes[tx_index].second; + ++tx_index; - if (!parse_and_validate_tx_from_blob(tx_blob, tx, tx_hash, tx_prefix_hash)) - SCAN_TABLE_QUIT("Could not parse tx from incoming blocks."); - - ++total_txs; auto its = m_scan_table.find(tx_prefix_hash); if (its == m_scan_table.end()) SCAN_TABLE_QUIT("Tx not found on scan table from incoming blocks."); @@ -4308,9 +4691,9 @@ bool Blockchain::prepare_handle_incoming_blocks(const std::listadd_txpool_tx(tx, meta); + m_db->add_txpool_tx(txid, blob, meta); } void Blockchain::update_txpool_tx(const crypto::hash &txid, const txpool_tx_meta_t &meta) @@ -4348,7 +4731,7 @@ bool Blockchain::for_all_txpool_txes(std::functionfor_all_txpool_txes(f, include_blob, include_unrelayed_txes); } -void Blockchain::set_user_options(uint64_t maxthreads, uint64_t blocks_per_sync, blockchain_db_sync_mode sync_mode, bool fast_sync) +void Blockchain::set_user_options(uint64_t maxthreads, bool sync_on_blocks, uint64_t sync_threshold, blockchain_db_sync_mode sync_mode, bool fast_sync) { if (sync_mode == db_defaultsync) { @@ -4357,7 +4740,8 @@ void Blockchain::set_user_options(uint64_t maxthreads, uint64_t blocks_per_sync, } m_db_sync_mode = sync_mode; m_fast_sync = fast_sync; - m_db_blocks_per_sync = blocks_per_sync; + m_db_sync_on_blocks = sync_on_blocks; + m_db_sync_threshold = sync_threshold; m_max_prepare_blocks_threads = maxthreads; } @@ -4378,6 +4762,39 @@ HardFork::State Blockchain::get_hard_fork_state() const return m_hardfork->get_state(); } +const std::vector& Blockchain::get_hard_fork_heights(network_type nettype) +{ + static const std::vector mainnet_heights = []() + { + std::vector heights; + for (const auto& i : mainnet_hard_forks) + heights.emplace_back(i.version, i.height, i.threshold, i.time); + return heights; + }(); + static const std::vector testnet_heights = []() + { + std::vector heights; + for (const auto& i : testnet_hard_forks) + heights.emplace_back(i.version, i.height, i.threshold, i.time); + return heights; + }(); + static const std::vector stagenet_heights = []() + { + std::vector heights; + for (const auto& i : stagenet_hard_forks) + heights.emplace_back(i.version, i.height, i.threshold, i.time); + return heights; + }(); + static const std::vector dummy; + switch (nettype) + { + case MAINNET: return mainnet_heights; + case TESTNET: return testnet_heights; + case STAGENET: return stagenet_heights; + default: return dummy; + } +} + bool Blockchain::get_hard_fork_voting_info(uint8_t version, uint32_t &window, uint32_t &votes, uint32_t &threshold, uint64_t &earliest_height, uint8_t &voting) const { return m_hardfork->get_voting_info(version, window, votes, threshold, earliest_height, voting); @@ -4393,9 +4810,9 @@ std::map> Blockchain:: get_ou return m_db->get_output_histogram(amounts, unlocked, recent_cutoff, min_count); } -std::list> Blockchain::get_alternative_chains() const +std::list>> Blockchain::get_alternative_chains() const { - std::list> chains; + std::list>> chains; for (const auto &i: m_alternative_chains) { @@ -4411,15 +4828,16 @@ std::list> Blockchain::get_a } if (!found) { - uint64_t length = 1; + std::vector chain; auto h = i.second.bl.prev_id; + chain.push_back(top); blocks_ext_by_hash::const_iterator prev; while ((prev = m_alternative_chains.find(h)) != m_alternative_chains.end()) { + chain.push_back(h); h = prev->second.bl.prev_id; - ++length; } - chains.push_back(std::make_pair(i.second, length)); + chains.push_back(std::make_pair(i.second, chain)); } } return chains; @@ -4431,20 +4849,22 @@ void Blockchain::cancel() } #if defined(PER_BLOCK_CHECKPOINT) -static const char expected_block_hashes_hash[] = "59261c03b54bcb21bd463f9fe40a94f40840a12642e9a3b3bfb11b35839a5fe3"; -void Blockchain::load_compiled_in_block_hashes() +static const char expected_block_hashes_hash[] = "7dafb40b414a0e59bfced6682ef519f0b416bc914dd3d622b72e0dd1a47117c2"; +void Blockchain::load_compiled_in_block_hashes(const GetCheckpointsCallback& get_checkpoints) { - const bool testnet = m_nettype == TESTNET; - const bool stagenet = m_nettype == STAGENET; - if (m_fast_sync && get_blocks_dat_start(testnet, stagenet) != nullptr && get_blocks_dat_size(testnet, stagenet) > 0) + if (get_checkpoints == nullptr || !m_fast_sync) { - MINFO("Loading precomputed blocks (" << get_blocks_dat_size(testnet, stagenet) << " bytes)"); - + return; + } + const epee::span &checkpoints = get_checkpoints(m_nettype); + if (!checkpoints.empty()) + { + MINFO("Loading precomputed blocks (" << checkpoints.size() << " bytes)"); if (m_nettype == MAINNET) { // first check hash crypto::hash hash; - if (!tools::sha256sum(get_blocks_dat_start(testnet, stagenet), get_blocks_dat_size(testnet, stagenet), hash)) + if (!tools::sha256sum(checkpoints.data(), checkpoints.size(), hash)) { MERROR("Failed to hash precomputed blocks data"); return; @@ -4464,9 +4884,9 @@ void Blockchain::load_compiled_in_block_hashes() } } - if (get_blocks_dat_size(testnet, stagenet) > 4) + if (checkpoints.size() > 4) { - const unsigned char *p = get_blocks_dat_start(testnet, stagenet); + const unsigned char *p = checkpoints.data(); const uint32_t nblocks = *p | ((*(p+1))<<8) | ((*(p+2))<<16) | ((*(p+3))<<24); if (nblocks > (std::numeric_limits::max() - 4) / sizeof(hash)) { @@ -4474,7 +4894,7 @@ void Blockchain::load_compiled_in_block_hashes() return; } const size_t size_needed = 4 + nblocks * sizeof(crypto::hash); - if(nblocks > 0 && nblocks > (m_db->height() + HASH_OF_HASHES_STEP - 1) / HASH_OF_HASHES_STEP && get_blocks_dat_size(testnet, stagenet) >= size_needed) + if(nblocks > 0 && nblocks > (m_db->height() + HASH_OF_HASHES_STEP - 1) / HASH_OF_HASHES_STEP && checkpoints.size() >= size_needed) { p += sizeof(uint32_t); m_blocks_hash_of_hashes.reserve(nblocks); @@ -4495,17 +4915,18 @@ void Blockchain::load_compiled_in_block_hashes() // for tx hashes will fail in handle_block_to_main_chain(..) CRITICAL_REGION_LOCAL(m_tx_pool); - std::list txs; + std::vector txs; m_tx_pool.get_transactions(txs); - size_t blob_size; + size_t tx_weight; uint64_t fee; bool relayed, do_not_relay, double_spend_seen; transaction pool_tx; + blobdata txblob; for(const transaction &tx : txs) { crypto::hash tx_hash = get_transaction_hash(tx); - m_tx_pool.take_tx(tx_hash, pool_tx, blob_size, fee, relayed, do_not_relay, double_spend_seen); + m_tx_pool.take_tx(tx_hash, pool_tx, txblob, tx_weight, fee, relayed, do_not_relay, double_spend_seen); } } } @@ -4557,7 +4978,27 @@ bool Blockchain::for_all_outputs(uint64_t amount, std::functionfor_all_outputs(amount, f);; } +void Blockchain::invalidate_block_template_cache() +{ + MDEBUG("Invalidating block template cache"); + m_btc_valid = false; +} + +void Blockchain::cache_block_template(const block &b, const cryptonote::account_public_address &address, const blobdata &nonce, const difficulty_type &diff, uint64_t height, uint64_t expected_reward, uint64_t pool_cookie) +{ + MDEBUG("Setting block template cache"); + m_btc = b; + m_btc_address = address; + m_btc_nonce = nonce; + m_btc_difficulty = diff; + m_btc_height = height; + m_btc_expected_reward = expected_reward; + m_btc_pool_cookie = pool_cookie; + m_btc_valid = true; +} + namespace cryptonote { -template bool Blockchain::get_transactions(const std::vector&, std::list&, std::list&) const; -template bool Blockchain::get_transactions_blobs(const std::vector&, std::list&, std::list&, bool) const; +template bool Blockchain::get_transactions(const std::vector&, std::vector&, std::vector&) const; +template bool Blockchain::get_transactions_blobs(const std::vector&, std::vector&, std::vector&, bool) const; +template bool Blockchain::get_split_transactions_blobs(const std::vector&, std::vector>&, std::vector&) const; } diff --git a/src/cryptonote_core/blockchain.h b/src/cryptonote_core/blockchain.h index 769e608caa4..32ed96b5ba6 100644 --- a/src/cryptonote_core/blockchain.h +++ b/src/cryptonote_core/blockchain.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -38,11 +38,14 @@ #include #include #include +#include #include #include +#include "span.h" #include "syncobj.h" #include "string_tools.h" +#include "rolling_median.h" #include "cryptonote_basic/cryptonote_basic.h" #include "common/util.h" #include "cryptonote_protocol/cryptonote_protocol_defs.h" @@ -55,6 +58,8 @@ #include "cryptonote_basic/hardfork.h" #include "blockchain_db/blockchain_db.h" +namespace tools { class Notify; } + namespace cryptonote { class tx_memory_pool; @@ -71,6 +76,15 @@ namespace cryptonote db_nosync //!< Leave syncing up to the backing db (safest, but slowest because of disk I/O) }; + /** + * @brief Callback routine that returns checkpoints data for specific network type + * + * @param network network type + * + * @return checkpoints data, empty span if there ain't any checkpoints for specific network type + */ + typedef std::function(cryptonote::network_type network)> GetCheckpointsCallback; + /************************************************************************/ /* */ /************************************************************************/ @@ -95,7 +109,7 @@ namespace cryptonote { block bl; //!< the block uint64_t height; //!< the height of the block in the blockchain - size_t block_cumulative_size; //!< the size (in bytes) of the block + size_t block_cumulative_weight; //!< the weight of the block difficulty_type cumulative_difficulty; //!< the accumulated difficulty after that block uint64_t already_generated_coins; //!< the total coins minted after that block }; @@ -107,6 +121,11 @@ namespace cryptonote */ Blockchain(tx_memory_pool& tx_pool); + /** + * @brief Blockchain destructor + */ + ~Blockchain(); + /** * @brief Initialize the Blockchain state * @@ -114,10 +133,12 @@ namespace cryptonote * @param nettype network type * @param offline true if running offline, else false * @param test_options test parameters + * @param fixed_difficulty fixed difficulty for testing purposes; 0 means disabled + * @param get_checkpoints if set, will be called to get checkpoints data * * @return true on success, false if any initialization steps fail */ - bool init(BlockchainDB* db, const network_type nettype = MAINNET, bool offline = false, const cryptonote::test_options *test_options = NULL); + bool init(BlockchainDB* db, const network_type nettype = MAINNET, bool offline = false, const cryptonote::test_options *test_options = NULL, difficulty_type fixed_difficulty = 0, const GetCheckpointsCallback& get_checkpoints = nullptr); /** * @brief Initialize the Blockchain state @@ -157,7 +178,7 @@ namespace cryptonote * * @return false if start_offset > blockchain height, else true */ - bool get_blocks(uint64_t start_offset, size_t count, std::list>& blocks, std::list& txs) const; + bool get_blocks(uint64_t start_offset, size_t count, std::vector>& blocks, std::vector& txs) const; /** * @brief get blocks from blocks based on start height and count @@ -168,7 +189,7 @@ namespace cryptonote * * @return false if start_offset > blockchain height, else true */ - bool get_blocks(uint64_t start_offset, size_t count, std::list>& blocks) const; + bool get_blocks(uint64_t start_offset, size_t count, std::vector>& blocks) const; /** * @brief compiles a list of all blocks stored as alternative chains @@ -177,7 +198,7 @@ namespace cryptonote * * @return true */ - bool get_alternative_blocks(std::list& blocks) const; + bool get_alternative_blocks(std::vector& blocks) const; /** * @brief returns the number of alternative blocks stored @@ -209,11 +230,12 @@ namespace cryptonote /** * @brief performs some preprocessing on a group of incoming blocks to speed up verification * - * @param blocks a list of incoming blocks + * @param blocks_entry a list of incoming blocks + * @param blocks the parsed blocks * * @return false on erroneous blocks, else true */ - bool prepare_handle_incoming_blocks(const std::list &blocks); + bool prepare_handle_incoming_blocks(const std::vector &blocks_entry, std::vector &blocks); /** * @brief incoming blocks post-processing, cleanup, and disk sync @@ -314,6 +336,7 @@ namespace cryptonote * @brief creates a new block to mine against * * @param b return-by-reference block to be filled in + * @param from_block optional block hash to start mining from (main chain tip if NULL) * @param miner_address address new coins for the block will go to * @param di return-by-reference tells the miner what the difficulty target is * @param height return-by-reference tells the miner what height it's mining against @@ -323,6 +346,7 @@ namespace cryptonote * @return true if block template filled in successfully, else false */ bool create_block_template(block& b, const account_public_address& miner_address, difficulty_type& di, uint64_t& height, uint64_t& expected_reward, const blobdata& ex_nonce); + bool create_block_template(block& b, const crypto::hash *from_block, const account_public_address& miner_address, difficulty_type& di, uint64_t& height, uint64_t& expected_reward, const blobdata& ex_nonce); /** * @brief checks if a block is known about with a given hash @@ -370,10 +394,11 @@ namespace cryptonote * @param hashes the hashes to be returned, return-by-reference * @param start_height the start height, return-by-reference * @param current_height the current blockchain height, return-by-reference + * @param clip_pruned whether to constrain results to unpruned data * * @return true if a block found in common, else false */ - bool find_blockchain_supplement(const std::list& qblock_ids, std::list& hashes, uint64_t& start_height, uint64_t& current_height) const; + bool find_blockchain_supplement(const std::list& qblock_ids, std::vector& hashes, uint64_t& start_height, uint64_t& current_height, bool clip_pruned) const; /** * @brief get recent block hashes for a foreign chain @@ -420,7 +445,7 @@ namespace cryptonote * * @return true if a block found in common or req_start_block specified, else false */ - bool find_blockchain_supplement(const uint64_t req_start_block, const std::list& qblock_ids, std::list > >& blocks, uint64_t& total_height, uint64_t& start_height, bool pruned, size_t max_count) const; + bool find_blockchain_supplement(const uint64_t req_start_block, const std::list& qblock_ids, std::vector, std::vector > > >& blocks, uint64_t& total_height, uint64_t& start_height, bool pruned, bool get_miner_tx_hash, size_t max_count) const; /** * @brief retrieves a set of blocks and their transactions, and possibly other transactions @@ -445,16 +470,6 @@ namespace cryptonote */ uint64_t get_num_mature_outputs(uint64_t amount) const; - /** - * @brief get random outputs (indices) for an amount - * - * @param amount the amount - * @param count the number of random outputs to choose - * - * @return the outputs' amount-global indices - */ - std::vector get_random_outputs(uint64_t amount, uint64_t count) const; - /** * @brief get the public key for an output * @@ -465,22 +480,6 @@ namespace cryptonote */ crypto::public_key get_output_key(uint64_t amount, uint64_t global_index) const; - /** - * @brief gets random outputs to mix with - * - * This function takes an RPC request for outputs to mix with - * and creates an RPC response with the resultant output indices. - * - * Outputs to mix with are randomly selected from the utxo set - * for each output amount in the request. - * - * @param req the output amounts and number of mixins to select - * @param res return-by-reference the resultant output indices - * - * @return true - */ - bool get_random_outs_for_amounts(const COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::request& req, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::response& res) const; - /** * @brief gets specific outputs to mix with * @@ -507,23 +506,6 @@ namespace cryptonote */ void get_output_key_mask_unlocked(const uint64_t& amount, const uint64_t& index, crypto::public_key& key, rct::key& mask, bool& unlocked) const; - /** - * @brief gets random ringct outputs to mix with - * - * This function takes an RPC request for outputs to mix with - * and creates an RPC response with the resultant output indices - * and the matching keys. - * - * Outputs to mix with are randomly selected from the utxo set - * for each output amount in the request. - * - * @param req the output amounts and number of mixins to select - * @param res return-by-reference the resultant output indices - * - * @return true - */ - bool get_random_rct_outs(const COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS::request& req, COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS::response& res) const; - /** * @brief gets per block distribution of outputs of a given amount * @@ -544,10 +526,12 @@ namespace cryptonote * * @param tx_id the hash of the transaction to fetch indices for * @param indexs return-by-reference the global indices for the transaction's outputs + * @param n_txes how many txes in a row to get results for * * @return false if the transaction does not exist, or if no indices are found, otherwise true */ bool get_tx_outputs_gindexs(const crypto::hash& tx_id, std::vector& indexs) const; + bool get_tx_outputs_gindexs(const crypto::hash& tx_id, size_t n_txes, std::vector>& indexs) const; /** * @brief stores the blockchain @@ -578,46 +562,57 @@ namespace cryptonote bool check_tx_inputs(transaction& tx, uint64_t& pmax_used_block_height, crypto::hash& max_used_block_id, tx_verification_context &tvc, bool kept_by_block = false); /** - * @brief get dynamic per kB fee for a given block size + * @brief get fee quantization mask + * + * The dynamic fee may be quantized, to mask out the last decimal places + * + * @return the fee quantized mask + */ + static uint64_t get_fee_quantization_mask(); + + /** + * @brief get dynamic per kB or byte fee for a given block weight * - * The dynamic fee is based on the block size in a past window, and - * the current block reward. It is expressed by kB. + * The dynamic fee is based on the block weight in a past window, and + * the current block reward. It is expressed by kB before v8, and + * per byte from v8. * * @param block_reward the current block reward - * @param median_block_size the median blob's size in the past window + * @param median_block_weight the median block weight in the past window * @param version hard fork version for rules and constants to use * - * @return the per kB fee + * @return the fee */ - static uint64_t get_dynamic_per_kb_fee(uint64_t block_reward, size_t median_block_size, uint8_t version); + static uint64_t get_dynamic_base_fee(uint64_t block_reward, size_t median_block_weight, uint8_t version); /** - * @brief get dynamic per kB fee estimate for the next few blocks + * @brief get dynamic per kB or byte fee estimate for the next few blocks * - * The dynamic fee is based on the block size in a past window, and - * the current block reward. It is expressed by kB. This function - * calculates an estimate for a dynamic fee which will be valid for - * the next grace_blocks + * The dynamic fee is based on the block weight in a past window, and + * the current block reward. It is expressed by kB before v8, and + * per byte from v8. + * This function calculates an estimate for a dynamic fee which will be + * valid for the next grace_blocks * * @param grace_blocks number of blocks we want the fee to be valid for * - * @return the per kB fee estimate + * @return the fee estimate */ - uint64_t get_dynamic_per_kb_fee_estimate(uint64_t grace_blocks) const; + uint64_t get_dynamic_base_fee_estimate(uint64_t grace_blocks) const; /** * @brief validate a transaction's fee * * This function validates the fee is enough for the transaction. - * This is based on the size of the transaction blob, and, after a - * height threshold, on the average size of transaction in a past window + * This is based on the weight of the transaction, and, after a + * height threshold, on the average weight of transaction in a past window * - * @param blob_size the transaction blob's size + * @param tx_weight the transaction weight * @param fee the fee * * @return true if the fee is enough, false otherwise */ - bool check_fee(size_t blob_size, uint64_t fee) const; + bool check_fee(size_t tx_weight, uint64_t fee) const; /** * @brief check that a transaction's outputs conform to current standards @@ -634,18 +629,25 @@ namespace cryptonote bool check_tx_outputs(const transaction& tx, tx_verification_context &tvc); /** - * @brief gets the blocksize limit based on recent blocks + * @brief gets the block weight limit based on recent blocks * * @return the limit */ - uint64_t get_current_cumulative_blocksize_limit() const; + uint64_t get_current_cumulative_block_weight_limit() const; + + /** + * @brief gets the long term block weight for a new block + * + * @return the long term block weight + */ + uint64_t get_next_long_term_block_weight(uint64_t block_weight) const; /** - * @brief gets the blocksize median based on recent blocks (same window as for the limit) + * @brief gets the block weight median based on recent blocks (same window as for the limit) * * @return the median */ - uint64_t get_current_cumulative_blocksize_median() const; + uint64_t get_current_cumulative_block_weight_median() const; /** * @brief gets the difficulty of the block with a given height @@ -654,7 +656,7 @@ namespace cryptonote * * @return the difficulty */ - uint64_t block_difficulty(uint64_t i) const; + difficulty_type block_difficulty(uint64_t i) const; /** * @brief gets blocks based on a list of block hashes @@ -687,6 +689,8 @@ namespace cryptonote template bool get_transactions_blobs(const t_ids_container& txs_ids, t_tx_container& txs, t_missed_container& missed_txs, bool pruned = false) const; template + bool get_split_transactions_blobs(const t_ids_container& txs_ids, t_tx_container& txs, t_missed_container& missed_txs) const; + template bool get_transactions(const t_ids_container& txs_ids, t_tx_container& txs, t_missed_container& missed_txs) const; //debug functions @@ -728,13 +732,28 @@ namespace cryptonote * @brief sets various performance options * * @param maxthreads max number of threads when preparing blocks for addition - * @param blocks_per_sync number of blocks to cache before syncing to database + * @param sync_on_blocks whether to sync based on blocks or bytes + * @param sync_threshold number of blocks/bytes to cache before syncing to database * @param sync_mode the ::blockchain_db_sync_mode to use * @param fast_sync sync using built-in block hashes as trusted */ - void set_user_options(uint64_t maxthreads, uint64_t blocks_per_sync, + void set_user_options(uint64_t maxthreads, bool sync_on_blocks, uint64_t sync_threshold, blockchain_db_sync_mode sync_mode, bool fast_sync); + /** + * @brief sets a block notify object to call for every new block + * + * @param notify the notify object to call at every new block + */ + void set_block_notify(const std::shared_ptr ¬ify) { m_block_notify = notify; } + + /** + * @brief sets a reorg notify object to call for every reorg + * + * @param notify the notify object to call at every reorg + */ + void set_reorg_notify(const std::shared_ptr ¬ify) { m_reorg_notify = notify; } + /** * @brief Put DB in safe sync mode */ @@ -754,6 +773,13 @@ namespace cryptonote */ HardFork::State get_hard_fork_state() const; + /** + * @brief gets the hardfork heights of given network + * + * @return the HardFork object + */ + static const std::vector& get_hard_fork_heights(network_type nettype); + /** * @brief gets the current hardfork version in use/voted for * @@ -794,6 +820,13 @@ namespace cryptonote */ uint8_t get_hard_fork_version(uint64_t height) const { return m_hardfork->get(height); } + /** + * @brief returns the earliest block a given version may activate + * + * @return the height + */ + uint64_t get_earliest_ideal_height_for_version(uint8_t version) const { return m_hardfork->get_earliest_ideal_height_for_version(version); } + /** * @brief get information about hardfork voting for a version * @@ -822,7 +855,7 @@ namespace cryptonote * * @return false if any removals fail, otherwise true */ - bool flush_txes_from_pool(const std::list &txids); + bool flush_txes_from_pool(const std::vector &txids); /** * @brief return a histogram of outputs on the blockchain @@ -911,11 +944,9 @@ namespace cryptonote * @param amount the amount * @param offsets the indices (indexed to the amount) of the outputs * @param outputs return-by-reference the outputs collected - * @param txs unused, candidate for removal */ void output_scan_worker(const uint64_t amount,const std::vector &offsets, - std::vector &outputs, std::unordered_map &txs) const; + std::vector &outputs) const; /** * @brief computes the "short" and "long" hashes for a set of blocks @@ -924,7 +955,7 @@ namespace cryptonote * @param blocks the blocks to be hashed * @param map return-by-reference the hashes for each block */ - void block_longhash_worker(uint64_t height, const std::vector &blocks, + void block_longhash_worker(uint64_t height, const epee::span &blocks, std::unordered_map &map) const; /** @@ -932,9 +963,9 @@ namespace cryptonote * * @return a list of chains */ - std::list> get_alternative_chains() const; + std::list>> get_alternative_chains() const; - void add_txpool_tx(transaction &tx, const txpool_tx_meta_t &meta); + void add_txpool_tx(const crypto::hash &txid, const cryptonote::blobdata &blob, const txpool_tx_meta_t &meta); void update_txpool_tx(const crypto::hash &txid, const txpool_tx_meta_t &meta); void remove_txpool_tx(const crypto::hash &txid); uint64_t get_txpool_tx_count(bool include_unrelayed_txes = true) const; @@ -945,7 +976,11 @@ namespace cryptonote bool is_within_compiled_block_hash_area(uint64_t height) const; bool is_within_compiled_block_hash_area() const { return is_within_compiled_block_hash_area(m_db->height()); } - uint64_t prevalidate_block_hashes(uint64_t height, const std::list &hashes); + uint64_t prevalidate_block_hashes(uint64_t height, const std::vector &hashes); + uint32_t get_blockchain_pruning_seed() const { return m_db->get_blockchain_pruning_seed(); } + bool prune_blockchain(uint32_t pruning_seed = 0); + bool update_blockchain_pruning(); + bool check_blockchain_pruning(); void lock(); void unlock(); @@ -959,7 +994,21 @@ namespace cryptonote */ void on_new_tx_from_block(const cryptonote::transaction &tx); + /** + * @brief returns the timestamps of the last N blocks + */ + std::vector get_last_block_timestamps(unsigned int blocks) const; + + /** + * @brief removes blocks from the top of the blockchain + * + * @param nblocks number of blocks to be removed + */ + void pop_blocks(uint64_t nblocks); + +#ifndef IN_UNIT_TESTS private: +#endif // TODO: evaluate whether or not each of these typedefs are left over from blockchain_storage typedef std::unordered_map blocks_by_id_index; @@ -985,8 +1034,8 @@ namespace cryptonote // main chain transactions_container m_transactions; - size_t m_current_block_cumul_sz_limit; - size_t m_current_block_cumul_sz_median; + size_t m_current_block_cumul_weight_limit; + size_t m_current_block_cumul_weight_median; // metadata containers std::unordered_map>> m_scan_table; @@ -1002,14 +1051,20 @@ namespace cryptonote bool m_fast_sync; bool m_show_time_stats; bool m_db_default_sync; - uint64_t m_db_blocks_per_sync; + bool m_db_sync_on_blocks; + uint64_t m_db_sync_threshold; uint64_t m_max_prepare_blocks_threads; uint64_t m_fake_pow_calc_time; uint64_t m_fake_scan_time; uint64_t m_sync_counter; + uint64_t m_bytes_to_sync; std::vector m_timestamps; std::vector m_difficulties; uint64_t m_timestamps_and_difficulties_height; + uint64_t m_long_term_block_weights_window; + uint64_t m_long_term_effective_median_block_weight; + mutable crypto::hash m_long_term_block_weights_cache_tip_hash; + mutable epee::misc_utils::rolling_median_t m_long_term_block_weights_cache_rolling_median; epee::critical_section m_difficulty_lock; crypto::hash m_difficulty_for_next_block_top_hash; @@ -1033,9 +1088,26 @@ namespace cryptonote network_type m_nettype; bool m_offline; + difficulty_type m_fixed_difficulty; std::atomic m_cancel; + // block template cache + block m_btc; + account_public_address m_btc_address; + blobdata m_btc_nonce; + difficulty_type m_btc_difficulty; + uint64_t m_btc_height; + uint64_t m_btc_pool_cookie; + uint64_t m_btc_expected_reward; + bool m_btc_valid; + + + bool m_batch_success; + + std::shared_ptr m_block_notify; + std::shared_ptr m_reorg_notify; + /** * @brief collects the keys for all outputs being "spent" as an input * @@ -1114,7 +1186,7 @@ namespace cryptonote * * @return false if the reorganization fails, otherwise true */ - bool switch_to_alternative_blockchain(std::list& alt_chain, bool discard_disconnected_chain); + bool switch_to_alternative_blockchain(std::list& alt_chain, bool discard_disconnected_chain); /** * @brief removes the most recent block from the blockchain @@ -1167,6 +1239,18 @@ namespace cryptonote */ bool handle_alternative_block(const block& b, const crypto::hash& id, block_verification_context& bvc); + /** + * @brief builds a list of blocks connecting a block to the main chain + * + * @param prev_id the block hash of the tip of the alt chain + * @param alt_chain the chain to be added to + * @param timestamps returns the timestamps of previous blocks + * @param bvc the block verification context for error return + * + * @return true on success, false otherwise + */ + bool build_alt_chain(const crypto::hash &prev_id, std::list& alt_chain, std::vector ×tamps, block_verification_context& bvc) const; + /** * @brief gets the difficulty requirement for a new block on an alternate chain * @@ -1175,7 +1259,7 @@ namespace cryptonote * * @return the difficulty requirement */ - difficulty_type get_next_difficulty_for_alternative_chain(const std::list& alt_chain, block_extended_info& bei) const; + difficulty_type get_next_difficulty_for_alternative_chain(const std::list& alt_chain, block_extended_info& bei) const; /** * @brief sanity checks a miner transaction before validating an entire block @@ -1197,7 +1281,7 @@ namespace cryptonote * and that his miner transaction totals reward + fee. * * @param b the block containing the miner transaction to be validated - * @param cumulative_block_size the block's size + * @param cumulative_block_weight the block's weight * @param fee the total fees collected in the block * @param base_reward return-by-reference the new block's generated coins * @param already_generated_coins the amount of currency generated prior to this block @@ -1206,7 +1290,7 @@ namespace cryptonote * * @return false if anything is found wrong with the miner transaction, otherwise true */ - bool validate_miner_transaction(const block& b, size_t cumulative_block_size, uint64_t fee, uint64_t& base_reward, uint64_t already_generated_coins, bool &partial_block_reward, uint8_t version); + bool validate_miner_transaction(const block& b, size_t cumulative_block_weight, uint64_t fee, uint64_t& base_reward, uint64_t already_generated_coins, bool &partial_block_reward, uint8_t version); /** * @brief reverts the blockchain to its previous state following a failed switch @@ -1223,32 +1307,26 @@ namespace cryptonote bool rollback_blockchain_switching(std::list& original_chain, uint64_t rollback_height); /** - * @brief gets recent block sizes for median calculation + * @brief gets recent block weights for median calculation * - * get the block sizes of the last blocks, and return by reference . + * get the block weights of the last blocks, and return by reference . * - * @param sz return-by-reference the list of sizes - * @param count the number of blocks to get sizes for + * @param weights return-by-reference the list of weights + * @param count the number of blocks to get weights for */ - void get_last_n_blocks_sizes(std::vector& sz, size_t count) const; + void get_last_n_blocks_weights(std::vector& weights, size_t count) const; /** - * @brief adds the given output to the requested set of random outputs + * @brief gets block long term weight median * - * @param result_outs return-by-reference the set the output is to be added to - * @param amount the output amount - * @param i the output index (indexed to amount) - */ - void add_out_to_get_random_outs(COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::outs_for_amount& result_outs, uint64_t amount, size_t i) const; - - /** - * @brief adds the given output to the requested set of random ringct outputs + * get the block long term weight median of blocks starting at + * + * @param start_height the block height of the first block to query + * @param count the number of blocks to get weights for * - * @param outs return-by-reference the set the output is to be added to - * @param amount the output amount (0 for rct inputs) - * @param i the rct output index + * @return the long term median block weight */ - void add_out_to_get_rct_random_outs(std::list& outs, uint64_t amount, size_t i) const; + uint64_t get_long_term_block_weight_median(uint64_t start_height, size_t count) const; /** * @brief checks if a transaction is unlocked (its outputs spendable) @@ -1342,15 +1420,17 @@ namespace cryptonote * * @return true unless start_height is greater than the current blockchain height */ - bool complete_timestamps_vector(uint64_t start_height, std::vector& timestamps); + bool complete_timestamps_vector(uint64_t start_height, std::vector& timestamps) const; /** - * @brief calculate the block size limit for the next block to be added + * @brief calculate the block weight limit for the next block to be added + * + * @param long_term_effective_median_block_weight optionally return that value * * @return true */ - bool update_next_cumulative_size_limit(); - void return_tx_to_pool(std::vector &txs); + bool update_next_cumulative_weight_limit(uint64_t *long_term_effective_median_block_weight = NULL); + void return_tx_to_pool(std::vector> &txs); /** * @brief make sure a transaction isn't attempting a double-spend @@ -1380,8 +1460,10 @@ namespace cryptonote * A (possibly empty) set of block hashes can be compiled into the * monero daemon binary. This function loads those hashes into * a useful state. + * + * @param get_checkpoints if set, will be called to get checkpoints data */ - void load_compiled_in_block_hashes(); + void load_compiled_in_block_hashes(const GetCheckpointsCallback& get_checkpoints); /** * @brief expands v2 transaction data from blockchain @@ -1391,5 +1473,17 @@ namespace cryptonote * that implicit data. */ bool expand_transaction_2(transaction &tx, const crypto::hash &tx_prefix_hash, const std::vector> &pubkeys); + + /** + * @brief invalidates any cached block template + */ + void invalidate_block_template_cache(); + + /** + * @brief stores a new cached block template + * + * At some point, may be used to push an update to miners + */ + void cache_block_template(const block &b, const cryptonote::account_public_address &address, const blobdata &nonce, const difficulty_type &diff, uint64_t height, uint64_t expected_reward, uint64_t pool_cookie); }; } // namespace cryptonote diff --git a/src/cryptonote_core/blockchain_storage_boost_serialization.h b/src/cryptonote_core/blockchain_storage_boost_serialization.h index f4f9f20cab0..cfeb5acfc5c 100644 --- a/src/cryptonote_core/blockchain_storage_boost_serialization.h +++ b/src/cryptonote_core/blockchain_storage_boost_serialization.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/cryptonote_core/cryptonote_core.cpp b/src/cryptonote_core/cryptonote_core.cpp index d2796deeb0f..13426230e20 100644 --- a/src/cryptonote_core/cryptonote_core.cpp +++ b/src/cryptonote_core/cryptonote_core.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -30,13 +30,11 @@ #include -#include "include_base_utils.h" #include "string_tools.h" using namespace epee; #include #include "cryptonote_core.h" -#include "common/command_line.h" #include "common/util.h" #include "common/updates.h" #include "common/download.h" @@ -45,7 +43,6 @@ using namespace epee; #include "warnings.h" #include "crypto/crypto.h" #include "cryptonote_config.h" -#include "cryptonote_tx_utils.h" #include "misc_language.h" #include "file_io_utils.h" #include @@ -53,6 +50,7 @@ using namespace epee; #include "ringct/rctTypes.h" #include "blockchain_db/blockchain_db.h" #include "ringct/rctSigs.h" +#include "common/notify.h" #include "version.h" #undef MONERO_DEFAULT_LOG_CATEGORY @@ -64,6 +62,9 @@ DISABLE_VS_WARNINGS(4355) #define BAD_SEMANTICS_TXES_MAX_SIZE 100 +// basically at least how many bytes the block itself serializes to without the miner tx +#define BLOCK_SIZE_SANITY_LEEWAY 100 + namespace cryptonote { const command_line::arg_descriptor arg_testnet_on = { @@ -76,6 +77,16 @@ namespace cryptonote , "Run on stagenet. The wallet must be launched with --stagenet flag." , false }; + const command_line::arg_descriptor arg_regtest_on = { + "regtest" + , "Run in a regression testing mode." + , false + }; + const command_line::arg_descriptor arg_fixed_difficulty = { + "fixed-difficulty" + , "Fixed difficulty used for testing." + , 0 + }; const command_line::arg_descriptor arg_data_dir = { "data-dir" , "Specify data directory" @@ -97,6 +108,11 @@ namespace cryptonote "disable-dns-checkpoints" , "Do not retrieve checkpoints from DNS" }; + const command_line::arg_descriptor arg_block_download_max_size = { + "block-download-max-size" + , "Set maximum size of block download queue in bytes (0 for default)" + , 0 + }; static const command_line::arg_descriptor arg_test_drop_download = { "test-drop-download" @@ -152,10 +168,45 @@ namespace cryptonote , "Relay blocks as normal blocks" , false }; - static const command_line::arg_descriptor arg_max_txpool_size = { - "max-txpool-size" - , "Set maximum txpool size in bytes." - , DEFAULT_TXPOOL_MAX_SIZE + static const command_line::arg_descriptor arg_pad_transactions = { + "pad-transactions" + , "Pad relayed transactions to help defend against traffic volume analysis" + , false + }; + static const command_line::arg_descriptor arg_max_txpool_weight = { + "max-txpool-weight" + , "Set maximum txpool weight in bytes." + , DEFAULT_TXPOOL_MAX_WEIGHT + }; + static const command_line::arg_descriptor arg_block_notify = { + "block-notify" + , "Run a program for each new block, '%s' will be replaced by the block hash" + , "" + }; + static const command_line::arg_descriptor arg_prune_blockchain = { + "prune-blockchain" + , "Prune blockchain" + , false + }; + static const command_line::arg_descriptor arg_reorg_notify = { + "reorg-notify" + , "Run a program for each reorg, '%s' will be replaced by the split height, " + "'%h' will be replaced by the new blockchain height, '%n' will be " + "replaced by the number of new blocks in the new chain, and '%d' will be " + "replaced by the number of blocks discarded from the old chain" + , "" + }; + static const command_line::arg_descriptor arg_block_rate_notify = { + "block-rate-notify" + , "Run a program when the block rate undergoes large fluctuations. This might " + "be a sign of large amounts of hash rate going on and off the Monero network, " + "and thus be of potential interest in predicting attacks. %t will be replaced " + "by the number of minutes for the observation window, %b by the number of " + "blocks observed within that window, and %e by the number of blocks that was " + "expected in that window. It is suggested that this notification is used to " + "automatically increase the number of confirmations required before a payment " + "is acted upon." + , "" }; //----------------------------------------------------------------------------------------------- @@ -170,9 +221,10 @@ namespace cryptonote m_last_dns_checkpoints_update(0), m_last_json_checkpoints_update(0), m_disable_dns_checkpoints(false), - m_threadpool(tools::threadpool::getInstance()), m_update_download(0), - m_nettype(UNDEFINED) + m_nettype(UNDEFINED), + m_update_available(false), + m_pad_transactions(false) { m_checkpoints_updating.clear(); set_cryptonote_protocol(pprotocol); @@ -231,6 +283,7 @@ namespace cryptonote //----------------------------------------------------------------------------------- void core::stop() { + m_miner.stop(); m_blockchain_storage.cancel(); tools::download_async_handle handle; @@ -252,6 +305,8 @@ namespace cryptonote command_line::add_arg(desc, arg_testnet_on); command_line::add_arg(desc, arg_stagenet_on); + command_line::add_arg(desc, arg_regtest_on); + command_line::add_arg(desc, arg_fixed_difficulty); command_line::add_arg(desc, arg_dns_checkpoints); command_line::add_arg(desc, arg_prep_blocks_threads); command_line::add_arg(desc, arg_fast_block_sync); @@ -263,7 +318,13 @@ namespace cryptonote command_line::add_arg(desc, arg_test_dbg_lock_sleep); command_line::add_arg(desc, arg_offline); command_line::add_arg(desc, arg_disable_dns_checkpoints); - command_line::add_arg(desc, arg_max_txpool_size); + command_line::add_arg(desc, arg_block_download_max_size); + command_line::add_arg(desc, arg_max_txpool_weight); + command_line::add_arg(desc, arg_pad_transactions); + command_line::add_arg(desc, arg_block_notify); + command_line::add_arg(desc, arg_prune_blockchain); + command_line::add_arg(desc, arg_reorg_notify); + command_line::add_arg(desc, arg_block_rate_notify); miner::init_options(desc); BlockchainDB::init_options(desc); @@ -301,6 +362,7 @@ namespace cryptonote set_enforce_dns_checkpoints(command_line::get_arg(vm, arg_dns_checkpoints)); test_drop_download_height(command_line::get_arg(vm, arg_test_drop_download_height)); m_fluffy_blocks_enabled = !get_arg(vm, arg_no_fluffy_blocks); + m_pad_transactions = get_arg(vm, arg_pad_transactions); m_offline = get_arg(vm, arg_offline); m_disable_dns_checkpoints = get_arg(vm, arg_disable_dns_checkpoints); if (!command_line::is_arg_defaulted(vm, arg_fluffy_blocks)) @@ -324,19 +386,19 @@ namespace cryptonote top_id = m_blockchain_storage.get_tail_id(height); } //----------------------------------------------------------------------------------------------- - bool core::get_blocks(uint64_t start_offset, size_t count, std::list>& blocks, std::list& txs) const + bool core::get_blocks(uint64_t start_offset, size_t count, std::vector>& blocks, std::vector& txs) const { return m_blockchain_storage.get_blocks(start_offset, count, blocks, txs); } //----------------------------------------------------------------------------------------------- - bool core::get_blocks(uint64_t start_offset, size_t count, std::list>& blocks) const + bool core::get_blocks(uint64_t start_offset, size_t count, std::vector>& blocks) const { return m_blockchain_storage.get_blocks(start_offset, count, blocks); } //----------------------------------------------------------------------------------------------- - bool core::get_blocks(uint64_t start_offset, size_t count, std::list& blocks) const + bool core::get_blocks(uint64_t start_offset, size_t count, std::vector& blocks) const { - std::list> bs; + std::vector> bs; if (!m_blockchain_storage.get_blocks(start_offset, count, bs)) return false; for (const auto &b: bs) @@ -344,23 +406,28 @@ namespace cryptonote return true; } //----------------------------------------------------------------------------------------------- - bool core::get_transactions(const std::vector& txs_ids, std::list& txs, std::list& missed_txs) const + bool core::get_transactions(const std::vector& txs_ids, std::vector& txs, std::vector& missed_txs) const { return m_blockchain_storage.get_transactions_blobs(txs_ids, txs, missed_txs); } //----------------------------------------------------------------------------------------------- + bool core::get_split_transactions_blobs(const std::vector& txs_ids, std::vector>& txs, std::vector& missed_txs) const + { + return m_blockchain_storage.get_split_transactions_blobs(txs_ids, txs, missed_txs); + } + //----------------------------------------------------------------------------------------------- bool core::get_txpool_backlog(std::vector& backlog) const { m_mempool.get_transaction_backlog(backlog); return true; } //----------------------------------------------------------------------------------------------- - bool core::get_transactions(const std::vector& txs_ids, std::list& txs, std::list& missed_txs) const + bool core::get_transactions(const std::vector& txs_ids, std::vector& txs, std::vector& missed_txs) const { return m_blockchain_storage.get_transactions(txs_ids, txs, missed_txs); } //----------------------------------------------------------------------------------------------- - bool core::get_alternative_blocks(std::list& blocks) const + bool core::get_alternative_blocks(std::vector& blocks) const { return m_blockchain_storage.get_alternative_blocks(blocks); } @@ -370,19 +437,16 @@ namespace cryptonote return m_blockchain_storage.get_alternative_blocks_count(); } //----------------------------------------------------------------------------------------------- - bool core::init(const boost::program_options::variables_map& vm, const char *config_subdir, const cryptonote::test_options *test_options) + bool core::init(const boost::program_options::variables_map& vm, const cryptonote::test_options *test_options, const GetCheckpointsCallback& get_checkpoints/* = nullptr */) { start_time = std::time(nullptr); - if (test_options != NULL) + const bool regtest = command_line::get_arg(vm, arg_regtest_on); + if (test_options != NULL || regtest) { m_nettype = FAKECHAIN; } bool r = handle_command_line(vm); - std::string m_config_folder_mempool = m_config_folder; - - if (config_subdir) - m_config_folder_mempool = m_config_folder_mempool + "/" + config_subdir; std::string db_type = command_line::get_arg(vm, cryptonote::arg_db_type); std::string db_sync_mode = command_line::get_arg(vm, cryptonote::arg_db_sync_mode); @@ -390,7 +454,8 @@ namespace cryptonote bool fast_sync = command_line::get_arg(vm, arg_fast_block_sync) != 0; uint64_t blocks_threads = command_line::get_arg(vm, arg_prep_blocks_threads); std::string check_updates_string = command_line::get_arg(vm, arg_check_updates); - size_t max_txpool_size = command_line::get_arg(vm, arg_max_txpool_size); + size_t max_txpool_weight = command_line::get_arg(vm, arg_max_txpool_weight); + bool prune_blockchain = command_line::get_arg(vm, arg_prune_blockchain); boost::filesystem::path folder(m_config_folder); if (m_nettype == FAKECHAIN) @@ -427,9 +492,20 @@ namespace cryptonote MGINFO("Loading blockchain from folder " << folder.string() << " ..."); const std::string filename = folder.string(); - // default to fast:async:1 + // default to fast:async:1 if overridden blockchain_db_sync_mode sync_mode = db_defaultsync; - uint64_t blocks_per_sync = 1; + bool sync_on_blocks = true; + uint64_t sync_threshold = 1; + + if (m_nettype == FAKECHAIN) + { + // reset the db by removing the database file before opening it + if (!db->remove_data_file(filename)) + { + MERROR("Failed to remove data file in " << filename); + return false; + } + } try { @@ -469,7 +545,7 @@ namespace cryptonote else if(options[0] == "fastest") { db_flags = DBF_FASTEST; - blocks_per_sync = 1000; // default to fastest:async:1000 + sync_threshold = 1000; // default to fastest:async:1000 sync_mode = db_sync_mode_is_default ? db_defaultsync : db_async; } else @@ -487,9 +563,22 @@ namespace cryptonote if(options.size() >= 3 && !safemode) { char *endptr; - uint64_t bps = strtoull(options[2].c_str(), &endptr, 0); - if (*endptr == '\0') - blocks_per_sync = bps; + uint64_t threshold = strtoull(options[2].c_str(), &endptr, 0); + if (*endptr == '\0' || !strcmp(endptr, "blocks")) + { + sync_on_blocks = true; + sync_threshold = threshold; + } + else if (!strcmp(endptr, "bytes")) + { + sync_on_blocks = false; + sync_threshold = threshold; + } + else + { + LOG_ERROR("Invalid db sync mode: " << options[2]); + return false; + } } if (db_salvage) @@ -506,11 +595,47 @@ namespace cryptonote } m_blockchain_storage.set_user_options(blocks_threads, - blocks_per_sync, sync_mode, fast_sync); + sync_on_blocks, sync_threshold, sync_mode, fast_sync); - r = m_blockchain_storage.init(db.release(), m_nettype, m_offline, test_options); + try + { + if (!command_line::is_arg_defaulted(vm, arg_block_notify)) + m_blockchain_storage.set_block_notify(std::shared_ptr(new tools::Notify(command_line::get_arg(vm, arg_block_notify).c_str()))); + } + catch (const std::exception &e) + { + MERROR("Failed to parse block notify spec: " << e.what()); + } - r = m_mempool.init(max_txpool_size); + try + { + if (!command_line::is_arg_defaulted(vm, arg_reorg_notify)) + m_blockchain_storage.set_reorg_notify(std::shared_ptr(new tools::Notify(command_line::get_arg(vm, arg_reorg_notify).c_str()))); + } + catch (const std::exception &e) + { + MERROR("Failed to parse reorg notify spec: " << e.what()); + } + + try + { + if (!command_line::is_arg_defaulted(vm, arg_block_rate_notify)) + m_block_rate_notify.reset(new tools::Notify(command_line::get_arg(vm, arg_block_rate_notify).c_str())); + } + catch (const std::exception &e) + { + MERROR("Failed to parse block rate notify spec: " << e.what()); + } + + const std::pair regtest_hard_forks[3] = {std::make_pair(1, 0), std::make_pair(Blockchain::get_hard_fork_heights(MAINNET).back().version, 1), std::make_pair(0, 0)}; + const cryptonote::test_options regtest_test_options = { + regtest_hard_forks, + 0 + }; + const difficulty_type fixed_difficulty = command_line::get_arg(vm, arg_fixed_difficulty); + r = m_blockchain_storage.init(db.release(), m_nettype, m_offline, regtest ? ®test_test_options : test_options, fixed_difficulty, get_checkpoints); + + r = m_mempool.init(max_txpool_weight); CHECK_AND_ASSERT_MES(r, false, "Failed to initialize memory pool"); // now that we have a valid m_blockchain_storage, we can clean out any @@ -546,6 +671,20 @@ namespace cryptonote r = m_miner.init(vm, m_nettype); CHECK_AND_ASSERT_MES(r, false, "Failed to initialize miner instance"); + if (prune_blockchain) + { + // display a message if the blockchain is not pruned yet + if (!m_blockchain_storage.get_blockchain_pruning_seed()) + { + MGINFO("Pruning blockchain..."); + CHECK_AND_ASSERT_MES(m_blockchain_storage.prune_blockchain(), false, "Failed to prune blockchain"); + } + else + { + CHECK_AND_ASSERT_MES(m_blockchain_storage.update_blockchain_pruning(), false, "Failed to update blockchain pruning"); + } + } + return load_state_data(); } //----------------------------------------------------------------------------------------------- @@ -594,7 +733,7 @@ namespace cryptonote return false; } //----------------------------------------------------------------------------------------------- - bool core::handle_incoming_tx_pre(const blobdata& tx_blob, tx_verification_context& tvc, cryptonote::transaction &tx, crypto::hash &tx_hash, crypto::hash &tx_prefixt_hash, bool keeped_by_block, bool relayed, bool do_not_relay) + bool core::handle_incoming_tx_pre(const blobdata& tx_blob, tx_verification_context& tvc, cryptonote::transaction &tx, crypto::hash &tx_hash, bool keeped_by_block, bool relayed, bool do_not_relay) { tvc = boost::value_initialized(); @@ -607,9 +746,8 @@ namespace cryptonote } tx_hash = crypto::null_hash; - tx_prefixt_hash = crypto::null_hash; - if(!parse_tx_from_blob(tx, tx_hash, tx_prefixt_hash, tx_blob)) + if(!parse_tx_from_blob(tx, tx_hash, tx_blob)) { LOG_PRINT_L1("WRONG TRANSACTION BLOB, Failed to parse, rejected"); tvc.m_verifivation_failed = true; @@ -642,7 +780,7 @@ namespace cryptonote return true; } //----------------------------------------------------------------------------------------------- - bool core::handle_incoming_tx_post(const blobdata& tx_blob, tx_verification_context& tvc, cryptonote::transaction &tx, crypto::hash &tx_hash, crypto::hash &tx_prefixt_hash, bool keeped_by_block, bool relayed, bool do_not_relay) + bool core::handle_incoming_tx_post(const blobdata& tx_blob, tx_verification_context& tvc, cryptonote::transaction &tx, crypto::hash &tx_hash, bool keeped_by_block, bool relayed, bool do_not_relay) { if(!check_tx_syntax(tx)) { @@ -651,80 +789,195 @@ namespace cryptonote return false; } + return true; + } + //----------------------------------------------------------------------------------------------- + void core::set_semantics_failed(const crypto::hash &tx_hash) + { + LOG_PRINT_L1("WRONG TRANSACTION BLOB, Failed to check tx " << tx_hash << " semantic, rejected"); + bad_semantics_txes_lock.lock(); + bad_semantics_txes[0].insert(tx_hash); + if (bad_semantics_txes[0].size() >= BAD_SEMANTICS_TXES_MAX_SIZE) + { + std::swap(bad_semantics_txes[0], bad_semantics_txes[1]); + bad_semantics_txes[0].clear(); + } + bad_semantics_txes_lock.unlock(); + } + //----------------------------------------------------------------------------------------------- + static bool is_canonical_bulletproof_layout(const std::vector &proofs) + { + if (proofs.size() != 1) + return false; + const size_t sz = proofs[0].V.size(); + if (sz == 0 || sz > BULLETPROOF_MAX_OUTPUTS) + return false; + return true; + } + //----------------------------------------------------------------------------------------------- + bool core::handle_incoming_tx_accumulated_batch(std::vector &tx_info, bool keeped_by_block) + { + bool ret = true; if (keeped_by_block && get_blockchain_storage().is_within_compiled_block_hash_area()) { MTRACE("Skipping semantics check for tx kept by block in embedded hash area"); + return true; } - else if(!check_tx_semantic(tx, keeped_by_block)) + + std::vector rvv; + for (size_t n = 0; n < tx_info.size(); ++n) { - LOG_PRINT_L1("WRONG TRANSACTION BLOB, Failed to check tx " << tx_hash << " semantic, rejected"); - tvc.m_verifivation_failed = true; - bad_semantics_txes_lock.lock(); - bad_semantics_txes[0].insert(tx_hash); - if (bad_semantics_txes[0].size() >= BAD_SEMANTICS_TXES_MAX_SIZE) + if (!check_tx_semantic(*tx_info[n].tx, keeped_by_block)) { - std::swap(bad_semantics_txes[0], bad_semantics_txes[1]); - bad_semantics_txes[0].clear(); + set_semantics_failed(tx_info[n].tx_hash); + tx_info[n].tvc.m_verifivation_failed = true; + tx_info[n].result = false; + continue; + } + + if (tx_info[n].tx->version < 2) + continue; + const rct::rctSig &rv = tx_info[n].tx->rct_signatures; + switch (rv.type) { + case rct::RCTTypeNull: + // coinbase should not come here, so we reject for all other types + MERROR_VER("Unexpected Null rctSig type"); + set_semantics_failed(tx_info[n].tx_hash); + tx_info[n].tvc.m_verifivation_failed = true; + tx_info[n].result = false; + break; + case rct::RCTTypeSimple: + if (!rct::verRctSemanticsSimple(rv)) + { + MERROR_VER("rct signature semantics check failed"); + set_semantics_failed(tx_info[n].tx_hash); + tx_info[n].tvc.m_verifivation_failed = true; + tx_info[n].result = false; + break; + } + break; + case rct::RCTTypeFull: + if (!rct::verRct(rv, true)) + { + MERROR_VER("rct signature semantics check failed"); + set_semantics_failed(tx_info[n].tx_hash); + tx_info[n].tvc.m_verifivation_failed = true; + tx_info[n].result = false; + break; + } + break; + case rct::RCTTypeBulletproof: + case rct::RCTTypeBulletproof2: + if (!is_canonical_bulletproof_layout(rv.p.bulletproofs)) + { + MERROR_VER("Bulletproof does not have canonical form"); + set_semantics_failed(tx_info[n].tx_hash); + tx_info[n].tvc.m_verifivation_failed = true; + tx_info[n].result = false; + break; + } + rvv.push_back(&rv); // delayed batch verification + break; + default: + MERROR_VER("Unknown rct type: " << rv.type); + set_semantics_failed(tx_info[n].tx_hash); + tx_info[n].tvc.m_verifivation_failed = true; + tx_info[n].result = false; + break; + } + } + if (!rvv.empty() && !rct::verRctSemanticsSimple(rvv)) + { + LOG_PRINT_L1("One transaction among this group has bad semantics, verifying one at a time"); + ret = false; + const bool assumed_bad = rvv.size() == 1; // if there's only one tx, it must be the bad one + for (size_t n = 0; n < tx_info.size(); ++n) + { + if (!tx_info[n].result) + continue; + if (tx_info[n].tx->rct_signatures.type != rct::RCTTypeBulletproof && tx_info[n].tx->rct_signatures.type != rct::RCTTypeBulletproof2) + continue; + if (assumed_bad || !rct::verRctSemanticsSimple(tx_info[n].tx->rct_signatures)) + { + set_semantics_failed(tx_info[n].tx_hash); + tx_info[n].tvc.m_verifivation_failed = true; + tx_info[n].result = false; + } } - bad_semantics_txes_lock.unlock(); - return false; } - return true; + return ret; } //----------------------------------------------------------------------------------------------- - bool core::handle_incoming_txs(const std::list& tx_blobs, std::vector& tvc, bool keeped_by_block, bool relayed, bool do_not_relay) + bool core::handle_incoming_txs(const std::vector& tx_blobs, std::vector& tvc, bool keeped_by_block, bool relayed, bool do_not_relay) { TRY_ENTRY(); + CRITICAL_REGION_LOCAL(m_incoming_tx_lock); - struct result { bool res; cryptonote::transaction tx; crypto::hash hash; crypto::hash prefix_hash; bool in_txpool; bool in_blockchain; }; + struct result { bool res; cryptonote::transaction tx; crypto::hash hash; }; std::vector results(tx_blobs.size()); tvc.resize(tx_blobs.size()); + tools::threadpool& tpool = tools::threadpool::getInstance(); tools::threadpool::waiter waiter; - std::list::const_iterator it = tx_blobs.begin(); + std::vector::const_iterator it = tx_blobs.begin(); for (size_t i = 0; i < tx_blobs.size(); i++, ++it) { - m_threadpool.submit(&waiter, [&, i, it] { + tpool.submit(&waiter, [&, i, it] { try { - results[i].res = handle_incoming_tx_pre(*it, tvc[i], results[i].tx, results[i].hash, results[i].prefix_hash, keeped_by_block, relayed, do_not_relay); + results[i].res = handle_incoming_tx_pre(*it, tvc[i], results[i].tx, results[i].hash, keeped_by_block, relayed, do_not_relay); } catch (const std::exception &e) { MERROR_VER("Exception in handle_incoming_tx_pre: " << e.what()); + tvc[i].m_verifivation_failed = true; results[i].res = false; } }); } - waiter.wait(); + waiter.wait(&tpool); it = tx_blobs.begin(); + std::vector already_have(tx_blobs.size(), false); for (size_t i = 0; i < tx_blobs.size(); i++, ++it) { if (!results[i].res) continue; if(m_mempool.have_tx(results[i].hash)) { LOG_PRINT_L2("tx " << results[i].hash << "already have transaction in tx_pool"); + already_have[i] = true; } else if(m_blockchain_storage.have_tx(results[i].hash)) { LOG_PRINT_L2("tx " << results[i].hash << " already have transaction in blockchain"); + already_have[i] = true; } else { - m_threadpool.submit(&waiter, [&, i, it] { + tpool.submit(&waiter, [&, i, it] { try { - results[i].res = handle_incoming_tx_post(*it, tvc[i], results[i].tx, results[i].hash, results[i].prefix_hash, keeped_by_block, relayed, do_not_relay); + results[i].res = handle_incoming_tx_post(*it, tvc[i], results[i].tx, results[i].hash, keeped_by_block, relayed, do_not_relay); } catch (const std::exception &e) { MERROR_VER("Exception in handle_incoming_tx_post: " << e.what()); + tvc[i].m_verifivation_failed = true; results[i].res = false; } }); } } - waiter.wait(); + waiter.wait(&tpool); + + std::vector tx_info; + tx_info.reserve(tx_blobs.size()); + for (size_t i = 0; i < tx_blobs.size(); i++) { + if (!results[i].res || already_have[i]) + continue; + tx_info.push_back({&results[i].tx, results[i].hash, tvc[i], results[i].res}); + } + if (!tx_info.empty()) + handle_incoming_tx_accumulated_batch(tx_info, keeped_by_block); bool ok = true; it = tx_blobs.begin(); @@ -734,8 +987,13 @@ namespace cryptonote ok = false; continue; } + if (keeped_by_block) + get_blockchain_storage().on_new_tx_from_block(results[i].tx); + if (already_have[i]) + continue; - ok &= add_new_tx(results[i].tx, results[i].hash, results[i].prefix_hash, it->size(), tvc[i], keeped_by_block, relayed, do_not_relay); + const size_t weight = get_transaction_weight(results[i].tx, it->size()); + ok &= add_new_tx(results[i].tx, results[i].hash, tx_blobs[i], weight, tvc[i], keeped_by_block, relayed, do_not_relay); if(tvc[i].m_verifivation_failed) {MERROR_VER("Transaction verification failed: " << results[i].hash);} else if(tvc[i].m_verifivation_impossible) @@ -751,7 +1009,7 @@ namespace cryptonote //----------------------------------------------------------------------------------------------- bool core::handle_incoming_tx(const blobdata& tx_blob, tx_verification_context& tvc, bool keeped_by_block, bool relayed, bool do_not_relay) { - std::list tx_blobs; + std::vector tx_blobs; tx_blobs.push_back(tx_blob); std::vector tvcv(1); bool r = handle_incoming_txs(tx_blobs, tvcv, keeped_by_block, relayed, do_not_relay); @@ -818,9 +1076,9 @@ namespace cryptonote } // for version > 1, ringct signatures check verifies amounts match - if(!keeped_by_block && get_object_blobsize(tx) >= m_blockchain_storage.get_current_cumulative_blocksize_limit() - CRYPTONOTE_COINBASE_BLOB_RESERVED_SIZE) + if(!keeped_by_block && get_transaction_weight(tx) >= m_blockchain_storage.get_current_cumulative_block_weight_limit() - CRYPTONOTE_COINBASE_BLOB_RESERVED_SIZE) { - MERROR_VER("tx is too large " << get_object_blobsize(tx) << ", expected not bigger than " << m_blockchain_storage.get_current_cumulative_blocksize_limit() - CRYPTONOTE_COINBASE_BLOB_RESERVED_SIZE); + MERROR_VER("tx is too large " << get_transaction_weight(tx) << ", expected not bigger than " << m_blockchain_storage.get_current_cumulative_block_weight_limit() - CRYPTONOTE_COINBASE_BLOB_RESERVED_SIZE); return false; } @@ -843,36 +1101,6 @@ namespace cryptonote return false; } - if (tx.version >= 2) - { - const rct::rctSig &rv = tx.rct_signatures; - switch (rv.type) { - case rct::RCTTypeNull: - // coinbase should not come here, so we reject for all other types - MERROR_VER("Unexpected Null rctSig type"); - return false; - case rct::RCTTypeSimple: - case rct::RCTTypeSimpleBulletproof: - if (!rct::verRctSimple(rv, true)) - { - MERROR_VER("rct signature semantics check failed"); - return false; - } - break; - case rct::RCTTypeFull: - case rct::RCTTypeFullBulletproof: - if (!rct::verRct(rv, true)) - { - MERROR_VER("rct signature semantics check failed"); - return false; - } - break; - default: - MERROR_VER("Unknown rct type: " << rv.type); - return false; - } - } - return true; } //----------------------------------------------------------------------------------------------- @@ -917,8 +1145,8 @@ namespace cryptonote const uint64_t end = start_offset + count - 1; m_blockchain_storage.for_blocks_range(start_offset, end, [this, &emission_amount, &total_fee_amount](uint64_t, const crypto::hash& hash, const block& b){ - std::list txs; - std::list missed_txs; + std::vector txs; + std::vector missed_txs; uint64_t coinbase_amount = get_outs_money_amount(b.miner_tx); this->get_transactions(b.tx_hashes, txs, missed_txs); uint64_t tx_fee_amount = 0; @@ -979,10 +1207,10 @@ namespace cryptonote bool core::add_new_tx(transaction& tx, tx_verification_context& tvc, bool keeped_by_block, bool relayed, bool do_not_relay) { crypto::hash tx_hash = get_transaction_hash(tx); - crypto::hash tx_prefix_hash = get_transaction_prefix_hash(tx); blobdata bl; t_serializable_object_to_blob(tx, bl); - return add_new_tx(tx, tx_hash, tx_prefix_hash, bl.size(), tvc, keeped_by_block, relayed, do_not_relay); + size_t tx_weight = get_transaction_weight(tx, bl.size()); + return add_new_tx(tx, tx_hash, bl, tx_weight, tvc, keeped_by_block, relayed, do_not_relay); } //----------------------------------------------------------------------------------------------- size_t core::get_blockchain_total_transactions() const @@ -990,11 +1218,8 @@ namespace cryptonote return m_blockchain_storage.get_total_transactions(); } //----------------------------------------------------------------------------------------------- - bool core::add_new_tx(transaction& tx, const crypto::hash& tx_hash, const crypto::hash& tx_prefix_hash, size_t blob_size, tx_verification_context& tvc, bool keeped_by_block, bool relayed, bool do_not_relay) + bool core::add_new_tx(transaction& tx, const crypto::hash& tx_hash, const cryptonote::blobdata &blob, size_t tx_weight, tx_verification_context& tvc, bool keeped_by_block, bool relayed, bool do_not_relay) { - if (keeped_by_block) - get_blockchain_storage().on_new_tx_from_block(tx); - if(m_mempool.have_tx(tx_hash)) { LOG_PRINT_L2("tx " << tx_hash << "already have transaction in tx_pool"); @@ -1008,13 +1233,13 @@ namespace cryptonote } uint8_t version = m_blockchain_storage.get_current_hard_fork_version(); - return m_mempool.add_tx(tx, tx_hash, blob_size, tvc, keeped_by_block, relayed, do_not_relay, version); + return m_mempool.add_tx(tx, tx_hash, blob, tx_weight, tvc, keeped_by_block, relayed, do_not_relay, version); } //----------------------------------------------------------------------------------------------- bool core::relay_txpool_transactions() { // we attempt to relay txes that should be relayed, but were not - std::list> txs; + std::vector> txs; if (m_mempool.get_relayable_transactions(txs) && !txs.empty()) { cryptonote_connection_context fake_context = AUTO_VAL_INIT(fake_context); @@ -1032,10 +1257,10 @@ namespace cryptonote //----------------------------------------------------------------------------------------------- void core::on_transaction_relayed(const cryptonote::blobdata& tx_blob) { - std::list> txs; + std::vector> txs; cryptonote::transaction tx; - crypto::hash tx_hash, tx_prefix_hash; - if (!parse_and_validate_tx_from_blob(tx_blob, tx, tx_hash, tx_prefix_hash)) + crypto::hash tx_hash; + if (!parse_and_validate_tx_from_blob(tx_blob, tx, tx_hash)) { LOG_ERROR("Failed to parse relayed transaction"); return; @@ -1049,19 +1274,19 @@ namespace cryptonote return m_blockchain_storage.create_block_template(b, adr, diffic, height, expected_reward, ex_nonce); } //----------------------------------------------------------------------------------------------- - bool core::find_blockchain_supplement(const std::list& qblock_ids, NOTIFY_RESPONSE_CHAIN_ENTRY::request& resp) const + bool core::get_block_template(block& b, const crypto::hash *prev_block, const account_public_address& adr, difficulty_type& diffic, uint64_t& height, uint64_t& expected_reward, const blobdata& ex_nonce) { - return m_blockchain_storage.find_blockchain_supplement(qblock_ids, resp); + return m_blockchain_storage.create_block_template(b, prev_block, adr, diffic, height, expected_reward, ex_nonce); } //----------------------------------------------------------------------------------------------- - bool core::find_blockchain_supplement(const uint64_t req_start_block, const std::list& qblock_ids, std::list > >& blocks, uint64_t& total_height, uint64_t& start_height, bool pruned, size_t max_count) const + bool core::find_blockchain_supplement(const std::list& qblock_ids, NOTIFY_RESPONSE_CHAIN_ENTRY::request& resp) const { - return m_blockchain_storage.find_blockchain_supplement(req_start_block, qblock_ids, blocks, total_height, start_height, pruned, max_count); + return m_blockchain_storage.find_blockchain_supplement(qblock_ids, resp); } //----------------------------------------------------------------------------------------------- - bool core::get_random_outs_for_amounts(const COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::request& req, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::response& res) const + bool core::find_blockchain_supplement(const uint64_t req_start_block, const std::list& qblock_ids, std::vector, std::vector > > >& blocks, uint64_t& total_height, uint64_t& start_height, bool pruned, bool get_miner_tx_hash, size_t max_count) const { - return m_blockchain_storage.get_random_outs_for_amounts(req, res); + return m_blockchain_storage.find_blockchain_supplement(req_start_block, qblock_ids, blocks, total_height, start_height, pruned, get_miner_tx_hash, max_count); } //----------------------------------------------------------------------------------------------- bool core::get_outs(const COMMAND_RPC_GET_OUTPUTS_BIN::request& req, COMMAND_RPC_GET_OUTPUTS_BIN::response& res) const @@ -1069,11 +1294,6 @@ namespace cryptonote return m_blockchain_storage.get_outs(req, res); } //----------------------------------------------------------------------------------------------- - bool core::get_random_rct_outs(const COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS::request& req, COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS::response& res) const - { - return m_blockchain_storage.get_random_rct_outs(req, res); - } - //----------------------------------------------------------------------------------------------- bool core::get_output_distribution(uint64_t amount, uint64_t from_height, uint64_t to_height, uint64_t &start_height, std::vector &distribution, uint64_t &base) const { return m_blockchain_storage.get_output_distribution(amount, from_height, to_height, start_height, distribution, base); @@ -1084,6 +1304,11 @@ namespace cryptonote return m_blockchain_storage.get_tx_outputs_gindexs(tx_id, indexs); } //----------------------------------------------------------------------------------------------- + bool core::get_tx_outputs_gindexs(const crypto::hash& tx_id, size_t n_txes, std::vector>& indexs) const + { + return m_blockchain_storage.get_tx_outputs_gindexs(tx_id, n_txes, indexs); + } + //----------------------------------------------------------------------------------------------- void core::pause_mine() { m_miner.pause(); @@ -1107,11 +1332,11 @@ namespace cryptonote return bce; } //----------------------------------------------------------------------------------------------- - bool core::handle_block_found(block& b) + bool core::handle_block_found(block& b, block_verification_context &bvc) { - block_verification_context bvc = boost::value_initialized(); + bvc = boost::value_initialized(); m_miner.pause(); - std::list blocks; + std::vector blocks; try { blocks.push_back(get_block_complete_entry(b, m_mempool)); @@ -1121,7 +1346,13 @@ namespace cryptonote m_miner.resume(); return false; } - prepare_handle_incoming_blocks(blocks); + std::vector pblocks; + if (!prepare_handle_incoming_blocks(blocks, pblocks)) + { + MERROR("Block found, but failed to prepare to add"); + m_miner.resume(); + return false; + } m_blockchain_storage.add_new_block(b, bvc); cleanup_handle_incoming_blocks(true); //anyway - update miner template @@ -1135,8 +1366,8 @@ namespace cryptonote cryptonote_connection_context exclude_context = boost::value_initialized(); NOTIFY_NEW_BLOCK::request arg = AUTO_VAL_INIT(arg); arg.current_blockchain_height = m_blockchain_storage.get_current_blockchain_height(); - std::list missed_txs; - std::list txs; + std::vector missed_txs; + std::vector txs; m_blockchain_storage.get_transactions_blobs(b.tx_hashes, txs, missed_txs); if(missed_txs.size() && m_blockchain_storage.get_block_id_by_height(get_block_height(b)) != get_block_hash(b)) { @@ -1153,7 +1384,7 @@ namespace cryptonote m_pprotocol->relay_block(arg, exclude_context); } - return bvc.m_added_to_main_chain; + return true; } //----------------------------------------------------------------------------------------------- void core::on_synchronized() @@ -1172,10 +1403,14 @@ namespace cryptonote } //----------------------------------------------------------------------------------------------- - bool core::prepare_handle_incoming_blocks(const std::list &blocks) + bool core::prepare_handle_incoming_blocks(const std::vector &blocks_entry, std::vector &blocks) { m_incoming_tx_lock.lock(); - m_blockchain_storage.prepare_handle_incoming_blocks(blocks); + if (!m_blockchain_storage.prepare_handle_incoming_blocks(blocks_entry, blocks)) + { + cleanup_handle_incoming_blocks(false); + return false; + } return true; } @@ -1192,30 +1427,38 @@ namespace cryptonote } //----------------------------------------------------------------------------------------------- - bool core::handle_incoming_block(const blobdata& block_blob, block_verification_context& bvc, bool update_miner_blocktemplate) + bool core::handle_incoming_block(const blobdata& block_blob, const block *b, block_verification_context& bvc, bool update_miner_blocktemplate) { TRY_ENTRY(); - // load json & DNS checkpoints every 10min/hour respectively, - // and verify them with respect to what blocks we already have - CHECK_AND_ASSERT_MES(update_checkpoints(), false, "One or more checkpoints loaded from json or dns conflicted with existing checkpoints."); - bvc = boost::value_initialized(); - if(block_blob.size() > get_max_block_size()) + + if (!check_incoming_block_size(block_blob)) { - LOG_PRINT_L1("WRONG BLOCK BLOB, too big size " << block_blob.size() << ", rejected"); bvc.m_verifivation_failed = true; return false; } - block b = AUTO_VAL_INIT(b); - if(!parse_and_validate_block_from_blob(block_blob, b)) + if (((size_t)-1) <= 0xffffffff && block_blob.size() >= 0x3fffffff) + MWARNING("This block's size is " << block_blob.size() << ", closing on the 32 bit limit"); + + // load json & DNS checkpoints every 10min/hour respectively, + // and verify them with respect to what blocks we already have + CHECK_AND_ASSERT_MES(update_checkpoints(), false, "One or more checkpoints loaded from json or dns conflicted with existing checkpoints."); + + block lb; + if (!b) { - LOG_PRINT_L1("Failed to parse and validate new block"); - bvc.m_verifivation_failed = true; - return false; + crypto::hash block_hash; + if(!parse_and_validate_block_from_blob(block_blob, lb, block_hash)) + { + LOG_PRINT_L1("Failed to parse and validate new block"); + bvc.m_verifivation_failed = true; + return false; + } + b = &lb; } - add_new_block(b, bvc); + add_new_block(*b, bvc); if(update_miner_blocktemplate && bvc.m_added_to_main_chain) update_miner_block_template(); return true; @@ -1227,9 +1470,13 @@ namespace cryptonote // block_blob bool core::check_incoming_block_size(const blobdata& block_blob) const { - if(block_blob.size() > get_max_block_size()) + // note: we assume block weight is always >= block blob size, so we check incoming + // blob size against the block weight limit, which acts as a sanity check without + // having to parse/weigh first; in fact, since the block blob is the block header + // plus the tx hashes, the weight will typically be much larger than the blob size + if(block_blob.size() > m_blockchain_storage.get_current_cumulative_block_weight_limit() + BLOCK_SIZE_SANITY_LEEWAY) { - LOG_PRINT_L1("WRONG BLOCK BLOB, too big size " << block_blob.size() << ", rejected"); + LOG_PRINT_L1("WRONG BLOCK BLOB, sanity check failed on size " << block_blob.size() << ", rejected"); return false; } return true; @@ -1255,9 +1502,9 @@ namespace cryptonote return m_blockchain_storage.have_block(id); } //----------------------------------------------------------------------------------------------- - bool core::parse_tx_from_blob(transaction& tx, crypto::hash& tx_hash, crypto::hash& tx_prefix_hash, const blobdata& blob) const + bool core::parse_tx_from_blob(transaction& tx, crypto::hash& tx_hash, const blobdata& blob) const { - return parse_and_validate_tx_from_blob(blob, tx, tx_hash, tx_prefix_hash); + return parse_and_validate_tx_from_blob(blob, tx, tx_hash); } //----------------------------------------------------------------------------------------------- bool core::check_tx_syntax(const transaction& tx) const @@ -1265,7 +1512,7 @@ namespace cryptonote return true; } //----------------------------------------------------------------------------------------------- - bool core::get_pool_transactions(std::list& txs, bool include_sensitive_data) const + bool core::get_pool_transactions(std::vector& txs, bool include_sensitive_data) const { m_mempool.get_transactions(txs, include_sensitive_data); return true; @@ -1359,6 +1606,8 @@ namespace cryptonote m_txpool_auto_relayer.do_call(boost::bind(&core::relay_txpool_transactions, this)); m_check_updates_interval.do_call(boost::bind(&core::check_updates, this)); m_check_disk_space_interval.do_call(boost::bind(&core::check_disk_space, this)); + m_block_rate_interval.do_call(boost::bind(&core::check_block_rate, this)); + m_blockchain_pruning_interval.do_call(boost::bind(&core::update_blockchain_pruning, this)); m_miner.on_idle(); m_mempool.on_idle(); return true; @@ -1366,6 +1615,9 @@ namespace cryptonote //----------------------------------------------------------------------------------------------- bool core::check_fork_time() { + if (m_nettype == FAKECHAIN) + return true; + HardFork::State state = m_blockchain_storage.get_hard_fork_state(); const el::Level level = el::Level::Warning; switch (state) { @@ -1401,6 +1653,11 @@ namespace cryptonote return get_blockchain_storage().get_hard_fork_version(height); } //----------------------------------------------------------------------------------------------- + uint64_t core::get_earliest_ideal_height_for_version(uint8_t version) const + { + return get_blockchain_storage().get_earliest_ideal_height_for_version(version); + } + //----------------------------------------------------------------------------------------------- bool core::check_updates() { static const char software[] = "monero"; @@ -1424,10 +1681,14 @@ namespace cryptonote return false; if (tools::vercmp(version.c_str(), MONERO_VERSION) <= 0) + { + m_update_available = false; return true; + } std::string url = tools::get_update_url(software, subdir, buildtag, version, true); MCLOG_CYAN(el::Level::Info, "global", "Version " << version << " of " << software << " for " << buildtag << " is available: " << url << ", SHA256 hash " << hash); + m_update_available = true; if (check_updates_level == UPDATES_NOTIFY) return true; @@ -1538,6 +1799,87 @@ namespace cryptonote return true; } //----------------------------------------------------------------------------------------------- + double factorial(unsigned int n) + { + if (n <= 1) + return 1.0; + double f = n; + while (n-- > 1) + f *= n; + return f; + } + //----------------------------------------------------------------------------------------------- + static double probability1(unsigned int blocks, unsigned int expected) + { + // https://www.umass.edu/wsp/resources/poisson/#computing + return pow(expected, blocks) / (factorial(blocks) * exp(expected)); + } + //----------------------------------------------------------------------------------------------- + static double probability(unsigned int blocks, unsigned int expected) + { + double p = 0.0; + if (blocks <= expected) + { + for (unsigned int b = 0; b <= blocks; ++b) + p += probability1(b, expected); + } + else if (blocks > expected) + { + for (unsigned int b = blocks; b <= expected * 3 /* close enough */; ++b) + p += probability1(b, expected); + } + return p; + } + //----------------------------------------------------------------------------------------------- + bool core::check_block_rate() + { + if (m_offline || m_nettype == FAKECHAIN || m_target_blockchain_height > get_current_blockchain_height()) + { + MDEBUG("Not checking block rate, offline or syncing"); + return true; + } + + static constexpr double threshold = 1. / (864000 / DIFFICULTY_TARGET_V2); // one false positive every 10 days + + const time_t now = time(NULL); + const std::vector timestamps = m_blockchain_storage.get_last_block_timestamps(60); + + static const unsigned int seconds[] = { 5400, 3600, 1800, 1200, 600 }; + for (size_t n = 0; n < sizeof(seconds)/sizeof(seconds[0]); ++n) + { + unsigned int b = 0; + const time_t time_boundary = now - static_cast(seconds[n]); + for (time_t ts: timestamps) b += ts >= time_boundary; + const double p = probability(b, seconds[n] / DIFFICULTY_TARGET_V2); + MDEBUG("blocks in the last " << seconds[n] / 60 << " minutes: " << b << " (probability " << p << ")"); + if (p < threshold) + { + MWARNING("There were " << b << " blocks in the last " << seconds[n] / 60 << " minutes, there might be large hash rate changes, or we might be partitioned, cut off from the Monero network or under attack. Or it could be just sheer bad luck."); + + std::shared_ptr block_rate_notify = m_block_rate_notify; + if (block_rate_notify) + { + auto expected = seconds[n] / DIFFICULTY_TARGET_V2; + block_rate_notify->notify("%t", std::to_string(seconds[n] / 60).c_str(), "%b", std::to_string(b).c_str(), "%e", std::to_string(expected).c_str(), NULL); + } + + break; // no need to look further + } + } + + return true; + } + //----------------------------------------------------------------------------------------------- + bool core::update_blockchain_pruning() + { + return m_blockchain_storage.update_blockchain_pruning(); + } + //----------------------------------------------------------------------------------------------- + bool core::check_blockchain_pruning() + { + return m_blockchain_storage.check_blockchain_pruning(); + } + //----------------------------------------------------------------------------------------------- void core::set_target_blockchain_height(uint64_t target_blockchain_height) { m_target_blockchain_height = target_blockchain_height; @@ -1548,7 +1890,7 @@ namespace cryptonote return m_target_blockchain_height; } //----------------------------------------------------------------------------------------------- - uint64_t core::prevalidate_block_hashes(uint64_t height, const std::list &hashes) + uint64_t core::prevalidate_block_hashes(uint64_t height, const std::vector &hashes) { return get_blockchain_storage().prevalidate_block_hashes(height, hashes); } @@ -1560,6 +1902,16 @@ namespace cryptonote return si.available; } //----------------------------------------------------------------------------------------------- + uint32_t core::get_blockchain_pruning_seed() const + { + return get_blockchain_storage().get_blockchain_pruning_seed(); + } + //----------------------------------------------------------------------------------------------- + bool core::prune_blockchain(uint32_t pruning_seed) + { + return get_blockchain_storage().prune_blockchain(pruning_seed); + } + //----------------------------------------------------------------------------------------------- std::time_t core::get_start_time() const { return start_time; diff --git a/src/cryptonote_core/cryptonote_core.h b/src/cryptonote_core/cryptonote_core.h index 17b5680e5e5..2fcf26a17dd 100644 --- a/src/cryptonote_core/cryptonote_core.h +++ b/src/cryptonote_core/cryptonote_core.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -34,12 +34,10 @@ #include #include -#include #include "cryptonote_protocol/cryptonote_protocol_handler_common.h" #include "storages/portable_storage_template_helper.h" #include "common/download.h" -#include "common/threadpool.h" #include "common/command_line.h" #include "tx_pool.h" #include "blockchain.h" @@ -56,12 +54,16 @@ namespace cryptonote { struct test_options { const std::pair *hard_forks; + const size_t long_term_block_weight_window; }; extern const command_line::arg_descriptor arg_data_dir; extern const command_line::arg_descriptor arg_testnet_on; extern const command_line::arg_descriptor arg_stagenet_on; + extern const command_line::arg_descriptor arg_regtest_on; + extern const command_line::arg_descriptor arg_fixed_difficulty; extern const command_line::arg_descriptor arg_offline; + extern const command_line::arg_descriptor arg_block_download_max_size; /************************************************************************/ /* */ @@ -116,7 +118,7 @@ namespace cryptonote * @param relayed whether or not the transaction was relayed to us * @param do_not_relay whether to prevent the transaction from being relayed * - * @return true if the transaction made it to the transaction pool, otherwise false + * @return true if the transaction was accepted, false otherwise */ bool handle_incoming_tx(const blobdata& tx_blob, tx_verification_context& tvc, bool keeped_by_block, bool relayed, bool do_not_relay); @@ -132,9 +134,9 @@ namespace cryptonote * @param relayed whether or not the transactions were relayed to us * @param do_not_relay whether to prevent the transactions from being relayed * - * @return true if the transactions made it to the transaction pool, otherwise false + * @return true if the transactions were accepted, false otherwise */ - bool handle_incoming_txs(const std::list& tx_blobs, std::vector& tvc, bool keeped_by_block, bool relayed, bool do_not_relay); + bool handle_incoming_txs(const std::vector& tx_blobs, std::vector& tvc, bool keeped_by_block, bool relayed, bool do_not_relay); /** * @brief handles an incoming block @@ -144,20 +146,21 @@ namespace cryptonote * optionally updates the miner's block template. * * @param block_blob the block to be added + * @param block the block to be added, or NULL * @param bvc return-by-reference metadata context about the block's validity * @param update_miner_blocktemplate whether or not to update the miner's block template * * @return false if loading new checkpoints fails, or the block is not * added, otherwise true */ - bool handle_incoming_block(const blobdata& block_blob, block_verification_context& bvc, bool update_miner_blocktemplate = true); + bool handle_incoming_block(const blobdata& block_blob, const block *b, block_verification_context& bvc, bool update_miner_blocktemplate = true); /** * @copydoc Blockchain::prepare_handle_incoming_blocks * * @note see Blockchain::prepare_handle_incoming_blocks */ - bool prepare_handle_incoming_blocks(const std::list &blocks); + bool prepare_handle_incoming_blocks(const std::vector &blocks_entry, std::vector &blocks); /** * @copydoc Blockchain::cleanup_handle_incoming_blocks @@ -192,10 +195,11 @@ namespace cryptonote * the network. * * @param b the block found + * @param bvc returns the block verification flags * * @return true if the block was added to the main chain, otherwise false */ - virtual bool handle_block_found( block& b); + virtual bool handle_block_found(block& b, block_verification_context &bvc); /** * @copydoc Blockchain::create_block_template @@ -203,6 +207,7 @@ namespace cryptonote * @note see Blockchain::create_block_template */ virtual bool get_block_template(block& b, const account_public_address& adr, difficulty_type& diffic, uint64_t& height, uint64_t& expected_reward, const blobdata& ex_nonce); + virtual bool get_block_template(block& b, const crypto::hash *prev_block, const account_public_address& adr, difficulty_type& diffic, uint64_t& height, uint64_t& expected_reward, const blobdata& ex_nonce); /** * @brief called when a transaction is relayed @@ -241,12 +246,12 @@ namespace cryptonote * a miner instance with parameters given on the command line (or defaults) * * @param vm command line parameters - * @param config_subdir subdirectory for config storage * @param test_options configuration options for testing + * @param get_checkpoints if set, will be called to get checkpoints data, must return checkpoints data pointer and size or nullptr if there ain't any checkpoints for specific network type * * @return false if one of the init steps fails, otherwise true */ - bool init(const boost::program_options::variables_map& vm, const char *config_subdir = NULL, const test_options *test_options = NULL); + bool init(const boost::program_options::variables_map& vm, const test_options *test_options = NULL, const GetCheckpointsCallback& get_checkpoints = nullptr); /** * @copydoc Blockchain::reset_and_set_genesis_block @@ -309,25 +314,25 @@ namespace cryptonote void get_blockchain_top(uint64_t& height, crypto::hash& top_id) const; /** - * @copydoc Blockchain::get_blocks(uint64_t, size_t, std::list>&, std::list&) const + * @copydoc Blockchain::get_blocks(uint64_t, size_t, std::vector>&, std::vector&) const * - * @note see Blockchain::get_blocks(uint64_t, size_t, std::list>&, std::list&) const + * @note see Blockchain::get_blocks(uint64_t, size_t, std::vector>&, std::vector&) const */ - bool get_blocks(uint64_t start_offset, size_t count, std::list>& blocks, std::list& txs) const; + bool get_blocks(uint64_t start_offset, size_t count, std::vector>& blocks, std::vector& txs) const; /** - * @copydoc Blockchain::get_blocks(uint64_t, size_t, std::list>&) const + * @copydoc Blockchain::get_blocks(uint64_t, size_t, std::vector>&) const * - * @note see Blockchain::get_blocks(uint64_t, size_t, std::list>&) const + * @note see Blockchain::get_blocks(uint64_t, size_t, std::vector>&) const */ - bool get_blocks(uint64_t start_offset, size_t count, std::list>& blocks) const; + bool get_blocks(uint64_t start_offset, size_t count, std::vector>& blocks) const; /** - * @copydoc Blockchain::get_blocks(uint64_t, size_t, std::list>&) const + * @copydoc Blockchain::get_blocks(uint64_t, size_t, std::vector>&) const * - * @note see Blockchain::get_blocks(uint64_t, size_t, std::list>&) const + * @note see Blockchain::get_blocks(uint64_t, size_t, std::vector>&) const */ - bool get_blocks(uint64_t start_offset, size_t count, std::list& blocks) const; + bool get_blocks(uint64_t start_offset, size_t count, std::vector& blocks) const; /** * @copydoc Blockchain::get_blocks(const t_ids_container&, t_blocks_container&, t_missed_container&) const @@ -352,14 +357,21 @@ namespace cryptonote * * @note see Blockchain::get_transactions */ - bool get_transactions(const std::vector& txs_ids, std::list& txs, std::list& missed_txs) const; + bool get_transactions(const std::vector& txs_ids, std::vector& txs, std::vector& missed_txs) const; /** * @copydoc Blockchain::get_transactions * * @note see Blockchain::get_transactions */ - bool get_transactions(const std::vector& txs_ids, std::list& txs, std::list& missed_txs) const; + bool get_split_transactions_blobs(const std::vector& txs_ids, std::vector>& txs, std::vector& missed_txs) const; + + /** + * @copydoc Blockchain::get_transactions + * + * @note see Blockchain::get_transactions + */ + bool get_transactions(const std::vector& txs_ids, std::vector& txs, std::vector& missed_txs) const; /** * @copydoc Blockchain::get_block_by_hash @@ -371,9 +383,9 @@ namespace cryptonote /** * @copydoc Blockchain::get_alternative_blocks * - * @note see Blockchain::get_alternative_blocks(std::list&) const + * @note see Blockchain::get_alternative_blocks(std::vector&) const */ - bool get_alternative_blocks(std::list& blocks) const; + bool get_alternative_blocks(std::vector& blocks) const; /** * @copydoc Blockchain::get_alternative_blocks_count @@ -430,7 +442,7 @@ namespace cryptonote * * @note see tx_memory_pool::get_transactions */ - bool get_pool_transactions(std::list& txs, bool include_unrelayed_txes = true) const; + bool get_pool_transactions(std::vector& txs, bool include_unrelayed_txes = true) const; /** * @copydoc tx_memory_pool::get_txpool_backlog @@ -513,11 +525,11 @@ namespace cryptonote bool find_blockchain_supplement(const std::list& qblock_ids, NOTIFY_RESPONSE_CHAIN_ENTRY::request& resp) const; /** - * @copydoc Blockchain::find_blockchain_supplement(const uint64_t, const std::list&, std::list > >&, uint64_t&, uint64_t&, size_t) const + * @copydoc Blockchain::find_blockchain_supplement(const uint64_t, const std::list&, std::vector > >&, uint64_t&, uint64_t&, size_t) const * - * @note see Blockchain::find_blockchain_supplement(const uint64_t, const std::list&, std::list > >&, uint64_t&, uint64_t&, size_t) const + * @note see Blockchain::find_blockchain_supplement(const uint64_t, const std::list&, std::vector > >&, uint64_t&, uint64_t&, size_t) const */ - bool find_blockchain_supplement(const uint64_t req_start_block, const std::list& qblock_ids, std::list > >& blocks, uint64_t& total_height, uint64_t& start_height, bool pruned, size_t max_count) const; + bool find_blockchain_supplement(const uint64_t req_start_block, const std::list& qblock_ids, std::vector, std::vector > > >& blocks, uint64_t& total_height, uint64_t& start_height, bool pruned, bool get_miner_tx_hash, size_t max_count) const; /** * @brief gets some stats about the daemon @@ -534,6 +546,7 @@ namespace cryptonote * @note see Blockchain::get_tx_outputs_gindexs */ bool get_tx_outputs_gindexs(const crypto::hash& tx_id, std::vector& indexs) const; + bool get_tx_outputs_gindexs(const crypto::hash& tx_id, size_t n_txes, std::vector>& indexs) const; /** * @copydoc Blockchain::get_tail_id @@ -549,13 +562,6 @@ namespace cryptonote */ difficulty_type get_block_cumulative_difficulty(uint64_t height) const; - /** - * @copydoc Blockchain::get_random_outs_for_amounts - * - * @note see Blockchain::get_random_outs_for_amounts - */ - bool get_random_outs_for_amounts(const COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::request& req, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::response& res) const; - /** * @copydoc Blockchain::get_outs * @@ -563,14 +569,6 @@ namespace cryptonote */ bool get_outs(const COMMAND_RPC_GET_OUTPUTS_BIN::request& req, COMMAND_RPC_GET_OUTPUTS_BIN::response& res) const; - /** - * - * @copydoc Blockchain::get_random_rct_outs - * - * @note see Blockchain::get_random_rct_outs - */ - bool get_random_rct_outs(const COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS::request& req, COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS::response& res) const; - /** * @copydoc Blockchain::get_output_distribution * @@ -662,6 +660,13 @@ namespace cryptonote */ uint8_t get_hard_fork_version(uint64_t height) const; + /** + * @brief return the earliest block a given version may activate + * + * @return what it says above + */ + uint64_t get_earliest_ideal_height_for_version(uint8_t version) const; + /** * @brief gets start_time * @@ -745,6 +750,16 @@ namespace cryptonote */ network_type get_nettype() const { return m_nettype; }; + /** + * @brief check whether an update is known to be available or not + * + * This does not actually trigger a check, but returns the result + * of the last check + * + * @return whether an update is known to be available or not + */ + bool is_update_available() const { return m_update_available; } + /** * @brief get whether fluffy blocks are enabled * @@ -752,12 +767,19 @@ namespace cryptonote */ bool fluffy_blocks_enabled() const { return m_fluffy_blocks_enabled; } + /** + * @brief get whether transaction relay should be padded + * + * @return whether transaction relay should be padded + */ + bool pad_transactions() const { return m_pad_transactions; } + /** * @brief check a set of hashes against the precompiled hash set * * @return number of usable blocks */ - uint64_t prevalidate_block_hashes(uint64_t height, const std::list &hashes); + uint64_t prevalidate_block_hashes(uint64_t height, const std::vector &hashes); /** * @brief get free disk space on the blockchain partition @@ -773,19 +795,49 @@ namespace cryptonote */ bool offline() const { return m_offline; } + /** + * @brief get the blockchain pruning seed + * + * @return the blockchain pruning seed + */ + uint32_t get_blockchain_pruning_seed() const; + + /** + * @brief prune the blockchain + * + * @param pruning_seed the seed to use to prune the chain (0 for default, highly recommended) + * + * @return true iff success + */ + bool prune_blockchain(uint32_t pruning_seed = 0); + + /** + * @brief incrementally prunes blockchain + * + * @return true on success, false otherwise + */ + bool update_blockchain_pruning(); + + /** + * @brief checks the blockchain pruning if enabled + * + * @return true on success, false otherwise + */ + bool check_blockchain_pruning(); + private: /** * @copydoc add_new_tx(transaction&, tx_verification_context&, bool) * * @param tx_hash the transaction's hash - * @param tx_prefix_hash the transaction prefix' hash - * @param blob_size the size of the transaction + * @param blob the transaction as a blob + * @param tx_weight the weight of the transaction * @param relayed whether or not the transaction was relayed to us * @param do_not_relay whether to prevent the transaction from being relayed * */ - bool add_new_tx(transaction& tx, const crypto::hash& tx_hash, const crypto::hash& tx_prefix_hash, size_t blob_size, tx_verification_context& tvc, bool keeped_by_block, bool relayed, bool do_not_relay); + bool add_new_tx(transaction& tx, const crypto::hash& tx_hash, const cryptonote::blobdata &blob, size_t tx_weight, tx_verification_context& tvc, bool keeped_by_block, bool relayed, bool do_not_relay); /** * @brief add a new transaction to the transaction pool @@ -825,7 +877,7 @@ namespace cryptonote * * @note see parse_tx_from_blob(transaction&, crypto::hash&, crypto::hash&, const blobdata&) const */ - bool parse_tx_from_blob(transaction& tx, crypto::hash& tx_hash, crypto::hash& tx_prefix_hash, const blobdata& blob) const; + bool parse_tx_from_blob(transaction& tx, crypto::hash& tx_hash, const blobdata& blob) const; /** * @brief check a transaction's syntax @@ -856,9 +908,12 @@ namespace cryptonote * @return true if all the checks pass, otherwise false */ bool check_tx_semantic(const transaction& tx, bool keeped_by_block) const; + void set_semantics_failed(const crypto::hash &tx_hash); - bool handle_incoming_tx_pre(const blobdata& tx_blob, tx_verification_context& tvc, cryptonote::transaction &tx, crypto::hash &tx_hash, crypto::hash &tx_prefixt_hash, bool keeped_by_block, bool relayed, bool do_not_relay); - bool handle_incoming_tx_post(const blobdata& tx_blob, tx_verification_context& tvc, cryptonote::transaction &tx, crypto::hash &tx_hash, crypto::hash &tx_prefixt_hash, bool keeped_by_block, bool relayed, bool do_not_relay); + bool handle_incoming_tx_pre(const blobdata& tx_blob, tx_verification_context& tvc, cryptonote::transaction &tx, crypto::hash &tx_hash, bool keeped_by_block, bool relayed, bool do_not_relay); + bool handle_incoming_tx_post(const blobdata& tx_blob, tx_verification_context& tvc, cryptonote::transaction &tx, crypto::hash &tx_hash, bool keeped_by_block, bool relayed, bool do_not_relay); + struct tx_verification_batch_info { const cryptonote::transaction *tx; crypto::hash tx_hash; tx_verification_context &tvc; bool &result; }; + bool handle_incoming_tx_accumulated_batch(std::vector &tx_info, bool keeped_by_block); /** * @copydoc miner::on_block_chain_update @@ -939,6 +994,13 @@ namespace cryptonote */ bool check_disk_space(); + /** + * @brief checks block rate, and warns if it's too slow + * + * @return true on success, false otherwise + */ + bool check_block_rate(); + bool m_test_drop_download = true; //!< whether or not to drop incoming blocks (for testing) uint64_t m_test_drop_download_height = 0; //!< height under which to drop incoming blocks, if doing so @@ -963,6 +1025,8 @@ namespace cryptonote epee::math_helper::once_a_time_seconds<60*2, false> m_txpool_auto_relayer; //!< interval for checking re-relaying txpool transactions epee::math_helper::once_a_time_seconds<60*60*12, true> m_check_updates_interval; //!< interval for checking for new versions epee::math_helper::once_a_time_seconds<60*10, true> m_check_disk_space_interval; //!< interval for checking for disk space + epee::math_helper::once_a_time_seconds<90, false> m_block_rate_interval; //!< interval for checking block rate + epee::math_helper::once_a_time_seconds<60*60*5, true> m_blockchain_pruning_interval; //!< interval for incremental blockchain pruning std::atomic m_starter_message_showed; //!< has the "daemon will sync now" message been shown? @@ -970,6 +1034,8 @@ namespace cryptonote network_type m_nettype; //!< which network are we on? + std::atomic m_update_available; + std::string m_checkpoints_path; //!< path to json checkpoints file time_t m_last_dns_checkpoints_update; //!< time when dns checkpoints were last updated time_t m_last_json_checkpoints_update; //!< time when json checkpoints were last updated @@ -984,8 +1050,6 @@ namespace cryptonote std::unordered_set bad_semantics_txes[2]; boost::mutex bad_semantics_txes_lock; - tools::threadpool& m_threadpool; - enum { UPDATES_DISABLED, UPDATES_NOTIFY, @@ -999,6 +1063,9 @@ namespace cryptonote bool m_fluffy_blocks_enabled; bool m_offline; + bool m_pad_transactions; + + std::shared_ptr m_block_rate_notify; }; } diff --git a/src/cryptonote_core/cryptonote_tx_utils.cpp b/src/cryptonote_core/cryptonote_tx_utils.cpp index 071ce591eef..854f3d1c58c 100644 --- a/src/cryptonote_core/cryptonote_tx_utils.cpp +++ b/src/cryptonote_core/cryptonote_tx_utils.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -38,6 +38,7 @@ using namespace epee; #include "cryptonote_tx_utils.h" #include "cryptonote_config.h" #include "cryptonote_basic/miner.h" +#include "cryptonote_basic/tx_extra.h" #include "crypto/crypto.h" #include "crypto/hash.h" #include "ringct/rctSigs.h" @@ -74,7 +75,7 @@ namespace cryptonote LOG_PRINT_L2("destinations include " << num_stdaddresses << " standard addresses and " << num_subaddresses << " subaddresses"); } //--------------------------------------------------------------- - bool construct_miner_tx(size_t height, size_t median_size, uint64_t already_generated_coins, size_t current_block_size, uint64_t fee, const account_public_address &miner_address, transaction& tx, const blobdata& extra_nonce, size_t max_outs, uint8_t hard_fork_version) { + bool construct_miner_tx(size_t height, size_t median_weight, uint64_t already_generated_coins, size_t current_block_weight, uint64_t fee, const account_public_address &miner_address, transaction& tx, const blobdata& extra_nonce, size_t max_outs, uint8_t hard_fork_version) { tx.vin.clear(); tx.vout.clear(); tx.extra.clear(); @@ -84,12 +85,14 @@ namespace cryptonote if(!extra_nonce.empty()) if(!add_extra_nonce_to_tx_extra(tx.extra, extra_nonce)) return false; + if (!sort_tx_extra(tx.extra, tx.extra)) + return false; txin_gen in; in.height = height; uint64_t block_reward; - if(!get_block_reward(median_size, current_block_size, already_generated_coins, block_reward, hard_fork_version)) + if(!get_block_reward(median_weight, current_block_weight, already_generated_coins, block_reward, hard_fork_version)) { LOG_PRINT_L0("Block is too big"); return false; @@ -127,7 +130,7 @@ namespace cryptonote out_amounts[1] += out_amounts[0]; for (size_t n = 1; n < out_amounts.size(); ++n) out_amounts[n - 1] = out_amounts[n]; - out_amounts.resize(out_amounts.size() - 1); + out_amounts.pop_back(); } } else @@ -195,7 +198,7 @@ namespace cryptonote return addr.m_view_public_key; } //--------------------------------------------------------------- - bool construct_tx_with_tx_key(const account_keys& sender_account_keys, const std::unordered_map& subaddresses, std::vector& sources, std::vector& destinations, const boost::optional& change_addr, std::vector extra, transaction& tx, uint64_t unlock_time, const crypto::secret_key &tx_key, const std::vector &additional_tx_keys, bool rct, bool bulletproof, rct::multisig_out *msout, bool shuffle_outs) + bool construct_tx_with_tx_key(const account_keys& sender_account_keys, const std::unordered_map& subaddresses, std::vector& sources, std::vector& destinations, const boost::optional& change_addr, const std::vector &extra, transaction& tx, uint64_t unlock_time, const crypto::secret_key &tx_key, const std::vector &additional_tx_keys, bool rct, const rct::RCTConfig &rct_config, rct::multisig_out *msout, bool shuffle_outs) { hw::device &hwdev = sender_account_keys.get_device(); @@ -223,13 +226,15 @@ namespace cryptonote std::vector tx_extra_fields; if (parse_tx_extra(tx.extra, tx_extra_fields)) { + bool add_dummy_payment_id = true; tx_extra_nonce extra_nonce; if (find_tx_extra_field_by_type(tx_extra_fields, extra_nonce)) { - crypto::hash8 payment_id = null_hash8; - if (get_encrypted_payment_id_from_tx_extra_nonce(extra_nonce.nonce, payment_id)) + crypto::hash payment_id = null_hash; + crypto::hash8 payment_id8 = null_hash8; + if (get_encrypted_payment_id_from_tx_extra_nonce(extra_nonce.nonce, payment_id8)) { - LOG_PRINT_L2("Encrypting payment id " << payment_id); + LOG_PRINT_L2("Encrypting payment id " << payment_id8); crypto::public_key view_key_pub = get_destination_view_key_pub(destinations, change_addr); if (view_key_pub == null_pkey) { @@ -237,28 +242,60 @@ namespace cryptonote return false; } - if (!hwdev.encrypt_payment_id(payment_id, view_key_pub, tx_key)) + if (!hwdev.encrypt_payment_id(payment_id8, view_key_pub, tx_key)) { LOG_ERROR("Failed to encrypt payment id"); return false; } std::string extra_nonce; - set_encrypted_payment_id_to_tx_extra_nonce(extra_nonce, payment_id); + set_encrypted_payment_id_to_tx_extra_nonce(extra_nonce, payment_id8); remove_field_from_tx_extra(tx.extra, typeid(tx_extra_nonce)); if (!add_extra_nonce_to_tx_extra(tx.extra, extra_nonce)) { LOG_ERROR("Failed to add encrypted payment id to tx extra"); return false; } - LOG_PRINT_L1("Encrypted payment ID: " << payment_id); + LOG_PRINT_L1("Encrypted payment ID: " << payment_id8); + add_dummy_payment_id = false; + } + else if (get_payment_id_from_tx_extra_nonce(extra_nonce.nonce, payment_id)) + { + add_dummy_payment_id = false; + } + } + + // we don't add one if we've got more than the usual 1 destination plus change + if (destinations.size() > 2) + add_dummy_payment_id = false; + + if (add_dummy_payment_id) + { + // if we have neither long nor short payment id, add a dummy short one, + // this should end up being the vast majority of txes as time goes on + std::string extra_nonce; + crypto::hash8 payment_id8 = null_hash8; + crypto::public_key view_key_pub = get_destination_view_key_pub(destinations, change_addr); + if (view_key_pub == null_pkey) + { + LOG_ERROR("Failed to get key to encrypt dummy payment id with"); + } + else + { + hwdev.encrypt_payment_id(payment_id8, view_key_pub, tx_key); + set_encrypted_payment_id_to_tx_extra_nonce(extra_nonce, payment_id8); + if (!add_extra_nonce_to_tx_extra(tx.extra, extra_nonce)) + { + LOG_ERROR("Failed to add dummy encrypted payment id to tx extra"); + // continue anyway + } } } } else { - LOG_ERROR("Failed to parse tx extra"); - return false; + MWARNING("Failed to parse tx extra"); + tx_extra_fields.clear(); } struct input_generation_context_data @@ -368,49 +405,12 @@ namespace cryptonote for(const tx_destination_entry& dst_entr: destinations) { CHECK_AND_ASSERT_MES(dst_entr.amount > 0 || tx.version > 1, false, "Destination with wrong amount: " << dst_entr.amount); - crypto::key_derivation derivation; crypto::public_key out_eph_public_key; - // make additional tx pubkey if necessary - keypair additional_txkey; - if (need_additional_txkeys) - { - additional_txkey.sec = additional_tx_keys[output_index]; - if (dst_entr.is_subaddress) - additional_txkey.pub = rct::rct2pk(hwdev.scalarmultKey(rct::pk2rct(dst_entr.addr.m_spend_public_key), rct::sk2rct(additional_txkey.sec))); - else - additional_txkey.pub = rct::rct2pk(hwdev.scalarmultBase(rct::sk2rct(additional_txkey.sec))); - } - - bool r; - if (change_addr && dst_entr.addr == *change_addr) - { - // sending change to yourself; derivation = a*R - r = hwdev.generate_key_derivation(txkey_pub, sender_account_keys.m_view_secret_key, derivation); - CHECK_AND_ASSERT_MES(r, false, "at creation outs: failed to generate_key_derivation(" << txkey_pub << ", " << sender_account_keys.m_view_secret_key << ")"); - } - else - { - // sending to the recipient; derivation = r*A (or s*C in the subaddress scheme) - r = hwdev.generate_key_derivation(dst_entr.addr.m_view_public_key, dst_entr.is_subaddress && need_additional_txkeys ? additional_txkey.sec : tx_key, derivation); - CHECK_AND_ASSERT_MES(r, false, "at creation outs: failed to generate_key_derivation(" << dst_entr.addr.m_view_public_key << ", " << (dst_entr.is_subaddress && need_additional_txkeys ? additional_txkey.sec : tx_key) << ")"); - } - - if (need_additional_txkeys) - { - additional_tx_public_keys.push_back(additional_txkey.pub); - } - - if (tx.version > 1) - { - crypto::secret_key scalar1; - hwdev.derivation_to_scalar(derivation, output_index, scalar1); - amount_keys.push_back(rct::sk2rct(scalar1)); - } - r = hwdev.derive_public_key(derivation, output_index, dst_entr.addr.m_spend_public_key, out_eph_public_key); - CHECK_AND_ASSERT_MES(r, false, "at creation outs: failed to derive_public_key(" << derivation << ", " << output_index << ", "<< dst_entr.addr.m_spend_public_key << ")"); - - hwdev.add_output_key_mapping(dst_entr.addr.m_view_public_key, dst_entr.addr.m_spend_public_key, dst_entr.is_subaddress, output_index, amount_keys.back(), out_eph_public_key); + hwdev.generate_output_ephemeral_keys(tx.version,sender_account_keys, txkey_pub, tx_key, + dst_entr, change_addr, output_index, + need_additional_txkeys, additional_tx_keys, + additional_tx_public_keys, amount_keys, out_eph_public_key); tx_out out; out.amount = dst_entr.amount; @@ -434,6 +434,9 @@ namespace cryptonote add_additional_tx_pub_keys_to_extra(tx.extra, additional_tx_public_keys); } + if (!sort_tx_extra(tx.extra, tx.extra)) + return false; + //check money if(summary_outs_money > summary_inputs_money ) { @@ -491,7 +494,7 @@ namespace cryptonote // the non-simple version is slightly smaller, but assumes all real inputs // are on the same index, so can only be used if there just one ring. - bool use_simple_rct = sources.size() > 1; + bool use_simple_rct = sources.size() > 1 || rct_config.range_proof_type != rct::RangeProofBorromean; if (!use_simple_rct) { @@ -516,6 +519,7 @@ namespace cryptonote uint64_t amount_in = 0, amount_out = 0; rct::ctkeyV inSk; + inSk.reserve(sources.size()); // mixRing indexing is done the other way round for simple rct::ctkeyM mixRing(use_simple_rct ? sources.size() : n_total_outs); rct::keyV destinations; @@ -532,6 +536,7 @@ namespace cryptonote ctkey.dest = rct::sk2rct(in_contexts[i].in_ephemeral.sec); ctkey.mask = sources[i].mask; inSk.push_back(ctkey); + memwipe(&ctkey, sizeof(rct::ctkey)); // inPk: (public key, commitment) // will be done when filling in mixRing if (msout) @@ -587,9 +592,10 @@ namespace cryptonote get_transaction_prefix_hash(tx, tx_prefix_hash); rct::ctkeyV outSk; if (use_simple_rct) - tx.rct_signatures = rct::genRctSimple(rct::hash2rct(tx_prefix_hash), inSk, destinations, inamounts, outamounts, amount_in - amount_out, mixRing, amount_keys, msout ? &kLRki : NULL, msout, index, outSk, bulletproof, hwdev); + tx.rct_signatures = rct::genRctSimple(rct::hash2rct(tx_prefix_hash), inSk, destinations, inamounts, outamounts, amount_in - amount_out, mixRing, amount_keys, msout ? &kLRki : NULL, msout, index, outSk, rct_config, hwdev); else - tx.rct_signatures = rct::genRct(rct::hash2rct(tx_prefix_hash), inSk, destinations, outamounts, mixRing, amount_keys, msout ? &kLRki[0] : NULL, msout, sources[0].real_output, outSk, bulletproof, hwdev); // same index assumption + tx.rct_signatures = rct::genRct(rct::hash2rct(tx_prefix_hash), inSk, destinations, outamounts, mixRing, amount_keys, msout ? &kLRki[0] : NULL, msout, sources[0].real_output, outSk, rct_config, hwdev); // same index assumption + memwipe(inSk.data(), inSk.size() * sizeof(rct::ctkey)); CHECK_AND_ASSERT_MES(tx.vout.size() == outSk.size(), false, "outSk size does not match vout"); @@ -601,7 +607,7 @@ namespace cryptonote return true; } //--------------------------------------------------------------- - bool construct_tx_and_get_tx_key(const account_keys& sender_account_keys, const std::unordered_map& subaddresses, std::vector& sources, std::vector& destinations, const boost::optional& change_addr, std::vector extra, transaction& tx, uint64_t unlock_time, crypto::secret_key &tx_key, std::vector &additional_tx_keys, bool rct, bool bulletproof, rct::multisig_out *msout) + bool construct_tx_and_get_tx_key(const account_keys& sender_account_keys, const std::unordered_map& subaddresses, std::vector& sources, std::vector& destinations, const boost::optional& change_addr, const std::vector &extra, transaction& tx, uint64_t unlock_time, crypto::secret_key &tx_key, std::vector &additional_tx_keys, bool rct, const rct::RCTConfig &rct_config, rct::multisig_out *msout) { hw::device &hwdev = sender_account_keys.get_device(); hwdev.open_tx(tx_key); @@ -619,19 +625,19 @@ namespace cryptonote additional_tx_keys.push_back(keypair::generate(sender_account_keys.get_device()).sec); } - bool r = construct_tx_with_tx_key(sender_account_keys, subaddresses, sources, destinations, change_addr, extra, tx, unlock_time, tx_key, additional_tx_keys, rct, bulletproof, msout); + bool r = construct_tx_with_tx_key(sender_account_keys, subaddresses, sources, destinations, change_addr, extra, tx, unlock_time, tx_key, additional_tx_keys, rct, rct_config, msout); hwdev.close_tx(); return r; } //--------------------------------------------------------------- - bool construct_tx(const account_keys& sender_account_keys, std::vector& sources, const std::vector& destinations, const boost::optional& change_addr, std::vector extra, transaction& tx, uint64_t unlock_time) + bool construct_tx(const account_keys& sender_account_keys, std::vector& sources, const std::vector& destinations, const boost::optional& change_addr, const std::vector &extra, transaction& tx, uint64_t unlock_time) { std::unordered_map subaddresses; subaddresses[sender_account_keys.m_account_address.m_spend_public_key] = {0,0}; crypto::secret_key tx_key; std::vector additional_tx_keys; std::vector destinations_copy = destinations; - return construct_tx_and_get_tx_key(sender_account_keys, subaddresses, sources, destinations_copy, change_addr, extra, tx, unlock_time, tx_key, additional_tx_keys, false, false, NULL); + return construct_tx_and_get_tx_key(sender_account_keys, subaddresses, sources, destinations_copy, change_addr, extra, tx, unlock_time, tx_key, additional_tx_keys, false, { rct::RangeProofBorromean, 0}, NULL); } //--------------------------------------------------------------- bool generate_genesis_block( diff --git a/src/cryptonote_core/cryptonote_tx_utils.h b/src/cryptonote_core/cryptonote_tx_utils.h index a5d149fcad4..d38aa747447 100644 --- a/src/cryptonote_core/cryptonote_tx_utils.h +++ b/src/cryptonote_core/cryptonote_tx_utils.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -37,7 +37,7 @@ namespace cryptonote { //--------------------------------------------------------------- - bool construct_miner_tx(size_t height, size_t median_size, uint64_t already_generated_coins, size_t current_block_size, uint64_t fee, const account_public_address &miner_address, transaction& tx, const blobdata& extra_nonce = blobdata(), size_t max_outs = 999, uint8_t hard_fork_version = 1); + bool construct_miner_tx(size_t height, size_t median_weight, uint64_t already_generated_coins, size_t current_block_weight, uint64_t fee, const account_public_address &miner_address, transaction& tx, const blobdata& extra_nonce = blobdata(), size_t max_outs = 999, uint8_t hard_fork_version = 1); struct tx_source_entry { @@ -73,25 +73,43 @@ namespace cryptonote struct tx_destination_entry { + std::string original; uint64_t amount; //money account_public_address addr; //destination address bool is_subaddress; + bool is_integrated; - tx_destination_entry() : amount(0), addr(AUTO_VAL_INIT(addr)), is_subaddress(false) { } - tx_destination_entry(uint64_t a, const account_public_address &ad, bool is_subaddress) : amount(a), addr(ad), is_subaddress(is_subaddress) { } + tx_destination_entry() : amount(0), addr(AUTO_VAL_INIT(addr)), is_subaddress(false), is_integrated(false) { } + tx_destination_entry(uint64_t a, const account_public_address &ad, bool is_subaddress) : amount(a), addr(ad), is_subaddress(is_subaddress), is_integrated(false) { } + tx_destination_entry(const std::string &o, uint64_t a, const account_public_address &ad, bool is_subaddress) : original(o), amount(a), addr(ad), is_subaddress(is_subaddress), is_integrated(false) { } BEGIN_SERIALIZE_OBJECT() + FIELD(original) VARINT_FIELD(amount) FIELD(addr) FIELD(is_subaddress) + FIELD(is_integrated) END_SERIALIZE() }; //--------------------------------------------------------------- crypto::public_key get_destination_view_key_pub(const std::vector &destinations, const boost::optional& change_addr); - bool construct_tx(const account_keys& sender_account_keys, std::vector &sources, const std::vector& destinations, const boost::optional& change_addr, std::vector extra, transaction& tx, uint64_t unlock_time); - bool construct_tx_with_tx_key(const account_keys& sender_account_keys, const std::unordered_map& subaddresses, std::vector& sources, std::vector& destinations, const boost::optional& change_addr, std::vector extra, transaction& tx, uint64_t unlock_time, const crypto::secret_key &tx_key, const std::vector &additional_tx_keys, bool rct = false, bool bulletproof = false, rct::multisig_out *msout = NULL, bool shuffle_outs = true); - bool construct_tx_and_get_tx_key(const account_keys& sender_account_keys, const std::unordered_map& subaddresses, std::vector& sources, std::vector& destinations, const boost::optional& change_addr, std::vector extra, transaction& tx, uint64_t unlock_time, crypto::secret_key &tx_key, std::vector &additional_tx_keys, bool rct = false, bool bulletproof = false, rct::multisig_out *msout = NULL); + bool construct_tx(const account_keys& sender_account_keys, std::vector &sources, const std::vector& destinations, const boost::optional& change_addr, const std::vector &extra, transaction& tx, uint64_t unlock_time); + bool construct_tx_with_tx_key(const account_keys& sender_account_keys, const std::unordered_map& subaddresses, std::vector& sources, std::vector& destinations, const boost::optional& change_addr, const std::vector &extra, transaction& tx, uint64_t unlock_time, const crypto::secret_key &tx_key, const std::vector &additional_tx_keys, bool rct = false, const rct::RCTConfig &rct_config = { rct::RangeProofBorromean, 0 }, rct::multisig_out *msout = NULL, bool shuffle_outs = true); + bool construct_tx_and_get_tx_key(const account_keys& sender_account_keys, const std::unordered_map& subaddresses, std::vector& sources, std::vector& destinations, const boost::optional& change_addr, const std::vector &extra, transaction& tx, uint64_t unlock_time, crypto::secret_key &tx_key, std::vector &additional_tx_keys, bool rct = false, const rct::RCTConfig &rct_config = { rct::RangeProofBorromean, 0 }, rct::multisig_out *msout = NULL); + bool generate_output_ephemeral_keys(const size_t tx_version, const cryptonote::account_keys &sender_account_keys, const crypto::public_key &txkey_pub, const crypto::secret_key &tx_key, + const cryptonote::tx_destination_entry &dst_entr, const boost::optional &change_addr, const size_t output_index, + const bool &need_additional_txkeys, const std::vector &additional_tx_keys, + std::vector &additional_tx_public_keys, + std::vector &amount_keys, + crypto::public_key &out_eph_public_key) ; + + bool generate_output_ephemeral_keys(const size_t tx_version, const cryptonote::account_keys &sender_account_keys, const crypto::public_key &txkey_pub, const crypto::secret_key &tx_key, + const cryptonote::tx_destination_entry &dst_entr, const boost::optional &change_addr, const size_t output_index, + const bool &need_additional_txkeys, const std::vector &additional_tx_keys, + std::vector &additional_tx_public_keys, + std::vector &amount_keys, + crypto::public_key &out_eph_public_key) ; bool generate_genesis_block( block& bl @@ -102,7 +120,7 @@ namespace cryptonote } BOOST_CLASS_VERSION(cryptonote::tx_source_entry, 1) -BOOST_CLASS_VERSION(cryptonote::tx_destination_entry, 1) +BOOST_CLASS_VERSION(cryptonote::tx_destination_entry, 2) namespace boost { @@ -132,6 +150,13 @@ namespace boost if (ver < 1) return; a & x.is_subaddress; + if (ver < 2) + { + x.is_integrated = false; + return; + } + a & x.original; + a & x.is_integrated; } } } diff --git a/src/cryptonote_core/tx_pool.cpp b/src/cryptonote_core/tx_pool.cpp index bf1fe476e8c..49d5a8ccc6c 100644 --- a/src/cryptonote_core/tx_pool.cpp +++ b/src/cryptonote_core/tx_pool.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -40,7 +40,7 @@ #include "blockchain.h" #include "blockchain_db/blockchain_db.h" #include "common/boost_serialization_helper.h" -#include "common/int-util.h" +#include "int-util.h" #include "misc_language.h" #include "warnings.h" #include "common/perf_timer.h" @@ -80,9 +80,13 @@ namespace cryptonote return amount * ACCEPT_THRESHOLD; } - uint64_t get_transaction_size_limit(uint8_t version) + uint64_t get_transaction_weight_limit(uint8_t version) { - return get_min_block_size(version) - CRYPTONOTE_COINBASE_BLOB_RESERVED_SIZE; + // from v8, limit a tx to 50% of the minimum block weight + if (version >= 8) + return get_min_block_weight(version) / 2 - CRYPTONOTE_COINBASE_BLOB_RESERVED_SIZE; + else + return get_min_block_weight(version) - CRYPTONOTE_COINBASE_BLOB_RESERVED_SIZE; } // This class is meant to create a batch when none currently exists. @@ -91,23 +95,27 @@ namespace cryptonote // the whole prepare/handle/cleanup incoming block sequence. class LockedTXN { public: - LockedTXN(Blockchain &b): m_blockchain(b), m_batch(false) { + LockedTXN(Blockchain &b): m_blockchain(b), m_batch(false), m_active(false) { m_batch = m_blockchain.get_db().batch_start(); + m_active = true; } - ~LockedTXN() { try { if (m_batch) { m_blockchain.get_db().batch_stop(); } } catch (const std::exception &e) { MWARNING("LockedTXN dtor filtering exception: " << e.what()); } } + void commit() { try { if (m_batch && m_active) { m_blockchain.get_db().batch_stop(); m_active = false; } } catch (const std::exception &e) { MWARNING("LockedTXN::commit filtering exception: " << e.what()); } } + void abort() { try { if (m_batch && m_active) { m_blockchain.get_db().batch_abort(); m_active = false; } } catch (const std::exception &e) { MWARNING("LockedTXN::abort filtering exception: " << e.what()); } } + ~LockedTXN() { abort(); } private: Blockchain &m_blockchain; bool m_batch; + bool m_active; }; } //--------------------------------------------------------------------------------- //--------------------------------------------------------------------------------- - tx_memory_pool::tx_memory_pool(Blockchain& bchs): m_blockchain(bchs), m_txpool_max_size(DEFAULT_TXPOOL_MAX_SIZE), m_txpool_size(0) + tx_memory_pool::tx_memory_pool(Blockchain& bchs): m_blockchain(bchs), m_txpool_max_weight(DEFAULT_TXPOOL_MAX_WEIGHT), m_txpool_weight(0), m_cookie(0) { } //--------------------------------------------------------------------------------- - bool tx_memory_pool::add_tx(transaction &tx, /*const crypto::hash& tx_prefix_hash,*/ const crypto::hash &id, size_t blob_size, tx_verification_context& tvc, bool kept_by_block, bool relayed, bool do_not_relay, uint8_t version) + bool tx_memory_pool::add_tx(transaction &tx, /*const crypto::hash& tx_prefix_hash,*/ const crypto::hash &id, const cryptonote::blobdata &blob, size_t tx_weight, tx_verification_context& tvc, bool kept_by_block, bool relayed, bool do_not_relay, uint8_t version) { // this should already be called with that lock, but let's make it explicit for clarity CRITICAL_REGION_LOCAL(m_transactions_lock); @@ -173,17 +181,17 @@ namespace cryptonote fee = tx.rct_signatures.txnFee; } - if (!kept_by_block && !m_blockchain.check_fee(blob_size, fee)) + if (!kept_by_block && !m_blockchain.check_fee(tx_weight, fee)) { tvc.m_verifivation_failed = true; tvc.m_fee_too_low = true; return false; } - size_t tx_size_limit = get_transaction_size_limit(version); - if (!kept_by_block && blob_size > tx_size_limit) + size_t tx_weight_limit = get_transaction_weight_limit(version); + if ((!kept_by_block || version >= HF_VERSION_PER_BYTE_FEE) && tx_weight > tx_weight_limit) { - LOG_PRINT_L1("transaction is too big: " << blob_size << " bytes, maximum size: " << tx_size_limit); + LOG_PRINT_L1("transaction is too heavy: " << tx_weight << " bytes, maximum weight: " << tx_weight_limit); tvc.m_verifivation_failed = true; tvc.m_too_big = true; return false; @@ -220,14 +228,14 @@ namespace cryptonote crypto::hash max_used_block_id = null_hash; uint64_t max_used_block_height = 0; cryptonote::txpool_tx_meta_t meta; - bool ch_inp_res = m_blockchain.check_tx_inputs(tx, max_used_block_height, max_used_block_id, tvc, kept_by_block); + bool ch_inp_res = check_tx_inputs([&tx]()->cryptonote::transaction&{ return tx; }, id, max_used_block_height, max_used_block_id, tvc, kept_by_block); if(!ch_inp_res) { // if the transaction was valid before (kept_by_block), then it // may become valid again, so ignore the failed inputs check. if(kept_by_block) { - meta.blob_size = blob_size; + meta.weight = tx_weight; meta.fee = fee; meta.max_used_block_id = null_hash; meta.max_used_block_height = 0; @@ -239,15 +247,19 @@ namespace cryptonote meta.relayed = relayed; meta.do_not_relay = do_not_relay; meta.double_spend_seen = have_tx_keyimges_as_spent(tx); + meta.bf_padding = 0; memset(meta.padding, 0, sizeof(meta.padding)); try { + if (kept_by_block) + m_parsed_tx_cache.insert(std::make_pair(id, tx)); CRITICAL_REGION_LOCAL1(m_blockchain); LockedTXN lock(m_blockchain); - m_blockchain.add_txpool_tx(tx, meta); - if (!insert_key_images(tx, kept_by_block)) + m_blockchain.add_txpool_tx(id, blob, meta); + if (!insert_key_images(tx, id, kept_by_block)) return false; - m_txs_by_fee_and_receive_time.emplace(std::pair(fee / (double)blob_size, receive_time), id); + m_txs_by_fee_and_receive_time.emplace(std::pair(fee / (double)tx_weight, receive_time), id); + lock.commit(); } catch (const std::exception &e) { @@ -266,7 +278,7 @@ namespace cryptonote }else { //update transactions container - meta.blob_size = blob_size; + meta.weight = tx_weight; meta.kept_by_block = kept_by_block; meta.fee = fee; meta.max_used_block_id = max_used_block_id; @@ -278,17 +290,21 @@ namespace cryptonote meta.relayed = relayed; meta.do_not_relay = do_not_relay; meta.double_spend_seen = false; + meta.bf_padding = 0; memset(meta.padding, 0, sizeof(meta.padding)); try { + if (kept_by_block) + m_parsed_tx_cache.insert(std::make_pair(id, tx)); CRITICAL_REGION_LOCAL1(m_blockchain); LockedTXN lock(m_blockchain); - m_blockchain.remove_txpool_tx(get_transaction_hash(tx)); - m_blockchain.add_txpool_tx(tx, meta); - if (!insert_key_images(tx, kept_by_block)) + m_blockchain.remove_txpool_tx(id); + m_blockchain.add_txpool_tx(id, blob, meta); + if (!insert_key_images(tx, id, kept_by_block)) return false; - m_txs_by_fee_and_receive_time.emplace(std::pair(fee / (double)blob_size, receive_time), id); + m_txs_by_fee_and_receive_time.emplace(std::pair(fee / (double)tx_weight, receive_time), id); + lock.commit(); } catch (const std::exception &e) { @@ -302,11 +318,13 @@ namespace cryptonote } tvc.m_verifivation_failed = false; - m_txpool_size += blob_size; + m_txpool_weight += tx_weight; + + ++m_cookie; - MINFO("Transaction added to pool: txid " << id << " bytes: " << blob_size << " fee/byte: " << (fee / (double)blob_size)); + MINFO("Transaction added to pool: txid " << id << " weight: " << tx_weight << " fee/byte: " << (fee / (double)tx_weight)); - prune(m_txpool_max_size); + prune(m_txpool_max_weight); return true; } @@ -315,36 +333,39 @@ namespace cryptonote { crypto::hash h = null_hash; size_t blob_size = 0; - if (!get_transaction_hash(tx, h, blob_size) || blob_size == 0) + cryptonote::blobdata bl; + t_serializable_object_to_blob(tx, bl); + if (bl.size() == 0 || !get_transaction_hash(tx, h)) return false; - return add_tx(tx, h, blob_size, tvc, keeped_by_block, relayed, do_not_relay, version); + return add_tx(tx, h, bl, get_transaction_weight(tx, bl.size()), tvc, keeped_by_block, relayed, do_not_relay, version); } //--------------------------------------------------------------------------------- - size_t tx_memory_pool::get_txpool_size() const + size_t tx_memory_pool::get_txpool_weight() const { CRITICAL_REGION_LOCAL(m_transactions_lock); - return m_txpool_size; + return m_txpool_weight; } //--------------------------------------------------------------------------------- - void tx_memory_pool::set_txpool_max_size(size_t bytes) + void tx_memory_pool::set_txpool_max_weight(size_t bytes) { CRITICAL_REGION_LOCAL(m_transactions_lock); - m_txpool_max_size = bytes; + m_txpool_max_weight = bytes; } //--------------------------------------------------------------------------------- void tx_memory_pool::prune(size_t bytes) { CRITICAL_REGION_LOCAL(m_transactions_lock); if (bytes == 0) - bytes = m_txpool_max_size; + bytes = m_txpool_max_weight; CRITICAL_REGION_LOCAL1(m_blockchain); LockedTXN lock(m_blockchain); + bool changed = false; // this will never remove the first one, but we don't care auto it = --m_txs_by_fee_and_receive_time.end(); while (it != m_txs_by_fee_and_receive_time.begin()) { - if (m_txpool_size <= bytes) + if (m_txpool_weight <= bytes) break; try { @@ -362,19 +383,20 @@ namespace cryptonote continue; } cryptonote::blobdata txblob = m_blockchain.get_txpool_tx_blob(txid); - cryptonote::transaction tx; - if (!parse_and_validate_tx_from_blob(txblob, tx)) + cryptonote::transaction_prefix tx; + if (!parse_and_validate_tx_prefix_from_blob(txblob, tx)) { MERROR("Failed to parse tx from txpool"); return; } // remove first, in case this throws, so key images aren't removed - MINFO("Pruning tx " << txid << " from txpool: size: " << it->first.second << ", fee/byte: " << it->first.first); + MINFO("Pruning tx " << txid << " from txpool: weight: " << meta.weight << ", fee/byte: " << it->first.first); m_blockchain.remove_txpool_tx(txid); - m_txpool_size -= txblob.size(); - remove_transaction_keyimages(tx); - MINFO("Pruned tx " << txid << " from txpool: size: " << it->first.second << ", fee/byte: " << it->first.first); + m_txpool_weight -= meta.weight; + remove_transaction_keyimages(tx, txid); + MINFO("Pruned tx " << txid << " from txpool: weight: " << meta.weight << ", fee/byte: " << it->first.first); m_txs_by_fee_and_receive_time.erase(it--); + changed = true; } catch (const std::exception &e) { @@ -382,15 +404,17 @@ namespace cryptonote return; } } - if (m_txpool_size > bytes) - MINFO("Pool size after pruning is larger than limit: " << m_txpool_size << "/" << bytes); + lock.commit(); + if (changed) + ++m_cookie; + if (m_txpool_weight > bytes) + MINFO("Pool weight after pruning is larger than limit: " << m_txpool_weight << "/" << bytes); } //--------------------------------------------------------------------------------- - bool tx_memory_pool::insert_key_images(const transaction &tx, bool kept_by_block) + bool tx_memory_pool::insert_key_images(const transaction_prefix &tx, const crypto::hash &id, bool kept_by_block) { for(const auto& in: tx.vin) { - const crypto::hash id = get_transaction_hash(tx); CHECKED_GET_SPECIFIC_VARIANT(in, const txin_to_key, txin, false); std::unordered_set& kei_image_set = m_spent_key_images[txin.k_image]; CHECK_AND_ASSERT_MES(kept_by_block || kei_image_set.size() == 0, false, "internal error: kept_by_block=" << kept_by_block @@ -399,25 +423,24 @@ namespace cryptonote auto ins_res = kei_image_set.insert(id); CHECK_AND_ASSERT_MES(ins_res.second, false, "internal error: try to insert duplicate iterator in key_image set"); } + ++m_cookie; return true; } //--------------------------------------------------------------------------------- //FIXME: Can return early before removal of all of the key images. // At the least, need to make sure that a false return here // is treated properly. Should probably not return early, however. - bool tx_memory_pool::remove_transaction_keyimages(const transaction& tx) + bool tx_memory_pool::remove_transaction_keyimages(const transaction_prefix& tx, const crypto::hash &actual_hash) { CRITICAL_REGION_LOCAL(m_transactions_lock); CRITICAL_REGION_LOCAL1(m_blockchain); // ND: Speedup - // 1. Move transaction hash calcuation outside of loop. ._. - crypto::hash actual_hash = get_transaction_hash(tx); for(const txin_v& vi: tx.vin) { CHECKED_GET_SPECIFIC_VARIANT(vi, const txin_to_key, txin, false); auto it = m_spent_key_images.find(txin.k_image); CHECK_AND_ASSERT_MES(it != m_spent_key_images.end(), false, "failed to find transaction input in key images. img=" << txin.k_image << ENDL - << "transaction id = " << get_transaction_hash(tx)); + << "transaction id = " << actual_hash); std::unordered_set& key_image_set = it->second; CHECK_AND_ASSERT_MES(key_image_set.size(), false, "empty key_image set, img=" << txin.k_image << ENDL << "transaction id = " << actual_hash); @@ -433,17 +456,16 @@ namespace cryptonote } } + ++m_cookie; return true; } //--------------------------------------------------------------------------------- - bool tx_memory_pool::take_tx(const crypto::hash &id, transaction &tx, size_t& blob_size, uint64_t& fee, bool &relayed, bool &do_not_relay, bool &double_spend_seen) + bool tx_memory_pool::take_tx(const crypto::hash &id, transaction &tx, cryptonote::blobdata &txblob, size_t& tx_weight, uint64_t& fee, bool &relayed, bool &do_not_relay, bool &double_spend_seen) { CRITICAL_REGION_LOCAL(m_transactions_lock); CRITICAL_REGION_LOCAL1(m_blockchain); auto sorted_it = find_tx_in_sorted_container(id); - if (sorted_it == m_txs_by_fee_and_receive_time.end()) - return false; try { @@ -454,13 +476,22 @@ namespace cryptonote MERROR("Failed to find tx in txpool"); return false; } - cryptonote::blobdata txblob = m_blockchain.get_txpool_tx_blob(id); - if (!parse_and_validate_tx_from_blob(txblob, tx)) + txblob = m_blockchain.get_txpool_tx_blob(id); + auto ci = m_parsed_tx_cache.find(id); + if (ci != m_parsed_tx_cache.end()) + { + tx = ci->second; + } + else if (!parse_and_validate_tx_from_blob(txblob, tx)) { MERROR("Failed to parse tx from txpool"); return false; } - blob_size = meta.blob_size; + else + { + tx.set_hash(id); + } + tx_weight = meta.weight; fee = meta.fee; relayed = meta.relayed; do_not_relay = meta.do_not_relay; @@ -468,8 +499,9 @@ namespace cryptonote // remove first, in case this throws, so key images aren't removed m_blockchain.remove_txpool_tx(id); - m_txpool_size -= blob_size; - remove_transaction_keyimages(tx); + m_txpool_weight -= tx_weight; + remove_transaction_keyimages(tx, id); + lock.commit(); } catch (const std::exception &e) { @@ -477,7 +509,9 @@ namespace cryptonote return false; } - m_txs_by_fee_and_receive_time.erase(sorted_it); + if (sorted_it != m_txs_by_fee_and_receive_time.end()) + m_txs_by_fee_and_receive_time.erase(sorted_it); + ++m_cookie; return true; } //--------------------------------------------------------------------------------- @@ -500,7 +534,7 @@ namespace cryptonote { CRITICAL_REGION_LOCAL(m_transactions_lock); CRITICAL_REGION_LOCAL1(m_blockchain); - std::unordered_set remove; + std::list> remove; m_blockchain.for_all_txpool_txes([this, &remove](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata*) { uint64_t tx_age = time(nullptr) - meta.receive_time; @@ -518,7 +552,7 @@ namespace cryptonote m_txs_by_fee_and_receive_time.erase(sorted_it); } m_timed_out_transactions.insert(txid); - remove.insert(txid); + remove.push_back(std::make_pair(txid, meta.weight)); } return true; }, false); @@ -526,13 +560,14 @@ namespace cryptonote if (!remove.empty()) { LockedTXN lock(m_blockchain); - for (const crypto::hash &txid: remove) + for (const std::pair &entry: remove) { + const crypto::hash &txid = entry.first; try { cryptonote::blobdata bd = m_blockchain.get_txpool_tx_blob(txid); - cryptonote::transaction tx; - if (!parse_and_validate_tx_from_blob(bd, tx)) + cryptonote::transaction_prefix tx; + if (!parse_and_validate_tx_prefix_from_blob(bd, tx)) { MERROR("Failed to parse tx from txpool"); // continue @@ -541,8 +576,8 @@ namespace cryptonote { // remove first, so we only remove key images if the tx removal succeeds m_blockchain.remove_txpool_tx(txid); - m_txpool_size -= bd.size(); - remove_transaction_keyimages(tx); + m_txpool_weight -= entry.second; + remove_transaction_keyimages(tx, txid); } } catch (const std::exception &e) @@ -551,16 +586,19 @@ namespace cryptonote // ignore error } } + lock.commit(); + ++m_cookie; } return true; } //--------------------------------------------------------------------------------- //TODO: investigate whether boolean return is appropriate - bool tx_memory_pool::get_relayable_transactions(std::list> &txs) const + bool tx_memory_pool::get_relayable_transactions(std::vector> &txs) const { CRITICAL_REGION_LOCAL(m_transactions_lock); CRITICAL_REGION_LOCAL1(m_blockchain); const uint64_t now = time(NULL); + txs.reserve(m_blockchain.get_txpool_tx_count()); m_blockchain.for_all_txpool_txes([this, now, &txs](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata *){ // 0 fee transactions are never relayed if(meta.fee > 0 && !meta.do_not_relay && now - meta.last_relayed_time > get_relay_delay(now, meta.receive_time)) @@ -588,7 +626,7 @@ namespace cryptonote return true; } //--------------------------------------------------------------------------------- - void tx_memory_pool::set_relayed(const std::list> &txs) + void tx_memory_pool::set_relayed(const std::vector> &txs) { CRITICAL_REGION_LOCAL(m_transactions_lock); CRITICAL_REGION_LOCAL1(m_blockchain); @@ -612,6 +650,7 @@ namespace cryptonote // continue } } + lock.commit(); } //--------------------------------------------------------------------------------- size_t tx_memory_pool::get_transactions_count(bool include_unrelayed_txes) const @@ -621,10 +660,11 @@ namespace cryptonote return m_blockchain.get_txpool_tx_count(include_unrelayed_txes); } //--------------------------------------------------------------------------------- - void tx_memory_pool::get_transactions(std::list& txs, bool include_unrelayed_txes) const + void tx_memory_pool::get_transactions(std::vector& txs, bool include_unrelayed_txes) const { CRITICAL_REGION_LOCAL(m_transactions_lock); CRITICAL_REGION_LOCAL1(m_blockchain); + txs.reserve(m_blockchain.get_txpool_tx_count(include_unrelayed_txes)); m_blockchain.for_all_txpool_txes([&txs](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata *bd){ transaction tx; if (!parse_and_validate_tx_from_blob(*bd, tx)) @@ -633,7 +673,8 @@ namespace cryptonote // continue return true; } - txs.push_back(tx); + tx.set_hash(txid); + txs.push_back(std::move(tx)); return true; }, true, include_unrelayed_txes); } @@ -642,6 +683,7 @@ namespace cryptonote { CRITICAL_REGION_LOCAL(m_transactions_lock); CRITICAL_REGION_LOCAL1(m_blockchain); + txs.reserve(m_blockchain.get_txpool_tx_count(include_unrelayed_txes)); m_blockchain.for_all_txpool_txes([&txs](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata *bd){ txs.push_back(txid); return true; @@ -653,8 +695,9 @@ namespace cryptonote CRITICAL_REGION_LOCAL(m_transactions_lock); CRITICAL_REGION_LOCAL1(m_blockchain); const uint64_t now = time(NULL); + backlog.reserve(m_blockchain.get_txpool_tx_count(include_unrelayed_txes)); m_blockchain.for_all_txpool_txes([&backlog, now](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata *bd){ - backlog.push_back({meta.blob_size, meta.fee, meta.receive_time - now}); + backlog.push_back({meta.weight, meta.fee, meta.receive_time - now}); return true; }, false, include_unrelayed_txes); } @@ -666,15 +709,15 @@ namespace cryptonote const uint64_t now = time(NULL); std::map agebytes; stats.txs_total = m_blockchain.get_txpool_tx_count(include_unrelayed_txes); - std::vector sizes; - sizes.reserve(stats.txs_total); - m_blockchain.for_all_txpool_txes([&stats, &sizes, now, &agebytes](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata *bd){ - sizes.push_back(meta.blob_size); - stats.bytes_total += meta.blob_size; - if (!stats.bytes_min || meta.blob_size < stats.bytes_min) - stats.bytes_min = meta.blob_size; - if (meta.blob_size > stats.bytes_max) - stats.bytes_max = meta.blob_size; + std::vector weights; + weights.reserve(stats.txs_total); + m_blockchain.for_all_txpool_txes([&stats, &weights, now, &agebytes](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata *bd){ + weights.push_back(meta.weight); + stats.bytes_total += meta.weight; + if (!stats.bytes_min || meta.weight < stats.bytes_min) + stats.bytes_min = meta.weight; + if (meta.weight > stats.bytes_max) + stats.bytes_max = meta.weight; if (!meta.relayed) stats.num_not_relayed++; stats.fee_total += meta.fee; @@ -686,12 +729,12 @@ namespace cryptonote stats.num_failing++; uint64_t age = now - meta.receive_time + (now == meta.receive_time); agebytes[age].txs++; - agebytes[age].bytes += meta.blob_size; + agebytes[age].bytes += meta.weight; if (meta.double_spend_seen) ++stats.num_double_spends; return true; }, false, include_unrelayed_txes); - stats.bytes_med = epee::misc_utils::median(sizes); + stats.bytes_med = epee::misc_utils::median(weights); if (stats.txs_total > 1) { /* looking for 98th percentile */ @@ -741,6 +784,8 @@ namespace cryptonote { CRITICAL_REGION_LOCAL(m_transactions_lock); CRITICAL_REGION_LOCAL1(m_blockchain); + tx_infos.reserve(m_blockchain.get_txpool_tx_count()); + key_image_infos.reserve(m_blockchain.get_txpool_tx_count()); m_blockchain.for_all_txpool_txes([&tx_infos, key_image_infos, include_sensitive_data](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata *bd){ tx_info txi; txi.id_hash = epee::string_tools::pod_to_hex(txid); @@ -752,8 +797,10 @@ namespace cryptonote // continue return true; } + tx.set_hash(txid); txi.tx_json = obj_to_json_str(tx); - txi.blob_size = meta.blob_size; + txi.blob_size = bd->size(); + txi.weight = meta.weight; txi.fee = meta.fee; txi.kept_by_block = meta.kept_by_block; txi.max_used_block_height = meta.max_used_block_height; @@ -767,7 +814,7 @@ namespace cryptonote txi.last_relayed_time = include_sensitive_data ? meta.last_relayed_time : 0; txi.do_not_relay = meta.do_not_relay; txi.double_spend_seen = meta.double_spend_seen; - tx_infos.push_back(txi); + tx_infos.push_back(std::move(txi)); return true; }, true, include_sensitive_data); @@ -811,18 +858,20 @@ namespace cryptonote { CRITICAL_REGION_LOCAL(m_transactions_lock); CRITICAL_REGION_LOCAL1(m_blockchain); + tx_infos.reserve(m_blockchain.get_txpool_tx_count()); + key_image_infos.reserve(m_blockchain.get_txpool_tx_count()); m_blockchain.for_all_txpool_txes([&tx_infos, key_image_infos](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata *bd){ cryptonote::rpc::tx_in_pool txi; txi.tx_hash = txid; - transaction tx; - if (!parse_and_validate_tx_from_blob(*bd, tx)) + if (!parse_and_validate_tx_from_blob(*bd, txi.tx)) { MERROR("Failed to parse tx from txpool"); // continue return true; } - txi.tx = tx; - txi.blob_size = meta.blob_size; + txi.tx.set_hash(txid); + txi.blob_size = bd->size(); + txi.weight = meta.weight; txi.fee = meta.fee; txi.kept_by_block = meta.kept_by_block; txi.max_used_block_height = meta.max_used_block_height; @@ -847,7 +896,7 @@ namespace cryptonote } const crypto::key_image& k_image = kee.first; - key_image_infos[k_image] = tx_hashes; + key_image_infos[k_image] = std::move(tx_hashes); } return true; } @@ -883,11 +932,17 @@ namespace cryptonote //--------------------------------------------------------------------------------- bool tx_memory_pool::on_blockchain_inc(uint64_t new_block_height, const crypto::hash& top_block_id) { + CRITICAL_REGION_LOCAL(m_transactions_lock); + m_input_cache.clear(); + m_parsed_tx_cache.clear(); return true; } //--------------------------------------------------------------------------------- bool tx_memory_pool::on_blockchain_dec(uint64_t new_block_height, const crypto::hash& top_block_id) { + CRITICAL_REGION_LOCAL(m_transactions_lock); + m_input_cache.clear(); + m_parsed_tx_cache.clear(); return true; } //--------------------------------------------------------------------------------- @@ -927,8 +982,47 @@ namespace cryptonote m_transactions_lock.unlock(); } //--------------------------------------------------------------------------------- - bool tx_memory_pool::is_transaction_ready_to_go(txpool_tx_meta_t& txd, transaction &tx) const + bool tx_memory_pool::check_tx_inputs(const std::function &get_tx, const crypto::hash &txid, uint64_t &max_used_block_height, crypto::hash &max_used_block_id, tx_verification_context &tvc, bool kept_by_block) const + { + if (!kept_by_block) + { + const std::unordered_map>::const_iterator i = m_input_cache.find(txid); + if (i != m_input_cache.end()) + { + max_used_block_height = std::get<2>(i->second); + max_used_block_id = std::get<3>(i->second); + tvc = std::get<1>(i->second); + return std::get<0>(i->second); + } + } + bool ret = m_blockchain.check_tx_inputs(get_tx(), max_used_block_height, max_used_block_id, tvc, kept_by_block); + if (!kept_by_block) + m_input_cache.insert(std::make_pair(txid, std::make_tuple(ret, tvc, max_used_block_height, max_used_block_id))); + return ret; + } + //--------------------------------------------------------------------------------- + bool tx_memory_pool::is_transaction_ready_to_go(txpool_tx_meta_t& txd, const crypto::hash &txid, const cryptonote::blobdata &txblob, transaction &tx) const { + struct transction_parser + { + transction_parser(const cryptonote::blobdata &txblob, const crypto::hash &txid, transaction &tx): txblob(txblob), txid(txid), tx(tx), parsed(false) {} + cryptonote::transaction &operator()() + { + if (!parsed) + { + if (!parse_and_validate_tx_from_blob(txblob, tx)) + throw std::runtime_error("failed to parse transaction blob"); + tx.set_hash(txid); + parsed = true; + } + return tx; + } + const cryptonote::blobdata &txblob; + const crypto::hash &txid; + transaction &tx; + bool parsed; + } lazy_tx(txblob, txid, tx); + //not the best implementation at this time, sorry :( //check is ring_signature already checked ? if(txd.max_used_block_id == null_hash) @@ -938,7 +1032,7 @@ namespace cryptonote return false;//we already sure that this tx is broken for this height tx_verification_context tvc; - if(!m_blockchain.check_tx_inputs(tx, txd.max_used_block_height, txd.max_used_block_id, tvc)) + if(!check_tx_inputs([&lazy_tx]()->cryptonote::transaction&{ return lazy_tx(); }, txid, txd.max_used_block_height, txd.max_used_block_id, tvc)) { txd.last_failed_height = m_blockchain.get_current_blockchain_height()-1; txd.last_failed_id = m_blockchain.get_block_id_by_height(txd.last_failed_height); @@ -955,7 +1049,7 @@ namespace cryptonote return false; //check ring signature again, it is possible (with very small chance) that this transaction become again valid tx_verification_context tvc; - if(!m_blockchain.check_tx_inputs(tx, txd.max_used_block_height, txd.max_used_block_id, tvc)) + if(!check_tx_inputs([&lazy_tx]()->cryptonote::transaction&{ return lazy_tx(); }, txid, txd.max_used_block_height, txd.max_used_block_id, tvc)) { txd.last_failed_height = m_blockchain.get_current_blockchain_height()-1; txd.last_failed_id = m_blockchain.get_block_id_by_height(txd.last_failed_height); @@ -964,7 +1058,7 @@ namespace cryptonote } } //if we here, transaction seems valid, but, anyway, check for key_images collisions with blockchain, just to be sure - if(m_blockchain.have_tx_keyimges_as_spent(tx)) + if(m_blockchain.have_tx_keyimges_as_spent(lazy_tx())) { txd.double_spend_seen = true; return false; @@ -974,7 +1068,7 @@ namespace cryptonote return true; } //--------------------------------------------------------------------------------- - bool tx_memory_pool::have_key_images(const std::unordered_set& k_images, const transaction& tx) + bool tx_memory_pool::have_key_images(const std::unordered_set& k_images, const transaction_prefix& tx) { for(size_t i = 0; i!= tx.vin.size(); i++) { @@ -985,7 +1079,7 @@ namespace cryptonote return false; } //--------------------------------------------------------------------------------- - bool tx_memory_pool::append_key_images(std::unordered_set& k_images, const transaction& tx) + bool tx_memory_pool::append_key_images(std::unordered_set& k_images, const transaction_prefix& tx) { for(size_t i = 0; i!= tx.vin.size(); i++) { @@ -1000,6 +1094,7 @@ namespace cryptonote { CRITICAL_REGION_LOCAL(m_transactions_lock); CRITICAL_REGION_LOCAL1(m_blockchain); + bool changed = false; LockedTXN lock(m_blockchain); for(size_t i = 0; i!= tx.vin.size(); i++) { @@ -1020,6 +1115,7 @@ namespace cryptonote { MDEBUG("Marking " << txid << " as double spending " << itk.k_image); meta.double_spend_seen = true; + changed = true; try { m_blockchain.update_txpool_tx(txid, meta); @@ -1033,6 +1129,9 @@ namespace cryptonote } } } + lock.commit(); + if (changed) + ++m_cookie; } //--------------------------------------------------------------------------------- std::string tx_memory_pool::print_pool(bool short_format) const @@ -1051,7 +1150,8 @@ namespace cryptonote } ss << obj_to_json_str(tx) << std::endl; } - ss << "blob_size: " << meta.blob_size << std::endl + ss << "blob_size: " << (short_format ? "-" : std::to_string(txblob->size())) << std::endl + << "weight: " << meta.weight << std::endl << "fee: " << print_money(meta.fee) << std::endl << "kept_by_block: " << (meta.kept_by_block ? 'T' : 'F') << std::endl << "double_spend_seen: " << (meta.double_spend_seen ? 'T' : 'F') << std::endl @@ -1066,30 +1166,30 @@ namespace cryptonote } //--------------------------------------------------------------------------------- //TODO: investigate whether boolean return is appropriate - bool tx_memory_pool::fill_block_template(block &bl, size_t median_size, uint64_t already_generated_coins, size_t &total_size, uint64_t &fee, uint64_t &expected_reward, uint8_t version) + bool tx_memory_pool::fill_block_template(block &bl, size_t median_weight, uint64_t already_generated_coins, size_t &total_weight, uint64_t &fee, uint64_t &expected_reward, uint8_t version) { CRITICAL_REGION_LOCAL(m_transactions_lock); CRITICAL_REGION_LOCAL1(m_blockchain); uint64_t best_coinbase = 0, coinbase = 0; - total_size = 0; + total_weight = 0; fee = 0; //baseline empty block - get_block_reward(median_size, total_size, already_generated_coins, best_coinbase, version); + get_block_reward(median_weight, total_weight, already_generated_coins, best_coinbase, version); - size_t max_total_size_pre_v5 = (130 * median_size) / 100 - CRYPTONOTE_COINBASE_BLOB_RESERVED_SIZE; - size_t max_total_size_v5 = 2 * median_size - CRYPTONOTE_COINBASE_BLOB_RESERVED_SIZE; - size_t max_total_size = version >= 5 ? max_total_size_v5 : max_total_size_pre_v5; + size_t max_total_weight_pre_v5 = (130 * median_weight) / 100 - CRYPTONOTE_COINBASE_BLOB_RESERVED_SIZE; + size_t max_total_weight_v5 = 2 * median_weight - CRYPTONOTE_COINBASE_BLOB_RESERVED_SIZE; + size_t max_total_weight = version >= 5 ? max_total_weight_v5 : max_total_weight_pre_v5; std::unordered_set k_images; - LOG_PRINT_L2("Filling block template, median size " << median_size << ", " << m_txs_by_fee_and_receive_time.size() << " txes in the pool"); + LOG_PRINT_L2("Filling block template, median weight " << median_weight << ", " << m_txs_by_fee_and_receive_time.size() << " txes in the pool"); LockedTXN lock(m_blockchain); auto sorted_it = m_txs_by_fee_and_receive_time.begin(); - while (sorted_it != m_txs_by_fee_and_receive_time.end()) + for (; sorted_it != m_txs_by_fee_and_receive_time.end(); ++sorted_it) { txpool_tx_meta_t meta; if (!m_blockchain.get_txpool_tx_meta(sorted_it->second, meta)) @@ -1097,13 +1197,12 @@ namespace cryptonote MERROR(" failed to find tx meta"); continue; } - LOG_PRINT_L2("Considering " << sorted_it->second << ", size " << meta.blob_size << ", current block size " << total_size << "/" << max_total_size << ", current coinbase " << print_money(best_coinbase)); + LOG_PRINT_L2("Considering " << sorted_it->second << ", weight " << meta.weight << ", current block weight " << total_weight << "/" << max_total_weight << ", current coinbase " << print_money(best_coinbase)); - // Can not exceed maximum block size - if (max_total_size < total_size + meta.blob_size) + // Can not exceed maximum block weight + if (max_total_weight < total_weight + meta.weight) { - LOG_PRINT_L2(" would exceed maximum block size"); - sorted_it++; + LOG_PRINT_L2(" would exceed maximum block weight"); continue; } @@ -1113,45 +1212,46 @@ namespace cryptonote // If we're getting lower coinbase tx, // stop including more tx uint64_t block_reward; - if(!get_block_reward(median_size, total_size + meta.blob_size, already_generated_coins, block_reward, version)) + if(!get_block_reward(median_weight, total_weight + meta.weight, already_generated_coins, block_reward, version)) { - LOG_PRINT_L2(" would exceed maximum block size"); - sorted_it++; + LOG_PRINT_L2(" would exceed maximum block weight"); continue; } coinbase = block_reward + fee + meta.fee; if (coinbase < template_accept_threshold(best_coinbase)) { LOG_PRINT_L2(" would decrease coinbase to " << print_money(coinbase)); - sorted_it++; continue; } } else { - // If we've exceeded the penalty free size, + // If we've exceeded the penalty free weight, // stop including more tx - if (total_size > median_size) + if (total_weight > median_weight) { - LOG_PRINT_L2(" would exceed median block size"); + LOG_PRINT_L2(" would exceed median block weight"); break; } } cryptonote::blobdata txblob = m_blockchain.get_txpool_tx_blob(sorted_it->second); cryptonote::transaction tx; - if (!parse_and_validate_tx_from_blob(txblob, tx)) - { - MERROR("Failed to parse tx from txpool"); - sorted_it++; - continue; - } // Skip transactions that are not ready to be // included into the blockchain or that are // missing key images const cryptonote::txpool_tx_meta_t original_meta = meta; - bool ready = is_transaction_ready_to_go(meta, tx); + bool ready = false; + try + { + ready = is_transaction_ready_to_go(meta, sorted_it->second, txblob, tx); + } + catch (const std::exception &e) + { + MERROR("Failed to check transaction readiness: " << e.what()); + // continue, not fatal + } if (memcmp(&original_meta, &meta, sizeof(meta))) { try @@ -1167,28 +1267,26 @@ namespace cryptonote if (!ready) { LOG_PRINT_L2(" not ready to go"); - sorted_it++; continue; } if (have_key_images(k_images, tx)) { LOG_PRINT_L2(" key images already seen"); - sorted_it++; continue; } bl.tx_hashes.push_back(sorted_it->second); - total_size += meta.blob_size; + total_weight += meta.weight; fee += meta.fee; best_coinbase = coinbase; append_key_images(k_images, tx); - sorted_it++; - LOG_PRINT_L2(" added, new block size " << total_size << "/" << max_total_size << ", coinbase " << print_money(best_coinbase)); + LOG_PRINT_L2(" added, new block weight " << total_weight << "/" << max_total_weight << ", coinbase " << print_money(best_coinbase)); } + lock.commit(); expected_reward = best_coinbase; - LOG_PRINT_L2("Block template filled with " << bl.tx_hashes.size() << " txes, size " - << total_size << "/" << max_total_size << ", coinbase " << print_money(best_coinbase) + LOG_PRINT_L2("Block template filled with " << bl.tx_hashes.size() << " txes, weight " + << total_weight << "/" << max_total_weight << ", coinbase " << print_money(best_coinbase) << " (including " << print_money(fee) << " in fees)"); return true; } @@ -1197,14 +1295,14 @@ namespace cryptonote { CRITICAL_REGION_LOCAL(m_transactions_lock); CRITICAL_REGION_LOCAL1(m_blockchain); - size_t tx_size_limit = get_transaction_size_limit(version); + size_t tx_weight_limit = get_transaction_weight_limit(version); std::unordered_set remove; - m_txpool_size = 0; - m_blockchain.for_all_txpool_txes([this, &remove, tx_size_limit](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata*) { - m_txpool_size += meta.blob_size; - if (meta.blob_size > tx_size_limit) { - LOG_PRINT_L1("Transaction " << txid << " is too big (" << meta.blob_size << " bytes), removing it from pool"); + m_txpool_weight = 0; + m_blockchain.for_all_txpool_txes([this, &remove, tx_weight_limit](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata*) { + m_txpool_weight += meta.weight; + if (meta.weight > tx_weight_limit) { + LOG_PRINT_L1("Transaction " << txid << " is too big (" << meta.weight << " bytes), removing it from pool"); remove.insert(txid); } else if (m_blockchain.have_tx(txid)) { @@ -1231,8 +1329,8 @@ namespace cryptonote } // remove tx from db first m_blockchain.remove_txpool_tx(txid); - m_txpool_size -= txblob.size(); - remove_transaction_keyimages(tx); + m_txpool_weight -= get_transaction_weight(tx, txblob.size()); + remove_transaction_keyimages(tx, txid); auto sorted_it = find_tx_in_sorted_container(txid); if (sorted_it == m_txs_by_fee_and_receive_time.end()) { @@ -1250,19 +1348,22 @@ namespace cryptonote // continue } } + lock.commit(); } + if (n_removed > 0) + ++m_cookie; return n_removed; } //--------------------------------------------------------------------------------- - bool tx_memory_pool::init(size_t max_txpool_size) + bool tx_memory_pool::init(size_t max_txpool_weight) { CRITICAL_REGION_LOCAL(m_transactions_lock); CRITICAL_REGION_LOCAL1(m_blockchain); - m_txpool_max_size = max_txpool_size ? max_txpool_size : DEFAULT_TXPOOL_MAX_SIZE; + m_txpool_max_weight = max_txpool_weight ? max_txpool_weight : DEFAULT_TXPOOL_MAX_WEIGHT; m_txs_by_fee_and_receive_time.clear(); m_spent_key_images.clear(); - m_txpool_size = 0; + m_txpool_weight = 0; std::vector remove; // first add the not kept by block, then the kept by block, @@ -1273,19 +1374,20 @@ namespace cryptonote bool r = m_blockchain.for_all_txpool_txes([this, &remove, kept](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata *bd) { if (!!kept != !!meta.kept_by_block) return true; - cryptonote::transaction tx; - if (!parse_and_validate_tx_from_blob(*bd, tx)) + cryptonote::transaction_prefix tx; + if (!parse_and_validate_tx_prefix_from_blob(*bd, tx)) { MWARNING("Failed to parse tx from txpool, removing"); remove.push_back(txid); + return true; } - if (!insert_key_images(tx, meta.kept_by_block)) + if (!insert_key_images(tx, txid, meta.kept_by_block)) { MFATAL("Failed to insert key images from txpool tx"); return false; } - m_txs_by_fee_and_receive_time.emplace(std::pair(meta.fee / (double)meta.blob_size, meta.receive_time), txid); - m_txpool_size += meta.blob_size; + m_txs_by_fee_and_receive_time.emplace(std::pair(meta.fee / (double)meta.weight, meta.receive_time), txid); + m_txpool_weight += meta.weight; return true; }, true); if (!r) @@ -1306,7 +1408,12 @@ namespace cryptonote // ignore error } } + lock.commit(); } + + m_cookie = 0; + + // Ignore deserialization error return true; } diff --git a/src/cryptonote_core/tx_pool.h b/src/cryptonote_core/tx_pool.h index 19cd83ed9cb..877f2b82f5e 100644 --- a/src/cryptonote_core/tx_pool.h +++ b/src/cryptonote_core/tx_pool.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -61,7 +61,7 @@ namespace cryptonote class txCompare { public: - bool operator()(const tx_by_fee_and_receive_time_entry& a, const tx_by_fee_and_receive_time_entry& b) + bool operator()(const tx_by_fee_and_receive_time_entry& a, const tx_by_fee_and_receive_time_entry& b) const { // sort by greatest first, not least if (a.first.first > b.first.first) return true; @@ -84,7 +84,7 @@ namespace cryptonote * * This handling includes: * storing the transactions - * organizing the transactions by fee per size + * organizing the transactions by fee per weight unit * taking/giving transactions to and from various other components * saving the transactions to disk on shutdown * helping create a new block template by choosing transactions for it @@ -105,9 +105,9 @@ namespace cryptonote * @copydoc add_tx(transaction&, tx_verification_context&, bool, bool, uint8_t) * * @param id the transaction's hash - * @param blob_size the transaction's size + * @param tx_weight the transaction's weight */ - bool add_tx(transaction &tx, const crypto::hash &id, size_t blob_size, tx_verification_context& tvc, bool kept_by_block, bool relayed, bool do_not_relay, uint8_t version); + bool add_tx(transaction &tx, const crypto::hash &id, const cryptonote::blobdata &blob, size_t tx_weight, tx_verification_context& tvc, bool kept_by_block, bool relayed, bool do_not_relay, uint8_t version); /** * @brief add a transaction to the transaction pool @@ -133,7 +133,8 @@ namespace cryptonote * * @param id the hash of the transaction * @param tx return-by-reference the transaction taken - * @param blob_size return-by-reference the transaction's size + * @param txblob return-by-reference the transaction as a blob + * @param tx_weight return-by-reference the transaction's weight * @param fee the transaction fee * @param relayed return-by-reference was transaction relayed to us by the network? * @param do_not_relay return-by-reference is transaction not to be relayed to the network? @@ -141,7 +142,7 @@ namespace cryptonote * * @return true unless the transaction cannot be found in the pool */ - bool take_tx(const crypto::hash &id, transaction &tx, size_t& blob_size, uint64_t& fee, bool &relayed, bool &do_not_relay, bool &double_spend_seen); + bool take_tx(const crypto::hash &id, transaction &tx, cryptonote::blobdata &txblob, size_t& tx_weight, uint64_t& fee, bool &relayed, bool &do_not_relay, bool &double_spend_seen); /** * @brief checks if the pool has a transaction with the given hash @@ -198,11 +199,11 @@ namespace cryptonote /** * @brief loads pool state (if any) from disk, and initializes pool * - * @param max_txpool_size the max size in bytes + * @param max_txpool_weight the max weight in bytes * * @return true */ - bool init(size_t max_txpool_size = 0); + bool init(size_t max_txpool_weight = 0); /** * @brief attempts to save the transaction pool state to disk @@ -219,16 +220,16 @@ namespace cryptonote * @brief Chooses transactions for a block to include * * @param bl return-by-reference the block to fill in with transactions - * @param median_size the current median block size + * @param median_weight the current median block weight * @param already_generated_coins the current total number of coins "minted" - * @param total_size return-by-reference the total size of the new block + * @param total_weight return-by-reference the total weight of the new block * @param fee return-by-reference the total of fees from the included transactions * @param expected_reward return-by-reference the total reward awarded to the miner finding this block, including transaction fees * @param version hard fork version to use for consensus rules * * @return true */ - bool fill_block_template(block &bl, size_t median_size, uint64_t already_generated_coins, size_t &total_size, uint64_t &fee, uint64_t &expected_reward, uint8_t version); + bool fill_block_template(block &bl, size_t median_weight, uint64_t already_generated_coins, size_t &total_weight, uint64_t &fee, uint64_t &expected_reward, uint8_t version); /** * @brief get a list of all transactions in the pool @@ -237,7 +238,7 @@ namespace cryptonote * @param include_unrelayed_txes include unrelayed txes in the result * */ - void get_transactions(std::list& txs, bool include_unrelayed_txes = true) const; + void get_transactions(std::vector& txs, bool include_unrelayed_txes = true) const; /** * @brief get a list of all transaction hashes in the pool @@ -249,7 +250,7 @@ namespace cryptonote void get_transaction_hashes(std::vector& txs, bool include_unrelayed_txes = true) const; /** - * @brief get (size, fee, receive time) for all transaction in the pool + * @brief get (weight, fee, receive time) for all transaction in the pool * * @param txs return-by-reference that data * @param include_unrelayed_txes include unrelayed txes in the result @@ -324,14 +325,14 @@ namespace cryptonote * * @return true */ - bool get_relayable_transactions(std::list>& txs) const; + bool get_relayable_transactions(std::vector>& txs) const; /** * @brief tell the pool that certain transactions were just relayed * * @param txs the list of transactions (and their hashes) */ - void set_relayed(const std::list>& txs); + void set_relayed(const std::vector>& txs); /** * @brief get the total number of transactions in the pool @@ -362,22 +363,29 @@ namespace cryptonote */ size_t validate(uint8_t version); + /** + * @brief return the cookie + * + * @return the cookie + */ + uint64_t cookie() const { return m_cookie; } + /** - * @brief get the cumulative txpool size in bytes + * @brief get the cumulative txpool weight in bytes * - * @return the cumulative txpool size in bytes + * @return the cumulative txpool weight in bytes */ - size_t get_txpool_size() const; + size_t get_txpool_weight() const; /** - * @brief set the max cumulative txpool size in bytes + * @brief set the max cumulative txpool weight in bytes * - * @param bytes the max cumulative txpool size in bytes + * @param bytes the max cumulative txpool weight in bytes */ - void set_txpool_max_size(size_t bytes); + void set_txpool_max_weight(size_t bytes); #define CURRENT_MEMPOOL_ARCHIVE_VER 11 -#define CURRENT_MEMPOOL_TX_DETAILS_ARCHIVE_VER 12 +#define CURRENT_MEMPOOL_TX_DETAILS_ARCHIVE_VER 13 /** * @brief information about a single transaction @@ -386,6 +394,7 @@ namespace cryptonote { transaction tx; //!< the transaction size_t blob_size; //!< the transaction's size + size_t weight; //!< the transaction's weight uint64_t fee; //!< the transaction's fee amount crypto::hash max_used_block_id; //!< the hash of the highest block referenced by an input uint64_t max_used_block_height; //!< the height of the highest block referenced by an input @@ -426,7 +435,7 @@ namespace cryptonote * * @return true on success, false on error */ - bool insert_key_images(const transaction &tx, bool kept_by_block); + bool insert_key_images(const transaction_prefix &tx, const crypto::hash &txid, bool kept_by_block); /** * @brief remove old transactions from the pool @@ -470,10 +479,11 @@ namespace cryptonote * a transaction from the pool. * * @param tx the transaction + * @param txid the transaction's hash * * @return false if any key images to be removed cannot be found, otherwise true */ - bool remove_transaction_keyimages(const transaction& tx); + bool remove_transaction_keyimages(const transaction_prefix& tx, const crypto::hash &txid); /** * @brief check if any of a transaction's spent key images are present in a given set @@ -483,7 +493,7 @@ namespace cryptonote * * @return true if any key images present in the set, otherwise false */ - static bool have_key_images(const std::unordered_set& kic, const transaction& tx); + static bool have_key_images(const std::unordered_set& kic, const transaction_prefix& tx); /** * @brief append the key images from a transaction to the given set @@ -493,16 +503,19 @@ namespace cryptonote * * @return false if any append fails, otherwise true */ - static bool append_key_images(std::unordered_set& kic, const transaction& tx); + static bool append_key_images(std::unordered_set& kic, const transaction_prefix& tx); /** * @brief check if a transaction is a valid candidate for inclusion in a block * * @param txd the transaction to check (and info about it) + * @param txid the txid of the transaction to check + * @param txblob the transaction blob to check + * @param tx the parsed transaction, if successful * * @return true if the transaction is good to go, otherwise false */ - bool is_transaction_ready_to_go(txpool_tx_meta_t& txd, transaction &tx) const; + bool is_transaction_ready_to_go(txpool_tx_meta_t& txd, const crypto::hash &txid, const cryptonote::blobdata &txblob, transaction&tx) const; /** * @brief mark all transactions double spending the one passed @@ -512,7 +525,7 @@ namespace cryptonote /** * @brief prune lowest fee/byte txes till we're not above bytes * - * if bytes is 0, use m_txpool_max_size + * if bytes is 0, use m_txpool_max_weight */ void prune(size_t bytes = 0); @@ -546,6 +559,8 @@ namespace cryptonote //!< container for transactions organized by fee per size and receive time sorted_tx_container m_txs_by_fee_and_receive_time; + std::atomic m_cookie; //!< incremented at each change + /** * @brief get an iterator to a transaction in the sorted container * @@ -555,6 +570,9 @@ namespace cryptonote */ sorted_tx_container::iterator find_tx_in_sorted_container(const crypto::hash& id) const; + //! cache/call Blockchain::check_tx_inputs results + bool check_tx_inputs(const std::function &get_tx, const crypto::hash &txid, uint64_t &max_used_block_height, crypto::hash &max_used_block_id, tx_verification_context &tvc, bool kept_by_block = false) const; + //! transactions which are unlikely to be included in blocks /*! These transactions are kept in RAM in case they *are* included * in a block eventually, but this container is not saved to disk. @@ -563,8 +581,12 @@ namespace cryptonote Blockchain& m_blockchain; //!< reference to the Blockchain object - size_t m_txpool_max_size; - size_t m_txpool_size; + size_t m_txpool_max_weight; + size_t m_txpool_weight; + + mutable std::unordered_map> m_input_cache; + + std::unordered_map m_parsed_tx_cache; }; } @@ -591,6 +613,9 @@ namespace boost if (version < 12) return; ar & td.do_not_relay; + if (version < 13) + return; + ar & td.weight; } } } diff --git a/src/cryptonote_core/tx_sanity_check.cpp b/src/cryptonote_core/tx_sanity_check.cpp new file mode 100644 index 00000000000..36be91f2cbe --- /dev/null +++ b/src/cryptonote_core/tx_sanity_check.cpp @@ -0,0 +1,100 @@ +// Copyright (c) 2019, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include +#include +#include "cryptonote_basic/cryptonote_basic_impl.h" +#include "cryptonote_basic/cryptonote_format_utils.h" +#include "blockchain.h" +#include "tx_sanity_check.h" + +#undef MONERO_DEFAULT_LOG_CATEGORY +#define MONERO_DEFAULT_LOG_CATEGORY "verify" + +namespace cryptonote +{ + +bool tx_sanity_check(Blockchain &blockchain, const cryptonote::blobdata &tx_blob) +{ + cryptonote::transaction tx; + + if (!cryptonote::parse_and_validate_tx_from_blob(tx_blob, tx)) + { + MERROR("Failed to parse transaction"); + return false; + } + + if (cryptonote::is_coinbase(tx)) + { + MERROR("Transaction is coinbase"); + return false; + } + std::set rct_indices; + size_t n_indices = 0; + + for (const auto &txin : tx.vin) + { + if (txin.type() != typeid(cryptonote::txin_to_key)) + continue; + const cryptonote::txin_to_key &in_to_key = boost::get(txin); + if (in_to_key.amount != 0) + continue; + const std::vector absolute = cryptonote::relative_output_offsets_to_absolute(in_to_key.key_offsets); + for (uint64_t offset: absolute) + rct_indices.insert(offset); + n_indices += in_to_key.key_offsets.size(); + } + + if (n_indices <= 10) + { + MERROR("n_indices is only " << n_indices); + return true; + } + + uint64_t n_available = blockchain.get_num_mature_outputs(0); + if (n_available < 10000) + return true; + + if (rct_indices.size() < n_indices * 8 / 10) + { + MERROR("unique indices is only " << rct_indices.size() << "/" << n_indices); + return false; + } + + std::vector offsets(rct_indices.begin(), rct_indices.end()); + uint64_t median = epee::misc_utils::median(offsets); + if (median < n_available * 6 / 10) + { + MERROR("median is " << median << "/" << n_available); + return false; + } + + return true; +} + +} diff --git a/src/cryptonote_core/tx_sanity_check.h b/src/cryptonote_core/tx_sanity_check.h new file mode 100644 index 00000000000..c12d1b0b104 --- /dev/null +++ b/src/cryptonote_core/tx_sanity_check.h @@ -0,0 +1,36 @@ +// Copyright (c) 2019, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "cryptonote_basic/blobdatatype.h" + +namespace cryptonote +{ + class Blockchain; + + bool tx_sanity_check(Blockchain &blockchain, const cryptonote::blobdata &tx_blob); +} diff --git a/src/cryptonote_protocol/CMakeLists.txt b/src/cryptonote_protocol/CMakeLists.txt index 1189ccf2243..bfcf4276700 100644 --- a/src/cryptonote_protocol/CMakeLists.txt +++ b/src/cryptonote_protocol/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2014-2018, The Monero Project +# Copyright (c) 2014-2019, The Monero Project # # All rights reserved. # diff --git a/src/cryptonote_protocol/block_queue.cpp b/src/cryptonote_protocol/block_queue.cpp index 9ae33d5401c..b4f9daa74dc 100644 --- a/src/cryptonote_protocol/block_queue.cpp +++ b/src/cryptonote_protocol/block_queue.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2018, The Monero Project +// Copyright (c) 2017-2019, The Monero Project // // All rights reserved. // @@ -31,8 +31,10 @@ #include #include #include +#include #include "string_tools.h" #include "cryptonote_protocol_defs.h" +#include "common/pruning.h" #include "block_queue.h" #undef MONERO_DEFAULT_LOG_CATEGORY @@ -50,14 +52,21 @@ namespace std { namespace cryptonote { -void block_queue::add_blocks(uint64_t height, std::list bcel, const boost::uuids::uuid &connection_id, float rate, size_t size) +void block_queue::add_blocks(uint64_t height, std::vector bcel, const boost::uuids::uuid &connection_id, float rate, size_t size) { boost::unique_lock lock(mutex); - std::list hashes; + std::vector hashes; bool has_hashes = remove_span(height, &hashes); blocks.insert(span(height, std::move(bcel), connection_id, rate, size)); if (has_hashes) + { + for (const crypto::hash &h: hashes) + { + requested_hashes.insert(h); + have_blocks.insert(h); + } set_span_hashes(height, connection_id, hashes); + } } void block_queue::add_blocks(uint64_t height, uint64_t nblocks, const boost::uuids::uuid &connection_id, boost::posix_time::ptime time) @@ -76,28 +85,37 @@ void block_queue::flush_spans(const boost::uuids::uuid &connection_id, bool all) block_map::iterator j = i++; if (j->connection_id == connection_id && (all || j->blocks.size() == 0)) { - blocks.erase(j); + erase_block(j); } } } +void block_queue::erase_block(block_map::iterator j) +{ + CHECK_AND_ASSERT_THROW_MES(j != blocks.end(), "Invalid iterator"); + for (const crypto::hash &h: j->hashes) + { + requested_hashes.erase(h); + have_blocks.erase(h); + } + blocks.erase(j); +} + void block_queue::flush_stale_spans(const std::set &live_connections) { boost::unique_lock lock(mutex); block_map::iterator i = blocks.begin(); - if (i != blocks.end() && is_blockchain_placeholder(*i)) - ++i; while (i != blocks.end()) { block_map::iterator j = i++; - if (live_connections.find(j->connection_id) == live_connections.end() && j->blocks.size() == 0) + if (j->blocks.empty() && live_connections.find(j->connection_id) == live_connections.end()) { - blocks.erase(j); + erase_block(j); } } } -bool block_queue::remove_span(uint64_t start_block_height, std::list *hashes) +bool block_queue::remove_span(uint64_t start_block_height, std::vector *hashes) { boost::unique_lock lock(mutex); for (block_map::iterator i = blocks.begin(); i != blocks.end(); ++i) @@ -106,7 +124,7 @@ bool block_queue::remove_span(uint64_t start_block_height, std::listhashes); - blocks.erase(i); + erase_block(i); return true; } } @@ -121,7 +139,7 @@ void block_queue::remove_spans(const boost::uuids::uuid &connection_id, uint64_t block_map::iterator j = i++; if (j->connection_id == connection_id && j->start_block_height <= start_block_height) { - blocks.erase(j); + erase_block(j); } } } @@ -139,105 +157,154 @@ uint64_t block_queue::get_max_block_height() const return height; } +uint64_t block_queue::get_next_needed_height(uint64_t blockchain_height) const +{ + boost::unique_lock lock(mutex); + if (blocks.empty()) + return blockchain_height; + uint64_t last_needed_height = blockchain_height; + bool first = true; + for (const auto &span: blocks) + { + if (span.start_block_height + span.nblocks - 1 < blockchain_height) + continue; + if (span.start_block_height != last_needed_height || (first && span.blocks.empty())) + return last_needed_height; + last_needed_height = span.start_block_height + span.nblocks; + first = false; + } + return last_needed_height; +} + void block_queue::print() const { boost::unique_lock lock(mutex); MDEBUG("Block queue has " << blocks.size() << " spans"); for (const auto &span: blocks) - MDEBUG(" " << span.start_block_height << " - " << (span.start_block_height+span.nblocks-1) << " (" << span.nblocks << ") - " << (is_blockchain_placeholder(span) ? "blockchain" : span.blocks.empty() ? "scheduled" : "filled ") << " " << span.connection_id << " (" << ((unsigned)(span.rate*10/1024.f))/10.f << " kB/s)"); + MDEBUG(" " << span.start_block_height << " - " << (span.start_block_height+span.nblocks-1) << " (" << span.nblocks << ") - " << (span.blocks.empty() ? "scheduled" : "filled ") << " " << span.connection_id << " (" << ((unsigned)(span.rate*10/1024.f))/10.f << " kB/s)"); } -std::string block_queue::get_overview() const +std::string block_queue::get_overview(uint64_t blockchain_height) const { boost::unique_lock lock(mutex); if (blocks.empty()) return "[]"; block_map::const_iterator i = blocks.begin(); - std::string s = std::string("[") + std::to_string(i->start_block_height + i->nblocks - 1) + ":"; - while (++i != blocks.end()) - s += i->blocks.empty() ? "." : "o"; + std::string s = std::string("["); + uint64_t expected = blockchain_height; + while (i != blocks.end()) + { + if (expected > i->start_block_height) + { + s += "<"; + } + else + { + if (expected < i->start_block_height) + s += std::string(std::max((uint64_t)1, (i->start_block_height - expected) / (i->nblocks ? i->nblocks : 1)), '_'); + s += i->blocks.empty() ? "." : i->start_block_height == blockchain_height ? "m" : "o"; + expected = i->start_block_height + i->nblocks; + } + ++i; + } s += "]"; return s; } +inline bool block_queue::requested_internal(const crypto::hash &hash) const +{ + return requested_hashes.find(hash) != requested_hashes.end(); +} + bool block_queue::requested(const crypto::hash &hash) const { boost::unique_lock lock(mutex); - for (const auto &span: blocks) - { - for (const auto &h: span.hashes) - if (h == hash) - return true; - } - return false; + return requested_internal(hash); } -std::pair block_queue::reserve_span(uint64_t first_block_height, uint64_t last_block_height, uint64_t max_blocks, const boost::uuids::uuid &connection_id, const std::list &block_hashes, boost::posix_time::ptime time) +bool block_queue::have(const crypto::hash &hash) const +{ + boost::unique_lock lock(mutex); + return have_blocks.find(hash) != have_blocks.end(); +} + +std::pair block_queue::reserve_span(uint64_t first_block_height, uint64_t last_block_height, uint64_t max_blocks, const boost::uuids::uuid &connection_id, uint32_t pruning_seed, uint64_t blockchain_height, const std::vector &block_hashes, boost::posix_time::ptime time) { boost::unique_lock lock(mutex); + MDEBUG("reserve_span: first_block_height " << first_block_height << ", last_block_height " << last_block_height + << ", max " << max_blocks << ", seed " << epee::string_tools::to_string_hex(pruning_seed) << ", blockchain_height " << + blockchain_height << ", block hashes size " << block_hashes.size()); if (last_block_height < first_block_height || max_blocks == 0) { MDEBUG("reserve_span: early out: first_block_height " << first_block_height << ", last_block_height " << last_block_height << ", max_blocks " << max_blocks); return std::make_pair(0, 0); } + if (block_hashes.size() > last_block_height) + { + MDEBUG("reserve_span: more block hashes than fit within last_block_height: " << block_hashes.size() << " and " << last_block_height); + return std::make_pair(0, 0); + } + // skip everything we've already requested uint64_t span_start_height = last_block_height - block_hashes.size() + 1; - std::list::const_iterator i = block_hashes.begin(); - while (i != block_hashes.end() && requested(*i)) + std::vector::const_iterator i = block_hashes.begin(); + while (i != block_hashes.end() && requested_internal(*i)) { ++i; ++span_start_height; } + + // if the peer's pruned for the starting block and its unpruned stripe comes next, start downloading from there + const uint32_t next_unpruned_height = tools::get_next_unpruned_block_height(span_start_height, blockchain_height, pruning_seed); + MDEBUG("reserve_span: next_unpruned_height " << next_unpruned_height << " from " << span_start_height << " and seed " + << epee::string_tools::to_string_hex(pruning_seed) << ", limit " << span_start_height + CRYPTONOTE_PRUNING_STRIPE_SIZE); + if (next_unpruned_height > span_start_height && next_unpruned_height < span_start_height + CRYPTONOTE_PRUNING_STRIPE_SIZE) + { + MDEBUG("We can download from next span: ideal height " << span_start_height << ", next unpruned height " << next_unpruned_height << + "(+" << next_unpruned_height - span_start_height << "), current seed " << pruning_seed); + span_start_height = next_unpruned_height; + } + MDEBUG("span_start_height: " <= block_hashes.size() + block_hashes_start_height) + { + MDEBUG("Out of hashes, cannot reserve"); + return std::make_pair(0, 0); + } + + i = block_hashes.begin() + span_start_height - block_hashes_start_height; + while (i != block_hashes.end() && requested_internal(*i)) + { + ++i; + ++span_start_height; + } + uint64_t span_length = 0; - std::list hashes; - while (i != block_hashes.end() && span_length < max_blocks) + std::vector hashes; + while (i != block_hashes.end() && span_length < max_blocks && tools::has_unpruned_block(span_start_height + span_length, blockchain_height, pruning_seed)) { hashes.push_back(*i); ++i; ++span_length; } if (span_length == 0) + { + MDEBUG("span_length 0, cannot reserve"); return std::make_pair(0, 0); + } MDEBUG("Reserving span " << span_start_height << " - " << (span_start_height + span_length - 1) << " for " << connection_id); add_blocks(span_start_height, span_length, connection_id, time); set_span_hashes(span_start_height, connection_id, hashes); return std::make_pair(span_start_height, span_length); } -bool block_queue::is_blockchain_placeholder(const span &span) const -{ - return span.connection_id == boost::uuids::nil_uuid(); -} - -std::pair block_queue::get_start_gap_span() const -{ - boost::unique_lock lock(mutex); - if (blocks.empty()) - return std::make_pair(0, 0); - block_map::const_iterator i = blocks.begin(); - if (!is_blockchain_placeholder(*i)) - return std::make_pair(0, 0); - uint64_t current_height = i->start_block_height + i->nblocks - 1; - ++i; - if (i == blocks.end()) - return std::make_pair(0, 0); - uint64_t first_span_height = i->start_block_height; - if (first_span_height <= current_height + 1) - return std::make_pair(0, 0); - MDEBUG("Found gap at start of spans: last blockchain block height " << current_height << ", first span's block height " << first_span_height); - print(); - return std::make_pair(current_height + 1, first_span_height - current_height - 1); -} - -std::pair block_queue::get_next_span_if_scheduled(std::list &hashes, boost::uuids::uuid &connection_id, boost::posix_time::ptime &time) const +std::pair block_queue::get_next_span_if_scheduled(std::vector &hashes, boost::uuids::uuid &connection_id, boost::posix_time::ptime &time) const { boost::unique_lock lock(mutex); if (blocks.empty()) return std::make_pair(0, 0); block_map::const_iterator i = blocks.begin(); - if (is_blockchain_placeholder(*i)) - ++i; if (i == blocks.end()) return std::make_pair(0, 0); if (!i->blocks.empty()) @@ -248,7 +315,17 @@ std::pair block_queue::get_next_span_if_scheduled(std::list< return std::make_pair(i->start_block_height, i->nblocks); } -void block_queue::set_span_hashes(uint64_t start_height, const boost::uuids::uuid &connection_id, std::list hashes) +void block_queue::reset_next_span_time(boost::posix_time::ptime t) +{ + boost::unique_lock lock(mutex); + CHECK_AND_ASSERT_THROW_MES(!blocks.empty(), "No next span to reset time"); + block_map::iterator i = blocks.begin(); + CHECK_AND_ASSERT_THROW_MES(i != blocks.end(), "No next span to reset time"); + CHECK_AND_ASSERT_THROW_MES(i->blocks.empty(), "Next span is not empty"); + (boost::posix_time::ptime&)i->time = t; // sod off, time doesn't influence sorting +} + +void block_queue::set_span_hashes(uint64_t start_height, const boost::uuids::uuid &connection_id, std::vector hashes) { boost::unique_lock lock(mutex); for (block_map::iterator i = blocks.begin(); i != blocks.end(); ++i) @@ -256,22 +333,22 @@ void block_queue::set_span_hashes(uint64_t start_height, const boost::uuids::uui if (i->start_block_height == start_height && i->connection_id == connection_id) { span s = *i; - blocks.erase(i); + erase_block(i); s.hashes = std::move(hashes); + for (const crypto::hash &h: s.hashes) + requested_hashes.insert(h); blocks.insert(s); return; } } } -bool block_queue::get_next_span(uint64_t &height, std::list &bcel, boost::uuids::uuid &connection_id, bool filled) const +bool block_queue::get_next_span(uint64_t &height, std::vector &bcel, boost::uuids::uuid &connection_id, bool filled) const { boost::unique_lock lock(mutex); if (blocks.empty()) return false; block_map::const_iterator i = blocks.begin(); - if (is_blockchain_placeholder(*i)) - ++i; for (; i != blocks.end(); ++i) { if (!filled || !i->blocks.empty()) @@ -285,19 +362,34 @@ bool block_queue::get_next_span(uint64_t &height, std::list lock(mutex); if (blocks.empty()) return false; block_map::const_iterator i = blocks.begin(); - if (is_blockchain_placeholder(*i)) - ++i; if (i == blocks.end()) return false; if (i->connection_id != connection_id) return false; filled = !i->blocks.empty(); + time = i->time; + return true; +} + +bool block_queue::has_next_span(uint64_t height, bool &filled, boost::posix_time::ptime &time, boost::uuids::uuid &connection_id) const +{ + boost::unique_lock lock(mutex); + if (blocks.empty()) + return false; + block_map::const_iterator i = blocks.begin(); + if (i == blocks.end()) + return false; + if (i->start_block_height > height) + return false; + filled = !i->blocks.empty(); + time = i->time; + connection_id = i->connection_id; return true; } @@ -317,8 +409,6 @@ size_t block_queue::get_num_filled_spans_prefix() const if (blocks.empty()) return 0; block_map::const_iterator i = blocks.begin(); - if (is_blockchain_placeholder(*i)) - ++i; size_t size = 0; while (i != blocks.end() && !i->blocks.empty()) { @@ -403,12 +493,35 @@ float block_queue::get_speed(const boost::uuids::uuid &connection_id) const return speed; } -bool block_queue::foreach(std::function f, bool include_blockchain_placeholder) const +float block_queue::get_download_rate(const boost::uuids::uuid &connection_id) const +{ + boost::unique_lock lock(mutex); + float conn_rate = -1.f; + for (const auto &span: blocks) + { + if (span.blocks.empty()) + continue; + if (span.connection_id != connection_id) + continue; + // note that the average below does not average over the whole set, but over the + // previous pseudo average and the latest rate: this gives much more importance + // to the latest measurements, which is fine here + if (conn_rate < 0.f) + conn_rate = span.rate; + else + conn_rate = (conn_rate + span.rate) / 2; + } + + if (conn_rate < 0) + conn_rate = 0.0f; + MTRACE("Download rate for " << connection_id << ": " << conn_rate << " b/s"); + return conn_rate; +} + +bool block_queue::foreach(std::function f) const { boost::unique_lock lock(mutex); block_map::const_iterator i = blocks.begin(); - if (!include_blockchain_placeholder && i != blocks.end() && is_blockchain_placeholder(*i)) - ++i; while (i != blocks.end()) if (!f(*i++)) return false; diff --git a/src/cryptonote_protocol/block_queue.h b/src/cryptonote_protocol/block_queue.h index 69ddaa4358f..1bef01d671d 100644 --- a/src/cryptonote_protocol/block_queue.h +++ b/src/cryptonote_protocol/block_queue.h @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2018, The Monero Project +// Copyright (c) 2017-2019, The Monero Project // // All rights reserved. // @@ -31,8 +31,9 @@ #pragma once #include -#include +#include #include +#include #include #include @@ -49,15 +50,15 @@ namespace cryptonote struct span { uint64_t start_block_height; - std::list hashes; - std::list blocks; + std::vector hashes; + std::vector blocks; boost::uuids::uuid connection_id; uint64_t nblocks; float rate; size_t size; boost::posix_time::ptime time; - span(uint64_t start_block_height, std::list blocks, const boost::uuids::uuid &connection_id, float rate, size_t size): + span(uint64_t start_block_height, std::vector blocks, const boost::uuids::uuid &connection_id, float rate, size_t size): start_block_height(start_block_height), blocks(std::move(blocks)), connection_id(connection_id), nblocks(this->blocks.size()), rate(rate), size(size), time() {} span(uint64_t start_block_height, uint64_t nblocks, const boost::uuids::uuid &connection_id, boost::posix_time::ptime time): start_block_height(start_block_height), connection_id(connection_id), nblocks(nblocks), rate(0.0f), size(0), time(time) {} @@ -67,33 +68,43 @@ namespace cryptonote typedef std::set block_map; public: - void add_blocks(uint64_t height, std::list bcel, const boost::uuids::uuid &connection_id, float rate, size_t size); + void add_blocks(uint64_t height, std::vector bcel, const boost::uuids::uuid &connection_id, float rate, size_t size); void add_blocks(uint64_t height, uint64_t nblocks, const boost::uuids::uuid &connection_id, boost::posix_time::ptime time = boost::date_time::min_date_time); void flush_spans(const boost::uuids::uuid &connection_id, bool all = false); void flush_stale_spans(const std::set &live_connections); - bool remove_span(uint64_t start_block_height, std::list *hashes = NULL); + bool remove_span(uint64_t start_block_height, std::vector *hashes = NULL); void remove_spans(const boost::uuids::uuid &connection_id, uint64_t start_block_height); uint64_t get_max_block_height() const; void print() const; - std::string get_overview() const; - std::pair reserve_span(uint64_t first_block_height, uint64_t last_block_height, uint64_t max_blocks, const boost::uuids::uuid &connection_id, const std::list &block_hashes, boost::posix_time::ptime time = boost::posix_time::microsec_clock::universal_time()); - bool is_blockchain_placeholder(const span &span) const; - std::pair get_start_gap_span() const; - std::pair get_next_span_if_scheduled(std::list &hashes, boost::uuids::uuid &connection_id, boost::posix_time::ptime &time) const; - void set_span_hashes(uint64_t start_height, const boost::uuids::uuid &connection_id, std::list hashes); - bool get_next_span(uint64_t &height, std::list &bcel, boost::uuids::uuid &connection_id, bool filled = true) const; - bool has_next_span(const boost::uuids::uuid &connection_id, bool &filled) const; + std::string get_overview(uint64_t blockchain_height) const; + bool has_unpruned_height(uint64_t block_height, uint64_t blockchain_height, uint32_t pruning_seed) const; + std::pair reserve_span(uint64_t first_block_height, uint64_t last_block_height, uint64_t max_blocks, const boost::uuids::uuid &connection_id, uint32_t pruning_seed, uint64_t blockchain_height, const std::vector &block_hashes, boost::posix_time::ptime time = boost::posix_time::microsec_clock::universal_time()); + uint64_t get_next_needed_height(uint64_t blockchain_height) const; + std::pair get_next_span_if_scheduled(std::vector &hashes, boost::uuids::uuid &connection_id, boost::posix_time::ptime &time) const; + void reset_next_span_time(boost::posix_time::ptime t = boost::posix_time::microsec_clock::universal_time()); + void set_span_hashes(uint64_t start_height, const boost::uuids::uuid &connection_id, std::vector hashes); + bool get_next_span(uint64_t &height, std::vector &bcel, boost::uuids::uuid &connection_id, bool filled = true) const; + bool has_next_span(const boost::uuids::uuid &connection_id, bool &filled, boost::posix_time::ptime &time) const; + bool has_next_span(uint64_t height, bool &filled, boost::posix_time::ptime &time, boost::uuids::uuid &connection_id) const; size_t get_data_size() const; size_t get_num_filled_spans_prefix() const; size_t get_num_filled_spans() const; crypto::hash get_last_known_hash(const boost::uuids::uuid &connection_id) const; bool has_spans(const boost::uuids::uuid &connection_id) const; float get_speed(const boost::uuids::uuid &connection_id) const; - bool foreach(std::function f, bool include_blockchain_placeholder = false) const; + float get_download_rate(const boost::uuids::uuid &connection_id) const; + bool foreach(std::function f) const; bool requested(const crypto::hash &hash) const; + bool have(const crypto::hash &hash) const; + + private: + void erase_block(block_map::iterator j); + inline bool requested_internal(const crypto::hash &hash) const; private: block_map blocks; mutable boost::recursive_mutex mutex; + std::unordered_set requested_hashes; + std::unordered_set have_blocks; }; } diff --git a/src/cryptonote_protocol/cryptonote_protocol_defs.h b/src/cryptonote_protocol/cryptonote_protocol_defs.h index cf0043287c0..3083a5b4ceb 100644 --- a/src/cryptonote_protocol/cryptonote_protocol_defs.h +++ b/src/cryptonote_protocol/cryptonote_protocol_defs.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -34,6 +34,7 @@ #include "serialization/keyvalue_serialization.h" #include "cryptonote_basic/cryptonote_basic.h" #include "cryptonote_basic/blobdatatype.h" + namespace cryptonote { @@ -48,11 +49,13 @@ namespace cryptonote bool incoming; bool localhost; bool local_ip; + bool ssl; std::string address; std::string host; std::string ip; std::string port; + uint16_t rpc_port; std::string peer_id; @@ -78,6 +81,8 @@ namespace cryptonote uint64_t height; + uint32_t pruning_seed; + BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(incoming) KV_SERIALIZE(localhost) @@ -86,6 +91,7 @@ namespace cryptonote KV_SERIALIZE(host) KV_SERIALIZE(ip) KV_SERIALIZE(port) + KV_SERIALIZE(rpc_port) KV_SERIALIZE(peer_id) KV_SERIALIZE(recv_count) KV_SERIALIZE(recv_idle_time) @@ -100,6 +106,7 @@ namespace cryptonote KV_SERIALIZE(support_flags) KV_SERIALIZE(connection_id) KV_SERIALIZE(height) + KV_SERIALIZE(pruning_seed) END_KV_SERIALIZE_MAP() }; @@ -109,7 +116,7 @@ namespace cryptonote struct block_complete_entry { blobdata block; - std::list txs; + std::vector txs; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(block) KV_SERIALIZE(txs) @@ -124,7 +131,7 @@ namespace cryptonote { const static int ID = BC_COMMANDS_POOL_BASE + 1; - struct request + struct request_t { block_complete_entry b; uint64_t current_blockchain_height; @@ -134,6 +141,7 @@ namespace cryptonote KV_SERIALIZE(current_blockchain_height) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init request; }; /************************************************************************/ @@ -143,14 +151,17 @@ namespace cryptonote { const static int ID = BC_COMMANDS_POOL_BASE + 2; - struct request + struct request_t { - std::list txs; + std::vector txs; + std::string _; // padding BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(txs) + KV_SERIALIZE(_) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init request; }; /************************************************************************/ /* */ @@ -159,27 +170,28 @@ namespace cryptonote { const static int ID = BC_COMMANDS_POOL_BASE + 3; - struct request + struct request_t { - std::list txs; - std::list blocks; + std::vector txs; + std::vector blocks; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE_CONTAINER_POD_AS_BLOB(txs) KV_SERIALIZE_CONTAINER_POD_AS_BLOB(blocks) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init request; }; struct NOTIFY_RESPONSE_GET_OBJECTS { const static int ID = BC_COMMANDS_POOL_BASE + 4; - struct request + struct request_t { - std::list txs; - std::list blocks; - std::list missed_ids; + std::vector txs; + std::vector blocks; + std::vector missed_ids; uint64_t current_blockchain_height; BEGIN_KV_SERIALIZE_MAP() @@ -189,6 +201,7 @@ namespace cryptonote KV_SERIALIZE(current_blockchain_height) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init request; }; @@ -196,14 +209,18 @@ namespace cryptonote { uint64_t current_height; uint64_t cumulative_difficulty; + uint64_t cumulative_difficulty_top64; crypto::hash top_id; uint8_t top_version; + uint32_t pruning_seed; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(current_height) KV_SERIALIZE(cumulative_difficulty) + KV_SERIALIZE(cumulative_difficulty_top64) KV_SERIALIZE_VAL_POD_AS_BLOB(top_id) KV_SERIALIZE_OPT(top_version, (uint8_t)0) + KV_SERIALIZE_OPT(pruning_seed, (uint32_t)0) END_KV_SERIALIZE_MAP() }; @@ -211,7 +228,7 @@ namespace cryptonote { const static int ID = BC_COMMANDS_POOL_BASE + 6; - struct request + struct request_t { std::list block_ids; /*IDs of the first 10 blocks are sequential, next goes with pow(2,n) offset, like 2, 4, 8, 16, 32, 64 and so on, and the last one is always genesis block */ @@ -219,26 +236,30 @@ namespace cryptonote KV_SERIALIZE_CONTAINER_POD_AS_BLOB(block_ids) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init request; }; struct NOTIFY_RESPONSE_CHAIN_ENTRY { const static int ID = BC_COMMANDS_POOL_BASE + 7; - struct request + struct request_t { uint64_t start_height; uint64_t total_height; uint64_t cumulative_difficulty; - std::list m_block_ids; + uint64_t cumulative_difficulty_top64; + std::vector m_block_ids; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(start_height) KV_SERIALIZE(total_height) KV_SERIALIZE(cumulative_difficulty) + KV_SERIALIZE(cumulative_difficulty_top64) KV_SERIALIZE_CONTAINER_POD_AS_BLOB(m_block_ids) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init request; }; /************************************************************************/ @@ -248,7 +269,7 @@ namespace cryptonote { const static int ID = BC_COMMANDS_POOL_BASE + 8; - struct request + struct request_t { block_complete_entry b; uint64_t current_blockchain_height; @@ -258,6 +279,7 @@ namespace cryptonote KV_SERIALIZE(current_blockchain_height) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init request; }; /************************************************************************/ @@ -267,7 +289,7 @@ namespace cryptonote { const static int ID = BC_COMMANDS_POOL_BASE + 9; - struct request + struct request_t { crypto::hash block_hash; uint64_t current_blockchain_height; @@ -279,6 +301,7 @@ namespace cryptonote KV_SERIALIZE_CONTAINER_POD_AS_BLOB(missing_tx_indices) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init request; }; } diff --git a/src/cryptonote_protocol/cryptonote_protocol_handler-base.cpp b/src/cryptonote_protocol/cryptonote_protocol_handler-base.cpp index c9fd40d88f8..225418980ed 100644 --- a/src/cryptonote_protocol/cryptonote_protocol_handler-base.cpp +++ b/src/cryptonote_protocol/cryptonote_protocol_handler-base.cpp @@ -2,7 +2,7 @@ /// @author rfree (current maintainer in monero.cc project) /// @brief This is the place to implement our handlers for protocol network actions, e.g. for ratelimit for download-requests -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -30,20 +30,8 @@ // STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF // THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -#include #include #include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include #include @@ -51,24 +39,14 @@ #include "net/net_utils_base.h" #include "misc_log_ex.h" -#include -#include -#include #include -#include -#include #include #include #include "misc_language.h" #include "pragma_comp_defs.h" -#include -#include #include -#include -#include - #include "cryptonote_protocol_handler.h" #include "net/network_throttle.hpp" diff --git a/src/cryptonote_protocol/cryptonote_protocol_handler.h b/src/cryptonote_protocol/cryptonote_protocol_handler.h index d7b74c06d6e..dcc5ec6edd6 100644 --- a/src/cryptonote_protocol/cryptonote_protocol_handler.h +++ b/src/cryptonote_protocol/cryptonote_protocol_handler.h @@ -2,7 +2,7 @@ /// @author rfree (current maintainer/user in monero.cc project - most of code is from CryptoNote) /// @brief This is the original cryptonote protocol network-events handler, modified by us -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -43,6 +43,7 @@ #include "cryptonote_protocol_defs.h" #include "cryptonote_protocol_handler_common.h" #include "block_queue.h" +#include "common/perf_timer.h" #include "cryptonote_basic/connection_context.h" #include "cryptonote_basic/cryptonote_stat_info.h" #include @@ -51,6 +52,7 @@ PUSH_WARNINGS DISABLE_VS_WARNINGS(4355) #define LOCALHOST_INT 2130706433 +#define CURRENCY_PROTOCOL_MAX_OBJECT_REQUEST_COUNT 500 namespace cryptonote { @@ -109,6 +111,11 @@ namespace cryptonote const block_queue &get_block_queue() const { return m_block_queue; } void stop(); void on_connection_close(cryptonote_connection_context &context); + void set_max_out_peers(unsigned int max) { m_max_out_peers = max; } + void set_no_sync(bool value) { m_no_sync = value; } + std::string get_peers_overview() const; + std::pair get_next_needed_pruning_stripe() const; + bool needs_new_sync_connections() const; private: //----------------- commands handlers ---------------------------------------------- int handle_notify_new_block(int command, NOTIFY_NEW_BLOCK::request& arg, cryptonote_connection_context& context); @@ -125,13 +132,18 @@ namespace cryptonote virtual bool relay_transactions(NOTIFY_NEW_TRANSACTIONS::request& arg, cryptonote_connection_context& exclude_context); //---------------------------------------------------------------------------------- //bool get_payload_sync_data(HANDSHAKE_DATA::request& hshd, cryptonote_connection_context& context); + bool should_drop_connection(cryptonote_connection_context& context, uint32_t next_stripe); bool request_missing_objects(cryptonote_connection_context& context, bool check_having_blocks, bool force_next_span = false); size_t get_synchronizing_connections_count(); bool on_connection_synchronized(); - bool should_download_next_span(cryptonote_connection_context& context) const; + bool should_download_next_span(cryptonote_connection_context& context, bool standby); void drop_connection(cryptonote_connection_context &context, bool add_fail, bool flush_all_spans); bool kick_idle_peers(); + bool check_standby_peers(); + bool update_sync_search(); int try_add_next_blocks(cryptonote_connection_context &context); + void notify_new_stripe(cryptonote_connection_context &context, uint32_t stripe); + void skip_unneeded_hashes(cryptonote_connection_context& context, bool check_block_queue) const; t_core& m_core; @@ -140,9 +152,18 @@ namespace cryptonote std::atomic m_syncronized_connections_count; std::atomic m_synchronized; std::atomic m_stopping; + std::atomic m_no_sync; boost::mutex m_sync_lock; block_queue m_block_queue; epee::math_helper::once_a_time_seconds<30> m_idle_peer_kicker; + epee::math_helper::once_a_time_milliseconds<100> m_standby_checker; + epee::math_helper::once_a_time_seconds<101> m_sync_search_checker; + std::atomic m_max_out_peers; + tools::PerformanceTimer m_sync_timer, m_add_timer; + uint64_t m_last_add_end_time; + uint64_t m_sync_spans_downloaded, m_sync_old_spans_downloaded, m_sync_bad_spans_downloaded; + uint64_t m_sync_download_chain_size, m_sync_download_objects_size; + size_t m_block_download_max_size; boost::mutex m_buffer_mutex; double get_avg_block_size(); @@ -155,16 +176,7 @@ namespace cryptonote std::string blob; epee::serialization::store_t_to_binary(arg, blob); //handler_response_blocks_now(blob.size()); // XXX - return m_p2p->invoke_notify_to_peer(t_parameter::ID, blob, context); - } - - template - bool relay_post_notify(typename t_parameter::request& arg, cryptonote_connection_context& exclude_context) - { - LOG_PRINT_L2("[" << epee::net_utils::print_connection_context_short(exclude_context) << "] post relay " << typeid(t_parameter).name() << " -->"); - std::string arg_buff; - epee::serialization::store_t_to_binary(arg, arg_buff); - return m_p2p->relay_notify_to_all(t_parameter::ID, arg_buff, exclude_context); + return m_p2p->invoke_notify_to_peer(t_parameter::ID, epee::strspan(blob), context); } }; diff --git a/src/cryptonote_protocol/cryptonote_protocol_handler.inl b/src/cryptonote_protocol/cryptonote_protocol_handler.inl index 2e1df8078a3..bbf2020d4d7 100644 --- a/src/cryptonote_protocol/cryptonote_protocol_handler.inl +++ b/src/cryptonote_protocol/cryptonote_protocol_handler.inl @@ -2,7 +2,7 @@ /// @author rfree (current maintainer/user in monero.cc project - most of code is from CryptoNote) /// @brief This is the original cryptonote protocol network-events handler, modified by us -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -42,17 +42,35 @@ #include "cryptonote_basic/cryptonote_format_utils.h" #include "profile_tools.h" #include "net/network_throttle-detail.hpp" +#include "common/pruning.h" #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "net.cn" #define MLOG_P2P_MESSAGE(x) MCINFO("net.p2p.msg", context << x) - -#define BLOCK_QUEUE_NBLOCKS_THRESHOLD 10 // chunks of N blocks +#define MLOGIF_P2P_MESSAGE(init, test, x) \ + do { \ + const auto level = el::Level::Info; \ + const char *cat = "net.p2p.msg"; \ + if (ELPP->vRegistry()->allowed(level, cat)) { \ + init; \ + if (test) \ + el::base::Writer(level, __FILE__, __LINE__, ELPP_FUNC, el::base::DispatchAction::NormalLog).construct(cat) << x; \ + } \ + } while(0) + +#define MLOG_PEER_STATE(x) \ + MCINFO(MONERO_DEFAULT_LOG_CATEGORY, context << "[" << epee::string_tools::to_string_hex(context.m_pruning_seed) << "] state: " << x << " in state " << cryptonote::get_protocol_state_string(context.m_state)) + +#define BLOCK_QUEUE_NSPANS_THRESHOLD 10 // chunks of N blocks #define BLOCK_QUEUE_SIZE_THRESHOLD (100*1024*1024) // MB -#define REQUEST_NEXT_SCHEDULED_SPAN_THRESHOLD (5 * 1000000) // microseconds +#define BLOCK_QUEUE_FORCE_DOWNLOAD_NEAR_BLOCKS 1000 +#define REQUEST_NEXT_SCHEDULED_SPAN_THRESHOLD_STANDBY (5 * 1000000) // microseconds +#define REQUEST_NEXT_SCHEDULED_SPAN_THRESHOLD (30 * 1000000) // microseconds #define IDLE_PEER_KICK_TIME (600 * 1000000) // microseconds #define PASSIVE_PEER_KICK_TIME (60 * 1000000) // microseconds +#define DROP_ON_SYNC_WEDGE_THRESHOLD (30 * 1000000000ull) // nanoseconds +#define LAST_ACTIVITY_STALL_THRESHOLD (2.0f) // seconds namespace cryptonote { @@ -65,7 +83,8 @@ namespace cryptonote m_p2p(p_net_layout), m_syncronized_connections_count(0), m_synchronized(offline), - m_stopping(false) + m_stopping(false), + m_no_sync(false) { if(!m_p2p) @@ -75,6 +94,19 @@ namespace cryptonote template bool t_cryptonote_protocol_handler::init(const boost::program_options::variables_map& vm) { + m_sync_timer.pause(); + m_sync_timer.reset(); + m_add_timer.pause(); + m_add_timer.reset(); + m_last_add_end_time = 0; + m_sync_spans_downloaded = 0; + m_sync_old_spans_downloaded = 0; + m_sync_bad_spans_downloaded = 0; + m_sync_download_chain_size = 0; + m_sync_download_objects_size = 0; + + m_block_download_max_size = command_line::get_arg(vm, cryptonote::arg_block_download_max_size); + return true; } //------------------------------------------------------------------------------------------------------------------------ @@ -103,9 +135,12 @@ namespace cryptonote if(context.m_state == cryptonote_connection_context::state_synchronizing) { NOTIFY_REQUEST_CHAIN::request r = boost::value_initialized(); + context.m_needed_objects.clear(); m_core.get_short_chain_history(r.block_ids); - LOG_PRINT_CCONTEXT_L2("-->>NOTIFY_REQUEST_CHAIN: m_block_ids.size()=" << r.block_ids.size() ); + handler_request_blocks_history( r.block_ids ); // change the limit(?), sleep(?) + MLOG_P2P_MESSAGE("-->>NOTIFY_REQUEST_CHAIN: m_block_ids.size()=" << r.block_ids.size() ); post_notify(r, context); + MLOG_PEER_STATE("requesting chain"); } else if(context.m_state == cryptonote_connection_context::state_standby) { @@ -203,11 +238,12 @@ namespace cryptonote cnx.host = cntxt.m_remote_address.host_str(); cnx.ip = ""; cnx.port = ""; - if (cntxt.m_remote_address.get_type_id() == epee::net_utils::ipv4_network_address::ID) + if (cntxt.m_remote_address.get_type_id() == epee::net_utils::ipv4_network_address::get_type_id()) { cnx.ip = cnx.host; cnx.port = std::to_string(cntxt.m_remote_address.as().port()); } + cnx.rpc_port = cntxt.m_rpc_port; std::stringstream peer_id_str; peer_id_str << std::hex << std::setw(16) << peer_id; @@ -245,8 +281,10 @@ namespace cryptonote cnx.current_upload = cntxt.m_current_speed_up / 1024; cnx.connection_id = epee::string_tools::pod_to_hex(cntxt.m_connection_id); + cnx.ssl = cntxt.m_ssl; cnx.height = cntxt.m_remote_blockchain_height; + cnx.pruning_seed = cntxt.m_pruning_seed; connections.push_back(cnx); @@ -279,7 +317,23 @@ namespace cryptonote } } + // reject weird pruning schemes + if (hshd.pruning_seed) + { + const uint32_t log_stripes = tools::get_pruning_log_stripes(hshd.pruning_seed); + if (log_stripes != CRYPTONOTE_PRUNING_LOG_STRIPES || tools::get_pruning_stripe(hshd.pruning_seed) > (1u << log_stripes)) + { + MWARNING(context << " peer claim unexpected pruning seed " << epee::string_tools::to_string_hex(hshd.pruning_seed) << ", disconnecting"); + return false; + } + } + context.m_remote_blockchain_height = hshd.current_height; + context.m_pruning_seed = hshd.pruning_seed; +#ifdef CRYPTONOTE_PRUNING_DEBUG_SPOOF_SEED + context.m_pruning_seed = tools::make_pruning_seed(1 + (context.m_remote_address.as().ip()) % (1 << CRYPTONOTE_PRUNING_LOG_STRIPES), CRYPTONOTE_PRUNING_LOG_STRIPES); + LOG_INFO_CC(context, "New connection posing as pruning seed " << epee::string_tools::to_string_hex(context.m_pruning_seed) << ", seed address " << &context.m_pruning_seed); +#endif uint64_t target = m_core.get_target_blockchain_height(); if (target == 0) @@ -287,18 +341,29 @@ namespace cryptonote if(m_core.have_block(hshd.top_id)) { + if (target > m_core.get_current_blockchain_height()) + { + MINFO(context << "peer is not ahead of us and we're syncing, disconnecting"); + return false; + } context.m_state = cryptonote_connection_context::state_normal; if(is_inital && target == m_core.get_current_blockchain_height()) on_connection_synchronized(); return true; } + // No chain synchronization over hidden networks (tor, i2p, etc.) + if(context.m_remote_address.get_zone() != epee::net_utils::zone::public_) + { + context.m_state = cryptonote_connection_context::state_normal; + return true; + } + if (hshd.current_height > target) { /* As I don't know if accessing hshd from core could be a good practice, I prefer pushing target height to the core at the same time it is pushed to the user. Nz. */ - m_core.set_target_blockchain_height((hshd.current_height)); int64_t diff = static_cast(hshd.current_height) - static_cast(m_core.get_current_blockchain_height()); uint64_t abs_diff = std::abs(diff); uint64_t max_block_height = std::max(hshd.current_height,m_core.get_current_blockchain_height()); @@ -309,14 +374,38 @@ namespace cryptonote << (0 <= diff ? std::string("behind") : std::string("ahead")) << "] " << ENDL << "SYNCHRONIZATION started"); if (hshd.current_height >= m_core.get_current_blockchain_height() + 5) // don't switch to unsafe mode just for a few blocks + { m_core.safesyncmode(false); + } + if (m_core.get_target_blockchain_height() == 0) // only when sync starts + { + m_sync_timer.resume(); + m_sync_timer.reset(); + m_add_timer.pause(); + m_add_timer.reset(); + m_last_add_end_time = 0; + m_sync_spans_downloaded = 0; + m_sync_old_spans_downloaded = 0; + m_sync_bad_spans_downloaded = 0; + m_sync_download_chain_size = 0; + m_sync_download_objects_size = 0; + } + m_core.set_target_blockchain_height((hshd.current_height)); } - LOG_PRINT_L1("Remote blockchain height: " << hshd.current_height << ", id: " << hshd.top_id); + MINFO(context << "Remote blockchain height: " << hshd.current_height << ", id: " << hshd.top_id); + + if (m_no_sync) + { + context.m_state = cryptonote_connection_context::state_normal; + return true; + } + context.m_state = cryptonote_connection_context::state_synchronizing; //let the socket to send response to handshake, but request callback, to let send request data after response LOG_PRINT_CCONTEXT_L2("requesting callback"); ++context.m_callback_request_count; m_p2p->request_callback(context); + MLOG_PEER_STATE("requesting callback"); return true; } //------------------------------------------------------------------------------------------------------------------------ @@ -325,8 +414,11 @@ namespace cryptonote { m_core.get_blockchain_top(hshd.current_height, hshd.top_id); hshd.top_version = m_core.get_ideal_hard_fork_version(hshd.current_height); - hshd.cumulative_difficulty = m_core.get_block_cumulative_difficulty(hshd.current_height); + difficulty_type wide_cumulative_difficulty = m_core.get_block_cumulative_difficulty(hshd.current_height); + hshd.cumulative_difficulty = (wide_cumulative_difficulty & 0xffffffffffffffff).convert_to(); + hshd.cumulative_difficulty_top64 = ((wide_cumulative_difficulty >> 64) & 0xffffffffffffffff).convert_to(); hshd.current_height +=1; + hshd.pruning_seed = m_core.get_blockchain_pruning_seed(); return true; } //------------------------------------------------------------------------------------------------------------------------ @@ -342,7 +434,7 @@ namespace cryptonote template int t_cryptonote_protocol_handler::handle_notify_new_block(int command, NOTIFY_NEW_BLOCK::request& arg, cryptonote_connection_context& context) { - MLOG_P2P_MESSAGE("Received NOTIFY_NEW_BLOCK (" << arg.b.txs.size() << " txes)"); + MLOGIF_P2P_MESSAGE(crypto::hash hash; cryptonote::block b; bool ret = cryptonote::parse_and_validate_block_from_blob(arg.b.block, b, &hash);, ret, "Received NOTIFY_NEW_BLOCK " << hash << " (height " << arg.current_blockchain_height << ", " << arg.b.txs.size() << " txes)"); if(context.m_state != cryptonote_connection_context::state_normal) return 1; if(!is_synchronized()) // can happen if a peer connection goes to normal but another thread still hasn't finished adding queued blocks @@ -351,9 +443,16 @@ namespace cryptonote return 1; } m_core.pause_mine(); - std::list blocks; + std::vector blocks; blocks.push_back(arg.b); - m_core.prepare_handle_incoming_blocks(blocks); + std::vector pblocks; + if (!m_core.prepare_handle_incoming_blocks(blocks, pblocks)) + { + LOG_PRINT_CCONTEXT_L1("Block verification failed: prepare_handle_incoming_blocks failed, dropping connection"); + drop_connection(context, false, false); + m_core.resume_mine(); + return 1; + } for(auto tx_blob_it = arg.b.txs.begin(); tx_blob_it!=arg.b.txs.end();tx_blob_it++) { cryptonote::tx_verification_context tvc = AUTO_VAL_INIT(tvc); @@ -369,7 +468,7 @@ namespace cryptonote } block_verification_context bvc = boost::value_initialized(); - m_core.handle_incoming_block(arg.b.block, bvc); // got block from handle_notify_new_block + m_core.handle_incoming_block(arg.b.block, pblocks.empty() ? NULL : &pblocks[0], bvc); // got block from handle_notify_new_block if (!m_core.cleanup_handle_incoming_blocks(true)) { LOG_PRINT_CCONTEXT_L0("Failure in cleanup_handle_incoming_blocks"); @@ -389,11 +488,14 @@ namespace cryptonote relay_block(arg, context); }else if(bvc.m_marked_as_orphaned) { + context.m_needed_objects.clear(); context.m_state = cryptonote_connection_context::state_synchronizing; NOTIFY_REQUEST_CHAIN::request r = boost::value_initialized(); m_core.get_short_chain_history(r.block_ids); - LOG_PRINT_CCONTEXT_L2("-->>NOTIFY_REQUEST_CHAIN: m_block_ids.size()=" << r.block_ids.size() ); + handler_request_blocks_history( r.block_ids ); // change the limit(?), sleep(?) + MLOG_P2P_MESSAGE("-->>NOTIFY_REQUEST_CHAIN: m_block_ids.size()=" << r.block_ids.size() ); post_notify(r, context); + MLOG_PEER_STATE("requesting chain"); } return 1; @@ -402,7 +504,7 @@ namespace cryptonote template int t_cryptonote_protocol_handler::handle_notify_new_fluffy_block(int command, NOTIFY_NEW_FLUFFY_BLOCK::request& arg, cryptonote_connection_context& context) { - MLOG_P2P_MESSAGE("Received NOTIFY_NEW_FLUFFY_BLOCK (height " << arg.current_blockchain_height << ", " << arg.b.txs.size() << " txes)"); + MLOGIF_P2P_MESSAGE(crypto::hash hash; cryptonote::block b; bool ret = cryptonote::parse_and_validate_block_from_blob(arg.b.block, b, &hash);, ret, "Received NOTIFY_NEW_FLUFFY_BLOCK " << hash << " (height " << arg.current_blockchain_height << ", " << arg.b.txs.size() << " txes)"); if(context.m_state != cryptonote_connection_context::state_normal) return 1; if(!is_synchronized()) // can happen if a peer connection goes to normal but another thread still hasn't finished adding queued blocks @@ -438,7 +540,7 @@ namespace cryptonote } } - std::list have_tx; + std::vector have_tx; // Instead of requesting missing transactions by hash like BTC, // we do it by index (thanks to a suggestion from moneromooo) because @@ -578,8 +680,8 @@ namespace cryptonote else { std::vector tx_ids; - std::list txes; - std::list missing; + std::vector txes; + std::vector missing; tx_ids.push_back(tx_hash); if (m_core.get_transactions(tx_ids, txes, missing) && missing.empty()) { @@ -616,6 +718,7 @@ namespace cryptonote missing_tx_req.missing_tx_indices = std::move(need_tx_indices); m_core.resume_mine(); + MLOG_P2P_MESSAGE("-->>NOTIFY_REQUEST_FLUFFY_MISSING_TX: missing_tx_indices.size()=" << missing_tx_req.missing_tx_indices.size() ); post_notify(missing_tx_req, context); } else // whoo-hoo we've got em all .. @@ -626,12 +729,18 @@ namespace cryptonote b.block = arg.b.block; b.txs = have_tx; - std::list blocks; + std::vector blocks; blocks.push_back(b); - m_core.prepare_handle_incoming_blocks(blocks); + std::vector pblocks; + if (!m_core.prepare_handle_incoming_blocks(blocks, pblocks)) + { + LOG_PRINT_CCONTEXT_L0("Failure in prepare_handle_incoming_blocks"); + m_core.resume_mine(); + return 1; + } block_verification_context bvc = boost::value_initialized(); - m_core.handle_incoming_block(arg.b.block, bvc); // got block from handle_notify_new_block + m_core.handle_incoming_block(arg.b.block, pblocks.empty() ? NULL : &pblocks[0], bvc); // got block from handle_notify_new_block if (!m_core.cleanup_handle_incoming_blocks(true)) { LOG_PRINT_CCONTEXT_L0("Failure in cleanup_handle_incoming_blocks"); @@ -656,11 +765,14 @@ namespace cryptonote } else if( bvc.m_marked_as_orphaned ) { + context.m_needed_objects.clear(); context.m_state = cryptonote_connection_context::state_synchronizing; NOTIFY_REQUEST_CHAIN::request r = boost::value_initialized(); m_core.get_short_chain_history(r.block_ids); - LOG_PRINT_CCONTEXT_L2("-->>NOTIFY_REQUEST_CHAIN: m_block_ids.size()=" << r.block_ids.size() ); + handler_request_blocks_history( r.block_ids ); // change the limit(?), sleep(?) + MLOG_P2P_MESSAGE("-->>NOTIFY_REQUEST_CHAIN: m_block_ids.size()=" << r.block_ids.size() ); post_notify(r, context); + MLOG_PEER_STATE("requesting chain"); } } } @@ -687,8 +799,8 @@ namespace cryptonote { MLOG_P2P_MESSAGE("Received NOTIFY_REQUEST_FLUFFY_MISSING_TX (" << arg.missing_tx_indices.size() << " txes), block hash " << arg.block_hash); - std::list> local_blocks; - std::list local_txs; + std::vector> local_blocks; + std::vector local_txs; block b; if (!m_core.get_block_by_hash(arg.block_hash, b)) @@ -702,12 +814,27 @@ namespace cryptonote NOTIFY_NEW_FLUFFY_BLOCK::request fluffy_response; fluffy_response.b.block = t_serializable_object_to_blob(b); fluffy_response.current_blockchain_height = arg.current_blockchain_height; + std::vector seen(b.tx_hashes.size(), false); for(auto& tx_idx: arg.missing_tx_indices) { if(tx_idx < b.tx_hashes.size()) { MDEBUG(" tx " << b.tx_hashes[tx_idx]); + if (seen[tx_idx]) + { + LOG_ERROR_CCONTEXT + ( + "Failed to handle request NOTIFY_REQUEST_FLUFFY_MISSING_TX" + << ", request is asking for duplicate tx " + << ", tx index = " << tx_idx << ", block tx count " << b.tx_hashes.size() + << ", block_height = " << arg.current_blockchain_height + << ", dropping connection" + ); + drop_connection(context, true, false); + return 1; + } txids.push_back(b.tx_hashes[tx_idx]); + seen[tx_idx] = true; } else { @@ -725,8 +852,8 @@ namespace cryptonote } } - std::list txs; - std::list missed; + std::vector txs; + std::vector missed; if (!m_core.get_transactions(txids, txs, missed)) { LOG_ERROR_CCONTEXT("Failed to handle request NOTIFY_REQUEST_FLUFFY_MISSING_TX, " @@ -747,7 +874,7 @@ namespace cryptonote fluffy_response.b.txs.push_back(t_serializable_object_to_blob(tx)); } - LOG_PRINT_CCONTEXT_L2 + MLOG_P2P_MESSAGE ( "-->>NOTIFY_RESPONSE_FLUFFY_MISSING_TX: " << ", txs.size()=" << fluffy_response.b.txs.size() @@ -762,6 +889,9 @@ namespace cryptonote int t_cryptonote_protocol_handler::handle_notify_new_transactions(int command, NOTIFY_NEW_TRANSACTIONS::request& arg, cryptonote_connection_context& context) { MLOG_P2P_MESSAGE("Received NOTIFY_NEW_TRANSACTIONS (" << arg.txs.size() << " txes)"); + for (const auto &blob: arg.txs) + MLOGIF_P2P_MESSAGE(cryptonote::transaction tx; crypto::hash hash; bool ret = cryptonote::parse_and_validate_tx_from_blob(blob, tx, hash);, ret, "Including transaction " << hash); + if(context.m_state != cryptonote_connection_context::state_normal) return 1; @@ -774,10 +904,12 @@ namespace cryptonote return 1; } - for(auto tx_blob_it = arg.txs.begin(); tx_blob_it!=arg.txs.end();) + std::vector newtxs; + newtxs.reserve(arg.txs.size()); + for (size_t i = 0; i < arg.txs.size(); ++i) { cryptonote::tx_verification_context tvc = AUTO_VAL_INIT(tvc); - m_core.handle_incoming_tx(*tx_blob_it, tvc, false, true, false); + m_core.handle_incoming_tx(arg.txs[i], tvc, false, true, false); if(tvc.m_verifivation_failed) { LOG_PRINT_CCONTEXT_L1("Tx verification failed, dropping connection"); @@ -785,10 +917,9 @@ namespace cryptonote return 1; } if(tvc.m_should_be_relayed) - ++tx_blob_it; - else - arg.txs.erase(tx_blob_it++); + newtxs.push_back(std::move(arg.txs[i])); } + arg.txs = std::move(newtxs); if(arg.txs.size()) { @@ -803,6 +934,17 @@ namespace cryptonote int t_cryptonote_protocol_handler::handle_request_get_objects(int command, NOTIFY_REQUEST_GET_OBJECTS::request& arg, cryptonote_connection_context& context) { MLOG_P2P_MESSAGE("Received NOTIFY_REQUEST_GET_OBJECTS (" << arg.blocks.size() << " blocks, " << arg.txs.size() << " txes)"); + + if (arg.blocks.size() + arg.txs.size() > CURRENCY_PROTOCOL_MAX_OBJECT_REQUEST_COUNT) + { + LOG_ERROR_CCONTEXT( + "Requested objects count is too big (" + << arg.blocks.size() + arg.txs.size() << ") expected not more then " + << CURRENCY_PROTOCOL_MAX_OBJECT_REQUEST_COUNT); + drop_connection(context, false, false); + return 1; + } + NOTIFY_RESPONSE_GET_OBJECTS::request rsp; if(!m_core.handle_get_objects(arg, rsp, context)) { @@ -810,7 +952,7 @@ namespace cryptonote drop_connection(context, false, false); return 1; } - LOG_PRINT_CCONTEXT_L2("-->>NOTIFY_RESPONSE_GET_OBJECTS: blocks.size()=" << rsp.blocks.size() << ", txs.size()=" << rsp.txs.size() + MLOG_P2P_MESSAGE("-->>NOTIFY_RESPONSE_GET_OBJECTS: blocks.size()=" << rsp.blocks.size() << ", txs.size()=" << rsp.txs.size() << ", rsp.m_current_blockchain_height=" << rsp.current_blockchain_height << ", missed_ids.size()=" << rsp.missed_ids.size()); post_notify(rsp, context); //handler_response_blocks_now(sizeof(rsp)); // XXX @@ -833,11 +975,14 @@ namespace cryptonote return avg / m_avg_buffer.size(); } - template int t_cryptonote_protocol_handler::handle_response_get_objects(int command, NOTIFY_RESPONSE_GET_OBJECTS::request& arg, cryptonote_connection_context& context) { MLOG_P2P_MESSAGE("Received NOTIFY_RESPONSE_GET_OBJECTS (" << arg.blocks.size() << " blocks, " << arg.txs.size() << " txes)"); + MLOG_PEER_STATE("received objects"); + + boost::posix_time::ptime request_time = context.m_last_request_time; + context.m_last_request_time = boost::date_time::not_a_date_time; // calculate size of request size_t size = 0; @@ -859,6 +1004,8 @@ namespace cryptonote CRITICAL_REGION_LOCAL(m_buffer_mutex); m_avg_buffer.push_back(size); } + ++m_sync_spans_downloaded; + m_sync_download_objects_size += size; MDEBUG(context << " downloaded " << size << " bytes worth of blocks"); /*using namespace boost::chrono; @@ -873,6 +1020,7 @@ namespace cryptonote LOG_ERROR_CCONTEXT("sent wrong NOTIFY_HAVE_OBJECTS: arg.m_current_blockchain_height=" << arg.current_blockchain_height << " < m_last_response_height=" << context.m_last_response_height << ", dropping connection"); drop_connection(context, false, false); + ++m_sync_bad_spans_downloaded; return 1; } @@ -892,11 +1040,13 @@ namespace cryptonote return 1; } - if(!parse_and_validate_block_from_blob(block_entry.block, b)) + crypto::hash block_hash; + if(!parse_and_validate_block_from_blob(block_entry.block, b, block_hash)) { LOG_ERROR_CCONTEXT("sent wrong block: failed to parse and validate block: " << epee::string_tools::buff_to_hex_nodelimer(block_entry.block) << ", dropping connection"); drop_connection(context, false, false); + ++m_sync_bad_spans_downloaded; return 1; } if (b.miner_tx.vin.size() != 1 || b.miner_tx.vin.front().type() != typeid(txin_gen)) @@ -904,18 +1054,19 @@ namespace cryptonote LOG_ERROR_CCONTEXT("sent wrong block: block: miner tx does not have exactly one txin_gen input" << epee::string_tools::buff_to_hex_nodelimer(block_entry.block) << ", dropping connection"); drop_connection(context, false, false); + ++m_sync_bad_spans_downloaded; return 1; } if (start_height == std::numeric_limits::max()) start_height = boost::get(b.miner_tx.vin[0]).height; - const crypto::hash block_hash = get_block_hash(b); auto req_it = context.m_requested_objects.find(block_hash); if(req_it == context.m_requested_objects.end()) { LOG_ERROR_CCONTEXT("sent wrong NOTIFY_RESPONSE_GET_OBJECTS: block with id=" << epee::string_tools::pod_to_hex(get_blob_hash(block_entry.block)) << " wasn't requested, dropping connection"); drop_connection(context, false, false); + ++m_sync_bad_spans_downloaded; return 1; } if(b.tx_hashes.size() != block_entry.txs.size()) @@ -923,6 +1074,7 @@ namespace cryptonote LOG_ERROR_CCONTEXT("sent wrong NOTIFY_RESPONSE_GET_OBJECTS: block with id=" << epee::string_tools::pod_to_hex(get_blob_hash(block_entry.block)) << ", tx_hashes.size()=" << b.tx_hashes.size() << " mismatch with block_complete_entry.m_txs.size()=" << block_entry.txs.size() << ", dropping connection"); drop_connection(context, false, false); + ++m_sync_bad_spans_downloaded; return 1; } @@ -930,34 +1082,27 @@ namespace cryptonote block_hashes.push_back(block_hash); } - if(context.m_requested_objects.size()) + if(!context.m_requested_objects.empty()) { - MERROR("returned not all requested objects (context.m_requested_objects.size()=" + MERROR(context << "returned not all requested objects (context.m_requested_objects.size()=" << context.m_requested_objects.size() << "), dropping connection"); drop_connection(context, false, false); + ++m_sync_bad_spans_downloaded; return 1; } - // get the last parsed block, which should be the highest one - const crypto::hash last_block_hash = cryptonote::get_block_hash(b); - if(m_core.have_block(last_block_hash)) - { - const uint64_t subchain_height = start_height + arg.blocks.size(); - LOG_DEBUG_CC(context, "These are old blocks, ignoring: blocks " << start_height << " - " << (subchain_height-1) << ", blockchain height " << m_core.get_current_blockchain_height()); - m_block_queue.remove_spans(context.m_connection_id, start_height); - goto skip; - } - { MLOG_YELLOW(el::Level::Debug, context << " Got NEW BLOCKS inside of " << __FUNCTION__ << ": size: " << arg.blocks.size() - << ", blocks: " << start_height << " - " << (start_height + arg.blocks.size() - 1)); + << ", blocks: " << start_height << " - " << (start_height + arg.blocks.size() - 1) << + " (pruning seed " << epee::string_tools::to_string_hex(context.m_pruning_seed) << ")"); // add that new span to the block queue - const boost::posix_time::time_duration dt = now - context.m_last_request_time; + const boost::posix_time::time_duration dt = now - request_time; const float rate = size * 1e6 / (dt.total_microseconds() + 1); - MDEBUG(context << " adding span: " << arg.blocks.size() << " at height " << start_height << ", " << dt.total_microseconds()/1e6 << " seconds, " << (rate/1e3) << " kB/s, size now " << (m_block_queue.get_data_size() + blocks_size) / 1048576.f << " MB"); + MDEBUG(context << " adding span: " << arg.blocks.size() << " at height " << start_height << ", " << dt.total_microseconds()/1e6 << " seconds, " << (rate/1024) << " kB/s, size now " << (m_block_queue.get_data_size() + blocks_size) / 1048576.f << " MB"); m_block_queue.add_blocks(start_height, arg.blocks, context.m_connection_id, rate, blocks_size); + const crypto::hash last_block_hash = cryptonote::get_block_hash(b); context.m_last_known_hash = last_block_hash; if (!m_core.get_test_drop_download() || !m_core.get_test_drop_download_height()) { // DISCARD BLOCKS for testing @@ -965,7 +1110,6 @@ namespace cryptonote } } -skip: try_add_next_blocks(context); return 1; } @@ -982,43 +1126,66 @@ skip: const boost::unique_lock sync{m_sync_lock, boost::try_to_lock}; if (!sync.owns_lock()) { - MINFO("Failed to lock m_sync_lock, going back to download"); + MINFO(context << "Failed to lock m_sync_lock, going back to download"); goto skip; } MDEBUG(context << " lock m_sync_lock, adding blocks to chain..."); + MLOG_PEER_STATE("adding blocks"); { m_core.pause_mine(); - epee::misc_utils::auto_scope_leave_caller scope_exit_handler = epee::misc_utils::create_scope_leave_handler( - boost::bind(&t_core::resume_mine, &m_core)); + m_add_timer.resume(); + bool starting = true; + epee::misc_utils::auto_scope_leave_caller scope_exit_handler = epee::misc_utils::create_scope_leave_handler([this, &starting]() { + m_add_timer.pause(); + m_core.resume_mine(); + if (!starting) + m_last_add_end_time = tools::get_tick_count(); + }); while (1) { const uint64_t previous_height = m_core.get_current_blockchain_height(); uint64_t start_height; - std::list blocks; + std::vector blocks; boost::uuids::uuid span_connection_id; if (!m_block_queue.get_next_span(start_height, blocks, span_connection_id)) { MDEBUG(context << " no next span found, going back to download"); break; } - MDEBUG(context << " next span in the queue has blocks " << start_height << "-" << (start_height + blocks.size() - 1) - << ", we need " << previous_height); if (blocks.empty()) { - MERROR("Next span has no blocks"); + MERROR(context << "Next span has no blocks"); m_block_queue.remove_spans(span_connection_id, start_height); - break; + continue; } + MDEBUG(context << " next span in the queue has blocks " << start_height << "-" << (start_height + blocks.size() - 1) + << ", we need " << previous_height); + block new_block; + crypto::hash last_block_hash; + if (!parse_and_validate_block_from_blob(blocks.back().block, new_block, last_block_hash)) + { + MERROR(context << "Failed to parse block, but it should already have been parsed"); + m_block_queue.remove_spans(span_connection_id, start_height); + continue; + } + if (m_core.have_block(last_block_hash)) + { + const uint64_t subchain_height = start_height + blocks.size(); + LOG_DEBUG_CC(context, "These are old blocks, ignoring: blocks " << start_height << " - " << (subchain_height-1) << ", blockchain height " << m_core.get_current_blockchain_height()); + m_block_queue.remove_spans(span_connection_id, start_height); + ++m_sync_old_spans_downloaded; + continue; + } if (!parse_and_validate_block_from_blob(blocks.front().block, new_block)) { - MERROR("Failed to parse block, but it should already have been parsed"); + MERROR(context << "Failed to parse block, but it should already have been parsed"); m_block_queue.remove_spans(span_connection_id, start_height); - break; + continue; } bool parent_known = m_core.have_block(new_block.prev_id); if (!parent_known) @@ -1031,6 +1198,24 @@ skip: bool parent_requested = m_block_queue.requested(new_block.prev_id); if (!parent_requested) { + // we might be able to ask for that block directly, as we now can request out of order, + // otherwise we continue out of order, unless this block is the one we need, in which + // case we request block hashes, though it might be safer to disconnect ? + if (start_height > previous_height) + { + if (should_drop_connection(context, get_next_needed_pruning_stripe().first)) + { + MDEBUG(context << "Got block with unknown parent which was not requested, but peer does not have that block - dropping connection"); + if (!context.m_is_income) + m_p2p->add_used_stripe_peer(context); + drop_connection(context, false, true); + return 1; + } + MDEBUG(context << "Got block with unknown parent which was not requested, but peer does not have that block - back to download"); + + goto skip; + } + // this can happen if a connection was sicced onto a late span, if it did not have those blocks, // since we don't know that at the sic time LOG_ERROR_CCONTEXT("Got block with unknown parent which was not requested - querying block hashes"); @@ -1046,12 +1231,33 @@ skip: } const boost::posix_time::ptime start = boost::posix_time::microsec_clock::universal_time(); - context.m_last_request_time = start; - m_core.prepare_handle_incoming_blocks(blocks); + if (starting) + { + starting = false; + if (m_last_add_end_time) + { + const uint64_t tnow = tools::get_tick_count(); + const uint64_t ns = tools::ticks_to_ns(tnow - m_last_add_end_time); + MINFO("Restarting adding block after idle for " << ns/1e9 << " seconds"); + } + } + + std::vector pblocks; + if (!m_core.prepare_handle_incoming_blocks(blocks, pblocks)) + { + LOG_ERROR_CCONTEXT("Failure in prepare_handle_incoming_blocks"); + return 1; + } + if (!pblocks.empty() && pblocks.size() != blocks.size()) + { + m_core.cleanup_handle_incoming_blocks(); + LOG_ERROR_CCONTEXT("Internal error: blocks.size() != block_entry.txs.size()"); + return 1; + } uint64_t block_process_time_full = 0, transactions_process_time_full = 0; - size_t num_txs = 0; + size_t num_txs = 0, blockidx = 0; for(const block_complete_entry& block_entry: blocks) { if (m_stopping) @@ -1070,14 +1276,16 @@ skip: LOG_ERROR_CCONTEXT("Internal error: tvc.size() != block_entry.txs.size()"); return 1; } - std::list::const_iterator it = block_entry.txs.begin(); + std::vector::const_iterator it = block_entry.txs.begin(); for (size_t i = 0; i < tvc.size(); ++i, ++it) { if(tvc[i].m_verifivation_failed) { if (!m_p2p->for_connection(span_connection_id, [&](cryptonote_connection_context& context, nodetool::peerid_type peer_id, uint32_t f)->bool{ + cryptonote::transaction tx; + parse_and_validate_tx_from_blob(*it, tx); // must succeed if we got here LOG_ERROR_CCONTEXT("transaction verification failed on NOTIFY_RESPONSE_GET_OBJECTS, tx_id = " - << epee::string_tools::pod_to_hex(get_blob_hash(*it)) << ", dropping connection"); + << epee::string_tools::pod_to_hex(cryptonote::get_transaction_hash(tx)) << ", dropping connection"); drop_connection(context, false, true); return 1; })) @@ -1101,7 +1309,7 @@ skip: TIME_MEASURE_START(block_process_time); block_verification_context bvc = boost::value_initialized(); - m_core.handle_incoming_block(block_entry.block, bvc, false); // <--- process block + m_core.handle_incoming_block(block_entry.block, pblocks.empty() ? NULL : &pblocks[blockidx], bvc, false); // <--- process block if(bvc.m_verifivation_failed) { @@ -1144,10 +1352,11 @@ skip: TIME_MEASURE_FINISH(block_process_time); block_process_time_full += block_process_time; + ++blockidx; } // each download block - MCINFO("sync-info", "Block process time (" << blocks.size() << " blocks, " << num_txs << " txs): " << block_process_time_full + transactions_process_time_full << " (" << transactions_process_time_full << "/" << block_process_time_full << ") ms"); + MDEBUG(context << "Block process time (" << blocks.size() << " blocks, " << num_txs << " txs): " << block_process_time_full + transactions_process_time_full << " (" << transactions_process_time_full << "/" << block_process_time_full << ") ms"); if (!m_core.cleanup_handle_incoming_blocks()) { @@ -1157,28 +1366,54 @@ skip: m_block_queue.remove_spans(span_connection_id, start_height); - if (m_core.get_current_blockchain_height() > previous_height) + const uint64_t current_blockchain_height = m_core.get_current_blockchain_height(); + if (current_blockchain_height > previous_height) { + const uint64_t target_blockchain_height = m_core.get_target_blockchain_height(); const boost::posix_time::time_duration dt = boost::posix_time::microsec_clock::universal_time() - start; + std::string progress_message = ""; + if (current_blockchain_height < target_blockchain_height) + { + uint64_t completion_percent = (current_blockchain_height * 100 / target_blockchain_height); + if (completion_percent == 100) // never show 100% if not actually up to date + completion_percent = 99; + progress_message = " (" + std::to_string(completion_percent) + "%, " + + std::to_string(target_blockchain_height - current_blockchain_height) + " left)"; + } + const uint32_t previous_stripe = tools::get_pruning_stripe(previous_height, target_blockchain_height, CRYPTONOTE_PRUNING_LOG_STRIPES); + const uint32_t current_stripe = tools::get_pruning_stripe(current_blockchain_height, target_blockchain_height, CRYPTONOTE_PRUNING_LOG_STRIPES); std::string timing_message = ""; if (ELPP->vRegistry()->allowed(el::Level::Info, "sync-info")) timing_message = std::string(" (") + std::to_string(dt.total_microseconds()/1e6) + " sec, " - + std::to_string((m_core.get_current_blockchain_height() - previous_height) * 1e6 / dt.total_microseconds()) - + " blocks/sec), " + std::to_string(m_block_queue.get_data_size() / 1048576.f) + " MB queued"; + + std::to_string((current_blockchain_height - previous_height) * 1e6 / dt.total_microseconds()) + + " blocks/sec), " + std::to_string(m_block_queue.get_data_size() / 1048576.f) + " MB queued in " + + std::to_string(m_block_queue.get_num_filled_spans()) + " spans, stripe " + + std::to_string(previous_stripe) + " -> " + std::to_string(current_stripe); if (ELPP->vRegistry()->allowed(el::Level::Debug, "sync-info")) - timing_message += std::string(": ") + m_block_queue.get_overview(); - MGINFO_YELLOW(context << " Synced " << m_core.get_current_blockchain_height() << "/" << m_core.get_target_blockchain_height() - << timing_message); + timing_message += std::string(": ") + m_block_queue.get_overview(current_blockchain_height); + MGINFO_YELLOW("Synced " << current_blockchain_height << "/" << target_blockchain_height + << progress_message << timing_message); + if (previous_stripe != current_stripe) + notify_new_stripe(context, current_stripe); } } } - if (should_download_next_span(context)) + MLOG_PEER_STATE("stopping adding blocks"); + + if (should_download_next_span(context, false)) { - context.m_needed_objects.clear(); - context.m_last_response_height = 0; force_next_span = true; } + else if (should_drop_connection(context, get_next_needed_pruning_stripe().first)) + { + if (!context.m_is_income) + { + m_p2p->add_used_stripe_peer(context); + drop_connection(context, false, false); + } + return 1; + } } skip: @@ -1192,9 +1427,33 @@ skip: } //------------------------------------------------------------------------------------------------------------------------ template + void t_cryptonote_protocol_handler::notify_new_stripe(cryptonote_connection_context& cntxt, uint32_t stripe) + { + m_p2p->for_each_connection([&](cryptonote_connection_context& context, nodetool::peerid_type peer_id, uint32_t support_flags)->bool + { + if (cntxt.m_connection_id == context.m_connection_id) + return true; + if (context.m_state == cryptonote_connection_context::state_normal) + { + const uint32_t peer_stripe = tools::get_pruning_stripe(context.m_pruning_seed); + if (stripe && peer_stripe && peer_stripe != stripe) + return true; + context.m_state = cryptonote_connection_context::state_synchronizing; + LOG_PRINT_CCONTEXT_L2("requesting callback"); + ++context.m_callback_request_count; + m_p2p->request_callback(context); + MLOG_PEER_STATE("requesting callback"); + } + return true; + }); + } + //------------------------------------------------------------------------------------------------------------------------ + template bool t_cryptonote_protocol_handler::on_idle() { m_idle_peer_kicker.do_call(boost::bind(&t_cryptonote_protocol_handler::kick_idle_peers, this)); + m_standby_checker.do_call(boost::bind(&t_cryptonote_protocol_handler::check_standby_peers, this)); + m_sync_search_checker.do_call(boost::bind(&t_cryptonote_protocol_handler::update_sync_search, this)); return m_core.on_idle(); } //------------------------------------------------------------------------------------------------------------------------ @@ -1202,30 +1461,81 @@ skip: bool t_cryptonote_protocol_handler::kick_idle_peers() { MTRACE("Checking for idle peers..."); - std::vector kick_connections; m_p2p->for_each_connection([&](cryptonote_connection_context& context, nodetool::peerid_type peer_id, uint32_t support_flags)->bool { - if (context.m_state == cryptonote_connection_context::state_synchronizing || context.m_state == cryptonote_connection_context::state_before_handshake) + if (context.m_state == cryptonote_connection_context::state_synchronizing && context.m_last_request_time != boost::date_time::not_a_date_time) { - const bool passive = context.m_state == cryptonote_connection_context::state_before_handshake; const boost::posix_time::ptime now = boost::posix_time::microsec_clock::universal_time(); const boost::posix_time::time_duration dt = now - context.m_last_request_time; - const int64_t threshold = passive ? PASSIVE_PEER_KICK_TIME : IDLE_PEER_KICK_TIME; - if (dt.total_microseconds() > threshold) + if (dt.total_microseconds() > IDLE_PEER_KICK_TIME) { - MINFO(context << " kicking " << (passive ? "passive" : "idle") << " peer"); - kick_connections.push_back(context.m_connection_id); + MINFO(context << " kicking idle peer, last update " << (dt.total_microseconds() / 1.e6) << " seconds ago"); + LOG_PRINT_CCONTEXT_L2("requesting callback"); + context.m_last_request_time = boost::date_time::not_a_date_time; + context.m_state = cryptonote_connection_context::state_standby; // we'll go back to adding, then (if we can't), download + ++context.m_callback_request_count; + m_p2p->request_callback(context); } } return true; }); - for (const boost::uuids::uuid &conn_id: kick_connections) + return true; + } + //------------------------------------------------------------------------------------------------------------------------ + template + bool t_cryptonote_protocol_handler::update_sync_search() + { + const uint64_t target = m_core.get_target_blockchain_height(); + const uint64_t height = m_core.get_current_blockchain_height(); + if (target > height) // if we're not synced yet, don't do it + return true; + + MTRACE("Checking for outgoing syncing peers..."); + unsigned n_syncing = 0, n_synced = 0; + boost::uuids::uuid last_synced_peer_id(boost::uuids::nil_uuid()); + m_p2p->for_each_connection([&](cryptonote_connection_context& context, nodetool::peerid_type peer_id, uint32_t support_flags)->bool { - m_p2p->for_connection(conn_id, [this](cryptonote_connection_context& context, nodetool::peerid_type peer_id, uint32_t support_flags) { - drop_connection(context, false, false); + if (!peer_id || context.m_is_income) // only consider connected outgoing peers return true; - }); + if (context.m_state == cryptonote_connection_context::state_synchronizing) + ++n_syncing; + if (context.m_state == cryptonote_connection_context::state_normal) + { + ++n_synced; + if (!context.m_anchor) + last_synced_peer_id = context.m_connection_id; + } + return true; + }); + MTRACE(n_syncing << " syncing, " << n_synced << " synced"); + + // if we're at max out peers, and not enough are syncing + if (n_synced + n_syncing >= m_max_out_peers && n_syncing < P2P_DEFAULT_SYNC_SEARCH_CONNECTIONS_COUNT && last_synced_peer_id != boost::uuids::nil_uuid()) + { + if (!m_p2p->for_connection(last_synced_peer_id, [&](cryptonote_connection_context& ctx, nodetool::peerid_type peer_id, uint32_t f)->bool{ + MINFO(ctx << "dropping synced peer, " << n_syncing << " syncing, " << n_synced << " synced"); + drop_connection(ctx, false, false); + return true; + })) + MDEBUG("Failed to find peer we wanted to drop"); } + + return true; + } + //------------------------------------------------------------------------------------------------------------------------ + template + bool t_cryptonote_protocol_handler::check_standby_peers() + { + m_p2p->for_each_connection([&](cryptonote_connection_context& context, nodetool::peerid_type peer_id, uint32_t support_flags)->bool + { + if (context.m_state == cryptonote_connection_context::state_standby) + { + LOG_PRINT_CCONTEXT_L2("requesting callback"); + ++context.m_callback_request_count; + m_p2p->request_callback(context); + } + return true; + }); return true; } //------------------------------------------------------------------------------------------------------------------------ @@ -1240,75 +1550,174 @@ skip: drop_connection(context, false, false); return 1; } - LOG_PRINT_CCONTEXT_L2("-->>NOTIFY_RESPONSE_CHAIN_ENTRY: m_start_height=" << r.start_height << ", m_total_height=" << r.total_height << ", m_block_ids.size()=" << r.m_block_ids.size()); + MLOG_P2P_MESSAGE("-->>NOTIFY_RESPONSE_CHAIN_ENTRY: m_start_height=" << r.start_height << ", m_total_height=" << r.total_height << ", m_block_ids.size()=" << r.m_block_ids.size()); post_notify(r, context); return 1; } //------------------------------------------------------------------------------------------------------------------------ template - bool t_cryptonote_protocol_handler::should_download_next_span(cryptonote_connection_context& context) const + bool t_cryptonote_protocol_handler::should_download_next_span(cryptonote_connection_context& context, bool standby) { - std::list hashes; + std::vector hashes; boost::uuids::uuid span_connection_id; boost::posix_time::ptime request_time; + boost::uuids::uuid connection_id; std::pair span; + bool filled; - span = m_block_queue.get_start_gap_span(); - if (span.second > 0) - { - MDEBUG(context << " we should download it as there is a gap"); - return true; - } - - // if the next span is not scheduled (or there is none) - span = m_block_queue.get_next_span_if_scheduled(hashes, span_connection_id, request_time); - if (span.second == 0) + const uint64_t blockchain_height = m_core.get_current_blockchain_height(); + if (context.m_remote_blockchain_height <= blockchain_height) + return false; + const boost::posix_time::ptime now = boost::posix_time::microsec_clock::universal_time(); + const bool has_next_block = tools::has_unpruned_block(blockchain_height, context.m_remote_blockchain_height, context.m_pruning_seed); + if (has_next_block) { - // we might be in a weird case where there is a filled next span, - // but it starts higher than the current height - uint64_t height; - std::list bcel; - if (!m_block_queue.get_next_span(height, bcel, span_connection_id, true)) - return false; - if (height > m_core.get_current_blockchain_height()) + if (!m_block_queue.has_next_span(blockchain_height, filled, request_time, connection_id)) { - MDEBUG(context << " we should download it as the next block isn't scheduled"); + MDEBUG(context << " we should download it as no peer reserved it"); return true; } + if (!filled) + { + const long dt = (now - request_time).total_microseconds(); + if (dt >= REQUEST_NEXT_SCHEDULED_SPAN_THRESHOLD) + { + MDEBUG(context << " we should download it as it's not been received yet after " << dt/1e6); + return true; + } + + // in standby, be ready to double download early since we're idling anyway + // let the fastest peer trigger first + long threshold; + const double dl_speed = context.m_max_speed_down; + if (standby && dt >= REQUEST_NEXT_SCHEDULED_SPAN_THRESHOLD_STANDBY && dl_speed > 0) + { + bool download = false; + if (m_p2p->for_connection(connection_id, [&](cryptonote_connection_context& ctx, nodetool::peerid_type peer_id, uint32_t f)->bool{ + const time_t nowt = time(NULL); + const time_t time_since_last_recv = nowt - ctx.m_last_recv; + const float last_activity = std::min((float)time_since_last_recv, dt/1e6f); + const bool stalled = last_activity > LAST_ACTIVITY_STALL_THRESHOLD; + if (stalled) + { + MDEBUG(context << " we should download it as the downloading peer is stalling for " << nowt - ctx.m_last_recv << " seconds"); + download = true; + return true; + } + + // estimate the standby peer can give us 80% of its max speed + // and let it download if that speed is > N times as fast as the current one + // N starts at 10 after REQUEST_NEXT_SCHEDULED_SPAN_THRESHOLD_STANDBY, + // decreases to 1.25 at REQUEST_NEXT_SCHEDULED_SPAN_THRESHOLD, + // so that at times goes without the download being done, a retry becomes easier + const float max_multiplier = 10.f; + const float min_multiplier = 1.25f; + float multiplier = max_multiplier; + if (dt/1e6 >= REQUEST_NEXT_SCHEDULED_SPAN_THRESHOLD_STANDBY) + { + multiplier = max_multiplier - (dt/1e6-REQUEST_NEXT_SCHEDULED_SPAN_THRESHOLD_STANDBY) * (max_multiplier - min_multiplier) / (REQUEST_NEXT_SCHEDULED_SPAN_THRESHOLD - REQUEST_NEXT_SCHEDULED_SPAN_THRESHOLD_STANDBY); + multiplier = std::min(max_multiplier, std::max(min_multiplier, multiplier)); + } + if (dl_speed * .8f > ctx.m_current_speed_down * multiplier) + { + MDEBUG(context << " we should download it as we are substantially faster (" << dl_speed << " vs " + << ctx.m_current_speed_down << ", multiplier " << multiplier << " after " << dt/1e6 << " seconds)"); + download = true; + return true; + } + return true; + })) + { + if (download) + return true; + } + else + { + MWARNING(context << " we should download it as the downloading peer is unexpectedly not known to us"); + return true; + } + } + } + } + + return false; + } + //------------------------------------------------------------------------------------------------------------------------ + template + bool t_cryptonote_protocol_handler::should_drop_connection(cryptonote_connection_context& context, uint32_t next_stripe) + { + if (context.m_anchor) + { + MDEBUG(context << "This is an anchor peer, not dropping"); return false; } - // if it was scheduled by this particular peer - if (span_connection_id == context.m_connection_id) + if (context.m_pruning_seed == 0) + { + MDEBUG(context << "This peer is not striped, not dropping"); return false; + } - float span_speed = m_block_queue.get_speed(span_connection_id); - float speed = m_block_queue.get_speed(context.m_connection_id); - MDEBUG(context << " next span is scheduled for " << span_connection_id << ", speed " << span_speed << ", ours " << speed); - - // we try for that span too if: - // - we're substantially faster, or: - // - we're the fastest and the other one isn't (avoids a peer being waaaay slow but yet unmeasured) - // - the other one asked at least 5 seconds ago - if (span_speed < .25 && speed > .75f) + const uint32_t peer_stripe = tools::get_pruning_stripe(context.m_pruning_seed); + if (next_stripe == peer_stripe) { - MDEBUG(context << " we should download it as we're substantially faster"); - return true; + MDEBUG(context << "This peer has needed stripe " << peer_stripe << ", not dropping"); + return false; } - if (speed == 1.0f && span_speed != 1.0f) + + if (!context.m_needed_objects.empty()) { - MDEBUG(context << " we should download it as we're the fastest peer"); - return true; + const uint64_t next_available_block_height = context.m_last_response_height - context.m_needed_objects.size() + 1; + if (tools::has_unpruned_block(next_available_block_height, context.m_remote_blockchain_height, context.m_pruning_seed)) + { + MDEBUG(context << "This peer has unpruned next block at height " << next_available_block_height << ", not dropping"); + return false; + } } - const boost::posix_time::ptime now = boost::posix_time::microsec_clock::universal_time(); - if ((now - request_time).total_microseconds() > REQUEST_NEXT_SCHEDULED_SPAN_THRESHOLD) + + if (next_stripe > 0) { - MDEBUG(context << " we should download it as this span was requested long ago"); - return true; + unsigned int n_out_peers = 0, n_peers_on_next_stripe = 0; + m_p2p->for_each_connection([&](cryptonote_connection_context& ctx, nodetool::peerid_type peer_id, uint32_t support_flags)->bool{ + if (!ctx.m_is_income) + ++n_out_peers; + if (ctx.m_state >= cryptonote_connection_context::state_synchronizing && tools::get_pruning_stripe(ctx.m_pruning_seed) == next_stripe) + ++n_peers_on_next_stripe; + return true; + }); + const uint32_t distance = (peer_stripe + (1<= m_max_out_peers && n_peers_on_next_stripe == 0) || (distance > 1 && n_peers_on_next_stripe <= 2) || distance > 2) + { + MDEBUG(context << "we want seed " << next_stripe << ", and either " << n_out_peers << " is at max out peers (" + << m_max_out_peers << ") or distance " << distance << " from " << next_stripe << " to " << peer_stripe << + " is too large and we have only " << n_peers_on_next_stripe << " peers on next seed, dropping connection to make space"); + return true; + } } + MDEBUG(context << "End of checks, not dropping"); return false; } //------------------------------------------------------------------------------------------------------------------------ template + void t_cryptonote_protocol_handler::skip_unneeded_hashes(cryptonote_connection_context& context, bool check_block_queue) const + { + // take out blocks we already have + size_t skip = 0; + while (skip < context.m_needed_objects.size() && (m_core.have_block(context.m_needed_objects[skip]) || (check_block_queue && m_block_queue.have(context.m_needed_objects[skip])))) + { + // if we're popping the last hash, record it so we can ask again from that hash, + // this prevents never being able to progress on peers we get old hash lists from + if (skip + 1 == context.m_needed_objects.size()) + context.m_last_known_hash = context.m_needed_objects[skip]; + ++skip; + } + if (skip > 0) + { + MDEBUG(context << "skipping " << skip << "/" << context.m_needed_objects.size() << " blocks"); + context.m_needed_objects = std::vector(context.m_needed_objects.begin() + skip, context.m_needed_objects.end()); + } + } + //------------------------------------------------------------------------------------------------------------------------ + template bool t_cryptonote_protocol_handler::request_missing_objects(cryptonote_connection_context& context, bool check_having_blocks, bool force_next_span) { // flush stale spans @@ -1323,64 +1732,114 @@ skip: bool start_from_current_chain = false; if (!force_next_span) { - bool first = true; - while (1) + do { - size_t nblocks = m_block_queue.get_num_filled_spans(); + size_t nspans = m_block_queue.get_num_filled_spans(); size_t size = m_block_queue.get_data_size(); - if (nblocks < BLOCK_QUEUE_NBLOCKS_THRESHOLD || size < BLOCK_QUEUE_SIZE_THRESHOLD) + const uint64_t bc_height = m_core.get_current_blockchain_height(); + const auto next_needed_pruning_stripe = get_next_needed_pruning_stripe(); + const uint32_t add_stripe = tools::get_pruning_stripe(bc_height, context.m_remote_blockchain_height, CRYPTONOTE_PRUNING_LOG_STRIPES); + const uint32_t peer_stripe = tools::get_pruning_stripe(context.m_pruning_seed); + const size_t block_queue_size_threshold = m_block_download_max_size ? m_block_download_max_size : BLOCK_QUEUE_SIZE_THRESHOLD; + bool queue_proceed = nspans < BLOCK_QUEUE_NSPANS_THRESHOLD || size < block_queue_size_threshold; + // get rid of blocks we already requested, or already have + skip_unneeded_hashes(context, true); + uint64_t next_needed_height = m_block_queue.get_next_needed_height(bc_height); + uint64_t next_block_height; + if (context.m_needed_objects.empty()) + next_block_height = next_needed_height; + else + next_block_height = context.m_last_response_height - context.m_needed_objects.size() + 1; + bool stripe_proceed_main = (add_stripe == 0 || peer_stripe == 0 || add_stripe == peer_stripe) && (next_block_height < bc_height + BLOCK_QUEUE_FORCE_DOWNLOAD_NEAR_BLOCKS || next_needed_height < bc_height + BLOCK_QUEUE_FORCE_DOWNLOAD_NEAR_BLOCKS); + bool stripe_proceed_secondary = tools::has_unpruned_block(next_block_height, context.m_remote_blockchain_height, context.m_pruning_seed); + bool proceed = stripe_proceed_main || (queue_proceed && stripe_proceed_secondary); + if (!stripe_proceed_main && !stripe_proceed_secondary && should_drop_connection(context, tools::get_pruning_stripe(next_block_height, context.m_remote_blockchain_height, CRYPTONOTE_PRUNING_LOG_STRIPES))) { - if (!first) - { - LOG_DEBUG_CC(context, "Block queue is " << nblocks << " and " << size << ", resuming"); - } - break; + if (!context.m_is_income) + m_p2p->add_used_stripe_peer(context); + return false; // drop outgoing connections } - // this one triggers if all threads are in standby, which should not happen, - // but happened at least once, so we unblock at least one thread if so - const boost::unique_lock sync{m_sync_lock, boost::try_to_lock}; - if (sync.owns_lock()) - { - LOG_DEBUG_CC(context, "No other thread is adding blocks, resuming"); - break; - } + MDEBUG(context << "proceed " << proceed << " (queue " << queue_proceed << ", stripe " << stripe_proceed_main << "/" << + stripe_proceed_secondary << "), " << next_needed_pruning_stripe.first << "-" << next_needed_pruning_stripe.second << + " needed, bc add stripe " << add_stripe << ", we have " << peer_stripe << "), bc_height " << bc_height); + MDEBUG(context << " - next_block_height " << next_block_height << ", seed " << epee::string_tools::to_string_hex(context.m_pruning_seed) << + ", next_needed_height "<< next_needed_height); + MDEBUG(context << " - last_response_height " << context.m_last_response_height << ", m_needed_objects size " << context.m_needed_objects.size()); - if (should_download_next_span(context)) + // if we're waiting for next span, try to get it before unblocking threads below, + // or a runaway downloading of future spans might happen + if (stripe_proceed_main && should_download_next_span(context, true)) { MDEBUG(context << " we should try for that next span too, we think we could get it faster, resuming"); force_next_span = true; + MLOG_PEER_STATE("resuming"); break; } - if (first) + if (proceed) { - LOG_DEBUG_CC(context, "Block queue is " << nblocks << " and " << size << ", pausing"); - first = false; - context.m_state = cryptonote_connection_context::state_standby; + if (context.m_state != cryptonote_connection_context::state_standby) + { + LOG_DEBUG_CC(context, "Block queue is " << nspans << " and " << size << ", resuming"); + MLOG_PEER_STATE("resuming"); + } + break; } - // this needs doing after we went to standby, so the callback knows what to do - bool filled; - if (m_block_queue.has_next_span(context.m_connection_id, filled) && !filled) + // this one triggers if all threads are in standby, which should not happen, + // but happened at least once, so we unblock at least one thread if so + boost::unique_lock sync{m_sync_lock, boost::try_to_lock}; + if (sync.owns_lock()) { - MDEBUG(context << " we have the next span, and it is scheduled, resuming"); - ++context.m_callback_request_count; - m_p2p->request_callback(context); - return true; + bool filled = false; + boost::posix_time::ptime time; + boost::uuids::uuid connection_id; + if (m_block_queue.has_next_span(m_core.get_current_blockchain_height(), filled, time, connection_id) && filled) + { + LOG_DEBUG_CC(context, "No other thread is adding blocks, and next span needed is ready, resuming"); + MLOG_PEER_STATE("resuming"); + context.m_state = cryptonote_connection_context::state_standby; + ++context.m_callback_request_count; + m_p2p->request_callback(context); + return true; + } + else + { + sync.unlock(); + + // if this has gone on for too long, drop incoming connection to guard against some wedge state + if (!context.m_is_income) + { + const uint64_t now = tools::get_tick_count(); + const uint64_t dt = now - m_last_add_end_time; + if (tools::ticks_to_ns(dt) >= DROP_ON_SYNC_WEDGE_THRESHOLD) + { + MDEBUG(context << "Block addition seems to have wedged, dropping connection"); + return false; + } + } + } } - for (size_t n = 0; n < 50; ++n) + if (context.m_state != cryptonote_connection_context::state_standby) { - if (m_stopping) - return true; - boost::this_thread::sleep_for(boost::chrono::milliseconds(100)); + if (!queue_proceed) + LOG_DEBUG_CC(context, "Block queue is " << nspans << " and " << size << ", pausing"); + else if (!stripe_proceed_main && !stripe_proceed_secondary) + LOG_DEBUG_CC(context, "We do not have the stripe required to download another block, pausing"); + context.m_state = cryptonote_connection_context::state_standby; + MLOG_PEER_STATE("pausing"); } - } + + return true; + } while(0); context.m_state = cryptonote_connection_context::state_synchronizing; } - MDEBUG(context << " request_missing_objects: check " << check_having_blocks << ", force_next_span " << force_next_span << ", m_needed_objects " << context.m_needed_objects.size() << " lrh " << context.m_last_response_height << ", chain " << m_core.get_current_blockchain_height()); + MDEBUG(context << " request_missing_objects: check " << check_having_blocks << ", force_next_span " << force_next_span + << ", m_needed_objects " << context.m_needed_objects.size() << " lrh " << context.m_last_response_height << ", chain " + << m_core.get_current_blockchain_height() << ", pruning seed " << epee::string_tools::to_string_hex(context.m_pruning_seed)); if(context.m_needed_objects.size() || force_next_span) { //we know objects that we need, request this objects @@ -1389,33 +1848,11 @@ skip: size_t count = 0; const size_t count_limit = m_core.get_block_sync_size(m_core.get_current_blockchain_height()); std::pair span = std::make_pair(0, 0); - { - MDEBUG(context << " checking for gap"); - span = m_block_queue.get_start_gap_span(); - if (span.second > 0) - { - const uint64_t first_block_height_known = context.m_last_response_height - context.m_needed_objects.size() + 1; - const uint64_t last_block_height_known = context.m_last_response_height; - const uint64_t first_block_height_needed = span.first; - const uint64_t last_block_height_needed = span.first + std::min(span.second, (uint64_t)count_limit) - 1; - MDEBUG(context << " gap found, span: " << span.first << " - " << span.first + span.second - 1 << " (" << last_block_height_needed << ")"); - MDEBUG(context << " current known hashes from " << first_block_height_known << " to " << last_block_height_known); - if (first_block_height_needed < first_block_height_known || last_block_height_needed > last_block_height_known) - { - MDEBUG(context << " we are missing some of the necessary hashes for this gap, requesting chain again"); - context.m_needed_objects.clear(); - context.m_last_response_height = 0; - start_from_current_chain = true; - goto skip; - } - MDEBUG(context << " we have the hashes for this gap"); - } - } if (force_next_span) { if (span.second == 0) { - std::list hashes; + std::vector hashes; boost::uuids::uuid span_connection_id; boost::posix_time::ptime time; span = m_block_queue.get_next_span_if_scheduled(hashes, span_connection_id, time); @@ -1427,6 +1864,7 @@ skip: req.blocks.push_back(hash); context.m_requested_objects.insert(hash); } + m_block_queue.reset_next_span_time(); } } } @@ -1440,26 +1878,30 @@ skip: context.m_last_response_height = 0; goto skip; } - // take out blocks we already have - while (!context.m_needed_objects.empty() && m_core.have_block(context.m_needed_objects.front())) - { - // if we're popping the last hash, record it so we can ask again from that hash, - // this prevents never being able to progress on peers we get old hash lists from - if (context.m_needed_objects.size() == 1) - context.m_last_known_hash = context.m_needed_objects.front(); - context.m_needed_objects.pop_front(); - } + skip_unneeded_hashes(context, false); + const uint64_t first_block_height = context.m_last_response_height - context.m_needed_objects.size() + 1; - span = m_block_queue.reserve_span(first_block_height, context.m_last_response_height, count_limit, context.m_connection_id, context.m_needed_objects); + span = m_block_queue.reserve_span(first_block_height, context.m_last_response_height, count_limit, context.m_connection_id, context.m_pruning_seed, context.m_remote_blockchain_height, context.m_needed_objects); MDEBUG(context << " span from " << first_block_height << ": " << span.first << "/" << span.second); + if (span.second > 0) + { + const uint32_t stripe = tools::get_pruning_stripe(span.first, context.m_remote_blockchain_height, CRYPTONOTE_PRUNING_LOG_STRIPES); + if (context.m_pruning_seed && stripe != tools::get_pruning_stripe(context.m_pruning_seed)) + { + MDEBUG(context << " starting early on next seed (" << span.first << " with stripe " << stripe << + ", context seed " << epee::string_tools::to_string_hex(context.m_pruning_seed) << ")"); + } + } } if (span.second == 0 && !force_next_span) { MDEBUG(context << " still no span reserved, we may be in the corner case of next span scheduled and everything else scheduled/filled"); - std::list hashes; + std::vector hashes; boost::uuids::uuid span_connection_id; boost::posix_time::ptime time; span = m_block_queue.get_next_span_if_scheduled(hashes, span_connection_id, time); + if (span.second > 0 && !tools::has_unpruned_block(span.first, context.m_remote_blockchain_height, context.m_pruning_seed)) + span = std::make_pair(0, 0); if (span.second > 0) { is_next = true; @@ -1487,37 +1929,85 @@ skip: MERROR("ERROR: skip " << skip << ", m_needed_objects " << context.m_needed_objects.size() << ", first_context_block_height" << first_context_block_height); return false; } - while (skip--) - context.m_needed_objects.pop_front(); + if (skip > 0) + context.m_needed_objects = std::vector(context.m_needed_objects.begin() + skip, context.m_needed_objects.end()); if (context.m_needed_objects.size() < span.second) { MERROR("ERROR: span " << span.first << "/" << span.second << ", m_needed_objects " << context.m_needed_objects.size()); return false; } - auto it = context.m_needed_objects.begin(); for (size_t n = 0; n < span.second; ++n) { - req.blocks.push_back(*it); + req.blocks.push_back(context.m_needed_objects[n]); ++count; - context.m_requested_objects.insert(*it); - auto j = it++; - context.m_needed_objects.erase(j); + context.m_requested_objects.insert(context.m_needed_objects[n]); } + context.m_needed_objects = std::vector(context.m_needed_objects.begin() + span.second, context.m_needed_objects.end()); } context.m_last_request_time = boost::posix_time::microsec_clock::universal_time(); - LOG_PRINT_CCONTEXT_L1("-->>NOTIFY_REQUEST_GET_OBJECTS: blocks.size()=" << req.blocks.size() << ", txs.size()=" << req.txs.size() + MLOG_P2P_MESSAGE("-->>NOTIFY_REQUEST_GET_OBJECTS: blocks.size()=" << req.blocks.size() << ", txs.size()=" << req.txs.size() << "requested blocks count=" << count << " / " << count_limit << " from " << span.first << ", first hash " << req.blocks.front()); //epee::net_utils::network_throttle_manager::get_global_throttle_inreq().logger_handle_net("log/dr-monero/net/req-all.data", sec, get_avg_block_size()); post_notify(req, context); + MLOG_PEER_STATE("requesting objects"); + return true; + } + + // if we're still around, we might be at a point where the peer is pruned, so we could either + // drop it to make space for other peers, or ask for a span further down the line + const uint32_t next_stripe = get_next_needed_pruning_stripe().first; + const uint32_t peer_stripe = tools::get_pruning_stripe(context.m_pruning_seed); + if (next_stripe && peer_stripe && next_stripe != peer_stripe) + { + // at this point, we have to either close the connection, or start getting blocks past the + // current point, or become dormant + MDEBUG(context << "this peer is pruned at seed " << epee::string_tools::to_string_hex(context.m_pruning_seed) << + ", next stripe needed is " << next_stripe); + if (!context.m_is_income) + { + if (should_drop_connection(context, next_stripe)) + { + m_p2p->add_used_stripe_peer(context); + return false; // drop outgoing connections + } + } + // we'll get back stuck waiting for the go ahead + context.m_state = cryptonote_connection_context::state_normal; + MLOG_PEER_STATE("Nothing to do for now, switching to normal state"); return true; } } skip: context.m_needed_objects.clear(); + + // we might have been called from the "received chain entry" handler, and end up + // here because we can't use any of those blocks (maybe because all of them are + // actually already requested). In this case, if we can add blocks instead, do so + if (m_core.get_current_blockchain_height() < m_core.get_target_blockchain_height()) + { + const boost::unique_lock sync{m_sync_lock, boost::try_to_lock}; + if (sync.owns_lock()) + { + uint64_t start_height; + std::vector blocks; + boost::uuids::uuid span_connection_id; + bool filled = false; + if (m_block_queue.get_next_span(start_height, blocks, span_connection_id, filled) && filled) + { + LOG_DEBUG_CC(context, "No other thread is adding blocks, resuming"); + MLOG_PEER_STATE("will try to add blocks next"); + context.m_state = cryptonote_connection_context::state_standby; + ++context.m_callback_request_count; + m_p2p->request_callback(context); + return true; + } + } + } + if(context.m_last_response_height < context.m_remote_blockchain_height-1) {//we have to fetch more objects ids, request blockchain entry @@ -1540,8 +2030,9 @@ skip: //LOG_PRINT_CCONTEXT_L1("r = " << 200); context.m_last_request_time = boost::posix_time::microsec_clock::universal_time(); - LOG_PRINT_CCONTEXT_L1("-->>NOTIFY_REQUEST_CHAIN: m_block_ids.size()=" << r.block_ids.size() << ", start_from_current_chain " << start_from_current_chain); + MLOG_P2P_MESSAGE("-->>NOTIFY_REQUEST_CHAIN: m_block_ids.size()=" << r.block_ids.size() << ", start_from_current_chain " << start_from_current_chain); post_notify(r, context); + MLOG_PEER_STATE("requesting chain"); }else { CHECK_AND_ASSERT_MES(context.m_last_response_height == context.m_remote_blockchain_height-1 @@ -1581,9 +2072,25 @@ skip: << ENDL << "Use the \"help\" command to see the list of available commands." << ENDL << "**********************************************************************"); + m_sync_timer.pause(); + if (ELPP->vRegistry()->allowed(el::Level::Info, "sync-info")) + { + const uint64_t sync_time = m_sync_timer.value(); + const uint64_t add_time = m_add_timer.value(); + if (sync_time && add_time) + { + MCLOG_YELLOW(el::Level::Info, "sync-info", "Sync time: " << sync_time/1e9/60 << " min, idle time " << + (100.f * (1.0f - add_time / (float)sync_time)) << "%" << ", " << + (10 * m_sync_download_objects_size / 1024 / 1024) / 10.f << " + " << + (10 * m_sync_download_chain_size / 1024 / 1024) / 10.f << " MB downloaded, " << + 100.0f * m_sync_old_spans_downloaded / m_sync_spans_downloaded << "% old spans, " << + 100.0f * m_sync_bad_spans_downloaded / m_sync_spans_downloaded << "% bad spans"); + } + } m_core.on_synchronized(); } m_core.safesyncmode(true); + m_p2p->clear_used_stripe_peers(); return true; } //------------------------------------------------------------------------------------------------------------------------ @@ -1604,6 +2111,11 @@ skip: { MLOG_P2P_MESSAGE("Received NOTIFY_RESPONSE_CHAIN_ENTRY: m_block_ids.size()=" << arg.m_block_ids.size() << ", m_start_height=" << arg.start_height << ", m_total_height=" << arg.total_height); + MLOG_PEER_STATE("received chain"); + + context.m_last_request_time = boost::date_time::not_a_date_time; + + m_sync_download_chain_size += arg.m_block_ids.size() * sizeof(crypto::hash); if(!arg.m_block_ids.size()) { @@ -1617,7 +2129,14 @@ skip: drop_connection(context, true, false); return 1; } + MDEBUG(context << "first block hash " << arg.m_block_ids.front() << ", last " << arg.m_block_ids.back()); + if (arg.total_height >= CRYPTONOTE_MAX_BLOCK_NUMBER || arg.m_block_ids.size() >= CRYPTONOTE_MAX_BLOCK_NUMBER) + { + LOG_ERROR_CCONTEXT("sent wrong NOTIFY_RESPONSE_CHAIN_ENTRY, with total_height=" << arg.total_height << " and block_ids=" << arg.m_block_ids.size()); + drop_connection(context, false, false); + return 1; + } context.m_remote_blockchain_height = arg.total_height; context.m_last_response_height = arg.start_height + arg.m_block_ids.size()-1; if(context.m_last_response_height > context.m_remote_blockchain_height) @@ -1637,6 +2156,7 @@ skip: return 1; } + context.m_needed_objects.clear(); uint64_t added = 0; for(auto& bl_id: arg.m_block_ids) { @@ -1664,38 +2184,43 @@ skip: { NOTIFY_NEW_FLUFFY_BLOCK::request fluffy_arg = AUTO_VAL_INIT(fluffy_arg); fluffy_arg.current_blockchain_height = arg.current_blockchain_height; - std::list fluffy_txs; + std::vector fluffy_txs; fluffy_arg.b = arg.b; fluffy_arg.b.txs = fluffy_txs; - // pre-serialize them - std::string fullBlob, fluffyBlob; - epee::serialization::store_t_to_binary(arg, fullBlob); - epee::serialization::store_t_to_binary(fluffy_arg, fluffyBlob); - // sort peers between fluffy ones and others - std::list fullConnections, fluffyConnections; + std::vector> fullConnections, fluffyConnections; m_p2p->for_each_connection([this, &exclude_context, &fullConnections, &fluffyConnections](connection_context& context, nodetool::peerid_type peer_id, uint32_t support_flags) { - if (peer_id && exclude_context.m_connection_id != context.m_connection_id) + if (peer_id && exclude_context.m_connection_id != context.m_connection_id && context.m_remote_address.get_zone() == epee::net_utils::zone::public_) { if(m_core.fluffy_blocks_enabled() && (support_flags & P2P_SUPPORT_FLAG_FLUFFY_BLOCKS)) { LOG_DEBUG_CC(context, "PEER SUPPORTS FLUFFY BLOCKS - RELAYING THIN/COMPACT WHATEVER BLOCK"); - fluffyConnections.push_back(context.m_connection_id); + fluffyConnections.push_back({context.m_remote_address.get_zone(), context.m_connection_id}); } else { LOG_DEBUG_CC(context, "PEER DOESN'T SUPPORT FLUFFY BLOCKS - RELAYING FULL BLOCK"); - fullConnections.push_back(context.m_connection_id); + fullConnections.push_back({context.m_remote_address.get_zone(), context.m_connection_id}); } } return true; }); // send fluffy ones first, we want to encourage people to run that - m_p2p->relay_notify_to_list(NOTIFY_NEW_FLUFFY_BLOCK::ID, fluffyBlob, fluffyConnections); - m_p2p->relay_notify_to_list(NOTIFY_NEW_BLOCK::ID, fullBlob, fullConnections); + if (!fluffyConnections.empty()) + { + std::string fluffyBlob; + epee::serialization::store_t_to_binary(fluffy_arg, fluffyBlob); + m_p2p->relay_notify_to_list(NOTIFY_NEW_FLUFFY_BLOCK::ID, epee::strspan(fluffyBlob), std::move(fluffyConnections)); + } + if (!fullConnections.empty()) + { + std::string fullBlob; + epee::serialization::store_t_to_binary(arg, fullBlob); + m_p2p->relay_notify_to_list(NOTIFY_NEW_BLOCK::ID, epee::strspan(fullBlob), std::move(fullConnections)); + } return true; } @@ -1703,21 +2228,159 @@ skip: template bool t_cryptonote_protocol_handler::relay_transactions(NOTIFY_NEW_TRANSACTIONS::request& arg, cryptonote_connection_context& exclude_context) { + const bool hide_tx_broadcast = + 1 < m_p2p->get_zone_count() && exclude_context.m_remote_address.get_zone() == epee::net_utils::zone::invalid; + + if (hide_tx_broadcast) + MDEBUG("Attempting to conceal origin of tx via anonymity network connection(s)"); + // no check for success, so tell core they're relayed unconditionally + const bool pad_transactions = m_core.pad_transactions() || hide_tx_broadcast; + size_t bytes = pad_transactions ? 9 /* header */ + 4 /* 1 + 'txs' */ + tools::get_varint_data(arg.txs.size()).size() : 0; for(auto tx_blob_it = arg.txs.begin(); tx_blob_it!=arg.txs.end(); ++tx_blob_it) + { m_core.on_transaction_relayed(*tx_blob_it); - return relay_post_notify(arg, exclude_context); + if (pad_transactions) + bytes += tools::get_varint_data(tx_blob_it->size()).size() + tx_blob_it->size(); + } + + if (pad_transactions) + { + // stuff some dummy bytes in to stay safe from traffic volume analysis + static constexpr size_t granularity = 1024; + size_t padding = granularity - bytes % granularity; + const size_t overhead = 2 /* 1 + '_' */ + tools::get_varint_data(padding).size(); + if (overhead > padding) + padding = 0; + else + padding -= overhead; + arg._ = std::string(padding, ' '); + + std::string arg_buff; + epee::serialization::store_t_to_binary(arg, arg_buff); + + // we probably lowballed the payload size a bit, so added a but too much. Fix this now. + size_t remove = arg_buff.size() % granularity; + if (remove > arg._.size()) + arg._.clear(); + else + arg._.resize(arg._.size() - remove); + // if the size of _ moved enough, we might lose byte in size encoding, we don't care + } + + std::vector> connections; + m_p2p->for_each_connection([hide_tx_broadcast, &exclude_context, &connections](connection_context& context, nodetool::peerid_type peer_id, uint32_t support_flags) + { + const epee::net_utils::zone current_zone = context.m_remote_address.get_zone(); + const bool broadcast_to_peer = + peer_id && + (hide_tx_broadcast != bool(current_zone == epee::net_utils::zone::public_)) && + exclude_context.m_connection_id != context.m_connection_id; + + if (broadcast_to_peer) + connections.push_back({current_zone, context.m_connection_id}); + + return true; + }); + + if (connections.empty()) + MERROR("Transaction not relayed - no" << (hide_tx_broadcast ? " privacy": "") << " peers available"); + else + { + std::string fullBlob; + epee::serialization::store_t_to_binary(arg, fullBlob); + m_p2p->relay_notify_to_list(NOTIFY_NEW_TRANSACTIONS::ID, epee::strspan(fullBlob), std::move(connections)); + } + return true; + } + //------------------------------------------------------------------------------------------------------------------------ + template + std::string t_cryptonote_protocol_handler::get_peers_overview() const + { + std::stringstream ss; + const boost::posix_time::ptime now = boost::posix_time::microsec_clock::universal_time(); + m_p2p->for_each_connection([&](const connection_context &ctx, nodetool::peerid_type peer_id, uint32_t support_flags) { + const uint32_t stripe = tools::get_pruning_stripe(ctx.m_pruning_seed); + char state_char = cryptonote::get_protocol_state_char(ctx.m_state); + ss << stripe + state_char; + if (ctx.m_last_request_time != boost::date_time::not_a_date_time) + ss << (((now - ctx.m_last_request_time).total_microseconds() > IDLE_PEER_KICK_TIME) ? "!" : "?"); + ss << + " "; + return true; + }); + return ss.str(); + } + //------------------------------------------------------------------------------------------------------------------------ + template + std::pair t_cryptonote_protocol_handler::get_next_needed_pruning_stripe() const + { + const uint64_t want_height_from_blockchain = m_core.get_current_blockchain_height(); + const uint64_t want_height_from_block_queue = m_block_queue.get_next_needed_height(want_height_from_blockchain); + const uint64_t want_height = std::max(want_height_from_blockchain, want_height_from_block_queue); + uint64_t blockchain_height = m_core.get_target_blockchain_height(); + // if we don't know the remote chain size yet, assume infinitely large so we get the right stripe if we're not near the tip + if (blockchain_height == 0) + blockchain_height = CRYPTONOTE_MAX_BLOCK_NUMBER; + const uint32_t next_pruning_stripe = tools::get_pruning_stripe(want_height, blockchain_height, CRYPTONOTE_PRUNING_LOG_STRIPES); + if (next_pruning_stripe == 0) + return std::make_pair(0, 0); + // if we already have a few peers on this stripe, but none on next one, try next one + unsigned int n_next = 0, n_subsequent = 0, n_others = 0; + const uint32_t subsequent_pruning_stripe = 1 + next_pruning_stripe % (1<for_each_connection([&](const connection_context &context, nodetool::peerid_type peer_id, uint32_t support_flags) { + if (context.m_state >= cryptonote_connection_context::state_synchronizing) + { + if (context.m_pruning_seed == 0 || tools::get_pruning_stripe(context.m_pruning_seed) == next_pruning_stripe) + ++n_next; + else if (tools::get_pruning_stripe(context.m_pruning_seed) == subsequent_pruning_stripe) + ++n_subsequent; + else + ++n_others; + } + return true; + }); + const bool use_next = (n_next > m_max_out_peers / 2 && n_subsequent <= 1) || (n_next > 2 && n_subsequent == 0); + const uint32_t ret_stripe = use_next ? subsequent_pruning_stripe: next_pruning_stripe; + MIDEBUG(const std::string po = get_peers_overview(), "get_next_needed_pruning_stripe: want height " << want_height << " (" << + want_height_from_blockchain << " from blockchain, " << want_height_from_block_queue << " from block queue), stripe " << + next_pruning_stripe << " (" << n_next << "/" << m_max_out_peers << " on it and " << n_subsequent << " on " << + subsequent_pruning_stripe << ", " << n_others << " others) -> " << ret_stripe << " (+" << + (ret_stripe - next_pruning_stripe + (1 << CRYPTONOTE_PRUNING_LOG_STRIPES)) % (1 << CRYPTONOTE_PRUNING_LOG_STRIPES) << + "), current peers " << po); + return std::make_pair(next_pruning_stripe, ret_stripe); + } + //------------------------------------------------------------------------------------------------------------------------ + template + bool t_cryptonote_protocol_handler::needs_new_sync_connections() const + { + const uint64_t target = m_core.get_target_blockchain_height(); + const uint64_t height = m_core.get_current_blockchain_height(); + if (target && target <= height) + return false; + size_t n_out_peers = 0; + m_p2p->for_each_connection([&](cryptonote_connection_context& ctx, nodetool::peerid_type peer_id, uint32_t support_flags)->bool{ + if (!ctx.m_is_income) + ++n_out_peers; + return true; + }); + if (n_out_peers >= m_max_out_peers) + return false; + return true; } //------------------------------------------------------------------------------------------------------------------------ template void t_cryptonote_protocol_handler::drop_connection(cryptonote_connection_context &context, bool add_fail, bool flush_all_spans) { + LOG_DEBUG_CC(context, "dropping connection id " << context.m_connection_id << " (pruning seed " << + epee::string_tools::to_string_hex(context.m_pruning_seed) << + "), add_fail " << add_fail << ", flush_all_spans " << flush_all_spans); + if (add_fail) m_p2p->add_host_fail(context.m_remote_address); - m_p2p->drop_connection(context); - m_block_queue.flush_spans(context.m_connection_id, flush_all_spans); + + m_p2p->drop_connection(context); } //------------------------------------------------------------------------------------------------------------------------ template @@ -1734,9 +2397,12 @@ skip: { MINFO("Target height decreasing from " << previous_target << " to " << target); m_core.set_target_blockchain_height(target); + if (target == 0 && context.m_state > cryptonote_connection_context::state_before_handshake && !m_stopping) + MCWARNING("global", "monerod is now disconnected from the network"); } m_block_queue.flush_spans(context.m_connection_id, false); + MLOG_PEER_STATE("closed"); } //------------------------------------------------------------------------------------------------------------------------ @@ -1747,3 +2413,4 @@ skip: m_core.stop(); } } // namespace + diff --git a/src/cryptonote_protocol/cryptonote_protocol_handler_common.h b/src/cryptonote_protocol/cryptonote_protocol_handler_common.h index 2b9f201ec16..a67178c5211 100644 --- a/src/cryptonote_protocol/cryptonote_protocol_handler_common.h +++ b/src/cryptonote_protocol/cryptonote_protocol_handler_common.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/daemon/CMakeLists.txt b/src/daemon/CMakeLists.txt index 3b1d0d8262b..d9bfd9a20ae 100644 --- a/src/daemon/CMakeLists.txt +++ b/src/daemon/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2014-2018, The Monero Project +# Copyright (c) 2014-2019, The Monero Project # # All rights reserved. # @@ -26,16 +26,6 @@ # STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF # THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -set(blocksdat "") -if(PER_BLOCK_CHECKPOINT) - if(APPLE) - add_custom_command(OUTPUT blocksdat.o MAIN_DEPENDENCY ../blocks/checkpoints.dat COMMAND cd ${CMAKE_CURRENT_SOURCE_DIR} && touch stub.c && ${CMAKE_C_COMPILER} -o stub.o -c stub.c COMMAND cd ${CMAKE_CURRENT_SOURCE_DIR} && ${CMAKE_LINKER} ${LD_RAW_FLAGS} -r -sectcreate __DATA __blocks_dat ../blocks/checkpoints.dat -o ${CMAKE_CURRENT_BINARY_DIR}/blocksdat.o stub.o && rm -f stub.*) - else() - add_custom_command(OUTPUT blocksdat.o MAIN_DEPENDENCY ../blocks/checkpoints.dat COMMAND cd ${CMAKE_CURRENT_SOURCE_DIR} && cp ../blocks/checkpoints.dat blocks.dat && ${CMAKE_LINKER} ${LD_RAW_FLAGS} -r -b binary -o ${CMAKE_CURRENT_BINARY_DIR}/blocksdat.o blocks.dat && rm -f blocks.dat) - endif() - set(blocksdat "blocksdat.o") -endif() - set(daemon_sources command_parser_executor.cpp command_server.cpp @@ -77,9 +67,7 @@ monero_private_headers(daemon monero_add_executable(daemon ${daemon_sources} ${daemon_headers} - ${daemon_private_headers} - ${blocksdat} -) + ${daemon_private_headers}) target_link_libraries(daemon PRIVATE rpc @@ -92,7 +80,6 @@ target_link_libraries(daemon daemonizer serialization daemon_rpc_server - epee ${EPEE_READLINE} version ${Boost_CHRONO_LIBRARY} @@ -103,7 +90,8 @@ target_link_libraries(daemon ${CMAKE_THREAD_LIBS_INIT} ${ZMQ_LIB} ${GNU_READLINE_LIBRARY} - ${EXTRA_LIBRARIES}) + ${EXTRA_LIBRARIES} + ${Blocks}) set_property(TARGET daemon PROPERTY OUTPUT_NAME "monerod") diff --git a/src/daemon/command_line_args.h b/src/daemon/command_line_args.h index 4673590aa63..32fdca5ea52 100644 --- a/src/daemon/command_line_args.h +++ b/src/daemon/command_line_args.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -72,6 +72,11 @@ namespace daemon_args , "Specify maximum log file size [B]" , MAX_LOG_FILE_SIZE }; + const command_line::arg_descriptor arg_max_log_files = { + "max-log-files" + , "Specify maximum number of rotated log files to be saved (no limit by setting to 0)" + , MAX_LOG_FILES + }; const command_line::arg_descriptor arg_log_level = { "log-level" , "" @@ -91,6 +96,12 @@ namespace daemon_args , 0 }; + const command_line::arg_descriptor arg_public_node = { + "public-node" + , "Allow other users to use the node as a remote (restricted RPC mode, view-only commands) and advertise it over P2P" + , false + }; + const command_line::arg_descriptor arg_zmq_rpc_bind_ip = { "zmq-rpc-bind-ip" , "IP for ZMQ RPC server to listen on" diff --git a/src/daemon/command_parser_executor.cpp b/src/daemon/command_parser_executor.cpp index 34d9fb4c8b1..0b452800e94 100644 --- a/src/daemon/command_parser_executor.cpp +++ b/src/daemon/command_parser_executor.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -27,6 +27,7 @@ // THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "common/dns_utils.h" +#include "common/command_line.h" #include "version.h" #include "daemon/command_parser_executor.h" @@ -39,17 +40,43 @@ t_command_parser_executor::t_command_parser_executor( uint32_t ip , uint16_t port , const boost::optional& login + , const epee::net_utils::ssl_options_t& ssl_options , bool is_rpc , cryptonote::core_rpc_server* rpc_server ) - : m_executor(ip, port, login, is_rpc, rpc_server) + : m_executor(ip, port, login, ssl_options, is_rpc, rpc_server) {} bool t_command_parser_executor::print_peer_list(const std::vector& args) { - if (!args.empty()) return false; + if (args.size() > 3) + { + std::cout << "use: print_pl [white] [gray] []" << std::endl; + return true; + } - return m_executor.print_peer_list(); + bool white = false; + bool gray = false; + size_t limit = 0; + for (size_t i = 0; i < args.size(); ++i) + { + if (args[i] == "white") + { + white = true; + } + else if (args[i] == "gray") + { + gray = true; + } + else if (!epee::string_tools::get_xtype_from_string(limit, args[i])) + { + std::cout << "unexpected argument: " << args[i] << std::endl; + return true; + } + } + + const bool print_both = !white && !gray; + return m_executor.print_peer_list(white | print_both, gray | print_both, limit); } bool t_command_parser_executor::print_peer_list_stats(const std::vector& args) @@ -101,6 +128,13 @@ bool t_command_parser_executor::print_connections(const std::vector return m_executor.print_connections(); } +bool t_command_parser_executor::print_net_stats(const std::vector& args) +{ + if (!args.empty()) return false; + + return m_executor.print_net_stats(); +} + bool t_command_parser_executor::print_blockchain_info(const std::vector& args) { if(!args.size()) @@ -162,9 +196,21 @@ bool t_command_parser_executor::print_height(const std::vector& arg bool t_command_parser_executor::print_block(const std::vector& args) { + bool include_hex = false; + + // Assumes that optional flags come after mandatory argument + for (unsigned int i = 1; i < args.size(); ++i) { + if (args[i] == "+hex") + include_hex = true; + else + { + std::cout << "unexpected argument: " << args[i] << std::endl; + return true; + } + } if (args.empty()) { - std::cout << "expected: print_block ( | )" << std::endl; + std::cout << "expected: print_block ( | ) [+hex]" << std::endl; return false; } @@ -172,14 +218,14 @@ bool t_command_parser_executor::print_block(const std::vector& args try { uint64_t height = boost::lexical_cast(arg); - return m_executor.print_block_by_height(height); + return m_executor.print_block_by_height(height, include_hex); } catch (const boost::bad_lexical_cast&) { crypto::hash block_hash; if (parse_hash256(arg, block_hash)) { - return m_executor.print_block_by_hash(block_hash); + return m_executor.print_block_by_hash(block_hash, include_hex); } } @@ -264,7 +310,7 @@ bool t_command_parser_executor::start_mining(const std::vector& arg { if(!args.size()) { - std::cout << "Please specify a wallet address to mine for: start_mining []" << std::endl; + std::cout << "Please specify a wallet address to mine for: start_mining [|auto]" << std::endl; return true; } @@ -326,18 +372,39 @@ bool t_command_parser_executor::start_mining(const std::vector& arg if(args.size() == 4) { - ignore_battery = args[3] == "true"; + if(args[3] == "true" || command_line::is_yes(args[3]) || args[3] == "1") + { + ignore_battery = true; + } + else if(args[3] != "false" && !command_line::is_no(args[3]) && args[3] != "0") + { + return false; + } } if(args.size() >= 3) { - do_background_mining = args[2] == "true"; + if(args[2] == "true" || command_line::is_yes(args[2]) || args[2] == "1") + { + do_background_mining = true; + } + else if(args[2] != "false" && !command_line::is_no(args[2]) && args[2] != "0") + { + return false; + } } if(args.size() >= 2) { - bool ok = epee::string_tools::get_xtype_from_string(threads_count, args[1]); - threads_count = (ok && 0 < threads_count) ? threads_count : 1; + if (args[1] == "auto" || args[1] == "autodetect") + { + threads_count = 0; + } + else + { + bool ok = epee::string_tools::get_xtype_from_string(threads_count, args[1]); + threads_count = (ok && 0 < threads_count) ? threads_count : 1; + } } m_executor.start_mining(info.address, threads_count, nettype, do_background_mining, ignore_battery); @@ -352,6 +419,11 @@ bool t_command_parser_executor::stop_mining(const std::vector& args return m_executor.stop_mining(); } +bool t_command_parser_executor::mining_status(const std::vector& args) +{ + return m_executor.mining_status(); +} + bool t_command_parser_executor::stop_daemon(const std::vector& args) { if (!args.empty()) return false; @@ -599,13 +671,13 @@ bool t_command_parser_executor::print_coinbase_tx_sum(const std::vector& args) { - if(args.size()) + if(args.size() > 1) { - std::cout << "No parameters allowed" << std::endl; + std::cout << "usage: alt_chain_info [block_hash]" << std::endl; return false; } - return m_executor.alt_chain_info(); + return m_executor.alt_chain_info(args.size() == 1 ? args[0] : ""); } bool t_command_parser_executor::print_blockchain_dynamic_stats(const std::vector& args) @@ -659,10 +731,58 @@ bool t_command_parser_executor::sync_info(const std::vector& args) return m_executor.sync_info(); } +bool t_command_parser_executor::pop_blocks(const std::vector& args) +{ + if (args.size() != 1) + { + std::cout << "Exactly one parameter is needed" << std::endl; + return false; + } + + try + { + uint64_t nblocks = boost::lexical_cast(args[0]); + if (nblocks < 1) + { + std::cout << "number of blocks must be greater than 0" << std::endl; + return false; + } + return m_executor.pop_blocks(nblocks); + } + catch (const boost::bad_lexical_cast&) + { + std::cout << "number of blocks must be a number greater than 0" << std::endl; + } + return false; +} + bool t_command_parser_executor::version(const std::vector& args) { std::cout << "Monero '" << MONERO_RELEASE_NAME << "' (v" << MONERO_VERSION_FULL << ")" << std::endl; return true; } +bool t_command_parser_executor::prune_blockchain(const std::vector& args) +{ + if (args.size() > 1) return false; + + if (args.empty() || args[0] != "confirm") + { + std::cout << "Warning: pruning from within monerod will not shrink the database file size." << std::endl; + std::cout << "Instead, parts of the file will be marked as free, so the file will not grow" << std::endl; + std::cout << "until that newly free space is used up. If you want a smaller file size now," << std::endl; + std::cout << "exit monerod and run monero-blockchain-prune (you will temporarily need more" << std::endl; + std::cout << "disk space for the database conversion though). If you are OK with the database" << std::endl; + std::cout << "file keeping the same size, re-run this command with the \"confirm\" parameter." << std::endl; + return true; + } + + return m_executor.prune_blockchain(); +} + +bool t_command_parser_executor::check_blockchain_pruning(const std::vector& args) +{ + return m_executor.check_blockchain_pruning(); +} + } // namespace daemonize diff --git a/src/daemon/command_parser_executor.h b/src/daemon/command_parser_executor.h index a700701716c..2efd78ec03d 100644 --- a/src/daemon/command_parser_executor.h +++ b/src/daemon/command_parser_executor.h @@ -6,7 +6,7 @@ */ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -40,6 +40,7 @@ #include "daemon/rpc_command_executor.h" #include "common/common_fwd.h" +#include "net/net_fwd.h" #include "rpc/core_rpc_server.h" namespace daemonize { @@ -53,6 +54,7 @@ class t_command_parser_executor final uint32_t ip , uint16_t port , const boost::optional& login + , const epee::net_utils::ssl_options_t& ssl_options , bool is_rpc , cryptonote::core_rpc_server* rpc_server = NULL ); @@ -97,6 +99,8 @@ class t_command_parser_executor final bool stop_mining(const std::vector& args); + bool mining_status(const std::vector& args); + bool stop_daemon(const std::vector& args); bool print_status(const std::vector& args); @@ -139,7 +143,15 @@ class t_command_parser_executor final bool sync_info(const std::vector& args); + bool pop_blocks(const std::vector& args); + bool version(const std::vector& args); + + bool prune_blockchain(const std::vector& args); + + bool check_blockchain_pruning(const std::vector& args); + + bool print_net_stats(const std::vector& args); }; } // namespace daemonize diff --git a/src/daemon/command_server.cpp b/src/daemon/command_server.cpp index 14460359725..f665eec9c37 100644 --- a/src/daemon/command_server.cpp +++ b/src/daemon/command_server.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -43,10 +43,11 @@ t_command_server::t_command_server( uint32_t ip , uint16_t port , const boost::optional& login + , const epee::net_utils::ssl_options_t& ssl_options , bool is_rpc , cryptonote::core_rpc_server* rpc_server ) - : m_parser(ip, port, login, is_rpc, rpc_server) + : m_parser(ip, port, login, ssl_options, is_rpc, rpc_server) , m_command_lookup() , m_is_rpc(is_rpc) { @@ -64,6 +65,7 @@ t_command_server::t_command_server( m_command_lookup.set_handler( "print_pl" , std::bind(&t_command_parser_executor::print_peer_list, &m_parser, p::_1) + , "print_pl [white] [gray] []" , "Print the current peer list." ); m_command_lookup.set_handler( @@ -76,6 +78,11 @@ t_command_server::t_command_server( , std::bind(&t_command_parser_executor::print_connections, &m_parser, p::_1) , "Print the current connections." ); + m_command_lookup.set_handler( + "print_net_stats" + , std::bind(&t_command_parser_executor::print_net_stats, &m_parser, p::_1) + , "Print network statistics." + ); m_command_lookup.set_handler( "print_bc" , std::bind(&t_command_parser_executor::print_blockchain_info, &m_parser, p::_1) @@ -103,14 +110,19 @@ t_command_server::t_command_server( m_command_lookup.set_handler( "start_mining" , std::bind(&t_command_parser_executor::start_mining, &m_parser, p::_1) - , "start_mining [] [do_background_mining] [ignore_battery]" - , "Start mining for specified address. Defaults to 1 thread and no background mining." + , "start_mining [|auto] [do_background_mining] [ignore_battery]" + , "Start mining for specified address. Defaults to 1 thread and no background mining. Use \"auto\" to autodetect optimal number of threads." ); m_command_lookup.set_handler( "stop_mining" , std::bind(&t_command_parser_executor::stop_mining, &m_parser, p::_1) , "Stop mining." ); + m_command_lookup.set_handler( + "mining_status" + , std::bind(&t_command_parser_executor::mining_status, &m_parser, p::_1) + , "Show current mining status." + ); m_command_lookup.set_handler( "print_pool" , std::bind(&t_command_parser_executor::print_transaction_pool_long, &m_parser, p::_1) @@ -255,6 +267,7 @@ t_command_server::t_command_server( m_command_lookup.set_handler( "alt_chain_info" , std::bind(&t_command_parser_executor::alt_chain_info, &m_parser, p::_1) + , "alt_chain_info [blockhash]" , "Print the information about alternative chains." ); m_command_lookup.set_handler( @@ -280,11 +293,27 @@ t_command_server::t_command_server( , std::bind(&t_command_parser_executor::sync_info, &m_parser, p::_1) , "Print information about the blockchain sync state." ); + m_command_lookup.set_handler( + "pop_blocks" + , std::bind(&t_command_parser_executor::pop_blocks, &m_parser, p::_1) + , "pop_blocks " + , "Remove blocks from end of blockchain" + ); m_command_lookup.set_handler( "version" , std::bind(&t_command_parser_executor::version, &m_parser, p::_1) , "Print version information." ); + m_command_lookup.set_handler( + "prune_blockchain" + , std::bind(&t_command_parser_executor::prune_blockchain, &m_parser, p::_1) + , "Prune the blockchain." + ); + m_command_lookup.set_handler( + "check_blockchain_pruning" + , std::bind(&t_command_parser_executor::check_blockchain_pruning, &m_parser, p::_1) + , "Check the blockchain pruning." + ); } bool t_command_server::process_command_str(const std::string& cmd) diff --git a/src/daemon/command_server.h b/src/daemon/command_server.h index aff74da45e7..da532223e56 100644 --- a/src/daemon/command_server.h +++ b/src/daemon/command_server.h @@ -9,7 +9,7 @@ Passing RPC commands: */ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -43,6 +43,7 @@ Passing RPC commands: #include "common/common_fwd.h" #include "console_handler.h" #include "daemon/command_parser_executor.h" +#include "net/net_fwd.h" namespace daemonize { @@ -57,6 +58,7 @@ class t_command_server { uint32_t ip , uint16_t port , const boost::optional& login + , const epee::net_utils::ssl_options_t& ssl_options , bool is_rpc = true , cryptonote::core_rpc_server* rpc_server = NULL ); diff --git a/src/daemon/core.h b/src/daemon/core.h index 475f418d650..91dbb7a4b38 100644 --- a/src/daemon/core.h +++ b/src/daemon/core.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -28,6 +28,7 @@ #pragma once +#include "blocks/blocks.h" #include "cryptonote_core/cryptonote_core.h" #include "cryptonote_protocol/cryptonote_protocol_handler.h" #include "misc_log_ex.h" @@ -66,26 +67,16 @@ class t_core final m_core.set_cryptonote_protocol(&protocol); } - std::string get_config_subdir() const - { - bool testnet = command_line::get_arg(m_vm_HACK, cryptonote::arg_testnet_on); - bool stagenet = command_line::get_arg(m_vm_HACK, cryptonote::arg_stagenet_on); - bool mainnet = !testnet && !stagenet; - std::string port = command_line::get_arg(m_vm_HACK, nodetool::arg_p2p_bind_port); - if ((mainnet && port != std::to_string(::config::P2P_DEFAULT_PORT)) - || (testnet && port != std::to_string(::config::testnet::P2P_DEFAULT_PORT)) - || (stagenet && port != std::to_string(::config::stagenet::P2P_DEFAULT_PORT))) { - return port; - } - return std::string(); - } - bool run() { //initialize core here MGINFO("Initializing core..."); - std::string config_subdir = get_config_subdir(); - if (!m_core.init(m_vm_HACK, config_subdir.empty() ? NULL : config_subdir.c_str())) +#if defined(PER_BLOCK_CHECKPOINT) + const cryptonote::GetCheckpointsCallback& get_checkpoints = blocks::GetCheckpointsData; +#else + const cryptonote::GetCheckpointsCallback& get_checkpoints = nullptr; +#endif + if (!m_core.init(m_vm_HACK, nullptr, get_checkpoints)) { return false; } diff --git a/src/daemon/daemon.cpp b/src/daemon/daemon.cpp index 48671f190c5..5084b628362 100644 --- a/src/daemon/daemon.cpp +++ b/src/daemon/daemon.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -45,6 +45,7 @@ #include "daemon/command_server.h" #include "daemon/command_server.h" #include "daemon/command_line_args.h" +#include "net/net_ssl.h" #include "version.h" using namespace epee; @@ -75,17 +76,15 @@ struct t_internals { protocol.set_p2p_endpoint(p2p.get()); core.set_protocol(protocol.get()); - const auto testnet = command_line::get_arg(vm, cryptonote::arg_testnet_on); - const auto stagenet = command_line::get_arg(vm, cryptonote::arg_stagenet_on); const auto restricted = command_line::get_arg(vm, cryptonote::core_rpc_server::arg_restricted_rpc); const auto main_rpc_port = command_line::get_arg(vm, cryptonote::core_rpc_server::arg_rpc_bind_port); - rpcs.emplace_back(new t_rpc{vm, core, p2p, restricted, testnet ? cryptonote::TESTNET : stagenet ? cryptonote::STAGENET : cryptonote::MAINNET, main_rpc_port, "core"}); + rpcs.emplace_back(new t_rpc{vm, core, p2p, restricted, main_rpc_port, "core"}); auto restricted_rpc_port_arg = cryptonote::core_rpc_server::arg_rpc_restricted_bind_port; if(!command_line::is_arg_defaulted(vm, restricted_rpc_port_arg)) { auto restricted_rpc_port = command_line::get_arg(vm, restricted_rpc_port_arg); - rpcs.emplace_back(new t_rpc{vm, core, p2p, true, testnet ? cryptonote::TESTNET : stagenet ? cryptonote::STAGENET : cryptonote::MAINNET, restricted_rpc_port, "restricted"}); + rpcs.emplace_back(new t_rpc{vm, core, p2p, true, restricted_rpc_port, "restricted"}); } } }; @@ -98,9 +97,11 @@ void t_daemon::init_options(boost::program_options::options_description & option } t_daemon::t_daemon( - boost::program_options::variables_map const & vm + boost::program_options::variables_map const & vm, + uint16_t public_rpc_port ) - : mp_internals{new t_internals{vm}} + : mp_internals{new t_internals{vm}}, + public_rpc_port(public_rpc_port) { zmq_rpc_bind_port = command_line::get_arg(vm, daemon_args::arg_zmq_rpc_bind_port); zmq_rpc_bind_address = command_line::get_arg(vm, daemon_args::arg_zmq_rpc_bind_ip); @@ -115,6 +116,7 @@ t_daemon::t_daemon(t_daemon && other) { mp_internals = std::move(other.mp_internals); other.mp_internals.reset(nullptr); + public_rpc_port = other.public_rpc_port; } } @@ -125,6 +127,7 @@ t_daemon & t_daemon::operator=(t_daemon && other) { mp_internals = std::move(other.mp_internals); other.mp_internals.reset(nullptr); + public_rpc_port = other.public_rpc_port; } return *this; } @@ -135,7 +138,19 @@ bool t_daemon::run(bool interactive) { throw std::runtime_error{"Can't run stopped daemon"}; } - tools::signal_handler::install(std::bind(&daemonize::t_daemon::stop_p2p, this)); + + std::atomic stop(false), shutdown(false); + boost::thread stop_thread = boost::thread([&stop, &shutdown, this] { + while (!stop) + epee::misc_utils::sleep_no_w(100); + if (shutdown) + this->stop_p2p(); + }); + epee::misc_utils::auto_scope_leave_caller scope_exit_handler = epee::misc_utils::create_scope_leave_handler([&](){ + stop = true; + stop_thread.join(); + }); + tools::signal_handler::install([&stop, &shutdown](int){ stop = shutdown = true; }); try { @@ -149,7 +164,7 @@ bool t_daemon::run(bool interactive) if (interactive && mp_internals->rpcs.size()) { // The first three variables are not used when the fourth is false - rpc_commands.reset(new daemonize::t_command_server(0, 0, boost::none, false, mp_internals->rpcs.front()->get_server())); + rpc_commands.reset(new daemonize::t_command_server(0, 0, boost::none, epee::net_utils::ssl_support_t::e_ssl_support_disabled, false, mp_internals->rpcs.front()->get_server())); rpc_commands->start_handling(std::bind(&daemonize::t_daemon::stop_p2p, this)); } @@ -176,6 +191,12 @@ bool t_daemon::run(bool interactive) MINFO(std::string("ZMQ server started at ") + zmq_rpc_bind_address + ":" + zmq_rpc_bind_port + "."); + if (public_rpc_port > 0) + { + MGINFO("Public RPC port " << public_rpc_port << " will be advertised to other peers over P2P"); + mp_internals->p2p.get().set_rpc_port(public_rpc_port); + } + mp_internals->p2p.run(); // blocks until p2p goes down if (rpc_commands) @@ -185,7 +206,6 @@ bool t_daemon::run(bool interactive) for(auto& rpc : mp_internals->rpcs) rpc->stop(); - mp_internals->core.get().get_miner().stop(); MGINFO("Node stopped."); return true; } @@ -207,7 +227,6 @@ void t_daemon::stop() { throw std::runtime_error{"Can't stop stopped daemon"}; } - mp_internals->core.get().get_miner().stop(); mp_internals->p2p.stop(); for(auto& rpc : mp_internals->rpcs) rpc->stop(); diff --git a/src/daemon/daemon.h b/src/daemon/daemon.h index 1e356ef5f22..d44173177eb 100644 --- a/src/daemon/daemon.h +++ b/src/daemon/daemon.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -43,11 +43,13 @@ class t_daemon final { void stop_p2p(); private: std::unique_ptr mp_internals; + uint16_t public_rpc_port; std::string zmq_rpc_bind_address; std::string zmq_rpc_bind_port; public: t_daemon( - boost::program_options::variables_map const & vm + boost::program_options::variables_map const & vm, + uint16_t public_rpc_port = 0 ); t_daemon(t_daemon && other); t_daemon & operator=(t_daemon && other); diff --git a/src/daemon/executor.cpp b/src/daemon/executor.cpp index fbc7d04fd69..3719a253d4a 100644 --- a/src/daemon/executor.cpp +++ b/src/daemon/executor.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -59,21 +59,21 @@ namespace daemonize ) { LOG_PRINT_L0("Monero '" << MONERO_RELEASE_NAME << "' (v" << MONERO_VERSION_FULL << ") Daemonised"); - return t_daemon{vm}; + return t_daemon{vm, public_rpc_port}; } bool t_executor::run_non_interactive( boost::program_options::variables_map const & vm ) { - return t_daemon{vm}.run(false); + return t_daemon{vm, public_rpc_port}.run(false); } bool t_executor::run_interactive( boost::program_options::variables_map const & vm ) { - return t_daemon{vm}.run(true); + return t_daemon{vm, public_rpc_port}.run(true); } } diff --git a/src/daemon/executor.h b/src/daemon/executor.h index 79d70567aae..61e4e1bbf1f 100644 --- a/src/daemon/executor.h +++ b/src/daemon/executor.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -45,6 +45,10 @@ namespace daemonize static std::string const NAME; + t_executor(uint16_t public_rpc_port = 0) : public_rpc_port(public_rpc_port) + { + } + static void init_options( boost::program_options::options_description & configurable_options ); @@ -62,5 +66,8 @@ namespace daemonize bool run_interactive( boost::program_options::variables_map const & vm ); + + private: + uint16_t public_rpc_port; }; } diff --git a/src/daemon/main.cpp b/src/daemon/main.cpp index 49494e8899c..690d4d60e58 100644 --- a/src/daemon/main.cpp +++ b/src/daemon/main.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -39,6 +39,7 @@ #include "daemon/executor.h" #include "daemonizer/daemonizer.h" #include "misc_log_ex.h" +#include "net/parse.h" #include "p2p/net_node.h" #include "rpc/core_rpc_server.h" #include "rpc/rpc_args.h" @@ -56,6 +57,57 @@ namespace po = boost::program_options; namespace bf = boost::filesystem; +uint16_t parse_public_rpc_port(const po::variables_map &vm) +{ + const auto &public_node_arg = daemon_args::arg_public_node; + const bool public_node = command_line::get_arg(vm, public_node_arg); + if (!public_node) + { + return 0; + } + + std::string rpc_port_str; + const auto &restricted_rpc_port = cryptonote::core_rpc_server::arg_rpc_restricted_bind_port; + if (!command_line::is_arg_defaulted(vm, restricted_rpc_port)) + { + rpc_port_str = command_line::get_arg(vm, restricted_rpc_port);; + } + else if (command_line::get_arg(vm, cryptonote::core_rpc_server::arg_restricted_rpc)) + { + rpc_port_str = command_line::get_arg(vm, cryptonote::core_rpc_server::arg_rpc_bind_port); + } + else + { + throw std::runtime_error("restricted RPC mode is required"); + } + + uint16_t rpc_port; + if (!string_tools::get_xtype_from_string(rpc_port, rpc_port_str)) + { + throw std::runtime_error("invalid RPC port " + rpc_port_str); + } + + const auto rpc_bind_address = command_line::get_arg(vm, cryptonote::rpc_args::descriptors().rpc_bind_ip); + const auto address = net::get_network_address(rpc_bind_address, rpc_port); + if (!address) { + throw std::runtime_error("failed to parse RPC bind address"); + } + if (address->get_zone() != epee::net_utils::zone::public_) + { + throw std::runtime_error(std::string(zone_to_string(address->get_zone())) + + " network zone is not supported, please check RPC server bind address"); + } + + if (address->is_loopback() || address->is_local()) + { + MLOG_RED(el::Level::Warning, "--" << public_node_arg.name + << " is enabled, but RPC server " << address->str() + << " may be unreachable from outside, please check RPC server bind address"); + } + + return rpc_port; +} + int main(int argc, char const * argv[]) { try { @@ -84,7 +136,9 @@ int main(int argc, char const * argv[]) command_line::add_arg(core_settings, daemon_args::arg_log_file); command_line::add_arg(core_settings, daemon_args::arg_log_level); command_line::add_arg(core_settings, daemon_args::arg_max_log_file_size); + command_line::add_arg(core_settings, daemon_args::arg_max_log_files); command_line::add_arg(core_settings, daemon_args::arg_max_concurrency); + command_line::add_arg(core_settings, daemon_args::arg_public_node); command_line::add_arg(core_settings, daemon_args::arg_zmq_rpc_bind_ip); command_line::add_arg(core_settings, daemon_args::arg_zmq_rpc_bind_port); @@ -162,9 +216,10 @@ int main(int argc, char const * argv[]) const bool testnet = command_line::get_arg(vm, cryptonote::arg_testnet_on); const bool stagenet = command_line::get_arg(vm, cryptonote::arg_stagenet_on); - if (testnet && stagenet) + const bool regtest = command_line::get_arg(vm, cryptonote::arg_regtest_on); + if (testnet + stagenet + regtest > 1) { - std::cerr << "Can't specify more than one of --tesnet and --stagenet" << ENDL; + std::cerr << "Can't specify more than one of --tesnet and --stagenet and --regtest" << ENDL; return 1; } @@ -202,8 +257,9 @@ int main(int argc, char const * argv[]) bf::path log_file_path {data_dir / std::string(CRYPTONOTE_NAME ".log")}; if (!command_line::is_arg_defaulted(vm, daemon_args::arg_log_file)) log_file_path = command_line::get_arg(vm, daemon_args::arg_log_file); - log_file_path = bf::absolute(log_file_path, relative_path_base); - mlog_configure(log_file_path.string(), true, command_line::get_arg(vm, daemon_args::arg_max_log_file_size)); + if (!log_file_path.has_parent_path()) + log_file_path = bf::absolute(log_file_path, relative_path_base); + mlog_configure(log_file_path.string(), true, command_line::get_arg(vm, daemon_args::arg_max_log_file_size), command_line::get_arg(vm, daemon_args::arg_max_log_files)); // Set log level if (!command_line::is_arg_defaulted(vm, daemon_args::arg_log_level)) @@ -214,6 +270,16 @@ int main(int argc, char const * argv[]) // after logs initialized tools::create_directories_if_necessary(data_dir.string()); +#ifdef STACK_TRACE + tools::set_stack_trace_log(log_file_path.filename().string()); +#endif // STACK_TRACE + + if (!command_line::is_arg_defaulted(vm, daemon_args::arg_max_concurrency)) + tools::set_max_concurrency(command_line::get_arg(vm, daemon_args::arg_max_concurrency)); + + // logging is now set up + MGINFO("Monero '" << MONERO_RELEASE_NAME << "' (v" << MONERO_VERSION_FULL << ")"); + // If there are positional options, we're running a daemon command { auto command = command_line::get_arg(vm, daemon_args::arg_command); @@ -237,11 +303,14 @@ int main(int argc, char const * argv[]) return 1; } + const char *env_rpc_login = nullptr; + const bool has_rpc_arg = command_line::has_arg(vm, arg.rpc_login); + const bool use_rpc_env = !has_rpc_arg && (env_rpc_login = getenv("RPC_LOGIN")) != nullptr && strlen(env_rpc_login) > 0; boost::optional login{}; - if (command_line::has_arg(vm, arg.rpc_login)) + if (has_rpc_arg || use_rpc_env) { login = tools::login::parse( - command_line::get_arg(vm, arg.rpc_login), false, [](bool verify) { + has_rpc_arg ? command_line::get_arg(vm, arg.rpc_login) : std::string(env_rpc_login), false, [](bool verify) { #ifdef HAVE_READLINE rdln::suspend_readline pause_readline; #endif @@ -255,32 +324,29 @@ int main(int argc, char const * argv[]) } } - daemonize::t_command_server rpc_commands{rpc_ip, rpc_port, std::move(login)}; + auto ssl_options = cryptonote::rpc_args::process_ssl(vm, true); + if (!ssl_options) + return 1; + + daemonize::t_command_server rpc_commands{rpc_ip, rpc_port, std::move(login), std::move(*ssl_options)}; if (rpc_commands.process_command_vec(command)) { return 0; } else { +#ifdef HAVE_READLINE + rdln::suspend_readline pause_readline; +#endif std::cerr << "Unknown command: " << command.front() << std::endl; return 1; } } } -#ifdef STACK_TRACE - tools::set_stack_trace_log(log_file_path.filename().string()); -#endif // STACK_TRACE - - if (!command_line::is_arg_defaulted(vm, daemon_args::arg_max_concurrency)) - tools::set_max_concurrency(command_line::get_arg(vm, daemon_args::arg_max_concurrency)); - - // logging is now set up - MGINFO("Monero '" << MONERO_RELEASE_NAME << "' (v" << MONERO_VERSION_FULL << ")"); - MINFO("Moving from main() into the daemonize now."); - return daemonizer::daemonize(argc, argv, daemonize::t_executor{}, vm) ? 0 : 1; + return daemonizer::daemonize(argc, argv, daemonize::t_executor{parse_public_rpc_port(vm)}, vm) ? 0 : 1; } catch (std::exception const & ex) { diff --git a/src/daemon/p2p.h b/src/daemon/p2p.h index 0f01c746df3..267b99c89a2 100644 --- a/src/daemon/p2p.h +++ b/src/daemon/p2p.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/daemon/protocol.h b/src/daemon/protocol.h index fd1d1b63862..51a2bce1fa0 100644 --- a/src/daemon/protocol.h +++ b/src/daemon/protocol.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/daemon/rpc.h b/src/daemon/rpc.h index 9621b0d0154..213593aa79d 100644 --- a/src/daemon/rpc.h +++ b/src/daemon/rpc.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -54,7 +54,6 @@ class t_rpc final , t_core & core , t_p2p & p2p , const bool restricted - , const cryptonote::network_type nettype , const std::string & port , const std::string & description ) @@ -62,7 +61,7 @@ class t_rpc final { MGINFO("Initializing " << m_description << " RPC server..."); - if (!m_server.init(vm, restricted, nettype, port)) + if (!m_server.init(vm, restricted, port)) { throw std::runtime_error("Failed to initialize " + m_description + " RPC server."); } diff --git a/src/daemon/rpc_command_executor.cpp b/src/daemon/rpc_command_executor.cpp index 48785344172..cca0f75f93e 100644 --- a/src/daemon/rpc_command_executor.cpp +++ b/src/daemon/rpc_command_executor.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -31,6 +31,7 @@ #include "string_tools.h" #include "common/password.h" #include "common/scoped_message_writer.h" +#include "common/pruning.h" #include "daemon/rpc_command_executor.h" #include "rpc/core_rpc_server_commands_defs.h" #include "cryptonote_core/cryptonote_core.h" @@ -60,13 +61,15 @@ namespace { peer_id_str >> id_str; epee::string_tools::xtype_to_string(peer.port, port_str); std::string addr_str = ip_str + ":" + port_str; - tools::msg_writer() << boost::format("%-10s %-25s %-25s %s") % prefix % id_str % addr_str % elapsed; + std::string rpc_port = peer.rpc_port ? std::to_string(peer.rpc_port) : "-"; + std::string pruning_seed = epee::string_tools::to_string_hex(peer.pruning_seed); + tools::msg_writer() << boost::format("%-10s %-25s %-25s %-5s %-4s %s") % prefix % id_str % addr_str % rpc_port % pruning_seed % elapsed; } void print_block_header(cryptonote::block_header_response const & header) { tools::success_msg_writer() - << "timestamp: " << boost::lexical_cast(header.timestamp) << std::endl + << "timestamp: " << boost::lexical_cast(header.timestamp) << " (" << tools::get_human_readable_timestamp(header.timestamp) << ")" << std::endl << "previous hash: " << header.prev_hash << std::endl << "nonce: " << boost::lexical_cast(header.nonce) << std::endl << "is orphan: " << header.orphan_status << std::endl @@ -75,7 +78,11 @@ namespace { << "hash: " << header.hash << std::endl << "difficulty: " << boost::lexical_cast(header.difficulty) << std::endl << "POW hash: " << header.pow_hash << std::endl - << "reward: " << boost::lexical_cast(header.reward); + << "block size: " << header.block_size << std::endl + << "block weight: " << header.block_weight << std::endl + << "long term weight: " << header.long_term_weight << std::endl + << "num txes: " << header.num_txes << std::endl + << "reward: " << cryptonote::print_money(header.reward); } std::string get_human_time_ago(time_t t, time_t now) @@ -120,6 +127,7 @@ t_rpc_command_executor::t_rpc_command_executor( uint32_t ip , uint16_t port , const boost::optional& login + , const epee::net_utils::ssl_options_t& ssl_options , bool is_rpc , cryptonote::core_rpc_server* rpc_server ) @@ -130,7 +138,7 @@ t_rpc_command_executor::t_rpc_command_executor( boost::optional http_login{}; if (login) http_login.emplace(login->username, login->password.password()); - m_rpc_client = new tools::t_rpc_client(ip, port, std::move(http_login)); + m_rpc_client = new tools::t_rpc_client(ip, port, std::move(http_login), ssl_options); } else { @@ -151,7 +159,7 @@ t_rpc_command_executor::~t_rpc_command_executor() } } -bool t_rpc_command_executor::print_peer_list() { +bool t_rpc_command_executor::print_peer_list(bool white, bool gray, size_t limit) { cryptonote::COMMAND_RPC_GET_PEER_LIST::request req; cryptonote::COMMAND_RPC_GET_PEER_LIST::response res; @@ -172,14 +180,24 @@ bool t_rpc_command_executor::print_peer_list() { } } - for (auto & peer : res.white_list) + if (white) { - print_peer("white", peer); + auto peer = res.white_list.cbegin(); + const auto end = limit ? peer + std::min(limit, res.white_list.size()) : res.white_list.cend(); + for (; peer != end; ++peer) + { + print_peer("white", *peer); + } } - for (auto & peer : res.gray_list) + if (gray) { - print_peer("gray", peer); + auto peer = res.gray_list.cbegin(); + const auto end = limit ? peer + std::min(limit, res.gray_list.size()) : res.gray_list.cend(); + for (; peer != end; ++peer) + { + print_peer("gray", *peer); + } } return true; @@ -439,25 +457,111 @@ bool t_rpc_command_executor::show_status() { } } - tools::success_msg_writer() << boost::format("Height: %llu/%llu (%.1f%%) on %s%s, %s, net hash %s, v%u%s, %s, %u(out)+%u(in) connections, uptime %ud %uh %um %us") + std::stringstream str; + str << boost::format("Height: %llu/%llu (%.1f%%) on %s%s, %s, net hash %s, v%u%s, %s, %u(out)+%u(in) connections") % (unsigned long long)ires.height % (unsigned long long)net_height % get_sync_percentage(ires) % (ires.testnet ? "testnet" : ires.stagenet ? "stagenet" : "mainnet") % bootstrap_msg - % (!has_mining_info ? "mining info unavailable" : mining_busy ? "syncing" : mres.active ? ( ( mres.is_background_mining_enabled ? "smart " : "" ) + std::string("mining at ") + get_mining_speed(mres.speed) ) : "not mining") + % (!has_mining_info ? "mining info unavailable" : mining_busy ? "syncing" : mres.active ? ( ( mres.is_background_mining_enabled ? "smart " : "" ) + std::string("mining at ") + get_mining_speed(mres.speed)) : "not mining") % get_mining_speed(ires.difficulty / ires.target) % (unsigned)hfres.version % get_fork_extra_info(hfres.earliest_height, net_height, ires.target) % (hfres.state == cryptonote::HardFork::Ready ? "up to date" : hfres.state == cryptonote::HardFork::UpdateNeeded ? "update needed" : "out of date, likely forked") % (unsigned)ires.outgoing_connections_count % (unsigned)ires.incoming_connections_count - % (unsigned int)floor(uptime / 60.0 / 60.0 / 24.0) - % (unsigned int)floor(fmod((uptime / 60.0 / 60.0), 24.0)) - % (unsigned int)floor(fmod((uptime / 60.0), 60.0)) - % (unsigned int)fmod(uptime, 60.0) ; + // restricted RPC does not disclose start time + if (ires.start_time) + { + str << boost::format(", uptime %ud %uh %um %us") + % (unsigned int)floor(uptime / 60.0 / 60.0 / 24.0) + % (unsigned int)floor(fmod((uptime / 60.0 / 60.0), 24.0)) + % (unsigned int)floor(fmod((uptime / 60.0), 60.0)) + % (unsigned int)fmod(uptime, 60.0) + ; + } + + tools::success_msg_writer() << str.str(); + + return true; +} + +bool t_rpc_command_executor::mining_status() { + cryptonote::COMMAND_RPC_MINING_STATUS::request mreq; + cryptonote::COMMAND_RPC_MINING_STATUS::response mres; + epee::json_rpc::error error_resp; + bool has_mining_info = true; + + std::string fail_message = "Problem fetching info"; + + bool mining_busy = false; + if (m_is_rpc) + { + // mining info is only available non unrestricted RPC mode + has_mining_info = m_rpc_client->rpc_request(mreq, mres, "/mining_status", fail_message.c_str()); + } + else + { + if (!m_rpc_server->on_mining_status(mreq, mres)) + { + tools::fail_msg_writer() << fail_message.c_str(); + return true; + } + + if (mres.status == CORE_RPC_STATUS_BUSY) + { + mining_busy = true; + } + else if (mres.status != CORE_RPC_STATUS_OK) + { + tools::fail_msg_writer() << make_error(fail_message, mres.status); + return true; + } + } + + if (!has_mining_info) + { + tools::fail_msg_writer() << "Mining info unavailable"; + return true; + } + + if (mining_busy || !mres.active) + { + tools::msg_writer() << "Not currently mining"; + } + else + { + tools::msg_writer() << "Mining at " << get_mining_speed(mres.speed) << " with " << mres.threads_count << " threads"; + } + + if (mres.active || mres.is_background_mining_enabled) + { + tools::msg_writer() << "PoW algorithm: " << mres.pow_algorithm; + tools::msg_writer() << "Mining address: " << mres.address; + } + + if (mres.is_background_mining_enabled) + { + tools::msg_writer() << "Smart mining enabled:"; + tools::msg_writer() << " Target: " << (unsigned)mres.bg_target << "% CPU"; + tools::msg_writer() << " Idle threshold: " << (unsigned)mres.bg_idle_threshold << "% CPU"; + tools::msg_writer() << " Min idle time: " << (unsigned)mres.bg_min_idle_seconds << " seconds"; + tools::msg_writer() << " Ignore battery: " << (mres.bg_ignore_battery ? "yes" : "no"); + } + + if (!mining_busy && mres.active && mres.speed > 0 && mres.block_target > 0 && mres.difficulty > 0) + { + double ratio = mres.speed * mres.block_target / (double)mres.difficulty; + uint64_t daily = 86400ull / mres.block_target * mres.block_reward * ratio; + uint64_t monthly = 86400ull / mres.block_target * 30.5 * mres.block_reward * ratio; + uint64_t yearly = 86400ull / mres.block_target * 356 * mres.block_reward * ratio; + tools::msg_writer() << "Expected: " << cryptonote::print_money(daily) << " monero daily, " + << cryptonote::print_money(monthly) << " monero monthly, " << cryptonote::print_money(yearly) << " yearly"; + } + return true; } @@ -485,6 +589,7 @@ bool t_rpc_command_executor::print_connections() { } tools::msg_writer() << std::setw(30) << std::left << "Remote Host" + << std::setw(6) << "SSL" << std::setw(20) << "Peer id" << std::setw(20) << "Support Flags" << std::setw(30) << "Recv/Sent (inactive,sec)" @@ -504,6 +609,7 @@ bool t_rpc_command_executor::print_connections() { tools::msg_writer() //<< std::setw(30) << std::left << in_out << std::setw(30) << std::left << address + << std::setw(6) << (info.ssl ? "yes" : "no") << std::setw(20) << epee::string_tools::pad_string(info.peer_id, 16, '0', true) << std::setw(20) << info.support_flags << std::setw(30) << std::to_string(info.recv_count) + "(" + std::to_string(info.recv_idle_time) + ")/" + std::to_string(info.send_count) + "(" + std::to_string(info.send_idle_time) + ")" @@ -523,6 +629,66 @@ bool t_rpc_command_executor::print_connections() { return true; } +bool t_rpc_command_executor::print_net_stats() +{ + cryptonote::COMMAND_RPC_GET_NET_STATS::request net_stats_req; + cryptonote::COMMAND_RPC_GET_NET_STATS::response net_stats_res; + cryptonote::COMMAND_RPC_GET_LIMIT::request limit_req; + cryptonote::COMMAND_RPC_GET_LIMIT::response limit_res; + + std::string fail_message = "Unsuccessful"; + + if (m_is_rpc) + { + if (!m_rpc_client->json_rpc_request(net_stats_req, net_stats_res, "get_net_stats", fail_message.c_str())) + { + return true; + } + if (!m_rpc_client->json_rpc_request(limit_req, limit_res, "get_limit", fail_message.c_str())) + { + return true; + } + } + else + { + if (!m_rpc_server->on_get_net_stats(net_stats_req, net_stats_res) || net_stats_res.status != CORE_RPC_STATUS_OK) + { + tools::fail_msg_writer() << make_error(fail_message, net_stats_res.status); + return true; + } + if (!m_rpc_server->on_get_limit(limit_req, limit_res) || limit_res.status != CORE_RPC_STATUS_OK) + { + tools::fail_msg_writer() << make_error(fail_message, limit_res.status); + return true; + } + } + + uint64_t seconds = (uint64_t)time(NULL) - net_stats_res.start_time; + uint64_t average = seconds > 0 ? net_stats_res.total_bytes_in / seconds : 0; + uint64_t limit = limit_res.limit_down * 1024; // convert to bytes, as limits are always kB/s + double percent = (double)average / (double)limit * 100.0; + tools::success_msg_writer() << boost::format("Received %u bytes (%s) in %u packets, average %s/s = %.2f%% of the limit of %s/s") + % net_stats_res.total_bytes_in + % tools::get_human_readable_bytes(net_stats_res.total_bytes_in) + % net_stats_res.total_packets_in + % tools::get_human_readable_bytes(average) + % percent + % tools::get_human_readable_bytes(limit); + + average = seconds > 0 ? net_stats_res.total_bytes_out / seconds : 0; + limit = limit_res.limit_up * 1024; + percent = (double)average / (double)limit * 100.0; + tools::success_msg_writer() << boost::format("Sent %u bytes (%s) in %u packets, average %s/s = %.2f%% of the limit of %s/s") + % net_stats_res.total_bytes_out + % tools::get_human_readable_bytes(net_stats_res.total_bytes_out) + % net_stats_res.total_packets_out + % tools::get_human_readable_bytes(average) + % percent + % tools::get_human_readable_bytes(limit); + + return true; +} + bool t_rpc_command_executor::print_blockchain_info(uint64_t start_block_index, uint64_t end_block_index) { cryptonote::COMMAND_RPC_GET_BLOCK_HEADERS_RANGE::request req; cryptonote::COMMAND_RPC_GET_BLOCK_HEADERS_RANGE::response res; @@ -530,6 +696,7 @@ bool t_rpc_command_executor::print_blockchain_info(uint64_t start_block_index, u req.start_height = start_block_index; req.end_height = end_block_index; + req.fill_pow_hash = false; std::string fail_message = "Unsuccessful"; @@ -553,10 +720,10 @@ bool t_rpc_command_executor::print_blockchain_info(uint64_t start_block_index, u for (auto & header : res.headers) { if (!first) - std::cout << std::endl; - std::cout - << "height: " << header.height << ", timestamp: " << header.timestamp << ", difficulty: " << header.difficulty - << ", size: " << header.block_size << ", transactions: " << header.num_txes << std::endl + tools::msg_writer() << "" << std::endl; + tools::msg_writer() + << "height: " << header.height << ", timestamp: " << header.timestamp << " (" << tools::get_human_readable_timestamp(header.timestamp) << ")" + << ", size: " << header.block_size << ", weight: " << header.block_weight << " (long term " << header.long_term_weight << "), transactions: " << header.num_txes << std::endl << "major version: " << (unsigned)header.major_version << ", minor version: " << (unsigned)header.minor_version << std::endl << "block id: " << header.hash << ", previous block id: " << header.prev_hash << std::endl << "difficulty: " << header.difficulty << ", nonce " << header.nonce << ", reward " << cryptonote::print_money(header.reward) << std::endl; @@ -649,7 +816,7 @@ bool t_rpc_command_executor::print_height() { return true; } -bool t_rpc_command_executor::print_block_by_hash(crypto::hash block_hash) { +bool t_rpc_command_executor::print_block_by_hash(crypto::hash block_hash, bool include_hex) { cryptonote::COMMAND_RPC_GET_BLOCK::request req; cryptonote::COMMAND_RPC_GET_BLOCK::response res; epee::json_rpc::error error_resp; @@ -675,13 +842,15 @@ bool t_rpc_command_executor::print_block_by_hash(crypto::hash block_hash) { } } + if (include_hex) + tools::success_msg_writer() << res.blob << std::endl; print_block_header(res.block_header); tools::success_msg_writer() << res.json << ENDL; return true; } -bool t_rpc_command_executor::print_block_by_height(uint64_t height) { +bool t_rpc_command_executor::print_block_by_height(uint64_t height, bool include_hex) { cryptonote::COMMAND_RPC_GET_BLOCK::request req; cryptonote::COMMAND_RPC_GET_BLOCK::response res; epee::json_rpc::error error_resp; @@ -707,6 +876,8 @@ bool t_rpc_command_executor::print_block_by_height(uint64_t height) { } } + if (include_hex) + tools::success_msg_writer() << res.blob << std::endl; print_block_header(res.block_header); tools::success_msg_writer() << res.json << ENDL; @@ -723,6 +894,7 @@ bool t_rpc_command_executor::print_transaction(crypto::hash transaction_hash, req.txs_hashes.push_back(epee::string_tools::pod_to_hex(transaction_hash)); req.decode_as_json = false; + req.split = true; req.prune = false; if (m_is_rpc) { @@ -748,13 +920,25 @@ bool t_rpc_command_executor::print_transaction(crypto::hash transaction_hash, if (res.txs.front().in_pool) tools::success_msg_writer() << "Found in pool"; else - tools::success_msg_writer() << "Found in blockchain at height " << res.txs.front().block_height; + tools::success_msg_writer() << "Found in blockchain at height " << res.txs.front().block_height << (res.txs.front().prunable_as_hex.empty() ? " (pruned)" : ""); } const std::string &as_hex = (1 == res.txs.size()) ? res.txs.front().as_hex : res.txs_as_hex.front(); + const std::string &pruned_as_hex = (1 == res.txs.size()) ? res.txs.front().pruned_as_hex : ""; + const std::string &prunable_as_hex = (1 == res.txs.size()) ? res.txs.front().prunable_as_hex : ""; // Print raw hex if requested if (include_hex) - tools::success_msg_writer() << as_hex << std::endl; + { + if (!as_hex.empty()) + { + tools::success_msg_writer() << as_hex << std::endl; + } + else + { + std::string output = pruned_as_hex + prunable_as_hex; + tools::success_msg_writer() << output << std::endl; + } + } // Print json if requested if (include_json) @@ -762,17 +946,27 @@ bool t_rpc_command_executor::print_transaction(crypto::hash transaction_hash, crypto::hash tx_hash, tx_prefix_hash; cryptonote::transaction tx; cryptonote::blobdata blob; - if (!string_tools::parse_hexstr_to_binbuff(as_hex, blob)) + std::string source = as_hex.empty() ? pruned_as_hex + prunable_as_hex : as_hex; + bool pruned = !pruned_as_hex.empty() && prunable_as_hex.empty(); + if (!string_tools::parse_hexstr_to_binbuff(source, blob)) { tools::fail_msg_writer() << "Failed to parse tx to get json format"; } - else if (!cryptonote::parse_and_validate_tx_from_blob(blob, tx, tx_hash, tx_prefix_hash)) - { - tools::fail_msg_writer() << "Failed to parse tx blob to get json format"; - } else { - tools::success_msg_writer() << cryptonote::obj_to_json_str(tx) << std::endl; + bool ret; + if (pruned) + ret = cryptonote::parse_and_validate_tx_base_from_blob(blob, tx); + else + ret = cryptonote::parse_and_validate_tx_from_blob(blob, tx); + if (!ret) + { + tools::fail_msg_writer() << "Failed to parse tx blob to get json format"; + } + else + { + tools::success_msg_writer() << cryptonote::obj_to_json_str(tx) << std::endl; + } } } } @@ -835,7 +1029,7 @@ bool t_rpc_command_executor::print_transaction_pool_long() { } else { - if (!m_rpc_server->on_get_transaction_pool(req, res, false) || res.status != CORE_RPC_STATUS_OK) + if (!m_rpc_server->on_get_transaction_pool(req, res) || res.status != CORE_RPC_STATUS_OK) { tools::fail_msg_writer() << make_error(fail_message, res.status); return true; @@ -855,8 +1049,9 @@ bool t_rpc_command_executor::print_transaction_pool_long() { tools::msg_writer() << "id: " << tx_info.id_hash << std::endl << tx_info.tx_json << std::endl << "blob_size: " << tx_info.blob_size << std::endl + << "weight: " << tx_info.weight << std::endl << "fee: " << cryptonote::print_money(tx_info.fee) << std::endl - << "fee/byte: " << cryptonote::print_money(tx_info.fee / (double)tx_info.blob_size) << std::endl + << "fee/byte: " << cryptonote::print_money(tx_info.fee / (double)tx_info.weight) << std::endl << "receive_time: " << tx_info.receive_time << " (" << get_human_time_ago(tx_info.receive_time, now) << ")" << std::endl << "relayed: " << [&](const cryptonote::tx_info &tx_info)->std::string { if (!tx_info.relayed) return "no"; return boost::lexical_cast(tx_info.last_relayed_time) + " (" + get_human_time_ago(tx_info.last_relayed_time, now) + ")"; } (tx_info) << std::endl << "do_not_relay: " << (tx_info.do_not_relay ? 'T' : 'F') << std::endl @@ -920,7 +1115,7 @@ bool t_rpc_command_executor::print_transaction_pool_short() { } else { - if (!m_rpc_server->on_get_transaction_pool(req, res, false) || res.status != CORE_RPC_STATUS_OK) + if (!m_rpc_server->on_get_transaction_pool(req, res) || res.status != CORE_RPC_STATUS_OK) { tools::fail_msg_writer() << make_error(fail_message, res.status); return true; @@ -938,8 +1133,9 @@ bool t_rpc_command_executor::print_transaction_pool_short() { { tools::msg_writer() << "id: " << tx_info.id_hash << std::endl << "blob_size: " << tx_info.blob_size << std::endl + << "weight: " << tx_info.weight << std::endl << "fee: " << cryptonote::print_money(tx_info.fee) << std::endl - << "fee/byte: " << cryptonote::print_money(tx_info.fee / (double)tx_info.blob_size) << std::endl + << "fee/byte: " << cryptonote::print_money(tx_info.fee / (double)tx_info.weight) << std::endl << "receive_time: " << tx_info.receive_time << " (" << get_human_time_ago(tx_info.receive_time, now) << ")" << std::endl << "relayed: " << [&](const cryptonote::tx_info &tx_info)->std::string { if (!tx_info.relayed) return "no"; return boost::lexical_cast(tx_info.last_relayed_time) + " (" + get_human_time_ago(tx_info.last_relayed_time, now) + ")"; } (tx_info) << std::endl << "do_not_relay: " << (tx_info.do_not_relay ? 'T' : 'F') << std::endl @@ -976,8 +1172,8 @@ bool t_rpc_command_executor::print_transaction_pool_stats() { } else { - memset(&res.pool_stats, 0, sizeof(res.pool_stats)); - if (!m_rpc_server->on_get_transaction_pool_stats(req, res, false) || res.status != CORE_RPC_STATUS_OK) + res.pool_stats = {}; + if (!m_rpc_server->on_get_transaction_pool_stats(req, res) || res.status != CORE_RPC_STATUS_OK) { tools::fail_msg_writer() << make_error(fail_message, res.status); return true; @@ -994,7 +1190,7 @@ bool t_rpc_command_executor::print_transaction_pool_stats() { size_t avg_bytes = n_transactions ? res.pool_stats.bytes_total / n_transactions : 0; std::string backlog_message; - const uint64_t full_reward_zone = ires.block_size_limit / 2; + const uint64_t full_reward_zone = ires.block_weight_limit / 2; if (res.pool_stats.bytes_total <= full_reward_zone) { backlog_message = "no backlog"; @@ -1297,7 +1493,7 @@ bool t_rpc_command_executor::out_peers(uint64_t limit) } } - std::cout << "Max number of out peers set to " << limit << std::endl; + tools::msg_writer() << "Max number of out peers set to " << limit << std::endl; return true; } @@ -1329,7 +1525,7 @@ bool t_rpc_command_executor::in_peers(uint64_t limit) } } - std::cout << "Max number of in peers set to " << limit << std::endl; + tools::msg_writer() << "Max number of in peers set to " << limit << std::endl; return true; } @@ -1628,7 +1824,7 @@ bool t_rpc_command_executor::print_coinbase_tx_sum(uint64_t height, uint64_t cou return true; } -bool t_rpc_command_executor::alt_chain_info() +bool t_rpc_command_executor::alt_chain_info(const std::string &tip) { cryptonote::COMMAND_RPC_GET_INFO::request ireq; cryptonote::COMMAND_RPC_GET_INFO::response ires; @@ -1663,12 +1859,32 @@ bool t_rpc_command_executor::alt_chain_info() } } - tools::msg_writer() << boost::lexical_cast(res.chains.size()) << " alternate chains found:"; - for (const auto &chain: res.chains) + if (tip.empty()) + { + tools::msg_writer() << boost::lexical_cast(res.chains.size()) << " alternate chains found:"; + for (const auto &chain: res.chains) + { + uint64_t start_height = (chain.height - chain.length + 1); + tools::msg_writer() << chain.length << " blocks long, from height " << start_height << " (" << (ires.height - start_height - 1) + << " deep), diff " << chain.difficulty << ": " << chain.block_hash; + } + } + else { - uint64_t start_height = (chain.height - chain.length + 1); - tools::msg_writer() << chain.length << " blocks long, from height " << start_height << " (" << (ires.height - start_height - 1) - << " deep), diff " << chain.difficulty << ": " << chain.block_hash; + const auto i = std::find_if(res.chains.begin(), res.chains.end(), [&tip](cryptonote::COMMAND_RPC_GET_ALTERNATE_CHAINS::chain_info &info){ return info.block_hash == tip; }); + if (i != res.chains.end()) + { + const auto &chain = *i; + tools::success_msg_writer() << "Found alternate chain with tip " << tip; + uint64_t start_height = (chain.height - chain.length + 1); + tools::msg_writer() << chain.length << " blocks long, from height " << start_height << " (" << (ires.height - start_height - 1) + << " deep), diff " << chain.difficulty << ":"; + for (const std::string &block_id: chain.block_hashes) + tools::msg_writer() << " " << block_id; + tools::msg_writer() << "Chain parent on main chain: " << chain.main_chain_parent_block; + } + else + tools::fail_msg_writer() << "Block hash " << tip << " is not the tip of any known alternate chain"; } return true; } @@ -1679,13 +1895,16 @@ bool t_rpc_command_executor::print_blockchain_dynamic_stats(uint64_t nblocks) cryptonote::COMMAND_RPC_GET_INFO::response ires; cryptonote::COMMAND_RPC_GET_BLOCK_HEADERS_RANGE::request bhreq; cryptonote::COMMAND_RPC_GET_BLOCK_HEADERS_RANGE::response bhres; - cryptonote::COMMAND_RPC_GET_PER_KB_FEE_ESTIMATE::request fereq; - cryptonote::COMMAND_RPC_GET_PER_KB_FEE_ESTIMATE::response feres; + cryptonote::COMMAND_RPC_GET_BASE_FEE_ESTIMATE::request fereq; + cryptonote::COMMAND_RPC_GET_BASE_FEE_ESTIMATE::response feres; + cryptonote::COMMAND_RPC_HARD_FORK_INFO::request hfreq; + cryptonote::COMMAND_RPC_HARD_FORK_INFO::response hfres; epee::json_rpc::error error_resp; std::string fail_message = "Problem fetching info"; fereq.grace_blocks = 0; + hfreq.version = HF_VERSION_PER_BYTE_FEE; if (m_is_rpc) { if (!m_rpc_client->rpc_request(ireq, ires, "/getinfo", fail_message.c_str())) @@ -1696,6 +1915,10 @@ bool t_rpc_command_executor::print_blockchain_dynamic_stats(uint64_t nblocks) { return true; } + if (!m_rpc_client->json_rpc_request(hfreq, hfres, "hard_fork_info", fail_message.c_str())) + { + return true; + } } else { @@ -1704,15 +1927,20 @@ bool t_rpc_command_executor::print_blockchain_dynamic_stats(uint64_t nblocks) tools::fail_msg_writer() << make_error(fail_message, ires.status); return true; } - if (!m_rpc_server->on_get_per_kb_fee_estimate(fereq, feres, error_resp) || feres.status != CORE_RPC_STATUS_OK) + if (!m_rpc_server->on_get_base_fee_estimate(fereq, feres, error_resp) || feres.status != CORE_RPC_STATUS_OK) { tools::fail_msg_writer() << make_error(fail_message, feres.status); return true; } + if (!m_rpc_server->on_hard_fork_info(hfreq, hfres, error_resp) || hfres.status != CORE_RPC_STATUS_OK) + { + tools::fail_msg_writer() << make_error(fail_message, hfres.status); + return true; + } } tools::msg_writer() << "Height: " << ires.height << ", diff " << ires.difficulty << ", cum. diff " << ires.cumulative_difficulty - << ", target " << ires.target << " sec" << ", dyn fee " << cryptonote::print_money(feres.fee) << "/kB"; + << ", target " << ires.target << " sec" << ", dyn fee " << cryptonote::print_money(feres.fee) << "/" << (hfres.enabled ? "byte" : "kB"); if (nblocks > 0) { @@ -1721,6 +1949,7 @@ bool t_rpc_command_executor::print_blockchain_dynamic_stats(uint64_t nblocks) bhreq.start_height = ires.height - nblocks; bhreq.end_height = ires.height - 1; + bhreq.fill_pow_hash = false; if (m_is_rpc) { if (!m_rpc_client->json_rpc_request(bhreq, bhres, "getblockheadersrange", fail_message.c_str())) @@ -1740,8 +1969,8 @@ bool t_rpc_command_executor::print_blockchain_dynamic_stats(uint64_t nblocks) double avgdiff = 0; double avgnumtxes = 0; double avgreward = 0; - std::vector sizes; - sizes.reserve(nblocks); + std::vector weights; + weights.reserve(nblocks); uint64_t earliest = std::numeric_limits::max(), latest = 0; std::vector major_versions(256, 0), minor_versions(256, 0); for (const auto &bhr: bhres.headers) @@ -1749,7 +1978,7 @@ bool t_rpc_command_executor::print_blockchain_dynamic_stats(uint64_t nblocks) avgdiff += bhr.difficulty; avgnumtxes += bhr.num_txes; avgreward += bhr.reward; - sizes.push_back(bhr.block_size); + weights.push_back(bhr.block_weight); static_assert(sizeof(bhr.major_version) == 1, "major_version expected to be uint8_t"); static_assert(sizeof(bhr.minor_version) == 1, "major_version expected to be uint8_t"); major_versions[(unsigned)bhr.major_version]++; @@ -1760,9 +1989,9 @@ bool t_rpc_command_executor::print_blockchain_dynamic_stats(uint64_t nblocks) avgdiff /= nblocks; avgnumtxes /= nblocks; avgreward /= nblocks; - uint64_t median_block_size = epee::misc_utils::median(sizes); + uint64_t median_block_weight = epee::misc_utils::median(weights); tools::msg_writer() << "Last " << nblocks << ": avg. diff " << (uint64_t)avgdiff << ", " << (latest - earliest) / nblocks << " avg sec/block, avg num txes " << avgnumtxes - << ", avg. reward " << cryptonote::print_money(avgreward) << ", median block size " << median_block_size; + << ", avg. reward " << cryptonote::print_money(avgreward) << ", median block weight " << median_block_weight; unsigned int max_major = 256, max_minor = 256; while (max_major > 0 && !major_versions[--max_major]); @@ -1886,6 +2115,8 @@ bool t_rpc_command_executor::sync_info() for (const auto &p: res.peers) current_download += p.info.current_download; tools::success_msg_writer() << "Downloading at " << current_download << " kB/s"; + if (res.next_needed_pruning_seed) + tools::success_msg_writer() << "Next needed pruning seed: " << res.next_needed_pruning_seed; tools::success_msg_writer() << std::to_string(res.peers.size()) << " peers"; for (const auto &p: res.peers) @@ -1893,29 +2124,126 @@ bool t_rpc_command_executor::sync_info() std::string address = epee::string_tools::pad_string(p.info.address, 24); uint64_t nblocks = 0, size = 0; for (const auto &s: res.spans) - if (s.rate > 0.0f && s.connection_id == p.info.connection_id) + if (s.connection_id == p.info.connection_id) nblocks += s.nblocks, size += s.size; - tools::success_msg_writer() << address << " " << epee::string_tools::pad_string(p.info.peer_id, 16, '0', true) << " " << p.info.height << " " << p.info.current_download << " kB/s, " << nblocks << " blocks / " << size/1e6 << " MB queued"; + tools::success_msg_writer() << address << " " << epee::string_tools::pad_string(p.info.peer_id, 16, '0', true) << " " << + epee::string_tools::pad_string(p.info.state, 16) << " " << + epee::string_tools::pad_string(epee::string_tools::to_string_hex(p.info.pruning_seed), 8) << " " << p.info.height << " " << + p.info.current_download << " kB/s, " << nblocks << " blocks / " << size/1e6 << " MB queued"; } uint64_t total_size = 0; for (const auto &s: res.spans) total_size += s.size; tools::success_msg_writer() << std::to_string(res.spans.size()) << " spans, " << total_size/1e6 << " MB"; + tools::success_msg_writer() << res.overview; for (const auto &s: res.spans) { std::string address = epee::string_tools::pad_string(s.remote_address, 24); + std::string pruning_seed = epee::string_tools::to_string_hex(tools::get_pruning_seed(s.start_block_height, std::numeric_limits::max(), CRYPTONOTE_PRUNING_LOG_STRIPES)); if (s.size == 0) { - tools::success_msg_writer() << address << " " << s.nblocks << " (" << s.start_block_height << " - " << (s.start_block_height + s.nblocks - 1) << ") -"; + tools::success_msg_writer() << address << " " << s.nblocks << "/" << pruning_seed << " (" << s.start_block_height << " - " << (s.start_block_height + s.nblocks - 1) << ") -"; } else { - tools::success_msg_writer() << address << " " << s.nblocks << " (" << s.start_block_height << " - " << (s.start_block_height + s.nblocks - 1) << ", " << (uint64_t)(s.size/1e3) << " kB) " << (unsigned)(s.rate/1e3) << " kB/s (" << s.speed/100.0f << ")"; + tools::success_msg_writer() << address << " " << s.nblocks << "/" << pruning_seed << " (" << s.start_block_height << " - " << (s.start_block_height + s.nblocks - 1) << ", " << (uint64_t)(s.size/1e3) << " kB) " << (unsigned)(s.rate/1e3) << " kB/s (" << s.speed/100.0f << ")"; } } return true; } +bool t_rpc_command_executor::pop_blocks(uint64_t num_blocks) +{ + cryptonote::COMMAND_RPC_POP_BLOCKS::request req; + cryptonote::COMMAND_RPC_POP_BLOCKS::response res; + std::string fail_message = "pop_blocks failed"; + + req.nblocks = num_blocks; + if (m_is_rpc) + { + if (!m_rpc_client->rpc_request(req, res, "/pop_blocks", fail_message.c_str())) + { + return true; + } + } + else + { + if (!m_rpc_server->on_pop_blocks(req, res) || res.status != CORE_RPC_STATUS_OK) + { + tools::fail_msg_writer() << make_error(fail_message, res.status); + return true; + } + } + tools::success_msg_writer() << "new height: " << res.height; + + return true; +} + +bool t_rpc_command_executor::prune_blockchain() +{ + cryptonote::COMMAND_RPC_PRUNE_BLOCKCHAIN::request req; + cryptonote::COMMAND_RPC_PRUNE_BLOCKCHAIN::response res; + std::string fail_message = "Unsuccessful"; + epee::json_rpc::error error_resp; + + req.check = false; + + if (m_is_rpc) + { + if (!m_rpc_client->json_rpc_request(req, res, "prune_blockchain", fail_message.c_str())) + { + return true; + } + } + else + { + if (!m_rpc_server->on_prune_blockchain(req, res, error_resp) || res.status != CORE_RPC_STATUS_OK) + { + tools::fail_msg_writer() << make_error(fail_message, res.status); + return true; + } + } + + tools::success_msg_writer() << "Blockchain pruned"; + return true; +} + +bool t_rpc_command_executor::check_blockchain_pruning() +{ + cryptonote::COMMAND_RPC_PRUNE_BLOCKCHAIN::request req; + cryptonote::COMMAND_RPC_PRUNE_BLOCKCHAIN::response res; + std::string fail_message = "Unsuccessful"; + epee::json_rpc::error error_resp; + + req.check = true; + + if (m_is_rpc) + { + if (!m_rpc_client->json_rpc_request(req, res, "prune_blockchain", fail_message.c_str())) + { + return true; + } + } + else + { + if (!m_rpc_server->on_prune_blockchain(req, res, error_resp) || res.status != CORE_RPC_STATUS_OK) + { + tools::fail_msg_writer() << make_error(fail_message, res.status); + return true; + } + } + + if (res.pruning_seed) + { + tools::success_msg_writer() << "Blockchain is pruned"; + } + else + { + tools::success_msg_writer() << "Blockchain is not pruned"; + } + return true; +} + }// namespace daemonize diff --git a/src/daemon/rpc_command_executor.h b/src/daemon/rpc_command_executor.h index 46168c93b9e..df2894d0937 100644 --- a/src/daemon/rpc_command_executor.h +++ b/src/daemon/rpc_command_executor.h @@ -6,7 +6,7 @@ */ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -43,6 +43,7 @@ #include "common/common_fwd.h" #include "common/rpc_client.h" #include "cryptonote_basic/cryptonote_basic.h" +#include "net/net_fwd.h" #include "rpc/core_rpc_server.h" #undef MONERO_DEFAULT_LOG_CATEGORY @@ -61,13 +62,14 @@ class t_rpc_command_executor final { uint32_t ip , uint16_t port , const boost::optional& user + , const epee::net_utils::ssl_options_t& ssl_options , bool is_rpc = true , cryptonote::core_rpc_server* rpc_server = NULL ); ~t_rpc_command_executor(); - bool print_peer_list(); + bool print_peer_list(bool white = true, bool gray = true, size_t limit = 0); bool print_peer_list_stats(); @@ -91,9 +93,9 @@ class t_rpc_command_executor final { bool print_height(); - bool print_block_by_hash(crypto::hash block_hash); + bool print_block_by_hash(crypto::hash block_hash, bool include_hex); - bool print_block_by_height(uint64_t height); + bool print_block_by_height(uint64_t height, bool include_hex); bool print_transaction(crypto::hash transaction_hash, bool include_hex, bool include_json); @@ -109,6 +111,8 @@ class t_rpc_command_executor final { bool stop_mining(); + bool mining_status(); + bool stop_daemon(); bool print_status(); @@ -143,7 +147,7 @@ class t_rpc_command_executor final { bool print_coinbase_tx_sum(uint64_t height, uint64_t count); - bool alt_chain_info(); + bool alt_chain_info(const std::string &tip); bool print_blockchain_dynamic_stats(uint64_t nblocks); @@ -152,6 +156,14 @@ class t_rpc_command_executor final { bool relay_tx(const std::string &txid); bool sync_info(); + + bool pop_blocks(uint64_t num_blocks); + + bool prune_blockchain(); + + bool check_blockchain_pruning(); + + bool print_net_stats(); }; } // namespace daemonize diff --git a/src/daemonizer/CMakeLists.txt b/src/daemonizer/CMakeLists.txt index 2753d0003c1..d540adc10d4 100644 --- a/src/daemonizer/CMakeLists.txt +++ b/src/daemonizer/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2014-2018, The Monero Project +# Copyright (c) 2014-2019, The Monero Project # # All rights reserved. # diff --git a/src/daemonizer/daemonizer.h b/src/daemonizer/daemonizer.h index c5852b59cf4..a45a2b44fc7 100644 --- a/src/daemonizer/daemonizer.h +++ b/src/daemonizer/daemonizer.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/daemonizer/posix_daemonizer.inl b/src/daemonizer/posix_daemonizer.inl index b3f3f262f3e..9ec09c67858 100644 --- a/src/daemonizer/posix_daemonizer.inl +++ b/src/daemonizer/posix_daemonizer.inl @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/daemonizer/posix_fork.cpp b/src/daemonizer/posix_fork.cpp index 3cbee9c5147..5af4e1a4a3b 100644 --- a/src/daemonizer/posix_fork.cpp +++ b/src/daemonizer/posix_fork.cpp @@ -12,7 +12,6 @@ #include #include #include -#include #ifndef TMPDIR #define TMPDIR "/tmp" diff --git a/src/daemonizer/posix_fork.h b/src/daemonizer/posix_fork.h index 9294b00e244..27b4ac18d02 100644 --- a/src/daemonizer/posix_fork.h +++ b/src/daemonizer/posix_fork.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/daemonizer/windows_daemonizer.inl b/src/daemonizer/windows_daemonizer.inl index 8077f29fb60..c6cd474fd4a 100644 --- a/src/daemonizer/windows_daemonizer.inl +++ b/src/daemonizer/windows_daemonizer.inl @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -31,6 +31,7 @@ #include "common/util.h" #include "daemonizer/windows_service.h" #include "daemonizer/windows_service_runner.h" +#include "cryptonote_core/cryptonote_core.h" #include #include @@ -60,6 +61,10 @@ namespace daemonizer "run-as-service" , "Hidden -- true if running as windows service" }; + const command_line::arg_descriptor arg_non_interactive = { + "non-interactive" + , "Run non-interactive" + }; std::string get_argument_string(int argc, char const * argv[]) { @@ -82,6 +87,7 @@ namespace daemonizer command_line::add_arg(normal_options, arg_start_service); command_line::add_arg(normal_options, arg_stop_service); command_line::add_arg(hidden_options, arg_is_service); + command_line::add_arg(hidden_options, arg_non_interactive); } inline boost::filesystem::path get_default_data_dir() @@ -176,7 +182,10 @@ namespace daemonizer else // interactive { //LOG_PRINT_L0("Monero '" << MONERO_RELEASE_NAME << "' (v" << MONERO_VERSION_FULL); - return executor.run_interactive(vm); + if (command_line::has_arg(vm, arg_non_interactive)) + return executor.run_non_interactive(vm); + else + return executor.run_interactive(vm); } return false; diff --git a/src/daemonizer/windows_service.cpp b/src/daemonizer/windows_service.cpp index b344e1a4b19..7d53b07c778 100644 --- a/src/daemonizer/windows_service.cpp +++ b/src/daemonizer/windows_service.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -70,8 +70,9 @@ namespace { } else { - return std::string{p_error_text}; + std::string ret{p_error_text}; LocalFree(p_error_text); + return ret; } } diff --git a/src/daemonizer/windows_service.h b/src/daemonizer/windows_service.h index aacf3d0395b..12429c78a7a 100644 --- a/src/daemonizer/windows_service.h +++ b/src/daemonizer/windows_service.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/daemonizer/windows_service_runner.h b/src/daemonizer/windows_service_runner.h index 06e180823ba..703795ce95e 100644 --- a/src/daemonizer/windows_service_runner.h +++ b/src/daemonizer/windows_service_runner.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/debug_utilities/CMakeLists.txt b/src/debug_utilities/CMakeLists.txt index 1bcbfd0cf68..7bc2c324fd5 100644 --- a/src/debug_utilities/CMakeLists.txt +++ b/src/debug_utilities/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2014-2018, The Monero Project +# Copyright (c) 2014-2019, The Monero Project # # All rights reserved. # diff --git a/src/debug_utilities/cn_deserialize.cpp b/src/debug_utilities/cn_deserialize.cpp index dfbd3b8645e..89c9db02ee8 100644 --- a/src/debug_utilities/cn_deserialize.cpp +++ b/src/debug_utilities/cn_deserialize.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -27,6 +27,8 @@ // THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include +#include +#include #include "cryptonote_basic/cryptonote_basic.h" #include "cryptonote_basic/tx_extra.h" #include "cryptonote_core/blockchain.h" @@ -41,6 +43,15 @@ using namespace epee; using namespace cryptonote; +static std::string extra_nonce_to_string(const cryptonote::tx_extra_nonce &extra_nonce) +{ + if (extra_nonce.nonce.size() == 9 && extra_nonce.nonce[0] == TX_EXTRA_NONCE_ENCRYPTED_PAYMENT_ID) + return "encrypted payment ID: " + epee::string_tools::buff_to_hex_nodelimer(extra_nonce.nonce.substr(1)); + if (extra_nonce.nonce.size() == 33 && extra_nonce.nonce[0] == TX_EXTRA_NONCE_PAYMENT_ID) + return "plaintext payment ID: " + epee::string_tools::buff_to_hex_nodelimer(extra_nonce.nonce.substr(1)); + return epee::string_tools::buff_to_hex_nodelimer(extra_nonce.nonce); +} + static void print_extra_fields(const std::vector &fields) { std::cout << "tx_extra has " << fields.size() << " field(s)" << std::endl; @@ -49,8 +60,9 @@ static void print_extra_fields(const std::vector &fi std::cout << "field " << n << ": "; if (typeid(cryptonote::tx_extra_padding) == fields[n].type()) std::cout << "extra padding: " << boost::get(fields[n]).size << " bytes"; else if (typeid(cryptonote::tx_extra_pub_key) == fields[n].type()) std::cout << "extra pub key: " << boost::get(fields[n]).pub_key; - else if (typeid(cryptonote::tx_extra_nonce) == fields[n].type()) std::cout << "extra nonce: " << epee::string_tools::buff_to_hex_nodelimer(boost::get(fields[n]).nonce); + else if (typeid(cryptonote::tx_extra_nonce) == fields[n].type()) std::cout << "extra nonce: " << extra_nonce_to_string(boost::get(fields[n])); else if (typeid(cryptonote::tx_extra_merge_mining_tag) == fields[n].type()) std::cout << "extra merge mining tag: depth " << boost::get(fields[n]).depth << ", merkle root " << boost::get(fields[n]).merkle_root; + else if (typeid(cryptonote::tx_extra_additional_pub_keys) == fields[n].type()) std::cout << "additional tx pubkeys: " << boost::join(boost::get(fields[n]).data | boost::adaptors::transformed([](const crypto::public_key &key){ return epee::string_tools::pod_to_hex(key); }), ", " ); else if (typeid(cryptonote::tx_extra_mysterious_minergate) == fields[n].type()) std::cout << "extra minergate custom: " << epee::string_tools::buff_to_hex_nodelimer(boost::get(fields[n]).data); else std::cout << "unknown"; std::cout << std::endl; @@ -68,11 +80,9 @@ int main(int argc, char* argv[]) po::options_description desc_cmd_only("Command line options"); po::options_description desc_cmd_sett("Command line options and settings options"); - const command_line::arg_descriptor arg_output_file = {"output-file", "Specify output file", "", true}; const command_line::arg_descriptor arg_log_level = {"log-level", "", log_level}; const command_line::arg_descriptor arg_input = {"input", "Specify input has a hexadecimal string", ""}; - command_line::add_arg(desc_cmd_sett, arg_output_file); command_line::add_arg(desc_cmd_sett, arg_log_level); command_line::add_arg(desc_cmd_sett, arg_input); @@ -108,55 +118,14 @@ int main(int argc, char* argv[]) mlog_configure("", true); - std::string m_config_folder; - - std::ostream *output; - std::ofstream *raw_data_file = NULL; - if (command_line::has_arg(vm, arg_output_file)) - { - output_file_path = boost::filesystem::path(command_line::get_arg(vm, arg_output_file)); - - const boost::filesystem::path dir_path = output_file_path.parent_path(); - if (!dir_path.empty()) - { - if (boost::filesystem::exists(dir_path)) - { - if (!boost::filesystem::is_directory(dir_path)) - { - std::cerr << "output directory path is a file: " << dir_path << std::endl; - return 1; - } - } - else - { - if (!boost::filesystem::create_directory(dir_path)) - { - std::cerr << "Failed to create directory " << dir_path << std::endl; - return 1; - } - } - } - - raw_data_file = new std::ofstream(); - raw_data_file->open(output_file_path.string(), std::ios_base::out | std::ios::trunc); - if (raw_data_file->fail()) - return 1; - output = raw_data_file; - } - else - { - output_file_path = ""; - output = &std::cout; - } - cryptonote::blobdata blob; if (!epee::string_tools::parse_hexstr_to_binbuff(input, blob)) { std::cerr << "Invalid hex input" << std::endl; - std::cerr << "Invalid hex input: " << input << std::endl; return 1; } + bool full; cryptonote::block block; cryptonote::transaction tx; std::vector fields; @@ -165,9 +134,12 @@ int main(int argc, char* argv[]) std::cout << "Parsed block:" << std::endl; std::cout << cryptonote::obj_to_json_str(block) << std::endl; } - else if (cryptonote::parse_and_validate_tx_from_blob(blob, tx)) + else if (cryptonote::parse_and_validate_tx_from_blob(blob, tx) || cryptonote::parse_and_validate_tx_base_from_blob(blob, tx)) { - std::cout << "Parsed transaction:" << std::endl; + if (tx.pruned) + std::cout << "Parsed pruned transaction:" << std::endl; + else + std::cout << "Parsed transaction:" << std::endl; std::cout << cryptonote::obj_to_json_str(tx) << std::endl; bool parsed = cryptonote::parse_tx_extra(tx.extra, fields); @@ -183,9 +155,9 @@ int main(int argc, char* argv[]) std::cout << "No fields were found in tx_extra" << std::endl; } } - else if (cryptonote::parse_tx_extra(std::vector(blob.begin(), blob.end()), fields) && !fields.empty()) + else if (((full = cryptonote::parse_tx_extra(std::vector(blob.begin(), blob.end()), fields)) || true) && !fields.empty()) { - std::cout << "Parsed tx_extra:" << std::endl; + std::cout << "Parsed" << (full ? "" : " partial") << " tx_extra:" << std::endl; print_extra_fields(fields); } else @@ -196,11 +168,5 @@ int main(int argc, char* argv[]) - if (output->fail()) - return 1; - output->flush(); - if (raw_data_file) - delete raw_data_file; - return 0; } diff --git a/src/debug_utilities/object_sizes.cpp b/src/debug_utilities/object_sizes.cpp index a3d037a59df..302f0a2060a 100644 --- a/src/debug_utilities/object_sizes.cpp +++ b/src/debug_utilities/object_sizes.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2018, The Monero Project +// Copyright (c) 2017-2019, The Monero Project // // All rights reserved. // @@ -29,6 +29,7 @@ #include #include "cryptonote_basic/cryptonote_basic.h" #include "cryptonote_basic/tx_extra.h" +#include "cryptonote_core/cryptonote_core.h" #include "cryptonote_core/blockchain.h" #include "p2p/p2p_protocol_defs.h" #include "net/connection_basic.hpp" @@ -94,6 +95,11 @@ int main(int argc, char* argv[]) SL(nodetool::node_server>); SL(nodetool::p2p_connection_context_t::connection_context>); SL(nodetool::network_address_old); + SL(nodetool::peerlist_entry_base); + + SL(nodetool::network_config); + SL(nodetool::basic_node_data); + SL(cryptonote::CORE_SYNC_DATA); SL(tools::wallet2::transfer_details); SL(tools::wallet2::payment_details); diff --git a/src/device/CMakeLists.txt b/src/device/CMakeLists.txt index cc2d20a54ed..0b0686f612b 100644 --- a/src/device/CMakeLists.txt +++ b/src/device/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2014-2017, The Monero Project +# Copyright (c) 2014-2019, The Monero Project # # All rights reserved. # @@ -32,29 +32,33 @@ set(device_sources log.cpp ) -if(PCSC_FOUND) - set(device_sources ${device_sources} device_ledger.cpp) +if(HIDAPI_FOUND) + set(device_sources + ${device_sources} + device_ledger.cpp + device_io_hid.cpp + ) endif() set(device_headers device.hpp + device_io.hpp device_default.hpp + device_cold.hpp log.hpp ) -if(PCSC_FOUND) - set(device_headers ${device_headers} device_ledger.hpp) +if(HIDAPI_FOUND) + set(device_headers + ${device_headers} + device_ledger.hpp + device_io_hid.hpp + ) endif() set(device_private_headers) -if(PER_BLOCK_CHECKPOINT) - set(Blocks "blocks") -else() - set(Blocks "") -endif() - monero_private_headers(device ${device_private_headers}) @@ -65,10 +69,12 @@ monero_add_library(device target_link_libraries(device PUBLIC - ${PCSC_LIBRARIES} + ${HIDAPI_LIBRARIES} cncrypto ringct_basic ${OPENSSL_CRYPTO_LIBRARIES} + ${Boost_SERIALIZATION_LIBRARY} PRIVATE + version ${Blocks} ${EXTRA_LIBRARIES}) diff --git a/src/device/device.cpp b/src/device/device.cpp index 983f59b60ea..fbd77dab9a3 100644 --- a/src/device/device.cpp +++ b/src/device/device.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2018, The Monero Project +// Copyright (c) 2017-2019, The Monero Project // // All rights reserved. // @@ -29,7 +29,7 @@ #include "device.hpp" #include "device_default.hpp" -#ifdef HAVE_PCSC +#ifdef WITH_DEVICE_LEDGER #include "device_ledger.hpp" #endif #include "misc_log_ex.h" @@ -39,32 +39,67 @@ namespace hw { /* ======================================================================= */ /* SETUP */ - /* ======================================================================= */ - device& get_device(const std::string device_descriptor) { - - struct s_devices { - std::map> registry; - s_devices() : registry() { - hw::core::register_all(registry); - #ifdef HAVE_PCSC - hw::ledger::register_all(registry); - #endif - }; - }; - - static const s_devices devices; + /* ======================================================================= */ + + static device_registry *get_device_registry(bool clear = false){ + static device_registry *registry = new device_registry(); + if (clear) + { + delete registry; + registry = NULL; + } + return registry; + } + + static void clear_device_registry(){ + get_device_registry(true); + } + + device_registry::device_registry(){ + hw::core::register_all(registry); + #ifdef WITH_DEVICE_LEDGER + hw::ledger::register_all(registry); + #endif + atexit(clear_device_registry); + } + + bool device_registry::register_device(const std::string & device_name, device * hw_device){ + auto search = registry.find(device_name); + if (search != registry.end()){ + return false; + } - auto device = devices.registry.find(device_descriptor); - if (device == devices.registry.end()) { - MERROR("device not found in registry: '" << device_descriptor << "'\n" << - "known devices:"); - - for( const auto& sm_pair : devices.registry ) { + registry.insert(std::make_pair(device_name, std::unique_ptr(hw_device))); + return true; + } + + device& device_registry::get_device(const std::string & device_descriptor){ + // Device descriptor can contain further specs after first : + auto delim = device_descriptor.find(':'); + auto device_descriptor_lookup = device_descriptor; + if (delim != std::string::npos) { + device_descriptor_lookup = device_descriptor.substr(0, delim); + } + + auto device = registry.find(device_descriptor_lookup); + if (device == registry.end()) { + MERROR("Device not found in registry: '" << device_descriptor << "'. Known devices: "); + for( const auto& sm_pair : registry ) { MERROR(" - " << sm_pair.first); } - throw std::runtime_error("device not found: "+ device_descriptor); + throw std::runtime_error("device not found: " + device_descriptor); } return *device->second; } + device& get_device(const std::string & device_descriptor) { + device_registry *registry = get_device_registry(); + return registry->get_device(device_descriptor); + } + + bool register_device(const std::string & device_name, device * hw_device){ + device_registry *registry = get_device_registry(); + return registry->register_device(device_name, hw_device); + } + } diff --git a/src/device/device.hpp b/src/device/device.hpp index 9df0cb39d54..866e2c676f0 100644 --- a/src/device/device.hpp +++ b/src/device/device.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2018, The Monero Project +// Copyright (c) 2017-2019, The Monero Project // // All rights reserved. // @@ -27,32 +27,19 @@ // THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // - -/* Note about debug: - * To debug Device you can def the following : - * #define DEBUG_HWDEVICE - * Activate debug mechanism: - * - Add more trace - * - All computation done by device are checked by default device. - * Required IODUMMYCRYPT_HWDEVICE or IONOCRYPT_HWDEVICE for fully working - * #define IODUMMYCRYPT_HWDEVICE 1 - * - It assumes sensitive data encryption is is off on device side. a XOR with 0x55. This allow Ledger Class to make check on clear value - * #define IONOCRYPT_HWDEVICE 1 - * - It assumes sensitive data encryption is off on device side. - */ - - #pragma once #include "crypto/crypto.h" #include "crypto/chacha.h" #include "ringct/rctTypes.h" +#include "cryptonote_config.h" + #ifndef USE_DEVICE_LEDGER #define USE_DEVICE_LEDGER 1 #endif -#if !defined(HAVE_PCSC) +#if !defined(HAVE_HIDAPI) #undef USE_DEVICE_LEDGER #define USE_DEVICE_LEDGER 0 #endif @@ -67,6 +54,8 @@ namespace cryptonote struct account_public_address; struct account_keys; struct subaddress_index; + struct tx_destination_entry; + struct keypair; } namespace hw { @@ -78,6 +67,21 @@ namespace hw { return false; } + class device_progress { + public: + virtual double progress() const { return 0; } + virtual bool indeterminate() const { return false; } + }; + + class i_device_callback { + public: + virtual void on_button_request(uint64_t code=0) {} + virtual void on_button_pressed() {} + virtual boost::optional on_pin_request() { return boost::none; } + virtual boost::optional on_passphrase_request(bool on_device) { return boost::none; } + virtual void on_progress(const device_progress& event) {} + virtual ~i_device_callback() = default; + }; class device { protected: @@ -85,7 +89,7 @@ namespace hw { public: - device() {} + device(): mode(NONE) {} device(const device &hwdev) {} virtual ~device() {} @@ -96,6 +100,19 @@ namespace hw { TRANSACTION_CREATE_FAKE, TRANSACTION_PARSE }; + enum device_type + { + SOFTWARE = 0, + LEDGER = 1, + TREZOR = 2 + }; + + + enum device_protocol_t { + PROTOCOL_DEFAULT, + PROTOCOL_PROXY, // Originally defined by Ledger + PROTOCOL_COLD, // Originally defined by Trezor + }; /* ======================================================================= */ /* SETUP/TEARDOWN */ @@ -109,8 +126,17 @@ namespace hw { virtual bool connect(void) = 0; virtual bool disconnect(void) = 0; - virtual bool set_mode(device_mode mode) = 0; + virtual bool set_mode(device_mode mode) { this->mode = mode; return true; } + virtual device_mode get_mode() const { return mode; } + + virtual device_type get_type() const = 0; + virtual device_protocol_t device_protocol() const { return PROTOCOL_DEFAULT; }; + virtual void set_callback(i_device_callback * callback) {}; + virtual void set_derivation_path(const std::string &derivation_path) {}; + + virtual void set_pin(const epee::wipeable_string & pin) {} + virtual void set_passphrase(const epee::wipeable_string & passphrase) {} /* ======================================================================= */ /* LOCKER */ @@ -125,7 +151,7 @@ namespace hw { /* ======================================================================= */ virtual bool get_public_address(cryptonote::account_public_address &pubkey) = 0; virtual bool get_secret_keys(crypto::secret_key &viewkey , crypto::secret_key &spendkey) = 0; - virtual bool generate_chacha_key(const cryptonote::account_keys &keys, crypto::chacha_key &key) = 0; + virtual bool generate_chacha_key(const cryptonote::account_keys &keys, crypto::chacha_key &key, uint64_t kdf_rounds) = 0; /* ======================================================================= */ /* SUB ADDRESS */ @@ -171,6 +197,10 @@ namespace hw { /* TRANSACTION */ /* ======================================================================= */ + virtual void generate_tx_proof(const crypto::hash &prefix_hash, + const crypto::public_key &R, const crypto::public_key &A, const boost::optional &B, const crypto::public_key &D, const crypto::secret_key &r, + crypto::signature &sig) = 0; + virtual bool open_tx(crypto::secret_key &tx_key) = 0; virtual bool encrypt_payment_id(crypto::hash8 &payment_id, const crypto::public_key &public_key, const crypto::secret_key &secret_key) = 0; @@ -180,12 +210,17 @@ namespace hw { return encrypt_payment_id(payment_id, public_key, secret_key); } - virtual bool ecdhEncode(rct::ecdhTuple & unmasked, const rct::key & sharedSec) = 0; - virtual bool ecdhDecode(rct::ecdhTuple & masked, const rct::key & sharedSec) = 0; + virtual rct::key genCommitmentMask(const rct::key &amount_key) = 0; - virtual bool add_output_key_mapping(const crypto::public_key &Aout, const crypto::public_key &Bout, const bool is_subaddress, const size_t real_output_index, - const rct::key &amount_key, const crypto::public_key &out_eph_public_key) = 0; + virtual bool ecdhEncode(rct::ecdhTuple & unmasked, const rct::key & sharedSec, bool short_amount) = 0; + virtual bool ecdhDecode(rct::ecdhTuple & masked, const rct::key & sharedSec, bool short_amount) = 0; + virtual bool generate_output_ephemeral_keys(const size_t tx_version, const cryptonote::account_keys &sender_account_keys, const crypto::public_key &txkey_pub, const crypto::secret_key &tx_key, + const cryptonote::tx_destination_entry &dst_entr, const boost::optional &change_addr, const size_t output_index, + const bool &need_additional_txkeys, const std::vector &additional_tx_keys, + std::vector &additional_tx_public_keys, + std::vector &amount_keys, + crypto::public_key &out_eph_public_key) = 0; virtual bool mlsag_prehash(const std::string &blob, size_t inputs_size, size_t outputs_size, const rct::keyV &hashes, const rct::ctkeyV &outPk, rct::key &prehash) = 0; virtual bool mlsag_prepare(const rct::key &H, const rct::key &xx, rct::key &a, rct::key &aG, rct::key &aHP, rct::key &rvII) = 0; @@ -194,6 +229,16 @@ namespace hw { virtual bool mlsag_sign(const rct::key &c, const rct::keyV &xx, const rct::keyV &alpha, const size_t rows, const size_t dsRows, rct::keyV &ss) = 0; virtual bool close_tx(void) = 0; + + virtual bool has_ki_cold_sync(void) const { return false; } + virtual bool has_tx_cold_sign(void) const { return false; } + virtual bool has_ki_live_refresh(void) const { return true; } + virtual bool compute_key_image(const cryptonote::account_keys& ack, const crypto::public_key& out_key, const crypto::key_derivation& recv_derivation, size_t real_output_index, const cryptonote::subaddress_index& received_index, cryptonote::keypair& in_ephemeral, crypto::key_image& ki) { return false; } + virtual void computing_key_images(bool started) {}; + virtual void set_network_type(cryptonote::network_type network_type) { } + + protected: + device_mode mode; } ; struct reset_mode { @@ -202,6 +247,17 @@ namespace hw { ~reset_mode() { hwref.set_mode(hw::device::NONE);} }; - device& get_device(const std::string device_descriptor) ; + class device_registry { + private: + std::map> registry; + + public: + device_registry(); + bool register_device(const std::string & device_name, device * hw_device); + device& get_device(const std::string & device_descriptor); + }; + + device& get_device(const std::string & device_descriptor); + bool register_device(const std::string & device_name, device * hw_device); } diff --git a/src/device/device_cold.hpp b/src/device/device_cold.hpp new file mode 100644 index 00000000000..31b1504abd4 --- /dev/null +++ b/src/device/device_cold.hpp @@ -0,0 +1,167 @@ +// Copyright (c) 2017-2019, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +#ifndef MONERO_DEVICE_COLD_H +#define MONERO_DEVICE_COLD_H + +#include "wallet/wallet2.h" +#include +#include + + +namespace hw { + + typedef struct wallet_shim { + boost::function get_tx_pub_key_from_received_outs; + } wallet_shim; + + class tx_aux_data { + public: + std::vector tx_device_aux; // device generated aux data + std::vector tx_recipients; // as entered by user + boost::optional bp_version; // BP version to use + boost::optional client_version; // Signing client version to use (testing) + }; + + class device_cold { + public: + + using exported_key_image = std::vector>; + + class op_progress : public hw::device_progress { + public: + op_progress():m_progress(0), m_indeterminate(false) {}; + explicit op_progress(double progress, bool indeterminate=false): m_progress(progress), m_indeterminate(indeterminate){} + + double progress() const override { return m_progress; } + bool indeterminate() const override { return m_indeterminate; } + protected: + double m_progress; + bool m_indeterminate; + }; + + class tx_progress : public op_progress { + public: + tx_progress(): + m_cur_tx(0), m_max_tx(1), + m_cur_step(0), m_max_step(1), + m_cur_substep(0), m_max_substep(1){}; + + tx_progress(size_t cur_tx, size_t max_tx, size_t cur_step, size_t max_step, size_t cur_substep, size_t max_substep): + m_cur_tx(cur_tx), m_max_tx(max_tx), + m_cur_step(cur_tx), m_max_step(max_tx), + m_cur_substep(cur_tx), m_max_substep(max_tx){} + + double progress() const override { + return std::max(1.0, (double)m_cur_tx / m_max_tx + + (double)m_cur_step / (m_max_tx * m_max_step) + + (double)m_cur_substep / (m_max_tx * m_max_step * m_max_substep)); + } + bool indeterminate() const override { return false; } + + protected: + size_t m_cur_tx; + size_t m_max_tx; + size_t m_cur_step; + size_t m_max_step; + size_t m_cur_substep; + size_t m_max_substep; + }; + + typedef struct { + std::string salt1; + std::string salt2; + std::string tx_enc_keys; + std::string tx_prefix_hash; + } tx_key_data_t; + + /** + * Key image sync with the cold protocol. + */ + virtual void ki_sync(wallet_shim * wallet, + const std::vector<::tools::wallet2::transfer_details> & transfers, + exported_key_image & ski) =0; + + /** + * Signs unsigned transaction with the cold protocol. + */ + virtual void tx_sign(wallet_shim * wallet, + const ::tools::wallet2::unsigned_tx_set & unsigned_tx, + ::tools::wallet2::signed_tx_set & signed_tx, + tx_aux_data & aux_data) =0; + + /** + * Get tx key support check. + */ + virtual bool is_get_tx_key_supported() const { return false; } + + /** + * Loads TX aux data required for tx key. + */ + virtual void load_tx_key_data(tx_key_data_t & res, const std::string & tx_aux_data) =0; + + /** + * Decrypts TX keys. + */ + virtual void get_tx_key( + std::vector<::crypto::secret_key> & tx_keys, + const tx_key_data_t & tx_aux_data, + const ::crypto::secret_key & view_key_priv) =0; + + /** + * Live refresh support check + */ + virtual bool is_live_refresh_supported() const { return false; }; + + /** + * Starts live refresh process with the device + */ + virtual void live_refresh_start() =0; + + /** + * One live refresh step + */ + virtual void live_refresh( + const ::crypto::secret_key & view_key_priv, + const crypto::public_key& out_key, + const crypto::key_derivation& recv_derivation, + size_t real_output_index, + const cryptonote::subaddress_index& received_index, + cryptonote::keypair& in_ephemeral, + crypto::key_image& ki + ) =0; + + /** + * Live refresh process termination + */ + virtual void live_refresh_finish() =0; + }; +} + +#endif //MONERO_DEVICE_COLD_H diff --git a/src/device/device_default.cpp b/src/device/device_default.cpp index 0071f7d4f1b..dc06ce23744 100644 --- a/src/device/device_default.cpp +++ b/src/device/device_default.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2018, The Monero Project +// Copyright (c) 2017-2019, The Monero Project // // All rights reserved. // @@ -31,9 +31,10 @@ #include "device_default.hpp" -#include "common/int-util.h" +#include "int-util.h" #include "cryptonote_basic/account.h" #include "cryptonote_basic/subaddress_index.h" +#include "cryptonote_core/cryptonote_tx_utils.h" #include "ringct/rctOps.h" #define ENCRYPTED_PAYMENT_ID_TAIL 0x8d @@ -69,21 +70,21 @@ namespace hw { } bool device_default::init(void) { - dfns(); + return true; } bool device_default::release() { - dfns(); + return true; } bool device_default::connect(void) { - dfns(); + return true; } bool device_default::disconnect() { - dfns(); + return true; } bool device_default::set_mode(device_mode mode) { - return true; + return device::set_mode(mode); } /* ======================================================================= */ @@ -100,14 +101,14 @@ namespace hw { /* WALLET & ADDRESS */ /* ======================================================================= */ - bool device_default::generate_chacha_key(const cryptonote::account_keys &keys, crypto::chacha_key &key) { + bool device_default::generate_chacha_key(const cryptonote::account_keys &keys, crypto::chacha_key &key, uint64_t kdf_rounds) { const crypto::secret_key &view_key = keys.m_view_secret_key; const crypto::secret_key &spend_key = keys.m_spend_secret_key; - tools::scrubbed_arr data; + epee::mlocked> data; memcpy(data.data(), &view_key, sizeof(view_key)); memcpy(data.data() + sizeof(view_key), &spend_key, sizeof(spend_key)); data[sizeof(data) - 1] = CHACHA8_KEY_TAIL; - crypto::generate_chacha_key(data.data(), sizeof(data), key); + crypto::generate_chacha_key(data.data(), sizeof(data), key, kdf_rounds); return true; } bool device_default::get_public_address(cryptonote::account_public_address &pubkey) { @@ -271,6 +272,11 @@ namespace hw { /* ======================================================================= */ /* TRANSACTION */ /* ======================================================================= */ + void device_default::generate_tx_proof(const crypto::hash &prefix_hash, + const crypto::public_key &R, const crypto::public_key &A, const boost::optional &B, const crypto::public_key &D, const crypto::secret_key &r, + crypto::signature &sig) { + crypto::generate_tx_proof(prefix_hash, R, A, B, D, r, sig); + } bool device_default::open_tx(crypto::secret_key &tx_key) { cryptonote::keypair txkey = cryptonote::keypair::generate(*this); @@ -278,10 +284,55 @@ namespace hw { return true; } + bool device_default::generate_output_ephemeral_keys(const size_t tx_version, + const cryptonote::account_keys &sender_account_keys, const crypto::public_key &txkey_pub, const crypto::secret_key &tx_key, + const cryptonote::tx_destination_entry &dst_entr, const boost::optional &change_addr, const size_t output_index, + const bool &need_additional_txkeys, const std::vector &additional_tx_keys, + std::vector &additional_tx_public_keys, + std::vector &amount_keys, crypto::public_key &out_eph_public_key) { - bool device_default::add_output_key_mapping(const crypto::public_key &Aout, const crypto::public_key &Bout, const bool is_subaddress, const size_t real_output_index, - const rct::key &amount_key, const crypto::public_key &out_eph_public_key) { - return true; + crypto::key_derivation derivation; + + // make additional tx pubkey if necessary + cryptonote::keypair additional_txkey; + if (need_additional_txkeys) + { + additional_txkey.sec = additional_tx_keys[output_index]; + if (dst_entr.is_subaddress) + additional_txkey.pub = rct::rct2pk(rct::scalarmultKey(rct::pk2rct(dst_entr.addr.m_spend_public_key), rct::sk2rct(additional_txkey.sec))); + else + additional_txkey.pub = rct::rct2pk(rct::scalarmultBase(rct::sk2rct(additional_txkey.sec))); + } + + bool r; + if (change_addr && dst_entr.addr == *change_addr) + { + // sending change to yourself; derivation = a*R + r = generate_key_derivation(txkey_pub, sender_account_keys.m_view_secret_key, derivation); + CHECK_AND_ASSERT_MES(r, false, "at creation outs: failed to generate_key_derivation(" << txkey_pub << ", " << sender_account_keys.m_view_secret_key << ")"); + } + else + { + // sending to the recipient; derivation = r*A (or s*C in the subaddress scheme) + r = generate_key_derivation(dst_entr.addr.m_view_public_key, dst_entr.is_subaddress && need_additional_txkeys ? additional_txkey.sec : tx_key, derivation); + CHECK_AND_ASSERT_MES(r, false, "at creation outs: failed to generate_key_derivation(" << dst_entr.addr.m_view_public_key << ", " << (dst_entr.is_subaddress && need_additional_txkeys ? additional_txkey.sec : tx_key) << ")"); + } + + if (need_additional_txkeys) + { + additional_tx_public_keys.push_back(additional_txkey.pub); + } + + if (tx_version > 1) + { + crypto::secret_key scalar1; + derivation_to_scalar(derivation, output_index, scalar1); + amount_keys.push_back(rct::sk2rct(scalar1)); + } + r = derive_public_key(derivation, output_index, dst_entr.addr.m_spend_public_key, out_eph_public_key); + CHECK_AND_ASSERT_MES(r, false, "at creation outs: failed to derive_public_key(" << derivation << ", " << output_index << ", "<< dst_entr.addr.m_spend_public_key << ")"); + + return r; } bool device_default::encrypt_payment_id(crypto::hash8 &payment_id, const crypto::public_key &public_key, const crypto::secret_key &secret_key) { @@ -302,13 +353,17 @@ namespace hw { return true; } - bool device_default::ecdhEncode(rct::ecdhTuple & unmasked, const rct::key & sharedSec) { - rct::ecdhEncode(unmasked, sharedSec); + rct::key device_default::genCommitmentMask(const rct::key &amount_key) { + return rct::genCommitmentMask(amount_key); + } + + bool device_default::ecdhEncode(rct::ecdhTuple & unmasked, const rct::key & sharedSec, bool short_amount) { + rct::ecdhEncode(unmasked, sharedSec, short_amount); return true; } - bool device_default::ecdhDecode(rct::ecdhTuple & masked, const rct::key & sharedSec) { - rct::ecdhDecode(masked, sharedSec); + bool device_default::ecdhDecode(rct::ecdhTuple & masked, const rct::key & sharedSec, bool short_amount) { + rct::ecdhDecode(masked, sharedSec, short_amount); return true; } diff --git a/src/device/device_default.hpp b/src/device/device_default.hpp index 771fbba7215..5252d4129e7 100644 --- a/src/device/device_default.hpp +++ b/src/device/device_default.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2018, The Monero Project +// Copyright (c) 2017-2019, The Monero Project // // All rights reserved. // @@ -61,6 +61,8 @@ namespace hw { bool set_mode(device_mode mode) override; + device_type get_type() const override {return device_type::SOFTWARE;}; + /* ======================================================================= */ /* LOCKER */ /* ======================================================================= */ @@ -73,7 +75,7 @@ namespace hw { /* ======================================================================= */ bool get_public_address(cryptonote::account_public_address &pubkey) override; bool get_secret_keys(crypto::secret_key &viewkey , crypto::secret_key &spendkey) override; - bool generate_chacha_key(const cryptonote::account_keys &keys, crypto::chacha_key &key) override; + bool generate_chacha_key(const cryptonote::account_keys &keys, crypto::chacha_key &key, uint64_t kdf_rounds) override; /* ======================================================================= */ /* SUB ADDRESS */ @@ -105,16 +107,25 @@ namespace hw { /* TRANSACTION */ /* ======================================================================= */ + void generate_tx_proof(const crypto::hash &prefix_hash, + const crypto::public_key &R, const crypto::public_key &A, const boost::optional &B, const crypto::public_key &D, const crypto::secret_key &r, + crypto::signature &sig) override; + bool open_tx(crypto::secret_key &tx_key) override; bool encrypt_payment_id(crypto::hash8 &payment_id, const crypto::public_key &public_key, const crypto::secret_key &secret_key) override; - bool ecdhEncode(rct::ecdhTuple & unmasked, const rct::key & sharedSec) override; - bool ecdhDecode(rct::ecdhTuple & masked, const rct::key & sharedSec) override; + rct::key genCommitmentMask(const rct::key &amount_key) override; - bool add_output_key_mapping(const crypto::public_key &Aout, const crypto::public_key &Bout, const bool is_subaddress, const size_t real_output_index, - const rct::key &amount_key, const crypto::public_key &out_eph_public_key) override; + bool ecdhEncode(rct::ecdhTuple & unmasked, const rct::key & sharedSec, bool short_amount) override; + bool ecdhDecode(rct::ecdhTuple & masked, const rct::key & sharedSec, bool short_amount) override; + bool generate_output_ephemeral_keys(const size_t tx_version, const cryptonote::account_keys &sender_account_keys, const crypto::public_key &txkey_pub, const crypto::secret_key &tx_key, + const cryptonote::tx_destination_entry &dst_entr, const boost::optional &change_addr, const size_t output_index, + const bool &need_additional_txkeys, const std::vector &additional_tx_keys, + std::vector &additional_tx_public_keys, + std::vector &amount_keys, + crypto::public_key &out_eph_public_key) override; bool mlsag_prehash(const std::string &blob, size_t inputs_size, size_t outputs_size, const rct::keyV &hashes, const rct::ctkeyV &outPk, rct::key &prehash) override; bool mlsag_prepare(const rct::key &H, const rct::key &xx, rct::key &a, rct::key &aG, rct::key &aHP, rct::key &rvII) override; diff --git a/src/device/device_io.hpp b/src/device/device_io.hpp new file mode 100644 index 00000000000..fe66736f746 --- /dev/null +++ b/src/device/device_io.hpp @@ -0,0 +1,56 @@ +// Copyright (c) 2017-2019, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + + + + +#pragma once + + +namespace hw { + namespace io { + + class device_io { + + public: + + device_io() {}; + ~device_io() {}; + + virtual void init() = 0; + virtual void release() = 0; + + virtual void connect(void *parms) = 0; + virtual void disconnect() = 0; + virtual bool connected() const = 0; + + virtual int exchange(unsigned char *command, unsigned int cmd_len, unsigned char *response, unsigned int max_resp_len, bool user_input) = 0; + }; + }; +}; diff --git a/src/device/device_io_hid.cpp b/src/device/device_io_hid.cpp new file mode 100644 index 00000000000..721bed9ca6e --- /dev/null +++ b/src/device/device_io_hid.cpp @@ -0,0 +1,350 @@ +// Copyright (c) 2017-2019, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +#if defined(HAVE_HIDAPI) + +#include +#include "log.hpp" +#include "device_io_hid.hpp" + +namespace hw { + namespace io { + + #undef MONERO_DEFAULT_LOG_CATEGORY + #define MONERO_DEFAULT_LOG_CATEGORY "device.io" + + #define ASSERT_X(exp,msg) CHECK_AND_ASSERT_THROW_MES(exp, msg); + + #define MAX_BLOCK 64 + + static std::string safe_hid_error(hid_device *hwdev) { + if (hwdev) { + return std::string((char*)hid_error(hwdev)); + } + return std::string("NULL device"); + } + + static std::string safe_hid_path(const hid_device_info *hwdev_info) { + if (hwdev_info && hwdev_info->path) { + return std::string(hwdev_info->path); + } + return std::string("NULL path"); + } + + device_io_hid::device_io_hid(unsigned short c, unsigned char t, unsigned int ps, unsigned int to) : + channel(c), + tag(t), + packet_size(ps), + timeout(to), + usb_vid(0), + usb_pid(0), + usb_device(NULL) { + } + + device_io_hid::device_io_hid() : device_io_hid(DEFAULT_CHANNEL, DEFAULT_TAG, DEFAULT_PACKET_SIZE, DEFAULT_TIMEOUT) { + } + + void device_io_hid::io_hid_log(int read, unsigned char* buffer, int block_len) { + if (hid_verbose) { + char strbuffer[1024]; + hw::buffer_to_str(strbuffer, sizeof(strbuffer), (char*)buffer, block_len); + MDEBUG( "HID " << (read?"<":">") <<" : "<=0, "Unable to init hidapi library. Error "+std::to_string(r)+": "+safe_hid_error(this->usb_device)); + } + + void device_io_hid::connect(void *params) { + hid_conn_params *p = (struct hid_conn_params*)params; + if (!this->connect(p->vid, p->pid, p->interface_number, p->usage_page)) { + ASSERT_X(false, "No device found"); + } + } + + void device_io_hid::connect(const std::vector &hcpV) { + for (auto p: hcpV) { + if (this->connect(p.vid, p.pid, p.interface_number, p.usage_page)) { + return; + } + } + ASSERT_X(false, "No device found"); + } + + hid_device_info *device_io_hid::find_device(hid_device_info *devices_list, boost::optional interface_number, boost::optional usage_page) { + bool select_any = !interface_number && !usage_page; + + MDEBUG( "Looking for " << + (select_any ? "any HID Device" : "HID Device with") << + (interface_number ? (" interface_number " + std::to_string(interface_number.value())) : "") << + ((interface_number && usage_page) ? " or" : "") << + (usage_page ? (" usage_page " + std::to_string(usage_page.value())) : "")); + + hid_device_info *result = nullptr; + for (; devices_list != nullptr; devices_list = devices_list->next) { + BOOST_SCOPE_EXIT(&devices_list, &result) { + MDEBUG( (result == devices_list ? "SELECTED" : "SKIPPED ") << + " HID Device" << + " path " << safe_hid_path(devices_list) << + " interface_number " << devices_list->interface_number << + " usage_page " << devices_list->usage_page); + } + BOOST_SCOPE_EXIT_END + + if (result != nullptr) { + continue; + } + + if (select_any) { + result = devices_list; + } else if (interface_number && devices_list->interface_number == interface_number.value()) { + result = devices_list; + } else if (usage_page && devices_list->usage_page == usage_page.value()) { + result = devices_list; + } + } + + return result; + } + + hid_device *device_io_hid::connect(unsigned int vid, unsigned int pid, boost::optional interface_number, boost::optional usage_page) { + hid_device_info *hwdev_info_list; + hid_device *hwdev; + + this->disconnect(); + + hwdev_info_list = hid_enumerate(vid, pid); + if (!hwdev_info_list) { + MDEBUG("Unable to enumerate device "+std::to_string(vid)+":"+std::to_string(vid)+ ": "+ safe_hid_error(this->usb_device)); + return NULL; + } + hwdev = NULL; + if (hid_device_info *device = find_device(hwdev_info_list, interface_number, usage_page)) { + hwdev = hid_open_path(device->path); + } + hid_free_enumeration(hwdev_info_list); + ASSERT_X(hwdev, "Unable to open device "+std::to_string(pid)+":"+std::to_string(vid)); + this->usb_vid = vid; + this->usb_pid = pid; + this->usb_device = hwdev; + return hwdev; + } + + + bool device_io_hid::connected() const { + return this->usb_device != NULL; + } + + int device_io_hid::exchange(unsigned char *command, unsigned int cmd_len, unsigned char *response, unsigned int max_resp_len, bool user_input) { + unsigned char buffer[400]; + unsigned char padding_buffer[MAX_BLOCK+1]; + unsigned int result; + int hid_ret; + unsigned int sw_offset; + unsigned int remaining; + unsigned int offset = 0; + + ASSERT_X(this->usb_device,"No device opened"); + + //Split command in several HID packet + memset(buffer, 0, sizeof(buffer)); + result = this->wrapCommand(command, cmd_len, buffer, sizeof(buffer)); + remaining = result; + + while (remaining > 0) { + int block_size = (remaining > MAX_BLOCK ? MAX_BLOCK : remaining); + memset(padding_buffer, 0, sizeof(padding_buffer)); + memcpy(padding_buffer+1, buffer + offset, block_size); + io_hid_log(0, padding_buffer, block_size+1); + hid_ret = hid_write(this->usb_device, padding_buffer, block_size+1); + ASSERT_X(hid_ret>=0, "Unable to send hidapi command. Error "+std::to_string(result)+": "+ safe_hid_error(this->usb_device)); + offset += block_size; + remaining -= block_size; + } + + //get first response + memset(buffer, 0, sizeof(buffer)); + if (!user_input) { + hid_ret = hid_read_timeout(this->usb_device, buffer, MAX_BLOCK, this->timeout); + } else { + hid_ret = hid_read(this->usb_device, buffer, MAX_BLOCK); + } + ASSERT_X(hid_ret>=0, "Unable to read hidapi response. Error "+std::to_string(result)+": "+ safe_hid_error(this->usb_device)); + result = (unsigned int)hid_ret; + io_hid_log(1, buffer, result); + offset = MAX_BLOCK; + //parse first response and get others if any + for (;;) { + result = this->unwrapReponse(buffer, offset, response, max_resp_len); + if (result != 0) { + break; + } + hid_ret = hid_read_timeout(this->usb_device, buffer + offset, MAX_BLOCK, this->timeout); + ASSERT_X(hid_ret>=0, "Unable to receive hidapi response. Error "+std::to_string(result)+": "+ safe_hid_error(this->usb_device)); + result = (unsigned int)hid_ret; + io_hid_log(1, buffer + offset, result); + offset += MAX_BLOCK; + } + return result; + } + + void device_io_hid::disconnect(void) { + if (this->usb_device) { + hid_close(this->usb_device); + } + this->usb_vid = 0; + this->usb_pid = 0; + this->usb_device = NULL; + } + + void device_io_hid::release() { + /* Do not exit, as the lib context is global*/ + //hid_exit(); + } + + unsigned int device_io_hid::wrapCommand(const unsigned char *command, size_t command_len, unsigned char *out, size_t out_len) { + unsigned int sequence_idx = 0; + unsigned int offset = 0; + unsigned int offset_out = 0; + unsigned int block_size; + + ASSERT_X(this->packet_size >= 3, "Invalid Packet size: "+std::to_string(this->packet_size)) ; + ASSERT_X(out_len >= 7, "out_len too short: "+std::to_string(out_len)); + + out_len -= 7; + out[offset_out++] = ((this->channel >> 8) & 0xff); + out[offset_out++] = (this->channel & 0xff); + out[offset_out++] = this->tag; + out[offset_out++] = ((sequence_idx >> 8) & 0xff); + out[offset_out++] = (sequence_idx & 0xff); + sequence_idx++; + out[offset_out++] = ((command_len >> 8) & 0xff); + out[offset_out++] = (command_len & 0xff); + block_size = (command_len > this->packet_size - 7 ? this->packet_size - 7 : command_len); + ASSERT_X(out_len >= block_size, "out_len too short: "+std::to_string(out_len)); + out_len -= block_size; + memcpy(out + offset_out, command + offset, block_size); + offset_out += block_size; + offset += block_size; + while (offset != command_len) { + ASSERT_X(out_len >= 5, "out_len too short: "+std::to_string(out_len)); + out_len -= 5; + out[offset_out++] = ((this->channel >> 8) & 0xff); + out[offset_out++] = (this->channel & 0xff); + out[offset_out++] = this->tag; + out[offset_out++] = ((sequence_idx >> 8) & 0xff); + out[offset_out++] = (sequence_idx & 0xff); + sequence_idx++; + block_size = ((command_len - offset) > this->packet_size - 5 ? this->packet_size - 5 : command_len - offset); + ASSERT_X(out_len >= block_size, "out_len too short: "+std::to_string(out_len)); + out_len -= block_size; + memcpy(out + offset_out, command + offset, block_size); + offset_out += block_size; + offset += block_size; + } + while ((offset_out % this->packet_size) != 0) { + ASSERT_X(out_len >= 1, "out_len too short: "+std::to_string(out_len)); + out_len--; + out[offset_out++] = 0; + } + return offset_out; + } + + /* + * return 0 if more data are needed + * >0 if response is fully available + */ + unsigned int device_io_hid::unwrapReponse(const unsigned char *data, size_t data_len, unsigned char *out, size_t out_len) { + unsigned int sequence_idx = 0; + unsigned int offset = 0; + unsigned int offset_out = 0; + unsigned int response_len; + unsigned int block_size; + unsigned int val; + + //end? + if ((data == NULL) || (data_len < 7 + 5)) { + return 0; + } + + //check hid header + val = (data[offset]<<8) + data[offset+1]; + offset += 2; + ASSERT_X(val == this->channel, "Wrong Channel"); + val = data[offset]; + offset++; + ASSERT_X(val == this->tag, "Wrong TAG"); + val = (data[offset]<<8) + data[offset+1]; + offset += 2; + ASSERT_X(val == sequence_idx, "Wrong sequence_idx"); + + //fetch + response_len = (data[offset++] << 8); + response_len |= data[offset++]; + ASSERT_X(out_len >= response_len, "Out Buffer too short"); + if (data_len < (7 + response_len)) { + return 0; + } + block_size = (response_len > (this->packet_size - 7) ? this->packet_size - 7 : response_len); + memcpy(out + offset_out, data + offset, block_size); + offset += block_size; + offset_out += block_size; + while (offset_out != response_len) { + sequence_idx++; + if (offset == data_len) { + return 0; + } + val = (data[offset]<<8) + data[offset+1]; + offset += 2; + ASSERT_X(val == this->channel, "Wrong Channel"); + val = data[offset]; + offset++; + ASSERT_X(val == this->tag, "Wrong TAG"); + val = (data[offset]<<8) + data[offset+1]; + offset += 2; + ASSERT_X(val == sequence_idx, "Wrong sequence_idx"); + + block_size = ((response_len - offset_out) > this->packet_size - 5 ? this->packet_size - 5 : response_len - offset_out); + if (block_size > (data_len - offset)) { + return 0; + } + memcpy(out + offset_out, data + offset, block_size); + offset += block_size; + offset_out += block_size; + } + return offset_out; + } + + + } +} + +#endif //#if defined(HAVE_HIDAPI) diff --git a/src/device/device_io_hid.hpp b/src/device/device_io_hid.hpp new file mode 100644 index 00000000000..96cb8d99362 --- /dev/null +++ b/src/device/device_io_hid.hpp @@ -0,0 +1,111 @@ +// Copyright (c) 2017-2019, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +#if defined(HAVE_HIDAPI) + +#include +#include +#include "device_io.hpp" + +#pragma once + +namespace hw { + namespace io { + + + + /** HID class base. Commands are formated as follow: + * + * |----------------------------------------------------------| + * | 2 bytes | 1 byte | 2 bytes | 2 bytes | len bytes | + * |-----------|----------|-----------|----------|------------| + * | channel | tag | sequence | len | payload | + * |----------------------------------------------------------| + */ + + + struct hid_conn_params { + unsigned int vid; + unsigned int pid; + int interface_number; + unsigned short usage_page; + }; + + + class device_io_hid: device_io { + + + private: + + + unsigned short channel; + unsigned char tag; + unsigned int packet_size; + unsigned int timeout; + + unsigned int usb_vid; + unsigned int usb_pid; + hid_device *usb_device; + + void io_hid_log(int read, unsigned char* buf, int buf_len); + void io_hid_init(); + void io_hid_exit() ; + void io_hid_open(int vid, int pid, int mode); + void io_hid_close (void); + + unsigned int wrapCommand(const unsigned char *command, size_t command_len, unsigned char *out, size_t out_len); + unsigned int unwrapReponse(const unsigned char *data, size_t data_len, unsigned char *out, size_t out_len); + + hid_device_info *find_device(hid_device_info *devices_list, boost::optional interface_number, boost::optional usage_page); + + public: + bool hid_verbose = false; + + static const unsigned short DEFAULT_CHANNEL = 0x0001; + static const unsigned char DEFAULT_TAG = 0x01; + static const unsigned int DEFAULT_PACKET_SIZE = 64; + static const unsigned int DEFAULT_TIMEOUT = 120000; + + device_io_hid(unsigned short channel, unsigned char tag, unsigned int packet_zize, unsigned int timeout); + device_io_hid(); + ~device_io_hid() {}; + + void init(); + void connect(void *params); + void connect(const std::vector &conn); + hid_device *connect(unsigned int vid, unsigned int pid, boost::optional interface_number, boost::optional usage_page); + bool connected() const; + int exchange(unsigned char *command, unsigned int cmd_len, unsigned char *response, unsigned int max_resp_len, bool user_input); + void disconnect(); + void release(); + }; + }; +}; + +#endif //#if defined(HAVE_HIDAPI) diff --git a/src/device/device_ledger.cpp b/src/device/device_ledger.cpp index 3b9ab674400..eba633da8a7 100644 --- a/src/device/device_ledger.cpp +++ b/src/device/device_ledger.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2018, The Monero Project +// Copyright (c) 2017-2019, The Monero Project // // All rights reserved. // @@ -27,11 +27,12 @@ // THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // +#include "version.h" #include "device_ledger.hpp" -#include "log.hpp" #include "ringct/rctOps.h" #include "cryptonote_basic/account.h" #include "cryptonote_basic/subaddress_index.h" +#include "cryptonote_core/cryptonote_tx_utils.h" #include #include @@ -48,12 +49,12 @@ namespace hw { /* ===================================================================== */ /* === Debug ==== */ /* ===================================================================== */ + void set_apdu_verbose(bool verbose) { apdu_verbose = verbose; } #define TRACKD MTRACE("hw") - #define ASSERT_RV(rv) CHECK_AND_ASSERT_THROW_MES((rv)==SCARD_S_SUCCESS, "Fail SCard API : (" << (rv) << ") "<< pcsc_stringify_error(rv)<<" Device="<id<<", hCard="<buffer_send[0], - this->buffer_send[1], - this->buffer_send[2], - this->buffer_send[3], - this->buffer_send[4] - ); - buffer_to_str(strbuffer+strlen(strbuffer), sizeof(strbuffer), (char*)(this->buffer_send+5), this->length_send-5); - MDEBUG( "CMD :" << strbuffer); - } - } - - void device_ledger::logRESP() { - if (apdu_verbose) { - char strbuffer[1024]; - sprintf(strbuffer, "%.02x%.02x ", - this->buffer_recv[this->length_recv-2], - this->buffer_recv[this->length_recv-1] - ); - buffer_to_str(strbuffer+strlen(strbuffer), sizeof(strbuffer), (char*)(this->buffer_recv), this->length_recv-2); - MDEBUG( "RESP :" << strbuffer); - - } - } - - /* -------------------------------------------------------------- */ - device_ledger::device_ledger() { + device_ledger::device_ledger(): hw_device(0x0101, 0x05, 64, 2000) { this->id = device_id++; - this->hCard = 0; - this->hContext = 0; this->reset_buffer(); this->mode = NONE; this->has_view_key = false; @@ -260,45 +254,121 @@ namespace hw { MDEBUG( "Device "<name << " UNLOCKed"); } + /* ======================================================================= */ - /* MISC */ + /* IO */ /* ======================================================================= */ - bool device_ledger::reset() { - int offset; - reset_buffer(); - - this->buffer_send[0] = 0x00; - this->buffer_send[1] = INS_RESET; - this->buffer_send[2] = 0x00; - this->buffer_send[3] = 0x00; - this->buffer_send[4] = 0x00; - offset = 5; - //options - this->buffer_send[offset] = 0x00; - this->buffer_send[4] = offset-5; - this->length_send = offset; + #define IO_SW_DENY 0x6982 + #define IO_SECRET_KEY 0x02 + + void device_ledger::logCMD() { + if (apdu_verbose) { + char strbuffer[1024]; + snprintf(strbuffer, sizeof(strbuffer), "%.02x %.02x %.02x %.02x %.02x ", + this->buffer_send[0], + this->buffer_send[1], + this->buffer_send[2], + this->buffer_send[3], + this->buffer_send[4] + ); + const size_t len = strlen(strbuffer); + buffer_to_str(strbuffer+len, sizeof(strbuffer)-len, (char*)(this->buffer_send+5), this->length_send-5); + MDEBUG( "CMD : " << strbuffer); + } + } + + void device_ledger::logRESP() { + if (apdu_verbose) { + char strbuffer[1024]; + snprintf(strbuffer, sizeof(strbuffer), "%.04x ", this->sw); + const size_t len = strlen(strbuffer); + buffer_to_str(strbuffer+len, sizeof(strbuffer)-len, (char*)(this->buffer_recv), this->length_recv); + MDEBUG( "RESP : " << strbuffer); + + } + } + + int device_ledger::set_command_header(unsigned char ins, unsigned char p1, unsigned char p2) { + reset_buffer(); + this->buffer_send[0] = PROTOCOL_VERSION; + this->buffer_send[1] = ins; + this->buffer_send[2] = p1; + this->buffer_send[3] = p2; + this->buffer_send[4] = 0x00; + return 5; + } + + int device_ledger::set_command_header_noopt(unsigned char ins, unsigned char p1, unsigned char p2) { + int offset = set_command_header(ins, p1, p2); + //options + this->buffer_send[offset++] = 0; + this->buffer_send[4] = offset - 5; + return offset; + } + + void device_ledger::send_simple(unsigned char ins, unsigned char p1) { + this->length_send = set_command_header_noopt(ins, p1); + if (ins == INS_GET_KEY && p1 == IO_SECRET_KEY) { + // export view key user input + this->exchange_wait_on_input(); + } else { this->exchange(); - return true; + } + } + + bool device_ledger::reset() { + reset_buffer(); + int offset = set_command_header_noopt(INS_RESET); + memmove(this->buffer_send+offset, MONERO_VERSION, strlen(MONERO_VERSION)); + offset += strlen(MONERO_VERSION); + this->buffer_send[4] = offset-5; + this->length_send = offset; + this->exchange(); + + ASSERT_X(this->length_recv>=3, "Communication error, less than three bytes received. Check your application version."); + + unsigned int device_version = 0; + device_version = VERSION(this->buffer_recv[0], this->buffer_recv[1], this->buffer_recv[2]); + + ASSERT_X (device_version >= MINIMAL_APP_VERSION, + "Unsupported device application version: " << VERSION_MAJOR(device_version)<<"."<length_send <= BUFFER_SEND_SIZE); logCMD(); - this->length_recv = BUFFER_RECV_SIZE; - rv = SCardTransmit(this->hCard, - SCARD_PCI_T0, this->buffer_send, this->length_send, - NULL, this->buffer_recv, &this->length_recv); - ASSERT_RV(rv); - ASSERT_T0(this->length_recv <= BUFFER_RECV_SIZE); + + this->length_recv = hw_device.exchange(this->buffer_send, this->length_send, this->buffer_recv, BUFFER_SEND_SIZE, false); + ASSERT_X(this->length_recv>=2, "Communication error, less than tow bytes received"); + + this->length_recv -= 2; + this->sw = (this->buffer_recv[length_recv]<<8) | this->buffer_recv[length_recv+1]; logRESP(); + ASSERT_SW(this->sw,ok,msk); + + return this->sw; + } - sw = (this->buffer_recv[this->length_recv-2]<<8) | this->buffer_recv[this->length_recv-1]; - ASSERT_SW(sw,ok,msk); - return sw; + unsigned int device_ledger::exchange_wait_on_input(unsigned int ok, unsigned int mask) { + logCMD(); + unsigned int deny = 0; + this->length_recv = hw_device.exchange(this->buffer_send, this->length_send, this->buffer_recv, BUFFER_SEND_SIZE, true); + ASSERT_X(this->length_recv>=2, "Communication error, less than two bytes received"); + + this->length_recv -= 2; + this->sw = (this->buffer_recv[length_recv]<<8) | this->buffer_recv[length_recv+1]; + if (this->sw == IO_SW_DENY) { + // cancel on device + deny = 1; + } else { + ASSERT_SW(this->sw,ok,msk); + } + + logRESP(); + return deny; } void device_ledger::reset_buffer() { @@ -318,101 +388,30 @@ namespace hw { } const std::string device_ledger::get_name() const { - if (this->full_name.empty() || (this->hCard == 0)) { + if (!this->connected()) { return std::string("name).append(">"); } - return this->full_name; + return this->name; } bool device_ledger::init(void) { #ifdef DEBUG_HWDEVICE this->controle_device = &hw::get_device("default"); #endif - LONG rv; this->release(); - rv = SCardEstablishContext(SCARD_SCOPE_SYSTEM,0,0, &this->hContext); - ASSERT_RV(rv); - MDEBUG( "Device "<id <<" SCardContext created: hContext="<hContext); - this->hCard = 0; - return true; - } - - bool device_ledger::release() { - this->disconnect(); - if (this->hContext) { - SCardReleaseContext(this->hContext); - MDEBUG( "Device "<id <<" SCardContext released: hContext="<hContext); - this->hContext = 0; - this->full_name.clear(); - } + hw_device.init(); + MDEBUG( "Device "<id <<" HIDUSB inited"); return true; } + + static const std::vector known_devices { + {0x2c97, 0x0001, 0, 0xffa0}, + {0x2c97, 0x0004, 0, 0xffa0}, + }; bool device_ledger::connect(void) { - BYTE pbAtr[MAX_ATR_SIZE]; - LPSTR mszReaders; - DWORD dwReaders; - LONG rv; - DWORD dwState, dwProtocol, dwAtrLen, dwReaderLen; - this->disconnect(); -#ifdef SCARD_AUTOALLOCATE - dwReaders = SCARD_AUTOALLOCATE; - rv = SCardListReaders(this->hContext, NULL, (LPSTR)&mszReaders, &dwReaders); -#else - dwReaders = 0; - rv = SCardListReaders(this->hContext, NULL, NULL, &dwReaders); - if (rv != SCARD_S_SUCCESS) - return false; - mszReaders = (LPSTR)calloc(dwReaders, sizeof(char)); - rv = SCardListReaders(this->hContext, NULL, mszReaders, &dwReaders); -#endif - if (rv == SCARD_S_SUCCESS) { - char* p; - const char* prefix = this->name.c_str(); - - p = mszReaders; - MDEBUG( "Looking for " << std::string(prefix)); - while (*p) { - MDEBUG( "Device Found: " << std::string(p)); - if ((strncmp(prefix, p, strlen(prefix))==0)) { - MDEBUG( "Device Match: " << std::string(p)); - if ((rv = SCardConnect(this->hContext, - p, SCARD_SHARE_EXCLUSIVE, SCARD_PROTOCOL_T0, - &this->hCard, &dwProtocol))!=SCARD_S_SUCCESS) { - break; - } - MDEBUG( "Device "<id <<" Connected: hCard="<hCard); - dwAtrLen = sizeof(pbAtr); - if ((rv = SCardStatus(this->hCard, NULL, &dwReaderLen, &dwState, &dwProtocol, pbAtr, &dwAtrLen))!=SCARD_S_SUCCESS) { - break; - } - MDEBUG( "Device "<id <<" Status OK"); - rv = SCARD_S_SUCCESS ; - this->full_name = std::string(p); - break; - } - p += strlen(p) +1; - } - } - - if (mszReaders) { - #ifdef SCARD_AUTOALLOCATE - SCardFreeMemory(this->hContext, mszReaders); - #else - free(mszReaders); - #endif - mszReaders = NULL; - } - if (rv != SCARD_S_SUCCESS) { - if ( hCard) { - SCardDisconnect(this->hCard, SCARD_UNPOWER_CARD); - MDEBUG( "Device "<id <<" disconnected: hCard="<hCard); - this->hCard = 0; - } - } - ASSERT_RV(rv); - + hw_device.connect(known_devices); this->reset(); #ifdef DEBUG_HWDEVICE cryptonote::account_public_address pubkey; @@ -422,15 +421,21 @@ namespace hw { crypto::secret_key skey; this->get_secret_keys(vkey,skey); - return rv==SCARD_S_SUCCESS; + return true; + } + + bool device_ledger::connected(void) const { + return hw_device.connected(); } bool device_ledger::disconnect() { - if (this->hCard) { - SCardDisconnect(this->hCard, SCARD_UNPOWER_CARD); - MDEBUG( "Device "<id <<" disconnected: hCard="<hCard); - this->hCard = 0; - } + hw_device.disconnect(); + return true; + } + + bool device_ledger::release() { + this->disconnect(); + hw_device.release(); return true; } @@ -439,20 +444,10 @@ namespace hw { int offset; - reset_buffer(); - switch(mode) { case TRANSACTION_CREATE_REAL: case TRANSACTION_CREATE_FAKE: - this->buffer_send[0] = 0x00; - this->buffer_send[1] = INS_SET_SIGNATURE_MODE; - this->buffer_send[2] = 0x01; - this->buffer_send[3] = 0x00; - this->buffer_send[4] = 0x00; - offset = 5; - //options - this->buffer_send[offset] = 0x00; - offset += 1; + offset = set_command_header_noopt(INS_SET_SIGNATURE_MODE, 1); //account this->buffer_send[offset] = mode; offset += 1; @@ -472,7 +467,7 @@ namespace hw { CHECK_AND_ASSERT_THROW_MES(false, " device_ledger::set_mode(unsigned int mode): invalid mode: "<buffer_send[0] = 0x00; - this->buffer_send[1] = INS_GET_KEY; - this->buffer_send[2] = 0x01; - this->buffer_send[3] = 0x00; - this->buffer_send[4] = 0x00; - offset = 5; - //options - this->buffer_send[offset] = 0x00; - offset += 1; - - this->buffer_send[4] = offset-5; - this->length_send = offset; - this->exchange(); + send_simple(INS_GET_KEY, 1); memmove(pubkey.m_view_public_key.data, this->buffer_recv, 32); memmove(pubkey.m_spend_public_key.data, this->buffer_recv+32, 32); @@ -514,22 +494,7 @@ namespace hw { memset(skey.data, 0xFF, 32); //spcialkey, normal conf handled in decrypt - int offset; - reset_buffer(); - - this->buffer_send[0] = 0x00; - this->buffer_send[1] = INS_GET_KEY; - this->buffer_send[2] = 0x02; - this->buffer_send[3] = 0x00; - this->buffer_send[4] = 0x00; - offset = 5; - //options - this->buffer_send[offset] = 0x00; - offset += 1; - - this->buffer_send[4] = offset-5; - this->length_send = offset; - this->exchange(); + send_simple(INS_GET_KEY, 0x02); //View key is retrievied, if allowed, to speed up blockchain parsing memmove(this->viewkey.data, this->buffer_recv+0, 32); @@ -549,34 +514,20 @@ namespace hw { return true; } - bool device_ledger::generate_chacha_key(const cryptonote::account_keys &keys, crypto::chacha_key &key) { + bool device_ledger::generate_chacha_key(const cryptonote::account_keys &keys, crypto::chacha_key &key, uint64_t kdf_rounds) { AUTO_LOCK_CMD(); - int offset; #ifdef DEBUG_HWDEVICE crypto::chacha_key key_x; cryptonote::account_keys keys_x = hw::ledger::decrypt(keys); - this->controle_device->generate_chacha_key(keys_x, key_x); + this->controle_device->generate_chacha_key(keys_x, key_x, kdf_rounds); #endif - reset_buffer(); - this->buffer_send[0] = 0x00; - this->buffer_send[1] = INS_GET_CHACHA8_PREKEY; - this->buffer_send[2] = 0x00; - this->buffer_send[3] = 0x00; - this->buffer_send[4] = 0x00; - offset = 5; - //options - this->buffer_send[offset] = 0x00; - offset += 1; - - this->buffer_send[4] = offset-5; - this->length_send = offset; - this->exchange(); + send_simple(INS_GET_CHACHA8_PREKEY); char prekey[200]; memmove(prekey, &this->buffer_recv[0], 200); - crypto::generate_chacha_key_prehashed(&prekey[0], sizeof(prekey), key); + crypto::generate_chacha_key_prehashed(&prekey[0], sizeof(prekey), key, kdf_rounds); #ifdef DEBUG_HWDEVICE hw::ledger::check32("generate_chacha_key_prehashed", "key", (char*)key_x.data(), (char*)key.data()); @@ -601,11 +552,11 @@ namespace hw { } const std::size_t output_index_x = output_index; crypto::public_key derived_pub_x; - hw::ledger::log_hexbuffer("derive_subaddress_public_key: [[IN]] pub ", pub_x.data, 32); - hw::ledger::log_hexbuffer("derive_subaddress_public_key: [[IN]] derivation", derivation_x.data, 32); - hw::ledger::log_message ("derive_subaddress_public_key: [[IN]] index ", std::to_string((int)output_index_x)); + log_hexbuffer("derive_subaddress_public_key: [[IN]] pub ", pub_x.data, 32); + log_hexbuffer("derive_subaddress_public_key: [[IN]] derivation", derivation_x.data, 32); + log_message ("derive_subaddress_public_key: [[IN]] index ", std::to_string((int)output_index_x)); this->controle_device->derive_subaddress_public_key(pub_x, derivation_x,output_index_x,derived_pub_x); - hw::ledger::log_hexbuffer("derive_subaddress_public_key: [[OUT]] derived_pub", derived_pub_x.data, 32); + log_hexbuffer("derive_subaddress_public_key: [[OUT]] derived_pub", derived_pub_x.data, 32); #endif if ((this->mode == TRANSACTION_PARSE) && has_view_key) { @@ -615,19 +566,7 @@ namespace hw { crypto::derive_subaddress_public_key(pub, derivation, output_index,derived_pub); } else { - int offset =0; - - reset_buffer(); - - this->buffer_send[0] = 0x00; - this->buffer_send[1] = INS_DERIVE_SUBADDRESS_PUBLIC_KEY; - this->buffer_send[2] = 0x00; - this->buffer_send[3] = 0x00; - this->buffer_send[4] = 0x00; - offset = 5; - //options - this->buffer_send[offset] = 0; - offset += 1; + int offset = set_command_header_noopt(INS_DERIVE_SUBADDRESS_PUBLIC_KEY); //pub memmove(this->buffer_send+offset, pub.data, 32); offset += 32; @@ -658,34 +597,23 @@ namespace hw { crypto::public_key device_ledger::get_subaddress_spend_public_key(const cryptonote::account_keys& keys, const cryptonote::subaddress_index &index) { AUTO_LOCK_CMD(); crypto::public_key D; - int offset; #ifdef DEBUG_HWDEVICE const cryptonote::account_keys keys_x = hw::ledger::decrypt(keys); const cryptonote::subaddress_index index_x = index; crypto::public_key D_x; - hw::ledger::log_hexbuffer("get_subaddress_spend_public_key: [[IN]] keys.m_view_secret_key ", keys_x.m_view_secret_key.data,32); - hw::ledger::log_hexbuffer("get_subaddress_spend_public_key: [[IN]] keys.m_spend_secret_key", keys_x.m_spend_secret_key.data,32); - hw::ledger::log_message ("get_subaddress_spend_public_key: [[IN]] index ", std::to_string(index_x.major)+"."+std::to_string(index_x.minor)); + log_hexbuffer("get_subaddress_spend_public_key: [[IN]] keys.m_view_secret_key ", keys_x.m_view_secret_key.data,32); + log_hexbuffer("get_subaddress_spend_public_key: [[IN]] keys.m_spend_secret_key", keys_x.m_spend_secret_key.data,32); + log_message ("get_subaddress_spend_public_key: [[IN]] index ", std::to_string(index_x.major)+"."+std::to_string(index_x.minor)); D_x = this->controle_device->get_subaddress_spend_public_key(keys_x, index_x); - hw::ledger::log_hexbuffer("get_subaddress_spend_public_key: [[OUT]] derivation ", D_x.data, 32); + log_hexbuffer("get_subaddress_spend_public_key: [[OUT]] derivation ", D_x.data, 32); #endif if (index.is_zero()) { D = keys.m_account_address.m_spend_public_key; } else { - reset_buffer(); - - this->buffer_send[0] = 0x00; - this->buffer_send[1] = INS_GET_SUBADDRESS_SPEND_PUBLIC_KEY; - this->buffer_send[2] = 0x00; - this->buffer_send[3] = 0x00; - this->buffer_send[4] = 0x00; - offset = 5; - //options - this->buffer_send[offset] = 0x00; - offset += 1; + int offset = set_command_header_noopt(INS_GET_SUBADDRESS_SPEND_PUBLIC_KEY); //index static_assert(sizeof(cryptonote::subaddress_index) == 8, "cryptonote::subaddress_index shall be 8 bytes length"); memmove(this->buffer_send+offset, &index, sizeof(cryptonote::subaddress_index)); @@ -720,36 +648,25 @@ namespace hw { cryptonote::account_public_address device_ledger::get_subaddress(const cryptonote::account_keys& keys, const cryptonote::subaddress_index &index) { AUTO_LOCK_CMD(); cryptonote::account_public_address address; - int offset; #ifdef DEBUG_HWDEVICE const cryptonote::account_keys keys_x = hw::ledger::decrypt(keys); const cryptonote::subaddress_index index_x = index; cryptonote::account_public_address address_x; - hw::ledger::log_hexbuffer("get_subaddress: [[IN]] keys.m_view_secret_key ", keys_x.m_view_secret_key.data, 32); - hw::ledger::log_hexbuffer("get_subaddress: [[IN]] keys.m_view_public_key", keys_x.m_account_address.m_view_public_key.data, 32); - hw::ledger::log_hexbuffer("get_subaddress: [[IN]] keys.m_view_secret_key ", keys_x.m_view_secret_key.data, 32); - hw::ledger::log_hexbuffer("get_subaddress: [[IN]] keys.m_spend_public_key", keys_x.m_account_address.m_spend_public_key.data, 32); - hw::ledger::log_message ("get_subaddress: [[IN]] index ", std::to_string(index_x.major)+"."+std::to_string(index_x.minor)); + log_hexbuffer("get_subaddress: [[IN]] keys.m_view_secret_key ", keys_x.m_view_secret_key.data, 32); + log_hexbuffer("get_subaddress: [[IN]] keys.m_view_public_key", keys_x.m_account_address.m_view_public_key.data, 32); + log_hexbuffer("get_subaddress: [[IN]] keys.m_view_secret_key ", keys_x.m_view_secret_key.data, 32); + log_hexbuffer("get_subaddress: [[IN]] keys.m_spend_public_key", keys_x.m_account_address.m_spend_public_key.data, 32); + log_message ("get_subaddress: [[IN]] index ", std::to_string(index_x.major)+"."+std::to_string(index_x.minor)); address_x = this->controle_device->get_subaddress(keys_x, index_x); - hw::ledger::log_hexbuffer("get_subaddress: [[OUT]] keys.m_view_public_key ", address_x.m_view_public_key.data, 32); - hw::ledger::log_hexbuffer("get_subaddress: [[OUT]] keys.m_spend_public_key", address_x.m_spend_public_key.data, 32); + log_hexbuffer("get_subaddress: [[OUT]] keys.m_view_public_key ", address_x.m_view_public_key.data, 32); + log_hexbuffer("get_subaddress: [[OUT]] keys.m_spend_public_key", address_x.m_spend_public_key.data, 32); #endif if (index.is_zero()) { address = keys.m_account_address; } else { - reset_buffer(); - - this->buffer_send[0] = 0x00; - this->buffer_send[1] = INS_GET_SUBADDRESS; - this->buffer_send[2] = 0x00; - this->buffer_send[3] = 0x00; - this->buffer_send[4] = 0x00; - offset = 5; - //options - this->buffer_send[offset] = 0x00; - offset += 1; + int offset = set_command_header_noopt(INS_GET_SUBADDRESS); //index static_assert(sizeof(cryptonote::subaddress_index) == 8, "cryptonote::subaddress_index shall be 8 bytes length"); memmove(this->buffer_send+offset, &index, sizeof(cryptonote::subaddress_index)); @@ -774,29 +691,18 @@ namespace hw { crypto::secret_key device_ledger::get_subaddress_secret_key(const crypto::secret_key &sec, const cryptonote::subaddress_index &index) { AUTO_LOCK_CMD(); crypto::secret_key sub_sec; - int offset; #ifdef DEBUG_HWDEVICE const crypto::secret_key sec_x = hw::ledger::decrypt(sec); const cryptonote::subaddress_index index_x = index; crypto::secret_key sub_sec_x; - hw::ledger::log_message ("get_subaddress_secret_key: [[IN]] index ", std::to_string(index.major)+"."+std::to_string(index.minor)); - hw::ledger::log_hexbuffer("get_subaddress_secret_key: [[IN]] sec ", sec_x.data, 32); + log_message ("get_subaddress_secret_key: [[IN]] index ", std::to_string(index.major)+"."+std::to_string(index.minor)); + log_hexbuffer("get_subaddress_secret_key: [[IN]] sec ", sec_x.data, 32); sub_sec_x = this->controle_device->get_subaddress_secret_key(sec_x, index_x); - hw::ledger::log_hexbuffer("get_subaddress_secret_key: [[OUT]] sub_sec", sub_sec_x.data, 32); + log_hexbuffer("get_subaddress_secret_key: [[OUT]] sub_sec", sub_sec_x.data, 32); #endif - reset_buffer(); - - this->buffer_send[0] = 0x00; - this->buffer_send[1] = INS_GET_SUBADDRESS_SECRET_KEY; - this->buffer_send[2] = 0x00; - this->buffer_send[3] = 0x00; - this->buffer_send[4] = 0x00; - offset = 5; - //options - this->buffer_send[offset] = 0; - offset += 1; + int offset = set_command_header_noopt(INS_GET_SUBADDRESS_SECRET_KEY); //sec memmove(this->buffer_send+offset, sec.data, 32); offset += 32; @@ -825,18 +731,9 @@ namespace hw { bool device_ledger::verify_keys(const crypto::secret_key &secret_key, const crypto::public_key &public_key) { AUTO_LOCK_CMD(); - int offset, sw; - - reset_buffer(); - this->buffer_send[0] = 0x00; - this->buffer_send[1] = INS_VERIFY_KEY; - this->buffer_send[2] = 0x00; - this->buffer_send[3] = 0x00; - this->buffer_send[4] = 0x00; - offset = 5; - //options - this->buffer_send[offset] = 0x00; - offset += 1; + int offset; + + offset = set_command_header_noopt(INS_VERIFY_KEY); //sec memmove(this->buffer_send+offset, secret_key.data, 32); offset += 32; @@ -859,29 +756,18 @@ namespace hw { bool device_ledger::scalarmultKey(rct::key & aP, const rct::key &P, const rct::key &a) { AUTO_LOCK_CMD(); - int offset; #ifdef DEBUG_HWDEVICE const rct::key P_x = P; const rct::key a_x = hw::ledger::decrypt(a); rct::key aP_x; - hw::ledger::log_hexbuffer("scalarmultKey: [[IN]] P ", (char*)P_x.bytes, 32); - hw::ledger::log_hexbuffer("scalarmultKey: [[IN]] a ", (char*)a_x.bytes, 32); + log_hexbuffer("scalarmultKey: [[IN]] P ", (char*)P_x.bytes, 32); + log_hexbuffer("scalarmultKey: [[IN]] a ", (char*)a_x.bytes, 32); this->controle_device->scalarmultKey(aP_x, P_x, a_x); - hw::ledger::log_hexbuffer("scalarmultKey: [[OUT]] aP", (char*)aP_x.bytes, 32); + log_hexbuffer("scalarmultKey: [[OUT]] aP", (char*)aP_x.bytes, 32); #endif - reset_buffer(); - - this->buffer_send[0] = 0x00; - this->buffer_send[1] = INS_SECRET_SCAL_MUL_KEY; - this->buffer_send[2] = 0x00; - this->buffer_send[3] = 0x00; - this->buffer_send[4] = 0x00; - offset = 5; - //options - this->buffer_send[offset] = 0; - offset += 1; + int offset = set_command_header_noopt(INS_SECRET_SCAL_MUL_KEY); //pub memmove(this->buffer_send+offset, P.bytes, 32); offset += 32; @@ -906,27 +792,16 @@ namespace hw { bool device_ledger::scalarmultBase(rct::key &aG, const rct::key &a) { AUTO_LOCK_CMD(); - int offset; #ifdef DEBUG_HWDEVICE const rct::key a_x = hw::ledger::decrypt(a); rct::key aG_x; - hw::ledger::log_hexbuffer("scalarmultKey: [[IN]] a ", (char*)a_x.bytes, 32); + log_hexbuffer("scalarmultKey: [[IN]] a ", (char*)a_x.bytes, 32); this->controle_device->scalarmultBase(aG_x, a_x); - hw::ledger::log_hexbuffer("scalarmultKey: [[OUT]] aG", (char*)aG_x.bytes, 32); + log_hexbuffer("scalarmultKey: [[OUT]] aG", (char*)aG_x.bytes, 32); #endif - reset_buffer(); - - this->buffer_send[0] = 0x00; - this->buffer_send[1] = INS_SECRET_SCAL_MUL_BASE; - this->buffer_send[2] = 0x00; - this->buffer_send[3] = 0x00; - this->buffer_send[4] = 0x00; - offset = 5; - //options - this->buffer_send[offset] = 0; - offset += 1; + int offset = set_command_header_noopt(INS_SECRET_SCAL_MUL_BASE); //sec memmove(this->buffer_send+offset, a.bytes, 32); offset += 32; @@ -947,26 +822,19 @@ namespace hw { bool device_ledger::sc_secret_add( crypto::secret_key &r, const crypto::secret_key &a, const crypto::secret_key &b) { AUTO_LOCK_CMD(); - int offset; #ifdef DEBUG_HWDEVICE const crypto::secret_key a_x = hw::ledger::decrypt(a); const crypto::secret_key b_x = hw::ledger::decrypt(b); crypto::secret_key r_x; + rct::key aG_x; + log_hexbuffer("sc_secret_add: [[IN]] a ", (char*)a_x.data, 32); + log_hexbuffer("sc_secret_add: [[IN]] b ", (char*)b_x.data, 32); this->controle_device->sc_secret_add(r_x, a_x, b_x); + log_hexbuffer("sc_secret_add: [[OUT]] aG", (char*)r_x.data, 32); #endif - reset_buffer(); - - this->buffer_send[0] = 0x00; - this->buffer_send[1] = INS_SECRET_KEY_ADD; - this->buffer_send[2] = 0x00; - this->buffer_send[3] = 0x00; - this->buffer_send[4] = 0x00; - offset = 5; - //options - this->buffer_send[offset] = 0; - offset += 1; + int offset = set_command_header_noopt(INS_SECRET_KEY_ADD); //sec key memmove(this->buffer_send+offset, a.data, 32); offset += 32; @@ -995,30 +863,17 @@ namespace hw { throw std::runtime_error("device generate key does not support recover"); } - int offset; - #ifdef DEBUG_HWDEVICE - bool recover_x = recover; - const crypto::secret_key recovery_key_x = recovery_key; crypto::public_key pub_x; crypto::secret_key sec_x; + crypto::secret_key recovery_key_x; + if (recover) { + recovery_key_x = hw::ledger::decrypt(recovery_key); + log_hexbuffer("generate_keys: [[IN]] pub", (char*)recovery_key_x.data, 32); + } #endif - reset_buffer(); - - this->buffer_send[0] = 0x00; - this->buffer_send[1] = INS_GENERATE_KEYPAIR; - this->buffer_send[2] = 0x00; - this->buffer_send[3] = 0x00; - this->buffer_send[4] = 0x00; - offset = 5; - //options - this->buffer_send[offset] = 0; - offset += 1; - - this->buffer_send[4] = offset-5; - this->length_send = offset; - this->exchange(); + send_simple(INS_GENERATE_KEYPAIR); //pub key memmove(pub.data, &this->buffer_recv[0], 32); @@ -1027,6 +882,9 @@ namespace hw { #ifdef DEBUG_HWDEVICE crypto::secret_key sec_clear = hw::ledger::decrypt(sec); sec_x = sec_clear; + log_hexbuffer("generate_keys: [[OUT]] pub", (char*)pub.data, 32); + log_hexbuffer("generate_keys: [[OUT]] sec", (char*)sec_clear.data, 32); + crypto::secret_key_to_public_key(sec_x,pub_x); hw::ledger::check32("generate_keys", "pub", pub_x.data, pub.data); #endif @@ -1041,12 +899,12 @@ namespace hw { #ifdef DEBUG_HWDEVICE const crypto::public_key pub_x = pub; - const crypto::secret_key sec_x = hw::ledger::decrypt(sec); + const crypto::secret_key sec_x = (sec == rct::rct2sk(rct::I)) ? sec: hw::ledger::decrypt(sec); crypto::key_derivation derivation_x; - hw::ledger::log_hexbuffer("generate_key_derivation: [[IN]] pub ", pub_x.data, 32); - hw::ledger::log_hexbuffer("generate_key_derivation: [[IN]] sec ", sec_x.data, 32); + log_hexbuffer("generate_key_derivation: [[IN]] pub ", pub_x.data, 32); + log_hexbuffer("generate_key_derivation: [[IN]] sec ", sec_x.data, 32); this->controle_device->generate_key_derivation(pub_x, sec_x, derivation_x); - hw::ledger::log_hexbuffer("generate_key_derivation: [[OUT]] derivation", derivation_x.data, 32); + log_hexbuffer("generate_key_derivation: [[OUT]] derivation", derivation_x.data, 32); #endif if ((this->mode == TRANSACTION_PARSE) && has_view_key) { @@ -1057,20 +915,7 @@ namespace hw { assert(is_fake_view_key(sec)); r = crypto::generate_key_derivation(pub, this->viewkey, derivation); } else { - - int offset; - - reset_buffer(); - - this->buffer_send[0] = 0x00; - this->buffer_send[1] = INS_GEN_KEY_DERIVATION; - this->buffer_send[2] = 0x00; - this->buffer_send[3] = 0x00; - this->buffer_send[4] = 0x00; - offset = 5; - //options - this->buffer_send[offset] = 0; - offset += 1; + int offset = set_command_header_noopt(INS_GEN_KEY_DERIVATION); //pub memmove(this->buffer_send+offset, pub.data, 32); offset += 32; @@ -1088,11 +933,11 @@ namespace hw { } #ifdef DEBUG_HWDEVICE crypto::key_derivation derivation_clear ; - if ((this->mode == TRANSACTION_PARSE) && has_view_key) { - derivation_clear = derivation; - }else { - derivation_clear = hw::ledger::decrypt(derivation); - } + if ((this->mode == TRANSACTION_PARSE) && has_view_key) { + derivation_clear = derivation; + } else { + derivation_clear = hw::ledger::decrypt(derivation); + } hw::ledger::check32("generate_key_derivation", "derivation", derivation_x.data, derivation_clear.data); #endif @@ -1119,29 +964,18 @@ namespace hw { bool device_ledger::derivation_to_scalar(const crypto::key_derivation &derivation, const size_t output_index, crypto::ec_scalar &res) { AUTO_LOCK_CMD(); - int offset; #ifdef DEBUG_HWDEVICE const crypto::key_derivation derivation_x = hw::ledger::decrypt(derivation); const size_t output_index_x = output_index; crypto::ec_scalar res_x; - hw::ledger::log_hexbuffer("derivation_to_scalar: [[IN]] derivation ", derivation_x.data, 32); - hw::ledger::log_message ("derivation_to_scalar: [[IN]] output_index ", std::to_string(output_index_x)); + log_hexbuffer("derivation_to_scalar: [[IN]] derivation ", derivation_x.data, 32); + log_message ("derivation_to_scalar: [[IN]] output_index ", std::to_string(output_index_x)); this->controle_device->derivation_to_scalar(derivation_x, output_index_x, res_x); - hw::ledger::log_hexbuffer("derivation_to_scalar: [[OUT]] res ", res_x.data, 32); + log_hexbuffer("derivation_to_scalar: [[OUT]] res ", res_x.data, 32); #endif - reset_buffer(); - - this->buffer_send[0] = 0x00; - this->buffer_send[1] = INS_DERIVATION_TO_SCALAR; - this->buffer_send[2] = 0x00; - this->buffer_send[3] = 0x00; - this->buffer_send[4] = 0x00; - offset = 5; - //options - this->buffer_send[offset] = 0; - offset += 1; + int offset = set_command_header_noopt(INS_DERIVATION_TO_SCALAR); //derivattion memmove(this->buffer_send+offset, derivation.data, 32); offset += 32; @@ -1169,31 +1003,20 @@ namespace hw { bool device_ledger::derive_secret_key(const crypto::key_derivation &derivation, const std::size_t output_index, const crypto::secret_key &sec, crypto::secret_key &derived_sec) { AUTO_LOCK_CMD(); - int offset; #ifdef DEBUG_HWDEVICE const crypto::key_derivation derivation_x = hw::ledger::decrypt(derivation); const std::size_t output_index_x = output_index; const crypto::secret_key sec_x = hw::ledger::decrypt(sec); crypto::secret_key derived_sec_x; - hw::ledger::log_hexbuffer("derive_secret_key: [[IN]] derivation ", derivation_x.data, 32); - hw::ledger::log_message ("derive_secret_key: [[IN]] index ", std::to_string(output_index_x)); - hw::ledger::log_hexbuffer("derive_secret_key: [[IN]] sec ", sec_x.data, 32); + log_hexbuffer("derive_secret_key: [[IN]] derivation ", derivation_x.data, 32); + log_message ("derive_secret_key: [[IN]] index ", std::to_string(output_index_x)); + log_hexbuffer("derive_secret_key: [[IN]] sec ", sec_x.data, 32); this->controle_device->derive_secret_key(derivation_x, output_index_x, sec_x, derived_sec_x); - hw::ledger::log_hexbuffer("derive_secret_key: [[OUT]] derived_sec", derived_sec_x.data, 32); + log_hexbuffer("derive_secret_key: [[OUT]] derived_sec", derived_sec_x.data, 32); #endif - reset_buffer(); - - this->buffer_send[0] = 0x00; - this->buffer_send[1] = INS_DERIVE_SECRET_KEY; - this->buffer_send[2] = 0x00; - this->buffer_send[3] = 0x00; - this->buffer_send[4] = 0x00; - offset = 5; - //options - this->buffer_send[offset] = 0; - offset += 1; + int offset = set_command_header_noopt(INS_DERIVE_SECRET_KEY); //derivation memmove(this->buffer_send+offset, derivation.data, 32); offset += 32; @@ -1224,31 +1047,20 @@ namespace hw { bool device_ledger::derive_public_key(const crypto::key_derivation &derivation, const std::size_t output_index, const crypto::public_key &pub, crypto::public_key &derived_pub){ AUTO_LOCK_CMD(); - int offset; #ifdef DEBUG_HWDEVICE const crypto::key_derivation derivation_x = hw::ledger::decrypt(derivation); const std::size_t output_index_x = output_index; const crypto::public_key pub_x = pub; crypto::public_key derived_pub_x; - hw::ledger::log_hexbuffer("derive_public_key: [[IN]] derivation ", derivation_x.data, 32); - hw::ledger::log_message ("derive_public_key: [[IN]] output_index", std::to_string(output_index_x)); - hw::ledger::log_hexbuffer("derive_public_key: [[IN]] pub ", pub_x.data, 32); + log_hexbuffer("derive_public_key: [[IN]] derivation ", derivation_x.data, 32); + log_message ("derive_public_key: [[IN]] output_index", std::to_string(output_index_x)); + log_hexbuffer("derive_public_key: [[IN]] pub ", pub_x.data, 32); this->controle_device->derive_public_key(derivation_x, output_index_x, pub_x, derived_pub_x); - hw::ledger::log_hexbuffer("derive_public_key: [[OUT]] derived_pub ", derived_pub_x.data, 32); + log_hexbuffer("derive_public_key: [[OUT]] derived_pub ", derived_pub_x.data, 32); #endif - reset_buffer(); - - this->buffer_send[0] = 0x00; - this->buffer_send[1] = INS_DERIVE_PUBLIC_KEY; - this->buffer_send[2] = 0x00; - this->buffer_send[3] = 0x00; - this->buffer_send[4] = 0x00; - offset = 5; - //options - this->buffer_send[offset] = 0; - offset += 1; + int offset = set_command_header_noopt(INS_DERIVE_PUBLIC_KEY); //derivation memmove(this->buffer_send+offset, derivation.data, 32); offset += 32; @@ -1278,30 +1090,19 @@ namespace hw { bool device_ledger::secret_key_to_public_key(const crypto::secret_key &sec, crypto::public_key &pub) { AUTO_LOCK_CMD(); - int offset; #ifdef DEBUG_HWDEVICE const crypto::secret_key sec_x = hw::ledger::decrypt(sec); crypto::public_key pub_x; - hw::ledger::log_hexbuffer("secret_key_to_public_key: [[IN]] sec ", sec_x.data, 32); + log_hexbuffer("secret_key_to_public_key: [[IN]] sec ", sec_x.data, 32); bool rc = this->controle_device->secret_key_to_public_key(sec_x, pub_x); - hw::ledger::log_hexbuffer("secret_key_to_public_key: [[OUT]] pub", pub_x.data, 32); + log_hexbuffer("secret_key_to_public_key: [[OUT]] pub", pub_x.data, 32); if (!rc){ - hw::ledger::log_message("secret_key_to_public_key", "secret_key rejected"); + log_message("FAIL secret_key_to_public_key", "secret_key rejected"); } #endif - reset_buffer(); - - this->buffer_send[0] = 0x00; - this->buffer_send[1] = INS_SECRET_KEY_TO_PUBLIC_KEY; - this->buffer_send[2] = 0x00; - this->buffer_send[3] = 0x00; - this->buffer_send[4] = 0x00; - offset = 5; - //options - this->buffer_send[offset] = 0; - offset += 1; + int offset = set_command_header_noopt(INS_SECRET_KEY_TO_PUBLIC_KEY); //sec key memmove(this->buffer_send+offset, sec.data, 32); offset += 32; @@ -1322,29 +1123,18 @@ namespace hw { bool device_ledger::generate_key_image(const crypto::public_key &pub, const crypto::secret_key &sec, crypto::key_image &image){ AUTO_LOCK_CMD(); - int offset; #ifdef DEBUG_HWDEVICE const crypto::public_key pub_x = pub; const crypto::secret_key sec_x = hw::ledger::decrypt(sec); crypto::key_image image_x; - hw::ledger::log_hexbuffer("generate_key_image: [[IN]] pub ", pub_x.data, 32); - hw::ledger::log_hexbuffer("generate_key_image: [[IN]] sec ", sec_x.data, 32); + log_hexbuffer("generate_key_image: [[IN]] pub ", pub_x.data, 32); + log_hexbuffer("generate_key_image: [[IN]] sec ", sec_x.data, 32); this->controle_device->generate_key_image(pub_x, sec_x, image_x); - hw::ledger::log_hexbuffer("generate_key_image: [[OUT]] image ", image_x.data, 32); + log_hexbuffer("generate_key_image: [[OUT]] image ", image_x.data, 32); #endif - reset_buffer(); - - this->buffer_send[0] = 0x00; - this->buffer_send[1] = INS_GEN_KEY_IMAGE; - this->buffer_send[2] = 0x00; - this->buffer_send[3] = 0x00; - this->buffer_send[4] = 0x00; - offset = 5; - //options - this->buffer_send[offset] = 0; - offset += 1; + int offset = set_command_header_noopt(INS_GEN_KEY_IMAGE); //pub memmove(this->buffer_send+offset, pub.data, 32); offset += 32; @@ -1370,22 +1160,81 @@ namespace hw { /* TRANSACTION */ /* ======================================================================= */ + void device_ledger::generate_tx_proof(const crypto::hash &prefix_hash, + const crypto::public_key &R, const crypto::public_key &A, const boost::optional &B, const crypto::public_key &D, const crypto::secret_key &r, + crypto::signature &sig) { + + AUTO_LOCK_CMD(); + + #ifdef DEBUG_HWDEVICE + const crypto::hash prefix_hash_x = prefix_hash; + const crypto::public_key R_x = R; + const crypto::public_key A_x = A; + const boost::optional B_x = B; + const crypto::public_key D_x = D; + const crypto::secret_key r_x = hw::ledger::decrypt(r); + crypto::signature sig_x; + log_hexbuffer("generate_tx_proof: [[IN]] prefix_hash ", prefix_hash_x.data, 32); + log_hexbuffer("generate_tx_proof: [[IN]] R ", R_x.data, 32); + log_hexbuffer("generate_tx_proof: [[IN]] A ", A_x.data, 32); + if (B_x) { + log_hexbuffer("generate_tx_proof: [[IN]] B ", (*B_x).data, 32); + } + log_hexbuffer("generate_tx_proof: [[IN]] D ", D_x.data, 32); + log_hexbuffer("generate_tx_proof: [[IN]] r ", r_x.data, 32); + #endif + + + int offset = set_command_header(INS_GET_TX_PROOF); + //options + this->buffer_send[offset] = B?0x01:0x00; + offset += 1; + //prefix_hash + memmove(&this->buffer_send[offset], prefix_hash.data, 32); + offset += 32; + // R + memmove(&this->buffer_send[offset], R.data, 32); + offset += 32; + // A + memmove(&this->buffer_send[offset], A.data, 32); + offset += 32; + // B + if (B) { + memmove(&this->buffer_send[offset], (*B).data, 32); + } else { + memset(&this->buffer_send[offset], 0, 32); + } + offset += 32; + // D + memmove(&this->buffer_send[offset], D.data, 32); + offset += 32; + // r + memmove(&this->buffer_send[offset], r.data, 32); + offset += 32; + + this->buffer_send[4] = offset-5; + this->length_send = offset; + this->exchange(); + + memmove(sig.c.data, &this->buffer_recv[0], 32); + memmove(sig.r.data, &this->buffer_recv[32], 32); + #ifdef DEBUG_HWDEVICE + log_hexbuffer("GENERATE_TX_PROOF: **c** ", sig.c.data, sizeof( sig.c.data)); + log_hexbuffer("GENERATE_TX_PROOF: **r** ", sig.r.data, sizeof( sig.r.data)); + + this->controle_device->generate_tx_proof(prefix_hash_x, R_x, A_x, B_x, D_x, r_x, sig_x); + hw::ledger::check32("generate_tx_proof", "c", sig_x.c.data, sig.c.data); + hw::ledger::check32("generate_tx_proof", "r", sig_x.r.data, sig.r.data); + + #endif + } + bool device_ledger::open_tx(crypto::secret_key &tx_key) { AUTO_LOCK_CMD(); - int offset; - reset_buffer(); key_map.clear(); + int offset = set_command_header_noopt(INS_OPEN_TX, 0x01); - this->buffer_send[0] = 0x00; - this->buffer_send[1] = INS_OPEN_TX; - this->buffer_send[2] = 0x01; - this->buffer_send[3] = 0x00; - this->buffer_send[4] = 0x00; - offset = 5; - //options - this->buffer_send[offset] = 0x00; - offset += 1; //account this->buffer_send[offset+0] = 0x00; this->buffer_send[offset+1] = 0x00; @@ -1398,32 +1247,29 @@ namespace hw { this->exchange(); memmove(tx_key.data, &this->buffer_recv[32], 32); - + #ifdef DEBUG_HWDEVICE + const crypto::secret_key r_x = hw::ledger::decrypt(tx_key); + log_hexbuffer("open_tx: [[OUT]] R ", (char*)&this->buffer_recv[0], 32); + log_hexbuffer("open_tx: [[OUT]] r ", r_x.data, 32); + #endif return true; } bool device_ledger::encrypt_payment_id(crypto::hash8 &payment_id, const crypto::public_key &public_key, const crypto::secret_key &secret_key) { AUTO_LOCK_CMD(); - int offset; #ifdef DEBUG_HWDEVICE const crypto::public_key public_key_x = public_key; const crypto::secret_key secret_key_x = hw::ledger::decrypt(secret_key); crypto::hash8 payment_id_x = payment_id; + log_hexbuffer("encrypt_payment_id: [[IN]] payment_id ", payment_id_x.data, 32); + log_hexbuffer("encrypt_payment_id: [[IN]] public_key ", public_key_x.data, 32); + log_hexbuffer("encrypt_payment_id: [[IN]] secret_key ", secret_key_x.data, 32); this->controle_device->encrypt_payment_id(payment_id_x, public_key_x, secret_key_x); + log_hexbuffer("encrypt_payment_id: [[OUT]] payment_id ", payment_id_x.data, 32); #endif - reset_buffer(); - - this->buffer_send[0] = 0x00; - this->buffer_send[1] = INS_STEALTH; - this->buffer_send[2] = 0x00; - this->buffer_send[3] = 0x00; - this->buffer_send[4] = 0x00; - offset = 5; - //options - this->buffer_send[offset] = 0x00; - offset += 1; + int offset = set_command_header_noopt(INS_STEALTH); //pub memmove(&this->buffer_send[offset], public_key.data, 32); offset += 32; @@ -1446,34 +1292,205 @@ namespace hw { return true; } - bool device_ledger::add_output_key_mapping(const crypto::public_key &Aout, const crypto::public_key &Bout, const bool is_subaddress, const size_t real_output_index, - const rct::key &amount_key, const crypto::public_key &out_eph_public_key) { - AUTO_LOCK_CMD(); - key_map.add(ABPkeys(rct::pk2rct(Aout),rct::pk2rct(Bout), is_subaddress, real_output_index, rct::pk2rct(out_eph_public_key), amount_key)); + + bool device_ledger::generate_output_ephemeral_keys(const size_t tx_version, const cryptonote::account_keys &sender_account_keys, const crypto::public_key &txkey_pub, const crypto::secret_key &tx_key, + const cryptonote::tx_destination_entry &dst_entr, const boost::optional &change_addr, const size_t output_index, + const bool &need_additional_txkeys, const std::vector &additional_tx_keys, + std::vector &additional_tx_public_keys, + std::vector &amount_keys, + crypto::public_key &out_eph_public_key) { + AUTO_LOCK_CMD(); + + #ifdef DEBUG_HWDEVICE + const size_t &tx_version_x = tx_version; + const cryptonote::account_keys sender_account_keys_x = hw::ledger::decrypt(sender_account_keys); + memmove((void*)sender_account_keys_x.m_view_secret_key.data, dbg_viewkey.data, 32); + + const crypto::public_key txkey_pub_x = txkey_pub; + const crypto::secret_key tx_key_x = hw::ledger::decrypt(tx_key); + const cryptonote::tx_destination_entry dst_entr_x = dst_entr; + const boost::optional change_addr_x = change_addr; + const size_t output_index_x = output_index; + const bool need_additional_txkeys_x = need_additional_txkeys; + + std::vector additional_tx_keys_x; + for (const auto k: additional_tx_keys) { + additional_tx_keys_x.push_back(hw::ledger::decrypt(k)); + } + + std::vector additional_tx_public_keys_x; + std::vector amount_keys_x; + crypto::public_key out_eph_public_key_x; + + log_message ("generate_output_ephemeral_keys: [[IN]] tx_version", std::to_string(tx_version_x)); + //log_hexbuffer("generate_output_ephemeral_keys: [[IN]] sender_account_keys.view", sender_account_keys.m_sview_secret_key.data, 32); + //log_hexbuffer("generate_output_ephemeral_keys: [[IN]] sender_account_keys.spend", sender_account_keys.m_spend_secret_key.data, 32); + log_hexbuffer("generate_output_ephemeral_keys: [[IN]] txkey_pub", txkey_pub_x.data, 32); + log_hexbuffer("generate_output_ephemeral_keys: [[IN]] tx_key", tx_key_x.data, 32); + log_hexbuffer("generate_output_ephemeral_keys: [[IN]] dst_entr.view", dst_entr_x.addr.m_view_public_key.data, 32); + log_hexbuffer("generate_output_ephemeral_keys: [[IN]] dst_entr.spend", dst_entr_x.addr.m_spend_public_key.data, 32); + if (change_addr) { + log_hexbuffer("generate_output_ephemeral_keys: [[IN]] change_addr.view", (*change_addr_x).m_view_public_key.data, 32); + log_hexbuffer("generate_output_ephemeral_keys: [[IN]] change_addr.spend", (*change_addr_x).m_spend_public_key.data, 32); + } + log_message ("generate_output_ephemeral_keys: [[IN]] output_index", std::to_string(output_index_x)); + log_message ("generate_output_ephemeral_keys: [[IN]] need_additional_txkeys", std::to_string(need_additional_txkeys_x)); + if(need_additional_txkeys_x) { + log_hexbuffer("generate_output_ephemeral_keys: [[IN]] additional_tx_keys[oi]", additional_tx_keys_x[output_index].data, 32); + } + this->controle_device->generate_output_ephemeral_keys(tx_version_x, sender_account_keys_x, txkey_pub_x, tx_key_x, dst_entr_x, change_addr_x, output_index_x, need_additional_txkeys_x, additional_tx_keys_x, + additional_tx_public_keys_x, amount_keys_x, out_eph_public_key_x); + if(need_additional_txkeys_x) { + log_hexbuffer("additional_tx_public_keys_x: [[OUT]] additional_tx_public_keys_x", additional_tx_public_keys_x.back().data, 32); + } + log_hexbuffer("generate_output_ephemeral_keys: [[OUT]] amount_keys ", (char*)amount_keys_x.back().bytes, 32); + log_hexbuffer("generate_output_ephemeral_keys: [[OUT]] out_eph_public_key ", out_eph_public_key_x.data, 32); + #endif + + ASSERT_X(tx_version > 1, "TX version not supported"<buffer_send[offset+0] = tx_version>>24; + this->buffer_send[offset+1] = tx_version>>16; + this->buffer_send[offset+2] = tx_version>>8; + this->buffer_send[offset+3] = tx_version>>0; + offset += 4; + //tx_key + memmove(&this->buffer_send[offset], tx_key.data, 32); + offset += 32; + //txkey_pub + memmove(&this->buffer_send[offset], txkey_pub.data, 32); + offset += 32; + //Aout + memmove(&this->buffer_send[offset], dst_entr.addr.m_view_public_key.data, 32); + offset += 32; + //Bout + memmove(&this->buffer_send[offset], dst_entr.addr.m_spend_public_key.data, 32); + offset += 32; + //output index + this->buffer_send[offset+0] = output_index>>24; + this->buffer_send[offset+1] = output_index>>16; + this->buffer_send[offset+2] = output_index>>8; + this->buffer_send[offset+3] = output_index>>0; + offset += 4; + //is_change, + bool is_change = (change_addr && dst_entr.addr == *change_addr); + this->buffer_send[offset] = is_change; + offset++; + //is_subaddress + this->buffer_send[offset] = dst_entr.is_subaddress; + offset++; + //need_additional_key + this->buffer_send[offset] = need_additional_txkeys; + offset++; + //additional_tx_key + if (need_additional_txkeys) { + memmove(&this->buffer_send[offset], additional_txkey.sec.data, 32); + } else { + memset(&this->buffer_send[offset], 0, 32); + } + offset += 32; + + this->buffer_send[4] = offset-5; + this->length_send = offset; + this->exchange(); + + offset = 0; + unsigned int recv_len = this->length_recv; + + //if (tx_version > 1) + { + ASSERT_X(recv_len>=32, "Not enought data from device"); + crypto::secret_key scalar1; + memmove(scalar1.data, &this->buffer_recv[offset],32); + amount_keys.push_back(rct::sk2rct(scalar1)); + offset += 32; + recv_len -= 32; + } + ASSERT_X(recv_len>=32, "Not enought data from device"); + memmove(out_eph_public_key.data, &this->buffer_recv[offset], 32); + recv_len -= 32; + offset += 32; + + if (need_additional_txkeys) + { + ASSERT_X(recv_len>=32, "Not enought data from device"); + memmove(additional_txkey.pub.data, &this->buffer_recv[offset], 32); + additional_tx_public_keys.push_back(additional_txkey.pub); + offset += 32; + recv_len -= 32; + } + + // add ABPkeys + this->add_output_key_mapping(dst_entr.addr.m_view_public_key, dst_entr.addr.m_spend_public_key, dst_entr.is_subaddress, is_change, + need_additional_txkeys, output_index, + amount_keys.back(), out_eph_public_key); + + #ifdef DEBUG_HWDEVICE + log_hexbuffer("generate_output_ephemeral_keys: clear amount_key", (const char*)hw::ledger::decrypt(amount_keys.back()).bytes, 32); + hw::ledger::check32("generate_output_ephemeral_keys", "amount_key", (const char*)amount_keys_x.back().bytes, (const char*)hw::ledger::decrypt(amount_keys.back()).bytes); + if (need_additional_txkeys) { + hw::ledger::check32("generate_output_ephemeral_keys", "additional_tx_key", additional_tx_public_keys_x.back().data, additional_tx_public_keys.back().data); + } + hw::ledger::check32("generate_output_ephemeral_keys", "out_eph_public_key", out_eph_public_key_x.data, out_eph_public_key.data); + #endif + + return true; + } + + bool device_ledger::add_output_key_mapping(const crypto::public_key &Aout, const crypto::public_key &Bout, const bool is_subaddress, const bool is_change, + const bool need_additional, const size_t real_output_index, + const rct::key &amount_key, const crypto::public_key &out_eph_public_key) { + key_map.add(ABPkeys(rct::pk2rct(Aout),rct::pk2rct(Bout), is_subaddress, is_change, need_additional, real_output_index, rct::pk2rct(out_eph_public_key), amount_key)); return true; } - bool device_ledger::ecdhEncode(rct::ecdhTuple & unmasked, const rct::key & AKout) { + rct::key device_ledger::genCommitmentMask(const rct::key &AKout) { + #ifdef DEBUG_HWDEVICE + const rct::key AKout_x = hw::ledger::decrypt(AKout); + rct::key mask_x; + mask_x = this->controle_device->genCommitmentMask(AKout_x); + #endif + + rct::key mask; + int offset = set_command_header_noopt(INS_GEN_COMMITMENT_MASK); + // AKout + memmove(this->buffer_send+offset, AKout.bytes, 32); + offset += 32; + + this->buffer_send[4] = offset-5; + this->length_send = offset; + this->exchange(); + + memmove(mask.bytes, &this->buffer_recv[0], 32); + + #ifdef DEBUG_HWDEVICE + hw::ledger::check32("genCommitmentMask", "mask", (const char*)mask_x.bytes, (const char*)mask.bytes); + #endif + + return mask; + } + + bool device_ledger::ecdhEncode(rct::ecdhTuple & unmasked, const rct::key & AKout, bool short_amount) { AUTO_LOCK_CMD(); - int offset; #ifdef DEBUG_HWDEVICE const rct::key AKout_x = hw::ledger::decrypt(AKout); rct::ecdhTuple unmasked_x = unmasked; - this->controle_device->ecdhEncode(unmasked_x, AKout_x); + this->controle_device->ecdhEncode(unmasked_x, AKout_x, short_amount); #endif - reset_buffer(); - - this->buffer_send[0] = 0x00; - this->buffer_send[1] = INS_BLIND; - this->buffer_send[2] = 0x00; - this->buffer_send[3] = 0x00; - this->buffer_send[4] = 0x00; - offset = 5; + int offset = set_command_header(INS_BLIND); //options - this->buffer_send[offset] = 0x00; - offset += 1; + this->buffer_send[offset] = short_amount?0x02:0x00; + offset += 1; // AKout memmove(this->buffer_send+offset, AKout.bytes, 32); offset += 32; @@ -1492,36 +1509,29 @@ namespace hw { memmove(unmasked.mask.bytes, &this->buffer_recv[32], 32); #ifdef DEBUG_HWDEVICE + MDEBUG("ecdhEncode: Akout: "<buffer_recv[64], 3*32); + log_hexbuffer("Blind AKV input", (char*)&this->buffer_recv[64], 3*32); #endif return true; } - bool device_ledger::ecdhDecode(rct::ecdhTuple & masked, const rct::key & AKout) { + bool device_ledger::ecdhDecode(rct::ecdhTuple & masked, const rct::key & AKout, bool short_amount) { AUTO_LOCK_CMD(); - int offset; #ifdef DEBUG_HWDEVICE const rct::key AKout_x = hw::ledger::decrypt(AKout); rct::ecdhTuple masked_x = masked; - this->controle_device->ecdhDecode(masked_x, AKout_x); + this->controle_device->ecdhDecode(masked_x, AKout_x, short_amount); #endif - reset_buffer(); - - this->buffer_send[0] = 0x00; - this->buffer_send[1] = INS_UNBLIND; - this->buffer_send[2] = 0x00; - this->buffer_send[3] = 0x00; - this->buffer_send[4] = 0x00; - offset = 5; + int offset = set_command_header(INS_UNBLIND); //options - this->buffer_send[offset] = 0x00; - offset += 1; + this->buffer_send[offset] = short_amount?0x02:0x00; + offset += 1; // AKout memmove(this->buffer_send+offset, AKout.bytes, 32); offset += 32; @@ -1540,6 +1550,7 @@ namespace hw { memmove(masked.mask.bytes, &this->buffer_recv[32], 32); #ifdef DEBUG_HWDEVICE + MDEBUG("ecdhEncode: Akout: "<buffer_send[0] = 0x00; - this->buffer_send[1] = INS_VALIDATE; - this->buffer_send[2] = 0x01; - this->buffer_send[3] = 0x01; - this->buffer_send[4] = 0x00; - offset = 5; + int offset = set_command_header(INS_VALIDATE, 0x01, 0x01); //options this->buffer_send[offset] = (inputs_size == 0)?0x00:0x80; offset += 1; @@ -1603,18 +1605,13 @@ namespace hw { this->buffer_send[4] = offset-5; this->length_send = offset; - this->exchange(); + // check fee user input + CHECK_AND_ASSERT_THROW_MES(this->exchange_wait_on_input() == 0, "Fee denied on device."); //pseudoOuts - if ((type == rct::RCTTypeSimple) || (type == rct::RCTTypeSimpleBulletproof)) { + if (type == rct::RCTTypeSimple) { for ( i = 0; i < inputs_size; i++) { - reset_buffer(); - this->buffer_send[0] = 0x00; - this->buffer_send[1] = INS_VALIDATE; - this->buffer_send[2] = 0x01; - this->buffer_send[3] = i+2; - this->buffer_send[4] = 0x00; - offset = 5; + offset = set_command_header(INS_VALIDATE, 0x01, i+2); //options this->buffer_send[offset] = (i==inputs_size-1)? 0x00:0x80; offset += 1; @@ -1631,7 +1628,11 @@ namespace hw { // ====== Aout, Bout, AKout, C, v, k ====== kv_offset = data_offset; - C_offset = kv_offset+ (32*2)*outputs_size; + if (type==rct::RCTTypeBulletproof2) { + C_offset = kv_offset+ (8)*outputs_size; + } else { + C_offset = kv_offset+ (32+32)*outputs_size; + } for ( i = 0; i < outputs_size; i++) { ABPkeys outKeys; bool found; @@ -1641,21 +1642,18 @@ namespace hw { log_hexbuffer("Pout not found", (char*)outPk[i].dest.bytes, 32); CHECK_AND_ASSERT_THROW_MES(found, "Pout not found"); } - reset_buffer(); - - this->buffer_send[0] = 0x00; - this->buffer_send[1] = INS_VALIDATE; - this->buffer_send[2] = 0x02; - this->buffer_send[3] = i+1; - this->buffer_send[4] = 0x00; - offset = 5; + offset = set_command_header(INS_VALIDATE, 0x02, i+1); //options this->buffer_send[offset] = (i==outputs_size-1)? 0x00:0x80 ; + this->buffer_send[offset] |= (type==rct::RCTTypeBulletproof2)?0x02:0x00; offset += 1; if (found) { //is_subaddress this->buffer_send[offset] = outKeys.is_subaddress; offset++; + //is_change_address + this->buffer_send[offset] = outKeys.is_change_address; + offset++; //Aout memmove(this->buffer_send+offset, outKeys.Aout.bytes, 32); offset+=32; @@ -1667,40 +1665,45 @@ namespace hw { offset+=32; } else { // dummy: is_subaddress Aout Bout AKout - offset += 1+32*3; + offset += 2+32*3; } //C memmove(this->buffer_send+offset, data+C_offset,32); offset += 32; C_offset += 32; - //k - memmove(this->buffer_send+offset, data+kv_offset,32); - offset += 32; - //v - kv_offset += 32; - memmove(this->buffer_send+offset, data+kv_offset,32); - offset += 32; - kv_offset += 32; + if (type==rct::RCTTypeBulletproof2) { + //k + memset(this->buffer_send+offset, 0, 32); + offset += 32; + //v + memset(this->buffer_send+offset, 0, 32); + memmove(this->buffer_send+offset, data+kv_offset,8); + offset += 32; + kv_offset += 8; + } else { + //k + memmove(this->buffer_send+offset, data+kv_offset,32); + offset += 32; + kv_offset += 32; + //v + memmove(this->buffer_send+offset, data+kv_offset,32); + offset += 32; + kv_offset += 32; + } this->buffer_send[4] = offset-5; this->length_send = offset; - this->exchange(); + // check transaction user input + CHECK_AND_ASSERT_THROW_MES(this->exchange_wait_on_input() == 0, "Transaction denied on device."); #ifdef DEBUG_HWDEVICE - hw::ledger::log_hexbuffer("Prehash AKV input", (char*)&this->buffer_recv[64], 3*32); + log_hexbuffer("Prehash AKV input", (char*)&this->buffer_recv[64], 3*32); #endif } // ====== C[], message, proof====== C_offset = kv_offset; for (i = 0; i < outputs_size; i++) { - reset_buffer(); - - this->buffer_send[0] = 0x00; - this->buffer_send[1] = INS_VALIDATE; - this->buffer_send[2] = 0x03; - this->buffer_send[3] = i+1; - this->buffer_send[4] = 0x00; - offset = 5; + offset = set_command_header(INS_VALIDATE, 0x03, i+1); //options this->buffer_send[offset] = 0x80 ; offset += 1; @@ -1715,17 +1718,7 @@ namespace hw { } - reset_buffer(); - - this->buffer_send[0] = 0x00; - this->buffer_send[1] = INS_VALIDATE; - this->buffer_send[2] = 0x03; - this->buffer_send[3] = i+1; - this->buffer_send[4] = 0x00; - offset = 5; - //options - this->buffer_send[offset] = 0x00; - offset += 1; + offset = set_command_header_noopt(INS_VALIDATE, 0x03, i+1); //message memmove(this->buffer_send+offset, hashes[0].bytes,32); offset += 32; @@ -1750,8 +1743,6 @@ namespace hw { bool device_ledger::mlsag_prepare(const rct::key &H, const rct::key &xx, rct::key &a, rct::key &aG, rct::key &aHP, rct::key &II) { AUTO_LOCK_CMD(); - int offset; - unsigned char options; #ifdef DEBUG_HWDEVICE const rct::key H_x = H; @@ -1762,17 +1753,7 @@ namespace hw { rct::key II_x; #endif - reset_buffer(); - - this->buffer_send[0] = 0x00; - this->buffer_send[1] = INS_MLSAG; - this->buffer_send[2] = 0x01; - this->buffer_send[3] = 0x00; - this->buffer_send[4] = 0x00; - offset = 5; - //options - this->buffer_send[offset] = 0x00; - offset += 1; + int offset = set_command_header_noopt(INS_MLSAG, 0x01); //value H memmove(this->buffer_send+offset, H.bytes, 32); offset += 32; @@ -1805,29 +1786,13 @@ namespace hw { bool device_ledger::mlsag_prepare(rct::key &a, rct::key &aG) { AUTO_LOCK_CMD(); - int offset; - unsigned char options; #ifdef DEBUG_HWDEVICE rct::key a_x; rct::key aG_x; #endif - reset_buffer(); - - this->buffer_send[0] = 0x00; - this->buffer_send[1] = INS_MLSAG; - this->buffer_send[2] = 0x01; - this->buffer_send[3] = 0x00; - this->buffer_send[4] = 0x00; - offset = 5; - //options - this->buffer_send[offset] = 0x00; - offset += 1; - - this->buffer_send[4] = offset-5; - this->length_send = offset; - this->exchange(); + send_simple(INS_MLSAG, 0x01); memmove(a.bytes, &this->buffer_recv[32*0], 32); memmove(aG.bytes, &this->buffer_recv[32*1], 32); @@ -1843,8 +1808,6 @@ namespace hw { bool device_ledger::mlsag_hash(const rct::keyV &long_message, rct::key &c) { AUTO_LOCK_CMD(); - int offset; - unsigned char options; size_t cnt; #ifdef DEBUG_HWDEVICE @@ -1855,14 +1818,7 @@ namespace hw { cnt = long_message.size(); for (size_t i = 0; ibuffer_send[0] = 0x00; - this->buffer_send[1] = INS_MLSAG; - this->buffer_send[2] = 0x02; - this->buffer_send[3] = i+1; - this->buffer_send[4] = 0x00; - offset = 5; + int offset = set_command_header(INS_MLSAG, 0x02, i+1); //options this->buffer_send[offset] = (i==(cnt-1))?0x00:0x80; //last @@ -1887,7 +1843,6 @@ namespace hw { bool device_ledger::mlsag_sign(const rct::key &c, const rct::keyV &xx, const rct::keyV &alpha, const size_t rows, const size_t dsRows, rct::keyV &ss) { AUTO_LOCK_CMD(); - int offset; CHECK_AND_ASSERT_THROW_MES(dsRows<=rows, "dsRows greater than rows"); CHECK_AND_ASSERT_THROW_MES(xx.size() == rows, "xx size does not match rows"); @@ -1905,14 +1860,7 @@ namespace hw { #endif for (size_t j = 0; j < dsRows; j++) { - reset_buffer(); - - this->buffer_send[0] = 0x00; - this->buffer_send[1] = INS_MLSAG; - this->buffer_send[2] = 0x03; - this->buffer_send[3] = j+1; - this->buffer_send[4] = 0x00; - offset = 5; + int offset = set_command_header(INS_MLSAG, 0x03, j+1); //options this->buffer_send[offset] = 0x00; if (j==(dsRows-1)) { @@ -1949,24 +1897,7 @@ namespace hw { bool device_ledger::close_tx() { AUTO_LOCK_CMD(); - int offset; - - reset_buffer(); - - this->buffer_send[0] = 0x00; - this->buffer_send[1] = INS_CLOSE_TX; - this->buffer_send[2] = 0x00; - this->buffer_send[3] = 0x00; - this->buffer_send[4] = 0x00; - offset = 5; - //options - this->buffer_send[offset] = 0x00; - offset += 1; - - this->buffer_send[4] = offset-5; - this->length_send = offset; - this->exchange(); - + send_simple(INS_CLOSE_TX); return true; } diff --git a/src/device/device_ledger.hpp b/src/device/device_ledger.hpp index f1fcaab871d..fe9028733af 100644 --- a/src/device/device_ledger.hpp +++ b/src/device/device_ledger.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2018, The Monero Project +// Copyright (c) 2017-2019, The Monero Project // // All rights reserved. // @@ -33,8 +33,8 @@ #include #include #include "device.hpp" -#include -#include +#include "log.hpp" +#include "device_io_hid.hpp" #include #include @@ -42,6 +42,18 @@ namespace hw { namespace ledger { + /* Minimal supported version */ + #define MINIMAL_APP_VERSION_MAJOR 1 + #define MINIMAL_APP_VERSION_MINOR 3 + #define MINIMAL_APP_VERSION_MICRO 1 + + #define VERSION(M,m,u) ((M)<<16|(m)<<8|(u)) + #define VERSION_MAJOR(v) (((v)>>16)&0xFF) + #define VERSION_MINOR(v) (((v)>>8)&0xFF) + #define VERSION_MICRO(v) (((v)>>0)&0xFF) + + #define MINIMAL_APP_VERSION VERSION(MINIMAL_APP_VERSION_MAJOR, MINIMAL_APP_VERSION_MINOR, MINIMAL_APP_VERSION_MICRO) + void register_all(std::map> ®istry); #ifdef WITH_DEVICE_LEDGER @@ -57,12 +69,15 @@ namespace hw { rct::key Aout; rct::key Bout; bool is_subaddress; + bool is_change_address; + bool additional_key ; size_t index; rct::key Pout; rct::key AKout; - ABPkeys(const rct::key& A, const rct::key& B, const bool is_subaddr, size_t index, const rct::key& P,const rct::key& AK); + ABPkeys(const rct::key& A, const rct::key& B, const bool is_subaddr, bool is_subaddress, bool is_change_address, size_t index, const rct::key& P,const rct::key& AK); ABPkeys(const ABPkeys& keys) ; - ABPkeys() {index=0;is_subaddress=false;} + ABPkeys() {index=0;is_subaddress=false;is_subaddress=false;is_change_address=false;} + ABPkeys &operator=(const ABPkeys &keys); }; class Keymap { @@ -84,25 +99,31 @@ namespace hw { mutable boost::recursive_mutex device_locker; mutable boost::mutex command_locker; - //PCSC management - std::string full_name; - SCARDCONTEXT hContext; - SCARDHANDLE hCard; - DWORD length_send; - BYTE buffer_send[BUFFER_SEND_SIZE]; - DWORD length_recv; - BYTE buffer_recv[BUFFER_RECV_SIZE]; - unsigned int id; + //IO + hw::io::device_io_hid hw_device; + unsigned int length_send; + unsigned char buffer_send[BUFFER_SEND_SIZE]; + unsigned int length_recv; + unsigned char buffer_recv[BUFFER_RECV_SIZE]; + unsigned int sw; + unsigned int id; void logCMD(void); void logRESP(void); - unsigned int exchange(unsigned int ok=0x9000, unsigned int mask=0xFFFF); + unsigned int exchange(unsigned int ok=0x9000, unsigned int mask=0xFFFF); + unsigned int exchange_wait_on_input(unsigned int ok=0x9000, unsigned int mask=0xFFFF); void reset_buffer(void); + int set_command_header(unsigned char ins, unsigned char p1 = 0x00, unsigned char p2 = 0x00); + int set_command_header_noopt(unsigned char ins, unsigned char p1 = 0x00, unsigned char p2 = 0x00); + void send_simple(unsigned char ins, unsigned char p1 = 0x00); + // hw running mode device_mode mode; // map public destination key to ephemeral destination key Keymap key_map; - + bool add_output_key_mapping(const crypto::public_key &Aout, const crypto::public_key &Bout, const bool is_subaddress, const bool is_change, + const bool need_additional, const size_t real_output_index, + const rct::key &amount_key, const crypto::public_key &out_eph_public_key); // To speed up blockchain parsing the view key maybe handle here. crypto::secret_key viewkey; bool has_view_key; @@ -119,7 +140,7 @@ namespace hw { device_ledger(const device_ledger &device) = delete ; device_ledger& operator=(const device_ledger &device) = delete; - explicit operator bool() const override {return this->hContext != 0;} + explicit operator bool() const override {return this->connected(); } bool reset(void); @@ -133,8 +154,12 @@ namespace hw { bool release() override; bool connect(void) override; bool disconnect() override; + bool connected(void) const; - bool set_mode(device_mode mode) override; + bool set_mode(device_mode mode) override; + + device_type get_type() const override {return device_type::LEDGER;}; + device_protocol_t device_protocol() const override { return PROTOCOL_PROXY; }; /* ======================================================================= */ /* LOCKER */ @@ -148,7 +173,7 @@ namespace hw { /* ======================================================================= */ bool get_public_address(cryptonote::account_public_address &pubkey) override; bool get_secret_keys(crypto::secret_key &viewkey , crypto::secret_key &spendkey) override; - bool generate_chacha_key(const cryptonote::account_keys &keys, crypto::chacha_key &key) override; + bool generate_chacha_key(const cryptonote::account_keys &keys, crypto::chacha_key &key, uint64_t kdf_rounds) override; /* ======================================================================= */ @@ -179,17 +204,25 @@ namespace hw { /* ======================================================================= */ /* TRANSACTION */ /* ======================================================================= */ - + void generate_tx_proof(const crypto::hash &prefix_hash, + const crypto::public_key &R, const crypto::public_key &A, const boost::optional &B, const crypto::public_key &D, const crypto::secret_key &r, + crypto::signature &sig) override; + bool open_tx(crypto::secret_key &tx_key) override; bool encrypt_payment_id(crypto::hash8 &payment_id, const crypto::public_key &public_key, const crypto::secret_key &secret_key) override; - bool ecdhEncode(rct::ecdhTuple & unmasked, const rct::key & sharedSec) override; - bool ecdhDecode(rct::ecdhTuple & masked, const rct::key & sharedSec) override; + rct::key genCommitmentMask(const rct::key &amount_key) override; - bool add_output_key_mapping(const crypto::public_key &Aout, const crypto::public_key &Bout, const bool is_subaddress, const size_t real_output_index, - const rct::key &amount_key, const crypto::public_key &out_eph_public_key) override; + bool ecdhEncode(rct::ecdhTuple & unmasked, const rct::key & sharedSec, bool short_format) override; + bool ecdhDecode(rct::ecdhTuple & masked, const rct::key & sharedSec, bool short_format) override; + bool generate_output_ephemeral_keys(const size_t tx_version, const cryptonote::account_keys &sender_account_keys, const crypto::public_key &txkey_pub, const crypto::secret_key &tx_key, + const cryptonote::tx_destination_entry &dst_entr, const boost::optional &change_addr, const size_t output_index, + const bool &need_additional_txkeys, const std::vector &additional_tx_keys, + std::vector &additional_tx_public_keys, + std::vector &amount_keys, + crypto::public_key &out_eph_public_key) override; bool mlsag_prehash(const std::string &blob, size_t inputs_size, size_t outputs_size, const rct::keyV &hashes, const rct::ctkeyV &outPk, rct::key &prehash) override; bool mlsag_prepare(const rct::key &H, const rct::key &xx, rct::key &a, rct::key &aG, rct::key &aHP, rct::key &rvII) override; diff --git a/src/device/log.cpp b/src/device/log.cpp index cbbcfc95306..616ad8e9064 100644 --- a/src/device/log.cpp +++ b/src/device/log.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2018, The Monero Project +// Copyright (c) 2017-2019, The Monero Project // // All rights reserved. // @@ -32,36 +32,41 @@ namespace hw { - #ifdef WITH_DEVICE_LEDGER - namespace ledger { - - #undef MONERO_DEFAULT_LOG_CATEGORY - #define MONERO_DEFAULT_LOG_CATEGORY "device.ledger" + #undef MONERO_DEFAULT_LOG_CATEGORY + #define MONERO_DEFAULT_LOG_CATEGORY "device" - void buffer_to_str(char *to_buff, size_t to_len, const char *buff, size_t len) { - CHECK_AND_ASSERT_THROW_MES(to_len > (len*2), "destination buffer too short. At least" << (len*2+1) << " bytes required"); - for (size_t i=0; i (len*2), "destination buffer too short. At least" << (len*2+1) << " bytes required"); + for (size_t i=0; iset_name(HW_TREZOR_NAME); + } + return trezor_device; + } + + void register_all(std::map> ®istry) { + registry.insert(std::make_pair(HW_TREZOR_NAME, std::unique_ptr(ensure_trezor_device()))); + } + + void register_all() { + hw::register_device(HW_TREZOR_NAME, ensure_trezor_device()); + } + + device_trezor::device_trezor() { + m_live_refresh_in_progress = false; + m_live_refresh_enabled = true; + m_live_refresh_thread_running = false; + } + + device_trezor::~device_trezor() { + try { + disconnect(); + release(); + } catch(std::exception const& e){ + MWARNING("Could not disconnect and release: " << e.what()); + } + } + + bool device_trezor::init() + { + m_live_refresh_in_progress = false; + bool r = device_trezor_base::init(); + if (r && !m_live_refresh_thread) + { + m_live_refresh_thread_running = true; + m_live_refresh_thread.reset(new boost::thread(boost::bind(&device_trezor::live_refresh_thread_main, this))); + } + return r; + } + + bool device_trezor::release() + { + m_live_refresh_in_progress = false; + m_live_refresh_thread_running = false; + if (m_live_refresh_thread) + { + m_live_refresh_thread->join(); + m_live_refresh_thread = nullptr; + } + return device_trezor_base::release(); + } + + bool device_trezor::disconnect() + { + m_live_refresh_in_progress = false; + return device_trezor_base::disconnect(); + } + + void device_trezor::device_state_reset_unsafe() + { + require_connected(); + if (m_live_refresh_in_progress) + { + try + { + live_refresh_finish_unsafe(); + } + catch(const std::exception & e) + { + MERROR("Live refresh could not be terminated: " << e.what()); + } + } + + m_live_refresh_in_progress = false; + device_trezor_base::device_state_reset_unsafe(); + } + + void device_trezor::live_refresh_thread_main() + { + while(m_live_refresh_thread_running) + { + boost::this_thread::sleep_for(boost::chrono::milliseconds(100)); + if (!m_live_refresh_in_progress) + { + continue; + } + + TREZOR_AUTO_LOCK_DEVICE(); + if (!m_transport || !m_live_refresh_in_progress) + { + continue; + } + + auto current_time = std::chrono::steady_clock::now(); + if (current_time - m_last_live_refresh_time <= std::chrono::minutes(5)) + { + continue; + } + + MTRACE("Closing live refresh process due to inactivity"); + try + { + live_refresh_finish(); + } + catch(const std::exception &e) + { + MWARNING("Live refresh auto-finish failed: " << e.what()); + } + } + } + + /* ======================================================================= */ + /* WALLET & ADDRESS */ + /* ======================================================================= */ + + bool device_trezor::get_public_address(cryptonote::account_public_address &pubkey) { + try { + auto res = get_address(); + + cryptonote::address_parse_info info{}; + bool r = cryptonote::get_account_address_from_str(info, this->network_type, res->address()); + CHECK_AND_ASSERT_MES(r, false, "Could not parse returned address. Address parse failed: " + res->address()); + CHECK_AND_ASSERT_MES(!info.is_subaddress, false, "Trezor returned a sub address"); + + pubkey = info.address; + return true; + + } catch(std::exception const& e){ + MERROR("Get public address exception: " << e.what()); + return false; + } + } + + bool device_trezor::get_secret_keys(crypto::secret_key &viewkey , crypto::secret_key &spendkey) { + try { + MDEBUG("Loading view-only key from the Trezor. Please check the Trezor for a confirmation."); + auto res = get_view_key(); + CHECK_AND_ASSERT_MES(res->watch_key().size() == 32, false, "Trezor returned invalid view key"); + + // Trezor does not make use of spendkey of the device API. + // Ledger loads encrypted spendkey, Trezor loads null key (never leaves device). + // In the test (debugging mode) we need to leave this field intact as it is already set by + // the debugging code and need to remain same for the testing purposes. +#ifndef WITH_TREZOR_DEBUGGING + spendkey = crypto::null_skey; // not given +#endif + + memcpy(viewkey.data, res->watch_key().data(), 32); + + return true; + + } catch(std::exception const& e){ + MERROR("Get secret keys exception: " << e.what()); + return false; + } + } + + /* ======================================================================= */ + /* Helpers */ + /* ======================================================================= */ + + /* ======================================================================= */ + /* TREZOR PROTOCOL */ + /* ======================================================================= */ + + std::shared_ptr device_trezor::get_address( + const boost::optional> & path, + const boost::optional & network_type){ + TREZOR_AUTO_LOCK_CMD(); + require_connected(); + device_state_reset_unsafe(); + require_initialized(); + + auto req = std::make_shared(); + this->set_msg_addr(req.get(), path, network_type); + + auto response = this->client_exchange(req); + MTRACE("Get address response received"); + return response; + } + + std::shared_ptr device_trezor::get_view_key( + const boost::optional> & path, + const boost::optional & network_type){ + TREZOR_AUTO_LOCK_CMD(); + require_connected(); + device_state_reset_unsafe(); + require_initialized(); + + auto req = std::make_shared(); + this->set_msg_addr(req.get(), path, network_type); + + auto response = this->client_exchange(req); + MTRACE("Get watch key response received"); + return response; + } + + bool device_trezor::is_get_tx_key_supported() const + { + require_initialized(); + return get_version() > pack_version(2, 0, 10); + } + + void device_trezor::load_tx_key_data(::hw::device_cold::tx_key_data_t & res, const std::string & tx_aux_data) + { + protocol::tx::load_tx_key_data(res, tx_aux_data); + } + + void device_trezor::get_tx_key( + std::vector<::crypto::secret_key> & tx_keys, + const ::hw::device_cold::tx_key_data_t & tx_aux_data, + const ::crypto::secret_key & view_key_priv) + { + TREZOR_AUTO_LOCK_CMD(); + require_connected(); + device_state_reset_unsafe(); + require_initialized(); + + auto req = protocol::tx::get_tx_key(tx_aux_data); + this->set_msg_addr(req.get()); + + auto response = this->client_exchange(req); + MTRACE("Get TX key response received"); + + protocol::tx::get_tx_key_ack(tx_keys, tx_aux_data.tx_prefix_hash, view_key_priv, response); + } + + void device_trezor::ki_sync(wallet_shim * wallet, + const std::vector & transfers, + hw::device_cold::exported_key_image & ski) + { +#define EVENT_PROGRESS(P) do { if (m_callback) {(m_callback)->on_progress(device_cold::op_progress(P)); } }while(0) + + TREZOR_AUTO_LOCK_CMD(); + require_connected(); + device_state_reset_unsafe(); + require_initialized(); + + std::shared_ptr req; + + std::vector mtds; + std::vector kis; + protocol::ki::key_image_data(wallet, transfers, mtds); + protocol::ki::generate_commitment(mtds, transfers, req); + + EVENT_PROGRESS(0.); + this->set_msg_addr(req.get()); + auto ack1 = this->client_exchange(req); + + const auto batch_size = 10; + const auto num_batches = (mtds.size() + batch_size - 1) / batch_size; + for(uint64_t cur = 0; cur < num_batches; ++cur){ + auto step_req = std::make_shared(); + auto idx_finish = std::min(static_cast((cur + 1) * batch_size), static_cast(mtds.size())); + for(uint64_t idx = cur * batch_size; idx < idx_finish; ++idx){ + auto added_tdis = step_req->add_tdis(); + CHECK_AND_ASSERT_THROW_MES(idx < mtds.size(), "Invalid transfer detail index"); + *added_tdis = mtds[idx]; + } + + auto step_ack = this->client_exchange(step_req); + auto kis_size = step_ack->kis_size(); + kis.reserve(static_cast(kis_size)); + for(int i = 0; i < kis_size; ++i){ + auto ckis = step_ack->kis(i); + kis.push_back(ckis); + } + + MTRACE("Batch " << cur << " / " << num_batches << " batches processed"); + EVENT_PROGRESS((double)cur * batch_size / mtds.size()); + } + EVENT_PROGRESS(1.); + + auto final_req = std::make_shared(); + auto final_ack = this->client_exchange(final_req); + ski.reserve(kis.size()); + + for(auto & sub : kis){ + ::crypto::signature sig{}; + ::crypto::key_image ki; + char buff[sizeof(ki.data)*3]; + + size_t buff_len = sizeof(buff); + + protocol::crypto::chacha::decrypt(sub.blob().data(), sub.blob().size(), + reinterpret_cast(final_ack->enc_key().data()), + reinterpret_cast(sub.iv().data()), buff, &buff_len); + CHECK_AND_ASSERT_THROW_MES(buff_len == sizeof(buff), "Plaintext size invalid"); + + memcpy(ki.data, buff, sizeof(ki.data)); + memcpy(sig.c.data, buff + sizeof(ki.data), sizeof(ki.data)); + memcpy(sig.r.data, buff + 2*sizeof(ki.data), sizeof(ki.data)); + ski.push_back(std::make_pair(ki, sig)); + } +#undef EVENT_PROGRESS + } + + bool device_trezor::is_live_refresh_supported() const + { + require_initialized(); + return get_version() > pack_version(2, 0, 10); + } + + bool device_trezor::is_live_refresh_enabled() const + { + return is_live_refresh_supported() && (mode == NONE || mode == TRANSACTION_PARSE) && m_live_refresh_enabled; + } + + bool device_trezor::has_ki_live_refresh() const + { + try{ + return is_live_refresh_enabled(); + } catch(const std::exception & e){ + MERROR("Could not detect if live refresh is enabled: " << e.what()); + } + return false; + } + + void device_trezor::live_refresh_start() + { + TREZOR_AUTO_LOCK_CMD(); + require_connected(); + live_refresh_start_unsafe(); + } + + void device_trezor::live_refresh_start_unsafe() + { + device_state_reset_unsafe(); + require_initialized(); + + auto req = std::make_shared(); + this->set_msg_addr(req.get()); + this->client_exchange(req); + m_live_refresh_in_progress = true; + m_last_live_refresh_time = std::chrono::steady_clock::now(); + } + + void device_trezor::live_refresh( + const ::crypto::secret_key & view_key_priv, + const crypto::public_key& out_key, + const crypto::key_derivation& recv_derivation, + size_t real_output_index, + const cryptonote::subaddress_index& received_index, + cryptonote::keypair& in_ephemeral, + crypto::key_image& ki + ) + { + TREZOR_AUTO_LOCK_CMD(); + require_connected(); + + if (!m_live_refresh_in_progress) + { + live_refresh_start_unsafe(); + } + + m_last_live_refresh_time = std::chrono::steady_clock::now(); + + auto req = std::make_shared(); + req->set_out_key(out_key.data, 32); + req->set_recv_deriv(recv_derivation.data, 32); + req->set_real_out_idx(real_output_index); + req->set_sub_addr_major(received_index.major); + req->set_sub_addr_minor(received_index.minor); + + auto ack = this->client_exchange(req); + protocol::ki::live_refresh_ack(view_key_priv, out_key, ack, in_ephemeral, ki); + } + + void device_trezor::live_refresh_finish_unsafe() + { + auto req = std::make_shared(); + this->client_exchange(req); + m_live_refresh_in_progress = false; + } + + void device_trezor::live_refresh_finish() + { + TREZOR_AUTO_LOCK_CMD(); + require_connected(); + if (m_live_refresh_in_progress) + { + live_refresh_finish_unsafe(); + } + } + + void device_trezor::computing_key_images(bool started) + { + try + { + if (!is_live_refresh_enabled()) + { + return; + } + + // React only on termination as the process can auto-start itself. + if (!started && m_live_refresh_in_progress) + { + live_refresh_finish(); + } + } + catch(const std::exception & e) + { + MWARNING("KI computation state change failed, started: " << started << ", e: " << e.what()); + } + } + + bool device_trezor::compute_key_image( + const ::cryptonote::account_keys& ack, + const ::crypto::public_key& out_key, + const ::crypto::key_derivation& recv_derivation, + size_t real_output_index, + const ::cryptonote::subaddress_index& received_index, + ::cryptonote::keypair& in_ephemeral, + ::crypto::key_image& ki) + { + if (!is_live_refresh_enabled()) + { + return false; + } + + live_refresh(ack.m_view_secret_key, out_key, recv_derivation, real_output_index, received_index, in_ephemeral, ki); + return true; + } + + void device_trezor::tx_sign(wallet_shim * wallet, + const tools::wallet2::unsigned_tx_set & unsigned_tx, + tools::wallet2::signed_tx_set & signed_tx, + hw::tx_aux_data & aux_data) + { + CHECK_AND_ASSERT_THROW_MES(unsigned_tx.transfers.first == 0, "Unsuported non zero offset"); + + TREZOR_AUTO_LOCK_CMD(); + require_connected(); + device_state_reset_unsafe(); + require_initialized(); + transaction_versions_check(unsigned_tx, aux_data); + + const size_t num_tx = unsigned_tx.txes.size(); + m_num_transations_to_sign = num_tx; + signed_tx.key_images.clear(); + signed_tx.key_images.resize(unsigned_tx.transfers.second.size()); + + for(size_t tx_idx = 0; tx_idx < num_tx; ++tx_idx) { + std::shared_ptr signer; + tx_sign(wallet, unsigned_tx, tx_idx, aux_data, signer); + + auto & cdata = signer->tdata(); + auto aux_info_cur = signer->store_tx_aux_info(); + aux_data.tx_device_aux.emplace_back(aux_info_cur); + + // Pending tx reconstruction + signed_tx.ptx.emplace_back(); + auto & cpend = signed_tx.ptx.back(); + cpend.tx = cdata.tx; + cpend.dust = 0; + cpend.fee = 0; + cpend.dust_added_to_fee = false; + cpend.change_dts = cdata.tx_data.change_dts; + cpend.selected_transfers = cdata.tx_data.selected_transfers; + cpend.key_images = ""; + cpend.dests = cdata.tx_data.dests; + cpend.construction_data = cdata.tx_data; + + // Transaction check + try { + transaction_check(cdata, aux_data); + } catch(const std::exception &e){ + throw exc::ProtocolException(std::string("Transaction verification failed: ") + e.what()); + } + + std::string key_images; + bool all_are_txin_to_key = std::all_of(cdata.tx.vin.begin(), cdata.tx.vin.end(), [&](const cryptonote::txin_v& s_e) -> bool + { + CHECKED_GET_SPECIFIC_VARIANT(s_e, const cryptonote::txin_to_key, in, false); + key_images += boost::to_string(in.k_image) + " "; + return true; + }); + if(!all_are_txin_to_key) { + throw std::invalid_argument("Not all are txin_to_key"); + } + cpend.key_images = key_images; + + // KI sync + for(size_t cidx=0, trans_max=unsigned_tx.transfers.second.size(); cidx < trans_max; ++cidx){ + signed_tx.key_images[cidx] = unsigned_tx.transfers.second[cidx].m_key_image; + } + + size_t num_sources = cdata.tx_data.sources.size(); + CHECK_AND_ASSERT_THROW_MES(num_sources == cdata.source_permutation.size(), "Invalid permutation size"); + CHECK_AND_ASSERT_THROW_MES(num_sources == cdata.tx.vin.size(), "Invalid tx.vin size"); + for(size_t src_idx = 0; src_idx < num_sources; ++src_idx){ + size_t idx_mapped = cdata.source_permutation[src_idx]; + CHECK_AND_ASSERT_THROW_MES(idx_mapped < cdata.tx_data.selected_transfers.size(), "Invalid idx_mapped"); + CHECK_AND_ASSERT_THROW_MES(src_idx < cdata.tx.vin.size(), "Invalid idx_mapped"); + + size_t idx_map_src = cdata.tx_data.selected_transfers[idx_mapped]; + CHECK_AND_ASSERT_THROW_MES(idx_map_src >= unsigned_tx.transfers.first, "Invalid offset"); + + idx_map_src -= unsigned_tx.transfers.first; + CHECK_AND_ASSERT_THROW_MES(idx_map_src < signed_tx.key_images.size(), "Invalid key image index"); + + const auto vini = boost::get(cdata.tx.vin[src_idx]); + signed_tx.key_images[idx_map_src] = vini.k_image; + } + } + + if (m_callback){ + m_callback->on_progress(device_cold::tx_progress(m_num_transations_to_sign, m_num_transations_to_sign, 1, 1, 1, 1)); + } + } + + void device_trezor::tx_sign(wallet_shim * wallet, + const tools::wallet2::unsigned_tx_set & unsigned_tx, + size_t idx, + hw::tx_aux_data & aux_data, + std::shared_ptr & signer) + { +#define EVENT_PROGRESS(S, SUB, SUBMAX) do { if (m_callback) { \ + (m_callback)->on_progress(device_cold::tx_progress(idx, m_num_transations_to_sign, S, 10, SUB, SUBMAX)); \ +} }while(0) + + require_connected(); + if (idx > 0) + device_state_reset_unsafe(); + + require_initialized(); + EVENT_PROGRESS(0, 1, 1); + + CHECK_AND_ASSERT_THROW_MES(idx < unsigned_tx.txes.size(), "Invalid transaction index"); + signer = std::make_shared(wallet, &unsigned_tx, idx, &aux_data); + const tools::wallet2::tx_construction_data & cur_tx = unsigned_tx.txes[idx]; + unsigned long num_sources = cur_tx.sources.size(); + unsigned long num_outputs = cur_tx.splitted_dsts.size(); + + // Step: Init + auto init_msg = signer->step_init(); + this->set_msg_addr(init_msg.get()); + transaction_pre_check(init_msg); + EVENT_PROGRESS(1, 1, 1); + + auto response = this->client_exchange(init_msg); + signer->step_init_ack(response); + + // Step: Set transaction inputs + for(size_t cur_src = 0; cur_src < num_sources; ++cur_src){ + auto src = signer->step_set_input(cur_src); + auto ack = this->client_exchange(src); + signer->step_set_input_ack(ack); + EVENT_PROGRESS(2, cur_src, num_sources); + } + + // Step: sort + auto perm_req = signer->step_permutation(); + if (perm_req){ + auto perm_ack = this->client_exchange(perm_req); + signer->step_permutation_ack(perm_ack); + } + EVENT_PROGRESS(3, 1, 1); + + // Step: input_vini + for(size_t cur_src = 0; cur_src < num_sources; ++cur_src){ + auto src = signer->step_set_vini_input(cur_src); + auto ack = this->client_exchange(src); + signer->step_set_vini_input_ack(ack); + EVENT_PROGRESS(4, cur_src, num_sources); + } + + // Step: all inputs set + auto all_inputs_set = signer->step_all_inputs_set(); + auto ack_all_inputs = this->client_exchange(all_inputs_set); + signer->step_all_inputs_set_ack(ack_all_inputs); + EVENT_PROGRESS(5, 1, 1); + + // Step: outputs + for(size_t cur_dst = 0; cur_dst < num_outputs; ++cur_dst){ + auto src = signer->step_set_output(cur_dst); + auto ack = this->client_exchange(src); + signer->step_set_output_ack(ack); + + // If BP is offloaded to host, another step with computed BP may be needed. + auto offloaded_bp = signer->step_rsig(cur_dst); + if (offloaded_bp){ + auto bp_ack = this->client_exchange(offloaded_bp); + signer->step_set_rsig_ack(ack); + } + + EVENT_PROGRESS(6, cur_dst, num_outputs); + } + + // Step: all outs set + auto all_out_set = signer->step_all_outs_set(); + auto ack_all_out_set = this->client_exchange(all_out_set); + signer->step_all_outs_set_ack(ack_all_out_set, *this); + EVENT_PROGRESS(7, 1, 1); + + // Step: sign each input + for(size_t cur_src = 0; cur_src < num_sources; ++cur_src){ + auto src = signer->step_sign_input(cur_src); + auto ack_sign = this->client_exchange(src); + signer->step_sign_input_ack(ack_sign); + EVENT_PROGRESS(8, cur_src, num_sources); + } + + // Step: final + auto final_msg = signer->step_final(); + auto ack_final = this->client_exchange(final_msg); + signer->step_final_ack(ack_final); + EVENT_PROGRESS(9, 1, 1); +#undef EVENT_PROGRESS + } + + void device_trezor::transaction_versions_check(const ::tools::wallet2::unsigned_tx_set & unsigned_tx, hw::tx_aux_data & aux_data) + { + auto trezor_version = get_version(); + unsigned client_version = 1; // default client version for tx + + if (trezor_version <= pack_version(2, 0, 10)){ + client_version = 0; + } + + if (aux_data.client_version){ + auto wanted_client_version = aux_data.client_version.get(); + if (wanted_client_version > client_version){ + throw exc::TrezorException("Trezor firmware 2.0.10 and lower does not support current transaction sign protocol. Please update."); + } else { + client_version = wanted_client_version; + } + } + aux_data.client_version = client_version; + + if (client_version == 0 && aux_data.bp_version && aux_data.bp_version.get() != 1){ + throw exc::TrezorException("Trezor firmware 2.0.10 and lower does not support current transaction sign protocol (BPv2+). Please update."); + } + } + + void device_trezor::transaction_pre_check(std::shared_ptr init_msg) + { + CHECK_AND_ASSERT_THROW_MES(init_msg, "TransactionInitRequest is empty"); + CHECK_AND_ASSERT_THROW_MES(init_msg->has_tsx_data(), "TransactionInitRequest has no transaction data"); + CHECK_AND_ASSERT_THROW_MES(m_features, "Device state not initialized"); // make sure the caller did not reset features + const bool nonce_required = init_msg->tsx_data().has_payment_id() && init_msg->tsx_data().payment_id().size() > 0; + + if (nonce_required && init_msg->tsx_data().payment_id().size() == 8){ + // Versions 2.0.9 and lower do not support payment ID + if (get_version() <= pack_version(2, 0, 9)) { + throw exc::TrezorException("Trezor firmware 2.0.9 and lower does not support transactions with short payment IDs or integrated addresses. Please update."); + } + } + } + + void device_trezor::transaction_check(const protocol::tx::TData & tdata, const hw::tx_aux_data & aux_data) + { + // Simple serialization check + cryptonote::blobdata tx_blob; + cryptonote::transaction tx_deserialized; + bool r = cryptonote::t_serializable_object_to_blob(tdata.tx, tx_blob); + CHECK_AND_ASSERT_THROW_MES(r, "Transaction serialization failed"); + r = cryptonote::parse_and_validate_tx_from_blob(tx_blob, tx_deserialized); + CHECK_AND_ASSERT_THROW_MES(r, "Transaction deserialization failed"); + + // Extras check + std::vector tx_extra_fields; + cryptonote::tx_extra_nonce nonce; + + r = cryptonote::parse_tx_extra(tdata.tx.extra, tx_extra_fields); + CHECK_AND_ASSERT_THROW_MES(r, "tx.extra parsing failed"); + + const bool nonce_required = tdata.tsx_data.has_payment_id() && tdata.tsx_data.payment_id().size() > 0; + const bool has_nonce = cryptonote::find_tx_extra_field_by_type(tx_extra_fields, nonce); + CHECK_AND_ASSERT_THROW_MES(has_nonce || !nonce_required, "Transaction nonce not present"); + + if (nonce_required){ + const std::string & payment_id = tdata.tsx_data.payment_id(); + if (payment_id.size() == 32){ + crypto::hash payment_id_long{}; + CHECK_AND_ASSERT_THROW_MES(cryptonote::get_payment_id_from_tx_extra_nonce(nonce.nonce, payment_id_long), "Long payment ID not present"); + + } else if (payment_id.size() == 8){ + crypto::hash8 payment_id_short{}; + CHECK_AND_ASSERT_THROW_MES(cryptonote::get_encrypted_payment_id_from_tx_extra_nonce(nonce.nonce, payment_id_short), "Short payment ID not present"); + } + } + } + +#else //WITH_DEVICE_TREZOR + + void register_all(std::map> ®istry) { + } + + void register_all() { + } + +#endif //WITH_DEVICE_TREZOR +}} diff --git a/src/device_trezor/device_trezor.hpp b/src/device_trezor/device_trezor.hpp new file mode 100644 index 00000000000..0e91847dcc3 --- /dev/null +++ b/src/device_trezor/device_trezor.hpp @@ -0,0 +1,209 @@ +// Copyright (c) 2017-2019, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +#ifndef MONERO_DEVICE_TREZOR_H +#define MONERO_DEVICE_TREZOR_H + +#include "trezor.hpp" +#include "device/device.hpp" + +#ifdef WITH_DEVICE_TREZOR +#include +#include +#include +#include +#include + +#include "device/device_default.hpp" +#include "device/device_cold.hpp" +#include "cryptonote_config.h" +#include "device_trezor_base.hpp" +#endif + +namespace hw { +namespace trezor { + + void register_all(); + void register_all(std::map> ®istry); + +#ifdef WITH_DEVICE_TREZOR + class device_trezor; + + /** + * Main device + */ + class device_trezor : public hw::trezor::device_trezor_base, public hw::device_cold { + protected: + std::atomic m_live_refresh_in_progress; + std::chrono::steady_clock::time_point m_last_live_refresh_time; + std::unique_ptr m_live_refresh_thread; + std::atomic m_live_refresh_thread_running; + bool m_live_refresh_enabled; + size_t m_num_transations_to_sign; + + void transaction_versions_check(const ::tools::wallet2::unsigned_tx_set & unsigned_tx, hw::tx_aux_data & aux_data); + void transaction_pre_check(std::shared_ptr init_msg); + void transaction_check(const protocol::tx::TData & tdata, const hw::tx_aux_data & aux_data); + void device_state_reset_unsafe() override; + void live_refresh_start_unsafe(); + void live_refresh_finish_unsafe(); + void live_refresh_thread_main(); + + /** + * Signs particular transaction idx in the unsigned set, keeps state in the signer + */ + virtual void tx_sign(wallet_shim * wallet, + const ::tools::wallet2::unsigned_tx_set & unsigned_tx, + size_t idx, + hw::tx_aux_data & aux_data, + std::shared_ptr & signer); + + public: + device_trezor(); + virtual ~device_trezor() override; + + device_trezor(const device_trezor &device) = delete ; + device_trezor& operator=(const device_trezor &device) = delete; + + explicit operator bool() const override {return true;} + + bool init() override; + bool release() override; + bool disconnect() override; + + device_protocol_t device_protocol() const override { return PROTOCOL_COLD; }; + + bool has_ki_cold_sync() const override { return true; } + bool has_tx_cold_sign() const override { return true; } + void set_network_type(cryptonote::network_type network_type) override { this->network_type = network_type; } + void set_live_refresh_enabled(bool enabled) { m_live_refresh_enabled = enabled; } + bool live_refresh_enabled() const { return m_live_refresh_enabled; } + + /* ======================================================================= */ + /* WALLET & ADDRESS */ + /* ======================================================================= */ + bool get_public_address(cryptonote::account_public_address &pubkey) override; + bool get_secret_keys(crypto::secret_key &viewkey , crypto::secret_key &spendkey) override; + + /* ======================================================================= */ + /* TREZOR PROTOCOL */ + /* ======================================================================= */ + + /** + * Get address. Throws. + */ + std::shared_ptr get_address( + const boost::optional> & path = boost::none, + const boost::optional & network_type = boost::none); + + /** + * Get watch key from device. Throws. + */ + std::shared_ptr get_view_key( + const boost::optional> & path = boost::none, + const boost::optional & network_type = boost::none); + + /** + * Get_tx_key support check + */ + bool is_get_tx_key_supported() const override; + + /** + * Loads tx aux data + */ + void load_tx_key_data(::hw::device_cold::tx_key_data_t & res, const std::string & tx_aux_data) override; + + /** + * TX key load with the Trezor + */ + void get_tx_key( + std::vector<::crypto::secret_key> & tx_keys, + const ::hw::device_cold::tx_key_data_t & tx_aux_data, + const ::crypto::secret_key & view_key_priv) override; + + /** + * Key image sync with the Trezor. + */ + void ki_sync(wallet_shim * wallet, + const std::vector<::tools::wallet2::transfer_details> & transfers, + hw::device_cold::exported_key_image & ski) override; + + bool is_live_refresh_supported() const override; + + bool is_live_refresh_enabled() const; + + bool has_ki_live_refresh() const override; + + void live_refresh_start() override; + + void live_refresh( + const ::crypto::secret_key & view_key_priv, + const crypto::public_key& out_key, + const crypto::key_derivation& recv_derivation, + size_t real_output_index, + const cryptonote::subaddress_index& received_index, + cryptonote::keypair& in_ephemeral, + crypto::key_image& ki + ) override; + + void live_refresh_finish() override; + + /** + * Letting device know the KI computation started / ended. + * During refresh + */ + void computing_key_images(bool started) override; + + /** + * Implements hw::device interface + * called from generate_key_image_helper_precomp() + */ + bool compute_key_image( + const ::cryptonote::account_keys& ack, + const ::crypto::public_key& out_key, + const ::crypto::key_derivation& recv_derivation, + size_t real_output_index, + const ::cryptonote::subaddress_index& received_index, + ::cryptonote::keypair& in_ephemeral, + ::crypto::key_image& ki) override; + + /** + * Signs unsigned transaction with the Trezor. + */ + void tx_sign(wallet_shim * wallet, + const ::tools::wallet2::unsigned_tx_set & unsigned_tx, + ::tools::wallet2::signed_tx_set & signed_tx, + hw::tx_aux_data & aux_data) override; + }; + +#endif + +} +} +#endif //MONERO_DEVICE_TREZOR_H diff --git a/src/device_trezor/device_trezor_base.cpp b/src/device_trezor/device_trezor_base.cpp new file mode 100644 index 00000000000..b7adf433dd5 --- /dev/null +++ b/src/device_trezor/device_trezor_base.cpp @@ -0,0 +1,552 @@ +// Copyright (c) 2017-2019, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +#include "device_trezor_base.hpp" +#include +#include +#include + +namespace hw { +namespace trezor { + +#ifdef WITH_DEVICE_TREZOR + +#undef MONERO_DEFAULT_LOG_CATEGORY +#define MONERO_DEFAULT_LOG_CATEGORY "device.trezor" +#define TREZOR_BIP44_HARDENED_ZERO 0x80000000 + + const uint32_t device_trezor_base::DEFAULT_BIP44_PATH[] = {0x8000002c, 0x80000080}; + + device_trezor_base::device_trezor_base(): m_callback(nullptr), m_last_msg_type(messages::MessageType_Success) { +#ifdef WITH_TREZOR_DEBUGGING + m_debug = false; +#endif + } + + device_trezor_base::~device_trezor_base() { + try { + disconnect(); + release(); + } catch(std::exception const& e){ + MERROR("Could not disconnect and release: " << e.what()); + } + } + + /* ======================================================================= */ + /* SETUP/TEARDOWN */ + /* ======================================================================= */ + + bool device_trezor_base::reset() { + return false; + } + + bool device_trezor_base::set_name(const std::string & name) { + this->m_full_name = name; + this->name = ""; + + auto delim = name.find(':'); + if (delim != std::string::npos && delim + 1 < name.length()) { + this->name = name.substr(delim + 1); + } + + return true; + } + + const std::string device_trezor_base::get_name() const { + if (this->m_full_name.empty()) { + return std::string("name).append(">"); + } + return this->m_full_name; + } + + bool device_trezor_base::init() { + if (!release()){ + MERROR("Release failed"); + return false; + } + + return true; + } + + bool device_trezor_base::release() { + try { + disconnect(); + return true; + + } catch(std::exception const& e){ + MERROR("Release exception: " << e.what()); + return false; + } + } + + bool device_trezor_base::connect() { + disconnect(); + + // Enumerate all available devices + TREZOR_AUTO_LOCK_DEVICE(); + try { + hw::trezor::t_transport_vect trans; + + MDEBUG("Enumerating Trezor devices..."); + enumerate(trans); + sort_transports_by_env(trans); + + MDEBUG("Enumeration yielded " << trans.size() << " Trezor devices"); + for (auto &cur : trans) { + MDEBUG(" device: " << *(cur.get())); + } + + for (auto &cur : trans) { + std::string cur_path = cur->get_path(); + if (boost::starts_with(cur_path, this->name)) { + MDEBUG("Device Match: " << cur_path); + m_transport = cur; + break; + } + } + + if (!m_transport) { + MERROR("No matching Trezor device found. Device specifier: \"" + this->name + "\""); + return false; + } + + m_transport->open(); + +#ifdef WITH_TREZOR_DEBUGGING + setup_debug(); +#endif + return true; + + } catch(std::exception const& e){ + MERROR("Open exception: " << e.what()); + return false; + } + } + + bool device_trezor_base::disconnect() { + TREZOR_AUTO_LOCK_DEVICE(); + m_device_state.clear(); + m_features.reset(); + + if (m_transport){ + try { + m_transport->close(); + m_transport = nullptr; + + } catch(std::exception const& e){ + MERROR("Disconnect exception: " << e.what()); + m_transport = nullptr; + return false; + } + } + +#ifdef WITH_TREZOR_DEBUGGING + if (m_debug_callback) { + m_debug_callback->on_disconnect(); + m_debug_callback = nullptr; + } +#endif + return true; + } + + /* ======================================================================= */ + /* LOCKER */ + /* ======================================================================= */ + + //lock the device for a long sequence + void device_trezor_base::lock() { + MTRACE("Ask for LOCKING for device " << this->name << " in thread "); + device_locker.lock(); + MTRACE("Device " << this->name << " LOCKed"); + } + + //lock the device for a long sequence + bool device_trezor_base::try_lock() { + MTRACE("Ask for LOCKING(try) for device " << this->name << " in thread "); + bool r = device_locker.try_lock(); + if (r) { + MTRACE("Device " << this->name << " LOCKed(try)"); + } else { + MDEBUG("Device " << this->name << " not LOCKed(try)"); + } + return r; + } + + //unlock the device + void device_trezor_base::unlock() { + MTRACE("Ask for UNLOCKING for device " << this->name << " in thread "); + device_locker.unlock(); + MTRACE("Device " << this->name << " UNLOCKed"); + } + + /* ======================================================================= */ + /* Helpers */ + /* ======================================================================= */ + + void device_trezor_base::require_connected() const { + if (!m_transport){ + throw exc::NotConnectedException(); + } + } + + void device_trezor_base::require_initialized() const { + if (!m_features){ + throw exc::TrezorException("Device state not initialized"); + } + + if (m_features->has_bootloader_mode() && m_features->bootloader_mode()){ + throw exc::TrezorException("Device is in the bootloader mode"); + } + + if (m_features->has_firmware_present() && !m_features->firmware_present()){ + throw exc::TrezorException("Device has no firmware loaded"); + } + + // Hard requirement on initialized field, has to be there. + if (!m_features->has_initialized() || !m_features->initialized()){ + throw exc::TrezorException("Device is not initialized"); + } + } + + void device_trezor_base::call_ping_unsafe(){ + auto pingMsg = std::make_shared(); + pingMsg->set_message("PING"); + + auto success = this->client_exchange(pingMsg); // messages::MessageType_Success + MDEBUG("Ping response " << success->message()); + (void)success; + } + + void device_trezor_base::test_ping(){ + require_connected(); + + try { + this->call_ping_unsafe(); + + } catch(exc::TrezorException const& e){ + MINFO("Trezor does not respond: " << e.what()); + throw exc::DeviceNotResponsiveException(std::string("Trezor not responding: ") + e.what()); + } + } + + void device_trezor_base::write_raw(const google::protobuf::Message * msg){ + require_connected(); + CHECK_AND_ASSERT_THROW_MES(msg, "Empty message"); + this->get_transport()->write(*msg); + } + + GenericMessage device_trezor_base::read_raw(){ + require_connected(); + std::shared_ptr msg_resp; + hw::trezor::messages::MessageType msg_resp_type; + + this->get_transport()->read(msg_resp, &msg_resp_type); + return GenericMessage(msg_resp_type, msg_resp); + } + + GenericMessage device_trezor_base::call_raw(const google::protobuf::Message * msg) { + write_raw(msg); + return read_raw(); + } + + bool device_trezor_base::message_handler(GenericMessage & input){ + // Later if needed this generic message handler can be replaced by a pointer to + // a protocol message handler which by default points to the device class which implements + // the default handler. + + if (m_last_msg_type == messages::MessageType_ButtonRequest){ + on_button_pressed(); + } + m_last_msg_type = input.m_type; + + switch(input.m_type){ + case messages::MessageType_ButtonRequest: + on_button_request(input, dynamic_cast(input.m_msg.get())); + return true; + case messages::MessageType_PassphraseRequest: + on_passphrase_request(input, dynamic_cast(input.m_msg.get())); + return true; + case messages::MessageType_PassphraseStateRequest: + on_passphrase_state_request(input, dynamic_cast(input.m_msg.get())); + return true; + case messages::MessageType_PinMatrixRequest: + on_pin_request(input, dynamic_cast(input.m_msg.get())); + return true; + default: + return false; + } + } + + void device_trezor_base::ensure_derivation_path() noexcept { + if (m_wallet_deriv_path.empty()){ + m_wallet_deriv_path.push_back(TREZOR_BIP44_HARDENED_ZERO); // default 0' + } + } + + void device_trezor_base::set_derivation_path(const std::string &deriv_path){ + this->m_wallet_deriv_path.clear(); + + if (deriv_path.empty() || deriv_path == "-"){ + ensure_derivation_path(); + return; + } + + CHECK_AND_ASSERT_THROW_MES(deriv_path.size() <= 255, "Derivation path is too long"); + + std::vector fields; + boost::split(fields, deriv_path, boost::is_any_of("/")); + CHECK_AND_ASSERT_THROW_MES(fields.size() <= 10, "Derivation path is too long"); + + boost::regex rgx("^([0-9]+)'?$"); + boost::cmatch match; + + this->m_wallet_deriv_path.reserve(fields.size()); + for(const std::string & cur : fields){ + const bool ok = boost::regex_match(cur.c_str(), match, rgx); + CHECK_AND_ASSERT_THROW_MES(ok, "Invalid wallet code: " << deriv_path << ". Invalid path element: " << cur); + CHECK_AND_ASSERT_THROW_MES(match[0].length() > 0, "Invalid wallet code: " << deriv_path << ". Invalid path element: " << cur); + + const unsigned long cidx = std::stoul(match[0].str()) | TREZOR_BIP44_HARDENED_ZERO; + this->m_wallet_deriv_path.push_back((unsigned int)cidx); + } + } + + /* ======================================================================= */ + /* TREZOR PROTOCOL */ + /* ======================================================================= */ + + bool device_trezor_base::ping() { + TREZOR_AUTO_LOCK_CMD(); + if (!m_transport){ + MINFO("Ping failed, device not connected"); + return false; + } + + try { + this->call_ping_unsafe(); + return true; + + } catch(std::exception const& e) { + MERROR("Ping failed, exception thrown " << e.what()); + } catch(...){ + MERROR("Ping failed, general exception thrown" << boost::current_exception_diagnostic_information()); + } + + return false; + } + + void device_trezor_base::device_state_reset_unsafe() + { + require_connected(); + auto initMsg = std::make_shared(); + + if(!m_device_state.empty()) { + initMsg->set_allocated_state(&m_device_state); + } + + m_features = this->client_exchange(initMsg); + initMsg->release_state(); + } + + void device_trezor_base::device_state_reset() + { + TREZOR_AUTO_LOCK_CMD(); + device_state_reset_unsafe(); + } + +#ifdef WITH_TREZOR_DEBUGGING +#define TREZOR_CALLBACK(method, ...) do { \ + if (m_debug_callback) m_debug_callback->method(__VA_ARGS__); \ + if (m_callback) m_callback->method(__VA_ARGS__); \ +}while(0) +#define TREZOR_CALLBACK_GET(VAR, method, ...) do { \ + if (m_debug_callback) VAR = m_debug_callback->method(__VA_ARGS__); \ + if (m_callback) VAR = m_callback->method(__VA_ARGS__); \ +}while(0) + + void device_trezor_base::setup_debug(){ + if (!m_debug){ + return; + } + + if (!m_debug_callback){ + CHECK_AND_ASSERT_THROW_MES(m_transport, "Transport does not exist"); + auto debug_transport = m_transport->find_debug(); + if (debug_transport) { + m_debug_callback = std::make_shared(debug_transport); + } else { + MDEBUG("Transport does not have debug link option"); + } + } + } + +#else +#define TREZOR_CALLBACK(method, ...) do { if (m_callback) m_callback->method(__VA_ARGS__); } while(0) +#define TREZOR_CALLBACK_GET(VAR, method, ...) VAR = (m_callback ? m_callback->method(__VA_ARGS__) : boost::none) +#endif + + void device_trezor_base::on_button_request(GenericMessage & resp, const messages::common::ButtonRequest * msg) + { + CHECK_AND_ASSERT_THROW_MES(msg, "Empty message"); + MDEBUG("on_button_request, code: " << msg->code()); + + TREZOR_CALLBACK(on_button_request, msg->code()); + + messages::common::ButtonAck ack; + write_raw(&ack); + + resp = read_raw(); + } + + void device_trezor_base::on_button_pressed() + { + TREZOR_CALLBACK(on_button_pressed); + } + + void device_trezor_base::on_pin_request(GenericMessage & resp, const messages::common::PinMatrixRequest * msg) + { + MDEBUG("on_pin_request"); + CHECK_AND_ASSERT_THROW_MES(msg, "Empty message"); + + boost::optional pin; + TREZOR_CALLBACK_GET(pin, on_pin_request); + + if (!pin && m_pin){ + pin = m_pin; + } + + // TODO: remove PIN from memory + messages::common::PinMatrixAck m; + if (pin) { + m.set_pin(pin.get().data(), pin.get().size()); + } + resp = call_raw(&m); + } + + void device_trezor_base::on_passphrase_request(GenericMessage & resp, const messages::common::PassphraseRequest * msg) + { + CHECK_AND_ASSERT_THROW_MES(msg, "Empty message"); + MDEBUG("on_passhprase_request, on device: " << msg->on_device()); + boost::optional passphrase; + TREZOR_CALLBACK_GET(passphrase, on_passphrase_request, msg->on_device()); + + if (!passphrase && m_passphrase){ + passphrase = m_passphrase; + } + + m_passphrase = boost::none; + + messages::common::PassphraseAck m; + if (!msg->on_device() && passphrase){ + // TODO: remove passphrase from memory + m.set_passphrase(passphrase.get().data(), passphrase.get().size()); + } + + if (!m_device_state.empty()){ + m.set_allocated_state(&m_device_state); + } + + resp = call_raw(&m); + m.release_state(); + } + + void device_trezor_base::on_passphrase_state_request(GenericMessage & resp, const messages::common::PassphraseStateRequest * msg) + { + MDEBUG("on_passhprase_state_request"); + CHECK_AND_ASSERT_THROW_MES(msg, "Empty message"); + + m_device_state = msg->state(); + messages::common::PassphraseStateAck m; + resp = call_raw(&m); + } + +#ifdef WITH_TREZOR_DEBUGGING + void device_trezor_base::wipe_device() + { + auto msg = std::make_shared(); + auto ret = client_exchange(msg); + (void)ret; + init_device(); + } + + void device_trezor_base::init_device() + { + auto msg = std::make_shared(); + m_features = client_exchange(msg); + } + + void device_trezor_base::load_device(const std::string & mnemonic, const std::string & pin, + bool passphrase_protection, const std::string & label, const std::string & language, + bool skip_checksum, bool expand) + { + if (m_features && m_features->initialized()){ + throw std::runtime_error("Device is initialized already. Call device.wipe() and try again."); + } + + auto msg = std::make_shared(); + msg->set_mnemonic(mnemonic); + msg->set_pin(pin); + msg->set_passphrase_protection(passphrase_protection); + msg->set_label(label); + msg->set_language(language); + msg->set_skip_checksum(skip_checksum); + auto ret = client_exchange(msg); + (void)ret; + + init_device(); + } + + trezor_debug_callback::trezor_debug_callback(std::shared_ptr & debug_transport){ + m_debug_link = std::make_shared(); + m_debug_link->init(debug_transport); + } + + void trezor_debug_callback::on_button_request(uint64_t code) { + if (m_debug_link) m_debug_link->press_yes(); + } + + boost::optional trezor_debug_callback::on_pin_request() { + return boost::none; + } + + boost::optional trezor_debug_callback::on_passphrase_request(bool on_device) { + return boost::none; + } + + void trezor_debug_callback::on_passphrase_state_request(const std::string &state) { + + } + + void trezor_debug_callback::on_disconnect(){ + if (m_debug_link) m_debug_link->close(); + } +#endif + +#endif //WITH_DEVICE_TREZOR +}} diff --git a/src/device_trezor/device_trezor_base.hpp b/src/device_trezor/device_trezor_base.hpp new file mode 100644 index 00000000000..c106d20990c --- /dev/null +++ b/src/device_trezor/device_trezor_base.hpp @@ -0,0 +1,342 @@ +// Copyright (c) 2017-2019, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +#ifndef MONERO_DEVICE_TREZOR_BASE_H +#define MONERO_DEVICE_TREZOR_BASE_H + + +#include +#include +#include "device/device.hpp" +#include "device/device_default.hpp" +#include "device/device_cold.hpp" +#include +#include +#include +#include "cryptonote_config.h" +#include "trezor.hpp" + +#ifdef WITH_TREZOR_DEBUGGING +#include "trezor/debug_link.hpp" +#endif + +//automatic lock one more level on device ensuring the current thread is allowed to use it +#define TREZOR_AUTO_LOCK_CMD() \ + /* lock both mutexes without deadlock*/ \ + boost::lock(device_locker, command_locker); \ + /* make sure both already-locked mutexes are unlocked at the end of scope */ \ + boost::lock_guard lock1(device_locker, boost::adopt_lock); \ + boost::lock_guard lock2(command_locker, boost::adopt_lock) + +#define TREZOR_AUTO_LOCK_DEVICE() boost::lock_guard lock1_device(device_locker) + +namespace hw { +namespace trezor { + +#ifdef WITH_DEVICE_TREZOR + class device_trezor_base; + +#ifdef WITH_TREZOR_DEBUGGING + class trezor_debug_callback : public hw::i_device_callback { + public: + trezor_debug_callback()=default; + explicit trezor_debug_callback(std::shared_ptr & debug_transport); + + void on_button_request(uint64_t code=0) override; + boost::optional on_pin_request() override; + boost::optional on_passphrase_request(bool on_device) override; + void on_passphrase_state_request(const std::string &state); + void on_disconnect(); + protected: + std::shared_ptr m_debug_link; + }; + +#endif + + /** + * TREZOR device template with basic functions + */ + class device_trezor_base : public hw::core::device_default { + protected: + + // Locker for concurrent access + mutable boost::recursive_mutex device_locker; + mutable boost::mutex command_locker; + + std::shared_ptr m_transport; + i_device_callback * m_callback; + + std::string m_full_name; + std::vector m_wallet_deriv_path; + std::string m_device_state; // returned after passphrase entry, session + std::shared_ptr m_features; // features from the last device reset + boost::optional m_pin; + boost::optional m_passphrase; + messages::MessageType m_last_msg_type; + + cryptonote::network_type network_type; + +#ifdef WITH_TREZOR_DEBUGGING + std::shared_ptr m_debug_callback; + bool m_debug; + + void setup_debug(); +#endif + + // + // Internal methods + // + + void require_connected() const; + void require_initialized() const; + void call_ping_unsafe(); + void test_ping(); + virtual void device_state_reset_unsafe(); + void ensure_derivation_path() noexcept; + + // Communication methods + + void write_raw(const google::protobuf::Message * msg); + GenericMessage read_raw(); + GenericMessage call_raw(const google::protobuf::Message * msg); + + // Trezor message protocol handler. Handles specific signalling messages. + bool message_handler(GenericMessage & input); + + /** + * Client communication wrapper, handles specific Trezor protocol. + * + * @throws UnexpectedMessageException if the response message type is different than expected. + * Exception contains message type and the message itself. + */ + template + std::shared_ptr + client_exchange(const std::shared_ptr &req, + const boost::optional & resp_type = boost::none, + const boost::optional> & resp_types = boost::none, + const boost::optional & resp_type_ptr = boost::none, + bool open_session = false) + { + // Require strictly protocol buffers response in the template. + BOOST_STATIC_ASSERT(boost::is_base_of::value); + const bool accepting_base = boost::is_same::value; + if (resp_types && !accepting_base){ + throw std::invalid_argument("Cannot specify list of accepted types and not using generic response"); + } + + // Determine type of expected message response + const messages::MessageType required_type = accepting_base ? messages::MessageType_Success : + (resp_type ? resp_type.get() : MessageMapper::get_message_wire_number()); + + // Open session if required + if (open_session){ + try { + m_transport->open(); + } catch (const std::exception& e) { + std::throw_with_nested(exc::SessionException("Could not open session")); + } + } + + // Scoped session closer + BOOST_SCOPE_EXIT_ALL(&, this) { + if (open_session){ + this->get_transport()->close(); + } + }; + + // Write/read the request + CHECK_AND_ASSERT_THROW_MES(req, "Request is null"); + auto msg_resp = call_raw(req.get()); + + bool processed = false; + do { + processed = message_handler(msg_resp); + } while(processed); + + // Response section + if (resp_type_ptr){ + *(resp_type_ptr.get()) = msg_resp.m_type; + } + + if (msg_resp.m_type == messages::MessageType_Failure) { + throw_failure_exception(dynamic_cast(msg_resp.m_msg.get())); + + } else if (!accepting_base && msg_resp.m_type == required_type) { + return message_ptr_retype(msg_resp.m_msg); + + } else if (accepting_base && (!resp_types || + std::find(resp_types.get().begin(), resp_types.get().end(), msg_resp.m_type) != resp_types.get().end())) { + return message_ptr_retype(msg_resp.m_msg); + + } else { + throw exc::UnexpectedMessageException(msg_resp.m_type, msg_resp.m_msg); + } + } + + /** + * Utility method to set address_n and network type to the message requets. + */ + template + void set_msg_addr(t_message * msg, + const boost::optional> & path = boost::none, + const boost::optional & network_type = boost::none) + { + CHECK_AND_ASSERT_THROW_MES(msg, "Message is null"); + msg->clear_address_n(); + if (path){ + for(auto x : path.get()){ + msg->add_address_n(x); + } + } else { + ensure_derivation_path(); + for (unsigned int i : DEFAULT_BIP44_PATH) { + msg->add_address_n(i); + } + for (unsigned int i : m_wallet_deriv_path) { + msg->add_address_n(i); + } + } + + if (network_type){ + msg->set_network_type(static_cast(network_type.get())); + } else { + msg->set_network_type(static_cast(this->network_type)); + } + } + + public: + device_trezor_base(); + ~device_trezor_base() override; + + device_trezor_base(const device_trezor_base &device) = delete ; + device_trezor_base& operator=(const device_trezor_base &device) = delete; + + explicit operator bool() const override {return true;} + device_type get_type() const override {return device_type::TREZOR;}; + + bool reset(); + + // Default derivation path for Monero + static const uint32_t DEFAULT_BIP44_PATH[2]; + + std::shared_ptr get_transport(){ + return m_transport; + } + + void set_callback(i_device_callback * callback) override { + m_callback = callback; + } + + i_device_callback * get_callback(){ + return m_callback; + } + + std::shared_ptr & get_features() { + return m_features; + } + + uint64_t get_version() const { + CHECK_AND_ASSERT_THROW_MES(m_features, "Features not loaded"); + CHECK_AND_ASSERT_THROW_MES(m_features->has_major_version() && m_features->has_minor_version() && m_features->has_patch_version(), "Invalid Trezor firmware version information"); + return pack_version(m_features->major_version(), m_features->minor_version(), m_features->patch_version()); + } + + void set_derivation_path(const std::string &deriv_path) override; + + virtual bool has_ki_live_refresh(void) const override { return false; } + + virtual void set_pin(const epee::wipeable_string & pin) override { + m_pin = pin; + } + virtual void set_passphrase(const epee::wipeable_string & passphrase) override { + m_passphrase = passphrase; + } + + /* ======================================================================= */ + /* SETUP/TEARDOWN */ + /* ======================================================================= */ + bool set_name(const std::string &name) override; + + const std::string get_name() const override; + bool init() override; + bool release() override; + bool connect() override; + bool disconnect() override; + + /* ======================================================================= */ + /* LOCKER */ + /* ======================================================================= */ + void lock() override; + void unlock() override; + bool try_lock() override; + + /* ======================================================================= */ + /* TREZOR PROTOCOL */ + /* ======================================================================= */ + + /** + * Device ping, no-throw + */ + bool ping(); + + /** + * Performs Initialize call to the Trezor, resets to known state. + */ + void device_state_reset(); + + // Protocol callbacks + void on_button_request(GenericMessage & resp, const messages::common::ButtonRequest * msg); + void on_button_pressed(); + void on_pin_request(GenericMessage & resp, const messages::common::PinMatrixRequest * msg); + void on_passphrase_request(GenericMessage & resp, const messages::common::PassphraseRequest * msg); + void on_passphrase_state_request(GenericMessage & resp, const messages::common::PassphraseStateRequest * msg); + +#ifdef WITH_TREZOR_DEBUGGING + void set_debug(bool debug){ + m_debug = debug; + } + + void set_debug_callback(std::shared_ptr & debug_callback){ + m_debug_callback = debug_callback; + } + + void wipe_device(); + void init_device(); + void load_device(const std::string & mnemonic, const std::string & pin="", bool passphrase_protection=false, + const std::string & label="test", const std::string & language="english", + bool skip_checksum=false, bool expand=false); + +#endif + }; + +#endif + +} +} +#endif //MONERO_DEVICE_TREZOR_BASE_H diff --git a/src/device_trezor/trezor.hpp b/src/device_trezor/trezor.hpp new file mode 100644 index 00000000000..396a27534e7 --- /dev/null +++ b/src/device_trezor/trezor.hpp @@ -0,0 +1,44 @@ +// Copyright (c) 2017-2019, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +#ifndef MONERO_TREZOR_HPP +#define MONERO_TREZOR_HPP + +#include "trezor/trezor_defs.hpp" + +#ifdef WITH_DEVICE_TREZOR +#include "trezor/transport.hpp" +#include "trezor/messages/messages.pb.h" +#include "trezor/messages/messages-common.pb.h" +#include "trezor/messages/messages-management.pb.h" +#include "trezor/messages/messages-monero.pb.h" +#include "trezor/protocol.hpp" +#endif + +#endif //MONERO_TREZOR_HPP diff --git a/src/device_trezor/trezor/debug_link.cpp b/src/device_trezor/trezor/debug_link.cpp new file mode 100644 index 00000000000..c7ee59afe14 --- /dev/null +++ b/src/device_trezor/trezor/debug_link.cpp @@ -0,0 +1,90 @@ +// Copyright (c) 2017-2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +#include "debug_link.hpp" + +namespace hw{ +namespace trezor{ + + DebugLink::DebugLink(){ + + } + + DebugLink::~DebugLink(){ + if (m_transport){ + close(); + } + } + + void DebugLink::init(std::shared_ptr & transport){ + CHECK_AND_ASSERT_THROW_MES(!m_transport, "Already initialized"); + m_transport = transport; + m_transport->open(); + } + + void DebugLink::close(){ + CHECK_AND_ASSERT_THROW_MES(m_transport, "Not initialized"); + if (m_transport) m_transport->close(); + } + + std::shared_ptr DebugLink::state(){ + return call( + messages::debug::DebugLinkGetState(), + boost::make_optional(messages::MessageType_DebugLinkGetState)); + } + + void DebugLink::input_word(const std::string & word){ + messages::debug::DebugLinkDecision decision; + decision.set_input(word); + call(decision, boost::none, true); + } + + void DebugLink::input_button(bool button){ + messages::debug::DebugLinkDecision decision; + decision.set_yes_no(button); + call(decision, boost::none, true); + } + + void DebugLink::input_swipe(bool swipe){ + messages::debug::DebugLinkDecision decision; + decision.set_up_down(swipe); + call(decision, boost::none, true); + } + + void DebugLink::stop(){ + messages::debug::DebugLinkStop msg; + call(msg, boost::none, true); + } + + + + + +} +} \ No newline at end of file diff --git a/src/device_trezor/trezor/debug_link.hpp b/src/device_trezor/trezor/debug_link.hpp new file mode 100644 index 00000000000..adf5f1d8f0c --- /dev/null +++ b/src/device_trezor/trezor/debug_link.hpp @@ -0,0 +1,93 @@ +// Copyright (c) 2017-2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +#ifndef MONERO_DEBUG_LINK_H +#define MONERO_DEBUG_LINK_H + +#include "transport.hpp" +#include "messages/messages-debug.pb.h" + + +namespace hw { +namespace trezor { + + class DebugLink { + public: + + DebugLink(); + virtual ~DebugLink(); + + void init(std::shared_ptr & transport); + void close(); + + std::shared_ptr state(); + void input_word(const std::string & word); + void input_button(bool button); + void input_swipe(bool swipe); + void press_yes() { input_button(true); } + void press_no() { input_button(false); } + void stop(); + + template + std::shared_ptr call( + const google::protobuf::Message & req, + const boost::optional &resp_type = boost::none, + bool no_wait = false) + { + BOOST_STATIC_ASSERT(boost::is_base_of::value); + + m_transport->write(req); + if (no_wait){ + return nullptr; + } + + // Read the response + std::shared_ptr msg_resp; + hw::trezor::messages::MessageType msg_resp_type; + m_transport->read(msg_resp, &msg_resp_type); + + messages::MessageType required_type = resp_type ? resp_type.get() : MessageMapper::get_message_wire_number(); + if (msg_resp_type == required_type) { + return message_ptr_retype(msg_resp); + } else if (msg_resp_type == messages::MessageType_Failure){ + throw_failure_exception(dynamic_cast(msg_resp.get())); + } else { + throw exc::UnexpectedMessageException(msg_resp_type, msg_resp); + } + }; + + private: + std::shared_ptr m_transport; + + }; + +} +} + +#endif //MONERO_DEBUG_LINK_H diff --git a/src/device_trezor/trezor/exceptions.hpp b/src/device_trezor/trezor/exceptions.hpp new file mode 100644 index 00000000000..41e8c2875fd --- /dev/null +++ b/src/device_trezor/trezor/exceptions.hpp @@ -0,0 +1,193 @@ +// Copyright (c) 2017-2019, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +#ifndef MONERO_EXCEPTIONS_H +#define MONERO_EXCEPTIONS_H + +#include +#include +#include + +namespace hw { +namespace trezor { +namespace exc { + + class SecurityException : public std::exception { + protected: + boost::optional reason; + + public: + SecurityException(): reason("General Security exception"){} + explicit SecurityException(std::string what): reason(what){} + + virtual const char* what() const throw() { + return reason.get().c_str(); + } + }; + + class Poly1305TagInvalid: public SecurityException { + public: + using SecurityException::SecurityException; + Poly1305TagInvalid(): SecurityException("Poly1305 authentication tag invalid"){} + }; + + class TrezorException : public std::exception { + protected: + boost::optional reason; + + public: + TrezorException(): reason("General Trezor exception"){} + explicit TrezorException(std::string what): reason(what){} + + virtual const char* what() const throw() { + return reason.get().c_str(); + } + }; + + class CommunicationException: public TrezorException { + public: + using TrezorException::TrezorException; + CommunicationException(): TrezorException("Trezor communication error"){} + }; + + class EncodingException: public CommunicationException { + public: + using CommunicationException::CommunicationException; + EncodingException(): CommunicationException("Trezor message encoding error"){} + }; + + class NotConnectedException : public CommunicationException { + public: + using CommunicationException::CommunicationException; + NotConnectedException(): CommunicationException("Trezor not connected"){} + }; + + class DeviceNotResponsiveException : public CommunicationException { + public: + using CommunicationException::CommunicationException; + DeviceNotResponsiveException(): CommunicationException("Trezor does not respond to ping"){} + }; + + class DeviceAcquireException : public CommunicationException { + public: + using CommunicationException::CommunicationException; + DeviceAcquireException(): CommunicationException("Trezor could not be acquired"){} + }; + + class SessionException: public CommunicationException { + public: + using CommunicationException::CommunicationException; + SessionException(): CommunicationException("Trezor session expired"){} + }; + + class TimeoutException: public CommunicationException { + public: + using CommunicationException::CommunicationException; + TimeoutException(): CommunicationException("Trezor communication timeout"){} + }; + + class ProtocolException: public CommunicationException { + public: + using CommunicationException::CommunicationException; + ProtocolException(): CommunicationException("Trezor protocol error"){} + }; + + // Communication protocol namespace + // Separated to distinguish between client and Trezor side exceptions. +namespace proto { + + class SecurityException : public ProtocolException { + public: + using ProtocolException::ProtocolException; + SecurityException(): ProtocolException("Security assertion violated in the protocol"){} + }; + + class FailureException : public ProtocolException { + private: + boost::optional code; + boost::optional message; + public: + using ProtocolException::ProtocolException; + FailureException(): ProtocolException("Trezor returned failure"){} + FailureException(boost::optional code, + boost::optional message) + : code(code), message(message) { + reason = "Trezor returned failure: code=" + + (code ? std::to_string(code.get()) : "") + + ", message=" + (message ? message.get() : ""); + }; + }; + + class UnexpectedMessageException : public FailureException { + public: + using FailureException::FailureException; + UnexpectedMessageException(): FailureException("Trezor claims unexpected message received"){} + }; + + class CancelledException : public FailureException { + public: + using FailureException::FailureException; + CancelledException(): FailureException("Trezor returned: cancelled operation"){} + }; + + class PinExpectedException : public FailureException { + public: + using FailureException::FailureException; + PinExpectedException(): FailureException("Trezor claims PIN is expected"){} + }; + + class InvalidPinException : public FailureException { + public: + using FailureException::FailureException; + InvalidPinException(): FailureException("Trezor claims PIN is invalid"){} + }; + + class NotEnoughFundsException : public FailureException { + public: + using FailureException::FailureException; + NotEnoughFundsException(): FailureException("Trezor claims not enough funds"){} + }; + + class NotInitializedException : public FailureException { + public: + using FailureException::FailureException; + NotInitializedException(): FailureException("Trezor claims not initialized"){} + }; + + class FirmwareErrorException : public FailureException { + public: + using FailureException::FailureException; + FirmwareErrorException(): FailureException("Trezor returned firmware error"){} + }; + +} +} +} +} +#endif //MONERO_EXCEPTIONS_H diff --git a/src/device_trezor/trezor/messages/.gitignore b/src/device_trezor/trezor/messages/.gitignore new file mode 100644 index 00000000000..32f7a77e5e2 --- /dev/null +++ b/src/device_trezor/trezor/messages/.gitignore @@ -0,0 +1,2 @@ +# protobuf generated code +* diff --git a/src/device_trezor/trezor/messages_map.cpp b/src/device_trezor/trezor/messages_map.cpp new file mode 100644 index 00000000000..6956b369a12 --- /dev/null +++ b/src/device_trezor/trezor/messages_map.cpp @@ -0,0 +1,136 @@ +// Copyright (c) 2017-2019, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +#include "messages_map.hpp" +#include "messages/messages.pb.h" +#include "messages/messages-common.pb.h" +#include "messages/messages-management.pb.h" +#include "messages/messages-monero.pb.h" + +#ifdef WITH_TREZOR_DEBUGGING +#include "messages/messages-debug.pb.h" +#endif + +using namespace std; +using namespace hw::trezor; + +namespace hw{ +namespace trezor +{ + + const char * TYPE_PREFIX = "MessageType_"; + const char * PACKAGES[] = { + "hw.trezor.messages.", + "hw.trezor.messages.common.", + "hw.trezor.messages.management.", +#ifdef WITH_TREZOR_DEBUGGING + "hw.trezor.messages.debug.", +#endif + "hw.trezor.messages.monero." + }; + + google::protobuf::Message * MessageMapper::get_message(int wire_number) { + return MessageMapper::get_message(static_cast(wire_number)); + } + + google::protobuf::Message * MessageMapper::get_message(messages::MessageType wire_number) { + const string &messageTypeName = hw::trezor::messages::MessageType_Name(wire_number); + if (messageTypeName.empty()) { + throw exc::EncodingException(std::string("Message descriptor not found: ") + std::to_string(wire_number)); + } + + string messageName = messageTypeName.substr(strlen(TYPE_PREFIX)); + return MessageMapper::get_message(messageName); + } + + google::protobuf::Message * MessageMapper::get_message(const std::string & msg_name) { + // Each package instantiation so lookup works + hw::trezor::messages::common::Success::default_instance(); + hw::trezor::messages::management::Cancel::default_instance(); + hw::trezor::messages::monero::MoneroGetAddress::default_instance(); + +#ifdef WITH_TREZOR_DEBUGGING + hw::trezor::messages::debug::DebugLinkDecision::default_instance(); +#endif + + google::protobuf::Descriptor const * desc = nullptr; + for(const string &text : PACKAGES){ + desc = google::protobuf::DescriptorPool::generated_pool() + ->FindMessageTypeByName(text + msg_name); + if (desc != nullptr){ + break; + } + } + + if (desc == nullptr){ + throw exc::EncodingException(std::string("Message not found: ") + msg_name); + } + + google::protobuf::Message* message = + google::protobuf::MessageFactory::generated_factory() + ->GetPrototype(desc)->New(); + + return message; + +// // CODEGEN way, fast +// switch(wire_number){ +// case 501: +// return new messages::monero::MoneroTransactionSignRequest(); +// default: +// throw std::runtime_error("not implemented"); +// } +// +// // CODEGEN message -> number: specification +// // messages::MessageType get_message_wire_number(const messages::monero::MoneroTransactionSignRequest * msg) { return 501; } +// // messages::MessageType get_message_wire_number(const messages::management::ping * msg) +// + } + + messages::MessageType MessageMapper::get_message_wire_number(const google::protobuf::Message * msg){ + return MessageMapper::get_message_wire_number(msg->GetDescriptor()->name()); + } + + messages::MessageType MessageMapper::get_message_wire_number(const google::protobuf::Message & msg){ + return MessageMapper::get_message_wire_number(msg.GetDescriptor()->name()); + } + + messages::MessageType MessageMapper::get_message_wire_number(const std::string & msg_name){ + string enumMessageName = std::string(TYPE_PREFIX) + msg_name; + + messages::MessageType res; + bool r = hw::trezor::messages::MessageType_Parse(enumMessageName, &res); + if (!r){ + throw exc::EncodingException(std::string("Message ") + msg_name + " not found"); + } + + return res; + } + +} +} diff --git a/src/device_trezor/trezor/messages_map.hpp b/src/device_trezor/trezor/messages_map.hpp new file mode 100644 index 00000000000..28fe9f4c27d --- /dev/null +++ b/src/device_trezor/trezor/messages_map.hpp @@ -0,0 +1,94 @@ +// Copyright (c) 2017-2019, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +#ifndef MONERO_MESSAGES_MAP_H +#define MONERO_MESSAGES_MAP_H + +#include +#include +#include +#include "exceptions.hpp" + +#include "trezor_defs.hpp" + +#include +#include +#include +#include +#include +#include "google/protobuf/descriptor.pb.h" + +#include "messages/messages.pb.h" + +namespace hw { +namespace trezor { + + class MessageMapper{ + public: + MessageMapper() { + + } + + static ::google::protobuf::Message * get_message(int wire_number); + static ::google::protobuf::Message * get_message(messages::MessageType); + static ::google::protobuf::Message * get_message(const std::string & msg_name); + static messages::MessageType get_message_wire_number(const google::protobuf::Message * msg); + static messages::MessageType get_message_wire_number(const google::protobuf::Message & msg); + static messages::MessageType get_message_wire_number(const std::string & msg_name); + + template + static messages::MessageType get_message_wire_number() { + BOOST_STATIC_ASSERT(boost::is_base_of::value); + return get_message_wire_number(t_message::default_instance().GetDescriptor()->name()); + } + }; + + template + std::shared_ptr message_ptr_retype(std::shared_ptr & in){ + BOOST_STATIC_ASSERT(boost::is_base_of::value); + if (!in){ + return nullptr; + } + + return std::dynamic_pointer_cast(in); + } + + template + std::shared_ptr message_ptr_retype_static(std::shared_ptr & in){ + BOOST_STATIC_ASSERT(boost::is_base_of::value); + if (!in){ + return nullptr; + } + + return std::static_pointer_cast(in); + } + +}} + +#endif //MONERO_MESSAGES_MAP_H diff --git a/src/device_trezor/trezor/protocol.cpp b/src/device_trezor/trezor/protocol.cpp new file mode 100644 index 00000000000..5fe08abbe73 --- /dev/null +++ b/src/device_trezor/trezor/protocol.cpp @@ -0,0 +1,1087 @@ +// Copyright (c) 2017-2019, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +#include "protocol.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "cryptonote_config.h" +#include +#include +#include + +#define GET_FIELD_STRING(name, type, jtype) field_##name = std::string(json[#name].GetString(), json[#name].GetStringLength()) +#define GET_FIELD_OTHER(name, type, jtype) field_##name = static_cast(json[#name].Get##jtype()) + +#define GET_STRING_FROM_JSON(json, name, type, mandatory, def) \ + GET_FIELD_FROM_JSON_EX(json, name, type, String, mandatory, def, GET_FIELD_STRING) + +#define GET_FIELD_FROM_JSON(json, name, type, jtype, mandatory, def) \ + GET_FIELD_FROM_JSON_EX(json, name, type, jtype, mandatory, def, GET_FIELD_OTHER) + +#define GET_FIELD_FROM_JSON_EX(json, name, type, jtype, mandatory, def, VAL) \ + type field_##name = static_cast(def); \ + bool field_##name##_found = false; \ + (void)field_##name##_found; \ + do if (json.HasMember(#name)) \ + { \ + if (json[#name].Is##jtype()) \ + { \ + VAL(name, type, jtype); \ + field_##name##_found = true; \ + } \ + else \ + { \ + throw std::invalid_argument("Field " #name " found in JSON, but not " #jtype); \ + } \ + } \ + else if (mandatory) \ + { \ + throw std::invalid_argument("Field " #name " not found in JSON");\ + } while(0) + + +namespace hw{ +namespace trezor{ +namespace protocol{ + + std::string key_to_string(const ::crypto::ec_point & key){ + return std::string(key.data, sizeof(key.data)); + } + + std::string key_to_string(const ::crypto::ec_scalar & key){ + return std::string(key.data, sizeof(key.data)); + } + + std::string key_to_string(const ::crypto::hash & key){ + return std::string(key.data, sizeof(key.data)); + } + + std::string key_to_string(const ::rct::key & key){ + return std::string(reinterpret_cast(key.bytes), sizeof(key.bytes)); + } + + void string_to_key(::crypto::ec_scalar & key, const std::string & str){ + if (str.size() != sizeof(key.data)){ + throw std::invalid_argument(std::string("Key has to have ") + std::to_string(sizeof(key.data)) + " B"); + } + memcpy(key.data, str.data(), sizeof(key.data)); + } + + void string_to_key(::crypto::ec_point & key, const std::string & str){ + if (str.size() != sizeof(key.data)){ + throw std::invalid_argument(std::string("Key has to have ") + std::to_string(sizeof(key.data)) + " B"); + } + memcpy(key.data, str.data(), sizeof(key.data)); + } + + void string_to_key(::rct::key & key, const std::string & str){ + if (str.size() != sizeof(key.bytes)){ + throw std::invalid_argument(std::string("Key has to have ") + std::to_string(sizeof(key.bytes)) + " B"); + } + memcpy(key.bytes, str.data(), sizeof(key.bytes)); + } + +namespace crypto { +namespace chacha { + + void decrypt(const void* ciphertext, size_t length, const uint8_t* key, const uint8_t* iv, char* plaintext, size_t *plaintext_len){ + CHECK_AND_ASSERT_THROW_MES(length >= TAG_SIZE, "Ciphertext length too small"); + CHECK_AND_ASSERT_THROW_MES(!plaintext_len || *plaintext_len >= (length - TAG_SIZE), "Plaintext length too small"); + + unsigned long long int res_len = plaintext_len ? *plaintext_len : length; + auto r = crypto_aead_chacha20poly1305_ietf_decrypt( + reinterpret_cast(plaintext), &res_len, nullptr, + static_cast(ciphertext), length, nullptr, 0, iv, key); + + if (r != 0){ + throw exc::Poly1305TagInvalid(); + } + + if (plaintext_len){ + *plaintext_len = (size_t) res_len; + } + } + +} +} + + +// Cold Key image sync +namespace ki { + + bool key_image_data(wallet_shim * wallet, + const std::vector & transfers, + std::vector & res) + { + for(auto & td : transfers){ + ::crypto::public_key tx_pub_key = wallet->get_tx_pub_key_from_received_outs(td); + const std::vector<::crypto::public_key> additional_tx_pub_keys = cryptonote::get_additional_tx_pub_keys_from_extra(td.m_tx); + + res.emplace_back(); + auto & cres = res.back(); + + cres.set_out_key(key_to_string(boost::get(td.m_tx.vout[td.m_internal_output_index].target).key)); + cres.set_tx_pub_key(key_to_string(tx_pub_key)); + cres.set_internal_output_index(td.m_internal_output_index); + for(auto & aux : additional_tx_pub_keys){ + cres.add_additional_tx_pub_keys(key_to_string(aux)); + } + } + + return true; + } + + std::string compute_hash(const MoneroTransferDetails & rr){ + KECCAK_CTX kck; + uint8_t md[32]; + + CHECK_AND_ASSERT_THROW_MES(rr.out_key().size() == 32, "Invalid out_key size"); + CHECK_AND_ASSERT_THROW_MES(rr.tx_pub_key().size() == 32, "Invalid tx_pub_key size"); + + keccak_init(&kck); + keccak_update(&kck, reinterpret_cast(rr.out_key().data()), 32); + keccak_update(&kck, reinterpret_cast(rr.tx_pub_key().data()), 32); + for (const auto &aux : rr.additional_tx_pub_keys()){ + CHECK_AND_ASSERT_THROW_MES(aux.size() == 32, "Invalid aux size"); + keccak_update(&kck, reinterpret_cast(aux.data()), 32); + } + + auto index_serialized = tools::get_varint_data(rr.internal_output_index()); + keccak_update(&kck, reinterpret_cast(index_serialized.data()), index_serialized.size()); + keccak_finish(&kck, md); + return std::string(reinterpret_cast(md), sizeof(md)); + } + + void generate_commitment(std::vector & mtds, + const std::vector & transfers, + std::shared_ptr & req) + { + req = std::make_shared(); + + KECCAK_CTX kck; + uint8_t final_hash[32]; + keccak_init(&kck); + + for(auto &cur : mtds){ + auto hash = compute_hash(cur); + keccak_update(&kck, reinterpret_cast(hash.data()), hash.size()); + } + keccak_finish(&kck, final_hash); + + req = std::make_shared(); + req->set_hash(std::string(reinterpret_cast(final_hash), 32)); + req->set_num(transfers.size()); + + std::unordered_map> sub_indices; + for (auto &cur : transfers){ + auto search = sub_indices.emplace(cur.m_subaddr_index.major, std::set()); + auto & st = search.first->second; + st.insert(cur.m_subaddr_index.minor); + } + + for (auto& x: sub_indices){ + auto subs = req->add_subs(); + subs->set_account(x.first); + for(auto minor : x.second){ + subs->add_minor_indices(minor); + } + } + } + + void live_refresh_ack(const ::crypto::secret_key & view_key_priv, + const ::crypto::public_key& out_key, + const std::shared_ptr & ack, + ::cryptonote::keypair& in_ephemeral, + ::crypto::key_image& ki) + { + std::string str_out_key(out_key.data, sizeof(out_key.data)); + auto enc_key = protocol::tx::compute_enc_key(view_key_priv, str_out_key, ack->salt()); + + const size_t len_ciphertext = ack->key_image().size(); // IV || keys + CHECK_AND_ASSERT_THROW_MES(len_ciphertext > crypto::chacha::IV_SIZE + crypto::chacha::TAG_SIZE, "Invalid size"); + + size_t ki_len = len_ciphertext - crypto::chacha::IV_SIZE - crypto::chacha::TAG_SIZE; + std::unique_ptr plaintext(new uint8_t[ki_len]); + uint8_t * buff = plaintext.get(); + + protocol::crypto::chacha::decrypt( + ack->key_image().data() + crypto::chacha::IV_SIZE, + len_ciphertext - crypto::chacha::IV_SIZE, + reinterpret_cast(enc_key.data), + reinterpret_cast(ack->key_image().data()), + reinterpret_cast(buff), &ki_len); + + CHECK_AND_ASSERT_THROW_MES(ki_len == 3*32, "Invalid size"); + ::crypto::signature sig{}; + memcpy(ki.data, buff, 32); + memcpy(sig.c.data, buff + 32, 32); + memcpy(sig.r.data, buff + 64, 32); + in_ephemeral.pub = out_key; + in_ephemeral.sec = ::crypto::null_skey; + + // Verification + std::vector pkeys; + pkeys.push_back(&out_key); + + CHECK_AND_ASSERT_THROW_MES(rct::scalarmultKey(rct::ki2rct(ki), rct::curveOrder()) == rct::identity(), + "Key image out of validity domain: key image " << epee::string_tools::pod_to_hex(ki)); + + CHECK_AND_ASSERT_THROW_MES(::crypto::check_ring_signature((const ::crypto::hash&)ki, ki, pkeys, &sig), + "Signature failed for key image " << epee::string_tools::pod_to_hex(ki) + << ", signature " + epee::string_tools::pod_to_hex(sig) + << ", pubkey " + epee::string_tools::pod_to_hex(*pkeys[0])); + } +} + +// Cold transaction signing +namespace tx { + + void translate_address(MoneroAccountPublicAddress * dst, const cryptonote::account_public_address * src){ + dst->set_view_public_key(key_to_string(src->m_view_public_key)); + dst->set_spend_public_key(key_to_string(src->m_spend_public_key)); + } + + void translate_dst_entry(MoneroTransactionDestinationEntry * dst, const cryptonote::tx_destination_entry * src){ + dst->set_amount(src->amount); + dst->set_is_subaddress(src->is_subaddress); + dst->set_is_integrated(src->is_integrated); + dst->set_original(src->original); + translate_address(dst->mutable_addr(), &(src->addr)); + } + + void translate_src_entry(MoneroTransactionSourceEntry * dst, const cryptonote::tx_source_entry * src){ + for(auto & cur : src->outputs){ + auto out = dst->add_outputs(); + out->set_idx(cur.first); + translate_rct_key(out->mutable_key(), &(cur.second)); + } + + dst->set_real_output(src->real_output); + dst->set_real_out_tx_key(key_to_string(src->real_out_tx_key)); + for(auto & cur : src->real_out_additional_tx_keys){ + dst->add_real_out_additional_tx_keys(key_to_string(cur)); + } + + dst->set_real_output_in_tx_index(src->real_output_in_tx_index); + dst->set_amount(src->amount); + dst->set_rct(src->rct); + dst->set_mask(key_to_string(src->mask)); + translate_klrki(dst->mutable_multisig_klrki(), &(src->multisig_kLRki)); + } + + void translate_klrki(MoneroMultisigKLRki * dst, const rct::multisig_kLRki * src){ + dst->set_k(key_to_string(src->k)); + dst->set_l(key_to_string(src->L)); + dst->set_r(key_to_string(src->R)); + dst->set_ki(key_to_string(src->ki)); + } + + void translate_rct_key(MoneroRctKey * dst, const rct::ctkey * src){ + dst->set_dest(key_to_string(src->dest)); + dst->set_commitment(key_to_string(src->mask)); + } + + std::string hash_addr(const MoneroAccountPublicAddress * addr, boost::optional amount, boost::optional is_subaddr){ + return hash_addr(addr->spend_public_key(), addr->view_public_key(), amount, is_subaddr); + } + + std::string hash_addr(const std::string & spend_key, const std::string & view_key, boost::optional amount, boost::optional is_subaddr){ + ::crypto::public_key spend{}, view{}; + if (spend_key.size() != 32 || view_key.size() != 32){ + throw std::invalid_argument("Public keys have invalid sizes"); + } + + memcpy(spend.data, spend_key.data(), 32); + memcpy(view.data, view_key.data(), 32); + return hash_addr(&spend, &view, amount, is_subaddr); + } + + std::string hash_addr(const ::crypto::public_key * spend_key, const ::crypto::public_key * view_key, boost::optional amount, boost::optional is_subaddr){ + char buff[64+8+1]; + size_t offset = 0; + + memcpy(buff + offset, spend_key->data, 32); offset += 32; + memcpy(buff + offset, view_key->data, 32); offset += 32; + + if (amount){ + memcpy(buff + offset, (uint8_t*) &(amount.get()), sizeof(amount.get())); offset += sizeof(amount.get()); + } + + if (is_subaddr){ + buff[offset] = is_subaddr.get(); + offset += 1; + } + + return std::string(buff, offset); + } + + ::crypto::secret_key compute_enc_key(const ::crypto::secret_key & private_view_key, const std::string & aux, const std::string & salt) + { + uint8_t hash[32]; + KECCAK_CTX ctx; + ::crypto::secret_key res; + + keccak_init(&ctx); + keccak_update(&ctx, (const uint8_t *) private_view_key.data, sizeof(private_view_key.data)); + if (!aux.empty()){ + keccak_update(&ctx, (const uint8_t *) aux.data(), aux.size()); + } + keccak_finish(&ctx, hash); + keccak(hash, sizeof(hash), hash, sizeof(hash)); + + hmac_keccak_hash(hash, (const uint8_t *) salt.data(), salt.size(), hash, sizeof(hash)); + memcpy(res.data, hash, sizeof(hash)); + memwipe(hash, sizeof(hash)); + return res; + } + + TData::TData() { + rsig_type = 0; + bp_version = 0; + cur_input_idx = 0; + cur_output_idx = 0; + cur_batch_idx = 0; + cur_output_in_batch_idx = 0; + } + + Signer::Signer(wallet_shim *wallet2, const unsigned_tx_set * unsigned_tx, size_t tx_idx, hw::tx_aux_data * aux_data) { + m_wallet2 = wallet2; + m_unsigned_tx = unsigned_tx; + m_aux_data = aux_data; + m_tx_idx = tx_idx; + m_ct.tx_data = cur_tx(); + m_multisig = false; + m_client_version = 1; + } + + void Signer::extract_payment_id(){ + const std::vector& tx_extra = cur_tx().extra; + m_ct.tsx_data.set_payment_id(""); + + std::vector tx_extra_fields; + cryptonote::parse_tx_extra(tx_extra, tx_extra_fields); // ok if partially parsed + cryptonote::tx_extra_nonce extra_nonce; + + ::crypto::hash payment_id{}; + if (find_tx_extra_field_by_type(tx_extra_fields, extra_nonce)) + { + ::crypto::hash8 payment_id8{}; + if(cryptonote::get_encrypted_payment_id_from_tx_extra_nonce(extra_nonce.nonce, payment_id8)) + { + m_ct.tsx_data.set_payment_id(std::string(payment_id8.data, 8)); + } + else if (cryptonote::get_payment_id_from_tx_extra_nonce(extra_nonce.nonce, payment_id)) + { + m_ct.tsx_data.set_payment_id(std::string(payment_id.data, 32)); + } + } + } + + static unsigned get_rsig_type(const rct::RCTConfig &rct_config, size_t num_outputs){ + if (rct_config.range_proof_type == rct::RangeProofBorromean){ + return rct::RangeProofBorromean; + } else if (num_outputs > BULLETPROOF_MAX_OUTPUTS){ + return rct::RangeProofMultiOutputBulletproof; + } else { + return rct::RangeProofPaddedBulletproof; + } + } + + static void generate_rsig_batch_sizes(std::vector &batches, unsigned rsig_type, size_t num_outputs){ + size_t amount_batched = 0; + + while(amount_batched < num_outputs){ + if (rsig_type == rct::RangeProofBorromean || rsig_type == rct::RangeProofBulletproof) { + batches.push_back(1); + amount_batched += 1; + + } else if (rsig_type == rct::RangeProofPaddedBulletproof){ + if (num_outputs > BULLETPROOF_MAX_OUTPUTS){ + throw std::invalid_argument("BP padded can support only BULLETPROOF_MAX_OUTPUTS statements"); + } + batches.push_back(num_outputs); + amount_batched += num_outputs; + + } else if (rsig_type == rct::RangeProofMultiOutputBulletproof){ + size_t batch_size = 1; + while (batch_size * 2 + amount_batched <= num_outputs && batch_size * 2 <= BULLETPROOF_MAX_OUTPUTS){ + batch_size *= 2; + } + batch_size = std::min(batch_size, num_outputs - amount_batched); + batches.push_back(batch_size); + amount_batched += batch_size; + + } else { + throw std::invalid_argument("Unknown rsig type"); + } + } + } + + void Signer::compute_integrated_indices(TsxData * tsx_data){ + if (m_aux_data == nullptr || m_aux_data->tx_recipients.empty()){ + return; + } + + auto & chg = tsx_data->change_dts(); + std::string change_hash = hash_addr(&chg.addr(), chg.amount(), chg.is_subaddress()); + + std::vector integrated_indices; + std::set integrated_hashes; + for (auto & cur : m_aux_data->tx_recipients){ + if (!cur.has_payment_id){ + continue; + } + integrated_hashes.emplace(hash_addr(&cur.address.m_spend_public_key, &cur.address.m_view_public_key)); + } + + ssize_t idx = -1; + for (auto & cur : tsx_data->outputs()){ + idx += 1; + + std::string c_hash = hash_addr(&cur.addr(), cur.amount(), cur.is_subaddress()); + if (c_hash == change_hash || cur.is_subaddress()){ + continue; + } + + c_hash = hash_addr(&cur.addr()); + if (integrated_hashes.find(c_hash) != integrated_hashes.end()){ + integrated_indices.push_back((uint32_t)idx); + } + } + + if (!integrated_indices.empty()){ + assign_to_repeatable(tsx_data->mutable_integrated_indices(), integrated_indices.begin(), integrated_indices.end()); + } + } + + std::shared_ptr Signer::step_init(){ + // extract payment ID from construction data + auto & tsx_data = m_ct.tsx_data; + auto & tx = cur_tx(); + + m_ct.tx.version = 2; + m_ct.tx.unlock_time = tx.unlock_time; + m_client_version = (m_aux_data->client_version ? m_aux_data->client_version.get() : 1); + + tsx_data.set_version(1); + tsx_data.set_client_version(client_version()); + tsx_data.set_unlock_time(tx.unlock_time); + tsx_data.set_num_inputs(static_cast(tx.sources.size())); + tsx_data.set_mixin(static_cast(tx.sources[0].outputs.size() - 1)); + tsx_data.set_account(tx.subaddr_account); + assign_to_repeatable(tsx_data.mutable_minor_indices(), tx.subaddr_indices.begin(), tx.subaddr_indices.end()); + + // Rsig decision + auto rsig_data = tsx_data.mutable_rsig_data(); + m_ct.rsig_type = get_rsig_type(tx.rct_config, tx.splitted_dsts.size()); + rsig_data->set_rsig_type(m_ct.rsig_type); + if (tx.rct_config.range_proof_type != rct::RangeProofBorromean){ + m_ct.bp_version = (m_aux_data->bp_version ? m_aux_data->bp_version.get() : 1); + rsig_data->set_bp_version((uint32_t) m_ct.bp_version); + } + + generate_rsig_batch_sizes(m_ct.grouping_vct, m_ct.rsig_type, tx.splitted_dsts.size()); + assign_to_repeatable(rsig_data->mutable_grouping(), m_ct.grouping_vct.begin(), m_ct.grouping_vct.end()); + + translate_dst_entry(tsx_data.mutable_change_dts(), &(tx.change_dts)); + for(auto & cur : tx.splitted_dsts){ + auto dst = tsx_data.mutable_outputs()->Add(); + translate_dst_entry(dst, &cur); + } + + compute_integrated_indices(&tsx_data); + + int64_t fee = 0; + for(auto & cur_in : tx.sources){ + fee += cur_in.amount; + } + for(auto & cur_out : tx.splitted_dsts){ + fee -= cur_out.amount; + } + if (fee < 0){ + throw std::invalid_argument("Fee cannot be negative"); + } + + tsx_data.set_fee(static_cast(fee)); + this->extract_payment_id(); + + auto init_req = std::make_shared(); + init_req->set_version(0); + init_req->mutable_tsx_data()->CopyFrom(tsx_data); + return init_req; + } + + void Signer::step_init_ack(std::shared_ptr ack){ + if (ack->has_rsig_data()){ + m_ct.rsig_param = std::make_shared(ack->rsig_data()); + } + + assign_from_repeatable(&(m_ct.tx_out_entr_hmacs), ack->hmacs().begin(), ack->hmacs().end()); + } + + std::shared_ptr Signer::step_set_input(size_t idx){ + CHECK_AND_ASSERT_THROW_MES(idx < cur_tx().sources.size(), "Invalid source index"); + m_ct.cur_input_idx = idx; + auto res = std::make_shared(); + translate_src_entry(res->mutable_src_entr(), &(cur_tx().sources[idx])); + return res; + } + + void Signer::step_set_input_ack(std::shared_ptr ack){ + auto & vini_str = ack->vini(); + + cryptonote::txin_v vini; + if (!cn_deserialize(vini_str.data(), vini_str.size(), vini)){ + throw exc::ProtocolException("Cannot deserialize vin[i]"); + } + + m_ct.tx.vin.emplace_back(vini); + m_ct.tx_in_hmacs.push_back(ack->vini_hmac()); + m_ct.pseudo_outs.push_back(ack->pseudo_out()); + m_ct.pseudo_outs_hmac.push_back(ack->pseudo_out_hmac()); + m_ct.alphas.push_back(ack->pseudo_out_alpha()); + m_ct.spend_encs.push_back(ack->spend_key()); + } + + void Signer::sort_ki(){ + const size_t input_size = cur_tx().sources.size(); + + m_ct.source_permutation.clear(); + for (size_t n = 0; n < input_size; ++n){ + m_ct.source_permutation.push_back(n); + } + + CHECK_AND_ASSERT_THROW_MES(m_ct.tx.vin.size() == input_size, "Invalid vector size"); + std::sort(m_ct.source_permutation.begin(), m_ct.source_permutation.end(), [&](const size_t i0, const size_t i1) { + const cryptonote::txin_to_key &tk0 = boost::get(m_ct.tx.vin[i0]); + const cryptonote::txin_to_key &tk1 = boost::get(m_ct.tx.vin[i1]); + return memcmp(&tk0.k_image, &tk1.k_image, sizeof(tk0.k_image)) > 0; + }); + + CHECK_AND_ASSERT_THROW_MES(m_ct.tx_in_hmacs.size() == input_size, "Invalid vector size"); + CHECK_AND_ASSERT_THROW_MES(m_ct.pseudo_outs.size() == input_size, "Invalid vector size"); + CHECK_AND_ASSERT_THROW_MES(m_ct.pseudo_outs_hmac.size() == input_size, "Invalid vector size"); + CHECK_AND_ASSERT_THROW_MES(m_ct.alphas.size() == input_size, "Invalid vector size"); + CHECK_AND_ASSERT_THROW_MES(m_ct.spend_encs.size() == input_size, "Invalid vector size"); + CHECK_AND_ASSERT_THROW_MES(m_ct.tx_data.sources.size() == input_size, "Invalid vector size"); + + tools::apply_permutation(m_ct.source_permutation, [&](size_t i0, size_t i1){ + std::swap(m_ct.tx.vin[i0], m_ct.tx.vin[i1]); + std::swap(m_ct.tx_in_hmacs[i0], m_ct.tx_in_hmacs[i1]); + std::swap(m_ct.pseudo_outs[i0], m_ct.pseudo_outs[i1]); + std::swap(m_ct.pseudo_outs_hmac[i0], m_ct.pseudo_outs_hmac[i1]); + std::swap(m_ct.alphas[i0], m_ct.alphas[i1]); + std::swap(m_ct.spend_encs[i0], m_ct.spend_encs[i1]); + std::swap(m_ct.tx_data.sources[i0], m_ct.tx_data.sources[i1]); + }); + } + + std::shared_ptr Signer::step_permutation(){ + sort_ki(); + + auto res = std::make_shared(); + assign_to_repeatable(res->mutable_perm(), m_ct.source_permutation.begin(), m_ct.source_permutation.end()); + + return res; + } + + void Signer::step_permutation_ack(std::shared_ptr ack){ + + } + + std::shared_ptr Signer::step_set_vini_input(size_t idx){ + CHECK_AND_ASSERT_THROW_MES(idx < m_ct.tx_data.sources.size(), "Invalid transaction index"); + CHECK_AND_ASSERT_THROW_MES(idx < m_ct.tx.vin.size(), "Invalid transaction index"); + CHECK_AND_ASSERT_THROW_MES(idx < m_ct.tx_in_hmacs.size(), "Invalid transaction index"); + + m_ct.cur_input_idx = idx; + auto tx = m_ct.tx_data; + auto res = std::make_shared(); + auto & vini = m_ct.tx.vin[idx]; + translate_src_entry(res->mutable_src_entr(), &(tx.sources[idx])); + res->set_vini(cryptonote::t_serializable_object_to_blob(vini)); + res->set_vini_hmac(m_ct.tx_in_hmacs[idx]); + + if (client_version() == 0) { + CHECK_AND_ASSERT_THROW_MES(idx < m_ct.pseudo_outs.size(), "Invalid transaction index"); + CHECK_AND_ASSERT_THROW_MES(idx < m_ct.pseudo_outs_hmac.size(), "Invalid transaction index"); + res->set_pseudo_out(m_ct.pseudo_outs[idx]); + res->set_pseudo_out_hmac(m_ct.pseudo_outs_hmac[idx]); + } + + return res; + } + + void Signer::step_set_vini_input_ack(std::shared_ptr ack){ + + } + + std::shared_ptr Signer::step_all_inputs_set(){ + return std::make_shared(); + } + + void Signer::step_all_inputs_set_ack(std::shared_ptr ack){ + if (client_version() > 0 || !is_offloading()){ + return; + } + + // If offloading, expect rsig configuration. + if (!ack->has_rsig_data()){ + throw exc::ProtocolException("Rsig offloading requires rsig param"); + } + + auto & rsig_data = ack->rsig_data(); + if (!rsig_data.has_mask()){ + throw exc::ProtocolException("Gamma masks not present in offloaded version"); + } + + auto & mask = rsig_data.mask(); + if (mask.size() != 32 * num_outputs()){ + throw exc::ProtocolException("Invalid number of gamma masks"); + } + + m_ct.rsig_gamma.reserve(num_outputs()); + for(size_t c=0; c < num_outputs(); ++c){ + rct::key cmask{}; + memcpy(cmask.bytes, mask.data() + c * 32, 32); + m_ct.rsig_gamma.emplace_back(cmask); + } + } + + std::shared_ptr Signer::step_set_output(size_t idx){ + CHECK_AND_ASSERT_THROW_MES(idx < m_ct.tx_data.splitted_dsts.size(), "Invalid transaction index"); + CHECK_AND_ASSERT_THROW_MES(idx < m_ct.tx_out_entr_hmacs.size(), "Invalid transaction index"); + CHECK_AND_ASSERT_THROW_MES(is_req_bulletproof(), "Borromean rsig not supported"); + + m_ct.cur_output_idx = idx; + m_ct.cur_output_in_batch_idx += 1; // assumes sequential call to step_set_output() + + auto res = std::make_shared(); + auto & cur_dst = m_ct.tx_data.splitted_dsts[idx]; + translate_dst_entry(res->mutable_dst_entr(), &cur_dst); + res->set_dst_entr_hmac(m_ct.tx_out_entr_hmacs[idx]); + + // Range sig offloading to the host + // ClientV0 sends offloaded BP with the last message in the batch. + // ClientV1 needs additional message after the last message in the batch as BP uses deterministic masks. + if (client_version() == 0 && is_offloading() && should_compute_bp_now()) { + auto rsig_data = res->mutable_rsig_data(); + compute_bproof(*rsig_data); + } + + return res; + } + + void Signer::step_set_output_ack(std::shared_ptr ack){ + cryptonote::tx_out tx_out; + rct::Bulletproof bproof{}; + rct::ctkey out_pk{}; + rct::ecdhTuple ecdh{}; + + bool has_rsig = false; + std::string rsig_buff; + + if (ack->has_rsig_data()){ + auto & rsig_data = ack->rsig_data(); + + if (rsig_data.has_rsig() && !rsig_data.rsig().empty()){ + has_rsig = true; + rsig_buff = rsig_data.rsig(); + } + + if (client_version() >= 1 && rsig_data.has_mask()){ + rct::key cmask{}; + string_to_key(cmask, rsig_data.mask()); + m_ct.rsig_gamma.emplace_back(cmask); + } + } + + if (!cn_deserialize(ack->tx_out(), tx_out)){ + throw exc::ProtocolException("Cannot deserialize vout[i]"); + } + + if (!cn_deserialize(ack->out_pk(), out_pk)){ + throw exc::ProtocolException("Cannot deserialize out_pk"); + } + + if (m_ct.bp_version <= 1) { + if (!cn_deserialize(ack->ecdh_info(), ecdh)){ + throw exc::ProtocolException("Cannot deserialize ecdhtuple"); + } + } else { + CHECK_AND_ASSERT_THROW_MES(8 == ack->ecdh_info().size(), "Invalid ECDH.amount size"); + memcpy(ecdh.amount.bytes, ack->ecdh_info().data(), 8); + } + + if (has_rsig && is_req_bulletproof() && !cn_deserialize(rsig_buff, bproof)){ + throw exc::ProtocolException("Cannot deserialize bulletproof rangesig"); + } + + m_ct.tx.vout.emplace_back(tx_out); + m_ct.tx_out_hmacs.push_back(ack->vouti_hmac()); + m_ct.tx_out_pk.emplace_back(out_pk); + m_ct.tx_out_ecdh.emplace_back(ecdh); + + // ClientV0, if no rsig was generated on Trezor, do not continue. + // ClientV1+ generates BP after all masks in the current batch are generated + if (!has_rsig || (client_version() >= 1 && is_offloading())){ + return; + } + + process_bproof(bproof); + m_ct.cur_batch_idx += 1; + m_ct.cur_output_in_batch_idx = 0; + } + + bool Signer::should_compute_bp_now() const { + CHECK_AND_ASSERT_THROW_MES(m_ct.cur_batch_idx < m_ct.grouping_vct.size(), "Invalid batch index"); + return m_ct.grouping_vct[m_ct.cur_batch_idx] <= m_ct.cur_output_in_batch_idx; + } + + void Signer::compute_bproof(messages::monero::MoneroTransactionRsigData & rsig_data){ + auto batch_size = m_ct.grouping_vct[m_ct.cur_batch_idx]; + std::vector amounts; + rct::keyV masks; + CHECK_AND_ASSERT_THROW_MES(m_ct.cur_output_idx + 1 >= batch_size, "Invalid index for batching"); + + for(size_t i = 0; i < batch_size; ++i){ + const size_t bidx = 1 + m_ct.cur_output_idx - batch_size + i; + CHECK_AND_ASSERT_THROW_MES(bidx < m_ct.tx_data.splitted_dsts.size(), "Invalid gamma index"); + CHECK_AND_ASSERT_THROW_MES(bidx < m_ct.rsig_gamma.size(), "Invalid gamma index"); + + amounts.push_back(m_ct.tx_data.splitted_dsts[bidx].amount); + masks.push_back(m_ct.rsig_gamma[bidx]); + } + + auto bp = bulletproof_PROVE(amounts, masks); + auto serRsig = cn_serialize(bp); + m_ct.tx_out_rsigs.emplace_back(bp); + rsig_data.set_rsig(serRsig); + } + + void Signer::process_bproof(rct::Bulletproof & bproof){ + CHECK_AND_ASSERT_THROW_MES(m_ct.cur_batch_idx < m_ct.grouping_vct.size(), "Invalid batch index"); + auto batch_size = m_ct.grouping_vct[m_ct.cur_batch_idx]; + for (size_t i = 0; i < batch_size; ++i){ + const size_t bidx = 1 + m_ct.cur_output_idx - batch_size + i; + CHECK_AND_ASSERT_THROW_MES(bidx < m_ct.tx_out_pk.size(), "Invalid out index"); + + rct::key commitment = m_ct.tx_out_pk[bidx].mask; + commitment = rct::scalarmultKey(commitment, rct::INV_EIGHT); + bproof.V.push_back(commitment); + } + + m_ct.tx_out_rsigs.emplace_back(bproof); + if (!rct::bulletproof_VERIFY(boost::get(m_ct.tx_out_rsigs.back()))) { + throw exc::ProtocolException("Returned range signature is invalid"); + } + } + + std::shared_ptr Signer::step_rsig(size_t idx){ + if (client_version() == 0 || !is_offloading() || !should_compute_bp_now()){ + return nullptr; + } + + auto res = std::make_shared(); + auto & cur_dst = m_ct.tx_data.splitted_dsts[idx]; + translate_dst_entry(res->mutable_dst_entr(), &cur_dst); + res->set_dst_entr_hmac(m_ct.tx_out_entr_hmacs[idx]); + + compute_bproof(*(res->mutable_rsig_data())); + res->set_is_offloaded_bp(true); + return res; + } + + void Signer::step_set_rsig_ack(std::shared_ptr ack){ + m_ct.cur_batch_idx += 1; + m_ct.cur_output_in_batch_idx = 0; + } + + std::shared_ptr Signer::step_all_outs_set(){ + return std::make_shared(); + } + + void Signer::step_all_outs_set_ack(std::shared_ptr ack, hw::device &hwdev){ + m_ct.rv = std::make_shared(); + m_ct.rv->txnFee = ack->rv().txn_fee(); + m_ct.rv->type = static_cast(ack->rv().rv_type()); + string_to_key(m_ct.rv->message, ack->rv().message()); + + // Extra copy + m_ct.tx.extra.clear(); + auto extra = ack->extra(); + auto extra_data = extra.data(); + m_ct.tx.extra.reserve(extra.size()); + for(size_t i = 0; i < extra.size(); ++i){ + m_ct.tx.extra.push_back(static_cast(extra_data[i])); + } + + ::crypto::hash tx_prefix_hash{}; + cryptonote::get_transaction_prefix_hash(m_ct.tx, tx_prefix_hash); + m_ct.tx_prefix_hash = key_to_string(tx_prefix_hash); + if (crypto_verify_32(reinterpret_cast(tx_prefix_hash.data), + reinterpret_cast(ack->tx_prefix_hash().data()))){ + throw exc::proto::SecurityException("Transaction prefix has does not match to the computed value"); + } + + // RctSig + auto num_sources = m_ct.tx_data.sources.size(); + if (is_simple() || is_req_bulletproof()){ + auto dst = &m_ct.rv->pseudoOuts; + if (is_bulletproof()){ + dst = &m_ct.rv->p.pseudoOuts; + } + + dst->clear(); + for (const auto &pseudo_out : m_ct.pseudo_outs) { + dst->emplace_back(); + string_to_key(dst->back(), pseudo_out); + } + + m_ct.rv->mixRing.resize(num_sources); + } else { + m_ct.rv->mixRing.resize(m_ct.tsx_data.mixin()); + m_ct.rv->mixRing[0].resize(num_sources); + } + + CHECK_AND_ASSERT_THROW_MES(m_ct.tx_out_pk.size() == m_ct.tx_out_ecdh.size(), "Invalid vector sizes"); + for(size_t i = 0; i < m_ct.tx_out_ecdh.size(); ++i){ + m_ct.rv->outPk.push_back(m_ct.tx_out_pk[i]); + m_ct.rv->ecdhInfo.push_back(m_ct.tx_out_ecdh[i]); + } + + for(size_t i = 0; i < m_ct.tx_out_rsigs.size(); ++i){ + if (is_bulletproof()){ + m_ct.rv->p.bulletproofs.push_back(boost::get(m_ct.tx_out_rsigs[i])); + } else { + m_ct.rv->p.rangeSigs.push_back(boost::get(m_ct.tx_out_rsigs[i])); + } + } + + rct::key hash_computed = rct::get_pre_mlsag_hash(*(m_ct.rv), hwdev); + auto & hash = ack->full_message_hash(); + + if (hash.size() != 32){ + throw exc::ProtocolException("Returned mlsag hash has invalid size"); + } + + if (crypto_verify_32(reinterpret_cast(hash_computed.bytes), + reinterpret_cast(hash.data()))){ + throw exc::proto::SecurityException("Computed MLSAG does not match"); + } + } + + std::shared_ptr Signer::step_sign_input(size_t idx){ + m_ct.cur_input_idx = idx; + + CHECK_AND_ASSERT_THROW_MES(idx < m_ct.tx_data.sources.size(), "Invalid transaction index"); + CHECK_AND_ASSERT_THROW_MES(idx < m_ct.tx.vin.size(), "Invalid transaction index"); + CHECK_AND_ASSERT_THROW_MES(idx < m_ct.tx_in_hmacs.size(), "Invalid transaction index"); + CHECK_AND_ASSERT_THROW_MES(idx < m_ct.alphas.size(), "Invalid transaction index"); + CHECK_AND_ASSERT_THROW_MES(idx < m_ct.spend_encs.size(), "Invalid transaction index"); + + auto res = std::make_shared(); + translate_src_entry(res->mutable_src_entr(), &(m_ct.tx_data.sources[idx])); + res->set_vini(cryptonote::t_serializable_object_to_blob(m_ct.tx.vin[idx])); + res->set_vini_hmac(m_ct.tx_in_hmacs[idx]); + res->set_pseudo_out_alpha(m_ct.alphas[idx]); + res->set_spend_key(m_ct.spend_encs[idx]); + + CHECK_AND_ASSERT_THROW_MES(idx < m_ct.pseudo_outs.size(), "Invalid transaction index"); + CHECK_AND_ASSERT_THROW_MES(idx < m_ct.pseudo_outs_hmac.size(), "Invalid transaction index"); + res->set_pseudo_out(m_ct.pseudo_outs[idx]); + res->set_pseudo_out_hmac(m_ct.pseudo_outs_hmac[idx]); + return res; + } + + void Signer::step_sign_input_ack(std::shared_ptr ack){ + rct::mgSig mg; + if (!cn_deserialize(ack->signature(), mg)){ + throw exc::ProtocolException("Cannot deserialize mg[i]"); + } + + // Sync updated pseudo_outputs, client_version>=1, HF10+ + if (client_version() >= 1 && ack->has_pseudo_out()){ + CHECK_AND_ASSERT_THROW_MES(m_ct.cur_input_idx < m_ct.pseudo_outs.size(), "Invalid pseudo-out index"); + m_ct.pseudo_outs[m_ct.cur_input_idx] = ack->pseudo_out(); + if (is_bulletproof()){ + CHECK_AND_ASSERT_THROW_MES(m_ct.cur_input_idx < m_ct.rv->p.pseudoOuts.size(), "Invalid pseudo-out index"); + string_to_key(m_ct.rv->p.pseudoOuts[m_ct.cur_input_idx], ack->pseudo_out()); + } else { + CHECK_AND_ASSERT_THROW_MES(m_ct.cur_input_idx < m_ct.rv->pseudoOuts.size(), "Invalid pseudo-out index"); + string_to_key(m_ct.rv->pseudoOuts[m_ct.cur_input_idx], ack->pseudo_out()); + } + } + + m_ct.rv->p.MGs.push_back(mg); + } + + std::shared_ptr Signer::step_final(){ + m_ct.tx.rct_signatures = *(m_ct.rv); + return std::make_shared(); + } + + void Signer::step_final_ack(std::shared_ptr ack){ + if (m_multisig){ + auto & cout_key = ack->cout_key(); + for(auto & cur : m_ct.couts){ + if (cur.size() != crypto::chacha::IV_SIZE + 32){ + throw std::invalid_argument("Encrypted cout has invalid length"); + } + + char buff[32]; + auto data = cur.data(); + + crypto::chacha::decrypt(data + crypto::chacha::IV_SIZE, 32, reinterpret_cast(cout_key.data()), reinterpret_cast(data), buff); + m_ct.couts_dec.emplace_back(buff, 32); + } + } + + m_ct.enc_salt1 = ack->salt(); + m_ct.enc_salt2 = ack->rand_mult(); + m_ct.enc_keys = ack->tx_enc_keys(); + } + + std::string Signer::store_tx_aux_info(){ + rapidjson::StringBuffer sb; + rapidjson::Writer writer(sb); + + rapidjson::Document json; + json.SetObject(); + + rapidjson::Value valueS(rapidjson::kStringType); + rapidjson::Value valueI(rapidjson::kNumberType); + + valueI.SetInt(1); + json.AddMember("version", valueI, json.GetAllocator()); + + valueS.SetString(m_ct.enc_salt1.c_str(), m_ct.enc_salt1.size()); + json.AddMember("salt1", valueS, json.GetAllocator()); + + valueS.SetString(m_ct.enc_salt2.c_str(), m_ct.enc_salt2.size()); + json.AddMember("salt2", valueS, json.GetAllocator()); + + valueS.SetString(m_ct.tx_prefix_hash.c_str(), m_ct.tx_prefix_hash.size()); + json.AddMember("tx_prefix_hash", valueS, json.GetAllocator()); + + valueS.SetString(m_ct.enc_keys.c_str(), m_ct.enc_keys.size()); + json.AddMember("enc_keys", valueS, json.GetAllocator()); + + json.Accept(writer); + return sb.GetString(); + } + + void load_tx_key_data(hw::device_cold::tx_key_data_t & res, const std::string & data) + { + rapidjson::Document json; + + // The contents should be JSON if the wallet follows the new format. + if (json.Parse(data.c_str()).HasParseError()) + { + throw std::invalid_argument("Data parsing error"); + } + else if(!json.IsObject()) + { + throw std::invalid_argument("Data parsing error - not an object"); + } + + GET_FIELD_FROM_JSON(json, version, int, Int, true, -1); + GET_STRING_FROM_JSON(json, salt1, std::string, true, std::string()); + GET_STRING_FROM_JSON(json, salt2, std::string, true, std::string()); + GET_STRING_FROM_JSON(json, enc_keys, std::string, true, std::string()); + GET_STRING_FROM_JSON(json, tx_prefix_hash, std::string, false, std::string()); + + if (field_version != 1) + { + throw std::invalid_argument("Unknown version"); + } + + res.salt1 = field_salt1; + res.salt2 = field_salt2; + res.tx_enc_keys = field_enc_keys; + res.tx_prefix_hash = field_tx_prefix_hash; + } + + std::shared_ptr get_tx_key( + const hw::device_cold::tx_key_data_t & tx_data) + { + auto req = std::make_shared(); + req->set_salt1(tx_data.salt1); + req->set_salt2(tx_data.salt2); + req->set_tx_enc_keys(tx_data.tx_enc_keys); + req->set_tx_prefix_hash(tx_data.tx_prefix_hash); + req->set_reason(0); + + return req; + } + + void get_tx_key_ack( + std::vector<::crypto::secret_key> & tx_keys, + const std::string & tx_prefix_hash, + const ::crypto::secret_key & view_key_priv, + std::shared_ptr ack + ) + { + auto enc_key = protocol::tx::compute_enc_key(view_key_priv, tx_prefix_hash, ack->salt()); + auto & encrypted_keys = ack->has_tx_derivations() ? ack->tx_derivations() : ack->tx_keys(); + + const size_t len_ciphertext = encrypted_keys.size(); // IV || keys || TAG + CHECK_AND_ASSERT_THROW_MES(len_ciphertext > crypto::chacha::IV_SIZE + crypto::chacha::TAG_SIZE, "Invalid size"); + + size_t keys_len = len_ciphertext - crypto::chacha::IV_SIZE - crypto::chacha::TAG_SIZE; + std::unique_ptr plaintext(new uint8_t[keys_len]); + + protocol::crypto::chacha::decrypt( + encrypted_keys.data() + crypto::chacha::IV_SIZE, + len_ciphertext - crypto::chacha::IV_SIZE, + reinterpret_cast(enc_key.data), + reinterpret_cast(encrypted_keys.data()), + reinterpret_cast(plaintext.get()), &keys_len); + + CHECK_AND_ASSERT_THROW_MES(keys_len % 32 == 0, "Invalid size"); + tx_keys.resize(keys_len / 32); + + for(unsigned i = 0; i < keys_len / 32; ++i) + { + memcpy(tx_keys[i].data, plaintext.get() + 32 * i, 32); + } + memwipe(plaintext.get(), keys_len); + } + +} +} +} +} diff --git a/src/device_trezor/trezor/protocol.hpp b/src/device_trezor/trezor/protocol.hpp new file mode 100644 index 00000000000..f58bf1039fe --- /dev/null +++ b/src/device_trezor/trezor/protocol.hpp @@ -0,0 +1,331 @@ +// Copyright (c) 2017-2019, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +#ifndef MONERO_PROTOCOL_H +#define MONERO_PROTOCOL_H + +#include "trezor_defs.hpp" +#include "device/device_cold.hpp" +#include "messages_map.hpp" +#include "transport.hpp" +#include "wallet/wallet2.h" + +namespace hw{ +namespace trezor{ +namespace protocol{ + + std::string key_to_string(const ::crypto::ec_point & key); + std::string key_to_string(const ::crypto::ec_scalar & key); + std::string key_to_string(const ::crypto::hash & key); + std::string key_to_string(const ::rct::key & key); + + void string_to_key(::crypto::ec_scalar & key, const std::string & str); + void string_to_key(::crypto::ec_point & key, const std::string & str); + void string_to_key(::rct::key & key, const std::string & str); + + template + void assign_to_repeatable(::google::protobuf::RepeatedField * dst, const InputIterator begin, const InputIterator end){ + for (InputIterator it = begin; it != end; it++) { + auto s = dst->Add(); + *s = *it; + } + } + + template + void assign_from_repeatable(std::vector * dst, const InputIterator begin, const InputIterator end){ + for (InputIterator it = begin; it != end; it++) { + dst->push_back(*it); + } + }; + + template + bool cn_deserialize(const void * buff, size_t len, T & dst){ + std::stringstream ss; + ss.write(static_cast(buff), len); //ss << tx_blob; + binary_archive ba(ss); + bool r = ::serialization::serialize(ba, dst); + return r; + } + + template + bool cn_deserialize(const std::string & str, T & dst){ + return cn_deserialize(str.data(), str.size(), dst); + } + + template + std::string cn_serialize(T & obj){ + std::ostringstream oss; + binary_archive oar(oss); + bool success = ::serialization::serialize(oar, obj); + if (!success){ + throw exc::EncodingException("Could not CN serialize given object"); + } + return oss.str(); + } + +// Crypto / encryption +namespace crypto { +namespace chacha { + // Constants as defined in RFC 7539. + const unsigned IV_SIZE = 12; + const unsigned TAG_SIZE = 16; // crypto_aead_chacha20poly1305_IETF_ABYTES; + + /** + * Chacha20Poly1305 decryption with tag verification. RFC 7539. + */ + void decrypt(const void* ciphertext, size_t length, const uint8_t* key, const uint8_t* iv, char* plaintext, size_t *plaintext_len=nullptr); + +} +} + + +// Cold Key image sync +namespace ki { + + using MoneroTransferDetails = messages::monero::MoneroKeyImageSyncStepRequest_MoneroTransferDetails; + using MoneroSubAddressIndicesList = messages::monero::MoneroKeyImageExportInitRequest_MoneroSubAddressIndicesList; + using MoneroExportedKeyImage = messages::monero::MoneroKeyImageSyncStepAck_MoneroExportedKeyImage; + using exported_key_image = hw::device_cold::exported_key_image; + + /** + * Converts transfer details to the MoneroTransferDetails required for KI sync + */ + bool key_image_data(wallet_shim * wallet, + const std::vector & transfers, + std::vector & res); + + /** + * Computes a hash over MoneroTransferDetails. Commitment used in the KI sync. + */ + std::string compute_hash(const MoneroTransferDetails & rr); + + /** + * Generates KI sync request with commitments computed. + */ + void generate_commitment(std::vector & mtds, + const std::vector & transfers, + std::shared_ptr & req); + + /** + * Processes Live refresh step response, parses KI, checks the signature + */ + void live_refresh_ack(const ::crypto::secret_key & view_key_priv, + const ::crypto::public_key& out_key, + const std::shared_ptr & ack, + ::cryptonote::keypair& in_ephemeral, + ::crypto::key_image& ki); +} + +// Cold transaction signing +namespace tx { + using TsxData = messages::monero::MoneroTransactionInitRequest_MoneroTransactionData; + using MoneroTransactionDestinationEntry = messages::monero::MoneroTransactionDestinationEntry; + using MoneroAccountPublicAddress = messages::monero::MoneroTransactionDestinationEntry_MoneroAccountPublicAddress; + using MoneroTransactionSourceEntry = messages::monero::MoneroTransactionSourceEntry; + using MoneroMultisigKLRki = messages::monero::MoneroTransactionSourceEntry_MoneroMultisigKLRki; + using MoneroOutputEntry = messages::monero::MoneroTransactionSourceEntry_MoneroOutputEntry; + using MoneroRctKey = messages::monero::MoneroTransactionSourceEntry_MoneroOutputEntry_MoneroRctKeyPublic; + using MoneroRsigData = messages::monero::MoneroTransactionRsigData; + + using tx_construction_data = tools::wallet2::tx_construction_data; + using unsigned_tx_set = tools::wallet2::unsigned_tx_set; + + void translate_address(MoneroAccountPublicAddress * dst, const cryptonote::account_public_address * src); + void translate_dst_entry(MoneroTransactionDestinationEntry * dst, const cryptonote::tx_destination_entry * src); + void translate_src_entry(MoneroTransactionSourceEntry * dst, const cryptonote::tx_source_entry * src); + void translate_klrki(MoneroMultisigKLRki * dst, const rct::multisig_kLRki * src); + void translate_rct_key(MoneroRctKey * dst, const rct::ctkey * src); + std::string hash_addr(const MoneroAccountPublicAddress * addr, boost::optional amount = boost::none, boost::optional is_subaddr = boost::none); + std::string hash_addr(const std::string & spend_key, const std::string & view_key, boost::optional amount = boost::none, boost::optional is_subaddr = boost::none); + std::string hash_addr(const ::crypto::public_key * spend_key, const ::crypto::public_key * view_key, boost::optional amount = boost::none, boost::optional is_subaddr = boost::none); + ::crypto::secret_key compute_enc_key(const ::crypto::secret_key & private_view_key, const std::string & aux, const std::string & salt); + + typedef boost::variant rsig_v; + + /** + * Transaction signer state holder. + */ + class TData { + public: + TsxData tsx_data; + tx_construction_data tx_data; + cryptonote::transaction tx; + unsigned rsig_type; + int bp_version; + std::vector grouping_vct; + std::shared_ptr rsig_param; + size_t cur_input_idx; + size_t cur_output_idx; + size_t cur_batch_idx; + size_t cur_output_in_batch_idx; + + std::vector tx_in_hmacs; + std::vector tx_out_entr_hmacs; + std::vector tx_out_hmacs; + std::vector tx_out_rsigs; + std::vector tx_out_pk; + std::vector tx_out_ecdh; + std::vector source_permutation; + std::vector alphas; + std::vector spend_encs; + std::vector pseudo_outs; + std::vector pseudo_outs_hmac; + std::vector couts; + std::vector couts_dec; + std::vector rsig_gamma; + std::string tx_prefix_hash; + std::string enc_salt1; + std::string enc_salt2; + std::string enc_keys; + + std::shared_ptr rv; + + TData(); + }; + + class Signer { + private: + TData m_ct; + wallet_shim * m_wallet2; + + size_t m_tx_idx; + const unsigned_tx_set * m_unsigned_tx; + hw::tx_aux_data * m_aux_data; + + unsigned m_client_version; + bool m_multisig; + + const tx_construction_data & cur_tx(){ + CHECK_AND_ASSERT_THROW_MES(m_tx_idx < m_unsigned_tx->txes.size(), "Invalid transaction index"); + return m_unsigned_tx->txes[m_tx_idx]; + } + + void extract_payment_id(); + void compute_integrated_indices(TsxData * tsx_data); + bool should_compute_bp_now() const; + void compute_bproof(messages::monero::MoneroTransactionRsigData & rsig_data); + void process_bproof(rct::Bulletproof & bproof); + + public: + Signer(wallet_shim * wallet2, const unsigned_tx_set * unsigned_tx, size_t tx_idx = 0, hw::tx_aux_data * aux_data = nullptr); + + std::shared_ptr step_init(); + void step_init_ack(std::shared_ptr ack); + + std::shared_ptr step_set_input(size_t idx); + void step_set_input_ack(std::shared_ptr ack); + + void sort_ki(); + std::shared_ptr step_permutation(); + void step_permutation_ack(std::shared_ptr ack); + + std::shared_ptr step_set_vini_input(size_t idx); + void step_set_vini_input_ack(std::shared_ptr ack); + + std::shared_ptr step_all_inputs_set(); + void step_all_inputs_set_ack(std::shared_ptr ack); + + std::shared_ptr step_set_output(size_t idx); + void step_set_output_ack(std::shared_ptr ack); + + std::shared_ptr step_rsig(size_t idx); + void step_set_rsig_ack(std::shared_ptr ack); + + std::shared_ptr step_all_outs_set(); + void step_all_outs_set_ack(std::shared_ptr ack, hw::device &hwdev); + + std::shared_ptr step_sign_input(size_t idx); + void step_sign_input_ack(std::shared_ptr ack); + + std::shared_ptr step_final(); + void step_final_ack(std::shared_ptr ack); + + std::string store_tx_aux_info(); + + unsigned client_version() const { + return m_client_version; + } + + bool is_simple() const { + if (!m_ct.rv){ + throw std::invalid_argument("RV not initialized"); + } + auto tp = m_ct.rv->type; + return tp == rct::RCTTypeSimple; + } + + bool is_req_bulletproof() const { + return m_ct.tx_data.rct_config.range_proof_type != rct::RangeProofBorromean; + } + + bool is_bulletproof() const { + if (!m_ct.rv){ + throw std::invalid_argument("RV not initialized"); + } + auto tp = m_ct.rv->type; + return tp == rct::RCTTypeBulletproof || tp == rct::RCTTypeBulletproof2; + } + + bool is_offloading() const { + return m_ct.rsig_param && m_ct.rsig_param->offload_type() != 0; + } + + size_t num_outputs() const { + return m_ct.tx_data.splitted_dsts.size(); + } + + size_t num_inputs() const { + return m_ct.tx_data.sources.size(); + } + + const TData & tdata() const { + return m_ct; + } + }; + + // TX Key decryption + void load_tx_key_data(hw::device_cold::tx_key_data_t & res, const std::string & data); + + std::shared_ptr get_tx_key( + const hw::device_cold::tx_key_data_t & tx_data); + + void get_tx_key_ack( + std::vector<::crypto::secret_key> & tx_keys, + const std::string & tx_prefix_hash, + const ::crypto::secret_key & view_key_priv, + std::shared_ptr ack + ); +} + +} +} +} + + +#endif //MONERO_PROTOCOL_H diff --git a/src/device_trezor/trezor/tools/README.md b/src/device_trezor/trezor/tools/README.md new file mode 100644 index 00000000000..0802e734a09 --- /dev/null +++ b/src/device_trezor/trezor/tools/README.md @@ -0,0 +1,45 @@ +# Trezor + +## Messages rebuild + +Install `protoc` for your distribution. Requirements: + +- `protobuf-compiler` +- `libprotobuf-dev` +- `python` + + +Soft requirement: Python 3, can be easily installed with [pyenv]. +If Python 3 is used there are no additional python dependencies. + +Since Cmake 3.12 the `FindPython` module is used to locate the Python +interpreter in your system. It preferably searches for Python 3, if none +is found, it searches for Python 2. + +Lower version of the cmake uses another module which does not guarantee +ordering. If you want to override the selected python you can do it in +the following way: + +```bash +export TREZOR_PYTHON=`which python3` +``` + + +### Python 2.7+ + +Python 3 has `tempfile.TemporaryDirectory` available but Python 2 lacks +this class so the message generation code uses `backports.tempfile` package +bundled in the repository. + +The minimal Python versions are 2.7 and 3.4 + +### Regenerate messages + +```bash +cd src/device_trezor/trezor +python tools/build_protob.py +``` + +The messages regeneration is done also automatically via cmake. + +[pyenv]: https://github.com/pyenv/pyenv diff --git a/src/device_trezor/trezor/tools/build_protob.py b/src/device_trezor/trezor/tools/build_protob.py new file mode 100644 index 00000000000..eb32f6b4dd4 --- /dev/null +++ b/src/device_trezor/trezor/tools/build_protob.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python +import os +import subprocess +import sys +import argparse + + +parser = argparse.ArgumentParser() +parser.add_argument("-d", "--debug-msg", default=False, action="store_const", const=True, help="Build debug messages") +args = parser.parse_args() + +CWD = os.path.dirname(os.path.realpath(__file__)) +ROOT_DIR = os.path.abspath(os.path.join(CWD, "..", "..", "..", "..")) +TREZOR_COMMON = os.path.join(ROOT_DIR, "external", "trezor-common") +TREZOR_MESSAGES = os.path.join(CWD, "..", "messages") + +# check for existence of the submodule directory +common_defs = os.path.join(TREZOR_COMMON, "defs") +if not os.path.exists(common_defs): + raise ValueError( + "trezor-common submodule seems to be missing.\n" + + 'Use "git submodule update --init --recursive" to retrieve it.' + ) + +# regenerate messages +try: + selected = [ + "messages.proto", + "messages-common.proto", + "messages-management.proto", + "messages-monero.proto", + ] + + if args.debug_msg: + selected += ["messages-debug.proto"] + + proto_srcs = [os.path.join(TREZOR_COMMON, "protob", x) for x in selected] + exec_args = [ + sys.executable, + os.path.join(CWD, "pb2cpp.py"), + "-o", + TREZOR_MESSAGES, + ] + proto_srcs + + subprocess.check_call(exec_args) + +except Exception as e: + raise diff --git a/src/device_trezor/trezor/tools/pb2cpp.py b/src/device_trezor/trezor/tools/pb2cpp.py new file mode 100644 index 00000000000..3e0318ea552 --- /dev/null +++ b/src/device_trezor/trezor/tools/pb2cpp.py @@ -0,0 +1,219 @@ +#!/usr/bin/env python +# Converts Google's protobuf python definitions of TREZOR wire messages +# to plain-python objects as used in TREZOR Core and python-trezor + +import argparse +import logging +import os +import re +import shutil +import subprocess +import glob +import hashlib + +try: + from tempfile import TemporaryDirectory +except: + # Py2 backward compatibility, using bundled sources. + # Original source: pip install backports.tempfile + try: + # Try bundled python version + import sys + sys.path.append(os.path.dirname(__file__)) + from py2backports.tempfile import TemporaryDirectory + + except: + raise EnvironmentError('Python 2.7+ or 3.4+ is required. ' + 'TemporaryDirectory is not available in Python 2.' + 'Try to specify python to use, e.g.: "export TREZOR_PYTHON=`which python3`"') + + +AUTO_HEADER = "# Automatically generated by pb2cpp\n" + +# Fixing GCC7 compilation error +UNDEF_STATEMENT = """ +#ifdef minor +#undef minor +#endif +""" + +PROTOC = None +PROTOC_INCLUDE = None + + +def which(pgm): + path = os.getenv('PATH') + for p in path.split(os.path.pathsep): + p = os.path.join(p, pgm) + if os.path.exists(p) and os.access(p, os.X_OK): + return p + + +def namespace_file(fpath, package): + """Adds / replaces package name. Simple regex parsing, may use https://github.com/ph4r05/plyprotobuf later""" + with open(fpath) as fh: + fdata = fh.read() + + re_syntax = re.compile(r"^syntax\s*=") + re_package = re.compile(r"^package\s+([^;]+?)\s*;\s*$") + lines = fdata.split("\n") + + line_syntax = None + line_package = None + for idx, line in enumerate(lines): + if line_syntax is None and re_syntax.match(line): + line_syntax = idx + if line_package is None and re_package.match(line): + line_package = idx + + if package is None: + if line_package is None: + return + else: + lines.pop(line_package) + + else: + new_package = "package %s;" % package + if line_package is None: + lines.insert(line_syntax + 1 if line_syntax is not None else 0, new_package) + else: + lines[line_package] = new_package + + new_fdat = "\n".join(lines) + with open(fpath, "w+") as fh: + fh.write(new_fdat) + return new_fdat + + +def protoc(files, out_dir, additional_includes=(), package=None, force=False): + """Compile code with protoc and return the data.""" + + include_dirs = set() + include_dirs.add(PROTOC_INCLUDE) + if additional_includes: + include_dirs.update(additional_includes) + + with TemporaryDirectory() as tmpdir_protob, TemporaryDirectory() as tmpdir_out: + include_dirs.add(tmpdir_protob) + + new_files = [] + for file in files: + bname = os.path.basename(file) + tmp_file = os.path.join(tmpdir_protob, bname) + + shutil.copy(file, tmp_file) + if package is not None: + namespace_file(tmp_file, package) + new_files.append(tmp_file) + + protoc_includes = ["-I" + dir for dir in include_dirs if dir] + + exec_args = ( + [ + PROTOC, + "--cpp_out", + tmpdir_out, + ] + + protoc_includes + + new_files + ) + + subprocess.check_call(exec_args) + + # Fixing gcc compilation and clashes with "minor" field name + add_undef(tmpdir_out) + + # Scan output dir, check file differences + update_message_files(tmpdir_out, out_dir, force) + + +def update_message_files(tmpdir_out, out_dir, force=False): + files = glob.glob(os.path.join(tmpdir_out, '*.pb.*')) + for fname in files: + bname = os.path.basename(fname) + dest_file = os.path.join(out_dir, bname) + if not force and os.path.exists(dest_file): + data = open(fname, 'rb').read() + data_hash = hashlib.sha256(data).digest() + data_dest = open(dest_file, 'rb').read() + data_dest_hash = hashlib.sha256(data_dest).digest() + if data_hash == data_dest_hash: + continue + + shutil.copy(fname, dest_file) + + +def add_undef(out_dir): + files = glob.glob(os.path.join(out_dir, '*.pb.*')) + for fname in files: + with open(fname) as fh: + lines = fh.readlines() + + idx_insertion = None + for idx in range(len(lines)): + if '@@protoc_insertion_point(includes)' in lines[idx]: + idx_insertion = idx + break + + if idx_insertion is None: + pass + + lines.insert(idx_insertion + 1, UNDEF_STATEMENT) + with open(fname, 'w') as fh: + fh.write("".join(lines)) + + +def strip_leader(s, prefix): + """Remove given prefix from underscored name.""" + leader = prefix + "_" + if s.startswith(leader): + return s[len(leader) :] + else: + return s + + +def main(): + global PROTOC, PROTOC_INCLUDE + logging.basicConfig(level=logging.DEBUG) + + parser = argparse.ArgumentParser() + # fmt: off + parser.add_argument("proto", nargs="+", help="Protobuf definition files") + parser.add_argument("-o", "--out-dir", help="Directory for generated source code") + parser.add_argument("-n", "--namespace", default=None, help="Message namespace") + parser.add_argument("-I", "--protoc-include", action="append", help="protoc include path") + parser.add_argument("-P", "--protobuf-module", default="protobuf", help="Name of protobuf module") + parser.add_argument("-f", "--force", default=False, help="Overwrite existing files") + # fmt: on + args = parser.parse_args() + + protoc_includes = args.protoc_include or (os.environ.get("PROTOC_INCLUDE"),) + + PROTOBUF_INCLUDE_DIRS = os.getenv("PROTOBUF_INCLUDE_DIRS", None) + PROTOBUF_PROTOC_EXECUTABLE = os.getenv("PROTOBUF_PROTOC_EXECUTABLE", None) + + if PROTOBUF_PROTOC_EXECUTABLE and not os.path.exists(PROTOBUF_PROTOC_EXECUTABLE): + raise ValueError("PROTOBUF_PROTOC_EXECUTABLE set but not found: %s" % PROTOBUF_PROTOC_EXECUTABLE) + + elif PROTOBUF_PROTOC_EXECUTABLE: + PROTOC = PROTOBUF_PROTOC_EXECUTABLE + + else: + if os.name == "nt": + PROTOC = which("protoc.exe") + else: + PROTOC = which("protoc") + + if not PROTOC: + raise ValueError("protoc command not found. Set PROTOBUF_PROTOC_EXECUTABLE env var to the protoc binary and optionally PROTOBUF_INCLUDE_DIRS") + + PROTOC_PREFIX = os.path.dirname(os.path.dirname(PROTOC)) + PROTOC_INCLUDE = PROTOBUF_INCLUDE_DIRS if PROTOBUF_INCLUDE_DIRS else os.path.join(PROTOC_PREFIX, "include") + + protoc( + args.proto, args.out_dir, protoc_includes, package=args.namespace, force=args.force + ) + + +if __name__ == "__main__": + main() diff --git a/src/blocks/blocks.dat b/src/device_trezor/trezor/tools/py2backports/__init__.py similarity index 100% rename from src/blocks/blocks.dat rename to src/device_trezor/trezor/tools/py2backports/__init__.py diff --git a/src/device_trezor/trezor/tools/py2backports/tempfile.py b/src/device_trezor/trezor/tools/py2backports/tempfile.py new file mode 100644 index 00000000000..e259dea3f8a --- /dev/null +++ b/src/device_trezor/trezor/tools/py2backports/tempfile.py @@ -0,0 +1,72 @@ +""" +https://github.com/pjdelport/backports.tempfile/blob/master/src/backports/tempfile.py +Partial backport of Python 3.5's tempfile module: + TemporaryDirectory +Backport modifications are marked with marked with "XXX backport". +""" +from __future__ import absolute_import + +import sys +import warnings as _warnings +from shutil import rmtree as _rmtree + +from py2backports.weakref import finalize + + +# XXX backport: Rather than backporting all of mkdtemp(), we just create a +# thin wrapper implementing its Python 3.5 signature. +if sys.version_info < (3, 5): + from tempfile import mkdtemp as old_mkdtemp + + def mkdtemp(suffix=None, prefix=None, dir=None): + """ + Wrap `tempfile.mkdtemp()` to make the suffix and prefix optional (like Python 3.5). + """ + kwargs = {k: v for (k, v) in + dict(suffix=suffix, prefix=prefix, dir=dir).items() + if v is not None} + return old_mkdtemp(**kwargs) + +else: + from tempfile import mkdtemp + + +# XXX backport: ResourceWarning was added in Python 3.2. +# For earlier versions, fall back to RuntimeWarning instead. +_ResourceWarning = RuntimeWarning if sys.version_info < (3, 2) else ResourceWarning + + +class TemporaryDirectory(object): + """Create and return a temporary directory. This has the same + behavior as mkdtemp but can be used as a context manager. For + example: + with TemporaryDirectory() as tmpdir: + ... + Upon exiting the context, the directory and everything contained + in it are removed. + """ + + def __init__(self, suffix=None, prefix=None, dir=None): + self.name = mkdtemp(suffix, prefix, dir) + self._finalizer = finalize( + self, self._cleanup, self.name, + warn_message="Implicitly cleaning up {!r}".format(self)) + + @classmethod + def _cleanup(cls, name, warn_message): + _rmtree(name) + _warnings.warn(warn_message, _ResourceWarning) + + + def __repr__(self): + return "<{} {!r}>".format(self.__class__.__name__, self.name) + + def __enter__(self): + return self.name + + def __exit__(self, exc, value, tb): + self.cleanup() + + def cleanup(self): + if self._finalizer.detach(): + _rmtree(self.name) diff --git a/src/device_trezor/trezor/tools/py2backports/weakref.py b/src/device_trezor/trezor/tools/py2backports/weakref.py new file mode 100644 index 00000000000..eb646812b4c --- /dev/null +++ b/src/device_trezor/trezor/tools/py2backports/weakref.py @@ -0,0 +1,148 @@ +""" +https://github.com/pjdelport/backports.weakref/blob/master/src/backports/weakref.py +Partial backport of Python 3.6's weakref module: + finalize (new in Python 3.4) +Backport modifications are marked with "XXX backport". +""" +from __future__ import absolute_import + +import itertools +import sys +from weakref import ref + +__all__ = ['finalize'] + + +class finalize(object): + """Class for finalization of weakrefable objects + finalize(obj, func, *args, **kwargs) returns a callable finalizer + object which will be called when obj is garbage collected. The + first time the finalizer is called it evaluates func(*arg, **kwargs) + and returns the result. After this the finalizer is dead, and + calling it just returns None. + When the program exits any remaining finalizers for which the + atexit attribute is true will be run in reverse order of creation. + By default atexit is true. + """ + + # Finalizer objects don't have any state of their own. They are + # just used as keys to lookup _Info objects in the registry. This + # ensures that they cannot be part of a ref-cycle. + + __slots__ = () + _registry = {} + _shutdown = False + _index_iter = itertools.count() + _dirty = False + _registered_with_atexit = False + + class _Info(object): + __slots__ = ("weakref", "func", "args", "kwargs", "atexit", "index") + + def __init__(self, obj, func, *args, **kwargs): + if not self._registered_with_atexit: + # We may register the exit function more than once because + # of a thread race, but that is harmless + import atexit + atexit.register(self._exitfunc) + finalize._registered_with_atexit = True + info = self._Info() + info.weakref = ref(obj, self) + info.func = func + info.args = args + info.kwargs = kwargs or None + info.atexit = True + info.index = next(self._index_iter) + self._registry[self] = info + finalize._dirty = True + + def __call__(self, _=None): + """If alive then mark as dead and return func(*args, **kwargs); + otherwise return None""" + info = self._registry.pop(self, None) + if info and not self._shutdown: + return info.func(*info.args, **(info.kwargs or {})) + + def detach(self): + """If alive then mark as dead and return (obj, func, args, kwargs); + otherwise return None""" + info = self._registry.get(self) + obj = info and info.weakref() + if obj is not None and self._registry.pop(self, None): + return (obj, info.func, info.args, info.kwargs or {}) + + def peek(self): + """If alive then return (obj, func, args, kwargs); + otherwise return None""" + info = self._registry.get(self) + obj = info and info.weakref() + if obj is not None: + return (obj, info.func, info.args, info.kwargs or {}) + + @property + def alive(self): + """Whether finalizer is alive""" + return self in self._registry + + @property + def atexit(self): + """Whether finalizer should be called at exit""" + info = self._registry.get(self) + return bool(info) and info.atexit + + @atexit.setter + def atexit(self, value): + info = self._registry.get(self) + if info: + info.atexit = bool(value) + + def __repr__(self): + info = self._registry.get(self) + obj = info and info.weakref() + if obj is None: + return '<%s object at %#x; dead>' % (type(self).__name__, id(self)) + else: + return '<%s object at %#x; for %r at %#x>' % \ + (type(self).__name__, id(self), type(obj).__name__, id(obj)) + + @classmethod + def _select_for_exit(cls): + # Return live finalizers marked for exit, oldest first + L = [(f,i) for (f,i) in cls._registry.items() if i.atexit] + L.sort(key=lambda item:item[1].index) + return [f for (f,i) in L] + + @classmethod + def _exitfunc(cls): + # At shutdown invoke finalizers for which atexit is true. + # This is called once all other non-daemonic threads have been + # joined. + reenable_gc = False + try: + if cls._registry: + import gc + if gc.isenabled(): + reenable_gc = True + gc.disable() + pending = None + while True: + if pending is None or finalize._dirty: + pending = cls._select_for_exit() + finalize._dirty = False + if not pending: + break + f = pending.pop() + try: + # gc is disabled, so (assuming no daemonic + # threads) the following is the only line in + # this function which might trigger creation + # of a new finalizer + f() + except Exception: + sys.excepthook(*sys.exc_info()) + assert f not in cls._registry + finally: + # prevent any more finalizers from executing during shutdown + finalize._shutdown = True + if reenable_gc: + gc.enable() diff --git a/src/device_trezor/trezor/transport.cpp b/src/device_trezor/trezor/transport.cpp new file mode 100644 index 00000000000..59b281f138f --- /dev/null +++ b/src/device_trezor/trezor/transport.cpp @@ -0,0 +1,1264 @@ +// Copyright (c) 2017-2019, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +#ifdef WITH_DEVICE_TREZOR_WEBUSB +#include +#endif + +#include +#include +#include +#include +#include +#include +#include "common/apply_permutation.h" +#include "transport.hpp" +#include "messages/messages-common.pb.h" + +#undef MONERO_DEFAULT_LOG_CATEGORY +#define MONERO_DEFAULT_LOG_CATEGORY "device.trezor.transport" + +using namespace std; +using json = rapidjson::Document; + + +namespace hw{ +namespace trezor{ + + bool t_serialize(const std::string & in, std::string & out){ + out = in; + return true; + } + + bool t_serialize(const json_val & in, std::string & out){ + rapidjson::StringBuffer sb; + rapidjson::Writer writer(sb); + in.Accept(writer); + out = sb.GetString(); + return true; + } + + std::string t_serialize(const json_val & in){ + std::string ret; + t_serialize(in, ret); + return ret; + } + + bool t_deserialize(const std::string & in, std::string & out){ + out = in; + return true; + } + + bool t_deserialize(const std::string & in, json & out){ + if (out.Parse(in.c_str()).HasParseError()) { + throw exc::CommunicationException("JSON parse error"); + } + return true; + } + + static std::string json_get_string(const rapidjson::Value & in){ + return std::string(in.GetString()); + } + + uint64_t pack_version(uint32_t major, uint32_t minor, uint32_t patch) + { + // packing (major, minor, patch) to 64 B: 16 B | 24 B | 24 B + const unsigned bits_1 = 16; + const unsigned bits_2 = 24; + const uint32_t mask_1 = (1 << bits_1) - 1; + const uint32_t mask_2 = (1 << bits_2) - 1; + CHECK_AND_ASSERT_THROW_MES(major <= mask_1 && minor <= mask_2 && patch <= mask_2, "Version numbers overflow packing scheme"); + return patch | (((uint64_t)minor) << bits_2) | (((uint64_t)major) << (bits_1 + bits_2)); + } + + typedef struct { + uint16_t trezor_type; + uint16_t id_vendor; + uint16_t id_product; + } trezor_usb_desc_t; + + static trezor_usb_desc_t TREZOR_DESC_T1 = {1, 0x534C, 0x0001}; + static trezor_usb_desc_t TREZOR_DESC_T2 = {2, 0x1209, 0x53C1}; + static trezor_usb_desc_t TREZOR_DESC_T2_BL = {3, 0x1209, 0x53C0}; + + static trezor_usb_desc_t TREZOR_DESCS[] = { + TREZOR_DESC_T1, + TREZOR_DESC_T2, + TREZOR_DESC_T2_BL, + }; + + static size_t TREZOR_DESCS_LEN = sizeof(TREZOR_DESCS)/sizeof(TREZOR_DESCS[0]); + + static ssize_t get_device_idx(uint16_t id_vendor, uint16_t id_product){ + for(size_t i = 0; i < TREZOR_DESCS_LEN; ++i){ + if (TREZOR_DESCS[i].id_vendor == id_vendor && TREZOR_DESCS[i].id_product == id_product){ + return i; + } + } + + return -1; + } + + static bool is_device_supported(ssize_t device_idx){ + CHECK_AND_ASSERT_THROW_MES(device_idx < (ssize_t)TREZOR_DESCS_LEN, "Device desc idx too big"); + if (device_idx < 0){ + return false; + } + +#ifdef TREZOR_1_SUPPORTED + return true; +#else + return TREZOR_DESCS[device_idx].trezor_type != 1; +#endif + } + + // + // Helpers + // + +#define PROTO_HEADER_SIZE 6 + + static size_t message_size(const google::protobuf::Message &req){ + return static_cast(req.ByteSize()); + } + + static size_t serialize_message_buffer_size(size_t msg_size) { + return PROTO_HEADER_SIZE + msg_size; // tag 2B + len 4B + } + + static void serialize_message_header(void * buff, uint16_t tag, uint32_t len){ + uint16_t wire_tag = boost::endian::native_to_big(static_cast(tag)); + uint32_t wire_len = boost::endian::native_to_big(static_cast(len)); + memcpy(buff, (void *) &wire_tag, 2); + memcpy((uint8_t*)buff + 2, (void *) &wire_len, 4); + } + + static void deserialize_message_header(const void * buff, uint16_t & tag, uint32_t & len){ + uint16_t wire_tag; + uint32_t wire_len; + memcpy(&wire_tag, buff, 2); + memcpy(&wire_len, (uint8_t*)buff + 2, 4); + + tag = boost::endian::big_to_native(wire_tag); + len = boost::endian::big_to_native(wire_len); + } + + static void serialize_message(const google::protobuf::Message &req, size_t msg_size, uint8_t * buff, size_t buff_size) { + auto msg_wire_num = MessageMapper::get_message_wire_number(req); + const auto req_buffer_size = serialize_message_buffer_size(msg_size); + if (req_buffer_size > buff_size){ + throw std::invalid_argument("Buffer too small"); + } + + serialize_message_header(buff, msg_wire_num, msg_size); + if (!req.SerializeToArray(buff + 6, msg_size)){ + throw exc::EncodingException("Message serialization error"); + } + } + + // + // Communication protocol + // + +#define REPLEN 64 + + void ProtocolV1::write(Transport & transport, const google::protobuf::Message & req){ + const auto msg_size = message_size(req); + const auto buff_size = serialize_message_buffer_size(msg_size) + 2; + + std::unique_ptr req_buff(new uint8_t[buff_size]); + uint8_t * req_buff_raw = req_buff.get(); + req_buff_raw[0] = '#'; + req_buff_raw[1] = '#'; + + serialize_message(req, msg_size, req_buff_raw + 2, buff_size - 2); + + size_t offset = 0; + uint8_t chunk_buff[REPLEN]; + + // Chunk by chunk upload + while(offset < buff_size){ + auto to_copy = std::min((size_t)(buff_size - offset), (size_t)(REPLEN - 1)); + + chunk_buff[0] = '?'; + memcpy(chunk_buff + 1, req_buff_raw + offset, to_copy); + + // Pad with zeros + if (to_copy < REPLEN - 1){ + memset(chunk_buff + 1 + to_copy, 0, REPLEN - 1 - to_copy); + } + + transport.write_chunk(chunk_buff, REPLEN); + offset += REPLEN - 1; + } + } + + void ProtocolV1::read(Transport & transport, std::shared_ptr & msg, messages::MessageType * msg_type){ + char chunk[REPLEN]; + + // Initial chunk read + size_t nread = transport.read_chunk(chunk, REPLEN); + if (nread != REPLEN){ + throw exc::CommunicationException("Read chunk has invalid size"); + } + + if (strncmp(chunk, "?##", 3) != 0){ + throw exc::CommunicationException("Malformed chunk"); + } + + uint16_t tag; + uint32_t len; + nread -= 3 + 6; + deserialize_message_header(chunk + 3, tag, len); + + std::string data_acc(chunk + 3 + 6, nread); + data_acc.reserve(len); + + while(nread < len){ + const size_t cur = transport.read_chunk(chunk, REPLEN); + if (chunk[0] != '?'){ + throw exc::CommunicationException("Chunk malformed"); + } + + data_acc.append(chunk + 1, cur - 1); + nread += cur - 1; + } + + if (msg_type){ + *msg_type = static_cast(tag); + } + + if (nread < len){ + throw exc::CommunicationException("Response incomplete"); + } + + std::shared_ptr msg_wrap(MessageMapper::get_message(tag)); + if (!msg_wrap->ParseFromArray(data_acc.c_str(), len)){ + throw exc::CommunicationException("Message could not be parsed"); + } + + msg = msg_wrap; + } + + static void assert_port_number(uint32_t port) + { + CHECK_AND_ASSERT_THROW_MES(port >= 1024 && port < 65535, "Invalid port number: " << port); + } + + Transport::Transport(): m_open_counter(0) { + + } + + bool Transport::pre_open(){ + if (m_open_counter > 0){ + MTRACE("Already opened, count: " << m_open_counter); + m_open_counter += 1; + return false; + + } else if (m_open_counter < 0){ + MTRACE("Negative open value: " << m_open_counter); + + } + + // Caller should set m_open_counter to 1 after open + m_open_counter = 0; + return true; + } + + bool Transport::pre_close(){ + m_open_counter -= 1; + + if (m_open_counter < 0){ + MDEBUG("Already closed. Counter " << m_open_counter); + + } else if (m_open_counter == 0) { + return true; + + } + + return false; + } + + // + // Bridge transport + // + + const char * BridgeTransport::PATH_PREFIX = "bridge:"; + + BridgeTransport::BridgeTransport( + boost::optional device_path, + boost::optional bridge_host): + m_device_path(device_path), + m_bridge_host(bridge_host ? bridge_host.get() : DEFAULT_BRIDGE), + m_response(boost::none), + m_session(boost::none), + m_device_info(boost::none) + { + const char *env_bridge_port = nullptr; + if (!bridge_host && (env_bridge_port = getenv("TREZOR_BRIDGE_PORT")) != nullptr) + { + uint16_t bridge_port; + CHECK_AND_ASSERT_THROW_MES(epee::string_tools::get_xtype_from_string(bridge_port, env_bridge_port), "Invalid bridge port: " << env_bridge_port); + assert_port_number(bridge_port); + + m_bridge_host = std::string("127.0.0.1:") + boost::lexical_cast(env_bridge_port); + MDEBUG("Bridge host: " << m_bridge_host); + } + + m_http_client.set_server(m_bridge_host, boost::none, epee::net_utils::ssl_support_t::e_ssl_support_disabled); + } + + std::string BridgeTransport::get_path() const { + if (!m_device_path){ + return ""; + } + + std::string path(PATH_PREFIX); + return path + m_device_path.get(); + } + + void BridgeTransport::enumerate(t_transport_vect & res) { + json bridge_res; + std::string req; + + bool req_status = invoke_bridge_http("/enumerate", req, bridge_res, m_http_client); + if (!req_status){ + throw exc::CommunicationException("Bridge enumeration failed"); + } + + for(rapidjson::Value::ConstValueIterator itr = bridge_res.Begin(); itr != bridge_res.End(); ++itr){ + auto element = itr->GetObject(); + auto t = std::make_shared(boost::make_optional(json_get_string(element["path"]))); + + auto itr_vendor = element.FindMember("vendor"); + auto itr_product = element.FindMember("product"); + if (itr_vendor != element.MemberEnd() && itr_product != element.MemberEnd() + && itr_vendor->value.IsNumber() && itr_product->value.IsNumber()){ + try { + const auto id_vendor = (uint16_t) itr_vendor->value.GetUint64(); + const auto id_product = (uint16_t) itr_product->value.GetUint64(); + const auto device_idx = get_device_idx(id_vendor, id_product); + if (!is_device_supported(device_idx)){ + MDEBUG("Device with idx " << device_idx << " is not supported. Vendor: " << id_vendor << ", product: " << id_product); + continue; + } + } catch(const std::exception &e){ + MERROR("Could not detect vendor & product: " << e.what()); + } + } + + t->m_device_info.emplace(); + t->m_device_info->CopyFrom(*itr, t->m_device_info->GetAllocator()); + res.push_back(t); + } + } + + void BridgeTransport::open() { + if (!pre_open()){ + return; + } + + if (!m_device_path){ + throw exc::CommunicationException("Coud not open, empty device path"); + } + + std::string uri = "/acquire/" + m_device_path.get() + "/null"; + std::string req; + json bridge_res; + bool req_status = invoke_bridge_http(uri, req, bridge_res, m_http_client); + if (!req_status){ + throw exc::CommunicationException("Failed to acquire device"); + } + + m_session = boost::make_optional(json_get_string(bridge_res["session"])); + m_open_counter = 1; + } + + void BridgeTransport::close() { + if (!pre_close()){ + return; + } + + MTRACE("Closing Trezor:BridgeTransport"); + if (!m_device_path || !m_session){ + throw exc::CommunicationException("Device not open"); + } + + std::string uri = "/release/" + m_session.get(); + std::string req; + json bridge_res; + bool req_status = invoke_bridge_http(uri, req, bridge_res, m_http_client); + if (!req_status){ + throw exc::CommunicationException("Failed to release device"); + } + + m_session = boost::none; + } + + void BridgeTransport::write(const google::protobuf::Message &req) { + m_response = boost::none; + + const auto msg_size = message_size(req); + const auto buff_size = serialize_message_buffer_size(msg_size); + + std::unique_ptr req_buff(new uint8_t[buff_size]); + uint8_t * req_buff_raw = req_buff.get(); + + serialize_message(req, msg_size, req_buff_raw, buff_size); + + std::string uri = "/call/" + m_session.get(); + std::string req_hex = epee::to_hex::string(epee::span(req_buff_raw, buff_size)); + std::string res_hex; + + bool req_status = invoke_bridge_http(uri, req_hex, res_hex, m_http_client); + if (!req_status){ + throw exc::CommunicationException("Call method failed"); + } + + m_response = res_hex; + } + + void BridgeTransport::read(std::shared_ptr & msg, messages::MessageType * msg_type) { + if (!m_response){ + throw exc::CommunicationException("Could not read, no response stored"); + } + + std::string bin_data; + if (!epee::string_tools::parse_hexstr_to_binbuff(m_response.get(), bin_data)){ + throw exc::CommunicationException("Response is not well hexcoded"); + } + + uint16_t msg_tag; + uint32_t msg_len; + deserialize_message_header(bin_data.c_str(), msg_tag, msg_len); + if (bin_data.size() != msg_len + 6){ + throw exc::CommunicationException("Response is not well hexcoded"); + } + + if (msg_type){ + *msg_type = static_cast(msg_tag); + } + + std::shared_ptr msg_wrap(MessageMapper::get_message(msg_tag)); + if (!msg_wrap->ParseFromArray(bin_data.c_str() + 6, msg_len)){ + throw exc::EncodingException("Response is not well hexcoded"); + } + msg = msg_wrap; + } + + const boost::optional & BridgeTransport::device_info() const { + return m_device_info; + } + + std::ostream& BridgeTransport::dump(std::ostream& o) const { + return o << "BridgeTransport"; + } + + // + // UdpTransport + // + const char * UdpTransport::PATH_PREFIX = "udp:"; + const char * UdpTransport::DEFAULT_HOST = "127.0.0.1"; + const int UdpTransport::DEFAULT_PORT = 21324; + + static void parse_udp_path(std::string &host, int &port, std::string path) + { + if (boost::starts_with(path, UdpTransport::PATH_PREFIX)) + { + path = path.substr(strlen(UdpTransport::PATH_PREFIX)); + } + + auto delim = path.find(':'); + if (delim == std::string::npos) { + host = path; + } else { + host = path.substr(0, delim); + port = std::stoi(path.substr(delim + 1)); + } + } + + UdpTransport::UdpTransport(boost::optional device_path, + boost::optional> proto) : + m_io_service(), m_deadline(m_io_service) + { + m_device_host = DEFAULT_HOST; + m_device_port = DEFAULT_PORT; + const char *env_trezor_path = nullptr; + + if (device_path) { + parse_udp_path(m_device_host, m_device_port, device_path.get()); + } else if ((env_trezor_path = getenv("TREZOR_PATH")) != nullptr && boost::starts_with(env_trezor_path, UdpTransport::PATH_PREFIX)){ + parse_udp_path(m_device_host, m_device_port, std::string(env_trezor_path)); + MDEBUG("Applied TREZOR_PATH: " << m_device_host << ":" << m_device_port); + } else { + m_device_host = DEFAULT_HOST; + } + + assert_port_number((uint32_t)m_device_port); + if (m_device_host != "localhost" && m_device_host != DEFAULT_HOST){ + throw std::invalid_argument("Local endpoint allowed only"); + } + + m_proto = proto ? proto.get() : std::make_shared(); + } + + std::string UdpTransport::get_path() const { + std::string path(PATH_PREFIX); + return path + m_device_host + ":" + std::to_string(m_device_port); + } + + void UdpTransport::require_socket(){ + if (!m_socket){ + throw exc::NotConnectedException("Socket not connected"); + } + } + + bool UdpTransport::ping(){ + return ping_int(); + } + + bool UdpTransport::ping_int(boost::posix_time::time_duration timeout){ + require_socket(); + try { + std::string req = "PINGPING"; + char res[8]; + + m_socket->send_to(boost::asio::buffer(req.c_str(), req.size()), m_endpoint); + receive(res, 8, nullptr, false, timeout); + + return memcmp(res, "PONGPONG", 8) == 0; + + } catch(...){ + return false; + } + } + + void UdpTransport::enumerate(t_transport_vect & res) { + std::shared_ptr t = std::make_shared(); + bool t_works = false; + + try{ + t->open(); + t_works = t->ping(); + } catch(...) { + + } + t->close(); + if (t_works){ + res.push_back(t); + } + } + + void UdpTransport::open() { + if (!pre_open()){ + return; + } + + udp::resolver resolver(m_io_service); + udp::resolver::query query(udp::v4(), m_device_host, std::to_string(m_device_port)); + m_endpoint = *resolver.resolve(query); + + m_socket.reset(new udp::socket(m_io_service)); + m_socket->open(udp::v4()); + + m_deadline.expires_at(boost::posix_time::pos_infin); + check_deadline(); + + m_proto->session_begin(*this); + m_open_counter = 1; + } + + void UdpTransport::close() { + if (!pre_close()){ + return; + } + + MTRACE("Closing Trezor:UdpTransport"); + if (!m_socket) { + throw exc::CommunicationException("Socket is already closed"); + } + + m_proto->session_end(*this); + m_socket->close(); + m_socket = nullptr; + } + + std::shared_ptr UdpTransport::find_debug() { +#ifdef WITH_TREZOR_DEBUGGING + std::shared_ptr t = std::make_shared(); + t->m_proto = std::make_shared(); + t->m_device_host = m_device_host; + t->m_device_port = m_device_port + 1; + return t; +#else + MINFO("Debug link is disabled in production"); + return nullptr; +#endif + } + + void UdpTransport::write_chunk(const void * buff, size_t size){ + require_socket(); + + if (size != 64){ + throw exc::CommunicationException("Invalid chunk size"); + } + + auto written = m_socket->send_to(boost::asio::buffer(buff, size), m_endpoint); + if (size != written){ + throw exc::CommunicationException("Could not send the whole chunk"); + } + } + + size_t UdpTransport::read_chunk(void * buff, size_t size){ + require_socket(); + if (size < 64){ + throw std::invalid_argument("Buffer too small"); + } + + ssize_t len; + while(true) { + try { + boost::system::error_code ec; + len = receive(buff, size, &ec, true); + if (ec == boost::asio::error::operation_aborted) { + continue; + } else if (ec) { + throw exc::CommunicationException(std::string("Comm error: ") + ec.message()); + } + + if (len != 64) { + throw exc::CommunicationException("Invalid chunk size"); + } + + break; + + } catch(exc::CommunicationException const& e){ + throw; + } catch(std::exception const& e){ + MWARNING("Error reading chunk, reason: " << e.what()); + throw exc::CommunicationException(std::string("Chunk read error: ") + std::string(e.what())); + } + } + + return static_cast(len); + } + + ssize_t UdpTransport::receive(void * buff, size_t size, boost::system::error_code * error_code, bool no_throw, boost::posix_time::time_duration timeout){ + boost::system::error_code ec; + boost::asio::mutable_buffer buffer = boost::asio::buffer(buff, size); + + require_socket(); + + // Set a deadline for the asynchronous operation. + m_deadline.expires_from_now(timeout); + + // Set up the variables that receive the result of the asynchronous + // operation. The error code is set to would_block to signal that the + // operation is incomplete. Asio guarantees that its asynchronous + // operations will never fail with would_block, so any other value in + // ec indicates completion. + ec = boost::asio::error::would_block; + std::size_t length = 0; + + // Start the asynchronous operation itself. The handle_receive function + // used as a callback will update the ec and length variables. + m_socket->async_receive_from(boost::asio::buffer(buffer), m_endpoint, + boost::bind(&UdpTransport::handle_receive, _1, _2, &ec, &length)); + + // Block until the asynchronous operation has completed. + do { + m_io_service.run_one(); + } + while (ec == boost::asio::error::would_block); + + if (error_code){ + *error_code = ec; + } + + if (no_throw){ + return length; + } + + // Operation result + if (ec == boost::asio::error::operation_aborted){ + throw exc::TimeoutException(); + + } else if (ec) { + MWARNING("Reading from UDP socket failed: " << ec.message()); + throw exc::CommunicationException(); + + } + + return length; + } + + void UdpTransport::write(const google::protobuf::Message &req) { + m_proto->write(*this, req); + } + + void UdpTransport::read(std::shared_ptr & msg, messages::MessageType * msg_type) { + m_proto->read(*this, msg, msg_type); + } + + void UdpTransport::check_deadline(){ + if (!m_socket){ + return; // no active socket. + } + + // Check whether the deadline has passed. We compare the deadline against + // the current time since a new asynchronous operation may have moved the + // deadline before this actor had a chance to run. + if (m_deadline.expires_at() <= boost::asio::deadline_timer::traits_type::now()) + { + // The deadline has passed. The outstanding asynchronous operation needs + // to be cancelled so that the blocked receive() function will return. + // + // Please note that cancel() has portability issues on some versions of + // Microsoft Windows, and it may be necessary to use close() instead. + // Consult the documentation for cancel() for further information. + m_socket->cancel(); + + // There is no longer an active deadline. The expiry is set to positive + // infinity so that the actor takes no action until a new deadline is set. + m_deadline.expires_at(boost::posix_time::pos_infin); + } + + // Put the actor back to sleep. + m_deadline.async_wait(boost::bind(&UdpTransport::check_deadline, this)); + } + + void UdpTransport::handle_receive(const boost::system::error_code &ec, std::size_t length, + boost::system::error_code *out_ec, std::size_t *out_length) { + *out_ec = ec; + *out_length = length; + } + + std::ostream& UdpTransport::dump(std::ostream& o) const { + return o << "UdpTransport"; + } + +#ifdef WITH_DEVICE_TREZOR_WEBUSB + + static bool is_trezor1(libusb_device_descriptor * info){ + return info->idVendor == TREZOR_DESC_T1.id_vendor && info->idProduct == TREZOR_DESC_T1.id_product; + } + + static bool is_trezor2(libusb_device_descriptor * info){ + return info->idVendor == TREZOR_DESC_T2.id_vendor && info->idProduct == TREZOR_DESC_T2.id_product; + } + + static bool is_trezor2_bl(libusb_device_descriptor * info){ + return info->idVendor == TREZOR_DESC_T2_BL.id_vendor && info->idProduct == TREZOR_DESC_T2_BL.id_product; + } + + static ssize_t get_trezor_dev_id(libusb_device_descriptor *info){ + CHECK_AND_ASSERT_THROW_MES(info, "Empty device descriptor"); + return get_device_idx(info->idVendor, info->idProduct); + } + + static void set_libusb_log(libusb_context *ctx){ + CHECK_AND_ASSERT_THROW_MES(ctx, "Null libusb context"); + + // http://libusb.sourceforge.net/api-1.0/group__libusb__lib.html +#if defined(LIBUSB_API_VERSION) && (LIBUSB_API_VERSION >= 0x01000106) +# define TREZOR_LIBUSB_SET_DEBUG(ctx, level) libusb_set_option(ctx, LIBUSB_OPTION_LOG_LEVEL, level) +#else +# define TREZOR_LIBUSB_SET_DEBUG(ctx, level) libusb_set_debug(ctx, level) +#endif + + if (ELPP->vRegistry()->allowed(el::Level::Debug, MONERO_DEFAULT_LOG_CATEGORY)) + TREZOR_LIBUSB_SET_DEBUG(ctx, 3); + else if (ELPP->vRegistry()->allowed(el::Level::Warning, MONERO_DEFAULT_LOG_CATEGORY)) + TREZOR_LIBUSB_SET_DEBUG(ctx, 2); + else if (ELPP->vRegistry()->allowed(el::Level::Error, MONERO_DEFAULT_LOG_CATEGORY)) + TREZOR_LIBUSB_SET_DEBUG(ctx, 1); + +#undef TREZOR_LIBUSB_SET_DEBUG + } + + static int get_libusb_ports(libusb_device *dev, std::vector &path){ + uint8_t tmp_path[16]; + int r = libusb_get_port_numbers(dev, tmp_path, sizeof(tmp_path)); + CHECK_AND_ASSERT_MES(r != LIBUSB_ERROR_OVERFLOW, -1, "Libusb path array too small"); + CHECK_AND_ASSERT_MES(r >= 0, -1, "Libusb path array error"); + + path.resize(r); + for (int i = 0; i < r; i++){ + path[i] = tmp_path[i]; + } + + return 0; + } + + static std::string get_usb_path(uint8_t bus_id, const std::vector &path){ + std::stringstream ss; + ss << WebUsbTransport::PATH_PREFIX << (boost::format("%03d") % ((int)bus_id)); + for(uint8_t port : path){ + ss << ":" << ((int) port); + } + return ss.str(); + } + + const char * WebUsbTransport::PATH_PREFIX = "webusb:"; + + WebUsbTransport::WebUsbTransport( + boost::optional descriptor, + boost::optional> proto + ): m_usb_session(nullptr), m_usb_device(nullptr), m_usb_device_handle(nullptr), + m_bus_id(-1), m_device_addr(-1) + { + if (descriptor){ + libusb_device_descriptor * desc = new libusb_device_descriptor; + memcpy(desc, descriptor.get(), sizeof(libusb_device_descriptor)); + this->m_usb_device_desc.reset(desc); + } + + m_proto = proto ? proto.get() : std::make_shared(); + +#ifdef WITH_TREZOR_DEBUGGING + m_debug_mode = false; +#endif + } + + WebUsbTransport::~WebUsbTransport(){ + if (m_usb_device){ + close(); + } + + if (m_usb_session) { + libusb_exit(m_usb_session); + m_usb_session = nullptr; + } + } + + void WebUsbTransport::require_device() const{ + if (!m_usb_device_desc){ + throw std::runtime_error("No USB device specified"); + } + } + + void WebUsbTransport::require_connected() const{ + require_device(); + if (!m_usb_device_handle){ + throw std::runtime_error("USB Device not opened"); + } + } + + void WebUsbTransport::enumerate(t_transport_vect & res) { + int r; + libusb_device **devs; + libusb_context *ctx = nullptr; + + r = libusb_init(&ctx); + CHECK_AND_ASSERT_THROW_MES(r >= 0, "Unable to init libusb"); + + set_libusb_log(ctx); + + ssize_t cnt = libusb_get_device_list(ctx, &devs); + if (cnt < 0){ + libusb_exit(ctx); + throw std::runtime_error("Unable to enumerate libusb devices"); + } + + MTRACE("Libusb devices: " << cnt); + + for(ssize_t i = 0; i < cnt; i++) { + libusb_device_descriptor desc{}; + r = libusb_get_device_descriptor(devs[i], &desc); + if (r < 0){ + MERROR("Unable to get libusb device descriptor " << i); + continue; + } + + const auto trezor_dev_idx = get_trezor_dev_id(&desc); + if (!is_device_supported(trezor_dev_idx)){ + continue; + } + + MTRACE("Found Trezor device: " << desc.idVendor << ":" << desc.idProduct << " dev_idx " << (int)trezor_dev_idx); + + auto t = std::make_shared(boost::make_optional(&desc)); + t->m_bus_id = libusb_get_bus_number(devs[i]); + t->m_device_addr = libusb_get_device_address(devs[i]); + + // Port resolution may fail. Non-critical error, just addressing precision is decreased. + get_libusb_ports(devs[i], t->m_port_numbers); + + res.push_back(t); + } + + libusb_free_device_list(devs, 1); + libusb_exit(ctx); + } + + std::string WebUsbTransport::get_path() const { + if (!m_usb_device_desc){ + return ""; + } + + return get_usb_path(static_cast(m_bus_id), m_port_numbers); + }; + + void WebUsbTransport::open() { + if (!pre_open()){ + return; + } + const int interface = get_interface(); + +#define TREZOR_DESTROY_SESSION() do { libusb_exit(m_usb_session); m_usb_session = nullptr; } while(0) + + int r; + libusb_device **devs = nullptr; + + if (m_usb_session) { + TREZOR_DESTROY_SESSION(); + } + + r = libusb_init(&m_usb_session); + CHECK_AND_ASSERT_THROW_MES(r >= 0, "Unable to init libusb"); + set_libusb_log(m_usb_session); + + bool found = false; + int open_res = 0; + + ssize_t cnt = libusb_get_device_list(m_usb_session, &devs); + if (cnt < 0){ + TREZOR_DESTROY_SESSION(); + throw std::runtime_error("Unable to enumerate libusb devices"); + } + + for (ssize_t i = 0; i < cnt; i++) { + libusb_device_descriptor desc{}; + r = libusb_get_device_descriptor(devs[i], &desc); + if (r < 0){ + MERROR("Unable to get libusb device descriptor " << i); + continue; + } + + const auto trezor_dev_idx = get_trezor_dev_id(&desc); + if (!is_device_supported(trezor_dev_idx)){ + continue; + } + + auto bus_id = libusb_get_bus_number(devs[i]); + std::vector path; + + // Port resolution may fail. Non-critical error, just addressing precision is decreased. + get_libusb_ports(devs[i], path); + + MTRACE("Found Trezor device: " << desc.idVendor << ":" << desc.idProduct + << ", dev_idx: " << (int)trezor_dev_idx + << ". path: " << get_usb_path(bus_id, path)); + + if (bus_id == m_bus_id && path == m_port_numbers) { + found = true; + m_usb_device = devs[i]; + open_res = libusb_open(m_usb_device, &m_usb_device_handle); + break; + } + } + + libusb_free_device_list(devs, 1); + + if (!found){ + TREZOR_DESTROY_SESSION(); + throw exc::DeviceAcquireException("Device not found"); + + } else if (found && open_res != 0) { + m_usb_device_handle = nullptr; + m_usb_device = nullptr; + TREZOR_DESTROY_SESSION(); + throw exc::DeviceAcquireException("Unable to open libusb device"); + } + + r = libusb_claim_interface(m_usb_device_handle, interface); + + if (r != 0){ + libusb_close(m_usb_device_handle); + m_usb_device_handle = nullptr; + m_usb_device = nullptr; + TREZOR_DESTROY_SESSION(); + throw exc::DeviceAcquireException("Unable to claim libusb device"); + } + + m_open_counter = 1; + m_proto->session_begin(*this); + +#undef TREZOR_DESTROY_SESSION + }; + + void WebUsbTransport::close() { + if (!pre_close()){ + return; + } + + MTRACE("Closing Trezor:WebUsbTransport"); + m_proto->session_end(*this); + + int r = libusb_release_interface(m_usb_device_handle, get_interface()); + if (r != 0){ + MERROR("Could not release libusb interface: " << r); + } + + m_usb_device = nullptr; + if (m_usb_device_handle) { + libusb_close(m_usb_device_handle); + m_usb_device_handle = nullptr; + } + + if (m_usb_session) { + libusb_exit(m_usb_session); + m_usb_session = nullptr; + } + }; + + std::shared_ptr WebUsbTransport::find_debug() { +#ifdef WITH_TREZOR_DEBUGGING + require_device(); + auto t = std::make_shared(boost::make_optional(m_usb_device_desc.get())); + t->m_bus_id = m_bus_id; + t->m_device_addr = m_device_addr; + t->m_port_numbers = m_port_numbers; + t->m_debug_mode = true; + return t; +#else + MINFO("Debug link is disabled in production"); + return nullptr; +#endif + } + + int WebUsbTransport::get_interface() const{ + const int INTERFACE_NORMAL = 0; +#ifdef WITH_TREZOR_DEBUGGING + const int INTERFACE_DEBUG = 1; + return m_debug_mode ? INTERFACE_DEBUG : INTERFACE_NORMAL; +#else + return INTERFACE_NORMAL; +#endif + } + + unsigned char WebUsbTransport::get_endpoint() const{ + const unsigned char ENDPOINT_NORMAL = 1; +#ifdef WITH_TREZOR_DEBUGGING + const unsigned char ENDPOINT_DEBUG = 2; + return m_debug_mode ? ENDPOINT_DEBUG : ENDPOINT_NORMAL; +#else + return ENDPOINT_NORMAL; +#endif + } + + void WebUsbTransport::write(const google::protobuf::Message &req) { + m_proto->write(*this, req); + }; + + void WebUsbTransport::read(std::shared_ptr & msg, messages::MessageType * msg_type) { + m_proto->read(*this, msg, msg_type); + }; + + void WebUsbTransport::write_chunk(const void * buff, size_t size) { + require_connected(); + if (size != REPLEN){ + throw exc::CommunicationException("Invalid chunk size: "); + } + + unsigned char endpoint = get_endpoint(); + endpoint = (endpoint & ~LIBUSB_ENDPOINT_DIR_MASK) | LIBUSB_ENDPOINT_OUT; + + int transferred = 0; + int r = libusb_interrupt_transfer(m_usb_device_handle, endpoint, (unsigned char*)buff, (int)size, &transferred, 0); + CHECK_AND_ASSERT_THROW_MES(r == 0, "Unable to transfer, r: " << r); + if (transferred != (int)size){ + throw exc::CommunicationException("Could not transfer chunk"); + } + }; + + size_t WebUsbTransport::read_chunk(void * buff, size_t size) { + require_connected(); + unsigned char endpoint = get_endpoint(); + endpoint = (endpoint & ~LIBUSB_ENDPOINT_DIR_MASK) | LIBUSB_ENDPOINT_IN; + + int transferred = 0; + int r = libusb_interrupt_transfer(m_usb_device_handle, endpoint, (unsigned char*)buff, (int)size, &transferred, 0); + CHECK_AND_ASSERT_THROW_MES(r == 0, "Unable to transfer, r: " << r); + if (transferred != (int)size){ + throw exc::CommunicationException("Could not read the chunk"); + } + + return transferred; + }; + + std::ostream& WebUsbTransport::dump(std::ostream& o) const { + o << "WebUsbTransport"; + }; + +#endif // WITH_DEVICE_TREZOR_WEBUSB + + void enumerate(t_transport_vect & res){ + BridgeTransport bt; + try{ + bt.enumerate(res); + } catch (const std::exception & e){ + MERROR("BridgeTransport enumeration failed:" << e.what()); + } + +#ifdef WITH_DEVICE_TREZOR_WEBUSB + hw::trezor::WebUsbTransport btw; + try{ + btw.enumerate(res); + } catch (const std::exception & e){ + MERROR("WebUsbTransport enumeration failed:" << e.what()); + } +#endif + +#ifdef WITH_DEVICE_TREZOR_UDP + hw::trezor::UdpTransport btu; + try{ + btu.enumerate(res); + } catch (const std::exception & e){ + MERROR("UdpTransport enumeration failed:" << e.what()); + } +#endif + } + + void sort_transports_by_env(t_transport_vect & res){ + const char *env_trezor_path = getenv("TREZOR_PATH"); + if (!env_trezor_path){ + return; + } + + // Sort transports by the longest matching prefix with TREZOR_PATH + std::string trezor_path(env_trezor_path); + std::vector match_idx(res.size()); + std::vector path_permutation(res.size()); + + for(size_t i = 0; i < res.size(); ++i){ + auto cpath = res[i]->get_path(); + std::string * s1 = &trezor_path; + std::string * s2 = &cpath; + + // first has to be shorter in std::mismatch(). Returns first non-matching iterators. + if (s1->size() >= s2->size()){ + std::swap(s1, s2); + } + + const auto mism = std::mismatch(s1->begin(), s1->end(), s2->begin()); + match_idx[i] = mism.first - s1->begin(); + path_permutation[i] = i; + } + + std::sort(path_permutation.begin(), path_permutation.end(), [&](const size_t i0, const size_t i1) { + return match_idx[i0] > match_idx[i1]; + }); + + tools::apply_permutation(path_permutation, res); + } + + std::shared_ptr transport(const std::string & path){ + if (boost::starts_with(path, BridgeTransport::PATH_PREFIX)){ + return std::make_shared(path.substr(strlen(BridgeTransport::PATH_PREFIX))); + + } else if (boost::starts_with(path, UdpTransport::PATH_PREFIX)){ + return std::make_shared(path.substr(strlen(UdpTransport::PATH_PREFIX))); + + } else { + throw std::invalid_argument("Unknown Trezor device path: " + path); + + } + } + + void throw_failure_exception(const messages::common::Failure * failure) { + if (failure == nullptr){ + throw std::invalid_argument("Failure message cannot be null"); + } + + boost::optional message = failure->has_message() ? boost::make_optional(failure->message()) : boost::none; + boost::optional code = failure->has_code() ? boost::make_optional(static_cast(failure->code())) : boost::none; + if (!code){ + throw exc::proto::FailureException(code, message); + } + + auto ecode = failure->code(); + if (ecode == messages::common::Failure_FailureType_Failure_UnexpectedMessage){ + throw exc::proto::UnexpectedMessageException(code, message); + } else if (ecode == messages::common::Failure_FailureType_Failure_ActionCancelled){ + throw exc::proto::CancelledException(code, message); + } else if (ecode == messages::common::Failure_FailureType_Failure_PinExpected){ + throw exc::proto::PinExpectedException(code, message); + } else if (ecode == messages::common::Failure_FailureType_Failure_PinInvalid){ + throw exc::proto::InvalidPinException(code, message); + } else if (ecode == messages::common::Failure_FailureType_Failure_NotEnoughFunds){ + throw exc::proto::NotEnoughFundsException(code, message); + } else if (ecode == messages::common::Failure_FailureType_Failure_NotInitialized){ + throw exc::proto::NotInitializedException(code, message); + } else if (ecode == messages::common::Failure_FailureType_Failure_FirmwareError){ + throw exc::proto::FirmwareErrorException(code, message); + } else { + throw exc::proto::FailureException(code, message); + } + } + + GenericMessage::GenericMessage(messages::MessageType m_type, const shared_ptr &m_msg) + : m_type(m_type), m_msg(m_msg), m_empty(false) {} + + std::ostream& operator<<(std::ostream& o, hw::trezor::Transport const& t){ + return t.dump(o); + } + + std::ostream& operator<<(std::ostream& o, std::shared_ptr const& t){ + if (!t){ + return o << "None"; + } + + return t->dump(o); + } + +} +} + diff --git a/src/device_trezor/trezor/transport.hpp b/src/device_trezor/trezor/transport.hpp new file mode 100644 index 00000000000..affd9155358 --- /dev/null +++ b/src/device_trezor/trezor/transport.hpp @@ -0,0 +1,404 @@ +// Copyright (c) 2017-2019, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +#ifndef MONERO_TRANSPORT_H +#define MONERO_TRANSPORT_H + + +#include +#include +#include +#include + +#include +#include +#include "net/http_client.h" + +#include "rapidjson/document.h" +#include "rapidjson/writer.h" +#include "rapidjson/stringbuffer.h" + +#include "exceptions.hpp" +#include "trezor_defs.hpp" +#include "messages_map.hpp" + +#include "messages/messages.pb.h" +#include "messages/messages-common.pb.h" +#include "messages/messages-management.pb.h" +#include "messages/messages-monero.pb.h" + +namespace hw { +namespace trezor { + + using json = rapidjson::Document; + using json_val = rapidjson::Value; + namespace http = epee::net_utils::http; + + const std::string DEFAULT_BRIDGE = "127.0.0.1:21325"; + + uint64_t pack_version(uint32_t major, uint32_t minor=0, uint32_t patch=0); + + // Base HTTP comm serialization. + bool t_serialize(const std::string & in, std::string & out); + bool t_serialize(const json_val & in, std::string & out); + std::string t_serialize(const json_val & in); + + bool t_deserialize(const std::string & in, std::string & out); + bool t_deserialize(const std::string & in, json & out); + + // Flexible json serialization. HTTP client tailored for bridge API + template + bool invoke_bridge_http(const boost::string_ref uri, const t_req & out_struct, t_res & result_struct, t_transport& transport, const boost::string_ref method = "POST", std::chrono::milliseconds timeout = std::chrono::seconds(180)) + { + std::string req_param; + t_serialize(out_struct, req_param); + + http::fields_list additional_params; + additional_params.push_back(std::make_pair("Origin","https://monero.trezor.io")); + additional_params.push_back(std::make_pair("Content-Type","application/json; charset=utf-8")); + + const http::http_response_info* pri = nullptr; + if(!transport.invoke(uri, method, req_param, timeout, &pri, std::move(additional_params))) + { + MERROR("Failed to invoke http request to " << uri); + return false; + } + + if(!pri) + { + MERROR("Failed to invoke http request to " << uri << ", internal error (null response ptr)"); + return false; + } + + if(pri->m_response_code != 200) + { + MERROR("Failed to invoke http request to " << uri << ", wrong response code: " << pri->m_response_code + << " Response Body: " << pri->m_body); + return false; + } + + return t_deserialize(pri->m_body, result_struct); + } + + // Forward decl + class Transport; + class Protocol; + + // Communication protocol + class Protocol { + public: + Protocol() = default; + virtual ~Protocol() = default; + virtual void session_begin(Transport & transport){ }; + virtual void session_end(Transport & transport){ }; + virtual void write(Transport & transport, const google::protobuf::Message & req)= 0; + virtual void read(Transport & transport, std::shared_ptr & msg, messages::MessageType * msg_type=nullptr)= 0; + }; + + class ProtocolV1 : public Protocol { + public: + ProtocolV1() = default; + virtual ~ProtocolV1() = default; + + void write(Transport & transport, const google::protobuf::Message & req) override; + void read(Transport & transport, std::shared_ptr & msg, messages::MessageType * msg_type=nullptr) override; + }; + + + // Base transport + typedef std::vector> t_transport_vect; + + class Transport { + public: + Transport(); + virtual ~Transport() = default; + + virtual bool ping() { return false; }; + virtual std::string get_path() const { return ""; }; + virtual void enumerate(t_transport_vect & res){}; + virtual void open(){}; + virtual void close(){}; + virtual void write(const google::protobuf::Message & req) =0; + virtual void read(std::shared_ptr & msg, messages::MessageType * msg_type=nullptr) =0; + virtual std::shared_ptr find_debug() { return nullptr; }; + + virtual void write_chunk(const void * buff, size_t size) { }; + virtual size_t read_chunk(void * buff, size_t size) { return 0; }; + virtual std::ostream& dump(std::ostream& o) const { return o << "Transport<>"; } + protected: + long m_open_counter; + + virtual bool pre_open(); + virtual bool pre_close(); + }; + + // Bridge transport + class BridgeTransport : public Transport { + public: + BridgeTransport( + boost::optional device_path = boost::none, + boost::optional bridge_host = boost::none); + + virtual ~BridgeTransport() = default; + + static const char * PATH_PREFIX; + + std::string get_path() const override; + void enumerate(t_transport_vect & res) override; + + void open() override; + void close() override; + + void write(const google::protobuf::Message &req) override; + void read(std::shared_ptr & msg, messages::MessageType * msg_type=nullptr) override; + + const boost::optional & device_info() const; + std::ostream& dump(std::ostream& o) const override; + + private: + epee::net_utils::http::http_simple_client m_http_client; + std::string m_bridge_host; + boost::optional m_device_path; + boost::optional m_session; + boost::optional m_response; + boost::optional m_device_info; + }; + + // UdpTransport transport + using boost::asio::ip::udp; + + class UdpTransport : public Transport { + public: + + explicit UdpTransport( + boost::optional device_path=boost::none, + boost::optional> proto=boost::none); + + virtual ~UdpTransport() = default; + + static const char * PATH_PREFIX; + static const char * DEFAULT_HOST; + static const int DEFAULT_PORT; + + bool ping() override; + std::string get_path() const override; + void enumerate(t_transport_vect & res) override; + + void open() override; + void close() override; + std::shared_ptr find_debug() override; + + void write(const google::protobuf::Message &req) override; + void read(std::shared_ptr & msg, messages::MessageType * msg_type=nullptr) override; + + void write_chunk(const void * buff, size_t size) override; + size_t read_chunk(void * buff, size_t size) override; + + std::ostream& dump(std::ostream& o) const override; + + private: + void require_socket(); + ssize_t receive(void * buff, size_t size, boost::system::error_code * error_code=nullptr, bool no_throw=false, boost::posix_time::time_duration timeout=boost::posix_time::seconds(10)); + void check_deadline(); + static void handle_receive(const boost::system::error_code& ec, std::size_t length, + boost::system::error_code* out_ec, std::size_t* out_length); + bool ping_int(boost::posix_time::time_duration timeout=boost::posix_time::milliseconds(1500)); + + std::shared_ptr m_proto; + std::string m_device_host; + int m_device_port; + + std::unique_ptr m_socket; + boost::asio::io_service m_io_service; + boost::asio::deadline_timer m_deadline; + udp::endpoint m_endpoint; + }; + +#ifdef WITH_DEVICE_TREZOR_WEBUSB +#include + + class WebUsbTransport : public Transport { + public: + + explicit WebUsbTransport( + boost::optional descriptor = boost::none, + boost::optional> proto = boost::none + ); + + virtual ~WebUsbTransport(); + + static const char * PATH_PREFIX; + + std::string get_path() const override; + void enumerate(t_transport_vect & res) override; + + void open() override; + void close() override; + std::shared_ptr find_debug() override; + + void write(const google::protobuf::Message &req) override; + void read(std::shared_ptr & msg, messages::MessageType * msg_type=nullptr) override; + + void write_chunk(const void * buff, size_t size) override; + size_t read_chunk(void * buff, size_t size) override; + + std::ostream& dump(std::ostream& o) const override; + + private: + void require_device() const; + void require_connected() const; + int get_interface() const; + unsigned char get_endpoint() const; + + std::shared_ptr m_proto; + + libusb_context *m_usb_session; + libusb_device *m_usb_device; + libusb_device_handle *m_usb_device_handle; + std::unique_ptr m_usb_device_desc; + std::vector m_port_numbers; + int m_bus_id; + int m_device_addr; + +#ifdef WITH_TREZOR_DEBUGGING + bool m_debug_mode; +#endif + }; + +#endif + + // + // General helpers + // + + /** + * Enumerates all transports + */ + void enumerate(t_transport_vect & res); + + /** + * Sorts found transports by TREZOR_PATH environment variable. + */ + void sort_transports_by_env(t_transport_vect & res); + + /** + * Transforms path to the transport + */ + std::shared_ptr transport(const std::string & path); + + /** + * Transforms path to the particular transport + */ + template + std::shared_ptr transport_typed(const std::string & path){ + auto t = transport(path); + if (!t){ + return nullptr; + } + + return std::dynamic_pointer_cast(t); + } + + // Exception carries unexpected message being received + namespace exc { + class UnexpectedMessageException: public ProtocolException { + protected: + hw::trezor::messages::MessageType recvType; + std::shared_ptr recvMsg; + + public: + using ProtocolException::ProtocolException; + UnexpectedMessageException(): ProtocolException("Trezor returned unexpected message") {}; + UnexpectedMessageException(hw::trezor::messages::MessageType recvType, + const std::shared_ptr & recvMsg) + : recvType(recvType), recvMsg(recvMsg) { + reason = std::string("Trezor returned unexpected message: ") + std::to_string(recvType); + } + }; + } + + /** + * Throws corresponding failure exception. + */ + [[ noreturn ]] void throw_failure_exception(const messages::common::Failure * failure); + + /** + * Generic message holder, type + obj + */ + class GenericMessage { + public: + GenericMessage(): m_empty(true) {} + GenericMessage(messages::MessageType m_type, const std::shared_ptr &m_msg); + bool empty() const { return m_empty; } + + hw::trezor::messages::MessageType m_type; + std::shared_ptr m_msg; + bool m_empty; + }; + + /** + * Simple wrapper for write-read message exchange with expected message response type. + * + * @throws UnexpectedMessageException if the response message type is different than expected. + * Exception contains message type and the message itself. + */ + template + std::shared_ptr + exchange_message(Transport & transport, const google::protobuf::Message & req, + boost::optional resp_type = boost::none) + { + // Require strictly protocol buffers response in the template. + BOOST_STATIC_ASSERT(boost::is_base_of::value); + + // Write the request + transport.write(req); + + // Read the response + std::shared_ptr msg_resp; + hw::trezor::messages::MessageType msg_resp_type; + transport.read(msg_resp, &msg_resp_type); + + // Determine type of expected message response + messages::MessageType required_type = resp_type ? resp_type.get() : MessageMapper::get_message_wire_number(); + + if (msg_resp_type == required_type) { + return message_ptr_retype(msg_resp); + } else if (msg_resp_type == messages::MessageType_Failure){ + throw_failure_exception(dynamic_cast(msg_resp.get())); + } else { + throw exc::UnexpectedMessageException(msg_resp_type, msg_resp); + } + } + + std::ostream& operator<<(std::ostream& o, hw::trezor::Transport const& t); + std::ostream& operator<<(std::ostream& o, std::shared_ptr const& t); +}} + + +#endif //MONERO_TRANSPORT_H diff --git a/src/device_trezor/trezor/trezor_defs.hpp b/src/device_trezor/trezor/trezor_defs.hpp new file mode 100644 index 00000000000..f0697cdb54b --- /dev/null +++ b/src/device_trezor/trezor/trezor_defs.hpp @@ -0,0 +1,75 @@ +// Copyright (c) 2017-2019, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +#ifndef USE_DEVICE_TREZOR +#define USE_DEVICE_TREZOR 1 +#endif + +// HAVE_TREZOR_READY set by cmake after protobuf messages +// were generated successfully and all minimal dependencies are met. +#ifndef DEVICE_TREZOR_READY +#undef USE_DEVICE_TREZOR +#define USE_DEVICE_TREZOR 0 +#endif + +#if USE_DEVICE_TREZOR +#define WITH_DEVICE_TREZOR 1 +#endif + +#ifndef WITH_DEVICE_TREZOR +#undef WITH_DEVICE_TREZOR_LITE +#endif + +#if defined(HAVE_TREZOR_LIBUSB) && USE_DEVICE_TREZOR +#define WITH_DEVICE_TREZOR_WEBUSB 1 +#endif + +// Enable / disable UDP in the enumeration +#ifndef USE_DEVICE_TREZOR_UDP +#define USE_DEVICE_TREZOR_UDP 1 +#endif + +// Enable / disable UDP in the enumeration for release +#ifndef USE_DEVICE_TREZOR_UDP_RELEASE +#define USE_DEVICE_TREZOR_UDP_RELEASE 0 +#endif + +#if USE_DEVICE_TREZOR_UDP && (USE_DEVICE_TREZOR_UDP_RELEASE || defined(TREZOR_DEBUG)) +#define WITH_DEVICE_TREZOR_UDP 1 +#endif + +// Avoids protobuf undefined macro warning +#ifndef PROTOBUF_INLINE_NOT_IN_HEADERS +#define PROTOBUF_INLINE_NOT_IN_HEADERS 0 +#endif + +// Fixes gcc7 problem with minor macro defined clashing with minor() field. +#ifdef minor +#undef minor +#endif diff --git a/src/gen_multisig/CMakeLists.txt b/src/gen_multisig/CMakeLists.txt index 18a6a9efe73..3a5e2927301 100644 --- a/src/gen_multisig/CMakeLists.txt +++ b/src/gen_multisig/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2017-2018, The Monero Project +# Copyright (c) 2017-2019, The Monero Project # # All rights reserved. # diff --git a/src/gen_multisig/gen_multisig.cpp b/src/gen_multisig/gen_multisig.cpp index 03e0a79464d..06019c50e5b 100644 --- a/src/gen_multisig/gen_multisig.cpp +++ b/src/gen_multisig/gen_multisig.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2018, The Monero Project +// Copyright (c) 2017-2019, The Monero Project // // All rights reserved. // @@ -91,7 +91,7 @@ static bool generate_multisig(uint32_t threshold, uint32_t total, const std::str for (size_t n = 0; n < total; ++n) { std::string name = basename + "-" + std::to_string(n + 1); - wallets[n].reset(new tools::wallet2(nettype)); + wallets[n].reset(new tools::wallet2(nettype, 1, false)); wallets[n]->init(""); wallets[n]->generate(name, pwd_container->password(), rct::rct2sk(rct::skGen()), false, false, create_address_file); } @@ -101,11 +101,13 @@ static bool generate_multisig(uint32_t threshold, uint32_t total, const std::str std::vector pk(total); for (size_t n = 0; n < total; ++n) { + wallets[n]->decrypt_keys(pwd_container->password()); if (!tools::wallet2::verify_multisig_info(wallets[n]->get_multisig_info(), sk[n], pk[n])) { - tools::fail_msg_writer() << tr("Failed to verify multisig info"); + tools::fail_msg_writer() << genms::tr("Failed to verify multisig info"); return false; } + wallets[n]->encrypt_keys(pwd_container->password()); } // make the wallets multisig @@ -128,8 +130,8 @@ static bool generate_multisig(uint32_t threshold, uint32_t total, const std::str ss << " " << name << std::endl; } - // finalize step if needed - if (!extra_info[0].empty()) + //exchange keys unless exchange_multisig_keys returns no extra info + while (!extra_info[0].empty()) { std::unordered_set pkeys; std::vector signers(total); @@ -143,11 +145,7 @@ static bool generate_multisig(uint32_t threshold, uint32_t total, const std::str } for (size_t n = 0; n < total; ++n) { - if (!wallets[n]->finalize_multisig(pwd_container->password(), pkeys, signers)) - { - tools::fail_msg_writer() << genms::tr("Error finalizing multisig"); - return false; - } + extra_info[n] = wallets[n]->exchange_multisig_keys(pwd_container->password(), pkeys, signers); } } @@ -165,6 +163,8 @@ static bool generate_multisig(uint32_t threshold, uint32_t total, const std::str int main(int argc, char* argv[]) { + TRY_ENTRY(); + po::options_description desc_params(wallet_args::tr("Wallet options")); command_line::add_arg(desc_params, arg_filename_base); command_line::add_arg(desc_params, arg_scheme); @@ -242,15 +242,10 @@ int main(int argc, char* argv[]) return 1; } - if (threshold != total-1 && threshold != total) - { - tools::fail_msg_writer() << genms::tr("Error: unsupported scheme: only N/N and N-1/N are supported"); - return 1; - } bool create_address_file = command_line::get_arg(*vm, arg_create_address_file); if (!generate_multisig(threshold, total, basename, testnet ? TESTNET : stagenet ? STAGENET : MAINNET, create_address_file)) return 1; return 0; - //CATCH_ENTRY_L0("main", 1); + CATCH_ENTRY_L0("main", 1); } diff --git a/src/lmdb/CMakeLists.txt b/src/lmdb/CMakeLists.txt new file mode 100644 index 00000000000..1f369f11479 --- /dev/null +++ b/src/lmdb/CMakeLists.txt @@ -0,0 +1,33 @@ +# Copyright (c) 2014-2018, The Monero Project +# +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without modification, are +# permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this list of +# conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, this list +# of conditions and the following disclaimer in the documentation and/or other +# materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors may be +# used to endorse or promote products derived from this software without specific +# prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +# THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +# THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +set(lmdb_sources database.cpp error.cpp table.cpp value_stream.cpp) +set(lmdb_headers database.h error.h key_stream.h table.h transaction.h util.h value_stream.h) + +monero_add_library(lmdb_lib ${lmdb_sources} ${lmdb_headers}) +target_link_libraries(lmdb_lib common ${LMDB_LIBRARY}) diff --git a/src/lmdb/database.cpp b/src/lmdb/database.cpp new file mode 100644 index 00000000000..ccab1902ae0 --- /dev/null +++ b/src/lmdb/database.cpp @@ -0,0 +1,187 @@ +// Copyright (c) 2014-2018, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#include "database.h" +#include "lmdb/error.h" +#include "lmdb/util.h" + +#ifdef _WIN32 +namespace +{ + constexpr const mdb_mode_t open_flags = 0; +} +#else +#include + +namespace +{ + constexpr const mdb_mode_t open_flags = (S_IRUSR | S_IWUSR); +} +#endif + +namespace lmdb +{ + namespace + { + constexpr const mdb_size_t max_resize = 1 * 1024 * 1024 * 1024; // 1 GB + void acquire_context(context& ctx) noexcept + { + while (ctx.lock.test_and_set()); + ++(ctx.active); + ctx.lock.clear(); + } + + void release_context(context& ctx) noexcept + { + --(ctx.active); + } + } + + void release_read_txn::operator()(MDB_txn* ptr) const noexcept + { + if (ptr) + { + MDB_env* const env = mdb_txn_env(ptr); + abort_txn{}(ptr); + if (env) + { + context* ctx = reinterpret_cast(mdb_env_get_userctx(env)); + if (ctx) + release_context(*ctx); + } + } + } + + expect open_environment(const char* path, MDB_dbi max_dbs) noexcept + { + MONERO_PRECOND(path != nullptr); + + MDB_env* obj = nullptr; + MONERO_LMDB_CHECK(mdb_env_create(std::addressof(obj))); + environment out{obj}; + + MONERO_LMDB_CHECK(mdb_env_set_maxdbs(out.get(), max_dbs)); + MONERO_LMDB_CHECK(mdb_env_open(out.get(), path, 0, open_flags)); + return {std::move(out)}; + } + + expect database::do_create_txn(unsigned int flags) noexcept + { + MONERO_PRECOND(handle() != nullptr); + + for (unsigned attempts = 0; attempts < 3; ++attempts) + { + acquire_context(ctx); + + MDB_txn* txn = nullptr; + const int err = + mdb_txn_begin(handle(), nullptr, flags, &txn); + if (!err && txn != nullptr) + return write_txn{txn}; + + release_context(ctx); + if (err != MDB_MAP_RESIZED) + return {lmdb::error(err)}; + MONERO_CHECK(this->resize()); + } + return {lmdb::error(MDB_MAP_RESIZED)}; + } + + database::database(environment env) + : env(std::move(env)), ctx{{}, ATOMIC_FLAG_INIT} + { + if (handle()) + { + const int err = mdb_env_set_userctx(handle(), std::addressof(ctx)); + if (err) + MONERO_THROW(lmdb::error(err), "Failed to set user context"); + } + } + + database::~database() noexcept + { + while (ctx.active); + } + + expect database::resize() noexcept + { + MONERO_PRECOND(handle() != nullptr); + + while (ctx.lock.test_and_set()); + while (ctx.active); + + MDB_envinfo info{}; + MONERO_LMDB_CHECK(mdb_env_info(handle(), &info)); + + const mdb_size_t resize = std::min(info.me_mapsize, max_resize); + const int err = mdb_env_set_mapsize(handle(), info.me_mapsize + resize); + ctx.lock.clear(); + if (err) + return {lmdb::error(err)}; + return success(); + } + + expect database::create_read_txn(suspended_txn txn) noexcept + { + if (txn) + { + acquire_context(ctx); + const int err = mdb_txn_renew(txn.get()); + if (err) + { + release_context(ctx); + return {lmdb::error(err)}; + } + return read_txn{txn.release()}; + } + auto new_txn = do_create_txn(MDB_RDONLY); + if (new_txn) + return read_txn{new_txn->release()}; + return new_txn.error(); + } + + expect database::reset_txn(read_txn txn) noexcept + { + MONERO_PRECOND(txn != nullptr); + mdb_txn_reset(txn.get()); + release_context(ctx); + return suspended_txn{txn.release()}; + } + + expect database::create_write_txn() noexcept + { + return do_create_txn(0); + } + + expect database::commit(write_txn txn) noexcept + { + MONERO_PRECOND(txn != nullptr); + MONERO_LMDB_CHECK(mdb_txn_commit(txn.get())); + txn.release(); + release_context(ctx); + return success(); + } +} // lmdb diff --git a/src/lmdb/database.h b/src/lmdb/database.h new file mode 100644 index 00000000000..269f8c8a136 --- /dev/null +++ b/src/lmdb/database.h @@ -0,0 +1,138 @@ +// Copyright (c) 2018, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#pragma once + +#include +#include +#include +#include +#include + +#include "common/expect.h" +#include "lmdb/error.h" +#include "lmdb/transaction.h" + +namespace lmdb +{ + //! Closes LMDB environment handle. + struct close_env + { + void operator()(MDB_env* ptr) const noexcept + { + if (ptr) + mdb_env_close(ptr); + } + }; + + using environment = std::unique_ptr; + + //! \return LMDB environment at `path` with a max of `max_dbs` tables. + expect open_environment(const char* path, MDB_dbi max_dbs) noexcept; + + //! Context given to LMDB. + struct context + { + std::atomic active; + std::atomic_flag lock; + }; + + //! Manages a LMDB environment for safe memory-map resizing. Thread-safe. + class database + { + environment env; + context ctx; + + //! \return The LMDB environment associated with the object. + MDB_env* handle() const noexcept { return env.get(); } + + expect do_create_txn(unsigned int flags) noexcept; + + public: + database(environment env); + + database(database&&) = delete; + database(database const&) = delete; + + virtual ~database() noexcept; + + database& operator=(database&&) = delete; + database& operator=(database const&) = delete; + + /*! + Resize the memory map for the LMDB environment. Will block until + all reads/writes on the environment complete. + */ + expect resize() noexcept; + + //! \return A read only LMDB transaction, reusing `txn` if provided. + expect create_read_txn(suspended_txn txn = nullptr) noexcept; + + //! \return `txn` after releasing context. + expect reset_txn(read_txn txn) noexcept; + + //! \return A read-write LMDB transaction. + expect create_write_txn() noexcept; + + //! Commit the read-write transaction. + expect commit(write_txn txn) noexcept; + + /*! + Create a write transaction, pass it to `f`, then try to commit + the write if `f` succeeds. + + \tparam F must be callable with signature `expect(MDB_txn&)`. + \param f must be re-startable if `lmdb::error(MDB_MAP_FULL)`. + + \return The result of calling `f`. + */ + template + typename std::result_of::type try_write(F f, unsigned attempts = 3) + { + for (unsigned i = 0; i < attempts; ++i) + { + expect txn = create_write_txn(); + if (!txn) + return txn.error(); + + MONERO_PRECOND(*txn != nullptr); + const auto wrote = f(*(*txn)); + if (wrote) + { + MONERO_CHECK(commit(std::move(*txn))); + return wrote; + } + if (wrote != lmdb::error(MDB_MAP_FULL)) + return wrote; + + txn->reset(); + MONERO_CHECK(this->resize()); + } + return {lmdb::error(MDB_MAP_FULL)}; + } + }; +} // lmdb + diff --git a/src/lmdb/error.cpp b/src/lmdb/error.cpp new file mode 100644 index 00000000000..91479521e93 --- /dev/null +++ b/src/lmdb/error.cpp @@ -0,0 +1,98 @@ +// Copyright (c) 2014-2018, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "error.h" + +#include +#include + +namespace { + struct category final : std::error_category + { + virtual const char* name() const noexcept override final + { + return "lmdb::error_category()"; + } + + virtual std::string message(int value) const override final + { + char const* const msg = mdb_strerror(value); + if (msg) + return msg; + return "Unknown lmdb::error_category() value"; + } + + virtual std::error_condition default_error_condition(int value) const noexcept override final + { + switch (value) + { + case MDB_KEYEXIST: + case MDB_NOTFOUND: + break; // map to nothing generic + case MDB_PAGE_NOTFOUND: + case MDB_CORRUPTED: + return std::errc::bad_address; + case MDB_PANIC: + case MDB_VERSION_MISMATCH: + case MDB_INVALID: + break; // map to nothing generic + case MDB_MAP_FULL: + return std::errc::no_buffer_space; + case MDB_DBS_FULL: + break; // map to nothing generic + case MDB_READERS_FULL: + case MDB_TLS_FULL: + return std::errc::no_lock_available; + case MDB_TXN_FULL: + case MDB_CURSOR_FULL: + case MDB_PAGE_FULL: + case MDB_MAP_RESIZED: + break; // map to nothing generic + case MDB_INCOMPATIBLE: + return std::errc::invalid_argument; + case MDB_BAD_RSLOT: + case MDB_BAD_TXN: + case MDB_BAD_VALSIZE: + case MDB_BAD_DBI: + return std::errc::invalid_argument; + default: + return std::error_condition{value, std::generic_category()}; + } + return std::error_condition{value, *this}; + } + }; +} + +namespace lmdb +{ + std::error_category const& error_category() noexcept + { + static const category instance{}; + return instance; + } +} + diff --git a/src/lmdb/error.h b/src/lmdb/error.h new file mode 100644 index 00000000000..2944adf785d --- /dev/null +++ b/src/lmdb/error.h @@ -0,0 +1,64 @@ +// Copyright (c) 2014-2018, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#pragma once + +#include +#include + +//! Executes a LMDB command, and returns errors via `lmdb::error` enum. +#define MONERO_LMDB_CHECK(...) \ + do \ + { \ + const int err = __VA_ARGS__ ; \ + if (err) \ + return {lmdb::error(err)}; \ + } while (0) + +namespace lmdb +{ + //! Tracks LMDB error codes. + enum class error : int + { + // 0 is reserved for no error, as per expect + // All other errors are the values reported by LMDB + }; + + std::error_category const& error_category() noexcept; + + inline std::error_code make_error_code(error value) noexcept + { + return std::error_code{int(value), error_category()}; + } +} + +namespace std +{ + template<> + struct is_error_code_enum<::lmdb::error> + : true_type + {}; +} diff --git a/src/lmdb/key_stream.h b/src/lmdb/key_stream.h new file mode 100644 index 00000000000..40434d3a1a7 --- /dev/null +++ b/src/lmdb/key_stream.h @@ -0,0 +1,264 @@ +// Copyright (c) 2018, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "lmdb/value_stream.h" +#include "span.h" + +namespace lmdb +{ + + /*! + An InputIterator for a fixed-sized LMDB key and value. `operator++` + iterates over keys. + + \tparam K Key type in database records. + \tparam V Value type in database records. + + \note This meets requirements for an InputIterator only. The iterator + can only be incremented and dereferenced. All copies of an iterator + share the same LMDB cursor, and therefore incrementing any copy will + change the cursor state for all (incrementing an iterator will + invalidate all prior copies of the iterator). Usage is identical + to `std::istream_iterator`. + */ + template + class key_iterator + { + MDB_cursor* cur; + epee::span key; + + void increment() + { + // MDB_NEXT_MULTIPLE doesn't work if only one value is stored :/ + if (cur) + key = lmdb::stream::get(*cur, MDB_NEXT_NODUP, sizeof(K), sizeof(V)).first; + } + + public: + using value_type = std::pair>>; + using reference = value_type; + using pointer = void; + using difference_type = std::size_t; + using iterator_category = std::input_iterator_tag; + + //! Construct an "end" iterator. + key_iterator() noexcept + : cur(nullptr), key() + {} + + /*! + \param cur Iterate over keys starting at this cursor position. + \throw std::system_error if unexpected LMDB error. This can happen + if `cur` is invalid. + */ + key_iterator(MDB_cursor* cur) + : cur(cur), key() + { + if (cur) + key = lmdb::stream::get(*cur, MDB_GET_CURRENT, sizeof(K), sizeof(V)).first; + } + + //! \return True if `this` is one-past the last key. + bool is_end() const noexcept { return key.empty(); } + + //! \return True iff `rhs` is referencing `this` key. + bool equal(key_iterator const& rhs) const noexcept + { + return + (key.empty() && rhs.key.empty()) || + key.data() == rhs.key.data(); + } + + /*! + Moves iterator to next key or end. Invalidates all prior copies of + the iterator. + */ + key_iterator& operator++() + { + increment(); + return *this; + } + + /*! + Moves iterator to next key or end. + + \return A copy that is already invalidated, ignore + */ + key_iterator operator++(int) + { + key_iterator out{*this}; + increment(); + return out; + } + + //! \pre `!is_end()` \return {current key, current value range} + value_type operator*() const + { + return {get_key(), make_value_range()}; + } + + //! \pre `!is_end()` \return Current key + K get_key() const noexcept + { + assert(!is_end()); + K out; + std::memcpy(std::addressof(out), key.data(), sizeof(out)); + return out; + } + + /*! + Return a C++ iterator over database values from current cursor + position that will reach `.is_end()` after the last duplicate key + record. Calling `make_iterator()` will return an iterator whose + `operator*` will return an entire value (`V`). + `make_iterator()` will return an + iterator whose `operator*` will return a `decltype(account.id)` + object - the other fields in the struct `account` are never copied + from the database. + + \throw std::system_error if LMDB has unexpected errors. + \return C++ iterator starting at current cursor position. + */ + template + value_iterator make_value_iterator() const + { + static_assert(std::is_same(), "bad MONERO_FIELD usage?"); + return {cur}; + } + + /*! + Return a range from current cursor position until last duplicate + key record. Useful in for-each range loops or in templated code + expecting a range of elements. Calling `make_range()` will return + a range of `T` objects. `make_range()` + will return a range of `decltype(account.id)` objects - the other + fields in the struct `account` are never copied from the database. + + \throw std::system_error if LMDB has unexpected errors. + \return An InputIterator range over values at cursor position. + */ + template + boost::iterator_range> make_value_range() const + { + return {make_value_iterator(), value_iterator{}}; + } + }; + + /*! + C++ wrapper for a LMDB read-only cursor on a fixed-sized key `K` and + value `V`. + + \tparam K key type being stored by each record. + \tparam V value type being stored by each record. + \tparam D cleanup functor for the cursor; usually unique per db/table. + */ + template + class key_stream + { + std::unique_ptr cur; + public: + + //! Take ownership of `cur` without changing position. `nullptr` valid. + explicit key_stream(std::unique_ptr cur) + : cur(std::move(cur)) + {} + + key_stream(key_stream&&) = default; + key_stream(key_stream const&) = delete; + ~key_stream() = default; + key_stream& operator=(key_stream&&) = default; + key_stream& operator=(key_stream const&) = delete; + + /*! + Give up ownership of the cursor. `make_iterator()` and + `make_range()` can still be invoked, but return the empty set. + + \return Currently owned LMDB cursor. + */ + std::unique_ptr give_cursor() noexcept + { + return {std::move(cur)}; + } + + /*! + Place the stream back at the first key/value. Newly created + iterators will start at the first value again. + + \note Invalidates all current iterators, including those created + with `make_iterator` or `make_range`. Also invalidates all + `value_iterator`s created with `key_iterator`. + */ + void reset() + { + if (cur) + lmdb::stream::get(*cur, MDB_FIRST, 0, 0); + } + + /*! + \throw std::system_error if LMDB has unexpected errors. + \return C++ iterator over database keys from current cursor + position that will reach `.is_end()` after the last key. + */ + key_iterator make_iterator() const + { + return {cur.get()}; + } + + /*! + \throw std::system_error if LMDB has unexpected errors. + \return Range from current cursor position until last key record. + Useful in for-each range loops or in templated code + */ + boost::iterator_range> make_range() const + { + return {make_iterator(), key_iterator{}}; + } + }; + + template + inline + bool operator==(key_iterator const& lhs, key_iterator const& rhs) noexcept + { + return lhs.equal(rhs); + } + + template + inline + bool operator!=(key_iterator const& lhs, key_iterator const& rhs) noexcept + { + return !lhs.equal(rhs); + } +} // lmdb + diff --git a/src/lmdb/table.cpp b/src/lmdb/table.cpp new file mode 100644 index 00000000000..0818b74e688 --- /dev/null +++ b/src/lmdb/table.cpp @@ -0,0 +1,43 @@ +// Copyright (c) 2018, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#include "table.h" + +namespace lmdb +{ + expect table::open(MDB_txn& write_txn) const noexcept + { + MONERO_PRECOND(name != nullptr); + + MDB_dbi out; + MONERO_LMDB_CHECK(mdb_dbi_open(&write_txn, name, flags, &out)); + if (key_cmp && !(flags & MDB_INTEGERKEY)) + MONERO_LMDB_CHECK(mdb_set_compare(&write_txn, out, key_cmp)); + if (value_cmp && !(flags & MDB_INTEGERDUP)) + MONERO_LMDB_CHECK(mdb_set_dupsort(&write_txn, out, value_cmp)); + return out; + } +} diff --git a/src/lmdb/table.h b/src/lmdb/table.h new file mode 100644 index 00000000000..41a3de2965e --- /dev/null +++ b/src/lmdb/table.h @@ -0,0 +1,120 @@ +#pragma once + +#include + +#include "common/expect.h" +#include "lmdb/error.h" +#include "lmdb/key_stream.h" +#include "lmdb/util.h" +#include "lmdb/value_stream.h" + +namespace lmdb +{ + //! Helper for grouping typical LMDB DBI options. + struct table + { + char const* const name; + const unsigned flags; + MDB_cmp_func const* const key_cmp; + MDB_cmp_func const* const value_cmp; + + //! \pre `name != nullptr` \return Open table. + expect open(MDB_txn& write_txn) const noexcept; + }; + + //! Helper for grouping typical LMDB DBI options when key and value are fixed types. + template + struct basic_table : table + { + using key_type = K; + using value_type = V; + + //! \return Additional LMDB flags based on `flags` value. + static constexpr unsigned compute_flags(const unsigned flags) noexcept + { + return flags | ((flags & MDB_DUPSORT) ? MDB_DUPFIXED : 0); + } + + constexpr explicit basic_table(const char* name, unsigned flags = 0, MDB_cmp_func value_cmp = nullptr) noexcept + : table{name, compute_flags(flags), &lmdb::less>, value_cmp} + {} + + /*! + \tparam U must be same as `V`; used for sanity checking. + \tparam F is the type within `U` that is being extracted. + \tparam offset to `F` within `U`. + + \note If using `F` and `offset` to retrieve a specific field, use + `MONERO_FIELD` macro in `src/lmdb/util.h` which calculates the + offset automatically. + + \return Value of type `F` at `offset` within `value` which has + type `U`. + */ + template + static expect get_value(MDB_val value) noexcept + { + static_assert(std::is_same(), "bad MONERO_FIELD?"); + static_assert(std::is_pod(), "F must be POD"); + static_assert(sizeof(F) + offset <= sizeof(U), "bad field type and/or offset"); + + if (value.mv_size != sizeof(U)) + return {lmdb::error(MDB_BAD_VALSIZE)}; + + F out; + std::memcpy(std::addressof(out), static_cast(value.mv_data) + offset, sizeof(out)); + return out; + } + + /*! + \pre `cur != nullptr`. + \param cur Active cursor on table. Returned in object on success, + otherwise destroyed. + \return A handle to the first key/value in the table linked + to `cur` or an empty `key_stream`. + */ + template + expect> + static get_key_stream(std::unique_ptr cur) noexcept + { + MONERO_PRECOND(cur != nullptr); + + MDB_val key; + MDB_val value; + const int err = mdb_cursor_get(cur.get(), &key, &value, MDB_FIRST); + if (err) + { + if (err != MDB_NOTFOUND) + return {lmdb::error(err)}; + cur.reset(); // return empty set + } + return key_stream{std::move(cur)}; + } + + /*! + \pre `cur != nullptr`. + \param cur Active cursor on table. Returned in object on success, + otherwise destroyed. + \return A handle to the first value at `key` in the table linked + to `cur` or an empty `value_stream`. + */ + template + expect> + static get_value_stream(K const& key, std::unique_ptr cur) noexcept + { + MONERO_PRECOND(cur != nullptr); + + MDB_val key_bytes = lmdb::to_val(key); + MDB_val value; + const int err = mdb_cursor_get(cur.get(), &key_bytes, &value, MDB_SET); + if (err) + { + if (err != MDB_NOTFOUND) + return {lmdb::error(err)}; + cur.reset(); // return empty set + } + return value_stream{std::move(cur)}; + } + }; +} // lmdb + diff --git a/src/lmdb/transaction.h b/src/lmdb/transaction.h new file mode 100644 index 00000000000..cdd80696c63 --- /dev/null +++ b/src/lmdb/transaction.h @@ -0,0 +1,95 @@ +// Copyright (c) 2018, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#pragma once + +#include +#include + +#include "lmdb/error.h" + +//! Uses C++ type system to differentiate between cursors +#define MONERO_CURSOR(name) \ + struct close_ ## name : ::lmdb::close_cursor {}; \ + using name = std::unique_ptr< MDB_cursor, close_ ## name >; + +namespace lmdb +{ + struct abort_txn + { + void operator()(MDB_txn* ptr) const noexcept + { + if (ptr) + mdb_txn_abort(ptr); + } + }; + + /*! + Only valid if used via `create_read_txn()`. Decrements active count in + associated `context`, and aborts a LMDB transaction (`mdb_txn_abort`). + */ + struct release_read_txn + { + void operator()(MDB_txn* ptr) const noexcept; + // implementation in database.cpp + }; + + /*! + Only valid if used via `create_write_txn()`. Decrements active count in + associated `context`, and aborts a LMDB transaction (`mdb_txn_abort`). + */ + struct abort_write_txn + { + void operator()(MDB_txn* ptr) const noexcept + { + release_read_txn{}(ptr); + } + }; + + struct close_cursor + { + void operator()(MDB_cursor* ptr) const noexcept + { + if (ptr) + mdb_cursor_close(ptr); + } + }; + + template + inline expect> + open_cursor(MDB_txn& txn, MDB_dbi tbl) noexcept + { + MDB_cursor* cur = nullptr; + MONERO_LMDB_CHECK(mdb_cursor_open(&txn, tbl, &cur)); + return std::unique_ptr{cur}; + } + + // The below use the C++ type system to designate `MDB_txn` status. + + using suspended_txn = std::unique_ptr; + using read_txn = std::unique_ptr; + using write_txn = std::unique_ptr; +} // lmdb diff --git a/src/lmdb/util.h b/src/lmdb/util.h new file mode 100644 index 00000000000..50162b7c84e --- /dev/null +++ b/src/lmdb/util.h @@ -0,0 +1,149 @@ +// Copyright (c) 2018, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#pragma once + +#include +#include +#include +#include +#include + +#include "span.h" + +/*! Calculates types and offset of struct field. Use in template arguments for + `table::get_value`, `value_iterator::get_value`, + `value_stream::make_iterator`, or `value_stream::make_range`. */ +#define MONERO_FIELD(obj, field) \ + obj , decltype(std::declval().field) , offsetof(obj, field) + +//! Expands to `lmdb::less` for the value `field` within `obj`. +#define MONERO_SORT_BY(obj, field) \ + &::lmdb::less< \ + lmdb::native_type().field)>, \ + offsetof(obj, field) \ + > + +//! Expands to `lmdb::compare` for the value `field` within `obj`. +#define MONERO_COMPARE(obj, field) \ + &::lmdb::compare< \ + decltype(std::declval().field), \ + offsetof(obj, field) \ + > + +namespace lmdb +{ + //! Prevent instantiation of `std::underlying_type` when `T` is not enum. + template + struct identity + { + using type = T; + }; + + /*! + Get the native type for enums, or return `T` unchanged. Useful for + merging generated machine code for templated functions that use enums + with identical size-widths without relying on aggressive identical + comdat folding (ICF) support in linker. So with enum defintion + `enum class enum_foo : unsigned long {};` will always yield + `assert(&func_foo == &func_foo>)`. + */ + template + using native_type = typename std::conditional< + std::is_enum::value, std::underlying_type, identity + >::type::type; + + //! \return `value` as its native type. + template::type> + inline constexpr U to_native(T value) noexcept + { + return U(value); + } + + //! \return `value` bytes in a LMDB `MDB_val` object. + template + inline MDB_val to_val(T&& value) noexcept + { + // lmdb does not touch user data, so const_cast is acceptable + static_assert(!std::is_rvalue_reference(), "cannot use temporary value"); + void const* const temp = reinterpret_cast(std::addressof(value)); + return MDB_val{sizeof(value), const_cast(temp)}; + } + + //! \return A span over the same chunk of memory as `value`. + inline constexpr epee::span to_byte_span(MDB_val value) noexcept + { + return {static_cast(value.mv_data), value.mv_size}; + } + + /*! + A LMDB comparison function that uses `operator<`. + + \tparam T has a defined `operator<` . + \tparam offset to `T` within the value. + + \return -1 if `left < right`, 1 if `right < left`, and 0 otherwise. + */ + template + inline int less(MDB_val const* left, MDB_val const* right) noexcept + { + if (!left || !right || left->mv_size < sizeof(T) + offset || right->mv_size < sizeof(T) + offset) + { + assert("invalid use of custom comparison" == 0); + return -1; + } + + T left_val; + T right_val; + std::memcpy(std::addressof(left_val), static_cast(left->mv_data) + offset, sizeof(T)); + std::memcpy(std::addressof(right_val), static_cast(right->mv_data) + offset, sizeof(T)); + return left_val < right_val ? -1 : bool(right_val < left_val); + } + + /*! + A LMDB comparison function that uses `std::memcmp`. + + \toaram T is `!epee::has_padding` + \tparam offset to `T` within the value. + + \return The result of `std::memcmp` over the value. + */ + template + inline int compare(MDB_val const* left, MDB_val const* right) noexcept + { + static_assert(!epee::has_padding(), "memcmp will not work"); + if (!left || !right || left->mv_size < sizeof(T) + offset || right->mv_size < sizeof(T) + offset) + { + assert("invalid use of custom comparison" == 0); + return -1; + } + return std::memcmp( + static_cast(left->mv_data) + offset, + static_cast(right->mv_data) + offset, + sizeof(T) + ); + } +} // lmdb diff --git a/src/lmdb/value_stream.cpp b/src/lmdb/value_stream.cpp new file mode 100644 index 00000000000..604140e4778 --- /dev/null +++ b/src/lmdb/value_stream.cpp @@ -0,0 +1,74 @@ +// Copyright (c) 2018, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#include "value_stream.h" + +#include + +#include "common/expect.h" +#include "lmdb/error.h" +#include "lmdb/util.h" + +namespace lmdb +{ + namespace stream + { + mdb_size_t count(MDB_cursor* cur) + { + mdb_size_t out = 0; + if (cur) + { + const int rc = mdb_cursor_count(cur, &out); + if (rc) + MONERO_THROW(lmdb::error(rc), "mdb_cursor_count"); + } + return out; + } + + std::pair, epee::span> + get(MDB_cursor& cur, MDB_cursor_op op, std::size_t key, std::size_t value) + { + MDB_val key_bytes{}; + MDB_val value_bytes{}; + const int rc = mdb_cursor_get(&cur, &key_bytes, &value_bytes, op); + if (rc) + { + if (rc == MDB_NOTFOUND) + return {}; + MONERO_THROW(lmdb::error(rc), "mdb_cursor_get"); + } + + if (key && key != key_bytes.mv_size) + MONERO_THROW(lmdb::error(MDB_BAD_VALSIZE), "mdb_cursor_get key"); + + if (value && (value_bytes.mv_size % value != 0 || value_bytes.mv_size == 0)) + MONERO_THROW(lmdb::error(MDB_BAD_VALSIZE), "mdb_cursor_get value"); + + return {lmdb::to_byte_span(key_bytes), lmdb::to_byte_span(value_bytes)}; + } + } +} + diff --git a/src/lmdb/value_stream.h b/src/lmdb/value_stream.h new file mode 100644 index 00000000000..01090aa67f8 --- /dev/null +++ b/src/lmdb/value_stream.h @@ -0,0 +1,287 @@ +// Copyright (c) 2018, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "span.h" + +namespace lmdb +{ + namespace stream + { + /* + \throw std::system_error if unexpected LMDB error. + \return 0 if `cur == nullptr`, otherwise count of values at current key. + */ + mdb_size_t count(MDB_cursor* cur); + + /*! + Calls `mdb_cursor_get` and does some error checking. + + \param cur is given to `mdb_cursor_get` without modification. + \param op is passed to `mdb_cursor_get` without modification. + \param key expected key size or 0 to skip key size check. + \param value expected value size or 0 to skip value size check. + + \throw std::system_error if `key != 0` and `key_.mv_size != key`. + \throw std::system_error if `value != 0` and `value_.mv_size != value`. + \throw std::system_error if `mdb_cursor_get` returns any error + other than `MDB_NOTFOUND`. + + \return {key bytes, value bytes} or two empty spans if `MDB_NOTFOUND`. + */ + std::pair, epee::span> + get(MDB_cursor& cur, MDB_cursor_op op, std::size_t key, std::size_t value); + } + + /*! + An InputIterator for a fixed-sized LMDB value at a specific key. + + \tparam T The value type at the specific key. + \tparam F The value type being returned when dereferenced. + \tparam offset to `F` within `T`. + + \note This meets requirements for an InputIterator only. The iterator + can only be incremented and dereferenced. All copies of an iterator + share the same LMDB cursor, and therefore incrementing any copy will + change the cursor state for all (incrementing an iterator will + invalidate all prior copies of the iterator). Usage is identical + to `std::istream_iterator`. + */ + template + class value_iterator + { + MDB_cursor* cur; + epee::span values; + + void increment() + { + values.remove_prefix(sizeof(T)); + if (values.empty() && cur) + values = lmdb::stream::get(*cur, MDB_NEXT_DUP, 0, sizeof(T)).second; + } + + public: + using value_type = F; + using reference = value_type; + using pointer = void; + using difference_type = std::size_t; + using iterator_category = std::input_iterator_tag; + + //! Construct an "end" iterator. + value_iterator() noexcept + : cur(nullptr), values() + {} + + /*! + \param cur Iterate over values starting at this cursor position. + \throw std::system_error if unexpected LMDB error. This can happen + if `cur` is invalid. + */ + value_iterator(MDB_cursor* cur) + : cur(cur), values() + { + if (cur) + values = lmdb::stream::get(*cur, MDB_GET_CURRENT, 0, sizeof(T)).second; + } + + value_iterator(value_iterator const&) = default; + ~value_iterator() = default; + value_iterator& operator=(value_iterator const&) = default; + + //! \return True if `this` is one-past the last value. + bool is_end() const noexcept { return values.empty(); } + + //! \return True iff `rhs` is referencing `this` value. + bool equal(value_iterator const& rhs) const noexcept + { + return + (values.empty() && rhs.values.empty()) || + values.data() == rhs.values.data(); + } + + //! Invalidates all prior copies of the iterator. + value_iterator& operator++() + { + increment(); + return *this; + } + + //! \return A copy that is already invalidated, ignore + value_iterator operator++(int) + { + value_iterator out{*this}; + increment(); + return out; + } + + /*! + Get a specific field within `F`. Default behavior is to return + the entirety of `U`, despite the filtering logic of `operator*`. + + \pre `!is_end()` + + \tparam U must match `T`, used for `MONERO_FIELD` sanity checking. + \tparam G field type to extract from the value + \tparam uoffset to `G` type, or `0` when `std::is_same()`. + + \return The field `G`, at `uoffset` within `U`. + */ + template + G get_value() const noexcept + { + static_assert(std::is_same(), "bad MONERO_FIELD usage?"); + static_assert(std::is_pod(), "value type must be pod"); + static_assert(std::is_pod(), "field type must be pod"); + static_assert(sizeof(G) + uoffset <= sizeof(U), "bad field and/or offset"); + assert(sizeof(G) + uoffset <= values.size()); + assert(!is_end()); + + G value; + std::memcpy(std::addressof(value), values.data() + uoffset, sizeof(value)); + return value; + } + + //! \pre `!is_end()` \return The field `F`, at `offset`, within `T`. + value_type operator*() const noexcept { return get_value(); } + }; + + /*! + C++ wrapper for a LMDB read-only cursor on a fixed-sized value `T`. + + \tparam T value type being stored by each record. + \tparam D cleanup functor for the cursor; usually unique per db/table. + */ + template + class value_stream + { + std::unique_ptr cur; + public: + + //! Take ownership of `cur` without changing position. `nullptr` valid. + explicit value_stream(std::unique_ptr cur) + : cur(std::move(cur)) + {} + + value_stream(value_stream&&) = default; + value_stream(value_stream const&) = delete; + ~value_stream() = default; + value_stream& operator=(value_stream&&) = default; + value_stream& operator=(value_stream const&) = delete; + + /*! + Give up ownership of the cursor. `count()`, `make_iterator()` and + `make_range()` can still be invoked, but return the empty set. + + \return Currently owned LMDB cursor. + */ + std::unique_ptr give_cursor() noexcept + { + return {std::move(cur)}; + } + + /*! + Place the stream back at the first value. Newly created iterators + will start at the first value again. + + \note Invalidates all current iterators from `this`, including + those created with `make_iterator` or `make_range`. + */ + void reset() + { + if (cur) + lmdb::stream::get(*cur, MDB_FIRST_DUP, 0, 0); + } + + /*! + \throw std::system_error if LMDB has unexpected errors. + \return Number of values at this key. + */ + std::size_t count() const + { + return lmdb::stream::count(cur.get()); + } + + /*! + Return a C++ iterator over database values from current cursor + position that will reach `.is_end()` after the last duplicate key + record. Calling `make_iterator()` will return an iterator whose + `operator*` will return entire value (`T`). + `make_iterator()` will return an + iterator whose `operator*` will return a `decltype(account.id)` + object - the other fields in the struct `account` are never copied + from the database. + + \throw std::system_error if LMDB has unexpected errors. + \return C++ iterator starting at current cursor position. + */ + template + value_iterator make_iterator() const + { + static_assert(std::is_same(), "was MONERO_FIELD used with wrong type?"); + return {cur.get()}; + } + + /*! + Return a range from current cursor position until last duplicate + key record. Useful in for-each range loops or in templated code + expecting a range of elements. Calling `make_range()` will return + a range of `T` objects. `make_range()` + will return a range of `decltype(account.id)` objects - the other + fields in the struct `account` are never copied from the database. + + \throw std::system_error if LMDB has unexpected errors. + \return An InputIterator range over values at cursor position. + */ + template + boost::iterator_range> make_range() const + { + return {make_iterator(), value_iterator{}}; + } + }; + + template + inline + bool operator==(value_iterator const& lhs, value_iterator const& rhs) noexcept + { + return lhs.equal(rhs); + } + + template + inline + bool operator!=(value_iterator const& lhs, value_iterator const& rhs) noexcept + { + return !lhs.equal(rhs); + } +} // lmdb + diff --git a/src/mnemonics/CMakeLists.txt b/src/mnemonics/CMakeLists.txt index e3836bccae2..acb9a43385f 100644 --- a/src/mnemonics/CMakeLists.txt +++ b/src/mnemonics/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2014-2018, The Monero Project +# Copyright (c) 2014-2019, The Monero Project # # All rights reserved. # diff --git a/src/mnemonics/chinese_simplified.h b/src/mnemonics/chinese_simplified.h index 1ae8c89d6a7..ff035dde050 100644 --- a/src/mnemonics/chinese_simplified.h +++ b/src/mnemonics/chinese_simplified.h @@ -21,7 +21,7 @@ // OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION // WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. // -// Code surrounding the word list is Copyright (c) 2014-2018, The Monero Project +// Code surrounding the word list is Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -72,7 +72,10 @@ namespace Language class Chinese_Simplified: public Base { public: - Chinese_Simplified(): Base("简体中文 (中国)", "Chinese (simplified)", std::vector({ + Chinese_Simplified(): Base("简体中文 (中国)", "Chinese (simplified)", {}, 1) + { + static constexpr const char * const words[NWORDS] = + { "的", "一", "是", @@ -1699,8 +1702,8 @@ namespace Language "秒", "浙", "貌" - }), 1) - { + }; + set_words(words); populate_maps(); } }; diff --git a/src/mnemonics/dutch.h b/src/mnemonics/dutch.h index c9806f4509e..14ba56a7ecf 100644 --- a/src/mnemonics/dutch.h +++ b/src/mnemonics/dutch.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -49,7 +49,10 @@ namespace Language class Dutch: public Base { public: - Dutch(): Base("Nederlands", "Dutch", std::vector({ + Dutch(): Base("Nederlands", "Dutch", {}, 4) + { + static constexpr const char * const words[NWORDS] = + { "aalglad", "aalscholver", "aambeeld", @@ -1676,8 +1679,8 @@ namespace Language "zwiep", "zwijmel", "zworen" - }), 4) - { + }; + set_words(words); populate_maps(); } }; diff --git a/src/mnemonics/electrum-words.cpp b/src/mnemonics/electrum-words.cpp index 6a2a3e0c46a..2dd40cc9a44 100644 --- a/src/mnemonics/electrum-words.cpp +++ b/src/mnemonics/electrum-words.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -37,19 +37,14 @@ */ #include -#include -#include #include #include #include -#include -#include "crypto/crypto.h" // for declaration of crypto::secret_key -#include +#include "wipeable_string.h" +#include "misc_language.h" +#include "int-util.h" #include "mnemonics/electrum-words.h" -#include -#include #include -#include #include "chinese_simplified.h" #include "english.h" @@ -67,11 +62,22 @@ #include "language_base.h" #include "singleton.h" +#undef MONERO_DEFAULT_LOG_CATEGORY +#define MONERO_DEFAULT_LOG_CATEGORY "mnemonic" + +namespace crypto +{ + namespace ElectrumWords + { + std::vector get_language_list(); + } +} + namespace { - uint32_t create_checksum_index(const std::vector &word_list, - uint32_t unique_prefix_length); - bool checksum_test(std::vector seed, uint32_t unique_prefix_length); + uint32_t create_checksum_index(const std::vector &word_list, + const Language::Base *language); + bool checksum_test(std::vector seed, const Language::Base *language); /*! * \brief Finds the word list that contains the seed words and puts the indices @@ -82,7 +88,7 @@ namespace * \param language Language instance pointer to write to after it is found. * \return true if all the words were present in some language false if not. */ - bool find_seed_language(const std::vector &seed, + bool find_seed_language(const std::vector &seed, bool has_checksum, std::vector &matched_indices, Language::Base **language) { // If there's a new language added, add an instance of it here. @@ -103,17 +109,19 @@ namespace }); Language::Base *fallback = NULL; + std::vector::const_iterator it2; + matched_indices.reserve(seed.size()); + // Iterate through all the languages and find a match for (std::vector::iterator it1 = language_instances.begin(); it1 != language_instances.end(); it1++) { - const std::unordered_map &word_map = (*it1)->get_word_map(); - const std::unordered_map &trimmed_word_map = (*it1)->get_trimmed_word_map(); + const std::unordered_map &word_map = (*it1)->get_word_map(); + const std::unordered_map &trimmed_word_map = (*it1)->get_trimmed_word_map(); // To iterate through seed words - std::vector::const_iterator it2; bool full_match = true; - std::string trimmed_word; + epee::wipeable_string trimmed_word; // Iterate through all the words and see if they're all present for (it2 = seed.begin(); it2 != seed.end(); it2++) { @@ -143,7 +151,7 @@ namespace // if we were using prefix only, and we have a checksum, check it now // to avoid false positives due to prefix set being too common if (has_checksum) - if (!checksum_test(seed, (*it1)->get_unique_prefix_length())) + if (!checksum_test(seed, *it1)) { fallback = *it1; full_match = false; @@ -152,9 +160,11 @@ namespace if (full_match) { *language = *it1; + MINFO("Full match for language " << (*language)->get_english_language_name()); return true; } // Some didn't match. Clear the index array. + memwipe(matched_indices.data(), matched_indices.size() * sizeof(matched_indices[0])); matched_indices.clear(); } @@ -164,9 +174,12 @@ namespace if (fallback) { *language = fallback; + MINFO("Fallback match for language " << (*language)->get_english_language_name()); return true; } + MINFO("No match found"); + memwipe(matched_indices.data(), matched_indices.size() * sizeof(matched_indices[0])); return false; } @@ -176,25 +189,25 @@ namespace * \param unique_prefix_length the prefix length of each word to use for checksum * \return Checksum index */ - uint32_t create_checksum_index(const std::vector &word_list, - uint32_t unique_prefix_length) + uint32_t create_checksum_index(const std::vector &word_list, + const Language::Base *language) { - std::string trimmed_words = ""; + epee::wipeable_string trimmed_words = "", word; - for (std::vector::const_iterator it = word_list.begin(); it != word_list.end(); it++) + const auto &word_map = language->get_word_map(); + const auto &trimmed_word_map = language->get_trimmed_word_map(); + const uint32_t unique_prefix_length = language->get_unique_prefix_length(); + for (std::vector::const_iterator it = word_list.begin(); it != word_list.end(); it++) { - if (it->length() > unique_prefix_length) - { - trimmed_words += Language::utf8prefix(*it, unique_prefix_length); - } - else - { - trimmed_words += *it; - } + word = Language::utf8prefix(*it, unique_prefix_length); + auto it2 = trimmed_word_map.find(word); + if (it2 == trimmed_word_map.end()) + throw std::runtime_error("Word \"" + std::string(word.data(), word.size()) + "\" not found in trimmed word map in " + language->get_english_language_name()); + trimmed_words += it2->first; } boost::crc_32_type result; result.process_bytes(trimmed_words.data(), trimmed_words.length()); - return result.checksum() % crypto::ElectrumWords::seed_length; + return result.checksum() % word_list.size(); } /*! @@ -203,21 +216,26 @@ namespace * \param unique_prefix_length the prefix length of each word to use for checksum * \return True if the test passed false if not. */ - bool checksum_test(std::vector seed, uint32_t unique_prefix_length) + bool checksum_test(std::vector seed, const Language::Base *language) { if (seed.empty()) return false; // The last word is the checksum. - std::string last_word = seed.back(); + epee::wipeable_string last_word = seed.back(); seed.pop_back(); - std::string checksum = seed[create_checksum_index(seed, unique_prefix_length)]; + const uint32_t unique_prefix_length = language->get_unique_prefix_length(); + + auto idx = create_checksum_index(seed, language); + epee::wipeable_string checksum = seed[idx]; - std::string trimmed_checksum = checksum.length() > unique_prefix_length ? Language::utf8prefix(checksum, unique_prefix_length) : + epee::wipeable_string trimmed_checksum = checksum.length() > unique_prefix_length ? Language::utf8prefix(checksum, unique_prefix_length) : checksum; - std::string trimmed_last_word = last_word.length() > unique_prefix_length ? Language::utf8prefix(last_word, unique_prefix_length) : + epee::wipeable_string trimmed_last_word = last_word.length() > unique_prefix_length ? Language::utf8prefix(last_word, unique_prefix_length) : last_word; - return trimmed_checksum == trimmed_last_word; + bool ret = Language::WordEqual()(trimmed_checksum, trimmed_last_word); + MINFO("Checksum is " << (ret ? "valid" : "invalid")); + return ret; } } @@ -244,16 +262,18 @@ namespace crypto * \param language_name Language of the seed as found gets written here. * \return false if not a multiple of 3 words, or if word is not in the words list */ - bool words_to_bytes(std::string words, std::string& dst, size_t len, bool duplicate, + bool words_to_bytes(const epee::wipeable_string &words, epee::wipeable_string& dst, size_t len, bool duplicate, std::string &language_name) { - std::vector seed; + std::vector seed; - boost::algorithm::trim(words); - boost::split(seed, words, boost::is_any_of(" "), boost::token_compress_on); + words.split(seed); if (len % 4) + { + MERROR("Invalid seed: not a multiple of 4"); return false; + } bool has_checksum = true; if (len) @@ -263,6 +283,7 @@ namespace crypto if (seed.size() != expected/2 && seed.size() != expected && seed.size() != expected + 1) { + MERROR("Invalid seed: unexpected number of words"); return false; } @@ -271,9 +292,11 @@ namespace crypto } std::vector matched_indices; + auto wiper = epee::misc_utils::create_scope_leave_handler([&](){memwipe(matched_indices.data(), matched_indices.size() * sizeof(matched_indices[0]));}); Language::Base *language; if (!find_seed_language(seed, has_checksum, matched_indices, &language)) { + MERROR("Invalid seed: language not found"); return false; } language_name = language->get_language_name(); @@ -281,9 +304,10 @@ namespace crypto if (has_checksum) { - if (!checksum_test(seed, language->get_unique_prefix_length())) + if (!checksum_test(seed, language)) { // Checksum fail + MERROR("Invalid seed: invalid checksum"); return false; } seed.pop_back(); @@ -291,29 +315,32 @@ namespace crypto for (unsigned int i=0; i < seed.size() / 3; i++) { - uint32_t val; - uint32_t w1, w2, w3; - w1 = matched_indices[i*3]; - w2 = matched_indices[i*3 + 1]; - w3 = matched_indices[i*3 + 2]; + uint32_t w[4]; + w[1] = matched_indices[i*3]; + w[2] = matched_indices[i*3 + 1]; + w[3] = matched_indices[i*3 + 2]; - val = w1 + word_list_length * (((word_list_length - w1) + w2) % word_list_length) + - word_list_length * word_list_length * (((word_list_length - w2) + w3) % word_list_length); + w[0]= w[1] + word_list_length * (((word_list_length - w[1]) + w[2]) % word_list_length) + + word_list_length * word_list_length * (((word_list_length - w[2]) + w[3]) % word_list_length); - if (!(val % word_list_length == w1)) return false; + if (!(w[0]% word_list_length == w[1])) + { + memwipe(w, sizeof(w)); + MERROR("Invalid seed: mumble mumble"); + return false; + } - dst.append((const char*)&val, 4); // copy 4 bytes to position + w[0] = SWAP32LE(w[0]); + dst.append((const char*)&w[0], 4); // copy 4 bytes to position + memwipe(w, sizeof(w)); } if (len > 0 && duplicate) { const size_t expected = len * 3 / 32; - std::string wlist_copy = words; if (seed.size() == expected/2) { - dst.append(dst); // if electrum 12-word seed, duplicate - wlist_copy += ' '; - wlist_copy += words; + dst.append(dst.data(), dst.size()); // if electrum 12-word seed, duplicate } } @@ -327,14 +354,20 @@ namespace crypto * \param language_name Language of the seed as found gets written here. * \return false if not a multiple of 3 words, or if word is not in the words list */ - bool words_to_bytes(std::string words, crypto::secret_key& dst, + bool words_to_bytes(const epee::wipeable_string &words, crypto::secret_key& dst, std::string &language_name) { - std::string s; + epee::wipeable_string s; if (!words_to_bytes(words, s, sizeof(dst), true, language_name)) + { + MERROR("Invalid seed: failed to convert words to bytes"); return false; + } if (s.size() != sizeof(dst)) + { + MERROR("Invalid seed: wrong output size"); return false; + } dst = *(const crypto::secret_key*)s.data(); return true; } @@ -346,100 +379,57 @@ namespace crypto * \param language_name Seed language name * \return true if successful false if not. Unsuccessful if wrong key size. */ - bool bytes_to_words(const char *src, size_t len, std::string& words, + bool bytes_to_words(const char *src, size_t len, epee::wipeable_string& words, const std::string &language_name) { if (len % 4 != 0 || len == 0) return false; - Language::Base *language; - if (language_name == "English") + const Language::Base *language = NULL; + const std::vector language_list = crypto::ElectrumWords::get_language_list(); + for (const Language::Base *l: language_list) { - language = Language::Singleton::instance(); + if (language_name == l->get_language_name() || language_name == l->get_english_language_name()) + language = l; } - else if (language_name == "Nederlands") - { - language = Language::Singleton::instance(); - } - else if (language_name == "Français") - { - language = Language::Singleton::instance(); - } - else if (language_name == "Español") - { - language = Language::Singleton::instance(); - } - else if (language_name == "Português") - { - language = Language::Singleton::instance(); - } - else if (language_name == "日本語") - { - language = Language::Singleton::instance(); - } - else if (language_name == "Italiano") - { - language = Language::Singleton::instance(); - } - else if (language_name == "Deutsch") - { - language = Language::Singleton::instance(); - } - else if (language_name == "русский язык") - { - language = Language::Singleton::instance(); - } - else if (language_name == "简体中文 (中国)") - { - language = Language::Singleton::instance(); - } - else if (language_name == "Esperanto") - { - language = Language::Singleton::instance(); - } - else if (language_name == "Lojban") - { - language = Language::Singleton::instance(); - } - else + if (!language) { return false; } const std::vector &word_list = language->get_word_list(); // To store the words for random access to add the checksum word later. - std::vector words_store; + std::vector words_store; uint32_t word_list_length = word_list.size(); // 4 bytes -> 3 words. 8 digits base 16 -> 3 digits base 1626 - for (unsigned int i=0; i < len/4; i++, words += ' ') + for (unsigned int i=0; i < len/4; i++, words.push_back(' ')) { - uint32_t w1, w2, w3; - - uint32_t val; + uint32_t w[4]; - memcpy(&val, src + (i * 4), 4); + w[0] = SWAP32LE(*(const uint32_t*)(src + (i * 4))); - w1 = val % word_list_length; - w2 = ((val / word_list_length) + w1) % word_list_length; - w3 = (((val / word_list_length) / word_list_length) + w2) % word_list_length; + w[1] = w[0] % word_list_length; + w[2] = ((w[0] / word_list_length) + w[1]) % word_list_length; + w[3] = (((w[0] / word_list_length) / word_list_length) + w[2]) % word_list_length; - words += word_list[w1]; + words += word_list[w[1]]; words += ' '; - words += word_list[w2]; + words += word_list[w[2]]; words += ' '; - words += word_list[w3]; + words += word_list[w[3]]; + + words_store.push_back(word_list[w[1]]); + words_store.push_back(word_list[w[2]]); + words_store.push_back(word_list[w[3]]); - words_store.push_back(word_list[w1]); - words_store.push_back(word_list[w2]); - words_store.push_back(word_list[w3]); + memwipe(w, sizeof(w)); } - words.pop_back(); - words += (' ' + words_store[create_checksum_index(words_store, language->get_unique_prefix_length())]); + words += words_store[create_checksum_index(words_store, language)]; return true; } - bool bytes_to_words(const crypto::secret_key& src, std::string& words, + bool bytes_to_words(const crypto::secret_key& src, epee::wipeable_string& words, const std::string &language_name) { return bytes_to_words(src.data, sizeof(src), words, language_name); @@ -483,11 +473,10 @@ namespace crypto * \param seed The seed to check (a space delimited concatenated word list) * \return true if the seed passed is a old style seed false if not. */ - bool get_is_old_style_seed(std::string seed) + bool get_is_old_style_seed(const epee::wipeable_string &seed) { - std::vector word_list; - boost::algorithm::trim(seed); - boost::split(word_list, seed, boost::is_any_of(" "), boost::token_compress_on); + std::vector word_list; + seed.split(word_list); return word_list.size() != (seed_length + 1); } diff --git a/src/mnemonics/electrum-words.h b/src/mnemonics/electrum-words.h index 856edb92a41..9aa727e4522 100644 --- a/src/mnemonics/electrum-words.h +++ b/src/mnemonics/electrum-words.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -41,9 +41,10 @@ #include #include -#include #include "crypto/crypto.h" // for declaration of crypto::secret_key +namespace epee { class wipeable_string; } + /*! * \namespace crypto * @@ -70,7 +71,7 @@ namespace crypto * \param language_name Language of the seed as found gets written here. * \return false if not a multiple of 3 words, or if word is not in the words list */ - bool words_to_bytes(std::string words, std::string& dst, size_t len, bool duplicate, + bool words_to_bytes(const epee::wipeable_string &words, epee::wipeable_string& dst, size_t len, bool duplicate, std::string &language_name); /*! * \brief Converts seed words to bytes (secret key). @@ -79,7 +80,7 @@ namespace crypto * \param language_name Language of the seed as found gets written here. * \return false if not a multiple of 3 words, or if word is not in the words list */ - bool words_to_bytes(std::string words, crypto::secret_key& dst, + bool words_to_bytes(const epee::wipeable_string &words, crypto::secret_key& dst, std::string &language_name); /*! @@ -90,7 +91,7 @@ namespace crypto * \param language_name Seed language name * \return true if successful false if not. Unsuccessful if wrong key size. */ - bool bytes_to_words(const char *src, size_t len, std::string& words, + bool bytes_to_words(const char *src, size_t len, epee::wipeable_string& words, const std::string &language_name); /*! @@ -100,7 +101,7 @@ namespace crypto * \param language_name Seed language name * \return true if successful false if not. Unsuccessful if wrong key size. */ - bool bytes_to_words(const crypto::secret_key& src, std::string& words, + bool bytes_to_words(const crypto::secret_key& src, epee::wipeable_string& words, const std::string &language_name); /*! @@ -115,7 +116,7 @@ namespace crypto * \param seed The seed to check (a space delimited concatenated word list) * \return true if the seed passed is a old style seed false if not. */ - bool get_is_old_style_seed(std::string seed); + bool get_is_old_style_seed(const epee::wipeable_string &seed); /*! * \brief Returns the name of a language in English diff --git a/src/mnemonics/english.h b/src/mnemonics/english.h index ee087674de3..677b70052f5 100644 --- a/src/mnemonics/english.h +++ b/src/mnemonics/english.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -49,7 +49,10 @@ namespace Language class English: public Base { public: - English(): Base("English", "English", std::vector({ + English(): Base("English", "English", {}, 3) + { + static constexpr const char * const words[NWORDS] = + { "abbey", "abducts", "ability", @@ -1676,8 +1679,8 @@ namespace Language "zombie", "zones", "zoom" - }), 3) - { + }; + set_words(words); populate_maps(); } }; diff --git a/src/mnemonics/english_old.h b/src/mnemonics/english_old.h index b3149164607..179fc45ec95 100644 --- a/src/mnemonics/english_old.h +++ b/src/mnemonics/english_old.h @@ -1,6 +1,6 @@ // Word list originally created as part of the Electrum project, Copyright (C) 2014 Thomas Voegtlin // -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -51,7 +51,10 @@ namespace Language class EnglishOld: public Base { public: - EnglishOld(): Base("EnglishOld", "English (old)", std::vector({ + EnglishOld(): Base("EnglishOld", "English (old)", {}, 4) + { + static constexpr const char * const words[NWORDS] = + { "like", "just", "love", @@ -1678,8 +1681,8 @@ namespace Language "unseen", "weapon", "weary" - }), 4) - { + }; + set_words(words); populate_maps(ALLOW_DUPLICATE_PREFIXES | ALLOW_SHORT_WORDS); } }; diff --git a/src/mnemonics/esperanto.h b/src/mnemonics/esperanto.h index cb377a58ea3..ef63a2235ab 100644 --- a/src/mnemonics/esperanto.h +++ b/src/mnemonics/esperanto.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -37,7 +37,7 @@ * Sources: * Baza Radikaro Oficiala * Reta Vortaro (http://www.reta-vortaro.de/revo/) - * Esperanto Panorama - Esperanto-English Dictionary (http://www.esperanto-panorama.net/vortaro/eoen.htm) + * Esperanto Panorama - Esperanto-English Dictionary (https://www.esperanto-panorama.net/vortaro/eoen.htm) * ESPDIC - Paul Denisowski (http://www.denisowski.org/Esperanto/ESPDIC/espdic.txt) */ @@ -58,7 +58,10 @@ namespace Language class Esperanto: public Base { public: - Esperanto(): Base("Esperanto", "Esperanto", std::vector({ + Esperanto(): Base("Esperanto", "Esperanto", {}, 4) + { + static constexpr const char * const words[NWORDS] = + { "abako", "abdiki", "abelo", @@ -1685,8 +1688,8 @@ namespace Language "zorgi", "zukino", "zumilo", - }), 4) - { + }; + set_words(words); populate_maps(); } }; diff --git a/src/mnemonics/french.h b/src/mnemonics/french.h index 7eaf456501b..85925aa530b 100644 --- a/src/mnemonics/french.h +++ b/src/mnemonics/french.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -49,7 +49,10 @@ namespace Language class French: public Base { public: - French(): Base("Français", "French", std::vector({ + French(): Base("Français", "French", {}, 4) + { + static constexpr const char * const words[NWORDS] = + { "abandon", "abattre", "aboi", @@ -1676,8 +1679,8 @@ namespace Language "zinc", "zone", "zoom" - }), 4) - { + }; + set_words(words); populate_maps(); } }; diff --git a/src/mnemonics/german.h b/src/mnemonics/german.h index 8eff435233c..40116970c37 100644 --- a/src/mnemonics/german.h +++ b/src/mnemonics/german.h @@ -1,6 +1,6 @@ // Word list created by Monero contributor Shrikez // -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -51,7 +51,10 @@ namespace Language class German: public Base { public: - German(): Base("Deutsch", "German", std::vector({ + German(): Base("Deutsch", "German", {}, 4) + { + static constexpr const char * const words[NWORDS] = + { "Abakus", "Abart", "abbilden", @@ -1678,8 +1681,8 @@ namespace Language "Zündung", "Zweck", "Zyklop" - }), 4) - { + }; + set_words(words); populate_maps(); } }; diff --git a/src/mnemonics/italian.h b/src/mnemonics/italian.h index d5ecb74f465..204fdcb7813 100644 --- a/src/mnemonics/italian.h +++ b/src/mnemonics/italian.h @@ -1,6 +1,6 @@ // Word list created by Monero contributor Shrikez // -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -51,7 +51,10 @@ namespace Language class Italian: public Base { public: - Italian(): Base("Italiano", "Italian", std::vector({ + Italian(): Base("Italiano", "Italian", {}, 4) + { + static constexpr const char * const words[NWORDS] = + { "abbinare", "abbonato", "abisso", @@ -1678,8 +1681,8 @@ namespace Language "zolfo", "zombie", "zucchero" - }), 4) - { + }; + set_words(words); populate_maps(); } }; diff --git a/src/mnemonics/japanese.h b/src/mnemonics/japanese.h index f3b3e4924a3..6213c4976f9 100644 --- a/src/mnemonics/japanese.h +++ b/src/mnemonics/japanese.h @@ -21,7 +21,7 @@ // OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION // WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. // -// Code surrounding the word list is Copyright (c) 2014-2018, The Monero Project +// Code surrounding the word list is Copyright (c) 2014-2019, The Monero Project // All rights reserved. // // Redistribution and use in source and binary forms, with or without modification, are @@ -71,7 +71,10 @@ namespace Language class Japanese: public Base { public: - Japanese(): Base("日本語", "Japanese", std::vector({ + Japanese(): Base("日本語", "Japanese", {}, 3) + { + static constexpr const char * const words[NWORDS] = + { "あいこくしん", "あいさつ", "あいだ", @@ -1698,8 +1701,8 @@ namespace Language "ひさん", "びじゅつかん", "ひしょ" - }), 3) - { + }; + set_words(words); populate_maps(); } }; diff --git a/src/mnemonics/language_base.h b/src/mnemonics/language_base.h index 2b0c37c6bc1..7d2599e9ab5 100644 --- a/src/mnemonics/language_base.h +++ b/src/mnemonics/language_base.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -38,7 +38,9 @@ #include #include #include +#include #include "misc_log_ex.h" +#include "fnv1.h" /*! * \namespace Language @@ -53,19 +55,110 @@ namespace Language * \param count How many characters to return. * \return A string consisting of the first count characters in s. */ - inline std::string utf8prefix(const std::string &s, size_t count) + template + inline T utf8prefix(const T &s, size_t count) { - std::string prefix = ""; - const char *ptr = s.c_str(); - while (count-- && *ptr) + T prefix = ""; + size_t avail = s.size(); + const char *ptr = s.data(); + while (count-- && avail--) { prefix += *ptr++; - while (((*ptr) & 0xc0) == 0x80) + while (avail && ((*ptr) & 0xc0) == 0x80) + { prefix += *ptr++; + --avail; + } } return prefix; } + template + inline T utf8canonical(const T &s) + { + T sc = ""; + size_t avail = s.size(); + const char *ptr = s.data(); + wint_t cp = 0; + int bytes = 1; + char wbuf[8], *wptr; + while (avail--) + { + if ((*ptr & 0x80) == 0) + { + cp = *ptr++; + bytes = 1; + } + else if ((*ptr & 0xe0) == 0xc0) + { + if (avail < 1) + throw std::runtime_error("Invalid UTF-8"); + cp = (*ptr++ & 0x1f) << 6; + cp |= *ptr++ & 0x3f; + --avail; + bytes = 2; + } + else if ((*ptr & 0xf0) == 0xe0) + { + if (avail < 2) + throw std::runtime_error("Invalid UTF-8"); + cp = (*ptr++ & 0xf) << 12; + cp |= (*ptr++ & 0x3f) << 6; + cp |= *ptr++ & 0x3f; + avail -= 2; + bytes = 3; + } + else if ((*ptr & 0xf8) == 0xf0) + { + if (avail < 3) + throw std::runtime_error("Invalid UTF-8"); + cp = (*ptr++ & 0x7) << 18; + cp |= (*ptr++ & 0x3f) << 12; + cp |= (*ptr++ & 0x3f) << 6; + cp |= *ptr++ & 0x3f; + avail -= 3; + bytes = 4; + } + else + throw std::runtime_error("Invalid UTF-8"); + + cp = std::towlower(cp); + wptr = wbuf; + switch (bytes) + { + case 1: *wptr++ = cp; break; + case 2: *wptr++ = 0xc0 | (cp >> 6); *wptr++ = 0x80 | (cp & 0x3f); break; + case 3: *wptr++ = 0xe0 | (cp >> 12); *wptr++ = 0x80 | ((cp >> 6) & 0x3f); *wptr++ = 0x80 | (cp & 0x3f); break; + case 4: *wptr++ = 0xf0 | (cp >> 18); *wptr++ = 0x80 | ((cp >> 12) & 0x3f); *wptr++ = 0x80 | ((cp >> 6) & 0x3f); *wptr++ = 0x80 | (cp & 0x3f); break; + default: throw std::runtime_error("Invalid UTF-8"); + } + *wptr = 0; + sc += T(wbuf, bytes); + cp = 0; + bytes = 1; + } + return sc; + } + + struct WordHash + { + std::size_t operator()(const epee::wipeable_string &s) const + { + const epee::wipeable_string sc = utf8canonical(s); + return epee::fnv::FNV1a(sc.data(), sc.size()); + } + }; + + struct WordEqual + { + bool operator()(const epee::wipeable_string &s0, const epee::wipeable_string &s1) const + { + const epee::wipeable_string s0c = utf8canonical(s0); + const epee::wipeable_string s1c = utf8canonical(s1); + return s0c == s1c; + } + }; + /*! * \class Base * \brief A base language class which all languages have to inherit from for @@ -78,9 +171,12 @@ namespace Language ALLOW_SHORT_WORDS = 1<<0, ALLOW_DUPLICATE_PREFIXES = 1<<1, }; - const std::vector word_list; /*!< A pointer to the array of words */ - std::unordered_map word_map; /*!< hash table to find word's index */ - std::unordered_map trimmed_word_map; /*!< hash table to find word's trimmed index */ + enum { + NWORDS = 1626 + }; + std::vector word_list; /*!< A pointer to the array of words */ + std::unordered_map word_map; /*!< hash table to find word's index */ + std::unordered_map trimmed_word_map; /*!< hash table to find word's trimmed index */ std::string language_name; /*!< Name of language */ std::string english_language_name; /*!< Name of language */ uint32_t unique_prefix_length; /*!< Number of unique starting characters to trim the wordlist to when matching */ @@ -91,7 +187,7 @@ namespace Language { int ii; std::vector::const_iterator it; - if (word_list.size () != 1626) + if (word_list.size () != NWORDS) throw std::runtime_error("Wrong word list length for " + language_name); for (it = word_list.begin(), ii = 0; it != word_list.end(); it++, ii++) { @@ -103,7 +199,7 @@ namespace Language else throw std::runtime_error("Too short word in " + language_name + " word list: " + *it); } - std::string trimmed; + epee::wipeable_string trimmed; if (it->length() > unique_prefix_length) { trimmed = utf8prefix(*it, unique_prefix_length); @@ -115,9 +211,9 @@ namespace Language if (trimmed_word_map.find(trimmed) != trimmed_word_map.end()) { if (flags & ALLOW_DUPLICATE_PREFIXES) - MWARNING("Duplicate prefix in " << language_name << " word list: " << trimmed); + MWARNING("Duplicate prefix in " << language_name << " word list: " << std::string(trimmed.data(), trimmed.size())); else - throw std::runtime_error("Duplicate prefix in " + language_name + " word list: " + trimmed); + throw std::runtime_error("Duplicate prefix in " + language_name + " word list: " + std::string(trimmed.data(), trimmed.size())); } trimmed_word_map[trimmed] = ii; } @@ -133,6 +229,12 @@ namespace Language virtual ~Base() { } + void set_words(const char * const words[]) + { + word_list.resize(NWORDS); + for (size_t i = 0; i < NWORDS; ++i) + word_list[i] = words[i]; + } /*! * \brief Returns a pointer to the word list. * \return A pointer to the word list. @@ -145,7 +247,7 @@ namespace Language * \brief Returns a pointer to the word map. * \return A pointer to the word map. */ - const std::unordered_map& get_word_map() const + const std::unordered_map& get_word_map() const { return word_map; } @@ -153,7 +255,7 @@ namespace Language * \brief Returns a pointer to the trimmed word map. * \return A pointer to the trimmed word map. */ - const std::unordered_map& get_trimmed_word_map() const + const std::unordered_map& get_trimmed_word_map() const { return trimmed_word_map; } diff --git a/src/mnemonics/lojban.h b/src/mnemonics/lojban.h index 412531a2307..b6baf46c478 100644 --- a/src/mnemonics/lojban.h +++ b/src/mnemonics/lojban.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -35,7 +35,7 @@ /* * Word list authored by: sorpaas * Sources: - * lo gimste jo'u lo ma'oste (http://guskant.github.io/lojbo/gismu-cmavo.html) + * lo gimste jo'u lo ma'oste (https://guskant.github.io/lojbo/gismu-cmavo.html) * N-grams of Lojban corpus (https://mw.lojban.org/papri/N-grams_of_Lojban_corpus) */ @@ -56,7 +56,10 @@ namespace Language class Lojban: public Base { public: - Lojban(): Base("Lojban", "Lojban", std::vector({ + Lojban(): Base("Lojban", "Lojban", {}, 4) + { + static constexpr const char * const words[NWORDS] = + { "backi", "bacru", "badna", @@ -1683,8 +1686,8 @@ namespace Language "noltruti'u", "samtci", "snaxa'a", - }), 4) - { + }; + set_words(words); populate_maps(); } }; diff --git a/src/mnemonics/portuguese.h b/src/mnemonics/portuguese.h index 0195389ceb2..dbc51fb2a31 100644 --- a/src/mnemonics/portuguese.h +++ b/src/mnemonics/portuguese.h @@ -21,7 +21,7 @@ // OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION // WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. // -// Code surrounding the word list is Copyright (c) 2014-2018, The Monero Project +// Code surrounding the word list is Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -72,7 +72,10 @@ namespace Language class Portuguese: public Base { public: - Portuguese(): Base("Português", "Portuguese", std::vector({ + Portuguese(): Base("Português", "Portuguese", {}, 4) + { + static constexpr const char * const words[NWORDS] = + { "abaular", "abdominal", "abeto", @@ -1699,8 +1702,8 @@ namespace Language "zeloso", "zenite", "zumbi" - }), 4) - { + }; + set_words(words); populate_maps(); } }; diff --git a/src/mnemonics/russian.h b/src/mnemonics/russian.h index d5dd556ef97..d36942a9b5f 100644 --- a/src/mnemonics/russian.h +++ b/src/mnemonics/russian.h @@ -1,6 +1,6 @@ // Word list created by Monero contributor sammy007 // -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -51,7 +51,10 @@ namespace Language class Russian: public Base { public: - Russian(): Base("русский язык", "Russian", std::vector({ + Russian(): Base("русский язык", "Russian", {}, 4) + { + static constexpr const char * const words[NWORDS] = + { "абажур", "абзац", "абонент", @@ -1678,8 +1681,8 @@ namespace Language "яхта", "ячейка", "ящик" - }), 4) - { + }; + set_words(words); populate_maps(); } }; diff --git a/src/mnemonics/singleton.h b/src/mnemonics/singleton.h index a15c2b9ae02..ffe3fe4debc 100644 --- a/src/mnemonics/singleton.h +++ b/src/mnemonics/singleton.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -50,8 +50,8 @@ namespace Language class Singleton { Singleton() {} - Singleton(Singleton &s) {} - Singleton& operator=(const Singleton&) {} + Singleton(Singleton &s) = delete; + Singleton& operator=(const Singleton&) = delete; public: static T* instance() { diff --git a/src/mnemonics/spanish.h b/src/mnemonics/spanish.h index 51f38fede7a..a0548577557 100644 --- a/src/mnemonics/spanish.h +++ b/src/mnemonics/spanish.h @@ -21,7 +21,7 @@ // OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION // WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. // -// Code surrounding the word list is Copyright (c) 2014-2018, The Monero Project +// Code surrounding the word list is Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -72,7 +72,10 @@ namespace Language class Spanish: public Base { public: - Spanish(): Base("Español", "Spanish", std::vector({ + Spanish(): Base("Español", "Spanish", {}, 4) + { + static constexpr const char * const words[NWORDS] = + { "ábaco", "abdomen", "abeja", @@ -1699,8 +1702,8 @@ namespace Language "risa", "ritmo", "rito" - }), 4) - { + }; + set_words(words); populate_maps(ALLOW_SHORT_WORDS); } }; diff --git a/src/multisig/CMakeLists.txt b/src/multisig/CMakeLists.txt index a770c6dc5c5..4631b75ef63 100644 --- a/src/multisig/CMakeLists.txt +++ b/src/multisig/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2017-2018, The Monero Project +# Copyright (c) 2017-2019, The Monero Project # # All rights reserved. # diff --git a/src/multisig/multisig.cpp b/src/multisig/multisig.cpp index d85c477720d..14df4d55495 100644 --- a/src/multisig/multisig.cpp +++ b/src/multisig/multisig.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2018, The Monero Project +// Copyright (c) 2017-2019, The Monero Project // // All rights reserved. // @@ -47,9 +47,12 @@ namespace cryptonote crypto::secret_key get_multisig_blinded_secret_key(const crypto::secret_key &key) { rct::keyV data; + data.reserve(2); data.push_back(rct::sk2rct(key)); data.push_back(multisig_salt); - return rct::rct2sk(rct::hash_to_scalar(data)); + crypto::secret_key result = rct::rct2sk(rct::hash_to_scalar(data)); + memwipe(&data[0], sizeof(rct::key)); + return result; } //----------------------------------------------------------------- void generate_multisig_N_N(const account_keys &keys, const std::vector &spend_keys, std::vector &multisig_keys, rct::key &spend_skey, rct::key &spend_pkey) @@ -81,6 +84,43 @@ namespace cryptonote } } //----------------------------------------------------------------- + std::vector generate_multisig_derivations(const account_keys &keys, const std::vector &derivations) + { + std::vector multisig_keys; + crypto::secret_key blinded_skey = get_multisig_blinded_secret_key(keys.m_spend_secret_key); + for (const auto &k: derivations) + { + rct::key d = rct::scalarmultKey(rct::pk2rct(k), rct::sk2rct(blinded_skey)); + multisig_keys.push_back(rct::rct2pk(d)); + } + + return multisig_keys; + } + //----------------------------------------------------------------- + crypto::secret_key calculate_multisig_signer_key(const std::vector& multisig_keys) + { + rct::key secret_key = rct::zero(); + for (const auto &k: multisig_keys) + { + sc_add(secret_key.bytes, secret_key.bytes, (const unsigned char*)k.data); + } + + return rct::rct2sk(secret_key); + } + //----------------------------------------------------------------- + std::vector calculate_multisig_keys(const std::vector& derivations) + { + std::vector multisig_keys; + multisig_keys.reserve(derivations.size()); + + for (const auto &k: derivations) + { + multisig_keys.emplace_back(get_multisig_blinded_secret_key(rct::rct2sk(rct::pk2rct(k)))); + } + + return multisig_keys; + } + //----------------------------------------------------------------- crypto::secret_key generate_multisig_view_secret_key(const crypto::secret_key &skey, const std::vector &skeys) { rct::key view_skey = rct::sk2rct(get_multisig_blinded_secret_key(skey)); @@ -89,7 +129,7 @@ namespace cryptonote return rct::rct2sk(view_skey); } //----------------------------------------------------------------- - crypto::public_key generate_multisig_N1_N_spend_public_key(const std::vector &pkeys) + crypto::public_key generate_multisig_M_N_spend_public_key(const std::vector &pkeys) { rct::key spend_public_key = rct::identity(); for (const auto &pk: pkeys) @@ -138,4 +178,9 @@ namespace cryptonote return true; } //----------------------------------------------------------------- + uint32_t multisig_rounds_required(uint32_t participants, uint32_t threshold) + { + CHECK_AND_ASSERT_THROW_MES(participants >= threshold, "participants must be greater or equal than threshold"); + return participants - threshold + 1; + } } diff --git a/src/multisig/multisig.h b/src/multisig/multisig.h index f95611441b1..bc6edbb05d6 100644 --- a/src/multisig/multisig.h +++ b/src/multisig/multisig.h @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2018, The Monero Project +// Copyright (c) 2017-2019, The Monero Project // // All rights reserved. // @@ -41,9 +41,31 @@ namespace cryptonote crypto::secret_key get_multisig_blinded_secret_key(const crypto::secret_key &key); void generate_multisig_N_N(const account_keys &keys, const std::vector &spend_keys, std::vector &multisig_keys, rct::key &spend_skey, rct::key &spend_pkey); void generate_multisig_N1_N(const account_keys &keys, const std::vector &spend_keys, std::vector &multisig_keys, rct::key &spend_skey, rct::key &spend_pkey); + /** + * @brief generate_multisig_derivations performs common DH key derivation. + * Each middle round in M/N scheme is DH exchange of public multisig keys of other participants multiplied by secret spend key of current participant. + * this functions does the following: new multisig key = secret spend * public multisig key + * @param keys - current wallet's keys + * @param derivations - public multisig keys of other participants + * @return new public multisig keys derived from previous round. This data needs to be exchange with other participants + */ + std::vector generate_multisig_derivations(const account_keys &keys, const std::vector &derivations); + crypto::secret_key calculate_multisig_signer_key(const std::vector& derivations); + /** + * @brief calculate_multisig_keys. Calculates secret multisig keys from others' participants ones as follows: mi = H(Mi) + * @param derivations - others' participants public multisig keys. + * @return vector of current wallet's multisig secret keys + */ + std::vector calculate_multisig_keys(const std::vector& derivations); crypto::secret_key generate_multisig_view_secret_key(const crypto::secret_key &skey, const std::vector &skeys); - crypto::public_key generate_multisig_N1_N_spend_public_key(const std::vector &pkeys); + /** + * @brief generate_multisig_M_N_spend_public_key calculates multisig wallet's spend public key by summing all of public multisig keys + * @param pkeys unique public multisig keys + * @return multisig wallet's spend public key + */ + crypto::public_key generate_multisig_M_N_spend_public_key(const std::vector &pkeys); bool generate_multisig_key_image(const account_keys &keys, size_t multisig_key_index, const crypto::public_key& out_key, crypto::key_image& ki); void generate_multisig_LR(const crypto::public_key pkey, const crypto::secret_key &k, crypto::public_key &L, crypto::public_key &R); bool generate_multisig_composite_key_image(const account_keys &keys, const std::unordered_map& subaddresses, const crypto::public_key& out_key, const crypto::public_key &tx_public_key, const std::vector& additional_tx_public_keys, size_t real_output_index, const std::vector &pkis, crypto::key_image &ki); + uint32_t multisig_rounds_required(uint32_t participants, uint32_t threshold); } diff --git a/src/net/CMakeLists.txt b/src/net/CMakeLists.txt new file mode 100644 index 00000000000..738f858f012 --- /dev/null +++ b/src/net/CMakeLists.txt @@ -0,0 +1,34 @@ +# Copyright (c) 2018, The Monero Project +# +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without modification, are +# permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this list of +# conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, this list +# of conditions and the following disclaimer in the documentation and/or other +# materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors may be +# used to endorse or promote products derived from this software without specific +# prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +# THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +# THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +set(net_sources error.cpp i2p_address.cpp parse.cpp socks.cpp socks_connect.cpp tor_address.cpp) +set(net_headers error.h i2p_address.h parse.h socks.h socks_connect.h tor_address.h) + +monero_add_library(net ${net_sources} ${net_headers}) +target_link_libraries(net common epee ${Boost_ASIO_LIBRARY}) + diff --git a/src/net/error.cpp b/src/net/error.cpp new file mode 100644 index 00000000000..037f44d52c6 --- /dev/null +++ b/src/net/error.cpp @@ -0,0 +1,92 @@ +// Copyright (c) 2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "error.h" + +#include + +namespace +{ + struct net_category : std::error_category + { + net_category() noexcept + : std::error_category() + {} + + const char* name() const noexcept override + { + return "net::error_category"; + } + + std::string message(int value) const override + { + switch (net::error(value)) + { + case net::error::expected_tld: + return "Expected top-level domain"; + case net::error::invalid_host: + return "Host value is not valid"; + case net::error::invalid_i2p_address: + return "Invalid I2P address"; + case net::error::invalid_port: + return "Invalid port value (expected 0-65535)"; + case net::error::invalid_tor_address: + return "Invalid Tor address"; + case net::error::unsupported_address: + return "Network address not supported"; + default: + break; + } + + return "Unknown net::error"; + } + + std::error_condition default_error_condition(int value) const noexcept override + { + switch (net::error(value)) + { + case net::error::invalid_port: + return std::errc::result_out_of_range; + case net::error::expected_tld: + case net::error::invalid_tor_address: + default: + break; + } + return std::error_condition{value, *this}; + } + }; +} // anonymous + +namespace net +{ + std::error_category const& error_category() noexcept + { + static const net_category instance{}; + return instance; + } +} diff --git a/src/net/error.h b/src/net/error.h new file mode 100644 index 00000000000..c8338f7e2f0 --- /dev/null +++ b/src/net/error.h @@ -0,0 +1,64 @@ +// Copyright (c) 2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +#include +#include + +namespace net +{ + //! General net errors + enum class error : int + { + // 0 reserved for success (as per expect) + expected_tld = 1, //!< Expected a tld + invalid_host, //!< Hostname is not valid + invalid_i2p_address, + invalid_port, //!< Outside of 0-65535 range + invalid_tor_address,//!< Invalid base32 or length + unsupported_address //!< Type not supported by `get_network_address` + }; + + //! \return `std::error_category` for `net` namespace. + std::error_category const& error_category() noexcept; + + //! \return `net::error` as a `std::error_code` value. + inline std::error_code make_error_code(error value) noexcept + { + return std::error_code{int(value), error_category()}; + } +} + +namespace std +{ + template<> + struct is_error_code_enum<::net::error> + : true_type + {}; +} diff --git a/src/net/fwd.h b/src/net/fwd.h new file mode 100644 index 00000000000..7cae882513f --- /dev/null +++ b/src/net/fwd.h @@ -0,0 +1,46 @@ +// Copyright (c) 2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +#include + +namespace net +{ + enum class error : int; + class tor_address; + class i2p_address; + + namespace socks + { + class client; + template class connect_handler; + enum class error : int; + enum class version : std::uint8_t; + } +} diff --git a/src/net/i2p_address.cpp b/src/net/i2p_address.cpp new file mode 100644 index 00000000000..cba829d3fd0 --- /dev/null +++ b/src/net/i2p_address.cpp @@ -0,0 +1,200 @@ +// Copyright (c) 2019, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "i2p_address.h" + +#include +#include +#include +#include +#include +#include + +#include "net/error.h" +#include "serialization/keyvalue_serialization.h" +#include "storages/portable_storage.h" +#include "string_tools.h" + +namespace net +{ + namespace + { + // !TODO only b32 addresses right now + constexpr const char tld[] = u8".b32.i2p"; + constexpr const char unknown_host[] = ""; + + constexpr const unsigned b32_length = 52; + + constexpr const char base32_alphabet[] = + u8"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz234567"; + + expect host_check(boost::string_ref host) noexcept + { + if (!host.ends_with(tld)) + return {net::error::expected_tld}; + + host.remove_suffix(sizeof(tld) - 1); + + if (host.size() != b32_length) + return {net::error::invalid_i2p_address}; + if (host.find_first_not_of(base32_alphabet) != boost::string_ref::npos) + return {net::error::invalid_i2p_address}; + + return success(); + } + + struct i2p_serialized + { + std::string host; + std::uint16_t port; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(host) + KV_SERIALIZE(port) + END_KV_SERIALIZE_MAP() + }; + } + + i2p_address::i2p_address(const boost::string_ref host, const std::uint16_t port) noexcept + : port_(port) + { + // this is a private constructor, throw if moved to public + assert(host.size() < sizeof(host_)); + + const std::size_t length = std::min(sizeof(host_) - 1, host.size()); + std::memcpy(host_, host.data(), length); + std::memset(host_ + length, 0, sizeof(host_) - length); + } + + const char* i2p_address::unknown_str() noexcept + { + return unknown_host; + } + + i2p_address::i2p_address() noexcept + : port_(0) + { + static_assert(sizeof(unknown_host) <= sizeof(host_), "bad buffer size"); + std::memcpy(host_, unknown_host, sizeof(unknown_host)); + std::memset(host_ + sizeof(unknown_host), 0, sizeof(host_) - sizeof(unknown_host)); + } + + expect i2p_address::make(const boost::string_ref address, const std::uint16_t default_port) + { + boost::string_ref host = address.substr(0, address.rfind(':')); + const boost::string_ref port = + address.substr(host.size() + (host.size() == address.size() ? 0 : 1)); + + MONERO_CHECK(host_check(host)); + + std::uint16_t porti = default_port; + if (!port.empty() && !epee::string_tools::get_xtype_from_string(porti, std::string{port})) + return {net::error::invalid_port}; + + static_assert(b32_length + sizeof(tld) == sizeof(i2p_address::host_), "bad internal host size"); + return i2p_address{host, porti}; + } + + bool i2p_address::_load(epee::serialization::portable_storage& src, epee::serialization::section* hparent) + { + i2p_serialized in{}; + if (in._load(src, hparent) && in.host.size() < sizeof(host_) && (in.host == unknown_host || !host_check(in.host).has_error())) + { + std::memcpy(host_, in.host.data(), in.host.size()); + std::memset(host_ + in.host.size(), 0, sizeof(host_) - in.host.size()); + port_ = in.port; + return true; + } + static_assert(sizeof(unknown_host) <= sizeof(host_), "bad buffer size"); + std::memcpy(host_, unknown_host, sizeof(unknown_host)); // include null terminator + port_ = 0; + return false; + } + + bool i2p_address::store(epee::serialization::portable_storage& dest, epee::serialization::section* hparent) const + { + const i2p_serialized out{std::string{host_}, port_}; + return out.store(dest, hparent); + } + + i2p_address::i2p_address(const i2p_address& rhs) noexcept + : port_(rhs.port_) + { + std::memcpy(host_, rhs.host_, sizeof(host_)); + } + + i2p_address& i2p_address::operator=(const i2p_address& rhs) noexcept + { + if (this != std::addressof(rhs)) + { + port_ = rhs.port_; + std::memcpy(host_, rhs.host_, sizeof(host_)); + } + return *this; + } + + bool i2p_address::is_unknown() const noexcept + { + static_assert(1 <= sizeof(host_), "host size too small"); + return host_[0] == '<'; // character is not allowed otherwise + } + + bool i2p_address::equal(const i2p_address& rhs) const noexcept + { + return port_ == rhs.port_ && is_same_host(rhs); + } + + bool i2p_address::less(const i2p_address& rhs) const noexcept + { + return std::strcmp(host_str(), rhs.host_str()) < 0 || port() < rhs.port(); + } + + bool i2p_address::is_same_host(const i2p_address& rhs) const noexcept + { + return std::strcmp(host_str(), rhs.host_str()) == 0; + } + + std::string i2p_address::str() const + { + const std::size_t host_length = std::strlen(host_str()); + const std::size_t port_length = + port_ == 0 ? 0 : std::numeric_limits::digits10 + 2; + + std::string out{}; + out.reserve(host_length + port_length); + out.assign(host_str(), host_length); + + if (port_ != 0) + { + out.push_back(':'); + namespace karma = boost::spirit::karma; + karma::generate(std::back_inserter(out), karma::ushort_, port()); + } + return out; + } +} diff --git a/src/net/i2p_address.h b/src/net/i2p_address.h new file mode 100644 index 00000000000..28a1118ba5b --- /dev/null +++ b/src/net/i2p_address.h @@ -0,0 +1,140 @@ +// Copyright (c) 2019, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +#include +#include +#include + +#include "common/expect.h" +#include "net/enums.h" +#include "net/error.h" + +namespace epee +{ +namespace serialization +{ + class portable_storage; + struct section; +} +} + +namespace net +{ + //! b32 i2p address; internal format not condensed/decoded. + class i2p_address + { + std::uint16_t port_; + char host_[61]; // null-terminated + + //! Keep in private, `host.size()` has no runtime check + i2p_address(boost::string_ref host, std::uint16_t port) noexcept; + + public: + //! \return Size of internal buffer for host. + static constexpr std::size_t buffer_size() noexcept { return sizeof(host_); } + + //! \return ``. + static const char* unknown_str() noexcept; + + //! An object with `port() == 0` and `host_str() == unknown_str()`. + i2p_address() noexcept; + + //! \return A default constructed `i2p_address` object. + static i2p_address unknown() noexcept { return i2p_address{}; } + + /*! + Parse `address` in b32 i2p format (i.e. x.b32.i2p:80) + with `default_port` being used if port is not specified in + `address`. + */ + static expect make(boost::string_ref address, std::uint16_t default_port = 0); + + //! Load from epee p2p format, and \return false if not valid tor address + bool _load(epee::serialization::portable_storage& src, epee::serialization::section* hparent); + + //! Store in epee p2p format + bool store(epee::serialization::portable_storage& dest, epee::serialization::section* hparent) const; + + // Moves and copies are currently identical + + i2p_address(const i2p_address& rhs) noexcept; + ~i2p_address() = default; + i2p_address& operator=(const i2p_address& rhs) noexcept; + + //! \return True if default constructed or via `unknown()`. + bool is_unknown() const noexcept; + + bool equal(const i2p_address& rhs) const noexcept; + bool less(const i2p_address& rhs) const noexcept; + + //! \return True if i2p addresses are identical. + bool is_same_host(const i2p_address& rhs) const noexcept; + + //! \return `x.b32.i2p` or `x.b32.i2p:z` if `port() != 0`. + std::string str() const; + + //! \return Null-terminated `x.b32.i2p` value or `unknown_str()`. + const char* host_str() const noexcept { return host_; } + + //! \return Port value or `0` if unspecified. + std::uint16_t port() const noexcept { return port_; } + + static constexpr bool is_loopback() noexcept { return false; } + static constexpr bool is_local() noexcept { return false; } + + static constexpr epee::net_utils::address_type get_type_id() noexcept + { + return epee::net_utils::address_type::i2p; + } + + static constexpr epee::net_utils::zone get_zone() noexcept + { + return epee::net_utils::zone::i2p; + } + + //! \return `!is_unknown()`. + bool is_blockable() const noexcept { return !is_unknown(); } + }; + + inline bool operator==(const i2p_address& lhs, const i2p_address& rhs) noexcept + { + return lhs.equal(rhs); + } + + inline bool operator!=(const i2p_address& lhs, const i2p_address& rhs) noexcept + { + return !lhs.equal(rhs); + } + + inline bool operator<(const i2p_address& lhs, const i2p_address& rhs) noexcept + { + return lhs.less(rhs); + } +} // net diff --git a/src/net/parse.cpp b/src/net/parse.cpp new file mode 100644 index 00000000000..eaaadb67eff --- /dev/null +++ b/src/net/parse.cpp @@ -0,0 +1,61 @@ +// Copyright (c) 2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "parse.h" + +#include "net/tor_address.h" +#include "net/i2p_address.h" +#include "string_tools.h" + +namespace net +{ + expect + get_network_address(const boost::string_ref address, const std::uint16_t default_port) + { + const boost::string_ref host = address.substr(0, address.rfind(':')); + + if (host.empty()) + return make_error_code(net::error::invalid_host); + if (host.ends_with(".onion")) + return tor_address::make(address, default_port); + if (host.ends_with(".i2p")) + return i2p_address::make(address, default_port); + + std::uint16_t port = default_port; + if (host.size() < address.size()) + { + if (!epee::string_tools::get_xtype_from_string(port, std::string{address.substr(host.size() + 1)})) + return make_error_code(net::error::invalid_port); + } + + std::uint32_t ip = 0; + if (epee::string_tools::get_ip_int32_from_string(ip, std::string{host})) + return {epee::net_utils::ipv4_network_address{ip, port}}; + return make_error_code(net::error::unsupported_address); + } +} diff --git a/src/net/parse.h b/src/net/parse.h new file mode 100644 index 00000000000..5804c41285a --- /dev/null +++ b/src/net/parse.h @@ -0,0 +1,54 @@ +// Copyright (c) 2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +#include +#include + +#include "common/expect.h" +#include "net/net_utils_base.h" + +namespace net +{ + /*! + Identifies onion, i2p and IPv4 addresses and returns them as a generic + `network_address`. If the type is unsupported, it might be a hostname, + and `error() == net::error::kUnsupportedAddress` is returned. + + \param address An onion address, i2p address, ipv4 address or hostname. Hostname + will return an error. + \param default_port If `address` does not specify a port, this value + will be used. + + \return A tor or IPv4 address, else error. + */ + expect + get_network_address(boost::string_ref address, std::uint16_t default_port); +} + diff --git a/src/net/socks.cpp b/src/net/socks.cpp new file mode 100644 index 00000000000..5a27e16f47b --- /dev/null +++ b/src/net/socks.cpp @@ -0,0 +1,332 @@ +// Copyright (c) 2018-2019, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "socks.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "net/net_utils_base.h" +#include "net/tor_address.h" +#include "net/i2p_address.h" + +namespace net +{ +namespace socks +{ + namespace + { + constexpr const unsigned v4_reply_size = 8; + constexpr const std::uint8_t v4_connect_command = 1; + constexpr const std::uint8_t v4tor_resolve_command = 0xf0; + constexpr const std::uint8_t v4_request_granted = 90; + + struct v4_header + { + std::uint8_t version; + std::uint8_t command_code; + boost::endian::big_uint16_t port; + boost::endian::big_uint32_t ip; + }; + + std::size_t write_domain_header(epee::span out, const std::uint8_t command, const std::uint16_t port, const boost::string_ref domain) + { + if (std::numeric_limits::max() - sizeof(v4_header) - 2 < domain.size()) + return 0; + + const std::size_t buf_size = sizeof(v4_header) + domain.size() + 2; + if (out.size() < buf_size) + return 0; + + // version 4, 1 indicates invalid ip for domain extension + const v4_header temp{4, command, port, std::uint32_t(1)}; + std::memcpy(out.data(), std::addressof(temp), sizeof(temp)); + out.remove_prefix(sizeof(temp)); + + *(out.data()) = 0; + out.remove_prefix(1); + + std::memcpy(out.data(), domain.data(), domain.size()); + out.remove_prefix(domain.size()); + + *(out.data()) = 0; + return buf_size; + } + + struct socks_category : boost::system::error_category + { + explicit socks_category() noexcept + : boost::system::error_category() + {} + + const char* name() const noexcept override + { + return "net::socks::error_category"; + } + + virtual std::string message(int value) const override + { + switch (socks::error(value)) + { + case socks::error::rejected: + return "Socks request rejected or failed"; + case socks::error::identd_connection: + return "Socks request rejected because server cannot connect to identd on the client"; + case socks::error::identd_user: + return "Socks request rejected because the client program and identd report different user-ids"; + + case socks::error::bad_read: + return "Socks boost::async_read read fewer bytes than expected"; + case socks::error::bad_write: + return "Socks boost::async_write wrote fewer bytes than expected"; + case socks::error::unexpected_version: + return "Socks server returned unexpected version in reply"; + + default: + break; + } + return "Unknown net::socks::error"; + } + + boost::system::error_condition default_error_condition(int value) const noexcept override + { + switch (socks::error(value)) + { + case socks::error::bad_read: + case socks::error::bad_write: + return boost::system::errc::io_error; + case socks::error::unexpected_version: + return boost::system::errc::protocol_error; + default: + break; + }; + if (1 <= value && value <= 256) + return boost::system::errc::protocol_error; + + return boost::system::error_condition{value, *this}; + } + }; + } + + const boost::system::error_category& error_category() noexcept + { + static const socks_category instance{}; + return instance; + } + + struct client::completed + { + std::shared_ptr self_; + + void operator()(const boost::system::error_code error, const std::size_t bytes) const + { + static_assert(1 < sizeof(self_->buffer_), "buffer too small for v4 response"); + + if (self_) + { + client& self = *self_; + self.buffer_size_ = std::min(bytes, sizeof(self.buffer_)); + + if (error) + self.done(error, std::move(self_)); + else if (self.buffer().size() < sizeof(v4_header)) + self.done(socks::error::bad_read, std::move(self_)); + else if (self.buffer_[0] != 0) // response version + self.done(socks::error::unexpected_version, std::move(self_)); + else if (self.buffer_[1] != v4_request_granted) + self.done(socks::error(int(self.buffer_[1]) + 1), std::move(self_)); + else + self.done(boost::system::error_code{}, std::move(self_)); + } + } + }; + + struct client::read + { + std::shared_ptr self_; + + static boost::asio::mutable_buffers_1 get_buffer(client& self) noexcept + { + static_assert(sizeof(v4_header) <= sizeof(self.buffer_), "buffer too small for v4 response"); + return boost::asio::buffer(self.buffer_, sizeof(v4_header)); + } + + void operator()(const boost::system::error_code error, const std::size_t bytes) + { + if (self_) + { + client& self = *self_; + if (error) + self.done(error, std::move(self_)); + else if (bytes < self.buffer().size()) + self.done(socks::error::bad_write, std::move(self_)); + else + boost::asio::async_read(self.proxy_, get_buffer(self), self.strand_.wrap(completed{std::move(self_)})); + } + } + }; + + struct client::write + { + std::shared_ptr self_; + + static boost::asio::const_buffers_1 get_buffer(client const& self) noexcept + { + return boost::asio::buffer(self.buffer_, self.buffer_size_); + } + + void operator()(const boost::system::error_code error) + { + if (self_) + { + client& self = *self_; + if (error) + self.done(error, std::move(self_)); + else + boost::asio::async_write(self.proxy_, get_buffer(self), self.strand_.wrap(read{std::move(self_)})); + } + } + }; + + client::client(stream_type::socket&& proxy, socks::version ver) + : proxy_(std::move(proxy)), strand_(GET_IO_SERVICE(proxy_)), buffer_size_(0), buffer_(), ver_(ver) + {} + + client::~client() {} + + bool client::set_connect_command(const epee::net_utils::ipv4_network_address& address) + { + switch (socks_version()) + { + case version::v4: + case version::v4a: + case version::v4a_tor: + break; + default: + return false; + } + + static_assert(sizeof(v4_header) < sizeof(buffer_), "buffer size too small for request"); + static_assert(0 < sizeof(buffer_), "buffer size too small for null termination"); + + // version 4 + const v4_header temp{4, v4_connect_command, address.port(), boost::endian::big_to_native(address.ip())}; + std::memcpy(std::addressof(buffer_), std::addressof(temp), sizeof(temp)); + buffer_[sizeof(temp)] = 0; + buffer_size_ = sizeof(temp) + 1; + + return true; + } + + bool client::set_connect_command(const boost::string_ref domain, std::uint16_t port) + { + switch (socks_version()) + { + case version::v4a: + case version::v4a_tor: + break; + + default: + return false; + } + + const std::size_t buf_used = write_domain_header(buffer_, v4_connect_command, port, domain); + buffer_size_ = buf_used; + return buf_used != 0; + } + + bool client::set_connect_command(const net::tor_address& address) + { + if (!address.is_unknown()) + return set_connect_command(address.host_str(), address.port()); + return false; + } + + bool client::set_connect_command(const net::i2p_address& address) + { + if (!address.is_unknown()) + return set_connect_command(address.host_str(), address.port()); + return false; + } + + bool client::set_resolve_command(boost::string_ref domain) + { + if (socks_version() != version::v4a_tor) + return false; + + const std::size_t buf_used = write_domain_header(buffer_, v4tor_resolve_command, 0, domain); + buffer_size_ = buf_used; + return buf_used != 0; + } + + bool client::connect_and_send(std::shared_ptr self, const stream_type::endpoint& proxy_address) + { + if (self && !self->buffer().empty()) + { + client& alias = *self; + alias.proxy_.async_connect(proxy_address, alias.strand_.wrap(write{std::move(self)})); + return true; + } + return false; + } + + bool client::send(std::shared_ptr self) + { + if (self && !self->buffer().empty()) + { + client& alias = *self; + boost::asio::async_write(alias.proxy_, write::get_buffer(alias), alias.strand_.wrap(read{std::move(self)})); + return true; + } + return false; + } + + void client::async_close::operator()(boost::system::error_code error) + { + if (self_ && error != boost::system::errc::operation_canceled) + { + const std::shared_ptr self = std::move(self_); + self->strand_.dispatch([self] () + { + if (self && self->proxy_.is_open()) + { + self->proxy_.shutdown(boost::asio::ip::tcp::socket::shutdown_both); + self->proxy_.close(); + } + }); + } + } +} // socks +} // net diff --git a/src/net/socks.h b/src/net/socks.h new file mode 100644 index 00000000000..4d1d34e9e36 --- /dev/null +++ b/src/net/socks.h @@ -0,0 +1,242 @@ +// Copyright (c) 2018-2019, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "net/fwd.h" +#include "span.h" + +namespace epee +{ +namespace net_utils +{ + class ipv4_network_address; +} +} + +namespace net +{ +namespace socks +{ + //! Supported socks variants. + enum class version : std::uint8_t + { + v4 = 0, + v4a, + v4a_tor //!< Extensions defined in Tor codebase + }; + + //! Possible errors with socks communication. Defined in https://www.openssh.com/txt/socks4.protocol + enum class error : int + { + // 0 is reserved for success value + // 1-256 -> reserved for error values from socks server (+1 from wire value). + rejected = 92, + identd_connection, + identd_user, + // Specific to application + bad_read = 257, + bad_write, + unexpected_version + }; + + /* boost::system::error_code is extended for easier compatibility with + boost::asio errors. If std::error_code is needed (with expect for + instance), then upgrade to boost 1.65+ or use conversion code in + develop branch at boost/system/detail/std_interoperability.hpp */ + + //! \return boost::system::error_category for net::socks namespace + const boost::system::error_category& error_category() noexcept; + + //! \return net::socks::error as a boost::system::error_code. + inline boost::system::error_code make_error_code(error value) noexcept + { + return boost::system::error_code{int(value), socks::error_category()}; + } + + //! Client support for socks connect and resolve commands. + class client + { + boost::asio::ip::tcp::socket proxy_; + boost::asio::io_service::strand strand_; + std::uint16_t buffer_size_; + std::uint8_t buffer_[1024]; + socks::version ver_; + + /*! + Only invoked after `*send(...)` function completes or fails. + `bool(error) == false` indicates success; `self.get()` is always + `this` and allows implementations to skip + `std::enable_shared_from_this` (ASIO callbacks need shared_ptr). + The design saves space and reduces cycles (everything uses moves, + so no atomic operations are ever necessary). + + \param error when processing last command (if any). + \param self `shared_ptr` handle to `this`. + */ + virtual void done(boost::system::error_code error, std::shared_ptr self) = 0; + + public: + using stream_type = boost::asio::ip::tcp; + + // defined in cpp + struct write; + struct read; + struct completed; + + /*! + \param proxy ownership is passed into `this`. Does not have to be + in connected state. + \param ver socks version for the connection. + */ + explicit client(stream_type::socket&& proxy, socks::version ver); + + client(const client&) = delete; + virtual ~client(); + client& operator=(const client&) = delete; + + //! \return Ownership of socks client socket object. + stream_type::socket take_socket() + { + return stream_type::socket{std::move(proxy_)}; + } + + //! \return Socks version. + socks::version socks_version() const noexcept { return ver_; } + + //! \return Contents of internal buffer. + epee::span buffer() const noexcept + { + return {buffer_, buffer_size_}; + } + + //! \post `buffer.empty()`. + void clear_command() noexcept { buffer_size_ = 0; } + + //! Try to set `address` as remote connection request. + bool set_connect_command(const epee::net_utils::ipv4_network_address& address); + + //! Try to set `domain` + `port` as remote connection request. + bool set_connect_command(boost::string_ref domain, std::uint16_t port); + + //! Try to set `address` as remote Tor hidden service connection request. + bool set_connect_command(const net::tor_address& address); + + //! Try to set `address` as remote i2p hidden service connection request. + bool set_connect_command(const net::i2p_address& address); + + //! Try to set `domain` as remote DNS A record lookup request. + bool set_resolve_command(boost::string_ref domain); + + /*! + Asynchronously connect to `proxy_address` then issue command in + `buffer()`. The `done(...)` method will be invoked upon completion + with `self` and potential `error`s. + + \note Must use one of the `self->set_*_command` calls before using + this function. + \note Only `async_close` can be invoked on `self` until the `done` + callback is invoked. + + \param self ownership of object is given to function. + \param proxy_address of the socks server. + \return False if `self->buffer().empty()` (no command set). + */ + static bool connect_and_send(std::shared_ptr self, const stream_type::endpoint& proxy_address); + + /*! + Assume existing connection to proxy server; asynchronously issue + command in `buffer()`. The `done(...)` method will be invoked + upon completion with `self` and potential `error`s. + + \note Must use one of the `self->set_*_command` calls before using + the function. + \note Only `async_close` can be invoked on `self` until the `done` + callback is invoked. + + \param self ownership of object is given to function. + \return False if `self->buffer().empty()` (no command set). + */ + static bool send(std::shared_ptr self); + + /*! Callback for closing socket. Thread-safe with `*send` functions; + never blocks (uses strands). */ + struct async_close + { + std::shared_ptr self_; + void operator()(boost::system::error_code error = boost::system::error_code{}); + }; + }; + + template + class connect_client : public client + { + Handler handler_; + + virtual void done(boost::system::error_code error, std::shared_ptr) override + { + handler_(error, take_socket()); + } + + public: + explicit connect_client(stream_type::socket&& proxy, socks::version ver, Handler&& handler) + : client(std::move(proxy), ver), handler_(std::move(handler)) + {} + + virtual ~connect_client() override {} + }; + + template + inline std::shared_ptr + make_connect_client(client::stream_type::socket&& proxy, socks::version ver, Handler handler) + { + return std::make_shared>(std::move(proxy), ver, std::move(handler)); + } +} // socks +} // net + +namespace boost +{ +namespace system +{ + template<> + struct is_error_code_enum + : true_type + {}; +} // system +} // boost diff --git a/src/net/socks_connect.cpp b/src/net/socks_connect.cpp new file mode 100644 index 00000000000..a5557f6f8f0 --- /dev/null +++ b/src/net/socks_connect.cpp @@ -0,0 +1,90 @@ +// Copyright (c) 2019, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "socks_connect.h" + +#include +#include +#include +#include +#include + +#include "net/error.h" +#include "net/net_utils_base.h" +#include "net/socks.h" +#include "string_tools.h" + +namespace net +{ +namespace socks +{ + boost::unique_future + connector::operator()(const std::string& remote_host, const std::string& remote_port, boost::asio::steady_timer& timeout) const + { + struct future_socket + { + boost::promise result_; + + void operator()(boost::system::error_code error, boost::asio::ip::tcp::socket&& socket) + { + if (error) + result_.set_exception(boost::system::system_error{error}); + else + result_.set_value(std::move(socket)); + } + }; + + boost::unique_future out{}; + { + std::uint16_t port = 0; + if (!epee::string_tools::get_xtype_from_string(port, remote_port)) + throw std::system_error{net::error::invalid_port, "Remote port for socks proxy"}; + + bool is_set = false; + std::uint32_t ip_address = 0; + boost::promise result{}; + out = result.get_future(); + const auto proxy = net::socks::make_connect_client( + boost::asio::ip::tcp::socket{GET_IO_SERVICE(timeout)}, net::socks::version::v4a, future_socket{std::move(result)} + ); + + if (epee::string_tools::get_ip_int32_from_string(ip_address, remote_host)) + is_set = proxy->set_connect_command(epee::net_utils::ipv4_network_address{ip_address, port}); + else + is_set = proxy->set_connect_command(remote_host, port); + + if (!is_set || !net::socks::client::connect_and_send(proxy, proxy_address)) + throw std::system_error{net::error::invalid_host, "Address for socks proxy"}; + + timeout.async_wait(net::socks::client::async_close{std::move(proxy)}); + } + + return out; + } +} // socks +} // net diff --git a/src/net/socks_connect.h b/src/net/socks_connect.h new file mode 100644 index 00000000000..44b0fa2b3b2 --- /dev/null +++ b/src/net/socks_connect.h @@ -0,0 +1,55 @@ +// Copyright (c) 2019, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +#include +#include +#include +#include + +namespace net +{ +namespace socks +{ + //! Primarily for use with `epee::net_utils::http_client`. + struct connector + { + boost::asio::ip::tcp::endpoint proxy_address; + + /*! Creates a new socket, asynchronously connects to `proxy_address`, + and requests a connection to `remote_host` on `remote_port`. Sets + socket as closed if `timeout` is reached. + + \return The socket if successful, and exception in the future with + error otherwise. */ + boost::unique_future + operator()(const std::string& remote_host, const std::string& remote_port, boost::asio::steady_timer& timeout) const; + }; +} // socks +} // net diff --git a/src/net/tor_address.cpp b/src/net/tor_address.cpp new file mode 100644 index 00000000000..904a9a0fc8f --- /dev/null +++ b/src/net/tor_address.cpp @@ -0,0 +1,203 @@ +// Copyright (c) 2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "tor_address.h" + +#include +#include +#include +#include +#include +#include + +#include "net/error.h" +#include "serialization/keyvalue_serialization.h" +#include "storages/portable_storage.h" +#include "string_tools.h" + +namespace net +{ + namespace + { + constexpr const char tld[] = u8".onion"; + constexpr const char unknown_host[] = ""; + + constexpr const unsigned v2_length = 16; + constexpr const unsigned v3_length = 56; + + constexpr const char base32_alphabet[] = + u8"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz234567"; + + expect host_check(boost::string_ref host) noexcept + { + if (!host.ends_with(tld)) + return {net::error::expected_tld}; + + host.remove_suffix(sizeof(tld) - 1); + + //! \TODO v3 has checksum, base32 decoding is required to verify it + if (host.size() != v2_length && host.size() != v3_length) + return {net::error::invalid_tor_address}; + if (host.find_first_not_of(base32_alphabet) != boost::string_ref::npos) + return {net::error::invalid_tor_address}; + + return success(); + } + + struct tor_serialized + { + std::string host; + std::uint16_t port; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(host) + KV_SERIALIZE(port) + END_KV_SERIALIZE_MAP() + }; + } + + tor_address::tor_address(const boost::string_ref host, const std::uint16_t port) noexcept + : port_(port) + { + // this is a private constructor, throw if moved to public + assert(host.size() < sizeof(host_)); + + const std::size_t length = std::min(sizeof(host_) - 1, host.size()); + std::memcpy(host_, host.data(), length); + std::memset(host_ + length, 0, sizeof(host_) - length); + } + + const char* tor_address::unknown_str() noexcept + { + return unknown_host; + } + + tor_address::tor_address() noexcept + : port_(0) + { + static_assert(sizeof(unknown_host) <= sizeof(host_), "bad buffer size"); + std::memcpy(host_, unknown_host, sizeof(unknown_host)); + std::memset(host_ + sizeof(unknown_host), 0, sizeof(host_) - sizeof(unknown_host)); + } + + expect tor_address::make(const boost::string_ref address, const std::uint16_t default_port) + { + boost::string_ref host = address.substr(0, address.rfind(':')); + const boost::string_ref port = + address.substr(host.size() + (host.size() == address.size() ? 0 : 1)); + + MONERO_CHECK(host_check(host)); + + std::uint16_t porti = default_port; + if (!port.empty() && !epee::string_tools::get_xtype_from_string(porti, std::string{port})) + return {net::error::invalid_port}; + + static_assert(v2_length <= v3_length, "bad internal host size"); + static_assert(v3_length + sizeof(tld) == sizeof(tor_address::host_), "bad internal host size"); + return tor_address{host, porti}; + } + + bool tor_address::_load(epee::serialization::portable_storage& src, epee::serialization::section* hparent) + { + tor_serialized in{}; + if (in._load(src, hparent) && in.host.size() < sizeof(host_) && (in.host == unknown_host || !host_check(in.host).has_error())) + { + std::memcpy(host_, in.host.data(), in.host.size()); + std::memset(host_ + in.host.size(), 0, sizeof(host_) - in.host.size()); + port_ = in.port; + return true; + } + static_assert(sizeof(unknown_host) <= sizeof(host_), "bad buffer size"); + std::memcpy(host_, unknown_host, sizeof(unknown_host)); // include null terminator + port_ = 0; + return false; + } + + bool tor_address::store(epee::serialization::portable_storage& dest, epee::serialization::section* hparent) const + { + const tor_serialized out{std::string{host_}, port_}; + return out.store(dest, hparent); + } + + tor_address::tor_address(const tor_address& rhs) noexcept + : port_(rhs.port_) + { + std::memcpy(host_, rhs.host_, sizeof(host_)); + } + + tor_address& tor_address::operator=(const tor_address& rhs) noexcept + { + if (this != std::addressof(rhs)) + { + port_ = rhs.port_; + std::memcpy(host_, rhs.host_, sizeof(host_)); + } + return *this; + } + + bool tor_address::is_unknown() const noexcept + { + static_assert(1 <= sizeof(host_), "host size too small"); + return host_[0] == '<'; // character is not allowed otherwise + } + + bool tor_address::equal(const tor_address& rhs) const noexcept + { + return port_ == rhs.port_ && is_same_host(rhs); + } + + bool tor_address::less(const tor_address& rhs) const noexcept + { + return std::strcmp(host_str(), rhs.host_str()) < 0 || port() < rhs.port(); + } + + bool tor_address::is_same_host(const tor_address& rhs) const noexcept + { + //! \TODO v2 and v3 should be comparable - requires base32 + return std::strcmp(host_str(), rhs.host_str()) == 0; + } + + std::string tor_address::str() const + { + const std::size_t host_length = std::strlen(host_str()); + const std::size_t port_length = + port_ == 0 ? 0 : std::numeric_limits::digits10 + 2; + + std::string out{}; + out.reserve(host_length + port_length); + out.assign(host_str(), host_length); + + if (port_ != 0) + { + out.push_back(':'); + namespace karma = boost::spirit::karma; + karma::generate(std::back_inserter(out), karma::ushort_, port()); + } + return out; + } +} diff --git a/src/net/tor_address.h b/src/net/tor_address.h new file mode 100644 index 00000000000..22d8cc11976 --- /dev/null +++ b/src/net/tor_address.h @@ -0,0 +1,140 @@ +// Copyright (c) 2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +#include +#include +#include + +#include "common/expect.h" +#include "net/enums.h" +#include "net/error.h" + +namespace epee +{ +namespace serialization +{ + class portable_storage; + struct section; +} +} + +namespace net +{ + //! Tor onion address; internal format not condensed/decoded. + class tor_address + { + std::uint16_t port_; + char host_[63]; // null-terminated + + //! Keep in private, `host.size()` has no runtime check + tor_address(boost::string_ref host, std::uint16_t port) noexcept; + + public: + //! \return Size of internal buffer for host. + static constexpr std::size_t buffer_size() noexcept { return sizeof(host_); } + + //! \return ``. + static const char* unknown_str() noexcept; + + //! An object with `port() == 0` and `host_str() == unknown_str()`. + tor_address() noexcept; + + //! \return A default constructed `tor_address` object. + static tor_address unknown() noexcept { return tor_address{}; } + + /*! + Parse `address` in onion v2 or v3 format with (i.e. x.onion:80) + with `default_port` being used iff port is not specified in + `address`. + */ + static expect make(boost::string_ref address, std::uint16_t default_port = 0); + + //! Load from epee p2p format, and \return false if not valid tor address + bool _load(epee::serialization::portable_storage& src, epee::serialization::section* hparent); + + //! Store in epee p2p format + bool store(epee::serialization::portable_storage& dest, epee::serialization::section* hparent) const; + + // Moves and copies are currently identical + + tor_address(const tor_address& rhs) noexcept; + ~tor_address() = default; + tor_address& operator=(const tor_address& rhs) noexcept; + + //! \return True if default constructed or via `unknown()`. + bool is_unknown() const noexcept; + + bool equal(const tor_address& rhs) const noexcept; + bool less(const tor_address& rhs) const noexcept; + + //! \return True if onion addresses are identical. + bool is_same_host(const tor_address& rhs) const noexcept; + + //! \return `x.onion` or `x.onion:z` if `port() != 0`. + std::string str() const; + + //! \return Null-terminated `x.onion` value or `unknown_str()`. + const char* host_str() const noexcept { return host_; } + + //! \return Port value or `0` if unspecified. + std::uint16_t port() const noexcept { return port_; } + + static constexpr bool is_loopback() noexcept { return false; } + static constexpr bool is_local() noexcept { return false; } + + static constexpr epee::net_utils::address_type get_type_id() noexcept + { + return epee::net_utils::address_type::tor; + } + + static constexpr epee::net_utils::zone get_zone() noexcept + { + return epee::net_utils::zone::tor; + } + + //! \return `!is_unknown()`. + bool is_blockable() const noexcept { return !is_unknown(); } + }; + + inline bool operator==(const tor_address& lhs, const tor_address& rhs) noexcept + { + return lhs.equal(rhs); + } + + inline bool operator!=(const tor_address& lhs, const tor_address& rhs) noexcept + { + return !lhs.equal(rhs); + } + + inline bool operator<(const tor_address& lhs, const tor_address& rhs) noexcept + { + return lhs.less(rhs); + } +} // net diff --git a/src/p2p/CMakeLists.txt b/src/p2p/CMakeLists.txt index 83bdffab5f7..3aecc3cf94b 100644 --- a/src/p2p/CMakeLists.txt +++ b/src/p2p/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2014-2018, The Monero Project +# Copyright (c) 2014-2019, The Monero Project # # All rights reserved. # @@ -38,9 +38,9 @@ source_group(p2p FILES ${P2P}) monero_add_library(p2p ${P2P}) target_link_libraries(p2p PUBLIC - epee version cryptonote_core + net ${UPNP_LIBRARIES} ${Boost_CHRONO_LIBRARY} ${Boost_PROGRAM_OPTIONS_LIBRARY} diff --git a/src/p2p/net_node.cpp b/src/p2p/net_node.cpp index c9ca63f4337..fcbcce58ceb 100644 --- a/src/p2p/net_node.cpp +++ b/src/p2p/net_node.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -28,8 +28,83 @@ // // Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers +#include +#include +#include +#include +#include +#include +#include +#include +#include + #include "common/command_line.h" +#include "cryptonote_core/cryptonote_core.h" +#include "cryptonote_protocol/cryptonote_protocol_defs.h" #include "net_node.h" +#include "net/net_utils_base.h" +#include "net/socks.h" +#include "net/parse.h" +#include "net/tor_address.h" +#include "net/i2p_address.h" +#include "p2p/p2p_protocol_defs.h" +#include "string_tools.h" + +namespace +{ + constexpr const boost::chrono::milliseconds future_poll_interval{500}; + constexpr const std::chrono::seconds socks_connect_timeout{P2P_DEFAULT_SOCKS_CONNECT_TIMEOUT}; + + std::int64_t get_max_connections(const boost::iterator_range value) noexcept + { + // -1 is default, 0 is error + if (value.empty()) + return -1; + + std::uint32_t out = 0; + if (epee::string_tools::get_xtype_from_string(out, std::string{value.begin(), value.end()})) + return out; + return 0; + } + + template + epee::net_utils::network_address get_address(const boost::string_ref value) + { + expect address = T::make(value); + if (!address) + { + MERROR( + "Failed to parse " << epee::net_utils::zone_to_string(T::get_zone()) << " address \"" << value << "\": " << address.error().message() + ); + return {}; + } + return {std::move(*address)}; + } + + bool start_socks(std::shared_ptr client, const boost::asio::ip::tcp::endpoint& proxy, const epee::net_utils::network_address& remote) + { + CHECK_AND_ASSERT_MES(client != nullptr, false, "Unexpected null client"); + + bool set = false; + switch (remote.get_type_id()) + { + case net::tor_address::get_type_id(): + set = client->set_connect_command(remote.as()); + break; + case net::i2p_address::get_type_id(): + set = client->set_connect_command(remote.as()); + break; + default: + MERROR("Unsupported network address in socks_connect"); + return false; + } + + const bool sent = + set && net::socks::client::connect_and_send(std::move(client), proxy); + CHECK_AND_ASSERT_MES(sent, false, "Unexpected failure to init socks client"); + return true; + } +} namespace nodetool { @@ -54,16 +129,218 @@ namespace nodetool const command_line::arg_descriptor > arg_p2p_add_exclusive_node = {"add-exclusive-node", "Specify list of peers to connect to only." " If this option is given the options add-priority-node and seed-node are ignored"}; const command_line::arg_descriptor > arg_p2p_seed_node = {"seed-node", "Connect to a node to retrieve peer addresses, and disconnect"}; + const command_line::arg_descriptor > arg_proxy = {"proxy", ",[,max_connections] i.e. \"tor,127.0.0.1:9050,100\""}; + const command_line::arg_descriptor > arg_anonymous_inbound = {"anonymous-inbound", ",<[bind-ip:]port>[,max_connections] i.e. \"x.onion,127.0.0.1:18083,100\""}; const command_line::arg_descriptor arg_p2p_hide_my_port = {"hide-my-port", "Do not announce yourself as peerlist candidate", false, true}; + const command_line::arg_descriptor arg_no_sync = {"no-sync", "Don't synchronize the blockchain with other peers", false}; const command_line::arg_descriptor arg_no_igd = {"no-igd", "Disable UPnP port mapping"}; const command_line::arg_descriptor arg_out_peers = {"out-peers", "set max number of out peers", -1}; const command_line::arg_descriptor arg_in_peers = {"in-peers", "set max number of in peers", -1}; const command_line::arg_descriptor arg_tos_flag = {"tos-flag", "set TOS flag", -1}; - const command_line::arg_descriptor arg_limit_rate_up = {"limit-rate-up", "set limit-rate-up [kB/s]", -1}; - const command_line::arg_descriptor arg_limit_rate_down = {"limit-rate-down", "set limit-rate-down [kB/s]", -1}; + const command_line::arg_descriptor arg_limit_rate_up = {"limit-rate-up", "set limit-rate-up [kB/s]", P2P_DEFAULT_LIMIT_RATE_UP}; + const command_line::arg_descriptor arg_limit_rate_down = {"limit-rate-down", "set limit-rate-down [kB/s]", P2P_DEFAULT_LIMIT_RATE_DOWN}; const command_line::arg_descriptor arg_limit_rate = {"limit-rate", "set limit-rate [kB/s]", -1}; const command_line::arg_descriptor arg_save_graph = {"save-graph", "Save data for dr monero", false}; + + boost::optional> get_proxies(boost::program_options::variables_map const& vm) + { + namespace ip = boost::asio::ip; + + std::vector proxies{}; + + const std::vector args = command_line::get_arg(vm, arg_proxy); + proxies.reserve(args.size()); + + for (const boost::string_ref arg : args) + { + proxies.emplace_back(); + + auto next = boost::algorithm::make_split_iterator(arg, boost::algorithm::first_finder(",")); + CHECK_AND_ASSERT_MES(!next.eof() && !next->empty(), boost::none, "No network type for --" << arg_proxy.name); + const boost::string_ref zone{next->begin(), next->size()}; + + ++next; + CHECK_AND_ASSERT_MES(!next.eof() && !next->empty(), boost::none, "No ipv4:port given for --" << arg_proxy.name); + const boost::string_ref proxy{next->begin(), next->size()}; + + ++next; + if (!next.eof()) + { + proxies.back().max_connections = get_max_connections(*next); + if (proxies.back().max_connections == 0) + { + MERROR("Invalid max connections given to --" << arg_proxy.name); + return boost::none; + } + } + + switch (epee::net_utils::zone_from_string(zone)) + { + case epee::net_utils::zone::tor: + proxies.back().zone = epee::net_utils::zone::tor; + break; + case epee::net_utils::zone::i2p: + proxies.back().zone = epee::net_utils::zone::i2p; + break; + default: + MERROR("Invalid network for --" << arg_proxy.name); + return boost::none; + } + + std::uint32_t ip = 0; + std::uint16_t port = 0; + if (!epee::string_tools::parse_peer_from_string(ip, port, std::string{proxy}) || port == 0) + { + MERROR("Invalid ipv4:port given for --" << arg_proxy.name); + return boost::none; + } + proxies.back().address = ip::tcp::endpoint{ip::address_v4{boost::endian::native_to_big(ip)}, port}; + } + + return proxies; + } + + boost::optional> get_anonymous_inbounds(boost::program_options::variables_map const& vm) + { + std::vector inbounds{}; + + const std::vector args = command_line::get_arg(vm, arg_anonymous_inbound); + inbounds.reserve(args.size()); + + for (const boost::string_ref arg : args) + { + inbounds.emplace_back(); + + auto next = boost::algorithm::make_split_iterator(arg, boost::algorithm::first_finder(",")); + CHECK_AND_ASSERT_MES(!next.eof() && !next->empty(), boost::none, "No inbound address for --" << arg_anonymous_inbound.name); + const boost::string_ref address{next->begin(), next->size()}; + + ++next; + CHECK_AND_ASSERT_MES(!next.eof() && !next->empty(), boost::none, "No local ipv4:port given for --" << arg_anonymous_inbound.name); + const boost::string_ref bind{next->begin(), next->size()}; + + const std::size_t colon = bind.find_first_of(':'); + CHECK_AND_ASSERT_MES(colon < bind.size(), boost::none, "No local port given for --" << arg_anonymous_inbound.name); + + ++next; + if (!next.eof()) + { + inbounds.back().max_connections = get_max_connections(*next); + if (inbounds.back().max_connections == 0) + { + MERROR("Invalid max connections given to --" << arg_proxy.name); + return boost::none; + } + } + + expect our_address = net::get_network_address(address, 0); + switch (our_address ? our_address->get_type_id() : epee::net_utils::address_type::invalid) + { + case net::tor_address::get_type_id(): + inbounds.back().our_address = std::move(*our_address); + inbounds.back().default_remote = net::tor_address::unknown(); + break; + case net::i2p_address::get_type_id(): + inbounds.back().our_address = std::move(*our_address); + inbounds.back().default_remote = net::i2p_address::unknown(); + break; + default: + MERROR("Invalid inbound address (" << address << ") for --" << arg_anonymous_inbound.name << ": " << (our_address ? "invalid type" : our_address.error().message())); + return boost::none; + } + + // get_address returns default constructed address on error + if (inbounds.back().our_address == epee::net_utils::network_address{}) + return boost::none; + + std::uint32_t ip = 0; + std::uint16_t port = 0; + if (!epee::string_tools::parse_peer_from_string(ip, port, std::string{bind})) + { + MERROR("Invalid ipv4:port given for --" << arg_anonymous_inbound.name); + return boost::none; + } + inbounds.back().local_ip = std::string{bind.substr(0, colon)}; + inbounds.back().local_port = std::string{bind.substr(colon + 1)}; + } + + return inbounds; + } + + bool is_filtered_command(const epee::net_utils::network_address& address, int command) + { + switch (command) + { + case nodetool::COMMAND_HANDSHAKE_T::ID: + case nodetool::COMMAND_TIMED_SYNC_T::ID: + case cryptonote::NOTIFY_NEW_TRANSACTIONS::ID: + return false; + default: + break; + } + + if (address.get_zone() == epee::net_utils::zone::public_) + return false; + + MWARNING("Filtered command (#" << command << ") to/from " << address.str()); + return true; + } + + boost::optional + socks_connect_internal(const std::atomic& stop_signal, boost::asio::io_service& service, const boost::asio::ip::tcp::endpoint& proxy, const epee::net_utils::network_address& remote) + { + using socket_type = net::socks::client::stream_type::socket; + using client_result = std::pair; + + struct notify + { + boost::promise socks_promise; + + void operator()(boost::system::error_code error, socket_type&& sock) + { + socks_promise.set_value(std::make_pair(error, std::move(sock))); + } + }; + + boost::unique_future socks_result{}; + { + boost::promise socks_promise{}; + socks_result = socks_promise.get_future(); + + auto client = net::socks::make_connect_client( + boost::asio::ip::tcp::socket{service}, net::socks::version::v4a, notify{std::move(socks_promise)} + ); + if (!start_socks(std::move(client), proxy, remote)) + return boost::none; + } + + const auto start = std::chrono::steady_clock::now(); + while (socks_result.wait_for(future_poll_interval) == boost::future_status::timeout) + { + if (socks_connect_timeout < std::chrono::steady_clock::now() - start) + { + MERROR("Timeout on socks connect (" << proxy << " to " << remote.str() << ")"); + return boost::none; + } + + if (stop_signal) + return boost::none; + } + + try + { + auto result = socks_result.get(); + if (!result.first) + return {std::move(result.second)}; + + MERROR("Failed to make socks connection to " << remote.str() << " (via " << proxy << "): " << result.first.message()); + } + catch (boost::broken_promise const&) + {} + + return boost::none; + } } diff --git a/src/p2p/net_node.h b/src/p2p/net_node.h index 4606f66ee98..42bb3b06182 100644 --- a/src/p2p/net_node.h +++ b/src/p2p/net_node.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -29,21 +29,31 @@ // Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers #pragma once +#include +#include +#include +#include #include +#include #include #include -#include #include -#include +#include +#include +#include #include "cryptonote_config.h" #include "warnings.h" -#include "net/levin_server_cp2.h" +#include "net/abstract_tcp_server2.h" +#include "net/levin_protocol_handler.h" +#include "net/levin_protocol_handler_async.h" #include "p2p_protocol_defs.h" #include "storages/levin_abstract_invoke2.h" #include "net_peerlist.h" #include "math_helper.h" #include "net_node_common.h" +#include "net/enums.h" +#include "net/fwd.h" #include "common/command_line.h" PUSH_WARNINGS @@ -51,6 +61,47 @@ DISABLE_VS_WARNINGS(4355) namespace nodetool { + struct proxy + { + proxy() + : max_connections(-1), + address(), + zone(epee::net_utils::zone::invalid) + {} + + std::int64_t max_connections; + boost::asio::ip::tcp::endpoint address; + epee::net_utils::zone zone; + }; + + struct anonymous_inbound + { + anonymous_inbound() + : max_connections(-1), + local_ip(), + local_port(), + our_address(), + default_remote() + {} + + std::int64_t max_connections; + std::string local_ip; + std::string local_port; + epee::net_utils::network_address our_address; + epee::net_utils::network_address default_remote; + }; + + boost::optional> get_proxies(const boost::program_options::variables_map& vm); + boost::optional> get_anonymous_inbounds(const boost::program_options::variables_map& vm); + + //! \return True if `commnd` is filtered (ignored/dropped) for `address` + bool is_filtered_command(epee::net_utils::network_address const& address, int command); + + // hides boost::future and chrono stuff from mondo template file + boost::optional + socks_connect_internal(const std::atomic& stop_signal, boost::asio::io_service& service, const boost::asio::ip::tcp::endpoint& proxy, const epee::net_utils::network_address& remote); + + template struct p2p_connection_context_t: base_type //t_payload_net_handler::connection_context //public net_utils::connection_context_base { @@ -75,56 +126,134 @@ namespace nodetool typedef COMMAND_HANDSHAKE_T COMMAND_HANDSHAKE; typedef COMMAND_TIMED_SYNC_T COMMAND_TIMED_SYNC; + typedef epee::net_utils::boosted_tcp_server> net_server; + + struct network_zone; + using connect_func = boost::optional(network_zone&, epee::net_utils::network_address const&, epee::net_utils::ssl_support_t); + + struct config_t + { + config_t() + : m_net_config(), + m_peer_id(crypto::rand()), + m_support_flags(0) + {} + + network_config m_net_config; + uint64_t m_peer_id; + uint32_t m_support_flags; + }; + typedef epee::misc_utils::struct_init config; + + struct network_zone + { + network_zone() + : m_connect(nullptr), + m_net_server(epee::net_utils::e_connection_type_P2P), + m_bind_ip(), + m_port(), + m_our_address(), + m_peerlist(), + m_config{}, + m_proxy_address(), + m_current_number_of_out_peers(0), + m_current_number_of_in_peers(0), + m_can_pingback(false) + { + set_config_defaults(); + } + + network_zone(boost::asio::io_service& public_service) + : m_connect(nullptr), + m_net_server(public_service, epee::net_utils::e_connection_type_P2P), + m_bind_ip(), + m_port(), + m_our_address(), + m_peerlist(), + m_config{}, + m_proxy_address(), + m_current_number_of_out_peers(0), + m_current_number_of_in_peers(0), + m_can_pingback(false) + { + set_config_defaults(); + } + + connect_func* m_connect; + net_server m_net_server; + std::string m_bind_ip; + std::string m_port; + epee::net_utils::network_address m_our_address; // in anonymity networks + peerlist_manager m_peerlist; + config m_config; + boost::asio::ip::tcp::endpoint m_proxy_address; + std::atomic m_current_number_of_out_peers; + std::atomic m_current_number_of_in_peers; + bool m_can_pingback; + + private: + void set_config_defaults() noexcept + { + // at this moment we have a hardcoded config + m_config.m_net_config.handshake_interval = P2P_DEFAULT_HANDSHAKE_INTERVAL; + m_config.m_net_config.packet_max_size = P2P_DEFAULT_PACKET_MAX_SIZE; + m_config.m_net_config.config_id = 0; + m_config.m_net_config.connection_timeout = P2P_DEFAULT_CONNECTION_TIMEOUT; + m_config.m_net_config.ping_connection_timeout = P2P_DEFAULT_PING_CONNECTION_TIMEOUT; + m_config.m_net_config.send_peerlist_sz = P2P_DEFAULT_PEERS_IN_HANDSHAKE; + m_config.m_support_flags = 0; // only set in public zone + } + }; + public: typedef t_payload_net_handler payload_net_handler; node_server(t_payload_net_handler& payload_handler) - :m_payload_handler(payload_handler), - m_current_number_of_out_peers(0), - m_current_number_of_in_peers(0), - m_allow_local_ip(false), - m_hide_my_port(false), - m_no_igd(false), - m_offline(false), - m_save_graph(false), - is_closing(false), - m_net_server( epee::net_utils::e_connection_type_P2P ) // this is a P2P connection of the main p2p node server, because this is class node_server<> - {} - virtual ~node_server() + : m_payload_handler(payload_handler), + m_external_port(0), + m_rpc_port(0), + m_allow_local_ip(false), + m_hide_my_port(false), + m_no_igd(false), + m_offline(false), + m_save_graph(false), + is_closing(false), + m_network_id() {} + virtual ~node_server(); static void init_options(boost::program_options::options_description& desc); bool run(); + network_zone& add_zone(epee::net_utils::zone zone); bool init(const boost::program_options::variables_map& vm); bool deinit(); bool send_stop_signal(); uint32_t get_this_peer_port(){return m_listening_port;} t_payload_net_handler& get_payload_object(); - template - void serialize(Archive &a, const t_version_type ver) - { - a & m_peerlist; - if (ver == 0) - { - // from v1, we do not store the peer id anymore - peerid_type peer_id = AUTO_VAL_INIT (peer_id); - a & peer_id; - } - } // debug functions bool log_peerlist(); bool log_connections(); - virtual uint64_t get_connections_count(); - size_t get_outgoing_connections_count(); - size_t get_incoming_connections_count(); - peerlist_manager& get_peerlist_manager(){return m_peerlist;} - void delete_out_connections(size_t count); - void delete_in_connections(size_t count); + + // These functions only return information for the "public" zone + virtual uint64_t get_public_connections_count(); + size_t get_public_outgoing_connections_count(); + size_t get_public_white_peers_count(); + size_t get_public_gray_peers_count(); + void get_public_peerlist(std::vector& gray, std::vector& white); + size_t get_zone_count() const { return m_network_zones.size(); } + + void change_max_out_public_peers(size_t count); + void change_max_in_public_peers(size_t count); virtual bool block_host(const epee::net_utils::network_address &adress, time_t seconds = P2P_IP_BLOCKTIME); virtual bool unblock_host(const epee::net_utils::network_address &address); virtual std::map get_blocked_hosts() { CRITICAL_REGION_LOCAL(m_blocked_hosts_lock); return m_blocked_hosts; } + + virtual void add_used_stripe_peer(const typename t_payload_net_handler::connection_context &context); + virtual void remove_used_stripe_peer(const typename t_payload_net_handler::connection_context &context); + virtual void clear_used_stripe_peers(); + private: const std::vector m_seed_nodes_list = { "seeds.moneroseeds.se" @@ -142,6 +271,9 @@ namespace nodetool CHAIN_LEVIN_NOTIFY_MAP2(p2p_connection_context); //move levin_commands_handler interface notify(...) callbacks into nothing BEGIN_INVOKE_MAP2(node_server) + if (is_filtered_command(context.m_remote_address, command)) + return LEVIN_ERROR_CONNECTION_HANDLER_NOT_DEFINED; + HANDLE_INVOKE_T2(COMMAND_HANDSHAKE, &node_server::handle_handshake) HANDLE_INVOKE_T2(COMMAND_TIMED_SYNC, &node_server::handle_timed_sync) HANDLE_INVOKE_T2(COMMAND_PING, &node_server::handle_ping) @@ -170,7 +302,7 @@ namespace nodetool bool make_default_peer_id(); bool make_default_config(); bool store_config(); - bool check_trust(const proof_of_trust& tr); + bool check_trust(const proof_of_trust& tr, epee::net_utils::zone zone_type); //----------------- levin_commands_handler ------------------------------------------------------------- @@ -178,10 +310,9 @@ namespace nodetool virtual void on_connection_close(p2p_connection_context& context); virtual void callback(p2p_connection_context& context); //----------------- i_p2p_endpoint ------------------------------------------------------------- - virtual bool relay_notify_to_list(int command, const std::string& data_buff, const std::list &connections); - virtual bool relay_notify_to_all(int command, const std::string& data_buff, const epee::net_utils::connection_context_base& context); - virtual bool invoke_command_to_peer(int command, const std::string& req_buff, std::string& resp_buff, const epee::net_utils::connection_context_base& context); - virtual bool invoke_notify_to_peer(int command, const std::string& req_buff, const epee::net_utils::connection_context_base& context); + virtual bool relay_notify_to_list(int command, const epee::span data_buff, std::vector> connections); + virtual bool invoke_command_to_peer(int command, const epee::span req_buff, std::string& resp_buff, const epee::net_utils::connection_context_base& context); + virtual bool invoke_notify_to_peer(int command, const epee::span req_buff, const epee::net_utils::connection_context_base& context); virtual bool drop_connection(const epee::net_utils::connection_context_base& context); virtual void request_callback(const epee::net_utils::connection_context_base& context); virtual void for_each_connection(std::function f); @@ -195,12 +326,12 @@ namespace nodetool const boost::program_options::variables_map& vm ); bool idle_worker(); - bool handle_remote_peerlist(const std::list& peerlist, time_t local_time, const epee::net_utils::connection_context_base& context); - bool get_local_node_data(basic_node_data& node_data); + bool handle_remote_peerlist(const std::vector& peerlist, time_t local_time, const epee::net_utils::connection_context_base& context); + bool get_local_node_data(basic_node_data& node_data, const network_zone& zone); //bool get_local_handshake_data(handshake_data& hshd); - bool merge_peerlist_with_local(const std::list& bs); - bool fix_time_delta(std::list& local_peerlist, time_t local_time, int64_t& delta); + bool merge_peerlist_with_local(const std::vector& bs); + bool fix_time_delta(std::vector& local_peerlist, time_t local_time, int64_t& delta); bool connections_maker(); bool peer_sync_idle_maker(); @@ -208,7 +339,7 @@ namespace nodetool bool do_peer_timed_sync(const epee::net_utils::connection_context_base& context, peerid_type peer_id); bool make_new_connection_from_anchor_peerlist(const std::vector& anchor_peerlist); - bool make_new_connection_from_peerlist(bool use_white_list); + bool make_new_connection_from_peerlist(network_zone& zone, bool use_white_list); bool try_to_connect_and_handshake_with_new_peer(const epee::net_utils::network_address& na, bool just_take_peerlist = false, uint64_t last_seen_stamp = 0, PeerType peer_type = white, uint64_t first_seen_stamp = 0); size_t get_random_index_with_fixed_probability(size_t max_index); bool is_peer_used(const peerlist_entry& peer); @@ -219,7 +350,7 @@ namespace nodetool template bool try_ping(basic_node_data& node_data, p2p_connection_context& context, const t_callback &cb); bool try_get_support_flags(const p2p_connection_context& context, std::function f); - bool make_expected_connections_count(PeerType peer_type, size_t expected_connections); + bool make_expected_connections_count(network_zone& zone, PeerType peer_type, size_t expected_connections); void cache_connect_fail_info(const epee::net_utils::network_address& addr); bool is_addr_recently_failed(const epee::net_utils::network_address& addr); bool is_priority_node(const epee::net_utils::network_address& na); @@ -232,8 +363,8 @@ namespace nodetool template bool parse_peers_and_add_to_container(const boost::program_options::variables_map& vm, const command_line::arg_descriptor > & arg, Container& container); - bool set_max_out_peers(const boost::program_options::variables_map& vm, int64_t max); - bool set_max_in_peers(const boost::program_options::variables_map& vm, int64_t max); + bool set_max_out_peers(network_zone& zone, int64_t max); + bool set_max_in_peers(network_zone& zone, int64_t max); bool set_tos_flag(const boost::program_options::variables_map& vm, int limit); bool set_rate_up_limit(const boost::program_options::variables_map& vm, int64_t limit); @@ -241,9 +372,15 @@ namespace nodetool bool set_rate_limit(const boost::program_options::variables_map& vm, int64_t limit); bool has_too_many_connections(const epee::net_utils::network_address &address); + uint64_t get_connections_count(); + size_t get_incoming_connections_count(); + size_t get_incoming_connections_count(network_zone&); + size_t get_outgoing_connections_count(); + size_t get_outgoing_connections_count(network_zone&); bool check_connection_and_handshake_with_peer(const epee::net_utils::network_address& na, uint64_t last_seen_stamp); bool gray_peerlist_housekeeping(); + bool check_incoming_connections(); void kill() { ///< will be called e.g. from deinit() _info("Killing the net_node"); @@ -257,31 +394,19 @@ namespace nodetool std::string print_connections_container(); - typedef epee::net_utils::boosted_tcp_server > net_server; - - struct config - { - network_config m_net_config; - uint64_t m_peer_id; - uint32_t m_support_flags; - - BEGIN_KV_SERIALIZE_MAP() - KV_SERIALIZE(m_net_config) - KV_SERIALIZE(m_peer_id) - KV_SERIALIZE(m_support_flags) - END_KV_SERIALIZE_MAP() - }; - public: - config m_config; // TODO was private, add getters? - std::atomic m_current_number_of_out_peers; - std::atomic m_current_number_of_in_peers; void set_save_graph(bool save_graph) { m_save_graph = save_graph; epee::net_utils::connection_basic::set_save_graph(save_graph); } + + void set_rpc_port(uint16_t rpc_port) + { + m_rpc_port = rpc_port; + } + private: std::string m_config_folder; @@ -289,7 +414,7 @@ namespace nodetool bool m_first_connection_maker_call; uint32_t m_listening_port; uint32_t m_external_port; - uint32_t m_ip_address; + uint16_t m_rpc_port; bool m_allow_local_ip; bool m_hide_my_port; bool m_no_igd; @@ -301,26 +426,37 @@ namespace nodetool //connections_indexed_container m_connections; t_payload_net_handler& m_payload_handler; - peerlist_manager m_peerlist; + peerlist_storage m_peerlist_storage; epee::math_helper::once_a_time_seconds m_peer_handshake_idle_maker_interval; epee::math_helper::once_a_time_seconds<1> m_connections_maker_interval; epee::math_helper::once_a_time_seconds<60*30, false> m_peerlist_store_interval; epee::math_helper::once_a_time_seconds<60> m_gray_peerlist_housekeeping_interval; + epee::math_helper::once_a_time_seconds<3600, false> m_incoming_connections_interval; - std::string m_bind_ip; - std::string m_port; #ifdef ALLOW_DEBUG_COMMANDS uint64_t m_last_stat_request_time; #endif std::list m_priority_peers; std::vector m_exclusive_peers; std::vector m_seed_nodes; - std::list m_command_line_peers; + bool m_fallback_seed_nodes_added; + std::vector m_command_line_peers; uint64_t m_peer_livetime; //keep connections to initiate some interactions - net_server m_net_server; - boost::uuids::uuid m_network_id; + + + static boost::optional public_connect(network_zone&, epee::net_utils::network_address const&, epee::net_utils::ssl_support_t); + static boost::optional socks_connect(network_zone&, epee::net_utils::network_address const&, epee::net_utils::ssl_support_t); + + + /* A `std::map` provides constant iterators and key/value pointers even with + inserts/erases to _other_ elements. This makes the configuration step easier + since references can safely be stored on the stack. Do not insert/erase + after configuration and before destruction, lock safety would need to be + added. `std::map::operator[]` WILL insert! */ + std::map m_network_zones; + std::map m_conn_fails_cache; epee::critical_section m_conn_fails_cache_lock; @@ -331,11 +467,17 @@ namespace nodetool epee::critical_section m_host_fails_score_lock; std::map m_host_fails_score; + boost::mutex m_used_stripe_peers_mutex; + std::array, 1 << CRYPTONOTE_PRUNING_LOG_STRIPES> m_used_stripe_peers; + + boost::uuids::uuid m_network_id; cryptonote::network_type m_nettype; + + epee::net_utils::ssl_support_t m_ssl_support; }; - const int64_t default_limit_up = 2048; // kB/s - const int64_t default_limit_down = 8192; // kB/s + const int64_t default_limit_up = P2P_DEFAULT_LIMIT_RATE_UP; // kB/s + const int64_t default_limit_down = P2P_DEFAULT_LIMIT_RATE_DOWN; // kB/s extern const command_line::arg_descriptor arg_p2p_bind_ip; extern const command_line::arg_descriptor arg_p2p_bind_port; extern const command_line::arg_descriptor arg_p2p_external_port; @@ -344,7 +486,10 @@ namespace nodetool extern const command_line::arg_descriptor > arg_p2p_add_priority_node; extern const command_line::arg_descriptor > arg_p2p_add_exclusive_node; extern const command_line::arg_descriptor > arg_p2p_seed_node; + extern const command_line::arg_descriptor > arg_proxy; + extern const command_line::arg_descriptor > arg_anonymous_inbound; extern const command_line::arg_descriptor arg_p2p_hide_my_port; + extern const command_line::arg_descriptor arg_no_sync; extern const command_line::arg_descriptor arg_no_igd; extern const command_line::arg_descriptor arg_offline; @@ -360,3 +505,4 @@ namespace nodetool } POP_WARNINGS + diff --git a/src/p2p/net_node.inl b/src/p2p/net_node.inl index 9b21705ec15..cec832231d3 100644 --- a/src/p2p/net_node.inl +++ b/src/p2p/net_node.inl @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -31,23 +31,34 @@ // IP blocking adapted from Boolberry #include +#include #include +#include +#include #include -#include +#include #include +#include +#include +#include +#include +#include #include "version.h" #include "string_tools.h" #include "common/util.h" #include "common/dns_utils.h" +#include "common/pruning.h" +#include "net/error.h" #include "net/net_helper.h" #include "math_helper.h" +#include "misc_log_ex.h" #include "p2p_protocol_defs.h" -#include "net_peerlist_boost_serialization.h" #include "net/local_ip.h" #include "crypto/crypto.h" #include "storages/levin_abstract_invoke2.h" #include "cryptonote_core/cryptonote_core.h" +#include "net/parse.h" #include #include @@ -62,6 +73,21 @@ namespace nodetool { + template + node_server::~node_server() + { + // tcp server uses io_service in destructor, and every zone uses + // io_service from public zone. + for (auto current = m_network_zones.begin(); current != m_network_zones.end(); /* below */) + { + if (current->first != epee::net_utils::zone::public_) + current = m_network_zones.erase(current); + else + ++current; + } + } + //----------------------------------------------------------------------------------- + inline bool append_net_address(std::vector & seed_nodes, std::string const & addr, uint16_t default_port); //----------------------------------------------------------------------------------- template void node_server::init_options(boost::program_options::options_description& desc) @@ -74,7 +100,10 @@ namespace nodetool command_line::add_arg(desc, arg_p2p_add_priority_node); command_line::add_arg(desc, arg_p2p_add_exclusive_node); command_line::add_arg(desc, arg_p2p_seed_node); + command_line::add_arg(desc, arg_proxy); + command_line::add_arg(desc, arg_anonymous_inbound); command_line::add_arg(desc, arg_p2p_hide_my_port); + command_line::add_arg(desc, arg_no_sync); command_line::add_arg(desc, arg_no_igd); command_line::add_arg(desc, arg_out_peers); command_line::add_arg(desc, arg_in_peers); @@ -88,62 +117,14 @@ namespace nodetool template bool node_server::init_config() { - // TRY_ENTRY(); - std::string state_file_path = m_config_folder + "/" + P2P_NET_DATA_FILENAME; - std::ifstream p2p_data; - p2p_data.open( state_file_path , std::ios_base::binary | std::ios_base::in); - if(!p2p_data.fail()) - { - try - { - // first try reading in portable mode - boost::archive::portable_binary_iarchive a(p2p_data); - a >> *this; - } - catch (...) - { - // if failed, try reading in unportable mode - boost::filesystem::copy_file(state_file_path, state_file_path + ".unportable", boost::filesystem::copy_option::overwrite_if_exists); - p2p_data.close(); - p2p_data.open( state_file_path , std::ios_base::binary | std::ios_base::in); - if(!p2p_data.fail()) - { - try - { - boost::archive::binary_iarchive a(p2p_data); - a >> *this; - } - catch (const std::exception &e) - { - MWARNING("Failed to load p2p config file, falling back to default config"); - m_peerlist = peerlist_manager(); // it was probably half clobbered by the failed load - make_default_config(); - } - } - else - { - make_default_config(); - } - } - }else - { - make_default_config(); - } - - // always recreate a new peer id - make_default_peer_id(); - - //at this moment we have hardcoded config - m_config.m_net_config.handshake_interval = P2P_DEFAULT_HANDSHAKE_INTERVAL; - m_config.m_net_config.packet_max_size = P2P_DEFAULT_PACKET_MAX_SIZE; //20 MB limit - m_config.m_net_config.config_id = 0; // initial config - m_config.m_net_config.connection_timeout = P2P_DEFAULT_CONNECTION_TIMEOUT; - m_config.m_net_config.ping_connection_timeout = P2P_DEFAULT_PING_CONNECTION_TIMEOUT; - m_config.m_net_config.send_peerlist_sz = P2P_DEFAULT_PEERS_IN_HANDSHAKE; - m_config.m_support_flags = P2P_SUPPORT_FLAGS; + auto storage = peerlist_storage::open(m_config_folder + "/" + P2P_NET_DATA_FILENAME); + if (storage) + m_peerlist_storage = std::move(*storage); + m_network_zones[epee::net_utils::zone::public_].m_config.m_support_flags = P2P_SUPPORT_FLAGS; m_first_connection_maker_call = true; + CATCH_ENTRY_L0("node_server::init_config", false); return true; } @@ -151,17 +132,26 @@ namespace nodetool template void node_server::for_each_connection(std::function f) { - m_net_server.get_config_object().foreach_connection([&](p2p_connection_context& cntx){ - return f(cntx, cntx.peer_id, cntx.support_flags); - }); + for(auto& zone : m_network_zones) + { + zone.second.m_net_server.get_config_object().foreach_connection([&](p2p_connection_context& cntx){ + return f(cntx, cntx.peer_id, cntx.support_flags); + }); + } } //----------------------------------------------------------------------------------- template bool node_server::for_connection(const boost::uuids::uuid &connection_id, std::function f) { - return m_net_server.get_config_object().for_connection(connection_id, [&](p2p_connection_context& cntx){ - return f(cntx, cntx.peer_id, cntx.support_flags); - }); + for(auto& zone : m_network_zones) + { + const bool result = zone.second.m_net_server.get_config_object().for_connection(connection_id, [&](p2p_connection_context& cntx){ + return f(cntx, cntx.peer_id, cntx.support_flags); + }); + if (result) + return true; + } + return false; } //----------------------------------------------------------------------------------- template @@ -181,36 +171,40 @@ namespace nodetool } //----------------------------------------------------------------------------------- template - bool node_server::make_default_peer_id() - { - m_config.m_peer_id = crypto::rand(); - return true; - } - //----------------------------------------------------------------------------------- - template - bool node_server::make_default_config() - { - return make_default_peer_id(); - } - //----------------------------------------------------------------------------------- - template bool node_server::block_host(const epee::net_utils::network_address &addr, time_t seconds) { + if(!addr.is_blockable()) + return false; + + const time_t now = time(nullptr); + CRITICAL_REGION_LOCAL(m_blocked_hosts_lock); - m_blocked_hosts[addr.host_str()] = time(nullptr) + seconds; + time_t limit; + if (now > std::numeric_limits::max() - seconds) + limit = std::numeric_limits::max(); + else + limit = now + seconds; + m_blocked_hosts[addr.host_str()] = limit; - // drop any connection to that IP - std::list conns; - m_net_server.get_config_object().foreach_connection([&](const p2p_connection_context& cntxt) + // drop any connection to that address. This should only have to look into + // the zone related to the connection, but really make sure everything is + // swept ... + std::vector conns; + for(auto& zone : m_network_zones) { - if (cntxt.m_remote_address.is_same_host(addr)) + zone.second.m_net_server.get_config_object().foreach_connection([&](const p2p_connection_context& cntxt) { - conns.push_back(cntxt.m_connection_id); - } - return true; - }); - for (const auto &c: conns) - m_net_server.get_config_object().close(c); + if (cntxt.m_remote_address.is_same_host(addr)) + { + conns.push_back(cntxt.m_connection_id); + } + return true; + }); + for (const auto &c: conns) + zone.second.m_net_server.get_config_object().close(c); + + conns.clear(); + } MCLOG_CYAN(el::Level::Info, "global", "Host " << addr.host_str() << " blocked."); return true; @@ -231,6 +225,9 @@ namespace nodetool template bool node_server::add_host_fail(const epee::net_utils::network_address &address) { + if(!address.is_blockable()) + return false; + CRITICAL_REGION_LOCAL(m_host_fails_score_lock); uint64_t fails = ++m_host_fails_score[address.host_str()]; MDEBUG("Host " << address.host_str() << " fail score=" << fails); @@ -245,12 +242,6 @@ namespace nodetool } //----------------------------------------------------------------------------------- template - bool node_server::parse_peer_from_string(epee::net_utils::network_address& pe, const std::string& node_addr, uint16_t default_port) - { - return epee::net_utils::create_network_address(pe, node_addr, default_port); - } - //----------------------------------------------------------------------------------- - template bool node_server::handle_command_line( const boost::program_options::variables_map& vm ) @@ -259,8 +250,11 @@ namespace nodetool bool stagenet = command_line::get_arg(vm, cryptonote::arg_stagenet_on); m_nettype = testnet ? cryptonote::TESTNET : stagenet ? cryptonote::STAGENET : cryptonote::MAINNET; - m_bind_ip = command_line::get_arg(vm, arg_p2p_bind_ip); - m_port = command_line::get_arg(vm, arg_p2p_bind_port); + network_zone& public_zone = m_network_zones[epee::net_utils::zone::public_]; + public_zone.m_connect = &public_connect; + public_zone.m_bind_ip = command_line::get_arg(vm, arg_p2p_bind_ip); + public_zone.m_port = command_line::get_arg(vm, arg_p2p_bind_port); + public_zone.m_can_pingback = true; m_external_port = command_line::get_arg(vm, arg_p2p_external_port); m_allow_local_ip = command_line::get_arg(vm, arg_p2p_allow_local_ip); m_no_igd = command_line::get_arg(vm, arg_no_igd); @@ -273,10 +267,28 @@ namespace nodetool { nodetool::peerlist_entry pe = AUTO_VAL_INIT(pe); pe.id = crypto::rand(); - const uint16_t default_port = testnet ? ::config::testnet::P2P_DEFAULT_PORT : stagenet ? ::config::stagenet::P2P_DEFAULT_PORT : ::config::P2P_DEFAULT_PORT; - bool r = parse_peer_from_string(pe.adr, pr_str, default_port); - CHECK_AND_ASSERT_MES(r, false, "Failed to parse address from string: " << pr_str); - m_command_line_peers.push_back(pe); + const uint16_t default_port = cryptonote::get_config(m_nettype).P2P_DEFAULT_PORT; + expect adr = net::get_network_address(pr_str, default_port); + if (adr) + { + add_zone(adr->get_zone()); + pe.adr = std::move(*adr); + m_command_line_peers.push_back(std::move(pe)); + continue; + } + CHECK_AND_ASSERT_MES( + adr == net::error::unsupported_address, false, "Bad address (\"" << pr_str << "\"): " << adr.error().message() + ); + + std::vector resolved_addrs; + bool r = append_net_address(resolved_addrs, pr_str, default_port); + CHECK_AND_ASSERT_MES(r, false, "Failed to parse or resolve address from string: " << pr_str); + for (const epee::net_utils::network_address& addr : resolved_addrs) + { + pe.id = crypto::rand(); + pe.adr = addr; + m_command_line_peers.push_back(pe); + } } } @@ -306,10 +318,16 @@ namespace nodetool if(command_line::has_arg(vm, arg_p2p_hide_my_port)) m_hide_my_port = true; - if ( !set_max_out_peers(vm, command_line::get_arg(vm, arg_out_peers) ) ) + if (command_line::has_arg(vm, arg_no_sync)) + m_payload_handler.set_no_sync(true); + + if ( !set_max_out_peers(public_zone, command_line::get_arg(vm, arg_out_peers) ) ) return false; + else + m_payload_handler.set_max_out_peers(public_zone.m_config.m_net_config.max_out_connection_count); + - if ( !set_max_in_peers(vm, command_line::get_arg(vm, arg_in_peers) ) ) + if ( !set_max_in_peers(public_zone, command_line::get_arg(vm, arg_in_peers) ) ) return false; if ( !set_tos_flag(vm, command_line::get_arg(vm, arg_tos_flag) ) ) @@ -324,27 +342,86 @@ namespace nodetool if ( !set_rate_limit(vm, command_line::get_arg(vm, arg_limit_rate) ) ) return false; + + auto proxies = get_proxies(vm); + if (!proxies) + return false; + + for (auto& proxy : *proxies) + { + network_zone& zone = add_zone(proxy.zone); + if (zone.m_connect != nullptr) + { + MERROR("Listed --" << arg_proxy.name << " twice with " << epee::net_utils::zone_to_string(proxy.zone)); + return false; + } + zone.m_connect = &socks_connect; + zone.m_proxy_address = std::move(proxy.address); + + if (!set_max_out_peers(zone, proxy.max_connections)) + return false; + } + + for (const auto& zone : m_network_zones) + { + if (zone.second.m_connect == nullptr) + { + MERROR("Set outgoing peer for " << epee::net_utils::zone_to_string(zone.first) << " but did not set --" << arg_proxy.name); + return false; + } + } + + auto inbounds = get_anonymous_inbounds(vm); + if (!inbounds) + return false; + + for (auto& inbound : *inbounds) + { + network_zone& zone = add_zone(inbound.our_address.get_zone()); + + if (!zone.m_bind_ip.empty()) + { + MERROR("Listed --" << arg_anonymous_inbound.name << " twice with " << epee::net_utils::zone_to_string(inbound.our_address.get_zone()) << " network"); + return false; + } + + zone.m_bind_ip = std::move(inbound.local_ip); + zone.m_port = std::move(inbound.local_port); + zone.m_net_server.set_default_remote(std::move(inbound.default_remote)); + zone.m_our_address = std::move(inbound.our_address); + + if (!set_max_in_peers(zone, inbound.max_connections)) + return false; + } + return true; } //----------------------------------------------------------------------------------- - inline void append_net_address( + inline bool append_net_address( std::vector & seed_nodes , std::string const & addr + , uint16_t default_port ) { using namespace boost::asio; + std::string host = addr; + std::string port = std::to_string(default_port); size_t pos = addr.find_last_of(':'); - CHECK_AND_ASSERT_MES_NO_RET(std::string::npos != pos && addr.length() - 1 != pos && 0 != pos, "Failed to parse seed address from string: '" << addr << '\''); - std::string host = addr.substr(0, pos); - std::string port = addr.substr(pos + 1); + if (std::string::npos != pos) + { + CHECK_AND_ASSERT_MES(addr.length() - 1 != pos && 0 != pos, false, "Failed to parse seed address from string: '" << addr << '\''); + host = addr.substr(0, pos); + port = addr.substr(pos + 1); + } + MINFO("Resolving node address: host=" << host << ", port=" << port); io_service io_srv; ip::tcp::resolver resolver(io_srv); ip::tcp::resolver::query query(host, port, boost::asio::ip::tcp::resolver::query::canonical_name); boost::system::error_code ec; ip::tcp::resolver::iterator i = resolver.resolve(query, ec); - CHECK_AND_ASSERT_MES_NO_RET(!ec, "Failed to resolve host name '" << host << "': " << ec.message() << ':' << ec.value()); + CHECK_AND_ASSERT_MES(!ec, false, "Failed to resolve host name '" << host << "': " << ec.message() << ':' << ec.value()); ip::tcp::resolver::iterator iend; for (; i != iend; ++i) @@ -354,14 +431,14 @@ namespace nodetool { epee::net_utils::network_address na{epee::net_utils::ipv4_network_address{boost::asio::detail::socket_ops::host_to_network_long(endpoint.address().to_v4().to_ulong()), endpoint.port()}}; seed_nodes.push_back(na); - MINFO("Added seed node: " << na.str()); + MINFO("Added node: " << na.str()); } else { MWARNING("IPv6 unsupported, skip '" << host << "' -> " << endpoint.address().to_v6().to_string(ec)); - throw std::runtime_error("IPv6 unsupported"); } } + return true; } //----------------------------------------------------------------------------------- @@ -382,6 +459,9 @@ namespace nodetool full_addrs.insert("162.210.173.150:38080"); full_addrs.insert("162.210.173.151:38080"); } + else if (nettype == cryptonote::FAKECHAIN) + { + } else { full_addrs.insert("107.152.130.98:18080"); @@ -395,7 +475,17 @@ namespace nodetool } return full_addrs; } + //----------------------------------------------------------------------------------- + template + typename node_server::network_zone& node_server::add_zone(const epee::net_utils::zone zone) + { + const auto zone_ = m_network_zones.lower_bound(zone); + if (zone_ != m_network_zones.end() && zone_->first == zone) + return zone_->second; + network_zone& public_zone = m_network_zones[epee::net_utils::zone::public_]; + return m_network_zones.emplace_hint(zone_, std::piecewise_construct, std::make_tuple(zone), std::tie(public_zone.m_net_server.get_io_service()))->second; + } //----------------------------------------------------------------------------------- template bool node_server::init(const boost::program_options::variables_map& vm) @@ -405,6 +495,7 @@ namespace nodetool bool res = handle_command_line(vm); CHECK_AND_ASSERT_MES(res, false, "Failed to handle command line"); + m_fallback_seed_nodes_added = false; if (m_nettype == cryptonote::TESTNET) { memcpy(&m_network_id, &::config::testnet::NETWORK_ID, 16); @@ -418,7 +509,7 @@ namespace nodetool else { memcpy(&m_network_id, &::config::NETWORK_ID, 16); - if (m_exclusive_peers.empty()) + if (m_exclusive_peers.empty() && !m_offline) { // for each hostname in the seed nodes list, attempt to DNS resolve and // add the result addresses as seed nodes @@ -483,7 +574,7 @@ namespace nodetool if (result.size()) { for (const auto& addr_string : result) - full_addrs.insert(addr_string + ":" + std::to_string(m_nettype == cryptonote::TESTNET ? ::config::testnet::P2P_DEFAULT_PORT : m_nettype == cryptonote::STAGENET ? ::config::stagenet::P2P_DEFAULT_PORT : ::config::P2P_DEFAULT_PORT)); + full_addrs.insert(addr_string + ":" + std::to_string(cryptonote::get_config(m_nettype).P2P_DEFAULT_PORT)); } ++i; } @@ -498,6 +589,7 @@ namespace nodetool for (const auto &peer: get_seed_nodes(cryptonote::MAINNET)) full_addrs.insert(peer); + m_fallback_seed_nodes_added = true; } } } @@ -505,50 +597,87 @@ namespace nodetool for (const auto& full_addr : full_addrs) { MDEBUG("Seed node: " << full_addr); - append_net_address(m_seed_nodes, full_addr); + append_net_address(m_seed_nodes, full_addr, cryptonote::get_config(m_nettype).P2P_DEFAULT_PORT); } MDEBUG("Number of seed nodes: " << m_seed_nodes.size()); m_config_folder = command_line::get_arg(vm, cryptonote::arg_data_dir); + network_zone& public_zone = m_network_zones.at(epee::net_utils::zone::public_); - if ((m_nettype == cryptonote::MAINNET && m_port != std::to_string(::config::P2P_DEFAULT_PORT)) - || (m_nettype == cryptonote::TESTNET && m_port != std::to_string(::config::testnet::P2P_DEFAULT_PORT)) - || (m_nettype == cryptonote::STAGENET && m_port != std::to_string(::config::stagenet::P2P_DEFAULT_PORT))) { - m_config_folder = m_config_folder + "/" + m_port; + if ((m_nettype == cryptonote::MAINNET && public_zone.m_port != std::to_string(::config::P2P_DEFAULT_PORT)) + || (m_nettype == cryptonote::TESTNET && public_zone.m_port != std::to_string(::config::testnet::P2P_DEFAULT_PORT)) + || (m_nettype == cryptonote::STAGENET && public_zone.m_port != std::to_string(::config::stagenet::P2P_DEFAULT_PORT))) { + m_config_folder = m_config_folder + "/" + public_zone.m_port; } res = init_config(); CHECK_AND_ASSERT_MES(res, false, "Failed to init config."); - res = m_peerlist.init(m_allow_local_ip); - CHECK_AND_ASSERT_MES(res, false, "Failed to init peerlist."); + for (auto& zone : m_network_zones) + { + res = zone.second.m_peerlist.init(m_peerlist_storage.take_zone(zone.first), m_allow_local_ip); + CHECK_AND_ASSERT_MES(res, false, "Failed to init peerlist."); + } + for(const auto& p: m_command_line_peers) + m_network_zones.at(p.adr.get_zone()).m_peerlist.append_with_peer_white(p); - for(auto& p: m_command_line_peers) - m_peerlist.append_with_peer_white(p); +// all peers are now setup +#ifdef CRYPTONOTE_PRUNING_DEBUG_SPOOF_SEED + for (auto& zone : m_network_zones) + { + std::list plw; + while (zone.second.m_peerlist.get_white_peers_count()) + { + plw.push_back(peerlist_entry()); + zone.second.m_peerlist.get_white_peer_by_index(plw.back(), 0); + zone.second.m_peerlist.remove_from_peer_white(plw.back()); + } + for (auto &e:plw) + zone.second.m_peerlist.append_with_peer_white(e); + + std::list plg; + while (zone.second.m_peerlist.get_gray_peers_count()) + { + plg.push_back(peerlist_entry()); + zone.second.m_peerlist.get_gray_peer_by_index(plg.back(), 0); + zone.second.m_peerlist.remove_from_peer_gray(plg.back()); + } + for (auto &e:plg) + zone.second.m_peerlist.append_with_peer_gray(e); + } +#endif //only in case if we really sure that we have external visible ip m_have_address = true; - m_ip_address = 0; m_last_stat_request_time = 0; //configure self - m_net_server.set_threads_prefix("P2P"); - m_net_server.get_config_object().set_handler(this); - m_net_server.get_config_object().m_invoke_timeout = P2P_DEFAULT_INVOKE_TIMEOUT; - m_net_server.set_connection_filter(this); + + public_zone.m_net_server.set_threads_prefix("P2P"); // all zones use these threads/asio::io_service // from here onwards, it's online stuff if (m_offline) return res; //try to bind - MINFO("Binding on " << m_bind_ip << ":" << m_port); - res = m_net_server.init_server(m_port, m_bind_ip); - CHECK_AND_ASSERT_MES(res, false, "Failed to bind server"); + m_ssl_support = epee::net_utils::ssl_support_t::e_ssl_support_disabled; + for (auto& zone : m_network_zones) + { + zone.second.m_net_server.get_config_object().set_handler(this); + zone.second.m_net_server.get_config_object().m_invoke_timeout = P2P_DEFAULT_INVOKE_TIMEOUT; - m_listening_port = m_net_server.get_binded_port(); - MLOG_GREEN(el::Level::Info, "Net service bound to " << m_bind_ip << ":" << m_listening_port); + if (!zone.second.m_bind_ip.empty()) + { + zone.second.m_net_server.set_connection_filter(this); + MINFO("Binding on " << zone.second.m_bind_ip << ":" << zone.second.m_port); + res = zone.second.m_net_server.init_server(zone.second.m_port, zone.second.m_bind_ip, epee::net_utils::ssl_support_t::e_ssl_support_disabled); + CHECK_AND_ASSERT_MES(res, false, "Failed to bind server"); + } + } + + m_listening_port = public_zone.m_net_server.get_binded_port(); + MLOG_GREEN(el::Level::Info, "Net service bound to " << public_zone.m_bind_ip << ":" << m_listening_port); if(m_external_port) MDEBUG("External port defined as " << m_external_port); @@ -572,44 +701,45 @@ namespace nodetool mPeersLoggerThread.reset(new boost::thread([&]() { _note("Thread monitor number of peers - start"); - while (!is_closing && !m_net_server.is_stop_signal_sent()) + const network_zone& public_zone = m_network_zones.at(epee::net_utils::zone::public_); + while (!is_closing && !public_zone.m_net_server.is_stop_signal_sent()) { // main loop of thread //number_of_peers = m_net_server.get_config_object().get_connections_count(); - unsigned int number_of_in_peers = 0; - unsigned int number_of_out_peers = 0; - m_net_server.get_config_object().foreach_connection([&](const p2p_connection_context& cntxt) + for (auto& zone : m_network_zones) { - if (cntxt.m_is_income) + unsigned int number_of_in_peers = 0; + unsigned int number_of_out_peers = 0; + zone.second.m_net_server.get_config_object().foreach_connection([&](const p2p_connection_context& cntxt) { - ++number_of_in_peers; - } - else - { - ++number_of_out_peers; - } - return true; - }); // lambda - - m_current_number_of_in_peers = number_of_in_peers; - m_current_number_of_out_peers = number_of_out_peers; - + if (cntxt.m_is_income) + { + ++number_of_in_peers; + } + else + { + ++number_of_out_peers; + } + return true; + }); // lambda + zone.second.m_current_number_of_in_peers = number_of_in_peers; + zone.second.m_current_number_of_out_peers = number_of_out_peers; + } boost::this_thread::sleep_for(boost::chrono::seconds(1)); } // main loop of thread _note("Thread monitor number of peers - done"); })); // lambda + network_zone& public_zone = m_network_zones.at(epee::net_utils::zone::public_); + public_zone.m_net_server.add_idle_handler(boost::bind(&node_server::idle_worker, this), 1000); + public_zone.m_net_server.add_idle_handler(boost::bind(&t_payload_net_handler::on_idle, &m_payload_handler), 1000); + //here you can set worker threads count int thrds_count = 10; - - m_net_server.add_idle_handler(boost::bind(&node_server::idle_worker, this), 1000); - m_net_server.add_idle_handler(boost::bind(&t_payload_net_handler::on_idle, &m_payload_handler), 1000); - boost::thread::attributes attrs; attrs.set_stack_size(THREAD_STACK_SIZE); - //go to loop MINFO("Run net_service loop( " << thrds_count << " threads)..."); - if(!m_net_server.run_server(thrds_count, true, attrs)) + if(!public_zone.m_net_server.run_server(thrds_count, true, attrs)) { LOG_ERROR("Failed to run net tcp server!"); } @@ -617,51 +747,63 @@ namespace nodetool MINFO("net_service loop stopped."); return true; } - + //----------------------------------------------------------------------------------- + template + uint64_t node_server::get_public_connections_count() + { + auto public_zone = m_network_zones.find(epee::net_utils::zone::public_); + if (public_zone == m_network_zones.end()) + return 0; + return public_zone->second.m_net_server.get_config_object().get_connections_count(); + } //----------------------------------------------------------------------------------- template uint64_t node_server::get_connections_count() { - return m_net_server.get_config_object().get_connections_count(); + std::uint64_t count = 0; + for (auto& zone : m_network_zones) + count += zone.second.m_net_server.get_config_object().get_connections_count(); + return count; } //----------------------------------------------------------------------------------- template bool node_server::deinit() { kill(); - m_peerlist.deinit(); - m_net_server.deinit_server(); - // remove UPnP port mapping - if(!m_no_igd) - delete_upnp_port_mapping(m_listening_port); + + if (!m_offline) + { + for(auto& zone : m_network_zones) + zone.second.m_net_server.deinit_server(); + // remove UPnP port mapping + if(!m_no_igd) + delete_upnp_port_mapping(m_listening_port); + } return store_config(); } //----------------------------------------------------------------------------------- template bool node_server::store_config() { - TRY_ENTRY(); + if (!tools::create_directories_if_necessary(m_config_folder)) { - MWARNING("Failed to create data directory: " << m_config_folder); + MWARNING("Failed to create data directory \"" << m_config_folder); return false; } - std::string state_file_path = m_config_folder + "/" + P2P_NET_DATA_FILENAME; - std::ofstream p2p_data; - p2p_data.open( state_file_path , std::ios_base::binary | std::ios_base::out| std::ios::trunc); - if(p2p_data.fail()) + peerlist_types active{}; + for (auto& zone : m_network_zones) + zone.second.m_peerlist.get_peerlist(active); + + const std::string state_file_path = m_config_folder + "/" + P2P_NET_DATA_FILENAME; + if (!m_peerlist_storage.store(state_file_path, active)) { MWARNING("Failed to save config to file " << state_file_path); return false; - }; - - boost::archive::portable_binary_oarchive a(p2p_data); - a << *this; - return true; - CATCH_ENTRY_L0("blockchain_storage::save", false); - + } + CATCH_ENTRY_L0("node_server::store", false); return true; } //----------------------------------------------------------------------------------- @@ -669,37 +811,39 @@ namespace nodetool bool node_server::send_stop_signal() { MDEBUG("[node] sending stop signal"); - m_net_server.send_stop_signal(); + for (auto& zone : m_network_zones) + zone.second.m_net_server.send_stop_signal(); MDEBUG("[node] Stop signal sent"); - std::list connection_ids; - m_net_server.get_config_object().foreach_connection([&](const p2p_connection_context& cntxt) { - connection_ids.push_back(cntxt.m_connection_id); - return true; - }); - for (const auto &connection_id: connection_ids) - m_net_server.get_config_object().close(connection_id); - + for (auto& zone : m_network_zones) + { + std::list connection_ids; + zone.second.m_net_server.get_config_object().foreach_connection([&](const p2p_connection_context& cntxt) { + connection_ids.push_back(cntxt.m_connection_id); + return true; + }); + for (const auto &connection_id: connection_ids) + zone.second.m_net_server.get_config_object().close(connection_id); + } m_payload_handler.stop(); - return true; } //----------------------------------------------------------------------------------- - - template bool node_server::do_handshake_with_peer(peerid_type& pi, p2p_connection_context& context_, bool just_take_peerlist) { + network_zone& zone = m_network_zones.at(context_.m_remote_address.get_zone()); + typename COMMAND_HANDSHAKE::request arg; typename COMMAND_HANDSHAKE::response rsp; - get_local_node_data(arg.node_data); + get_local_node_data(arg.node_data, zone); m_payload_handler.get_payload_sync_data(arg.payload_data); epee::simple_event ev; std::atomic hsh_result(false); - bool r = epee::net_utils::async_invoke_remote_command2(context_.m_connection_id, COMMAND_HANDSHAKE::ID, arg, m_net_server.get_config_object(), - [this, &pi, &ev, &hsh_result, &just_take_peerlist](int code, const typename COMMAND_HANDSHAKE::response& rsp, p2p_connection_context& context) + bool r = epee::net_utils::async_invoke_remote_command2(context_.m_connection_id, COMMAND_HANDSHAKE::ID, arg, zone.m_net_server.get_config_object(), + [this, &pi, &ev, &hsh_result, &just_take_peerlist, &context_](int code, const typename COMMAND_HANDSHAKE::response& rsp, p2p_connection_context& context) { epee::misc_utils::auto_scope_leave_caller scope_exit_handler = epee::misc_utils::create_scope_leave_handler([&](){ev.raise();}); @@ -711,7 +855,7 @@ namespace nodetool if(rsp.node_data.network_id != m_network_id) { - LOG_WARNING_CC(context, "COMMAND_HANDSHAKE Failed, wrong network! (" << epee::string_tools::get_str_from_guid_a(rsp.node_data.network_id) << "), closing connection."); + LOG_WARNING_CC(context, "COMMAND_HANDSHAKE Failed, wrong network! (" << rsp.node_data.network_id << "), closing connection."); return; } @@ -732,19 +876,26 @@ namespace nodetool } pi = context.peer_id = rsp.node_data.peer_id; - m_peerlist.set_peer_just_seen(rsp.node_data.peer_id, context.m_remote_address); + context.m_rpc_port = rsp.node_data.rpc_port; + m_network_zones.at(context.m_remote_address.get_zone()).m_peerlist.set_peer_just_seen(rsp.node_data.peer_id, context.m_remote_address, context.m_pruning_seed, context.m_rpc_port); - if(rsp.node_data.peer_id == m_config.m_peer_id) + // move + for (auto const& zone : m_network_zones) { - LOG_DEBUG_CC(context, "Connection to self detected, dropping connection"); - hsh_result = false; - return; + if(rsp.node_data.peer_id == zone.second.m_config.m_peer_id) + { + LOG_DEBUG_CC(context, "Connection to self detected, dropping connection"); + hsh_result = false; + return; + } } + LOG_INFO_CC(context, "New connection handshaked, pruning seed " << epee::string_tools::to_string_hex(context.m_pruning_seed)); LOG_DEBUG_CC(context, " COMMAND_HANDSHAKE INVOKED OK"); }else { LOG_DEBUG_CC(context, " COMMAND_HANDSHAKE(AND CLOSE) INVOKED OK"); } + context_ = context; }, P2P_DEFAULT_HANDSHAKE_INVOKE_TIMEOUT); if(r) @@ -755,7 +906,7 @@ namespace nodetool if(!hsh_result) { LOG_WARNING_CC(context_, "COMMAND_HANDSHAKE Failed"); - m_net_server.get_config_object().close(context_.m_connection_id); + m_network_zones.at(context_.m_remote_address.get_zone()).m_net_server.get_config_object().close(context_.m_connection_id); } else { @@ -774,7 +925,8 @@ namespace nodetool typename COMMAND_TIMED_SYNC::request arg = AUTO_VAL_INIT(arg); m_payload_handler.get_payload_sync_data(arg.payload_data); - bool r = epee::net_utils::async_invoke_remote_command2(context_.m_connection_id, COMMAND_TIMED_SYNC::ID, arg, m_net_server.get_config_object(), + network_zone& zone = m_network_zones.at(context_.m_remote_address.get_zone()); + bool r = epee::net_utils::async_invoke_remote_command2(context_.m_connection_id, COMMAND_TIMED_SYNC::ID, arg, zone.m_net_server.get_config_object(), [this](int code, const typename COMMAND_TIMED_SYNC::response& rsp, p2p_connection_context& context) { context.m_in_timedsync = false; @@ -787,12 +939,15 @@ namespace nodetool if(!handle_remote_peerlist(rsp.local_peerlist_new, rsp.local_time, context)) { LOG_WARNING_CC(context, "COMMAND_TIMED_SYNC: failed to handle_remote_peerlist(...), closing connection."); - m_net_server.get_config_object().close(context.m_connection_id ); + m_network_zones.at(context.m_remote_address.get_zone()).m_net_server.get_config_object().close(context.m_connection_id ); add_host_fail(context.m_remote_address); } if(!context.m_is_income) - m_peerlist.set_peer_just_seen(context.peer_id, context.m_remote_address); - m_payload_handler.process_payload_sync_data(rsp.payload_data, context, false); + m_network_zones.at(context.m_remote_address.get_zone()).m_peerlist.set_peer_just_seen(context.peer_id, context.m_remote_address, context.m_pruning_seed, context.m_rpc_port); + if (!m_payload_handler.process_payload_sync_data(rsp.payload_data, context, false)) + { + m_network_zones.at(context.m_remote_address.get_zone()).m_net_server.get_config_object().close(context.m_connection_id ); + } }); if(!r) @@ -819,53 +974,61 @@ namespace nodetool template bool node_server::is_peer_used(const peerlist_entry& peer) { - - if(m_config.m_peer_id == peer.id) - return true;//dont make connections to ourself + for(const auto& zone : m_network_zones) + if(zone.second.m_config.m_peer_id == peer.id) + return true;//dont make connections to ourself bool used = false; - m_net_server.get_config_object().foreach_connection([&](const p2p_connection_context& cntxt) + for(auto& zone : m_network_zones) { - if(cntxt.peer_id == peer.id || (!cntxt.m_is_income && peer.adr == cntxt.m_remote_address)) + zone.second.m_net_server.get_config_object().foreach_connection([&](const p2p_connection_context& cntxt) { - used = true; - return false;//stop enumerating - } - return true; - }); + if(cntxt.peer_id == peer.id || (!cntxt.m_is_income && peer.adr == cntxt.m_remote_address)) + { + used = true; + return false;//stop enumerating + } + return true; + }); - return used; + if(used) + return true; + } + return false; } //----------------------------------------------------------------------------------- template bool node_server::is_peer_used(const anchor_peerlist_entry& peer) { - if(m_config.m_peer_id == peer.id) { - return true;//dont make connections to ourself - } - - bool used = false; - - m_net_server.get_config_object().foreach_connection([&](const p2p_connection_context& cntxt) - { - if(cntxt.peer_id == peer.id || (!cntxt.m_is_income && peer.adr == cntxt.m_remote_address)) - { - used = true; - - return false;//stop enumerating + for(auto& zone : m_network_zones) { + if(zone.second.m_config.m_peer_id == peer.id) { + return true;//dont make connections to ourself } - - return true; - }); - - return used; + bool used = false; + zone.second.m_net_server.get_config_object().foreach_connection([&](const p2p_connection_context& cntxt) + { + if(cntxt.peer_id == peer.id || (!cntxt.m_is_income && peer.adr == cntxt.m_remote_address)) + { + used = true; + return false;//stop enumerating + } + return true; + }); + if (used) + return true; + } + return false; } //----------------------------------------------------------------------------------- template bool node_server::is_addr_connected(const epee::net_utils::network_address& peer) { + const auto zone = m_network_zones.find(peer.get_zone()); + if (zone == m_network_zones.end()) + return false; + bool connected = false; - m_net_server.get_config_object().foreach_connection([&](const p2p_connection_context& cntxt) + zone->second.m_net_server.get_config_object().foreach_connection([&](const p2p_connection_context& cntxt) { if(!cntxt.m_is_income && peer == cntxt.m_remote_address) { @@ -890,55 +1053,54 @@ namespace nodetool template bool node_server::try_to_connect_and_handshake_with_new_peer(const epee::net_utils::network_address& na, bool just_take_peerlist, uint64_t last_seen_stamp, PeerType peer_type, uint64_t first_seen_stamp) { - if (m_current_number_of_out_peers == m_config.m_net_config.max_out_connection_count) // out peers limit + network_zone& zone = m_network_zones.at(na.get_zone()); + if (zone.m_connect == nullptr) // outgoing connections in zone not possible + return false; + + if (zone.m_current_number_of_out_peers == zone.m_config.m_net_config.max_out_connection_count) // out peers limit { return false; } - else if (m_current_number_of_out_peers > m_config.m_net_config.max_out_connection_count) + else if (zone.m_current_number_of_out_peers > zone.m_config.m_net_config.max_out_connection_count) { - m_net_server.get_config_object().del_out_connections(1); - m_current_number_of_out_peers --; // atomic variable, update time = 1s + zone.m_net_server.get_config_object().del_out_connections(1); + --(zone.m_current_number_of_out_peers); // atomic variable, update time = 1s return false; } + + MDEBUG("Connecting to " << na.str() << "(peer_type=" << peer_type << ", last_seen: " << (last_seen_stamp ? epee::misc_utils::get_time_interval_string(time(NULL) - last_seen_stamp):"never") << ")..."); - CHECK_AND_ASSERT_MES(na.get_type_id() == epee::net_utils::ipv4_network_address::ID, false, - "Only IPv4 addresses are supported here"); - const epee::net_utils::ipv4_network_address &ipv4 = na.as(); - - typename net_server::t_connection_context con = AUTO_VAL_INIT(con); - bool res = m_net_server.connect(epee::string_tools::get_ip_string_from_int32(ipv4.ip()), - epee::string_tools::num_to_string_fast(ipv4.port()), - m_config.m_net_config.connection_timeout, - con); - - if(!res) + auto con = zone.m_connect(zone, na, m_ssl_support); + if(!con) { bool is_priority = is_priority_node(na); - LOG_PRINT_CC_PRIORITY_NODE(is_priority, con, "Connect failed to " << na.str() + LOG_PRINT_CC_PRIORITY_NODE(is_priority, bool(con), "Connect failed to " << na.str() /*<< ", try " << try_count*/); //m_peerlist.set_peer_unreachable(pe); return false; } + con->m_anchor = peer_type == anchor; peerid_type pi = AUTO_VAL_INIT(pi); - res = do_handshake_with_peer(pi, con, just_take_peerlist); + bool res = do_handshake_with_peer(pi, *con, just_take_peerlist); if(!res) { bool is_priority = is_priority_node(na); - LOG_PRINT_CC_PRIORITY_NODE(is_priority, con, "Failed to HANDSHAKE with peer " + LOG_PRINT_CC_PRIORITY_NODE(is_priority, *con, "Failed to HANDSHAKE with peer " << na.str() /*<< ", try " << try_count*/); + zone.m_net_server.get_config_object().close(con->m_connection_id); return false; } if(just_take_peerlist) { - m_net_server.get_config_object().close(con.m_connection_id); - LOG_DEBUG_CC(con, "CONNECTION HANDSHAKED OK AND CLOSED."); + zone.m_net_server.get_config_object().close(con->m_connection_id); + LOG_DEBUG_CC(*con, "CONNECTION HANDSHAKED OK AND CLOSED."); return true; } @@ -948,7 +1110,9 @@ namespace nodetool time_t last_seen; time(&last_seen); pe_local.last_seen = static_cast(last_seen); - m_peerlist.append_with_peer_white(pe_local); + pe_local.pruning_seed = con->m_pruning_seed; + pe_local.rpc_port = con->m_rpc_port; + zone.m_peerlist.append_with_peer_white(pe_local); //update last seen and push it to peerlist manager anchor_peerlist_entry ape = AUTO_VAL_INIT(ape); @@ -956,51 +1120,46 @@ namespace nodetool ape.id = pi; ape.first_seen = first_seen_stamp ? first_seen_stamp : time(nullptr); - m_peerlist.append_with_peer_anchor(ape); + zone.m_peerlist.append_with_peer_anchor(ape); - LOG_DEBUG_CC(con, "CONNECTION HANDSHAKED OK."); + LOG_DEBUG_CC(*con, "CONNECTION HANDSHAKED OK."); return true; } template bool node_server::check_connection_and_handshake_with_peer(const epee::net_utils::network_address& na, uint64_t last_seen_stamp) { + network_zone& zone = m_network_zones.at(na.get_zone()); + if (zone.m_connect == nullptr) + return false; + LOG_PRINT_L1("Connecting to " << na.str() << "(last_seen: " << (last_seen_stamp ? epee::misc_utils::get_time_interval_string(time(NULL) - last_seen_stamp):"never") << ")..."); - CHECK_AND_ASSERT_MES(na.get_type_id() == epee::net_utils::ipv4_network_address::ID, false, - "Only IPv4 addresses are supported here"); - const epee::net_utils::ipv4_network_address &ipv4 = na.as(); - - typename net_server::t_connection_context con = AUTO_VAL_INIT(con); - bool res = m_net_server.connect(epee::string_tools::get_ip_string_from_int32(ipv4.ip()), - epee::string_tools::num_to_string_fast(ipv4.port()), - m_config.m_net_config.connection_timeout, - con); - - if (!res) { + auto con = zone.m_connect(zone, na, m_ssl_support); + if (!con) { bool is_priority = is_priority_node(na); - LOG_PRINT_CC_PRIORITY_NODE(is_priority, con, "Connect failed to " << na.str()); + LOG_PRINT_CC_PRIORITY_NODE(is_priority, p2p_connection_context{}, "Connect failed to " << na.str()); return false; } + con->m_anchor = false; peerid_type pi = AUTO_VAL_INIT(pi); - res = do_handshake_with_peer(pi, con, true); - + const bool res = do_handshake_with_peer(pi, *con, true); if (!res) { bool is_priority = is_priority_node(na); - LOG_PRINT_CC_PRIORITY_NODE(is_priority, con, "Failed to HANDSHAKE with peer " << na.str()); - + LOG_PRINT_CC_PRIORITY_NODE(is_priority, *con, "Failed to HANDSHAKE with peer " << na.str()); + zone.m_net_server.get_config_object().close(con->m_connection_id); return false; } - m_net_server.get_config_object().close(con.m_connection_id); + zone.m_net_server.get_config_object().close(con->m_connection_id); - LOG_DEBUG_CC(con, "CONNECTION HANDSHAKED OK AND CLOSED."); + LOG_DEBUG_CC(*con, "CONNECTION HANDSHAKED OK AND CLOSED."); return true; } @@ -1026,7 +1185,7 @@ namespace nodetool bool node_server::make_new_connection_from_anchor_peerlist(const std::vector& anchor_peerlist) { for (const auto& pe: anchor_peerlist) { - _note("Considering connecting (out) to peer: " << peerid_type(pe.id) << " " << pe.adr.str()); + _note("Considering connecting (out) to anchor peer: " << peerid_type(pe.id) << " " << pe.adr.str()); if(is_peer_used(pe)) { _note("Peer is used"); @@ -1057,49 +1216,114 @@ namespace nodetool } //----------------------------------------------------------------------------------- template - bool node_server::make_new_connection_from_peerlist(bool use_white_list) + bool node_server::make_new_connection_from_peerlist(network_zone& zone, bool use_white_list) { - size_t local_peers_count = use_white_list ? m_peerlist.get_white_peers_count():m_peerlist.get_gray_peers_count(); - if(!local_peers_count) - return false;//no peers - - size_t max_random_index = std::min(local_peers_count -1, 20); + size_t max_random_index = 0; std::set tried_peers; size_t try_count = 0; size_t rand_count = 0; - while(rand_count < (max_random_index+1)*3 && try_count < 10 && !m_net_server.is_stop_signal_sent()) + while(rand_count < (max_random_index+1)*3 && try_count < 10 && !zone.m_net_server.is_stop_signal_sent()) { ++rand_count; size_t random_index; + const uint32_t next_needed_pruning_stripe = m_payload_handler.get_next_needed_pruning_stripe().second; - if (use_white_list) { - local_peers_count = m_peerlist.get_white_peers_count(); - if (!local_peers_count) - return false; - max_random_index = std::min(local_peers_count -1, 20); - random_index = get_random_index_with_fixed_probability(max_random_index); - } else { - local_peers_count = m_peerlist.get_gray_peers_count(); - if (!local_peers_count) - return false; - random_index = crypto::rand() % local_peers_count; + // build a set of all the /16 we're connected to, and prefer a peer that's not in that set + std::set classB; + if (&zone == &m_network_zones.at(epee::net_utils::zone::public_)) // at returns reference, not copy + { + zone.m_net_server.get_config_object().foreach_connection([&](const p2p_connection_context& cntxt) + { + if (cntxt.m_remote_address.get_type_id() == epee::net_utils::ipv4_network_address::get_type_id()) + { + + const epee::net_utils::network_address na = cntxt.m_remote_address; + const uint32_t actual_ip = na.as().ip(); + classB.insert(actual_ip & 0x0000ffff); + } + return true; + }); } - CHECK_AND_ASSERT_MES(random_index < local_peers_count, false, "random_starter_index < peers_local.size() failed!!"); + std::deque filtered; + const size_t limit = use_white_list ? 20 : std::numeric_limits::max(); + size_t idx = 0, skipped = 0; + for (int step = 0; step < 2; ++step) + { + bool skip_duplicate_class_B = step == 0; + zone.m_peerlist.foreach (use_white_list, [&classB, &filtered, &idx, &skipped, skip_duplicate_class_B, limit, next_needed_pruning_stripe](const peerlist_entry &pe){ + if (filtered.size() >= limit) + return false; + bool skip = false; + if (skip_duplicate_class_B && pe.adr.get_type_id() == epee::net_utils::ipv4_network_address::get_type_id()) + { + const epee::net_utils::network_address na = pe.adr; + uint32_t actual_ip = na.as().ip(); + skip = classB.find(actual_ip & 0x0000ffff) != classB.end(); + } + if (skip) + ++skipped; + else if (next_needed_pruning_stripe == 0 || pe.pruning_seed == 0) + filtered.push_back(idx); + else if (next_needed_pruning_stripe == tools::get_pruning_stripe(pe.pruning_seed)) + filtered.push_front(idx); + ++idx; + return true; + }); + if (skipped == 0 || !filtered.empty()) + break; + if (skipped) + MGINFO("Skipping " << skipped << " possible peers as they share a class B with existing peers"); + } + if (filtered.empty()) + { + MDEBUG("No available peer in " << (use_white_list ? "white" : "gray") << " list filtered by " << next_needed_pruning_stripe); + return false; + } + if (use_white_list) + { + // if using the white list, we first pick in the set of peers we've already been using earlier + random_index = get_random_index_with_fixed_probability(std::min(filtered.size() - 1, 20)); + CRITICAL_REGION_LOCAL(m_used_stripe_peers_mutex); + if (next_needed_pruning_stripe > 0 && next_needed_pruning_stripe <= (1ul << CRYPTONOTE_PRUNING_LOG_STRIPES) && !m_used_stripe_peers[next_needed_pruning_stripe-1].empty()) + { + const epee::net_utils::network_address na = m_used_stripe_peers[next_needed_pruning_stripe-1].front(); + m_used_stripe_peers[next_needed_pruning_stripe-1].pop_front(); + for (size_t i = 0; i < filtered.size(); ++i) + { + peerlist_entry pe; + if (zone.m_peerlist.get_white_peer_by_index(pe, filtered[i]) && pe.adr == na) + { + MDEBUG("Reusing stripe " << next_needed_pruning_stripe << " peer " << pe.adr.str()); + random_index = i; + break; + } + } + } + } + else + random_index = crypto::rand_idx(filtered.size()); + + CHECK_AND_ASSERT_MES(random_index < filtered.size(), false, "random_index < filtered.size() failed!!"); + random_index = filtered[random_index]; + CHECK_AND_ASSERT_MES(random_index < (use_white_list ? zone.m_peerlist.get_white_peers_count() : zone.m_peerlist.get_gray_peers_count()), + false, "random_index < peers size failed!!"); if(tried_peers.count(random_index)) continue; tried_peers.insert(random_index); peerlist_entry pe = AUTO_VAL_INIT(pe); - bool r = use_white_list ? m_peerlist.get_white_peer_by_index(pe, random_index):m_peerlist.get_gray_peer_by_index(pe, random_index); + bool r = use_white_list ? zone.m_peerlist.get_white_peer_by_index(pe, random_index):zone.m_peerlist.get_gray_peer_by_index(pe, random_index); CHECK_AND_ASSERT_MES(r, false, "Failed to get random peer from peerlist(white:" << use_white_list << ")"); ++try_count; - _note("Considering connecting (out) to peer: " << peerid_to_string(pe.id) << " " << pe.adr.str()); + _note("Considering connecting (out) to " << (use_white_list ? "white" : "gray") << " list peer: " << + peerid_to_string(pe.id) << " " << pe.adr.str() << ", pruning seed " << epee::string_tools::to_string_hex(pe.pruning_seed) << + " (stripe " << next_needed_pruning_stripe << " needed)"); if(is_peer_used(pe)) { _note("Peer is used"); @@ -1113,6 +1337,7 @@ namespace nodetool continue; MDEBUG("Selected peer: " << peerid_to_string(pe.id) << " " << pe.adr.str() + << ", pruning seed " << epee::string_tools::to_string_hex(pe.pruning_seed) << " " << "[peer_list=" << (use_white_list ? white : gray) << "] last_seen: " << (pe.last_seen ? epee::misc_utils::get_time_interval_string(time(NULL) - pe.last_seen) : "never")); @@ -1133,26 +1358,32 @@ namespace nodetool return true; size_t try_count = 0; - size_t current_index = crypto::rand()%m_seed_nodes.size(); - bool fallback_nodes_added = false; + size_t current_index = crypto::rand_idx(m_seed_nodes.size()); + const net_server& server = m_network_zones.at(epee::net_utils::zone::public_).m_net_server; while(true) { - if(m_net_server.is_stop_signal_sent()) + if(server.is_stop_signal_sent()) return false; if(try_to_connect_and_handshake_with_new_peer(m_seed_nodes[current_index], true)) break; if(++try_count > m_seed_nodes.size()) { - if (!fallback_nodes_added) + if (!m_fallback_seed_nodes_added) { MWARNING("Failed to connect to any of seed peers, trying fallback seeds"); + current_index = m_seed_nodes.size(); for (const auto &peer: get_seed_nodes(m_nettype)) { MDEBUG("Fallback seed node: " << peer); - append_net_address(m_seed_nodes, peer); + append_net_address(m_seed_nodes, peer, cryptonote::get_config(m_nettype).P2P_DEFAULT_PORT); + } + m_fallback_seed_nodes_added = true; + if (current_index == m_seed_nodes.size()) + { + MWARNING("No fallback seeds, continuing without seeds"); + break; } - fallback_nodes_added = true; // continue for another few cycles } else @@ -1170,13 +1401,17 @@ namespace nodetool template bool node_server::connections_maker() { + using zone_type = epee::net_utils::zone; + if (m_offline) return true; if (!connect_to_peerlist(m_exclusive_peers)) return false; if (!m_exclusive_peers.empty()) return true; - size_t start_conn_count = get_outgoing_connections_count(); - if(!m_peerlist.get_white_peers_count() && m_seed_nodes.size()) + // Only have seeds in the public zone right now. + + size_t start_conn_count = get_public_outgoing_connections_count(); + if(!get_public_white_peers_count() && m_seed_nodes.size()) { if (!connect_to_seed()) return false; @@ -1184,34 +1419,49 @@ namespace nodetool if (!connect_to_peerlist(m_priority_peers)) return false; - size_t expected_white_connections = (m_config.m_net_config.max_out_connection_count*P2P_DEFAULT_WHITELIST_CONNECTIONS_PERCENT)/100; - - size_t conn_count = get_outgoing_connections_count(); - if(conn_count < m_config.m_net_config.max_out_connection_count) + for(auto& zone : m_network_zones) { - if(conn_count < expected_white_connections) - { - //start from anchor list - if(!make_expected_connections_count(anchor, P2P_DEFAULT_ANCHOR_CONNECTIONS_COUNT)) - return false; - //then do white list - if(!make_expected_connections_count(white, expected_white_connections)) - return false; - //then do grey list - if(!make_expected_connections_count(gray, m_config.m_net_config.max_out_connection_count)) - return false; - }else + size_t base_expected_white_connections = (zone.second.m_config.m_net_config.max_out_connection_count*P2P_DEFAULT_WHITELIST_CONNECTIONS_PERCENT)/100; + + size_t conn_count = get_outgoing_connections_count(zone.second); + while(conn_count < zone.second.m_config.m_net_config.max_out_connection_count) { - //start from grey list - if(!make_expected_connections_count(gray, m_config.m_net_config.max_out_connection_count)) - return false; - //and then do white list - if(!make_expected_connections_count(white, m_config.m_net_config.max_out_connection_count)) + const size_t expected_white_connections = m_payload_handler.get_next_needed_pruning_stripe().second ? zone.second.m_config.m_net_config.max_out_connection_count : base_expected_white_connections; + if(conn_count < expected_white_connections) + { + //start from anchor list + while (get_outgoing_connections_count(zone.second) < P2P_DEFAULT_ANCHOR_CONNECTIONS_COUNT + && make_expected_connections_count(zone.second, anchor, P2P_DEFAULT_ANCHOR_CONNECTIONS_COUNT)); + //then do white list + while (get_outgoing_connections_count(zone.second) < expected_white_connections + && make_expected_connections_count(zone.second, white, expected_white_connections)); + //then do grey list + while (get_outgoing_connections_count(zone.second) < zone.second.m_config.m_net_config.max_out_connection_count + && make_expected_connections_count(zone.second, gray, zone.second.m_config.m_net_config.max_out_connection_count)); + }else + { + //start from grey list + while (get_outgoing_connections_count(zone.second) < zone.second.m_config.m_net_config.max_out_connection_count + && make_expected_connections_count(zone.second, gray, zone.second.m_config.m_net_config.max_out_connection_count)); + //and then do white list + while (get_outgoing_connections_count(zone.second) < zone.second.m_config.m_net_config.max_out_connection_count + && make_expected_connections_count(zone.second, white, zone.second.m_config.m_net_config.max_out_connection_count)); + } + if(zone.second.m_net_server.is_stop_signal_sent()) return false; + size_t new_conn_count = get_outgoing_connections_count(zone.second); + if (new_conn_count <= conn_count) + { + // we did not make any connection, sleep a bit to avoid a busy loop in case we don't have + // any peers to try, then break so we will try seeds to get more peers + boost::this_thread::sleep_for(boost::chrono::seconds(1)); + break; + } + conn_count = new_conn_count; } } - if (start_conn_count == get_outgoing_connections_count() && start_conn_count < m_config.m_net_config.max_out_connection_count) + if (start_conn_count == get_public_outgoing_connections_count() && start_conn_count < m_network_zones.at(zone_type::public_).m_config.m_net_config.max_out_connection_count) { MINFO("Failed to connect to any, trying seeds"); if (!connect_to_seed()) @@ -1222,77 +1472,157 @@ namespace nodetool } //----------------------------------------------------------------------------------- template - bool node_server::make_expected_connections_count(PeerType peer_type, size_t expected_connections) + bool node_server::make_expected_connections_count(network_zone& zone, PeerType peer_type, size_t expected_connections) { if (m_offline) - return true; + return false; std::vector apl; if (peer_type == anchor) { - m_peerlist.get_and_empty_anchor_peerlist(apl); + zone.m_peerlist.get_and_empty_anchor_peerlist(apl); } - size_t conn_count = get_outgoing_connections_count(); + size_t conn_count = get_outgoing_connections_count(zone); //add new connections from white peers - while(conn_count < expected_connections) + if(conn_count < expected_connections) { - if(m_net_server.is_stop_signal_sent()) + if(zone.m_net_server.is_stop_signal_sent()) return false; + MDEBUG("Making expected connection, type " << peer_type << ", " << conn_count << "/" << expected_connections << " connections"); + if (peer_type == anchor && !make_new_connection_from_anchor_peerlist(apl)) { - break; + return false; } - if (peer_type == white && !make_new_connection_from_peerlist(true)) { - break; + if (peer_type == white && !make_new_connection_from_peerlist(zone, true)) { + return false; } - if (peer_type == gray && !make_new_connection_from_peerlist(false)) { - break; + if (peer_type == gray && !make_new_connection_from_peerlist(zone, false)) { + return false; } - - conn_count = get_outgoing_connections_count(); } return true; } - //----------------------------------------------------------------------------------- template - size_t node_server::get_outgoing_connections_count() + size_t node_server::get_public_outgoing_connections_count() + { + auto public_zone = m_network_zones.find(epee::net_utils::zone::public_); + if (public_zone == m_network_zones.end()) + return 0; + return get_outgoing_connections_count(public_zone->second); + } + //----------------------------------------------------------------------------------- + template + size_t node_server::get_incoming_connections_count(network_zone& zone) { size_t count = 0; - m_net_server.get_config_object().foreach_connection([&](const p2p_connection_context& cntxt) + zone.m_net_server.get_config_object().foreach_connection([&](const p2p_connection_context& cntxt) { - if(!cntxt.m_is_income) + if(cntxt.m_is_income) ++count; return true; }); - return count; } //----------------------------------------------------------------------------------- template - size_t node_server::get_incoming_connections_count() + size_t node_server::get_outgoing_connections_count(network_zone& zone) { size_t count = 0; - m_net_server.get_config_object().foreach_connection([&](const p2p_connection_context& cntxt) + zone.m_net_server.get_config_object().foreach_connection([&](const p2p_connection_context& cntxt) { - if(cntxt.m_is_income) + if(!cntxt.m_is_income) ++count; return true; }); - return count; } //----------------------------------------------------------------------------------- template + size_t node_server::get_outgoing_connections_count() + { + size_t count = 0; + for(auto& zone : m_network_zones) + count += get_outgoing_connections_count(zone.second); + return count; + } + //----------------------------------------------------------------------------------- + template + size_t node_server::get_incoming_connections_count() + { + size_t count = 0; + for (auto& zone : m_network_zones) + { + zone.second.m_net_server.get_config_object().foreach_connection([&](const p2p_connection_context& cntxt) + { + if(cntxt.m_is_income) + ++count; + return true; + }); + } + return count; + } + //----------------------------------------------------------------------------------- + template + size_t node_server::get_public_white_peers_count() + { + auto public_zone = m_network_zones.find(epee::net_utils::zone::public_); + if (public_zone == m_network_zones.end()) + return 0; + return public_zone->second.m_peerlist.get_white_peers_count(); + } + //----------------------------------------------------------------------------------- + template + size_t node_server::get_public_gray_peers_count() + { + auto public_zone = m_network_zones.find(epee::net_utils::zone::public_); + if (public_zone == m_network_zones.end()) + return 0; + return public_zone->second.m_peerlist.get_gray_peers_count(); + } + //----------------------------------------------------------------------------------- + template + void node_server::get_public_peerlist(std::vector& gray, std::vector& white) + { + auto public_zone = m_network_zones.find(epee::net_utils::zone::public_); + if (public_zone != m_network_zones.end()) + public_zone->second.m_peerlist.get_peerlist(gray, white); + } + //----------------------------------------------------------------------------------- + template bool node_server::idle_worker() { m_peer_handshake_idle_maker_interval.do_call(boost::bind(&node_server::peer_sync_idle_maker, this)); m_connections_maker_interval.do_call(boost::bind(&node_server::connections_maker, this)); m_gray_peerlist_housekeeping_interval.do_call(boost::bind(&node_server::gray_peerlist_housekeeping, this)); m_peerlist_store_interval.do_call(boost::bind(&node_server::store_config, this)); + m_incoming_connections_interval.do_call(boost::bind(&node_server::check_incoming_connections, this)); + return true; + } + //----------------------------------------------------------------------------------- + template + bool node_server::check_incoming_connections() + { + if (m_offline) + return true; + + const auto public_zone = m_network_zones.find(epee::net_utils::zone::public_); + if (public_zone != m_network_zones.end() && get_incoming_connections_count(public_zone->second) == 0) + { + if (m_hide_my_port || public_zone->second.m_config.m_net_config.max_in_connection_count == 0) + { + MGINFO("Incoming connections disabled, enable them for full connectivity"); + } + else + { + const el::Level level = el::Level::Warning; + MCLOG_RED(level, "global", "No incoming connections - check firewalls/routers allow port " << get_this_peer_port()); + } + } return true; } //----------------------------------------------------------------------------------- @@ -1302,15 +1632,18 @@ namespace nodetool MDEBUG("STARTED PEERLIST IDLE HANDSHAKE"); typedef std::list > local_connects_type; local_connects_type cncts; - m_net_server.get_config_object().foreach_connection([&](p2p_connection_context& cntxt) + for(auto& zone : m_network_zones) { - if(cntxt.peer_id && !cntxt.m_in_timedsync) + zone.second.m_net_server.get_config_object().foreach_connection([&](p2p_connection_context& cntxt) { - cntxt.m_in_timedsync = true; - cncts.push_back(local_connects_type::value_type(cntxt, cntxt.peer_id));//do idle sync only with handshaked connections - } - return true; - }); + if(cntxt.peer_id && !cntxt.m_in_timedsync) + { + cntxt.m_in_timedsync = true; + cncts.push_back(local_connects_type::value_type(cntxt, cntxt.peer_id));//do idle sync only with handshaked connections + } + return true; + }); + } std::for_each(cncts.begin(), cncts.end(), [&](const typename local_connects_type::value_type& vl){do_peer_timed_sync(vl.first, vl.second);}); @@ -1319,7 +1652,7 @@ namespace nodetool } //----------------------------------------------------------------------------------- template - bool node_server::fix_time_delta(std::list& local_peerlist, time_t local_time, int64_t& delta) + bool node_server::fix_time_delta(std::vector& local_peerlist, time_t local_time, int64_t& delta) { //fix time delta time_t now = 0; @@ -1334,40 +1667,55 @@ namespace nodetool return false; } be.last_seen += delta; +#ifdef CRYPTONOTE_PRUNING_DEBUG_SPOOF_SEED + be.pruning_seed = tools::make_pruning_seed(1 + (be.adr.as().ip()) % (1ul << CRYPTONOTE_PRUNING_LOG_STRIPES), CRYPTONOTE_PRUNING_LOG_STRIPES); +#endif } return true; } //----------------------------------------------------------------------------------- template - bool node_server::handle_remote_peerlist(const std::list& peerlist, time_t local_time, const epee::net_utils::connection_context_base& context) + bool node_server::handle_remote_peerlist(const std::vector& peerlist, time_t local_time, const epee::net_utils::connection_context_base& context) { int64_t delta = 0; - std::list peerlist_ = peerlist; + std::vector peerlist_ = peerlist; if(!fix_time_delta(peerlist_, local_time, delta)) return false; + + const epee::net_utils::zone zone = context.m_remote_address.get_zone(); + for(const auto& peer : peerlist_) + { + if(peer.adr.get_zone() != zone) + { + MWARNING(context << " sent peerlist from another zone, dropping"); + return false; + } + } + LOG_DEBUG_CC(context, "REMOTE PEERLIST: TIME_DELTA: " << delta << ", remote peerlist size=" << peerlist_.size()); LOG_DEBUG_CC(context, "REMOTE PEERLIST: " << print_peerlist_to_string(peerlist_)); - return m_peerlist.merge_peerlist(peerlist_); + return m_network_zones.at(context.m_remote_address.get_zone()).m_peerlist.merge_peerlist(peerlist_); } //----------------------------------------------------------------------------------- template - bool node_server::get_local_node_data(basic_node_data& node_data) + bool node_server::get_local_node_data(basic_node_data& node_data, const network_zone& zone) { time_t local_time; time(&local_time); - node_data.local_time = local_time; - node_data.peer_id = m_config.m_peer_id; - if(!m_hide_my_port) + node_data.local_time = local_time; // \TODO This can be an identifying value across zones (public internet to tor/i2p) ... + node_data.peer_id = zone.m_config.m_peer_id; + if(!m_hide_my_port && zone.m_can_pingback) node_data.my_port = m_external_port ? m_external_port : m_listening_port; else node_data.my_port = 0; + node_data.rpc_port = zone.m_can_pingback ? m_rpc_port : 0; node_data.network_id = m_network_id; return true; } //----------------------------------------------------------------------------------- #ifdef ALLOW_DEBUG_COMMANDS template - bool node_server::check_trust(const proof_of_trust& tr) + bool node_server::check_trust(const proof_of_trust& tr, const epee::net_utils::zone zone_type) { uint64_t local_time = time(NULL); uint64_t time_delata = local_time > tr.time ? local_time - tr.time: tr.time - local_time; @@ -1381,9 +1729,11 @@ namespace nodetool MWARNING("check_trust failed to check time conditions, last_stat_request_time=" << m_last_stat_request_time << ", proof_time=" << tr.time); return false; } - if(m_config.m_peer_id != tr.peer_id) + + const network_zone& zone = m_network_zones.at(zone_type); + if(zone.m_config.m_peer_id != tr.peer_id) { - MWARNING("check_trust failed: peer_id mismatch (passed " << tr.peer_id << ", expected " << m_config.m_peer_id<< ")"); + MWARNING("check_trust failed: peer_id mismatch (passed " << tr.peer_id << ", expected " << zone.m_config.m_peer_id<< ")"); return false; } crypto::public_key pk = AUTO_VAL_INIT(pk); @@ -1402,12 +1752,12 @@ namespace nodetool template int node_server::handle_get_stat_info(int command, typename COMMAND_REQUEST_STAT_INFO::request& arg, typename COMMAND_REQUEST_STAT_INFO::response& rsp, p2p_connection_context& context) { - if(!check_trust(arg.tr)) + if(!check_trust(arg.tr, context.m_remote_address.get_zone())) { drop_connection(context); return 1; } - rsp.connections_count = m_net_server.get_config_object().get_connections_count(); + rsp.connections_count = get_connections_count(); rsp.incoming_connections_count = rsp.connections_count - get_outgoing_connections_count(); rsp.version = MONERO_VERSION_FULL; rsp.os_version = tools::get_os_version_string(); @@ -1418,12 +1768,12 @@ namespace nodetool template int node_server::handle_get_network_state(int command, COMMAND_REQUEST_NETWORK_STATE::request& arg, COMMAND_REQUEST_NETWORK_STATE::response& rsp, p2p_connection_context& context) { - if(!check_trust(arg.tr)) + if(!check_trust(arg.tr, context.m_remote_address.get_zone())) { drop_connection(context); return 1; } - m_net_server.get_config_object().foreach_connection([&](const p2p_connection_context& cntxt) + m_network_zones.at(epee::net_utils::zone::public_).m_net_server.get_config_object().foreach_connection([&](const p2p_connection_context& cntxt) { connection_entry ce; ce.adr = cntxt.m_remote_address; @@ -1433,8 +1783,9 @@ namespace nodetool return true; }); - m_peerlist.get_peerlist_full(rsp.local_peerlist_gray, rsp.local_peerlist_white); - rsp.my_id = m_config.m_peer_id; + network_zone& zone = m_network_zones.at(context.m_remote_address.get_zone()); + zone.m_peerlist.get_peerlist(rsp.local_peerlist_gray, rsp.local_peerlist_white); + rsp.my_id = zone.m_config.m_peer_id; rsp.local_time = time(NULL); return 1; } @@ -1442,7 +1793,7 @@ namespace nodetool template int node_server::handle_get_peer_id(int command, COMMAND_REQUEST_PEER_ID::request& arg, COMMAND_REQUEST_PEER_ID::response& rsp, p2p_connection_context& context) { - rsp.my_id = m_config.m_peer_id; + rsp.my_id = m_network_zones.at(context.m_remote_address.get_zone()).m_config.m_peer_id; return 1; } #endif @@ -1450,63 +1801,73 @@ namespace nodetool template int node_server::handle_get_support_flags(int command, COMMAND_REQUEST_SUPPORT_FLAGS::request& arg, COMMAND_REQUEST_SUPPORT_FLAGS::response& rsp, p2p_connection_context& context) { - rsp.support_flags = m_config.m_support_flags; + rsp.support_flags = m_network_zones.at(context.m_remote_address.get_zone()).m_config.m_support_flags; return 1; } //----------------------------------------------------------------------------------- template void node_server::request_callback(const epee::net_utils::connection_context_base& context) { - m_net_server.get_config_object().request_callback(context.m_connection_id); + m_network_zones.at(context.m_remote_address.get_zone()).m_net_server.get_config_object().request_callback(context.m_connection_id); } //----------------------------------------------------------------------------------- template - bool node_server::relay_notify_to_list(int command, const std::string& data_buff, const std::list &connections) + bool node_server::relay_notify_to_list(int command, const epee::span data_buff, std::vector> connections) { + std::sort(connections.begin(), connections.end()); + auto zone = m_network_zones.begin(); for(const auto& c_id: connections) { - m_net_server.get_config_object().notify(command, data_buff, c_id); + for (;;) + { + if (zone == m_network_zones.end()) + { + MWARNING("Unable to relay all messages, " << epee::net_utils::zone_to_string(c_id.first) << " not available"); + return false; + } + if (c_id.first <= zone->first) + break; + + ++zone; + } + if (zone->first == c_id.first) + zone->second.m_net_server.get_config_object().notify(command, data_buff, c_id.second); } return true; } //----------------------------------------------------------------------------------- template - bool node_server::relay_notify_to_all(int command, const std::string& data_buff, const epee::net_utils::connection_context_base& context) - { - std::list connections; - m_net_server.get_config_object().foreach_connection([&](const p2p_connection_context& cntxt) - { - if(cntxt.peer_id && context.m_connection_id != cntxt.m_connection_id) - connections.push_back(cntxt.m_connection_id); - return true; - }); - return relay_notify_to_list(command, data_buff, connections); - } - //----------------------------------------------------------------------------------- - template void node_server::callback(p2p_connection_context& context) { m_payload_handler.on_callback(context); } //----------------------------------------------------------------------------------- template - bool node_server::invoke_notify_to_peer(int command, const std::string& req_buff, const epee::net_utils::connection_context_base& context) + bool node_server::invoke_notify_to_peer(int command, const epee::span req_buff, const epee::net_utils::connection_context_base& context) { - int res = m_net_server.get_config_object().notify(command, req_buff, context.m_connection_id); + if(is_filtered_command(context.m_remote_address, command)) + return false; + + network_zone& zone = m_network_zones.at(context.m_remote_address.get_zone()); + int res = zone.m_net_server.get_config_object().notify(command, req_buff, context.m_connection_id); return res > 0; } //----------------------------------------------------------------------------------- template - bool node_server::invoke_command_to_peer(int command, const std::string& req_buff, std::string& resp_buff, const epee::net_utils::connection_context_base& context) + bool node_server::invoke_command_to_peer(int command, const epee::span req_buff, std::string& resp_buff, const epee::net_utils::connection_context_base& context) { - int res = m_net_server.get_config_object().invoke(command, req_buff, resp_buff, context.m_connection_id); + if(is_filtered_command(context.m_remote_address, command)) + return false; + + network_zone& zone = m_network_zones.at(context.m_remote_address.get_zone()); + int res = zone.m_net_server.get_config_object().invoke(command, req_buff, resp_buff, context.m_connection_id); return res > 0; } //----------------------------------------------------------------------------------- template bool node_server::drop_connection(const epee::net_utils::connection_context_base& context) { - m_net_server.get_config_object().close(context.m_connection_id); + m_network_zones.at(context.m_remote_address.get_zone()).m_net_server.get_config_object().close(context.m_connection_id); return true; } //----------------------------------------------------------------------------------- @@ -1516,18 +1877,21 @@ namespace nodetool if(!node_data.my_port) return false; - CHECK_AND_ASSERT_MES(context.m_remote_address.get_type_id() == epee::net_utils::ipv4_network_address::ID, false, + CHECK_AND_ASSERT_MES(context.m_remote_address.get_type_id() == epee::net_utils::ipv4_network_address::get_type_id(), false, "Only IPv4 addresses are supported here"); const epee::net_utils::network_address na = context.m_remote_address; uint32_t actual_ip = na.as().ip(); - if(!m_peerlist.is_host_allowed(context.m_remote_address)) + network_zone& zone = m_network_zones.at(na.get_zone()); + + if(!zone.m_peerlist.is_host_allowed(context.m_remote_address)) return false; + std::string ip = epee::string_tools::get_ip_string_from_int32(actual_ip); std::string port = epee::string_tools::num_to_string_fast(node_data.my_port); epee::net_utils::network_address address{epee::net_utils::ipv4_network_address(actual_ip, node_data.my_port)}; peerid_type pr = node_data.peer_id; - bool r = m_net_server.connect_async(ip, port, m_config.m_net_config.ping_connection_timeout, [cb, /*context,*/ address, pr, this]( + bool r = zone.m_net_server.connect_async(ip, port, zone.m_config.m_net_config.ping_connection_timeout, [cb, /*context,*/ address, pr, this]( const typename net_server::t_connection_context& ping_context, const boost::system::error_code& ec)->bool { @@ -1547,7 +1911,9 @@ namespace nodetool // GCC 5.1.0 gives error with second use of uint64_t (peerid_type) variable. peerid_type pr_ = pr; - bool inv_call_res = epee::net_utils::async_invoke_remote_command2(ping_context.m_connection_id, COMMAND_PING::ID, req, m_net_server.get_config_object(), + network_zone& zone = m_network_zones.at(address.get_zone()); + + bool inv_call_res = epee::net_utils::async_invoke_remote_command2(ping_context.m_connection_id, COMMAND_PING::ID, req, zone.m_net_server.get_config_object(), [=](int code, const COMMAND_PING::response& rsp, p2p_connection_context& context) { if(code <= 0) @@ -1556,20 +1922,21 @@ namespace nodetool return; } + network_zone& zone = m_network_zones.at(address.get_zone()); if(rsp.status != PING_OK_RESPONSE_STATUS_TEXT || pr != rsp.peer_id) { LOG_WARNING_CC(ping_context, "back ping invoke wrong response \"" << rsp.status << "\" from" << address.str() << ", hsh_peer_id=" << pr_ << ", rsp.peer_id=" << rsp.peer_id); - m_net_server.get_config_object().close(ping_context.m_connection_id); + zone.m_net_server.get_config_object().close(ping_context.m_connection_id); return; } - m_net_server.get_config_object().close(ping_context.m_connection_id); + zone.m_net_server.get_config_object().close(ping_context.m_connection_id); cb(); }); if(!inv_call_res) { LOG_WARNING_CC(ping_context, "back ping invoke failed to " << address.str()); - m_net_server.get_config_object().close(ping_context.m_connection_id); + zone.m_net_server.get_config_object().close(ping_context.m_connection_id); return false; } return true; @@ -1584,13 +1951,16 @@ namespace nodetool template bool node_server::try_get_support_flags(const p2p_connection_context& context, std::function f) { + if(context.m_remote_address.get_zone() != epee::net_utils::zone::public_) + return false; + COMMAND_REQUEST_SUPPORT_FLAGS::request support_flags_request; bool r = epee::net_utils::async_invoke_remote_command2 ( context.m_connection_id, COMMAND_REQUEST_SUPPORT_FLAGS::ID, support_flags_request, - m_net_server.get_config_object(), + m_network_zones.at(epee::net_utils::zone::public_).m_net_server.get_config_object(), [=](int code, const typename COMMAND_REQUEST_SUPPORT_FLAGS::response& rsp, p2p_connection_context& context_) { if(code < 0) @@ -1619,8 +1989,24 @@ namespace nodetool //fill response rsp.local_time = time(NULL); - m_peerlist.get_peerlist_head(rsp.local_peerlist_new); + + const epee::net_utils::zone zone_type = context.m_remote_address.get_zone(); + network_zone& zone = m_network_zones.at(zone_type); + + zone.m_peerlist.get_peerlist_head(rsp.local_peerlist_new, true); m_payload_handler.get_payload_sync_data(rsp.payload_data); + + /* Tor/I2P nodes receiving connections via forwarding (from tor/i2p daemon) + do not know the address of the connecting peer. This is relayed to them, + iff the node has setup an inbound hidden service. The other peer will have + to use the random peer_id value to link the two. My initial thought is that + the inbound peer should leave the other side marked as ``, + etc., because someone could give faulty addresses over Tor/I2P to get the + real peer with that identity banned/blacklisted. */ + + if(!context.m_is_income && zone.m_our_address.get_zone() == zone_type) + rsp.local_peerlist_new.push_back(peerlist_entry{zone.m_our_address, zone.m_config.m_peer_id, std::time(nullptr)}); + LOG_DEBUG_CC(context, "COMMAND_TIMED_SYNC"); return 1; } @@ -1631,7 +2017,7 @@ namespace nodetool if(arg.node_data.network_id != m_network_id) { - LOG_INFO_CC(context, "WRONG NETWORK AGENT CONNECTED! id=" << epee::string_tools::get_str_from_guid_a(arg.node_data.network_id)); + LOG_INFO_CC(context, "WRONG NETWORK AGENT CONNECTED! id=" << arg.node_data.network_id); drop_connection(context); add_host_fail(context.m_remote_address); return 1; @@ -1652,7 +2038,9 @@ namespace nodetool return 1; } - if (m_current_number_of_in_peers >= m_config.m_net_config.max_in_connection_count) // in peers limit + network_zone& zone = m_network_zones.at(context.m_remote_address.get_zone()); + + if (zone.m_current_number_of_in_peers >= zone.m_config.m_net_config.max_in_connection_count) // in peers limit { LOG_WARNING_CC(context, "COMMAND_HANDSHAKE came, but already have max incoming connections, so dropping this one."); drop_connection(context); @@ -1676,15 +2064,16 @@ namespace nodetool //associate peer_id with this connection context.peer_id = arg.node_data.peer_id; context.m_in_timedsync = false; + context.m_rpc_port = arg.node_data.rpc_port; - if(arg.node_data.peer_id != m_config.m_peer_id && arg.node_data.my_port) + if(arg.node_data.peer_id != zone.m_config.m_peer_id && arg.node_data.my_port && zone.m_can_pingback) { peerid_type peer_id_l = arg.node_data.peer_id; uint32_t port_l = arg.node_data.my_port; //try ping to be sure that we can add this peer to peer_list try_ping(arg.node_data, context, [peer_id_l, port_l, context, this]() { - CHECK_AND_ASSERT_MES(context.m_remote_address.get_type_id() == epee::net_utils::ipv4_network_address::ID, void(), + CHECK_AND_ASSERT_MES(context.m_remote_address.get_type_id() == epee::net_utils::ipv4_network_address::get_type_id(), void(), "Only IPv4 addresses are supported here"); //called only(!) if success pinged, update local peerlist peerlist_entry pe; @@ -1694,7 +2083,9 @@ namespace nodetool time(&last_seen); pe.last_seen = static_cast(last_seen); pe.id = peer_id_l; - this->m_peerlist.append_with_peer_white(pe); + pe.pruning_seed = context.m_pruning_seed; + pe.rpc_port = context.m_rpc_port; + this->m_network_zones.at(context.m_remote_address.get_zone()).m_peerlist.append_with_peer_white(pe); LOG_DEBUG_CC(context, "PING SUCCESS " << context.m_remote_address.host_str() << ":" << port_l); }); } @@ -1705,8 +2096,8 @@ namespace nodetool }); //fill response - m_peerlist.get_peerlist_head(rsp.local_peerlist_new); - get_local_node_data(rsp.node_data); + zone.m_peerlist.get_peerlist_head(rsp.local_peerlist_new, true); + get_local_node_data(rsp.node_data, zone); m_payload_handler.get_payload_sync_data(rsp.payload_data); LOG_DEBUG_CC(context, "COMMAND_HANDSHAKE"); return 1; @@ -1717,16 +2108,17 @@ namespace nodetool { LOG_DEBUG_CC(context, "COMMAND_PING"); rsp.status = PING_OK_RESPONSE_STATUS_TEXT; - rsp.peer_id = m_config.m_peer_id; + rsp.peer_id = m_network_zones.at(context.m_remote_address.get_zone()).m_config.m_peer_id; return 1; } //----------------------------------------------------------------------------------- template bool node_server::log_peerlist() { - std::list pl_white; - std::list pl_gray; - m_peerlist.get_peerlist_full(pl_gray, pl_white); + std::vector pl_white; + std::vector pl_gray; + for (auto& zone : m_network_zones) + zone.second.m_peerlist.get_peerlist(pl_gray, pl_white); MINFO(ENDL << "Peerlist white:" << ENDL << print_peerlist_to_string(pl_white) << ENDL << "Peerlist gray:" << ENDL << print_peerlist_to_string(pl_gray) ); return true; } @@ -1743,14 +2135,17 @@ namespace nodetool { std::stringstream ss; - m_net_server.get_config_object().foreach_connection([&](const p2p_connection_context& cntxt) + for (auto& zone : m_network_zones) { - ss << cntxt.m_remote_address.str() - << " \t\tpeer_id " << cntxt.peer_id - << " \t\tconn_id " << epee::string_tools::get_str_from_guid_a(cntxt.m_connection_id) << (cntxt.m_is_income ? " INC":" OUT") - << std::endl; - return true; - }); + zone.second.m_net_server.get_config_object().foreach_connection([&](const p2p_connection_context& cntxt) + { + ss << cntxt.m_remote_address.str() + << " \t\tpeer_id " << cntxt.peer_id + << " \t\tconn_id " << cntxt.m_connection_id << (cntxt.m_is_income ? " INC":" OUT") + << std::endl; + return true; + }); + } std::string s = ss.str(); return s; } @@ -1764,11 +2159,12 @@ namespace nodetool template void node_server::on_connection_close(p2p_connection_context& context) { - if (!m_net_server.is_stop_signal_sent() && !context.m_is_income) { + network_zone& zone = m_network_zones.at(context.m_remote_address.get_zone()); + if (!zone.m_net_server.is_stop_signal_sent() && !context.m_is_income) { epee::net_utils::network_address na = AUTO_VAL_INIT(na); na = context.m_remote_address; - m_peerlist.remove_from_peer_anchor(na); + zone.m_peerlist.remove_from_peer_anchor(na); } m_payload_handler.on_connection_close(context); @@ -1785,9 +2181,10 @@ namespace nodetool template template bool node_server::connect_to_peerlist(const Container& peers) { + const network_zone& public_zone = m_network_zones.at(epee::net_utils::zone::public_); for(const epee::net_utils::network_address& na: peers) { - if(m_net_server.is_stop_signal_sent()) + if(public_zone.m_net_server.is_stop_signal_sent()) return false; if(is_addr_connected(na)) @@ -1806,48 +2203,68 @@ namespace nodetool for(const std::string& pr_str: perrs) { - epee::net_utils::network_address na = AUTO_VAL_INIT(na); - const uint16_t default_port = m_nettype == cryptonote::TESTNET ? ::config::testnet::P2P_DEFAULT_PORT : m_nettype == cryptonote::STAGENET ? ::config::stagenet::P2P_DEFAULT_PORT : ::config::P2P_DEFAULT_PORT; - bool r = parse_peer_from_string(na, pr_str, default_port); - CHECK_AND_ASSERT_MES(r, false, "Failed to parse address from string: " << pr_str); - container.push_back(na); + const uint16_t default_port = cryptonote::get_config(m_nettype).P2P_DEFAULT_PORT; + expect adr = net::get_network_address(pr_str, default_port); + if (adr) + { + add_zone(adr->get_zone()); + container.push_back(std::move(*adr)); + continue; + } + std::vector resolved_addrs; + bool r = append_net_address(resolved_addrs, pr_str, default_port); + CHECK_AND_ASSERT_MES(r, false, "Failed to parse or resolve address from string: " << pr_str); + for (const epee::net_utils::network_address& addr : resolved_addrs) + { + container.push_back(addr); + } } return true; } template - bool node_server::set_max_out_peers(const boost::program_options::variables_map& vm, int64_t max) + bool node_server::set_max_out_peers(network_zone& zone, int64_t max) { if(max == -1) { - m_config.m_net_config.max_out_connection_count = P2P_DEFAULT_CONNECTIONS_COUNT; + zone.m_config.m_net_config.max_out_connection_count = P2P_DEFAULT_CONNECTIONS_COUNT; return true; } - m_config.m_net_config.max_out_connection_count = max; + zone.m_config.m_net_config.max_out_connection_count = max; return true; } template - bool node_server::set_max_in_peers(const boost::program_options::variables_map& vm, int64_t max) + bool node_server::set_max_in_peers(network_zone& zone, int64_t max) { - if(max == -1) { - m_config.m_net_config.max_in_connection_count = -1; - return true; - } - m_config.m_net_config.max_in_connection_count = max; + zone.m_config.m_net_config.max_in_connection_count = max; return true; } template - void node_server::delete_out_connections(size_t count) + void node_server::change_max_out_public_peers(size_t count) { - m_net_server.get_config_object().del_out_connections(count); + auto public_zone = m_network_zones.find(epee::net_utils::zone::public_); + if (public_zone != m_network_zones.end()) + { + const auto current = public_zone->second.m_config.m_net_config.max_out_connection_count; + public_zone->second.m_config.m_net_config.max_out_connection_count = count; + if(current > count) + public_zone->second.m_net_server.get_config_object().del_out_connections(current - count); + } } template - void node_server::delete_in_connections(size_t count) + void node_server::change_max_in_public_peers(size_t count) { - m_net_server.get_config_object().del_in_connections(count); + auto public_zone = m_network_zones.find(epee::net_utils::zone::public_); + if (public_zone != m_network_zones.end()) + { + const auto current = public_zone->second.m_config.m_net_config.max_in_connection_count; + public_zone->second.m_config.m_net_config.max_in_connection_count = count; + if(current > count) + public_zone->second.m_net_server.get_config_object().del_in_connections(current - count); + } } template @@ -1864,11 +2281,10 @@ namespace nodetool template bool node_server::set_rate_up_limit(const boost::program_options::variables_map& vm, int64_t limit) { - this->islimitup=true; + this->islimitup=(limit != -1) && (limit != default_limit_up); if (limit==-1) { limit=default_limit_up; - this->islimitup=false; } epee::net_utils::connection >::set_rate_up_limit( limit ); @@ -1879,10 +2295,9 @@ namespace nodetool template bool node_server::set_rate_down_limit(const boost::program_options::variables_map& vm, int64_t limit) { - this->islimitdown=true; + this->islimitdown=(limit != -1) && (limit != default_limit_down); if(limit==-1) { limit=default_limit_down; - this->islimitdown=false; } epee::net_utils::connection >::set_rate_down_limit( limit ); MINFO("Set limit-down to " << limit << " kB/s"); @@ -1920,10 +2335,13 @@ namespace nodetool template bool node_server::has_too_many_connections(const epee::net_utils::network_address &address) { + if (address.get_zone() != epee::net_utils::zone::public_) + return false; // Unable to determine how many connections from host + const size_t max_connections = 1; size_t count = 0; - m_net_server.get_config_object().foreach_connection([&](const p2p_connection_context& cntxt) + m_network_zones.at(epee::net_utils::zone::public_).m_net_server.get_config_object().foreach_connection([&](const p2p_connection_context& cntxt) { if (cntxt.m_is_income && cntxt.m_remote_address.is_same_host(address)) { count++; @@ -1944,31 +2362,68 @@ namespace nodetool { if (m_offline) return true; if (!m_exclusive_peers.empty()) return true; + if (m_payload_handler.needs_new_sync_connections()) return true; - peerlist_entry pe = AUTO_VAL_INIT(pe); - - if (m_net_server.is_stop_signal_sent()) - return false; - - if (!m_peerlist.get_random_gray_peer(pe)) { + for (auto& zone : m_network_zones) + { + if (zone.second.m_net_server.is_stop_signal_sent()) return false; - } - - bool success = check_connection_and_handshake_with_peer(pe.adr, pe.last_seen); - if (!success) { - m_peerlist.remove_from_peer_gray(pe); + if (zone.second.m_connect == nullptr) + continue; - LOG_PRINT_L2("PEER EVICTED FROM GRAY PEER LIST IP address: " << pe.adr.host_str() << " Peer ID: " << peerid_type(pe.id)); + peerlist_entry pe{}; + if (!zone.second.m_peerlist.get_random_gray_peer(pe)) + continue; - return true; + if (!check_connection_and_handshake_with_peer(pe.adr, pe.last_seen)) + { + zone.second.m_peerlist.remove_from_peer_gray(pe); + LOG_PRINT_L2("PEER EVICTED FROM GRAY PEER LIST IP address: " << pe.adr.host_str() << " Peer ID: " << peerid_type(pe.id)); + } + else + { + zone.second.m_peerlist.set_peer_just_seen(pe.id, pe.adr, pe.pruning_seed, pe.rpc_port); + LOG_PRINT_L2("PEER PROMOTED TO WHITE PEER LIST IP address: " << pe.adr.host_str() << " Peer ID: " << peerid_type(pe.id)); + } } + return true; + } - m_peerlist.set_peer_just_seen(pe.id, pe.adr); + template + void node_server::add_used_stripe_peer(const typename t_payload_net_handler::connection_context &context) + { + const uint32_t stripe = tools::get_pruning_stripe(context.m_pruning_seed); + if (stripe == 0 || stripe > (1ul << CRYPTONOTE_PRUNING_LOG_STRIPES)) + return; + const uint32_t index = stripe - 1; + CRITICAL_REGION_LOCAL(m_used_stripe_peers_mutex); + MINFO("adding stripe " << stripe << " peer: " << context.m_remote_address.str()); + std::remove_if(m_used_stripe_peers[index].begin(), m_used_stripe_peers[index].end(), + [&context](const epee::net_utils::network_address &na){ return context.m_remote_address == na; }); + m_used_stripe_peers[index].push_back(context.m_remote_address); + } - LOG_PRINT_L2("PEER PROMOTED TO WHITE PEER LIST IP address: " << pe.adr.host_str() << " Peer ID: " << peerid_type(pe.id)); + template + void node_server::remove_used_stripe_peer(const typename t_payload_net_handler::connection_context &context) + { + const uint32_t stripe = tools::get_pruning_stripe(context.m_pruning_seed); + if (stripe == 0 || stripe > (1ul << CRYPTONOTE_PRUNING_LOG_STRIPES)) + return; + const uint32_t index = stripe - 1; + CRITICAL_REGION_LOCAL(m_used_stripe_peers_mutex); + MINFO("removing stripe " << stripe << " peer: " << context.m_remote_address.str()); + std::remove_if(m_used_stripe_peers[index].begin(), m_used_stripe_peers[index].end(), + [&context](const epee::net_utils::network_address &na){ return context.m_remote_address == na; }); + } - return true; + template + void node_server::clear_used_stripe_peers() + { + CRITICAL_REGION_LOCAL(m_used_stripe_peers_mutex); + MINFO("clearing used stripe peers"); + for (auto &e: m_used_stripe_peers) + e.clear(); } template @@ -1988,7 +2443,7 @@ namespace nodetool char lanAddress[64]; result = UPNP_GetValidIGD(deviceList, &urls, &igdData, lanAddress, sizeof lanAddress); freeUPNPDevlist(deviceList); - if (result != 0) { + if (result > 0) { if (result == 1) { std::ostringstream portString; portString << port; @@ -2034,7 +2489,7 @@ namespace nodetool char lanAddress[64]; result = UPNP_GetValidIGD(deviceList, &urls, &igdData, lanAddress, sizeof lanAddress); freeUPNPDevlist(deviceList); - if (result != 0) { + if (result > 0) { if (result == 1) { std::ostringstream portString; portString << port; @@ -2059,4 +2514,37 @@ namespace nodetool MINFO("No IGD was found."); } } + + template + boost::optional> + node_server::socks_connect(network_zone& zone, const epee::net_utils::network_address& remote, epee::net_utils::ssl_support_t ssl_support) + { + auto result = socks_connect_internal(zone.m_net_server.get_stop_signal(), zone.m_net_server.get_io_service(), zone.m_proxy_address, remote); + if (result) // if no error + { + p2p_connection_context context{}; + if (zone.m_net_server.add_connection(context, std::move(*result), remote, ssl_support)) + return {std::move(context)}; + } + return boost::none; + } + + template + boost::optional> + node_server::public_connect(network_zone& zone, epee::net_utils::network_address const& na, epee::net_utils::ssl_support_t ssl_support) + { + CHECK_AND_ASSERT_MES(na.get_type_id() == epee::net_utils::ipv4_network_address::get_type_id(), boost::none, + "Only IPv4 addresses are supported here"); + const epee::net_utils::ipv4_network_address &ipv4 = na.as(); + + typename net_server::t_connection_context con{}; + const bool res = zone.m_net_server.connect(epee::string_tools::get_ip_string_from_int32(ipv4.ip()), + epee::string_tools::num_to_string_fast(ipv4.port()), + zone.m_config.m_net_config.connection_timeout, + con, "0.0.0.0", ssl_support); + + if (res) + return {std::move(con)}; + return boost::none; + } } diff --git a/src/p2p/net_node_common.h b/src/p2p/net_node_common.h index 218250efa2e..26451b33330 100644 --- a/src/p2p/net_node_common.h +++ b/src/p2p/net_node_common.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -31,6 +31,8 @@ #pragma once #include +#include +#include #include "net/net_utils_base.h" #include "p2p_protocol_defs.h" @@ -43,37 +45,36 @@ namespace nodetool template struct i_p2p_endpoint { - virtual bool relay_notify_to_list(int command, const std::string& data_buff, const std::list& connections)=0; - virtual bool relay_notify_to_all(int command, const std::string& data_buff, const epee::net_utils::connection_context_base& context)=0; - virtual bool invoke_command_to_peer(int command, const std::string& req_buff, std::string& resp_buff, const epee::net_utils::connection_context_base& context)=0; - virtual bool invoke_notify_to_peer(int command, const std::string& req_buff, const epee::net_utils::connection_context_base& context)=0; + virtual bool relay_notify_to_list(int command, const epee::span data_buff, std::vector> connections)=0; + virtual bool invoke_command_to_peer(int command, const epee::span req_buff, std::string& resp_buff, const epee::net_utils::connection_context_base& context)=0; + virtual bool invoke_notify_to_peer(int command, const epee::span req_buff, const epee::net_utils::connection_context_base& context)=0; virtual bool drop_connection(const epee::net_utils::connection_context_base& context)=0; virtual void request_callback(const epee::net_utils::connection_context_base& context)=0; - virtual uint64_t get_connections_count()=0; + virtual uint64_t get_public_connections_count()=0; + virtual size_t get_zone_count() const=0; virtual void for_each_connection(std::function f)=0; virtual bool for_connection(const boost::uuids::uuid&, std::function f)=0; virtual bool block_host(const epee::net_utils::network_address &address, time_t seconds = 0)=0; virtual bool unblock_host(const epee::net_utils::network_address &address)=0; virtual std::map get_blocked_hosts()=0; virtual bool add_host_fail(const epee::net_utils::network_address &address)=0; + virtual void add_used_stripe_peer(const t_connection_context &context)=0; + virtual void remove_used_stripe_peer(const t_connection_context &context)=0; + virtual void clear_used_stripe_peers()=0; }; template struct p2p_endpoint_stub: public i_p2p_endpoint { - virtual bool relay_notify_to_list(int command, const std::string& data_buff, const std::list& connections) + virtual bool relay_notify_to_list(int command, const epee::span data_buff, std::vector> connections) { return false; } - virtual bool relay_notify_to_all(int command, const std::string& data_buff, const epee::net_utils::connection_context_base& context) + virtual bool invoke_command_to_peer(int command, const epee::span req_buff, std::string& resp_buff, const epee::net_utils::connection_context_base& context) { return false; } - virtual bool invoke_command_to_peer(int command, const std::string& req_buff, std::string& resp_buff, const epee::net_utils::connection_context_base& context) - { - return false; - } - virtual bool invoke_notify_to_peer(int command, const std::string& req_buff, const epee::net_utils::connection_context_base& context) + virtual bool invoke_notify_to_peer(int command, const epee::span req_buff, const epee::net_utils::connection_context_base& context) { return true; } @@ -94,7 +95,12 @@ namespace nodetool return false; } - virtual uint64_t get_connections_count() + virtual size_t get_zone_count() const + { + return 0; + } + + virtual uint64_t get_public_connections_count() { return false; } @@ -114,5 +120,14 @@ namespace nodetool { return true; } + virtual void add_used_stripe_peer(const t_connection_context &context) + { + } + virtual void remove_used_stripe_peer(const t_connection_context &context) + { + } + virtual void clear_used_stripe_peers() + { + } }; } diff --git a/src/p2p/net_peerlist.cpp b/src/p2p/net_peerlist.cpp new file mode 100644 index 00000000000..ce5c67fe5c5 --- /dev/null +++ b/src/p2p/net_peerlist.cpp @@ -0,0 +1,295 @@ +// Copyright (c) 2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "net_peerlist.h" + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "net_peerlist_boost_serialization.h" + + +namespace nodetool +{ + namespace + { + constexpr unsigned CURRENT_PEERLIST_STORAGE_ARCHIVE_VER = 6; + + struct by_zone + { + using zone = epee::net_utils::zone; + + template + bool operator()(const T& left, const zone right) const + { + return left.adr.get_zone() < right; + } + + template + bool operator()(const zone left, const T& right) const + { + return left < right.adr.get_zone(); + } + + template + bool operator()(const T& left, const U& right) const + { + return left.adr.get_zone() < right.adr.get_zone(); + } + }; + + template + std::vector load_peers(Archive& a, unsigned ver) + { + // at v6, we drop existing peerlists, because annoying change + if (ver < 6) + return {}; + + uint64_t size = 0; + a & size; + + Elem ple{}; + + std::vector elems{}; + elems.reserve(size); + while (size--) + { + a & ple; + elems.push_back(std::move(ple)); + } + + return elems; + } + + template + void save_peers(Archive& a, const Range& elems) + { + const uint64_t size = elems.size(); + a & size; + for (const auto& elem : elems) + a & elem; + } + + template + std::vector do_take_zone(std::vector& src, epee::net_utils::zone zone) + { + const auto start = std::lower_bound(src.begin(), src.end(), zone, by_zone{}); + const auto end = std::upper_bound(start, src.end(), zone, by_zone{}); + + std::vector out{}; + out.assign(std::make_move_iterator(start), std::make_move_iterator(end)); + src.erase(start, end); + return out; + } + + template + void add_peers(Container& dest, std::vector&& src) + { + dest.insert(std::make_move_iterator(src.begin()), std::make_move_iterator(src.end())); + } + + template + void copy_peers(Container& dest, const Range& src) + { + std::copy(src.begin(), src.end(), std::back_inserter(dest)); + } + } // anonymous + + struct peerlist_join + { + const peerlist_types& ours; + const peerlist_types& other; + }; + + template + void serialize(Archive& a, peerlist_types& elem, unsigned ver) + { + elem.white = load_peers(a, ver); + elem.gray = load_peers(a, ver); + elem.anchor = load_peers(a, ver); + + if (ver == 0) + { + // from v1, we do not store the peer id anymore + peerid_type peer_id{}; + a & peer_id; + } + } + + template + void serialize(Archive& a, peerlist_join elem, unsigned ver) + { + save_peers(a, boost::range::join(elem.ours.white, elem.other.white)); + save_peers(a, boost::range::join(elem.ours.gray, elem.other.gray)); + save_peers(a, boost::range::join(elem.ours.anchor, elem.other.anchor)); + } + + boost::optional peerlist_storage::open(std::istream& src, const bool new_format) + { + try + { + peerlist_storage out{}; + if (new_format) + { + boost::archive::portable_binary_iarchive a{src}; + a >> out.m_types; + } + else + { + boost::archive::binary_iarchive a{src}; + a >> out.m_types; + } + + if (src.good()) + { + std::sort(out.m_types.white.begin(), out.m_types.white.end(), by_zone{}); + std::sort(out.m_types.gray.begin(), out.m_types.gray.end(), by_zone{}); + std::sort(out.m_types.anchor.begin(), out.m_types.anchor.end(), by_zone{}); + return {std::move(out)}; + } + } + catch (const std::exception& e) + {} + + return boost::none; + } + + boost::optional peerlist_storage::open(const std::string& path) + { + std::ifstream src_file{}; + src_file.open( path , std::ios_base::binary | std::ios_base::in); + if(src_file.fail()) + return boost::none; + + boost::optional out = open(src_file, true); + if (!out) + { + // if failed, try reading in unportable mode + boost::filesystem::copy_file(path, path + ".unportable", boost::filesystem::copy_option::overwrite_if_exists); + src_file.close(); + src_file.open( path , std::ios_base::binary | std::ios_base::in); + if(src_file.fail()) + return boost::none; + + out = open(src_file, false); + if (!out) + { + // This is different from the `return boost::none` cases above. Those + // cases could fail due to bad file permissions, so a shutdown is + // likely more appropriate. + MWARNING("Failed to load p2p config file, falling back to default config"); + out.emplace(); + } + } + + return out; + } + + peerlist_storage::~peerlist_storage() noexcept + {} + + bool peerlist_storage::store(std::ostream& dest, const peerlist_types& other) const + { + try + { + boost::archive::portable_binary_oarchive a{dest}; + const peerlist_join pj{std::cref(m_types), std::cref(other)}; + a << pj; + return dest.good(); + } + catch (const boost::archive::archive_exception& e) + {} + + return false; + } + + bool peerlist_storage::store(const std::string& path, const peerlist_types& other) const + { + std::ofstream dest_file{}; + dest_file.open( path , std::ios_base::binary | std::ios_base::out| std::ios::trunc); + if(dest_file.fail()) + return false; + + return store(dest_file, other); + } + + peerlist_types peerlist_storage::take_zone(epee::net_utils::zone zone) + { + peerlist_types out{}; + out.white = do_take_zone(m_types.white, zone); + out.gray = do_take_zone(m_types.gray, zone); + out.anchor = do_take_zone(m_types.anchor, zone); + return out; + } + + bool peerlist_manager::init(peerlist_types&& peers, bool allow_local_ip) + { + CRITICAL_REGION_LOCAL(m_peerlist_lock); + + if (!m_peers_white.empty() || !m_peers_gray.empty() || !m_peers_anchor.empty()) + return false; + + add_peers(m_peers_white.get(), std::move(peers.white)); + add_peers(m_peers_gray.get(), std::move(peers.gray)); + add_peers(m_peers_anchor.get(), std::move(peers.anchor)); + m_allow_local_ip = allow_local_ip; + return true; + } + + void peerlist_manager::get_peerlist(std::vector& pl_gray, std::vector& pl_white) + { + CRITICAL_REGION_LOCAL(m_peerlist_lock); + copy_peers(pl_gray, m_peers_gray.get()); + copy_peers(pl_white, m_peers_white.get()); + } + + void peerlist_manager::get_peerlist(peerlist_types& peers) + { + CRITICAL_REGION_LOCAL(m_peerlist_lock); + peers.white.reserve(peers.white.size() + m_peers_white.size()); + peers.gray.reserve(peers.gray.size() + m_peers_gray.size()); + peers.anchor.reserve(peers.anchor.size() + m_peers_anchor.size()); + + copy_peers(peers.white, m_peers_white.get()); + copy_peers(peers.gray, m_peers_gray.get()); + copy_peers(peers.anchor, m_peers_anchor.get()); + } +} + +BOOST_CLASS_VERSION(nodetool::peerlist_types, nodetool::CURRENT_PEERLIST_STORAGE_ARCHIVE_VER); +BOOST_CLASS_VERSION(nodetool::peerlist_join, nodetool::CURRENT_PEERLIST_STORAGE_ARCHIVE_VER); + diff --git a/src/p2p/net_peerlist.h b/src/p2p/net_peerlist.h index 1d609a37e1a..883997fd6e0 100644 --- a/src/p2p/net_peerlist.h +++ b/src/p2p/net_peerlist.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -30,33 +30,67 @@ #pragma once +#include #include -#include -#include -#include -#include -#include -#include +#include +#include #include #include #include #include +#include #include -#include "syncobj.h" +#include "cryptonote_config.h" +#include "net/enums.h" #include "net/local_ip.h" #include "p2p_protocol_defs.h" -#include "cryptonote_config.h" -#include "net_peerlist_boost_serialization.h" - - -#define CURRENT_PEERLIST_STORAGE_ARCHIVE_VER 6 +#include "syncobj.h" namespace nodetool { + struct peerlist_types + { + std::vector white; + std::vector gray; + std::vector anchor; + }; + + class peerlist_storage + { + public: + peerlist_storage() + : m_types{} + {} + + //! \return Peers stored in stream `src` in `new_format` (portable archive or older non-portable). + static boost::optional open(std::istream& src, const bool new_format); + //! \return Peers stored in file at `path` + static boost::optional open(const std::string& path); + + peerlist_storage(peerlist_storage&&) = default; + peerlist_storage(const peerlist_storage&) = delete; + + ~peerlist_storage() noexcept; + + peerlist_storage& operator=(peerlist_storage&&) = default; + peerlist_storage& operator=(const peerlist_storage&) = delete; + + //! Save peers from `this` and `other` in stream `dest`. + bool store(std::ostream& dest, const peerlist_types& other) const; + + //! Save peers from `this` and `other` in one file at `path`. + bool store(const std::string& path, const peerlist_types& other) const; + + //! \return Peers in `zone` and from remove from `this`. + peerlist_types take_zone(epee::net_utils::zone zone); + + private: + peerlist_types m_types; + }; /************************************************************************/ /* */ @@ -64,25 +98,27 @@ namespace nodetool class peerlist_manager { public: - bool init(bool allow_local_ip); - bool deinit(); + bool init(peerlist_types&& peers, bool allow_local_ip); size_t get_white_peers_count(){CRITICAL_REGION_LOCAL(m_peerlist_lock); return m_peers_white.size();} size_t get_gray_peers_count(){CRITICAL_REGION_LOCAL(m_peerlist_lock); return m_peers_gray.size();} - bool merge_peerlist(const std::list& outer_bs); - bool get_peerlist_head(std::list& bs_head, uint32_t depth = P2P_DEFAULT_PEERS_IN_HANDSHAKE); - bool get_peerlist_full(std::list& pl_gray, std::list& pl_white); + bool merge_peerlist(const std::vector& outer_bs); + bool get_peerlist_head(std::vector& bs_head, bool anonymize, uint32_t depth = P2P_DEFAULT_PEERS_IN_HANDSHAKE); + void get_peerlist(std::vector& pl_gray, std::vector& pl_white); + void get_peerlist(peerlist_types& peers); bool get_white_peer_by_index(peerlist_entry& p, size_t i); bool get_gray_peer_by_index(peerlist_entry& p, size_t i); + template bool foreach(bool white, const F &f); bool append_with_peer_white(const peerlist_entry& pr); bool append_with_peer_gray(const peerlist_entry& pr); bool append_with_peer_anchor(const anchor_peerlist_entry& ple); - bool set_peer_just_seen(peerid_type peer, const epee::net_utils::network_address& addr); + bool set_peer_just_seen(peerid_type peer, const epee::net_utils::network_address& addr, uint32_t pruning_seed, uint16_t rpc_port); bool set_peer_unreachable(const peerlist_entry& pr); bool is_host_allowed(const epee::net_utils::network_address &address); bool get_random_gray_peer(peerlist_entry& pe); bool remove_from_peer_gray(const peerlist_entry& pe); bool get_and_empty_anchor_peerlist(std::vector& apl); bool remove_from_peer_anchor(const epee::net_utils::network_address& addr); + bool remove_from_peer_white(const peerlist_entry& pe); private: struct by_time{}; @@ -133,18 +169,6 @@ namespace nodetool > > peers_indexed; - typedef boost::multi_index_container< - peerlist_entry, - boost::multi_index::indexed_by< - // access by peerlist_entry::id< - boost::multi_index::ordered_unique, boost::multi_index::member >, - // access by peerlist_entry::net_adress - boost::multi_index::ordered_unique, boost::multi_index::member >, - // sort by peerlist_entry::last_seen< - boost::multi_index::ordered_non_unique, boost::multi_index::member > - > - > peers_indexed_old; - typedef boost::multi_index_container< anchor_peerlist_entry, boost::multi_index::indexed_by< @@ -154,56 +178,8 @@ namespace nodetool boost::multi_index::ordered_non_unique, boost::multi_index::member > > > anchor_peers_indexed; - public: - - template - void serialize_peers(Archive &a, List &list, Element ple, const t_version_type ver) - { - if (typename Archive::is_saving()) - { - uint64_t size = list.size(); - a & size; - for (auto p: list) - { - a & p; - } - } - else - { - uint64_t size; - a & size; - list.clear(); - while (size--) - { - a & ple; - list.insert(ple); - } - } - } - - template - void serialize(Archive &a, const t_version_type ver) - { - // at v6, we drop existing peerlists, because annoying change - if (ver < 6) - return; - - CRITICAL_REGION_LOCAL(m_peerlist_lock); - -#if 0 - // trouble loading more than one peer, can't find why - a & m_peers_white; - a & m_peers_gray; - a & m_peers_anchor; -#else - serialize_peers(a, m_peers_white, peerlist_entry(), ver); - serialize_peers(a, m_peers_gray, peerlist_entry(), ver); - serialize_peers(a, m_peers_anchor, anchor_peerlist_entry(), ver); -#endif - } private: - bool peers_indexed_from_old(const peers_indexed_old& pio, peers_indexed& pi); void trim_white_peerlist(); void trim_gray_peerlist(); @@ -218,34 +194,6 @@ namespace nodetool anchor_peers_indexed m_peers_anchor; }; //-------------------------------------------------------------------------------------------------- - inline - bool peerlist_manager::init(bool allow_local_ip) - { - m_allow_local_ip = allow_local_ip; - return true; - } - //-------------------------------------------------------------------------------------------------- - inline - bool peerlist_manager::deinit() - { - return true; - } - //-------------------------------------------------------------------------------------------------- - inline - bool peerlist_manager::peers_indexed_from_old(const peers_indexed_old& pio, peers_indexed& pi) - { - for(auto x: pio) - { - auto by_addr_it = pi.get().find(x.adr); - if(by_addr_it == pi.get().end()) - { - pi.insert(x); - } - } - - return true; - } - //-------------------------------------------------------------------------------------------------- inline void peerlist_manager::trim_gray_peerlist() { while(m_peers_gray.size() > P2P_LOCAL_GRAY_PEERLIST_LIMIT) @@ -265,7 +213,7 @@ namespace nodetool } //-------------------------------------------------------------------------------------------------- inline - bool peerlist_manager::merge_peerlist(const std::list& outer_bs) + bool peerlist_manager::merge_peerlist(const std::vector& outer_bs) { CRITICAL_REGION_LOCAL(m_peerlist_lock); for(const peerlist_entry& be: outer_bs) @@ -315,46 +263,56 @@ namespace nodetool } //-------------------------------------------------------------------------------------------------- inline - bool peerlist_manager::get_peerlist_head(std::list& bs_head, uint32_t depth) + bool peerlist_manager::get_peerlist_head(std::vector& bs_head, bool anonymize, uint32_t depth) { - CRITICAL_REGION_LOCAL(m_peerlist_lock); peers_indexed::index::type& by_time_index=m_peers_white.get(); uint32_t cnt = 0; + + // picks a random set of peers within the first 120%, rather than a set of the first 100%. + // The intent is that if someone asks twice, they can't easily tell: + // - this address was not in the first list, but is in the second, so the only way this can be + // is if its last_seen was recently reset, so this means the target node recently had a new + // connection to that address + // - this address was in the first list, and not in the second, which means either the address + // was moved to the gray list (if it's not accessibe, which the attacker can check if + // the address accepts incoming connections) or it was the oldest to still fit in the 250 items, + // so its last_seen is old. + const uint32_t pick_depth = anonymize ? depth + depth / 5 : depth; + bs_head.reserve(pick_depth); for(const peers_indexed::value_type& vl: boost::adaptors::reverse(by_time_index)) { - if(!vl.last_seen) - continue; - - if(cnt++ >= depth) + if(cnt++ >= pick_depth) break; bs_head.push_back(vl); } - return true; - } - //-------------------------------------------------------------------------------------------------- - inline - bool peerlist_manager::get_peerlist_full(std::list& pl_gray, std::list& pl_white) - { - CRITICAL_REGION_LOCAL(m_peerlist_lock); - peers_indexed::index::type& by_time_index_gr=m_peers_gray.get(); - for(const peers_indexed::value_type& vl: boost::adaptors::reverse(by_time_index_gr)) - { - pl_gray.push_back(vl); - } - peers_indexed::index::type& by_time_index_wt=m_peers_white.get(); - for(const peers_indexed::value_type& vl: boost::adaptors::reverse(by_time_index_wt)) + if (anonymize) { - pl_white.push_back(vl); + std::random_shuffle(bs_head.begin(), bs_head.end()); + if (bs_head.size() > depth) + bs_head.resize(depth); + for (auto &e: bs_head) + e.last_seen = 0; } return true; } //-------------------------------------------------------------------------------------------------- + template inline + bool peerlist_manager::foreach(bool white, const F &f) + { + CRITICAL_REGION_LOCAL(m_peerlist_lock); + peers_indexed::index::type& by_time_index = white ? m_peers_white.get() : m_peers_gray.get(); + for(const peers_indexed::value_type& vl: boost::adaptors::reverse(by_time_index)) + if (!f(vl)) + return false; + return true; + } + //-------------------------------------------------------------------------------------------------- inline - bool peerlist_manager::set_peer_just_seen(peerid_type peer, const epee::net_utils::network_address& addr) + bool peerlist_manager::set_peer_just_seen(peerid_type peer, const epee::net_utils::network_address& addr, uint32_t pruning_seed, uint16_t rpc_port) { TRY_ENTRY(); CRITICAL_REGION_LOCAL(m_peerlist_lock); @@ -363,6 +321,8 @@ namespace nodetool ple.adr = addr; ple.id = peer; ple.last_seen = time(NULL); + ple.pruning_seed = pruning_seed; + ple.rpc_port = rpc_port; return append_with_peer_white(ple); CATCH_ENTRY_L0("peerlist_manager::set_peer_just_seen()", false); } @@ -384,8 +344,14 @@ namespace nodetool trim_white_peerlist(); }else { - //update record in white list - m_peers_white.replace(by_addr_it_wt, ple); + //update record in white list + peerlist_entry new_ple = ple; + if (by_addr_it_wt->pruning_seed && ple.pruning_seed == 0) // guard against older nodes not passing pruning info around + new_ple.pruning_seed = by_addr_it_wt->pruning_seed; + if (by_addr_it_wt->rpc_port && ple.rpc_port == 0) // guard against older nodes not passing RPC port around + new_ple.rpc_port = by_addr_it_wt->rpc_port; + new_ple.last_seen = by_addr_it_wt->last_seen; // do not overwrite the last seen timestamp, incoming peer list are untrusted + m_peers_white.replace(by_addr_it_wt, new_ple); } //remove from gray list, if need auto by_addr_it_gr = m_peers_gray.get().find(ple.adr); @@ -419,8 +385,14 @@ namespace nodetool trim_gray_peerlist(); }else { - //update record in white list - m_peers_gray.replace(by_addr_it_gr, ple); + //update record in gray list + peerlist_entry new_ple = ple; + if (by_addr_it_gr->pruning_seed && ple.pruning_seed == 0) // guard against older nodes not passing pruning info around + new_ple.pruning_seed = by_addr_it_gr->pruning_seed; + if (by_addr_it_gr->rpc_port && ple.rpc_port == 0) // guard against older nodes not passing RPC port around + new_ple.rpc_port = by_addr_it_gr->rpc_port; + new_ple.last_seen = by_addr_it_gr->last_seen; // do not overwrite the last seen timestamp, incoming peer list are untrusted + m_peers_gray.replace(by_addr_it_gr, new_ple); } return true; CATCH_ENTRY_L0("peerlist_manager::append_with_peer_gray()", false); @@ -455,7 +427,7 @@ namespace nodetool return false; } - size_t random_index = crypto::rand() % m_peers_gray.size(); + size_t random_index = crypto::rand_idx(m_peers_gray.size()); peers_indexed::index::type& by_time_index = m_peers_gray.get(); pe = *epee::misc_utils::move_it_backward(--by_time_index.end(), random_index); @@ -466,6 +438,24 @@ namespace nodetool } //-------------------------------------------------------------------------------------------------- inline + bool peerlist_manager::remove_from_peer_white(const peerlist_entry& pe) + { + TRY_ENTRY(); + + CRITICAL_REGION_LOCAL(m_peerlist_lock); + + peers_indexed::index_iterator::type iterator = m_peers_white.get().find(pe.adr); + + if (iterator != m_peers_white.get().end()) { + m_peers_white.erase(iterator); + } + + return true; + + CATCH_ENTRY_L0("peerlist_manager::remove_from_peer_white()", false); + } + //-------------------------------------------------------------------------------------------------- + inline bool peerlist_manager::remove_from_peer_gray(const peerlist_entry& pe) { TRY_ENTRY(); @@ -524,4 +514,3 @@ namespace nodetool //-------------------------------------------------------------------------------------------------- } -BOOST_CLASS_VERSION(nodetool::peerlist_manager, CURRENT_PEERLIST_STORAGE_ARCHIVE_VER) diff --git a/src/p2p/net_peerlist_boost_serialization.h b/src/p2p/net_peerlist_boost_serialization.h index e79207888a1..32f30adca3b 100644 --- a/src/p2p/net_peerlist_boost_serialization.h +++ b/src/p2p/net_peerlist_boost_serialization.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -30,32 +30,59 @@ #pragma once +#include + +#include "common/expect.h" #include "net/net_utils_base.h" +#include "net/tor_address.h" +#include "net/i2p_address.h" #include "p2p/p2p_protocol_defs.h" +#ifdef CRYPTONOTE_PRUNING_DEBUG_SPOOF_SEED +#include "common/pruning.h" +#endif + +BOOST_CLASS_VERSION(nodetool::peerlist_entry, 2) + namespace boost { namespace serialization { template - inline void do_serialize(Archive &a, epee::net_utils::network_address& na, T local) + inline void do_serialize(boost::mpl::false_, Archive &a, epee::net_utils::network_address& na) + { + T addr{}; + a & addr; + na = std::move(addr); + } + + template + inline void do_serialize(boost::mpl::true_, Archive &a, const epee::net_utils::network_address& na) { - if (typename Archive::is_saving()) local = na.as(); - a & local; - if (!typename Archive::is_saving()) na = local; + a & na.as(); } + template inline void serialize(Archive &a, epee::net_utils::network_address& na, const ver_type ver) { + static constexpr const typename Archive::is_saving is_saving{}; + uint8_t type; - if (typename Archive::is_saving()) - type = na.get_type_id(); + if (is_saving) + type = uint8_t(na.get_type_id()); a & type; - switch (type) + switch (epee::net_utils::address_type(type)) { - case epee::net_utils::ipv4_network_address::ID: - do_serialize(a, na, epee::net_utils::ipv4_network_address{0, 0}); - break; + case epee::net_utils::ipv4_network_address::get_type_id(): + do_serialize(is_saving, a, na); + break; + case net::tor_address::get_type_id(): + do_serialize(is_saving, a, na); + break; + case net::i2p_address::get_type_id(): + do_serialize(is_saving, a, na); + break; + case epee::net_utils::address_type::invalid: default: throw std::runtime_error("Unsupported network address type"); } @@ -71,12 +98,116 @@ namespace boost na = epee::net_utils::ipv4_network_address{ip, port}; } + template + inline void save(Archive& a, const net::tor_address& na, const ver_type) + { + const size_t length = std::strlen(na.host_str()); + if (length > 255) + MONERO_THROW(net::error::invalid_tor_address, "Tor address too long"); + + const uint16_t port{na.port()}; + const uint8_t len = length; + a & port; + a & len; + a.save_binary(na.host_str(), length); + } + + template + inline void save(Archive& a, const net::i2p_address& na, const ver_type) + { + const size_t length = std::strlen(na.host_str()); + if (length > 255) + MONERO_THROW(net::error::invalid_i2p_address, "i2p address too long"); + + const uint16_t port{na.port()}; + const uint8_t len = length; + a & port; + a & len; + a.save_binary(na.host_str(), length); + } + + template + inline void load(Archive& a, net::tor_address& na, const ver_type) + { + uint16_t port = 0; + uint8_t length = 0; + a & port; + a & length; + + const size_t buffer_size = net::tor_address::buffer_size(); + if (length > buffer_size) + MONERO_THROW(net::error::invalid_tor_address, "Tor address too long"); + + char host[buffer_size] = {0}; + a.load_binary(host, length); + host[sizeof(host) - 1] = 0; + + if (std::strcmp(host, net::tor_address::unknown_str()) == 0) + na = net::tor_address::unknown(); + else + na = MONERO_UNWRAP(net::tor_address::make(host, port)); + } + + template + inline void load(Archive& a, net::i2p_address& na, const ver_type) + { + uint16_t port = 0; + uint8_t length = 0; + a & port; + a & length; + + const size_t buffer_size = net::i2p_address::buffer_size(); + if (length > buffer_size) + MONERO_THROW(net::error::invalid_i2p_address, "i2p address too long"); + + char host[buffer_size] = {0}; + a.load_binary(host, length); + host[sizeof(host) - 1] = 0; + + if (std::strcmp(host, net::i2p_address::unknown_str()) == 0) + na = net::i2p_address::unknown(); + else + na = MONERO_UNWRAP(net::i2p_address::make(host, port)); + } + + template + inline void serialize(Archive &a, net::tor_address& na, const ver_type ver) + { + boost::serialization::split_free(a, na, ver); + } + + template + inline void serialize(Archive &a, net::i2p_address& na, const ver_type ver) + { + boost::serialization::split_free(a, na, ver); + } + template inline void serialize(Archive &a, nodetool::peerlist_entry& pl, const ver_type ver) { a & pl.adr; a & pl.id; a & pl.last_seen; + if (ver < 1) + { + if (!typename Archive::is_saving()) + pl.pruning_seed = 0; + return; + } + a & pl.pruning_seed; +#ifdef CRYPTONOTE_PRUNING_DEBUG_SPOOF_SEED + if (!typename Archive::is_saving()) + { + pl.pruning_seed = tools::make_pruning_seed(1+pl.adr.as().ip() % (1< diff --git a/src/p2p/p2p_protocol_defs.h b/src/p2p/p2p_protocol_defs.h index e793e19b63d..85774fcd517 100644 --- a/src/p2p/p2p_protocol_defs.h +++ b/src/p2p/p2p_protocol_defs.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -31,8 +31,11 @@ #pragma once #include +#include #include "serialization/keyvalue_serialization.h" #include "net/net_utils_base.h" +#include "net/tor_address.h" // needed for serialization +#include "net/i2p_address.h" // needed for serialization #include "misc_language.h" #include "string_tools.h" #include "time_helper.h" @@ -72,11 +75,16 @@ namespace nodetool AddressType adr; peerid_type id; int64_t last_seen; + uint32_t pruning_seed; + uint16_t rpc_port; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(adr) KV_SERIALIZE(id) - KV_SERIALIZE(last_seen) + if (!is_store || this_ref.last_seen != 0) + KV_SERIALIZE_OPT(last_seen, (int64_t)0) + KV_SERIALIZE_OPT(pruning_seed, (uint32_t)0) + KV_SERIALIZE_OPT(rpc_port, (uint16_t)0) END_KV_SERIALIZE_MAP() }; typedef peerlist_entry_base peerlist_entry; @@ -114,7 +122,7 @@ namespace nodetool #pragma pack(pop) inline - std::string print_peerlist_to_string(const std::list& pl) + std::string print_peerlist_to_string(const std::vector& pl) { time_t now_time = 0; time(&now_time); @@ -122,7 +130,11 @@ namespace nodetool ss << std::setfill ('0') << std::setw (8) << std::hex << std::noshowbase; for(const peerlist_entry& pe: pl) { - ss << pe.id << "\t" << pe.adr.str() << " \tlast_seen: " << epee::misc_utils::get_time_interval_string(now_time - pe.last_seen) << std::endl; + ss << pe.id << "\t" << pe.adr.str() + << " \trpc port " << (pe.rpc_port > 0 ? std::to_string(pe.rpc_port) : "-") + << " \tpruning seed " << pe.pruning_seed + << " \tlast_seen: " << (pe.last_seen == 0 ? std::string("never") : epee::misc_utils::get_time_interval_string(now_time - pe.last_seen)) + << std::endl; } return ss.str(); } @@ -153,6 +165,7 @@ namespace nodetool uuid network_id; uint64_t local_time; uint32_t my_port; + uint16_t rpc_port; peerid_type peer_id; BEGIN_KV_SERIALIZE_MAP() @@ -160,6 +173,7 @@ namespace nodetool KV_SERIALIZE(peer_id) KV_SERIALIZE(local_time) KV_SERIALIZE(my_port) + KV_SERIALIZE_OPT(rpc_port, (uint16_t)(0)) END_KV_SERIALIZE_MAP() }; @@ -174,7 +188,7 @@ namespace nodetool { const static int ID = P2P_COMMANDS_POOL_BASE + 1; - struct request + struct request_t { basic_node_data node_data; t_playload_type payload_data; @@ -184,12 +198,13 @@ namespace nodetool KV_SERIALIZE(payload_data) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init request; - struct response + struct response_t { basic_node_data node_data; t_playload_type payload_data; - std::list local_peerlist_new; + std::vector local_peerlist_new; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(node_data) @@ -198,14 +213,14 @@ namespace nodetool { // saving: save both, so old and new peers can understand it KV_SERIALIZE(local_peerlist_new) - std::list> local_peerlist; + std::vector> local_peerlist; for (const auto &p: this_ref.local_peerlist_new) { - if (p.adr.get_type_id() == epee::net_utils::ipv4_network_address::ID) + if (p.adr.get_type_id() == epee::net_utils::ipv4_network_address::get_type_id()) { const epee::net_utils::network_address &na = p.adr; const epee::net_utils::ipv4_network_address &ipv4 = na.as(); - local_peerlist.push_back(peerlist_entry_base({{ipv4.ip(), ipv4.port()}, p.id, p.last_seen})); + local_peerlist.push_back(peerlist_entry_base({{ipv4.ip(), ipv4.port()}, p.id, p.last_seen, p.pruning_seed, p.rpc_port})); } else MDEBUG("Not including in legacy peer list: " << p.adr.str()); @@ -217,15 +232,16 @@ namespace nodetool // loading: load old list only if there is no new one if (!epee::serialization::selector::serialize(this_ref.local_peerlist_new, stg, hparent_section, "local_peerlist_new")) { - std::list> local_peerlist; + std::vector> local_peerlist; epee::serialization::selector::serialize_stl_container_pod_val_as_blob(local_peerlist, stg, hparent_section, "local_peerlist"); for (const auto &p: local_peerlist) - ((response&)this_ref).local_peerlist_new.push_back(peerlist_entry({epee::net_utils::ipv4_network_address(p.adr.ip, p.adr.port), p.id, p.last_seen})); + ((response&)this_ref).local_peerlist_new.push_back(peerlist_entry({epee::net_utils::ipv4_network_address(p.adr.ip, p.adr.port), p.id, p.last_seen, p.pruning_seed, p.rpc_port})); } } END_KV_SERIALIZE_MAP() }; - }; + typedef epee::misc_utils::struct_init response; + }; /************************************************************************/ @@ -236,19 +252,20 @@ namespace nodetool { const static int ID = P2P_COMMANDS_POOL_BASE + 2; - struct request + struct request_t { t_playload_type payload_data; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(payload_data) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init request; - struct response + struct response_t { uint64_t local_time; t_playload_type payload_data; - std::list local_peerlist_new; + std::vector local_peerlist_new; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(local_time) @@ -257,10 +274,10 @@ namespace nodetool { // saving: save both, so old and new peers can understand it KV_SERIALIZE(local_peerlist_new) - std::list> local_peerlist; + std::vector> local_peerlist; for (const auto &p: this_ref.local_peerlist_new) { - if (p.adr.get_type_id() == epee::net_utils::ipv4_network_address::ID) + if (p.adr.get_type_id() == epee::net_utils::ipv4_network_address::get_type_id()) { const epee::net_utils::network_address &na = p.adr; const epee::net_utils::ipv4_network_address &ipv4 = na.as(); @@ -276,7 +293,7 @@ namespace nodetool // loading: load old list only if there is no new one if (!epee::serialization::selector::serialize(this_ref.local_peerlist_new, stg, hparent_section, "local_peerlist_new")) { - std::list> local_peerlist; + std::vector> local_peerlist; epee::serialization::selector::serialize_stl_container_pod_val_as_blob(local_peerlist, stg, hparent_section, "local_peerlist"); for (const auto &p: local_peerlist) ((response&)this_ref).local_peerlist_new.push_back(peerlist_entry({epee::net_utils::ipv4_network_address(p.adr.ip, p.adr.port), p.id, p.last_seen})); @@ -284,6 +301,7 @@ namespace nodetool } END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init response; }; /************************************************************************/ @@ -301,15 +319,16 @@ namespace nodetool #define PING_OK_RESPONSE_STATUS_TEXT "OK" - struct request + struct request_t { /*actually we don't need to send any real data*/ BEGIN_KV_SERIALIZE_MAP() END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init request; - struct response + struct response_t { std::string status; peerid_type peer_id; @@ -319,6 +338,7 @@ namespace nodetool KV_SERIALIZE(peer_id) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init response; }; @@ -345,15 +365,16 @@ namespace nodetool { const static int ID = P2P_COMMANDS_POOL_BASE + 4; - struct request + struct request_t { proof_of_trust tr; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(tr) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init request; - struct response + struct response_t { std::string version; std::string os_version; @@ -369,6 +390,7 @@ namespace nodetool KV_SERIALIZE(payload_info) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init response; }; @@ -379,19 +401,20 @@ namespace nodetool { const static int ID = P2P_COMMANDS_POOL_BASE + 5; - struct request + struct request_t { proof_of_trust tr; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(tr) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init request; - struct response + struct response_t { - std::list local_peerlist_white; - std::list local_peerlist_gray; - std::list connections_list; + std::vector local_peerlist_white; + std::vector local_peerlist_gray; + std::vector connections_list; peerid_type my_id; uint64_t local_time; BEGIN_KV_SERIALIZE_MAP() @@ -402,6 +425,7 @@ namespace nodetool KV_SERIALIZE(local_time) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init response; }; /************************************************************************/ @@ -411,13 +435,14 @@ namespace nodetool { const static int ID = P2P_COMMANDS_POOL_BASE + 6; - struct request + struct request_t { BEGIN_KV_SERIALIZE_MAP() END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init request; - struct response + struct response_t { peerid_type my_id; @@ -425,6 +450,7 @@ namespace nodetool KV_SERIALIZE(my_id) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init response; }; /************************************************************************/ @@ -434,13 +460,14 @@ namespace nodetool { const static int ID = P2P_COMMANDS_POOL_BASE + 7; - struct request + struct request_t { BEGIN_KV_SERIALIZE_MAP() END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init request; - struct response + struct response_t { uint32_t support_flags; @@ -448,6 +475,7 @@ namespace nodetool KV_SERIALIZE(support_flags) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init response; }; #endif @@ -462,6 +490,3 @@ namespace nodetool } } - - - diff --git a/src/p2p/stdafx.h b/src/p2p/stdafx.h index b6ff3781109..12a3717023b 100644 --- a/src/p2p/stdafx.h +++ b/src/p2p/stdafx.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/platform/mingw/alloca.h b/src/platform/mingw/alloca.h index 71934b19ad6..e4722af2a38 100644 --- a/src/platform/mingw/alloca.h +++ b/src/platform/mingw/alloca.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/platform/msc/alloca.h b/src/platform/msc/alloca.h index 89743e12bc6..fcf1731b08e 100644 --- a/src/platform/msc/alloca.h +++ b/src/platform/msc/alloca.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/platform/msc/inline_c.h b/src/platform/msc/inline_c.h index b274f3ec2f6..23c11bd06b1 100644 --- a/src/platform/msc/inline_c.h +++ b/src/platform/msc/inline_c.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/platform/msc/stdbool.h b/src/platform/msc/stdbool.h index 63e4200b2b0..4d24995bd33 100644 --- a/src/platform/msc/stdbool.h +++ b/src/platform/msc/stdbool.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/platform/msc/sys/param.h b/src/platform/msc/sys/param.h index ca9c9282d9e..e3cfb7e8634 100644 --- a/src/platform/msc/sys/param.h +++ b/src/platform/msc/sys/param.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/ringct/CMakeLists.txt b/src/ringct/CMakeLists.txt index c8dcdca26a9..0192aa93129 100644 --- a/src/ringct/CMakeLists.txt +++ b/src/ringct/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2016-2018, The Monero Project +# Copyright (c) 2016-2019, The Monero Project # # All rights reserved. # @@ -30,11 +30,13 @@ set(ringct_basic_sources rctOps.cpp rctTypes.cpp rctCryptoOps.c + multiexp.cc bulletproofs.cc) set(ringct_basic_private_headers rctOps.h rctTypes.h + multiexp.h bulletproofs.h) monero_private_headers(ringct_basic diff --git a/src/ringct/bulletproofs.cc b/src/ringct/bulletproofs.cc index fd15ffbc451..6270d4d1460 100644 --- a/src/ringct/bulletproofs.cc +++ b/src/ringct/bulletproofs.cc @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2018, The Monero Project +// Copyright (c) 2017-2019, The Monero Project // // All rights reserved. // @@ -29,15 +29,18 @@ // Adapted from Java code by Sarang Noether #include -#include #include +#include #include "misc_log_ex.h" +#include "span.h" #include "common/perf_timer.h" +#include "cryptonote_config.h" extern "C" { #include "crypto/crypto-ops.h" } #include "rctOps.h" +#include "multiexp.h" #include "bulletproofs.h" #undef MONERO_DEFAULT_LOG_CATEGORY @@ -45,29 +48,62 @@ extern "C" //#define DEBUG_BP +#if 0 #define PERF_TIMER_START_BP(x) PERF_TIMER_START_UNIT(x, 1000000) +#define PERF_TIMER_STOP_BP(x) PERF_TIMER_STOP(x) +#else +#define PERF_TIMER_START_BP(x) ((void)0) +#define PERF_TIMER_STOP_BP(x) ((void)0) +#endif + +#define STRAUS_SIZE_LIMIT 232 +#define PIPPENGER_SIZE_LIMIT 0 namespace rct { static rct::key vector_exponent(const rct::keyV &a, const rct::keyV &b); -static rct::keyV vector_powers(rct::key x, size_t n); +static rct::keyV vector_powers(const rct::key &x, size_t n); +static rct::keyV vector_dup(const rct::key &x, size_t n); static rct::key inner_product(const rct::keyV &a, const rct::keyV &b); static constexpr size_t maxN = 64; -static rct::key Hi[maxN], Gi[maxN]; -static ge_dsmp Gprecomp[64], Hprecomp[64]; +static constexpr size_t maxM = BULLETPROOF_MAX_OUTPUTS; +static rct::key Hi[maxN*maxM], Gi[maxN*maxM]; +static ge_p3 Hi_p3[maxN*maxM], Gi_p3[maxN*maxM]; +static std::shared_ptr straus_HiGi_cache; +static std::shared_ptr pippenger_HiGi_cache; static const rct::key TWO = { {0x02, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 } }; -static const rct::keyV oneN = vector_powers(rct::identity(), maxN); +static const rct::key MINUS_ONE = { { 0xec, 0xd3, 0xf5, 0x5c, 0x1a, 0x63, 0x12, 0x58, 0xd6, 0x9c, 0xf7, 0xa2, 0xde, 0xf9, 0xde, 0x14, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10 } }; +static const rct::key MINUS_INV_EIGHT = { { 0x74, 0xa4, 0x19, 0x7a, 0xf0, 0x7d, 0x0b, 0xf7, 0x05, 0xc2, 0xda, 0x25, 0x2b, 0x5c, 0x0b, 0x0d, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0a } }; +static const rct::keyV oneN = vector_dup(rct::identity(), maxN); static const rct::keyV twoN = vector_powers(TWO, maxN); static const rct::key ip12 = inner_product(oneN, twoN); static boost::mutex init_mutex; +static inline rct::key multiexp(const std::vector &data, size_t HiGi_size) +{ + if (HiGi_size > 0) + { + static_assert(232 <= STRAUS_SIZE_LIMIT, "Straus in precalc mode can only be calculated till STRAUS_SIZE_LIMIT"); + return HiGi_size <= 232 && data.size() == HiGi_size ? straus(data, straus_HiGi_cache, 0) : pippenger(data, pippenger_HiGi_cache, HiGi_size, get_pippenger_c(data.size())); + } + else + return data.size() <= 95 ? straus(data, NULL, 0) : pippenger(data, NULL, 0, get_pippenger_c(data.size())); +} + +static inline bool is_reduced(const rct::key &scalar) +{ + return sc_check(scalar.bytes) == 0; +} + static rct::key get_exponent(const rct::key &base, size_t idx) { static const std::string salt("bulletproof"); std::string hashed = std::string((const char*)base.bytes, sizeof(base)) + salt + tools::get_varint_data(idx); - return rct::hashToPoint(rct::hash2rct(crypto::cn_fast_hash(hashed.data(), hashed.size()))); + const rct::key e = rct::hashToPoint(rct::hash2rct(crypto::cn_fast_hash(hashed.data(), hashed.size()))); + CHECK_AND_ASSERT_THROW_MES(!(e == rct::identity()), "Exponent is point at infinity"); + return e; } static void init_exponents() @@ -77,13 +113,28 @@ static void init_exponents() static bool init_done = false; if (init_done) return; - for (size_t i = 0; i < maxN; ++i) + std::vector data; + data.reserve(maxN*maxM*2); + for (size_t i = 0; i < maxN*maxM; ++i) { Hi[i] = get_exponent(rct::H, i * 2); - rct::precomp(Hprecomp[i], Hi[i]); + CHECK_AND_ASSERT_THROW_MES(ge_frombytes_vartime(&Hi_p3[i], Hi[i].bytes) == 0, "ge_frombytes_vartime failed"); Gi[i] = get_exponent(rct::H, i * 2 + 1); - rct::precomp(Gprecomp[i], Gi[i]); + CHECK_AND_ASSERT_THROW_MES(ge_frombytes_vartime(&Gi_p3[i], Gi[i].bytes) == 0, "ge_frombytes_vartime failed"); + + data.push_back({rct::zero(), Gi_p3[i]}); + data.push_back({rct::zero(), Hi_p3[i]}); } + + straus_HiGi_cache = straus_init_cache(data, STRAUS_SIZE_LIMIT); + pippenger_HiGi_cache = pippenger_init_cache(data, 0, PIPPENGER_SIZE_LIMIT); + + MINFO("Hi/Gi cache size: " << (sizeof(Hi)+sizeof(Gi))/1024 << " kB"); + MINFO("Hi_p3/Gi_p3 cache size: " << (sizeof(Hi_p3)+sizeof(Gi_p3))/1024 << " kB"); + MINFO("Straus cache size: " << straus_get_cache_size(straus_HiGi_cache)/1024 << " kB"); + MINFO("Pippenger cache size: " << pippenger_get_cache_size(pippenger_HiGi_cache)/1024 << " kB"); + size_t cache_size = (sizeof(Hi)+sizeof(Hi_p3))*2 + straus_get_cache_size(straus_HiGi_cache) + pippenger_get_cache_size(pippenger_HiGi_cache); + MINFO("Total cache size: " << cache_size/1024 << "kB"); init_done = true; } @@ -91,61 +142,50 @@ static void init_exponents() static rct::key vector_exponent(const rct::keyV &a, const rct::keyV &b) { CHECK_AND_ASSERT_THROW_MES(a.size() == b.size(), "Incompatible sizes of a and b"); - CHECK_AND_ASSERT_THROW_MES(a.size() <= maxN, "Incompatible sizes of a and maxN"); - rct::key res = rct::identity(); + CHECK_AND_ASSERT_THROW_MES(a.size() <= maxN*maxM, "Incompatible sizes of a and maxN"); + + std::vector multiexp_data; + multiexp_data.reserve(a.size()*2); for (size_t i = 0; i < a.size(); ++i) { - rct::key term; - rct::addKeys3(term, a[i], Gprecomp[i], b[i], Hprecomp[i]); - rct::addKeys(res, res, term); + multiexp_data.emplace_back(a[i], Gi_p3[i]); + multiexp_data.emplace_back(b[i], Hi_p3[i]); } - return res; + return multiexp(multiexp_data, 2 * a.size()); } /* Compute a custom vector-scalar commitment */ -static rct::key vector_exponent_custom(const rct::keyV &A, const rct::keyV &B, const rct::keyV &a, const rct::keyV &b) +static rct::key cross_vector_exponent8(size_t size, const std::vector &A, size_t Ao, const std::vector &B, size_t Bo, const rct::keyV &a, size_t ao, const rct::keyV &b, size_t bo, const rct::keyV *scale, const ge_p3 *extra_point, const rct::key *extra_scalar) { - CHECK_AND_ASSERT_THROW_MES(A.size() == B.size(), "Incompatible sizes of A and B"); - CHECK_AND_ASSERT_THROW_MES(a.size() == b.size(), "Incompatible sizes of a and b"); - CHECK_AND_ASSERT_THROW_MES(a.size() == A.size(), "Incompatible sizes of a and A"); - CHECK_AND_ASSERT_THROW_MES(a.size() <= maxN, "Incompatible sizes of a and maxN"); - rct::key res = rct::identity(); - for (size_t i = 0; i < a.size(); ++i) + CHECK_AND_ASSERT_THROW_MES(size + Ao <= A.size(), "Incompatible size for A"); + CHECK_AND_ASSERT_THROW_MES(size + Bo <= B.size(), "Incompatible size for B"); + CHECK_AND_ASSERT_THROW_MES(size + ao <= a.size(), "Incompatible size for a"); + CHECK_AND_ASSERT_THROW_MES(size + bo <= b.size(), "Incompatible size for b"); + CHECK_AND_ASSERT_THROW_MES(size <= maxN*maxM, "size is too large"); + CHECK_AND_ASSERT_THROW_MES(!scale || size == scale->size() / 2, "Incompatible size for scale"); + CHECK_AND_ASSERT_THROW_MES(!!extra_point == !!extra_scalar, "only one of extra point/scalar present"); + + std::vector multiexp_data; + multiexp_data.resize(size*2 + (!!extra_point)); + for (size_t i = 0; i < size; ++i) { - rct::key term; -#if 0 - // we happen to know where A and B might fall, so don't bother checking the rest - ge_dsmp *Acache = NULL, *Bcache = NULL; - ge_dsmp Acache_custom[1], Bcache_custom[1]; - if (Gi[i] == A[i]) - Acache = Gprecomp + i; - else if (i<32 && Gi[i+32] == A[i]) - Acache = Gprecomp + i + 32; - else - { - rct::precomp(Acache_custom[0], A[i]); - Acache = Acache_custom; - } - if (i == 0 && B[i] == Hi[0]) - Bcache = Hprecomp; - else - { - rct::precomp(Bcache_custom[0], B[i]); - Bcache = Bcache_custom; - } - rct::addKeys3(term, a[i], *Acache, b[i], *Bcache); -#else - ge_dsmp Acache, Bcache; - rct::precomp(Bcache, B[i]); - rct::addKeys3(term, a[i], A[i], b[i], Bcache); -#endif - rct::addKeys(res, res, term); + sc_mul(multiexp_data[i*2].scalar.bytes, a[ao+i].bytes, INV_EIGHT.bytes);; + multiexp_data[i*2].point = A[Ao+i]; + sc_mul(multiexp_data[i*2+1].scalar.bytes, b[bo+i].bytes, INV_EIGHT.bytes); + if (scale) + sc_mul(multiexp_data[i*2+1].scalar.bytes, multiexp_data[i*2+1].scalar.bytes, (*scale)[Bo+i].bytes); + multiexp_data[i*2+1].point = B[Bo+i]; } - return res; + if (extra_point) + { + sc_mul(multiexp_data.back().scalar.bytes, extra_scalar->bytes, INV_EIGHT.bytes); + multiexp_data.back().point = *extra_point; + } + return multiexp(multiexp_data, 0); } /* Given a scalar, construct a vector of powers */ -static rct::keyV vector_powers(rct::key x, size_t n) +static rct::keyV vector_powers(const rct::key &x, size_t n) { rct::keyV res(n); if (n == 0) @@ -161,8 +201,42 @@ static rct::keyV vector_powers(rct::key x, size_t n) return res; } +/* Given a scalar, return the sum of its powers from 0 to n-1 */ +static rct::key vector_power_sum(rct::key x, size_t n) +{ + if (n == 0) + return rct::zero(); + rct::key res = rct::identity(); + if (n == 1) + return res; + + const bool is_power_of_2 = (n & (n - 1)) == 0; + if (is_power_of_2) + { + sc_add(res.bytes, res.bytes, x.bytes); + while (n > 2) + { + sc_mul(x.bytes, x.bytes, x.bytes); + sc_muladd(res.bytes, x.bytes, res.bytes, res.bytes); + n /= 2; + } + } + else + { + rct::key prev = x; + for (size_t i = 1; i < n; ++i) + { + if (i > 1) + sc_mul(prev.bytes, prev.bytes, x.bytes); + sc_add(res.bytes, res.bytes, prev.bytes); + } + } + + return res; +} + /* Given two scalar arrays, construct the inner product */ -static rct::key inner_product(const rct::keyV &a, const rct::keyV &b) +static rct::key inner_product(const epee::span &a, const epee::span &b) { CHECK_AND_ASSERT_THROW_MES(a.size() == b.size(), "Incompatible sizes of a and b"); rct::key res = rct::zero(); @@ -173,6 +247,11 @@ static rct::key inner_product(const rct::keyV &a, const rct::keyV &b) return res; } +static rct::key inner_product(const rct::keyV &a, const rct::keyV &b) +{ + return inner_product(epee::span(a.data(), a.size()), epee::span(b.data(), b.size())); +} + /* Given two scalar arrays, construct the Hadamard product */ static rct::keyV hadamard(const rct::keyV &a, const rct::keyV &b) { @@ -185,16 +264,22 @@ static rct::keyV hadamard(const rct::keyV &a, const rct::keyV &b) return res; } -/* Given two curvepoint arrays, construct the Hadamard product */ -static rct::keyV hadamard2(const rct::keyV &a, const rct::keyV &b) +/* folds a curvepoint array using a two way scaled Hadamard product */ +static void hadamard_fold(std::vector &v, const rct::keyV *scale, const rct::key &a, const rct::key &b) { - CHECK_AND_ASSERT_THROW_MES(a.size() == b.size(), "Incompatible sizes of a and b"); - rct::keyV res(a.size()); - for (size_t i = 0; i < a.size(); ++i) + CHECK_AND_ASSERT_THROW_MES((v.size() & 1) == 0, "Vector size should be even"); + const size_t sz = v.size() / 2; + for (size_t n = 0; n < sz; ++n) { - rct::addKeys(res[i], a[i], b[i]); + ge_dsmp c[2]; + ge_dsm_precomp(c[0], &v[n]); + ge_dsm_precomp(c[1], &v[sz + n]); + rct::key sa, sb; + if (scale) sc_mul(sa.bytes, a.bytes, (*scale)[n].bytes); else sa = a; + if (scale) sc_mul(sb.bytes, b.bytes, (*scale)[sz + n].bytes); else sb = b; + ge_double_scalarmult_precomp_vartime2_p3(&v[n], sa.bytes, c[0], sb.bytes, c[1]); } - return res; + v.resize(sz); } /* Add two vectors */ @@ -209,71 +294,103 @@ static rct::keyV vector_add(const rct::keyV &a, const rct::keyV &b) return res; } -/* Subtract two vectors */ -static rct::keyV vector_subtract(const rct::keyV &a, const rct::keyV &b) +/* Add a scalar to all elements of a vector */ +static rct::keyV vector_add(const rct::keyV &a, const rct::key &b) { - CHECK_AND_ASSERT_THROW_MES(a.size() == b.size(), "Incompatible sizes of a and b"); rct::keyV res(a.size()); for (size_t i = 0; i < a.size(); ++i) { - sc_sub(res[i].bytes, a[i].bytes, b[i].bytes); + sc_add(res[i].bytes, a[i].bytes, b.bytes); } return res; } -/* Multiply a scalar and a vector */ -static rct::keyV vector_scalar(const rct::keyV &a, const rct::key &x) +/* Subtract a scalar from all elements of a vector */ +static rct::keyV vector_subtract(const rct::keyV &a, const rct::key &b) { rct::keyV res(a.size()); for (size_t i = 0; i < a.size(); ++i) { - sc_mul(res[i].bytes, a[i].bytes, x.bytes); + sc_sub(res[i].bytes, a[i].bytes, b.bytes); } return res; } -/* Exponentiate a curve vector by a scalar */ -static rct::keyV vector_scalar2(const rct::keyV &a, const rct::key &x) +/* Multiply a scalar and a vector */ +static rct::keyV vector_scalar(const epee::span &a, const rct::key &x) { rct::keyV res(a.size()); for (size_t i = 0; i < a.size(); ++i) { - rct::scalarmultKey(res[i], a[i], x); + sc_mul(res[i].bytes, a[i].bytes, x.bytes); } return res; } -static rct::key switch_endianness(rct::key k) +static rct::keyV vector_scalar(const rct::keyV &a, const rct::key &x) { - std::reverse(k.bytes, k.bytes + sizeof(k)); - return k; + return vector_scalar(epee::span(a.data(), a.size()), x); } -/* Compute the inverse of a scalar, the stupid way */ -static rct::key invert(const rct::key &x) +/* Create a vector from copies of a single value */ +static rct::keyV vector_dup(const rct::key &x, size_t N) { - rct::key inv; - - BN_CTX *ctx = BN_CTX_new(); - BIGNUM *X = BN_new(); - BIGNUM *L = BN_new(); - BIGNUM *I = BN_new(); - - BN_bin2bn(switch_endianness(x).bytes, sizeof(rct::key), X); - BN_bin2bn(switch_endianness(rct::curveOrder()).bytes, sizeof(rct::key), L); + return rct::keyV(N, x); +} - CHECK_AND_ASSERT_THROW_MES(BN_mod_inverse(I, X, L, ctx), "Failed to invert"); +static rct::key sm(rct::key y, int n, const rct::key &x) +{ + while (n--) + sc_mul(y.bytes, y.bytes, y.bytes); + sc_mul(y.bytes, y.bytes, x.bytes); + return y; +} - const int len = BN_num_bytes(I); - CHECK_AND_ASSERT_THROW_MES((size_t)len <= sizeof(rct::key), "Invalid number length"); - inv = rct::zero(); - BN_bn2bin(I, inv.bytes); - std::reverse(inv.bytes, inv.bytes + len); +/* Compute the inverse of a scalar, the clever way */ +static rct::key invert(const rct::key &x) +{ + rct::key _1, _10, _100, _11, _101, _111, _1001, _1011, _1111; + + _1 = x; + sc_mul(_10.bytes, _1.bytes, _1.bytes); + sc_mul(_100.bytes, _10.bytes, _10.bytes); + sc_mul(_11.bytes, _10.bytes, _1.bytes); + sc_mul(_101.bytes, _10.bytes, _11.bytes); + sc_mul(_111.bytes, _10.bytes, _101.bytes); + sc_mul(_1001.bytes, _10.bytes, _111.bytes); + sc_mul(_1011.bytes, _10.bytes, _1001.bytes); + sc_mul(_1111.bytes, _100.bytes, _1011.bytes); - BN_free(I); - BN_free(L); - BN_free(X); - BN_CTX_free(ctx); + rct::key inv; + sc_mul(inv.bytes, _1111.bytes, _1.bytes); + + inv = sm(inv, 123 + 3, _101); + inv = sm(inv, 2 + 2, _11); + inv = sm(inv, 1 + 4, _1111); + inv = sm(inv, 1 + 4, _1111); + inv = sm(inv, 4, _1001); + inv = sm(inv, 2, _11); + inv = sm(inv, 1 + 4, _1111); + inv = sm(inv, 1 + 3, _101); + inv = sm(inv, 3 + 3, _101); + inv = sm(inv, 3, _111); + inv = sm(inv, 1 + 4, _1111); + inv = sm(inv, 2 + 3, _111); + inv = sm(inv, 2 + 2, _11); + inv = sm(inv, 1 + 4, _1011); + inv = sm(inv, 2 + 4, _1011); + inv = sm(inv, 6 + 4, _1001); + inv = sm(inv, 2 + 2, _11); + inv = sm(inv, 3 + 2, _11); + inv = sm(inv, 3 + 2, _11); + inv = sm(inv, 1 + 4, _1001); + inv = sm(inv, 1 + 3, _111); + inv = sm(inv, 2 + 4, _1111); + inv = sm(inv, 1 + 4, _1011); + inv = sm(inv, 3, _101); + inv = sm(inv, 2 + 4, _1111); + inv = sm(inv, 3, _101); + inv = sm(inv, 1 + 2, _11); #ifdef DEBUG_BP rct::key tmp; @@ -283,197 +400,280 @@ static rct::key invert(const rct::key &x) return inv; } +static rct::keyV invert(rct::keyV x) +{ + rct::keyV scratch; + scratch.reserve(x.size()); + + rct::key acc = rct::identity(); + for (size_t n = 0; n < x.size(); ++n) + { + scratch.push_back(acc); + if (n == 0) + acc = x[0]; + else + sc_mul(acc.bytes, acc.bytes, x[n].bytes); + } + + acc = invert(acc); + + rct::key tmp; + for (int i = x.size(); i-- > 0; ) + { + sc_mul(tmp.bytes, acc.bytes, x[i].bytes); + sc_mul(x[i].bytes, acc.bytes, scratch[i].bytes); + acc = tmp; + } + + return x; +} + /* Compute the slice of a vector */ -static rct::keyV slice(const rct::keyV &a, size_t start, size_t stop) +static epee::span slice(const rct::keyV &a, size_t start, size_t stop) { CHECK_AND_ASSERT_THROW_MES(start < a.size(), "Invalid start index"); CHECK_AND_ASSERT_THROW_MES(stop <= a.size(), "Invalid stop index"); CHECK_AND_ASSERT_THROW_MES(start < stop, "Invalid start/stop indices"); - rct::keyV res(stop - start); - for (size_t i = start; i < stop; ++i) - { - res[i - start] = a[i]; - } - return res; + return epee::span(&a[start], stop - start); } static rct::key hash_cache_mash(rct::key &hash_cache, const rct::key &mash0, const rct::key &mash1) { - rct::keyV data; - data.reserve(3); - data.push_back(hash_cache); - data.push_back(mash0); - data.push_back(mash1); - return hash_cache = rct::hash_to_scalar(data); + rct::key data[3]; + data[0] = hash_cache; + data[1] = mash0; + data[2] = mash1; + rct::hash_to_scalar(hash_cache, data, sizeof(data)); + return hash_cache; } static rct::key hash_cache_mash(rct::key &hash_cache, const rct::key &mash0, const rct::key &mash1, const rct::key &mash2) { - rct::keyV data; - data.reserve(4); - data.push_back(hash_cache); - data.push_back(mash0); - data.push_back(mash1); - data.push_back(mash2); - return hash_cache = rct::hash_to_scalar(data); + rct::key data[4]; + data[0] = hash_cache; + data[1] = mash0; + data[2] = mash1; + data[3] = mash2; + rct::hash_to_scalar(hash_cache, data, sizeof(data)); + return hash_cache; } static rct::key hash_cache_mash(rct::key &hash_cache, const rct::key &mash0, const rct::key &mash1, const rct::key &mash2, const rct::key &mash3) { - rct::keyV data; - data.reserve(5); - data.push_back(hash_cache); - data.push_back(mash0); - data.push_back(mash1); - data.push_back(mash2); - data.push_back(mash3); - return hash_cache = rct::hash_to_scalar(data); + rct::key data[5]; + data[0] = hash_cache; + data[1] = mash0; + data[2] = mash1; + data[3] = mash2; + data[4] = mash3; + rct::hash_to_scalar(hash_cache, data, sizeof(data)); + return hash_cache; } /* Given a value v (0..2^N-1) and a mask gamma, construct a range proof */ Bulletproof bulletproof_PROVE(const rct::key &sv, const rct::key &gamma) { + return bulletproof_PROVE(rct::keyV(1, sv), rct::keyV(1, gamma)); +} + +Bulletproof bulletproof_PROVE(uint64_t v, const rct::key &gamma) +{ + return bulletproof_PROVE(std::vector(1, v), rct::keyV(1, gamma)); +} + +/* Given a set of values v (0..2^N-1) and masks gamma, construct a range proof */ +Bulletproof bulletproof_PROVE(const rct::keyV &sv, const rct::keyV &gamma) +{ + CHECK_AND_ASSERT_THROW_MES(sv.size() == gamma.size(), "Incompatible sizes of sv and gamma"); + CHECK_AND_ASSERT_THROW_MES(!sv.empty(), "sv is empty"); + for (const rct::key &sve: sv) + CHECK_AND_ASSERT_THROW_MES(is_reduced(sve), "Invalid sv input"); + for (const rct::key &g: gamma) + CHECK_AND_ASSERT_THROW_MES(is_reduced(g), "Invalid gamma input"); + init_exponents(); PERF_TIMER_UNIT(PROVE, 1000000); constexpr size_t logN = 6; // log2(64) constexpr size_t N = 1< 0; ) + for (size_t j = 0; j < M; ++j) { - if (sv[i/8] & (((uint64_t)1)<<(i%8))) - { - aL[i] = rct::identity(); - } - else + for (size_t i = N; i-- > 0; ) { - aL[i] = rct::zero(); + if (j < sv.size() && (sv[j][i/8] & (((uint64_t)1)<<(i%8)))) + { + aL[j*N+i] = rct::identity(); + aL8[j*N+i] = INV_EIGHT; + aR[j*N+i] = aR8[j*N+i] = rct::zero(); + } + else + { + aL[j*N+i] = aL8[j*N+i] = rct::zero(); + aR[j*N+i] = MINUS_ONE; + aR8[j*N+i] = MINUS_INV_EIGHT; + } } - sc_sub(aR[i].bytes, aL[i].bytes, rct::identity().bytes); } - PERF_TIMER_STOP(PROVE_aLaR); - - rct::key hash_cache = rct::hash_to_scalar(V); + PERF_TIMER_STOP_BP(PROVE_aLaR); // DEBUG: Test to ensure this recovers the value #ifdef DEBUG_BP - uint64_t test_aL = 0, test_aR = 0; - for (size_t i = 0; i < N; ++i) - { - if (aL[i] == rct::identity()) - test_aL += ((uint64_t)1)<= (j-1)*N && i < j*N) + { + CHECK_AND_ASSERT_THROW_MES(1+j < zpow.size(), "invalid zpow index"); + CHECK_AND_ASSERT_THROW_MES(i-(j-1)*N < twoN.size(), "invalid twoN index"); + sc_muladd(zero_twos[i].bytes, zpow[1+j].bytes, twoN[i-(j-1)*N].bytes, zero_twos[i].bytes); + } + } + } - rct::key k = rct::zero(); - sc_mulsub(k.bytes, zsq.bytes, ip1y.bytes, k.bytes); + rct::keyV r0 = vector_add(aR, z); + const auto yMN = vector_powers(y, MN); + r0 = hadamard(r0, yMN); + r0 = vector_add(r0, zero_twos); + rct::keyV r1 = hadamard(yMN, sR); - rct::key zcu; - sc_mul(zcu.bytes, zsq.bytes, z.bytes); - sc_mulsub(k.bytes, zcu.bytes, ip12.bytes, k.bytes); - sc_add(t0.bytes, t0.bytes, k.bytes); + // Polynomial construction before PAPER LINE 46 + rct::key t1_1 = inner_product(l0, r1); + rct::key t1_2 = inner_product(l1, r0); + rct::key t1; + sc_add(t1.bytes, t1_1.bytes, t1_2.bytes); + rct::key t2 = inner_product(l1, r1); - // DEBUG: Test the value of t0 has the correct form -#ifdef DEBUG_BP - rct::key test_t0 = rct::zero(); - rct::key iph = inner_product(aL, hadamard(aR, yN)); - sc_add(test_t0.bytes, test_t0.bytes, iph.bytes); - rct::key ips = inner_product(vector_subtract(aL, aR), yN); - sc_muladd(test_t0.bytes, z.bytes, ips.bytes, test_t0.bytes); - rct::key ipt = inner_product(twoN, aL); - sc_muladd(test_t0.bytes, zsq.bytes, ipt.bytes, test_t0.bytes); - sc_add(test_t0.bytes, test_t0.bytes, k.bytes); - CHECK_AND_ASSERT_THROW_MES(t0 == test_t0, "t0 check failed"); -#endif - PERF_TIMER_STOP(PROVE_step1); + PERF_TIMER_STOP_BP(PROVE_step1); PERF_TIMER_START_BP(PROVE_step2); - const auto HyNsR = hadamard(yN, sR); - const auto vpIz = vector_scalar(oneN, z); - const auto vp2zsq = vector_scalar(twoN, zsq); - const auto aL_vpIz = vector_subtract(aL, vpIz); - const auto aR_vpIz = vector_add(aR, vpIz); - - rct::key ip1 = inner_product(aL_vpIz, HyNsR); - sc_add(t1.bytes, t1.bytes, ip1.bytes); - - rct::key ip2 = inner_product(sL, vector_add(hadamard(yN, aR_vpIz), vp2zsq)); - sc_add(t1.bytes, t1.bytes, ip2.bytes); - - rct::key ip3 = inner_product(sL, HyNsR); - sc_add(t2.bytes, t2.bytes, ip3.bytes); - // PAPER LINES 47-48 rct::key tau1 = rct::skGen(), tau2 = rct::skGen(); - rct::key T1 = rct::addKeys(rct::scalarmultKey(rct::H, t1), rct::scalarmultBase(tau1)); - rct::key T2 = rct::addKeys(rct::scalarmultKey(rct::H, t2), rct::scalarmultBase(tau2)); + rct::key T1, T2; + ge_p3 p3; + sc_mul(tmp.bytes, t1.bytes, INV_EIGHT.bytes); + sc_mul(tmp2.bytes, tau1.bytes, INV_EIGHT.bytes); + ge_double_scalarmult_base_vartime_p3(&p3, tmp.bytes, &ge_p3_H, tmp2.bytes); + ge_p3_tobytes(T1.bytes, &p3); + sc_mul(tmp.bytes, t2.bytes, INV_EIGHT.bytes); + sc_mul(tmp2.bytes, tau2.bytes, INV_EIGHT.bytes); + ge_double_scalarmult_base_vartime_p3(&p3, tmp.bytes, &ge_p3_H, tmp2.bytes); + ge_p3_tobytes(T2.bytes, &p3); // PAPER LINES 49-51 rct::key x = hash_cache_mash(hash_cache, z, T1, T2); + if (x == rct::zero()) + { + PERF_TIMER_STOP_BP(PROVE_step2); + MINFO("x is 0, trying again"); + goto try_again; + } // PAPER LINES 52-53 - rct::key taux = rct::zero(); + rct::key taux; sc_mul(taux.bytes, tau1.bytes, x.bytes); rct::key xsq; sc_mul(xsq.bytes, x.bytes, x.bytes); sc_muladd(taux.bytes, tau2.bytes, xsq.bytes, taux.bytes); - sc_muladd(taux.bytes, gamma.bytes, zsq.bytes, taux.bytes); + for (size_t j = 1; j <= sv.size(); ++j) + { + CHECK_AND_ASSERT_THROW_MES(j+1 < zpow.size(), "invalid zpow index"); + sc_muladd(taux.bytes, zpow[j+1].bytes, gamma[j-1].bytes, taux.bytes); + } rct::key mu; sc_muladd(mu.bytes, x.bytes, rho.bytes, alpha.bytes); // PAPER LINES 54-57 - rct::keyV l = vector_add(aL_vpIz, vector_scalar(sL, x)); - rct::keyV r = vector_add(hadamard(yN, vector_add(aR_vpIz, vector_scalar(sR, x))), vp2zsq); - PERF_TIMER_STOP(PROVE_step2); + rct::keyV l = l0; + l = vector_add(l, vector_scalar(l1, x)); + rct::keyV r = r0; + r = vector_add(r, vector_scalar(r1, x)); + PERF_TIMER_STOP_BP(PROVE_step2); PERF_TIMER_START_BP(PROVE_step3); rct::key t = inner_product(l, r); @@ -481,6 +681,7 @@ Bulletproof bulletproof_PROVE(const rct::key &sv, const rct::key &gamma) // DEBUG: Test if the l and r vectors match the polynomial forms #ifdef DEBUG_BP rct::key test_t; + const rct::key t0 = inner_product(l0, r0); sc_muladd(test_t.bytes, t1.bytes, x.bytes, t0.bytes); sc_muladd(test_t.bytes, t2.bytes, xsq.bytes, test_t.bytes); CHECK_AND_ASSERT_THROW_MES(test_t == t, "test_t check failed"); @@ -488,265 +689,412 @@ Bulletproof bulletproof_PROVE(const rct::key &sv, const rct::key &gamma) // PAPER LINES 32-33 rct::key x_ip = hash_cache_mash(hash_cache, x, taux, mu, t); + if (x_ip == rct::zero()) + { + PERF_TIMER_STOP_BP(PROVE_step3); + MINFO("x_ip is 0, trying again"); + goto try_again; + } // These are used in the inner product rounds - size_t nprime = N; - rct::keyV Gprime(N); - rct::keyV Hprime(N); - rct::keyV aprime(N); - rct::keyV bprime(N); + size_t nprime = MN; + std::vector Gprime(MN); + std::vector Hprime(MN); + rct::keyV aprime(MN); + rct::keyV bprime(MN); const rct::key yinv = invert(y); - rct::key yinvpow = rct::identity(); - for (size_t i = 0; i < N; ++i) + rct::keyV yinvpow(MN); + yinvpow[0] = rct::identity(); + yinvpow[1] = yinv; + for (size_t i = 0; i < MN; ++i) { - Gprime[i] = Gi[i]; - Hprime[i] = scalarmultKey(Hi[i], yinvpow); - sc_mul(yinvpow.bytes, yinvpow.bytes, yinv.bytes); + Gprime[i] = Gi_p3[i]; + Hprime[i] = Hi_p3[i]; + if (i > 1) + sc_mul(yinvpow[i].bytes, yinvpow[i-1].bytes, yinv.bytes); aprime[i] = l[i]; bprime[i] = r[i]; } - rct::keyV L(logN); - rct::keyV R(logN); + rct::keyV L(logMN); + rct::keyV R(logMN); int round = 0; - rct::keyV w(logN); // this is the challenge x in the inner product protocol - PERF_TIMER_STOP(PROVE_step3); + rct::keyV w(logMN); // this is the challenge x in the inner product protocol + PERF_TIMER_STOP_BP(PROVE_step3); PERF_TIMER_START_BP(PROVE_step4); // PAPER LINE 13 + const rct::keyV *scale = &yinvpow; while (nprime > 1) { // PAPER LINE 15 nprime /= 2; // PAPER LINES 16-17 + PERF_TIMER_START_BP(PROVE_inner_product); rct::key cL = inner_product(slice(aprime, 0, nprime), slice(bprime, nprime, bprime.size())); rct::key cR = inner_product(slice(aprime, nprime, aprime.size()), slice(bprime, 0, nprime)); + PERF_TIMER_STOP_BP(PROVE_inner_product); // PAPER LINES 18-19 - L[round] = vector_exponent_custom(slice(Gprime, nprime, Gprime.size()), slice(Hprime, 0, nprime), slice(aprime, 0, nprime), slice(bprime, nprime, bprime.size())); + PERF_TIMER_START_BP(PROVE_LR); sc_mul(tmp.bytes, cL.bytes, x_ip.bytes); - rct::addKeys(L[round], L[round], rct::scalarmultKey(rct::H, tmp)); - R[round] = vector_exponent_custom(slice(Gprime, 0, nprime), slice(Hprime, nprime, Hprime.size()), slice(aprime, nprime, aprime.size()), slice(bprime, 0, nprime)); + L[round] = cross_vector_exponent8(nprime, Gprime, nprime, Hprime, 0, aprime, 0, bprime, nprime, scale, &ge_p3_H, &tmp); sc_mul(tmp.bytes, cR.bytes, x_ip.bytes); - rct::addKeys(R[round], R[round], rct::scalarmultKey(rct::H, tmp)); + R[round] = cross_vector_exponent8(nprime, Gprime, 0, Hprime, nprime, aprime, nprime, bprime, 0, scale, &ge_p3_H, &tmp); + PERF_TIMER_STOP_BP(PROVE_LR); // PAPER LINES 21-22 w[round] = hash_cache_mash(hash_cache, L[round], R[round]); + if (w[round] == rct::zero()) + { + PERF_TIMER_STOP_BP(PROVE_step4); + MINFO("w[round] is 0, trying again"); + goto try_again; + } // PAPER LINES 24-25 const rct::key winv = invert(w[round]); - Gprime = hadamard2(vector_scalar2(slice(Gprime, 0, nprime), winv), vector_scalar2(slice(Gprime, nprime, Gprime.size()), w[round])); - Hprime = hadamard2(vector_scalar2(slice(Hprime, 0, nprime), w[round]), vector_scalar2(slice(Hprime, nprime, Hprime.size()), winv)); + if (nprime > 1) + { + PERF_TIMER_START_BP(PROVE_hadamard2); + hadamard_fold(Gprime, NULL, winv, w[round]); + hadamard_fold(Hprime, scale, w[round], winv); + PERF_TIMER_STOP_BP(PROVE_hadamard2); + } // PAPER LINES 28-29 + PERF_TIMER_START_BP(PROVE_prime); aprime = vector_add(vector_scalar(slice(aprime, 0, nprime), w[round]), vector_scalar(slice(aprime, nprime, aprime.size()), winv)); bprime = vector_add(vector_scalar(slice(bprime, 0, nprime), winv), vector_scalar(slice(bprime, nprime, bprime.size()), w[round])); + PERF_TIMER_STOP_BP(PROVE_prime); + scale = NULL; ++round; } - PERF_TIMER_STOP(PROVE_step4); + PERF_TIMER_STOP_BP(PROVE_step4); // PAPER LINE 58 (with inclusions from PAPER LINE 8 and PAPER LINE 20) - return Bulletproof(V, A, S, T1, T2, taux, mu, L, R, aprime[0], bprime[0], t); + return Bulletproof(std::move(V), A, S, T1, T2, taux, mu, std::move(L), std::move(R), aprime[0], bprime[0], t); } -Bulletproof bulletproof_PROVE(uint64_t v, const rct::key &gamma) +Bulletproof bulletproof_PROVE(const std::vector &v, const rct::keyV &gamma) { + CHECK_AND_ASSERT_THROW_MES(v.size() == gamma.size(), "Incompatible sizes of v and gamma"); + // vG + gammaH PERF_TIMER_START_BP(PROVE_v); - rct::key sv = rct::zero(); - sv.bytes[0] = v & 255; - sv.bytes[1] = (v >> 8) & 255; - sv.bytes[2] = (v >> 16) & 255; - sv.bytes[3] = (v >> 24) & 255; - sv.bytes[4] = (v >> 32) & 255; - sv.bytes[5] = (v >> 40) & 255; - sv.bytes[6] = (v >> 48) & 255; - sv.bytes[7] = (v >> 56) & 255; - PERF_TIMER_STOP(PROVE_v); + rct::keyV sv(v.size()); + for (size_t i = 0; i < v.size(); ++i) + { + sv[i] = rct::zero(); + sv[i].bytes[0] = v[i] & 255; + sv[i].bytes[1] = (v[i] >> 8) & 255; + sv[i].bytes[2] = (v[i] >> 16) & 255; + sv[i].bytes[3] = (v[i] >> 24) & 255; + sv[i].bytes[4] = (v[i] >> 32) & 255; + sv[i].bytes[5] = (v[i] >> 40) & 255; + sv[i].bytes[6] = (v[i] >> 48) & 255; + sv[i].bytes[7] = (v[i] >> 56) & 255; + } + PERF_TIMER_STOP_BP(PROVE_v); return bulletproof_PROVE(sv, gamma); } +struct proof_data_t +{ + rct::key x, y, z, x_ip; + std::vector w; + size_t logM, inv_offset; +}; + /* Given a range proof, determine if it is valid */ -bool bulletproof_VERIFY(const Bulletproof &proof) +bool bulletproof_VERIFY(const std::vector &proofs) { init_exponents(); - CHECK_AND_ASSERT_MES(proof.V.size() == 1, false, "V does not have exactly one element"); - CHECK_AND_ASSERT_MES(proof.L.size() == proof.R.size(), false, "Mismatched L and R sizes"); - CHECK_AND_ASSERT_MES(proof.L.size() > 0, false, "Empty proof"); - CHECK_AND_ASSERT_MES(proof.L.size() == 6, false, "Proof is not for 64 bits"); - - const size_t logN = proof.L.size(); - const size_t N = 1 << logN; - - // Reconstruct the challenges PERF_TIMER_START_BP(VERIFY); - PERF_TIMER_START_BP(VERIFY_start); - rct::key hash_cache = rct::hash_to_scalar(proof.V[0]); - rct::key y = hash_cache_mash(hash_cache, proof.A, proof.S); - rct::key z = hash_cache = rct::hash_to_scalar(y); - rct::key x = hash_cache_mash(hash_cache, z, proof.T1, proof.T2); - PERF_TIMER_STOP(VERIFY_start); - - PERF_TIMER_START_BP(VERIFY_line_60); - // Reconstruct the challenges - rct::key x_ip = hash_cache_mash(hash_cache, x, proof.taux, proof.mu, proof.t); - PERF_TIMER_STOP(VERIFY_line_60); - - PERF_TIMER_START_BP(VERIFY_line_61); - // PAPER LINE 61 - rct::key L61Left = rct::addKeys(rct::scalarmultBase(proof.taux), rct::scalarmultKey(rct::H, proof.t)); - - rct::key k = rct::zero(); - const auto yN = vector_powers(y, N); - rct::key ip1y = inner_product(oneN, yN); - rct::key zsq; - sc_mul(zsq.bytes, z.bytes, z.bytes); - rct::key tmp, tmp2; - sc_mulsub(k.bytes, zsq.bytes, ip1y.bytes, k.bytes); - rct::key zcu; - sc_mul(zcu.bytes, zsq.bytes, z.bytes); - sc_mulsub(k.bytes, zcu.bytes, ip12.bytes, k.bytes); - PERF_TIMER_STOP(VERIFY_line_61); - - PERF_TIMER_START_BP(VERIFY_line_61rl); - sc_muladd(tmp.bytes, z.bytes, ip1y.bytes, k.bytes); - rct::key L61Right = rct::scalarmultKey(rct::H, tmp); - - CHECK_AND_ASSERT_MES(proof.V.size() == 1, false, "proof.V does not have exactly one element"); - tmp = rct::scalarmultKey(proof.V[0], zsq); - rct::addKeys(L61Right, L61Right, tmp); - tmp = rct::scalarmultKey(proof.T1, x); - rct::addKeys(L61Right, L61Right, tmp); - - rct::key xsq; - sc_mul(xsq.bytes, x.bytes, x.bytes); - tmp = rct::scalarmultKey(proof.T2, xsq); - rct::addKeys(L61Right, L61Right, tmp); - PERF_TIMER_STOP(VERIFY_line_61rl); + const size_t logN = 6; + const size_t N = 1 << logN; - if (!(L61Right == L61Left)) + // sanity and figure out which proof is longest + size_t max_length = 0; + size_t nV = 0; + std::vector proof_data; + proof_data.reserve(proofs.size()); + size_t inv_offset = 0; + std::vector to_invert; + to_invert.reserve(11 * sizeof(proofs)); + size_t max_logM = 0; + for (const Bulletproof *p: proofs) { - MERROR("Verification failure at step 1"); - return false; - } - - PERF_TIMER_START_BP(VERIFY_line_62); - // PAPER LINE 62 - rct::key P = rct::addKeys(proof.A, rct::scalarmultKey(proof.S, x)); - PERF_TIMER_STOP(VERIFY_line_62); - - // Compute the number of rounds for the inner product - const size_t rounds = proof.L.size(); - CHECK_AND_ASSERT_MES(rounds > 0, false, "Zero rounds"); + const Bulletproof &proof = *p; + + // check scalar range + CHECK_AND_ASSERT_MES(is_reduced(proof.taux), false, "Input scalar not in range"); + CHECK_AND_ASSERT_MES(is_reduced(proof.mu), false, "Input scalar not in range"); + CHECK_AND_ASSERT_MES(is_reduced(proof.a), false, "Input scalar not in range"); + CHECK_AND_ASSERT_MES(is_reduced(proof.b), false, "Input scalar not in range"); + CHECK_AND_ASSERT_MES(is_reduced(proof.t), false, "Input scalar not in range"); + + CHECK_AND_ASSERT_MES(proof.V.size() >= 1, false, "V does not have at least one element"); + CHECK_AND_ASSERT_MES(proof.L.size() == proof.R.size(), false, "Mismatched L and R sizes"); + CHECK_AND_ASSERT_MES(proof.L.size() > 0, false, "Empty proof"); + + max_length = std::max(max_length, proof.L.size()); + nV += proof.V.size(); + + // Reconstruct the challenges + PERF_TIMER_START_BP(VERIFY_start); + proof_data.resize(proof_data.size() + 1); + proof_data_t &pd = proof_data.back(); + rct::key hash_cache = rct::hash_to_scalar(proof.V); + pd.y = hash_cache_mash(hash_cache, proof.A, proof.S); + CHECK_AND_ASSERT_MES(!(pd.y == rct::zero()), false, "y == 0"); + pd.z = hash_cache = rct::hash_to_scalar(pd.y); + CHECK_AND_ASSERT_MES(!(pd.z == rct::zero()), false, "z == 0"); + pd.x = hash_cache_mash(hash_cache, pd.z, proof.T1, proof.T2); + CHECK_AND_ASSERT_MES(!(pd.x == rct::zero()), false, "x == 0"); + pd.x_ip = hash_cache_mash(hash_cache, pd.x, proof.taux, proof.mu, proof.t); + CHECK_AND_ASSERT_MES(!(pd.x_ip == rct::zero()), false, "x_ip == 0"); + PERF_TIMER_STOP_BP(VERIFY_start); + + size_t M; + for (pd.logM = 0; (M = 1< 0, false, "Zero rounds"); + + PERF_TIMER_START_BP(VERIFY_line_21_22); + // PAPER LINES 21-22 + // The inner product challenges are computed per round + pd.w.resize(rounds); + for (size_t i = 0; i < rounds; ++i) + { + pd.w[i] = hash_cache_mash(hash_cache, proof.L[i], proof.R[i]); + CHECK_AND_ASSERT_MES(!(pd.w[i] == rct::zero()), false, "w[i] == 0"); + } + PERF_TIMER_STOP_BP(VERIFY_line_21_22); - PERF_TIMER_START_BP(VERIFY_line_21_22); - // PAPER LINES 21-22 - // The inner product challenges are computed per round - rct::keyV w(rounds); - for (size_t i = 0; i < rounds; ++i) - { - w[i] = hash_cache_mash(hash_cache, proof.L[i], proof.R[i]); + pd.inv_offset = inv_offset; + for (size_t i = 0; i < rounds; ++i) + to_invert.push_back(pd.w[i]); + to_invert.push_back(pd.y); + inv_offset += rounds + 1; } - PERF_TIMER_STOP(VERIFY_line_21_22); + CHECK_AND_ASSERT_MES(max_length < 32, false, "At least one proof is too large"); + size_t maxMN = 1u << max_length; - PERF_TIMER_START_BP(VERIFY_line_24_25); - // Basically PAPER LINES 24-25 - // Compute the curvepoints from G[i] and H[i] - rct::key inner_prod = rct::identity(); - rct::key yinvpow = rct::identity(); - rct::key ypow = rct::identity(); + rct::key tmp; - PERF_TIMER_START_BP(VERIFY_line_24_25_invert); - const rct::key yinv = invert(y); - rct::keyV winv(rounds); - for (size_t i = 0; i < rounds; ++i) - winv[i] = invert(w[i]); - PERF_TIMER_STOP(VERIFY_line_24_25_invert); + std::vector multiexp_data; + multiexp_data.reserve(nV + (2 * (max_logM + logN) + 4) * proofs.size() + 2 * maxMN); + multiexp_data.resize(2 * maxMN); - for (size_t i = 0; i < N; ++i) + PERF_TIMER_START_BP(VERIFY_line_24_25_invert); + const std::vector inverses = invert(to_invert); + PERF_TIMER_STOP_BP(VERIFY_line_24_25_invert); + + // setup weighted aggregates + rct::key z1 = rct::zero(); + rct::key z3 = rct::zero(); + rct::keyV m_z4(maxMN, rct::zero()), m_z5(maxMN, rct::zero()); + rct::key m_y0 = rct::zero(), y1 = rct::zero(); + int proof_data_index = 0; + rct::keyV w_cache; + rct::keyV proof8_V, proof8_L, proof8_R; + for (const Bulletproof *p: proofs) { - // Convert the index to binary IN REVERSE and construct the scalar exponent - rct::key g_scalar = proof.a; - rct::key h_scalar; - sc_mul(h_scalar.bytes, proof.b.bytes, yinvpow.bytes); + const Bulletproof &proof = *p; + const proof_data_t &pd = proof_data[proof_data_index++]; + + CHECK_AND_ASSERT_MES(proof.L.size() == 6+pd.logM, false, "Proof is not the expected size"); + const size_t M = 1 << pd.logM; + const size_t MN = M*N; + const rct::key weight_y = rct::skGen(); + const rct::key weight_z = rct::skGen(); + + // pre-multiply some points by 8 + proof8_V.resize(proof.V.size()); for (size_t i = 0; i < proof.V.size(); ++i) proof8_V[i] = rct::scalarmult8(proof.V[i]); + proof8_L.resize(proof.L.size()); for (size_t i = 0; i < proof.L.size(); ++i) proof8_L[i] = rct::scalarmult8(proof.L[i]); + proof8_R.resize(proof.R.size()); for (size_t i = 0; i < proof.R.size(); ++i) proof8_R[i] = rct::scalarmult8(proof.R[i]); + rct::key proof8_T1 = rct::scalarmult8(proof.T1); + rct::key proof8_T2 = rct::scalarmult8(proof.T2); + rct::key proof8_S = rct::scalarmult8(proof.S); + rct::key proof8_A = rct::scalarmult8(proof.A); + + PERF_TIMER_START_BP(VERIFY_line_61); + // PAPER LINE 61 + sc_mulsub(m_y0.bytes, proof.taux.bytes, weight_y.bytes, m_y0.bytes); + + const rct::keyV zpow = vector_powers(pd.z, M+3); + + rct::key k; + const rct::key ip1y = vector_power_sum(pd.y, MN); + sc_mulsub(k.bytes, zpow[2].bytes, ip1y.bytes, rct::zero().bytes); + for (size_t j = 1; j <= M; ++j) + { + CHECK_AND_ASSERT_MES(j+2 < zpow.size(), false, "invalid zpow index"); + sc_mulsub(k.bytes, zpow[j+2].bytes, ip12.bytes, k.bytes); + } + PERF_TIMER_STOP_BP(VERIFY_line_61); - for (size_t j = rounds; j-- > 0; ) + PERF_TIMER_START_BP(VERIFY_line_61rl_new); + sc_muladd(tmp.bytes, pd.z.bytes, ip1y.bytes, k.bytes); + sc_sub(tmp.bytes, proof.t.bytes, tmp.bytes); + sc_muladd(y1.bytes, tmp.bytes, weight_y.bytes, y1.bytes); + for (size_t j = 0; j < proof8_V.size(); j++) + { + sc_mul(tmp.bytes, zpow[j+2].bytes, weight_y.bytes); + multiexp_data.emplace_back(tmp, proof8_V[j]); + } + sc_mul(tmp.bytes, pd.x.bytes, weight_y.bytes); + multiexp_data.emplace_back(tmp, proof8_T1); + rct::key xsq; + sc_mul(xsq.bytes, pd.x.bytes, pd.x.bytes); + sc_mul(tmp.bytes, xsq.bytes, weight_y.bytes); + multiexp_data.emplace_back(tmp, proof8_T2); + PERF_TIMER_STOP_BP(VERIFY_line_61rl_new); + + PERF_TIMER_START_BP(VERIFY_line_62); + // PAPER LINE 62 + multiexp_data.emplace_back(weight_z, proof8_A); + sc_mul(tmp.bytes, pd.x.bytes, weight_z.bytes); + multiexp_data.emplace_back(tmp, proof8_S); + PERF_TIMER_STOP_BP(VERIFY_line_62); + + // Compute the number of rounds for the inner product + const size_t rounds = pd.logM+logN; + CHECK_AND_ASSERT_MES(rounds > 0, false, "Zero rounds"); + + PERF_TIMER_START_BP(VERIFY_line_24_25); + // Basically PAPER LINES 24-25 + // Compute the curvepoints from G[i] and H[i] + rct::key yinvpow = rct::identity(); + rct::key ypow = rct::identity(); + + const rct::key *winv = &inverses[pd.inv_offset]; + const rct::key yinv = inverses[pd.inv_offset + rounds]; + + // precalc + PERF_TIMER_START_BP(VERIFY_line_24_25_precalc); + w_cache.resize(1< 0; --s) + { + sc_mul(w_cache[s].bytes, w_cache[s/2].bytes, pd.w[j].bytes); + sc_mul(w_cache[s-1].bytes, w_cache[s/2].bytes, winv[j].bytes); + } + } + PERF_TIMER_STOP_BP(VERIFY_line_24_25_precalc); - if ((i & (((size_t)1)< &proofs) +{ + std::vector proof_pointers; + proof_pointers.reserve(proofs.size()); + for (const Bulletproof &proof: proofs) + proof_pointers.push_back(&proof); + return bulletproof_VERIFY(proof_pointers); +} + +bool bulletproof_VERIFY(const Bulletproof &proof) +{ + std::vector proofs; + proofs.push_back(&proof); + return bulletproof_VERIFY(proofs); +} + } diff --git a/src/ringct/bulletproofs.h b/src/ringct/bulletproofs.h index 3061d272e29..21d49483406 100644 --- a/src/ringct/bulletproofs.h +++ b/src/ringct/bulletproofs.h @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2018, The Monero Project +// Copyright (c) 2017-2019, The Monero Project // // All rights reserved. // @@ -40,7 +40,11 @@ namespace rct Bulletproof bulletproof_PROVE(const rct::key &v, const rct::key &gamma); Bulletproof bulletproof_PROVE(uint64_t v, const rct::key &gamma); +Bulletproof bulletproof_PROVE(const rct::keyV &v, const rct::keyV &gamma); +Bulletproof bulletproof_PROVE(const std::vector &v, const rct::keyV &gamma); bool bulletproof_VERIFY(const Bulletproof &proof); +bool bulletproof_VERIFY(const std::vector &proofs); +bool bulletproof_VERIFY(const std::vector &proofs); } diff --git a/src/ringct/multiexp.cc b/src/ringct/multiexp.cc new file mode 100644 index 00000000000..6f77fed3450 --- /dev/null +++ b/src/ringct/multiexp.cc @@ -0,0 +1,711 @@ +// Copyright (c) 2017, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Adapted from Python code by Sarang Noether + +#include "misc_log_ex.h" +#include "common/perf_timer.h" +extern "C" +{ +#include "crypto/crypto-ops.h" +} +#include "common/aligned.h" +#include "rctOps.h" +#include "multiexp.h" + +#undef MONERO_DEFAULT_LOG_CATEGORY +#define MONERO_DEFAULT_LOG_CATEGORY "multiexp" + +//#define MULTIEXP_PERF(x) x +#define MULTIEXP_PERF(x) + +#define RAW_MEMORY_BLOCK +//#define ALTERNATE_LAYOUT +//#define TRACK_STRAUS_ZERO_IDENTITY + +// per points us for N/B points (B point bands) +// raw alt 128/192 4096/192 4096/4096 +// 0 0 52.6 71 71.2 +// 0 1 53.2 72.2 72.4 +// 1 0 52.7 67 67.1 +// 1 1 52.8 70.4 70.2 + +// Pippenger: +// 1 2 3 4 5 6 7 8 9 bestN +// 2 555 598 621 804 1038 1733 2486 5020 8304 1 +// 4 783 747 800 1006 1428 2132 3285 5185 9806 2 +// 8 1174 1071 1095 1286 1640 2398 3869 6378 12080 2 +// 16 2279 1874 1745 1739 2144 2831 4209 6964 12007 4 +// 32 3910 3706 2588 2477 2782 3467 4856 7489 12618 4 +// 64 7184 5429 4710 4368 4010 4672 6027 8559 13684 5 +// 128 14097 10574 8452 7297 6841 6718 8615 10580 15641 6 +// 256 27715 20800 16000 13550 11875 11400 11505 14090 18460 6 +// 512 55100 41250 31740 26570 22030 19830 20760 21380 25215 6 +// 1024 111520 79000 61080 49720 43080 38320 37600 35040 36750 8 +// 2048 219480 162680 122120 102080 83760 70360 66600 63920 66160 8 +// 4096 453320 323080 247240 210200 180040 150240 132440 114920 110560 9 + +// 2 4 8 16 32 64 128 256 512 1024 2048 4096 +// Bos Coster 858 994 1316 1949 3183 5512 9865 17830 33485 63160 124280 246320 +// Straus 226 341 548 980 1870 3538 7039 14490 29020 57200 118640 233640 +// Straus/cached 226 315 485 785 1514 2858 5753 11065 22970 45120 98880 194840 +// Pippenger 555 747 1071 1739 2477 4010 6718 11400 19830 35040 63920 110560 + +// Best/cached Straus Straus Straus Straus Straus Straus Straus Straus Pip Pip Pip Pip +// Best/uncached Straus Straus Straus Straus Straus Straus Pip Pip Pip Pip Pip Pip + +// New timings: +// Pippenger: +// 2/1 always +// 3/2 at ~13 +// 4/3 at ~29 +// 5/4 at ~83 +// 6/5 < 200 +// 7/6 at ~470 +// 8/7 at ~1180 +// 9/8 at ~2290 +// Cached Pippenger: +// 6/5 < 200 +// 7/6 at 460 +// 8/7 at 1180 +// 9/8 at 2300 +// +// Cached Straus/Pippenger cross at 232 +// + +namespace rct +{ + +static inline bool operator<(const rct::key &k0, const rct::key&k1) +{ + for (int n = 31; n >= 0; --n) + { + if (k0.bytes[n] < k1.bytes[n]) + return true; + if (k0.bytes[n] > k1.bytes[n]) + return false; + } + return false; +} + +static inline rct::key div2(const rct::key &k) +{ + rct::key res; + int carry = 0; + for (int n = 31; n >= 0; --n) + { + int new_carry = (k.bytes[n] & 1) << 7; + res.bytes[n] = k.bytes[n] / 2 + carry; + carry = new_carry; + } + return res; +} + +static inline rct::key pow2(size_t n) +{ + CHECK_AND_ASSERT_THROW_MES(n < 256, "Invalid pow2 argument"); + rct::key res = rct::zero(); + res[n >> 3] |= 1<<(n&7); + return res; +} + +static inline int test(const rct::key &k, size_t n) +{ + if (n >= 256) return 0; + return k[n >> 3] & (1 << (n & 7)); +} + +static inline void add(ge_p3 &p3, const ge_cached &other) +{ + ge_p1p1 p1; + ge_add(&p1, &p3, &other); + ge_p1p1_to_p3(&p3, &p1); +} + +static inline void add(ge_p3 &p3, const ge_p3 &other) +{ + ge_cached cached; + ge_p3_to_cached(&cached, &other); + add(p3, cached); +} + +rct::key bos_coster_heap_conv(std::vector data) +{ + MULTIEXP_PERF(PERF_TIMER_START_UNIT(bos_coster, 1000000)); + MULTIEXP_PERF(PERF_TIMER_START_UNIT(setup, 1000000)); + size_t points = data.size(); + CHECK_AND_ASSERT_THROW_MES(points > 1, "Not enough points"); + std::vector heap(points); + for (size_t n = 0; n < points; ++n) + heap[n] = n; + + auto Comp = [&](size_t e0, size_t e1) { return data[e0].scalar < data[e1].scalar; }; + std::make_heap(heap.begin(), heap.end(), Comp); + MULTIEXP_PERF(PERF_TIMER_STOP(setup)); + + MULTIEXP_PERF(PERF_TIMER_START_UNIT(loop, 1000000)); + MULTIEXP_PERF(PERF_TIMER_START_UNIT(pop, 1000000)); MULTIEXP_PERF(PERF_TIMER_PAUSE(pop)); + MULTIEXP_PERF(PERF_TIMER_START_UNIT(add, 1000000)); MULTIEXP_PERF(PERF_TIMER_PAUSE(add)); + MULTIEXP_PERF(PERF_TIMER_START_UNIT(sub, 1000000)); MULTIEXP_PERF(PERF_TIMER_PAUSE(sub)); + MULTIEXP_PERF(PERF_TIMER_START_UNIT(push, 1000000)); MULTIEXP_PERF(PERF_TIMER_PAUSE(push)); + while (heap.size() > 1) + { + MULTIEXP_PERF(PERF_TIMER_RESUME(pop)); + std::pop_heap(heap.begin(), heap.end(), Comp); + size_t index1 = heap.back(); + heap.pop_back(); + std::pop_heap(heap.begin(), heap.end(), Comp); + size_t index2 = heap.back(); + heap.pop_back(); + MULTIEXP_PERF(PERF_TIMER_PAUSE(pop)); + + MULTIEXP_PERF(PERF_TIMER_RESUME(add)); + ge_cached cached; + ge_p3_to_cached(&cached, &data[index1].point); + ge_p1p1 p1; + ge_add(&p1, &data[index2].point, &cached); + ge_p1p1_to_p3(&data[index2].point, &p1); + MULTIEXP_PERF(PERF_TIMER_PAUSE(add)); + + MULTIEXP_PERF(PERF_TIMER_RESUME(sub)); + sc_sub(data[index1].scalar.bytes, data[index1].scalar.bytes, data[index2].scalar.bytes); + MULTIEXP_PERF(PERF_TIMER_PAUSE(sub)); + + MULTIEXP_PERF(PERF_TIMER_RESUME(push)); + if (!(data[index1].scalar == rct::zero())) + { + heap.push_back(index1); + std::push_heap(heap.begin(), heap.end(), Comp); + } + + heap.push_back(index2); + std::push_heap(heap.begin(), heap.end(), Comp); + MULTIEXP_PERF(PERF_TIMER_PAUSE(push)); + } + MULTIEXP_PERF(PERF_TIMER_STOP(push)); + MULTIEXP_PERF(PERF_TIMER_STOP(sub)); + MULTIEXP_PERF(PERF_TIMER_STOP(add)); + MULTIEXP_PERF(PERF_TIMER_STOP(pop)); + MULTIEXP_PERF(PERF_TIMER_STOP(loop)); + + MULTIEXP_PERF(PERF_TIMER_START_UNIT(end, 1000000)); + //return rct::scalarmultKey(data[index1].point, data[index1].scalar); + std::pop_heap(heap.begin(), heap.end(), Comp); + size_t index1 = heap.back(); + heap.pop_back(); + ge_p2 p2; + ge_scalarmult(&p2, data[index1].scalar.bytes, &data[index1].point); + rct::key res; + ge_tobytes(res.bytes, &p2); + return res; +} + +rct::key bos_coster_heap_conv_robust(std::vector data) +{ + MULTIEXP_PERF(PERF_TIMER_START_UNIT(bos_coster, 1000000)); + MULTIEXP_PERF(PERF_TIMER_START_UNIT(setup, 1000000)); + size_t points = data.size(); + CHECK_AND_ASSERT_THROW_MES(points > 0, "Not enough points"); + std::vector heap; + heap.reserve(points); + for (size_t n = 0; n < points; ++n) + { + if (!(data[n].scalar == rct::zero()) && !ge_p3_is_point_at_infinity(&data[n].point)) + heap.push_back(n); + } + points = heap.size(); + if (points == 0) + return rct::identity(); + + auto Comp = [&](size_t e0, size_t e1) { return data[e0].scalar < data[e1].scalar; }; + std::make_heap(heap.begin(), heap.end(), Comp); + + if (points < 2) + { + std::pop_heap(heap.begin(), heap.end(), Comp); + size_t index1 = heap.back(); + ge_p2 p2; + ge_scalarmult(&p2, data[index1].scalar.bytes, &data[index1].point); + rct::key res; + ge_tobytes(res.bytes, &p2); + return res; + } + + MULTIEXP_PERF(PERF_TIMER_STOP(setup)); + + MULTIEXP_PERF(PERF_TIMER_START_UNIT(loop, 1000000)); + MULTIEXP_PERF(PERF_TIMER_START_UNIT(pop, 1000000)); MULTIEXP_PERF(PERF_TIMER_PAUSE(pop)); + MULTIEXP_PERF(PERF_TIMER_START_UNIT(div, 1000000)); MULTIEXP_PERF(PERF_TIMER_PAUSE(div)); + MULTIEXP_PERF(PERF_TIMER_START_UNIT(add, 1000000)); MULTIEXP_PERF(PERF_TIMER_PAUSE(add)); + MULTIEXP_PERF(PERF_TIMER_START_UNIT(sub, 1000000)); MULTIEXP_PERF(PERF_TIMER_PAUSE(sub)); + MULTIEXP_PERF(PERF_TIMER_START_UNIT(push, 1000000)); MULTIEXP_PERF(PERF_TIMER_PAUSE(push)); + while (heap.size() > 1) + { + MULTIEXP_PERF(PERF_TIMER_RESUME(pop)); + std::pop_heap(heap.begin(), heap.end(), Comp); + size_t index1 = heap.back(); + heap.pop_back(); + std::pop_heap(heap.begin(), heap.end(), Comp); + size_t index2 = heap.back(); + heap.pop_back(); + MULTIEXP_PERF(PERF_TIMER_PAUSE(pop)); + + ge_cached cached; + ge_p1p1 p1; + ge_p2 p2; + + MULTIEXP_PERF(PERF_TIMER_RESUME(div)); + while (1) + { + rct::key s1_2 = div2(data[index1].scalar); + if (!(data[index2].scalar < s1_2)) + break; + if (data[index1].scalar.bytes[0] & 1) + { + data.resize(data.size()+1); + data.back().scalar = rct::identity(); + data.back().point = data[index1].point; + heap.push_back(data.size() - 1); + std::push_heap(heap.begin(), heap.end(), Comp); + } + data[index1].scalar = div2(data[index1].scalar); + ge_p3_to_p2(&p2, &data[index1].point); + ge_p2_dbl(&p1, &p2); + ge_p1p1_to_p3(&data[index1].point, &p1); + } + MULTIEXP_PERF(PERF_TIMER_PAUSE(div)); + + MULTIEXP_PERF(PERF_TIMER_RESUME(add)); + ge_p3_to_cached(&cached, &data[index1].point); + ge_add(&p1, &data[index2].point, &cached); + ge_p1p1_to_p3(&data[index2].point, &p1); + MULTIEXP_PERF(PERF_TIMER_PAUSE(add)); + + MULTIEXP_PERF(PERF_TIMER_RESUME(sub)); + sc_sub(data[index1].scalar.bytes, data[index1].scalar.bytes, data[index2].scalar.bytes); + MULTIEXP_PERF(PERF_TIMER_PAUSE(sub)); + + MULTIEXP_PERF(PERF_TIMER_RESUME(push)); + if (!(data[index1].scalar == rct::zero())) + { + heap.push_back(index1); + std::push_heap(heap.begin(), heap.end(), Comp); + } + + heap.push_back(index2); + std::push_heap(heap.begin(), heap.end(), Comp); + MULTIEXP_PERF(PERF_TIMER_PAUSE(push)); + } + MULTIEXP_PERF(PERF_TIMER_STOP(push)); + MULTIEXP_PERF(PERF_TIMER_STOP(sub)); + MULTIEXP_PERF(PERF_TIMER_STOP(add)); + MULTIEXP_PERF(PERF_TIMER_STOP(pop)); + MULTIEXP_PERF(PERF_TIMER_STOP(loop)); + + MULTIEXP_PERF(PERF_TIMER_START_UNIT(end, 1000000)); + //return rct::scalarmultKey(data[index1].point, data[index1].scalar); + std::pop_heap(heap.begin(), heap.end(), Comp); + size_t index1 = heap.back(); + heap.pop_back(); + ge_p2 p2; + ge_scalarmult(&p2, data[index1].scalar.bytes, &data[index1].point); + rct::key res; + ge_tobytes(res.bytes, &p2); + return res; +} + +#define STRAUS_C 4 + +struct straus_cached_data +{ +#ifdef RAW_MEMORY_BLOCK + size_t size; + ge_cached *multiples; + straus_cached_data(): size(0), multiples(NULL) {} + ~straus_cached_data() { aligned_free(multiples); } +#else + std::vector> multiples; +#endif +}; +#ifdef RAW_MEMORY_BLOCK +#ifdef ALTERNATE_LAYOUT +#define CACHE_OFFSET(cache,point,digit) cache->multiples[(point)*((1<multiples[(point)+cache->size*((digit)-1)] +#endif +#else +#ifdef ALTERNATE_LAYOUT +#define CACHE_OFFSET(cache,point,digit) local_cache->multiples[j][digit-1] +#else +#define CACHE_OFFSET(cache,point,digit) local_cache->multiples[digit][j] +#endif +#endif + +std::shared_ptr straus_init_cache(const std::vector &data, size_t N) +{ + MULTIEXP_PERF(PERF_TIMER_START_UNIT(multiples, 1000000)); + if (N == 0) + N = data.size(); + CHECK_AND_ASSERT_THROW_MES(N <= data.size(), "Bad cache base data"); + ge_cached cached; + ge_p1p1 p1; + ge_p3 p3; + std::shared_ptr cache(new straus_cached_data()); + +#ifdef RAW_MEMORY_BLOCK + const size_t offset = cache->size; + cache->multiples = (ge_cached*)aligned_realloc(cache->multiples, sizeof(ge_cached) * ((1<multiples, "Out of memory"); + cache->size = N; + for (size_t j=offset;jmultiples.size(); + cache->multiples.resize(std::max(offset, N)); + for (size_t i = offset; i < N; ++i) + { + cache->multiples[i].resize((1<multiples[i][0], &data[i].point); + for (size_t j=2;j<1<multiples[i][j-2]); + ge_p1p1_to_p3(&p3, &p1); + ge_p3_to_cached(&cache->multiples[i][j-1], &p3); + } + } +#else + cache->multiples.resize(1<multiples[1].size(); + cache->multiples[1].resize(std::max(offset, N)); + for (size_t i = offset; i < N; ++i) + ge_p3_to_cached(&cache->multiples[1][i], &data[i].point); + for (size_t i=2;i<1<multiples[i].resize(std::max(offset, N)); + for (size_t j=offset;jmultiples[i-1][j]); + ge_p1p1_to_p3(&p3, &p1); + ge_p3_to_cached(&cache->multiples[i][j], &p3); + } + } +#endif +#endif + MULTIEXP_PERF(PERF_TIMER_STOP(multiples)); + + return cache; +} + +size_t straus_get_cache_size(const std::shared_ptr &cache) +{ + size_t sz = 0; +#ifdef RAW_MEMORY_BLOCK + sz += cache->size * sizeof(ge_cached) * ((1<multiples) + sz += e0.size() * sizeof(ge_cached); +#endif + return sz; +} + +rct::key straus(const std::vector &data, const std::shared_ptr &cache, size_t STEP) +{ + CHECK_AND_ASSERT_THROW_MES(cache == NULL || cache->size >= data.size(), "Cache is too small"); + MULTIEXP_PERF(PERF_TIMER_UNIT(straus, 1000000)); + bool HiGi = cache != NULL; + STEP = STEP ? STEP : 192; + + MULTIEXP_PERF(PERF_TIMER_START_UNIT(setup, 1000000)); + static constexpr unsigned int mask = (1< local_cache = cache == NULL ? straus_init_cache(data) : cache; + ge_cached cached; + ge_p1p1 p1; + ge_p3 p3; + +#ifdef TRACK_STRAUS_ZERO_IDENTITY + MULTIEXP_PERF(PERF_TIMER_START_UNIT(skip, 1000000)); + std::vector skip(data.size()); + for (size_t i = 0; i < data.size(); ++i) + skip[i] = data[i].scalar == rct::zero() || ge_p3_is_point_at_infinity(&data[i].point); + MULTIEXP_PERF(PERF_TIMER_STOP(skip)); +#endif + + MULTIEXP_PERF(PERF_TIMER_START_UNIT(digits, 1000000)); +#if STRAUS_C==4 + std::unique_ptr digits{new uint8_t[64 * data.size()]}; +#else + std::unique_ptr digits{new uint8_t[256 * data.size()]}; +#endif + for (size_t j = 0; j < data.size(); ++j) + { + const unsigned char *bytes = data[j].scalar.bytes; +#if STRAUS_C==4 + unsigned int i; + for (i = 0; i < 64; i += 2, bytes++) + { + digits[j*64+i] = bytes[0] & 0xf; + digits[j*64+i+1] = bytes[0] >> 4; + } +#elif 1 + unsigned char bytes33[33]; + memcpy(bytes33, data[j].scalar.bytes, 32); + bytes33[32] = 0; + bytes = bytes33; + for (size_t i = 0; i < 256; ++i) + digits[j*256+i] = ((bytes[i>>3] | (bytes[(i>>3)+1]<<8)) >> (i&7)) & mask; +#else + rct::key shifted = data[j].scalar; + for (size_t i = 0; i < 256; ++i) + { + digits[j*256+i] = shifted.bytes[0] & 0xf; + shifted = div2(shifted, (256-i)>>3); + } +#endif + } + MULTIEXP_PERF(PERF_TIMER_STOP(digits)); + + rct::key maxscalar = rct::zero(); + for (size_t i = 0; i < data.size(); ++i) + if (maxscalar < data[i].scalar) + maxscalar = data[i].scalar; + size_t start_i = 0; + while (start_i < 256 && !(maxscalar < pow2(start_i))) + start_i += STRAUS_C; + MULTIEXP_PERF(PERF_TIMER_STOP(setup)); + + ge_p3 res_p3 = ge_p3_identity; + + for (size_t start_offset = 0; start_offset < data.size(); start_offset += STEP) + { + const size_t num_points = std::min(data.size() - start_offset, STEP); + + ge_p3 band_p3 = ge_p3_identity; + size_t i = start_i; + if (!(i < STRAUS_C)) + goto skipfirst; + while (!(i < STRAUS_C)) + { + ge_p2 p2; + ge_p3_to_p2(&p2, &band_p3); + for (size_t j = 0; j < STRAUS_C; ++j) + { + ge_p2_dbl(&p1, &p2); + if (j == STRAUS_C - 1) + ge_p1p1_to_p3(&band_p3, &p1); + else + ge_p1p1_to_p2(&p2, &p1); + } +skipfirst: + i -= STRAUS_C; + for (size_t j = start_offset; j < start_offset + num_points; ++j) + { +#ifdef TRACK_STRAUS_ZERO_IDENTITY + if (skip[j]) + continue; +#endif +#if STRAUS_C==4 + const uint8_t digit = digits[j*64+i/4]; +#else + const uint8_t digit = digits[j*256+i]; +#endif + if (digit) + { + ge_add(&p1, &band_p3, &CACHE_OFFSET(local_cache, j, digit)); + ge_p1p1_to_p3(&band_p3, &p1); + } + } + } + + ge_p3_to_cached(&cached, &band_p3); + ge_add(&p1, &res_p3, &cached); + ge_p1p1_to_p3(&res_p3, &p1); + } + + rct::key res; + ge_p3_tobytes(res.bytes, &res_p3); + return res; +} + +size_t get_pippenger_c(size_t N) +{ + if (N <= 13) return 2; + if (N <= 29) return 3; + if (N <= 83) return 4; + if (N <= 185) return 5; + if (N <= 465) return 6; + if (N <= 1180) return 7; + if (N <= 2295) return 8; + return 9; +} + +struct pippenger_cached_data +{ + size_t size; + ge_cached *cached; + pippenger_cached_data(): size(0), cached(NULL) {} + ~pippenger_cached_data() { aligned_free(cached); } +}; + +std::shared_ptr pippenger_init_cache(const std::vector &data, size_t start_offset, size_t N) +{ + MULTIEXP_PERF(PERF_TIMER_START_UNIT(pippenger_init_cache, 1000000)); + CHECK_AND_ASSERT_THROW_MES(start_offset <= data.size(), "Bad cache base data"); + if (N == 0) + N = data.size() - start_offset; + CHECK_AND_ASSERT_THROW_MES(N <= data.size() - start_offset, "Bad cache base data"); + ge_cached cached; + std::shared_ptr cache(new pippenger_cached_data()); + + cache->size = N; + cache->cached = (ge_cached*)aligned_realloc(cache->cached, N * sizeof(ge_cached), 4096); + CHECK_AND_ASSERT_THROW_MES(cache->cached, "Out of memory"); + for (size_t i = 0; i < N; ++i) + ge_p3_to_cached(&cache->cached[i], &data[i+start_offset].point); + + MULTIEXP_PERF(PERF_TIMER_STOP(pippenger_init_cache)); + return cache; +} + +size_t pippenger_get_cache_size(const std::shared_ptr &cache) +{ + return cache->size * sizeof(*cache->cached); +} + +rct::key pippenger(const std::vector &data, const std::shared_ptr &cache, size_t cache_size, size_t c) +{ + if (cache != NULL && cache_size == 0) + cache_size = cache->size; + CHECK_AND_ASSERT_THROW_MES(cache == NULL || cache_size <= cache->size, "Cache is too small"); + if (c == 0) + c = get_pippenger_c(data.size()); + CHECK_AND_ASSERT_THROW_MES(c <= 9, "c is too large"); + + ge_p3 result = ge_p3_identity; + bool result_init = false; + std::unique_ptr buckets{new ge_p3[1< local_cache = cache == NULL ? pippenger_init_cache(data) : cache; + std::shared_ptr local_cache_2 = data.size() > cache_size ? pippenger_init_cache(data, cache_size) : NULL; + + rct::key maxscalar = rct::zero(); + for (size_t i = 0; i < data.size(); ++i) + { + if (maxscalar < data[i].scalar) + maxscalar = data[i].scalar; + } + size_t groups = 0; + while (groups < 256 && !(maxscalar < pow2(groups))) + ++groups; + groups = (groups + c - 1) / c; + + for (size_t k = groups; k-- > 0; ) + { + if (result_init) + { + ge_p2 p2; + ge_p3_to_p2(&p2, &result); + for (size_t i = 0; i < c; ++i) + { + ge_p1p1 p1; + ge_p2_dbl(&p1, &p2); + if (i == c - 1) + ge_p1p1_to_p3(&result, &p1); + else + ge_p1p1_to_p2(&p2, &p1); + } + } + memset(buckets_init, 0, 1u<cached[i]); + else + add(buckets[bucket], local_cache_2->cached[i - cache_size]); + } + else + { + buckets[bucket] = data[i].point; + buckets_init[bucket] = true; + } + } + + // sum the buckets + ge_p3 pail; + bool pail_init = false; + for (size_t i = (1< 0; --i) + { + if (buckets_init[i]) + { + if (pail_init) + add(pail, buckets[i]); + else + { + pail = buckets[i]; + pail_init = true; + } + } + if (pail_init) + { + if (result_init) + add(result, pail); + else + { + result = pail; + result_init = true; + } + } + } + } + + rct::key res; + ge_p3_tobytes(res.bytes, &result); + return res; +} + +} diff --git a/src/ringct/multiexp.h b/src/ringct/multiexp.h new file mode 100644 index 00000000000..b5270793343 --- /dev/null +++ b/src/ringct/multiexp.h @@ -0,0 +1,71 @@ +// Copyright (c) 2017, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Adapted from Python code by Sarang Noether + +#pragma once + +#ifndef MULTIEXP_H +#define MULTIEXP_H + +#include +#include "crypto/crypto.h" +#include "rctTypes.h" +#include "misc_log_ex.h" + +namespace rct +{ + +struct MultiexpData { + rct::key scalar; + ge_p3 point; + + MultiexpData() {} + MultiexpData(const rct::key &s, const ge_p3 &p): scalar(s), point(p) {} + MultiexpData(const rct::key &s, const rct::key &p): scalar(s) + { + CHECK_AND_ASSERT_THROW_MES(ge_frombytes_vartime(&point, p.bytes) == 0, "ge_frombytes_vartime failed"); + } +}; + +struct straus_cached_data; +struct pippenger_cached_data; + +rct::key bos_coster_heap_conv(std::vector data); +rct::key bos_coster_heap_conv_robust(std::vector data); +std::shared_ptr straus_init_cache(const std::vector &data, size_t N =0); +size_t straus_get_cache_size(const std::shared_ptr &cache); +rct::key straus(const std::vector &data, const std::shared_ptr &cache = NULL, size_t STEP = 0); +std::shared_ptr pippenger_init_cache(const std::vector &data, size_t start_offset = 0, size_t N =0); +size_t pippenger_get_cache_size(const std::shared_ptr &cache); +size_t get_pippenger_c(size_t N); +rct::key pippenger(const std::vector &data, const std::shared_ptr &cache = NULL, size_t cache_size = 0, size_t c = 0); + +} + +#endif diff --git a/src/ringct/rctCryptoOps.c b/src/ringct/rctCryptoOps.c index 6fdd17f6bd9..fbbf6f9bd04 100644 --- a/src/ringct/rctCryptoOps.c +++ b/src/ringct/rctCryptoOps.c @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/ringct/rctCryptoOps.h b/src/ringct/rctCryptoOps.h index e5c1c987a1d..2a25d13a763 100644 --- a/src/ringct/rctCryptoOps.h +++ b/src/ringct/rctCryptoOps.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/ringct/rctOps.cpp b/src/ringct/rctOps.cpp index cc46d0aa78a..b5499262fa1 100644 --- a/src/ringct/rctOps.cpp +++ b/src/ringct/rctOps.cpp @@ -30,6 +30,7 @@ #include #include "misc_log_ex.h" +#include "cryptonote_basic/cryptonote_format_utils.h" #include "rctOps.h" using namespace crypto; using namespace std; @@ -39,6 +40,183 @@ using namespace std; #define CHECK_AND_ASSERT_THROW_MES_L1(expr, message) {if(!(expr)) {MWARNING(message); throw std::runtime_error(message);}} +struct zero_commitment { uint64_t amount; rct::key commitment; }; +static const zero_commitment zero_commitments[] = { + { (uint64_t)0ull, {{0x58, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66}} }, + { (uint64_t)1ull, {{0x17, 0x38, 0xeb, 0x7a, 0x67, 0x7c, 0x61, 0x49, 0x22, 0x8a, 0x2b, 0xea, 0xa2, 0x1b, 0xea, 0x9e, 0x33, 0x70, 0x80, 0x2d, 0x72, 0xa3, 0xee, 0xc7, 0x90, 0x11, 0x95, 0x80, 0xe0, 0x2b, 0xd5, 0x22}} }, + { (uint64_t)2ull, {{0x76, 0x24, 0x84, 0x63, 0xa, 0x6, 0x17, 0x17, 0x8d, 0xe, 0x33, 0xf3, 0x2e, 0xe, 0x11, 0x3e, 0xa8, 0x46, 0x86, 0x9d, 0x46, 0x4b, 0xb, 0x6f, 0xf1, 0x3b, 0x29, 0x97, 0x4, 0x9c, 0xda, 0x7d}} }, + { (uint64_t)3ull, {{0xcf, 0xf7, 0x7b, 0x56, 0x62, 0x1c, 0x4f, 0xef, 0x74, 0xcf, 0x37, 0xc1, 0x78, 0xd4, 0xb5, 0x8a, 0xf4, 0xad, 0x8c, 0xd4, 0x35, 0xfc, 0xb9, 0x62, 0x76, 0xbc, 0x15, 0x9c, 0x7c, 0x6a, 0x28, 0x8c}} }, + { (uint64_t)4ull, {{0x9a, 0xb8, 0x6c, 0x31, 0xf4, 0x22, 0xd8, 0x21, 0xb5, 0x22, 0x57, 0x30, 0xd1, 0xbf, 0x73, 0xa, 0x9b, 0x91, 0xd2, 0xee, 0xe3, 0x14, 0xb8, 0x4e, 0xbd, 0x4b, 0x93, 0xa6, 0x81, 0x61, 0x82, 0x66}} }, + { (uint64_t)5ull, {{0x32, 0xee, 0x2f, 0x65, 0x9a, 0xf6, 0x38, 0x58, 0xc2, 0xf7, 0xdc, 0x11, 0x1b, 0x3b, 0xb8, 0xfe, 0xc0, 0x2c, 0xac, 0x42, 0x38, 0x3b, 0xb7, 0x36, 0xde, 0x1, 0x8, 0x6f, 0x38, 0xf0, 0x12, 0x3c}} }, + { (uint64_t)6ull, {{0x47, 0x26, 0x2b, 0x1e, 0xa6, 0x43, 0x1, 0x6e, 0x38, 0x24, 0x17, 0x53, 0xa4, 0xfb, 0x39, 0x92, 0x9e, 0x31, 0xea, 0x9b, 0xd3, 0x41, 0x1a, 0xb1, 0x7f, 0x16, 0x6e, 0x61, 0xf6, 0xc, 0xe5, 0xa7}} }, + { (uint64_t)7ull, {{0xc6, 0x32, 0x93, 0x68, 0x79, 0x9a, 0xd, 0xed, 0x4c, 0x20, 0x25, 0x6b, 0xff, 0xe6, 0x45, 0x47, 0xf1, 0x7b, 0xc4, 0x23, 0x95, 0x4, 0xbe, 0x82, 0x4d, 0xff, 0x8a, 0x2b, 0xe1, 0xaf, 0xe3, 0xcd}} }, + { (uint64_t)8ull, {{0xd5, 0xf1, 0x50, 0x74, 0x33, 0x46, 0x19, 0xf, 0x84, 0x2b, 0x6, 0xb8, 0xfa, 0xe1, 0x20, 0xeb, 0x85, 0x24, 0x7e, 0x9f, 0x6d, 0xec, 0x88, 0xff, 0xa2, 0x23, 0xbf, 0x69, 0x94, 0xe9, 0xc8, 0xc2}} }, + { (uint64_t)9ull, {{0x56, 00, 0x23, 0x32, 0x9e, 0xc0, 0xfa, 0xf3, 0x3b, 0x5e, 0x3a, 0x5c, 0xb4, 0xea, 0xef, 0xee, 0x38, 0xf8, 0x96, 0x1c, 0x88, 0xb6, 0x6a, 0x2f, 0x19, 0xd4, 0x59, 0x51, 0x96, 0x9c, 0x6d, 0x1f}} }, + { (uint64_t)10ull, {{0x3, 0x80, 0xdc, 0x24, 0xcc, 0x97, 0xcc, 0xe6, 0x58, 0xc3, 0xa9, 0x47, 0xc5, 0x10, 0x25, 0xde, 0x1a, 0x69, 0x80, 0x3b, 0xdb, 0x50, 0x5, 0xe3, 0xb7, 0xdd, 0xa9, 0xd, 0x68, 0x59, 0xb0, 0x1c}} }, + { (uint64_t)20ull, {{0x9, 0x3, 0xf6, 0x2e, 0x97, 0x76, 0x47, 0x58, 0xfe, 0xf8, 0x9e, 0x5b, 0xec, 0x29, 0xef, 0x4f, 0xc5, 0xe6, 0x45, 0x4b, 0x2d, 0x47, 0x44, 0x47, 0x36, 0x4, 0x4c, 0x25, 0x2e, 0xe2, 0x8e, 0xba}} }, + { (uint64_t)30ull, {{0xa2, 0x8b, 0x89, 0xe0, 0xb, 0xed, 0x62, 0x31, 0x68, 0x5b, 0xf9, 0x74, 0x36, 0xf2, 0xba, 0x51, 0xa2, 0x51, 0x55, 0x7f, 0x8d, 0x17, 0xa, 0x78, 0xe3, 0x12, 0xd6, 0x24, 0xbf, 0x60, 0xff, 0xfe}} }, + { (uint64_t)40ull, {{0xb5, 0xc6, 0x95, 0x55, 0x6a, 0x28, 0x47, 0xb2, 0xe, 0x1c, 0xbb, 0x26, 0xe6, 0xa9, 0xc6, 0x8a, 0x61, 0xc5, 0x50, 0xce, 0xb7, 0xc3, 0x4, 0xfe, 0x92, 0x28, 0x3d, 0x29, 0xa9, 0xb2, 0x43, 0xcb}} }, + { (uint64_t)50ull, {{0x12, 0x8e, 0xc6, 0xcd, 0xc0, 0x6b, 0x43, 0xc5, 0xd0, 0x9c, 0x3f, 0x65, 0x2a, 0xe3, 0x44, 0x7f, 0x9b, 0x3f, 0x2c, 0x30, 0x91, 0x2d, 0xf0, 0x80, 0x37, 00, 0x85, 0xbc, 0xc, 0x9, 0xef, 0x78}} }, + { (uint64_t)60ull, {{0x1f, 0x9f, 0x40, 0x3a, 0xae, 0xa7, 0x16, 0xfb, 0xe2, 0x98, 0xa8, 0x14, 0xf1, 0xee, 0xbc, 0x1b, 0x73, 0x16, 0x8c, 0x37, 0xfa, 0xe3, 0x16, 0xeb, 0x65, 0x5, 0x81, 0x6f, 0xc2, 0x20, 0xeb, 0xfb}} }, + { (uint64_t)70ull, {{0x10, 0xa2, 0x38, 0xc5, 0xe4, 0x8e, 0x4b, 0x93, 0x99, 0xdb, 0xa6, 0xcb, 0xd9, 0x8e, 0x63, 0x54, 0x41, 0x59, 0xe9, 0x8c, 0x93, 0x5a, 0xc0, 0x60, 0x3d, 0x72, 0xde, 0xf, 0xff, 0x31, 0x53, 0xbb}} }, + { (uint64_t)80ull, {{0x75, 0xab, 0x78, 0xc7, 0x28, 0x1f, 0x69, 0x28, 0xf0, 0x94, 0x86, 0x5, 0x7a, 0x63, 0x64, 0x18, 0x27, 0xc5, 0x74, 0x84, 0xe3, 0xe9, 0x9a, 0x39, 0xf3, 0x12, 0xa4, 0x3a, 0x51, 0x9b, 0xda, 0x8}} }, + { (uint64_t)90ull, {{0xe9, 0x56, 0x7b, 0xa7, 0x88, 0xb8, 0x5b, 0x82, 0xc8, 0x65, 0x7a, 0x15, 0xa5, 0x48, 0x99, 0x5c, 0xf6, 0xb0, 0xbd, 0xd1, 0xc6, 0x2a, 0xda, 0x77, 0x55, 0xf2, 0x32, 0x3a, 0xd8, 0xa4, 0x8, 0x51}} }, + { (uint64_t)100ull, {{0xb6, 0x17, 0x36, 0xd5, 0xf2, 0x8d, 0xef, 0x28, 0x61, 0x6a, 0xfc, 0x47, 0x93, 0xe9, 0x9b, 0x27, 0xcd, 0x3e, 0x89, 0xfb, 0x91, 0xc1, 0x13, 0xd4, 0x30, 0x73, 0x65, 0xfb, 0x75, 0xde, 0xdf, 0x88}} }, + { (uint64_t)200ull, {{0x71, 0x3, 0xeb, 0x72, 0x19, 0x28, 0xd7, 0x91, 0x99, 0x87, 0xf3, 0x50, 0xca, 0xa5, 0x7a, 0xe7, 0xb0, 0x81, 0x57, 0x15, 0x3b, 0x4c, 0x43, 0xd, 0x3e, 0xde, 0xc0, 0xc2, 0x3, 0x7, 0x97, 0x44}} }, + { (uint64_t)300ull, {{0x24, 0x40, 0x9e, 0x92, 0x2e, 0xce, 0xd1, 0xa0, 0x5e, 0x4e, 0xac, 0xa3, 0xdf, 0x91, 0x19, 0xc3, 0x8a, 0x92, 0x2e, 0xb, 0x66, 0xd0, 0x2d, 0x9d, 0xd2, 0xfb, 0x1d, 0xcc, 0x20, 0xb9, 0xaf, 0xc7}} }, + { (uint64_t)400ull, {{0xa7, 0x72, 0x9f, 0xa9, 0x32, 0x81, 0x82, 0x99, 0x34, 0x11, 0x5d, 0x47, 0x5a, 0x67, 0x86, 0xa, 0x14, 0x12, 0xc5, 0xe5, 0x95, 0x12, 0x20, 0xd9, 0x60, 0xc2, 0x41, 0xa0, 0x19, 0x1a, 0x9e, 0x65}} }, + { (uint64_t)500ull, {{0x2e, 0x53, 0xc, 0x6, 0x1c, 0x6d, 0x9e, 0x97, 0xab, 0xaf, 0x46, 0x8c, 0x32, 0xb0, 0xad, 0xa7, 0x49, 0x22, 0x57, 0x72, 0xfc, 0xd1, 0x17, 0x41, 0xcb, 0x5c, 0x3, 0x5c, 0xdd, 0x26, 0x14, 0xe}} }, + { (uint64_t)600ull, {{0xa5, 0xb, 0x91, 0x9, 0x9d, 0xf1, 0xb1, 0x69, 0x4f, 0x30, 0xb5, 0x8f, 0xe6, 0x77, 0x68, 0x50, 0xdb, 0xdb, 0xf4, 0x6c, 0xed, 0x99, 0x7f, 0x52, 0x62, 0xa8, 0x51, 0x59, 0x40, 0x74, 0xa5, 0x9d}} }, + { (uint64_t)700ull, {{0x51, 0x2c, 0xf, 0xae, 0xcc, 0xbe, 0xf2, 0xfe, 0xe5, 0x75, 0x4c, 0x6a, 0x45, 0xfd, 0xc0, 0x75, 0x2d, 0x4f, 0x15, 0x22, 0xe7, 0x7f, 0xf0, 0xc4, 0x8d, 0xcb, 0x19, 0x91, 0x8a, 0x68, 0x84, 0xe0}} }, + { (uint64_t)800ull, {{0xda, 0xa9, 0xf9, 0xa5, 0xb9, 0x71, 0x33, 0x33, 0xe9, 0x8c, 0x5, 0xac, 0xe7, 0x27, 0xcc, 0xe, 0x7d, 0xc3, 0xf1, 0x59, 0x49, 0xe1, 0xef, 0x4d, 0x94, 0xfa, 0x47, 0xd6, 0x8a, 0x34, 0xc6, 0x75}} }, + { (uint64_t)900ull, {{0xc7, 0x2b, 0x18, 0xc9, 0x17, 0xcd, 0x43, 0xee, 0x78, 0x40, 0x5e, 0x39, 0x83, 0x98, 0xb8, 0x3a, 0xc0, 0x97, 0x7b, 0x25, 0x19, 0x90, 0xd8, 0x13, 0xc, 0x38, 0xba, 0x53, 0xb6, 0x3d, 0xb4, 0xf7}} }, + { (uint64_t)1000ull, {{0x1, 0xdf, 0x60, 0x91, 0xeb, 0x6a, 0x48, 0xe9, 0xe4, 0x22, 0x25, 0xb, 0xe3, 0x83, 0x88, 0xc8, 0x61, 0xb6, 0x55, 0x55, 0xa7, 0x20, 0xad, 0x15, 0x35, 0x86, 0xfe, 0x2b, 0xd2, 0x2f, 0xa2, 0x3d}} }, + { (uint64_t)2000ull, {{0x24, 0xf5, 0xb1, 0x34, 0x78, 0x46, 0xaf, 0x22, 0xb5, 0x6f, 0x41, 0x25, 0xb3, 0xe7, 0x67, 0x8c, 0xf8, 0x4b, 0x4f, 0xd2, 0xf9, 0x2e, 0x1c, 0x40, 0xaa, 0x3a, 0x1b, 0xe0, 0xc7, 0x4d, 0x95, 0xe6}} }, + { (uint64_t)3000ull, {{0xa7, 0x1c, 0x9a, 0x8f, 0x40, 0xc1, 0x25, 0x9c, 0x36, 0x26, 0x27, 0x73, 0xe0, 0x8, 0x20, 0x18, 0x3e, 0x6b, 0x59, 0xe0, 0x71, 0xc9, 0x9b, 0x34, 0x9b, 0xef, 0x8f, 0x7e, 0xd2, 0xc6, 0xad, 0xb9}} }, + { (uint64_t)4000ull, {{0x98, 0xdc, 0x74, 0xaf, 0x19, 0x89, 0xd3, 0x4b, 0x64, 0x2e, 0xb3, 0x6, 0x2d, 0xbc, 0x9d, 0xca, 0xd8, 0x1, 0xc5, 0x65, 0x27, 0x6, 0x93, 0x99, 0xe7, 0xc4, 0x11, 0xad, 0x14, 0x28, 0x82, 0xf6}} }, + { (uint64_t)5000ull, {{0x61, 0x76, 0xac, 0x4a, 0xc0, 0x6, 0x5e, 0x49, 0xd6, 0xc4, 0x41, 0xcf, 0x40, 0x4f, 0xad, 0xda, 0xad, 0x44, 0x93, 0xe, 0xf0, 0x3c, 0x68, 0x9, 0xad, 0xd7, 0x77, 0xe4, 0x2f, 0xee, 0x7f, 0x10}} }, + { (uint64_t)6000ull, {{0x78, 0x79, 0x4, 0x65, 0xf6, 0x60, 0x5b, 0x5a, 0x84, 0x77, 0x36, 0x5a, 0xa6, 0xc2, 0xa4, 0xa5, 0x84, 0x91, 0xc, 0x23, 0x95, 0x2, 0x92, 0x97, 0x52, 0x49, 0xa1, 0xad, 0x7d, 0xf0, 0xf7, 0xe8}} }, + { (uint64_t)7000ull, {{0x20, 0xa5, 0x60, 0x6b, 0x60, 0x23, 0x95, 0xd6, 0x8e, 0x2f, 0xad, 0x8e, 0xc6, 0x7f, 0x92, 0xde, 0x89, 0xc6, 0x3e, 0x1e, 0x7f, 0xc1, 0xdd, 0x7f, 0x92, 0xff, 0xed, 0xb8, 0xf6, 0x55, 0xfb, 0xd}} }, + { (uint64_t)8000ull, {{0x9a, 0x78, 0x97, 0x43, 0x98, 0x65, 0x17, 0xd9, 0x5f, 0x4e, 0x80, 0x8b, 0xeb, 0xe6, 0x52, 0xd, 0xe6, 0xcf, 0x8c, 0x51, 0x35, 0xab, 0x36, 0x8, 0x7e, 0x87, 0xe2, 0x76, 0xac, 0x6a, 0x34, 0x1}} }, + { (uint64_t)9000ull, {{0x5f, 0xc7, 0xaa, 0x48, 0xbb, 0x19, 0x13, 0x58, 0xc7, 0xe3, 0x4d, 0x24, 0xcf, 0x9c, 0x31, 0x16, 0x74, 0x12, 0x7a, 0xb2, 0x45, 0xd0, 0x8f, 0x4e, 0x2c, 0xfd, 0xbf, 0x8f, 0x5, 0xc9, 0x5b, 0xf5}} }, + { (uint64_t)10000ull, {{0x61, 0x20, 0xe7, 0x76, 0xe9, 0x12, 0xab, 0x10, 0x5a, 0x49, 0xf9, 0xda, 0x2, 0xa6, 0x75, 0x17, 0xc0, 0xa9, 0xb, 0x2b, 0x3e, 0x2d, 0xa3, 0xd, 0xff, 0x34, 0x39, 0x93, 0xdb, 0xec, 0x95, 0x97}} }, + { (uint64_t)20000ull, {{0x77, 0xbf, 0xb5, 0x37, 0xac, 0xa, 0xbc, 0x41, 0xaa, 0x21, 0xd0, 0xec, 0xd9, 0x18, 0x13, 0x34, 0xd8, 0x6b, 0xa7, 0x86, 0x5a, 0x94, 0x47, 0xf5, 0xc1, 0x58, 0x9a, 0x81, 0xd7, 0xef, 0xb3, 0xbb}} }, + { (uint64_t)30000ull, {{0x35, 0xf4, 0x5, 0xa9, 0x5f, 0x75, 0x19, 0x2a, 0xe9, 0xc0, 0xd4, 0xf5, 0x88, 0x84, 0x47, 0x14, 0xf6, 0x85, 0x1b, 0x97, 0xce, 0xbd, 0x9f, 0x7c, 0x2, 0xc5, 0xdd, 0xd7, 0xbf, 0x58, 0xff, 0x31}} }, + { (uint64_t)40000ull, {{0x77, 0x55, 0xbb, 0x3f, 0x38, 0x7c, 0x21, 0xb8, 0xa0, 0xf4, 0x48, 0x1f, 0xbf, 0xa8, 0x8a, 0xbe, 0xee, 0xce, 0xc7, 0x56, 0x53, 0xfc, 0xa1, 0x89, 0x58, 0x39, 0xc1, 0xba, 0x6, 0x47, 0x9f, 0x96}} }, + { (uint64_t)50000ull, {{0x8b, 0x7e, 0x84, 0xa3, 0x37, 0xb7, 0xb9, 0xcd, 0x5d, 0xb3, 0x63, 0x33, 0x8, 0xad, 0x51, 0x86, 0xa3, 0x59, 0xd, 0xff, 0xb8, 0x23, 0x1e, 0x2f, 0x31, 0xfd, 0x20, 0x42, 0x54, 0x9f, 0xfb, 0xe2}} }, + { (uint64_t)60000ull, {{0xef, 0xfd, 0xa6, 0x25, 0x15, 0xea, 0xb1, 0xbc, 0x1e, 0xbd, 0x74, 0x92, 0x94, 0x9b, 0x1, 0x22, 0xc3, 0x9f, 0x71, 0xa, 0x65, 0x16, 0xec, 0x66, 0x8c, 0x37, 0x61, 0xe6, 0xcc, 0x36, 0x1f, 0x25}} }, + { (uint64_t)70000ull, {{0x16, 0xba, 0x89, 00, 0xf3, 0x6f, 0xf, 0x6c, 0x46, 0x1c, 0xb, 0xe7, 0x64, 0xae, 0xee, 0x48, 0x86, 0x6, 0xb0, 0x53, 0xed, 0xdc, 0x10, 0xb5, 0x9a, 0x3e, 0xde, 0xcd, 0x23, 0xd4, 0x4f, 0xc0}} }, + { (uint64_t)80000ull, {{0x4d, 0xd4, 0x70, 0x3b, 0x7b, 0x7f, 0xcf, 0xe7, 0x2a, 0x2e, 0x4f, 0x31, 0xa4, 0x34, 0x17, 0xf9, 0xc0, 0xda, 0x64, 0x2f, 0xd0, 0xa9, 0x29, 0xb8, 0xf5, 0xed, 0xd8, 0x3, 0x7f, 0x93, 0xc5, 0xb3}} }, + { (uint64_t)90000ull, {{0x8e, 0xfc, 0x3, 0x20, 0x40, 0xbd, 0x90, 0x41, 0xda, 0x3d, 0xb0, 0x9b, 0xa1, 0x3d, 0xa2, 0xa5, 0xd1, 0xb8, 0x12, 0x3, 0xa, 0x5a, 0x36, 0x7c, 0x58, 0x94, 0xbd, 0x54, 0x11, 0x9, 0xe7, 0x30}} }, + { (uint64_t)100000ull, {{0xbd, 0x2e, 0xb1, 0x97, 0x83, 0x57, 0x1c, 0xf2, 0x22, 0x2c, 0x81, 0xb, 0x69, 0xf, 0xc7, 0x66, 0x64, 0x57, 0xae, 0x20, 0x92, 0x5b, 0x90, 0x5, 0xce, 0xe6, 0x1d, 0xf2, 0x66, 0x6f, 0xdc, 0xb7}} }, + { (uint64_t)200000ull, {{0x83, 0xd4, 0xcd, 0xdd, 0xc1, 0x44, 0x87, 0x32, 0xf2, 0x97, 0x7c, 0x41, 0xaa, 0xa7, 0x1f, 0xe6, 0xde, 0x9c, 0x17, 0x6d, 0xa8, 0x99, 0xee, 0xbf, 0xfc, 0x1b, 0xb, 0xa9, 0xea, 0x92, 0x97, 0x90}} }, + { (uint64_t)300000ull, {{0xcc, 0xc0, 0x6b, 0x44, 0xc3, 0x1, 0x38, 0x6, 0x30, 0x45, 0xed, 0x1, 0xd2, 0x45, 0xd8, 0x14, 0x3, 0xb6, 0x36, 0x52, 0xeb, 0xc4, 0xf9, 0x96, 0x7f, 0xd, 0x7f, 0x38, 0x69, 0x7f, 0x46, 0x16}} }, + { (uint64_t)400000ull, {{0x1b, 0xbf, 0xe7, 0xe, 0xca, 0xf1, 0xdd, 0xd7, 0xf1, 0x2, 0x36, 0xf6, 0x8a, 0x41, 00, 0xb, 0x5d, 0xab, 0x2d, 0x47, 0x5c, 0xb9, 0x2f, 0x62, 0xc2, 0xd6, 0x84, 0xcf, 0x57, 0x69, 0xfb, 0x84}} }, + { (uint64_t)500000ull, {{0xf1, 0xb1, 0xcd, 0xaa, 0x78, 0x14, 0x95, 0x36, 0xf, 0x53, 0x31, 0x81, 0xaa, 0x58, 0xc8, 0xbd, 0xae, 0x6a, 0x77, 0x98, 0xd0, 0x2d, 0xab, 0x6d, 0x56, 0x26, 0x81, 0x27, 0x67, 0x9, 0xe7, 0x1}} }, + { (uint64_t)600000ull, {{0xd5, 0x26, 0x7d, 0x60, 0xd4, 0xfe, 0x9b, 0xc5, 0xfe, 0xfa, 0x7d, 0x3f, 0xe0, 0x7c, 0xd1, 0xfa, 0xd4, 0x55, 0x73, 0xd5, 0xae, 0x19, 0x10, 0xda, 0x7, 0x3e, 0x6d, 0x2d, 0xf9, 0xe2, 0x4, 0x39}} }, + { (uint64_t)700000ull, {{0xb, 0x58, 0x11, 0x25, 0xc2, 0xc4, 0x83, 0xc9, 0xa3, 0xd8, 0xbc, 0x8, 0x32, 0x2f, 0x26, 0xaa, 0x1f, 0xc5, 0xe, 0x41, 0x53, 0x2c, 0x1b, 0x9d, 0xf6, 0x26, 0xb0, 0x9, 0xd7, 0x88, 0x67, 0xcf}} }, + { (uint64_t)800000ull, {{0xf5, 0xb3, 0xd1, 0x8f, 0x66, 0xd0, 0xf9, 0x17, 0x5c, 0x30, 0x83, 0xb5, 0xf8, 0x7, 0x8e, 0xaf, 0xa8, 0x9e, 0xf8, 0x1d, 0xe7, 0x15, 0x8, 0xbc, 0x25, 0x1f, 0x5c, 0x5f, 0xe7, 0x25, 0x2e, 0x6}} }, + { (uint64_t)900000ull, {{0x1, 0xde, 0x40, 0x2c, 0x4b, 00, 0x43, 0x4, 0x2e, 0xae, 0x9e, 0xde, 0xa1, 0x49, 0x2b, 0x9d, 0x82, 0xb7, 0xbc, 0x36, 0x68, 0xe9, 0xb5, 0x84, 0xb0, 0x31, 0x3d, 0x44, 0x50, 0x53, 0x40, 0x74}} }, + { (uint64_t)1000000ull, {{0x53, 0x4b, 0x85, 0xc7, 0x89, 0x3f, 0x66, 0xf0, 0x26, 0xb6, 0x5e, 0xd7, 0xe7, 0xa4, 0xb8, 0xc9, 0xf4, 0xb, 0xe3, 0x1b, 0xcd, 0xa, 0x3d, 0xcd, 0x27, 0xc4, 0x71, 0x2, 0x56, 0x51, 0x65, 0x3}} }, + { (uint64_t)2000000ull, {{0xcd, 0xb5, 0xda, 0xfa, 0x53, 0x10, 0xf5, 0x26, 0x2f, 0xfc, 0x9, 0x26, 0xd0, 0xdf, 0x6e, 0xeb, 0xee, 0x2d, 0x52, 0xa9, 0x8d, 0xc6, 0x9f, 0xd, 0xc5, 0xe4, 0xeb, 0xf0, 0xc1, 0xa8, 0x77, 0x2e}} }, + { (uint64_t)3000000ull, {{0x9e, 0x75, 0x63, 0xf0, 0x33, 0x59, 0xea, 0x31, 0xe2, 0x91, 0xe7, 0xf0, 0xb8, 0x74, 0x17, 0xbc, 0xf5, 0xb2, 0x34, 0xee, 0x8b, 0x7e, 0x5b, 0x4, 0x41, 0x73, 0xbf, 00, 0x46, 0x86, 0x7c, 0x57}} }, + { (uint64_t)4000000ull, {{0xfd, 0x2a, 0xeb, 0xd, 0x5e, 0xe5, 0x3b, 0x77, 0xf2, 0xb1, 0xe3, 0xac, 0x75, 0x2d, 0x19, 0x38, 0x9f, 0xc5, 0xba, 0xa0, 0xf8, 0xd7, 0x64, 0x48, 0xa5, 0x9f, 0x99, 0x85, 0xa4, 0x8d, 0xa, 0x25}} }, + { (uint64_t)5000000ull, {{0xc0, 0xbe, 0x4f, 0xb8, 0x77, 0xb9, 0xce, 0x50, 0x87, 0x71, 0x32, 0x3b, 0xcf, 0x1f, 0xb9, 0x48, 0x47, 0x10, 0xee, 0x23, 0x2, 00, 0x6, 0xc3, 0xe8, 0xca, 0xac, 0x6e, 0x4f, 0x2, 0xfa, 0xbf}} }, + { (uint64_t)6000000ull, {{0xfc, 0x44, 0x5c, 0xa3, 0x84, 0xf3, 0x3e, 0x55, 0x8d, 0xc1, 0x56, 0x44, 0x9d, 0x3f, 0xba, 0x6a, 0xfd, 0x54, 0xc3, 0x42, 0xe6, 0x35, 0x11, 0xf, 0xe7, 0x9c, 0x16, 0xc7, 0x17, 0xf7, 0xd4, 0xf7}} }, + { (uint64_t)7000000ull, {{0xd8, 0x9, 0x2b, 0x8d, 0x45, 0xdb, 0x54, 0xa5, 0x6d, 0x64, 0xe8, 0x9, 0x4a, 0x6, 0x22, 0xe2, 0x6e, 0x8a, 0x2e, 0xec, 0xb9, 0x3, 0xb2, 0xe1, 0xf7, 0x5a, 0x83, 0x7b, 0x3a, 0xd8, 0x55, 0x4a}} }, + { (uint64_t)8000000ull, {{0x10, 0x4, 0x5c, 0x91, 0xdb, 0xad, 0x8a, 0x6a, 0x81, 0x62, 0x4a, 0xe0, 0xcf, 0x20, 0x5d, 0xb9, 0x97, 0x3e, 0xe8, 0x42, 0x3e, 0x97, 0xaf, 0x58, 0xa6, 0x1c, 0xfa, 0x7a, 0x78, 0x66, 0xf4, 0x1}} }, + { (uint64_t)9000000ull, {{0x11, 0x5c, 0x20, 0x9e, 0xe1, 0xde, 0xf3, 0x10, 0xce, 0xc9, 0xa6, 0xd1, 0x6c, 0xe6, 0x27, 0xec, 0xbd, 0xb9, 0xff, 0x2c, 0x23, 0x9, 0x3c, 0x24, 0xc8, 0x6c, 0x1b, 0xf2, 0x50, 0xd4, 0xb5, 0x85}} }, + { (uint64_t)10000000ull, {{0x2f, 0x99, 0xd9, 0x74, 0x44, 0x18, 0x66, 0x9, 0x49, 0xba, 0x43, 0x35, 0x61, 0xc6, 0x5, 0xb6, 0xf7, 0xbe, 0x8f, 0x82, 0xa, 0x93, 0xcb, 0x2a, 0xed, 0xa9, 0x7c, 0x87, 0x32, 0x92, 0x56, 0x49}} }, + { (uint64_t)20000000ull, {{0xc6, 0x77, 0x1f, 0xab, 0x14, 0xb, 0x75, 0xf4, 0xef, 0xd0, 0x97, 0xfc, 0xe1, 0x82, 0x6b, 0x80, 0xba, 0xe3, 0x16, 0xbc, 0xec, 0x28, 0x86, 0x9b, 0x3a, 0x1b, 0xf1, 0xbc, 0x6e, 0x4d, 0x20, 0x43}} }, + { (uint64_t)30000000ull, {{0xa1, 0xe9, 0xed, 0x3e, 0xf6, 0x5a, 0x9d, 0x52, 0x6c, 0xc2, 0x62, 0x5, 0x88, 0x12, 0x1, 0xd8, 0xa8, 0xf2, 0xc4, 0x40, 0x9f, 0xa3, 0x64, 0x10, 0x72, 0x96, 0xb9, 0xf9, 0x6a, 0x61, 0xb3, 0x58}} }, + { (uint64_t)40000000ull, {{0xb5, 0x31, 0x2d, 0xc7, 0x72, 0x94, 0xab, 0x9b, 0xc8, 0xbf, 0xd1, 0x39, 0x1e, 0x9a, 0xca, 0x92, 0x45, 0xe2, 0x28, 0xf7, 0x4b, 0x49, 0x74, 0xfc, 0x29, 0xad, 0x1c, 0x31, 0xcb, 0xe3, 0xe6, 0xa3}} }, + { (uint64_t)50000000ull, {{0xb8, 0xab, 0xc9, 0xff, 0xf6, 0x84, 0x1d, 0x2e, 0xa0, 0x13, 0x5a, 0x21, 0x72, 0xd3, 0xa7, 0xb, 0xfc, 0x2b, 0x70, 0x22, 0x8, 0xcd, 0x4a, 0x43, 0xc6, 0x30, 0xbe, 0xb1, 0xb8, 0xa0, 0x32, 0x8b}} }, + { (uint64_t)60000000ull, {{0x63, 0x90, 0xe1, 0xdb, 0x81, 0xb0, 0xea, 0x5c, 0xe2, 0x73, 0x94, 0x14, 0xe5, 0x2b, 0x7, 0x98, 0xd8, 0x2e, 0xb8, 0xe9, 0xae, 0xc5, 0x6d, 0xfe, 0x7e, 0x2c, 0x64, 0x11, 0xab, 0x79, 0x41, 0x87}} }, + { (uint64_t)70000000ull, {{0x7e, 0x51, 0xaf, 0xee, 0x5b, 0xc9, 0x71, 0x52, 0x9d, 0x64, 0x4d, 0xcd, 0x7f, 0x2a, 0x2a, 0xb0, 0x26, 0x69, 0xce, 0x2c, 0xb5, 0x7, 0xa6, 0x2d, 0xfc, 0x93, 0x17, 0x6c, 0xb6, 0xdf, 0x41, 0x38}} }, + { (uint64_t)80000000ull, {{0xf3, 0x7b, 0x94, 0x6b, 0x8b, 0x24, 0x88, 0xeb, 0xee, 0x1c, 0x6, 0xc1, 0x27, 0xfb, 0xe5, 0xfa, 0x5e, 0xfd, 0x62, 0x36, 0x9d, 0xd5, 0xaa, 0xda, 0xed, 0xd8, 0x88, 0x50, 0x1d, 0x3b, 0x7e, 0x3b}} }, + { (uint64_t)90000000ull, {{0x46, 0xcb, 0x76, 0x57, 0xf6, 0x1c, 0x83, 0x7c, 0xec, 0x80, 0x74, 0xbb, 0xb0, 0xf5, 0x2e, 0x7f, 0xc5, 0x9a, 0xd, 0x94, 0xe0, 0x17, 00, 0x9a, 0xbe, 0x25, 0x65, 0x2e, 0x4a, 0xd2, 0xe5, 0x3d}} }, + { (uint64_t)100000000ull, {{0x66, 0x7b, 0x8e, 0x6f, 0x6a, 0x4b, 0x91, 0x89, 0x76, 0xd9, 0x73, 0x5a, 0x43, 0x36, 0x7d, 0xc7, 0x59, 0x2c, 0x87, 0xd0, 0xa1, 0xf8, 0x15, 0xc6, 0xe8, 0x7d, 0xf1, 0x1a, 0x13, 0x50, 0x9f, 0xb2}} }, + { (uint64_t)200000000ull, {{0x3b, 0xcb, 0x51, 0x48, 0x1, 0x64, 0x1b, 0x62, 0x55, 0x93, 0x8c, 0xc5, 0x3, 0x76, 0x2d, 0x35, 0xce, 0x6, 0xd7, 0x5f, 0xe9, 0x50, 0x95, 0x9a, 0x1a, 0xab, 0x21, 0x4b, 0x50, 0x9b, 0x10, 0xb}} }, + { (uint64_t)300000000ull, {{0xa5, 0x92, 0x6f, 0x3, 0x1e, 0x6b, 0x15, 0xeb, 0x86, 0x23, 0x51, 0x8, 0xab, 0xb1, 0xaf, 0x90, 0xc5, 0xb1, 0x62, 0xc3, 0x99, 0x8c, 0x8b, 0xbb, 0x3f, 0xfb, 0xb0, 0x72, 0x9d, 0xa9, 0x45, 0x7b}} }, + { (uint64_t)400000000ull, {{0xfe, 0x35, 0xb6, 0x99, 0x44, 0x41, 0xe, 0xaf, 0x81, 0x5b, 0xdc, 0xd0, 0xa4, 0xd7, 0x1e, 0xf9, 0xfc, 0x66, 0x86, 0x48, 0xad, 0x43, 0x74, 0x3b, 0x3, 0x5a, 0xed, 0x2c, 0x17, 0xc1, 0x38, 0x7a}} }, + { (uint64_t)500000000ull, {{0x22, 0x22, 0xd6, 0x70, 0xb8, 0x7d, 0x9b, 0x47, 0xb8, 0xb9, 0x5c, 0x8c, 0x39, 0x7b, 0xc5, 0x2e, 0x2b, 0x46, 0xa6, 0x48, 0xb0, 0x2, 0xa0, 0x48, 0x5a, 0x37, 0x5c, 0xd8, 0x1f, 0x4a, 0x54, 0x5f}} }, + { (uint64_t)600000000ull, {{0xd3, 0x23, 0x8a, 0x4a, 0x8b, 0x71, 0xab, 0x46, 0xd1, 0x53, 0x4, 0xac, 0xfa, 0x2f, 0x40, 0xbf, 0x5e, 0xa6, 0x3b, 0x3d, 0x86, 0x4a, 0x79, 0xfa, 0x84, 0x25, 0xd2, 0x65, 0x5a, 0xe7, 0x7, 0x6f}} }, + { (uint64_t)700000000ull, {{0xa8, 0xff, 0x28, 0x3f, 0xcf, 0xf0, 0x53, 0xd3, 0x44, 0xc8, 0xf7, 0x56, 0x4f, 0x40, 0x24, 0xb6, 0x6b, 0xfa, 0x45, 0x9f, 0x47, 0x6f, 0xd, 0x73, 0xc, 0x91, 0x39, 0x90, 0x8b, 0x2d, 0x64, 0x7e}} }, + { (uint64_t)800000000ull, {{0xf2, 0xda, 0xf8, 0x88, 0xc4, 0x46, 0x57, 0x1, 0xc0, 0xe6, 0x1e, 0x12, 0xc3, 0xfb, 0xd4, 0xea, 0x79, 0xc7, 0xec, 0xb4, 0xf0, 0xc4, 0xb1, 0x54, 0xc5, 0x1a, 0x24, 0xd1, 0xe9, 0x21, 0x28, 0xba}} }, + { (uint64_t)900000000ull, {{0x11, 0x6a, 0xe5, 0xd2, 0x9c, 0xec, 0x72, 0xaa, 0xc5, 0x57, 0xcb, 0x14, 0xe2, 0xcd, 0xd5, 0x53, 0xe5, 0x88, 0xff, 0x8b, 0x81, 0x78, 0x26, 0x1, 0x99, 0xc4, 0xc, 0xae, 0xa2, 0x12, 0xcb, 0x63}} }, + { (uint64_t)1000000000ull, {{0x8c, 0xe6, 0x48, 0x33, 0xce, 0xc9, 00, 0xcb, 0x6d, 0x5a, 0xc4, 0x6f, 0xc0, 0x23, 0x7d, 0x8f, 0x24, 0x39, 0xc3, 0xdf, 0xa2, 0x38, 0xba, 0xf9, 0xcc, 0x94, 0x16, 0x6a, 0xd2, 0xe8, 0x98, 0x87}} }, + { (uint64_t)2000000000ull, {{0x37, 0x8d, 0x3c, 0x5d, 0xbb, 0xa4, 0x82, 0x3d, 0x33, 0x12, 0xbb, 0x61, 0xfc, 0x6, 0x75, 0xa1, 0xbb, 0x39, 0x89, 0xf3, 0x97, 0x1, 0xeb, 0xd, 0x5c, 0xe4, 0xde, 0x5b, 0xd, 0x90, 0x74, 0x72}} }, + { (uint64_t)3000000000ull, {{0x7f, 0xa2, 0xd0, 0xa5, 0x99, 0xe7, 0x97, 0x2e, 0x74, 0xcb, 0x75, 0xf9, 0x8a, 0xf4, 0x84, 0xfc, 0x85, 0x19, 0xcb, 0x7e, 0x25, 0xb9, 0x84, 0xa7, 0x6d, 0x8b, 0xc2, 0xba, 0x8d, 0xaf, 0xde, 0xd8}} }, + { (uint64_t)4000000000ull, {{0xda, 0x3a, 0xcb, 00, 0xab, 0x2d, 0x8d, 0xcc, 0xac, 0xec, 0x8f, 0x77, 0x59, 0x21, 0xc4, 0xe, 0x26, 0xb1, 0xff, 0xbe, 0xca, 0x9e, 0xb7, 0xe6, 0x57, 0x25, 0x6f, 0x59, 0x68, 0xf2, 0x34, 0x1c}} }, + { (uint64_t)5000000000ull, {{0x34, 0x6, 0xd7, 0x9a, 0x50, 0xd8, 0x14, 0xa9, 0xcc, 0xed, 0x3b, 0x24, 0x4, 0xed, 0x3e, 0x1b, 0x8d, 0xa6, 0x21, 0x98, 0x8c, 0x43, 0xb1, 0x93, 0x69, 0x42, 0xf4, 0x94, 0xa, 0xc5, 0xbf, 0x6a}} }, + { (uint64_t)6000000000ull, {{0xf8, 0x3e, 0xe8, 0xc1, 0x62, 0xfc, 0x52, 0xa0, 0x8, 0x9f, 0x46, 0xe8, 0x29, 0xc2, 0xea, 0xf6, 0xa1, 0x9f, 0xd5, 0x96, 0xcd, 0x12, 0xb3, 0xe8, 0x19, 0xd5, 0x67, 0x69, 0x44, 0xf, 0x7b, 0x4e}} }, + { (uint64_t)7000000000ull, {{0x8c, 0x72, 0x7d, 0x24, 0x57, 0xf3, 0x4b, 0x2f, 0xdb, 0x6a, 0xdf, 0x69, 0x1a, 0xb3, 0x5f, 0xaa, 0xe4, 0xff, 0x23, 0x4c, 0x28, 0xb4, 0x4e, 0x9f, 0xd3, 0x71, 0x8e, 0xef, 0xec, 0x41, 0x75, 0x80}} }, + { (uint64_t)8000000000ull, {{0x4a, 0x2e, 0x2f, 0x76, 0xe3, 0x5d, 0xcb, 0xa8, 0x97, 0xa3, 0xae, 0x72, 0xc4, 0x27, 0xd, 0x9c, 0x13, 0x17, 0x14, 0xed, 0x19, 0x1b, 0x55, 0x5c, 0x5e, 0x1, 0xe4, 0x75, 0x7c, 0xba, 0xe7, 0x2c}} }, + { (uint64_t)9000000000ull, {{0x3f, 0x9f, 0xc, 0x4, 0xc0, 0xb9, 0xec, 0x9b, 0x4d, 0x11, 0x7c, 0x5f, 0xc9, 0xf1, 0x8a, 0x20, 0xf2, 0xb3, 0xfa, 0xcc, 0xa4, 0xc8, 0xae, 0x41, 0xaf, 0x7c, 0x8, 0xe9, 0xe0, 0xef, 0xb9, 0x81}} }, + { (uint64_t)10000000000ull, {{0x97, 0xc9, 0x2a, 0x29, 0x1, 0x5e, 0xcb, 0x49, 0xf8, 0x9, 0x5, 0x45, 0xe0, 0x1f, 0xf9, 0x78, 0x6c, 0xae, 0x40, 0x57, 0x73, 0x47, 0x61, 0x18, 0x24, 0xf4, 0xb6, 0x59, 0x9f, 0xf5, 0xd3, 0x64}} }, + { (uint64_t)20000000000ull, {{0x9, 0xba, 0xed, 0x9a, 0x3c, 0x44, 0xb2, 0x22, 0x85, 0xa0, 0xae, 0xa4, 0x14, 0x8c, 0xa7, 0xde, 0x9b, 0xea, 0x96, 0x3c, 0xf6, 0x96, 0x23, 0xb6, 0x83, 0x44, 0x5c, 0xa, 0x10, 0xa5, 0x86, 0x77}} }, + { (uint64_t)30000000000ull, {{0x45, 0xac, 0xaf, 0x1d, 0xe2, 0x89, 0x6d, 0xe8, 0x72, 0x84, 0xff, 0xed, 0x57, 0x8b, 0x77, 0x14, 0xf5, 0x18, 0xa6, 0x18, 0xe2, 0xae, 0x6f, 0x90, 0xae, 0x4f, 0x70, 0x13, 0xa2, 0x8e, 0x99, 0xe0}} }, + { (uint64_t)40000000000ull, {{0x8, 0xb8, 0x47, 0x36, 0x42, 0x24, 0xe2, 0x9c, 0xe3, 0x36, 0x63, 0x93, 0xc2, 0xe1, 0x1e, 0xfc, 0x75, 0x55, 0xde, 0xe1, 0xa0, 0x5f, 0x91, 0xa7, 0x2e, 0x61, 0x11, 0x76, 0x84, 0xdd, 0xbe, 0x29}} }, + { (uint64_t)50000000000ull, {{0x6c, 0x8e, 0xe, 0x4a, 0x63, 0x4f, 0x85, 0x9a, 0x31, 0xab, 0x2f, 0x7a, 0x78, 0xc0, 0xc4, 0xa5, 0x93, 0x8c, 0xb7, 0x7f, 0x3, 0x35, 0x50, 0xa4, 0x7d, 0x7e, 0x31, 0x81, 0xb6, 0xb2, 0x6e, 0xc0}} }, + { (uint64_t)60000000000ull, {{0x66, 0xc2, 0xa0, 0x9, 0x65, 0xf9, 0xbf, 0xcb, 0xb1, 0x1e, 0xa0, 0x3c, 0xf1, 0xd6, 0x31, 0xb0, 0xe, 0x8a, 0x1e, 0xf7, 0xa6, 0xb, 0x1b, 0xe4, 0xa5, 0xac, 0x9, 0x23, 0xb, 0xf8, 0x17, 0x3f}} }, + { (uint64_t)70000000000ull, {{0x63, 0x51, 0xd7, 0x74, 0xc0, 0x2c, 0x5a, 0x9d, 0xee, 0xcf, 0xdb, 0xab, 0x70, 0x96, 0x68, 0x59, 0x8c, 0x47, 0xe4, 0xb1, 0x78, 0x2c, 0xe5, 0xae, 0x31, 0x6a, 0xf7, 0x40, 0xa6, 0x6f, 0x7e, 0x30}} }, + { (uint64_t)80000000000ull, {{0x5a, 0xcc, 0xfd, 0x16, 0x22, 0x79, 0xa5, 0x1c, 0x8b, 0x3b, 0xd5, 0xd3, 0x67, 0x9e, 0x91, 0x89, 0x67, 0xa2, 0x64, 0xea, 0x6, 0x3d, 0x37, 0xdf, 0xf5, 0xe3, 0x45, 0x7e, 0xc3, 0x7, 0xd4, 0x57}} }, + { (uint64_t)90000000000ull, {{0xb7, 0x47, 0xfc, 0x1, 0xc6, 0xf0, 0xc7, 0x49, 0x67, 0x3a, 0x29, 0x10, 0x25, 0xc, 0x2e, 0x23, 0xcb, 0x38, 0x27, 0x4d, 0x63, 0xb4, 0x2f, 0x52, 0x1b, 0x84, 0x63, 0x56, 0xe4, 0x13, 0x61, 0x8f}} }, + { (uint64_t)100000000000ull, {{0x9, 0x42, 0x84, 0x3d, 0x6f, 0x69, 0xe1, 0xcf, 0x3d, 0x99, 0xc9, 0x9f, 0xc, 0x97, 0xc0, 0xe6, 0xe5, 0x78, 0x93, 0x5a, 0xf6, 0xa8, 0xbd, 0xb8, 0xf8, 0x1d, 0x5b, 0x90, 0xbd, 0xe7, 0xcc, 0x10}} }, + { (uint64_t)200000000000ull, {{0x56, 0x4c, 0x64, 0xea, 0x50, 0xe4, 0xbd, 0x20, 0xdb, 0x58, 0x5d, 0xb5, 0x87, 0xb1, 0xf7, 0x64, 0xa2, 0x62, 0xd8, 0x46, 0xa6, 0xb0, 0xa2, 0x4b, 0x43, 0x27, 0x60, 0xd2, 0xf9, 0xde, 0x66, 0x5b}} }, + { (uint64_t)300000000000ull, {{0xac, 0x65, 0x83, 0x41, 0x5b, 0xd6, 0x4c, 0x3, 0x35, 0x97, 0xf9, 0x28, 0xa4, 0xb5, 0xd4, 0xf4, 0x78, 0x9e, 0xa8, 0xb2, 0x87, 0x82, 0x73, 0x89, 0xa8, 0x1e, 0xb6, 0x62, 0x9e, 0xc5, 0xb8, 0x50}} }, + { (uint64_t)400000000000ull, {{0x52, 0xf4, 0x9d, 0x89, 0xcf, 0x74, 0x13, 0x2f, 0xc7, 0x43, 0x2e, 0x6a, 0x6b, 0xef, 0xcf, 0xf3, 0xfd, 0x13, 0xd6, 0x3b, 0x51, 0x60, 0xab, 0x1c, 0xe6, 0x4a, 0xb0, 0xd1, 0x21, 0xcd, 0xa9, 0x9a}} }, + { (uint64_t)500000000000ull, {{0xe9, 0xaa, 0x7c, 0x81, 0xcd, 0xb5, 0xb3, 0x14, 0x8f, 0xb7, 0x62, 0x80, 0x63, 0xcd, 0x7a, 0x7, 0xd1, 0xad, 0xd1, 0x64, 0x3c, 0xed, 0xd3, 0xfa, 0x34, 0x47, 0x9d, 0x85, 0x9c, 0xc5, 0x62, 0x65}} }, + { (uint64_t)600000000000ull, {{0x98, 0x27, 0xae, 0x31, 0xe5, 0xc2, 0xa7, 0x78, 0x39, 0xf6, 0xb, 0x83, 0xab, 0x45, 0x78, 0xe2, 0xa0, 0x1e, 0xfa, 0x4b, 0x3b, 0x14, 0xcc, 0x72, 0x73, 0x14, 0xff, 0xd7, 0x15, 0x53, 0x63, 0xbf}} }, + { (uint64_t)700000000000ull, {{0x72, 0x91, 0x6a, 0x79, 0x27, 0xff, 0x13, 0x24, 0xd4, 0x98, 0x40, 0xec, 0xc0, 0x98, 0x68, 0xb8, 0xf3, 0x15, 0xe4, 0xf1, 0xf6, 0xd4, 0x45, 0x8d, 0x37, 0x5e, 0xc7, 0x45, 0xfc, 0x2e, 0x63, 0x53}} }, + { (uint64_t)800000000000ull, {{0x66, 0x76, 0xe0, 0x4, 0xf, 0xa4, 0xb8, 0x22, 0x9c, 0x61, 0x69, 0xc, 0x71, 0x32, 0x22, 0xcf, 0x3d, 0x37, 0xb9, 0x49, 0x3b, 0x49, 0x6, 0x80, 0xbb, 0x48, 0xd8, 0xd5, 0x1a, 0xde, 0x95, 0xf2}} }, + { (uint64_t)900000000000ull, {{0x41, 0x54, 0xb3, 0x46, 0x5a, 0x43, 0x72, 0x67, 0x1e, 0xa9, 0xe0, 0x64, 0xa7, 0xca, 0xa6, 0x6e, 0x14, 0xb4, 0x98, 0x6a, 0x46, 0x68, 0x91, 0x8a, 0xfa, 0x57, 0x9b, 0xf1, 0xed, 0x25, 0x6, 0xdd}} }, + { (uint64_t)1000000000000ull, {{0xbb, 0x6f, 0x70, 0x62, 0xca, 0x30, 0x6d, 0x67, 0x2a, 0x73, 0xe, 0x2a, 0x2f, 0x21, 0x9b, 0xdb, 0xe4, 0xc, 0x9f, 0xb3, 0xfe, 0x4d, 0x60, 0x13, 0x69, 0x2a, 0xf9, 0x3c, 0xdb, 0x2e, 0xc, 0xd1}} }, + { (uint64_t)2000000000000ull, {{0xbc, 0xe, 0xae, 0x5b, 0x9c, 0x6a, 0xd6, 0x38, 0x7a, 0x41, 0x19, 0x3c, 0x46, 0xf3, 0xc1, 0xd0, 0x71, 0x6d, 0x77, 0xd6, 0x4e, 0x22, 0xb2, 0xe0, 0x7b, 0x4b, 0xce, 0x75, 0x67, 0x65, 0xa2, 0xb}} }, + { (uint64_t)3000000000000ull, {{0x10, 0xc, 0x6f, 0x13, 0x42, 0xb7, 0x1b, 0x73, 0xed, 0xdd, 0xc5, 0x49, 0x2b, 0xe9, 0x23, 0x18, 0x2f, 00, 0xa6, 0x83, 0x48, 0x8e, 0xc3, 0xa2, 0xa1, 0xc7, 0xa9, 0x49, 0xcb, 0xe5, 0x77, 0x68}} }, + { (uint64_t)4000000000000ull, {{0x41, 0x9f, 0x7c, 0x94, 0x91, 0x4, 0x34, 0xf, 0xd3, 0xce, 0x85, 0x94, 0x8d, 0x2e, 0xf9, 0xf0, 0xdd, 0x4b, 0xb3, 0xd9, 0x2f, 0x5a, 0x78, 0x2c, 0x5f, 0x78, 0x4, 0xb7, 0x52, 0x9a, 0x13, 0xc6}} }, + { (uint64_t)5000000000000ull, {{0x40, 0x65, 0x34, 0x98, 0xbe, 0xa0, 0x22, 0xe3, 0x36, 0x5a, 0x3, 0xe5, 0x75, 0x25, 0xba, 0x65, 0x96, 0x53, 0x76, 0x24, 0x4f, 0xff, 0x10, 0x73, 0xe, 0xd9, 0x7a, 0x73, 0xb7, 0x53, 0x1, 0x91}} }, + { (uint64_t)6000000000000ull, {{0xdb, 0x1c, 0x7c, 0xf6, 0x8, 0x91, 0xf9, 0x65, 0xeb, 0xa9, 0xc6, 0x2, 0x24, 00, 0x63, 0xe, 00, 0x47, 0x95, 0x34, 0xe6, 0xf5, 0xb5, 0x33, 0xdc, 0xfc, 0x83, 0x19, 0x38, 0x52, 0x2c, 0x78}} }, + { (uint64_t)7000000000000ull, {{0x59, 0xa0, 0x3a, 0x31, 0x53, 0xa9, 0x94, 0xd7, 0x23, 0x27, 0xe4, 0xd9, 0x24, 0x21, 0xd3, 0xe3, 0x29, 0x1b, 0x1f, 0xa1, 0xb2, 0x40, 0xde, 0x44, 0xb9, 0x2d, 0x7f, 0x62, 0xec, 0x1, 0x28, 0xf1}} }, + { (uint64_t)8000000000000ull, {{0xb2, 0x80, 0xb9, 0x3b, 0x1e, 0x43, 0x88, 00, 0x73, 0xea, 0x4a, 0xa0, 0xef, 0x11, 0x4, 0xf8, 0x24, 0xbd, 0x12, 0x7a, 0x4a, 0x3d, 0xa2, 0x13, 0x92, 0x65, 0xf, 0xe8, 0xc6, 0x55, 0xb6, 0xc5}} }, + { (uint64_t)9000000000000ull, {{0xda, 0xf0, 0xd3, 0xe9, 0x32, 0x17, 0xd8, 0xe9, 0x5a, 0xbf, 0xdd, 0xf1, 0x3b, 0x7f, 0xd4, 0x8e, 0x34, 0x47, 0xad, 0x9, 0x23, 0x26, 0xb8, 0x99, 0xed, 0x58, 0x1f, 0xd5, 0xf8, 0x6, 0xc5, 0x6}} }, + { (uint64_t)10000000000000ull, {{0x16, 0x3d, 0xd6, 0x82, 0xec, 0x97, 0x7c, 0xdd, 0xa5, 0x95, 0x31, 0xda, 0x3f, 0xfa, 0x72, 0x99, 0x8a, 0x6f, 0x88, 0x37, 0xab, 0xad, 0xc6, 0x36, 0xaa, 0xed, 0xc8, 0xbe, 0x19, 0xb2, 0xd7, 0xc7}} }, + { (uint64_t)20000000000000ull, {{0x2, 0xfa, 0x35, 0x3a, 0xa8, 0x4e, 0xa8, 0xc4, 0x4c, 0x80, 0x23, 0x6, 0x5d, 0x79, 0x41, 0x60, 0x6b, 0x1f, 0xa5, 0xc2, 0x64, 0xdc, 0xcf, 0x46, 0xdc, 0x64, 0x94, 0xeb, 0xe9, 0x60, 0x6f, 0x20}} }, + { (uint64_t)30000000000000ull, {{0x87, 0x5, 0xd, 0xab, 0xf5, 0xb2, 0x3e, 0x8b, 0x79, 0x81, 0x3f, 0x4e, 0xd7, 0x6a, 0xa4, 0xad, 0xd2, 0x25, 0xdd, 0x2a, 0x50, 0x89, 0xaf, 0x6, 0x7d, 0xa7, 0x7c, 0xcb, 0x6e, 0xc5, 0x59, 0x46}} }, + { (uint64_t)40000000000000ull, {{0xaa, 0xe6, 0xb2, 0xc8, 0xa2, 0x9e, 0x4d, 0xbc, 0x63, 0x76, 0xc1, 0x72, 0x5, 0xfb, 0x2, 0x85, 0xe7, 0xd7, 0xd3, 0x25, 0x32, 0x3c, 0xd5, 0x26, 0xf, 0x98, 0xad, 0xff, 0xf7, 0xd4, 0xd4, 0xfb}} }, + { (uint64_t)50000000000000ull, {{0x9d, 0x79, 0x28, 0x82, 0x12, 0xa1, 0xe2, 0x3c, 0x9, 0x9f, 0xb2, 0xd8, 0xf0, 0xd0, 0xdb, 0xd3, 0xc2, 0xec, 0xd7, 0x58, 0xb9, 0xe6, 0xb5, 0xb4, 0xf2, 0x90, 0x60, 0x7, 0x9f, 0x19, 0x66, 0x9f}} }, + { (uint64_t)60000000000000ull, {{0x18, 0x90, 0x10, 0x6f, 0x1b, 0x97, 0xbc, 0x2d, 0xa, 0xe3, 0x96, 0xe5, 0xe5, 0x5e, 0xbf, 0xcc, 0x8e, 0xf6, 0x91, 0x7f, 0xb1, 0x96, 0xcb, 0x2b, 0x1e, 0x80, 0x25, 0x5d, 0x54, 0xb6, 0x87, 0x10}} }, + { (uint64_t)70000000000000ull, {{0x57, 0xd3, 0x4e, 0xf7, 0x54, 0x3b, 0xe4, 0x7b, 0x7b, 0xf4, 0x97, 0xce, 0x4a, 0x17, 0x6e, 0x78, 0xc6, 0xd6, 0x5c, 0xd3, 0x27, 0xf6, 0x4b, 0xa7, 0x5c, 0x27, 0xd1, 0x57, 0xb3, 0x37, 0x12, 0x5d}} }, + { (uint64_t)80000000000000ull, {{0x5e, 0xcb, 0x10, 0x15, 0x4b, 0x96, 0xca, 0xb5, 0x5e, 0x9, 0x46, 0x83, 0xf8, 0xdb, 0xff, 0x7f, 0x56, 0x63, 0x5f, 0xa6, 0x64, 0x97, 0xee, 0x9e, 0x24, 0xe, 0x83, 0x63, 0x7c, 0x7c, 0x87, 0x72}} }, + { (uint64_t)90000000000000ull, {{0x42, 0x32, 0x69, 0x98, 0x51, 0x30, 0xf1, 0x66, 0x51, 0x6a, 0x5b, 0xa8, 0x61, 0x9, 0x6d, 0x72, 0xec, 0xcc, 0x67, 0xad, 0xab, 0xa4, 0x5e, 0xb3, 0x73, 0x9a, 0xe, 0xbc, 0x61, 0xa3, 0x20, 0xae}} }, + { (uint64_t)100000000000000ull, {{0xa8, 0xd1, 0x60, 0x95, 0x91, 0x49, 0x8f, 0xa7, 0xc2, 0x94, 0x27, 0xad, 0x89, 0x31, 0xaf, 0x36, 0xc5, 0x2d, 0xc9, 0x7b, 0x4a, 0x11, 0xe7, 0x47, 0xa9, 0x56, 0xc2, 0x8c, 0x42, 0x54, 0xcf, 0xd4}} }, + { (uint64_t)200000000000000ull, {{0x23, 0x14, 0x49, 00, 0xa8, 0x66, 0xe8, 0xc1, 0xbf, 0x40, 0x98, 0xda, 0xa9, 0x48, 0xb9, 0x86, 0xf3, 0x84, 0xe, 0x5a, 0x7d, 0x21, 0x5e, 0xf0, 0xd5, 0x64, 0xef, 0xd8, 0xbe, 0xc6, 0x83, 0x15}} }, + { (uint64_t)300000000000000ull, {{0x6a, 0x51, 0x47, 0x3c, 0x86, 0xed, 0xad, 0x53, 0x51, 0x4b, 0x3f, 0x95, 0x97, 0xed, 0x21, 0xae, 00, 0x81, 0x51, 0xa0, 0x9e, 0x43, 0xad, 0xdd, 0x45, 0xd1, 0x74, 0x63, 0xc5, 0x34, 0x3, 0x97}} }, + { (uint64_t)400000000000000ull, {{0x8, 0xbd, 0xd4, 0xc3, 0xe4, 0x53, 0x1b, 0x29, 0x7a, 0x70, 00, 0x1e, 0xb8, 0xa4, 0xf1, 0x98, 0xdc, 0x3b, 0xd4, 0xf1, 0xf5, 0x60, 0x9a, 0xda, 0x98, 0xf6, 0xd9, 0x5f, 0x9a, 0x1a, 0x30, 0x2e}} }, + { (uint64_t)500000000000000ull, {{0x97, 0x55, 0x70, 0xea, 0x12, 0xde, 0x5a, 0xf5, 0xc5, 0x36, 0xbd, 0xb6, 0x83, 0x54, 0xfb, 0xc8, 0x32, 0x21, 0x50, 0xfc, 0x56, 0x83, 0x7c, 0x4b, 0x78, 0xa9, 0x85, 0x76, 0x5d, 0x2a, 0x70, 0x99}} }, + { (uint64_t)600000000000000ull, {{0xa7, 0xa6, 0x39, 0x93, 0x41, 0xcb, 0x4d, 0x67, 0x76, 0xcd, 0x94, 0xd, 0x1d, 0x6a, 0xb0, 0xac, 0xa, 0xbf, 0x56, 0x93, 0x6a, 0x35, 0x31, 0xdf, 0xe9, 0x6c, 0x23, 0x69, 0x97, 0x8e, 0x49, 0xfa}} }, + { (uint64_t)700000000000000ull, {{0x55, 0x9, 0x3e, 0x5e, 0xeb, 0xca, 0x3, 0x88, 0x48, 0xdc, 0x99, 0x7e, 0x31, 0x95, 0xec, 0xc5, 0x8f, 0xb4, 0xa5, 0x71, 0xb9, 0x52, 0x56, 0xc0, 0xff, 0x49, 0xbe, 0xd0, 0xf1, 0x65, 0x22, 0xbd}} }, + { (uint64_t)800000000000000ull, {{0xbb, 0xc6, 0x18, 0x2, 0x24, 0xaf, 0xd3, 0x38, 0xa6, 0xf4, 0xa0, 0x6b, 0x11, 0x98, 0x40, 0x68, 0xeb, 0x36, 0x35, 0xe7, 0xe5, 0x47, 0x66, 0x69, 0x78, 0x83, 0xaf, 0xbd, 0xce, 0xad, 0x2f, 0x31}} }, + { (uint64_t)900000000000000ull, {{0x61, 0x3a, 0xa1, 0x2c, 0xc0, 0xa1, 0x9b, 0xc8, 0x43, 0x63, 0x50, 0xbb, 0xc0, 0xf6, 0x16, 0x32, 0x6e, 0x64, 0x85, 0x83, 0x33, 0x4a, 0x32, 0x65, 0x16, 0x29, 0xe9, 0x5, 0xc5, 0x20, 0x62, 0x69}} }, + { (uint64_t)1000000000000000ull, {{0x52, 0xdd, 0xf8, 0x81, 0x13, 0xa0, 0xfc, 0xf2, 0x12, 0x90, 0x95, 0xc6, 0x18, 0x91, 0xbe, 0x88, 0x5c, 0x9, 0x30, 0x8, 0xeb, 0xc4, 0x65, 0xc, 0xb0, 0xee, 0xa5, 0x60, 0xcd, 0x4d, 0x75, 0x1b}} }, + { (uint64_t)2000000000000000ull, {{0x75, 0xbd, 0xfc, 0x35, 0xa6, 0xdf, 0x76, 0xe5, 0x98, 0x8e, 0xd9, 0xe3, 0x10, 0xa5, 0x89, 0x16, 0xae, 0xf0, 0xc5, 0xf0, 0x5b, 0x89, 0x22, 0xea, 0xae, 0x2c, 0xf9, 0x8f, 0x58, 0x42, 0x3c, 0xe3}} }, + { (uint64_t)3000000000000000ull, {{0x88, 0x98, 0x93, 0xe8, 0x7d, 0x56, 0x9f, 0x14, 0xb2, 0x48, 0xd1, 0xed, 0x93, 0xe8, 0xce, 0x60, 0xbb, 0xe3, 0x73, 0x69, 0xb0, 0xd6, 0xc7, 0xa1, 0x86, 0x89, 0x33, 0xd3, 0xc3, 0xda, 0x9a, 0x72}} }, + { (uint64_t)4000000000000000ull, {{0x88, 0x3e, 0xf3, 0x4b, 0xa2, 0xc1, 0x91, 0xf4, 0x9d, 0x3c, 0xc6, 0xad, 0xa0, 0xaf, 0xf1, 0xcf, 0xb1, 0x77, 0xbd, 0x9e, 0xd4, 0xb3, 0xa5, 0x37, 0x84, 0xb7, 0xf1, 0x62, 0x9b, 0xed, 0x17, 0x41}} }, + { (uint64_t)5000000000000000ull, {{0xa2, 0x90, 0x7c, 0x39, 0x84, 0xb1, 0x4a, 0xb1, 0xf4, 0xda, 0x58, 0xc2, 0xc8, 0x2d, 0x6b, 0x24, 0xf1, 0x29, 0x49, 0x9, 0x75, 0xfc, 0x4a, 0x33, 0x3d, 0x25, 0xa1, 0xf9, 0x2b, 0xc4, 0x32, 0xb6}} }, + { (uint64_t)6000000000000000ull, {{0xa0, 0x7d, 0x9f, 0x18, 0x95, 0x1f, 0xf2, 0x32, 0xcf, 0x4e, 0xc0, 0xee, 0x2f, 0xbc, 0xc3, 0xe1, 0x1b, 0x2c, 0xaf, 0xc9, 0x57, 0x65, 0x82, 0x10, 0x38, 0x1e, 0x3e, 0xe4, 0xed, 0xec, 0x2e, 0x7a}} }, + { (uint64_t)7000000000000000ull, {{0x66, 0x80, 0x21, 0xd5, 0xde, 0x8c, 0xa4, 0xc1, 0x8f, 0x5a, 0x74, 0xf2, 0x78, 0x69, 0xc4, 0xd6, 0xd4, 0x93, 0xa3, 0x30, 0x39, 0x3c, 0xf0, 0x26, 0x41, 0xff, 0xa8, 0x56, 0x7b, 0xa5, 0x36, 0x20}} }, + { (uint64_t)8000000000000000ull, {{0xe0, 0x48, 0x7a, 0xc4, 0x5a, 0x82, 0x59, 0xe3, 0xe5, 0xf2, 0xd9, 0xb8, 0xf6, 0xb8, 0xfa, 0x26, 0x9a, 0x63, 0x49, 0x71, 0xa2, 0xf7, 0xc2, 0x1a, 0x54, 0x17, 0x76, 0x81, 0xeb, 0x2, 0xbd, 0x4a}} }, + { (uint64_t)9000000000000000ull, {{0x98, 0x92, 0x6a, 0x3a, 0xf0, 0x5b, 0xf4, 0xa9, 0x8d, 0xf9, 0xf6, 0x4a, 0xe7, 0xb9, 0xda, 0x45, 0xa7, 0x6, 0xc3, 0xf8, 0x39, 0x5e, 0x47, 0x1f, 0x96, 0xed, 0x3c, 0x6, 0x6, 0xbe, 0xbb, 0x71}} }, + { (uint64_t)10000000000000000ull, {{0x80, 0xad, 0xb7, 0xd, 0x46, 0xf6, 0x3a, 0x75, 0x64, 0xa3, 0xf6, 0x71, 0xd9, 0xba, 0x95, 0x71, 0xb7, 0xf7, 0x95, 0xa9, 0x63, 0x38, 0x2a, 0x4d, 0x9f, 0xaf, 0x2d, 0x54, 0xf6, 0xc6, 0x84, 0x29}} }, + { (uint64_t)20000000000000000ull, {{0xae, 0xbd, 0x97, 0x42, 0x1f, 0x3f, 0xca, 0xe8, 0x95, 0x18, 0x60, 0xe6, 0xd9, 0xd1, 0xf3, 0xec, 0x59, 0x73, 0xa2, 0xf7, 0x66, 0x88, 0x4b, 0xfe, 0x17, 0x50, 0x79, 0x51, 0xe4, 0x62, 0xc6, 0x63}} }, + { (uint64_t)30000000000000000ull, {{0x61, 0x2, 0x6c, 0x84, 0x2a, 0x6a, 0x22, 0x25, 0x74, 0x6b, 0x19, 0x6b, 0x56, 0x89, 0xe1, 0x18, 0xf5, 0x41, 0x34, 0x15, 0x98, 0x1d, 0x7, 0x73, 0x62, 0xb2, 0xe7, 0xb9, 0xac, 0xa5, 0x28, 0x16}} }, + { (uint64_t)40000000000000000ull, {{0x52, 0x54, 0xb5, 0x78, 0xe8, 0x57, 0x9a, 0x27, 0x3b, 0x89, 0x8e, 0x65, 0x9d, 0xd3, 0xe1, 0xa1, 0xcf, 0xba, 0x12, 0x47, 0x26, 0x64, 0xbd, 0x4e, 0x7f, 0x9a, 0x13, 0xb1, 0xfc, 0xee, 0x2, 0x93}} }, + { (uint64_t)50000000000000000ull, {{0x7b, 0x2a, 0xb, 00, 0xcf, 0xdc, 0xa9, 0x51, 0x46, 0xcf, 0x80, 0x95, 0xdd, 0x2b, 0x82, 0x90, 0x91, 0xb0, 0xf2, 0xd5, 0xbb, 0xc, 0x33, 0x82, 0x2d, 0x8b, 0x43, 0x42, 0x69, 0xcd, 0x2a, 0x42}} }, + { (uint64_t)60000000000000000ull, {{0x21, 0x57, 0x4f, 0xed, 0x15, 0x1a, 0x2f, 0x9f, 0x64, 0xa4, 0x5b, 0xe2, 0x8a, 0x3a, 0xf5, 0x88, 0xe9, 0xf2, 0xd1, 0x71, 0x35, 0xa3, 0x53, 0x7f, 0x7, 0xfd, 0x6a, 0xef, 0xa2, 0x9f, 0x2, 0xaf}} }, + { (uint64_t)70000000000000000ull, {{0x1a, 0xf2, 0x41, 0xe1, 0x38, 0x27, 0x98, 0x29, 0xac, 0x6a, 0xe6, 0x2f, 0xf, 0x33, 0x20, 0x4b, 0xb2, 0x8a, 0xfd, 0x6, 0x5c, 0x42, 0x59, 0x3b, 0xdc, 0x79, 0x14, 0x85, 0x97, 0x5b, 0x26, 0x95}} }, + { (uint64_t)80000000000000000ull, {{0xa8, 0xc8, 0xb8, 0x7b, 0x51, 0x2d, 0xef, 0x9b, 0x5e, 0x50, 0xe, 0xb4, 0x98, 0xaf, 0x86, 0xaa, 0xd2, 0x46, 0x4a, 0xea, 0xe7, 0x6d, 0xb1, 0xf6, 0x5d, 0x23, 0x26, 0xce, 0x90, 0x26, 0xec, 0x69}} }, + { (uint64_t)90000000000000000ull, {{0x3d, 0x78, 0x73, 0x63, 0x95, 0xf1, 0xd7, 0xde, 0x8e, 0x16, 0xc0, 0xb5, 0xa9, 0x9f, 0x4d, 0xc4, 0xeb, 0x8f, 0x22, 0xac, 0xc1, 0x5b, 0x21, 0x42, 0x44, 0x1d, 0xbd, 0x8d, 0x2c, 0x31, 0xb9, 0xce}} }, + { (uint64_t)100000000000000000ull, {{0x27, 0x27, 0xd4, 0x93, 0x2f, 0x98, 0x39, 0xe4, 0x3b, 0x6b, 0xf5, 0xfb, 0x29, 0xa3, 0xbe, 0x4c, 0x9, 0xb, 0x6e, 0xb9, 0x31, 00, 0xbb, 0x92, 0x58, 0x1a, 0xdb, 0x8d, 0xd2, 0xb6, 0x61, 0x54}} }, + { (uint64_t)200000000000000000ull, {{0xae, 0x96, 0x78, 0x2e, 0xf2, 0xc4, 0xdf, 0x7d, 0x2e, 0x4, 0xcc, 0xf9, 0xef, 0x76, 0x23, 0x7f, 0x17, 0xc, 0x97, 0x3, 0xb4, 0x92, 0xc0, 0x78, 0x52, 0x6e, 0xb1, 0xf6, 0x85, 0x3d, 0xb1, 0x33}} }, + { (uint64_t)300000000000000000ull, {{0x17, 0x43, 0xfe, 0xab, 0x12, 0xad, 0xe5, 0xfe, 0x12, 0x53, 0x22, 0x27, 0x2f, 0xd1, 0x40, 0x6b, 0x74, 0xe8, 0x19, 0x70, 0x32, 0x68, 0x46, 0x22, 0xee, 0x79, 0xab, 0xcd, 0x94, 0x93, 0x66, 0x4c}} }, + { (uint64_t)400000000000000000ull, {{0x7, 0x9b, 0xf2, 0xa9, 0x6e, 0x16, 0x6e, 0xf9, 0xe6, 0xb2, 0x23, 0x1d, 0xb9, 0x85, 0x8b, 0x99, 0x98, 0x7f, 0x49, 0x33, 0x87, 0xde, 0xeb, 0xd5, 0x17, 0x48, 0x54, 0x9a, 0xd, 0xf7, 0xdc, 0x44}} }, + { (uint64_t)500000000000000000ull, {{0xca, 0xba, 0x97, 0x98, 0x51, 0x6d, 0xad, 0x3, 0x38, 0xd0, 0x6e, 0x10, 0x6d, 0x76, 0xa2, 0x1, 0x93, 0x7a, 0xce, 0x4c, 0x91, 0x53, 0x9e, 0x61, 0x7d, 0x89, 0x28, 0x73, 0x6, 0xa3, 0x92, 0xb1}} }, + { (uint64_t)600000000000000000ull, {{0x6b, 0x8, 0x7f, 0x48, 0xb3, 0xd7, 0xaa, 0xc9, 0x57, 0xc4, 0x52, 0xe5, 0x1a, 0x18, 0xd7, 0x26, 0xb, 0xf8, 0xc8, 0x56, 0xc4, 0xc7, 0x1e, 0x48, 0xf6, 0x49, 0xae, 00, 0x4a, 0xf6, 0x8f, 0x13}} }, + { (uint64_t)700000000000000000ull, {{0x9e, 0xed, 0x8b, 0x23, 0x1f, 0x79, 0x4c, 0x46, 0x5c, 0xbe, 0x88, 0x40, 0xd0, 0xf1, 0x6f, 0x7b, 0x9f, 0x9c, 0x6e, 0xb4, 0x9c, 0x20, 0x7d, 0xe9, 0xd8, 0x55, 0x11, 0x83, 0xd0, 0xc7, 0x6e, 0x43}} }, + { (uint64_t)800000000000000000ull, {{0x58, 0x4a, 0x78, 0x93, 0x13, 0x7e, 0xbd, 0x2, 0x8b, 0xa7, 0x59, 0x82, 0xc3, 0x39, 0xb7, 0x66, 0xaa, 0xda, 0xad, 0xf9, 0x14, 0x50, 0xf9, 0x40, 0x7d, 0x2a, 0x97, 0xd7, 0xf6, 0xb1, 0x93, 0x5e}} }, + { (uint64_t)900000000000000000ull, {{0x7, 0xce, 0x54, 0xb1, 0x18, 0x26, 0xa1, 0x75, 0x23, 0x13, 0x55, 0x1a, 00, 0x20, 0xfd, 0x79, 0x8a, 00, 0x9e, 0x20, 0xcd, 0xb2, 0x40, 0x1d, 0x52, 0x51, 0xc1, 0x55, 0x8e, 0xea, 0xd2, 0x6c}} }, + { (uint64_t)1000000000000000000ull, {{0x39, 0x80, 0x7f, 0x3d, 0xce, 0xb0, 0xa6, 0xfe, 0x34, 0xa7, 0xa1, 0xed, 0xc6, 0x9b, 0x78, 0xff, 0xbe, 0xd5, 0xa7, 0x8c, 0x6c, 0x87, 0x5d, 0xda, 0x96, 0x69, 0xdb, 0xb2, 0x95, 0x70, 0xf0, 0xf4}} }, + { (uint64_t)2000000000000000000ull, {{0xda, 0x74, 00, 0x86, 0xf1, 0x5c, 0xe8, 0x21, 0xe9, 0xd, 0x50, 0xaf, 0xcf, 0x80, 0x9c, 0x7e, 0x18, 0x51, 0x90, 0x1b, 0xa3, 0x5f, 0x9f, 0x63, 0x78, 0xd6, 0x40, 0x7c, 0xb9, 0xc7, 0xa2, 0x75}} }, + { (uint64_t)3000000000000000000ull, {{0x7, 0xa1, 0x75, 0x63, 0xae, 0xf5, 0xcf, 0xd0, 0x36, 0xfa, 0x64, 0xd4, 0xb1, 0x97, 0xa9, 0x51, 0xc0, 0xd2, 0x87, 0x2b, 0xd, 0xb6, 0xf9, 0xbe, 0x47, 0xe6, 0x7c, 0xa6, 0xb5, 0x35, 0xe2, 0x6e}} }, + { (uint64_t)4000000000000000000ull, {{0xe3, 0x49, 0xf7, 0xeb, 0xe5, 0x11, 0x39, 0xfe, 0xd5, 0x69, 0x40, 0x37, 0xd1, 0x14, 0xb7, 0xbd, 0x45, 0xdd, 0xa, 0x6a, 0xf0, 0x4b, 0x62, 0xec, 0xa4, 0xd8, 0xcd, 0x55, 0x2a, 0x14, 0xe3, 0xfb}} }, + { (uint64_t)5000000000000000000ull, {{0x8d, 0x59, 0x7e, 0xa9, 0xf5, 0x79, 0x9a, 0x4d, 0x15, 0x3d, 0x82, 0xd6, 0xf7, 0xbe, 0xa0, 0x2e, 0x52, 0x40, 0xa2, 0xc8, 0x9b, 0x4, 0x1e, 0x6, 0x2f, 0x37, 0xbc, 0x7b, 0x82, 0xa0, 0xac, 0x55}} }, + { (uint64_t)6000000000000000000ull, {{0xa3, 0x43, 0xa7, 0xe1, 0x14, 0x4d, 0x33, 0x50, 0xf, 0x3e, 0xfd, 0x38, 0x15, 0x82, 0xdd, 0xc5, 0xd0, 0x18, 0x3e, 0x5d, 0xcf, 0x8a, 0xfa, 0x64, 0xbb, 0x67, 0x6c, 0x97, 0x3e, 0x3d, 0x1a, 0xb1}} }, + { (uint64_t)7000000000000000000ull, {{0x89, 0xe9, 0x3e, 0xe9, 0xf2, 0x4d, 0x72, 0x61, 0xe5, 0x44, 0xca, 0x8f, 0x9, 0xa7, 0x40, 0x4e, 0xe3, 0xa9, 0xe, 0xe2, 0x50, 0x7d, 0xda, 0xcf, 0x41, 0x2a, 0x58, 0xc, 0x9, 0x65, 0x1c, 0x53}} }, + { (uint64_t)8000000000000000000ull, {{0xc5, 0x94, 0x10, 0x81, 0x54, 0x69, 0xf4, 0x59, 0xd1, 0x5a, 0x6f, 0xe3, 0xf2, 0xa1, 0x1b, 0xa6, 0x31, 0x12, 0xfa, 0xaa, 0xc5, 0x3d, 0xbc, 0x52, 0x5d, 0x3c, 0xfa, 0xb1, 0xfa, 0x9c, 0x3d, 0xdb}} }, + { (uint64_t)9000000000000000000ull, {{0x9d, 0xe7, 0xcb, 0xb, 0x8d, 0x7b, 0xac, 0x47, 0xff, 0xd3, 0x93, 0x1b, 0xcd, 0x82, 0xcd, 0xd5, 0x35, 0xc, 0x29, 0x34, 0xb1, 0x6e, 0xb, 0x64, 0x32, 0xab, 0xf7, 0xcb, 0x4b, 0x5c, 0x37, 0x6d}} }, + { (uint64_t)10000000000000000000ull, {{0x65, 0x8d, 0x1, 0x37, 0x6d, 0x18, 0x63, 0xe7, 0x7b, 0x9, 0x6f, 0x98, 0xe6, 0xe5, 0x13, 0xc2, 0x4, 0x10, 0xf5, 0xc7, 0xfb, 0x18, 0xa6, 0xe5, 0x9a, 0x52, 0x66, 0x84, 0x5c, 0xd9, 0xb1, 0xe3}} }, +}; + namespace rct { //Various key initialization functions @@ -60,16 +238,26 @@ namespace rct { //Various key generation functions + bool toPointCheckOrder(ge_p3 *P, const unsigned char *data) + { + if (ge_frombytes_vartime(P, data)) + return false; + ge_p2 R; + ge_scalarmult(&R, curveOrder().bytes, P); + key tmp; + ge_tobytes(tmp.bytes, &R); + return tmp == identity(); + } + //generates a random scalar which can be used as a secret key or mask void skGen(key &sk) { - sk = crypto::rand(); - sc_reduce32(sk.bytes); + random32_unbiased(sk.bytes); } //generates a random scalar which can be used as a secret key or mask key skGen() { - key sk = crypto::rand(); - sc_reduce32(sk.bytes); + key sk; + skGen(sk); return sk; } @@ -79,9 +267,8 @@ namespace rct { CHECK_AND_ASSERT_THROW_MES(rows > 0, "0 keys requested"); keyV rv(rows); size_t i = 0; - crypto::rand(rows * sizeof(key), (uint8_t*)&rv[0]); for (i = 0 ; i < rows ; i++) { - sc_reduce32(rv[i].bytes); + skGen(rv[i]); } return rv; } @@ -108,8 +295,7 @@ namespace rct { //generates C =aG + bH from b, a is given.. void genC(key & C, const key & a, xmr_amount amount) { - key bH = scalarmultH(d2h(amount)); - addKeys1(C, a, bH); + addKeys2(C, a, d2h(amount), rct::H); } //generates a / Pedersen commitment to the amount @@ -134,19 +320,22 @@ namespace rct { } key zeroCommit(xmr_amount amount) { - key mask = identity(); - mask = scalarmultBase(mask); + const zero_commitment *begin = zero_commitments; + const zero_commitment *end = zero_commitments + sizeof(zero_commitments) / sizeof(zero_commitments[0]); + const zero_commitment value{amount, rct::zero()}; + const auto it = std::lower_bound(begin, end, value, [](const zero_commitment &e0, const zero_commitment &e1){ return e0.amount < e1.amount; }); + if (it != end && it->amount == amount) + { + return it->commitment; + } key am = d2h(amount); key bH = scalarmultH(am); - addKeys(mask, mask, bH); - return mask; + return addKeys(G, bH); } key commit(xmr_amount amount, const key &mask) { - key c = scalarmultBase(mask); - key am = d2h(amount); - key bH = scalarmultH(am); - addKeys(c, c, bH); + key c; + genC(c, mask, amount); return c; } @@ -198,15 +387,33 @@ namespace rct { //Computes aH where H= toPoint(cn_fast_hash(G)), G the basepoint key scalarmultH(const key & a) { - ge_p3 A; ge_p2 R; - CHECK_AND_ASSERT_THROW_MES_L1(ge_frombytes_vartime(&A, H.bytes) == 0, "ge_frombytes_vartime failed at "+boost::lexical_cast(__LINE__)); - ge_scalarmult(&R, a.bytes, &A); + ge_scalarmult(&R, a.bytes, &ge_p3_H); key aP; ge_tobytes(aP.bytes, &R); return aP; } + //Computes 8P + key scalarmult8(const key & P) { + ge_p3 p3; + CHECK_AND_ASSERT_THROW_MES_L1(ge_frombytes_vartime(&p3, P.bytes) == 0, "ge_frombytes_vartime failed at "+boost::lexical_cast(__LINE__)); + ge_p2 p2; + ge_p3_to_p2(&p2, &p3); + ge_p1p1 p1; + ge_mul8(&p1, &p2); + ge_p1p1_to_p2(&p2, &p1); + rct::key res; + ge_tobytes(res.bytes, &p2); + return res; + } + + //Computes lA where l is the curve order + bool isInMainSubgroup(const key & A) { + ge_p3 p3; + return toPointCheckOrder(&p3, A.bytes); + } + //Curve addition / subtractions //for curve points: AB = A + B @@ -228,6 +435,25 @@ namespace rct { return k; } + rct::key addKeys(const keyV &A) { + if (A.empty()) + return rct::identity(); + ge_p3 p3, tmp; + CHECK_AND_ASSERT_THROW_MES_L1(ge_frombytes_vartime(&p3, A[0].bytes) == 0, "ge_frombytes_vartime failed at "+boost::lexical_cast(__LINE__)); + for (size_t i = 1; i < A.size(); ++i) + { + CHECK_AND_ASSERT_THROW_MES_L1(ge_frombytes_vartime(&tmp, A[i].bytes) == 0, "ge_frombytes_vartime failed at "+boost::lexical_cast(__LINE__)); + ge_cached p2; + ge_p3_to_cached(&p2, &tmp); + ge_p1p1 p1; + ge_add(&p1, &p3, &p2); + ge_p1p1_to_p3(&p3, &p1); + } + rct::key res; + ge_p3_tobytes(res.bytes, &p3); + return res; + } + //addKeys1 //aGB = aG + B where a is a scalar, G is the basepoint, and B is a point void addKeys1(key &aGB, const key &a, const key & B) { @@ -444,18 +670,58 @@ namespace rct { //Elliptic Curve Diffie Helman: encodes and decodes the amount b and mask a // where C= aG + bH - void ecdhEncode(ecdhTuple & unmasked, const key & sharedSec) { - key sharedSec1 = hash_to_scalar(sharedSec); - key sharedSec2 = hash_to_scalar(sharedSec1); + static key ecdhHash(const key &k) + { + char data[38]; + rct::key hash; + memcpy(data, "amount", 6); + memcpy(data + 6, &k, sizeof(k)); + cn_fast_hash(hash, data, sizeof(data)); + return hash; + } + static void xor8(key &v, const key &k) + { + for (int i = 0; i < 8; ++i) + v.bytes[i] ^= k.bytes[i]; + } + key genCommitmentMask(const key &sk) + { + char data[15 + sizeof(key)]; + memcpy(data, "commitment_mask", 15); + memcpy(data + 15, &sk, sizeof(sk)); + key scalar; + hash_to_scalar(scalar, data, sizeof(data)); + return scalar; + } + + void ecdhEncode(ecdhTuple & unmasked, const key & sharedSec, bool v2) { //encode - sc_add(unmasked.mask.bytes, unmasked.mask.bytes, sharedSec1.bytes); - sc_add(unmasked.amount.bytes, unmasked.amount.bytes, sharedSec2.bytes); + if (v2) + { + unmasked.mask = zero(); + xor8(unmasked.amount, ecdhHash(sharedSec)); + } + else + { + key sharedSec1 = hash_to_scalar(sharedSec); + key sharedSec2 = hash_to_scalar(sharedSec1); + sc_add(unmasked.mask.bytes, unmasked.mask.bytes, sharedSec1.bytes); + sc_add(unmasked.amount.bytes, unmasked.amount.bytes, sharedSec2.bytes); + } } - void ecdhDecode(ecdhTuple & masked, const key & sharedSec) { - key sharedSec1 = hash_to_scalar(sharedSec); - key sharedSec2 = hash_to_scalar(sharedSec1); + void ecdhDecode(ecdhTuple & masked, const key & sharedSec, bool v2) { //decode - sc_sub(masked.mask.bytes, masked.mask.bytes, sharedSec1.bytes); - sc_sub(masked.amount.bytes, masked.amount.bytes, sharedSec2.bytes); + if (v2) + { + masked.mask = genCommitmentMask(sharedSec); + xor8(masked.amount, ecdhHash(sharedSec)); + } + else + { + key sharedSec1 = hash_to_scalar(sharedSec); + key sharedSec2 = hash_to_scalar(sharedSec1); + sc_sub(masked.mask.bytes, masked.mask.bytes, sharedSec1.bytes); + sc_sub(masked.amount.bytes, masked.amount.bytes, sharedSec2.bytes); + } } } diff --git a/src/ringct/rctOps.h b/src/ringct/rctOps.h index 3f8f6955c62..dd6d8759383 100644 --- a/src/ringct/rctOps.h +++ b/src/ringct/rctOps.h @@ -62,6 +62,9 @@ namespace rct { static const key Z = { {0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 } }; static const key I = { {0x01, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 } }; static const key L = { {0xed, 0xd3, 0xf5, 0x5c, 0x1a, 0x63, 0x12, 0x58, 0xd6, 0x9c, 0xf7, 0xa2, 0xde, 0xf9, 0xde, 0x14, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10 } }; + static const key G = { {0x58, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66 } }; + static const key EIGHT = { {0x08, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 } }; + static const key INV_EIGHT = { { 0x79, 0x2f, 0xdc, 0xe2, 0x29, 0xe5, 0x06, 0x61, 0xd0, 0xda, 0x1c, 0x7d, 0xb3, 0x9d, 0xd3, 0x07, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x06 } }; //Creates a zero scalar inline key zero() { return Z; } @@ -82,6 +85,7 @@ namespace rct { keyM keyMInit(size_t rows, size_t cols); //Various key generation functions + bool toPointCheckOrder(ge_p3 *P, const unsigned char *data); //generates a random scalar which can be used as a secret key or mask key skGen(); @@ -118,12 +122,17 @@ namespace rct { key scalarmultKey(const key &P, const key &a); //Computes aH where H= toPoint(cn_fast_hash(G)), G the basepoint key scalarmultH(const key & a); + // multiplies a point by 8 + key scalarmult8(const key & P); + // checks a is in the main subgroup (ie, not a small one) + bool isInMainSubgroup(const key & a); //Curve addition / subtractions //for curve points: AB = A + B void addKeys(key &AB, const key &A, const key &B); rct::key addKeys(const key &A, const key &B); + rct::key addKeys(const keyV &A); //aGB = aG + B where a is a scalar, G is the basepoint, and B is a point void addKeys1(key &aGB, const key &a, const key & B); //aGbB = aG + bB where a, b are scalars, G is the basepoint and B is a point @@ -173,7 +182,8 @@ namespace rct { //Elliptic Curve Diffie Helman: encodes and decodes the amount b and mask a // where C= aG + bH - void ecdhEncode(ecdhTuple & unmasked, const key & sharedSec); - void ecdhDecode(ecdhTuple & masked, const key & sharedSec); + key genCommitmentMask(const key &sk); + void ecdhEncode(ecdhTuple & unmasked, const key & sharedSec, bool v2); + void ecdhDecode(ecdhTuple & masked, const key & sharedSec, bool v2); } #endif /* RCTOPS_H */ diff --git a/src/ringct/rctSigs.cpp b/src/ringct/rctSigs.cpp index 777b4d13aa3..ff2a81d4366 100644 --- a/src/ringct/rctSigs.cpp +++ b/src/ringct/rctSigs.cpp @@ -44,37 +44,50 @@ using namespace std; #define CHECK_AND_ASSERT_MES_L1(expr, ret, message) {if(!(expr)) {MCERROR("verify", message); return ret;}} -namespace rct { - bool is_simple(int type) +namespace +{ + rct::Bulletproof make_dummy_bulletproof(const std::vector &outamounts, rct::keyV &C, rct::keyV &masks) { - switch (type) + const size_t n_outs = outamounts.size(); + const rct::key I = rct::identity(); + size_t nrl = 0; + while ((1u << nrl) < n_outs) + ++nrl; + nrl += 6; + + C.resize(n_outs); + masks.resize(n_outs); + for (size_t i = 0; i < n_outs; ++i) { - case RCTTypeSimple: - case RCTTypeSimpleBulletproof: - return true; - default: - return false; + masks[i] = I; + rct::key sv8, sv; + sv = rct::zero(); + sv.bytes[0] = outamounts[i] & 255; + sv.bytes[1] = (outamounts[i] >> 8) & 255; + sv.bytes[2] = (outamounts[i] >> 16) & 255; + sv.bytes[3] = (outamounts[i] >> 24) & 255; + sv.bytes[4] = (outamounts[i] >> 32) & 255; + sv.bytes[5] = (outamounts[i] >> 40) & 255; + sv.bytes[6] = (outamounts[i] >> 48) & 255; + sv.bytes[7] = (outamounts[i] >> 56) & 255; + sc_mul(sv8.bytes, sv.bytes, rct::INV_EIGHT.bytes); + rct::addKeys2(C[i], rct::INV_EIGHT, sv8, rct::H); } - } - bool is_bulletproof(int type) - { - switch (type) - { - case RCTTypeSimpleBulletproof: - case RCTTypeFullBulletproof: - return true; - default: - return false; - } + return rct::Bulletproof{rct::keyV(n_outs, I), I, I, I, I, I, I, rct::keyV(nrl, I), rct::keyV(nrl, I), I, I, I}; } +} - Bulletproof proveRangeBulletproof(key &C, key &mask, uint64_t amount) +namespace rct { + Bulletproof proveRangeBulletproof(keyV &C, keyV &masks, const std::vector &amounts, epee::span sk, hw::device &hwdev) { - mask = rct::skGen(); - Bulletproof proof = bulletproof_PROVE(amount, mask); - CHECK_AND_ASSERT_THROW_MES(proof.V.size() == 1, "V has not exactly one element"); - C = proof.V[0]; + CHECK_AND_ASSERT_THROW_MES(amounts.size() == sk.size(), "Invalid amounts/sk sizes"); + masks.resize(amounts.size()); + for (size_t i = 0; i < masks.size(); ++i) + masks[i] = hwdev.genCommitmentMask(sk[i]); + Bulletproof proof = bulletproof_PROVE(amounts, masks); + CHECK_AND_ASSERT_THROW_MES(proof.V.size() == amounts.size(), "V does not have the expected size"); + C = proof.V; return proof; } @@ -85,6 +98,13 @@ namespace rct { catch (...) { return false; } } + bool verBulletproof(const std::vector &proofs) + { + try { return bulletproof_VERIFY(proofs); } + // we can get deep throws from ge_frombytes_vartime if input isn't valid + catch (...) { return false; } + } + //Borromean (c.f. gmax/andytoshi's paper) boroSig genBorromean(const key64 x, const key64 P1, const key64 P2, const bits indices) { key64 L[2], alpha; @@ -147,7 +167,7 @@ namespace rct { //This is a just slghtly more efficient version than the ones described below //(will be explained in more detail in Ring Multisig paper //These are aka MG signatutes in earlier drafts of the ring ct paper - // c.f. http://eprint.iacr.org/2015/1098 section 2. + // c.f. https://eprint.iacr.org/2015/1098 section 2. // Gen creates a signature which proves that for some column in the keymatrix "pk" // the signer knows a secret key for each row in that column // Ver verifies that the MG sig was created correctly @@ -244,7 +264,7 @@ namespace rct { //This is a just slghtly more efficient version than the ones described below //(will be explained in more detail in Ring Multisig paper //These are aka MG signatutes in earlier drafts of the ring ct paper - // c.f. http://eprint.iacr.org/2015/1098 section 2. + // c.f. https://eprint.iacr.org/2015/1098 section 2. // Gen creates a signature which proves that for some column in the keymatrix "pk" // the signer knows a secret key for each row in that column // Ver verifies that the MG sig was created correctly @@ -285,6 +305,7 @@ namespace rct { for (j = 0; j < dsRows; j++) { addKeys2(L, rv.ss[i][j], c_old, pk[i][j]); hashToPoint(Hi, pk[i][j]); + CHECK_AND_ASSERT_MES(!(Hi == rct::identity()), false, "Data hashed to point at infinity"); addKeys3(R, rv.ss[i][j], Hi, c_old, Ip[j].k); toHash[3 * j + 1] = pk[i][j]; toHash[3 * j + 2] = L; @@ -307,7 +328,7 @@ namespace rct { //proveRange and verRange //proveRange gives C, and mask such that \sumCi = C - // c.f. http://eprint.iacr.org/2015/1098 section 5.1 + // c.f. https://eprint.iacr.org/2015/1098 section 5.1 // and Ci is a commitment to either 0 or 2^i, i=0,...,63 // thus this proves that "amount" is in [0, 2^64] // mask is a such that C = aG + bH, and b = amount @@ -339,7 +360,7 @@ namespace rct { //proveRange and verRange //proveRange gives C, and mask such that \sumCi = C - // c.f. http://eprint.iacr.org/2015/1098 section 5.1 + // c.f. https://eprint.iacr.org/2015/1098 section 5.1 // and Ci is a commitment to either 0 or 2^i, i=0,...,63 // thus this proves that "amount" is in [0, 2^64] // mask is a such that C = aG + bH, and b = amount @@ -389,7 +410,7 @@ namespace rct { std::stringstream ss; binary_archive ba(ss); CHECK_AND_ASSERT_THROW_MES(!rv.mixRing.empty(), "Empty mixRing"); - const size_t inputs = is_simple(rv.type) ? rv.mixRing.size() : rv.mixRing[0].size(); + const size_t inputs = is_rct_simple(rv.type) ? rv.mixRing.size() : rv.mixRing[0].size(); const size_t outputs = rv.ecdhInfo.size(); key prehash; CHECK_AND_ASSERT_THROW_MES(const_cast(rv).serialize_rctsig_base(ba, inputs, outputs), @@ -398,7 +419,7 @@ namespace rct { hashes.push_back(hash2rct(h)); keyV kv; - if (rv.type == RCTTypeSimpleBulletproof || rv.type == RCTTypeFullBulletproof) + if (rv.type == RCTTypeBulletproof || rv.type == RCTTypeBulletproof2) { kv.reserve((6*2+9) * rv.p.bulletproofs.size()); for (const auto &p: rv.p.bulletproofs) @@ -441,14 +462,13 @@ namespace rct { //Ring-ct MG sigs //Prove: - // c.f. http://eprint.iacr.org/2015/1098 section 4. definition 10. + // c.f. https://eprint.iacr.org/2015/1098 section 4. definition 10. // This does the MG sig on the "dest" part of the given key matrix, and // the last row is the sum of input commitments from that column - sum output commitments // this shows that sum inputs = sum outputs //Ver: // verifies the above sig is created corretly - mgSig proveRctMG(const key &message, const ctkeyM & pubs, const ctkeyV & inSk, const ctkeyV &outSk, const ctkeyV & outPk, const multisig_kLRki *kLRki, key *mscout, unsigned int index, key txnFeeKey, hw::device &hwdev) { - mgSig mg; + mgSig proveRctMG(const key &message, const ctkeyM & pubs, const ctkeyV & inSk, const ctkeyV &outSk, const ctkeyV & outPk, const multisig_kLRki *kLRki, key *mscout, unsigned int index, const key &txnFeeKey, hw::device &hwdev) { //setup vars size_t cols = pubs.size(); CHECK_AND_ASSERT_THROW_MES(cols >= 1, "Empty pubs"); @@ -492,7 +512,9 @@ namespace rct { for (size_t j = 0; j < outPk.size(); j++) { sc_sub(sk[rows].bytes, sk[rows].bytes, outSk[j].mask.bytes); //subtract output masks in last row.. } - return MLSAG_Gen(message, M, sk, kLRki, mscout, index, rows, hwdev); + mgSig result = MLSAG_Gen(message, M, sk, kLRki, mscout, index, rows, hwdev); + memwipe(sk.data(), sk.size() * sizeof(key)); + return result; } @@ -504,7 +526,6 @@ namespace rct { // a_out, Cout is for the output commitment // index is the signing index.. mgSig proveRctMGSimple(const key &message, const ctkeyV & pubs, const ctkey & inSk, const key &a , const key &Cout, const multisig_kLRki *kLRki, key *mscout, unsigned int index, hw::device &hwdev) { - mgSig mg; //setup vars size_t rows = 1; size_t cols = pubs.size(); @@ -521,19 +542,21 @@ namespace rct { M[i][0] = pubs[i].dest; subKeys(M[i][1], pubs[i].mask, Cout); } - return MLSAG_Gen(message, M, sk, kLRki, mscout, index, rows, hwdev); + mgSig result = MLSAG_Gen(message, M, sk, kLRki, mscout, index, rows, hwdev); + memwipe(&sk[0], sizeof(key)); + return result; } //Ring-ct MG sigs //Prove: - // c.f. http://eprint.iacr.org/2015/1098 section 4. definition 10. + // c.f. https://eprint.iacr.org/2015/1098 section 4. definition 10. // This does the MG sig on the "dest" part of the given key matrix, and // the last row is the sum of input commitments from that column - sum output commitments // this shows that sum inputs = sum outputs //Ver: // verifies the above sig is created corretly - bool verRctMG(const mgSig &mg, const ctkeyM & pubs, const ctkeyV & outPk, key txnFeeKey, const key &message) { + bool verRctMG(const mgSig &mg, const ctkeyM & pubs, const ctkeyV & outPk, const key &txnFeeKey, const key &message) { PERF_TIMER(verRctMG); //setup vars size_t cols = pubs.size(); @@ -583,10 +606,19 @@ namespace rct { keyV tmp(rows + 1); size_t i; keyM M(cols, tmp); + ge_p3 Cp3; + CHECK_AND_ASSERT_MES_L1(ge_frombytes_vartime(&Cp3, C.bytes) == 0, false, "point conv failed"); + ge_cached Ccached; + ge_p3_to_cached(&Ccached, &Cp3); + ge_p1p1 p1; //create the matrix to mg sig for (i = 0; i < cols; i++) { M[i][0] = pubs[i].dest; - subKeys(M[i][1], pubs[i].mask, C); + ge_p3 p3; + CHECK_AND_ASSERT_MES_L1(ge_frombytes_vartime(&p3, pubs[i].mask.bytes) == 0, false, "point conv failed"); + ge_sub(&p1, &p3, &Ccached); + ge_p1p1_to_p3(&p3, &p1); + ge_p3_tobytes(M[i][1].bytes, &p3); } //DP(C); return MLSAG_Ver(message, M, mg, rows); @@ -650,12 +682,12 @@ namespace rct { // Also contains masked "amount" and "mask" so the receiver can see how much they received //verRct: // verifies that all signatures (rangeProogs, MG sig, sum inputs = outputs) are correct - //decodeRct: (c.f. http://eprint.iacr.org/2015/1098 section 5.1.1) + //decodeRct: (c.f. https://eprint.iacr.org/2015/1098 section 5.1.1) // uses the attached ecdh info to find the amounts represented by each output commitment // must know the destination private key to find the correct amount, else will return a random number // Note: For txn fees, the last index in the amounts vector should contain that // Thus the amounts vector will be "one" longer than the destinations vectort - rctSig genRct(const key &message, const ctkeyV & inSk, const keyV & destinations, const vector & amounts, const ctkeyM &mixRing, const keyV &amount_keys, const multisig_kLRki *kLRki, multisig_out *msout, unsigned int index, ctkeyV &outSk, bool bulletproof, hw::device &hwdev) { + rctSig genRct(const key &message, const ctkeyV & inSk, const keyV & destinations, const vector & amounts, const ctkeyM &mixRing, const keyV &amount_keys, const multisig_kLRki *kLRki, multisig_out *msout, unsigned int index, ctkeyV &outSk, const RCTConfig &rct_config, hw::device &hwdev) { CHECK_AND_ASSERT_THROW_MES(amounts.size() == destinations.size() || amounts.size() == destinations.size() + 1, "Different number of amounts/destinations"); CHECK_AND_ASSERT_THROW_MES(amount_keys.size() == destinations.size(), "Different number of amount_keys/destinations"); CHECK_AND_ASSERT_THROW_MES(index < mixRing.size(), "Bad index into mixRing"); @@ -663,15 +695,13 @@ namespace rct { CHECK_AND_ASSERT_THROW_MES(mixRing[n].size() == inSk.size(), "Bad mixRing size"); } CHECK_AND_ASSERT_THROW_MES((kLRki && msout) || (!kLRki && !msout), "Only one of kLRki/msout is present"); + CHECK_AND_ASSERT_THROW_MES(inSk.size() < 2, "genRct is not suitable for 2+ rings"); rctSig rv; - rv.type = bulletproof ? RCTTypeFullBulletproof : RCTTypeFull; + rv.type = RCTTypeFull; rv.message = message; rv.outPk.resize(destinations.size()); - if (bulletproof) - rv.p.bulletproofs.resize(destinations.size()); - else - rv.p.rangeSigs.resize(destinations.size()); + rv.p.rangeSigs.resize(destinations.size()); rv.ecdhInfo.resize(destinations.size()); size_t i = 0; @@ -681,21 +711,14 @@ namespace rct { //add destination to sig rv.outPk[i].dest = copy(destinations[i]); //compute range proof - if (bulletproof) - rv.p.bulletproofs[i] = proveRangeBulletproof(rv.outPk[i].mask, outSk[i].mask, amounts[i]); - else - rv.p.rangeSigs[i] = proveRange(rv.outPk[i].mask, outSk[i].mask, amounts[i]); + rv.p.rangeSigs[i] = proveRange(rv.outPk[i].mask, outSk[i].mask, amounts[i]); #ifdef DBG - if (bulletproof) - CHECK_AND_ASSERT_THROW_MES(verBulletproof(rv.p.bulletproofs[i]), "verBulletproof failed on newly created proof"); - else - CHECK_AND_ASSERT_THROW_MES(verRange(rv.outPk[i].mask, rv.p.rangeSigs[i]), "verRange failed on newly created proof"); + CHECK_AND_ASSERT_THROW_MES(verRange(rv.outPk[i].mask, rv.p.rangeSigs[i]), "verRange failed on newly created proof"); #endif - //mask amount and mask rv.ecdhInfo[i].mask = copy(outSk[i].mask); rv.ecdhInfo[i].amount = d2h(amounts[i]); - hwdev.ecdhEncode(rv.ecdhInfo[i], amount_keys[i]); + hwdev.ecdhEncode(rv.ecdhInfo[i], amount_keys[i], rv.type == RCTTypeBulletproof2); } //set txn fee @@ -716,17 +739,18 @@ namespace rct { return rv; } - rctSig genRct(const key &message, const ctkeyV & inSk, const ctkeyV & inPk, const keyV & destinations, const vector & amounts, const keyV &amount_keys, const multisig_kLRki *kLRki, multisig_out *msout, const int mixin, hw::device &hwdev) { + rctSig genRct(const key &message, const ctkeyV & inSk, const ctkeyV & inPk, const keyV & destinations, const vector & amounts, const keyV &amount_keys, const multisig_kLRki *kLRki, multisig_out *msout, const int mixin, const RCTConfig &rct_config, hw::device &hwdev) { unsigned int index; ctkeyM mixRing; ctkeyV outSk; tie(mixRing, index) = populateFromBlockchain(inPk, mixin); - return genRct(message, inSk, destinations, amounts, mixRing, amount_keys, kLRki, msout, index, outSk, false, hwdev); + return genRct(message, inSk, destinations, amounts, mixRing, amount_keys, kLRki, msout, index, outSk, rct_config, hwdev); } //RCT simple //for post-rct only - rctSig genRctSimple(const key &message, const ctkeyV & inSk, const keyV & destinations, const vector &inamounts, const vector &outamounts, xmr_amount txnFee, const ctkeyM & mixRing, const keyV &amount_keys, const std::vector *kLRki, multisig_out *msout, const std::vector & index, ctkeyV &outSk, bool bulletproof, hw::device &hwdev) { + rctSig genRctSimple(const key &message, const ctkeyV & inSk, const keyV & destinations, const vector &inamounts, const vector &outamounts, xmr_amount txnFee, const ctkeyM & mixRing, const keyV &amount_keys, const std::vector *kLRki, multisig_out *msout, const std::vector & index, ctkeyV &outSk, const RCTConfig &rct_config, hw::device &hwdev) { + const bool bulletproof = rct_config.range_proof_type != RangeProofBorromean; CHECK_AND_ASSERT_THROW_MES(inamounts.size() > 0, "Empty inamounts"); CHECK_AND_ASSERT_THROW_MES(inamounts.size() == inSk.size(), "Different number of inamounts/inSk"); CHECK_AND_ASSERT_THROW_MES(outamounts.size() == destinations.size(), "Different number of amounts/destinations"); @@ -742,41 +766,97 @@ namespace rct { } rctSig rv; - rv.type = bulletproof ? RCTTypeSimpleBulletproof : RCTTypeSimple; + rv.type = bulletproof ? (rct_config.bp_version == 0 || rct_config.bp_version >= 2 ? RCTTypeBulletproof2 : RCTTypeBulletproof) : RCTTypeSimple; rv.message = message; rv.outPk.resize(destinations.size()); - if (bulletproof) - rv.p.bulletproofs.resize(destinations.size()); - else + if (!bulletproof) rv.p.rangeSigs.resize(destinations.size()); rv.ecdhInfo.resize(destinations.size()); size_t i; keyV masks(destinations.size()); //sk mask.. outSk.resize(destinations.size()); - key sumout = zero(); for (i = 0; i < destinations.size(); i++) { //add destination to sig rv.outPk[i].dest = copy(destinations[i]); //compute range proof - if (bulletproof) - rv.p.bulletproofs[i] = proveRangeBulletproof(rv.outPk[i].mask, outSk[i].mask, outamounts[i]); - else + if (!bulletproof) rv.p.rangeSigs[i] = proveRange(rv.outPk[i].mask, outSk[i].mask, outamounts[i]); #ifdef DBG - if (bulletproof) - CHECK_AND_ASSERT_THROW_MES(verBulletproof(rv.p.bulletproofs[i]), "verBulletproof failed on newly created proof"); - else + if (!bulletproof) CHECK_AND_ASSERT_THROW_MES(verRange(rv.outPk[i].mask, rv.p.rangeSigs[i]), "verRange failed on newly created proof"); #endif - + } + + rv.p.bulletproofs.clear(); + if (bulletproof) + { + size_t n_amounts = outamounts.size(); + size_t amounts_proved = 0; + if (rct_config.range_proof_type == RangeProofPaddedBulletproof) + { + rct::keyV C, masks; + if (hwdev.get_mode() == hw::device::TRANSACTION_CREATE_FAKE) + { + // use a fake bulletproof for speed + rv.p.bulletproofs.push_back(make_dummy_bulletproof(outamounts, C, masks)); + } + else + { + const epee::span keys{&amount_keys[0], amount_keys.size()}; + rv.p.bulletproofs.push_back(proveRangeBulletproof(C, masks, outamounts, keys, hwdev)); + #ifdef DBG + CHECK_AND_ASSERT_THROW_MES(verBulletproof(rv.p.bulletproofs.back()), "verBulletproof failed on newly created proof"); + #endif + } + for (i = 0; i < outamounts.size(); ++i) + { + rv.outPk[i].mask = rct::scalarmult8(C[i]); + outSk[i].mask = masks[i]; + } + } + else while (amounts_proved < n_amounts) + { + size_t batch_size = 1; + if (rct_config.range_proof_type == RangeProofMultiOutputBulletproof) + while (batch_size * 2 + amounts_proved <= n_amounts && batch_size * 2 <= BULLETPROOF_MAX_OUTPUTS) + batch_size *= 2; + rct::keyV C, masks; + std::vector batch_amounts(batch_size); + for (i = 0; i < batch_size; ++i) + batch_amounts[i] = outamounts[i + amounts_proved]; + if (hwdev.get_mode() == hw::device::TRANSACTION_CREATE_FAKE) + { + // use a fake bulletproof for speed + rv.p.bulletproofs.push_back(make_dummy_bulletproof(batch_amounts, C, masks)); + } + else + { + const epee::span keys{&amount_keys[amounts_proved], batch_size}; + rv.p.bulletproofs.push_back(proveRangeBulletproof(C, masks, batch_amounts, keys, hwdev)); + #ifdef DBG + CHECK_AND_ASSERT_THROW_MES(verBulletproof(rv.p.bulletproofs.back()), "verBulletproof failed on newly created proof"); + #endif + } + for (i = 0; i < batch_size; ++i) + { + rv.outPk[i + amounts_proved].mask = rct::scalarmult8(C[i]); + outSk[i + amounts_proved].mask = masks[i]; + } + amounts_proved += batch_size; + } + } + + key sumout = zero(); + for (i = 0; i < outSk.size(); ++i) + { sc_add(sumout.bytes, outSk[i].mask.bytes, sumout.bytes); //mask amount and mask rv.ecdhInfo[i].mask = copy(outSk[i].mask); rv.ecdhInfo[i].amount = d2h(outamounts[i]); - hwdev.ecdhEncode(rv.ecdhInfo[i], amount_keys[i]); + hwdev.ecdhEncode(rv.ecdhInfo[i], amount_keys[i], rv.type == RCTTypeBulletproof2); } //set txn fee @@ -794,7 +874,6 @@ namespace rct { sc_add(sumpouts.bytes, a[i].bytes, sumpouts.bytes); genC(pseudoOuts[i], a[i], inamounts[i]); } - rv.mixRing = mixRing; sc_sub(a[i].bytes, sumout.bytes, sumpouts.bytes); genC(pseudoOuts[i], a[i], inamounts[i]); DP(pseudoOuts[i]); @@ -808,7 +887,7 @@ namespace rct { return rv; } - rctSig genRctSimple(const key &message, const ctkeyV & inSk, const ctkeyV & inPk, const keyV & destinations, const vector &inamounts, const vector &outamounts, const keyV &amount_keys, const std::vector *kLRki, multisig_out *msout, xmr_amount txnFee, unsigned int mixin, hw::device &hwdev) { + rctSig genRctSimple(const key &message, const ctkeyV & inSk, const ctkeyV & inPk, const keyV & destinations, const vector &inamounts, const vector &outamounts, const keyV &amount_keys, const std::vector *kLRki, multisig_out *msout, xmr_amount txnFee, unsigned int mixin, const RCTConfig &rct_config, hw::device &hwdev) { std::vector index; index.resize(inPk.size()); ctkeyM mixRing; @@ -818,7 +897,7 @@ namespace rct { mixRing[i].resize(mixin+1); index[i] = populateFromBlockchainSimple(mixRing[i], inPk[i], mixin); } - return genRctSimple(message, inSk, destinations, inamounts, outamounts, txnFee, mixRing, amount_keys, kLRki, msout, index, outSk, false, hwdev); + return genRctSimple(message, inSk, destinations, inamounts, outamounts, txnFee, mixRing, amount_keys, kLRki, msout, index, outSk, rct_config, hwdev); } //RingCT protocol @@ -828,18 +907,15 @@ namespace rct { // Also contains masked "amount" and "mask" so the receiver can see how much they received //verRct: // verifies that all signatures (rangeProogs, MG sig, sum inputs = outputs) are correct - //decodeRct: (c.f. http://eprint.iacr.org/2015/1098 section 5.1.1) + //decodeRct: (c.f. https://eprint.iacr.org/2015/1098 section 5.1.1) // uses the attached ecdh info to find the amounts represented by each output commitment // must know the destination private key to find the correct amount, else will return a random number bool verRct(const rctSig & rv, bool semantics) { PERF_TIMER(verRct); - CHECK_AND_ASSERT_MES(rv.type == RCTTypeFull || rv.type == RCTTypeFullBulletproof, false, "verRct called on non-full rctSig"); + CHECK_AND_ASSERT_MES(rv.type == RCTTypeFull, false, "verRct called on non-full rctSig"); if (semantics) { - if (rv.type == RCTTypeFullBulletproof) - CHECK_AND_ASSERT_MES(rv.outPk.size() == rv.p.bulletproofs.size(), false, "Mismatched sizes of outPk and rv.p.bulletproofs"); - else - CHECK_AND_ASSERT_MES(rv.outPk.size() == rv.p.rangeSigs.size(), false, "Mismatched sizes of outPk and rv.p.rangeSigs"); + CHECK_AND_ASSERT_MES(rv.outPk.size() == rv.p.rangeSigs.size(), false, "Mismatched sizes of outPk and rv.p.rangeSigs"); CHECK_AND_ASSERT_MES(rv.outPk.size() == rv.ecdhInfo.size(), false, "Mismatched sizes of outPk and rv.ecdhInfo"); CHECK_AND_ASSERT_MES(rv.p.MGs.size() == 1, false, "full rctSig has not one MG"); } @@ -856,19 +932,13 @@ namespace rct { tools::threadpool::waiter waiter; std::deque results(rv.outPk.size(), false); DP("range proofs verified?"); - for (size_t i = 0; i < rv.outPk.size(); i++) { - tpool.submit(&waiter, [&, i] { - if (rv.p.rangeSigs.empty()) - results[i] = verBulletproof(rv.p.bulletproofs[i]); - else - results[i] = verRange(rv.outPk[i].mask, rv.p.rangeSigs[i]); - }); - } - waiter.wait(); + for (size_t i = 0; i < rv.outPk.size(); i++) + tpool.submit(&waiter, [&, i] { results[i] = verRange(rv.outPk[i].mask, rv.p.rangeSigs[i]); }); + waiter.wait(&tpool); - for (size_t i = 0; i < rv.outPk.size(); ++i) { + for (size_t i = 0; i < results.size(); ++i) { if (!results[i]) { - LOG_PRINT_L1("Range proof verified failed for output " << i); + LOG_PRINT_L1("Range proof verified failed for proof " << i); return false; } } @@ -902,17 +972,27 @@ namespace rct { //ver RingCT simple //assumes only post-rct style inputs (at least for max anonymity) - bool verRctSimple(const rctSig & rv, bool semantics) { + bool verRctSemanticsSimple(const std::vector & rvv) { try { - PERF_TIMER(verRctSimple); + PERF_TIMER(verRctSemanticsSimple); - CHECK_AND_ASSERT_MES(rv.type == RCTTypeSimple || rv.type == RCTTypeSimpleBulletproof, false, "verRctSimple called on non simple rctSig"); - if (semantics) + tools::threadpool& tpool = tools::threadpool::getInstance(); + tools::threadpool::waiter waiter; + std::deque results; + std::vector proofs; + size_t max_non_bp_proofs = 0, offset = 0; + + for (const rctSig *rvp: rvv) { - if (rv.type == RCTTypeSimpleBulletproof) + CHECK_AND_ASSERT_MES(rvp, false, "rctSig pointer is NULL"); + const rctSig &rv = *rvp; + CHECK_AND_ASSERT_MES(rv.type == RCTTypeSimple || rv.type == RCTTypeBulletproof || rv.type == RCTTypeBulletproof2, + false, "verRctSemanticsSimple called on non simple rctSig"); + const bool bulletproof = is_rct_bulletproof(rv.type); + if (bulletproof) { - CHECK_AND_ASSERT_MES(rv.outPk.size() == rv.p.bulletproofs.size(), false, "Mismatched sizes of outPk and rv.p.bulletproofs"); + CHECK_AND_ASSERT_MES(rv.outPk.size() == n_bulletproof_amounts(rv.p.bulletproofs), false, "Mismatched sizes of outPk and bulletproofs"); CHECK_AND_ASSERT_MES(rv.p.pseudoOuts.size() == rv.p.MGs.size(), false, "Mismatched sizes of rv.p.pseudoOuts and rv.p.MGs"); CHECK_AND_ASSERT_MES(rv.pseudoOuts.empty(), false, "rv.pseudoOuts is not empty"); } @@ -923,81 +1003,122 @@ namespace rct { CHECK_AND_ASSERT_MES(rv.p.pseudoOuts.empty(), false, "rv.p.pseudoOuts is not empty"); } CHECK_AND_ASSERT_MES(rv.outPk.size() == rv.ecdhInfo.size(), false, "Mismatched sizes of outPk and rv.ecdhInfo"); - } - else - { - // semantics check is early, and mixRing/MGs aren't resolved yet - if (rv.type == RCTTypeSimpleBulletproof) - CHECK_AND_ASSERT_MES(rv.p.pseudoOuts.size() == rv.mixRing.size(), false, "Mismatched sizes of rv.p.pseudoOuts and mixRing"); - else - CHECK_AND_ASSERT_MES(rv.pseudoOuts.size() == rv.mixRing.size(), false, "Mismatched sizes of rv.pseudoOuts and mixRing"); - } - const size_t threads = std::max(rv.outPk.size(), rv.mixRing.size()); + if (!bulletproof) + max_non_bp_proofs += rv.p.rangeSigs.size(); + } - std::deque results(threads); - tools::threadpool& tpool = tools::threadpool::getInstance(); - tools::threadpool::waiter waiter; + results.resize(max_non_bp_proofs); + for (const rctSig *rvp: rvv) + { + const rctSig &rv = *rvp; - const keyV &pseudoOuts = is_bulletproof(rv.type) ? rv.p.pseudoOuts : rv.pseudoOuts; + const bool bulletproof = is_rct_bulletproof(rv.type); + const keyV &pseudoOuts = bulletproof ? rv.p.pseudoOuts : rv.pseudoOuts; - if (semantics) { - key sumOutpks = identity(); + rct::keyV masks(rv.outPk.size()); for (size_t i = 0; i < rv.outPk.size(); i++) { - addKeys(sumOutpks, sumOutpks, rv.outPk[i].mask); + masks[i] = rv.outPk[i].mask; } + key sumOutpks = addKeys(masks); DP(sumOutpks); - key txnFeeKey = scalarmultH(d2h(rv.txnFee)); + const key txnFeeKey = scalarmultH(d2h(rv.txnFee)); addKeys(sumOutpks, txnFeeKey, sumOutpks); - key sumPseudoOuts = identity(); - for (size_t i = 0 ; i < pseudoOuts.size() ; i++) { - addKeys(sumPseudoOuts, sumPseudoOuts, pseudoOuts[i]); - } + key sumPseudoOuts = addKeys(pseudoOuts); DP(sumPseudoOuts); //check pseudoOuts vs Outs.. if (!equalKeys(sumPseudoOuts, sumOutpks)) { - LOG_PRINT_L1("Sum check failed"); - return false; + LOG_PRINT_L1("Sum check failed"); + return false; } - results.clear(); - results.resize(rv.outPk.size()); - for (size_t i = 0; i < rv.outPk.size(); i++) { - tpool.submit(&waiter, [&, i] { - if (rv.p.rangeSigs.empty()) - results[i] = verBulletproof(rv.p.bulletproofs[i]); - else - results[i] = verRange(rv.outPk[i].mask, rv.p.rangeSigs[i]); - }); + if (bulletproof) + { + for (size_t i = 0; i < rv.p.bulletproofs.size(); i++) + proofs.push_back(&rv.p.bulletproofs[i]); } - waiter.wait(); - - for (size_t i = 0; i < results.size(); ++i) { - if (!results[i]) { - LOG_PRINT_L1("Range proof verified failed for output " << i); - return false; - } + else + { + for (size_t i = 0; i < rv.p.rangeSigs.size(); i++) + tpool.submit(&waiter, [&, i, offset] { results[i+offset] = verRange(rv.outPk[i].mask, rv.p.rangeSigs[i]); }); + offset += rv.p.rangeSigs.size(); } } - else { - const key message = get_pre_mlsag_hash(rv, hw::get_device("default")); - - results.clear(); - results.resize(rv.mixRing.size()); - for (size_t i = 0 ; i < rv.mixRing.size() ; i++) { - tpool.submit(&waiter, [&, i] { - results[i] = verRctMGSimple(message, rv.p.MGs[i], rv.mixRing[i], pseudoOuts[i]); - }); + if (!proofs.empty() && !verBulletproof(proofs)) + { + LOG_PRINT_L1("Aggregate range proof verified failed"); + return false; + } + + waiter.wait(&tpool); + for (size_t i = 0; i < results.size(); ++i) { + if (!results[i]) { + LOG_PRINT_L1("Range proof verified failed for proof " << i); + return false; } - waiter.wait(); + } - for (size_t i = 0; i < results.size(); ++i) { - if (!results[i]) { - LOG_PRINT_L1("verRctMGSimple failed for input " << i); - return false; - } + return true; + } + // we can get deep throws from ge_frombytes_vartime if input isn't valid + catch (const std::exception &e) + { + LOG_PRINT_L1("Error in verRctSemanticsSimple: " << e.what()); + return false; + } + catch (...) + { + LOG_PRINT_L1("Error in verRctSemanticsSimple, but not an actual exception"); + return false; + } + } + + bool verRctSemanticsSimple(const rctSig & rv) + { + return verRctSemanticsSimple(std::vector(1, &rv)); + } + + //ver RingCT simple + //assumes only post-rct style inputs (at least for max anonymity) + bool verRctNonSemanticsSimple(const rctSig & rv) { + try + { + PERF_TIMER(verRctNonSemanticsSimple); + + CHECK_AND_ASSERT_MES(rv.type == RCTTypeSimple || rv.type == RCTTypeBulletproof || rv.type == RCTTypeBulletproof2, + false, "verRctNonSemanticsSimple called on non simple rctSig"); + const bool bulletproof = is_rct_bulletproof(rv.type); + // semantics check is early, and mixRing/MGs aren't resolved yet + if (bulletproof) + CHECK_AND_ASSERT_MES(rv.p.pseudoOuts.size() == rv.mixRing.size(), false, "Mismatched sizes of rv.p.pseudoOuts and mixRing"); + else + CHECK_AND_ASSERT_MES(rv.pseudoOuts.size() == rv.mixRing.size(), false, "Mismatched sizes of rv.pseudoOuts and mixRing"); + + const size_t threads = std::max(rv.outPk.size(), rv.mixRing.size()); + + std::deque results(threads); + tools::threadpool& tpool = tools::threadpool::getInstance(); + tools::threadpool::waiter waiter; + + const keyV &pseudoOuts = bulletproof ? rv.p.pseudoOuts : rv.pseudoOuts; + + const key message = get_pre_mlsag_hash(rv, hw::get_device("default")); + + results.clear(); + results.resize(rv.mixRing.size()); + for (size_t i = 0 ; i < rv.mixRing.size() ; i++) { + tpool.submit(&waiter, [&, i] { + results[i] = verRctMGSimple(message, rv.p.MGs[i], rv.mixRing[i], pseudoOuts[i]); + }); + } + waiter.wait(&tpool); + + for (size_t i = 0; i < results.size(); ++i) { + if (!results[i]) { + LOG_PRINT_L1("verRctMGSimple failed for input " << i); + return false; } } @@ -1006,12 +1127,12 @@ namespace rct { // we can get deep throws from ge_frombytes_vartime if input isn't valid catch (const std::exception &e) { - LOG_PRINT_L1("Error in verRct: " << e.what()); + LOG_PRINT_L1("Error in verRctNonSemanticsSimple: " << e.what()); return false; } catch (...) { - LOG_PRINT_L1("Error in verRct, but not an actual exception"); + LOG_PRINT_L1("Error in verRctNonSemanticsSimple, but not an actual exception"); return false; } } @@ -1023,23 +1144,25 @@ namespace rct { // Also contains masked "amount" and "mask" so the receiver can see how much they received //verRct: // verifies that all signatures (rangeProogs, MG sig, sum inputs = outputs) are correct - //decodeRct: (c.f. http://eprint.iacr.org/2015/1098 section 5.1.1) + //decodeRct: (c.f. https://eprint.iacr.org/2015/1098 section 5.1.1) // uses the attached ecdh info to find the amounts represented by each output commitment // must know the destination private key to find the correct amount, else will return a random number xmr_amount decodeRct(const rctSig & rv, const key & sk, unsigned int i, key & mask, hw::device &hwdev) { - CHECK_AND_ASSERT_MES(rv.type == RCTTypeFull || rv.type == RCTTypeFullBulletproof, false, "decodeRct called on non-full rctSig"); + CHECK_AND_ASSERT_MES(rv.type == RCTTypeFull, false, "decodeRct called on non-full rctSig"); CHECK_AND_ASSERT_THROW_MES(i < rv.ecdhInfo.size(), "Bad index"); CHECK_AND_ASSERT_THROW_MES(rv.outPk.size() == rv.ecdhInfo.size(), "Mismatched sizes of rv.outPk and rv.ecdhInfo"); //mask amount and mask ecdhTuple ecdh_info = rv.ecdhInfo[i]; - hwdev.ecdhDecode(ecdh_info, sk); + hwdev.ecdhDecode(ecdh_info, sk, rv.type == RCTTypeBulletproof2); mask = ecdh_info.mask; key amount = ecdh_info.amount; key C = rv.outPk[i].mask; DP("C"); DP(C); key Ctmp; + CHECK_AND_ASSERT_THROW_MES(sc_check(mask.bytes) == 0, "warning, bad ECDH mask"); + CHECK_AND_ASSERT_THROW_MES(sc_check(amount.bytes) == 0, "warning, bad ECDH amount"); addKeys2(Ctmp, mask, amount, H); DP("Ctmp"); DP(Ctmp); @@ -1055,19 +1178,21 @@ namespace rct { } xmr_amount decodeRctSimple(const rctSig & rv, const key & sk, unsigned int i, key &mask, hw::device &hwdev) { - CHECK_AND_ASSERT_MES(rv.type == RCTTypeSimple || rv.type == RCTTypeSimpleBulletproof, false, "decodeRct called on non simple rctSig"); + CHECK_AND_ASSERT_MES(rv.type == RCTTypeSimple || rv.type == RCTTypeBulletproof || rv.type == RCTTypeBulletproof2, false, "decodeRct called on non simple rctSig"); CHECK_AND_ASSERT_THROW_MES(i < rv.ecdhInfo.size(), "Bad index"); CHECK_AND_ASSERT_THROW_MES(rv.outPk.size() == rv.ecdhInfo.size(), "Mismatched sizes of rv.outPk and rv.ecdhInfo"); //mask amount and mask ecdhTuple ecdh_info = rv.ecdhInfo[i]; - hwdev.ecdhDecode(ecdh_info, sk); + hwdev.ecdhDecode(ecdh_info, sk, rv.type == RCTTypeBulletproof2); mask = ecdh_info.mask; key amount = ecdh_info.amount; key C = rv.outPk[i].mask; DP("C"); DP(C); key Ctmp; + CHECK_AND_ASSERT_THROW_MES(sc_check(mask.bytes) == 0, "warning, bad ECDH mask"); + CHECK_AND_ASSERT_THROW_MES(sc_check(amount.bytes) == 0, "warning, bad ECDH amount"); addKeys2(Ctmp, mask, amount, H); DP("Ctmp"); DP(Ctmp); @@ -1083,12 +1208,12 @@ namespace rct { } bool signMultisig(rctSig &rv, const std::vector &indices, const keyV &k, const multisig_out &msout, const key &secret_key) { - CHECK_AND_ASSERT_MES(rv.type == RCTTypeFull || rv.type == RCTTypeSimple || rv.type == RCTTypeFullBulletproof || rv.type == RCTTypeSimpleBulletproof, + CHECK_AND_ASSERT_MES(rv.type == RCTTypeFull || rv.type == RCTTypeSimple || rv.type == RCTTypeBulletproof || rv.type == RCTTypeBulletproof2, false, "unsupported rct type"); CHECK_AND_ASSERT_MES(indices.size() == k.size(), false, "Mismatched k/indices sizes"); CHECK_AND_ASSERT_MES(k.size() == rv.p.MGs.size(), false, "Mismatched k/MGs size"); CHECK_AND_ASSERT_MES(k.size() == msout.c.size(), false, "Mismatched k/msout.c size"); - if (rv.type == RCTTypeFull || rv.type == RCTTypeFullBulletproof) + if (rv.type == RCTTypeFull) { CHECK_AND_ASSERT_MES(rv.p.MGs.size() == 1, false, "MGs not a single element"); } diff --git a/src/ringct/rctSigs.h b/src/ringct/rctSigs.h index b8aab0f11a5..9227eab1e94 100644 --- a/src/ringct/rctSigs.h +++ b/src/ringct/rctSigs.h @@ -70,7 +70,7 @@ namespace rct { //Multilayered Spontaneous Anonymous Group Signatures (MLSAG signatures) //These are aka MG signatutes in earlier drafts of the ring ct paper - // c.f. http://eprint.iacr.org/2015/1098 section 2. + // c.f. https://eprint.iacr.org/2015/1098 section 2. // Gen creates a signature which proves that for some column in the keymatrix "pk" // the signer knows a secret key for each row in that column // Ver verifies that the MG sig was created correctly @@ -80,7 +80,7 @@ namespace rct { //proveRange and verRange //proveRange gives C, and mask such that \sumCi = C - // c.f. http://eprint.iacr.org/2015/1098 section 5.1 + // c.f. https://eprint.iacr.org/2015/1098 section 5.1 // and Ci is a commitment to either 0 or 2^i, i=0,...,63 // thus this proves that "amount" is in [0, 2^64] // mask is a such that C = aG + bH, and b = amount @@ -90,15 +90,15 @@ namespace rct { //Ring-ct MG sigs //Prove: - // c.f. http://eprint.iacr.org/2015/1098 section 4. definition 10. + // c.f. https://eprint.iacr.org/2015/1098 section 4. definition 10. // This does the MG sig on the "dest" part of the given key matrix, and // the last row is the sum of input commitments from that column - sum output commitments // this shows that sum inputs = sum outputs //Ver: // verifies the above sig is created corretly - mgSig proveRctMG(const ctkeyM & pubs, const ctkeyV & inSk, const keyV &outMasks, const ctkeyV & outPk, const multisig_kLRki *kLRki, key *mscout, unsigned int index, key txnFee, const key &message, hw::device &hwdev); + mgSig proveRctMG(const ctkeyM & pubs, const ctkeyV & inSk, const keyV &outMasks, const ctkeyV & outPk, const multisig_kLRki *kLRki, key *mscout, unsigned int index, const key &txnFee, const key &message, hw::device &hwdev); mgSig proveRctMGSimple(const key & message, const ctkeyV & pubs, const ctkey & inSk, const key &a , const key &Cout, const multisig_kLRki *kLRki, key *mscout, unsigned int index, hw::device &hwdev); - bool verRctMG(const mgSig &mg, const ctkeyM & pubs, const ctkeyV & outPk, key txnFee, const key &message); + bool verRctMG(const mgSig &mg, const ctkeyM & pubs, const ctkeyV & outPk, const key &txnFee, const key &message); bool verRctMGSimple(const key &message, const mgSig &mg, const ctkeyV & pubs, const key & C); //These functions get keys from blockchain @@ -116,22 +116,24 @@ namespace rct { // Also contains masked "amount" and "mask" so the receiver can see how much they received //verRct: // verifies that all signatures (rangeProogs, MG sig, sum inputs = outputs) are correct - //decodeRct: (c.f. http://eprint.iacr.org/2015/1098 section 5.1.1) + //decodeRct: (c.f. https://eprint.iacr.org/2015/1098 section 5.1.1) // uses the attached ecdh info to find the amounts represented by each output commitment // must know the destination private key to find the correct amount, else will return a random number - rctSig genRct(const key &message, const ctkeyV & inSk, const keyV & destinations, const std::vector & amounts, const ctkeyM &mixRing, const keyV &amount_keys, const multisig_kLRki *kLRki, multisig_out *msout, unsigned int index, ctkeyV &outSk, bool bulletproof, hw::device &hwdev); - rctSig genRct(const key &message, const ctkeyV & inSk, const ctkeyV & inPk, const keyV & destinations, const std::vector & amounts, const keyV &amount_keys, const multisig_kLRki *kLRki, multisig_out *msout, const int mixin, hw::device &hwdev); - rctSig genRctSimple(const key & message, const ctkeyV & inSk, const ctkeyV & inPk, const keyV & destinations, const std::vector & inamounts, const std::vector & outamounts, const keyV &amount_keys, const std::vector *kLRki, multisig_out *msout, xmr_amount txnFee, unsigned int mixin, hw::device &hwdev); - rctSig genRctSimple(const key & message, const ctkeyV & inSk, const keyV & destinations, const std::vector & inamounts, const std::vector & outamounts, xmr_amount txnFee, const ctkeyM & mixRing, const keyV &amount_keys, const std::vector *kLRki, multisig_out *msout, const std::vector & index, ctkeyV &outSk, bool bulletproof, hw::device &hwdev); + rctSig genRct(const key &message, const ctkeyV & inSk, const keyV & destinations, const std::vector & amounts, const ctkeyM &mixRing, const keyV &amount_keys, const multisig_kLRki *kLRki, multisig_out *msout, unsigned int index, ctkeyV &outSk, const RCTConfig &rct_config, hw::device &hwdev); + rctSig genRct(const key &message, const ctkeyV & inSk, const ctkeyV & inPk, const keyV & destinations, const std::vector & amounts, const keyV &amount_keys, const multisig_kLRki *kLRki, multisig_out *msout, const int mixin, const RCTConfig &rct_config, hw::device &hwdev); + rctSig genRctSimple(const key & message, const ctkeyV & inSk, const ctkeyV & inPk, const keyV & destinations, const std::vector & inamounts, const std::vector & outamounts, const keyV &amount_keys, const std::vector *kLRki, multisig_out *msout, xmr_amount txnFee, unsigned int mixin, const RCTConfig &rct_config, hw::device &hwdev); + rctSig genRctSimple(const key & message, const ctkeyV & inSk, const keyV & destinations, const std::vector & inamounts, const std::vector & outamounts, xmr_amount txnFee, const ctkeyM & mixRing, const keyV &amount_keys, const std::vector *kLRki, multisig_out *msout, const std::vector & index, ctkeyV &outSk, const RCTConfig &rct_config, hw::device &hwdev); bool verRct(const rctSig & rv, bool semantics); static inline bool verRct(const rctSig & rv) { return verRct(rv, true) && verRct(rv, false); } - bool verRctSimple(const rctSig & rv, bool semantics); - static inline bool verRctSimple(const rctSig & rv) { return verRctSimple(rv, true) && verRctSimple(rv, false); } + bool verRctSemanticsSimple(const rctSig & rv); + bool verRctSemanticsSimple(const std::vector & rv); + bool verRctNonSemanticsSimple(const rctSig & rv); + static inline bool verRctSimple(const rctSig & rv) { return verRctSemanticsSimple(rv) && verRctNonSemanticsSimple(rv); } xmr_amount decodeRct(const rctSig & rv, const key & sk, unsigned int i, key & mask, hw::device &hwdev); xmr_amount decodeRct(const rctSig & rv, const key & sk, unsigned int i, hw::device &hwdev); xmr_amount decodeRctSimple(const rctSig & rv, const key & sk, unsigned int i, key & mask, hw::device &hwdev); xmr_amount decodeRctSimple(const rctSig & rv, const key & sk, unsigned int i, hw::device &hwdev); - + key get_pre_mlsag_hash(const rctSig &rv, hw::device &hwdev); bool signMultisig(rctSig &rv, const std::vector &indices, const keyV &k, const multisig_out &msout, const key &secret_key); } #endif /* RCTSIGS_H */ diff --git a/src/ringct/rctTypes.cpp b/src/ringct/rctTypes.cpp index 5650b3ba11f..f01e683cb74 100644 --- a/src/ringct/rctTypes.cpp +++ b/src/ringct/rctTypes.cpp @@ -28,6 +28,8 @@ // STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF // THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#include "misc_log_ex.h" +#include "cryptonote_config.h" #include "rctTypes.h" using namespace crypto; using namespace std; @@ -209,4 +211,92 @@ namespace rct { return vali; } + bool is_rct_simple(int type) + { + switch (type) + { + case RCTTypeSimple: + case RCTTypeBulletproof: + case RCTTypeBulletproof2: + return true; + default: + return false; + } + } + + bool is_rct_bulletproof(int type) + { + switch (type) + { + case RCTTypeBulletproof: + case RCTTypeBulletproof2: + return true; + default: + return false; + } + } + + bool is_rct_borromean(int type) + { + switch (type) + { + case RCTTypeSimple: + case RCTTypeFull: + return true; + default: + return false; + } + } + + size_t n_bulletproof_amounts(const Bulletproof &proof) + { + CHECK_AND_ASSERT_MES(proof.L.size() >= 6, 0, "Invalid bulletproof L size"); + CHECK_AND_ASSERT_MES(proof.L.size() == proof.R.size(), 0, "Mismatched bulletproof L/R size"); + static const size_t extra_bits = 4; + static_assert((1 << extra_bits) == BULLETPROOF_MAX_OUTPUTS, "log2(BULLETPROOF_MAX_OUTPUTS) is out of date"); + CHECK_AND_ASSERT_MES(proof.L.size() <= 6 + extra_bits, 0, "Invalid bulletproof L size"); + CHECK_AND_ASSERT_MES(proof.V.size() <= (1u<<(proof.L.size()-6)), 0, "Invalid bulletproof V/L"); + CHECK_AND_ASSERT_MES(proof.V.size() * 2 > (1u<<(proof.L.size()-6)), 0, "Invalid bulletproof V/L"); + CHECK_AND_ASSERT_MES(proof.V.size() > 0, 0, "Empty bulletproof"); + return proof.V.size(); + } + + size_t n_bulletproof_amounts(const std::vector &proofs) + { + size_t n = 0; + for (const Bulletproof &proof: proofs) + { + size_t n2 = n_bulletproof_amounts(proof); + CHECK_AND_ASSERT_MES(n2 < std::numeric_limits::max() - n, 0, "Invalid number of bulletproofs"); + if (n2 == 0) + return 0; + n += n2; + } + return n; + } + + size_t n_bulletproof_max_amounts(const Bulletproof &proof) + { + CHECK_AND_ASSERT_MES(proof.L.size() >= 6, 0, "Invalid bulletproof L size"); + CHECK_AND_ASSERT_MES(proof.L.size() == proof.R.size(), 0, "Mismatched bulletproof L/R size"); + static const size_t extra_bits = 4; + static_assert((1 << extra_bits) == BULLETPROOF_MAX_OUTPUTS, "log2(BULLETPROOF_MAX_OUTPUTS) is out of date"); + CHECK_AND_ASSERT_MES(proof.L.size() <= 6 + extra_bits, 0, "Invalid bulletproof L size"); + return 1 << (proof.L.size() - 6); + } + + size_t n_bulletproof_max_amounts(const std::vector &proofs) + { + size_t n = 0; + for (const Bulletproof &proof: proofs) + { + size_t n2 = n_bulletproof_max_amounts(proof); + CHECK_AND_ASSERT_MES(n2 < std::numeric_limits::max() - n, 0, "Invalid number of bulletproofs"); + if (n2 == 0) + return 0; + n += n2; + } + return n; + } + } diff --git a/src/ringct/rctTypes.h b/src/ringct/rctTypes.h index eba1e3d93fc..f8729b8729d 100644 --- a/src/ringct/rctTypes.h +++ b/src/ringct/rctTypes.h @@ -36,6 +36,7 @@ #include #include #include +#include extern "C" { #include "crypto/crypto-ops.h" @@ -81,7 +82,7 @@ namespace rct { unsigned char operator[](int i) const { return bytes[i]; } - bool operator==(const key &k) const { return !memcmp(bytes, k.bytes, sizeof(bytes)); } + bool operator==(const key &k) const { return !crypto_verify_32(bytes, k.bytes); } unsigned char bytes[32]; }; typedef std::vector keyV; //vector of keys @@ -119,17 +120,14 @@ namespace rct { // If the pedersen commitment to an amount is C = aG + bH, // "mask" contains a 32 byte key a // "amount" contains a hex representation (in 32 bytes) of a 64 bit number - // "senderPk" is not the senders actual public key, but a one-time public key generated for // the purpose of the ECDH exchange struct ecdhTuple { key mask; key amount; - key senderPk; BEGIN_SERIALIZE_OBJECT() - FIELD(mask) + FIELD(mask) // not saved from v2 BPs FIELD(amount) - // FIELD(senderPk) // not serialized, as we do not use it in monero currently END_SERIALIZE() }; @@ -150,7 +148,7 @@ namespace rct { }; //just contains the necessary keys to represent MLSAG sigs - //c.f. http://eprint.iacr.org/2015/1098 + //c.f. https://eprint.iacr.org/2015/1098 struct mgSig { keyM ss; key cc; @@ -186,9 +184,14 @@ namespace rct { rct::keyV L, R; rct::key a, b, t; - Bulletproof() {} + Bulletproof(): + A({}), S({}), T1({}), T2({}), taux({}), mu({}), a({}), b({}), t({}) {} Bulletproof(const rct::key &V, const rct::key &A, const rct::key &S, const rct::key &T1, const rct::key &T2, const rct::key &taux, const rct::key &mu, const rct::keyV &L, const rct::keyV &R, const rct::key &a, const rct::key &b, const rct::key &t): V({V}), A(A), S(S), T1(T1), T2(T2), taux(taux), mu(mu), L(L), R(R), a(a), b(b), t(t) {} + Bulletproof(const rct::keyV &V, const rct::key &A, const rct::key &S, const rct::key &T1, const rct::key &T2, const rct::key &taux, const rct::key &mu, const rct::keyV &L, const rct::keyV &R, const rct::key &a, const rct::key &b, const rct::key &t): + V(V), A(A), S(S), T1(T1), T2(T2), taux(taux), mu(mu), L(L), R(R), a(a), b(b), t(t) {} + + bool operator==(const Bulletproof &other) const { return V == other.V && A == other.A && S == other.S && T1 == other.T1 && T2 == other.T2 && taux == other.taux && mu == other.mu && L == other.L && R == other.R && a == other.a && b == other.b && t == other.t; } BEGIN_SERIALIZE_OBJECT() // Commitments aren't saved, they're restored via outPk @@ -210,6 +213,11 @@ namespace rct { END_SERIALIZE() }; + size_t n_bulletproof_amounts(const Bulletproof &proof); + size_t n_bulletproof_max_amounts(const Bulletproof &proof); + size_t n_bulletproof_amounts(const std::vector &proofs); + size_t n_bulletproof_max_amounts(const std::vector &proofs); + //A container to hold all signatures necessary for RingCT // rangeSigs holds all the rangeproof data of a transaction // MG holds the MLSAG signature of a transaction @@ -221,8 +229,13 @@ namespace rct { RCTTypeNull = 0, RCTTypeFull = 1, RCTTypeSimple = 2, - RCTTypeFullBulletproof = 3, - RCTTypeSimpleBulletproof = 4, + RCTTypeBulletproof = 3, + RCTTypeBulletproof2 = 4, + }; + enum RangeProofType { RangeProofBorromean, RangeProofBulletproof, RangeProofMultiOutputBulletproof, RangeProofPaddedBulletproof }; + struct RCTConfig { + RangeProofType range_proof_type; + int bp_version; }; struct rctSigBase { uint8_t type; @@ -239,8 +252,8 @@ namespace rct { { FIELD(type) if (type == RCTTypeNull) - return true; - if (type != RCTTypeFull && type != RCTTypeFullBulletproof && type != RCTTypeSimple && type != RCTTypeSimpleBulletproof) + return ar.stream().good(); + if (type != RCTTypeFull && type != RCTTypeSimple && type != RCTTypeBulletproof && type != RCTTypeBulletproof2) return false; VARINT_FIELD(txnFee) // inputs/outputs not saved, only here for serialization help @@ -269,7 +282,19 @@ namespace rct { return false; for (size_t i = 0; i < outputs; ++i) { - FIELDS(ecdhInfo[i]) + if (type == RCTTypeBulletproof2) + { + ar.begin_object(); + if (!typename Archive::is_saving()) + memset(ecdhInfo[i].amount.bytes, 0, sizeof(ecdhInfo[i].amount.bytes)); + crypto::hash8 &amount = (crypto::hash8&)ecdhInfo[i].amount; + FIELD(amount); + ar.end_object(); + } + else + { + FIELDS(ecdhInfo[i]) + } if (outputs - i > 1) ar.delimit_array(); } @@ -287,7 +312,7 @@ namespace rct { ar.delimit_array(); } ar.end_array(); - return true; + return ar.stream().good(); } }; struct rctSigPrunable { @@ -300,22 +325,29 @@ namespace rct { bool serialize_rctsig_prunable(Archive &ar, uint8_t type, size_t inputs, size_t outputs, size_t mixin) { if (type == RCTTypeNull) - return true; - if (type != RCTTypeFull && type != RCTTypeFullBulletproof && type != RCTTypeSimple && type != RCTTypeSimpleBulletproof) + return ar.stream().good(); + if (type != RCTTypeFull && type != RCTTypeSimple && type != RCTTypeBulletproof && type != RCTTypeBulletproof2) return false; - if (type == RCTTypeSimpleBulletproof || type == RCTTypeFullBulletproof) + if (type == RCTTypeBulletproof || type == RCTTypeBulletproof2) { + uint32_t nbp = bulletproofs.size(); + if (type == RCTTypeBulletproof2) + VARINT_FIELD(nbp) + else + FIELD(nbp) ar.tag("bp"); ar.begin_array(); - PREPARE_CUSTOM_VECTOR_SERIALIZATION(outputs, bulletproofs); - if (bulletproofs.size() != outputs) + if (nbp > outputs) return false; - for (size_t i = 0; i < outputs; ++i) + PREPARE_CUSTOM_VECTOR_SERIALIZATION(nbp, bulletproofs); + for (size_t i = 0; i < nbp; ++i) { FIELDS(bulletproofs[i]) - if (outputs - i > 1) + if (nbp - i > 1) ar.delimit_array(); } + if (n_bulletproof_max_amounts(bulletproofs) < outputs) + return false; ar.end_array(); } else @@ -338,7 +370,7 @@ namespace rct { ar.begin_array(); // we keep a byte for size of MGs, because we don't know whether this is // a simple or full rct signature, and it's starting to annoy the hell out of me - size_t mg_elements = (type == RCTTypeSimple || type == RCTTypeSimpleBulletproof) ? inputs : 1; + size_t mg_elements = (type == RCTTypeSimple || type == RCTTypeBulletproof || type == RCTTypeBulletproof2) ? inputs : 1; PREPARE_CUSTOM_VECTOR_SERIALIZATION(mg_elements, MGs); if (MGs.size() != mg_elements) return false; @@ -356,7 +388,7 @@ namespace rct { for (size_t j = 0; j < mixin + 1; ++j) { ar.begin_array(); - size_t mg_ss2_elements = ((type == RCTTypeSimple || type == RCTTypeSimpleBulletproof) ? 1 : inputs) + 1; + size_t mg_ss2_elements = ((type == RCTTypeSimple || type == RCTTypeBulletproof || type == RCTTypeBulletproof2) ? 1 : inputs) + 1; PREPARE_CUSTOM_VECTOR_SERIALIZATION(mg_ss2_elements, MGs[i].ss[j]); if (MGs[i].ss[j].size() != mg_ss2_elements) return false; @@ -382,7 +414,7 @@ namespace rct { ar.delimit_array(); } ar.end_array(); - if (type == RCTTypeSimpleBulletproof) + if (type == RCTTypeBulletproof || type == RCTTypeBulletproof2) { ar.tag("pseudoOuts"); ar.begin_array(); @@ -397,12 +429,22 @@ namespace rct { } ar.end_array(); } - return true; + return ar.stream().good(); } }; struct rctSig: public rctSigBase { rctSigPrunable p; + + keyV& get_pseudo_outs() + { + return type == RCTTypeBulletproof || type == RCTTypeBulletproof2 ? p.pseudoOuts : pseudoOuts; + } + + keyV const& get_pseudo_outs() const + { + return type == RCTTypeBulletproof || type == RCTTypeBulletproof2 ? p.pseudoOuts : pseudoOuts; + } }; //other basepoint H = toPoint(cn_fast_hash(G)), G the basepoint @@ -506,24 +548,28 @@ namespace rct { //int[64] to uint long long xmr_amount b2d(bits amountb); - static inline const rct::key pk2rct(const crypto::public_key &pk) { return (const rct::key&)pk; } - static inline const rct::key sk2rct(const crypto::secret_key &sk) { return (const rct::key&)sk; } - static inline const rct::key ki2rct(const crypto::key_image &ki) { return (const rct::key&)ki; } - static inline const rct::key hash2rct(const crypto::hash &h) { return (const rct::key&)h; } - static inline const crypto::public_key rct2pk(const rct::key &k) { return (const crypto::public_key&)k; } - static inline const crypto::secret_key rct2sk(const rct::key &k) { return (const crypto::secret_key&)k; } - static inline const crypto::key_image rct2ki(const rct::key &k) { return (const crypto::key_image&)k; } - static inline const crypto::hash rct2hash(const rct::key &k) { return (const crypto::hash&)k; } - static inline bool operator==(const rct::key &k0, const crypto::public_key &k1) { return !memcmp(&k0, &k1, 32); } - static inline bool operator!=(const rct::key &k0, const crypto::public_key &k1) { return memcmp(&k0, &k1, 32); } + bool is_rct_simple(int type); + bool is_rct_bulletproof(int type); + bool is_rct_borromean(int type); + + static inline const rct::key &pk2rct(const crypto::public_key &pk) { return (const rct::key&)pk; } + static inline const rct::key &sk2rct(const crypto::secret_key &sk) { return (const rct::key&)sk; } + static inline const rct::key &ki2rct(const crypto::key_image &ki) { return (const rct::key&)ki; } + static inline const rct::key &hash2rct(const crypto::hash &h) { return (const rct::key&)h; } + static inline const crypto::public_key &rct2pk(const rct::key &k) { return (const crypto::public_key&)k; } + static inline const crypto::secret_key &rct2sk(const rct::key &k) { return (const crypto::secret_key&)k; } + static inline const crypto::key_image &rct2ki(const rct::key &k) { return (const crypto::key_image&)k; } + static inline const crypto::hash &rct2hash(const rct::key &k) { return (const crypto::hash&)k; } + static inline bool operator==(const rct::key &k0, const crypto::public_key &k1) { return !crypto_verify_32(k0.bytes, (const unsigned char*)&k1); } + static inline bool operator!=(const rct::key &k0, const crypto::public_key &k1) { return crypto_verify_32(k0.bytes, (const unsigned char*)&k1); } } namespace cryptonote { - static inline bool operator==(const crypto::public_key &k0, const rct::key &k1) { return !memcmp(&k0, &k1, 32); } - static inline bool operator!=(const crypto::public_key &k0, const rct::key &k1) { return memcmp(&k0, &k1, 32); } - static inline bool operator==(const crypto::secret_key &k0, const rct::key &k1) { return !memcmp(&k0, &k1, 32); } - static inline bool operator!=(const crypto::secret_key &k0, const rct::key &k1) { return memcmp(&k0, &k1, 32); } + static inline bool operator==(const crypto::public_key &k0, const rct::key &k1) { return !crypto_verify_32((const unsigned char*)&k0, k1.bytes); } + static inline bool operator!=(const crypto::public_key &k0, const rct::key &k1) { return crypto_verify_32((const unsigned char*)&k0, k1.bytes); } + static inline bool operator==(const crypto::secret_key &k0, const rct::key &k1) { return !crypto_verify_32((const unsigned char*)&k0, k1.bytes); } + static inline bool operator!=(const crypto::secret_key &k0, const rct::key &k1) { return crypto_verify_32((const unsigned char*)&k0, k1.bytes); } } namespace rct { diff --git a/src/rpc/CMakeLists.txt b/src/rpc/CMakeLists.txt index 7162317ed6d..cffe8e1eba0 100644 --- a/src/rpc/CMakeLists.txt +++ b/src/rpc/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2014-2018, The Monero Project +# Copyright (c) 2014-2019, The Monero Project # # All rights reserved. # @@ -31,6 +31,7 @@ set(rpc_base_sources set(rpc_sources core_rpc_server.cpp + rpc_handler.cpp instanciations) set(daemon_messages_sources @@ -45,7 +46,8 @@ set(daemon_rpc_server_sources set(rpc_base_headers rpc_args.h) -set(rpc_headers) +set(rpc_headers + rpc_handler.cpp) set(daemon_rpc_server_headers) @@ -63,7 +65,6 @@ set(daemon_rpc_server_private_headers message.h daemon_messages.h daemon_handler.h - rpc_handler.h zmq_server.h) @@ -111,7 +112,8 @@ target_link_libraries(rpc common cryptonote_core cryptonote_protocol - epee + net + version ${Boost_REGEX_LIBRARY} ${Boost_THREAD_LIBRARY} PRIVATE @@ -121,6 +123,7 @@ target_link_libraries(daemon_messages LINK_PRIVATE cryptonote_core cryptonote_protocol + version serialization ${EXTRA_LIBRARIES}) @@ -129,6 +132,7 @@ target_link_libraries(daemon_rpc_server rpc cryptonote_core cryptonote_protocol + version daemon_messages serialization ${Boost_CHRONO_LIBRARY} diff --git a/src/rpc/core_rpc_server.cpp b/src/rpc/core_rpc_server.cpp index dc7b6b30fa2..b6836e636e2 100644 --- a/src/rpc/core_rpc_server.cpp +++ b/src/rpc/core_rpc_server.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -28,6 +28,7 @@ // // Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers +#include #include "include_base_utils.h" #include "string_tools.h" using namespace epee; @@ -41,13 +42,15 @@ using namespace epee; #include "cryptonote_basic/cryptonote_format_utils.h" #include "cryptonote_basic/account.h" #include "cryptonote_basic/cryptonote_basic_impl.h" +#include "cryptonote_core/tx_sanity_check.h" #include "misc_language.h" +#include "net/parse.h" #include "storages/http_abstract_invoke.h" #include "crypto/hash.h" #include "rpc/rpc_args.h" +#include "rpc/rpc_handler.h" #include "core_rpc_server_error_codes.h" #include "p2p/net_node.h" -#include "get_output_distribution_cache.h" #include "version.h" #undef MONERO_DEFAULT_LOG_CATEGORY @@ -56,6 +59,8 @@ using namespace epee; #define MAX_RESTRICTED_FAKE_OUTS_COUNT 40 #define MAX_RESTRICTED_GLOBAL_FAKE_OUTS_COUNT 5000 +#define OUTPUT_HISTOGRAM_RECENT_CUTOFF_RESTRICTION (3 * 86400) // 3 days max, the wallet requests 1.8 days + namespace { void add_reason(std::string &reasons, const char *reason) @@ -64,6 +69,18 @@ namespace reasons += ", "; reasons += reason; } + + uint64_t round_up(uint64_t value, uint64_t quantum) + { + return (value + quantum - 1) / quantum * quantum; + } + + void store_difficulty(cryptonote::difficulty_type difficulty, uint64_t &sdiff, std::string &swdiff, uint64_t &stop64) + { + sdiff = (difficulty & 0xffffffffffffffff).convert_to(); + swdiff = cryptonote::hex(difficulty); + stop64 = ((difficulty >> 64) & 0xffffffffffffffff).convert_to(); + } } namespace cryptonote @@ -77,7 +94,7 @@ namespace cryptonote command_line::add_arg(desc, arg_restricted_rpc); command_line::add_arg(desc, arg_bootstrap_daemon_address); command_line::add_arg(desc, arg_bootstrap_daemon_login); - cryptonote::rpc_args::init_options(desc); + cryptonote::rpc_args::init_options(desc, true); } //------------------------------------------------------------------------------------------------------------------------------ core_rpc_server::core_rpc_server( @@ -91,15 +108,13 @@ namespace cryptonote bool core_rpc_server::init( const boost::program_options::variables_map& vm , const bool restricted - , const network_type nettype , const std::string& port ) { m_restricted = restricted; - m_nettype = nettype; m_net_server.set_threads_prefix("RPC"); - auto rpc_config = cryptonote::rpc_args::process(vm); + auto rpc_config = cryptonote::rpc_args::process(vm, true); if (!rpc_config) return false; @@ -113,11 +128,11 @@ namespace cryptonote epee::net_utils::http::login login; login.username = bootstrap_daemon_login.substr(0, loc); login.password = bootstrap_daemon_login.substr(loc + 1); - m_http_client.set_server(m_bootstrap_daemon_address, login, false); + m_http_client.set_server(m_bootstrap_daemon_address, login, epee::net_utils::ssl_support_t::e_ssl_support_autodetect); } else { - m_http_client.set_server(m_bootstrap_daemon_address, boost::none, false); + m_http_client.set_server(m_bootstrap_daemon_address, boost::none, epee::net_utils::ssl_support_t::e_ssl_support_autodetect); } m_should_use_bootstrap_daemon = true; } @@ -134,7 +149,7 @@ namespace cryptonote auto rng = [](size_t len, uint8_t *ptr){ return crypto::rand(len, ptr); }; return epee::http_server_impl_base::init( - rng, std::move(port), std::move(rpc_config->bind_ip), std::move(rpc_config->access_control_origins), std::move(http_login) + rng, std::move(port), std::move(rpc_config->bind_ip), std::move(rpc_config->access_control_origins), std::move(http_login), std::move(rpc_config->ssl_options) ); } //------------------------------------------------------------------------------------------------------------------------------ @@ -149,19 +164,22 @@ namespace cryptonote #define CHECK_CORE_READY() do { if(!check_core_ready()){res.status = CORE_RPC_STATUS_BUSY;return true;} } while(0) //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_get_height(const COMMAND_RPC_GET_HEIGHT::request& req, COMMAND_RPC_GET_HEIGHT::response& res) + bool core_rpc_server::on_get_height(const COMMAND_RPC_GET_HEIGHT::request& req, COMMAND_RPC_GET_HEIGHT::response& res, const connection_context *ctx) { PERF_TIMER(on_get_height); bool r; if (use_bootstrap_daemon_if_necessary(invoke_http_mode::JON, "/getheight", req, res, r)) return r; - res.height = m_core.get_current_blockchain_height(); + crypto::hash hash; + m_core.get_blockchain_top(res.height, hash); + ++res.height; // block height to chain height + res.hash = string_tools::pod_to_hex(hash); res.status = CORE_RPC_STATUS_OK; return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_get_info(const COMMAND_RPC_GET_INFO::request& req, COMMAND_RPC_GET_INFO::response& res) + bool core_rpc_server::on_get_info(const COMMAND_RPC_GET_INFO::request& req, COMMAND_RPC_GET_INFO::response& res, const connection_context *ctx) { PERF_TIMER(on_get_info); bool r; @@ -175,103 +193,139 @@ namespace cryptonote return r; } + const bool restricted = m_restricted && ctx; + crypto::hash top_hash; m_core.get_blockchain_top(res.height, top_hash); ++res.height; // turn top block height into blockchain height res.top_block_hash = string_tools::pod_to_hex(top_hash); res.target_height = m_core.get_target_blockchain_height(); - res.difficulty = m_core.get_blockchain_storage().get_difficulty_for_next_block(); + store_difficulty(m_core.get_blockchain_storage().get_difficulty_for_next_block(), res.difficulty, res.wide_difficulty, res.difficulty_top64); res.target = m_core.get_blockchain_storage().get_difficulty_target(); res.tx_count = m_core.get_blockchain_storage().get_total_transactions() - res.height; //without coinbase res.tx_pool_size = m_core.get_pool_transactions_count(); - res.alt_blocks_count = m_core.get_blockchain_storage().get_alternative_blocks_count(); - uint64_t total_conn = m_p2p.get_connections_count(); - res.outgoing_connections_count = m_p2p.get_outgoing_connections_count(); - res.incoming_connections_count = total_conn - res.outgoing_connections_count; - res.rpc_connections_count = get_connections_count(); - res.white_peerlist_size = m_p2p.get_peerlist_manager().get_white_peers_count(); - res.grey_peerlist_size = m_p2p.get_peerlist_manager().get_gray_peers_count(); - res.mainnet = m_nettype == MAINNET; - res.testnet = m_nettype == TESTNET; - res.stagenet = m_nettype == STAGENET; - res.cumulative_difficulty = m_core.get_blockchain_storage().get_db().get_block_cumulative_difficulty(res.height - 1); - res.block_size_limit = m_core.get_blockchain_storage().get_current_cumulative_blocksize_limit(); - res.block_size_median = m_core.get_blockchain_storage().get_current_cumulative_blocksize_median(); - res.status = CORE_RPC_STATUS_OK; - res.start_time = (uint64_t)m_core.get_start_time(); - res.free_space = m_restricted ? std::numeric_limits::max() : m_core.get_free_space(); + res.alt_blocks_count = restricted ? 0 : m_core.get_blockchain_storage().get_alternative_blocks_count(); + uint64_t total_conn = restricted ? 0 : m_p2p.get_public_connections_count(); + res.outgoing_connections_count = restricted ? 0 : m_p2p.get_public_outgoing_connections_count(); + res.incoming_connections_count = restricted ? 0 : (total_conn - res.outgoing_connections_count); + res.rpc_connections_count = restricted ? 0 : get_connections_count(); + res.white_peerlist_size = restricted ? 0 : m_p2p.get_public_white_peers_count(); + res.grey_peerlist_size = restricted ? 0 : m_p2p.get_public_gray_peers_count(); + + cryptonote::network_type net_type = nettype(); + res.mainnet = net_type == MAINNET; + res.testnet = net_type == TESTNET; + res.stagenet = net_type == STAGENET; + res.nettype = net_type == MAINNET ? "mainnet" : net_type == TESTNET ? "testnet" : net_type == STAGENET ? "stagenet" : "fakechain"; + store_difficulty(m_core.get_blockchain_storage().get_db().get_block_cumulative_difficulty(res.height - 1), + res.cumulative_difficulty, res.wide_cumulative_difficulty, res.cumulative_difficulty_top64); + res.block_size_limit = res.block_weight_limit = m_core.get_blockchain_storage().get_current_cumulative_block_weight_limit(); + res.block_size_median = res.block_weight_median = m_core.get_blockchain_storage().get_current_cumulative_block_weight_median(); + res.start_time = restricted ? 0 : (uint64_t)m_core.get_start_time(); + res.free_space = restricted ? std::numeric_limits::max() : m_core.get_free_space(); res.offline = m_core.offline(); - res.bootstrap_daemon_address = m_bootstrap_daemon_address; - res.height_without_bootstrap = res.height; + res.bootstrap_daemon_address = restricted ? "" : m_bootstrap_daemon_address; + res.height_without_bootstrap = restricted ? 0 : res.height; + if (restricted) + res.was_bootstrap_ever_used = false; + else { boost::shared_lock lock(m_bootstrap_daemon_mutex); res.was_bootstrap_ever_used = m_was_bootstrap_ever_used; } + res.database_size = m_core.get_blockchain_storage().get_db().get_database_size(); + if (restricted) + res.database_size = round_up(res.database_size, 5ull* 1024 * 1024 * 1024); + res.update_available = restricted ? false : m_core.is_update_available(); + res.version = restricted ? "" : MONERO_VERSION; + + res.status = CORE_RPC_STATUS_OK; return true; } //------------------------------------------------------------------------------------------------------------------------------ - static cryptonote::blobdata get_pruned_tx_blob(cryptonote::transaction &tx) + bool core_rpc_server::on_get_net_stats(const COMMAND_RPC_GET_NET_STATS::request& req, COMMAND_RPC_GET_NET_STATS::response& res, const connection_context *ctx) { - std::stringstream ss; - binary_archive ba(ss); - bool r = tx.serialize_base(ba); - CHECK_AND_ASSERT_MES(r, cryptonote::blobdata(), "Failed to serialize rct signatures base"); - return ss.str(); + PERF_TIMER(on_get_net_stats); + // No bootstrap daemon check: Only ever get stats about local server + res.start_time = (uint64_t)m_core.get_start_time(); + { + CRITICAL_REGION_LOCAL(epee::net_utils::network_throttle_manager::m_lock_get_global_throttle_in); + epee::net_utils::network_throttle_manager::get_global_throttle_in().get_stats(res.total_packets_in, res.total_bytes_in); + } + { + CRITICAL_REGION_LOCAL(epee::net_utils::network_throttle_manager::m_lock_get_global_throttle_out); + epee::net_utils::network_throttle_manager::get_global_throttle_out().get_stats(res.total_packets_out, res.total_bytes_out); + } + res.status = CORE_RPC_STATUS_OK; + return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_get_blocks(const COMMAND_RPC_GET_BLOCKS_FAST::request& req, COMMAND_RPC_GET_BLOCKS_FAST::response& res) + class pruned_transaction { + transaction& tx; + public: + pruned_transaction(transaction& tx) : tx(tx) {} + BEGIN_SERIALIZE_OBJECT() + bool r = tx.serialize_base(ar); + if (!r) return false; + END_SERIALIZE() + }; + //------------------------------------------------------------------------------------------------------------------------------ + bool core_rpc_server::on_get_blocks(const COMMAND_RPC_GET_BLOCKS_FAST::request& req, COMMAND_RPC_GET_BLOCKS_FAST::response& res, const connection_context *ctx) { PERF_TIMER(on_get_blocks); bool r; if (use_bootstrap_daemon_if_necessary(invoke_http_mode::BIN, "/getblocks.bin", req, res, r)) return r; - std::list > > bs; + std::vector, std::vector > > > bs; - if(!m_core.find_blockchain_supplement(req.start_height, req.block_ids, bs, res.current_height, res.start_height, req.prune, COMMAND_RPC_GET_BLOCKS_FAST_MAX_COUNT)) + if(!m_core.find_blockchain_supplement(req.start_height, req.block_ids, bs, res.current_height, res.start_height, req.prune, !req.no_miner_tx, COMMAND_RPC_GET_BLOCKS_FAST_MAX_COUNT)) { res.status = "Failed"; return false; } size_t pruned_size = 0, unpruned_size = 0, ntxes = 0; + res.blocks.reserve(bs.size()); + res.output_indices.reserve(bs.size()); for(auto& bd: bs) { res.blocks.resize(res.blocks.size()+1); - res.blocks.back().block = bd.first; - pruned_size += bd.first.size(); - unpruned_size += bd.first.size(); + res.blocks.back().block = bd.first.first; + pruned_size += bd.first.first.size(); + unpruned_size += bd.first.first.size(); res.output_indices.push_back(COMMAND_RPC_GET_BLOCKS_FAST::block_output_indices()); - res.output_indices.back().indices.push_back(COMMAND_RPC_GET_BLOCKS_FAST::tx_output_indices()); - block b; - if (!parse_and_validate_block_from_blob(bd.first, b)) - { - res.status = "Invalid block"; - return false; - } - bool r = m_core.get_tx_outputs_gindexs(get_transaction_hash(b.miner_tx), res.output_indices.back().indices.back().indices); - if (!r) - { - res.status = "Failed"; - return false; - } - size_t txidx = 0; ntxes += bd.second.size(); - for (std::list::iterator i = bd.second.begin(); i != bd.second.end(); ++i) + res.output_indices.back().indices.reserve(1 + bd.second.size()); + if (req.no_miner_tx) + res.output_indices.back().indices.push_back(COMMAND_RPC_GET_BLOCKS_FAST::tx_output_indices()); + res.blocks.back().txs.reserve(bd.second.size()); + for (std::vector>::iterator i = bd.second.begin(); i != bd.second.end(); ++i) { - unpruned_size += i->size(); - res.blocks.back().txs.push_back(std::move(*i)); - i->clear(); - i->shrink_to_fit(); + unpruned_size += i->second.size(); + res.blocks.back().txs.push_back(std::move(i->second)); + i->second.clear(); + i->second.shrink_to_fit(); pruned_size += res.blocks.back().txs.back().size(); + } - res.output_indices.back().indices.push_back(COMMAND_RPC_GET_BLOCKS_FAST::tx_output_indices()); - bool r = m_core.get_tx_outputs_gindexs(b.tx_hashes[txidx++], res.output_indices.back().indices.back().indices); + const size_t n_txes_to_lookup = bd.second.size() + (req.no_miner_tx ? 0 : 1); + if (n_txes_to_lookup > 0) + { + std::vector> indices; + bool r = m_core.get_tx_outputs_gindexs(req.no_miner_tx ? bd.second.front().first : bd.first.second, n_txes_to_lookup, indices); if (!r) { res.status = "Failed"; return false; } + if (indices.size() != n_txes_to_lookup || res.output_indices.back().indices.size() != (req.no_miner_tx ? 1 : 0)) + { + res.status = "Failed"; + return false; + } + for (size_t i = 0; i < indices.size(); ++i) + res.output_indices.back().indices.push_back({std::move(indices[i])}); } } @@ -279,14 +333,14 @@ namespace cryptonote res.status = CORE_RPC_STATUS_OK; return true; } - bool core_rpc_server::on_get_alt_blocks_hashes(const COMMAND_RPC_GET_ALT_BLOCKS_HASHES::request& req, COMMAND_RPC_GET_ALT_BLOCKS_HASHES::response& res) + bool core_rpc_server::on_get_alt_blocks_hashes(const COMMAND_RPC_GET_ALT_BLOCKS_HASHES::request& req, COMMAND_RPC_GET_ALT_BLOCKS_HASHES::response& res, const connection_context *ctx) { PERF_TIMER(on_get_alt_blocks_hashes); bool r; if (use_bootstrap_daemon_if_necessary(invoke_http_mode::JON, "/get_alt_blocks_hashes", req, res, r)) return r; - std::list blks; + std::vector blks; if(!m_core.get_alternative_blocks(blks)) { @@ -306,7 +360,7 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_get_blocks_by_height(const COMMAND_RPC_GET_BLOCKS_BY_HEIGHT::request& req, COMMAND_RPC_GET_BLOCKS_BY_HEIGHT::response& res) + bool core_rpc_server::on_get_blocks_by_height(const COMMAND_RPC_GET_BLOCKS_BY_HEIGHT::request& req, COMMAND_RPC_GET_BLOCKS_BY_HEIGHT::response& res, const connection_context *ctx) { PERF_TIMER(on_get_blocks_by_height); bool r; @@ -328,8 +382,8 @@ namespace cryptonote res.status = "Error retrieving block at height " + std::to_string(height); return true; } - std::list txs; - std::list missed_txs; + std::vector txs; + std::vector missed_txs; m_core.get_transactions(blk.tx_hashes, txs, missed_txs); res.blocks.resize(res.blocks.size() + 1); res.blocks.back().block = block_to_blob(blk); @@ -340,73 +394,25 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_get_hashes(const COMMAND_RPC_GET_HASHES_FAST::request& req, COMMAND_RPC_GET_HASHES_FAST::response& res) + bool core_rpc_server::on_get_hashes(const COMMAND_RPC_GET_HASHES_FAST::request& req, COMMAND_RPC_GET_HASHES_FAST::response& res, const connection_context *ctx) { PERF_TIMER(on_get_hashes); bool r; if (use_bootstrap_daemon_if_necessary(invoke_http_mode::BIN, "/gethashes.bin", req, res, r)) return r; - NOTIFY_RESPONSE_CHAIN_ENTRY::request resp; - - resp.start_height = req.start_height; - if(!m_core.find_blockchain_supplement(req.block_ids, resp)) + res.start_height = req.start_height; + if(!m_core.get_blockchain_storage().find_blockchain_supplement(req.block_ids, res.m_block_ids, res.start_height, res.current_height, false)) { res.status = "Failed"; return false; } - res.current_height = resp.total_height; - res.start_height = resp.start_height; - res.m_block_ids = std::move(resp.m_block_ids); - - res.status = CORE_RPC_STATUS_OK; - return true; - } - //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_get_random_outs(const COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::request& req, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::response& res) - { - PERF_TIMER(on_get_random_outs); - bool r; - if (use_bootstrap_daemon_if_necessary(invoke_http_mode::BIN, "/getrandom_outs.bin", req, res, r)) - return r; - - res.status = "Failed"; - - if (m_restricted) - { - if (req.amounts.size() > 100 || req.outs_count > MAX_RESTRICTED_FAKE_OUTS_COUNT) - { - res.status = "Too many outs requested"; - return true; - } - } - if(!m_core.get_random_outs_for_amounts(req, res)) - { - return true; - } - - res.status = CORE_RPC_STATUS_OK; - std::stringstream ss; - typedef COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::outs_for_amount outs_for_amount; - typedef COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::out_entry out_entry; - std::for_each(res.outs.begin(), res.outs.end(), [&](outs_for_amount& ofa) - { - ss << "[" << ofa.amount << "]:"; - CHECK_AND_ASSERT_MES(ofa.outs.size(), ;, "internal error: ofa.outs.size() is empty for amount " << ofa.amount); - std::for_each(ofa.outs.begin(), ofa.outs.end(), [&](out_entry& oe) - { - ss << oe.global_amount_index << " "; - }); - ss << ENDL; - }); - std::string s = ss.str(); - LOG_PRINT_L2("COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS: " << ENDL << s); res.status = CORE_RPC_STATUS_OK; return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_get_outs_bin(const COMMAND_RPC_GET_OUTPUTS_BIN::request& req, COMMAND_RPC_GET_OUTPUTS_BIN::response& res) + bool core_rpc_server::on_get_outs_bin(const COMMAND_RPC_GET_OUTPUTS_BIN::request& req, COMMAND_RPC_GET_OUTPUTS_BIN::response& res, const connection_context *ctx) { PERF_TIMER(on_get_outs_bin); bool r; @@ -415,7 +421,8 @@ namespace cryptonote res.status = "Failed"; - if (m_restricted) + const bool restricted = m_restricted && ctx; + if (restricted) { if (req.outputs.size() > MAX_RESTRICTED_GLOBAL_FAKE_OUTS_COUNT) { @@ -433,7 +440,7 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_get_outs(const COMMAND_RPC_GET_OUTPUTS::request& req, COMMAND_RPC_GET_OUTPUTS::response& res) + bool core_rpc_server::on_get_outs(const COMMAND_RPC_GET_OUTPUTS::request& req, COMMAND_RPC_GET_OUTPUTS::response& res, const connection_context *ctx) { PERF_TIMER(on_get_outs); bool r; @@ -442,7 +449,8 @@ namespace cryptonote res.status = "Failed"; - if (m_restricted) + const bool restricted = m_restricted && ctx; + if (restricted) { if (req.outputs.size() > MAX_RESTRICTED_GLOBAL_FAKE_OUTS_COUNT) { @@ -453,6 +461,7 @@ namespace cryptonote cryptonote::COMMAND_RPC_GET_OUTPUTS_BIN::request req_bin; req_bin.outputs = req.outputs; + req_bin.get_txid = req.get_txid; cryptonote::COMMAND_RPC_GET_OUTPUTS_BIN::response res_bin; if(!m_core.get_outs(req_bin, res_bin)) { @@ -475,35 +484,7 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_get_random_rct_outs(const COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS::request& req, COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS::response& res) - { - PERF_TIMER(on_get_random_rct_outs); - bool r; - if (use_bootstrap_daemon_if_necessary(invoke_http_mode::BIN, "/getrandom_rctouts.bin", req, res, r)) - return r; - - res.status = "Failed"; - if(!m_core.get_random_rct_outs(req, res)) - { - return true; - } - - res.status = CORE_RPC_STATUS_OK; - std::stringstream ss; - typedef COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS::out_entry out_entry; - CHECK_AND_ASSERT_MES(res.outs.size(), true, "internal error: res.outs.size() is empty"); - std::for_each(res.outs.begin(), res.outs.end(), [&](out_entry& oe) - { - ss << oe.global_amount_index << " "; - }); - ss << ENDL; - std::string s = ss.str(); - LOG_PRINT_L2("COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS: " << ENDL << s); - res.status = CORE_RPC_STATUS_OK; - return true; - } - //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_get_indexes(const COMMAND_RPC_GET_TX_GLOBAL_OUTPUTS_INDEXES::request& req, COMMAND_RPC_GET_TX_GLOBAL_OUTPUTS_INDEXES::response& res) + bool core_rpc_server::on_get_indexes(const COMMAND_RPC_GET_TX_GLOBAL_OUTPUTS_INDEXES::request& req, COMMAND_RPC_GET_TX_GLOBAL_OUTPUTS_INDEXES::response& res, const connection_context *ctx) { PERF_TIMER(on_get_indexes); bool ok; @@ -521,7 +502,7 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_get_transactions(const COMMAND_RPC_GET_TRANSACTIONS::request& req, COMMAND_RPC_GET_TRANSACTIONS::response& res) + bool core_rpc_server::on_get_transactions(const COMMAND_RPC_GET_TRANSACTIONS::request& req, COMMAND_RPC_GET_TRANSACTIONS::response& res, const connection_context *ctx) { PERF_TIMER(on_get_transactions); bool ok; @@ -544,9 +525,9 @@ namespace cryptonote } vh.push_back(*reinterpret_cast(b.data())); } - std::list missed_txs; - std::list txs; - bool r = m_core.get_transactions(vh, txs, missed_txs); + std::vector missed_txs; + std::vector> txs; + bool r = m_core.get_split_transactions_blobs(vh, txs, missed_txs); if(!r) { res.status = "Failed"; @@ -557,7 +538,7 @@ namespace cryptonote // try the pool for any missing txes size_t found_in_pool = 0; std::unordered_set pool_tx_hashes; - std::unordered_map double_spend_seen; + std::unordered_map per_tx_pool_tx_info; if (!missed_txs.empty()) { std::vector pool_tx_info; @@ -566,25 +547,26 @@ namespace cryptonote if(r) { // sort to match original request - std::list sorted_txs; + std::vector> sorted_txs; std::vector::const_iterator i; + unsigned txs_processed = 0; for (const crypto::hash &h: vh) { if (std::find(missed_txs.begin(), missed_txs.end(), h) == missed_txs.end()) { - if (txs.empty()) + if (txs.size() == txs_processed) { res.status = "Failed: internal error - txs is empty"; return true; } // core returns the ones it finds in the right order - if (get_transaction_hash(txs.front()) != h) + if (std::get<0>(txs[txs_processed]) != h) { res.status = "Failed: tx hash mismatch"; return true; } - sorted_txs.push_back(std::move(txs.front())); - txs.pop_front(); + sorted_txs.push_back(std::move(txs[txs_processed])); + ++txs_processed; } else if ((i = std::find_if(pool_tx_info.begin(), pool_tx_info.end(), [h](const tx_info &txi) { return epee::string_tools::pod_to_hex(h) == txi.id_hash; })) != pool_tx_info.end()) { @@ -594,15 +576,24 @@ namespace cryptonote res.status = "Failed to parse and validate tx from blob"; return true; } - sorted_txs.push_back(tx); - missed_txs.remove(h); + std::stringstream ss; + binary_archive ba(ss); + bool r = const_cast(tx).serialize_base(ba); + if (!r) + { + res.status = "Failed to serialize transaction base"; + return true; + } + const cryptonote::blobdata pruned = ss.str(); + sorted_txs.push_back(std::make_tuple(h, pruned, get_transaction_prunable_hash(tx), std::string(i->tx_blob, pruned.size()))); + missed_txs.erase(std::find(missed_txs.begin(), missed_txs.end(), h)); pool_tx_hashes.insert(h); const std::string hash_string = epee::string_tools::pod_to_hex(h); for (const auto &ti: pool_tx_info) { if (ti.id_hash == hash_string) { - double_spend_seen.insert(std::make_pair(h, ti.double_spend_seen)); + per_tx_pool_tx_info.insert(std::make_pair(h, ti)); break; } } @@ -614,7 +605,7 @@ namespace cryptonote LOG_PRINT_L2("Found " << found_in_pool << "/" << vh.size() << " transactions in the pool"); } - std::list::const_iterator txhi = req.txs_hashes.begin(); + std::vector::const_iterator txhi = req.txs_hashes.begin(); std::vector::const_iterator vhi = vh.begin(); for(auto& tx: txs) { @@ -623,22 +614,82 @@ namespace cryptonote crypto::hash tx_hash = *vhi++; e.tx_hash = *txhi++; - blobdata blob = req.prune ? get_pruned_tx_blob(tx) : t_serializable_object_to_blob(tx); - e.as_hex = string_tools::buff_to_hex_nodelimer(blob); - if (req.decode_as_json) - e.as_json = obj_to_json_str(tx); + e.prunable_hash = epee::string_tools::pod_to_hex(std::get<2>(tx)); + if (req.split || req.prune || std::get<3>(tx).empty()) + { + // use splitted form with pruned and prunable (filled only when prune=false and the daemon has it), leaving as_hex as empty + e.pruned_as_hex = string_tools::buff_to_hex_nodelimer(std::get<1>(tx)); + if (!req.prune) + e.prunable_as_hex = string_tools::buff_to_hex_nodelimer(std::get<3>(tx)); + if (req.decode_as_json) + { + cryptonote::blobdata tx_data; + cryptonote::transaction t; + if (req.prune || std::get<3>(tx).empty()) + { + // decode pruned tx to JSON + tx_data = std::get<1>(tx); + if (cryptonote::parse_and_validate_tx_base_from_blob(tx_data, t)) + { + pruned_transaction pruned_tx{t}; + e.as_json = obj_to_json_str(pruned_tx); + } + else + { + res.status = "Failed to parse and validate pruned tx from blob"; + return true; + } + } + else + { + // decode full tx to JSON + tx_data = std::get<1>(tx) + std::get<3>(tx); + if (cryptonote::parse_and_validate_tx_from_blob(tx_data, t)) + { + e.as_json = obj_to_json_str(t); + } + else + { + res.status = "Failed to parse and validate tx from blob"; + return true; + } + } + } + } + else + { + // use non-splitted form, leaving pruned_as_hex and prunable_as_hex as empty + cryptonote::blobdata tx_data = std::get<1>(tx) + std::get<3>(tx); + e.as_hex = string_tools::buff_to_hex_nodelimer(tx_data); + if (req.decode_as_json) + { + cryptonote::transaction t; + if (cryptonote::parse_and_validate_tx_from_blob(tx_data, t)) + { + e.as_json = obj_to_json_str(t); + } + else + { + res.status = "Failed to parse and validate tx from blob"; + return true; + } + } + } e.in_pool = pool_tx_hashes.find(tx_hash) != pool_tx_hashes.end(); if (e.in_pool) { e.block_height = e.block_timestamp = std::numeric_limits::max(); - if (double_spend_seen.find(tx_hash) != double_spend_seen.end()) + auto it = per_tx_pool_tx_info.find(tx_hash); + if (it != per_tx_pool_tx_info.end()) { - e.double_spend_seen = double_spend_seen[tx_hash]; + e.double_spend_seen = it->second.double_spend_seen; + e.relayed = it->second.relayed; } else { - MERROR("Failed to determine double spend status for " << tx_hash); + MERROR("Failed to determine pool info for " << tx_hash); e.double_spend_seen = false; + e.relayed = false; } } else @@ -646,6 +697,7 @@ namespace cryptonote e.block_height = m_core.get_blockchain_storage().get_db().get_tx_block_height(tx_hash); e.block_timestamp = m_core.get_blockchain_storage().get_db().get_block_timestamp(e.block_height); e.double_spend_seen = false; + e.relayed = false; } // fill up old style responses too, in case an old wallet asks @@ -675,13 +727,15 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_is_key_image_spent(const COMMAND_RPC_IS_KEY_IMAGE_SPENT::request& req, COMMAND_RPC_IS_KEY_IMAGE_SPENT::response& res, bool request_has_rpc_origin) + bool core_rpc_server::on_is_key_image_spent(const COMMAND_RPC_IS_KEY_IMAGE_SPENT::request& req, COMMAND_RPC_IS_KEY_IMAGE_SPENT::response& res, const connection_context *ctx) { PERF_TIMER(on_is_key_image_spent); bool ok; if (use_bootstrap_daemon_if_necessary(invoke_http_mode::JON, "/is_key_image_spent", req, res, ok)) return ok; + const bool restricted = m_restricted && ctx; + const bool request_has_rpc_origin = ctx != NULL; std::vector key_images; for(const auto& ki_hex_str: req.key_images) { @@ -711,7 +765,7 @@ namespace cryptonote // check the pool too std::vector txs; std::vector ki; - r = m_core.get_pool_transactions_and_spent_keys_info(txs, ki, !request_has_rpc_origin || !m_restricted); + r = m_core.get_pool_transactions_and_spent_keys_info(txs, ki, !request_has_rpc_origin || !restricted); if(!r) { res.status = "Failed"; @@ -742,7 +796,7 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_send_raw_tx(const COMMAND_RPC_SEND_RAW_TX::request& req, COMMAND_RPC_SEND_RAW_TX::response& res) + bool core_rpc_server::on_send_raw_tx(const COMMAND_RPC_SEND_RAW_TX::request& req, COMMAND_RPC_SEND_RAW_TX::response& res, const connection_context *ctx) { PERF_TIMER(on_send_raw_tx); bool ok; @@ -759,36 +813,45 @@ namespace cryptonote return true; } + if (req.do_sanity_checks && !cryptonote::tx_sanity_check(m_core.get_blockchain_storage(), tx_blob)) + { + res.status = "Failed"; + res.reason = "Sanity check failed"; + res.sanity_check_failed = true; + return true; + } + res.sanity_check_failed = false; + cryptonote_connection_context fake_context = AUTO_VAL_INIT(fake_context); tx_verification_context tvc = AUTO_VAL_INIT(tvc); if(!m_core.handle_incoming_tx(tx_blob, tvc, false, false, req.do_not_relay) || tvc.m_verifivation_failed) { res.status = "Failed"; - res.reason = ""; + std::string reason = ""; if ((res.low_mixin = tvc.m_low_mixin)) - add_reason(res.reason, "ring size too small"); + add_reason(reason, "bad ring size"); if ((res.double_spend = tvc.m_double_spend)) - add_reason(res.reason, "double spend"); + add_reason(reason, "double spend"); if ((res.invalid_input = tvc.m_invalid_input)) - add_reason(res.reason, "invalid input"); + add_reason(reason, "invalid input"); if ((res.invalid_output = tvc.m_invalid_output)) - add_reason(res.reason, "invalid output"); + add_reason(reason, "invalid output"); if ((res.too_big = tvc.m_too_big)) - add_reason(res.reason, "too big"); + add_reason(reason, "too big"); if ((res.overspend = tvc.m_overspend)) - add_reason(res.reason, "overspend"); + add_reason(reason, "overspend"); if ((res.fee_too_low = tvc.m_fee_too_low)) - add_reason(res.reason, "fee too low"); + add_reason(reason, "fee too low"); if ((res.not_rct = tvc.m_not_rct)) - add_reason(res.reason, "tx is not ringct"); - const std::string punctuation = res.reason.empty() ? "" : ": "; + add_reason(reason, "tx is not ringct"); + const std::string punctuation = reason.empty() ? "" : ": "; if (tvc.m_verifivation_failed) { - LOG_PRINT_L0("[on_send_raw_tx]: tx verification failed" << punctuation << res.reason); + LOG_PRINT_L0("[on_send_raw_tx]: tx verification failed" << punctuation << reason); } else { - LOG_PRINT_L0("[on_send_raw_tx]: Failed to process tx" << punctuation << res.reason); + LOG_PRINT_L0("[on_send_raw_tx]: Failed to process tx" << punctuation << reason); } return true; } @@ -810,12 +873,12 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_start_mining(const COMMAND_RPC_START_MINING::request& req, COMMAND_RPC_START_MINING::response& res) + bool core_rpc_server::on_start_mining(const COMMAND_RPC_START_MINING::request& req, COMMAND_RPC_START_MINING::response& res, const connection_context *ctx) { PERF_TIMER(on_start_mining); CHECK_CORE_READY(); cryptonote::address_parse_info info; - if(!get_account_address_from_str(info, m_nettype, req.miner_address)) + if(!get_account_address_from_str(info, nettype(), req.miner_address)) { res.status = "Failed, wrong address"; LOG_PRINT_L0(res.status); @@ -845,10 +908,13 @@ namespace cryptonote return true; } - boost::thread::attributes attrs; - attrs.set_stack_size(THREAD_STACK_SIZE); - - if(!m_core.get_miner().start(info.address, static_cast(req.threads_count), attrs, req.do_background_mining, req.ignore_battery)) + cryptonote::miner &miner= m_core.get_miner(); + if (miner.is_mining()) + { + res.status = "Already mining"; + return true; + } + if(!miner.start(info.address, static_cast(req.threads_count), req.do_background_mining, req.ignore_battery)) { res.status = "Failed, mining not started"; LOG_PRINT_L0(res.status); @@ -858,10 +924,17 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_stop_mining(const COMMAND_RPC_STOP_MINING::request& req, COMMAND_RPC_STOP_MINING::response& res) + bool core_rpc_server::on_stop_mining(const COMMAND_RPC_STOP_MINING::request& req, COMMAND_RPC_STOP_MINING::response& res, const connection_context *ctx) { PERF_TIMER(on_stop_mining); - if(!m_core.get_miner().stop()) + cryptonote::miner &miner= m_core.get_miner(); + if(!miner.is_mining()) + { + res.status = "Mining never started"; + LOG_PRINT_L0(res.status); + return true; + } + if(!miner.stop()) { res.status = "Failed, mining not stopped"; LOG_PRINT_L0(res.status); @@ -871,26 +944,46 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_mining_status(const COMMAND_RPC_MINING_STATUS::request& req, COMMAND_RPC_MINING_STATUS::response& res) + bool core_rpc_server::on_mining_status(const COMMAND_RPC_MINING_STATUS::request& req, COMMAND_RPC_MINING_STATUS::response& res, const connection_context *ctx) { PERF_TIMER(on_mining_status); const miner& lMiner = m_core.get_miner(); res.active = lMiner.is_mining(); res.is_background_mining_enabled = lMiner.get_is_background_mining_enabled(); + store_difficulty(m_core.get_blockchain_storage().get_difficulty_for_next_block(), res.difficulty, res.wide_difficulty, res.difficulty_top64); + res.block_target = m_core.get_blockchain_storage().get_current_hard_fork_version() < 2 ? DIFFICULTY_TARGET_V1 : DIFFICULTY_TARGET_V2; if ( lMiner.is_mining() ) { res.speed = lMiner.get_speed(); res.threads_count = lMiner.get_threads_count(); - const account_public_address& lMiningAdr = lMiner.get_mining_address(); - res.address = get_account_address_as_str(m_nettype, false, lMiningAdr); + res.block_reward = lMiner.get_block_reward(); + } + const account_public_address& lMiningAdr = lMiner.get_mining_address(); + res.address = get_account_address_as_str(nettype(), false, lMiningAdr); + const uint8_t major_version = m_core.get_blockchain_storage().get_current_hard_fork_version(); + const unsigned variant = major_version >= 7 ? major_version - 6 : 0; + switch (variant) + { + case 0: res.pow_algorithm = "Cryptonight"; break; + case 1: res.pow_algorithm = "CNv1 (Cryptonight variant 1)"; break; + case 2: case 3: res.pow_algorithm = "CNv2 (Cryptonight variant 2)"; break; + case 4: case 5: res.pow_algorithm = "CNv4 (Cryptonight variant 4)"; break; + default: res.pow_algorithm = "I'm not sure actually"; break; + } + if (res.is_background_mining_enabled) + { + res.bg_idle_threshold = lMiner.get_idle_threshold(); + res.bg_min_idle_seconds = lMiner.get_min_idle_seconds(); + res.bg_ignore_battery = lMiner.get_ignore_battery(); + res.bg_target = lMiner.get_mining_target(); } res.status = CORE_RPC_STATUS_OK; return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_save_bc(const COMMAND_RPC_SAVE_BC::request& req, COMMAND_RPC_SAVE_BC::response& res) + bool core_rpc_server::on_save_bc(const COMMAND_RPC_SAVE_BC::request& req, COMMAND_RPC_SAVE_BC::response& res, const connection_context *ctx) { PERF_TIMER(on_save_bc); if( !m_core.get_blockchain_storage().store_blockchain() ) @@ -902,37 +995,38 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_get_peer_list(const COMMAND_RPC_GET_PEER_LIST::request& req, COMMAND_RPC_GET_PEER_LIST::response& res) + bool core_rpc_server::on_get_peer_list(const COMMAND_RPC_GET_PEER_LIST::request& req, COMMAND_RPC_GET_PEER_LIST::response& res, const connection_context *ctx) { PERF_TIMER(on_get_peer_list); - std::list white_list; - std::list gray_list; - m_p2p.get_peerlist_manager().get_peerlist_full(gray_list, white_list); - + std::vector white_list; + std::vector gray_list; + m_p2p.get_public_peerlist(gray_list, white_list); + res.white_list.reserve(white_list.size()); for (auto & entry : white_list) { - if (entry.adr.get_type_id() == epee::net_utils::ipv4_network_address::ID) + if (entry.adr.get_type_id() == epee::net_utils::ipv4_network_address::get_type_id()) res.white_list.emplace_back(entry.id, entry.adr.as().ip(), - entry.adr.as().port(), entry.last_seen); + entry.adr.as().port(), entry.last_seen, entry.pruning_seed, entry.rpc_port); else - res.white_list.emplace_back(entry.id, entry.adr.str(), entry.last_seen); + res.white_list.emplace_back(entry.id, entry.adr.str(), entry.last_seen, entry.pruning_seed, entry.rpc_port); } + res.gray_list.reserve(gray_list.size()); for (auto & entry : gray_list) { - if (entry.adr.get_type_id() == epee::net_utils::ipv4_network_address::ID) + if (entry.adr.get_type_id() == epee::net_utils::ipv4_network_address::get_type_id()) res.gray_list.emplace_back(entry.id, entry.adr.as().ip(), - entry.adr.as().port(), entry.last_seen); + entry.adr.as().port(), entry.last_seen, entry.pruning_seed, entry.rpc_port); else - res.gray_list.emplace_back(entry.id, entry.adr.str(), entry.last_seen); + res.gray_list.emplace_back(entry.id, entry.adr.str(), entry.last_seen, entry.pruning_seed, entry.rpc_port); } res.status = CORE_RPC_STATUS_OK; return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_set_log_hash_rate(const COMMAND_RPC_SET_LOG_HASH_RATE::request& req, COMMAND_RPC_SET_LOG_HASH_RATE::response& res) + bool core_rpc_server::on_set_log_hash_rate(const COMMAND_RPC_SET_LOG_HASH_RATE::request& req, COMMAND_RPC_SET_LOG_HASH_RATE::response& res, const connection_context *ctx) { PERF_TIMER(on_set_log_hash_rate); if(m_core.get_miner().is_mining()) @@ -947,7 +1041,7 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_set_log_level(const COMMAND_RPC_SET_LOG_LEVEL::request& req, COMMAND_RPC_SET_LOG_LEVEL::response& res) + bool core_rpc_server::on_set_log_level(const COMMAND_RPC_SET_LOG_LEVEL::request& req, COMMAND_RPC_SET_LOG_LEVEL::response& res, const connection_context *ctx) { PERF_TIMER(on_set_log_level); if (req.level < 0 || req.level > 4) @@ -960,7 +1054,7 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_set_log_categories(const COMMAND_RPC_SET_LOG_CATEGORIES::request& req, COMMAND_RPC_SET_LOG_CATEGORIES::response& res) + bool core_rpc_server::on_set_log_categories(const COMMAND_RPC_SET_LOG_CATEGORIES::request& req, COMMAND_RPC_SET_LOG_CATEGORIES::response& res, const connection_context *ctx) { PERF_TIMER(on_set_log_categories); mlog_set_log(req.categories.c_str()); @@ -969,43 +1063,69 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_get_transaction_pool(const COMMAND_RPC_GET_TRANSACTION_POOL::request& req, COMMAND_RPC_GET_TRANSACTION_POOL::response& res, bool request_has_rpc_origin) + bool core_rpc_server::on_get_transaction_pool(const COMMAND_RPC_GET_TRANSACTION_POOL::request& req, COMMAND_RPC_GET_TRANSACTION_POOL::response& res, const connection_context *ctx) { PERF_TIMER(on_get_transaction_pool); bool r; if (use_bootstrap_daemon_if_necessary(invoke_http_mode::JON, "/get_transaction_pool", req, res, r)) return r; - m_core.get_pool_transactions_and_spent_keys_info(res.transactions, res.spent_key_images, !request_has_rpc_origin || !m_restricted); + const bool restricted = m_restricted && ctx; + const bool request_has_rpc_origin = ctx != NULL; + m_core.get_pool_transactions_and_spent_keys_info(res.transactions, res.spent_key_images, !request_has_rpc_origin || !restricted); + for (tx_info& txi : res.transactions) + txi.tx_blob = epee::string_tools::buff_to_hex_nodelimer(txi.tx_blob); + res.status = CORE_RPC_STATUS_OK; + return true; + } + //------------------------------------------------------------------------------------------------------------------------------ + bool core_rpc_server::on_get_transaction_pool_hashes_bin(const COMMAND_RPC_GET_TRANSACTION_POOL_HASHES_BIN::request& req, COMMAND_RPC_GET_TRANSACTION_POOL_HASHES_BIN::response& res, const connection_context *ctx) + { + PERF_TIMER(on_get_transaction_pool_hashes); + bool r; + if (use_bootstrap_daemon_if_necessary(invoke_http_mode::JON, "/get_transaction_pool_hashes.bin", req, res, r)) + return r; + + const bool restricted = m_restricted && ctx; + const bool request_has_rpc_origin = ctx != NULL; + m_core.get_pool_transaction_hashes(res.tx_hashes, !request_has_rpc_origin || !restricted); res.status = CORE_RPC_STATUS_OK; return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_get_transaction_pool_hashes(const COMMAND_RPC_GET_TRANSACTION_POOL_HASHES::request& req, COMMAND_RPC_GET_TRANSACTION_POOL_HASHES::response& res, bool request_has_rpc_origin) + bool core_rpc_server::on_get_transaction_pool_hashes(const COMMAND_RPC_GET_TRANSACTION_POOL_HASHES::request& req, COMMAND_RPC_GET_TRANSACTION_POOL_HASHES::response& res, const connection_context *ctx) { PERF_TIMER(on_get_transaction_pool_hashes); bool r; - if (use_bootstrap_daemon_if_necessary(invoke_http_mode::JON, "/get_transaction_pool_hashes.bin", req, res, r)) + if (use_bootstrap_daemon_if_necessary(invoke_http_mode::JON, "/get_transaction_pool_hashes", req, res, r)) return r; - m_core.get_pool_transaction_hashes(res.tx_hashes, !request_has_rpc_origin || !m_restricted); + const bool restricted = m_restricted && ctx; + const bool request_has_rpc_origin = ctx != NULL; + std::vector tx_hashes; + m_core.get_pool_transaction_hashes(tx_hashes, !request_has_rpc_origin || !restricted); + res.tx_hashes.reserve(tx_hashes.size()); + for (const crypto::hash &tx_hash: tx_hashes) + res.tx_hashes.push_back(epee::string_tools::pod_to_hex(tx_hash)); res.status = CORE_RPC_STATUS_OK; return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_get_transaction_pool_stats(const COMMAND_RPC_GET_TRANSACTION_POOL_STATS::request& req, COMMAND_RPC_GET_TRANSACTION_POOL_STATS::response& res, bool request_has_rpc_origin) + bool core_rpc_server::on_get_transaction_pool_stats(const COMMAND_RPC_GET_TRANSACTION_POOL_STATS::request& req, COMMAND_RPC_GET_TRANSACTION_POOL_STATS::response& res, const connection_context *ctx) { PERF_TIMER(on_get_transaction_pool_stats); bool r; if (use_bootstrap_daemon_if_necessary(invoke_http_mode::JON, "/get_transaction_pool_stats", req, res, r)) return r; - m_core.get_pool_transaction_stats(res.pool_stats, !request_has_rpc_origin || !m_restricted); + const bool restricted = m_restricted && ctx; + const bool request_has_rpc_origin = ctx != NULL; + m_core.get_pool_transaction_stats(res.pool_stats, !request_has_rpc_origin || !restricted); res.status = CORE_RPC_STATUS_OK; return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_stop_daemon(const COMMAND_RPC_STOP_DAEMON::request& req, COMMAND_RPC_STOP_DAEMON::response& res) + bool core_rpc_server::on_stop_daemon(const COMMAND_RPC_STOP_DAEMON::request& req, COMMAND_RPC_STOP_DAEMON::response& res, const connection_context *ctx) { PERF_TIMER(on_stop_daemon); // FIXME: replace back to original m_p2p.send_stop_signal() after @@ -1015,7 +1135,7 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_getblockcount(const COMMAND_RPC_GETBLOCKCOUNT::request& req, COMMAND_RPC_GETBLOCKCOUNT::response& res) + bool core_rpc_server::on_getblockcount(const COMMAND_RPC_GETBLOCKCOUNT::request& req, COMMAND_RPC_GETBLOCKCOUNT::response& res, const connection_context *ctx) { PERF_TIMER(on_getblockcount); { @@ -1031,7 +1151,7 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_getblockhash(const COMMAND_RPC_GETBLOCKHASH::request& req, COMMAND_RPC_GETBLOCKHASH::response& res, epee::json_rpc::error& error_resp) + bool core_rpc_server::on_getblockhash(const COMMAND_RPC_GETBLOCKHASH::request& req, COMMAND_RPC_GETBLOCKHASH::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx) { PERF_TIMER(on_getblockhash); { @@ -1052,7 +1172,7 @@ namespace cryptonote if(m_core.get_current_blockchain_height() <= h) { error_resp.code = CORE_RPC_ERROR_CODE_TOO_BIG_HEIGHT; - error_resp.message = std::string("Too big height: ") + std::to_string(h) + ", current blockchain height = " + std::to_string(m_core.get_current_blockchain_height()); + error_resp.message = std::string("Requested block height: ") + std::to_string(h) + " greater than current top block height: " + std::to_string(m_core.get_current_blockchain_height() - 1); } res = string_tools::pod_to_hex(m_core.get_block_id_by_height(h)); return true; @@ -1060,7 +1180,7 @@ namespace cryptonote //------------------------------------------------------------------------------------------------------------------------------ // equivalent of strstr, but with arbitrary bytes (ie, NULs) // This does not differentiate between "not found" and "found at offset 0" - uint64_t slow_memmem(const void* start_buff, size_t buflen,const void* pat,size_t patlen) + size_t slow_memmem(const void* start_buff, size_t buflen,const void* pat,size_t patlen) { const void* buf = start_buff; const void* end=(const char*)buf+buflen; @@ -1075,7 +1195,7 @@ namespace cryptonote return 0; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_getblocktemplate(const COMMAND_RPC_GETBLOCKTEMPLATE::request& req, COMMAND_RPC_GETBLOCKTEMPLATE::response& res, epee::json_rpc::error& error_resp) + bool core_rpc_server::on_getblocktemplate(const COMMAND_RPC_GETBLOCKTEMPLATE::request& req, COMMAND_RPC_GETBLOCKTEMPLATE::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx) { PERF_TIMER(on_getblocktemplate); bool r; @@ -1098,7 +1218,7 @@ namespace cryptonote cryptonote::address_parse_info info; - if(!req.wallet_address.size() || !cryptonote::get_account_address_from_str(info, m_nettype, req.wallet_address)) + if(!req.wallet_address.size() || !cryptonote::get_account_address_from_str(info, nettype(), req.wallet_address)) { error_resp.code = CORE_RPC_ERROR_CODE_WRONG_WALLET_ADDRESS; error_resp.message = "Failed to parse wallet address"; @@ -1111,23 +1231,35 @@ namespace cryptonote return false; } - block b = AUTO_VAL_INIT(b); + block b; cryptonote::blobdata blob_reserve; blob_reserve.resize(req.reserve_size, 0); - if(!m_core.get_block_template(b, info.address, res.difficulty, res.height, res.expected_reward, blob_reserve)) + cryptonote::difficulty_type wdiff; + crypto::hash prev_block; + if (!req.prev_block.empty()) + { + if (!epee::string_tools::hex_to_pod(req.prev_block, prev_block)) + { + error_resp.code = CORE_RPC_ERROR_CODE_INTERNAL_ERROR; + error_resp.message = "Invalid prev_block"; + return false; + } + } + if(!m_core.get_block_template(b, req.prev_block.empty() ? NULL : &prev_block, info.address, wdiff, res.height, res.expected_reward, blob_reserve)) { error_resp.code = CORE_RPC_ERROR_CODE_INTERNAL_ERROR; error_resp.message = "Internal error: failed to create block template"; LOG_ERROR("Failed to create block template"); return false; } + store_difficulty(wdiff, res.difficulty, res.wide_difficulty, res.difficulty_top64); blobdata block_blob = t_serializable_object_to_blob(b); crypto::public_key tx_pub_key = cryptonote::get_tx_pub_key_from_extra(b.miner_tx); if(tx_pub_key == crypto::null_pkey) { error_resp.code = CORE_RPC_ERROR_CODE_INTERNAL_ERROR; error_resp.message = "Internal error: failed to create block template"; - LOG_ERROR("Failed to tx pub key in coinbase extra"); + LOG_ERROR("Failed to get tx pub key in coinbase extra"); return false; } res.reserved_offset = slow_memmem((void*)block_blob.data(), block_blob.size(), &tx_pub_key, sizeof(tx_pub_key)); @@ -1138,7 +1270,10 @@ namespace cryptonote LOG_ERROR("Failed to find tx pub key in blockblob"); return false; } - res.reserved_offset += sizeof(tx_pub_key) + 2; //2 bytes: tag for TX_EXTRA_NONCE(1 byte), counter in TX_EXTRA_NONCE(1 byte) + if (req.reserve_size) + res.reserved_offset += sizeof(tx_pub_key) + 2; //2 bytes: tag for TX_EXTRA_NONCE(1 byte), counter in TX_EXTRA_NONCE(1 byte) + else + res.reserved_offset = 0; if(res.reserved_offset + req.reserve_size > block_blob.size()) { error_resp.code = CORE_RPC_ERROR_CODE_INTERNAL_ERROR; @@ -1154,7 +1289,7 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_submitblock(const COMMAND_RPC_SUBMITBLOCK::request& req, COMMAND_RPC_SUBMITBLOCK::response& res, epee::json_rpc::error& error_resp) + bool core_rpc_server::on_submitblock(const COMMAND_RPC_SUBMITBLOCK::request& req, COMMAND_RPC_SUBMITBLOCK::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx) { PERF_TIMER(on_submitblock); { @@ -1182,7 +1317,7 @@ namespace cryptonote // Fixing of high orphan issue for most pools // Thanks Boolberry! - block b = AUTO_VAL_INIT(b); + block b; if(!parse_and_validate_block_from_blob(blockblob, b)) { error_resp.code = CORE_RPC_ERROR_CODE_WRONG_BLOCKBLOB; @@ -1199,7 +1334,8 @@ namespace cryptonote return false; } - if(!m_core.handle_block_found(b)) + block_verification_context bvc; + if(!m_core.handle_block_found(b, bvc)) { error_resp.code = CORE_RPC_ERROR_CODE_BLOCK_NOT_ACCEPTED; error_resp.message = "Block not accepted"; @@ -1209,6 +1345,71 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ + bool core_rpc_server::on_generateblocks(const COMMAND_RPC_GENERATEBLOCKS::request& req, COMMAND_RPC_GENERATEBLOCKS::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx) + { + PERF_TIMER(on_generateblocks); + + CHECK_CORE_READY(); + + res.status = CORE_RPC_STATUS_OK; + + if(m_core.get_nettype() != FAKECHAIN) + { + error_resp.code = CORE_RPC_ERROR_CODE_REGTEST_REQUIRED; + error_resp.message = "Regtest required when generating blocks"; + return false; + } + + COMMAND_RPC_GETBLOCKTEMPLATE::request template_req; + COMMAND_RPC_GETBLOCKTEMPLATE::response template_res; + COMMAND_RPC_SUBMITBLOCK::request submit_req; + COMMAND_RPC_SUBMITBLOCK::response submit_res; + + template_req.reserve_size = 1; + template_req.wallet_address = req.wallet_address; + template_req.prev_block = req.prev_block; + submit_req.push_back(boost::value_initialized()); + res.height = m_core.get_blockchain_storage().get_current_blockchain_height(); + + for(size_t i = 0; i < req.amount_of_blocks; i++) + { + bool r = on_getblocktemplate(template_req, template_res, error_resp, ctx); + res.status = template_res.status; + template_req.prev_block.clear(); + + if (!r) return false; + + blobdata blockblob; + if(!string_tools::parse_hexstr_to_binbuff(template_res.blocktemplate_blob, blockblob)) + { + error_resp.code = CORE_RPC_ERROR_CODE_WRONG_BLOCKBLOB; + error_resp.message = "Wrong block blob"; + return false; + } + block b; + if(!parse_and_validate_block_from_blob(blockblob, b)) + { + error_resp.code = CORE_RPC_ERROR_CODE_WRONG_BLOCKBLOB; + error_resp.message = "Wrong block blob"; + return false; + } + b.nonce = req.starting_nonce; + miner::find_nonce_for_given_block(b, template_res.difficulty, template_res.height); + + submit_req.front() = string_tools::buff_to_hex_nodelimer(block_to_blob(b)); + r = on_submitblock(submit_req, submit_res, error_resp, ctx); + res.status = submit_res.status; + + if (!r) return false; + + res.blocks.push_back(epee::string_tools::pod_to_hex(get_block_hash(b))); + template_req.prev_block = res.blocks.back(); + res.height = template_res.height; + } + + return true; + } + //------------------------------------------------------------------------------------------------------------------------------ uint64_t core_rpc_server::get_block_reward(const block& blk) { uint64_t reward = 0; @@ -1231,11 +1432,16 @@ namespace cryptonote response.height = height; response.depth = m_core.get_current_blockchain_height() - height - 1; response.hash = string_tools::pod_to_hex(hash); - response.difficulty = m_core.get_blockchain_storage().block_difficulty(height); + store_difficulty(m_core.get_blockchain_storage().block_difficulty(height), + response.difficulty, response.wide_difficulty, response.difficulty_top64); + store_difficulty(m_core.get_blockchain_storage().get_db().get_block_cumulative_difficulty(height), + response.cumulative_difficulty, response.wide_cumulative_difficulty, response.cumulative_difficulty_top64); response.reward = get_block_reward(blk); - response.block_size = m_core.get_blockchain_storage().get_db().get_block_size(height); + response.block_size = response.block_weight = m_core.get_blockchain_storage().get_db().get_block_weight(height); response.num_txes = blk.tx_hashes.size(); response.pow_hash = fill_pow_hash ? string_tools::pod_to_hex(get_block_longhash(blk, height)) : ""; + response.long_term_weight = m_core.get_blockchain_storage().get_db().get_block_long_term_weight(height); + response.miner_tx_hash = string_tools::pod_to_hex(cryptonote::get_transaction_hash(blk.miner_tx)); return true; } //------------------------------------------------------------------------------------------------------------------------------ @@ -1306,7 +1512,7 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_get_last_block_header(const COMMAND_RPC_GET_LAST_BLOCK_HEADER::request& req, COMMAND_RPC_GET_LAST_BLOCK_HEADER::response& res, epee::json_rpc::error& error_resp) + bool core_rpc_server::on_get_last_block_header(const COMMAND_RPC_GET_LAST_BLOCK_HEADER::request& req, COMMAND_RPC_GET_LAST_BLOCK_HEADER::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx) { PERF_TIMER(on_get_last_block_header); bool r; @@ -1325,7 +1531,8 @@ namespace cryptonote error_resp.message = "Internal error: can't get last block."; return false; } - bool response_filled = fill_block_header_response(last_block, false, last_block_height, last_block_hash, res.block_header, req.fill_pow_hash); + const bool restricted = m_restricted && ctx; + bool response_filled = fill_block_header_response(last_block, false, last_block_height, last_block_hash, res.block_header, req.fill_pow_hash && !restricted); if (!response_filled) { error_resp.code = CORE_RPC_ERROR_CODE_INTERNAL_ERROR; @@ -1336,7 +1543,8 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_get_block_header_by_hash(const COMMAND_RPC_GET_BLOCK_HEADER_BY_HASH::request& req, COMMAND_RPC_GET_BLOCK_HEADER_BY_HASH::response& res, epee::json_rpc::error& error_resp){ + bool core_rpc_server::on_get_block_header_by_hash(const COMMAND_RPC_GET_BLOCK_HEADER_BY_HASH::request& req, COMMAND_RPC_GET_BLOCK_HEADER_BY_HASH::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx) + { PERF_TIMER(on_get_block_header_by_hash); bool r; if (use_bootstrap_daemon_if_necessary(invoke_http_mode::JON_RPC, "getblockheaderbyhash", req, res, r)) @@ -1366,7 +1574,8 @@ namespace cryptonote return false; } uint64_t block_height = boost::get(blk.miner_tx.vin.front()).height; - bool response_filled = fill_block_header_response(blk, orphan, block_height, block_hash, res.block_header, req.fill_pow_hash); + const bool restricted = m_restricted && ctx; + bool response_filled = fill_block_header_response(blk, orphan, block_height, block_hash, res.block_header, req.fill_pow_hash && !restricted); if (!response_filled) { error_resp.code = CORE_RPC_ERROR_CODE_INTERNAL_ERROR; @@ -1377,7 +1586,8 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_get_block_headers_range(const COMMAND_RPC_GET_BLOCK_HEADERS_RANGE::request& req, COMMAND_RPC_GET_BLOCK_HEADERS_RANGE::response& res, epee::json_rpc::error& error_resp){ + bool core_rpc_server::on_get_block_headers_range(const COMMAND_RPC_GET_BLOCK_HEADERS_RANGE::request& req, COMMAND_RPC_GET_BLOCK_HEADERS_RANGE::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx) + { PERF_TIMER(on_get_block_headers_range); bool r; if (use_bootstrap_daemon_if_necessary(invoke_http_mode::JON_RPC, "getblockheadersrange", req, res, r)) @@ -1415,7 +1625,8 @@ namespace cryptonote return false; } res.headers.push_back(block_header_response()); - bool response_filled = fill_block_header_response(blk, false, block_height, block_hash, res.headers.back(), req.fill_pow_hash); + const bool restricted = m_restricted && ctx; + bool response_filled = fill_block_header_response(blk, false, block_height, block_hash, res.headers.back(), req.fill_pow_hash && !restricted); if (!response_filled) { error_resp.code = CORE_RPC_ERROR_CODE_INTERNAL_ERROR; @@ -1427,7 +1638,8 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_get_block_header_by_height(const COMMAND_RPC_GET_BLOCK_HEADER_BY_HEIGHT::request& req, COMMAND_RPC_GET_BLOCK_HEADER_BY_HEIGHT::response& res, epee::json_rpc::error& error_resp){ + bool core_rpc_server::on_get_block_header_by_height(const COMMAND_RPC_GET_BLOCK_HEADER_BY_HEIGHT::request& req, COMMAND_RPC_GET_BLOCK_HEADER_BY_HEIGHT::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx) + { PERF_TIMER(on_get_block_header_by_height); bool r; if (use_bootstrap_daemon_if_necessary(invoke_http_mode::JON_RPC, "getblockheaderbyheight", req, res, r)) @@ -1436,7 +1648,7 @@ namespace cryptonote if(m_core.get_current_blockchain_height() <= req.height) { error_resp.code = CORE_RPC_ERROR_CODE_TOO_BIG_HEIGHT; - error_resp.message = std::string("Too big height: ") + std::to_string(req.height) + ", current blockchain height = " + std::to_string(m_core.get_current_blockchain_height()); + error_resp.message = std::string("Requested block height: ") + std::to_string(req.height) + " greater than current top block height: " + std::to_string(m_core.get_current_blockchain_height() - 1); return false; } crypto::hash block_hash = m_core.get_block_id_by_height(req.height); @@ -1448,7 +1660,8 @@ namespace cryptonote error_resp.message = "Internal error: can't get block by height. Height = " + std::to_string(req.height) + '.'; return false; } - bool response_filled = fill_block_header_response(blk, false, req.height, block_hash, res.block_header, req.fill_pow_hash); + const bool restricted = m_restricted && ctx; + bool response_filled = fill_block_header_response(blk, false, req.height, block_hash, res.block_header, req.fill_pow_hash && !restricted); if (!response_filled) { error_resp.code = CORE_RPC_ERROR_CODE_INTERNAL_ERROR; @@ -1459,7 +1672,8 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_get_block(const COMMAND_RPC_GET_BLOCK::request& req, COMMAND_RPC_GET_BLOCK::response& res, epee::json_rpc::error& error_resp){ + bool core_rpc_server::on_get_block(const COMMAND_RPC_GET_BLOCK::request& req, COMMAND_RPC_GET_BLOCK::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx) + { PERF_TIMER(on_get_block); bool r; if (use_bootstrap_daemon_if_necessary(invoke_http_mode::JON_RPC, "getblock", req, res, r)) @@ -1481,7 +1695,7 @@ namespace cryptonote if(m_core.get_current_blockchain_height() <= req.height) { error_resp.code = CORE_RPC_ERROR_CODE_TOO_BIG_HEIGHT; - error_resp.message = std::string("Too big height: ") + std::to_string(req.height) + ", current blockchain height = " + std::to_string(m_core.get_current_blockchain_height()); + error_resp.message = std::string("Requested block height: ") + std::to_string(req.height) + " greater than current top block height: " + std::to_string(m_core.get_current_blockchain_height() - 1); return false; } block_hash = m_core.get_block_id_by_height(req.height); @@ -1502,7 +1716,8 @@ namespace cryptonote return false; } uint64_t block_height = boost::get(blk.miner_tx.vin.front()).height; - bool response_filled = fill_block_header_response(blk, orphan, block_height, block_hash, res.block_header, req.fill_pow_hash); + const bool restricted = m_restricted && ctx; + bool response_filled = fill_block_header_response(blk, orphan, block_height, block_hash, res.block_header, req.fill_pow_hash && !restricted); if (!response_filled) { error_resp.code = CORE_RPC_ERROR_CODE_INTERNAL_ERROR; @@ -1520,7 +1735,7 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_get_connections(const COMMAND_RPC_GET_CONNECTIONS::request& req, COMMAND_RPC_GET_CONNECTIONS::response& res, epee::json_rpc::error& error_resp) + bool core_rpc_server::on_get_connections(const COMMAND_RPC_GET_CONNECTIONS::request& req, COMMAND_RPC_GET_CONNECTIONS::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx) { PERF_TIMER(on_get_connections); @@ -1531,56 +1746,12 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_get_info_json(const COMMAND_RPC_GET_INFO::request& req, COMMAND_RPC_GET_INFO::response& res, epee::json_rpc::error& error_resp) + bool core_rpc_server::on_get_info_json(const COMMAND_RPC_GET_INFO::request& req, COMMAND_RPC_GET_INFO::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx) { - PERF_TIMER(on_get_info_json); - bool r; - if (use_bootstrap_daemon_if_necessary(invoke_http_mode::JON_RPC, "get_info", req, res, r)) - { - res.bootstrap_daemon_address = m_bootstrap_daemon_address; - crypto::hash top_hash; - m_core.get_blockchain_top(res.height_without_bootstrap, top_hash); - ++res.height_without_bootstrap; // turn top block height into blockchain height - res.was_bootstrap_ever_used = true; - return r; - } - - crypto::hash top_hash; - m_core.get_blockchain_top(res.height, top_hash); - ++res.height; // turn top block height into blockchain height - res.top_block_hash = string_tools::pod_to_hex(top_hash); - res.target_height = m_core.get_target_blockchain_height(); - res.difficulty = m_core.get_blockchain_storage().get_difficulty_for_next_block(); - res.target = m_core.get_blockchain_storage().get_current_hard_fork_version() < 2 ? DIFFICULTY_TARGET_V1 : DIFFICULTY_TARGET_V2; - res.tx_count = m_core.get_blockchain_storage().get_total_transactions() - res.height; //without coinbase - res.tx_pool_size = m_core.get_pool_transactions_count(); - res.alt_blocks_count = m_core.get_blockchain_storage().get_alternative_blocks_count(); - uint64_t total_conn = m_p2p.get_connections_count(); - res.outgoing_connections_count = m_p2p.get_outgoing_connections_count(); - res.incoming_connections_count = total_conn - res.outgoing_connections_count; - res.rpc_connections_count = get_connections_count(); - res.white_peerlist_size = m_p2p.get_peerlist_manager().get_white_peers_count(); - res.grey_peerlist_size = m_p2p.get_peerlist_manager().get_gray_peers_count(); - res.mainnet = m_nettype == MAINNET; - res.testnet = m_nettype == TESTNET; - res.stagenet = m_nettype == STAGENET; - res.cumulative_difficulty = m_core.get_blockchain_storage().get_db().get_block_cumulative_difficulty(res.height - 1); - res.block_size_limit = m_core.get_blockchain_storage().get_current_cumulative_blocksize_limit(); - res.block_size_median = m_core.get_blockchain_storage().get_current_cumulative_blocksize_median(); - res.status = CORE_RPC_STATUS_OK; - res.start_time = (uint64_t)m_core.get_start_time(); - res.free_space = m_restricted ? std::numeric_limits::max() : m_core.get_free_space(); - res.offline = m_core.offline(); - res.bootstrap_daemon_address = m_bootstrap_daemon_address; - res.height_without_bootstrap = res.height; - { - boost::shared_lock lock(m_bootstrap_daemon_mutex); - res.was_bootstrap_ever_used = m_was_bootstrap_ever_used; - } - return true; + return on_get_info(req, res, ctx); } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_hard_fork_info(const COMMAND_RPC_HARD_FORK_INFO::request& req, COMMAND_RPC_HARD_FORK_INFO::response& res, epee::json_rpc::error& error_resp) + bool core_rpc_server::on_hard_fork_info(const COMMAND_RPC_HARD_FORK_INFO::request& req, COMMAND_RPC_HARD_FORK_INFO::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx) { PERF_TIMER(on_hard_fork_info); bool r; @@ -1596,7 +1767,7 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_get_bans(const COMMAND_RPC_GETBANS::request& req, COMMAND_RPC_GETBANS::response& res, epee::json_rpc::error& error_resp) + bool core_rpc_server::on_get_bans(const COMMAND_RPC_GETBANS::request& req, COMMAND_RPC_GETBANS::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx) { PERF_TIMER(on_get_bans); @@ -1620,7 +1791,7 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_set_bans(const COMMAND_RPC_SETBANS::request& req, COMMAND_RPC_SETBANS::response& res, epee::json_rpc::error& error_resp) + bool core_rpc_server::on_set_bans(const COMMAND_RPC_SETBANS::request& req, COMMAND_RPC_SETBANS::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx) { PERF_TIMER(on_set_bans); @@ -1629,12 +1800,14 @@ namespace cryptonote epee::net_utils::network_address na; if (!i->host.empty()) { - if (!epee::net_utils::create_network_address(na, i->host)) + auto na_parsed = net::get_network_address(i->host, 0); + if (!na_parsed) { error_resp.code = CORE_RPC_ERROR_CODE_WRONG_PARAM; error_resp.message = "Unsupported host type"; return false; } + na = std::move(*na_parsed); } else { @@ -1650,15 +1823,15 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_flush_txpool(const COMMAND_RPC_FLUSH_TRANSACTION_POOL::request& req, COMMAND_RPC_FLUSH_TRANSACTION_POOL::response& res, epee::json_rpc::error& error_resp) + bool core_rpc_server::on_flush_txpool(const COMMAND_RPC_FLUSH_TRANSACTION_POOL::request& req, COMMAND_RPC_FLUSH_TRANSACTION_POOL::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx) { PERF_TIMER(on_flush_txpool); bool failed = false; - std::list txids; + std::vector txids; if (req.txids.empty()) { - std::list pool_txs; + std::vector pool_txs; bool r = m_core.get_pool_transactions(pool_txs); if (!r) { @@ -1705,13 +1878,20 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_get_output_histogram(const COMMAND_RPC_GET_OUTPUT_HISTOGRAM::request& req, COMMAND_RPC_GET_OUTPUT_HISTOGRAM::response& res, epee::json_rpc::error& error_resp) + bool core_rpc_server::on_get_output_histogram(const COMMAND_RPC_GET_OUTPUT_HISTOGRAM::request& req, COMMAND_RPC_GET_OUTPUT_HISTOGRAM::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx) { PERF_TIMER(on_get_output_histogram); bool r; if (use_bootstrap_daemon_if_necessary(invoke_http_mode::JON_RPC, "get_output_histogram", req, res, r)) return r; + const bool restricted = m_restricted && ctx; + if (restricted && req.recent_cutoff > 0 && req.recent_cutoff < (uint64_t)time(NULL) - OUTPUT_HISTOGRAM_RECENT_CUTOFF_RESTRICTION) + { + res.status = "Recent cutoff is too old"; + return true; + } + std::map> histogram; try { @@ -1735,7 +1915,7 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_get_version(const COMMAND_RPC_GET_VERSION::request& req, COMMAND_RPC_GET_VERSION::response& res, epee::json_rpc::error& error_resp) + bool core_rpc_server::on_get_version(const COMMAND_RPC_GET_VERSION::request& req, COMMAND_RPC_GET_VERSION::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx) { PERF_TIMER(on_get_version); bool r; @@ -1747,7 +1927,7 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_get_coinbase_tx_sum(const COMMAND_RPC_GET_COINBASE_TX_SUM::request& req, COMMAND_RPC_GET_COINBASE_TX_SUM::response& res, epee::json_rpc::error& error_resp) + bool core_rpc_server::on_get_coinbase_tx_sum(const COMMAND_RPC_GET_COINBASE_TX_SUM::request& req, COMMAND_RPC_GET_COINBASE_TX_SUM::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx) { PERF_TIMER(on_get_coinbase_tx_sum); std::pair amounts = m_core.get_coinbase_tx_sum(req.height, req.count); @@ -1757,27 +1937,42 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_get_per_kb_fee_estimate(const COMMAND_RPC_GET_PER_KB_FEE_ESTIMATE::request& req, COMMAND_RPC_GET_PER_KB_FEE_ESTIMATE::response& res, epee::json_rpc::error& error_resp) + bool core_rpc_server::on_get_base_fee_estimate(const COMMAND_RPC_GET_BASE_FEE_ESTIMATE::request& req, COMMAND_RPC_GET_BASE_FEE_ESTIMATE::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx) { - PERF_TIMER(on_get_per_kb_fee_estimate); + PERF_TIMER(on_get_base_fee_estimate); bool r; - if (use_bootstrap_daemon_if_necessary(invoke_http_mode::JON_RPC, "get_fee_estimate", req, res, r)) + if (use_bootstrap_daemon_if_necessary(invoke_http_mode::JON_RPC, "get_fee_estimate", req, res, r)) return r; - res.fee = m_core.get_blockchain_storage().get_dynamic_per_kb_fee_estimate(req.grace_blocks); + res.fee = m_core.get_blockchain_storage().get_dynamic_base_fee_estimate(req.grace_blocks); + res.quantization_mask = Blockchain::get_fee_quantization_mask(); res.status = CORE_RPC_STATUS_OK; return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_get_alternate_chains(const COMMAND_RPC_GET_ALTERNATE_CHAINS::request& req, COMMAND_RPC_GET_ALTERNATE_CHAINS::response& res, epee::json_rpc::error& error_resp) + bool core_rpc_server::on_get_alternate_chains(const COMMAND_RPC_GET_ALTERNATE_CHAINS::request& req, COMMAND_RPC_GET_ALTERNATE_CHAINS::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx) { PERF_TIMER(on_get_alternate_chains); try { - std::list> chains = m_core.get_blockchain_storage().get_alternative_chains(); + std::list>> chains = m_core.get_blockchain_storage().get_alternative_chains(); for (const auto &i: chains) { - res.chains.push_back(COMMAND_RPC_GET_ALTERNATE_CHAINS::chain_info{epee::string_tools::pod_to_hex(get_block_hash(i.first.bl)), i.first.height, i.second, i.first.cumulative_difficulty}); + difficulty_type wdiff = i.first.cumulative_difficulty; + res.chains.push_back(COMMAND_RPC_GET_ALTERNATE_CHAINS::chain_info{epee::string_tools::pod_to_hex(get_block_hash(i.first.bl)), i.first.height, i.second.size(), 0, "", 0, {}, std::string()}); + store_difficulty(wdiff, res.chains.back().difficulty, res.chains.back().wide_difficulty, res.chains.back().difficulty_top64); + res.chains.back().block_hashes.reserve(i.second.size()); + for (const crypto::hash &block_id: i.second) + res.chains.back().block_hashes.push_back(epee::string_tools::pod_to_hex(block_id)); + if (i.first.height < i.second.size()) + { + res.status = "Error finding alternate chain attachment point"; + return true; + } + cryptonote::block main_chain_parent_block; + try { main_chain_parent_block = m_core.get_blockchain_storage().get_db().get_block_from_height(i.first.height - i.second.size()); } + catch (const std::exception &e) { res.status = "Error finding alternate chain attachment point"; return true; } + res.chains.back().main_chain_parent_block = epee::string_tools::pod_to_hex(get_block_hash(main_chain_parent_block)); } res.status = CORE_RPC_STATUS_OK; } @@ -1788,7 +1983,7 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_get_limit(const COMMAND_RPC_GET_LIMIT::request& req, COMMAND_RPC_GET_LIMIT::response& res) + bool core_rpc_server::on_get_limit(const COMMAND_RPC_GET_LIMIT::request& req, COMMAND_RPC_GET_LIMIT::response& res, const connection_context *ctx) { PERF_TIMER(on_get_limit); bool r; @@ -1801,7 +1996,7 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_set_limit(const COMMAND_RPC_SET_LIMIT::request& req, COMMAND_RPC_SET_LIMIT::response& res) + bool core_rpc_server::on_set_limit(const COMMAND_RPC_SET_LIMIT::request& req, COMMAND_RPC_SET_LIMIT::response& res, const connection_context *ctx) { PERF_TIMER(on_set_limit); // -1 = reset to default @@ -1841,31 +2036,23 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_out_peers(const COMMAND_RPC_OUT_PEERS::request& req, COMMAND_RPC_OUT_PEERS::response& res) + bool core_rpc_server::on_out_peers(const COMMAND_RPC_OUT_PEERS::request& req, COMMAND_RPC_OUT_PEERS::response& res, const connection_context *ctx) { PERF_TIMER(on_out_peers); - size_t n_connections = m_p2p.get_outgoing_connections_count(); - size_t n_delete = (n_connections > req.out_peers) ? n_connections - req.out_peers : 0; - m_p2p.m_config.m_net_config.max_out_connection_count = req.out_peers; - if (n_delete) - m_p2p.delete_out_connections(n_delete); + m_p2p.change_max_out_public_peers(req.out_peers); res.status = CORE_RPC_STATUS_OK; return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_in_peers(const COMMAND_RPC_IN_PEERS::request& req, COMMAND_RPC_IN_PEERS::response& res) + bool core_rpc_server::on_in_peers(const COMMAND_RPC_IN_PEERS::request& req, COMMAND_RPC_IN_PEERS::response& res, const connection_context *ctx) { PERF_TIMER(on_in_peers); - size_t n_connections = m_p2p.get_incoming_connections_count(); - size_t n_delete = (n_connections > req.in_peers) ? n_connections - req.in_peers : 0; - m_p2p.m_config.m_net_config.max_in_connection_count = req.in_peers; - if (n_delete) - m_p2p.delete_in_connections(n_delete); + m_p2p.change_max_in_public_peers(req.in_peers); res.status = CORE_RPC_STATUS_OK; return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_start_save_graph(const COMMAND_RPC_START_SAVE_GRAPH::request& req, COMMAND_RPC_START_SAVE_GRAPH::response& res) + bool core_rpc_server::on_start_save_graph(const COMMAND_RPC_START_SAVE_GRAPH::request& req, COMMAND_RPC_START_SAVE_GRAPH::response& res, const connection_context *ctx) { PERF_TIMER(on_start_save_graph); m_p2p.set_save_graph(true); @@ -1873,7 +2060,7 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_stop_save_graph(const COMMAND_RPC_STOP_SAVE_GRAPH::request& req, COMMAND_RPC_STOP_SAVE_GRAPH::response& res) + bool core_rpc_server::on_stop_save_graph(const COMMAND_RPC_STOP_SAVE_GRAPH::request& req, COMMAND_RPC_STOP_SAVE_GRAPH::response& res, const connection_context *ctx) { PERF_TIMER(on_stop_save_graph); m_p2p.set_save_graph(false); @@ -1881,9 +2068,16 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_update(const COMMAND_RPC_UPDATE::request& req, COMMAND_RPC_UPDATE::response& res) + bool core_rpc_server::on_update(const COMMAND_RPC_UPDATE::request& req, COMMAND_RPC_UPDATE::response& res, const connection_context *ctx) { PERF_TIMER(on_update); + + if (m_core.offline()) + { + res.status = "Daemon is running offline"; + return true; + } + static const char software[] = "monero"; #ifdef BUILD_TAG static const char buildtag[] = BOOST_PP_STRINGIZE(BUILD_TAG); @@ -1976,7 +2170,19 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_relay_tx(const COMMAND_RPC_RELAY_TX::request& req, COMMAND_RPC_RELAY_TX::response& res, epee::json_rpc::error& error_resp) + bool core_rpc_server::on_pop_blocks(const COMMAND_RPC_POP_BLOCKS::request& req, COMMAND_RPC_POP_BLOCKS::response& res, const connection_context *ctx) + { + PERF_TIMER(on_pop_blocks); + + m_core.get_blockchain_storage().pop_blocks(req.nblocks); + + res.height = m_core.get_current_blockchain_height(); + res.status = CORE_RPC_STATUS_OK; + + return true; + } + //------------------------------------------------------------------------------------------------------------------------------ + bool core_rpc_server::on_relay_tx(const COMMAND_RPC_RELAY_TX::request& req, COMMAND_RPC_RELAY_TX::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx) { PERF_TIMER(on_relay_tx); @@ -2022,7 +2228,7 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_sync_info(const COMMAND_RPC_SYNC_INFO::request& req, COMMAND_RPC_SYNC_INFO::response& res, epee::json_rpc::error& error_resp) + bool core_rpc_server::on_sync_info(const COMMAND_RPC_SYNC_INFO::request& req, COMMAND_RPC_SYNC_INFO::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx) { PERF_TIMER(on_sync_info); @@ -2030,6 +2236,7 @@ namespace cryptonote m_core.get_blockchain_top(res.height, top_hash); ++res.height; // turn top block height into blockchain height res.target_height = m_core.get_target_blockchain_height(); + res.next_needed_pruning_seed = m_p2p.get_payload_object().get_next_needed_pruning_stripe().second; for (const auto &c: m_p2p.get_payload_object().get_connections()) res.peers.push_back({c}); @@ -2044,12 +2251,13 @@ namespace cryptonote res.spans.push_back({span.start_block_height, span.nblocks, span_connection_id, (uint32_t)(span.rate + 0.5f), speed, span.size, address}); return true; }); + res.overview = block_queue.get_overview(res.height); res.status = CORE_RPC_STATUS_OK; return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_get_txpool_backlog(const COMMAND_RPC_GET_TRANSACTION_POOL_BACKLOG::request& req, COMMAND_RPC_GET_TRANSACTION_POOL_BACKLOG::response& res, epee::json_rpc::error& error_resp) + bool core_rpc_server::on_get_txpool_backlog(const COMMAND_RPC_GET_TRANSACTION_POOL_BACKLOG::request& req, COMMAND_RPC_GET_TRANSACTION_POOL_BACKLOG::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx) { PERF_TIMER(on_get_txpool_backlog); bool r; @@ -2067,96 +2275,99 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_get_output_distribution(const COMMAND_RPC_GET_OUTPUT_DISTRIBUTION::request& req, COMMAND_RPC_GET_OUTPUT_DISTRIBUTION::response& res, epee::json_rpc::error& error_resp) + bool core_rpc_server::on_get_output_distribution(const COMMAND_RPC_GET_OUTPUT_DISTRIBUTION::request& req, COMMAND_RPC_GET_OUTPUT_DISTRIBUTION::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx) { PERF_TIMER(on_get_output_distribution); + bool r; + if (use_bootstrap_daemon_if_necessary(invoke_http_mode::JON_RPC, "get_output_distribution", req, res, r)) + return r; + try { + // 0 is placeholder for the whole chain + const uint64_t req_to_height = req.to_height ? req.to_height : (m_core.get_current_blockchain_height() - 1); for (uint64_t amount: req.amounts) { - static struct D - { - boost::mutex mutex; - std::vector cached_distribution; - uint64_t cached_from, cached_to, cached_start_height, cached_base; - bool cached; - D(): cached_from(0), cached_to(0), cached_start_height(0), cached_base(0), cached(false) {} - } d; - boost::unique_lock lock(d.mutex); - - if (d.cached && amount == 0 && d.cached_from == req.from_height && d.cached_to == req.to_height) - { - res.distributions.push_back({amount, d.cached_start_height, d.cached_distribution, d.cached_base}); - if (req.cumulative) - { - auto &distribution = res.distributions.back().distribution; - distribution[0] += d.cached_base; - for (size_t n = 1; n < distribution.size(); ++n) - distribution[n] += distribution[n-1]; - } - continue; - } - - // this is a slow operation, so we have precomputed caches of common cases - bool found = false; - for (const auto &slot: get_output_distribution_cache) - { - if (slot.amount == amount && slot.from_height == req.from_height && slot.to_height == req.to_height) - { - res.distributions.push_back({amount, slot.start_height, slot.distribution, slot.base}); - found = true; - if (req.cumulative) - { - auto &distribution = res.distributions.back().distribution; - distribution[0] += slot.base; - for (size_t n = 1; n < distribution.size(); ++n) - distribution[n] += distribution[n-1]; - } - break; - } - } - if (found) - continue; - - std::vector distribution; - uint64_t start_height, base; - if (!m_core.get_output_distribution(amount, req.from_height, req.to_height, start_height, distribution, base)) + auto data = rpc::RpcHandler::get_output_distribution([this](uint64_t amount, uint64_t from, uint64_t to, uint64_t &start_height, std::vector &distribution, uint64_t &base) { return m_core.get_output_distribution(amount, from, to, start_height, distribution, base); }, amount, req.from_height, req_to_height, [this](uint64_t height) { return m_core.get_blockchain_storage().get_db().get_block_hash_from_height(height); }, req.cumulative, m_core.get_current_blockchain_height()); + if (!data) { error_resp.code = CORE_RPC_ERROR_CODE_INTERNAL_ERROR; - error_resp.message = "Failed to get rct distribution"; + error_resp.message = "Failed to get output distribution"; return false; } - if (req.to_height > 0 && req.to_height >= req.from_height) - { - uint64_t offset = std::max(req.from_height, start_height); - if (offset <= req.to_height && req.to_height - offset + 1 < distribution.size()) - distribution.resize(req.to_height - offset + 1); - } - if (amount == 0) - { - d.cached_from = req.from_height; - d.cached_to = req.to_height; - d.cached_distribution = distribution; - d.cached_start_height = start_height; - d.cached_base = base; - d.cached = true; - } + res.distributions.push_back({std::move(*data), amount, "", req.binary, req.compress}); + } + } + catch (const std::exception &e) + { + error_resp.code = CORE_RPC_ERROR_CODE_INTERNAL_ERROR; + error_resp.message = "Failed to get output distribution"; + return false; + } - if (req.cumulative) + res.status = CORE_RPC_STATUS_OK; + return true; + } + //------------------------------------------------------------------------------------------------------------------------------ + bool core_rpc_server::on_get_output_distribution_bin(const COMMAND_RPC_GET_OUTPUT_DISTRIBUTION::request& req, COMMAND_RPC_GET_OUTPUT_DISTRIBUTION::response& res, const connection_context *ctx) + { + PERF_TIMER(on_get_output_distribution_bin); + + bool r; + if (use_bootstrap_daemon_if_necessary(invoke_http_mode::BIN, "/get_output_distribution.bin", req, res, r)) + return r; + + res.status = "Failed"; + + if (!req.binary) + { + res.status = "Binary only call"; + return false; + } + try + { + // 0 is placeholder for the whole chain + const uint64_t req_to_height = req.to_height ? req.to_height : (m_core.get_current_blockchain_height() - 1); + for (uint64_t amount: req.amounts) + { + auto data = rpc::RpcHandler::get_output_distribution([this](uint64_t amount, uint64_t from, uint64_t to, uint64_t &start_height, std::vector &distribution, uint64_t &base) { return m_core.get_output_distribution(amount, from, to, start_height, distribution, base); }, amount, req.from_height, req_to_height, [this](uint64_t height) { return m_core.get_blockchain_storage().get_db().get_block_hash_from_height(height); }, req.cumulative, m_core.get_current_blockchain_height()); + if (!data) { - distribution[0] += base; - for (size_t n = 1; n < distribution.size(); ++n) - distribution[n] += distribution[n-1]; + res.status = "Failed to get output distribution"; + return false; } - res.distributions.push_back({amount, start_height, std::move(distribution), base}); + res.distributions.push_back({std::move(*data), amount, "", req.binary, req.compress}); } } catch (const std::exception &e) + { + res.status = "Failed to get output distribution"; + return false; + } + + res.status = CORE_RPC_STATUS_OK; + return true; + } + //------------------------------------------------------------------------------------------------------------------------------ + bool core_rpc_server::on_prune_blockchain(const COMMAND_RPC_PRUNE_BLOCKCHAIN::request& req, COMMAND_RPC_PRUNE_BLOCKCHAIN::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx) + { + try + { + if (!(req.check ? m_core.check_blockchain_pruning() : m_core.prune_blockchain())) + { + error_resp.code = CORE_RPC_ERROR_CODE_INTERNAL_ERROR; + error_resp.message = req.check ? "Failed to check blockchain pruning" : "Failed to prune blockchain"; + return false; + } + res.pruning_seed = m_core.get_blockchain_pruning_seed(); + res.pruned = res.pruning_seed != 0; + } + catch (const std::exception &e) { error_resp.code = CORE_RPC_ERROR_CODE_INTERNAL_ERROR; - error_resp.message = "Failed to get output distribution"; + error_resp.message = "Failed to prune blockchain"; return false; } diff --git a/src/rpc/core_rpc_server.h b/src/rpc/core_rpc_server.h index 324f219f8da..e4683bbe23c 100644 --- a/src/rpc/core_rpc_server.h +++ b/src/rpc/core_rpc_server.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -40,6 +40,9 @@ #include "p2p/net_node.h" #include "cryptonote_protocol/cryptonote_protocol_handler.h" +#undef MONERO_DEFAULT_LOG_CATEGORY +#define MONERO_DEFAULT_LOG_CATEGORY "daemon.rpc" + // yes, epee doesn't properly use its full namespace when calling its // functions from macros. *sigh* using namespace epee; @@ -53,9 +56,16 @@ namespace cryptonote { public: + static const command_line::arg_descriptor arg_public_node; static const command_line::arg_descriptor arg_rpc_bind_port; static const command_line::arg_descriptor arg_rpc_restricted_bind_port; static const command_line::arg_descriptor arg_restricted_rpc; + static const command_line::arg_descriptor arg_rpc_ssl; + static const command_line::arg_descriptor arg_rpc_ssl_private_key; + static const command_line::arg_descriptor arg_rpc_ssl_certificate; + static const command_line::arg_descriptor arg_rpc_ssl_ca_certificates; + static const command_line::arg_descriptor> arg_rpc_ssl_allowed_fingerprints; + static const command_line::arg_descriptor arg_rpc_ssl_allow_any_cert; static const command_line::arg_descriptor arg_bootstrap_daemon_address; static const command_line::arg_descriptor arg_bootstrap_daemon_login; @@ -70,10 +80,9 @@ namespace cryptonote bool init( const boost::program_options::variables_map& vm, const bool restricted, - const network_type nettype, const std::string& port ); - network_type nettype() const { return m_nettype; } + network_type nettype() const { return m_core.get_nettype(); } CHAIN_HTTP_TO_MAP2(connection_context); //forward http requests to uri map @@ -87,11 +96,7 @@ namespace cryptonote MAP_URI_AUTO_BIN2("/get_hashes.bin", on_get_hashes, COMMAND_RPC_GET_HASHES_FAST) MAP_URI_AUTO_BIN2("/gethashes.bin", on_get_hashes, COMMAND_RPC_GET_HASHES_FAST) MAP_URI_AUTO_BIN2("/get_o_indexes.bin", on_get_indexes, COMMAND_RPC_GET_TX_GLOBAL_OUTPUTS_INDEXES) - MAP_URI_AUTO_BIN2("/get_random_outs.bin", on_get_random_outs, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS) - MAP_URI_AUTO_BIN2("/getrandom_outs.bin", on_get_random_outs, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS) MAP_URI_AUTO_BIN2("/get_outs.bin", on_get_outs_bin, COMMAND_RPC_GET_OUTPUTS_BIN) - MAP_URI_AUTO_BIN2("/get_random_rctouts.bin", on_get_random_rct_outs, COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS) - MAP_URI_AUTO_BIN2("/getrandom_rctouts.bin", on_get_random_rct_outs, COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS) MAP_URI_AUTO_JON2("/get_transactions", on_get_transactions, COMMAND_RPC_GET_TRANSACTIONS) MAP_URI_AUTO_JON2("/gettransactions", on_get_transactions, COMMAND_RPC_GET_TRANSACTIONS) MAP_URI_AUTO_JON2("/get_alt_blocks_hashes", on_get_alt_blocks_hashes, COMMAND_RPC_GET_ALT_BLOCKS_HASHES) @@ -107,11 +112,13 @@ namespace cryptonote MAP_URI_AUTO_JON2_IF("/set_log_level", on_set_log_level, COMMAND_RPC_SET_LOG_LEVEL, !m_restricted) MAP_URI_AUTO_JON2_IF("/set_log_categories", on_set_log_categories, COMMAND_RPC_SET_LOG_CATEGORIES, !m_restricted) MAP_URI_AUTO_JON2("/get_transaction_pool", on_get_transaction_pool, COMMAND_RPC_GET_TRANSACTION_POOL) - MAP_URI_AUTO_JON2("/get_transaction_pool_hashes.bin", on_get_transaction_pool_hashes, COMMAND_RPC_GET_TRANSACTION_POOL_HASHES) + MAP_URI_AUTO_JON2("/get_transaction_pool_hashes.bin", on_get_transaction_pool_hashes_bin, COMMAND_RPC_GET_TRANSACTION_POOL_HASHES_BIN) + MAP_URI_AUTO_JON2("/get_transaction_pool_hashes", on_get_transaction_pool_hashes, COMMAND_RPC_GET_TRANSACTION_POOL_HASHES) MAP_URI_AUTO_JON2("/get_transaction_pool_stats", on_get_transaction_pool_stats, COMMAND_RPC_GET_TRANSACTION_POOL_STATS) MAP_URI_AUTO_JON2_IF("/stop_daemon", on_stop_daemon, COMMAND_RPC_STOP_DAEMON, !m_restricted) MAP_URI_AUTO_JON2("/get_info", on_get_info, COMMAND_RPC_GET_INFO) MAP_URI_AUTO_JON2("/getinfo", on_get_info, COMMAND_RPC_GET_INFO) + MAP_URI_AUTO_JON2_IF("/get_net_stats", on_get_net_stats, COMMAND_RPC_GET_NET_STATS, !m_restricted) MAP_URI_AUTO_JON2("/get_limit", on_get_limit, COMMAND_RPC_GET_LIMIT) MAP_URI_AUTO_JON2_IF("/set_limit", on_set_limit, COMMAND_RPC_SET_LIMIT, !m_restricted) MAP_URI_AUTO_JON2_IF("/out_peers", on_out_peers, COMMAND_RPC_OUT_PEERS, !m_restricted) @@ -120,6 +127,8 @@ namespace cryptonote MAP_URI_AUTO_JON2_IF("/stop_save_graph", on_stop_save_graph, COMMAND_RPC_STOP_SAVE_GRAPH, !m_restricted) MAP_URI_AUTO_JON2("/get_outs", on_get_outs, COMMAND_RPC_GET_OUTPUTS) MAP_URI_AUTO_JON2_IF("/update", on_update, COMMAND_RPC_UPDATE, !m_restricted) + MAP_URI_AUTO_BIN2("/get_output_distribution.bin", on_get_output_distribution_bin, COMMAND_RPC_GET_OUTPUT_DISTRIBUTION) + MAP_URI_AUTO_JON2_IF("/pop_blocks", on_pop_blocks, COMMAND_RPC_POP_BLOCKS, !m_restricted) BEGIN_JSON_RPC_MAP("/json_rpc") MAP_JON_RPC("get_block_count", on_getblockcount, COMMAND_RPC_GETBLOCKCOUNT) MAP_JON_RPC("getblockcount", on_getblockcount, COMMAND_RPC_GETBLOCKCOUNT) @@ -129,6 +138,7 @@ namespace cryptonote MAP_JON_RPC_WE("getblocktemplate", on_getblocktemplate, COMMAND_RPC_GETBLOCKTEMPLATE) MAP_JON_RPC_WE("submit_block", on_submitblock, COMMAND_RPC_SUBMITBLOCK) MAP_JON_RPC_WE("submitblock", on_submitblock, COMMAND_RPC_SUBMITBLOCK) + MAP_JON_RPC_WE_IF("generateblocks", on_generateblocks, COMMAND_RPC_GENERATEBLOCKS, !m_restricted) MAP_JON_RPC_WE("get_last_block_header", on_get_last_block_header, COMMAND_RPC_GET_LAST_BLOCK_HEADER) MAP_JON_RPC_WE("getlastblockheader", on_get_last_block_header, COMMAND_RPC_GET_LAST_BLOCK_HEADER) MAP_JON_RPC_WE("get_block_header_by_hash", on_get_block_header_by_hash, COMMAND_RPC_GET_BLOCK_HEADER_BY_HASH) @@ -148,74 +158,79 @@ namespace cryptonote MAP_JON_RPC_WE("get_output_histogram", on_get_output_histogram, COMMAND_RPC_GET_OUTPUT_HISTOGRAM) MAP_JON_RPC_WE("get_version", on_get_version, COMMAND_RPC_GET_VERSION) MAP_JON_RPC_WE_IF("get_coinbase_tx_sum", on_get_coinbase_tx_sum, COMMAND_RPC_GET_COINBASE_TX_SUM, !m_restricted) - MAP_JON_RPC_WE("get_fee_estimate", on_get_per_kb_fee_estimate, COMMAND_RPC_GET_PER_KB_FEE_ESTIMATE) + MAP_JON_RPC_WE("get_fee_estimate", on_get_base_fee_estimate, COMMAND_RPC_GET_BASE_FEE_ESTIMATE) MAP_JON_RPC_WE_IF("get_alternate_chains",on_get_alternate_chains, COMMAND_RPC_GET_ALTERNATE_CHAINS, !m_restricted) MAP_JON_RPC_WE_IF("relay_tx", on_relay_tx, COMMAND_RPC_RELAY_TX, !m_restricted) MAP_JON_RPC_WE_IF("sync_info", on_sync_info, COMMAND_RPC_SYNC_INFO, !m_restricted) MAP_JON_RPC_WE("get_txpool_backlog", on_get_txpool_backlog, COMMAND_RPC_GET_TRANSACTION_POOL_BACKLOG) MAP_JON_RPC_WE("get_output_distribution", on_get_output_distribution, COMMAND_RPC_GET_OUTPUT_DISTRIBUTION) + MAP_JON_RPC_WE_IF("prune_blockchain", on_prune_blockchain, COMMAND_RPC_PRUNE_BLOCKCHAIN, !m_restricted) END_JSON_RPC_MAP() END_URI_MAP2() - bool on_get_height(const COMMAND_RPC_GET_HEIGHT::request& req, COMMAND_RPC_GET_HEIGHT::response& res); - bool on_get_blocks(const COMMAND_RPC_GET_BLOCKS_FAST::request& req, COMMAND_RPC_GET_BLOCKS_FAST::response& res); - bool on_get_alt_blocks_hashes(const COMMAND_RPC_GET_ALT_BLOCKS_HASHES::request& req, COMMAND_RPC_GET_ALT_BLOCKS_HASHES::response& res); - bool on_get_blocks_by_height(const COMMAND_RPC_GET_BLOCKS_BY_HEIGHT::request& req, COMMAND_RPC_GET_BLOCKS_BY_HEIGHT::response& res); - bool on_get_hashes(const COMMAND_RPC_GET_HASHES_FAST::request& req, COMMAND_RPC_GET_HASHES_FAST::response& res); - bool on_get_transactions(const COMMAND_RPC_GET_TRANSACTIONS::request& req, COMMAND_RPC_GET_TRANSACTIONS::response& res); - bool on_is_key_image_spent(const COMMAND_RPC_IS_KEY_IMAGE_SPENT::request& req, COMMAND_RPC_IS_KEY_IMAGE_SPENT::response& res, bool request_has_rpc_origin = true); - bool on_get_indexes(const COMMAND_RPC_GET_TX_GLOBAL_OUTPUTS_INDEXES::request& req, COMMAND_RPC_GET_TX_GLOBAL_OUTPUTS_INDEXES::response& res); - bool on_send_raw_tx(const COMMAND_RPC_SEND_RAW_TX::request& req, COMMAND_RPC_SEND_RAW_TX::response& res); - bool on_start_mining(const COMMAND_RPC_START_MINING::request& req, COMMAND_RPC_START_MINING::response& res); - bool on_stop_mining(const COMMAND_RPC_STOP_MINING::request& req, COMMAND_RPC_STOP_MINING::response& res); - bool on_mining_status(const COMMAND_RPC_MINING_STATUS::request& req, COMMAND_RPC_MINING_STATUS::response& res); - bool on_get_random_outs(const COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::request& req, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::response& res); - bool on_get_outs_bin(const COMMAND_RPC_GET_OUTPUTS_BIN::request& req, COMMAND_RPC_GET_OUTPUTS_BIN::response& res); - bool on_get_outs(const COMMAND_RPC_GET_OUTPUTS::request& req, COMMAND_RPC_GET_OUTPUTS::response& res); - bool on_get_random_rct_outs(const COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS::request& req, COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS::response& res); - bool on_get_info(const COMMAND_RPC_GET_INFO::request& req, COMMAND_RPC_GET_INFO::response& res); - bool on_save_bc(const COMMAND_RPC_SAVE_BC::request& req, COMMAND_RPC_SAVE_BC::response& res); - bool on_get_peer_list(const COMMAND_RPC_GET_PEER_LIST::request& req, COMMAND_RPC_GET_PEER_LIST::response& res); - bool on_set_log_hash_rate(const COMMAND_RPC_SET_LOG_HASH_RATE::request& req, COMMAND_RPC_SET_LOG_HASH_RATE::response& res); - bool on_set_log_level(const COMMAND_RPC_SET_LOG_LEVEL::request& req, COMMAND_RPC_SET_LOG_LEVEL::response& res); - bool on_set_log_categories(const COMMAND_RPC_SET_LOG_CATEGORIES::request& req, COMMAND_RPC_SET_LOG_CATEGORIES::response& res); - bool on_get_transaction_pool(const COMMAND_RPC_GET_TRANSACTION_POOL::request& req, COMMAND_RPC_GET_TRANSACTION_POOL::response& res, bool request_has_rpc_origin = true); - bool on_get_transaction_pool_hashes(const COMMAND_RPC_GET_TRANSACTION_POOL_HASHES::request& req, COMMAND_RPC_GET_TRANSACTION_POOL_HASHES::response& res, bool request_has_rpc_origin = true); - bool on_get_transaction_pool_stats(const COMMAND_RPC_GET_TRANSACTION_POOL_STATS::request& req, COMMAND_RPC_GET_TRANSACTION_POOL_STATS::response& res, bool request_has_rpc_origin = true); - bool on_stop_daemon(const COMMAND_RPC_STOP_DAEMON::request& req, COMMAND_RPC_STOP_DAEMON::response& res); - bool on_get_limit(const COMMAND_RPC_GET_LIMIT::request& req, COMMAND_RPC_GET_LIMIT::response& res); - bool on_set_limit(const COMMAND_RPC_SET_LIMIT::request& req, COMMAND_RPC_SET_LIMIT::response& res); - bool on_out_peers(const COMMAND_RPC_OUT_PEERS::request& req, COMMAND_RPC_OUT_PEERS::response& res); - bool on_in_peers(const COMMAND_RPC_IN_PEERS::request& req, COMMAND_RPC_IN_PEERS::response& res); - bool on_start_save_graph(const COMMAND_RPC_START_SAVE_GRAPH::request& req, COMMAND_RPC_START_SAVE_GRAPH::response& res); - bool on_stop_save_graph(const COMMAND_RPC_STOP_SAVE_GRAPH::request& req, COMMAND_RPC_STOP_SAVE_GRAPH::response& res); - bool on_update(const COMMAND_RPC_UPDATE::request& req, COMMAND_RPC_UPDATE::response& res); + bool on_get_height(const COMMAND_RPC_GET_HEIGHT::request& req, COMMAND_RPC_GET_HEIGHT::response& res, const connection_context *ctx = NULL); + bool on_get_blocks(const COMMAND_RPC_GET_BLOCKS_FAST::request& req, COMMAND_RPC_GET_BLOCKS_FAST::response& res, const connection_context *ctx = NULL); + bool on_get_alt_blocks_hashes(const COMMAND_RPC_GET_ALT_BLOCKS_HASHES::request& req, COMMAND_RPC_GET_ALT_BLOCKS_HASHES::response& res, const connection_context *ctx = NULL); + bool on_get_blocks_by_height(const COMMAND_RPC_GET_BLOCKS_BY_HEIGHT::request& req, COMMAND_RPC_GET_BLOCKS_BY_HEIGHT::response& res, const connection_context *ctx = NULL); + bool on_get_hashes(const COMMAND_RPC_GET_HASHES_FAST::request& req, COMMAND_RPC_GET_HASHES_FAST::response& res, const connection_context *ctx = NULL); + bool on_get_transactions(const COMMAND_RPC_GET_TRANSACTIONS::request& req, COMMAND_RPC_GET_TRANSACTIONS::response& res, const connection_context *ctx = NULL); + bool on_is_key_image_spent(const COMMAND_RPC_IS_KEY_IMAGE_SPENT::request& req, COMMAND_RPC_IS_KEY_IMAGE_SPENT::response& res, const connection_context *ctx = NULL); + bool on_get_indexes(const COMMAND_RPC_GET_TX_GLOBAL_OUTPUTS_INDEXES::request& req, COMMAND_RPC_GET_TX_GLOBAL_OUTPUTS_INDEXES::response& res, const connection_context *ctx = NULL); + bool on_send_raw_tx(const COMMAND_RPC_SEND_RAW_TX::request& req, COMMAND_RPC_SEND_RAW_TX::response& res, const connection_context *ctx = NULL); + bool on_start_mining(const COMMAND_RPC_START_MINING::request& req, COMMAND_RPC_START_MINING::response& res, const connection_context *ctx = NULL); + bool on_stop_mining(const COMMAND_RPC_STOP_MINING::request& req, COMMAND_RPC_STOP_MINING::response& res, const connection_context *ctx = NULL); + bool on_mining_status(const COMMAND_RPC_MINING_STATUS::request& req, COMMAND_RPC_MINING_STATUS::response& res, const connection_context *ctx = NULL); + bool on_get_outs_bin(const COMMAND_RPC_GET_OUTPUTS_BIN::request& req, COMMAND_RPC_GET_OUTPUTS_BIN::response& res, const connection_context *ctx = NULL); + bool on_get_outs(const COMMAND_RPC_GET_OUTPUTS::request& req, COMMAND_RPC_GET_OUTPUTS::response& res, const connection_context *ctx = NULL); + bool on_get_info(const COMMAND_RPC_GET_INFO::request& req, COMMAND_RPC_GET_INFO::response& res, const connection_context *ctx = NULL); + bool on_get_net_stats(const COMMAND_RPC_GET_NET_STATS::request& req, COMMAND_RPC_GET_NET_STATS::response& res, const connection_context *ctx = NULL); + bool on_save_bc(const COMMAND_RPC_SAVE_BC::request& req, COMMAND_RPC_SAVE_BC::response& res, const connection_context *ctx = NULL); + bool on_get_peer_list(const COMMAND_RPC_GET_PEER_LIST::request& req, COMMAND_RPC_GET_PEER_LIST::response& res, const connection_context *ctx = NULL); + bool on_set_log_hash_rate(const COMMAND_RPC_SET_LOG_HASH_RATE::request& req, COMMAND_RPC_SET_LOG_HASH_RATE::response& res, const connection_context *ctx = NULL); + bool on_set_log_level(const COMMAND_RPC_SET_LOG_LEVEL::request& req, COMMAND_RPC_SET_LOG_LEVEL::response& res, const connection_context *ctx = NULL); + bool on_set_log_categories(const COMMAND_RPC_SET_LOG_CATEGORIES::request& req, COMMAND_RPC_SET_LOG_CATEGORIES::response& res, const connection_context *ctx = NULL); + bool on_get_transaction_pool(const COMMAND_RPC_GET_TRANSACTION_POOL::request& req, COMMAND_RPC_GET_TRANSACTION_POOL::response& res, const connection_context *ctx = NULL); + bool on_get_transaction_pool_hashes_bin(const COMMAND_RPC_GET_TRANSACTION_POOL_HASHES_BIN::request& req, COMMAND_RPC_GET_TRANSACTION_POOL_HASHES_BIN::response& res, const connection_context *ctx = NULL); + bool on_get_transaction_pool_hashes(const COMMAND_RPC_GET_TRANSACTION_POOL_HASHES::request& req, COMMAND_RPC_GET_TRANSACTION_POOL_HASHES::response& res, const connection_context *ctx = NULL); + bool on_get_transaction_pool_stats(const COMMAND_RPC_GET_TRANSACTION_POOL_STATS::request& req, COMMAND_RPC_GET_TRANSACTION_POOL_STATS::response& res, const connection_context *ctx = NULL); + bool on_stop_daemon(const COMMAND_RPC_STOP_DAEMON::request& req, COMMAND_RPC_STOP_DAEMON::response& res, const connection_context *ctx = NULL); + bool on_get_limit(const COMMAND_RPC_GET_LIMIT::request& req, COMMAND_RPC_GET_LIMIT::response& res, const connection_context *ctx = NULL); + bool on_set_limit(const COMMAND_RPC_SET_LIMIT::request& req, COMMAND_RPC_SET_LIMIT::response& res, const connection_context *ctx = NULL); + bool on_out_peers(const COMMAND_RPC_OUT_PEERS::request& req, COMMAND_RPC_OUT_PEERS::response& res, const connection_context *ctx = NULL); + bool on_in_peers(const COMMAND_RPC_IN_PEERS::request& req, COMMAND_RPC_IN_PEERS::response& res, const connection_context *ctx = NULL); + bool on_start_save_graph(const COMMAND_RPC_START_SAVE_GRAPH::request& req, COMMAND_RPC_START_SAVE_GRAPH::response& res, const connection_context *ctx = NULL); + bool on_stop_save_graph(const COMMAND_RPC_STOP_SAVE_GRAPH::request& req, COMMAND_RPC_STOP_SAVE_GRAPH::response& res, const connection_context *ctx = NULL); + bool on_update(const COMMAND_RPC_UPDATE::request& req, COMMAND_RPC_UPDATE::response& res, const connection_context *ctx = NULL); + bool on_get_output_distribution_bin(const COMMAND_RPC_GET_OUTPUT_DISTRIBUTION::request& req, COMMAND_RPC_GET_OUTPUT_DISTRIBUTION::response& res, const connection_context *ctx = NULL); + bool on_pop_blocks(const COMMAND_RPC_POP_BLOCKS::request& req, COMMAND_RPC_POP_BLOCKS::response& res, const connection_context *ctx = NULL); //json_rpc - bool on_getblockcount(const COMMAND_RPC_GETBLOCKCOUNT::request& req, COMMAND_RPC_GETBLOCKCOUNT::response& res); - bool on_getblockhash(const COMMAND_RPC_GETBLOCKHASH::request& req, COMMAND_RPC_GETBLOCKHASH::response& res, epee::json_rpc::error& error_resp); - bool on_getblocktemplate(const COMMAND_RPC_GETBLOCKTEMPLATE::request& req, COMMAND_RPC_GETBLOCKTEMPLATE::response& res, epee::json_rpc::error& error_resp); - bool on_submitblock(const COMMAND_RPC_SUBMITBLOCK::request& req, COMMAND_RPC_SUBMITBLOCK::response& res, epee::json_rpc::error& error_resp); - bool on_get_last_block_header(const COMMAND_RPC_GET_LAST_BLOCK_HEADER::request& req, COMMAND_RPC_GET_LAST_BLOCK_HEADER::response& res, epee::json_rpc::error& error_resp); - bool on_get_block_header_by_hash(const COMMAND_RPC_GET_BLOCK_HEADER_BY_HASH::request& req, COMMAND_RPC_GET_BLOCK_HEADER_BY_HASH::response& res, epee::json_rpc::error& error_resp); - bool on_get_block_header_by_height(const COMMAND_RPC_GET_BLOCK_HEADER_BY_HEIGHT::request& req, COMMAND_RPC_GET_BLOCK_HEADER_BY_HEIGHT::response& res, epee::json_rpc::error& error_resp); - bool on_get_block_headers_range(const COMMAND_RPC_GET_BLOCK_HEADERS_RANGE::request& req, COMMAND_RPC_GET_BLOCK_HEADERS_RANGE::response& res, epee::json_rpc::error& error_resp); - bool on_get_block(const COMMAND_RPC_GET_BLOCK::request& req, COMMAND_RPC_GET_BLOCK::response& res, epee::json_rpc::error& error_resp); - bool on_get_connections(const COMMAND_RPC_GET_CONNECTIONS::request& req, COMMAND_RPC_GET_CONNECTIONS::response& res, epee::json_rpc::error& error_resp); - bool on_get_info_json(const COMMAND_RPC_GET_INFO::request& req, COMMAND_RPC_GET_INFO::response& res, epee::json_rpc::error& error_resp); - bool on_hard_fork_info(const COMMAND_RPC_HARD_FORK_INFO::request& req, COMMAND_RPC_HARD_FORK_INFO::response& res, epee::json_rpc::error& error_resp); - bool on_set_bans(const COMMAND_RPC_SETBANS::request& req, COMMAND_RPC_SETBANS::response& res, epee::json_rpc::error& error_resp); - bool on_get_bans(const COMMAND_RPC_GETBANS::request& req, COMMAND_RPC_GETBANS::response& res, epee::json_rpc::error& error_resp); - bool on_flush_txpool(const COMMAND_RPC_FLUSH_TRANSACTION_POOL::request& req, COMMAND_RPC_FLUSH_TRANSACTION_POOL::response& res, epee::json_rpc::error& error_resp); - bool on_get_output_histogram(const COMMAND_RPC_GET_OUTPUT_HISTOGRAM::request& req, COMMAND_RPC_GET_OUTPUT_HISTOGRAM::response& res, epee::json_rpc::error& error_resp); - bool on_get_version(const COMMAND_RPC_GET_VERSION::request& req, COMMAND_RPC_GET_VERSION::response& res, epee::json_rpc::error& error_resp); - bool on_get_coinbase_tx_sum(const COMMAND_RPC_GET_COINBASE_TX_SUM::request& req, COMMAND_RPC_GET_COINBASE_TX_SUM::response& res, epee::json_rpc::error& error_resp); - bool on_get_per_kb_fee_estimate(const COMMAND_RPC_GET_PER_KB_FEE_ESTIMATE::request& req, COMMAND_RPC_GET_PER_KB_FEE_ESTIMATE::response& res, epee::json_rpc::error& error_resp); - bool on_get_alternate_chains(const COMMAND_RPC_GET_ALTERNATE_CHAINS::request& req, COMMAND_RPC_GET_ALTERNATE_CHAINS::response& res, epee::json_rpc::error& error_resp); - bool on_relay_tx(const COMMAND_RPC_RELAY_TX::request& req, COMMAND_RPC_RELAY_TX::response& res, epee::json_rpc::error& error_resp); - bool on_sync_info(const COMMAND_RPC_SYNC_INFO::request& req, COMMAND_RPC_SYNC_INFO::response& res, epee::json_rpc::error& error_resp); - bool on_get_txpool_backlog(const COMMAND_RPC_GET_TRANSACTION_POOL_BACKLOG::request& req, COMMAND_RPC_GET_TRANSACTION_POOL_BACKLOG::response& res, epee::json_rpc::error& error_resp); - bool on_get_output_distribution(const COMMAND_RPC_GET_OUTPUT_DISTRIBUTION::request& req, COMMAND_RPC_GET_OUTPUT_DISTRIBUTION::response& res, epee::json_rpc::error& error_resp); + bool on_getblockcount(const COMMAND_RPC_GETBLOCKCOUNT::request& req, COMMAND_RPC_GETBLOCKCOUNT::response& res, const connection_context *ctx = NULL); + bool on_getblockhash(const COMMAND_RPC_GETBLOCKHASH::request& req, COMMAND_RPC_GETBLOCKHASH::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL); + bool on_getblocktemplate(const COMMAND_RPC_GETBLOCKTEMPLATE::request& req, COMMAND_RPC_GETBLOCKTEMPLATE::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL); + bool on_submitblock(const COMMAND_RPC_SUBMITBLOCK::request& req, COMMAND_RPC_SUBMITBLOCK::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL); + bool on_generateblocks(const COMMAND_RPC_GENERATEBLOCKS::request& req, COMMAND_RPC_GENERATEBLOCKS::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL); + bool on_get_last_block_header(const COMMAND_RPC_GET_LAST_BLOCK_HEADER::request& req, COMMAND_RPC_GET_LAST_BLOCK_HEADER::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL); + bool on_get_block_header_by_hash(const COMMAND_RPC_GET_BLOCK_HEADER_BY_HASH::request& req, COMMAND_RPC_GET_BLOCK_HEADER_BY_HASH::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL); + bool on_get_block_header_by_height(const COMMAND_RPC_GET_BLOCK_HEADER_BY_HEIGHT::request& req, COMMAND_RPC_GET_BLOCK_HEADER_BY_HEIGHT::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL); + bool on_get_block_headers_range(const COMMAND_RPC_GET_BLOCK_HEADERS_RANGE::request& req, COMMAND_RPC_GET_BLOCK_HEADERS_RANGE::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL); + bool on_get_block(const COMMAND_RPC_GET_BLOCK::request& req, COMMAND_RPC_GET_BLOCK::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL); + bool on_get_connections(const COMMAND_RPC_GET_CONNECTIONS::request& req, COMMAND_RPC_GET_CONNECTIONS::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL); + bool on_get_info_json(const COMMAND_RPC_GET_INFO::request& req, COMMAND_RPC_GET_INFO::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL); + bool on_hard_fork_info(const COMMAND_RPC_HARD_FORK_INFO::request& req, COMMAND_RPC_HARD_FORK_INFO::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL); + bool on_set_bans(const COMMAND_RPC_SETBANS::request& req, COMMAND_RPC_SETBANS::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL); + bool on_get_bans(const COMMAND_RPC_GETBANS::request& req, COMMAND_RPC_GETBANS::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL); + bool on_flush_txpool(const COMMAND_RPC_FLUSH_TRANSACTION_POOL::request& req, COMMAND_RPC_FLUSH_TRANSACTION_POOL::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL); + bool on_get_output_histogram(const COMMAND_RPC_GET_OUTPUT_HISTOGRAM::request& req, COMMAND_RPC_GET_OUTPUT_HISTOGRAM::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL); + bool on_get_version(const COMMAND_RPC_GET_VERSION::request& req, COMMAND_RPC_GET_VERSION::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL); + bool on_get_coinbase_tx_sum(const COMMAND_RPC_GET_COINBASE_TX_SUM::request& req, COMMAND_RPC_GET_COINBASE_TX_SUM::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL); + bool on_get_base_fee_estimate(const COMMAND_RPC_GET_BASE_FEE_ESTIMATE::request& req, COMMAND_RPC_GET_BASE_FEE_ESTIMATE::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL); + bool on_get_alternate_chains(const COMMAND_RPC_GET_ALTERNATE_CHAINS::request& req, COMMAND_RPC_GET_ALTERNATE_CHAINS::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL); + bool on_relay_tx(const COMMAND_RPC_RELAY_TX::request& req, COMMAND_RPC_RELAY_TX::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL); + bool on_sync_info(const COMMAND_RPC_SYNC_INFO::request& req, COMMAND_RPC_SYNC_INFO::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL); + bool on_get_txpool_backlog(const COMMAND_RPC_GET_TRANSACTION_POOL_BACKLOG::request& req, COMMAND_RPC_GET_TRANSACTION_POOL_BACKLOG::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL); + bool on_get_output_distribution(const COMMAND_RPC_GET_OUTPUT_DISTRIBUTION::request& req, COMMAND_RPC_GET_OUTPUT_DISTRIBUTION::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL); + bool on_prune_blockchain(const COMMAND_RPC_PRUNE_BLOCKCHAIN::request& req, COMMAND_RPC_PRUNE_BLOCKCHAIN::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL); //----------------------- private: diff --git a/src/rpc/core_rpc_server_commands_defs.h b/src/rpc/core_rpc_server_commands_defs.h index 70e18684871..cfe4bbf2343 100644 --- a/src/rpc/core_rpc_server_commands_defs.h +++ b/src/rpc/core_rpc_server_commands_defs.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -33,6 +33,41 @@ #include "cryptonote_basic/cryptonote_basic.h" #include "cryptonote_basic/difficulty.h" #include "crypto/hash.h" +#include "rpc/rpc_handler.h" +#include "common/varint.h" +#include "common/perf_timer.h" + +namespace +{ + template + std::string compress_integer_array(const std::vector &v) + { + std::string s; + s.resize(v.size() * (sizeof(T) * 8 / 7 + 1)); + char *ptr = (char*)s.data(); + for (const T &t: v) + tools::write_varint(ptr, t); + s.resize(ptr - s.data()); + return s; + } + + template + std::vector decompress_integer_array(const std::string &s) + { + std::vector v; + v.reserve(s.size()); + int read = 0; + const std::string::const_iterator end = s.end(); + for (std::string::const_iterator i = s.begin(); i != end; std::advance(i, read)) + { + T t; + read = tools::read_varint(std::string::const_iterator(i), s.end(), t); + CHECK_AND_ASSERT_THROW_MES(read > 0 && read <= 256, "Error decompressing data"); + v.push_back(t); + } + return v; + } +} namespace cryptonote { @@ -48,47 +83,54 @@ namespace cryptonote // whether they can talk to a given daemon without having to know in // advance which version they will stop working with // Don't go over 32767 for any of these -#define CORE_RPC_VERSION_MAJOR 1 -#define CORE_RPC_VERSION_MINOR 20 +#define CORE_RPC_VERSION_MAJOR 2 +#define CORE_RPC_VERSION_MINOR 6 #define MAKE_CORE_RPC_VERSION(major,minor) (((major)<<16)|(minor)) #define CORE_RPC_VERSION MAKE_CORE_RPC_VERSION(CORE_RPC_VERSION_MAJOR, CORE_RPC_VERSION_MINOR) struct COMMAND_RPC_GET_HEIGHT { - struct request + struct request_t { BEGIN_KV_SERIALIZE_MAP() END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init request; - struct response + struct response_t { uint64_t height; std::string status; bool untrusted; + std::string hash; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(height) KV_SERIALIZE(status) KV_SERIALIZE(untrusted) + KV_SERIALIZE(hash) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init response; }; struct COMMAND_RPC_GET_BLOCKS_FAST { - struct request + struct request_t { std::list block_ids; //*first 10 blocks id goes sequential, next goes in pow(2,n) offset, like 2, 4, 8, 16, 32, 64 and so on, and the last one is always genesis block */ uint64_t start_height; bool prune; + bool no_miner_tx; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE_CONTAINER_POD_AS_BLOB(block_ids) KV_SERIALIZE(start_height) KV_SERIALIZE(prune) + KV_SERIALIZE_OPT(no_miner_tx, false) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init request; struct tx_output_indices { @@ -108,9 +150,9 @@ namespace cryptonote END_KV_SERIALIZE_MAP() }; - struct response + struct response_t { - std::list blocks; + std::vector blocks; uint64_t start_height; uint64_t current_height; std::string status; @@ -126,19 +168,21 @@ namespace cryptonote KV_SERIALIZE(untrusted) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init response; }; struct COMMAND_RPC_GET_BLOCKS_BY_HEIGHT { - struct request + struct request_t { std::vector heights; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(heights) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init request; - struct response + struct response_t { std::vector blocks; std::string status; @@ -150,17 +194,19 @@ namespace cryptonote KV_SERIALIZE(untrusted) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init response; }; struct COMMAND_RPC_GET_ALT_BLOCKS_HASHES { - struct request + struct request_t { BEGIN_KV_SERIALIZE_MAP() END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init request; - struct response + struct response_t { std::vector blks_hashes; std::string status; @@ -172,11 +218,12 @@ namespace cryptonote KV_SERIALIZE(untrusted) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init response; }; struct COMMAND_RPC_GET_HASHES_FAST { - struct request + struct request_t { std::list block_ids; //*first 10 blocks id goes sequential, next goes in pow(2,n) offset, like 2, 4, 8, 16, 32, 64 and so on, and the last one is always genesis block */ uint64_t start_height; @@ -185,10 +232,11 @@ namespace cryptonote KV_SERIALIZE(start_height) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init request; - struct response + struct response_t { - std::list m_block_ids; + std::vector m_block_ids; uint64_t start_height; uint64_t current_height; std::string status; @@ -202,225 +250,13 @@ namespace cryptonote KV_SERIALIZE(untrusted) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init response; }; - //----------------------------------------------- - struct COMMAND_RPC_GET_ADDRESS_TXS - { - struct request - { - std::string address; - std::string view_key; - - BEGIN_KV_SERIALIZE_MAP() - KV_SERIALIZE(address) - KV_SERIALIZE(view_key) - END_KV_SERIALIZE_MAP() - }; - - struct spent_output { - uint64_t amount; - std::string key_image; - std::string tx_pub_key; - uint64_t out_index; - uint32_t mixin; - - - BEGIN_KV_SERIALIZE_MAP() - KV_SERIALIZE(amount) - KV_SERIALIZE(key_image) - KV_SERIALIZE(tx_pub_key) - KV_SERIALIZE(out_index) - KV_SERIALIZE(mixin) - END_KV_SERIALIZE_MAP() - }; - - struct transaction - { - uint64_t id; - std::string hash; - uint64_t timestamp; - uint64_t total_received; - uint64_t total_sent; - uint64_t unlock_time; - uint64_t height; - std::list spent_outputs; - std::string payment_id; - bool coinbase; - bool mempool; - uint32_t mixin; - - BEGIN_KV_SERIALIZE_MAP() - KV_SERIALIZE(id) - KV_SERIALIZE(hash) - KV_SERIALIZE(timestamp) - KV_SERIALIZE(total_received) - KV_SERIALIZE(total_sent) - KV_SERIALIZE(unlock_time) - KV_SERIALIZE(height) - KV_SERIALIZE(spent_outputs) - KV_SERIALIZE(payment_id) - KV_SERIALIZE(coinbase) - KV_SERIALIZE(mempool) - KV_SERIALIZE(mixin) - END_KV_SERIALIZE_MAP() - }; - - - struct response - { - //std::list txs_as_json; - uint64_t total_received; - uint64_t total_received_unlocked = 0; // OpenMonero only - uint64_t scanned_height; - std::list transactions; - uint64_t blockchain_height; - uint64_t scanned_block_height; - std::string status; - BEGIN_KV_SERIALIZE_MAP() - KV_SERIALIZE(total_received) - KV_SERIALIZE(total_received_unlocked) - KV_SERIALIZE(scanned_height) - KV_SERIALIZE(transactions) - KV_SERIALIZE(blockchain_height) - KV_SERIALIZE(scanned_block_height) - KV_SERIALIZE(status) - END_KV_SERIALIZE_MAP() - }; - }; - - //----------------------------------------------- - struct COMMAND_RPC_GET_ADDRESS_INFO - { - struct request - { - std::string address; - std::string view_key; - - BEGIN_KV_SERIALIZE_MAP() - KV_SERIALIZE(address) - KV_SERIALIZE(view_key) - END_KV_SERIALIZE_MAP() - }; - - struct spent_output - { - uint64_t amount; - std::string key_image; - std::string tx_pub_key; - uint64_t out_index; - uint32_t mixin; - - BEGIN_KV_SERIALIZE_MAP() - KV_SERIALIZE(amount) - KV_SERIALIZE(key_image) - KV_SERIALIZE(tx_pub_key) - KV_SERIALIZE(out_index) - KV_SERIALIZE(mixin) - END_KV_SERIALIZE_MAP() - }; - - - - struct response - { - uint64_t locked_funds; - uint64_t total_received; - uint64_t total_sent; - uint64_t scanned_height; - uint64_t scanned_block_height; - uint64_t start_height; - uint64_t transaction_height; - uint64_t blockchain_height; - std::list spent_outputs; - BEGIN_KV_SERIALIZE_MAP() - KV_SERIALIZE(locked_funds) - KV_SERIALIZE(total_received) - KV_SERIALIZE(total_sent) - KV_SERIALIZE(scanned_height) - KV_SERIALIZE(scanned_block_height) - KV_SERIALIZE(start_height) - KV_SERIALIZE(transaction_height) - KV_SERIALIZE(blockchain_height) - KV_SERIALIZE(spent_outputs) - END_KV_SERIALIZE_MAP() - }; - }; - - //----------------------------------------------- - struct COMMAND_RPC_GET_UNSPENT_OUTS - { - struct request - { - std::string amount; - std::string address; - std::string view_key; - // OpenMonero specific - uint64_t mixin; - bool use_dust; - std::string dust_threshold; - - BEGIN_KV_SERIALIZE_MAP() - KV_SERIALIZE(amount) - KV_SERIALIZE(address) - KV_SERIALIZE(view_key) - KV_SERIALIZE(mixin) - KV_SERIALIZE(use_dust) - KV_SERIALIZE(dust_threshold) - END_KV_SERIALIZE_MAP() - }; - - - struct output { - uint64_t amount; - std::string public_key; - uint64_t index; - uint64_t global_index; - std::string rct; - std::string tx_hash; - std::string tx_pub_key; - std::string tx_prefix_hash; - std::vector spend_key_images; - uint64_t timestamp; - uint64_t height; - - - BEGIN_KV_SERIALIZE_MAP() - KV_SERIALIZE(amount) - KV_SERIALIZE(public_key) - KV_SERIALIZE(index) - KV_SERIALIZE(global_index) - KV_SERIALIZE(rct) - KV_SERIALIZE(tx_hash) - KV_SERIALIZE(tx_pub_key) - KV_SERIALIZE(tx_prefix_hash) - KV_SERIALIZE(spend_key_images) - KV_SERIALIZE(timestamp) - KV_SERIALIZE(height) - END_KV_SERIALIZE_MAP() - }; - - struct response - { - uint64_t amount; - std::list outputs; - uint64_t per_kb_fee; - std::string status; - std::string reason; - BEGIN_KV_SERIALIZE_MAP() - KV_SERIALIZE(amount) - KV_SERIALIZE(outputs) - KV_SERIALIZE(per_kb_fee) - KV_SERIALIZE(status) - KV_SERIALIZE(reason) - END_KV_SERIALIZE_MAP() - }; - }; - //----------------------------------------------- struct COMMAND_RPC_GET_RANDOM_OUTS { - struct request + struct request_t { std::vector amounts; uint32_t count; @@ -430,6 +266,7 @@ namespace cryptonote KV_SERIALIZE(count) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init request; struct output { @@ -454,7 +291,7 @@ namespace cryptonote }; - struct response + struct response_t { std::vector amount_outs; std::string Error; @@ -463,11 +300,12 @@ namespace cryptonote KV_SERIALIZE(Error) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init response; }; //----------------------------------------------- struct COMMAND_RPC_SUBMIT_RAW_TX { - struct request + struct request_t { std::string address; std::string view_key; @@ -479,9 +317,10 @@ namespace cryptonote KV_SERIALIZE(tx) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init request; - struct response + struct response_t { std::string status; std::string error; @@ -491,118 +330,72 @@ namespace cryptonote KV_SERIALIZE(error) END_KV_SERIALIZE_MAP() }; - }; - //----------------------------------------------- - struct COMMAND_RPC_LOGIN - { - struct request - { - std::string address; - std::string view_key; - bool create_account; - - BEGIN_KV_SERIALIZE_MAP() - KV_SERIALIZE(address) - KV_SERIALIZE(view_key) - KV_SERIALIZE(create_account) - END_KV_SERIALIZE_MAP() - }; - - - struct response - { - std::string status; - std::string reason; - bool new_address; - - BEGIN_KV_SERIALIZE_MAP() - KV_SERIALIZE(status) - KV_SERIALIZE(reason) - KV_SERIALIZE(new_address) - END_KV_SERIALIZE_MAP() - }; - }; - //----------------------------------------------- - struct COMMAND_RPC_IMPORT_WALLET_REQUEST - { - struct request - { - std::string address; - std::string view_key; - - BEGIN_KV_SERIALIZE_MAP() - KV_SERIALIZE(address) - KV_SERIALIZE(view_key) - END_KV_SERIALIZE_MAP() - }; - - - struct response - { - std::string payment_id; - uint64_t import_fee; - bool new_request; - bool request_fulfilled; - std::string payment_address; - std::string status; - - BEGIN_KV_SERIALIZE_MAP() - KV_SERIALIZE(payment_id) - KV_SERIALIZE(import_fee) - KV_SERIALIZE(new_request) - KV_SERIALIZE(request_fulfilled) - KV_SERIALIZE(payment_address) - KV_SERIALIZE(status) - END_KV_SERIALIZE_MAP() - }; + typedef epee::misc_utils::struct_init response; }; //----------------------------------------------- struct COMMAND_RPC_GET_TRANSACTIONS { - struct request + struct request_t { - std::list txs_hashes; + std::vector txs_hashes; bool decode_as_json; bool prune; + bool split; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(txs_hashes) KV_SERIALIZE(decode_as_json) KV_SERIALIZE_OPT(prune, false) + KV_SERIALIZE_OPT(split, false) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init request; struct entry { std::string tx_hash; std::string as_hex; + std::string pruned_as_hex; + std::string prunable_as_hex; + std::string prunable_hash; std::string as_json; bool in_pool; bool double_spend_seen; uint64_t block_height; uint64_t block_timestamp; std::vector output_indices; + bool relayed; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(tx_hash) KV_SERIALIZE(as_hex) + KV_SERIALIZE(pruned_as_hex) + KV_SERIALIZE(prunable_as_hex) + KV_SERIALIZE(prunable_hash) KV_SERIALIZE(as_json) KV_SERIALIZE(in_pool) KV_SERIALIZE(double_spend_seen) - KV_SERIALIZE(block_height) - KV_SERIALIZE(block_timestamp) - KV_SERIALIZE(output_indices) + if (!this_ref.in_pool) + { + KV_SERIALIZE(block_height) + KV_SERIALIZE(block_timestamp) + KV_SERIALIZE(output_indices) + } + else + { + KV_SERIALIZE(relayed) + } END_KV_SERIALIZE_MAP() }; - struct response + struct response_t { // older compatibility stuff - std::list txs_as_hex; //transactions blobs as hex (old compat) - std::list txs_as_json; //transactions decoded as json (old compat) + std::vector txs_as_hex; //transactions blobs as hex (old compat) + std::vector txs_as_json; //transactions decoded as json (old compat) // in both old and new - std::list missed_tx; //not found transactions + std::vector missed_tx; //not found transactions // new style std::vector txs; @@ -618,6 +411,7 @@ namespace cryptonote KV_SERIALIZE(untrusted) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init response; }; //----------------------------------------------- @@ -629,7 +423,7 @@ namespace cryptonote SPENT_IN_POOL = 2, }; - struct request + struct request_t { std::vector key_images; @@ -637,9 +431,10 @@ namespace cryptonote KV_SERIALIZE(key_images) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init request; - struct response + struct response_t { std::vector spent_status; std::string status; @@ -651,21 +446,23 @@ namespace cryptonote KV_SERIALIZE(untrusted) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init response; }; //----------------------------------------------- struct COMMAND_RPC_GET_TX_GLOBAL_OUTPUTS_INDEXES { - struct request + struct request_t { crypto::hash txid; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE_VAL_POD_AS_BLOB(txid) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init request; - struct response + struct response_t { std::vector o_indexes; std::string status; @@ -676,50 +473,7 @@ namespace cryptonote KV_SERIALIZE(untrusted) END_KV_SERIALIZE_MAP() }; - }; - //----------------------------------------------- - struct COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS - { - struct request - { - std::vector amounts; - uint64_t outs_count; - BEGIN_KV_SERIALIZE_MAP() - KV_SERIALIZE(amounts) - KV_SERIALIZE(outs_count) - END_KV_SERIALIZE_MAP() - }; - -#pragma pack (push, 1) - struct out_entry - { - uint64_t global_amount_index; - crypto::public_key out_key; - }; -#pragma pack(pop) - - struct outs_for_amount - { - uint64_t amount; - std::list outs; - - BEGIN_KV_SERIALIZE_MAP() - KV_SERIALIZE(amount) - KV_SERIALIZE_CONTAINER_POD_AS_BLOB(outs) - END_KV_SERIALIZE_MAP() - }; - - struct response - { - std::vector outs; - std::string status; - bool untrusted; - BEGIN_KV_SERIALIZE_MAP() - KV_SERIALIZE(outs) - KV_SERIALIZE(status) - KV_SERIALIZE(untrusted) - END_KV_SERIALIZE_MAP() - }; + typedef epee::misc_utils::struct_init response; }; //----------------------------------------------- struct get_outputs_out @@ -735,14 +489,17 @@ namespace cryptonote struct COMMAND_RPC_GET_OUTPUTS_BIN { - struct request + struct request_t { std::vector outputs; + bool get_txid; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(outputs) + KV_SERIALIZE_OPT(get_txid, true) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init request; struct outkey { @@ -761,7 +518,7 @@ namespace cryptonote END_KV_SERIALIZE_MAP() }; - struct response + struct response_t { std::vector outs; std::string status; @@ -773,18 +530,22 @@ namespace cryptonote KV_SERIALIZE(untrusted) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init response; }; //----------------------------------------------- struct COMMAND_RPC_GET_OUTPUTS { - struct request + struct request_t { std::vector outputs; + bool get_txid; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(outputs) + KV_SERIALIZE(get_txid) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init request; struct outkey { @@ -803,7 +564,7 @@ namespace cryptonote END_KV_SERIALIZE_MAP() }; - struct response + struct response_t { std::vector outs; std::string status; @@ -815,59 +576,27 @@ namespace cryptonote KV_SERIALIZE(untrusted) END_KV_SERIALIZE_MAP() }; - }; - - struct COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS - { - struct request - { - uint64_t outs_count; - BEGIN_KV_SERIALIZE_MAP() - KV_SERIALIZE(outs_count) - END_KV_SERIALIZE_MAP() - }; - -#pragma pack (push, 1) - struct out_entry - { - uint64_t amount; - uint64_t global_amount_index; - crypto::public_key out_key; - rct::key commitment; - }; -#pragma pack(pop) - - struct response - { - std::list outs; - std::string status; - bool untrusted; - BEGIN_KV_SERIALIZE_MAP() - KV_SERIALIZE_CONTAINER_POD_AS_BLOB(outs) - KV_SERIALIZE(status) - KV_SERIALIZE(untrusted) - END_KV_SERIALIZE_MAP() - }; + typedef epee::misc_utils::struct_init response; }; //----------------------------------------------- struct COMMAND_RPC_SEND_RAW_TX { - struct request + struct request_t { std::string tx_as_hex; bool do_not_relay; - - request() {} - explicit request(const transaction &); + bool do_sanity_checks; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(tx_as_hex) KV_SERIALIZE_OPT(do_not_relay, false) + KV_SERIALIZE_OPT(do_sanity_checks, true) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init request; - struct response + struct response_t { std::string status; std::string reason; @@ -880,6 +609,7 @@ namespace cryptonote bool overspend; bool fee_too_low; bool not_rct; + bool sanity_check_failed; bool untrusted; BEGIN_KV_SERIALIZE_MAP() @@ -894,14 +624,16 @@ namespace cryptonote KV_SERIALIZE(overspend) KV_SERIALIZE(fee_too_low) KV_SERIALIZE(not_rct) + KV_SERIALIZE(sanity_check_failed) KV_SERIALIZE(untrusted) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init response; }; //----------------------------------------------- struct COMMAND_RPC_START_MINING { - struct request + struct request_t { std::string miner_address; uint64_t threads_count; @@ -915,8 +647,9 @@ namespace cryptonote KV_SERIALIZE(ignore_battery) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init request; - struct response + struct response_t { std::string status; @@ -924,23 +657,27 @@ namespace cryptonote KV_SERIALIZE(status) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init response; }; //----------------------------------------------- struct COMMAND_RPC_GET_INFO { - struct request + struct request_t { BEGIN_KV_SERIALIZE_MAP() END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init request; - struct response + struct response_t { std::string status; uint64_t height; uint64_t target_height; uint64_t difficulty; + std::string wide_difficulty; + uint64_t difficulty_top64; uint64_t target; uint64_t tx_count; uint64_t tx_pool_size; @@ -953,10 +690,15 @@ namespace cryptonote bool mainnet; bool testnet; bool stagenet; + std::string nettype; std::string top_block_hash; uint64_t cumulative_difficulty; + std::string wide_cumulative_difficulty; + uint64_t cumulative_difficulty_top64; uint64_t block_size_limit; + uint64_t block_weight_limit; uint64_t block_size_median; + uint64_t block_weight_median; uint64_t start_time; uint64_t free_space; bool offline; @@ -964,12 +706,17 @@ namespace cryptonote std::string bootstrap_daemon_address; uint64_t height_without_bootstrap; bool was_bootstrap_ever_used; + uint64_t database_size; + bool update_available; + std::string version; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(status) KV_SERIALIZE(height) KV_SERIALIZE(target_height) KV_SERIALIZE(difficulty) + KV_SERIALIZE(wide_difficulty) + KV_SERIALIZE(difficulty_top64) KV_SERIALIZE(target) KV_SERIALIZE(tx_count) KV_SERIALIZE(tx_pool_size) @@ -982,10 +729,15 @@ namespace cryptonote KV_SERIALIZE(mainnet) KV_SERIALIZE(testnet) KV_SERIALIZE(stagenet) + KV_SERIALIZE(nettype) KV_SERIALIZE(top_block_hash) KV_SERIALIZE(cumulative_difficulty) + KV_SERIALIZE(wide_cumulative_difficulty) + KV_SERIALIZE(cumulative_difficulty_top64) KV_SERIALIZE(block_size_limit) + KV_SERIALIZE_OPT(block_weight_limit, (uint64_t)0) KV_SERIALIZE(block_size_median) + KV_SERIALIZE_OPT(block_weight_median, (uint64_t)0) KV_SERIALIZE(start_time) KV_SERIALIZE(free_space) KV_SERIALIZE(offline) @@ -993,23 +745,61 @@ namespace cryptonote KV_SERIALIZE(bootstrap_daemon_address) KV_SERIALIZE(height_without_bootstrap) KV_SERIALIZE(was_bootstrap_ever_used) + KV_SERIALIZE(database_size) + KV_SERIALIZE(update_available) + KV_SERIALIZE(version) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init response; }; + //----------------------------------------------- + struct COMMAND_RPC_GET_NET_STATS + { + struct request_t + { + + BEGIN_KV_SERIALIZE_MAP() + END_KV_SERIALIZE_MAP() + }; + typedef epee::misc_utils::struct_init request; + + + struct response_t + { + std::string status; + uint64_t start_time; + uint64_t total_packets_in; + uint64_t total_bytes_in; + uint64_t total_packets_out; + uint64_t total_bytes_out; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(status) + KV_SERIALIZE(start_time) + KV_SERIALIZE(total_packets_in) + KV_SERIALIZE(total_bytes_in) + KV_SERIALIZE(total_packets_out) + KV_SERIALIZE(total_bytes_out) + END_KV_SERIALIZE_MAP() + }; + typedef epee::misc_utils::struct_init response; + }; + //----------------------------------------------- struct COMMAND_RPC_STOP_MINING { - struct request + struct request_t { BEGIN_KV_SERIALIZE_MAP() END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init request; - struct response + struct response_t { std::string status; @@ -1017,27 +807,39 @@ namespace cryptonote KV_SERIALIZE(status) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init response; }; //----------------------------------------------- struct COMMAND_RPC_MINING_STATUS { - struct request + struct request_t { BEGIN_KV_SERIALIZE_MAP() END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init request; - struct response + struct response_t { std::string status; bool active; uint64_t speed; uint32_t threads_count; std::string address; + std::string pow_algorithm; bool is_background_mining_enabled; + uint8_t bg_idle_threshold; + uint8_t bg_min_idle_seconds; + bool bg_ignore_battery; + uint8_t bg_target; + uint32_t block_target; + uint64_t block_reward; + uint64_t difficulty; + std::string wide_difficulty; + uint64_t difficulty_top64; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(status) @@ -1045,23 +847,35 @@ namespace cryptonote KV_SERIALIZE(speed) KV_SERIALIZE(threads_count) KV_SERIALIZE(address) + KV_SERIALIZE(pow_algorithm) KV_SERIALIZE(is_background_mining_enabled) + KV_SERIALIZE(bg_idle_threshold) + KV_SERIALIZE(bg_min_idle_seconds) + KV_SERIALIZE(bg_ignore_battery) + KV_SERIALIZE(bg_target) + KV_SERIALIZE(block_target) + KV_SERIALIZE(block_reward) + KV_SERIALIZE(difficulty) + KV_SERIALIZE(wide_difficulty) + KV_SERIALIZE(difficulty_top64) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init response; }; //----------------------------------------------- struct COMMAND_RPC_SAVE_BC { - struct request + struct request_t { BEGIN_KV_SERIALIZE_MAP() END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init request; - struct response + struct response_t { std::string status; @@ -1069,6 +883,7 @@ namespace cryptonote KV_SERIALIZE(status) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init response; }; // @@ -1076,7 +891,7 @@ namespace cryptonote { typedef std::list request; - struct response + struct response_t { uint64_t count; std::string status; @@ -1086,7 +901,7 @@ namespace cryptonote KV_SERIALIZE(status) END_KV_SERIALIZE_MAP() }; - + typedef epee::misc_utils::struct_init response; }; struct COMMAND_RPC_GETBLOCKHASH @@ -1099,20 +914,25 @@ namespace cryptonote struct COMMAND_RPC_GETBLOCKTEMPLATE { - struct request + struct request_t { uint64_t reserve_size; //max 255 bytes std::string wallet_address; + std::string prev_block; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(reserve_size) KV_SERIALIZE(wallet_address) + KV_SERIALIZE(prev_block) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init request; - struct response + struct response_t { uint64_t difficulty; + std::string wide_difficulty; + uint64_t difficulty_top64; uint64_t height; uint64_t reserved_offset; uint64_t expected_reward; @@ -1124,6 +944,8 @@ namespace cryptonote BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(difficulty) + KV_SERIALIZE(wide_difficulty) + KV_SERIALIZE(difficulty_top64) KV_SERIALIZE(height) KV_SERIALIZE(reserved_offset) KV_SERIALIZE(expected_reward) @@ -1134,13 +956,14 @@ namespace cryptonote KV_SERIALIZE(untrusted) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init response; }; struct COMMAND_RPC_SUBMITBLOCK { typedef std::vector request; - struct response + struct response_t { std::string status; @@ -1148,6 +971,40 @@ namespace cryptonote KV_SERIALIZE(status) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init response; + }; + + struct COMMAND_RPC_GENERATEBLOCKS + { + struct request_t + { + uint64_t amount_of_blocks; + std::string wallet_address; + std::string prev_block; + uint32_t starting_nonce; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(amount_of_blocks) + KV_SERIALIZE(wallet_address) + KV_SERIALIZE(prev_block) + KV_SERIALIZE_OPT(starting_nonce, (uint32_t)0) + END_KV_SERIALIZE_MAP() + }; + typedef epee::misc_utils::struct_init request; + + struct response_t + { + uint64_t height; + std::vector blocks; + std::string status; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(height) + KV_SERIALIZE(blocks) + KV_SERIALIZE(status) + END_KV_SERIALIZE_MAP() + }; + typedef epee::misc_utils::struct_init response; }; struct block_header_response @@ -1161,11 +1018,19 @@ namespace cryptonote uint64_t height; uint64_t depth; std::string hash; - difficulty_type difficulty; + uint64_t difficulty; + std::string wide_difficulty; + uint64_t difficulty_top64; + uint64_t cumulative_difficulty; + std::string wide_cumulative_difficulty; + uint64_t cumulative_difficulty_top64; uint64_t reward; uint64_t block_size; + uint64_t block_weight; uint64_t num_txes; std::string pow_hash; + uint64_t long_term_weight; + std::string miner_tx_hash; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(major_version) @@ -1178,16 +1043,24 @@ namespace cryptonote KV_SERIALIZE(depth) KV_SERIALIZE(hash) KV_SERIALIZE(difficulty) + KV_SERIALIZE(wide_difficulty) + KV_SERIALIZE(difficulty_top64) + KV_SERIALIZE(cumulative_difficulty) + KV_SERIALIZE(wide_cumulative_difficulty) + KV_SERIALIZE(cumulative_difficulty_top64) KV_SERIALIZE(reward) KV_SERIALIZE(block_size) + KV_SERIALIZE_OPT(block_weight, (uint64_t)0) KV_SERIALIZE(num_txes) KV_SERIALIZE(pow_hash) + KV_SERIALIZE_OPT(long_term_weight, (uint64_t)0) + KV_SERIALIZE(miner_tx_hash) END_KV_SERIALIZE_MAP() }; struct COMMAND_RPC_GET_LAST_BLOCK_HEADER { - struct request + struct request_t { bool fill_pow_hash; @@ -1195,8 +1068,9 @@ namespace cryptonote KV_SERIALIZE_OPT(fill_pow_hash, false); END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init request; - struct response + struct response_t { std::string status; block_header_response block_header; @@ -1208,12 +1082,13 @@ namespace cryptonote KV_SERIALIZE(untrusted) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init response; }; struct COMMAND_RPC_GET_BLOCK_HEADER_BY_HASH { - struct request + struct request_t { std::string hash; bool fill_pow_hash; @@ -1223,8 +1098,9 @@ namespace cryptonote KV_SERIALIZE_OPT(fill_pow_hash, false); END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init request; - struct response + struct response_t { std::string status; block_header_response block_header; @@ -1236,12 +1112,12 @@ namespace cryptonote KV_SERIALIZE(untrusted) END_KV_SERIALIZE_MAP() }; - + typedef epee::misc_utils::struct_init response; }; struct COMMAND_RPC_GET_BLOCK_HEADER_BY_HEIGHT { - struct request + struct request_t { uint64_t height; bool fill_pow_hash; @@ -1251,8 +1127,9 @@ namespace cryptonote KV_SERIALIZE_OPT(fill_pow_hash, false); END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init request; - struct response + struct response_t { std::string status; block_header_response block_header; @@ -1264,12 +1141,12 @@ namespace cryptonote KV_SERIALIZE(untrusted) END_KV_SERIALIZE_MAP() }; - + typedef epee::misc_utils::struct_init response; }; struct COMMAND_RPC_GET_BLOCK { - struct request + struct request_t { std::string hash; uint64_t height; @@ -1281,8 +1158,9 @@ namespace cryptonote KV_SERIALIZE_OPT(fill_pow_hash, false); END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init request; - struct response + struct response_t { std::string status; block_header_response block_header; @@ -1302,7 +1180,7 @@ namespace cryptonote KV_SERIALIZE(untrusted) END_KV_SERIALIZE_MAP() }; - + typedef epee::misc_utils::struct_init response; }; struct peer { @@ -1310,15 +1188,17 @@ namespace cryptonote std::string host; uint32_t ip; uint16_t port; + uint16_t rpc_port; uint64_t last_seen; + uint32_t pruning_seed; peer() = default; - peer(uint64_t id, const std::string &host, uint64_t last_seen) - : id(id), host(host), ip(0), port(0), last_seen(last_seen) + peer(uint64_t id, const std::string &host, uint64_t last_seen, uint32_t pruning_seed, uint16_t rpc_port) + : id(id), host(host), ip(0), port(0), rpc_port(rpc_port), last_seen(last_seen), pruning_seed(pruning_seed) {} - peer(uint64_t id, uint32_t ip, uint16_t port, uint64_t last_seen) - : id(id), host(std::to_string(ip)), ip(ip), port(port), last_seen(last_seen) + peer(uint64_t id, uint32_t ip, uint16_t port, uint64_t last_seen, uint32_t pruning_seed, uint16_t rpc_port) + : id(id), host(std::to_string(ip)), ip(ip), port(port), rpc_port(rpc_port), last_seen(last_seen), pruning_seed(pruning_seed) {} BEGIN_KV_SERIALIZE_MAP() @@ -1326,19 +1206,22 @@ namespace cryptonote KV_SERIALIZE(host) KV_SERIALIZE(ip) KV_SERIALIZE(port) + KV_SERIALIZE_OPT(rpc_port, (uint16_t)0) KV_SERIALIZE(last_seen) + KV_SERIALIZE_OPT(pruning_seed, (uint32_t)0) END_KV_SERIALIZE_MAP() }; struct COMMAND_RPC_GET_PEER_LIST { - struct request + struct request_t { BEGIN_KV_SERIALIZE_MAP() END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init request; - struct response + struct response_t { std::string status; std::vector white_list; @@ -1350,11 +1233,12 @@ namespace cryptonote KV_SERIALIZE(gray_list) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init response; }; struct COMMAND_RPC_SET_LOG_HASH_RATE { - struct request + struct request_t { bool visible; @@ -1362,19 +1246,21 @@ namespace cryptonote KV_SERIALIZE(visible) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init request; - struct response + struct response_t { std::string status; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(status) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init response; }; struct COMMAND_RPC_SET_LOG_LEVEL { - struct request + struct request_t { int8_t level; @@ -1382,19 +1268,21 @@ namespace cryptonote KV_SERIALIZE(level) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init request; - struct response + struct response_t { std::string status; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(status) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init response; }; struct COMMAND_RPC_SET_LOG_CATEGORIES { - struct request + struct request_t { std::string categories; @@ -1402,8 +1290,9 @@ namespace cryptonote KV_SERIALIZE(categories) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init request; - struct response + struct response_t { std::string status; std::string categories; @@ -1413,6 +1302,7 @@ namespace cryptonote KV_SERIALIZE(categories) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init response; }; struct tx_info @@ -1420,6 +1310,7 @@ namespace cryptonote std::string id_hash; std::string tx_json; // TODO - expose this data directly uint64_t blob_size; + uint64_t weight; uint64_t fee; std::string max_used_block_id_hash; uint64_t max_used_block_height; @@ -1437,6 +1328,7 @@ namespace cryptonote KV_SERIALIZE(id_hash) KV_SERIALIZE(tx_json) KV_SERIALIZE(blob_size) + KV_SERIALIZE_OPT(weight, (uint64_t)0) KV_SERIALIZE(fee) KV_SERIALIZE(max_used_block_id_hash) KV_SERIALIZE(max_used_block_height) @@ -1465,13 +1357,14 @@ namespace cryptonote struct COMMAND_RPC_GET_TRANSACTION_POOL { - struct request + struct request_t { BEGIN_KV_SERIALIZE_MAP() END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init request; - struct response + struct response_t { std::string status; std::vector transactions; @@ -1485,17 +1378,19 @@ namespace cryptonote KV_SERIALIZE(untrusted) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init response; }; - struct COMMAND_RPC_GET_TRANSACTION_POOL_HASHES + struct COMMAND_RPC_GET_TRANSACTION_POOL_HASHES_BIN { - struct request + struct request_t { BEGIN_KV_SERIALIZE_MAP() END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init request; - struct response + struct response_t { std::string status; std::vector tx_hashes; @@ -1507,24 +1402,50 @@ namespace cryptonote KV_SERIALIZE(untrusted) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init response; + }; + + struct COMMAND_RPC_GET_TRANSACTION_POOL_HASHES + { + struct request_t + { + BEGIN_KV_SERIALIZE_MAP() + END_KV_SERIALIZE_MAP() + }; + typedef epee::misc_utils::struct_init request; + + struct response_t + { + std::string status; + std::vector tx_hashes; + bool untrusted; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(status) + KV_SERIALIZE(tx_hashes) + KV_SERIALIZE(untrusted) + END_KV_SERIALIZE_MAP() + }; + typedef epee::misc_utils::struct_init response; }; struct tx_backlog_entry { - uint64_t blob_size; + uint64_t weight; uint64_t fee; uint64_t time_in_pool; }; struct COMMAND_RPC_GET_TRANSACTION_POOL_BACKLOG { - struct request + struct request_t { BEGIN_KV_SERIALIZE_MAP() END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init request; - struct response + struct response_t { std::string status; std::vector backlog; @@ -1536,6 +1457,7 @@ namespace cryptonote KV_SERIALIZE(untrusted) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init response; }; struct txpool_histo @@ -1565,6 +1487,8 @@ namespace cryptonote std::vector histo; uint32_t num_double_spends; + txpool_stats(): bytes_total(0), bytes_min(0), bytes_max(0), bytes_med(0), fee_total(0), oldest(0), txs_total(0), num_failing(0), num_10m(0), num_not_relayed(0), histo_98pc(0), num_double_spends(0) {} + BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(bytes_total) KV_SERIALIZE(bytes_min) @@ -1584,13 +1508,14 @@ namespace cryptonote struct COMMAND_RPC_GET_TRANSACTION_POOL_STATS { - struct request + struct request_t { BEGIN_KV_SERIALIZE_MAP() END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init request; - struct response + struct response_t { std::string status; txpool_stats pool_stats; @@ -1602,17 +1527,19 @@ namespace cryptonote KV_SERIALIZE(untrusted) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init response; }; struct COMMAND_RPC_GET_CONNECTIONS { - struct request + struct request_t { BEGIN_KV_SERIALIZE_MAP() END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init request; - struct response + struct response_t { std::string status; std::list connections; @@ -1622,12 +1549,12 @@ namespace cryptonote KV_SERIALIZE(connections) END_KV_SERIALIZE_MAP() }; - + typedef epee::misc_utils::struct_init response; }; struct COMMAND_RPC_GET_BLOCK_HEADERS_RANGE { - struct request + struct request_t { uint64_t start_height; uint64_t end_height; @@ -1639,8 +1566,9 @@ namespace cryptonote KV_SERIALIZE_OPT(fill_pow_hash, false); END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init request; - struct response + struct response_t { std::string status; std::vector headers; @@ -1652,17 +1580,19 @@ namespace cryptonote KV_SERIALIZE(untrusted) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init response; }; struct COMMAND_RPC_STOP_DAEMON { - struct request + struct request_t { BEGIN_KV_SERIALIZE_MAP() END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init request; - struct response + struct response_t { std::string status; @@ -1670,17 +1600,19 @@ namespace cryptonote KV_SERIALIZE(status) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init response; }; struct COMMAND_RPC_FAST_EXIT { - struct request + struct request_t { BEGIN_KV_SERIALIZE_MAP() END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init request; - struct response + struct response_t { std::string status; @@ -1688,17 +1620,19 @@ namespace cryptonote KV_SERIALIZE(status) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init response; }; struct COMMAND_RPC_GET_LIMIT { - struct request + struct request_t { BEGIN_KV_SERIALIZE_MAP() END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init request; - struct response + struct response_t { std::string status; uint64_t limit_up; @@ -1712,11 +1646,12 @@ namespace cryptonote KV_SERIALIZE(untrusted) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init response; }; struct COMMAND_RPC_SET_LIMIT { - struct request + struct request_t { int64_t limit_down; // all limits (for get and set) are kB/s int64_t limit_up; @@ -1726,8 +1661,9 @@ namespace cryptonote KV_SERIALIZE(limit_up) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init request; - struct response + struct response_t { std::string status; int64_t limit_up; @@ -1739,19 +1675,21 @@ namespace cryptonote KV_SERIALIZE(limit_down) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init response; }; struct COMMAND_RPC_OUT_PEERS { - struct request + struct request_t { uint64_t out_peers; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(out_peers) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init request; - struct response + struct response_t { std::string status; @@ -1759,19 +1697,21 @@ namespace cryptonote KV_SERIALIZE(status) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init response; }; struct COMMAND_RPC_IN_PEERS { - struct request + struct request_t { uint64_t in_peers; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(in_peers) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init request; - struct response + struct response_t { std::string status; @@ -1779,17 +1719,19 @@ namespace cryptonote KV_SERIALIZE(status) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init response; }; struct COMMAND_RPC_START_SAVE_GRAPH { - struct request + struct request_t { BEGIN_KV_SERIALIZE_MAP() END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init request; - struct response + struct response_t { std::string status; @@ -1797,17 +1739,19 @@ namespace cryptonote KV_SERIALIZE(status) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init response; }; struct COMMAND_RPC_STOP_SAVE_GRAPH { - struct request + struct request_t { BEGIN_KV_SERIALIZE_MAP() END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init request; - struct response + struct response_t { std::string status; @@ -1815,11 +1759,12 @@ namespace cryptonote KV_SERIALIZE(status) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init response; }; struct COMMAND_RPC_HARD_FORK_INFO { - struct request + struct request_t { uint8_t version; @@ -1827,8 +1772,9 @@ namespace cryptonote KV_SERIALIZE(version) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init request; - struct response + struct response_t { uint8_t version; bool enabled; @@ -1854,6 +1800,7 @@ namespace cryptonote KV_SERIALIZE(untrusted) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init response; }; struct COMMAND_RPC_GETBANS @@ -1871,13 +1818,14 @@ namespace cryptonote END_KV_SERIALIZE_MAP() }; - struct request + struct request_t { BEGIN_KV_SERIALIZE_MAP() END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init request; - struct response + struct response_t { std::string status; std::vector bans; @@ -1887,6 +1835,7 @@ namespace cryptonote KV_SERIALIZE(bans) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init response; }; struct COMMAND_RPC_SETBANS @@ -1906,7 +1855,7 @@ namespace cryptonote END_KV_SERIALIZE_MAP() }; - struct request + struct request_t { std::vector bans; @@ -1914,8 +1863,9 @@ namespace cryptonote KV_SERIALIZE(bans) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init request; - struct response + struct response_t { std::string status; @@ -1923,20 +1873,22 @@ namespace cryptonote KV_SERIALIZE(status) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init response; }; struct COMMAND_RPC_FLUSH_TRANSACTION_POOL { - struct request + struct request_t { - std::list txids; + std::vector txids; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(txids) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init request; - struct response + struct response_t { std::string status; @@ -1944,11 +1896,12 @@ namespace cryptonote KV_SERIALIZE(status) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init response; }; struct COMMAND_RPC_GET_OUTPUT_HISTOGRAM { - struct request + struct request_t { std::vector amounts; uint64_t min_count; @@ -1964,6 +1917,7 @@ namespace cryptonote KV_SERIALIZE(recent_cutoff); END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init request; struct entry { @@ -1984,7 +1938,7 @@ namespace cryptonote entry() {} }; - struct response + struct response_t { std::string status; std::vector histogram; @@ -1996,17 +1950,19 @@ namespace cryptonote KV_SERIALIZE(untrusted) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init response; }; struct COMMAND_RPC_GET_VERSION { - struct request + struct request_t { BEGIN_KV_SERIALIZE_MAP() END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init request; - struct response + struct response_t { std::string status; uint32_t version; @@ -2018,11 +1974,12 @@ namespace cryptonote KV_SERIALIZE(untrusted) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init response; }; struct COMMAND_RPC_GET_COINBASE_TX_SUM { - struct request + struct request_t { uint64_t height; uint64_t count; @@ -2032,8 +1989,9 @@ namespace cryptonote KV_SERIALIZE(count); END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init request; - struct response + struct response_t { std::string status; uint64_t emission_amount; @@ -2045,11 +2003,12 @@ namespace cryptonote KV_SERIALIZE(fee_amount) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init response; }; - struct COMMAND_RPC_GET_PER_KB_FEE_ESTIMATE + struct COMMAND_RPC_GET_BASE_FEE_ESTIMATE { - struct request + struct request_t { uint64_t grace_blocks; @@ -2057,28 +2016,33 @@ namespace cryptonote KV_SERIALIZE(grace_blocks) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init request; - struct response + struct response_t { std::string status; uint64_t fee; + uint64_t quantization_mask; bool untrusted; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(status) KV_SERIALIZE(fee) + KV_SERIALIZE_OPT(quantization_mask, (uint64_t)1) KV_SERIALIZE(untrusted) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init response; }; struct COMMAND_RPC_GET_ALTERNATE_CHAINS { - struct request + struct request_t { BEGIN_KV_SERIALIZE_MAP() END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init request; struct chain_info { @@ -2086,16 +2050,24 @@ namespace cryptonote uint64_t height; uint64_t length; uint64_t difficulty; + std::string wide_difficulty; + uint64_t difficulty_top64; + std::vector block_hashes; + std::string main_chain_parent_block; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(block_hash) KV_SERIALIZE(height) KV_SERIALIZE(length) KV_SERIALIZE(difficulty) + KV_SERIALIZE(wide_difficulty) + KV_SERIALIZE(difficulty_top64) + KV_SERIALIZE(block_hashes) + KV_SERIALIZE(main_chain_parent_block) END_KV_SERIALIZE_MAP() }; - struct response + struct response_t { std::string status; std::list chains; @@ -2105,11 +2077,12 @@ namespace cryptonote KV_SERIALIZE(chains) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init response; }; struct COMMAND_RPC_UPDATE { - struct request + struct request_t { std::string command; std::string path; @@ -2119,8 +2092,9 @@ namespace cryptonote KV_SERIALIZE(path); END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init request; - struct response + struct response_t { std::string status; bool update; @@ -2140,20 +2114,22 @@ namespace cryptonote KV_SERIALIZE(path) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init response; }; struct COMMAND_RPC_RELAY_TX { - struct request + struct request_t { - std::list txids; + std::vector txids; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(txids) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init request; - struct response + struct response_t { std::string status; @@ -2161,15 +2137,17 @@ namespace cryptonote KV_SERIALIZE(status) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init response; }; struct COMMAND_RPC_SYNC_INFO { - struct request + struct request_t { BEGIN_KV_SERIALIZE_MAP() END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init request; struct peer { @@ -2201,66 +2179,158 @@ namespace cryptonote END_KV_SERIALIZE_MAP() }; - struct response + struct response_t { std::string status; uint64_t height; uint64_t target_height; + uint32_t next_needed_pruning_seed; std::list peers; std::list spans; + std::string overview; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(status) KV_SERIALIZE(height) KV_SERIALIZE(target_height) + KV_SERIALIZE(next_needed_pruning_seed) KV_SERIALIZE(peers) KV_SERIALIZE(spans) + KV_SERIALIZE(overview) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init response; }; struct COMMAND_RPC_GET_OUTPUT_DISTRIBUTION { - struct request + struct request_t { std::vector amounts; uint64_t from_height; uint64_t to_height; bool cumulative; + bool binary; + bool compress; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(amounts) KV_SERIALIZE_OPT(from_height, (uint64_t)0) KV_SERIALIZE_OPT(to_height, (uint64_t)0) KV_SERIALIZE_OPT(cumulative, false) + KV_SERIALIZE_OPT(binary, true) + KV_SERIALIZE_OPT(compress, false) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init request; struct distribution { + rpc::output_distribution_data data; uint64_t amount; - uint64_t start_height; - std::vector distribution; - uint64_t base; + std::string compressed_data; + bool binary; + bool compress; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(amount) - KV_SERIALIZE(start_height) - KV_SERIALIZE_CONTAINER_POD_AS_BLOB(distribution) - KV_SERIALIZE(base) + KV_SERIALIZE_N(data.start_height, "start_height") + KV_SERIALIZE(binary) + KV_SERIALIZE(compress) + if (this_ref.binary) + { + if (is_store) + { + if (this_ref.compress) + { + const_cast(this_ref.compressed_data) = compress_integer_array(this_ref.data.distribution); + KV_SERIALIZE(compressed_data) + } + else + KV_SERIALIZE_CONTAINER_POD_AS_BLOB_N(data.distribution, "distribution") + } + else + { + if (this_ref.compress) + { + KV_SERIALIZE(compressed_data) + const_cast&>(this_ref.data.distribution) = decompress_integer_array(this_ref.compressed_data); + } + else + KV_SERIALIZE_CONTAINER_POD_AS_BLOB_N(data.distribution, "distribution") + } + } + else + KV_SERIALIZE_N(data.distribution, "distribution") + KV_SERIALIZE_N(data.base, "base") + END_KV_SERIALIZE_MAP() + }; + + struct response_t + { + std::string status; + std::vector distributions; + bool untrusted; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(status) + KV_SERIALIZE(distributions) + KV_SERIALIZE(untrusted) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init response; + }; + + struct COMMAND_RPC_POP_BLOCKS + { + struct request_t + { + uint64_t nblocks; - struct response + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(nblocks); + END_KV_SERIALIZE_MAP() + }; + typedef epee::misc_utils::struct_init request; + + struct response_t { std::string status; - std::vector distributions; + uint64_t height; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(status) - KV_SERIALIZE(distributions) + KV_SERIALIZE(height) + END_KV_SERIALIZE_MAP() + }; + typedef epee::misc_utils::struct_init response; + }; + + struct COMMAND_RPC_PRUNE_BLOCKCHAIN + { + struct request_t + { + bool check; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE_OPT(check, false) + END_KV_SERIALIZE_MAP() + }; + typedef epee::misc_utils::struct_init request; + + struct response_t + { + bool pruned; + uint32_t pruning_seed; + std::string status; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(status) + KV_SERIALIZE(pruned) + KV_SERIALIZE(pruning_seed) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init response; }; } diff --git a/src/rpc/core_rpc_server_error_codes.h b/src/rpc/core_rpc_server_error_codes.h index 69caaa6a63d..b13049e61c8 100644 --- a/src/rpc/core_rpc_server_error_codes.h +++ b/src/rpc/core_rpc_server_error_codes.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -42,5 +42,6 @@ #define CORE_RPC_ERROR_CODE_WRONG_BLOCKBLOB_SIZE -10 #define CORE_RPC_ERROR_CODE_UNSUPPORTED_RPC -11 #define CORE_RPC_ERROR_CODE_MINING_TO_SUBADDRESS -12 +#define CORE_RPC_ERROR_CODE_REGTEST_REQUIRED -13 diff --git a/src/rpc/daemon_handler.cpp b/src/rpc/daemon_handler.cpp index 39f169cdf9f..612b2cab66e 100644 --- a/src/rpc/daemon_handler.cpp +++ b/src/rpc/daemon_handler.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2018, The Monero Project +// Copyright (c) 2017-2019, The Monero Project // // All rights reserved. // @@ -34,6 +34,7 @@ #include "cryptonote_basic/cryptonote_format_utils.h" #include "cryptonote_basic/blobdatatype.h" #include "ringct/rctSigs.h" +#include "version.h" namespace cryptonote { @@ -50,9 +51,9 @@ namespace rpc void DaemonHandler::handle(const GetBlocksFast::Request& req, GetBlocksFast::Response& res) { - std::list > > blocks; + std::vector, std::vector > > > blocks; - if(!m_core.find_blockchain_supplement(req.start_height, req.block_ids, blocks, res.current_height, res.start_height, req.prune, COMMAND_RPC_GET_BLOCKS_FAST_MAX_COUNT)) + if(!m_core.find_blockchain_supplement(req.start_height, req.block_ids, blocks, res.current_height, res.start_height, req.prune, true, COMMAND_RPC_GET_BLOCKS_FAST_MAX_COUNT)) { res.status = Message::STATUS_FAILED; res.error_details = "core::find_blockchain_supplement() returned false"; @@ -62,9 +63,6 @@ namespace rpc res.blocks.resize(blocks.size()); res.output_indices.resize(blocks.size()); - //TODO: really need to switch uses of std::list to std::vector unless - // it's a huge performance concern - auto it = blocks.begin(); uint64_t block_count = 0; @@ -72,7 +70,7 @@ namespace rpc { cryptonote::rpc::block_with_transactions& bwt = res.blocks[block_count]; - if (!parse_and_validate_block_from_blob(it->first, bwt.block)) + if (!parse_and_validate_block_from_blob(it->first.first, bwt.block)) { res.blocks.clear(); res.output_indices.clear(); @@ -89,53 +87,45 @@ namespace rpc res.error_details = "incorrect number of transactions retrieved for block"; return; } - std::list txs; - for (const auto& blob : it->second) - { - txs.resize(txs.size() + 1); - if (!parse_and_validate_tx_from_blob(blob, txs.back())) - { - res.blocks.clear(); - res.output_indices.clear(); - res.status = Message::STATUS_FAILED; - res.error_details = "failed retrieving a requested transaction"; - return; - } - } cryptonote::rpc::block_output_indices& indices = res.output_indices[block_count]; // miner tx output indices { cryptonote::rpc::tx_output_indices tx_indices; - bool r = m_core.get_tx_outputs_gindexs(get_transaction_hash(bwt.block.miner_tx), tx_indices); - if (!r) + if (!m_core.get_tx_outputs_gindexs(get_transaction_hash(bwt.block.miner_tx), tx_indices)) { res.status = Message::STATUS_FAILED; res.error_details = "core::get_tx_outputs_gindexs() returned false"; return; } - indices.push_back(tx_indices); + indices.push_back(std::move(tx_indices)); } - // assume each block returned is returned with all its transactions - // in the correct order. - auto tx_it = txs.begin(); - for (const crypto::hash& h : bwt.block.tx_hashes) + auto hash_it = bwt.block.tx_hashes.begin(); + bwt.transactions.reserve(it->second.size()); + for (const auto& blob : it->second) { - bwt.transactions.emplace(h, *tx_it); - tx_it++; + bwt.transactions.emplace_back(); + if (!parse_and_validate_tx_from_blob(blob.second, bwt.transactions.back())) + { + res.blocks.clear(); + res.output_indices.clear(); + res.status = Message::STATUS_FAILED; + res.error_details = "failed retrieving a requested transaction"; + return; + } cryptonote::rpc::tx_output_indices tx_indices; - bool r = m_core.get_tx_outputs_gindexs(h, tx_indices); - if (!r) + if (!m_core.get_tx_outputs_gindexs(*hash_it, tx_indices)) { res.status = Message::STATUS_FAILED; res.error_details = "core::get_tx_outputs_gindexs() returned false"; return; } - indices.push_back(tx_indices); + indices.push_back(std::move(tx_indices)); + ++hash_it; } it++; @@ -151,7 +141,7 @@ namespace rpc auto& chain = m_core.get_blockchain_storage(); - if (!chain.find_blockchain_supplement(req.known_hashes, res.hashes, res.start_height, res.current_height)) + if (!chain.find_blockchain_supplement(req.known_hashes, res.hashes, res.start_height, res.current_height, false)) { res.status = Message::STATUS_FAILED; res.error_details = "Blockchain::find_blockchain_supplement() returned false"; @@ -163,10 +153,10 @@ namespace rpc void DaemonHandler::handle(const GetTransactions::Request& req, GetTransactions::Response& res) { - std::list found_txs; - std::list missed_hashes; + std::vector found_txs_vec; + std::vector missed_vec; - bool r = m_core.get_transactions(req.tx_hashes, found_txs, missed_hashes); + bool r = m_core.get_transactions(req.tx_hashes, found_txs_vec, missed_vec); // TODO: consider fixing core::get_transactions to not hide exceptions if (!r) @@ -176,20 +166,7 @@ namespace rpc return; } - size_t num_found = found_txs.size(); - - // std::list is annoying - std::vector found_txs_vec - { - std::make_move_iterator(std::begin(found_txs)), - std::make_move_iterator(std::end(found_txs)) - }; - - std::vector missed_vec - { - std::make_move_iterator(std::begin(missed_hashes)), - std::make_move_iterator(std::end(missed_hashes)) - }; + size_t num_found = found_txs_vec.size(); std::vector heights(num_found); std::vector in_pool(num_found, false); @@ -204,7 +181,7 @@ namespace rpc // if any missing from blockchain, check in tx pool if (!missed_vec.empty()) { - std::list pool_txs; + std::vector pool_txs; m_core.get_pool_transactions(pool_txs); @@ -284,60 +261,45 @@ namespace rpc } - //TODO: handle "restricted" RPC - void DaemonHandler::handle(const GetRandomOutputsForAmounts::Request& req, GetRandomOutputsForAmounts::Response& res) + void DaemonHandler::handle(const SendRawTx::Request& req, SendRawTx::Response& res) { - auto& chain = m_core.get_blockchain_storage(); - - try - { - for (const uint64_t& amount : req.amounts) - { - std::vector indices = chain.get_random_outputs(amount, req.count); - - outputs_for_amount ofa; - - ofa.resize(indices.size()); - - for (size_t i = 0; i < indices.size(); i++) - { - crypto::public_key key = chain.get_output_key(amount, indices[i]); - ofa[i].amount_index = indices[i]; - ofa[i].key = key; - } - - amount_with_random_outputs amt; - amt.amount = amount; - amt.outputs = ofa; - - res.amounts_with_outputs.push_back(amt); - } + handleTxBlob(cryptonote::tx_to_blob(req.tx), req.relay, res); + } - res.status = Message::STATUS_OK; - } - catch (const std::exception& e) + void DaemonHandler::handle(const SendRawTxHex::Request& req, SendRawTxHex::Response& res) + { + std::string tx_blob; + if(!epee::string_tools::parse_hexstr_to_binbuff(req.tx_as_hex, tx_blob)) { + MERROR("[SendRawTxHex]: Failed to parse tx from hexbuff: " << req.tx_as_hex); res.status = Message::STATUS_FAILED; - res.error_details = e.what(); + res.error_details = "Invalid hex"; + return; } + handleTxBlob(tx_blob, req.relay, res); } - void DaemonHandler::handle(const SendRawTx::Request& req, SendRawTx::Response& res) + void DaemonHandler::handleTxBlob(const std::string& tx_blob, bool relay, SendRawTx::Response& res) { - auto tx_blob = cryptonote::tx_to_blob(req.tx); + if (!m_p2p.get_payload_object().is_synchronized()) + { + res.status = Message::STATUS_FAILED; + res.error_details = "Not ready to accept transactions; try again later"; + return; + } cryptonote_connection_context fake_context = AUTO_VAL_INIT(fake_context); tx_verification_context tvc = AUTO_VAL_INIT(tvc); - if(!m_core.handle_incoming_tx(tx_blob, tvc, false, false, !req.relay) || tvc.m_verifivation_failed) + if(!m_core.handle_incoming_tx(tx_blob, tvc, false, false, !relay) || tvc.m_verifivation_failed) { if (tvc.m_verifivation_failed) { - LOG_PRINT_L0("[on_send_raw_tx]: tx verification failed"); + MERROR("[SendRawTx]: tx verification failed"); } else { - LOG_PRINT_L0("[on_send_raw_tx]: Failed to process tx"); + MERROR("[SendRawTx]: Failed to process tx"); } res.status = Message::STATUS_FAILED; res.error_details = ""; @@ -389,9 +351,9 @@ namespace rpc return; } - if(!tvc.m_should_be_relayed || !req.relay) + if(!tvc.m_should_be_relayed || !relay) { - LOG_PRINT_L0("[on_send_raw_tx]: tx accepted, but not relayed"); + MERROR("[SendRawTx]: tx accepted, but not relayed"); res.error_details = "Not relayed"; res.relayed = false; res.status = Message::STATUS_OK; @@ -446,10 +408,7 @@ namespace rpc return; } - boost::thread::attributes attrs; - attrs.set_stack_size(THREAD_STACK_SIZE); - - if(!m_core.get_miner().start(info.address, static_cast(req.threads_count), attrs, req.do_background_mining, req.ignore_battery)) + if(!m_core.get_miner().start(info.address, static_cast(req.threads_count), req.do_background_mining, req.ignore_battery)) { res.error_details = "Failed, mining not started"; LOG_PRINT_L0(res.error_details); @@ -474,7 +433,8 @@ namespace rpc auto& chain = m_core.get_blockchain_storage(); - res.info.difficulty = chain.get_difficulty_for_next_block(); + res.info.wide_difficulty = chain.get_difficulty_for_next_block(); + res.info.difficulty = (res.info.wide_difficulty & 0xffffffffffffffff).convert_to(); res.info.target = chain.get_difficulty_target(); @@ -484,20 +444,23 @@ namespace rpc res.info.alt_blocks_count = chain.get_alternative_blocks_count(); - uint64_t total_conn = m_p2p.get_connections_count(); - res.info.outgoing_connections_count = m_p2p.get_outgoing_connections_count(); + uint64_t total_conn = m_p2p.get_public_connections_count(); + res.info.outgoing_connections_count = m_p2p.get_public_outgoing_connections_count(); res.info.incoming_connections_count = total_conn - res.info.outgoing_connections_count; - res.info.white_peerlist_size = m_p2p.get_peerlist_manager().get_white_peers_count(); + res.info.white_peerlist_size = m_p2p.get_public_white_peers_count(); - res.info.grey_peerlist_size = m_p2p.get_peerlist_manager().get_gray_peers_count(); + res.info.grey_peerlist_size = m_p2p.get_public_gray_peers_count(); res.info.mainnet = m_core.get_nettype() == MAINNET; res.info.testnet = m_core.get_nettype() == TESTNET; res.info.stagenet = m_core.get_nettype() == STAGENET; - res.info.cumulative_difficulty = m_core.get_blockchain_storage().get_db().get_block_cumulative_difficulty(res.info.height - 1); - res.info.block_size_limit = m_core.get_blockchain_storage().get_current_cumulative_blocksize_limit(); + res.info.wide_cumulative_difficulty = m_core.get_blockchain_storage().get_db().get_block_cumulative_difficulty(res.info.height - 1); + res.info.cumulative_difficulty = (res.info.wide_cumulative_difficulty & 0xffffffffffffffff).convert_to(); + res.info.block_size_limit = res.info.block_weight_limit = m_core.get_blockchain_storage().get_current_cumulative_block_weight_limit(); + res.info.block_size_median = res.info.block_weight_median = m_core.get_blockchain_storage().get_current_cumulative_block_weight_median(); res.info.start_time = (uint64_t)m_core.get_start_time(); + res.info.version = MONERO_VERSION; res.status = Message::STATUS_OK; res.error_details = ""; @@ -785,12 +748,53 @@ namespace rpc res.status = Message::STATUS_OK; } - void DaemonHandler::handle(const GetPerKBFeeEstimate::Request& req, GetPerKBFeeEstimate::Response& res) + void DaemonHandler::handle(const GetFeeEstimate::Request& req, GetFeeEstimate::Response& res) { - res.estimated_fee_per_kb = m_core.get_blockchain_storage().get_dynamic_per_kb_fee_estimate(req.num_grace_blocks); + res.hard_fork_version = m_core.get_blockchain_storage().get_current_hard_fork_version(); + res.estimated_base_fee = m_core.get_blockchain_storage().get_dynamic_base_fee_estimate(req.num_grace_blocks); + + if (res.hard_fork_version < HF_VERSION_PER_BYTE_FEE) + { + res.size_scale = 1024; // per KiB fee + res.fee_mask = 1; + } + else + { + res.size_scale = 1; // per byte fee + res.fee_mask = Blockchain::get_fee_quantization_mask(); + } res.status = Message::STATUS_OK; } + void DaemonHandler::handle(const GetOutputDistribution::Request& req, GetOutputDistribution::Response& res) + { + try + { + res.distributions.reserve(req.amounts.size()); + + const uint64_t req_to_height = req.to_height ? req.to_height : (m_core.get_current_blockchain_height() - 1); + for (std::uint64_t amount : req.amounts) + { + auto data = rpc::RpcHandler::get_output_distribution([this](uint64_t amount, uint64_t from, uint64_t to, uint64_t &start_height, std::vector &distribution, uint64_t &base) { return m_core.get_output_distribution(amount, from, to, start_height, distribution, base); }, amount, req.from_height, req_to_height, [this](uint64_t height) { return m_core.get_blockchain_storage().get_db().get_block_hash_from_height(height); }, req.cumulative, m_core.get_current_blockchain_height()); + if (!data) + { + res.distributions.clear(); + res.status = Message::STATUS_FAILED; + res.error_details = "Failed to get output distribution"; + return; + } + res.distributions.push_back(output_distribution{std::move(*data), amount, req.cumulative}); + } + res.status = Message::STATUS_OK; + } + catch (const std::exception& e) + { + res.distributions.clear(); + res.status = Message::STATUS_FAILED; + res.error_details = e.what(); + } + } + bool DaemonHandler::getBlockHeaderByHash(const crypto::hash& hash_in, cryptonote::rpc::BlockHeaderResponse& header) { block b; @@ -821,7 +825,8 @@ namespace rpc header.reward += out.amount; } - header.difficulty = m_core.get_blockchain_storage().block_difficulty(header.height); + header.wide_difficulty = m_core.get_blockchain_storage().block_difficulty(header.height); + header.difficulty = (header.wide_difficulty & 0xffffffffffffffff).convert_to(); return true; } @@ -847,8 +852,8 @@ namespace rpc REQ_RESP_TYPES_MACRO(request_type, GetTransactions, req_json, resp_message, handle); REQ_RESP_TYPES_MACRO(request_type, KeyImagesSpent, req_json, resp_message, handle); REQ_RESP_TYPES_MACRO(request_type, GetTxGlobalOutputIndices, req_json, resp_message, handle); - REQ_RESP_TYPES_MACRO(request_type, GetRandomOutputsForAmounts, req_json, resp_message, handle); REQ_RESP_TYPES_MACRO(request_type, SendRawTx, req_json, resp_message, handle); + REQ_RESP_TYPES_MACRO(request_type, SendRawTxHex, req_json, resp_message, handle); REQ_RESP_TYPES_MACRO(request_type, GetInfo, req_json, resp_message, handle); REQ_RESP_TYPES_MACRO(request_type, StartMining, req_json, resp_message, handle); REQ_RESP_TYPES_MACRO(request_type, StopMining, req_json, resp_message, handle); @@ -866,7 +871,8 @@ namespace rpc REQ_RESP_TYPES_MACRO(request_type, GetOutputHistogram, req_json, resp_message, handle); REQ_RESP_TYPES_MACRO(request_type, GetOutputKeys, req_json, resp_message, handle); REQ_RESP_TYPES_MACRO(request_type, GetRPCVersion, req_json, resp_message, handle); - REQ_RESP_TYPES_MACRO(request_type, GetPerKBFeeEstimate, req_json, resp_message, handle); + REQ_RESP_TYPES_MACRO(request_type, GetFeeEstimate, req_json, resp_message, handle); + REQ_RESP_TYPES_MACRO(request_type, GetOutputDistribution, req_json, resp_message, handle); // if none of the request types matches if (resp_message == NULL) diff --git a/src/rpc/daemon_handler.h b/src/rpc/daemon_handler.h index f4371164048..34723f676ec 100644 --- a/src/rpc/daemon_handler.h +++ b/src/rpc/daemon_handler.h @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2018, The Monero Project +// Copyright (c) 2017-2019, The Monero Project // // All rights reserved. // @@ -66,10 +66,10 @@ class DaemonHandler : public RpcHandler void handle(const GetTxGlobalOutputIndices::Request& req, GetTxGlobalOutputIndices::Response& res); - void handle(const GetRandomOutputsForAmounts::Request& req, GetRandomOutputsForAmounts::Response& res); - void handle(const SendRawTx::Request& req, SendRawTx::Response& res); + void handle(const SendRawTxHex::Request& req, SendRawTxHex::Response& res); + void handle(const StartMining::Request& req, StartMining::Response& res); void handle(const GetInfo::Request& req, GetInfo::Response& res); @@ -128,7 +128,9 @@ class DaemonHandler : public RpcHandler void handle(const GetRPCVersion::Request& req, GetRPCVersion::Response& res); - void handle(const GetPerKBFeeEstimate::Request& req, GetPerKBFeeEstimate::Response& res); + void handle(const GetFeeEstimate::Request& req, GetFeeEstimate::Response& res); + + void handle(const GetOutputDistribution::Request& req, GetOutputDistribution::Response& res); std::string handle(const std::string& request); @@ -136,6 +138,8 @@ class DaemonHandler : public RpcHandler bool getBlockHeaderByHash(const crypto::hash& hash_in, cryptonote::rpc::BlockHeaderResponse& response); + void handleTxBlob(const std::string& tx_blob, bool relay, SendRawTx::Response& res); + cryptonote::core& m_core; t_p2p& m_p2p; }; diff --git a/src/rpc/daemon_messages.cpp b/src/rpc/daemon_messages.cpp index e5fb9781c44..cf0f5ece174 100644 --- a/src/rpc/daemon_messages.cpp +++ b/src/rpc/daemon_messages.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2018, The Monero Project +// Copyright (c) 2016-2019, The Monero Project // // All rights reserved. // @@ -41,8 +41,8 @@ const char* const GetHashesFast::name = "get_hashes_fast"; const char* const GetTransactions::name = "get_transactions"; const char* const KeyImagesSpent::name = "key_images_spent"; const char* const GetTxGlobalOutputIndices::name = "get_tx_global_output_indices"; -const char* const GetRandomOutputsForAmounts::name = "get_random_outputs_for_amounts"; const char* const SendRawTx::name = "send_raw_tx"; +const char* const SendRawTxHex::name = "send_raw_tx_hex"; const char* const StartMining::name = "start_mining"; const char* const StopMining::name = "stop_mining"; const char* const MiningStatus::name = "mining_status"; @@ -60,7 +60,8 @@ const char* const HardForkInfo::name = "hard_fork_info"; const char* const GetOutputHistogram::name = "get_output_histogram"; const char* const GetOutputKeys::name = "get_output_keys"; const char* const GetRPCVersion::name = "get_rpc_version"; -const char* const GetPerKBFeeEstimate::name = "get_dynamic_per_kb_fee_estimate"; +const char* const GetFeeEstimate::name = "get_dynamic_fee_estimate"; +const char* const GetOutputDistribution::name = "get_output_distribution"; @@ -177,8 +178,6 @@ rapidjson::Value GetTransactions::Request::toJson(rapidjson::Document& doc) cons { auto val = Message::toJson(doc); - auto& al = doc.GetAllocator(); - INSERT_INTO_JSON_OBJECT(val, doc, tx_hashes, tx_hashes); return val; @@ -193,8 +192,6 @@ rapidjson::Value GetTransactions::Response::toJson(rapidjson::Document& doc) con { rapidjson::Value val(rapidjson::kObjectType); - auto& al = doc.GetAllocator(); - INSERT_INTO_JSON_OBJECT(val, doc, txs, txs); INSERT_INTO_JSON_OBJECT(val, doc, missed_hashes, missed_hashes); @@ -212,8 +209,6 @@ rapidjson::Value KeyImagesSpent::Request::toJson(rapidjson::Document& doc) const { auto val = Message::toJson(doc); - auto& al = doc.GetAllocator(); - INSERT_INTO_JSON_OBJECT(val, doc, key_images, key_images); return val; @@ -228,8 +223,6 @@ rapidjson::Value KeyImagesSpent::Response::toJson(rapidjson::Document& doc) cons { auto val = Message::toJson(doc); - auto& al = doc.GetAllocator(); - INSERT_INTO_JSON_OBJECT(val, doc, spent_status, spent_status); return val; @@ -245,8 +238,6 @@ rapidjson::Value GetTxGlobalOutputIndices::Request::toJson(rapidjson::Document& { auto val = Message::toJson(doc); - auto& al = doc.GetAllocator(); - INSERT_INTO_JSON_OBJECT(val, doc, tx_hash, tx_hash); return val; @@ -261,8 +252,6 @@ rapidjson::Value GetTxGlobalOutputIndices::Response::toJson(rapidjson::Document& { auto val = Message::toJson(doc); - auto& al = doc.GetAllocator(); - INSERT_INTO_JSON_OBJECT(val, doc, output_indices, output_indices); return val; @@ -273,83 +262,57 @@ void GetTxGlobalOutputIndices::Response::fromJson(rapidjson::Value& val) GET_FROM_JSON_OBJECT(val, output_indices, output_indices); } - -rapidjson::Value GetRandomOutputsForAmounts::Request::toJson(rapidjson::Document& doc) const +rapidjson::Value SendRawTx::Request::toJson(rapidjson::Document& doc) const { auto val = Message::toJson(doc); - auto& al = doc.GetAllocator(); - - INSERT_INTO_JSON_OBJECT(val, doc, amounts, amounts); - INSERT_INTO_JSON_OBJECT(val, doc, count, count); + INSERT_INTO_JSON_OBJECT(val, doc, tx, tx); + INSERT_INTO_JSON_OBJECT(val, doc, relay, relay); return val; } -void GetRandomOutputsForAmounts::Request::fromJson(rapidjson::Value& val) +void SendRawTx::Request::fromJson(rapidjson::Value& val) { - GET_FROM_JSON_OBJECT(val, amounts, amounts); - GET_FROM_JSON_OBJECT(val, count, count); + GET_FROM_JSON_OBJECT(val, tx, tx); + GET_FROM_JSON_OBJECT(val, relay, relay); } -rapidjson::Value GetRandomOutputsForAmounts::Response::toJson(rapidjson::Document& doc) const +rapidjson::Value SendRawTx::Response::toJson(rapidjson::Document& doc) const { auto val = Message::toJson(doc); - auto& al = doc.GetAllocator(); - - INSERT_INTO_JSON_OBJECT(val, doc, amounts_with_outputs, amounts_with_outputs); + INSERT_INTO_JSON_OBJECT(val, doc, relayed, relayed); return val; } -void GetRandomOutputsForAmounts::Response::fromJson(rapidjson::Value& val) + +void SendRawTx::Response::fromJson(rapidjson::Value& val) { - GET_FROM_JSON_OBJECT(val, amounts_with_outputs, amounts_with_outputs); + GET_FROM_JSON_OBJECT(val, relayed, relayed); } - -rapidjson::Value SendRawTx::Request::toJson(rapidjson::Document& doc) const +rapidjson::Value SendRawTxHex::Request::toJson(rapidjson::Document& doc) const { auto val = Message::toJson(doc); - auto& al = doc.GetAllocator(); - - INSERT_INTO_JSON_OBJECT(val, doc, tx, tx); + INSERT_INTO_JSON_OBJECT(val, doc, tx_as_hex, tx_as_hex); INSERT_INTO_JSON_OBJECT(val, doc, relay, relay); return val; } -void SendRawTx::Request::fromJson(rapidjson::Value& val) +void SendRawTxHex::Request::fromJson(rapidjson::Value& val) { - GET_FROM_JSON_OBJECT(val, tx, tx); + GET_FROM_JSON_OBJECT(val, tx_as_hex, tx_as_hex); GET_FROM_JSON_OBJECT(val, relay, relay); } -rapidjson::Value SendRawTx::Response::toJson(rapidjson::Document& doc) const -{ - auto val = Message::toJson(doc); - - auto& al = doc.GetAllocator(); - - INSERT_INTO_JSON_OBJECT(val, doc, relayed, relayed); - - return val; -} - - -void SendRawTx::Response::fromJson(rapidjson::Value& val) -{ - GET_FROM_JSON_OBJECT(val, relayed, relayed); -} - rapidjson::Value StartMining::Request::toJson(rapidjson::Document& doc) const { auto val = Message::toJson(doc); - auto& al = doc.GetAllocator(); - INSERT_INTO_JSON_OBJECT(val, doc, miner_address, miner_address); INSERT_INTO_JSON_OBJECT(val, doc, threads_count, threads_count); INSERT_INTO_JSON_OBJECT(val, doc, do_background_mining, do_background_mining); @@ -408,8 +371,6 @@ rapidjson::Value MiningStatus::Response::toJson(rapidjson::Document& doc) const { auto val = Message::toJson(doc); - auto& al = doc.GetAllocator(); - INSERT_INTO_JSON_OBJECT(val, doc, active, active); INSERT_INTO_JSON_OBJECT(val, doc, speed, speed); INSERT_INTO_JSON_OBJECT(val, doc, threads_count, threads_count); @@ -442,8 +403,6 @@ rapidjson::Value GetInfo::Response::toJson(rapidjson::Document& doc) const { auto val = Message::toJson(doc); - auto& al = doc.GetAllocator(); - INSERT_INTO_JSON_OBJECT(val, doc, info, info); return val; @@ -459,8 +418,6 @@ rapidjson::Value SaveBC::Request::toJson(rapidjson::Document& doc) const { auto val = Message::toJson(doc); - auto& al = doc.GetAllocator(); - return val; } @@ -472,8 +429,6 @@ rapidjson::Value SaveBC::Response::toJson(rapidjson::Document& doc) const { auto val = Message::toJson(doc); - auto& al = doc.GetAllocator(); - return val; } @@ -486,8 +441,6 @@ rapidjson::Value GetBlockHash::Request::toJson(rapidjson::Document& doc) const { auto val = Message::toJson(doc); - auto& al = doc.GetAllocator(); - INSERT_INTO_JSON_OBJECT(val, doc, height, height); return val; @@ -502,8 +455,6 @@ rapidjson::Value GetBlockHash::Response::toJson(rapidjson::Document& doc) const { auto val = Message::toJson(doc); - auto& al = doc.GetAllocator(); - INSERT_INTO_JSON_OBJECT(val, doc, hash, hash); return val; @@ -519,8 +470,6 @@ rapidjson::Value GetLastBlockHeader::Request::toJson(rapidjson::Document& doc) c { auto val = Message::toJson(doc); - auto& al = doc.GetAllocator(); - return val; } @@ -532,8 +481,6 @@ rapidjson::Value GetLastBlockHeader::Response::toJson(rapidjson::Document& doc) { auto val = Message::toJson(doc); - auto& al = doc.GetAllocator(); - INSERT_INTO_JSON_OBJECT(val, doc, header, header); return val; @@ -549,8 +496,6 @@ rapidjson::Value GetBlockHeaderByHash::Request::toJson(rapidjson::Document& doc) { auto val = Message::toJson(doc); - auto& al = doc.GetAllocator(); - INSERT_INTO_JSON_OBJECT(val, doc, hash, hash); return val; @@ -565,8 +510,6 @@ rapidjson::Value GetBlockHeaderByHash::Response::toJson(rapidjson::Document& doc { auto val = Message::toJson(doc); - auto& al = doc.GetAllocator(); - INSERT_INTO_JSON_OBJECT(val, doc, header, header); return val; @@ -582,8 +525,6 @@ rapidjson::Value GetBlockHeaderByHeight::Request::toJson(rapidjson::Document& do { auto val = Message::toJson(doc); - auto& al = doc.GetAllocator(); - INSERT_INTO_JSON_OBJECT(val, doc, height, height); return val; @@ -598,8 +539,6 @@ rapidjson::Value GetBlockHeaderByHeight::Response::toJson(rapidjson::Document& d { auto val = Message::toJson(doc); - auto& al = doc.GetAllocator(); - INSERT_INTO_JSON_OBJECT(val, doc, header, header); return val; @@ -615,8 +554,6 @@ rapidjson::Value GetBlockHeadersByHeight::Request::toJson(rapidjson::Document& d { auto val = Message::toJson(doc); - auto& al = doc.GetAllocator(); - INSERT_INTO_JSON_OBJECT(val, doc, heights, heights); return val; @@ -631,8 +568,6 @@ rapidjson::Value GetBlockHeadersByHeight::Response::toJson(rapidjson::Document& { auto val = Message::toJson(doc); - auto& al = doc.GetAllocator(); - INSERT_INTO_JSON_OBJECT(val, doc, headers, headers); return val; @@ -648,8 +583,6 @@ rapidjson::Value GetPeerList::Request::toJson(rapidjson::Document& doc) const { auto val = Message::toJson(doc); - auto& al = doc.GetAllocator(); - return val; } @@ -661,8 +594,6 @@ rapidjson::Value GetPeerList::Response::toJson(rapidjson::Document& doc) const { auto val = Message::toJson(doc); - auto& al = doc.GetAllocator(); - INSERT_INTO_JSON_OBJECT(val, doc, white_list, white_list); INSERT_INTO_JSON_OBJECT(val, doc, gray_list, gray_list); @@ -715,8 +646,6 @@ rapidjson::Value GetTransactionPool::Response::toJson(rapidjson::Document& doc) { auto val = Message::toJson(doc); - auto& al = doc.GetAllocator(); - INSERT_INTO_JSON_OBJECT(val, doc, transactions, transactions); INSERT_INTO_JSON_OBJECT(val, doc, key_images, key_images); @@ -734,8 +663,6 @@ rapidjson::Value HardForkInfo::Request::toJson(rapidjson::Document& doc) const { auto val = Message::toJson(doc); - auto& al = doc.GetAllocator(); - INSERT_INTO_JSON_OBJECT(val, doc, version, version); return val; @@ -750,8 +677,6 @@ rapidjson::Value HardForkInfo::Response::toJson(rapidjson::Document& doc) const { auto val = Message::toJson(doc); - auto& al = doc.GetAllocator(); - INSERT_INTO_JSON_OBJECT(val, doc, info, info); return val; @@ -767,8 +692,6 @@ rapidjson::Value GetOutputHistogram::Request::toJson(rapidjson::Document& doc) c { auto val = Message::toJson(doc); - auto& al = doc.GetAllocator(); - INSERT_INTO_JSON_OBJECT(val, doc, amounts, amounts); INSERT_INTO_JSON_OBJECT(val, doc, min_count, min_count); INSERT_INTO_JSON_OBJECT(val, doc, max_count, max_count); @@ -791,8 +714,6 @@ rapidjson::Value GetOutputHistogram::Response::toJson(rapidjson::Document& doc) { auto val = Message::toJson(doc); - auto& al = doc.GetAllocator(); - INSERT_INTO_JSON_OBJECT(val, doc, histogram, histogram); return val; @@ -808,8 +729,6 @@ rapidjson::Value GetOutputKeys::Request::toJson(rapidjson::Document& doc) const { auto val = Message::toJson(doc); - auto& al = doc.GetAllocator(); - INSERT_INTO_JSON_OBJECT(val, doc, outputs, outputs); return val; @@ -824,8 +743,6 @@ rapidjson::Value GetOutputKeys::Response::toJson(rapidjson::Document& doc) const { auto val = Message::toJson(doc); - auto& al = doc.GetAllocator(); - INSERT_INTO_JSON_OBJECT(val, doc, keys, keys); return val; @@ -850,8 +767,6 @@ rapidjson::Value GetRPCVersion::Response::toJson(rapidjson::Document& doc) const { auto val = Message::toJson(doc); - auto& al = doc.GetAllocator(); - INSERT_INTO_JSON_OBJECT(val, doc, version, version); return val; @@ -862,38 +777,75 @@ void GetRPCVersion::Response::fromJson(rapidjson::Value& val) GET_FROM_JSON_OBJECT(val, version, version); } -rapidjson::Value GetPerKBFeeEstimate::Request::toJson(rapidjson::Document& doc) const +rapidjson::Value GetFeeEstimate::Request::toJson(rapidjson::Document& doc) const { auto val = Message::toJson(doc); - auto& al = doc.GetAllocator(); - INSERT_INTO_JSON_OBJECT(val, doc, num_grace_blocks, num_grace_blocks); return val; } -void GetPerKBFeeEstimate::Request::fromJson(rapidjson::Value& val) +void GetFeeEstimate::Request::fromJson(rapidjson::Value& val) { GET_FROM_JSON_OBJECT(val, num_grace_blocks, num_grace_blocks); } -rapidjson::Value GetPerKBFeeEstimate::Response::toJson(rapidjson::Document& doc) const +rapidjson::Value GetFeeEstimate::Response::toJson(rapidjson::Document& doc) const { auto val = Message::toJson(doc); - auto& al = doc.GetAllocator(); + INSERT_INTO_JSON_OBJECT(val, doc, estimated_base_fee, estimated_base_fee); + INSERT_INTO_JSON_OBJECT(val, doc, fee_mask, fee_mask); + INSERT_INTO_JSON_OBJECT(val, doc, size_scale, size_scale); + INSERT_INTO_JSON_OBJECT(val, doc, hard_fork_version, hard_fork_version); + + return val; +} - INSERT_INTO_JSON_OBJECT(val, doc, estimated_fee_per_kb, estimated_fee_per_kb); +void GetFeeEstimate::Response::fromJson(rapidjson::Value& val) +{ + GET_FROM_JSON_OBJECT(val, estimated_base_fee, estimated_base_fee); + GET_FROM_JSON_OBJECT(val, fee_mask, fee_mask); + GET_FROM_JSON_OBJECT(val, size_scale, size_scale); + GET_FROM_JSON_OBJECT(val, hard_fork_version, hard_fork_version); +} + +rapidjson::Value GetOutputDistribution::Request::toJson(rapidjson::Document& doc) const +{ + auto val = Message::toJson(doc); + + INSERT_INTO_JSON_OBJECT(val, doc, amounts, amounts); + INSERT_INTO_JSON_OBJECT(val, doc, from_height, from_height); + INSERT_INTO_JSON_OBJECT(val, doc, to_height, to_height); + INSERT_INTO_JSON_OBJECT(val, doc, cumulative, cumulative); return val; } -void GetPerKBFeeEstimate::Response::fromJson(rapidjson::Value& val) +void GetOutputDistribution::Request::fromJson(rapidjson::Value& val) { - GET_FROM_JSON_OBJECT(val, estimated_fee_per_kb, estimated_fee_per_kb); + GET_FROM_JSON_OBJECT(val, amounts, amounts); + GET_FROM_JSON_OBJECT(val, from_height, from_height); + GET_FROM_JSON_OBJECT(val, to_height, to_height); + GET_FROM_JSON_OBJECT(val, cumulative, cumulative); } +rapidjson::Value GetOutputDistribution::Response::toJson(rapidjson::Document& doc) const +{ + auto val = Message::toJson(doc); + + INSERT_INTO_JSON_OBJECT(val, doc, status, status); + INSERT_INTO_JSON_OBJECT(val, doc, distributions, distributions); + + return val; +} + +void GetOutputDistribution::Response::fromJson(rapidjson::Value& val) +{ + GET_FROM_JSON_OBJECT(val, status, status); + GET_FROM_JSON_OBJECT(val, distributions, distributions); +} } // namespace rpc diff --git a/src/rpc/daemon_messages.h b/src/rpc/daemon_messages.h index 1495c845fc7..c0d9aed0ae0 100644 --- a/src/rpc/daemon_messages.h +++ b/src/rpc/daemon_messages.h @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2018, The Monero Project +// Copyright (c) 2016-2019, The Monero Project // // All rights reserved. // @@ -28,6 +28,9 @@ #pragma once +#include +#include + #include "message.h" #include "cryptonote_protocol/cryptonote_protocol_defs.h" #include "rpc/message_data_structs.h" @@ -62,12 +65,10 @@ class classname \ #define END_RPC_MESSAGE_RESPONSE }; #define END_RPC_MESSAGE_CLASS }; -#define COMMA() , - // NOTE: when using a type with multiple template parameters, // replace any comma in the template specifier with the macro // above, or the preprocessor will eat the comma in a bad way. -#define RPC_MESSAGE_MEMBER(type, name) type name; +#define RPC_MESSAGE_MEMBER(type, name) type name = {} namespace cryptonote @@ -106,7 +107,7 @@ BEGIN_RPC_MESSAGE_CLASS(GetHashesFast); RPC_MESSAGE_MEMBER(uint64_t, start_height); END_RPC_MESSAGE_REQUEST; BEGIN_RPC_MESSAGE_RESPONSE; - RPC_MESSAGE_MEMBER(std::list, hashes); + RPC_MESSAGE_MEMBER(std::vector, hashes); RPC_MESSAGE_MEMBER(uint64_t, start_height); RPC_MESSAGE_MEMBER(uint64_t, current_height); END_RPC_MESSAGE_RESPONSE; @@ -118,7 +119,8 @@ BEGIN_RPC_MESSAGE_CLASS(GetTransactions); RPC_MESSAGE_MEMBER(std::vector, tx_hashes); END_RPC_MESSAGE_REQUEST; BEGIN_RPC_MESSAGE_RESPONSE; - RPC_MESSAGE_MEMBER(std::unordered_map, txs); + using txes_map = std::unordered_map; + RPC_MESSAGE_MEMBER(txes_map, txs); RPC_MESSAGE_MEMBER(std::vector, missed_hashes); END_RPC_MESSAGE_RESPONSE; END_RPC_MESSAGE_CLASS; @@ -169,6 +171,14 @@ BEGIN_RPC_MESSAGE_CLASS(SendRawTx); END_RPC_MESSAGE_RESPONSE; END_RPC_MESSAGE_CLASS; +BEGIN_RPC_MESSAGE_CLASS(SendRawTxHex); + BEGIN_RPC_MESSAGE_REQUEST; + RPC_MESSAGE_MEMBER(std::string, tx_as_hex); + RPC_MESSAGE_MEMBER(bool, relay); + END_RPC_MESSAGE_REQUEST; + using Response = SendRawTx::Response; +END_RPC_MESSAGE_CLASS; + BEGIN_RPC_MESSAGE_CLASS(StartMining); BEGIN_RPC_MESSAGE_REQUEST; RPC_MESSAGE_MEMBER(std::string, miner_address); @@ -407,12 +417,27 @@ BEGIN_RPC_MESSAGE_CLASS(GetRPCVersion); END_RPC_MESSAGE_RESPONSE; END_RPC_MESSAGE_CLASS; -BEGIN_RPC_MESSAGE_CLASS(GetPerKBFeeEstimate); +BEGIN_RPC_MESSAGE_CLASS(GetFeeEstimate); BEGIN_RPC_MESSAGE_REQUEST; RPC_MESSAGE_MEMBER(uint64_t, num_grace_blocks); END_RPC_MESSAGE_REQUEST; BEGIN_RPC_MESSAGE_RESPONSE; - RPC_MESSAGE_MEMBER(uint64_t, estimated_fee_per_kb); + RPC_MESSAGE_MEMBER(uint64_t, estimated_base_fee); + RPC_MESSAGE_MEMBER(uint64_t, fee_mask); + RPC_MESSAGE_MEMBER(uint32_t, size_scale); + RPC_MESSAGE_MEMBER(uint8_t, hard_fork_version); + END_RPC_MESSAGE_RESPONSE; +END_RPC_MESSAGE_CLASS; + +BEGIN_RPC_MESSAGE_CLASS(GetOutputDistribution); + BEGIN_RPC_MESSAGE_REQUEST; + RPC_MESSAGE_MEMBER(std::vector, amounts); + RPC_MESSAGE_MEMBER(uint64_t, from_height); + RPC_MESSAGE_MEMBER(uint64_t, to_height); + RPC_MESSAGE_MEMBER(bool, cumulative); + END_RPC_MESSAGE_REQUEST; + BEGIN_RPC_MESSAGE_RESPONSE; + RPC_MESSAGE_MEMBER(std::vector, distributions); END_RPC_MESSAGE_RESPONSE; END_RPC_MESSAGE_CLASS; diff --git a/src/rpc/daemon_rpc_version.h b/src/rpc/daemon_rpc_version.h index e20af5b21b2..b3eb9699bb5 100644 --- a/src/rpc/daemon_rpc_version.h +++ b/src/rpc/daemon_rpc_version.h @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2018, The Monero Project +// Copyright (c) 2016-2019, The Monero Project // // All rights reserved. // diff --git a/src/rpc/get_output_distribution_cache.h b/src/rpc/get_output_distribution_cache.h deleted file mode 100644 index 6495e7d4cea..00000000000 --- a/src/rpc/get_output_distribution_cache.h +++ /dev/null @@ -1,113 +0,0 @@ -static const struct -{ - uint64_t amount; - uint64_t from_height; - uint64_t to_height; - uint64_t start_height; - uint64_t base; - std::vector distribution; -} -get_output_distribution_cache[] = -{ - { - 0, - 1544704, - 1546001, - 1544704, - 5143500, - { - 5, 38, 37, 33, 39, 7, 1, 1, 5, 9, 7, 5, 17, 5, 3, 9, 3, 17, 5, 17, 1, 1, 15, 13, 3, 10, 5, 3, 34, 1, 45, 7, - 5, 17, 5, 22, 3, 1, 17, 16, 5, 1, 3, 43, 5, 13, 3, 23, 9, 7, 9, 13, 1, 11, 1, 17, 1, 3, 16, 11, 5, 11, 7, 7, - 33, 11, 7, 1, 5, 1, 21, 19, 1, 17, 1, 49, 17, 3, 3, 9, 35, 46, 46, 39, 26, 33, 21, 3, 23, 3, 9, 37, 1, 33, 11, 32, - 1, 13, 16, 12, 3, 21, 1, 18, 3, 19, 1, 25, 5, 3, 18, 7, 17, 5, 9, 15, 7, 7, 11, 9, 9, 17, 5, 16, 1, 3, 13, 3, - 5, 5, 5, 13, 5, 9, 5, 13, 3, 17, 15, 36, 13, 3, 20, 12, 6, 23, 17, 10, 22, 23, 1, 7, 21, 6, 23, 1, 3, 19, 13, 1, - 3, 43, 35, 13, 1, 31, 7, 3, 17, 1, 15, 5, 11, 15, 24, 1, 18, 13, 5, 15, 1, 29, 3, 3, 13, 3, 15, 7, 17, 3, 1, 1, - 17, 1, 1, 45, 39, 27, 45, 46, 34, 7, 3, 3, 9, 3, 3, 11, 7, 5, 9, 25, 19, 3, 33, 1, 5, 17, 1, 45, 4, 1, 45, 11, - 44, 32, 3, 1, 3, 7, 17, 15, 5, 45, 35, 41, 1, 35, 3, 3, 19, 1, 9, 17, 29, 29, 3, 1, 13, 1, 3, 47, 21, 13, 7, 1, - 7, 5, 1, 11, 1, 40, 9, 7, 3, 3, 13, 25, 1, 47, 5, 7, 3, 7, 31, 40, 34, 6, 3, 15, 3, 31, 5, 13, 27, 9, 12, 21, - 3, 1, 19, 1, 19, 5, 47, 49, 47, 42, 50, 34, 29, 23, 1, 5, 9, 16, 11, 7, 1, 19, 7, 5, 1, 15, 1, 1, 9, 13, 9, 5, - 27, 3, 3, 29, 1, 33, 3, 9, 5, 35, 5, 1, 17, 7, 3, 39, 3, 28, 19, 1, 1, 9, 1, 3, 27, 1, 37, 3, 1, 1, 16, 3, - 25, 11, 5, 3, 33, 45, 17, 11, 7, 22, 9, 1, 5, 5, 5, 15, 1, 15, 9, 7, 11, 13, 37, 49, 46, 38, 11, 1, 25, 1, 13, 18, - 3, 7, 39, 3, 37, 19, 35, 3, 1, 3, 19, 1, 3, 15, 21, 3, 27, 1, 45, 48, 1, 13, 29, 9, 1, 1, 46, 43, 5, 15, 3, 7, - 29, 26, 5, 5, 21, 37, 17, 21, 3, 13, 1, 5, 1, 17, 5, 31, 13, 1, 11, 3, 46, 9, 3, 7, 1, 1, 41, 1, 21, 1, 5, 12, - 7, 13, 9, 25, 1, 47, 47, 48, 48, 48, 48, 48, 47, 48, 45, 45, 33, 52, 50, 46, 45, 47, 35, 41, 38, 35, 42, 38, 34, 41, 39, 35, - 51, 51, 45, 43, 49, 52, 53, 45, 42, 46, 37, 53, 49, 41, 46, 49, 46, 47, 48, 37, 41, 33, 43, 38, 15, 3, 3, 27, 11, 5, 23, 13, - 1, 1, 37, 3, 15, 3, 30, 13, 3, 45, 12, 3, 5, 11, 1, 1, 21, 9, 11, 19, 1, 1, 1, 25, 5, 21, 3, 1, 32, 44, 3, 33, - 11, 7, 5, 23, 1, 37, 47, 48, 48, 48, 48, 48, 48, 46, 47, 47, 50, 45, 49, 50, 46, 47, 49, 45, 51, 49, 50, 49, 49, 46, 47, 48, - 46, 48, 46, 50, 46, 43, 46, 46, 48, 47, 46, 47, 45, 49, 46, 43, 50, 45, 45, 49, 45, 48, 45, 49, 48, 45, 45, 51, 45, 51, 45, 46, - 52, 45, 45, 51, 51, 52, 44, 45, 52, 50, 50, 46, 47, 51, 51, 46, 47, 47, 47, 50, 47, 51, 48, 49, 51, 50, 48, 48, 48, 50, 49, 49, - 52, 52, 49, 50, 49, 49, 49, 51, 52, 49, 52, 50, 49, 47, 29, 15, 39, 17, 31, 5, 40, 5, 18, 23, 25, 7, 35, 26, 5, 31, 49, 22, - 3, 17, 7, 49, 7, 49, 47, 12, 44, 46, 36, 15, 3, 1, 47, 13, 35, 40, 5, 21, 19, 39, 21, 33, 31, 29, 1, 1, 37, 1, 15, 47, - 7, 7, 47, 41, 13, 3, 47, 31, 9, 33, 13, 43, 29, 5, 1, 9, 33, 7, 27, 15, 15, 25, 5, 43, 22, 31, 7, 1, 47, 1, 15, 27, - 3, 27, 45, 15, 1, 36, 17, 1, 23, 39, 38, 45, 7, 7, 19, 7, 11, 47, 33, 16, 3, 45, 45, 45, 9, 27, 3, 3, 21, 3, 7, 21, - 7, 3, 43, 1, 17, 1, 45, 37, 46, 5, 5, 13, 46, 40, 48, 48, 45, 34, 1, 46, 19, 25, 9, 7, 47, 23, 37, 31, 3, 25, 13, 46, - 31, 25, 5, 46, 35, 52, 11, 23, 27, 4, 15, 11, 11, 11, 9, 34, 7, 9, 15, 34, 9, 27, 37, 28, 25, 45, 13, 30, 5, 25, 15, 7, - 3, 19, 27, 1, 7, 11, 1, 32, 3, 45, 11, 9, 21, 25, 9, 13, 13, 1, 7, 1, 33, 11, 5, 3, 3, 27, 27, 5, 3, 37, 17, 17, - 3, 7, 5, 13, 1, 3, 44, 45, 26, 25, 1, 13, 3, 13, 3, 11, 1, 11, 7, 45, 3, 3, 1, 43, 1, 19, 3, 1, 15, 5, 39, 7, - 7, 1, 9, 1, 11, 19, 3, 35, 29, 7, 15, 11, 40, 7, 44, 38, 34, 7, 9, 7, 1, 27, 1, 9, 5, 45, 1, 21, 3, 1, 5, 9, - 3, 21, 23, 33, 3, 1, 7, 3, 3, 7, 41, 9, 7, 1, 5, 31, 9, 7, 1, 1, 11, 41, 51, 20, 9, 47, 39, 17, 9, 35, 1, 41, - 1, 19, 1, 19, 15, 1, 13, 5, 23, 15, 9, 15, 17, 1, 15, 27, 33, 31, 29, 7, 13, 1, 5, 45, 5, 1, 1, 11, 1, 13, 3, 7, - 9, 1, 13, 39, 3, 33, 5, 3, 7, 7, 5, 29, 11, 1, 7, 1, 15, 3, 13, 3, 15, 3, 3, 1, 5, 1, 9, 1, 44, 49, 24, 25, - 1, 1, 34, 22, 7, 5, 5, 5, 10, 9, 13, 3, 9, 1, 9, 19, 7, 43, 48, 7, 11, 7, 3, 3, 7, 21, 1, 1, 3, 3, 11, 31, - 1, 1, 13, 22, 23, 7, 27, 9, 3, 3, 21, 1, 35, 21, 9, 11, 13, 39, 1, 3, 7, 23, 3, 28, 3, 45, 47, 38, 32, 37, 34, 1, - 23, 3, 3, 1, 19, 19, 1, 5, 13, 1, 5, 11, 38, 3, 1, 36, 13, 1, 1, 23, 5, 17, 11, 1, 13, 1, 3, 7, 11, 3, 33, 7, - 19, 5, 5, 1, 1, 3, 5, 41, 1, 3, 25, 1, 7, 7, 9, 3, 11, 3, 13, 5, 7, 1, 3, 9, 1, 1, 43, 47, 47, 47, 17, 7, - 17, 3, 19, 1, 9, 9, 33, 22, 1, 25, 1, 3, 3, 32, 5, 1, 23, 9, 5, 1, 31, 5, 9, 1, 3, 7, 19, 1, 12, 11, 5, 1, - 1, 9, 25, 15, 15, 13, 5, 3, 15, 1, 17, 1, 1, 5, 5, 41, 11, 15, 7, 3, 21, 21, 35, 22, 46, 35, 3, 27, 5, 3, 45, 22, - 27, 1, 19, 9, 1, 25, 1, 29, 3, 5, 25, 17, 27, 5, 19, 5, 25, 7, 19, 1, 9, 21, 3, 7, 29, 27, 17, 3, 3, 15, 7, 19, - 5, 25, 1, 23, 45, 4, 31, 1, 37, 14, 29, 3, 29, 1, 23, 29, 19, 11, 1, 13, 13, 9, 1, 25, 1, 33, 1, 37, 37, 23, 7, 21, - 7, 3, 13, 7, 3, 7, 21, 11, 9, 1, 31, 3, 1, 7, 39, 46, 3, 30, - }, - }, - { - 0, - 1562704, - 1564001, - 1562704, - 5521986, - { - 35, 45, 23, 3, 44, 47, 44, 3, 17, 35, 7, 11, 7, 29, 43, 27, 11, 7, 5, 31, 13, 9, 45, 45, 7, 42, 17, 15, 19, 11, 45, 19, - 45, 46, 45, 46, 32, 34, 43, 34, 46, 47, 45, 30, 17, 45, 46, 36, 35, 38, 19, 9, 23, 17, 3, 19, 31, 41, 35, 24, 15, 45, 15, 5, - 11, 5, 19, 11, 11, 7, 15, 19, 45, 34, 7, 7, 29, 1, 9, 36, 7, 44, 45, 33, 25, 8, 17, 7, 44, 43, 48, 45, 42, 46, 40, 44, - 1, 43, 45, 46, 46, 35, 19, 19, 23, 5, 13, 19, 7, 16, 9, 3, 25, 34, 3, 27, 9, 39, 3, 43, 21, 1, 45, 45, 39, 25, 23, 13, - 39, 39, 3, 45, 43, 46, 44, 40, 39, 33, 45, 47, 38, 45, 45, 39, 47, 47, 45, 46, 35, 45, 43, 47, 45, 40, 34, 42, 42, 48, 49, 47, - 47, 48, 47, 45, 43, 48, 37, 48, 41, 45, 48, 34, 42, 44, 9, 19, 27, 1, 47, 47, 43, 25, 29, 5, 5, 21, 39, 35, 43, 37, 13, 45, - 25, 31, 26, 47, 45, 23, 23, 39, 32, 25, 44, 47, 35, 47, 15, 17, 7, 9, 5, 35, 31, 3, 45, 47, 46, 13, 17, 48, 45, 9, 13, 45, - 45, 31, 1, 53, 44, 46, 39, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 47, 47, 47, 47, 47, 47, 47, 47, 53, 49, 45, 45, 50, - 50, 27, 43, 46, 47, 46, 45, 48, 36, 42, 42, 46, 45, 47, 41, 45, 43, 47, 38, 48, 47, 33, 11, 45, 46, 34, 42, 32, 3, 45, 37, 45, - 15, 3, 45, 29, 31, 9, 5, 3, 27, 5, 21, 25, 7, 15, 46, 34, 5, 3, 17, 3, 9, 13, 7, 11, 3, 1, 34, 13, 7, 45, 33, 26, - 9, 5, 9, 41, 43, 34, 3, 35, 3, 17, 37, 11, 17, 3, 15, 27, 15, 45, 46, 13, 26, 16, 11, 7, 5, 45, 38, 45, 45, 22, 44, 44, - 43, 6, 11, 35, 15, 44, 17, 27, 13, 3, 40, 5, 9, 7, 35, 19, 5, 5, 1, 28, 33, 15, 45, 5, 29, 3, 31, 12, 5, 32, 24, 3, - 23, 13, 47, 45, 42, 46, 39, 21, 21, 1, 44, 44, 47, 41, 5, 1, 11, 36, 20, 5, 45, 39, 45, 45, 44, 45, 32, 22, 40, 11, 38, 1, - 45, 46, 37, 9, 23, 9, 15, 44, 7, 16, 38, 46, 11, 14, 24, 19, 19, 7, 26, 25, 45, 37, 17, 1, 35, 1, 3, 28, 3, 11, 22, 13, - 3, 1, 7, 38, 5, 3, 1, 26, 1, 3, 43, 44, 46, 45, 21, 11, 1, 44, 27, 1, 11, 26, 10, 44, 45, 45, 45, 47, 47, 45, 48, 45, - 38, 9, 5, 44, 46, 27, 3, 12, 1, 11, 3, 44, 43, 1, 5, 2, 46, 17, 13, 19, 1, 12, 7, 23, 1, 17, 6, 13, 3, 5, 27, 7, - 46, 36, 19, 25, 1, 1, 3, 8, 15, 3, 45, 45, 45, 37, 6, 15, 3, 5, 38, 14, 41, 1, 13, 45, 45, 39, 44, 29, 43, 48, 51, 50, - 37, 5, 17, 46, 47, 31, 5, 42, 49, 38, 39, 24, 7, 11, 44, 35, 21, 6, 15, 5, 47, 13, 28, 45, 34, 27, 24, 15, 35, 13, 7, 25, - 43, 13, 14, 5, 3, 5, 46, 45, 45, 35, 5, 21, 28, 3, 13, 4, 30, 19, 45, 45, 40, 37, 5, 40, 17, 9, 3, 9, 13, 4, 17, 33, - 44, 39, 17, 45, 28, 5, 11, 45, 19, 45, 21, 44, 31, 43, 49, 48, 15, 3, 1, 44, 45, 43, 11, 1, 1, 27, 43, 45, 39, 3, 1, 3, - 5, 31, 1, 43, 23, 19, 7, 5, 45, 31, 11, 7, 3, 9, 5, 7, 13, 43, 47, 45, 46, 47, 14, 3, 25, 45, 7, 17, 32, 21, 3, 17, - 5, 11, 31, 40, 45, 20, 45, 45, 32, 38, 47, 38, 45, 41, 49, 30, 45, 5, 36, 31, 22, 3, 46, 45, 13, 21, 23, 5, 46, 45, 33, 19, - 25, 1, 7, 13, 5, 44, 23, 29, 23, 24, 7, 5, 37, 13, 29, 13, 7, 17, 7, 43, 3, 21, 7, 44, 1, 19, 15, 9, 34, 43, 26, 3, - 17, 5, 6, 5, 7, 3, 33, 35, 43, 41, 48, 47, 30, 45, 19, 7, 5, 33, 11, 34, 25, 1, 21, 11, 29, 7, 1, 4, 5, 10, 14, 3, - 44, 11, 47, 45, 33, 3, 9, 7, 40, 23, 9, 1, 3, 1, 7, 5, 9, 9, 6, 11, 45, 41, 45, 19, 5, 11, 10, 39, 9, 19, 3, 11, - 43, 42, 1, 13, 35, 5, 32, 7, 5, 5, 43, 37, 3, 32, 17, 3, 23, 1, 13, 45, 17, 1, 21, 45, 43, 46, 49, 47, 45, 30, 9, 31, - 19, 42, 19, 44, 17, 14, 19, 25, 1, 7, 5, 45, 19, 13, 7, 3, 1, 1, 9, 21, 37, 9, 11, 1, 3, 37, 27, 13, 5, 21, 33, 3, - 27, 9, 27, 1, 39, 1, 46, 21, 10, 13, 21, 40, 22, 35, 41, 9, 33, 3, 17, 8, 45, 46, 42, 46, 47, 47, 47, 48, 48, 47, 43, 48, - 37, 39, 50, 35, 3, 40, 45, 40, 46, 36, 34, 28, 9, 9, 37, 9, 5, 7, 13, 31, 1, 7, 3, 3, 44, 45, 25, 15, 1, 21, 43, 25, - 1, 38, 34, 42, 31, 23, 33, 35, 37, 20, 7, 15, 3, 7, 7, 27, 45, 45, 48, 47, 45, 44, 47, 23, 25, 36, 11, 3, 18, 24, 27, 13, - 41, 13, 5, 5, 7, 19, 15, 7, 5, 14, 45, 45, 37, 1, 5, 17, 21, 41, 17, 37, 53, 41, 45, 45, 45, 45, 45, 45, 45, 45, 43, 47, - 47, 48, 53, 47, 47, 47, 49, 27, 45, 47, 47, 47, 47, 45, 45, 45, 47, 43, 48, 34, 34, 43, 46, 15, 37, 21, 5, 27, 11, 1, 9, 7, - 19, 15, 1, 1, 19, 3, 36, 27, 29, 13, 21, 9, 17, 5, 16, 45, 23, 34, 3, 1, 7, 25, 28, 13, 29, 15, 11, 19, 17, 1, 27, 23, - 31, 19, 41, 41, 40, 47, 28, 31, 26, 26, 36, 17, 5, 1, 23, 1, 45, 34, 49, 51, 34, 43, 37, 5, 41, 15, 5, 21, 1, 7, 9, 19, - 5, 11, 39, 19, 45, 45, 38, 17, 9, 1, 15, 11, 5, 13, 47, 46, 48, 45, 19, 32, 7, 19, 5, 7, 23, 29, 5, 45, 41, 37, 1, 5, - 27, 5, 5, 7, 19, 9, 1, 35, 48, 38, 38, 39, 42, 43, 21, 23, 43, 41, 7, 3, 7, 13, 1, 46, 47, 46, 23, 46, 45, 25, 7, 9, - 21, 7, 41, 13, 20, 1, 21, 15, 37, 5, 40, 45, 45, 5, 45, 46, 15, 33, 46, 12, 13, 7, 24, 7, 5, 30, 7, 46, 13, 8, 44, 35, - 45, 33, 40, 36, 47, 47, 29, 43, 36, 43, 45, 42, 36, 19, 7, 7, 43, 3, 44, 25, 48, 29, 11, 45, 30, 1, 17, 13, 25, 1, 48, 45, - 45, 45, 44, 49, 37, 9, 21, 17, 15, 7, 15, 25, 1, 1, 9, 43, 33, 11, 3, 29, 45, 45, 9, 7, 7, 27, 47, 45, 47, 48, 45, 47, - 26, 1, 43, 15, 45, 17, 1, 5, 35, 31, 9, 3, 9, 19, 9, 21, 43, 5, 27, 1, 5, 9, 4, 34, 19, 3, 7, 11, 45, 46, 45, 45, - 46, 47, 47, 44, 45, 43, 27, 9, 17, 15, 19, 44, 45, 46, 47, 47, 45, 45, - } - } -}; - diff --git a/src/rpc/instanciations.cpp b/src/rpc/instanciations.cpp index ec888298268..94fdb5f18c0 100644 --- a/src/rpc/instanciations.cpp +++ b/src/rpc/instanciations.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2018, The Monero Project +// Copyright (c) 2017-2019, The Monero Project // // All rights reserved. // diff --git a/src/rpc/message.cpp b/src/rpc/message.cpp index 0ebe34efedb..158b58005bc 100644 --- a/src/rpc/message.cpp +++ b/src/rpc/message.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2018, The Monero Project +// Copyright (c) 2016-2019, The Monero Project // // All rights reserved. // diff --git a/src/rpc/message.h b/src/rpc/message.h index 16b8e92fc9a..2b7b61ab3f1 100644 --- a/src/rpc/message.h +++ b/src/rpc/message.h @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2018, The Monero Project +// Copyright (c) 2016-2019, The Monero Project // // All rights reserved. // @@ -65,7 +65,7 @@ namespace rpc static const char* STATUS_BAD_REQUEST; static const char* STATUS_BAD_JSON; - Message() : status(STATUS_OK) { } + Message() : status(STATUS_OK), rpc_version(0) { } virtual ~Message() { } diff --git a/src/rpc/message_data_structs.h b/src/rpc/message_data_structs.h index 17ae9629fba..2a43811cf06 100644 --- a/src/rpc/message_data_structs.h +++ b/src/rpc/message_data_structs.h @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2018, The Monero Project +// Copyright (c) 2016-2019, The Monero Project // // All rights reserved. // @@ -30,7 +30,9 @@ #include "crypto/hash.h" #include "cryptonote_basic/cryptonote_basic.h" +#include "cryptonote_basic/difficulty.h" #include "ringct/rctSigs.h" +#include "rpc/rpc_handler.h" #include #include @@ -44,7 +46,7 @@ namespace rpc struct block_with_transactions { cryptonote::block block; - std::unordered_map transactions; + std::vector transactions; }; typedef std::vector tx_output_indices; @@ -77,7 +79,9 @@ namespace rpc uint64_t id; uint32_t ip; uint16_t port; + uint16_t rpc_port; uint64_t last_seen; + uint32_t pruning_seed; }; struct tx_in_pool @@ -85,6 +89,7 @@ namespace rpc cryptonote::transaction tx; crypto::hash tx_hash; uint64_t blob_size; + uint64_t weight; uint64_t fee; crypto::hash max_used_block_hash; uint64_t max_used_block_height; @@ -161,6 +166,7 @@ namespace rpc uint64_t height; uint64_t depth; crypto::hash hash; + cryptonote::difficulty_type wide_difficulty; uint64_t difficulty; uint64_t reward; }; @@ -169,6 +175,7 @@ namespace rpc { uint64_t height; uint64_t target_height; + cryptonote::difficulty_type wide_difficulty; uint64_t difficulty; uint64_t target; uint64_t tx_count; @@ -181,12 +188,24 @@ namespace rpc bool mainnet; bool testnet; bool stagenet; + std::string nettype; crypto::hash top_block_hash; + cryptonote::difficulty_type wide_cumulative_difficulty; uint64_t cumulative_difficulty; uint64_t block_size_limit; + uint64_t block_weight_limit; + uint64_t block_size_median; + uint64_t block_weight_median; uint64_t start_time; + std::string version; }; + struct output_distribution + { + output_distribution_data data; + uint64_t amount; + bool cumulative; + }; } // namespace rpc } // namespace cryptonote diff --git a/src/rpc/rpc_args.cpp b/src/rpc/rpc_args.cpp index d4a6138ba14..4479bd1f1a0 100644 --- a/src/rpc/rpc_args.cpp +++ b/src/rpc/rpc_args.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -33,28 +33,95 @@ #include #include "common/command_line.h" #include "common/i18n.h" +#include "hex.h" namespace cryptonote { + namespace + { + boost::optional do_process_ssl(const boost::program_options::variables_map& vm, const rpc_args::descriptors& arg, const bool any_cert_option) + { + bool ssl_required = false; + epee::net_utils::ssl_options_t ssl_options = epee::net_utils::ssl_support_t::e_ssl_support_enabled; + if (any_cert_option && command_line::get_arg(vm, arg.rpc_ssl_allow_any_cert)) + ssl_options.verification = epee::net_utils::ssl_verification_t::none; + else + { + std::string ssl_ca_file = command_line::get_arg(vm, arg.rpc_ssl_ca_certificates); + const std::vector ssl_allowed_fingerprints = command_line::get_arg(vm, arg.rpc_ssl_allowed_fingerprints); + + std::vector> allowed_fingerprints{ ssl_allowed_fingerprints.size() }; + std::transform(ssl_allowed_fingerprints.begin(), ssl_allowed_fingerprints.end(), allowed_fingerprints.begin(), epee::from_hex::vector); + for (const auto &fpr: allowed_fingerprints) + { + if (fpr.size() != SSL_FINGERPRINT_SIZE) + { + MERROR("SHA-256 fingerprint should be " BOOST_PP_STRINGIZE(SSL_FINGERPRINT_SIZE) " bytes long."); + return boost::none; + } + } + + if (!allowed_fingerprints.empty() || !ssl_ca_file.empty()) + { + ssl_required = true; + ssl_options = epee::net_utils::ssl_options_t{ + std::move(allowed_fingerprints), std::move(ssl_ca_file) + }; + + if (command_line::get_arg(vm, arg.rpc_ssl_allow_chained)) + ssl_options.verification = epee::net_utils::ssl_verification_t::user_ca; + } + } + + // user specified CA file or fingeprints implies enabled SSL by default + if (!ssl_required && !epee::net_utils::ssl_support_from_string(ssl_options.support, command_line::get_arg(vm, arg.rpc_ssl))) + { + MERROR("Invalid argument for " << std::string(arg.rpc_ssl.name)); + return boost::none; + } + + ssl_options.auth = epee::net_utils::ssl_authentication_t{ + command_line::get_arg(vm, arg.rpc_ssl_private_key), command_line::get_arg(vm, arg.rpc_ssl_certificate) + }; + + return {std::move(ssl_options)}; + } + } // anonymous + rpc_args::descriptors::descriptors() : rpc_bind_ip({"rpc-bind-ip", rpc_args::tr("Specify IP to bind RPC server"), "127.0.0.1"}) , rpc_login({"rpc-login", rpc_args::tr("Specify username[:password] required for RPC server"), "", true}) , confirm_external_bind({"confirm-external-bind", rpc_args::tr("Confirm rpc-bind-ip value is NOT a loopback (local) IP")}) , rpc_access_control_origins({"rpc-access-control-origins", rpc_args::tr("Specify a comma separated list of origins to allow cross origin resource sharing"), ""}) + , rpc_ssl({"rpc-ssl", rpc_args::tr("Enable SSL on RPC connections: enabled|disabled|autodetect"), "autodetect"}) + , rpc_ssl_private_key({"rpc-ssl-private-key", rpc_args::tr("Path to a PEM format private key"), ""}) + , rpc_ssl_certificate({"rpc-ssl-certificate", rpc_args::tr("Path to a PEM format certificate"), ""}) + , rpc_ssl_ca_certificates({"rpc-ssl-ca-certificates", rpc_args::tr("Path to file containing concatenated PEM format certificate(s) to replace system CA(s)."), ""}) + , rpc_ssl_allowed_fingerprints({"rpc-ssl-allowed-fingerprints", rpc_args::tr("List of certificate fingerprints to allow")}) + , rpc_ssl_allow_chained({"rpc-ssl-allow-chained", rpc_args::tr("Allow user (via --rpc-ssl-certificates) chain certificates"), false}) + , rpc_ssl_allow_any_cert({"rpc-ssl-allow-any-cert", rpc_args::tr("Allow any peer certificate"), false}) {} const char* rpc_args::tr(const char* str) { return i18n_translate(str, "cryptonote::rpc_args"); } - void rpc_args::init_options(boost::program_options::options_description& desc) + void rpc_args::init_options(boost::program_options::options_description& desc, const bool any_cert_option) { const descriptors arg{}; command_line::add_arg(desc, arg.rpc_bind_ip); command_line::add_arg(desc, arg.rpc_login); command_line::add_arg(desc, arg.confirm_external_bind); command_line::add_arg(desc, arg.rpc_access_control_origins); + command_line::add_arg(desc, arg.rpc_ssl); + command_line::add_arg(desc, arg.rpc_ssl_private_key); + command_line::add_arg(desc, arg.rpc_ssl_certificate); + command_line::add_arg(desc, arg.rpc_ssl_ca_certificates); + command_line::add_arg(desc, arg.rpc_ssl_allowed_fingerprints); + command_line::add_arg(desc, arg.rpc_ssl_allow_chained); + if (any_cert_option) + command_line::add_arg(desc, arg.rpc_ssl_allow_any_cert); } - boost::optional rpc_args::process(const boost::program_options::variables_map& vm) + boost::optional rpc_args::process(const boost::program_options::variables_map& vm, const bool any_cert_option) { const descriptors arg{}; rpc_args config{}; @@ -82,11 +149,17 @@ namespace cryptonote } } - if (command_line::has_arg(vm, arg.rpc_login)) + const char *env_rpc_login = nullptr; + const bool has_rpc_arg = command_line::has_arg(vm, arg.rpc_login); + const bool use_rpc_env = !has_rpc_arg && (env_rpc_login = getenv("RPC_LOGIN")) != nullptr && strlen(env_rpc_login) > 0; + boost::optional login{}; + if (has_rpc_arg || use_rpc_env) { - config.login = tools::login::parse(command_line::get_arg(vm, arg.rpc_login), true, [](bool verify) { - return tools::password_container::prompt(verify, "RPC server password"); - }); + config.login = tools::login::parse( + has_rpc_arg ? command_line::get_arg(vm, arg.rpc_login) : std::string(env_rpc_login), true, [](bool verify) { + return tools::password_container::prompt(verify, "RPC server password"); + }); + if (!config.login) return boost::none; @@ -102,7 +175,7 @@ namespace cryptonote { if (!config.login) { - LOG_ERROR(arg.rpc_access_control_origins.name << tr(" requires RFC server password --") << arg.rpc_login.name << tr(" cannot be empty")); + LOG_ERROR(arg.rpc_access_control_origins.name << tr(" requires RPC server password --") << arg.rpc_login.name << tr(" cannot be empty")); return boost::none; } @@ -112,6 +185,17 @@ namespace cryptonote config.access_control_origins = std::move(access_control_origins); } + auto ssl_options = do_process_ssl(vm, arg, any_cert_option); + if (!ssl_options) + return boost::none; + config.ssl_options = std::move(*ssl_options); + return {std::move(config)}; } + + boost::optional rpc_args::process_ssl(const boost::program_options::variables_map& vm, const bool any_cert_option) + { + const descriptors arg{}; + return do_process_ssl(vm, arg, any_cert_option); + } } diff --git a/src/rpc/rpc_args.h b/src/rpc/rpc_args.h index 8e375385b87..619f02b42dc 100644 --- a/src/rpc/rpc_args.h +++ b/src/rpc/rpc_args.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -35,6 +35,7 @@ #include "common/command_line.h" #include "common/password.h" +#include "net/net_ssl.h" namespace cryptonote { @@ -54,16 +55,29 @@ namespace cryptonote const command_line::arg_descriptor rpc_login; const command_line::arg_descriptor confirm_external_bind; const command_line::arg_descriptor rpc_access_control_origins; + const command_line::arg_descriptor rpc_ssl; + const command_line::arg_descriptor rpc_ssl_private_key; + const command_line::arg_descriptor rpc_ssl_certificate; + const command_line::arg_descriptor rpc_ssl_ca_certificates; + const command_line::arg_descriptor> rpc_ssl_allowed_fingerprints; + const command_line::arg_descriptor rpc_ssl_allow_chained; + const command_line::arg_descriptor rpc_ssl_allow_any_cert; }; + // `allow_any_cert` bool toggles `--rpc-ssl-allow-any-cert` configuration + static const char* tr(const char* str); - static void init_options(boost::program_options::options_description& desc); + static void init_options(boost::program_options::options_description& desc, const bool any_cert_option = false); //! \return Arguments specified by user, or `boost::none` if error - static boost::optional process(const boost::program_options::variables_map& vm); + static boost::optional process(const boost::program_options::variables_map& vm, const bool any_cert_option = false); + + //! \return SSL arguments specified by user, or `boost::none` if error + static boost::optional process_ssl(const boost::program_options::variables_map& vm, const bool any_cert_option = false); std::string bind_ip; std::vector access_control_origins; boost::optional login; // currently `boost::none` if unspecified by user + epee::net_utils::ssl_options_t ssl_options = epee::net_utils::ssl_support_t::e_ssl_support_enabled; }; } diff --git a/src/rpc/rpc_handler.cpp b/src/rpc/rpc_handler.cpp new file mode 100644 index 00000000000..af5cb98a30b --- /dev/null +++ b/src/rpc/rpc_handler.cpp @@ -0,0 +1,111 @@ + +#include +#include +#include + +#include "cryptonote_core/cryptonote_core.h" + +namespace cryptonote +{ +namespace rpc +{ + namespace + { + output_distribution_data + process_distribution(bool cumulative, std::uint64_t start_height, std::vector distribution, std::uint64_t base) + { + if (!cumulative && !distribution.empty()) + { + for (std::size_t n = distribution.size() - 1; 0 < n; --n) + distribution[n] -= distribution[n - 1]; + distribution[0] -= base; + } + + return {std::move(distribution), start_height, base}; + } + } + + boost::optional + RpcHandler::get_output_distribution(const std::function&, uint64_t&)> &f, uint64_t amount, uint64_t from_height, uint64_t to_height, const std::function &get_hash, bool cumulative, uint64_t blockchain_height) + { + static struct D + { + boost::mutex mutex; + std::vector cached_distribution; + std::uint64_t cached_from, cached_to, cached_start_height, cached_base; + crypto::hash cached_m10_hash; + crypto::hash cached_top_hash; + bool cached; + D(): cached_from(0), cached_to(0), cached_start_height(0), cached_base(0), cached_m10_hash(crypto::null_hash), cached_top_hash(crypto::null_hash), cached(false) {} + } d; + const boost::unique_lock lock(d.mutex); + + crypto::hash top_hash = crypto::null_hash; + if (d.cached_to < blockchain_height) + top_hash = get_hash(d.cached_to); + if (d.cached && amount == 0 && d.cached_from == from_height && d.cached_to == to_height && d.cached_top_hash == top_hash) + return process_distribution(cumulative, d.cached_start_height, d.cached_distribution, d.cached_base); + + std::vector distribution; + std::uint64_t start_height, base; + + // see if we can extend the cache - a common case + bool can_extend = d.cached && amount == 0 && d.cached_from == from_height && to_height > d.cached_to && top_hash == d.cached_top_hash; + if (!can_extend) + { + // we kept track of the hash 10 blocks below, if it exists, so if it matches, + // we can still pop the last 10 cached slots and try again + if (d.cached && amount == 0 && d.cached_from == from_height && d.cached_to - d.cached_from >= 10 && to_height > d.cached_to - 10) + { + crypto::hash hash10 = get_hash(d.cached_to - 10); + if (hash10 == d.cached_m10_hash) + { + d.cached_to -= 10; + d.cached_top_hash = hash10; + d.cached_m10_hash = crypto::null_hash; + d.cached_distribution.resize(d.cached_distribution.size() - 10); + can_extend = true; + } + } + } + if (can_extend) + { + std::vector new_distribution; + if (!f(amount, d.cached_to + 1, to_height, start_height, new_distribution, base)) + return boost::none; + distribution = d.cached_distribution; + distribution.reserve(distribution.size() + new_distribution.size()); + for (const auto &e: new_distribution) + distribution.push_back(e); + start_height = d.cached_start_height; + base = d.cached_base; + } + else + { + if (!f(amount, from_height, to_height, start_height, distribution, base)) + return boost::none; + } + + if (to_height > 0 && to_height >= from_height) + { + const std::uint64_t offset = std::max(from_height, start_height); + if (offset <= to_height && to_height - offset + 1 < distribution.size()) + distribution.resize(to_height - offset + 1); + } + + if (amount == 0) + { + d.cached_from = from_height; + d.cached_to = to_height; + d.cached_top_hash = get_hash(d.cached_to); + d.cached_m10_hash = d.cached_to >= 10 ? get_hash(d.cached_to - 10) : crypto::null_hash; + d.cached_distribution = distribution; + d.cached_start_height = start_height; + d.cached_base = base; + d.cached = true; + } + + return process_distribution(cumulative, start_height, std::move(distribution), base); + } +} // rpc +} // cryptonote diff --git a/src/rpc/rpc_handler.h b/src/rpc/rpc_handler.h index 64bade5a819..b81983d28ec 100644 --- a/src/rpc/rpc_handler.h +++ b/src/rpc/rpc_handler.h @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2018, The Monero Project +// Copyright (c) 2016-2019, The Monero Project // // All rights reserved. // @@ -28,24 +28,36 @@ #pragma once +#include +#include #include +#include +#include "crypto/hash.h" namespace cryptonote { +class core; namespace rpc { +struct output_distribution_data +{ + std::vector distribution; + std::uint64_t start_height; + std::uint64_t base; +}; class RpcHandler { public: + RpcHandler() { } + virtual ~RpcHandler() { } virtual std::string handle(const std::string& request) = 0; - RpcHandler() { } - - virtual ~RpcHandler() { } + static boost::optional + get_output_distribution(const std::function&, uint64_t&)> &f, uint64_t amount, uint64_t from_height, uint64_t to_height, const std::function &get_hash, bool cumulative, uint64_t blockchain_height); }; diff --git a/src/rpc/zmq_server.cpp b/src/rpc/zmq_server.cpp index 3aee8c4c7cb..ae748e052c4 100644 --- a/src/rpc/zmq_server.cpp +++ b/src/rpc/zmq_server.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2018, The Monero Project +// Copyright (c) 2016-2019, The Monero Project // // All rights reserved. // @@ -27,7 +27,6 @@ // THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "zmq_server.h" -#include namespace cryptonote { @@ -104,6 +103,10 @@ bool ZmqServer::addTCPSocket(std::string address, std::string port) rep_socket->setsockopt(ZMQ_RCVTIMEO, &DEFAULT_RPC_RECV_TIMEOUT_MS, sizeof(DEFAULT_RPC_RECV_TIMEOUT_MS)); + if (address.empty()) + address = "*"; + if (port.empty()) + port = "*"; std::string bind_address = addr_prefix + address + std::string(":") + port; rep_socket->bind(bind_address.c_str()); } diff --git a/src/rpc/zmq_server.h b/src/rpc/zmq_server.h index 0cd906a3f54..1b1e4c7cf70 100644 --- a/src/rpc/zmq_server.h +++ b/src/rpc/zmq_server.h @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2018, The Monero Project +// Copyright (c) 2016-2019, The Monero Project // // All rights reserved. // diff --git a/src/serialization/CMakeLists.txt b/src/serialization/CMakeLists.txt index 5a6bebf096f..28b775a37ad 100644 --- a/src/serialization/CMakeLists.txt +++ b/src/serialization/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2016-2018, The Monero Project +# Copyright (c) 2016-2019, The Monero Project # # All rights reserved. # diff --git a/src/serialization/binary_archive.h b/src/serialization/binary_archive.h index f47a4494d70..9f60cf311ca 100644 --- a/src/serialization/binary_archive.h +++ b/src/serialization/binary_archive.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -99,7 +99,7 @@ struct binary_archive : public binary_archive_base { explicit binary_archive(stream_type &s) : base_type(s) { - stream_type::streampos pos = stream_.tellg(); + stream_type::pos_type pos = stream_.tellg(); stream_.seekg(0, std::ios_base::end); eof_pos_ = stream_.tellg(); stream_.seekg(pos); @@ -146,7 +146,8 @@ struct binary_archive : public binary_archive_base void serialize_uvarint(T &v) { typedef std::istreambuf_iterator it; - tools::read_varint(it(stream_), it(), v); // XXX handle failure + if (tools::read_varint(it(stream_), it(), v) < 0) + stream_.setstate(std::ios_base::failbit); } void begin_array(size_t &s) diff --git a/src/serialization/binary_utils.h b/src/serialization/binary_utils.h index 79b30b337bc..790661e496f 100644 --- a/src/serialization/binary_utils.h +++ b/src/serialization/binary_utils.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/serialization/container.h b/src/serialization/container.h index 978a59d2a8a..dfe0f969135 100644 --- a/src/serialization/container.h +++ b/src/serialization/container.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2017, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/serialization/crypto.h b/src/serialization/crypto.h index 87172834f95..d2537146ef0 100644 --- a/src/serialization/crypto.h +++ b/src/serialization/crypto.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/serialization/debug_archive.h b/src/serialization/debug_archive.h index c6033a39996..093ca41eb12 100644 --- a/src/serialization/debug_archive.h +++ b/src/serialization/debug_archive.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/serialization/deque.h b/src/serialization/deque.h index 994d3f19517..97f16d0a5c3 100644 --- a/src/serialization/deque.h +++ b/src/serialization/deque.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2017, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/serialization/difficulty_type.h b/src/serialization/difficulty_type.h new file mode 100644 index 00000000000..c551095f946 --- /dev/null +++ b/src/serialization/difficulty_type.h @@ -0,0 +1,65 @@ +// Copyright (c) 2019, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +#include "cryptonote_basic/difficulty.h" +#include "serialization.h" + +template<> struct is_basic_type { typedef boost::true_type type; }; + +template