Skip to content

backport: Merge bitcoin/bitcoin#28815, 28931, 28009, 26679 #7092

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
vijaydasmp wants to merge 4 commits into
base: develop
Choose a base branch
from
Draft

backport: Merge bitcoin/bitcoin#28815, 28931, 28009, 26679 #7092

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
db74b18
Merge bitcoin/bitcoin#28815: fuzz: Avoid timeout and bloat in fuzz ta…
fanquake Nov 8, 2023
64ff0b3
Merge bitcoin/bitcoin#28931: fuzz: Limit fuzz buffer size in script_f…
fanquake Nov 26, 2023
f00dec1
Merge bitcoin/bitcoin#28009: script, test: python typing and linter u…
fanquake Jun 30, 2023
0bed3e3
Merge bitcoin/bitcoin#26679: wallet: Skip rescanning if wallet is mor…
achow101 Jan 11, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions ci/lint/04_install.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,11 +33,11 @@ if [ -z "${SKIP_PYTHON_INSTALL}" ]; then
python3 --version
fi

${CI_RETRY_EXE} pip3 install codespell==2.2.1
${CI_RETRY_EXE} pip3 install flake8==4.0.1
${CI_RETRY_EXE} pip3 install lief==0.13.1
${CI_RETRY_EXE} pip3 install mypy==0.981
${CI_RETRY_EXE} pip3 install pyzmq==24.0.1
${CI_RETRY_EXE} pip3 install codespell==2.2.5
${CI_RETRY_EXE} pip3 install flake8==6.0.0
${CI_RETRY_EXE} pip3 install lief==0.13.2
${CI_RETRY_EXE} pip3 install mypy==1.4.1
${CI_RETRY_EXE} pip3 install pyzmq==25.1.0
${CI_RETRY_EXE} pip3 install vulture==2.6

SHELLCHECK_VERSION=v0.8.0
Expand Down
13 changes: 9 additions & 4 deletions src/script/interpreter.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,16 +7,21 @@
#define BITCOIN_SCRIPT_INTERPRETER_H

#include <script/script_error.h>
#include <consensus/amount.h>
#include <primitives/transaction.h>
#include <script/script_error.h> // IWYU pragma: export
#include <span.h>
#include <uint256.h>

#include <cstddef>
#include <cstdint>
#include <vector>
#include <stdint.h>

class CPubKey;
class CScript;
class CTransaction;
class CTxOut;
class uint256;
class CScriptNum;
class XOnlyPubKey;
struct CScriptWitness;

/** Signature hash types/flags */
enum
Expand Down
10 changes: 7 additions & 3 deletions src/test/fuzz/bloom_filter.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,21 +10,22 @@
#include <uint256.h>

#include <cassert>
#include <cstdint>
#include <limits>
#include <optional>
#include <string>
#include <vector>

FUZZ_TARGET(bloom_filter)
{
FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size());
bool good_data{true};

CBloomFilter bloom_filter{
fuzzed_data_provider.ConsumeIntegralInRange<unsigned int>(1, 10000000),
1.0 / fuzzed_data_provider.ConsumeIntegralInRange<unsigned int>(1, std::numeric_limits<unsigned int>::max()),
fuzzed_data_provider.ConsumeIntegral<unsigned int>(),
static_cast<unsigned char>(fuzzed_data_provider.PickValueInArray({BLOOM_UPDATE_NONE, BLOOM_UPDATE_ALL, BLOOM_UPDATE_P2PUBKEY_ONLY, BLOOM_UPDATE_MASK}))};
LIMITED_WHILE(fuzzed_data_provider.remaining_bytes() > 0, 10000) {
LIMITED_WHILE(good_data && fuzzed_data_provider.remaining_bytes() > 0, 10'000)
{
CallOneOf(
fuzzed_data_provider,
[&] {
Expand All @@ -37,6 +38,7 @@ FUZZ_TARGET(bloom_filter)
[&] {
const std::optional<COutPoint> out_point = ConsumeDeserializable<COutPoint>(fuzzed_data_provider);
if (!out_point) {
good_data = false;
return;
}
(void)bloom_filter.contains(*out_point);
Expand All @@ -47,6 +49,7 @@ FUZZ_TARGET(bloom_filter)
[&] {
const std::optional<uint256> u256 = ConsumeDeserializable<uint256>(fuzzed_data_provider);
if (!u256) {
good_data = false;
return;
}
(void)bloom_filter.contains(*u256);
Expand All @@ -57,6 +60,7 @@ FUZZ_TARGET(bloom_filter)
[&] {
const std::optional<CMutableTransaction> mut_tx = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider);
if (!mut_tx) {
good_data = false;
return;
}
const CTransaction tx{*mut_tx};
Expand Down
23 changes: 17 additions & 6 deletions src/test/fuzz/coins_view.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,28 +2,30 @@
// Distributed under the MIT software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.

#include <chainparams.h>
#include <chainparamsbase.h>
#include <coins.h>
#include <consensus/amount.h>
#include <consensus/tx_check.h>
#include <consensus/tx_verify.h>
#include <consensus/validation.h>
#include <key.h>
#include <policy/policy.h>
#include <primitives/transaction.h>
#include <pubkey.h>
#include <script/interpreter.h>
#include <sync.h>
#include <test/fuzz/FuzzedDataProvider.h>
#include <test/fuzz/fuzz.h>
#include <test/fuzz/util.h>
#include <test/util/setup_common.h>
#include <validation.h>
#include <util/hasher.h>

#include <cassert>
#include <cstdint>
#include <limits>
#include <memory>
#include <optional>
#include <stdexcept>
#include <string>
#include <utility>
#include <vector>

namespace {
Expand All @@ -46,12 +48,15 @@ void initialize_coins_view()
FUZZ_TARGET(coins_view, .init = initialize_coins_view)
{
FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()};
bool good_data{true};

CCoinsView backend_coins_view;
CCoinsViewCache coins_view_cache{&backend_coins_view, /*deterministic=*/true};
COutPoint random_out_point;
Coin random_coin;
CMutableTransaction random_mutable_transaction;
LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 10000) {
LIMITED_WHILE(good_data && fuzzed_data_provider.ConsumeBool(), 10'000)
{
CallOneOf(
fuzzed_data_provider,
[&] {
Expand Down Expand Up @@ -97,35 +102,41 @@ FUZZ_TARGET(coins_view, .init = initialize_coins_view)
[&] {
const std::optional<COutPoint> opt_out_point = ConsumeDeserializable<COutPoint>(fuzzed_data_provider);
if (!opt_out_point) {
good_data = false;
return;
}
random_out_point = *opt_out_point;
},
[&] {
const std::optional<Coin> opt_coin = ConsumeDeserializable<Coin>(fuzzed_data_provider);
if (!opt_coin) {
good_data = false;
return;
}
random_coin = *opt_coin;
},
[&] {
const std::optional<CMutableTransaction> opt_mutable_transaction = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider);
if (!opt_mutable_transaction) {
good_data = false;
return;
}
random_mutable_transaction = *opt_mutable_transaction;
},
[&] {
CCoinsMapMemoryResource resource;
CCoinsMap coins_map{0, SaltedOutpointHasher{/*deterministic=*/true}, CCoinsMap::key_equal{}, &resource};
LIMITED_WHILE (fuzzed_data_provider.ConsumeBool(), 10000) {

LIMITED_WHILE (good_data && fuzzed_data_provider.ConsumeBool(), 10'000)
{
CCoinsCacheEntry coins_cache_entry;
coins_cache_entry.flags = fuzzed_data_provider.ConsumeIntegral<unsigned char>();
if (fuzzed_data_provider.ConsumeBool()) {
coins_cache_entry.coin = random_coin;
} else {
const std::optional<Coin> opt_coin = ConsumeDeserializable<Coin>(fuzzed_data_provider);
if (!opt_coin) {
good_data = false;
return;
}
coins_cache_entry.coin = *opt_coin;
Expand Down
5 changes: 5 additions & 0 deletions src/test/fuzz/fuzz.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,11 @@
/**
* Can be used to limit a theoretically unbounded loop. This caps the runtime
* to avoid timeouts or OOMs.
*
* This can be used in combination with a check in the condition to confirm
* whether the fuzz engine provided "good" data. If the fuzz input contains
* invalid data, the loop aborts early. This will teach the fuzz engine to look
* for useful data and avoids bloating the fuzz input folder with useless data.
*/
#define LIMITED_WHILE(condition, limit) \
for (unsigned _count{limit}; (condition) && _count; --_count)
Expand Down
12 changes: 7 additions & 5 deletions src/test/fuzz/policy_estimator.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,15 +4,13 @@

#include <policy/fees.h>
#include <primitives/transaction.h>
#include <streams.h>
#include <test/fuzz/FuzzedDataProvider.h>
#include <test/fuzz/fuzz.h>
#include <test/fuzz/util.h>
#include <test/util/setup_common.h>
#include <txmempool.h>

#include <cstdint>
#include <optional>
#include <string>
#include <vector>

void initialize_policy_estimator()
Expand All @@ -23,13 +21,15 @@ void initialize_policy_estimator()
FUZZ_TARGET(policy_estimator, .init = initialize_policy_estimator)
{
FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size());
bool good_data{true};
CBlockPolicyEstimator block_policy_estimator;
LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 10000) {
LIMITED_WHILE(good_data && fuzzed_data_provider.ConsumeBool(), 10'000) {
CallOneOf(
fuzzed_data_provider,
[&] {
const std::optional<CMutableTransaction> mtx = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider);
if (!mtx) {
good_data = false;
return;
}
const CTransaction tx{*mtx};
Expand All @@ -40,9 +40,11 @@ FUZZ_TARGET(policy_estimator, .init = initialize_policy_estimator)
},
[&] {
std::vector<CTxMemPoolEntry> mempool_entries;
LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 10000) {
LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 10000)
{
const std::optional<CMutableTransaction> mtx = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider);
if (!mtx) {
good_data = false;
break;
}
const CTransaction tx{*mtx};
Expand Down
36 changes: 23 additions & 13 deletions src/test/fuzz/rpc.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,37 +3,38 @@
// file COPYING or http://www.opensource.org/licenses/mit-license.php.

#include <base58.h>
#include <core_io.h>
#include <key.h>
#include <key_io.h>
#include <node/context.h>
#include <primitives/block.h>
#include <primitives/transaction.h>
#include <psbt.h>
#include <rpc/blockchain.h>
#include <rpc/client.h>
#include <rpc/request.h>
#include <rpc/server.h>
#include <rpc/util.h>
#include <span.h>
#include <streams.h>
#include <test/fuzz/FuzzedDataProvider.h>
#include <test/fuzz/fuzz.h>
#include <test/fuzz/util.h>
#include <test/util/setup_common.h>
#include <tinyformat.h>
#include <uint256.h>
#include <univalue.h>
#include <util/strencodings.h>
#include <util/string.h>
#include <util/time.h>

#include <algorithm>
#include <cassert>
#include <cstdint>
#include <cstdlib>
#include <exception>
#include <iostream>
#include <memory>
#include <optional>
#include <stdexcept>
#include <string>
#include <vector>
enum class ChainType;

namespace {
struct RPCFuzzTestingSetup : public TestingSetup {
Expand Down Expand Up @@ -173,7 +174,7 @@ const std::vector<std::string> RPC_COMMANDS_SAFE_FOR_FUZZING{
"waitfornewblock",
};

std::string ConsumeScalarRPCArgument(FuzzedDataProvider& fuzzed_data_provider)
std::string ConsumeScalarRPCArgument(FuzzedDataProvider& fuzzed_data_provider, bool& good_data)
{
const size_t max_string_length = 4096;
const size_t max_base58_bytes_length{64};
Expand Down Expand Up @@ -240,6 +241,7 @@ std::string ConsumeScalarRPCArgument(FuzzedDataProvider& fuzzed_data_provider)
// hex encoded block
std::optional<CBlock> opt_block = ConsumeDeserializable<CBlock>(fuzzed_data_provider);
if (!opt_block) {
good_data = false;
return;
}
CDataStream data_stream{SER_NETWORK, PROTOCOL_VERSION};
Expand All @@ -250,6 +252,7 @@ std::string ConsumeScalarRPCArgument(FuzzedDataProvider& fuzzed_data_provider)
// hex encoded block header
std::optional<CBlockHeader> opt_block_header = ConsumeDeserializable<CBlockHeader>(fuzzed_data_provider);
if (!opt_block_header) {
good_data = false;
return;
}
CDataStream data_stream{SER_NETWORK, PROTOCOL_VERSION};
Expand All @@ -260,6 +263,7 @@ std::string ConsumeScalarRPCArgument(FuzzedDataProvider& fuzzed_data_provider)
// hex encoded tx
std::optional<CMutableTransaction> opt_tx = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider);
if (!opt_tx) {
good_data = false;
return;
}
CDataStream data_stream{SER_NETWORK, PROTOCOL_VERSION};
Expand All @@ -270,6 +274,7 @@ std::string ConsumeScalarRPCArgument(FuzzedDataProvider& fuzzed_data_provider)
// base64 encoded psbt
std::optional<PartiallySignedTransaction> opt_psbt = ConsumeDeserializable<PartiallySignedTransaction>(fuzzed_data_provider);
if (!opt_psbt) {
good_data = false;
return;
}
CDataStream data_stream{SER_NETWORK, PROTOCOL_VERSION};
Expand All @@ -280,6 +285,7 @@ std::string ConsumeScalarRPCArgument(FuzzedDataProvider& fuzzed_data_provider)
// base58 encoded key
CKey key = ConsumePrivateKey(fuzzed_data_provider);
if (!key.IsValid()) {
good_data = false;
return;
}
r = EncodeSecret(key);
Expand All @@ -288,25 +294,27 @@ std::string ConsumeScalarRPCArgument(FuzzedDataProvider& fuzzed_data_provider)
// hex encoded pubkey
CKey key = ConsumePrivateKey(fuzzed_data_provider);
if (!key.IsValid()) {
good_data = false;
return;
}
r = HexStr(key.GetPubKey());
});
return r;
}

std::string ConsumeArrayRPCArgument(FuzzedDataProvider& fuzzed_data_provider)
std::string ConsumeArrayRPCArgument(FuzzedDataProvider& fuzzed_data_provider, bool& good_data)
{
std::vector<std::string> scalar_arguments;
LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 100) {
scalar_arguments.push_back(ConsumeScalarRPCArgument(fuzzed_data_provider));
LIMITED_WHILE(good_data && fuzzed_data_provider.ConsumeBool(), 100)
{
scalar_arguments.push_back(ConsumeScalarRPCArgument(fuzzed_data_provider, good_data));
}
return "[\"" + Join(scalar_arguments, "\",\"") + "\"]";
}

std::string ConsumeRPCArgument(FuzzedDataProvider& fuzzed_data_provider)
std::string ConsumeRPCArgument(FuzzedDataProvider& fuzzed_data_provider, bool& good_data)
{
return fuzzed_data_provider.ConsumeBool() ? ConsumeScalarRPCArgument(fuzzed_data_provider) : ConsumeArrayRPCArgument(fuzzed_data_provider);
return fuzzed_data_provider.ConsumeBool() ? ConsumeScalarRPCArgument(fuzzed_data_provider, good_data) : ConsumeArrayRPCArgument(fuzzed_data_provider, good_data);
}

RPCFuzzTestingSetup* InitializeRPCFuzzTestingSetup()
Expand Down Expand Up @@ -342,6 +350,7 @@ void initialize_rpc()
FUZZ_TARGET(rpc, .init = initialize_rpc)
{
FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()};
bool good_data{true};
SetMockTime(ConsumeTime(fuzzed_data_provider));
const std::string rpc_command = fuzzed_data_provider.ConsumeRandomLengthString(64);
if (!g_limit_to_rpc_command.empty() && rpc_command != g_limit_to_rpc_command) {
Expand All @@ -352,8 +361,9 @@ FUZZ_TARGET(rpc, .init = initialize_rpc)
return;
}
std::vector<std::string> arguments;
LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 100) {
arguments.push_back(ConsumeRPCArgument(fuzzed_data_provider));
LIMITED_WHILE(good_data && fuzzed_data_provider.ConsumeBool(), 100)
{
arguments.push_back(ConsumeRPCArgument(fuzzed_data_provider, good_data));
}
try {
rpc_testing_setup->CallRPC(rpc_command, arguments);
Expand Down
Loading
Loading