Skip to content

Refactor coverage out of GlobalRunnerState with the purpose of sharing it with the Rust FuzzTest framework for emitting features. #1775

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
copybara-service wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Refactor coverage out of GlobalRunnerState with the purpose of sharing it with the Rust FuzzTest framework for emitting features. #1775

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 36 additions & 0 deletions centipede/BUILD
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1024,6 +1024,11 @@ cc_library(
# e.g. feature.cc. These files are compiled by the engine and the runner
# separately, with different compiler flags.
RUNNER_SOURCES_NO_MAIN = [
"sancov_cpp.cc",
"shared_coverage_state.cc",
"shared_coverage_state.h",
"sancov_interface.h",
"sancov_shared.cc",
"byte_array_mutator.cc",
"byte_array_mutator.h",
"callstack.h",
Expand Down Expand Up @@ -1202,6 +1207,37 @@ cc_library(
],
)

cc_library(
name = "shared_coverage",
srcs = [
"runner_dl_info.cc",
"runner_sancov.cc",
"runner_utils.cc",
"sancov_shared.cc",
"shared_coverage_state.cc",
"@com_google_fuzztest//common:defs.h",
],
hdrs = [
"runner_dl_info.h",
"runner_interface.h",
"runner_utils.h",
"sancov_interface.h",
"shared_coverage_state.h",
],
deps = [
":callstack",
":feature",
":int_utils",
":mutation_input",
":pc_info",
":runner_cmp_trace",
"@abseil-cpp//absl/base:core_headers",
"@abseil-cpp//absl/base:nullability",
"@abseil-cpp//absl/numeric:bits",
"@abseil-cpp//absl/types:span",
],
)

# Flags for :seed_corpus_maker.
cc_library(
name = "seed_corpus_maker_flags",
Expand Down
213 changes: 100 additions & 113 deletions centipede/runner.cc
View file
Open in desktop

Large diffs are not rendered by default.

186 changes: 1 addition & 185 deletions centipede/runner.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,28 +19,21 @@
#include <string.h>
#include <time.h>

#include <algorithm>
#include <atomic>
#include <cstddef>
#include <cstdint>
#include <cstdlib>

#include "absl/base/const_init.h"
#include "absl/base/nullability.h"
#include "absl/numeric/bits.h"
#include "./centipede/byte_array_mutator.h"
#include "./centipede/callstack.h"
#include "./centipede/concurrent_bitset.h"
#include "./centipede/concurrent_byteset.h"
#include "./centipede/feature.h"
#include "./centipede/hashed_ring_buffer.h"
#include "./centipede/knobs.h"
#include "./centipede/reverse_pc_table.h"
#include "./centipede/runner_cmp_trace.h"
#include "./centipede/runner_dl_info.h"
#include "./centipede/runner_interface.h"
#include "./centipede/runner_result.h"
#include "./centipede/runner_sancov_object.h"
#include "./centipede/shared_coverage_state.h"

namespace fuzztest::internal {

Expand All @@ -54,79 +47,6 @@ class LockGuard {
pthread_mutex_t &mu_;
};

// Flags derived from CENTIPEDE_RUNNER_FLAGS.
// Flags used in instrumentation callbacks are bit-packed for efficiency.
struct RunTimeFlags {
uint64_t path_level : 8;
uint64_t use_pc_features : 1;
uint64_t use_dataflow_features : 1;
uint64_t use_cmp_features : 1;
uint64_t callstack_level : 8;
uint64_t use_counter_features : 1;
uint64_t use_auto_dictionary : 1;
std::atomic<uint64_t> timeout_per_input;
uint64_t timeout_per_batch;
std::atomic<uint64_t> stack_limit_kb;
std::atomic<uint64_t> rss_limit_mb;
uint64_t crossover_level;
uint64_t skip_seen_features : 1;
uint64_t ignore_timeout_reports : 1;
uint64_t max_len;
};

// One such object is created in runner's TLS.
// There is no CTOR, since we don't want to use the brittle and lazy TLS CTORs.
// All data members are zero-initialized during thread creation.
struct ThreadLocalRunnerState {
// Traces the memory comparison of `n` bytes at `s1` and `s2` called at
// `caller_pc` with `is_equal` indicating whether the two memory regions have
// equal contents. May add cmp features and auto-dictionary entries if
// enabled.
void TraceMemCmp(uintptr_t caller_pc, const uint8_t *s1, const uint8_t *s2,
size_t n, bool is_equal);

// Intrusive doubly-linked list of TLS objects.
// Guarded by state.tls_list_mu.
ThreadLocalRunnerState *next, *prev;

// The pthread_create() interceptor calls OnThreadStart() before the thread
// callback. The main thread also calls OnThreadStart(). OnThreadStop() will
// be called when thread termination is detected internally - see runner.cc.
void OnThreadStart();
void OnThreadStop();

// Whether OnThreadStart() is called on this thread. This is used as a proxy
// of the readiness of the lower-level runtime.
bool started;

// Paths are thread-local, so we maintain the current bounded path here.
// We allow paths of up to 100, controlled at run-time via the "path_level".
static constexpr uint64_t kBoundedPathLength = 100;
HashedRingBuffer<kBoundedPathLength> path_ring_buffer;

// Value of SP in the top call frame of the thread, computed in OnThreadStart.
uintptr_t top_frame_sp;
// The lower bound of the stack region of this thread. 0 means unknown.
uintptr_t stack_region_low;
// Lowest observed value of SP.
uintptr_t lowest_sp;

// The (imprecise) call stack is updated by the PC callback.
CallStack<> call_stack;

// Cmp traces capture the arguments of CMP instructions, memcmp, etc.
// We have dedicated traces for 2-, 4-, and 8-byte comparison, and
// a catch-all `cmp_traceN` trace for memcmp, etc.
CmpTrace<2, 64> cmp_trace2;
CmpTrace<4, 64> cmp_trace4;
CmpTrace<8, 64> cmp_trace8;
CmpTrace<0, 64> cmp_traceN;

// Set this to true if the thread needs to be ignored in ForEachTLS.
// It should be always false if the state is in the global detached_tls_list.
bool ignore;
};

// One global object of this type is created by the runner at start up.
// All data members will be initialized to zero, unless they have initializers.
// Accesses to the subobjects should be fast, so we are trying to avoid
Expand All @@ -144,79 +64,6 @@ struct GlobalRunnerState {
GlobalRunnerState();
~GlobalRunnerState();

// Runner reads flags from CentipedeGetRunnerFlags(). We don't use flags
// passed via argv so that argv flags can be passed directly to
// LLVMFuzzerInitialize, w/o filtering. The flags are separated with
// ':' on both sides, i.e. like this: ":flag1:flag2:flag3=value3".
// We do it this way to make the flag parsing code extremely simple. The
// interface is private between Centipede and the runner and may change.
//
// Note that this field reflects the initial runner flags. But some
// flags can change later (if wrapped with std::atomic).
const char *centipede_runner_flags = CentipedeGetRunnerFlags();
const char *arg1 = GetStringFlag(":arg1=");
const char *arg2 = GetStringFlag(":arg2=");
const char *arg3 = GetStringFlag(":arg3=");
// The path to a file where the runner may write the description of failure.
const char *failure_description_path =
GetStringFlag(":failure_description_path=");

// Flags.
RunTimeFlags run_time_flags = {
/*path_level=*/std::min(ThreadLocalRunnerState::kBoundedPathLength,
HasIntFlag(":path_level=", 0)),
/*use_pc_features=*/HasFlag(":use_pc_features:"),
/*use_dataflow_features=*/HasFlag(":use_dataflow_features:"),
/*use_cmp_features=*/HasFlag(":use_cmp_features:"),
/*callstack_level=*/HasIntFlag(":callstack_level=", 0),
/*use_counter_features=*/HasFlag(":use_counter_features:"),
/*use_auto_dictionary=*/HasFlag(":use_auto_dictionary:"),
/*timeout_per_input=*/HasIntFlag(":timeout_per_input=", 0),
/*timeout_per_batch=*/HasIntFlag(":timeout_per_batch=", 0),
/*stack_limit_kb=*/HasIntFlag(":stack_limit_kb=", 0),
/*rss_limit_mb=*/HasIntFlag(":rss_limit_mb=", 0),
/*crossover_level=*/HasIntFlag(":crossover_level=", 50),
/*skip_seen_features=*/HasFlag(":skip_seen_features:"),
/*ignore_timeout_reports=*/HasFlag(":ignore_timeout_reports:"),
/*max_len=*/HasIntFlag(":max_len=", 4000),
};

// Returns true iff `flag` is present.
// Typical usage: pass ":some_flag:", i.e. the flag name surrounded with ':'.
// TODO(ussuri): Refactor `char *` into a `string_view`.
bool HasFlag(const char *absl_nonnull flag) const {
if (!centipede_runner_flags) return false;
return strstr(centipede_runner_flags, flag) != nullptr;
}

// If a flag=value pair is present, returns value,
// otherwise returns `default_value`.
// Typical usage: pass ":some_flag=".
// TODO(ussuri): Refactor `char *` into a `string_view`.
uint64_t HasIntFlag(const char *absl_nonnull flag,
uint64_t default_value) const {
if (!centipede_runner_flags) return default_value;
const char *beg = strstr(centipede_runner_flags, flag);
if (!beg) return default_value;
return atoll(beg + strlen(flag)); // NOLINT: can't use strto64, etc.
}

// If a :flag=value: pair is present returns value, otherwise returns nullptr.
// The result is obtained by calling strndup, so make sure to save
// it in `this` to avoid a leak.
// Typical usage: pass ":some_flag=".
// TODO(ussuri): Refactor `char *` into a `string_view`.
const char *absl_nullable GetStringFlag(const char *absl_nonnull flag) const {
if (!centipede_runner_flags) return nullptr;
// Extract "value" from ":flag=value:" inside centipede_runner_flags.
const char *beg = strstr(centipede_runner_flags, flag);
if (!beg) return nullptr;
const char *value_beg = beg + strlen(flag);
const char *end = strstr(value_beg, ":");
if (!end) return nullptr;
return strndup(value_beg, end - value_beg);
}

pthread_mutex_t execution_result_override_mu = PTHREAD_MUTEX_INITIALIZER;
// If not nullptr, it points to a batch result with either zero or one
// execution. When an execution result present, it will be passed as the
Expand Down Expand Up @@ -247,20 +94,6 @@ struct GlobalRunnerState {
// Reclaims all TLSs in detached_tls_list and cleans up the list.
void CleanUpDetachedTls();

// Computed by DlInfo().
// Usually, the main object is the executable binary containing main()
// and most of the executable code (we assume that the target is
// built in mostly-static mode, i.e. -dynamic_mode=off).
// When the `dl_path_suffix` runner flag is provided, the main_object refers
// to the dynamic library (DSO) pointed to by this flag.
//
// Note: this runner currently does not support more than one instrumented
// DSO in the process, i.e. you either instrument the main binary, or one DSO.
// Supporting more than one DSO will require major changes,
// major added complexity, and potentially cause slowdown.
// There is currently no motivation for such a change.
DlInfo main_object;

// State for SanitizerCoverage.
// See https://clang.llvm.org/docs/SanitizerCoverage.html.
SanCovObjectArray sancov_objects;
Expand All @@ -272,14 +105,8 @@ struct GlobalRunnerState {
// Tracing CMP instructions, capture events from these domains:
// kCMPEq, kCMPModDiff, kCMPHamming, kCMPModDiffLog, kCMPMsbEq.
// See https://clang.llvm.org/docs/SanitizerCoverage.html#tracing-data-flow.
// An arbitrarily large size.
static constexpr size_t kCmpFeatureSetSize = 1 << 18;
// TODO(kcc): remove cmp_feature_set.
ConcurrentBitSet<kCmpFeatureSetSize> cmp_feature_set{absl::kConstInit};
ConcurrentBitSet<kCmpFeatureSetSize> cmp_eq_set{absl::kConstInit};
ConcurrentBitSet<kCmpFeatureSetSize> cmp_moddiff_set{absl::kConstInit};
ConcurrentBitSet<kCmpFeatureSetSize> cmp_hamming_set{absl::kConstInit};
ConcurrentBitSet<kCmpFeatureSetSize> cmp_difflog_set{absl::kConstInit};

// We think that call stack produces rich signal, so we give a few bits to it.
static constexpr size_t kCallStackFeatureSetSize = 1 << 24;
Expand Down Expand Up @@ -346,20 +173,9 @@ struct GlobalRunnerState {

// The Watchdog thread sets this to true.
std::atomic<bool> watchdog_thread_started;

// An arbitrarily large size.
static const size_t kMaxFeatures = 1 << 20;
// FeatureArray used to accumulate features from all sources.
FeatureArray<kMaxFeatures> g_features;

// Features that were seen before.
static constexpr size_t kSeenFeatureSetSize =
absl::bit_ceil(feature_domains::kLastDomain.end());
ConcurrentBitSet<kSeenFeatureSetSize> seen_features{absl::kConstInit};
};

extern GlobalRunnerState state;
extern __thread ThreadLocalRunnerState tls;

// Check for stack limit for the stack pointer `sp` in the current thread.
void CheckStackLimit(uintptr_t sp);
Expand Down
2 changes: 1 addition & 1 deletion centipede/runner_interceptors.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@

#include "absl/base/nullability.h"
#include "absl/base/optimization.h"
#include "./centipede/runner.h"
#include "./centipede/shared_coverage_state.h"

using fuzztest::internal::tls;

Expand Down
1 change: 0 additions & 1 deletion centipede/runner_interface.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@
#include <functional>
#include <memory>
#include <string>
#include <string_view>
#include <vector>

#include "absl/base/nullability.h"
Expand Down
Loading