From 35d2dee41b732ba93e2ca709a1c8bb94187f49d8 Mon Sep 17 00:00:00 2001 From: Qubasa Date: Mon, 24 Mar 2025 23:37:03 +0100 Subject: [PATCH 1/2] add cpu usage report, add multi core support. Add optional undefined behaviour sanitizer --- .gitignore | 2 + CMakeLists.txt | 29 +++++++++- README.md | 95 +++++++++++++++++--------------- client.c | 3 ++ client_stream.c | 17 +++++- common.c | 3 -- common.h | 8 +-- cpu-usage.c | 141 ++++++++++++++++++++++++++++++++++++++++++++++++ cpu-usage.h | 1 + main.c | 55 +++++++++++++++++-- server.c | 3 ++ server_stream.c | 14 ++++- 12 files changed, 314 insertions(+), 57 deletions(-) create mode 100644 cpu-usage.c create mode 100644 cpu-usage.h diff --git a/.gitignore b/.gitignore index d17ee65..1bbb202 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,5 @@ build* *.cache compile_commands.json +test_client.sh +test_server.sh \ No newline at end of file diff --git a/CMakeLists.txt b/CMakeLists.txt index 793b9b1..bec2579 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3,6 +3,29 @@ project(qperf VERSION 1.0.0 LANGUAGES C) set(CMAKE_EXPORT_COMPILE_COMMANDS ON) +# Set linker flags to use lld +if(DEFINED SANITIZE) + set(CMAKE_EXE_LINKER_FLAGS "-fuse-ld=lld") + set(CMAKE_MODULE_LINKER_FLAGS "-fuse-ld=lld") + set(CMAKE_SHARED_LINKER_FLAGS "-fuse-ld=lld") + + # Set the sanitize options + set(SANITIZE_FLAGS + -fsanitize=address + -fsanitize=undefined + -fsanitize=signed-integer-overflow + -fsanitize=shift + -fsanitize=integer-divide-by-zero + -fsanitize=array-bounds + -fsanitize=implicit-conversion + -fsanitize=nullability + -fsanitize=integer + -fsanitize-address-use-after-return=always + -fsanitize-address-use-after-scope + ) + set(BETTER_STACKTRACE -g -O1 -fno-omit-frame-pointer -fno-optimize-sibling-calls) +endif() + add_subdirectory(extern) add_executable(qperf main.c @@ -10,13 +33,15 @@ add_executable(qperf main.c client_stream.h client_stream.c server.h server.c server_stream.h server_stream.c - common.h common.c) + common.h common.c + cpu-usage.h cpu-usage.c) -target_link_libraries(qperf PRIVATE quicly ev picotls) +target_link_libraries(qperf PRIVATE quicly ev picotls ${SANITIZE_FLAGS} ${BETTER_STACKTRACE}) target_compile_definitions(qperf PRIVATE QPERF_VERSION="${PROJECT_VERSION}") target_compile_options(qperf PRIVATE -Werror=implicit-function-declaration -Werror=incompatible-pointer-types -Werror=shift-count-overflow + ${SANITIZE_FLAGS} ${BETTER_STACKTRACE} ) diff --git a/README.md b/README.md index 62d40b6..3ef5571 100644 --- a/README.md +++ b/README.md @@ -7,56 +7,67 @@ Uses https://github.com/h2o/quicly Usage: ./qperf [options] Options: - -c target run as client and connect to target server - --cc [reno,cubic] congestion control algorithm to use (default reno) - -e measure time for connection establishment and first byte only - -g enable UDP generic segmentation offload - --iw initial-window initial window to use (default 10) - -l log-file file to log tls secrets - -p port to listen on/connect to (default 18080) - -s run as server - -t time (s) run for X seconds (default 10s) - -h print this help + -c target run as client and connect to target server + --cc [reno,cubic] congestion control algorithm to use (default reno) + -e measure time for connection establishment and first byte only + -g enable UDP generic segmentation offload + --iw initial-window initial window to use (default 10) + -l log-file file to log tls secrets + -p port to listen on/connect to (default 18080) + -s address listen as server on address + -t time (s) run for X seconds (default 10s) + -n num_cores number of cores to use (default nproc) + -h print this help ``` -server + +#### client ``` -./qperf -s -starting server with pid 5624 on port 18080 -got new connection -request received, sending data -connection 0 second 0 send window: 1112923 packets sent: 364792 packets lost: 373 -connection 0 second 1 send window: 1238055 packets sent: 377515 packets lost: 123 -connection 0 second 2 send window: 583352 packets sent: 355482 packets lost: 862 -connection 0 second 3 send window: 275563 packets sent: 367538 packets lost: 607 -connection 0 second 4 send window: 1100261 packets sent: 366005 packets lost: 20 -connection 0 second 5 send window: 633010 packets sent: 356021 packets lost: 857 -connection 0 second 6 send window: 1266610 packets sent: 367866 packets lost: 0 -connection 0 second 7 send window: 1668530 packets sent: 360649 packets lost: 0 -connection 0 second 8 send window: 1994930 packets sent: 364087 packets lost: 0 -connection 0 second 9 send window: 1779683 packets sent: 374804 packets lost: 80 -connection 0 total packets sent: 3654759 total packets lost: 2922 +./qperf -c 2a01:4f8:1c1b:fd7a::1 -p 18000 -n 1 +starting client with host 2a01:4f8:1c1b:fd7a::1, port 18000, runtime 10s, cc reno, iw 10 +connection establishment time: 16ms +time to first byte: 16ms +second 0: 1.158 gbit/s, cpu 0: 21.65% +second 1: 1.161 gbit/s, cpu 2: 50.00% +second 2: 1.205 gbit/s, cpu 0: 32.29% +second 3: 1.162 gbit/s, cpu 2: 52.87% +second 4: 1.258 gbit/s, cpu 1: 34.04% +second 5: 1.387 gbit/s, cpu 1: 50.00% +second 6: 1.393 gbit/s, cpu 2: 15.96% +second 7: 1.366 gbit/s, cpu 0: 17.35% +second 8: 1.195 gbit/s, cpu 0: 46.67% +second 9: 1.226 gbit/s, cpu 1: 15.62% +connection closed ``` -*Note*: The server looks for a TLS certificate and key in the current working dir named "server.crt" and "server.key" respectively([See TLS](#TLS)). You can use a self signed certificate; the client doesn't validate it. -client +#### server + ``` -./qperf -c 127.0.0.1 -running client with host=127.0.0.1 and runtime=10s -connection establishment time: 6ms -time to first byte: 7ms -second 0: 3.144 gbit/s (422030372 bytes received) -second 1: 3.444 gbit/s (462189378 bytes received) -second 2: 3.184 gbit/s (427337822 bytes received) -second 3: 3.333 gbit/s (447304096 bytes received) -second 4: 2.996 gbit/s (402100242 bytes received) -second 5: 3.274 gbit/s (439462608 bytes received) -second 6: 3.083 gbit/s (413746021 bytes received) -second 7: 3.336 gbit/s (447686682 bytes received) -second 8: 3.034 gbit/s (407235597 bytes received) -second 9: 3.02 gbit/s (405314061 bytes received) +./qperf -s :: -p 18000 +starting server with pid 15291,address ::, port 18000, cc reno, iw 10 +starting server with pid 15293,address ::, port 18002, cc reno, iw 10 +starting server with pid 15292,address ::, port 18001, cc reno, iw 10 +starting server with pid 15294,address ::, port 18003, cc reno, iw 10 +got new connection +request received, sending data +connection 0 second 0 send window: 622682 packets sent: 133403 packets lost: 2289, cpu 3: 1.99% +connection 0 second 1 send window: 716581 packets sent: 126095 packets lost: 23, cpu 3: 98.99% +connection 0 second 2 send window: 652943 packets sent: 131597 packets lost: 37, cpu 3: 97.98% +connection 0 second 3 send window: 563577 packets sent: 138904 packets lost: 35, cpu 3: 96.97% +connection 0 second 4 send window: 637191 packets sent: 127947 packets lost: 43, cpu 3: 100.00% +connection 0 second 5 send window: 727889 packets sent: 119218 packets lost: 13, cpu 3: 99.00% +connection 0 second 6 send window: 577401 packets sent: 111255 packets lost: 62, cpu 2: 53.33% +connection 0 second 7 send window: 807801 packets sent: 97623 packets lost: 0, cpu 2: 100.00% +connection 0 second 8 send window: 976761 packets sent: 92145 packets lost: 0, cpu 3: 53.54% +connection 0 second 9 send window: 890324 packets sent: 104484 packets lost: 11, cpu 3: 100.00% +transport close:code=0x0;frame=0;reason= +connection 0 second 10 send window: 891604 packets sent: 0 packets lost: 0, cpu 3: -nan% +connection 0 total packets sent: 1182671 total packets lost: 2513 +connection closed ``` +*Note*: The server looks for a TLS certificate and key in the current working dir named "server.crt" and "server.key" respectively([See TLS](#TLS)). You can use a self signed certificate; the client doesn't validate it. + # how to build ## 1. Install required dependencies diff --git a/client.c b/client.c index 4fef638..cd8dbeb 100755 --- a/client.c +++ b/client.c @@ -1,6 +1,7 @@ #include "client.h" #include "client_stream.h" #include "common.h" +#include "cpu-usage.h" #include #include @@ -13,6 +14,7 @@ #include #include + #include static int client_socket = -1; @@ -125,6 +127,7 @@ int run_client(const char *port, bool gso, const char *logfile, const char *cc, { setup_session_cache(get_tlsctx()); quicly_amend_ptls_context(get_tlsctx()); + getcpuusage(); client_ctx = quicly_spec_context; client_ctx.tls = get_tlsctx(); diff --git a/client_stream.c b/client_stream.c index f3a7a39..3091a8c 100644 --- a/client_stream.c +++ b/client_stream.c @@ -1,9 +1,17 @@ + +#define _GNU_SOURCE #include "client_stream.h" #include "client.h" #include "common.h" #include #include #include +#include +#include +#include +#include "cpu-usage.h" + + static int current_second = 0; static uint64_t bytes_received = 0; @@ -28,8 +36,15 @@ static void report_cb(EV_P_ ev_timer *w, int revents) { char size_str[100]; format_size(size_str, bytes_received); + double *percent = getcpuusage(); + + int cpu_core = sched_getcpu(); + if (cpu_core == -1) { + perror("sched_getcpu"); + exit(1); + } - printf("second %i: %s (%lu bytes received)\n", current_second, size_str, bytes_received); + printf("second %i: %s, cpu %d: %.2f%%\n", current_second, size_str, cpu_core, percent[cpu_core]); fflush(stdout); ++current_second; bytes_received = 0; diff --git a/common.c b/common.c index 1a4776c..8bf8e9f 100644 --- a/common.c +++ b/common.c @@ -1,5 +1,4 @@ #include "common.h" - #include #include #include @@ -22,8 +21,6 @@ struct addrinfo *get_address(const char *host, const char *port) struct addrinfo hints; struct addrinfo *result; - printf("resolving %s:%s\n", host, port); - memset(&hints, 0, sizeof(struct addrinfo)); hints.ai_family = AF_UNSPEC; // Let getaddrinfo decide if it's a hostname. diff --git a/common.h b/common.h index 280db6f..fc6f285 100644 --- a/common.h +++ b/common.h @@ -6,6 +6,8 @@ #include #include +#define BREAKPOINT printf("%s() at %s:%d\n", __func__, __FILE__, __LINE__); __asm__("int3; nop" ::: "memory"); + ptls_context_t *get_tlsctx(); struct addrinfo *get_address(const char *host, const char *port); @@ -13,7 +15,6 @@ void enable_gso(); bool send_pending(quicly_context_t *ctx, int fd, quicly_conn_t *conn); void print_escaped(const char *src, size_t len); - static inline int64_t min_int64(int64_t a, int64_t b) { if(a < b) { @@ -44,7 +45,7 @@ static inline int64_t clamp_int64(int64_t val, int64_t min, int64_t max) static inline uint64_t get_current_pid() { - uint64_t pid; + uint64_t pid; #ifdef __APPLE__ pthread_threadid_np(NULL, &pid); @@ -53,4 +54,5 @@ static inline uint64_t get_current_pid() #endif return pid; -} \ No newline at end of file +} + diff --git a/cpu-usage.c b/cpu-usage.c new file mode 100644 index 0000000..20dab6e --- /dev/null +++ b/cpu-usage.c @@ -0,0 +1,141 @@ +#include +#include +#include +#include +#include +#include +#include + +#define PROCSTATFILE "/proc/stat" + +void eprintf(const char *fmt, ...) { + va_list ap; + va_start(ap, fmt); + vfprintf(stderr, fmt, ap); + va_end(ap); + exit(EXIT_FAILURE); +} + +void *emalloc(size_t size) { + void *p; + + p = malloc(size); + if (!p) + eprintf("out of memory\n"); + return p; +} + +void *ecalloc(size_t nmemb, size_t size) { + void *p; + + p = calloc(nmemb, size); + if (!p) + eprintf("out of memory\n"); + return p; +} + +int getcpucount(void) { + int i; + FILE *fp; + char buf[BUFSIZ]; + + fp = fopen(PROCSTATFILE, "r"); + if (!fp) + eprintf("can't open %s\n", PROCSTATFILE); + + if(!fgets(buf, BUFSIZ, fp)){ + eprintf("error reading %s\n", PROCSTATFILE); + exit(1); + } + + for (i = 0; fgets(buf, BUFSIZ, fp); i++) + if (!!strncmp("cpu", buf, 3)) + break; + + fclose(fp); + + return i; +} + +double *getcpuusage(void) { + int i; + char buf[BUFSIZ]; + int cpucount; + unsigned long long total; + static double *percent; + FILE *fp; + + struct cpudata { + int cpuid; + unsigned long long user; + unsigned long long nice; + unsigned long long system; + unsigned long long idle; + }; + + struct cpudata *cd; + static struct cpudata *prev; + + cpucount = getcpucount(); + + if (!percent) + percent = ecalloc((cpucount + 1), sizeof(double)); + + if (!prev) + prev = emalloc(cpucount * sizeof(struct cpudata)); + + cd = emalloc(cpucount * sizeof(struct cpudata)); + + fp = fopen(PROCSTATFILE, "r"); + if (!fp) + eprintf("can't open %s\n", PROCSTATFILE); + if (!fgets(buf, BUFSIZ, fp)) { + eprintf("error reading %s\n", PROCSTATFILE); + exit(1); + } + + for (i = 0; i < cpucount; i++) { + if (prev) { + if (!fgets(buf, BUFSIZ, fp)) { + eprintf("error reading %s\n", PROCSTATFILE); + exit(1); + } + sscanf(buf, "cpu%d %llu %llu %llu %llu", + &cd[i].cpuid, &cd[i].user, &cd[i].nice, &cd[i].system, &cd[i].idle); + + total = 0; + total += (cd[i].user - prev[i].user); + total += (cd[i].nice - prev[i].nice); + total += (cd[i].system - prev[i].system); + percent[i] = total; + total += (cd[i].idle - prev[i].idle); + percent[i] /= total; + percent[i] *= 100; + } + } + + + fclose(fp); + + fp = fopen(PROCSTATFILE, "r"); + if (!fp) + eprintf("can't open %s\n", PROCSTATFILE); + if(!fgets(buf, BUFSIZ, fp)) { + eprintf("error reading %s\n", PROCSTATFILE); + exit(1); + } + + for (i = 0; i < cpucount; i++) { + if(!fgets(buf, BUFSIZ, fp)) { + eprintf("error reading %s\n", PROCSTATFILE); + exit(1); + } + sscanf(buf, "cpu%d %llu %llu %llu %llu", + &cd[i].cpuid, &prev[i].user, &prev[i].nice, &prev[i].system, &prev[i].idle); + } + + free(cd); + fclose(fp); + + return percent; +} diff --git a/cpu-usage.h b/cpu-usage.h new file mode 100644 index 0000000..3ae00de --- /dev/null +++ b/cpu-usage.h @@ -0,0 +1 @@ +double *getcpuusage(void); \ No newline at end of file diff --git a/main.c b/main.c index 213c212..9c5010f 100755 --- a/main.c +++ b/main.c @@ -2,7 +2,8 @@ #include #include #include - +#include + #include #include "server.h" #include "client.h" @@ -21,6 +22,7 @@ static void usage(const char *cmd) " -p port to listen on/connect to (default 18080)\n" " -s address listen as server on address\n" " -t time (s) run for X seconds (default 10s)\n" + " -n num_cores number of cores to use (default nproc)\n" " -h print this help\n" "\n", cmd); @@ -46,8 +48,12 @@ int main(int argc, char** argv) const char *logfile = NULL; const char *cc = "reno"; int iw = 10; + int num_cores = sysconf(_SC_NPROCESSORS_ONLN); + if (num_cores < 1) { + num_cores = 1; + } - while ((ch = getopt_long(argc, argv, "c:egl:p:s:t:h", long_options, NULL)) != -1) { + while ((ch = getopt_long(argc, argv, "c:egl:p:s:t:n:h", long_options, NULL)) != -1) { switch (ch) { case 0: if(strcmp(optarg, "reno") != 0 && strcmp(optarg, "cubic") != 0) { @@ -82,6 +88,7 @@ int main(int argc, char** argv) logfile = optarg; break; case 'p': + // if(sscanf(optarg, "%u", &port) < 0 || port > 65535) { fprintf(stderr, "invalid argument passed to -p\n"); exit(1); @@ -97,6 +104,13 @@ int main(int argc, char** argv) exit(1); } break; + case 'n': + if(sscanf(optarg, "%u", &num_cores) != 1 || num_cores < 1) { + fprintf(stderr, "invalid argument passed to -n\n"); + exit(1); + } + break; + default: usage(argv[0]); exit(1); @@ -116,7 +130,38 @@ int main(int argc, char** argv) char port_char[16]; sprintf(port_char, "%d", port); - return server_mode ? - run_server(address, port_char, gso, logfile, cc, iw, "server.crt", "server.key") : - run_client(port_char, gso, logfile, cc, iw, host, runtime_s, ttfb_only); + + pid_t *child_pids = malloc(num_cores * sizeof(pid_t)); + if (!child_pids) { + fprintf(stderr, "failed to allocate memory for child pids\n"); + exit(1); + } + for (int i = 0; i < num_cores; i++) { + int new_port = port + i; + char new_port_char[16]; + sprintf(new_port_char, "%d", new_port); + + pid_t pid = fork(); + if (pid < 0) { + fprintf(stderr, "fork failed\n"); + exit(1); + } else if (pid == 0) { + // Child process: run as server or client with new_port + if (server_mode) { + exit(run_server(address, new_port_char, gso, logfile, cc, iw, "server.crt", "server.key")); + } else { + exit(run_client(new_port_char, gso, logfile, cc, iw, host, runtime_s, ttfb_only)); + } + } else { + child_pids[i] = pid; + } + } + + // Parent process waits for all child processes to finish. + for (int i = 0; i < num_cores; i++) { + int status; + waitpid(child_pids[i], &status, 0); + } + free(child_pids); + return 0; } \ No newline at end of file diff --git a/server.c b/server.c index 83ddce9..00cbeb7 100755 --- a/server.c +++ b/server.c @@ -1,6 +1,7 @@ #include "server.h" #include "server_stream.h" #include "common.h" +#include "cpu-usage.h" #include #include @@ -49,6 +50,7 @@ static int udp_listen(struct addrinfo *addr) return -1; } + static inline quicly_conn_t *find_conn(struct sockaddr_storage *sa, socklen_t salen, quicly_decoded_packet_t *packet) { for(size_t i = 0; i < num_conns; ++i) { @@ -178,6 +180,7 @@ int run_server(const char* address, const char *port, bool gso, const char *logf { setup_session_cache(get_tlsctx()); quicly_amend_ptls_context(get_tlsctx()); + getcpuusage(); server_ctx = quicly_spec_context; server_ctx.tls = get_tlsctx(); diff --git a/server_stream.c b/server_stream.c index f76f088..2a3a53f 100644 --- a/server_stream.c +++ b/server_stream.c @@ -1,5 +1,8 @@ +#define _GNU_SOURCE #include "server_stream.h" +#include "cpu-usage.h" +#include #include #include #include @@ -28,7 +31,16 @@ static void print_report(server_stream *s) s->report_num_packets_lost = stats.num_packets.lost - s->total_num_packets_lost; s->total_num_packets_sent = stats.num_packets.sent; s->total_num_packets_lost = stats.num_packets.lost; - printf("connection %i second %i send window: %"PRIu32" packets sent: %"PRIu64" packets lost: %"PRIu64"\n", s->report_id, s->report_second, stats.cc.cwnd, s->report_num_packets_sent, s->report_num_packets_lost); + double *percent = getcpuusage(); + + int cpu_core = sched_getcpu(); + if (cpu_core == -1) { + perror("sched_getcpu"); + exit(1); + } + + + printf("connection %i second %i send window: %"PRIu32" packets sent: %"PRIu64" packets lost: %"PRIu64", cpu %d: %.2f%%\n", s->report_id, s->report_second, stats.cc.cwnd, s->report_num_packets_sent, s->report_num_packets_lost, cpu_core, percent[cpu_core]); fflush(stdout); ++s->report_second; } From 423098cdc67f6b100b7413af1a876ef51722460d Mon Sep 17 00:00:00 2001 From: Qubasa Date: Mon, 31 Mar 2025 15:33:07 +0200 Subject: [PATCH 2/2] renamed -n num_cores to -i num_instances --- README.md | 2 +- main.c | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 3ef5571..51db9cf 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ Options: -p port to listen on/connect to (default 18080) -s address listen as server on address -t time (s) run for X seconds (default 10s) - -n num_cores number of cores to use (default nproc) + -i num_instances number of instances to start (default nproc) -h print this help ``` diff --git a/main.c b/main.c index 9c5010f..4657d72 100755 --- a/main.c +++ b/main.c @@ -22,7 +22,7 @@ static void usage(const char *cmd) " -p port to listen on/connect to (default 18080)\n" " -s address listen as server on address\n" " -t time (s) run for X seconds (default 10s)\n" - " -n num_cores number of cores to use (default nproc)\n" + " -i num_instances number of instances to start (default nproc)\n" " -h print this help\n" "\n", cmd); @@ -53,7 +53,7 @@ int main(int argc, char** argv) num_cores = 1; } - while ((ch = getopt_long(argc, argv, "c:egl:p:s:t:n:h", long_options, NULL)) != -1) { + while ((ch = getopt_long(argc, argv, "c:egl:p:s:t:i:h", long_options, NULL)) != -1) { switch (ch) { case 0: if(strcmp(optarg, "reno") != 0 && strcmp(optarg, "cubic") != 0) { @@ -78,7 +78,7 @@ int main(int argc, char** argv) case 'g': #ifdef __linux__ gso = true; - printf("using UDP GSO, requires kernel >= 4.18\n"); + fprintf(stderr, "using UDP GSO, requires kernel >= 4.18\n"); #else fprintf(stderr, "UDP GSO only supported on linux\n"); exit(1); @@ -104,9 +104,9 @@ int main(int argc, char** argv) exit(1); } break; - case 'n': + case 'i': if(sscanf(optarg, "%u", &num_cores) != 1 || num_cores < 1) { - fprintf(stderr, "invalid argument passed to -n\n"); + fprintf(stderr, "invalid argument passed to -i\n"); exit(1); } break; @@ -164,4 +164,4 @@ int main(int argc, char** argv) } free(child_pids); return 0; -} \ No newline at end of file +}