From 588c546ba907ca79390d3be5a0dae0da247664dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Linus=20L=C3=BCssing?= Date: Sun, 20 Dec 2020 19:52:13 +0100 Subject: [PATCH 1/5] Makefile: allow overriding CC, CFLAGS and LDFLAGS from env MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Useful if compiling against a library from a custom directory, for instance a modified lipcap. Signed-off-by: Linus Lüssing --- Makefile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 203122d..fad2dd1 100644 --- a/Makefile +++ b/Makefile @@ -1,14 +1,14 @@ -CC=gcc +CC?=gcc -CFLAGS=-Wall -Werror -ggdb -LDFLAGS=-lpcap +CFLAGS+=-Wall -Werror -ggdb +LDFLAGS?=-lpcap NAME=bpfcountd PREFIX?=/usr/local CONFDIR?=${PREFIX}/etc/bpfcountd bpfcountd: main.o list.o usock.o filters.o util.o - $(CC) main.o list.o usock.o filters.o util.o -o ${NAME} ${LDFLAGS} + $(CC) ${CFLAGS} main.o list.o usock.o filters.o util.o -o ${NAME} ${LDFLAGS} all: test bpfcountd From 73092a0f8c67645dc0125374a08e1966537d6394 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Linus=20L=C3=BCssing?= Date: Sun, 20 Dec 2020 20:50:45 +0100 Subject: [PATCH 2/5] filter: introducing rule substitutions via ${RULENAME} MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This allows to reuse a previously defined filter. For instance like: MDNS;udp port 5353 MDNS4;ip && ${MDNS} MDNS6;ip6 && ${MDNS} Signed-off-by: Linus Lüssing --- filters.c | 28 +++++++++++++++++++++++----- filters.h | 3 ++- util.c | 14 ++++++++++++-- 3 files changed, 37 insertions(+), 8 deletions(-) diff --git a/filters.c b/filters.c index c52f84d..2cfb74d 100644 --- a/filters.c +++ b/filters.c @@ -25,12 +25,31 @@ void filters_finish(filters_ctx *ctx) { list_free(ctx->filters); } -void filters_add(filters_ctx *ctx, const char *id, const char* bpf_str) { +static void filters_bpfstr_unwind(filters_ctx *ctx, char *bpf_str) +{ + char token[1024 + strlen("${}")]; + + list_foreach(ctx->filters, f) { + struct filter *tmp = list_data(f, struct filter); + + snprintf(token, sizeof(token), "${%s}", tmp->id); + strnrepl(token, tmp->bpf_str, bpf_str, 4096); + } +} + +static void filters_bpfstr_unwind_finish(filters_ctx *ctx) +{ + list_foreach(ctx->filters, f) + free(list_data(f, struct filter)->bpf_str); +} + +void filters_add(filters_ctx *ctx, const char *id, char *bpf_str) { struct filter *instance = malloc(sizeof(*instance)); // TODO: document maximum len of id strncpy(instance->id, id, 1023); instance->id[1023] = '\0'; + instance->bpf_str = bpf_str; instance->bpf = malloc(sizeof(struct bpf_program)); instance->packets_count = 0; @@ -79,17 +98,16 @@ void filters_load(filters_ctx *ctx, const char *filterfile_path, const char *mac // // TODO: here is dump!! char *bpf = malloc(4096); - strncpy(bpf, bpf_tmp, 4096); + strncpy(bpf, bpf_tmp, 4095); strnrepl("$MAC", mac_addr, bpf, 4096); + filters_bpfstr_unwind(ctx, bpf); fprintf(stderr, "id: %s; bpf: \"%s\";\n", id, bpf); filters_add(ctx, id, bpf); - - // the bpf is not needed anymore - free(bpf); } + filters_bpfstr_unwind_finish(ctx); free(line); fclose(fp); } diff --git a/filters.h b/filters.h index 93e0eec..366546a 100644 --- a/filters.h +++ b/filters.h @@ -7,6 +7,7 @@ struct filter { char id[1024]; + char *bpf_str; struct bpf_program *bpf; unsigned long long packets_count; unsigned long long bytes_count; @@ -19,7 +20,7 @@ typedef struct { void filters_init(filters_ctx *ctx, pcap_t* pcap_ctx); void filters_finish(filters_ctx *ctx); -void filters_add(filters_ctx *ctx, const char *id, const char* bpf_str); +void filters_add(filters_ctx *ctx, const char *id, char *bpf_str); void filters_load(filters_ctx *ctx, const char *filterfile_path, const char* mac_addr); void filters_process(filters_ctx *ctx, const struct pcap_pkthdr *pkthdr, const u_char *packet); diff --git a/util.c b/util.c index 245e63d..48367a7 100644 --- a/util.c +++ b/util.c @@ -23,6 +23,7 @@ void get_mac(char *dest, const char *iface) { } void strnrepl(const char *token, const char *replace, char *str, size_t n) { + const size_t str_br_len = strlen("()"); char *ptr = str; char *p_behind; size_t length_new = strlen(str); @@ -36,15 +37,24 @@ void strnrepl(const char *token, const char *replace, char *str, size_t n) { return; // calculate the new length and check for overflow - length_new += strlen(replace) - strlen(token); + length_new += strlen(replace) - strlen(token) + str_br_len; if (length_new >= n) return; // this points to the position behind the token p_behind = ptr + strlen(token); - memmove(ptr + strlen(replace), p_behind, strlen(p_behind)); + // making room for replacement + memmove(ptr + strlen(replace) + str_br_len, p_behind, + strlen(p_behind)); + + // inserting replacement + *ptr = '('; + ptr += 1; memcpy(ptr, replace, strlen(replace)); + ptr += strlen(replace); + *ptr = ')'; + str[length_new] = 0x00; } } From 53faac0ca54cea4dbf9e747ab33fe34d6b7d058a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Linus=20L=C3=BCssing?= Date: Mon, 21 Dec 2020 04:51:06 +0100 Subject: [PATCH 3/5] Fix hangs on quiet interface with epoll for pcap handler + unix socket MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previously if no packets were received on an interface bpfcountd would hang: It would neither respond to a Ctrl+c (SIGINT) nor to a unix socket client because it would keep blocking on pcap_dispatch(). Avoid this by introducing epoll for the pcap handler and unix socket. epoll_wait() will return when there is a new (batch of) packets from the pcap handler, if there is a connection request from the unix socket or if there is an interrupt from a signal handler. This should also make the unix socket a lot more responsive. epoll infrastructure will also help us to move the pcap filtering from userspace to kernelspace later, to improve performance. That is epoll allows us to easily have and handle multiple, per filter rule pcap handlers later. Signed-off-by: Linus Lüssing --- main.c | 89 ++++++++++++++++++++++++++++++++++++++++++--------------- usock.c | 13 ++++++++- usock.h | 2 +- 3 files changed, 79 insertions(+), 25 deletions(-) diff --git a/main.c b/main.c index 483e79a..8253d5c 100644 --- a/main.c +++ b/main.c @@ -3,12 +3,16 @@ #include #include #include +#include // for epoll_create1(), epoll_ctl(), struct epoll_event #include "util.h" #include "filters.h" #include "usock.h" #include "list.h" +#define MAX_EVENTS 32 + +struct epoll_event events[MAX_EVENTS]; struct config { const char *device; // TODO: rename @@ -22,10 +26,12 @@ typedef struct { filters_ctx filters_ctx; pcap_t *pcap_ctx; + int fd; } bpfcountd_ctx; -void prepare_pcap(bpfcountd_ctx *ctx, const char* device) { +void prepare_pcap(bpfcountd_ctx *ctx, const char* device, int epoll_fd) { + struct epoll_event event; char errbuf[PCAP_ERRBUF_SIZE]; memset(&errbuf, 0, sizeof(errbuf)); @@ -37,6 +43,26 @@ void prepare_pcap(bpfcountd_ctx *ctx, const char* device) { exit(1); } + if (pcap_setnonblock(ctx->pcap_ctx, 1, errbuf) == PCAP_ERROR) { + fprintf(stderr, "Can't set pcap handler nonblocking.\n"); + exit(1); + } + + ctx->fd = pcap_get_selectable_fd(ctx->pcap_ctx); + if (ctx->fd == PCAP_ERROR) { + fprintf(stderr, "Can't get file descriptor from pcap handler."); + exit(1); + } + + memset(&event, 0, sizeof(event)); + event.events = EPOLLIN; + event.data.ptr = ctx->pcap_ctx; + + if (epoll_ctl(epoll_fd, EPOLL_CTL_ADD, ctx->fd, &event)) { + fprintf(stderr, "Can't add pcap to epoll.\n"); + exit(1); + } + fprintf(stderr, "Device: %s\n", device); } @@ -112,11 +138,11 @@ void sigint_handler(int signo) { term = 1; } -void bpfcountd_init(bpfcountd_ctx *ctx, int argc, char *argv[]) { +void bpfcountd_init(bpfcountd_ctx *ctx, int argc, char *argv[], int epoll_fd) { prepare_config(&ctx->config, argc, argv); // get a pcap handle - prepare_pcap(ctx, ctx->config.device); + prepare_pcap(ctx, ctx->config.device, epoll_fd); // initialize the filter unit filters_init(&ctx->filters_ctx, ctx->pcap_ctx); @@ -128,16 +154,24 @@ void bpfcountd_init(bpfcountd_ctx *ctx, int argc, char *argv[]) { } void bpfcountd_finish(bpfcountd_ctx *ctx) { + close(ctx->fd); pcap_close(ctx->pcap_ctx); filters_finish(&ctx->filters_ctx); } int main(int argc, char *argv[]) { + int epoll_fd = epoll_create1(0); + + if (epoll_fd < 0) { + fprintf(stderr, "Can't create epoll file descriptor.\n"); + exit(1); + } + bpfcountd_ctx ctx = {}; - bpfcountd_init(&ctx, argc, argv); + bpfcountd_init(&ctx, argc, argv, epoll_fd); - int usock = usock_prepare(ctx.config.usock_path); + int usock = usock_prepare(ctx.config.usock_path, epoll_fd); int usock_client; int result = 0; @@ -147,30 +181,39 @@ int main(int argc, char *argv[]) { // TODO: find out if the method drops packets while(!term) { - int res = pcap_dispatch(ctx.pcap_ctx, 100, callback, (u_char *) &ctx); - if (res == -1) { - printf("ERROR: %s\n", pcap_geterr(ctx.pcap_ctx)); - result = 1; - break; - } - - if((usock_client = usock_accept(usock)) != -1) { - - list_foreach(ctx.filters_ctx.filters, f) { - struct filter *tmp = list_data(f, struct filter); - char buf[1067]; - memset(buf, 0x00, sizeof(buf)); - - snprintf(buf, 1067, "%s:%llu:%llu\n", tmp->id, tmp->bytes_count, tmp->packets_count); - usock_sendstr(usock_client, buf); + int ev_count = epoll_wait(epoll_fd, events, MAX_EVENTS, -1); + + for(int i = 0; i < ev_count; i++) { + pcap_t *pcap_ctx = events[i].data.ptr; + + if (pcap_ctx) { + int res = pcap_dispatch(pcap_ctx, 100, callback, (u_char *) &ctx); + if (res == -1) { + printf("ERROR: %s\n", pcap_geterr(pcap_ctx)); + result = 1; + break; + } + // pcap_ctx == NULL indicates unix socket event + } else if ((usock_client = usock_accept(usock)) != -1) { + list_foreach(ctx.filters_ctx.filters, f) { + struct filter *tmp = list_data(f, struct filter); + char buf[1067]; + memset(buf, 0x00, sizeof(buf)); + + snprintf(buf, 1067, "%s:%llu:%llu\n", tmp->id, tmp->bytes_count, tmp->packets_count); + usock_sendstr(usock_client, buf); + } + + usock_finish(usock_client); } - - usock_finish(usock_client); } + } usock_finish(usock); unlink(ctx.config.usock_path); bpfcountd_finish(&ctx); + close(epoll_fd); + return result; } diff --git a/usock.c b/usock.c index 52eddd1..fe556fb 100644 --- a/usock.c +++ b/usock.c @@ -2,14 +2,16 @@ #include #include #include +#include // for epoll_ctl(), struct epoll_event #include #include "usock.h" -int usock_prepare(const char *path) { +int usock_prepare(const char *path, const int epoll_fd) { int sock; struct sockaddr_un local; + struct epoll_event event; if ((sock = socket(AF_UNIX, SOCK_STREAM | SOCK_NONBLOCK, 0)) == -1) { perror("socket"); @@ -35,6 +37,15 @@ int usock_prepare(const char *path) { exit(1); } + memset(&event, 0, sizeof(event)); + event.events = EPOLLIN; + event.data.ptr = NULL; + + if (epoll_ctl(epoll_fd, EPOLL_CTL_ADD, sock, &event)) { + fprintf(stderr, "Can't add pcap to epoll.\n"); + exit(1); + } + return sock; } diff --git a/usock.h b/usock.h index c7a1a30..a0019b3 100644 --- a/usock.h +++ b/usock.h @@ -4,7 +4,7 @@ #include #include -int usock_prepare(const char* path); +int usock_prepare(const char* path, const int epoll_fd); int usock_accept(int sock); void usock_finish(int sock); void usock_sendstr(int client_sock, const char* str); From 80b455e7ea978fc2c9d6a4b533092f5106562fc8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Linus=20L=C3=BCssing?= Date: Mon, 21 Dec 2020 07:58:00 +0100 Subject: [PATCH 4/5] WIP / DONT MERGE: Switch from userspace to kernelspace BPF filtering To increase performance, use pcap_setfilter() instead of pcap_offline_filter(): With the latter all packets are copied to userspace, to bpfcountd, and the filtering is then done within bpfcountd. The former avoids this and applies filtering in kernelspace already. Which then, together with a snaplen of zero, allows to avoid copying the actual payload packet. Furthermore, any traffic which does not match any filter rule of bpfcountd should now be affected less performance wise as it does not create any event to bpfcountd at all. +++ TODO/WIP: +++ The latter actually does not seem to be the case in all situations. This only seems to be true when using very few filter rules. In my tests on an Intel i7 laptop with a veth interface pair and iperf3 my throughput results were affected by kernelspace filtering bpfcountd with n 'arp' filter rules in the following way: 1 rule: -2.8%, 5 rules: -4.7%, 50 rules: -37.9%, 500 rules: -92.7% While for userspace filtering I had about a constant 43% less throughput until bpfcountd reached 100% CPU usage (with each added rule CPU usage of bpfcountd increases). Then throughput actually got faster again, probably due to packets dropped in libpcap. For kernelspace filtering, bpfcountd slumbered at 0% CPU usage, as expected. --- filters.c | 111 ++++++++++++++++++++++++++++++++++++++++++++---------- filters.h | 12 +++--- main.c | 86 +++++++++++------------------------------- 3 files changed, 120 insertions(+), 89 deletions(-) diff --git a/filters.c b/filters.c index 2cfb74d..1e51258 100644 --- a/filters.c +++ b/filters.c @@ -2,13 +2,14 @@ #include #include +#include // for epoll_ctl(), struct epoll_event +#include #include "util.h" -void filters_init(filters_ctx *ctx, pcap_t* pcap_ctx) { +void filters_init(filters_ctx *ctx) { ctx->filters = list_new(); - ctx->pcap_ctx = pcap_ctx; } void filters_finish(filters_ctx *ctx) { @@ -17,8 +18,8 @@ void filters_finish(filters_ctx *ctx) { list_foreach(ctx->filters, f) { struct filter *tmp = list_data(f, struct filter); - pcap_freecode(tmp->bpf); - free(tmp->bpf); + close(tmp->fd); + pcap_close(tmp->pcap_ctx); free(tmp); } @@ -43,27 +44,93 @@ static void filters_bpfstr_unwind_finish(filters_ctx *ctx) free(list_data(f, struct filter)->bpf_str); } -void filters_add(filters_ctx *ctx, const char *id, char *bpf_str) { +static void filters_prepare_pcap(struct filter *filter, const char* device, int epoll_fd) { + struct bpf_program bpf; + struct epoll_event event; + char errbuf[PCAP_ERRBUF_SIZE]; + + memset(&errbuf, 0, sizeof(errbuf)); + + filter->pcap_ctx = pcap_create(device, errbuf); + // TODO: there could be a warning in errbuf even if handle != NULL + if (!filter->pcap_ctx) { + fprintf(stderr, "Couldn't open device %s\n", errbuf); + exit(1); + } + + if (pcap_set_snaplen(filter->pcap_ctx, 0)) { + fprintf(stderr, "Error at setting zero snaplen\n"); + exit(1); + } + + if (pcap_set_buffer_size(filter->pcap_ctx, BUFSIZ)) { + fprintf(stderr, "Error at setting pcap buffer size\n"); + exit(1); + } + + if (pcap_set_timeout(filter->pcap_ctx, 1000)) { + fprintf(stderr, "Error at setting pcap timeout\n"); + exit(1); + } + + if (pcap_setnonblock(filter->pcap_ctx, 1, errbuf) == PCAP_ERROR) { + fprintf(stderr, "Can't set pcap handler nonblocking.\n"); + exit(1); + } + + if (pcap_activate(filter->pcap_ctx)) { + fprintf(stderr, "Can't activate pcap handler.\n"); + exit(1); + } + + if (pcap_compile(filter->pcap_ctx, &bpf, filter->bpf_str, 0, PCAP_NETMASK_UNKNOWN) == -1) { + fprintf(stderr, "Error at compiling bpf \"%s\": %s\n", filter->bpf_str, pcap_geterr(filter->pcap_ctx)); + exit(1); + } + + if (pcap_setfilter(filter->pcap_ctx, &bpf)) { + pcap_freecode(&bpf); + fprintf(stderr, "Can't set pcap filter\n"); + exit(1); + } + + pcap_freecode(&bpf); + + filter->fd = pcap_get_selectable_fd(filter->pcap_ctx); + if (filter->fd == PCAP_ERROR) { + fprintf(stderr, "Can't get file descriptor from pcap handler."); + exit(1); + } + + memset(&event, 0, sizeof(event)); + event.events = EPOLLIN; + event.data.ptr = filter; + + if (epoll_ctl(epoll_fd, EPOLL_CTL_ADD, filter->fd, &event)) { + fprintf(stderr, "Can't add pcap to epoll.\n"); + exit(1); + } +} + + + +void filters_add(filters_ctx *ctx, const char *id, char *bpf_str, const char* device, int epoll_fd) { struct filter *instance = malloc(sizeof(*instance)); // TODO: document maximum len of id strncpy(instance->id, id, 1023); instance->id[1023] = '\0'; instance->bpf_str = bpf_str; - instance->bpf = malloc(sizeof(struct bpf_program)); instance->packets_count = 0; instance->bytes_count = 0; - if (pcap_compile(ctx->pcap_ctx, instance->bpf, bpf_str, 0, PCAP_NETMASK_UNKNOWN) == -1) { - fprintf(stderr, "Error at compiling bpf \"%s\": %s\n", bpf_str, pcap_geterr(ctx->pcap_ctx)); - exit(1); - } + filters_prepare_pcap(instance, device, epoll_fd); list_insert(ctx->filters, instance); } -void filters_load(filters_ctx *ctx, const char *filterfile_path, const char *mac_addr) { +void filters_load(filters_ctx *ctx, const char *filterfile_path, const char *mac_addr, const char* device, int epoll_fd) { // TODO: test with /dev/random as input FILE *fp = fopen(filterfile_path, "r"); @@ -71,6 +138,8 @@ void filters_load(filters_ctx *ctx, const char *filterfile_path, const char *mac size_t read = 0; int line_no = 0; + fprintf(stderr, "Device: %s\n", device); + if (fp == NULL) { fprintf(stderr, "Error while opening the filterfile '%s': ", filterfile_path); perror(""); @@ -104,7 +173,7 @@ void filters_load(filters_ctx *ctx, const char *filterfile_path, const char *mac fprintf(stderr, "id: %s; bpf: \"%s\";\n", id, bpf); - filters_add(ctx, id, bpf); + filters_add(ctx, id, bpf, device, epoll_fd); } filters_bpfstr_unwind_finish(ctx); @@ -112,13 +181,17 @@ void filters_load(filters_ctx *ctx, const char *filterfile_path, const char *mac fclose(fp); } -void filters_process(filters_ctx *ctx, const struct pcap_pkthdr *pkthdr, const u_char *packet) { - list_foreach(ctx->filters, f) { - struct filter *tmp = list_data(f, struct filter); +void filters_process(u_char *ptr, const struct pcap_pkthdr *pkthdr, const u_char *packet) { + struct filter *filter = (struct filter *)ptr; - if (pcap_offline_filter(tmp->bpf, pkthdr, packet)) { - tmp->packets_count += 1; - tmp->bytes_count += pkthdr->len; - } + filter->packets_count += 1; + filter->bytes_count += pkthdr->len; +} + +void filters_break(filters_ctx *filters_ctx) { + list_foreach(filters_ctx->filters, f) { + struct filter *filter = list_data(f, struct filter); + + pcap_breakloop(filter->pcap_ctx); } } diff --git a/filters.h b/filters.h index 366546a..6bc6cde 100644 --- a/filters.h +++ b/filters.h @@ -8,9 +8,10 @@ struct filter { char id[1024]; char *bpf_str; - struct bpf_program *bpf; unsigned long long packets_count; unsigned long long bytes_count; + pcap_t *pcap_ctx; + int fd; }; typedef struct { @@ -18,11 +19,12 @@ typedef struct { struct list *filters; } filters_ctx; -void filters_init(filters_ctx *ctx, pcap_t* pcap_ctx); +void filters_init(filters_ctx *ctx); void filters_finish(filters_ctx *ctx); -void filters_add(filters_ctx *ctx, const char *id, char *bpf_str); -void filters_load(filters_ctx *ctx, const char *filterfile_path, const char* mac_addr); +void filters_add(filters_ctx *ctx, const char *id, char *bpf_str, const char* device, int epoll_fd); +void filters_load(filters_ctx *ctx, const char *filterfile_path, const char* mac_addr, const char* device, int epoll_fd); -void filters_process(filters_ctx *ctx, const struct pcap_pkthdr *pkthdr, const u_char *packet); +void filters_process(u_char *ptr, const struct pcap_pkthdr *pkthdr, const u_char *packet); +void filters_break(filters_ctx *filters_ctx); #endif diff --git a/main.c b/main.c index 8253d5c..97b7234 100644 --- a/main.c +++ b/main.c @@ -24,47 +24,9 @@ struct config { typedef struct { struct config config; filters_ctx filters_ctx; - - pcap_t *pcap_ctx; - int fd; } bpfcountd_ctx; - -void prepare_pcap(bpfcountd_ctx *ctx, const char* device, int epoll_fd) { - struct epoll_event event; - char errbuf[PCAP_ERRBUF_SIZE]; - - memset(&errbuf, 0, sizeof(errbuf)); - - ctx->pcap_ctx = pcap_open_live(device, BUFSIZ, 1, 1000, errbuf); - // TODO: there could be a warning in errbuf even if handle != NULL - if (!ctx->pcap_ctx) { - fprintf(stderr, "Couldn't open device %s\n", errbuf); - exit(1); - } - - if (pcap_setnonblock(ctx->pcap_ctx, 1, errbuf) == PCAP_ERROR) { - fprintf(stderr, "Can't set pcap handler nonblocking.\n"); - exit(1); - } - - ctx->fd = pcap_get_selectable_fd(ctx->pcap_ctx); - if (ctx->fd == PCAP_ERROR) { - fprintf(stderr, "Can't get file descriptor from pcap handler."); - exit(1); - } - - memset(&event, 0, sizeof(event)); - event.events = EPOLLIN; - event.data.ptr = ctx->pcap_ctx; - - if (epoll_ctl(epoll_fd, EPOLL_CTL_ADD, ctx->fd, &event)) { - fprintf(stderr, "Can't add pcap to epoll.\n"); - exit(1); - } - - fprintf(stderr, "Device: %s\n", device); -} +bpfcountd_ctx ctx; void help(const char* path) { fprintf(stderr, "%s -i -f [-u ] [-h]\n\n", path); @@ -126,39 +88,27 @@ void prepare_config(struct config *cfg, int argc, char *argv[]){ } -void callback(u_char *ptr, const struct pcap_pkthdr *pkthdr, const u_char *packet) -{ - bpfcountd_ctx *ctx = (bpfcountd_ctx *) ptr; - filters_process(&ctx->filters_ctx, pkthdr, packet); -} - int term = 0; void sigint_handler(int signo) { term = 1; + filters_break(&ctx.filters_ctx); } void bpfcountd_init(bpfcountd_ctx *ctx, int argc, char *argv[], int epoll_fd) { prepare_config(&ctx->config, argc, argv); - // get a pcap handle - prepare_pcap(ctx, ctx->config.device, epoll_fd); - // initialize the filter unit - filters_init(&ctx->filters_ctx, ctx->pcap_ctx); + filters_init(&ctx->filters_ctx); filters_load( &ctx->filters_ctx, ctx->config.filters_path, - ctx->config.mac_addr + ctx->config.mac_addr, + ctx->config.device, + epoll_fd ); } -void bpfcountd_finish(bpfcountd_ctx *ctx) { - close(ctx->fd); - pcap_close(ctx->pcap_ctx); - filters_finish(&ctx->filters_ctx); -} - int main(int argc, char *argv[]) { int epoll_fd = epoll_create1(0); @@ -167,8 +117,6 @@ int main(int argc, char *argv[]) { exit(1); } - bpfcountd_ctx ctx = {}; - bpfcountd_init(&ctx, argc, argv, epoll_fd); int usock = usock_prepare(ctx.config.usock_path, epoll_fd); @@ -183,17 +131,23 @@ int main(int argc, char *argv[]) { while(!term) { int ev_count = epoll_wait(epoll_fd, events, MAX_EVENTS, -1); + if (term) + break; + for(int i = 0; i < ev_count; i++) { - pcap_t *pcap_ctx = events[i].data.ptr; + struct filter *filter = events[i].data.ptr; - if (pcap_ctx) { - int res = pcap_dispatch(pcap_ctx, 100, callback, (u_char *) &ctx); - if (res == -1) { - printf("ERROR: %s\n", pcap_geterr(pcap_ctx)); + if (filter) { + int res = pcap_dispatch(filter->pcap_ctx, 100, filters_process, (u_char *) filter); + switch (res) { + case PCAP_ERROR: + printf("ERROR: %s\n", pcap_geterr(filter->pcap_ctx)); result = 1; + /* fall through */ + case PCAP_ERROR_BREAK: break; } - // pcap_ctx == NULL indicates unix socket event + // filter == NULL indicates unix socket event } else if ((usock_client = usock_accept(usock)) != -1) { list_foreach(ctx.filters_ctx.filters, f) { struct filter *tmp = list_data(f, struct filter); @@ -210,10 +164,12 @@ int main(int argc, char *argv[]) { } + fprintf(stderr, "Shutting down...\n"); usock_finish(usock); unlink(ctx.config.usock_path); - bpfcountd_finish(&ctx); + filters_finish(&ctx.filters_ctx); close(epoll_fd); + fprintf(stderr, "Shutdown finished, bye\n"); return result; } From 477605484295c4ceb099384ee53e0628a9f6d030 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Linus=20L=C3=BCssing?= Date: Tue, 22 Dec 2020 05:10:33 +0100 Subject: [PATCH 5/5] Speedup shutdown via pthreads pcap_close() is rather slow, it needs about 100 millseconds. When using many rules and calling pcap_close() for each of those then this accumulates quickly to a noticeable delay on shutdown. Parallelising the pcap_close() calls via pthreads helps. --- Makefile | 2 +- filters.c | 37 +++++++++++++++++++++++++++++++++---- filters.h | 1 + 3 files changed, 35 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index fad2dd1..e282654 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ CC?=gcc CFLAGS+=-Wall -Werror -ggdb -LDFLAGS?=-lpcap +LDFLAGS?=-lpcap -lpthread NAME=bpfcountd PREFIX?=/usr/local diff --git a/filters.c b/filters.c index 1e51258..2462541 100644 --- a/filters.c +++ b/filters.c @@ -4,25 +4,53 @@ #include #include // for epoll_ctl(), struct epoll_event #include +#include #include "util.h" void filters_init(filters_ctx *ctx) { ctx->filters = list_new(); + ctx->count = 0; +} + +void *filters_finish_thread(void *args) { + struct filter *filter = args; + + close(filter->fd); + pcap_close(filter->pcap_ctx); + filter->pcap_ctx = NULL; + + return NULL; } void filters_finish(filters_ctx *ctx) { + pthread_t *id = calloc(ctx->count, sizeof(*id)); + int i = 0, j = 0; + + // parallelize shutdown, pcap_close() is slow unfortunately... + if (id) { + list_foreach(ctx->filters, f) { + struct filter *tmp = list_data(f, struct filter); + + pthread_create(&id[i++], NULL, &filters_finish_thread, tmp); + } + } // free the filters - list_foreach(ctx->filters, f) { - struct filter *tmp = list_data(f, struct filter); + list_foreach(ctx->filters, f2) { + struct filter *tmp = list_data(f2, struct filter); + + pthread_join(id[j++], NULL); + + // thread creation might have failed + if (tmp->pcap_ctx) + filters_finish_thread(tmp); - close(tmp->fd); - pcap_close(tmp->pcap_ctx); free(tmp); } + free(id); list_free(ctx->filters); } @@ -128,6 +156,7 @@ void filters_add(filters_ctx *ctx, const char *id, char *bpf_str, const char* de filters_prepare_pcap(instance, device, epoll_fd); list_insert(ctx->filters, instance); + ctx->count++; } void filters_load(filters_ctx *ctx, const char *filterfile_path, const char *mac_addr, const char* device, int epoll_fd) { diff --git a/filters.h b/filters.h index 6bc6cde..140149c 100644 --- a/filters.h +++ b/filters.h @@ -17,6 +17,7 @@ struct filter { typedef struct { pcap_t *pcap_ctx; struct list *filters; + int count; } filters_ctx; void filters_init(filters_ctx *ctx);