From 627f5cbdb21e6c00785c092e9e59c8cde634b14b Mon Sep 17 00:00:00 2001 From: Sergey Yakovlev <3gnees@gmail.com> Date: Tue, 26 Aug 2025 21:50:18 +0300 Subject: [PATCH 1/3] bench with dpdk --- common/memory.h | 14 +- common/registry.h | 6 +- common/value.h | 3 +- dataplane/unittest/config/main.c | 11 +- filter/attribute/net4.h | 166 +++++-------------- filter/attribute/port.h | 18 +- filter/attribute/proto.h | 8 +- filter/bench/dpdk.c | 102 ++++++++++++ filter/bench/dpdk/def.h | 100 +++++++++++ filter/bench/dpdk/gen.c | 39 +++++ filter/bench/dpdk/gen.h | 8 + filter/bench/dpdk/main.c | 144 ++++++++++++++++ filter/bench/dpdk/util.c | 108 ++++++++++++ filter/bench/dpdk/util.h | 40 +++++ filter/bench/meson.build | 9 + filter/filter.h | 3 + filter/helper.c | 135 +++++++++++++-- filter/helper.h | 13 ++ filter/meson.build | 1 + filter/tests/basic_proto.c | 8 +- filter/tests/basic_vlan.c | 1 + filter/tests/ipv4_tuple.c | 159 ++++++++++++++++++ filter/tests/meson.build | 20 +++ filter/tests/trie.c | 105 ++++++++++++ filter/tests/vs_prev.c | 3 + filter/trie.c | 275 +++++++++++++++++++++++++++++++ filter/trie.h | 61 +++++++ filter/utils/utils.c | 2 +- filter/utils/utils.h | 4 +- tests/common/radix_test.c | 3 + 30 files changed, 1401 insertions(+), 168 deletions(-) create mode 100644 filter/bench/dpdk.c create mode 100644 filter/bench/dpdk/def.h create mode 100644 filter/bench/dpdk/gen.c create mode 100644 filter/bench/dpdk/gen.h create mode 100644 filter/bench/dpdk/main.c create mode 100644 filter/bench/dpdk/util.c create mode 100644 filter/bench/dpdk/util.h create mode 100644 filter/tests/ipv4_tuple.c create mode 100644 filter/tests/trie.c create mode 100644 filter/trie.c create mode 100644 filter/trie.h diff --git a/common/memory.h b/common/memory.h index b4598f579..b72993b9c 100644 --- a/common/memory.h +++ b/common/memory.h @@ -2,6 +2,7 @@ #include #include +#include #include #include "memory_address.h" @@ -98,12 +99,15 @@ memory_brealloc( void *new_data = memory_balloc(context, new_size); if (new_data == NULL) return NULL; - if (old_size < new_size) - memcpy(new_data, data, old_size); - else - memcpy(new_data, data, new_size); - if (old_size) + if (old_size) { + if (old_size < new_size) + memcpy(new_data, data, old_size); + else + memcpy(new_data, data, new_size); + memory_bfree(context, data, old_size); + } + return new_data; } diff --git a/common/registry.h b/common/registry.h index 2907ac687..fb24c6f23 100644 --- a/common/registry.h +++ b/common/registry.h @@ -2,7 +2,6 @@ #include #include -#include #include #include "common/exp_array.h" @@ -292,7 +291,10 @@ value_registry_join_range( uint32_t v1 = ADDR_OF(®istry1->values)[idx1]; uint32_t v2 = ADDR_OF(®istry2->values)[idx2]; - join_func(v1, v2, range_idx, join_func_data); + int ret = join_func(v1, v2, range_idx, join_func_data); + if (ret < 0) { + return ret; + } } } return 0; diff --git a/common/value.h b/common/value.h index 1a6002306..6ffdd241c 100644 --- a/common/value.h +++ b/common/value.h @@ -37,13 +37,14 @@ value_table_init( uint32_t *values = (uint32_t *)memory_balloc( memory_context, h_dim * v_dim * sizeof(uint32_t) ); - memset(values, 0, h_dim * v_dim * sizeof(uint32_t)); if (values == NULL) { remap_table_free(&value_table->remap_table); return -1; } + memset(values, 0, h_dim * v_dim * sizeof(uint32_t)); + SET_OFFSET_OF(&value_table->values, values); value_table->h_dim = h_dim; diff --git a/dataplane/unittest/config/main.c b/dataplane/unittest/config/main.c index f04ceddf4..d3536b4e2 100644 --- a/dataplane/unittest/config/main.c +++ b/dataplane/unittest/config/main.c @@ -18,10 +18,7 @@ check_instance( } int -main(int argc, char **argv) { - (void)argc; - (void)argv; - +main() { FILE *dataplane_config_file = fopen(CONFIG_PATH, "r"); struct dataplane_config *config = NULL; @@ -33,4 +30,10 @@ main(int argc, char **argv) { check_instance(config->instances, 0, 1024, 2048); check_instance(config->instances + 1, 1, 512, 128); check_instance(config->instances + 2, 0, 123, 124); + + dataplane_config_free(config); + + puts("OK"); + + return 0; } \ No newline at end of file diff --git a/filter/attribute/net4.h b/filter/attribute/net4.h index 2963d2a93..c5a05f043 100644 --- a/filter/attribute/net4.h +++ b/filter/attribute/net4.h @@ -1,10 +1,8 @@ #include "../helper.h" #include "../rule.h" -#include "common/lpm.h" -#include "common/range_collector.h" #include "dataplane/packet/packet.h" -#include "util.h" +#include "filter/trie.h" #include #include @@ -31,137 +29,54 @@ action_get_net4_dst( *count = action->net4.dst_count; } -static inline void -net4_collect_values( - struct net4 *start, - uint32_t count, - struct lpm *lpm, - struct value_table *table -) { - for (struct net4 *net4 = start; net4 < start + count; ++net4) { - uint32_t addr = htobe32(net4->addr); - uint32_t mask = htobe32(net4->mask); - uint32_t to = addr | ~mask; - lpm4_collect_values( - lpm, - (uint8_t *)&addr, - (uint8_t *)&to, - lpm_collect_value_iterator, - table - ); - } -} - -static inline void -net4_collect_registry( - struct net4 *start, - uint32_t count, - struct lpm *lpm, - struct value_registry *registry -) { - for (struct net4 *net4 = start; net4 < start + count; ++net4) { - uint32_t addr = htobe32(net4->addr); - uint32_t mask = htobe32(net4->mask); - uint32_t to = addr | ~mask; - lpm4_collect_values( - lpm, - (uint8_t *)&addr, - (uint8_t *)&to, - lpm_collect_registry_iterator, - registry - ); - } -} +//////////////////////////////////////////////////////////////////////////////// static inline int collect_net4_values( struct memory_context *memory_context, - const struct filter_rule *actions, + const struct filter_rule *rules, uint32_t count, rule_get_net4_func get_net4, - struct lpm *lpm, + struct trie *trie, struct value_registry *registry ) { - struct range_collector collector; - if (range_collector_init(&collector, memory_context)) - goto error; + int ret = trie_init(trie, memory_context); + if (ret < 0) { + return -1; + } - for (const struct filter_rule *action = actions; - action < actions + count; - ++action) { - if (action->net4.src_count == 0 && - action->net4.dst_count == 0) { + for (const struct filter_rule *rule = rules; rule < rules + count; + ++rule) { + if (rule->net4.src_count == 0 && rule->net4.dst_count == 0) { continue; } struct net4 *nets; uint32_t net_count; - get_net4(action, &nets, &net_count); + get_net4(rule, &nets, &net_count); for (struct net4 *net4 = nets; net4 < nets + net_count; ++net4) { uint32_t addr = htobe32(net4->addr); - if (range4_collector_add( - &collector, - (uint8_t *)&addr, - __builtin_popcountll(net4->mask) - )) - goto error_collector; + ret = trie_add( + trie, + (uint8_t *)&addr, + __builtin_popcountll(net4->mask), + rule - rules + ); + if (ret < 0) { + return -1; + } } } - if (lpm_init(lpm, memory_context)) { - goto error_lpm; - } - if (range_collector_collect(&collector, 4, lpm)) { - goto error_collector; - } - - struct value_table table; - if (value_table_init(&table, memory_context, 1, collector.count)) - goto error_vtab; - - for (const struct filter_rule *action = actions; - action < actions + count; - ++action) { - - value_table_new_gen(&table); - - struct net4 *nets; - uint32_t net_count; - get_net4(action, &nets, &net_count); - - net4_collect_values(nets, net_count, lpm, &table); - } - - value_table_compact(&table); - lpm4_remap(lpm, &table); - lpm4_compact(lpm); - - for (const struct filter_rule *action = actions; - action < actions + count; - ++action) { - value_registry_start(registry); - struct net4 *nets; - uint32_t net_count; - get_net4(action, &nets, &net_count); + // build trie classifiers + trie_build_classifiers(trie); - net4_collect_registry(nets, net_count, lpm, registry); - } - - value_table_free(&table); - return 0; - -error_collector: - range_collector_free(&collector, 4); -error_lpm: - lpm_free(lpm); - -error_vtab: - -error: - return -1; + return fill_rule_registry_by_trie( + trie, count, registry, memory_context + ); } //////////////////////////////////////////////////////////////////////////////// @@ -175,14 +90,14 @@ init_net4_src( size_t actions_count, struct memory_context *memory_context ) { - struct lpm *lpm = memory_balloc(memory_context, sizeof(struct lpm)); - *data = lpm; + struct trie *trie = memory_balloc(memory_context, sizeof(struct trie)); + *data = trie; return collect_net4_values( memory_context, actions, actions_count, action_get_net4_src, - lpm, + trie, registry ); } @@ -191,11 +106,14 @@ init_net4_src( static inline uint32_t lookup_net4_src(struct packet *packet, void *data) { struct rte_mbuf *mbuf = packet_to_mbuf(packet); + struct rte_ipv4_hdr *ipv4_hdr = rte_pktmbuf_mtod_offset( mbuf, struct rte_ipv4_hdr *, packet->network_header.offset ); - struct lpm *lpm = (struct lpm *)data; - return lpm4_lookup(lpm, (uint8_t *)&ipv4_hdr->src_addr); + + struct trie *trie = (struct trie *)data; + uint32_t res = trie_classify(trie, (uint8_t *)&ipv4_hdr->src_addr, 4); + return res; } // Allows to initialize attribute for IPv4 destination address. @@ -207,14 +125,14 @@ init_net4_dst( size_t actions_count, struct memory_context *memory_context ) { - struct lpm *lpm = memory_balloc(memory_context, sizeof(struct lpm)); - *data = lpm; + struct trie *trie = memory_balloc(memory_context, sizeof(struct trie)); + *data = trie; return collect_net4_values( memory_context, actions, actions_count, action_get_net4_dst, - lpm, + trie, registry ); } @@ -228,15 +146,15 @@ lookup_net4_dst(struct packet *packet, void *data) { mbuf, struct rte_ipv4_hdr *, packet->network_header.offset ); - struct lpm *lpm = (struct lpm *)data; - - return lpm4_lookup(lpm, (uint8_t *)&ipv4_hdr->dst_addr); + struct trie *trie = (struct trie *)data; + uint32_t res = trie_classify(trie, (uint8_t *)&ipv4_hdr->dst_addr, 4); + return res; } // Allows to free data for IPv4 classification. static inline void free_net4(void *data, struct memory_context *memory_context) { (void)memory_context; - struct lpm *lpm = (struct lpm *)data; - lpm_free(lpm); + struct trie *trie = (struct trie *)data; + trie_free_mem(trie); } \ No newline at end of file diff --git a/filter/attribute/port.h b/filter/attribute/port.h index d581d9b4a..2e026886b 100644 --- a/filter/attribute/port.h +++ b/filter/attribute/port.h @@ -59,7 +59,8 @@ static inline uint32_t lookup_port_src(struct packet *packet, void *data) { (void)packet; struct value_table *table = data; - return value_table_get(table, 0, packet_src_port(packet)); + uint32_t res = value_table_get(table, 0, packet_src_port(packet)); + return res; } typedef void (*action_get_port_range_func)( @@ -71,7 +72,7 @@ typedef void (*action_get_port_range_func)( static inline int collect_port_values( struct memory_context *memory_context, - const struct filter_rule *actions, + const struct filter_rule *rules, uint32_t count, action_get_port_range_func get_port_range, struct value_table *table, @@ -80,8 +81,7 @@ collect_port_values( if (value_table_init(table, memory_context, 1, 65536)) return -1; - for (const struct filter_rule *action = actions; - action < actions + count; + for (const struct filter_rule *action = rules; action < rules + count; ++action) { value_table_new_gen(table); @@ -103,14 +103,13 @@ collect_port_values( value_table_compact(table); - for (const struct filter_rule *action = actions; - action < actions + count; - ++action) { + for (const struct filter_rule *rule = rules; rule < rules + count; + ++rule) { value_registry_start(registry); struct filter_port_range *port_ranges; uint32_t port_range_count; - get_port_range(action, &port_ranges, &port_range_count); + get_port_range(rule, &port_ranges, &port_range_count); for (struct filter_port_range *ports = port_ranges; ports < port_ranges + port_range_count; ++ports) { @@ -150,7 +149,8 @@ get_port_range_dst( static inline uint32_t lookup_port_dst(struct packet *packet, void *data) { struct value_table *table = data; - return value_table_get(table, 0, packet_dst_port(packet)); + uint32_t res = value_table_get(table, 0, packet_dst_port(packet)); + return res; } static inline int diff --git a/filter/attribute/proto.h b/filter/attribute/proto.h index 10aee6373..4a5a8b5d7 100644 --- a/filter/attribute/proto.h +++ b/filter/attribute/proto.h @@ -140,15 +140,17 @@ lookup_proto(struct packet *packet, void *data) { rte_pktmbuf_mtod(packet->mbuf, struct rte_ether_hdr *); struct rte_ipv4_hdr *ip_hdr = (struct rte_ipv4_hdr *)(eth_hdr + 1); + uint32_t class; if (ip_hdr->next_proto_id == IPPROTO_UDP) { - return c->max_tcp_class + 1; + class = c->max_tcp_class + 1; } else if (ip_hdr->next_proto_id == IPPROTO_ICMP) { - return c->max_tcp_class + 2; + class = c->max_tcp_class + 2; } else { // TCP struct rte_tcp_hdr *tcp_hdr = (struct rte_tcp_hdr *)(ip_hdr + 1); - return value_table_get(&c->tcp_flags, 0, tcp_hdr->tcp_flags); + class = value_table_get(&c->tcp_flags, 0, tcp_hdr->tcp_flags); } + return class; } static inline void diff --git a/filter/bench/dpdk.c b/filter/bench/dpdk.c new file mode 100644 index 000000000..c0cd5d17c --- /dev/null +++ b/filter/bench/dpdk.c @@ -0,0 +1,102 @@ +#include +#include +#include + +#include +#include + +//////////////////////////////////////////////////////////////////////////////// + +struct ipv4_5tuple { + uint8_t proto; + uint32_t src_ip; + uint32_t dst_ip; + uint16_t src_port; + uint16_t dst_port; +}; + +//////////////////////////////////////////////////////////////////////////////// + +struct rte_acl_field_def defs[1] = { + { + .type = RTE_ACL_FIELD_TYPE_BITMASK, + .size = sizeof(uint8_t), + .field_index = 0, + .input_index = 0, + .offset = offsetof(struct ipv4_5tuple, proto), + }, +}; + +RTE_ACL_RULE_DEF(acl_ipv4_rule, RTE_DIM(defs)); + +struct rte_acl_param dpdk_acl_params = { + .name = "ACL_example", + .socket_id = SOCKET_ID_ANY, + .rule_size = RTE_ACL_RULE_SZ(RTE_DIM(defs)), + + /* number of fields per rule. */ + .max_rule_num = 8, /* maximum number of rules in the AC context. */ +}; + +int +main(int argc, char **argv) { + int ret = rte_eal_init(argc, argv); + if (ret < 0) { + printf("ret=%d\n", ret); + rte_panic("Cannot init EAL\n"); + } + + const struct acl_ipv4_rule acl_rules[] = { + {.data = {.userdata = 1, .category_mask = 3, .priority = 1}, + + .field[0] = {.value.u8 = 15, .mask_range.u8 = 0b1010}} + }; + + struct rte_acl_ctx *acx; + struct rte_acl_config cfg; + + /* create an empty AC context */ + if ((acx = rte_acl_create(&dpdk_acl_params)) == NULL) { + printf("ret=%d\n", ret); + rte_panic("Cannot create ACL\n"); + } + + /* add rules to the context */ + ret = rte_acl_add_rules( + acx, (const struct rte_acl_rule *)&acl_rules, RTE_DIM(acl_rules) + ); + if (ret != 0) { + printf("ret=%d\n", ret); + rte_panic("Cannot add ACL rules\n"); + } + + cfg.num_categories = 2; + cfg.num_fields = RTE_DIM(defs); + + memcpy(cfg.defs, defs, sizeof(defs)); + + /* build the runtime structures for added rules, with 2 categories. */ + ret = rte_acl_build(acx, &cfg); + if (ret != 0) { + printf("ret=%d\n", ret); + rte_panic("Cannot build ACL\n"); + } + + uint32_t results[4]; + struct ipv4_5tuple data = {.proto = 15}; + const uint8_t *datas = (const uint8_t *)&data; + ret = rte_acl_classify(acx, &datas, results, 1, 4); + if (ret != 0) { + printf("ret=%d\n", ret); + rte_panic("Cannot classify\n"); + } + + puts("result="); + for (size_t i = 0; i < 4; ++i) { + printf("%u ", results[i]); + } + puts(""); + + puts("OK"); + return 0; +} \ No newline at end of file diff --git a/filter/bench/dpdk/def.h b/filter/bench/dpdk/def.h new file mode 100644 index 000000000..c572a33d7 --- /dev/null +++ b/filter/bench/dpdk/def.h @@ -0,0 +1,100 @@ +#pragma once + +#include +#include +#include + +#include +#include + +#include +#include +#include +#include +#include + +#include "common/network.h" +#include "filter/rule.h" + +//////////////////////////////////////////////////////////////////////////////// + +#define MAX_RULE_NUM 10000 + +//////////////////////////////////////////////////////////////////////////////// + +#define IP_HDR_OFFSET \ + (RTE_PKTMBUF_HEADROOM + sizeof(struct rte_mbuf) + \ + sizeof(struct rte_ether_hdr)) +#define TRANSPORT_HDR_OFFSET (IP_HDR_OFFSET + sizeof(struct rte_ipv4_hdr)) + +static_assert( + IP_HDR_OFFSET == 398, "offset of the IP header calculated incorrect" +); + +//////////////////////////////////////////////////////////////////////////////// + +static const struct rte_acl_field_def dpdk_acl_field_defs[5] = { + { + .type = RTE_ACL_FIELD_TYPE_BITMASK, + .size = sizeof(uint8_t), + .field_index = 0, + .input_index = 0, + .offset = IP_HDR_OFFSET + + offsetof(struct rte_ipv4_hdr, next_proto_id), + }, + {.type = RTE_ACL_FIELD_TYPE_MASK, + .size = sizeof(uint32_t), + .field_index = 1, + .input_index = 1, + .offset = IP_HDR_OFFSET + offsetof(struct rte_ipv4_hdr, src_addr)}, + {.type = RTE_ACL_FIELD_TYPE_MASK, + .size = sizeof(uint32_t), + .field_index = 2, + .input_index = 2, + .offset = IP_HDR_OFFSET + offsetof(struct rte_ipv4_hdr, dst_addr)}, + {.type = RTE_ACL_FIELD_TYPE_RANGE, + .size = sizeof(uint16_t), + .field_index = 3, + .input_index = 3, + + // both udp and tcp headers start with src port following dst port + .offset = TRANSPORT_HDR_OFFSET}, + {.type = RTE_ACL_FIELD_TYPE_RANGE, + .size = sizeof(uint16_t), + .field_index = 4, + .input_index = 4, + .offset = TRANSPORT_HDR_OFFSET + sizeof(uint16_t)} +}; + +//////////////////////////////////////////////////////////////////////////////// + +RTE_ACL_RULE_DEF(dpdk_acl_rule, RTE_DIM(dpdk_acl_field_defs)); + +//////////////////////////////////////////////////////////////////////////////// + +static const struct rte_acl_param dpdk_acl_params = { + .name = "DPDK_ACL", + .socket_id = SOCKET_ID_ANY, + .rule_size = RTE_ACL_RULE_SZ(RTE_DIM(dpdk_acl_field_defs)), + + // maximum number of rules + .max_rule_num = MAX_RULE_NUM, +}; + +//////////////////////////////////////////////////////////////////////////////// + +struct dpdk_acl { + struct dpdk_acl_rule rules[MAX_RULE_NUM]; + struct rte_acl_ctx *ctx; + struct rte_acl_config cfg; +}; + +//////////////////////////////////////////////////////////////////////////////// + +struct filter_rule_holder { + struct net4 nets_src[MAX_RULE_NUM]; + struct net4 nets_dst[MAX_RULE_NUM]; + struct filter_port_range ports_src[MAX_RULE_NUM]; + struct filter_port_range ports_dst[MAX_RULE_NUM]; + struct filter_rule rules[MAX_RULE_NUM]; +}; \ No newline at end of file diff --git a/filter/bench/dpdk/gen.c b/filter/bench/dpdk/gen.c new file mode 100644 index 000000000..11bf138ef --- /dev/null +++ b/filter/bench/dpdk/gen.c @@ -0,0 +1,39 @@ +#include "gen.h" +#include "rule.h" +#include "utils.h" +#include + +//////////////////////////////////////////////////////////////////////////////// + +void +generate_rules(struct filter_rule_holder *holder, uint32_t count) { + for (uint32_t i = 0; i < count; ++i) { + struct filter_rule *rule = &holder->rules[i]; + + holder->nets_src[i].addr = ip(0xff, 0xff, i & 0xff, 0); + holder->nets_src[i].mask = ip(0xff, 0xff, 0xff, 0); + + holder->nets_dst[i].addr = ip(0xff, 0xff, (i + 1) & 0xff, 0); + holder->nets_dst[i].mask = ip(0xff, 0xff, 0xff, 0); + + rule->net4.dst_count = rule->net4.src_count = 1; + rule->net4.srcs = &holder->nets_src[i]; + rule->net4.dsts = &holder->nets_dst[i]; + + holder->ports_src[i] = (struct filter_port_range + ){.from = i & 127, .to = 500 + (i & 255)}; + holder->ports_dst[i] = (struct filter_port_range + ){.from = 100 + (i & 127), .to = 600 + (i & 255)}; + + rule->transport.src_count = rule->transport.dst_count = 1; + rule->transport.srcs = &holder->ports_src[i]; + rule->transport.dsts = &holder->ports_dst[i]; + + rule->transport.proto = (struct filter_proto + ){.proto = IPPROTO_UDP, .enable_bits = 0, .disable_bits = 0}; + + rule->action = i + 1; + } +} + +//////////////////////////////////////////////////////////////////////////////// \ No newline at end of file diff --git a/filter/bench/dpdk/gen.h b/filter/bench/dpdk/gen.h new file mode 100644 index 000000000..8dc3b2c23 --- /dev/null +++ b/filter/bench/dpdk/gen.h @@ -0,0 +1,8 @@ +#include "def.h" + +//////////////////////////////////////////////////////////////////////////////// + +void +generate_rules(struct filter_rule_holder *holder, uint32_t count); + +//////////////////////////////////////////////////////////////////////////////// \ No newline at end of file diff --git a/filter/bench/dpdk/main.c b/filter/bench/dpdk/main.c new file mode 100644 index 000000000..831e27e1a --- /dev/null +++ b/filter/bench/dpdk/main.c @@ -0,0 +1,144 @@ +#include "attribute.h" +#include "def.h" +#include "filter.h" +#include "gen.h" +#include "util.h" +#include "utils.h" +#include +#include +#include + +// #define RULES_COUNT 289 +#define RULES_COUNT 3000 +// #define PACKETS 1000000 +#define PACKETS 10000000 +#define MEMORY ((size_t)1ull << 30) + +int +main(int argc, char **argv) { + (void)argc; + (void)argv; + // Generate rules + struct filter_rule_holder rule_holder; + generate_rules(&rule_holder, RULES_COUNT); + + // Init filter and memory + struct block_allocator allocator; + block_allocator_init(&allocator); + void *memory = malloc(MEMORY); + block_allocator_put_arena(&allocator, memory, MEMORY); + + struct memory_context memory_context; + int ret = memory_context_init(&memory_context, "bench", &allocator); + assert(ret == 0); + + puts("filter init..."); + + FILTER_DECLARE( + sign, + &attribute_proto, + &attribute_net4_src, + &attribute_net4_dst, + &attribute_port_src, + &attribute_port_dst + ); + + struct filter filter; + clock_t tp = clock(); + FILTER_INIT( + &filter, + sign, + rule_holder.rules, + RULES_COUNT, + &memory_context, + &ret + ); + assert(ret == 0); + double filter_init_time = (double)(clock() - tp) / CLOCKS_PER_SEC; + printf("Filter init time: %.2fs\n", filter_init_time); + + puts("dpdk init..."); + + // Init dpdk rte + ret = dpdk_init(argc, argv); + if (ret != 0) { + printf("failed to init dpdk (error code %d)\n", ret); + return 1; + } + + puts("dpdk acl init..."); + + // Init dpdk acl + struct dpdk_acl dpdk_acl; + + tp = clock(); + ret = dpdk_acl_init(&dpdk_acl, rule_holder.rules, RULES_COUNT); + double dpdk_init_time = (double)(clock() - tp) / CLOCKS_PER_SEC; + printf("DPDK ACL init time: %.2fs\n", dpdk_init_time); + + if (ret != 0) { + printf("failed to init dpdk acl (error code %d)\n", ret); + return 1; + } + + struct packet *packets = malloc(PACKETS * sizeof(struct packet)); + for (size_t i = 0; i < PACKETS; ++i) { + packets[i] = make_packet( + ip(0xff, 0xff, i & 0xff, 1), + ip(0xff, 0xff, (i + 1) & 0xff, 1), + i & 127, + (i * 17) & 255, + IPPROTO_UDP, + 0, + 0 + ); + } + + // query dpdk acl + uint32_t h = 0; + tp = clock(); + for (size_t i = 0; i < PACKETS; ++i) { + uint32_t action = dpdk_acl_classify(&dpdk_acl, &packets[i]); + (void)action; +#ifdef DPDK_ACL_STRESS + h ^= action; +#endif + } + double dpdk_classify_time = (double)(clock() - tp) / CLOCKS_PER_SEC; + printf("DPDK ACL classify time: %.2fs\n", dpdk_classify_time); + printf("DPDK ACL perfomance: %.2f MP/s\n", + (double)PACKETS / 1e6 / dpdk_classify_time); + + // query filter + tp = clock(); + uint32_t *actions; + uint32_t count; + + for (size_t i = 0; i < PACKETS; ++i) { + FILTER_QUERY(&filter, sign, &packets[i], &actions, &count); +#ifdef STRESS + if (count > 0) { + assert(count == 1); + h ^= actions[0]; + } +#endif + } + + double filter_classify_time = (double)(clock() - tp) / CLOCKS_PER_SEC; + printf("Filter classify time: %.2fs\n", filter_classify_time); + printf("Filter perfomance: %.2f MP/s\n", + (double)PACKETS / 1e6 / filter_classify_time); + + assert(h == 0); + + for (size_t i = 0; i < PACKETS; ++i) { + free_packet(&packets[i]); + } + free(packets); + + puts("OK"); + + free(memory); + + return 0; +} \ No newline at end of file diff --git a/filter/bench/dpdk/util.c b/filter/bench/dpdk/util.c new file mode 100644 index 000000000..653b5c5fa --- /dev/null +++ b/filter/bench/dpdk/util.c @@ -0,0 +1,108 @@ +#include "util.h" +#include "dpdk/def.h" +#include "rte_bitops.h" +#include "rte_common.h" + +//////////////////////////////////////////////////////////////////////////////// + +int +dpdk_init(int argc, char **argv) { + return rte_eal_init(argc, argv); +} + +//////////////////////////////////////////////////////////////////////////////// + +static struct dpdk_acl_rule +transform_rule(const struct filter_rule *rule, int32_t prior) { + return (struct dpdk_acl_rule + ){// user data + .data = + {.userdata = rule->action, + .category_mask = 0b1, + .priority = prior}, + + // protocol + .field[0] = + {.value.u8 = rule->transport.proto.proto, + .mask_range.u8 = 0xff}, + + // src ip + .field[1] = + {.value.u32 = rule->net4.srcs[0].addr, + .mask_range.u32 = rte_popcount32(rule->net4.srcs[0].mask)}, + + // dst ip + .field[2] = + {.value.u32 = rule->net4.dsts[0].addr, + .mask_range.u32 = rte_popcount32(rule->net4.dsts[0].mask)}, + + // src port + .field[3] = + {.value.u32 = rule->transport.srcs[0].from, + .mask_range.u32 = rule->transport.srcs[0].to}, + + // dst port + .field[4] = + {.value.u32 = rule->transport.dsts[0].from, + .mask_range.u32 = rule->transport.dsts[0].to} + }; +} + +//////////////////////////////////////////////////////////////////////////////// + +int +dpdk_acl_init( + struct dpdk_acl *acl, + const struct filter_rule *rules, + uint32_t rule_count +) { + if ((acl->ctx = rte_acl_create(&dpdk_acl_params)) == NULL) { + puts("failed to create DPDK ACL context"); + return -1; + } + + for (uint32_t i = 0; i < rule_count; ++i) { + acl->rules[i] = transform_rule(&rules[i], rule_count - i); + } + + int ret = rte_acl_add_rules( + acl->ctx, (const struct rte_acl_rule *)acl->rules, rule_count + ); + if (ret != 0) { + puts("failed to add rules into the DPDK ACL context"); + return ret; + } + + acl->cfg.num_categories = 1; + acl->cfg.num_fields = RTE_DIM(dpdk_acl_field_defs); + acl->cfg.max_size = 0; + memcpy(acl->cfg.defs, dpdk_acl_field_defs, sizeof(dpdk_acl_field_defs)); + + ret = rte_acl_build(acl->ctx, &acl->cfg); + if (ret != 0) { + puts("failed to build DPDK ACL"); + return ret; + } + + return 0; +} + +//////////////////////////////////////////////////////////////////////////////// + +void +dpdk_acl_debug_fields(const uint8_t *data) { + for (size_t i = 0; i < RTE_DIM(dpdk_acl_field_defs); ++i) { + const struct rte_acl_field_def *field = &dpdk_acl_field_defs[i]; + printf("field_index=%u\n", field->field_index); + printf("input_index=%u\n", field->input_index); + printf("bytes: ["); + for (uint32_t b = 0; b < field->size; ++b) { + uint8_t byte = data[field->offset + b]; + printf("%x", byte); + if (b + 1 < field->size) { + printf(" "); + } + } + printf("]\n\n"); + } +} \ No newline at end of file diff --git a/filter/bench/dpdk/util.h b/filter/bench/dpdk/util.h new file mode 100644 index 000000000..7352b91da --- /dev/null +++ b/filter/bench/dpdk/util.h @@ -0,0 +1,40 @@ +#pragma once + +#include "filter/rule.h" +#include "lib/dataplane/packet/packet.h" + +#include "def.h" +#include + +//////////////////////////////////////////////////////////////////////////////// + +int +dpdk_init(int argc, char **argv); + +int +dpdk_acl_init( + struct dpdk_acl *acl, + const struct filter_rule *rules, + uint32_t rule_count +); + +//////////////////////////////////////////////////////////////////////////////// + +void +dpdk_acl_debug_fields(const uint8_t *data); + +//////////////////////////////////////////////////////////////////////////////// + +static inline uint32_t +dpdk_acl_classify(struct dpdk_acl *acl, struct packet *packet) { + uint32_t result; + const uint8_t *packet_bytes = (const uint8_t *)packet->mbuf; + + // debug packet bytes used in classification + + int res = rte_acl_classify(acl->ctx, &packet_bytes, &result, 1, 1); + assert(res == 0); + return result; +} + +//////////////////////////////////////////////////////////////////////////////// \ No newline at end of file diff --git a/filter/bench/meson.build b/filter/bench/meson.build index c4cfe2403..4d3e66ece 100644 --- a/filter/bench/meson.build +++ b/filter/bench/meson.build @@ -25,4 +25,13 @@ executable('macros', link_args: yanet_link_args, dependencies: deps, link_language: 'c' +) + +# bench vs dpdk +executable('dpdk', + ['dpdk/gen.c', 'dpdk/util.c', 'dpdk/main.c'], + c_args: yanet_c_args, + link_args: yanet_link_args, + dependencies: deps, + link_language: 'c' ) \ No newline at end of file diff --git a/filter/filter.h b/filter/filter.h index d65b7baf0..8c647b768 100644 --- a/filter/filter.h +++ b/filter/filter.h @@ -322,3 +322,6 @@ filter_free(struct filter *filter); } \ } while (0); \ free_finish: + +void +value_registry_print(struct value_registry *r, uint32_t value); \ No newline at end of file diff --git a/filter/helper.c b/filter/helper.c index 615019032..86649b6aa 100644 --- a/filter/helper.c +++ b/filter/helper.c @@ -1,5 +1,9 @@ #include "helper.h" +#include "common/exp_array.h" +#include "common/memory.h" #include "common/registry.h" +#include +#include int init_dummy_registry( @@ -151,25 +155,29 @@ merge_registry_values( struct value_registry *registry2, struct value_table *table ) { - if (value_table_init( - table, - memory_context, - value_registry_capacity(registry1), - value_registry_capacity(registry2) - )) { + int ret = value_table_init( + table, + memory_context, + value_registry_capacity(registry1), + value_registry_capacity(registry2) + ); + if (ret < 0) { return -1; } for (uint32_t range_idx = 0; range_idx < registry1->range_count; ++range_idx) { value_table_new_gen(table); - value_registry_join_range( + ret = value_registry_join_range( registry1, registry2, range_idx, value_table_touch_action, table ); + if (ret < 0) { + return -1; + } } value_table_compact(table); @@ -191,8 +199,6 @@ value_table_collect_action(uint32_t v1, uint32_t v2, uint32_t idx, void *data) { collect_ctx->registry, value_table_get(collect_ctx->table, v1, v2) ); - - return 0; } static int @@ -203,9 +209,7 @@ collect_registry_values( struct value_table *table, struct value_registry *registry ) { - if (value_registry_init(registry, memory_context)) { - return -1; - } + int ret = value_registry_init(registry, memory_context); struct value_collect_ctx collect_ctx; collect_ctx.table = table; @@ -213,14 +217,20 @@ collect_registry_values( for (uint32_t range_idx = 0; range_idx < registry1->range_count; ++range_idx) { - value_registry_start(registry); - value_registry_join_range( + ret = value_registry_start(registry); + if (ret < 0) { + return -1; + } + ret = value_registry_join_range( registry1, registry2, range_idx, value_table_collect_action, &collect_ctx ); + if (ret < 0) { + return -1; + } } return 0; @@ -248,4 +258,101 @@ merge_and_collect_registry( } return 0; +} + +int +on_found_rule_classifier(const struct trie_vertex *v, void *data) { + struct rule_classifiers *cls = data; + for (uint64_t i = 0; i < v->rule_count; ++i) { + uint32_t rule = v->rules[i]; + if (cls->classifiers[rule] != NULL && + cls->classifiers[rule][cls->count[rule] - 1] == + v->classifier) { + continue; + } + void *classifiers = cls->classifiers[rule]; + int ret = mem_array_expand_exp( + cls->mctx, + &classifiers, + sizeof(uint32_t), + &cls->count[rule] + ); + if (ret < 0) { + return ret; + } + cls->classifiers[rule] = classifiers; + cls->classifiers[rule][cls->count[rule] - 1] = v->classifier; + } + return 0; +} + +//////////////////////////////////////////////////////////////////////////////// + +static inline int +compare_ints(const void *a, const void *b) { + uint32_t arg1 = *(const uint32_t *)a; + uint32_t arg2 = *(const uint32_t *)b; + return (int)(arg1 > arg2) - (int)(arg1 < arg2); +} + +int +fill_rule_registry_by_trie( + const struct trie *trie, + uint32_t rule_count, + struct value_registry *registry, + struct memory_context *mctx +) { + uint32_t **classifiers = + memory_balloc(mctx, rule_count * sizeof(uint32_t *)); + if (classifiers == NULL) { + return -1; + } + uint64_t *count = memory_balloc(mctx, rule_count * sizeof(uint64_t)); + if (count == NULL) { + memory_bfree( + mctx, classifiers, rule_count * sizeof(uint32_t *) + ); + return -1; + } + + for (uint32_t i = 0; i < rule_count; ++i) { + classifiers[i] = NULL; + } + memset(count, 0, sizeof(uint64_t) * rule_count); + + struct rule_classifiers cls = { + .classifiers = classifiers, .count = count, .mctx = mctx + }; + + int ret = trie_collect_rule_classifiers(trie, &cls); + if (ret < 0) { + goto free; + } + + for (uint32_t rule = 0; rule < rule_count; ++rule) { + ret = value_registry_start(registry); + if (ret < 0) { + goto free; + } + qsort(cls.classifiers[rule], + cls.count[rule], + sizeof(uint32_t), + compare_ints); + for (uint32_t i = 0; i < cls.count[rule]; ++i) { + uint32_t c = cls.classifiers[rule][i]; + if (i > 0 && c == cls.classifiers[rule][i - 1]) { + continue; + } + ret = value_registry_collect(registry, c); + if (ret < 0) { + goto free; + } + } + } + +free: + memory_bfree(mctx, classifiers, rule_count * sizeof(uint32_t *)); + memory_bfree(mctx, count, rule_count * sizeof(uint64_t)); + + return ret; } \ No newline at end of file diff --git a/filter/helper.h b/filter/helper.h index 63a4920cc..23946cbc7 100644 --- a/filter/helper.h +++ b/filter/helper.h @@ -1,7 +1,10 @@ +#pragma once + #include "common/memory.h" #include "common/registry.h" #include "rule.h" +#include "trie.h" int merge_and_collect_registry( @@ -27,4 +30,14 @@ init_dummy_registry( struct memory_context *memory_context, uint32_t actions, struct value_registry *registry +); + +//////////////////////////////////////////////////////////////////////////////// + +int +fill_rule_registry_by_trie( + const struct trie *trie, + uint32_t rule_count, + struct value_registry *registry, + struct memory_context *mctx ); \ No newline at end of file diff --git a/filter/meson.build b/filter/meson.build index 1cf82a5d9..a2a2429c3 100644 --- a/filter/meson.build +++ b/filter/meson.build @@ -6,6 +6,7 @@ sources = files( 'ipfw.c', 'filter.c', 'helper.c', + 'trie.c' ) lib_filter = static_library( diff --git a/filter/tests/basic_proto.c b/filter/tests/basic_proto.c index a89f299c1..16e978323 100644 --- a/filter/tests/basic_proto.c +++ b/filter/tests/basic_proto.c @@ -21,6 +21,7 @@ query_tcp_packet(struct filter *filter, uint16_t flags, uint32_t expected) { filter_query(filter, &packet, &actions, &actions_count); assert(actions_count == 1); assert(actions[0] == expected); + free_packet(&packet); } void @@ -31,6 +32,7 @@ query_udp_packet(struct filter *filter, uint32_t expected) { filter_query(filter, &packet, &actions, &actions_count); assert(actions_count == 1); assert(actions[0] == expected); + free_packet(&packet); } //////////////////////////////////////////////////////////////////////////////// @@ -47,15 +49,15 @@ test_proto_1(void *memory) { assert(res == 0); struct filter_rule_builder b1; - builer_set_proto(&b1, IPPROTO_TCP, 0b101, 0b010); + builder_set_proto(&b1, IPPROTO_TCP, 0b101, 0b010); struct filter_rule r1 = build_rule(&b1, 1); struct filter_rule_builder b2; - builer_set_proto(&b2, IPPROTO_UDP, 0, 0); + builder_set_proto(&b2, IPPROTO_UDP, 0, 0); struct filter_rule r2 = build_rule(&b2, 2); struct filter_rule_builder b3; - builer_set_proto(&b3, PROTO_UNSPEC, 0, 0); + builder_set_proto(&b3, PROTO_UNSPEC, 0, 0); struct filter_rule r3 = build_rule(&b3, 3); struct filter_rule rules[3] = {r1, r2, r3}; diff --git a/filter/tests/basic_vlan.c b/filter/tests/basic_vlan.c index 807ffbcc9..7a6a833ec 100644 --- a/filter/tests/basic_vlan.c +++ b/filter/tests/basic_vlan.c @@ -21,6 +21,7 @@ query_packet(struct filter *filter, uint16_t vlan, uint32_t expected) { filter_query(filter, &packet, &actions, &actions_count); assert(actions_count == 1); assert(actions[0] == expected); + free_packet(&packet); } //////////////////////////////////////////////////////////////////////////////// diff --git a/filter/tests/ipv4_tuple.c b/filter/tests/ipv4_tuple.c new file mode 100644 index 000000000..cf16e642a --- /dev/null +++ b/filter/tests/ipv4_tuple.c @@ -0,0 +1,159 @@ +#include "attribute.h" +#include "common/memory_block.h" +#include "filter/filter.h" +#include "utils.h" +#include + +//////////////////////////////////////////////////////////////////////////////// + +#define MEMORY (size_t)(1ull << 30) + +//////////////////////////////////////////////////////////////////////////////// + +void +test1(void *memory) { + // init memory + struct block_allocator allocator; + block_allocator_init(&allocator); + block_allocator_put_arena(&allocator, memory, MEMORY); + + struct memory_context memory_context; + int ret = memory_context_init(&memory_context, "test", &allocator); + assert(ret == 0); + + // build rules + struct filter_rule_builder b; + builder_init(&b); + builder_add_net4_src( + &b, ip(0xff, 0xff, 32, 0), ip(0xff, 0xff, 0xff, 0) + ); + builder_add_net4_dst( + &b, ip(0xff, 0xff, 33, 0), ip(0xff, 0xff, 0xff, 0) + ); + builder_add_port_src_range(&b, 288, 788); + builder_add_port_dst_range(&b, 388, 888); + struct filter_rule rule = build_rule(&b, 289); + + // filter attributes + const struct filter_attribute *attrs[] = { + &attribute_proto, + &attribute_net4_src, + &attribute_net4_dst, + &attribute_port_src, + &attribute_port_dst + }; + struct filter filter; + ret = filter_init(&filter, attrs, 5, &rule, 1, &memory_context); + assert(ret == 0); + + struct packet packet = make_packet( + ip(0xff, 0xff, 32, 1), + ip(0xff, 0xff, 33, 1), + 288, + 800, + IPPROTO_UDP, + 0, + 0 + ); + uint32_t *actions; + uint32_t count; + ret = filter_query(&filter, &packet, &actions, &count); + assert(ret == 0); + assert(count == 1); + assert(actions[0] == 289); + + free_packet(&packet); + filter_free(&filter); +} + +//////////////////////////////////////////////////////////////////////////////// + +#define RULES 708 + +//////////////////////////////////////////////////////////////////////////////// + +void +test2(void *memory) { + // build rules + struct filter_rule rules[RULES]; + struct filter_rule_builder builders[RULES]; + + for (uint32_t i = 0; i < RULES; ++i) { + struct filter_rule_builder *b = &builders[i]; + + builder_init(b); + builder_add_port_src_range(b, i & 127, 500 + (i & 255)); + builder_add_port_dst_range(b, 100 + (i & 127), 600 + (i & 255)); + builder_add_net4_src( + b, ip(0xff, 0xff, i & 0xff, 0), ip(0xff, 0xff, 0xff, 0) + ); + builder_add_net4_dst( + b, + ip(0xff, 0xff, (i + 1) & 0xff, 0), + ip(0xff, 0xff, 0xff, 0) + ); + builder_set_proto(b, IPPROTO_UDP, 0, 0); + + rules[i] = build_rule(b, i + 1); + } + + // init memory + struct block_allocator allocator; + block_allocator_init(&allocator); + block_allocator_put_arena(&allocator, memory, MEMORY); + + struct memory_context mctx; + int ret = memory_context_init(&mctx, "test", &allocator); + assert(ret == 0); + + // build filter + const struct filter_attribute *attrs[] = { + &attribute_proto, + &attribute_net4_src, + &attribute_net4_dst, + &attribute_port_src, + &attribute_port_dst + }; + struct filter filter; + ret = filter_init(&filter, attrs, 5, rules, RULES, &mctx); + assert(ret == 0); + + // query packet + struct packet packet = make_packet( + ip(0xff, 0xff, 32, 1), + ip(0xff, 0xff, 33, 1), + 32, + 132, + IPPROTO_UDP, + 0, + 0 + ); + uint32_t *actions; + uint32_t count; + ret = filter_query(&filter, &packet, &actions, &count); + assert(ret == 0); + assert(count == 1); + assert(actions[0] == 33); + + free_packet(&packet); + filter_free(&filter); +} + +//////////////////////////////////////////////////////////////////////////////// + +int +main() { + void *memory = malloc(MEMORY); + + puts("test1..."); + test1(memory); + + puts("test2..."); + test2(memory); + + free(memory); + + puts("OK"); + + return 0; +} diff --git a/filter/tests/meson.build b/filter/tests/meson.build index 752b9631b..950d6596a 100644 --- a/filter/tests/meson.build +++ b/filter/tests/meson.build @@ -102,3 +102,23 @@ filter_macros = executable('macros', link_language: 'c' ) test('macros', filter_macros, suite: ['filter']) + + +# test ipv4 5-tuple +ipv4_tuple = executable('ipv4_tuple', ['ipv4_tuple.c'], +c_args: yanet_test_c_args, + link_args: yanet_link_args, + dependencies: test_deps, + link_language: 'c' +) +test('ipv4_tuple', ipv4_tuple, suite: ['filter']) + + +# test trie +trie = executable('trie', ['trie.c'], + c_args: yanet_test_c_args, + link_args: yanet_link_args, + dependencies: test_deps, + link_language: 'c' +) +test('trie', trie, suite: ['filter']) \ No newline at end of file diff --git a/filter/tests/trie.c b/filter/tests/trie.c new file mode 100644 index 000000000..287634f7f --- /dev/null +++ b/filter/tests/trie.c @@ -0,0 +1,105 @@ +#include "filter/trie.h" +#include "common/memory.h" +#include "common/memory_address.h" +#include "common/registry.h" +#include "filter/helper.h" +#include +#include +#include + +//////////////////////////////////////////////////////////////////////////////// + +#define MEMORY (1 << 24) + +//////////////////////////////////////////////////////////////////////////////// + +void +test1(void *memory) { + // init memory + struct block_allocator allocator; + block_allocator_init(&allocator); + block_allocator_put_arena(&allocator, memory, MEMORY); + + struct memory_context mctx; + int ret = memory_context_init(&mctx, "test", &allocator); + assert(ret == 0); + + // init trie + struct trie trie; + ret = trie_init(&trie, &mctx); + assert(ret == 0); + + // add rules in trie + + const uint8_t v1[] = {0x10, 0x20, 0xff, 0xff}; + ret = trie_add(&trie, v1, 16, 0); + assert(ret == 0); + + const uint8_t v2[] = {0x10, 0x20, 0xff, 0xff}; + ret = trie_add(&trie, v2, 20, 1); + assert(ret == 0); + + const uint8_t v3[] = {0x10, 0x20, 0xee, 0xee}; + ret = trie_add(&trie, v3, 20, 2); + assert(ret == 0); + + // build classifiers + trie_build_classifiers(&trie); + + // make requests + const uint8_t r1[] = {0x10, 0x20, 0xff, 0xff}; // only 1 and 2 + uint32_t c1 = trie_classify(&trie, r1, 4); + + const uint8_t r2[] = {0x10, 0x20, 0xee, 0}; // only 1 and 3 + uint32_t c2 = trie_classify(&trie, r2, 4); + + const uint8_t r3[] = {0x10, 0x20, 0, 0}; // only 1 + uint32_t c3 = trie_classify(&trie, r3, 4); + + const uint8_t r4[] = {0x11, 0x20, 0, 0}; // nothing + uint32_t c4 = trie_classify(&trie, r4, 4); + + assert(c1 != c2); + assert(c1 != c3); + assert(c2 != c3); + assert(c4 != c1); + assert(c4 != c2); + assert(c4 != c3); + + // collect rule classifiers in registry + struct value_registry registry; + ret = value_registry_init(®istry, &mctx); + assert(ret == 0); + + ret = fill_rule_registry_by_trie(&trie, 3, ®istry, &mctx); + assert(ret == 0); + + assert(registry.range_count == 3); + uint32_t ref_range_counts[] = {3, 1, 1}; + + for (uint64_t i = 0; i < registry.range_count; ++i) { + struct value_range range = ADDR_OF(®istry.ranges)[i]; + assert(range.count == ref_range_counts[i]); + } + + // free registry + value_registry_free(®istry); + + // free trie + trie_free_mem(&trie); +} + +//////////////////////////////////////////////////////////////////////////////// + +int +main() { + void *memory = malloc(MEMORY); + + puts("test1..."); + test1(memory); + + free(memory); + + puts("OK"); + return 0; +} \ No newline at end of file diff --git a/filter/tests/vs_prev.c b/filter/tests/vs_prev.c index 2f2d98a17..0570c265d 100644 --- a/filter/tests/vs_prev.c +++ b/filter/tests/vs_prev.c @@ -258,5 +258,8 @@ main() { } free(packets); + free(rules); + free(builders); + return 0; } \ No newline at end of file diff --git a/filter/trie.c b/filter/trie.c new file mode 100644 index 000000000..575600739 --- /dev/null +++ b/filter/trie.c @@ -0,0 +1,275 @@ +#include "trie.h" +#include "common/exp_array.h" +#include "common/memory.h" +#include + +//////////////////////////////////////////////////////////////////////////////// + +static void +init_vertex(struct trie_vertex *v) { + v->rules = NULL; + v->rule_count = 0; + v->classifier = 0; + memset(v->next, TRIE_VERTEX_UNDEF, 256 * sizeof(uint32_t)); +} + +static int +add_vertex(struct trie *trie) { + void *data = trie->vertices; + int ret = mem_array_expand_exp( + trie->mctx, + &data, + sizeof(struct trie_vertex), + &trie->vertex_count + ); + if (ret < 0) { + return -1; + } + trie->vertices = data; + init_vertex(&trie->vertices[trie->vertex_count - 1]); + return 0; +} + +static inline int +transition( + struct trie *trie, uint32_t vertex_id, uint8_t next, uint32_t *next_v +) { + struct trie_vertex *v = &trie->vertices[vertex_id]; + if (v->next[next] == TRIE_VERTEX_UNDEF) { + int ret = add_vertex(trie); + if (ret < 0) { + return -1; + } + v = &trie->vertices[vertex_id]; + v->next[next] = trie->vertex_count - 1; + } + *next_v = v->next[next]; + return 0; +} + +static int +append_rule(struct trie *trie, uint32_t vertex_id, uint32_t rule) { + struct trie_vertex *v = &trie->vertices[vertex_id]; + void *data = v->rules; + int ret = mem_array_expand_exp( + trie->mctx, &data, sizeof(uint32_t), &v->rule_count + ); + if (ret < 0) { + return ret; + } + v->rules = data; + v->rules[v->rule_count - 1] = rule; + return 0; +} + +//////////////////////////////////////////////////////////////////////////////// + +int +trie_init(struct trie *trie, struct memory_context *mctx) { + trie->vertex_count = 0; + trie->vertices = NULL; + trie->mctx = mctx; + return add_vertex(trie); +} + +//////////////////////////////////////////////////////////////////////////////// + +static int +add_rec(struct trie *trie, + uint32_t vertex_id, + const uint8_t *value, + uint32_t prefix_len, + uint32_t rule) { + if (prefix_len == 0) { + return append_rule(trie, vertex_id, rule); + } + int ret; + if (prefix_len >= 8) { + uint32_t next_vertex_id; + ret = transition(trie, vertex_id, value[0], &next_vertex_id); + if (ret < 0) { + return -1; + } + return add_rec( + trie, next_vertex_id, value + 1, prefix_len - 8, rule + ); + } + uint8_t mask = ~((1 << (8 - prefix_len)) - 1); + uint8_t need = value[0] & mask; + for (uint16_t next = 0; next <= 0xFF; ++next) { + if ((next & mask) != need) { + continue; + } + uint32_t next_vertex_id; + ret = transition(trie, vertex_id, next, &next_vertex_id); + if (ret < 0) { + return -1; + } + ret = add_rec(trie, next_vertex_id, NULL, 0, rule); + if (ret < 0) { + return -1; + } + } + return 0; +} + +int +trie_add( + struct trie *trie, + const uint8_t *value, + uint32_t prefix_len, + uint32_t rule +) { + return add_rec(trie, 0, value, prefix_len, rule); +} + +//////////////////////////////////////////////////////////////////////////////// + +static inline uint32_t +next_classifier( + struct trie *trie, + uint32_t v_id, + uint32_t par_id, + uint8_t from, + uint32_t *next_class +) { + if (par_id == TRIE_VERTEX_UNDEF) { + return (*next_class)++; + } + struct trie_vertex *v = &trie->vertices[v_id]; + struct trie_vertex *par = &trie->vertices[par_id]; + if (v->rule_count == 0) { + return par->classifier; + } + for (uint8_t simb = 0; simb < from; ++simb) { + uint32_t s_id = par->next[simb]; + if (s_id != TRIE_VERTEX_UNDEF) { + struct trie_vertex *s = &trie->vertices[s_id]; + if (s->rule_count == v->rule_count && + memcmp(s->rules, + v->rules, + s->rule_count * sizeof(uint32_t)) == 0) { + return s->classifier; + } + } + } + return (*next_class)++; +} + +static void +build_classifiers_rec( + struct trie *trie, + uint32_t v_id, + uint32_t par_id, + uint8_t from, + uint32_t *next_class +) { + struct trie_vertex *v = &trie->vertices[v_id]; + v->classifier = next_classifier(trie, v_id, par_id, from, next_class); + for (uint16_t next = 0; next <= 0xFF; ++next) { + uint32_t next_id = v->next[next]; + if (next_id != TRIE_VERTEX_UNDEF) { + build_classifiers_rec( + trie, next_id, v_id, next, next_class + ); + } + } +} + +void +trie_build_classifiers(struct trie *trie) { + uint32_t class = 0; + build_classifiers_rec(trie, 0, TRIE_VERTEX_UNDEF, 0, &class); +} + +//////////////////////////////////////////////////////////////////////////////// + +uint32_t +trie_classify(const struct trie *trie, const uint8_t *value, uint32_t n) { + const struct trie_vertex *v = &trie->vertices[0]; + for (uint32_t b = 0; b < n; ++b) { + uint32_t next = v->next[value[b]]; + if (next == TRIE_VERTEX_UNDEF) { + break; + } + v = &trie->vertices[next]; + } + return v->classifier; +} + +//////////////////////////////////////////////////////////////////////////////// + +static int +collect_rule_classifiers_rec( + const struct trie *trie, + uint32_t v_id, + struct rule_classifiers *cls, + uint32_t *stack, + uint32_t stack_size +) { + stack[stack_size++] = v_id; + const struct trie_vertex *v = &trie->vertices[v_id]; + if (v->rule_count > 0) { + for (uint32_t i = 0; i < stack_size; ++i) { + const struct trie_vertex *stack_v = + &trie->vertices[stack[i]]; + for (uint32_t j = 0; j < stack_v->rule_count; ++j) { + uint32_t rule = stack_v->rules[j]; + + void *data = cls->classifiers[rule]; + int ret = mem_array_expand_exp( + cls->mctx, + &data, + sizeof(uint32_t), + &cls->count[rule] + ); + if (ret < 0) { + return -1; + } + cls->classifiers[rule] = data; + cls->classifiers[rule][cls->count[rule] - 1] = + v->classifier; + } + } + } + for (uint16_t next = 0; next <= 0xFF; ++next) { + uint32_t next_id = v->next[next]; + if (next_id != TRIE_VERTEX_UNDEF) { + int ret = collect_rule_classifiers_rec( + trie, next_id, cls, stack, stack_size + ); + if (ret < 0) { + return ret; + } + } + } + return 0; +} + +#define MAX_TRIE_DEPTH 32 + +int +trie_collect_rule_classifiers( + const struct trie *trie, struct rule_classifiers *cls +) { + uint32_t stack[MAX_TRIE_DEPTH]; + return collect_rule_classifiers_rec(trie, 0, cls, stack, 0); +} + +//////////////////////////////////////////////////////////////////////////////// + +void +trie_free_mem(struct trie *trie) { + for (size_t i = 0; i < trie->vertex_count; ++i) { + memory_bfree( + trie->mctx, + trie->vertices[i].rules, + trie->vertices[i].rule_count * sizeof(uint32_t) + ); + } + memory_bfree( + trie->mctx, + trie->vertices, + trie->vertex_count * sizeof(struct trie_vertex) + ); +} \ No newline at end of file diff --git a/filter/trie.h b/filter/trie.h new file mode 100644 index 000000000..5cc44e19b --- /dev/null +++ b/filter/trie.h @@ -0,0 +1,61 @@ +#pragma once + +#include + +//////////////////////////////////////////////////////////////////////////////// + +#define TRIE_VERTEX_UNDEF ((uint32_t)-1) + +//////////////////////////////////////////////////////////////////////////////// + +struct trie_vertex { + uint32_t next[256]; + uint32_t *rules; + uint64_t rule_count; + uint32_t classifier; +}; + +//////////////////////////////////////////////////////////////////////////////// + +struct trie { + struct memory_context *mctx; + struct trie_vertex *vertices; + uint64_t vertex_count; +}; + +//////////////////////////////////////////////////////////////////////////////// + +int +trie_init(struct trie *trie, struct memory_context *mctx); + +int +trie_add( + struct trie *trie, + const uint8_t *value, + uint32_t prefix_len, + uint32_t rule +); + +void +trie_build_classifiers(struct trie *trie); + +uint32_t +trie_classify(const struct trie *trie, const uint8_t *value, uint32_t n); + +//////////////////////////////////////////////////////////////////////////////// + +struct rule_classifiers { + uint32_t **classifiers; // rule -> [classifier] + uint64_t *count; // rule -> len([classifier]) + struct memory_context *mctx; +}; + +int +trie_collect_rule_classifiers( + const struct trie *trie, struct rule_classifiers *cls +); + +//////////////////////////////////////////////////////////////////////////////// + +void +trie_free_mem(struct trie *trie); \ No newline at end of file diff --git a/filter/utils/utils.c b/filter/utils/utils.c index eeb491bc7..d39103edb 100644 --- a/filter/utils/utils.c +++ b/filter/utils/utils.c @@ -248,7 +248,7 @@ builder_add_port_src_range( } void -builer_set_proto( +builder_set_proto( struct filter_rule_builder *builder, uint8_t proto, uint16_t enable_bits, diff --git a/filter/utils/utils.h b/filter/utils/utils.h index 8bfd285bb..f49da875f 100644 --- a/filter/utils/utils.h +++ b/filter/utils/utils.h @@ -96,7 +96,7 @@ builder_add_port_src_range( ); void -builer_set_proto( +builder_set_proto( struct filter_rule_builder *builder, uint8_t proto, uint16_t enable_bits, @@ -112,6 +112,6 @@ build_rule(struct filter_rule_builder *builder, uint32_t action); //////////////////////////////////////////////////////////////////////////////// inline static uint32_t -ip(uint8_t a, uint8_t b, uint8_t c, uint8_t d) { +ip(uint32_t a, uint32_t b, uint32_t c, uint32_t d) { return (a << 24) | (b << 16) | (c << 8) | d; } \ No newline at end of file diff --git a/tests/common/radix_test.c b/tests/common/radix_test.c index 91207f8d7..eb9e1cd13 100644 --- a/tests/common/radix_test.c +++ b/tests/common/radix_test.c @@ -78,6 +78,9 @@ main() { radix_free(&radix); + free(arena0); + puts("OK!"); + return 0; } \ No newline at end of file From b415093f2c110576a5055958e77aa6e80a000140 Mon Sep 17 00:00:00 2001 From: Sergey Yakovlev <3gnees@gmail.com> Date: Wed, 27 Aug 2025 20:20:16 +0300 Subject: [PATCH 2/3] trie -> lpm --- filter/attribute/net4.h | 167 +++++++++++++++++++++++++++++----------- filter/trie.c | 15 ---- filter/trie.h | 31 +++++++- 3 files changed, 153 insertions(+), 60 deletions(-) diff --git a/filter/attribute/net4.h b/filter/attribute/net4.h index c5a05f043..0fc0092f5 100644 --- a/filter/attribute/net4.h +++ b/filter/attribute/net4.h @@ -1,8 +1,9 @@ -#include "../helper.h" #include "../rule.h" +#include "common/lpm.h" +#include "common/range_collector.h" #include "dataplane/packet/packet.h" -#include "filter/trie.h" +#include "util.h" #include #include @@ -29,54 +30,137 @@ action_get_net4_dst( *count = action->net4.dst_count; } -//////////////////////////////////////////////////////////////////////////////// +static inline void +net4_collect_values( + struct net4 *start, + uint32_t count, + struct lpm *lpm, + struct value_table *table +) { + for (struct net4 *net4 = start; net4 < start + count; ++net4) { + uint32_t addr = htobe32(net4->addr); + uint32_t mask = htobe32(net4->mask); + uint32_t to = addr | ~mask; + lpm4_collect_values( + lpm, + (uint8_t *)&addr, + (uint8_t *)&to, + lpm_collect_value_iterator, + table + ); + } +} + +static inline void +net4_collect_registry( + struct net4 *start, + uint32_t count, + struct lpm *lpm, + struct value_registry *registry +) { + for (struct net4 *net4 = start; net4 < start + count; ++net4) { + uint32_t addr = htobe32(net4->addr); + uint32_t mask = htobe32(net4->mask); + uint32_t to = addr | ~mask; + lpm4_collect_values( + lpm, + (uint8_t *)&addr, + (uint8_t *)&to, + lpm_collect_registry_iterator, + registry + ); + } +} static inline int collect_net4_values( struct memory_context *memory_context, - const struct filter_rule *rules, + const struct filter_rule *actions, uint32_t count, rule_get_net4_func get_net4, - struct trie *trie, + struct lpm *lpm, struct value_registry *registry ) { - int ret = trie_init(trie, memory_context); - if (ret < 0) { - return -1; - } + struct range_collector collector; + if (range_collector_init(&collector, memory_context)) + goto error; - for (const struct filter_rule *rule = rules; rule < rules + count; - ++rule) { - if (rule->net4.src_count == 0 && rule->net4.dst_count == 0) { + for (const struct filter_rule *action = actions; + action < actions + count; + ++action) { + if (action->net4.src_count == 0 && + action->net4.dst_count == 0) { continue; } struct net4 *nets; uint32_t net_count; - get_net4(rule, &nets, &net_count); + get_net4(action, &nets, &net_count); for (struct net4 *net4 = nets; net4 < nets + net_count; ++net4) { uint32_t addr = htobe32(net4->addr); - ret = trie_add( - trie, - (uint8_t *)&addr, - __builtin_popcountll(net4->mask), - rule - rules - ); - if (ret < 0) { - return -1; - } + if (range4_collector_add( + &collector, + (uint8_t *)&addr, + __builtin_popcountll(net4->mask) + )) + goto error_collector; } } + if (lpm_init(lpm, memory_context)) { + goto error_lpm; + } + if (range_collector_collect(&collector, 4, lpm)) { + goto error_collector; + } - // build trie classifiers - trie_build_classifiers(trie); + struct value_table table; + if (value_table_init(&table, memory_context, 1, collector.count)) + goto error_vtab; - return fill_rule_registry_by_trie( - trie, count, registry, memory_context - ); + for (const struct filter_rule *action = actions; + action < actions + count; + ++action) { + + value_table_new_gen(&table); + + struct net4 *nets; + uint32_t net_count; + get_net4(action, &nets, &net_count); + + net4_collect_values(nets, net_count, lpm, &table); + } + + value_table_compact(&table); + lpm4_remap(lpm, &table); + lpm4_compact(lpm); + + for (const struct filter_rule *action = actions; + action < actions + count; + ++action) { + value_registry_start(registry); + + struct net4 *nets; + uint32_t net_count; + get_net4(action, &nets, &net_count); + + net4_collect_registry(nets, net_count, lpm, registry); + } + + value_table_free(&table); + return 0; + +error_collector: + range_collector_free(&collector, 4); +error_lpm: + lpm_free(lpm); + +error_vtab: + +error: + return -1; } //////////////////////////////////////////////////////////////////////////////// @@ -90,14 +174,14 @@ init_net4_src( size_t actions_count, struct memory_context *memory_context ) { - struct trie *trie = memory_balloc(memory_context, sizeof(struct trie)); - *data = trie; + struct lpm *lpm = memory_balloc(memory_context, sizeof(struct lpm)); + *data = lpm; return collect_net4_values( memory_context, actions, actions_count, action_get_net4_src, - trie, + lpm, registry ); } @@ -106,14 +190,11 @@ init_net4_src( static inline uint32_t lookup_net4_src(struct packet *packet, void *data) { struct rte_mbuf *mbuf = packet_to_mbuf(packet); - struct rte_ipv4_hdr *ipv4_hdr = rte_pktmbuf_mtod_offset( mbuf, struct rte_ipv4_hdr *, packet->network_header.offset ); - - struct trie *trie = (struct trie *)data; - uint32_t res = trie_classify(trie, (uint8_t *)&ipv4_hdr->src_addr, 4); - return res; + struct lpm *lpm = (struct lpm *)data; + return lpm4_lookup(lpm, (uint8_t *)&ipv4_hdr->src_addr); } // Allows to initialize attribute for IPv4 destination address. @@ -125,14 +206,14 @@ init_net4_dst( size_t actions_count, struct memory_context *memory_context ) { - struct trie *trie = memory_balloc(memory_context, sizeof(struct trie)); - *data = trie; + struct lpm *lpm = memory_balloc(memory_context, sizeof(struct lpm)); + *data = lpm; return collect_net4_values( memory_context, actions, actions_count, action_get_net4_dst, - trie, + lpm, registry ); } @@ -146,15 +227,15 @@ lookup_net4_dst(struct packet *packet, void *data) { mbuf, struct rte_ipv4_hdr *, packet->network_header.offset ); - struct trie *trie = (struct trie *)data; - uint32_t res = trie_classify(trie, (uint8_t *)&ipv4_hdr->dst_addr, 4); - return res; + struct lpm *lpm = (struct lpm *)data; + + return lpm4_lookup(lpm, (uint8_t *)&ipv4_hdr->dst_addr); } // Allows to free data for IPv4 classification. static inline void free_net4(void *data, struct memory_context *memory_context) { (void)memory_context; - struct trie *trie = (struct trie *)data; - trie_free_mem(trie); + struct lpm *lpm = (struct lpm *)data; + lpm_free(lpm); } \ No newline at end of file diff --git a/filter/trie.c b/filter/trie.c index 575600739..2317397b2 100644 --- a/filter/trie.c +++ b/filter/trie.c @@ -184,21 +184,6 @@ trie_build_classifiers(struct trie *trie) { //////////////////////////////////////////////////////////////////////////////// -uint32_t -trie_classify(const struct trie *trie, const uint8_t *value, uint32_t n) { - const struct trie_vertex *v = &trie->vertices[0]; - for (uint32_t b = 0; b < n; ++b) { - uint32_t next = v->next[value[b]]; - if (next == TRIE_VERTEX_UNDEF) { - break; - } - v = &trie->vertices[next]; - } - return v->classifier; -} - -//////////////////////////////////////////////////////////////////////////////// - static int collect_rule_classifiers_rec( const struct trie *trie, diff --git a/filter/trie.h b/filter/trie.h index 5cc44e19b..136e0247d 100644 --- a/filter/trie.h +++ b/filter/trie.h @@ -36,11 +36,38 @@ trie_add( uint32_t rule ); +//////////////////////////////////////////////////////////////////////////////// + void trie_build_classifiers(struct trie *trie); -uint32_t -trie_classify(const struct trie *trie, const uint8_t *value, uint32_t n); +//////////////////////////////////////////////////////////////////////////////// + +static inline uint32_t +trie_classify4(const struct trie *trie, const uint8_t *value) { + const struct trie_vertex *v = &trie->vertices[0]; + for (uint32_t b = 0; b < 4; ++b) { + uint32_t next = v->next[value[b]]; + if (next == TRIE_VERTEX_UNDEF) { + break; + } + v = &trie->vertices[next]; + } + return v->classifier; +} + +static inline uint32_t +trie_classify(const struct trie *trie, const uint8_t *value, uint32_t n) { + const struct trie_vertex *v = &trie->vertices[0]; + for (uint32_t b = 0; b < n; ++b) { + uint32_t next = v->next[value[b]]; + if (next == TRIE_VERTEX_UNDEF) { + break; + } + v = &trie->vertices[next]; + } + return v->classifier; +} //////////////////////////////////////////////////////////////////////////////// From 29eefe968a4d3b5ee253fbc11bfb702901c87902 Mon Sep 17 00:00:00 2001 From: Sergey Yakovlev <3gnees@gmail.com> Date: Wed, 27 Aug 2025 20:26:01 +0300 Subject: [PATCH 3/3] removed unused files --- filter/bench/dpdk.c | 102 --------------------------------------- filter/bench/meson.build | 2 +- 2 files changed, 1 insertion(+), 103 deletions(-) delete mode 100644 filter/bench/dpdk.c diff --git a/filter/bench/dpdk.c b/filter/bench/dpdk.c deleted file mode 100644 index c0cd5d17c..000000000 --- a/filter/bench/dpdk.c +++ /dev/null @@ -1,102 +0,0 @@ -#include -#include -#include - -#include -#include - -//////////////////////////////////////////////////////////////////////////////// - -struct ipv4_5tuple { - uint8_t proto; - uint32_t src_ip; - uint32_t dst_ip; - uint16_t src_port; - uint16_t dst_port; -}; - -//////////////////////////////////////////////////////////////////////////////// - -struct rte_acl_field_def defs[1] = { - { - .type = RTE_ACL_FIELD_TYPE_BITMASK, - .size = sizeof(uint8_t), - .field_index = 0, - .input_index = 0, - .offset = offsetof(struct ipv4_5tuple, proto), - }, -}; - -RTE_ACL_RULE_DEF(acl_ipv4_rule, RTE_DIM(defs)); - -struct rte_acl_param dpdk_acl_params = { - .name = "ACL_example", - .socket_id = SOCKET_ID_ANY, - .rule_size = RTE_ACL_RULE_SZ(RTE_DIM(defs)), - - /* number of fields per rule. */ - .max_rule_num = 8, /* maximum number of rules in the AC context. */ -}; - -int -main(int argc, char **argv) { - int ret = rte_eal_init(argc, argv); - if (ret < 0) { - printf("ret=%d\n", ret); - rte_panic("Cannot init EAL\n"); - } - - const struct acl_ipv4_rule acl_rules[] = { - {.data = {.userdata = 1, .category_mask = 3, .priority = 1}, - - .field[0] = {.value.u8 = 15, .mask_range.u8 = 0b1010}} - }; - - struct rte_acl_ctx *acx; - struct rte_acl_config cfg; - - /* create an empty AC context */ - if ((acx = rte_acl_create(&dpdk_acl_params)) == NULL) { - printf("ret=%d\n", ret); - rte_panic("Cannot create ACL\n"); - } - - /* add rules to the context */ - ret = rte_acl_add_rules( - acx, (const struct rte_acl_rule *)&acl_rules, RTE_DIM(acl_rules) - ); - if (ret != 0) { - printf("ret=%d\n", ret); - rte_panic("Cannot add ACL rules\n"); - } - - cfg.num_categories = 2; - cfg.num_fields = RTE_DIM(defs); - - memcpy(cfg.defs, defs, sizeof(defs)); - - /* build the runtime structures for added rules, with 2 categories. */ - ret = rte_acl_build(acx, &cfg); - if (ret != 0) { - printf("ret=%d\n", ret); - rte_panic("Cannot build ACL\n"); - } - - uint32_t results[4]; - struct ipv4_5tuple data = {.proto = 15}; - const uint8_t *datas = (const uint8_t *)&data; - ret = rte_acl_classify(acx, &datas, results, 1, 4); - if (ret != 0) { - printf("ret=%d\n", ret); - rte_panic("Cannot classify\n"); - } - - puts("result="); - for (size_t i = 0; i < 4; ++i) { - printf("%u ", results[i]); - } - puts(""); - - puts("OK"); - return 0; -} \ No newline at end of file diff --git a/filter/bench/meson.build b/filter/bench/meson.build index 4d3e66ece..122ef85d3 100644 --- a/filter/bench/meson.build +++ b/filter/bench/meson.build @@ -30,7 +30,7 @@ executable('macros', # bench vs dpdk executable('dpdk', ['dpdk/gen.c', 'dpdk/util.c', 'dpdk/main.c'], - c_args: yanet_c_args, + c_args: yanet_c_args + ['-O3'], link_args: yanet_link_args, dependencies: deps, link_language: 'c'