From 2133300a43bfadce3f3e9bf8f54c3f1189822216 Mon Sep 17 00:00:00 2001 From: Janne Peltonen Date: Fri, 30 May 2025 15:43:13 +0300 Subject: [PATCH 1/7] test: bench_packet: add more tests Add new variants of odp_packet_parse() tests with and without L4 checksumming. Add new variants of odp_event_user_area(), odp_event_user_flag_set() and odp_event_user_area_and_flag() tests. Add tests for odp_packet_l{2,3,4}_type() functions and for the packet flag getter functions. Use packets/events of multiple types in a pseudorandom order in the new tests to make things less predictable for branchy code and hopefully slightly more realistic. Signed-off-by: Janne Peltonen --- test/performance/odp_bench_packet.c | 305 +++++++++++++++++++++++++++- 1 file changed, 298 insertions(+), 7 deletions(-) diff --git a/test/performance/odp_bench_packet.c b/test/performance/odp_bench_packet.c index a698b7c510..e20b4c9caa 100644 --- a/test/performance/odp_bench_packet.c +++ b/test/performance/odp_bench_packet.c @@ -63,7 +63,7 @@ #define TEST_DEF_BURST 8 /** Maximum number of results to be held */ -#define TEST_MAX_BENCH 100 +#define TEST_MAX_BENCH 200 #define TEST_MAX_SIZES 7 @@ -105,6 +105,12 @@ typedef struct { bench_suite_t suite; /** Packet pool */ odp_pool_t pool; + /** Buffer pool */ + odp_pool_t pool_buf; + /** Timeout pool */ + odp_pool_t pool_tmo; + /** Event vector pool */ + odp_pool_t pool_evv; struct { /** Test packet length */ uint32_t len; @@ -286,6 +292,67 @@ static void alloc_packets_twice(void) TEST_REPEAT_COUNT); } +typedef struct test_packet_t { + const uint8_t *data; + int len; +} test_packet_t; + +test_packet_t test_packets[] = { + {test_packet_arp, sizeof(test_packet_arp)}, + {test_packet_ipv4_icmp, sizeof(test_packet_ipv4_icmp)}, + {test_packet_ipv6_icmp, sizeof(test_packet_ipv6_icmp)}, + {test_packet_ipv4_tcp, sizeof(test_packet_ipv4_tcp)}, + {test_packet_ipv6_tcp, sizeof(test_packet_ipv6_tcp)}, + {test_packet_ipv4_udp, sizeof(test_packet_ipv4_udp)}, + {test_packet_ipv6_udp, sizeof(test_packet_ipv6_udp)}, + {test_packet_ipv4_sctp, sizeof(test_packet_ipv4_sctp)}, + {test_packet_ipv6_sctp, sizeof(test_packet_ipv6_sctp)}, + {test_packet_vlan_ipv4_udp, sizeof(test_packet_vlan_ipv4_udp)}, + {test_packet_mcast_eth_ipv4_udp, sizeof(test_packet_mcast_eth_ipv4_udp)}, + {test_packet_mcast_eth_ipv6_udp, sizeof(test_packet_mcast_eth_ipv6_udp)}, + {test_packet_ipv4_udp_last_frag, sizeof(test_packet_ipv4_udp_last_frag)}, +}; + +static uint32_t rnd(void) +{ + static uint64_t s = 1; + uint64_t prime = 0x7fffffff; + + s = (0x12345678 * s) % prime; + return s; +} + +static void create_packets_misc(void) +{ + int num_types = ODPH_ARRAY_SIZE(test_packets); + + for (int i = 0; i < TEST_REPEAT_COUNT; i++) { + test_packet_t *tp = &test_packets[rnd() % num_types]; + odp_packet_t *pkt = &gbl_args->pkt_tbl[i]; + + *pkt = odp_packet_alloc(gbl_args->pool, tp->len); + if (*pkt == ODP_PACKET_INVALID) + ODPH_ABORT("Allocating test packets failed\n"); + if (odp_packet_copy_from_mem(*pkt, 0, tp->len, tp->data)) + ODPH_ABORT("Copying test packet failed\n"); + } +} + +static void create_packets_misc_parsed(void) +{ + odp_packet_parse_param_t param = { + .proto = ODP_PROTO_ETH, + .last_layer = ODP_PROTO_LAYER_ALL, + .chksums.all_chksum = 0 + }; + + create_packets_misc(); + for (int i = 0; i < TEST_REPEAT_COUNT; i++) { + if (odp_packet_parse(gbl_args->pkt_tbl[i], 0, ¶m)) + ODPH_ABORT("Packet parsing failed\n"); + } +} + static void alloc_parse_packets(const void *pkt_data, uint32_t len) { int i; @@ -403,6 +470,54 @@ static void create_packets(void) gbl_args->pkt.seg_len = min_seg_len; } +static int num_misc_event_types = 4; + +static void create_events_misc(void) +{ + odp_packet_t pkt; + odp_buffer_t buf; + odp_timeout_t tmo; + odp_event_vector_t evv; + odp_event_t events[num_misc_event_types]; + int i; + + pkt = odp_packet_alloc(gbl_args->pool, gbl_args->pkt.len); + if (pkt == ODP_PACKET_INVALID) + ODPH_ABORT("Packet alloc failed\n"); + + buf = odp_buffer_alloc(gbl_args->pool_buf); + if (buf == ODP_BUFFER_INVALID) + ODPH_ABORT("Buffer alloc failed\n"); + + tmo = odp_timeout_alloc(gbl_args->pool_tmo); + if (tmo == ODP_TIMEOUT_INVALID) + ODPH_ABORT("Timeout alloc failed\n"); + + events[0] = odp_packet_to_event(pkt); + events[1] = odp_buffer_to_event(buf); + events[2] = odp_timeout_to_event(tmo); + + if (gbl_args->pool_evv != ODP_POOL_INVALID) { + evv = odp_event_vector_alloc(gbl_args->pool_evv); + if (evv == ODP_EVENT_VECTOR_INVALID) + ODPH_ABORT("Event vector alloc failed\n"); + events[3] = odp_event_vector_to_event(evv); + } else { + num_misc_event_types = 3; + } + + for (i = 0; i < num_misc_event_types; i++) + gbl_args->event_tbl[i] = events[i]; + for (; i < TEST_REPEAT_COUNT; i++) + gbl_args->event_tbl[i] = events[rnd() % num_misc_event_types]; +} + +static void free_events_misc(void) +{ + for (int i = 0; i < num_misc_event_types; i++) + odp_event_free(gbl_args->event_tbl[i]); +} + static void create_events(void) { int i; @@ -1265,6 +1380,55 @@ static int event_user_flag_set(void) return i; } +#define DEF_FLAG_TEST_FUN(name, func) \ +static int name(void) \ +{ \ + uint32_t ret = 0; \ + \ + for (int i = 0; i < TEST_REPEAT_COUNT; i++) \ + ret += func(gbl_args->pkt_tbl[i]); \ + \ + return ret <= TEST_REPEAT_COUNT; \ +} + +DEF_FLAG_TEST_FUN(packet_has_error, odp_packet_has_error) +DEF_FLAG_TEST_FUN(packet_has_l2_error, odp_packet_has_l2_error) +DEF_FLAG_TEST_FUN(packet_has_l3_error, odp_packet_has_l3_error) +DEF_FLAG_TEST_FUN(packet_has_l4_error, odp_packet_has_l4_error) +DEF_FLAG_TEST_FUN(packet_has_l2, odp_packet_has_l2) +DEF_FLAG_TEST_FUN(packet_has_l3, odp_packet_has_l3) +DEF_FLAG_TEST_FUN(packet_has_l4, odp_packet_has_l4) +DEF_FLAG_TEST_FUN(packet_has_eth, odp_packet_has_eth) +DEF_FLAG_TEST_FUN(packet_has_eth_bcast, odp_packet_has_eth_bcast) +DEF_FLAG_TEST_FUN(packet_has_eth_mcast, odp_packet_has_eth_mcast) +DEF_FLAG_TEST_FUN(packet_has_jumbo, odp_packet_has_jumbo) +DEF_FLAG_TEST_FUN(packet_has_vlan, odp_packet_has_vlan) +DEF_FLAG_TEST_FUN(packet_has_vlan_qinq, odp_packet_has_vlan_qinq) +DEF_FLAG_TEST_FUN(packet_has_arp, odp_packet_has_arp) +DEF_FLAG_TEST_FUN(packet_has_ipv4, odp_packet_has_ipv4) +DEF_FLAG_TEST_FUN(packet_has_ipv6, odp_packet_has_ipv6) +DEF_FLAG_TEST_FUN(packet_has_ip_bcast, odp_packet_has_ip_bcast) +DEF_FLAG_TEST_FUN(packet_has_ip_mcast, odp_packet_has_ip_mcast) +DEF_FLAG_TEST_FUN(packet_has_ipfrag, odp_packet_has_ipfrag) +DEF_FLAG_TEST_FUN(packet_has_ipopt, odp_packet_has_ipopt) +DEF_FLAG_TEST_FUN(packet_has_ipsec, odp_packet_has_ipsec) +DEF_FLAG_TEST_FUN(packet_has_udp, odp_packet_has_udp) +DEF_FLAG_TEST_FUN(packet_has_tcp, odp_packet_has_tcp) +DEF_FLAG_TEST_FUN(packet_has_sctp, odp_packet_has_sctp) +DEF_FLAG_TEST_FUN(packet_has_icmp, odp_packet_has_icmp) +DEF_FLAG_TEST_FUN(packet_has_flow_hash, odp_packet_has_flow_hash) +DEF_FLAG_TEST_FUN(packet_has_ts, odp_packet_has_ts) + +static int packet_l2_type(void) +{ + uint32_t ret = 0; + + for (int i = 0; i < TEST_REPEAT_COUNT; i++) + ret += odp_packet_l2_type(gbl_args->pkt_tbl[i]); + + return ret; +} + static int packet_l2_ptr(void) { int i; @@ -1298,6 +1462,16 @@ static int packet_l2_offset_set(void) return !ret; } +static int packet_l3_type(void) +{ + uint32_t ret = 0; + + for (int i = 0; i < TEST_REPEAT_COUNT; i++) + ret += odp_packet_l3_type(gbl_args->pkt_tbl[i]); + + return ret; +} + static int packet_l3_ptr(void) { int i; @@ -1331,6 +1505,16 @@ static int packet_l3_offset_set(void) return !ret; } +static int packet_l4_type(void) +{ + uint32_t ret = 0; + + for (int i = 0; i < TEST_REPEAT_COUNT; i++) + ret += odp_packet_l4_type(gbl_args->pkt_tbl[i]); + + return ret; +} + static int packet_l4_ptr(void) { int i; @@ -1478,7 +1662,7 @@ static int event_subtype(void) return i; } -static int packet_parse(void) +static int do_packet_parse(int chksum) { odp_packet_parse_param_t param; odp_packet_t *pkt_tbl = gbl_args->pkt_tbl; @@ -1488,9 +1672,9 @@ static int packet_parse(void) memset(¶m, 0, sizeof(odp_packet_parse_param_t)); param.proto = ODP_PROTO_ETH; param.last_layer = ODP_PROTO_LAYER_ALL; - param.chksums.chksum.ipv4 = 1; - param.chksums.chksum.tcp = 1; - param.chksums.chksum.udp = 1; + param.chksums.chksum.ipv4 = !!chksum; + param.chksums.chksum.tcp = !!chksum; + param.chksums.chksum.udp = !!chksum; for (i = 0; i < TEST_REPEAT_COUNT; i++) ret += odp_packet_parse(pkt_tbl[i], 0, ¶m); @@ -1498,6 +1682,16 @@ static int packet_parse(void) return !ret; } +static int packet_parse(void) +{ + return do_packet_parse(1); +} + +static int packet_parse_no_chksum(void) +{ + return do_packet_parse(0); +} + static int packet_parse_multi(void) { int burst_size = gbl_args->appl.burst_size; @@ -1699,17 +1893,56 @@ bench_info_t test_suite[] = { BENCH_INFO(packet_user_ptr_set, create_packets, free_packets, NULL), BENCH_INFO(packet_user_area, create_packets, free_packets, NULL), BENCH_INFO(event_user_area, create_events, free_packets, NULL), + BENCH_INFO(event_user_area, create_events_misc, free_events_misc, + "event_user_area misc"), BENCH_INFO(packet_user_area_size, create_packets, free_packets, NULL), BENCH_INFO(packet_user_flag, create_packets, free_packets, NULL), BENCH_INFO(event_user_area_and_flag, create_events, free_packets, NULL), + BENCH_INFO(event_user_area_and_flag, create_events_misc, free_events_misc, + "event_user_area_and_flag misc"), BENCH_INFO(packet_user_flag_set, create_packets, free_packets, NULL), BENCH_INFO(event_user_flag_set, create_events, free_packets, NULL), + BENCH_INFO(event_user_flag_set, create_events_misc, free_events_misc, + "event user_flag_set misc"), + +#define BINFO(fun) BENCH_INFO(fun, create_packets_misc_parsed, free_packets, NULL) + BINFO(packet_has_error), + BINFO(packet_has_l2_error), + BINFO(packet_has_l3_error), + BINFO(packet_has_l4_error), + BINFO(packet_has_l2), + BINFO(packet_has_l3), + BINFO(packet_has_l4), + BINFO(packet_has_eth), + BINFO(packet_has_eth_bcast), + BINFO(packet_has_eth_mcast), + BINFO(packet_has_jumbo), + BINFO(packet_has_vlan), + BINFO(packet_has_vlan_qinq), + BINFO(packet_has_arp), + BINFO(packet_has_ipv4), + BINFO(packet_has_ipv6), + BINFO(packet_has_ip_bcast), + BINFO(packet_has_ip_mcast), + BINFO(packet_has_ipfrag), + BINFO(packet_has_ipopt), + BINFO(packet_has_ipsec), + BINFO(packet_has_udp), + BINFO(packet_has_tcp), + BINFO(packet_has_sctp), + BINFO(packet_has_icmp), + BINFO(packet_has_flow_hash), + BINFO(packet_has_ts), + + BENCH_INFO(packet_l2_type, create_packets_misc_parsed, free_packets, NULL), BENCH_INFO(packet_l2_ptr, create_packets, free_packets, NULL), BENCH_INFO(packet_l2_offset, create_packets, free_packets, NULL), BENCH_INFO(packet_l2_offset_set, create_packets, free_packets, NULL), + BENCH_INFO(packet_l3_type, create_packets_misc_parsed, free_packets, NULL), BENCH_INFO(packet_l3_ptr, create_packets, free_packets, NULL), BENCH_INFO(packet_l3_offset, create_packets, free_packets, NULL), BENCH_INFO(packet_l3_offset_set, create_packets, free_packets, NULL), + BENCH_INFO(packet_l4_type, create_packets_misc_parsed, free_packets, NULL), BENCH_INFO(packet_l4_ptr, create_packets, free_packets, NULL), BENCH_INFO(packet_l4_offset, create_packets, free_packets, NULL), BENCH_INFO(packet_l4_offset_set, create_packets, free_packets, NULL), @@ -1731,6 +1964,10 @@ bench_info_t test_suite[] = { "packet_parse ipv6/tcp"), BENCH_INFO(packet_parse, alloc_parse_packets_ipv6_udp, free_packets, "packet_parse ipv6/udp"), + BENCH_INFO(packet_parse, create_packets_misc, free_packets, + "packet_parse misc"), + BENCH_INFO(packet_parse_no_chksum, create_packets_misc, free_packets, + "packet_parse misc no csum"), BENCH_INFO(packet_parse_multi, alloc_parse_packets_multi_ipv4_tcp, free_packets_multi, "packet_parse_multi ipv4/tcp"), BENCH_INFO(packet_parse_multi, alloc_parse_packets_multi_ipv4_udp, free_packets_multi, @@ -1744,6 +1981,55 @@ bench_info_t test_suite[] = { ODP_STATIC_ASSERT(ODPH_ARRAY_SIZE(test_suite) < TEST_MAX_BENCH, "Result array is too small to hold all the results"); +static odp_pool_t create_pool(const char *name, const odp_pool_param_t *param) +{ + odp_pool_t pool = odp_pool_create(name, param); + + if (pool == ODP_POOL_INVALID) { + ODPH_ERR("Error: %s pool creation failed.\n", name); + exit(EXIT_FAILURE); + } + return pool; +} + +static odp_pool_t create_buffer_pool(const odp_pool_capability_t *capa) +{ + odp_pool_param_t param; + + odp_pool_param_init(¶m); + param.type = ODP_POOL_BUFFER; + param.buf.num = 1; + param.buf.size = 1; + param.buf.uarea_size = capa->buf.max_uarea_size > 0 ? 1 : 0; + return create_pool("buf", ¶m); +} + +static odp_pool_t create_timeout_pool(const odp_pool_capability_t *capa) +{ + odp_pool_param_t param; + + odp_pool_param_init(¶m); + param.type = ODP_POOL_TIMEOUT; + param.tmo.num = 1; + param.tmo.uarea_size = capa->tmo.max_uarea_size > 0 ? 1 : 0; + return create_pool("tmo", ¶m); +} + +static odp_pool_t create_evv_pool(const odp_pool_capability_t *capa) +{ + odp_pool_param_t param; + + if (capa->event_vector.max_pools == 0) + return ODP_POOL_INVALID; + + odp_pool_param_init(¶m); + param.type = ODP_POOL_EVENT_VECTOR; + param.event_vector.num = 1; + param.event_vector.max_size = capa->event_vector.max_size; + param.event_vector.uarea_size = capa->event_vector.max_uarea_size > 0 ? 1 : 0; + return create_pool("evv", ¶m); +} + /** * ODP packet microbenchmark application */ @@ -1887,11 +2173,13 @@ int main(int argc, char *argv[]) params.type = ODP_POOL_PACKET; gbl_args->pool = odp_pool_create("packet pool", ¶ms); - if (gbl_args->pool == ODP_POOL_INVALID) { ODPH_ERR("Error: packet pool create failed.\n"); exit(EXIT_FAILURE); } + gbl_args->pool_buf = create_buffer_pool(&capa); + gbl_args->pool_tmo = create_timeout_pool(&capa); + gbl_args->pool_evv = create_evv_pool(&capa); printf("CPU: %i\n", odp_cpumask_first(&cpumask)); printf("CPU mask: %s\n", cpumaskstr); @@ -1934,7 +2222,10 @@ int main(int argc, char *argv[]) ret = gbl_args->suite.retval; - if (odp_pool_destroy(gbl_args->pool)) { + if (odp_pool_destroy(gbl_args->pool) || + odp_pool_destroy(gbl_args->pool_buf) || + odp_pool_destroy(gbl_args->pool_tmo) || + (gbl_args->pool_evv != ODP_POOL_INVALID && odp_pool_destroy(gbl_args->pool_evv))) { ODPH_ERR("Error: pool destroy\n"); exit(EXIT_FAILURE); } From 8d47722c0dd7e47e94df688ccd8d9cc22ff7a4cc Mon Sep 17 00:00:00 2001 From: Janne Peltonen Date: Fri, 30 May 2025 15:43:14 +0300 Subject: [PATCH 2/7] linux-gen: event: remove unused pool type field from event hdr Remove _odp_event_hdr_t::type which is not used for anything. Signed-off-by: Janne Peltonen --- platform/linux-generic/include/odp_event_internal.h | 3 --- platform/linux-generic/odp_pool.c | 1 - 2 files changed, 4 deletions(-) diff --git a/platform/linux-generic/include/odp_event_internal.h b/platform/linux-generic/include/odp_event_internal.h index 893b8b63c5..e1e5cd3b4e 100644 --- a/platform/linux-generic/include/odp_event_internal.h +++ b/platform/linux-generic/include/odp_event_internal.h @@ -57,9 +57,6 @@ typedef struct _odp_event_hdr_t { /* Combined pool and event index */ _odp_event_index_t index; - /* Pool type */ - int8_t type; - /* Event type. Maybe different than pool type (crypto compl event) */ int8_t event_type; diff --git a/platform/linux-generic/odp_pool.c b/platform/linux-generic/odp_pool.c index 18482a042f..b4651d83db 100644 --- a/platform/linux-generic/odp_pool.c +++ b/platform/linux-generic/odp_pool.c @@ -497,7 +497,6 @@ static void init_event_hdr(pool_t *pool, _odp_event_hdr_t *event_hdr, uint32_t e /* Initialize common event metadata */ event_hdr->index.pool = pool->pool_idx; event_hdr->index.event = event_index; - event_hdr->type = type; event_hdr->event_type = type; event_hdr->subtype = ODP_EVENT_NO_SUBTYPE; event_hdr->pool = _odp_pool_handle(pool); From 0107a9f33e6d9434ea636918cda2bee5dcc9a191 Mon Sep 17 00:00:00 2001 From: Janne Peltonen Date: Fri, 30 May 2025 15:43:15 +0300 Subject: [PATCH 3/7] linux-gen: packet: remove unused packet input flags Remove unused flags from _odp_packet_input_flags_t. The flags are set but never read. Signed-off-by: Janne Peltonen --- .../linux-generic/include/odp/api/plat/packet_inline_types.h | 3 --- platform/linux-generic/odp_classification.c | 1 - platform/linux-generic/odp_ipsec.c | 1 - platform/linux-generic/odp_parse.c | 5 +---- 4 files changed, 1 insertion(+), 9 deletions(-) diff --git a/platform/linux-generic/include/odp/api/plat/packet_inline_types.h b/platform/linux-generic/include/odp/api/plat/packet_inline_types.h index 8c5dff00be..ef7ea58093 100644 --- a/platform/linux-generic/include/odp/api/plat/packet_inline_types.h +++ b/platform/linux-generic/include/odp/api/plat/packet_inline_types.h @@ -63,7 +63,6 @@ typedef union { /* Individual input flags */ struct { - uint64_t dst_queue:1; /* Dst queue present */ uint64_t cls_mark: 1; /* Classifier mark value present*/ uint64_t flow_hash:1; /* Flow hash present */ @@ -80,7 +79,6 @@ typedef union { uint64_t vlan:1; /* VLAN hdr found */ uint64_t vlan_qinq:1; /* Stacked VLAN found, QinQ */ - uint64_t snap:1; /* SNAP */ uint64_t arp:1; /* ARP */ uint64_t ipv4:1; /* IPv4 */ @@ -106,7 +104,6 @@ typedef union { uint64_t l3_chksum_done:1; /* L3 checksum validation done */ uint64_t l4_chksum_done:1; /* L4 checksum validation done */ - uint64_t ipsec_udp:1; /* UDP-encapsulated IPsec packet */ uint64_t udp_chksum_zero:1; /* UDP header had 0 as chksum */ }; diff --git a/platform/linux-generic/odp_classification.c b/platform/linux-generic/odp_classification.c index b90e119579..66467c06df 100644 --- a/platform/linux-generic/odp_classification.c +++ b/platform/linux-generic/odp_classification.c @@ -1738,7 +1738,6 @@ int _odp_cls_classify_packet(pktio_entry_t *entry, const uint8_t *base, if (*pool == ODP_POOL_INVALID) *pool = entry->pool; - pkt_hdr->p.input_flags.dst_queue = 1; pkt_hdr->cos = cos->index; if (!cos->queue_group) { diff --git a/platform/linux-generic/odp_ipsec.c b/platform/linux-generic/odp_ipsec.c index 6e68e2eba6..2ea13463ee 100644 --- a/platform/linux-generic/odp_ipsec.c +++ b/platform/linux-generic/odp_ipsec.c @@ -2368,7 +2368,6 @@ int _odp_ipsec_try_inline(odp_packet_t *pkt) result->flag.inline_mode = 1; pkt_hdr = packet_hdr(*pkt); - pkt_hdr->p.input_flags.dst_queue = 1; pkt_hdr->dst_queue = ipsec_sa->queue; /* Distinguish inline IPsec packets from classifier packets */ pkt_hdr->cos = CLS_COS_IDX_NONE; diff --git a/platform/linux-generic/odp_parse.c b/platform/linux-generic/odp_parse.c index 60e8e5922d..98dd652a2c 100644 --- a/platform/linux-generic/odp_parse.c +++ b/platform/linux-generic/odp_parse.c @@ -59,7 +59,6 @@ uint16_t _odp_parse_eth(packet_parser_t *prs, const uint8_t **parseptr, /* Check for SNAP vs. DIX */ if (odp_unlikely(ethtype < _ODP_ETH_LEN_MAX)) { - input_flags.snap = 1; if (ethtype > frame_len - *offset) { prs->flags.snap_len_err = 1; ethtype = 0; @@ -312,10 +311,8 @@ static inline void parse_udp(packet_parser_t *prs, const uint8_t **parseptr, uint32_t val; memcpy(&val, udp + 1, 4); - if (val != 0) { + if (val != 0) prs->input_flags.ipsec = 1; - prs->input_flags.ipsec_udp = 1; - } } *parseptr += sizeof(_odp_udphdr_t); From bc7966cb5f1ee8605d88d58dddc491f2cf86f919 Mon Sep 17 00:00:00 2001 From: Janne Peltonen Date: Fri, 30 May 2025 15:43:16 +0300 Subject: [PATCH 4/7] linux-gen: packet: store L4 protocol number instead of multiple flags Remove L4 protocol flags (TCP, UDP, etc) from the packet input flags stored in the packet header and store the L4 protocol number directly. This slightly simplifies odp_packet_l4_type() and packet parsing and makes it possible to make _odp_packet_input_flags_t smaller to make room for other things in the packet header. This causes an application visible change that multiple L4 packet flags (e.g. both TCP and UDP) can no longer be set at the same time, which is probably more in line with the spirit of the ODP API, even though the old behaviour seemed to be allowed too. Signed-off-by: Janne Peltonen --- .../odp/api/plat/packet_flag_inlines.h | 25 ++++++----------- .../odp/api/plat/packet_inline_types.h | 11 ++------ .../include/odp/api/plat/packet_inlines.h | 25 ++--------------- .../include/odp_packet_internal.h | 6 +++- platform/linux-generic/odp_classification.c | 22 +++++++-------- platform/linux-generic/odp_packet.c | 17 ++++------- platform/linux-generic/odp_packet_flags.c | 23 ++++++++++++--- platform/linux-generic/odp_parse.c | 28 ++----------------- 8 files changed, 55 insertions(+), 102 deletions(-) diff --git a/platform/linux-generic/include/odp/api/plat/packet_flag_inlines.h b/platform/linux-generic/include/odp/api/plat/packet_flag_inlines.h index f075a07147..f2cec5be89 100644 --- a/platform/linux-generic/include/odp/api/plat/packet_flag_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/packet_flag_inlines.h @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: BSD-3-Clause * Copyright (c) 2017-2018 Linaro Limited - * Copyright (c) 2022-2024 Nokia + * Copyright (c) 2022-2025 Nokia */ /** @@ -12,8 +12,8 @@ #ifndef _ODP_PLAT_PACKET_FLAG_INLINES_H_ #define _ODP_PLAT_PACKET_FLAG_INLINES_H_ -#include #include +#include #include #ifdef __cplusplus @@ -225,34 +225,25 @@ _ODP_INLINE int odp_packet_has_ipopt(odp_packet_t pkt) _ODP_INLINE int odp_packet_has_udp(odp_packet_t pkt) { - _odp_packet_input_flags_t flags; - - flags.all = _odp_packet_input_flags(pkt); - return flags.udp; + return _odp_pkt_get(pkt, odp_proto_l4_type_t, l4_type) == ODP_PROTO_L4_TYPE_UDP; } _ODP_INLINE int odp_packet_has_tcp(odp_packet_t pkt) { - _odp_packet_input_flags_t flags; - - flags.all = _odp_packet_input_flags(pkt); - return flags.tcp; + return _odp_pkt_get(pkt, odp_proto_l4_type_t, l4_type) == ODP_PROTO_L4_TYPE_TCP; } _ODP_INLINE int odp_packet_has_sctp(odp_packet_t pkt) { - _odp_packet_input_flags_t flags; - - flags.all = _odp_packet_input_flags(pkt); - return flags.sctp; + return _odp_pkt_get(pkt, odp_proto_l4_type_t, l4_type) == ODP_PROTO_L4_TYPE_SCTP; } _ODP_INLINE int odp_packet_has_icmp(odp_packet_t pkt) { - _odp_packet_input_flags_t flags; + odp_proto_l4_type_t l4_type = _odp_pkt_get(pkt, odp_proto_l4_type_t, l4_type); - flags.all = _odp_packet_input_flags(pkt); - return flags.icmp; + return (l4_type == ODP_PROTO_L4_TYPE_ICMPV4 || + l4_type == ODP_PROTO_L4_TYPE_ICMPV6); } _ODP_INLINE int odp_packet_has_error(odp_packet_t pkt) diff --git a/platform/linux-generic/include/odp/api/plat/packet_inline_types.h b/platform/linux-generic/include/odp/api/plat/packet_inline_types.h index ef7ea58093..44e5b663c3 100644 --- a/platform/linux-generic/include/odp/api/plat/packet_inline_types.h +++ b/platform/linux-generic/include/odp/api/plat/packet_inline_types.h @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: BSD-3-Clause * Copyright (c) 2015-2018 Linaro Limited - * Copyright (c) 2019-2022 Nokia + * Copyright (c) 2019-2025 Nokia */ @@ -48,6 +48,7 @@ typedef struct _odp_packet_inline_offset_t { uint16_t timestamp; uint16_t input_flags; uint16_t flags; + uint16_t l4_type; uint16_t cls_mark; uint16_t ipsec_ctx; uint16_t crypto_op; @@ -90,14 +91,6 @@ typedef union { uint64_t ipsec:1; /* IPSec packet. Required by the odp_packet_has_ipsec_set() func. */ - uint64_t ipsec_ah:1; /* IPSec authentication header */ - uint64_t ipsec_esp:1; /* IPSec encapsulating security - payload */ - uint64_t udp:1; /* UDP */ - uint64_t tcp:1; /* TCP */ - uint64_t sctp:1; /* SCTP */ - uint64_t icmp:1; /* ICMP */ - uint64_t no_next_hdr:1; /* No Next Header */ uint64_t color:2; /* Packet color for traffic mgmt */ uint64_t nodrop:1; /* Drop eligibility status */ diff --git a/platform/linux-generic/include/odp/api/plat/packet_inlines.h b/platform/linux-generic/include/odp/api/plat/packet_inlines.h index 37723ae563..3d7f7b8ee6 100644 --- a/platform/linux-generic/include/odp/api/plat/packet_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/packet_inlines.h @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: BSD-3-Clause * Copyright (c) 2017-2018 Linaro Limited - * Copyright (c) 2019-2022 Nokia + * Copyright (c) 2019-2025 Nokia */ /** @@ -362,28 +362,7 @@ _ODP_INLINE odp_proto_l3_type_t odp_packet_l3_type(odp_packet_t pkt) _ODP_INLINE odp_proto_l4_type_t odp_packet_l4_type(odp_packet_t pkt) { - _odp_packet_input_flags_t input_flags; - - input_flags.all = _odp_pkt_get(pkt, uint64_t, input_flags); - - if (input_flags.tcp) - return ODP_PROTO_L4_TYPE_TCP; - else if (input_flags.udp) - return ODP_PROTO_L4_TYPE_UDP; - else if (input_flags.sctp) - return ODP_PROTO_L4_TYPE_SCTP; - else if (input_flags.ipsec_ah) - return ODP_PROTO_L4_TYPE_AH; - else if (input_flags.ipsec_esp) - return ODP_PROTO_L4_TYPE_ESP; - else if (input_flags.icmp && input_flags.ipv4) - return ODP_PROTO_L4_TYPE_ICMPV4; - else if (input_flags.icmp && input_flags.ipv6) - return ODP_PROTO_L4_TYPE_ICMPV6; - else if (input_flags.no_next_hdr) - return ODP_PROTO_L4_TYPE_NO_NEXT; - - return ODP_PROTO_L4_TYPE_NONE; + return _odp_pkt_get(pkt, uint8_t, l4_type); } _ODP_INLINE odp_packet_chksum_status_t odp_packet_l3_chksum_status(odp_packet_t pkt) diff --git a/platform/linux-generic/include/odp_packet_internal.h b/platform/linux-generic/include/odp_packet_internal.h index 60b49a89bb..3d50e3fd9e 100644 --- a/platform/linux-generic/include/odp_packet_internal.h +++ b/platform/linux-generic/include/odp_packet_internal.h @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: BSD-3-Clause * Copyright (c) 2013-2018 Linaro Limited - * Copyright (c) 2019-2022 Nokia + * Copyright (c) 2019-2025 Nokia */ /** @@ -67,6 +67,9 @@ typedef struct { /* offset to L4 hdr (TCP, UDP, SCTP, also ICMP) */ uint16_t l4_offset; + + /* l4 protocol type (TCP, UDP, ...) */ + odp_proto_l4_type_t l4_type; } packet_parser_t; /** @@ -224,6 +227,7 @@ static inline void _odp_packet_reset_md(odp_packet_hdr_t *pkt_hdr) /* Clear all flags. Resets also return value of cls_mark, user_ptr, etc. */ pkt_hdr->p.input_flags.all = 0; pkt_hdr->p.flags.all_flags = 0; + pkt_hdr->p.l4_type = ODP_PROTO_L4_TYPE_NONE; pkt_hdr->p.l2_offset = 0; pkt_hdr->p.l3_offset = ODP_PACKET_OFFSET_INVALID; diff --git a/platform/linux-generic/odp_classification.c b/platform/linux-generic/odp_classification.c index 66467c06df..5c3a7b5e44 100644 --- a/platform/linux-generic/odp_classification.c +++ b/platform/linux-generic/odp_classification.c @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: BSD-3-Clause * Copyright (c) 2014-2018 Linaro Limited - * Copyright (c) 2019-2024 Nokia + * Copyright (c) 2019-2025 Nokia */ #include @@ -1001,7 +1001,7 @@ static inline int verify_pmr_tcp_sport(const uint8_t *pkt_addr, uint16_t sport; const _odp_tcphdr_t *tcp; - if (!pkt_hdr->p.input_flags.tcp) + if (pkt_hdr->p.l4_type != ODP_PROTO_L4_TYPE_TCP) return 0; tcp = (const _odp_tcphdr_t *)(pkt_addr + pkt_hdr->p.l4_offset); sport = tcp->src_port; @@ -1018,7 +1018,7 @@ static inline int verify_pmr_tcp_dport(const uint8_t *pkt_addr, uint16_t dport; const _odp_tcphdr_t *tcp; - if (!pkt_hdr->p.input_flags.tcp) + if (pkt_hdr->p.l4_type != ODP_PROTO_L4_TYPE_TCP) return 0; tcp = (const _odp_tcphdr_t *)(pkt_addr + pkt_hdr->p.l4_offset); dport = tcp->dst_port; @@ -1035,7 +1035,7 @@ static inline int verify_pmr_udp_dport(const uint8_t *pkt_addr, uint16_t dport; const _odp_udphdr_t *udp; - if (!pkt_hdr->p.input_flags.udp) + if (pkt_hdr->p.l4_type != ODP_PROTO_L4_TYPE_UDP) return 0; udp = (const _odp_udphdr_t *)(pkt_addr + pkt_hdr->p.l4_offset); dport = udp->dst_port; @@ -1052,7 +1052,7 @@ static inline int verify_pmr_udp_sport(const uint8_t *pkt_addr, uint16_t sport; const _odp_udphdr_t *udp; - if (!pkt_hdr->p.input_flags.udp) + if (pkt_hdr->p.l4_type != ODP_PROTO_L4_TYPE_UDP) return 0; udp = (const _odp_udphdr_t *)(pkt_addr + pkt_hdr->p.l4_offset); sport = udp->src_port; @@ -1212,9 +1212,9 @@ static inline int verify_pmr_ipsec_spi(const uint8_t *pkt_addr, pkt_addr += pkt_hdr->p.l4_offset; - if (pkt_hdr->p.input_flags.ipsec_ah) + if (pkt_hdr->p.l4_type == ODP_PROTO_L4_TYPE_AH) spi = ((const _odp_ahhdr_t *)pkt_addr)->spi; - else if (pkt_hdr->p.input_flags.ipsec_esp) + else if (pkt_hdr->p.l4_type == ODP_PROTO_L4_TYPE_ESP) spi = ((const _odp_esphdr_t *)pkt_addr)->spi; else return 0; @@ -1774,14 +1774,14 @@ static uint32_t packet_rss_hash(odp_packet_hdr_t *pkt_hdr, tuple_len += 2; } - if (pkt_hdr->p.input_flags.tcp && hash_proto.tcp) { + if (hash_proto.tcp && pkt_hdr->p.l4_type == ODP_PROTO_L4_TYPE_TCP) { /* add tcp */ tcp = (const _odp_tcphdr_t *)(base + pkt_hdr->p.l4_offset); tuple.v4.sport = tcp->src_port; tuple.v4.dport = tcp->dst_port; tuple_len += 1; - } else if (pkt_hdr->p.input_flags.udp && hash_proto.udp) { + } else if (hash_proto.udp && pkt_hdr->p.l4_type == ODP_PROTO_L4_TYPE_UDP) { /* add udp */ udp = (const _odp_udphdr_t *)(base + pkt_hdr->p.l4_offset); @@ -1797,13 +1797,13 @@ static uint32_t packet_rss_hash(odp_packet_hdr_t *pkt_hdr, thash_load_ipv6_addr(ipv6, &tuple); tuple_len += 8; } - if (pkt_hdr->p.input_flags.tcp && hash_proto.tcp) { + if (hash_proto.tcp && pkt_hdr->p.l4_type == ODP_PROTO_L4_TYPE_TCP) { tcp = (const _odp_tcphdr_t *)(base + pkt_hdr->p.l4_offset); tuple.v6.sport = tcp->src_port; tuple.v6.dport = tcp->dst_port; tuple_len += 1; - } else if (pkt_hdr->p.input_flags.udp && hash_proto.udp) { + } else if (hash_proto.udp && pkt_hdr->p.l4_type == ODP_PROTO_L4_TYPE_UDP) { /* add udp */ udp = (const _odp_udphdr_t *)(base + pkt_hdr->p.l4_offset); diff --git a/platform/linux-generic/odp_packet.c b/platform/linux-generic/odp_packet.c index 114d1172e9..001cdd1d5b 100644 --- a/platform/linux-generic/odp_packet.c +++ b/platform/linux-generic/odp_packet.c @@ -65,6 +65,7 @@ const _odp_packet_inline_offset_t _odp_packet_inline ODP_ALIGNED_CACHE = { .timestamp = offsetof(odp_packet_hdr_t, timestamp), .input_flags = offsetof(odp_packet_hdr_t, p.input_flags), .flags = offsetof(odp_packet_hdr_t, p.flags), + .l4_type = offsetof(odp_packet_hdr_t, p.l4_type), .cls_mark = offsetof(odp_packet_hdr_t, cls_mark), .ipsec_ctx = offsetof(odp_packet_hdr_t, ipsec_ctx), .crypto_op = offsetof(odp_packet_hdr_t, crypto_op_result), @@ -1474,14 +1475,6 @@ static int packet_print_input_flags(odp_packet_hdr_t *hdr, char *str, int max) len += _odp_snprint(&str[len], max - len, "ipv6 "); if (hdr->p.input_flags.ipsec) len += _odp_snprint(&str[len], max - len, "ipsec "); - if (hdr->p.input_flags.udp) - len += _odp_snprint(&str[len], max - len, "udp "); - if (hdr->p.input_flags.tcp) - len += _odp_snprint(&str[len], max - len, "tcp "); - if (hdr->p.input_flags.sctp) - len += _odp_snprint(&str[len], max - len, "sctp "); - if (hdr->p.input_flags.icmp) - len += _odp_snprint(&str[len], max - len, "icmp "); return len; } @@ -1512,6 +1505,8 @@ void odp_packet_print(odp_packet_t pkt) } len += _odp_snprint(&str[len], n - len, " flags 0x%" PRIx32 "\n", hdr->p.flags.all_flags); + len += _odp_snprint(&str[len], n - len, + " l4_type %u\n", hdr->p.l4_type); len += _odp_snprint(&str[len], n - len, " cls_mark %" PRIu64 "\n", odp_packet_cls_mark(pkt)); len += _odp_snprint(&str[len], n - len, @@ -1908,7 +1903,7 @@ int _odp_packet_l4_chksum(odp_packet_hdr_t *pkt_hdr, { /* UDP chksum == 0 case is covered in parse_udp() */ if (opt.bit.udp_chksum && - pkt_hdr->p.input_flags.udp && + pkt_hdr->p.l4_type == _ODP_IPPROTO_UDP && !pkt_hdr->p.input_flags.ipfrag && !pkt_hdr->p.input_flags.udp_chksum_zero) { uint16_t sum = ~packet_sum(pkt_hdr, @@ -1929,7 +1924,7 @@ int _odp_packet_l4_chksum(odp_packet_hdr_t *pkt_hdr, } if (opt.bit.tcp_chksum && - pkt_hdr->p.input_flags.tcp && + pkt_hdr->p.l4_type == _ODP_IPPROTO_TCP && !pkt_hdr->p.input_flags.ipfrag) { uint16_t sum = ~packet_sum(pkt_hdr, pkt_hdr->p.l3_offset, @@ -1949,7 +1944,7 @@ int _odp_packet_l4_chksum(odp_packet_hdr_t *pkt_hdr, } if (opt.bit.sctp_chksum && - pkt_hdr->p.input_flags.sctp && + pkt_hdr->p.l4_type == _ODP_IPPROTO_SCTP && !pkt_hdr->p.input_flags.ipfrag) { uint32_t seg_len = 0; _odp_sctphdr_t hdr_copy; diff --git a/platform/linux-generic/odp_packet_flags.c b/platform/linux-generic/odp_packet_flags.c index f02babf4a3..5342fcec31 100644 --- a/platform/linux-generic/odp_packet_flags.c +++ b/platform/linux-generic/odp_packet_flags.c @@ -1,5 +1,6 @@ /* SPDX-License-Identifier: BSD-3-Clause * Copyright (c) 2014-2018 Linaro Limited + * Copyright (c) 2025 Nokia */ #include @@ -119,24 +120,38 @@ void odp_packet_has_ipsec_set(odp_packet_t pkt, int val) setflag(pkt, input_flags.ipsec, val); } +static inline void set_type(odp_packet_t pkt, odp_proto_l4_type_t type, int val) +{ + packet_hdr(pkt)->p.l4_type = val ? type : ODP_PROTO_L4_TYPE_NONE; +} + void odp_packet_has_udp_set(odp_packet_t pkt, int val) { - setflag(pkt, input_flags.udp, val); + set_type(pkt, ODP_PROTO_L4_TYPE_UDP, val); } void odp_packet_has_tcp_set(odp_packet_t pkt, int val) { - setflag(pkt, input_flags.tcp, val); + set_type(pkt, ODP_PROTO_L4_TYPE_TCP, val); } void odp_packet_has_sctp_set(odp_packet_t pkt, int val) { - setflag(pkt, input_flags.sctp, val); + set_type(pkt, ODP_PROTO_L4_TYPE_SCTP, val); } void odp_packet_has_icmp_set(odp_packet_t pkt, int val) { - setflag(pkt, input_flags.icmp, val); + odp_proto_l4_type_t type = ODP_PROTO_L4_TYPE_NONE; + odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); + + if (val) { + if (pkt_hdr->p.input_flags.ipv6) + type = ODP_PROTO_L4_TYPE_ICMPV6; + else + type = ODP_PROTO_L4_TYPE_ICMPV4; + } + pkt_hdr->p.l4_type = type; } void odp_packet_has_ts_clr(odp_packet_t pkt) diff --git a/platform/linux-generic/odp_parse.c b/platform/linux-generic/odp_parse.c index 98dd652a2c..67fc7e1388 100644 --- a/platform/linux-generic/odp_parse.c +++ b/platform/linux-generic/odp_parse.c @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: BSD-3-Clause * Copyright (c) 2013-2018 Linaro Limited - * Copyright (c) 2019-2022 Nokia + * Copyright (c) 2019-2025 Nokia */ #include @@ -406,26 +406,14 @@ int _odp_packet_parse_common_l3_l4(packet_parser_t *prs, if (layer == ODP_PROTO_LAYER_L3) return prs->flags.all.error != 0; - /* Set l4 flag only for known ip_proto */ prs->input_flags.l4 = 1; + prs->l4_type = ip_proto; /* Parse Layer 4 headers */ switch (ip_proto) { - case _ODP_IPPROTO_ICMPV4: - /* Fall through */ - - case _ODP_IPPROTO_ICMPV6: - prs->input_flags.icmp = 1; - break; - - case _ODP_IPPROTO_IPIP: - /* Do nothing */ - break; - case _ODP_IPPROTO_TCP: if (odp_unlikely(offset + _ODP_TCPHDR_LEN > seg_end)) return -1; - prs->input_flags.tcp = 1; parse_tcp(prs, &parseptr, frame_len - prs->l4_offset, opt, l4_part_sum); if (prs->flags.tcp_err && opt.bit.drop_tcp_err) @@ -435,36 +423,24 @@ int _odp_packet_parse_common_l3_l4(packet_parser_t *prs, case _ODP_IPPROTO_UDP: if (odp_unlikely(offset + _ODP_UDPHDR_LEN > seg_end)) return -1; - prs->input_flags.udp = 1; parse_udp(prs, &parseptr, opt, l4_part_sum); if (prs->flags.udp_err && opt.bit.drop_udp_err) return -1; /* drop */ break; case _ODP_IPPROTO_AH: - prs->input_flags.ipsec = 1; - prs->input_flags.ipsec_ah = 1; - break; - case _ODP_IPPROTO_ESP: prs->input_flags.ipsec = 1; - prs->input_flags.ipsec_esp = 1; break; case _ODP_IPPROTO_SCTP: - prs->input_flags.sctp = 1; parse_sctp(prs, &parseptr, frame_len - prs->l4_offset, opt, l4_part_sum); if (prs->flags.sctp_err && opt.bit.drop_sctp_err) return -1; /* drop */ break; - case _ODP_IPPROTO_NO_NEXT: - prs->input_flags.no_next_hdr = 1; - break; - default: - prs->input_flags.l4 = 0; break; } From 40c74498b167a4dda9382349fe1f40be39a9a172 Mon Sep 17 00:00:00 2001 From: Janne Peltonen Date: Fri, 30 May 2025 15:43:17 +0300 Subject: [PATCH 5/7] linux-gen: packet: shrink _odp_packet_input_flags_t to 32 bits Now with fewer packet input flags, shrink _odp_packet_input_flags_t from 8 to 4 bytes. This reduces the size of packet_parset_t by 8 bytes since structure padding there gets reduced too. Add explicit unused padding bytes in packet header (for now) to avoid moving other packet header fields between cache lines. Signed-off-by: Janne Peltonen --- .../odp/api/plat/packet_flag_inlines.h | 4 +- .../odp/api/plat/packet_inline_types.h | 52 +++++++++---------- .../include/odp/api/plat/packet_inlines.h | 14 ++--- .../include/odp_packet_internal.h | 4 +- 4 files changed, 38 insertions(+), 36 deletions(-) diff --git a/platform/linux-generic/include/odp/api/plat/packet_flag_inlines.h b/platform/linux-generic/include/odp/api/plat/packet_flag_inlines.h index f2cec5be89..ad6cf8f24e 100644 --- a/platform/linux-generic/include/odp/api/plat/packet_flag_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/packet_flag_inlines.h @@ -22,9 +22,9 @@ extern "C" { /** @cond _ODP_HIDE_FROM_DOXYGEN_ */ -static inline uint64_t _odp_packet_input_flags(odp_packet_t pkt) +static inline uint32_t _odp_packet_input_flags(odp_packet_t pkt) { - return _odp_pkt_get(pkt, uint64_t, input_flags); + return _odp_pkt_get(pkt, uint32_t, input_flags); } #ifndef _ODP_NO_INLINE diff --git a/platform/linux-generic/include/odp/api/plat/packet_inline_types.h b/platform/linux-generic/include/odp/api/plat/packet_inline_types.h index 44e5b663c3..19bf774fcd 100644 --- a/platform/linux-generic/include/odp/api/plat/packet_inline_types.h +++ b/platform/linux-generic/include/odp/api/plat/packet_inline_types.h @@ -60,44 +60,44 @@ extern const _odp_packet_inline_offset_t _odp_packet_inline; /* Packet input & protocol flags */ typedef union { /* All input flags */ - uint64_t all; + uint32_t all; /* Individual input flags */ struct { - uint64_t cls_mark: 1; /* Classifier mark value present*/ + uint32_t cls_mark: 1; /* Classifier mark value present*/ - uint64_t flow_hash:1; /* Flow hash present */ - uint64_t timestamp:1; /* Timestamp present */ + uint32_t flow_hash:1; /* Flow hash present */ + uint32_t timestamp:1; /* Timestamp present */ - uint64_t l2:1; /* known L2 protocol present */ - uint64_t l3:1; /* known L3 protocol present */ - uint64_t l4:1; /* known L4 protocol present */ + uint32_t l2:1; /* known L2 protocol present */ + uint32_t l3:1; /* known L3 protocol present */ + uint32_t l4:1; /* known L4 protocol present */ - uint64_t eth:1; /* Ethernet */ - uint64_t eth_bcast:1; /* Ethernet broadcast */ - uint64_t eth_mcast:1; /* Ethernet multicast */ - uint64_t jumbo:1; /* Jumbo frame */ - uint64_t vlan:1; /* VLAN hdr found */ - uint64_t vlan_qinq:1; /* Stacked VLAN found, QinQ */ + uint32_t eth:1; /* Ethernet */ + uint32_t eth_bcast:1; /* Ethernet broadcast */ + uint32_t eth_mcast:1; /* Ethernet multicast */ + uint32_t jumbo:1; /* Jumbo frame */ + uint32_t vlan:1; /* VLAN hdr found */ + uint32_t vlan_qinq:1; /* Stacked VLAN found, QinQ */ - uint64_t arp:1; /* ARP */ + uint32_t arp:1; /* ARP */ - uint64_t ipv4:1; /* IPv4 */ - uint64_t ipv6:1; /* IPv6 */ - uint64_t ip_bcast:1; /* IP broadcast */ - uint64_t ip_mcast:1; /* IP multicast */ - uint64_t ipfrag:1; /* IP fragment */ - uint64_t ipopt:1; /* IP optional headers */ + uint32_t ipv4:1; /* IPv4 */ + uint32_t ipv6:1; /* IPv6 */ + uint32_t ip_bcast:1; /* IP broadcast */ + uint32_t ip_mcast:1; /* IP multicast */ + uint32_t ipfrag:1; /* IP fragment */ + uint32_t ipopt:1; /* IP optional headers */ - uint64_t ipsec:1; /* IPSec packet. Required by the + uint32_t ipsec:1; /* IPSec packet. Required by the odp_packet_has_ipsec_set() func. */ - uint64_t color:2; /* Packet color for traffic mgmt */ - uint64_t nodrop:1; /* Drop eligibility status */ + uint32_t color:2; /* Packet color for traffic mgmt */ + uint32_t nodrop:1; /* Drop eligibility status */ - uint64_t l3_chksum_done:1; /* L3 checksum validation done */ - uint64_t l4_chksum_done:1; /* L4 checksum validation done */ - uint64_t udp_chksum_zero:1; /* UDP header had 0 as chksum */ + uint32_t l3_chksum_done:1; /* L3 checksum validation done */ + uint32_t l4_chksum_done:1; /* L4 checksum validation done */ + uint32_t udp_chksum_zero:1; /* UDP header had 0 as chksum */ }; } _odp_packet_input_flags_t; diff --git a/platform/linux-generic/include/odp/api/plat/packet_inlines.h b/platform/linux-generic/include/odp/api/plat/packet_inlines.h index 3d7f7b8ee6..5c7f203b93 100644 --- a/platform/linux-generic/include/odp/api/plat/packet_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/packet_inlines.h @@ -339,7 +339,7 @@ _ODP_INLINE odp_proto_l2_type_t odp_packet_l2_type(odp_packet_t pkt) { _odp_packet_input_flags_t input_flags; - input_flags.all = _odp_pkt_get(pkt, uint64_t, input_flags); + input_flags.all = _odp_pkt_get(pkt, uint32_t, input_flags); return input_flags.eth ? ODP_PROTO_L2_TYPE_ETH : ODP_PROTO_L2_TYPE_NONE; } @@ -348,7 +348,7 @@ _ODP_INLINE odp_proto_l3_type_t odp_packet_l3_type(odp_packet_t pkt) { _odp_packet_input_flags_t input_flags; - input_flags.all = _odp_pkt_get(pkt, uint64_t, input_flags); + input_flags.all = _odp_pkt_get(pkt, uint32_t, input_flags); if (input_flags.ipv4) return ODP_PROTO_L3_TYPE_IPV4; @@ -371,7 +371,7 @@ _ODP_INLINE odp_packet_chksum_status_t odp_packet_l3_chksum_status(odp_packet_t _odp_packet_input_flags_t input_flags; flags.all_flags = _odp_pkt_get(pkt, uint32_t, flags); - input_flags.all = _odp_pkt_get(pkt, uint64_t, input_flags); + input_flags.all = _odp_pkt_get(pkt, uint32_t, input_flags); if (!input_flags.l3_chksum_done) return ODP_PACKET_CHKSUM_UNKNOWN; @@ -388,7 +388,7 @@ _ODP_INLINE odp_packet_chksum_status_t odp_packet_l4_chksum_status(odp_packet_t _odp_packet_input_flags_t input_flags; flags.all_flags = _odp_pkt_get(pkt, uint32_t, flags); - input_flags.all = _odp_pkt_get(pkt, uint64_t, input_flags); + input_flags.all = _odp_pkt_get(pkt, uint32_t, input_flags); if (!input_flags.l4_chksum_done) return ODP_PACKET_CHKSUM_UNKNOWN; @@ -574,7 +574,7 @@ _ODP_INLINE odp_packet_color_t odp_packet_color(odp_packet_t pkt) { _odp_packet_input_flags_t input_flags; - input_flags.all = _odp_pkt_get(pkt, uint64_t, input_flags); + input_flags.all = _odp_pkt_get(pkt, uint32_t, input_flags); return (odp_packet_color_t)input_flags.color; } @@ -583,7 +583,7 @@ _ODP_INLINE odp_bool_t odp_packet_drop_eligible(odp_packet_t pkt) { _odp_packet_input_flags_t input_flags; - input_flags.all = _odp_pkt_get(pkt, uint64_t, input_flags); + input_flags.all = _odp_pkt_get(pkt, uint32_t, input_flags); return !input_flags.nodrop; } @@ -601,7 +601,7 @@ _ODP_INLINE uint64_t odp_packet_cls_mark(odp_packet_t pkt) { _odp_packet_input_flags_t input_flags; - input_flags.all = _odp_pkt_get(pkt, uint64_t, input_flags); + input_flags.all = _odp_pkt_get(pkt, uint32_t, input_flags); return input_flags.cls_mark ? _odp_pkt_get(pkt, uint16_t, cls_mark) : 0; } diff --git a/platform/linux-generic/include/odp_packet_internal.h b/platform/linux-generic/include/odp_packet_internal.h index 3d50e3fd9e..a0303856d2 100644 --- a/platform/linux-generic/include/odp_packet_internal.h +++ b/platform/linux-generic/include/odp_packet_internal.h @@ -38,7 +38,7 @@ extern "C" { #include #include -ODP_STATIC_ASSERT(sizeof(_odp_packet_input_flags_t) == sizeof(uint64_t), +ODP_STATIC_ASSERT(sizeof(_odp_packet_input_flags_t) == sizeof(uint32_t), "INPUT_FLAGS_SIZE_ERROR"); ODP_STATIC_ASSERT(sizeof(_odp_packet_flags_t) == sizeof(uint32_t), @@ -88,6 +88,8 @@ typedef struct ODP_ALIGNED_CACHE odp_packet_hdr_t { packet_parser_t p; + uint64_t unused_padding; + /* --- 64-byte cache line boundary --- */ odp_pktio_t input; From 9602c237f6e2939f8fd94c0c0732513c512b89e8 Mon Sep 17 00:00:00 2001 From: Janne Peltonen Date: Fri, 30 May 2025 15:43:18 +0300 Subject: [PATCH 6/7] linux-gen: event: move user area pointer to the common event header Add user area pointer in the common event header and remove it from event type specific headers. This simplifies and speeds up functions like odp_event_user_area() that operate on generic events. This does not change which event types support user area. Signed-off-by: Janne Peltonen --- .../odp/api/plat/buffer_inline_types.h | 13 -------- .../include/odp/api/plat/buffer_inlines.h | 4 +-- .../include/odp/api/plat/dma_inlines.h | 2 +- .../include/odp/api/plat/event_inline_types.h | 3 +- .../include/odp/api/plat/event_inlines.h | 31 ++++--------------- .../odp/api/plat/event_vector_inline_types.h | 1 - .../odp/api/plat/event_vector_inlines.h | 3 +- .../odp/api/plat/packet_inline_types.h | 1 - .../include/odp/api/plat/packet_inlines.h | 2 +- .../odp/api/plat/packet_vector_inlines.h | 5 +-- .../include/odp/api/plat/timer_inline_types.h | 1 - .../include/odp/api/plat/timer_inlines.h | 5 +-- .../include/odp_buffer_internal.h | 3 -- .../include/odp_event_internal.h | 5 ++- .../include/odp_event_vector_internal.h | 3 -- .../include/odp_packet_internal.h | 22 ++++++------- .../include/odp_timer_internal.h | 3 -- platform/linux-generic/odp_buffer.c | 9 ------ platform/linux-generic/odp_event_offsets.c | 3 +- platform/linux-generic/odp_event_vector.c | 4 +-- platform/linux-generic/odp_packet.c | 3 +- platform/linux-generic/odp_packet_vector.c | 5 +-- platform/linux-generic/odp_pool.c | 21 +------------ platform/linux-generic/odp_timer.c | 4 +-- 24 files changed, 44 insertions(+), 112 deletions(-) diff --git a/platform/linux-generic/include/odp/api/plat/buffer_inline_types.h b/platform/linux-generic/include/odp/api/plat/buffer_inline_types.h index a4843da550..3c732e85b1 100644 --- a/platform/linux-generic/include/odp/api/plat/buffer_inline_types.h +++ b/platform/linux-generic/include/odp/api/plat/buffer_inline_types.h @@ -13,19 +13,6 @@ extern "C" { /** @cond _ODP_HIDE_FROM_DOXYGEN_ */ -/* Buffer header field accessors */ -#define _odp_buffer_get(buffer_hdr, cast, field) \ - (*(cast *)(uintptr_t)((uint8_t *)buffer_hdr + \ - _odp_buffer_inline_offset.field)) - -/* Buffer header field offsets for inline functions */ -typedef struct _odp_buffer_inline_offset_t { - uint16_t uarea_addr; - -} _odp_buffer_inline_offset_t; - -extern const _odp_buffer_inline_offset_t _odp_buffer_inline_offset; - /** @endcond */ #ifdef __cplusplus diff --git a/platform/linux-generic/include/odp/api/plat/buffer_inlines.h b/platform/linux-generic/include/odp/api/plat/buffer_inlines.h index 05820256d8..cc333fe6f5 100644 --- a/platform/linux-generic/include/odp/api/plat/buffer_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/buffer_inlines.h @@ -1,5 +1,5 @@ /* SPDX-License-Identifier: BSD-3-Clause - * Copyright (c) 2019-2023 Nokia + * Copyright (c) 2019-2025 Nokia */ #ifndef ODP_PLAT_BUFFER_INLINES_H_ @@ -78,7 +78,7 @@ _ODP_INLINE odp_pool_t odp_buffer_pool(odp_buffer_t buf) _ODP_INLINE void *odp_buffer_user_area(odp_buffer_t buf) { - return _odp_buffer_get(buf, void *, uarea_addr); + return _odp_event_hdr_field((odp_event_t)(uintptr_t)buf, void *, user_area); } /** @endcond */ diff --git a/platform/linux-generic/include/odp/api/plat/dma_inlines.h b/platform/linux-generic/include/odp/api/plat/dma_inlines.h index b69f327c37..0a085b04b4 100644 --- a/platform/linux-generic/include/odp/api/plat/dma_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/dma_inlines.h @@ -51,7 +51,7 @@ _ODP_INLINE odp_event_t odp_dma_compl_to_event(odp_dma_compl_t dma_compl) _ODP_INLINE void *odp_dma_compl_user_area(odp_dma_compl_t dma_compl) { - return odp_buffer_user_area((odp_buffer_t)(uintptr_t)dma_compl); + return _odp_event_hdr_field((odp_event_t)(uintptr_t)dma_compl, void *, user_area); } _ODP_INLINE void odp_dma_transfer_param_init(odp_dma_transfer_param_t *trs_param) diff --git a/platform/linux-generic/include/odp/api/plat/event_inline_types.h b/platform/linux-generic/include/odp/api/plat/event_inline_types.h index ee22614957..3e32809840 100644 --- a/platform/linux-generic/include/odp/api/plat/event_inline_types.h +++ b/platform/linux-generic/include/odp/api/plat/event_inline_types.h @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: BSD-3-Clause * Copyright (c) 2018 Linaro Limited - * Copyright (c) 2022 Nokia + * Copyright (c) 2022-2025 Nokia */ #ifndef ODP_PLAT_EVENT_INLINE_TYPES_H_ @@ -26,6 +26,7 @@ extern "C" { typedef struct _odp_event_inline_offset_t { uint16_t event_type; uint16_t base_data; + uint16_t user_area; uint16_t subtype; uint16_t flow_id; uint16_t pool; diff --git a/platform/linux-generic/include/odp/api/plat/event_inlines.h b/platform/linux-generic/include/odp/api/plat/event_inlines.h index f9a0e6a271..51c0e8bd84 100644 --- a/platform/linux-generic/include/odp/api/plat/event_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/event_inlines.h @@ -6,18 +6,14 @@ #ifndef ODP_PLAT_EVENT_INLINES_H_ #define ODP_PLAT_EVENT_INLINES_H_ -#include #include #include #include -#include -#include #include #include #include #include -#include #ifdef __cplusplus extern "C" { @@ -99,28 +95,13 @@ _ODP_INLINE odp_pool_t odp_event_pool(odp_event_t event) _ODP_INLINE void *odp_event_user_area(odp_event_t event) { - const odp_event_type_t type = __odp_event_type_get(event); - - switch (type) { - case ODP_EVENT_BUFFER: - case ODP_EVENT_ML_COMPL: - case ODP_EVENT_DMA_COMPL: - return _odp_buffer_get((odp_buffer_t)event, void *, uarea_addr); - case ODP_EVENT_PACKET: - return _odp_pkt_get((odp_packet_t)event, void *, user_area); - case ODP_EVENT_VECTOR: - case ODP_EVENT_PACKET_VECTOR: - return _odp_event_vect_get(event, void *, uarea_addr); - case ODP_EVENT_TIMEOUT: - return _odp_timeout_hdr_field((odp_timeout_t)event, void *, uarea_addr); - default: - return NULL; - } + return _odp_event_hdr_field(event, void *, user_area); } _ODP_INLINE void *odp_event_user_area_and_flag(odp_event_t event, int *flag) { const odp_event_type_t type = __odp_event_type_get(event); + void *uarea = _odp_event_hdr_field(event, void *, user_area); _ODP_ASSERT(flag != NULL); @@ -129,7 +110,7 @@ _ODP_INLINE void *odp_event_user_area_and_flag(odp_event_t event, int *flag) case ODP_EVENT_DMA_COMPL: case ODP_EVENT_ML_COMPL: *flag = -1; - return _odp_buffer_get((odp_buffer_t)event, void *, uarea_addr); + return uarea; case ODP_EVENT_PACKET: { _odp_packet_flags_t pkt_flags; @@ -138,7 +119,7 @@ _ODP_INLINE void *odp_event_user_area_and_flag(odp_event_t event, int *flag) pkt_flags.all_flags = _odp_pkt_get(pkt, uint32_t, flags); *flag = pkt_flags.user_flag; - return _odp_pkt_get(pkt, void *, user_area); + return uarea; } case ODP_EVENT_VECTOR: case ODP_EVENT_PACKET_VECTOR: @@ -148,11 +129,11 @@ _ODP_INLINE void *odp_event_user_area_and_flag(odp_event_t event, int *flag) v_flags.all_flags = _odp_event_vect_get(event, uint32_t, flags); *flag = v_flags.user_flag; - return _odp_event_vect_get(event, void *, uarea_addr); + return uarea; } case ODP_EVENT_TIMEOUT: *flag = -1; - return _odp_timeout_hdr_field((odp_timeout_t)event, void *, uarea_addr); + return uarea; default: *flag = -1; return NULL; diff --git a/platform/linux-generic/include/odp/api/plat/event_vector_inline_types.h b/platform/linux-generic/include/odp/api/plat/event_vector_inline_types.h index d7045ec47c..c0fd9b7c88 100644 --- a/platform/linux-generic/include/odp/api/plat/event_vector_inline_types.h +++ b/platform/linux-generic/include/odp/api/plat/event_vector_inline_types.h @@ -33,7 +33,6 @@ typedef struct _odp_event_vector_inline_offset_t { uint16_t event; uint16_t pool; uint16_t size; - uint16_t uarea_addr; uint16_t flags; } _odp_event_vector_inline_offset_t; diff --git a/platform/linux-generic/include/odp/api/plat/event_vector_inlines.h b/platform/linux-generic/include/odp/api/plat/event_vector_inlines.h index c08ebbb5ec..65c24dd80c 100644 --- a/platform/linux-generic/include/odp/api/plat/event_vector_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/event_vector_inlines.h @@ -16,6 +16,7 @@ #include #include +#include #include #include @@ -81,7 +82,7 @@ _ODP_INLINE void odp_event_vector_size_set(odp_event_vector_t evv, uint32_t size _ODP_INLINE void *odp_event_vector_user_area(odp_event_vector_t evv) { - return _odp_event_vect_get(evv, void *, uarea_addr); + return _odp_event_hdr_field((odp_event_t)(uintptr_t)evv, void *, user_area); } _ODP_INLINE int odp_event_vector_user_flag(odp_event_vector_t evv) diff --git a/platform/linux-generic/include/odp/api/plat/packet_inline_types.h b/platform/linux-generic/include/odp/api/plat/packet_inline_types.h index 19bf774fcd..f6ce7347b3 100644 --- a/platform/linux-generic/include/odp/api/plat/packet_inline_types.h +++ b/platform/linux-generic/include/odp/api/plat/packet_inline_types.h @@ -40,7 +40,6 @@ typedef struct _odp_packet_inline_offset_t { uint16_t input; uint16_t seg_count; uint16_t user_ptr; - uint16_t user_area; uint16_t l2_offset; uint16_t l3_offset; uint16_t l4_offset; diff --git a/platform/linux-generic/include/odp/api/plat/packet_inlines.h b/platform/linux-generic/include/odp/api/plat/packet_inlines.h index 5c7f203b93..b81fd28dd7 100644 --- a/platform/linux-generic/include/odp/api/plat/packet_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/packet_inlines.h @@ -204,7 +204,7 @@ _ODP_INLINE void odp_packet_user_ptr_set(odp_packet_t pkt, const void *ptr) _ODP_INLINE void *odp_packet_user_area(odp_packet_t pkt) { - return _odp_pkt_get(pkt, void *, user_area); + return _odp_event_hdr_field((odp_event_t)(uintptr_t)pkt, void *, user_area); } _ODP_INLINE uint32_t odp_packet_user_area_size(odp_packet_t pkt) diff --git a/platform/linux-generic/include/odp/api/plat/packet_vector_inlines.h b/platform/linux-generic/include/odp/api/plat/packet_vector_inlines.h index f29951ef7c..72c6c0ffd6 100644 --- a/platform/linux-generic/include/odp/api/plat/packet_vector_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/packet_vector_inlines.h @@ -1,5 +1,5 @@ /* SPDX-License-Identifier: BSD-3-Clause - * Copyright (c) 2020-2022 Nokia + * Copyright (c) 2020-2025 Nokia */ /** @@ -16,6 +16,7 @@ #include #include +#include #include #include @@ -81,7 +82,7 @@ _ODP_INLINE void odp_packet_vector_size_set(odp_packet_vector_t pktv, uint32_t s _ODP_INLINE void *odp_packet_vector_user_area(odp_packet_vector_t pktv) { - return _odp_event_vect_get(pktv, void *, uarea_addr); + return _odp_event_hdr_field((odp_event_t)(uintptr_t)pktv, void *, user_area); } _ODP_INLINE int odp_packet_vector_user_flag(odp_packet_vector_t pktv) diff --git a/platform/linux-generic/include/odp/api/plat/timer_inline_types.h b/platform/linux-generic/include/odp/api/plat/timer_inline_types.h index 249b67bbef..fe881f63ff 100644 --- a/platform/linux-generic/include/odp/api/plat/timer_inline_types.h +++ b/platform/linux-generic/include/odp/api/plat/timer_inline_types.h @@ -23,7 +23,6 @@ typedef struct _odp_timeout_inline_offset_t { uint16_t expiration; uint16_t timer; uint16_t user_ptr; - uint16_t uarea_addr; } _odp_timeout_inline_offset_t; diff --git a/platform/linux-generic/include/odp/api/plat/timer_inlines.h b/platform/linux-generic/include/odp/api/plat/timer_inlines.h index 32bfb20dd3..e3edb304d8 100644 --- a/platform/linux-generic/include/odp/api/plat/timer_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/timer_inlines.h @@ -1,5 +1,5 @@ /* SPDX-License-Identifier: BSD-3-Clause - * Copyright (c) 2022-2023 Nokia + * Copyright (c) 2022-2025 Nokia */ #ifndef ODP_PLAT_TIMER_INLINES_H_ @@ -11,6 +11,7 @@ #include #include +#include #include #include @@ -55,7 +56,7 @@ _ODP_INLINE void *odp_timeout_user_ptr(odp_timeout_t tmo) _ODP_INLINE void *odp_timeout_user_area(odp_timeout_t tmo) { - return _odp_timeout_hdr_field(tmo, void *, uarea_addr); + return _odp_event_hdr_field((odp_event_t)(uintptr_t)tmo, void *, user_area); } _ODP_INLINE uint64_t odp_timer_current_tick(odp_timer_pool_t tpid) diff --git a/platform/linux-generic/include/odp_buffer_internal.h b/platform/linux-generic/include/odp_buffer_internal.h index 92b6beb123..e325c0792b 100644 --- a/platform/linux-generic/include/odp_buffer_internal.h +++ b/platform/linux-generic/include/odp_buffer_internal.h @@ -38,9 +38,6 @@ typedef struct ODP_ALIGNED_CACHE odp_buffer_hdr_t { /* Common event header */ _odp_event_hdr_t event_hdr; - /* User area pointer */ - void *uarea_addr; - /* Data */ uint8_t data[]; } odp_buffer_hdr_t; diff --git a/platform/linux-generic/include/odp_event_internal.h b/platform/linux-generic/include/odp_event_internal.h index e1e5cd3b4e..c3a39020a7 100644 --- a/platform/linux-generic/include/odp_event_internal.h +++ b/platform/linux-generic/include/odp_event_internal.h @@ -1,5 +1,5 @@ /* SPDX-License-Identifier: BSD-3-Clause - * Copyright (c) 2021-2022 Nokia + * Copyright (c) 2021-2025 Nokia */ /** @@ -54,6 +54,9 @@ typedef struct _odp_event_hdr_t { /* Initial buffer tail pointer and endmark location (if enabled) */ uint8_t *buf_end; + /* User area pointer */ + uint8_t *user_area; + /* Combined pool and event index */ _odp_event_index_t index; diff --git a/platform/linux-generic/include/odp_event_vector_internal.h b/platform/linux-generic/include/odp_event_vector_internal.h index 5c6913d297..97a37858ba 100644 --- a/platform/linux-generic/include/odp_event_vector_internal.h +++ b/platform/linux-generic/include/odp_event_vector_internal.h @@ -29,9 +29,6 @@ typedef struct ODP_ALIGNED_CACHE odp_event_vector_hdr_t { /* Common event header */ _odp_event_hdr_t event_hdr; - /* User area pointer */ - void *uarea_addr; - /* Event vector size */ uint32_t size; diff --git a/platform/linux-generic/include/odp_packet_internal.h b/platform/linux-generic/include/odp_packet_internal.h index a0303856d2..c994d816d9 100644 --- a/platform/linux-generic/include/odp_packet_internal.h +++ b/platform/linux-generic/include/odp_packet_internal.h @@ -88,8 +88,6 @@ typedef struct ODP_ALIGNED_CACHE odp_packet_hdr_t { packet_parser_t p; - uint64_t unused_padding; - /* --- 64-byte cache line boundary --- */ odp_pktio_t input; @@ -123,17 +121,14 @@ typedef struct ODP_ALIGNED_CACHE odp_packet_hdr_t { /* Flow hash value */ uint32_t flow_hash; - /* User area pointer */ - void *uarea_addr; - /* User context pointer */ const void *user_ptr; - /* --- 64-byte cache line boundary --- */ - /* Timestamp value */ odp_time_t timestamp; + /* --- 64-byte cache line boundary --- */ + /* Classifier mark */ uint16_t cls_mark; @@ -349,26 +344,27 @@ static inline void _odp_packet_copy_md(odp_packet_hdr_t *dst_hdr, dst_hdr->p = src_hdr->p; - if (src_hdr->uarea_addr) { + if (src_hdr->event_hdr.user_area) { if (uarea_copy) { const pool_t *src_pool = _odp_pool_entry(src_hdr->event_hdr.pool); const pool_t *dst_pool = _odp_pool_entry(dst_hdr->event_hdr.pool); const uint32_t src_uarea_size = src_pool->param_uarea_size; const uint32_t dst_uarea_size = dst_pool->param_uarea_size; - _ODP_ASSERT(dst_hdr->uarea_addr != NULL); + _ODP_ASSERT(dst_hdr->event_hdr.user_area != NULL); _ODP_ASSERT(dst_uarea_size >= src_uarea_size); - memcpy(dst_hdr->uarea_addr, src_hdr->uarea_addr, src_uarea_size); + memcpy(dst_hdr->event_hdr.user_area, + src_hdr->event_hdr.user_area, src_uarea_size); } else { - void *src_uarea = src_hdr->uarea_addr; + void *src_uarea = src_hdr->event_hdr.user_area; /* If user area exists, packets should always be from the same pool, so * user area pointers can simply be swapped. */ _ODP_ASSERT(dst_hdr->event_hdr.pool == src_hdr->event_hdr.pool); - src_hdr->uarea_addr = dst_hdr->uarea_addr; - dst_hdr->uarea_addr = src_uarea; + src_hdr->event_hdr.user_area = dst_hdr->event_hdr.user_area; + dst_hdr->event_hdr.user_area = src_uarea; } } diff --git a/platform/linux-generic/include/odp_timer_internal.h b/platform/linux-generic/include/odp_timer_internal.h index 7059db0c49..de71d9d6fd 100644 --- a/platform/linux-generic/include/odp_timer_internal.h +++ b/platform/linux-generic/include/odp_timer_internal.h @@ -39,9 +39,6 @@ typedef struct ODP_ALIGNED_CACHE odp_timeout_hdr_t { /* User ptr inherited from parent timer */ const void *user_ptr; - /* User area pointer */ - void *uarea_addr; - /* Parent timer */ odp_timer_t timer; diff --git a/platform/linux-generic/odp_buffer.c b/platform/linux-generic/odp_buffer.c index a319f96e4b..05d6f7ea1a 100644 --- a/platform/linux-generic/odp_buffer.c +++ b/platform/linux-generic/odp_buffer.c @@ -17,15 +17,6 @@ #include #include -#include - -/* Buffer header field offsets for inline functions */ -const _odp_buffer_inline_offset_t _odp_buffer_inline_offset ODP_ALIGNED_CACHE = { - .uarea_addr = offsetof(odp_buffer_hdr_t, uarea_addr) -}; - -#include - void odp_buffer_print(odp_buffer_t buf) { int len = 0; diff --git a/platform/linux-generic/odp_event_offsets.c b/platform/linux-generic/odp_event_offsets.c index f8826d4c46..6c2d9a4e52 100644 --- a/platform/linux-generic/odp_event_offsets.c +++ b/platform/linux-generic/odp_event_offsets.c @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: BSD-3-Clause * Copyright (c) 2015-2018 Linaro Limited - * Copyright (c) 2020-2024 Nokia + * Copyright (c) 2020-2025 Nokia */ #include @@ -14,6 +14,7 @@ const _odp_event_inline_offset_t _odp_event_inline_offset ODP_ALIGNED_CACHE = { .event_type = offsetof(_odp_event_hdr_t, event_type), .base_data = offsetof(_odp_event_hdr_t, base_data), + .user_area = offsetof(_odp_event_hdr_t, user_area), .subtype = offsetof(_odp_event_hdr_t, subtype), .flow_id = offsetof(_odp_event_hdr_t, flow_id), .pool = offsetof(_odp_event_hdr_t, pool), diff --git a/platform/linux-generic/odp_event_vector.c b/platform/linux-generic/odp_event_vector.c index 4aeb8bba5c..78a77c2f02 100644 --- a/platform/linux-generic/odp_event_vector.c +++ b/platform/linux-generic/odp_event_vector.c @@ -24,7 +24,6 @@ const _odp_event_vector_inline_offset_t _odp_event_vector_inline ODP_ALIGNED_CAC .event = offsetof(odp_event_vector_hdr_t, event), .pool = offsetof(odp_event_vector_hdr_t, event_hdr.pool), .size = offsetof(odp_event_vector_hdr_t, size), - .uarea_addr = offsetof(odp_event_vector_hdr_t, uarea_addr), .flags = offsetof(odp_event_vector_hdr_t, flags) }; @@ -109,7 +108,8 @@ void odp_event_vector_print(odp_event_vector_t evv) len += _odp_snprint(&str[len], n - len, " size %" PRIu32 "\n", evv_hdr->size); len += _odp_snprint(&str[len], n - len, " flags 0x%" PRIx32 "\n", evv_hdr->flags.all_flags); - len += _odp_snprint(&str[len], n - len, " user area %p\n", evv_hdr->uarea_addr); + len += _odp_snprint(&str[len], n - len, " user area %p\n", + evv_hdr->event_hdr.user_area); for (i = 0; i < evv_hdr->size; i++) { odp_event_t ev = evv_hdr->event[i]; diff --git a/platform/linux-generic/odp_packet.c b/platform/linux-generic/odp_packet.c index 001cdd1d5b..2d8bee8941 100644 --- a/platform/linux-generic/odp_packet.c +++ b/platform/linux-generic/odp_packet.c @@ -57,7 +57,6 @@ const _odp_packet_inline_offset_t _odp_packet_inline ODP_ALIGNED_CACHE = { .input = offsetof(odp_packet_hdr_t, input), .seg_count = offsetof(odp_packet_hdr_t, seg_count), .user_ptr = offsetof(odp_packet_hdr_t, user_ptr), - .user_area = offsetof(odp_packet_hdr_t, uarea_addr), .l2_offset = offsetof(odp_packet_hdr_t, p.l2_offset), .l3_offset = offsetof(odp_packet_hdr_t, p.l3_offset), .l4_offset = offsetof(odp_packet_hdr_t, p.l4_offset), @@ -1512,7 +1511,7 @@ void odp_packet_print(odp_packet_t pkt) len += _odp_snprint(&str[len], n - len, " user ptr %p\n", hdr->user_ptr); len += _odp_snprint(&str[len], n - len, - " user area %p\n", hdr->uarea_addr); + " user area %p\n", hdr->event_hdr.user_area); len += _odp_snprint(&str[len], n - len, " l2_offset %" PRIu32 "\n", hdr->p.l2_offset); len += _odp_snprint(&str[len], n - len, diff --git a/platform/linux-generic/odp_packet_vector.c b/platform/linux-generic/odp_packet_vector.c index efc40581a1..1a5d2bbc6b 100644 --- a/platform/linux-generic/odp_packet_vector.c +++ b/platform/linux-generic/odp_packet_vector.c @@ -1,5 +1,5 @@ /* SPDX-License-Identifier: BSD-3-Clause - * Copyright (c) 2020-2022 Nokia + * Copyright (c) 2020-2025 Nokia */ #include @@ -100,7 +100,8 @@ void odp_packet_vector_print(odp_packet_vector_t pktv) len += _odp_snprint(&str[len], n - len, " size %" PRIu32 "\n", pktv_hdr->size); len += _odp_snprint(&str[len], n - len, " flags 0x%" PRIx32 "\n", pktv_hdr->flags.all_flags); - len += _odp_snprint(&str[len], n - len, " user area %p\n", pktv_hdr->uarea_addr); + len += _odp_snprint(&str[len], n - len, " user area %p\n", + pktv_hdr->event_hdr.user_area); for (i = 0; i < pktv_hdr->size; i++) { odp_packet_t pkt = odp_packet_from_event(pktv_hdr->event[i]); diff --git a/platform/linux-generic/odp_pool.c b/platform/linux-generic/odp_pool.c index b4651d83db..fdfde2a5f9 100644 --- a/platform/linux-generic/odp_pool.c +++ b/platform/linux-generic/odp_pool.c @@ -500,6 +500,7 @@ static void init_event_hdr(pool_t *pool, _odp_event_hdr_t *event_hdr, uint32_t e event_hdr->event_type = type; event_hdr->subtype = ODP_EVENT_NO_SUBTYPE; event_hdr->pool = _odp_pool_handle(pool); + event_hdr->user_area = uarea; /* Store base values for fast init */ if (type == ODP_POOL_BUFFER || type == ODP_POOL_PACKET) { @@ -508,18 +509,11 @@ static void init_event_hdr(pool_t *pool, _odp_event_hdr_t *event_hdr, uint32_t e _odp_event_endmark_set(_odp_event_from_hdr(event_hdr)); } - if (type == ODP_POOL_BUFFER) { - odp_buffer_hdr_t *buf_hdr = (void *)event_hdr; - - buf_hdr->uarea_addr = uarea; - } - /* Initialize segmentation metadata */ if (type == ODP_POOL_PACKET) { odp_packet_hdr_t *pkt_hdr = (void *)event_hdr; pkt_hdr->user_ptr = NULL; - pkt_hdr->uarea_addr = uarea; pkt_hdr->seg_data = data_ptr; pkt_hdr->seg_len = pool->seg_len; pkt_hdr->seg_count = 1; @@ -530,25 +524,12 @@ static void init_event_hdr(pool_t *pool, _odp_event_hdr_t *event_hdr, uint32_t e /* Initialize packet vector metadata */ if (type == ODP_POOL_VECTOR) { - odp_event_vector_hdr_t *vect_hdr = (void *)event_hdr; - event_hdr->event_type = ODP_EVENT_PACKET_VECTOR; - vect_hdr->uarea_addr = uarea; } /* Initialize event vector metadata */ if (type == ODP_POOL_EVENT_VECTOR) { - odp_event_vector_hdr_t *vect_hdr = (void *)event_hdr; - event_hdr->event_type = ODP_EVENT_VECTOR; - vect_hdr->uarea_addr = uarea; - } - - /* Initialize timeout metadata */ - if (type == ODP_POOL_TIMEOUT) { - odp_timeout_hdr_t *tmo_hdr = (void *)event_hdr; - - tmo_hdr->uarea_addr = uarea; } } diff --git a/platform/linux-generic/odp_timer.c b/platform/linux-generic/odp_timer.c index d9bd4a85a1..a8eee786e2 100644 --- a/platform/linux-generic/odp_timer.c +++ b/platform/linux-generic/odp_timer.c @@ -114,7 +114,6 @@ _odp_timeout_inline_offset ODP_ALIGNED_CACHE = { .expiration = offsetof(odp_timeout_hdr_t, expiration), .timer = offsetof(odp_timeout_hdr_t, timer), .user_ptr = offsetof(odp_timeout_hdr_t, user_ptr), - .uarea_addr = offsetof(odp_timeout_hdr_t, uarea_addr), }; #include @@ -1968,7 +1967,8 @@ void odp_timeout_print(odp_timeout_t tmo) len += _odp_snprint(&str[len], n - len, " expiration %" PRIu64 "\n", tmo_hdr->expiration); len += _odp_snprint(&str[len], n - len, " user ptr %p\n", tmo_hdr->user_ptr); - len += _odp_snprint(&str[len], n - len, " user area %p\n", tmo_hdr->uarea_addr); + len += _odp_snprint(&str[len], n - len, " user area %p\n", + tmo_hdr->event_hdr.user_area); if (timer != ODP_TIMER_INVALID) { timer_pool_t *tp = handle_to_tp(timer); From 51619e370fb48db661b77d3e333ed38608815264 Mon Sep 17 00:00:00 2001 From: Janne Peltonen Date: Fri, 30 May 2025 15:43:21 +0300 Subject: [PATCH 7/7] linux-gen: event: move user flag to the common event header Add user flag in the common event header and remove it from event type specific headers. This simplifies and speeds up functions like odp_event_user_flag() that operate on generic events. This does not change which event types have a user flag visible through ODP API. Signed-off-by: Janne Peltonen --- .../include/odp/api/plat/event_inline_types.h | 1 + .../include/odp/api/plat/event_inlines.h | 68 ++----------------- .../odp/api/plat/event_vector_inline_types.h | 10 --- .../odp/api/plat/event_vector_inlines.h | 12 ++-- .../odp/api/plat/packet_inline_types.h | 7 +- .../include/odp/api/plat/packet_inlines.h | 11 ++- .../odp/api/plat/packet_vector_inlines.h | 12 ++-- .../include/odp_event_internal.h | 3 + .../include/odp_event_vector_internal.h | 3 - .../include/odp_packet_internal.h | 8 +-- platform/linux-generic/odp_event_offsets.c | 1 + platform/linux-generic/odp_event_vector.c | 9 ++- platform/linux-generic/odp_packet.c | 2 + platform/linux-generic/odp_packet_vector.c | 8 +-- platform/linux-generic/odp_pool.c | 3 + 15 files changed, 42 insertions(+), 116 deletions(-) diff --git a/platform/linux-generic/include/odp/api/plat/event_inline_types.h b/platform/linux-generic/include/odp/api/plat/event_inline_types.h index 3e32809840..98163035ad 100644 --- a/platform/linux-generic/include/odp/api/plat/event_inline_types.h +++ b/platform/linux-generic/include/odp/api/plat/event_inline_types.h @@ -30,6 +30,7 @@ typedef struct _odp_event_inline_offset_t { uint16_t subtype; uint16_t flow_id; uint16_t pool; + uint16_t user_flag; } _odp_event_inline_offset_t; diff --git a/platform/linux-generic/include/odp/api/plat/event_inlines.h b/platform/linux-generic/include/odp/api/plat/event_inlines.h index 51c0e8bd84..2dd7975a13 100644 --- a/platform/linux-generic/include/odp/api/plat/event_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/event_inlines.h @@ -7,13 +7,10 @@ #define ODP_PLAT_EVENT_INLINES_H_ #include -#include #include #include #include -#include -#include #ifdef __cplusplus extern "C" { @@ -100,72 +97,17 @@ _ODP_INLINE void *odp_event_user_area(odp_event_t event) _ODP_INLINE void *odp_event_user_area_and_flag(odp_event_t event, int *flag) { - const odp_event_type_t type = __odp_event_type_get(event); - void *uarea = _odp_event_hdr_field(event, void *, user_area); - _ODP_ASSERT(flag != NULL); - - switch (type) { - case ODP_EVENT_BUFFER: - case ODP_EVENT_DMA_COMPL: - case ODP_EVENT_ML_COMPL: - *flag = -1; - return uarea; - case ODP_EVENT_PACKET: - { - _odp_packet_flags_t pkt_flags; - odp_packet_t pkt = (odp_packet_t)event; - - pkt_flags.all_flags = _odp_pkt_get(pkt, uint32_t, flags); - *flag = pkt_flags.user_flag; - - return uarea; - } - case ODP_EVENT_VECTOR: - case ODP_EVENT_PACKET_VECTOR: - { - _odp_event_vector_flags_t v_flags; - - v_flags.all_flags = _odp_event_vect_get(event, uint32_t, flags); - *flag = v_flags.user_flag; - - return uarea; - } - case ODP_EVENT_TIMEOUT: - *flag = -1; - return uarea; - default: - *flag = -1; - return NULL; - } + *flag = _odp_event_hdr_field(event, int8_t, user_flag); + return _odp_event_hdr_field(event, void *, user_area); } _ODP_INLINE void odp_event_user_flag_set(odp_event_t event, int val) { - const odp_event_type_t type = __odp_event_type_get(event); - - switch (type) { - case ODP_EVENT_PACKET: - { - odp_packet_t pkt = (odp_packet_t)event; - _odp_packet_flags_t *flags = _odp_pkt_get_ptr(pkt, _odp_packet_flags_t, flags); + int8_t *user_flag = _odp_event_hdr_ptr(event, int8_t, user_flag); - flags->user_flag = !!val; - return; - } - case ODP_EVENT_VECTOR: - case ODP_EVENT_PACKET_VECTOR: - { - _odp_event_vector_flags_t *flags = - _odp_event_vect_get_ptr(event, _odp_event_vector_flags_t, flags); - - flags->user_flag = !!val; - return; - } - default: - /* Nothing to do */ - return; - } + if (odp_likely(*user_flag != -1)) + *user_flag = !!val; } _ODP_INLINE odp_event_subtype_t odp_event_subtype(odp_event_t event) diff --git a/platform/linux-generic/include/odp/api/plat/event_vector_inline_types.h b/platform/linux-generic/include/odp/api/plat/event_vector_inline_types.h index c0fd9b7c88..27c0c341aa 100644 --- a/platform/linux-generic/include/odp/api/plat/event_vector_inline_types.h +++ b/platform/linux-generic/include/odp/api/plat/event_vector_inline_types.h @@ -13,15 +13,6 @@ extern "C" { /** @cond _ODP_HIDE_FROM_DOXYGEN_ */ -typedef union { - uint32_t all_flags; - - struct { - uint32_t user_flag : 1; - }; - -} _odp_event_vector_flags_t; - /* Event vector field accessors */ #define _odp_event_vect_get(vect, cast, field) \ (*(cast *)(uintptr_t)((uint8_t *)vect + _odp_event_vector_inline.field)) @@ -33,7 +24,6 @@ typedef struct _odp_event_vector_inline_offset_t { uint16_t event; uint16_t pool; uint16_t size; - uint16_t flags; } _odp_event_vector_inline_offset_t; diff --git a/platform/linux-generic/include/odp/api/plat/event_vector_inlines.h b/platform/linux-generic/include/odp/api/plat/event_vector_inlines.h index 65c24dd80c..6cbf12ed2e 100644 --- a/platform/linux-generic/include/odp/api/plat/event_vector_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/event_vector_inlines.h @@ -87,19 +87,15 @@ _ODP_INLINE void *odp_event_vector_user_area(odp_event_vector_t evv) _ODP_INLINE int odp_event_vector_user_flag(odp_event_vector_t evv) { - _odp_event_vector_flags_t flags; - - flags.all_flags = _odp_event_vect_get(evv, uint32_t, flags); - - return flags.user_flag; + return _odp_event_hdr_field((odp_event_t)(uintptr_t)evv, int8_t, user_flag); } _ODP_INLINE void odp_event_vector_user_flag_set(odp_event_vector_t evv, int val) { - _odp_event_vector_flags_t *flags = _odp_event_vect_get_ptr(evv, _odp_event_vector_flags_t, - flags); + odp_event_t event = (odp_event_t)(uintptr_t)evv; + int8_t *user_flag = _odp_event_hdr_ptr(event, int8_t, user_flag); - flags->user_flag = !!val; + *user_flag = !!val; } /** @endcond */ diff --git a/platform/linux-generic/include/odp/api/plat/packet_inline_types.h b/platform/linux-generic/include/odp/api/plat/packet_inline_types.h index f6ce7347b3..1de068ac0b 100644 --- a/platform/linux-generic/include/odp/api/plat/packet_inline_types.h +++ b/platform/linux-generic/include/odp/api/plat/packet_inline_types.h @@ -109,13 +109,12 @@ typedef union { uint32_t all_flags; struct { - uint32_t reserved1: 4; + uint32_t reserved1: 5; /* * Init flags */ uint32_t user_ptr_set: 1; /* User has set a non-NULL value */ - uint32_t user_flag: 1; /* * Packet output flags @@ -147,8 +146,8 @@ typedef union { /* Flag groups */ struct { - uint32_t reserved2: 4; - uint32_t other: 21; /* All other flags */ + uint32_t reserved2: 5; + uint32_t other: 20; /* All other flags */ uint32_t error: 7; /* All error flags */ } all; diff --git a/platform/linux-generic/include/odp/api/plat/packet_inlines.h b/platform/linux-generic/include/odp/api/plat/packet_inlines.h index b81fd28dd7..e28bd6dccd 100644 --- a/platform/linux-generic/include/odp/api/plat/packet_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/packet_inlines.h @@ -216,18 +216,15 @@ _ODP_INLINE uint32_t odp_packet_user_area_size(odp_packet_t pkt) _ODP_INLINE int odp_packet_user_flag(odp_packet_t pkt) { - _odp_packet_flags_t flags; - - flags.all_flags = _odp_pkt_get(pkt, uint32_t, flags); - - return flags.user_flag; + return _odp_event_hdr_field((odp_event_t)(uintptr_t)pkt, int8_t, user_flag); } _ODP_INLINE void odp_packet_user_flag_set(odp_packet_t pkt, int val) { - _odp_packet_flags_t *flags = _odp_pkt_get_ptr(pkt, _odp_packet_flags_t, flags); + odp_event_t event = (odp_event_t)(uintptr_t)pkt; + int8_t *user_flag = _odp_event_hdr_ptr(event, int8_t, user_flag); - flags->user_flag = !!val; + *user_flag = !!val; } _ODP_INLINE uint32_t odp_packet_l2_offset(odp_packet_t pkt) diff --git a/platform/linux-generic/include/odp/api/plat/packet_vector_inlines.h b/platform/linux-generic/include/odp/api/plat/packet_vector_inlines.h index 72c6c0ffd6..806ef7a6f8 100644 --- a/platform/linux-generic/include/odp/api/plat/packet_vector_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/packet_vector_inlines.h @@ -87,19 +87,15 @@ _ODP_INLINE void *odp_packet_vector_user_area(odp_packet_vector_t pktv) _ODP_INLINE int odp_packet_vector_user_flag(odp_packet_vector_t pktv) { - _odp_event_vector_flags_t flags; - - flags.all_flags = _odp_event_vect_get(pktv, uint32_t, flags); - - return flags.user_flag; + return _odp_event_hdr_field((odp_event_t)(uintptr_t)pktv, int8_t, user_flag); } _ODP_INLINE void odp_packet_vector_user_flag_set(odp_packet_vector_t pktv, int val) { - _odp_event_vector_flags_t *flags = _odp_event_vect_get_ptr(pktv, _odp_event_vector_flags_t, - flags); + odp_event_t event = (odp_event_t)(uintptr_t)pktv; + int8_t *user_flag = _odp_event_hdr_ptr(event, int8_t, user_flag); - flags->user_flag = !!val; + *user_flag = !!val; } /** @endcond */ diff --git a/platform/linux-generic/include/odp_event_internal.h b/platform/linux-generic/include/odp_event_internal.h index c3a39020a7..faa3d69510 100644 --- a/platform/linux-generic/include/odp_event_internal.h +++ b/platform/linux-generic/include/odp_event_internal.h @@ -69,6 +69,9 @@ typedef struct _odp_event_hdr_t { /* Event flow id */ uint8_t flow_id; + /* User flag (0 or 1, or -1 if not supported) */ + int8_t user_flag; + } _odp_event_hdr_t; static inline odp_event_t _odp_event_from_hdr(_odp_event_hdr_t *hdr) diff --git a/platform/linux-generic/include/odp_event_vector_internal.h b/platform/linux-generic/include/odp_event_vector_internal.h index 97a37858ba..85047d0ac3 100644 --- a/platform/linux-generic/include/odp_event_vector_internal.h +++ b/platform/linux-generic/include/odp_event_vector_internal.h @@ -32,9 +32,6 @@ typedef struct ODP_ALIGNED_CACHE odp_event_vector_hdr_t { /* Event vector size */ uint32_t size; - /* Flags */ - _odp_event_vector_flags_t flags; - /* Vector of event handles */ odp_event_t event[]; diff --git a/platform/linux-generic/include/odp_packet_internal.h b/platform/linux-generic/include/odp_packet_internal.h index c994d816d9..5379bbaa55 100644 --- a/platform/linux-generic/include/odp_packet_internal.h +++ b/platform/linux-generic/include/odp_packet_internal.h @@ -221,7 +221,7 @@ static inline void packet_subtype_set(odp_packet_t pkt, int subtype) */ static inline void _odp_packet_reset_md(odp_packet_hdr_t *pkt_hdr) { - /* Clear all flags. Resets also return value of cls_mark, user_ptr, etc. */ + /* Clear all flags. Resets also return value of cls_mark, etc. */ pkt_hdr->p.input_flags.all = 0; pkt_hdr->p.flags.all_flags = 0; pkt_hdr->p.l4_type = ODP_PROTO_L4_TYPE_NONE; @@ -230,9 +230,8 @@ static inline void _odp_packet_reset_md(odp_packet_hdr_t *pkt_hdr) pkt_hdr->p.l3_offset = ODP_PACKET_OFFSET_INVALID; pkt_hdr->p.l4_offset = ODP_PACKET_OFFSET_INVALID; - if (odp_unlikely(pkt_hdr->event_hdr.subtype != ODP_EVENT_PACKET_BASIC)) - pkt_hdr->event_hdr.subtype = ODP_EVENT_PACKET_BASIC; - + pkt_hdr->event_hdr.subtype = ODP_EVENT_PACKET_BASIC; + pkt_hdr->event_hdr.user_flag = 0; pkt_hdr->input = ODP_PKTIO_INVALID; } @@ -323,6 +322,7 @@ static inline void _odp_packet_copy_md(odp_packet_hdr_t *dst_hdr, */ dst_hdr->input = src_hdr->input; dst_hdr->event_hdr.subtype = subtype; + dst_hdr->event_hdr.user_flag = src_hdr->event_hdr.user_flag; dst_hdr->dst_queue = src_hdr->dst_queue; dst_hdr->cos = src_hdr->cos; dst_hdr->cls_mark = src_hdr->cls_mark; diff --git a/platform/linux-generic/odp_event_offsets.c b/platform/linux-generic/odp_event_offsets.c index 6c2d9a4e52..5efe77a536 100644 --- a/platform/linux-generic/odp_event_offsets.c +++ b/platform/linux-generic/odp_event_offsets.c @@ -18,6 +18,7 @@ _odp_event_inline_offset ODP_ALIGNED_CACHE = { .subtype = offsetof(_odp_event_hdr_t, subtype), .flow_id = offsetof(_odp_event_hdr_t, flow_id), .pool = offsetof(_odp_event_hdr_t, pool), + .user_flag = offsetof(_odp_event_hdr_t, user_flag), }; #include diff --git a/platform/linux-generic/odp_event_vector.c b/platform/linux-generic/odp_event_vector.c index 78a77c2f02..d9264b85e7 100644 --- a/platform/linux-generic/odp_event_vector.c +++ b/platform/linux-generic/odp_event_vector.c @@ -24,7 +24,6 @@ const _odp_event_vector_inline_offset_t _odp_event_vector_inline ODP_ALIGNED_CAC .event = offsetof(odp_event_vector_hdr_t, event), .pool = offsetof(odp_event_vector_hdr_t, event_hdr.pool), .size = offsetof(odp_event_vector_hdr_t, size), - .flags = offsetof(odp_event_vector_hdr_t, flags) }; #include @@ -50,6 +49,7 @@ odp_event_vector_t odp_event_vector_alloc(odp_pool_t pool_hdl) return ODP_EVENT_VECTOR_INVALID; _ODP_ASSERT(event_vector_hdr_from_event(event)->size == 0); + _ODP_ASSERT(event_vector_hdr_from_event(event)->event_hdr.user_flag == 0); return odp_event_vector_from_event(event); } @@ -59,8 +59,7 @@ void odp_event_vector_free(odp_event_vector_t evv) odp_event_vector_hdr_t *evv_hdr = _odp_event_vector_hdr(evv); evv_hdr->size = 0; - evv_hdr->flags.all_flags = 0; - + evv_hdr->event_hdr.user_flag = 0; _odp_event_free(odp_event_vector_to_event(evv)); } @@ -106,8 +105,8 @@ void odp_event_vector_print(odp_event_vector_t evv) evv_hdr = _odp_event_vector_hdr(evv); len += _odp_snprint(&str[len], n - len, " size %" PRIu32 "\n", evv_hdr->size); - len += _odp_snprint(&str[len], n - len, " flags 0x%" PRIx32 "\n", - evv_hdr->flags.all_flags); + len += _odp_snprint(&str[len], n - len, " user flag %d\n", + evv_hdr->event_hdr.user_flag); len += _odp_snprint(&str[len], n - len, " user area %p\n", evv_hdr->event_hdr.user_area); diff --git a/platform/linux-generic/odp_packet.c b/platform/linux-generic/odp_packet.c index 2d8bee8941..7bdc8827bb 100644 --- a/platform/linux-generic/odp_packet.c +++ b/platform/linux-generic/odp_packet.c @@ -1508,6 +1508,8 @@ void odp_packet_print(odp_packet_t pkt) " l4_type %u\n", hdr->p.l4_type); len += _odp_snprint(&str[len], n - len, " cls_mark %" PRIu64 "\n", odp_packet_cls_mark(pkt)); + len += _odp_snprint(&str[len], n - len, + " user flag %d\n", hdr->event_hdr.user_flag); len += _odp_snprint(&str[len], n - len, " user ptr %p\n", hdr->user_ptr); len += _odp_snprint(&str[len], n - len, diff --git a/platform/linux-generic/odp_packet_vector.c b/platform/linux-generic/odp_packet_vector.c index 1a5d2bbc6b..dbff56c6ba 100644 --- a/platform/linux-generic/odp_packet_vector.c +++ b/platform/linux-generic/odp_packet_vector.c @@ -38,6 +38,7 @@ odp_packet_vector_t odp_packet_vector_alloc(odp_pool_t pool_hdl) return ODP_PACKET_VECTOR_INVALID; _ODP_ASSERT(event_vector_hdr_from_event(event)->size == 0); + _ODP_ASSERT(event_vector_hdr_from_event(event)->event_hdr.user_flag == 0); return odp_packet_vector_from_event(event); } @@ -47,8 +48,7 @@ void odp_packet_vector_free(odp_packet_vector_t pktv) odp_event_vector_hdr_t *pktv_hdr = _odp_packet_vector_hdr(pktv); pktv_hdr->size = 0; - pktv_hdr->flags.all_flags = 0; - + pktv_hdr->event_hdr.user_flag = 0; _odp_event_free(odp_packet_vector_to_event(pktv)); } @@ -98,8 +98,8 @@ void odp_packet_vector_print(odp_packet_vector_t pktv) len += _odp_snprint(&str[len], n - len, " handle 0x%" PRIx64 "\n", odp_packet_vector_to_u64(pktv)); len += _odp_snprint(&str[len], n - len, " size %" PRIu32 "\n", pktv_hdr->size); - len += _odp_snprint(&str[len], n - len, " flags 0x%" PRIx32 "\n", - pktv_hdr->flags.all_flags); + len += _odp_snprint(&str[len], n - len, " user flag %d\n", + pktv_hdr->event_hdr.user_flag); len += _odp_snprint(&str[len], n - len, " user area %p\n", pktv_hdr->event_hdr.user_area); diff --git a/platform/linux-generic/odp_pool.c b/platform/linux-generic/odp_pool.c index fdfde2a5f9..3bf94bed76 100644 --- a/platform/linux-generic/odp_pool.c +++ b/platform/linux-generic/odp_pool.c @@ -501,6 +501,7 @@ static void init_event_hdr(pool_t *pool, _odp_event_hdr_t *event_hdr, uint32_t e event_hdr->subtype = ODP_EVENT_NO_SUBTYPE; event_hdr->pool = _odp_pool_handle(pool); event_hdr->user_area = uarea; + event_hdr->user_flag = -1; /* Store base values for fast init */ if (type == ODP_POOL_BUFFER || type == ODP_POOL_PACKET) { @@ -525,11 +526,13 @@ static void init_event_hdr(pool_t *pool, _odp_event_hdr_t *event_hdr, uint32_t e /* Initialize packet vector metadata */ if (type == ODP_POOL_VECTOR) { event_hdr->event_type = ODP_EVENT_PACKET_VECTOR; + event_hdr->user_flag = 0; } /* Initialize event vector metadata */ if (type == ODP_POOL_EVENT_VECTOR) { event_hdr->event_type = ODP_EVENT_VECTOR; + event_hdr->user_flag = 0; } }