diff --git a/platform/linux-generic/include/odp/api/plat/buffer_inline_types.h b/platform/linux-generic/include/odp/api/plat/buffer_inline_types.h index a4843da550..3c732e85b1 100644 --- a/platform/linux-generic/include/odp/api/plat/buffer_inline_types.h +++ b/platform/linux-generic/include/odp/api/plat/buffer_inline_types.h @@ -13,19 +13,6 @@ extern "C" { /** @cond _ODP_HIDE_FROM_DOXYGEN_ */ -/* Buffer header field accessors */ -#define _odp_buffer_get(buffer_hdr, cast, field) \ - (*(cast *)(uintptr_t)((uint8_t *)buffer_hdr + \ - _odp_buffer_inline_offset.field)) - -/* Buffer header field offsets for inline functions */ -typedef struct _odp_buffer_inline_offset_t { - uint16_t uarea_addr; - -} _odp_buffer_inline_offset_t; - -extern const _odp_buffer_inline_offset_t _odp_buffer_inline_offset; - /** @endcond */ #ifdef __cplusplus diff --git a/platform/linux-generic/include/odp/api/plat/buffer_inlines.h b/platform/linux-generic/include/odp/api/plat/buffer_inlines.h index 05820256d8..cc333fe6f5 100644 --- a/platform/linux-generic/include/odp/api/plat/buffer_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/buffer_inlines.h @@ -1,5 +1,5 @@ /* SPDX-License-Identifier: BSD-3-Clause - * Copyright (c) 2019-2023 Nokia + * Copyright (c) 2019-2025 Nokia */ #ifndef ODP_PLAT_BUFFER_INLINES_H_ @@ -78,7 +78,7 @@ _ODP_INLINE odp_pool_t odp_buffer_pool(odp_buffer_t buf) _ODP_INLINE void *odp_buffer_user_area(odp_buffer_t buf) { - return _odp_buffer_get(buf, void *, uarea_addr); + return _odp_event_hdr_field((odp_event_t)(uintptr_t)buf, void *, user_area); } /** @endcond */ diff --git a/platform/linux-generic/include/odp/api/plat/dma_inlines.h b/platform/linux-generic/include/odp/api/plat/dma_inlines.h index b69f327c37..0a085b04b4 100644 --- a/platform/linux-generic/include/odp/api/plat/dma_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/dma_inlines.h @@ -51,7 +51,7 @@ _ODP_INLINE odp_event_t odp_dma_compl_to_event(odp_dma_compl_t dma_compl) _ODP_INLINE void *odp_dma_compl_user_area(odp_dma_compl_t dma_compl) { - return odp_buffer_user_area((odp_buffer_t)(uintptr_t)dma_compl); + return _odp_event_hdr_field((odp_event_t)(uintptr_t)dma_compl, void *, user_area); } _ODP_INLINE void odp_dma_transfer_param_init(odp_dma_transfer_param_t *trs_param) diff --git a/platform/linux-generic/include/odp/api/plat/event_inline_types.h b/platform/linux-generic/include/odp/api/plat/event_inline_types.h index ee22614957..98163035ad 100644 --- a/platform/linux-generic/include/odp/api/plat/event_inline_types.h +++ b/platform/linux-generic/include/odp/api/plat/event_inline_types.h @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: BSD-3-Clause * Copyright (c) 2018 Linaro Limited - * Copyright (c) 2022 Nokia + * Copyright (c) 2022-2025 Nokia */ #ifndef ODP_PLAT_EVENT_INLINE_TYPES_H_ @@ -26,9 +26,11 @@ extern "C" { typedef struct _odp_event_inline_offset_t { uint16_t event_type; uint16_t base_data; + uint16_t user_area; uint16_t subtype; uint16_t flow_id; uint16_t pool; + uint16_t user_flag; } _odp_event_inline_offset_t; diff --git a/platform/linux-generic/include/odp/api/plat/event_inlines.h b/platform/linux-generic/include/odp/api/plat/event_inlines.h index f9a0e6a271..2dd7975a13 100644 --- a/platform/linux-generic/include/odp/api/plat/event_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/event_inlines.h @@ -6,18 +6,11 @@ #ifndef ODP_PLAT_EVENT_INLINES_H_ #define ODP_PLAT_EVENT_INLINES_H_ -#include #include -#include #include -#include -#include #include #include -#include -#include -#include #ifdef __cplusplus extern "C" { @@ -99,92 +92,22 @@ _ODP_INLINE odp_pool_t odp_event_pool(odp_event_t event) _ODP_INLINE void *odp_event_user_area(odp_event_t event) { - const odp_event_type_t type = __odp_event_type_get(event); - - switch (type) { - case ODP_EVENT_BUFFER: - case ODP_EVENT_ML_COMPL: - case ODP_EVENT_DMA_COMPL: - return _odp_buffer_get((odp_buffer_t)event, void *, uarea_addr); - case ODP_EVENT_PACKET: - return _odp_pkt_get((odp_packet_t)event, void *, user_area); - case ODP_EVENT_VECTOR: - case ODP_EVENT_PACKET_VECTOR: - return _odp_event_vect_get(event, void *, uarea_addr); - case ODP_EVENT_TIMEOUT: - return _odp_timeout_hdr_field((odp_timeout_t)event, void *, uarea_addr); - default: - return NULL; - } + return _odp_event_hdr_field(event, void *, user_area); } _ODP_INLINE void *odp_event_user_area_and_flag(odp_event_t event, int *flag) { - const odp_event_type_t type = __odp_event_type_get(event); - _ODP_ASSERT(flag != NULL); - - switch (type) { - case ODP_EVENT_BUFFER: - case ODP_EVENT_DMA_COMPL: - case ODP_EVENT_ML_COMPL: - *flag = -1; - return _odp_buffer_get((odp_buffer_t)event, void *, uarea_addr); - case ODP_EVENT_PACKET: - { - _odp_packet_flags_t pkt_flags; - odp_packet_t pkt = (odp_packet_t)event; - - pkt_flags.all_flags = _odp_pkt_get(pkt, uint32_t, flags); - *flag = pkt_flags.user_flag; - - return _odp_pkt_get(pkt, void *, user_area); - } - case ODP_EVENT_VECTOR: - case ODP_EVENT_PACKET_VECTOR: - { - _odp_event_vector_flags_t v_flags; - - v_flags.all_flags = _odp_event_vect_get(event, uint32_t, flags); - *flag = v_flags.user_flag; - - return _odp_event_vect_get(event, void *, uarea_addr); - } - case ODP_EVENT_TIMEOUT: - *flag = -1; - return _odp_timeout_hdr_field((odp_timeout_t)event, void *, uarea_addr); - default: - *flag = -1; - return NULL; - } + *flag = _odp_event_hdr_field(event, int8_t, user_flag); + return _odp_event_hdr_field(event, void *, user_area); } _ODP_INLINE void odp_event_user_flag_set(odp_event_t event, int val) { - const odp_event_type_t type = __odp_event_type_get(event); + int8_t *user_flag = _odp_event_hdr_ptr(event, int8_t, user_flag); - switch (type) { - case ODP_EVENT_PACKET: - { - odp_packet_t pkt = (odp_packet_t)event; - _odp_packet_flags_t *flags = _odp_pkt_get_ptr(pkt, _odp_packet_flags_t, flags); - - flags->user_flag = !!val; - return; - } - case ODP_EVENT_VECTOR: - case ODP_EVENT_PACKET_VECTOR: - { - _odp_event_vector_flags_t *flags = - _odp_event_vect_get_ptr(event, _odp_event_vector_flags_t, flags); - - flags->user_flag = !!val; - return; - } - default: - /* Nothing to do */ - return; - } + if (odp_likely(*user_flag != -1)) + *user_flag = !!val; } _ODP_INLINE odp_event_subtype_t odp_event_subtype(odp_event_t event) diff --git a/platform/linux-generic/include/odp/api/plat/event_vector_inline_types.h b/platform/linux-generic/include/odp/api/plat/event_vector_inline_types.h index d7045ec47c..27c0c341aa 100644 --- a/platform/linux-generic/include/odp/api/plat/event_vector_inline_types.h +++ b/platform/linux-generic/include/odp/api/plat/event_vector_inline_types.h @@ -13,15 +13,6 @@ extern "C" { /** @cond _ODP_HIDE_FROM_DOXYGEN_ */ -typedef union { - uint32_t all_flags; - - struct { - uint32_t user_flag : 1; - }; - -} _odp_event_vector_flags_t; - /* Event vector field accessors */ #define _odp_event_vect_get(vect, cast, field) \ (*(cast *)(uintptr_t)((uint8_t *)vect + _odp_event_vector_inline.field)) @@ -33,8 +24,6 @@ typedef struct _odp_event_vector_inline_offset_t { uint16_t event; uint16_t pool; uint16_t size; - uint16_t uarea_addr; - uint16_t flags; } _odp_event_vector_inline_offset_t; diff --git a/platform/linux-generic/include/odp/api/plat/event_vector_inlines.h b/platform/linux-generic/include/odp/api/plat/event_vector_inlines.h index c08ebbb5ec..6cbf12ed2e 100644 --- a/platform/linux-generic/include/odp/api/plat/event_vector_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/event_vector_inlines.h @@ -16,6 +16,7 @@ #include #include +#include #include #include @@ -81,24 +82,20 @@ _ODP_INLINE void odp_event_vector_size_set(odp_event_vector_t evv, uint32_t size _ODP_INLINE void *odp_event_vector_user_area(odp_event_vector_t evv) { - return _odp_event_vect_get(evv, void *, uarea_addr); + return _odp_event_hdr_field((odp_event_t)(uintptr_t)evv, void *, user_area); } _ODP_INLINE int odp_event_vector_user_flag(odp_event_vector_t evv) { - _odp_event_vector_flags_t flags; - - flags.all_flags = _odp_event_vect_get(evv, uint32_t, flags); - - return flags.user_flag; + return _odp_event_hdr_field((odp_event_t)(uintptr_t)evv, int8_t, user_flag); } _ODP_INLINE void odp_event_vector_user_flag_set(odp_event_vector_t evv, int val) { - _odp_event_vector_flags_t *flags = _odp_event_vect_get_ptr(evv, _odp_event_vector_flags_t, - flags); + odp_event_t event = (odp_event_t)(uintptr_t)evv; + int8_t *user_flag = _odp_event_hdr_ptr(event, int8_t, user_flag); - flags->user_flag = !!val; + *user_flag = !!val; } /** @endcond */ diff --git a/platform/linux-generic/include/odp/api/plat/packet_flag_inlines.h b/platform/linux-generic/include/odp/api/plat/packet_flag_inlines.h index f075a07147..ad6cf8f24e 100644 --- a/platform/linux-generic/include/odp/api/plat/packet_flag_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/packet_flag_inlines.h @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: BSD-3-Clause * Copyright (c) 2017-2018 Linaro Limited - * Copyright (c) 2022-2024 Nokia + * Copyright (c) 2022-2025 Nokia */ /** @@ -12,8 +12,8 @@ #ifndef _ODP_PLAT_PACKET_FLAG_INLINES_H_ #define _ODP_PLAT_PACKET_FLAG_INLINES_H_ -#include #include +#include #include #ifdef __cplusplus @@ -22,9 +22,9 @@ extern "C" { /** @cond _ODP_HIDE_FROM_DOXYGEN_ */ -static inline uint64_t _odp_packet_input_flags(odp_packet_t pkt) +static inline uint32_t _odp_packet_input_flags(odp_packet_t pkt) { - return _odp_pkt_get(pkt, uint64_t, input_flags); + return _odp_pkt_get(pkt, uint32_t, input_flags); } #ifndef _ODP_NO_INLINE @@ -225,34 +225,25 @@ _ODP_INLINE int odp_packet_has_ipopt(odp_packet_t pkt) _ODP_INLINE int odp_packet_has_udp(odp_packet_t pkt) { - _odp_packet_input_flags_t flags; - - flags.all = _odp_packet_input_flags(pkt); - return flags.udp; + return _odp_pkt_get(pkt, odp_proto_l4_type_t, l4_type) == ODP_PROTO_L4_TYPE_UDP; } _ODP_INLINE int odp_packet_has_tcp(odp_packet_t pkt) { - _odp_packet_input_flags_t flags; - - flags.all = _odp_packet_input_flags(pkt); - return flags.tcp; + return _odp_pkt_get(pkt, odp_proto_l4_type_t, l4_type) == ODP_PROTO_L4_TYPE_TCP; } _ODP_INLINE int odp_packet_has_sctp(odp_packet_t pkt) { - _odp_packet_input_flags_t flags; - - flags.all = _odp_packet_input_flags(pkt); - return flags.sctp; + return _odp_pkt_get(pkt, odp_proto_l4_type_t, l4_type) == ODP_PROTO_L4_TYPE_SCTP; } _ODP_INLINE int odp_packet_has_icmp(odp_packet_t pkt) { - _odp_packet_input_flags_t flags; + odp_proto_l4_type_t l4_type = _odp_pkt_get(pkt, odp_proto_l4_type_t, l4_type); - flags.all = _odp_packet_input_flags(pkt); - return flags.icmp; + return (l4_type == ODP_PROTO_L4_TYPE_ICMPV4 || + l4_type == ODP_PROTO_L4_TYPE_ICMPV6); } _ODP_INLINE int odp_packet_has_error(odp_packet_t pkt) diff --git a/platform/linux-generic/include/odp/api/plat/packet_inline_types.h b/platform/linux-generic/include/odp/api/plat/packet_inline_types.h index 8c5dff00be..1de068ac0b 100644 --- a/platform/linux-generic/include/odp/api/plat/packet_inline_types.h +++ b/platform/linux-generic/include/odp/api/plat/packet_inline_types.h @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: BSD-3-Clause * Copyright (c) 2015-2018 Linaro Limited - * Copyright (c) 2019-2022 Nokia + * Copyright (c) 2019-2025 Nokia */ @@ -40,7 +40,6 @@ typedef struct _odp_packet_inline_offset_t { uint16_t input; uint16_t seg_count; uint16_t user_ptr; - uint16_t user_area; uint16_t l2_offset; uint16_t l3_offset; uint16_t l4_offset; @@ -48,6 +47,7 @@ typedef struct _odp_packet_inline_offset_t { uint16_t timestamp; uint16_t input_flags; uint16_t flags; + uint16_t l4_type; uint16_t cls_mark; uint16_t ipsec_ctx; uint16_t crypto_op; @@ -59,55 +59,44 @@ extern const _odp_packet_inline_offset_t _odp_packet_inline; /* Packet input & protocol flags */ typedef union { /* All input flags */ - uint64_t all; + uint32_t all; /* Individual input flags */ struct { - uint64_t dst_queue:1; /* Dst queue present */ - uint64_t cls_mark: 1; /* Classifier mark value present*/ - - uint64_t flow_hash:1; /* Flow hash present */ - uint64_t timestamp:1; /* Timestamp present */ - - uint64_t l2:1; /* known L2 protocol present */ - uint64_t l3:1; /* known L3 protocol present */ - uint64_t l4:1; /* known L4 protocol present */ - - uint64_t eth:1; /* Ethernet */ - uint64_t eth_bcast:1; /* Ethernet broadcast */ - uint64_t eth_mcast:1; /* Ethernet multicast */ - uint64_t jumbo:1; /* Jumbo frame */ - uint64_t vlan:1; /* VLAN hdr found */ - uint64_t vlan_qinq:1; /* Stacked VLAN found, QinQ */ - - uint64_t snap:1; /* SNAP */ - uint64_t arp:1; /* ARP */ - - uint64_t ipv4:1; /* IPv4 */ - uint64_t ipv6:1; /* IPv6 */ - uint64_t ip_bcast:1; /* IP broadcast */ - uint64_t ip_mcast:1; /* IP multicast */ - uint64_t ipfrag:1; /* IP fragment */ - uint64_t ipopt:1; /* IP optional headers */ - - uint64_t ipsec:1; /* IPSec packet. Required by the + uint32_t cls_mark: 1; /* Classifier mark value present*/ + + uint32_t flow_hash:1; /* Flow hash present */ + uint32_t timestamp:1; /* Timestamp present */ + + uint32_t l2:1; /* known L2 protocol present */ + uint32_t l3:1; /* known L3 protocol present */ + uint32_t l4:1; /* known L4 protocol present */ + + uint32_t eth:1; /* Ethernet */ + uint32_t eth_bcast:1; /* Ethernet broadcast */ + uint32_t eth_mcast:1; /* Ethernet multicast */ + uint32_t jumbo:1; /* Jumbo frame */ + uint32_t vlan:1; /* VLAN hdr found */ + uint32_t vlan_qinq:1; /* Stacked VLAN found, QinQ */ + + uint32_t arp:1; /* ARP */ + + uint32_t ipv4:1; /* IPv4 */ + uint32_t ipv6:1; /* IPv6 */ + uint32_t ip_bcast:1; /* IP broadcast */ + uint32_t ip_mcast:1; /* IP multicast */ + uint32_t ipfrag:1; /* IP fragment */ + uint32_t ipopt:1; /* IP optional headers */ + + uint32_t ipsec:1; /* IPSec packet. Required by the odp_packet_has_ipsec_set() func. */ - uint64_t ipsec_ah:1; /* IPSec authentication header */ - uint64_t ipsec_esp:1; /* IPSec encapsulating security - payload */ - uint64_t udp:1; /* UDP */ - uint64_t tcp:1; /* TCP */ - uint64_t sctp:1; /* SCTP */ - uint64_t icmp:1; /* ICMP */ - uint64_t no_next_hdr:1; /* No Next Header */ - - uint64_t color:2; /* Packet color for traffic mgmt */ - uint64_t nodrop:1; /* Drop eligibility status */ - - uint64_t l3_chksum_done:1; /* L3 checksum validation done */ - uint64_t l4_chksum_done:1; /* L4 checksum validation done */ - uint64_t ipsec_udp:1; /* UDP-encapsulated IPsec packet */ - uint64_t udp_chksum_zero:1; /* UDP header had 0 as chksum */ + + uint32_t color:2; /* Packet color for traffic mgmt */ + uint32_t nodrop:1; /* Drop eligibility status */ + + uint32_t l3_chksum_done:1; /* L3 checksum validation done */ + uint32_t l4_chksum_done:1; /* L4 checksum validation done */ + uint32_t udp_chksum_zero:1; /* UDP header had 0 as chksum */ }; } _odp_packet_input_flags_t; @@ -120,13 +109,12 @@ typedef union { uint32_t all_flags; struct { - uint32_t reserved1: 4; + uint32_t reserved1: 5; /* * Init flags */ uint32_t user_ptr_set: 1; /* User has set a non-NULL value */ - uint32_t user_flag: 1; /* * Packet output flags @@ -158,8 +146,8 @@ typedef union { /* Flag groups */ struct { - uint32_t reserved2: 4; - uint32_t other: 21; /* All other flags */ + uint32_t reserved2: 5; + uint32_t other: 20; /* All other flags */ uint32_t error: 7; /* All error flags */ } all; diff --git a/platform/linux-generic/include/odp/api/plat/packet_inlines.h b/platform/linux-generic/include/odp/api/plat/packet_inlines.h index 37723ae563..e28bd6dccd 100644 --- a/platform/linux-generic/include/odp/api/plat/packet_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/packet_inlines.h @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: BSD-3-Clause * Copyright (c) 2017-2018 Linaro Limited - * Copyright (c) 2019-2022 Nokia + * Copyright (c) 2019-2025 Nokia */ /** @@ -204,7 +204,7 @@ _ODP_INLINE void odp_packet_user_ptr_set(odp_packet_t pkt, const void *ptr) _ODP_INLINE void *odp_packet_user_area(odp_packet_t pkt) { - return _odp_pkt_get(pkt, void *, user_area); + return _odp_event_hdr_field((odp_event_t)(uintptr_t)pkt, void *, user_area); } _ODP_INLINE uint32_t odp_packet_user_area_size(odp_packet_t pkt) @@ -216,18 +216,15 @@ _ODP_INLINE uint32_t odp_packet_user_area_size(odp_packet_t pkt) _ODP_INLINE int odp_packet_user_flag(odp_packet_t pkt) { - _odp_packet_flags_t flags; - - flags.all_flags = _odp_pkt_get(pkt, uint32_t, flags); - - return flags.user_flag; + return _odp_event_hdr_field((odp_event_t)(uintptr_t)pkt, int8_t, user_flag); } _ODP_INLINE void odp_packet_user_flag_set(odp_packet_t pkt, int val) { - _odp_packet_flags_t *flags = _odp_pkt_get_ptr(pkt, _odp_packet_flags_t, flags); + odp_event_t event = (odp_event_t)(uintptr_t)pkt; + int8_t *user_flag = _odp_event_hdr_ptr(event, int8_t, user_flag); - flags->user_flag = !!val; + *user_flag = !!val; } _ODP_INLINE uint32_t odp_packet_l2_offset(odp_packet_t pkt) @@ -339,7 +336,7 @@ _ODP_INLINE odp_proto_l2_type_t odp_packet_l2_type(odp_packet_t pkt) { _odp_packet_input_flags_t input_flags; - input_flags.all = _odp_pkt_get(pkt, uint64_t, input_flags); + input_flags.all = _odp_pkt_get(pkt, uint32_t, input_flags); return input_flags.eth ? ODP_PROTO_L2_TYPE_ETH : ODP_PROTO_L2_TYPE_NONE; } @@ -348,7 +345,7 @@ _ODP_INLINE odp_proto_l3_type_t odp_packet_l3_type(odp_packet_t pkt) { _odp_packet_input_flags_t input_flags; - input_flags.all = _odp_pkt_get(pkt, uint64_t, input_flags); + input_flags.all = _odp_pkt_get(pkt, uint32_t, input_flags); if (input_flags.ipv4) return ODP_PROTO_L3_TYPE_IPV4; @@ -362,28 +359,7 @@ _ODP_INLINE odp_proto_l3_type_t odp_packet_l3_type(odp_packet_t pkt) _ODP_INLINE odp_proto_l4_type_t odp_packet_l4_type(odp_packet_t pkt) { - _odp_packet_input_flags_t input_flags; - - input_flags.all = _odp_pkt_get(pkt, uint64_t, input_flags); - - if (input_flags.tcp) - return ODP_PROTO_L4_TYPE_TCP; - else if (input_flags.udp) - return ODP_PROTO_L4_TYPE_UDP; - else if (input_flags.sctp) - return ODP_PROTO_L4_TYPE_SCTP; - else if (input_flags.ipsec_ah) - return ODP_PROTO_L4_TYPE_AH; - else if (input_flags.ipsec_esp) - return ODP_PROTO_L4_TYPE_ESP; - else if (input_flags.icmp && input_flags.ipv4) - return ODP_PROTO_L4_TYPE_ICMPV4; - else if (input_flags.icmp && input_flags.ipv6) - return ODP_PROTO_L4_TYPE_ICMPV6; - else if (input_flags.no_next_hdr) - return ODP_PROTO_L4_TYPE_NO_NEXT; - - return ODP_PROTO_L4_TYPE_NONE; + return _odp_pkt_get(pkt, uint8_t, l4_type); } _ODP_INLINE odp_packet_chksum_status_t odp_packet_l3_chksum_status(odp_packet_t pkt) @@ -392,7 +368,7 @@ _ODP_INLINE odp_packet_chksum_status_t odp_packet_l3_chksum_status(odp_packet_t _odp_packet_input_flags_t input_flags; flags.all_flags = _odp_pkt_get(pkt, uint32_t, flags); - input_flags.all = _odp_pkt_get(pkt, uint64_t, input_flags); + input_flags.all = _odp_pkt_get(pkt, uint32_t, input_flags); if (!input_flags.l3_chksum_done) return ODP_PACKET_CHKSUM_UNKNOWN; @@ -409,7 +385,7 @@ _ODP_INLINE odp_packet_chksum_status_t odp_packet_l4_chksum_status(odp_packet_t _odp_packet_input_flags_t input_flags; flags.all_flags = _odp_pkt_get(pkt, uint32_t, flags); - input_flags.all = _odp_pkt_get(pkt, uint64_t, input_flags); + input_flags.all = _odp_pkt_get(pkt, uint32_t, input_flags); if (!input_flags.l4_chksum_done) return ODP_PACKET_CHKSUM_UNKNOWN; @@ -595,7 +571,7 @@ _ODP_INLINE odp_packet_color_t odp_packet_color(odp_packet_t pkt) { _odp_packet_input_flags_t input_flags; - input_flags.all = _odp_pkt_get(pkt, uint64_t, input_flags); + input_flags.all = _odp_pkt_get(pkt, uint32_t, input_flags); return (odp_packet_color_t)input_flags.color; } @@ -604,7 +580,7 @@ _ODP_INLINE odp_bool_t odp_packet_drop_eligible(odp_packet_t pkt) { _odp_packet_input_flags_t input_flags; - input_flags.all = _odp_pkt_get(pkt, uint64_t, input_flags); + input_flags.all = _odp_pkt_get(pkt, uint32_t, input_flags); return !input_flags.nodrop; } @@ -622,7 +598,7 @@ _ODP_INLINE uint64_t odp_packet_cls_mark(odp_packet_t pkt) { _odp_packet_input_flags_t input_flags; - input_flags.all = _odp_pkt_get(pkt, uint64_t, input_flags); + input_flags.all = _odp_pkt_get(pkt, uint32_t, input_flags); return input_flags.cls_mark ? _odp_pkt_get(pkt, uint16_t, cls_mark) : 0; } diff --git a/platform/linux-generic/include/odp/api/plat/packet_vector_inlines.h b/platform/linux-generic/include/odp/api/plat/packet_vector_inlines.h index f29951ef7c..806ef7a6f8 100644 --- a/platform/linux-generic/include/odp/api/plat/packet_vector_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/packet_vector_inlines.h @@ -1,5 +1,5 @@ /* SPDX-License-Identifier: BSD-3-Clause - * Copyright (c) 2020-2022 Nokia + * Copyright (c) 2020-2025 Nokia */ /** @@ -16,6 +16,7 @@ #include #include +#include #include #include @@ -81,24 +82,20 @@ _ODP_INLINE void odp_packet_vector_size_set(odp_packet_vector_t pktv, uint32_t s _ODP_INLINE void *odp_packet_vector_user_area(odp_packet_vector_t pktv) { - return _odp_event_vect_get(pktv, void *, uarea_addr); + return _odp_event_hdr_field((odp_event_t)(uintptr_t)pktv, void *, user_area); } _ODP_INLINE int odp_packet_vector_user_flag(odp_packet_vector_t pktv) { - _odp_event_vector_flags_t flags; - - flags.all_flags = _odp_event_vect_get(pktv, uint32_t, flags); - - return flags.user_flag; + return _odp_event_hdr_field((odp_event_t)(uintptr_t)pktv, int8_t, user_flag); } _ODP_INLINE void odp_packet_vector_user_flag_set(odp_packet_vector_t pktv, int val) { - _odp_event_vector_flags_t *flags = _odp_event_vect_get_ptr(pktv, _odp_event_vector_flags_t, - flags); + odp_event_t event = (odp_event_t)(uintptr_t)pktv; + int8_t *user_flag = _odp_event_hdr_ptr(event, int8_t, user_flag); - flags->user_flag = !!val; + *user_flag = !!val; } /** @endcond */ diff --git a/platform/linux-generic/include/odp/api/plat/timer_inline_types.h b/platform/linux-generic/include/odp/api/plat/timer_inline_types.h index 249b67bbef..fe881f63ff 100644 --- a/platform/linux-generic/include/odp/api/plat/timer_inline_types.h +++ b/platform/linux-generic/include/odp/api/plat/timer_inline_types.h @@ -23,7 +23,6 @@ typedef struct _odp_timeout_inline_offset_t { uint16_t expiration; uint16_t timer; uint16_t user_ptr; - uint16_t uarea_addr; } _odp_timeout_inline_offset_t; diff --git a/platform/linux-generic/include/odp/api/plat/timer_inlines.h b/platform/linux-generic/include/odp/api/plat/timer_inlines.h index 32bfb20dd3..e3edb304d8 100644 --- a/platform/linux-generic/include/odp/api/plat/timer_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/timer_inlines.h @@ -1,5 +1,5 @@ /* SPDX-License-Identifier: BSD-3-Clause - * Copyright (c) 2022-2023 Nokia + * Copyright (c) 2022-2025 Nokia */ #ifndef ODP_PLAT_TIMER_INLINES_H_ @@ -11,6 +11,7 @@ #include #include +#include #include #include @@ -55,7 +56,7 @@ _ODP_INLINE void *odp_timeout_user_ptr(odp_timeout_t tmo) _ODP_INLINE void *odp_timeout_user_area(odp_timeout_t tmo) { - return _odp_timeout_hdr_field(tmo, void *, uarea_addr); + return _odp_event_hdr_field((odp_event_t)(uintptr_t)tmo, void *, user_area); } _ODP_INLINE uint64_t odp_timer_current_tick(odp_timer_pool_t tpid) diff --git a/platform/linux-generic/include/odp_buffer_internal.h b/platform/linux-generic/include/odp_buffer_internal.h index 92b6beb123..e325c0792b 100644 --- a/platform/linux-generic/include/odp_buffer_internal.h +++ b/platform/linux-generic/include/odp_buffer_internal.h @@ -38,9 +38,6 @@ typedef struct ODP_ALIGNED_CACHE odp_buffer_hdr_t { /* Common event header */ _odp_event_hdr_t event_hdr; - /* User area pointer */ - void *uarea_addr; - /* Data */ uint8_t data[]; } odp_buffer_hdr_t; diff --git a/platform/linux-generic/include/odp_event_internal.h b/platform/linux-generic/include/odp_event_internal.h index 893b8b63c5..faa3d69510 100644 --- a/platform/linux-generic/include/odp_event_internal.h +++ b/platform/linux-generic/include/odp_event_internal.h @@ -1,5 +1,5 @@ /* SPDX-License-Identifier: BSD-3-Clause - * Copyright (c) 2021-2022 Nokia + * Copyright (c) 2021-2025 Nokia */ /** @@ -54,12 +54,12 @@ typedef struct _odp_event_hdr_t { /* Initial buffer tail pointer and endmark location (if enabled) */ uint8_t *buf_end; + /* User area pointer */ + uint8_t *user_area; + /* Combined pool and event index */ _odp_event_index_t index; - /* Pool type */ - int8_t type; - /* Event type. Maybe different than pool type (crypto compl event) */ int8_t event_type; @@ -69,6 +69,9 @@ typedef struct _odp_event_hdr_t { /* Event flow id */ uint8_t flow_id; + /* User flag (0 or 1, or -1 if not supported) */ + int8_t user_flag; + } _odp_event_hdr_t; static inline odp_event_t _odp_event_from_hdr(_odp_event_hdr_t *hdr) diff --git a/platform/linux-generic/include/odp_event_vector_internal.h b/platform/linux-generic/include/odp_event_vector_internal.h index 5c6913d297..85047d0ac3 100644 --- a/platform/linux-generic/include/odp_event_vector_internal.h +++ b/platform/linux-generic/include/odp_event_vector_internal.h @@ -29,15 +29,9 @@ typedef struct ODP_ALIGNED_CACHE odp_event_vector_hdr_t { /* Common event header */ _odp_event_hdr_t event_hdr; - /* User area pointer */ - void *uarea_addr; - /* Event vector size */ uint32_t size; - /* Flags */ - _odp_event_vector_flags_t flags; - /* Vector of event handles */ odp_event_t event[]; diff --git a/platform/linux-generic/include/odp_packet_internal.h b/platform/linux-generic/include/odp_packet_internal.h index 60b49a89bb..5379bbaa55 100644 --- a/platform/linux-generic/include/odp_packet_internal.h +++ b/platform/linux-generic/include/odp_packet_internal.h @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: BSD-3-Clause * Copyright (c) 2013-2018 Linaro Limited - * Copyright (c) 2019-2022 Nokia + * Copyright (c) 2019-2025 Nokia */ /** @@ -38,7 +38,7 @@ extern "C" { #include #include -ODP_STATIC_ASSERT(sizeof(_odp_packet_input_flags_t) == sizeof(uint64_t), +ODP_STATIC_ASSERT(sizeof(_odp_packet_input_flags_t) == sizeof(uint32_t), "INPUT_FLAGS_SIZE_ERROR"); ODP_STATIC_ASSERT(sizeof(_odp_packet_flags_t) == sizeof(uint32_t), @@ -67,6 +67,9 @@ typedef struct { /* offset to L4 hdr (TCP, UDP, SCTP, also ICMP) */ uint16_t l4_offset; + + /* l4 protocol type (TCP, UDP, ...) */ + odp_proto_l4_type_t l4_type; } packet_parser_t; /** @@ -118,17 +121,14 @@ typedef struct ODP_ALIGNED_CACHE odp_packet_hdr_t { /* Flow hash value */ uint32_t flow_hash; - /* User area pointer */ - void *uarea_addr; - /* User context pointer */ const void *user_ptr; - /* --- 64-byte cache line boundary --- */ - /* Timestamp value */ odp_time_t timestamp; + /* --- 64-byte cache line boundary --- */ + /* Classifier mark */ uint16_t cls_mark; @@ -221,17 +221,17 @@ static inline void packet_subtype_set(odp_packet_t pkt, int subtype) */ static inline void _odp_packet_reset_md(odp_packet_hdr_t *pkt_hdr) { - /* Clear all flags. Resets also return value of cls_mark, user_ptr, etc. */ + /* Clear all flags. Resets also return value of cls_mark, etc. */ pkt_hdr->p.input_flags.all = 0; pkt_hdr->p.flags.all_flags = 0; + pkt_hdr->p.l4_type = ODP_PROTO_L4_TYPE_NONE; pkt_hdr->p.l2_offset = 0; pkt_hdr->p.l3_offset = ODP_PACKET_OFFSET_INVALID; pkt_hdr->p.l4_offset = ODP_PACKET_OFFSET_INVALID; - if (odp_unlikely(pkt_hdr->event_hdr.subtype != ODP_EVENT_PACKET_BASIC)) - pkt_hdr->event_hdr.subtype = ODP_EVENT_PACKET_BASIC; - + pkt_hdr->event_hdr.subtype = ODP_EVENT_PACKET_BASIC; + pkt_hdr->event_hdr.user_flag = 0; pkt_hdr->input = ODP_PKTIO_INVALID; } @@ -322,6 +322,7 @@ static inline void _odp_packet_copy_md(odp_packet_hdr_t *dst_hdr, */ dst_hdr->input = src_hdr->input; dst_hdr->event_hdr.subtype = subtype; + dst_hdr->event_hdr.user_flag = src_hdr->event_hdr.user_flag; dst_hdr->dst_queue = src_hdr->dst_queue; dst_hdr->cos = src_hdr->cos; dst_hdr->cls_mark = src_hdr->cls_mark; @@ -343,26 +344,27 @@ static inline void _odp_packet_copy_md(odp_packet_hdr_t *dst_hdr, dst_hdr->p = src_hdr->p; - if (src_hdr->uarea_addr) { + if (src_hdr->event_hdr.user_area) { if (uarea_copy) { const pool_t *src_pool = _odp_pool_entry(src_hdr->event_hdr.pool); const pool_t *dst_pool = _odp_pool_entry(dst_hdr->event_hdr.pool); const uint32_t src_uarea_size = src_pool->param_uarea_size; const uint32_t dst_uarea_size = dst_pool->param_uarea_size; - _ODP_ASSERT(dst_hdr->uarea_addr != NULL); + _ODP_ASSERT(dst_hdr->event_hdr.user_area != NULL); _ODP_ASSERT(dst_uarea_size >= src_uarea_size); - memcpy(dst_hdr->uarea_addr, src_hdr->uarea_addr, src_uarea_size); + memcpy(dst_hdr->event_hdr.user_area, + src_hdr->event_hdr.user_area, src_uarea_size); } else { - void *src_uarea = src_hdr->uarea_addr; + void *src_uarea = src_hdr->event_hdr.user_area; /* If user area exists, packets should always be from the same pool, so * user area pointers can simply be swapped. */ _ODP_ASSERT(dst_hdr->event_hdr.pool == src_hdr->event_hdr.pool); - src_hdr->uarea_addr = dst_hdr->uarea_addr; - dst_hdr->uarea_addr = src_uarea; + src_hdr->event_hdr.user_area = dst_hdr->event_hdr.user_area; + dst_hdr->event_hdr.user_area = src_uarea; } } diff --git a/platform/linux-generic/include/odp_timer_internal.h b/platform/linux-generic/include/odp_timer_internal.h index 7059db0c49..de71d9d6fd 100644 --- a/platform/linux-generic/include/odp_timer_internal.h +++ b/platform/linux-generic/include/odp_timer_internal.h @@ -39,9 +39,6 @@ typedef struct ODP_ALIGNED_CACHE odp_timeout_hdr_t { /* User ptr inherited from parent timer */ const void *user_ptr; - /* User area pointer */ - void *uarea_addr; - /* Parent timer */ odp_timer_t timer; diff --git a/platform/linux-generic/odp_buffer.c b/platform/linux-generic/odp_buffer.c index a319f96e4b..05d6f7ea1a 100644 --- a/platform/linux-generic/odp_buffer.c +++ b/platform/linux-generic/odp_buffer.c @@ -17,15 +17,6 @@ #include #include -#include - -/* Buffer header field offsets for inline functions */ -const _odp_buffer_inline_offset_t _odp_buffer_inline_offset ODP_ALIGNED_CACHE = { - .uarea_addr = offsetof(odp_buffer_hdr_t, uarea_addr) -}; - -#include - void odp_buffer_print(odp_buffer_t buf) { int len = 0; diff --git a/platform/linux-generic/odp_classification.c b/platform/linux-generic/odp_classification.c index b90e119579..5c3a7b5e44 100644 --- a/platform/linux-generic/odp_classification.c +++ b/platform/linux-generic/odp_classification.c @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: BSD-3-Clause * Copyright (c) 2014-2018 Linaro Limited - * Copyright (c) 2019-2024 Nokia + * Copyright (c) 2019-2025 Nokia */ #include @@ -1001,7 +1001,7 @@ static inline int verify_pmr_tcp_sport(const uint8_t *pkt_addr, uint16_t sport; const _odp_tcphdr_t *tcp; - if (!pkt_hdr->p.input_flags.tcp) + if (pkt_hdr->p.l4_type != ODP_PROTO_L4_TYPE_TCP) return 0; tcp = (const _odp_tcphdr_t *)(pkt_addr + pkt_hdr->p.l4_offset); sport = tcp->src_port; @@ -1018,7 +1018,7 @@ static inline int verify_pmr_tcp_dport(const uint8_t *pkt_addr, uint16_t dport; const _odp_tcphdr_t *tcp; - if (!pkt_hdr->p.input_flags.tcp) + if (pkt_hdr->p.l4_type != ODP_PROTO_L4_TYPE_TCP) return 0; tcp = (const _odp_tcphdr_t *)(pkt_addr + pkt_hdr->p.l4_offset); dport = tcp->dst_port; @@ -1035,7 +1035,7 @@ static inline int verify_pmr_udp_dport(const uint8_t *pkt_addr, uint16_t dport; const _odp_udphdr_t *udp; - if (!pkt_hdr->p.input_flags.udp) + if (pkt_hdr->p.l4_type != ODP_PROTO_L4_TYPE_UDP) return 0; udp = (const _odp_udphdr_t *)(pkt_addr + pkt_hdr->p.l4_offset); dport = udp->dst_port; @@ -1052,7 +1052,7 @@ static inline int verify_pmr_udp_sport(const uint8_t *pkt_addr, uint16_t sport; const _odp_udphdr_t *udp; - if (!pkt_hdr->p.input_flags.udp) + if (pkt_hdr->p.l4_type != ODP_PROTO_L4_TYPE_UDP) return 0; udp = (const _odp_udphdr_t *)(pkt_addr + pkt_hdr->p.l4_offset); sport = udp->src_port; @@ -1212,9 +1212,9 @@ static inline int verify_pmr_ipsec_spi(const uint8_t *pkt_addr, pkt_addr += pkt_hdr->p.l4_offset; - if (pkt_hdr->p.input_flags.ipsec_ah) + if (pkt_hdr->p.l4_type == ODP_PROTO_L4_TYPE_AH) spi = ((const _odp_ahhdr_t *)pkt_addr)->spi; - else if (pkt_hdr->p.input_flags.ipsec_esp) + else if (pkt_hdr->p.l4_type == ODP_PROTO_L4_TYPE_ESP) spi = ((const _odp_esphdr_t *)pkt_addr)->spi; else return 0; @@ -1738,7 +1738,6 @@ int _odp_cls_classify_packet(pktio_entry_t *entry, const uint8_t *base, if (*pool == ODP_POOL_INVALID) *pool = entry->pool; - pkt_hdr->p.input_flags.dst_queue = 1; pkt_hdr->cos = cos->index; if (!cos->queue_group) { @@ -1775,14 +1774,14 @@ static uint32_t packet_rss_hash(odp_packet_hdr_t *pkt_hdr, tuple_len += 2; } - if (pkt_hdr->p.input_flags.tcp && hash_proto.tcp) { + if (hash_proto.tcp && pkt_hdr->p.l4_type == ODP_PROTO_L4_TYPE_TCP) { /* add tcp */ tcp = (const _odp_tcphdr_t *)(base + pkt_hdr->p.l4_offset); tuple.v4.sport = tcp->src_port; tuple.v4.dport = tcp->dst_port; tuple_len += 1; - } else if (pkt_hdr->p.input_flags.udp && hash_proto.udp) { + } else if (hash_proto.udp && pkt_hdr->p.l4_type == ODP_PROTO_L4_TYPE_UDP) { /* add udp */ udp = (const _odp_udphdr_t *)(base + pkt_hdr->p.l4_offset); @@ -1798,13 +1797,13 @@ static uint32_t packet_rss_hash(odp_packet_hdr_t *pkt_hdr, thash_load_ipv6_addr(ipv6, &tuple); tuple_len += 8; } - if (pkt_hdr->p.input_flags.tcp && hash_proto.tcp) { + if (hash_proto.tcp && pkt_hdr->p.l4_type == ODP_PROTO_L4_TYPE_TCP) { tcp = (const _odp_tcphdr_t *)(base + pkt_hdr->p.l4_offset); tuple.v6.sport = tcp->src_port; tuple.v6.dport = tcp->dst_port; tuple_len += 1; - } else if (pkt_hdr->p.input_flags.udp && hash_proto.udp) { + } else if (hash_proto.udp && pkt_hdr->p.l4_type == ODP_PROTO_L4_TYPE_UDP) { /* add udp */ udp = (const _odp_udphdr_t *)(base + pkt_hdr->p.l4_offset); diff --git a/platform/linux-generic/odp_event_offsets.c b/platform/linux-generic/odp_event_offsets.c index f8826d4c46..5efe77a536 100644 --- a/platform/linux-generic/odp_event_offsets.c +++ b/platform/linux-generic/odp_event_offsets.c @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: BSD-3-Clause * Copyright (c) 2015-2018 Linaro Limited - * Copyright (c) 2020-2024 Nokia + * Copyright (c) 2020-2025 Nokia */ #include @@ -14,9 +14,11 @@ const _odp_event_inline_offset_t _odp_event_inline_offset ODP_ALIGNED_CACHE = { .event_type = offsetof(_odp_event_hdr_t, event_type), .base_data = offsetof(_odp_event_hdr_t, base_data), + .user_area = offsetof(_odp_event_hdr_t, user_area), .subtype = offsetof(_odp_event_hdr_t, subtype), .flow_id = offsetof(_odp_event_hdr_t, flow_id), .pool = offsetof(_odp_event_hdr_t, pool), + .user_flag = offsetof(_odp_event_hdr_t, user_flag), }; #include diff --git a/platform/linux-generic/odp_event_vector.c b/platform/linux-generic/odp_event_vector.c index 4aeb8bba5c..d9264b85e7 100644 --- a/platform/linux-generic/odp_event_vector.c +++ b/platform/linux-generic/odp_event_vector.c @@ -24,8 +24,6 @@ const _odp_event_vector_inline_offset_t _odp_event_vector_inline ODP_ALIGNED_CAC .event = offsetof(odp_event_vector_hdr_t, event), .pool = offsetof(odp_event_vector_hdr_t, event_hdr.pool), .size = offsetof(odp_event_vector_hdr_t, size), - .uarea_addr = offsetof(odp_event_vector_hdr_t, uarea_addr), - .flags = offsetof(odp_event_vector_hdr_t, flags) }; #include @@ -51,6 +49,7 @@ odp_event_vector_t odp_event_vector_alloc(odp_pool_t pool_hdl) return ODP_EVENT_VECTOR_INVALID; _ODP_ASSERT(event_vector_hdr_from_event(event)->size == 0); + _ODP_ASSERT(event_vector_hdr_from_event(event)->event_hdr.user_flag == 0); return odp_event_vector_from_event(event); } @@ -60,8 +59,7 @@ void odp_event_vector_free(odp_event_vector_t evv) odp_event_vector_hdr_t *evv_hdr = _odp_event_vector_hdr(evv); evv_hdr->size = 0; - evv_hdr->flags.all_flags = 0; - + evv_hdr->event_hdr.user_flag = 0; _odp_event_free(odp_event_vector_to_event(evv)); } @@ -107,9 +105,10 @@ void odp_event_vector_print(odp_event_vector_t evv) evv_hdr = _odp_event_vector_hdr(evv); len += _odp_snprint(&str[len], n - len, " size %" PRIu32 "\n", evv_hdr->size); - len += _odp_snprint(&str[len], n - len, " flags 0x%" PRIx32 "\n", - evv_hdr->flags.all_flags); - len += _odp_snprint(&str[len], n - len, " user area %p\n", evv_hdr->uarea_addr); + len += _odp_snprint(&str[len], n - len, " user flag %d\n", + evv_hdr->event_hdr.user_flag); + len += _odp_snprint(&str[len], n - len, " user area %p\n", + evv_hdr->event_hdr.user_area); for (i = 0; i < evv_hdr->size; i++) { odp_event_t ev = evv_hdr->event[i]; diff --git a/platform/linux-generic/odp_ipsec.c b/platform/linux-generic/odp_ipsec.c index 6e68e2eba6..2ea13463ee 100644 --- a/platform/linux-generic/odp_ipsec.c +++ b/platform/linux-generic/odp_ipsec.c @@ -2368,7 +2368,6 @@ int _odp_ipsec_try_inline(odp_packet_t *pkt) result->flag.inline_mode = 1; pkt_hdr = packet_hdr(*pkt); - pkt_hdr->p.input_flags.dst_queue = 1; pkt_hdr->dst_queue = ipsec_sa->queue; /* Distinguish inline IPsec packets from classifier packets */ pkt_hdr->cos = CLS_COS_IDX_NONE; diff --git a/platform/linux-generic/odp_packet.c b/platform/linux-generic/odp_packet.c index 114d1172e9..7bdc8827bb 100644 --- a/platform/linux-generic/odp_packet.c +++ b/platform/linux-generic/odp_packet.c @@ -57,7 +57,6 @@ const _odp_packet_inline_offset_t _odp_packet_inline ODP_ALIGNED_CACHE = { .input = offsetof(odp_packet_hdr_t, input), .seg_count = offsetof(odp_packet_hdr_t, seg_count), .user_ptr = offsetof(odp_packet_hdr_t, user_ptr), - .user_area = offsetof(odp_packet_hdr_t, uarea_addr), .l2_offset = offsetof(odp_packet_hdr_t, p.l2_offset), .l3_offset = offsetof(odp_packet_hdr_t, p.l3_offset), .l4_offset = offsetof(odp_packet_hdr_t, p.l4_offset), @@ -65,6 +64,7 @@ const _odp_packet_inline_offset_t _odp_packet_inline ODP_ALIGNED_CACHE = { .timestamp = offsetof(odp_packet_hdr_t, timestamp), .input_flags = offsetof(odp_packet_hdr_t, p.input_flags), .flags = offsetof(odp_packet_hdr_t, p.flags), + .l4_type = offsetof(odp_packet_hdr_t, p.l4_type), .cls_mark = offsetof(odp_packet_hdr_t, cls_mark), .ipsec_ctx = offsetof(odp_packet_hdr_t, ipsec_ctx), .crypto_op = offsetof(odp_packet_hdr_t, crypto_op_result), @@ -1474,14 +1474,6 @@ static int packet_print_input_flags(odp_packet_hdr_t *hdr, char *str, int max) len += _odp_snprint(&str[len], max - len, "ipv6 "); if (hdr->p.input_flags.ipsec) len += _odp_snprint(&str[len], max - len, "ipsec "); - if (hdr->p.input_flags.udp) - len += _odp_snprint(&str[len], max - len, "udp "); - if (hdr->p.input_flags.tcp) - len += _odp_snprint(&str[len], max - len, "tcp "); - if (hdr->p.input_flags.sctp) - len += _odp_snprint(&str[len], max - len, "sctp "); - if (hdr->p.input_flags.icmp) - len += _odp_snprint(&str[len], max - len, "icmp "); return len; } @@ -1512,12 +1504,16 @@ void odp_packet_print(odp_packet_t pkt) } len += _odp_snprint(&str[len], n - len, " flags 0x%" PRIx32 "\n", hdr->p.flags.all_flags); + len += _odp_snprint(&str[len], n - len, + " l4_type %u\n", hdr->p.l4_type); len += _odp_snprint(&str[len], n - len, " cls_mark %" PRIu64 "\n", odp_packet_cls_mark(pkt)); + len += _odp_snprint(&str[len], n - len, + " user flag %d\n", hdr->event_hdr.user_flag); len += _odp_snprint(&str[len], n - len, " user ptr %p\n", hdr->user_ptr); len += _odp_snprint(&str[len], n - len, - " user area %p\n", hdr->uarea_addr); + " user area %p\n", hdr->event_hdr.user_area); len += _odp_snprint(&str[len], n - len, " l2_offset %" PRIu32 "\n", hdr->p.l2_offset); len += _odp_snprint(&str[len], n - len, @@ -1908,7 +1904,7 @@ int _odp_packet_l4_chksum(odp_packet_hdr_t *pkt_hdr, { /* UDP chksum == 0 case is covered in parse_udp() */ if (opt.bit.udp_chksum && - pkt_hdr->p.input_flags.udp && + pkt_hdr->p.l4_type == _ODP_IPPROTO_UDP && !pkt_hdr->p.input_flags.ipfrag && !pkt_hdr->p.input_flags.udp_chksum_zero) { uint16_t sum = ~packet_sum(pkt_hdr, @@ -1929,7 +1925,7 @@ int _odp_packet_l4_chksum(odp_packet_hdr_t *pkt_hdr, } if (opt.bit.tcp_chksum && - pkt_hdr->p.input_flags.tcp && + pkt_hdr->p.l4_type == _ODP_IPPROTO_TCP && !pkt_hdr->p.input_flags.ipfrag) { uint16_t sum = ~packet_sum(pkt_hdr, pkt_hdr->p.l3_offset, @@ -1949,7 +1945,7 @@ int _odp_packet_l4_chksum(odp_packet_hdr_t *pkt_hdr, } if (opt.bit.sctp_chksum && - pkt_hdr->p.input_flags.sctp && + pkt_hdr->p.l4_type == _ODP_IPPROTO_SCTP && !pkt_hdr->p.input_flags.ipfrag) { uint32_t seg_len = 0; _odp_sctphdr_t hdr_copy; diff --git a/platform/linux-generic/odp_packet_flags.c b/platform/linux-generic/odp_packet_flags.c index f02babf4a3..5342fcec31 100644 --- a/platform/linux-generic/odp_packet_flags.c +++ b/platform/linux-generic/odp_packet_flags.c @@ -1,5 +1,6 @@ /* SPDX-License-Identifier: BSD-3-Clause * Copyright (c) 2014-2018 Linaro Limited + * Copyright (c) 2025 Nokia */ #include @@ -119,24 +120,38 @@ void odp_packet_has_ipsec_set(odp_packet_t pkt, int val) setflag(pkt, input_flags.ipsec, val); } +static inline void set_type(odp_packet_t pkt, odp_proto_l4_type_t type, int val) +{ + packet_hdr(pkt)->p.l4_type = val ? type : ODP_PROTO_L4_TYPE_NONE; +} + void odp_packet_has_udp_set(odp_packet_t pkt, int val) { - setflag(pkt, input_flags.udp, val); + set_type(pkt, ODP_PROTO_L4_TYPE_UDP, val); } void odp_packet_has_tcp_set(odp_packet_t pkt, int val) { - setflag(pkt, input_flags.tcp, val); + set_type(pkt, ODP_PROTO_L4_TYPE_TCP, val); } void odp_packet_has_sctp_set(odp_packet_t pkt, int val) { - setflag(pkt, input_flags.sctp, val); + set_type(pkt, ODP_PROTO_L4_TYPE_SCTP, val); } void odp_packet_has_icmp_set(odp_packet_t pkt, int val) { - setflag(pkt, input_flags.icmp, val); + odp_proto_l4_type_t type = ODP_PROTO_L4_TYPE_NONE; + odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); + + if (val) { + if (pkt_hdr->p.input_flags.ipv6) + type = ODP_PROTO_L4_TYPE_ICMPV6; + else + type = ODP_PROTO_L4_TYPE_ICMPV4; + } + pkt_hdr->p.l4_type = type; } void odp_packet_has_ts_clr(odp_packet_t pkt) diff --git a/platform/linux-generic/odp_packet_vector.c b/platform/linux-generic/odp_packet_vector.c index efc40581a1..dbff56c6ba 100644 --- a/platform/linux-generic/odp_packet_vector.c +++ b/platform/linux-generic/odp_packet_vector.c @@ -1,5 +1,5 @@ /* SPDX-License-Identifier: BSD-3-Clause - * Copyright (c) 2020-2022 Nokia + * Copyright (c) 2020-2025 Nokia */ #include @@ -38,6 +38,7 @@ odp_packet_vector_t odp_packet_vector_alloc(odp_pool_t pool_hdl) return ODP_PACKET_VECTOR_INVALID; _ODP_ASSERT(event_vector_hdr_from_event(event)->size == 0); + _ODP_ASSERT(event_vector_hdr_from_event(event)->event_hdr.user_flag == 0); return odp_packet_vector_from_event(event); } @@ -47,8 +48,7 @@ void odp_packet_vector_free(odp_packet_vector_t pktv) odp_event_vector_hdr_t *pktv_hdr = _odp_packet_vector_hdr(pktv); pktv_hdr->size = 0; - pktv_hdr->flags.all_flags = 0; - + pktv_hdr->event_hdr.user_flag = 0; _odp_event_free(odp_packet_vector_to_event(pktv)); } @@ -98,9 +98,10 @@ void odp_packet_vector_print(odp_packet_vector_t pktv) len += _odp_snprint(&str[len], n - len, " handle 0x%" PRIx64 "\n", odp_packet_vector_to_u64(pktv)); len += _odp_snprint(&str[len], n - len, " size %" PRIu32 "\n", pktv_hdr->size); - len += _odp_snprint(&str[len], n - len, " flags 0x%" PRIx32 "\n", - pktv_hdr->flags.all_flags); - len += _odp_snprint(&str[len], n - len, " user area %p\n", pktv_hdr->uarea_addr); + len += _odp_snprint(&str[len], n - len, " user flag %d\n", + pktv_hdr->event_hdr.user_flag); + len += _odp_snprint(&str[len], n - len, " user area %p\n", + pktv_hdr->event_hdr.user_area); for (i = 0; i < pktv_hdr->size; i++) { odp_packet_t pkt = odp_packet_from_event(pktv_hdr->event[i]); diff --git a/platform/linux-generic/odp_parse.c b/platform/linux-generic/odp_parse.c index 60e8e5922d..67fc7e1388 100644 --- a/platform/linux-generic/odp_parse.c +++ b/platform/linux-generic/odp_parse.c @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: BSD-3-Clause * Copyright (c) 2013-2018 Linaro Limited - * Copyright (c) 2019-2022 Nokia + * Copyright (c) 2019-2025 Nokia */ #include @@ -59,7 +59,6 @@ uint16_t _odp_parse_eth(packet_parser_t *prs, const uint8_t **parseptr, /* Check for SNAP vs. DIX */ if (odp_unlikely(ethtype < _ODP_ETH_LEN_MAX)) { - input_flags.snap = 1; if (ethtype > frame_len - *offset) { prs->flags.snap_len_err = 1; ethtype = 0; @@ -312,10 +311,8 @@ static inline void parse_udp(packet_parser_t *prs, const uint8_t **parseptr, uint32_t val; memcpy(&val, udp + 1, 4); - if (val != 0) { + if (val != 0) prs->input_flags.ipsec = 1; - prs->input_flags.ipsec_udp = 1; - } } *parseptr += sizeof(_odp_udphdr_t); @@ -409,26 +406,14 @@ int _odp_packet_parse_common_l3_l4(packet_parser_t *prs, if (layer == ODP_PROTO_LAYER_L3) return prs->flags.all.error != 0; - /* Set l4 flag only for known ip_proto */ prs->input_flags.l4 = 1; + prs->l4_type = ip_proto; /* Parse Layer 4 headers */ switch (ip_proto) { - case _ODP_IPPROTO_ICMPV4: - /* Fall through */ - - case _ODP_IPPROTO_ICMPV6: - prs->input_flags.icmp = 1; - break; - - case _ODP_IPPROTO_IPIP: - /* Do nothing */ - break; - case _ODP_IPPROTO_TCP: if (odp_unlikely(offset + _ODP_TCPHDR_LEN > seg_end)) return -1; - prs->input_flags.tcp = 1; parse_tcp(prs, &parseptr, frame_len - prs->l4_offset, opt, l4_part_sum); if (prs->flags.tcp_err && opt.bit.drop_tcp_err) @@ -438,36 +423,24 @@ int _odp_packet_parse_common_l3_l4(packet_parser_t *prs, case _ODP_IPPROTO_UDP: if (odp_unlikely(offset + _ODP_UDPHDR_LEN > seg_end)) return -1; - prs->input_flags.udp = 1; parse_udp(prs, &parseptr, opt, l4_part_sum); if (prs->flags.udp_err && opt.bit.drop_udp_err) return -1; /* drop */ break; case _ODP_IPPROTO_AH: - prs->input_flags.ipsec = 1; - prs->input_flags.ipsec_ah = 1; - break; - case _ODP_IPPROTO_ESP: prs->input_flags.ipsec = 1; - prs->input_flags.ipsec_esp = 1; break; case _ODP_IPPROTO_SCTP: - prs->input_flags.sctp = 1; parse_sctp(prs, &parseptr, frame_len - prs->l4_offset, opt, l4_part_sum); if (prs->flags.sctp_err && opt.bit.drop_sctp_err) return -1; /* drop */ break; - case _ODP_IPPROTO_NO_NEXT: - prs->input_flags.no_next_hdr = 1; - break; - default: - prs->input_flags.l4 = 0; break; } diff --git a/platform/linux-generic/odp_pool.c b/platform/linux-generic/odp_pool.c index 18482a042f..3bf94bed76 100644 --- a/platform/linux-generic/odp_pool.c +++ b/platform/linux-generic/odp_pool.c @@ -497,10 +497,11 @@ static void init_event_hdr(pool_t *pool, _odp_event_hdr_t *event_hdr, uint32_t e /* Initialize common event metadata */ event_hdr->index.pool = pool->pool_idx; event_hdr->index.event = event_index; - event_hdr->type = type; event_hdr->event_type = type; event_hdr->subtype = ODP_EVENT_NO_SUBTYPE; event_hdr->pool = _odp_pool_handle(pool); + event_hdr->user_area = uarea; + event_hdr->user_flag = -1; /* Store base values for fast init */ if (type == ODP_POOL_BUFFER || type == ODP_POOL_PACKET) { @@ -509,18 +510,11 @@ static void init_event_hdr(pool_t *pool, _odp_event_hdr_t *event_hdr, uint32_t e _odp_event_endmark_set(_odp_event_from_hdr(event_hdr)); } - if (type == ODP_POOL_BUFFER) { - odp_buffer_hdr_t *buf_hdr = (void *)event_hdr; - - buf_hdr->uarea_addr = uarea; - } - /* Initialize segmentation metadata */ if (type == ODP_POOL_PACKET) { odp_packet_hdr_t *pkt_hdr = (void *)event_hdr; pkt_hdr->user_ptr = NULL; - pkt_hdr->uarea_addr = uarea; pkt_hdr->seg_data = data_ptr; pkt_hdr->seg_len = pool->seg_len; pkt_hdr->seg_count = 1; @@ -531,25 +525,14 @@ static void init_event_hdr(pool_t *pool, _odp_event_hdr_t *event_hdr, uint32_t e /* Initialize packet vector metadata */ if (type == ODP_POOL_VECTOR) { - odp_event_vector_hdr_t *vect_hdr = (void *)event_hdr; - event_hdr->event_type = ODP_EVENT_PACKET_VECTOR; - vect_hdr->uarea_addr = uarea; + event_hdr->user_flag = 0; } /* Initialize event vector metadata */ if (type == ODP_POOL_EVENT_VECTOR) { - odp_event_vector_hdr_t *vect_hdr = (void *)event_hdr; - event_hdr->event_type = ODP_EVENT_VECTOR; - vect_hdr->uarea_addr = uarea; - } - - /* Initialize timeout metadata */ - if (type == ODP_POOL_TIMEOUT) { - odp_timeout_hdr_t *tmo_hdr = (void *)event_hdr; - - tmo_hdr->uarea_addr = uarea; + event_hdr->user_flag = 0; } } diff --git a/platform/linux-generic/odp_timer.c b/platform/linux-generic/odp_timer.c index d9bd4a85a1..a8eee786e2 100644 --- a/platform/linux-generic/odp_timer.c +++ b/platform/linux-generic/odp_timer.c @@ -114,7 +114,6 @@ _odp_timeout_inline_offset ODP_ALIGNED_CACHE = { .expiration = offsetof(odp_timeout_hdr_t, expiration), .timer = offsetof(odp_timeout_hdr_t, timer), .user_ptr = offsetof(odp_timeout_hdr_t, user_ptr), - .uarea_addr = offsetof(odp_timeout_hdr_t, uarea_addr), }; #include @@ -1968,7 +1967,8 @@ void odp_timeout_print(odp_timeout_t tmo) len += _odp_snprint(&str[len], n - len, " expiration %" PRIu64 "\n", tmo_hdr->expiration); len += _odp_snprint(&str[len], n - len, " user ptr %p\n", tmo_hdr->user_ptr); - len += _odp_snprint(&str[len], n - len, " user area %p\n", tmo_hdr->uarea_addr); + len += _odp_snprint(&str[len], n - len, " user area %p\n", + tmo_hdr->event_hdr.user_area); if (timer != ODP_TIMER_INVALID) { timer_pool_t *tp = handle_to_tp(timer); diff --git a/test/performance/odp_bench_packet.c b/test/performance/odp_bench_packet.c index a698b7c510..e20b4c9caa 100644 --- a/test/performance/odp_bench_packet.c +++ b/test/performance/odp_bench_packet.c @@ -63,7 +63,7 @@ #define TEST_DEF_BURST 8 /** Maximum number of results to be held */ -#define TEST_MAX_BENCH 100 +#define TEST_MAX_BENCH 200 #define TEST_MAX_SIZES 7 @@ -105,6 +105,12 @@ typedef struct { bench_suite_t suite; /** Packet pool */ odp_pool_t pool; + /** Buffer pool */ + odp_pool_t pool_buf; + /** Timeout pool */ + odp_pool_t pool_tmo; + /** Event vector pool */ + odp_pool_t pool_evv; struct { /** Test packet length */ uint32_t len; @@ -286,6 +292,67 @@ static void alloc_packets_twice(void) TEST_REPEAT_COUNT); } +typedef struct test_packet_t { + const uint8_t *data; + int len; +} test_packet_t; + +test_packet_t test_packets[] = { + {test_packet_arp, sizeof(test_packet_arp)}, + {test_packet_ipv4_icmp, sizeof(test_packet_ipv4_icmp)}, + {test_packet_ipv6_icmp, sizeof(test_packet_ipv6_icmp)}, + {test_packet_ipv4_tcp, sizeof(test_packet_ipv4_tcp)}, + {test_packet_ipv6_tcp, sizeof(test_packet_ipv6_tcp)}, + {test_packet_ipv4_udp, sizeof(test_packet_ipv4_udp)}, + {test_packet_ipv6_udp, sizeof(test_packet_ipv6_udp)}, + {test_packet_ipv4_sctp, sizeof(test_packet_ipv4_sctp)}, + {test_packet_ipv6_sctp, sizeof(test_packet_ipv6_sctp)}, + {test_packet_vlan_ipv4_udp, sizeof(test_packet_vlan_ipv4_udp)}, + {test_packet_mcast_eth_ipv4_udp, sizeof(test_packet_mcast_eth_ipv4_udp)}, + {test_packet_mcast_eth_ipv6_udp, sizeof(test_packet_mcast_eth_ipv6_udp)}, + {test_packet_ipv4_udp_last_frag, sizeof(test_packet_ipv4_udp_last_frag)}, +}; + +static uint32_t rnd(void) +{ + static uint64_t s = 1; + uint64_t prime = 0x7fffffff; + + s = (0x12345678 * s) % prime; + return s; +} + +static void create_packets_misc(void) +{ + int num_types = ODPH_ARRAY_SIZE(test_packets); + + for (int i = 0; i < TEST_REPEAT_COUNT; i++) { + test_packet_t *tp = &test_packets[rnd() % num_types]; + odp_packet_t *pkt = &gbl_args->pkt_tbl[i]; + + *pkt = odp_packet_alloc(gbl_args->pool, tp->len); + if (*pkt == ODP_PACKET_INVALID) + ODPH_ABORT("Allocating test packets failed\n"); + if (odp_packet_copy_from_mem(*pkt, 0, tp->len, tp->data)) + ODPH_ABORT("Copying test packet failed\n"); + } +} + +static void create_packets_misc_parsed(void) +{ + odp_packet_parse_param_t param = { + .proto = ODP_PROTO_ETH, + .last_layer = ODP_PROTO_LAYER_ALL, + .chksums.all_chksum = 0 + }; + + create_packets_misc(); + for (int i = 0; i < TEST_REPEAT_COUNT; i++) { + if (odp_packet_parse(gbl_args->pkt_tbl[i], 0, ¶m)) + ODPH_ABORT("Packet parsing failed\n"); + } +} + static void alloc_parse_packets(const void *pkt_data, uint32_t len) { int i; @@ -403,6 +470,54 @@ static void create_packets(void) gbl_args->pkt.seg_len = min_seg_len; } +static int num_misc_event_types = 4; + +static void create_events_misc(void) +{ + odp_packet_t pkt; + odp_buffer_t buf; + odp_timeout_t tmo; + odp_event_vector_t evv; + odp_event_t events[num_misc_event_types]; + int i; + + pkt = odp_packet_alloc(gbl_args->pool, gbl_args->pkt.len); + if (pkt == ODP_PACKET_INVALID) + ODPH_ABORT("Packet alloc failed\n"); + + buf = odp_buffer_alloc(gbl_args->pool_buf); + if (buf == ODP_BUFFER_INVALID) + ODPH_ABORT("Buffer alloc failed\n"); + + tmo = odp_timeout_alloc(gbl_args->pool_tmo); + if (tmo == ODP_TIMEOUT_INVALID) + ODPH_ABORT("Timeout alloc failed\n"); + + events[0] = odp_packet_to_event(pkt); + events[1] = odp_buffer_to_event(buf); + events[2] = odp_timeout_to_event(tmo); + + if (gbl_args->pool_evv != ODP_POOL_INVALID) { + evv = odp_event_vector_alloc(gbl_args->pool_evv); + if (evv == ODP_EVENT_VECTOR_INVALID) + ODPH_ABORT("Event vector alloc failed\n"); + events[3] = odp_event_vector_to_event(evv); + } else { + num_misc_event_types = 3; + } + + for (i = 0; i < num_misc_event_types; i++) + gbl_args->event_tbl[i] = events[i]; + for (; i < TEST_REPEAT_COUNT; i++) + gbl_args->event_tbl[i] = events[rnd() % num_misc_event_types]; +} + +static void free_events_misc(void) +{ + for (int i = 0; i < num_misc_event_types; i++) + odp_event_free(gbl_args->event_tbl[i]); +} + static void create_events(void) { int i; @@ -1265,6 +1380,55 @@ static int event_user_flag_set(void) return i; } +#define DEF_FLAG_TEST_FUN(name, func) \ +static int name(void) \ +{ \ + uint32_t ret = 0; \ + \ + for (int i = 0; i < TEST_REPEAT_COUNT; i++) \ + ret += func(gbl_args->pkt_tbl[i]); \ + \ + return ret <= TEST_REPEAT_COUNT; \ +} + +DEF_FLAG_TEST_FUN(packet_has_error, odp_packet_has_error) +DEF_FLAG_TEST_FUN(packet_has_l2_error, odp_packet_has_l2_error) +DEF_FLAG_TEST_FUN(packet_has_l3_error, odp_packet_has_l3_error) +DEF_FLAG_TEST_FUN(packet_has_l4_error, odp_packet_has_l4_error) +DEF_FLAG_TEST_FUN(packet_has_l2, odp_packet_has_l2) +DEF_FLAG_TEST_FUN(packet_has_l3, odp_packet_has_l3) +DEF_FLAG_TEST_FUN(packet_has_l4, odp_packet_has_l4) +DEF_FLAG_TEST_FUN(packet_has_eth, odp_packet_has_eth) +DEF_FLAG_TEST_FUN(packet_has_eth_bcast, odp_packet_has_eth_bcast) +DEF_FLAG_TEST_FUN(packet_has_eth_mcast, odp_packet_has_eth_mcast) +DEF_FLAG_TEST_FUN(packet_has_jumbo, odp_packet_has_jumbo) +DEF_FLAG_TEST_FUN(packet_has_vlan, odp_packet_has_vlan) +DEF_FLAG_TEST_FUN(packet_has_vlan_qinq, odp_packet_has_vlan_qinq) +DEF_FLAG_TEST_FUN(packet_has_arp, odp_packet_has_arp) +DEF_FLAG_TEST_FUN(packet_has_ipv4, odp_packet_has_ipv4) +DEF_FLAG_TEST_FUN(packet_has_ipv6, odp_packet_has_ipv6) +DEF_FLAG_TEST_FUN(packet_has_ip_bcast, odp_packet_has_ip_bcast) +DEF_FLAG_TEST_FUN(packet_has_ip_mcast, odp_packet_has_ip_mcast) +DEF_FLAG_TEST_FUN(packet_has_ipfrag, odp_packet_has_ipfrag) +DEF_FLAG_TEST_FUN(packet_has_ipopt, odp_packet_has_ipopt) +DEF_FLAG_TEST_FUN(packet_has_ipsec, odp_packet_has_ipsec) +DEF_FLAG_TEST_FUN(packet_has_udp, odp_packet_has_udp) +DEF_FLAG_TEST_FUN(packet_has_tcp, odp_packet_has_tcp) +DEF_FLAG_TEST_FUN(packet_has_sctp, odp_packet_has_sctp) +DEF_FLAG_TEST_FUN(packet_has_icmp, odp_packet_has_icmp) +DEF_FLAG_TEST_FUN(packet_has_flow_hash, odp_packet_has_flow_hash) +DEF_FLAG_TEST_FUN(packet_has_ts, odp_packet_has_ts) + +static int packet_l2_type(void) +{ + uint32_t ret = 0; + + for (int i = 0; i < TEST_REPEAT_COUNT; i++) + ret += odp_packet_l2_type(gbl_args->pkt_tbl[i]); + + return ret; +} + static int packet_l2_ptr(void) { int i; @@ -1298,6 +1462,16 @@ static int packet_l2_offset_set(void) return !ret; } +static int packet_l3_type(void) +{ + uint32_t ret = 0; + + for (int i = 0; i < TEST_REPEAT_COUNT; i++) + ret += odp_packet_l3_type(gbl_args->pkt_tbl[i]); + + return ret; +} + static int packet_l3_ptr(void) { int i; @@ -1331,6 +1505,16 @@ static int packet_l3_offset_set(void) return !ret; } +static int packet_l4_type(void) +{ + uint32_t ret = 0; + + for (int i = 0; i < TEST_REPEAT_COUNT; i++) + ret += odp_packet_l4_type(gbl_args->pkt_tbl[i]); + + return ret; +} + static int packet_l4_ptr(void) { int i; @@ -1478,7 +1662,7 @@ static int event_subtype(void) return i; } -static int packet_parse(void) +static int do_packet_parse(int chksum) { odp_packet_parse_param_t param; odp_packet_t *pkt_tbl = gbl_args->pkt_tbl; @@ -1488,9 +1672,9 @@ static int packet_parse(void) memset(¶m, 0, sizeof(odp_packet_parse_param_t)); param.proto = ODP_PROTO_ETH; param.last_layer = ODP_PROTO_LAYER_ALL; - param.chksums.chksum.ipv4 = 1; - param.chksums.chksum.tcp = 1; - param.chksums.chksum.udp = 1; + param.chksums.chksum.ipv4 = !!chksum; + param.chksums.chksum.tcp = !!chksum; + param.chksums.chksum.udp = !!chksum; for (i = 0; i < TEST_REPEAT_COUNT; i++) ret += odp_packet_parse(pkt_tbl[i], 0, ¶m); @@ -1498,6 +1682,16 @@ static int packet_parse(void) return !ret; } +static int packet_parse(void) +{ + return do_packet_parse(1); +} + +static int packet_parse_no_chksum(void) +{ + return do_packet_parse(0); +} + static int packet_parse_multi(void) { int burst_size = gbl_args->appl.burst_size; @@ -1699,17 +1893,56 @@ bench_info_t test_suite[] = { BENCH_INFO(packet_user_ptr_set, create_packets, free_packets, NULL), BENCH_INFO(packet_user_area, create_packets, free_packets, NULL), BENCH_INFO(event_user_area, create_events, free_packets, NULL), + BENCH_INFO(event_user_area, create_events_misc, free_events_misc, + "event_user_area misc"), BENCH_INFO(packet_user_area_size, create_packets, free_packets, NULL), BENCH_INFO(packet_user_flag, create_packets, free_packets, NULL), BENCH_INFO(event_user_area_and_flag, create_events, free_packets, NULL), + BENCH_INFO(event_user_area_and_flag, create_events_misc, free_events_misc, + "event_user_area_and_flag misc"), BENCH_INFO(packet_user_flag_set, create_packets, free_packets, NULL), BENCH_INFO(event_user_flag_set, create_events, free_packets, NULL), + BENCH_INFO(event_user_flag_set, create_events_misc, free_events_misc, + "event user_flag_set misc"), + +#define BINFO(fun) BENCH_INFO(fun, create_packets_misc_parsed, free_packets, NULL) + BINFO(packet_has_error), + BINFO(packet_has_l2_error), + BINFO(packet_has_l3_error), + BINFO(packet_has_l4_error), + BINFO(packet_has_l2), + BINFO(packet_has_l3), + BINFO(packet_has_l4), + BINFO(packet_has_eth), + BINFO(packet_has_eth_bcast), + BINFO(packet_has_eth_mcast), + BINFO(packet_has_jumbo), + BINFO(packet_has_vlan), + BINFO(packet_has_vlan_qinq), + BINFO(packet_has_arp), + BINFO(packet_has_ipv4), + BINFO(packet_has_ipv6), + BINFO(packet_has_ip_bcast), + BINFO(packet_has_ip_mcast), + BINFO(packet_has_ipfrag), + BINFO(packet_has_ipopt), + BINFO(packet_has_ipsec), + BINFO(packet_has_udp), + BINFO(packet_has_tcp), + BINFO(packet_has_sctp), + BINFO(packet_has_icmp), + BINFO(packet_has_flow_hash), + BINFO(packet_has_ts), + + BENCH_INFO(packet_l2_type, create_packets_misc_parsed, free_packets, NULL), BENCH_INFO(packet_l2_ptr, create_packets, free_packets, NULL), BENCH_INFO(packet_l2_offset, create_packets, free_packets, NULL), BENCH_INFO(packet_l2_offset_set, create_packets, free_packets, NULL), + BENCH_INFO(packet_l3_type, create_packets_misc_parsed, free_packets, NULL), BENCH_INFO(packet_l3_ptr, create_packets, free_packets, NULL), BENCH_INFO(packet_l3_offset, create_packets, free_packets, NULL), BENCH_INFO(packet_l3_offset_set, create_packets, free_packets, NULL), + BENCH_INFO(packet_l4_type, create_packets_misc_parsed, free_packets, NULL), BENCH_INFO(packet_l4_ptr, create_packets, free_packets, NULL), BENCH_INFO(packet_l4_offset, create_packets, free_packets, NULL), BENCH_INFO(packet_l4_offset_set, create_packets, free_packets, NULL), @@ -1731,6 +1964,10 @@ bench_info_t test_suite[] = { "packet_parse ipv6/tcp"), BENCH_INFO(packet_parse, alloc_parse_packets_ipv6_udp, free_packets, "packet_parse ipv6/udp"), + BENCH_INFO(packet_parse, create_packets_misc, free_packets, + "packet_parse misc"), + BENCH_INFO(packet_parse_no_chksum, create_packets_misc, free_packets, + "packet_parse misc no csum"), BENCH_INFO(packet_parse_multi, alloc_parse_packets_multi_ipv4_tcp, free_packets_multi, "packet_parse_multi ipv4/tcp"), BENCH_INFO(packet_parse_multi, alloc_parse_packets_multi_ipv4_udp, free_packets_multi, @@ -1744,6 +1981,55 @@ bench_info_t test_suite[] = { ODP_STATIC_ASSERT(ODPH_ARRAY_SIZE(test_suite) < TEST_MAX_BENCH, "Result array is too small to hold all the results"); +static odp_pool_t create_pool(const char *name, const odp_pool_param_t *param) +{ + odp_pool_t pool = odp_pool_create(name, param); + + if (pool == ODP_POOL_INVALID) { + ODPH_ERR("Error: %s pool creation failed.\n", name); + exit(EXIT_FAILURE); + } + return pool; +} + +static odp_pool_t create_buffer_pool(const odp_pool_capability_t *capa) +{ + odp_pool_param_t param; + + odp_pool_param_init(¶m); + param.type = ODP_POOL_BUFFER; + param.buf.num = 1; + param.buf.size = 1; + param.buf.uarea_size = capa->buf.max_uarea_size > 0 ? 1 : 0; + return create_pool("buf", ¶m); +} + +static odp_pool_t create_timeout_pool(const odp_pool_capability_t *capa) +{ + odp_pool_param_t param; + + odp_pool_param_init(¶m); + param.type = ODP_POOL_TIMEOUT; + param.tmo.num = 1; + param.tmo.uarea_size = capa->tmo.max_uarea_size > 0 ? 1 : 0; + return create_pool("tmo", ¶m); +} + +static odp_pool_t create_evv_pool(const odp_pool_capability_t *capa) +{ + odp_pool_param_t param; + + if (capa->event_vector.max_pools == 0) + return ODP_POOL_INVALID; + + odp_pool_param_init(¶m); + param.type = ODP_POOL_EVENT_VECTOR; + param.event_vector.num = 1; + param.event_vector.max_size = capa->event_vector.max_size; + param.event_vector.uarea_size = capa->event_vector.max_uarea_size > 0 ? 1 : 0; + return create_pool("evv", ¶m); +} + /** * ODP packet microbenchmark application */ @@ -1887,11 +2173,13 @@ int main(int argc, char *argv[]) params.type = ODP_POOL_PACKET; gbl_args->pool = odp_pool_create("packet pool", ¶ms); - if (gbl_args->pool == ODP_POOL_INVALID) { ODPH_ERR("Error: packet pool create failed.\n"); exit(EXIT_FAILURE); } + gbl_args->pool_buf = create_buffer_pool(&capa); + gbl_args->pool_tmo = create_timeout_pool(&capa); + gbl_args->pool_evv = create_evv_pool(&capa); printf("CPU: %i\n", odp_cpumask_first(&cpumask)); printf("CPU mask: %s\n", cpumaskstr); @@ -1934,7 +2222,10 @@ int main(int argc, char *argv[]) ret = gbl_args->suite.retval; - if (odp_pool_destroy(gbl_args->pool)) { + if (odp_pool_destroy(gbl_args->pool) || + odp_pool_destroy(gbl_args->pool_buf) || + odp_pool_destroy(gbl_args->pool_tmo) || + (gbl_args->pool_evv != ODP_POOL_INVALID && odp_pool_destroy(gbl_args->pool_evv))) { ODPH_ERR("Error: pool destroy\n"); exit(EXIT_FAILURE); }