From 3a56e86a7379229a416f70e30ad7e8b700819dc7 Mon Sep 17 00:00:00 2001 From: Andrew Yourtchenko Date: Wed, 24 Jan 2024 12:41:23 +0000 Subject: [PATCH 001/271] misc: Initial changes for stable/2402 branch Type: docs Change-Id: I820bbb54597a8f640ed6b854d20d0b572c5f255b Signed-off-by: Andrew Yourtchenko --- .gitreview | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitreview b/.gitreview index 1db08df202..fadff209a0 100644 --- a/.gitreview +++ b/.gitreview @@ -2,3 +2,4 @@ host=gerrit.fd.io port=29418 project=vpp +defaultbranch=stable/2402 From 8cbf84dce02102ae1e9e6c2545fdea6c5673bc22 Mon Sep 17 00:00:00 2001 From: Florin Coras Date: Mon, 12 Feb 2024 22:33:41 -0500 Subject: [PATCH 002/271] vcl: fix epollet test for unhandled evts Argument to vcl_epoll_ctl_add_unhandled_event is often the result of an and between events and EPOLLET which is larger than u8 Type: fix Change-Id: I8c98f557fa1db9f3eb79c90ecdd60ac9366d4d40 Signed-off-by: Florin Coras (cherry picked from commit e81f27ffb2a698737eae607b111d0611d221222f) --- src/vcl/vppcom.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/vcl/vppcom.c b/src/vcl/vppcom.c index d9c3b30afe..5174fe844e 100644 --- a/src/vcl/vppcom.c +++ b/src/vcl/vppcom.c @@ -2830,7 +2830,7 @@ vppcom_epoll_create (void) static void vcl_epoll_ctl_add_unhandled_event (vcl_worker_t *wrk, vcl_session_t *s, - u8 is_epollet, session_evt_type_t evt) + u32 is_epollet, session_evt_type_t evt) { if (!is_epollet) { From ccfc24f7451b2b6a6f8eabf735812d45ec4a4bf9 Mon Sep 17 00:00:00 2001 From: Vratko Polak Date: Tue, 6 Feb 2024 12:45:59 +0100 Subject: [PATCH 003/271] buffers: bring back cache occupancy improvement The improvement was removed in 40129, causing 5-40% regressions in AVF tests. There is a memory-speed trade-off, this change prefers speed over memory efficiency. Ideally, the choice should be configurable, but that is not easy to achieve, considering how early is vlib_buffer_main_init called. Type: fix Fixes: 038dad7ef29b0b724071edb5f8cc7a9845584454 Change-Id: I4746f3634abe6d233c9d092a372de05b3d1ae4b6 Signed-off-by: Vratko Polak (cherry picked from commit 04fd51c03c428859bae949a8294ee0f9c062a44b) --- src/vlib/buffer.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/vlib/buffer.c b/src/vlib/buffer.c index 82fe641278..b5200ba502 100644 --- a/src/vlib/buffer.c +++ b/src/vlib/buffer.c @@ -473,6 +473,10 @@ vlib_buffer_alloc_size (uword ext_hdr_size, uword data_size) uword alloc_size = ext_hdr_size + sizeof (vlib_buffer_t) + data_size; alloc_size = round_pow2 (alloc_size, VLIB_BUFFER_ALIGN); + /* in case when we have even number of 'cachelines', we add one more for + * better cache occupancy */ + alloc_size |= VLIB_BUFFER_ALIGN; + return alloc_size; } From 455960759b5417c767ed331748c7ee76662ffd18 Mon Sep 17 00:00:00 2001 From: Andrew Yourtchenko Date: Tue, 20 Feb 2024 15:19:56 +0000 Subject: [PATCH 004/271] misc: VPP 24.02 Release Notes Type: docs Change-Id: I4209b4edf387f2d7b88ecc338cca3b4901210ed2 Signed-off-by: Andrew Yourtchenko --- docs/aboutvpp/releasenotes/index.rst | 1 + docs/aboutvpp/releasenotes/v24.02.rst | 567 ++++++++++++++++++++++++++ 2 files changed, 568 insertions(+) create mode 100644 docs/aboutvpp/releasenotes/v24.02.rst diff --git a/docs/aboutvpp/releasenotes/index.rst b/docs/aboutvpp/releasenotes/index.rst index b22febfa92..6c3a830cbd 100644 --- a/docs/aboutvpp/releasenotes/index.rst +++ b/docs/aboutvpp/releasenotes/index.rst @@ -6,6 +6,7 @@ Release notes .. toctree:: :maxdepth: 2 + v24.02 v23.10 v23.06 v23.02 diff --git a/docs/aboutvpp/releasenotes/v24.02.rst b/docs/aboutvpp/releasenotes/v24.02.rst new file mode 100644 index 0000000000..61051ce759 --- /dev/null +++ b/docs/aboutvpp/releasenotes/v24.02.rst @@ -0,0 +1,567 @@ +Release notes for VPP 24.02 +=========================== + +More than 262 commits since the previous release, including 123 fixes. + +Features +-------- + +- Build System + + - Modify N\_PREFETCH on Arm N2 to achieve best perf (`bef2d6da4 `_) + - Add ability to disable some plugins from packaging and tests (`bc37878ec `_) + +- Infrastructure Library + + - Native AES-CTR implementation (`9caef2a35 `_) + +- Plugins + + - Amazon Elastic Network Adapter (ENA) device driver + + - Amazon Elastic Network Adapter (ENA) native driver (`2d725c612 `_) + + - CNat + + - Add flow hash config to cnat translation (`589fe7ca6 `_) + + - Crypto - ipsecmb + + - Bump intel-ipsec-mb version to 1.4 (`40242b88e `_) + - Bump intel-ipsec-mb version to 1.5 (`adb2c6799 `_) + + - Crypto - native + + - Add AES-CTR (`da3771c25 `_) + + - DPDK + + - Add ConnectX-6LX and ConnectX-7 support (`029f039d5 `_) + - Add Mellanox BlueField NICs (`006c071b0 `_) + - Bump to DPDK 23.11 (`327c32306 `_) + - Bump rdma-core to 49.0 (`b1a1209ce `_) + - Add ID for QAT 4xxx series VF support (`ebe2371e6 `_) + + - GTPU + + - Support non-G-PDU packets and PDU Session (`f9ab6985d `_) + + - IAVF Device driver + + - New driver using new dev infra (`47447f1f5 `_) + + - IPv6 Segment Routing Mobile + + - Implement SRv6 mobile API funcs (`68ac24428 `_) + + - Marvell Octeon device driver + + - Native driver for Marvell Octeon SoC (`01fe7ab88 `_) + + - NPTv6 + + - Icmp6 alg to handle icmp6 error messages (`ff344a98a `_) + +- VNET + + - FLOW + + - Add support for using l2tpv3 as RSS type (`6cb727394 `_) + + - IPSec + + - Allow receiving encrypted IP packets with TFC padding (`8fce54637 `_) + + - New Device Drivers Infra + + - New device driver infra (`38c619115 `_) + + - Session Layer + + - Make port range configurable (`e111bbd12 `_) + + +Known issues +------------ + +For the full list of issues please refer to fd.io `JIRA `_. + +Fixed issues +------------ + +For the full list of fixed issues please refer to: +- fd.io `JIRA `_ +- git `commit log `_ + + +API changes +----------- + +Description of results: + +- *Definition changed*: indicates that the API file was modified between releases. +- *Only in image*: indicates the API is new for this release. +- *Only in file*: indicates the API has been removed in this release. + +============================================================= ================== +Message Name Result +============================================================= ================== +cnat_translation_details definition changed +cnat_translation_update definition changed +dev_attach only in image +dev_attach_reply only in image +dev_create_port_if only in image +dev_create_port_if_reply only in image +dev_detach only in image +dev_detach_reply only in image +dev_remove_port_if only in image +dev_remove_port_if_reply only in image +dhcp_client_detect_enable_disable only in image +dhcp_client_detect_enable_disable_reply only in image +gtpu_add_del_forward only in image +gtpu_add_del_forward_reply only in image +gtpu_add_del_tunnel_v2 only in image +gtpu_add_del_tunnel_v2_reply only in image +gtpu_get_transfer_counts only in image +gtpu_get_transfer_counts_reply only in image +gtpu_tunnel_v2_details only in image +gtpu_tunnel_v2_dump only in image +ipsec_sa_v5_details only in image +ipsec_sa_v5_dump only in image +ipsec_sad_entry_add_v2 only in image +ipsec_sad_entry_add_v2_reply only in image +lldp_details only in image +lldp_dump only in image +lldp_dump_reply only in image +ping_finished_event only in image +rdma_create_v4 only in image +rdma_create_v4_reply only in image +sr_mobile_localsid_add_del only in image +sr_mobile_localsid_add_del_reply only in image +sr_mobile_policy_add only in image +sr_mobile_policy_add_reply only in image +urpf_interface_details only in image +urpf_interface_dump only in image +want_ping_finished_events only in image +want_ping_finished_events_reply only in image +============================================================= ================== + +Found 38 api message signature differences + + +Newly deprecated API messages +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +These messages are still there in the API, but can and probably +will disappear in the next release. + +- rdma_create_v3 + +In-progress API messages +~~~~~~~~~~~~~~~~~~~~~~~~ + +These messages are provided for testing and experimentation only. +They are *not* subject to any compatibility process, +and therefore can arbitrarily change or disappear at *any* moment. +Also they may have less than satisfactory testing, making +them unsuitable for other use than the technology preview. +If you are intending to use these messages in production projects, +please collaborate with the feature maintainer on their productization. + +- abf_itf_attach_add_del +- abf_itf_attach_add_del_reply +- abf_itf_attach_details +- abf_itf_attach_dump +- abf_plugin_get_version +- abf_plugin_get_version_reply +- abf_policy_add_del +- abf_policy_add_del_reply +- abf_policy_details +- abf_policy_dump +- acl_plugin_use_hash_lookup_get +- acl_plugin_use_hash_lookup_get_reply +- acl_plugin_use_hash_lookup_set +- acl_plugin_use_hash_lookup_set_reply +- bpf_trace_filter_set +- bpf_trace_filter_set_reply +- cnat_get_snat_addresses +- cnat_get_snat_addresses_reply +- cnat_session_details +- cnat_session_dump +- cnat_session_purge +- cnat_session_purge_reply +- cnat_set_snat_addresses +- cnat_set_snat_addresses_reply +- cnat_set_snat_policy +- cnat_set_snat_policy_reply +- cnat_snat_policy_add_del_exclude_pfx +- cnat_snat_policy_add_del_exclude_pfx_reply +- cnat_snat_policy_add_del_if +- cnat_snat_policy_add_del_if_reply +- cnat_translation_del +- cnat_translation_del_reply +- cnat_translation_details +- cnat_translation_dump +- cnat_translation_update +- cnat_translation_update_reply +- det44_get_timeouts_reply +- det44_set_timeouts +- det44_set_timeouts_reply +- dev_attach +- dev_attach_reply +- dev_create_port_if +- dev_create_port_if_reply +- dev_detach +- dev_detach_reply +- dev_remove_port_if +- dev_remove_port_if_reply +- flowprobe_get_params +- flowprobe_get_params_reply +- flowprobe_interface_add_del +- flowprobe_interface_add_del_reply +- flowprobe_interface_details +- flowprobe_interface_dump +- flowprobe_set_params +- flowprobe_set_params_reply +- gbp_bridge_domain_add +- gbp_bridge_domain_add_reply +- gbp_bridge_domain_del +- gbp_bridge_domain_del_reply +- gbp_bridge_domain_details +- gbp_bridge_domain_dump +- gbp_bridge_domain_dump_reply +- gbp_contract_add_del +- gbp_contract_add_del_reply +- gbp_contract_details +- gbp_contract_dump +- gbp_endpoint_add +- gbp_endpoint_add_reply +- gbp_endpoint_del +- gbp_endpoint_del_reply +- gbp_endpoint_details +- gbp_endpoint_dump +- gbp_endpoint_group_add +- gbp_endpoint_group_add_reply +- gbp_endpoint_group_del +- gbp_endpoint_group_del_reply +- gbp_endpoint_group_details +- gbp_endpoint_group_dump +- gbp_ext_itf_add_del +- gbp_ext_itf_add_del_reply +- gbp_ext_itf_details +- gbp_ext_itf_dump +- gbp_recirc_add_del +- gbp_recirc_add_del_reply +- gbp_recirc_details +- gbp_recirc_dump +- gbp_route_domain_add +- gbp_route_domain_add_reply +- gbp_route_domain_del +- gbp_route_domain_del_reply +- gbp_route_domain_details +- gbp_route_domain_dump +- gbp_route_domain_dump_reply +- gbp_subnet_add_del +- gbp_subnet_add_del_reply +- gbp_subnet_details +- gbp_subnet_dump +- gbp_vxlan_tunnel_add +- gbp_vxlan_tunnel_add_reply +- gbp_vxlan_tunnel_del +- gbp_vxlan_tunnel_del_reply +- gbp_vxlan_tunnel_details +- gbp_vxlan_tunnel_dump +- gtpu_add_del_forward +- gtpu_add_del_forward_reply +- gtpu_add_del_tunnel_v2 +- gtpu_add_del_tunnel_v2_reply +- gtpu_get_transfer_counts +- gtpu_get_transfer_counts_reply +- gtpu_tunnel_v2_details +- gtpu_tunnel_v2_dump +- ikev2_child_sa_details +- ikev2_child_sa_dump +- ikev2_initiate_del_child_sa +- ikev2_initiate_del_child_sa_reply +- ikev2_initiate_del_ike_sa +- ikev2_initiate_del_ike_sa_reply +- ikev2_initiate_rekey_child_sa +- ikev2_initiate_rekey_child_sa_reply +- ikev2_initiate_sa_init +- ikev2_initiate_sa_init_reply +- ikev2_nonce_get +- ikev2_nonce_get_reply +- ikev2_profile_add_del +- ikev2_profile_add_del_reply +- ikev2_profile_details +- ikev2_profile_disable_natt +- ikev2_profile_disable_natt_reply +- ikev2_profile_dump +- ikev2_profile_set_auth +- ikev2_profile_set_auth_reply +- ikev2_profile_set_id +- ikev2_profile_set_id_reply +- ikev2_profile_set_ipsec_udp_port +- ikev2_profile_set_ipsec_udp_port_reply +- ikev2_profile_set_liveness +- ikev2_profile_set_liveness_reply +- ikev2_profile_set_ts +- ikev2_profile_set_ts_reply +- ikev2_profile_set_udp_encap +- ikev2_profile_set_udp_encap_reply +- ikev2_sa_details +- ikev2_sa_dump +- ikev2_set_esp_transforms +- ikev2_set_esp_transforms_reply +- ikev2_set_ike_transforms +- ikev2_set_ike_transforms_reply +- ikev2_set_local_key +- ikev2_set_local_key_reply +- ikev2_set_responder +- ikev2_set_responder_hostname +- ikev2_set_responder_hostname_reply +- ikev2_set_responder_reply +- ikev2_set_sa_lifetime +- ikev2_set_sa_lifetime_reply +- ikev2_set_tunnel_interface +- ikev2_set_tunnel_interface_reply +- ikev2_traffic_selector_details +- ikev2_traffic_selector_dump +- ip_neighbor_config_get +- ip_neighbor_config_get_reply +- ip_route_add_del_v2 +- ip_route_add_del_v2_reply +- ip_route_lookup_v2 +- ip_route_lookup_v2_reply +- ip_route_v2_details +- ip_route_v2_dump +- ip_session_redirect_add +- ip_session_redirect_add_reply +- ip_session_redirect_add_v2 +- ip_session_redirect_add_v2_reply +- ip_session_redirect_del +- ip_session_redirect_del_reply +- l2_emulation +- l2_emulation_reply +- lcp_default_ns_get_reply +- lcp_default_ns_set +- lcp_default_ns_set_reply +- lcp_itf_pair_add_del_v2 +- lcp_itf_pair_add_del_v2_reply +- lcp_itf_pair_details +- lldp_details +- mdata_enable_disable +- mdata_enable_disable_reply +- nat44_ed_vrf_tables_v2_details +- nat44_ed_vrf_tables_v2_dump +- nat44_ei_add_del_address_range +- nat44_ei_add_del_address_range_reply +- nat44_ei_add_del_static_mapping +- nat44_ei_add_del_static_mapping_reply +- nat44_ei_address_details +- nat44_ei_address_dump +- nat44_ei_del_session +- nat44_ei_del_session_reply +- nat44_ei_del_user +- nat44_ei_del_user_reply +- nat44_ei_forwarding_enable_disable +- nat44_ei_forwarding_enable_disable_reply +- nat44_ei_ha_flush +- nat44_ei_ha_flush_reply +- nat44_ei_ha_resync +- nat44_ei_ha_resync_completed_event +- nat44_ei_ha_resync_reply +- nat44_ei_ha_set_failover +- nat44_ei_ha_set_failover_reply +- nat44_ei_ha_set_listener +- nat44_ei_ha_set_listener_reply +- nat44_ei_interface_add_del_feature +- nat44_ei_interface_add_del_feature_reply +- nat44_ei_interface_details +- nat44_ei_interface_dump +- nat44_ei_ipfix_enable_disable +- nat44_ei_ipfix_enable_disable_reply +- nat44_ei_plugin_enable_disable +- nat44_ei_plugin_enable_disable_reply +- nat44_ei_set_addr_and_port_alloc_alg +- nat44_ei_set_addr_and_port_alloc_alg_reply +- nat44_ei_set_fq_options +- nat44_ei_set_fq_options_reply +- nat44_ei_set_mss_clamping +- nat44_ei_set_mss_clamping_reply +- nat44_ei_set_timeouts +- nat44_ei_set_timeouts_reply +- nat44_ei_set_workers +- nat44_ei_set_workers_reply +- nat44_ei_show_fq_options +- nat44_ei_show_fq_options_reply +- nat44_ei_show_running_config +- nat44_ei_show_running_config_reply +- nat44_ei_static_mapping_details +- nat44_ei_static_mapping_dump +- nat44_ei_user_details +- nat44_ei_user_dump +- nat44_ei_user_session_details +- nat44_ei_user_session_dump +- nat44_ei_user_session_v2_details +- nat44_ei_user_session_v2_dump +- nat44_ei_worker_details +- nat44_ei_worker_dump +- nat64_plugin_enable_disable +- nat64_plugin_enable_disable_reply +- npt66_binding_add_del +- npt66_binding_add_del_reply +- oddbuf_enable_disable +- oddbuf_enable_disable_reply +- pg_interface_enable_disable_coalesce +- pg_interface_enable_disable_coalesce_reply +- ping_finished_event +- pnat_binding_add +- pnat_binding_add_reply +- pnat_binding_add_v2 +- pnat_binding_add_v2_reply +- pnat_binding_attach +- pnat_binding_attach_reply +- pnat_binding_del +- pnat_binding_del_reply +- pnat_binding_detach +- pnat_binding_detach_reply +- pnat_bindings_details +- pnat_bindings_get +- pnat_bindings_get_reply +- pnat_interfaces_details +- pnat_interfaces_get +- pnat_interfaces_get_reply +- sample_macswap_enable_disable +- sample_macswap_enable_disable_reply +- set_ip_flow_hash_v3 +- set_ip_flow_hash_v3_reply +- sr_localsids_with_packet_stats_details +- sr_localsids_with_packet_stats_dump +- sr_mobile_localsid_add_del +- sr_mobile_localsid_add_del_reply +- sr_mobile_policy_add +- sr_mobile_policy_add_reply +- sr_policies_with_sl_index_details +- sr_policies_with_sl_index_dump +- sr_policy_add_v2 +- sr_policy_add_v2_reply +- sr_policy_mod_v2 +- sr_policy_mod_v2_reply +- sw_interface_ip6nd_ra_details +- sw_interface_ip6nd_ra_dump +- sw_interface_set_vxlan_gbp_bypass +- sw_interface_set_vxlan_gbp_bypass_reply +- test_addresses +- test_addresses2 +- test_addresses2_reply +- test_addresses3 +- test_addresses3_reply +- test_addresses_reply +- test_empty +- test_empty_reply +- test_enum +- test_enum_reply +- test_interface +- test_interface_reply +- test_prefix +- test_prefix_reply +- test_string +- test_string2 +- test_string2_reply +- test_string_reply +- test_vla +- test_vla2 +- test_vla2_reply +- test_vla3 +- test_vla3_reply +- test_vla4 +- test_vla4_reply +- test_vla5 +- test_vla5_reply +- test_vla_reply +- trace_capture_packets +- trace_capture_packets_reply +- trace_clear_cache +- trace_clear_cache_reply +- trace_clear_capture +- trace_clear_capture_reply +- trace_details +- trace_dump +- trace_dump_reply +- trace_filter_function_details +- trace_filter_function_dump +- trace_set_filter_function +- trace_set_filter_function_reply +- trace_set_filters +- trace_set_filters_reply +- trace_v2_details +- trace_v2_dump +- tracenode_enable_disable +- tracenode_enable_disable_reply +- vxlan_gbp_tunnel_add_del +- vxlan_gbp_tunnel_add_del_reply +- vxlan_gbp_tunnel_details +- vxlan_gbp_tunnel_dump +- want_ping_finished_events +- want_ping_finished_events_reply + +Patches that changed API definitions +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + +``src/plugins/urpf/urpf.api`` + +* `2fa69effc `_ urpf: add interface dump to API + +``src/plugins/srv6-mobile/sr_mobile_types.api`` + +* `68ac24428 `_ srv6-mobile: Implement SRv6 mobile API funcs + +``src/plugins/srv6-mobile/sr_mobile.api`` + +* `68ac24428 `_ srv6-mobile: Implement SRv6 mobile API funcs + +``src/plugins/npt66/npt66.api`` + +* `bdeee2194 `_ npt66: add show command and rx/tx counters + +``src/plugins/gtpu/gtpu.api`` + +* `f9ab6985d `_ gtpu: support non-G-PDU packets and PDU Session + +``src/plugins/dhcp/dhcp.api`` + +* `f3be34e44 `_ dhcp: api to enable client detect on interface + +``src/plugins/ping/ping.api`` + +* `bb1cde678 `_ ping: Simple binary API for running ping based on events + +``src/plugins/lldp/lldp.api`` + +* `9f8d3b9b2 `_ lldp: dump api + +``src/plugins/rdma/rdma.api`` + +* `04d262d1e `_ rdma: add rdma_create_v4 that handles flags properly + +``src/plugins/cnat/cnat.api`` + +* `589fe7ca6 `_ cnat: add flow hash config to cnat translation + +``src/vnet/dev/dev.api`` + +* `ddf6cec37 `_ dev: initial set of APIs + +``src/vnet/ipsec/ipsec.api`` + +* `0e2f188f7 `_ ipsec: huge anti-replay window support + +``src/vnet/ipsec/ipsec_types.api`` + +* `0e2f188f7 `_ ipsec: huge anti-replay window support + +``src/vnet/devices/virtio/virtio.api`` + +* `00c59e496 `_ virtio: virtio_flags api use enumflag instead of enum From 7453b5dfe8a52d0e033ec675eba3ada26ba771a3 Mon Sep 17 00:00:00 2001 From: Florin Coras Date: Wed, 14 Feb 2024 16:14:46 -0800 Subject: [PATCH 005/271] session: postpone ct cleanup if rx evt pending Type: fix Change-Id: I8cfaa62abd38d5356263b0ffd428638d1a027617 Signed-off-by: Florin Coras (cherry picked from commit 3efcbaf3b1119b4312ae1f3a1c59dea2d746bec4) --- src/vnet/session/application_local.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/vnet/session/application_local.c b/src/vnet/session/application_local.c index 8c6cf8a93f..5bd1471b6f 100644 --- a/src/vnet/session/application_local.c +++ b/src/vnet/session/application_local.c @@ -1132,10 +1132,10 @@ ct_handle_cleanups (void *args) clib_fifo_sub2 (wrk->pending_cleanups, req); ct = ct_connection_get (req->ct_index, thread_index); s = session_get (ct->c_s_index, ct->c_thread_index); - if (!svm_fifo_has_event (s->tx_fifo)) - ct_session_postponed_cleanup (ct); - else + if (svm_fifo_has_event (s->tx_fifo) || (s->flags & SESSION_F_RX_EVT)) clib_fifo_add1 (wrk->pending_cleanups, *req); + else + ct_session_postponed_cleanup (ct); n_to_handle -= 1; } @@ -1411,6 +1411,7 @@ ct_session_tx (session_t * s) peer_s = session_get (peer_ct->c_s_index, peer_ct->c_thread_index); if (peer_s->session_state >= SESSION_STATE_TRANSPORT_CLOSING) return 0; + peer_s->flags |= SESSION_F_RX_EVT; return session_enqueue_notify (peer_s); } From e4ec1584ec1dbc0f2d0343d4cf9c91455059789b Mon Sep 17 00:00:00 2001 From: Florin Coras Date: Tue, 13 Feb 2024 15:37:20 -0500 Subject: [PATCH 006/271] tls: mark ho done atomically after ctx init Make sure ctx is initialized before ho is marked as done. Type: fix Change-Id: If0525a9890a56e289e2ab006c669a9d64dc6505d Signed-off-by: Florin Coras (cherry picked from commit 0ded4890beaa3aa1f36c61ff6125d19582b25391) --- src/vnet/tls/tls.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/vnet/tls/tls.c b/src/vnet/tls/tls.c index a27d731aca..358e3a7b2e 100644 --- a/src/vnet/tls/tls.c +++ b/src/vnet/tls/tls.c @@ -594,12 +594,13 @@ tls_session_connected_cb (u32 tls_app_index, u32 ho_ctx_index, u32 ctx_handle; ho_ctx = tls_ctx_half_open_get (ho_ctx_index); - ho_ctx->flags |= TLS_CONN_F_HO_DONE; ctx_handle = tls_ctx_alloc (ho_ctx->tls_ctx_engine); ctx = tls_ctx_get (ctx_handle); clib_memcpy_fast (ctx, ho_ctx, sizeof (*ctx)); + /* Half-open freed on tcp half-open cleanup notification */ + __atomic_fetch_or (&ho_ctx->flags, TLS_CONN_F_HO_DONE, __ATOMIC_RELEASE); ctx->c_thread_index = vlib_get_thread_index (); ctx->tls_ctx_handle = ctx_handle; From 4991354309cd71dee8f9e8c2964e6c7f117d5119 Mon Sep 17 00:00:00 2001 From: Matthew Smith Date: Mon, 5 Feb 2024 22:57:55 +0000 Subject: [PATCH 007/271] ip: force full reassembly before virtual Type: improvement The vnet buffer metadata for full IP reassembly and shallow virtual reassembly overlaps. If you have full reassembly and virtual reassembly enabled on the same interface and virtual reassembly happens to process packets first, full reassembly will stomp on the metadata populated by virtual reassembly. Virtual reassembly gets enabled implicitly when NAT feature nodes are enabled. Those NAT feature nodes rely on the virtual reassembly metadata being populated correctly in order to find L4 proto & ports. When NAT and IP full reassembly are both enabled on an interface, NAT can drop fragmented packets because the virtual reassembly metadata can be overwritten by full reassembly. Ensure that full reassembly runs before virtual reassembly. Add a runs_before dependency to ensure that ip4-full-reassembly-feature runs before ip4-sv-reassembly-feature. There was a duplicate VNET_FEATURE_INIT() for ip4-full-reassembly-feature. It seems to have been intended for enabling ip4-full-reassembly-custom as a feature node, but its contents are identical to the earlier VNET_FEATURE_INIT() for ip4-full-reassembly-feature. Removed the duplicate. Change-Id: Ie600b854d4ceb90a7cb736810140d410b8f72447 Signed-off-by: Matthew Smith (cherry picked from commit 205ed8f8845a8ea36f38ed29df158a5a07c2e2c3) --- src/vnet/ip/reass/ip4_full_reass.c | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/src/vnet/ip/reass/ip4_full_reass.c b/src/vnet/ip/reass/ip4_full_reass.c index c65f349e5b..7f0b8d90c1 100644 --- a/src/vnet/ip/reass/ip4_full_reass.c +++ b/src/vnet/ip/reass/ip4_full_reass.c @@ -1424,11 +1424,11 @@ VLIB_REGISTER_NODE (ip4_full_reass_node_feature) = { }; VNET_FEATURE_INIT (ip4_full_reass_feature, static) = { - .arc_name = "ip4-unicast", - .node_name = "ip4-full-reassembly-feature", - .runs_before = VNET_FEATURES ("ip4-lookup", - "ipsec4-input-feature"), - .runs_after = 0, + .arc_name = "ip4-unicast", + .node_name = "ip4-full-reassembly-feature", + .runs_before = VNET_FEATURES ("ip4-lookup", "ipsec4-input-feature", + "ip4-sv-reassembly-feature"), + .runs_after = 0, }; VLIB_NODE_FN (ip4_full_reass_node_custom) (vlib_main_t * vm, @@ -1453,15 +1453,6 @@ VLIB_REGISTER_NODE (ip4_full_reass_node_custom) = { }, }; -VNET_FEATURE_INIT (ip4_full_reass_custom, static) = { - .arc_name = "ip4-unicast", - .node_name = "ip4-full-reassembly-feature", - .runs_before = VNET_FEATURES ("ip4-lookup", - "ipsec4-input-feature"), - .runs_after = 0, -}; - - #ifndef CLIB_MARCH_VARIANT uword ip4_full_reass_custom_register_next_node (uword node_index) From 1200c799d0b346f28deb76daa8f643e771976da7 Mon Sep 17 00:00:00 2001 From: Anton Nikolaev Date: Tue, 30 Jan 2024 11:54:00 +0000 Subject: [PATCH 008/271] linux-cp: add add_del_v3 and get_v2 methods Added vl_api_lcp_itf_pair_add_del_v3_t_handler method, it can return vif_index in reply. Also added vl_api_lcp_itf_pair_get_v2_t_handler methods, this method is able to dump only one lcp pair or dump all lcp pairs via stream_msg. Type: improvement Change-Id: I1d25344ee57f8fac8b857bb3a9a03116230b4d2c Signed-off-by: Anton Nikolaev (cherry picked from commit 83ad79d69a09f504ba6ce3325fc165648eb55daa) --- src/plugins/linux-cp/lcp.api | 39 ++++++++++++++++++ src/plugins/linux-cp/lcp_api.c | 75 +++++++++++++++++++++++++++++++--- 2 files changed, 108 insertions(+), 6 deletions(-) diff --git a/src/plugins/linux-cp/lcp.api b/src/plugins/linux-cp/lcp.api index adef1b8515..e7eaa5a366 100644 --- a/src/plugins/linux-cp/lcp.api +++ b/src/plugins/linux-cp/lcp.api @@ -103,6 +103,27 @@ define lcp_itf_pair_add_del_v2_reply i32 retval; vl_api_interface_index_t host_sw_if_index; }; +autoendian define lcp_itf_pair_add_del_v3 +{ + option in_progress; + + u32 client_index; + u32 context; + bool is_add; + vl_api_interface_index_t sw_if_index; + string host_if_name[16]; /* IFNAMSIZ */ + vl_api_lcp_itf_host_type_t host_if_type; + string netns[32]; /* LCP_NS_LEN */ +}; +define lcp_itf_pair_add_del_v3_reply +{ + option in_progress; + + u32 context; + i32 retval; + u32 vif_index; + vl_api_interface_index_t host_sw_if_index; +}; /** \brief Dump Linux Control Plane interface pair data @param client_index - opaque cookie to identify the sender @@ -121,6 +142,19 @@ autoendian define lcp_itf_pair_get_reply i32 retval; u32 cursor; }; +autoendian define lcp_itf_pair_get_v2 +{ + u32 client_index; + u32 context; + u32 cursor; + vl_api_interface_index_t sw_if_index; +}; +autoendian define lcp_itf_pair_get_v2_reply +{ + u32 context; + i32 retval; + u32 cursor; +}; /** \brief Linux Control Plane interface pair dump response @param context - sender context which was passed in the request @@ -148,6 +182,11 @@ service { stream lcp_itf_pair_details; }; +service { + rpc lcp_itf_pair_get_v2 returns lcp_itf_pair_get_v2_reply + stream lcp_itf_pair_details; +}; + /** \brief Replace end/begin */ autoreply define lcp_itf_pair_replace_begin diff --git a/src/plugins/linux-cp/lcp_api.c b/src/plugins/linux-cp/lcp_api.c index a217aa708f..991516a3ec 100644 --- a/src/plugins/linux-cp/lcp_api.c +++ b/src/plugins/linux-cp/lcp_api.c @@ -45,7 +45,7 @@ static int vl_api_lcp_itf_pair_add (u32 phy_sw_if_index, lip_host_type_t lip_host_type, u8 *mp_host_if_name, size_t sizeof_host_if_name, u8 *mp_namespace, size_t sizeof_mp_namespace, - u32 *host_sw_if_index_p) + u32 *host_sw_if_index_p, u32 *vif_index_p) { u8 *host_if_name, *netns; int host_len, netns_len, rv; @@ -64,6 +64,13 @@ vl_api_lcp_itf_pair_add (u32 phy_sw_if_index, lip_host_type_t lip_host_type, rv = lcp_itf_pair_create (phy_sw_if_index, host_if_name, lip_host_type, netns, host_sw_if_index_p); + if (!rv && (vif_index_p != NULL)) + { + lcp_itf_pair_t *pair = + lcp_itf_pair_get (lcp_itf_pair_find_by_phy (phy_sw_if_index)); + *vif_index_p = pair->lip_vif_index; + } + vec_free (host_if_name); vec_free (netns); @@ -86,7 +93,7 @@ vl_api_lcp_itf_pair_add_del_t_handler (vl_api_lcp_itf_pair_add_del_t *mp) { rv = vl_api_lcp_itf_pair_add ( phy_sw_if_index, lip_host_type, mp->host_if_name, - sizeof (mp->host_if_name), mp->netns, sizeof (mp->netns), NULL); + sizeof (mp->host_if_name), mp->netns, sizeof (mp->netns), NULL, NULL); } else { @@ -111,10 +118,10 @@ vl_api_lcp_itf_pair_add_del_v2_t_handler (vl_api_lcp_itf_pair_add_del_v2_t *mp) lip_host_type = api_decode_host_type (mp->host_if_type); if (mp->is_add) { - rv = vl_api_lcp_itf_pair_add (phy_sw_if_index, lip_host_type, - mp->host_if_name, - sizeof (mp->host_if_name), mp->netns, - sizeof (mp->netns), &host_sw_if_index); + rv = vl_api_lcp_itf_pair_add ( + phy_sw_if_index, lip_host_type, mp->host_if_name, + sizeof (mp->host_if_name), mp->netns, sizeof (mp->netns), + &host_sw_if_index, NULL); } else { @@ -126,6 +133,37 @@ vl_api_lcp_itf_pair_add_del_v2_t_handler (vl_api_lcp_itf_pair_add_del_v2_t *mp) { rmp->host_sw_if_index = host_sw_if_index; }); } +static void +vl_api_lcp_itf_pair_add_del_v3_t_handler (vl_api_lcp_itf_pair_add_del_v3_t *mp) +{ + u32 phy_sw_if_index, host_sw_if_index = ~0, vif_index = ~0; + vl_api_lcp_itf_pair_add_del_v3_reply_t *rmp; + lip_host_type_t lip_host_type; + int rv; + + VALIDATE_SW_IF_INDEX_END (mp); + + phy_sw_if_index = mp->sw_if_index; + lip_host_type = api_decode_host_type (mp->host_if_type); + if (mp->is_add) + { + rv = vl_api_lcp_itf_pair_add ( + phy_sw_if_index, lip_host_type, mp->host_if_name, + sizeof (mp->host_if_name), mp->netns, sizeof (mp->netns), + &host_sw_if_index, &vif_index); + } + else + { + rv = lcp_itf_pair_delete (phy_sw_if_index); + } + + BAD_SW_IF_INDEX_LABEL; + REPLY_MACRO2_END (VL_API_LCP_ITF_PAIR_ADD_DEL_V3_REPLY, ({ + rmp->host_sw_if_index = host_sw_if_index; + rmp->vif_index = vif_index; + })); +} + static void send_lcp_itf_pair_details (index_t lipi, vl_api_registration_t *rp, u32 context) @@ -161,6 +199,31 @@ vl_api_lcp_itf_pair_get_t_handler (vl_api_lcp_itf_pair_get_t *mp) ({ send_lcp_itf_pair_details (cursor, rp, mp->context); })); } +static void +vl_api_lcp_itf_pair_get_v2_t_handler (vl_api_lcp_itf_pair_get_v2_t *mp) +{ + vl_api_lcp_itf_pair_get_v2_reply_t *rmp; + i32 rv = 0; + + if (mp->sw_if_index == ~0) + { + REPLY_AND_DETAILS_MACRO_END ( + VL_API_LCP_ITF_PAIR_GET_REPLY, lcp_itf_pair_pool, + ({ send_lcp_itf_pair_details (cursor, rp, mp->context); })); + } + else + { + VALIDATE_SW_IF_INDEX_END (mp); + send_lcp_itf_pair_details ( + lcp_itf_pair_find_by_phy (mp->sw_if_index), + vl_api_client_index_to_registration (mp->client_index), mp->context); + + BAD_SW_IF_INDEX_LABEL; + REPLY_MACRO2_END (VL_API_LCP_ITF_PAIR_GET_V2_REPLY, + ({ rmp->cursor = ~0; })); + } +} + static void vl_api_lcp_default_ns_set_t_handler (vl_api_lcp_default_ns_set_t *mp) { From a541cfd31dca351a736362b4e06b626236f340dd Mon Sep 17 00:00:00 2001 From: Matthew Smith Date: Mon, 12 Feb 2024 18:39:21 +0000 Subject: [PATCH 009/271] ipsec: check each packet for no algs in esp-encrypt In esp_encrypt_inline(), if two or more consecutive packets are associated with the same SA which has no crypto or integrity algorithms set, only the first one gets dropped. Subsequent packets either get sent (synchronous crypto) or cause a segv (asynchronous crypto). The current SA's index and pool entry are cached before it can be determined whether the packet should be dropped due to no algorithms being set. The check for no algorithms is only performed when the cached SA index is different than the SA index for the current packet. So packets after the first one associated with the "none" alg SA aren't handled properly. This was broken by my previous commit ("ipsec: keep esp encrypt pointer and index synced") which fixed a segv that occurred under a different set of circumstances. Check whether each packet should be dropped instead of only checking when a new SA is encountered. Update unit tests: - Add a test for no algs on tunnel interface which enables asynchronous crypto. - Send more than one packet in the tests for no algs. Type: fix Fixes: dac9e566cd16fc375fff14280b37cb5135584fc6 Signed-off-by: Matthew Smith Change-Id: I69e951f22044051eb8557da187cb58f5535b54bf (cherry picked from commit ff71939c30ae81241808da1843e82cf2dfa92344) --- src/vnet/ipsec/esp_encrypt.c | 23 +++++++++++++---------- test/test_ipsec_tun_if_esp.py | 24 +++++++++++++++++++++++- 2 files changed, 36 insertions(+), 11 deletions(-) diff --git a/src/vnet/ipsec/esp_encrypt.c b/src/vnet/ipsec/esp_encrypt.c index 8a1ac7e1eb..46f7fe6cd0 100644 --- a/src/vnet/ipsec/esp_encrypt.c +++ b/src/vnet/ipsec/esp_encrypt.c @@ -607,6 +607,7 @@ esp_encrypt_inline (vlib_main_t *vm, vlib_node_runtime_t *node, u32 current_sa_bytes = 0, spi = 0; u8 esp_align = 4, iv_sz = 0, icv_sz = 0; ipsec_sa_t *sa0 = 0; + u8 sa_drop_no_crypto = 0; vlib_buffer_t *lb; vnet_crypto_op_t **crypto_ops = &ptd->crypto_ops; vnet_crypto_op_t **integ_ops = &ptd->integ_ops; @@ -692,16 +693,10 @@ esp_encrypt_inline (vlib_main_t *vm, vlib_node_runtime_t *node, sa0 = ipsec_sa_get (sa_index0); current_sa_index = sa_index0; - if (PREDICT_FALSE ((sa0->crypto_alg == IPSEC_CRYPTO_ALG_NONE && - sa0->integ_alg == IPSEC_INTEG_ALG_NONE) && - !ipsec_sa_is_set_NO_ALGO_NO_DROP (sa0))) - { - err = ESP_ENCRYPT_ERROR_NO_ENCRYPTION; - esp_encrypt_set_next_index (b[0], node, thread_index, err, - n_noop, noop_nexts, drop_next, - sa_index0); - goto trace; - } + sa_drop_no_crypto = ((sa0->crypto_alg == IPSEC_CRYPTO_ALG_NONE && + sa0->integ_alg == IPSEC_INTEG_ALG_NONE) && + !ipsec_sa_is_set_NO_ALGO_NO_DROP (sa0)); + vlib_prefetch_combined_counter (&ipsec_sa_counters, thread_index, current_sa_index); @@ -715,6 +710,14 @@ esp_encrypt_inline (vlib_main_t *vm, vlib_node_runtime_t *node, is_async = im->async_mode | ipsec_sa_is_set_IS_ASYNC (sa0); } + if (PREDICT_FALSE (sa_drop_no_crypto != 0)) + { + err = ESP_ENCRYPT_ERROR_NO_ENCRYPTION; + esp_encrypt_set_next_index (b[0], node, thread_index, err, n_noop, + noop_nexts, drop_next, sa_index0); + goto trace; + } + if (PREDICT_FALSE ((u16) ~0 == sa0->thread_index)) { /* this is the first packet to use this SA, claim the SA diff --git a/test/test_ipsec_tun_if_esp.py b/test/test_ipsec_tun_if_esp.py index 5131fbefe7..a7f91b9e96 100644 --- a/test/test_ipsec_tun_if_esp.py +++ b/test/test_ipsec_tun_if_esp.py @@ -1231,13 +1231,35 @@ def test_tun_44(self): self.config_sa_tra(p) self.config_protect(p) - tx = self.gen_pkts(self.pg1, src=self.pg1.remote_ip4, dst=p.remote_tun_if_host) + tx = self.gen_pkts( + self.pg1, src=self.pg1.remote_ip4, dst=p.remote_tun_if_host, count=127 + ) self.send_and_assert_no_replies(self.pg1, tx) self.unconfig_protect(p) self.unconfig_sa(p) self.unconfig_network(p) + def test_tun_44_async(self): + """IPSec SA with NULL algos using async crypto""" + p = self.ipv4_params + + self.vapi.ipsec_set_async_mode(async_enable=True) + self.config_network(p) + self.config_sa_tra(p) + self.config_protect(p) + + tx = self.gen_pkts( + self.pg1, src=self.pg1.remote_ip4, dst=p.remote_tun_if_host, count=127 + ) + self.send_and_assert_no_replies(self.pg1, tx) + + self.unconfig_protect(p) + self.unconfig_sa(p) + self.unconfig_network(p) + + self.vapi.ipsec_set_async_mode(async_enable=False) + @tag_fixme_vpp_workers class TestIpsec6MultiTunIfEsp(TemplateIpsec6TunProtect, TemplateIpsec, IpsecTun6): From 500ac0596126576e278e65a64597e8b87fdc55f8 Mon Sep 17 00:00:00 2001 From: Alexander Chernavin Date: Wed, 27 Dec 2023 12:37:16 +0000 Subject: [PATCH 010/271] flowprobe: fix flush callbacks when multiple workers IPFIX buffers are stored on a per worker thread basis. Currently, the flush callbacks will flush only buffers stored for the main thread. And buffers for worker threads will not be sent until their size reach the path MTU configured for the exporter. So if traffic is constant, the problem will unlikely to be visible. Buffers will be sent once they reach the maximum size. However, if traffic stops at some point and flush is triggered in order to make the plugin send all currently buffered data, this will not happen. And collectors will not receive that data. The plugin will keep the remaining data until traffic starts again, the buffers reach the maximum size, and be sent. With this fix, flush buffers for worker threads and for the main thread when the flush callbacks are triggered. This will allow to remove @tag_fixme_vpp_workers from the unit tests that don't set timers. The tests that set timers will still be failing for other multi-worker related problems. Type: fix Change-Id: I9a7d9cef8ddbec7ee68c79309e48e7bc0953d488 Signed-off-by: Alexander Chernavin (cherry picked from commit 4c7305f124cfa9c649ec6c9231eaf608fe336f1b) --- src/plugins/flowprobe/node.c | 86 ++++++++++++++++++++++++++++++++++++ test/test_flowprobe.py | 2 - 2 files changed, 86 insertions(+), 2 deletions(-) diff --git a/src/plugins/flowprobe/node.c b/src/plugins/flowprobe/node.c index 194664962e..cf03449e13 100644 --- a/src/plugins/flowprobe/node.c +++ b/src/plugins/flowprobe/node.c @@ -105,6 +105,9 @@ vlib_node_registration_t flowprobe_input_l2_node; vlib_node_registration_t flowprobe_output_ip4_node; vlib_node_registration_t flowprobe_output_ip6_node; vlib_node_registration_t flowprobe_output_l2_node; +vlib_node_registration_t flowprobe_flush_ip4_node; +vlib_node_registration_t flowprobe_flush_ip6_node; +vlib_node_registration_t flowprobe_flush_l2_node; /* No counters at the moment */ #define foreach_flowprobe_error \ @@ -945,18 +948,57 @@ flush_record (flowprobe_variant_t which) void flowprobe_flush_callback_ip4 (void) { + vlib_main_t *worker_vm; + u32 i; + + /* Flush for each worker thread */ + for (i = 1; i < vlib_get_n_threads (); i++) + { + worker_vm = vlib_get_main_by_index (i); + if (worker_vm) + vlib_node_set_interrupt_pending (worker_vm, + flowprobe_flush_ip4_node.index); + } + + /* Flush for the main thread */ flush_record (FLOW_VARIANT_IP4); } void flowprobe_flush_callback_ip6 (void) { + vlib_main_t *worker_vm; + u32 i; + + /* Flush for each worker thread */ + for (i = 1; i < vlib_get_n_threads (); i++) + { + worker_vm = vlib_get_main_by_index (i); + if (worker_vm) + vlib_node_set_interrupt_pending (worker_vm, + flowprobe_flush_ip6_node.index); + } + + /* Flush for the main thread */ flush_record (FLOW_VARIANT_IP6); } void flowprobe_flush_callback_l2 (void) { + vlib_main_t *worker_vm; + u32 i; + + /* Flush for each worker thread */ + for (i = 1; i < vlib_get_n_threads (); i++) + { + worker_vm = vlib_get_main_by_index (i); + if (worker_vm) + vlib_node_set_interrupt_pending (worker_vm, + flowprobe_flush_l2_node.index); + } + + /* Flush for the main thread */ flush_record (FLOW_VARIANT_L2); flush_record (FLOW_VARIANT_L2_IP4); flush_record (FLOW_VARIANT_L2_IP6); @@ -1062,6 +1104,32 @@ flowprobe_walker_process (vlib_main_t * vm, return 0; } +static uword +flowprobe_flush_ip4 (vlib_main_t *vm, vlib_node_runtime_t *rt, vlib_frame_t *f) +{ + flush_record (FLOW_VARIANT_IP4); + + return 0; +} + +static uword +flowprobe_flush_ip6 (vlib_main_t *vm, vlib_node_runtime_t *rt, vlib_frame_t *f) +{ + flush_record (FLOW_VARIANT_IP6); + + return 0; +} + +static uword +flowprobe_flush_l2 (vlib_main_t *vm, vlib_node_runtime_t *rt, vlib_frame_t *f) +{ + flush_record (FLOW_VARIANT_L2); + flush_record (FLOW_VARIANT_L2_IP4); + flush_record (FLOW_VARIANT_L2_IP6); + + return 0; +} + /* *INDENT-OFF* */ VLIB_REGISTER_NODE (flowprobe_input_ip4_node) = { .function = flowprobe_input_ip4_node_fn, @@ -1135,6 +1203,24 @@ VLIB_REGISTER_NODE (flowprobe_walker_node) = { .type = VLIB_NODE_TYPE_INPUT, .state = VLIB_NODE_STATE_INTERRUPT, }; +VLIB_REGISTER_NODE (flowprobe_flush_ip4_node) = { + .function = flowprobe_flush_ip4, + .name = "flowprobe-flush-ip4", + .type = VLIB_NODE_TYPE_INPUT, + .state = VLIB_NODE_STATE_INTERRUPT, +}; +VLIB_REGISTER_NODE (flowprobe_flush_ip6_node) = { + .function = flowprobe_flush_ip6, + .name = "flowprobe-flush-ip6", + .type = VLIB_NODE_TYPE_INPUT, + .state = VLIB_NODE_STATE_INTERRUPT, +}; +VLIB_REGISTER_NODE (flowprobe_flush_l2_node) = { + .function = flowprobe_flush_l2, + .name = "flowprobe-flush-l2", + .type = VLIB_NODE_TYPE_INPUT, + .state = VLIB_NODE_STATE_INTERRUPT, +}; /* *INDENT-ON* */ /* diff --git a/test/test_flowprobe.py b/test/test_flowprobe.py index 609099980f..ac0433abc0 100644 --- a/test/test_flowprobe.py +++ b/test/test_flowprobe.py @@ -1228,7 +1228,6 @@ def test_0002(self): self.logger.info("FFP_TEST_FINISH_0002") -@tag_fixme_vpp_workers class DatapathTx(MethodHolder, DatapathTestsHolder): """Collect info on Ethernet, IP4 and IP6 datapath (TX) (no timers)""" @@ -1309,7 +1308,6 @@ def test_rewritten_traffic(self): ipfix.remove_vpp_config() -@tag_fixme_vpp_workers class DatapathRx(MethodHolder, DatapathTestsHolder): """Collect info on Ethernet, IP4 and IP6 datapath (RX) (no timers)""" From dd59e1b43250fac37f31ad2e2e65ae25eefb85f8 Mon Sep 17 00:00:00 2001 From: Dave Wallace Date: Thu, 14 Mar 2024 21:41:00 -0400 Subject: [PATCH 011/271] misc: in crcchecker.py, don't check for uncommitted changes in CI Type: fix Change-Id: I63260a953e54518b3084b62fccdb4af81315b229 Signed-off-by: Dave Wallace (cherry picked from commit 3a0d7d2c95e8b8087c20b99fed5bcf62fac027d9) --- extras/scripts/crcchecker.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/extras/scripts/crcchecker.py b/extras/scripts/crcchecker.py index 01cb02523d..7dcdb681e1 100755 --- a/extras/scripts/crcchecker.py +++ b/extras/scripts/crcchecker.py @@ -82,13 +82,15 @@ def filelist_from_git_ls(): def is_uncommitted_changes(): """Returns true if there are uncommitted changes in the repo""" - git_status = "git status --porcelain -uno" - returncode = run(git_status.split(), stdout=PIPE, stderr=PIPE) - if returncode.returncode != 0: - sys.exit(returncode.returncode) - - if returncode.stdout: - return True + # Don't run this check in the Jenkins CI + if os.getenv("FDIOTOOLS_IMAGE") is None: + git_status = "git status --porcelain -uno" + returncode = run(git_status.split(), stdout=PIPE, stderr=PIPE) + if returncode.returncode != 0: + sys.exit(returncode.returncode) + + if returncode.stdout: + return True return False From 80e9503819a64f8360bb9b055660089ba67957f6 Mon Sep 17 00:00:00 2001 From: Vratko Polak Date: Thu, 11 Apr 2024 16:06:29 +0200 Subject: [PATCH 012/271] sr: use correct reply to sr_policy_add_v2 Type: fix Fixes: c4c205b091934d96a173f4c0d75ef7e888298ac7 Change-Id: I110729601a9f19451297883b781ec56e2b31465b Signed-off-by: Vratko Polak (cherry picked from commit 4332082093c267818899476d563c73298a491014) --- src/vnet/srv6/sr_api.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/vnet/srv6/sr_api.c b/src/vnet/srv6/sr_api.c index 5594fed44f..6a209dc7af 100644 --- a/src/vnet/srv6/sr_api.c +++ b/src/vnet/srv6/sr_api.c @@ -163,7 +163,7 @@ vl_api_sr_policy_add_v2_t_handler (vl_api_sr_policy_add_v2_t *mp) mp->type, ntohl (mp->fib_table), mp->is_encap, 0, NULL); vec_free (segments); - REPLY_MACRO (VL_API_SR_POLICY_ADD_REPLY); + REPLY_MACRO (VL_API_SR_POLICY_ADD_V2_REPLY); } static void From e3bd25946a5edb1d55e939638fb305d5bc900cf4 Mon Sep 17 00:00:00 2001 From: Abed Mohammad Kamaluddin Date: Mon, 20 May 2024 16:21:28 +0530 Subject: [PATCH 013/271] build: update git files for marvell repo Type: improvement Signed-off-by: Abed Mohammad Kamaluddin --- .github/workflows/close_prs.yml | 21 --------------------- .gitreview | 5 ----- 2 files changed, 26 deletions(-) delete mode 100644 .github/workflows/close_prs.yml diff --git a/.github/workflows/close_prs.yml b/.github/workflows/close_prs.yml deleted file mode 100644 index f8abc37f02..0000000000 --- a/.github/workflows/close_prs.yml +++ /dev/null @@ -1,21 +0,0 @@ ---- -name: Close Pull Request - -on: - pull_request_target: - types: [opened] - -jobs: - run: - runs-on: ubuntu-latest - steps: - - uses: superbrothers/close-pull-request@v3 - with: - # Optional. Post a issue comment just before closing a pull request. - comment: "Thank you so much for your interest! VPP takes patches at https://gerrit.fd.io/ - -``` -git clone https://gerrit.fd.io/r/vpp -``` - -Using [git review](https://www.mediawiki.org/wiki/Gerrit/git-review) to contribute patches is recommended" diff --git a/.gitreview b/.gitreview index fadff209a0..e69de29bb2 100644 --- a/.gitreview +++ b/.gitreview @@ -1,5 +0,0 @@ -[gerrit] -host=gerrit.fd.io -port=29418 -project=vpp -defaultbranch=stable/2402 From ddc9566b7438f29754b41af5916787f9fbfd68da Mon Sep 17 00:00:00 2001 From: Monendra Singh Kushwaha Date: Sun, 18 Feb 2024 23:33:41 +0530 Subject: [PATCH 014/271] build: bump octeon-roc version to 0.3 This patch updates cache line size in octeon-roc library. Type: fix Signed-off-by: Monendra Singh Kushwaha Change-Id: Ifbb6e7d2a5436a88ef10d22c414112edc23e0b35 Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/123217 Tested-by: sa_ip-sw-jenkins --- build/external/packages/octeon-roc.mk | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build/external/packages/octeon-roc.mk b/build/external/packages/octeon-roc.mk index 4bdeabeac2..62f5d823bd 100644 --- a/build/external/packages/octeon-roc.mk +++ b/build/external/packages/octeon-roc.mk @@ -2,9 +2,9 @@ # SPDX-License-Identifier: Apache-2.0 # https://spdx.org/licenses/Apache-2.0.html -octeon-roc_version := 0.2 +octeon-roc_version := 0.3 octeon-roc_tarball := octeon-roc-v$(octeon-roc_version).tar.gz -octeon-roc_tarball_md5sum := a72bb3b27fd3bbaf58aadd99514620e1 +octeon-roc_tarball_md5sum := e4a16beb76a6c63af1600dd4d1d752b8 octeon-roc_tarball_strip_dirs := 1 octeon-roc_url := https://github.com/MarvellEmbeddedProcessors/marvell-vpp/archive/refs/tags/$(octeon-roc_tarball) From d91675becb1a017518d5c2fd1101b60d258cefc1 Mon Sep 17 00:00:00 2001 From: Monendra Singh Kushwaha Date: Thu, 15 Feb 2024 23:07:39 +0530 Subject: [PATCH 015/271] dev: fix type for uint32 arg value Type: fix Change-Id: Ib09d6a0dfc95d82ecfd2ff123be9004cb038d0d4 Signed-off-by: Monendra Singh Kushwaha Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/126557 Tested-by: Shiva Shankar Kommula --- src/vnet/dev/args.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/vnet/dev/args.h b/src/vnet/dev/args.h index 0c49d1fcfe..a256cfe8e0 100644 --- a/src/vnet/dev/args.h +++ b/src/vnet/dev/args.h @@ -24,7 +24,7 @@ typedef enum typedef union { u8 boolean; - u8 uint32; + u32 uint32; u8 *string; } vnet_dev_arg_value_t; From 7b0c958f153b358c9c2e41f714d564cc425b174a Mon Sep 17 00:00:00 2001 From: Monendra Singh Kushwaha Date: Tue, 6 Feb 2024 14:02:43 +0530 Subject: [PATCH 016/271] dev: add per-port vnet flow Type: feature Change-Id: If63f39211288ab2eba8bc1ab50a2a4c7755abc66 Signed-off-by: Monendra Singh Kushwaha Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/126558 Tested-by: Shiva Shankar Kommula --- src/vnet/dev/dev.c | 1 + src/vnet/dev/dev.h | 14 +++++++++++++- src/vnet/dev/error.c | 25 ++++++++++++++++++++++++ src/vnet/dev/errors.h | 4 +++- src/vnet/dev/format.c | 15 +++++++++++++++ src/vnet/dev/handlers.c | 42 +++++++++++++++++++++++++++++++++++++++-- 6 files changed, 97 insertions(+), 4 deletions(-) diff --git a/src/vnet/dev/dev.c b/src/vnet/dev/dev.c index 0e04e9ab64..e04fa161ce 100644 --- a/src/vnet/dev/dev.c +++ b/src/vnet/dev/dev.c @@ -399,6 +399,7 @@ vnet_dev_main_init (vlib_main_t *vm) .mac_addr_change_function = vnet_dev_port_mac_change, .mac_addr_add_del_function = vnet_dev_add_del_mac_address, .flow_ops_function = vnet_dev_flow_ops_fn, + .format_flow = format_vnet_dev_flow, .set_rss_queues_function = vnet_dev_interface_set_rss_queues, }; driver->dev_class_index = vnet_register_device_class (vm, dev_class); diff --git a/src/vnet/dev/dev.h b/src/vnet/dev/dev.h index e7c6ca45fb..bbf2f9dff2 100644 --- a/src/vnet/dev/dev.h +++ b/src/vnet/dev/dev.h @@ -144,7 +144,11 @@ typedef struct _ (ADD_SECONDARY_HW_ADDR) \ _ (REMOVE_SECONDARY_HW_ADDR) \ _ (RXQ_INTR_MODE_ENABLE) \ - _ (RXQ_INTR_MODE_DISABLE) + _ (RXQ_INTR_MODE_DISABLE) \ + _ (ADD_RX_FLOW) \ + _ (DEL_RX_FLOW) \ + _ (GET_RX_FLOW_COUNTER) \ + _ (RESET_RX_FLOW_COUNTER) typedef enum { @@ -166,6 +170,11 @@ typedef struct vnet_dev_port_cfg_change_req vnet_dev_hw_addr_t addr; u16 max_rx_frame_size; vnet_dev_queue_id_t queue_id; + struct + { + u32 flow_index; + uword *private_data; + }; }; } vnet_dev_port_cfg_change_req_t; @@ -237,6 +246,7 @@ typedef struct vnet_dev_port_op_no_rv_t *deinit; vnet_dev_port_op_no_rv_t *free; format_function_t *format_status; + format_function_t *format_flow; } vnet_dev_port_ops_t; typedef union @@ -535,6 +545,7 @@ void *vnet_dev_get_device_info (vlib_main_t *, vnet_dev_device_id_t); /* error.c */ clib_error_t *vnet_dev_port_err (vlib_main_t *, vnet_dev_port_t *, vnet_dev_rv_t, char *, ...); +int vnet_dev_flow_err (vlib_main_t *, vnet_dev_rv_t); /* handlers.c */ clib_error_t *vnet_dev_port_set_max_frame_size (vnet_main_t *, @@ -654,6 +665,7 @@ format_function_t format_vnet_dev_port_tx_offloads; format_function_t format_vnet_dev_rv; format_function_t format_vnet_dev_rx_queue_info; format_function_t format_vnet_dev_tx_queue_info; +format_function_t format_vnet_dev_flow; unformat_function_t unformat_vnet_dev_flags; unformat_function_t unformat_vnet_dev_port_flags; diff --git a/src/vnet/dev/error.c b/src/vnet/dev/error.c index df9c6d364e..4e057010af 100644 --- a/src/vnet/dev/error.c +++ b/src/vnet/dev/error.c @@ -6,6 +6,7 @@ #include #include #include +#include clib_error_t * vnet_dev_port_err (vlib_main_t *vm, vnet_dev_port_t *port, vnet_dev_rv_t rv, @@ -27,3 +28,27 @@ vnet_dev_port_err (vlib_main_t *vm, vnet_dev_port_t *port, vnet_dev_rv_t rv, vec_free (s); return err; } + +int +vnet_dev_flow_err (vlib_main_t *vm, vnet_dev_rv_t rv) +{ + if (rv == VNET_DEV_OK) + return 0; + + switch (rv) + { + /* clang-format off */ +#define _(n, e, s) \ + case VNET_DEV_ERR_##e: \ + return VNET_FLOW_ERROR_##e; + foreach_flow_error; +#undef _ + /* clang-format on */ + default: + ASSERT (0); + } + + ASSERT (0); + + return 0; +} diff --git a/src/vnet/dev/errors.h b/src/vnet/dev/errors.h index 47e72957da..430a6aef28 100644 --- a/src/vnet/dev/errors.h +++ b/src/vnet/dev/errors.h @@ -39,6 +39,8 @@ _ (UNKNOWN_INTERFACE, "unknown interface") \ _ (UNSUPPORTED_CONFIG, "unsupported config") \ _ (UNSUPPORTED_DEVICE, "unsupported device") \ - _ (UNSUPPORTED_DEVICE_VER, "unsupported device version") + _ (UNSUPPORTED_DEVICE_VER, "unsupported device version") \ + _ (ALREADY_DONE, "already done") \ + _ (NO_SUCH_INTERFACE, "no such interface") #endif /* _VNET_DEV_ERRORS_H_ */ diff --git a/src/vnet/dev/format.c b/src/vnet/dev/format.c index 848cd13a6c..ed83a0eba9 100644 --- a/src/vnet/dev/format.c +++ b/src/vnet/dev/format.c @@ -490,3 +490,18 @@ format_vnet_dev_port_tx_offloads (u8 *s, va_list *args) return s; } + +u8 * +format_vnet_dev_flow (u8 *s, va_list *args) +{ + u32 dev_instance = va_arg (*args, u32); + u32 flow_index = va_arg (*args, u32); + uword private_data = va_arg (*args, uword); + vnet_dev_port_t *port = vnet_dev_get_port_from_dev_instance (dev_instance); + + if (port->port_ops.format_flow) + s = format (s, "%U", port->port_ops.format_flow, port, flow_index, + private_data); + + return s; +} diff --git a/src/vnet/dev/handlers.c b/src/vnet/dev/handlers.c index fcaef14221..2a55affe3e 100644 --- a/src/vnet/dev/handlers.c +++ b/src/vnet/dev/handlers.c @@ -146,9 +146,47 @@ int vnet_dev_flow_ops_fn (vnet_main_t *vnm, vnet_flow_dev_op_t op, u32 dev_instance, u32 flow_index, uword *private_data) { + vlib_main_t *vm = vlib_get_main (); vnet_dev_port_t *p = vnet_dev_get_port_from_dev_instance (dev_instance); - log_warn (p->dev, "unsupported request for flow_ops received"); - return VNET_FLOW_ERROR_NOT_SUPPORTED; + vnet_dev_port_cfg_change_req_t req; + vnet_dev_rv_t rv; + + switch (op) + { + case VNET_FLOW_DEV_OP_ADD_FLOW: + req.type = VNET_DEV_PORT_CFG_ADD_RX_FLOW; + break; + case VNET_FLOW_DEV_OP_DEL_FLOW: + req.type = VNET_DEV_PORT_CFG_DEL_RX_FLOW; + break; + case VNET_FLOW_DEV_OP_GET_COUNTER: + req.type = VNET_DEV_PORT_CFG_GET_RX_FLOW_COUNTER; + break; + case VNET_FLOW_DEV_OP_RESET_COUNTER: + req.type = VNET_DEV_PORT_CFG_RESET_RX_FLOW_COUNTER; + break; + default: + log_warn (p->dev, "unsupported request for flow_ops received"); + return VNET_FLOW_ERROR_NOT_SUPPORTED; + } + + req.flow_index = flow_index; + req.private_data = private_data; + + rv = vnet_dev_port_cfg_change_req_validate (vm, p, &req); + if (rv != VNET_DEV_OK) + { + log_err (p->dev, "validation failed for flow_ops"); + return VNET_FLOW_ERROR_NOT_SUPPORTED; + } + + if ((rv = vnet_dev_process_port_cfg_change_req (vm, p, &req)) != VNET_DEV_OK) + { + log_err (p->dev, "request for flow_ops failed"); + return vnet_dev_flow_err (vm, rv); + } + + return 0; } clib_error_t * From 0fdf459cea8239fc7fd39b45b4f93e88df5af249 Mon Sep 17 00:00:00 2001 From: Monendra Singh Kushwaha Date: Wed, 14 Feb 2024 16:18:24 +0530 Subject: [PATCH 017/271] octeon: add flow offload infra Type: feature Change-Id: I3485e1627eafc5125255985003573247e7562db2 Signed-off-by: Kommula Shiva Shankar Signed-off-by: Monendra Singh Kushwaha Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/126559 --- src/plugins/dev_octeon/CMakeLists.txt | 1 + src/plugins/dev_octeon/flow.c | 505 ++++++++++++++++++++++++++ src/plugins/dev_octeon/format.c | 19 + src/plugins/dev_octeon/init.c | 2 + src/plugins/dev_octeon/octeon.h | 26 ++ src/plugins/dev_octeon/port.c | 26 +- src/plugins/dev_octeon/rx_node.c | 5 + 7 files changed, 581 insertions(+), 3 deletions(-) create mode 100644 src/plugins/dev_octeon/flow.c diff --git a/src/plugins/dev_octeon/CMakeLists.txt b/src/plugins/dev_octeon/CMakeLists.txt index b7c25fe040..e8abf1a338 100644 --- a/src/plugins/dev_octeon/CMakeLists.txt +++ b/src/plugins/dev_octeon/CMakeLists.txt @@ -30,6 +30,7 @@ add_vpp_plugin(dev_octeon roc_helper.c rx_node.c tx_node.c + flow.c MULTIARCH_SOURCES rx_node.c diff --git a/src/plugins/dev_octeon/flow.c b/src/plugins/dev_octeon/flow.c new file mode 100644 index 0000000000..1c367a036a --- /dev/null +++ b/src/plugins/dev_octeon/flow.c @@ -0,0 +1,505 @@ +/* + * Copyright (c) 2024 Marvell. + * SPDX-License-Identifier: Apache-2.0 + * https://spdx.org/licenses/Apache-2.0.html + */ + +#include +#include + +VLIB_REGISTER_LOG_CLASS (oct_log, static) = { + .class_name = "octeon", + .subclass_name = "flow", +}; + +#define FLOW_IS_ETHERNET_CLASS(f) (f->type == VNET_FLOW_TYPE_ETHERNET) + +#define FLOW_IS_IPV4_CLASS(f) \ + ((f->type == VNET_FLOW_TYPE_IP4) || \ + (f->type == VNET_FLOW_TYPE_IP4_N_TUPLE) || \ + (f->type == VNET_FLOW_TYPE_IP4_N_TUPLE_TAGGED) || \ + (f->type == VNET_FLOW_TYPE_IP4_VXLAN) || \ + (f->type == VNET_FLOW_TYPE_IP4_GTPC) || \ + (f->type == VNET_FLOW_TYPE_IP4_GTPU) || \ + (f->type == VNET_FLOW_TYPE_IP4_L2TPV3OIP) || \ + (f->type == VNET_FLOW_TYPE_IP4_IPSEC_ESP) || \ + (f->type == VNET_FLOW_TYPE_IP4_IPSEC_AH)) + +#define FLOW_IS_IPV6_CLASS(f) \ + ((f->type == VNET_FLOW_TYPE_IP6) || \ + (f->type == VNET_FLOW_TYPE_IP6_N_TUPLE) || \ + (f->type == VNET_FLOW_TYPE_IP6_N_TUPLE_TAGGED) || \ + (f->type == VNET_FLOW_TYPE_IP6_VXLAN)) + +#define FLOW_IS_L3_TYPE(f) \ + ((f->type == VNET_FLOW_TYPE_IP4) || (f->type == VNET_FLOW_TYPE_IP6)) + +#define FLOW_IS_L4_TYPE(f) \ + ((f->type == VNET_FLOW_TYPE_IP4_N_TUPLE) || \ + (f->type == VNET_FLOW_TYPE_IP6_N_TUPLE) || \ + (f->type == VNET_FLOW_TYPE_IP4_N_TUPLE_TAGGED) || \ + (f->type == VNET_FLOW_TYPE_IP6_N_TUPLE_TAGGED)) + +#define FLOW_IS_L4_TUNNEL_TYPE(f) \ + ((f->type == VNET_FLOW_TYPE_IP4_VXLAN) || \ + (f->type == VNET_FLOW_TYPE_IP6_VXLAN) || \ + (f->type == VNET_FLOW_TYPE_IP4_GTPC) || \ + (f->type == VNET_FLOW_TYPE_IP4_GTPU)) + +#define OCT_FLOW_UNSUPPORTED_ACTIONS(f) \ + ((f->actions == VNET_FLOW_ACTION_BUFFER_ADVANCE) || \ + (f->actions == VNET_FLOW_ACTION_REDIRECT_TO_NODE)) + +/* Keep values in sync with vnet/flow.h */ +#define foreach_oct_flow_rss_types \ + _ (1, FLOW_KEY_TYPE_IPV4 | FLOW_KEY_TYPE_TCP, "ipv4-tcp") \ + _ (2, FLOW_KEY_TYPE_IPV4 | FLOW_KEY_TYPE_UDP, "ipv4-udp") \ + _ (3, FLOW_KEY_TYPE_IPV4 | FLOW_KEY_TYPE_SCTP, "ipv4-sctp") \ + _ (5, FLOW_KEY_TYPE_IPV4, "ipv4") \ + _ (9, FLOW_KEY_TYPE_IPV6 | FLOW_KEY_TYPE_TCP, "ipv6-tcp") \ + _ (10, FLOW_KEY_TYPE_IPV6 | FLOW_KEY_TYPE_UDP, "ipv6-udp") \ + _ (11, FLOW_KEY_TYPE_IPV6 | FLOW_KEY_TYPE_SCTP, "ipv6-sctp") \ + _ (13, FLOW_KEY_TYPE_IPV6_EXT, "ipv6-ex") \ + _ (14, FLOW_KEY_TYPE_IPV6, "ipv6") \ + _ (16, FLOW_KEY_TYPE_PORT, "port") \ + _ (17, FLOW_KEY_TYPE_VXLAN, "vxlan") \ + _ (18, FLOW_KEY_TYPE_GENEVE, "geneve") \ + _ (19, FLOW_KEY_TYPE_NVGRE, "nvgre") \ + _ (20, FLOW_KEY_TYPE_GTPU, "gtpu") \ + _ (60, FLOW_KEY_TYPE_L4_DST, "l4-dst-only") \ + _ (61, FLOW_KEY_TYPE_L4_SRC, "l4-src-only") \ + _ (62, FLOW_KEY_TYPE_L3_DST, "l3-dst-only") \ + _ (63, FLOW_KEY_TYPE_L3_SRC, "l3-src-only") + +typedef struct +{ + u16 src_port; + u16 dst_port; + u32 verification_tag; + u32 cksum; +} sctp_header_t; + +typedef struct +{ + u8 ver_flags; + u8 type; + u16 length; + u32 teid; +} gtpu_header_t; + +static void +oct_flow_convert_rss_types (u64 *key, u64 rss_types) +{ +#define _(a, b, c) \ + if (rss_types & (1UL << a)) \ + *key |= b; + + foreach_oct_flow_rss_types +#undef _ + + return; +} + +vnet_dev_rv_t +oct_flow_validate_params (vlib_main_t *vm, vnet_dev_port_t *port, + vnet_dev_port_cfg_type_t type, u32 flow_index, + uword *priv_data) +{ + vnet_flow_t *flow = vnet_get_flow (flow_index); + u32 last_queue; + u32 qid; + + if (type == VNET_DEV_PORT_CFG_GET_RX_FLOW_COUNTER || + type == VNET_DEV_PORT_CFG_RESET_RX_FLOW_COUNTER) + { + log_err (port->dev, "Unsupported request type"); + return VNET_DEV_ERR_NOT_SUPPORTED; + } + + if (OCT_FLOW_UNSUPPORTED_ACTIONS (flow)) + { + log_err (port->dev, "Unsupported flow action"); + return VNET_DEV_ERR_NOT_SUPPORTED; + } + + if (flow->actions & VNET_FLOW_ACTION_REDIRECT_TO_QUEUE) + { + qid = flow->redirect_queue; + if (qid > port->intf.num_rx_queues - 1 || qid < 0) + { + log_err (port->dev, + "Given Q(%d) is invalid, supported range is %d-%d", qid, 0, + port->intf.num_rx_queues - 1); + return VNET_DEV_ERR_NOT_SUPPORTED; + } + } + + if (flow->actions & VNET_FLOW_ACTION_RSS) + { + last_queue = flow->queue_index + flow->queue_num; + if (last_queue > port->intf.num_rx_queues - 1) + { + log_err (port->dev, + "Given Q range(%d-%d) is invalid, supported range is %d-%d", + flow->queue_index, flow->queue_index + flow->queue_num, 0, + port->intf.num_rx_queues - 1); + return VNET_DEV_ERR_NOT_SUPPORTED; + } + } + return VNET_DEV_OK; +} + +static vnet_dev_rv_t +oct_flow_rule_create (vnet_dev_port_t *port, struct roc_npc_action *actions, + struct roc_npc_item_info *item_info, vnet_flow_t *flow, + uword *private_data) +{ + oct_port_t *oct_port = vnet_dev_get_port_data (port); + struct roc_npc_attr attr = { .priority = 1, .ingress = 1 }; + struct roc_npc_flow *npc_flow; + oct_flow_entry_t *flow_entry; + struct roc_npc *npc; + int rv = 0; + + npc = &oct_port->npc; + + npc_flow = + roc_npc_flow_create (npc, &attr, item_info, actions, npc->pf_func, &rv); + if (rv) + { + log_err (port->dev, "roc_npc_flow_create failed with '%s' error", + roc_error_msg_get (rv)); + return VNET_DEV_ERR_NOT_SUPPORTED; + } + roc_npc_mcam_clear_counter (npc, npc_flow->ctr_id); + + pool_get_zero (oct_port->flow_entries, flow_entry); + flow_entry->index = flow_entry - oct_port->flow_entries; + flow_entry->vnet_flow_index = flow->index; + flow_entry->npc_flow = npc_flow; + + *private_data = flow_entry->index; + + return VNET_DEV_OK; +} + +static vnet_dev_rv_t +oct_flow_add (vlib_main_t *vm, vnet_dev_port_t *port, vnet_flow_t *flow, + uword *private_data) +{ + struct roc_npc_item_info item_info[ROC_NPC_ITEM_TYPE_END] = {}; + struct roc_npc_action actions[ROC_NPC_ITEM_TYPE_END] = {}; + oct_port_t *oct_port = vnet_dev_get_port_data (port); + u16 l4_src_port = 0, l4_dst_port = 0; + u16 l4_src_mask = 0, l4_dst_mask = 0; + struct roc_npc_action_rss rss_conf = {}; + struct roc_npc_action_queue conf = {}; + struct roc_npc_action_mark mark = {}; + struct roc_npc *npc = &oct_port->npc; + vnet_dev_rv_t rv = VNET_DEV_OK; + int layer = 0, index = 0; + u16 *queues = NULL; + u64 flow_key = 0; + u8 proto = 0; + u16 action = 0; + + if (FLOW_IS_ETHERNET_CLASS (flow)) + { + ethernet_header_t eth_spec = { .type = clib_host_to_net_u16 ( + flow->ethernet.eth_hdr.type) }, + eth_mask = { .type = 0xFFFF }; + + item_info[layer].spec = (void *) ð_spec; + item_info[layer].mask = (void *) ð_mask; + item_info[layer].size = sizeof (ethernet_header_t); + item_info[layer].type = ROC_NPC_ITEM_TYPE_ETH; + layer++; + } + + else if (FLOW_IS_IPV4_CLASS (flow)) + { + vnet_flow_ip4_t *ip4_hdr = &flow->ip4; + proto = ip4_hdr->protocol.prot; + ip4_header_t ip4_spec = { .src_address = ip4_hdr->src_addr.addr, + .dst_address = ip4_hdr->dst_addr.addr }, + ip4_mask = { .src_address = ip4_hdr->src_addr.mask, + .dst_address = ip4_hdr->dst_addr.mask }; + + item_info[layer].spec = (void *) &ip4_spec; + item_info[layer].mask = (void *) &ip4_mask; + item_info[layer].size = sizeof (ip4_header_t); + item_info[layer].type = ROC_NPC_ITEM_TYPE_IPV4; + layer++; + + if (FLOW_IS_L4_TYPE (flow)) + { + vnet_flow_ip4_n_tuple_t *ip4_tuple_hdr = &flow->ip4_n_tuple; + + l4_src_port = clib_host_to_net_u16 (ip4_tuple_hdr->src_port.port); + l4_dst_port = clib_host_to_net_u16 (ip4_tuple_hdr->dst_port.port); + l4_src_mask = clib_host_to_net_u16 (ip4_tuple_hdr->src_port.mask); + l4_dst_mask = clib_host_to_net_u16 (ip4_tuple_hdr->dst_port.mask); + } + } + else if (FLOW_IS_IPV6_CLASS (flow)) + { + vnet_flow_ip6_t *ip6_hdr = &flow->ip6; + proto = ip6_hdr->protocol.prot; + ip6_header_t ip6_spec = { .src_address = ip6_hdr->src_addr.addr, + .dst_address = ip6_hdr->dst_addr.addr }, + ip6_mask = { .src_address = ip6_hdr->src_addr.mask, + .dst_address = ip6_hdr->dst_addr.mask }; + + item_info[layer].spec = (void *) &ip6_spec; + item_info[layer].mask = (void *) &ip6_mask; + item_info[layer].size = sizeof (ip6_header_t); + item_info[layer].type = ROC_NPC_ITEM_TYPE_IPV6; + layer++; + + if (FLOW_IS_L4_TYPE (flow)) + { + vnet_flow_ip6_n_tuple_t *ip6_tuple_hdr = &flow->ip6_n_tuple; + + l4_src_port = clib_host_to_net_u16 (ip6_tuple_hdr->src_port.port); + l4_dst_port = clib_host_to_net_u16 (ip6_tuple_hdr->dst_port.port); + l4_src_mask = clib_host_to_net_u16 (ip6_tuple_hdr->src_port.mask); + l4_dst_mask = clib_host_to_net_u16 (ip6_tuple_hdr->dst_port.mask); + } + } + + if (!proto) + goto end_item_info; + + switch (proto) + { + case IP_PROTOCOL_UDP: + item_info[layer].type = ROC_NPC_ITEM_TYPE_UDP; + + udp_header_t udp_spec = { .src_port = l4_src_port, + .dst_port = l4_dst_port }, + udp_mask = { .src_port = l4_src_mask, + .dst_port = l4_dst_mask }; + + item_info[layer].spec = (void *) &udp_spec; + item_info[layer].mask = (void *) &udp_mask; + item_info[layer].size = sizeof (udp_header_t); + layer++; + + if (FLOW_IS_L4_TUNNEL_TYPE (flow)) + { + switch (flow->type) + { + case VNET_FLOW_TYPE_IP4_GTPU: + item_info[layer].type = ROC_NPC_ITEM_TYPE_GTPU; + gtpu_header_t gtpu_spec = { .teid = clib_host_to_net_u32 ( + flow->ip4_gtpu.teid) }, + gtpu_mask = { .teid = 0XFFFFFFFF }; + + item_info[layer].spec = (void *) >pu_spec; + item_info[layer].mask = (void *) >pu_mask; + item_info[layer].size = sizeof (gtpu_header_t); + layer++; + break; + + default: + log_err (port->dev, "Unsupported L4 tunnel type"); + return VNET_DEV_ERR_NOT_SUPPORTED; + } + } /* FLOW_IS_L4_TUNNEL_TYPE */ + break; + + case IP_PROTOCOL_TCP: + item_info[layer].type = ROC_NPC_ITEM_TYPE_TCP; + + tcp_header_t tcp_spec = { .src_port = l4_src_port, + .dst_port = l4_dst_port }, + tcp_mask = { .src_port = l4_src_mask, + .dst_port = l4_dst_mask }; + + item_info[layer].spec = (void *) &tcp_spec; + item_info[layer].mask = (void *) &tcp_mask; + item_info[layer].size = sizeof (tcp_header_t); + layer++; + break; + + case IP_PROTOCOL_SCTP: + item_info[layer].type = ROC_NPC_ITEM_TYPE_SCTP; + + sctp_header_t sctp_spec = { .src_port = l4_src_port, + .dst_port = l4_dst_port }, + sctp_mask = { .src_port = l4_src_mask, + .dst_port = l4_dst_mask }; + + item_info[layer].spec = (void *) &sctp_spec; + item_info[layer].mask = (void *) &sctp_mask; + item_info[layer].size = sizeof (sctp_header_t); + layer++; + break; + + case IP_PROTOCOL_IPSEC_ESP: + item_info[layer].type = ROC_NPC_ITEM_TYPE_ESP; + esp_header_t esp_spec = { .spi = clib_host_to_net_u32 ( + flow->ip4_ipsec_esp.spi) }, + esp_mask = { .spi = 0xFFFFFFFF }; + + item_info[layer].spec = (void *) &esp_spec; + item_info[layer].mask = (void *) &esp_mask; + item_info[layer].size = sizeof (u32); + layer++; + break; + + default: + log_err (port->dev, "Unsupported IP protocol '%U'", format_ip_protocol, + proto); + return VNET_DEV_ERR_NOT_SUPPORTED; + } + +end_item_info: + item_info[layer].type = ROC_NPC_ITEM_TYPE_END; + + if (flow->actions & VNET_FLOW_ACTION_REDIRECT_TO_QUEUE) + { + conf.index = flow->redirect_queue; + actions[action].type = ROC_NPC_ACTION_TYPE_QUEUE; + actions[action].conf = &conf; + action++; + } + + else if (flow->actions & VNET_FLOW_ACTION_DROP) + { + actions[action].type = ROC_NPC_ACTION_TYPE_DROP; + action++; + } + + else if (flow->actions & VNET_FLOW_ACTION_RSS) + { + if (!flow->queue_num) + { + log_err (port->dev, "RSS action has no queues"); + return VNET_DEV_ERR_NOT_SUPPORTED; + } + queues = clib_mem_alloc (sizeof (u16) * port->intf.num_rx_queues); + + for (index = 0; index < flow->queue_num; index++) + queues[index] = flow->queue_index++; + + oct_flow_convert_rss_types (&flow_key, flow->rss_types); + if (!flow_key) + { + log_err (port->dev, "Invalid RSS hash function"); + return VNET_DEV_ERR_NOT_SUPPORTED; + } + npc->flowkey_cfg_state = flow_key; + rss_conf.queue_num = flow->queue_num; + rss_conf.queue = queues; + + actions[action].type = ROC_NPC_ACTION_TYPE_RSS; + actions[action].conf = &rss_conf; + action++; + } + + if (flow->actions & VNET_FLOW_ACTION_MARK) + { + if (flow->mark_flow_id == 0 || + flow->mark_flow_id > (NPC_FLOW_FLAG_VAL - 2)) + { + log_err (port->dev, "mark flow id must be > 0 and < 0xfffe"); + return VNET_DEV_ERR_NOT_SUPPORTED; + } + /* RoC library adds 1 to id, so subtract 1 */ + mark.id = flow->mark_flow_id - 1; + actions[action].type = ROC_NPC_ACTION_TYPE_MARK; + actions[action].conf = &mark; + action++; + } + + /* make count as default action */ + actions[action].type = ROC_NPC_ACTION_TYPE_COUNT; + actions[action + 1].type = ROC_NPC_ACTION_TYPE_END; + + rv = oct_flow_rule_create (port, actions, item_info, flow, private_data); + + if (queues) + clib_mem_free (queues); + + return rv; +} + +static vnet_dev_rv_t +oct_flow_del (vlib_main_t *vm, vnet_dev_port_t *port, vnet_flow_t *flow, + uword *private_data) +{ + oct_port_t *oct_port = vnet_dev_get_port_data (port); + struct roc_npc *npc = &oct_port->npc; + struct roc_npc_flow *npc_flow; + oct_flow_entry_t *flow_entry; + int rv = 0, index; + + index = *private_data; + flow_entry = pool_elt_at_index (oct_port->flow_entries, index); + npc_flow = flow_entry->npc_flow; + rv = roc_npc_flow_destroy (npc, npc_flow); + if (rv) + { + log_err (port->dev, "roc_npc_flow_destroy failed with '%s' error", + roc_error_msg_get (rv)); + return VNET_DEV_ERR_NOT_SUPPORTED; + } + pool_put (oct_port->flow_entries, flow_entry); + + return VNET_DEV_OK; +} + +vnet_dev_rv_t +oct_flow_query (vlib_main_t *vm, vnet_dev_port_t *port, u32 flow_index, + uword private_data, u64 *hits) +{ + oct_port_t *oct_port = vnet_dev_get_port_data (port); + struct roc_npc *npc = &oct_port->npc; + struct roc_npc_flow *npc_flow; + oct_flow_entry_t *flow_entry; + i32 flow_count; + int rv = 0; + + flow_count = pool_elts (oct_port->flow_entries); + if (!flow_count) + { + log_err (port->dev, "Flow entry pool is empty"); + return VNET_DEV_ERR_NOT_SUPPORTED; + } + + flow_entry = pool_elt_at_index (oct_port->flow_entries, private_data); + npc_flow = flow_entry->npc_flow; + if (npc_flow->ctr_id == NPC_COUNTER_NONE) + { + log_err (port->dev, "Counters are not available for given flow id (%u)", + flow_index); + return VNET_DEV_ERR_NOT_SUPPORTED; + } + + rv = roc_npc_mcam_read_counter (npc, npc_flow->ctr_id, hits); + if (rv != 0) + { + log_err (port->dev, "Error reading flow counter for given flow id (%u)", + flow_index); + return VNET_DEV_ERR_INTERNAL; + } + + return VNET_DEV_OK; +} + +vnet_dev_rv_t +oct_flow_ops_fn (vlib_main_t *vm, vnet_dev_port_t *port, + vnet_dev_port_cfg_type_t type, u32 flow_index, + uword *priv_data) +{ + vnet_flow_t *flow = vnet_get_flow (flow_index); + + if (type == VNET_DEV_PORT_CFG_ADD_RX_FLOW) + return oct_flow_add (vm, port, flow, priv_data); + + if (type == VNET_DEV_PORT_CFG_DEL_RX_FLOW) + return oct_flow_del (vm, port, flow, priv_data); + + return VNET_DEV_ERR_NOT_SUPPORTED; +} diff --git a/src/plugins/dev_octeon/format.c b/src/plugins/dev_octeon/format.c index 5ee956ad4f..e624b84f54 100644 --- a/src/plugins/dev_octeon/format.c +++ b/src/plugins/dev_octeon/format.c @@ -162,3 +162,22 @@ format_oct_tx_trace (u8 *s, va_list *args) return s; } + +u8 * +format_oct_port_flow (u8 *s, va_list *args) +{ + vlib_main_t *vm = vlib_get_main (); + vnet_dev_port_t *port = va_arg (*args, vnet_dev_port_t *); + u32 flow_index = va_arg (*args, u32); + uword private_data = va_arg (*args, uword); + u64 hits; + + if (flow_index == ~0) + return s; + + if (oct_flow_query (vm, port, flow_index, private_data, &hits) == + VNET_DEV_OK) + s = format (s, "flow (%u) hit count: %lu", flow_index, hits); + + return s; +} diff --git a/src/plugins/dev_octeon/init.c b/src/plugins/dev_octeon/init.c index bee449f212..d3b6814e50 100644 --- a/src/plugins/dev_octeon/init.c +++ b/src/plugins/dev_octeon/init.c @@ -135,7 +135,9 @@ oct_init_nix (vlib_main_t *vm, vnet_dev_t *dev) .start = oct_port_start, .stop = oct_port_stop, .config_change = oct_port_cfg_change, + .config_change_validate = oct_port_cfg_change_validate, .format_status = format_oct_port_status, + .format_flow = format_oct_port_flow, }, .data_size = sizeof (oct_port_t), .initial_data = &oct_port, diff --git a/src/plugins/dev_octeon/octeon.h b/src/plugins/dev_octeon/octeon.h index a87330c6f9..39501d0410 100644 --- a/src/plugins/dev_octeon/octeon.h +++ b/src/plugins/dev_octeon/octeon.h @@ -9,6 +9,9 @@ #include #include #include +#include +#include +#include #include #include @@ -31,12 +34,23 @@ typedef struct struct roc_nix *nix; } oct_device_t; +typedef struct +{ + /* vnet flow index */ + u32 vnet_flow_index; + + u32 index; + /* Internal flow object */ + struct roc_npc_flow *npc_flow; +} oct_flow_entry_t; + typedef struct { u8 lf_allocated : 1; u8 tm_initialized : 1; u8 npc_initialized : 1; struct roc_npc npc; + oct_flow_entry_t *flow_entries; } oct_port_t; typedef struct @@ -87,6 +101,7 @@ typedef struct format_function_t format_oct_port_status; format_function_t format_oct_rx_trace; format_function_t format_oct_tx_trace; +format_function_t format_oct_port_flow; /* port.c */ vnet_dev_rv_t oct_port_init (vlib_main_t *, vnet_dev_port_t *); @@ -95,6 +110,8 @@ void oct_port_stop (vlib_main_t *, vnet_dev_port_t *); void oct_port_deinit (vlib_main_t *, vnet_dev_port_t *); vnet_dev_rv_t oct_port_cfg_change (vlib_main_t *, vnet_dev_port_t *, vnet_dev_port_cfg_change_req_t *); +vnet_dev_rv_t oct_port_cfg_change_validate (vlib_main_t *, vnet_dev_port_t *, + vnet_dev_port_cfg_change_req_t *); /* queue.c */ vnet_dev_rv_t oct_rx_queue_alloc (vlib_main_t *, vnet_dev_rx_queue_t *); @@ -108,6 +125,15 @@ void oct_txq_deinit (vlib_main_t *, vnet_dev_tx_queue_t *); format_function_t format_oct_rxq_info; format_function_t format_oct_txq_info; +/* flow.c */ +vnet_dev_rv_t oct_flow_ops_fn (vlib_main_t *, vnet_dev_port_t *, + vnet_dev_port_cfg_type_t, u32, uword *); +vnet_dev_rv_t oct_flow_validate_params (vlib_main_t *, vnet_dev_port_t *, + vnet_dev_port_cfg_type_t, u32, + uword *); +vnet_dev_rv_t oct_flow_query (vlib_main_t *, vnet_dev_port_t *, u32, uword, + u64 *); + #define log_debug(dev, f, ...) \ vlib_log (VLIB_LOG_LEVEL_DEBUG, oct_log.class, "%U: " f, \ format_vnet_dev_addr, (dev), ##__VA_ARGS__) diff --git a/src/plugins/dev_octeon/port.c b/src/plugins/dev_octeon/port.c index 5857bc15f7..f1ae15aadb 100644 --- a/src/plugins/dev_octeon/port.c +++ b/src/plugins/dev_octeon/port.c @@ -10,6 +10,9 @@ #include #include +#define OCT_FLOW_PREALLOC_SIZE 1 +#define OCT_FLOW_MAX_PRIORITY 7 + VLIB_REGISTER_LOG_CLASS (oct_log, static) = { .class_name = "octeon", .subclass_name = "port", @@ -95,6 +98,8 @@ oct_port_init (vlib_main_t *vm, vnet_dev_port_t *port) roc_nix_rss_key_set (nix, default_rss_key); cp->npc.roc_nix = nix; + cp->npc.flow_prealloc_size = OCT_FLOW_PREALLOC_SIZE; + cp->npc.flow_max_priority = OCT_FLOW_MAX_PRIORITY; if ((rrv = roc_npc_init (&cp->npc))) { oct_port_deinit (vm, port); @@ -360,7 +365,7 @@ oct_port_stop (vlib_main_t *vm, vnet_dev_port_t *port) } vnet_dev_rv_t -oct_port_cfg_change_precheck (vlib_main_t *vm, vnet_dev_port_t *port, +oct_port_cfg_change_validate (vlib_main_t *vm, vnet_dev_port_t *port, vnet_dev_port_cfg_change_req_t *req) { vnet_dev_rv_t rv = VNET_DEV_OK; @@ -378,6 +383,14 @@ oct_port_cfg_change_precheck (vlib_main_t *vm, vnet_dev_port_t *port, case VNET_DEV_PORT_CFG_REMOVE_SECONDARY_HW_ADDR: break; + case VNET_DEV_PORT_CFG_ADD_RX_FLOW: + case VNET_DEV_PORT_CFG_DEL_RX_FLOW: + case VNET_DEV_PORT_CFG_GET_RX_FLOW_COUNTER: + case VNET_DEV_PORT_CFG_RESET_RX_FLOW_COUNTER: + rv = oct_flow_validate_params (vm, port, req->type, req->flow_index, + req->private_data); + break; + default: rv = VNET_DEV_ERR_NOT_SUPPORTED; }; @@ -394,8 +407,6 @@ oct_port_cfg_change (vlib_main_t *vm, vnet_dev_port_t *port, switch (req->type) { case VNET_DEV_PORT_CFG_PROMISC_MODE: - { - } break; case VNET_DEV_PORT_CFG_CHANGE_PRIMARY_HW_ADDR: @@ -410,6 +421,15 @@ oct_port_cfg_change (vlib_main_t *vm, vnet_dev_port_t *port, case VNET_DEV_PORT_CFG_MAX_RX_FRAME_SIZE: break; + case VNET_DEV_PORT_CFG_ADD_RX_FLOW: + case VNET_DEV_PORT_CFG_DEL_RX_FLOW: + case VNET_DEV_PORT_CFG_GET_RX_FLOW_COUNTER: + case VNET_DEV_PORT_CFG_RESET_RX_FLOW_COUNTER: + rv = oct_flow_ops_fn (vm, port, req->type, req->flow_index, + req->private_data); + + break; + default: return VNET_DEV_ERR_NOT_SUPPORTED; }; diff --git a/src/plugins/dev_octeon/rx_node.c b/src/plugins/dev_octeon/rx_node.c index c1c4771795..5f7e5a8469 100644 --- a/src/plugins/dev_octeon/rx_node.c +++ b/src/plugins/dev_octeon/rx_node.c @@ -131,6 +131,10 @@ oct_rx_batch (vlib_main_t *vm, oct_rx_node_ctx_t *ctx, ctx->n_rx_bytes += b[1]->current_length = d[1].sg0.seg1_size; ctx->n_rx_bytes += b[2]->current_length = d[2].sg0.seg1_size; ctx->n_rx_bytes += b[3]->current_length = d[3].sg0.seg1_size; + b[0]->flow_id = d[0].parse.w[3] >> 48; + b[1]->flow_id = d[1].parse.w[3] >> 48; + b[2]->flow_id = d[2].parse.w[3] >> 48; + b[3]->flow_id = d[3].parse.w[3] >> 48; ctx->n_segs += 4; segs = d[0].sg0.segs + d[1].sg0.segs + d[2].sg0.segs + d[3].sg0.segs; @@ -149,6 +153,7 @@ oct_rx_batch (vlib_main_t *vm, oct_rx_node_ctx_t *ctx, ctx->to_next[0] = vlib_get_buffer_index (vm, b[0]); b[0]->template = bt; ctx->n_rx_bytes += b[0]->current_length = d[0].sg0.seg1_size; + b[0]->flow_id = d[0].parse.w[3] >> 48; ctx->n_segs += 1; if (d[0].sg0.segs > 1) oct_rx_attach_tail (vm, ctx, b[0], d + 0); From 55465da8f1ec8fac1319c36c5c4f3aad5d5b537e Mon Sep 17 00:00:00 2001 From: Monendra Singh Kushwaha Date: Fri, 23 Feb 2024 12:59:47 +0530 Subject: [PATCH 018/271] octeon: add support for VF device This patch adds support for VF and loopback device. Type: feature Change-Id: I1ea92f3a1161851957206300ab921c27290b0305 Signed-off-by: Monendra Singh Kushwaha Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/126560 Tested-by: Shiva Shankar Kommula --- src/plugins/dev_octeon/init.c | 3 ++- src/plugins/dev_octeon/octeon.h | 1 + src/plugins/dev_octeon/port.c | 21 ++++++++++++++++----- 3 files changed, 19 insertions(+), 6 deletions(-) diff --git a/src/plugins/dev_octeon/init.c b/src/plugins/dev_octeon/init.c index d3b6814e50..47be8a8f9a 100644 --- a/src/plugins/dev_octeon/init.c +++ b/src/plugins/dev_octeon/init.c @@ -51,6 +51,7 @@ static struct } _ (0xa063, RVU_PF, "Marvell Octeon Resource Virtualization Unit PF"), + _ (0xa0f8, RVU_VF, "Marvell Octeon Resource Virtualization Unit VF"), _ (0xa0f3, CPT_VF, "Marvell Octeon Cryptographic Accelerator Unit VF"), #undef _ }; @@ -238,7 +239,7 @@ oct_init (vlib_main_t *vm, vnet_dev_t *dev) strncpy ((char *) cd->plt_pci_dev.name, dev->device_id, sizeof (cd->plt_pci_dev.name) - 1); - if (cd->type == OCT_DEVICE_TYPE_RVU_PF) + if (cd->type == OCT_DEVICE_TYPE_RVU_PF || cd->type == OCT_DEVICE_TYPE_RVU_VF) return oct_init_nix (vm, dev); else if (cd->type == OCT_DEVICE_TYPE_CPT_VF) return oct_init_cpt (vm, dev); diff --git a/src/plugins/dev_octeon/octeon.h b/src/plugins/dev_octeon/octeon.h index 39501d0410..fd8a92c7b3 100644 --- a/src/plugins/dev_octeon/octeon.h +++ b/src/plugins/dev_octeon/octeon.h @@ -19,6 +19,7 @@ typedef enum { OCT_DEVICE_TYPE_UNKNOWN = 0, OCT_DEVICE_TYPE_RVU_PF, + OCT_DEVICE_TYPE_RVU_VF, OCT_DEVICE_TYPE_CPT_VF, } __clib_packed oct_device_type_t; diff --git a/src/plugins/dev_octeon/port.c b/src/plugins/dev_octeon/port.c index f1ae15aadb..00ad8b9c47 100644 --- a/src/plugins/dev_octeon/port.c +++ b/src/plugins/dev_octeon/port.c @@ -10,8 +10,9 @@ #include #include -#define OCT_FLOW_PREALLOC_SIZE 1 -#define OCT_FLOW_MAX_PRIORITY 7 +#define OCT_FLOW_PREALLOC_SIZE 1 +#define OCT_FLOW_MAX_PRIORITY 7 +#define OCT_ETH_LINK_SPEED_100G 100000 /**< 100 Gbps */ VLIB_REGISTER_LOG_CLASS (oct_log, static) = { .class_name = "octeon", @@ -171,9 +172,19 @@ oct_port_poll (vlib_main_t *vm, vnet_dev_port_t *port) vnet_dev_port_state_changes_t changes = {}; int rrv; - rrv = roc_nix_mac_link_info_get (nix, &link_info); - if (rrv) - return; + if (roc_nix_is_lbk (nix)) + { + link_info.status = 1; + link_info.full_duplex = 1; + link_info.autoneg = 0; + link_info.speed = OCT_ETH_LINK_SPEED_100G; + } + else + { + rrv = roc_nix_mac_link_info_get (nix, &link_info); + if (rrv) + return; + } if (cd->status != link_info.status) { From 68081afaaa90b76f71a56ddc2e1d4f0e7c83bc9e Mon Sep 17 00:00:00 2001 From: Monendra Singh Kushwaha Date: Mon, 15 Apr 2024 11:41:45 +0530 Subject: [PATCH 019/271] octeon: add support for SDP device This patch adds support for SDP (System DPI Packet Interface Unit) device. Type: feature Change-Id: Idf1f53b151edf2992613746d5818409187b4b051 Signed-off-by: Monendra Singh Kushwaha Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/126561 Tested-by: Shiva Shankar Kommula --- src/plugins/dev_octeon/init.c | 20 ++++++++++++++------ src/plugins/dev_octeon/octeon.h | 1 + 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/src/plugins/dev_octeon/init.c b/src/plugins/dev_octeon/init.c index 47be8a8f9a..03a1d59f7c 100644 --- a/src/plugins/dev_octeon/init.c +++ b/src/plugins/dev_octeon/init.c @@ -52,6 +52,7 @@ static struct _ (0xa063, RVU_PF, "Marvell Octeon Resource Virtualization Unit PF"), _ (0xa0f8, RVU_VF, "Marvell Octeon Resource Virtualization Unit VF"), + _ (0xa0f7, SDP_VF, "Marvell Octeon System DPI Packet Interface Unit VF"), _ (0xa0f3, CPT_VF, "Marvell Octeon Cryptographic Accelerator Unit VF"), #undef _ }; @@ -239,12 +240,19 @@ oct_init (vlib_main_t *vm, vnet_dev_t *dev) strncpy ((char *) cd->plt_pci_dev.name, dev->device_id, sizeof (cd->plt_pci_dev.name) - 1); - if (cd->type == OCT_DEVICE_TYPE_RVU_PF || cd->type == OCT_DEVICE_TYPE_RVU_VF) - return oct_init_nix (vm, dev); - else if (cd->type == OCT_DEVICE_TYPE_CPT_VF) - return oct_init_cpt (vm, dev); - else - return VNET_DEV_ERR_UNSUPPORTED_DEVICE; + switch (cd->type) + { + case OCT_DEVICE_TYPE_RVU_PF: + case OCT_DEVICE_TYPE_RVU_VF: + case OCT_DEVICE_TYPE_SDP_VF: + return oct_init_nix (vm, dev); + + case OCT_DEVICE_TYPE_CPT_VF: + return oct_init_cpt (vm, dev); + + default: + return VNET_DEV_ERR_UNSUPPORTED_DEVICE; + } return 0; } diff --git a/src/plugins/dev_octeon/octeon.h b/src/plugins/dev_octeon/octeon.h index fd8a92c7b3..2168216526 100644 --- a/src/plugins/dev_octeon/octeon.h +++ b/src/plugins/dev_octeon/octeon.h @@ -20,6 +20,7 @@ typedef enum OCT_DEVICE_TYPE_UNKNOWN = 0, OCT_DEVICE_TYPE_RVU_PF, OCT_DEVICE_TYPE_RVU_VF, + OCT_DEVICE_TYPE_SDP_VF, OCT_DEVICE_TYPE_CPT_VF, } __clib_packed oct_device_type_t; From 9dd8bcab7464630a758af393adf339751ad20443 Mon Sep 17 00:00:00 2001 From: Damjan Marion Date: Wed, 27 Mar 2024 22:19:15 +0100 Subject: [PATCH 020/271] octeon: refill even if nothing dequeued from rx queue It may happen that rx queue is empty of free buffers due to previous alloc failure. Type: fix Fixes: 01fe7ab Change-Id: I344dcda11525444bd1358b3d36ffdf8ab9aa2677 Signed-off-by: Damjan Marion Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/126564 Reviewed-by: Monendra Singh Kushwaha Tested-by: Shiva Shankar Kommula --- src/plugins/dev_octeon/rx_node.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/plugins/dev_octeon/rx_node.c b/src/plugins/dev_octeon/rx_node.c index 5f7e5a8469..35776c8b71 100644 --- a/src/plugins/dev_octeon/rx_node.c +++ b/src/plugins/dev_octeon/rx_node.c @@ -313,7 +313,7 @@ oct_rx_node_inline (vlib_main_t *vm, vlib_node_runtime_t *node, n_desc = (status.tail - head) & cq_mask; if (n_desc == 0) - return 0; + goto refill; vlib_get_new_next_frame (vm, node, ctx->next_index, ctx->to_next, ctx->n_left_to_next); @@ -365,14 +365,15 @@ oct_rx_node_inline (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_put_next_frame (vm, node, ctx->next_index, ctx->n_left_to_next); - n_enq = crq->n_enq - ctx->n_segs; - n_enq += oct_rxq_refill (vm, rxq, rxq->size - n_enq); - crq->n_enq = n_enq; - vlib_increment_combined_counter ( vnm->interface_main.combined_sw_if_counters + VNET_INTERFACE_COUNTER_RX, thr_idx, ctx->hw_if_index, ctx->n_rx_pkts, ctx->n_rx_bytes); +refill: + n_enq = crq->n_enq - ctx->n_segs; + n_enq += oct_rxq_refill (vm, rxq, rxq->size - n_enq); + crq->n_enq = n_enq; + return ctx->n_rx_pkts; } From fc7215be246c376e89e070a49aaaab3988599256 Mon Sep 17 00:00:00 2001 From: Damjan Marion Date: Wed, 27 Mar 2024 22:53:21 +0100 Subject: [PATCH 021/271] octeon: specify pool when alloc buffers Type: fix Fixes: 01fe7ab Change-Id: I72fdaca250468d91a31efcce2fb447c97ba49dc7 Signed-off-by: Damjan Marion Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/126565 Reviewed-by: Monendra Singh Kushwaha Tested-by: Shiva Shankar Kommula --- src/plugins/dev_octeon/rx_node.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/plugins/dev_octeon/rx_node.c b/src/plugins/dev_octeon/rx_node.c index 35776c8b71..997f135619 100644 --- a/src/plugins/dev_octeon/rx_node.c +++ b/src/plugins/dev_octeon/rx_node.c @@ -217,7 +217,8 @@ oct_rxq_refill (vlib_main_t *vm, vnet_dev_rx_queue_t *rxq, u16 n_refill) while (n_lines >= batch_max_lines) { - n_alloc = vlib_buffer_alloc (vm, buffer_indices, batch_max_bufs); + n_alloc = + vlib_buffer_alloc_from_pool (vm, buffer_indices, batch_max_bufs, bpi); if (PREDICT_FALSE (n_alloc < batch_max_bufs)) goto alloc_fail; oct_rxq_refill_batch (vm, lmt_id, addr, lines, buffer_indices, w0, From 9da1cd3f5f94864c731dbdf81c559b3c8370a125 Mon Sep 17 00:00:00 2001 From: Damjan Marion Date: Sat, 30 Mar 2024 14:56:51 +0100 Subject: [PATCH 022/271] octeon: fix buffer free on full tx ring Type: fix Fixes: 01fe7ab Change-Id: I4419107c4bcb7f85b76addfc62178b6e75e10a52 Signed-off-by: Damjan Marion Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/126566 Reviewed-by: Monendra Singh Kushwaha Tested-by: Shiva Shankar Kommula --- src/plugins/dev_octeon/tx_node.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/plugins/dev_octeon/tx_node.c b/src/plugins/dev_octeon/tx_node.c index 5deaa82a0c..28e8f25adb 100644 --- a/src/plugins/dev_octeon/tx_node.c +++ b/src/plugins/dev_octeon/tx_node.c @@ -392,11 +392,11 @@ VNET_DEV_NODE_FN (oct_tx_node) if (n < n_pkts) { - n = n_pkts - n; - vlib_buffer_free (vm, from + n, n); + u32 n_free = n_pkts - n; + vlib_buffer_free (vm, from + n, n_free); vlib_error_count (vm, node->node_index, OCT_TX_NODE_CTR_NO_FREE_SLOTS, - n); - n_pkts -= ctx.n_drop; + n_free); + n_pkts -= n_free; } if (ctx.n_drop) From febafdeefb7d9c92dac6842d5c50f73c0f4d1c6e Mon Sep 17 00:00:00 2001 From: Damjan Marion Date: Fri, 29 Mar 2024 21:33:03 +0100 Subject: [PATCH 023/271] octeon: fix memory ordering issue in tx batch free Type: fix Fixes: 01fe7ab Change-Id: I4425e809f0977521ddecf91b58b26fe4519dd6e0 Signed-off-by: Damjan Marion Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/126567 Reviewed-by: Monendra Singh Kushwaha Tested-by: Shiva Shankar Kommula --- src/plugins/dev_octeon/octeon.h | 14 +++++++++----- src/plugins/dev_octeon/port.c | 7 +++++-- src/plugins/dev_octeon/queue.c | 14 +++++++++++--- src/plugins/dev_octeon/tx_node.c | 26 +++++++++++++++++--------- 4 files changed, 42 insertions(+), 19 deletions(-) diff --git a/src/plugins/dev_octeon/octeon.h b/src/plugins/dev_octeon/octeon.h index 2168216526..92ec953ed2 100644 --- a/src/plugins/dev_octeon/octeon.h +++ b/src/plugins/dev_octeon/octeon.h @@ -15,6 +15,8 @@ #include #include +#define OCT_BATCH_ALLOC_IOVA0_MASK 0xFFFFFFFFFFFFFF80 + typedef enum { OCT_DEVICE_TYPE_UNKNOWN = 0, @@ -73,13 +75,15 @@ typedef struct typedef struct { CLIB_ALIGN_MARK (cl, 128); - union - { - struct npa_batch_alloc_status_s status; - u64 iova[16]; - }; + u64 iova[16]; } oct_npa_batch_alloc_cl128_t; +typedef union +{ + struct npa_batch_alloc_status_s status; + u64 as_u64; +} oct_npa_batch_alloc_status_t; + STATIC_ASSERT_SIZEOF (oct_npa_batch_alloc_cl128_t, 128); typedef struct diff --git a/src/plugins/dev_octeon/port.c b/src/plugins/dev_octeon/port.c index 00ad8b9c47..a82e48004b 100644 --- a/src/plugins/dev_octeon/port.c +++ b/src/plugins/dev_octeon/port.c @@ -284,8 +284,11 @@ oct_txq_stop (vlib_main_t *vm, vnet_dev_tx_queue_t *txq) for (n = ctq->ba_num_cl, cl = ctq->ba_buffer + ctq->ba_first_cl; n; cl++, n--) { - if (cl->status.ccode != 0) - for (u32 i = 0; i < cl->status.count; i++) + oct_npa_batch_alloc_status_t st; + + st.as_u64 = __atomic_load_n (cl->iova, __ATOMIC_ACQUIRE); + if (st.status.ccode != ALLOC_CCODE_INVAL) + for (u32 i = 0; i < st.status.count; i++) { vlib_buffer_t *b = (vlib_buffer_t *) (cl->iova[i] + off); vlib_buffer_free_one (vm, vlib_get_buffer_index (vm, b)); diff --git a/src/plugins/dev_octeon/queue.c b/src/plugins/dev_octeon/queue.c index 9378fc3b7c..d6ae794fb8 100644 --- a/src/plugins/dev_octeon/queue.c +++ b/src/plugins/dev_octeon/queue.c @@ -57,12 +57,20 @@ oct_tx_queue_alloc (vlib_main_t *vm, vnet_dev_tx_queue_t *txq) oct_txq_t *ctq = vnet_dev_get_tx_queue_data (txq); vnet_dev_port_t *port = txq->port; vnet_dev_t *dev = port->dev; + u32 sz = sizeof (void *) * ROC_CN10K_NPA_BATCH_ALLOC_MAX_PTRS; + vnet_dev_rv_t rv; log_debug (dev, "tx_queue_alloc: queue %u alocated", txq->queue_id); - return vnet_dev_dma_mem_alloc ( - vm, dev, sizeof (void *) * ROC_CN10K_NPA_BATCH_ALLOC_MAX_PTRS, 128, - (void **) &ctq->ba_buffer); + rv = vnet_dev_dma_mem_alloc (vm, dev, sz, 128, (void **) &ctq->ba_buffer); + + if (rv != VNET_DEV_OK) + return rv; + + clib_memset_u64 (ctq->ba_buffer, OCT_BATCH_ALLOC_IOVA0_MASK, + ROC_CN10K_NPA_BATCH_ALLOC_MAX_PTRS); + + return rv; } void diff --git a/src/plugins/dev_octeon/tx_node.c b/src/plugins/dev_octeon/tx_node.c index 28e8f25adb..0dbf8759d3 100644 --- a/src/plugins/dev_octeon/tx_node.c +++ b/src/plugins/dev_octeon/tx_node.c @@ -46,9 +46,12 @@ oct_batch_free (vlib_main_t *vm, oct_tx_ctx_t *ctx, vnet_dev_tx_queue_t *txq) for (cl = ctq->ba_buffer + ctq->ba_first_cl; num_cl > 0; num_cl--, cl++) { - u8 count; - if (cl->status.ccode == ALLOC_CCODE_INVAL) + oct_npa_batch_alloc_status_t st; + + if ((st.as_u64 = __atomic_load_n (cl->iova, __ATOMIC_RELAXED)) == + OCT_BATCH_ALLOC_IOVA0_MASK + ALLOC_CCODE_INVAL) { + cl_not_ready: ctx->batch_alloc_not_ready++; n_freed = bi - (u32 *) ctq->ba_buffer; if (n_freed > 0) @@ -63,11 +66,15 @@ oct_batch_free (vlib_main_t *vm, oct_tx_ctx_t *ctx, vnet_dev_tx_queue_t *txq) return 0; } - count = cl->status.count; + if (st.status.count > 8 && + __atomic_load_n (cl->iova + 8, __ATOMIC_RELAXED) == + OCT_BATCH_ALLOC_IOVA0_MASK) + goto cl_not_ready; + #if (CLIB_DEBUG > 0) - cl->status.count = cl->status.ccode = 0; + cl->iova[0] &= OCT_BATCH_ALLOC_IOVA0_MASK; #endif - if (PREDICT_TRUE (count == 16)) + if (PREDICT_TRUE (st.status.count == 16)) { /* optimize for likely case where cacheline is full */ vlib_get_buffer_indices_with_offset (vm, (void **) cl, bi, 16, @@ -76,9 +83,9 @@ oct_batch_free (vlib_main_t *vm, oct_tx_ctx_t *ctx, vnet_dev_tx_queue_t *txq) } else { - vlib_get_buffer_indices_with_offset (vm, (void **) cl, bi, count, - off); - bi += count; + vlib_get_buffer_indices_with_offset (vm, (void **) cl, bi, + st.status.count, off); + bi += st.status.count; } } @@ -89,7 +96,8 @@ oct_batch_free (vlib_main_t *vm, oct_tx_ctx_t *ctx, vnet_dev_tx_queue_t *txq) /* clear status bits in each cacheline */ n = cl - ctq->ba_buffer; for (u32 i = 0; i < n; i++) - ctq->ba_buffer[i].iova[0] = 0; + ctq->ba_buffer[i].iova[0] = ctq->ba_buffer[i].iova[8] = + OCT_BATCH_ALLOC_IOVA0_MASK; ctq->ba_num_cl = ctq->ba_first_cl = 0; } From 09bd57dc554432d112dd884d3f866acbeda4b693 Mon Sep 17 00:00:00 2001 From: Damjan Marion Date: Wed, 24 Apr 2024 20:50:01 +0200 Subject: [PATCH 024/271] octeon: fix roc_nix_npc_mac_addr_get() return value check Type: fix Fixes: 01fe7ab Change-Id: I46782c69773085f9e6a8dfff798bb42b1e822c32 Signed-off-by: Damjan Marion Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/126568 Reviewed-by: Monendra Singh Kushwaha Tested-by: Shiva Shankar Kommula --- src/plugins/dev_octeon/init.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plugins/dev_octeon/init.c b/src/plugins/dev_octeon/init.c index 03a1d59f7c..8c5ed95b06 100644 --- a/src/plugins/dev_octeon/init.c +++ b/src/plugins/dev_octeon/init.c @@ -114,7 +114,7 @@ oct_init_nix (vlib_main_t *vm, vnet_dev_t *dev) if ((rrv = roc_nix_dev_init (cd->nix))) return cnx_return_roc_err (dev, rrv, "roc_nix_dev_init"); - if (roc_nix_npc_mac_addr_get (cd->nix, mac_addr)) + if ((rrv = roc_nix_npc_mac_addr_get (cd->nix, mac_addr))) return cnx_return_roc_err (dev, rrv, "roc_nix_npc_mac_addr_get"); vnet_dev_port_add_args_t port_add_args = { From 70330d41103598760ecaa3d78c703a0570dea10b Mon Sep 17 00:00:00 2001 From: Monendra Singh Kushwaha Date: Thu, 25 Apr 2024 15:36:21 +0530 Subject: [PATCH 025/271] octeon: fix buffer free for more than 6 segment Type: fix Fixes: 01fe7ab Change-Id: I4423d287e8148344754b2f6a13886c093a1384e4 Signed-off-by: Monendra Singh Kushwaha Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/126727 Tested-by: sa_ip-sw-jenkins Reviewed-by: Nithinsen Kaithakadan --- src/plugins/dev_octeon/tx_node.c | 64 ++++++++++++++++++-------------- 1 file changed, 37 insertions(+), 27 deletions(-) diff --git a/src/plugins/dev_octeon/tx_node.c b/src/plugins/dev_octeon/tx_node.c index 0dbf8759d3..ce8709959a 100644 --- a/src/plugins/dev_octeon/tx_node.c +++ b/src/plugins/dev_octeon/tx_node.c @@ -133,7 +133,8 @@ oct_batch_free (vlib_main_t *vm, oct_tx_ctx_t *ctx, vnet_dev_tx_queue_t *txq) static_always_inline u8 oct_tx_enq1 (vlib_main_t *vm, oct_tx_ctx_t *ctx, vlib_buffer_t *b, - lmt_line_t *line, u32 flags, int simple, int trace) + lmt_line_t *line, u32 flags, int simple, int trace, u32 *n, + u8 *dpl) { u8 n_dwords = 2; u32 total_len = 0; @@ -159,7 +160,7 @@ oct_tx_enq1 (vlib_main_t *vm, oct_tx_ctx_t *ctx, vlib_buffer_t *b, tail_segs[n_tail_segs++] = t; if (n_tail_segs > 5) { - ctx->drop[ctx->n_drop++] = t; + ctx->drop[ctx->n_drop++] = b; return 0; } } @@ -231,6 +232,9 @@ oct_tx_enq1 (vlib_main_t *vm, oct_tx_ctx_t *ctx, vlib_buffer_t *b, for (u32 i = 0; i < n_dwords; i++) line->dwords[i] = d.as_u128[i]; + *dpl = n_dwords; + *n = *n + 1; + return n_dwords; } @@ -240,7 +244,7 @@ oct_tx_enq16 (vlib_main_t *vm, oct_tx_ctx_t *ctx, vnet_dev_tx_queue_t *txq, { u8 dwords_per_line[16], *dpl = dwords_per_line; u64 lmt_arg, ioaddr, n_lines; - u32 n_left, or_flags_16 = 0; + u32 n_left, or_flags_16 = 0, n = 0; const u32 not_simple_flags = VLIB_BUFFER_NEXT_PRESENT | VNET_BUFFER_F_OFFLOAD; lmt_line_t *l = ctx->lmt_lines; @@ -248,7 +252,7 @@ oct_tx_enq16 (vlib_main_t *vm, oct_tx_ctx_t *ctx, vnet_dev_tx_queue_t *txq, /* Data Store Memory Barrier - outer shareable domain */ asm volatile("dmb oshst" ::: "memory"); - for (n_left = n_pkts; n_left >= 8; n_left -= 8, b += 8, l += 8) + for (n_left = n_pkts; n_left >= 8; n_left -= 8, b += 8) { u32 f0, f1, f2, f3, f4, f5, f6, f7, or_f = 0; vlib_prefetch_buffer_header (b[8], LOAD); @@ -269,48 +273,54 @@ oct_tx_enq16 (vlib_main_t *vm, oct_tx_ctx_t *ctx, vnet_dev_tx_queue_t *txq, if ((or_f & not_simple_flags) == 0) { int simple = 1; - oct_tx_enq1 (vm, ctx, b[0], l, f0, simple, trace); - oct_tx_enq1 (vm, ctx, b[1], l + 1, f1, simple, trace); + oct_tx_enq1 (vm, ctx, b[0], l, f0, simple, trace, &n, &dpl[n]); + oct_tx_enq1 (vm, ctx, b[1], l + n, f1, simple, trace, &n, &dpl[n]); vlib_prefetch_buffer_header (b[13], LOAD); - oct_tx_enq1 (vm, ctx, b[2], l + 2, f2, simple, trace); - oct_tx_enq1 (vm, ctx, b[3], l + 3, f3, simple, trace); + oct_tx_enq1 (vm, ctx, b[2], l + n, f2, simple, trace, &n, &dpl[n]); + oct_tx_enq1 (vm, ctx, b[3], l + n, f3, simple, trace, &n, &dpl[n]); vlib_prefetch_buffer_header (b[14], LOAD); - oct_tx_enq1 (vm, ctx, b[4], l + 4, f4, simple, trace); - oct_tx_enq1 (vm, ctx, b[5], l + 5, f5, simple, trace); + oct_tx_enq1 (vm, ctx, b[4], l + n, f4, simple, trace, &n, &dpl[n]); + oct_tx_enq1 (vm, ctx, b[5], l + n, f5, simple, trace, &n, &dpl[n]); vlib_prefetch_buffer_header (b[15], LOAD); - oct_tx_enq1 (vm, ctx, b[6], l + 6, f6, simple, trace); - oct_tx_enq1 (vm, ctx, b[7], l + 7, f7, simple, trace); - dpl[0] = dpl[1] = dpl[2] = dpl[3] = 2; - dpl[4] = dpl[5] = dpl[6] = dpl[7] = 2; + oct_tx_enq1 (vm, ctx, b[6], l + n, f6, simple, trace, &n, &dpl[n]); + oct_tx_enq1 (vm, ctx, b[7], l + n, f7, simple, trace, &n, &dpl[n]); } else { int simple = 0; - dpl[0] = oct_tx_enq1 (vm, ctx, b[0], l, f0, simple, trace); - dpl[1] = oct_tx_enq1 (vm, ctx, b[1], l + 1, f1, simple, trace); + oct_tx_enq1 (vm, ctx, b[0], l, f0, simple, trace, &n, &dpl[n]); + oct_tx_enq1 (vm, ctx, b[1], l + n, f1, simple, trace, &n, &dpl[n]); vlib_prefetch_buffer_header (b[13], LOAD); - dpl[2] = oct_tx_enq1 (vm, ctx, b[2], l + 2, f2, simple, trace); - dpl[3] = oct_tx_enq1 (vm, ctx, b[3], l + 3, f3, simple, trace); + oct_tx_enq1 (vm, ctx, b[2], l + n, f2, simple, trace, &n, &dpl[n]); + oct_tx_enq1 (vm, ctx, b[3], l + n, f3, simple, trace, &n, &dpl[n]); vlib_prefetch_buffer_header (b[14], LOAD); - dpl[4] = oct_tx_enq1 (vm, ctx, b[4], l + 4, f4, simple, trace); - dpl[5] = oct_tx_enq1 (vm, ctx, b[5], l + 5, f5, simple, trace); + oct_tx_enq1 (vm, ctx, b[4], l + n, f4, simple, trace, &n, &dpl[n]); + oct_tx_enq1 (vm, ctx, b[5], l + n, f5, simple, trace, &n, &dpl[n]); vlib_prefetch_buffer_header (b[15], LOAD); - dpl[6] = oct_tx_enq1 (vm, ctx, b[6], l + 6, f6, simple, trace); - dpl[7] = oct_tx_enq1 (vm, ctx, b[7], l + 7, f7, simple, trace); + oct_tx_enq1 (vm, ctx, b[6], l + n, f6, simple, trace, &n, &dpl[n]); + oct_tx_enq1 (vm, ctx, b[7], l + n, f7, simple, trace, &n, &dpl[n]); } - dpl += 8; + dpl += n; + l += n; + n = 0; } - for (; n_left > 0; n_left -= 1, b += 1, l += 1) + for (; n_left > 0; n_left -= 1, b += 1) { u32 f0 = b[0]->flags; - dpl++[0] = oct_tx_enq1 (vm, ctx, b[0], l, f0, 0, trace); + oct_tx_enq1 (vm, ctx, b[0], l, f0, 0, trace, &n, &dpl[n]); or_flags_16 |= f0; + dpl += n; + l += n; + n = 0; } lmt_arg = ctx->lmt_id; ioaddr = ctx->lmt_ioaddr; - n_lines = n_pkts; + n_lines = dpl - dwords_per_line; + + if (PREDICT_FALSE (!n_lines)) + return n_pkts; if (PREDICT_FALSE (or_flags_16 & VLIB_BUFFER_NEXT_PRESENT)) { @@ -396,7 +406,7 @@ VNET_DEV_NODE_FN (oct_tx_node) n += oct_tx_enq16 (vm, &ctx, txq, b, n_left, /* trace */ 0); } - ctq->n_enq = n_enq + n; + ctq->n_enq = n_enq + n - ctx.n_drop; if (n < n_pkts) { From 988ebe2431f7735fcce56a771db275f42bd3f19a Mon Sep 17 00:00:00 2001 From: Monendra Singh Kushwaha Date: Fri, 19 Apr 2024 16:34:49 +0530 Subject: [PATCH 026/271] octeon: add max packet length check This patch compares packet length with maximum supported packet length and drops the packet accordingly. Type: fix Change-Id: I80ef453d43149818936649e1e58ae90c84a34ab9 Signed-off-by: Monendra Singh Kushwaha Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/126728 Tested-by: sa_ip-sw-jenkins Reviewed-by: Nithinsen Kaithakadan --- src/plugins/dev_octeon/octeon.h | 3 ++- src/plugins/dev_octeon/tx_node.c | 26 +++++++++++++++++++++++++- 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/src/plugins/dev_octeon/octeon.h b/src/plugins/dev_octeon/octeon.h index 92ec953ed2..a1280379bf 100644 --- a/src/plugins/dev_octeon/octeon.h +++ b/src/plugins/dev_octeon/octeon.h @@ -162,7 +162,8 @@ vnet_dev_rv_t oct_flow_query (vlib_main_t *, vnet_dev_port_t *, u32, uword, _ (AURA_BATCH_ALLOC_ISSUE_FAIL, aura_batch_alloc_issue_fail, ERROR, \ "aura batch alloc issue failed") \ _ (AURA_BATCH_ALLOC_NOT_READY, aura_batch_alloc_not_ready, ERROR, \ - "aura batch alloc not ready") + "aura batch alloc not ready") \ + _ (MTU_EXCEEDED, mtu_exceeded, ERROR, "mtu exceeded") typedef enum { diff --git a/src/plugins/dev_octeon/tx_node.c b/src/plugins/dev_octeon/tx_node.c index ce8709959a..a2e4b07de8 100644 --- a/src/plugins/dev_octeon/tx_node.c +++ b/src/plugins/dev_octeon/tx_node.c @@ -22,8 +22,11 @@ typedef struct u32 n_tx_bytes; u32 n_drop; vlib_buffer_t *drop[VLIB_FRAME_SIZE]; + u32 n_exd_mtu; + vlib_buffer_t *exd_mtu[VLIB_FRAME_SIZE]; u32 batch_alloc_not_ready; u32 batch_alloc_issue_fail; + int max_pkt_len; u16 lmt_id; u64 lmt_ioaddr; lmt_line_t *lmt_lines; @@ -149,6 +152,12 @@ oct_tx_enq1 (vlib_main_t *vm, oct_tx_ctx_t *ctx, vlib_buffer_t *b, }, }; + if (PREDICT_FALSE (vlib_buffer_length_in_chain (vm, b) > ctx->max_pkt_len)) + { + ctx->exd_mtu[ctx->n_exd_mtu++] = b; + return 0; + } + if (!simple && flags & VLIB_BUFFER_NEXT_PRESENT) { u8 n_tail_segs = 0; @@ -360,6 +369,8 @@ VNET_DEV_NODE_FN (oct_tx_node) vnet_dev_tx_node_runtime_t *rt = vnet_dev_get_tx_node_runtime (node); vnet_dev_tx_queue_t *txq = rt->tx_queue; oct_txq_t *ctq = vnet_dev_get_tx_queue_data (txq); + vnet_dev_t *dev = txq->port->dev; + oct_device_t *cd = vnet_dev_get_data (dev); u32 node_index = node->node_index; u32 *from = vlib_frame_vector_args (frame); u32 n, n_enq, n_left, n_pkts = frame->n_vectors; @@ -373,6 +384,7 @@ VNET_DEV_NODE_FN (oct_tx_node) .sq = ctq->sq.qid, .sizem1 = 1, }, + .max_pkt_len = roc_nix_max_pkt_len (cd->nix), .lmt_id = lmt_id, .lmt_ioaddr = ctq->io_addr, .lmt_lines = ctq->lmt_addr + (lmt_id << ROC_LMT_LINE_SIZE_LOG2), @@ -406,7 +418,7 @@ VNET_DEV_NODE_FN (oct_tx_node) n += oct_tx_enq16 (vm, &ctx, txq, b, n_left, /* trace */ 0); } - ctq->n_enq = n_enq + n - ctx.n_drop; + ctq->n_enq = n_enq + n - ctx.n_drop - ctx.n_exd_mtu; if (n < n_pkts) { @@ -421,6 +433,10 @@ VNET_DEV_NODE_FN (oct_tx_node) vlib_error_count (vm, node->node_index, OCT_TX_NODE_CTR_CHAIN_TOO_LONG, ctx.n_drop); + if (PREDICT_FALSE (ctx.n_exd_mtu)) + vlib_error_count (vm, node->node_index, OCT_TX_NODE_CTR_MTU_EXCEEDED, + ctx.n_exd_mtu); + if (ctx.batch_alloc_not_ready) vlib_error_count (vm, node_index, OCT_TX_NODE_CTR_AURA_BATCH_ALLOC_NOT_READY, @@ -441,5 +457,13 @@ VNET_DEV_NODE_FN (oct_tx_node) n_pkts -= ctx.n_drop; } + if (PREDICT_FALSE (ctx.n_exd_mtu)) + { + u32 bi[VLIB_FRAME_SIZE]; + vlib_get_buffer_indices (vm, ctx.exd_mtu, bi, ctx.n_exd_mtu); + vlib_buffer_free (vm, bi, ctx.n_exd_mtu); + n_pkts -= ctx.n_exd_mtu; + } + return n_pkts; } From c2758adc19ea3863f225c198707bd54a3fee56b2 Mon Sep 17 00:00:00 2001 From: Monendra Singh Kushwaha Date: Mon, 6 May 2024 14:57:28 +0530 Subject: [PATCH 027/271] octeon: fix rvu vf device id Type: fix Fixes: I1ea92f Change-Id: Idf91b3f4cb3cc93081a282a14ffe2421a3628509 Signed-off-by: Monendra Singh Kushwaha Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/126726 Tested-by: sa_ip-toolkits-Jenkins Tested-by: sa_ip-sw-jenkins Reviewed-by: Nithinsen Kaithakadan --- src/plugins/dev_octeon/init.c | 3 ++- src/plugins/dev_octeon/octeon.h | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/plugins/dev_octeon/init.c b/src/plugins/dev_octeon/init.c index 8c5ed95b06..97a11e0d0d 100644 --- a/src/plugins/dev_octeon/init.c +++ b/src/plugins/dev_octeon/init.c @@ -51,7 +51,8 @@ static struct } _ (0xa063, RVU_PF, "Marvell Octeon Resource Virtualization Unit PF"), - _ (0xa0f8, RVU_VF, "Marvell Octeon Resource Virtualization Unit VF"), + _ (0xa064, RVU_VF, "Marvell Octeon Resource Virtualization Unit VF"), + _ (0xa0f8, LBK_VF, "Marvell Octeon Loopback Unit VF"), _ (0xa0f7, SDP_VF, "Marvell Octeon System DPI Packet Interface Unit VF"), _ (0xa0f3, CPT_VF, "Marvell Octeon Cryptographic Accelerator Unit VF"), #undef _ diff --git a/src/plugins/dev_octeon/octeon.h b/src/plugins/dev_octeon/octeon.h index a1280379bf..e43cde0a35 100644 --- a/src/plugins/dev_octeon/octeon.h +++ b/src/plugins/dev_octeon/octeon.h @@ -22,6 +22,7 @@ typedef enum OCT_DEVICE_TYPE_UNKNOWN = 0, OCT_DEVICE_TYPE_RVU_PF, OCT_DEVICE_TYPE_RVU_VF, + OCT_DEVICE_TYPE_LBK_VF, OCT_DEVICE_TYPE_SDP_VF, OCT_DEVICE_TYPE_CPT_VF, } __clib_packed oct_device_type_t; From 766cc824d5495fc8bedd3c5ace5a0d5be208edd7 Mon Sep 17 00:00:00 2001 From: Guillaume Solignac Date: Tue, 9 Apr 2024 11:40:51 +0200 Subject: [PATCH 028/271] octeon: add promisc support We add the capability to enable and disable promiscous mode on the octeon port. Type: feature Signed-off-by: Guillaume Solignac Change-Id: I9a1464d2e1e8a0570ff16e221a4896aedc3ed6f8 Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/127828 Reviewed-by: Monendra Singh Kushwaha Reviewed-by: Nithinsen Kaithakadan Tested-by: sa_ip-sw-jenkins --- src/plugins/dev_octeon/port.c | 41 +++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/src/plugins/dev_octeon/port.c b/src/plugins/dev_octeon/port.c index a82e48004b..d5f78301ad 100644 --- a/src/plugins/dev_octeon/port.c +++ b/src/plugins/dev_octeon/port.c @@ -378,6 +378,44 @@ oct_port_stop (vlib_main_t *vm, vnet_dev_port_t *port) oct_txq_stop (vm, q); } +vnet_dev_rv_t +oct_validate_config_promisc_mode (vnet_dev_port_t *port, int enable) +{ + vnet_dev_t *dev = port->dev; + oct_device_t *cd = vnet_dev_get_data (dev); + struct roc_nix *nix = cd->nix; + + if (roc_nix_is_vf_or_sdp (nix)) + return VNET_DEV_ERR_UNSUPPORTED_DEVICE; + + return VNET_DEV_OK; +} + +vnet_dev_rv_t +oct_op_config_promisc_mode (vlib_main_t *vm, vnet_dev_port_t *port, int enable) +{ + vnet_dev_t *dev = port->dev; + oct_device_t *cd = vnet_dev_get_data (dev); + struct roc_nix *nix = cd->nix; + int rv; + + rv = roc_nix_npc_promisc_ena_dis (nix, enable); + if (rv) + { + return oct_roc_err (dev, rv, "roc_nix_npc_promisc_ena_dis failed"); + } + + rv = roc_nix_mac_promisc_mode_enable (nix, enable); + if (rv) + { + return oct_roc_err (dev, rv, + "roc_nix_mac_promisc_mode_enable(%s) failed", + enable ? "true" : "false"); + } + + return VNET_DEV_OK; +} + vnet_dev_rv_t oct_port_cfg_change_validate (vlib_main_t *vm, vnet_dev_port_t *port, vnet_dev_port_cfg_change_req_t *req) @@ -392,6 +430,8 @@ oct_port_cfg_change_validate (vlib_main_t *vm, vnet_dev_port_t *port, break; case VNET_DEV_PORT_CFG_PROMISC_MODE: + rv = oct_validate_config_promisc_mode (port, req->promisc); + break; case VNET_DEV_PORT_CFG_CHANGE_PRIMARY_HW_ADDR: case VNET_DEV_PORT_CFG_ADD_SECONDARY_HW_ADDR: case VNET_DEV_PORT_CFG_REMOVE_SECONDARY_HW_ADDR: @@ -421,6 +461,7 @@ oct_port_cfg_change (vlib_main_t *vm, vnet_dev_port_t *port, switch (req->type) { case VNET_DEV_PORT_CFG_PROMISC_MODE: + rv = oct_op_config_promisc_mode (vm, port, req->promisc); break; case VNET_DEV_PORT_CFG_CHANGE_PRIMARY_HW_ADDR: From ac16ead67a7f8bc0cc5c4d80e5ed24eb8c6512e1 Mon Sep 17 00:00:00 2001 From: Alok Mishra Date: Wed, 10 Apr 2024 13:04:51 +0530 Subject: [PATCH 029/271] octeon: add support for mac address update This patch adds support for updating primary hardware address on octeon port. Type: feature Change-Id: Ib0ff6aaa6dafc209eb71c44b8a6504d3df9aa5c8 Signed-off-by: Alok Mishra Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/127829 Reviewed-by: Monendra Singh Kushwaha Reviewed-by: Nithinsen Kaithakadan Tested-by: sa_ip-toolkits-Jenkins Tested-by: sa_ip-sw-jenkins --- src/plugins/dev_octeon/port.c | 41 +++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/src/plugins/dev_octeon/port.c b/src/plugins/dev_octeon/port.c index d5f78301ad..98a4c28b37 100644 --- a/src/plugins/dev_octeon/port.c +++ b/src/plugins/dev_octeon/port.c @@ -416,6 +416,44 @@ oct_op_config_promisc_mode (vlib_main_t *vm, vnet_dev_port_t *port, int enable) return VNET_DEV_OK; } +static vnet_dev_rv_t +oct_port_add_del_eth_addr (vlib_main_t *vm, vnet_dev_port_t *port, + vnet_dev_hw_addr_t *addr, int is_add, + int is_primary) +{ + vnet_dev_t *dev = port->dev; + oct_device_t *cd = vnet_dev_get_data (dev); + struct roc_nix *nix = cd->nix; + vnet_dev_rv_t rv = VNET_DEV_OK; + + i32 rrv; + + if (is_primary) + { + if (is_add) + { + /* Update mac address at NPC */ + rrv = roc_nix_npc_mac_addr_set (nix, (u8 *) addr); + if (rrv) + rv = oct_roc_err (dev, rrv, "roc_nix_npc_mac_addr_set() failed"); + + /* Update mac address at CGX for PFs only */ + if (!roc_nix_is_vf_or_sdp (nix)) + { + rrv = roc_nix_mac_addr_set (nix, (u8 *) addr); + if (rrv) + { + /* Rollback to previous mac address */ + roc_nix_npc_mac_addr_set (nix, + (u8 *) &port->primary_hw_addr); + rv = oct_roc_err (dev, rrv, "roc_nix_mac_addr_set() failed"); + } + } + } + } + return rv; +} + vnet_dev_rv_t oct_port_cfg_change_validate (vlib_main_t *vm, vnet_dev_port_t *port, vnet_dev_port_cfg_change_req_t *req) @@ -465,6 +503,9 @@ oct_port_cfg_change (vlib_main_t *vm, vnet_dev_port_t *port, break; case VNET_DEV_PORT_CFG_CHANGE_PRIMARY_HW_ADDR: + rv = oct_port_add_del_eth_addr (vm, port, &req->addr, + /* is_add */ 1, + /* is_primary */ 1); break; case VNET_DEV_PORT_CFG_ADD_SECONDARY_HW_ADDR: From e617e8a41e0ef192b324db8f134cda8b2bb4292d Mon Sep 17 00:00:00 2001 From: Alok Mishra Date: Fri, 26 Apr 2024 18:40:13 +0500 Subject: [PATCH 030/271] dev: fix mac address dump in trace output "port->attr.hw_addr" is set during the port initialization. It won't be updated if the mac address of the port is changed via the vpp command line. Use "port->primary_hw_addr" to dump the updated mac address correclty. Type: fix Fixes: 38c619115b Change-Id: I6c99fcfdfae67efb3606e17f36781c56716ff7ea Signed-off-by: Alok Mishra Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/127830 Reviewed-by: Monendra Singh Kushwaha Reviewed-by: Nithinsen Kaithakadan Tested-by: sa_ip-sw-jenkins Tested-by: sa_ip-toolkits-Jenkins --- src/vnet/dev/format.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/vnet/dev/format.c b/src/vnet/dev/format.c index ed83a0eba9..f599c0f8b8 100644 --- a/src/vnet/dev/format.c +++ b/src/vnet/dev/format.c @@ -101,7 +101,7 @@ format_vnet_dev_port_info (u8 *s, va_list *args) u32 indent = format_get_indent (s); s = format (s, "Hardware Address is %U", format_vnet_dev_hw_addr, - &port->attr.hw_addr); + &port->primary_hw_addr); s = format (s, ", %u RX queues (max %u), %u TX queues (max %u)", pool_elts (port->rx_queues), port->attr.max_rx_queues, pool_elts (port->tx_queues), port->attr.max_tx_queues); From 7602a5d9db48aaa142455c54d534b31064e758dc Mon Sep 17 00:00:00 2001 From: Monendra Singh Kushwaha Date: Wed, 28 Feb 2024 13:09:06 +0530 Subject: [PATCH 031/271] octeon: add support for Marvell Octeon9 SoC Type: feature Signed-off-by: Monendra Singh Kushwaha Change-Id: I5db58b8ec41b45596bc03b4a336a184c17871294 Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/127831 Tested-by: sa_ip-sw-jenkins --- build/external/packages/octeon-roc.mk | 4 +- src/plugins/dev_octeon/CMakeLists.txt | 6 ++- src/plugins/dev_octeon/roc_helper.c | 7 ++++ src/plugins/dev_octeon/rx_node.c | 33 ++++++++++++++++ src/plugins/dev_octeon/tx_node.c | 56 ++++++++++++++++++++++++++- 5 files changed, 102 insertions(+), 4 deletions(-) diff --git a/build/external/packages/octeon-roc.mk b/build/external/packages/octeon-roc.mk index 62f5d823bd..f96cd4c6e3 100644 --- a/build/external/packages/octeon-roc.mk +++ b/build/external/packages/octeon-roc.mk @@ -2,9 +2,9 @@ # SPDX-License-Identifier: Apache-2.0 # https://spdx.org/licenses/Apache-2.0.html -octeon-roc_version := 0.3 +octeon-roc_version := 0.4 octeon-roc_tarball := octeon-roc-v$(octeon-roc_version).tar.gz -octeon-roc_tarball_md5sum := e4a16beb76a6c63af1600dd4d1d752b8 +octeon-roc_tarball_md5sum := f3772f6e05d71cf2709c00ff1a1929ca octeon-roc_tarball_strip_dirs := 1 octeon-roc_url := https://github.com/MarvellEmbeddedProcessors/marvell-vpp/archive/refs/tags/$(octeon-roc_tarball) diff --git a/src/plugins/dev_octeon/CMakeLists.txt b/src/plugins/dev_octeon/CMakeLists.txt index e8abf1a338..0f6b32bbec 100644 --- a/src/plugins/dev_octeon/CMakeLists.txt +++ b/src/plugins/dev_octeon/CMakeLists.txt @@ -1,7 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # Copyright(c) 2022 Cisco Systems, Inc. -if (NOT VPP_PLATFORM_NAME STREQUAL "octeon10") +if (NOT VPP_PLATFORM_NAME STREQUAL "octeon10" AND NOT VPP_PLATFORM_NAME STREQUAL "octeon9") return() endif() @@ -21,6 +21,10 @@ endif() include_directories (${OCTEON_ROC_DIR}/) +if (VPP_PLATFORM_NAME STREQUAL "octeon9") + add_compile_definitions(PLATFORM_OCTEON9) +endif() + add_vpp_plugin(dev_octeon SOURCES init.c diff --git a/src/plugins/dev_octeon/roc_helper.c b/src/plugins/dev_octeon/roc_helper.c index f10c2cb578..16e0a871a9 100644 --- a/src/plugins/dev_octeon/roc_helper.c +++ b/src/plugins/dev_octeon/roc_helper.c @@ -49,6 +49,12 @@ oct_plt_get_thread_index (void) return __os_thread_index; } +static u64 +oct_plt_get_cache_line_size (void) +{ + return CLIB_CACHE_LINE_BYTES; +} + static void oct_drv_physmem_free (vlib_main_t *vm, void *mem) { @@ -178,4 +184,5 @@ oct_plt_init_param_t oct_plt_init_param = { .oct_plt_spinlock_unlock = oct_plt_spinlock_unlock, .oct_plt_spinlock_trylock = oct_plt_spinlock_trylock, .oct_plt_get_thread_index = oct_plt_get_thread_index, + .oct_plt_get_cache_line_size = oct_plt_get_cache_line_size, }; diff --git a/src/plugins/dev_octeon/rx_node.c b/src/plugins/dev_octeon/rx_node.c index 997f135619..1f8d5d93fa 100644 --- a/src/plugins/dev_octeon/rx_node.c +++ b/src/plugins/dev_octeon/rx_node.c @@ -165,6 +165,38 @@ oct_rx_batch (vlib_main_t *vm, oct_rx_node_ctx_t *ctx, return n; } +#ifdef PLATFORM_OCTEON9 +static_always_inline u32 +oct_rxq_refill (vlib_main_t *vm, vnet_dev_rx_queue_t *rxq, u16 n_refill) +{ + u32 n_alloc, n_free; + u32 buffer_indices[n_refill]; + vlib_buffer_t *buffers[n_refill]; + u8 bpi = vnet_dev_get_rx_queue_buffer_pool_index (rxq); + oct_rxq_t *crq = vnet_dev_get_rx_queue_data (rxq); + u64 aura = roc_npa_aura_handle_to_aura (crq->aura_handle); + const uint64_t addr = + roc_npa_aura_handle_to_base (crq->aura_handle) + NPA_LF_AURA_OP_FREE0; + + if (n_refill < 256) + return 0; + + n_alloc = vlib_buffer_alloc (vm, buffer_indices, n_refill); + if (PREDICT_FALSE (n_alloc < n_refill)) + goto alloc_fail; + + vlib_get_buffers (vm, buffer_indices, (vlib_buffer_t **) buffers, n_alloc); + + for (n_free = 0; n_free < n_alloc; n_free++) + roc_store_pair ((u64) buffers[n_free], aura, addr); + + return n_alloc; + +alloc_fail: + vlib_buffer_unalloc_to_pool (vm, buffer_indices, n_alloc, bpi); + return 0; +} +#else static_always_inline void oct_rxq_refill_batch (vlib_main_t *vm, u64 lmt_id, u64 addr, oct_npa_lf_aura_batch_free_line_t *lines, u32 *bi, @@ -260,6 +292,7 @@ oct_rxq_refill (vlib_main_t *vm, vnet_dev_rx_queue_t *rxq, u16 n_refill) return n_enq; } +#endif static_always_inline void oct_rx_trace (vlib_main_t *vm, vlib_node_runtime_t *node, diff --git a/src/plugins/dev_octeon/tx_node.c b/src/plugins/dev_octeon/tx_node.c index a2e4b07de8..0907493814 100644 --- a/src/plugins/dev_octeon/tx_node.c +++ b/src/plugins/dev_octeon/tx_node.c @@ -32,6 +32,44 @@ typedef struct lmt_line_t *lmt_lines; } oct_tx_ctx_t; +#ifdef PLATFORM_OCTEON9 +static_always_inline u32 +oct_batch_free (vlib_main_t *vm, oct_tx_ctx_t *ctx, vnet_dev_tx_queue_t *txq) +{ + oct_txq_t *ctq = vnet_dev_get_tx_queue_data (txq); + u16 off = ctq->hdr_off; + u64 ah = ctq->aura_handle; + u32 n_freed = 0, n; + + ah = ctq->aura_handle; + + if ((n = roc_npa_aura_op_available (ah)) >= 32) + { + u64 buffers[n]; + u32 bi[n]; + + n_freed = roc_npa_aura_op_bulk_alloc (ah, buffers, n, 0, 1); + vlib_get_buffer_indices_with_offset (vm, (void **) &buffers, bi, n_freed, + off); + vlib_buffer_free_no_next (vm, bi, n_freed); + } + + return n_freed; +} + +static_always_inline void +oct_lmt_copy (void *lmt_addr, u64 io_addr, void *desc, u64 dwords) +{ + u64 lmt_status; + + do + { + roc_lmt_mov_seg (lmt_addr, desc, dwords); + lmt_status = roc_lmt_submit_ldeor (io_addr); + } + while (lmt_status == 0); +} +#else static_always_inline u32 oct_batch_free (vlib_main_t *vm, oct_tx_ctx_t *ctx, vnet_dev_tx_queue_t *txq) { @@ -133,6 +171,7 @@ oct_batch_free (vlib_main_t *vm, oct_tx_ctx_t *ctx, vnet_dev_tx_queue_t *txq) return n_freed; } +#endif static_always_inline u8 oct_tx_enq1 (vlib_main_t *vm, oct_tx_ctx_t *ctx, vlib_buffer_t *b, @@ -158,6 +197,11 @@ oct_tx_enq1 (vlib_main_t *vm, oct_tx_ctx_t *ctx, vlib_buffer_t *b, return 0; } +#ifdef PLATFORM_OCTEON9 + /* Override line for Octeon9 */ + line = ctx->lmt_lines; +#endif + if (!simple && flags & VLIB_BUFFER_NEXT_PRESENT) { u8 n_tail_segs = 0; @@ -238,8 +282,12 @@ oct_tx_enq1 (vlib_main_t *vm, oct_tx_ctx_t *ctx, vlib_buffer_t *b, t->sw_if_index = vnet_buffer (b)->sw_if_index[VLIB_TX]; } +#ifdef PLATFORM_OCTEON9 + oct_lmt_copy (line, ctx->lmt_ioaddr, &d, n_dwords); +#else for (u32 i = 0; i < n_dwords; i++) line->dwords[i] = d.as_u128[i]; +#endif *dpl = n_dwords; *n = *n + 1; @@ -252,7 +300,7 @@ oct_tx_enq16 (vlib_main_t *vm, oct_tx_ctx_t *ctx, vnet_dev_tx_queue_t *txq, vlib_buffer_t **b, u32 n_pkts, int trace) { u8 dwords_per_line[16], *dpl = dwords_per_line; - u64 lmt_arg, ioaddr, n_lines; + u64 __attribute__ ((unused)) lmt_arg, ioaddr, n_lines; u32 n_left, or_flags_16 = 0, n = 0; const u32 not_simple_flags = VLIB_BUFFER_NEXT_PRESENT | VNET_BUFFER_F_OFFLOAD; @@ -331,6 +379,7 @@ oct_tx_enq16 (vlib_main_t *vm, oct_tx_ctx_t *ctx, vnet_dev_tx_queue_t *txq, if (PREDICT_FALSE (!n_lines)) return n_pkts; +#ifndef PLATFORM_OCTEON9 if (PREDICT_FALSE (or_flags_16 & VLIB_BUFFER_NEXT_PRESENT)) { dpl = dwords_per_line; @@ -359,6 +408,7 @@ oct_tx_enq16 (vlib_main_t *vm, oct_tx_ctx_t *ctx, vnet_dev_tx_queue_t *txq, } roc_lmt_submit_steorl (lmt_arg, ioaddr); +#endif return n_pkts; } @@ -375,7 +425,11 @@ VNET_DEV_NODE_FN (oct_tx_node) u32 *from = vlib_frame_vector_args (frame); u32 n, n_enq, n_left, n_pkts = frame->n_vectors; vlib_buffer_t *buffers[VLIB_FRAME_SIZE + 8], **b = buffers; +#ifdef PLATFORM_OCTEON9 + u64 lmt_id = 0; +#else u64 lmt_id = vm->thread_index << ROC_LMT_LINES_PER_CORE_LOG2; +#endif oct_tx_ctx_t ctx = { .node = node, From c4c64e94d004daf3471bf10df83e2ef61ac006e8 Mon Sep 17 00:00:00 2001 From: Sriram Vatala Date: Tue, 12 Mar 2024 04:22:00 +0000 Subject: [PATCH 032/271] octeon: add support for vnet generic flow type This patch adds vnet generic flow type support in OCTEON plugin, which extends the existing vnet flow types supported. It allows users to configure additional match patterns like 802.1q tag fields, 802.1ad tag fields, MPLS fields, IP DSCP etc., if supported by the underlying hardware. On OCTEON various match patterns including user defined custom protocol types can be supported depending on the programmable classification profile. Generic flows operate based on hexadecimal strings representing packet data bytes and corresponding mask data bytes. The mask data bytes, with bits set to '1', selectively identify the data bytes used for hardware flow matching. To configure generic flow rules, packetforge tool is recommended which accepts inputs in a user readable and friendly format. This tool is available in VPP tree under `extras/packetforge`. Detailed instructions can be found in the documentation under `extras/packetforge`. Additionally user can use existing vnet flow CLI and binary API interfaces to configure rules manually. Type: feature Change-Id: I8198536cf1fe0a4719542a8b54c599230c7852e9 Signed-off-by: Sriram Vatala --- src/plugins/dev_octeon/flow.c | 386 ++++++++++++++++++++++++++++++++ src/plugins/dev_octeon/octeon.h | 6 + 2 files changed, 392 insertions(+) diff --git a/src/plugins/dev_octeon/flow.c b/src/plugins/dev_octeon/flow.c index 1c367a036a..35aabde76a 100644 --- a/src/plugins/dev_octeon/flow.c +++ b/src/plugins/dev_octeon/flow.c @@ -46,6 +46,8 @@ VLIB_REGISTER_LOG_CLASS (oct_log, static) = { (f->type == VNET_FLOW_TYPE_IP4_GTPC) || \ (f->type == VNET_FLOW_TYPE_IP4_GTPU)) +#define FLOW_IS_GENERIC_TYPE(f) (f->type == VNET_FLOW_TYPE_GENERIC) + #define OCT_FLOW_UNSUPPORTED_ACTIONS(f) \ ((f->actions == VNET_FLOW_ACTION_BUFFER_ADVANCE) || \ (f->actions == VNET_FLOW_ACTION_REDIRECT_TO_NODE)) @@ -71,6 +73,9 @@ VLIB_REGISTER_LOG_CLASS (oct_log, static) = { _ (62, FLOW_KEY_TYPE_L3_DST, "l3-dst-only") \ _ (63, FLOW_KEY_TYPE_L3_SRC, "l3-src-only") +#define GTPU_PORT 2152 +#define VXLAN_PORT 4789 + typedef struct { u16 src_port; @@ -87,6 +92,27 @@ typedef struct u32 teid; } gtpu_header_t; +typedef struct +{ + u8 layer; + u16 nxt_proto; + vnet_dev_port_t *port; + struct roc_npc_item_info *items; + struct + { + u8 *spec; + u8 *mask; + u16 off; + } oct_drv; + struct + { + u8 *spec; + u8 *mask; + u16 off; + u16 len; + } generic; +} oct_flow_parse_state; + static void oct_flow_convert_rss_types (u64 *key, u64 rss_types) { @@ -183,6 +209,320 @@ oct_flow_rule_create (vnet_dev_port_t *port, struct roc_npc_action *actions, return VNET_DEV_OK; } +static int +oct_parse_l2 (oct_flow_parse_state *pst) +{ + struct roc_npc_flow_item_eth *eth_spec = + (struct roc_npc_flow_item_eth *) &pst->oct_drv.spec[pst->oct_drv.off]; + struct roc_npc_flow_item_eth *eth_mask = + (struct roc_npc_flow_item_eth *) &pst->oct_drv.mask[pst->oct_drv.off]; + ethernet_header_t *eth_hdr_mask = + (ethernet_header_t *) &pst->generic.mask[pst->generic.off]; + ethernet_header_t *eth_hdr = + (ethernet_header_t *) &pst->generic.spec[pst->generic.off]; + u16 tpid, etype; + + tpid = etype = clib_net_to_host_u16 (eth_hdr->type); + clib_memcpy_fast (eth_spec, eth_hdr, sizeof (ethernet_header_t)); + clib_memcpy_fast (eth_mask, eth_hdr_mask, sizeof (ethernet_header_t)); + eth_spec->has_vlan = 0; + + pst->items[pst->layer].spec = (void *) eth_spec; + pst->items[pst->layer].mask = (void *) eth_mask; + pst->items[pst->layer].size = sizeof (ethernet_header_t); + pst->items[pst->layer].type = ROC_NPC_ITEM_TYPE_ETH; + pst->generic.off += sizeof (ethernet_header_t); + pst->oct_drv.off += sizeof (struct roc_npc_flow_item_eth); + pst->layer++; + + /* Parse VLAN Tags if any */ + struct roc_npc_flow_item_vlan *vlan_spec = + (struct roc_npc_flow_item_vlan *) &pst->oct_drv.spec[pst->oct_drv.off]; + struct roc_npc_flow_item_vlan *vlan_mask = + (struct roc_npc_flow_item_vlan *) &pst->oct_drv.mask[pst->oct_drv.off]; + ethernet_vlan_header_t *vlan_hdr, *vlan_hdr_mask; + u8 vlan_cnt = 0; + + while (tpid == ETHERNET_TYPE_DOT1AD || tpid == ETHERNET_TYPE_VLAN) + { + if (pst->generic.off >= pst->generic.len) + break; + + vlan_hdr = + (ethernet_vlan_header_t *) &pst->generic.spec[pst->generic.off]; + vlan_hdr_mask = + (ethernet_vlan_header_t *) &pst->generic.mask[pst->generic.off]; + tpid = etype = clib_net_to_host_u16 (vlan_hdr->type); + clib_memcpy (&vlan_spec[vlan_cnt], vlan_hdr, + sizeof (ethernet_vlan_header_t)); + clib_memcpy (&vlan_mask[vlan_cnt], vlan_hdr_mask, + sizeof (ethernet_vlan_header_t)); + pst->items[pst->layer].spec = (void *) &vlan_spec[vlan_cnt]; + pst->items[pst->layer].mask = (void *) &vlan_mask[vlan_cnt]; + pst->items[pst->layer].size = sizeof (ethernet_vlan_header_t); + pst->items[pst->layer].type = ROC_NPC_ITEM_TYPE_VLAN; + pst->generic.off += sizeof (ethernet_vlan_header_t); + pst->oct_drv.off += sizeof (struct roc_npc_flow_item_vlan); + pst->layer++; + vlan_cnt++; + } + + /* Inner most vlan tag */ + if (vlan_cnt) + vlan_spec[vlan_cnt - 1].has_more_vlan = 0; + + pst->nxt_proto = etype; + return 0; +} + +static int +oct_parse_l3 (oct_flow_parse_state *pst) +{ + + if (pst->generic.off >= pst->generic.len || pst->nxt_proto == 0) + return 0; + + if (pst->nxt_proto == ETHERNET_TYPE_MPLS) + { + int label_stack_bottom = 0; + do + { + + u8 *mpls_spec = &pst->generic.spec[pst->generic.off]; + u8 *mpls_mask = &pst->generic.mask[pst->generic.off]; + + label_stack_bottom = mpls_spec[2] & 1; + pst->items[pst->layer].spec = (void *) mpls_spec; + pst->items[pst->layer].mask = (void *) mpls_mask; + pst->items[pst->layer].size = sizeof (u32); + pst->items[pst->layer].type = ROC_NPC_ITEM_TYPE_MPLS; + pst->generic.off += sizeof (u32); + pst->layer++; + } + while (label_stack_bottom); + + pst->nxt_proto = 0; + return 0; + } + else if (pst->nxt_proto == ETHERNET_TYPE_IP4) + { + ip4_header_t *ip4_spec = + (ip4_header_t *) &pst->generic.spec[pst->generic.off]; + ip4_header_t *ip4_mask = + (ip4_header_t *) &pst->generic.mask[pst->generic.off]; + pst->items[pst->layer].spec = (void *) ip4_spec; + pst->items[pst->layer].mask = (void *) ip4_mask; + pst->items[pst->layer].size = sizeof (ip4_header_t); + pst->items[pst->layer].type = ROC_NPC_ITEM_TYPE_IPV4; + pst->generic.off += sizeof (ip4_header_t); + pst->layer++; + pst->nxt_proto = ip4_spec->protocol; + } + else if (pst->nxt_proto == ETHERNET_TYPE_IP6) + { + struct roc_npc_flow_item_ipv6 *ip6_spec = + (struct roc_npc_flow_item_ipv6 *) &pst->oct_drv.spec[pst->oct_drv.off]; + struct roc_npc_flow_item_ipv6 *ip6_mask = + (struct roc_npc_flow_item_ipv6 *) &pst->oct_drv.mask[pst->oct_drv.off]; + ip6_header_t *ip6_hdr_mask = + (ip6_header_t *) &pst->generic.mask[pst->generic.off]; + ip6_header_t *ip6_hdr = + (ip6_header_t *) &pst->generic.spec[pst->generic.off]; + u8 nxt_hdr = ip6_hdr->protocol; + + clib_memcpy (ip6_spec, ip6_hdr, sizeof (ip6_header_t)); + clib_memcpy (ip6_mask, ip6_hdr_mask, sizeof (ip6_header_t)); + pst->items[pst->layer].spec = (void *) ip6_spec; + pst->items[pst->layer].mask = (void *) ip6_mask; + pst->items[pst->layer].size = sizeof (ip6_header_t); + pst->items[pst->layer].type = ROC_NPC_ITEM_TYPE_IPV6; + pst->generic.off += sizeof (ip6_header_t); + pst->oct_drv.off += sizeof (struct roc_npc_flow_item_ipv6); + pst->layer++; + + while (nxt_hdr == IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS || + nxt_hdr == IP_PROTOCOL_IP6_DESTINATION_OPTIONS || + nxt_hdr == IP_PROTOCOL_IPV6_ROUTE) + { + if (pst->generic.off >= pst->generic.len) + return 0; + + ip6_ext_header_t *ip6_ext_spec = + (ip6_ext_header_t *) &pst->generic.spec[pst->generic.off]; + ip6_ext_header_t *ip6_ext_mask = + (ip6_ext_header_t *) &pst->generic.mask[pst->generic.off]; + nxt_hdr = ip6_ext_spec->next_hdr; + + pst->items[pst->layer].spec = (void *) ip6_ext_spec; + pst->items[pst->layer].mask = (void *) ip6_ext_mask; + pst->items[pst->layer].type = ROC_NPC_ITEM_TYPE_IPV6_EXT; + pst->generic.off += ip6_ext_header_len (ip6_ext_spec); + pst->layer++; + } + + if (pst->generic.off >= pst->generic.len) + return 0; + + if (nxt_hdr == IP_PROTOCOL_IPV6_FRAGMENTATION) + { + ip6_frag_hdr_t *ip6_ext_frag_spec = + (ip6_frag_hdr_t *) &pst->generic.spec[pst->generic.off]; + ip6_frag_hdr_t *ip6_ext_frag_mask = + (ip6_frag_hdr_t *) &pst->generic.mask[pst->generic.off]; + + pst->items[pst->layer].spec = (void *) ip6_ext_frag_spec; + pst->items[pst->layer].mask = (void *) ip6_ext_frag_mask; + pst->items[pst->layer].size = sizeof (ip6_frag_hdr_t); + pst->items[pst->layer].type = ROC_NPC_ITEM_TYPE_IPV6_FRAG_EXT; + pst->generic.off += sizeof (ip6_frag_hdr_t); + pst->layer++; + } + + pst->nxt_proto = nxt_hdr; + } + /* Unsupported L3. */ + else + return -1; + + return 0; +} + +static int +oct_parse_l4 (oct_flow_parse_state *pst) +{ + + if (pst->generic.off >= pst->generic.len || pst->nxt_proto == 0) + return 0; + +#define _(protocol_t, protocol_value, ltype) \ + if (pst->nxt_proto == protocol_value) \ + \ + { \ + \ + protocol_t *spec = (protocol_t *) &pst->generic.spec[pst->generic.off]; \ + protocol_t *mask = (protocol_t *) &pst->generic.mask[pst->generic.off]; \ + pst->items[pst->layer].spec = spec; \ + pst->items[pst->layer].mask = mask; \ + \ + pst->items[pst->layer].size = sizeof (protocol_t); \ + \ + pst->items[pst->layer].type = ROC_NPC_ITEM_TYPE_##ltype; \ + pst->generic.off += sizeof (protocol_t); \ + pst->layer++; \ + return 0; \ + } + + _ (esp_header_t, IP_PROTOCOL_IPSEC_ESP, ESP) + _ (udp_header_t, IP_PROTOCOL_UDP, UDP) + _ (tcp_header_t, IP_PROTOCOL_TCP, TCP) + _ (sctp_header_t, IP_PROTOCOL_SCTP, SCTP) + _ (icmp46_header_t, IP_PROTOCOL_ICMP, ICMP) + _ (icmp46_header_t, IP_PROTOCOL_ICMP6, ICMP) + _ (igmp_header_t, IP_PROTOCOL_IGMP, IGMP) + _ (gre_header_t, IP_PROTOCOL_GRE, GRE) + + /* Unsupported L4. */ + return -1; +} + +static int +oct_parse_tunnel (oct_flow_parse_state *pst) +{ + if (pst->generic.off >= pst->generic.len) + return 0; + + if (pst->items[pst->layer - 1].type == ROC_NPC_ITEM_TYPE_GRE) + { + gre_header_t *gre_hdr = (gre_header_t *) pst->items[pst->layer - 1].spec; + pst->nxt_proto = clib_net_to_host_u16 (gre_hdr->protocol); + goto parse_l3; + } + + else if (pst->items[pst->layer - 1].type == ROC_NPC_ITEM_TYPE_UDP) + { + udp_header_t *udp_h = (udp_header_t *) pst->items[pst->layer - 1].spec; + u16 dport = clib_net_to_host_u16 (udp_h->dst_port); + + if (dport == GTPU_PORT) + { + gtpu_header_t *gtpu_spec = + (gtpu_header_t *) &pst->generic.spec[pst->generic.off]; + gtpu_header_t *gtpu_mask = + (gtpu_header_t *) &pst->generic.mask[pst->generic.off]; + pst->items[pst->layer].spec = (void *) gtpu_spec; + pst->items[pst->layer].mask = (void *) gtpu_mask; + pst->items[pst->layer].size = sizeof (gtpu_header_t); + pst->items[pst->layer].type = ROC_NPC_ITEM_TYPE_GTPU; + pst->generic.off += sizeof (gtpu_header_t); + pst->layer++; + pst->nxt_proto = 0; + return 0; + } + else if (dport == VXLAN_PORT) + { + vxlan_header_t *vxlan_spec = + (vxlan_header_t *) &pst->generic.spec[pst->generic.off]; + vxlan_header_t *vxlan_mask = + (vxlan_header_t *) &pst->generic.spec[pst->generic.off]; + pst->items[pst->layer].spec = (void *) vxlan_spec; + pst->items[pst->layer].mask = (void *) vxlan_mask; + pst->items[pst->layer].size = sizeof (vxlan_header_t); + pst->items[pst->layer].type = ROC_NPC_ITEM_TYPE_VXLAN; + pst->generic.off += sizeof (vxlan_header_t); + pst->layer++; + pst->nxt_proto = 0; + goto parse_l2; + } + } + /* No supported Tunnel detected. */ + else + { + log_err (pst->port->dev, + "Partially parsed till offset %u, not able to parse further", + pst->generic.off); + return 0; + } +parse_l2: + if (oct_parse_l2 (pst)) + return -1; +parse_l3: + if (oct_parse_l3 (pst)) + return -1; + + return oct_parse_l4 (pst); +} + +static vnet_dev_rv_t +oct_flow_generic_pattern_parse (oct_flow_parse_state *pst) +{ + + if (oct_parse_l2 (pst)) + goto err; + + if (oct_parse_l3 (pst)) + goto err; + + if (oct_parse_l4 (pst)) + goto err; + + if (oct_parse_tunnel (pst)) + goto err; + + if (pst->generic.off < pst->generic.len) + { + log_err (pst->port->dev, + "Partially parsed till offset %u, not able to parse further", + pst->generic.off); + goto err; + } + + pst->items[pst->layer].type = ROC_NPC_ITEM_TYPE_END; + return VNET_DEV_OK; + +err: + return VNET_DEV_ERR_NOT_SUPPORTED; +} + static vnet_dev_rv_t oct_flow_add (vlib_main_t *vm, vnet_dev_port_t *port, vnet_flow_t *flow, uword *private_data) @@ -196,6 +536,7 @@ oct_flow_add (vlib_main_t *vm, vnet_dev_port_t *port, vnet_flow_t *flow, struct roc_npc_action_queue conf = {}; struct roc_npc_action_mark mark = {}; struct roc_npc *npc = &oct_port->npc; + u8 *flow_spec = 0, *flow_mask = 0; vnet_dev_rv_t rv = VNET_DEV_OK; int layer = 0, index = 0; u16 *queues = NULL; @@ -203,6 +544,45 @@ oct_flow_add (vlib_main_t *vm, vnet_dev_port_t *port, vnet_flow_t *flow, u8 proto = 0; u16 action = 0; + if (FLOW_IS_GENERIC_TYPE (flow)) + { + u8 drv_item_spec[1024] = { 0 }, drv_item_mask[1024] = { 0 }; + unformat_input_t input; + int rc; + + unformat_init_string ( + &input, (const char *) flow->generic.pattern.spec, + strlen ((const char *) flow->generic.pattern.spec)); + unformat_user (&input, unformat_hex_string, &flow_spec); + unformat_free (&input); + + unformat_init_string ( + &input, (const char *) flow->generic.pattern.mask, + strlen ((const char *) flow->generic.pattern.mask)); + unformat_user (&input, unformat_hex_string, &flow_mask); + unformat_free (&input); + + oct_flow_parse_state pst = { + .nxt_proto = 0, + .port = port, + .items = item_info, + .oct_drv = { .spec = drv_item_spec, .mask = drv_item_mask }, + .generic = { .spec = flow_spec, + .mask = flow_mask, + .len = vec_len (flow_spec) }, + }; + + rc = oct_flow_generic_pattern_parse (&pst); + if (rc) + { + vec_free (flow_spec); + vec_free (flow_mask); + return VNET_DEV_ERR_NOT_SUPPORTED; + } + + goto parse_flow_actions; + } + if (FLOW_IS_ETHERNET_CLASS (flow)) { ethernet_header_t eth_spec = { .type = clib_host_to_net_u16 ( @@ -357,6 +737,7 @@ oct_flow_add (vlib_main_t *vm, vnet_dev_port_t *port, vnet_flow_t *flow, end_item_info: item_info[layer].type = ROC_NPC_ITEM_TYPE_END; +parse_flow_actions: if (flow->actions & VNET_FLOW_ACTION_REDIRECT_TO_QUEUE) { conf.index = flow->redirect_queue; @@ -422,6 +803,11 @@ oct_flow_add (vlib_main_t *vm, vnet_dev_port_t *port, vnet_flow_t *flow, if (queues) clib_mem_free (queues); + if (flow_spec) + vec_free (flow_spec); + if (flow_mask) + vec_free (flow_mask); + return rv; } diff --git a/src/plugins/dev_octeon/octeon.h b/src/plugins/dev_octeon/octeon.h index e43cde0a35..cd17d5abc4 100644 --- a/src/plugins/dev_octeon/octeon.h +++ b/src/plugins/dev_octeon/octeon.h @@ -12,6 +12,12 @@ #include #include #include +#include +#include +#include +#include +#include +#include #include #include From 3c4b60a7c5a4f4739c2c7d91b3c6feb324afdbd9 Mon Sep 17 00:00:00 2001 From: Monendra Singh Kushwaha Date: Thu, 16 May 2024 12:24:37 +0530 Subject: [PATCH 033/271] dev: fix counter_start in counter clear routine This patch fixes counter_start value, as it should hold the cumulative counter value whenever counter clear routine is called. Type: fix Fixes: 38c619115b Change-Id: I50bf8ddcde419caf1170dfacdea03ff3d93a3327 Signed-off-by: Monendra Singh Kushwaha --- src/vnet/dev/counters.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/vnet/dev/counters.c b/src/vnet/dev/counters.c index 0a1e0a7419..d02839d664 100644 --- a/src/vnet/dev/counters.c +++ b/src/vnet/dev/counters.c @@ -54,7 +54,7 @@ vnet_dev_counters_clear (vlib_main_t *vm, vnet_dev_counter_main_t *cm) { for (int i = 0; i < cm->n_counters; i++) { - cm->counter_start[i] = cm->counter_data[i]; + cm->counter_start[i] += cm->counter_data[i]; cm->counter_data[i] = 0; } } From 1e8e0cd6936368bc9495d51c7b7c59e9ed0c3512 Mon Sep 17 00:00:00 2001 From: Monendra Singh Kushwaha Date: Thu, 16 May 2024 14:15:40 +0530 Subject: [PATCH 034/271] octeon: add counters support for port and queue Type: feature Change-Id: I5d52d78a93c7d0a12b9cee16fe7ebabdc2b19f0a Signed-off-by: Monendra Singh Kushwaha --- src/plugins/dev_octeon/CMakeLists.txt | 1 + src/plugins/dev_octeon/counter.c | 296 ++++++++++++++++++++++++++ src/plugins/dev_octeon/octeon.h | 8 + src/plugins/dev_octeon/port.c | 17 ++ 4 files changed, 322 insertions(+) create mode 100644 src/plugins/dev_octeon/counter.c diff --git a/src/plugins/dev_octeon/CMakeLists.txt b/src/plugins/dev_octeon/CMakeLists.txt index 0f6b32bbec..c6271ecdfb 100644 --- a/src/plugins/dev_octeon/CMakeLists.txt +++ b/src/plugins/dev_octeon/CMakeLists.txt @@ -35,6 +35,7 @@ add_vpp_plugin(dev_octeon rx_node.c tx_node.c flow.c + counter.c MULTIARCH_SOURCES rx_node.c diff --git a/src/plugins/dev_octeon/counter.c b/src/plugins/dev_octeon/counter.c new file mode 100644 index 0000000000..5763237d10 --- /dev/null +++ b/src/plugins/dev_octeon/counter.c @@ -0,0 +1,296 @@ +/* + * Copyright (c) 2024 Marvell. + * SPDX-License-Identifier: Apache-2.0 + * https://spdx.org/licenses/Apache-2.0.html + */ + +#include +#include +#include +#include +#include +#include + +VLIB_REGISTER_LOG_CLASS (oct_log, static) = { + .class_name = "oct", + .subclass_name = "counters", +}; + +typedef enum +{ + OCT_PORT_CTR_RX_BYTES, + OCT_PORT_CTR_TX_BYTES, + OCT_PORT_CTR_RX_PACKETS, + OCT_PORT_CTR_TX_PACKETS, + OCT_PORT_CTR_RX_DROPS, + OCT_PORT_CTR_TX_DROPS, + OCT_PORT_CTR_RX_DROP_BYTES, + OCT_PORT_CTR_RX_UCAST, + OCT_PORT_CTR_TX_UCAST, + OCT_PORT_CTR_RX_MCAST, + OCT_PORT_CTR_TX_MCAST, + OCT_PORT_CTR_RX_BCAST, + OCT_PORT_CTR_TX_BCAST, + OCT_PORT_CTR_RX_FCS, + OCT_PORT_CTR_RX_ERR, + OCT_PORT_CTR_RX_DROP_MCAST, + OCT_PORT_CTR_RX_DROP_BCAST, + OCT_PORT_CTR_RX_DROP_L3_MCAST, + OCT_PORT_CTR_RX_DROP_L3_BCAST, +} oct_port_counter_id_t; + +vnet_dev_counter_t oct_port_counters[] = { + VNET_DEV_CTR_RX_BYTES (OCT_PORT_CTR_RX_BYTES), + VNET_DEV_CTR_RX_PACKETS (OCT_PORT_CTR_RX_PACKETS), + VNET_DEV_CTR_RX_DROPS (OCT_PORT_CTR_RX_DROPS), + VNET_DEV_CTR_VENDOR (OCT_PORT_CTR_RX_DROP_BYTES, RX, BYTES, "drop bytes"), + VNET_DEV_CTR_VENDOR (OCT_PORT_CTR_RX_UCAST, RX, PACKETS, "unicast"), + VNET_DEV_CTR_VENDOR (OCT_PORT_CTR_RX_MCAST, RX, PACKETS, "multicast"), + VNET_DEV_CTR_VENDOR (OCT_PORT_CTR_RX_BCAST, RX, PACKETS, "broadcast"), + VNET_DEV_CTR_VENDOR (OCT_PORT_CTR_RX_FCS, RX, PACKETS, "fcs"), + VNET_DEV_CTR_VENDOR (OCT_PORT_CTR_RX_ERR, RX, PACKETS, "error"), + VNET_DEV_CTR_VENDOR (OCT_PORT_CTR_RX_DROP_MCAST, RX, PACKETS, + "drop multicast"), + VNET_DEV_CTR_VENDOR (OCT_PORT_CTR_RX_DROP_BCAST, RX, PACKETS, + "drop broadcast"), + VNET_DEV_CTR_VENDOR (OCT_PORT_CTR_RX_DROP_L3_MCAST, RX, PACKETS, + "drop L3 multicast"), + VNET_DEV_CTR_VENDOR (OCT_PORT_CTR_RX_DROP_L3_BCAST, RX, PACKETS, + "drop L3 broadcast"), + + VNET_DEV_CTR_TX_BYTES (OCT_PORT_CTR_TX_BYTES), + VNET_DEV_CTR_TX_PACKETS (OCT_PORT_CTR_TX_PACKETS), + VNET_DEV_CTR_TX_DROPS (OCT_PORT_CTR_TX_DROPS), + VNET_DEV_CTR_VENDOR (OCT_PORT_CTR_TX_UCAST, TX, PACKETS, "unicast"), + VNET_DEV_CTR_VENDOR (OCT_PORT_CTR_TX_MCAST, TX, PACKETS, "multicast"), + VNET_DEV_CTR_VENDOR (OCT_PORT_CTR_TX_BCAST, TX, PACKETS, "broadcast"), +}; + +typedef enum +{ + OCT_RXQ_CTR_BYTES, + OCT_RXQ_CTR_PKTS, + OCT_RXQ_CTR_DROPS, + OCT_RXQ_CTR_DROP_BYTES, + OCT_RXQ_CTR_ERR, +} oct_rxq_counter_id_t; + +vnet_dev_counter_t oct_rxq_counters[] = { + VNET_DEV_CTR_RX_BYTES (OCT_RXQ_CTR_BYTES), + VNET_DEV_CTR_RX_PACKETS (OCT_RXQ_CTR_PKTS), + VNET_DEV_CTR_RX_DROPS (OCT_RXQ_CTR_DROPS), + VNET_DEV_CTR_VENDOR (OCT_RXQ_CTR_DROP_BYTES, RX, BYTES, "drop bytes"), + VNET_DEV_CTR_VENDOR (OCT_RXQ_CTR_ERR, RX, PACKETS, "error"), +}; + +typedef enum +{ + OCT_TXQ_CTR_BYTES, + OCT_TXQ_CTR_PKTS, + OCT_TXQ_CTR_DROPS, + OCT_TXQ_CTR_DROP_BYTES, +} oct_txq_counter_id_t; + +vnet_dev_counter_t oct_txq_counters[] = { + VNET_DEV_CTR_TX_BYTES (OCT_TXQ_CTR_BYTES), + VNET_DEV_CTR_TX_PACKETS (OCT_TXQ_CTR_PKTS), + VNET_DEV_CTR_TX_DROPS (OCT_TXQ_CTR_DROPS), + VNET_DEV_CTR_VENDOR (OCT_TXQ_CTR_DROP_BYTES, TX, BYTES, "drop bytes"), +}; + +static vnet_dev_rv_t +oct_roc_err (vnet_dev_t *dev, int rv, char *fmt, ...) +{ + u8 *s = 0; + va_list va; + + va_start (va, fmt); + s = va_format (s, fmt, &va); + va_end (va); + + log_err (dev, "%v - ROC error %s (%d)", s, roc_error_msg_get (rv), rv); + + vec_free (s); + return VNET_DEV_ERR_INTERNAL; +} + +void +oct_port_add_counters (vlib_main_t *vm, vnet_dev_port_t *port) +{ + vnet_dev_port_add_counters (vm, port, oct_port_counters, + ARRAY_LEN (oct_port_counters)); + + foreach_vnet_dev_port_rx_queue (rxq, port) + { + vnet_dev_rx_queue_add_counters (vm, rxq, oct_rxq_counters, + ARRAY_LEN (oct_rxq_counters)); + } + + foreach_vnet_dev_port_tx_queue (txq, port) + { + vnet_dev_tx_queue_add_counters (vm, txq, oct_txq_counters, + ARRAY_LEN (oct_txq_counters)); + } +} + +vnet_dev_rv_t +oct_port_get_stats (vlib_main_t *vm, vnet_dev_port_t *port) +{ + vnet_dev_t *dev = port->dev; + oct_device_t *cd = vnet_dev_get_data (dev); + struct roc_nix *nix = cd->nix; + int rrv; + struct roc_nix_stats stats; + + if ((rrv = roc_nix_stats_get (nix, &stats))) + return oct_roc_err (dev, rrv, "roc_nix_stats_get() failed"); + + foreach_vnet_dev_counter (c, port->counter_main) + { + switch (c->user_data) + { + case OCT_PORT_CTR_RX_BYTES: + vnet_dev_counter_value_update (vm, c, stats.rx_octs); + break; + case OCT_PORT_CTR_TX_BYTES: + vnet_dev_counter_value_update (vm, c, stats.tx_octs); + break; + case OCT_PORT_CTR_RX_PACKETS: + vnet_dev_counter_value_update ( + vm, c, stats.rx_ucast + stats.rx_bcast + stats.rx_mcast); + break; + case OCT_PORT_CTR_TX_PACKETS: + vnet_dev_counter_value_update ( + vm, c, stats.tx_ucast + stats.tx_bcast + stats.tx_mcast); + break; + case OCT_PORT_CTR_RX_DROPS: + vnet_dev_counter_value_update (vm, c, stats.rx_drop); + break; + case OCT_PORT_CTR_TX_DROPS: + vnet_dev_counter_value_update (vm, c, stats.tx_drop); + break; + case OCT_PORT_CTR_RX_DROP_BYTES: + vnet_dev_counter_value_update (vm, c, stats.rx_drop_octs); + break; + case OCT_PORT_CTR_RX_UCAST: + vnet_dev_counter_value_update (vm, c, stats.rx_ucast); + break; + case OCT_PORT_CTR_TX_UCAST: + vnet_dev_counter_value_update (vm, c, stats.tx_ucast); + break; + case OCT_PORT_CTR_RX_MCAST: + vnet_dev_counter_value_update (vm, c, stats.rx_mcast); + break; + case OCT_PORT_CTR_TX_MCAST: + vnet_dev_counter_value_update (vm, c, stats.tx_mcast); + break; + case OCT_PORT_CTR_RX_BCAST: + vnet_dev_counter_value_update (vm, c, stats.rx_bcast); + break; + case OCT_PORT_CTR_TX_BCAST: + vnet_dev_counter_value_update (vm, c, stats.tx_bcast); + break; + case OCT_PORT_CTR_RX_FCS: + vnet_dev_counter_value_update (vm, c, stats.rx_fcs); + break; + case OCT_PORT_CTR_RX_ERR: + vnet_dev_counter_value_update (vm, c, stats.rx_err); + break; + case OCT_PORT_CTR_RX_DROP_MCAST: + vnet_dev_counter_value_update (vm, c, stats.rx_drop_mcast); + break; + case OCT_PORT_CTR_RX_DROP_BCAST: + vnet_dev_counter_value_update (vm, c, stats.rx_drop_bcast); + break; + case OCT_PORT_CTR_RX_DROP_L3_MCAST: + vnet_dev_counter_value_update (vm, c, stats.rx_drop_l3_mcast); + break; + case OCT_PORT_CTR_RX_DROP_L3_BCAST: + vnet_dev_counter_value_update (vm, c, stats.rx_drop_l3_bcast); + break; + default: + ASSERT (0); + } + } + + return VNET_DEV_OK; +} + +vnet_dev_rv_t +oct_rxq_get_stats (vlib_main_t *vm, vnet_dev_port_t *port, + vnet_dev_rx_queue_t *rxq) +{ + oct_rxq_t *crq = vnet_dev_get_rx_queue_data (rxq); + struct roc_nix_stats_queue qstats; + vnet_dev_t *dev = port->dev; + oct_device_t *cd = vnet_dev_get_data (dev); + struct roc_nix *nix = cd->nix; + int rrv; + + if ((rrv = roc_nix_stats_queue_get (nix, crq->rq.qid, 1, &qstats))) + return oct_roc_err (dev, rrv, "roc_nix_stats_queue_get() failed"); + + foreach_vnet_dev_counter (c, rxq->counter_main) + { + switch (c->user_data) + { + case OCT_RXQ_CTR_BYTES: + vnet_dev_counter_value_update (vm, c, qstats.rx_octs); + break; + case OCT_RXQ_CTR_PKTS: + vnet_dev_counter_value_update (vm, c, qstats.rx_pkts); + break; + case OCT_RXQ_CTR_DROPS: + vnet_dev_counter_value_update (vm, c, qstats.rx_drop_pkts); + break; + case OCT_RXQ_CTR_DROP_BYTES: + vnet_dev_counter_value_update (vm, c, qstats.rx_drop_octs); + break; + case OCT_RXQ_CTR_ERR: + vnet_dev_counter_value_update (vm, c, qstats.rx_error_pkts); + break; + default: + ASSERT (0); + } + } + + return VNET_DEV_OK; +} + +vnet_dev_rv_t +oct_txq_get_stats (vlib_main_t *vm, vnet_dev_port_t *port, + vnet_dev_tx_queue_t *txq) +{ + oct_txq_t *ctq = vnet_dev_get_tx_queue_data (txq); + struct roc_nix_stats_queue qstats; + vnet_dev_t *dev = port->dev; + oct_device_t *cd = vnet_dev_get_data (dev); + struct roc_nix *nix = cd->nix; + int rrv; + + if ((rrv = roc_nix_stats_queue_get (nix, ctq->sq.qid, 0, &qstats))) + return oct_roc_err (dev, rrv, "roc_nix_stats_queue_get() failed"); + + foreach_vnet_dev_counter (c, txq->counter_main) + { + switch (c->user_data) + { + case OCT_TXQ_CTR_BYTES: + vnet_dev_counter_value_update (vm, c, qstats.tx_octs); + break; + case OCT_TXQ_CTR_PKTS: + vnet_dev_counter_value_update (vm, c, qstats.tx_pkts); + break; + case OCT_TXQ_CTR_DROPS: + vnet_dev_counter_value_update (vm, c, qstats.tx_drop_pkts); + break; + case OCT_TXQ_CTR_DROP_BYTES: + vnet_dev_counter_value_update (vm, c, qstats.tx_drop_octs); + break; + default: + ASSERT (0); + } + } + + return VNET_DEV_OK; +} diff --git a/src/plugins/dev_octeon/octeon.h b/src/plugins/dev_octeon/octeon.h index cd17d5abc4..4926f1afe7 100644 --- a/src/plugins/dev_octeon/octeon.h +++ b/src/plugins/dev_octeon/octeon.h @@ -147,6 +147,14 @@ vnet_dev_rv_t oct_flow_validate_params (vlib_main_t *, vnet_dev_port_t *, vnet_dev_rv_t oct_flow_query (vlib_main_t *, vnet_dev_port_t *, u32, uword, u64 *); +/* counter.c */ +void oct_port_add_counters (vlib_main_t *, vnet_dev_port_t *); +vnet_dev_rv_t oct_port_get_stats (vlib_main_t *, vnet_dev_port_t *); +vnet_dev_rv_t oct_rxq_get_stats (vlib_main_t *, vnet_dev_port_t *, + vnet_dev_rx_queue_t *); +vnet_dev_rv_t oct_txq_get_stats (vlib_main_t *, vnet_dev_port_t *, + vnet_dev_tx_queue_t *); + #define log_debug(dev, f, ...) \ vlib_log (VLIB_LOG_LEVEL_DEBUG, oct_log.class, "%U: " f, \ format_vnet_dev_addr, (dev), ##__VA_ARGS__) diff --git a/src/plugins/dev_octeon/port.c b/src/plugins/dev_octeon/port.c index 98a4c28b37..bfda6f632b 100644 --- a/src/plugins/dev_octeon/port.c +++ b/src/plugins/dev_octeon/port.c @@ -124,6 +124,8 @@ oct_port_init (vlib_main_t *vm, vnet_dev_port_t *port) return rv; } + oct_port_add_counters (vm, port); + return VNET_DEV_OK; } @@ -172,6 +174,21 @@ oct_port_poll (vlib_main_t *vm, vnet_dev_port_t *port) vnet_dev_port_state_changes_t changes = {}; int rrv; + if (oct_port_get_stats (vm, port)) + return; + + foreach_vnet_dev_port_rx_queue (q, port) + { + if (oct_rxq_get_stats (vm, port, q)) + return; + } + + foreach_vnet_dev_port_tx_queue (q, port) + { + if (oct_txq_get_stats (vm, port, q)) + return; + } + if (roc_nix_is_lbk (nix)) { link_info.status = 1; From a5f1219ad022c43f3f63220f18fbf1d865eb4d68 Mon Sep 17 00:00:00 2001 From: Pavan Nikhilesh Date: Tue, 28 May 2024 21:29:45 +0530 Subject: [PATCH 035/271] ci: add github workflow Signed-off-by: Pavan Nikhilesh --- .github/workflows/build.yml | 181 ++++++++++++++++++++++++++++++++++++ DPDK_VERSION | 2 + MRVL_VERSION | 1 + 3 files changed, 184 insertions(+) create mode 100644 .github/workflows/build.yml create mode 100644 DPDK_VERSION create mode 100644 MRVL_VERSION diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000000..a82b954802 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,181 @@ +name: build-cn10k + +on: + push: + schedule: + - cron: "0 0 * * *" + pull_request: + +permissions: + contents: write + pages: write + id-token: write + packages: write + +jobs: + ubuntu-cn10k-build: + name: ubuntu-cn10k-arm64 + runs-on: ubuntu-latest + + strategy: + fail-fast: true + matrix: + include: + - arch: aarch64 + distro: ubuntu22.04 + compiler: gcc + library: static + + steps: + - name: Checkout sources + uses: actions/checkout@v4.1.6 + with: + fetch-depth: 0 + fetch-tags: true + - name: Generate cache keys + id: get_ref_keys + run: | + echo 'ccache=ccache-${{ matrix.distro }}-${{ matrix.compiler }}-${{ matrix.arch }}-'$(date -u +%Y-w%W) >> $GITHUB_OUTPUT + - name: Retrieve ccache cache + uses: actions/cache@v4 + with: + path: ~/.ccache + key: ${{ steps.get_ref_keys.outputs.ccache }}-${{ github.ref }} + restore-keys: | + ${{ steps.get_ref_keys.outputs.ccache }}-refs/heads/main + - name: Extract version details + id: version + run: | + mkdir -p "${PWD}/artifacts" + git tag --points-at HEAD > /tmp/tags + [ -s /tmp/tags ] && PKG_POSTFIX= || PKG_POSTFIX=-devel + [ -s /tmp/tags ] && NIGHTLY=false || NIGHTLY=true + echo "PKG_VERSION_NAME=`./src/scripts/version | awk -F '-' '{print $1}'`" >> "${PWD}/artifacts/env" + echo "MRVL_PKG_VERSION=`cat MRVL_VERSION`" >> "${PWD}/artifacts/env" + echo "DPDK_PKG_VERSION=`cat DPDK_VERSION | grep RELEASE_VERSION | awk -F'=' '{print $2}'`" >> "${PWD}/artifacts/env" + echo "DPDK_BASE_PKG_VERSION=`cat DPDK_VERSION | grep BASE_VERSION | awk -F'=' '{print $2}' | awk -F'.' '{print $1"."$2}'`" >> "${PWD}/artifacts/env" + echo "PKG_POSTFIX=${PKG_POSTFIX}" >> "${PWD}/artifacts/env" + source "${PWD}/artifacts/env" + echo "NIGHTLY=${NIGHTLY}" >> $GITHUB_OUTPUT + echo "PKG_VERSION_NAME=${PKG_VERSION_NAME}" >> $GITHUB_OUTPUT + echo "MRVL_PKG_VERSION=${MRVL_PKG_VERSION}" >> $GITHUB_OUTPUT + echo "DPDK_PKG_VERSION=${DPDK_PKG_VERSION}" >> $GITHUB_OUTPUT + - uses: uraimo/run-on-arch-action@v2.7.2 + name: Build VPP and generate package + id: build + with: + arch: ${{ matrix.arch }} + distro: ${{ matrix.distro }} + githubToken: ${{ github.token }} + setup: | + mkdir -p ~/.ccache + dockerRunArgs: | + --volume "${PWD}/artifacts:/artifacts" + --volume "${HOME}/.ccache:/root/.ccache" + shell: /bin/bash + install: | + apt-get update -q -y + apt-get install -y apt-utils sudo make dialog ccache git + apt-get install -y aspell aspell-en autopoint autotools-dev binfmt-support binutils binutils-aarch64-linux-gnu + apt-get install -y binutils-common bsdextrautils bzip2 ca-certificates clang-14 cmake-data cpp cpp-11 cpp-12 dbus + apt-get install -y dctrl-tools debugedit dh-autoreconf dh-elpa-helper dh-strip-nondeterminism dictionaries-common dirmngr + apt-get install -y distro-info-data dpkg-dev dwz emacsen-common fakeroot file fontconfig-config fonts-dejavu-core g++ g++-11 + apt-get install -y gcc gcc-11 gcc-11-base gcc-12 gettext gettext-base git-man gnupg gnupg-l10n gnupg-utils gpg gpg-agent + apt-get install -y gpg-wks-client gpg-wks-server gpgconf gpgsm groff-base hunspell-en-us icu-devtools intltool-debian + apt-get install -y javascript-common kmod less libalgorithm-diff-perl libalgorithm-diff-xs-perl libalgorithm-merge-perl + apt-get install -y libapparmor1 libapr1 libarchive-cpio-perl libarchive-zip-perl libarchive13 libasan6 libasan8 libaspell15 + apt-get install -y libassuan0 libatomic1 libbcg729-0 libbinutils libbrotli1 libbsd0 libc-ares2 libc-dev-bin libc-devtools + apt-get install -y libc6-dev libcap2-bin libcbor0.8 libcc1-0 libclang-common-14-dev libclang-cpp11 libclang-cpp14 + apt-get install -y libclang1-14 libcommon-sense-perl libconfuse-common libconfuse-doc libconfuse2 libcrypt-dev libctf-nobfd0 + apt-get install -y libctf0 libcurl3-gnutls libcurl4 libdbus-1-3 libdbus-1-dev libdebhelper-perl libdeflate0 libdpkg-perl + apt-get install -y libdw1 libedit2 libelf1 libenchant-2-2 liberror-perl libexpat1 libexpat1-dev libfakeroot libfido2-1 + apt-get install -y libfile-fcntllock-perl libfile-stripnondeterminism-perl libfontconfig1 libfreetype6 libgc1 libgcc-11-dev + apt-get install -y libgcc-12-dev libgd-perl libgd3 libgdbm-compat4 libgdbm6 libglib2.0-0 libglib2.0-data libgomp1 + apt-get install -y libhiredis0.14 libhunspell-1.7-0 libhwasan0 libicu-dev libicu70 libiperf0 libisl23 libitm1 libjbig0 + apt-get install -y libjpeg-turbo8 libjpeg8 libjs-jquery libjs-sphinxdoc libjs-underscore libjson-perl libjson-xs-perl + apt-get install -y libjsoncpp25 libkmod2 libksba8 libldap-2.5-0 libldap-common libllvm11 libllvm14 liblocale-gettext-perl + apt-get install -y liblsan0 libltdl-dev libltdl7 liblua5.2-0 libmagic-mgc libmagic1 libmail-sendmail-perl libmaxminddb0 + apt-get install -y libmd0 libmnl0 libmpc3 libmpdec3 libmpfr6 libncurses-dev libnghttp2-14 libnl-3-200 libnl-genl-3-200 + apt-get install -y libnl-route-3-200 libnpth0 libnsl-dev libnuma1 libobjc-11-dev libobjc4 libpam-cap libpcap0.8 + apt-get install -y libpcap0.8-dev libperl5.34 libperlio-gzip-perl libpfm4 libpipeline1 libpng16-16 libpsl5 libpython3-dev + apt-get install -y libpython3-stdlib libpython3.10 libpython3.10-dev libpython3.10-minimal libpython3.10-stdlib libreadline8 + apt-get install -y librhash0 librtmp1 libsasl2-2 libsasl2-modules libsasl2-modules-db libsbc1 libsctp-dev libsctp1 + apt-get install -y libsigsegv2 libsmi2ldbl libsnappy1v5 libspandsp2 libspeexdsp1 libsqlite3-0 libssh-4 libssh-gcrypt-4 + apt-get install -y libstdc++-11-dev libsub-override-perl libsubunit-dev libsubunit0 libsys-hostname-long-perl + apt-get install -y libtext-iconv-perl libtiff5 libtinfo-dev libtirpc-dev libtsan0 libtsan2 libtypes-serialiser-perl + apt-get install -y libubsan1 libuchardet0 libuv1 libwebp7 libwireshark-data libwireshark15 libwiretap12 libwsutil13 libx11-6 + apt-get install -y libx11-data libxau6 libxcb1 libxdmcp6 libxext6 libxml2 libxml2-dev libxmuu1 libxpm4 libxslt1.1 + apt-get install -y libyaml-0-2 libz3-4 libz3-dev linux-headers-5.15.0-107 linux-headers-5.15.0-107-generic + apt-get install -y linux-headers-generic linux-libc-dev llvm-14 llvm-14-dev llvm-14-linker-tools llvm-14-runtime + apt-get install -y llvm-14-tools lsb-release lto-disabled-list m4 man-db manpages manpages-dev media-types netbase + apt-get install -y openssh-client openssl patch perl perl-modules-5.34 pinentry-curses po-debconf publicsuffix + apt-get install -y python-babel-localedata python3 python3-attr python3-babel python3-bs4 python3-certifi python3-chardet + apt-get install -y python3-distlib python3-distutils python3-filelock python3-html5lib python3-idna + apt-get install -y python3-importlib-metadata python3-jinja2 python3-lib2to3 python3-lxml python3-markupsafe python3-minimal + apt-get install -y python3-more-itertools python3-pip-whl python3-pkg-resources python3-platformdirs python3-pygments + apt-get install -y python3-pyrsistent python3-requests python3-setuptools-whl python3-six python3-soupsieve python3-tz + apt-get install -y python3-urllib3 python3-webencodings python3-wheel python3-wheel-whl python3-zipp python3.10 + apt-get install -y python3.10-dev python3.10-minimal python3.10-venv readline-common rpcsvc-proto shared-mime-info tzdata + apt-get install -y ucf uuid-dev wireshark-common xauth xdg-user-dirs xz-utils zlib1g-dev + apt-get install -y check chrpath clang clang-format-11 cmake cscope debhelper dh-python dkms + apt-get install -y ethtool exuberant-ctags gcovr git-review iperf iperf3 lcov libapr1-dev + apt-get install -y libconfuse-dev libelf-dev libffi7 libmnl-dev libnl-3-dev libnl-route-3-dev + apt-get install -y libnuma-dev libpcap-dev libssl-dev nasm ninja-build python3-all python3-dev + apt-get install -y python3-jsonschema python3-pip python3-ply python3-setuptools python3-venv + apt-get install -y python3-virtualenv tshark + run: | + source /artifacts/env + echo "cache_dir = /root/.ccache" > /root/.ccache/ccache.conf + ccache -p + git config --global --add safe.directory "${PWD}" + APT_ARGS='-y -q' make install-deps + make build-release VPP_PLATFORM=octeon10 + mkdir -p "${PWD}/install/DEBIAN" + mv build-root/install-vpp-native/vpp/* install/. + cd "${PWD}/install" + echo 'Package: vpp-'$PKG_VERSION_NAME'-cn10k'$PKG_POSTFIX >> DEBIAN/control + echo 'Version: '$MRVL_PKG_VERSION >> DEBIAN/control + echo "Maintainer: Jerin Jacob (jerinj@marvell.com)" >> DEBIAN/control + echo 'Depends: python3, python3-ply, dpdk-'$DPDK_BASE_PKG_VERSION'-cn10k (= '$DPDK_PKG_VERSION')' >> DEBIAN/control + echo "Architecture: arm64" >> DEBIAN/control + echo "Homepage: https://wiki.fd.io/view/VPP" >> DEBIAN/control + echo "Description: Vector Packet Processing (VPP) for Octeon10" >> DEBIAN/control + cd - + mv "${PWD}/install" "${PWD}/vpp-${PKG_VERSION_NAME}-cn10k${PKG_POSTFIX}_${MRVL_PKG_VERSION}_arm64" + dpkg --build "vpp-${PKG_VERSION_NAME}-cn10k${PKG_POSTFIX}_${MRVL_PKG_VERSION}_arm64" + cp "vpp-${PKG_VERSION_NAME}-cn10k${PKG_POSTFIX}_${MRVL_PKG_VERSION}_arm64.deb" /artifacts/. + - name: Export version name + id: artifacts + run: | + source "${PWD}/artifacts/env" + echo $PKG_VERSION_NAME + echo "PKG_VERSION_NAME=${PKG_VERSION_NAME}" >> "$GITHUB_OUTPUT" + echo $MRVL_PKG_VERSION + echo "MRVL_PKG_VERSION=${MRVL_PKG_VERSION}" >> "$GITHUB_OUTPUT" + echo $PKG_POSTFIX + echo "PKG_POSTFIX=${PKG_POSTFIX}" >> "$GITHUB_OUTPUT" + [[ "$PKG_POSTFIX" == "-devel" ]] && TAG=devel || TAG=${PKG_VERSION_NAME} + echo "TAG=${TAG}" >> "$GITHUB_OUTPUT" + - name: Upload debian package as artifact + uses: actions/upload-artifact@v4.3.1 + if: ${{ github.event_name == 'push' }} + with: + name: vpp-${{ steps.artifacts.outputs.PKG_VERSION_NAME }}-cn10k${{ steps.artifacts.outputs.PKG_POSTFIX }}_${{ steps.artifacts.outputs.MRVL_PKG_VERSION }}_arm64.deb + path: ${{ github.workspace }}/artifacts/vpp-${{ steps.artifacts.outputs.PKG_VERSION_NAME }}-cn10k${{ steps.artifacts.outputs.PKG_POSTFIX }}_${{ steps.artifacts.outputs.MRVL_PKG_VERSION }}_arm64.deb + - name: Release VPP cn10k package + uses: softprops/action-gh-release@v2.0.4 + if: ${{ github.event_name == 'push' }} + with: + tag_name: ${{ steps.artifacts.outputs.TAG }} + files: | + ${{ github.workspace }}/artifacts/vpp-${{ steps.artifacts.outputs.PKG_VERSION_NAME }}-cn10k${{ steps.artifacts.outputs.PKG_POSTFIX }}_${{ steps.artifacts.outputs.MRVL_PKG_VERSION }}_arm64.deb + - name: Dispatch package update event + if: ${{ github.event_name == 'push' }} + run: | + curl -L \ + -X POST \ + -H "Accept: application/vnd.github+json" \ + -H "Authorization: Bearer ${{ secrets.PPA_REPO_SECRET }}" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + https://api.github.com/repos/marvellembeddedprocessors/packages/dispatches \ + -d '{"event_type":"dispatch-event", "client_payload": {"package" : "vpp", "tag": "${{ steps.artifacts.outputs.TAG }}", "dpdk_tag" : "${{ steps.version.outputs.DPDK_PKG_VERSION }}", "has_dpdk" : "true"}}' diff --git a/DPDK_VERSION b/DPDK_VERSION new file mode 100644 index 0000000000..488ae33498 --- /dev/null +++ b/DPDK_VERSION @@ -0,0 +1,2 @@ +BASE_VERSION=23.11.0 +RELEASE_VERSION=24.05.0 diff --git a/MRVL_VERSION b/MRVL_VERSION new file mode 100644 index 0000000000..aa5d9023dd --- /dev/null +++ b/MRVL_VERSION @@ -0,0 +1 @@ +24.05.0 From 03f6b006b6d3c7d88447695797ab0cd7d8cbca1b Mon Sep 17 00:00:00 2001 From: Monendra Singh Kushwaha Date: Tue, 21 May 2024 03:18:26 +0530 Subject: [PATCH 036/271] dev: add port counter clear operation Type: feature Change-Id: Ibd876c5251fc2f9d87816d235fff2de22be4b21c Signed-off-by: Monendra Singh Kushwaha Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/128474 Tested-by: sa_ip-toolkits-Jenkins Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/128637 Reviewed-by: Nithinsen Kaithakadan --- src/vnet/dev/dev.h | 1 + src/vnet/dev/port.c | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/src/vnet/dev/dev.h b/src/vnet/dev/dev.h index bbf2f9dff2..07a99bd132 100644 --- a/src/vnet/dev/dev.h +++ b/src/vnet/dev/dev.h @@ -245,6 +245,7 @@ typedef struct vnet_dev_port_op_no_rv_t *stop; vnet_dev_port_op_no_rv_t *deinit; vnet_dev_port_op_no_rv_t *free; + vnet_dev_port_op_no_rv_t *clear_counters; format_function_t *format_status; format_function_t *format_flow; } vnet_dev_port_ops_t; diff --git a/src/vnet/dev/port.c b/src/vnet/dev/port.c index 8a6df54cbc..7450084d14 100644 --- a/src/vnet/dev/port.c +++ b/src/vnet/dev/port.c @@ -733,6 +733,12 @@ vnet_dev_port_if_remove (vlib_main_t *vm, vnet_dev_port_t *port) void vnet_dev_port_clear_counters (vlib_main_t *vm, vnet_dev_port_t *port) { + if (port->port_ops.clear_counters) + { + port->port_ops.clear_counters (vm, port); + return; + } + if (port->counter_main) vnet_dev_counters_clear (vm, port->counter_main); From 39efbd78e0f0efe0730c91ffad2b51495af1b449 Mon Sep 17 00:00:00 2001 From: Monendra Singh Kushwaha Date: Tue, 21 May 2024 03:26:49 +0530 Subject: [PATCH 037/271] octeon: add clear counters support for port Type: feature Change-Id: Ie36be41694e7bd5341b4239dcba2ae6834c4a73f Signed-off-by: Monendra Singh Kushwaha Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/128475 Tested-by: sa_ip-toolkits-Jenkins Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/128638 Tested-by: sa_ip-sw-jenkins Reviewed-by: Nithinsen Kaithakadan --- src/plugins/dev_octeon/counter.c | 34 ++++++++++++++++++++++++++++++++ src/plugins/dev_octeon/init.c | 1 + src/plugins/dev_octeon/octeon.h | 1 + 3 files changed, 36 insertions(+) diff --git a/src/plugins/dev_octeon/counter.c b/src/plugins/dev_octeon/counter.c index 5763237d10..16fc9477a1 100644 --- a/src/plugins/dev_octeon/counter.c +++ b/src/plugins/dev_octeon/counter.c @@ -294,3 +294,37 @@ oct_txq_get_stats (vlib_main_t *vm, vnet_dev_port_t *port, return VNET_DEV_OK; } + +void +oct_port_clear_counters (vlib_main_t *vm, vnet_dev_port_t *port) +{ + vnet_dev_t *dev = port->dev; + oct_device_t *cd = vnet_dev_get_data (dev); + struct roc_nix *nix = cd->nix; + oct_rxq_t *crq; + oct_txq_t *ctq; + int rrv; + + if ((rrv = roc_nix_stats_reset (nix))) + oct_roc_err (dev, rrv, "roc_nix_stats_reset() failed"); + + foreach_vnet_dev_port_rx_queue (rxq, port) + { + crq = vnet_dev_get_rx_queue_data (rxq); + + if ((rrv = roc_nix_stats_queue_reset (nix, crq->rq.qid, 1))) + oct_roc_err (dev, rrv, + "roc_nix_stats_queue_reset() failed for rx queue %u", + rxq->queue_id); + } + + foreach_vnet_dev_port_tx_queue (txq, port) + { + ctq = vnet_dev_get_tx_queue_data (txq); + + if ((rrv = roc_nix_stats_queue_reset (nix, ctq->sq.qid, 0))) + oct_roc_err (dev, rrv, + "roc_nix_stats_queue_reset() failed for tx queue %u", + txq->queue_id); + } +} diff --git a/src/plugins/dev_octeon/init.c b/src/plugins/dev_octeon/init.c index 97a11e0d0d..2f0f4d3468 100644 --- a/src/plugins/dev_octeon/init.c +++ b/src/plugins/dev_octeon/init.c @@ -141,6 +141,7 @@ oct_init_nix (vlib_main_t *vm, vnet_dev_t *dev) .config_change_validate = oct_port_cfg_change_validate, .format_status = format_oct_port_status, .format_flow = format_oct_port_flow, + .clear_counters = oct_port_clear_counters, }, .data_size = sizeof (oct_port_t), .initial_data = &oct_port, diff --git a/src/plugins/dev_octeon/octeon.h b/src/plugins/dev_octeon/octeon.h index 4926f1afe7..52c8fa03f5 100644 --- a/src/plugins/dev_octeon/octeon.h +++ b/src/plugins/dev_octeon/octeon.h @@ -149,6 +149,7 @@ vnet_dev_rv_t oct_flow_query (vlib_main_t *, vnet_dev_port_t *, u32, uword, /* counter.c */ void oct_port_add_counters (vlib_main_t *, vnet_dev_port_t *); +void oct_port_clear_counters (vlib_main_t *, vnet_dev_port_t *); vnet_dev_rv_t oct_port_get_stats (vlib_main_t *, vnet_dev_port_t *); vnet_dev_rv_t oct_rxq_get_stats (vlib_main_t *, vnet_dev_port_t *, vnet_dev_rx_queue_t *); From 8f2b0cea579911bee7f21367428bbb59e31bc3d5 Mon Sep 17 00:00:00 2001 From: Monendra Singh Kushwaha Date: Wed, 22 May 2024 14:17:21 +0530 Subject: [PATCH 038/271] octeon: update trace for flow redirection This patch updates trace to use "qid" and "PKT_RX_FDIR" keywords. These keywords are used by CSIT flow related test cases to verify redirect to queue action and mark action. Type: feature Change-Id: Ib0f41a8a1a93cbbbf8c59304924d4e68efff48fd Signed-off-by: Monendra Singh Kushwaha Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/128476 Tested-by: sa_ip-toolkits-Jenkins Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/128640 Reviewed-by: Nithinsen Kaithakadan --- src/plugins/dev_octeon/format.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/plugins/dev_octeon/format.c b/src/plugins/dev_octeon/format.c index e624b84f54..dac512b8dd 100644 --- a/src/plugins/dev_octeon/format.c +++ b/src/plugins/dev_octeon/format.c @@ -25,7 +25,7 @@ format_oct_nix_rx_cqe_desc (u8 *s, va_list *args) typeof (d->sg0) *sg0 = &d->sg0; typeof (d->sg0) *sg1 = &d->sg1; - s = format (s, "hdr: cqe_type %u nude %u q %u tag 0x%x", h->cqe_type, + s = format (s, "hdr: cqe_type %u nude %u qid %u tag 0x%x", h->cqe_type, h->node, h->q, h->tag); s = format (s, "\n%Uparse:", format_white_space, indent); #define _(n, f) s = format (s, " " #n " " f, p->n) @@ -41,6 +41,8 @@ format_oct_nix_rx_cqe_desc (u8 *s, va_list *args) _ (flow_key_alg, "%u"); _ (eoh_ptr, "%u"); _ (match_id, "0x%x"); + if (p->match_id) + s = format (s, " PKT_RX_FDIR 0x%x", p->match_id); s = format (s, "\n%U ", format_white_space, indent); _ (wqe_aura, "0x%x"); _ (pb_aura, "0x%x"); From 82e404fc1cb771a01caedc0b99d9791f18c78039 Mon Sep 17 00:00:00 2001 From: Monendra Singh Kushwaha Date: Thu, 23 May 2024 16:16:42 +0530 Subject: [PATCH 039/271] octeon: fix lbk vf initialization Type: fix Fixes: 0a2fdc56 Change-Id: I5d232a86be66edeec8b740a883104f5a22516697 Signed-off-by: Monendra Singh Kushwaha Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/128477 Tested-by: sa_ip-toolkits-Jenkins Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/128642 Tested-by: sa_ip-sw-jenkins Reviewed-by: Nithinsen Kaithakadan --- src/plugins/dev_octeon/init.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/plugins/dev_octeon/init.c b/src/plugins/dev_octeon/init.c index 2f0f4d3468..7aed543f61 100644 --- a/src/plugins/dev_octeon/init.c +++ b/src/plugins/dev_octeon/init.c @@ -246,6 +246,7 @@ oct_init (vlib_main_t *vm, vnet_dev_t *dev) { case OCT_DEVICE_TYPE_RVU_PF: case OCT_DEVICE_TYPE_RVU_VF: + case OCT_DEVICE_TYPE_LBK_VF: case OCT_DEVICE_TYPE_SDP_VF: return oct_init_nix (vm, dev); From 576f99e0cd1148a37a9be27afe30ae9e409e8ef4 Mon Sep 17 00:00:00 2001 From: Alok Mishra Date: Fri, 26 Apr 2024 15:45:49 +0530 Subject: [PATCH 040/271] octeon: add support for max_rx_frame_size update This patch adds capability to update max_rx_frame_size on octeon port. Initial MTU value is being set in the "oct_port_start", which is invoked every time the Ethernet interface is brought up, thus overwriting any MTU value set by VPP CLI. Moved the MTU initialization to "oct_port_init" to address this. Type: feature Change-Id: I00d0d52bc7711062cde47b8fe52e6823bb718d08 Signed-off-by: Alok Mishra Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/128478 Tested-by: sa_ip-toolkits-Jenkins Reviewed-by: Monendra Singh Kushwaha Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/128643 Reviewed-by: Nithinsen Kaithakadan --- src/plugins/dev_octeon/octeon.h | 7 ++++ src/plugins/dev_octeon/port.c | 71 ++++++++++++++++++++++++++++----- 2 files changed, 69 insertions(+), 9 deletions(-) diff --git a/src/plugins/dev_octeon/octeon.h b/src/plugins/dev_octeon/octeon.h index 52c8fa03f5..b5d48c7ffa 100644 --- a/src/plugins/dev_octeon/octeon.h +++ b/src/plugins/dev_octeon/octeon.h @@ -23,6 +23,13 @@ #define OCT_BATCH_ALLOC_IOVA0_MASK 0xFFFFFFFFFFFFFF80 +/* + * L2 header size includes + * ETH_HDR_LEN + ETH_CRC_LEN + * 2 VLAN TAGS + */ +#define OCT_PKTIO_MAX_L2_SIZE 26 + typedef enum { OCT_DEVICE_TYPE_UNKNOWN = 0, diff --git a/src/plugins/dev_octeon/port.c b/src/plugins/dev_octeon/port.c index bfda6f632b..ad3720e3fc 100644 --- a/src/plugins/dev_octeon/port.c +++ b/src/plugins/dev_octeon/port.c @@ -124,6 +124,12 @@ oct_port_init (vlib_main_t *vm, vnet_dev_port_t *port) return rv; } + if ((rrv = roc_nix_mac_mtu_set (nix, 9200))) + { + rv = oct_roc_err (dev, rrv, "roc_nix_mac_mtu_set() failed"); + return rv; + } + oct_port_add_counters (vm, port); return VNET_DEV_OK; @@ -344,12 +350,6 @@ oct_port_start (vlib_main_t *vm, vnet_dev_port_t *port) ctq->n_enq = 0; } - if ((rrv = roc_nix_mac_mtu_set (nix, 9200))) - { - rv = oct_roc_err (dev, rrv, "roc_nix_mac_mtu_set() failed"); - goto done; - } - if ((rrv = roc_nix_npc_rx_ena_dis (nix, true))) { rv = oct_roc_err (dev, rrv, "roc_nix_npc_rx_ena_dis() failed"); @@ -442,7 +442,6 @@ oct_port_add_del_eth_addr (vlib_main_t *vm, vnet_dev_port_t *port, oct_device_t *cd = vnet_dev_get_data (dev); struct roc_nix *nix = cd->nix; vnet_dev_rv_t rv = VNET_DEV_OK; - i32 rrv; if (is_primary) @@ -468,6 +467,60 @@ oct_port_add_del_eth_addr (vlib_main_t *vm, vnet_dev_port_t *port, } } } + + return rv; +} + +vnet_dev_rv_t +oct_validate_config_max_rx_len (vlib_main_t *vm, vnet_dev_port_t *port, + u32 rx_frame_size) +{ + vnet_dev_t *dev = port->dev; + oct_device_t *cd = vnet_dev_get_data (dev); + struct roc_nix *nix = cd->nix; + vnet_dev_rv_t rv = VNET_DEV_OK; + + u32 max_len; + i32 min_len; + + if (port->started) + return VNET_DEV_ERR_PORT_STARTED; + + min_len = (i32) rx_frame_size - OCT_PKTIO_MAX_L2_SIZE; + if (min_len < 0 || min_len < NIX_MIN_HW_FRS) + { + log_err ( + dev, + "Requested rx_frame_size is lower than the minimum supported value."); + return VNET_DEV_ERR_INVALID_VALUE; + } + + max_len = roc_nix_max_pkt_len (nix); + if (rx_frame_size > max_len) + { + log_err ( + dev, + "Requested rx_frame_size is higher than the max supported value."); + return VNET_DEV_ERR_INVALID_VALUE; + } + + return rv; +} + +vnet_dev_rv_t +oct_op_config_max_rx_len (vlib_main_t *vm, vnet_dev_port_t *port, + u32 rx_frame_size) +{ + vnet_dev_t *dev = port->dev; + oct_device_t *cd = vnet_dev_get_data (dev); + struct roc_nix *nix = cd->nix; + vnet_dev_rv_t rv = VNET_DEV_OK; + i32 rrv; + + rrv = roc_nix_mac_max_rx_len_set (nix, rx_frame_size); + if (rrv) + rv = oct_roc_err (dev, rrv, "roc_nix_mac_max_rx_len_set() failed"); + return rv; } @@ -480,8 +533,7 @@ oct_port_cfg_change_validate (vlib_main_t *vm, vnet_dev_port_t *port, switch (req->type) { case VNET_DEV_PORT_CFG_MAX_RX_FRAME_SIZE: - if (port->started) - rv = VNET_DEV_ERR_PORT_STARTED; + rv = oct_validate_config_max_rx_len (vm, port, req->max_rx_frame_size); break; case VNET_DEV_PORT_CFG_PROMISC_MODE: @@ -532,6 +584,7 @@ oct_port_cfg_change (vlib_main_t *vm, vnet_dev_port_t *port, break; case VNET_DEV_PORT_CFG_MAX_RX_FRAME_SIZE: + rv = oct_op_config_max_rx_len (vm, port, req->max_rx_frame_size); break; case VNET_DEV_PORT_CFG_ADD_RX_FLOW: From 616f6edbc78dc292a3f3bd84c833f2fe4c744a50 Mon Sep 17 00:00:00 2001 From: Nithinsen Kaithakadan Date: Wed, 29 May 2024 13:43:38 +0530 Subject: [PATCH 041/271] misc: add baseline.txt for ci add baseline.txt file containing the base vpp commmit_id Type: feature Signed-off-by: Nithinsen Kaithakadan Change-Id: I7eb5f00a700edd00c37b50a771d8f81097e545db Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/128649 Tested-by: sa_ip-toolkits-Jenkins Tested-by: sa_ip-sw-jenkins Reviewed-by: Monendra Singh Kushwaha Reviewed-by: Ashish Gupta Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/128959 Reviewed-by: Nithinsen Kaithakadan --- scripts/ci/baseline.txt | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 scripts/ci/baseline.txt diff --git a/scripts/ci/baseline.txt b/scripts/ci/baseline.txt new file mode 100644 index 0000000000..3a849b80c4 --- /dev/null +++ b/scripts/ci/baseline.txt @@ -0,0 +1,9 @@ +# +# SPDX-License-Identifier: BSD-3-Clause +# https://spdx.org/licenses +# Copyright (c) 2018 Marvell. +# +# commit id of base tag for this branch +# Upstream version v24.02 release +500ac0596126576e278e65a64597e8b87fdc55f8 +flowprobe: fix flush callbacks when multiple workers From b0078959a081f9fa24643af456f3b01577451aa3 Mon Sep 17 00:00:00 2001 From: Monendra Singh Kushwaha Date: Mon, 3 Jun 2024 17:33:41 +0530 Subject: [PATCH 042/271] build: update octeon roc version Type: feature Change-Id: I221f64ba0bb768191e2228dab132166a8a3326c3 Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/128924 Tested-by: sa_ip-toolkits-Jenkins Tested-by: sa_ip-sw-jenkins Reviewed-by: Monendra Singh Kushwaha Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/129016 Reviewed-by: Nithinsen Kaithakadan --- build/external/packages/octeon-roc.mk | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build/external/packages/octeon-roc.mk b/build/external/packages/octeon-roc.mk index f96cd4c6e3..a742c99240 100644 --- a/build/external/packages/octeon-roc.mk +++ b/build/external/packages/octeon-roc.mk @@ -2,9 +2,9 @@ # SPDX-License-Identifier: Apache-2.0 # https://spdx.org/licenses/Apache-2.0.html -octeon-roc_version := 0.4 +octeon-roc_version := 0.5 octeon-roc_tarball := octeon-roc-v$(octeon-roc_version).tar.gz -octeon-roc_tarball_md5sum := f3772f6e05d71cf2709c00ff1a1929ca +octeon-roc_tarball_md5sum := f25fca80b4394e06716ac12ec54574be octeon-roc_tarball_strip_dirs := 1 octeon-roc_url := https://github.com/MarvellEmbeddedProcessors/marvell-vpp/archive/refs/tags/$(octeon-roc_tarball) From 2381ef51dce57e860078fcc5d44964e7e7b29171 Mon Sep 17 00:00:00 2001 From: Monendra Singh Kushwaha Date: Wed, 29 May 2024 17:17:07 +0530 Subject: [PATCH 043/271] octeon: convert link speed from Mbps to Kbps Type: fix Fixes: 01fe7ab88e Change-Id: I88d03adcd4ef2a585ed77834b3bf8ef9d50b15c9 Signed-off-by: Monendra Singh Kushwaha Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/128664 Tested-by: sa_ip-toolkits-Jenkins Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/129018 Tested-by: sa_ip-sw-jenkins Reviewed-by: Nithinsen Kaithakadan --- src/plugins/dev_octeon/port.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/plugins/dev_octeon/port.c b/src/plugins/dev_octeon/port.c index ad3720e3fc..6b73987e75 100644 --- a/src/plugins/dev_octeon/port.c +++ b/src/plugins/dev_octeon/port.c @@ -226,7 +226,8 @@ oct_port_poll (vlib_main_t *vm, vnet_dev_port_t *port) if (cd->speed != link_info.speed) { changes.change.link_speed = 1; - changes.link_speed = link_info.speed; + /* Convert to Kbps */ + changes.link_speed = link_info.speed * 1000; cd->speed = link_info.speed; } From a6203fc65d34b5366b98abef9bf26b75319eb52a Mon Sep 17 00:00:00 2001 From: Nithinsen Kaithakadan Date: Fri, 16 Feb 2024 11:58:04 +0530 Subject: [PATCH 044/271] octeon: add crypto framework Configure crypto device. Add crypto support in control plane and data plane. Control plane - Handle vnet crypto key add and delete - Register crypto async enqueue and dequeue handlers Data plane - Add encryption and decryption support for - AES-GCM - AES-CBC hmac sha1/256/384/512 Type: feature Signed-off-by: Nithinsen Kaithakadan Change-Id: Ia9e16c61ed84800a59e0c932a4ba6aa1423c1ec8 Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/127839 Tested-by: sa_ip-sw-jenkins Reviewed-by: Monendra Singh Kushwaha Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/129038 Tested-by: sa_ip-toolkits-Jenkins Reviewed-by: Nithinsen Kaithakadan --- src/plugins/dev_octeon/CMakeLists.txt | 1 + src/plugins/dev_octeon/crypto.c | 1276 +++++++++++++++++++++++++ src/plugins/dev_octeon/crypto.h | 160 ++++ src/plugins/dev_octeon/init.c | 83 +- src/plugins/dev_octeon/octeon.h | 2 - 5 files changed, 1516 insertions(+), 6 deletions(-) create mode 100644 src/plugins/dev_octeon/crypto.c create mode 100644 src/plugins/dev_octeon/crypto.h diff --git a/src/plugins/dev_octeon/CMakeLists.txt b/src/plugins/dev_octeon/CMakeLists.txt index c6271ecdfb..6109de57a7 100644 --- a/src/plugins/dev_octeon/CMakeLists.txt +++ b/src/plugins/dev_octeon/CMakeLists.txt @@ -36,6 +36,7 @@ add_vpp_plugin(dev_octeon tx_node.c flow.c counter.c + crypto.c MULTIARCH_SOURCES rx_node.c diff --git a/src/plugins/dev_octeon/crypto.c b/src/plugins/dev_octeon/crypto.c new file mode 100644 index 0000000000..2c840784cf --- /dev/null +++ b/src/plugins/dev_octeon/crypto.c @@ -0,0 +1,1276 @@ +/* + * Copyright (c) 2024 Marvell. + * SPDX-License-Identifier: Apache-2.0 + * https://spdx.org/licenses/Apache-2.0.html + */ + +#include +#include +#include +#include +#include +#include + +oct_crypto_t oct_crypto; +oct_crypto_dev_t oct_crypto_dev; + +VLIB_REGISTER_LOG_CLASS (oct_log, static) = { + .class_name = "octeon", + .subclass_name = "init", +}; + +void +oct_crypto_key_del_handler (vlib_main_t *vm, vnet_crypto_key_index_t key_index) +{ + extern oct_plt_init_param_t oct_plt_init_param; + oct_crypto_key_t *ckey; + + vec_validate (oct_crypto.keys[VNET_CRYPTO_OP_TYPE_ENCRYPT], key_index); + ckey = + vec_elt_at_index (oct_crypto.keys[VNET_CRYPTO_OP_TYPE_ENCRYPT], key_index); + if (ckey->sess) + { + oct_plt_init_param.oct_plt_free (ckey->sess); + ckey->sess = NULL; + return; + } + + ckey = + vec_elt_at_index (oct_crypto.keys[VNET_CRYPTO_OP_TYPE_DECRYPT], key_index); + if (ckey->sess) + { + oct_plt_init_param.oct_plt_free (ckey->sess); + ckey->sess = NULL; + return; + } +} + +void +oct_crypto_key_add_handler (vlib_main_t *vm, vnet_crypto_key_index_t key_index) +{ + oct_crypto_key_t *ckey; + + vec_validate (oct_crypto.keys[VNET_CRYPTO_OP_TYPE_ENCRYPT], key_index); + ckey = + vec_elt_at_index (oct_crypto.keys[VNET_CRYPTO_OP_TYPE_ENCRYPT], key_index); + ckey->sess = NULL; + + vec_validate (oct_crypto.keys[VNET_CRYPTO_OP_TYPE_DECRYPT], key_index); + ckey = + vec_elt_at_index (oct_crypto.keys[VNET_CRYPTO_OP_TYPE_DECRYPT], key_index); + ckey->sess = NULL; +} + +void +oct_crypto_key_handler (vlib_main_t *vm, vnet_crypto_key_op_t kop, + vnet_crypto_key_index_t idx) +{ + if (kop == VNET_CRYPTO_KEY_OP_DEL) + { + oct_crypto_key_del_handler (vm, idx); + return; + } + oct_crypto_key_add_handler (vm, idx); +} + +static_always_inline oct_crypto_sess_t * +oct_crypto_session_alloc (vlib_main_t *vm) +{ + extern oct_plt_init_param_t oct_plt_init_param; + oct_crypto_sess_t *addr = NULL; + u32 size; + + size = sizeof (oct_crypto_sess_t); + addr = oct_plt_init_param.oct_plt_zmalloc (size, CLIB_CACHE_LINE_BYTES); + if (addr == NULL) + { + log_err (oct_crypto_dev.dev, "Failed to allocate crypto session memory"); + return NULL; + } + + return addr; +} + +static_always_inline void +oct_crypto_session_free (vlib_main_t *vm, oct_crypto_sess_t *sess) +{ + extern oct_plt_init_param_t oct_plt_init_param; + + oct_plt_init_param.oct_plt_free (sess); + return; +} + +void +oct_crypto_burst_submit (oct_crypto_dev_t *crypto_dev, struct cpt_inst_s *inst, + u32 n_left) +{ + u64 *lmt_line[OCT_MAX_LMT_SZ]; + u64 lmt_arg, core_lmt_id; + u64 lmt_base; + u64 io_addr; + u32 count; + + lmt_base = crypto_dev->lmtline.lmt_base; + io_addr = crypto_dev->lmtline.io_addr; + + ROC_LMT_CPT_BASE_ID_GET (lmt_base, core_lmt_id); + + for (count = 0; count < 16; count++) + { + lmt_line[count] = OCT_CPT_LMT_GET_LINE_ADDR (lmt_base, count); + } + + while (n_left > OCT_MAX_LMT_SZ) + { + + /* + * Add a memory barrier so that LMTLINEs from the previous iteration + * can be reused for a subsequent transfer. + */ + asm volatile("dmb oshst" ::: "memory"); + + lmt_arg = ROC_CN10K_CPT_LMT_ARG | (u64) core_lmt_id; + + for (count = 0; count < 16; count++) + { + roc_lmt_mov_seg ((void *) lmt_line[count], inst + count, + CPT_LMT_SIZE_COPY); + } + + /* Set number of LMTSTs, excluding the first */ + lmt_arg |= (OCT_MAX_LMT_SZ - 1) << 12; + + roc_lmt_submit_steorl (lmt_arg, io_addr); + + inst += OCT_MAX_LMT_SZ; + n_left -= OCT_MAX_LMT_SZ; + } + + if (n_left > 0) + { + /* + * Add a memory barrier so that LMTLINEs from the previous iteration + * can be reused for a subsequent transfer. + */ + asm volatile("dmb oshst" ::: "memory"); + + lmt_arg = ROC_CN10K_CPT_LMT_ARG | (u64) core_lmt_id; + + for (count = 0; count < n_left; count++) + { + roc_lmt_mov_seg ((void *) lmt_line[count], inst + count, + CPT_LMT_SIZE_COPY); + } + + /* Set number of LMTSTs, excluding the first */ + lmt_arg |= (n_left - 1) << 12; + + roc_lmt_submit_steorl (lmt_arg, io_addr); + } +} + +static_always_inline u32 +oct_crypto_fill_sg2_comp_from_iov (struct roc_sg2list_comp *list, u32 i, + struct roc_se_iov_ptr *from, + u32 from_offset, u32 *psize, + struct roc_se_buf_ptr *extra_buf, + u32 extra_offset) +{ + u32 extra_len = extra_buf ? extra_buf->size : 0; + u32 size = *psize, buf_sz, e_len, next_len; + struct roc_sg2list_comp *to; + u64 e_vaddr, next_vaddr; + void *vaddr; + i32 j; + + for (j = 0; j < from->buf_cnt; j++) + { + to = &list[i / 3]; + buf_sz = from->bufs[j].size; + vaddr = from->bufs[j].vaddr; + + if (PREDICT_FALSE (from_offset)) + { + if (from_offset >= buf_sz) + { + from_offset -= buf_sz; + continue; + } + e_vaddr = (u64) vaddr + from_offset; + e_len = clib_min ((buf_sz - from_offset), size); + from_offset = 0; + } + else + { + e_vaddr = (u64) vaddr; + e_len = clib_min (buf_sz, size); + } + + to->u.s.len[i % 3] = (e_len); + to->ptr[i % 3] = (e_vaddr); + to->u.s.valid_segs = (i % 3) + 1; + + if (extra_len && (e_len >= extra_offset)) + { + /* Break the data at given offset */ + next_len = e_len - extra_offset; + next_vaddr = e_vaddr + extra_offset; + + if (!extra_offset) + i--; + else + { + e_len = extra_offset; + size -= e_len; + to->u.s.len[i % 3] = (e_len); + } + + extra_len = clib_min (extra_len, size); + /* Insert extra data ptr */ + if (extra_len) + { + i++; + to = &list[i / 3]; + to->u.s.len[i % 3] = (extra_len); + to->ptr[i % 3] = ((u64) extra_buf->vaddr); + to->u.s.valid_segs = (i % 3) + 1; + size -= extra_len; + } + + next_len = clib_min (next_len, size); + /* insert the rest of the data */ + if (next_len) + { + i++; + to = &list[i / 3]; + to->u.s.len[i % 3] = (next_len); + to->ptr[i % 3] = (next_vaddr); + to->u.s.valid_segs = (i % 3) + 1; + size -= next_len; + } + extra_len = 0; + } + else + size -= e_len; + + if (extra_offset) + extra_offset -= size; + + i++; + + if (PREDICT_FALSE (!size)) + break; + } + + *psize = size; + return (u32) i; +} + +static_always_inline u32 +oct_crypto_fill_sg2_comp (struct roc_sg2list_comp *list, u32 index, + u64 dma_addr, u32 size) +{ + struct roc_sg2list_comp *to = &list[index / 3]; + + to->u.s.len[index % 3] = (size); + to->ptr[index % 3] = (dma_addr); + to->u.s.valid_segs = (index % 3) + 1; + return ++index; +} + +static_always_inline u32 +oct_crypto_fill_sg2_comp_from_buf (struct roc_sg2list_comp *list, u32 index, + struct roc_se_buf_ptr *from) +{ + struct roc_sg2list_comp *to = &list[index / 3]; + + to->u.s.len[index % 3] = (from->size); + to->ptr[index % 3] = ((u64) from->vaddr); + to->u.s.valid_segs = (index % 3) + 1; + return ++index; +} + +static_always_inline int +oct_crypto_sg2_inst_prep (struct roc_se_fc_params *params, + struct cpt_inst_s *inst, u64 offset_ctrl, + const u8 *iv_s, int iv_len, u8 pack_iv, + u8 pdcp_alg_type, i32 inputlen, i32 outputlen, + u32 passthrough_len, u32 req_flags, int pdcp_flag, + int decrypt) +{ + u32 mac_len = 0, aad_len = 0, size, index, g_size_bytes; + struct roc_sg2list_comp *gather_comp, *scatter_comp; + void *m_vaddr = params->meta_buf.vaddr; + struct roc_se_buf_ptr *aad_buf = NULL; + union cpt_inst_w5 cpt_inst_w5; + union cpt_inst_w6 cpt_inst_w6; + u16 scatter_sz, gather_sz; + struct roc_se_ctx *se_ctx; + u64 *offset_vaddr; + int ret = 0; + u8 *iv_d; + + se_ctx = params->ctx; + mac_len = se_ctx->mac_len; + + if (PREDICT_FALSE (req_flags & ROC_SE_VALID_AAD_BUF)) + { + /* We don't support both AAD and auth data separately */ + aad_len = params->aad_buf.size; + aad_buf = ¶ms->aad_buf; + } + + /* save space for iv */ + offset_vaddr = m_vaddr; + + m_vaddr = (u8 *) m_vaddr + ROC_SE_OFF_CTRL_LEN + PLT_ALIGN_CEIL (iv_len, 8); + + inst->w4.s.opcode_major |= (u64) ROC_DMA_MODE_SG; + + /* This is DPTR len in case of SG mode */ + inst->w4.s.dlen = inputlen + ROC_SE_OFF_CTRL_LEN; + + /* iv offset is 0 */ + *offset_vaddr = offset_ctrl; + iv_d = ((u8 *) offset_vaddr + ROC_SE_OFF_CTRL_LEN); + + if (PREDICT_TRUE (iv_len)) + clib_memcpy (iv_d, iv_s, iv_len); + + /* DPTR has SG list */ + + gather_comp = (struct roc_sg2list_comp *) ((u8 *) m_vaddr); + + /* + * Input Gather List + */ + index = 0; + + /* Offset control word followed by iv */ + + index = oct_crypto_fill_sg2_comp (gather_comp, index, (u64) offset_vaddr, + ROC_SE_OFF_CTRL_LEN + iv_len); + + /* Add input data */ + if (decrypt && (req_flags & ROC_SE_VALID_MAC_BUF)) + { + size = inputlen - iv_len - mac_len; + if (size) + { + /* input data only */ + u32 aad_offset = aad_len ? passthrough_len : 0; + + index = oct_crypto_fill_sg2_comp_from_iov (gather_comp, index, + params->src_iov, 0, &size, + aad_buf, aad_offset); + + if (PREDICT_FALSE (size)) + { + log_err (oct_crypto_dev.dev, + "Insufficient buffer" + " space, size %d needed", + size); + return -1; + } + } + + /* mac data */ + if (mac_len) + index = oct_crypto_fill_sg2_comp_from_buf (gather_comp, index, + ¶ms->mac_buf); + } + else + { + /* input data */ + size = inputlen - iv_len; + if (size) + { + u32 aad_offset = aad_len ? passthrough_len : 0; + + index = oct_crypto_fill_sg2_comp_from_iov (gather_comp, index, + params->src_iov, 0, &size, + aad_buf, aad_offset); + if (PREDICT_FALSE (size)) + { + log_err (oct_crypto_dev.dev, + "Insufficient buffer space," + " size %d needed", + size); + return -1; + } + } + } + + gather_sz = (index + 2) / 3; + g_size_bytes = gather_sz * sizeof (struct roc_sg2list_comp); + + /* + * Output Scatter List + */ + + index = 0; + scatter_comp = + (struct roc_sg2list_comp *) ((u8 *) gather_comp + g_size_bytes); + + index = oct_crypto_fill_sg2_comp ( + scatter_comp, index, (u64) offset_vaddr + ROC_SE_OFF_CTRL_LEN, iv_len); + + /* Add output data */ + if ((!decrypt) && (req_flags & ROC_SE_VALID_MAC_BUF)) + { + size = outputlen - iv_len - mac_len; + if (size) + { + + u32 aad_offset = aad_len ? passthrough_len : 0; + + index = oct_crypto_fill_sg2_comp_from_iov (scatter_comp, index, + params->dst_iov, 0, &size, + aad_buf, aad_offset); + if (PREDICT_FALSE (size)) + { + log_err (oct_crypto_dev.dev, + "Insufficient buffer space," + " size %d needed", + size); + return -1; + } + } + + /* mac data */ + if (mac_len) + index = oct_crypto_fill_sg2_comp_from_buf (scatter_comp, index, + ¶ms->mac_buf); + } + else + { + /* Output including mac */ + size = outputlen - iv_len; + if (size) + { + u32 aad_offset = aad_len ? passthrough_len : 0; + + index = oct_crypto_fill_sg2_comp_from_iov (scatter_comp, index, + params->dst_iov, 0, &size, + aad_buf, aad_offset); + + if (PREDICT_FALSE (size)) + { + log_err (oct_crypto_dev.dev, + "Insufficient buffer space," + " size %d needed", + size); + return -1; + } + } + } + + scatter_sz = (index + 2) / 3; + + cpt_inst_w5.s.gather_sz = gather_sz; + cpt_inst_w6.s.scatter_sz = scatter_sz; + + cpt_inst_w5.s.dptr = (u64) gather_comp; + cpt_inst_w6.s.rptr = (u64) scatter_comp; + + inst->w5.u64 = cpt_inst_w5.u64; + inst->w6.u64 = cpt_inst_w6.u64; + + if (PREDICT_FALSE ((scatter_sz >> 4) || (gather_sz >> 4))) + { + log_err (oct_crypto_dev.dev, + "Exceeds max supported components. Reduce segments"); + ret = -1; + } + + return ret; +} + +static_always_inline int +oct_crypto_cpt_hmac_prep (u32 flags, u64 d_offs, u64 d_lens, + struct roc_se_fc_params *fc_params, + struct cpt_inst_s *inst, u8 is_decrypt) +{ + u32 encr_data_len, auth_data_len, aad_len = 0; + i32 inputlen, outputlen, enc_dlen, auth_dlen; + u32 encr_offset, auth_offset, iv_offset = 0; + union cpt_inst_w4 cpt_inst_w4; + struct roc_se_ctx *se_ctx; + u32 passthrough_len = 0; + const u8 *src = NULL; + u32 cipher_type; + u64 offset_ctrl; + u8 iv_len = 16; + u8 op_minor; + u32 mac_len; + int ret; + + encr_offset = ROC_SE_ENCR_OFFSET (d_offs); + auth_offset = ROC_SE_AUTH_OFFSET (d_offs); + encr_data_len = ROC_SE_ENCR_DLEN (d_lens); + auth_data_len = ROC_SE_AUTH_DLEN (d_lens); + + if (PREDICT_FALSE (flags & ROC_SE_VALID_AAD_BUF)) + { + /* We don't support both AAD and auth data separately */ + auth_data_len = 0; + auth_offset = 0; + aad_len = fc_params->aad_buf.size; + } + + se_ctx = fc_params->ctx; + cipher_type = se_ctx->enc_cipher; + mac_len = se_ctx->mac_len; + cpt_inst_w4.u64 = se_ctx->template_w4.u64; + op_minor = cpt_inst_w4.s.opcode_minor; + + if (PREDICT_FALSE (flags & ROC_SE_VALID_AAD_BUF)) + { + /* + * When AAD is given, data above encr_offset is pass through + * Since AAD is given as separate pointer and not as offset, + * this is a special case as we need to fragment input data + * into passthrough + encr_data and then insert AAD in between. + */ + passthrough_len = encr_offset; + auth_offset = passthrough_len + iv_len; + encr_offset = passthrough_len + aad_len + iv_len; + auth_data_len = aad_len + encr_data_len; + } + else + { + encr_offset += iv_len; + auth_offset += iv_len; + } + + auth_dlen = auth_offset + auth_data_len; + enc_dlen = encr_data_len + encr_offset; + + cpt_inst_w4.s.opcode_major = ROC_SE_MAJOR_OP_FC; + + if (is_decrypt) + { + cpt_inst_w4.s.opcode_minor |= ROC_SE_FC_MINOR_OP_DECRYPT; + + if (auth_dlen > enc_dlen) + { + inputlen = auth_dlen + mac_len; + outputlen = auth_dlen; + } + else + { + inputlen = enc_dlen + mac_len; + outputlen = enc_dlen; + } + } + else + { + cpt_inst_w4.s.opcode_minor |= ROC_SE_FC_MINOR_OP_ENCRYPT; + + /* Round up to 16 bytes alignment */ + if (PREDICT_FALSE (encr_data_len & 0xf)) + { + if (PREDICT_TRUE (cipher_type == ROC_SE_AES_CBC)) + enc_dlen = PLT_ALIGN_CEIL (encr_data_len, 8) + encr_offset; + } + + /* + * auth_dlen is larger than enc_dlen in Authentication cases + * like AES GMAC Authentication + */ + if (PREDICT_FALSE (auth_dlen > enc_dlen)) + { + inputlen = auth_dlen; + outputlen = auth_dlen + mac_len; + } + else + { + inputlen = enc_dlen; + outputlen = enc_dlen + mac_len; + } + } + + if (op_minor & ROC_SE_FC_MINOR_OP_HMAC_FIRST) + outputlen = enc_dlen; + + cpt_inst_w4.s.param1 = encr_data_len; + cpt_inst_w4.s.param2 = auth_data_len; + + if (PREDICT_FALSE ((encr_offset >> 16) || (iv_offset >> 8) || + (auth_offset >> 8))) + { + log_err (oct_crypto_dev.dev, "Offset not supported"); + log_err (oct_crypto_dev.dev, + "enc_offset: %d, iv_offset : %d, auth_offset: %d", encr_offset, + iv_offset, auth_offset); + return -1; + } + + offset_ctrl = clib_host_to_net_u64 ( + ((u64) encr_offset << 16) | ((u64) iv_offset << 8) | ((u64) auth_offset)); + + src = fc_params->iv_buf; + + inst->w4.u64 = cpt_inst_w4.u64; + + ret = oct_crypto_sg2_inst_prep (fc_params, inst, offset_ctrl, src, iv_len, 0, + 0, inputlen, outputlen, passthrough_len, + flags, 0, is_decrypt); + + if (PREDICT_FALSE (ret)) + { + log_err (oct_crypto_dev.dev, "sg prep failed"); + return -1; + } + + return 0; +} + +static_always_inline void +oct_crypto_fill_fc_params (oct_crypto_sess_t *sess, struct cpt_inst_s *inst, + const bool is_aead, u8 aad_length, u8 *payload, + vnet_crypto_async_frame_elt_t *elts, void *mdata, + u32 cipher_data_length, u32 cipher_data_offset, + u32 auth_data_length, u32 auth_data_offset, + vlib_buffer_t *b, u16 adj_len) +{ + struct roc_se_fc_params fc_params = { 0 }; + struct roc_se_ctx *ctx = &sess->cpt_ctx; + u64 d_offs = 0, d_lens = 0; + vlib_buffer_t *buffer = b; + u32 flags = 0, index = 0; + u8 op_minor = 0, cpt_op; + char src[SRC_IOV_SIZE]; + u32 *iv_buf; + + cpt_op = sess->cpt_op; + + if (is_aead) + { + flags |= ROC_SE_VALID_IV_BUF; + iv_buf = (u32 *) elts->iv; + iv_buf[3] = clib_host_to_net_u32 (0x1); + fc_params.iv_buf = elts->iv; + + d_offs = cipher_data_offset; + d_offs = d_offs << 16; + + d_lens = cipher_data_length; + d_lens = d_lens << 32; + + fc_params.aad_buf.vaddr = elts->aad; + fc_params.aad_buf.size = aad_length; + flags |= ROC_SE_VALID_AAD_BUF; + + if (sess->cpt_ctx.mac_len) + { + flags |= ROC_SE_VALID_MAC_BUF; + fc_params.mac_buf.size = sess->cpt_ctx.mac_len; + fc_params.mac_buf.vaddr = elts->tag; + } + } + else + { + op_minor = ctx->template_w4.s.opcode_minor; + + flags |= ROC_SE_VALID_IV_BUF; + + fc_params.iv_buf = elts->iv; + + d_offs = cipher_data_offset; + d_offs = (d_offs << 16) | auth_data_offset; + + d_lens = cipher_data_length; + d_lens = (d_lens << 32) | auth_data_length; + + if (PREDICT_TRUE (sess->cpt_ctx.mac_len)) + { + if (!(op_minor & ROC_SE_FC_MINOR_OP_HMAC_FIRST)) + { + flags |= ROC_SE_VALID_MAC_BUF; + fc_params.mac_buf.size = sess->cpt_ctx.mac_len; + fc_params.mac_buf.vaddr = elts->digest; + } + } + } + + fc_params.ctx = &sess->cpt_ctx; + + fc_params.src_iov = (void *) src; + + fc_params.src_iov->bufs[index].vaddr = payload; + fc_params.src_iov->bufs[index].size = b->current_length - adj_len; + index++; + + while (buffer->flags & VLIB_BUFFER_NEXT_PRESENT) + { + buffer = vlib_get_buffer (vlib_get_main (), buffer->next_buffer); + fc_params.src_iov->bufs[index].vaddr = + buffer->data + buffer->current_data; + fc_params.src_iov->bufs[index].size = buffer->current_length; + index++; + } + + fc_params.src_iov->buf_cnt = index; + + fc_params.dst_iov = (void *) src; + + fc_params.meta_buf.vaddr = mdata; + fc_params.meta_buf.size = OCT_SCATTER_GATHER_BUFFER_SIZE; + + oct_crypto_cpt_hmac_prep (flags, d_offs, d_lens, &fc_params, inst, cpt_op); +} + +static_always_inline u64 +oct_cpt_inst_w7_get (oct_crypto_sess_t *sess, struct roc_cpt *roc_cpt) +{ + union cpt_inst_w7 inst_w7; + + inst_w7.u64 = 0; + inst_w7.s.cptr = (u64) &sess->cpt_ctx.se_ctx.fctx; + /* Set the engine group */ + inst_w7.s.egrp = roc_cpt->eng_grp[CPT_ENG_TYPE_IE]; + + return inst_w7.u64; +} + +static_always_inline void +oct_map_keyindex_to_session (oct_crypto_sess_t *sess, u32 key_index, u8 type) +{ + oct_crypto_key_t *ckey; + + ckey = vec_elt_at_index (oct_crypto.keys[type], key_index); + + ckey->sess = sess; +} + +static_always_inline i32 +oct_crypto_link_session_update (vlib_main_t *vm, oct_crypto_sess_t *sess, + u32 key_index, u8 type) +{ + vnet_crypto_key_t *crypto_key, *auth_key; + roc_se_cipher_type enc_type = 0; + roc_se_auth_type auth_type = 0; + vnet_crypto_key_t *key; + u32 digest_len = ~0; + i32 rv = 0; + + key = vnet_crypto_get_key (key_index); + + switch (key->async_alg) + { + case VNET_CRYPTO_ALG_AES_128_CBC_SHA1_TAG12: + case VNET_CRYPTO_ALG_AES_192_CBC_SHA1_TAG12: + case VNET_CRYPTO_ALG_AES_256_CBC_SHA1_TAG12: + enc_type = ROC_SE_AES_CBC; + auth_type = ROC_SE_SHA1_TYPE; + digest_len = 12; + break; + case VNET_CRYPTO_ALG_AES_128_CBC_SHA224_TAG14: + case VNET_CRYPTO_ALG_AES_192_CBC_SHA224_TAG14: + case VNET_CRYPTO_ALG_AES_256_CBC_SHA224_TAG14: + enc_type = ROC_SE_AES_CBC; + auth_type = ROC_SE_SHA2_SHA224; + digest_len = 14; + break; + case VNET_CRYPTO_ALG_AES_128_CBC_SHA256_TAG16: + case VNET_CRYPTO_ALG_AES_192_CBC_SHA256_TAG16: + case VNET_CRYPTO_ALG_AES_256_CBC_SHA256_TAG16: + enc_type = ROC_SE_AES_CBC; + auth_type = ROC_SE_SHA2_SHA256; + digest_len = 16; + break; + case VNET_CRYPTO_ALG_AES_128_CBC_SHA384_TAG24: + case VNET_CRYPTO_ALG_AES_192_CBC_SHA384_TAG24: + case VNET_CRYPTO_ALG_AES_256_CBC_SHA384_TAG24: + enc_type = ROC_SE_AES_CBC; + auth_type = ROC_SE_SHA2_SHA384; + digest_len = 24; + break; + case VNET_CRYPTO_ALG_AES_128_CBC_SHA512_TAG32: + case VNET_CRYPTO_ALG_AES_192_CBC_SHA512_TAG32: + case VNET_CRYPTO_ALG_AES_256_CBC_SHA512_TAG32: + enc_type = ROC_SE_AES_CBC; + auth_type = ROC_SE_SHA2_SHA512; + digest_len = 32; + break; + default: + log_err (oct_crypto_dev.dev, + "Crypto: Undefined link algo %u specified. Key index %u", + key->async_alg, key_index); + return -1; + } + + if (type == VNET_CRYPTO_OP_TYPE_ENCRYPT) + sess->cpt_ctx.ciph_then_auth = true; + else + sess->cpt_ctx.auth_then_ciph = true; + + sess->iv_length = 16; + sess->cpt_op = type; + + crypto_key = vnet_crypto_get_key (key->index_crypto); + rv = roc_se_ciph_key_set (&sess->cpt_ctx, enc_type, crypto_key->data, + vec_len (crypto_key->data)); + if (rv) + { + log_err (oct_crypto_dev.dev, + "Error in setting cipher key for enc type %u", enc_type); + return -1; + } + + auth_key = vnet_crypto_get_key (key->index_integ); + + rv = roc_se_auth_key_set (&sess->cpt_ctx, auth_type, auth_key->data, + vec_len (auth_key->data), digest_len); + if (rv) + { + log_err (oct_crypto_dev.dev, + "Error in setting auth key for auth type %u", auth_type); + return -1; + } + + oct_map_keyindex_to_session (sess, key_index, type); + /* + * Map session to crypto key index also. This entry can be referred + * while deleting key + */ + oct_map_keyindex_to_session (sess, key->index_crypto, type); + + return 0; +} + +static_always_inline i32 +oct_crypto_aead_session_update (vlib_main_t *vm, oct_crypto_sess_t *sess, + u32 key_index, u8 type) +{ + vnet_crypto_key_t *key = vnet_crypto_get_key (key_index); + roc_se_cipher_type enc_type = 0; + roc_se_auth_type auth_type = 0; + u32 digest_len = ~0; + i32 rv = 0; + + switch (key->async_alg) + { + case VNET_CRYPTO_ALG_AES_128_GCM: + case VNET_CRYPTO_ALG_AES_192_GCM: + case VNET_CRYPTO_ALG_AES_256_GCM: + enc_type = ROC_SE_AES_GCM; + sess->aes_gcm = 1; + sess->iv_offset = 0; + sess->iv_length = 16; + sess->cpt_ctx.mac_len = 16; + sess->cpt_op = type; + digest_len = 16; + break; + default: + log_err (oct_crypto_dev.dev, + "Crypto: Undefined cipher algo %u specified. Key index %u", + key->async_alg, key_index); + return -1; + } + + rv = roc_se_ciph_key_set (&sess->cpt_ctx, enc_type, key->data, + vec_len (key->data)); + if (rv) + { + log_err (oct_crypto_dev.dev, + "Error in setting cipher key for enc type %u", enc_type); + return -1; + } + + rv = roc_se_auth_key_set (&sess->cpt_ctx, auth_type, NULL, 0, digest_len); + if (rv) + { + log_err (oct_crypto_dev.dev, + "Error in setting auth key for auth type %u", auth_type); + return -1; + } + + oct_map_keyindex_to_session (sess, key_index, type); + + return 0; +} + +i32 +oct_crypto_session_create (vlib_main_t *vm, vnet_crypto_key_index_t key_index, + int op_type) +{ + oct_crypto_sess_t *session; + vnet_crypto_key_t *key; + i32 rv = 0; + + key = vnet_crypto_get_key (key_index); + + session = oct_crypto_session_alloc (vm); + if (session == NULL) + return -1; + + if (key->type == VNET_CRYPTO_KEY_TYPE_LINK) + rv = oct_crypto_link_session_update (vm, session, key_index, op_type); + else + rv = oct_crypto_aead_session_update (vm, session, key_index, op_type); + + if (rv) + { + oct_crypto_session_free (vm, session); + return -1; + } + + session->crypto_dev = &oct_crypto_dev; + + session->cpt_inst_w7 = + oct_cpt_inst_w7_get (session, session->crypto_dev->roc_cpt); + + return 0; +} + +static_always_inline void +oct_crypto_update_frame_error_status (vnet_crypto_async_frame_t *f, + vnet_crypto_op_status_t s) +{ + u32 i; + + for (i = 0; i < f->n_elts; i++) + f->elts[i].status = s; + + f->state = VNET_CRYPTO_FRAME_STATE_NOT_PROCESSED; +} + +int +oct_crypto_enqueue_enc_dec (vlib_main_t *vm, vnet_crypto_async_frame_t *frame, + const u8 is_aead, u8 aad_len, const u8 type) +{ + struct cpt_inst_s inst[VNET_CRYPTO_FRAME_SIZE]; + u32 i, enq_tail, enc_auth_len, buffer_index; + u32 crypto_start_offset, integ_start_offset; + vnet_crypto_async_frame_elt_t *elts; + oct_crypto_dev_t *crypto_dev = NULL; + oct_crypto_inflight_req_t *infl_req; + oct_crypto_pending_queue_t *pend_q; + u64 dptr_start_ptr, curr_ptr; + oct_crypto_sess_t *sess; + u32 crypto_total_length; + oct_crypto_key_t *key; + vlib_buffer_t *buffer; + u16 adj_len; + + /* GCM packets having 8 bytes of aad and 8 bytes of iv */ + u8 aad_iv = 8 + 8; + + pend_q = &oct_crypto.pend_q[vlib_get_thread_index ()]; + + enq_tail = pend_q->enq_tail; + + infl_req = &pend_q->req_queue[enq_tail]; + infl_req->frame = frame; + + for (i = 0; i < frame->n_elts; i++) + { + elts = &frame->elts[i]; + buffer_index = frame->buffer_indices[i]; + key = vec_elt_at_index (oct_crypto.keys[type], elts->key_index); + + if (!key->sess) + { + if (oct_crypto_session_create (vm, elts->key_index, type) == -1) + { + oct_crypto_update_frame_error_status ( + frame, VNET_CRYPTO_OP_STATUS_FAIL_ENGINE_ERR); + return -1; + } + } + sess = key->sess; + crypto_dev = sess->crypto_dev; + + memset (inst + i, 0, sizeof (struct cpt_inst_s)); + + buffer = vlib_get_buffer (vm, buffer_index); + + if (is_aead) + { + dptr_start_ptr = + (u64) (buffer->data + (elts->crypto_start_offset - aad_iv)); + curr_ptr = (u64) (buffer->data + buffer->current_data); + adj_len = (u16) (dptr_start_ptr - curr_ptr); + + crypto_total_length = elts->crypto_total_length; + crypto_start_offset = aad_iv; + integ_start_offset = 0; + + oct_crypto_fill_fc_params ( + sess, inst + i, is_aead, aad_len, (u8 *) dptr_start_ptr, elts, + (oct_crypto_scatter_gather_t *) (infl_req->sg_data) + i, + crypto_total_length /* cipher_len */, + crypto_start_offset /* cipher_offset */, 0 /* auth_len */, + integ_start_offset /* auth_off */, buffer, adj_len); + } + else + { + dptr_start_ptr = (u64) (buffer->data + elts->crypto_start_offset - + elts->integ_length_adj); + enc_auth_len = elts->crypto_total_length + elts->integ_length_adj; + + curr_ptr = (u64) (buffer->data + buffer->current_data); + adj_len = (u16) (dptr_start_ptr - curr_ptr); + + crypto_total_length = elts->crypto_total_length; + crypto_start_offset = + elts->crypto_start_offset - elts->integ_start_offset; + integ_start_offset = 0; + + oct_crypto_fill_fc_params ( + sess, inst + i, is_aead, aad_len, (u8 *) dptr_start_ptr, elts, + (oct_crypto_scatter_gather_t *) (infl_req->sg_data) + i, + crypto_total_length /* cipher_len */, + crypto_start_offset /* cipher_offset */, + enc_auth_len /* auth_len */, integ_start_offset /* auth_off */, + buffer, adj_len); + } + + inst[i].w7.u64 = sess->cpt_inst_w7; + inst[i].res_addr = (u64) &infl_req->res[i]; + } + + oct_crypto_burst_submit (crypto_dev, inst, frame->n_elts); + + infl_req->elts = frame->n_elts; + OCT_MOD_INC (pend_q->enq_tail, pend_q->n_desc); + pend_q->n_crypto_inflight++; + + return 0; +} + +int +oct_crypto_enqueue_linked_alg_enc (vlib_main_t *vm, + vnet_crypto_async_frame_t *frame) +{ + oct_crypto_enqueue_enc_dec (vm, frame, 0 /* is_aead */, 0 /* aad_len */, + VNET_CRYPTO_OP_TYPE_ENCRYPT); + return 0; +} + +int +oct_crypto_enqueue_linked_alg_dec (vlib_main_t *vm, + vnet_crypto_async_frame_t *frame) +{ + oct_crypto_enqueue_enc_dec (vm, frame, 0 /* is_aead */, 0 /* aad_len */, + VNET_CRYPTO_OP_TYPE_DECRYPT); + return 0; +} + +int +oct_crypto_enqueue_aead_aad_enc (vlib_main_t *vm, + vnet_crypto_async_frame_t *frame, u8 aad_len) +{ + oct_crypto_enqueue_enc_dec (vm, frame, 1 /* is_aead */, aad_len, + VNET_CRYPTO_OP_TYPE_ENCRYPT); + + return 0; +} + +int +oct_crypto_enqueue_aead_aad_dec (vlib_main_t *vm, + vnet_crypto_async_frame_t *frame, u8 aad_len) +{ + oct_crypto_enqueue_enc_dec (vm, frame, 1 /* is_aead */, aad_len, + VNET_CRYPTO_OP_TYPE_DECRYPT); + + return 0; +} + +int +oct_crypto_enqueue_aead_aad_8_enc (vlib_main_t *vm, + vnet_crypto_async_frame_t *frame) +{ + return oct_crypto_enqueue_aead_aad_enc (vm, frame, 8); +} + +int +oct_crypto_enqueue_aead_aad_12_enc (vlib_main_t *vm, + vnet_crypto_async_frame_t *frame) +{ + return oct_crypto_enqueue_aead_aad_enc (vm, frame, 12); +} + +int +oct_crypto_enqueue_aead_aad_8_dec (vlib_main_t *vm, + vnet_crypto_async_frame_t *frame) +{ + return oct_crypto_enqueue_aead_aad_dec (vm, frame, 8); +} + +int +oct_crypto_enqueue_aead_aad_12_dec (vlib_main_t *vm, + vnet_crypto_async_frame_t *frame) +{ + return oct_crypto_enqueue_aead_aad_dec (vm, frame, 12); +} + +vnet_crypto_async_frame_t * +oct_crypto_frame_dequeue (vlib_main_t *vm, u32 *nb_elts_processed, + u32 *enqueue_thread_idx) +{ + u32 deq_head, status = VNET_CRYPTO_OP_STATUS_COMPLETED; + vnet_crypto_async_frame_elt_t *fe = NULL; + oct_crypto_inflight_req_t *infl_req; + oct_crypto_pending_queue_t *pend_q; + vnet_crypto_async_frame_t *frame; + volatile union cpt_res_s *res; + int i; + + pend_q = &oct_crypto.pend_q[vlib_get_thread_index ()]; + + if (!pend_q->n_crypto_inflight) + return NULL; + + deq_head = pend_q->deq_head; + infl_req = &pend_q->req_queue[deq_head]; + frame = infl_req->frame; + + fe = frame->elts; + + for (i = infl_req->deq_elts; i < infl_req->elts; ++i) + { + res = &infl_req->res[i]; + + if (PREDICT_FALSE (res->cn10k.compcode == CPT_COMP_NOT_DONE)) + return NULL; + + if (PREDICT_FALSE (res->cn10k.uc_compcode)) + { + if (res->cn10k.uc_compcode == ROC_SE_ERR_GC_ICV_MISCOMPARE) + status = fe[i].status = VNET_CRYPTO_OP_STATUS_FAIL_BAD_HMAC; + else + status = fe[i].status = VNET_CRYPTO_OP_STATUS_FAIL_ENGINE_ERR; + } + + infl_req->deq_elts++; + } + + clib_memset ((void *) infl_req->res, 0, + sizeof (union cpt_res_s) * VNET_CRYPTO_FRAME_SIZE); + + OCT_MOD_INC (pend_q->deq_head, pend_q->n_desc); + pend_q->n_crypto_inflight--; + + frame->state = status == VNET_CRYPTO_OP_STATUS_COMPLETED ? + VNET_CRYPTO_FRAME_STATE_SUCCESS : + VNET_CRYPTO_FRAME_STATE_ELT_ERROR; + + *nb_elts_processed = frame->n_elts; + *enqueue_thread_idx = frame->enqueue_thread_index; + + infl_req->deq_elts = 0; + infl_req->elts = 0; + + return frame; +} + +int +oct_init_crypto_engine_handlers (vlib_main_t *vm, vnet_dev_t *dev) +{ + u32 engine_index; + + engine_index = vnet_crypto_register_engine (vm, "oct_cryptodev", 100, + "OCT Cryptodev Engine"); + +#define _(n, k, t, a) \ + vnet_crypto_register_enqueue_handler ( \ + vm, engine_index, VNET_CRYPTO_OP_##n##_TAG##t##_AAD##a##_ENC, \ + oct_crypto_enqueue_aead_aad_##a##_enc); \ + vnet_crypto_register_enqueue_handler ( \ + vm, engine_index, VNET_CRYPTO_OP_##n##_TAG##t##_AAD##a##_DEC, \ + oct_crypto_enqueue_aead_aad_##a##_dec); + foreach_oct_crypto_aead_async_alg +#undef _ + +#define _(c, h, k, d) \ + vnet_crypto_register_enqueue_handler ( \ + vm, engine_index, VNET_CRYPTO_OP_##c##_##h##_TAG##d##_ENC, \ + oct_crypto_enqueue_linked_alg_enc); \ + vnet_crypto_register_enqueue_handler ( \ + vm, engine_index, VNET_CRYPTO_OP_##c##_##h##_TAG##d##_DEC, \ + oct_crypto_enqueue_linked_alg_dec); + foreach_oct_crypto_link_async_alg; +#undef _ + + vnet_crypto_register_dequeue_handler (vm, engine_index, + oct_crypto_frame_dequeue); + + vnet_crypto_register_key_handler (vm, engine_index, oct_crypto_key_handler); + + return 0; +} + +int +oct_conf_sw_queue (vlib_main_t *vm, vnet_dev_t *dev) +{ + extern oct_plt_init_param_t oct_plt_init_param; + vnet_device_main_t *vdm = &vnet_device_main; + oct_crypto_inflight_req_t *infl_req_queue; + u8 num_worker_cores; + int i, j = 0; + + num_worker_cores = + vdm->last_worker_thread_index - vdm->first_worker_thread_index + 1; + + oct_crypto.pend_q = oct_plt_init_param.oct_plt_zmalloc ( + num_worker_cores * sizeof (oct_crypto_pending_queue_t), + CLIB_CACHE_LINE_BYTES); + if (oct_crypto.pend_q == NULL) + { + log_err (dev, "Failed to allocate memory for crypto pending queue"); + return -1; + } + + for (i = 0; i <= num_worker_cores; ++i) + { + oct_crypto.pend_q[i].n_desc = OCT_CRYPTO_DEFAULT_SW_ASYNC_FRAME_COUNT; + + oct_crypto.pend_q[i].req_queue = oct_plt_init_param.oct_plt_zmalloc ( + OCT_CRYPTO_DEFAULT_SW_ASYNC_FRAME_COUNT * + sizeof (oct_crypto_inflight_req_t), + CLIB_CACHE_LINE_BYTES); + if (oct_crypto.pend_q[i].req_queue == NULL) + { + log_err (dev, + "Failed to allocate memory for crypto inflight request"); + goto free; + } + + for (j = 0; j <= oct_crypto.pend_q[i].n_desc; ++j) + { + infl_req_queue = &oct_crypto.pend_q[i].req_queue[j]; + + infl_req_queue->sg_data = oct_plt_init_param.oct_plt_zmalloc ( + OCT_SCATTER_GATHER_BUFFER_SIZE * VNET_CRYPTO_FRAME_SIZE, + CLIB_CACHE_LINE_BYTES); + if (infl_req_queue->sg_data == NULL) + { + log_err (dev, "Failed to allocate crypto scatter gather memory"); + goto free; + } + } + } + return 0; +free: + for (; i >= 0; i--) + { + if (oct_crypto.pend_q[i].req_queue == NULL) + continue; + for (; j >= 0; j--) + { + infl_req_queue = &oct_crypto.pend_q[i].req_queue[j]; + + if (infl_req_queue->sg_data == NULL) + continue; + + oct_plt_init_param.oct_plt_free (infl_req_queue->sg_data); + } + oct_plt_init_param.oct_plt_free (oct_crypto.pend_q[i].req_queue); + } + oct_plt_init_param.oct_plt_free (oct_crypto.pend_q); + + return -1; +} diff --git a/src/plugins/dev_octeon/crypto.h b/src/plugins/dev_octeon/crypto.h new file mode 100644 index 0000000000..33e33d6340 --- /dev/null +++ b/src/plugins/dev_octeon/crypto.h @@ -0,0 +1,160 @@ +/* + * Copyright (c) 2024 Marvell. + * SPDX-License-Identifier: Apache-2.0 + * https://spdx.org/licenses/Apache-2.0.html + */ + +#ifndef _CRYPTO_H_ +#define _CRYPTO_H_ +#include +#include + +/* CRYPTO_ID, KEY_LENGTH_IN_BYTES, TAG_LEN, AAD_LEN */ +#define foreach_oct_crypto_aead_async_alg \ + _ (AES_128_GCM, 16, 16, 8) \ + _ (AES_128_GCM, 16, 16, 12) \ + _ (AES_192_GCM, 24, 16, 8) \ + _ (AES_192_GCM, 24, 16, 12) \ + _ (AES_256_GCM, 32, 16, 8) \ + _ (AES_256_GCM, 32, 16, 12) + +/* CRYPTO_ID, INTEG_ID, KEY_LENGTH_IN_BYTES, DIGEST_LEN */ +#define foreach_oct_crypto_link_async_alg \ + _ (AES_128_CBC, SHA1, 16, 12) \ + _ (AES_192_CBC, SHA1, 24, 12) \ + _ (AES_256_CBC, SHA1, 32, 12) \ + _ (AES_128_CBC, SHA256, 16, 16) \ + _ (AES_192_CBC, SHA256, 24, 16) \ + _ (AES_256_CBC, SHA256, 32, 16) \ + _ (AES_128_CBC, SHA384, 16, 24) \ + _ (AES_192_CBC, SHA384, 24, 24) \ + _ (AES_256_CBC, SHA384, 32, 24) \ + _ (AES_128_CBC, SHA512, 16, 32) \ + _ (AES_192_CBC, SHA512, 24, 32) \ + _ (AES_256_CBC, SHA512, 32, 32) + +#define OCT_MOD_INC(i, l) ((i) == (l - 1) ? (i) = 0 : (i)++) + +#define OCT_SCATTER_GATHER_BUFFER_SIZE 1024 +#define OCT_CRYPTO_DEFAULT_SW_ASYNC_FRAME_COUNT 256 + +#define CPT_LMT_SIZE_COPY (sizeof (struct cpt_inst_s) / 16) +#define OCT_MAX_LMT_SZ 16 + +#define SRC_IOV_SIZE \ + (sizeof (struct roc_se_iov_ptr) + \ + (sizeof (struct roc_se_buf_ptr) * ROC_MAX_SG_CNT)) + +#define OCT_CPT_LMT_GET_LINE_ADDR(lmt_addr, lmt_num) \ + (void *) ((u64) (lmt_addr) + ((u64) (lmt_num) << ROC_LMT_LINE_SIZE_LOG2)) + +typedef struct +{ + CLIB_CACHE_LINE_ALIGN_MARK (cacheline0); + struct roc_cpt *roc_cpt; + struct roc_cpt_lmtline lmtline; + struct roc_cpt_lf lf; + vnet_dev_t *dev; +} oct_crypto_dev_t; + +typedef struct +{ + CLIB_CACHE_LINE_ALIGN_MARK (cacheline0); + /** CPT opcode */ + u16 cpt_op : 4; + /** Flag for AES GCM */ + u16 aes_gcm : 1; + /** IV length in bytes */ + u8 iv_length; + /** Auth IV length in bytes */ + u8 auth_iv_length; + /** IV offset in bytes */ + u16 iv_offset; + /** Auth IV offset in bytes */ + u16 auth_iv_offset; + /** CPT inst word 7 */ + u64 cpt_inst_w7; + oct_crypto_dev_t *crypto_dev; + struct roc_se_ctx cpt_ctx; +} oct_crypto_sess_t; + +typedef struct +{ + CLIB_CACHE_LINE_ALIGN_MARK (cacheline0); + oct_crypto_sess_t *sess; + oct_crypto_dev_t *crypto_dev; +} oct_crypto_key_t; + +typedef struct oct_crypto_scatter_gather +{ + u8 buf[OCT_SCATTER_GATHER_BUFFER_SIZE]; +} oct_crypto_scatter_gather_t; + +typedef struct +{ + CLIB_CACHE_LINE_ALIGN_MARK (cacheline0); + /** Result data of all entries in the frame */ + volatile union cpt_res_s res[VNET_CRYPTO_FRAME_SIZE]; + /** Scatter gather data */ + void *sg_data; + /** Frame pointer */ + vnet_crypto_async_frame_t *frame; + /** Number of async elements in frame */ + u16 elts; + /** Next read entry in frame, when dequeue */ + u16 deq_elts; +} oct_crypto_inflight_req_t; + +typedef struct +{ + /** Array of pending request */ + oct_crypto_inflight_req_t *req_queue; + /** Number of inflight operations in queue */ + u32 n_crypto_inflight; + /** Tail of queue to be used for enqueue */ + u16 enq_tail; + /** Head of queue to be used for dequeue */ + u16 deq_head; + /** Number of descriptors */ + u16 n_desc; +} oct_crypto_pending_queue_t; + +typedef struct +{ + oct_crypto_key_t *keys[VNET_CRYPTO_ASYNC_OP_N_TYPES]; + oct_crypto_pending_queue_t *pend_q; +} oct_crypto_t; + +void oct_crypto_key_del_handler (vlib_main_t *vm, + vnet_crypto_key_index_t key_index); + +void oct_crypto_key_add_handler (vlib_main_t *vm, + vnet_crypto_key_index_t key_index); + +void oct_crypto_key_handler (vlib_main_t *vm, vnet_crypto_key_op_t kop, + vnet_crypto_key_index_t idx); + +int oct_crypto_enqueue_linked_alg_enc (vlib_main_t *vm, + vnet_crypto_async_frame_t *frame); +int oct_crypto_enqueue_linked_alg_dec (vlib_main_t *vm, + vnet_crypto_async_frame_t *frame); +int oct_crypto_enqueue_aead_aad_enc (vlib_main_t *vm, + vnet_crypto_async_frame_t *frame, + u8 aad_len); +int oct_crypto_enqueue_aead_aad_dec (vlib_main_t *vm, + vnet_crypto_async_frame_t *frame, + u8 aad_len); +int oct_crypto_enqueue_aead_aad_8_enc (vlib_main_t *vm, + vnet_crypto_async_frame_t *frame); +int oct_crypto_enqueue_aead_aad_12_enc (vlib_main_t *vm, + vnet_crypto_async_frame_t *frame); +int oct_crypto_enqueue_aead_aad_8_dec (vlib_main_t *vm, + vnet_crypto_async_frame_t *frame); +int oct_crypto_enqueue_aead_aad_12_dec (vlib_main_t *vm, + vnet_crypto_async_frame_t *frame); +vnet_crypto_async_frame_t *oct_crypto_frame_dequeue (vlib_main_t *vm, + u32 *nb_elts_processed, + u32 *enqueue_thread_idx); +int oct_init_crypto_engine_handlers (vlib_main_t *vm, vnet_dev_t *dev); +int oct_conf_sw_queue (vlib_main_t *vm, vnet_dev_t *dev); +#endif /* _CRYPTO_H_ */ diff --git a/src/plugins/dev_octeon/init.c b/src/plugins/dev_octeon/init.c index 7aed543f61..24af469609 100644 --- a/src/plugins/dev_octeon/init.c +++ b/src/plugins/dev_octeon/init.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include @@ -185,17 +186,91 @@ oct_init_nix (vlib_main_t *vm, vnet_dev_t *dev) return vnet_dev_port_add (vm, dev, 0, &port_add_args); } +static int +oct_conf_cpt (vlib_main_t *vm, vnet_dev_t *dev, struct roc_cpt *roc_cpt, + int nb_lf) +{ + int rrv; + if ((rrv = roc_cpt_eng_grp_add (roc_cpt, CPT_ENG_TYPE_SE)) < 0) + { + log_err (dev, "Could not add CPT SE engines"); + return cnx_return_roc_err (dev, rrv, "roc_cpt_eng_grp_add"); + } + if ((rrv = roc_cpt_eng_grp_add (roc_cpt, CPT_ENG_TYPE_IE)) < 0) + { + log_err (dev, "Could not add CPT IE engines"); + return cnx_return_roc_err (dev, rrv, "roc_cpt_eng_grp_add"); + } + if (roc_cpt->eng_grp[CPT_ENG_TYPE_IE] != ROC_CPT_DFLT_ENG_GRP_SE_IE) + { + log_err (dev, "Invalid CPT IE engine group configuration"); + return -1; + } + if (roc_cpt->eng_grp[CPT_ENG_TYPE_SE] != ROC_CPT_DFLT_ENG_GRP_SE) + { + log_err (dev, "Invalid CPT SE engine group configuration"); + return -1; + } + if ((rrv = roc_cpt_dev_configure (roc_cpt, nb_lf, false, 0)) < 0) + { + log_err (dev, "could not configure crypto device %U", + format_vlib_pci_addr, roc_cpt->pci_dev->addr); + return cnx_return_roc_err (dev, rrv, "roc_cpt_dev_configure"); + } + return 0; +} + +static vnet_dev_rv_t +oct_conf_cpt_queue (vlib_main_t *vm, vnet_dev_t *dev, struct roc_cpt *roc_cpt) +{ + extern oct_crypto_dev_t oct_crypto_dev; + struct roc_cpt_lmtline *cpt_lmtline; + struct roc_cpt_lf *cpt_lf; + int rrv; + + cpt_lf = &oct_crypto_dev.lf; + cpt_lmtline = &oct_crypto_dev.lmtline; + + cpt_lf->nb_desc = 8192; + cpt_lf->lf_id = 0; + if ((rrv = roc_cpt_lf_init (roc_cpt, cpt_lf)) < 0) + return cnx_return_roc_err (dev, rrv, "roc_cpt_lf_init"); + + roc_cpt_iq_enable (cpt_lf); + + if ((rrv = roc_cpt_lmtline_init (roc_cpt, cpt_lmtline, 0) < 0)) + return cnx_return_roc_err (dev, rrv, "roc_cpt_lmtline_init"); + + return 0; +} + static vnet_dev_rv_t oct_init_cpt (vlib_main_t *vm, vnet_dev_t *dev) { + extern oct_plt_init_param_t oct_plt_init_param; oct_device_t *cd = vnet_dev_get_data (dev); + extern oct_crypto_dev_t oct_crypto_dev; int rrv; - struct roc_cpt cpt = { - .pci_dev = &cd->plt_pci_dev, - }; - if ((rrv = roc_cpt_dev_init (&cpt))) + oct_crypto_dev.roc_cpt = oct_plt_init_param.oct_plt_zmalloc ( + sizeof (struct roc_cpt), CLIB_CACHE_LINE_BYTES); + oct_crypto_dev.roc_cpt->pci_dev = &cd->plt_pci_dev; + + oct_crypto_dev.dev = dev; + + if ((rrv = roc_cpt_dev_init (oct_crypto_dev.roc_cpt))) return cnx_return_roc_err (dev, rrv, "roc_cpt_dev_init"); + + if ((rrv = oct_conf_cpt (vm, dev, oct_crypto_dev.roc_cpt, 1))) + return rrv; + + if ((rrv = oct_conf_cpt_queue (vm, dev, oct_crypto_dev.roc_cpt))) + return rrv; + + oct_conf_sw_queue (vm, dev); + + oct_init_crypto_engine_handlers (vm, dev); + return VNET_DEV_OK; } diff --git a/src/plugins/dev_octeon/octeon.h b/src/plugins/dev_octeon/octeon.h index b5d48c7ffa..53908c94f4 100644 --- a/src/plugins/dev_octeon/octeon.h +++ b/src/plugins/dev_octeon/octeon.h @@ -48,7 +48,6 @@ typedef struct u8 full_duplex : 1; u32 speed; struct plt_pci_device plt_pci_dev; - struct roc_cpt cpt; struct roc_nix *nix; } oct_device_t; @@ -109,7 +108,6 @@ typedef struct u64 aura_handle; u64 io_addr; void *lmt_addr; - oct_npa_batch_alloc_cl128_t *ba_buffer; u8 ba_first_cl; u8 ba_num_cl; From b17de386e1d7cae06713a1e9c950032d92d4ab64 Mon Sep 17 00:00:00 2001 From: Monendra Singh Kushwaha Date: Wed, 5 Jun 2024 16:15:28 +0530 Subject: [PATCH 045/271] octeon: add crypto framework for O9k Configure multiple crypto device. Data plane - Add encryption and decryption support for - AES-GCM Type: feature Change-Id: I30e304067c0fe45d29f3d1b3b7f9818afde218d0 Signed-off-by: Monendra Singh Kushwaha Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/129289 Tested-by: sa_ip-toolkits-Jenkins Tested-by: sa_ip-sw-jenkins Reviewed-by: Nithinsen Kaithakadan --- src/plugins/dev_octeon/crypto.c | 468 ++++++++++++++++++++++++++++---- src/plugins/dev_octeon/crypto.h | 7 +- src/plugins/dev_octeon/init.c | 50 +++- src/plugins/dev_octeon/octeon.h | 3 +- 4 files changed, 455 insertions(+), 73 deletions(-) diff --git a/src/plugins/dev_octeon/crypto.c b/src/plugins/dev_octeon/crypto.c index 2c840784cf..b8552f0618 100644 --- a/src/plugins/dev_octeon/crypto.c +++ b/src/plugins/dev_octeon/crypto.c @@ -11,8 +11,7 @@ #include #include -oct_crypto_t oct_crypto; -oct_crypto_dev_t oct_crypto_dev; +oct_crypto_main_t oct_crypto_main; VLIB_REGISTER_LOG_CLASS (oct_log, static) = { .class_name = "octeon", @@ -22,12 +21,12 @@ VLIB_REGISTER_LOG_CLASS (oct_log, static) = { void oct_crypto_key_del_handler (vlib_main_t *vm, vnet_crypto_key_index_t key_index) { + oct_crypto_main_t *ocm = &oct_crypto_main; extern oct_plt_init_param_t oct_plt_init_param; oct_crypto_key_t *ckey; - vec_validate (oct_crypto.keys[VNET_CRYPTO_OP_TYPE_ENCRYPT], key_index); - ckey = - vec_elt_at_index (oct_crypto.keys[VNET_CRYPTO_OP_TYPE_ENCRYPT], key_index); + vec_validate (ocm->keys[VNET_CRYPTO_OP_TYPE_ENCRYPT], key_index); + ckey = vec_elt_at_index (ocm->keys[VNET_CRYPTO_OP_TYPE_ENCRYPT], key_index); if (ckey->sess) { oct_plt_init_param.oct_plt_free (ckey->sess); @@ -35,8 +34,7 @@ oct_crypto_key_del_handler (vlib_main_t *vm, vnet_crypto_key_index_t key_index) return; } - ckey = - vec_elt_at_index (oct_crypto.keys[VNET_CRYPTO_OP_TYPE_DECRYPT], key_index); + ckey = vec_elt_at_index (ocm->keys[VNET_CRYPTO_OP_TYPE_DECRYPT], key_index); if (ckey->sess) { oct_plt_init_param.oct_plt_free (ckey->sess); @@ -48,16 +46,15 @@ oct_crypto_key_del_handler (vlib_main_t *vm, vnet_crypto_key_index_t key_index) void oct_crypto_key_add_handler (vlib_main_t *vm, vnet_crypto_key_index_t key_index) { + oct_crypto_main_t *ocm = &oct_crypto_main; oct_crypto_key_t *ckey; - vec_validate (oct_crypto.keys[VNET_CRYPTO_OP_TYPE_ENCRYPT], key_index); - ckey = - vec_elt_at_index (oct_crypto.keys[VNET_CRYPTO_OP_TYPE_ENCRYPT], key_index); + vec_validate (ocm->keys[VNET_CRYPTO_OP_TYPE_ENCRYPT], key_index); + ckey = vec_elt_at_index (ocm->keys[VNET_CRYPTO_OP_TYPE_ENCRYPT], key_index); ckey->sess = NULL; - vec_validate (oct_crypto.keys[VNET_CRYPTO_OP_TYPE_DECRYPT], key_index); - ckey = - vec_elt_at_index (oct_crypto.keys[VNET_CRYPTO_OP_TYPE_DECRYPT], key_index); + vec_validate (ocm->keys[VNET_CRYPTO_OP_TYPE_DECRYPT], key_index); + ckey = vec_elt_at_index (ocm->keys[VNET_CRYPTO_OP_TYPE_DECRYPT], key_index); ckey->sess = NULL; } @@ -65,17 +62,23 @@ void oct_crypto_key_handler (vlib_main_t *vm, vnet_crypto_key_op_t kop, vnet_crypto_key_index_t idx) { + oct_crypto_main_t *ocm = &oct_crypto_main; + if (kop == VNET_CRYPTO_KEY_OP_DEL) { oct_crypto_key_del_handler (vm, idx); return; } oct_crypto_key_add_handler (vm, idx); + + ocm->started = 1; } static_always_inline oct_crypto_sess_t * -oct_crypto_session_alloc (vlib_main_t *vm) +oct_crypto_session_alloc (vlib_main_t *vm, u8 type) { + oct_crypto_main_t *ocm = &oct_crypto_main; + oct_crypto_dev_t *ocd = ocm->crypto_dev[type]; extern oct_plt_init_param_t oct_plt_init_param; oct_crypto_sess_t *addr = NULL; u32 size; @@ -84,7 +87,7 @@ oct_crypto_session_alloc (vlib_main_t *vm) addr = oct_plt_init_param.oct_plt_zmalloc (size, CLIB_CACHE_LINE_BYTES); if (addr == NULL) { - log_err (oct_crypto_dev.dev, "Failed to allocate crypto session memory"); + log_err (ocd->dev, "Failed to allocate crypto session memory"); return NULL; } @@ -100,16 +103,49 @@ oct_crypto_session_free (vlib_main_t *vm, oct_crypto_sess_t *sess) return; } +#ifdef PLATFORM_OCTEON9 +static inline void +oct_cpt_inst_submit (struct cpt_inst_s *inst, uint64_t lmtline, + uint64_t io_addr) +{ + uint64_t lmt_status; + + do + { + /* Copy CPT command to LMTLINE */ + roc_lmt_mov64 ((void *) lmtline, inst); + + /* + * Make sure compiler does not reorder memcpy and ldeor. + * LMTST transactions are always flushed from the write + * buffer immediately, a DMB is not required to push out + * LMTSTs. + */ + asm volatile("dmb oshst" : : : "memory"); + lmt_status = roc_lmt_submit_ldeor (io_addr); + } + while (lmt_status == 0); +} +#endif + void oct_crypto_burst_submit (oct_crypto_dev_t *crypto_dev, struct cpt_inst_s *inst, u32 n_left) { - u64 *lmt_line[OCT_MAX_LMT_SZ]; - u64 lmt_arg, core_lmt_id; u64 lmt_base; u64 io_addr; u32 count; +#ifdef PLATFORM_OCTEON9 + lmt_base = crypto_dev->lf.lmt_base; + io_addr = crypto_dev->lf.io_addr; + + for (count = 0; count < n_left; count++) + oct_cpt_inst_submit (inst + count, lmt_base, io_addr); +#else + u64 *lmt_line[OCT_MAX_LMT_SZ]; + u64 lmt_arg, core_lmt_id; + lmt_base = crypto_dev->lmtline.lmt_base; io_addr = crypto_dev->lmtline.io_addr; @@ -167,6 +203,103 @@ oct_crypto_burst_submit (oct_crypto_dev_t *crypto_dev, struct cpt_inst_s *inst, roc_lmt_submit_steorl (lmt_arg, io_addr); } +#endif +} + +static_always_inline uint32_t +oct_crypto_fill_sg_comp_from_iov (struct roc_sglist_comp *list, uint32_t i, + struct roc_se_iov_ptr *from, + uint32_t from_offset, uint32_t *psize, + struct roc_se_buf_ptr *extra_buf, + uint32_t extra_offset) +{ + uint32_t extra_len = extra_buf ? extra_buf->size : 0; + uint32_t size = *psize; + int32_t j; + + for (j = 0; j < from->buf_cnt; j++) + { + struct roc_sglist_comp *to = &list[i >> 2]; + uint32_t buf_sz = from->bufs[j].size; + void *vaddr = from->bufs[j].vaddr; + uint64_t e_vaddr; + uint32_t e_len; + + if (PREDICT_FALSE (from_offset)) + { + if (from_offset >= buf_sz) + { + from_offset -= buf_sz; + continue; + } + e_vaddr = (uint64_t) vaddr + from_offset; + e_len = clib_min ((buf_sz - from_offset), size); + from_offset = 0; + } + else + { + e_vaddr = (uint64_t) vaddr; + e_len = clib_min (buf_sz, size); + } + + to->u.s.len[i % 4] = clib_host_to_net_u16 (e_len); + to->ptr[i % 4] = clib_host_to_net_u64 (e_vaddr); + + if (extra_len && (e_len >= extra_offset)) + { + /* Break the data at given offset */ + uint32_t next_len = e_len - extra_offset; + uint64_t next_vaddr = e_vaddr + extra_offset; + + if (!extra_offset) + { + i--; + } + else + { + e_len = extra_offset; + size -= e_len; + to->u.s.len[i % 4] = clib_host_to_net_u16 (e_len); + } + + extra_len = clib_min (extra_len, size); + /* Insert extra data ptr */ + if (extra_len) + { + i++; + to = &list[i >> 2]; + to->u.s.len[i % 4] = clib_host_to_net_u16 (extra_len); + to->ptr[i % 4] = + clib_host_to_net_u64 ((uint64_t) extra_buf->vaddr); + size -= extra_len; + } + + next_len = clib_min (next_len, size); + /* insert the rest of the data */ + if (next_len) + { + i++; + to = &list[i >> 2]; + to->u.s.len[i % 4] = clib_host_to_net_u16 (next_len); + to->ptr[i % 4] = clib_host_to_net_u64 (next_vaddr); + size -= next_len; + } + extra_len = 0; + } + else + { + size -= e_len; + } + if (extra_offset) + extra_offset -= size; + i++; + + if (PREDICT_FALSE (!size)) + break; + } + + *psize = size; + return (uint32_t) i; } static_always_inline u32 @@ -266,6 +399,28 @@ oct_crypto_fill_sg2_comp_from_iov (struct roc_sg2list_comp *list, u32 i, return (u32) i; } +static_always_inline uint32_t +oct_crypto_fill_sg_comp_from_buf (struct roc_sglist_comp *list, uint32_t i, + struct roc_se_buf_ptr *from) +{ + struct roc_sglist_comp *to = &list[i >> 2]; + + to->u.s.len[i % 4] = clib_host_to_net_u16 (from->size); + to->ptr[i % 4] = clib_host_to_net_u64 ((uint64_t) from->vaddr); + return ++i; +} + +static_always_inline uint32_t +oct_crypto_fill_sg_comp (struct roc_sglist_comp *list, uint32_t i, + uint64_t dma_addr, uint32_t size) +{ + struct roc_sglist_comp *to = &list[i >> 2]; + + to->u.s.len[i % 4] = clib_host_to_net_u16 (size); + to->ptr[i % 4] = clib_host_to_net_u64 (dma_addr); + return ++i; +} + static_always_inline u32 oct_crypto_fill_sg2_comp (struct roc_sg2list_comp *list, u32 index, u64 dma_addr, u32 size) @@ -290,7 +445,190 @@ oct_crypto_fill_sg2_comp_from_buf (struct roc_sg2list_comp *list, u32 index, return ++index; } -static_always_inline int +static_always_inline int __attribute__ ((unused)) +oct_crypto_sg_inst_prep (struct roc_se_fc_params *params, + struct cpt_inst_s *inst, uint64_t offset_ctrl, + const uint8_t *iv_s, int iv_len, uint8_t pack_iv, + uint8_t pdcp_alg_type, int32_t inputlen, + int32_t outputlen, uint32_t passthrough_len, + uint32_t req_flags, int pdcp_flag, int decrypt) +{ + oct_crypto_main_t *ocm = &oct_crypto_main; + oct_crypto_dev_t *ocd = ocm->crypto_dev[decrypt]; + struct roc_sglist_comp *gather_comp, *scatter_comp; + void *m_vaddr = params->meta_buf.vaddr; + struct roc_se_buf_ptr *aad_buf = NULL; + uint32_t mac_len = 0, aad_len = 0; + struct roc_se_ctx *se_ctx; + uint32_t i, g_size_bytes; + uint64_t *offset_vaddr; + uint32_t s_size_bytes; + uint8_t *in_buffer; + uint32_t size; + uint8_t *iv_d; + int ret = 0; + + se_ctx = params->ctx; + mac_len = se_ctx->mac_len; + + if (PREDICT_FALSE (req_flags & ROC_SE_VALID_AAD_BUF)) + { + /* We don't support both AAD and auth data separately */ + aad_len = params->aad_buf.size; + aad_buf = ¶ms->aad_buf; + } + + /* save space for iv */ + offset_vaddr = m_vaddr; + + m_vaddr = + (uint8_t *) m_vaddr + ROC_SE_OFF_CTRL_LEN + PLT_ALIGN_CEIL (iv_len, 8); + + inst->w4.s.opcode_major |= (uint64_t) ROC_DMA_MODE_SG; + + /* iv offset is 0 */ + *offset_vaddr = offset_ctrl; + + iv_d = ((uint8_t *) offset_vaddr + ROC_SE_OFF_CTRL_LEN); + + if (PREDICT_TRUE (iv_len)) + memcpy (iv_d, iv_s, iv_len); + + /* DPTR has SG list */ + + /* TODO Add error check if space will be sufficient */ + gather_comp = (struct roc_sglist_comp *) ((uint8_t *) m_vaddr + 8); + + /* + * Input Gather List + */ + i = 0; + + /* Offset control word followed by iv */ + + i = oct_crypto_fill_sg_comp (gather_comp, i, (uint64_t) offset_vaddr, + ROC_SE_OFF_CTRL_LEN + iv_len); + + /* Add input data */ + if (decrypt && (req_flags & ROC_SE_VALID_MAC_BUF)) + { + size = inputlen - iv_len - mac_len; + + if (PREDICT_TRUE (size)) + { + uint32_t aad_offset = aad_len ? passthrough_len : 0; + i = oct_crypto_fill_sg_comp_from_iov ( + gather_comp, i, params->src_iov, 0, &size, aad_buf, aad_offset); + if (PREDICT_FALSE (size)) + { + log_err (ocd->dev, "Insufficient buffer space, size %d needed", + size); + return -1; + } + } + + if (mac_len) + i = + oct_crypto_fill_sg_comp_from_buf (gather_comp, i, ¶ms->mac_buf); + } + else + { + /* input data */ + size = inputlen - iv_len; + if (size) + { + uint32_t aad_offset = aad_len ? passthrough_len : 0; + i = oct_crypto_fill_sg_comp_from_iov ( + gather_comp, i, params->src_iov, 0, &size, aad_buf, aad_offset); + if (PREDICT_FALSE (size)) + { + log_err (ocd->dev, "Insufficient buffer space, size %d needed", + size); + return -1; + } + } + } + + in_buffer = m_vaddr; + ((uint16_t *) in_buffer)[0] = 0; + ((uint16_t *) in_buffer)[1] = 0; + ((uint16_t *) in_buffer)[2] = clib_host_to_net_u16 (i); + + g_size_bytes = ((i + 3) / 4) * sizeof (struct roc_sglist_comp); + /* + * Output Scatter List + */ + + i = 0; + scatter_comp = + (struct roc_sglist_comp *) ((uint8_t *) gather_comp + g_size_bytes); + + i = oct_crypto_fill_sg_comp ( + scatter_comp, i, (uint64_t) offset_vaddr + ROC_SE_OFF_CTRL_LEN, iv_len); + + /* Add output data */ + if ((!decrypt) && (req_flags & ROC_SE_VALID_MAC_BUF)) + { + size = outputlen - iv_len - mac_len; + if (size) + { + + uint32_t aad_offset = aad_len ? passthrough_len : 0; + + i = oct_crypto_fill_sg_comp_from_iov ( + scatter_comp, i, params->dst_iov, 0, &size, aad_buf, aad_offset); + if (PREDICT_FALSE (size)) + { + log_err (ocd->dev, "Insufficient buffer space, size %d needed", + size); + return -1; + } + } + + /* mac data */ + if (mac_len) + i = + oct_crypto_fill_sg_comp_from_buf (scatter_comp, i, ¶ms->mac_buf); + } + else + { + /* Output including mac */ + size = outputlen - iv_len; + + if (size) + { + uint32_t aad_offset = aad_len ? passthrough_len : 0; + + i = oct_crypto_fill_sg_comp_from_iov ( + scatter_comp, i, params->dst_iov, 0, &size, aad_buf, aad_offset); + + if (PREDICT_FALSE (size)) + { + log_err (ocd->dev, "Insufficient buffer space, size %d needed", + size); + return -1; + } + } + } + ((uint16_t *) in_buffer)[3] = clib_host_to_net_u16 (i); + s_size_bytes = ((i + 3) / 4) * sizeof (struct roc_sglist_comp); + + size = g_size_bytes + s_size_bytes + ROC_SG_LIST_HDR_SIZE; + + /* This is DPTR len in case of SG mode */ + inst->w4.s.dlen = size; + + if (PREDICT_FALSE (size > ROC_SG_MAX_DLEN_SIZE)) + { + log_err (ocd->dev, "Exceeds max supported components. Reduce segments"); + ret = -1; + } + + inst->dptr = (uint64_t) in_buffer; + return ret; +} + +static_always_inline int __attribute__ ((unused)) oct_crypto_sg2_inst_prep (struct roc_se_fc_params *params, struct cpt_inst_s *inst, u64 offset_ctrl, const u8 *iv_s, int iv_len, u8 pack_iv, @@ -298,6 +636,8 @@ oct_crypto_sg2_inst_prep (struct roc_se_fc_params *params, u32 passthrough_len, u32 req_flags, int pdcp_flag, int decrypt) { + oct_crypto_main_t *ocm = &oct_crypto_main; + oct_crypto_dev_t *ocd = ocm->crypto_dev[decrypt]; u32 mac_len = 0, aad_len = 0, size, index, g_size_bytes; struct roc_sg2list_comp *gather_comp, *scatter_comp; void *m_vaddr = params->meta_buf.vaddr; @@ -366,7 +706,7 @@ oct_crypto_sg2_inst_prep (struct roc_se_fc_params *params, if (PREDICT_FALSE (size)) { - log_err (oct_crypto_dev.dev, + log_err (ocd->dev, "Insufficient buffer" " space, size %d needed", size); @@ -392,7 +732,7 @@ oct_crypto_sg2_inst_prep (struct roc_se_fc_params *params, aad_buf, aad_offset); if (PREDICT_FALSE (size)) { - log_err (oct_crypto_dev.dev, + log_err (ocd->dev, "Insufficient buffer space," " size %d needed", size); @@ -429,7 +769,7 @@ oct_crypto_sg2_inst_prep (struct roc_se_fc_params *params, aad_buf, aad_offset); if (PREDICT_FALSE (size)) { - log_err (oct_crypto_dev.dev, + log_err (ocd->dev, "Insufficient buffer space," " size %d needed", size); @@ -456,7 +796,7 @@ oct_crypto_sg2_inst_prep (struct roc_se_fc_params *params, if (PREDICT_FALSE (size)) { - log_err (oct_crypto_dev.dev, + log_err (ocd->dev, "Insufficient buffer space," " size %d needed", size); @@ -478,8 +818,7 @@ oct_crypto_sg2_inst_prep (struct roc_se_fc_params *params, if (PREDICT_FALSE ((scatter_sz >> 4) || (gather_sz >> 4))) { - log_err (oct_crypto_dev.dev, - "Exceeds max supported components. Reduce segments"); + log_err (ocd->dev, "Exceeds max supported components. Reduce segments"); ret = -1; } @@ -491,6 +830,8 @@ oct_crypto_cpt_hmac_prep (u32 flags, u64 d_offs, u64 d_lens, struct roc_se_fc_params *fc_params, struct cpt_inst_s *inst, u8 is_decrypt) { + oct_crypto_main_t *ocm = &oct_crypto_main; + oct_crypto_dev_t *ocd = ocm->crypto_dev[is_decrypt]; u32 encr_data_len, auth_data_len, aad_len = 0; i32 inputlen, outputlen, enc_dlen, auth_dlen; u32 encr_offset, auth_offset, iv_offset = 0; @@ -599,10 +940,9 @@ oct_crypto_cpt_hmac_prep (u32 flags, u64 d_offs, u64 d_lens, if (PREDICT_FALSE ((encr_offset >> 16) || (iv_offset >> 8) || (auth_offset >> 8))) { - log_err (oct_crypto_dev.dev, "Offset not supported"); - log_err (oct_crypto_dev.dev, - "enc_offset: %d, iv_offset : %d, auth_offset: %d", encr_offset, - iv_offset, auth_offset); + log_err (ocd->dev, "Offset not supported"); + log_err (ocd->dev, "enc_offset: %d, iv_offset : %d, auth_offset: %d", + encr_offset, iv_offset, auth_offset); return -1; } @@ -613,13 +953,19 @@ oct_crypto_cpt_hmac_prep (u32 flags, u64 d_offs, u64 d_lens, inst->w4.u64 = cpt_inst_w4.u64; +#ifdef PLATFORM_OCTEON9 + ret = oct_crypto_sg_inst_prep (fc_params, inst, offset_ctrl, src, iv_len, 0, + 0, inputlen, outputlen, passthrough_len, + flags, 0, is_decrypt); +#else ret = oct_crypto_sg2_inst_prep (fc_params, inst, offset_ctrl, src, iv_len, 0, 0, inputlen, outputlen, passthrough_len, flags, 0, is_decrypt); +#endif if (PREDICT_FALSE (ret)) { - log_err (oct_crypto_dev.dev, "sg prep failed"); + log_err (ocd->dev, "sg prep failed"); return -1; } @@ -737,9 +1083,10 @@ oct_cpt_inst_w7_get (oct_crypto_sess_t *sess, struct roc_cpt *roc_cpt) static_always_inline void oct_map_keyindex_to_session (oct_crypto_sess_t *sess, u32 key_index, u8 type) { + oct_crypto_main_t *ocm = &oct_crypto_main; oct_crypto_key_t *ckey; - ckey = vec_elt_at_index (oct_crypto.keys[type], key_index); + ckey = vec_elt_at_index (ocm->keys[type], key_index); ckey->sess = sess; } @@ -748,6 +1095,8 @@ static_always_inline i32 oct_crypto_link_session_update (vlib_main_t *vm, oct_crypto_sess_t *sess, u32 key_index, u8 type) { + oct_crypto_main_t *ocm = &oct_crypto_main; + oct_crypto_dev_t *ocd = ocm->crypto_dev[type]; vnet_crypto_key_t *crypto_key, *auth_key; roc_se_cipher_type enc_type = 0; roc_se_auth_type auth_type = 0; @@ -795,7 +1144,7 @@ oct_crypto_link_session_update (vlib_main_t *vm, oct_crypto_sess_t *sess, digest_len = 32; break; default: - log_err (oct_crypto_dev.dev, + log_err (ocd->dev, "Crypto: Undefined link algo %u specified. Key index %u", key->async_alg, key_index); return -1; @@ -814,8 +1163,8 @@ oct_crypto_link_session_update (vlib_main_t *vm, oct_crypto_sess_t *sess, vec_len (crypto_key->data)); if (rv) { - log_err (oct_crypto_dev.dev, - "Error in setting cipher key for enc type %u", enc_type); + log_err (ocd->dev, "Error in setting cipher key for enc type %u", + enc_type); return -1; } @@ -825,8 +1174,8 @@ oct_crypto_link_session_update (vlib_main_t *vm, oct_crypto_sess_t *sess, vec_len (auth_key->data), digest_len); if (rv) { - log_err (oct_crypto_dev.dev, - "Error in setting auth key for auth type %u", auth_type); + log_err (ocd->dev, "Error in setting auth key for auth type %u", + auth_type); return -1; } @@ -844,6 +1193,8 @@ static_always_inline i32 oct_crypto_aead_session_update (vlib_main_t *vm, oct_crypto_sess_t *sess, u32 key_index, u8 type) { + oct_crypto_main_t *ocm = &oct_crypto_main; + oct_crypto_dev_t *ocd = ocm->crypto_dev[type]; vnet_crypto_key_t *key = vnet_crypto_get_key (key_index); roc_se_cipher_type enc_type = 0; roc_se_auth_type auth_type = 0; @@ -864,7 +1215,7 @@ oct_crypto_aead_session_update (vlib_main_t *vm, oct_crypto_sess_t *sess, digest_len = 16; break; default: - log_err (oct_crypto_dev.dev, + log_err (ocd->dev, "Crypto: Undefined cipher algo %u specified. Key index %u", key->async_alg, key_index); return -1; @@ -874,16 +1225,16 @@ oct_crypto_aead_session_update (vlib_main_t *vm, oct_crypto_sess_t *sess, vec_len (key->data)); if (rv) { - log_err (oct_crypto_dev.dev, - "Error in setting cipher key for enc type %u", enc_type); + log_err (ocd->dev, "Error in setting cipher key for enc type %u", + enc_type); return -1; } rv = roc_se_auth_key_set (&sess->cpt_ctx, auth_type, NULL, 0, digest_len); if (rv) { - log_err (oct_crypto_dev.dev, - "Error in setting auth key for auth type %u", auth_type); + log_err (ocd->dev, "Error in setting auth key for auth type %u", + auth_type); return -1; } @@ -896,13 +1247,15 @@ i32 oct_crypto_session_create (vlib_main_t *vm, vnet_crypto_key_index_t key_index, int op_type) { + oct_crypto_main_t *ocm = &oct_crypto_main; + oct_crypto_dev_t *ocd = ocm->crypto_dev[op_type]; oct_crypto_sess_t *session; vnet_crypto_key_t *key; i32 rv = 0; key = vnet_crypto_get_key (key_index); - session = oct_crypto_session_alloc (vm); + session = oct_crypto_session_alloc (vm, op_type); if (session == NULL) return -1; @@ -917,7 +1270,7 @@ oct_crypto_session_create (vlib_main_t *vm, vnet_crypto_key_index_t key_index, return -1; } - session->crypto_dev = &oct_crypto_dev; + session->crypto_dev = ocd; session->cpt_inst_w7 = oct_cpt_inst_w7_get (session, session->crypto_dev->roc_cpt); @@ -941,6 +1294,7 @@ int oct_crypto_enqueue_enc_dec (vlib_main_t *vm, vnet_crypto_async_frame_t *frame, const u8 is_aead, u8 aad_len, const u8 type) { + oct_crypto_main_t *ocm = &oct_crypto_main; struct cpt_inst_s inst[VNET_CRYPTO_FRAME_SIZE]; u32 i, enq_tail, enc_auth_len, buffer_index; u32 crypto_start_offset, integ_start_offset; @@ -958,7 +1312,7 @@ oct_crypto_enqueue_enc_dec (vlib_main_t *vm, vnet_crypto_async_frame_t *frame, /* GCM packets having 8 bytes of aad and 8 bytes of iv */ u8 aad_iv = 8 + 8; - pend_q = &oct_crypto.pend_q[vlib_get_thread_index ()]; + pend_q = &ocm->pend_q[vlib_get_thread_index ()]; enq_tail = pend_q->enq_tail; @@ -969,7 +1323,7 @@ oct_crypto_enqueue_enc_dec (vlib_main_t *vm, vnet_crypto_async_frame_t *frame, { elts = &frame->elts[i]; buffer_index = frame->buffer_indices[i]; - key = vec_elt_at_index (oct_crypto.keys[type], elts->key_index); + key = vec_elt_at_index (ocm->keys[type], elts->key_index); if (!key->sess) { @@ -1111,6 +1465,7 @@ vnet_crypto_async_frame_t * oct_crypto_frame_dequeue (vlib_main_t *vm, u32 *nb_elts_processed, u32 *enqueue_thread_idx) { + oct_crypto_main_t *ocm = &oct_crypto_main; u32 deq_head, status = VNET_CRYPTO_OP_STATUS_COMPLETED; vnet_crypto_async_frame_elt_t *fe = NULL; oct_crypto_inflight_req_t *infl_req; @@ -1119,7 +1474,7 @@ oct_crypto_frame_dequeue (vlib_main_t *vm, u32 *nb_elts_processed, volatile union cpt_res_s *res; int i; - pend_q = &oct_crypto.pend_q[vlib_get_thread_index ()]; + pend_q = &ocm->pend_q[vlib_get_thread_index ()]; if (!pend_q->n_crypto_inflight) return NULL; @@ -1206,6 +1561,7 @@ oct_init_crypto_engine_handlers (vlib_main_t *vm, vnet_dev_t *dev) int oct_conf_sw_queue (vlib_main_t *vm, vnet_dev_t *dev) { + oct_crypto_main_t *ocm = &oct_crypto_main; extern oct_plt_init_param_t oct_plt_init_param; vnet_device_main_t *vdm = &vnet_device_main; oct_crypto_inflight_req_t *infl_req_queue; @@ -1215,10 +1571,10 @@ oct_conf_sw_queue (vlib_main_t *vm, vnet_dev_t *dev) num_worker_cores = vdm->last_worker_thread_index - vdm->first_worker_thread_index + 1; - oct_crypto.pend_q = oct_plt_init_param.oct_plt_zmalloc ( + ocm->pend_q = oct_plt_init_param.oct_plt_zmalloc ( num_worker_cores * sizeof (oct_crypto_pending_queue_t), CLIB_CACHE_LINE_BYTES); - if (oct_crypto.pend_q == NULL) + if (ocm->pend_q == NULL) { log_err (dev, "Failed to allocate memory for crypto pending queue"); return -1; @@ -1226,22 +1582,22 @@ oct_conf_sw_queue (vlib_main_t *vm, vnet_dev_t *dev) for (i = 0; i <= num_worker_cores; ++i) { - oct_crypto.pend_q[i].n_desc = OCT_CRYPTO_DEFAULT_SW_ASYNC_FRAME_COUNT; + ocm->pend_q[i].n_desc = OCT_CRYPTO_DEFAULT_SW_ASYNC_FRAME_COUNT; - oct_crypto.pend_q[i].req_queue = oct_plt_init_param.oct_plt_zmalloc ( + ocm->pend_q[i].req_queue = oct_plt_init_param.oct_plt_zmalloc ( OCT_CRYPTO_DEFAULT_SW_ASYNC_FRAME_COUNT * sizeof (oct_crypto_inflight_req_t), CLIB_CACHE_LINE_BYTES); - if (oct_crypto.pend_q[i].req_queue == NULL) + if (ocm->pend_q[i].req_queue == NULL) { log_err (dev, "Failed to allocate memory for crypto inflight request"); goto free; } - for (j = 0; j <= oct_crypto.pend_q[i].n_desc; ++j) + for (j = 0; j <= ocm->pend_q[i].n_desc; ++j) { - infl_req_queue = &oct_crypto.pend_q[i].req_queue[j]; + infl_req_queue = &ocm->pend_q[i].req_queue[j]; infl_req_queue->sg_data = oct_plt_init_param.oct_plt_zmalloc ( OCT_SCATTER_GATHER_BUFFER_SIZE * VNET_CRYPTO_FRAME_SIZE, @@ -1257,20 +1613,20 @@ oct_conf_sw_queue (vlib_main_t *vm, vnet_dev_t *dev) free: for (; i >= 0; i--) { - if (oct_crypto.pend_q[i].req_queue == NULL) + if (ocm->pend_q[i].req_queue == NULL) continue; for (; j >= 0; j--) { - infl_req_queue = &oct_crypto.pend_q[i].req_queue[j]; + infl_req_queue = &ocm->pend_q[i].req_queue[j]; if (infl_req_queue->sg_data == NULL) continue; oct_plt_init_param.oct_plt_free (infl_req_queue->sg_data); } - oct_plt_init_param.oct_plt_free (oct_crypto.pend_q[i].req_queue); + oct_plt_init_param.oct_plt_free (ocm->pend_q[i].req_queue); } - oct_plt_init_param.oct_plt_free (oct_crypto.pend_q); + oct_plt_init_param.oct_plt_free (ocm->pend_q); return -1; } diff --git a/src/plugins/dev_octeon/crypto.h b/src/plugins/dev_octeon/crypto.h index 33e33d6340..d668d14134 100644 --- a/src/plugins/dev_octeon/crypto.h +++ b/src/plugins/dev_octeon/crypto.h @@ -9,6 +9,8 @@ #include #include +#define OCT_MAX_N_CPT_DEV 2 + /* CRYPTO_ID, KEY_LENGTH_IN_BYTES, TAG_LEN, AAD_LEN */ #define foreach_oct_crypto_aead_async_alg \ _ (AES_128_GCM, 16, 16, 8) \ @@ -121,9 +123,12 @@ typedef struct typedef struct { + oct_crypto_dev_t *crypto_dev[OCT_MAX_N_CPT_DEV]; oct_crypto_key_t *keys[VNET_CRYPTO_ASYNC_OP_N_TYPES]; oct_crypto_pending_queue_t *pend_q; -} oct_crypto_t; + int n_cpt; + u8 started; +} oct_crypto_main_t; void oct_crypto_key_del_handler (vlib_main_t *vm, vnet_crypto_key_index_t key_index); diff --git a/src/plugins/dev_octeon/init.c b/src/plugins/dev_octeon/init.c index 24af469609..5782fc2394 100644 --- a/src/plugins/dev_octeon/init.c +++ b/src/plugins/dev_octeon/init.c @@ -16,6 +16,7 @@ #include struct roc_model oct_model; +extern oct_crypto_main_t oct_crypto_main; VLIB_REGISTER_LOG_CLASS (oct_log, static) = { .class_name = "octeon", @@ -55,7 +56,9 @@ static struct _ (0xa064, RVU_VF, "Marvell Octeon Resource Virtualization Unit VF"), _ (0xa0f8, LBK_VF, "Marvell Octeon Loopback Unit VF"), _ (0xa0f7, SDP_VF, "Marvell Octeon System DPI Packet Interface Unit VF"), - _ (0xa0f3, CPT_VF, "Marvell Octeon Cryptographic Accelerator Unit VF"), + _ (0xa0f3, O10K_CPT_VF, + "Marvell Octeon-10 Cryptographic Accelerator Unit VF"), + _ (0xa0fe, O9K_CPT_VF, "Marvell Octeon-9 Cryptographic Accelerator Unit VF"), #undef _ }; @@ -187,10 +190,12 @@ oct_init_nix (vlib_main_t *vm, vnet_dev_t *dev) } static int -oct_conf_cpt (vlib_main_t *vm, vnet_dev_t *dev, struct roc_cpt *roc_cpt, +oct_conf_cpt (vlib_main_t *vm, vnet_dev_t *dev, oct_crypto_dev_t *ocd, int nb_lf) { + struct roc_cpt *roc_cpt = ocd->roc_cpt; int rrv; + if ((rrv = roc_cpt_eng_grp_add (roc_cpt, CPT_ENG_TYPE_SE)) < 0) { log_err (dev, "Could not add CPT SE engines"); @@ -221,15 +226,15 @@ oct_conf_cpt (vlib_main_t *vm, vnet_dev_t *dev, struct roc_cpt *roc_cpt, } static vnet_dev_rv_t -oct_conf_cpt_queue (vlib_main_t *vm, vnet_dev_t *dev, struct roc_cpt *roc_cpt) +oct_conf_cpt_queue (vlib_main_t *vm, vnet_dev_t *dev, oct_crypto_dev_t *ocd) { - extern oct_crypto_dev_t oct_crypto_dev; + struct roc_cpt *roc_cpt = ocd->roc_cpt; struct roc_cpt_lmtline *cpt_lmtline; struct roc_cpt_lf *cpt_lf; int rrv; - cpt_lf = &oct_crypto_dev.lf; - cpt_lmtline = &oct_crypto_dev.lmtline; + cpt_lf = &ocd->lf; + cpt_lmtline = &ocd->lmtline; cpt_lf->nb_desc = 8192; cpt_lf->lf_id = 0; @@ -247,30 +252,44 @@ oct_conf_cpt_queue (vlib_main_t *vm, vnet_dev_t *dev, struct roc_cpt *roc_cpt) static vnet_dev_rv_t oct_init_cpt (vlib_main_t *vm, vnet_dev_t *dev) { + oct_crypto_main_t *ocm = &oct_crypto_main; extern oct_plt_init_param_t oct_plt_init_param; oct_device_t *cd = vnet_dev_get_data (dev); - extern oct_crypto_dev_t oct_crypto_dev; + oct_crypto_dev_t *ocd = NULL; int rrv; - oct_crypto_dev.roc_cpt = oct_plt_init_param.oct_plt_zmalloc ( - sizeof (struct roc_cpt), CLIB_CACHE_LINE_BYTES); - oct_crypto_dev.roc_cpt->pci_dev = &cd->plt_pci_dev; + if (ocm->n_cpt == OCT_MAX_N_CPT_DEV || ocm->started) + return VNET_DEV_ERR_NOT_SUPPORTED; + + ocd = oct_plt_init_param.oct_plt_zmalloc (sizeof (oct_crypto_dev_t), + CLIB_CACHE_LINE_BYTES); + + ocd->roc_cpt = oct_plt_init_param.oct_plt_zmalloc (sizeof (struct roc_cpt), + CLIB_CACHE_LINE_BYTES); + ocd->roc_cpt->pci_dev = &cd->plt_pci_dev; - oct_crypto_dev.dev = dev; + ocd->dev = dev; - if ((rrv = roc_cpt_dev_init (oct_crypto_dev.roc_cpt))) + if ((rrv = roc_cpt_dev_init (ocd->roc_cpt))) return cnx_return_roc_err (dev, rrv, "roc_cpt_dev_init"); - if ((rrv = oct_conf_cpt (vm, dev, oct_crypto_dev.roc_cpt, 1))) + if ((rrv = oct_conf_cpt (vm, dev, ocd, 1))) return rrv; - if ((rrv = oct_conf_cpt_queue (vm, dev, oct_crypto_dev.roc_cpt))) + if ((rrv = oct_conf_cpt_queue (vm, dev, ocd))) return rrv; oct_conf_sw_queue (vm, dev); oct_init_crypto_engine_handlers (vm, dev); + if (!ocm->n_cpt) + ocm->crypto_dev[0] = ocd; + + ocm->crypto_dev[1] = ocd; + + ocm->n_cpt++; + return VNET_DEV_OK; } @@ -325,7 +344,8 @@ oct_init (vlib_main_t *vm, vnet_dev_t *dev) case OCT_DEVICE_TYPE_SDP_VF: return oct_init_nix (vm, dev); - case OCT_DEVICE_TYPE_CPT_VF: + case OCT_DEVICE_TYPE_O10K_CPT_VF: + case OCT_DEVICE_TYPE_O9K_CPT_VF: return oct_init_cpt (vm, dev); default: diff --git a/src/plugins/dev_octeon/octeon.h b/src/plugins/dev_octeon/octeon.h index 53908c94f4..b5fd223e76 100644 --- a/src/plugins/dev_octeon/octeon.h +++ b/src/plugins/dev_octeon/octeon.h @@ -37,7 +37,8 @@ typedef enum OCT_DEVICE_TYPE_RVU_VF, OCT_DEVICE_TYPE_LBK_VF, OCT_DEVICE_TYPE_SDP_VF, - OCT_DEVICE_TYPE_CPT_VF, + OCT_DEVICE_TYPE_O10K_CPT_VF, + OCT_DEVICE_TYPE_O9K_CPT_VF, } __clib_packed oct_device_type_t; typedef struct From 84394b33efe471421776a2dfa651924caa5d1fe4 Mon Sep 17 00:00:00 2001 From: Nithinsen Kaithakadan Date: Thu, 18 Apr 2024 15:16:01 +0530 Subject: [PATCH 046/271] octeon: add readme for dev_octeon plugin This patch adds summary, configuration and steps to launch VPP with dev_octeon. Type: docs Signed-off-by: Nithinsen Kaithakadan Signed-off-by: Monendra Singh Kushwaha Change-Id: Ibdd7cff31a74530e9e2a32b7b4f4e0293aab3b90 Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/129293 Tested-by: sa_ip-toolkits-Jenkins Tested-by: sa_ip-sw-jenkins Reviewed-by: Nithinsen Kaithakadan --- src/plugins/dev_octeon/README.md | 87 ++++++++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) create mode 100644 src/plugins/dev_octeon/README.md diff --git a/src/plugins/dev_octeon/README.md b/src/plugins/dev_octeon/README.md new file mode 100644 index 0000000000..eb07076b6d --- /dev/null +++ b/src/plugins/dev_octeon/README.md @@ -0,0 +1,87 @@ +# Octeon device plugin for VPP {#dev_octeon_doc} + +## Overview +This plugin provides native device support for Marvell OCTEON-10 SoCs. +This OCTEON native implementation optimizes the interface between hardware +and VPP fast-path data structures. It integrates the following hardware +accelerators into VPP: +- Network interface controller (aka NIX) for packet ingress and egress + +## Supported SoC +- OCTEON-10 +- OCTEON-9 + + +## Usage +The following steps demonstrate how you may bring up VPP with dev_octeon, on the +OCTEON platform. + +### Setup + +#### Configure NIX on OCTEON +-# Determine NIX PF on OCTEON +``` +# lspci -d 177d::0200 | grep 'a063' + 0002:02:00.0 Ethernet controller: Cavium, Inc. Device a063 (rev 08) + 0002:03:00.0 Ethernet controller: Cavium, Inc. Device a063 (rev 08) +``` + +-# Bind NIX VF to vfio-pci driver +``` +echo 0002:03:00.0 > /sys/bus/pci/devices/0002:03:00.0/driver/unbind +echo 0002:02:00.0 > /sys/bus/pci/devices/0002:02:00.0/driver/unbind + +echo 177d a063 > /sys/bus/pci/drivers/vfio-pci/new_id + +echo 0002:02:00.0 > /sys/bus/pci/drivers/vfio-pci/bind +echo 0002:03:00.0 > /sys/bus/pci/drivers/vfio-pci/bind + +``` + +### Launch VPP +VPP device bringup with dev_octeon is possible either through vppctl commands or +startup conf. + +#### Device bringup using vppctl +Launch VPP with startup conf. + +``` +# vpp -c /etc/vpp/startup.conf +# vppctl -s /run/vpp/cli.sock + _______ _ _ _____ ___ + __/ __/ _ \ (_)__ | | / / _ \/ _ \ + _/ _// // / / / _ \ | |/ / ___/ ___/ + /_/ /____(_)_/\___/ |___/_/ /_/ + + vpp# device attach pci/0002:02:00.0 driver octeon + vpp# device create-interface pci/0002:02:00.0 port 0 num-rx-queues 4 + vpp# device attach pci/0002:03:00.0 driver octeon + vpp# device create-interface pci/0002:03:00.0 port 0 num-rx-queues 4 +``` + +#### Device bringup using startup.conf device section +``` +devices { + dev pci/0002:02:00.0 + { + driver octeon + port 0 + { + name eth0 + num-rx-queues 4 + num-tx-queues 4 + } + } + + dev pci/0002:03:00.0 + { + driver octeon + port 0 + { + name eth1 + num-rx-queues 5 + num-tx-queues 5 + } + } +} +``` From c80bc3e7aba184596fcb25ac2ff881caaa34a409 Mon Sep 17 00:00:00 2001 From: Monendra Singh Kushwaha Date: Mon, 10 Jun 2024 16:40:35 +0530 Subject: [PATCH 047/271] build: update octeon-roc url Type: fix Change-Id: I2c5b6c95e621df3bea1a687151d0ab23d3e0fcb9 Signed-off-by: Monendra Singh Kushwaha Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/129328 Tested-by: sa_ip-toolkits-Jenkins Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/129340 Tested-by: sa_ip-sw-jenkins Reviewed-by: Nithinsen Kaithakadan --- build/external/packages/octeon-roc.mk | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/build/external/packages/octeon-roc.mk b/build/external/packages/octeon-roc.mk index a742c99240..98eb29b95a 100644 --- a/build/external/packages/octeon-roc.mk +++ b/build/external/packages/octeon-roc.mk @@ -3,11 +3,11 @@ # https://spdx.org/licenses/Apache-2.0.html octeon-roc_version := 0.5 -octeon-roc_tarball := octeon-roc-v$(octeon-roc_version).tar.gz -octeon-roc_tarball_md5sum := f25fca80b4394e06716ac12ec54574be +octeon-roc_tarball := v$(octeon-roc_version).tar.gz +octeon-roc_tarball_md5sum := 76bc56c84935da944bbf340fe5283ef0 octeon-roc_tarball_strip_dirs := 1 -octeon-roc_url := https://github.com/MarvellEmbeddedProcessors/marvell-vpp/archive/refs/tags/$(octeon-roc_tarball) +octeon-roc_url := https://github.com/MarvellEmbeddedProcessors/marvell-octeon-roc/archive/refs/tags/$(octeon-roc_tarball) define octeon-roc_config_cmds @true From ac6d4c4a9e084ae8a3e111c77f71b4e9b8cee9f0 Mon Sep 17 00:00:00 2001 From: Harish Malik Date: Fri, 7 Jun 2024 17:04:46 +0530 Subject: [PATCH 048/271] octeon: enable vf device promiscuous mode feature This patch enables promiscuous mode on vf devices except SDP vf and LBK devices Type: feature Change-Id: I2e18c63590f22f99c69500f10b42b64d899249ce Signed-off-by: Kommula Shiva Shankar Signed-off-by: Harish Malik Signed-off-by: Monendra Singh Kushwaha Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/129263 Tested-by: sa_ip-toolkits-Jenkins Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/129334 Tested-by: sa_ip-sw-jenkins Reviewed-by: Nithinsen Kaithakadan --- src/plugins/dev_octeon/port.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/plugins/dev_octeon/port.c b/src/plugins/dev_octeon/port.c index 6b73987e75..b2ab948e0f 100644 --- a/src/plugins/dev_octeon/port.c +++ b/src/plugins/dev_octeon/port.c @@ -403,7 +403,7 @@ oct_validate_config_promisc_mode (vnet_dev_port_t *port, int enable) oct_device_t *cd = vnet_dev_get_data (dev); struct roc_nix *nix = cd->nix; - if (roc_nix_is_vf_or_sdp (nix)) + if (roc_nix_is_sdp (nix) || roc_nix_is_lbk (nix)) return VNET_DEV_ERR_UNSUPPORTED_DEVICE; return VNET_DEV_OK; @@ -423,6 +423,9 @@ oct_op_config_promisc_mode (vlib_main_t *vm, vnet_dev_port_t *port, int enable) return oct_roc_err (dev, rv, "roc_nix_npc_promisc_ena_dis failed"); } + if (!roc_nix_is_pf (nix)) + return VNET_DEV_OK; + rv = roc_nix_mac_promisc_mode_enable (nix, enable); if (rv) { From 93531b065c7afb0598c7a79fb8f426144622b021 Mon Sep 17 00:00:00 2001 From: Nithinsen Kaithakadan Date: Thu, 13 Jun 2024 10:55:11 +0530 Subject: [PATCH 049/271] octeon: fix memory alignment for physmem alloc This patch fix memory alignment to 128byte when allocating memory using physmem alloc in dev_octeon plugin. Type: fix Signed-off-by: Nithinsen Kaithakadan Change-Id: Iab2d372f6a576dc2243c7606e2997516400d2877 Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/129664 Tested-by: sa_ip-sw-jenkins Tested-by: sa_ip-toolkits-Jenkins Reviewed-by: Nithinsen Kaithakadan --- src/plugins/dev_octeon/roc_helper.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/plugins/dev_octeon/roc_helper.c b/src/plugins/dev_octeon/roc_helper.c index 16e0a871a9..c1166b654c 100644 --- a/src/plugins/dev_octeon/roc_helper.c +++ b/src/plugins/dev_octeon/roc_helper.c @@ -75,13 +75,12 @@ oct_drv_physmem_alloc (vlib_main_t *vm, u32 size, u32 align) if (align) { - /* Force cache line alloc in case alignment is less than cache line */ - align = align < CLIB_CACHE_LINE_BYTES ? CLIB_CACHE_LINE_BYTES : align; + /* Force ROC align alloc in case alignment is less than ROC align */ + align = align < ROC_ALIGN ? ROC_ALIGN : align; mem = vlib_physmem_alloc_aligned_on_numa (vm, size, align, 0); } else - mem = - vlib_physmem_alloc_aligned_on_numa (vm, size, CLIB_CACHE_LINE_BYTES, 0); + mem = vlib_physmem_alloc_aligned_on_numa (vm, size, ROC_ALIGN, 0); if (!mem) return NULL; From 31c13aa328037ec4eb48016bc73bc4f130360ade Mon Sep 17 00:00:00 2001 From: Nagendra T P Date: Fri, 23 Aug 2024 13:56:57 +0530 Subject: [PATCH 050/271] ci: add cpt firmware dependency for VPP package Added automatic CPT Firmware dependency in VPP workflow Signed-off-by: Nagendra T P --- .github/workflows/build.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a82b954802..9a30f890b7 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -49,6 +49,11 @@ jobs: mkdir -p "${PWD}/artifacts" git tag --points-at HEAD > /tmp/tags [ -s /tmp/tags ] && PKG_POSTFIX= || PKG_POSTFIX=-devel + if [ $PKG_POSTFIX = -devel ]; then + FW_PKG_POSTFIX= + else + FW_PKG_POSTFIX=$PKG_POSTFIX + fi [ -s /tmp/tags ] && NIGHTLY=false || NIGHTLY=true echo "PKG_VERSION_NAME=`./src/scripts/version | awk -F '-' '{print $1}'`" >> "${PWD}/artifacts/env" echo "MRVL_PKG_VERSION=`cat MRVL_VERSION`" >> "${PWD}/artifacts/env" @@ -136,7 +141,7 @@ jobs: echo 'Package: vpp-'$PKG_VERSION_NAME'-cn10k'$PKG_POSTFIX >> DEBIAN/control echo 'Version: '$MRVL_PKG_VERSION >> DEBIAN/control echo "Maintainer: Jerin Jacob (jerinj@marvell.com)" >> DEBIAN/control - echo 'Depends: python3, python3-ply, dpdk-'$DPDK_BASE_PKG_VERSION'-cn10k (= '$DPDK_PKG_VERSION')' >> DEBIAN/control + echo 'Depends: python3, python3-ply, dpdk-'$DPDK_BASE_PKG_VERSION'-cn10k (= '$DPDK_PKG_VERSION'), cpt-firmware-cn10k'${FW_PKG_POSTFIX} (= '$MRVL_PKG_VERSION')' >> DEBIAN/control echo "Architecture: arm64" >> DEBIAN/control echo "Homepage: https://wiki.fd.io/view/VPP" >> DEBIAN/control echo "Description: Vector Packet Processing (VPP) for Octeon10" >> DEBIAN/control From 8b4e052c837a742a578bb495069b9e79bbaf0e75 Mon Sep 17 00:00:00 2001 From: Nagendra T P Date: Fri, 23 Aug 2024 14:03:07 +0530 Subject: [PATCH 051/271] ci: update dpdk release version Bump DPDK Version to 24.07.0 Signed-off-by: Venkata Ravichandra Mynidi Signed-off-by: Nagendra T P --- DPDK_VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DPDK_VERSION b/DPDK_VERSION index 488ae33498..c6b9cb79cb 100644 --- a/DPDK_VERSION +++ b/DPDK_VERSION @@ -1,2 +1,2 @@ BASE_VERSION=23.11.0 -RELEASE_VERSION=24.05.0 +RELEASE_VERSION=24.07.0 From 4d03704f6b475875e2740bb6517e5f68c96ec278 Mon Sep 17 00:00:00 2001 From: Nagendra T P Date: Mon, 26 Aug 2024 10:34:35 +0530 Subject: [PATCH 052/271] ci: fix depends field Fixed Depends field to resolve packaging issue Signed-off-by: Nagendra T P --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9a30f890b7..c55486a82b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -141,7 +141,7 @@ jobs: echo 'Package: vpp-'$PKG_VERSION_NAME'-cn10k'$PKG_POSTFIX >> DEBIAN/control echo 'Version: '$MRVL_PKG_VERSION >> DEBIAN/control echo "Maintainer: Jerin Jacob (jerinj@marvell.com)" >> DEBIAN/control - echo 'Depends: python3, python3-ply, dpdk-'$DPDK_BASE_PKG_VERSION'-cn10k (= '$DPDK_PKG_VERSION'), cpt-firmware-cn10k'${FW_PKG_POSTFIX} (= '$MRVL_PKG_VERSION')' >> DEBIAN/control + echo 'Depends: python3, python3-ply, dpdk-'$DPDK_BASE_PKG_VERSION'-cn10k (= '$DPDK_PKG_VERSION'), cpt-firmware-cn10k'${FW_PKG_POSTFIX}' (= '$MRVL_PKG_VERSION')' >> DEBIAN/control echo "Architecture: arm64" >> DEBIAN/control echo "Homepage: https://wiki.fd.io/view/VPP" >> DEBIAN/control echo "Description: Vector Packet Processing (VPP) for Octeon10" >> DEBIAN/control From 17c211540a7896b9e139399c7eaf1c17f4a851bb Mon Sep 17 00:00:00 2001 From: Nagendra T P Date: Tue, 27 Aug 2024 16:06:57 +0530 Subject: [PATCH 053/271] ci: fix uninitilized FW_PKG_POSTFIX error Initialize FW_PKG_POSTFIX to give it global scope Signed-off-by: Nagendra T P --- .github/workflows/build.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c55486a82b..ac7dd738e8 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -49,8 +49,9 @@ jobs: mkdir -p "${PWD}/artifacts" git tag --points-at HEAD > /tmp/tags [ -s /tmp/tags ] && PKG_POSTFIX= || PKG_POSTFIX=-devel + FW_PKG_POSTFIX="" if [ $PKG_POSTFIX = -devel ]; then - FW_PKG_POSTFIX= + FW_PKG_POSTFIX="" else FW_PKG_POSTFIX=$PKG_POSTFIX fi From 9a01407a28ac45c4a6e8c49f4982c8662778a9ae Mon Sep 17 00:00:00 2001 From: Nagendra T P Date: Tue, 27 Aug 2024 19:36:30 +0530 Subject: [PATCH 054/271] ci: add FW_PKG_POSTFIX to artifacts Signed-off-by: Nagendra T P --- .github/workflows/build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ac7dd738e8..f4ae5ef522 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -55,6 +55,7 @@ jobs: else FW_PKG_POSTFIX=$PKG_POSTFIX fi + echo "FW_PKG_POSTFIX=${FW_PKG_POSTFIX}" >> "${PWD}/artifacts/env" [ -s /tmp/tags ] && NIGHTLY=false || NIGHTLY=true echo "PKG_VERSION_NAME=`./src/scripts/version | awk -F '-' '{print $1}'`" >> "${PWD}/artifacts/env" echo "MRVL_PKG_VERSION=`cat MRVL_VERSION`" >> "${PWD}/artifacts/env" From 1fda612b471c9a5d408fe64ce83b4985eb6a45c4 Mon Sep 17 00:00:00 2001 From: Aakash Date: Mon, 1 Aug 2022 04:59:02 -0400 Subject: [PATCH 055/271] linux-cp: add support for xfrm netlink notifcation This patch contains changes to add support for handlng xfrm notifications in linux-cp plugin. Type: feature JIRA: https://essjira.marvell.com/browse/IPBUSW-39410 Signed-off-by: Bheemappa Agasimundin Signed-off-by: Aakash Change-Id: I49d5fbaae26608490c43ff01bacabcda9528e3bf Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/82978 Tested-by: sa_ip-sw-jenkins Reviewed-by: Vijay Ram Inavolu Reviewed-by: Ashish Gupta Reviewed-by: Narayana Prasad Raju Athreya Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/117665 Reviewed-by: Monendra Singh Kushwaha (cherry picked from commit aaa2a7584292cb438b586ee94e8ca36b3642f1f6) Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/121113 Reviewed-by: Shiva Shankar Kommula Tested-by: Shiva Shankar Kommula --- src/plugins/linux-cp/CMakeLists.txt | 4 + src/plugins/linux-cp/lcp_ipsec.c | 1574 +++++++++++++++++++++++++++ src/plugins/linux-cp/lcp_xfrm.h | 159 +++ src/plugins/linux-cp/lcp_xfrm_nl.c | 664 +++++++++++ 4 files changed, 2401 insertions(+) create mode 100644 src/plugins/linux-cp/lcp_ipsec.c create mode 100644 src/plugins/linux-cp/lcp_xfrm.h create mode 100644 src/plugins/linux-cp/lcp_xfrm_nl.c diff --git a/src/plugins/linux-cp/CMakeLists.txt b/src/plugins/linux-cp/CMakeLists.txt index c891689b4b..9ffb7e29fa 100644 --- a/src/plugins/linux-cp/CMakeLists.txt +++ b/src/plugins/linux-cp/CMakeLists.txt @@ -26,6 +26,7 @@ endif() vpp_plugin_find_library(linux-cp LIBNL3_LIB libnl-3.so) vpp_plugin_find_library(linux-cp LIBNL3_ROUTE_LIB libnl-route-3.so.200) +vpp_plugin_find_library(linux-cp LIBNL3_XFRM_LIB libnl-xfrm-3.so.200) include_directories(${LIBNL3_INCLUDE_DIR}/libnl3) include_directories(${LIBMNL_INCLUDE_DIR}) @@ -72,7 +73,10 @@ add_vpp_plugin(linux_nl SOURCES lcp_router.c lcp_nl.c + lcp_ipsec.c + lcp_xfrm_nl.c LINK_LIBRARIES + ${LIBNL3_XFRM_LIB} lcp ) diff --git a/src/plugins/linux-cp/lcp_ipsec.c b/src/plugins/linux-cp/lcp_ipsec.c new file mode 100644 index 0000000000..5b3a460580 --- /dev/null +++ b/src/plugins/linux-cp/lcp_ipsec.c @@ -0,0 +1,1574 @@ +/* + * Copyright (c) 2022 Cisco and/or its affiliates. + * Copyright (c) 2022 Marvell Technology, Inc and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#define NL_XFRM_DBG(...) vlib_log_debug (lcp_xfrm_logger, __VA_ARGS__) +#define NL_XFRM_INFO(...) vlib_log_notice (lcp_xfrm_logger, __VA_ARGS__) +#define NL_XFRM_WARN(...) vlib_log_warn (lcp_xfrm_logger, __VA_ARGS__) +#define NL_XFRM_ERR(...) vlib_log_err (lcp_xfrm_logger, __VA_ARGS__) + +/* Keeping size in sync with libnl lib */ +#define ALGO_NAME 64 +#define INB_PROTECT_POL_PRIO 9999 +#define IS_TUNNEL_MODE_ENABLED !!nm->is_tunnel_mode +/* Size in bytes */ +#define GCM_SALT_SIZE 4 + +#define cpu_to_be(x, bits) \ + if ((bits) == 16) \ + x = clib_host_to_net_u16 (x); \ + else if ((bits) == 32) \ + x = clib_host_to_net_u32 (x); \ + else \ + x = clib_host_to_net_u64 (x); + +/* Random seq number */ +static u32 g_seq = 0; +static vlib_log_class_t lcp_xfrm_logger; +uword *lifetime_by_sa_id; +static int config_bypass = 0; +uword *tun_idx_by_sel_daddr; + +typedef struct sa_life_limits +{ + u64 soft_byte_limit; + u64 hard_byte_limit; + u64 soft_packet_limit; + u64 hard_packet_limit; + u32 sa_id; + + /* Used in tunnel mode */ + int tun_sw_if_idx; + u8 sa_in_tunnel; +} sa_life_limits_t; + +typedef struct policy_db +{ + int tun_sw_if_idx; +} policy_db_t; + +typedef struct sa_expire_req +{ + struct nlmsghdr nlmsg_hdr; + struct xfrm_user_expire xfrm_expire; +} sa_expire_req_nl_t; + +u32 * +get_mcast_addr (u32 *ip6addr) +{ + u32 *maddr = malloc (sizeof (ip6_address_t)); + + maddr[0] = 0xff020000; + maddr[1] = 0x0; + maddr[2] = 0x1; + maddr[3] = (0xff000000) | (0x00ffffff & clib_net_to_host_u32 (ip6addr[3])); + for (int i = 0; i < 4; i++) + cpu_to_be (maddr[i], 32); + return maddr; +} + +static inline u32 +lcp_xfrm_ipsec_sa_id_table (u32 spi, ip_address_t *addr) +{ + u32 hash = addr->version ^ spi; + + switch (addr->version) + { + case AF_IP4: + hash ^= addr->ip.ip4.as_u32; + break; + case AF_IP6: + hash ^= addr->ip.ip6.as_u32[0] ^ addr->ip.ip6.as_u32[1] ^ + addr->ip.ip6.as_u32[2] ^ addr->ip.ip6.as_u32[3]; + break; + } + hash ^= (hash >> 16); + NL_XFRM_DBG ("### sa_id : %x", hash); + return hash; +} + +static inline fib_protocol_t +lcp_xfrm_mk_proto (uint32_t k) +{ + if (AF_INET6 == k) + return (FIB_PROTOCOL_IP6); + return (FIB_PROTOCOL_IP4); +} + +static inline void +lcp_xfrm_mk_ipaddr (const struct nl_addr *xa, ip_address_t *ia) +{ + fib_protocol_t fproto; + + ip_address_reset (ia); + fproto = lcp_xfrm_mk_proto (nl_addr_get_family (xa)); + + ip_address_set (ia, nl_addr_get_binary_addr (xa), + FIB_PROTOCOL_IP4 == fproto ? AF_IP4 : AF_IP6); +} + +void +get_auth_algo (char *alg, int len, ipsec_integ_alg_t *val) +{ + if (!strcmp (alg, "hmac(md5)")) + *val = IPSEC_INTEG_ALG_MD5_96; + else if (!strcmp (alg, "hmac(sha1)")) + *val = IPSEC_INTEG_ALG_SHA1_96; + else if (!strcmp (alg, "hmac(sha256)")) + *val = IPSEC_INTEG_ALG_SHA_256_128; + else if (!strcmp (alg, "hmac(sha384)")) + *val = IPSEC_INTEG_ALG_SHA_384_192; + else if (!strcmp (alg, "hmac(sha512)")) + *val = IPSEC_INTEG_ALG_SHA_512_256; + else + *val = IPSEC_INTEG_N_ALG; +} + +void +get_crypto_algo (char *alg, int len, ipsec_crypto_alg_t *val) +{ + if (!strcmp (alg, "cbc(aes)")) + { + if (len == 128) + *val = IPSEC_CRYPTO_ALG_AES_CBC_128; + else if (len == 192) + *val = IPSEC_CRYPTO_ALG_AES_CBC_192; + else if (len == 256) + *val = IPSEC_CRYPTO_ALG_AES_CBC_256; + else + *val = IPSEC_CRYPTO_N_ALG; + } + + else if (!strcmp (alg, "rfc4106(gcm(aes))")) + { + /* + * Len includes 4Bsalt as well. So remove it to get actual cipher keylen + */ + len -= GCM_SALT_SIZE * 8; + if (len == 128) + *val = IPSEC_CRYPTO_ALG_AES_GCM_128; + else if (len == 192) + *val = IPSEC_CRYPTO_ALG_AES_GCM_192; + else if (len == 256) + *val = IPSEC_CRYPTO_ALG_AES_GCM_256; + else + *val = IPSEC_CRYPTO_N_ALG; + } + + else if (!strcmp (alg, "ctr(aes)")) + { + if (len == 128) + *val = IPSEC_CRYPTO_ALG_AES_CTR_128; + else if (len == 192) + *val = IPSEC_CRYPTO_ALG_AES_CTR_192; + else if (len == 256) + *val = IPSEC_CRYPTO_ALG_AES_CTR_256; + else + *val = IPSEC_CRYPTO_N_ALG; + } + + else if (!strcmp (alg, "cbc(des)")) + *val = IPSEC_CRYPTO_ALG_DES_CBC; + + else if (!strcmp (alg, "cbc(des3-cede)")) + *val = IPSEC_CRYPTO_ALG_3DES_CBC; + else + *val = IPSEC_CRYPTO_N_ALG; +} + +static inline void +update_port_details (ipsec_policy_t *p, u16 sport, u16 dport, u16 sportmask, + u16 dportmask) +{ + if (!sportmask) + { + p->lport.start = 0; + p->lport.stop = 65535; + } + else if (sportmask == 0xffff) + { + /* Linux XFRM doesn't support port ranges */ + p->lport.start = sport; + p->lport.stop = sport; + } + + if (!dportmask) + { + p->rport.start = 0; + p->rport.stop = 65535; + } + else if (dportmask == 0xffff) + { + p->rport.start = dport; + p->rport.stop = dport; + } +} + +static inline void +update_bypass_policy_addrs (ipsec_policy_t *policy) +{ + ip46_address_t start; + ip46_address_t stop; + + clib_memset (&start, (u8) 0, sizeof (ip46_address_t)); + clib_memset (&stop, (u8) ~0, sizeof (ip46_address_t)); + + clib_memcpy_fast (&policy->laddr.start, &start, sizeof (ip46_address_t)); + clib_memcpy_fast (&policy->laddr.stop, &stop, sizeof (ip46_address_t)); + clib_memcpy_fast (&policy->raddr.start, &start, sizeof (ip46_address_t)); + clib_memcpy_fast (&policy->raddr.stop, &stop, sizeof (ip46_address_t)); +} + +static inline void +lcp_xfrm_config_bypass_policies (u32 spd_id, u8 is_add, u8 is_ip6) +{ + int rv; + u32 p_idx; + vlib_main_t *vm = vlib_get_main (); + ipsec_policy_t policy, policy1; + + /* Bypass policies configured only once across all connections */ + if (config_bypass == 1 && is_add) + return; + + /* + * Adding bypass policy one in inb and one in outb direction + * allowing all ranges 0.0.0.0 - 255.255.255.255 + */ + update_bypass_policy_addrs (&policy); + update_port_details (&policy, 0, 65535, 0, 0); + policy.policy = IPSEC_POLICY_ACTION_BYPASS; + policy.protocol = 0; + policy.sa_id = 0; + policy.is_ipv6 = is_ip6; + policy.id = spd_id; + /* + * Setting the least priority for the bypass, which means in outb + * if packet doesn't match any PROTECT policies, then it will always + * hit the BYPASS. + */ + policy.priority = 0; + + clib_memcpy_fast (&policy1, &policy, sizeof (policy1)); + ipsec_policy_mk_type (1, policy.is_ipv6, policy.policy, &policy.type); + + rv = ipsec_add_del_policy (vm, &policy, is_add, &p_idx); + if (!rv) + NL_XFRM_DBG ("bypass policy-index:%d", p_idx); + else + NL_XFRM_ERR ("bypass policy error:%d", rv); + + ipsec_policy_mk_type (0, policy1.is_ipv6, policy1.policy, &policy1.type); + rv = ipsec_add_del_policy (vm, &policy1, is_add, &p_idx); + if (!rv) + NL_XFRM_DBG ("bypass policy-index:%d", p_idx); + else + NL_XFRM_ERR ("bypass policy error:%d", rv); + + config_bypass = !!is_add; + + NL_XFRM_INFO ("Bypass policies %s successfull", + (is_add == 1) ? "addition" : "deletion"); +} + +static inline void +lcp_xfrm_inb_policy_cfg (ip_address_t *t_saddr, ip_address_t *t_daddr, + u32 sa_id, u32 spd_id, u8 is_add) +{ + int rv; + u32 p_idx; + vlib_main_t *vm = vlib_get_main (); + ipsec_policy_t policy; + + if (t_saddr->version == AF_IP4) + { + clib_memcpy_fast (&policy.laddr.start.ip4.as_u32, + &t_saddr->ip.ip4.as_u32, sizeof (ip4_address_t)); + clib_memcpy_fast (&policy.laddr.stop.ip4.as_u32, &t_saddr->ip.ip4.as_u32, + sizeof (ip4_address_t)); + clib_memcpy_fast (&policy.raddr.start.ip4.as_u32, + &t_daddr->ip.ip4.as_u32, sizeof (ip4_address_t)); + clib_memcpy_fast (&policy.raddr.stop.ip4.as_u32, &t_daddr->ip.ip4.as_u32, + sizeof (ip4_address_t)); + policy.is_ipv6 = 0; + } + else + { + clib_memcpy_fast (&policy.laddr.start.ip6.as_u32, + &t_saddr->ip.ip6.as_u32, sizeof (ip6_address_t)); + clib_memcpy_fast (&policy.laddr.stop.ip6.as_u32, &t_saddr->ip.ip6.as_u32, + sizeof (ip6_address_t)); + clib_memcpy_fast (&policy.raddr.start.ip6.as_u32, + &t_daddr->ip.ip6.as_u32, sizeof (ip6_address_t)); + clib_memcpy_fast (&policy.raddr.stop.ip6.as_u32, &t_daddr->ip.ip6.as_u32, + sizeof (ip6_address_t)); + policy.is_ipv6 = 1; + } + + policy.policy = IPSEC_POLICY_ACTION_PROTECT; + /*SA doesn't have details of inner protocol. So set 0 (means accept any)*/ + policy.protocol = 0; + policy.sa_id = sa_id; + policy.id = spd_id; + policy.priority = INB_PROTECT_POL_PRIO; + update_port_details (&policy, 0, 65535, 0, 0); + + ipsec_policy_mk_type (0, policy.is_ipv6, policy.policy, &policy.type); + rv = ipsec_add_del_policy (vm, &policy, is_add, &p_idx); + if (!rv) + NL_XFRM_INFO ("ipsec inb policy %s success %U -> %U sa_id: %x spd_id: %x", + ((is_add) ? "add" : "del"), format_ip_address, t_saddr, + format_ip_address, t_daddr, sa_id, spd_id); + else + NL_XFRM_ERR ( + "ipsec inb policy %s fail(err: %d) %U -> %U sa_id: %x spd_id: %x", + ((is_add) ? "add" : "del"), rv, format_ip_address, t_saddr, + format_ip_address, t_daddr, sa_id, spd_id); +} + +static inline int +lcp_xfrm_get_matching_iface (ip46_address_t *addr, u8 is_ipv6) +{ + vnet_main_t *vnm = vnet_get_main (); + vnet_sw_interface_t *sif; + u8 iface_found; + + pool_foreach (sif, vnm->interface_main.sw_interfaces) + { + iface_found = ip_interface_has_address (sif->sw_if_index, addr, is_ipv6); + if (iface_found) + { + NL_XFRM_DBG ("Found matching IP on interface index :%u", + sif->sw_if_index); + return sif->sw_if_index; + } + } + return ~0; +} + +static inline u32 +lcp_xfrm_create_spd (ip_address_t *saddr, ip_address_t *daddr, u32 spi, + int *sw_if_index, u8 is_outb) +{ + vlib_main_t *vm = vlib_get_main (); + ipsec_main_t *im = &ipsec_main; + int rv, is_ipv6; + ipsec_spd_t *spd0; + u32 spd_id = 0; + uword *ptr; + ip46_address_t *ip46 = NULL; + + ip46 = (is_outb) ? (&saddr->ip) : (&daddr->ip); + is_ipv6 = (saddr->version == AF_IP4) ? 0 : 1; + + *sw_if_index = lcp_xfrm_get_matching_iface (ip46, !is_ipv6); + if (*sw_if_index == ~0) + return ~0; + + ptr = hash_get (im->spd_index_by_sw_if_index, *sw_if_index); + if (ptr) + { + spd0 = pool_elt_at_index (im->spds, ptr[0]); + NL_XFRM_DBG ("Interface already bound to spd_id: %x", spd0->id); + spd_id = spd0->id; + } + else + { + spd_id = lcp_xfrm_ipsec_sa_id_table (spi, daddr); + rv = ipsec_add_del_spd (vm, spd_id, 1); + if (rv) + { + NL_XFRM_ERR ("spd creation failed"); + return ~0; + } + rv = ipsec_set_interface_spd (vm, *sw_if_index, spd_id, 1); + switch (rv) + { + case VNET_API_ERROR_SYSCALL_ERROR_1: + NL_XFRM_ERR ("no such spd-id"); + return ~0; + case VNET_API_ERROR_SYSCALL_ERROR_2: + NL_XFRM_DBG ("spd already assigned"); + break; + } + NL_XFRM_INFO ("Adding bypass for src %U dst %U", format_ip_address, + saddr, format_ip_address, daddr); + lcp_xfrm_config_bypass_policies (spd_id, 1, is_ipv6); + } + return spd_id; +} + +static inline void +lcp_xfrm_delete_spd (u32 spd_id, ip_address_t *saddr, ip_address_t *daddr, + u8 is_ip6, u8 is_outb) +{ + vlib_main_t *vm = vlib_get_main (); + ipsec_main_t *im = &ipsec_main; + u32 spd_index, *policies; + ipsec_spd_t *spd; + ipsec_policy_t *p0 = NULL; + uword *p; + + p = hash_get (im->spd_index_by_spd_id, spd_id); + if (!p) + return; + spd_index = p[0]; + spd = pool_elt_at_index (im->spds, spd_index); +#define _(t, v) \ + vec_foreach (policies, spd->policies[IPSEC_SPD_POLICY_##t]) \ + { \ + p0 = pool_elt_at_index (im->policies, *policies); \ + if (p0) \ + { \ + if (IPSEC_POLICY_ACTION_BYPASS != p0->policy) \ + return; \ + } \ + } + foreach_ipsec_spd_policy_type +#undef _ + if (!p0) return; + /* + * There will not be a notification indicating the termination of a + * tunnel. So once we detect that no more PROTECT policies are left + * in our database, we delete the bypass policies (which was addded + * internally by us) and delete the SPD as well. + */ + lcp_xfrm_config_bypass_policies (spd->id, 0, is_ip6); + ipsec_add_del_spd (vm, spd->id, 0); +} + +static inline u8 +lcp_xfrm_get_sa_direction (ip46_address_t *saddr, ip46_address_t *daddr, + int *sw_if_index, u8 is_ipv6) +{ + u8 is_outb = 0; + + *sw_if_index = lcp_xfrm_get_matching_iface (saddr, !is_ipv6); + if (~0 == *sw_if_index) + { + *sw_if_index = lcp_xfrm_get_matching_iface (daddr, !is_ipv6); + if (~0 == *sw_if_index) + clib_error_return (0, "SA notification doesn't belong to VPP iface"); + } + else + is_outb = 1; + + return is_outb; +} + +static inline int +find_tunnel_db (ip_address_t *saddr, ip_address_t *daddr, u8 dir, u8 is_ipv6) +{ + ipip_tunnel_key_t key; + ipip_tunnel_t *tun; + + key.mode = IPIP_MODE_P2P; + key.fib_index = fib_table_find (fib_ip_proto (is_ipv6), 0); + + if (is_ipv6) + key.transport = IPIP_TRANSPORT_IP6; + else + key.transport = IPIP_TRANSPORT_IP4; + + if (dir) + { + clib_memcpy_fast (&key.src, &saddr->ip, sizeof (ip46_address_t)); + clib_memcpy_fast (&key.dst, &daddr->ip, sizeof (ip46_address_t)); + } + else + { + clib_memcpy_fast (&key.src, &daddr->ip, sizeof (ip46_address_t)); + clib_memcpy_fast (&key.dst, &saddr->ip, sizeof (ip46_address_t)); + } + + tun = ipip_tunnel_db_find (&key); + if (!tun) + { + NL_XFRM_ERR ("Tunnel iface not found"); + return ~0; + } + + NL_XFRM_DBG ("Tunnel iface found in tunnel DB iterface index: %x", + tun->sw_if_index); + return tun->sw_if_index; +} + +static inline void +lcp_xfrm_update_tunnel (ip_address_t *saddr, ip_address_t *daddr, u8 dir, + u32 sa_id, u8 is_ipv6) +{ + u32 sa_out = 0, *sa_ins = NULL; + ipsec_sa_t *sai = NULL, *sao = NULL; + index_t itpi; + u32 sw_if_index; + int rv; + + if (dir) + return; + + sw_if_index = find_tunnel_db (saddr, daddr, dir, is_ipv6); + if (sw_if_index == ~0) + return; + + pool_foreach_index (itpi, ipsec_tun_protect_pool) + { + ipsec_tun_protect_t *itp = + pool_elt_at_index (ipsec_tun_protect_pool, itpi); + if (!itp || (itp->itp_sw_if_index != sw_if_index)) + continue; + sao = ipsec_sa_get (itp->itp_out_sa); + sa_out = sao->id; + FOR_EACH_IPSEC_PROTECT_INPUT_SA ( + itp, sai, if (sa_id != sai->id) vec_add1 (sa_ins, sai->id);) + } + + rv = ipsec_tun_protect_update (sw_if_index, NULL, sa_out, sa_ins); + if (rv) + { + NL_XFRM_ERR ("SA del: Tunnel protect update failure (err: %d)", rv); + return; + } + NL_XFRM_INFO ("Tunnel protect update success (index: %x)", sw_if_index); +} + +void +nl_xfrm_sa_del (struct xfrmnl_sa *sa) +{ + int is_hard = xfrmnl_sa_is_hardexpiry_reached (sa); + u8 fam = xfrmnl_sa_get_family (sa); + u32 spi = xfrmnl_sa_get_spi (sa); + struct nl_addr *dst = xfrmnl_sa_get_daddr (sa); + struct nl_addr *src = xfrmnl_sa_get_saddr (sa); + u8 is_ip6 = (fam == AF_INET) ? 0 : 1; + ip_address_t daddr, saddr; + u8 is_outb = 1; + u32 id = 0; + int sw_if_index, rv = 0; + u32 spd_id = ~0; + + /* + * Dont need to handle EXPIRE due to soft limit as the rekeying will take + * care of installing new and deleting old one. But for hard limit, we + * need to delete SA as part of EXPIRE notification + */ + if ((nl_object_get_msgtype ((struct nl_object *) sa) == XFRM_MSG_EXPIRE) && + (!is_hard)) + return; + + lcp_xfrm_mk_ipaddr (dst, &daddr); + lcp_xfrm_mk_ipaddr (src, &saddr); + + id = lcp_xfrm_ipsec_sa_id_table (spi, &daddr); + + is_outb = + lcp_xfrm_get_sa_direction (&saddr.ip, &daddr.ip, &sw_if_index, is_ip6); + + if (IS_TUNNEL_MODE_ENABLED) + lcp_xfrm_update_tunnel (&saddr, &daddr, is_outb, id, is_ip6); + else if ((sw_if_index != ~0) && (is_outb == 0)) + { + is_outb = 0; + spd_id = lcp_xfrm_create_spd (&saddr, &daddr, spi, &rv, is_outb); + lcp_xfrm_inb_policy_cfg (&saddr, &daddr, id, spd_id, 0); + } + + rv = ipsec_sa_unlock_id (id); + if (rv) + { + NL_XFRM_ERR ("ipsec sa %x del failure(err: %d) %U -> %U", id, rv, + format_ip_address, &saddr, format_ip_address, &daddr); + } + else + { + hash_unset (lifetime_by_sa_id, id); + NL_XFRM_INFO ("ipsec sa %x del success %U -> %U", id, format_ip_address, + &saddr, format_ip_address, &daddr); + } + + if (!IS_TUNNEL_MODE_ENABLED) + lcp_xfrm_delete_spd (spd_id, &saddr, &daddr, is_ip6, is_outb); + return; +} + +ipsec_sa_t * +get_sa_by_sa_id (u32 sa_id) +{ + ipsec_sa_t *sa; + u32 sai; + + pool_foreach_index (sai, ipsec_sa_pool) + { + sa = ipsec_sa_get (sai); + if (!sa) + return NULL; + if (sa_id == sa->id) + return sa; + } + return NULL; +} + +ipsec_sa_t * +get_reverse_sa_by_tun_ip (ip_address_t *saddr, ip_address_t *daddr, u8 is_ipv6, + u8 dir) +{ + sa_life_limits_t *life = NULL; + ipsec_sa_t *sa; + u8 found = 0; + uword *p = NULL; + u32 sai; + + pool_foreach_index (sai, ipsec_sa_pool) + { + sa = ipsec_sa_get (sai); + if (!sa) + return NULL; + + p = hash_get (lifetime_by_sa_id, sa->id); + if (!p) + continue; + + life = (sa_life_limits_t *) p[0]; + + if (!is_ipv6 && + !ip4_address_compare (&daddr->ip.ip4, &sa->tunnel.t_src.ip.ip4) && + !ip4_address_compare (&saddr->ip.ip4, &sa->tunnel.t_dst.ip.ip4) && + !life->sa_in_tunnel) + found = 1; + else if (!ip6_address_compare (&daddr->ip.ip6, + &sa->tunnel.t_src.ip.ip6) && + !ip6_address_compare (&saddr->ip.ip6, + &sa->tunnel.t_dst.ip.ip6) && + !life->sa_in_tunnel) + found = 1; + + if (found) + { + if (dir) + life->sa_in_tunnel = !dir; + else + /* Its enabled only for inb sa */ + life->sa_in_tunnel = 1; + return sa; + } + } + return NULL; +} + +static inline int +lcp_xfrm_create_tunnel (struct xfrmnl_sa *sa, int *sw_if_index, u8 is_ipv6, + u8 dir, ip_address_t *saddr, ip_address_t *daddr, + u32 phy_sw_if_index) +{ + tunnel_encap_decap_flags_t tflags = TUNNEL_ENCAP_DECAP_FLAG_NONE; + u8 fib_index = 0, instance = 0; + clib_error_t *ret; + int rv = 0; + int reqid = xfrmnl_sa_get_reqid (sa); + + /* + * Reqid will be unique and constant for a given connection. + * Hence using the same as tunnel instance + */ + instance = reqid; + + fib_index = fib_table_find (fib_ip_proto (is_ipv6), 0); + + /* If inb sa, then swap the IPs */ + if (!dir) + rv = ipip_add_tunnel (is_ipv6 ? IPIP_TRANSPORT_IP6 : IPIP_TRANSPORT_IP4, + instance, &daddr->ip, &saddr->ip, fib_index, tflags, + IP_DSCP_CS0, TUNNEL_MODE_P2P, (u32 *) sw_if_index); + + else + rv = ipip_add_tunnel (is_ipv6 ? IPIP_TRANSPORT_IP6 : IPIP_TRANSPORT_IP4, + instance, &saddr->ip, &daddr->ip, fib_index, tflags, + IP_DSCP_CS0, TUNNEL_MODE_P2P, (u32 *) sw_if_index); + + if (rv == VNET_API_ERROR_IF_ALREADY_EXISTS) + { + NL_XFRM_DBG ("Tunnel instance %x exists sw_if_idx: %x", instance, + sw_if_index[0]); + return 0; + } + else if (rv == VNET_API_ERROR_INSTANCE_IN_USE) + { + NL_XFRM_ERR ("Tunnel instance %x already in use", instance); + return -1; + } + else if (rv < 0) + { + NL_XFRM_ERR ("Tunnel addition failed(err: %d) for %U->%U", rv, + format_ip_address, saddr, format_ip_address, daddr); + return -1; + } + + NL_XFRM_INFO ("Tunnel instance %x created succesfully.. index: %x", instance, + *sw_if_index); + ret = vnet_sw_interface_set_flags (vnet_get_main (), *sw_if_index, + VNET_SW_INTERFACE_FLAG_ADMIN_UP); + if (ret) + { + NL_XFRM_ERR ("Error setting flags on tunnel"); + return -1; + } + + vnet_sw_interface_update_unnumbered (*sw_if_index, phy_sw_if_index, 1); + return 0; +} + +static inline void +lcp_xfrm_protect_tunnel (int sw_if_index, u8 is_ipv6, u32 sa_id, u8 dir, + ip_address_t *saddr, ip_address_t *daddr) +{ + u32 sa_out = 0, sa_in = 0, *sa_ins = NULL; + ipsec_sa_t *sa; + + if (dir == 0) + { + sa_in = sa_id; + sa = get_reverse_sa_by_tun_ip (saddr, daddr, is_ipv6, dir); + /* Outb sa is not yet configured */ + if (!sa) + { + return; + } + sa_out = sa->id; + } + else + { + sa_out = sa_id; + sa = get_reverse_sa_by_tun_ip (saddr, daddr, is_ipv6, dir); + /* Inb sa is not yet configured */ + if (!sa) + { + return; + } + sa_in = sa->id; + } + + index_t itpi; + ipsec_sa_t *sai = NULL; + pool_foreach_index (itpi, ipsec_tun_protect_pool) + { + ipsec_tun_protect_t *itp = + pool_elt_at_index (ipsec_tun_protect_pool, itpi); + if (!itp || (itp->itp_sw_if_index != sw_if_index)) + continue; + FOR_EACH_IPSEC_PROTECT_INPUT_SA ( + itp, sai, if (sai && (sa_in != sai->id)) { + if (vec_len (sa_ins) < ITP_MAX_N_SA_IN) + vec_add1 (sa_ins, sai->id); + }) + } + + /* Adding the curent inb sa to tunnel */ + vec_add1 (sa_ins, sa_in); + + int rv = ipsec_tun_protect_update (sw_if_index, NULL, sa_out, sa_ins); + if (rv) + NL_XFRM_ERR ("Tunnel protect update failure (err: %d)", rv); + else + NL_XFRM_INFO ("Tunnel protect update success for index : %d)", + sw_if_index); +} + +static inline void +lcp_xfrm_configure_route_mode (struct xfrmnl_sa *sa, u8 is_ipv6, + ip_address_t *saddr, ip_address_t *daddr, + u32 sa_id, u8 dir, u32 phy_sw_if_idx) +{ + int ret, sw_if_index = -1; + uword *p = NULL; + + ret = lcp_xfrm_create_tunnel (sa, &sw_if_index, is_ipv6, dir, saddr, daddr, + phy_sw_if_idx); + if (ret < 0) + return; + + p = hash_get (lifetime_by_sa_id, sa_id); + if (!p) + return; + ((sa_life_limits_t *) p[0])->tun_sw_if_idx = sw_if_index; + + lcp_xfrm_protect_tunnel (sw_if_index, is_ipv6, sa_id, dir, saddr, daddr); +} + +static inline void +nl_xfrm_sa_add (struct xfrmnl_sa *sa) +{ + struct xfrmnl_ltime_cfg *lifetimes = xfrmnl_sa_get_lifetime_cfg (sa); + ipsec_crypto_alg_t crypto_alg = IPSEC_CRYPTO_ALG_NONE; + ipsec_integ_alg_t integ_alg = IPSEC_INTEG_ALG_NONE; + ipsec_sa_flags_t flags = IPSEC_SA_FLAG_NONE; + char key[IPSEC_KEY_MAX_LEN], auth_key[IPSEC_KEY_MAX_LEN]; + char alg_name[ALGO_NAME], auth_alg_name[ALGO_NAME]; + struct nl_addr *dst = xfrmnl_sa_get_daddr (sa); + struct nl_addr *src = xfrmnl_sa_get_saddr (sa); + unsigned int udp_src, udp_dst, encap_type; + sa_life_limits_t *life = NULL, lifetime; + unsigned int key_len, auth_key_len; + ipsec_key_t ck = { 0 }, ik = { 0 }; + u32 spi = xfrmnl_sa_get_spi (sa); + struct nl_addr *encap_oa = NULL; + u8 ip_family, mode, is_ipv6, dir; + u32 salt = 0, icv = 0, sai = 0, id = 0; + ipsec_protocol_t proto = 0; + ip_address_t saddr, daddr; + int if_idx, sw_if_index; + tunnel_t tun = {}; + u32 spd_id; + int rv; + + lcp_xfrm_mk_ipaddr (dst, &daddr); + lcp_xfrm_mk_ipaddr (src, &saddr); + + id = lcp_xfrm_ipsec_sa_id_table (spi, &daddr); + + /* + * Ideal case, this scenaio will never be hit. But when reading SA + * notification from sk_xfrm socket fails and we initate a sync,then there is + * a possibility that we get notification for the one already present in VPP. + * Hence the check + */ + if (get_sa_by_sa_id (id)) + goto error; + + if (xfrmnl_sa_get_flags (sa) & XFRM_STATE_ESN) + { + flags |= IPSEC_SA_FLAG_USE_ESN; + } + /* + * Kernel SA XFRM doesnt have a flag for AR config. So a non-zero + * replay window size indicates AR is enabled.Also replay window + * size is fixed to 64 in VPP (IPSEC_SA_ANTI_REPLAY_WINDOW_SIZE). + * So its not configurable + */ + + if (xfrmnl_sa_get_replay_window (sa)) + { + flags |= IPSEC_SA_FLAG_USE_ANTI_REPLAY; + } + + xfrmnl_sa_get_encap_tmpl (sa, &encap_type, &udp_src, &udp_dst, &encap_oa); + if (encap_type == UDP_ENCAP_ESPINUDP) + { + flags |= IPSEC_SA_FLAG_UDP_ENCAP; + } + else + udp_src = udp_dst = IPSEC_UDP_PORT_NONE; + + lifetime.soft_byte_limit = xfrmnl_ltime_cfg_get_soft_bytelimit (lifetimes); + lifetime.hard_byte_limit = xfrmnl_ltime_cfg_get_hard_bytelimit (lifetimes); + lifetime.soft_packet_limit = + xfrmnl_ltime_cfg_get_soft_packetlimit (lifetimes); + lifetime.hard_packet_limit = + xfrmnl_ltime_cfg_get_hard_packetlimit (lifetimes); + lifetime.sa_id = id; + lifetime.sa_in_tunnel = 0; + lifetime.tun_sw_if_idx = ~0; + + proto = + (50 == xfrmnl_sa_get_proto (sa)) ? IPSEC_PROTOCOL_ESP : IPSEC_PROTOCOL_AH; + ip_family = xfrmnl_sa_get_family (sa); + + if (-1 != xfrmnl_sa_get_aead_params (sa, alg_name, &key_len, &icv, key)) + flags |= IPSEC_SA_FLAG_IS_AEAD; + else + { + if (-1 == xfrmnl_sa_get_crypto_params (sa, alg_name, &key_len, key)) + { + NL_XFRM_ERR ("crypto param extraction failed"); + goto error; + } + if (-1 == xfrmnl_sa_get_auth_params (sa, auth_alg_name, &auth_key_len, + NULL, auth_key)) + { + NL_XFRM_ERR ("auth param extraction failed"); + goto error; + } + + get_auth_algo (auth_alg_name, auth_key_len, &integ_alg); + if (integ_alg == IPSEC_INTEG_N_ALG) + { + NL_XFRM_ERR ("Invalid/Unsupported integ algo: %s keylen: %u", + auth_alg_name, auth_key_len); + goto error; + } + ik.len = auth_key_len / 8; + clib_memcpy_fast (ik.data, (u8 *) auth_key, (auth_key_len / 8)); + } + + get_crypto_algo (alg_name, key_len, &crypto_alg); + if (crypto_alg == IPSEC_CRYPTO_N_ALG) + { + NL_XFRM_ERR ("Invalid/Unsupported crypto algo: %s keylen: %u", alg_name, + key_len); + goto error; + } + + /* + * Key_len/key here includes salt size/value. As per rfc5282 + * GCM salt size will be 4B which will be after cipher key + */ + if (IPSEC_CRYPTO_ALG_IS_GCM (crypto_alg)) + { + key_len -= GCM_SALT_SIZE * 8; + clib_memcpy_fast (&salt, ((u8 *) key) + (key_len / 8), GCM_SALT_SIZE); + } + /* + * Else for CCM if supported, salt size would be 3B and needs + * to be handled here accordingly + */ + ck.len = key_len / 8; + clib_memcpy_fast (ck.data, (u8 *) key, (key_len / 8)); + + is_ipv6 = (ip_family == AF_INET) ? 0 : 1; + + dir = + lcp_xfrm_get_sa_direction (&saddr.ip, &daddr.ip, &sw_if_index, is_ipv6); + + if (!dir) + flags |= IPSEC_SA_FLAG_IS_INBOUND; + + if (IS_TUNNEL_MODE_ENABLED) + { + /* + * Other tunnel flags of VPP defined under + * foreach_tunnel_encap_decap_flag are not supported by Strongswan/XFRM + * NLs. + */ + if (!(xfrmnl_sa_get_flags (sa) & XFRM_SA_XFLAG_DONT_ENCAP_DSCP)) + { + tun.t_encap_decap_flags |= TUNNEL_ENCAP_DECAP_FLAG_ENCAP_COPY_DSCP; + } + if (xfrmnl_sa_get_flags (sa) & XFRM_STATE_NOPMTUDISC) + { + tun.t_encap_decap_flags |= TUNNEL_ENCAP_DECAP_FLAG_ENCAP_COPY_DF; + } + if (xfrmnl_sa_get_flags (sa) & XFRM_STATE_NOECN) + { + if (!dir) + tun.t_encap_decap_flags |= TUNNEL_ENCAP_DECAP_FLAG_DECAP_COPY_ECN; + else + tun.t_encap_decap_flags |= TUNNEL_ENCAP_DECAP_FLAG_ENCAP_COPY_ECN; + } + } + else + { + mode = (XFRM_MODE_TRANSPORT == xfrmnl_sa_get_mode (sa)) ? 0 : 1; + if (mode) + { + flags |= IPSEC_SA_FLAG_IS_TUNNEL; + if (AF_INET6 == ip_family) + flags |= IPSEC_SA_FLAG_IS_TUNNEL_V6; + } + flags |= IPSEC_SA_FLAG_IS_PROTECT; + } + if (ip_family == AF_INET) + { + is_ipv6 = 0; + tun.t_src.version = tun.t_dst.version = AF_IP4; + clib_memcpy_fast (&tun.t_src.ip.ip4.as_u32, &saddr.ip.ip4.as_u32, + sizeof (ip4_address_t)); + clib_memcpy_fast (&tun.t_dst.ip.ip4.as_u32, &daddr.ip.ip4.as_u32, + sizeof (ip4_address_t)); + } + else + { + is_ipv6 = 1; + tun.t_src.version = tun.t_dst.version = AF_IP6; + clib_memcpy_fast (tun.t_src.ip.ip6.as_u32, &saddr.ip.ip6.as_u32, + sizeof (ip6_address_t)); + clib_memcpy_fast (tun.t_dst.ip.ip6.as_u32, &daddr.ip.ip6.as_u32, + sizeof (ip6_address_t)); + } + + rv = ipsec_sa_add_and_lock (id, spi, proto, crypto_alg, &ck, integ_alg, &ik, + flags, salt, udp_src, udp_dst, &tun, &sai); + if (rv) + { + NL_XFRM_ERR ("ipsec sa add %x failure(err: %d) %U -> %U", id, rv, + format_ip_address, &saddr, format_ip_address, &daddr); + goto error; + } + vec_add1 (life, lifetime); + hash_set (lifetime_by_sa_id, id, life); + + NL_XFRM_INFO ("ipsec sa add %x success %U -> %U", id, format_ip_address, + &saddr, format_ip_address, &daddr); + + if (IS_TUNNEL_MODE_ENABLED) + lcp_xfrm_configure_route_mode (sa, is_ipv6, &saddr, &daddr, id, dir, + sw_if_index); + + else if ((sw_if_index != ~0) && (dir == 0)) + { + spd_id = lcp_xfrm_create_spd (&saddr, &daddr, spi, &if_idx, dir); + lcp_xfrm_inb_policy_cfg (&saddr, &daddr, id, spd_id, 1); + } + +error: + return; +} + +static inline void +get_max_addresses_by_prefix (ip_address_t *orig, u8 prefix, ip_address_t *max, + u8 is_ip6) +{ + if (!is_ip6) + { + ip4_prefix_max_address_host_order (&orig->ip.ip4, prefix, &max->ip.ip4); + cpu_to_be (max->ip.ip4.as_u32, 32); + } + else + { + ip6_preflen_to_mask (prefix, &max->ip.ip6); + + for (int i = 0; i < 4; i++) + { + max->ip.ip6.as_u32[i] = ~(max->ip.ip6.as_u32[i]); + max->ip.ip6.as_u32[i] |= orig->ip.ip6.as_u32[i]; + } + } +} + +static inline void +lcp_xfrm_update_addr_ranges (ipsec_policy_t *p, ip_address_t *sel_saddr, + ip_address_t *sel_daddr, u8 sel_saddr_prefix, + u8 sel_daddr_prefix) +{ + ip_address_t sel_stop_saddr, sel_stop_daddr; + + get_max_addresses_by_prefix (sel_saddr, sel_saddr_prefix, &sel_stop_saddr, + p->is_ipv6); + get_max_addresses_by_prefix (sel_daddr, sel_daddr_prefix, &sel_stop_daddr, + p->is_ipv6); + + if (!p->is_ipv6) + { + clib_memcpy_fast (&p->laddr.start.ip4.as_u32, &sel_saddr->ip.ip4.as_u32, + sizeof (ip4_address_t)); + clib_memcpy_fast (&p->laddr.stop.ip4.as_u32, + &sel_stop_saddr.ip.ip4.as_u32, sizeof (ip4_address_t)); + clib_memcpy_fast (&p->raddr.start.ip4.as_u32, &sel_daddr->ip.ip4.as_u32, + sizeof (ip4_address_t)); + clib_memcpy_fast (&p->raddr.stop.ip4.as_u32, + &sel_stop_daddr.ip.ip4.as_u32, sizeof (ip4_address_t)); + } + else + { + clib_memcpy_fast (&p->laddr.start.ip6.as_u32, &sel_saddr->ip.ip6.as_u32, + sizeof (ip6_address_t)); + clib_memcpy_fast (&p->laddr.stop.ip6.as_u32, + &sel_stop_saddr.ip.ip6.as_u32, sizeof (ip6_address_t)); + clib_memcpy_fast (&p->raddr.start.ip6.as_u32, &sel_daddr->ip.ip6.as_u32, + sizeof (ip6_address_t)); + clib_memcpy_fast (&p->raddr.stop.ip6.as_u32, + &sel_stop_daddr.ip.ip6.as_u32, sizeof (ip6_address_t)); + } +} + +static inline int +find_matching_sp (ipsec_policy_t *p, ip_address_t *saddr, ip_address_t *daddr, + u8 sprefix, u8 dprefix, ipsec_spd_policy_type_t type) +{ + ip_address_t saddr_stop, daddr_stop; + u8 matched = 0; + u8 is_ip6 = (saddr->version == AF_IP6) ? 1 : 0; + + get_max_addresses_by_prefix (saddr, sprefix, &saddr_stop, is_ip6); + get_max_addresses_by_prefix (daddr, dprefix, &daddr_stop, is_ip6); + + if (!is_ip6 && !ip4_address_compare (&p->laddr.start.ip4, &saddr->ip.ip4) && + !ip4_address_compare (&p->laddr.stop.ip4, &saddr_stop.ip.ip4) && + !ip4_address_compare (&p->raddr.start.ip4, &daddr->ip.ip4) && + !ip4_address_compare (&p->raddr.stop.ip4, &daddr_stop.ip.ip4) && + (p->type == type)) + matched = 1; + + else if (!ip6_address_compare (&p->laddr.start.ip6, &saddr->ip.ip6) && + !ip6_address_compare (&p->laddr.stop.ip6, &saddr_stop.ip.ip6) && + !ip6_address_compare (&p->raddr.start.ip6, &daddr->ip.ip6) && + !ip6_address_compare (&p->raddr.stop.ip6, &daddr_stop.ip.ip6) && + (p->type == type)) + matched = 1; + if (matched) + { + NL_XFRM_DBG ("Found Matchin policy. Delete it"); + return 1; + } + return 0; +} + +static inline void +lcp_xfrm_del_old_sp (ip_address_t *s_saddr, ip_address_t *s_daddr, + u8 s_sprefix, u8 s_dprefix, ipsec_spd_policy_type_t type) +{ + ipsec_main_t *im = &ipsec_main; + u32 spd_idx, *policies; + ipsec_spd_t *spd; + ipsec_policy_t *p0 = NULL; + int r = 0; + u32 p_idx; + vlib_main_t *vm = vlib_get_main (); + + pool_foreach_index (spd_idx, im->spds) + { + spd = pool_elt_at_index (im->spds, spd_idx); + if (!spd) + return; +#define _(t, v) \ + vec_foreach (policies, spd->policies[IPSEC_SPD_POLICY_##t]) \ + { \ + p0 = pool_elt_at_index (im->policies, *policies); \ + if (!p0) \ + return; \ + if (p0->policy == IPSEC_POLICY_ACTION_BYPASS) \ + continue; \ + r = \ + find_matching_sp (p0, s_saddr, s_daddr, s_sprefix, s_dprefix, type); \ + if (r) \ + goto found; \ + } + foreach_ipsec_spd_policy_type +#undef _ + } + +found: + if (!r) + return; + NL_XFRM_DBG ("Deleting Pol wth spdid:%x and sa_id:%x", p0->id, p0->sa_id); + r = ipsec_add_del_policy (vm, p0, 0, &p_idx); + if (!r) + NL_XFRM_INFO ("ipsec inb policy del success %U -> %U", format_ip_address, + s_saddr, format_ip_address, s_daddr); + else + NL_XFRM_ERR ("ipsec inb policy del fail(err: %d) %U -> %U", r, + format_ip_address, s_saddr, format_ip_address, s_daddr); +} + +static inline void +fib_entry_cfg (ip_address_t *sel_daddr, u8 sel_daddr_prefix, u32 if_idx, + u8 ip6, u8 is_add) +{ + u8 fib_index = 0; + fib_route_path_t *rpath = NULL, path; + fib_prefix_t rpfx; + fib_source_t fib_src = FIB_SOURCE_API; + + clib_memset (&path, 0, sizeof (path)); + path.frp_weight = 1; + path.frp_sw_if_index = if_idx; + vec_add1 (rpath, path); + + rpfx.fp_len = sel_daddr_prefix; + if (!ip6) + { + rpfx.fp_proto = FIB_PROTOCOL_IP4; + memcpy (&rpfx.fp_addr.ip4, &sel_daddr->ip.ip4, sizeof (ip4_address_t)); + } + else + { + rpfx.fp_proto = FIB_PROTOCOL_IP6; + memcpy (&rpfx.fp_addr.ip6, &sel_daddr->ip.ip6, sizeof (ip6_address_t)); + } + + if (!is_add) + fib_table_entry_path_remove2 (fib_index, &rpfx, fib_src, rpath); + else + fib_table_entry_path_add2 (fib_index, &rpfx, fib_src, FIB_ENTRY_FLAG_NONE, + rpath); + vec_free (rpath); +} + +static inline void +lcp_xfrm_tun_cfg_destroy (ip_address_t *sel_daddr, u8 sel_daddr_prefix, + u8 is_ipv6) +{ + int sw_if_index; + uword *p = NULL; + int rv; + + if (!is_ipv6) + p = hash_get (tun_idx_by_sel_daddr, sel_daddr->ip.ip4.as_u32); + else + p = hash_get (tun_idx_by_sel_daddr, + ip6_address_hash_to_u32 (&sel_daddr->ip.ip6)); + + if (!p) + return; + sw_if_index = ((policy_db_t *) p[0])->tun_sw_if_idx; + + if (sw_if_index == ~0) + return; + fib_entry_cfg (sel_daddr, sel_daddr_prefix, sw_if_index, is_ipv6, 0); + + if (!is_ipv6) + hash_unset (tun_idx_by_sel_daddr, sel_daddr->ip.ip4.as_u32); + else + hash_unset (tun_idx_by_sel_daddr, sel_daddr->ip.ip6.as_u32); + + rv = ipsec_tun_protect_del (sw_if_index, NULL); + if (rv) + { + NL_XFRM_ERR ("Tunnel protect del failure (err: %d)", rv); + return; + } + + rv = ipip_del_tunnel (sw_if_index); + if (rv) + NL_XFRM_ERR ("Tunnel deletion failure (err: %d)", rv); + return; +} + +static inline void +nl_xfrm_sp_del (struct xfrmnl_sp *sp) +{ + struct xfrmnl_sel *sel = xfrmnl_sp_get_sel (sp); + struct nl_addr *sel_src = xfrmnl_sel_get_saddr (sel); + struct nl_addr *sel_dst = xfrmnl_sel_get_daddr (sel); + u8 fam = xfrmnl_sel_get_family (sel); + u8 sel_saddr_prefix = xfrmnl_sel_get_prefixlen_s (sel); + u8 sel_daddr_prefix = xfrmnl_sel_get_prefixlen_d (sel); + ip_address_t sel_saddr, sel_daddr; + u8 dir = xfrmnl_sp_get_dir (sp); + ipsec_spd_policy_type_t type; + u8 is_ip6; + + if (dir != XFRM_POLICY_OUT) + return; + + is_ip6 = (fam == AF_INET6) ? 1 : 0; + + lcp_xfrm_mk_ipaddr (sel_dst, &sel_daddr); + lcp_xfrm_mk_ipaddr (sel_src, &sel_saddr); + + if (IS_TUNNEL_MODE_ENABLED) + { + lcp_xfrm_tun_cfg_destroy (&sel_daddr, sel_daddr_prefix, is_ip6); + } + else + { + ipsec_policy_mk_type (1, is_ip6, IPSEC_POLICY_ACTION_PROTECT, &type); + + lcp_xfrm_del_old_sp (&sel_saddr, &sel_daddr, sel_saddr_prefix, + sel_daddr_prefix, type); + } + + return; +} + +static inline void +lcp_xfrm_tun_update_fib (u32 sa_id, ip_address_t *sel_daddr, + u8 sel_daddr_prefix, u8 is_ipv6) +{ + int sw_if_index = -1; + uword *p = NULL; + policy_db_t *pols = NULL, pol; + + p = hash_get (lifetime_by_sa_id, sa_id); + if (!p) + return; + sw_if_index = ((sa_life_limits_t *) p[0])->tun_sw_if_idx; + + pol.tun_sw_if_idx = sw_if_index; + vec_add1 (pols, pol); + if (!is_ipv6) + hash_set (tun_idx_by_sel_daddr, sel_daddr->ip.ip4.as_u32, pols); + else + hash_set (tun_idx_by_sel_daddr, + ip6_address_hash_to_u32 (&sel_daddr->ip.ip6), pols); + + if (sw_if_index != ~0) + fib_entry_cfg (sel_daddr, sel_daddr_prefix, sw_if_index, is_ipv6, 1); +} + +static inline void +nl_xfrm_sp_add (struct xfrmnl_sp *sp, u8 num) +{ + /* User template(tunnel) variables */ + struct xfrmnl_user_tmpl *u_tmpl = xfrmnl_sp_usertemplate_n (sp, (num - 1)); + struct nl_addr *src = xfrmnl_user_tmpl_get_saddr (u_tmpl); + struct nl_addr *dst = xfrmnl_user_tmpl_get_daddr (u_tmpl); + u8 fam = xfrmnl_user_tmpl_get_family (u_tmpl); + u32 spi = xfrmnl_user_tmpl_get_spi (u_tmpl); + ip_address_t saddr, daddr; + u32 sa_id; + + /* Selector variables */ + struct xfrmnl_sel *sel = xfrmnl_sp_get_sel (sp); + struct nl_addr *sel_src = xfrmnl_sel_get_saddr (sel); + struct nl_addr *sel_dst = xfrmnl_sel_get_daddr (sel); + u8 sel_saddr_prefix = xfrmnl_sel_get_prefixlen_s (sel); + u8 sel_daddr_prefix = xfrmnl_sel_get_prefixlen_d (sel); + u16 sel_sportmask = xfrmnl_sel_get_sportmask (sel); + u16 sel_dportmask = xfrmnl_sel_get_dportmask (sel); + u16 sel_dport = xfrmnl_sel_get_dport (sel); + u16 sel_sport = xfrmnl_sel_get_sport (sel); + u8 proto = xfrmnl_sel_get_proto (sel); + ip_address_t sel_saddr, sel_daddr; + + u32 prio = xfrmnl_sp_get_priority (sp); + u8 dir = xfrmnl_sp_get_dir (sp); + u8 is_ipv6 = 0, is_outbound = 0; + vlib_main_t *vm = vlib_get_main (); + ipsec_spd_policy_type_t type; + u32 p_idx, spd_id = 0; + int rv, sw_if_index; + ipsec_policy_t p; + + /* + * Inbound policy additions are handled as part of SA addition. + * Hence we ignore inbound policy notifications from kernel + */ + if (dir != XFRM_POLICY_OUT) + return; + + is_ipv6 = (fam == AF_INET6) ? 1 : 0; + + lcp_xfrm_mk_ipaddr (sel_dst, &sel_daddr); + lcp_xfrm_mk_ipaddr (sel_src, &sel_saddr); + lcp_xfrm_mk_ipaddr (dst, &daddr); + lcp_xfrm_mk_ipaddr (src, &saddr); + + sa_id = lcp_xfrm_ipsec_sa_id_table (spi, &daddr); + + if (IS_TUNNEL_MODE_ENABLED) + { + /* + * Add a fib entry for dest tun selectors via ipipX interface. + */ + lcp_xfrm_tun_update_fib (sa_id, &sel_daddr, sel_daddr_prefix, is_ipv6); + return; + } + + is_outbound = 1; + spd_id = + lcp_xfrm_create_spd (&saddr, &daddr, spi, &sw_if_index, is_outbound); + + if (sw_if_index == ~0) + { + NL_XFRM_ERR ("SP add Notfn is not for vpp interfaces"); + return; + } + + ipsec_policy_mk_type (is_outbound, is_ipv6, IPSEC_POLICY_ACTION_PROTECT, + &type); + lcp_xfrm_del_old_sp (&sel_saddr, &sel_daddr, sel_saddr_prefix, + sel_daddr_prefix, type); + + p.id = spd_id; + p.priority = prio; + p.is_ipv6 = is_ipv6; + + lcp_xfrm_update_addr_ranges (&p, &sel_saddr, &sel_daddr, sel_saddr_prefix, + sel_daddr_prefix); + p.protocol = proto; + update_port_details (&p, sel_sport, sel_dport, sel_sportmask, sel_dportmask); + p.sa_id = sa_id; + p.policy = IPSEC_POLICY_ACTION_PROTECT; + + ipsec_policy_mk_type (is_outbound, p.is_ipv6, p.policy, &p.type); + rv = ipsec_add_del_policy (vm, &p, 1, &p_idx); + if (!rv) + NL_XFRM_INFO ("ipsec %s policy add success %U -> %U sa_id: %x spd_id: %x", + (!is_outbound ? "inb" : "outb"), format_ip_address, + &sel_saddr, format_ip_address, &sel_daddr, sa_id, spd_id); + else + NL_XFRM_ERR ( + "ipsec %s policy add fail(err: %d) %U -> %U sa_id: %x spd_id: %x", + (!is_outbound ? "inb" : "outb"), rv, format_ip_address, &sel_saddr, + format_ip_address, &sel_daddr, sa_id, spd_id); + + return; +} + +void +nl_xfrm_sa_cfg (struct xfrmnl_sa *sa) +{ + switch (nl_object_get_msgtype ((struct nl_object *) sa)) + { + case XFRM_MSG_UPDSA: + case XFRM_MSG_NEWSA: + nl_xfrm_sa_add (sa); + break; + + case XFRM_MSG_EXPIRE: + case XFRM_MSG_DELSA: + nl_xfrm_sa_del (sa); + break; + } +} + +void +nl_xfrm_sp_cfg (struct xfrmnl_sp *sp) +{ + u8 num_user_tmpl = 0; + + switch (nl_object_get_msgtype ((struct nl_object *) sp)) + { + case XFRM_MSG_UPDPOLICY: + case XFRM_MSG_NEWPOLICY: + num_user_tmpl = xfrmnl_sp_get_nusertemplates (sp); + if (!num_user_tmpl) + { + NL_XFRM_DBG ( + "Don't support allow/drop policies notification from Kernel. \ + Number of user template (%u) should be 1. If more than 1 \ + template is found, we choose only the first one", + num_user_tmpl); + return; + } + nl_xfrm_sp_add (sp, num_user_tmpl); + break; + + case XFRM_MSG_DELPOLICY: + /* DEL notification will not have the user template */ + nl_xfrm_sp_del (sp); + break; + } +} + +static inline u8 +build_nl_expire_msg (ipsec_sa_t *sa, u8 is_hard) +{ + sa_expire_req_nl_t expire_req; + + memset (&expire_req, 0, sizeof (expire_req)); + + /* Fill up the nl header */ + expire_req.nlmsg_hdr.nlmsg_len = + NLMSG_LENGTH (sizeof (expire_req.xfrm_expire)); + expire_req.nlmsg_hdr.nlmsg_flags = NLM_F_REQUEST | NLM_F_ACK; + expire_req.nlmsg_hdr.nlmsg_type = XFRM_MSG_EXPIRE; + expire_req.nlmsg_hdr.nlmsg_seq = ++g_seq; + expire_req.nlmsg_hdr.nlmsg_pid = nl_socket_get_local_port (nm->sk_xfrm); + + /* Fill up the xfrm_user_expire with the SA info that expired */ + expire_req.xfrm_expire.hard = is_hard; + expire_req.xfrm_expire.state.flags = XFRM_STATE_AF_UNSPEC; + expire_req.xfrm_expire.state.id.spi = clib_host_to_net_u32 (sa->spi); + expire_req.xfrm_expire.state.mode = (sa->flags & IPSEC_SA_FLAG_IS_TUNNEL) ? + XFRM_MODE_TUNNEL : + XFRM_MODE_TRANSPORT; + expire_req.xfrm_expire.state.id.proto = + (sa->protocol == IPSEC_PROTOCOL_ESP) ? IPPROTO_ESP : IPPROTO_AH; + + expire_req.xfrm_expire.state.family = + (sa->tunnel.t_dst.version == AF_IP4) ? AF_INET : AF_INET6; + + (sa->tunnel.t_dst.version == AF_IP4) ? + clib_memcpy_fast (&expire_req.xfrm_expire.state.id.daddr.a4, + &sa->tunnel.t_dst.ip.ip4.as_u32, + sizeof (ip4_address_t)) : + clib_memcpy_fast (&expire_req.xfrm_expire.state.id.daddr.a6, + &sa->tunnel.t_dst.ip.ip6.as_u32, sizeof (ip6_address_t)); + + (sa->tunnel.t_src.version == AF_IP4) ? + clib_memcpy_fast (&expire_req.xfrm_expire.state.saddr.a4, + &sa->tunnel.t_src.ip.ip4.as_u32, + sizeof (ip4_address_t)) : + clib_memcpy_fast (&expire_req.xfrm_expire.state.saddr.a6, + &sa->tunnel.t_src.ip.ip6.as_u32, sizeof (ip6_address_t)); + + return send_nl_msg (&expire_req.nlmsg_hdr, XFRMGRP_EXPIRE, XFRM_MSG_EXPIRE); +} + +u8 +check_for_expiry () +{ + sa_life_limits_t *life = NULL; + vlib_counter_t count; + uword *p = NULL; + ipsec_sa_t *sa = NULL; + int rv = 0; + + pool_foreach (sa, ipsec_sa_pool) + { + p = hash_get (lifetime_by_sa_id, sa->id); + if (!p) + continue; + life = (sa_life_limits_t *) p[0]; + vlib_get_combined_counter (&ipsec_sa_counters, sa->stat_index, &count); + + if ((count.packets >= life->hard_packet_limit) || + (count.bytes >= life->hard_byte_limit)) + { + NL_XFRM_INFO ( + "HARD EXPIRY said : %x CntPkt: %u SoftPkt: %u HardPkt: %u", sa->id, + count.packets, life->soft_packet_limit, life->hard_packet_limit); + rv = build_nl_expire_msg (sa, 1); + } + else if ((count.packets >= life->soft_packet_limit) || + (count.bytes >= life->soft_byte_limit)) + { + NL_XFRM_INFO ( + "SOFT EXPIRY said : %x CntPkt: %u SoftPkt: %u HardPkt: %u", sa->id, + count.packets, life->soft_packet_limit, life->hard_packet_limit); + rv = build_nl_expire_msg (sa, 0); + } + if (rv) + vlib_zero_combined_counter (&ipsec_sa_counters, sa->stat_index); + } + return 0; +} + +const nl_xfrm_vft_t lcp_xfrm_vft = { + .nvl_rt_xfrm_sa_cfg = { .is_mp_safe = 0, .cb = nl_xfrm_sa_cfg }, + .nvl_rt_xfrm_sp_cfg = { .is_mp_safe = 0, .cb = nl_xfrm_sp_cfg }, +}; + +static clib_error_t * +lcp_xfrm_init (vlib_main_t *vm) +{ + lcp_xfrm_logger = vlib_log_register_class ("linux-cp", "ipsec"); + + nl_xfrm_register_vft (&lcp_xfrm_vft); + + lifetime_by_sa_id = hash_create (0, sizeof (uword)); + tun_idx_by_sel_daddr = hash_create (0, sizeof (uword)); + + return (NULL); +} + +VLIB_INIT_FUNCTION (lcp_xfrm_init) = { + .runs_before = VLIB_INITS ("lcp_nl_xfrm_init"), +}; + +uword +ipsec_xfrm_expire_process (vlib_main_t *vm, vlib_node_runtime_t *node, + vlib_frame_t *frame) +{ + /* init will wake it up */ + vlib_process_wait_for_event (vm); + + while (1) + { + vlib_process_wait_for_event_or_clock (vm, 2); + vlib_process_get_events (vm, NULL); + check_for_expiry (); + } + return 0; +} + +/* + * fd.io coding-style-patch-verification: ON + * + * Local Variables: + * eval: (c-set-style "gnu") + * End: + */ diff --git a/src/plugins/linux-cp/lcp_xfrm.h b/src/plugins/linux-cp/lcp_xfrm.h new file mode 100644 index 0000000000..4d41709df9 --- /dev/null +++ b/src/plugins/linux-cp/lcp_xfrm.h @@ -0,0 +1,159 @@ +/* + * Copyright (c) 2022 Cisco and/or its affiliates. + * Copyright (c) 2022 Marvell Technology, Inc and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#define NL_RT_COMMON uword is_mp_safe +#define NL_RX_BUF_SIZE_DEF (1 << 28) /* 256 MB */ +#define NL_TX_BUF_SIZE_DEF (1 << 20) /* 1 MB */ +#define NL_BATCH_SIZE_DEF (1 << 11) /* 2048 */ +#define NL_BATCH_DELAY_MS_DEF 50 /* 50 ms, max 20 batch/s */ +#define NL_SYNC_BATCH_LIMIT_DEF (1 << 10) /* 1024 */ +#define NL_SYNC_BATCH_DELAY_MS_DEF 20 /* 20ms, max 50 batch/s */ +#define NL_SYNC_ATTEMPT_DELAY_MS_DEF 2000 /* 2s */ + +#define DAY_F64 (1.0 * (24 * 60 * 60)) + +#define NL_DBG(...) vlib_log_debug (nl_xfrm_main.nl_logger, __VA_ARGS__); +#define NL_WARN(...) vlib_log_warn (nl_xfrm_main.nl_logger, __VA_ARGS__); +#define NL_INFO(...) vlib_log_notice (nl_xfrm_main.nl_logger, __VA_ARGS__); +#define NL_ERROR(...) vlib_log_err (nl_xfrm_main.nl_logger, __VA_ARGS__); + +#define FOREACH_XFRM_VFT(__func, __arg) \ + { \ + nl_xfrm_main_t *nm = &nl_xfrm_main; \ + nl_xfrm_vft_t *__nv; \ + vec_foreach (__nv, nm->nl_xfrm_vfts) \ + { \ + if (!__nv->__func.cb) \ + continue; \ + \ + if (!__nv->__func.is_mp_safe) \ + vlib_worker_thread_barrier_sync (vlib_get_main ()); \ + \ + __nv->__func.cb (__arg); \ + \ + if (!__nv->__func.is_mp_safe) \ + vlib_worker_thread_barrier_release (vlib_get_main ()); \ + } \ + } + +typedef void (*nl_rt_sa_cb_t) (struct xfrmnl_sa *sa); +typedef void (*nl_rt_sp_cb_t) (struct xfrmnl_sp *sp); + +typedef struct nl_rt_sa_cfg_t_ +{ + NL_RT_COMMON; + + nl_rt_sa_cb_t cb; +} nl_rt_sa_cfg_t; + +typedef struct nl_rt_sp_cfg_t_ +{ + NL_RT_COMMON; + + nl_rt_sp_cb_t cb; +} nl_rt_sp_cfg_t; + +typedef struct nl_xfrm_vft_t_ +{ + nl_rt_sa_cfg_t nvl_rt_xfrm_sa_cfg; + nl_rt_sp_cfg_t nvl_rt_xfrm_sp_cfg; +} nl_xfrm_vft_t; + +typedef enum nl_status_t_ +{ + NL_STATUS_NOTIF_PROC, + NL_STATUS_SYNC, +} nl_status_t; + +typedef enum nl_event_type_t_ +{ + NL_EVENT_READ, + NL_EVENT_ERR, +} nl_event_type_t; + +typedef struct nl_msg_info +{ + struct nl_msg *msg; +} nl_msg_info_t; + +typedef struct nl_xfrm_main +{ + + nl_status_t nl_status; + struct nl_sock *sk_xfrm; + u8 xfrm_fd; + u8 is_tunnel_mode; + vlib_log_class_t nl_logger; + nl_xfrm_vft_t *nl_xfrm_vfts; + nl_msg_info_t *nl_msg_queue; + uword clib_file_index; + + u32 rx_buf_size; + u32 tx_buf_size; + u32 batch_size; + u32 batch_delay_ms; + + u32 sync_batch_limit; + u32 sync_batch_delay_ms; + u32 sync_attempt_delay_ms; +} nl_xfrm_main_t; + +extern nl_xfrm_main_t *nm; + +extern void nl_xfrm_register_vft (const nl_xfrm_vft_t *nv); +void nl_xfrm_sa_cfg (struct xfrmnl_sa *sa); +void nl_xfrm_sp_cfg (struct xfrmnl_sp *sp); +u8 check_for_expiry (); +int send_nl_msg (struct nlmsghdr *nl_hdr, unsigned int groups, u8 msg_type); +uword ipsec_xfrm_expire_process (vlib_main_t *vm, vlib_node_runtime_t *node, + vlib_frame_t *frame); + +/* + * fd.io coding-style-patch-verification: ON + * + * Local Variables: + * eval: (c-set-style "gnu") + * End: + */ diff --git a/src/plugins/linux-cp/lcp_xfrm_nl.c b/src/plugins/linux-cp/lcp_xfrm_nl.c new file mode 100644 index 0000000000..21497cde9f --- /dev/null +++ b/src/plugins/linux-cp/lcp_xfrm_nl.c @@ -0,0 +1,664 @@ +/* + * Copyright (c) 2022 Cisco and/or its affiliates. + * Copyright (c) 2022 Marvell Technology, Inc and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define _GNU_SOURCE +#include +#include + +#include +#include +#include +#include +#include + +static nl_xfrm_main_t nl_xfrm_main = { + .rx_buf_size = NL_RX_BUF_SIZE_DEF, + .tx_buf_size = NL_TX_BUF_SIZE_DEF, + .batch_size = NL_BATCH_SIZE_DEF, + .batch_delay_ms = NL_BATCH_DELAY_MS_DEF, + .sync_batch_limit = NL_SYNC_BATCH_LIMIT_DEF, + .sync_batch_delay_ms = NL_SYNC_BATCH_DELAY_MS_DEF, + .sync_attempt_delay_ms = NL_SYNC_ATTEMPT_DELAY_MS_DEF, +}; + +nl_xfrm_main_t *nm = &nl_xfrm_main; + +static void lcp_xfrm_nl_open_sync_socket (); +static void lcp_xfrm_nl_close_sync_socket (); +static void lcp_xfrm_nl_open_socket (); +static void lcp_xfrm_nl_close_socket (); + +void +nl_xfrm_register_vft (const nl_xfrm_vft_t *nv) +{ + vec_add1 (nm->nl_xfrm_vfts, *nv); +} + +static void +nl_sp_cfg (struct xfrmnl_sp *sp, void *arg) +{ + FOREACH_XFRM_VFT (nvl_rt_xfrm_sp_cfg, sp); +} +static void +nl_sa_cfg (struct xfrmnl_sa *sa, void *arg) +{ + FOREACH_XFRM_VFT (nvl_rt_xfrm_sa_cfg, sa); +} + +static void +nl_xfrm_dispatch (struct nl_object *obj, void *arg) +{ + /* nothing can be done without interface mappings */ + if (!lcp_itf_num_pairs ()) + return; + + switch (nl_object_get_msgtype (obj)) + { + case XFRM_MSG_EXPIRE: + case XFRM_MSG_UPDSA: + case XFRM_MSG_NEWSA: + case XFRM_MSG_DELSA: + NL_DBG ("######### SA Notification ######### "); + nl_sa_cfg ((struct xfrmnl_sa *) obj, arg); + break; + + case XFRM_MSG_UPDPOLICY: + case XFRM_MSG_NEWPOLICY: + case XFRM_MSG_DELPOLICY: + NL_DBG ("######### SP Notification ######### "); + nl_sp_cfg ((struct xfrmnl_sp *) obj, arg); + break; + + default: + NL_ERROR ("unhandled xfrm notfn: %s %x", nl_object_get_type (obj)); + break; + } +} + +int +send_nl_msg (struct nlmsghdr *nl_hdr, unsigned int groups, u8 msg_type) +{ + int status; + struct nl_sock *sk_xfrm = nm->sk_xfrm; + struct sockaddr_nl nl_addr; + struct iovec iov; + struct msghdr msg; + struct nl_msg *nlmsg; + + nlmsg = nlmsg_alloc_simple (msg_type, NLM_F_REQUEST); + + memset (&msg, 0, sizeof (struct msghdr)); + memset (&iov, 0, sizeof (struct iovec)); + + iov.iov_base = (void *) nl_hdr; + iov.iov_len = nl_hdr->nlmsg_len; + + msg.msg_name = &nl_addr; + msg.msg_namelen = sizeof (nl_addr); + msg.msg_iov = &iov; + msg.msg_iovlen = 1; + + memset (&nl_addr, 0, sizeof (nl_addr)); + nl_addr.nl_family = AF_NETLINK; + nl_addr.nl_groups = groups; + nl_addr.nl_pid = 0; + + status = nl_sendmsg (sk_xfrm, nlmsg, &msg); + if (status < 0) + { + NL_ERROR ("Expiry send failed"); + return 0; + } + + return 1; +} + +static int +nl_xfrm_process_msgs (void) +{ + nl_msg_info_t *msg_info; + int err, n_msgs = 0; + + /* process a batch of messages. break if we hit our limit */ + vec_foreach (msg_info, nm->nl_msg_queue) + { + if ((err = nl_msg_parse (msg_info->msg, nl_xfrm_dispatch, msg_info)) < 0) + NL_INFO ("Unable to parse object: %s", nl_geterror (err)); + nlmsg_free (msg_info->msg); + if (++n_msgs >= nm->batch_size) + break; + } + + /* remove the messages we processed from the head of the queue */ + if (n_msgs) + vec_delete (nm->nl_msg_queue, n_msgs, 0); + + NL_INFO ("Processed %u messages", n_msgs); + + return n_msgs; +} + +static int +lcp_xfrm_nl_send_dump_req (int msg_type) +{ + struct nl_sock *sk_xfrm = nm->sk_xfrm; + int err; + struct rtgenmsg rt_hdr = { + .rtgen_family = AF_UNSPEC, + }; + + err = + nl_send_simple (sk_xfrm, msg_type, NLM_F_DUMP, &rt_hdr, sizeof (rt_hdr)); + + if (err < 0) + { + NL_ERROR ("Unable to send a dump request: %s", nl_geterror (err)); + } + else + NL_INFO ("Dump request sent via socket %d ", nl_socket_get_fd (sk_xfrm)); + + return err; +} + +static int +lcp_xfrm_nl_dump_cb (struct nl_msg *msg, void *arg) +{ + int err; + + if ((err = nl_msg_parse (msg, nl_xfrm_dispatch, NULL)) < 0) + NL_ERROR ("Unable to parse object: %s", nl_geterror (err)); + + return NL_OK; +} + +static int +lcp_xfrm_nl_read (int msg_limit, int *is_done_rcvd) +{ + struct nl_sock *sk_xfrm = nm->sk_xfrm; + struct sockaddr_nl nla; + uint8_t *buf = NULL; + int n_bytes; + struct nlmsghdr *hdr; + struct nl_msg *msg = NULL; + int err = 0; + int done = 0; + int n_msgs = 0; + +continue_reading: + n_bytes = nl_recv (sk_xfrm, &nla, &buf, /* creds */ NULL); + if (n_bytes <= 0) + return n_bytes; + + hdr = (struct nlmsghdr *) buf; + while (nlmsg_ok (hdr, n_bytes)) + { + nlmsg_free (msg); + msg = nlmsg_convert (hdr); + if (!msg) + { + err = -NLE_NOMEM; + goto out; + } + + n_msgs++; + + nlmsg_set_proto (msg, NETLINK_XFRM); + nlmsg_set_src (msg, &nla); + + /* Message that terminates a multipart message. Finish parsing and signal + * the caller that all dump replies have been received + */ + if (hdr->nlmsg_type == NLMSG_DONE) + { + done = 1; + goto out; + } + /* Message to be ignored. Continue parsing */ + else if (hdr->nlmsg_type == NLMSG_NOOP) + ; + /* Message that indicates data was lost. Finish parsing and return an + * error + */ + else if (hdr->nlmsg_type == NLMSG_OVERRUN) + { + err = -NLE_MSG_OVERFLOW; + goto out; + } + /* Message that indicates an error. Finish parsing, extract the error + * code, and return it */ + else if (hdr->nlmsg_type == NLMSG_ERROR) + { + struct nlmsgerr *e = nlmsg_data (hdr); + + if (hdr->nlmsg_len < nlmsg_size (sizeof (*e))) + { + err = -NLE_MSG_TRUNC; + goto out; + } + else if (e->error) + { + err = -nl_syserr2nlerr (e->error); + goto out; + } + /* Message is an acknowledgement (err_code = 0). Continue parsing */ + else + ; + } + /* Message that contains the requested data. Pass it for processing and + * continue parsing + */ + else + { + lcp_xfrm_nl_dump_cb (msg, NULL); + } + + hdr = nlmsg_next (hdr, &n_bytes); + } + + nlmsg_free (msg); + free (buf); + msg = NULL; + buf = NULL; + + if (!done && n_msgs < msg_limit) + goto continue_reading; + +out: + nlmsg_free (msg); + free (buf); + + if (err) + return err; + + *is_done_rcvd = done; + + return n_msgs; +} + +static void +lcp_xfrm_nl_close_sync_socket () +{ + struct nl_sock *sk_xfrm = nm->sk_xfrm; + + if (sk_xfrm) + { + NL_INFO ("Closing netlink synchronization socket %d", + nl_socket_get_fd (sk_xfrm)); + nl_socket_free (sk_xfrm); + nm->sk_xfrm = NULL; + } +} + +static void +lcp_xfrm_nl_open_sync_socket () +{ + struct nl_sock *sk_xfrm; + + /* Allocate a new blocking socket for XFRM that will be used for dump + * requests. Buffer sizes are left default because replies to dump requests + * are flow-controlled and the kernel will not overflow the socket by sending + * these + */ + + nm->sk_xfrm = sk_xfrm = nl_socket_alloc (); + + nl_connect (sk_xfrm, NETLINK_XFRM); + + NL_INFO ("Opened netlink synchronization socket %d", + nl_socket_get_fd (sk_xfrm)); +} + +static inline void +lcp_xfrm_nl_recv_dump_replies () +{ + int is_done = 0, n_msgs; + + do + { + n_msgs = lcp_xfrm_nl_read (nm->sync_batch_limit, &is_done); + if (n_msgs < 0) + { + NL_ERROR ("Error receiving dump replies " + ": %s (%d)", + nl_geterror (n_msgs), n_msgs); + break; + } + else if (n_msgs == 0) + { + NL_ERROR ("EOF while receiving dump replies"); + break; + } + else + NL_INFO ("Processed %u dump replies", n_msgs); + } + while (!is_done); +} + +static inline void +lcp_xfrm_nl_sync () +{ + /* close the xfrm socket listening on XFRM notifications */ + lcp_xfrm_nl_close_socket (); + /* create a new xfrm sync socket only to initiate a DUMP request */ + lcp_xfrm_nl_open_sync_socket (); + + /* get all xfrm cfgs from linux and cfg the same here*/ + lcp_xfrm_nl_send_dump_req (XFRM_MSG_GETSA); + lcp_xfrm_nl_recv_dump_replies (); + lcp_xfrm_nl_send_dump_req (XFRM_MSG_GETPOLICY); + lcp_xfrm_nl_recv_dump_replies (); + + /* close the xfrm sync socket since dump request is handled by now */ + lcp_xfrm_nl_close_sync_socket (); + /* create the xfrm socket to handle XFRM notifications */ + lcp_xfrm_nl_open_socket (); +} + +static uword +nl_xfrm_process (vlib_main_t *vm, vlib_node_runtime_t *node, + vlib_frame_t *frame) +{ + uword event_type; + uword *event_data = 0; + f64 wait_time = DAY_F64; + + while (1) + { + if (nm->nl_status == NL_STATUS_NOTIF_PROC) + { + /* If we process a batch of messages and stop because we reached the + * batch size limit, we want to wake up after the batch delay and + * process more. Otherwise we just want to wait for a read event. + */ + vlib_process_wait_for_event_or_clock (vm, wait_time); + event_type = vlib_process_get_events (vm, &event_data); + vec_reset_length (event_data); + + switch (event_type) + { + /* Process batch of queued messages on timeout or read event + * signal + */ + case ~0: + case NL_EVENT_READ: + nl_xfrm_process_msgs (); + wait_time = (vec_len (nm->nl_msg_queue) != 0) ? + nm->batch_delay_ms * 1e-3 : + DAY_F64; + + break; + case NL_EVENT_ERR: + nm->nl_status = NL_STATUS_SYNC; + break; + default: + NL_ERROR ("Unknown event type: %u", (u32) event_type); + } + } + else if (nm->nl_status == NL_STATUS_SYNC) + { + NL_INFO ("Start sync"); + lcp_xfrm_nl_sync (); + nm->nl_status = NL_STATUS_NOTIF_PROC; + NL_INFO ("Sync done"); + } + else + NL_ERROR ("Unknown status: %d", nm->nl_status); + } + return frame->n_vectors; +} + +VLIB_REGISTER_NODE (nl_xfrm_process_node, static) = { + .function = nl_xfrm_process, + .name = "linux-cp-netlink-xfrm-process", + .type = VLIB_NODE_TYPE_PROCESS, + .process_log2_n_stack_bytes = 17, +}; + +VLIB_REGISTER_NODE (ipsec_xfrm_expire_process_node, static) = { + .function = ipsec_xfrm_expire_process, + .name = "ipsec-xfrm-expire-process", + .type = VLIB_NODE_TYPE_PROCESS, + .process_log2_n_stack_bytes = 17, +}; + +static int +nl_xfrm_cb (struct nl_msg *msg, void *arg) +{ + nl_msg_info_t *msg_info = 0; + + /* queue for later */ + vec_add2 (nm->nl_msg_queue, msg_info, 1); + + msg_info->msg = msg; + nlmsg_get (msg); + + return 0; +} + +int +lcp_nl_xfrm_drain_messages (void) +{ + int err; + + /* Read until there's an error */ + while ((err = nl_recvmsgs_default (nm->sk_xfrm)) > -1) + ; + + /* If there was an error other then EAGAIN, signal process node */ + if (err != -NLE_AGAIN) + vlib_process_signal_event (vlib_get_main (), nl_xfrm_process_node.index, + NL_EVENT_ERR, 0); + else + { + /* If netlink notification processing is active, signal process node + * there were notifications read + */ + if (nm->nl_status == NL_STATUS_NOTIF_PROC) + { + vlib_process_signal_event ( + vlib_get_main (), nl_xfrm_process_node.index, NL_EVENT_READ, 0); + } + } + + return err; +} + +static clib_error_t * +nl_xfrm_read_cb (clib_file_t *f) +{ + int err; + + err = lcp_nl_xfrm_drain_messages (); + if (err < 0 && err != -NLE_AGAIN) + NL_ERROR ("Error reading netlink socket (fd %d): %s (%d)", + f->file_descriptor, nl_geterror (err), err); + + return 0; +} + +static clib_error_t * +nl_xfrm_error_cb (clib_file_t *f) +{ + NL_ERROR ("Error polling netlink socket (fd %d)", f->file_descriptor); + + /* notify process node */ + vlib_process_signal_event (vlib_get_main (), nl_xfrm_process_node.index, + NL_EVENT_ERR, 0); + + return clib_error_return (0, "Error polling netlink socket %d", + f->file_descriptor); +} + +/* Set the RX buffer size to be used on the netlink socket */ +void +lcp_xfrm_nl_set_buffer_size (u32 buf_size) +{ + nm->rx_buf_size = buf_size; + + if (nm->sk_xfrm) + nl_socket_set_buffer_size (nm->sk_xfrm, nm->rx_buf_size, nm->tx_buf_size); +} + +/* Set the batch size - maximum netlink messages to process at one time */ +void +lcp_xfrm_nl_set_batch_size (u32 batch_size) +{ + nm->batch_size = batch_size; +} + +/* Set the batch delay - how long to wait in ms between processing batches */ +void +lcp_xfrm_nl_set_batch_delay (u32 batch_delay_ms) +{ + nm->batch_delay_ms = batch_delay_ms; +} + +static clib_error_t * +lcp_xfrm_itf_pair_config (vlib_main_t *vm, unformat_input_t *input) +{ + u32 buf_size, batch_size, batch_delay_ms; + + while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) + { + if (unformat (input, "enable-route-mode-ipsec")) + nm->is_tunnel_mode = 1; + else if (unformat (input, "nl-rx-buffer-size %u", &buf_size)) + lcp_xfrm_nl_set_buffer_size (buf_size); + else if (unformat (input, "nl-batch-size %u", &batch_size)) + lcp_xfrm_nl_set_batch_size (batch_size); + else if (unformat (input, "nl-batch-delay-ms %u", &batch_delay_ms)) + lcp_xfrm_nl_set_batch_delay (batch_delay_ms); + else + return clib_error_return (0, "invalid netlink option: %U", + format_unformat_error, input); + } + + return NULL; +} + +VLIB_CONFIG_FUNCTION (lcp_xfrm_itf_pair_config, "linux-xfrm-nl"); + +static void +lcp_xfrm_nl_close_socket (void) +{ + /* delete existing fd from epoll fd set */ + if (nm->clib_file_index != ~0) + { + clib_file_main_t *fm = &file_main; + clib_file_t *f = clib_file_get (fm, nm->clib_file_index); + + if (f) + { + NL_INFO ("Stopping poll of fd %u", f->file_descriptor); + fm->file_update (f, UNIX_FILE_UPDATE_DELETE); + } + else + /* stored index was not a valid file, reset stored index to ~0 */ + nm->clib_file_index = ~0; + } + /* If we already created a socket, close/free it */ + if (nm->sk_xfrm) + { + NL_INFO ("Closing netlink socket %d", nl_socket_get_fd (nm->sk_xfrm)); + nl_socket_free (nm->sk_xfrm); + nm->sk_xfrm = NULL; + } +} + +static void +lcp_xfrm_nl_open_socket (void) +{ + int dest_ns_fd = 0, curr_ns_fd = 0; + /* + * Allocate a new socket for xfrm. Notifications do not use sequence + * numbers, disable sequence number checking. + * Define a callback function, which will be called for each + * notification received + */ + dest_ns_fd = lcp_get_default_ns_fd (); + if (dest_ns_fd) + { + curr_ns_fd = open ("/proc/self/ns/net", O_RDONLY); + setns (dest_ns_fd, CLONE_NEWNET); + } + + nm->sk_xfrm = nl_socket_alloc (); + nm->xfrm_fd = nl_socket_get_fd (nm->sk_xfrm); + nl_socket_disable_seq_check (nm->sk_xfrm); + nl_join_groups (nm->sk_xfrm, XFRMGRP_SA | XFRMGRP_POLICY | XFRMGRP_EXPIRE); + nl_connect (nm->sk_xfrm, NETLINK_XFRM); + + /* Set socket in nonblocking mode and increase buffer sizes */ + nl_socket_set_nonblocking (nm->sk_xfrm); + nl_socket_set_buffer_size (nm->sk_xfrm, nm->rx_buf_size, nm->tx_buf_size); + + if (dest_ns_fd && curr_ns_fd >= 0) + { + setns (curr_ns_fd, CLONE_NEWNET); + close (curr_ns_fd); + } + if (nm->clib_file_index == ~0) + { + clib_file_t rt_file = { + .read_function = nl_xfrm_read_cb, + .error_function = nl_xfrm_error_cb, + .file_descriptor = nl_socket_get_fd (nm->sk_xfrm), + .description = format (0, "linux-cp netlink route socket"), + }; + + nm->clib_file_index = clib_file_add (&file_main, &rt_file); + NL_INFO ("Added file %u", nm->clib_file_index); + } + else + /* clib file already created and socket was closed due to error */ + { + clib_file_main_t *fm = &file_main; + clib_file_t *f = clib_file_get (fm, nm->clib_file_index); + + f->file_descriptor = nl_socket_get_fd (nm->sk_xfrm); + fm->file_update (f, UNIX_FILE_UPDATE_ADD); + NL_INFO ("Starting poll of %d", f->file_descriptor); + } + + nl_socket_modify_cb (nm->sk_xfrm, NL_CB_VALID, NL_CB_CUSTOM, nl_xfrm_cb, + NULL); + NL_INFO ("Opened netlink socket %d", nl_socket_get_fd (nm->sk_xfrm)); +} + +clib_error_t * +lcp_nl_xfrm_init (vlib_main_t *vm) +{ + + nm->nl_status = NL_STATUS_NOTIF_PROC; + nm->clib_file_index = ~0; + nm->nl_logger = vlib_log_register_class ("nl", "xfrm"); + + lcp_xfrm_nl_open_socket (); + vlib_process_signal_event (vlib_get_main (), + ipsec_xfrm_expire_process_node.index, 0, 0); + return NULL; +} + +VLIB_INIT_FUNCTION (lcp_nl_xfrm_init) = { + .runs_after = VLIB_INITS ("ipsec_init"), +}; +/* + * fd.io coding-style-patch-verification: ON + * + * Local Variables: + * eval: (c-set-style "gnu") + * End: + */ From 7b778c5bdda51f3f51dea82985ed1e4c20905414 Mon Sep 17 00:00:00 2001 From: Kommula Shiva Shankar Date: Tue, 6 Feb 2024 23:23:14 +0530 Subject: [PATCH 056/271] linux-cp: update code to support api proto changes Type: fix Signed-off-by: Kommula Shiva Shankar Change-Id: I8c1b075dc753155c7bc59712fc51aa3bcdda9d38 Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/121382 Reviewed-by: Monendra Singh Kushwaha --- src/plugins/linux-cp/lcp_ipsec.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plugins/linux-cp/lcp_ipsec.c b/src/plugins/linux-cp/lcp_ipsec.c index 5b3a460580..3e5904454f 100644 --- a/src/plugins/linux-cp/lcp_ipsec.c +++ b/src/plugins/linux-cp/lcp_ipsec.c @@ -998,7 +998,7 @@ nl_xfrm_sa_add (struct xfrmnl_sa *sa) } rv = ipsec_sa_add_and_lock (id, spi, proto, crypto_alg, &ck, integ_alg, &ik, - flags, salt, udp_src, udp_dst, &tun, &sai); + flags, salt, udp_src, udp_dst, 0, &tun, &sai); if (rv) { NL_XFRM_ERR ("ipsec sa add %x failure(err: %d) %U -> %U", id, rv, From 62820752da7d04f5ca5eee13278ee3eb32c90b9a Mon Sep 17 00:00:00 2001 From: Bheemappa Agasimundin Date: Tue, 5 Dec 2023 18:25:33 +0000 Subject: [PATCH 057/271] linux-cp: add ipsec interface support for xfrm This patch adds ipsec interface support for strongswan based SA configuration. Type: feature JIRA: https://essjira.marvell.com/browse/IPBUSW-39410 Signed-off-by: Bheemappa Agasimundin Change-Id: Idf3d965a6e0fe8e2d141b7559b36394d4a4d715d Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/117670 Tested-by: sa_ip-sw-jenkins Reviewed-by: Aakash Kumar Shankarappa Reviewed-by: Monendra Singh Kushwaha (cherry picked from commit 0fc72be05d8839eafaedb686bfe5f28b6042dbe0) Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/121114 Reviewed-by: Shiva Shankar Kommula Tested-by: Shiva Shankar Kommula --- src/plugins/linux-cp/lcp_ipsec.c | 106 ++++++++++++++++++++++++----- src/plugins/linux-cp/lcp_xfrm.h | 10 ++- src/plugins/linux-cp/lcp_xfrm_nl.c | 14 +++- 3 files changed, 111 insertions(+), 19 deletions(-) diff --git a/src/plugins/linux-cp/lcp_ipsec.c b/src/plugins/linux-cp/lcp_ipsec.c index 3e5904454f..79194c2265 100644 --- a/src/plugins/linux-cp/lcp_ipsec.c +++ b/src/plugins/linux-cp/lcp_ipsec.c @@ -24,8 +24,8 @@ /* Keeping size in sync with libnl lib */ #define ALGO_NAME 64 #define INB_PROTECT_POL_PRIO 9999 -#define IS_TUNNEL_MODE_ENABLED !!nm->is_tunnel_mode -/* Size in bytes */ +#define IS_ROUTE_MODE_ENABLED !!nm->is_route_mode +/* size in bytes */ #define GCM_SALT_SIZE 4 #define cpu_to_be(x, bits) \ @@ -67,6 +67,23 @@ typedef struct sa_expire_req struct xfrm_user_expire xfrm_expire; } sa_expire_req_nl_t; +static inline int +lcp_xfrm_is_ipsec_intf_exist (u8 *if_name, u32 *sw_if_index) +{ + vnet_main_t *vnm = vnet_get_main (); + uword *p; + vnet_hw_interface_t *hi; + + p = hash_get (vnm->interface_main.hw_interface_by_name, if_name); + if (!p) + return 0; + + hi = vnet_get_hw_interface (vnm, p[0]); + sw_if_index[0] = hi->sw_if_index; + + return 1; +} + u32 * get_mcast_addr (u32 *ip6addr) { @@ -509,18 +526,27 @@ find_tunnel_db (ip_address_t *saddr, ip_address_t *daddr, u8 dir, u8 is_ipv6) static inline void lcp_xfrm_update_tunnel (ip_address_t *saddr, ip_address_t *daddr, u8 dir, - u32 sa_id, u8 is_ipv6) + u32 sa_id, u8 is_ipv6, struct xfrmnl_sa *sa) { u32 sa_out = 0, *sa_ins = NULL; ipsec_sa_t *sai = NULL, *sao = NULL; index_t itpi; u32 sw_if_index; int rv; + u8 *s = NULL; + u8 instance = xfrmnl_sa_get_reqid (sa); if (dir) return; - sw_if_index = find_tunnel_db (saddr, daddr, dir, is_ipv6); + if (nm->interface_type == NL_INTERFACE_TYPE_IPIP) + sw_if_index = find_tunnel_db (saddr, daddr, dir, is_ipv6); + else + { + s = format (s, "ipsec%d", instance); + lcp_xfrm_is_ipsec_intf_exist (s, &sw_if_index); + vec_free (s); + } if (sw_if_index == ~0) return; @@ -577,8 +603,8 @@ nl_xfrm_sa_del (struct xfrmnl_sa *sa) is_outb = lcp_xfrm_get_sa_direction (&saddr.ip, &daddr.ip, &sw_if_index, is_ip6); - if (IS_TUNNEL_MODE_ENABLED) - lcp_xfrm_update_tunnel (&saddr, &daddr, is_outb, id, is_ip6); + if (IS_ROUTE_MODE_ENABLED) + lcp_xfrm_update_tunnel (&saddr, &daddr, is_outb, id, is_ip6, sa); else if ((sw_if_index != ~0) && (is_outb == 0)) { is_outb = 0; @@ -599,7 +625,7 @@ nl_xfrm_sa_del (struct xfrmnl_sa *sa) &saddr, format_ip_address, &daddr); } - if (!IS_TUNNEL_MODE_ENABLED) + if (!IS_ROUTE_MODE_ENABLED) lcp_xfrm_delete_spd (spd_id, &saddr, &daddr, is_ip6, is_outb); return; } @@ -669,9 +695,48 @@ get_reverse_sa_by_tun_ip (ip_address_t *saddr, ip_address_t *daddr, u8 is_ipv6, } static inline int -lcp_xfrm_create_tunnel (struct xfrmnl_sa *sa, int *sw_if_index, u8 is_ipv6, - u8 dir, ip_address_t *saddr, ip_address_t *daddr, - u32 phy_sw_if_index) +lcp_xfrm_create_ipsec_tunnel (struct xfrmnl_sa *sa, int *sw_if_index, + u32 phy_sw_if_index) +{ + clib_error_t *ret; + int rv = 0; + u8 *s = NULL; + int instance = xfrmnl_sa_get_reqid (sa); + + s = format (s, "ipsec%d", instance); + rv = lcp_xfrm_is_ipsec_intf_exist (s, (u32 *) sw_if_index); + vec_free (s); + + if (rv) + return 0; + + rv = ipsec_itf_create (instance, TUNNEL_MODE_P2P, (u32 *) sw_if_index); + if (rv == VNET_API_ERROR_INVALID_REGISTRATION) + { + NL_XFRM_ERR ("Tunnel instance %x exists sw_if_idx: %x", instance, + sw_if_index[0]); + return 0; + } + + NL_XFRM_DBG ("Tunnel instance %x created succesfully.. index: %x", instance, + *sw_if_index); + + ret = vnet_sw_interface_set_flags (vnet_get_main (), *sw_if_index, + VNET_SW_INTERFACE_FLAG_ADMIN_UP); + if (ret) + { + NL_XFRM_ERR ("Error setting flags on tunnel"); + return -1; + } + + vnet_sw_interface_update_unnumbered (*sw_if_index, phy_sw_if_index, 1); + return 0; +} + +static inline int +lcp_xfrm_create_ipip_tunnel (struct xfrmnl_sa *sa, int *sw_if_index, + u8 is_ipv6, u8 dir, ip_address_t *saddr, + ip_address_t *daddr, u32 phy_sw_if_index) { tunnel_encap_decap_flags_t tflags = TUNNEL_ENCAP_DECAP_FLAG_NONE; u8 fib_index = 0, instance = 0; @@ -794,8 +859,12 @@ lcp_xfrm_configure_route_mode (struct xfrmnl_sa *sa, u8 is_ipv6, int ret, sw_if_index = -1; uword *p = NULL; - ret = lcp_xfrm_create_tunnel (sa, &sw_if_index, is_ipv6, dir, saddr, daddr, - phy_sw_if_idx); + if (nm->interface_type == NL_INTERFACE_TYPE_IPIP) + ret = lcp_xfrm_create_ipip_tunnel (sa, &sw_if_index, is_ipv6, dir, saddr, + daddr, phy_sw_if_idx); + else + ret = lcp_xfrm_create_ipsec_tunnel (sa, &sw_if_index, phy_sw_if_idx); + if (ret < 0) return; @@ -944,7 +1013,7 @@ nl_xfrm_sa_add (struct xfrmnl_sa *sa) if (!dir) flags |= IPSEC_SA_FLAG_IS_INBOUND; - if (IS_TUNNEL_MODE_ENABLED) + if (nm->interface_type == NL_INTERFACE_TYPE_IPIP) { /* * Other tunnel flags of VPP defined under @@ -1011,7 +1080,7 @@ nl_xfrm_sa_add (struct xfrmnl_sa *sa) NL_XFRM_INFO ("ipsec sa add %x success %U -> %U", id, format_ip_address, &saddr, format_ip_address, &daddr); - if (IS_TUNNEL_MODE_ENABLED) + if (IS_ROUTE_MODE_ENABLED) lcp_xfrm_configure_route_mode (sa, is_ipv6, &saddr, &daddr, id, dir, sw_if_index); @@ -1229,7 +1298,10 @@ lcp_xfrm_tun_cfg_destroy (ip_address_t *sel_daddr, u8 sel_daddr_prefix, return; } - rv = ipip_del_tunnel (sw_if_index); + if (nm->interface_type == NL_INTERFACE_TYPE_IPIP) + rv = ipip_del_tunnel (sw_if_index); + else + rv = ipsec_itf_delete (sw_if_index); if (rv) NL_XFRM_ERR ("Tunnel deletion failure (err: %d)", rv); return; @@ -1257,7 +1329,7 @@ nl_xfrm_sp_del (struct xfrmnl_sp *sp) lcp_xfrm_mk_ipaddr (sel_dst, &sel_daddr); lcp_xfrm_mk_ipaddr (sel_src, &sel_saddr); - if (IS_TUNNEL_MODE_ENABLED) + if (IS_ROUTE_MODE_ENABLED) { lcp_xfrm_tun_cfg_destroy (&sel_daddr, sel_daddr_prefix, is_ip6); } @@ -1347,7 +1419,7 @@ nl_xfrm_sp_add (struct xfrmnl_sp *sp, u8 num) sa_id = lcp_xfrm_ipsec_sa_id_table (spi, &daddr); - if (IS_TUNNEL_MODE_ENABLED) + if (IS_ROUTE_MODE_ENABLED) { /* * Add a fib entry for dest tun selectors via ipipX interface. diff --git a/src/plugins/linux-cp/lcp_xfrm.h b/src/plugins/linux-cp/lcp_xfrm.h index 4d41709df9..e7257e431f 100644 --- a/src/plugins/linux-cp/lcp_xfrm.h +++ b/src/plugins/linux-cp/lcp_xfrm.h @@ -34,6 +34,7 @@ #include #include #include +#include #include #include @@ -113,6 +114,12 @@ typedef enum nl_event_type_t_ NL_EVENT_ERR, } nl_event_type_t; +typedef enum nl_interface_type_t_ +{ + NL_INTERFACE_TYPE_IPIP = 1, + NL_INTERFACE_TYPE_IPSEC +} nl_interface_type_t; + typedef struct nl_msg_info { struct nl_msg *msg; @@ -124,7 +131,8 @@ typedef struct nl_xfrm_main nl_status_t nl_status; struct nl_sock *sk_xfrm; u8 xfrm_fd; - u8 is_tunnel_mode; + u8 is_route_mode; + nl_interface_type_t interface_type; vlib_log_class_t nl_logger; nl_xfrm_vft_t *nl_xfrm_vfts; nl_msg_info_t *nl_msg_queue; diff --git a/src/plugins/linux-cp/lcp_xfrm_nl.c b/src/plugins/linux-cp/lcp_xfrm_nl.c index 21497cde9f..4eabf40e6a 100644 --- a/src/plugins/linux-cp/lcp_xfrm_nl.c +++ b/src/plugins/linux-cp/lcp_xfrm_nl.c @@ -530,22 +530,34 @@ static clib_error_t * lcp_xfrm_itf_pair_config (vlib_main_t *vm, unformat_input_t *input) { u32 buf_size, batch_size, batch_delay_ms; + char *tunnel_name = NULL; while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) { if (unformat (input, "enable-route-mode-ipsec")) - nm->is_tunnel_mode = 1; + nm->is_route_mode = 1; else if (unformat (input, "nl-rx-buffer-size %u", &buf_size)) lcp_xfrm_nl_set_buffer_size (buf_size); else if (unformat (input, "nl-batch-size %u", &batch_size)) lcp_xfrm_nl_set_batch_size (batch_size); else if (unformat (input, "nl-batch-delay-ms %u", &batch_delay_ms)) lcp_xfrm_nl_set_batch_delay (batch_delay_ms); + else if (unformat (input, "interface %s", tunnel_name)) + { + if (!clib_strcmp (tunnel_name, "ipsec")) + nm->interface_type = NL_INTERFACE_TYPE_IPSEC; + + vec_free (tunnel_name); + } else return clib_error_return (0, "invalid netlink option: %U", format_unformat_error, input); } + if (nm->interface_type && !nm->is_route_mode) + return clib_error_return ( + 0, "enable-route-mode-ipsec configuration is missing"); + return NULL; } From 5ffb52403e7866297b8bca49a3a5a5ceb3f7bd49 Mon Sep 17 00:00:00 2001 From: Kommula Shiva Shankar Date: Wed, 7 Feb 2024 11:58:17 +0530 Subject: [PATCH 058/271] linux-cp: initialize sw_if_index variable Type: fix Signed-off-by: Kommula Shiva Shankar Change-Id: I4c45f9e8010008c98f511a938cdab48335e4a57b Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/121437 Reviewed-by: Monendra Singh Kushwaha --- src/plugins/linux-cp/lcp_ipsec.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plugins/linux-cp/lcp_ipsec.c b/src/plugins/linux-cp/lcp_ipsec.c index 79194c2265..fe83a948ae 100644 --- a/src/plugins/linux-cp/lcp_ipsec.c +++ b/src/plugins/linux-cp/lcp_ipsec.c @@ -531,7 +531,7 @@ lcp_xfrm_update_tunnel (ip_address_t *saddr, ip_address_t *daddr, u8 dir, u32 sa_out = 0, *sa_ins = NULL; ipsec_sa_t *sai = NULL, *sao = NULL; index_t itpi; - u32 sw_if_index; + u32 sw_if_index = ~0; int rv; u8 *s = NULL; u8 instance = xfrmnl_sa_get_reqid (sa); From b83d5a4f8bcbe340c2986758f43ce6337b134870 Mon Sep 17 00:00:00 2001 From: Mohamed Feroz Date: Thu, 20 Jun 2024 05:43:35 +0000 Subject: [PATCH 059/271] vcl: workaround for multi-threaded vcl application This patch provides a multi-threaded application to share single vcl worker context across multiple threads as long as only single thread handles the session layer for different application sessions Type: fix Signed-off-by: Mohamed Feroz Change-Id: If9ea4803c4b3ec83b4ccf405cb696c5bad984d97 Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/129910 Reviewed-by: Venkata Ravichandra Mynidi Tested-by: sa_ip-sw-jenkins (cherry picked from commit 00dbd75bc1e89fb7748bfa9110ac85bc08e2fd36) Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/130553 Reviewed-by: Nithinsen Kaithakadan --- src/vcl/ldp.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/vcl/ldp.c b/src/vcl/ldp.c index 64155ea3a4..752aa6b076 100644 --- a/src/vcl/ldp.c +++ b/src/vcl/ldp.c @@ -342,13 +342,11 @@ close (int fd) epfd = vls_attr (vlsh, VPPCOM_ATTR_GET_LIBC_EPFD, 0, 0); if (epfd > 0) { - ldp_worker_ctx_t *ldpw = ldp_worker_get_current (); u32 size = sizeof (epfd); LDBG (0, "fd %d: calling libc_close: epfd %u", fd, epfd); libc_close (epfd); - ldpw->mq_epfd_added = 0; epfd = 0; (void) vls_attr (vlsh, VPPCOM_ATTR_SET_LIBC_EPFD, &epfd, &size); From 0babfdb777190c9ac44572d21882aee036d60674 Mon Sep 17 00:00:00 2001 From: Alok Mishra Date: Mon, 24 Jun 2024 15:56:16 +0500 Subject: [PATCH 060/271] octeon: fix link down issue An issue is noticed when the link state of the ethernet interface is toggled to down and again brought up. The link status shows "up" in the "show interface", but shows "down" in "show hardware-interface". This issue does not occur when the link is brought up for the first time, but only manifests when the interface is brought down and up again. This happens because the device status and VNET_HW_INTERFACE_FLAG_LINK_UP aren't correctly updated during the port stop, preventing them from being updated during the port start. This patch fixes this issue in the port stop function. Type: fix Signed-off-by: Alok Mishra Change-Id: Idbc5a6c31409a3d1c5b19e14a3c9509291175677 Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/130090 Reviewed-by: Monendra Singh Kushwaha Tested-by: sa_ip-sw-jenkins Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/130568 Reviewed-by: Nithinsen Kaithakadan --- src/plugins/dev_octeon/port.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/plugins/dev_octeon/port.c b/src/plugins/dev_octeon/port.c index b2ab948e0f..d6a31a3df2 100644 --- a/src/plugins/dev_octeon/port.c +++ b/src/plugins/dev_octeon/port.c @@ -394,6 +394,18 @@ oct_port_stop (vlib_main_t *vm, vnet_dev_port_t *port) foreach_vnet_dev_port_tx_queue (q, port) oct_txq_stop (vm, q); + + vnet_dev_port_state_change (vm, port, + (vnet_dev_port_state_changes_t){ + .change.link_state = 1, + .change.link_speed = 1, + .link_speed = 0, + .link_state = 0, + }); + + /* Update the device status */ + cd->status = 0; + cd->speed = 0; } vnet_dev_rv_t From bcfe1afde1db2e46dc155c6e82fb59d9b8ee8f95 Mon Sep 17 00:00:00 2001 From: Nithinsen Kaithakadan Date: Mon, 27 May 2024 08:06:49 +0530 Subject: [PATCH 061/271] octeon: add crypto support for 3descbc and aesctr Add crypto encryption and decryption support for following algorithms in control plane and data plane. - AES-CTR sha1 - 3DES-CBC md5 sha1/256/384/512 Type: feature Signed-off-by: Nithinsen Kaithakadan Change-Id: I5345cf7ed2b913ca09d0964edd272af174943297 Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/128465 Tested-by: sa_ip-sw-jenkins Reviewed-by: Monendra Singh Kushwaha Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/130647 Reviewed-by: Nithinsen Kaithakadan Tested-by: Devapraba Muthumani --- src/plugins/dev_octeon/crypto.c | 40 ++++++++++++++++++++++++++++++++- src/plugins/dev_octeon/crypto.h | 10 ++++++++- 2 files changed, 48 insertions(+), 2 deletions(-) diff --git a/src/plugins/dev_octeon/crypto.c b/src/plugins/dev_octeon/crypto.c index b8552f0618..000949615b 100644 --- a/src/plugins/dev_octeon/crypto.c +++ b/src/plugins/dev_octeon/crypto.c @@ -911,7 +911,8 @@ oct_crypto_cpt_hmac_prep (u32 flags, u64 d_offs, u64 d_lens, /* Round up to 16 bytes alignment */ if (PREDICT_FALSE (encr_data_len & 0xf)) { - if (PREDICT_TRUE (cipher_type == ROC_SE_AES_CBC)) + if (PREDICT_TRUE (cipher_type == ROC_SE_AES_CBC) || + (cipher_type == ROC_SE_DES3_CBC)) enc_dlen = PLT_ALIGN_CEIL (encr_data_len, 8) + encr_offset; } @@ -1143,6 +1144,43 @@ oct_crypto_link_session_update (vlib_main_t *vm, oct_crypto_sess_t *sess, auth_type = ROC_SE_SHA2_SHA512; digest_len = 32; break; + case VNET_CRYPTO_ALG_AES_128_CTR_SHA1_TAG12: + case VNET_CRYPTO_ALG_AES_192_CTR_SHA1_TAG12: + case VNET_CRYPTO_ALG_AES_256_CTR_SHA1_TAG12: + enc_type = ROC_SE_AES_CTR; + auth_type = ROC_SE_SHA1_TYPE; + digest_len = 12; + break; + case VNET_CRYPTO_ALG_3DES_CBC_MD5_TAG12: + enc_type = ROC_SE_DES3_CBC; + auth_type = ROC_SE_MD5_TYPE; + digest_len = 12; + break; + case VNET_CRYPTO_ALG_3DES_CBC_SHA1_TAG12: + enc_type = ROC_SE_DES3_CBC; + auth_type = ROC_SE_SHA1_TYPE; + digest_len = 12; + break; + case VNET_CRYPTO_ALG_3DES_CBC_SHA224_TAG14: + enc_type = ROC_SE_DES3_CBC; + auth_type = ROC_SE_SHA2_SHA224; + digest_len = 14; + break; + case VNET_CRYPTO_ALG_3DES_CBC_SHA256_TAG16: + enc_type = ROC_SE_DES3_CBC; + auth_type = ROC_SE_SHA2_SHA256; + digest_len = 16; + break; + case VNET_CRYPTO_ALG_3DES_CBC_SHA384_TAG24: + enc_type = ROC_SE_DES3_CBC; + auth_type = ROC_SE_SHA2_SHA384; + digest_len = 24; + break; + case VNET_CRYPTO_ALG_3DES_CBC_SHA512_TAG32: + enc_type = ROC_SE_DES3_CBC; + auth_type = ROC_SE_SHA2_SHA512; + digest_len = 32; + break; default: log_err (ocd->dev, "Crypto: Undefined link algo %u specified. Key index %u", diff --git a/src/plugins/dev_octeon/crypto.h b/src/plugins/dev_octeon/crypto.h index d668d14134..75e2539118 100644 --- a/src/plugins/dev_octeon/crypto.h +++ b/src/plugins/dev_octeon/crypto.h @@ -33,7 +33,15 @@ _ (AES_256_CBC, SHA384, 32, 24) \ _ (AES_128_CBC, SHA512, 16, 32) \ _ (AES_192_CBC, SHA512, 24, 32) \ - _ (AES_256_CBC, SHA512, 32, 32) + _ (AES_256_CBC, SHA512, 32, 32) \ + _ (3DES_CBC, MD5, 24, 12) \ + _ (3DES_CBC, SHA1, 24, 12) \ + _ (3DES_CBC, SHA256, 24, 16) \ + _ (3DES_CBC, SHA384, 24, 24) \ + _ (3DES_CBC, SHA512, 24, 32) \ + _ (AES_128_CTR, SHA1, 16, 12) \ + _ (AES_192_CTR, SHA1, 24, 12) \ + _ (AES_256_CTR, SHA1, 32, 12) #define OCT_MOD_INC(i, l) ((i) == (l - 1) ? (i) = 0 : (i)++) From bcbfc7999578585ea9322468c7b609476a5a9158 Mon Sep 17 00:00:00 2001 From: Nithinsen Kaithakadan Date: Fri, 12 Jul 2024 12:04:21 +0530 Subject: [PATCH 062/271] octeon: allocate pending queue for all cores This patch allocates memory for pending queue for all cores including main-core which fixes crash in crypto offload when accessing queue using worker core index. Type: fix Signed-off-by: Nithinsen Kaithakadan Change-Id: I2df9ecba0a11074b8b17b4b3feb44c748abf1c3c Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/131231 Tested-by: sa_ip-sw-jenkins Reviewed-by: Nithinsen Kaithakadan --- src/plugins/dev_octeon/crypto.c | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/src/plugins/dev_octeon/crypto.c b/src/plugins/dev_octeon/crypto.c index 000949615b..18ab84b088 100644 --- a/src/plugins/dev_octeon/crypto.c +++ b/src/plugins/dev_octeon/crypto.c @@ -1599,18 +1599,14 @@ oct_init_crypto_engine_handlers (vlib_main_t *vm, vnet_dev_t *dev) int oct_conf_sw_queue (vlib_main_t *vm, vnet_dev_t *dev) { + vlib_thread_main_t *tm = vlib_get_thread_main (); oct_crypto_main_t *ocm = &oct_crypto_main; extern oct_plt_init_param_t oct_plt_init_param; - vnet_device_main_t *vdm = &vnet_device_main; oct_crypto_inflight_req_t *infl_req_queue; - u8 num_worker_cores; int i, j = 0; - num_worker_cores = - vdm->last_worker_thread_index - vdm->first_worker_thread_index + 1; - ocm->pend_q = oct_plt_init_param.oct_plt_zmalloc ( - num_worker_cores * sizeof (oct_crypto_pending_queue_t), + tm->n_vlib_mains * sizeof (oct_crypto_pending_queue_t), CLIB_CACHE_LINE_BYTES); if (ocm->pend_q == NULL) { @@ -1618,7 +1614,7 @@ oct_conf_sw_queue (vlib_main_t *vm, vnet_dev_t *dev) return -1; } - for (i = 0; i <= num_worker_cores; ++i) + for (i = 0; i < tm->n_vlib_mains; ++i) { ocm->pend_q[i].n_desc = OCT_CRYPTO_DEFAULT_SW_ASYNC_FRAME_COUNT; From c023a57a1a42fb61c9b089394cc78b42a9a9d158 Mon Sep 17 00:00:00 2001 From: Nithinsen Kaithakadan Date: Tue, 16 Jul 2024 21:58:27 +0530 Subject: [PATCH 063/271] octeon: alloc crypto session context in key add This patch fixes physmem alloc crash in dataplane as part of session creation by moving session alloc into crypto key add handler. Type: fix Signed-off-by: Nithinsen Kaithakadan Change-Id: I07e3a258288a674d84ca2350a902fc83baaab2ef Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/131470 Tested-by: sa_ip-sw-jenkins Reviewed-by: Nithinsen Kaithakadan --- src/plugins/dev_octeon/crypto.c | 186 ++++++++++++++++++++++---------- src/plugins/dev_octeon/crypto.h | 10 +- 2 files changed, 131 insertions(+), 65 deletions(-) diff --git a/src/plugins/dev_octeon/crypto.c b/src/plugins/dev_octeon/crypto.c index 18ab84b088..b96d1b3cdb 100644 --- a/src/plugins/dev_octeon/crypto.c +++ b/src/plugins/dev_octeon/crypto.c @@ -18,28 +18,122 @@ VLIB_REGISTER_LOG_CLASS (oct_log, static) = { .subclass_name = "init", }; +static_always_inline void +oct_map_keyindex_to_session (oct_crypto_sess_t *sess, u32 key_index, u8 type) +{ + oct_crypto_main_t *ocm = &oct_crypto_main; + oct_crypto_key_t *ckey; + + ckey = vec_elt_at_index (ocm->keys[type], key_index); + + ckey->sess = sess; + sess->key_index = key_index; +} + +static_always_inline oct_crypto_sess_t * +oct_crypto_session_alloc (vlib_main_t *vm, u8 type) +{ + extern oct_plt_init_param_t oct_plt_init_param; + oct_crypto_sess_t *addr = NULL; + oct_crypto_main_t *ocm; + oct_crypto_dev_t *ocd; + u32 size; + + ocm = &oct_crypto_main; + ocd = ocm->crypto_dev[type]; + + size = sizeof (oct_crypto_sess_t); + + addr = oct_plt_init_param.oct_plt_zmalloc (size, CLIB_CACHE_LINE_BYTES); + if (addr == NULL) + { + log_err (ocd->dev, "Failed to allocate crypto session memory"); + return NULL; + } + + return addr; +} + +static_always_inline i32 +oct_crypto_session_create (vlib_main_t *vm, vnet_crypto_key_index_t key_index, + int op_type) +{ + oct_crypto_main_t *ocm = &oct_crypto_main; + oct_crypto_sess_t *session; + vnet_crypto_key_t *key; + oct_crypto_key_t *ckey; + + key = vnet_crypto_get_key (key_index); + + if (key->type == VNET_CRYPTO_KEY_TYPE_LINK) + { + /* + * Read crypto or integ key session. And map link key index to same. + */ + if (key->index_crypto != UINT32_MAX) + { + ckey = vec_elt_at_index (ocm->keys[op_type], key->index_crypto); + session = ckey->sess; + } + else if (key->index_integ != UINT32_MAX) + { + ckey = vec_elt_at_index (ocm->keys[op_type], key->index_integ); + session = ckey->sess; + } + else + return -1; + } + else + { + session = oct_crypto_session_alloc (vm, op_type); + if (session == NULL) + return -1; + } + + oct_map_keyindex_to_session (session, key_index, op_type); + return 0; +} + void oct_crypto_key_del_handler (vlib_main_t *vm, vnet_crypto_key_index_t key_index) { - oct_crypto_main_t *ocm = &oct_crypto_main; extern oct_plt_init_param_t oct_plt_init_param; + oct_crypto_main_t *ocm = &oct_crypto_main; + oct_crypto_key_t *ckey_linked; oct_crypto_key_t *ckey; vec_validate (ocm->keys[VNET_CRYPTO_OP_TYPE_ENCRYPT], key_index); + ckey = vec_elt_at_index (ocm->keys[VNET_CRYPTO_OP_TYPE_ENCRYPT], key_index); if (ckey->sess) { + /* + * If in case link algo is pointing to same sesison, reset the pointer. + */ + if (ckey->sess->key_index != key_index) + { + ckey_linked = vec_elt_at_index ( + ocm->keys[VNET_CRYPTO_OP_TYPE_ENCRYPT], ckey->sess->key_index); + ckey_linked->sess = NULL; + } oct_plt_init_param.oct_plt_free (ckey->sess); ckey->sess = NULL; - return; } ckey = vec_elt_at_index (ocm->keys[VNET_CRYPTO_OP_TYPE_DECRYPT], key_index); if (ckey->sess) { + /* + * If in case link algo is pointing to same sesison, reset the pointer. + */ + if (ckey->sess->key_index != key_index) + { + ckey_linked = vec_elt_at_index ( + ocm->keys[VNET_CRYPTO_OP_TYPE_DECRYPT], ckey->sess->key_index); + ckey_linked->sess = NULL; + } oct_plt_init_param.oct_plt_free (ckey->sess); ckey->sess = NULL; - return; } } @@ -51,11 +145,27 @@ oct_crypto_key_add_handler (vlib_main_t *vm, vnet_crypto_key_index_t key_index) vec_validate (ocm->keys[VNET_CRYPTO_OP_TYPE_ENCRYPT], key_index); ckey = vec_elt_at_index (ocm->keys[VNET_CRYPTO_OP_TYPE_ENCRYPT], key_index); - ckey->sess = NULL; + if (ckey->sess == NULL) + { + if (oct_crypto_session_create (vm, key_index, + VNET_CRYPTO_OP_TYPE_ENCRYPT)) + { + clib_warning ("Unable to create crypto session"); + return; + } + } vec_validate (ocm->keys[VNET_CRYPTO_OP_TYPE_DECRYPT], key_index); ckey = vec_elt_at_index (ocm->keys[VNET_CRYPTO_OP_TYPE_DECRYPT], key_index); - ckey->sess = NULL; + if (ckey->sess == NULL) + { + if (oct_crypto_session_create (vm, key_index, + VNET_CRYPTO_OP_TYPE_DECRYPT)) + { + clib_warning ("Unable to create crypto session"); + return; + } + } } void @@ -74,26 +184,6 @@ oct_crypto_key_handler (vlib_main_t *vm, vnet_crypto_key_op_t kop, ocm->started = 1; } -static_always_inline oct_crypto_sess_t * -oct_crypto_session_alloc (vlib_main_t *vm, u8 type) -{ - oct_crypto_main_t *ocm = &oct_crypto_main; - oct_crypto_dev_t *ocd = ocm->crypto_dev[type]; - extern oct_plt_init_param_t oct_plt_init_param; - oct_crypto_sess_t *addr = NULL; - u32 size; - - size = sizeof (oct_crypto_sess_t); - addr = oct_plt_init_param.oct_plt_zmalloc (size, CLIB_CACHE_LINE_BYTES); - if (addr == NULL) - { - log_err (ocd->dev, "Failed to allocate crypto session memory"); - return NULL; - } - - return addr; -} - static_always_inline void oct_crypto_session_free (vlib_main_t *vm, oct_crypto_sess_t *sess) { @@ -128,7 +218,7 @@ oct_cpt_inst_submit (struct cpt_inst_s *inst, uint64_t lmtline, } #endif -void +static_always_inline void oct_crypto_burst_submit (oct_crypto_dev_t *crypto_dev, struct cpt_inst_s *inst, u32 n_left) { @@ -1081,17 +1171,6 @@ oct_cpt_inst_w7_get (oct_crypto_sess_t *sess, struct roc_cpt *roc_cpt) return inst_w7.u64; } -static_always_inline void -oct_map_keyindex_to_session (oct_crypto_sess_t *sess, u32 key_index, u8 type) -{ - oct_crypto_main_t *ocm = &oct_crypto_main; - oct_crypto_key_t *ckey; - - ckey = vec_elt_at_index (ocm->keys[type], key_index); - - ckey->sess = sess; -} - static_always_inline i32 oct_crypto_link_session_update (vlib_main_t *vm, oct_crypto_sess_t *sess, u32 key_index, u8 type) @@ -1217,13 +1296,6 @@ oct_crypto_link_session_update (vlib_main_t *vm, oct_crypto_sess_t *sess, return -1; } - oct_map_keyindex_to_session (sess, key_index, type); - /* - * Map session to crypto key index also. This entry can be referred - * while deleting key - */ - oct_map_keyindex_to_session (sess, key->index_crypto, type); - return 0; } @@ -1276,27 +1348,20 @@ oct_crypto_aead_session_update (vlib_main_t *vm, oct_crypto_sess_t *sess, return -1; } - oct_map_keyindex_to_session (sess, key_index, type); - return 0; } -i32 -oct_crypto_session_create (vlib_main_t *vm, vnet_crypto_key_index_t key_index, - int op_type) +static_always_inline i32 +oct_crypto_session_init (vlib_main_t *vm, oct_crypto_sess_t *session, + vnet_crypto_key_index_t key_index, int op_type) { oct_crypto_main_t *ocm = &oct_crypto_main; oct_crypto_dev_t *ocd = ocm->crypto_dev[op_type]; - oct_crypto_sess_t *session; vnet_crypto_key_t *key; i32 rv = 0; key = vnet_crypto_get_key (key_index); - session = oct_crypto_session_alloc (vm, op_type); - if (session == NULL) - return -1; - if (key->type == VNET_CRYPTO_KEY_TYPE_LINK) rv = oct_crypto_link_session_update (vm, session, key_index, op_type); else @@ -1313,6 +1378,8 @@ oct_crypto_session_create (vlib_main_t *vm, vnet_crypto_key_index_t key_index, session->cpt_inst_w7 = oct_cpt_inst_w7_get (session, session->crypto_dev->roc_cpt); + session->initialised = 1; + return 0; } @@ -1328,7 +1395,7 @@ oct_crypto_update_frame_error_status (vnet_crypto_async_frame_t *f, f->state = VNET_CRYPTO_FRAME_STATE_NOT_PROCESSED; } -int +static_always_inline int oct_crypto_enqueue_enc_dec (vlib_main_t *vm, vnet_crypto_async_frame_t *frame, const u8 is_aead, u8 aad_len, const u8 type) { @@ -1365,14 +1432,15 @@ oct_crypto_enqueue_enc_dec (vlib_main_t *vm, vnet_crypto_async_frame_t *frame, if (!key->sess) { - if (oct_crypto_session_create (vm, elts->key_index, type) == -1) - { oct_crypto_update_frame_error_status ( frame, VNET_CRYPTO_OP_STATUS_FAIL_ENGINE_ERR); return -1; - } } sess = key->sess; + if (PREDICT_FALSE (!sess->initialised)) + { + oct_crypto_session_init (vm, sess, elts->key_index, type); + } crypto_dev = sess->crypto_dev; memset (inst + i, 0, sizeof (struct cpt_inst_s)); @@ -1461,7 +1529,7 @@ oct_crypto_enqueue_aead_aad_enc (vlib_main_t *vm, return 0; } -int +static_always_inline int oct_crypto_enqueue_aead_aad_dec (vlib_main_t *vm, vnet_crypto_async_frame_t *frame, u8 aad_len) { diff --git a/src/plugins/dev_octeon/crypto.h b/src/plugins/dev_octeon/crypto.h index 75e2539118..57d1e92f77 100644 --- a/src/plugins/dev_octeon/crypto.h +++ b/src/plugins/dev_octeon/crypto.h @@ -84,6 +84,10 @@ typedef struct u16 auth_iv_offset; /** CPT inst word 7 */ u64 cpt_inst_w7; + /* initialise as part of first packet */ + u8 initialised; + /* store link key index in case of linked algo */ + vnet_crypto_key_index_t key_index; oct_crypto_dev_t *crypto_dev; struct roc_se_ctx cpt_ctx; } oct_crypto_sess_t; @@ -151,12 +155,6 @@ int oct_crypto_enqueue_linked_alg_enc (vlib_main_t *vm, vnet_crypto_async_frame_t *frame); int oct_crypto_enqueue_linked_alg_dec (vlib_main_t *vm, vnet_crypto_async_frame_t *frame); -int oct_crypto_enqueue_aead_aad_enc (vlib_main_t *vm, - vnet_crypto_async_frame_t *frame, - u8 aad_len); -int oct_crypto_enqueue_aead_aad_dec (vlib_main_t *vm, - vnet_crypto_async_frame_t *frame, - u8 aad_len); int oct_crypto_enqueue_aead_aad_8_enc (vlib_main_t *vm, vnet_crypto_async_frame_t *frame); int oct_crypto_enqueue_aead_aad_12_enc (vlib_main_t *vm, From e8d99c639e6f7d41226f540043bf6408ff6b192c Mon Sep 17 00:00:00 2001 From: Monendra Singh Kushwaha Date: Thu, 18 Jul 2024 12:23:59 +0530 Subject: [PATCH 064/271] octeon: add crypto counters This patch adds crypto counters for pending pkts, inflight operations and successfully dequeued pkts in queue. Type: feature JIRA: https://essjira.marvell.com/browse/IPBUSW-49522 Signed-off-by: Monendra Singh Kushwaha Change-Id: I1fc578d2566f195b763c5ed22466d03a7420c7b5 Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/131527 Tested-by: sa_ip-sw-jenkins Reviewed-by: Nithinsen Kaithakadan (cherry picked from commit fb2ab1262c9d6b55d5a94f3942a01b8449f64543) Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/131785 --- src/plugins/dev_octeon/CMakeLists.txt | 1 + src/plugins/dev_octeon/cli.c | 200 ++++++++++++++++++++++++++ src/plugins/dev_octeon/crypto.c | 17 +++ src/plugins/dev_octeon/crypto.h | 18 +++ src/plugins/dev_octeon/init.c | 12 +- 5 files changed, 247 insertions(+), 1 deletion(-) create mode 100644 src/plugins/dev_octeon/cli.c diff --git a/src/plugins/dev_octeon/CMakeLists.txt b/src/plugins/dev_octeon/CMakeLists.txt index 6109de57a7..1a60d3a28b 100644 --- a/src/plugins/dev_octeon/CMakeLists.txt +++ b/src/plugins/dev_octeon/CMakeLists.txt @@ -28,6 +28,7 @@ endif() add_vpp_plugin(dev_octeon SOURCES init.c + cli.c format.c port.c queue.c diff --git a/src/plugins/dev_octeon/cli.c b/src/plugins/dev_octeon/cli.c new file mode 100644 index 0000000000..bd88849c08 --- /dev/null +++ b/src/plugins/dev_octeon/cli.c @@ -0,0 +1,200 @@ +/* + * Copyright (c) 2024 Marvell. + * SPDX-License-Identifier: Apache-2.0 + * https://spdx.org/licenses/Apache-2.0.html + */ + +/** + * @file + * @brief OCTEON CLI implementation. + */ + +#include +#include +#include +#include + +static const char *ul = "=====================================================" + "========================="; + +static void +oct_print_global_counters (vlib_main_t *vm, u64 **stat, u32 n_threads) +{ + u64 global_stat[OCT_MAX_CRYPTO_COUNTERS] = { 0 }; + oct_crypto_main_t *ocm = &oct_crypto_main; + unsigned int n_global_stats = 0; + vlib_simple_counter_main_t *cm; + u32 cnt_idx, thread_idx = 0; + + for (thread_idx = 0; thread_idx < n_threads; thread_idx++) + { + for (cnt_idx = 0; cnt_idx < OCT_MAX_CRYPTO_COUNTERS; cnt_idx++) + { + if (stat[cnt_idx][thread_idx]) + { + global_stat[cnt_idx] += stat[cnt_idx][thread_idx]; + n_global_stats++; + } + } + } + + if (!n_global_stats) + return; + + /* Display cumulative counters */ + vlib_cli_output (vm, "%-16s %-40s %-20s", "", "Global counter", "Value"); + vlib_cli_output (vm, "%-16s %-.40s %-.20s", "", ul, ul); + +#define _(i, s, d) \ + cm = &ocm->s##_counter; \ + if (global_stat[i]) \ + vlib_cli_output (vm, "%-16s %-40s %20Ld", "", cm->name, global_stat[i]); + foreach_crypto_counter; +#undef _ +} + +unsigned int +oct_get_per_thread_stats (u64 **stat, u32 n_threads, u64 *threads_with_stats) +{ + unsigned int cnt_idx, thread_idx = 0, n_threads_with_stats = 0; + + /* Identify threads that have non-zero Octeon crypto counters */ + for (thread_idx = 0; thread_idx < n_threads; thread_idx++) + { + for (cnt_idx = 0; cnt_idx < OCT_MAX_CRYPTO_COUNTERS; cnt_idx++) + { + if (stat[cnt_idx][thread_idx]) + { + threads_with_stats[n_threads_with_stats++] = thread_idx; + break; + } + } + } + + return n_threads_with_stats; +} + +static void +oct_print_per_thread_counters (vlib_main_t *vm, u64 **stat, u32 n_threads) +{ + unsigned int idx, thread_idx = 0, n_threads_with_stats = 0; + oct_crypto_main_t *ocm = &oct_crypto_main; + u64 threads_with_stats[n_threads]; + vlib_simple_counter_main_t *cm; + + n_threads_with_stats = + oct_get_per_thread_stats (stat, n_threads, threads_with_stats); + + if (!n_threads_with_stats) + return; + + vlib_cli_output (vm, "%-16s %-40s %-20s", "Thread", "Per-thread counter", + "Value"); + vlib_cli_output (vm, "%-.16s %-.40s %-.20s", ul, ul, ul); + + for (idx = 0; idx < n_threads_with_stats; idx++) + { + thread_idx = threads_with_stats[idx]; + + vlib_cli_output (vm, "%-16s", vlib_worker_threads[thread_idx].name); + + /* clang-format off */ +#define _(i, s, d) \ + cm = &ocm->s##_counter; \ + if (stat[i][thread_idx]) \ + vlib_cli_output (vm, "%-16s %-40s %20Ld", "", cm->name, \ + stat[i][thread_idx]); + foreach_crypto_counter; +#undef _ + /* clang-format on */ + } + + vlib_cli_output (vm, "\n"); + + return; +} + +static clib_error_t * +oct_crypto_counters_command_fn (vlib_main_t *vm, unformat_input_t *input, + vlib_cli_command_t *cmd) +{ + unsigned int cnt_idx = 0, thread_idx = 0; + oct_crypto_main_t *ocm = &oct_crypto_main; + vlib_simple_counter_main_t *cm; + u64 *stat[OCT_MAX_CRYPTO_COUNTERS] = { 0 }; + counter_t *counters = NULL; + u32 n_threads = vlib_get_n_threads (); + +#define _(i, s, d) \ + cm = &ocm->s##_counter; \ + vec_validate_init_empty (stat[i], n_threads, 0); \ + for (thread_idx = 0; thread_idx < n_threads; thread_idx++) \ + { \ + counters = cm->counters[thread_idx]; \ + stat[i][thread_idx] = counters[0]; \ + } + foreach_crypto_counter; +#undef _ + + oct_print_per_thread_counters (vm, stat, n_threads); + + oct_print_global_counters (vm, stat, n_threads); + + for (cnt_idx = 0; cnt_idx < OCT_MAX_CRYPTO_COUNTERS; cnt_idx++) + vec_free (stat[cnt_idx]); + + return 0; +} + +/*? + * This command displays Octeon crypto counters + * + * @cliexpar + * Example of how to display Octeon crypto counters: + * @cliexstart{show octeon crypto counters} + * Per-thread counter Value + * ======================================== ==================== + * + * crypto-inflight-operations 8 + * crypto-success-packets 8 + * + * Global counter Value + * ======================================== ==================== + * crypto-inflight-operations 8 + * crypto-success-packets 8 + * @cliexend +?*/ +VLIB_CLI_COMMAND (oct_crypto_counters_command, static) = { + .path = "show octeon crypto counters", + .short_help = "show octeon crypto counters [verbose]", + .function = oct_crypto_counters_command_fn, +}; + +static clib_error_t * +oct_crypto_counters_clear_command_fn (vlib_main_t *vm, unformat_input_t *input, + vlib_cli_command_t *cmd) +{ + vlib_simple_counter_main_t *cm; + oct_crypto_main_t *ocm = &oct_crypto_main; + +#define _(i, s, d) \ + cm = &ocm->s##_counter; \ + vlib_clear_simple_counters (cm); + foreach_crypto_counter; +#undef _ + + return 0; +} + +/*? + * This command clears Octeon crypto counters + * + * @cliexpar + * @cliexstart{clear octeon crypto counters} + * @cliexend +?*/ +VLIB_CLI_COMMAND (oct_crypto_counters_clear_command, static) = { + .path = "clear octeon crypto counters", + .short_help = "clear octeon crypto counters", + .function = oct_crypto_counters_clear_command_fn, +}; diff --git a/src/plugins/dev_octeon/crypto.c b/src/plugins/dev_octeon/crypto.c index b96d1b3cdb..b9b20ade21 100644 --- a/src/plugins/dev_octeon/crypto.c +++ b/src/plugins/dev_octeon/crypto.c @@ -1498,6 +1498,11 @@ oct_crypto_enqueue_enc_dec (vlib_main_t *vm, vnet_crypto_async_frame_t *frame, OCT_MOD_INC (pend_q->enq_tail, pend_q->n_desc); pend_q->n_crypto_inflight++; + vlib_increment_simple_counter (pend_q->pending_packets, vm->thread_index, 0, + frame->n_elts); + vlib_increment_simple_counter (pend_q->crypto_inflight, vm->thread_index, 0, + 1); + return 0; } @@ -1609,11 +1614,18 @@ oct_crypto_frame_dequeue (vlib_main_t *vm, u32 *nb_elts_processed, infl_req->deq_elts++; } + vlib_decrement_simple_counter (pend_q->pending_packets, vm->thread_index, 0, + infl_req->elts); + vlib_increment_simple_counter (pend_q->success_packets, vm->thread_index, 0, + infl_req->elts); + clib_memset ((void *) infl_req->res, 0, sizeof (union cpt_res_s) * VNET_CRYPTO_FRAME_SIZE); OCT_MOD_INC (pend_q->deq_head, pend_q->n_desc); pend_q->n_crypto_inflight--; + vlib_decrement_simple_counter (pend_q->crypto_inflight, vm->thread_index, 0, + 1); frame->state = status == VNET_CRYPTO_OP_STATUS_COMPLETED ? VNET_CRYPTO_FRAME_STATE_SUCCESS : @@ -1710,8 +1722,13 @@ oct_conf_sw_queue (vlib_main_t *vm, vnet_dev_t *dev) goto free; } } + +#define _(n, s, d) ocm->pend_q[i].s = &ocm->s##_counter; + foreach_crypto_counter; } + return 0; + free: for (; i >= 0; i--) { diff --git a/src/plugins/dev_octeon/crypto.h b/src/plugins/dev_octeon/crypto.h index 57d1e92f77..32cbd66bfb 100644 --- a/src/plugins/dev_octeon/crypto.h +++ b/src/plugins/dev_octeon/crypto.h @@ -11,6 +11,14 @@ #define OCT_MAX_N_CPT_DEV 2 +#define OCT_MAX_CRYPTO_COUNTERS 3 + +/* counter, name, verbose */ +#define foreach_crypto_counter \ + _ (0, pending_packets, "crypto-pending-packets") \ + _ (1, crypto_inflight, "crypto-inflight-operations") \ + _ (2, success_packets, "crypto-success-packets") + /* CRYPTO_ID, KEY_LENGTH_IN_BYTES, TAG_LEN, AAD_LEN */ #define foreach_oct_crypto_aead_async_alg \ _ (AES_128_GCM, 16, 16, 8) \ @@ -131,6 +139,11 @@ typedef struct u16 deq_head; /** Number of descriptors */ u16 n_desc; + /** Crypto counters for pending pkts, inflight operations + * and successfully dequeued pkts in queue */ +#define _(i, s, d) vlib_simple_counter_main_t *s; + foreach_crypto_counter; +#undef _ } oct_crypto_pending_queue_t; typedef struct @@ -138,10 +151,15 @@ typedef struct oct_crypto_dev_t *crypto_dev[OCT_MAX_N_CPT_DEV]; oct_crypto_key_t *keys[VNET_CRYPTO_ASYNC_OP_N_TYPES]; oct_crypto_pending_queue_t *pend_q; +#define _(i, s, d) vlib_simple_counter_main_t s##_counter; + foreach_crypto_counter; +#undef _ int n_cpt; u8 started; } oct_crypto_main_t; +extern oct_crypto_main_t oct_crypto_main; + void oct_crypto_key_del_handler (vlib_main_t *vm, vnet_crypto_key_index_t key_index); diff --git a/src/plugins/dev_octeon/init.c b/src/plugins/dev_octeon/init.c index 5782fc2394..9b7751f420 100644 --- a/src/plugins/dev_octeon/init.c +++ b/src/plugins/dev_octeon/init.c @@ -284,7 +284,17 @@ oct_init_cpt (vlib_main_t *vm, vnet_dev_t *dev) oct_init_crypto_engine_handlers (vm, dev); if (!ocm->n_cpt) - ocm->crypto_dev[0] = ocd; + { + ocm->crypto_dev[0] = ocd; + /* Initialize counters */ +#define _(i, s, str) \ + ocm->s##_counter.name = str; \ + ocm->s##_counter.stat_segment_name = "/octeon/" str "_counters"; \ + vlib_validate_simple_counter (&ocm->s##_counter, 0); \ + vlib_zero_simple_counter (&ocm->s##_counter, 0); + foreach_crypto_counter; +#undef _ + } ocm->crypto_dev[1] = ocd; From a67342382b24cee39a205ded989a1c9772458dab Mon Sep 17 00:00:00 2001 From: Monendra Singh Kushwaha Date: Fri, 12 Jul 2024 16:28:52 +0530 Subject: [PATCH 065/271] octeon: sync mac address to cgx/rpm table This patch syncs mac address to cgx/rpm table at initialization, which sets interface in unicast mode. Type: fix Change-Id: Icf64b91ce5c74cc399cad3d3358b951fb2a65297 Signed-off-by: Monendra Singh Kushwaha Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/131188 Tested-by: sa_ip-sw-jenkins Reviewed-by: Nithinsen Kaithakadan (cherry picked from commit 596ad4abd1e04a9bc560973ce6466ccf3bacafd4) Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/131787 --- src/plugins/dev_octeon/port.c | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/plugins/dev_octeon/port.c b/src/plugins/dev_octeon/port.c index d6a31a3df2..bc24b6b02d 100644 --- a/src/plugins/dev_octeon/port.c +++ b/src/plugins/dev_octeon/port.c @@ -59,6 +59,7 @@ oct_port_init (vlib_main_t *vm, vnet_dev_port_t *port) vnet_dev_t *dev = port->dev; oct_device_t *cd = vnet_dev_get_data (dev); oct_port_t *cp = vnet_dev_get_port_data (port); + u8 mac_addr[PLT_ETHER_ADDR_LEN]; struct roc_nix *nix = cd->nix; vnet_dev_rv_t rv; int rrv; @@ -76,6 +77,22 @@ oct_port_init (vlib_main_t *vm, vnet_dev_port_t *port) } cp->lf_allocated = 1; + if (!roc_nix_is_vf_or_sdp (nix)) + { + if ((rrv = roc_nix_npc_mac_addr_get (nix, mac_addr))) + { + oct_port_deinit (vm, port); + return oct_roc_err (dev, rrv, "roc_nix_npc_mac_addr_get failed"); + } + + /* Sync MAC address to CGX/RPM table */ + if ((rrv = roc_nix_mac_addr_set (nix, mac_addr))) + { + oct_port_deinit (vm, port); + return oct_roc_err (dev, rrv, "roc_nix_mac_addr_set failed"); + } + } + if ((rrv = roc_nix_tm_init (nix))) { oct_port_deinit (vm, port); From 5ef91e36d3b1960ab478aceade1cb046e444d574 Mon Sep 17 00:00:00 2001 From: Bheemappa Agasimundin Date: Sun, 24 Mar 2024 08:36:48 +0000 Subject: [PATCH 066/271] linux-cp: add readme for xfrm implementation This patch adds REAMDE for XFRM changes design and startup.conf configuration details. Type: feature JIRA: https://essjira.marvell.com/browse/IPBUSW-43828 Signed-off-by: Bheemappa Agasimundin Change-Id: Icddedc14fcd3c78d3a0c1ccc2c1eba3cd139e656 Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/124378 Tested-by: sa_ip-sw-jenkins Reviewed-by: Aakash Kumar Shankarappa Reviewed-by: Abed Mohammad Kamaluddin Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/132674 Reviewed-by: Nithinsen Kaithakadan --- src/plugins/linux-cp/README | 169 ++++++++++++++++++++++++++++++++++++ 1 file changed, 169 insertions(+) create mode 100644 src/plugins/linux-cp/README diff --git a/src/plugins/linux-cp/README b/src/plugins/linux-cp/README new file mode 100644 index 0000000000..9e63646dd6 --- /dev/null +++ b/src/plugins/linux-cp/README @@ -0,0 +1,169 @@ +XFRM NETLINK SUPPORT IN LINUX NL PLUGIN: +======================================== + +Introduction: +------------- + +The purpose of introducing Linux XFRM netlink support in the linux_nl_plugin +is to mirror Linux XFRM configurations to the VPP IPsec subsystem. These +configurations can be manually set using ip commands or via keying daemons +like StrongSwan. In both cases, the netlink notifications generated from +Linux are read by this XFRM module and translated into VPP's IPsec configuration. + +Highlights: +=========== + +1. The XFRM module is part of the existing linux-nl plugin, added as a new XFRM + process type node.From now on, it will be referred to as the XFRM node/plugin. +2. The XFRM node piggybacks on the libnl-xfrm system library for parsing/extracting + netlink messages. +3. The XFRM node will depend on Kernel XFRM netlink notifications. To read these + messages, the XFRM node registers to XFRMGRP_SA, XFRMGRP_POLICY, and XFRMGRP_EXPIRE + groups of the NETLINK_XFRM protocol of the AF_NETLINK family. +3. The XFRM node will support both policy-based and tunnel/route-based IPsec. The mode + can be selected via startup.conf. +4. The plugin will support packet and byte-based soft life and hard life expiry as the + datapath will be handled in VPP. + +Design: +======= + +The XFRM module is divided into three components: reading XFRM NL notifications +configuring VPP IPsec based on NL notifications, and handling SA expiry. + +1. Reading the XFRM NL notifications: +===================================== + + In this design, a process similar to the Linux netlink plugin is followed. It + includes creating a netlink socket of type NETLINK_XFRM and registering it to + multicast groups such as XFRMGRP_SA, XFRMGRP_POLICY, and XFRMGRP_EXPIRE. Once + the netlink (NL) messages are read from the NETLINK_XFRM socket, they are handled + based on their message type. This handling process is where the second part of the + plugin comes into play: configuring VPP IPsec + +2. Configuring VPP IPsec: +======================== + + Based on the startup configuration, VPP IPsec will be configured to run in one of + the two IPsec modes. + + Policy based IPsec: + ------------------- + + a.In VPP IPsec, the Security Association (SA) and Policy are closely linked + during configuration. Each SA has a unique SA ID. However, the XFRM kernel + does not inherently recognize SA identifiers. In this plugin, we generate a + unique 32-bit SA ID based on the Security Parameter Index (SPI), Destination + IP (DIP), and Protocol (Proto) for a given SA. This ID is then used to associate + the SA with a policy. + + b.Because of a limitation (as described in point 3 of the limitations), adding + an SA message will also handle inbound policy addition. However, outbound policy + handling is part of the Policy notification process. + + + c.Using the tunnel endpoint IP address, we determine the VPP interface on which + IPsec/SPD (Security Policy Database) needs to be enabled. + + d.After creating and enabling the Security Policy Database (SPD) on a specific + interface, all packets passing through that interface undergo policy lookup. + If a packet doesn't match any existing policies, we internally add "allow all" + bypass policies. These bypass policies allow essential messages such as IKE (Internet + Key Exchange), Neighbor Discovery, and keep-alive messages to pass through. + + e.The configuration of bypass policies is handled internally, and there won't be a Netlink + (NL) notification for it. These bypass policies are added when the first Security + Association (SA) or protect policy notification is handled. They are subsequently + deleted when there are no more protect policies in the system. + + + Route based IPsec: + ------------------ + + a.In a manner similar to the policy-based scheme, we derive Security Association (SA) + IDs here. Additionally, as part of the SA notification process, we create an IPIP + tunnel or an IPsec interface. This tunnel is then protected with inbound (inb) and + outbound (outb) SAs. It's important to note that a maximum of 4 inbound SAs and + 1 outbound SA can be bound to a tunnel. + + b.In the context of Policy notifications, we only handle FIB (Forwarding Information Base) + entries via tunnel interfaces. We do not add policies in route mode. + +3. SA Expiry: +============= + +VPP (Vector Packet Processing) does not support soft/hard byte or packet-based expiries directly. +To address this limitation, we've implemented a process node that continuously polls all +available SA (Security Association) counters at fixed intervals. Here's how it works: + + a. Expiry Values and Message Creation: + + 1. The plugin receives expiry values from the XFRM SA (Linux Security Association) Netlink (NL) + notifications. + 2. Based on these values, the plugin constructs an NL SA expiry message (XFRM_MSG_EXPIRE). + 3. The message specifies whether the expiry is hard or soft. + + b. Kernel Interaction: + + 1. The plugin sends the NL SA expiry message to the kernel. + 2. The kernel validates this message against its database. + 3. If the expiry is soft, the kernel initiates rekeying automatically. In this case, our plugin + doesn't need to handle the message further. + 4. If the expiry is hard, there is no separate notification to delete the SA. + + c.Handling Hard Expiry: + + When the plugin receives a hard expiry notification, it deletes the corresponding SA. +This approach ensures that SA management remains consistent even without direct soft/hard expiry support in VPP. + +Limitations: +=========== + +1.The plugin does not support on-demand Security Association (SA) creation by installing trap policies. + This limitation arises because VPP cannot install a policy without a valid SA ID. Consequently, the + strongSwan configuration option "auto = route" cannot be supported. + +2.Given that VPP does not allow configuring the anti-replay window size (which remains fixed at 64), + the plugin does not take into account the replay-window size configured in strongSwan. However, it + does handle enabling or disabling the use-anti-replay flag in VPP. + +3.During negotiations between peers, there's a possibility that a peer sends an ESP packet with an old + Security Association (SA) (SPI X). Meanwhile, the strongSwan on the device under test (DUT) has updated + its inbound (INB) policy to point to a newly negotiated SA (SPI Y). This situation could lead to packet + drops in the DUT's inbound direction until the peer updates its policy to use the new SA (SPI Y). + + To avoid this behavior, the plugin handles INB policy addition and deletion as part of SA addition and + deletion. As long as there is an SA in VPP, all INB packets matching that SA/SPI are accepted. Whenever + we receive a delete SA notification from the kernel, we remove the SA and its associated INB policy + +4. a.We do not handle inbound (INB) policy notifications from the kernel. + b.Forward (FWD) policies are not handled in the plugin because there is no use case for them. + c.The plugin intentionally avoids handling XFRM Policy notifications for BYPASS and DROP actions. + This design optimization ensures compatibility with daemons like strongSwan. + d.In NL (Netlink) policy notifications, we expect only one user template to be present. + +5. The plugin has been tested with AES-GCM and AES-CBC encryption algorithms only. + +6. In route/tunnel mode, VPP supports a maximum of 4 inbound (INB) SAs and 1 outbound (OUTB) SA + bound to the tunnel. + +7. In route/tunnel mode, if StrongSwan is configured such that more than one connection uses the same + tunnel endpoint, it leads to undefined behavior. This is because the tunnel (IPIP/IPsec) instance + is created based on tunnel endpoints and tunnel type. Since the endpoints and type of tunnel are + the same for both connections, it could happen that the second connection (tunnel creation) fails in VPP. + +Startup.conf section: +==================== + +linux-xfrm-nl{ + # Following parameter enables route mode IPsec. + enable-route-mode-ipsec, + # Specifies Ipsec interface type "ipsec" or "ipip". + interface <"interface_type">, + # Set the RX buffer size to be used on the netlink socket. + nl-rx-buffer-size <>, + # Set the batch size - maximum netlink messages to process at one time. + nl-batch-size <>, + # Set the batch delay - how long to wait in ms between processing batches. + nl-batch-delay-ms <> +} From 1ee9d826c81fa27904efd9343f4a1821e80b7198 Mon Sep 17 00:00:00 2001 From: Bheemappa Agasimundin Date: Mon, 20 May 2024 07:50:11 +0000 Subject: [PATCH 067/271] octeon: add virtio support This patch contains following changes: 1.Adds new bus support as virtio. 2.Adds new octeon-virtio driver support. Type: feature JIRA: https://essjira.marvell.com/browse/IPBUSW-49354 Signed-off-by: Bheemappa Agasimundin Change-Id: I2ac8cabc8ede96131d04b1470863e42d66f11a5d Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/128093 Tested-by: sa_ip-sw-jenkins Reviewed-by: Monendra Singh Kushwaha Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/132675 Reviewed-by: Nithinsen Kaithakadan --- build/external/Makefile | 3 +- build/external/packages/dpdk.mk | 31 +- build/external/packages/octeon-dao.mk | 59 +++ .../001-disable-dao-apps.patch | 12 + src/plugins/dev_octeon/CMakeLists.txt | 6 +- src/plugins/dev_octeon/dev_octeon_virtio.mk | 47 ++ src/plugins/dev_octeon/oct_virtio.h | 146 ++++++ src/plugins/dev_octeon/virtio.c | 437 ++++++++++++++++++ src/plugins/dev_octeon/virtio_bus.c | 117 +++++ src/plugins/dev_octeon/virtio_bus.h | 25 + src/plugins/dev_octeon/virtio_ctrl.c | 294 ++++++++++++ src/plugins/dev_octeon/virtio_format.c | 41 ++ src/plugins/dev_octeon/virtio_port.c | 169 +++++++ src/plugins/dev_octeon/virtio_rx_node.c | 350 ++++++++++++++ src/plugins/dev_octeon/virtio_tx_node.c | 182 ++++++++ 15 files changed, 1889 insertions(+), 30 deletions(-) create mode 100644 build/external/packages/octeon-dao.mk create mode 100644 build/external/patches/octeon-dao_24.05.2/001-disable-dao-apps.patch create mode 100644 src/plugins/dev_octeon/dev_octeon_virtio.mk create mode 100644 src/plugins/dev_octeon/oct_virtio.h create mode 100644 src/plugins/dev_octeon/virtio.c create mode 100644 src/plugins/dev_octeon/virtio_bus.c create mode 100644 src/plugins/dev_octeon/virtio_bus.h create mode 100644 src/plugins/dev_octeon/virtio_ctrl.c create mode 100644 src/plugins/dev_octeon/virtio_format.c create mode 100644 src/plugins/dev_octeon/virtio_port.c create mode 100644 src/plugins/dev_octeon/virtio_rx_node.c create mode 100644 src/plugins/dev_octeon/virtio_tx_node.c diff --git a/build/external/Makefile b/build/external/Makefile index af2512c461..be0b7d2437 100644 --- a/build/external/Makefile +++ b/build/external/Makefile @@ -43,13 +43,14 @@ include packages/rdma-core.mk include packages/dpdk.mk include packages/xdp-tools.mk include packages/octeon-roc.mk +include packages/octeon-dao.mk .PHONY: clean clean: @rm -rf $(B) $(I) .PHONY: install -install: $(if $(ARCH_X86_64), ipsec-mb-install) dpdk-install rdma-core-install quicly-install xdp-tools-install $(if $(AARCH64), octeon-roc-install) +install: $(if $(ARCH_X86_64), ipsec-mb-install) dpdk-install rdma-core-install quicly-install xdp-tools-install $(if $(AARCH64), octeon-roc-install, octeon-dao-install) .PHONY: config config: $(if $(ARCH_X86_64), ipsec-mb-config) dpdk-config rdma-core-config quicly-build diff --git a/build/external/packages/dpdk.mk b/build/external/packages/dpdk.mk index 2c5a903027..5179b53bc9 100644 --- a/build/external/packages/dpdk.mk +++ b/build/external/packages/dpdk.mk @@ -48,12 +48,9 @@ endif DPDK_DRIVERS_DISABLED := baseband/\*, \ bus/dpaa, \ bus/ifpga, \ - common/cnxk, \ compress/isal, \ - compress/octeontx, \ compress/zlib, \ crypto/ccp, \ - crypto/cnxk, \ crypto/dpaa_sec, \ crypto/openssl, \ crypto/aesni_mb, \ @@ -61,16 +58,11 @@ DPDK_DRIVERS_DISABLED := baseband/\*, \ crypto/kasumi, \ crypto/snow3g, \ crypto/zuc, \ - event/\*, \ mempool/dpaa, \ - mempool/cnxk, \ - net/af_packet, \ net/bnx2x, \ - net/bonding, \ - net/cnxk, \ net/ipn3ke, \ net/liquidio, \ - net/pcap, \ + net/pcap, \ net/pfe, \ net/sfc, \ net/softnic, \ @@ -81,28 +73,11 @@ DPDK_DRIVERS_DISABLED := baseband/\*, \ DPDK_LIBS_DISABLED := acl, \ bbdev, \ bitratestats, \ - bpf, \ - cfgfile, \ - cnxk, \ - distributor, \ - efd, \ - fib, \ - flow_classify, \ - graph, \ - gro, \ - gso, \ jobstats, \ kni, \ - latencystats, \ - lpm, \ + table, \ member, \ - node, \ - pipeline, \ - port, \ - power, \ - rawdev, \ - rib, \ - table + pipeline DPDK_MLX_CONFIG_FLAG := diff --git a/build/external/packages/octeon-dao.mk b/build/external/packages/octeon-dao.mk new file mode 100644 index 0000000000..cb8f1aaea6 --- /dev/null +++ b/build/external/packages/octeon-dao.mk @@ -0,0 +1,59 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2024 Marvell. + +DAO_DEBUG ?= n +octeon-dao_version := 24.05.2 +octeon-dao_tarball := $(octeon-dao_version).tar.gz +octeon-dao_tarball_md5sum := d052c1bf98b232c149aad250d6cab1f6 + +octeon-dao_tarball_strip_dirs := 1 +octeon-dao_url := https://github.com/MarvellEmbeddedProcessors/dpu-accelerator-offload/archive/refs/tags/$(octeon-dao_tarball) + +octeon_dao_cmake_args ?= + +DAO_BUILD_TYPE:=release +ifeq ($(DAO_DEBUG), y) +DAO_BUILD_TYPE:=debug +endif + +DAO_MESON_ARGS = \ + --default-library static \ + -Dprefer_static=True \ + --buildtype=$(DAO_BUILD_TYPE)\ + -Denable_kmods=false\ + -Dc_link_args='-lnuma' + +PREFIX = $(CNXK_SDK_SYSROOT) +ifeq (,$(findstring $(OCTEON_VERSION),cn10k cn9k)) + DAO_MESON_ARGS += -Dplatform=native + DAO_MESON_ARGS += --prefix $(octeon-dao_install_dir) + PREFIX = $(octeon-dao_install_dir) +else ifeq ($(OCTEON_VERSION), cn10k) + DAO_MESON_ARGS += --cross-file=$(octeon-dao_src_dir)/config/arm64_cn10k_linux_gcc + DAO_MESON_ARGS += --prefix $(CNXK_SDK_SYSROOT) +else ifeq ($(OCTEON_VERSION), cn9k) + DAO_MESON_ARGS += --cross-file=$(octeon-dao_src_dir)/config/arm64_cn9k_linux_gcc + DAO_MESON_ARGS += --prefix $(CNXK_SDK_SYSROOT) +endif + +PIP_DOWNLOAD_DIR = $(CURDIR)/downloads/ +define octeon-dao_config_cmds + PKG_CONFIG_PATH=${PREFIX}/lib/pkgconfig meson setup $(octeon-dao_src_dir) \ + $(octeon-dao_build_dir) \ + $(DAO_MESON_ARGS)\ + | tee $(dao_config_log) && \ + echo "DAO post meson configuration" +endef + +define octeon-dao_build_cmds + cd ${octeon-dao_build_dir} && rm -f $(octeon-dao_build_log) && \ + meson compile -C ${octeon-dao_build_dir} | tee $(octeon-dao_build_log) +endef + +define octeon-dao_install_cmds + cd ${octeon-dao_build_dir} && \ + meson install &&\ + echo "meson installed directory ${octeon-dao_install_dir}" +endef + +$(eval $(call package,octeon-dao)) diff --git a/build/external/patches/octeon-dao_24.05.2/001-disable-dao-apps.patch b/build/external/patches/octeon-dao_24.05.2/001-disable-dao-apps.patch new file mode 100644 index 0000000000..f72da2f9f2 --- /dev/null +++ b/build/external/patches/octeon-dao_24.05.2/001-disable-dao-apps.patch @@ -0,0 +1,12 @@ +diff --git a/meson.build b/meson.build +index 1dcad58..0564087 100644 +--- a/meson.build ++++ b/meson.build +@@ -36,8 +36,6 @@ subdir('config') + if host_machine.cpu_family() == 'aarch64' + if DAO_BUILD_CONF.has('DAO_LIBDPDK_DEP') + subdir('lib') +- subdir('app') +- subdir('tests') + endif + endif diff --git a/src/plugins/dev_octeon/CMakeLists.txt b/src/plugins/dev_octeon/CMakeLists.txt index 1a60d3a28b..877c7b8d27 100644 --- a/src/plugins/dev_octeon/CMakeLists.txt +++ b/src/plugins/dev_octeon/CMakeLists.txt @@ -11,11 +11,14 @@ vpp_plugin_find_library(dev-octeon OCTEON_ROC_LIB "libocteon-roc.a") if (NOT OCTEON_ROC_DIR) message("OCTEON ROC files not found - Marvell OCTEON device plugin disabled") - return() endif() if (NOT OCTEON_ROC_LIB) message("OCTEON ROC library (libocteon-roc.a) not found - Marvell OCTEON device plugin disabled") +endif() + +if (NOT OCTEON_ROC_DIR OR NOT OCTEON_ROC_LIB) + include (dev_octeon_virtio.mk) return () endif() @@ -47,3 +50,4 @@ add_vpp_plugin(dev_octeon ${OCTEON_ROC_LIB} ) +include (dev_octeon_virtio.mk) diff --git a/src/plugins/dev_octeon/dev_octeon_virtio.mk b/src/plugins/dev_octeon/dev_octeon_virtio.mk new file mode 100644 index 0000000000..91ea0eb084 --- /dev/null +++ b/src/plugins/dev_octeon/dev_octeon_virtio.mk @@ -0,0 +1,47 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright(c) 2024 Marvell. + +# Find OCTEON roc files +vpp_plugin_find_library(dev-octeon-virtio DAO_PAL_LIB "libdao_pal.a") +vpp_plugin_find_library(dev-octeon-virtio DAO_VIRT_LIB "libdao_virtio.a") +vpp_plugin_find_library(dev-octeon-virtio DAO_VIRT_NET_LIB "libdao_virtio_net.a") +vpp_plugin_find_library(dev-octeon-virtio DAO_VFIO_LIB "libdao_vfio.a") +vpp_plugin_find_library(dev-octeon-virtio DAO_PEM_LIB "libdao_pem.a") +vpp_plugin_find_library(dev-octeon-virtio DAO_COMM_LIB "libdao_common.a") +vpp_plugin_find_library(dev-octeon-virtio DAO_DPDK_LIB "libdpdk.a") + +vpp_find_path(DAO_NETDEV_INCLUDE_DIR NAMES dao_virtio_netdev.h) + +if (NOT DAO_NETDEV_INCLUDE_DIR) + message("OCTEON VIRTIO DAO files not found - Marvell OCTEON virtio device plugin disabled") + return() +endif() + +set(DAO_CONFG_INCLUDE_DIR "${DAO_NETDEV_INCLUDE_DIR}/..") + +if (NOT DAO_PAL_LIB OR NOT DAO_VIRT_LIB OR NOT DAO_VIRT_NET_LIB OR NOT DAO_VFIO_LIB OR NOT DAO_PEM_LIB OR NOT DAO_COMM_LIB) + message("OCTEON VIRTIO DAO LIBS are not found - Marvell OCTEON virtio device plugin disabled") + return() +endif() + +set (DAO_LIBS ${DAO_PAL_LIB} ${DAO_VIRT_LIB} ${DAO_VIRT_NET_LIB} ${DAO_VFIO_LIB} ${DAO_PEM_LIB} ${DAO_COMM_LIB} ${DAO_DPDK_LIB}) + +unset(DAO_LINK_FLAGS) +string_append(DAO_LINK_FLAGS "-Wl,--whole-archive,${DAO_PAL_LIB},${DAO_VIRT_LIB},${DAO_VIRT_NET_LIB},${DAO_VFIO_LIB},${DAO_PEM_LIB},${DAO_COMM_LIB},${DAO_DPDK_LIB},--no-whole-archive") + +include_directories (${DAO_NETDEV_INCLUDE_DIR}/) +include_directories (${DAO_CONFG_INCLUDE_DIR}/) + +add_vpp_plugin(dev_octeon_virtio + SOURCES + virtio.c + virtio_bus.c + virtio_port.c + virtio_ctrl.c + virtio_tx_node.c + virtio_rx_node.c + virtio_format.c + + LINK_FLAGS + "${DAO_LINK_FLAGS}" +) diff --git a/src/plugins/dev_octeon/oct_virtio.h b/src/plugins/dev_octeon/oct_virtio.h new file mode 100644 index 0000000000..04b54230b0 --- /dev/null +++ b/src/plugins/dev_octeon/oct_virtio.h @@ -0,0 +1,146 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright (c) 2024 Marvell. + */ +#ifndef _OCTEON_VIRTIO_H_ +#define _OCTEON_VIRTIO_H_ + +#undef always_inline + +#include +#include +#include +#include +#include +#include + +#define always_inline static inline __attribute__ ((__always_inline__)) + +#include +#include +#include +#include +#include + +#define VIRTIO_NET_RSS_RETA_SIZE 128 +#define OCT_VIRTIO_DEVICE_ID 0xa70d +#define MAX_JUMBO_PKT_LEN 9600 + +#define foreach_oct_virt_tx_node_counter \ + _ (ENQUE_FAIL, enque_fail, ERROR, "Virtio enqueue failed") + +typedef enum +{ +#define _(f, n, s, d) OCT_VIRT_TX_NODE_CTR_##f, + foreach_oct_virt_tx_node_counter +#undef _ +} oct_tx_node_counter_t; + +typedef struct +{ + u32 sw_if_index; + u16 virtio_id; + u64 tx_q_map; +} oct_virt_tx_trace_t; + +typedef struct +{ + u32 sw_if_index; + u16 virtio_id; + u16 queue_id; + u64 rx_q_map; +} oct_virt_rx_trace_t; + +always_inline vlib_buffer_t * +oct_virt_to_bp (void *b) +{ + return (vlib_buffer_t *) ((u8 *) b + sizeof (struct dao_virtio_net_hdr) - + sizeof (vlib_buffer_t)); +} + +always_inline void * +oct_bp_to_virt (vlib_buffer_t *b) +{ + return (void *) ((u8 *) vlib_buffer_get_current (b) - + sizeof (struct dao_virtio_net_hdr)); +} + +typedef struct +{ + u8 status : 1; + u8 full_duplex : 1; + u16 virtio_id; + u32 pem_devid; + u32 speed; +} oct_virtio_device_t; + +typedef struct +{ + u16 reta_size; + u16 vchan_id; + u16 virtio_id; +} oct_virtio_port_t; + +typedef struct +{ + u64 wrkr_cpu_mask; + u64 netdev_map; + u16 netdev_qp_count[DAO_VIRTIO_DEV_MAX]; + u8 dao_lib_initialized; +} oct_virtio_main_t; + +typedef struct +{ + u8 state; +} oct_virtio_port_map_t; + +typedef struct +{ + u64 qmap; + u16 last_rx_q; + u16 last_tx_q; +} oct_virtio_q_info_t; + +typedef struct +{ + u8 initialized; + u64 netdev_map; + oct_virtio_q_info_t q_map[DAO_VIRTIO_DEV_MAX]; +} oct_virtio_per_thread_data_t; + +int oct_virtio_dev_status_cb (u16 virtio_devid, u8 status); +int oct_virito_rss_reta_configure (u16 virtio_devid, + struct virtio_net_ctrl_rss *rss); +int oct_virtio_configure_promisc (u16 virtio_devid, u8 enable); +int oct_virtio_configure_allmulti (u16 virtio_devid, u8 enabl); +int oct_virtio_mac_addr_set (u16 virtio_devid, u8 *mac); +int oct_virtio_mac_addr_add (u16 virtio_devid, + struct virtio_net_ctrl_mac *mac_tbl, u8 type); +int oct_virtio_mq_configure (u16 virtio_devid, bool qmap_set); +int oct_virtio_vlib_buffer_alloc (u16 devid, void *buffs[], u16 nb_buffs); +int oct_virtio_vlib_buffer_free (u16 devid, void *buffs[], u16 nb_buffs); + +vnet_dev_rv_t oct_virtio_port_init (vlib_main_t *vm, vnet_dev_port_t *port); +void oct_virtio_port_deinit (vlib_main_t *vm, vnet_dev_port_t *port); + +vnet_dev_rv_t oct_virtio_port_start (vlib_main_t *vm, vnet_dev_port_t *port); + +void oct_virtio_port_stop (vlib_main_t *vm, vnet_dev_port_t *port); + +format_function_t format_oct_virt_rx_trace; +format_function_t format_oct_virt_tx_trace; +u8 *format_oct_virt_port_status (u8 *s, va_list *args); +void oct_virt_buffer_pool_dma_map (vlib_main_t *vm); + +#define log_debug(fmt, ...) \ + vlib_log_debug (oct_virt_log.class, fmt, ##__VA_ARGS__) +#define log_info(fmt, ...) \ + vlib_log_info (oct_virt_log.class, fmt, ##__VA_ARGS__) +#define log_notice(fmt, ...) \ + vlib_log_info (oct_virt_log.class, fmt, ##__VA_ARGS__) +#define log_warn(fmt, ...) \ + vlib_log_info (oct_virt_log.class, fmt, ##__VA_ARGS__) +#define log_err(fmt, ...) \ + vlib_log_info (oct_virt_log.class, fmt, ##__VA_ARGS__) + +#endif /* _OCTEON_VIRTIO_H_ */ diff --git a/src/plugins/dev_octeon/virtio.c b/src/plugins/dev_octeon/virtio.c new file mode 100644 index 0000000000..5437060bc7 --- /dev/null +++ b/src/plugins/dev_octeon/virtio.c @@ -0,0 +1,437 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright (c) 2024 Marvell. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#define OCTEON_VIRTIO_DEV "Marvell Octeon virtio network device" + +oct_virtio_main_t *oct_virtio_main = NULL; +oct_virtio_port_map_t *virtio_port_map = NULL; +oct_virtio_per_thread_data_t *oct_virt_thread_data = NULL; + +VLIB_REGISTER_LOG_CLASS (oct_virt_log, static) = { + .class_name = "octeon", + .subclass_name = "virtio_init", +}; + +enum oct_virtio_dev_args_types +{ + DEV_ARG_VIRT_NB_VIRTIO_DEVICES = 1, + DEV_ARG_VIRT_DMA_DEVICE_LIST, + DEV_ARG_VIRT_MISC_DEVICE, + DEV_ARG_VIRT_END +}; + +static vnet_dev_arg_t oct_virtio_dev_args[] = { + { + .id = DEV_ARG_VIRT_NB_VIRTIO_DEVICES, + .name = "nb_virtio", + .desc = "Number of virtio device", + .type = VNET_DEV_ARG_TYPE_UINT32, + .default_val.uint32 = 1, + }, + { + .id = DEV_ARG_VIRT_DMA_DEVICE_LIST, + .name = "dma", + .desc = "DMA device list", + .type = VNET_DEV_ARG_TYPE_STRING, + }, + { + .id = DEV_ARG_VIRT_MISC_DEVICE, + .name = "misc", + .desc = "Miscellaneous device list", + .type = VNET_DEV_ARG_TYPE_STRING, + }, + { + .id = DEV_ARG_VIRT_END, + .name = "end", + .desc = "Argument end", + .type = VNET_DEV_ARG_END, + }, +}; + +#define _(f, n, s, d) \ + { .name = #n, .desc = d, .severity = VL_COUNTER_SEVERITY_##s }, + +vlib_error_desc_t oct_virtio_tx_node_counters[] = { + foreach_oct_virt_tx_node_counter +}; +#undef _ + +vnet_dev_node_t oct_virtio_rx_node = { + .format_trace = format_oct_virt_rx_trace, +}; + +vnet_dev_node_t oct_virtio_tx_node = { + .format_trace = format_oct_virt_tx_trace, + .error_counters = oct_virtio_tx_node_counters, + .n_error_counters = ARRAY_LEN (oct_virtio_tx_node_counters), +}; + +void +oct_virt_buffer_pool_dma_map (vlib_main_t *vm) +{ + uword i; + size_t page_sz; + vlib_physmem_map_t *pm; + vlib_buffer_pool_t *bp; + int iova_mode = rte_eal_iova_mode (); + + vec_foreach (bp, vm->buffer_main->buffer_pools) + { + if (bp->start) + { + pm = vlib_physmem_get_map (vm, bp->physmem_map_index); + page_sz = 1ULL << pm->log2_page_size; + for (i = 0; i < pm->n_pages; i++) + { + char *va = ((char *) pm->base) + i * page_sz; + uword pa = (iova_mode == RTE_IOVA_VA) ? pointer_to_uword (va) : + pm->page_table[i]; + + dao_pal_vfio_dma_map (pointer_to_uword (va), pa, page_sz); + } + } + } +} + +static clib_error_t * +dao_log_read (clib_file_t *uf) +{ + unformat_input_t input; + u8 *line, *s = 0; + int n, n_try; + + n = n_try = 4096; + while (n == n_try) + { + uword len = vec_len (s); + vec_resize (s, len + n_try); + + n = read (uf->file_descriptor, s + len, n_try); + if (n < 0 && errno != EAGAIN) + return clib_error_return_unix (0, "read"); + vec_set_len (s, len + (n < 0 ? 0 : n)); + } + + unformat_init_vector (&input, s); + + while (unformat_user (&input, unformat_line, &line)) + { + vec_add1 (line, 0); + vec_pop (line); + clib_warning ("%v", line); + vec_free (line); + } + + unformat_free (&input); + return 0; +} + +static void +dao_lib_logging (void) +{ + int log_fds[2] = { 0 }; + + if (pipe (log_fds) == 0) + { + if (fcntl (log_fds[0], F_SETFL, O_NONBLOCK) == 0 && + fcntl (log_fds[1], F_SETFL, O_NONBLOCK) == 0) + { + FILE *f = fdopen (log_fds[1], "a"); + if (f && rte_openlog_stream (f) == 0) + { + clib_file_t t = { 0 }; + t.read_function = dao_log_read; + t.file_descriptor = log_fds[0]; + t.description = format (0, "DAO logging pipe"); + clib_file_add (&file_main, &t); + } + } + else + { + close (log_fds[0]); + close (log_fds[1]); + } + } +} + +static u8 * +oct_virtio_probe (vlib_main_t *vm, vnet_dev_bus_index_t bus_index, + void *dev_info) +{ + oct_dev_bus_virtio_device_info_t *di = dev_info; + + if (di->vendor_id != 0x177d || di->device_id != OCT_VIRTIO_DEVICE_ID) + return 0; + + return format (0, "%s", OCTEON_VIRTIO_DEV); +} + +static char ** +oct_populate_dma_device_list (u16 *nb_elem, u8 *dma_list) +{ + char *device = NULL; + char **vec = NULL; + char *saveptr; + u16 count = 0; + + device = strtok_r ((char *) dma_list, ",", &saveptr); + while (device) + { + vec = reallocarray (vec, count + 1, sizeof (vec)); + vec[count] = strdup (device); + count++; + device = strtok_r (saveptr, ",", &saveptr); + } + *nb_elem = count; + return vec; +} + +static void +oct_virtio_parse_arguments (dao_pal_global_conf_t *conf, vnet_dev_arg_t *args) +{ + int i = 0; + vnet_dev_arg_t *a = args; + + for (; a < vec_end (args) && a->val_set; a++) + { + switch (a->id) + { + case DEV_ARG_VIRT_NB_VIRTIO_DEVICES: + conf->nb_virtio_devs = a->val.uint32; + break; + case DEV_ARG_VIRT_DMA_DEVICE_LIST: + conf->dma_devices = + oct_populate_dma_device_list (&conf->nb_dma_devs, a->val.string); + break; + case DEV_ARG_VIRT_MISC_DEVICE: + conf->misc_devices = oct_populate_dma_device_list ( + &conf->nb_misc_devices, a->val.string); + break; + default: + log_info ("Invalid virtio device arguments received\n"); + } + + i++; + } +} + +static void +oct_dev_virtio_mac_addr_get (u16 dev_id, u8 mac_addr[]) +{ + mac_addr[0] = 0x00; + mac_addr[1] = 0x0f; + mac_addr[2] = 0xb7; + mac_addr[3] = 0x11; + mac_addr[4] = (dev_id & PEM_PFVF_DEV_ID_VF_MASK) >> PEM_PFVF_DEV_ID_PF_SHIFT; + mac_addr[5] = + ((dev_id & PEM_PFVF_DEV_ID_VF_MASK) >> PEM_PFVF_DEV_ID_VF_SHIFT) + 1; +} + +static vnet_dev_rv_t +oct_virtio_init (vlib_main_t *vm, vnet_dev_t *dev) +{ + u8 mac_addr[6]; + vnet_dev_rv_t rv; + uint64_t lcore_mask; + oct_virtio_port_t ovp = {}; + dao_pal_global_conf_t conf = { 0 }; + struct dao_virtio_netdev_cbs cbs = {}; + oct_dev_bus_virtio_device_data_t *bus_data; + oct_virtio_device_t *device_data = vnet_dev_get_data (dev); + + bus_data = oct_get_bus_virtio_device_data (dev); + + if (!oct_virtio_main->dao_lib_initialized) + { + /** + * The initialization of the DAO library will be carried out using the + * arguments provided during the first initialization of the virtio + * interface. Any arguments provided from the second virtio device + * onwards will be disregarded. + */ + oct_virtio_parse_arguments (&conf, dev->args); + + if (dao_pal_global_init (&conf)) + { + log_err ("dao_pal_global_init failed\n"); + return VNET_DEV_ERR_UNSUPPORTED_CONFIG; + } + + /* Update lcore_mask with main core */ + lcore_mask = DAO_BIT_ULL (vm->cpu_id) | oct_virtio_main->wrkr_cpu_mask; + + log_debug ("lcore_mask %lu\n", lcore_mask); + if (dao_pal_dma_dev_setup (lcore_mask)) + { + log_err ("dao_pal_dma_dev_setup failed\n"); + rv = VNET_DEV_ERR_UNSUPPORTED_CONFIG; + goto finish; + } + + /* Set main core DMA devices for virtio control */ + if (dao_pal_dma_ctrl_dev_set (vm->cpu_id)) + { + log_err ("dao_pal_dma_dev_setup failed\n"); + rv = VNET_DEV_ERR_UNSUPPORTED_CONFIG; + goto finish; + } + + oct_virt_buffer_pool_dma_map (vm); + + cbs.status_cb = oct_virtio_dev_status_cb; + cbs.rss_cb = oct_virito_rss_reta_configure; + cbs.promisc_cb = oct_virtio_configure_promisc; + cbs.allmulti_cb = oct_virtio_configure_allmulti; + cbs.mac_set = oct_virtio_mac_addr_set; + cbs.mac_add = oct_virtio_mac_addr_add; + cbs.mq_configure = oct_virtio_mq_configure; + cbs.extbuf_get = oct_virtio_vlib_buffer_alloc; + cbs.extbuf_put = oct_virtio_vlib_buffer_free; + + dao_virtio_netdev_cb_register (&cbs); + + oct_virtio_main->dao_lib_initialized = 1; + } + + oct_dev_virtio_mac_addr_get (bus_data->virtio_dev.virtio_id, mac_addr); + + device_data->virtio_id = bus_data->virtio_dev.virtio_id; + ovp.virtio_id = bus_data->virtio_dev.virtio_id; + ovp.reta_size = VIRTIO_NET_RSS_RETA_SIZE; + + vnet_dev_port_add_args_t port_add_args = { + .port = { + .attr = { + .type = VNET_DEV_PORT_TYPE_ETHERNET, + .max_rx_queues = DAO_VIRTIO_MAX_QUEUES, + .max_tx_queues = DAO_VIRTIO_MAX_QUEUES, + .max_supported_rx_frame_size = MAX_JUMBO_PKT_LEN, + .caps = { + .rss = 1, + }, + .rx_offloads = { + .ip4_cksum = 1, + }, + }, + .ops = { + .init = oct_virtio_port_init, + .deinit = oct_virtio_port_deinit, + .start = oct_virtio_port_start, + .stop = oct_virtio_port_stop, + .config_change = NULL, + .format_status = format_oct_virt_port_status, + }, + .data_size = sizeof (oct_virtio_port_t), + .initial_data = &ovp, + }, + .rx_node = &oct_virtio_rx_node, + .tx_node = &oct_virtio_tx_node, + .rx_queue = { + .config = { + .data_size = 0, + .default_size = 1024, + .multiplier = 32, + .min_size = 256, + .max_size = 16384, + }, + .ops = { + .alloc = NULL, + .free = NULL, + .format_info = NULL, + }, + }, + .tx_queue = { + .config = { + .data_size = 0, + .default_size = 1024, + .multiplier = 32, + .min_size = 256, + .max_size = 16384, + }, + .ops = { + .alloc = NULL, + .free = NULL, + .format_info = NULL, + }, + }, + }; + + vnet_dev_set_hw_addr_eth_mac (&port_add_args.port.attr.hw_addr, mac_addr); + + log_info ("MAC address is %U", format_ethernet_address, mac_addr); + + rv = vnet_dev_port_add (vm, dev, 0, &port_add_args); + + return rv; + +finish: + dao_pal_global_fini (); + return rv; +} + +static clib_error_t * +oct_virtio_worker_init (vlib_main_t *vm) +{ + u16 cpu_id = clib_get_current_cpu_id (); + + oct_virtio_main->wrkr_cpu_mask |= DAO_BIT_ULL (cpu_id); + + return 0; +} + +static clib_error_t * +oct_virtio_exit (vlib_main_t *vm) +{ + dao_pal_global_fini (); + return 0; +} + +static clib_error_t * +oct_virtio_plugin_init (vlib_main_t *vm) +{ + dao_lib_logging (); + vec_validate (virtio_port_map, DAO_VIRTIO_DEV_MAX); + vec_validate (oct_virt_thread_data, DAO_PAL_MAX_WORKERS); + vec_validate_aligned (oct_virtio_main, 1, CLIB_CACHE_LINE_BYTES); + return NULL; +} + +static void +oct_virtio_deinit (vlib_main_t *vm, vnet_dev_t *dev) +{ + log_info ("Device unlinitialized\n"); +} + +VLIB_INIT_FUNCTION (oct_virtio_plugin_init); + +VLIB_WORKER_INIT_FUNCTION (oct_virtio_worker_init); + +VLIB_MAIN_LOOP_EXIT_FUNCTION (oct_virtio_exit); + +VNET_DEV_REGISTER_DRIVER (octeon_virtio) = { + .name = "octeon_virtio", + .bus = "virtio", + .device_data_sz = sizeof (oct_virtio_device_t), + .ops = { + .init = oct_virtio_init, + .deinit = oct_virtio_deinit, + .probe = oct_virtio_probe, + }, + .args = oct_virtio_dev_args, +}; + +VLIB_PLUGIN_REGISTER () = { + .version = VPP_BUILD_VER, + .description = "dev_octeon_virtio", +}; diff --git a/src/plugins/dev_octeon/virtio_bus.c b/src/plugins/dev_octeon/virtio_bus.c new file mode 100644 index 0000000000..1a515c9399 --- /dev/null +++ b/src/plugins/dev_octeon/virtio_bus.c @@ -0,0 +1,117 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright (c) 2024 Marvell. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +oct_dev_bus_virtio_device_data_t * +oct_get_bus_virtio_device_data (vnet_dev_t *dev) +{ + return (void *) dev->bus_data; +} + +static int +oct_dev_bus_virtio_device_id_to_virtio_id (u32 *addr, char *str) +{ + unformat_input_t input; + uword rv; + unformat_init_string (&input, str, strlen (str)); + + rv = + unformat (&input, "virtio" VNET_DEV_DEVICE_ID_PREFIX_DELIMITER "%u", addr); + unformat_free (&input); + return rv; +} + +static void * +oct_dev_bus_virtio_get_device_info (vlib_main_t *vm, char *device) +{ + oct_dev_bus_virtio_device_info_t *info; + u32 device_id = 0; + + if (oct_dev_bus_virtio_device_id_to_virtio_id (&device_id, device) == 0) + return 0; + + info = clib_mem_alloc (sizeof (oct_dev_bus_virtio_device_info_t)); + info->virtio_id = device_id; + info->vendor_id = 0x177d; + info->device_id = OCT_VIRTIO_DEVICE_ID; + + return info; +} + +static void +oct_dev_bus_virtio_free_device_info (vlib_main_t *vm, void *dev_info) +{ + clib_mem_free (dev_info); +} + +static vnet_dev_rv_t +oct_dev_bus_virtio_dev_open (vlib_main_t *vm, vnet_dev_t *dev) +{ + oct_dev_bus_virtio_device_info_t *info; + oct_dev_bus_virtio_device_data_t *pd = oct_get_bus_virtio_device_data (dev); + + if ((info = oct_dev_bus_virtio_get_device_info (vm, dev->device_id)) == 0) + return VNET_DEV_ERR_INVALID_DEVICE_ID; + + dev->numa_node = 0; + dev->va_dma = 1; + pd->virtio_dev.device_id = info->device_id; + pd->virtio_dev.vendor_id = info->vendor_id; + pd->virtio_dev.virtio_id = info->virtio_id; + + clib_mem_free (info); + + return VNET_DEV_OK; +} + +static void +oct_bus_virtio_dev_close (vlib_main_t *vm, vnet_dev_t *dev) +{ +} + +static u8 * +format_oct_virtio_device_info (u8 *s, va_list *args) +{ + va_arg (*args, vnet_dev_format_args_t *); + vnet_dev_t *dev = va_arg (*args, vnet_dev_t *); + oct_dev_bus_virtio_device_data_t *pdd = oct_get_bus_virtio_device_data (dev); + + s = format (s, "Virtio ID is %u", pdd->virtio_dev.device_id); + + return s; +} + +static u8 * +format_oct_virtio_device_addr (u8 *s, va_list *args) +{ + vnet_dev_t *dev = va_arg (*args, vnet_dev_t *); + oct_dev_bus_virtio_device_data_t *pdd; + + pdd = oct_get_bus_virtio_device_data (dev); + return format (s, "virtio/%u", pdd->virtio_dev.virtio_id); +} + +VNET_DEV_REGISTER_BUS (virtio) = { + .name = "virtio", + .device_data_size = sizeof (oct_dev_bus_virtio_device_info_t), + .ops = { + .device_open = oct_dev_bus_virtio_dev_open, + .device_close = oct_bus_virtio_dev_close, + .get_device_info = oct_dev_bus_virtio_get_device_info, + .free_device_info = oct_dev_bus_virtio_free_device_info, + .dma_mem_alloc_fn = NULL, + .dma_mem_free_fn = NULL, + .format_device_info = format_oct_virtio_device_info, + .format_device_addr = format_oct_virtio_device_addr, + }, +}; diff --git a/src/plugins/dev_octeon/virtio_bus.h b/src/plugins/dev_octeon/virtio_bus.h new file mode 100644 index 0000000000..c039ca96b2 --- /dev/null +++ b/src/plugins/dev_octeon/virtio_bus.h @@ -0,0 +1,25 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright (c) 2024 Marvell. + */ +#ifndef _VIRTIO_BUS_H_ +#define _VIRTIO_BUS_H_ +#include + +typedef struct +{ + u16 device_id; + u16 vendor_id; + u16 virtio_id; + u16 reserved; +} oct_dev_bus_virtio_device_info_t; + +typedef struct +{ + oct_dev_bus_virtio_device_info_t virtio_dev; +} oct_dev_bus_virtio_device_data_t; + +oct_dev_bus_virtio_device_data_t * +oct_get_bus_virtio_device_data (vnet_dev_t *dev); + +#endif //_VIRTIO_BUS_H diff --git a/src/plugins/dev_octeon/virtio_ctrl.c b/src/plugins/dev_octeon/virtio_ctrl.c new file mode 100644 index 0000000000..cf25d78147 --- /dev/null +++ b/src/plugins/dev_octeon/virtio_ctrl.c @@ -0,0 +1,294 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright (c) 2024 Marvell. + */ + +#include +#include +#include +#include + +#define OCT_VIRTIO_MAX_WRKS 24 + +extern oct_virtio_main_t *oct_virtio_main; +extern oct_virtio_port_map_t *virtio_port_map; +extern oct_virtio_per_thread_data_t *oct_virt_thread_data; + +VLIB_REGISTER_LOG_CLASS (oct_virt_log, static) = { + .class_name = "octeon", + .subclass_name = "virtio_ctl", +}; + +int +oct_virtio_vlib_buffer_free (u16 devid, void *buffs[], u16 nb_buffs) +{ + int i = 0; + u32 bi[nb_buffs]; + vlib_buffer_t *b[nb_buffs]; + vlib_main_t *vm = vlib_get_first_main (); + + for (i = 0; i < nb_buffs; i++) + b[i] = oct_virt_to_bp (buffs[i]); + + vlib_get_buffer_indices (vm, b, bi, nb_buffs); + vlib_buffer_free_no_next (vm, bi, nb_buffs); + + return 0; +} + +int +oct_virtio_vlib_buffer_alloc (u16 devid, void *buffs[], u16 nb_buffs) +{ + int i = 0; + u16 allocated; + u32 vbuf_idxs[nb_buffs]; + vlib_buffer_t *b[nb_buffs]; + vlib_main_t *vm = vlib_get_first_main (); + + allocated = vlib_buffer_alloc (vm, vbuf_idxs, nb_buffs); + if (allocated != nb_buffs) + { + vlib_buffer_free_no_next (vm, vbuf_idxs, allocated); + return -1; + } + vlib_get_buffers (vm, vbuf_idxs, b, nb_buffs); + + for (i = 0; i < nb_buffs; i++) + buffs[i] = oct_bp_to_virt (b[i]); + + return 0; +} + +int +oct_virtio_mac_addr_add (u16 virtio_devid, struct virtio_net_ctrl_mac *mac_tbl, + u8 type) +{ + /* Not supported */ + return 0; +} + +int +oct_virtio_mac_addr_set (u16 virtio_devid, u8 *mac) +{ + /* Not supported */ + return 0; +} + +int +oct_virtio_configure_allmulti (u16 virtio_devid, u8 enable) +{ + /* Not supported */ + return 0; +} + +int +oct_virtio_configure_promisc (u16 virtio_devid, u8 enable) +{ + /* Not supported */ + return 0; +} + +static_always_inline void +oct_virtio_clear_lcore_queue_mapping (u16 virtio_devid) +{ + u32 cpu_id = 0; + oct_virtio_main_t *ovm = oct_virtio_main; + oct_virtio_per_thread_data_t *ptd = oct_virt_thread_data; + u64 wrkr_cpu_mask = ovm->wrkr_cpu_mask; + + ovm->netdev_map &= ~(DAO_BIT (virtio_devid)); + while (wrkr_cpu_mask) + { + if (!(wrkr_cpu_mask & (1 << cpu_id))) + { + cpu_id++; + continue; + } + ptd[cpu_id].netdev_map &= ~(DAO_BIT (virtio_devid)); + ptd[cpu_id].q_map[virtio_devid].qmap = 0; + wrkr_cpu_mask &= ~(1 << cpu_id); + cpu_id++; + } + + ovm->netdev_qp_count[virtio_devid] = 0; +} + +static_always_inline int +oct_virtio_setup_worker_queue_mapping (u16 virtio_devid, u16 virt_q_count) +{ + u32 cpu_id = 0; + u16 virt_rx_q, q_id; + oct_virtio_main_t *ovm = oct_virtio_main; + u64 wrkr_cpu_mask = ovm->wrkr_cpu_mask; + oct_virtio_per_thread_data_t *ptd = oct_virt_thread_data; + + virt_rx_q = virt_q_count / 2; + q_id = 0; + for (q_id = 0; q_id < virt_rx_q && ovm->wrkr_cpu_mask; q_id++) + { + while (!(wrkr_cpu_mask & DAO_BIT_ULL (cpu_id))) + cpu_id++; + + ptd[cpu_id].q_map[virtio_devid].qmap |= DAO_BIT_ULL (q_id); + ptd[cpu_id].netdev_map |= DAO_BIT (virtio_devid); + CLIB_MEMORY_BARRIER (); + + wrkr_cpu_mask &= ~DAO_BIT_ULL (cpu_id); + cpu_id++; + if (!wrkr_cpu_mask) + { + cpu_id = 0; + wrkr_cpu_mask = ovm->wrkr_cpu_mask; + } + } + + ovm->netdev_qp_count[virtio_devid] = virt_q_count / 2; + ovm->netdev_map |= DAO_BIT (virtio_devid); + CLIB_MEMORY_BARRIER (); + + return 0; +} + +int +oct_virtio_mq_configure (u16 virtio_devid, bool qmap_set) +{ + u16 virt_q_count; + + oct_virtio_clear_lcore_queue_mapping (virtio_devid); + if (!qmap_set) + return 0; + + virt_q_count = dao_virtio_netdev_queue_count (virtio_devid); + log_info ("virtio_dev=%u: virt_q_count=%u\n", virtio_devid, virt_q_count); + if (virt_q_count <= 0 || virt_q_count & 0x1 || + virt_q_count >= (DAO_VIRTIO_MAX_QUEUES - 1)) + { + log_err ("virtio_dev=%d: invalid virt_q_count=%d\n", virtio_devid, + virt_q_count); + return -EIO; + } + + oct_virtio_setup_worker_queue_mapping (virtio_devid, virt_q_count); + + return 0; +} + +int +oct_virito_rss_reta_configure (u16 virtio_devid, + struct virtio_net_ctrl_rss *rss) +{ + u16 virt_q_count; + + oct_virtio_clear_lcore_queue_mapping (virtio_devid); + + if (rss == NULL) + return 0; + + /* Get active virt queue count */ + virt_q_count = dao_virtio_netdev_queue_count (virtio_devid); + + if (virt_q_count <= 0 || virt_q_count & 0x1 || + virt_q_count >= (DAO_VIRTIO_MAX_QUEUES - 1)) + { + log_err ("virtio_dev=%d: invalid virt_q_count=%d\n", virtio_devid, + virt_q_count); + return -EIO; + } + + oct_virtio_setup_worker_queue_mapping (virtio_devid, virt_q_count); + + return 0; +} + +int +oct_virtio_dev_status_cb (u16 virtio_devid, u8 status) +{ + u16 virt_q_count; + + log_debug ("[%s] virtio_dev=%d: status=%s\n", __func__, virtio_devid, + dao_virtio_dev_status_to_str (status)); + + switch (status) + { + case VIRTIO_DEV_RESET: + case VIRTIO_DEV_NEEDS_RESET: + virtio_port_map[virtio_devid].state = 0; + CLIB_MEMORY_BARRIER (); + oct_virtio_clear_lcore_queue_mapping (virtio_devid); + break; + case VIRTIO_DEV_DRIVER_OK: + + /* Get active virt queue count */ + virt_q_count = dao_virtio_netdev_queue_count (virtio_devid); + + if (virt_q_count <= 0 || virt_q_count & 0x1 || + virt_q_count >= (DAO_VIRTIO_MAX_QUEUES - 1)) + { + log_err ("virtio_dev=%d: invalid virt_q_count=%d\n", virtio_devid, + virt_q_count); + return -EIO; + } + + oct_virtio_setup_worker_queue_mapping (virtio_devid, virt_q_count); + virtio_port_map[virtio_devid].state = 1; + CLIB_MEMORY_BARRIER (); + break; + default: + break; + } + + return 0; +} + +static_always_inline void +oct_virtio_desc_process (u64 netdev_map, u16 *netdev_qp_count) +{ + u16 dev_id = 0; + + while (netdev_map) + { + if (!(netdev_map & 0x1)) + { + netdev_map >>= 1; + dev_id++; + continue; + } + dao_virtio_net_desc_manage (dev_id, netdev_qp_count[dev_id]); + netdev_map >>= 1; + dev_id++; + } +} + +void +virtio_ctrl_thread_fn (void *args) +{ + vlib_worker_thread_t *w = (vlib_worker_thread_t *) args; + oct_virtio_main_t *ovm = oct_virtio_main; + u32 cpu_id = clib_get_current_cpu_id (); + + vlib_worker_thread_init (w); + ovm->wrkr_cpu_mask |= DAO_BIT (cpu_id); + /* Wait till Octeon virtio DAO lib init is complete */ + while (!ovm || !ovm->dao_lib_initialized) + CLIB_PAUSE (); + + /* Assign DMA devices per lcore */ + dao_pal_thread_init (cpu_id); + ovm->wrkr_cpu_mask &= ~(DAO_BIT (cpu_id)); + + while (1) + { + /* Process virtio descriptors */ + oct_virtio_desc_process (ovm->netdev_map, ovm->netdev_qp_count); + + /* Flush and submit DMA ops */ + dao_dma_flush_submit (); + } +} + +VLIB_REGISTER_THREAD (virtio_ctrl_thread_reg, static) = { + .name = "virtio-ctrl", + .short_name = "virt-ctl", + .function = virtio_ctrl_thread_fn, + .no_data_structure_clone = 1, +}; diff --git a/src/plugins/dev_octeon/virtio_format.c b/src/plugins/dev_octeon/virtio_format.c new file mode 100644 index 0000000000..f21e634882 --- /dev/null +++ b/src/plugins/dev_octeon/virtio_format.c @@ -0,0 +1,41 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright (c) 2024 Marvell. + */ + +#include "vlib/pci/pci.h" +#include "vnet/error.h" +#include "vppinfra/error.h" +#include +#include +#include + +u8 * +format_oct_virt_port_status (u8 *s, va_list *args) +{ + return s; +} + +u8 * +format_oct_virt_rx_trace (u8 *s, va_list *args) +{ + va_arg (*args, vlib_main_t *); + va_arg (*args, vlib_node_t *); + oct_virt_rx_trace_t *t = va_arg (*args, oct_virt_rx_trace_t *); + + s = format (s, "octeon-virt-rx: virtio_id %u sw_if_index %u rx_q_map %lu", + t->virtio_id, t->sw_if_index, t->rx_q_map); + return s; +} + +u8 * +format_oct_virt_tx_trace (u8 *s, va_list *args) +{ + va_arg (*args, vlib_main_t *); + va_arg (*args, vlib_node_t *); + oct_virt_tx_trace_t *t = va_arg (*args, oct_virt_tx_trace_t *); + + s = format (s, "octeon-virt-tx: virtio_id %u sw_if_index %u tx_q_map %lu ", + t->virtio_id, t->sw_if_index, t->tx_q_map); + return s; +} diff --git a/src/plugins/dev_octeon/virtio_port.c b/src/plugins/dev_octeon/virtio_port.c new file mode 100644 index 0000000000..3ebc1caf64 --- /dev/null +++ b/src/plugins/dev_octeon/virtio_port.c @@ -0,0 +1,169 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright (c) 2024 Marvell. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#define VIRTO_NETDEV_SPEED_NUM_UNKNOWN UINT32_MAX /**< Unknown */ +#define OCT_VIRTIO_NIX_RSS_KEY_LEN 48 + +static u64 vchan_bitmap[2] = { 0 }; +extern oct_virtio_port_map_t *virtio_port_map; + +VLIB_REGISTER_LOG_CLASS (oct_virt_log, static) = { + .class_name = "octeon", + .subclass_name = "virtio_port", +}; + +int +oct_virtio_dma_vchan_id_allocate (void) +{ + int idx; + int pos; + + for (int i = 0; i < DAO_VIRTIO_DEV_MAX; i++) + { + idx = i / 64; + pos = i % 64; + if (!(vchan_bitmap[idx] & (1ULL << pos))) + { + vchan_bitmap[idx] |= (1ULL << pos); + return i; + } + } + return -1; +} + +void +oct_virtio_dma_vchan_id_free (int id) +{ + int idx; + int pos; + + if (id >= 0 && id < DAO_VIRTIO_DEV_MAX) + { + idx = id / 64; + pos = id % 64; + vchan_bitmap[idx] &= ~(1ULL << pos); + } +} + +vnet_dev_rv_t +oct_virtio_port_init (vlib_main_t *vm, vnet_dev_port_t *port) +{ + int rrv; + u16 virtio_devid; + u8 buffer_pool_index; + vlib_buffer_pool_t *bp; + vnet_dev_t *dev = port->dev; + struct dao_virtio_netdev_conf netdev_conf = { 0 }; + oct_virtio_device_t *ovd = vnet_dev_get_data (dev); + oct_virtio_port_t *ovp = vnet_dev_get_port_data (port); + + buffer_pool_index = + vlib_buffer_pool_get_default_for_numa (vm, dev->numa_node); + bp = vlib_get_buffer_pool (vm, buffer_pool_index); + virtio_devid = ovd->virtio_id; + + netdev_conf.pem_devid = ovd->pem_devid; + netdev_conf.flags |= DAO_VIRTIO_NETDEV_EXTBUF; + netdev_conf.dataroom_size = bp->data_size; + netdev_conf.reta_size = VIRTIO_NET_RSS_RETA_SIZE; + netdev_conf.link_info.status = 0; + netdev_conf.link_info.speed = VIRTO_NETDEV_SPEED_NUM_UNKNOWN; + netdev_conf.link_info.duplex = 0xFF; + netdev_conf.hash_key_size = OCT_VIRTIO_NIX_RSS_KEY_LEN; + netdev_conf.dma_vchan = oct_virtio_dma_vchan_id_allocate (); + memcpy (netdev_conf.mac, port->attr.hw_addr.eth_mac, + sizeof (netdev_conf.mac)); + log_debug ("port start: port %u, virtio_id %u, vchan_id %d\n", port->port_id, + virtio_devid, netdev_conf.dma_vchan); + + dao_pal_dma_vchan_setup (virtio_devid, netdev_conf.dma_vchan, NULL); + /* Initialize virtio net device */ + rrv = dao_virtio_netdev_init (virtio_devid, &netdev_conf); + if (rrv) + { + log_err ("[%s] dao_virtio_netdev_init failed \n", __func__); + oct_virtio_dma_vchan_id_free (netdev_conf.dma_vchan); + return VNET_DEV_ERR_INTERNAL; + } + ovp->vchan_id = netdev_conf.dma_vchan; + + return VNET_DEV_OK; +} + +void +oct_virtio_port_deinit (vlib_main_t *vm, vnet_dev_port_t *port) +{ + vnet_dev_t *dev = port->dev; + oct_virtio_device_t *ovd = vnet_dev_get_data (dev); + oct_virtio_port_t *ovp = vnet_dev_get_port_data (port); + + log_debug ("clear data for virtio id %u\n", ovd->virtio_id); + dao_virtio_netdev_fini (ovd->virtio_id); + oct_virtio_dma_vchan_id_free (ovp->vchan_id); +} + +void +oct_virt_port_poll (vlib_main_t *vm, vnet_dev_port_t *port) +{ + vnet_dev_t *dev = port->dev; + vnet_dev_port_state_changes_t changes = {}; + oct_virtio_device_t *ovd = vnet_dev_get_data (dev); + u16 virtio_devid = ovd->virtio_id; + + if (ovd->status != virtio_port_map[virtio_devid].state) + { + changes.change.link_state = 1; + changes.link_state = virtio_port_map[virtio_devid].state; + ovd->status = virtio_port_map[virtio_devid].state; + } + else + return; + + vnet_dev_port_state_change (vm, port, changes); +} + +vnet_dev_rv_t +oct_virtio_port_start (vlib_main_t *vm, vnet_dev_port_t *port) +{ + vnet_dev_t *dev = port->dev; + oct_virtio_device_t *ovd = vnet_dev_get_data (dev); + struct dao_virtio_netdev_link_info link_info = { 0 }; + + log_debug ("State up for virtio device %u\n", ovd->virtio_id); + link_info.status = 0x1; + link_info.duplex = 0xFF; + link_info.speed = VIRTO_NETDEV_SPEED_NUM_UNKNOWN; + dao_virtio_netdev_link_sts_update (ovd->virtio_id, &link_info); + + vnet_dev_poll_port_add (vm, port, 0.5, oct_virt_port_poll); + + return VNET_DEV_OK; +} + +void +oct_virtio_port_stop (vlib_main_t *vm, vnet_dev_port_t *port) +{ + vnet_dev_t *dev = port->dev; + oct_virtio_device_t *ovd = vnet_dev_get_data (dev); + struct dao_virtio_netdev_link_info link_info = { 0 }; + + log_debug ("[%s] received dev stop port id %d virtio_id %u\n", __func__, + port->port_id, ovd->virtio_id); + + link_info.status = 0x0; + link_info.duplex = 0xFF; + link_info.speed = VIRTO_NETDEV_SPEED_NUM_UNKNOWN; + dao_virtio_netdev_link_sts_update (ovd->virtio_id, &link_info); + vnet_dev_poll_port_remove (vm, port, oct_virt_port_poll); +} diff --git a/src/plugins/dev_octeon/virtio_rx_node.c b/src/plugins/dev_octeon/virtio_rx_node.c new file mode 100644 index 0000000000..96cc44356f --- /dev/null +++ b/src/plugins/dev_octeon/virtio_rx_node.c @@ -0,0 +1,350 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright (c) 2024 Marvell. + */ + +#include +#include +#include +#include + +#define OCT_VIRT_LENGTH(h) (h->desc_data[1] & 0x00000000FFFFFFFF) +#define OCT_VIRT_NEXT_FLAG(h) \ + (h->desc_data[1] & DAO_BIT_ULL (VRING_DESC_F_NEXT)) + +extern oct_virtio_main_t *oct_virtio_main; +extern oct_virtio_per_thread_data_t *oct_virt_thread_data; + +static_always_inline u32 +oct_virt_trace_rx_buffers (vlib_main_t *vm, vlib_node_runtime_t *node, + vlib_buffer_t **b, u16 nb_pkts, u32 nb_trace, + u64 rx_q_map, u16 virtio_id, u16 next_index) +{ + int idx = 0; + u32 n_traced = 0; + for (idx = 0; idx < nb_pkts && idx < nb_trace; idx++) + { + if (PREDICT_TRUE (vlib_trace_buffer (vm, node, next_index, b[idx], 0))) + { + oct_virt_rx_trace_t *tr = + vlib_add_trace (vm, node, b[idx], sizeof (*tr)); + tr->rx_q_map = rx_q_map; + tr->virtio_id = virtio_id; + n_traced++; + } + } + + return n_traced; +} + +static_always_inline vlib_buffer_t * +oct_virt_populate_inner_segments (vlib_main_t *vm, vlib_buffer_t *head, + void *p, u32 pool_idx) +{ + vlib_buffer_t *prev, *b; + struct dao_virtio_net_hdr *v_hdr; + + v_hdr = (struct dao_virtio_net_hdr *) p; + b = oct_virt_to_bp (p); + head->total_length_not_including_first_buffer += b->current_length; + prev = b; + while (v_hdr->desc_data[0]) + { + v_hdr = (struct dao_virtio_net_hdr *) v_hdr->desc_data[0]; + b = oct_virt_to_bp ((void *) v_hdr->desc_data[0]); + b->current_length = OCT_VIRT_LENGTH (v_hdr); + b->current_data = 0; + prev->flags |= VLIB_BUFFER_NEXT_PRESENT; + head->total_length_not_including_first_buffer += b->current_length; + prev->buffer_pool_index = pool_idx; + prev->next_buffer = vlib_get_buffer_index (vm, b); + prev = b; + } + head->flags |= VLIB_BUFFER_TOTAL_LENGTH_VALID; + + return b; +} + +static_always_inline vlib_buffer_t * +oct_virt_process_chained_packets (vlib_main_t *vm, void **pkts, + u32 *nb_pkts_chain, u32 *n_rx_bytes) +{ + u32 vhdr_len = sizeof (struct virtio_net_hdr); + struct dao_virtio_net_hdr *v_hdr; + vlib_buffer_t *b, *head, *prev = NULL; + u32 buffer_index = 0, len; + u8 pool_idx = 0; + int idx = 0; + + pool_idx = vlib_buffer_pool_get_default_for_numa (vm, 0); + v_hdr = (struct dao_virtio_net_hdr *) pkts[idx]; + len = OCT_VIRT_LENGTH (v_hdr); + head = oct_virt_to_bp (pkts[idx]); + + /** + * If Host uses linux virtio interface skip first buffer as it contains + * only virtio header details + */ + if (len == vhdr_len) + { + buffer_index = vlib_get_buffer_index (vm, head); + vlib_buffer_free_no_next (vm, &buffer_index, 1); + + idx++; + head = oct_virt_to_bp (pkts[idx]); + head->buffer_pool_index = pool_idx; + } + + do + { + v_hdr = (struct dao_virtio_net_hdr *) pkts[idx]; + b = oct_virt_to_bp (pkts[idx]); + b->current_length = OCT_VIRT_LENGTH (v_hdr); + b->current_data = 0; + /* Check for DPU side segmentation */ + if (PREDICT_FALSE ((v_hdr->desc_data[0]))) + b = oct_virt_populate_inner_segments (vm, head, pkts[idx], pool_idx); + + if (prev) + { + prev->flags |= VLIB_BUFFER_NEXT_PRESENT; + head->flags |= VLIB_BUFFER_TOTAL_LENGTH_VALID; + head->total_length_not_including_first_buffer += b->current_length; + prev->buffer_pool_index = pool_idx; + prev->next_buffer = vlib_get_buffer_index (vm, b); + prev = b; + } + prev = b; + idx++; + } + while (OCT_VIRT_NEXT_FLAG (v_hdr)); + + *nb_pkts_chain = idx; + *n_rx_bytes = + head->current_length + head->total_length_not_including_first_buffer; + + return head; +} + +static_always_inline u32 +oct_virtio_process_virtio_packets (vlib_main_t *vm, void **pkts, + vlib_buffer_t **b, u32 *n_rx_pkts, + u32 *to_next, vnet_dev_rx_queue_t *rxq) +{ + u8 flags = 0; + int idx = 0; + u32 nb_pkts = *n_rx_pkts, next_nb_pkts = 0; + u32 n_rx_bytes = 0, nb_pkts_chain = 0; + struct dao_virtio_net_hdr *v_hdr[4]; + vlib_buffer_template_t bt = rxq->buffer_template; + + while (nb_pkts >= 8) + { + v_hdr[0] = (struct dao_virtio_net_hdr *) pkts[idx + 0]; + v_hdr[1] = (struct dao_virtio_net_hdr *) pkts[idx + 1]; + v_hdr[2] = (struct dao_virtio_net_hdr *) pkts[idx + 2]; + v_hdr[3] = (struct dao_virtio_net_hdr *) pkts[idx + 3]; + + flags |= OCT_VIRT_NEXT_FLAG (v_hdr[0]); + flags |= OCT_VIRT_NEXT_FLAG (v_hdr[1]); + flags |= OCT_VIRT_NEXT_FLAG (v_hdr[2]); + flags |= OCT_VIRT_NEXT_FLAG (v_hdr[3]); + + if (PREDICT_FALSE (flags)) + break; + + b[0] = oct_virt_to_bp (pkts[idx + 0]); + b[1] = oct_virt_to_bp (pkts[idx + 1]); + b[2] = oct_virt_to_bp (pkts[idx + 2]); + b[3] = oct_virt_to_bp (pkts[idx + 3]); + + clib_prefetch_store (oct_virt_to_bp (pkts[idx + 4])); + clib_prefetch_store (oct_virt_to_bp (pkts[idx + 5])); + clib_prefetch_store (oct_virt_to_bp (pkts[idx + 6])); + clib_prefetch_store (oct_virt_to_bp (pkts[idx + 7])); + + b[0]->template = bt; + b[1]->template = bt; + b[2]->template = bt; + b[3]->template = bt; + + b[0]->current_length = OCT_VIRT_LENGTH (v_hdr[0]); + b[1]->current_length = OCT_VIRT_LENGTH (v_hdr[1]); + b[2]->current_length = OCT_VIRT_LENGTH (v_hdr[2]); + b[3]->current_length = OCT_VIRT_LENGTH (v_hdr[3]); + + n_rx_bytes += b[0]->current_length; + n_rx_bytes += b[1]->current_length; + n_rx_bytes += b[2]->current_length; + n_rx_bytes += b[3]->current_length; + + to_next[0] = vlib_get_buffer_index (vm, b[0]); + to_next[1] = vlib_get_buffer_index (vm, b[1]); + to_next[2] = vlib_get_buffer_index (vm, b[2]); + to_next[3] = vlib_get_buffer_index (vm, b[3]); + + b += 4; + idx += 4; + to_next += 4; + nb_pkts -= 4; + next_nb_pkts += 4; + } + + while (nb_pkts) + { + nb_pkts_chain = 1; + v_hdr[0] = (struct dao_virtio_net_hdr *) pkts[idx]; + b[0] = oct_virt_to_bp (pkts[idx]); + b[0]->template = bt; + + if (OCT_VIRT_NEXT_FLAG (v_hdr[0])) + b[0] = oct_virt_process_chained_packets (vm, &pkts[idx], + &nb_pkts_chain, &n_rx_bytes); + else + { + b[0]->current_length = OCT_VIRT_LENGTH (v_hdr[0]); + n_rx_bytes += b[0]->current_length; + b[0]->current_data = 0; + } + + to_next[0] = vlib_get_buffer_index (vm, b[0]); + b++; + idx += nb_pkts_chain; + to_next++; + nb_pkts -= nb_pkts_chain; + next_nb_pkts++; + } + + *n_rx_pkts = next_nb_pkts; + + return n_rx_bytes; +} + +static_always_inline uword +oct_virtio_rx_node_inline (vlib_main_t *vm, vlib_node_runtime_t *node, + vlib_frame_t *frame, vnet_dev_port_t *port, + vnet_dev_rx_queue_t *rxq) +{ + u64 q_map; + u64 rx_q_map; + u32 trace_count; + u32 trace_enabled; + oct_virtio_port_t *ovp; + u32 cpu_id = vm->cpu_id; + void *pkts[VLIB_FRAME_SIZE]; + u32 *to_next, n_left_to_next; + u16 queue, virt_q, virtio_id; + vlib_buffer_t *b[VLIB_FRAME_SIZE]; + vnet_main_t *vnm = vnet_get_main (); + u32 thr_idx = vlib_get_thread_index (); + oct_virtio_main_t *ovm = oct_virtio_main; + u32 n_rx_pkts, n_rx_bytes = 0, rx_pkts_total = 0; + oct_virtio_per_thread_data_t *ptd = oct_virt_thread_data; + + trace_enabled = trace_count = vlib_get_trace_count (vm, node); + ovp = vnet_dev_get_port_data (port); + virtio_id = ovp->virtio_id; + + if (PREDICT_FALSE (!ovm || !ovm->dao_lib_initialized)) + return 0; + /* Assign DMA devices per lcore */ + if (PREDICT_FALSE (!ptd[cpu_id].initialized)) + { + dao_pal_thread_init (cpu_id); + dao_pal_dma_lcore_mem2dev_autofree_set (cpu_id, false); + ptd[cpu_id].initialized = 1; + } + + rx_q_map = ptd[cpu_id].q_map[virtio_id].qmap; + q_map = ptd[cpu_id].q_map[virtio_id].qmap; + + if (!(ptd[cpu_id].netdev_map & (DAO_BIT (virtio_id))) || !q_map) + return 0; + + /* Flush and submit DMA ops */ + dao_dma_flush_submit (); + + queue = ptd[cpu_id].q_map[virtio_id].last_rx_q; + + while (rx_q_map) + { + if (!(rx_q_map & DAO_BIT (queue))) + goto next; + + rx_q_map &= ~DAO_BIT (queue); + virt_q = (queue << 1) + 1; + + n_rx_pkts = dao_virtio_net_dequeue_burst_ext (virtio_id, virt_q, pkts, + VLIB_FRAME_SIZE); + + if (!n_rx_pkts) + goto next; + + vlib_get_new_next_frame (vm, node, rxq->next_index, to_next, + n_left_to_next); + + n_rx_bytes += oct_virtio_process_virtio_packets (vm, pkts, b, &n_rx_pkts, + to_next, rxq); + + if (PREDICT_FALSE (trace_count)) + trace_count -= + oct_virt_trace_rx_buffers (vm, node, b, n_rx_pkts, trace_count, + q_map, virtio_id, rxq->next_index); + + if (PREDICT_TRUE (rxq->next_index == + VNET_DEV_ETH_RX_PORT_NEXT_ETH_INPUT)) + { + vlib_next_frame_t *nf; + vlib_frame_t *f; + ethernet_input_frame_t *ef; + nf = vlib_node_runtime_get_next_frame (vm, node, rxq->next_index); + f = vlib_get_frame (vm, nf->frame); + f->flags = ETH_INPUT_FRAME_F_SINGLE_SW_IF_IDX; + /* checksum offload not supported yet */ + f->flags |= ETH_INPUT_FRAME_F_IP4_CKSUM_OK; + + ef = vlib_frame_scalar_args (f); + ef->sw_if_index = port->intf.sw_if_index; + ef->hw_if_index = port->intf.hw_if_index; + + vlib_frame_no_append (f); + } + + n_left_to_next -= n_rx_pkts; + + vlib_put_next_frame (vm, node, rxq->next_index, n_left_to_next); + rx_pkts_total += n_rx_pkts; + + if (rx_pkts_total == VLIB_FRAME_SIZE) + break; + next: + queue = queue + 1; + if (DAO_BIT (queue) > q_map) + queue = 0; + } + + ptd[cpu_id].q_map[virtio_id].last_rx_q = queue; + + if (PREDICT_FALSE (trace_enabled)) + vlib_set_trace_count (vm, node, trace_count); + + vlib_increment_combined_counter ( + vnm->interface_main.combined_sw_if_counters + VNET_INTERFACE_COUNTER_RX, + thr_idx, port->intf.hw_if_index, rx_pkts_total, n_rx_bytes); + + return rx_pkts_total; +} + +VNET_DEV_NODE_FN (oct_virtio_rx_node) +(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame) +{ + u32 n_rx = 0; + + foreach_vnet_dev_rx_queue_runtime (rxq, node) + { + vnet_dev_port_t *port = rxq->port; + n_rx += oct_virtio_rx_node_inline (vm, node, frame, port, rxq); + } + + return n_rx; +} diff --git a/src/plugins/dev_octeon/virtio_tx_node.c b/src/plugins/dev_octeon/virtio_tx_node.c new file mode 100644 index 0000000000..03e0bb492b --- /dev/null +++ b/src/plugins/dev_octeon/virtio_tx_node.c @@ -0,0 +1,182 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright (c) 2024 Marvell. + */ +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +extern oct_virtio_main_t *oct_virtio_main; +extern oct_virtio_per_thread_data_t *oct_virt_thread_data; + +static_always_inline u32 +oct_virtio_enqueue (vlib_main_t *vm, vlib_buffer_t **b, u16 nb_pkts, + u16 virtio_devid) +{ + int idx = 0; + u64 tx_q_map; + u64 q_map; + u32 cpu_id = vm->cpu_id; + u16 nb_pkts_left = nb_pkts; + u16 queue, virt_q, sent = 0, cur_sent = 0; + void *virt_b[VLIB_FRAME_SIZE]; + struct dao_virtio_net_hdr *v_hdr[4]; + struct dao_virtio_net_hdr vhdr_init = {}; + oct_virtio_per_thread_data_t *ptd = oct_virt_thread_data; + + tx_q_map = ptd[cpu_id].q_map[virtio_devid].qmap; + q_map = ptd[cpu_id].q_map[virtio_devid].qmap; + + while (nb_pkts >= 8) + { + v_hdr[0] = oct_bp_to_virt (b[0]); + v_hdr[1] = oct_bp_to_virt (b[1]); + v_hdr[2] = oct_bp_to_virt (b[2]); + v_hdr[3] = oct_bp_to_virt (b[3]); + + *v_hdr[0] = vhdr_init; + *v_hdr[1] = vhdr_init; + *v_hdr[2] = vhdr_init; + *v_hdr[3] = vhdr_init; + + virt_b[idx + 0] = (void *) v_hdr[0]; + virt_b[idx + 1] = (void *) v_hdr[1]; + virt_b[idx + 2] = (void *) v_hdr[2]; + virt_b[idx + 3] = (void *) v_hdr[3]; + + clib_prefetch_store (oct_bp_to_virt (b[4])); + clib_prefetch_store (oct_bp_to_virt (b[5])); + clib_prefetch_store (oct_bp_to_virt (b[6])); + clib_prefetch_store (oct_bp_to_virt (b[7])); + + vlib_prefetch_buffer_header (b[4], LOAD); + vlib_prefetch_buffer_header (b[5], LOAD); + vlib_prefetch_buffer_header (b[6], LOAD); + vlib_prefetch_buffer_header (b[7], LOAD); + + v_hdr[0]->desc_data[1] = b[0]->current_length; + v_hdr[1]->desc_data[1] = b[1]->current_length; + v_hdr[2]->desc_data[1] = b[2]->current_length; + v_hdr[3]->desc_data[1] = b[3]->current_length; + + v_hdr[0]->desc_data[0] = ~b[0]->current_data + 1; + v_hdr[1]->desc_data[0] = ~b[1]->current_data + 1; + v_hdr[2]->desc_data[0] = ~b[2]->current_data + 1; + v_hdr[3]->desc_data[0] = ~b[3]->current_data + 1; + + v_hdr[0]->hdr.num_buffers = 1; + v_hdr[1]->hdr.num_buffers = 1; + v_hdr[2]->hdr.num_buffers = 1; + v_hdr[3]->hdr.num_buffers = 1; + + b += 4; + idx += 4; + nb_pkts -= 4; + } + + while (nb_pkts) + { + v_hdr[0] = oct_bp_to_virt (b[0]); + *v_hdr[0] = vhdr_init; + v_hdr[0]->hdr.num_buffers = 1; + virt_b[idx] = (void *) v_hdr[0]; + v_hdr[0]->desc_data[1] = b[0]->current_length; + v_hdr[0]->desc_data[0] = ~b[0]->current_data + 1; + + b++; + idx++; + nb_pkts--; + } + + queue = ptd[cpu_id].q_map[virtio_devid].last_tx_q; + + while (tx_q_map && nb_pkts_left) + { + if (!(tx_q_map & DAO_BIT (queue))) + goto next; + + tx_q_map &= ~(DAO_BIT (queue)); + virt_q = queue << 1; + cur_sent = dao_virtio_net_enqueue_burst_ext ( + virtio_devid, virt_q, &virt_b[sent], nb_pkts_left); + nb_pkts_left -= cur_sent; + sent += cur_sent; + + next: + queue = queue + 1; + if (DAO_BIT (queue) > q_map) + queue = 0; + } + ptd[cpu_id].q_map[virtio_devid].last_tx_q = queue; + + /* Flush and submit DMA ops */ + dao_dma_flush_submit (); + + return sent; +} + +static_always_inline void +oct_virtio_trace_buffers (vlib_main_t *vm, vlib_node_runtime_t *node, + oct_virtio_port_t *ovp, vlib_buffer_t **b, + u16 n_pkts, u16 virtio_id) +{ + u32 i; + u64 tx_q_map; + u32 cpu_id = clib_get_current_cpu_id (); + oct_virtio_per_thread_data_t *ptd = oct_virt_thread_data; + + tx_q_map = ptd[cpu_id].q_map[virtio_id].qmap; + + for (i = 0; i < n_pkts; i++) + { + if (!(b[i]->flags & VLIB_BUFFER_IS_TRACED)) + continue; + oct_virt_tx_trace_t *t = vlib_add_trace (vm, node, b[i], sizeof (*t)); + t->virtio_id = virtio_id; + t->sw_if_index = vnet_buffer (b[i])->sw_if_index[VLIB_TX]; + t->tx_q_map = tx_q_map; + } +} + +VNET_DEV_NODE_FN (oct_virtio_tx_node) +(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame) +{ + u32 n_tx_pkts; + u16 virtio_id; + oct_virtio_port_t *ovp; + u32 *from = vlib_frame_vector_args (frame); + u16 n_pkts = frame->n_vectors; + vlib_buffer_t *buffers[VLIB_FRAME_SIZE + 8], **b = buffers; + vnet_dev_tx_node_runtime_t *rt = vnet_dev_get_tx_node_runtime (node); + vnet_dev_tx_queue_t *txq = rt->tx_queue; + + if (!txq) + return 0; + + ovp = vnet_dev_get_port_data (txq->port); + virtio_id = ovp->virtio_id; + + vlib_get_buffers (vm, from, b, n_pkts); + + if (PREDICT_FALSE (node->flags & VLIB_NODE_FLAG_TRACE)) + oct_virtio_trace_buffers (vm, node, ovp, b, n_pkts, virtio_id); + + n_tx_pkts = oct_virtio_enqueue (vm, b, n_pkts, virtio_id); + if (n_tx_pkts < n_pkts) + { + u32 n_free = n_pkts - n_tx_pkts; + vlib_buffer_free_no_next (vm, from + n_tx_pkts, n_free); + vlib_error_count (vm, node->node_index, OCT_VIRT_TX_NODE_CTR_ENQUE_FAIL, + n_free); + } + + return n_tx_pkts; +} From 9eb0d6bfe287d8886fd8a5ec594bcfe563327b1e Mon Sep 17 00:00:00 2001 From: Nithinsen Kaithakadan Date: Fri, 26 Jul 2024 13:16:16 +0530 Subject: [PATCH 068/271] octeon: fix crypto pending queue overflow This patch fixes overflow in crypto pending queue entries, by dividing cpt descriptors fairly across worker cores, as software pending queues. Type: fix Signed-off-by: Nithinsen Kaithakadan Change-Id: I01fe1f0e0b482629ad6bae7894f76fcbad0e020b Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/132120 Tested-by: sa_ip-sw-jenkins Reviewed-by: Monendra Singh Kushwaha Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/132688 Reviewed-by: Nithinsen Kaithakadan --- src/plugins/dev_octeon/crypto.c | 62 ++++++++++++++++++++------------- src/plugins/dev_octeon/crypto.h | 3 +- src/plugins/dev_octeon/init.c | 14 +++++--- 3 files changed, 48 insertions(+), 31 deletions(-) diff --git a/src/plugins/dev_octeon/crypto.c b/src/plugins/dev_octeon/crypto.c index b9b20ade21..aab0f969d5 100644 --- a/src/plugins/dev_octeon/crypto.c +++ b/src/plugins/dev_octeon/crypto.c @@ -1356,10 +1356,12 @@ oct_crypto_session_init (vlib_main_t *vm, oct_crypto_sess_t *session, vnet_crypto_key_index_t key_index, int op_type) { oct_crypto_main_t *ocm = &oct_crypto_main; - oct_crypto_dev_t *ocd = ocm->crypto_dev[op_type]; vnet_crypto_key_t *key; + oct_crypto_dev_t *ocd; i32 rv = 0; + ocd = ocm->crypto_dev[op_type]; + key = vnet_crypto_get_key (key_index); if (key->type == VNET_CRYPTO_KEY_TYPE_LINK) @@ -1399,10 +1401,10 @@ static_always_inline int oct_crypto_enqueue_enc_dec (vlib_main_t *vm, vnet_crypto_async_frame_t *frame, const u8 is_aead, u8 aad_len, const u8 type) { - oct_crypto_main_t *ocm = &oct_crypto_main; + u32 i, enq_tail, enc_auth_len, buffer_index, nb_infl_allowed; struct cpt_inst_s inst[VNET_CRYPTO_FRAME_SIZE]; - u32 i, enq_tail, enc_auth_len, buffer_index; u32 crypto_start_offset, integ_start_offset; + oct_crypto_main_t *ocm = &oct_crypto_main; vnet_crypto_async_frame_elt_t *elts; oct_crypto_dev_t *crypto_dev = NULL; oct_crypto_inflight_req_t *infl_req; @@ -1421,6 +1423,14 @@ oct_crypto_enqueue_enc_dec (vlib_main_t *vm, vnet_crypto_async_frame_t *frame, enq_tail = pend_q->enq_tail; + nb_infl_allowed = pend_q->n_desc - pend_q->n_crypto_inflight; + if (PREDICT_FALSE (nb_infl_allowed == 0)) + { + oct_crypto_update_frame_error_status ( + frame, VNET_CRYPTO_OP_STATUS_FAIL_ENGINE_ERR); + return -1; + } + infl_req = &pend_q->req_queue[enq_tail]; infl_req->frame = frame; @@ -1430,20 +1440,21 @@ oct_crypto_enqueue_enc_dec (vlib_main_t *vm, vnet_crypto_async_frame_t *frame, buffer_index = frame->buffer_indices[i]; key = vec_elt_at_index (ocm->keys[type], elts->key_index); - if (!key->sess) + if (PREDICT_FALSE (!key->sess)) { oct_crypto_update_frame_error_status ( frame, VNET_CRYPTO_OP_STATUS_FAIL_ENGINE_ERR); return -1; } + sess = key->sess; + if (PREDICT_FALSE (!sess->initialised)) - { oct_crypto_session_init (vm, sess, elts->key_index, type); - } + crypto_dev = sess->crypto_dev; - memset (inst + i, 0, sizeof (struct cpt_inst_s)); + clib_memset (inst + i, 0, sizeof (struct cpt_inst_s)); buffer = vlib_get_buffer (vm, buffer_index); @@ -1510,38 +1521,32 @@ int oct_crypto_enqueue_linked_alg_enc (vlib_main_t *vm, vnet_crypto_async_frame_t *frame) { - oct_crypto_enqueue_enc_dec (vm, frame, 0 /* is_aead */, 0 /* aad_len */, - VNET_CRYPTO_OP_TYPE_ENCRYPT); - return 0; + return oct_crypto_enqueue_enc_dec ( + vm, frame, 0 /* is_aead */, 0 /* aad_len */, VNET_CRYPTO_OP_TYPE_ENCRYPT); } int oct_crypto_enqueue_linked_alg_dec (vlib_main_t *vm, vnet_crypto_async_frame_t *frame) { - oct_crypto_enqueue_enc_dec (vm, frame, 0 /* is_aead */, 0 /* aad_len */, - VNET_CRYPTO_OP_TYPE_DECRYPT); - return 0; + return oct_crypto_enqueue_enc_dec ( + vm, frame, 0 /* is_aead */, 0 /* aad_len */, VNET_CRYPTO_OP_TYPE_DECRYPT); } int oct_crypto_enqueue_aead_aad_enc (vlib_main_t *vm, vnet_crypto_async_frame_t *frame, u8 aad_len) { - oct_crypto_enqueue_enc_dec (vm, frame, 1 /* is_aead */, aad_len, - VNET_CRYPTO_OP_TYPE_ENCRYPT); - - return 0; + return oct_crypto_enqueue_enc_dec (vm, frame, 1 /* is_aead */, aad_len, + VNET_CRYPTO_OP_TYPE_ENCRYPT); } static_always_inline int oct_crypto_enqueue_aead_aad_dec (vlib_main_t *vm, vnet_crypto_async_frame_t *frame, u8 aad_len) { - oct_crypto_enqueue_enc_dec (vm, frame, 1 /* is_aead */, aad_len, - VNET_CRYPTO_OP_TYPE_DECRYPT); - - return 0; + return oct_crypto_enqueue_enc_dec (vm, frame, 1 /* is_aead */, aad_len, + VNET_CRYPTO_OP_TYPE_DECRYPT); } int @@ -1680,9 +1685,10 @@ int oct_conf_sw_queue (vlib_main_t *vm, vnet_dev_t *dev) { vlib_thread_main_t *tm = vlib_get_thread_main (); - oct_crypto_main_t *ocm = &oct_crypto_main; extern oct_plt_init_param_t oct_plt_init_param; + oct_crypto_main_t *ocm = &oct_crypto_main; oct_crypto_inflight_req_t *infl_req_queue; + u32 n_inflight_req; int i, j = 0; ocm->pend_q = oct_plt_init_param.oct_plt_zmalloc ( @@ -1694,13 +1700,19 @@ oct_conf_sw_queue (vlib_main_t *vm, vnet_dev_t *dev) return -1; } + /* + * Each pending queue will get number of cpt desc / number of cores. + * And that desc count is shared across inflight entries. + */ + n_inflight_req = + (OCT_CPT_LF_MAX_NB_DESC / tm->n_vlib_mains) / VNET_CRYPTO_FRAME_SIZE; + for (i = 0; i < tm->n_vlib_mains; ++i) { - ocm->pend_q[i].n_desc = OCT_CRYPTO_DEFAULT_SW_ASYNC_FRAME_COUNT; + ocm->pend_q[i].n_desc = n_inflight_req; ocm->pend_q[i].req_queue = oct_plt_init_param.oct_plt_zmalloc ( - OCT_CRYPTO_DEFAULT_SW_ASYNC_FRAME_COUNT * - sizeof (oct_crypto_inflight_req_t), + ocm->pend_q[i].n_desc * sizeof (oct_crypto_inflight_req_t), CLIB_CACHE_LINE_BYTES); if (ocm->pend_q[i].req_queue == NULL) { diff --git a/src/plugins/dev_octeon/crypto.h b/src/plugins/dev_octeon/crypto.h index 32cbd66bfb..6cc9f39782 100644 --- a/src/plugins/dev_octeon/crypto.h +++ b/src/plugins/dev_octeon/crypto.h @@ -11,6 +11,8 @@ #define OCT_MAX_N_CPT_DEV 2 +#define OCT_CPT_LF_MAX_NB_DESC 128000 + #define OCT_MAX_CRYPTO_COUNTERS 3 /* counter, name, verbose */ @@ -54,7 +56,6 @@ #define OCT_MOD_INC(i, l) ((i) == (l - 1) ? (i) = 0 : (i)++) #define OCT_SCATTER_GATHER_BUFFER_SIZE 1024 -#define OCT_CRYPTO_DEFAULT_SW_ASYNC_FRAME_COUNT 256 #define CPT_LMT_SIZE_COPY (sizeof (struct cpt_inst_s) / 16) #define OCT_MAX_LMT_SZ 16 diff --git a/src/plugins/dev_octeon/init.c b/src/plugins/dev_octeon/init.c index 9b7751f420..c7d2ed9707 100644 --- a/src/plugins/dev_octeon/init.c +++ b/src/plugins/dev_octeon/init.c @@ -236,7 +236,7 @@ oct_conf_cpt_queue (vlib_main_t *vm, vnet_dev_t *dev, oct_crypto_dev_t *ocd) cpt_lf = &ocd->lf; cpt_lmtline = &ocd->lmtline; - cpt_lf->nb_desc = 8192; + cpt_lf->nb_desc = OCT_CPT_LF_MAX_NB_DESC; cpt_lf->lf_id = 0; if ((rrv = roc_cpt_lf_init (roc_cpt, cpt_lf)) < 0) return cnx_return_roc_err (dev, rrv, "roc_cpt_lf_init"); @@ -279,12 +279,14 @@ oct_init_cpt (vlib_main_t *vm, vnet_dev_t *dev) if ((rrv = oct_conf_cpt_queue (vm, dev, ocd))) return rrv; - oct_conf_sw_queue (vm, dev); - - oct_init_crypto_engine_handlers (vm, dev); - if (!ocm->n_cpt) { + /* + * Initialize s/w queues, which are common across multiple + * crypto devices + */ + oct_conf_sw_queue (vm, dev); + ocm->crypto_dev[0] = ocd; /* Initialize counters */ #define _(i, s, str) \ @@ -298,6 +300,8 @@ oct_init_cpt (vlib_main_t *vm, vnet_dev_t *dev) ocm->crypto_dev[1] = ocd; + oct_init_crypto_engine_handlers (vm, dev); + ocm->n_cpt++; return VNET_DEV_OK; From c0e11867cc10a79a7d4ffde664b3aecce98dfacb Mon Sep 17 00:00:00 2001 From: Monendra Singh Kushwaha Date: Mon, 5 Aug 2024 15:30:22 +0530 Subject: [PATCH 069/271] octeon: enable tx checksum offload capability Type: feature Signed-off-by: Monendra Singh Kushwaha Change-Id: I666f4000cb51c4979567a52255377dbe15ffb202 Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/132833 Tested-by: sa_ip-sw-jenkins Reviewed-by: Nithinsen Kaithakadan --- src/plugins/dev_octeon/init.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/plugins/dev_octeon/init.c b/src/plugins/dev_octeon/init.c index c7d2ed9707..b766eadf53 100644 --- a/src/plugins/dev_octeon/init.c +++ b/src/plugins/dev_octeon/init.c @@ -135,6 +135,9 @@ oct_init_nix (vlib_main_t *vm, vnet_dev_t *dev) .rx_offloads = { .ip4_cksum = 1, }, + .tx_offloads = { + .ip4_cksum = 1, + }, }, .ops = { .init = oct_port_init, From 63b19cef5697fa81779e40968a878e58b2195768 Mon Sep 17 00:00:00 2001 From: Monendra Singh Kushwaha Date: Tue, 6 Aug 2024 14:29:20 +0530 Subject: [PATCH 070/271] octeon: fix tx checksum offload This patch fixes tx checksum offload by setting correct l3/l4 pointer. Type: fix Signed-off-by: Monendra Singh Kushwaha Change-Id: I5d7407c1c2ea7f4c5138c2c304fda829b5e1ffc6 Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/132890 Tested-by: sa_ip-sw-jenkins Reviewed-by: Nithinsen Kaithakadan --- src/plugins/dev_octeon/tx_node.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/plugins/dev_octeon/tx_node.c b/src/plugins/dev_octeon/tx_node.c index 0907493814..6d239836a2 100644 --- a/src/plugins/dev_octeon/tx_node.c +++ b/src/plugins/dev_octeon/tx_node.c @@ -255,19 +255,18 @@ oct_tx_enq1 (vlib_main_t *vm, oct_tx_ctx_t *ctx, vlib_buffer_t *b, if (oflags & VNET_BUFFER_OFFLOAD_F_IP_CKSUM) { d.hdr_w1.ol3type = NIX_SENDL3TYPE_IP4_CKSUM; - d.hdr_w1.ol3ptr = vnet_buffer (b)->l3_hdr_offset; - d.hdr_w1.ol4ptr = - vnet_buffer (b)->l3_hdr_offset + sizeof (ip4_header_t); + d.hdr_w1.ol3ptr = vnet_buffer (b)->l3_hdr_offset - b->current_data; + d.hdr_w1.ol4ptr = d.hdr_w1.ol3ptr + sizeof (ip4_header_t); } if (oflags & VNET_BUFFER_OFFLOAD_F_UDP_CKSUM) { d.hdr_w1.ol4type = NIX_SENDL4TYPE_UDP_CKSUM; - d.hdr_w1.ol4ptr = vnet_buffer (b)->l4_hdr_offset; + d.hdr_w1.ol4ptr = vnet_buffer (b)->l4_hdr_offset - b->current_data; } else if (oflags & VNET_BUFFER_OFFLOAD_F_TCP_CKSUM) { d.hdr_w1.ol4type = NIX_SENDL4TYPE_TCP_CKSUM; - d.hdr_w1.ol4ptr = vnet_buffer (b)->l4_hdr_offset; + d.hdr_w1.ol4ptr = vnet_buffer (b)->l4_hdr_offset - b->current_data; } } From f5f754be580b46cb1df33228b1d4d63acdcfb2b0 Mon Sep 17 00:00:00 2001 From: Bheemappa Agasimundin Date: Fri, 9 Aug 2024 09:48:23 +0000 Subject: [PATCH 071/271] octeon: fix dao checksum This patch updates DAO tar file checksum. Type: fix JIRA: https://essjira.marvell.com/browse/IPBUSW-51694 Signed-off-by: Bheemappa Agasimundin Change-Id: I5f170a023e9a6187cbb7483651bc6a141fe5925a Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/133246 Tested-by: sa_ip-sw-jenkins Reviewed-by: Ashish Gupta --- build/external/packages/octeon-dao.mk | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build/external/packages/octeon-dao.mk b/build/external/packages/octeon-dao.mk index cb8f1aaea6..9207a2dc7e 100644 --- a/build/external/packages/octeon-dao.mk +++ b/build/external/packages/octeon-dao.mk @@ -4,10 +4,10 @@ DAO_DEBUG ?= n octeon-dao_version := 24.05.2 octeon-dao_tarball := $(octeon-dao_version).tar.gz -octeon-dao_tarball_md5sum := d052c1bf98b232c149aad250d6cab1f6 +octeon-dao_tarball_md5sum := a06dda68ab8cb91734c305fc17e2a5ae octeon-dao_tarball_strip_dirs := 1 -octeon-dao_url := https://github.com/MarvellEmbeddedProcessors/dpu-accelerator-offload/archive/refs/tags/$(octeon-dao_tarball) +octeon-dao_url := https://github.com/MarvellEmbeddedProcessors/dao/archive/refs/tags/$(octeon-dao_tarball) octeon_dao_cmake_args ?= From 26e207732d00e73bea13e908757526b3a3e5ef83 Mon Sep 17 00:00:00 2001 From: Nagendra T P Date: Wed, 4 Sep 2024 09:28:54 +0530 Subject: [PATCH 072/271] ci: fix vpp package name in releases section Fix VPP package nameing in releases section of GitHub Signed-off-by: Nagendra T P --- .github/workflows/build.yml | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f4ae5ef522..838938fae9 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -64,9 +64,8 @@ jobs: echo "PKG_POSTFIX=${PKG_POSTFIX}" >> "${PWD}/artifacts/env" source "${PWD}/artifacts/env" echo "NIGHTLY=${NIGHTLY}" >> $GITHUB_OUTPUT - echo "PKG_VERSION_NAME=${PKG_VERSION_NAME}" >> $GITHUB_OUTPUT - echo "MRVL_PKG_VERSION=${MRVL_PKG_VERSION}" >> $GITHUB_OUTPUT echo "DPDK_PKG_VERSION=${DPDK_PKG_VERSION}" >> $GITHUB_OUTPUT + echo "DPDK_BASE_PKG_VERSION=${DPDK_BASE_PKG_VERSION}" >> $GITHUB_OUTPUT - uses: uraimo/run-on-arch-action@v2.7.2 name: Build VPP and generate package id: build @@ -129,9 +128,11 @@ jobs: apt-get install -y libconfuse-dev libelf-dev libffi7 libmnl-dev libnl-3-dev libnl-route-3-dev apt-get install -y libnuma-dev libpcap-dev libssl-dev nasm ninja-build python3-all python3-dev apt-get install -y python3-jsonschema python3-pip python3-ply python3-setuptools python3-venv - apt-get install -y python3-virtualenv tshark + apt-get install -y python3-virtualenv tshark lsb-release run: | source /artifacts/env + DISTRO=ubuntu-`lsb_release -rs` + echo "DISTRO=${DISTRO}" >> /artifacts/env echo "cache_dir = /root/.ccache" > /root/.ccache/ccache.conf ccache -p git config --global --add safe.directory "${PWD}" @@ -159,6 +160,8 @@ jobs: echo "PKG_VERSION_NAME=${PKG_VERSION_NAME}" >> "$GITHUB_OUTPUT" echo $MRVL_PKG_VERSION echo "MRVL_PKG_VERSION=${MRVL_PKG_VERSION}" >> "$GITHUB_OUTPUT" + echo $DISTRO + echo "DISTRO=${DISTRO}" >> "$GITHUB_OUTPUT" echo $PKG_POSTFIX echo "PKG_POSTFIX=${PKG_POSTFIX}" >> "$GITHUB_OUTPUT" [[ "$PKG_POSTFIX" == "-devel" ]] && TAG=devel || TAG=${PKG_VERSION_NAME} @@ -169,11 +172,21 @@ jobs: with: name: vpp-${{ steps.artifacts.outputs.PKG_VERSION_NAME }}-cn10k${{ steps.artifacts.outputs.PKG_POSTFIX }}_${{ steps.artifacts.outputs.MRVL_PKG_VERSION }}_arm64.deb path: ${{ github.workspace }}/artifacts/vpp-${{ steps.artifacts.outputs.PKG_VERSION_NAME }}-cn10k${{ steps.artifacts.outputs.PKG_POSTFIX }}_${{ steps.artifacts.outputs.MRVL_PKG_VERSION }}_arm64.deb + - name: Delete existing release + if: ${{ github.event_name == 'push' }} + env: + GH_TOKEN: ${{ github.token }} + run: | + if gh release view vpp-${{ steps.artifacts.outputs.PKG_VERSION_NAME }}-${{ steps.artifacts.outputs.MRVL_PKG_VERSION }}-${{ steps.artifacts.outputs.DISTRO }}-${{ steps.artifacts.outputs.TAG }}; then + gh release delete vpp-${{ steps.artifacts.outputs.PKG_VERSION_NAME }}-${{ steps.artifacts.outputs.MRVL_PKG_VERSION }}-${{ steps.artifacts.outputs.DISTRO }}-${{ steps.artifacts.outputs.TAG }} --cleanup-tag -y + else + echo "Release not found" + fi - name: Release VPP cn10k package uses: softprops/action-gh-release@v2.0.4 if: ${{ github.event_name == 'push' }} with: - tag_name: ${{ steps.artifacts.outputs.TAG }} + tag_name: vpp-${{ steps.artifacts.outputs.PKG_VERSION_NAME }}-${{ steps.artifacts.outputs.MRVL_PKG_VERSION }}-${{ steps.artifacts.outputs.DISTRO }}-${{ steps.artifacts.outputs.TAG }} files: | ${{ github.workspace }}/artifacts/vpp-${{ steps.artifacts.outputs.PKG_VERSION_NAME }}-cn10k${{ steps.artifacts.outputs.PKG_POSTFIX }}_${{ steps.artifacts.outputs.MRVL_PKG_VERSION }}_arm64.deb - name: Dispatch package update event @@ -185,4 +198,7 @@ jobs: -H "Authorization: Bearer ${{ secrets.PPA_REPO_SECRET }}" \ -H "X-GitHub-Api-Version: 2022-11-28" \ https://api.github.com/repos/marvellembeddedprocessors/packages/dispatches \ - -d '{"event_type":"dispatch-event", "client_payload": {"package" : "vpp", "tag": "${{ steps.artifacts.outputs.TAG }}", "dpdk_tag" : "${{ steps.version.outputs.DPDK_PKG_VERSION }}", "has_dpdk" : "true"}}' + -d '{"event_type":"dispatch-event", "client_payload": {"package" : "vpp", + "tag": "vpp-${{ steps.artifacts.outputs.PKG_VERSION_NAME }}-${{ steps.artifacts.outputs.MRVL_PKG_VERSION }}-${{ steps.artifacts.outputs.DISTRO }}-${{ steps.artifacts.outputs.TAG }}", + "dpdk_tag" : "dpdk-${{ steps.version.outputs.DPDK_BASE_PKG_VERSION }}_${{ steps.version.outputs.DPDK_PKG_VERSION }}-${{ steps.artifacts.outputs.DISTRO }}-${{ steps.version.outputs.DPDK_PKG_VERSION }}", + "has_dpdk" : "true", "distro" : "${{ steps.artifacts.outputs.DISTRO }}"}}' \ No newline at end of file From 937d001d8c5e9a60b85f3612a8873adf46177c5c Mon Sep 17 00:00:00 2001 From: Bheemappa Agasimundin Date: Thu, 2 Nov 2023 16:07:09 +0000 Subject: [PATCH 073/271] octep-cp: add support to read control msg from host This patch adds a new PCIe octeon end point control path plugin which reads mbox messages using octep library. Once messages are read, corresponding actions are applied to datapath. Type: feature JIRA: https://essjira.marvell.com/browse/IPBUSW-37643 https://essjira.marvell.com/browse/IPBUSW-39413 Signed-off-by: Bheemappa Agasimundin Signed-off-by: Aakash Change-Id: Iadd5971185d02ce558c26058cc72577e545d367e Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/115172 Tested-by: sa_ip-sw-jenkins Reviewed-by: Monendra Singh Kushwaha (cherry picked from commit 854c7fb98973a667411f123365ae32a13922748e) Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/121115 Reviewed-by: Shiva Shankar Kommula Tested-by: Shiva Shankar Kommula --- MAINTAINERS | 5 + src/plugins/octep_cp/CMakeLists.txt | 53 ++ .../octep_cp/compat/aarch64/cp_compat.h | 105 +++ .../octep_cp/docs/octep_cp_plugin_doc.md | 47 ++ src/plugins/octep_cp/octep_action.c | 33 + src/plugins/octep_cp/octep_action.h | 49 ++ src/plugins/octep_cp/octep_config.c | 612 ++++++++++++++++++ src/plugins/octep_cp/octep_config.h | 151 +++++ src/plugins/octep_cp/octep_ctrl.c | 236 +++++++ src/plugins/octep_cp/octep_input.c | 352 ++++++++++ src/plugins/octep_cp/octep_input.h | 42 ++ 11 files changed, 1685 insertions(+) create mode 100644 src/plugins/octep_cp/CMakeLists.txt create mode 100644 src/plugins/octep_cp/compat/aarch64/cp_compat.h create mode 100644 src/plugins/octep_cp/docs/octep_cp_plugin_doc.md create mode 100644 src/plugins/octep_cp/octep_action.c create mode 100644 src/plugins/octep_cp/octep_action.h create mode 100644 src/plugins/octep_cp/octep_config.c create mode 100644 src/plugins/octep_cp/octep_config.h create mode 100644 src/plugins/octep_cp/octep_ctrl.c create mode 100644 src/plugins/octep_cp/octep_input.c create mode 100644 src/plugins/octep_cp/octep_input.h diff --git a/MAINTAINERS b/MAINTAINERS index c310e72822..abd8e5bb97 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -431,6 +431,11 @@ I: dispatch-trace M: Dave Barach F: src/plugins/dispatch-trace +Plugin - Marvell OCTEON End Point Control Agent +I: octep-cp +M: Bheemappa Agasimundin +F: src/plugins/octep_cp/ + Plugin - DPDK I: dpdk M: Damjan Marion diff --git a/src/plugins/octep_cp/CMakeLists.txt b/src/plugins/octep_cp/CMakeLists.txt new file mode 100644 index 0000000000..814ba4363c --- /dev/null +++ b/src/plugins/octep_cp/CMakeLists.txt @@ -0,0 +1,53 @@ +# Copyright (c) 2023 Marvell. +# SPDX-License-Identifier: Apache-2.0 +# https://spdx.org/licenses/Apache-2.0.html +# + +unset(OCTEP_INCLUDE_DIR) +unset(OCTEP_LINK_FLAGS) +unset(OCTEP_LIB_DIR) + +set(OCTEP_LIB_DIR_HINT "/usr/marvell") +set(OCTEP_INC_DIR_HINT "/usr/marvell/include") +set(PLAT "aarch64") + +if(NOT CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64.*") + return() +endif() + +vpp_find_path(OCTEP_INCLUDE_DIR octep_cp_lib.h HINTS ${OCTEP_INC_DIR_HINT}) +vpp_plugin_find_library(octep_cp OCTEP_LIB "liboctep_cp.a" HINTS ${OCTEP_LIB_DIR_HINT}) + +if (NOT OCTEP_INCLUDE_DIR) + message(WARNING "-- Octep headers not found - Octep-cp plugin disabled") + return() +endif() + +if (NOT OCTEP_LIB) + message(WARNING "-- Octep lib not found - Octep-cp plugin disabled") + return() +else() + message(WARNING "-- Octep lib found - Octep-cp plugin enabled") +endif() + +set(CMAKE_C_FLAGS "-I${CMAKE_CURRENT_SOURCE_DIR}/compat/${PLAT}") + +get_filename_component(OCTEP_LIB_DIR ${OCTEP_LIB} DIRECTORY) +string_append(OCTEP_LINK_FLAGS "-L${OCTEP_LIB_DIR}") +string_append(OCTEP_LINK_FLAGS "-Wl,--whole-archive,${OCTEP_LIB},-lconfig,--no-whole-archive") + +include_directories (${OCTEP_INCLUDE_DIR}) + +message(STATUS "OCTEP header ${OCTEP_INCLUDE_DIR}") +message(STATUS "OCTEP lib ${OCTEP_LIB_DIR}") +message(STATUS "OCTEP link flag ${OCTEP_LINK_FLAGS}") + +add_vpp_plugin(octep_cp + SOURCES + octep_ctrl.c + octep_config.c + octep_action.c + octep_input.c + LINK_FLAGS + "${OCTEP_LINK_FLAGS}" +) diff --git a/src/plugins/octep_cp/compat/aarch64/cp_compat.h b/src/plugins/octep_cp/compat/aarch64/cp_compat.h new file mode 100644 index 0000000000..a1c1db004d --- /dev/null +++ b/src/plugins/octep_cp/compat/aarch64/cp_compat.h @@ -0,0 +1,105 @@ +/* + * Copyright (c) 2023 Marvell. + * SPDX-License-Identifier: Apache-2.0 + * https://spdx.org/licenses/Apache-2.0.html + */ + +#ifndef __CP_COMPAT_H__ +#define __CP_COMPAT_H__ + +#include +#include + +#define CP_ETHER_ADDR_LEN 6 /**< Length of Ethernet address. */ +#define CP_ETHER_GROUP_ADDR 0x01 /**< Mcast or bcast Eth. addr. */ +#define CP_ETHER_LOCAL_ADMIN_ADDR 0x02 /**< Locally assigned Eth. addr. */ + +#define __cp_always_inline inline __attribute__ ((always_inline)) + +#define cp_rmb() asm volatile("dmb oshld" : : : "memory") +#define cp_io_rmb() cp_rmb () +#define cp_wmb() asm volatile("dmb oshst" : : : "memory") +#define cp_io_wmb() cp_wmb () + +static __cp_always_inline uint32_t +cp_read32_relaxed (const volatile void *addr) +{ + uint32_t val; + + asm volatile("ldr %w[val], [%x[addr]]" : [val] "=r"(val) : [addr] "r"(addr)); + return val; +} + +static __cp_always_inline uint32_t +cp_read32 (const volatile void *addr) +{ + uint32_t val; + val = cp_read32_relaxed (addr); + cp_io_rmb (); + return val; +} + +static __cp_always_inline uint64_t +cp_read64_relaxed (const volatile void *addr) +{ + uint64_t val; + + asm volatile("ldr %x[val], [%x[addr]]" : [val] "=r"(val) : [addr] "r"(addr)); + return val; +} + +static __cp_always_inline uint64_t +cp_read64 (const volatile void *addr) +{ + uint64_t val; + val = cp_read64_relaxed (addr); + cp_io_rmb (); + return val; +} + +static __cp_always_inline void +cp_write32_relaxed (uint32_t val, volatile void *addr) +{ + asm volatile("str %w[val], [%x[addr]]" : : [val] "r"(val), [addr] "r"(addr)); +} + +static __cp_always_inline void +cp_write32 (uint32_t value, volatile void *addr) +{ + cp_io_wmb (); + cp_write32_relaxed (value, addr); +} + +static __cp_always_inline void +cp_write64_relaxed (uint64_t val, volatile void *addr) +{ + asm volatile("str %x[val], [%x[addr]]" : : [val] "r"(val), [addr] "r"(addr)); +} + +static __cp_always_inline void +cp_write64 (uint64_t value, volatile void *addr) +{ + cp_io_wmb (); + cp_write64_relaxed (value, addr); +} + +static inline void +cp_eth_random_addr (uint8_t *addr) +{ + uint64_t r = rand (); + uint8_t *p = (uint8_t *) &r; + + memcpy (addr, p, CP_ETHER_ADDR_LEN); + addr[0] &= (uint8_t) ~CP_ETHER_GROUP_ADDR; /* clear multicast bit */ + addr[0] |= CP_ETHER_LOCAL_ADMIN_ADDR; /* set local assignment bit */ +} + +#endif /* __CP_COMPAT_H__ */ + +/* + * fd.io coding-style-patch-verification: ON + * + * Local Variables: + * eval: (c-set-style "gnu") + * End: + */ diff --git a/src/plugins/octep_cp/docs/octep_cp_plugin_doc.md b/src/plugins/octep_cp/docs/octep_cp_plugin_doc.md new file mode 100644 index 0000000000..43390b5b74 --- /dev/null +++ b/src/plugins/octep_cp/docs/octep_cp_plugin_doc.md @@ -0,0 +1,47 @@ +# OCTEON end point control plain plugin (OCTEP-CP) for VPP {#octep_cp_plugin_doc} + +## Overview +This plugin implements Marvell OCTEON PCIe end point control plane protocol. +Marvell OCTEON firmware provides convenience user library liboctep.so to +setup and interact with the host over mailbox. This octep_cp plugin uses +liboctep.so library to read/send control message from/to host over mailbox. + +## Supported SoC +- OCTEON CN10KXX + +## Usage +The following steps demonstrate how you may bring up VPP with octep_cp, on the +OCTEON connected to host. +1. Enable octep_cp plugin in VPP startup.conf file +2. octep_cp plugin initializes liboctep.so library which initializes SDP firmware. +3. If there are any messages from host firmware puts them into mailbox. +4. octep_cp plugin regularly calls liboctep.so API's to check mailbox. +5. octep_cp plugin applies configuration action requested by host and replies + success or failure to host. + +### Setup +1. OCTEON should be connected to host via SDP interface. +2. Determine SDP interface on OCTEON + "lspci | grep SDP" OR "dmesg | grep sdp" + 0002:1f:00.0 Ethernet controller: Cavium, Inc. Octeon Tx2 SDP Physical Function (rev 51) + 0002:1f:00.1 Ethernet controller: Cavium, Inc. Octeon Tx2 SDP Virtual Function (rev 51) +3. Bind SDP VF to vfio-pci driver + dpdk-devbind.py -b vfio-pci 0002:1f:00.1 +4. Modify startup.conf + - Enable octep_cp plugin + plugins { + plugin octep_cp_plugin.so { enable } + } + - Add SDP VF device under `onp` section + onp { + dev 0002:1f:00.1 + } +5. Determine SDP interface on HOST side + - lspci | grep Cavium + 17:00.0 Network controller: Cavium, Inc. Device b900 + - load OCTEON PF and VF driver, insmod octeon_ep.ko octeon_ep_vf.ko + - create required VF's with 'echo 1 > /sys/bus/pci/devices/0000\:17\:00.0/sriov_numvfs' + +#### Configuration +This plugin uses /usr/bin/cn106xx.cfg configuration file to configure +PCIe end point. diff --git a/src/plugins/octep_cp/octep_action.c b/src/plugins/octep_cp/octep_action.c new file mode 100644 index 0000000000..f00ab17e70 --- /dev/null +++ b/src/plugins/octep_cp/octep_action.c @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2023 Marvell. + * SPDX-License-Identifier: Apache-2.0 + * https://spdx.org/licenses/Apache-2.0.html + */ + +#include +#include +#include "octep_action.h" +#include "octep_ctrl_net.h" +#include +#include + +void +octep_update_pktio (uint8_t cmd, uint32_t value) +{ + vnet_main_t *vnm = vnet_get_main (); + vnet_interface_main_t *im = &vnm->interface_main; + vnet_sw_interface_t *si; + + pool_foreach (si, im->sw_interfaces) + { + vnet_hw_interface_set_mtu (vnm, si->hw_if_index, value); + } +} + +/* + * fd.io coding-style-patch-verification: ON + * + * Local Variables: + * eval: (c-set-style "gnu") + * End: + */ diff --git a/src/plugins/octep_cp/octep_action.h b/src/plugins/octep_cp/octep_action.h new file mode 100644 index 0000000000..3cf9e01522 --- /dev/null +++ b/src/plugins/octep_cp/octep_action.h @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2023 Marvell. + * SPDX-License-Identifier: Apache-2.0 + * https://spdx.org/licenses/Apache-2.0.html + */ + +#ifndef __OCTEP_ACTION_H__ +#define __OCTEP_ACTION_H__ + +#include +#include +#include +#include +#include + +struct pf_config +{ + int n_vfs; +}; + +struct pem_config +{ + int n_pfs; + struct pf_config pfconfig[64]; +}; + +struct octep_pf_vf_cfg +{ + struct pem_config pemconfig[4]; + int n_pems; + int pem_idx; + int pf_idx; + int is_vf; + int vf_idx; +}; + +extern struct octep_pf_vf_cfg cfg_idx; + +void octep_update_pktio (uint8_t cmd, uint32_t value); + +#endif + +/* + * fd.io coding-style-patch-verification: ON + * + * Local Variables: + * eval: (c-set-style "gnu") + * End: + */ diff --git a/src/plugins/octep_cp/octep_config.c b/src/plugins/octep_cp/octep_config.c new file mode 100644 index 0000000000..f704902f3b --- /dev/null +++ b/src/plugins/octep_cp/octep_config.c @@ -0,0 +1,612 @@ +/* + * Copyright (c) 2023 Marvell. + * SPDX-License-Identifier: Apache-2.0 + * https://spdx.org/licenses/Apache-2.0.html + */ + +#include +#include +#include +#include + +#include "octep_cp_lib.h" +#include "octep_config.h" + +struct app_cfg cfg; + +/** + * Object heirarchy + * *(0 or more), +(1 or more) + * + * soc = { pem* }; + * pem = { idx, pf* }; + * pf = { idx, if, info, vf* }; + * vf = { idx, if, info }; + * if = { mtu, mac_addr, link_state, rx_state, autoneg, pause_mode, speed, + * supported_modes, advertisedd_modes}; + * info = { pkind, hb_interval, hb_miss_count }; + */ + +#define CFG_TOKEN_SOC "soc" +#define CFG_TOKEN_BASE_SOC "base_soc" +#define CFG_TOKEN_PEMS "pems" +#define CFG_TOKEN_PFS "pfs" +#define CFG_TOKEN_VFS "vfs" +#define CFG_TOKEN_IDX "idx" +#define CFG_TOKEN_IF_MTU "mtu" +#define CFG_TOKEN_IF_MAC_ADDR "mac_addr" +#define CFG_TOKEN_IF_LSTATE "link_state" +#define CFG_TOKEN_IF_RSTATE "rx_state" +#define CFG_TOKEN_IF_AUTONEG "autoneg" +#define CFG_TOKEN_IF_PMODE "pause_mode" +#define CFG_TOKEN_IF_SPEED "speed" +#define CFG_TOKEN_IF_SMODES "supported_modes" +#define CFG_TOKEN_IF_AMODES "advertised_modes" +#define CFG_TOKEN_INFO_PKIND "pkind" +#define CFG_TOKEN_INFO_HB_INTERVAL "hb_interval" +#define CFG_TOKEN_INFO_HB_MISS_COUNT "hb_miss_count" + +static void +print_if (struct if_cfg *iface) +{ + clib_warning ("mac_addr: %02x:%02x:%02x:%02x:%02x:%02x\n", + iface->mac_addr[0], iface->mac_addr[1], iface->mac_addr[2], + iface->mac_addr[3], iface->mac_addr[4], iface->mac_addr[5]); + clib_warning ("mtu: %d, link: %d, rx: %d, autoneg: 0x%x\n", iface->mtu, + iface->link_state, iface->rx_state, iface->autoneg); + clib_warning ("pause_mode: 0x%x, speed: %d\n", iface->pause_mode, + iface->speed); + clib_warning ("supported_modes: 0x%lx, advertised_modes: 0x%lx\n", + iface->supported_modes, iface->advertised_modes); +} + +static void +print_info (struct octep_fw_info *info) +{ + clib_warning ("pkind: %u, hbi: %u, hbmc: %u\n", info->pkind, + info->hb_interval, info->hb_miss_count); +} + +static void +print_config () +{ + struct pem_cfg *pem; + struct pf_cfg *pf; + struct vf_cfg *vf; + + pem = cfg.pems; + while (pem) + { + pf = pem->pfs; + while (pf) + { + clib_warning ("[%d]:[%d]\n", pem->idx, pf->idx); + print_if (&pf->iface); + print_info (&pf->info); + vf = pf->vfs; + while (vf) + { + clib_warning ("[%d]:[%d]:[%d]\n", pem->idx, pf->idx, vf->idx); + print_if (&vf->iface); + print_info (&vf->info); + vf = vf->next; + } + pf = pf->next; + } + pem = pem->next; + } +} + +static struct pem_cfg * +create_pem (int idx) +{ + struct pem_cfg *pem, *p; + + pem = calloc (sizeof (struct pem_cfg), 1); + if (!pem) + return NULL; + + pem->idx = idx; + if (cfg.pems) + { + p = cfg.pems; + while (p->next) + p = p->next; + + p->next = pem; + } + else + cfg.pems = pem; + + cfg.npem++; + + return pem; +} + +static struct pem_cfg * +get_pem (int idx) +{ + struct pem_cfg *pem; + + if (!cfg.pems) + return NULL; + + pem = cfg.pems; + while (pem) + { + if (pem->idx == idx) + return pem; + pem = pem->next; + } + + return NULL; +} + +static struct pf_cfg * +create_pf (struct pem_cfg *pemcfg, int idx) +{ + struct pf_cfg *pf, *p; + + pf = calloc (sizeof (struct pf_cfg), 1); + if (!pf) + return NULL; + + pf->idx = idx; + if (pemcfg->pfs) + { + p = pemcfg->pfs; + while (p->next) + p = p->next; + + p->next = pf; + } + else + pemcfg->pfs = pf; + + pemcfg->npf++; + + return pf; +} + +static struct pf_cfg * +get_pf (struct pem_cfg *pemcfg, int idx) +{ + struct pf_cfg *pf; + + if (!pemcfg->pfs) + return NULL; + + pf = pemcfg->pfs; + while (pf) + { + if (pf->idx == idx) + return pf; + pf = pf->next; + } + + return NULL; +} + +static struct vf_cfg * +create_vf (struct pf_cfg *pfcfg, int idx) +{ + struct vf_cfg *vf, *p; + + vf = calloc (sizeof (struct vf_cfg), 1); + if (!vf) + return NULL; + + vf->idx = idx; + if (pfcfg->vfs) + { + p = pfcfg->vfs; + while (p->next) + p = p->next; + + p->next = vf; + } + else + pfcfg->vfs = vf; + + pfcfg->nvf++; + + return vf; +} + +static struct vf_cfg * +get_vf (struct pf_cfg *pfcfg, int idx) +{ + struct vf_cfg *vf; + + if (!pfcfg->vfs) + return NULL; + + vf = pfcfg->vfs; + while (vf) + { + if (vf->idx == idx) + return vf; + vf = vf->next; + } + + return vf; +} + +int +get_max_rx_pktlen (void) +{ + struct octep_cp_lib_info info; + + octep_cp_lib_get_info (&info); + if (info.soc_model.flag & + (OCTEP_CP_SOC_MODEL_CN96xx_Ax | OCTEP_CP_SOC_MODEL_CNF95xxN_A0)) + return (16 * 1024); + + return ((64 * 1024) - 1); +} + +static int +parse_if (config_setting_t *lcfg, struct if_cfg *iface) +{ + config_setting_t *mac; + int ival, i, n; + + if (config_setting_lookup_int (lcfg, CFG_TOKEN_IF_MTU, &ival)) + iface->mtu = ival; + + mac = config_setting_get_member (lcfg, CFG_TOKEN_IF_MAC_ADDR); + if (mac) + { + n = config_setting_length (mac); + if (n > ETH_ALEN) + n = ETH_ALEN; + for (i = 0; i < n; i++) + iface->mac_addr[i] = config_setting_get_int_elem (mac, i); + } + if (config_setting_lookup_int (lcfg, CFG_TOKEN_IF_LSTATE, &ival)) + iface->link_state = ival; + if (config_setting_lookup_int (lcfg, CFG_TOKEN_IF_RSTATE, &ival)) + iface->rx_state = ival; + if (config_setting_lookup_int (lcfg, CFG_TOKEN_IF_AUTONEG, &ival)) + iface->autoneg = ival; + if (config_setting_lookup_int (lcfg, CFG_TOKEN_IF_PMODE, &ival)) + iface->pause_mode = ival; + if (config_setting_lookup_int (lcfg, CFG_TOKEN_IF_SPEED, &ival)) + iface->speed = ival; + if (config_setting_lookup_int (lcfg, CFG_TOKEN_IF_SMODES, &ival)) + iface->supported_modes = ival; + if (config_setting_lookup_int (lcfg, CFG_TOKEN_IF_AMODES, &ival)) + iface->advertised_modes = ival; + iface->max_rx_pktlen = get_max_rx_pktlen (); + + return 0; +} + +static int +parse_info (config_setting_t *lcfg, struct octep_fw_info *info) +{ + int ival, ret; + + ret = config_setting_lookup_int (lcfg, CFG_TOKEN_INFO_PKIND, &ival); + info->pkind = (ret == CONFIG_TRUE) ? ival : 0; + + ret = config_setting_lookup_int (lcfg, CFG_TOKEN_INFO_HB_INTERVAL, &ival); + info->hb_interval = (ret == CONFIG_TRUE) ? ival : DEFAULT_HB_INTERVAL_MSECS; + + ret = config_setting_lookup_int (lcfg, CFG_TOKEN_INFO_HB_MISS_COUNT, &ival); + info->hb_miss_count = (ret == CONFIG_TRUE) ? ival : DEFAULT_HB_MISS_COUNT; + + return 0; +} + +static int +parse_pf (config_setting_t *pf, struct pf_cfg *pfcfg) +{ + config_setting_t *vfs, *vf; + int nvfs, i, idx, err; + struct vf_cfg *vfcfg; + + err = parse_if (pf, &pfcfg->iface); + if (err) + return err; + + err = parse_info (pf, &pfcfg->info); + if (err) + return err; + + vfs = config_setting_get_member (pf, CFG_TOKEN_VFS); + if (!vfs) + return 0; + nvfs = config_setting_length (vfs); + cfg_idx.pemconfig[0].pfconfig[pfcfg->idx].n_vfs = nvfs; + for (i = 0; i < nvfs; i++) + { + vf = config_setting_get_elem (vfs, i); + if (!vf) + continue; + if (config_setting_lookup_int (vf, CFG_TOKEN_IDX, &idx) == CONFIG_FALSE) + continue; + vfcfg = get_vf (pfcfg, idx); + if (!vfcfg) + { + vfcfg = create_vf (pfcfg, idx); + if (!vfcfg) + { + clib_warning ("Oom for pf[%d]vf[%d]\n", pfcfg->idx, idx); + continue; + } + } + err = parse_if (vf, &vfcfg->iface); + if (err) + return err; + err = parse_info (vf, &vfcfg->info); + if (err) + return err; + } + + return 0; +} + +static int +parse_pem (config_setting_t *pem, struct pem_cfg *pemcfg) +{ + config_setting_t *pfs, *pf; + int npfs, i, idx, err; + struct pf_cfg *pfcfg; + + pfs = config_setting_get_member (pem, CFG_TOKEN_PFS); + if (!pfs) + return 0; + + npfs = config_setting_length (pfs); + for (i = 0; i < npfs; i++) + { + pf = config_setting_get_elem (pfs, i); + if (!pf) + continue; + if (config_setting_lookup_int (pf, CFG_TOKEN_IDX, &idx) == CONFIG_FALSE) + continue; + pfcfg = get_pf (pemcfg, idx); + if (!pfcfg) + { + pfcfg = create_pf (pemcfg, idx); + if (!pfcfg) + { + clib_warning ("Oom for pem[%d]pf[%d]\n", pemcfg->idx, idx); + continue; + } + } + err = parse_pf (pf, pfcfg); + if (err) + return err; + } + + return 0; +} + +static int +parse_pems (config_setting_t *pems) +{ + config_setting_t *pem; + int npems, i, idx, err; + struct pem_cfg *pemcfg; + + npems = config_setting_length (pems); + for (i = 0; i < npems; i++) + { + pem = config_setting_get_elem (pems, i); + if (!pem) + continue; + if (config_setting_lookup_int (pem, CFG_TOKEN_IDX, &idx) == CONFIG_FALSE) + continue; + pemcfg = get_pem (idx); + if (!pemcfg) + { + pemcfg = create_pem (idx); + if (!pemcfg) + { + clib_warning ("Oom for pem[%d]\n", idx); + continue; + } + } + err = parse_pem (pem, pemcfg); + if (err) + return err; + } + + return 0; +} + +static int +parse_base_config (const char *cfg_file_path) +{ + config_setting_t *lcfg, *pems; + config_t fcfg; + int err; + + clib_warning ("base config file : %s\n", cfg_file_path); + config_init (&fcfg); + if (!config_read_file (&fcfg, cfg_file_path)) + { + clib_warning ("%s:%d - %s\n", config_error_file (&fcfg), + config_error_line (&fcfg), config_error_text (&fcfg)); + config_destroy (&fcfg); + return (EXIT_FAILURE); + } + + lcfg = config_lookup (&fcfg, CFG_TOKEN_SOC); + if (!lcfg) + { + config_destroy (&fcfg); + return -EINVAL; + } + + pems = config_setting_get_member (lcfg, CFG_TOKEN_PEMS); + if (pems) + { + err = parse_pems (pems); + if (err) + { + config_destroy (&fcfg); + return err; + } + } + + config_destroy (&fcfg); + + return 0; +} + +int +octep_cp_config_init (const char *cfg_file_path) +{ + config_setting_t *lcfg, *pems; + const char *str; + config_t fcfg; + int err; + + clib_warning ("config file : %s\n", cfg_file_path); + config_init (&fcfg); + if (!config_read_file (&fcfg, cfg_file_path)) + { + clib_warning ("%s:%d - %s\n", config_error_file (&fcfg), + config_error_line (&fcfg), config_error_text (&fcfg)); + config_destroy (&fcfg); + return -EINVAL; + } + + lcfg = config_lookup (&fcfg, CFG_TOKEN_SOC); + if (!lcfg) + { + config_destroy (&fcfg); + return -EINVAL; + } + + if (config_setting_lookup_string (lcfg, CFG_TOKEN_BASE_SOC, &str)) + { + err = parse_base_config (str); + if (err) + { + config_destroy (&fcfg); + return err; + } + } + + pems = config_setting_get_member (lcfg, CFG_TOKEN_PEMS); + if (pems) + { + err = parse_pems (pems); + if (err) + { + config_destroy (&fcfg); + return err; + } + } + + config_destroy (&fcfg); + + print_config (); + + return 0; +} + +int +app_config_get_if_from_msg_info (union octep_cp_msg_info *ctx_info, + union octep_cp_msg_info *msg_info, + struct if_cfg **iface, + struct if_stats **ifstats, + struct octep_fw_info **info) +{ + struct pem_cfg *pem = cfg.pems; + struct pf_cfg *pf; + struct vf_cfg *vf; + + while (pem) + { + if (pem->idx == ctx_info->s.pem_idx) + { + pf = pem->pfs; + while (pf) + { + if (pf->idx == ctx_info->s.pf_idx) + { + if (!msg_info->s.is_vf) + { + clib_warning ("pem[%u] pf[%u]\n", pem->idx, pf->idx); + cfg_idx.pem_idx = pem->idx; + cfg_idx.pf_idx = pf->idx; + cfg_idx.is_vf = 0; + cfg_idx.vf_idx = 0; + *iface = &pf->iface; + *ifstats = &pf->ifstats; + *info = &pf->info; + return 0; + } + vf = pf->vfs; + while (vf) + { + if (vf->idx == msg_info->s.vf_idx) + { + clib_warning ("pem[%u] pf[%u] vf[%u]\n", pem->idx, + pf->idx, vf->idx); + cfg_idx.pem_idx = pem->idx; + cfg_idx.pf_idx = pf->idx; + cfg_idx.is_vf = 1; + cfg_idx.vf_idx = vf->idx; + *iface = &vf->iface; + *ifstats = &vf->ifstats; + *info = &vf->info; + return 0; + } + vf = vf->next; + } + } + pf = pf->next; + } + } + pem = pem->next; + } + + return -ENOENT; +} + +int +octep_cp_config_uninit () +{ + struct pem_cfg *pem, *pp; + struct pf_cfg *pf, *pfp; + struct vf_cfg *vf, *vfp; + + clib_warning ("config uninit\n"); + pem = cfg.pems; + while (pem) + { + pf = pem->pfs; + while (pf) + { + vf = pf->vfs; + while (vf) + { + vfp = vf->next; + free (vf); + vf = vfp; + } + pfp = pf->next; + free (pf); + pf = pfp; + } + pp = pem->next; + free (pem); + pem = pp; + } + + return 0; +} + +/* + * fd.io coding-style-patch-verification: ON + * + * Local Variables: + * eval: (c-set-style "gnu") + * End: + */ diff --git a/src/plugins/octep_cp/octep_config.h b/src/plugins/octep_cp/octep_config.h new file mode 100644 index 0000000000..1e28395f2e --- /dev/null +++ b/src/plugins/octep_cp/octep_config.h @@ -0,0 +1,151 @@ +/* + * Copyright (c) 2023 Marvell. + * SPDX-License-Identifier: Apache-2.0 + * https://spdx.org/licenses/Apache-2.0.html + */ + +#ifndef __OCTEP_CONFIG_H__ +#define __OCTEP_CONFIG_H__ + +#include + +#include +#include "octep_action.h" + +#ifndef ETH_ALEN +#define ETH_ALEN 6 +#endif + +#define MIN_HB_INTERVAL_MSECS 1000 +#define MAX_HB_INTERVAL_MSECS 15000 +#define DEFAULT_HB_INTERVAL_MSECS MIN_HB_INTERVAL_MSECS + +#define DEFAULT_HB_MISS_COUNT 20 + +/* Network interface stats */ +struct if_stats +{ + struct octep_iface_rx_stats rx_stats; + struct octep_iface_tx_stats tx_stats; +}; + +/* Network interface data */ +struct if_cfg +{ + u16 idx; + u16 host_if_id; + /* Current MTU of the interface */ + u16 mtu; + /* Max Receive packet length of the interface */ + u16 max_rx_pktlen; + u8 mac_addr[ETH_ALEN]; + /* Enum octep_ctrl_net_state */ + u16 link_state; + /* Enum octep_ctrl_net_state */ + u16 rx_state; + /* OCTEP_LINK_MODE_XXX */ + u8 autoneg; + /* OCTEP_LINK_MODE_XXX */ + u8 pause_mode; + /* SPEED_XXX */ + u32 speed; + /* OCTEP_LINK_MODE_XXX */ + u64 supported_modes; + /* OCTEP_LINK_MODE_XXX */ + u64 advertised_modes; +}; + +/* Virtual function configuration */ +struct vf_cfg +{ + /* VF index */ + int idx; + /* Network interface data */ + struct if_cfg iface; + struct if_stats ifstats; + struct octep_fw_info info; + struct vf_cfg *next; +}; + +/* Physical function configuration */ +struct pf_cfg +{ + /* PF index */ + int idx; + /* Network interface data */ + struct if_cfg iface; + struct if_stats ifstats; + struct octep_fw_info info; + /* Number of vf's */ + int nvf; + /* Configuration for vf's */ + struct vf_cfg *vfs; + struct pf_cfg *next; +}; + +/* PEM configuration */ +struct pem_cfg +{ + /* PEM index */ + int idx; + /* Number of pf's */ + int npf; + /* Nonfiguration for pf's */ + struct pf_cfg *pfs; + struct pem_cfg *next; +}; + +/* App configuration */ +struct app_cfg +{ + /* Number of pem's */ + int npem; + /* Nonfiguration for pem's */ + struct pem_cfg *pems; +}; + +extern struct app_cfg cfg; + +/* + * Parse file and populate configuration. + * + * @param cfg_file_path: Path to configuration file. + * + * return value: 0 on success, -errno on failure. + */ +int octep_cp_config_init (const char *cfg_file_path); + +/* + * Get interface based on information in message header. + * + * @param ctx_info: non-null pointer to message context info. This is the + * pem->pf context used to poll for messages. + * @param msg_info: non-null pointer to message info. This is the info from + * received message. + * @param iface: non-null pointer to struct if_cfg *. + * @param ifstats: non-null pointer to struct if_stats *. + * @param info: non-null pointer to struct octep_fw_info *. + * + * return value: 0 on success, -errno on failure. + */ +int app_config_get_if_from_msg_info (union octep_cp_msg_info *ctx_info, + union octep_cp_msg_info *msg_info, + struct if_cfg **iface, + struct if_stats **ifstats, + struct octep_fw_info **info); + +/* + * Free allocated configuration artifacts. + * return value: 0 on success, -errno on failure. + */ +int octep_cp_config_uninit (); + +#endif /* __OCTEP_CONFIG_H__ */ + +/* + * fd.io coding-style-patch-verification: ON + * + * Local Variables: + * eval: (c-set-style "gnu") + * End: + */ diff --git a/src/plugins/octep_cp/octep_ctrl.c b/src/plugins/octep_cp/octep_ctrl.c new file mode 100644 index 0000000000..776853f176 --- /dev/null +++ b/src/plugins/octep_cp/octep_ctrl.c @@ -0,0 +1,236 @@ +/* + * Copyright (c) 2023 Marvell. + * SPDX-License-Identifier: Apache-2.0 + * https://spdx.org/licenses/Apache-2.0.html + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "octep_cp_lib.h" +#include "octep_input.h" +#include "octep_config.h" +#include +#include +#include +#include + +/* Control plane version */ +#define CP_VERSION_MAJOR 1 +#define CP_VERSION_MINOR 0 +#define CP_VERSION_VARIANT 0 +#define MAX_EVENTS 6 + +#define CP_VERSION_CURRENT \ + (OCTEP_CP_VERSION (CP_VERSION_MAJOR, CP_VERSION_MINOR, CP_VERSION_VARIANT)) + +#define SOC_CFG_PATH "/usr/bin/cn106xx.cfg" +struct octep_pf_vf_cfg cfg_idx; + +/* + * The PERST# (PCI Express Reset) signal is an open drain, active low output + * from the root port. It is released when all power rails and the REFCLK + * signal have stabilized. + */ +static volatile int perst = 0; +static int hb_interval = 0; + +struct octep_cp_lib_cfg cp_lib_cfg = { 0 }; + +static int +process_events () +{ + struct octep_cp_event_info e[MAX_EVENTS]; + int n, i; + + n = octep_cp_lib_recv_event (e, MAX_EVENTS); + if (n < 0) + return n; + + for (i = 0; i < n; i++) + { + if (e[i].e == OCTEP_CP_EVENT_TYPE_PERST) + { + clib_warning ("Event: perst on dom[%d]\n", e[i].u.perst.dom_idx); + perst = 1; + } + } + + return 0; +} + +static int +send_heartbeat () +{ + struct octep_cp_event_info info; + int i, j; + + info.e = OCTEP_CP_EVENT_TYPE_HEARTBEAT; + for (i = 0; i < cp_lib_cfg.ndoms; i++) + { + info.u.hbeat.dom_idx = cp_lib_cfg.doms[i].idx; + for (j = 0; j < cp_lib_cfg.doms[i].npfs; j++) + { + info.u.hbeat.pf_idx = cp_lib_cfg.doms[i].pfs[j].idx; + octep_cp_lib_send_event (&info); + } + } + + return 0; +} + +void +sigint_handler (int sig_num) +{ + if (sig_num == SIGALRM) + { + if (perst) + return; + send_heartbeat (); + alarm (hb_interval); + } +} + +static int +set_fw_ready (int ready) +{ + struct octep_cp_event_info info; + int i, j; + + info.e = OCTEP_CP_EVENT_TYPE_FW_READY; + info.u.fw_ready.ready = ready; + for (i = 0; i < cp_lib_cfg.ndoms; i++) + { + info.u.fw_ready.dom_idx = cp_lib_cfg.doms[i].idx; + for (j = 0; j < cp_lib_cfg.doms[i].npfs; j++) + { + info.u.fw_ready.pf_idx = cp_lib_cfg.doms[i].pfs[j].idx; + octep_cp_lib_send_event (&info); + } + } + + return 0; +} + +static uword +octep_cp_process (vlib_main_t *vm, vlib_node_runtime_t *node, + vlib_frame_t *frame) +{ + int err = 0, i, j; + struct pem_cfg *pem; + struct pf_cfg *pf; + const char *soc_cfg = SOC_CFG_PATH; + + /* init will wake it up */ + vlib_process_wait_for_event (vm); + + err = octep_cp_config_init (soc_cfg); + if (err) + return err; + + // signal(SIGINT, sigint_handler); + signal (SIGALRM, sigint_handler); + + cp_lib_cfg.ndoms = cfg.npem; + cp_lib_cfg.min_version = CP_VERSION_CURRENT; + cp_lib_cfg.max_version = CP_VERSION_CURRENT; + cfg_idx.n_pems = cfg.npem; + pem = cfg.pems; + i = 0; + while (pem) + { + cp_lib_cfg.doms[i].idx = pem->idx; + cp_lib_cfg.doms[i].npfs = pem->npf; + cfg_idx.pemconfig[i].n_pfs = pem->npf; + pf = pem->pfs; + j = 0; + while (pf) + { + cp_lib_cfg.doms[i].pfs[j++].idx = pf->idx; + if (hb_interval == 0 || pf->info.hb_interval < hb_interval) + hb_interval = pf->info.hb_interval; + + pf = pf->next; + } + pem = pem->next; + i++; + } + err = octep_cp_lib_init (&cp_lib_cfg); + if (err) + return err; + + err = octep_cp_initialize_receive_vector (); + if (err) + { + octep_cp_lib_uninit (); + return err; + } + + set_fw_ready (1); + clib_warning ("Heartbeat interval : %u msecs\n", hb_interval); + hb_interval /= 1000; + alarm (hb_interval); + while (1) + { + /* + * Host PF driver has a timeout of 500ms, so keeping polling interval + * less than that(100ms). Else the host PF driver octeon_ep.ko timesout + */ + vlib_process_wait_for_event_or_clock (vm, 0.1); + vlib_process_get_events (vm, NULL); + loop_process_msgs (); + process_events (); + } + return 0; +} + +static clib_error_t * +octep_cp_process_exit (vlib_main_t *vm) +{ + set_fw_ready (0); + octep_cp_lib_uninit (); + octep_cp_initialize_receive_vector (); + octep_cp_config_uninit (); + return 0; +} + +VLIB_REGISTER_NODE (octep_cp_process_node, static) = { + .function = octep_cp_process, + .name = "octep-cp-process", + .type = VLIB_NODE_TYPE_PROCESS, + .process_log2_n_stack_bytes = 17, +}; + +clib_error_t * +octep_cp_init (vlib_main_t *vm) +{ + + vlib_process_signal_event (vlib_get_main (), octep_cp_process_node.index, 0, + 0); + return NULL; +} + +VLIB_INIT_FUNCTION (octep_cp_init) = { + .runs_after = VLIB_INITS ("onp_plugin_init"), +}; + +VLIB_PLUGIN_REGISTER () = { + .version = VPP_BUILD_VER, + .description = "Octep Control Agent", + .default_disabled = 1, +}; + +VLIB_MAIN_LOOP_EXIT_FUNCTION (octep_cp_process_exit); + +/* + * fd.io coding-style-patch-verification: ON + * + * Local Variables: + * eval: (c-set-style "gnu") + * End: + */ diff --git a/src/plugins/octep_cp/octep_input.c b/src/plugins/octep_cp/octep_input.c new file mode 100644 index 0000000000..9ce9d2428f --- /dev/null +++ b/src/plugins/octep_cp/octep_input.c @@ -0,0 +1,352 @@ +/* + * Copyright (c) 2023 Marvell. + * SPDX-License-Identifier: Apache-2.0 + * https://spdx.org/licenses/Apache-2.0.html + */ +#include +#include +#include +#include +#include + +#include "octep_cp_lib.h" +#include "cp_compat.h" +#include "octep_ctrl_net.h" +#include "octep_hw.h" +#include "octep_input.h" +#include "octep_action.h" +#include "octep_config.h" + +#define LOOP_RX_BUF_CNT 6 + +static struct octep_cp_msg rx_msg[LOOP_RX_BUF_CNT]; +static int rx_num = LOOP_RX_BUF_CNT; +static int max_msg_sz = sizeof (union octep_ctrl_net_max_data); + +extern struct octep_cp_lib_cfg cp_lib_cfg; +extern struct octep_pf_vf_cfg cfg_idx; + +static const uint32_t resp_hdr_sz = sizeof (union octep_ctrl_net_resp_hdr); +static const uint32_t mtu_sz = sizeof (struct octep_ctrl_net_h2f_resp_cmd_mtu); +static const uint32_t mac_sz = sizeof (struct octep_ctrl_net_h2f_resp_cmd_mac); +static const uint32_t state_sz = + sizeof (struct octep_ctrl_net_h2f_resp_cmd_state); +static const uint32_t link_info_sz = sizeof (struct octep_ctrl_net_link_info); +static const uint32_t if_stats_sz = + sizeof (struct octep_ctrl_net_h2f_resp_cmd_get_stats); +static const uint32_t info_sz = + sizeof (struct octep_ctrl_net_h2f_resp_cmd_get_info); + +/* + * Initialize max receive burst size and each message size. + * + */ + +int +octep_cp_initialize_receive_vector () +{ + int i, j; + struct octep_cp_msg *msg; + + clib_warning ("Loop: Init\n"); + /* For now only support single buffer messages */ + for (i = 0; i < cp_lib_cfg.ndoms; i++) + { + for (j = 0; j < cp_lib_cfg.doms[i].npfs; j++) + { + if (cp_lib_cfg.doms[i].pfs[j].max_msg_sz < max_msg_sz) + return -EINVAL; + } + } + + for (i = 0; i < rx_num; i++) + { + msg = &rx_msg[i]; + msg->info.s.sz = max_msg_sz; + msg->sg_num = 1; + msg->sg_list[0].sz = max_msg_sz; + msg->sg_list[0].msg = calloc (1, max_msg_sz); + if (!msg->sg_list[0].msg) + goto mem_alloc_fail; + } + + clib_warning ("Loop: using single buffer with msg sz %u.\n", max_msg_sz); + + return 0; + +mem_alloc_fail: + for (i = 0; i < LOOP_RX_BUF_CNT; i++) + { + msg = &rx_msg[i]; + if (msg->sg_list[0].msg) + free (msg->sg_list[0].msg); + msg->sg_list[0].sz = 0; + msg->sg_num = 0; + } + + return -ENOMEM; +} + +static int +process_mtu (struct if_cfg *iface, struct octep_ctrl_net_h2f_req *req, + struct octep_ctrl_net_h2f_resp *resp) +{ + int ret = 0; + + if (req->mtu.cmd == OCTEP_CTRL_NET_CMD_GET) + { + resp->mtu.val = iface->max_rx_pktlen; + clib_warning ("Cmd: get mtu : %u\n", resp->mtu.val); + ret = mtu_sz; + } + else + { + iface->mtu = req->mtu.val; + clib_warning ("Cmd: set mtu : %u\n", req->mtu.val); + octep_update_pktio (req->mtu.cmd, req->mtu.val); + } + resp->hdr.s.reply = OCTEP_CTRL_NET_REPLY_OK; + + return ret; +} + +static int +process_mac (struct if_cfg *iface, struct octep_ctrl_net_h2f_req *req, + struct octep_ctrl_net_h2f_resp *resp) +{ + int ret = 0; + + if (req->mac.cmd == OCTEP_CTRL_NET_CMD_GET) + { + memcpy (&resp->mac.addr, &iface->mac_addr, ETH_ALEN); + ret = mac_sz; + clib_warning ("Cmd: get mac : %02x:%02x:%02x:%02x:%02x:%02x\n", + resp->mac.addr[0], resp->mac.addr[1], resp->mac.addr[2], + resp->mac.addr[3], resp->mac.addr[4], resp->mac.addr[5]); + } + else + { + memcpy (&iface->mac_addr, &req->mac.addr, ETH_ALEN); + clib_warning ("Cmd: set mac : %02x:%02x:%02x:%02x:%02x:%02x\n", + req->mac.addr[0], req->mac.addr[1], req->mac.addr[2], + req->mac.addr[3], req->mac.addr[4], req->mac.addr[5]); + } + resp->hdr.s.reply = OCTEP_CTRL_NET_REPLY_OK; + + return ret; +} + +static int +process_get_if_stats (struct if_stats *ifstats, + struct octep_ctrl_net_h2f_req *req, + struct octep_ctrl_net_h2f_resp *resp) +{ + /* Struct if_stats = struct octep_ctrl_net_h2f_resp_cmd_get_stats */ + memcpy (&resp->if_stats, ifstats, if_stats_sz); + resp->hdr.s.reply = OCTEP_CTRL_NET_REPLY_OK; + clib_warning ("Cmd: get if stats\n"); + + return if_stats_sz; +} + +static int +process_link_status (struct if_cfg *iface, struct octep_ctrl_net_h2f_req *req, + struct octep_ctrl_net_h2f_resp *resp) +{ + int ret = 0; + + if (req->link.cmd == OCTEP_CTRL_NET_CMD_GET) + { + resp->link.state = iface->link_state; + ret = state_sz; + } + else + iface->link_state = req->link.state; + + resp->hdr.s.reply = OCTEP_CTRL_NET_REPLY_OK; + + return ret; +} + +static int +process_rx_state (struct if_cfg *iface, struct octep_ctrl_net_h2f_req *req, + struct octep_ctrl_net_h2f_resp *resp) +{ + int ret = 0; + + if (req->rx.cmd == OCTEP_CTRL_NET_CMD_GET) + { + resp->rx.state = iface->rx_state; + ret = state_sz; + } + else + iface->rx_state = req->rx.state; + + resp->hdr.s.reply = OCTEP_CTRL_NET_REPLY_OK; + + return ret; +} + +static int +process_link_info (struct if_cfg *iface, struct octep_ctrl_net_h2f_req *req, + struct octep_ctrl_net_h2f_resp *resp) +{ + int ret = 0; + + if (req->link_info.cmd == OCTEP_CTRL_NET_CMD_GET) + { + resp->link_info.supported_modes = iface->supported_modes; + resp->link_info.advertised_modes = iface->advertised_modes; + resp->link_info.autoneg = iface->autoneg; + resp->link_info.pause = iface->pause_mode; + resp->link_info.speed = iface->speed; + ret = link_info_sz; + } + else + { + iface->advertised_modes = req->link_info.info.advertised_modes; + iface->autoneg = req->link_info.info.autoneg; + iface->pause_mode = req->link_info.info.pause; + iface->speed = req->link_info.info.speed; + // clib_warning ("Cmd: set link info: am:%lx a:%x p:%x s:%x\n", + // req->link_info.info.advertised_modes, + // req->link_info.info.autoneg, req->link_info.info.pause, + // req->link_info.info.speed); + } + resp->hdr.s.reply = OCTEP_CTRL_NET_REPLY_OK; + + return ret; +} + +static int +process_get_info (struct octep_fw_info *info, + struct octep_ctrl_net_h2f_req *req, + struct octep_ctrl_net_h2f_resp *resp) +{ + memcpy (&resp->info.fw_info, info, sizeof (struct octep_fw_info)); + resp->hdr.s.reply = OCTEP_CTRL_NET_REPLY_OK; + + return info_sz; +} + +static int +process_msg (union octep_cp_msg_info *ctx, struct octep_cp_msg *msg) +{ + struct octep_ctrl_net_h2f_req *req; + struct octep_ctrl_net_h2f_resp resp = { 0 }; + struct octep_cp_msg resp_msg; + struct if_cfg *iface; + struct if_stats *ifdata; + struct octep_fw_info *info; + int err = 0, resp_sz = 0; + + err = + app_config_get_if_from_msg_info (ctx, &msg->info, &iface, &ifdata, &info); + if (err) + { + clib_warning ("Invalid msg[%lx]\n", msg->info.words[0]); + return err; + } + + req = (struct octep_ctrl_net_h2f_req *) msg->sg_list[0].msg; + resp.hdr.words[0] = req->hdr.words[0]; + iface->host_if_id = req->hdr.s.sender; + resp_sz = resp_hdr_sz; + switch (req->hdr.s.cmd) + { + case OCTEP_CTRL_NET_H2F_CMD_MTU: + resp_sz += process_mtu (iface, req, &resp); + break; + case OCTEP_CTRL_NET_H2F_CMD_MAC: + resp_sz += process_mac (iface, req, &resp); + break; + case OCTEP_CTRL_NET_H2F_CMD_GET_IF_STATS: + resp_sz += process_get_if_stats (ifdata, req, &resp); + break; + case OCTEP_CTRL_NET_H2F_CMD_LINK_STATUS: + resp_sz += process_link_status (iface, req, &resp); + break; + case OCTEP_CTRL_NET_H2F_CMD_RX_STATE: + resp_sz += process_rx_state (iface, req, &resp); + break; + case OCTEP_CTRL_NET_H2F_CMD_LINK_INFO: + resp_sz += process_link_info (iface, req, &resp); + break; + case OCTEP_CTRL_NET_H2F_CMD_GET_INFO: + resp_sz += process_get_info (info, req, &resp); + break; + default: + clib_warning ("Unhandled Cmd : %u\n", req->hdr.s.cmd); + resp_sz = 0; + break; + } + + if (resp_sz >= resp_hdr_sz) + { + resp_msg.info = msg->info; + resp_msg.info.s.sz = resp_sz; + resp_msg.sg_num = 1; + resp_msg.sg_list[0].sz = resp_sz; + resp_msg.sg_list[0].msg = &resp; + octep_cp_lib_send_msg_resp (ctx, &resp_msg, 1); + ifdata->tx_stats.pkts++; + ifdata->tx_stats.octs += resp_sz; + } + + ifdata->rx_stats.pkts++; + ifdata->rx_stats.octets += msg->info.s.sz; + + return 0; +} + +int +loop_process_msgs () +{ + union octep_cp_msg_info ctx; + struct octep_cp_msg *msg; + int ret, i, j, m; + + for (i = 0; i < cp_lib_cfg.ndoms; i++) + { + ctx.s.pem_idx = cp_lib_cfg.doms[i].idx; + for (j = 0; j < cp_lib_cfg.doms[i].npfs; j++) + { + ctx.s.pf_idx = cp_lib_cfg.doms[i].pfs[j].idx; + ret = octep_cp_lib_recv_msg (&ctx, rx_msg, rx_num); + for (m = 0; m < ret; m++) + { + msg = &rx_msg[m]; + process_msg (&ctx, msg); + /* Library will overwrite msg size in header so reset it */ + msg->info.s.sz = max_msg_sz; + } + } + } + + return 0; +} + +int +octep_cp_uninitialize_receive_vector () +{ + int i; + + clib_warning ("%s\n", __func__); + + for (i = 0; i < rx_num; i++) + { + if (rx_msg[i].sg_list[0].msg) + free (rx_msg[i].sg_list[0].msg); + rx_msg[i].sg_list[0].sz = 0; + } + + return 0; +} + +/* + * fd.io coding-style-patch-verification: ON + * + * Local Variables: + * eval: (c-set-style "gnu") + * End: + */ diff --git a/src/plugins/octep_cp/octep_input.h b/src/plugins/octep_cp/octep_input.h new file mode 100644 index 0000000000..7980009083 --- /dev/null +++ b/src/plugins/octep_cp/octep_input.h @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2023 Marvell. + * SPDX-License-Identifier: Apache-2.0 + * https://spdx.org/licenses/Apache-2.0.html + */ + +#ifndef __OCTEP_INPUT_H__ +#define __OCTEP_INPUT_H__ + +/* + * Initialize loop mode implementation. + * return value: 0 on success, -errno on failure. + */ +int octep_cp_initialize_receive_vector (); + +/* + * Process interrupts and host messages. + * return value: size of response in words on success, -errno on failure. + */ +int loop_process_msgs (); + +/* + * Process user interrupt signal. + * return value: 0 on success, -errno on failure. + */ +int loop_process_sigusr1 (); + +/* + * Uninitialize loop mode implementation. + * return value: 0 on success, -errno on failure. + */ +int octep_cp_uninitialize_receive_vector (); + +#endif /* __OCTEP_INPUT_H__ */ + +/* + * fd.io coding-style-patch-verification: ON + * + * Local Variables: + * eval: (c-set-style "gnu") + * End: + */ From 45bf8aae04d8ef11886700140a6dfb56c6fc3a6b Mon Sep 17 00:00:00 2001 From: Bheemappa Agasimundin Date: Wed, 10 Jan 2024 10:39:31 +0000 Subject: [PATCH 074/271] octep-cp: add support for host checksum offload This patch contains changes to provide OCTEON offload capabilities to HOST. Type: feature JIRA: https://essjira.marvell.com/browse/IPBUSW-37647 Signed-off-by: Bheemappa Agasimundin Change-Id: I3461a9a50897b768750b519eb6c85b6b3a3af3ac Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/119477 Reviewed-by: Abed Mohammad Kamaluddin Tested-by: sa_ip-sw-jenkins (cherry picked from commit 27d2aab102f0c6fdd40f9c8c39d7d21c821214d3) Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/121129 Reviewed-by: Shiva Shankar Kommula Tested-by: Shiva Shankar Kommula --- src/plugins/octep_cp/octep_config.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/plugins/octep_cp/octep_config.c b/src/plugins/octep_cp/octep_config.c index f704902f3b..602dcda2a7 100644 --- a/src/plugins/octep_cp/octep_config.c +++ b/src/plugins/octep_cp/octep_config.c @@ -285,10 +285,16 @@ parse_if (config_setting_t *lcfg, struct if_cfg *iface) static int parse_info (config_setting_t *lcfg, struct octep_fw_info *info) { - int ival, ret; + int ival = 0, ret; ret = config_setting_lookup_int (lcfg, CFG_TOKEN_INFO_PKIND, &ival); - info->pkind = (ret == CONFIG_TRUE) ? ival : 0; + if (ival) + { + info->pkind = OCTEP_PKIND_OL_SUPPORTED; + info->fsz = OCTEP_FSZ_OL_SUPPORTED; + info->rx_offloads = OCTEP_RX_OFFLOAD_CKSUM; + info->tx_offloads = OCTEP_TX_OFFLOAD_CKSUM; + } ret = config_setting_lookup_int (lcfg, CFG_TOKEN_INFO_HB_INTERVAL, &ival); info->hb_interval = (ret == CONFIG_TRUE) ? ival : DEFAULT_HB_INTERVAL_MSECS; From 0feb1d2a3f6896e83d9976e599a984e95f66ec3f Mon Sep 17 00:00:00 2001 From: Bheemappa Agasimundin Date: Thu, 11 Jan 2024 09:53:47 +0000 Subject: [PATCH 075/271] octep-cp: add generic octep configuration file This patch adds generic configuration file for OCTEON end point instead of having specific configuration like cn106xx or cn103xx. Type: feature JIRA: https://essjira.marvell.com/browse/IPBUSW-37647 Signed-off-by: Bheemappa Agasimundin Change-Id: Ieec6bac5f5017523cef09df37ac12d389fb48e93 Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/119565 Reviewed-by: Abed Mohammad Kamaluddin Tested-by: sa_ip-sw-jenkins (cherry picked from commit 2060c1a1da3ec24037bda90a22d5c4271bfa75e4) Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/121130 Reviewed-by: Shiva Shankar Kommula Tested-by: Shiva Shankar Kommula --- src/plugins/octep_cp/CMakeLists.txt | 1 + .../octep_cp/docs/octep_cp_plugin_doc.md | 2 +- src/plugins/octep_cp/octep_cp_cn10kxx.cfg | 796 ++++++++++++++++++ src/plugins/octep_cp/octep_ctrl.c | 4 +- 4 files changed, 800 insertions(+), 3 deletions(-) create mode 100755 src/plugins/octep_cp/octep_cp_cn10kxx.cfg diff --git a/src/plugins/octep_cp/CMakeLists.txt b/src/plugins/octep_cp/CMakeLists.txt index 814ba4363c..6043618677 100644 --- a/src/plugins/octep_cp/CMakeLists.txt +++ b/src/plugins/octep_cp/CMakeLists.txt @@ -51,3 +51,4 @@ add_vpp_plugin(octep_cp LINK_FLAGS "${OCTEP_LINK_FLAGS}" ) +install(FILES octep_cp_cn10kxx.cfg DESTINATION etc/vpp/ COMPONENT vpp-plugin-octep_cp) diff --git a/src/plugins/octep_cp/docs/octep_cp_plugin_doc.md b/src/plugins/octep_cp/docs/octep_cp_plugin_doc.md index 43390b5b74..8ed49805e1 100644 --- a/src/plugins/octep_cp/docs/octep_cp_plugin_doc.md +++ b/src/plugins/octep_cp/docs/octep_cp_plugin_doc.md @@ -43,5 +43,5 @@ OCTEON connected to host. - create required VF's with 'echo 1 > /sys/bus/pci/devices/0000\:17\:00.0/sriov_numvfs' #### Configuration -This plugin uses /usr/bin/cn106xx.cfg configuration file to configure +This plugin uses /usr/bin/cn10kxx.cfg configuration file to configure PCIe end point. diff --git a/src/plugins/octep_cp/octep_cp_cn10kxx.cfg b/src/plugins/octep_cp/octep_cp_cn10kxx.cfg new file mode 100755 index 0000000000..db5d740904 --- /dev/null +++ b/src/plugins/octep_cp/octep_cp_cn10kxx.cfg @@ -0,0 +1,796 @@ +soc = { + /* 1 pem */ + pems = ( + { + idx = 0; + /* 1 pf per pem */ + pfs = ( + { + idx = 0; + mac_addr = [0x00, 0x00, 0x00, 0x01, 0x01]; + link_state = 0; + rx_state = 0; + autoneg = 0x3; + pause_mode = 0x3; + speed = 10000; + supported_modes = 0x1; + advertised_modes = 0x1; + hb_interval = 1000; + hb_miss_count = 20; + pkind = 0; + /* 64 vf's per pf */ + vfs = ( + { + idx = 0; + mac_addr = [0x00, 0x00, 0x00, 0x01, 0x01, 0x01]; + link_state = 0; + rx_state = 0; + autoneg = 0x3; + pause_mode = 0x3; + speed = 10000; + supported_modes = 0x1; + advertised_modes = 0x1; + pkind = 0; + }, + { + idx = 1; + mac_addr = [0x00, 0x00, 0x00, 0x02, 0x01, 0x01]; + link_state = 0; + rx_state = 0; + autoneg = 0x3; + pause_mode = 0x3; + speed = 10000; + supported_modes = 0x1; + advertised_modes = 0x1; + pkind = 0; + }, + { + idx = 2; + mac_addr = [0x00, 0x00, 0x00, 0x03, 0x01, 0x01]; + link_state = 0; + rx_state = 0; + autoneg = 0x3; + pause_mode = 0x3; + speed = 10000; + supported_modes = 0x1; + advertised_modes = 0x1; + pkind = 0; + }, + { + idx = 3; + mac_addr = [0x00, 0x00, 0x00, 0x04, 0x01, 0x01]; + link_state = 0; + rx_state = 0; + autoneg = 0x3; + pause_mode = 0x3; + speed = 10000; + supported_modes = 0x1; + advertised_modes = 0x1; + pkind = 0; + }, + { + idx = 4; + mac_addr = [0x00, 0x00, 0x00, 0x05, 0x01, 0x01]; + link_state = 0; + rx_state = 0; + autoneg = 0x3; + pause_mode = 0x3; + speed = 10000; + supported_modes = 0x1; + advertised_modes = 0x1; + pkind = 0; + }, + { + idx = 5; + mac_addr = [0x00, 0x00, 0x00, 0x06, 0x01, 0x01]; + link_state = 0; + rx_state = 0; + autoneg = 0x3; + pause_mode = 0x3; + speed = 10000; + supported_modes = 0x1; + advertised_modes = 0x1; + pkind = 0; + }, + { + idx = 6; + mac_addr = [0x00, 0x00, 0x00, 0x07, 0x01, 0x01]; + link_state = 0; + rx_state = 0; + autoneg = 0x3; + pause_mode = 0x3; + speed = 10000; + supported_modes = 0x1; + advertised_modes = 0x1; + pkind = 0; + }, + { + idx = 7; + mac_addr = [0x00, 0x00, 0x00, 0x08, 0x01, 0x01]; + link_state = 0; + rx_state = 0; + autoneg = 0x3; + pause_mode = 0x3; + speed = 10000; + supported_modes = 0x1; + advertised_modes = 0x1; + pkind = 0; + }, + { + idx = 8; + mac_addr = [0x00, 0x00, 0x00, 0x09, 0x01, 0x01]; + link_state = 0; + rx_state = 0; + autoneg = 0x3; + pause_mode = 0x3; + speed = 10000; + supported_modes = 0x1; + advertised_modes = 0x1; + pkind = 0; + }, + { + idx = 9; + mac_addr = [0x00, 0x00, 0x00, 0x0a, 0x01, 0x01]; + link_state = 0; + rx_state = 0; + autoneg = 0x3; + pause_mode = 0x3; + speed = 10000; + supported_modes = 0x1; + advertised_modes = 0x1; + pkind = 0; + }, + { + idx = 10; + mac_addr = [0x00, 0x00, 0x00, 0x0b, 0x01, 0x01]; + link_state = 0; + rx_state = 0; + autoneg = 0x3; + pause_mode = 0x3; + speed = 10000; + supported_modes = 0x1; + advertised_modes = 0x1; + pkind = 0; + }, + { + idx = 11; + mac_addr = [0x00, 0x00, 0x00, 0x0c, 0x01, 0x01]; + link_state = 0; + rx_state = 0; + autoneg = 0x3; + pause_mode = 0x3; + speed = 10000; + supported_modes = 0x1; + advertised_modes = 0x1; + pkind = 0; + }, + { + idx = 12; + mac_addr = [0x00, 0x00, 0x00, 0x0d, 0x01, 0x01]; + link_state = 0; + rx_state = 0; + autoneg = 0x3; + pause_mode = 0x3; + speed = 10000; + supported_modes = 0x1; + advertised_modes = 0x1; + pkind = 0; + }, + { + idx = 13; + mac_addr = [0x00, 0x00, 0x00, 0x0e, 0x01, 0x01]; + link_state = 0; + rx_state = 0; + autoneg = 0x3; + pause_mode = 0x3; + speed = 10000; + supported_modes = 0x1; + advertised_modes = 0x1; + pkind = 0; + }, + { + idx = 14; + mac_addr = [0x00, 0x00, 0x00, 0x0f, 0x01, 0x01]; + link_state = 0; + rx_state = 0; + autoneg = 0x3; + pause_mode = 0x3; + speed = 10000; + supported_modes = 0x1; + advertised_modes = 0x1; + pkind = 0; + }, + { + idx = 15; + mac_addr = [0x00, 0x00, 0x00, 0x10, 0x01, 0x01]; + link_state = 0; + rx_state = 0; + autoneg = 0x3; + pause_mode = 0x3; + speed = 10000; + supported_modes = 0x1; + advertised_modes = 0x1; + pkind = 0; + }, + { + idx = 16; + mac_addr = [0x00, 0x00, 0x00, 0x11, 0x01, 0x01]; + link_state = 0; + rx_state = 0; + autoneg = 0x3; + pause_mode = 0x3; + speed = 10000; + supported_modes = 0x1; + advertised_modes = 0x1; + pkind = 0; + }, + { + idx = 17; + mac_addr = [0x00, 0x00, 0x00, 0x12, 0x01, 0x01]; + link_state = 0; + rx_state = 0; + autoneg = 0x3; + pause_mode = 0x3; + speed = 10000; + supported_modes = 0x1; + advertised_modes = 0x1; + pkind = 0; + }, + { + idx = 18; + mac_addr = [0x00, 0x00, 0x00, 0x13, 0x01, 0x01]; + link_state = 0; + rx_state = 0; + autoneg = 0x3; + pause_mode = 0x3; + speed = 10000; + supported_modes = 0x1; + advertised_modes = 0x1; + pkind = 0; + }, + { + idx = 19; + mac_addr = [0x00, 0x00, 0x00, 0x14, 0x01, 0x01]; + link_state = 0; + rx_state = 0; + autoneg = 0x3; + pause_mode = 0x3; + speed = 10000; + supported_modes = 0x1; + advertised_modes = 0x1; + pkind = 0; + }, + { + idx = 20; + mac_addr = [0x00, 0x00, 0x00, 0x15, 0x01, 0x01]; + link_state = 0; + rx_state = 0; + autoneg = 0x3; + pause_mode = 0x3; + speed = 10000; + supported_modes = 0x1; + advertised_modes = 0x1; + pkind = 0; + }, + { + idx = 21; + mac_addr = [0x00, 0x00, 0x00, 0x16, 0x01, 0x01]; + link_state = 0; + rx_state = 0; + autoneg = 0x3; + pause_mode = 0x3; + speed = 10000; + supported_modes = 0x1; + advertised_modes = 0x1; + pkind = 0; + }, + { + idx = 22; + mac_addr = [0x00, 0x00, 0x00, 0x17, 0x01, 0x01]; + link_state = 0; + rx_state = 0; + autoneg = 0x3; + pause_mode = 0x3; + speed = 10000; + supported_modes = 0x1; + advertised_modes = 0x1; + pkind = 0; + }, + { + idx = 23; + mac_addr = [0x00, 0x00, 0x00, 0x18, 0x01, 0x01]; + link_state = 0; + rx_state = 0; + autoneg = 0x3; + pause_mode = 0x3; + speed = 10000; + supported_modes = 0x1; + advertised_modes = 0x1; + pkind = 0; + }, + { + idx = 24; + mac_addr = [0x00, 0x00, 0x00, 0x19, 0x01, 0x01]; + link_state = 0; + rx_state = 0; + autoneg = 0x3; + pause_mode = 0x3; + speed = 10000; + supported_modes = 0x1; + advertised_modes = 0x1; + pkind = 0; + }, + { + idx = 25; + mac_addr = [0x00, 0x00, 0x00, 0x1a, 0x01, 0x01]; + link_state = 0; + rx_state = 0; + autoneg = 0x3; + pause_mode = 0x3; + speed = 10000; + supported_modes = 0x1; + advertised_modes = 0x1; + pkind = 0; + }, + { + idx = 26; + mac_addr = [0x00, 0x00, 0x00, 0x1b, 0x01, 0x01]; + link_state = 0; + rx_state = 0; + autoneg = 0x3; + pause_mode = 0x3; + speed = 10000; + supported_modes = 0x1; + advertised_modes = 0x1; + pkind = 0; + }, + { + idx = 27; + mac_addr = [0x00, 0x00, 0x00, 0x1c, 0x01, 0x01]; + link_state = 0; + rx_state = 0; + autoneg = 0x3; + pause_mode = 0x3; + speed = 10000; + supported_modes = 0x1; + advertised_modes = 0x1; + pkind = 0; + }, + { + idx = 28; + mac_addr = [0x00, 0x00, 0x00, 0x1d, 0x01, 0x01]; + link_state = 0; + rx_state = 0; + autoneg = 0x3; + pause_mode = 0x3; + speed = 10000; + supported_modes = 0x1; + advertised_modes = 0x1; + pkind = 0; + }, + { + idx = 29; + mac_addr = [0x00, 0x00, 0x00, 0x1e, 0x01, 0x01]; + link_state = 0; + rx_state = 0; + autoneg = 0x3; + pause_mode = 0x3; + speed = 10000; + supported_modes = 0x1; + advertised_modes = 0x1; + pkind = 0; + }, + { + idx = 30; + mac_addr = [0x00, 0x00, 0x00, 0x1f, 0x01, 0x01]; + link_state = 0; + rx_state = 0; + autoneg = 0x3; + pause_mode = 0x3; + speed = 10000; + supported_modes = 0x1; + advertised_modes = 0x1; + pkind = 0; + }, + { + idx = 31; + mac_addr = [0x00, 0x00, 0x00, 0x20, 0x01, 0x01]; + link_state = 0; + rx_state = 0; + autoneg = 0x3; + pause_mode = 0x3; + speed = 10000; + supported_modes = 0x1; + advertised_modes = 0x1; + pkind = 0; + }, + { + idx = 32; + mac_addr = [0x00, 0x00, 0x00, 0x21, 0x01, 0x01]; + link_state = 0; + rx_state = 0; + autoneg = 0x3; + pause_mode = 0x3; + speed = 10000; + supported_modes = 0x1; + advertised_modes = 0x1; + pkind = 0; + }, + { + idx = 33; + mac_addr = [0x00, 0x00, 0x00, 0x22, 0x01, 0x01]; + link_state = 0; + rx_state = 0; + autoneg = 0x3; + pause_mode = 0x3; + speed = 10000; + supported_modes = 0x1; + advertised_modes = 0x1; + pkind = 0; + }, + { + idx = 34; + mac_addr = [0x00, 0x00, 0x00, 0x23, 0x01, 0x01]; + link_state = 0; + rx_state = 0; + autoneg = 0x3; + pause_mode = 0x3; + speed = 10000; + supported_modes = 0x1; + advertised_modes = 0x1; + pkind = 0; + }, + { + idx = 35; + mac_addr = [0x00, 0x00, 0x00, 0x24, 0x01, 0x01]; + link_state = 0; + rx_state = 0; + autoneg = 0x3; + pause_mode = 0x3; + speed = 10000; + supported_modes = 0x1; + advertised_modes = 0x1; + pkind = 0; + }, + { + idx = 36; + mac_addr = [0x00, 0x00, 0x00, 0x25, 0x01, 0x01]; + link_state = 0; + rx_state = 0; + autoneg = 0x3; + pause_mode = 0x3; + speed = 10000; + supported_modes = 0x1; + advertised_modes = 0x1; + pkind = 0; + }, + { + idx = 37; + mac_addr = [0x00, 0x00, 0x00, 0x26, 0x01, 0x01]; + link_state = 0; + rx_state = 0; + autoneg = 0x3; + pause_mode = 0x3; + speed = 10000; + supported_modes = 0x1; + advertised_modes = 0x1; + pkind = 0; + }, + { + idx = 38; + mac_addr = [0x00, 0x00, 0x00, 0x27, 0x01, 0x01]; + link_state = 0; + rx_state = 0; + autoneg = 0x3; + pause_mode = 0x3; + speed = 10000; + supported_modes = 0x1; + advertised_modes = 0x1; + pkind = 0; + }, + { + idx = 39; + mac_addr = [0x00, 0x00, 0x00, 0x28, 0x01, 0x01]; + link_state = 0; + rx_state = 0; + autoneg = 0x3; + pause_mode = 0x3; + speed = 10000; + supported_modes = 0x1; + advertised_modes = 0x1; + pkind = 0; + }, + { + idx = 40; + mac_addr = [0x00, 0x00, 0x00, 0x29, 0x01, 0x01]; + link_state = 0; + rx_state = 0; + autoneg = 0x3; + pause_mode = 0x3; + speed = 10000; + supported_modes = 0x1; + advertised_modes = 0x1; + pkind = 0; + }, + { + idx = 41; + mac_addr = [0x00, 0x00, 0x00, 0x2a, 0x01, 0x01]; + link_state = 0; + rx_state = 0; + autoneg = 0x3; + pause_mode = 0x3; + speed = 10000; + supported_modes = 0x1; + advertised_modes = 0x1; + pkind = 0; + }, + { + idx = 42; + mac_addr = [0x00, 0x00, 0x00, 0x2b, 0x01, 0x01]; + link_state = 0; + rx_state = 0; + autoneg = 0x3; + pause_mode = 0x3; + speed = 10000; + supported_modes = 0x1; + advertised_modes = 0x1; + pkind = 0; + }, + { + idx = 43; + mac_addr = [0x00, 0x00, 0x00, 0x2c, 0x01, 0x01]; + link_state = 0; + rx_state = 0; + autoneg = 0x3; + pause_mode = 0x3; + speed = 10000; + supported_modes = 0x1; + advertised_modes = 0x1; + pkind = 0; + }, + { + idx = 44; + mac_addr = [0x00, 0x00, 0x00, 0x2d, 0x01, 0x01]; + link_state = 0; + rx_state = 0; + autoneg = 0x3; + pause_mode = 0x3; + speed = 10000; + supported_modes = 0x1; + advertised_modes = 0x1; + pkind = 0; + }, + { + idx = 45; + mac_addr = [0x00, 0x00, 0x00, 0x2e, 0x01, 0x01]; + link_state = 0; + rx_state = 0; + autoneg = 0x3; + pause_mode = 0x3; + speed = 10000; + supported_modes = 0x1; + advertised_modes = 0x1; + pkind = 0; + }, + { + idx = 46; + mac_addr = [0x00, 0x00, 0x00, 0x2f, 0x01, 0x01]; + link_state = 0; + rx_state = 0; + autoneg = 0x3; + pause_mode = 0x3; + speed = 10000; + supported_modes = 0x1; + advertised_modes = 0x1; + pkind = 0; + }, + { + idx = 47; + mac_addr = [0x00, 0x00, 0x00, 0x30, 0x01, 0x01]; + link_state = 0; + rx_state = 0; + autoneg = 0x3; + pause_mode = 0x3; + speed = 10000; + supported_modes = 0x1; + advertised_modes = 0x1; + pkind = 0; + }, + { + idx = 48; + mac_addr = [0x00, 0x00, 0x00, 0x31, 0x01, 0x01]; + link_state = 0; + rx_state = 0; + autoneg = 0x3; + pause_mode = 0x3; + speed = 10000; + supported_modes = 0x1; + advertised_modes = 0x1; + pkind = 0; + }, + { + idx = 49; + mac_addr = [0x00, 0x00, 0x00, 0x32, 0x01, 0x01]; + link_state = 0; + rx_state = 0; + autoneg = 0x3; + pause_mode = 0x3; + speed = 10000; + supported_modes = 0x1; + advertised_modes = 0x1; + pkind = 0; + }, + { + idx = 50; + mac_addr = [0x00, 0x00, 0x00, 0x33, 0x01, 0x01]; + link_state = 0; + rx_state = 0; + autoneg = 0x3; + pause_mode = 0x3; + speed = 10000; + supported_modes = 0x1; + advertised_modes = 0x1; + pkind = 0; + }, + { + idx = 51; + mac_addr = [0x00, 0x00, 0x00, 0x34, 0x01, 0x01]; + link_state = 0; + rx_state = 0; + autoneg = 0x3; + pause_mode = 0x3; + speed = 10000; + supported_modes = 0x1; + advertised_modes = 0x1; + pkind = 0; + }, + { + idx = 52; + mac_addr = [0x00, 0x00, 0x00, 0x35, 0x01, 0x01]; + link_state = 0; + rx_state = 0; + autoneg = 0x3; + pause_mode = 0x3; + speed = 10000; + supported_modes = 0x1; + advertised_modes = 0x1; + pkind = 0; + }, + { + idx = 53; + mac_addr = [0x00, 0x00, 0x00, 0x36, 0x01, 0x01]; + link_state = 0; + rx_state = 0; + autoneg = 0x3; + pause_mode = 0x3; + speed = 10000; + supported_modes = 0x1; + advertised_modes = 0x1; + pkind = 0; + }, + { + idx = 54; + mac_addr = [0x00, 0x00, 0x00, 0x37, 0x01, 0x01]; + link_state = 0; + rx_state = 0; + autoneg = 0x3; + pause_mode = 0x3; + speed = 10000; + supported_modes = 0x1; + advertised_modes = 0x1; + pkind = 0; + }, + { + idx = 55; + mac_addr = [0x00, 0x00, 0x00, 0x38, 0x01, 0x01]; + link_state = 0; + rx_state = 0; + autoneg = 0x3; + pause_mode = 0x3; + speed = 10000; + supported_modes = 0x1; + advertised_modes = 0x1; + pkind = 0; + }, + { + idx = 56; + mac_addr = [0x00, 0x00, 0x00, 0x39, 0x01, 0x01]; + link_state = 0; + rx_state = 0; + autoneg = 0x3; + pause_mode = 0x3; + speed = 10000; + supported_modes = 0x1; + advertised_modes = 0x1; + pkind = 0; + }, + { + idx = 57; + mac_addr = [0x00, 0x00, 0x00, 0x3a, 0x01, 0x01]; + link_state = 0; + rx_state = 0; + autoneg = 0x3; + pause_mode = 0x3; + speed = 10000; + supported_modes = 0x1; + advertised_modes = 0x1; + pkind = 0; + }, + { + idx = 58; + mac_addr = [0x00, 0x00, 0x00, 0x3b, 0x01, 0x01]; + link_state = 0; + rx_state = 0; + autoneg = 0x3; + pause_mode = 0x3; + speed = 10000; + supported_modes = 0x1; + advertised_modes = 0x1; + pkind = 0; + }, + { + idx = 59; + mac_addr = [0x00, 0x00, 0x00, 0x3c, 0x01, 0x01]; + link_state = 0; + rx_state = 0; + autoneg = 0x3; + pause_mode = 0x3; + speed = 10000; + supported_modes = 0x1; + advertised_modes = 0x1; + pkind = 0; + }, + { + idx = 60; + mac_addr = [0x00, 0x00, 0x00, 0x3d, 0x01, 0x01]; + link_state = 0; + rx_state = 0; + autoneg = 0x3; + pause_mode = 0x3; + speed = 10000; + supported_modes = 0x1; + advertised_modes = 0x1; + pkind = 0; + }, + { + idx = 61; + mac_addr = [0x00, 0x00, 0x00, 0x3e, 0x01, 0x01]; + link_state = 0; + rx_state = 0; + autoneg = 0x3; + pause_mode = 0x3; + speed = 10000; + supported_modes = 0x1; + advertised_modes = 0x1; + pkind = 0; + }, + { + idx = 62; + mac_addr = [0x00, 0x00, 0x00, 0x3f, 0x01, 0x01]; + link_state = 0; + rx_state = 0; + autoneg = 0x3; + pause_mode = 0x3; + speed = 10000; + supported_modes = 0x1; + advertised_modes = 0x1; + pkind = 0; + }, + { + idx = 63; + mac_addr = [0x00, 0x00, 0x00, 0x40, 0x01, 0x01]; + link_state = 0; + rx_state = 0; + autoneg = 0x3; + pause_mode = 0x3; + speed = 10000; + supported_modes = 0x1; + advertised_modes = 0x1; + pkind = 0; + } + ); + } + ); + } + ); +}; diff --git a/src/plugins/octep_cp/octep_ctrl.c b/src/plugins/octep_cp/octep_ctrl.c index 776853f176..ca8a7f31f6 100644 --- a/src/plugins/octep_cp/octep_ctrl.c +++ b/src/plugins/octep_cp/octep_ctrl.c @@ -29,7 +29,7 @@ #define CP_VERSION_CURRENT \ (OCTEP_CP_VERSION (CP_VERSION_MAJOR, CP_VERSION_MINOR, CP_VERSION_VARIANT)) -#define SOC_CFG_PATH "/usr/bin/cn106xx.cfg" +#define SOC_CFG_PATH "/etc/vpp/octep_cp_cn10kxx.cfg" struct octep_pf_vf_cfg cfg_idx; /* @@ -221,7 +221,7 @@ VLIB_INIT_FUNCTION (octep_cp_init) = { VLIB_PLUGIN_REGISTER () = { .version = VPP_BUILD_VER, - .description = "Octep Control Agent", + .description = "OCTEON PCI End-point Control Agent", .default_disabled = 1, }; From d165e172d1f6bc6d98b96498297e31312b79e50f Mon Sep 17 00:00:00 2001 From: Bheemappa Agasimundin Date: Thu, 8 Feb 2024 15:26:44 +0000 Subject: [PATCH 076/271] octep-cp: support runtime checksum offload on/off This patch adds support for runtime checksum offload ON/OFF from HOST. Type: feature JIRA:https://essjira.marvell.com/browse/IPBUSW-41157 Signed-off-by: Bheemappa Agasimundin Change-Id: Icc1fc7023f363fe61e5e96cab3c6e92b0d6ef143 Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/122465 Tested-by: sa_ip-sw-jenkins Reviewed-by: Shiva Shankar Kommula --- src/plugins/octep_cp/octep_config.c | 6 ++ src/plugins/octep_cp/octep_config.h | 4 +- src/plugins/octep_cp/octep_cp_cn10kxx.cfg | 8 ++ src/plugins/octep_cp/octep_ctrl.c | 18 ++-- src/plugins/octep_cp/octep_input.c | 109 +++++++++++++++++++++- src/plugins/octep_cp/octep_input.h | 4 + 6 files changed, 138 insertions(+), 11 deletions(-) diff --git a/src/plugins/octep_cp/octep_config.c b/src/plugins/octep_cp/octep_config.c index 602dcda2a7..349989d136 100644 --- a/src/plugins/octep_cp/octep_config.c +++ b/src/plugins/octep_cp/octep_config.c @@ -42,6 +42,7 @@ struct app_cfg cfg; #define CFG_TOKEN_IF_SPEED "speed" #define CFG_TOKEN_IF_SMODES "supported_modes" #define CFG_TOKEN_IF_AMODES "advertised_modes" +#define CFG_TOKEN_IF_NAME "if_name" #define CFG_TOKEN_INFO_PKIND "pkind" #define CFG_TOKEN_INFO_HB_INTERVAL "hb_interval" #define CFG_TOKEN_INFO_HB_MISS_COUNT "hb_miss_count" @@ -250,6 +251,7 @@ parse_if (config_setting_t *lcfg, struct if_cfg *iface) { config_setting_t *mac; int ival, i, n; + char *if_name = NULL; if (config_setting_lookup_int (lcfg, CFG_TOKEN_IF_MTU, &ival)) iface->mtu = ival; @@ -277,6 +279,10 @@ parse_if (config_setting_t *lcfg, struct if_cfg *iface) iface->supported_modes = ival; if (config_setting_lookup_int (lcfg, CFG_TOKEN_IF_AMODES, &ival)) iface->advertised_modes = ival; + if (config_setting_lookup_string (lcfg, CFG_TOKEN_IF_NAME, + (const char **) &if_name)) + clib_memcpy (iface->if_name, if_name, strlen (if_name)); + iface->max_rx_pktlen = get_max_rx_pktlen (); return 0; diff --git a/src/plugins/octep_cp/octep_config.h b/src/plugins/octep_cp/octep_config.h index 1e28395f2e..ee84968f1b 100644 --- a/src/plugins/octep_cp/octep_config.h +++ b/src/plugins/octep_cp/octep_config.h @@ -19,7 +19,7 @@ #define MIN_HB_INTERVAL_MSECS 1000 #define MAX_HB_INTERVAL_MSECS 15000 #define DEFAULT_HB_INTERVAL_MSECS MIN_HB_INTERVAL_MSECS - +#define IF_NAME_MAX_LEN 256 #define DEFAULT_HB_MISS_COUNT 20 /* Network interface stats */ @@ -53,6 +53,8 @@ struct if_cfg u64 supported_modes; /* OCTEP_LINK_MODE_XXX */ u64 advertised_modes; + /* Interface name */ + u8 if_name[IF_NAME_MAX_LEN]; }; /* Virtual function configuration */ diff --git a/src/plugins/octep_cp/octep_cp_cn10kxx.cfg b/src/plugins/octep_cp/octep_cp_cn10kxx.cfg index db5d740904..bd03983039 100755 --- a/src/plugins/octep_cp/octep_cp_cn10kxx.cfg +++ b/src/plugins/octep_cp/octep_cp_cn10kxx.cfg @@ -31,6 +31,7 @@ soc = { supported_modes = 0x1; advertised_modes = 0x1; pkind = 0; + if_name = "eth1"; }, { idx = 1; @@ -43,6 +44,7 @@ soc = { supported_modes = 0x1; advertised_modes = 0x1; pkind = 0; + if_name = "eth2"; }, { idx = 2; @@ -55,6 +57,7 @@ soc = { supported_modes = 0x1; advertised_modes = 0x1; pkind = 0; + if_name = "eth3"; }, { idx = 3; @@ -67,6 +70,7 @@ soc = { supported_modes = 0x1; advertised_modes = 0x1; pkind = 0; + if_name = "eth4"; }, { idx = 4; @@ -79,6 +83,7 @@ soc = { supported_modes = 0x1; advertised_modes = 0x1; pkind = 0; + if_name = "eth5"; }, { idx = 5; @@ -91,6 +96,7 @@ soc = { supported_modes = 0x1; advertised_modes = 0x1; pkind = 0; + if_name = "eth6"; }, { idx = 6; @@ -103,6 +109,7 @@ soc = { supported_modes = 0x1; advertised_modes = 0x1; pkind = 0; + if_name = "eth7"; }, { idx = 7; @@ -115,6 +122,7 @@ soc = { supported_modes = 0x1; advertised_modes = 0x1; pkind = 0; + if_name = "eth8"; }, { idx = 8; diff --git a/src/plugins/octep_cp/octep_ctrl.c b/src/plugins/octep_cp/octep_ctrl.c index ca8a7f31f6..68a71b37db 100644 --- a/src/plugins/octep_cp/octep_ctrl.c +++ b/src/plugins/octep_cp/octep_ctrl.c @@ -23,11 +23,17 @@ /* Control plane version */ #define CP_VERSION_MAJOR 1 #define CP_VERSION_MINOR 0 -#define CP_VERSION_VARIANT 0 -#define MAX_EVENTS 6 +#define CP_VERSION_VARIANT_MIN 0 +#define CP_VERSION_VARIANT_CUR 1 +#define MAX_EVENTS 16 -#define CP_VERSION_CURRENT \ - (OCTEP_CP_VERSION (CP_VERSION_MAJOR, CP_VERSION_MINOR, CP_VERSION_VARIANT)) +#define CP_VERSION_MIN \ + (OCTEP_CP_VERSION (CP_VERSION_MAJOR, CP_VERSION_MINOR, \ + CP_VERSION_VARIANT_MIN)) + +#define CP_VERSION_MAX \ + (OCTEP_CP_VERSION (CP_VERSION_MAJOR, CP_VERSION_MINOR, \ + CP_VERSION_VARIANT_CUR)) #define SOC_CFG_PATH "/etc/vpp/octep_cp_cn10kxx.cfg" struct octep_pf_vf_cfg cfg_idx; @@ -137,8 +143,8 @@ octep_cp_process (vlib_main_t *vm, vlib_node_runtime_t *node, signal (SIGALRM, sigint_handler); cp_lib_cfg.ndoms = cfg.npem; - cp_lib_cfg.min_version = CP_VERSION_CURRENT; - cp_lib_cfg.max_version = CP_VERSION_CURRENT; + cp_lib_cfg.min_version = CP_VERSION_MIN; + cp_lib_cfg.max_version = CP_VERSION_MAX; cfg_idx.n_pems = cfg.npem; pem = cfg.pems; i = 0; diff --git a/src/plugins/octep_cp/octep_input.c b/src/plugins/octep_cp/octep_input.c index 9ce9d2428f..fd1be1e198 100644 --- a/src/plugins/octep_cp/octep_input.c +++ b/src/plugins/octep_cp/octep_input.c @@ -37,6 +37,8 @@ static const uint32_t if_stats_sz = static const uint32_t info_sz = sizeof (struct octep_ctrl_net_h2f_resp_cmd_get_info); +#define CTRL_NET_RESP_OFFLOADS_SZ sizeof (struct octep_ctrl_net_offloads) + /* * Initialize max receive burst size and each message size. * @@ -208,10 +210,6 @@ process_link_info (struct if_cfg *iface, struct octep_ctrl_net_h2f_req *req, iface->autoneg = req->link_info.info.autoneg; iface->pause_mode = req->link_info.info.pause; iface->speed = req->link_info.info.speed; - // clib_warning ("Cmd: set link info: am:%lx a:%x p:%x s:%x\n", - // req->link_info.info.advertised_modes, - // req->link_info.info.autoneg, req->link_info.info.pause, - // req->link_info.info.speed); } resp->hdr.s.reply = OCTEP_CTRL_NET_REPLY_OK; @@ -229,6 +227,106 @@ process_get_info (struct octep_fw_info *info, return info_sz; } +clib_error_t * +octep_enable_disable_offload_feature_arc (u8 *if_name, bool enable) +{ + uword *p; + u32 hw_if_index; + clib_error_t *error = NULL; + vnet_hw_interface_t *hi = NULL; + vnet_feature_registration_t *reg; + vnet_main_t *vnm = vnet_get_main (); + + if (!(p = hash_get (vnm->interface_main.hw_interface_by_name, if_name))) + return clib_error_return (0, "Unknown interfacse name (%s)... ", + (const char *) if_name); + + hw_if_index = p[0]; + hi = vnet_get_hw_interface (vnm, hw_if_index); + + reg = vnet_get_feature_reg ((const char *) DEVICE_INPUT, + (const char *) DPU_INPUT_NODE); + if (reg == 0) + { + error = clib_error_return ( + 0, + "Feature (%s) not registered to arc (%s)... See 'show " + "features verbose' for valid feature/arc combinations. ", + DPU_INPUT_NODE, DEVICE_INPUT); + return error; + } + + if (reg->enable_disable_cb) + error = reg->enable_disable_cb (hi->sw_if_index, enable); + + if (error) + return error; + + vnet_feature_enable_disable ((const char *) DEVICE_INPUT, + (const char *) DPU_INPUT_NODE, hi->sw_if_index, + enable, 0, 0); + + reg = vnet_get_feature_reg ((const char *) DEVICE_OUTPUT, + (const char *) DPU_OUTPUT_NODE); + if (reg == 0) + return clib_error_return ( + 0, + "Feature (%s) not registered to arc (%s)... See 'show " + "features verbose' for valid feature/arc combinations. ", + DPU_OUTPUT_NODE, DEVICE_OUTPUT); + + if (reg->enable_disable_cb) + error = reg->enable_disable_cb (hi->sw_if_index, enable); + + if (error) + return error; + + vnet_feature_enable_disable ((const char *) DEVICE_OUTPUT, + (const char *) DPU_OUTPUT_NODE, hi->sw_if_index, + enable, 0, 0); + + return error; +} + +static int +process_offloads (struct octep_fw_info *info, + struct octep_ctrl_net_h2f_req *req, + struct octep_ctrl_net_h2f_resp *resp, struct if_cfg *iface) +{ + + if (req->offloads.cmd == OCTEP_CTRL_NET_CMD_GET) + { + resp->offloads.rx_offloads = info->rx_offloads; + resp->offloads.tx_offloads = info->tx_offloads; + resp->offloads.ext_offloads = info->ext_offloads; + resp->hdr.s.reply = OCTEP_CTRL_NET_REPLY_OK; + return CTRL_NET_RESP_OFFLOADS_SZ; + } + + /** + * Disable/enable feature arc based on Host request or existing config. + */ + if (!req->offloads.offloads.rx_offloads && + !req->offloads.offloads.tx_offloads) + { + if (octep_enable_disable_offload_feature_arc (iface->if_name, 0)) + return 0; + } + else if (!info->rx_offloads && !info->tx_offloads) + { + if (octep_enable_disable_offload_feature_arc (iface->if_name, 1)) + return 0; + } + + info->rx_offloads = req->offloads.offloads.rx_offloads; + info->tx_offloads = req->offloads.offloads.tx_offloads; + info->ext_offloads = req->offloads.offloads.ext_offloads; + + resp->hdr.s.reply = OCTEP_CTRL_NET_REPLY_OK; + + return CTRL_NET_RESP_OFFLOADS_SZ; +} + static int process_msg (union octep_cp_msg_info *ctx, struct octep_cp_msg *msg) { @@ -275,6 +373,9 @@ process_msg (union octep_cp_msg_info *ctx, struct octep_cp_msg *msg) case OCTEP_CTRL_NET_H2F_CMD_GET_INFO: resp_sz += process_get_info (info, req, &resp); break; + case OCTEP_CTRL_NET_H2F_CMD_OFFLOADS: + resp_sz += process_offloads (info, req, &resp, iface); + break; default: clib_warning ("Unhandled Cmd : %u\n", req->hdr.s.cmd); resp_sz = 0; diff --git a/src/plugins/octep_cp/octep_input.h b/src/plugins/octep_cp/octep_input.h index 7980009083..810316a3cd 100644 --- a/src/plugins/octep_cp/octep_input.h +++ b/src/plugins/octep_cp/octep_input.h @@ -7,6 +7,10 @@ #ifndef __OCTEP_INPUT_H__ #define __OCTEP_INPUT_H__ +#define DEVICE_INPUT "device-input" +#define DPU_INPUT_NODE "h2d-input" +#define DEVICE_OUTPUT "interface-output" +#define DPU_OUTPUT_NODE "d2h-output" /* * Initialize loop mode implementation. * return value: 0 on success, -errno on failure. From 38f952ca23ed4abe982fcd0b7764d715ddbc530d Mon Sep 17 00:00:00 2001 From: Kommula Shiva Shankar Date: Wed, 7 Feb 2024 15:53:27 +0530 Subject: [PATCH 077/271] vpp: add octeon9 platform specific cmake file Type: feature Signed-off-by: Kommula Shiva Shankar Change-Id: I012c9755bb614fc9b15431d6c9fda7c51fa939ea Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/121457 Tested-by: sa_ip-toolkits-Jenkins Reviewed-by: Monendra Singh Kushwaha --- src/cmake/platform/octeon9.cmake | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 src/cmake/platform/octeon9.cmake diff --git a/src/cmake/platform/octeon9.cmake b/src/cmake/platform/octeon9.cmake new file mode 100644 index 0000000000..b81d1705d8 --- /dev/null +++ b/src/cmake/platform/octeon9.cmake @@ -0,0 +1,4 @@ +set(VPP_PLATFORM_CACHE_LINE_SIZE 128) +set(VPP_PLATFORM_MARCH_FLAGS -march=armv8.2-a+crc+crypto+lse) +set(VPP_PLATFORM_BUFFER_ALIGN 128) +set(VPP_PLATFORM_N_PREFETCHES 6) From 42f0ea8861c9d986b96d9511e6be9318598db51c Mon Sep 17 00:00:00 2001 From: Sriram Vatala Date: Thu, 8 Aug 2024 09:57:36 +0000 Subject: [PATCH 078/271] octeon: use proper refs for roc item spec and mask vnet flow enable is failing due to bogus bytes pointed by spec, mask variables of roc_npc_flow_item structure. Using reference to local variables defined in block scope is causing this. Moving the variable declarations to function block scope fixes this issue. Fixes: 064762e20 Type: fix Signed-off-by: Sriram Vatala Change-Id: I3904199b5b2bd88cd02ada5604059ab6fd12eef7 Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/133108 Tested-by: sa_ip-sw-jenkins Reviewed-by: Monendra Singh Kushwaha (cherry picked from commit 0670542d8a6eda382d7b82de3df2d2cea1046a4f) Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/134661 Reviewed-by: Nithinsen Kaithakadan --- src/plugins/dev_octeon/flow.c | 102 ++++++++++++++++++++-------------- 1 file changed, 59 insertions(+), 43 deletions(-) diff --git a/src/plugins/dev_octeon/flow.c b/src/plugins/dev_octeon/flow.c index 35aabde76a..5bef25f536 100644 --- a/src/plugins/dev_octeon/flow.c +++ b/src/plugins/dev_octeon/flow.c @@ -189,6 +189,14 @@ oct_flow_rule_create (vnet_dev_port_t *port, struct roc_npc_action *actions, npc = &oct_port->npc; + for (int i = 0; item_info[i].type != ROC_NPC_ITEM_TYPE_END; i++) + { + log_debug (port->dev, "Flow[%d] Item[%d] type %d spec 0x%U mask 0x%U", + flow->index, i, item_info[i].type, format_hex_bytes, + item_info[i].spec, item_info[i].size, format_hex_bytes, + item_info[i].mask, item_info[i].size); + } + npc_flow = roc_npc_flow_create (npc, &attr, item_info, actions, npc->pf_func, &rv); if (rv) @@ -530,6 +538,14 @@ oct_flow_add (vlib_main_t *vm, vnet_dev_port_t *port, vnet_flow_t *flow, struct roc_npc_item_info item_info[ROC_NPC_ITEM_TYPE_END] = {}; struct roc_npc_action actions[ROC_NPC_ITEM_TYPE_END] = {}; oct_port_t *oct_port = vnet_dev_get_port_data (port); + ethernet_header_t eth_spec = {}, eth_mask = {}; + sctp_header_t sctp_spec = {}, sctp_mask = {}; + gtpu_header_t gtpu_spec = {}, gtpu_mask = {}; + ip4_header_t ip4_spec = {}, ip4_mask = {}; + ip6_header_t ip6_spec = {}, ip6_mask = {}; + udp_header_t udp_spec = {}, udp_mask = {}; + tcp_header_t tcp_spec = {}, tcp_mask = {}; + esp_header_t esp_spec = {}, esp_mask = {}; u16 l4_src_port = 0, l4_dst_port = 0; u16 l4_src_mask = 0, l4_dst_mask = 0; struct roc_npc_action_rss rss_conf = {}; @@ -537,6 +553,7 @@ oct_flow_add (vlib_main_t *vm, vnet_dev_port_t *port, vnet_flow_t *flow, struct roc_npc_action_mark mark = {}; struct roc_npc *npc = &oct_port->npc; u8 *flow_spec = 0, *flow_mask = 0; + u8 *drv_spec = 0, *drv_mask = 0; vnet_dev_rv_t rv = VNET_DEV_OK; int layer = 0, index = 0; u16 *queues = NULL; @@ -546,7 +563,6 @@ oct_flow_add (vlib_main_t *vm, vnet_dev_port_t *port, vnet_flow_t *flow, if (FLOW_IS_GENERIC_TYPE (flow)) { - u8 drv_item_spec[1024] = { 0 }, drv_item_mask[1024] = { 0 }; unformat_input_t input; int rc; @@ -562,11 +578,13 @@ oct_flow_add (vlib_main_t *vm, vnet_dev_port_t *port, vnet_flow_t *flow, unformat_user (&input, unformat_hex_string, &flow_mask); unformat_free (&input); + vec_validate (drv_spec, 1024); + vec_validate (drv_mask, 1024); oct_flow_parse_state pst = { .nxt_proto = 0, .port = port, .items = item_info, - .oct_drv = { .spec = drv_item_spec, .mask = drv_item_mask }, + .oct_drv = { .spec = drv_spec, .mask = drv_mask }, .generic = { .spec = flow_spec, .mask = flow_mask, .len = vec_len (flow_spec) }, @@ -577,6 +595,8 @@ oct_flow_add (vlib_main_t *vm, vnet_dev_port_t *port, vnet_flow_t *flow, { vec_free (flow_spec); vec_free (flow_mask); + vec_free (drv_spec); + vec_free (drv_mask); return VNET_DEV_ERR_NOT_SUPPORTED; } @@ -585,9 +605,8 @@ oct_flow_add (vlib_main_t *vm, vnet_dev_port_t *port, vnet_flow_t *flow, if (FLOW_IS_ETHERNET_CLASS (flow)) { - ethernet_header_t eth_spec = { .type = clib_host_to_net_u16 ( - flow->ethernet.eth_hdr.type) }, - eth_mask = { .type = 0xFFFF }; + eth_spec.type = clib_host_to_net_u16 (flow->ethernet.eth_hdr.type); + eth_mask.type = 0xFFFF; item_info[layer].spec = (void *) ð_spec; item_info[layer].mask = (void *) ð_mask; @@ -600,10 +619,11 @@ oct_flow_add (vlib_main_t *vm, vnet_dev_port_t *port, vnet_flow_t *flow, { vnet_flow_ip4_t *ip4_hdr = &flow->ip4; proto = ip4_hdr->protocol.prot; - ip4_header_t ip4_spec = { .src_address = ip4_hdr->src_addr.addr, - .dst_address = ip4_hdr->dst_addr.addr }, - ip4_mask = { .src_address = ip4_hdr->src_addr.mask, - .dst_address = ip4_hdr->dst_addr.mask }; + + ip4_spec.src_address = ip4_hdr->src_addr.addr; + ip4_spec.dst_address = ip4_hdr->dst_addr.addr; + ip4_mask.src_address = ip4_hdr->src_addr.mask; + ip4_mask.dst_address = ip4_hdr->dst_addr.mask; item_info[layer].spec = (void *) &ip4_spec; item_info[layer].mask = (void *) &ip4_mask; @@ -625,10 +645,11 @@ oct_flow_add (vlib_main_t *vm, vnet_dev_port_t *port, vnet_flow_t *flow, { vnet_flow_ip6_t *ip6_hdr = &flow->ip6; proto = ip6_hdr->protocol.prot; - ip6_header_t ip6_spec = { .src_address = ip6_hdr->src_addr.addr, - .dst_address = ip6_hdr->dst_addr.addr }, - ip6_mask = { .src_address = ip6_hdr->src_addr.mask, - .dst_address = ip6_hdr->dst_addr.mask }; + + ip6_spec.src_address = ip6_hdr->src_addr.addr; + ip6_spec.dst_address = ip6_hdr->dst_addr.addr; + ip6_mask.src_address = ip6_hdr->src_addr.mask; + ip6_mask.dst_address = ip6_hdr->dst_addr.mask; item_info[layer].spec = (void *) &ip6_spec; item_info[layer].mask = (void *) &ip6_mask; @@ -653,16 +674,15 @@ oct_flow_add (vlib_main_t *vm, vnet_dev_port_t *port, vnet_flow_t *flow, switch (proto) { case IP_PROTOCOL_UDP: - item_info[layer].type = ROC_NPC_ITEM_TYPE_UDP; - - udp_header_t udp_spec = { .src_port = l4_src_port, - .dst_port = l4_dst_port }, - udp_mask = { .src_port = l4_src_mask, - .dst_port = l4_dst_mask }; + udp_spec.src_port = l4_src_port; + udp_spec.dst_port = l4_dst_port; + udp_mask.src_port = l4_src_mask; + udp_mask.dst_port = l4_dst_mask; item_info[layer].spec = (void *) &udp_spec; item_info[layer].mask = (void *) &udp_mask; item_info[layer].size = sizeof (udp_header_t); + item_info[layer].type = ROC_NPC_ITEM_TYPE_UDP; layer++; if (FLOW_IS_L4_TUNNEL_TYPE (flow)) @@ -670,14 +690,13 @@ oct_flow_add (vlib_main_t *vm, vnet_dev_port_t *port, vnet_flow_t *flow, switch (flow->type) { case VNET_FLOW_TYPE_IP4_GTPU: - item_info[layer].type = ROC_NPC_ITEM_TYPE_GTPU; - gtpu_header_t gtpu_spec = { .teid = clib_host_to_net_u32 ( - flow->ip4_gtpu.teid) }, - gtpu_mask = { .teid = 0XFFFFFFFF }; + gtpu_spec.teid = clib_host_to_net_u32 (flow->ip4_gtpu.teid); + gtpu_mask.teid = 0XFFFFFFFF; item_info[layer].spec = (void *) >pu_spec; item_info[layer].mask = (void *) >pu_mask; item_info[layer].size = sizeof (gtpu_header_t); + item_info[layer].type = ROC_NPC_ITEM_TYPE_GTPU; layer++; break; @@ -689,42 +708,39 @@ oct_flow_add (vlib_main_t *vm, vnet_dev_port_t *port, vnet_flow_t *flow, break; case IP_PROTOCOL_TCP: - item_info[layer].type = ROC_NPC_ITEM_TYPE_TCP; - - tcp_header_t tcp_spec = { .src_port = l4_src_port, - .dst_port = l4_dst_port }, - tcp_mask = { .src_port = l4_src_mask, - .dst_port = l4_dst_mask }; + tcp_spec.src_port = l4_src_port; + tcp_spec.dst_port = l4_dst_port; + tcp_mask.src_port = l4_src_mask; + tcp_mask.dst_port = l4_dst_mask; item_info[layer].spec = (void *) &tcp_spec; item_info[layer].mask = (void *) &tcp_mask; item_info[layer].size = sizeof (tcp_header_t); + item_info[layer].type = ROC_NPC_ITEM_TYPE_TCP; layer++; break; case IP_PROTOCOL_SCTP: - item_info[layer].type = ROC_NPC_ITEM_TYPE_SCTP; - - sctp_header_t sctp_spec = { .src_port = l4_src_port, - .dst_port = l4_dst_port }, - sctp_mask = { .src_port = l4_src_mask, - .dst_port = l4_dst_mask }; + sctp_spec.src_port = l4_src_port; + sctp_spec.dst_port = l4_dst_port; + sctp_mask.src_port = l4_src_mask; + sctp_mask.dst_port = l4_dst_mask; item_info[layer].spec = (void *) &sctp_spec; item_info[layer].mask = (void *) &sctp_mask; item_info[layer].size = sizeof (sctp_header_t); + item_info[layer].type = ROC_NPC_ITEM_TYPE_SCTP; layer++; break; case IP_PROTOCOL_IPSEC_ESP: - item_info[layer].type = ROC_NPC_ITEM_TYPE_ESP; - esp_header_t esp_spec = { .spi = clib_host_to_net_u32 ( - flow->ip4_ipsec_esp.spi) }, - esp_mask = { .spi = 0xFFFFFFFF }; + esp_spec.spi = clib_host_to_net_u32 (flow->ip4_ipsec_esp.spi); + esp_mask.spi = 0xFFFFFFFF; item_info[layer].spec = (void *) &esp_spec; item_info[layer].mask = (void *) &esp_mask; item_info[layer].size = sizeof (u32); + item_info[layer].type = ROC_NPC_ITEM_TYPE_ESP; layer++; break; @@ -803,10 +819,10 @@ oct_flow_add (vlib_main_t *vm, vnet_dev_port_t *port, vnet_flow_t *flow, if (queues) clib_mem_free (queues); - if (flow_spec) - vec_free (flow_spec); - if (flow_mask) - vec_free (flow_mask); + vec_free (flow_spec); + vec_free (flow_mask); + vec_free (drv_spec); + vec_free (drv_mask); return rv; } From d1692ef2a75996aebba464584068c91f4d5e2106 Mon Sep 17 00:00:00 2001 From: Bheemappa Agasimundin Date: Mon, 5 Aug 2024 10:01:40 +0530 Subject: [PATCH 079/271] octeon: update virtio header length The length of the Virtio header fluctuates depending on the features that have been negotiated with the host. This patch integrates this modification. Type: fix JIRA: https://essjira.marvell.com/browse/IPBUSW-49355 Signed-off-by: Bheemappa Agasimundin Change-Id: Icd448b14a4a4a1bb3590ba99b4e33260c6d5c320 Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/132773 Tested-by: sa_ip-sw-jenkins Reviewed-by: Monendra Singh Kushwaha (cherry picked from commit 16d63c8047974e03b2856508bf858feb663c5d1b) Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/134662 Reviewed-by: Nithinsen Kaithakadan --- src/plugins/dev_octeon/oct_virtio.h | 15 ++++-- src/plugins/dev_octeon/virtio_ctrl.c | 41 ++++++++++++++-- src/plugins/dev_octeon/virtio_rx_node.c | 64 +++++++++++++------------ src/plugins/dev_octeon/virtio_tx_node.c | 28 ++++++----- 4 files changed, 96 insertions(+), 52 deletions(-) diff --git a/src/plugins/dev_octeon/oct_virtio.h b/src/plugins/dev_octeon/oct_virtio.h index 04b54230b0..b8f852b179 100644 --- a/src/plugins/dev_octeon/oct_virtio.h +++ b/src/plugins/dev_octeon/oct_virtio.h @@ -52,17 +52,20 @@ typedef struct } oct_virt_rx_trace_t; always_inline vlib_buffer_t * -oct_virt_to_bp (void *b) +oct_virt_to_bp (void *b, u16 hdr_len) { - return (vlib_buffer_t *) ((u8 *) b + sizeof (struct dao_virtio_net_hdr) - - sizeof (vlib_buffer_t)); + return ( + vlib_buffer_t *) ((u8 *) b + + sizeof (((struct dao_virtio_net_hdr *) 0)->desc_data) + + hdr_len - sizeof (vlib_buffer_t)); } always_inline void * -oct_bp_to_virt (vlib_buffer_t *b) +oct_bp_to_virt (vlib_buffer_t *b, u16 hdr_len) { return (void *) ((u8 *) vlib_buffer_get_current (b) - - sizeof (struct dao_virtio_net_hdr)); + sizeof (((struct dao_virtio_net_hdr *) 0)->desc_data) - + hdr_len); } typedef struct @@ -99,11 +102,13 @@ typedef struct u64 qmap; u16 last_rx_q; u16 last_tx_q; + u16 virtio_hdr_sz; } oct_virtio_q_info_t; typedef struct { u8 initialized; + u16 service_core; u64 netdev_map; oct_virtio_q_info_t q_map[DAO_VIRTIO_DEV_MAX]; } oct_virtio_per_thread_data_t; diff --git a/src/plugins/dev_octeon/virtio_ctrl.c b/src/plugins/dev_octeon/virtio_ctrl.c index cf25d78147..4be643b861 100644 --- a/src/plugins/dev_octeon/virtio_ctrl.c +++ b/src/plugins/dev_octeon/virtio_ctrl.c @@ -23,12 +23,16 @@ int oct_virtio_vlib_buffer_free (u16 devid, void *buffs[], u16 nb_buffs) { int i = 0; + u16 hdr_len; u32 bi[nb_buffs]; vlib_buffer_t *b[nb_buffs]; + u32 cpu_id = clib_get_current_cpu_id (); vlib_main_t *vm = vlib_get_first_main (); + oct_virtio_per_thread_data_t *ptd = oct_virt_thread_data; + hdr_len = ptd[cpu_id].q_map[devid].virtio_hdr_sz; for (i = 0; i < nb_buffs; i++) - b[i] = oct_virt_to_bp (buffs[i]); + b[i] = oct_virt_to_bp (buffs[i], hdr_len); vlib_get_buffer_indices (vm, b, bi, nb_buffs); vlib_buffer_free_no_next (vm, bi, nb_buffs); @@ -40,11 +44,15 @@ int oct_virtio_vlib_buffer_alloc (u16 devid, void *buffs[], u16 nb_buffs) { int i = 0; + u16 hdr_len; u16 allocated; u32 vbuf_idxs[nb_buffs]; vlib_buffer_t *b[nb_buffs]; + u32 cpu_id = clib_get_current_cpu_id (); vlib_main_t *vm = vlib_get_first_main (); + oct_virtio_per_thread_data_t *ptd = oct_virt_thread_data; + hdr_len = ptd[cpu_id].q_map[devid].virtio_hdr_sz; allocated = vlib_buffer_alloc (vm, vbuf_idxs, nb_buffs); if (allocated != nb_buffs) { @@ -54,7 +62,7 @@ oct_virtio_vlib_buffer_alloc (u16 devid, void *buffs[], u16 nb_buffs) vlib_get_buffers (vm, vbuf_idxs, b, nb_buffs); for (i = 0; i < nb_buffs; i++) - buffs[i] = oct_bp_to_virt (b[i]); + buffs[i] = oct_bp_to_virt (b[i], hdr_len); return 0; } @@ -113,6 +121,24 @@ oct_virtio_clear_lcore_queue_mapping (u16 virtio_devid) ovm->netdev_qp_count[virtio_devid] = 0; } +static_always_inline u16 +oct_virtio_netdev_hdrlen_get (u16 virtio_devid) +{ + struct virtio_net_hdr vnet_hdr; + u16 virtio_hdr_sz = 0; + u64 feature_bits = 0; + + feature_bits = dao_virtio_netdev_feature_bits_get (virtio_devid); + + if (feature_bits & DAO_BIT_ULL (VIRTIO_NET_F_HASH_REPORT)) + virtio_hdr_sz = offsetof (struct virtio_net_hdr, padding_reserved) + + sizeof (vnet_hdr.padding_reserved); + else + virtio_hdr_sz = offsetof (struct virtio_net_hdr, num_buffers) + + sizeof (vnet_hdr.num_buffers); + return virtio_hdr_sz; +} + static_always_inline int oct_virtio_setup_worker_queue_mapping (u16 virtio_devid, u16 virt_q_count) { @@ -121,6 +147,10 @@ oct_virtio_setup_worker_queue_mapping (u16 virtio_devid, u16 virt_q_count) oct_virtio_main_t *ovm = oct_virtio_main; u64 wrkr_cpu_mask = ovm->wrkr_cpu_mask; oct_virtio_per_thread_data_t *ptd = oct_virt_thread_data; + u16 virtio_hdr_sz = 0; + + virtio_hdr_sz = oct_virtio_netdev_hdrlen_get (virtio_devid); + ptd[ptd->service_core].q_map[virtio_devid].virtio_hdr_sz = virtio_hdr_sz; virt_rx_q = virt_q_count / 2; q_id = 0; @@ -130,8 +160,9 @@ oct_virtio_setup_worker_queue_mapping (u16 virtio_devid, u16 virt_q_count) cpu_id++; ptd[cpu_id].q_map[virtio_devid].qmap |= DAO_BIT_ULL (q_id); - ptd[cpu_id].netdev_map |= DAO_BIT (virtio_devid); + ptd[cpu_id].q_map[virtio_devid].virtio_hdr_sz = virtio_hdr_sz; CLIB_MEMORY_BARRIER (); + ptd[cpu_id].netdev_map |= DAO_BIT (virtio_devid); wrkr_cpu_mask &= ~DAO_BIT_ULL (cpu_id); cpu_id++; @@ -143,8 +174,8 @@ oct_virtio_setup_worker_queue_mapping (u16 virtio_devid, u16 virt_q_count) } ovm->netdev_qp_count[virtio_devid] = virt_q_count / 2; - ovm->netdev_map |= DAO_BIT (virtio_devid); CLIB_MEMORY_BARRIER (); + ovm->netdev_map |= DAO_BIT (virtio_devid); return 0; } @@ -264,10 +295,12 @@ virtio_ctrl_thread_fn (void *args) { vlib_worker_thread_t *w = (vlib_worker_thread_t *) args; oct_virtio_main_t *ovm = oct_virtio_main; + oct_virtio_per_thread_data_t *ptd = oct_virt_thread_data; u32 cpu_id = clib_get_current_cpu_id (); vlib_worker_thread_init (w); ovm->wrkr_cpu_mask |= DAO_BIT (cpu_id); + ptd->service_core = cpu_id; /* Wait till Octeon virtio DAO lib init is complete */ while (!ovm || !ovm->dao_lib_initialized) CLIB_PAUSE (); diff --git a/src/plugins/dev_octeon/virtio_rx_node.c b/src/plugins/dev_octeon/virtio_rx_node.c index 96cc44356f..b027da3ccb 100644 --- a/src/plugins/dev_octeon/virtio_rx_node.c +++ b/src/plugins/dev_octeon/virtio_rx_node.c @@ -39,20 +39,20 @@ oct_virt_trace_rx_buffers (vlib_main_t *vm, vlib_node_runtime_t *node, static_always_inline vlib_buffer_t * oct_virt_populate_inner_segments (vlib_main_t *vm, vlib_buffer_t *head, - void *p, u32 pool_idx) + void *p, u32 pool_idx, u16 hdr_len) { vlib_buffer_t *prev, *b; struct dao_virtio_net_hdr *v_hdr; v_hdr = (struct dao_virtio_net_hdr *) p; - b = oct_virt_to_bp (p); + b = oct_virt_to_bp (p, hdr_len); head->total_length_not_including_first_buffer += b->current_length; prev = b; while (v_hdr->desc_data[0]) { v_hdr = (struct dao_virtio_net_hdr *) v_hdr->desc_data[0]; - b = oct_virt_to_bp ((void *) v_hdr->desc_data[0]); - b->current_length = OCT_VIRT_LENGTH (v_hdr); + b = oct_virt_to_bp ((void *) v_hdr->desc_data[0], hdr_len); + b->current_length = OCT_VIRT_LENGTH (v_hdr) - hdr_len; b->current_data = 0; prev->flags |= VLIB_BUFFER_NEXT_PRESENT; head->total_length_not_including_first_buffer += b->current_length; @@ -67,7 +67,8 @@ oct_virt_populate_inner_segments (vlib_main_t *vm, vlib_buffer_t *head, static_always_inline vlib_buffer_t * oct_virt_process_chained_packets (vlib_main_t *vm, void **pkts, - u32 *nb_pkts_chain, u32 *n_rx_bytes) + u32 *nb_pkts_chain, u32 *n_rx_bytes, + u16 hdr_len) { u32 vhdr_len = sizeof (struct virtio_net_hdr); struct dao_virtio_net_hdr *v_hdr; @@ -78,8 +79,8 @@ oct_virt_process_chained_packets (vlib_main_t *vm, void **pkts, pool_idx = vlib_buffer_pool_get_default_for_numa (vm, 0); v_hdr = (struct dao_virtio_net_hdr *) pkts[idx]; - len = OCT_VIRT_LENGTH (v_hdr); - head = oct_virt_to_bp (pkts[idx]); + len = OCT_VIRT_LENGTH (v_hdr) - hdr_len; + head = oct_virt_to_bp (pkts[idx], hdr_len); /** * If Host uses linux virtio interface skip first buffer as it contains @@ -91,19 +92,20 @@ oct_virt_process_chained_packets (vlib_main_t *vm, void **pkts, vlib_buffer_free_no_next (vm, &buffer_index, 1); idx++; - head = oct_virt_to_bp (pkts[idx]); + head = oct_virt_to_bp (pkts[idx], hdr_len); head->buffer_pool_index = pool_idx; } do { v_hdr = (struct dao_virtio_net_hdr *) pkts[idx]; - b = oct_virt_to_bp (pkts[idx]); - b->current_length = OCT_VIRT_LENGTH (v_hdr); + b = oct_virt_to_bp (pkts[idx], hdr_len); + b->current_length = OCT_VIRT_LENGTH (v_hdr) - hdr_len; b->current_data = 0; /* Check for DPU side segmentation */ if (PREDICT_FALSE ((v_hdr->desc_data[0]))) - b = oct_virt_populate_inner_segments (vm, head, pkts[idx], pool_idx); + b = oct_virt_populate_inner_segments (vm, head, pkts[idx], pool_idx, + hdr_len); if (prev) { @@ -129,7 +131,8 @@ oct_virt_process_chained_packets (vlib_main_t *vm, void **pkts, static_always_inline u32 oct_virtio_process_virtio_packets (vlib_main_t *vm, void **pkts, vlib_buffer_t **b, u32 *n_rx_pkts, - u32 *to_next, vnet_dev_rx_queue_t *rxq) + u32 *to_next, vnet_dev_rx_queue_t *rxq, + u16 hdr_len) { u8 flags = 0; int idx = 0; @@ -153,25 +156,25 @@ oct_virtio_process_virtio_packets (vlib_main_t *vm, void **pkts, if (PREDICT_FALSE (flags)) break; - b[0] = oct_virt_to_bp (pkts[idx + 0]); - b[1] = oct_virt_to_bp (pkts[idx + 1]); - b[2] = oct_virt_to_bp (pkts[idx + 2]); - b[3] = oct_virt_to_bp (pkts[idx + 3]); + b[0] = oct_virt_to_bp (pkts[idx + 0], hdr_len); + b[1] = oct_virt_to_bp (pkts[idx + 1], hdr_len); + b[2] = oct_virt_to_bp (pkts[idx + 2], hdr_len); + b[3] = oct_virt_to_bp (pkts[idx + 3], hdr_len); - clib_prefetch_store (oct_virt_to_bp (pkts[idx + 4])); - clib_prefetch_store (oct_virt_to_bp (pkts[idx + 5])); - clib_prefetch_store (oct_virt_to_bp (pkts[idx + 6])); - clib_prefetch_store (oct_virt_to_bp (pkts[idx + 7])); + clib_prefetch_store (oct_virt_to_bp (pkts[idx + 4], hdr_len)); + clib_prefetch_store (oct_virt_to_bp (pkts[idx + 5], hdr_len)); + clib_prefetch_store (oct_virt_to_bp (pkts[idx + 6], hdr_len)); + clib_prefetch_store (oct_virt_to_bp (pkts[idx + 7], hdr_len)); b[0]->template = bt; b[1]->template = bt; b[2]->template = bt; b[3]->template = bt; - b[0]->current_length = OCT_VIRT_LENGTH (v_hdr[0]); - b[1]->current_length = OCT_VIRT_LENGTH (v_hdr[1]); - b[2]->current_length = OCT_VIRT_LENGTH (v_hdr[2]); - b[3]->current_length = OCT_VIRT_LENGTH (v_hdr[3]); + b[0]->current_length = OCT_VIRT_LENGTH (v_hdr[0]) - hdr_len; + b[1]->current_length = OCT_VIRT_LENGTH (v_hdr[1]) - hdr_len; + b[2]->current_length = OCT_VIRT_LENGTH (v_hdr[2]) - hdr_len; + b[3]->current_length = OCT_VIRT_LENGTH (v_hdr[3]) - hdr_len; n_rx_bytes += b[0]->current_length; n_rx_bytes += b[1]->current_length; @@ -194,15 +197,15 @@ oct_virtio_process_virtio_packets (vlib_main_t *vm, void **pkts, { nb_pkts_chain = 1; v_hdr[0] = (struct dao_virtio_net_hdr *) pkts[idx]; - b[0] = oct_virt_to_bp (pkts[idx]); + b[0] = oct_virt_to_bp (pkts[idx], hdr_len); b[0]->template = bt; if (OCT_VIRT_NEXT_FLAG (v_hdr[0])) - b[0] = oct_virt_process_chained_packets (vm, &pkts[idx], - &nb_pkts_chain, &n_rx_bytes); + b[0] = oct_virt_process_chained_packets ( + vm, &pkts[idx], &nb_pkts_chain, &n_rx_bytes, hdr_len); else { - b[0]->current_length = OCT_VIRT_LENGTH (v_hdr[0]); + b[0]->current_length = OCT_VIRT_LENGTH (v_hdr[0]) - hdr_len; n_rx_bytes += b[0]->current_length; b[0]->current_data = 0; } @@ -233,9 +236,9 @@ oct_virtio_rx_node_inline (vlib_main_t *vm, vlib_node_runtime_t *node, u32 cpu_id = vm->cpu_id; void *pkts[VLIB_FRAME_SIZE]; u32 *to_next, n_left_to_next; - u16 queue, virt_q, virtio_id; vlib_buffer_t *b[VLIB_FRAME_SIZE]; vnet_main_t *vnm = vnet_get_main (); + u16 queue, virt_q, virtio_id, hdr_len; u32 thr_idx = vlib_get_thread_index (); oct_virtio_main_t *ovm = oct_virtio_main; u32 n_rx_pkts, n_rx_bytes = 0, rx_pkts_total = 0; @@ -265,6 +268,7 @@ oct_virtio_rx_node_inline (vlib_main_t *vm, vlib_node_runtime_t *node, dao_dma_flush_submit (); queue = ptd[cpu_id].q_map[virtio_id].last_rx_q; + hdr_len = ptd[cpu_id].q_map[virtio_id].virtio_hdr_sz; while (rx_q_map) { @@ -284,7 +288,7 @@ oct_virtio_rx_node_inline (vlib_main_t *vm, vlib_node_runtime_t *node, n_left_to_next); n_rx_bytes += oct_virtio_process_virtio_packets (vm, pkts, b, &n_rx_pkts, - to_next, rxq); + to_next, rxq, hdr_len); if (PREDICT_FALSE (trace_count)) trace_count -= diff --git a/src/plugins/dev_octeon/virtio_tx_node.c b/src/plugins/dev_octeon/virtio_tx_node.c index 03e0bb492b..26329202c2 100644 --- a/src/plugins/dev_octeon/virtio_tx_node.c +++ b/src/plugins/dev_octeon/virtio_tx_node.c @@ -22,25 +22,25 @@ oct_virtio_enqueue (vlib_main_t *vm, vlib_buffer_t **b, u16 nb_pkts, u16 virtio_devid) { int idx = 0; - u64 tx_q_map; - u64 q_map; + u64 tx_q_map, q_map; u32 cpu_id = vm->cpu_id; - u16 nb_pkts_left = nb_pkts; + u16 nb_pkts_left = nb_pkts, hdr_len; u16 queue, virt_q, sent = 0, cur_sent = 0; void *virt_b[VLIB_FRAME_SIZE]; struct dao_virtio_net_hdr *v_hdr[4]; - struct dao_virtio_net_hdr vhdr_init = {}; + struct dao_virtio_net_hdr vhdr_init = { 0 }; oct_virtio_per_thread_data_t *ptd = oct_virt_thread_data; tx_q_map = ptd[cpu_id].q_map[virtio_devid].qmap; q_map = ptd[cpu_id].q_map[virtio_devid].qmap; + hdr_len = ptd[cpu_id].q_map[virtio_devid].virtio_hdr_sz; while (nb_pkts >= 8) { - v_hdr[0] = oct_bp_to_virt (b[0]); - v_hdr[1] = oct_bp_to_virt (b[1]); - v_hdr[2] = oct_bp_to_virt (b[2]); - v_hdr[3] = oct_bp_to_virt (b[3]); + v_hdr[0] = oct_bp_to_virt (b[0], hdr_len); + v_hdr[1] = oct_bp_to_virt (b[1], hdr_len); + v_hdr[2] = oct_bp_to_virt (b[2], hdr_len); + v_hdr[3] = oct_bp_to_virt (b[3], hdr_len); *v_hdr[0] = vhdr_init; *v_hdr[1] = vhdr_init; @@ -52,10 +52,10 @@ oct_virtio_enqueue (vlib_main_t *vm, vlib_buffer_t **b, u16 nb_pkts, virt_b[idx + 2] = (void *) v_hdr[2]; virt_b[idx + 3] = (void *) v_hdr[3]; - clib_prefetch_store (oct_bp_to_virt (b[4])); - clib_prefetch_store (oct_bp_to_virt (b[5])); - clib_prefetch_store (oct_bp_to_virt (b[6])); - clib_prefetch_store (oct_bp_to_virt (b[7])); + clib_prefetch_store (oct_bp_to_virt (b[4], hdr_len)); + clib_prefetch_store (oct_bp_to_virt (b[5], hdr_len)); + clib_prefetch_store (oct_bp_to_virt (b[6], hdr_len)); + clib_prefetch_store (oct_bp_to_virt (b[7], hdr_len)); vlib_prefetch_buffer_header (b[4], LOAD); vlib_prefetch_buffer_header (b[5], LOAD); @@ -67,6 +67,7 @@ oct_virtio_enqueue (vlib_main_t *vm, vlib_buffer_t **b, u16 nb_pkts, v_hdr[2]->desc_data[1] = b[2]->current_length; v_hdr[3]->desc_data[1] = b[3]->current_length; + /* Number of bytes deviates (+/-) from vlib buffer current data */ v_hdr[0]->desc_data[0] = ~b[0]->current_data + 1; v_hdr[1]->desc_data[0] = ~b[1]->current_data + 1; v_hdr[2]->desc_data[0] = ~b[2]->current_data + 1; @@ -84,11 +85,12 @@ oct_virtio_enqueue (vlib_main_t *vm, vlib_buffer_t **b, u16 nb_pkts, while (nb_pkts) { - v_hdr[0] = oct_bp_to_virt (b[0]); + v_hdr[0] = oct_bp_to_virt (b[0], hdr_len); *v_hdr[0] = vhdr_init; v_hdr[0]->hdr.num_buffers = 1; virt_b[idx] = (void *) v_hdr[0]; v_hdr[0]->desc_data[1] = b[0]->current_length; + /* Number of bytes deviates (+/-) from vlib buffer current data */ v_hdr[0]->desc_data[0] = ~b[0]->current_data + 1; b++; From 0d0844da3015dee02b5be832c6592d1f3e203070 Mon Sep 17 00:00:00 2001 From: Bheemappa Agasimundin Date: Wed, 14 Aug 2024 22:12:51 +0530 Subject: [PATCH 080/271] octeon: add egress chaining support for virtio This patch adds octeon virtio egress chaining support. Type: feature JIRA: https://essjira.marvell.com/browse/IPBUSW-50557 Signed-off-by: Bheemappa Agasimundin Change-Id: Iae2328560584f8b4da21e10e9771ad557a2f0b78 Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/133543 Tested-by: sa_ip-sw-jenkins Reviewed-by: Monendra Singh Kushwaha Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/134664 Reviewed-by: Nithinsen Kaithakadan --- src/plugins/dev_octeon/virtio.c | 14 +---- src/plugins/dev_octeon/virtio_ctrl.c | 5 +- src/plugins/dev_octeon/virtio_tx_node.c | 82 ++++++++++++++++++------- 3 files changed, 64 insertions(+), 37 deletions(-) diff --git a/src/plugins/dev_octeon/virtio.c b/src/plugins/dev_octeon/virtio.c index 5437060bc7..c4f41e672f 100644 --- a/src/plugins/dev_octeon/virtio.c +++ b/src/plugins/dev_octeon/virtio.c @@ -226,18 +226,6 @@ oct_virtio_parse_arguments (dao_pal_global_conf_t *conf, vnet_dev_arg_t *args) } } -static void -oct_dev_virtio_mac_addr_get (u16 dev_id, u8 mac_addr[]) -{ - mac_addr[0] = 0x00; - mac_addr[1] = 0x0f; - mac_addr[2] = 0xb7; - mac_addr[3] = 0x11; - mac_addr[4] = (dev_id & PEM_PFVF_DEV_ID_VF_MASK) >> PEM_PFVF_DEV_ID_PF_SHIFT; - mac_addr[5] = - ((dev_id & PEM_PFVF_DEV_ID_VF_MASK) >> PEM_PFVF_DEV_ID_VF_SHIFT) + 1; -} - static vnet_dev_rv_t oct_virtio_init (vlib_main_t *vm, vnet_dev_t *dev) { @@ -304,7 +292,7 @@ oct_virtio_init (vlib_main_t *vm, vnet_dev_t *dev) oct_virtio_main->dao_lib_initialized = 1; } - oct_dev_virtio_mac_addr_get (bus_data->virtio_dev.virtio_id, mac_addr); + ethernet_mac_address_generate (mac_addr); device_data->virtio_id = bus_data->virtio_dev.virtio_id; ovp.virtio_id = bus_data->virtio_dev.virtio_id; diff --git a/src/plugins/dev_octeon/virtio_ctrl.c b/src/plugins/dev_octeon/virtio_ctrl.c index 4be643b861..435eb373d6 100644 --- a/src/plugins/dev_octeon/virtio_ctrl.c +++ b/src/plugins/dev_octeon/virtio_ctrl.c @@ -150,7 +150,6 @@ oct_virtio_setup_worker_queue_mapping (u16 virtio_devid, u16 virt_q_count) u16 virtio_hdr_sz = 0; virtio_hdr_sz = oct_virtio_netdev_hdrlen_get (virtio_devid); - ptd[ptd->service_core].q_map[virtio_devid].virtio_hdr_sz = virtio_hdr_sz; virt_rx_q = virt_q_count / 2; q_id = 0; @@ -160,7 +159,6 @@ oct_virtio_setup_worker_queue_mapping (u16 virtio_devid, u16 virt_q_count) cpu_id++; ptd[cpu_id].q_map[virtio_devid].qmap |= DAO_BIT_ULL (q_id); - ptd[cpu_id].q_map[virtio_devid].virtio_hdr_sz = virtio_hdr_sz; CLIB_MEMORY_BARRIER (); ptd[cpu_id].netdev_map |= DAO_BIT (virtio_devid); @@ -173,6 +171,9 @@ oct_virtio_setup_worker_queue_mapping (u16 virtio_devid, u16 virt_q_count) } } + for (cpu_id = 0; cpu_id < DAO_PAL_MAX_WORKERS; cpu_id++) + ptd[cpu_id].q_map[virtio_devid].virtio_hdr_sz = virtio_hdr_sz; + ovm->netdev_qp_count[virtio_devid] = virt_q_count / 2; CLIB_MEMORY_BARRIER (); ovm->netdev_map |= DAO_BIT (virtio_devid); diff --git a/src/plugins/dev_octeon/virtio_tx_node.c b/src/plugins/dev_octeon/virtio_tx_node.c index 26329202c2..b9862831eb 100644 --- a/src/plugins/dev_octeon/virtio_tx_node.c +++ b/src/plugins/dev_octeon/virtio_tx_node.c @@ -13,21 +13,44 @@ #include #include #include +#define OCT_VIRT_MAX_FRAGS 6 extern oct_virtio_main_t *oct_virtio_main; extern oct_virtio_per_thread_data_t *oct_virt_thread_data; +static_always_inline void +oct_virt_free_to_vlib (vlib_main_t *vm, vlib_node_runtime_t *node, + void *virt_b[], u16 nb_free, u16 hdr_len) +{ + u16 idx = 0; + vlib_buffer_t *b; + u32 b_index; + + while (idx < nb_free) + { + b = oct_virt_to_bp (virt_b[idx], hdr_len); + b_index = vlib_get_buffer_index (vm, b); + vlib_buffer_free_no_next (vm, &b_index, 1); + idx++; + } + + vlib_error_count (vm, node->node_index, OCT_VIRT_TX_NODE_CTR_ENQUE_FAIL, + nb_free); +} + static_always_inline u32 -oct_virtio_enqueue (vlib_main_t *vm, vlib_buffer_t **b, u16 nb_pkts, - u16 virtio_devid) +oct_virtio_enqueue (vlib_main_t *vm, vlib_node_runtime_t *node, + vlib_buffer_t **b, u16 nb_pkts, u16 virtio_devid) { - int idx = 0; + vlib_buffer_t *bp; + bool next_present; u64 tx_q_map, q_map; + u16 idx = 0, nb_frags = 0; u32 cpu_id = vm->cpu_id; u16 nb_pkts_left = nb_pkts, hdr_len; u16 queue, virt_q, sent = 0, cur_sent = 0; - void *virt_b[VLIB_FRAME_SIZE]; - struct dao_virtio_net_hdr *v_hdr[4]; + void *virt_b[VLIB_FRAME_SIZE * OCT_VIRT_MAX_FRAGS]; + struct dao_virtio_net_hdr *v_hdr[4], *head; struct dao_virtio_net_hdr vhdr_init = { 0 }; oct_virtio_per_thread_data_t *ptd = oct_virt_thread_data; @@ -37,6 +60,13 @@ oct_virtio_enqueue (vlib_main_t *vm, vlib_buffer_t **b, u16 nb_pkts, while (nb_pkts >= 8) { + next_present = b[0]->flags & VLIB_BUFFER_NEXT_PRESENT || + b[1]->flags & VLIB_BUFFER_NEXT_PRESENT || + b[2]->flags & VLIB_BUFFER_NEXT_PRESENT || + b[3]->flags & VLIB_BUFFER_NEXT_PRESENT; + if (PREDICT_FALSE (next_present)) + break; + v_hdr[0] = oct_bp_to_virt (b[0], hdr_len); v_hdr[1] = oct_bp_to_virt (b[1], hdr_len); v_hdr[2] = oct_bp_to_virt (b[2], hdr_len); @@ -85,20 +115,31 @@ oct_virtio_enqueue (vlib_main_t *vm, vlib_buffer_t **b, u16 nb_pkts, while (nb_pkts) { - v_hdr[0] = oct_bp_to_virt (b[0], hdr_len); - *v_hdr[0] = vhdr_init; - v_hdr[0]->hdr.num_buffers = 1; - virt_b[idx] = (void *) v_hdr[0]; - v_hdr[0]->desc_data[1] = b[0]->current_length; - /* Number of bytes deviates (+/-) from vlib buffer current data */ - v_hdr[0]->desc_data[0] = ~b[0]->current_data + 1; - + bp = b[0]; + head = oct_bp_to_virt (bp, hdr_len); + do + { + v_hdr[0] = oct_bp_to_virt (bp, hdr_len); + *v_hdr[0] = vhdr_init; + virt_b[idx] = (void *) v_hdr[0]; + v_hdr[0]->desc_data[1] = bp->current_length; + /* Number of bytes deviates (+/-) from vlib buffer current data */ + v_hdr[0]->desc_data[0] = ~bp->current_data + 1; + next_present = bp->flags & VLIB_BUFFER_NEXT_PRESENT; + v_hdr[0]->hdr.num_buffers = 1; + idx++; + nb_frags++; + } + while (next_present && (bp = vlib_get_buffer (vm, bp->next_buffer))); + + head->hdr.num_buffers = nb_frags; b++; - idx++; nb_pkts--; + nb_frags = 0; } queue = ptd[cpu_id].q_map[virtio_devid].last_tx_q; + nb_pkts_left = idx; while (tx_q_map && nb_pkts_left) { @@ -117,8 +158,12 @@ oct_virtio_enqueue (vlib_main_t *vm, vlib_buffer_t **b, u16 nb_pkts, if (DAO_BIT (queue) > q_map) queue = 0; } + ptd[cpu_id].q_map[virtio_devid].last_tx_q = queue; + if (PREDICT_FALSE (nb_pkts_left)) + oct_virt_free_to_vlib (vm, node, &virt_b[sent], nb_pkts_left, hdr_len); + /* Flush and submit DMA ops */ dao_dma_flush_submit (); @@ -171,14 +216,7 @@ VNET_DEV_NODE_FN (oct_virtio_tx_node) if (PREDICT_FALSE (node->flags & VLIB_NODE_FLAG_TRACE)) oct_virtio_trace_buffers (vm, node, ovp, b, n_pkts, virtio_id); - n_tx_pkts = oct_virtio_enqueue (vm, b, n_pkts, virtio_id); - if (n_tx_pkts < n_pkts) - { - u32 n_free = n_pkts - n_tx_pkts; - vlib_buffer_free_no_next (vm, from + n_tx_pkts, n_free); - vlib_error_count (vm, node->node_index, OCT_VIRT_TX_NODE_CTR_ENQUE_FAIL, - n_free); - } + n_tx_pkts = oct_virtio_enqueue (vm, node, b, n_pkts, virtio_id); return n_tx_pkts; } From d5c8dd5a77981cb23a19cbecea0a8d688cffe288 Mon Sep 17 00:00:00 2001 From: Bheemappa Agasimundin Date: Wed, 21 Aug 2024 18:45:03 +0530 Subject: [PATCH 081/271] octeon: fix library link issue This patch fixes OCTEON DAO library link issue. Type: fix JIRA: https://essjira.marvell.com/browse/IPBUSW-52142 Signed-off-by: Bheemappa Agasimundin Change-Id: Ic8f85d3ba5cbd0ddb3946fcb60b36bd530f8c9a1 Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/133895 Tested-by: sa_ip-sw-jenkins Reviewed-by: Monendra Singh Kushwaha (cherry picked from commit 82f6e4fbd8f6d5f3611f4aad103dc4e238634924) Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/134663 Reviewed-by: Nithinsen Kaithakadan --- build/external/packages/octeon-dao.mk | 3 +-- src/plugins/dev_octeon/dev_octeon_virtio.mk | 13 +++++++++++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/build/external/packages/octeon-dao.mk b/build/external/packages/octeon-dao.mk index 9207a2dc7e..eb56659fc4 100644 --- a/build/external/packages/octeon-dao.mk +++ b/build/external/packages/octeon-dao.mk @@ -20,8 +20,7 @@ DAO_MESON_ARGS = \ --default-library static \ -Dprefer_static=True \ --buildtype=$(DAO_BUILD_TYPE)\ - -Denable_kmods=false\ - -Dc_link_args='-lnuma' + -Denable_kmods=false PREFIX = $(CNXK_SDK_SYSROOT) ifeq (,$(findstring $(OCTEON_VERSION),cn10k cn9k)) diff --git a/src/plugins/dev_octeon/dev_octeon_virtio.mk b/src/plugins/dev_octeon/dev_octeon_virtio.mk index 91ea0eb084..5f84a7f4b4 100644 --- a/src/plugins/dev_octeon/dev_octeon_virtio.mk +++ b/src/plugins/dev_octeon/dev_octeon_virtio.mk @@ -24,9 +24,18 @@ if (NOT DAO_PAL_LIB OR NOT DAO_VIRT_LIB OR NOT DAO_VIRT_NET_LIB OR NOT DAO_VFIO_ return() endif() -set (DAO_LIBS ${DAO_PAL_LIB} ${DAO_VIRT_LIB} ${DAO_VIRT_NET_LIB} ${DAO_VFIO_LIB} ${DAO_PEM_LIB} ${DAO_COMM_LIB} ${DAO_DPDK_LIB}) - unset(DAO_LINK_FLAGS) + +get_filename_component(DAO_DPDK_LIB_DIR ${DAO_DPDK_LIB} DIRECTORY) + +link_directories(${DAO_DPDK_LIB_DIR}) +string_append(DAO_LINK_FLAGS "-L${DAO_DPDK_LIB_DIR}") +string_append(DAO_LINK_FLAGS "-lnuma -lz -lelf -lpcap -ljansson") +if(OPENSSL_FOUND) + string_append(DAO_LINK_FLAGS "-lssl") + string_append(DAO_LINK_FLAGS "-lcrypto") +endif() + string_append(DAO_LINK_FLAGS "-Wl,--whole-archive,${DAO_PAL_LIB},${DAO_VIRT_LIB},${DAO_VIRT_NET_LIB},${DAO_VFIO_LIB},${DAO_PEM_LIB},${DAO_COMM_LIB},${DAO_DPDK_LIB},--no-whole-archive") include_directories (${DAO_NETDEV_INCLUDE_DIR}/) From 79ad18571b6a210b1df62c0ede3fa718691bdef7 Mon Sep 17 00:00:00 2001 From: Bheemappa Agasimundin Date: Thu, 1 Aug 2024 14:44:17 +0530 Subject: [PATCH 082/271] octeon: add readme for octeon-virtio plugin This document provides an overview of the OCTEON virtio plugin and includes configuration details for using the plugin. Type: feature JIRA: https://essjira.marvell.com/browse/IPBUSW-50902 Signed-off-by: Bheemappa Agasimundin Change-Id: Ie151d8d84fe5c19b3862e93013a04acdef9c8810 Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/132586 Reviewed-by: Abed Mohammad Kamaluddin Tested-by: sa_ip-sw-jenkins (cherry picked from commit 432105b7e391c21b065a0610569b0352967e1ced) Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/134673 Reviewed-by: Nithinsen Kaithakadan --- src/plugins/dev_octeon/README_VIRTIO.md | 123 ++++++++++++++++++++++++ 1 file changed, 123 insertions(+) create mode 100644 src/plugins/dev_octeon/README_VIRTIO.md diff --git a/src/plugins/dev_octeon/README_VIRTIO.md b/src/plugins/dev_octeon/README_VIRTIO.md new file mode 100644 index 0000000000..dbf03dfd3c --- /dev/null +++ b/src/plugins/dev_octeon/README_VIRTIO.md @@ -0,0 +1,123 @@ +# Octeon virtio device plugin for VPP {#dev_octeon_virtio_doc} + +## Overview + +This plugin is a virtio device plugin for VPP, supporting packet input and output to +and from the HOST virtio interface over PCIe, with the Marvell OCTEON SoC operating +in endpoint mode. + +This plugin uses DAO library to communicate with the HOST device. The DAO library +employs a platform device to transmit and receive data to and from the HOST device. +Platform devices aren't situated on standard buses such as PCI or USB, this plugin +enlists a virtual bus with VPP, identifying it as virtio.To enhance performance, +this plugin utilizes a dedicated core to transfer host descriptors to Octeon. + +An alternate way for Host-OCTEON communication is using the SDP interface. The +primary difference is that SDP interfaces utilize NIX device bandwidth, thereby +limiting the device to 50Gbps when used in endpoint NIC mode. In contrast, by +using the Virtio plugin, OCTEON can function as a 100Gbps NIC. + +## Supported SoC +- OCTEON-10 + +## Usage +The following steps demonstrate how you may bring up VPP with dev_octeon_virtio, on the +OCTEON platform. + +### Setup + +#### Configure DMA and NPA devices on OCTEON +-# Determine DMA/DPI device on OCTEON. +``` +# lspci -d 177d:a080:0880 + 0000:06:00.0 System peripheral: Cavium, Inc. Device a080 +``` +-# Bind and Create (2 + 2 * number of workers) DMA devices. +``` +echo 0000:06:00.0 > /sys/bus/pci/devices/0000:06:00.0/driver/unbind +echo octeontx2-dpi > /sys/bus/pci/devices/0000:06:00.0/driver_override +echo 0000:06:00.0 > /sys/bus/pci/drivers_probe +echo 32 >/sys/bus/pci/devices/0000:06:00.0/sriov_numvfs + +``` +-# Determine NPA PCI on OCTEON and bind to vfio-pci. +``` +#lspci -d 177d:a0fb:0880 +0002:17:00.0 System peripheral: Cavium, Inc. Device a0fb (rev 54) + +echo 0002:17:00.0 > /sys/bus/pci/devices/0002:17:00.0/driver/unbind +echo 177d a0fb > /sys/bus/pci/drivers/vfio-pci/new_id +echo 0002:17:00.0 > /sys/bus/pci/drivers/vfio-pci/bind +``` +-# Bind platform devices pem0-bar4-mem and dpi_sdp_regs to vfio-platform +``` +echo "vfio-platform" | sudo tee "/sys/bus/platform/devices/*pem0-bar4-mem/driver_override" > /dev/null +echo "*pem0-bar4-mem" | sudo tee "/sys/bus/platform/drivers/vfio-platform/bind" > /dev/null +echo "vfio-platform" | sudo tee "/sys/bus/platform/devices/*dpi_sdp_regs/driver_override" > /dev/null +echo "*dpi_sdp_regs" | sudo tee "/sys/bus/platform/drivers/vfio-platform/bind" > /dev/null +Note: Replace * with actual runtime address attached with platform device. + +``` +### Launch VPP +VPP device bringup with dev_octeon_virtio is possible either through vppctl commands or +startup conf.This plugin takes following device arguments for the first device attach.And +arguments passed on next devices are ignored. + +nb_virtio - Max number of virtio devices will be configured. +dma - List of all DMA devices. +misc - List of all miscellaneous devices (example NPA device). + +DMA devices needed is calculated as: + +2 (for control) + 2 (for virtio service thread) + 2 x (number of workers) + +#### Device bringup using startup.conf device section +``` +cpu { + main-core 1 + corelist-workers 8-9 + corelist-virtio-ctrl 7 +} + +devices { + dev virtio/0 + { + driver octeon_virtio + port 0 + { + name oct_virtio/0 + num-rx-queues 4 + num-tx-queues 4 + } + args 'nb_virtio=2,dma=\"0000:06:00.1,0000:06:00.2,0000:06:00.3,0000:06:00.4,0000:06:00.5,0000:06:00.6,0000:06:00.7,0000:06:01.1,0000:06:01.2,0000:06:01.3\",misc=\"0002:17:00.0\"' + } + + dev virtio/1 + { + driver octeon_virtio + port 1 + { + name oct_virtio/1 + num-rx-queues 2 + num-tx-queues 3 + } + } +} +``` + +#### Device bringup using vppctl +Launch VPP with startup conf. + +``` +# vpp -c /etc/vpp/startup.conf +# vppctl -s /run/vpp/cli.sock + _______ _ _ _____ ___ + __/ __/ _ \ (_)__ | | / / _ \/ _ \ + _/ _// // / / / _ \ | |/ / ___/ ___/ + /_/ /____(_)_/\___/ |___/_/ /_/ + + vpp# vppctl device attach virtio/0 driver octeon_virtio args nb_virtio=2,dma=\"0000:06:00.1,0000:06:00.2,0000:06:00.3,0000:06:00.4,0000:06:00.5,0000:06:00.6,0000:06:00.7,0000:06:01.1,0000:06:01.2,0000:06:01.3\",misc=\"0002:17:00.0\" + vpp# vppctl device create-interface virtio/0 port 0 num-rx-queues 2 num-tx-queues 3 + vpp# vppctl device attach virtio/1 driver octeon_virtio + vpp# vppctl device create-interface virtio/1 port 1 num-rx-queues 2 num-tx-queues 3 +``` From 11bc4e788d884dedcc3d85ef57c71e1aa240cd00 Mon Sep 17 00:00:00 2001 From: Bheemappa Agasimundin Date: Tue, 27 Aug 2024 17:29:30 +0000 Subject: [PATCH 083/271] linux-cp: fix esn and anti-replay issue This patch enables anti-replay when ESN is enabled on a Security Association (SA) configured via strongSwan. Type: fix JIRA: https://essjira.marvell.com/browse/IPBUSW-52363 Signed-off-by: Bheemappa Agasimundin Change-Id: I0efd5ba77088214245eb3b19b35309eed2ab66ca Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/134340 Tested-by: sa_ip-sw-jenkins Reviewed-by: Monendra Singh Kushwaha (cherry picked from commit 88d83f4567f7441184143a4e400af554d1819853) Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/134672 Reviewed-by: Nithinsen Kaithakadan --- src/plugins/linux-cp/lcp_ipsec.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plugins/linux-cp/lcp_ipsec.c b/src/plugins/linux-cp/lcp_ipsec.c index fe83a948ae..c847e699f8 100644 --- a/src/plugins/linux-cp/lcp_ipsec.c +++ b/src/plugins/linux-cp/lcp_ipsec.c @@ -918,7 +918,7 @@ nl_xfrm_sa_add (struct xfrmnl_sa *sa) if (xfrmnl_sa_get_flags (sa) & XFRM_STATE_ESN) { - flags |= IPSEC_SA_FLAG_USE_ESN; + flags |= IPSEC_SA_FLAG_USE_ESN | IPSEC_SA_FLAG_USE_ANTI_REPLAY; } /* * Kernel SA XFRM doesnt have a flag for AR config. So a non-zero From 9cb11c67d0b13c359d1fb3fcac86397340d3c487 Mon Sep 17 00:00:00 2001 From: Damjan Marion Date: Mon, 26 Feb 2024 16:19:56 +0000 Subject: [PATCH 084/271] crypto-native: fix AES-CBC encrypt loop Type: fix JIRA: https://essjira.marvell.com/browse/IPBUSW-47134 Change-Id: I11cc52ff3867277e6591efb061f96cadfcc70c88 Signed-off-by: Damjan Marion Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/134381 Tested-by: sa_ip-sw-jenkins Reviewed-by: Monendra Singh Kushwaha (cherry picked from commit 9a1fb77f89207cd1cc865a899a9568219b6dc0f2) Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/134713 Reviewed-by: Nithinsen Kaithakadan --- src/plugins/crypto_native/aes_cbc.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/plugins/crypto_native/aes_cbc.c b/src/plugins/crypto_native/aes_cbc.c index f2d700a079..c84390c310 100644 --- a/src/plugins/crypto_native/aes_cbc.c +++ b/src/plugins/crypto_native/aes_cbc.c @@ -55,17 +55,17 @@ aes_ops_enc_aes_cbc (vlib_main_t * vm, vnet_crypto_op_t * ops[], u32 i, j, count, n_left = n_ops; u32xN placeholder_mask = { }; u32xN len = { }; - vnet_crypto_key_index_t key_index[N_AES_BYTES]; - u8 *src[N_AES_BYTES] = {}; - u8 *dst[N_AES_BYTES] = {}; + vnet_crypto_key_index_t key_index[4 * N_AES_LANES]; + u8 *src[4 * N_AES_LANES] = {}; + u8 *dst[4 * N_AES_LANES] = {}; u8xN r[4] = {}; u8xN k[15][4] = {}; - for (i = 0; i < N_AES_BYTES; i++) + for (i = 0; i < 4 * N_AES_LANES; i++) key_index[i] = ~0; more: - for (i = 0; i < N_AES_BYTES; i++) + for (i = 0; i < 4 * N_AES_LANES; i++) if (len[i] == 0) { if (n_left == 0) @@ -198,7 +198,7 @@ aes_ops_enc_aes_cbc (vlib_main_t * vm, vnet_crypto_op_t * ops[], len -= u32xN_splat (count); - for (i = 0; i < N_AES_BYTES; i++) + for (i = 0; i < 4 * N_AES_LANES; i++) { src[i] += count; dst[i] += count; From f105cab51a376ffe048b4c757328bfea80675c78 Mon Sep 17 00:00:00 2001 From: Varun Rapelly Date: Thu, 8 Aug 2024 05:44:51 +0000 Subject: [PATCH 085/271] svm: add api to shrink allocated fifo This patch adds a new API to shrink fifos allocated in fifo segments. In svm_fifo_clear_deq_ntf, calls svm_fifo_del_want_deq_ntf when fifo is empty. Type: improvement JIRA: https://essjira.marvell.com/browse/IPBUSW-50810 Signed-off-by: Varun Rapelly Change-Id: Ic2f66730d949f2ea6d59b8c1c016ec261a818a1a Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/133077 Reviewed-by: Monendra Singh Kushwaha Reviewed-by: Venkata Ravichandra Mynidi Tested-by: sa_ip-sw-jenkins (cherry picked from commit d0c2bb324731f60ab136afed14cf283556f2cc64) Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/134723 Reviewed-by: Nithinsen Kaithakadan --- src/svm/fifo_segment.c | 38 ++++++++++++++++++++++++++++++++++++++ src/svm/fifo_segment.h | 8 ++++++++ src/svm/svm_fifo.h | 2 ++ 3 files changed, 48 insertions(+) diff --git a/src/svm/fifo_segment.c b/src/svm/fifo_segment.c index d5f6291308..d86f012ba1 100644 --- a/src/svm/fifo_segment.c +++ b/src/svm/fifo_segment.c @@ -917,6 +917,44 @@ fifo_segment_duplicate_fifo (fifo_segment_t *fs, svm_fifo_t *f) return nf; } +/** + * Shrink fifo allocated in fifo segment + */ +void +fifo_segment_shrink_fifo (fifo_segment_t *fs, svm_fifo_t *f) +{ + fifo_segment_header_t *fsh = fs->h; + fifo_segment_slice_t *fss; + svm_fifo_shared_t *sf; + svm_fifo_chunk_t *c; + + ASSERT (f->refcnt > 0); + + /* Do not try to shrink buffers if the FIFO contains data */ + if (!svm_fifo_is_empty (f)) + return; + + sf = f->shr; + fss = fsh_slice_get (fsh, sf->slice_index); + + /* Free fifo chunks */ + fsh_slice_collect_chunks (fsh, fss, fs_chunk_ptr (fsh, f->shr->start_chunk)); + + sf->start_chunk = sf->end_chunk = 0; + sf->head_chunk = sf->tail_chunk = 0; + + c = fsh_try_alloc_chunk (fsh, fss, 0); + if (!c) + return; + + sf->head_chunk = sf->start_chunk = fs_chunk_sptr (fsh, c); + while (c->next) + c = fs_chunk_ptr (fsh, c->next); + sf->tail_chunk = sf->end_chunk = fs_chunk_sptr (fsh, c); + + svm_fifo_set_size (f, 0); +} + /** * Free fifo allocated in fifo segment */ diff --git a/src/svm/fifo_segment.h b/src/svm/fifo_segment.h index ec18420726..781c5bb1b5 100644 --- a/src/svm/fifo_segment.h +++ b/src/svm/fifo_segment.h @@ -142,6 +142,14 @@ svm_fifo_t *fifo_segment_alloc_fifo_w_offset (fifo_segment_t *fs, uword offset); svm_fifo_t *fifo_segment_duplicate_fifo (fifo_segment_t *fs, svm_fifo_t *f); +/** + * Shrink fifo allocated in fifo segment + * + * @param fs fifo segment for fifo + * @param f fifo to be shrinked + */ +void fifo_segment_shrink_fifo (fifo_segment_t *fs, svm_fifo_t *f); + /** * Free fifo allocated in fifo segment * diff --git a/src/svm/svm_fifo.h b/src/svm/svm_fifo.h index 7ea114f870..14c395f63a 100644 --- a/src/svm/svm_fifo.h +++ b/src/svm/svm_fifo.h @@ -811,6 +811,8 @@ svm_fifo_clear_deq_ntf (svm_fifo_t * f) clib_atomic_store_rel_n (&f->shr->has_deq_ntf, 1); if (want_deq_ntf & SVM_FIFO_WANT_DEQ_NOTIF) svm_fifo_del_want_deq_ntf (f, SVM_FIFO_WANT_DEQ_NOTIF); + if (want_deq_ntf & SVM_FIFO_WANT_DEQ_NOTIF_IF_EMPTY) + svm_fifo_del_want_deq_ntf (f, SVM_FIFO_WANT_DEQ_NOTIF_IF_EMPTY); } /** From 943190fece875b9823f6262a755ac76a99a639c3 Mon Sep 17 00:00:00 2001 From: Varun Rapelly Date: Sat, 17 Aug 2024 12:08:34 +0000 Subject: [PATCH 086/271] session: add api to shrink fifos This patch adds new API to shrink fifos in session layer. Shrinks svm_fifo chunks when the TCP connection is in TCP_STATE_FIN_WAIT_2 and TCP_STATE_CLOSING states. The current problem with the TIME_WAIT TCP connection state in VPP is that it keeps all session resources (including RX/TX svm_fifos and their chunks) for the duration of the 2MSL timeout as required by the RFC. Such behavior might introduce the out-of-memory scenario when running many TCP sessions. The session in the TIME_STATE_CLOSING and TCP_STATE_FIN_WAIT_2 states does not need svm_fifo chunks anymore because it doesn't expect any data transfer. Deallocating svm_fifo chunks allows memory saving while keeping svm_fifo control structures used for signaling and notifications. Type: improvement JIRA: https://essjira.marvell.com/browse/IPBUSW-50810 Signed-off-by: Varun Rapelly Change-Id: Ibc92c021e8b47928b351059c3ed46ae77458a72e Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/133676 Tested-by: sa_ip-sw-jenkins Reviewed-by: Monendra Singh Kushwaha Reviewed-by: Venkata Ravichandra Mynidi (cherry picked from commit 0e746baddfcd19f59291384bc3b2b3607f63177b) Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/134724 Reviewed-by: Nithinsen Kaithakadan --- src/vnet/session/segment_manager.c | 24 ++++++++++++++++++++++++ src/vnet/session/segment_manager.h | 1 + src/vnet/session/session.c | 6 ++++++ src/vnet/session/session.h | 1 + src/vnet/tcp/tcp_input.c | 10 ++++++++++ 5 files changed, 42 insertions(+) diff --git a/src/vnet/session/segment_manager.c b/src/vnet/session/segment_manager.c index d459b73650..e8a20c9fb4 100644 --- a/src/vnet/session/segment_manager.c +++ b/src/vnet/session/segment_manager.c @@ -853,6 +853,30 @@ segment_manager_alloc_session_fifos (segment_manager_t * sm, return 0; } +void +segment_manager_shrink_fifos (svm_fifo_t *rx_fifo, svm_fifo_t *tx_fifo) +{ + segment_manager_t *sm; + fifo_segment_t *fs; + u32 segment_index; + + if (!rx_fifo || !tx_fifo) + return; + + /* + * It's possible to have no segment manager if the session was removed + * as result of a detach. + */ + if (!(sm = segment_manager_get_if_valid (rx_fifo->segment_manager))) + return; + + segment_index = rx_fifo->segment_index; + fs = segment_manager_get_segment_w_lock (sm, segment_index); + fifo_segment_shrink_fifo (fs, rx_fifo); + fifo_segment_shrink_fifo (fs, tx_fifo); + segment_manager_segment_reader_unlock (sm); +} + void segment_manager_dealloc_fifos (svm_fifo_t * rx_fifo, svm_fifo_t * tx_fifo) { diff --git a/src/vnet/session/segment_manager.h b/src/vnet/session/segment_manager.h index 1e99c4605a..947fbbe226 100644 --- a/src/vnet/session/segment_manager.h +++ b/src/vnet/session/segment_manager.h @@ -148,6 +148,7 @@ int segment_manager_try_alloc_fifos (fifo_segment_t * fs, u32 rx_fifo_size, u32 tx_fifo_size, svm_fifo_t ** rx_fifo, svm_fifo_t ** tx_fifo); +void segment_manager_shrink_fifos (svm_fifo_t *rx_fifo, svm_fifo_t *tx_fifo); void segment_manager_dealloc_fifos (svm_fifo_t * rx_fifo, svm_fifo_t * tx_fifo); void segment_manager_detach_fifo (segment_manager_t *sm, svm_fifo_t **f); diff --git a/src/vnet/session/session.c b/src/vnet/session/session.c index 79681829e5..46aa1844b6 100644 --- a/src/vnet/session/session.c +++ b/src/vnet/session/session.c @@ -284,6 +284,12 @@ session_program_cleanup (session_t *s) session_cleanup_notify (s, SESSION_CLEANUP_SESSION); } +void +session_shrink_fifos (session_t *s) +{ + segment_manager_shrink_fifos (s->rx_fifo, s->tx_fifo); +} + /** * Cleans up session and lookup table. * diff --git a/src/vnet/session/session.h b/src/vnet/session/session.h index 78158d5f3e..2d1e6cf7f9 100644 --- a/src/vnet/session/session.h +++ b/src/vnet/session/session.h @@ -368,6 +368,7 @@ session_t *session_alloc (u32 thread_index); void session_free (session_t * s); void session_cleanup (session_t *s); void session_program_cleanup (session_t *s); +void session_shrink_fifos (session_t *s); void session_cleanup_half_open (session_handle_t ho_handle); u8 session_is_valid (u32 si, u8 thread_index); diff --git a/src/vnet/tcp/tcp_input.c b/src/vnet/tcp/tcp_input.c index c069a9b08d..e08437efbd 100644 --- a/src/vnet/tcp/tcp_input.c +++ b/src/vnet/tcp/tcp_input.c @@ -2252,6 +2252,11 @@ tcp46_rcv_process_inline (vlib_main_t *vm, vlib_node_runtime_t *node, tcp_timer_set (&wrk->timer_wheel, tc, TCP_TIMER_WAITCLOSE, tcp_cfg.timewait_time); session_transport_closed_notify (&tc->connection); + /* Shrink FIFOs */ + session_t *s = session_get_if_valid (tc->connection.s_index, + tc->connection.thread_index); + if (s) + session_shrink_fifos (s); goto drop; break; @@ -2389,6 +2394,11 @@ tcp46_rcv_process_inline (vlib_main_t *vm, vlib_node_runtime_t *node, tcp_cfg.timewait_time); tcp_program_ack (tc); session_transport_closed_notify (&tc->connection); + /* Shrink FIFOs */ + session_t *s = session_get_if_valid (tc->connection.s_index, + tc->connection.thread_index); + if (s) + session_shrink_fifos (s); break; case TCP_STATE_TIME_WAIT: /* Remain in the TIME-WAIT state. Restart the time-wait From 3cd38e86e5007350d8d4203cba0a6912e45b3fea Mon Sep 17 00:00:00 2001 From: Varun Rapelly Date: Thu, 8 Aug 2024 05:36:30 +0000 Subject: [PATCH 087/271] vppinfra: export utils for non vppinfra clib users Exports below functions for non src/vppinfra clib users: 1. clib_file_n_bytes 2. sprintf_clib_timebase_time Type: improvement JIRA: https://essjira.marvell.com/browse/IPBUSW-50810 Signed-off-by: Varun Rapelly Change-Id: Ia8356e15559290625b4ef0e22507d112abf9b900 Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/133076 Tested-by: sa_ip-sw-jenkins Reviewed-by: Venkata Ravichandra Mynidi Reviewed-by: Monendra Singh Kushwaha (cherry picked from commit 342e3cb50e02a02a6476e275ecc33c39fc4c331f) Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/134725 Reviewed-by: Nithinsen Kaithakadan --- src/vppinfra/time_range.c | 13 +++++++++++++ src/vppinfra/time_range.h | 2 ++ src/vppinfra/unix-misc.c | 4 ++-- 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/src/vppinfra/time_range.c b/src/vppinfra/time_range.c index 4b5e130376..076832b271 100644 --- a/src/vppinfra/time_range.c +++ b/src/vppinfra/time_range.c @@ -272,6 +272,19 @@ format_clib_timebase_time (u8 * s, va_list * args) return (s); } +__clib_export void +sprintf_clib_timebase_time (u8 *s, f64 now) +{ + clib_timebase_component_t _c, *cp = &_c; + + clib_timebase_time_to_components (now, cp); + + sprintf ((char *) s, "%s, %u %s %u %u:%02u:%02u", + day_names_epoch_order[cp->day_name_index], cp->day, + month_short_names[cp->month], cp->year, cp->hour, cp->minute, + cp->second); +} + uword unformat_clib_timebase_range_hms (unformat_input_t * input, va_list * args) { diff --git a/src/vppinfra/time_range.h b/src/vppinfra/time_range.h index 993bf9c360..7e95dcb398 100644 --- a/src/vppinfra/time_range.h +++ b/src/vppinfra/time_range.h @@ -68,6 +68,8 @@ unformat_function_t unformat_clib_timebase_range_hms; unformat_function_t unformat_clib_timebase_range_vector; format_function_t format_clib_timebase_time; +void sprintf_clib_timebase_time (u8 *s, f64 now); +#define CLIB_TIMEBASE_STR_MAX_SZ 60 static inline f64 clib_timebase_summer_offset_fastpath (clib_timebase_t * tb, f64 now) diff --git a/src/vppinfra/unix-misc.c b/src/vppinfra/unix-misc.c index 6a7328bd87..e158526ecc 100644 --- a/src/vppinfra/unix-misc.c +++ b/src/vppinfra/unix-misc.c @@ -50,8 +50,8 @@ __clib_export __thread uword __os_thread_index = 0; __clib_export __thread uword __os_numa_index = 0; -clib_error_t * -clib_file_n_bytes (char *file, uword * result) +__clib_export clib_error_t * +clib_file_n_bytes (char *file, uword *result) { struct stat s; From 2c41976e11d94fafb5f4c8334444cc3b5f34ac03 Mon Sep 17 00:00:00 2001 From: Monendra Singh Kushwaha Date: Wed, 4 Sep 2024 16:03:58 +0530 Subject: [PATCH 088/271] misc: update version Type: feature Signed-off-by: Monendra Singh Kushwaha --- MRVL_VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MRVL_VERSION b/MRVL_VERSION index aa5d9023dd..5480d0d067 100644 --- a/MRVL_VERSION +++ b/MRVL_VERSION @@ -1 +1 @@ -24.05.0 +24.09.0 From ca34e79bfa3374ff8e1ae42ab9df0ec7bf5bf385 Mon Sep 17 00:00:00 2001 From: Alok Mishra Date: Mon, 2 Sep 2024 18:28:21 +0500 Subject: [PATCH 089/271] octeon: enable h/w vlan tagging support This patch enables h/w vlan tagging by setting MAX_VTAG_INS field in NIX_AF_SMQ_CFG register. This is required to configure VLAN tag insertion by the hardware for egress packets Type: fix JIRA: https://essjira.marvell.com/browse/IPBUSW-52862 Signed-off-by: Alok Mishra Change-Id: Ifcdf9c1e5b0b8ddc27cabab597ae91780ceca095 Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/134899 Tested-by: sa_ip-sw-jenkins Reviewed-by: Nithinsen Kaithakadan --- src/plugins/dev_octeon/init.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/plugins/dev_octeon/init.c b/src/plugins/dev_octeon/init.c index b766eadf53..2acca6d2cf 100644 --- a/src/plugins/dev_octeon/init.c +++ b/src/plugins/dev_octeon/init.c @@ -114,6 +114,7 @@ oct_init_nix (vlib_main_t *vm, vnet_dev_t *dev) .reta_sz = ROC_NIX_RSS_RETA_SZ_256, .max_sqb_count = 512, .pci_dev = &cd->plt_pci_dev, + .hw_vlan_ins = true, }; if ((rrv = roc_nix_dev_init (cd->nix))) From 95f489fafe113bdbb2490ffa9acbdaf7ee398bd9 Mon Sep 17 00:00:00 2001 From: Nagendra T P Date: Thu, 5 Sep 2024 14:21:55 +0530 Subject: [PATCH 090/271] doc: bump dpdk version --- DPDK_VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DPDK_VERSION b/DPDK_VERSION index c6b9cb79cb..b4fb8b86ab 100644 --- a/DPDK_VERSION +++ b/DPDK_VERSION @@ -1,2 +1,2 @@ BASE_VERSION=23.11.0 -RELEASE_VERSION=24.07.0 +RELEASE_VERSION=24.08.0 From 1dc1d081fa71c2996a820c2867d159b2843e1901 Mon Sep 17 00:00:00 2001 From: Nagendra T P Date: Mon, 9 Sep 2024 19:20:51 +0530 Subject: [PATCH 091/271] ci: fix TAG in workflow MRVL_PKG_VERSION shall replace PKG_VERSION_NAME in TAG Signed-off-by: Nagendra T P --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 838938fae9..c1dcd973de 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -164,7 +164,7 @@ jobs: echo "DISTRO=${DISTRO}" >> "$GITHUB_OUTPUT" echo $PKG_POSTFIX echo "PKG_POSTFIX=${PKG_POSTFIX}" >> "$GITHUB_OUTPUT" - [[ "$PKG_POSTFIX" == "-devel" ]] && TAG=devel || TAG=${PKG_VERSION_NAME} + [[ "$PKG_POSTFIX" == "-devel" ]] && TAG=devel || TAG=${MRVL_PKG_VERSION} echo "TAG=${TAG}" >> "$GITHUB_OUTPUT" - name: Upload debian package as artifact uses: actions/upload-artifact@v4.3.1 From c39c2a998b75068d69bd15eb23eb4f2c6d161fe6 Mon Sep 17 00:00:00 2001 From: Pavan Nikhilesh Date: Wed, 11 Sep 2024 01:45:38 +0530 Subject: [PATCH 092/271] ci: update workflow to include platform name Update platform name in release tag. Signed-off-by: Pavan Nikhilesh --- .github/workflows/build.yml | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c1dcd973de..e09bdd71e9 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -165,7 +165,9 @@ jobs: echo $PKG_POSTFIX echo "PKG_POSTFIX=${PKG_POSTFIX}" >> "$GITHUB_OUTPUT" [[ "$PKG_POSTFIX" == "-devel" ]] && TAG=devel || TAG=${MRVL_PKG_VERSION} + [[ "$PKG_POSTFIX" == "-devel" ]] && IS_DEVEL="true" || IS_DEVEL="false" echo "TAG=${TAG}" >> "$GITHUB_OUTPUT" + echo "IS_DEVEL=${IS_DEVEL}" >> "$GITHUB_OUTPUT" - name: Upload debian package as artifact uses: actions/upload-artifact@v4.3.1 if: ${{ github.event_name == 'push' }} @@ -177,8 +179,8 @@ jobs: env: GH_TOKEN: ${{ github.token }} run: | - if gh release view vpp-${{ steps.artifacts.outputs.PKG_VERSION_NAME }}-${{ steps.artifacts.outputs.MRVL_PKG_VERSION }}-${{ steps.artifacts.outputs.DISTRO }}-${{ steps.artifacts.outputs.TAG }}; then - gh release delete vpp-${{ steps.artifacts.outputs.PKG_VERSION_NAME }}-${{ steps.artifacts.outputs.MRVL_PKG_VERSION }}-${{ steps.artifacts.outputs.DISTRO }}-${{ steps.artifacts.outputs.TAG }} --cleanup-tag -y + if gh release view vpp-${{ steps.artifacts.outputs.PKG_VERSION_NAME }}-cn10k-${{ steps.artifacts.outputs.MRVL_PKG_VERSION }}-${{ steps.artifacts.outputs.DISTRO }}-${{ steps.artifacts.outputs.TAG }}; then + gh release delete vpp-${{ steps.artifacts.outputs.PKG_VERSION_NAME }}-cn10k-${{ steps.artifacts.outputs.MRVL_PKG_VERSION }}-${{ steps.artifacts.outputs.DISTRO }}-${{ steps.artifacts.outputs.TAG }} --cleanup-tag -y else echo "Release not found" fi @@ -186,7 +188,8 @@ jobs: uses: softprops/action-gh-release@v2.0.4 if: ${{ github.event_name == 'push' }} with: - tag_name: vpp-${{ steps.artifacts.outputs.PKG_VERSION_NAME }}-${{ steps.artifacts.outputs.MRVL_PKG_VERSION }}-${{ steps.artifacts.outputs.DISTRO }}-${{ steps.artifacts.outputs.TAG }} + draft: false + tag_name: vpp-${{ steps.artifacts.outputs.PKG_VERSION_NAME }}-cn10k-${{ steps.artifacts.outputs.MRVL_PKG_VERSION }}-${{ steps.artifacts.outputs.DISTRO }}-${{ steps.artifacts.outputs.TAG }} files: | ${{ github.workspace }}/artifacts/vpp-${{ steps.artifacts.outputs.PKG_VERSION_NAME }}-cn10k${{ steps.artifacts.outputs.PKG_POSTFIX }}_${{ steps.artifacts.outputs.MRVL_PKG_VERSION }}_arm64.deb - name: Dispatch package update event @@ -200,5 +203,7 @@ jobs: https://api.github.com/repos/marvellembeddedprocessors/packages/dispatches \ -d '{"event_type":"dispatch-event", "client_payload": {"package" : "vpp", "tag": "vpp-${{ steps.artifacts.outputs.PKG_VERSION_NAME }}-${{ steps.artifacts.outputs.MRVL_PKG_VERSION }}-${{ steps.artifacts.outputs.DISTRO }}-${{ steps.artifacts.outputs.TAG }}", - "dpdk_tag" : "dpdk-${{ steps.version.outputs.DPDK_BASE_PKG_VERSION }}_${{ steps.version.outputs.DPDK_PKG_VERSION }}-${{ steps.artifacts.outputs.DISTRO }}-${{ steps.version.outputs.DPDK_PKG_VERSION }}", - "has_dpdk" : "true", "distro" : "${{ steps.artifacts.outputs.DISTRO }}"}}' \ No newline at end of file + "dpdk_tag" : "dpdk-cn10k-${{ steps.version.outputs.DPDK_BASE_PKG_VERSION }}_${{ steps.version.outputs.DPDK_PKG_VERSION }}-${{ steps.artifacts.outputs.DISTRO }}-${{ steps.version.outputs.DPDK_PKG_VERSION }}", + "has_dpdk" : "true", "distro" : "${{ steps.artifacts.outputs.DISTRO }}", + "platform" : "cn10k", + "devel": "${{ steps.artifacts.outputs.IS_DEVEL }}"}}' From 4116e182a603b21808ef45b7684ce2791b881ca0 Mon Sep 17 00:00:00 2001 From: Mohamed Feroz Date: Thu, 29 Aug 2024 08:04:02 +0000 Subject: [PATCH 093/271] vcl: fix vcl issue in multi-thread-workers mode In some multi-process/thread applications, a connection FD is created on main process and actual data processing is done in worker process. The main process does nothing. For such applications, when multi-thread-workers is enabled, in current VLS code, the session events are still enqueued to the VCL worker corresponding to Main process. As the main process is not dequeuing any event, application doesn't move forward. This patch fixes this issue by enabling VCL worker corresponding to the Worker process as listener of the session. Type: improvement Signed-off-by: Mohamed Feroz Change-Id: Ia7e6270c1acbce7feeafbf281c661285d63c3b22 --- src/vcl/vcl_locked.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/vcl/vcl_locked.c b/src/vcl/vcl_locked.c index 412db8def3..2a15f6cedf 100644 --- a/src/vcl/vcl_locked.c +++ b/src/vcl/vcl_locked.c @@ -374,6 +374,12 @@ vls_worker_get_current (void) return pool_elt_at_index (vlsm->workers, vls_get_worker_index ()); } +static inline u8 +vls_n_workers (void) +{ + return pool_elts (vlsm->workers); +} + static void vls_worker_alloc (void) { @@ -1300,7 +1306,7 @@ vls_mp_checks (vcl_locked_session_t * vls, int is_add) vcl_session_t *s; u32 owner_wrk; - if (vls_mt_wrk_supported ()) + if (vls_mt_wrk_supported () && vls_n_workers () <= 1) return; ASSERT (wrk->wrk_index == vls->vcl_wrk_index); From 886f224efc8b63814ee5ac4fd88c599c45768967 Mon Sep 17 00:00:00 2001 From: Garvit Date: Mon, 26 Aug 2024 12:13:10 +0530 Subject: [PATCH 094/271] ipsec: remove redundant code in ipsec_tun_in trace This patch removes redundant sa index value in format_ipsec_tun_protect_input_trace() as sa index value already added by format_ipsec4_tunnel_kv(). Type: fix JIRA: https://essjira.marvell.com/browse/IPBUSW-47134 Change-Id: I7834bda57ba0f7cfe5084b671f6f7da0749ba64d Signed-off-by: Garvit Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/134112 Tested-by: sa_ip-sw-jenkins Reviewed-by: Monendra Singh Kushwaha (cherry picked from commit 41e185704d75168dddad5fef33a3e9b72cb7b4e5) --- src/vnet/ipsec/ipsec_tun_in.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/vnet/ipsec/ipsec_tun_in.c b/src/vnet/ipsec/ipsec_tun_in.c index a419d8c4fe..1f8f914275 100644 --- a/src/vnet/ipsec/ipsec_tun_in.c +++ b/src/vnet/ipsec/ipsec_tun_in.c @@ -60,8 +60,8 @@ format_ipsec_tun_protect_input_trace (u8 * s, va_list * args) s = format (s, "IPSec: %U seq %u", format_ipsec6_tunnel_kv, &t->kv6, t->seq); else - s = format (s, "IPSec: %U seq %u sa %d", - format_ipsec4_tunnel_kv, &t->kv4, t->seq); + s = + format (s, "IPSec: %U seq %u", format_ipsec4_tunnel_kv, &t->kv4, t->seq); return s; } From 8413bd71f4ef9a7b37a628e2551e03326a21d0e1 Mon Sep 17 00:00:00 2001 From: Monendra Singh Kushwaha Date: Fri, 13 Sep 2024 16:10:37 +0530 Subject: [PATCH 095/271] octeon: fix compilation on octeon9 Type: fix Signed-off-by: Monendra Singh Kushwaha Change-Id: I9ffa78122dcd6b98cad9902e43ba8432fcd572b1 Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/135542 Tested-by: sa_ip-sw-jenkins (cherry picked from commit 38226183ca104358ca3827c37325330df051b178) --- src/plugins/dev_octeon/tx_node.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/plugins/dev_octeon/tx_node.c b/src/plugins/dev_octeon/tx_node.c index 6d239836a2..f42f18d989 100644 --- a/src/plugins/dev_octeon/tx_node.c +++ b/src/plugins/dev_octeon/tx_node.c @@ -300,7 +300,8 @@ oct_tx_enq16 (vlib_main_t *vm, oct_tx_ctx_t *ctx, vnet_dev_tx_queue_t *txq, { u8 dwords_per_line[16], *dpl = dwords_per_line; u64 __attribute__ ((unused)) lmt_arg, ioaddr, n_lines; - u32 n_left, or_flags_16 = 0, n = 0; + u32 __attribute__ ((unused)) or_flags_16 = 0; + u32 n_left, n = 0; const u32 not_simple_flags = VLIB_BUFFER_NEXT_PRESENT | VNET_BUFFER_F_OFFLOAD; lmt_line_t *l = ctx->lmt_lines; From 7e565a0c6bed37b945632d8a42c5c8aa8dc536a9 Mon Sep 17 00:00:00 2001 From: Nagendra T P Date: Fri, 20 Sep 2024 11:12:48 +0530 Subject: [PATCH 096/271] workflow for cn9k platform --- .github/workflows/build-cn9k.yml | 208 +++++++++++++++++++++++++++++++ 1 file changed, 208 insertions(+) create mode 100644 .github/workflows/build-cn9k.yml diff --git a/.github/workflows/build-cn9k.yml b/.github/workflows/build-cn9k.yml new file mode 100644 index 0000000000..2bf0291e25 --- /dev/null +++ b/.github/workflows/build-cn9k.yml @@ -0,0 +1,208 @@ +name: build-cn9k + +on: + push: + schedule: + - cron: "0 0 * * *" + pull_request: + +permissions: + contents: write + pages: write + id-token: write + packages: write + +jobs: + ubuntu-cn9k-build: + name: ubuntu-cn9k-arm64 + runs-on: ubuntu-latest + + strategy: + fail-fast: true + matrix: + include: + - arch: aarch64 + distro: ubuntu22.04 + compiler: gcc + library: static + + steps: + - name: Checkout sources + uses: actions/checkout@v4.1.6 + with: + fetch-depth: 0 + fetch-tags: true + - name: Generate cache keys + id: get_ref_keys + run: | + echo 'ccache=ccache-${{ matrix.distro }}-${{ matrix.compiler }}-${{ matrix.arch }}-'$(date -u +%Y-w%W) >> $GITHUB_OUTPUT + - name: Retrieve ccache cache + uses: actions/cache@v4 + with: + path: ~/.ccache + key: ${{ steps.get_ref_keys.outputs.ccache }}-${{ github.ref }} + restore-keys: | + ${{ steps.get_ref_keys.outputs.ccache }}-refs/heads/main + - name: Extract version details + id: version + run: | + mkdir -p "${PWD}/artifacts" + git tag --points-at HEAD > /tmp/tags + [ -s /tmp/tags ] && PKG_POSTFIX= || PKG_POSTFIX=-devel + FW_PKG_POSTFIX="" + if [ ${PKG_POSTFIX} == "-devel" ]; then + FW_PKG_POSTFIX="" + else + FW_PKG_POSTFIX=$PKG_POSTFIX + fi + echo "FW_PKG_POSTFIX=${FW_PKG_POSTFIX}" >> "${PWD}/artifacts/env" + [ -s /tmp/tags ] && NIGHTLY=false || NIGHTLY=true + echo "PKG_VERSION_NAME=`./src/scripts/version | awk -F '-' '{print $1}'`" >> "${PWD}/artifacts/env" + echo "MRVL_PKG_VERSION=`cat MRVL_VERSION`" >> "${PWD}/artifacts/env" + echo "DPDK_PKG_VERSION=`cat DPDK_VERSION | grep RELEASE_VERSION | awk -F'=' '{print $2}'`" >> "${PWD}/artifacts/env" + echo "DPDK_BASE_PKG_VERSION=`cat DPDK_VERSION | grep BASE_VERSION | awk -F'=' '{print $2}' | awk -F'.' '{print $1"."$2}'`" >> "${PWD}/artifacts/env" + echo "PKG_POSTFIX=${PKG_POSTFIX}" >> "${PWD}/artifacts/env" + source "${PWD}/artifacts/env" + echo "NIGHTLY=${NIGHTLY}" >> $GITHUB_OUTPUT + echo "DPDK_PKG_VERSION=${DPDK_PKG_VERSION}" >> $GITHUB_OUTPUT + echo "DPDK_BASE_PKG_VERSION=${DPDK_BASE_PKG_VERSION}" >> $GITHUB_OUTPUT + - uses: uraimo/run-on-arch-action@v2.7.2 + name: Build VPP and generate package + id: build + with: + arch: ${{ matrix.arch }} + distro: ${{ matrix.distro }} + githubToken: ${{ github.token }} + setup: | + mkdir -p ~/.ccache + dockerRunArgs: | + --volume "${PWD}/artifacts:/artifacts" + --volume "${HOME}/.ccache:/root/.ccache" + shell: /bin/bash + install: | + apt-get update -q -y + apt-get install -y apt-utils sudo make dialog ccache git + apt-get install -y aspell aspell-en autopoint autotools-dev binfmt-support binutils binutils-aarch64-linux-gnu + apt-get install -y binutils-common bsdextrautils bzip2 ca-certificates clang-14 cmake-data cpp cpp-11 cpp-12 dbus + apt-get install -y dctrl-tools debugedit dh-autoreconf dh-elpa-helper dh-strip-nondeterminism dictionaries-common dirmngr + apt-get install -y distro-info-data dpkg-dev dwz emacsen-common fakeroot file fontconfig-config fonts-dejavu-core g++ g++-11 + apt-get install -y gcc gcc-11 gcc-11-base gcc-12 gettext gettext-base git-man gnupg gnupg-l10n gnupg-utils gpg gpg-agent + apt-get install -y gpg-wks-client gpg-wks-server gpgconf gpgsm groff-base hunspell-en-us icu-devtools intltool-debian + apt-get install -y javascript-common kmod less libalgorithm-diff-perl libalgorithm-diff-xs-perl libalgorithm-merge-perl + apt-get install -y libapparmor1 libapr1 libarchive-cpio-perl libarchive-zip-perl libarchive13 libasan6 libasan8 libaspell15 + apt-get install -y libassuan0 libatomic1 libbcg729-0 libbinutils libbrotli1 libbsd0 libc-ares2 libc-dev-bin libc-devtools + apt-get install -y libc6-dev libcap2-bin libcbor0.8 libcc1-0 libclang-common-14-dev libclang-cpp11 libclang-cpp14 + apt-get install -y libclang1-14 libcommon-sense-perl libconfuse-common libconfuse-doc libconfuse2 libcrypt-dev libctf-nobfd0 + apt-get install -y libctf0 libcurl3-gnutls libcurl4 libdbus-1-3 libdbus-1-dev libdebhelper-perl libdeflate0 libdpkg-perl + apt-get install -y libdw1 libedit2 libelf1 libenchant-2-2 liberror-perl libexpat1 libexpat1-dev libfakeroot libfido2-1 + apt-get install -y libfile-fcntllock-perl libfile-stripnondeterminism-perl libfontconfig1 libfreetype6 libgc1 libgcc-11-dev + apt-get install -y libgcc-12-dev libgd-perl libgd3 libgdbm-compat4 libgdbm6 libglib2.0-0 libglib2.0-data libgomp1 + apt-get install -y libhiredis0.14 libhunspell-1.7-0 libhwasan0 libicu-dev libicu70 libiperf0 libisl23 libitm1 libjbig0 + apt-get install -y libjpeg-turbo8 libjpeg8 libjs-jquery libjs-sphinxdoc libjs-underscore libjson-perl libjson-xs-perl + apt-get install -y libjsoncpp25 libkmod2 libksba8 libldap-2.5-0 libldap-common libllvm11 libllvm14 liblocale-gettext-perl + apt-get install -y liblsan0 libltdl-dev libltdl7 liblua5.2-0 libmagic-mgc libmagic1 libmail-sendmail-perl libmaxminddb0 + apt-get install -y libmd0 libmnl0 libmpc3 libmpdec3 libmpfr6 libncurses-dev libnghttp2-14 libnl-3-200 libnl-genl-3-200 + apt-get install -y libnl-route-3-200 libnpth0 libnsl-dev libnuma1 libobjc-11-dev libobjc4 libpam-cap libpcap0.8 + apt-get install -y libpcap0.8-dev libperl5.34 libperlio-gzip-perl libpfm4 libpipeline1 libpng16-16 libpsl5 libpython3-dev + apt-get install -y libpython3-stdlib libpython3.10 libpython3.10-dev libpython3.10-minimal libpython3.10-stdlib libreadline8 + apt-get install -y librhash0 librtmp1 libsasl2-2 libsasl2-modules libsasl2-modules-db libsbc1 libsctp-dev libsctp1 + apt-get install -y libsigsegv2 libsmi2ldbl libsnappy1v5 libspandsp2 libspeexdsp1 libsqlite3-0 libssh-4 libssh-gcrypt-4 + apt-get install -y libstdc++-11-dev libsub-override-perl libsubunit-dev libsubunit0 libsys-hostname-long-perl + apt-get install -y libtext-iconv-perl libtiff5 libtinfo-dev libtirpc-dev libtsan0 libtsan2 libtypes-serialiser-perl + apt-get install -y libubsan1 libuchardet0 libuv1 libwebp7 libwireshark-data libwireshark15 libwiretap12 libwsutil13 libx11-6 + apt-get install -y libx11-data libxau6 libxcb1 libxdmcp6 libxext6 libxml2 libxml2-dev libxmuu1 libxpm4 libxslt1.1 + apt-get install -y libyaml-0-2 libz3-4 libz3-dev linux-headers-5.15.0-107 linux-headers-5.15.0-107-generic + apt-get install -y linux-headers-generic linux-libc-dev llvm-14 llvm-14-dev llvm-14-linker-tools llvm-14-runtime + apt-get install -y llvm-14-tools lsb-release lto-disabled-list m4 man-db manpages manpages-dev media-types netbase + apt-get install -y openssh-client openssl patch perl perl-modules-5.34 pinentry-curses po-debconf publicsuffix + apt-get install -y python-babel-localedata python3 python3-attr python3-babel python3-bs4 python3-certifi python3-chardet + apt-get install -y python3-distlib python3-distutils python3-filelock python3-html5lib python3-idna + apt-get install -y python3-importlib-metadata python3-jinja2 python3-lib2to3 python3-lxml python3-markupsafe python3-minimal + apt-get install -y python3-more-itertools python3-pip-whl python3-pkg-resources python3-platformdirs python3-pygments + apt-get install -y python3-pyrsistent python3-requests python3-setuptools-whl python3-six python3-soupsieve python3-tz + apt-get install -y python3-urllib3 python3-webencodings python3-wheel python3-wheel-whl python3-zipp python3.10 + apt-get install -y python3.10-dev python3.10-minimal python3.10-venv readline-common rpcsvc-proto shared-mime-info tzdata + apt-get install -y ucf uuid-dev wireshark-common xauth xdg-user-dirs xz-utils zlib1g-dev + apt-get install -y check chrpath clang clang-format-11 cmake cscope debhelper dh-python dkms + apt-get install -y ethtool exuberant-ctags gcovr git-review iperf iperf3 lcov libapr1-dev + apt-get install -y libconfuse-dev libelf-dev libffi7 libmnl-dev libnl-3-dev libnl-route-3-dev + apt-get install -y libnuma-dev libpcap-dev libssl-dev nasm ninja-build python3-all python3-dev + apt-get install -y python3-jsonschema python3-pip python3-ply python3-setuptools python3-venv + apt-get install -y python3-virtualenv tshark lsb-release + run: | + source /artifacts/env + DISTRO=ubuntu-`lsb_release -rs` + echo "DISTRO=${DISTRO}" >> /artifacts/env + echo "cache_dir = /root/.ccache" > /root/.ccache/ccache.conf + ccache -p + git config --global --add safe.directory "${PWD}" + APT_ARGS='-y -q' make install-deps + make build-release VPP_PLATFORM=octeon9 + mkdir -p "${PWD}/install/DEBIAN" + mv build-root/install-vpp-native/vpp/* install/. + cd "${PWD}/install" + echo 'Package: vpp-'$PKG_VERSION_NAME'-cn9k'$PKG_POSTFIX >> DEBIAN/control + echo 'Version: '$MRVL_PKG_VERSION >> DEBIAN/control + echo "Maintainer: Jerin Jacob (jerinj@marvell.com)" >> DEBIAN/control + echo 'Depends: python3, python3-ply, dpdk-'$DPDK_BASE_PKG_VERSION'-cn9k (= '$DPDK_PKG_VERSION'), cpt-firmware-cn9k'${FW_PKG_POSTFIX}' (= '$MRVL_PKG_VERSION')' >> DEBIAN/control + echo "Architecture: arm64" >> DEBIAN/control + echo "Homepage: https://wiki.fd.io/view/VPP" >> DEBIAN/control + echo "Description: Vector Packet Processing (VPP) for Octeon9" >> DEBIAN/control + cd - + mv "${PWD}/install" "${PWD}/vpp-${PKG_VERSION_NAME}-cn9k${PKG_POSTFIX}_${MRVL_PKG_VERSION}_arm64" + dpkg --build "vpp-${PKG_VERSION_NAME}-cn9k${PKG_POSTFIX}_${MRVL_PKG_VERSION}_arm64" + cp "vpp-${PKG_VERSION_NAME}-cn9k${PKG_POSTFIX}_${MRVL_PKG_VERSION}_arm64.deb" /artifacts/. + - name: Export version name + id: artifacts + run: | + source "${PWD}/artifacts/env" + echo $PKG_VERSION_NAME + echo "PKG_VERSION_NAME=${PKG_VERSION_NAME}" >> "$GITHUB_OUTPUT" + echo $MRVL_PKG_VERSION + echo "MRVL_PKG_VERSION=${MRVL_PKG_VERSION}" >> "$GITHUB_OUTPUT" + echo $DISTRO + echo "DISTRO=${DISTRO}" >> "$GITHUB_OUTPUT" + echo $PKG_POSTFIX + echo "PKG_POSTFIX=${PKG_POSTFIX}" >> "$GITHUB_OUTPUT" + [[ "$PKG_POSTFIX" == "-devel" ]] && TAG=devel || TAG=${MRVL_PKG_VERSION} + [[ "$PKG_POSTFIX" == "-devel" ]] && IS_DEVEL="true" || IS_DEVEL="false" + echo "TAG=${TAG}" >> "$GITHUB_OUTPUT" + echo "IS_DEVEL=${IS_DEVEL}" >> "$GITHUB_OUTPUT" + - name: Upload debian package as artifact + uses: actions/upload-artifact@v4.3.1 + if: ${{ github.event_name == 'push' }} + with: + name: vpp-${{ steps.artifacts.outputs.PKG_VERSION_NAME }}-cn9k${{ steps.artifacts.outputs.PKG_POSTFIX }}_${{ steps.artifacts.outputs.MRVL_PKG_VERSION }}_arm64.deb + path: ${{ github.workspace }}/artifacts/vpp-${{ steps.artifacts.outputs.PKG_VERSION_NAME }}-cn9k${{ steps.artifacts.outputs.PKG_POSTFIX }}_${{ steps.artifacts.outputs.MRVL_PKG_VERSION }}_arm64.deb + - name: Delete existing release + if: ${{ github.event_name == 'push' }} + env: + GH_TOKEN: ${{ github.token }} + run: | + if gh release view vpp-${{ steps.artifacts.outputs.PKG_VERSION_NAME }}-cn9k-${{ steps.artifacts.outputs.MRVL_PKG_VERSION }}-${{ steps.artifacts.outputs.DISTRO }}-${{ steps.artifacts.outputs.TAG }}; then + gh release delete vpp-${{ steps.artifacts.outputs.PKG_VERSION_NAME }}-cn9k-${{ steps.artifacts.outputs.MRVL_PKG_VERSION }}-${{ steps.artifacts.outputs.DISTRO }}-${{ steps.artifacts.outputs.TAG }} --cleanup-tag -y + else + echo "Release not found" + fi + - name: Release VPP cn9k package + uses: softprops/action-gh-release@v2.0.4 + if: ${{ github.event_name == 'push' }} + with: + tag_name: vpp-${{ steps.artifacts.outputs.PKG_VERSION_NAME }}-cn9k-${{ steps.artifacts.outputs.MRVL_PKG_VERSION }}-${{ steps.artifacts.outputs.DISTRO }}-${{ steps.artifacts.outputs.TAG }} + files: | + ${{ github.workspace }}/artifacts/vpp-${{ steps.artifacts.outputs.PKG_VERSION_NAME }}-cn9k${{ steps.artifacts.outputs.PKG_POSTFIX }}_${{ steps.artifacts.outputs.MRVL_PKG_VERSION }}_arm64.deb + - name: Dispatch package update event + if: ${{ github.event_name == 'push' }} + run: | + curl -L \ + -X POST \ + -H "Accept: application/vnd.github+json" \ + -H "Authorization: Bearer ${{ secrets.PPA_REPO_SECRET }}" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + https://api.github.com/repos/marvellembeddedprocessors/packages/dispatches \ + -d '{"event_type":"dispatch-event", "client_payload": {"package" : "vpp", + "tag": "vpp-${{ steps.artifacts.outputs.PKG_VERSION_NAME }}-${{ steps.artifacts.outputs.MRVL_PKG_VERSION }}-${{ steps.artifacts.outputs.DISTRO }}-${{ steps.artifacts.outputs.TAG }}", + "dpdk_tag" : "dpdk-cn9k-${{ steps.version.outputs.DPDK_BASE_PKG_VERSION }}_${{ steps.version.outputs.DPDK_PKG_VERSION }}-${{ steps.artifacts.outputs.DISTRO }}-${{ steps.version.outputs.DPDK_PKG_VERSION }}", + "has_dpdk" : "true", "distro" : "${{ steps.artifacts.outputs.DISTRO }}", + "platform" : "cn9k", + "devel": "${{ steps.artifacts.outputs.IS_DEVEL }}"}}' \ No newline at end of file From a58c3bb5bcb2e401a8347891a57ef3979fdc295b Mon Sep 17 00:00:00 2001 From: Nagendra T P Date: Mon, 9 Dec 2024 10:22:26 +0530 Subject: [PATCH 097/271] ci: update docker plugin version Update docker plugin version. Signed-off-by: Nagendra T P --- .github/workflows/build-cn9k.yml | 4 ++-- .github/workflows/build.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-cn9k.yml b/.github/workflows/build-cn9k.yml index 2bf0291e25..40b636250a 100644 --- a/.github/workflows/build-cn9k.yml +++ b/.github/workflows/build-cn9k.yml @@ -66,7 +66,7 @@ jobs: echo "NIGHTLY=${NIGHTLY}" >> $GITHUB_OUTPUT echo "DPDK_PKG_VERSION=${DPDK_PKG_VERSION}" >> $GITHUB_OUTPUT echo "DPDK_BASE_PKG_VERSION=${DPDK_BASE_PKG_VERSION}" >> $GITHUB_OUTPUT - - uses: uraimo/run-on-arch-action@v2.7.2 + - uses: uraimo/run-on-arch-action@v2.8.1 name: Build VPP and generate package id: build with: @@ -205,4 +205,4 @@ jobs: "dpdk_tag" : "dpdk-cn9k-${{ steps.version.outputs.DPDK_BASE_PKG_VERSION }}_${{ steps.version.outputs.DPDK_PKG_VERSION }}-${{ steps.artifacts.outputs.DISTRO }}-${{ steps.version.outputs.DPDK_PKG_VERSION }}", "has_dpdk" : "true", "distro" : "${{ steps.artifacts.outputs.DISTRO }}", "platform" : "cn9k", - "devel": "${{ steps.artifacts.outputs.IS_DEVEL }}"}}' \ No newline at end of file + "devel": "${{ steps.artifacts.outputs.IS_DEVEL }}"}}' diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e09bdd71e9..d603f2f380 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -66,7 +66,7 @@ jobs: echo "NIGHTLY=${NIGHTLY}" >> $GITHUB_OUTPUT echo "DPDK_PKG_VERSION=${DPDK_PKG_VERSION}" >> $GITHUB_OUTPUT echo "DPDK_BASE_PKG_VERSION=${DPDK_BASE_PKG_VERSION}" >> $GITHUB_OUTPUT - - uses: uraimo/run-on-arch-action@v2.7.2 + - uses: uraimo/run-on-arch-action@v2.8.1 name: Build VPP and generate package id: build with: From 791eed47f3492250760195bd09070896adedee41 Mon Sep 17 00:00:00 2001 From: Monendra Singh Kushwaha Date: Wed, 18 Sep 2024 15:54:28 +0530 Subject: [PATCH 098/271] octeon: update crypto counter cli This patch disables crypto counters if no crypto device attached to dev-octeon plugin. Type: fix Change-Id: I762db9f6fe290a378760e4c64f0066b7aca838b6 Signed-off-by: Monendra Singh Kushwaha Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/135795 Tested-by: sa_ip-sw-jenkins Reviewed-by: Nithinsen Kaithakadan (cherry picked from commit 3948543dc4c43fb2fef510db2fd20756ea7abc25) Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/136335 --- src/plugins/dev_octeon/cli.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/plugins/dev_octeon/cli.c b/src/plugins/dev_octeon/cli.c index bd88849c08..b5ed7e8313 100644 --- a/src/plugins/dev_octeon/cli.c +++ b/src/plugins/dev_octeon/cli.c @@ -125,6 +125,10 @@ oct_crypto_counters_command_fn (vlib_main_t *vm, unformat_input_t *input, counter_t *counters = NULL; u32 n_threads = vlib_get_n_threads (); + if (!ocm->n_cpt) + return clib_error_create ( + "No Crypto device attached to dev-octeon plugin"); + #define _(i, s, d) \ cm = &ocm->s##_counter; \ vec_validate_init_empty (stat[i], n_threads, 0); \ @@ -177,6 +181,10 @@ oct_crypto_counters_clear_command_fn (vlib_main_t *vm, unformat_input_t *input, vlib_simple_counter_main_t *cm; oct_crypto_main_t *ocm = &oct_crypto_main; + if (!ocm->n_cpt) + return clib_error_create ( + "No Crypto device attached to dev-octeon plugin"); + #define _(i, s, d) \ cm = &ocm->s##_counter; \ vlib_clear_simple_counters (cm); From 3a1550d1c392cc71f805184bedac00a21b3b2d91 Mon Sep 17 00:00:00 2001 From: Monendra Singh Kushwaha Date: Mon, 23 Sep 2024 15:10:10 +0530 Subject: [PATCH 099/271] octeon: fix SDP device link information This patch fixes SDP (System DPI Packet Interface Unit) device link information. Type: fix Change-Id: I4563094601d9bb24132e4dc712cde14daa7f4364 Signed-off-by: Monendra Singh Kushwaha Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/136124 Tested-by: sa_ip-sw-jenkins (cherry picked from commit a6663b51da784b174aecf5ee47a2ed60dbc24ce6) Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/136336 Reviewed-by: Nithinsen Kaithakadan --- src/plugins/dev_octeon/port.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plugins/dev_octeon/port.c b/src/plugins/dev_octeon/port.c index bc24b6b02d..f619116f73 100644 --- a/src/plugins/dev_octeon/port.c +++ b/src/plugins/dev_octeon/port.c @@ -212,7 +212,7 @@ oct_port_poll (vlib_main_t *vm, vnet_dev_port_t *port) return; } - if (roc_nix_is_lbk (nix)) + if (roc_nix_is_lbk (nix) || roc_nix_is_sdp (nix)) { link_info.status = 1; link_info.full_duplex = 1; From a20021abf4c0789d26f0c0949011733620d53031 Mon Sep 17 00:00:00 2001 From: Alok Mishra Date: Fri, 9 Aug 2024 13:29:24 +0500 Subject: [PATCH 100/271] octeon: enable ethernet pause frame support This patch adds support for MAC pause flow control. By default, pause flow control is enabled in the device configuration. Type: feature JIRA: https://essjira.marvell.com/browse/IPBUSW-52629 Signed-off-by: Alok Mishra Change-Id: I0f448479a38fae615d87af7e736c6053ada89cca Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/133468 Reviewed-by: Sunil Kumar Kori Tested-by: sa_ip-sw-jenkins (cherry picked from commit 2ecaee43502adc76ab867b0a8854b6002caaa2c2) Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/136337 Reviewed-by: Nithinsen Kaithakadan --- src/plugins/dev_octeon/port.c | 73 +++++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) diff --git a/src/plugins/dev_octeon/port.c b/src/plugins/dev_octeon/port.c index f619116f73..84cc06fd52 100644 --- a/src/plugins/dev_octeon/port.c +++ b/src/plugins/dev_octeon/port.c @@ -53,6 +53,72 @@ oct_roc_err (vnet_dev_t *dev, int rv, char *fmt, ...) return VNET_DEV_ERR_INTERNAL; } +vnet_dev_rv_t +oct_port_pause_flow_control_init (vlib_main_t *vm, vnet_dev_port_t *port) +{ + vnet_dev_t *dev = port->dev; + oct_device_t *cd = vnet_dev_get_data (dev); + struct roc_nix *nix = cd->nix; + struct roc_nix_fc_cfg fc_cfg; + struct roc_nix_sq *sq; + struct roc_nix_cq *cq; + struct roc_nix_rq *rq; + int rrv; + + if (roc_nix_is_sdp (nix) || roc_nix_is_lbk (nix)) + return VNET_DEV_ERR_UNSUPPORTED_DEVICE; + + fc_cfg.type = ROC_NIX_FC_RXCHAN_CFG; + fc_cfg.rxchan_cfg.enable = true; + rrv = roc_nix_fc_config_set (nix, &fc_cfg); + if (rrv) + return oct_roc_err (dev, rrv, "roc_nix_fc_config_set failed"); + + memset (&fc_cfg, 0, sizeof (struct roc_nix_fc_cfg)); + fc_cfg.type = ROC_NIX_FC_RQ_CFG; + fc_cfg.rq_cfg.enable = true; + fc_cfg.rq_cfg.tc = 0; + + foreach_vnet_dev_port_rx_queue (rxq, port) + { + oct_rxq_t *crq = vnet_dev_get_rx_queue_data (rxq); + + rq = &crq->rq; + cq = &crq->cq; + + fc_cfg.rq_cfg.rq = rq->qid; + fc_cfg.rq_cfg.cq_drop = cq->drop_thresh; + + rrv = roc_nix_fc_config_set (nix, &fc_cfg); + if (rrv) + return oct_roc_err (dev, rrv, "roc_nix_fc_config_set failed"); + } + + memset (&fc_cfg, 0, sizeof (struct roc_nix_fc_cfg)); + fc_cfg.type = ROC_NIX_FC_TM_CFG; + fc_cfg.tm_cfg.tc = 0; + fc_cfg.tm_cfg.enable = true; + + foreach_vnet_dev_port_tx_queue (txq, port) + { + oct_txq_t *ctq = vnet_dev_get_tx_queue_data (txq); + + sq = &ctq->sq; + + fc_cfg.tm_cfg.sq = sq->qid; + rrv = roc_nix_fc_config_set (nix, &fc_cfg); + if (rrv) + return oct_roc_err (dev, rrv, "roc_nix_fc_config_set failed"); + } + + /* By default, enable pause frame flow control */ + rrv = roc_nix_fc_mode_set (nix, ROC_NIX_FC_FULL); + if (rrv) + return oct_roc_err (dev, rrv, "roc_nix_fc_mode_set failed"); + + return VNET_DEV_OK; +} + vnet_dev_rv_t oct_port_init (vlib_main_t *vm, vnet_dev_port_t *port) { @@ -147,6 +213,13 @@ oct_port_init (vlib_main_t *vm, vnet_dev_port_t *port) return rv; } + /* Configure pause frame flow control*/ + if ((rv = oct_port_pause_flow_control_init (vm, port))) + { + oct_port_deinit (vm, port); + return rv; + } + oct_port_add_counters (vm, port); return VNET_DEV_OK; From b39c6fec5cecb2c1c87c5f6f93738b330d5d7877 Mon Sep 17 00:00:00 2001 From: Varun Rapelly Date: Sat, 17 Aug 2024 16:19:29 +0000 Subject: [PATCH 101/271] tls: add async processing support Adds support for tls async processing using OpenSSL. Adds new CLI command to configure OpenSSL TLS configurations used by OpenSSL context and session. New CLI format is: tls openssl set-tls [record-size ] [record-split-size ] [max-pipelines ] This patch sets default values to first_seg_size(32MB) and add_seg_size(256MB). If the user configures these values, the defaults will be overwritten with the user-specified values. Type: feature JIRA: https://essjira.marvell.com/browse/IPBUSW-50810 Signed-off-by: Varun Rapelly Change-Id: I990be31fced9e258fdb036f5751cd67594b0bce7 Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/136352 Tested-by: sa_ip-sw-jenkins Reviewed-by: Nithinsen Kaithakadan --- src/plugins/tlsopenssl/tls_async.c | 590 +++++++++++++++++++++++++-- src/plugins/tlsopenssl/tls_openssl.c | 224 ++++++++-- src/plugins/tlsopenssl/tls_openssl.h | 38 +- src/vnet/tls/tls.c | 4 + src/vnet/tls/tls.h | 17 +- 5 files changed, 799 insertions(+), 74 deletions(-) diff --git a/src/plugins/tlsopenssl/tls_async.c b/src/plugins/tlsopenssl/tls_async.c index 89b4f77e33..7e9100fa5a 100644 --- a/src/plugins/tlsopenssl/tls_async.c +++ b/src/plugins/tlsopenssl/tls_async.c @@ -17,12 +17,31 @@ #include #include #include +#include -#define SSL_ASYNC_INFLIGHT 1 -#define SSL_ASYNC_READY 2 -#define SSL_ASYNC_REENTER 3 #define MAX_VECTOR_ASYNC 256 +#define SSL_WANT_NAMES \ + { \ + [0] = "N/A", [SSL_NOTHING] = "SSL_NOTHING", \ + [SSL_WRITING] = "SSL_WRITING", [SSL_READING] = "SSL_READING", \ + [SSL_X509_LOOKUP] = "SSL_X509_LOOKUP", \ + [SSL_ASYNC_PAUSED] = "SSL_ASYNC_PAUSED", \ + [SSL_ASYNC_NO_JOBS] = "SSL_ASYNC_NO_JOBS", \ + [SSL_CLIENT_HELLO_CB] = "SSL_CLIENT_HELLO_CB", \ + } + +static const char *ssl_want[] = SSL_WANT_NAMES; + +typedef enum ssl_evt_status_type_ +{ + SSL_ASYNC_INVALID_STATUS = 0, + SSL_ASYNC_INFLIGHT = 1, + SSL_ASYNC_READY = 2, + SSL_ASYNC_REENTER = 3, + SSL_ASYNC_MAX_STATUS +} ssl_evt_status_type_t; + typedef struct openssl_tls_callback_arg_ { int thread_index; @@ -33,7 +52,8 @@ typedef struct openssl_event_ { u32 ctx_index; int session_index; - u8 status; + ssl_evt_status_type_t status; + ssl_async_evt_type_t type; openssl_resume_handler *handler; openssl_tls_callback_arg_t cb_args; @@ -46,12 +66,15 @@ typedef struct openssl_async_queue_ { int evt_run_head; int evt_run_tail; + int depth; + int max_depth; } openssl_async_queue_t; typedef struct openssl_async_ { openssl_evt_t ***evt_pool; openssl_async_queue_t *queue; + openssl_async_queue_t *queue_in_init; void (*polling) (void); u8 start_polling; ENGINE *engine; @@ -74,8 +97,8 @@ struct engine_polling void qat_init_thread (void *arg); struct engine_polling engine_list[] = { - {"qat", qat_polling, qat_pre_init, qat_init_thread}, - {"dasync", dasync_polling, NULL, NULL} + { "qat", qat_polling, qat_pre_init, qat_init_thread }, + { "dasync", dasync_polling, NULL, NULL } }; openssl_async_t openssl_async_main; @@ -98,6 +121,7 @@ evt_pool_init (vlib_main_t * vm) vec_validate (om->evt_pool, num_threads - 1); vec_validate (om->queue, num_threads - 1); + vec_validate (om->queue_in_init, num_threads - 1); om->start_polling = 0; om->engine = 0; @@ -106,6 +130,13 @@ evt_pool_init (vlib_main_t * vm) { om->queue[i].evt_run_head = -1; om->queue[i].evt_run_tail = -1; + om->queue[i].depth = 0; + om->queue[i].max_depth = 0; + + om->queue_in_init[i].evt_run_head = -1; + om->queue_in_init[i].evt_run_tail = -1; + om->queue_in_init[i].depth = 0; + om->queue_in_init[i].max_depth = 0; } om->polling = NULL; @@ -276,10 +307,503 @@ tls_async_openssl_callback (SSL * s, void *cb_arg) return 1; } +/* + * Continue an async SSL_write() call. + * This function is _only_ called when continuing an SSL_write() call + * that returned WANT_ASYNC. + * Since it continues the handling of an existing, paused SSL job + * (ASYNC_JOB*), the 'buf' and 'num' params to SSL_write() have + * already been set in the initial call, and are meaningless here. + * Therefore setting buf=null,num=0, to emphasize the point. + * On successful write, TLS context total_write bytes are updated. + */ +static int +openssl_async_write_from_fifo_into_ssl (svm_fifo_t *f, SSL *ssl, + tls_ctx_t *ctx) +{ + int wrote = 0; + + wrote = SSL_write (ssl, NULL, 0); + ossl_check_err_is_fatal (ssl, wrote); + + ctx->total_write -= wrote; + svm_fifo_dequeue_drop (f, wrote); + + return wrote; +} + +/* + * Perform SSL_write from TX FIFO head. + * On successful write, TLS context total_write bytes are updated. + */ +static_always_inline int +openssl_write_from_fifo_head_into_ssl (svm_fifo_t *f, SSL *ssl, tls_ctx_t *ctx, + u32 max_len) +{ + int wrote = 0, rv, i = 0, len; + u32 n_segs = 2; + svm_fifo_seg_t fs[n_segs]; + + max_len = clib_min (ctx->total_write, max_len); + + len = svm_fifo_segments (f, 0, fs, &n_segs, max_len); + if (len <= 0) + return 0; + + while (wrote < len && i < n_segs) + { + rv = SSL_write (ssl, fs[i].data, fs[i].len); + wrote += (rv > 0) ? rv : 0; + if (rv < (int) fs[i].len) + break; + i++; + } + + if (wrote) + { + ctx->total_write -= wrote; + svm_fifo_dequeue_drop (f, wrote); + } + + return wrote; +} + +static int +openssl_async_read_from_ssl_into_fifo (svm_fifo_t *f, SSL *ssl) +{ + int read; + + read = SSL_read (ssl, NULL, 0); + if (read <= 0) + return read; + + svm_fifo_enqueue_nocopy (f, read); + + return read; +} + +/* + * Pop the current event from queue and update tail if needed + */ +static void +tls_async_dequeue_update (openssl_evt_t *event, int *evt_run_head, + int *evt_run_tail, int *queue_depth) +{ + /* remove the event from queue head */ + *evt_run_head = event->next; + event->status = SSL_ASYNC_INVALID_STATUS; + event->next = -1; + + (*queue_depth)--; + + if (*evt_run_head < 0) + { + *evt_run_tail = -1; + if (*queue_depth) + ERRORPR ("queue empty but depth:%d\n", *queue_depth); + } +} + +static int +tls_async_dequeue_event (int thread_index) +{ + openssl_evt_t *event; + openssl_async_t *om = &openssl_async_main; + openssl_async_queue_t *queue = om->queue; + int *evt_run_tail = &queue[thread_index].evt_run_tail; + int *evt_run_head = &queue[thread_index].evt_run_head; + int dequeue_cnt = clib_min (queue[thread_index].depth, MAX_VECTOR_ASYNC); + + /* dequeue all pending events, events enqueued during this routine call, + * will be handled next time tls_async_dequeue_event is invoked */ + while (*evt_run_head >= 0 && dequeue_cnt--) + { + session_t *app_session, *tls_session; + openssl_ctx_t *oc; + tls_ctx_t *ctx; + SSL *ssl; + + event = openssl_evt_get_w_thread (*evt_run_head, thread_index); + ctx = openssl_ctx_get_w_thread (event->ctx_index, thread_index); + oc = (openssl_ctx_t *) ctx; + ssl = oc->ssl; + + if (event->type == SSL_ASYNC_EVT_RD) + { + /* read event */ + svm_fifo_t *app_rx_fifo, *tls_rx_fifo; + int read; + + app_session = session_get_from_handle (ctx->app_session_handle); + app_rx_fifo = app_session->rx_fifo; + + tls_session = session_get_from_handle (ctx->tls_session_handle); + tls_rx_fifo = tls_session->rx_fifo; + + /* continue the paused job */ + read = openssl_async_read_from_ssl_into_fifo (app_rx_fifo, ssl); + if (read < 0) + { + if (SSL_want_async (ssl)) + goto handle_later; + + tls_async_dequeue_update (event, evt_run_head, evt_run_tail, + &queue[thread_index].depth); + goto ev_rd_done; + } + + /* read finished or in error, remove the event from queue */ + tls_async_dequeue_update (event, evt_run_head, evt_run_tail, + &queue[thread_index].depth); + + /* Unrecoverable protocol error. Reset connection */ + if (PREDICT_FALSE ((read < 0) && + (SSL_get_error (ssl, read) == SSL_ERROR_SSL))) + { + tls_notify_app_io_error (ctx); + goto ev_rd_done; + } + + /* + * Managed to read some data. If handshake just completed, session + * may still be in accepting state. + */ + if (app_session->session_state >= SESSION_STATE_READY) + tls_notify_app_enqueue (ctx, app_session); + + /* managed to read, try to read more */ + while (read > 0) + { + read = openssl_read_from_ssl_into_fifo (app_rx_fifo, ssl); + if (read < 0) + { + if (SSL_want_async (ssl)) + { + vpp_tls_async_enqueue_event (ctx, SSL_ASYNC_EVT_RD, NULL, + 0); + goto ev_rd_queued; + } + } + + /* Unrecoverable protocol error. Reset connection */ + if (PREDICT_FALSE ((read < 0) && + (SSL_get_error (ssl, read) == SSL_ERROR_SSL))) + { + tls_notify_app_io_error (ctx); + goto ev_rd_done; + } + + /* If handshake just completed, session may still be in accepting + * state */ + if (read >= 0 && + app_session->session_state >= SESSION_STATE_READY) + tls_notify_app_enqueue (ctx, app_session); + } + + ev_rd_done: + /* read done */ + ctx->in_async_read = false; + + if ((SSL_pending (ssl) > 0) || + svm_fifo_max_dequeue_cons (tls_rx_fifo)) + { + tls_add_vpp_q_builtin_rx_evt (tls_session); + } + + ev_rd_queued: + continue; + } + else if (event->type == SSL_ASYNC_EVT_WR) + { + /* write event */ + int wrote, wrote_sum = 0; + u32 space, enq_buf; + svm_fifo_t *app_tx_fifo, *tls_tx_fifo; + transport_send_params_t *sp = + (transport_send_params_t *) event->handler; + + app_session = session_get_from_handle (ctx->app_session_handle); + app_tx_fifo = app_session->tx_fifo; + + /* continue the paused job */ + wrote = + openssl_async_write_from_fifo_into_ssl (app_tx_fifo, ssl, ctx); + if (!wrote) + { + if (SSL_want_async (ssl)) + /* paused job not ready, wait */ + goto handle_later; + ERRORPR ("[wrote:%d want:%s ctx:%d]\n", wrote, + ssl_want[SSL_want (ssl)], oc->openssl_ctx_index); + } + + /* paused job done, remove event, update queue */ + tls_async_dequeue_update (event, evt_run_head, evt_run_tail, + &queue[thread_index].depth); + + /* Unrecoverable protocol error. Reset connection */ + if (PREDICT_FALSE (wrote < 0)) + { + tls_notify_app_io_error (ctx); + ERRORPR ("Unrecoverable protocol error. Reset connection\n"); + goto ev_in_queue; + } + wrote_sum += wrote; + + tls_session = session_get_from_handle (ctx->tls_session_handle); + tls_tx_fifo = tls_session->tx_fifo; + + /* prepare for remaining write(s) */ + space = svm_fifo_max_enqueue_prod (tls_tx_fifo); + /* Leave a bit of extra space for tls ctrl data, if any needed */ + space = clib_max ((int) space - TLSO_CTRL_BYTES, 0); + + /* continue remaining openssl_ctx_write request */ + while (ctx->total_write) + { + int rv; + u32 deq_max = svm_fifo_max_dequeue_cons (app_tx_fifo); + + deq_max = clib_min (deq_max, space); + deq_max = clib_min (deq_max, sp->max_burst_size); + if (!deq_max) + goto check_tls_fifo; + + /* Make sure tcp's tx fifo can actually buffer all bytes to + * be dequeued. If under memory pressure, tls's fifo segment + * might not be able to allocate the chunks needed. This also + * avoids errors from the underlying custom bio to the ssl + * infra which at times can get stuck. */ + if (svm_fifo_provision_chunks (tls_tx_fifo, 0, 0, + deq_max + TLSO_CTRL_BYTES)) + goto check_tls_fifo; + + rv = openssl_write_from_fifo_head_into_ssl (app_tx_fifo, ssl, + ctx, deq_max); + + /* Unrecoverable protocol error. Reset connection */ + if (PREDICT_FALSE (rv < 0)) + { + tls_notify_app_io_error (ctx); + ERRORPR ("Unrecoverable protocol error. Reset connection\n"); + goto ev_in_queue; + } + + if (!rv) + { + if (SSL_want_async (ssl)) + { + /* new paused job, add queue event and wait */ + vpp_tls_async_enqueue_event (ctx, SSL_ASYNC_EVT_WR, sp, + 0); + goto ev_in_queue; + } + ERRORPR ("[rv:%d want:%s ctx:%d]\n", rv, + ssl_want[SSL_want (ssl)], oc->openssl_ctx_index); + break; + } + wrote_sum += rv; + } + + if (svm_fifo_needs_deq_ntf (app_tx_fifo, wrote)) + session_dequeue_notify (app_session); + + check_tls_fifo: + /* we got here, async write is done or not possible */ + ctx->total_write = 0; + + if (PREDICT_FALSE (ctx->app_closed && + BIO_ctrl_pending (oc->rbio) <= 0)) + openssl_confirm_app_close (ctx); + + /* Deschedule and wait for deq notification if fifo is almost full */ + enq_buf = + clib_min (svm_fifo_size (tls_tx_fifo) / 2, TLSO_MIN_ENQ_SPACE); + if (space < wrote_sum + enq_buf) + { + svm_fifo_add_want_deq_ntf (tls_tx_fifo, SVM_FIFO_WANT_DEQ_NOTIF); + transport_connection_deschedule (&ctx->connection); + sp->flags |= TRANSPORT_SND_F_DESCHED; + } + else + { + /* Request tx reschedule of the app session */ + app_session->flags |= SESSION_F_CUSTOM_TX; + transport_connection_reschedule (&ctx->connection); + } + + ev_in_queue: + /* job removed, openssl_ctx_write will resume */ + continue; + } + else + { + /* wrong event type */ + ERRORPR ("goto remove_event [event->type:%d]\n", event->type); + tls_async_dequeue_update (event, evt_run_head, evt_run_tail, + &queue[thread_index].depth); + } + } + +handle_later: + return 1; +} + +static int +tls_async_dequeue_event_in_init (int thread_index) +{ + openssl_evt_t *event; + openssl_async_t *om = &openssl_async_main; + openssl_async_queue_t *queue = om->queue_in_init; + int *evt_run_tail = &queue[thread_index].evt_run_tail; + int *evt_run_head = &queue[thread_index].evt_run_head; + + /* dequeue events if exists */ + while (*evt_run_head >= 0) + { + openssl_ctx_t *oc; + tls_ctx_t *ctx; + int rv, err; + + event = openssl_evt_get_w_thread (*evt_run_head, thread_index); + ctx = openssl_ctx_get_w_thread (event->ctx_index, thread_index); + oc = (openssl_ctx_t *) ctx; + + if (event->type != SSL_ASYNC_EVT_INIT) + { + /* wrong event type */ + ERRORPR ("goto remove_event [event->type:%d]\n", event->type); + goto remove_event; + } + + if (!SSL_in_init (oc->ssl)) + { + ERRORPR ("[!SSL_in_init() != ev->type:%d] th:%d ev:%d\n", + event->type, event->cb_args.thread_index, + event->cb_args.event_index); + goto remove_event; + } + + rv = SSL_do_handshake (oc->ssl); + err = SSL_get_error (oc->ssl, rv); + + /* Do not remove session from tail */ + if (err == SSL_ERROR_WANT_ASYNC) + goto handle_later; + + if (err == SSL_ERROR_SSL) + { + char buf[512]; + ERR_error_string (ERR_get_error (), buf); + ERRORPR ("Err: %s\n", buf); + openssl_handle_handshake_failure (ctx); + goto remove_event; + } + + if (err == SSL_ERROR_WANT_WRITE || err == SSL_ERROR_WANT_READ) + goto handle_later; + + /* client not supported */ + if (!SSL_is_server (oc->ssl)) + { + ERRORPR ("goto remove_event [!SSL_is_server]\n"); + goto remove_event; + } + + /* Need to check transport status */ + if (ctx->is_passive_close) + { + openssl_handle_handshake_failure (ctx); + goto remove_event; + } + + if (tls_notify_app_accept (ctx)) + { + ctx->c_s_index = SESSION_INVALID_INDEX; + tls_disconnect_transport (ctx); + } + + TLS_DBG (1, "Handshake for %u complete. TLS cipher is %s", + oc->openssl_ctx_index, SSL_get_cipher (oc->ssl)); + + remove_event: + *evt_run_head = event->next; + queue[thread_index].depth--; + + if (*evt_run_head < 0) + { + /* queue empty, bail out */ + *evt_run_tail = -1; + if (queue[thread_index].depth) + ERRORPR ("queue empty but depth:%d\n", queue[thread_index].depth); + break; + } + } + +handle_later: + return 1; +} + int -vpp_tls_async_init_event (tls_ctx_t * ctx, - openssl_resume_handler * handler, - session_t * session) +vpp_tls_async_enqueue_event (tls_ctx_t *ctx, int evt_type, + transport_send_params_t *sp, int size) +{ + openssl_evt_t *event; + openssl_async_t *om = &openssl_async_main; + openssl_async_queue_t *queue; + openssl_ctx_t *oc; + int thread_index; + int event_index; + int *evt_run_tail; + int *evt_run_head; + + event = openssl_evt_get (ctx->evt_index[evt_type]); + + thread_index = event->thread_idx; + event_index = event->event_idx; + + oc = (openssl_ctx_t *) ctx; + + /* set queue to be used */ + if (SSL_in_init (oc->ssl)) + queue = om->queue_in_init; + else + queue = om->queue; + + evt_run_tail = &queue[thread_index].evt_run_tail; + evt_run_head = &queue[thread_index].evt_run_head; + + event->type = SSL_ASYNC_INFLIGHT; + event->handler = (openssl_resume_handler *) sp; + event->next = -1; + + /* first we enqueue the request */ + if (*evt_run_tail >= 0) + { + openssl_evt_t *event_tail; + + /* queue not empty, append to tail event */ + event_tail = openssl_evt_get_w_thread (*evt_run_tail, thread_index); + event_tail->next = event_index; + } + + /* set tail to use new event index */ + *evt_run_tail = event_index; + + if (*evt_run_head < 0) + /* queue is empty, update head */ + *evt_run_head = event_index; + + queue[thread_index].depth++; + if (queue[thread_index].depth > queue[thread_index].max_depth) + queue[thread_index].max_depth = queue[thread_index].depth; + + return 1; +} + +static int +vpp_tls_async_init_event (tls_ctx_t *ctx, openssl_resume_handler *handler, + session_t *session, ssl_async_evt_type_t evt_type) { u32 eidx; openssl_evt_t *event; @@ -291,37 +815,53 @@ vpp_tls_async_init_event (tls_ctx_t * ctx, event->ctx_index = oc->openssl_ctx_index; event->event_idx = eidx; event->thread_idx = thread_id; - event->handler = handler; + event->handler = NULL; event->session_index = session->session_index; - event->status = 0; - ctx->evt_index = eidx; -#ifdef HAVE_OPENSSL_ASYNC - SSL_set_async_callback_arg (oc->ssl, &event->cb_args); -#endif + event->type = evt_type; + event->status = SSL_ASYNC_INVALID_STATUS; + ctx->evt_index[evt_type] = eidx; + + return 1; +} + +int +vpp_tls_async_init_events (tls_ctx_t *ctx, openssl_resume_handler *handler, + session_t *session) +{ + vpp_tls_async_init_event (ctx, handler, session, SSL_ASYNC_EVT_INIT); + vpp_tls_async_init_event (ctx, handler, session, SSL_ASYNC_EVT_RD); + vpp_tls_async_init_event (ctx, handler, session, SSL_ASYNC_EVT_WR); return 1; } int -vpp_openssl_is_inflight (tls_ctx_t * ctx) +vpp_openssl_is_inflight (tls_ctx_t *ctx) { u32 eidx; openssl_evt_t *event; - eidx = ctx->evt_index; - event = openssl_evt_get (eidx); + int i; + + for (i = SSL_ASYNC_EVT_INIT; i < SSL_ASYNC_EVT_MAX; i++) + { + eidx = ctx->evt_index[i]; + event = openssl_evt_get (eidx); + + if (event->status == SSL_ASYNC_INFLIGHT) + return 1; + } - if (event->status == SSL_ASYNC_INFLIGHT) - return 1; return 0; } int -vpp_tls_async_update_event (tls_ctx_t * ctx, int eagain) +vpp_tls_async_update_event (tls_ctx_t *ctx, int eagain, + ssl_async_evt_type_t type) { u32 eidx; openssl_evt_t *event; - eidx = ctx->evt_index; + eidx = ctx->evt_index[type]; event = openssl_evt_get (eidx); event->status = SSL_ASYNC_INFLIGHT; if (eagain) @@ -469,7 +1009,7 @@ tls_resume_from_crypto (int thread_index) continue; } - event->status = 0; + event->status = SSL_ASYNC_INVALID_STATUS; *evt_run_head = event->next; if (event->next < 0) @@ -501,8 +1041,8 @@ tls_async_process (vlib_main_t * vm, vlib_node_runtime_t * rt, thread_index = vlib_get_thread_index (); if (pool_elts (om->evt_pool[thread_index]) > 0) { - openssl_async_polling (); - tls_resume_from_crypto (thread_index); + tls_async_dequeue_event_in_init (thread_index); + tls_async_dequeue_event (thread_index); } return 0; diff --git a/src/plugins/tlsopenssl/tls_openssl.c b/src/plugins/tlsopenssl/tls_openssl.c index e63413a390..44100023a0 100644 --- a/src/plugins/tlsopenssl/tls_openssl.c +++ b/src/plugins/tlsopenssl/tls_openssl.c @@ -73,9 +73,16 @@ openssl_ctx_free (tls_ctx_t * ctx) SSL_free (oc->ssl); vec_free (ctx->srv_hostname); SSL_CTX_free (oc->client_ssl_ctx); -#ifdef HAVE_OPENSSL_ASYNC - openssl_evt_free (ctx->evt_index, ctx->c_thread_index); -#endif + + if (openssl_main.async) + { + openssl_evt_free (ctx->evt_index[SSL_ASYNC_EVT_INIT], + ctx->c_thread_index); + openssl_evt_free (ctx->evt_index[SSL_ASYNC_EVT_RD], + ctx->c_thread_index); + openssl_evt_free (ctx->evt_index[SSL_ASYNC_EVT_WR], + ctx->c_thread_index); + } } pool_put_index (openssl_main.ctx_pool[ctx->c_thread_index], @@ -158,12 +165,8 @@ openssl_lctx_get (u32 lctx_index) return pool_elt_at_index (openssl_main.lctx_pool, lctx_index); } -#define ossl_check_err_is_fatal(_ssl, _rv) \ - if (PREDICT_FALSE (_rv < 0 && SSL_get_error (_ssl, _rv) == SSL_ERROR_SSL)) \ - return -1; - -static int -openssl_read_from_ssl_into_fifo (svm_fifo_t *f, SSL *ssl, u32 max_len) +int +openssl_read_from_ssl_into_fifo (svm_fifo_t *f, SSL *ssl) { int read, rv, n_fs, i; const int n_segs = 2; @@ -174,7 +177,6 @@ openssl_read_from_ssl_into_fifo (svm_fifo_t *f, SSL *ssl, u32 max_len) if (!max_enq) return 0; - max_enq = clib_min (max_len, max_enq); n_fs = svm_fifo_provision_chunks (f, fs, n_segs, max_enq); if (n_fs < 0) return 0; @@ -184,7 +186,7 @@ openssl_read_from_ssl_into_fifo (svm_fifo_t *f, SSL *ssl, u32 max_len) if (read <= 0) { ossl_check_err_is_fatal (ssl, read); - return 0; + return read; } if (read == (int) fs[0].len) @@ -246,11 +248,15 @@ openssl_check_async_status (tls_ctx_t * ctx, openssl_resume_handler * handler, SSL_get_async_status (oc->ssl, &estatus); if (estatus == ASYNC_STATUS_EAGAIN) { - vpp_tls_async_update_event (ctx, 1); + vpp_tls_async_update_event (ctx, 1, SSL_ASYNC_EVT_INIT); + vpp_tls_async_update_event (ctx, 1, SSL_ASYNC_EVT_RD); + vpp_tls_async_update_event (ctx, 1, SSL_ASYNC_EVT_WR); } else { - vpp_tls_async_update_event (ctx, 0); + vpp_tls_async_update_event (ctx, 0, SSL_ASYNC_EVT_INIT); + vpp_tls_async_update_event (ctx, 0, SSL_ASYNC_EVT_RD); + vpp_tls_async_update_event (ctx, 0, SSL_ASYNC_EVT_WR); } return 1; @@ -259,8 +265,32 @@ openssl_check_async_status (tls_ctx_t * ctx, openssl_resume_handler * handler, #endif -static void -openssl_handle_handshake_failure (tls_ctx_t * ctx) +static int +openssl_handle_want_async (tls_ctx_t *ctx, int evt_type, + transport_send_params_t *sp, int size) +{ + int ret; + + if (evt_type >= SSL_ASYNC_EVT_MAX || evt_type == 0) + { + ERRORPR ("\\- return 0 [illegal evt_type value:%d]\n", evt_type); + return 0; + } + + if (evt_type == SSL_ASYNC_EVT_WR) + { + /* de-schedule transport connection */ + transport_connection_deschedule (&ctx->connection); + sp->flags |= TRANSPORT_SND_F_DESCHED; + ctx->total_write = size; + } + ret = vpp_tls_async_enqueue_event (ctx, evt_type, sp, size); + + return ret; +} + +void +openssl_handle_handshake_failure (tls_ctx_t *ctx) { session_t *app_session; @@ -307,19 +337,18 @@ openssl_ctx_handshake_rx (tls_ctx_t * ctx, session_t * tls_session) rv = SSL_do_handshake (oc->ssl); err = SSL_get_error (oc->ssl, rv); -#ifdef HAVE_OPENSSL_ASYNC - if (err == SSL_ERROR_WANT_ASYNC) + if (openssl_main.async && err == SSL_ERROR_WANT_ASYNC) { - openssl_check_async_status (ctx, openssl_ctx_handshake_rx, - tls_session); + openssl_handle_want_async (ctx, SSL_ASYNC_EVT_INIT, NULL, 0); + return -1; } -#endif + if (err == SSL_ERROR_SSL) { char buf[512]; ERR_error_string (ERR_get_error (), buf); clib_warning ("Err: %s", buf); - + ERRORPR ("Err: %s\n", buf); openssl_handle_handshake_failure (ctx); return -1; } @@ -384,13 +413,18 @@ openssl_ctx_handshake_rx (tls_ctx_t * ctx, session_t * tls_session) return rv; } -static void -openssl_confirm_app_close (tls_ctx_t * ctx) +void +openssl_confirm_app_close (tls_ctx_t *ctx) { openssl_ctx_t *oc = (openssl_ctx_t *) ctx; SSL_shutdown (oc->ssl); tls_disconnect_transport (ctx); session_transport_closed_notify (&ctx->connection); + session_t *app_session = session_get_from_handle (ctx->app_session_handle); + + /* Shrink FIFOs */ + if (app_session) + session_shrink_fifos (app_session); } static int @@ -434,7 +468,14 @@ openssl_ctx_write_tls (tls_ctx_t *ctx, session_t *app_session, } if (!wrote) - goto check_tls_fifo; + { + if (openssl_main.async && SSL_want_async (oc->ssl)) + { + openssl_handle_want_async (ctx, SSL_ASYNC_EVT_WR, sp, deq_max); + return 0; + } + goto check_tls_fifo; + } if (svm_fifo_needs_deq_ntf (f, wrote)) session_dequeue_notify (app_session); @@ -534,7 +575,6 @@ static inline int openssl_ctx_read_tls (tls_ctx_t *ctx, session_t *tls_session) { openssl_ctx_t *oc = (openssl_ctx_t *) ctx; - const u32 max_len = 128 << 10; session_t *app_session; svm_fifo_t *f; int read; @@ -548,26 +588,40 @@ openssl_ctx_read_tls (tls_ctx_t *ctx, session_t *tls_session) tls_session = session_get_from_handle (ctx->tls_session_handle); } + if (ctx->in_async_read) + return 0; + app_session = session_get_from_handle (ctx->app_session_handle); f = app_session->rx_fifo; - read = openssl_read_from_ssl_into_fifo (f, oc->ssl, max_len); + read = openssl_read_from_ssl_into_fifo (f, oc->ssl); + if (read < 0) + { + if (openssl_main.async && SSL_want_async (oc->ssl)) + { + ctx->in_async_read = true; + openssl_handle_want_async (ctx, SSL_ASYNC_EVT_RD, NULL, 0); + return 0; + } + } /* Unrecoverable protocol error. Reset connection */ - if (PREDICT_FALSE (read < 0)) + if (PREDICT_FALSE ((read < 0) && + (SSL_get_error (oc->ssl, read) == SSL_ERROR_SSL))) { tls_notify_app_io_error (ctx); return 0; } - if (read) + /* If handshake just completed, session may still be in accepting state */ + if (read > 0 && app_session->session_state >= SESSION_STATE_READY) tls_notify_app_enqueue (ctx, app_session); if ((SSL_pending (oc->ssl) > 0) || svm_fifo_max_dequeue_cons (tls_session->rx_fifo)) tls_add_vpp_q_builtin_rx_evt (tls_session); - return read; + return (read > 0) ? read : 0; } static inline int @@ -752,6 +806,10 @@ openssl_ctx_init_client (tls_ctx_t * ctx) if (om->async) SSL_CTX_set_mode (oc->client_ssl_ctx, SSL_MODE_ASYNC); #endif + + /* Set TLSv1_2 */ + SSL_CTX_set_max_proto_version (oc->client_ssl_ctx, TLS1_2_VERSION); + rv = SSL_CTX_set_cipher_list (oc->client_ssl_ctx, (const char *) om->ciphers); if (rv != 1) @@ -796,7 +854,17 @@ openssl_ctx_init_client (tls_ctx_t * ctx) { TLS_DBG (1, "Couldn't set client certificate-key pair"); } - + /* Set TLS Record size */ + if (om->record_size) + { + rv = SSL_CTX_set_max_send_fragment (oc->client_ssl_ctx, om->record_size); + if (rv != 1) + { + TLS_DBG (1, "Couldn't set TLS record-size"); + return -1; + } + TLS_DBG (1, "Using TLS record-size of %d", om->record_size); + } /* * 2. Do the first steps in the handshake. */ @@ -805,7 +873,7 @@ openssl_ctx_init_client (tls_ctx_t * ctx) #ifdef HAVE_OPENSSL_ASYNC session_t *tls_session = session_get_from_handle (ctx->tls_session_handle); - vpp_tls_async_init_event (ctx, openssl_ctx_handshake_rx, tls_session); + vpp_tls_async_init_events (ctx, openssl_ctx_handshake_rx, tls_session); #endif while (1) { @@ -889,6 +957,39 @@ openssl_start_listen (tls_ctx_t * lctx) TLS_DBG (1, "Couldn't set temp DH parameters"); return -1; } + /* Set TLSv1_2 */ + SSL_CTX_set_max_proto_version (ssl_ctx, TLS1_2_VERSION); + /* Set TLS Record size */ + if (om->record_size) + { + rv = SSL_CTX_set_max_send_fragment (ssl_ctx, om->record_size); + if (rv != 1) + { + TLS_DBG (1, "Couldn't set TLS record-size"); + return -1; + } + } + /* Set TLS Record Split size */ + if (om->record_split_size) + { + rv = SSL_CTX_set_split_send_fragment (ssl_ctx, om->record_split_size); + if (rv != 1) + { + TLS_DBG (1, "Couldn't set TLS record-split-size"); + return -1; + } + } + + /* Set TLS Max Pipeline count */ + if (om->max_pipelines) + { + rv = SSL_CTX_set_max_pipelines (ssl_ctx, om->max_pipelines); + if (rv != 1) + { + TLS_DBG (1, "Couldn't set TLS max-pipelines"); + return -1; + } + } /* * Set the key and cert @@ -1007,22 +1108,20 @@ openssl_ctx_init_server (tls_ctx_t * ctx) TLS_DBG (1, "Initiating handshake for [%u]%u", ctx->c_thread_index, oc->openssl_ctx_index); -#ifdef HAVE_OPENSSL_ASYNC session_t *tls_session = session_get_from_handle (ctx->tls_session_handle); - vpp_tls_async_init_event (ctx, openssl_ctx_handshake_rx, tls_session); -#endif + if (openssl_main.async) + vpp_tls_async_init_events (ctx, openssl_ctx_handshake_rx, tls_session); + while (1) { rv = SSL_do_handshake (oc->ssl); err = SSL_get_error (oc->ssl, rv); -#ifdef HAVE_OPENSSL_ASYNC - if (err == SSL_ERROR_WANT_ASYNC) + if (openssl_main.async && err == SSL_ERROR_WANT_ASYNC) { - openssl_check_async_status (ctx, openssl_ctx_handshake_rx, - tls_session); + openssl_handle_want_async (ctx, SSL_ASYNC_EVT_INIT, NULL, 0); + break; } -#endif if (err != SSL_ERROR_WANT_WRITE) break; } @@ -1044,10 +1143,8 @@ openssl_handshake_is_over (tls_ctx_t * ctx) static int openssl_transport_close (tls_ctx_t * ctx) { -#ifdef HAVE_OPENSSL_ASYNC - if (vpp_openssl_is_inflight (ctx)) + if (openssl_main.async && vpp_openssl_is_inflight (ctx)) return 0; -#endif if (!openssl_handshake_is_over (ctx)) { @@ -1221,7 +1318,6 @@ VLIB_INIT_FUNCTION (tls_openssl_init) = }; /* *INDENT-ON* */ -#ifdef HAVE_OPENSSL_ASYNC static clib_error_t * tls_openssl_set_command_fn (vlib_main_t * vm, unformat_input_t * input, vlib_cli_command_t * cmd) @@ -1298,7 +1394,47 @@ VLIB_CLI_COMMAND (tls_openssl_set_command, static) = .function = tls_openssl_set_command_fn, }; /* *INDENT-ON* */ -#endif + +static clib_error_t * +tls_openssl_set_tls_fn (vlib_main_t *vm, unformat_input_t *input, + vlib_cli_command_t *cmd) +{ + openssl_main_t *om = &openssl_main; + u32 tmp; + + while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) + { + if (unformat (input, "record-size %U", unformat_memory_size, &tmp)) + { + clib_warning ("Using TLS record-size of %d", tmp); + om->record_size = tmp; + } + else if (unformat (input, "record-split-size %U", unformat_memory_size, + &tmp)) + { + clib_warning ("Using TLS record-split-size of %d", tmp); + om->record_split_size = tmp; + } + else if (unformat (input, "max-pipelines %U", unformat_memory_size, + &tmp)) + { + clib_warning ("Using TLS max-pipelines of %d", tmp); + om->max_pipelines = tmp; + } + else + return clib_error_return (0, "failed: unknown input `%U'", + format_unformat_error, input); + } + + return 0; +} + +VLIB_CLI_COMMAND (tls_openssl_set_tls, static) = { + .path = "tls openssl set-tls", + .short_help = "tls openssl set-tls [record-size ] [record-split-size " + "] [max-pipelines ]", + .function = tls_openssl_set_tls_fn, +}; /* *INDENT-OFF* */ VLIB_PLUGIN_REGISTER () = { diff --git a/src/plugins/tlsopenssl/tls_openssl.h b/src/plugins/tlsopenssl/tls_openssl.h index 1600cd77ab..090cc33df6 100644 --- a/src/plugins/tlsopenssl/tls_openssl.h +++ b/src/plugins/tlsopenssl/tls_openssl.h @@ -29,6 +29,28 @@ #define DTLSO_MAX_DGRAM 2000 +#define ossl_check_err_is_fatal(_ssl, _rv) \ + if (PREDICT_FALSE (_rv < 0 && SSL_get_error (_ssl, _rv) == SSL_ERROR_SSL)) \ + return -1; + +#define ERRORPR(fmt_str, ...) \ + do \ + { \ + FILE *f = fopen ("/tmp/vpp-err.log", "a"); \ + if (f != NULL) \ + { \ + struct timespec ts; \ + clock_gettime (CLOCK_REALTIME, &ts); \ + fprintf (f, "[%ld.%06ld]vpp[%d:%lx:%02ld][%d:%s] " fmt_str, \ + ts.tv_sec, ts.tv_nsec / 1000, getpid (), \ + (unsigned long) pthread_self (), vlib_get_thread_index (), \ + __LINE__, __func__, ##__VA_ARGS__); \ + fflush (f); \ + fclose (f); \ + } \ + } \ + while (0) + typedef struct tls_ctx_openssl_ { tls_ctx_t ctx; /**< First */ @@ -63,15 +85,20 @@ typedef struct openssl_main_ u8 *ciphers; int engine_init; int async; + u32 record_size; + u32 record_split_size; + u32 max_pipelines; } openssl_main_t; typedef int openssl_resume_handler (tls_ctx_t * ctx, session_t * tls_session); tls_ctx_t *openssl_ctx_get_w_thread (u32 ctx_index, u8 thread_index); -int vpp_tls_async_init_event (tls_ctx_t * ctx, - openssl_resume_handler * handler, - session_t * session); -int vpp_tls_async_update_event (tls_ctx_t * ctx, int eagain); +int vpp_tls_async_init_events (tls_ctx_t *ctx, openssl_resume_handler *handler, + session_t *session); +int vpp_tls_async_update_event (tls_ctx_t *ctx, int eagain, + ssl_async_evt_type_t type); +int vpp_tls_async_enqueue_event (tls_ctx_t *ctx, int evt_type, + transport_send_params_t *sp, int size); int tls_async_openssl_callback (SSL * s, void *evt); int openssl_evt_free (int event_idx, u8 thread_index); void openssl_polling_start (ENGINE * engine); @@ -80,6 +107,9 @@ void openssl_async_node_enable_disable (u8 is_en); clib_error_t *tls_openssl_api_init (vlib_main_t * vm); int tls_openssl_set_ciphers (char *ciphers); int vpp_openssl_is_inflight (tls_ctx_t * ctx); +int openssl_read_from_ssl_into_fifo (svm_fifo_t *f, SSL *ssl); +void openssl_handle_handshake_failure (tls_ctx_t *ctx); +void openssl_confirm_app_close (tls_ctx_t *ctx); #endif /* SRC_PLUGINS_TLSOPENSSL_TLS_OPENSSL_H_ */ diff --git a/src/vnet/tls/tls.c b/src/vnet/tls/tls.c index 358e3a7b2e..66fcefd93b 100644 --- a/src/vnet/tls/tls.c +++ b/src/vnet/tls/tls.c @@ -1403,6 +1403,10 @@ tls_init (vlib_main_t * vm) vec_validate (tm->rx_bufs, num_threads - 1); vec_validate (tm->tx_bufs, num_threads - 1); + /* + * first_seg_size default value 32MB + * add_seg_size default value 256 MB + */ tm->first_seg_size = 32 << 20; tm->add_seg_size = 256 << 20; diff --git a/src/vnet/tls/tls.h b/src/vnet/tls/tls.h index 60f96ee5f4..c27857e58e 100644 --- a/src/vnet/tls/tls.h +++ b/src/vnet/tls/tls.h @@ -36,6 +36,19 @@ #define TLS_DBG(_lvl, _fmt, _args...) #endif +#define foreach_ssl_async_evt_type \ + _ (INIT, "SSL_in_init async event") \ + _ (RD, "Read async event") \ + _ (WR, "Write async event") \ + _ (MAX, "Maximum async event") + +typedef enum ssl_async_evt_type_ +{ +#define _(sym, str) SSL_ASYNC_EVT_##sym, + foreach_ssl_async_evt_type +#undef _ +} ssl_async_evt_type_t; + /* *INDENT-OFF* */ typedef struct tls_cxt_id_ { @@ -100,9 +113,11 @@ typedef struct tls_ctx_ u8 is_migrated; tls_conn_flags_t flags; u8 *srv_hostname; - u32 evt_index; + u32 evt_index[SSL_ASYNC_EVT_MAX]; + u32 total_write; u32 ckpair_index; transport_proto_t tls_type; + bool in_async_read; } tls_ctx_t; typedef struct tls_main_ From 916417840a5a0e72b77fcada10bd49707c414fbb Mon Sep 17 00:00:00 2001 From: Varun Rapelly Date: Tue, 27 Aug 2024 14:32:19 +0000 Subject: [PATCH 102/271] tls: configurable openssl engine support This patch enables OpenSSL engine support for tls plugin. New config parameters added in tls section to set the OpenSSL engine path. tls { engine-path } Type: improvement JIRA: https://essjira.marvell.com/browse/IPBUSW-52548 Signed-off-by: Varun Rapelly Change-Id: Ib85cc59ff75795db783c6f180aa2f7128f10e578 Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/136353 Tested-by: sa_ip-sw-jenkins Reviewed-by: Nithinsen Kaithakadan --- src/plugins/tlsopenssl/tls_async.c | 29 +++++++++++++++++++++++++++++ src/vnet/tls/tls.c | 2 ++ src/vnet/tls/tls.h | 1 + 3 files changed, 32 insertions(+) diff --git a/src/plugins/tlsopenssl/tls_async.c b/src/plugins/tlsopenssl/tls_async.c index 7e9100fa5a..c798833447 100644 --- a/src/plugins/tlsopenssl/tls_async.c +++ b/src/plugins/tlsopenssl/tls_async.c @@ -95,9 +95,11 @@ struct engine_polling }; void qat_init_thread (void *arg); +void dpdk_engine_init_thread (void *arg); struct engine_polling engine_list[] = { { "qat", qat_polling, qat_pre_init, qat_init_thread }, + { "dpdk_engine", dasync_polling, NULL, dpdk_engine_init_thread }, { "dasync", dasync_polling, NULL, NULL } }; @@ -1024,6 +1026,33 @@ tls_resume_from_crypto (int thread_index) } +void +dpdk_engine_init_thread (void *arg) +{ + vlib_main_t *vm = vlib_get_main (); + tls_main_t *tm = vnet_tls_get_main (); + void *handle = NULL; + + if (tm && tm->engine_path) + handle = dlopen ((char *) tm->engine_path, RTLD_LAZY); + + if (!handle) + { + clib_warning ("dpdk engline library not found"); + return; + } + + u64 *(*fn) () = dlsym (handle, "dpdk_rte_thread_register"); + + if (fn) + { + if (fn () < 0) + clib_warning ("dpdk: cannot register thread %u", vm->thread_index); + } + else + clib_warning ("dpdk_rte_thread_register symbol not found"); +} + static clib_error_t * tls_async_init (vlib_main_t * vm) { diff --git a/src/vnet/tls/tls.c b/src/vnet/tls/tls.c index 66fcefd93b..b2c365da2f 100644 --- a/src/vnet/tls/tls.c +++ b/src/vnet/tls/tls.c @@ -1450,6 +1450,8 @@ tls_config_fn (vlib_main_t * vm, unformat_input_t * input) } tm->fifo_size = tmp; } + else if (unformat (input, "engine-path %s", &tm->engine_path)) + ; else return clib_error_return (0, "unknown input `%U'", format_unformat_error, input); diff --git a/src/vnet/tls/tls.h b/src/vnet/tls/tls.h index c27857e58e..9658afc9b8 100644 --- a/src/vnet/tls/tls.h +++ b/src/vnet/tls/tls.h @@ -138,6 +138,7 @@ typedef struct tls_main_ u64 first_seg_size; u64 add_seg_size; u32 fifo_size; + char *engine_path; } tls_main_t; typedef struct tls_engine_vft_ From f718509be099a7a14e019ccb3af8a01730d58754 Mon Sep 17 00:00:00 2001 From: Alok Mishra Date: Tue, 24 Sep 2024 20:54:37 +0500 Subject: [PATCH 103/271] octeon: fix pause flow control for lbk/sdp devices Pause frame flow control is not supported for LBK (Loopback) and SDP (System DPI Packet Interface Unit) devices. This patch skips the pause flow configuration for these devices. Type: fix JIRA: https://essjira.marvell.com/browse/IPBUSW-52629 Signed-off-by: Alok Mishra Change-Id: I3096fcef9df4ad59d64bfabb83f91f13813128a8 Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/136273 Reviewed-by: Monendra Singh Kushwaha Tested-by: sa_ip-sw-jenkins (cherry picked from commit fc8d244c9ac93c282a9e57d41dde33bbd1f8f415) Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/136434 --- src/plugins/dev_octeon/port.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/plugins/dev_octeon/port.c b/src/plugins/dev_octeon/port.c index 84cc06fd52..9767557d30 100644 --- a/src/plugins/dev_octeon/port.c +++ b/src/plugins/dev_octeon/port.c @@ -65,8 +65,13 @@ oct_port_pause_flow_control_init (vlib_main_t *vm, vnet_dev_port_t *port) struct roc_nix_rq *rq; int rrv; + /* pause flow control is not supported on SDP/LBK devices */ if (roc_nix_is_sdp (nix) || roc_nix_is_lbk (nix)) - return VNET_DEV_ERR_UNSUPPORTED_DEVICE; + { + log_notice (dev, + "pause flow control is not supported on SDP/LBK devices"); + return VNET_DEV_OK; + } fc_cfg.type = ROC_NIX_FC_RXCHAN_CFG; fc_cfg.rxchan_cfg.enable = true; From a98597987fbd0dcb407a98daa54d6dbb9d565690 Mon Sep 17 00:00:00 2001 From: Monendra Singh Kushwaha Date: Fri, 20 Sep 2024 20:33:38 +0530 Subject: [PATCH 104/271] octeon: fix error handling for packet with error Type: fix JIRA: https://essjira.marvell.com/browse/IPBUSW-53410 Change-Id: Ieb97f1526939bcd732c155d3a7535dca71971258 Signed-off-by: Monendra Singh Kushwaha Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/136125 Tested-by: sa_ip-sw-jenkins (cherry picked from commit adb897df53e9d4b9053e295a7aa50bc37a53c7da) Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/136475 --- src/plugins/dev_octeon/rx_node.c | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/src/plugins/dev_octeon/rx_node.c b/src/plugins/dev_octeon/rx_node.c index 1f8d5d93fa..b057c4d704 100644 --- a/src/plugins/dev_octeon/rx_node.c +++ b/src/plugins/dev_octeon/rx_node.c @@ -104,7 +104,9 @@ oct_rx_batch (vlib_main_t *vm, oct_rx_node_ctx_t *ctx, { oct_rxq_t *crq = vnet_dev_get_rx_queue_data (rxq); vlib_buffer_template_t bt = rxq->buffer_template; - u32 n_left; + u32 b0_err_flags = 0, b1_err_flags = 0; + u32 b2_err_flags = 0, b3_err_flags = 0; + u32 n_left, err_flags = 0; oct_nix_rx_cqe_desc_t *d = ctx->next_desc; vlib_buffer_t *b[4]; @@ -145,6 +147,13 @@ oct_rx_batch (vlib_main_t *vm, oct_rx_node_ctx_t *ctx, oct_rx_attach_tail (vm, ctx, b[2], d + 2); oct_rx_attach_tail (vm, ctx, b[3], d + 3); } + + b0_err_flags = (d[0].parse.w[0] >> 20) & 0xFFF; + b1_err_flags = (d[1].parse.w[0] >> 20) & 0xFFF; + b2_err_flags = (d[2].parse.w[0] >> 20) & 0xFFF; + b3_err_flags = (d[3].parse.w[0] >> 20) & 0xFFF; + + err_flags |= b0_err_flags | b1_err_flags | b2_err_flags | b3_err_flags; } for (; n_left; d += 1, n_left -= 1, ctx->to_next += 1) @@ -157,11 +166,16 @@ oct_rx_batch (vlib_main_t *vm, oct_rx_node_ctx_t *ctx, ctx->n_segs += 1; if (d[0].sg0.segs > 1) oct_rx_attach_tail (vm, ctx, b[0], d + 0); + + err_flags |= ((d[0].parse.w[0] >> 20) & 0xFFF); } plt_write64 ((crq->cq.wdata | n), crq->cq.door); ctx->n_rx_pkts += n; ctx->n_left_to_next -= n; + if (err_flags) + ctx->parse_w0_or = (err_flags << 20); + return n; } From be173a9aa5a9f9f2dce440d9a5f3d47a617620da Mon Sep 17 00:00:00 2001 From: Varun Rapelly Date: Tue, 1 Oct 2024 09:11:55 +0000 Subject: [PATCH 105/271] tls: fix build issue with ssl async event type This patch fixes build issue in tls plugin with native compilation by using the correct enum type value. Type: fix JIRA: https://essjira.marvell.com/browse/IPBUSW-53659 Signed-off-by: Varun Rapelly Change-Id: I74a2bccbec09b0bab488cd349bb0a4a4d38fafcc Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/136487 Tested-by: sa_ip-sw-jenkins Reviewed-by: Venkata Ravichandra Mynidi --- src/plugins/tlsopenssl/tls_async.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plugins/tlsopenssl/tls_async.c b/src/plugins/tlsopenssl/tls_async.c index c798833447..0fe785bd53 100644 --- a/src/plugins/tlsopenssl/tls_async.c +++ b/src/plugins/tlsopenssl/tls_async.c @@ -775,7 +775,7 @@ vpp_tls_async_enqueue_event (tls_ctx_t *ctx, int evt_type, evt_run_tail = &queue[thread_index].evt_run_tail; evt_run_head = &queue[thread_index].evt_run_head; - event->type = SSL_ASYNC_INFLIGHT; + event->type = SSL_ASYNC_EVT_INIT; event->handler = (openssl_resume_handler *) sp; event->next = -1; From 5fab84889808437b2bf12de8f7a4709abf4ddd96 Mon Sep 17 00:00:00 2001 From: Monendra Singh Kushwaha Date: Tue, 1 Oct 2024 13:43:46 +0530 Subject: [PATCH 106/271] dev: add queue counter clear operation Type: feature Signed-off-by: Monendra Singh Kushwaha Change-Id: I0960625588569c81b90145ff5c81261b24650be4 Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/136479 Tested-by: sa_ip-sw-jenkins --- src/vnet/dev/dev.h | 2 ++ src/vnet/dev/port.c | 24 ++++++++++++++---------- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/src/vnet/dev/dev.h b/src/vnet/dev/dev.h index 07a99bd132..eb06eeba34 100644 --- a/src/vnet/dev/dev.h +++ b/src/vnet/dev/dev.h @@ -115,6 +115,7 @@ typedef struct vnet_dev_rx_queue_op_t *start; vnet_dev_rx_queue_op_no_rv_t *stop; vnet_dev_rx_queue_op_no_rv_t *free; + vnet_dev_rx_queue_op_no_rv_t *clear_counters; format_function_t *format_info; } vnet_dev_rx_queue_ops_t; @@ -124,6 +125,7 @@ typedef struct vnet_dev_tx_queue_op_t *start; vnet_dev_tx_queue_op_no_rv_t *stop; vnet_dev_tx_queue_op_no_rv_t *free; + vnet_dev_tx_queue_op_no_rv_t *clear_counters; format_function_t *format_info; } vnet_dev_tx_queue_ops_t; diff --git a/src/vnet/dev/port.c b/src/vnet/dev/port.c index 7450084d14..5b4b8cdc7b 100644 --- a/src/vnet/dev/port.c +++ b/src/vnet/dev/port.c @@ -734,21 +734,25 @@ void vnet_dev_port_clear_counters (vlib_main_t *vm, vnet_dev_port_t *port) { if (port->port_ops.clear_counters) - { - port->port_ops.clear_counters (vm, port); - return; - } - - if (port->counter_main) + port->port_ops.clear_counters (vm, port); + else if (port->counter_main) vnet_dev_counters_clear (vm, port->counter_main); foreach_vnet_dev_port_rx_queue (q, port) - if (q->counter_main) - vnet_dev_counters_clear (vm, q->counter_main); + { + if (port->rx_queue_ops.clear_counters) + port->rx_queue_ops.clear_counters (vm, q); + else if (q->counter_main) + vnet_dev_counters_clear (vm, q->counter_main); + } foreach_vnet_dev_port_tx_queue (q, port) - if (q->counter_main) - vnet_dev_counters_clear (vm, q->counter_main); + { + if (port->tx_queue_ops.clear_counters) + port->tx_queue_ops.clear_counters (vm, q); + else if (q->counter_main) + vnet_dev_counters_clear (vm, q->counter_main); + } log_notice (port->dev, "counters cleared on port %u", port->port_id); } From 0ca7f04daeba35efe0409a831e3e42cfb0afd115 Mon Sep 17 00:00:00 2001 From: Monendra Singh Kushwaha Date: Tue, 1 Oct 2024 13:47:29 +0530 Subject: [PATCH 107/271] octeon: add clear counters for queues Type: feature Signed-off-by: Monendra Singh Kushwaha Change-Id: Iec94247842561e11327871a45d1e24bf231037eb Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/136480 Tested-by: sa_ip-sw-jenkins --- src/plugins/dev_octeon/counter.c | 44 +++++++++++++++++++------------- 1 file changed, 26 insertions(+), 18 deletions(-) diff --git a/src/plugins/dev_octeon/counter.c b/src/plugins/dev_octeon/counter.c index 16fc9477a1..dd73684c38 100644 --- a/src/plugins/dev_octeon/counter.c +++ b/src/plugins/dev_octeon/counter.c @@ -301,30 +301,38 @@ oct_port_clear_counters (vlib_main_t *vm, vnet_dev_port_t *port) vnet_dev_t *dev = port->dev; oct_device_t *cd = vnet_dev_get_data (dev); struct roc_nix *nix = cd->nix; - oct_rxq_t *crq; - oct_txq_t *ctq; int rrv; if ((rrv = roc_nix_stats_reset (nix))) oct_roc_err (dev, rrv, "roc_nix_stats_reset() failed"); +} - foreach_vnet_dev_port_rx_queue (rxq, port) - { - crq = vnet_dev_get_rx_queue_data (rxq); +void +oct_rxq_clear_counters (vlib_main_t *vm, vnet_dev_rx_queue_t *rxq) +{ + oct_rxq_t *crq = vnet_dev_get_rx_queue_data (rxq); + vnet_dev_t *dev = rxq->port->dev; + oct_device_t *cd = vnet_dev_get_data (dev); + struct roc_nix *nix = cd->nix; + int rrv; - if ((rrv = roc_nix_stats_queue_reset (nix, crq->rq.qid, 1))) - oct_roc_err (dev, rrv, - "roc_nix_stats_queue_reset() failed for rx queue %u", - rxq->queue_id); - } + if ((rrv = roc_nix_stats_queue_reset (nix, crq->rq.qid, 1))) + oct_roc_err (dev, rrv, + "roc_nix_stats_queue_reset() failed for rx queue %u", + rxq->queue_id); +} - foreach_vnet_dev_port_tx_queue (txq, port) - { - ctq = vnet_dev_get_tx_queue_data (txq); +void +oct_txq_clear_counters (vlib_main_t *vm, vnet_dev_tx_queue_t *txq) +{ + oct_txq_t *ctq = vnet_dev_get_tx_queue_data (txq); + vnet_dev_t *dev = txq->port->dev; + oct_device_t *cd = vnet_dev_get_data (dev); + struct roc_nix *nix = cd->nix; + int rrv; - if ((rrv = roc_nix_stats_queue_reset (nix, ctq->sq.qid, 0))) - oct_roc_err (dev, rrv, - "roc_nix_stats_queue_reset() failed for tx queue %u", - txq->queue_id); - } + if ((rrv = roc_nix_stats_queue_reset (nix, ctq->sq.qid, 0))) + oct_roc_err (dev, rrv, + "roc_nix_stats_queue_reset() failed for tx queue %u", + txq->queue_id); } From f89d7d90f89ed47048e7b860a3fa7ccd76b36b10 Mon Sep 17 00:00:00 2001 From: Bheemappa Agasimundin Date: Tue, 1 Oct 2024 15:06:41 +0000 Subject: [PATCH 108/271] linux-cp: fix ipsec policy incorrect protocol type This patch changes protocol type 0 to IPSEC_POLICY_PROTOCOL_ANY to allow any transport protocol for protect/bypass. Type: fix JIRA: https://essjira.marvell.com/browse/IPBUSW-54360 Signed-off-by: Bheemappa Agasimundin Change-Id: Ic3c0ec6508a3a960127d4cf5cb5a803ed0cb707e Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/136502 Tested-by: sa_ip-sw-jenkins Reviewed-by: Monendra Singh Kushwaha (cherry picked from commit 75ca0c01793abef291e450d8c28b1a65d081485e) Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/136706 --- src/plugins/linux-cp/lcp_ipsec.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/plugins/linux-cp/lcp_ipsec.c b/src/plugins/linux-cp/lcp_ipsec.c index c847e699f8..ff6e79649c 100644 --- a/src/plugins/linux-cp/lcp_ipsec.c +++ b/src/plugins/linux-cp/lcp_ipsec.c @@ -269,7 +269,7 @@ lcp_xfrm_config_bypass_policies (u32 spd_id, u8 is_add, u8 is_ip6) update_bypass_policy_addrs (&policy); update_port_details (&policy, 0, 65535, 0, 0); policy.policy = IPSEC_POLICY_ACTION_BYPASS; - policy.protocol = 0; + policy.protocol = IPSEC_POLICY_PROTOCOL_ANY; policy.sa_id = 0; policy.is_ipv6 = is_ip6; policy.id = spd_id; @@ -338,7 +338,7 @@ lcp_xfrm_inb_policy_cfg (ip_address_t *t_saddr, ip_address_t *t_daddr, policy.policy = IPSEC_POLICY_ACTION_PROTECT; /*SA doesn't have details of inner protocol. So set 0 (means accept any)*/ - policy.protocol = 0; + policy.protocol = IPSEC_POLICY_PROTOCOL_ANY; policy.sa_id = sa_id; policy.id = spd_id; policy.priority = INB_PROTECT_POL_PRIO; @@ -1410,6 +1410,7 @@ nl_xfrm_sp_add (struct xfrmnl_sp *sp, u8 num) if (dir != XFRM_POLICY_OUT) return; + proto = !proto ? IPSEC_POLICY_PROTOCOL_ANY : proto; is_ipv6 = (fam == AF_INET6) ? 1 : 0; lcp_xfrm_mk_ipaddr (sel_dst, &sel_daddr); From fa2fec3779b77dc86611c576d741d831bcbd3724 Mon Sep 17 00:00:00 2001 From: Monendra Singh Kushwaha Date: Tue, 15 Oct 2024 15:38:44 +0530 Subject: [PATCH 109/271] octeon: add sample startup conf file Type: feature JIRA: https://essjira.marvell.com/browse/IPBUSW-54714 Signed-off-by: Monendra Singh Kushwaha Change-Id: Ia0ef71f7f333f251eebeb5179e06312fffce1552 Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/137203 Tested-by: sa_ip-sw-jenkins (cherry picked from commit 1e5ae8e608210e99514d74e226388e583877f521) Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/137387 Reviewed-by: Nithinsen Kaithakadan --- src/plugins/dev_octeon/octeon-startup.conf | 186 +++++++++++++++++++++ 1 file changed, 186 insertions(+) create mode 100644 src/plugins/dev_octeon/octeon-startup.conf diff --git a/src/plugins/dev_octeon/octeon-startup.conf b/src/plugins/dev_octeon/octeon-startup.conf new file mode 100644 index 0000000000..174826dfd4 --- /dev/null +++ b/src/plugins/dev_octeon/octeon-startup.conf @@ -0,0 +1,186 @@ +unix { + log /run/vpp/vpp.log + full-coredump + cli-listen /run/vpp/cli.sock + gid root + + ## run vpp in the interactive mode + # interactive + + ## do not use colors in terminal output + # nocolor + + ## do not display banner + # nobanner +} + +api-trace { + ## This stanza controls binary API tracing. Unless there is a very strong reason, + ## please leave this feature enabled. + on + ## Additional parameters: + ## + ## To set the number of binary API trace records in the circular buffer, configure nitems + ## + ## nitems + ## + ## To save the api message table decode tables, configure a filename. Results in /tmp/ + ## Very handy for understanding api message changes between versions, identifying missing + ## plugins, and so forth. + ## + ## save-api-table +} + +api-segment { + gid root + } + +socksvr { + #default + socket-name /run/vpp/vpp-api.sock +} + +# memory { + ## Set the main heap size, default is 1G + # main-heap-size 2G + + ## Set the main heap page size. Default page size is OS default page + ## which is in most cases 4K. if different page size is specified VPP + ## will try to allocate main heap by using specified page size. + ## special keyword 'default-hugepage' will use system default hugepage + ## size + # main-heap-page-size 1G + ## Set the default huge page size. + # default-hugepage-size 1G +#} + +cpu { + ## In the VPP there is one main thread and optionally the user can create worker(s) + ## The main thread and worker thread(s) can be pinned to CPU core(s) manually or automatically + + ## Manual pinning of thread(s) to CPU core(s) + + ## Set logical CPU core where main thread runs, if main core is not set + ## VPP will use core 1 if available + main-core 1 + + ## Set logical CPU core(s) where worker threads are running + corelist-workers 2-5 + + ## Automatic pinning of thread(s) to CPU core(s) + + ## Sets number of CPU core(s) to be skipped (1 ... N-1) + ## Skipped CPU core(s) are not used for pinning main thread and working thread(s). + ## The main thread is automatically pinned to the first available CPU core and worker(s) + ## are pinned to next free CPU core(s) after core assigned to main thread + # skip-cores 4 + + ## Specify a number of workers to be created + ## Workers are pinned to N consecutive CPU cores while skipping "skip-cores" CPU core(s) + ## and main thread's CPU core + # workers 2 + + ## Set scheduling policy and priority of main and worker threads + + ## Scheduling policy options are: other (SCHED_OTHER), batch (SCHED_BATCH) + ## idle (SCHED_IDLE), fifo (SCHED_FIFO), rr (SCHED_RR) + # scheduler-policy fifo + + ## Scheduling priority is used only for "real-time policies (fifo and rr), + ## and has to be in the range of priorities supported for a particular policy + # scheduler-priority 50 +} + +session +{ + event-queue-length 102400 +} + +buffers { + ## Increase number of buffers allocated, needed only in scenarios with + ## large number of interfaces and worker threads. Value is per numa node. + ## Default is 16384 (8192 if running unpriviledged) + buffers-per-numa 128000 + + ## Size of buffer data area + ## Default is 2048 + # default data-size 2048 + + ## Size of the memory pages allocated for buffer data + ## Default will try 'default-hugepage' then 'default' + ## you can also pass a size in K/M/G e.g. '8M' + page-size default-hugepage +} + +devices { + ## whitelist interface + dev pci/0002:04:00.0 + { + driver octeon + port 0 + { + name eth0 + num-rx-queues 4 + num-tx-queues 4 + } + } + + dev pci/0002:05:00.0 + { + driver octeon + port 0 + { + name eth1 + num-rx-queues 4 + num-tx-queues 4 + } + } +} + +plugins { + plugin dpdk_plugin.so { disable } + plugin onp_plugin.so { disable } + plugin dev_octeon_plugin.so { enable } +} + + +## Statistics Segment +statseg { + # socket-name , name of the stats segment socket + # defaults to /run/vpp/stats.sock + size 4G + # page-size , page size, ie. 2m, defaults to 4k + # per-node-counters on | off, defaults to none + # update-interval , sets the segment scrape / update interval +} + +## L2 FIB +# l2fib { + ## l2fib hash table size. + # table-size 512M + + ## l2fib hash table number of buckets. Must be power of 2. + # num-buckets 524288 +# } + +## ipsec +# { + # ip4 { + ## ipsec for ipv4 tunnel lookup hash number of buckets. + # num-buckets 524288 + # } + # ip6 { + ## ipsec for ipv6 tunnel lookup hash number of buckets. + # num-buckets 524288 + # } +# } + +# logging { + ## set default logging level for logging buffer + ## logging levels: emerg, alert,crit, error, warn, notice, info, debug, disabled + # default-log-level debug + ## set default logging level for syslog or stderr output + # default-syslog-log-level info + ## Set per-class configuration + # class dpdk/cryptodev { rate-limit 100 level debug syslog-level error } +# } From fab45de23a36a1c3aabd72e6a301be0f2972ad0f Mon Sep 17 00:00:00 2001 From: Nithinsen Kaithakadan Date: Sun, 13 Oct 2024 23:18:05 +0530 Subject: [PATCH 110/271] octeon: fix link algo crypto failures with esn This patch fixes encryption and decryption failures in case of esn enabled scenarios. Type: fix Signed-off-by: Nithinsen Kaithakadan Change-Id: I96e40c8e51dc2a9ca5edbe0137625d6386822d86 Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/137084 Tested-by: sa_ip-sw-jenkins Reviewed-by: Monendra Singh Kushwaha (cherry picked from commit c1cfb5943555b279c0184213dddb8d6be57914a0) Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/137427 Reviewed-by: Nithinsen Kaithakadan --- src/plugins/dev_octeon/crypto.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/plugins/dev_octeon/crypto.c b/src/plugins/dev_octeon/crypto.c index aab0f969d5..154defbe34 100644 --- a/src/plugins/dev_octeon/crypto.c +++ b/src/plugins/dev_octeon/crypto.c @@ -1478,8 +1478,8 @@ oct_crypto_enqueue_enc_dec (vlib_main_t *vm, vnet_crypto_async_frame_t *frame, } else { - dptr_start_ptr = (u64) (buffer->data + elts->crypto_start_offset - - elts->integ_length_adj); + dptr_start_ptr = (u64) (buffer->data + elts->integ_start_offset); + enc_auth_len = elts->crypto_total_length + elts->integ_length_adj; curr_ptr = (u64) (buffer->data + buffer->current_data); From 5ae8b79a7326123c0d9c2fc2c8d26feb2e0641c7 Mon Sep 17 00:00:00 2001 From: Monendra Singh Kushwaha Date: Wed, 9 Oct 2024 11:50:13 +0530 Subject: [PATCH 111/271] octeon: register callback to set max npa pools Type: fix Change-Id: I5d8f4f88e4d42323e63f12d4dd6bfa12bf06aef2 Signed-off-by: Monendra Singh Kushwaha Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/137085 Tested-by: sa_ip-sw-jenkins Reviewed-by: Ashwin Sekhar T K Reviewed-by: Nithin Kumar Dabilpuram Klocwork: Nithin Kumar Dabilpuram (cherry picked from commit 1fcd9dbb7aa20dae34036a7ee27f5cd563c68966) Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/138315 --- src/plugins/dev_octeon/init.c | 43 +++++++++++++++++++++++++++++++++ src/plugins/dev_octeon/octeon.h | 1 + 2 files changed, 44 insertions(+) diff --git a/src/plugins/dev_octeon/init.c b/src/plugins/dev_octeon/init.c index 2acca6d2cf..b3cc8f5371 100644 --- a/src/plugins/dev_octeon/init.c +++ b/src/plugins/dev_octeon/init.c @@ -16,6 +16,7 @@ #include struct roc_model oct_model; +u32 oct_npa_max_pools = OCT_NPA_MAX_POOLS; extern oct_crypto_main_t oct_crypto_main; VLIB_REGISTER_LOG_CLASS (oct_log, static) = { @@ -404,6 +405,13 @@ VNET_DEV_REGISTER_DRIVER (octeon) = { }, }; +static int +oct_npa_max_pools_set_cb (struct plt_pci_device *pci_dev) +{ + roc_idev_npa_maxpools_set (oct_npa_max_pools); + return 0; +} + static clib_error_t * oct_plugin_init (vlib_main_t *vm) { @@ -417,6 +425,9 @@ oct_plugin_init (vlib_main_t *vm) rv = roc_model_init (&oct_model); if (rv) return clib_error_return (0, "roc_model_init failed"); + + roc_npa_lf_init_cb_register (oct_npa_max_pools_set_cb); + return 0; } @@ -426,3 +437,35 @@ VLIB_PLUGIN_REGISTER () = { .version = VPP_BUILD_VER, .description = "dev_octeon", }; + +static clib_error_t * +oct_early_config (vlib_main_t *vm, unformat_input_t *input) +{ + unformat_input_t _line_input, *line_input = &_line_input; + clib_error_t *error = 0; + + if (!unformat_user (input, unformat_line_input, line_input)) + return 0; + + while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT) + { + if (unformat (line_input, "max-pools %u", &oct_npa_max_pools)) + ; + else + { + error = clib_error_return (0, "unknown input '%U'", + format_unformat_error, line_input); + goto done; + } + } + + if (oct_npa_max_pools < 128 || (oct_npa_max_pools > BIT_ULL (20))) + error = clib_error_return ( + 0, "Invalid max-pools value (%u), should be in range of (128 - %u)\n", + oct_npa_max_pools, BIT_ULL (20)); +done: + unformat_free (line_input); + return error; +} + +VLIB_EARLY_CONFIG_FUNCTION (oct_early_config, "dev_octeon"); diff --git a/src/plugins/dev_octeon/octeon.h b/src/plugins/dev_octeon/octeon.h index b5fd223e76..8526bb2cd4 100644 --- a/src/plugins/dev_octeon/octeon.h +++ b/src/plugins/dev_octeon/octeon.h @@ -21,6 +21,7 @@ #include #include +#define OCT_NPA_MAX_POOLS 8192 #define OCT_BATCH_ALLOC_IOVA0_MASK 0xFFFFFFFFFFFFFF80 /* From 71f1143a595ba58ef7f7f3830e27dcd94a9377ad Mon Sep 17 00:00:00 2001 From: Monendra Singh Kushwaha Date: Thu, 12 Dec 2024 13:38:08 +0530 Subject: [PATCH 112/271] build: update octeon-roc version Type: feature JIRA: https://essjira.marvell.com/browse/IPBUSW-53921 Change-Id: I58506c605465294f62fe2ab56f93c473ee5e5501 Signed-off-by: Monendra Singh Kushwaha Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/136708 Reviewed-by: Nithin Kumar Dabilpuram Reviewed-by: Anoob Joseph Klocwork: Nithin Kumar Dabilpuram (cherry picked from commit cdc552c8c0dd37480743785be0659073c43774d4) Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/138317 --- build/external/packages/octeon-roc.mk | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/build/external/packages/octeon-roc.mk b/build/external/packages/octeon-roc.mk index 98eb29b95a..eb902d1534 100644 --- a/build/external/packages/octeon-roc.mk +++ b/build/external/packages/octeon-roc.mk @@ -2,12 +2,13 @@ # SPDX-License-Identifier: Apache-2.0 # https://spdx.org/licenses/Apache-2.0.html -octeon-roc_version := 0.5 -octeon-roc_tarball := v$(octeon-roc_version).tar.gz -octeon-roc_tarball_md5sum := 76bc56c84935da944bbf340fe5283ef0 +octeon-roc_version := octeon-roc-SDK12.24.11 +octeon-roc_tarball := $(octeon-roc_version).tar.gz +octeon-roc_tarball_md5sum := a18c4bd52df9724821b42b391e45a6ea +octeon-roc_github := https://github.com/MarvellEmbeddedProcessors/marvell-octeon-roc octeon-roc_tarball_strip_dirs := 1 -octeon-roc_url := https://github.com/MarvellEmbeddedProcessors/marvell-octeon-roc/archive/refs/tags/$(octeon-roc_tarball) +octeon-roc_url := $(octeon-roc_github)/archive/refs/tags/$(octeon-roc_tarball) define octeon-roc_config_cmds @true From 1ded4d3a467ef84806073841a340c7795546a10d Mon Sep 17 00:00:00 2001 From: Monendra Singh Kushwaha Date: Fri, 11 Oct 2024 17:52:51 +0530 Subject: [PATCH 113/271] octeon: update roc function Type: fix Signed-off-by: Monendra Singh Kushwaha Change-Id: Ia055a5cbe5169c868852ecd6bf97af4bd3e557bd Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/137040 Reviewed-by: Anoob Joseph Reviewed-by: Nithin Kumar Dabilpuram Tested-by: sa_ip-sw-jenkins (cherry picked from commit 47bb577c63a81bd36c114275455c90aa6a3f003c) Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/138318 --- src/plugins/dev_octeon/init.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plugins/dev_octeon/init.c b/src/plugins/dev_octeon/init.c index b3cc8f5371..a2a9f23353 100644 --- a/src/plugins/dev_octeon/init.c +++ b/src/plugins/dev_octeon/init.c @@ -248,7 +248,7 @@ oct_conf_cpt_queue (vlib_main_t *vm, vnet_dev_t *dev, oct_crypto_dev_t *ocd) roc_cpt_iq_enable (cpt_lf); - if ((rrv = roc_cpt_lmtline_init (roc_cpt, cpt_lmtline, 0) < 0)) + if ((rrv = roc_cpt_lmtline_init (roc_cpt, cpt_lmtline, 0, false) < 0)) return cnx_return_roc_err (dev, rrv, "roc_cpt_lmtline_init"); return 0; From 737f00de6bb319491062941a0a7d1a4521e77782 Mon Sep 17 00:00:00 2001 From: Monendra Singh Kushwaha Date: Thu, 3 Oct 2024 16:15:06 +0530 Subject: [PATCH 114/271] octeon: add port argument to set allmulti mode This patch adds port argument to set allmulti mode and enables allmulti mode base on port argument. Type: feature JIRA: https://essjira.marvell.com/browse/IPBUSW-53922 Signed-off-by: Monendra Singh Kushwaha Change-Id: I88bdae1ebdf1ba7d2c1728d4dccf250a6148259d Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/136613 Tested-by: sa_ip-sw-jenkins Tested-by: Nithin Kumar Dabilpuram Reviewed-by: Nithin Kumar Dabilpuram Klocwork: Nithin Kumar Dabilpuram (cherry picked from commit ff3ae863465bf9115722363cf7e7aea768b25098) Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/138319 --- src/plugins/dev_octeon/init.c | 17 +++++++++++++++++ src/plugins/dev_octeon/octeon.h | 6 ++++++ src/plugins/dev_octeon/port.c | 14 ++++++++++++++ 3 files changed, 37 insertions(+) diff --git a/src/plugins/dev_octeon/init.c b/src/plugins/dev_octeon/init.c index a2a9f23353..1f10dc1a0c 100644 --- a/src/plugins/dev_octeon/init.c +++ b/src/plugins/dev_octeon/init.c @@ -63,6 +63,22 @@ static struct #undef _ }; +static vnet_dev_arg_t oct_port_args[] = { + { + .id = OCT_PORT_ARG_ALLMULTI_MODE, + .name = "allmulti", + .desc = "Set allmulti mode, applicable to network devices only", + .type = VNET_DEV_ARG_TYPE_BOOL, + .default_val.boolean = false, + }, + { + .id = OCT_PORT_ARG_END, + .name = "end", + .desc = "Argument end", + .type = VNET_DEV_ARG_END, + }, +}; + static u8 * oct_probe (vlib_main_t *vm, vnet_dev_bus_index_t bus_index, void *dev_info) { @@ -154,6 +170,7 @@ oct_init_nix (vlib_main_t *vm, vnet_dev_t *dev) }, .data_size = sizeof (oct_port_t), .initial_data = &oct_port, + .args = oct_port_args, }, .rx_node = &oct_rx_node, .tx_node = &oct_tx_node, diff --git a/src/plugins/dev_octeon/octeon.h b/src/plugins/dev_octeon/octeon.h index 8526bb2cd4..8e10255419 100644 --- a/src/plugins/dev_octeon/octeon.h +++ b/src/plugins/dev_octeon/octeon.h @@ -31,6 +31,12 @@ */ #define OCT_PKTIO_MAX_L2_SIZE 26 +typedef enum +{ + OCT_PORT_ARG_ALLMULTI_MODE = 1, + OCT_PORT_ARG_END +} oct_port_args_t; + typedef enum { OCT_DEVICE_TYPE_UNKNOWN = 0, diff --git a/src/plugins/dev_octeon/port.c b/src/plugins/dev_octeon/port.c index 9767557d30..2b2fdea698 100644 --- a/src/plugins/dev_octeon/port.c +++ b/src/plugins/dev_octeon/port.c @@ -164,6 +164,20 @@ oct_port_init (vlib_main_t *vm, vnet_dev_port_t *port) } } + /* Enable allmulti mode, if set by arg */ + foreach_vnet_dev_port_args (arg, port) + { + if (arg->id == OCT_PORT_ARG_ALLMULTI_MODE) + { + if ((rrv = roc_nix_npc_mcast_config (nix, 1, 0))) + { + oct_port_deinit (vm, port); + return oct_roc_err (dev, rrv, "roc_nix_mac_addr_set failed"); + } + break; + } + } + if ((rrv = roc_nix_tm_init (nix))) { oct_port_deinit (vm, port); From 15a116afcf9e8e4f81459c13930bc2330ace1e89 Mon Sep 17 00:00:00 2001 From: Kishor Dhanawade Date: Thu, 26 Sep 2024 15:48:34 +0530 Subject: [PATCH 115/271] octeon: add support for hmac_md5 and chachapoly Added support for following algorithms - aes-cbc hmac_md5 - chacha20_poly1305 Type: feature JIRA: https://essjira.marvell.com/browse/IPBUSW-50926 Signed-off-by: Kishor Dhanawade Signed-off-by: Nithinsen Kaithakadan Change-Id: I44702483dad8182d5f15aed39c6bb42f1ca15d3c Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/136363 Tested-by: sa_ip-sw-jenkins Reviewed-by: Tejasree Kondoj Reviewed-by: Nithinsen Kaithakadan (cherry picked from commit e48d6cf4cd69904349681ae5acffe0e4eaa12ffc) Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/138320 Reviewed-by: Monendra Singh Kushwaha --- src/plugins/dev_octeon/crypto.c | 28 ++++++++++++++++++++++++++++ src/plugins/dev_octeon/crypto.h | 12 +++++++++++- 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/src/plugins/dev_octeon/crypto.c b/src/plugins/dev_octeon/crypto.c index 154defbe34..a00cd839cc 100644 --- a/src/plugins/dev_octeon/crypto.c +++ b/src/plugins/dev_octeon/crypto.c @@ -1223,6 +1223,13 @@ oct_crypto_link_session_update (vlib_main_t *vm, oct_crypto_sess_t *sess, auth_type = ROC_SE_SHA2_SHA512; digest_len = 32; break; + case VNET_CRYPTO_ALG_AES_128_CBC_MD5_TAG12: + case VNET_CRYPTO_ALG_AES_192_CBC_MD5_TAG12: + case VNET_CRYPTO_ALG_AES_256_CBC_MD5_TAG12: + enc_type = ROC_SE_AES_CBC; + auth_type = ROC_SE_MD5_TYPE; + digest_len = 12; + break; case VNET_CRYPTO_ALG_AES_128_CTR_SHA1_TAG12: case VNET_CRYPTO_ALG_AES_192_CTR_SHA1_TAG12: case VNET_CRYPTO_ALG_AES_256_CTR_SHA1_TAG12: @@ -1324,6 +1331,10 @@ oct_crypto_aead_session_update (vlib_main_t *vm, oct_crypto_sess_t *sess, sess->cpt_op = type; digest_len = 16; break; + case VNET_CRYPTO_ALG_CHACHA20_POLY1305: + enc_type = ROC_SE_CHACHA20; + auth_type = ROC_SE_POLY1305; + break; default: log_err (ocd->dev, "Crypto: Undefined cipher algo %u specified. Key index %u", @@ -1348,6 +1359,9 @@ oct_crypto_aead_session_update (vlib_main_t *vm, oct_crypto_sess_t *sess, return -1; } + if (enc_type == ROC_SE_CHACHA20) + sess->cpt_ctx.template_w4.s.opcode_minor |= BIT (5); + return 0; } @@ -1563,6 +1577,13 @@ oct_crypto_enqueue_aead_aad_12_enc (vlib_main_t *vm, return oct_crypto_enqueue_aead_aad_enc (vm, frame, 12); } +int +oct_crypto_enqueue_aead_aad_0_enc (vlib_main_t *vm, + vnet_crypto_async_frame_t *frame) +{ + return oct_crypto_enqueue_aead_aad_enc (vm, frame, 0); +} + int oct_crypto_enqueue_aead_aad_8_dec (vlib_main_t *vm, vnet_crypto_async_frame_t *frame) @@ -1577,6 +1598,13 @@ oct_crypto_enqueue_aead_aad_12_dec (vlib_main_t *vm, return oct_crypto_enqueue_aead_aad_dec (vm, frame, 12); } +int +oct_crypto_enqueue_aead_aad_0_dec (vlib_main_t *vm, + vnet_crypto_async_frame_t *frame) +{ + return oct_crypto_enqueue_aead_aad_dec (vm, frame, 0); +} + vnet_crypto_async_frame_t * oct_crypto_frame_dequeue (vlib_main_t *vm, u32 *nb_elts_processed, u32 *enqueue_thread_idx) diff --git a/src/plugins/dev_octeon/crypto.h b/src/plugins/dev_octeon/crypto.h index 6cc9f39782..19d37c4caf 100644 --- a/src/plugins/dev_octeon/crypto.h +++ b/src/plugins/dev_octeon/crypto.h @@ -28,7 +28,10 @@ _ (AES_192_GCM, 24, 16, 8) \ _ (AES_192_GCM, 24, 16, 12) \ _ (AES_256_GCM, 32, 16, 8) \ - _ (AES_256_GCM, 32, 16, 12) + _ (AES_256_GCM, 32, 16, 12) \ + _ (CHACHA20_POLY1305, 32, 16, 8) \ + _ (CHACHA20_POLY1305, 32, 16, 12) \ + _ (CHACHA20_POLY1305, 32, 16, 0) /* CRYPTO_ID, INTEG_ID, KEY_LENGTH_IN_BYTES, DIGEST_LEN */ #define foreach_oct_crypto_link_async_alg \ @@ -44,6 +47,9 @@ _ (AES_128_CBC, SHA512, 16, 32) \ _ (AES_192_CBC, SHA512, 24, 32) \ _ (AES_256_CBC, SHA512, 32, 32) \ + _ (AES_128_CBC, MD5, 16, 12) \ + _ (AES_192_CBC, MD5, 24, 12) \ + _ (AES_256_CBC, MD5, 32, 12) \ _ (3DES_CBC, MD5, 24, 12) \ _ (3DES_CBC, SHA1, 24, 12) \ _ (3DES_CBC, SHA256, 24, 16) \ @@ -178,10 +184,14 @@ int oct_crypto_enqueue_aead_aad_8_enc (vlib_main_t *vm, vnet_crypto_async_frame_t *frame); int oct_crypto_enqueue_aead_aad_12_enc (vlib_main_t *vm, vnet_crypto_async_frame_t *frame); +int oct_crypto_enqueue_aead_aad_0_enc (vlib_main_t *vm, + vnet_crypto_async_frame_t *frame); int oct_crypto_enqueue_aead_aad_8_dec (vlib_main_t *vm, vnet_crypto_async_frame_t *frame); int oct_crypto_enqueue_aead_aad_12_dec (vlib_main_t *vm, vnet_crypto_async_frame_t *frame); +int oct_crypto_enqueue_aead_aad_0_dec (vlib_main_t *vm, + vnet_crypto_async_frame_t *frame); vnet_crypto_async_frame_t *oct_crypto_frame_dequeue (vlib_main_t *vm, u32 *nb_elts_processed, u32 *enqueue_thread_idx); From aa8d946cd1067f20fbcc8579867ff77a7f7b75fe Mon Sep 17 00:00:00 2001 From: Varun Rapelly Date: Thu, 24 Oct 2024 09:04:55 +0000 Subject: [PATCH 116/271] tls: configurable test cert/key User may want to test applications that use TLS using none RSA Private key and Certificate (for example ECDSA/DSS). VPP uses RSA key/cert pair embedded in code, this commit adds the alternative of replacing the RSA key/cert pair with user defined key/cert files. New configuration files should be used in the tls configuration section tls { test-srv-crt-path test-srv-key-path } Type: feature Signed-off-by: Varun Rapelly JIRA: https://essjira.marvell.com/browse/IPBUSW-53659 Change-Id: I386bc240b849a7695208167478b7311c7a5449e0 Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/137990 Tested-by: sa_ip-sw-jenkins Reviewed-by: Venkata Ravichandra Mynidi --- src/vnet/session/application.c | 18 ++++++++ src/vnet/session/application_interface.h | 1 + src/vnet/tls/tls.c | 52 ++++++++++++++++++++++++ src/vnet/tls/tls.h | 2 + 4 files changed, 73 insertions(+) diff --git a/src/vnet/session/application.c b/src/vnet/session/application.c index 2c69138931..8d82293a07 100644 --- a/src/vnet/session/application.c +++ b/src/vnet/session/application.c @@ -21,6 +21,24 @@ static app_main_t app_main; +/* + * test Certificate and Private key can be overwritten when using + * test-srv-crt-path and test-srv-key-path tls config section. + * If not specified, RSA Certificate and Private key are in use. + */ +static vnet_app_add_cert_key_pair_args_t test_srv_key_pair = { + .cert = (u8 *) test_srv_crt_rsa, + .cert_len = sizeof (test_srv_crt_rsa), + .key = (u8 *) test_srv_key_rsa, + .key_len = sizeof (test_srv_key_rsa) +}; + +vnet_app_add_cert_key_pair_args_t * +vnet_app_tls_get_test_srv_key_pair (void) +{ + return &test_srv_key_pair; +} + #define app_interface_check_thread_and_barrier(_fn, _arg) \ if (PREDICT_FALSE (!vlib_thread_is_main_w_barrier ())) \ { \ diff --git a/src/vnet/session/application_interface.h b/src/vnet/session/application_interface.h index b41f7a4c43..d223021329 100644 --- a/src/vnet/session/application_interface.h +++ b/src/vnet/session/application_interface.h @@ -290,6 +290,7 @@ int vnet_app_add_cert_key_pair (vnet_app_add_cert_key_pair_args_t * a); int vnet_app_del_cert_key_pair (u32 index); /** Ask for app cb on pair deletion */ int vnet_app_add_cert_key_interest (u32 index, u32 app_index); +vnet_app_add_cert_key_pair_args_t *vnet_app_tls_get_test_srv_key_pair (void); uword unformat_vnet_uri (unformat_input_t *input, va_list *args); diff --git a/src/vnet/tls/tls.c b/src/vnet/tls/tls.c index b2c365da2f..86c6ac76ed 100644 --- a/src/vnet/tls/tls.c +++ b/src/vnet/tls/tls.c @@ -15,6 +15,7 @@ #include #include +#include #include static tls_main_t tls_main; @@ -1388,6 +1389,27 @@ tls_register_engine (const tls_engine_vft_t * vft, crypto_engine_type_t type) tls_vfts[type] = *vft; } +static clib_error_t * +tls_get_test_file_len_content (char *file_name, u32 *len, u8 **content) +{ + clib_error_t *error; + uword n_bytes; + u8 *rv = 0; + + error = clib_file_n_bytes (file_name, &n_bytes); + if (error) + return error; + + error = clib_file_contents (file_name, &rv); + if (error) + return error; + + *len = n_bytes; + *content = rv; + + return error; +} + static clib_error_t * tls_init (vlib_main_t * vm) { @@ -1452,6 +1474,36 @@ tls_config_fn (vlib_main_t * vm, unformat_input_t * input) } else if (unformat (input, "engine-path %s", &tm->engine_path)) ; + else if (unformat (input, "test-srv-crt-path %s", + &tm->test_srv_crt_path)) + { + clib_error_t *err; + vnet_app_add_cert_key_pair_args_t *ck_pair = + vnet_app_tls_get_test_srv_key_pair (); + + err = tls_get_test_file_len_content ( + tm->test_srv_crt_path, &ck_pair->cert_len, &ck_pair->cert); + if (err) + { + vec_free (tm->test_srv_crt_path); + return err; + } + } + else if (unformat (input, "test-srv-key-path %s", + &tm->test_srv_key_path)) + { + clib_error_t *err; + vnet_app_add_cert_key_pair_args_t *ck_pair = + vnet_app_tls_get_test_srv_key_pair (); + + err = tls_get_test_file_len_content ( + tm->test_srv_key_path, &ck_pair->key_len, &ck_pair->key); + if (err) + { + vec_free (tm->test_srv_key_path); + return err; + } + } else return clib_error_return (0, "unknown input `%U'", format_unformat_error, input); diff --git a/src/vnet/tls/tls.h b/src/vnet/tls/tls.h index 9658afc9b8..741d775b9a 100644 --- a/src/vnet/tls/tls.h +++ b/src/vnet/tls/tls.h @@ -139,6 +139,8 @@ typedef struct tls_main_ u64 add_seg_size; u32 fifo_size; char *engine_path; + char *test_srv_key_path; + char *test_srv_crt_path; } tls_main_t; typedef struct tls_engine_vft_ From 9e8ddb18742de3b1035ada1a7766d0db7d33cd77 Mon Sep 17 00:00:00 2001 From: Varun Rapelly Date: Tue, 13 Aug 2024 11:22:28 +0000 Subject: [PATCH 117/271] http_static_l4: initial static http l4 code base A simple caching static http/https server with a built-in vpp host stack application. CLI configuration to enable static l4 server: http static l4 server www-root [prealloc-fifos ] [private-segment-size ] [fifo-size ] [uri ] Type: feature Signed-off-by: Ofer Heifetz JIRA: https://essjira.marvell.com/browse/IPBUSW-53659 Change-Id: I3e9833ceeabe57ac40af2be3075d31914d46c817 Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/137991 Tested-by: sa_ip-sw-jenkins Reviewed-by: Venkata Ravichandra Mynidi --- MAINTAINERS | 6 + src/plugins/http_static_l4/CMakeLists.txt | 26 + src/plugins/http_static_l4/FEATURE.yaml | 10 + src/plugins/http_static_l4/http_static.c | 84 + src/plugins/http_static_l4/http_static.h | 236 ++ src/plugins/http_static_l4/http_static_l4.api | 35 + src/plugins/http_static_l4/http_static_test.c | 135 ++ src/plugins/http_static_l4/static_server.c | 2072 +++++++++++++++++ 8 files changed, 2604 insertions(+) create mode 100644 src/plugins/http_static_l4/CMakeLists.txt create mode 100644 src/plugins/http_static_l4/FEATURE.yaml create mode 100644 src/plugins/http_static_l4/http_static.c create mode 100644 src/plugins/http_static_l4/http_static.h create mode 100644 src/plugins/http_static_l4/http_static_l4.api create mode 100644 src/plugins/http_static_l4/http_static_test.c create mode 100644 src/plugins/http_static_l4/static_server.c diff --git a/MAINTAINERS b/MAINTAINERS index abd8e5bb97..9802e3cb50 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -465,6 +465,12 @@ M: Dave Barach M: Florin Coras F: src/plugins/http_static/ +Plugin - http_static l4 +I: http_static_l4 +M: Dave Barach +M: Florin Coras +F: src/plugins/http_static_l4/ + Plugin - builtinurl I: builtinurl M: Dave Barach diff --git a/src/plugins/http_static_l4/CMakeLists.txt b/src/plugins/http_static_l4/CMakeLists.txt new file mode 100644 index 0000000000..845a849907 --- /dev/null +++ b/src/plugins/http_static_l4/CMakeLists.txt @@ -0,0 +1,26 @@ + +# Copyright (c) +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +add_vpp_plugin(http_static_l4 + SOURCES + http_static.c + static_server.c + http_static.h + + API_FILES + http_static_l4.api + + API_TEST_SOURCES + http_static_test.c +) diff --git a/src/plugins/http_static_l4/FEATURE.yaml b/src/plugins/http_static_l4/FEATURE.yaml new file mode 100644 index 0000000000..b99f2419db --- /dev/null +++ b/src/plugins/http_static_l4/FEATURE.yaml @@ -0,0 +1,10 @@ +--- +name: Static http https l4 server +maintainer: Dave Barach +features: + - An extensible static http/https server with caching +description: "A simple caching static http / https server + A built-in vpp host stack application. + Supports HTTP GET and HTTP POST methods." +state: production +properties: [API, CLI, MULTITHREAD] diff --git a/src/plugins/http_static_l4/http_static.c b/src/plugins/http_static_l4/http_static.c new file mode 100644 index 0000000000..5a87af507e --- /dev/null +++ b/src/plugins/http_static_l4/http_static.c @@ -0,0 +1,84 @@ +/* + * http_static.c - skeleton vpp engine plug-in + * + * Copyright (c) + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include + +#include +#include +#include + +/* define message IDs */ +#include +#include + +#include + +#define vl_print(handle, ...) vlib_cli_output (handle, __VA_ARGS__) + +#define REPLY_MSG_ID_BASE hmp->msg_id_base +#include + +http_static_l4_main_t http_static_l4_main; + +/* API message handler */ +static void +vl_api_http_static_l4_enable_t_handler (vl_api_http_static_l4_enable_t *mp) +{ + vl_api_http_static_l4_enable_reply_t *rmp; + http_static_l4_main_t *hmp = &http_static_l4_main; + int rv; + + mp->uri[ARRAY_LEN (mp->uri) - 1] = 0; + mp->www_root[ARRAY_LEN (mp->www_root) - 1] = 0; + + rv = http_static_l4_server_enable ( + ntohl (mp->fifo_size), ntohl (mp->cache_size_limit), + ntohl (mp->prealloc_fifos), ntohl (mp->private_segment_size), mp->www_root, + mp->uri); + + REPLY_MACRO (VL_API_HTTP_STATIC_L4_ENABLE_REPLY); +} + +#include +static clib_error_t * +http_static_l4_init (vlib_main_t *vm) +{ + http_static_l4_main_t *hmp = &http_static_l4_main; + + hmp->vlib_main = vm; + hmp->vnet_main = vnet_get_main (); + + /* Ask for a correctly-sized block of API message decode slots */ + hmp->msg_id_base = setup_message_id_table (); + + return 0; +} + +VLIB_INIT_FUNCTION (http_static_l4_init); + +VLIB_PLUGIN_REGISTER () = { .version = VPP_BUILD_VER, + .description = "HTTP Static l4 Server" }; + +/* + * fd.io coding-style-patch-verification: ON + * + * Local Variables: + * eval: (c-set-style "gnu") + * End: + */ diff --git a/src/plugins/http_static_l4/http_static.h b/src/plugins/http_static_l4/http_static.h new file mode 100644 index 0000000000..5fe92b5325 --- /dev/null +++ b/src/plugins/http_static_l4/http_static.h @@ -0,0 +1,236 @@ + +/* + * http_static.h - skeleton vpp engine plug-in header file + * + * Copyright (c) + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef __included_http_static_l4_h__ +#define __included_http_static_l4_h__ + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +/** @file http_static.h + * Static http server definitions + */ + +#define HTTP_STAT_LINE_LEN 64 +#define HTTP_PATH_LEN 10 + +typedef struct +{ + /* API message ID base */ + u16 msg_id_base; + + /* convenience */ + vlib_main_t *vlib_main; + vnet_main_t *vnet_main; +} http_static_l4_main_t; + +extern http_static_l4_main_t http_static_l4_main; + +/** \brief Session States + */ + +typedef enum +{ + /** Session is closed */ + HTTP_STATE_CLOSED, + /** Session is established */ + HTTP_STATE_ESTABLISHED, + /** Session has more data to receive */ + HTTP_STATE_RECEIVE_MORE_DATA, + /** Session has sent an OK response */ + HTTP_STATE_OK_SENT, + /** Session has sent an HTML response */ + HTTP_STATE_SEND_MORE_DATA, + /** Session has sent file, closes the connection */ + HTTP_STATE_CLOSING, + /** Number of states */ + HTTP_STATE_N_STATES, +} http_session_state_t; + +typedef enum +{ + CALLED_FROM_RX, + CALLED_FROM_TX, + CALLED_FROM_TIMER, +} http_state_machine_called_from_t; + +typedef enum +{ + HTTP_BUILTIN_METHOD_GET = 0, + HTTP_BUILTIN_METHOD_POST, +} http_builtin_method_type_t; + +/** \brief Application session + */ +typedef struct +{ + CLIB_CACHE_LINE_ALIGN_MARK (cacheline0); + /** Base class instance variables */ +#define _(type, name) type name; + foreach_app_session_field +#undef _ + /** rx thread index */ + u32 thread_index; + /** rx buffer */ + u8 *rx_buf; + /** vpp session index, handle */ + u32 vpp_session_index; + u64 vpp_session_handle; + /** Timeout timer handle */ + u32 timer_handle; + /** Fully-resolved file path */ + u8 path[HTTP_PATH_LEN]; + /** File data, a vector */ + u8 *data; + /** Current data send offset */ + u32 data_offset; + /** Need to free data in detach_cache_entry */ + int free_data; + + /** File cache pool index */ + u32 cache_pool_index; + /** state machine called from... */ + http_state_machine_called_from_t called_from; +} http_session_t; + +/** \brief In-memory file data cache entry + */ +typedef struct +{ + /** Name of the file */ + u8 filename[HTTP_PATH_LEN]; + /** Contents of the file, as a u8 * vector */ + u8 *data; + /** Last time the cache entry was used */ + f64 last_used; + /** Cache LRU links */ + u32 next_index; + u32 prev_index; + /** Reference count, so we don't recycle while referenced */ + int inuse; +} file_data_cache_t; + +/** \brief Main data structure + */ + +typedef struct +{ + /** Per thread vector of session pools */ + http_session_t **sessions; + /** Per thread Session pool reader writer lock */ + clib_rwlock_t *thr_sessions_lock; + /** vpp session to http session index map */ + u32 **session_to_http_session; + + u8 ***rx_buf_pool; + /** Enable debug messages */ + int debug_level; + + /** vpp message/event queue */ + svm_msg_q_t **vpp_queue; + + /** Unified file data cache pool */ + file_data_cache_t *cache_pool; + /** Hash table which maps file name to file data */ + BVT (clib_bihash) name_to_data; + + /** Hash tables for built-in GET and POST handlers */ + uword *get_url_handlers; + uword *post_url_handlers; + + /** Current cache size */ + u64 cache_size; + /** Max cache size in bytes */ + u64 cache_limit; + /** Number of cache evictions */ + u64 cache_evictions; + + /** Cache LRU listheads */ + u32 first_index; + u32 last_index; + + /** root path to be served */ + u8 *www_root; + + /** filename for index.html redirect */ + char redirect_file_name[HTTP_PATH_LEN]; + + /** Server's event queue */ + svm_queue_t *vl_input_queue; + + /** API client handle */ + u32 my_client_index; + + /** Application index */ + u32 app_index; + + /** Process node index for event scheduling */ + u32 node_index; + + /** Cert and key pair for tls */ + u32 ckpair_index; + + /** Session cleanup timer wheel */ + tw_timer_wheel_2t_1w_2048sl_t tw; + clib_spinlock_t tw_lock; + + /** Time base, so we can generate browser cache control http spew */ + clib_timebase_t timebase; + + /** Number of preallocated fifos, usually 0 */ + u32 prealloc_fifos; + /** Private segment size, usually 0 */ + u64 private_segment_size; + /** Size of the allocated rx, tx fifos, roughly 8K or so */ + u32 fifo_size; + /** fifo dequeue threshold */ + u32 fifo_deq_thresh; + /** The bind URI, defaults to tcp://0.0.0.0/80 */ + u8 *uri; + /** Keep Connection Alive after file fetch is done, default=0 */ + u8 keepalive; + vlib_main_t *vlib_main; +} http_static_l4_server_main_t; + +extern http_static_l4_server_main_t http_static_l4_server_main; + +int http_static_l4_server_enable (u32 fifo_size, u32 cache_limit, + u32 prealloc_fifos, u32 private_segment_size, + u8 *www_root, u8 *uri); + +void http_static_l4_server_register_builtin_handler (void *fp, char *url, + int type); + +#endif /* __included_http_static_l4_h__ */ + +/* + * fd.io coding-style-patch-verification: ON + * + * Local Variables: + * eval: (c-set-style "gnu") + * End: + */ diff --git a/src/plugins/http_static_l4/http_static_l4.api b/src/plugins/http_static_l4/http_static_l4.api new file mode 100644 index 0000000000..a43bb0079d --- /dev/null +++ b/src/plugins/http_static_l4/http_static_l4.api @@ -0,0 +1,35 @@ + +/** \file + This file defines static http l4 server control-plane API messages +*/ +option version = "2.1.0"; + +/** \brief Configure and enable the static http l4 server + @param client_index - opaque cookie to identify the sender + @param context - sender context, to match reply w/ request + @param fifo_size - size (in bytes) of the session FIFOs + @param cache_size_limit - size (in bytes) of the in-memory file data cache + @param prealloc_fifos - number of preallocated fifos (usually 0) + @param private_segment_size - fifo segment size (usually 0) + @param www_root - html root path + @param uri - bind URI, defaults to "tcp://0.0.0.0/80" +*/ + +autoreply define http_static_l4_enable { + /* Client identifier, set from api_main.my_client_index */ + u32 client_index; + + /* Arbitrary context, so client can match reply to request */ + u32 context; + /* Typical options */ + u32 fifo_size; + u32 cache_size_limit; + /* Unusual options */ + u32 prealloc_fifos; + u32 private_segment_size; + + /* Root of the html path */ + string www_root[256]; + /* The bind URI */ + string uri[256]; +}; diff --git a/src/plugins/http_static_l4/http_static_test.c b/src/plugins/http_static_l4/http_static_test.c new file mode 100644 index 0000000000..7d784a2810 --- /dev/null +++ b/src/plugins/http_static_l4/http_static_test.c @@ -0,0 +1,135 @@ +/* + * http_static.c - skeleton vpp-api-test plug-in + * + * Copyright (c) + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include +#include +#include + +uword unformat_sw_if_index (unformat_input_t *input, va_list *args); + +/* Declare message IDs */ +#include +#include + +typedef struct +{ + /* API message ID base */ + u16 msg_id_base; + vat_main_t *vat_main; +} http_static_l4_test_main_t; + +http_static_l4_test_main_t http_static_l4_test_main; + +#define __plugin_msg_base http_static_l4_test_main.msg_id_base +#include + +static int +api_http_static_l4_enable (vat_main_t *vam) +{ + unformat_input_t *line_input = vam->input; + vl_api_http_static_l4_enable_t *mp; + u64 tmp; + u8 *www_root = 0; + u8 *uri = 0; + u32 prealloc_fifos = 0; + u32 private_segment_size = 0; + u32 fifo_size = 8 << 10; + u32 cache_size_limit = 1 << 20; + int ret; + + /* Parse args required to build the message */ + while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT) + { + if (unformat (line_input, "www-root %s", &www_root)) + ; + else if (unformat (line_input, "prealloc-fifos %d", &prealloc_fifos)) + ; + else if (unformat (line_input, "private-segment-size %U", + unformat_memory_size, &tmp)) + { + if (tmp >= 0x100000000ULL) + { + errmsg ("private segment size %llu, too large", tmp); + return -99; + } + private_segment_size = (u32) tmp; + } + else if (unformat (line_input, "fifo-size %U", unformat_memory_size, + &tmp)) + { + if (tmp >= 0x100000000ULL) + { + errmsg ("fifo-size %llu, too large", tmp); + return -99; + } + fifo_size = (u32) tmp; + } + else if (unformat (line_input, "cache-size %U", unformat_memory_size, + &tmp)) + { + if (tmp < (128ULL << 10)) + { + errmsg ("cache-size must be at least 128kb"); + return -99; + } + cache_size_limit = (u32) tmp; + } + + else if (unformat (line_input, "uri %s", &uri)) + ; + else + { + errmsg ("unknown input `%U'", format_unformat_error, line_input); + return -99; + } + } + + if (www_root == 0) + { + errmsg ("Must specify www-root"); + return -99; + } + + if (uri == 0) + uri = format (0, "tcp://0.0.0.0/80%c", 0); + + /* Construct the API message */ + M (HTTP_STATIC_L4_ENABLE, mp); + strncpy_s ((char *) mp->www_root, 256, (const char *) www_root, 256); + strncpy_s ((char *) mp->uri, 256, (const char *) uri, 256); + mp->fifo_size = ntohl (fifo_size); + mp->cache_size_limit = ntohl (cache_size_limit); + mp->prealloc_fifos = ntohl (prealloc_fifos); + mp->private_segment_size = ntohl (private_segment_size); + + /* send it... */ + S (mp); + + /* Wait for a reply... */ + W (ret); + return ret; +} + +#include + +/* + * fd.io coding-style-patch-verification: ON + * + * Local Variables: + * eval: (c-set-style "gnu") + * End: + */ diff --git a/src/plugins/http_static_l4/static_server.c b/src/plugins/http_static_l4/static_server.c new file mode 100644 index 0000000000..718e1d8216 --- /dev/null +++ b/src/plugins/http_static_l4/static_server.c @@ -0,0 +1,2072 @@ +/* + * Copyright (c) 2017-2019 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#define HTTP_CONNECT "Connection:" + +#define HTTP_CONN_LEN (sizeof (HTTP_CONNECT) - 1) +#define HTTP_CONN_KA "Keep-Alive" +#define HTTP_CONN_CL "Close" +#define HTTP_CONN_CT_L "Content-Length: " +#define HTTP_VER_1_0 "HTTP/1.0" + +#define HTTP_BODY_PREFIX "\r\n\r\n" +#define HTTP_201 "HTTP/1.1 201 OK\r\n" +#define HTTP_201_LEN (sizeof (HTTP_201) - 1) +#define HTTP_200 "HTTP/1.1 200 OK\r\n" +#define HTTP_200_LEN (sizeof (HTTP_200) - 1) + +#define IND_HTML "index.html" +#define IND_HTML_LEN (sizeof (IND_HTML) - 1) + +#define HTTP_SESSION_CLOSED INT_MAX + +/** @file static_server.c + * Static http server, sufficient to + * serve .html / .css / .js content. + */ +/*? %%clicmd:group_label Static HTTP Server %% ?*/ + +#define HTTP_FIFO_DEF_THRESH (64 << 10) + +http_static_l4_server_main_t http_static_l4_server_main; + +/** \brief Format the called-from enum + */ + +static u8 * +format_state_machine_called_from (u8 *s, va_list *args) +{ + http_state_machine_called_from_t cf = + va_arg (*args, http_state_machine_called_from_t); + char *which = "bogus!"; + + switch (cf) + { + case CALLED_FROM_RX: + which = "from rx"; + break; + case CALLED_FROM_TX: + which = "from tx"; + break; + case CALLED_FROM_TIMER: + which = "from timer"; + break; + + default: + break; + } + + s = format (s, "%s", which); + return s; +} + +/** \brief Acquire reader lock on the sessions pools + */ +static void +http_static_server_thr_sessions_reader_lock (u32 thread_index) +{ + clib_rwlock_reader_lock ( + &http_static_l4_server_main.thr_sessions_lock[thread_index]); +} + +/** \brief Drop reader lock on the sessions pools + */ +static void +http_static_server_thr_sessions_reader_unlock (u32 thread_index) +{ + clib_rwlock_reader_unlock ( + &http_static_l4_server_main.thr_sessions_lock[thread_index]); +} + +/** \brief Acquire writer lock on the sessions pools + */ +static void +http_static_server_thr_sessions_writer_lock (u32 thread_index) +{ + clib_rwlock_writer_lock ( + &http_static_l4_server_main.thr_sessions_lock[thread_index]); +} + +/** \brief Drop writer lock on the sessions pools + */ +static void +http_static_server_thr_sessions_writer_unlock (u32 thread_index) +{ + clib_rwlock_writer_unlock ( + &http_static_l4_server_main.thr_sessions_lock[thread_index]); +} + +static void +http_static_server_all_sessions_writer_unlock (void) +{ + u32 thr_index; + http_static_l4_server_main_t *hsm = &http_static_l4_server_main; + + /* Lock sessions of all threads */ + for (thr_index = 0; thr_index <= vec_len (hsm->thr_sessions_lock); + thr_index++) + http_static_server_thr_sessions_writer_unlock (thr_index); +} + +static void +http_static_server_all_sessions_writer_lock (void) +{ + u32 thr_index; + http_static_l4_server_main_t *hsm = &http_static_l4_server_main; + + /* Lock sessions of all threads */ + for (thr_index = 0; thr_index <= vec_len (hsm->thr_sessions_lock); + thr_index++) + http_static_server_thr_sessions_writer_lock (thr_index); +} + +static void +http_static_server_all_sessions_reader_unlock (void) +{ + u32 thr_index; + http_static_l4_server_main_t *hsm = &http_static_l4_server_main; + + /* Unlock sessions of all threads */ + for (thr_index = 0; thr_index <= vec_len (hsm->thr_sessions_lock); + thr_index++) + http_static_server_thr_sessions_reader_unlock (thr_index); +} + +static void +http_static_server_all_sessions_reader_lock (void) +{ + u32 thr_index; + http_static_l4_server_main_t *hsm = &http_static_l4_server_main; + + /* Lock sessions of all threads */ + for (thr_index = 0; thr_index <= vec_len (hsm->thr_sessions_lock); + thr_index++) + http_static_server_thr_sessions_reader_lock (thr_index); +} + +/** \brief Start a session cleanup timer + */ +static void +http_static_server_session_timer_start (http_session_t *hs) +{ + http_static_l4_server_main_t *hsm = &http_static_l4_server_main; + u32 hs_handle; + + /* The session layer may fire a callback at a later date... */ + if (!pool_is_free (hsm->sessions[hs->thread_index], hs)) + { + hs_handle = hs->thread_index << 24 | hs->session_index; + clib_spinlock_lock (&http_static_l4_server_main.tw_lock); + hs->timer_handle = tw_timer_start_2t_1w_2048sl ( + &http_static_l4_server_main.tw, hs_handle, 0, 60); + clib_spinlock_unlock (&http_static_l4_server_main.tw_lock); + } +} + +/** \brief stop a session cleanup timer + */ +static void +http_static_server_session_timer_stop (http_session_t *hs) +{ + if (hs->timer_handle == ~0) + return; + clib_spinlock_lock (&http_static_l4_server_main.tw_lock); + tw_timer_stop_2t_1w_2048sl (&http_static_l4_server_main.tw, + hs->timer_handle); + clib_spinlock_unlock (&http_static_l4_server_main.tw_lock); +} + +/** \brief Allocate an http session + */ +static http_session_t * +http_static_server_session_alloc (u32 thread_index) +{ + http_static_l4_server_main_t *hsm = &http_static_l4_server_main; + http_session_t *hs = NULL; + + pool_get_aligned_zero (hsm->sessions[thread_index], hs, 0); + hs->session_index = hs - hsm->sessions[thread_index]; + hs->thread_index = thread_index; + hs->timer_handle = ~0; + hs->cache_pool_index = ~0; + return hs; +} + +/** \brief Get an http session by index + */ +static http_session_t * +http_static_server_session_get (u32 thread_index, u32 hs_index) +{ + http_static_l4_server_main_t *hsm = &http_static_l4_server_main; + if (pool_is_free_index (hsm->sessions[thread_index], hs_index)) + return 0; + return pool_elt_at_index (hsm->sessions[thread_index], hs_index); +} + +/** \brief Free an http session + */ +static void +http_static_server_session_free (http_session_t *hs) +{ + http_static_l4_server_main_t *hsm = &http_static_l4_server_main; + + /* Make sure the timer is stopped... */ + http_static_server_session_timer_stop (hs); + pool_put (hsm->sessions[hs->thread_index], hs); + + if (CLIB_DEBUG) + { + u32 save_thread_index; + save_thread_index = hs->thread_index; + /* Poison the entry, preserve timer state and thread index */ + memset (hs, 0xfa, sizeof (*hs)); + hs->timer_handle = ~0; + hs->thread_index = save_thread_index; + } +} + +/** \brief add a session to the vpp < -- > http session index map + */ +static void +http_static_server_session_lookup_add (u32 thread_index, u32 s_index, + u32 hs_index) +{ + http_static_l4_server_main_t *hsm = &http_static_l4_server_main; + vec_validate (hsm->session_to_http_session[thread_index], s_index); + hsm->session_to_http_session[thread_index][s_index] = hs_index; +} + +/** \brief Remove a session from the vpp < -- > http session index map + */ +static void +http_static_server_session_lookup_del (u32 thread_index, u32 s_index) +{ + http_static_l4_server_main_t *hsm = &http_static_l4_server_main; + hsm->session_to_http_session[thread_index][s_index] = ~0; +} + +/** \brief lookup a session in the vpp < -- > http session index map + */ + +static http_session_t * +http_static_server_session_lookup (u32 thread_index, u32 s_index) +{ + http_static_l4_server_main_t *hsm = &http_static_l4_server_main; + u32 hs_index; + + if (s_index < vec_len (hsm->session_to_http_session[thread_index])) + { + hs_index = hsm->session_to_http_session[thread_index][s_index]; + return http_static_server_session_get (thread_index, hs_index); + } + return 0; +} + +/** \brief Detach cache entry from session + */ + +static void +http_static_server_detach_cache_entry (http_session_t *hs) +{ + http_static_l4_server_main_t *hsm = &http_static_l4_server_main; + file_data_cache_t *ep; + + /* + * Decrement cache pool entry reference count + * Note that if e.g. a file lookup fails, the cache pool index + * won't be set + */ + if (hs->cache_pool_index != ~0) + { + ep = pool_elt_at_index (hsm->cache_pool, hs->cache_pool_index); + ep->inuse--; + if (hsm->debug_level > 1) + clib_warning ("index %d refcnt now %d", hs->cache_pool_index, + ep->inuse); + } + hs->cache_pool_index = ~0; + if (hs->free_data) + vec_free (hs->data); + hs->data = 0; + hs->data_offset = 0; + hs->free_data = 0; + hs->path[0] = 0; +} + +/** \brief Disconnect a session + */ +static void +http_static_server_session_disconnect (http_session_t *hs) +{ + vnet_disconnect_args_t _a = { 0 }, *a = &_a; + a->handle = hs->vpp_session_handle; + a->app_index = http_static_l4_server_main.app_index; + vnet_disconnect_session (a); +} + +/** \brief http error boilerplate + */ +static const char *http_error_template = "HTTP/1.1 %s\r\n" + "Date: %U GMT\r\n" + "Content-Type: text/html\r\n" + "Connection: close\r\n" + "Pragma: no-cache\r\n" + "Content-Length: 0\r\n\r\n"; + +#define HTTP_RESPONSE_STR_MAX_SZ 512 +/** \brief http response boilerplate + */ +static const char *http_response_template = "Date: %s GMT\r\n" + "Expires: %s GMT\r\n" + "Server: VPP Static\r\n" + "Content-Type: %s\r\n" + "Content-Length: %d\r\n\r\n"; + +/** \brief receive http data + @param hs - http session + @return -1 failed, 0 for success, 1 partly successful. +*/ +static u32 +static_receive_data (http_session_t *hs) +{ + u32 max_dequeue; + int n_read; + + max_dequeue = svm_fifo_max_dequeue (hs->rx_fifo); + if (PREDICT_FALSE (max_dequeue == 0)) + { + return -1; + } + + n_read = app_recv_stream_raw (hs->rx_fifo, &hs->data[hs->data_offset], + max_dequeue, 0, 0 /* peek */); + hs->data_offset += n_read; + if (n_read != max_dequeue) + clib_warning ("WARNING: max_dequeue %d bytes while read only %d bytes", + max_dequeue, n_read); + if (!svm_fifo_is_empty (hs->rx_fifo)) + return 1; + + svm_fifo_unset_event (hs->rx_fifo); + return 0; +} + +/** \brief send http data + @param hs - http session + @param data - the data vector to transmit + @param length - length of data + @param offset - transmit offset for this operation + @return offset for next transmit operation, may be unchanged w/ full fifo +*/ + +static u32 +static_send_data (http_session_t *hs, u8 *data, u32 length, u32 offset) +{ + u32 bytes_to_send; + http_static_l4_server_main_t *hsm = &http_static_l4_server_main; + + bytes_to_send = length - offset; + + while (bytes_to_send > 0) + { + int actual_transfer; + + actual_transfer = svm_fifo_enqueue ( + hs->tx_fifo, clib_min (bytes_to_send, 4 << 20), data + offset); + + /* Made any progress? */ + if (actual_transfer <= 0) + { + if (hsm->debug_level > 0 && bytes_to_send > 0) + clib_warning ("WARNING: still %d bytes to send", bytes_to_send); + return offset; + } + else + { + offset += actual_transfer; + bytes_to_send -= actual_transfer; + + if (hsm->debug_level && bytes_to_send > 0) + clib_warning ("WARNING: still %d bytes to send", bytes_to_send); + + if (svm_fifo_set_event (hs->tx_fifo)) + session_send_io_evt_to_thread (hs->tx_fifo, + SESSION_IO_EVT_TX_FLUSH); + return offset; + } + } + /* NOTREACHED */ + return ~0; +} + +/** \brief Send an http error string + @param hs - the http session + @param str - the error string, e.g. "404 Not Found" +*/ +static void +send_error (http_session_t *hs, char *str) +{ + http_static_l4_server_main_t *hsm = &http_static_l4_server_main; + u8 *data; + f64 now; + + now = clib_timebase_now (&hsm->timebase); + data = format (0, http_error_template, str, format_clib_timebase_time, now); + static_send_data (hs, data, vec_len (data), 0); + vec_free (data); +} + +/** \brief Retrieve data from the application layer + */ +static int +session_rx_request (http_session_t *hs) +{ + http_static_l4_server_main_t *hsm = &http_static_l4_server_main; + u32 max_dequeue, cursize; + int n_read; + + max_dequeue = svm_fifo_max_dequeue (hs->rx_fifo); + if (PREDICT_FALSE (max_dequeue == 0)) + return -1; + + cursize = vec_len (hs->rx_buf); + if (vec_mem_size (hs->rx_buf) == 0) + vec_pop2 (hsm->rx_buf_pool[hs->thread_index], hs->rx_buf); + + vec_validate (hs->rx_buf, cursize + max_dequeue - 1); + n_read = app_recv_stream_raw (hs->rx_fifo, hs->rx_buf + cursize, max_dequeue, + 0, 0 /* peek */); + ASSERT (n_read == max_dequeue); + if (svm_fifo_is_empty (hs->rx_fifo)) + svm_fifo_unset_event (hs->rx_fifo); + + vec_set_len (hs->rx_buf, cursize + n_read); + return 0; +} + +/** \brief Sanity-check the forward and reverse LRU lists + */ +static inline void +lru_validate (http_static_l4_server_main_t *hsm) +{ +#if CLIB_DEBUG > 0 + f64 last_timestamp; + u32 index; + int i; + file_data_cache_t *ep; + + last_timestamp = 1e70; + for (i = 1, index = hsm->first_index; index != ~0;) + { + ep = pool_elt_at_index (hsm->cache_pool, index); + index = ep->next_index; + /* Timestamps should be smaller (older) as we walk the fwd list */ + if (ep->last_used > last_timestamp) + { + clib_warning ("%d[%d]: last used %.6f, last_timestamp %.6f", + ep - hsm->cache_pool, i, ep->last_used, + last_timestamp); + } + last_timestamp = ep->last_used; + i++; + } + + last_timestamp = 0.0; + for (i = 1, index = hsm->last_index; index != ~0;) + { + ep = pool_elt_at_index (hsm->cache_pool, index); + index = ep->prev_index; + /* Timestamps should be larger (newer) as we walk the rev list */ + if (ep->last_used < last_timestamp) + { + clib_warning ("%d[%d]: last used %.6f, last_timestamp %.6f", + ep - hsm->cache_pool, i, ep->last_used, + last_timestamp); + } + last_timestamp = ep->last_used; + i++; + } +#endif +} + +/** \brief Remove a data cache entry from the LRU lists + */ +static inline void +lru_remove (http_static_l4_server_main_t *hsm, file_data_cache_t *ep) +{ + file_data_cache_t *next_ep, *prev_ep; + u32 ep_index; + + lru_validate (hsm); + + ep_index = ep - hsm->cache_pool; + + /* Deal with list heads */ + if (ep_index == hsm->first_index) + hsm->first_index = ep->next_index; + if (ep_index == hsm->last_index) + hsm->last_index = ep->prev_index; + + /* Fix next->prev */ + if (ep->next_index != ~0) + { + next_ep = pool_elt_at_index (hsm->cache_pool, ep->next_index); + next_ep->prev_index = ep->prev_index; + } + /* Fix prev->next */ + if (ep->prev_index != ~0) + { + prev_ep = pool_elt_at_index (hsm->cache_pool, ep->prev_index); + prev_ep->next_index = ep->next_index; + } + lru_validate (hsm); +} + +/** \brief Add an entry to the LRU lists, tag w/ supplied timestamp + */ +#if 0 +static inline void +lru_add (http_static_l4_server_main_t *hsm, file_data_cache_t *ep, f64 now) +{ + file_data_cache_t *next_ep; + u32 ep_index; + + lru_validate (hsm); + + ep_index = ep - hsm->cache_pool; + + /* + * Re-add at the head of the forward LRU list, + * tail of the reverse LRU list + */ + if (hsm->first_index != ~0) + { + next_ep = pool_elt_at_index (hsm->cache_pool, hsm->first_index); + next_ep->prev_index = ep_index; + } + + ep->prev_index = ~0; + + /* ep now the new head of the LRU forward list */ + ep->next_index = hsm->first_index; + hsm->first_index = ep_index; + + /* single session case: also the tail of the reverse LRU list */ + if (hsm->last_index == ~0) + hsm->last_index = ep_index; + ep->last_used = now; + + lru_validate (hsm); +} +#endif +/** \brief Remove and re-add a cache entry from/to the LRU lists + */ +#if 0 +static inline void +lru_update (http_static_l4_server_main_t *hsm, file_data_cache_t *ep, f64 now) +{ + lru_remove (hsm, ep); + lru_add (hsm, ep, now); +} +#endif +/** \brief Session-layer (main) data rx callback. + Parse the http request, and reply to it. + Future extensions might include POST processing, active content, etc. +*/ + +/* svm_fifo_add_want_deq_ntf (tx_fifo, SVM_FIFO_WANT_DEQ_NOTIF_IF_FULL) +get shoulder-tap when transport dequeues something, set in +xmit routine. */ + +/** \brief closed state - should never really get here + */ +static int +state_closed (session_t *s, http_session_t *hs, + http_state_machine_called_from_t cf) +{ + clib_warning ("WARNING: http session %d, called from %U", hs->session_index, + format_state_machine_called_from, cf); + return -1; +} + +static void +close_session (http_session_t *hs) +{ + http_static_server_session_timer_stop (hs); + hs->timer_handle = ~0; + http_static_server_session_disconnect (hs); +} + +/** \brief Register a builtin GET or POST handler + */ +__clib_export void +http_static_l4_server_register_builtin_handler (void *fp, char *url, + int request_type) +{ + http_static_l4_server_main_t *hsm = &http_static_l4_server_main; + uword *p, *builtin_table; + + builtin_table = (request_type == HTTP_BUILTIN_METHOD_GET) ? + hsm->get_url_handlers : + hsm->post_url_handlers; + + p = hash_get_mem (builtin_table, url); + + if (p) + { + clib_warning ("WARNING: attempt to replace handler for %s '%s' ignored", + (request_type == HTTP_BUILTIN_METHOD_GET) ? "GET" : "POST", + url); + return; + } + + hash_set_mem (builtin_table, url, (uword) fp); + + /* + * Need to update the hash table pointer in http_static_l4_server_main + * in case we just expanded it... + */ + if (request_type == HTTP_BUILTIN_METHOD_GET) + hsm->get_url_handlers = builtin_table; + else + hsm->post_url_handlers = builtin_table; +} + +static int +v_find_index (u8 *vec, char *str) +{ + int start_index; + u32 slen = (u32) strnlen_s_inline (str, 16); + u32 vlen = vec_len (vec); + + ASSERT (slen > 0); + + if (vlen <= slen) + return -1; + + for (start_index = 0; start_index < (vlen - slen); start_index++) + { + if (!memcmp (&vec[start_index], str, slen)) + return start_index; + } + + return -1; +} + +/** \brief Same func as above (v_find_index) just case-insensitive. + */ +static int +v_find_index_insensitive (u8 *vec, char *str, u32 start_pos) +{ + int start_index; + u32 slen = (u32) strnlen_s_inline (str, 16); + u32 vlen = vec_len (vec); + + ASSERT (slen > 0); + + if (PREDICT_FALSE (vlen - start_pos <= slen)) + return -1; + + for (start_index = start_pos; start_index < (vlen - slen); start_index++) + { + if (!strncasecmp ((const char *) &vec[start_index], str, slen)) + return start_index; + } + + return -1; +} + +/* Support files in following format : N[NN]c c=B|K|M, i.e. 1B,22K,789M */ +static u32 +parse_fileformat (u8 *vec, int *digits) +{ + u32 n_bytes = 0; + char *endptr; + + /* Avoid vpp format() function, it uses spinlocks. */ + if (vec != NULL) + { + n_bytes = (u32) strtol ((char *) vec, &endptr, 10); + } + else + clib_warning ("vec is null"); + if (n_bytes == 0) + return 0; + + *digits = endptr - (char *) vec; + + switch (vec[*digits]) + { + case 'B': + break; + case 'K': + n_bytes <<= 10; + break; + case 'M': + n_bytes <<= 20; + break; + default: + n_bytes = 0; + } + return n_bytes; +} + +/** \brief established state - waiting for GET, POST, etc. + */ +static int +state_established (session_t *s, http_session_t *hs, + http_state_machine_called_from_t cf) +{ + http_static_l4_server_main_t *hsm = &http_static_l4_server_main; + u8 *request = 0; + int i, c, p, rv; + u8 *nrp = NULL; /* new request pointer */ + u8 request_type = HTTP_BUILTIN_METHOD_GET; + int digits = 0; + u32 n_bytes = 0, num_lines, remainder; + u32 pcl = 0; /* Post Content Length */ + u32 pfc = 0; /* Post First Content */ + const char *line = + "001 002 003 004 005 006 007 008 009 010 011 012 013 014 015 016\n"; + u8 line_len = HTTP_STAT_LINE_LEN; + u8 *v = NULL; + u8 *key = NULL; + u8 *endptr; + + ASSERT (strlen (line) == HTTP_STAT_LINE_LEN); + + /* Read data from the sessison layer */ + rv = session_rx_request (hs); + + /* No data? Odd, but stay in this state and await further instructions */ + if (rv) + return 0; + + /* Process the client request */ + request = hs->rx_buf; + if (vec_len (request) < 8) + { + if (hsm->debug_level > 1) + clib_warning ("http request:%s too short", request); + send_error (hs, "400 Bad Request"); + close_session (hs); + return -1; + } + + if ((i = v_find_index (request, "GET ")) >= 0) + goto find_end; + else if ((i = v_find_index (request, "POST ")) >= 0) + { + request_type = HTTP_BUILTIN_METHOD_POST; + goto find_end; + } + + if (hsm->debug_level > 1) + clib_warning ("Unknown http method"); + + send_error (hs, "405 Method Not Allowed"); + close_session (hs); + return -1; + +find_end: + /* Lose "GET /" or "POST /" */ + vec_delete (request, + (request_type == HTTP_BUILTIN_METHOD_GET) ? + (i + sizeof ("GET /") - 1) : + (i + sizeof ("POST /") - 1), + 0); + /* Check the HTTP version, for keepalive enabling needs. */ + if (v_find_index (request, HTTP_VER_1_0) >= 0) + hsm->keepalive = 0; + else + hsm->keepalive = 1; + + /* The header connection request, define the respond */ + if ((c = v_find_index (request, HTTP_CONNECT)) >= 0) + { + if (v_find_index_insensitive (request, HTTP_CONN_KA, + c + HTTP_CONN_LEN) >= 0) + hsm->keepalive = 1; + else if (v_find_index_insensitive (request, HTTP_CONN_CL, + c + HTTP_CONN_LEN) >= 0) + hsm->keepalive = 0; + } + + /* Lose "GET /" or "POST /" */ + vec_delete (request, i + (sizeof ("GET /") - 1) + request_type, 0); + + /* Replace 'index.html' file with redirect_file_name + * Relies on sizeof(redirect_file_name) < "index.html" + */ + if (v_find_index (request, "index.html") == 0 && + vec_len (hsm->redirect_file_name)) + + if (request_type == HTTP_BUILTIN_METHOD_POST) + { + /* keep size of file by Content-Length for next rx queue msg read */ + if ((p = v_find_index_insensitive (request, HTTP_CONN_CT_L, + i + sizeof ("POST "))) >= 0) + { + nrp = &request[p + sizeof (HTTP_CONN_CT_L) - 1]; + pcl = (int) strtol ((char *) nrp, (char **) &endptr, 10); + digits = endptr - nrp; + if ((pcl == 0) || (digits > 8)) /* max support ~95MB. */ + { + clib_warning ("Requested file :%s failed length parsing)", + nrp); + close_session (hs); + return -1; + } + } + else + { + clib_warning ("POST length file:%s is invalid\n", nrp); + close_session (hs); + return -1; + } + if ((p = v_find_index_insensitive (nrp, HTTP_BODY_PREFIX, + (u32) (nrp - request))) < 0) + { + clib_warning ("POST body request was not found:%s is invalid\n", + request); + close_session (hs); + return -1; + } + else + { + nrp = nrp + p + sizeof (HTTP_BODY_PREFIX) - 1; + } + pfc = (vec_len (request) - (nrp - request)); + } + + /* find or read the file if we haven't done so yet. */ + if (hs->data == 0) + { + BVT (clib_bihash_kv) kv; + file_data_cache_t *dp; + u8 pl = v_find_index (request, " "); /* pl = path length */ + pl = clib_min (pl, sizeof (hs->path)); + + memset (hs->path, 0, sizeof (hs->path)); + strncpy ((char *) hs->path, (const char *) request, pl); + /* key must be vector for hash purpose */ + vec_resize (key, pl); + clib_memcpy_fast (key, hs->path, pl); + kv.key = (u64) key; + if (hsm->debug_level > 1) + clib_warning ("Using '%s' key for lookup table.\n", kv.key); + + if (hsm->debug_level > 1) + clib_warning ("hs->path:%s, Post Content Length:%d", hs->path, pcl); + + /* If the path name already exists in the cache, or not. */ + if (BV (clib_bihash_search) (&hsm->name_to_data, &kv, &kv) == 0) + { + if (hsm->debug_level > 1) + clib_warning ("lookup '%s' returned %lld", kv.key, kv.value); + + /* found the data.. */ + dp = pool_elt_at_index (hsm->cache_pool, kv.value); + hs->data = dp->data; + /* Update the cache entry, mark it in-use */ + hs->cache_pool_index = dp - hsm->cache_pool; + dp->inuse++; + if (hsm->debug_level > 1) + clib_warning ("index %d refcnt now %d", hs->cache_pool_index, + dp->inuse); + /* If POST request, need to put the replace info */ + if (request_type == HTTP_BUILTIN_METHOD_POST) + { + u32 min, max = 0; + min = clib_min (pcl, vec_bytes (dp->data)); + max = clib_max (pcl, vec_bytes (dp->data)); + vec_delete (dp->data, (u32) (max - min), min); + clib_memcpy_fast (dp->data, nrp, pfc); + /* check if all data has been copied or if there is more data to + * receive */ + if (pcl == pfc) + { + static_send_data (hs, (u8 *) "HTTP/1.1 201 OK\r\n", 17, 0); + hs->session_state = HTTP_STATE_OK_SENT; + } + else + { + hs->data_offset = pfc; + hs->session_state = HTTP_STATE_RECEIVE_MORE_DATA; + } + } + } + else + { + if (hsm->debug_level > 1) + clib_warning ("lookup '%s' failed", kv.key); + + /* if file not found in cache, GET request with format N[NN]C can be + * accepted. */ + if (request_type == HTTP_BUILTIN_METHOD_GET) + { + n_bytes = (int) strtol ((char *) request, (char **) &endptr, 10); + digits = endptr - request; + if ((n_bytes == 0) || (digits > 3)) + { + clib_warning ("Requested file:%s is invalid. valid format " + "is \"N[NN]C\", N{0-9} C{B|K|M}.\n", + request); + close_session (hs); + return -1; + } + + switch (request[digits]) + { + case 'B': + break; + case 'K': + n_bytes <<= 10; + break; + case 'M': + n_bytes <<= 20; + break; + default: + clib_warning ("Requested file:%s is invalid", request); + close_session (hs); + return -1; + } + + /* No recycling, fail if exceeding limit */ + if ((hsm->cache_size + n_bytes) > hsm->cache_limit) + { + clib_warning ("ERROR: cache-size:%llu + file-size:%u may " + "not exceed cache-limit:%llu", + hsm->cache_size, n_bytes, hsm->cache_limit); + close_session (hs); + return -1; + } + /* Read the "file" into memory, 64B lines */ + vec_resize (v, n_bytes); + num_lines = n_bytes / line_len; + remainder = n_bytes % line_len; + for (i = 0; i < num_lines; i++) + clib_memcpy_fast (v + i * line_len, line, strlen (line)); + clib_memcpy_fast (v + i * line_len, line, remainder); + } + else if (request_type == HTTP_BUILTIN_METHOD_POST) + { + /* pcl is the total content to be received by the POST request, + pfc is the current available data (not includes headers). */ + vec_resize (v, pcl); + clib_memcpy_fast (v, nrp, pfc); + if (pcl == pfc) + { + static_send_data (hs, (u8 *) "HTTP/1.1 201 OK\r\n", 17, 0); + hs->session_state = HTTP_STATE_OK_SENT; + } + else + { + hs->session_state = HTTP_STATE_RECEIVE_MORE_DATA; + } + } + + hs->data = v; + + /* Create a cache entry for it */ + pool_get (hsm->cache_pool, dp); + memset (dp, 0, sizeof (*dp)); + strcpy ((char *) dp->filename, (char *) hs->path); + dp->data = hs->data; + hs->cache_pool_index = dp - hsm->cache_pool; + dp->inuse++; + if (hsm->debug_level > 1) + clib_warning ("index %d refcnt now %d", hs->cache_pool_index, + dp->inuse); + /* clib_bihash_kv_vec8_8_t compares vecs, so in current code 'key' + * must be a vec. Creating a vec here does not effect performance, + * since this is the cache-miss slowpath. + */ + kv.value = dp - hsm->cache_pool; + /* Add to the lookup table */ + if (hsm->debug_level > 1) + clib_warning ("add '%s' value %lld", kv.key, kv.value); + + if (BV (clib_bihash_add_del) (&hsm->name_to_data, &kv, + 1 /* is_add */) < 0) + { + clib_warning ("BUG: add failed!"); + } + hsm->cache_size += vec_len (dp->data); + } + if (hs->session_state == HTTP_STATE_OK_SENT) + { + hs->data_offset = 0; + } + else if (hs->session_state == HTTP_STATE_RECEIVE_MORE_DATA) + { + hs->data_offset = pfc; + } + } + /* Keep vec-len-reset, because it was done in original code. */ + vec_reset_length (hs->rx_buf); + if (hs->session_state == HTTP_STATE_ESTABLISHED || + hs->session_state == HTTP_STATE_OK_SENT) + { + hs->session_state = HTTP_STATE_OK_SENT; + /* send 200 OK first */ + static_send_data (hs, (u8 *) "HTTP/1.1 200 OK\r\n", 17, 0); + } + + return 1; +} + +/** \brief receive more data state - aggregate more data from Rx fifo into the + * allocated data cache. In this state we manage data receive progress. In case + * no more data has been received after a while, we reattach the data cache + * page. To receive more data we will use the static_receive_data function. + * */ +static int +state_receive_more_data (session_t *s, http_session_t *hs, + http_state_machine_called_from_t cf) +{ + int rv; + http_static_l4_server_main_t *hsm = &http_static_l4_server_main; + /* Continue receives data */ + rv = static_receive_data (hs); + if (rv != 0) + { + return rv; + } + if (hs->data_offset < vec_len (hs->data)) + { + /* No: ask for a shoulder-tap when the rx fifo has more data */ + svm_fifo_add_want_deq_ntf (hs->rx_fifo, SVM_FIFO_WANT_DEQ_NOTIF); + hs->session_state = HTTP_STATE_RECEIVE_MORE_DATA; + return 0; + } + if (hs->data_offset == vec_len (hs->data)) + { + /* send 201 OK first */ + static_send_data (hs, (u8 *) "HTTP/1.1 201 OK\r\n", 17, 0); + hs->session_state = HTTP_STATE_OK_SENT; + } + + /* Let go of the file cache entry */ + http_static_server_detach_cache_entry (hs); + + /* Finished with this receive, move to CLOSING (no keepalive), + * or back to ESTABLISHED (keepalive). */ + if (hsm->keepalive) + hs->session_state = HTTP_STATE_ESTABLISHED; + else + { + svm_fifo_add_want_deq_ntf (hs->tx_fifo, + SVM_FIFO_WANT_DEQ_NOTIF_IF_EMPTY); + hs->session_state = HTTP_STATE_CLOSING; + } + + return 0; +} + +static int +state_send_more_data (session_t *s, http_session_t *hs, + http_state_machine_called_from_t cf) +{ + http_static_l4_server_main_t *hsm = &http_static_l4_server_main; + /* Start sending data */ + hs->data_offset = + static_send_data (hs, hs->data, vec_len (hs->data), hs->data_offset); + + /* Did we finish? */ + if (hs->data_offset < vec_len (hs->data)) + { + /* No: ask for a shoulder-tap when the tx fifo has space */ + svm_fifo_add_want_deq_ntf (hs->tx_fifo, SVM_FIFO_WANT_DEQ_NOTIF); + hs->session_state = HTTP_STATE_SEND_MORE_DATA; + return 0; + } + /* Finished with this transaction, move to CLOSING (no keepalive), + * or back to ESTABLISHED (keepalive). + */ + + /* Let go of the file cache entry */ + http_static_server_detach_cache_entry (hs); + if (hsm->keepalive) + hs->session_state = HTTP_STATE_ESTABLISHED; + else + { + svm_fifo_add_want_deq_ntf (hs->tx_fifo, + SVM_FIFO_WANT_DEQ_NOTIF_IF_EMPTY); + hs->session_state = HTTP_STATE_CLOSING; + } + + return 0; +} + +static int +state_sent_ok (session_t *s, http_session_t *hs, + http_state_machine_called_from_t cf) +{ + http_static_l4_server_main_t *hsm = &http_static_l4_server_main; + char *http_type; + u8 http_response[HTTP_RESPONSE_STR_MAX_SZ]; + f64 now; + u32 offset; + u8 date[CLIB_TIMEBASE_STR_MAX_SZ], expire[CLIB_TIMEBASE_STR_MAX_SZ]; + + http_type = "text/html"; + + if (hs->data == 0) + { + clib_warning ("BUG: hs->data not set for session %d", hs->session_index); + close_session (hs); + return 0; + } + + /* + * Send an http response, which needs the current time, + * the expiration time, and the data length + */ + now = clib_timebase_now (&hsm->timebase); + sprintf_clib_timebase_time (date, now); + sprintf_clib_timebase_time (expire, now + 600.0); + + sprintf ((char *) http_response, http_response_template, (char *) date, + (char *) expire, http_type, vec_len (hs->data)); + offset = + static_send_data (hs, http_response, strlen ((char *) http_response), 0); + if (offset != strlen ((char *) http_response)) + { + clib_warning ("BUG: couldn't send response header!"); + close_session (hs); + return 0; + } + + /* Send data from the beginning... */ + hs->data_offset = 0; + hs->session_state = HTTP_STATE_SEND_MORE_DATA; + return 1; +} + +static int +state_closing (session_t *s, http_session_t *hs, + http_state_machine_called_from_t cf) +{ + close_session (hs); + return HTTP_SESSION_CLOSED; +} + +static void *state_funcs[HTTP_STATE_N_STATES] = { + state_closed, + /* Waiting for GET, POST, etc. */ + state_established, + /* Received more data */ + state_receive_more_data, + /* Sent OK */ + state_sent_ok, + /* Send more data */ + state_send_more_data, + /* Sent all data, closing connection */ + state_closing +}; + +static inline int +http_static_server_rx_tx_callback (session_t *s, + http_state_machine_called_from_t cf) +{ + http_session_t *hs; + int (*fp) (session_t *, http_session_t *, http_state_machine_called_from_t); + int rv; + + /* Acquire a reader lock on the session table */ + http_static_server_thr_sessions_reader_lock (s->thread_index); + hs = http_static_server_session_lookup (s->thread_index, s->session_index); + + if (!hs) + { + clib_warning ("No http session for thread %d session_index %d", + s->thread_index, s->session_index); + http_static_server_thr_sessions_reader_unlock (s->thread_index); + return 0; + } + + /* Execute state machine for this session */ + do + { + fp = state_funcs[hs->session_state]; + rv = (*fp) (s, hs, cf); + if (rv < 0 || rv == HTTP_SESSION_CLOSED) + goto session_closed; + } + while (rv); + + /* Reset the session expiration timer */ + http_static_server_session_timer_stop (hs); + http_static_server_session_timer_start (hs); + +session_closed: + http_static_server_thr_sessions_reader_unlock (s->thread_index); + return 0; +} + +static int +http_static_server_rx_callback (session_t *s) +{ + return http_static_server_rx_tx_callback (s, CALLED_FROM_RX); +} + +static int +http_static_server_tx_callback (session_t *s) +{ + return http_static_server_rx_tx_callback (s, CALLED_FROM_TX); +} + +/** \brief Session accept callback + */ + +static int +http_static_server_session_accept_callback (session_t *s) +{ + http_static_l4_server_main_t *hsm = &http_static_l4_server_main; + http_session_t *hs; + u32 thresh; + + hsm->vpp_queue[s->thread_index] = + session_main_get_vpp_event_queue (s->thread_index); + + http_static_server_thr_sessions_writer_lock (s->thread_index); + + hs = http_static_server_session_alloc (s->thread_index); + http_static_server_session_lookup_add (s->thread_index, s->session_index, + hs->session_index); + hs->rx_fifo = s->rx_fifo; + hs->tx_fifo = s->tx_fifo; + hs->vpp_session_index = s->session_index; + hs->vpp_session_handle = session_handle (s); + hs->session_state = HTTP_STATE_ESTABLISHED; + http_static_server_session_timer_start (hs); + + http_static_server_thr_sessions_writer_unlock (s->thread_index); + + /* The application sets a threshold for it's fifo to get notified when + * additional data can be enqueued. We want to keep the TX fifo reasonably + * full, however avoid entering a state where the fifo is full all the time + * and small chunks of data are being enqueued each time. + * If the fifo and threshold use the same size, this means that a + * notification will be given when the fifo empties. + */ + thresh = svm_fifo_size (hs->tx_fifo) - hsm->fifo_deq_thresh; + svm_fifo_set_deq_thresh (hs->tx_fifo, thresh); + + s->session_state = SESSION_STATE_READY; + return 0; +} + +/** \brief Session disconnect callback + */ + +static void +http_static_server_session_disconnect_callback (session_t *s) +{ + http_static_l4_server_main_t *hsm = &http_static_l4_server_main; + vnet_disconnect_args_t _a = { 0 }, *a = &_a; + + a->handle = session_handle (s); + a->app_index = hsm->app_index; + vnet_disconnect_session (a); +} + +/** \brief Session reset callback + */ + +static void +http_static_server_session_reset_callback (session_t *s) +{ + http_static_l4_server_main_t *hsm = &http_static_l4_server_main; + vnet_disconnect_args_t _a = { 0 }, *a = &_a; + + a->handle = session_handle (s); + a->app_index = hsm->app_index; + vnet_disconnect_session (a); +} + +static int +http_static_server_session_connected_callback (u32 app_index, u32 api_context, + session_t *s, + session_error_t err) +{ + clib_warning ("called..."); + return -1; +} + +static int +http_static_server_add_segment_callback (u32 client_index, u64 segment_handle) +{ + return 0; +} + +static void +http_static_session_cleanup (session_t *s, session_cleanup_ntf_t ntf) +{ + http_session_t *hs; + http_static_l4_server_main_t *hsm = &http_static_l4_server_main; + + if (ntf == SESSION_CLEANUP_TRANSPORT) + return; + + http_static_server_thr_sessions_writer_lock (s->thread_index); + + hs = http_static_server_session_lookup (s->thread_index, s->session_index); + if (!hs) + goto done; + + http_static_server_detach_cache_entry (hs); + http_static_server_session_lookup_del (hs->thread_index, + hs->vpp_session_index); + + vec_reset_length (hs->rx_buf); + vec_add1 (hsm->rx_buf_pool[hs->thread_index], hs->rx_buf); + http_static_server_session_free (hs); + +done: + http_static_server_thr_sessions_writer_unlock (s->thread_index); +} + +/** \brief Session-layer virtual function table + */ +static session_cb_vft_t http_static_server_session_cb_vft = { + .session_accept_callback = http_static_server_session_accept_callback, + .session_disconnect_callback = + http_static_server_session_disconnect_callback, + .session_connected_callback = http_static_server_session_connected_callback, + .add_segment_callback = http_static_server_add_segment_callback, + .builtin_app_rx_callback = http_static_server_rx_callback, + .builtin_app_tx_callback = http_static_server_tx_callback, + .session_reset_callback = http_static_server_session_reset_callback, + .session_cleanup_callback = http_static_session_cleanup, +}; + +static int +http_static_server_attach () +{ + vnet_app_add_cert_key_pair_args_t *ck_pair = + vnet_app_tls_get_test_srv_key_pair (); + http_static_l4_server_main_t *hsm = &http_static_l4_server_main; + u64 options[APP_OPTIONS_N_OPTIONS]; + vnet_app_attach_args_t _a, *a = &_a; + u64 segment_size = 128 << 20; + + clib_memset (a, 0, sizeof (*a)); + clib_memset (options, 0, sizeof (options)); + + if (hsm->private_segment_size) + segment_size = hsm->private_segment_size; + + a->api_client_index = ~0; + a->name = format (0, "test_http_static_server"); + a->session_cb_vft = &http_static_server_session_cb_vft; + a->options = options; + a->options[APP_OPTIONS_SEGMENT_SIZE] = segment_size; + a->options[APP_OPTIONS_ADD_SEGMENT_SIZE] = segment_size; + a->options[APP_OPTIONS_RX_FIFO_SIZE] = + hsm->fifo_size ? hsm->fifo_size : 8 << 10; + a->options[APP_OPTIONS_TX_FIFO_SIZE] = + hsm->fifo_size ? hsm->fifo_size : 32 << 10; + a->options[APP_OPTIONS_FLAGS] = APP_OPTIONS_FLAGS_IS_BUILTIN; + a->options[APP_OPTIONS_PREALLOC_FIFO_PAIRS] = hsm->prealloc_fifos; + a->options[APP_OPTIONS_TLS_ENGINE] = CRYPTO_ENGINE_OPENSSL; + + if (vnet_application_attach (a)) + { + vec_free (a->name); + clib_warning ("failed to attach server"); + return -1; + } + vec_free (a->name); + hsm->app_index = a->app_index; + + vnet_app_add_cert_key_pair (ck_pair); + hsm->ckpair_index = ck_pair->index; + + return 0; +} + +static int +http_static_transport_needs_crypto (transport_proto_t proto) +{ + return proto == TRANSPORT_PROTO_TLS || proto == TRANSPORT_PROTO_DTLS || + proto == TRANSPORT_PROTO_QUIC; +} + +static int +http_static_server_listen () +{ + http_static_l4_server_main_t *hsm = &http_static_l4_server_main; + session_endpoint_cfg_t sep = SESSION_ENDPOINT_CFG_NULL; + vnet_listen_args_t _a, *a = &_a; + char *uri = "tcp://0.0.0.0/80"; + int rv; + + clib_memset (a, 0, sizeof (*a)); + a->app_index = hsm->app_index; + + if (hsm->uri) + uri = (char *) hsm->uri; + + if (parse_uri (uri, &sep)) + return -1; + + clib_memcpy (&a->sep_ext, &sep, sizeof (sep)); + if (http_static_transport_needs_crypto (a->sep_ext.transport_proto)) + { + session_endpoint_alloc_ext_cfg (&a->sep_ext, + TRANSPORT_ENDPT_EXT_CFG_CRYPTO); + a->sep_ext.ext_cfg->crypto.ckpair_index = hsm->ckpair_index; + } + + rv = vnet_listen (a); + if (a->sep_ext.ext_cfg) + clib_mem_free (a->sep_ext.ext_cfg); + return rv; +} + +static void +http_static_server_session_close_cb (void *hs_handlep) +{ + http_static_l4_server_main_t *hsm = &http_static_l4_server_main; + http_session_t *hs; + uword hs_handle; + hs_handle = pointer_to_uword (hs_handlep); + hs = + http_static_server_session_get (hs_handle >> 24, hs_handle & 0x00FFFFFF); + + if (hsm->debug_level > 1) + clib_warning ("terminate thread %d index %d hs %llx", hs_handle >> 24, + hs_handle & 0x00FFFFFF, hs); + if (!hs) + return; + hs->timer_handle = ~0; + http_static_server_session_disconnect (hs); +} + +/** \brief Expired session timer-wheel callback + */ +static void +http_expired_timers_dispatch (u32 *expired_timers) +{ + u32 hs_handle; + int i; + + for (i = 0; i < vec_len (expired_timers); i++) + { + /* Get session handle. The first bit is the timer id */ + hs_handle = expired_timers[i] & 0x7FFFFFFF; + session_send_rpc_evt_to_thread (hs_handle >> 24, + http_static_server_session_close_cb, + uword_to_pointer (hs_handle, void *)); + } +} + +/** \brief Timer-wheel expiration process + */ +static uword +http_static_server_process (vlib_main_t *vm, vlib_node_runtime_t *rt, + vlib_frame_t *f) +{ + http_static_l4_server_main_t *hsm = &http_static_l4_server_main; + f64 now, timeout = 1.0; + uword *event_data = 0; + uword __clib_unused event_type; + + while (1) + { + vlib_process_wait_for_event_or_clock (vm, timeout); + now = vlib_time_now (vm); + event_type = vlib_process_get_events (vm, (uword **) &event_data); + + /* expire timers */ + clib_spinlock_lock (&http_static_l4_server_main.tw_lock); + tw_timer_expire_timers_2t_1w_2048sl (&hsm->tw, now); + clib_spinlock_unlock (&http_static_l4_server_main.tw_lock); + + vec_reset_length (event_data); + } + return 0; +} + +VLIB_REGISTER_NODE (http_static_server_process_node) = { + .function = http_static_server_process, + .type = VLIB_NODE_TYPE_PROCESS, + .name = "static-http-server-process", + .state = VLIB_NODE_STATE_DISABLED, +}; + +static int +http_static_server_create (vlib_main_t *vm) +{ + vlib_thread_main_t *vtm = vlib_get_thread_main (); + http_static_l4_server_main_t *hsm = &http_static_l4_server_main; + u32 num_threads, i; + vlib_node_t *n; + + num_threads = 1 /* main thread */ + vtm->n_threads; + vec_validate (hsm->vpp_queue, num_threads - 1); + vec_validate (hsm->sessions, num_threads - 1); + vec_validate (hsm->session_to_http_session, num_threads - 1); + vec_validate (hsm->thr_sessions_lock, num_threads - 1); + vec_validate (hsm->rx_buf_pool, num_threads - 1); + + for (i = 0; i <= vec_len (hsm->thr_sessions_lock); i++) + clib_rwlock_init (&hsm->thr_sessions_lock[i]); + + clib_spinlock_init (&hsm->tw_lock); + + if (http_static_server_attach ()) + { + clib_warning ("failed to attach server"); + return -1; + } + if (http_static_server_listen ()) + { + clib_warning ("failed to start listening"); + return -1; + } + + /* Init path-to-cache hash table */ + BV (clib_bihash_init) (&hsm->name_to_data, "http cache", 128, 32 << 20); + + hsm->get_url_handlers = hash_create_string (0, sizeof (uword)); + hsm->post_url_handlers = hash_create_string (0, sizeof (uword)); + + /* Init timer wheel and process */ + tw_timer_wheel_init_2t_1w_2048sl (&hsm->tw, http_expired_timers_dispatch, + 1.0 /* timer interval */, ~0); + vlib_node_set_state (vm, http_static_server_process_node.index, + VLIB_NODE_STATE_POLLING); + n = vlib_get_node (vm, http_static_server_process_node.index); + vlib_start_process (vm, n->runtime_index); + + return 0; +} + +/** \brief API helper function for vl_api_http_static_enable_t messages + */ +int +http_static_l4_server_enable (u32 fifo_size, u32 cache_limit, + u32 prealloc_fifos, u32 private_segment_size, + u8 *www_root, u8 *uri) +{ + http_static_l4_server_main_t *hsm = &http_static_l4_server_main; + int rv; + + hsm->fifo_size = fifo_size; + hsm->cache_limit = cache_limit; + hsm->prealloc_fifos = prealloc_fifos; + hsm->private_segment_size = private_segment_size; + hsm->www_root = format (0, "%s%c", www_root, 0); + hsm->uri = format (0, "%s%c", uri, 0); + + if (vec_len (hsm->www_root) < 2) + return VNET_API_ERROR_INVALID_VALUE; + + if (hsm->my_client_index != ~0) + return VNET_API_ERROR_APP_ALREADY_ATTACHED; + + vnet_session_enable_disable (hsm->vlib_main, 1 /* turn on TCP, etc. */); + + rv = http_static_server_create (hsm->vlib_main); + switch (rv) + { + case 0: + break; + default: + vec_free (hsm->www_root); + vec_free (hsm->uri); + return VNET_API_ERROR_INIT_FAILED; + } + return 0; +} + +static clib_error_t * +http_static_l4_server_create_command_fn (vlib_main_t *vm, + unformat_input_t *input, + vlib_cli_command_t *cmd) +{ + http_static_l4_server_main_t *hsm = &http_static_l4_server_main; + unformat_input_t _line_input, *line_input = &_line_input; + u64 seg_size; + u64 fifo_size; + u64 fifo_deq_thresh; + u8 *www_root = 0; + int rv; + + hsm->prealloc_fifos = 0; + hsm->private_segment_size = 0; + hsm->fifo_size = 0; + hsm->fifo_deq_thresh = HTTP_FIFO_DEF_THRESH; + /* 10mb cache limit, before LRU occurs */ + hsm->cache_limit = 10 << 20; + + clib_warning ("Starting http_static_l4_server ..."); + /* Get a line of input. */ + if (!unformat_user (input, unformat_line_input, line_input)) + goto no_wwwroot; + + while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT) + { + if (unformat (line_input, "www-root %s", &www_root)) + ; + else if (unformat (line_input, "prealloc-fifos %d", + &hsm->prealloc_fifos)) + ; + else if (unformat (line_input, "private-segment-size %U", + unformat_memory_size, &seg_size)) + hsm->private_segment_size = seg_size; + else if (unformat (line_input, "fifo-size %U", unformat_memory_size, + &fifo_size)) + { + if (fifo_size > UINT_MAX) + return clib_error_return (0, "fifo-size can't be over 4gb"); + hsm->fifo_size = + (fifo_size < 1024) ? fifo_size << 10 : (u32) fifo_size; + vlib_cli_output (vm, "fifo-size set to :%u", hsm->fifo_size); + } + else if (unformat (line_input, "fifo-deq-thresh %U", + unformat_memory_size, &fifo_deq_thresh)) + { + hsm->fifo_deq_thresh = (fifo_deq_thresh < 1024) ? + fifo_deq_thresh << 10 : + (u32) fifo_deq_thresh; + vlib_cli_output (vm, "fifo-deq-thresh set to :%u", + hsm->fifo_deq_thresh); + } + else if (unformat (line_input, "cache-size %U", unformat_memory_size, + &hsm->cache_limit)) + { + if (hsm->cache_limit < (128 << 10)) + { + return clib_error_return (0, + "cache-size must be at least 128kb"); + } + } + + else if (unformat (line_input, "uri %s", &hsm->uri)) + ; + else if (unformat (line_input, "debug %d", &hsm->debug_level)) + ; + else if (unformat (line_input, "debug")) + hsm->debug_level = 1; + else + return clib_error_return (0, "unknown input `%U'", + format_unformat_error, line_input); + } + unformat_free (line_input); + + if (hsm->fifo_deq_thresh > hsm->fifo_size) + { + return clib_error_return ( + 0, "fifo-deq-thresh:%u can be bigger than fifo-size:%d", + hsm->fifo_deq_thresh, hsm->fifo_size); + } + + if (www_root == 0) + { + no_wwwroot: + return clib_error_return (0, "Must specify www-root "); + } + + if (hsm->my_client_index != (u32) ~0) + { + vec_free (www_root); + return clib_error_return (0, "http server already running..."); + } + + hsm->www_root = www_root; + + vnet_session_enable_disable (vm, 1 /* turn on TCP, etc. */); + + rv = http_static_server_create (vm); + switch (rv) + { + case 0: + break; + default: + vec_free (hsm->www_root); + return clib_error_return (0, "server_create returned %d", rv); + } + return 0; +} + +/*? + * Enable the static http server + * + * @cliexpar + * This command enables the static http server. Only the www-root + * parameter is required + * @clistart + * http static server www-root /tmp/www uri tcp://0.0.0.0/80 cache-size 2m + * @cliend + * @cliexcmd{http static server www-root [prealloc-fios ] + * [private-segment-size ] [fifo-size ] [uri ]} +?*/ +VLIB_CLI_COMMAND (http_static_l4_server_create_command, static) = { + .path = "http static l4 server", + .short_help = + "http static l4 server www-root [prealloc-fifos ]\n" + "[private-segment-size ] [fifo-size ] [uri ]\n" + "[debug [nn]]\n", + .function = http_static_l4_server_create_command_fn, +}; + +/** \brief format a file cache entry + */ +u8 * +format_hsm_l4_cache_entry (u8 *s, va_list *args) +{ + file_data_cache_t *ep = va_arg (*args, file_data_cache_t *); + f64 now = va_arg (*args, f64); + + /* Header */ + if (ep == 0) + { + s = format (s, "%40s%12s%20s", "File", "Size", "Age"); + return s; + } + s = format (s, "%40s%12lld%20.2f", ep->filename, vec_len (ep->data), + now - ep->last_used); + return s; +} + +u8 * +format_http_l4_session_state (u8 *s, va_list *args) +{ + http_session_state_t state = va_arg (*args, http_session_state_t); + char *state_string = "bogus!"; + + switch (state) + { + case HTTP_STATE_CLOSED: + state_string = "closed"; + break; + case HTTP_STATE_ESTABLISHED: + state_string = "established"; + break; + case HTTP_STATE_RECEIVE_MORE_DATA: + state_string = "received"; + break; + case HTTP_STATE_OK_SENT: + state_string = "ok sent"; + break; + case HTTP_STATE_SEND_MORE_DATA: + state_string = "send more data"; + break; + case HTTP_STATE_CLOSING: + state_string = "closing"; + break; + default: + break; + } + + return format (s, "%s", state_string); +} + +u8 * +format_http_l4_session (u8 *s, va_list *args) +{ + http_session_t *hs = va_arg (*args, http_session_t *); + int verbose = va_arg (*args, int); + + s = format (s, "[%d]: state %U", hs->session_index, + format_http_l4_session_state, hs->session_state); + if (verbose > 0) + { + s = format (s, "\n path %s, data length %u, data_offset %u", hs->path, + vec_len (hs->data), hs->data_offset); + } + return s; +} + +static clib_error_t * +http_show_static_l4_server_command_fn (vlib_main_t *vm, + unformat_input_t *input, + vlib_cli_command_t *cmd) +{ + http_static_l4_server_main_t *hsm = &http_static_l4_server_main; + file_data_cache_t *ep, **entries = 0; + int verbose = 0; + int show_cache = 0; + int show_sessions = 0; + u32 index; + f64 now; + + if (hsm->www_root == 0) + return clib_error_return (0, "Static server disabled"); + + while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) + { + if (unformat (input, "verbose %d", &verbose)) + ; + else if (unformat (input, "verbose")) + verbose = 1; + else if (unformat (input, "cache")) + show_cache = 1; + else if (unformat (input, "sessions")) + show_sessions = 1; + else + break; + } + + if ((show_cache + show_sessions) == 0) + return clib_error_return (0, "specify one or more of cache, sessions"); + + if (show_cache) + { + if (verbose == 0) + { + vlib_cli_output ( + vm, + "www_root %s, cache size %lld bytes, limit %lld bytes, " + "evictions %lld", + hsm->www_root, hsm->cache_size, hsm->cache_limit, + hsm->cache_evictions); + return 0; + } + + now = vlib_time_now (vm); + + vlib_cli_output (vm, "%U", format_hsm_l4_cache_entry, 0 /* header */, + now); + + for (index = hsm->first_index; index != ~0;) + { + ep = pool_elt_at_index (hsm->cache_pool, index); + index = ep->next_index; + vlib_cli_output (vm, "%U", format_hsm_l4_cache_entry, ep, now); + } + + vlib_cli_output (vm, "%40s%12lld", "Total Size", hsm->cache_size); + + vec_free (entries); + } + + if (show_sessions) + { + u32 *session_indices = 0; + http_session_t *hs; + int i, j; + + /* Lock for all threads */ + http_static_server_all_sessions_reader_lock (); + + for (i = 0; i < vec_len (hsm->sessions); i++) + { + pool_foreach (hs, hsm->sessions[i]) + { + vec_add1 (session_indices, hs - hsm->sessions[i]); + } + + for (j = 0; j < vec_len (session_indices); j++) + { + vlib_cli_output ( + vm, "%U", format_http_l4_session, + pool_elt_at_index (hsm->sessions[i], session_indices[j]), + verbose); + } + vec_reset_length (session_indices); + } + http_static_server_all_sessions_reader_unlock (); + vec_free (session_indices); + } + return 0; +} + +/*? + * Display static http server l4 cache statistics + * + * @cliexpar + * This command shows the contents of the static http l4 server cache + * @clistart + * show http static server + * @cliend + * @cliexcmd{show http static l4 server sessions cache [verbose [nn]]} +?*/ +VLIB_CLI_COMMAND (http_show_static_l4_server_command, static) = { + .path = "show http static l4 server", + .short_help = "show http static l4 server sessions cache [verbose []]", + .function = http_show_static_l4_server_command_fn, +}; + +static clib_error_t * +http_clear_static_cache_command_fn (vlib_main_t *vm, unformat_input_t *input, + vlib_cli_command_t *cmd) +{ + http_static_l4_server_main_t *hsm = &http_static_l4_server_main; + file_data_cache_t *dp; + u32 free_index; + u32 busy_items = 0; + BVT (clib_bihash_kv) kv; + + if (hsm->www_root == 0) + return clib_error_return (0, "Static server disabled"); + + /* Lock for all threads */ + http_static_server_all_sessions_reader_lock (); + + /* Walk the LRU list to find active entries */ + free_index = hsm->last_index; + while (free_index != ~0) + { + dp = pool_elt_at_index (hsm->cache_pool, free_index); + free_index = dp->prev_index; + /* Which could be in use... */ + if (dp->inuse) + { + busy_items++; + free_index = dp->next_index; + continue; + } + kv.key = (u64) (dp->filename); + kv.value = ~0ULL; + if (BV (clib_bihash_add_del) (&hsm->name_to_data, &kv, 0 /* is_add */) < + 0) + { + clib_warning ("BUG: cache clear delete '%s' FAILED!", dp->filename); + } + + lru_remove (hsm, dp); + hsm->cache_size -= vec_len (dp->data); + hsm->cache_evictions++; + dp->filename[0] = 0; + vec_free (dp->data); + if (hsm->debug_level > 1) + clib_warning ("pool put index %d", dp - hsm->cache_pool); + pool_put (hsm->cache_pool, dp); + free_index = hsm->last_index; + } + http_static_server_all_sessions_reader_unlock (); + if (busy_items > 0) + vlib_cli_output (vm, "Note: %d busy items still in cache...", busy_items); + else + vlib_cli_output (vm, "Cache cleared..."); + return 0; +} + +/*? + * Clear the static http l4 server cache, to force the server to + * reload content from backing files + * + * @cliexpar + * This command clear the static http server cache + * @clistart + * clear http static cache + * @cliend + * @cliexcmd{clear http static l4 cache} +?*/ +VLIB_CLI_COMMAND (clear_http_static_l4_cache_command, static) = { + .path = "clear http static l4 cache", + .short_help = "clear http static l4 cache", + .function = http_clear_static_cache_command_fn, +}; + +static clib_error_t * +http_set_static_l4_redirect_command_fn (vlib_main_t *vm, + unformat_input_t *input, + vlib_cli_command_t *cmd) +{ + http_static_l4_server_main_t *hsm = &http_static_l4_server_main; + unformat_input_t _line_input, *line_input = &_line_input; + u8 *redirect_file = NULL; + int digits = 0; + u32 n_bytes, num_m_args = 0; + + if (hsm->www_root == 0) + return clib_error_return (0, "Static server disabled"); + + /* Get a line of input. */ + if (!unformat_user (input, unformat_line_input, line_input)) + return 0; + + while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT) + { + if (unformat (line_input, "file %s", &redirect_file)) + num_m_args++; + else + return clib_error_return (0, "unknown input `%U'", + format_unformat_error, line_input); + } + if (num_m_args != 1) + return clib_error_return (0, "Too many params"); + + if (redirect_file == NULL) + return clib_error_return (0, "No file name", redirect_file); + else + clib_warning ("File Name : %s", redirect_file); + + n_bytes = parse_fileformat (redirect_file, &digits); + if (n_bytes == 0 || digits > 3) + return clib_error_return (0, "Bad File : %s, n_bytes=%u, digits=%u", + redirect_file, n_bytes, digits); + + redirect_file[digits + 1] = '\0'; + + /* Lock for all threads */ + http_static_server_all_sessions_writer_lock (); + strcpy (hsm->redirect_file_name, (char *) redirect_file); + http_static_server_all_sessions_writer_unlock (); + vlib_cli_output (vm, "redirect index.html to File Name : %s", redirect_file); + vec_free (redirect_file); + return 0; +} + +VLIB_CLI_COMMAND (http_set_static_l4_redirect_command, static) = { + .path = "set http static l4 redirect", + .short_help = "set http static l4 redirect file ", + .function = http_set_static_l4_redirect_command_fn, +}; + +static clib_error_t * +http_static_l4_server_main_init (vlib_main_t *vm) +{ + http_static_l4_server_main_t *hsm = &http_static_l4_server_main; + + hsm->my_client_index = ~0; + hsm->vlib_main = vm; + hsm->first_index = hsm->last_index = ~0; + + clib_timebase_init (&hsm->timebase, 0 /* GMT */, CLIB_TIMEBASE_DAYLIGHT_NONE, + &vm->clib_time /* share the system clock */); + + return 0; +} + +VLIB_INIT_FUNCTION (http_static_l4_server_main_init); + +/* + * fd.io coding-style-patch-verification: ON + * + * Local Variables: + * eval: (c-set-style "gnu") + * End: + */ From 15b93ce2c5d2eea79bd6e7a8072b0e7cb29696e9 Mon Sep 17 00:00:00 2001 From: Monendra Singh Kushwaha Date: Mon, 11 Nov 2024 12:49:30 +0530 Subject: [PATCH 118/271] octeon: use allmulti port argument value Type: fix Change-Id: Iab627cec02079866f81922dc3d397f1d3d0885f0 Signed-off-by: Monendra Singh Kushwaha Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/139028 Tested-by: sa_ip-sw-jenkins --- src/plugins/dev_octeon/port.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/plugins/dev_octeon/port.c b/src/plugins/dev_octeon/port.c index 2b2fdea698..983e0d70bc 100644 --- a/src/plugins/dev_octeon/port.c +++ b/src/plugins/dev_octeon/port.c @@ -167,9 +167,9 @@ oct_port_init (vlib_main_t *vm, vnet_dev_port_t *port) /* Enable allmulti mode, if set by arg */ foreach_vnet_dev_port_args (arg, port) { - if (arg->id == OCT_PORT_ARG_ALLMULTI_MODE) + if (arg->id == OCT_PORT_ARG_ALLMULTI_MODE && vnet_dev_arg_get_bool (arg)) { - if ((rrv = roc_nix_npc_mcast_config (nix, 1, 0))) + if ((rrv = roc_nix_npc_mcast_config (nix, true, false))) { oct_port_deinit (vm, port); return oct_roc_err (dev, rrv, "roc_nix_mac_addr_set failed"); From 840911e4d701f08e985a7e4a71d6c611e3eeb439 Mon Sep 17 00:00:00 2001 From: Monendra Singh Kushwaha Date: Thu, 7 Nov 2024 01:20:38 +0530 Subject: [PATCH 119/271] octeon: set rss flowkey after mac update Type: fix Change-Id: I1b087b9e1e7a17ca545f960e896ca48266cdcc93 Signed-off-by: Monendra Singh Kushwaha Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/138727 Reviewed-by: Nithin Kumar Dabilpuram Reviewed-by: Kiran Kumar Kokkilagadda Tested-by: sa_ip-sw-jenkins (cherry picked from commit 445791096f6a672ec65c5a2db91df82d283679f9) Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/139029 --- src/plugins/dev_octeon/port.c | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/plugins/dev_octeon/port.c b/src/plugins/dev_octeon/port.c index 983e0d70bc..103e513b64 100644 --- a/src/plugins/dev_octeon/port.c +++ b/src/plugins/dev_octeon/port.c @@ -443,6 +443,7 @@ oct_port_start (vlib_main_t *vm, vnet_dev_port_t *port) { vnet_dev_t *dev = port->dev; oct_device_t *cd = vnet_dev_get_data (dev); + oct_port_t *cp = vnet_dev_get_port_data (port); struct roc_nix *nix = cd->nix; struct roc_nix_eeprom_info eeprom_info = {}; vnet_dev_rv_t rv; @@ -466,6 +467,12 @@ oct_port_start (vlib_main_t *vm, vnet_dev_port_t *port) goto done; } + if ((rrv = roc_npc_mcam_enable_all_entries (&cp->npc, true))) + { + rv = oct_roc_err (dev, rrv, "roc_npc_mcam_enable_all_entries() failed"); + goto done; + } + vnet_dev_poll_port_add (vm, port, 0.5, oct_port_poll); if (roc_nix_eeprom_info_get (nix, &eeprom_info) == 0) @@ -484,6 +491,7 @@ oct_port_stop (vlib_main_t *vm, vnet_dev_port_t *port) { vnet_dev_t *dev = port->dev; oct_device_t *cd = vnet_dev_get_data (dev); + oct_port_t *cp = vnet_dev_get_port_data (port); struct roc_nix *nix = cd->nix; int rrv; @@ -491,6 +499,14 @@ oct_port_stop (vlib_main_t *vm, vnet_dev_port_t *port) vnet_dev_poll_port_remove (vm, port, oct_port_poll); + /* Disable all the NPC entries */ + rrv = roc_npc_mcam_enable_all_entries (&cp->npc, false); + if (rrv) + { + oct_roc_err (dev, rrv, "roc_npc_mcam_enable_all_entries() failed"); + return; + } + rrv = roc_nix_npc_rx_ena_dis (nix, false); if (rrv) { @@ -590,6 +606,10 @@ oct_port_add_del_eth_addr (vlib_main_t *vm, vnet_dev_port_t *port, rv = oct_roc_err (dev, rrv, "roc_nix_mac_addr_set() failed"); } } + + rrv = roc_nix_rss_default_setup (nix, default_rss_flowkey); + if (rrv) + rv = oct_roc_err (dev, rrv, "roc_nix_rss_default_setup() failed"); } } From c5188d9487345a0073f42ae2576ab296aa2f095b Mon Sep 17 00:00:00 2001 From: Monendra Singh Kushwaha Date: Wed, 20 Nov 2024 12:06:52 +0530 Subject: [PATCH 120/271] dev: assign tx queue to all threads This patch assigns tx queue to all thread and enables tx queue sharing if needed. Type: fix JIRA: https://essjira.marvell.com/browse/IPBUSW-53459 Signed-off-by: Monendra Singh Kushwaha Change-Id: I8cb561c29c2a508b8b478c646121b1caa61b8520 Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/139617 Tested-by: sa_ip-sw-jenkins Reviewed-by: Nithin Kumar Dabilpuram (cherry picked from commit 1a724f6be6157d71f1ec7fe14326e858c98e9e1a) Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/140369 --- src/vnet/dev/port.c | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/vnet/dev/port.c b/src/vnet/dev/port.c index 5b4b8cdc7b..a56cb86ea7 100644 --- a/src/vnet/dev/port.c +++ b/src/vnet/dev/port.c @@ -516,6 +516,7 @@ vnet_dev_port_if_create (vlib_main_t *vm, vnet_dev_port_t *port) u16 n_threads = vlib_get_n_threads (); vnet_dev_main_t *dm = &vnet_dev_main; vnet_dev_t *dev = port->dev; + vnet_dev_tx_queue_t *txq, **qp; vnet_dev_port_t **pp; vnet_dev_rv_t rv; u16 ti = 0; @@ -553,13 +554,15 @@ vnet_dev_port_if_create (vlib_main_t *vm, vnet_dev_port_t *port) VNET_DEV_OK) goto error; - foreach_vnet_dev_port_tx_queue (q, port) + for (ti = 0; ti < n_threads; ti++) { - q->assigned_threads = clib_bitmap_set (q->assigned_threads, ti, 1); + qp = pool_elt_at_index (port->tx_queues, ti % port->intf.num_tx_queues); + txq = qp[0]; + txq->assigned_threads = clib_bitmap_set (txq->assigned_threads, ti, 1); log_debug (dev, "port %u tx queue %u assigned to thread %u", - port->port_id, q->queue_id, ti); - if (++ti >= n_threads) - break; + port->port_id, txq->queue_id, ti); + if (clib_bitmap_count_set_bits (txq->assigned_threads) > 1) + txq->lock_needed = 1; } /* pool of port pointers helps us to assign unique dev_instance */ From 1be60d390d6ef705ac0953bcc04ffd77762ced78 Mon Sep 17 00:00:00 2001 From: Nithinsen Kaithakadan Date: Tue, 6 Aug 2024 17:18:52 +0530 Subject: [PATCH 121/271] octeon: add inline ipsec inbound support This patch add support for inline ipsec in inbound Type: feature Signed-off-by: Nithinsen Kaithakadan Signed-off-by: Monendra Singh Kushwaha Change-Id: Iac8aec02428c5eda26cec9bdba8cefc6d3b8f6a2 Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/132913 Tested-by: sa_ip-sw-jenkins Reviewed-by: Nithin Kumar Dabilpuram (cherry picked from commit f43a876aea28e9919128a5ab1bd54bad2e2f41f6) Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/140370 --- src/plugins/dev_octeon/CMakeLists.txt | 1 + src/plugins/dev_octeon/cli.c | 30 +- src/plugins/dev_octeon/flow.c | 33 + src/plugins/dev_octeon/init.c | 104 ++- src/plugins/dev_octeon/ipsec.c | 702 ++++++++++++++++++++ src/plugins/dev_octeon/ipsec.h | 133 ++++ src/plugins/dev_octeon/octeon.h | 30 +- src/plugins/dev_octeon/port.c | 21 +- src/plugins/dev_octeon/queue.c | 34 +- src/plugins/dev_octeon/rx_node.c | 897 ++++++++++++++++++++++++-- 10 files changed, 1913 insertions(+), 72 deletions(-) create mode 100644 src/plugins/dev_octeon/ipsec.c create mode 100644 src/plugins/dev_octeon/ipsec.h diff --git a/src/plugins/dev_octeon/CMakeLists.txt b/src/plugins/dev_octeon/CMakeLists.txt index 877c7b8d27..544901df84 100644 --- a/src/plugins/dev_octeon/CMakeLists.txt +++ b/src/plugins/dev_octeon/CMakeLists.txt @@ -41,6 +41,7 @@ add_vpp_plugin(dev_octeon flow.c counter.c crypto.c + ipsec.c MULTIARCH_SOURCES rx_node.c diff --git a/src/plugins/dev_octeon/cli.c b/src/plugins/dev_octeon/cli.c index b5ed7e8313..b390bb1d1e 100644 --- a/src/plugins/dev_octeon/cli.c +++ b/src/plugins/dev_octeon/cli.c @@ -170,7 +170,7 @@ oct_crypto_counters_command_fn (vlib_main_t *vm, unformat_input_t *input, ?*/ VLIB_CLI_COMMAND (oct_crypto_counters_command, static) = { .path = "show octeon crypto counters", - .short_help = "show octeon crypto counters [verbose]", + .short_help = "show octeon crypto counters", .function = oct_crypto_counters_command_fn, }; @@ -206,3 +206,31 @@ VLIB_CLI_COMMAND (oct_crypto_counters_clear_command, static) = { .short_help = "clear octeon crypto counters", .function = oct_crypto_counters_clear_command_fn, }; + +static clib_error_t * +oct_ipsec_inline_counters_command_fn (vlib_main_t *vm, unformat_input_t *input, + vlib_cli_command_t *cmd) +{ + struct roc_nix_stats stats; + + roc_nix_inl_dev_stats_get (&stats); + + vlib_cli_output (vm, "rx_ucast %u", stats.rx_ucast); + vlib_cli_output (vm, "rx_bcast %u", stats.rx_bcast); + vlib_cli_output (vm, "rx_mcast %u", stats.rx_mcast); + vlib_cli_output (vm, "rx_drop %u", stats.rx_drop); + vlib_cli_output (vm, "rx_fcs %u", stats.rx_fcs); + vlib_cli_output (vm, "rx_err %u", stats.rx_err); + vlib_cli_output (vm, "rx_drop_bcast %u", stats.rx_drop_bcast); + vlib_cli_output (vm, "rx_drop_mcast %u", stats.rx_drop_mcast); + vlib_cli_output (vm, "rx_drop_l3_bcast %u", stats.rx_drop_l3_bcast); + vlib_cli_output (vm, "rx_drop_l3_bcast %u", stats.rx_drop_l3_mcast); + + return 0; +} + +VLIB_CLI_COMMAND (oct_ipsec_inline_counters_command, static) = { + .path = "show octeon ipsec inline counters", + .short_help = "show ipsec inline counters", + .function = oct_ipsec_inline_counters_command_fn, +}; diff --git a/src/plugins/dev_octeon/flow.c b/src/plugins/dev_octeon/flow.c index 5bef25f536..3dde2a4c87 100644 --- a/src/plugins/dev_octeon/flow.c +++ b/src/plugins/dev_octeon/flow.c @@ -135,6 +135,10 @@ oct_flow_validate_params (vlib_main_t *vm, vnet_dev_port_t *port, u32 last_queue; u32 qid; + if ((flow->actions & VNET_FLOW_ACTION_REDIRECT_TO_QUEUE) && + flow->redirect_queue == ~0 && flow->type == VNET_FLOW_TYPE_IP4_IPSEC_ESP) + return VNET_DEV_OK; + if (type == VNET_DEV_PORT_CFG_GET_RX_FLOW_COUNTER || type == VNET_DEV_PORT_CFG_RESET_RX_FLOW_COUNTER) { @@ -535,6 +539,7 @@ static vnet_dev_rv_t oct_flow_add (vlib_main_t *vm, vnet_dev_port_t *port, vnet_flow_t *flow, uword *private_data) { + oct_main_t *om = &oct_main; struct roc_npc_item_info item_info[ROC_NPC_ITEM_TYPE_END] = {}; struct roc_npc_action actions[ROC_NPC_ITEM_TYPE_END] = {}; oct_port_t *oct_port = vnet_dev_get_port_data (port); @@ -546,6 +551,7 @@ oct_flow_add (vlib_main_t *vm, vnet_dev_port_t *port, vnet_flow_t *flow, udp_header_t udp_spec = {}, udp_mask = {}; tcp_header_t tcp_spec = {}, tcp_mask = {}; esp_header_t esp_spec = {}, esp_mask = {}; + vnet_flow_ip4_ipsec_esp_t *esp_hdr; u16 l4_src_port = 0, l4_dst_port = 0; u16 l4_src_mask = 0, l4_dst_mask = 0; struct roc_npc_action_rss rss_conf = {}; @@ -561,6 +567,33 @@ oct_flow_add (vlib_main_t *vm, vnet_dev_port_t *port, vnet_flow_t *flow, u8 proto = 0; u16 action = 0; + if ((flow->actions & VNET_FLOW_ACTION_REDIRECT_TO_QUEUE) && + flow->redirect_queue == ~0 && flow->type == VNET_FLOW_TYPE_IP4_IPSEC_ESP) + { + if (!om->inl_dev_initialized) + { + log_err (port->dev, "Inline device is not initialized"); + return VNET_DEV_ERR_NOT_SUPPORTED; + } + + esp_hdr = &flow->ip4_ipsec_esp; + esp_spec.spi = clib_host_to_net_u32 (esp_hdr->spi); + esp_mask.spi = 0; /* Any */ + item_info[0].spec = (void *) &esp_spec; + item_info[0].mask = (void *) &esp_mask; + item_info[0].size = sizeof (u32); + item_info[0].type = ROC_NPC_ITEM_TYPE_ESP; + item_info[1].type = ROC_NPC_ITEM_TYPE_END; + + actions[0].conf = (void *) NULL; + actions[0].type = ROC_NPC_ACTION_TYPE_SEC; + actions[1].type = ROC_NPC_ACTION_TYPE_COUNT; + actions[2].type = ROC_NPC_ACTION_TYPE_END; + + return oct_flow_rule_create (port, actions, item_info, flow, + private_data); + } + if (FLOW_IS_GENERIC_TYPE (flow)) { unformat_input_t input; diff --git a/src/plugins/dev_octeon/init.c b/src/plugins/dev_octeon/init.c index 1f10dc1a0c..5633694509 100644 --- a/src/plugins/dev_octeon/init.c +++ b/src/plugins/dev_octeon/init.c @@ -11,13 +11,16 @@ #include #include #include +#include #include #include struct roc_model oct_model; u32 oct_npa_max_pools = OCT_NPA_MAX_POOLS; +oct_main_t oct_main; extern oct_crypto_main_t oct_crypto_main; +extern oct_inl_dev_main_t oct_inl_dev_main; VLIB_REGISTER_LOG_CLASS (oct_log, static) = { .class_name = "octeon", @@ -27,11 +30,20 @@ VLIB_REGISTER_LOG_CLASS (oct_log, static) = { #define _(f, n, s, d) \ { .name = #n, .desc = d, .severity = VL_COUNTER_SEVERITY_##s }, +vlib_error_desc_t oct_rx_node_counters[] = { + /* clang-format off */ + foreach_octeon10_ipsec_ucc + foreach_oct_rx_node_counter + /* clang-format on */ +}; + vlib_error_desc_t oct_tx_node_counters[] = { foreach_oct_tx_node_counter }; #undef _ vnet_dev_node_t oct_rx_node = { .format_trace = format_oct_rx_trace, + .error_counters = oct_rx_node_counters, + .n_error_counters = ARRAY_LEN (oct_rx_node_counters), }; vnet_dev_node_t oct_tx_node = { @@ -60,6 +72,10 @@ static struct _ (0xa0f3, O10K_CPT_VF, "Marvell Octeon-10 Cryptographic Accelerator Unit VF"), _ (0xa0fe, O9K_CPT_VF, "Marvell Octeon-9 Cryptographic Accelerator Unit VF"), + _ (0xa0f0, RVU_INL_PF, + "Marvell Octeon Resource Virtualization Unit Inline Device PF"), + _ (0xa0f1, RVU_INL_VF, + "Marvell Octeon Resource Virtualization Unit Inline Device VF"), #undef _ }; @@ -79,6 +95,36 @@ static vnet_dev_arg_t oct_port_args[] = { }, }; +clib_error_t * +oct_inl_inb_ipsec_flow_enable (void) +{ + oct_inl_dev_main_t *inl_main = &oct_inl_dev_main; + vnet_dev_main_t *dm = &vnet_dev_main; + vnet_main_t *vnm = vnet_get_main (); + vnet_flow_t flow = { 0 }; + u32 flow_index = ~0; + + if (inl_main->is_inl_ipsec_flow_enabled) + return NULL; + + pool_foreach_pointer (port, dm->ports_by_dev_instance) + { + clib_memset (&flow, 0, sizeof (vnet_flow_t)); + + flow.index = ~0; + flow.actions = VNET_FLOW_ACTION_REDIRECT_TO_QUEUE; + flow.type = VNET_FLOW_TYPE_IP4_IPSEC_ESP; + flow.ip4_ipsec_esp.spi = 0; + flow.redirect_queue = ~0; + + vnet_flow_add (vnm, &flow, &flow_index); + vnet_flow_enable (vnm, flow_index, port->intf.hw_if_index); + } + + inl_main->is_inl_ipsec_flow_enabled = 1; + return NULL; +} + static u8 * oct_probe (vlib_main_t *vm, vnet_dev_bus_index_t bus_index, void *dev_info) { @@ -122,10 +168,13 @@ oct_alloc (vlib_main_t *vm, vnet_dev_t *dev) static vnet_dev_rv_t oct_init_nix (vlib_main_t *vm, vnet_dev_t *dev) { - oct_device_t *cd = vnet_dev_get_data (dev); + oct_main_t *om = &oct_main; + oct_inl_dev_main_t *oidm = &oct_inl_dev_main; + oct_device_t *cd = vnet_dev_get_data (dev), **oct_dev = 0; u8 mac_addr[6]; int rrv; oct_port_t oct_port = {}; + vnet_dev_rv_t rv; *cd->nix = (struct roc_nix){ .reta_sz = ROC_NIX_RSS_RETA_SZ_256, @@ -140,6 +189,10 @@ oct_init_nix (vlib_main_t *vm, vnet_dev_t *dev) if ((rrv = roc_nix_npc_mac_addr_get (cd->nix, mac_addr))) return cnx_return_roc_err (dev, rrv, "roc_nix_npc_mac_addr_get"); + if (oidm->inl_dev) + if ((rv = oct_init_nix_inline_ipsec (vm, oidm->vdev, dev))) + return rv; + vnet_dev_port_add_args_t port_add_args = { .port = { .attr = { @@ -177,7 +230,7 @@ oct_init_nix (vlib_main_t *vm, vnet_dev_t *dev) .rx_queue = { .config = { .data_size = sizeof (oct_rxq_t), - .default_size = 1024, + .default_size = 8192, .multiplier = 32, .min_size = 256, .max_size = 16384, @@ -191,7 +244,7 @@ oct_init_nix (vlib_main_t *vm, vnet_dev_t *dev) .tx_queue = { .config = { .data_size = sizeof (oct_txq_t), - .default_size = 1024, + .default_size = 8192, .multiplier = 32, .min_size = 256, .max_size = 16384, @@ -208,7 +261,13 @@ oct_init_nix (vlib_main_t *vm, vnet_dev_t *dev) log_info (dev, "MAC address is %U", format_ethernet_address, mac_addr); - return vnet_dev_port_add (vm, dev, 0, &port_add_args); + if ((rv = vnet_dev_port_add (vm, dev, 0, &port_add_args))) + return rv; + + pool_get (om->oct_dev, oct_dev); + oct_dev = vnet_dev_get_data (dev); + + return VNET_DEV_OK; } static int @@ -271,6 +330,30 @@ oct_conf_cpt_queue (vlib_main_t *vm, vnet_dev_t *dev, oct_crypto_dev_t *ocd) return 0; } +static vnet_dev_rv_t +oct_init_inl_dev (vlib_main_t *vm, vnet_dev_t *dev) +{ + extern oct_plt_init_param_t oct_plt_init_param; + oct_device_t *od = vnet_dev_get_data (dev); + oct_inl_dev_main_t *oidm = &oct_inl_dev_main; + vnet_dev_rv_t rv; + + oidm->inl_dev = oct_plt_init_param.oct_plt_zmalloc ( + sizeof (struct roc_nix_inl_dev), CLIB_CACHE_LINE_BYTES); + oidm->inl_dev->pci_dev = &od->plt_pci_dev; + oidm->vdev = dev; + + if ((rv = oct_early_init_inline_ipsec (vm, dev))) + return rv; + if ((rv = oct_init_ipsec_backend (vm, dev))) + return rv; + + oct_main.use_single_rx_aura = 1; + oct_main.inl_dev_initialized = 1; + + return VNET_DEV_OK; +} + static vnet_dev_rv_t oct_init_cpt (vlib_main_t *vm, vnet_dev_t *dev) { @@ -384,6 +467,10 @@ oct_init (vlib_main_t *vm, vnet_dev_t *dev) case OCT_DEVICE_TYPE_O9K_CPT_VF: return oct_init_cpt (vm, dev); + case OCT_DEVICE_TYPE_RVU_INL_PF: + case OCT_DEVICE_TYPE_RVU_INL_VF: + return oct_init_inl_dev (vm, dev); + default: return VNET_DEV_ERR_UNSUPPORTED_DEVICE; } @@ -461,6 +548,9 @@ oct_early_config (vlib_main_t *vm, unformat_input_t *input) unformat_input_t _line_input, *line_input = &_line_input; clib_error_t *error = 0; + oct_inl_dev_main.in_min_spi = 0; + oct_inl_dev_main.in_max_spi = 8192; + if (!unformat_user (input, unformat_line_input, line_input)) return 0; @@ -468,6 +558,12 @@ oct_early_config (vlib_main_t *vm, unformat_input_t *input) { if (unformat (line_input, "max-pools %u", &oct_npa_max_pools)) ; + else if (unformat (line_input, "ipsec_in_min_spi %u", + &oct_inl_dev_main.in_min_spi)) + ; + else if (unformat (line_input, "ipsec_in_max_spi %u", + &oct_inl_dev_main.in_max_spi)) + ; else { error = clib_error_return (0, "unknown input '%U'", diff --git a/src/plugins/dev_octeon/ipsec.c b/src/plugins/dev_octeon/ipsec.c new file mode 100644 index 0000000000..9de69585e6 --- /dev/null +++ b/src/plugins/dev_octeon/ipsec.c @@ -0,0 +1,702 @@ +/* + * Copyright (c) 2024 Marvell. + * SPDX-License-Identifier: Apache-2.0 + * https://spdx.org/licenses/Apache-2.0.html + */ + +#include +#include +#include + +#define OCT_NIX_INL_META_POOL_NAME "OCT_NIX_INL_META_POOL" + +oct_ipsec_main_t oct_ipsec_main; +oct_inl_dev_main_t oct_inl_dev_main; + +VLIB_REGISTER_LOG_CLASS (oct_log, static) = { + .class_name = "octeon", + .subclass_name = "ipsec", +}; + +static void +oct_cn10k_ipsec_hmac_opad_ipad_gen (ipsec_sa_t *sa, u8 *hmac_opad_ipad) +{ + u8 opad[128] = { [0 ... 127] = 0x5c }; + u8 ipad[128] = { [0 ... 127] = 0x36 }; + const u8 *key = sa->integ_key.data; + u32 length = sa->integ_key.len; + u32 i; + + /* HMAC OPAD and IPAD */ + for (i = 0; i < 128 && i < length; i++) + { + opad[i] = opad[i] ^ key[i]; + ipad[i] = ipad[i] ^ key[i]; + } + + /* + * Precompute hash of HMAC OPAD and IPAD to avoid + * per-packet computation + */ + switch (sa->integ_alg) + { + case IPSEC_INTEG_ALG_SHA1_96: + roc_hash_sha1_gen (opad, (u32 *) &hmac_opad_ipad[0]); + roc_hash_sha1_gen (ipad, (u32 *) &hmac_opad_ipad[24]); + break; + case IPSEC_INTEG_ALG_SHA_256_96: + case IPSEC_INTEG_ALG_SHA_256_128: + roc_hash_sha256_gen (opad, (u32 *) &hmac_opad_ipad[0], 256); + roc_hash_sha256_gen (ipad, (u32 *) &hmac_opad_ipad[64], 256); + break; + case IPSEC_INTEG_ALG_SHA_384_192: + roc_hash_sha512_gen (opad, (u64 *) &hmac_opad_ipad[0], 384); + roc_hash_sha512_gen (ipad, (u64 *) &hmac_opad_ipad[64], 384); + break; + case IPSEC_INTEG_ALG_SHA_512_256: + roc_hash_sha512_gen (opad, (u64 *) &hmac_opad_ipad[0], 512); + roc_hash_sha512_gen (ipad, (u64 *) &hmac_opad_ipad[64], 512); + break; + default: + break; + } +} + +static_always_inline u64 +oct_ipsec_crypto_inst_w7_get (void *sa) +{ + union cpt_inst_w7 w7; + + w7.u64 = 0; + w7.s.egrp = ROC_CPT_DFLT_ENG_GRP_SE_IE; + w7.s.ctx_val = 1; + w7.s.cptr = (u64) sa; + + return w7.u64; +} + +static_always_inline i32 +oct_ipsec_sa_common_param_fill (union roc_ot_ipsec_sa_word2 *w2, + u8 *cipher_key, u8 *salt_key, + u8 *hmac_opad_ipad, ipsec_sa_t *sa) +{ + u32 *tmp_salt; + u64 *tmp_key; + int i; + + if (ipsec_sa_is_set_UDP_ENCAP (sa)) + w2->s.encap_type = ROC_IE_OT_SA_ENCAP_UDP; + + /* Set protocol - ESP vs AH */ + if (sa->protocol == IPSEC_PROTOCOL_ESP) + w2->s.protocol = ROC_IE_SA_PROTOCOL_ESP; + else + w2->s.protocol = ROC_IE_SA_PROTOCOL_AH; + + /* Set mode - transport vs tunnel */ + if (ipsec_sa_is_set_IS_TUNNEL (sa)) + w2->s.mode = ROC_IE_SA_MODE_TUNNEL; + else + w2->s.mode = ROC_IE_SA_MODE_TRANSPORT; + + if (ipsec_sa_is_set_IS_CTR (sa)) + { + if (ipsec_sa_is_set_IS_AEAD (sa)) + { + /* AEAD is set for AES_GCM */ + if (IPSEC_CRYPTO_ALG_IS_GCM (sa->crypto_alg)) + { + w2->s.enc_type = ROC_IE_SA_ENC_AES_GCM; + w2->s.auth_type = ROC_IE_SA_AUTH_NULL; + } + else + { + clib_warning ("Unsupported AEAD algorithm"); + return -1; + } + } + else + w2->s.enc_type = ROC_IE_SA_ENC_AES_CTR; + } + else + { + switch (sa->crypto_alg) + { + case IPSEC_CRYPTO_ALG_NONE: + w2->s.enc_type = ROC_IE_SA_ENC_NULL; + break; + case IPSEC_CRYPTO_ALG_AES_CBC_128: + case IPSEC_CRYPTO_ALG_AES_CBC_192: + case IPSEC_CRYPTO_ALG_AES_CBC_256: + w2->s.enc_type = ROC_IE_SA_ENC_AES_CBC; + break; + default: + clib_warning ("Unsupported encryption algorithm"); + return -1; + } + } + + switch (sa->crypto_alg) + { + case IPSEC_CRYPTO_ALG_AES_GCM_128: + case IPSEC_CRYPTO_ALG_AES_CBC_128: + case IPSEC_CRYPTO_ALG_AES_CTR_128: + w2->s.aes_key_len = ROC_IE_SA_AES_KEY_LEN_128; + break; + case IPSEC_CRYPTO_ALG_AES_GCM_192: + case IPSEC_CRYPTO_ALG_AES_CBC_192: + case IPSEC_CRYPTO_ALG_AES_CTR_192: + w2->s.aes_key_len = ROC_IE_SA_AES_KEY_LEN_192; + break; + case IPSEC_CRYPTO_ALG_AES_GCM_256: + case IPSEC_CRYPTO_ALG_AES_CBC_256: + case IPSEC_CRYPTO_ALG_AES_CTR_256: + w2->s.aes_key_len = ROC_IE_SA_AES_KEY_LEN_256; + break; + default: + break; + } + + if (!ipsec_sa_is_set_IS_AEAD (sa)) + { + switch (sa->integ_alg) + { + case IPSEC_INTEG_ALG_NONE: + w2->s.auth_type = ROC_IE_SA_AUTH_NULL; + break; + case IPSEC_INTEG_ALG_SHA1_96: + w2->s.auth_type = ROC_IE_SA_AUTH_SHA1; + break; + case IPSEC_INTEG_ALG_SHA_256_96: + case IPSEC_INTEG_ALG_SHA_256_128: + w2->s.auth_type = ROC_IE_SA_AUTH_SHA2_256; + break; + case IPSEC_INTEG_ALG_SHA_384_192: + w2->s.auth_type = ROC_IE_SA_AUTH_SHA2_384; + break; + case IPSEC_INTEG_ALG_SHA_512_256: + w2->s.auth_type = ROC_IE_SA_AUTH_SHA2_512; + break; + default: + clib_warning ("Unsupported authentication algorithm"); + return -1; + } + } + + oct_cn10k_ipsec_hmac_opad_ipad_gen (sa, hmac_opad_ipad); + + tmp_key = (u64 *) hmac_opad_ipad; + for (i = 0; i < (int) (ROC_CTX_MAX_OPAD_IPAD_LEN / sizeof (u64)); i++) + tmp_key[i] = clib_net_to_host_u64 (tmp_key[i]); + + if (ipsec_sa_is_set_IS_AEAD (sa)) + { + if (IPSEC_CRYPTO_ALG_IS_GCM (sa->crypto_alg)) + clib_memcpy (salt_key, &sa->salt, OCT_ROC_SALT_LEN); + tmp_salt = (u32 *) salt_key; + *tmp_salt = clib_net_to_host_u32 (*tmp_salt); + } + + /* Populate encryption key */ + clib_memcpy (cipher_key, sa->crypto_key.data, sa->crypto_key.len); + tmp_key = (u64 *) cipher_key; + for (i = 0; i < (int) (ROC_CTX_MAX_CKEY_LEN / sizeof (u64)); i++) + tmp_key[i] = clib_net_to_host_u64 (tmp_key[i]); + + w2->s.spi = sa->spi; + + return 0; +} + +static_always_inline void +oct_ipsec_sa_len_precalc (ipsec_sa_t *sa, oct_ipsec_encap_len_t *encap) +{ + if (ipsec_sa_is_set_IS_TUNNEL_V6 (sa)) + encap->partial_len = ROC_CPT_TUNNEL_IPV6_HDR_LEN; + else + encap->partial_len = ROC_CPT_TUNNEL_IPV4_HDR_LEN; + + if (sa->protocol == IPSEC_PROTOCOL_ESP) + { + encap->partial_len += ROC_CPT_ESP_HDR_LEN; + encap->roundup_len = ROC_CPT_ESP_TRL_LEN; + encap->footer_len = ROC_CPT_ESP_TRL_LEN; + } + else + { + encap->partial_len = ROC_CPT_AH_HDR_LEN; + } + + encap->partial_len += sa->crypto_iv_size; + encap->partial_len += sa->integ_icv_size; + + encap->roundup_byte = sa->esp_block_align; + encap->icv_len = sa->integ_icv_size; + + if (ipsec_sa_is_set_UDP_ENCAP (sa)) + encap->partial_len += sizeof (udp_header_t); +} + +static size_t +oct_ipsec_inb_ctx_size (struct roc_ot_ipsec_inb_sa *sa) +{ + size_t size; + + /* Variable based on anti-replay window */ + size = offsetof (struct roc_ot_ipsec_inb_sa, ctx) + + offsetof (struct roc_ot_ipsec_inb_ctx_update_reg, ar_winbits); + + if (sa->w0.s.ar_win) + size += (1 << (sa->w0.s.ar_win - 1)) * sizeof (u64); + + return size; +} + +static_always_inline void +oct_ipsec_common_inst_param_fill (void *sa, oct_ipsec_session_t *sess) +{ + union cpt_inst_w3 w3; + + clib_memset (&sess->inst, 0, sizeof (struct cpt_inst_s)); + + sess->inst.w7.u64 = oct_ipsec_crypto_inst_w7_get (sa); + + /* Populate word3 in CPT instruction template */ + w3.u64 = 0; + w3.s.qord = 1; + sess->inst.w3.u64 = w3.u64; +} + +static i32 +oct_ipsec_inb_session_update (oct_ipsec_session_t *sess, ipsec_sa_t *sa) +{ + union roc_ot_ipsec_sa_word2 w2; + u32 min_spi, max_spi, spi_mask; + struct roc_ot_ipsec_inb_sa *roc_sa; + oct_ipsec_inb_sa_priv_data_t *inb_sa_priv; + union cpt_inst_w4 inst_w4; + union roc_ot_ipsec_inb_param1 param1; + size_t offset; + + /* Ensure SPI is within the range supported by inline pktio device */ + spi_mask = roc_nix_inl_inb_spi_range (NULL, true, &min_spi, &max_spi); + if (sa->spi < min_spi || sa->spi > max_spi) + { + clib_warning ("SPI %u is not within supported range %u-%u", sa->spi, + min_spi, max_spi); + return -1; + } + + roc_sa = (struct roc_ot_ipsec_inb_sa *) roc_nix_inl_inb_sa_get (NULL, true, + sa->spi); + if (!roc_sa) + { + clib_warning ("Failed to create inbound sa session"); + return -1; + } + + inb_sa_priv = roc_nix_inl_ot_ipsec_inb_sa_sw_rsvd (roc_sa); + inb_sa_priv->user_data = sa->stat_index; + + if (ipsec_sa_is_set_UDP_ENCAP (sa)) + { + roc_sa->w10.s.udp_dst_port = 4500; + roc_sa->w10.s.udp_src_port = 4500; + } + + w2.u64 = 0; + int rv = oct_ipsec_sa_common_param_fill ( + &w2, roc_sa->cipher_key, roc_sa->w8.s.salt, roc_sa->hmac_opad_ipad, sa); + if (rv) + return rv; + + oct_ipsec_sa_len_precalc (sa, &sess->encap); + + if (sa->flags & IPSEC_SA_FLAG_USE_ANTI_REPLAY) + roc_sa->w0.s.ar_win = max_log2 (IPSEC_SA_ANTI_REPLAY_WINDOW_SIZE (sa)) - 5; + + /* Set direction and enable ESN (if needed) */ + w2.s.dir = ROC_IE_SA_DIR_INBOUND; + if (ipsec_sa_is_set_USE_ESN (sa)) + w2.s.esn_en = 1; + + /* + * Default options for pkt_out and pkt_fmt are with + * second pass meta and no defrag. + */ + roc_sa->w0.s.pkt_format = ROC_IE_OT_SA_PKT_FMT_META; + roc_sa->w0.s.pkt_output = ROC_IE_OT_SA_PKT_OUTPUT_NO_FRAG; + roc_sa->w0.s.pkind = ROC_IE_OT_CPT_PKIND; + + offset = offsetof (struct roc_ot_ipsec_inb_sa, ctx); + roc_sa->w0.s.hw_ctx_off = offset / 8; + roc_sa->w0.s.ctx_push_size = roc_sa->w0.s.hw_ctx_off + 1; + + /* Set context size, in number of 128B units following the first 128B */ + roc_sa->w0.s.ctx_size = + (round_pow2 (oct_ipsec_inb_ctx_size (roc_sa), 128) >> 7) - 1; + + /* Save SA index/SPI in cookie for now */ + roc_sa->w1.s.cookie = plt_cpu_to_be_32 (sa->spi & spi_mask); + + /* Enable SA */ + w2.s.valid = 1; + roc_sa->w2.u64 = w2.u64; + + asm volatile("dmb oshst" ::: "memory"); + + oct_ipsec_common_inst_param_fill (roc_sa, sess); + + /* Populate word4 in CPT instruction template */ + inst_w4.u64 = 0; + inst_w4.s.opcode_major = ROC_IE_OT_MAJOR_OP_PROCESS_INBOUND_IPSEC; + param1.u16 = 0; + /* Disable IP checksum verification by default */ + param1.s.ip_csum_disable = ROC_IE_OT_SA_INNER_PKT_IP_CSUM_DISABLE; + /* Disable L4 checksum verification by default */ + param1.s.l4_csum_disable = ROC_IE_OT_SA_INNER_PKT_L4_CSUM_DISABLE; + param1.s.esp_trailer_disable = 0; + inst_w4.s.param1 = param1.u16; + sess->inst.w4.u64 = inst_w4.u64; + + rv = roc_nix_inl_ctx_write (NULL, roc_sa, roc_sa, true, + sizeof (struct roc_ot_ipsec_inb_sa)); + if (rv) + { + clib_warning ("roc_nix_inl_ctx_write failed with '%s' error", + roc_error_msg_get (rv)); + return rv; + } + + rv = roc_nix_inl_sa_sync (NULL, roc_sa, true, ROC_NIX_INL_SA_OP_FLUSH); + if (rv) + { + clib_warning ( + "roc_nix_inl_sa_sync flush operation failed with '%s' error", + roc_error_msg_get (rv)); + return rv; + } + + return 0; +} + +static i32 +oct_ipsec_session_create (u32 sa_index) +{ + oct_ipsec_main_t *oim = &oct_ipsec_main; + ipsec_sa_t *sa = ipsec_sa_get (sa_index); + oct_ipsec_session_t *session = NULL; + u32 sess_index; + int rv; + + pool_get_aligned (oim->inline_ipsec_sessions, session, ROC_ALIGN); + clib_memset (session, 0, sizeof (*session)); + sess_index = session - oim->inline_ipsec_sessions; + + ASSERT (sa_index == sess_index); + + if (sa->flags & IPSEC_SA_FLAG_IS_INBOUND) + { + rv = oct_ipsec_inb_session_update (session, sa); + if (rv) + return rv; + } + + /* Initialize the ITF details in ipsec_session for tunnel SAs */ + if (ipsec_sa_is_set_IS_TUNNEL (sa)) + session->itf_sw_idx = ~0; + return 0; +} + +static i32 +oct_ipsec_session_destroy (u32 sa_index) +{ + oct_ipsec_main_t *oim = &oct_ipsec_main; + ipsec_sa_t *sa = ipsec_sa_get (sa_index); + oct_ipsec_session_t *session = NULL; + struct roc_ot_ipsec_inb_sa *roc_sa; + void *sa_dptr = NULL; + int rv; + + session = pool_elt_at_index (oim->inline_ipsec_sessions, sa_index); + if (pool_is_free (oim->inline_ipsec_sessions, session)) + return -1; + + if (sa->flags & IPSEC_SA_FLAG_IS_INBOUND) + { + roc_sa = (struct roc_ot_ipsec_inb_sa *) roc_nix_inl_inb_sa_get ( + NULL, true, sa->spi); + if (!roc_sa) + { + clib_warning ("roc_nix_inl_inb_sa_get failed to get SA for spi %u", + sa->spi); + return -1; + } + + sa_dptr = plt_zmalloc (sizeof (struct roc_ot_ipsec_inb_sa), 8); + if (sa_dptr != NULL) + { + roc_ot_ipsec_inb_sa_init (sa_dptr, true); + rv = roc_nix_inl_ctx_write (NULL, sa_dptr, roc_sa, true, + sizeof (struct roc_ot_ipsec_inb_sa)); + if (rv) + { + clib_warning ("roc_nix_inl_ctx_write failed - ROC error %s (%d)", + roc_error_msg_get (rv), rv); + return rv; + } + plt_free (sa_dptr); + } + } + + clib_memset (session, 0, sizeof (oct_ipsec_session_t)); + return 0; +} + +static clib_error_t * +oct_add_del_session (u32 sa_index, u8 is_add) +{ + ipsec_sa_t *sa; + + if (!is_add) + { + if (oct_ipsec_session_destroy (sa_index) < 0) + { + return clib_error_create ( + "IPsec session destroy operation failed for IPsec " + "index %u", + sa_index); + } + return 0; + } + + if (oct_ipsec_session_create (sa_index) < 0) + return clib_error_create ("ipsec session create failed for sa index %u", + sa_index); + + sa = ipsec_sa_get (sa_index); + + if (sa->flags & IPSEC_SA_FLAG_IS_INBOUND) + return oct_inl_inb_ipsec_flow_enable (); + + return 0; +} + +static clib_error_t * +oct_ipsec_check_support (ipsec_sa_t *sa) +{ + oct_crypto_main_t *ocm = &oct_crypto_main; + oct_crypto_dev_t *ocd = ocm->crypto_dev[0]; + union cpt_eng_caps hw_caps = ocd->roc_cpt->hw_caps[CPT_ENG_TYPE_IE]; + u8 is_cipher_algo_supported; + u8 is_auth_algo_supported; + + if (!ipsec_sa_is_set_IS_TUNNEL (sa)) + return clib_error_create ( + "Transport mode SA is not supported in Inline IPsec operation"); + + switch (sa->crypto_alg) + { + case IPSEC_CRYPTO_ALG_NONE: + is_cipher_algo_supported = 1; + break; + case IPSEC_CRYPTO_ALG_AES_GCM_128: + case IPSEC_CRYPTO_ALG_AES_GCM_192: + case IPSEC_CRYPTO_ALG_AES_GCM_256: + case IPSEC_CRYPTO_ALG_AES_CBC_128: + case IPSEC_CRYPTO_ALG_AES_CBC_192: + case IPSEC_CRYPTO_ALG_AES_CBC_256: + case IPSEC_CRYPTO_ALG_AES_CTR_128: + case IPSEC_CRYPTO_ALG_AES_CTR_192: + case IPSEC_CRYPTO_ALG_AES_CTR_256: + is_cipher_algo_supported = hw_caps.aes; + break; + default: + is_cipher_algo_supported = 0; + break; + } + + switch (sa->integ_alg) + { + case IPSEC_INTEG_ALG_NONE: + is_auth_algo_supported = 1; + break; + case IPSEC_INTEG_ALG_MD5_96: + case IPSEC_INTEG_ALG_SHA1_96: + case IPSEC_INTEG_ALG_SHA_256_128: + case IPSEC_INTEG_ALG_SHA_384_192: + case IPSEC_INTEG_ALG_SHA_512_256: + is_auth_algo_supported = hw_caps.sha1_sha2; + break; + default: + is_auth_algo_supported = 0; + break; + } + + if (!is_cipher_algo_supported) + return clib_error_create ("crypto-alg %U not supported", + format_ipsec_crypto_alg, sa->crypto_alg); + + if (!is_auth_algo_supported) + return clib_error_create ("integ-alg %U not supported", + format_ipsec_integ_alg, sa->integ_alg); + + return 0; +} + +vnet_dev_rv_t +oct_init_ipsec_backend (vlib_main_t *vm, vnet_dev_t *dev) +{ + ipsec_main_t *im = &ipsec_main; + int rv; + u32 idx; + + idx = ipsec_register_esp_backend ( + vm, im, "octeon backend", "esp4-encrypt", "esp4-encrypt-tun", + "esp4-decrypt", "esp4-decrypt-tun", "esp6-encrypt", "esp6-encrypt-tun", + "esp6-decrypt", "esp6-decrypt-tun", "esp-mpls-encrypt-tun", + oct_ipsec_check_support, oct_add_del_session); + + rv = ipsec_select_esp_backend (im, idx); + if (rv) + { + log_err (dev, "IPsec ESP backend selection failed"); + return VNET_DEV_ERR_INTERNAL; + } + return VNET_DEV_OK; +} + +vnet_dev_rv_t +oct_ipsec_inl_dev_inb_cfg (vlib_main_t *vm, vnet_dev_t *dev, + oct_inl_dev_cfg_t *inl_dev_cfg) +{ + oct_inl_dev_main_t *inl_dev_main = &oct_inl_dev_main; + oct_device_t *cd = vnet_dev_get_data (dev); + int rrv; + + cd->nix->ipsec_in_min_spi = inl_dev_main->in_min_spi; + cd->nix->ipsec_in_max_spi = inl_dev_main->in_max_spi; + + if ((rrv = roc_nix_inl_inb_init (cd->nix))) + { + log_err (dev, "roc_nix_inl_inb_init: %s [%d]", roc_error_msg_get (rrv), + rrv); + return VNET_DEV_ERR_UNSUPPORTED_DEVICE; + } + + roc_nix_inb_mode_set (cd->nix, true); + roc_nix_inl_inb_set (cd->nix, true); + + inl_dev_main->inb_sa_base = roc_nix_inl_inb_sa_base_get (NULL, true); + inl_dev_main->inb_sa_sz = roc_nix_inl_inb_sa_sz (NULL, true); + + inl_dev_main->inb_spi_mask = + roc_nix_inl_inb_spi_range (NULL, true, NULL, NULL); + + return VNET_DEV_OK; +} + +static int +oct_pool_inl_meta_pool_cb (u64 *aura_handle, uintptr_t *mpool, u32 buf_sz, + u32 nb_bufs, bool destroy, const char *mempool_name) +{ + extern oct_plt_init_param_t oct_plt_init_param; + u64 mem_start, mem_end, elem_addr; + struct npa_pool_s npapool; + struct npa_aura_s aura; + const char *mp_name; + u32 i; + u64 total_sz; + u64 roc_aura_handle; + int rv; + + mp_name = mempool_name ? mempool_name : OCT_NIX_INL_META_POOL_NAME; + + if (destroy) + return 0; + + buf_sz = PLT_ALIGN (buf_sz, ROC_ALIGN); + total_sz = nb_bufs * buf_sz; + + mem_start = (u64) oct_plt_init_param.oct_plt_zmalloc (total_sz, ROC_ALIGN); + if (!mem_start) + { + clib_warning ("Failed to allocate physmem for pool %s", mp_name); + return -1; + } + + clib_memset (&aura, 0, sizeof (struct npa_aura_s)); + clib_memset (&npapool, 0, sizeof (struct npa_pool_s)); + + npapool.nat_align = 1; + + rv = roc_npa_pool_create (&roc_aura_handle, buf_sz, nb_bufs, &aura, &npapool, + ROC_NPA_ZERO_AURA_F); + if (rv) + { + clib_warning ("roc_npa_pool_create failed with '%s' error", + roc_error_msg_get (rv)); + return -1; + } + + mem_end = mem_start + total_sz; + + roc_npa_aura_op_range_set (roc_aura_handle, mem_start, mem_end); + + elem_addr = mem_start; + for (i = 0; i < nb_bufs; i++) + { + roc_npa_aura_op_free (roc_aura_handle, 0, elem_addr); + elem_addr += buf_sz; + } + + /* Read back to confirm pointers are freed */ + roc_npa_aura_op_available (roc_aura_handle); + + *aura_handle = roc_aura_handle; + *mpool = (uintptr_t) mem_start; + + return 0; +} +vnet_dev_rv_t +oct_early_init_inline_ipsec (vlib_main_t *vm, vnet_dev_t *dev) +{ + vlib_buffer_main_t *bm = vm->buffer_main; + u8 bp_index = vlib_buffer_pool_get_default_for_numa (vm, 0); + vlib_buffer_pool_t *bp = NULL; + extern oct_plt_init_param_t oct_plt_init_param; + oct_inl_dev_main_t *inl_dev_main = &oct_inl_dev_main; + int rrv; + + bp = vec_elt_at_index (bm->buffer_pools, bp_index); + + inl_dev_main->inl_dev->ipsec_in_min_spi = inl_dev_main->in_min_spi; + inl_dev_main->inl_dev->ipsec_in_max_spi = inl_dev_main->in_max_spi; + inl_dev_main->inl_dev->wqe_skip = 0; + inl_dev_main->inl_dev->nb_meta_bufs = bp->n_buffers; + inl_dev_main->inl_dev->attach_cptlf = true; + + if ((rrv = roc_nix_inl_dev_init (inl_dev_main->inl_dev)) < 0) + { + log_err (dev, "roc_nix_inl_dev_init: %s [%d]", roc_error_msg_get (rrv), + rrv); + return VNET_DEV_ERR_UNSUPPORTED_DEVICE; + } + + roc_nix_inl_meta_pool_cb_register (oct_pool_inl_meta_pool_cb); + + return VNET_DEV_OK; +} + +vnet_dev_rv_t +oct_init_nix_inline_ipsec (vlib_main_t *vm, vnet_dev_t *inl_dev, + vnet_dev_t *dev) +{ + oct_inl_dev_cfg_t inl_dev_cfg; + vnet_dev_rv_t rv; + + if ((rv = oct_ipsec_inl_dev_inb_cfg (vm, dev, &inl_dev_cfg))) + return rv; + + return VNET_DEV_OK; +} diff --git a/src/plugins/dev_octeon/ipsec.h b/src/plugins/dev_octeon/ipsec.h new file mode 100644 index 0000000000..483a67f952 --- /dev/null +++ b/src/plugins/dev_octeon/ipsec.h @@ -0,0 +1,133 @@ +/* + * Copyright (c) 2024 Marvell. + * SPDX-License-Identifier: Apache-2.0 + * https://spdx.org/licenses/Apache-2.0.html + */ + +#ifndef _OCTEON_IPSEC_H_ +#define _OCTEON_IPSEC_H_ + +#define OCT_ROC_SALT_LEN 4 +#define OCT_IPSEC_MAX_SESSION 3600 + +#define foreach_octeon10_ipsec_ucc \ + _ (SUCCESS, success, INFO, "Packet successfully processed") \ + _ (ERR_SA_INVAL, err_sa_inval, ERROR, "SA invalid") \ + _ (ERR_SA_EXPIRED, err_sa_expired, ERROR, "SA hard-expired") \ + _ (ERR_SA_OVERFLOW, err_sa_overflow, ERROR, "SA overflow") \ + _ (ERR_SA_ESP_BAD_ALGO, err_sa_esp_bad_algo, ERROR, "ESP bad algorithm") \ + _ (ERR_SA_AH_BAD_ALGO, err_sa_ah_bad_algo, ERROR, "SA AH bad algorithm") \ + _ (ERR_SA_BAD_CTX, err_sa_bad_ctx, ERROR, "Bad SA context received on CPT") \ + _ (SA_CTX_FLAG_MISMATCH, sa_ctx_flag_mismatch, ERROR, \ + "SA context flags mismatch") \ + _ (ERR_AOP_IPSEC, err_aop_ipsec, ERROR, "AOP logical error") \ + _ (ERR_PKT_IP, err_pkt_ip, ERROR, "Bad IP version or TTL") \ + _ (ERR_PKT_IP6_BAD_EXT, err_pkt_ip6_bad_ext, ERROR, \ + "IPv6 mobility extension not supported") \ + _ (ERR_PKT_IP6_HBH, err_pkt_ip6_hbh, ERROR, \ + "Error with IPv6 hop-by-hop header") \ + _ (ERR_PKT_IP6_BIGEXT, err_pkt_ip6_bigext, ERROR, \ + "IPv6 extension header length exceeded") \ + _ (ERR_PKT_IP_ULP, err_pkt_ip_ulp, ERROR, "Bad protocol in IP header") \ + _ (ERR_PKT_SA_MISMATCH, err_pkt_sa_mismatch, ERROR, \ + "IP address mismatch b/w SA and packet") \ + _ (ERR_PKT_SPI_MISMATCH, err_pkt_spi_mismatch, ERROR, \ + "SPI mismatch b/w SA and packet") \ + _ (ERR_PKT_ESP_BADPAD, err_pkt_esp_badpad, ERROR, \ + "Bad padding in ESP packet") \ + _ (ERR_PKT_BADICV, err_pkt_badicv, ERROR, "ICV verification failed") \ + _ (ERR_PKT_REPLAY_SEQ, err_pkt_replay_seq, ERROR, \ + "Sequence number out of anti-replay window") \ + _ (ERR_PKT_BADNH, err_pkt_badnh, ERROR, "Bad next-hop") \ + _ (ERR_PKT_SA_PORT_MISMATCH, err_pkt_sa_port_mismatch, ERROR, \ + "Port mismatch b/w packet and SA") \ + _ (ERR_PKT_BAD_DLEN, err_pkt_bad_dlen, ERROR, "Dlen mismatch") \ + _ (ERR_SA_ESP_BAD_KEYS, err_sa_esp_bad_keys, ERROR, \ + "Bad key-size for selected ESP algorithm") \ + _ (ERR_SA_AH_BAD_KEYS, err_sa_ah_bad_keys, ERROR, \ + "Bad key-size for selected AH algorithm") \ + _ (ERR_SA_BAD_IP, err_sa_bad_ip, ERROR, \ + "IP version mismatch b/w packet and SA") \ + _ (ERR_PKT_IP_FRAG, err_pkt_ip_frag, ERROR, \ + "IPsec packet is an outer-IP fragment") \ + _ (ERR_PKT_REPLAY_WINDOW, err_pkt_replay_window, ERROR, \ + "Sequence number already seen") \ + _ (SUCCESS_PKT_IP_BADCSUM, success_pkt_ip_badcsum, ERROR, \ + "Bad IP checksum ") \ + _ (SUCCESS_PKT_L4_GOODCSUM, success_pkt_l4_goodcsum, INFO, \ + "Good inner L4 checksum") \ + _ (SUCCESS_PKT_L4_BADCSUM, success_pkt_l4_badcsum, ERROR, \ + "Bad inner L4 checksum") \ + _ (SUCCESS_SA_SOFTEXP_FIRST, success_sa_softexp_first, WARN, \ + "SA soft-expired - first encounter") \ + _ (SUCCESS_PKT_UDPESP_NZCSUM, success_pkt_udpesp_nzcsum, ERROR, \ + "Non-zero UDP checksum in UDP-ESP packet") \ + _ (SUCCESS_SA_SOFTEXP_AGAIN, success_sa_softexp_again, WARN, \ + "SA soft-expired - subsequent encounter") \ + _ (SUCCESS_PKT_UDP_ZEROCSUM, success_pkt_udp_zerocsum, INFO, \ + "Zero UDP checksum") + +typedef struct +{ + uint8_t partial_len; + uint8_t roundup_len; + uint8_t footer_len; + uint8_t roundup_byte; + uint8_t icv_len; +} oct_ipsec_encap_len_t; + +typedef struct +{ + u64 user_data; +} oct_ipsec_inb_sa_priv_data_t; + +typedef struct +{ + CLIB_CACHE_LINE_ALIGN_MARK (cacheline0); + /* Outbound SA */ + struct roc_ot_ipsec_outb_sa out_sa; + CLIB_CACHE_LINE_ALIGN_MARK (cacheline1); + struct cpt_inst_s inst; + uint16_t iv_offset; + uint8_t iv_length; + u32 itf_sw_idx; + /* Packet length for IPsec encapsulation */ + oct_ipsec_encap_len_t encap; +} oct_ipsec_session_t; + +typedef struct +{ + CLIB_CACHE_LINE_ALIGN_MARK (cacheline0); + oct_ipsec_session_t *inline_ipsec_sessions; + +} oct_ipsec_main_t; + +typedef struct +{ +} oct_inl_dev_cfg_t; + +typedef struct +{ + CLIB_CACHE_LINE_ALIGN_MARK (cacheline0); + struct roc_nix_inl_dev *inl_dev; + vnet_dev_t *vdev; + uintptr_t inb_sa_base; + u32 inb_sa_sz; + u32 inb_spi_mask; + u8 is_inl_ipsec_flow_enabled; + u32 in_min_spi; + u32 in_max_spi; +} oct_inl_dev_main_t; + +extern oct_ipsec_main_t oct_ipsec_main; +extern oct_inl_dev_main_t oct_inl_dev_main; + +vnet_dev_rv_t oct_init_ipsec_backend (vlib_main_t *vm, vnet_dev_t *dev); + +vnet_dev_rv_t oct_early_init_inline_ipsec (vlib_main_t *vm, vnet_dev_t *dev); +vnet_dev_rv_t oct_init_nix_inline_ipsec (vlib_main_t *vm, vnet_dev_t *inl_dev, + vnet_dev_t *dev); + +clib_error_t *oct_inl_inb_ipsec_flow_enable (void); + +#endif /* _OCTEON_IPSEC_H_ */ diff --git a/src/plugins/dev_octeon/octeon.h b/src/plugins/dev_octeon/octeon.h index 8e10255419..2a6c72bffc 100644 --- a/src/plugins/dev_octeon/octeon.h +++ b/src/plugins/dev_octeon/octeon.h @@ -20,6 +20,7 @@ #include #include #include +#include #define OCT_NPA_MAX_POOLS 8192 #define OCT_BATCH_ALLOC_IOVA0_MASK 0xFFFFFFFFFFFFFF80 @@ -46,6 +47,8 @@ typedef enum OCT_DEVICE_TYPE_SDP_VF, OCT_DEVICE_TYPE_O10K_CPT_VF, OCT_DEVICE_TYPE_O9K_CPT_VF, + OCT_DEVICE_TYPE_RVU_INL_PF, + OCT_DEVICE_TYPE_RVU_INL_VF, } __clib_packed oct_device_type_t; typedef struct @@ -123,6 +126,17 @@ typedef struct struct roc_nix_sq sq; } oct_txq_t; +typedef struct +{ + oct_device_t **oct_dev; + u8 inl_dev_initialized : 1; + u8 use_single_rx_aura : 1; + u64 aura_handle; + +} oct_main_t; + +extern oct_main_t oct_main; + /* format.c */ format_function_t format_oct_port_status; format_function_t format_oct_rx_trace; @@ -144,7 +158,7 @@ vnet_dev_rv_t oct_rx_queue_alloc (vlib_main_t *, vnet_dev_rx_queue_t *); vnet_dev_rv_t oct_tx_queue_alloc (vlib_main_t *, vnet_dev_tx_queue_t *); void oct_rx_queue_free (vlib_main_t *, vnet_dev_rx_queue_t *); void oct_tx_queue_free (vlib_main_t *, vnet_dev_tx_queue_t *); -vnet_dev_rv_t oct_rxq_init (vlib_main_t *, vnet_dev_rx_queue_t *); +vnet_dev_rv_t oct_rxq_init (vlib_main_t *, vnet_dev_rx_queue_t *, u32); vnet_dev_rv_t oct_txq_init (vlib_main_t *, vnet_dev_tx_queue_t *); void oct_rxq_deinit (vlib_main_t *, vnet_dev_rx_queue_t *); void oct_txq_deinit (vlib_main_t *, vnet_dev_tx_queue_t *); @@ -185,6 +199,19 @@ vnet_dev_rv_t oct_txq_get_stats (vlib_main_t *, vnet_dev_port_t *, vlib_log (VLIB_LOG_LEVEL_ERR, oct_log.class, "%U: " f, \ format_vnet_dev_addr, (dev), ##__VA_ARGS__) +#define foreach_oct_rx_node_counter \ + _ (ERR_UNDEFINED, err_undefined, ERROR, "undefined decrypt error") + +/* clang-format off */ +typedef enum +{ +#define _(f, n, s, d) OCT_RX_NODE_CTR_##f, + foreach_octeon10_ipsec_ucc + foreach_oct_rx_node_counter +#undef _ +} oct_rx_node_counter_t; +/* clang-format on */ + #define foreach_oct_tx_node_counter \ _ (CHAIN_TOO_LONG, chain_too_long, ERROR, "drop due to buffer chain > 6") \ _ (NO_FREE_SLOTS, no_free_slots, ERROR, "no free tx slots") \ @@ -213,4 +240,5 @@ typedef struct u32 sw_if_index; oct_tx_desc_t desc; } oct_tx_trace_t; + #endif /* _OCTEON_H_ */ diff --git a/src/plugins/dev_octeon/port.c b/src/plugins/dev_octeon/port.c index 103e513b64..1d52e0fc02 100644 --- a/src/plugins/dev_octeon/port.c +++ b/src/plugins/dev_octeon/port.c @@ -8,12 +8,15 @@ #include #include #include +#include #include #define OCT_FLOW_PREALLOC_SIZE 1 #define OCT_FLOW_MAX_PRIORITY 7 #define OCT_ETH_LINK_SPEED_100G 100000 /**< 100 Gbps */ +extern oct_inl_dev_main_t oct_inl_dev_main; + VLIB_REGISTER_LOG_CLASS (oct_log, static) = { .class_name = "octeon", .subclass_name = "port", @@ -210,9 +213,13 @@ oct_port_init (vlib_main_t *vm, vnet_dev_port_t *port) } cp->npc_initialized = 1; + u64 total_sz = 0; + foreach_vnet_dev_port_rx_queue (q, port) + total_sz += q->size; + foreach_vnet_dev_port_rx_queue (q, port) if (q->enabled) - if ((rv = oct_rxq_init (vm, q))) + if ((rv = oct_rxq_init (vm, q, total_sz))) { oct_port_deinit (vm, port); return rv; @@ -441,6 +448,7 @@ oct_txq_stop (vlib_main_t *vm, vnet_dev_tx_queue_t *txq) vnet_dev_rv_t oct_port_start (vlib_main_t *vm, vnet_dev_port_t *port) { + oct_inl_dev_main_t *inl_main = &oct_inl_dev_main; vnet_dev_t *dev = port->dev; oct_device_t *cd = vnet_dev_get_data (dev); oct_port_t *cp = vnet_dev_get_port_data (port); @@ -449,12 +457,21 @@ oct_port_start (vlib_main_t *vm, vnet_dev_port_t *port) vnet_dev_rv_t rv; int rrv; - log_debug (port->dev, "port start: port %u", port->port_id); + log_info (port->dev, "port start: port %u", port->port_id); foreach_vnet_dev_port_rx_queue (q, port) if ((rv = oct_rxq_start (vm, q)) != VNET_DEV_OK) goto done; + if (inl_main->inl_dev) + { + if ((rrv = roc_nix_inl_rq_ena_dis (nix, true))) + { + rv = oct_roc_err (dev, rrv, "roc_nix_inl_rq_ena_dis failed"); + goto done; + } + } + foreach_vnet_dev_port_tx_queue (q, port) { oct_txq_t *ctq = vnet_dev_get_tx_queue_data (q); diff --git a/src/plugins/dev_octeon/queue.c b/src/plugins/dev_octeon/queue.c index d6ae794fb8..b9f19714c9 100644 --- a/src/plugins/dev_octeon/queue.c +++ b/src/plugins/dev_octeon/queue.c @@ -86,8 +86,9 @@ oct_tx_queue_free (vlib_main_t *vm, vnet_dev_tx_queue_t *txq) } vnet_dev_rv_t -oct_rxq_init (vlib_main_t *vm, vnet_dev_rx_queue_t *rxq) +oct_rxq_init (vlib_main_t *vm, vnet_dev_rx_queue_t *rxq, u32 total_sz) { + oct_main_t *om = &oct_main; oct_rxq_t *crq = vnet_dev_get_rx_queue_data (rxq); vnet_dev_t *dev = rxq->port->dev; oct_device_t *cd = vnet_dev_get_data (dev); @@ -99,11 +100,19 @@ oct_rxq_init (vlib_main_t *vm, vnet_dev_rx_queue_t *rxq) struct npa_aura_s aura = {}; struct npa_pool_s npapool = { .nat_align = 1 }; - if ((rrv = roc_npa_pool_create (&crq->aura_handle, bp->alloc_size, rxq->size, - &aura, &npapool, 0))) + if (!om->use_single_rx_aura || !om->aura_handle) { - oct_rxq_deinit (vm, rxq); - return oct_roc_err (dev, rrv, "roc_npa_pool_create() failed"); + if ((rrv = roc_npa_pool_create (&crq->aura_handle, bp->alloc_size, + total_sz, &aura, &npapool, 0))) + { + oct_rxq_deinit (vm, rxq); + return oct_roc_err (dev, rrv, "roc_npa_pool_create() failed"); + } + om->aura_handle = crq->aura_handle; + } + else + { + crq->aura_handle = om->aura_handle; } crq->npa_pool_initialized = 1; @@ -132,9 +141,9 @@ oct_rxq_init (vlib_main_t *vm, vnet_dev_rx_queue_t *rxq) .qid = rxq->queue_id, .cqid = crq->cq.qid, .aura_handle = crq->aura_handle, - .first_skip = crq->hdr_off + sizeof (vlib_buffer_t), - .later_skip = crq->hdr_off + sizeof (vlib_buffer_t), - .lpb_size = bp->data_size + crq->hdr_off + sizeof (vlib_buffer_t), + .first_skip = sizeof (vlib_buffer_t), + .later_skip = sizeof (vlib_buffer_t), + .lpb_size = bp->data_size + sizeof (vlib_buffer_t), .flow_tag_width = 32, }; @@ -153,6 +162,15 @@ oct_rxq_init (vlib_main_t *vm, vnet_dev_rx_queue_t *rxq) (0x7 << 4); log_debug (dev, "RQ %u initialised", crq->cq.qid); + /* Configure inline device rq */ + rrv = roc_nix_inl_dev_rq_get (&crq->rq, 0 /* disable */); + if (rrv) + { + clib_warning ("roc_nix_inl_dev_rq_get failed with '%s' error", + roc_error_msg_get (rrv)); + + return -1; + } return VNET_DEV_OK; } diff --git a/src/plugins/dev_octeon/rx_node.c b/src/plugins/dev_octeon/rx_node.c index b057c4d704..22152a9b48 100644 --- a/src/plugins/dev_octeon/rx_node.c +++ b/src/plugins/dev_octeon/rx_node.c @@ -5,6 +5,7 @@ #include #include #include +#include #include #include @@ -19,6 +20,7 @@ typedef struct u64 parse_w0_or; u32 n_left_to_next; u32 *to_next; + u16 *next; u32 n_rx_pkts; u32 n_rx_bytes; u32 n_segs; @@ -99,8 +101,410 @@ oct_rx_attach_tail (vlib_main_t *vm, oct_rx_node_ctx_t *ctx, vlib_buffer_t *h, } static_always_inline u32 -oct_rx_batch (vlib_main_t *vm, oct_rx_node_ctx_t *ctx, - vnet_dev_rx_queue_t *rxq, u32 n) +oct_ipsec_update_itf_sw_idx (oct_ipsec_session_t *session, u32 sa_idx) +{ + clib_bihash_kv_24_16_t bkey60 = { 0 }; + clib_bihash_kv_8_16_t bkey40 = { 0 }; + ipsec_tun_lkup_result_t *res; + ipsec4_tunnel_kv_t *key40; + ipsec6_tunnel_kv_t *key60; + ip_address_t *ip_addr; + ipsec_main_t *ipm; + ipsec_sa_t *sa; + i32 rv; + + sa = ipsec_sa_get (sa_idx); + ASSERT (sa); + + ipm = &ipsec_main; + ip_addr = &sa->tunnel.t_src; + + if (AF_IP4 == ip_addr->version) + { + key40 = (ipsec4_tunnel_kv_t *) &bkey40; + ipsec4_tunnel_mk_key (key40, &ip_addr->ip.ip4, + clib_host_to_net_u32 (sa->spi)); + rv = clib_bihash_search_inline_8_16 (&ipm->tun4_protect_by_key, &bkey40); + ASSERT (!rv); + + res = (ipsec_tun_lkup_result_t *) &bkey40.value; + } + else + { + + key60 = (ipsec6_tunnel_kv_t *) &bkey60; + key60->key.remote_ip = ip_addr->ip.ip6; + key60->key.spi = clib_host_to_net_u32 (sa->spi); + key60->key.__pad = 0; + + rv = + clib_bihash_search_inline_24_16 (&ipm->tun6_protect_by_key, &bkey60); + ASSERT (!rv); + + res = (ipsec_tun_lkup_result_t *) &bkey60.value; + } + + /* Store the ITF sw_if_index in the SA session to avoid duplicate + lookups for each packet */ + session->itf_sw_idx = res->sw_if_index; + + return res->sw_if_index; +} + +static_always_inline void +oct_rx_ipsec_update_sa_counters_x4 (vlib_main_t *vm, vlib_buffer_t *b0, + vlib_buffer_t *b1, vlib_buffer_t *b2, + vlib_buffer_t *b3, u32 ilen, u8 frag_cnt, + u32 oct_sa_idx) +{ + vlib_combined_counter_main_t *rx_counter; + ipsec_main_t *im = &ipsec_main; + oct_ipsec_session_t *session; + struct roc_ot_ipsec_inb_sa *roc_sa; + vnet_interface_main_t *vim; + oct_ipsec_main_t *oim = &oct_ipsec_main; + oct_inl_dev_main_t *oidm = &oct_inl_dev_main; + oct_ipsec_inb_sa_priv_data_t *inb_sa_priv; + u32 sa_idx, itf_sw_idx; + vnet_main_t *vnm; + + vnm = im->vnet_main; + vim = &vnm->interface_main; + rx_counter = vim->combined_sw_if_counters + VNET_INTERFACE_COUNTER_RX; + + roc_sa = roc_nix_inl_ot_ipsec_inb_sa (oidm->inb_sa_base, oct_sa_idx); + inb_sa_priv = roc_nix_inl_ot_ipsec_inb_sa_sw_rsvd (roc_sa); + + sa_idx = (u32) inb_sa_priv->user_data; + + vnet_buffer (b0)->ipsec.sad_index = sa_idx; + vnet_buffer (b1)->ipsec.sad_index = sa_idx; + vnet_buffer (b2)->ipsec.sad_index = sa_idx; + vnet_buffer (b3)->ipsec.sad_index = sa_idx; + + ASSERT (sa_idx < vec_len (oim->inline_ipsec_sessions)); + + session = pool_elt_at_index (oim->inline_ipsec_sessions, sa_idx); + itf_sw_idx = session->itf_sw_idx; + /* + * Check if itf_sw_idx is populated already. First packet on the SA + * populates the itf_sw_idx in the SA session + */ + if (PREDICT_FALSE (itf_sw_idx == ~0)) + itf_sw_idx = oct_ipsec_update_itf_sw_idx (session, sa_idx); + + /* Update IPsec counters with inner IP length */ + vlib_increment_combined_counter (&ipsec_sa_counters, vm->thread_index, + sa_idx, frag_cnt, ilen); + + /* Update ITF counters with inner IP length */ + vlib_increment_combined_counter (rx_counter, vm->thread_index, itf_sw_idx, + frag_cnt, ilen); +} +static_always_inline void +oct_rx_ipsec_update_counters_x4 (vlib_main_t *vm, vlib_buffer_t *b0, u32 ilen0, + u8 frag_cnt0, u32 idx0, vlib_buffer_t *b1, + u32 ilen1, u8 frag_cnt1, u32 idx1, + vlib_buffer_t *b2, u32 ilen2, u8 frag_cnt2, + u32 idx2, vlib_buffer_t *b3, u32 ilen3, + u8 frag_cnt3, u32 idx3) +{ + vlib_combined_counter_main_t *rx_counter; + oct_inl_dev_main_t *oidm = &oct_inl_dev_main; + oct_ipsec_main_t *oim = &oct_ipsec_main; + struct roc_ot_ipsec_inb_sa *roc_sa0, *roc_sa1; + struct roc_ot_ipsec_inb_sa *roc_sa2, *roc_sa3; + oct_ipsec_inb_sa_priv_data_t *inb_sa_priv; + ipsec_main_t *im = &ipsec_main; + oct_ipsec_session_t *session; + vnet_interface_main_t *vim; + u32 sa_idx0, itf_sw_idx0; + u32 sa_idx1, itf_sw_idx1; + u32 sa_idx2, itf_sw_idx2; + u32 sa_idx3, itf_sw_idx3; + vnet_main_t *vnm; + u32 idx_xor; + + idx_xor = idx0 ^ idx1; + idx_xor += idx1 ^ idx2; + idx_xor += idx2 ^ idx3; + + if (!idx_xor) + { + oct_rx_ipsec_update_sa_counters_x4 ( + vm, b0, b1, b2, b3, ilen0 + ilen1 + ilen2 + ilen3, + frag_cnt0 + frag_cnt1 + frag_cnt2 + frag_cnt3, idx0); + return; + } + + vnm = im->vnet_main; + vim = &vnm->interface_main; + rx_counter = vim->combined_sw_if_counters + VNET_INTERFACE_COUNTER_RX; + + roc_sa0 = roc_nix_inl_ot_ipsec_inb_sa (oidm->inb_sa_base, idx0); + inb_sa_priv = roc_nix_inl_ot_ipsec_inb_sa_sw_rsvd (roc_sa0); + + sa_idx0 = (u32) inb_sa_priv->user_data; + vnet_buffer (b0)->ipsec.sad_index = sa_idx0; + + roc_sa1 = roc_nix_inl_ot_ipsec_inb_sa (oidm->inb_sa_base, idx1); + inb_sa_priv = roc_nix_inl_ot_ipsec_inb_sa_sw_rsvd (roc_sa1); + + sa_idx1 = (u32) inb_sa_priv->user_data; + vnet_buffer (b1)->ipsec.sad_index = sa_idx1; + + roc_sa2 = roc_nix_inl_ot_ipsec_inb_sa (oidm->inb_sa_base, idx2); + inb_sa_priv = roc_nix_inl_ot_ipsec_inb_sa_sw_rsvd (roc_sa2); + + sa_idx2 = (u32) inb_sa_priv->user_data; + vnet_buffer (b2)->ipsec.sad_index = sa_idx2; + + roc_sa3 = roc_nix_inl_ot_ipsec_inb_sa (oidm->inb_sa_base, idx3); + inb_sa_priv = roc_nix_inl_ot_ipsec_inb_sa_sw_rsvd (roc_sa3); + + sa_idx3 = (u32) inb_sa_priv->user_data; + vnet_buffer (b3)->ipsec.sad_index = sa_idx3; + + ASSERT (sa_idx0 < vec_len (oim->inline_ipsec_sessions)); + ASSERT (sa_idx1 < vec_len (oim->inline_ipsec_sessions)); + ASSERT (sa_idx2 < vec_len (oim->inline_ipsec_sessions)); + ASSERT (sa_idx3 < vec_len (oim->inline_ipsec_sessions)); + + session = pool_elt_at_index (oim->inline_ipsec_sessions, sa_idx0); + itf_sw_idx0 = session->itf_sw_idx; + /* Check if itf_sw_idx is populated already. First packet on the SA + populates the itf_sw_idx in the SA session */ + if (PREDICT_FALSE (itf_sw_idx0 == ~0)) + itf_sw_idx0 = oct_ipsec_update_itf_sw_idx (session, sa_idx0); + + session = pool_elt_at_index (oim->inline_ipsec_sessions, sa_idx1); + itf_sw_idx1 = session->itf_sw_idx; + if (PREDICT_FALSE (itf_sw_idx1 == ~0)) + itf_sw_idx1 = oct_ipsec_update_itf_sw_idx (session, sa_idx1); + + session = pool_elt_at_index (oim->inline_ipsec_sessions, sa_idx2); + itf_sw_idx2 = session->itf_sw_idx; + if (PREDICT_FALSE (itf_sw_idx2 == ~0)) + itf_sw_idx2 = oct_ipsec_update_itf_sw_idx (session, sa_idx2); + + session = pool_elt_at_index (oim->inline_ipsec_sessions, sa_idx3); + itf_sw_idx3 = session->itf_sw_idx; + if (PREDICT_FALSE (itf_sw_idx3 == ~0)) + itf_sw_idx3 = oct_ipsec_update_itf_sw_idx (session, sa_idx3); + + /* Update IPsec counters with outer IP length */ + vlib_increment_combined_counter (&ipsec_sa_counters, vm->thread_index, + sa_idx0, frag_cnt0, ilen0); + vlib_increment_combined_counter (&ipsec_sa_counters, vm->thread_index, + sa_idx1, frag_cnt1, ilen1); + vlib_increment_combined_counter (&ipsec_sa_counters, vm->thread_index, + sa_idx2, frag_cnt2, ilen2); + vlib_increment_combined_counter (&ipsec_sa_counters, vm->thread_index, + sa_idx3, frag_cnt3, ilen3); + + /* Update ITF counters with inner IP length */ + vlib_increment_combined_counter (rx_counter, vm->thread_index, itf_sw_idx0, + frag_cnt0, ilen0); + vlib_increment_combined_counter (rx_counter, vm->thread_index, itf_sw_idx1, + frag_cnt1, ilen1); + vlib_increment_combined_counter (rx_counter, vm->thread_index, itf_sw_idx2, + frag_cnt2, ilen2); + vlib_increment_combined_counter (rx_counter, vm->thread_index, itf_sw_idx3, + frag_cnt3, ilen3); +} + +static_always_inline void +oct_rx_ipsec_update_counters (vlib_main_t *vm, vlib_buffer_t *b, u32 ilen, + u8 frag_cnt, u32 idx) +{ + vlib_combined_counter_main_t *rx_counter; + ipsec_main_t *im = &ipsec_main; + oct_ipsec_session_t *session; + struct roc_ot_ipsec_inb_sa *roc_sa; + oct_ipsec_inb_sa_priv_data_t *inb_sa_priv; + u32 sa_idx, itf_sw_idx; + vnet_interface_main_t *vim; + oct_ipsec_main_t *oim = &oct_ipsec_main; + oct_inl_dev_main_t *oidm = &oct_inl_dev_main; + vnet_main_t *vnm; + + vnm = im->vnet_main; + vim = &vnm->interface_main; + rx_counter = vim->combined_sw_if_counters + VNET_INTERFACE_COUNTER_RX; + + roc_sa = roc_nix_inl_ot_ipsec_inb_sa (oidm->inb_sa_base, idx); + inb_sa_priv = roc_nix_inl_ot_ipsec_inb_sa_sw_rsvd (roc_sa); + + sa_idx = (u32) inb_sa_priv->user_data; + + vnet_buffer (b)->ipsec.sad_index = sa_idx; + ASSERT (sa_idx < vec_len (oim->inline_ipsec_sessions)); + + session = pool_elt_at_index (oim->inline_ipsec_sessions, sa_idx); + itf_sw_idx = session->itf_sw_idx; + /* + * Check if itf_sw_idx is populated already. First packet on the SA + * populates the itf_sw_idx in the SA session. + */ + if (PREDICT_FALSE (itf_sw_idx == ~0)) + itf_sw_idx = oct_ipsec_update_itf_sw_idx (session, sa_idx); + + /* Update IPsec counters with inner IP length */ + vlib_increment_combined_counter (&ipsec_sa_counters, vm->thread_index, + sa_idx, frag_cnt, ilen); + + /* Update ITF counters with inner IP length */ + vlib_increment_combined_counter (rx_counter, vm->thread_index, itf_sw_idx, + frag_cnt, ilen); +} + +static_always_inline u8 +oct_is_packet_from_cpt (union nix_rx_parse_u *rxp) +{ + return rxp->chan >> 11; +} + +static_always_inline uword +oct_ipsec_is_inl_op_success (struct cpt_parse_hdr_s *cpt_hdr) +{ + return (((1U << cpt_hdr->w3.hw_ccode) & CPT_COMP_HWGOOD_MASK) && + roc_ie_ot_ucc_is_success (cpt_hdr->w3.uc_ccode)); +} + +static_always_inline u32 +oct_get_len_from_meta (struct cpt_parse_hdr_s *cpt_hdr, u64 w0, u64 w4) +{ + u32 len; + uintptr_t ip; + ip = (uintptr_t) cpt_hdr + ((w4 >> 16) & 0xFF); + ip += ((w0 >> 40) & 0x6); + len = plt_be_to_cpu_16 (*(u16 *) ip); + len += ((w4 >> 16) & 0xFF) - (w4 & 0xFF); + len += (w0 & BIT (42)) ? 40 : 0; + + return len; +} + +static_always_inline void +oct_rx_ipsec_set_error (vlib_main_t *vm, vlib_node_runtime_t *node, + vlib_buffer_t *b, u16 uc_err) +{ + switch (uc_err) + { + /* clang-format off */ +#define _(f, n, s, d) \ + case ROC_IE_OT_UCC_##f: \ + b->error = node->errors[OCT_RX_NODE_CTR_##f]; \ + break; + foreach_octeon10_ipsec_ucc; +#undef _ + /* clang-format on */ + default: + b->error = node->errors[OCT_RX_NODE_CTR_ERR_UNDEFINED]; + } +} + +static_always_inline u32 +oct_rx_inl_ipsec_vlib_from_cq (vlib_main_t *vm, vlib_node_runtime_t *node, + oct_nix_rx_cqe_desc_t *d, vlib_buffer_t **b, + oct_rx_node_ctx_t *ctx, + vlib_buffer_template_t *bt, + struct cpt_parse_hdr_s *cpt_hdr, + vlib_buffer_t **buffs, u32 *err_flags) +{ + union nix_rx_parse_u *orig_rxp; + u32 is_fail, olen, esp_sz, l2_ol3_sz, idx; + u64 *wqe_ptr; + + cpt_hdr = (struct cpt_parse_hdr_s *) *(((u64 *) d) + 9); + wqe_ptr = (u64 *) clib_net_to_host_u64 (cpt_hdr->wqe_ptr); + + b[0] = (vlib_buffer_t *) wqe_ptr; + orig_rxp = (union nix_rx_parse_u *) (wqe_ptr + 1); + l2_ol3_sz = orig_rxp->leptr - orig_rxp->laptr; + olen = orig_rxp->pkt_lenm1 + 1; + esp_sz = olen - l2_ol3_sz; + ctx->to_next[0] = vlib_get_buffer_index (vm, b[0]); + b[0]->template = *bt; + b[0]->flow_id = d[0].parse.w[3] >> 48; + *err_flags |= ((d[0].parse.w[0] >> 20) & 0xFFF); + ctx->n_segs += 1; + + is_fail = !oct_ipsec_is_inl_op_success (cpt_hdr); + + if (PREDICT_FALSE (is_fail)) + { + b[0]->current_length = olen; + ctx->next[0] = VNET_DEV_ETH_RX_PORT_NEXT_DROP; + oct_rx_ipsec_set_error (vm, node, b[0], cpt_hdr->w3.uc_ccode); + } + else + { + ctx->next[0] = ctx->next_index; + b[0]->current_length = + oct_get_len_from_meta (cpt_hdr, d[0].parse.w[0], d[0].parse.w[4]); + idx = cpt_hdr->w0.cookie; + oct_rx_ipsec_update_counters (vm, b[0], esp_sz, 1, idx); + } + ctx->n_rx_bytes += olen; + + buffs[0] = b[0]; + + return 0; +} +static_always_inline u32 +oct_rx_vlib_from_cq (vlib_main_t *vm, oct_nix_rx_cqe_desc_t *d, + vlib_buffer_t **b, oct_rx_node_ctx_t *ctx, + vlib_buffer_template_t *bt, u64 meta_aura_handle, + vlib_buffer_t **buffs, u32 *err_flags) +{ + b[0] = (vlib_buffer_t *) d->segs0[0] - 1; + ctx->to_next[0] = vlib_get_buffer_index (vm, b[0]); + ctx->next[0] = ctx->next_index; + b[0]->template = *bt; + ctx->n_rx_bytes += b[0]->current_length = d[0].sg0.seg1_size; + b[0]->flow_id = d[0].parse.w[3] >> 48; + *err_flags |= ((d[0].parse.w[0] >> 20) & 0xFFF); + ctx->n_segs += 1; + if (d[0].sg0.segs > 1) + oct_rx_attach_tail (vm, ctx, b[0], d + 0); + buffs[0] = b[0]; + return 0; +} + +#define OCT_PUSH_META_TO_FREE(_metabuf, _laddr, _loff_p) \ + do \ + { \ + *(u64 *) ((_laddr) + (*(_loff_p) << 3)) = (u64) _metabuf; \ + *(_loff_p) = *(_loff_p) + 1; \ + } \ + while (0) + +#define LMT_OFF(lmt_addr, lmt_num, offset) \ + (void *) ((uintptr_t) (lmt_addr) + \ + ((u64) (lmt_num) << ROC_LMT_LINE_SIZE_LOG2) + (offset)) + +static_always_inline void +oct_rx_flush_meta_burst (u16 lmt_id, u64 data, u16 lnum, uintptr_t aura_handle) +{ + u64 pa; + + /* Prepare PA and Data */ + pa = roc_npa_aura_handle_to_base (aura_handle) + NPA_LF_AURA_BATCH_FREE0; + pa |= ((data & 0x7) << 4); + + data >>= 3; + data <<= 19; + data |= (u64) lmt_id; + data |= (u64) (lnum - 1) << 12; + + roc_lmt_submit_steorl (data, pa); +} + +static_always_inline u32 +oct_rx_batch (vlib_main_t *vm, vlib_node_runtime_t *node, + oct_rx_node_ctx_t *ctx, vnet_dev_rx_queue_t *rxq, u32 n, + vlib_buffer_t **buffs) { oct_rxq_t *crq = vnet_dev_get_rx_queue_data (rxq); vlib_buffer_template_t bt = rxq->buffer_template; @@ -108,9 +512,34 @@ oct_rx_batch (vlib_main_t *vm, oct_rx_node_ctx_t *ctx, u32 b2_err_flags = 0, b3_err_flags = 0; u32 n_left, err_flags = 0; oct_nix_rx_cqe_desc_t *d = ctx->next_desc; + struct cpt_parse_hdr_s *cpt_hdr0, *cpt_hdr1; + struct cpt_parse_hdr_s *cpt_hdr2, *cpt_hdr3; + union nix_rx_parse_u *orig_rxp0, *orig_rxp1; + union nix_rx_parse_u *orig_rxp2, *orig_rxp3; + u8 is_b0_from_cpt, is_b1_from_cpt; + u8 is_b2_from_cpt, is_b3_from_cpt; + u64 *wqe_ptr0, *wqe_ptr1; + u64 *wqe_ptr2, *wqe_ptr3; + u32 is_fail0, is_fail1, is_fail2, is_fail3; + u32 olen0, olen1, olen2, olen3; + u32 esp_sz0, esp_sz1, esp_sz2, esp_sz3; + u32 l2_ol3_sz0, l2_ol3_sz1, l2_ol3_sz2, l2_ol3_sz3; + u32 idx0, idx1, idx2, idx3; vlib_buffer_t *b[4]; - - for (n_left = n; n_left >= 8; d += 4, n_left -= 4, ctx->to_next += 4) + u8 n_from_cpt, n_cpt_err; + u64 meta_aura_handle; + u64 lbase = crq->lmt_base_addr; + u8 loff = 0, lnum = 0, shft = 0; + u16 lmt_id; + u64 laddr; + + meta_aura_handle = crq->rq.meta_aura_handle; + ROC_LMT_BASE_ID_GET (lbase, lmt_id); + laddr = lbase; + laddr += 8; + + for (n_left = n; n_left >= 8; + d += 4, n_left -= 4, ctx->to_next += 4, ctx->next += 4) { u32 segs = 0; clib_prefetch_store (oct_seg_to_bp (d[4].segs0[0])); @@ -121,53 +550,377 @@ oct_rx_batch (vlib_main_t *vm, oct_rx_node_ctx_t *ctx, clib_prefetch_store (oct_seg_to_bp (d[7].segs0[0])); b[2] = oct_seg_to_bp (d[2].segs0[0]); b[3] = oct_seg_to_bp (d[3].segs0[0]); - ctx->to_next[0] = vlib_get_buffer_index (vm, b[0]); - ctx->to_next[1] = vlib_get_buffer_index (vm, b[1]); - ctx->to_next[2] = vlib_get_buffer_index (vm, b[2]); - ctx->to_next[3] = vlib_get_buffer_index (vm, b[3]); - b[0]->template = bt; - b[1]->template = bt; - b[2]->template = bt; - b[3]->template = bt; - ctx->n_rx_bytes += b[0]->current_length = d[0].sg0.seg1_size; - ctx->n_rx_bytes += b[1]->current_length = d[1].sg0.seg1_size; - ctx->n_rx_bytes += b[2]->current_length = d[2].sg0.seg1_size; - ctx->n_rx_bytes += b[3]->current_length = d[3].sg0.seg1_size; - b[0]->flow_id = d[0].parse.w[3] >> 48; - b[1]->flow_id = d[1].parse.w[3] >> 48; - b[2]->flow_id = d[2].parse.w[3] >> 48; - b[3]->flow_id = d[3].parse.w[3] >> 48; - ctx->n_segs += 4; - segs = d[0].sg0.segs + d[1].sg0.segs + d[2].sg0.segs + d[3].sg0.segs; - - if (PREDICT_FALSE (segs > 4)) + + is_b0_from_cpt = oct_is_packet_from_cpt (&d[0].parse.f); + is_b1_from_cpt = oct_is_packet_from_cpt (&d[1].parse.f); + is_b2_from_cpt = oct_is_packet_from_cpt (&d[2].parse.f); + is_b3_from_cpt = oct_is_packet_from_cpt (&d[3].parse.f); + + n_from_cpt = + is_b0_from_cpt + is_b1_from_cpt + is_b2_from_cpt + is_b3_from_cpt; + if (n_from_cpt == 0) { - oct_rx_attach_tail (vm, ctx, b[0], d + 0); - oct_rx_attach_tail (vm, ctx, b[1], d + 1); - oct_rx_attach_tail (vm, ctx, b[2], d + 2); - oct_rx_attach_tail (vm, ctx, b[3], d + 3); + ctx->to_next[0] = vlib_get_buffer_index (vm, b[0]); + ctx->to_next[1] = vlib_get_buffer_index (vm, b[1]); + ctx->to_next[2] = vlib_get_buffer_index (vm, b[2]); + ctx->to_next[3] = vlib_get_buffer_index (vm, b[3]); + + ctx->next[0] = ctx->next_index; + ctx->next[1] = ctx->next_index; + ctx->next[2] = ctx->next_index; + ctx->next[3] = ctx->next_index; + + b[0]->template = bt; + b[1]->template = bt; + b[2]->template = bt; + b[3]->template = bt; + + ctx->n_rx_bytes += b[0]->current_length = d[0].sg0.seg1_size; + ctx->n_rx_bytes += b[1]->current_length = d[1].sg0.seg1_size; + ctx->n_rx_bytes += b[2]->current_length = d[2].sg0.seg1_size; + ctx->n_rx_bytes += b[3]->current_length = d[3].sg0.seg1_size; + + b[0]->flow_id = d[0].parse.w[3] >> 48; + b[1]->flow_id = d[1].parse.w[3] >> 48; + b[2]->flow_id = d[2].parse.w[3] >> 48; + b[3]->flow_id = d[3].parse.w[3] >> 48; + + b0_err_flags = (d[0].parse.w[0] >> 20) & 0xFFF; + b1_err_flags = (d[1].parse.w[0] >> 20) & 0xFFF; + b2_err_flags = (d[2].parse.w[0] >> 20) & 0xFFF; + b3_err_flags = (d[3].parse.w[0] >> 20) & 0xFFF; + + err_flags |= + b0_err_flags | b1_err_flags | b2_err_flags | b3_err_flags; + + ctx->n_segs += 4; + segs = d[0].sg0.segs + d[1].sg0.segs + d[2].sg0.segs + d[3].sg0.segs; + + if (PREDICT_FALSE (segs > 4)) + { + oct_rx_attach_tail (vm, ctx, b[0], d + 0); + oct_rx_attach_tail (vm, ctx, b[1], d + 1); + oct_rx_attach_tail (vm, ctx, b[2], d + 2); + oct_rx_attach_tail (vm, ctx, b[3], d + 3); + } + + buffs[0] = b[0]; + buffs[1] = b[1]; + buffs[2] = b[2]; + buffs[3] = b[3]; + } + else if (n_from_cpt == 4) + { + /* All packets are from cpt */ + cpt_hdr0 = (struct cpt_parse_hdr_s *) *(((u64 *) &d[0]) + 9); + cpt_hdr1 = (struct cpt_parse_hdr_s *) *(((u64 *) &d[1]) + 9); + cpt_hdr2 = (struct cpt_parse_hdr_s *) *(((u64 *) &d[2]) + 9); + cpt_hdr3 = (struct cpt_parse_hdr_s *) *(((u64 *) &d[3]) + 9); + + wqe_ptr0 = (u64 *) clib_net_to_host_u64 (cpt_hdr0->wqe_ptr); + wqe_ptr1 = (u64 *) clib_net_to_host_u64 (cpt_hdr1->wqe_ptr); + wqe_ptr2 = (u64 *) clib_net_to_host_u64 (cpt_hdr2->wqe_ptr); + wqe_ptr3 = (u64 *) clib_net_to_host_u64 (cpt_hdr3->wqe_ptr); + + b[0] = (vlib_buffer_t *) wqe_ptr0; + b[1] = (vlib_buffer_t *) wqe_ptr1; + b[2] = (vlib_buffer_t *) wqe_ptr2; + b[3] = (vlib_buffer_t *) wqe_ptr3; + + orig_rxp0 = (union nix_rx_parse_u *) (wqe_ptr0 + 1); + orig_rxp1 = (union nix_rx_parse_u *) (wqe_ptr1 + 1); + orig_rxp2 = (union nix_rx_parse_u *) (wqe_ptr2 + 1); + orig_rxp3 = (union nix_rx_parse_u *) (wqe_ptr3 + 1); + + l2_ol3_sz0 = orig_rxp0->leptr - orig_rxp0->laptr; + l2_ol3_sz1 = orig_rxp1->leptr - orig_rxp1->laptr; + l2_ol3_sz2 = orig_rxp2->leptr - orig_rxp2->laptr; + l2_ol3_sz3 = orig_rxp3->leptr - orig_rxp3->laptr; + + olen0 = orig_rxp0->pkt_lenm1 + 1; + olen1 = orig_rxp1->pkt_lenm1 + 1; + olen2 = orig_rxp2->pkt_lenm1 + 1; + olen3 = orig_rxp3->pkt_lenm1 + 1; + + esp_sz0 = olen0 - l2_ol3_sz0; + esp_sz1 = olen1 - l2_ol3_sz1; + esp_sz2 = olen2 - l2_ol3_sz2; + esp_sz3 = olen3 - l2_ol3_sz3; + + ctx->to_next[0] = vlib_get_buffer_index (vm, b[0]); + ctx->to_next[1] = vlib_get_buffer_index (vm, b[1]); + ctx->to_next[2] = vlib_get_buffer_index (vm, b[2]); + ctx->to_next[3] = vlib_get_buffer_index (vm, b[3]); + + b[0]->template = bt; + b[1]->template = bt; + b[2]->template = bt; + b[3]->template = bt; + + is_fail0 = !oct_ipsec_is_inl_op_success (cpt_hdr0); + is_fail1 = !oct_ipsec_is_inl_op_success (cpt_hdr1); + is_fail2 = !oct_ipsec_is_inl_op_success (cpt_hdr2); + is_fail3 = !oct_ipsec_is_inl_op_success (cpt_hdr3); + n_cpt_err = is_fail0 + is_fail1 + is_fail2 + is_fail3; + + if (PREDICT_TRUE (!n_cpt_err)) + { + ctx->next[0] = ctx->next_index; + ctx->next[1] = ctx->next_index; + ctx->next[2] = ctx->next_index; + ctx->next[3] = ctx->next_index; + + b[0]->current_length = oct_get_len_from_meta ( + cpt_hdr0, d[0].parse.w[0], d[0].parse.w[4]); + b[1]->current_length = oct_get_len_from_meta ( + cpt_hdr0, d[1].parse.w[0], d[1].parse.w[4]); + b[2]->current_length = oct_get_len_from_meta ( + cpt_hdr0, d[2].parse.w[0], d[2].parse.w[4]); + b[3]->current_length = oct_get_len_from_meta ( + cpt_hdr0, d[3].parse.w[0], d[3].parse.w[4]); + + idx0 = cpt_hdr0->w0.cookie; + idx1 = cpt_hdr1->w0.cookie; + idx2 = cpt_hdr2->w0.cookie; + idx3 = cpt_hdr3->w0.cookie; + + oct_rx_ipsec_update_counters_x4 ( + vm, b[0], esp_sz0, 1, idx0, b[1], esp_sz1, 1, idx1, b[2], + esp_sz2, 1, idx2, b[3], esp_sz3, 1, idx3); + } + else + { + if (is_fail0) + { + b[0]->current_length = olen0; + ctx->next[0] = VNET_DEV_ETH_RX_PORT_NEXT_DROP; + oct_rx_ipsec_set_error (vm, node, b[0], + cpt_hdr0->w3.uc_ccode); + } + else + { + ctx->next[0] = ctx->next_index; + b[0]->current_length = oct_get_len_from_meta ( + cpt_hdr0, d[0].parse.w[0], d[0].parse.w[4]); + idx0 = cpt_hdr0->w0.cookie; + oct_rx_ipsec_update_counters (vm, b[0], esp_sz0, 1, idx0); + } + + if (is_fail1) + { + b[1]->current_length = olen1; + ctx->next[1] = VNET_DEV_ETH_RX_PORT_NEXT_DROP; + oct_rx_ipsec_set_error (vm, node, b[1], + cpt_hdr1->w3.uc_ccode); + } + else + { + ctx->next[1] = ctx->next_index; + b[1]->current_length = oct_get_len_from_meta ( + cpt_hdr1, d[1].parse.w[0], d[1].parse.w[4]); + idx1 = cpt_hdr1->w0.cookie; + oct_rx_ipsec_update_counters (vm, b[1], esp_sz1, 1, idx1); + } + + if (is_fail2) + { + b[2]->current_length = olen2; + ctx->next[2] = VNET_DEV_ETH_RX_PORT_NEXT_DROP; + oct_rx_ipsec_set_error (vm, node, b[2], + cpt_hdr2->w3.uc_ccode); + } + else + { + ctx->next[2] = ctx->next_index; + b[2]->current_length = oct_get_len_from_meta ( + cpt_hdr2, d[2].parse.w[0], d[2].parse.w[4]); + idx2 = cpt_hdr2->w0.cookie; + oct_rx_ipsec_update_counters (vm, b[2], esp_sz2, 1, idx2); + } + + if (is_fail3) + { + b[3]->current_length = olen3; + ctx->next[3] = VNET_DEV_ETH_RX_PORT_NEXT_DROP; + oct_rx_ipsec_set_error (vm, node, b[3], + cpt_hdr3->w3.uc_ccode); + } + else + { + ctx->next[3] = ctx->next_index; + b[3]->current_length = oct_get_len_from_meta ( + cpt_hdr3, d[3].parse.w[0], d[3].parse.w[4]); + idx3 = cpt_hdr3->w0.cookie; + oct_rx_ipsec_update_counters (vm, b[3], esp_sz3, 1, idx3); + } + } + ctx->n_rx_bytes += olen0 + olen1 + olen2 + olen3; + + b[0]->flow_id = d[0].parse.w[3] >> 48; + b[1]->flow_id = d[1].parse.w[3] >> 48; + b[2]->flow_id = d[2].parse.w[3] >> 48; + b[3]->flow_id = d[3].parse.w[3] >> 48; + + b0_err_flags = (d[0].parse.w[0] >> 20) & 0xFFF; + b1_err_flags = (d[1].parse.w[0] >> 20) & 0xFFF; + b2_err_flags = (d[2].parse.w[0] >> 20) & 0xFFF; + b3_err_flags = (d[3].parse.w[0] >> 20) & 0xFFF; + + err_flags |= + b0_err_flags | b1_err_flags | b2_err_flags | b3_err_flags; + + ctx->n_segs += 4; + + OCT_PUSH_META_TO_FREE ((u64) cpt_hdr0, laddr, &loff); + OCT_PUSH_META_TO_FREE ((u64) cpt_hdr1, laddr, &loff); + OCT_PUSH_META_TO_FREE ((u64) cpt_hdr2, laddr, &loff); + OCT_PUSH_META_TO_FREE ((u64) cpt_hdr3, laddr, &loff); + + buffs[0] = b[0]; + buffs[1] = b[1]; + buffs[2] = b[2]; + buffs[3] = b[3]; + } + else + { + /* CQ ring contains mix of packets from wire and CPT */ + if (is_b0_from_cpt) + { + cpt_hdr0 = (struct cpt_parse_hdr_s *) *(((u64 *) &d[0]) + 9); + oct_rx_inl_ipsec_vlib_from_cq (vm, node, &d[0], &b[0], ctx, &bt, + cpt_hdr0, buffs, &err_flags); + OCT_PUSH_META_TO_FREE ((u64) cpt_hdr0, laddr, &loff); + } + else + oct_rx_vlib_from_cq (vm, &d[0], &b[0], ctx, &bt, meta_aura_handle, + buffs, &err_flags); + + if (is_b1_from_cpt) + { + cpt_hdr1 = (struct cpt_parse_hdr_s *) *(((u64 *) &d[1]) + 9); + oct_rx_inl_ipsec_vlib_from_cq (vm, node, &d[1], &b[1], ctx, &bt, + cpt_hdr1, buffs, &err_flags); + OCT_PUSH_META_TO_FREE ((u64) cpt_hdr1, laddr, &loff); + } + else + oct_rx_vlib_from_cq (vm, &d[1], &b[1], ctx, &bt, meta_aura_handle, + buffs, &err_flags); + if (is_b2_from_cpt) + { + cpt_hdr2 = (struct cpt_parse_hdr_s *) *(((u64 *) &d[2]) + 9); + oct_rx_inl_ipsec_vlib_from_cq (vm, node, &d[2], &b[2], ctx, &bt, + cpt_hdr2, buffs, &err_flags); + OCT_PUSH_META_TO_FREE ((u64) cpt_hdr2, laddr, &loff); + } + else + oct_rx_vlib_from_cq (vm, &d[2], &b[2], ctx, &bt, meta_aura_handle, + buffs, &err_flags); + if (is_b3_from_cpt) + { + cpt_hdr3 = (struct cpt_parse_hdr_s *) *(((u64 *) &d[3]) + 9); + oct_rx_inl_ipsec_vlib_from_cq (vm, node, &d[3], &b[3], ctx, &bt, + cpt_hdr3, buffs, &err_flags); + OCT_PUSH_META_TO_FREE ((u64) cpt_hdr3, laddr, &loff); + } + else + oct_rx_vlib_from_cq (vm, &d[3], &b[3], ctx, &bt, meta_aura_handle, + buffs, &err_flags); + } + buffs += 4; + /* Check if lmtline border is crossed and adjust lnum */ + if (loff > 15) + { + /* Update aura handle */ + *(u64 *) (laddr - 8) = + (((u64) (15 & 0x1) << 32) | + roc_npa_aura_handle_to_aura (meta_aura_handle)); + loff = loff - 15; + shft += 3; + + lnum++; + laddr = (uintptr_t) LMT_OFF (lbase, lnum, 8); + /* Pick the pointer from 16th index and put it + * at end of this new line. + */ + *(u64 *) (laddr + (loff << 3) - 8) = *(u64 *) (laddr - 8); } - b0_err_flags = (d[0].parse.w[0] >> 20) & 0xFFF; - b1_err_flags = (d[1].parse.w[0] >> 20) & 0xFFF; - b2_err_flags = (d[2].parse.w[0] >> 20) & 0xFFF; - b3_err_flags = (d[3].parse.w[0] >> 20) & 0xFFF; - - err_flags |= b0_err_flags | b1_err_flags | b2_err_flags | b3_err_flags; + /* Flush it when we are in 16th line and might + * overflow it + */ + if (lnum >= 15 && loff >= 12) + { + /* 16 LMT Line size m1 */ + u64 data = BIT_ULL (48) - 1; + + /* Update aura handle */ + *(u64 *) (laddr - 8) = + (((u64) (loff & 0x1) << 32) | + roc_npa_aura_handle_to_aura (meta_aura_handle)); + + data = (data & ~(0x7UL << shft)) | (((u64) loff >> 1) << shft); + + /* Send up to 16 lmt lines of pointers */ + oct_rx_flush_meta_burst (lmt_id, data, lnum + 1, meta_aura_handle); + plt_wmb (); + lnum = 0; + loff = 0; + shft = 0; + /* First pointer starts at 8B offset */ + laddr = (uintptr_t) LMT_OFF (lbase, lnum, 8); + } + } + if (loff) + { + /* 16 LMT Line size m1 */ + u64 data = BIT_ULL (48) - 1; + + /* Update aura handle */ + *(u64 *) (laddr - 8) = (((u64) (loff & 0x1) << 32) | + roc_npa_aura_handle_to_aura (meta_aura_handle)); + + data = (data & ~(0x7UL << shft)) | (((u64) loff >> 1) << shft); + + /* Send up to 16 lmt lines of pointers */ + oct_rx_flush_meta_burst (lmt_id, data, lnum + 1, meta_aura_handle); + plt_wmb (); + lnum = 0; + loff = 0; + shft = 0; + /* First pointer starts at 8B offset */ + laddr = (uintptr_t) LMT_OFF (lbase, lnum, 8); } - for (; n_left; d += 1, n_left -= 1, ctx->to_next += 1) + for (; n_left; d += 1, n_left -= 1, ctx->to_next += 1, ctx->next += 1) + { + is_b0_from_cpt = oct_is_packet_from_cpt (&d[0].parse.f); + if (is_b0_from_cpt) + { + cpt_hdr0 = (struct cpt_parse_hdr_s *) *(((u64 *) &d[0]) + 9); + oct_rx_inl_ipsec_vlib_from_cq (vm, node, &d[0], &b[0], ctx, &bt, + cpt_hdr0, buffs, &err_flags); + OCT_PUSH_META_TO_FREE ((u64) cpt_hdr0, laddr, &loff); + } + else + + oct_rx_vlib_from_cq (vm, &d[0], &b[0], ctx, &bt, meta_aura_handle, + buffs, &err_flags); + buffs += 1; + } + if (loff) { - b[0] = (vlib_buffer_t *) d->segs0[0] - 1; - ctx->to_next[0] = vlib_get_buffer_index (vm, b[0]); - b[0]->template = bt; - ctx->n_rx_bytes += b[0]->current_length = d[0].sg0.seg1_size; - b[0]->flow_id = d[0].parse.w[3] >> 48; - ctx->n_segs += 1; - if (d[0].sg0.segs > 1) - oct_rx_attach_tail (vm, ctx, b[0], d + 0); - - err_flags |= ((d[0].parse.w[0] >> 20) & 0xFFF); + /* 16 LMT Line size m1 */ + u64 data = BIT_ULL (48) - 1; + + /* Update aura handle */ + *(u64 *) (laddr - 8) = (((u64) (loff & 0x1) << 32) | + roc_npa_aura_handle_to_aura (meta_aura_handle)); + + data = (data & ~(0x7UL << shft)) | (((u64) loff >> 1) << shft); + + /* Send up to 16 lmt lines of pointers */ + oct_rx_flush_meta_burst (lmt_id, data, lnum + 1, meta_aura_handle); + plt_wmb (); } plt_write64 ((crq->cq.wdata | n), crq->cq.door); @@ -189,7 +942,7 @@ oct_rxq_refill (vlib_main_t *vm, vnet_dev_rx_queue_t *rxq, u16 n_refill) u8 bpi = vnet_dev_get_rx_queue_buffer_pool_index (rxq); oct_rxq_t *crq = vnet_dev_get_rx_queue_data (rxq); u64 aura = roc_npa_aura_handle_to_aura (crq->aura_handle); - const uint64_t addr = + const u64 addr = roc_npa_aura_handle_to_base (crq->aura_handle) + NPA_LF_AURA_OP_FREE0; if (n_refill < 256) @@ -310,7 +1063,8 @@ oct_rxq_refill (vlib_main_t *vm, vnet_dev_rx_queue_t *rxq, u16 n_refill) static_always_inline void oct_rx_trace (vlib_main_t *vm, vlib_node_runtime_t *node, - oct_rx_node_ctx_t *ctx, oct_nix_rx_cqe_desc_t *d, u32 n_desc) + oct_rx_node_ctx_t *ctx, oct_nix_rx_cqe_desc_t *d, u32 n_desc, + vlib_buffer_t **buffs) { u32 i = 0; if (PREDICT_TRUE (ctx->trace_count == 0)) @@ -318,7 +1072,7 @@ oct_rx_trace (vlib_main_t *vm, vlib_node_runtime_t *node, while (ctx->n_traced < ctx->trace_count && i < n_desc) { - vlib_buffer_t *b = (vlib_buffer_t *) d[i].segs0[0] - 1; + vlib_buffer_t *b = *buffs; if (PREDICT_TRUE (vlib_trace_buffer (vm, node, ctx->next_index, b, /* follow_chain */ 0))) @@ -330,9 +1084,33 @@ oct_rx_trace (vlib_main_t *vm, vlib_node_runtime_t *node, ctx->n_traced++; } i++; + buffs++; } } +static_always_inline void +oct_rx_enq_to_next (vlib_main_t *vm, vlib_node_runtime_t *node, + oct_rx_node_ctx_t *ctx, u8 *is_single_next) +{ +#ifdef PLATFORM_OCTEON9 + vlib_buffer_enqueue_to_single_next (vm, node, ctx->to_next, ctx->next_index, + ctx->n_rx_pkts); +#else + u32 i; + + for (i = 0; i < ctx->n_rx_pkts; i++) + { + if (ctx->next[i] == VNET_DEV_ETH_RX_PORT_NEXT_DROP) + { + *is_single_next = 0; + break; + } + } + vlib_buffer_enqueue_to_next (vm, node, ctx->to_next, ctx->next, + ctx->n_rx_pkts); +#endif +} + static_always_inline uword oct_rx_node_inline (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame, vnet_dev_port_t *port, @@ -346,10 +1124,17 @@ oct_rx_node_inline (vlib_main_t *vm, vlib_node_runtime_t *node, u32 cq_mask = crq->cq.qmask; oct_nix_rx_cqe_desc_t *descs = crq->cq.desc_base; oct_nix_lf_cq_op_status_t status; + u32 to_next[VLIB_FRAME_SIZE]; + u16 next[VLIB_FRAME_SIZE]; + vlib_buffer_t *buffs[256]; + u8 is_single_next = 1; oct_rx_node_ctx_t _ctx = { .next_index = rxq->next_index, .sw_if_index = port->intf.sw_if_index, .hw_if_index = port->intf.hw_if_index, + .to_next = to_next, + .next = next, + .n_left_to_next = VLIB_FRAME_SIZE, }, *ctx = &_ctx; /* get head and tail from NIX_LF_CQ_OP_STATUS */ @@ -363,17 +1148,14 @@ oct_rx_node_inline (vlib_main_t *vm, vlib_node_runtime_t *node, if (n_desc == 0) goto refill; - vlib_get_new_next_frame (vm, node, ctx->next_index, ctx->to_next, - ctx->n_left_to_next); - ctx->trace_count = vlib_get_trace_count (vm, node); while (1) { ctx->next_desc = descs + head; n = clib_min (cq_size - head, clib_min (n_desc, ctx->n_left_to_next)); - n = oct_rx_batch (vm, ctx, rxq, n); - oct_rx_trace (vm, node, ctx, descs + head, n); + n = oct_rx_batch (vm, node, ctx, rxq, n, buffs); + oct_rx_trace (vm, node, ctx, descs + head, n, buffs); if (ctx->n_left_to_next == 0) break; @@ -387,11 +1169,16 @@ oct_rx_node_inline (vlib_main_t *vm, vlib_node_runtime_t *node, if (n_desc == 0) break; } + ctx->to_next = to_next; + ctx->next = next; + + oct_rx_enq_to_next (vm, node, ctx, &is_single_next); if (ctx->n_traced) vlib_set_trace_count (vm, node, ctx->trace_count - ctx->n_traced); - if (PREDICT_TRUE (ctx->next_index == VNET_DEV_ETH_RX_PORT_NEXT_ETH_INPUT)) + if (PREDICT_TRUE (is_single_next && + ctx->next_index == VNET_DEV_ETH_RX_PORT_NEXT_ETH_INPUT)) { vlib_next_frame_t *nf; vlib_frame_t *f; @@ -411,8 +1198,6 @@ oct_rx_node_inline (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_no_append (f); } - vlib_put_next_frame (vm, node, ctx->next_index, ctx->n_left_to_next); - vlib_increment_combined_counter ( vnm->interface_main.combined_sw_if_counters + VNET_INTERFACE_COUNTER_RX, thr_idx, ctx->hw_if_index, ctx->n_rx_pkts, ctx->n_rx_bytes); From 9d6c5a6a025b2a585b3dd76d0411c8c7a134834f Mon Sep 17 00:00:00 2001 From: Monendra Singh Kushwaha Date: Fri, 29 Nov 2024 12:01:27 +0530 Subject: [PATCH 122/271] octeon: update roc function Type: fix Change-Id: Ia8265607b4ee936be406777e38fdf36f3536f219 Signed-off-by: Monendra Singh Kushwaha Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/140359 (cherry picked from commit 18bd68c673b7bfa8d13ea0cd616b8899b24b71c6) Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/140371 --- src/plugins/dev_octeon/ipsec.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plugins/dev_octeon/ipsec.c b/src/plugins/dev_octeon/ipsec.c index 9de69585e6..5053b7ec27 100644 --- a/src/plugins/dev_octeon/ipsec.c +++ b/src/plugins/dev_octeon/ipsec.c @@ -436,7 +436,7 @@ oct_ipsec_session_destroy (u32 sa_index) sa_dptr = plt_zmalloc (sizeof (struct roc_ot_ipsec_inb_sa), 8); if (sa_dptr != NULL) { - roc_ot_ipsec_inb_sa_init (sa_dptr, true); + roc_ot_ipsec_inb_sa_init (sa_dptr); rv = roc_nix_inl_ctx_write (NULL, sa_dptr, roc_sa, true, sizeof (struct roc_ot_ipsec_inb_sa)); if (rv) From 92fffafd3f53729c6cf61115fa323a6a4c95627c Mon Sep 17 00:00:00 2001 From: Monendra Singh Kushwaha Date: Fri, 29 Nov 2024 15:43:45 +0530 Subject: [PATCH 123/271] build: update roc version Type: feature Change-Id: I5f2d3675ce1420a1c1097904556e96304aaba7ec Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/140386 Tested-by: sa_ip-sw-jenkins Reviewed-by: Monendra Singh Kushwaha --- build/external/packages/octeon-roc.mk | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build/external/packages/octeon-roc.mk b/build/external/packages/octeon-roc.mk index eb902d1534..5029f780c5 100644 --- a/build/external/packages/octeon-roc.mk +++ b/build/external/packages/octeon-roc.mk @@ -2,9 +2,9 @@ # SPDX-License-Identifier: Apache-2.0 # https://spdx.org/licenses/Apache-2.0.html -octeon-roc_version := octeon-roc-SDK12.24.11 +octeon-roc_version := octeon-roc-SDK12.24.12 octeon-roc_tarball := $(octeon-roc_version).tar.gz -octeon-roc_tarball_md5sum := a18c4bd52df9724821b42b391e45a6ea +octeon-roc_tarball_md5sum := 821ce30952222e5ef768f89f70f6d902 octeon-roc_github := https://github.com/MarvellEmbeddedProcessors/marvell-octeon-roc octeon-roc_tarball_strip_dirs := 1 From 8d372e96e033fe0194d86cb2c263013ccd4f06d0 Mon Sep 17 00:00:00 2001 From: Bheemappa Agasimundin Date: Thu, 5 Sep 2024 15:34:55 +0000 Subject: [PATCH 124/271] octeon: add host checksum offload support for sdp This patch adds host checksum offload support in OCTEON plugin for SDP interface. Type: feature JIRA: https://essjira.marvell.com/browse/IPBUSW-53018 Signed-off-by: Bheemappa Agasimundin Change-Id: Ic11ac4c4e1195e0f68f67739a321aa697d6b2c88 Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/135061 Tested-by: sa_ip-sw-jenkins Reviewed-by: Monendra Singh Kushwaha (cherry picked from commit 24caf142201b8386ca99412853c3e265a45cc22c) Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/140411 --- src/plugins/dev_octeon/CMakeLists.txt | 2 + src/plugins/dev_octeon/dpu/dpu.c | 302 ++++++++++++++++++++++++++ src/plugins/dev_octeon/dpu/dpu.h | 52 +++++ 3 files changed, 356 insertions(+) create mode 100644 src/plugins/dev_octeon/dpu/dpu.c create mode 100644 src/plugins/dev_octeon/dpu/dpu.h diff --git a/src/plugins/dev_octeon/CMakeLists.txt b/src/plugins/dev_octeon/CMakeLists.txt index 544901df84..d9cbbf6679 100644 --- a/src/plugins/dev_octeon/CMakeLists.txt +++ b/src/plugins/dev_octeon/CMakeLists.txt @@ -23,6 +23,7 @@ if (NOT OCTEON_ROC_DIR OR NOT OCTEON_ROC_LIB) endif() include_directories (${OCTEON_ROC_DIR}/) +include_directories (${CMAKE_CURRENT_SOURCE_DIR}/dpu/) if (VPP_PLATFORM_NAME STREQUAL "octeon9") add_compile_definitions(PLATFORM_OCTEON9) @@ -42,6 +43,7 @@ add_vpp_plugin(dev_octeon counter.c crypto.c ipsec.c + dpu/dpu.c MULTIARCH_SOURCES rx_node.c diff --git a/src/plugins/dev_octeon/dpu/dpu.c b/src/plugins/dev_octeon/dpu/dpu.c new file mode 100644 index 0000000000..b04ede4037 --- /dev/null +++ b/src/plugins/dev_octeon/dpu/dpu.c @@ -0,0 +1,302 @@ +/* + * Copyright (c) 2024 Marvell. + * SPDX-License-Identifier: Apache-2.0 + * https://spdx.org/licenses/Apache-2.0.html + */ + +/** + * @file + * @brief Host DPU interface. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +always_inline void +h2d_compute_checksum (vlib_main_t *vm, vlib_buffer_t *b) +{ + ethernet_header_t *e; + ip4_header_t *ip; + tcp_header_t *th; + udp_header_t *uh; + + e = vlib_buffer_get_current (b); + if (PREDICT_TRUE (clib_net_to_host_u16 (e->type) == ETHERNET_TYPE_IP4)) + { + ip = (ip4_header_t *) (((u8 *) e) + sizeof (ethernet_header_t)); + if (ip->protocol == IP_PROTOCOL_TCP) + { + th = (tcp_header_t *) (b->data + b->current_data + + sizeof (ethernet_header_t) + + ip4_header_bytes (ip)); + th->checksum = 0; + th->checksum = ip4_tcp_udp_compute_checksum (vm, b, ip); + } + else if (ip->protocol == IP_PROTOCOL_UDP) + { + uh = (udp_header_t *) (b->data + b->current_data + + sizeof (ethernet_header_t) + + ip4_header_bytes (ip)); + uh->checksum = 0; + uh->checksum = ip4_tcp_udp_compute_checksum (vm, b, ip); + } + } +} + +static u8 * +format_h2d_input_trace (u8 *s, va_list *args) +{ + CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *); + CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *); + + s = format (s, "h2d-input:\n"); + return s; +} + +VLIB_NODE_FN (h2d_input_node) +(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame) +{ + u32 n_left, next0, next1, next2, next3; + vlib_buffer_t *bufs[VLIB_FRAME_SIZE]; + u16 nexts[VLIB_FRAME_SIZE], *next; + vlib_buffer_t **b = bufs; + u32 *from; + + from = vlib_frame_vector_args (frame); + n_left = frame->n_vectors; + next = nexts; + vlib_get_buffers (vm, from, bufs, n_left); + + while (n_left >= 8) + { + vlib_buffer_advance (b[0], OCT_H2D_META_SIZE); + vlib_buffer_advance (b[1], OCT_H2D_META_SIZE); + vlib_buffer_advance (b[2], OCT_H2D_META_SIZE); + vlib_buffer_advance (b[3], OCT_H2D_META_SIZE); + + h2d_compute_checksum (vm, b[0]); + h2d_compute_checksum (vm, b[1]); + h2d_compute_checksum (vm, b[2]); + h2d_compute_checksum (vm, b[3]); + + vnet_feature_next (&next0, b[0]); + vnet_feature_next (&next1, b[1]); + vnet_feature_next (&next2, b[2]); + vnet_feature_next (&next3, b[3]); + + next[0] = (u16) next0; + next[1] = (u16) next1; + next[2] = (u16) next2; + next[3] = (u16) next3; + + if (b[0]->flags & VLIB_BUFFER_IS_TRACED) + vlib_add_trace (vm, node, b[0], sizeof (u32)); + if (b[1]->flags & VLIB_BUFFER_IS_TRACED) + vlib_add_trace (vm, node, b[1], sizeof (u32)); + if (b[2]->flags & VLIB_BUFFER_IS_TRACED) + vlib_add_trace (vm, node, b[2], sizeof (u32)); + if (b[3]->flags & VLIB_BUFFER_IS_TRACED) + vlib_add_trace (vm, node, b[3], sizeof (u32)); + + b += 4; + next += 4; + n_left -= 4; + } + + while (n_left) + { + vlib_buffer_advance (b[0], OCT_H2D_META_SIZE); + h2d_compute_checksum (vm, b[0]); + vnet_feature_next (&next0, b[0]); + next[0] = (u16) next0; + + if (b[0]->flags & VLIB_BUFFER_IS_TRACED) + vlib_add_trace (vm, node, b[0], sizeof (u32)); + b += 1; + next += 1; + n_left -= 1; + } + + vlib_buffer_enqueue_to_next (vm, node, from, nexts, frame->n_vectors); + return frame->n_vectors; +} + +VNET_FEATURE_INIT (h2d_input_node, static) = { + .arc_name = "port-rx-eth", + .node_name = "h2d-input", + .runs_before = VNET_FEATURES ("ethernet-input"), +}; + +VLIB_REGISTER_NODE (h2d_input_node) = { + .vector_size = sizeof (u32), + .format_trace = format_h2d_input_trace, + .type = VLIB_NODE_TYPE_INTERNAL, + .n_errors = 0, + .n_next_nodes = 0, + .name = "h2d-input", +}; + +always_inline u8 +d2h_validate_checksum (vlib_main_t *vm, vlib_buffer_t *b) +{ + u8 csum = OCT_D2H_CSUM_VERIFIED; + ethernet_header_t *e; + ip4_header_t *ip; + + e = vlib_buffer_get_current (b); + if (PREDICT_TRUE (clib_net_to_host_u16 (e->type) == ETHERNET_TYPE_IP4)) + { + vlib_buffer_advance (b, sizeof (ethernet_header_t)); + ip = vlib_buffer_get_current (b); + + if (ip->protocol == IP_PROTOCOL_TCP || ip->protocol == IP_PROTOCOL_UDP) + { + ip4_tcp_udp_validate_checksum (vm, b); + if (!(b->flags & VNET_BUFFER_F_L4_CHECKSUM_CORRECT)) + csum = OCT_D2H_CSUM_FAILED; + } + vlib_buffer_advance (b, -sizeof (ethernet_header_t)); + } + return csum; +} + +static u8 * +format_d2h_output_trace (u8 *s, va_list *args) +{ + CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *); + CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *); + + s = format (s, "d2h-output\n"); + return s; +} + +VLIB_NODE_FN (d2h_output_node) +(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame) +{ + oct_d2h_meta_t *hdr0, *hdr1, *hdr2, *hdr3; + u32 n_left, next0, next1, next2, next3; + vlib_buffer_t *bufs[VLIB_FRAME_SIZE]; + u16 nexts[VLIB_FRAME_SIZE], *next; + u8 csum0, csum1, csum2, csum3; + vlib_buffer_t **b = bufs; + u32 *from; + + from = vlib_frame_vector_args (frame); + n_left = frame->n_vectors; + next = nexts; + vlib_get_buffers (vm, from, bufs, n_left); + + while (n_left >= 8) + { + csum0 = d2h_validate_checksum (vm, b[0]); + csum1 = d2h_validate_checksum (vm, b[1]); + csum2 = d2h_validate_checksum (vm, b[2]); + csum3 = d2h_validate_checksum (vm, b[3]); + + vlib_buffer_advance (b[0], -OCT_D2H_META_SIZE); + vlib_buffer_advance (b[1], -OCT_D2H_META_SIZE); + vlib_buffer_advance (b[2], -OCT_D2H_META_SIZE); + vlib_buffer_advance (b[3], -OCT_D2H_META_SIZE); + + clib_prefetch_load ((u8 *) vlib_buffer_get_current (b[4]) - + OCT_D2H_META_SIZE); + clib_prefetch_load ((u8 *) vlib_buffer_get_current (b[5]) - + OCT_D2H_META_SIZE); + clib_prefetch_load ((u8 *) vlib_buffer_get_current (b[6]) - + OCT_D2H_META_SIZE); + clib_prefetch_load ((u8 *) vlib_buffer_get_current (b[7]) - + OCT_D2H_META_SIZE); + + hdr0 = vlib_buffer_get_current (b[0]); + hdr1 = vlib_buffer_get_current (b[1]); + hdr2 = vlib_buffer_get_current (b[2]); + hdr3 = vlib_buffer_get_current (b[3]); + + hdr0->as_u64 = 0; + hdr1->as_u64 = 0; + hdr2->as_u64 = 0; + hdr3->as_u64 = 0; + + hdr0->csum_verified = csum0; + hdr1->csum_verified = csum1; + hdr2->csum_verified = csum2; + hdr3->csum_verified = csum3; + + vnet_feature_next (&next0, b[0]); + vnet_feature_next (&next1, b[1]); + vnet_feature_next (&next2, b[2]); + vnet_feature_next (&next3, b[3]); + + if (b[0]->flags & VLIB_BUFFER_IS_TRACED) + vlib_add_trace (vm, node, b[0], sizeof (u32)); + if (b[1]->flags & VLIB_BUFFER_IS_TRACED) + vlib_add_trace (vm, node, b[1], sizeof (u32)); + if (b[2]->flags & VLIB_BUFFER_IS_TRACED) + vlib_add_trace (vm, node, b[2], sizeof (u32)); + if (b[3]->flags & VLIB_BUFFER_IS_TRACED) + vlib_add_trace (vm, node, b[3], sizeof (u32)); + + next[0] = (u16) next0; + next[1] = (u16) next1; + next[2] = (u16) next2; + next[3] = (u16) next3; + + b += 4; + next += 4; + n_left -= 4; + } + while (n_left) + { + csum0 = d2h_validate_checksum (vm, b[0]); + vlib_buffer_advance (b[0], -OCT_D2H_META_SIZE); + hdr0 = vlib_buffer_get_current (b[0]); + hdr0->as_u64 = 0; + hdr0->csum_verified = csum0; + + if (b[0]->flags & VLIB_BUFFER_IS_TRACED) + vlib_add_trace (vm, node, b[0], sizeof (u32)); + vnet_feature_next (&next0, b[0]); + next[0] = (u16) next0; + + b += 1; + next += 1; + n_left -= 1; + } + + vlib_buffer_enqueue_to_next (vm, node, from, nexts, frame->n_vectors); + return frame->n_vectors; +} + +VNET_FEATURE_INIT (d2h_output_node, static) = { + .arc_name = "interface-output", + .node_name = "d2h-output", + .runs_before = VNET_FEATURES ("interface-output-arc-end"), +}; + +VLIB_REGISTER_NODE (d2h_output_node) = { + .vector_size = sizeof (u32), + .format_trace = format_d2h_output_trace, + .type = VLIB_NODE_TYPE_INTERNAL, + .n_errors = 0, + .n_next_nodes = 0, + .name = "d2h-output", +}; + +/* + * fd.io coding-style-patch-verification: ON + * + * Local Variables: + * eval: (c-set-style "gnu") + * End: + */ diff --git a/src/plugins/dev_octeon/dpu/dpu.h b/src/plugins/dev_octeon/dpu/dpu.h new file mode 100644 index 0000000000..15586615bc --- /dev/null +++ b/src/plugins/dev_octeon/dpu/dpu.h @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2024 Marvell. + * SPDX-License-Identifier: Apache-2.0 + * https://spdx.org/licenses/Apache-2.0.html + */ + +#ifndef _DPU_H_ +#define _DPU_H_ + +/** + * HOST to DPU meta data + */ +typedef struct oct_h2d_meta +{ + u64 as_u64[3]; +} oct_h2d_meta_t; + +#define OCT_H2D_META_SIZE (sizeof (oct_h2d_meta_t)) + +/** + * DPU to HOST meta data + */ +typedef union oct_d2h_meta +{ + u64 as_u64; + struct + { + u64 request_id : 16; + u64 reserved : 2; + u64 csum_verified : 2; + u64 destqport : 22; + u64 sport : 6; + u64 opcode : 16; + }; +} oct_d2h_meta_t; + +#define OCT_D2H_META_SIZE (sizeof (oct_d2h_meta_t)) + +#define OCT_D2H_CSUM_FAILED 0x0 +#define OCT_D2H_L4SUM_VERIFIED 0x1 +#define OCT_D2H_IPSUM_VERIFIED 0x2 +#define OCT_D2H_CSUM_VERIFIED (OCT_D2H_L4SUM_VERIFIED | OCT_D2H_IPSUM_VERIFIED) + +#endif /* _DPU_H_ */ + +/* + * fd.io coding-style-patch-verification: ON + * + * Local Variables: + * eval: (c-set-style "gnu") + * End: + */ From 8f7c2f80d9656d6d46951790b86899a444492b8e Mon Sep 17 00:00:00 2001 From: Mohamed Feroz Date: Tue, 27 Feb 2024 21:19:10 +0530 Subject: [PATCH 125/271] tm: add tm framework for hw traffic management Type: feature 1. What is the patch about? The patch adds changes to VNET library in VPP which adds optional HW traffic management(TM) support to VNET device. When a user configures TM hierarchy with a VNET device, the packets sent on any of the send queues of the net device will flow through the TM elements(scheduler/Shaper/Policer) as per the configuration of TM hierarchy. 2. Need for the TM framework Traffic management on a network device is of utmost importance in optimizing network performance. It is done by controlling the rate(traffic shaping) of data transmission and prioritizing(scheduling) of some flows over the others. However, the shaping and scheduling are the tasks where a computing element keeps checking for availability of tokens or arbitrating over multiple priority flows at higher resolution. This is a less CPU intensive operation done at higher frequency which will eat up the CPU cycles when these operations are done in general purpose CPUs. Hence, it would be wiser to offload such tedious tasks to hardware modules which can implement the same traffic management tasks at higher resolution. This offers various other advantages such as a. Performance and Low Latency Hardware based Traffic managers(TM) handle packet processing tasks more efficiently than software based implementations running on a general purpose CPUs. HW traffic managers are good at maintaining better latency compared to software based solutions. b. Scalability As Network speeds increase and traffic volumes grow, hardware-based traffic managers can scale better than software based implementations. They can be designed with parallel processing capabilities and dedicated resources for numerous miniscule tasks. This allows for better handling of traffic without a compromise in performance/accuracy. c. Reliability Dedicated hardware components in a HW based traffic manager offer higher reliability and fault tolerance compared to software based solutions running on general purpose CPUs. They are less susceptible to fluctuations in performance and accuracy caused by other tasks running in the same system. Some of the HW TMs provide built-in redundancy ensuring continuous operation in case of component failures. d. Power Efficiency Traffic management of high rate traffic will result in higher power consumption on generic CPUs as these are not built specifically for this work. Custom built ASICs/FPGAs which contain special HW modules consume less power for traffic management. Overall power consumption can be very crucial in energy-conscious environments such as data centers. 3. Description of Framework changes The patch adds support to introduce a hook in the vnet device structure. This hook contains the required callbacks to create a TM hierarchy on a HW vnet device. Along with this, additional field in vnet_buffer_opaque2 is introduced to capture the TM node where a packet should be enqueued as per the flow priority of the packet. Change-Id: Iba588d44e56c971fc04272fce08be6d5238c83c3 Signed-off-by: Venkata Ravichandra Mynidi Signed-off-by: Mohamed Feroz Signed-off-by: Harish Malik Signed-off-by: Alok Mishra Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/133079 Tested-by: sa_ip-sw-jenkins Reviewed-by: Nithin Kumar Dabilpuram Reviewed-by: Monendra Singh Kushwaha (cherry picked from commit 3e0d9e257f2ecac4b3c24d395263780ef430d604) Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/140412 --- MAINTAINERS | 4 + src/vnet/CMakeLists.txt | 16 ++ src/vnet/interface.h | 2 + src/vnet/tm/tm.api | 355 +++++++++++++++++++++++++ src/vnet/tm/tm.c | 165 ++++++++++++ src/vnet/tm/tm.h | 299 +++++++++++++++++++++ src/vnet/tm/tm_api.c | 240 +++++++++++++++++ src/vnet/tm/tm_test.c | 567 ++++++++++++++++++++++++++++++++++++++++ 8 files changed, 1648 insertions(+) create mode 100644 src/vnet/tm/tm.api create mode 100644 src/vnet/tm/tm.c create mode 100644 src/vnet/tm/tm.h create mode 100644 src/vnet/tm/tm_api.c create mode 100644 src/vnet/tm/tm_test.c diff --git a/MAINTAINERS b/MAINTAINERS index 9802e3cb50..9cc05f2d9f 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -98,6 +98,10 @@ I: policer M: Neale Ranns F: src/vnet/policer/ +VNET Traffic Management +I: tm +F: src/vnet/tm/ + VNET New Device Drivers Infra I: dev M: Damjan Marion diff --git a/src/vnet/CMakeLists.txt b/src/vnet/CMakeLists.txt index 3225540165..b7963f5c83 100644 --- a/src/vnet/CMakeLists.txt +++ b/src/vnet/CMakeLists.txt @@ -1410,6 +1410,21 @@ list(APPEND VNET_HEADERS list(APPEND VNET_API_FILES teib/teib.api) +############################################################################## +# TM +############################################################################## + +list(APPEND VNET_SOURCES + tm/tm.c + tm/tm_api.c +) + +list(APPEND VNET_HEADERS + tm/tm.h +) + +list(APPEND VNET_API_FILES tm/tm.api) + ############################################################################## # ARP/ND ############################################################################## @@ -1477,6 +1492,7 @@ add_vat_test_library(vnet session/session_test.c l2/l2_test.c ipsec/ipsec_test.c + tm/tm_test.c ) ############################################################################## diff --git a/src/vnet/interface.h b/src/vnet/interface.h index f0cb540f97..bcbae3ba24 100644 --- a/src/vnet/interface.h +++ b/src/vnet/interface.h @@ -44,6 +44,7 @@ #include #include #include +#include #include struct vnet_main_t; @@ -290,6 +291,7 @@ typedef struct _vnet_device_class /* Interface to set rss queues of the interface */ vnet_interface_rss_queues_set_t *set_rss_queues_function; + tm_system_t *tm_sys_impl; } vnet_device_class_t; u32 vnet_register_device_class (vlib_main_t *, vnet_device_class_t *); diff --git a/src/vnet/tm/tm.api b/src/vnet/tm/tm.api new file mode 100644 index 0000000000..948530384c --- /dev/null +++ b/src/vnet/tm/tm.api @@ -0,0 +1,355 @@ +/* + * Copyright (c) 2024 Marvell. + * SPDX-License-Identifier: Apache-2.0 + * https://spdx.org/licenses/Apache-2.0.html + */ + +/** + * @brief Reply for adding a traffic management node. + * + * This structure specifies the parameters returned in response to add a new TM node. + * + * @param context - Sender context, to match reply with request. + * @param retval - Return value indicating success or failure of the operation. + * @param node_id - Identifier of the newly created TM node. + */ +define tm_sys_node_add_reply +{ + u32 context; + i32 retval; + u32 node_id; +}; + +/** + * @brief Add a new traffic management node. + * + * This structure outlines the necessary parameters for adding a new TM node within the VPP system. + * + * @param client_index - Opaque cookie to identify the sender. + * @param context - Sender context, to match reply with request. + * @param sw_if_idx - Software interface index. + * @param parent_node_id - Identifier of the parent node. + * @param node_id - Identifier for the new TM node. + * @param weight - Weight assigned to the new node. + * @param shaper_id - Identifier of the shaper profile to be applied. + * @param lvl - Level of the new node in the hierarchy. + * @param priority - Priority level of the new node. + */ +define tm_sys_node_add +{ + u32 client_index; + u32 context; + u32 sw_if_idx; + u32 parent_node_id; + u32 node_id; + u32 weight; + u32 shaper_id; + u32 lvl; + u32 priority; +}; + +/** + * @brief Reply for Suspending a traffic management node. + * + * This structure specifies the parameters returned in response for suspending a TM node. + * + * @param context - Sender context, to match reply with request. + * @param retval - Return value indicating success or failure of the operation. + * @param node_id - Index of the TM node to be suspended. + */ +define tm_sys_node_suspend_reply +{ + u32 context; + i32 retval; + u32 node_id; +}; + +/** + * @brief Suspend an existing traffic management node. + * + * @param client_index - Opaque cookie to identify the sender. + * @param context - Sender context, to match reply with request. + * @param hw_if_idx - Hardware interface index + * @param node_idx - Index of the TM node to be suspended. + */ +define tm_sys_node_suspend +{ + u32 client_index; + u32 context; + u32 sw_if_idx; + u32 tm_node_id; +}; + +/** + * @brief Reply for resuming a suspended traffic management node. + * + * This structure specifies the parameters returned in response for resuming a TM node. + * + * @param context - Sender context, to match reply with request. + * @param retval - Return value indicating success or failure of the operation. + * @param node_id - Index of the TM node to be resumed. + */ +define tm_sys_node_resume_reply +{ + u32 context; + i32 retval; + u32 node_id; +}; + +/** + * @brief Resume a suspended traffic management node. + * + * @param client_index - Opaque cookie to identify the sender. + * @param context - Sender context, to match reply with request. + * @param hw_if_idx - Hardware interface index + * @param node_idx - Index of the TM node to be resumed. + */ +define tm_sys_node_resume +{ + u32 client_index; + u32 context; + u32 sw_if_idx; + u32 tm_node_id; +}; + +/** + * @brief Reply for deleting a traffic management node. + * + * This structure specifies the parameters returned in response for deleting a TM node. + * + * @param context - Sender context, to match reply with request. + * @param retval - Return value indicating success or failure of the operation. + * @param node_id - Index of the TM node to be deleted. + */ +define tm_sys_node_delete_reply +{ + u32 context; + i32 retval; + u32 node_id; +}; + +/** + * @brief Delete a traffic management node. + * + * @param client_index - Opaque cookie to identify the sender. + * @param context - Sender context, to match reply with request. + * @param hw_if_idx - Hardware interface index + * @param node_idx - Index of the TM node to be deleted. + */ +define tm_sys_node_delete +{ + u32 client_index; + u32 context; + u32 sw_if_idx; + u32 tm_node_id; +}; + +/** + * @brief Reply for creating a shaper profile. + * + * This structure specifies the parameters returned in response to creating a new shaper profile. + * + * @param context - Sender context, to match reply with request. + * @param shaper_id - Identifier of the newly created shaper profile. + * @param retval - Return value indicating success or failure of the operation. + */ +define tm_sys_shaper_profile_create_reply +{ + u32 context; + u32 shaper_id; + i32 retval; +}; + +/** + * @brief Create a new shaper profile. + * + * This structure outlines the necessary parameters to create a new shaper profile. + * + * @param client_index - Opaque cookie to identify the sender. + * @param context - Sender context, to match reply with request. + * @param sw_if_idx - Software interface index. + * @param tm_shaper_id - Identifier for the new shaper profile. + * @param is_pkt_mode - packet_mode or byte_mode + * @param shaper_commit_rate - Committed rate for the shaper. + * @param shaper_commit_burst - Committed burst size for the shaper. + * @param shaper_peak_rate - Peak rate for the shaper. + * @param shaper_peak_burst - Peak burst size for the shaper. + * @param shaper_len_adjust - Length adjustment for the shaper. + */ +define tm_sys_shaper_profile_create +{ + u32 client_index; + u32 context; + u32 sw_if_idx; + u32 tm_shaper_id; + u8 is_pkt_mode; + u64 shaper_commit_rate; + u64 shaper_commit_burst; + u64 shaper_peak_rate; + u64 shaper_peak_burst; + i32 shaper_len_adjust; +}; + +/** + * @brief Reply for updating a node's shaper profile. + * + * This structure specifies the parameters returned in response to updating a shaper profile for a TM node. + * + * @param context - Sender context, to match reply with request. + * @param shaper_id - Identifier of the updated shaper profile. + * @param retval - Return value indicating success or failure of the operation. + */ +define tm_sys_node_shaper_update_reply +{ + u32 context; + u32 shaper_id; + i32 retval; +}; + +/** + * @brief Update a node's shaper profile. + * + * This structure outlines the necessary parameters to update the shaper profile of a TM node. + * + * @param client_index - Opaque cookie to identify the sender. + * @param context - Sender context, to match reply with request. + * @param sw_if_idx - Software interface index. + * @param shaper_id - Identifier of the shaper profile to be applied. + * @param node_id - Identifier of the TM node to be updated. + */ +define tm_sys_node_shaper_update +{ + u32 client_index; + u32 context; + u32 sw_if_idx; + u32 shaper_id; + u32 node_id; +}; + +/** + * @brief Reply for deleting a shaper profile. + * + * This structure specifies the parameters returned in response to deleting a shaper profile. + * + * @param context - Sender context, to match reply with request. + * @param retval - Return value indicating success or failure of the operation. + * @param shaper_id - Identifier of the deleted shaper profile. + */ +define tm_sys_shaper_profile_delete_reply +{ + u32 context; + i32 retval; + u32 shaper_id; +}; + +/** + * @brief Delete a shaper profile. + * + * This structure outlines the necessary parameters to delete a shaper profile. + * + * @param client_index - Opaque cookie to identify the sender. + * @param context - Sender context, to match reply with request. + * @param sw_if_idx - Software interface index. + * @param shaper_id - Identifier of the shaper profile to be deleted. + */ +define tm_sys_shaper_profile_delete +{ + u32 client_index; + u32 context; + u32 sw_if_idx; + u32 shaper_id; +}; + +/** + * @brief Reply for reading the statistics of a TM node. + * + * This structure specifies the parameters returned in response to reading the statistics of a TM node. + * + * @param context - Sender context, to match reply with request. + * @param node_id - Identifier of the TM node whose statistics are being read. + * @param retval - Return value indicating success or failure of the operation. + */ +define tm_sys_node_read_stats_reply +{ + u32 context; + u32 node_id; + i32 retval; +}; + +/** + * @brief Read the statistics of a TM node. + * + * This structure outlines the necessary parameters to read the statistics of a TM node. + * + * @param client_index - Opaque cookie to identify the sender. + * @param context - Sender context, to match reply with request. + * @param sw_if_idx - Software interface index. + * @param node_id - Identifier of the TM node whose statistics are to be read. + */ +define tm_sys_node_read_stats + { + u32 client_index; + u32 context; + u32 sw_if_idx; + u32 node_id; +}; + +/** + * @brief Reply for starting the traffic management (TM) system. + * + * This structure specifies the parameters returned in response to starting the TM system. + * + * @param context - Sender context, to match reply with request. + * @param retval - Return value indicating success or failure of the operation. + */ +define tm_sys_start_tm_reply +{ + u32 context; + i32 retval; +}; + +/** + * @brief Start the traffic management (TM) system. + * + * This structure outlines the necessary parameters to start the TM system. + * + * @param client_index - Opaque cookie to identify the sender. + * @param context - Sender context, to match reply with request. + * @param sw_if_idx - Software interface index. + */ +define tm_sys_start_tm +{ + u32 client_index; + u32 context; + u32 sw_if_idx; +}; + +/** + * @brief Reply for stoping the traffic management (TM) system. + * + * This structure specifies the parameters returned in response to stoping the TM system. + * + * @param context - Sender context, to match reply with request. + * @param retval - Return value indicating success or failure of the operation. + */ +define tm_sys_stop_tm_reply +{ + u32 context; + i32 retval; +}; + +/** + * @brief Stop the traffic management (TM) system. + * + * This structure outlines the necessary parameters to stop the TM system. + * + * @param client_index - Opaque cookie to identify the sender. + * @param context - Sender context, to match reply with request. + * @param sw_if_idx - Software interface index. + */ +define tm_sys_stop_tm +{ + u32 client_index; + u32 context; + u32 sw_if_idx; +}; diff --git a/src/vnet/tm/tm.c b/src/vnet/tm/tm.c new file mode 100644 index 0000000000..f5dfec64d8 --- /dev/null +++ b/src/vnet/tm/tm.c @@ -0,0 +1,165 @@ +/* + * Copyright (c) 2024 Marvell. + * SPDX-License-Identifier: Apache-2.0 + * https://spdx.org/licenses/Apache-2.0.html + */ + +#include + +tm_system_t tm_system_main; + +int +tm_system_register (tm_system_t *tm_sys, u32 hw_if_idx) +{ + vnet_main_t *vnm = vnet_get_main (); + + vnet_hw_interface_t *hi = vnet_get_hw_interface (vnm, hw_if_idx); + + vnet_device_class_t *dev_class = + vnet_get_device_class (vnm, hi->dev_class_index); + + dev_class->tm_sys_impl = tm_sys; + + return 0; +} + +int +tm_sys_node_add (u32 hw_if_idx, u32 node_id, u32 parent_node_id, u32 priority, + u32 weight, u32 lvl, tm_node_params_t *params) +{ + vnet_main_t *vnm = vnet_get_main (); + vnet_hw_interface_t *hi = vnet_get_hw_interface (vnm, hw_if_idx); + vnet_device_class_t *dev_class = + vnet_get_device_class (vnm, hi->dev_class_index); + + dev_class->tm_sys_impl->node_add (hw_if_idx, node_id, parent_node_id, + priority, weight, lvl, params); + + return 0; +} + +int +tm_sys_node_suspend (u32 hw_if_idx, u32 node_id) +{ + vnet_main_t *vnm = vnet_get_main (); + vnet_hw_interface_t *hi = vnet_get_hw_interface (vnm, hw_if_idx); + vnet_device_class_t *dev_class = + vnet_get_device_class (vnm, hi->dev_class_index); + + dev_class->tm_sys_impl->node_suspend (hw_if_idx, node_id); + + return 0; +} + +int +tm_sys_node_resume (u32 hw_if_idx, u32 node_id) +{ + vnet_main_t *vnm = vnet_get_main (); + vnet_hw_interface_t *hi = vnet_get_hw_interface (vnm, hw_if_idx); + vnet_device_class_t *dev_class = + vnet_get_device_class (vnm, hi->dev_class_index); + + dev_class->tm_sys_impl->node_resume (hw_if_idx, node_id); + + return 0; +} + +int +tm_sys_node_delete (u32 hw_if_idx, u32 node_idx) +{ + vnet_main_t *vnm = vnet_get_main (); + + vnet_hw_interface_t *hi = vnet_get_hw_interface (vnm, hw_if_idx); + + vnet_device_class_t *dev_class = + vnet_get_device_class (vnm, hi->dev_class_index); + + dev_class->tm_sys_impl->node_delete (hw_if_idx, node_idx); + + return 0; +} + +int +tm_sys_shaper_profile_create (u32 hw_if_idx, tm_shaper_params_t *param) +{ + vnet_main_t *vnm = vnet_get_main (); + + vnet_hw_interface_t *hi = vnet_get_hw_interface (vnm, hw_if_idx); + vnet_device_class_t *dev_class = + vnet_get_device_class (vnm, hi->dev_class_index); + + dev_class->tm_sys_impl->shaper_profile_create (hw_if_idx, param); + + return 0; +} + +int +tm_sys_node_shaper_update (u32 hw_if_idx, u32 node_id, u32 shaper_id) +{ + vnet_main_t *vnm = vnet_get_main (); + + vnet_hw_interface_t *hi = vnet_get_hw_interface (vnm, hw_if_idx); + vnet_device_class_t *dev_class = + vnet_get_device_class (vnm, hi->dev_class_index); + + dev_class->tm_sys_impl->node_shaper_update (hw_if_idx, node_id, shaper_id); + + return 0; +} + +int +tm_sys_shaper_profile_delete (u32 hw_if_idx, u32 shaper_id) +{ + vnet_main_t *vnm = vnet_get_main (); + + vnet_hw_interface_t *hi = vnet_get_hw_interface (vnm, hw_if_idx); + vnet_device_class_t *dev_class = + vnet_get_device_class (vnm, hi->dev_class_index); + + dev_class->tm_sys_impl->shaper_profile_delete (hw_if_idx, shaper_id); + + return 0; +} + +int +tm_sys_node_read_stats (u32 hw_if_idx, u32 node_idx, tm_stats_params_t *param) +{ + vnet_main_t *vnm = vnet_get_main (); + + vnet_hw_interface_t *hi = vnet_get_hw_interface (vnm, hw_if_idx); + + vnet_device_class_t *dev_class = + vnet_get_device_class (vnm, hi->dev_class_index); + + dev_class->tm_sys_impl->node_read_stats (hw_if_idx, node_idx, param); + + return 0; +} + +int +tm_sys_start_tm (u32 hw_if_idx) +{ + vnet_main_t *vnm = vnet_get_main (); + + vnet_hw_interface_t *hi = vnet_get_hw_interface (vnm, hw_if_idx); + vnet_device_class_t *dev_class = + vnet_get_device_class (vnm, hi->dev_class_index); + + dev_class->tm_sys_impl->start_tm (hw_if_idx); + + return 0; +} + +int +tm_sys_stop_tm (u32 hw_if_idx) +{ + vnet_main_t *vnm = vnet_get_main (); + + vnet_hw_interface_t *hi = vnet_get_hw_interface (vnm, hw_if_idx); + vnet_device_class_t *dev_class = + vnet_get_device_class (vnm, hi->dev_class_index); + + dev_class->tm_sys_impl->stop_tm (hw_if_idx); + + return 0; +} diff --git a/src/vnet/tm/tm.h b/src/vnet/tm/tm.h new file mode 100644 index 0000000000..c29af54632 --- /dev/null +++ b/src/vnet/tm/tm.h @@ -0,0 +1,299 @@ +/* + * Copyright (c) 2024 Marvell. + * SPDX-License-Identifier: Apache-2.0 + * https://spdx.org/licenses/Apache-2.0.html + */ + +#ifndef _TM_H_ +#define _TM_H_ + +#include +#include +#include +#include +#include + +typedef struct tm_node_params_ +{ + /* Shaper profile for the node. */ + u32 shaper_profile_id; + + union + { + struct + { + /* The ingress queue buffer length */ + u32 ingress_q_len; + } leaf; + + struct + { + /** Number of SP priorities. */ + u32 num_sp_priorities; + /* Is scheduling done with pkt mode(1) or byte mode(0). defined per sp + * priority */ + u8 *sched_pkt_mode; + } nonleaf; + }; + + /** Level Identifier of the node in the tm hierarchy */ + u32 level; + + /** Store Node specific data */ + void *data; + + /** TM Node id */ + u32 id; +} tm_node_params_t; + +typedef struct tm_shaper_params_ +{ + struct + { + /** Committed Information Rate. */ + u64 rate; + /** Max burst size for Committed information rate*/ + u64 burst_size; + } commit; + + struct + { + /** Peak Information Rate. */ + u64 rate; + /** Max burst size for Peak information rate. */ + u64 burst_size; + } peak; + + /** Value to be added to the length of each packet for the + * purpose of shaping. */ + i32 pkt_len_adj; + + /** Byte mode of Packet mode */ + u8 pkt_mode; + + /** Shaper profile ID */ + u32 shaper_id; +} tm_shaper_params_t; + +typedef enum +{ + TM_BYTE_BASED_WEIGHTS, + TM_FRAME_BASED_WEIGHTS +} tm_sched_mode_t; + +/** + * TM Color + */ +enum tm_color +{ + TM_COLOR_GREEN = 0, /**< Green */ + TM_COLOR_YELLOW, /**< Yellow */ + TM_COLOR_RED, /**< Red */ + TM_COLORS /**< Number of colors */ +}; + +/** + * The tm_node_stats_type enumeration lists possible packet or octet + * statistics at a tm node. + */ +typedef enum tm_node_stats_type_t +{ + /** Packets dropped by this node after scheduling/shaping at this node */ + TM_NODE_STATS_PKTS_DROPPED, + /** Octets dropped after scheduling/shaping at this node */ + TM_NODE_STATS_OCTETS_DROPPED, + /** Green packets that are sent through this tm node */ + TM_NODE_STATS_GREEN_PKTS, + /** Green octets that are sent through this tm node */ + TM_NODE_STATS_GREEN_OCTETS, + /** Yellow packets that are sent through this tm node */ + TM_NODE_STATS_YELLOW_PKTS, + /** Yellow octets that are sent through this tm node */ + TM_NODE_STATS_YELLOW_OCTETS, + /** Red packets that are sent through this tm node */ + TM_NODE_STATS_RED_PKTS, + /** Red octets that are sent through this tm node */ + TM_NODE_STATS_RED_OCTETS, + /** Node stats max */ + TM_NODE_STATS_MAX, +} tm_node_stats_type_t; + +/** + * Node statistics counters + */ +typedef struct tm_stats_params_ +{ + /** Number of packets scheduled from current node. */ + uint64_t n_pkts; + + /** Number of bytes scheduled from current node. */ + uint64_t n_bytes; + + /** Statistics counters for leaf nodes only. */ + struct + { + /** Number of packets dropped by current leaf node per each + * color. + */ + uint64_t n_pkts_dropped[TM_COLORS]; + + /** Number of bytes dropped by current leaf node per each + * color. + */ + uint64_t n_bytes_dropped[TM_COLORS]; + + /** Number of packets currently waiting in the packet queue of + * current leaf node. + */ + uint64_t n_pkts_queued; + /** Number of bytes currently waiting in the packet queue of + * current leaf node. + */ + uint64_t n_bytes_queued; + } leaf; +} tm_stats_params_t; + +typedef struct tm_system_t_ +{ + u32 hw_if_idx; + int (*node_add) (u32 hw_if_idx, u32 node_id, u32 parent_node_id, + u32 priority, u32 weight, u32 lvl, + tm_node_params_t *params); + + int (*node_suspend) (u32 hw_if_idx, u32 node_idx); + int (*node_resume) (u32 hw_if_idx, u32 node_idx); + int (*node_delete) (u32 hw_if_idx, u32 node_idx); + int (*shaper_profile_create) (u32 hw_if_idx, tm_shaper_params_t *param); + int (*shaper_profile_delete) (u32 hw_if_idx, u32 shaper_id); + int (*node_shaper_update) (u32 hw_if_idx, u32 node_id, + u32 shaper_profile_id); + int (*node_read_stats) (u32 hw_if_idx, u32 node_idx, + tm_stats_params_t *param); + int (*start_tm) (u32 hw_if_idx); + int (*stop_tm) (u32 hw_if_idx); +} tm_system_t; + +/** + * @brief Add a new traffic management node and connect it to an + * existing parent node. + * + * @param hw_if_idx - Hardware interface index. + * @param node_id - Identifier for the new TM node to be created. + * @param parent_node_id - Identifier of the existing parent node. + * @param priority - Priority level of the new node. + * @param weight - Weight assigned to the new node. + * @param lvl - Level of the new node in the hierarchy. + * @param params - Pointer to the structure containing additional parameters + * for the TM node. + * + * @return 0 on success. + */ +int tm_sys_node_add (u32 hw_if_idx, u32 node_id, u32 parent_node_id, + u32 priority, u32 weight, u32 lvl, + tm_node_params_t *params); + +/** + * @brief Suspend an existing traffic management node. + * + * @param hw_if_idx - Hardware interface index + * @param node_idx - Index of the TM node to be suspended. + * + * @return 0 on success. + */ +int tm_sys_node_suspend (u32 hw_if_idx, u32 node_idx); + +/** + * @brief Resume a suspended traffic management node. + * + * @param hw_if_idx - Hardware interface index + * @param node_idx - Index of the TM node to be resumed. + * + * @return 0 on success. + */ +int tm_sys_node_resume (u32 hw_if_idx, u32 node_idx); + +/** + * @brief Delete an existing traffic management node. + * A node can only be deleted if it has no child nodes + * connected to it. + * + * @param hw_if_idx - Hardware interface index + * @param node_idx - Index of the TM node to be deleted. + * + * @return 0 on success. + */ +int tm_sys_node_delete (u32 hw_if_idx, u32 node_idx); + +/** + * @brief Create a new shaper profile for traffic management. + * + * @param hw_if_idx - Hardware interface index. + * @param param - Pointer to the structure containing the shaper parameters. + * + * @return 0 on success. + */ +int tm_sys_shaper_profile_create (u32 hw_if_idx, tm_shaper_params_t *param); + +/** + * @brief Update the shaper profile id of a TM node. + * + * @param hw_if_idx - Hardware interface index. + * @param node_id - Identifier of the TM node to be updated. + * @param shaper_profile_id - Identifier of the new shaper profile to be + * applied. + * + * @return 0 on success. + */ +int tm_sys_node_shaper_update (u32 hw_if_idx, u32 node_id, + u32 shaper_profile_id); + +/** + * @brief Delete an existing shaper profile. + * + * @param hw_if_idx - Hardware interface index. + * @param shaper_id - Identifier of the shaper profile to be deleted. + * + * @return 0 on success. + */ +int tm_sys_shaper_profile_delete (u32 hw_if_idx, u32 shaper_id); + +/** + * @brief Read statistics for a specific traffic management node. + * + * @param hw_if_idx - Hardware interface index. + * @param node_idx - Index of the TM node whose statistics are to be read. + * @param param - Pointer to the structure where the statistics will be stored. + * + * @return 0 on success. + */ +int tm_sys_node_read_stats (u32 hw_if_idx, u32 node_idx, + tm_stats_params_t *param); + +/** + * @brief Start the traffic management system. + * + * @param hw_if_idx - Hardware interface index. + * + * @return 0 on success. + */ +int tm_sys_start_tm (u32 hw_if_idx); + +/** + * @brief Stop the traffic management system. + * + * @param hw_if_idx - Hardware interface index. + * + * @return 0 on success. + */ +int tm_sys_stop_tm (u32 hw_if_idx); + +/** + * @brief Register the traffic management (TM) system. + * + * @param tm_sys - Pointer to the TM system structure to be registered. + * @param hw_if_idx - Hardware interface index. + * + * @return 0 on success. + */ +int tm_system_register (tm_system_t *tm_sys, u32 hw_if_idx); +#endif diff --git a/src/vnet/tm/tm_api.c b/src/vnet/tm/tm_api.c new file mode 100644 index 0000000000..f5f716c838 --- /dev/null +++ b/src/vnet/tm/tm_api.c @@ -0,0 +1,240 @@ +/* + * Copyright (c) 2024 Marvell. + * SPDX-License-Identifier: Apache-2.0 + * https://spdx.org/licenses/Apache-2.0.html + */ + +#include +#include +#include +#include +#include +#include +#include + +/** + * Base message ID fot the plugin + */ +static u32 tm_base_msg_id; +#define REPLY_MSG_ID_BASE tm_base_msg_id + +#include + +void +vl_api_tm_sys_node_add_t_handler (vl_api_tm_sys_node_add_t *mp) +{ + vl_api_tm_sys_node_add_reply_t *rmp; + vnet_main_t *vnm = vnet_get_main (); + tm_node_params_t n_p; + int rv = -1; + u32 node_id = 0; + u32 parent_node_id = 0; + u32 priority = 0; + u32 weight = 0; + u32 lvl = 0; + vnet_sw_interface_t *sw = + vnet_get_sup_sw_interface (vnm, clib_net_to_host_u32 (mp->sw_if_idx)); + + node_id = clib_net_to_host_u32 (mp->node_id); + parent_node_id = clib_net_to_host_u32 (mp->parent_node_id); + n_p.shaper_profile_id = clib_net_to_host_u32 (mp->shaper_id); + weight = clib_net_to_host_u32 (mp->weight); + priority = clib_net_to_host_u32 (mp->priority); + lvl = clib_net_to_host_u32 (mp->lvl); + + rv = tm_sys_node_add (sw->hw_if_index, node_id, parent_node_id, priority, + weight, lvl, &n_p); + + REPLY_MACRO2 (VL_API_TM_SYS_NODE_ADD_REPLY, + ({ rmp->node_id = clib_host_to_net_u32 (node_id); })); +} + +void +vl_api_tm_sys_node_suspend_t_handler (vl_api_tm_sys_node_suspend_t *mp) +{ + vl_api_tm_sys_node_suspend_reply_t *rmp; + vnet_main_t *vnm = vnet_get_main (); + int rv = -1; + u32 node_id = 0; + vnet_sw_interface_t *sw = + vnet_get_sup_sw_interface (vnm, clib_net_to_host_u32 (mp->sw_if_idx)); + + node_id = clib_net_to_host_u32 (mp->tm_node_id); + + rv = tm_sys_node_suspend (sw->hw_if_index, node_id); + + REPLY_MACRO2 (VL_API_TM_SYS_NODE_SUSPEND_REPLY, + ({ rmp->node_id = clib_host_to_net_u32 (node_id); })); +} + +void +vl_api_tm_sys_node_resume_t_handler (vl_api_tm_sys_node_resume_t *mp) +{ + vl_api_tm_sys_node_resume_reply_t *rmp; + vnet_main_t *vnm = vnet_get_main (); + int rv = -1; + u32 node_id = 0; + vnet_sw_interface_t *sw = + vnet_get_sup_sw_interface (vnm, clib_net_to_host_u32 (mp->sw_if_idx)); + + node_id = clib_net_to_host_u32 (mp->tm_node_id); + rv = tm_sys_node_resume (sw->hw_if_index, node_id); + + REPLY_MACRO2 (VL_API_TM_SYS_NODE_RESUME_REPLY, + ({ rmp->node_id = clib_host_to_net_u32 (node_id); })); +} + +void +vl_api_tm_sys_node_delete_t_handler (vl_api_tm_sys_node_delete_t *mp) +{ + vl_api_tm_sys_node_delete_reply_t *rmp; + vnet_main_t *vnm = vnet_get_main (); + u32 node_id = 0; + int rv = -1; + + vnet_sw_interface_t *sw = + vnet_get_sup_sw_interface (vnm, clib_net_to_host_u32 (mp->sw_if_idx)); + + node_id = clib_net_to_host_u32 (mp->tm_node_id); + + rv = tm_sys_node_delete (sw->hw_if_index, node_id); + + REPLY_MACRO2 (VL_API_TM_SYS_NODE_DELETE_REPLY, + ({ rmp->node_id = clib_host_to_net_u32 (node_id); })); +} + +void +vl_api_tm_sys_shaper_profile_create_t_handler ( + vl_api_tm_sys_shaper_profile_create_t *mp) +{ + vl_api_tm_sys_shaper_profile_create_reply_t *rmp; + vnet_main_t *vnm = vnet_get_main (); + tm_shaper_params_t s_p; + int rv = -1; + + vnet_sw_interface_t *sw = + vnet_get_sup_sw_interface (vnm, clib_net_to_host_u32 (mp->sw_if_idx)); + + s_p.shaper_id = clib_net_to_host_u32 (mp->tm_shaper_id); + s_p.commit.rate = clib_net_to_host_u64 (mp->shaper_commit_rate); + s_p.commit.burst_size = clib_net_to_host_u64 (mp->shaper_commit_burst); + s_p.peak.rate = clib_net_to_host_u64 (mp->shaper_peak_rate); + s_p.peak.burst_size = clib_net_to_host_u64 (mp->shaper_peak_burst); + s_p.pkt_len_adj = clib_net_to_host_i64 (mp->shaper_len_adjust); + s_p.pkt_mode = mp->is_pkt_mode; + + rv = tm_sys_shaper_profile_create (sw->hw_if_index, &s_p); + + REPLY_MACRO2 (VL_API_TM_SYS_SHAPER_PROFILE_CREATE_REPLY, ({ + if (!rv) + rmp->shaper_id = clib_host_to_net_u32 (s_p.shaper_id); + })); +} + +void +vl_api_tm_sys_node_shaper_update_t_handler ( + vl_api_tm_sys_node_shaper_update_t *mp) +{ + vl_api_tm_sys_node_shaper_update_reply_t *rmp; + vnet_main_t *vnm = vnet_get_main (); + u32 shaper_profile_id = 0; + u32 node_id = 0; + int rv = -1; + + vnet_sw_interface_t *sw = + vnet_get_sup_sw_interface (vnm, clib_net_to_host_u32 (mp->sw_if_idx)); + + node_id = clib_net_to_host_u32 (mp->node_id); + shaper_profile_id = clib_net_to_host_u32 (mp->shaper_id); + + rv = tm_sys_node_shaper_update (sw->hw_if_index, node_id, shaper_profile_id); + + REPLY_MACRO2 (VL_API_TM_SYS_NODE_SHAPER_UPDATE_REPLY, ({ + if (!rv) + rmp->shaper_id = clib_host_to_net_u32 (shaper_profile_id); + })); +} + +void +vl_api_tm_sys_shaper_profile_delete_t_handler ( + vl_api_tm_sys_shaper_profile_delete_t *mp) +{ + vl_api_tm_sys_shaper_profile_delete_reply_t *rmp; + vnet_main_t *vnm = vnet_get_main (); + u32 shaper_id = 0; + int rv = -1; + + vnet_sw_interface_t *sw = + vnet_get_sup_sw_interface (vnm, clib_net_to_host_u32 (mp->sw_if_idx)); + shaper_id = clib_net_to_host_u32 (mp->shaper_id); + + rv = tm_sys_shaper_profile_delete (sw->hw_if_index, shaper_id); + + REPLY_MACRO2 (VL_API_TM_SYS_SHAPER_PROFILE_DELETE_REPLY, ({ + if (!rv) + rmp->shaper_id = clib_host_to_net_u32 (shaper_id); + })); +} + +void +vl_api_tm_sys_node_read_stats_t_handler (vl_api_tm_sys_node_read_stats_t *mp) +{ + vl_api_tm_sys_node_read_stats_reply_t *rmp; + vnet_main_t *vnm = vnet_get_main (); + tm_stats_params_t s_p = { 0 }; + u32 node_id = 0; + int rv = -1; + + vnet_sw_interface_t *sw = + vnet_get_sup_sw_interface (vnm, clib_net_to_host_u32 (mp->sw_if_idx)); + + node_id = clib_net_to_host_u32 (mp->node_id); + + rv = tm_sys_node_read_stats (sw->hw_if_index, node_id, &s_p); + + REPLY_MACRO2 (VL_API_TM_SYS_NODE_READ_STATS_REPLY, + ({ rmp->node_id = clib_host_to_net_u32 (node_id); })); +} + +void +vl_api_tm_sys_start_tm_t_handler (vl_api_tm_sys_start_tm_t *mp) +{ + vl_api_tm_sys_start_tm_reply_t *rmp; + vnet_main_t *vnm = vnet_get_main (); + int rv = -1; + + vnet_sw_interface_t *sw = + vnet_get_sup_sw_interface (vnm, clib_net_to_host_u32 (mp->sw_if_idx)); + + rv = tm_sys_start_tm (sw->hw_if_index); + + REPLY_MACRO (VL_API_TM_SYS_START_TM_REPLY); +} + +void +vl_api_tm_sys_stop_tm_t_handler (vl_api_tm_sys_stop_tm_t *mp) +{ + vl_api_tm_sys_stop_tm_reply_t *rmp; + vnet_main_t *vnm = vnet_get_main (); + int rv = -1; + + vnet_sw_interface_t *sw = + vnet_get_sup_sw_interface (vnm, clib_net_to_host_u32 (mp->sw_if_idx)); + + rv = tm_sys_stop_tm (sw->hw_if_index); + + REPLY_MACRO (VL_API_TM_SYS_STOP_TM_REPLY); +} + +#include + +static clib_error_t * +tm_api_init (vlib_main_t *vm) +{ + /* Ask for a correctly-sized block of API message decode slots */ + tm_base_msg_id = setup_message_id_table (); + + return 0; +} + +VLIB_INIT_FUNCTION (tm_api_init); diff --git a/src/vnet/tm/tm_test.c b/src/vnet/tm/tm_test.c new file mode 100644 index 0000000000..49b733e533 --- /dev/null +++ b/src/vnet/tm/tm_test.c @@ -0,0 +1,567 @@ +/* + * Copyright (c) 2024 Marvell. + * SPDX-License-Identifier: Apache-2.0 + * https://spdx.org/licenses/Apache-2.0.html + */ + +#include +#include +#include +#include + +#include + +#include + +typedef struct +{ + /* API message ID base */ + u16 msg_id_base; + u32 ping_id; + vat_main_t *vat_main; +} tm_test_main_t; + +tm_test_main_t tm_test_main; + +#define __plugin_msg_base tm_test_main.msg_id_base +#include +uword unformat_sw_if_index (unformat_input_t *input, va_list *args); + +/* Declare message IDs */ +#include +#include +#include + +static int +api_tm_sys_node_add (vat_main_t *vam) +{ + u32 level, priority, node_id, weight, parent_node_id; + u8 priority_set = 0, level_set = 0; + unformat_input_t *i = vam->input; + vl_api_tm_sys_node_add_t *mp; + u32 msg_size = sizeof (*mp); + u8 sw_if_idx_set = 0; + u32 sw_if_idx = 0; + u32 shaper_id = 0; + int ret; + + vam->result_ready = 0; + mp = vl_msg_api_alloc_as_if_client (msg_size); + + while (unformat_check_input (i) != UNFORMAT_END_OF_INPUT) + { + if (unformat (i, "sw_if_idx %u", &sw_if_idx)) + sw_if_idx_set = 1; + else if (unformat (i, "node_id %u", &node_id)) + ; + else if (unformat (i, "parent_node_id %d", &parent_node_id)) + ; + else if (unformat (i, "shaper_prof %d", &shaper_id)) + ; + else if (unformat (i, "weight %u", &weight)) + ; + else if (unformat (i, "priority %u", &priority)) + priority_set = 1; + else if (unformat (i, "level %u", &level)) + level_set = 1; + else + { + clib_warning ("Invalid input, unknown parameter"); + return -EINVAL; + } + } + + if (!sw_if_idx_set || !priority_set || !level_set) + return -EINVAL; + + M (TM_SYS_NODE_ADD, mp); + + mp->sw_if_idx = clib_host_to_net_u32 (sw_if_idx); + mp->node_id = clib_host_to_net_u32 (node_id); + mp->parent_node_id = clib_host_to_net_u32 (parent_node_id); + mp->shaper_id = clib_host_to_net_u32 (shaper_id); + mp->weight = clib_host_to_net_u32 (weight); + mp->priority = clib_host_to_net_u32 (priority); + mp->lvl = clib_host_to_net_u32 (level); + + S (mp); + W (ret); + return ret; +} + +static int +api_tm_sys_node_suspend (vat_main_t *vam) +{ + u8 sw_if_idx_set = 0, tm_node_idx_set = 0; + unformat_input_t *i = vam->input; + vl_api_tm_sys_node_suspend_t *mp; + u32 msg_size = sizeof (*mp); + u32 tm_node_id = 0; + u32 sw_if_idx = 0; + int ret; + + vam->result_ready = 0; + mp = vl_msg_api_alloc_as_if_client (msg_size); + + while (unformat_check_input (i) != UNFORMAT_END_OF_INPUT) + { + if (unformat (i, "sw_if_idx %u", &sw_if_idx)) + sw_if_idx_set = 1; + else if (unformat (i, "tm_node_id %u", &tm_node_id)) + tm_node_idx_set = 1; + else + { + clib_warning ("Invalid input, unknown parameter"); + return -EINVAL; + } + } + + if (!sw_if_idx_set || !tm_node_idx_set) + return -EINVAL; + + M (TM_SYS_NODE_SUSPEND, mp); + + mp->sw_if_idx = clib_host_to_net_u32 (sw_if_idx); + mp->tm_node_id = clib_host_to_net_u32 (tm_node_id); + + S (mp); + W (ret); + return ret; +} + +static int +api_tm_sys_node_resume (vat_main_t *vam) +{ + u8 sw_if_idx_set = 0, tm_node_idx_set = 0; + unformat_input_t *i = vam->input; + vl_api_tm_sys_node_resume_t *mp; + u32 msg_size = sizeof (*mp); + u32 tm_node_id = 0; + u32 sw_if_idx = 0; + int ret; + + vam->result_ready = 0; + mp = vl_msg_api_alloc_as_if_client (msg_size); + + while (unformat_check_input (i) != UNFORMAT_END_OF_INPUT) + { + if (unformat (i, "sw_if_idx %u", &sw_if_idx)) + sw_if_idx_set = 1; + else if (unformat (i, "tm_node_id %u", &tm_node_id)) + tm_node_idx_set = 1; + else + { + clib_warning ("Invalid input, unknown parameter"); + return -EINVAL; + } + } + + if (!sw_if_idx_set || !tm_node_idx_set) + return -EINVAL; + + M (TM_SYS_NODE_RESUME, mp); + + mp->sw_if_idx = clib_host_to_net_u32 (sw_if_idx); + mp->tm_node_id = clib_host_to_net_u32 (tm_node_id); + + S (mp); + W (ret); + return ret; +} + +static int +api_tm_sys_node_delete (vat_main_t *vam) +{ + u8 sw_if_idx_set = 0, tm_node_idx_set = 0; + unformat_input_t *i = vam->input; + vl_api_tm_sys_node_delete_t *mp; + u32 msg_size = sizeof (*mp); + u32 tm_node_id = 0; + u32 sw_if_idx = 0; + int ret; + + vam->result_ready = 0; + mp = vl_msg_api_alloc_as_if_client (msg_size); + + while (unformat_check_input (i) != UNFORMAT_END_OF_INPUT) + { + if (unformat (i, "sw_if_idx %u", &sw_if_idx)) + sw_if_idx_set = 1; + else if (unformat (i, "tm_node_id %u", &tm_node_id)) + tm_node_idx_set = 1; + else + { + clib_warning ("Invalid input, unknown parameter"); + return -EINVAL; + } + } + + if (!sw_if_idx_set || !tm_node_idx_set) + return -EINVAL; + + M (TM_SYS_NODE_DELETE, mp); + + mp->sw_if_idx = clib_host_to_net_u32 (sw_if_idx); + mp->tm_node_id = clib_host_to_net_u32 (tm_node_id); + + S (mp); + W (ret); + return ret; +} + +static int +api_tm_sys_shaper_profile_create (vat_main_t *vam) +{ + vl_api_tm_sys_shaper_profile_create_t *mp; + unformat_input_t *i = vam->input; + u32 msg_size = sizeof (*mp); + i32 shaper_len_adjust = 0; + u64 shaper_commit_rate = 0; + u64 shaper_commit_burst = 0; + u64 shaper_peak_rate = 0; + u64 shaper_peak_burst = 0; + u32 tm_shaper_id = 0; + u8 sw_if_idx_set = 0; + u32 is_packet_mode = 0; + u32 sw_if_idx = 0; + int ret; + + vam->result_ready = 0; + + mp = vl_msg_api_alloc_as_if_client (msg_size); + + while (unformat_check_input (i) != UNFORMAT_END_OF_INPUT) + { + if (unformat (i, "sw_if_idx %u", &sw_if_idx)) + sw_if_idx_set = 1; + else if (unformat (i, "shaper_id %u", &tm_shaper_id)) + ; + else if (unformat (i, "packet_mode %u", &is_packet_mode)) + ; + else if (unformat (i, "shaper_peak_burst %llu", &shaper_peak_burst)) + ; + else if (unformat (i, "shaper_commit_rate %llu", &shaper_commit_rate)) + ; + else if (unformat (i, "shaper_commit_burst %llu", &shaper_commit_burst)) + ; + else if (unformat (i, "shaper_peak_rate %llu", &shaper_peak_rate)) + ; + else if (unformat (i, "shaper_len_adjust %d", &shaper_len_adjust)) + ; + else + { + clib_warning ("Invalid input, unknown parameter"); + return -EINVAL; + } + } + + if (!sw_if_idx_set) + return -EINVAL; + + M (TM_SYS_SHAPER_PROFILE_CREATE, mp); + + mp->sw_if_idx = clib_host_to_net_u32 (sw_if_idx); + mp->tm_shaper_id = clib_host_to_net_u32 (tm_shaper_id); + mp->is_pkt_mode = (u8) is_packet_mode; + mp->shaper_commit_rate = clib_host_to_net_u64 (shaper_commit_rate); + mp->shaper_commit_burst = clib_host_to_net_u64 (shaper_commit_burst); + mp->shaper_peak_rate = clib_host_to_net_u64 (shaper_peak_rate); + mp->shaper_peak_burst = clib_host_to_net_u64 (shaper_peak_burst); + mp->shaper_len_adjust = clib_host_to_net_i64 (shaper_len_adjust); + + S (mp); + W (ret); + return ret; +} + +static int +api_tm_sys_node_shaper_update (vat_main_t *vam) +{ + u8 sw_if_idx_set = 0; + unformat_input_t *i = vam->input; + vl_api_tm_sys_node_shaper_update_t *mp; + u32 msg_size = sizeof (*mp); + u32 shaper_profile = 0, node_id = 0; + u32 sw_if_idx = 0; + int ret; + + vam->result_ready = 0; + mp = vl_msg_api_alloc_as_if_client (msg_size); + + while (unformat_check_input (i) != UNFORMAT_END_OF_INPUT) + { + if (unformat (i, "sw_if_idx %u", &sw_if_idx)) + sw_if_idx_set = 1; + else if (unformat (i, "shaper_profile %u", &shaper_profile)) + shaper_profile = 1; + else if (unformat (i, "node_id %u", &node_id)) + ; + else + { + clib_warning ("Invalid input, unknown parameter"); + return -EINVAL; + } + } + + if (!sw_if_idx_set || !shaper_profile) + return -EINVAL; + + M (TM_SYS_NODE_SHAPER_UPDATE, mp); + + mp->sw_if_idx = clib_host_to_net_u32 (sw_if_idx); + mp->shaper_id = clib_host_to_net_u32 (shaper_profile); + mp->node_id = clib_host_to_net_u32 (node_id); + + S (mp); + W (ret); + return ret; +} + +static int +api_tm_sys_shaper_profile_delete (vat_main_t *vam) +{ + vl_api_tm_sys_shaper_profile_delete_t *mp; + unformat_input_t *i = vam->input; + u32 msg_size = sizeof (*mp); + u8 sw_if_idx_set = 0; + u32 sw_if_idx = 0; + u32 shaper_id = 0; + int ret; + + vam->result_ready = 0; + mp = vl_msg_api_alloc_as_if_client (msg_size); + + while (unformat_check_input (i) != UNFORMAT_END_OF_INPUT) + { + if (unformat (i, "sw_if_idx %u", &sw_if_idx)) + sw_if_idx_set = 1; + + if (unformat (i, "shaper_id %u", &shaper_id)) + ; + else + { + clib_warning ("Invalid input, unknown parameter"); + return -EINVAL; + } + } + + if (!shaper_id && !sw_if_idx_set) + return -EINVAL; + + M (TM_SYS_SHAPER_PROFILE_DELETE, mp); + + mp->sw_if_idx = clib_host_to_net_u32 (sw_if_idx); + mp->shaper_id = clib_host_to_net_u32 (shaper_id); + + S (mp); + W (ret); + return ret; +} + +static int +api_tm_sys_node_read_stats (vat_main_t *vam) +{ + u8 sw_if_idx_set = 0, tm_node_idx_set = 0; + unformat_input_t *i = vam->input; + vl_api_tm_sys_node_read_stats_t *mp; + u32 msg_size = sizeof (*mp); + u32 tm_node_id = 0; + u32 sw_if_idx = 0; + int ret; + + vam->result_ready = 0; + mp = vl_msg_api_alloc_as_if_client (msg_size); + + while (unformat_check_input (i) != UNFORMAT_END_OF_INPUT) + { + if (unformat (i, "sw_if_idx %u", &sw_if_idx)) + sw_if_idx_set = 1; + else if (unformat (i, "tm_node_id %u", &tm_node_id)) + tm_node_idx_set = 1; + else + { + clib_warning ("Invalid input, unknown parameter"); + return -EINVAL; + } + } + + if (!sw_if_idx_set || !tm_node_idx_set) + return -EINVAL; + + M (TM_SYS_NODE_READ_STATS, mp); + + mp->sw_if_idx = clib_host_to_net_u32 (sw_if_idx); + mp->node_id = clib_host_to_net_u32 (tm_node_id); + + S (mp); + W (ret); + return ret; +} + +static int +api_tm_sys_start_tm (vat_main_t *vam) +{ + u8 sw_if_idx_set = 0; + unformat_input_t *i = vam->input; + vl_api_tm_sys_start_tm_t *mp; + u32 msg_size = sizeof (*mp); + u32 sw_if_idx = 0; + int ret; + + vam->result_ready = 0; + mp = vl_msg_api_alloc_as_if_client (msg_size); + + while (unformat_check_input (i) != UNFORMAT_END_OF_INPUT) + { + if (unformat (i, "sw_if_idx %u", &sw_if_idx)) + sw_if_idx_set = 1; + else + { + clib_warning ("Invalid input, unknown parameter"); + return -EINVAL; + } + } + + if (!sw_if_idx_set) + return -EINVAL; + + M (TM_SYS_START_TM, mp); + + mp->sw_if_idx = clib_host_to_net_u32 (sw_if_idx); + + S (mp); + W (ret); + return ret; +} + +static int +api_tm_sys_stop_tm (vat_main_t *vam) +{ + u8 sw_if_idx_set = 0; + unformat_input_t *i = vam->input; + vl_api_tm_sys_stop_tm_t *mp; + u32 msg_size = sizeof (*mp); + u32 sw_if_idx = 0; + int ret; + + vam->result_ready = 0; + mp = vl_msg_api_alloc_as_if_client (msg_size); + + while (unformat_check_input (i) != UNFORMAT_END_OF_INPUT) + { + if (unformat (i, "sw_if_idx %u", &sw_if_idx)) + sw_if_idx_set = 1; + else + { + clib_warning ("Invalid input, unknown parameter"); + return -EINVAL; + } + } + + if (!sw_if_idx_set) + return -EINVAL; + + M (TM_SYS_STOP_TM, mp); + + mp->sw_if_idx = clib_host_to_net_u32 (sw_if_idx); + + S (mp); + W (ret); + return ret; +} + +static void +vl_api_tm_sys_node_add_reply_t_handler (vl_api_tm_sys_node_add_reply_t *mp) +{ + vat_main_t *vam = tm_test_main.vat_main; + clib_warning ("TM node_add_id : %u\n", clib_net_to_host_u32 (mp->node_id)); + vam->result_ready = 1; +} + +static void +vl_api_tm_sys_node_suspend_reply_t_handler ( + vl_api_tm_sys_node_suspend_reply_t *mp) +{ + vat_main_t *vam = tm_test_main.vat_main; + clib_warning ("TM node_suspend_id : %u\n", + clib_net_to_host_u32 (mp->node_id)); + vam->result_ready = 1; +} + +static void +vl_api_tm_sys_node_resume_reply_t_handler ( + vl_api_tm_sys_node_resume_reply_t *mp) +{ + vat_main_t *vam = tm_test_main.vat_main; + clib_warning ("TM node_resume_ id : %u\n", + clib_net_to_host_u32 (mp->node_id)); + vam->result_ready = 1; +} + +static void +vl_api_tm_sys_node_delete_reply_t_handler ( + vl_api_tm_sys_node_delete_reply_t *mp) +{ + vat_main_t *vam = tm_test_main.vat_main; + clib_warning ("TM node_delete_id : %u\n", + clib_net_to_host_u32 (mp->node_id)); + vam->result_ready = 1; +} + +static void +vl_api_tm_sys_shaper_profile_create_reply_t_handler ( + vl_api_tm_sys_shaper_profile_create_reply_t *mp) +{ + vat_main_t *vam = tm_test_main.vat_main; + clib_warning ("Shaper profile id : %u\n", + clib_net_to_host_u32 (mp->shaper_id)); + vam->result_ready = 1; +} + +static void +vl_api_tm_sys_node_shaper_update_reply_t_handler ( + vl_api_tm_sys_node_shaper_update_reply_t *mp) +{ + vat_main_t *vam = tm_test_main.vat_main; + clib_warning ("TM node updated shaper id : %u\n", + clib_net_to_host_u32 (mp->shaper_id)); + + vam->result_ready = 1; +} + +static void +vl_api_tm_sys_shaper_profile_delete_reply_t_handler ( + vl_api_tm_sys_shaper_profile_delete_reply_t *mp) +{ + vat_main_t *vam = tm_test_main.vat_main; + clib_warning ("TM shaper profile delete id : %u\n", + clib_net_to_host_u32 (mp->shaper_id)); + vam->result_ready = 1; +} + +static void +vl_api_tm_sys_node_read_stats_reply_t_handler ( + vl_api_tm_sys_node_read_stats_reply_t *mp) +{ + vat_main_t *vam = tm_test_main.vat_main; + clib_warning ("TM stats for node id : %u\n", + clib_net_to_host_u32 (mp->node_id)); + + vam->result_ready = 1; +} + +static void +vl_api_tm_sys_start_tm_reply_t_handler (vl_api_tm_sys_start_tm_reply_t *mp) +{ + vat_main_t *vam = tm_test_main.vat_main; + vam->result_ready = 1; +} + +static void +vl_api_tm_sys_stop_tm_reply_t_handler (vl_api_tm_sys_stop_tm_reply_t *mp) +{ + vat_main_t *vam = tm_test_main.vat_main; + vam->result_ready = 1; +} + +#include From c3f2ef5fa7bcda35869af08e62355844ceb27033 Mon Sep 17 00:00:00 2001 From: Bheemappa Agasimundin Date: Tue, 5 Nov 2024 13:55:51 +0530 Subject: [PATCH 126/271] octep-cp: modify checksum offload support for octeon plugin This patch modifies input node name from device-input to port-rx-eth. Type: feature JIRA: https://essjira.marvell.com/browse/IPBUSW-53018 Signed-off-by: Bheemappa Agasimundin Change-Id: Ia8dae78652505c6170ef693992384af5d53cd953 Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/138550 Tested-by: sa_ip-sw-jenkins Reviewed-by: Monendra Singh Kushwaha (cherry picked from commit dbf7d8f0d66a063bbd65599e04eef53fef6c3bc7) Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/140413 --- src/plugins/octep_cp/octep_action.c | 2 -- src/plugins/octep_cp/octep_ctrl.c | 5 +---- src/plugins/octep_cp/octep_input.h | 2 +- 3 files changed, 2 insertions(+), 7 deletions(-) diff --git a/src/plugins/octep_cp/octep_action.c b/src/plugins/octep_cp/octep_action.c index f00ab17e70..1c6a04a358 100644 --- a/src/plugins/octep_cp/octep_action.c +++ b/src/plugins/octep_cp/octep_action.c @@ -3,8 +3,6 @@ * SPDX-License-Identifier: Apache-2.0 * https://spdx.org/licenses/Apache-2.0.html */ - -#include #include #include "octep_action.h" #include "octep_ctrl_net.h" diff --git a/src/plugins/octep_cp/octep_ctrl.c b/src/plugins/octep_cp/octep_ctrl.c index 68a71b37db..ea5f1f75e8 100644 --- a/src/plugins/octep_cp/octep_ctrl.c +++ b/src/plugins/octep_cp/octep_ctrl.c @@ -15,7 +15,6 @@ #include "octep_cp_lib.h" #include "octep_input.h" #include "octep_config.h" -#include #include #include #include @@ -221,9 +220,7 @@ octep_cp_init (vlib_main_t *vm) return NULL; } -VLIB_INIT_FUNCTION (octep_cp_init) = { - .runs_after = VLIB_INITS ("onp_plugin_init"), -}; +VLIB_INIT_FUNCTION (octep_cp_init); VLIB_PLUGIN_REGISTER () = { .version = VPP_BUILD_VER, diff --git a/src/plugins/octep_cp/octep_input.h b/src/plugins/octep_cp/octep_input.h index 810316a3cd..3f173d024f 100644 --- a/src/plugins/octep_cp/octep_input.h +++ b/src/plugins/octep_cp/octep_input.h @@ -7,7 +7,7 @@ #ifndef __OCTEP_INPUT_H__ #define __OCTEP_INPUT_H__ -#define DEVICE_INPUT "device-input" +#define DEVICE_INPUT "port-rx-eth" #define DPU_INPUT_NODE "h2d-input" #define DEVICE_OUTPUT "interface-output" #define DPU_OUTPUT_NODE "d2h-output" From f776c5137503cedcd34117c63e2e49ad8513724b Mon Sep 17 00:00:00 2001 From: Bheemappa Agasimundin Date: Wed, 6 Nov 2024 16:23:57 +0530 Subject: [PATCH 127/271] octeon: add host checksum offload support This patch adds host checksum offload support for OCTEON virtio interface. Type: feature JIRA: https://essjira.marvell.com/browse/IPBUSW-50903 Signed-off-by: Bheemappa Agasimundin Change-Id: I128040d6a77e2a85f58583ef167f25813ad4315a Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/138675 Tested-by: sa_ip-sw-jenkins Reviewed-by: Monendra Singh Kushwaha (cherry picked from commit b72880823a8f5ff9adce57bc0c96cc4d8f7f6788) Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/140414 --- src/plugins/dev_octeon/oct_virtio.h | 12 +++++++ src/plugins/dev_octeon/virtio.c | 19 ++++++++--- src/plugins/dev_octeon/virtio_ctrl.c | 44 ++++++++++++++++++++++++- src/plugins/dev_octeon/virtio_port.c | 2 ++ src/plugins/dev_octeon/virtio_rx_node.c | 7 +++- src/plugins/dev_octeon/virtio_tx_node.c | 6 ++++ 6 files changed, 84 insertions(+), 6 deletions(-) diff --git a/src/plugins/dev_octeon/oct_virtio.h b/src/plugins/dev_octeon/oct_virtio.h index b8f852b179..91e03df9ee 100644 --- a/src/plugins/dev_octeon/oct_virtio.h +++ b/src/plugins/dev_octeon/oct_virtio.h @@ -26,6 +26,10 @@ #define OCT_VIRTIO_DEVICE_ID 0xa70d #define MAX_JUMBO_PKT_LEN 9600 +#define OCT_ETH_TX_OFFLOAD_IPV4_CKSUM (1 << 0) +#define OCT_ETH_RX_OFFLOAD_CHECKSUM (1 << 1) +#define OCT_ETH_TX_OFFLOAD_TCP_TSO (1 << 2) + #define foreach_oct_virt_tx_node_counter \ _ (ENQUE_FAIL, enque_fail, ERROR, "Virtio enqueue failed") @@ -90,6 +94,7 @@ typedef struct u64 netdev_map; u16 netdev_qp_count[DAO_VIRTIO_DEV_MAX]; u8 dao_lib_initialized; + u8 ip4_csum_offload_enable; } oct_virtio_main_t; typedef struct @@ -105,11 +110,18 @@ typedef struct u16 virtio_hdr_sz; } oct_virtio_q_info_t; +typedef struct +{ + u64 rx_offloads; + u64 tx_offloads; +} oct_intf_offload_t; + typedef struct { u8 initialized; u16 service_core; u64 netdev_map; + oct_intf_offload_t intf[DAO_VIRTIO_DEV_MAX]; oct_virtio_q_info_t q_map[DAO_VIRTIO_DEV_MAX]; } oct_virtio_per_thread_data_t; diff --git a/src/plugins/dev_octeon/virtio.c b/src/plugins/dev_octeon/virtio.c index c4f41e672f..8279d77e94 100644 --- a/src/plugins/dev_octeon/virtio.c +++ b/src/plugins/dev_octeon/virtio.c @@ -28,6 +28,7 @@ enum oct_virtio_dev_args_types DEV_ARG_VIRT_NB_VIRTIO_DEVICES = 1, DEV_ARG_VIRT_DMA_DEVICE_LIST, DEV_ARG_VIRT_MISC_DEVICE, + DEV_ARG_VIRT_CSUM_OFFLD_EN, DEV_ARG_VIRT_END }; @@ -51,6 +52,13 @@ static vnet_dev_arg_t oct_virtio_dev_args[] = { .desc = "Miscellaneous device list", .type = VNET_DEV_ARG_TYPE_STRING, }, + { + .id = DEV_ARG_VIRT_CSUM_OFFLD_EN, + .name = "enable_csum_offld", + .desc = "Enable Host checksum offload", + .type = VNET_DEV_ARG_TYPE_BOOL, + .default_val.boolean = 0, + }, { .id = DEV_ARG_VIRT_END, .name = "end", @@ -208,15 +216,18 @@ oct_virtio_parse_arguments (dao_pal_global_conf_t *conf, vnet_dev_arg_t *args) switch (a->id) { case DEV_ARG_VIRT_NB_VIRTIO_DEVICES: - conf->nb_virtio_devs = a->val.uint32; + conf->nb_virtio_devs = vnet_dev_arg_get_uint32 (a); break; case DEV_ARG_VIRT_DMA_DEVICE_LIST: - conf->dma_devices = - oct_populate_dma_device_list (&conf->nb_dma_devs, a->val.string); + conf->dma_devices = oct_populate_dma_device_list ( + &conf->nb_dma_devs, vnet_dev_arg_get_string (a)); break; case DEV_ARG_VIRT_MISC_DEVICE: conf->misc_devices = oct_populate_dma_device_list ( - &conf->nb_misc_devices, a->val.string); + &conf->nb_misc_devices, vnet_dev_arg_get_string (a)); + break; + case DEV_ARG_VIRT_CSUM_OFFLD_EN: + oct_virtio_main->ip4_csum_offload_enable = vnet_dev_arg_get_bool (a); break; default: log_info ("Invalid virtio device arguments received\n"); diff --git a/src/plugins/dev_octeon/virtio_ctrl.c b/src/plugins/dev_octeon/virtio_ctrl.c index 435eb373d6..520e3754b8 100644 --- a/src/plugins/dev_octeon/virtio_ctrl.c +++ b/src/plugins/dev_octeon/virtio_ctrl.c @@ -9,6 +9,8 @@ #include #define OCT_VIRTIO_MAX_WRKS 24 +#define OCT_VIRTIO_CHECKSUM_OFFLOAD_MASK 0x3 +#define OCT_VIRTIO_TSO_OFFLOAD_MASK 0xFFFF extern oct_virtio_main_t *oct_virtio_main; extern oct_virtio_port_map_t *virtio_port_map; @@ -139,11 +141,45 @@ oct_virtio_netdev_hdrlen_get (u16 virtio_devid) return virtio_hdr_sz; } +static int +chksum_offload_configure (uint16_t virtio_devid, u64 *tx_offloads, + u64 *rx_offloads) +{ + u64 csum_offload, tso_offload; + + csum_offload = dao_virtio_netdev_feature_bits_get (virtio_devid) & + OCT_VIRTIO_CHECKSUM_OFFLOAD_MASK; + tso_offload = dao_virtio_netdev_feature_bits_get (virtio_devid) & + OCT_VIRTIO_TSO_OFFLOAD_MASK; + + if (csum_offload & DAO_BIT_ULL (VIRTIO_NET_F_CSUM)) + *tx_offloads |= OCT_ETH_TX_OFFLOAD_IPV4_CKSUM; + + if (tso_offload & DAO_BIT_ULL (VIRTIO_NET_F_HOST_TSO4) || + tso_offload & DAO_BIT_ULL (VIRTIO_NET_F_HOST_TSO6)) + { + *tx_offloads |= OCT_ETH_TX_OFFLOAD_TCP_TSO; + log_err ("TSO offload is not supported\n"); + } + + if (csum_offload & DAO_BIT_ULL (VIRTIO_NET_F_GUEST_CSUM)) + *rx_offloads |= OCT_ETH_RX_OFFLOAD_CHECKSUM; + + /** + * We need to configure out interface, but by default, OCTEON interfaces are + * enabled with RX and TX checksum enabled, and currently, we don’t have + * control to enable or disable them. For now, based on these flags, the + * correct flags will be set for the HOST. + */ + return 0; +} + static_always_inline int oct_virtio_setup_worker_queue_mapping (u16 virtio_devid, u16 virt_q_count) { u32 cpu_id = 0; u16 virt_rx_q, q_id; + u64 tx_offloads = 0, rx_offloads = 0; oct_virtio_main_t *ovm = oct_virtio_main; u64 wrkr_cpu_mask = ovm->wrkr_cpu_mask; oct_virtio_per_thread_data_t *ptd = oct_virt_thread_data; @@ -151,6 +187,8 @@ oct_virtio_setup_worker_queue_mapping (u16 virtio_devid, u16 virt_q_count) virtio_hdr_sz = oct_virtio_netdev_hdrlen_get (virtio_devid); + chksum_offload_configure (virtio_devid, &tx_offloads, &rx_offloads); + virt_rx_q = virt_q_count / 2; q_id = 0; for (q_id = 0; q_id < virt_rx_q && ovm->wrkr_cpu_mask; q_id++) @@ -162,6 +200,11 @@ oct_virtio_setup_worker_queue_mapping (u16 virtio_devid, u16 virt_q_count) CLIB_MEMORY_BARRIER (); ptd[cpu_id].netdev_map |= DAO_BIT (virtio_devid); + if (oct_virtio_main->ip4_csum_offload_enable) + { + ptd[cpu_id].intf[virtio_devid].tx_offloads = tx_offloads; + ptd[cpu_id].intf[virtio_devid].rx_offloads = rx_offloads; + } wrkr_cpu_mask &= ~DAO_BIT_ULL (cpu_id); cpu_id++; if (!wrkr_cpu_mask) @@ -249,7 +292,6 @@ oct_virtio_dev_status_cb (u16 virtio_devid, u8 status) oct_virtio_clear_lcore_queue_mapping (virtio_devid); break; case VIRTIO_DEV_DRIVER_OK: - /* Get active virt queue count */ virt_q_count = dao_virtio_netdev_queue_count (virtio_devid); diff --git a/src/plugins/dev_octeon/virtio_port.c b/src/plugins/dev_octeon/virtio_port.c index 3ebc1caf64..c7f00bb7c0 100644 --- a/src/plugins/dev_octeon/virtio_port.c +++ b/src/plugins/dev_octeon/virtio_port.c @@ -17,6 +17,7 @@ static u64 vchan_bitmap[2] = { 0 }; extern oct_virtio_port_map_t *virtio_port_map; +extern oct_virtio_main_t *oct_virtio_main; VLIB_REGISTER_LOG_CLASS (oct_virt_log, static) = { .class_name = "octeon", @@ -82,6 +83,7 @@ oct_virtio_port_init (vlib_main_t *vm, vnet_dev_port_t *port) netdev_conf.link_info.duplex = 0xFF; netdev_conf.hash_key_size = OCT_VIRTIO_NIX_RSS_KEY_LEN; netdev_conf.dma_vchan = oct_virtio_dma_vchan_id_allocate (); + netdev_conf.csum_en = oct_virtio_main->ip4_csum_offload_enable; memcpy (netdev_conf.mac, port->attr.hw_addr.eth_mac, sizeof (netdev_conf.mac)); log_debug ("port start: port %u, virtio_id %u, vchan_id %d\n", port->port_id, diff --git a/src/plugins/dev_octeon/virtio_rx_node.c b/src/plugins/dev_octeon/virtio_rx_node.c index b027da3ccb..e94a6930ff 100644 --- a/src/plugins/dev_octeon/virtio_rx_node.c +++ b/src/plugins/dev_octeon/virtio_rx_node.c @@ -304,7 +304,12 @@ oct_virtio_rx_node_inline (vlib_main_t *vm, vlib_node_runtime_t *node, nf = vlib_node_runtime_get_next_frame (vm, node, rxq->next_index); f = vlib_get_frame (vm, nf->frame); f->flags = ETH_INPUT_FRAME_F_SINGLE_SW_IF_IDX; - /* checksum offload not supported yet */ + /** + * We can set the checksum as OK because, in the host checksum + * offload case, OCTEON Tx will perform the checksum computation. In + * the host non-checksum offload case, the host computes the checksum + * and provides it to OCTEON. + */ f->flags |= ETH_INPUT_FRAME_F_IP4_CKSUM_OK; ef = vlib_frame_scalar_args (f); diff --git a/src/plugins/dev_octeon/virtio_tx_node.c b/src/plugins/dev_octeon/virtio_tx_node.c index b9862831eb..3afebe935b 100644 --- a/src/plugins/dev_octeon/virtio_tx_node.c +++ b/src/plugins/dev_octeon/virtio_tx_node.c @@ -42,6 +42,7 @@ static_always_inline u32 oct_virtio_enqueue (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_buffer_t **b, u16 nb_pkts, u16 virtio_devid) { + u64 rx_offload; vlib_buffer_t *bp; bool next_present; u64 tx_q_map, q_map; @@ -57,6 +58,11 @@ oct_virtio_enqueue (vlib_main_t *vm, vlib_node_runtime_t *node, tx_q_map = ptd[cpu_id].q_map[virtio_devid].qmap; q_map = ptd[cpu_id].q_map[virtio_devid].qmap; hdr_len = ptd[cpu_id].q_map[virtio_devid].virtio_hdr_sz; + rx_offload = ptd[cpu_id].intf[virtio_devid].rx_offloads; + + /* Packets reaching to tx node means we can assume the checksum is good. */ + if (rx_offload & OCT_ETH_RX_OFFLOAD_CHECKSUM) + vhdr_init.hdr.flags = VIRTIO_NET_HDR_F_DATA_VALID; while (nb_pkts >= 8) { From 11d86f3b35cd6a50db92703d7ce53eb59df14519 Mon Sep 17 00:00:00 2001 From: Harish Kumar Malik Date: Tue, 29 Oct 2024 16:54:58 +0500 Subject: [PATCH 128/271] octeon: add tm support for octeon Type: feature Signed-off-by: Harish Kumar Malik Signed-off-by: Monendra Singh Kushwaha Signed-off-by: Alok Mishra Change-Id: Ia057a9a935ac92f48d636dece2187c54c6ff4b61 Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/138139 Tested-by: sa_ip-sw-jenkins (cherry picked from commit 64e2ac298fe30364812eaedeffe13b62b984191f) Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/140442 --- src/plugins/dev_octeon/CMakeLists.txt | 1 + src/plugins/dev_octeon/octeon.h | 2 + src/plugins/dev_octeon/port.c | 16 +- src/plugins/dev_octeon/tm.c | 387 ++++++++++++++++++++++++++ src/plugins/dev_octeon/tm.h | 18 ++ 5 files changed, 422 insertions(+), 2 deletions(-) create mode 100644 src/plugins/dev_octeon/tm.c create mode 100644 src/plugins/dev_octeon/tm.h diff --git a/src/plugins/dev_octeon/CMakeLists.txt b/src/plugins/dev_octeon/CMakeLists.txt index d9cbbf6679..410317764f 100644 --- a/src/plugins/dev_octeon/CMakeLists.txt +++ b/src/plugins/dev_octeon/CMakeLists.txt @@ -44,6 +44,7 @@ add_vpp_plugin(dev_octeon crypto.c ipsec.c dpu/dpu.c + tm.c MULTIARCH_SOURCES rx_node.c diff --git a/src/plugins/dev_octeon/octeon.h b/src/plugins/dev_octeon/octeon.h index 2a6c72bffc..905e10253b 100644 --- a/src/plugins/dev_octeon/octeon.h +++ b/src/plugins/dev_octeon/octeon.h @@ -241,4 +241,6 @@ typedef struct oct_tx_desc_t desc; } oct_tx_trace_t; +extern tm_system_t dev_oct_tm_ops; + #endif /* _OCTEON_H_ */ diff --git a/src/plugins/dev_octeon/port.c b/src/plugins/dev_octeon/port.c index 1d52e0fc02..7274fee162 100644 --- a/src/plugins/dev_octeon/port.c +++ b/src/plugins/dev_octeon/port.c @@ -16,6 +16,7 @@ #define OCT_ETH_LINK_SPEED_100G 100000 /**< 100 Gbps */ extern oct_inl_dev_main_t oct_inl_dev_main; +tm_system_t tm_system_ops; VLIB_REGISTER_LOG_CLASS (oct_log, static) = { .class_name = "octeon", @@ -40,6 +41,14 @@ static const u64 rxq_cfg = ROC_NIX_LF_RX_CFG_LEN_OL3 | ROC_NIX_LF_RX_CFG_LEN_OL4 | ROC_NIX_LF_RX_CFG_LEN_IL3 | ROC_NIX_LF_RX_CFG_LEN_IL4; +static int +oct_init_tm_args (tm_system_t *tm) +{ + memset (tm, 0, sizeof (tm_system_t)); + memcpy (tm, &dev_oct_tm_ops, sizeof (tm_system_t)); + return 0; +} + static vnet_dev_rv_t oct_roc_err (vnet_dev_t *dev, int rv, char *fmt, ...) { @@ -135,10 +144,10 @@ oct_port_init (vlib_main_t *vm, vnet_dev_port_t *port) oct_port_t *cp = vnet_dev_get_port_data (port); u8 mac_addr[PLT_ETHER_ADDR_LEN]; struct roc_nix *nix = cd->nix; - vnet_dev_rv_t rv; + vnet_dev_rv_t rv = -1; int rrv; - log_debug (dev, "port init: port %u", port->port_id); + log_notice (dev, "port init: port %u", port->port_id); if ((rrv = roc_nix_lf_alloc (nix, port->intf.num_rx_queues, port->intf.num_tx_queues, rxq_cfg))) @@ -248,6 +257,9 @@ oct_port_init (vlib_main_t *vm, vnet_dev_port_t *port) oct_port_add_counters (vm, port); + oct_init_tm_args (&tm_system_ops); + tm_system_register (&tm_system_ops, port->intf.hw_if_index); + return VNET_DEV_OK; } diff --git a/src/plugins/dev_octeon/tm.c b/src/plugins/dev_octeon/tm.c new file mode 100644 index 0000000000..3fd1d1c943 --- /dev/null +++ b/src/plugins/dev_octeon/tm.c @@ -0,0 +1,387 @@ +/* + * Copyright (c) 2024 Marvell. + * SPDX-License-Identifier: Apache-2.0 + * https://spdx.org/licenses/Apache-2.0.html + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "tm.h" + +VLIB_REGISTER_LOG_CLASS (oct_log, static) = { + .class_name = "octeon", + .subclass_name = "tm", +}; + +static vnet_dev_rv_t +oct_roc_err (vnet_dev_t *dev, int rv, char *fmt, ...) +{ + u8 *s = 0; + va_list va; + + va_start (va, fmt); + s = va_format (s, fmt, &va); + va_end (va); + + log_err (dev, "%v - ROC error %s (%d)", s, roc_error_msg_get (rv), rv); + + vec_free (s); + return VNET_DEV_ERR_INTERNAL; +} + +int +oct_tm_sys_node_add (u32 hw_if_idx, u32 node_id, u32 parent_node_id, + u32 priority, u32 weight, u32 lvl, + tm_node_params_t *params) + +{ + vnet_main_t *vnm = vnet_get_main (); + vnet_hw_interface_t *hi = vnet_get_hw_interface (vnm, hw_if_idx); + vnet_dev_port_t *port = + vnet_dev_get_port_from_dev_instance (hi->dev_instance); + vnet_dev_t *dev = port->dev; + oct_device_t *cd = vnet_dev_get_data (dev); + struct roc_nix *nix = cd->nix; + struct roc_nix_tm_node *parent_node = NULL; + struct roc_nix_tm_node *tm_node = NULL; + struct roc_nix_tm_shaper_profile *profile = NULL; + int rc = 0; + + /* We don't support dynamic updates */ + if (roc_nix_tm_is_user_hierarchy_enabled (nix)) + { + rc = -ERANGE; + return oct_roc_err (dev, rc, "roc_nix_tm_dynamic update not supported"); + } + if (parent_node_id) + { + parent_node = roc_nix_tm_node_get (nix, parent_node_id); + } + + /* Find the right level */ + if (lvl != ROC_TM_LVL_ROOT && parent_node) + { + lvl = parent_node->lvl + 1; + } + else if (parent_node_id == ROC_NIX_TM_NODE_ID_INVALID) + { + lvl = ROC_TM_LVL_ROOT; + } + else + { + /* Neither proper parent nor proper level id given */ + rc = -ERANGE; + return oct_roc_err (dev, rc, "roc_nix_tm_invalid_parent-id_err"); + } + + tm_node = plt_zmalloc (sizeof (struct roc_nix_tm_node), 0); + if (!tm_node) + { + rc = -ENOMEM; + return oct_roc_err (dev, rc, "oct_nix_tm_node_alloc_failed"); + } + + tm_node->id = node_id; + tm_node->parent_id = parent_node_id; + tm_node->lvl = lvl; + tm_node->priority = priority; + tm_node->free_fn = plt_free; + tm_node->weight = weight; + tm_node->shaper_profile_id = params->shaper_profile_id; + + profile = roc_nix_tm_shaper_profile_get (nix, params->shaper_profile_id); + + rc = roc_nix_tm_node_add (nix, tm_node); + if (rc < 0) + { + plt_free (tm_node); + return oct_roc_err (dev, rc, "roc_nix_tm_node_add_err"); + } + + roc_nix_tm_shaper_default_red_algo (tm_node, profile); + return 0; +} + +int +oct_tm_sys_node_delete (u32 hw_if_idx, u32 node_id) +{ + vnet_main_t *vnm = vnet_get_main (); + vnet_hw_interface_t *hi = vnet_get_hw_interface (vnm, hw_if_idx); + vnet_dev_port_t *port = + vnet_dev_get_port_from_dev_instance (hi->dev_instance); + vnet_dev_t *dev = port->dev; + oct_device_t *cd = vnet_dev_get_data (dev); + struct roc_nix *nix = cd->nix; + struct roc_nix_tm_node *tm_node = NULL; + int rc; + bool free_node = 1; + + if ((rc = roc_nix_tm_is_user_hierarchy_enabled (nix))) + { + rc = -ERANGE; + return oct_roc_err (dev, rc, "roc_nix_tm_dynamic update not supported"); + } + if (node_id == ROC_NIX_TM_NODE_ID_INVALID) + { + rc = -EINVAL; + return oct_roc_err (dev, rc, "oct_tm_node_delete_invalid_node-id"); + } + + tm_node = roc_nix_tm_node_get (nix, node_id); + if (!tm_node) + { + rc = -EINVAL; + return oct_roc_err (dev, rc, "oct_tm_node_delete node-id not found"); + } + + rc = roc_nix_tm_node_delete (nix, tm_node->id, free_node); + if (rc) + { + return oct_roc_err (dev, rc, "roc_nix_tm_delete_failed"); + } + return 0; +} + +int +oct_tm_sys_shaper_profile_create (u32 hw_if_idx, tm_shaper_params_t *params) +{ + vnet_main_t *vnm = vnet_get_main (); + vnet_hw_interface_t *hi = vnet_get_hw_interface (vnm, hw_if_idx); + vnet_dev_port_t *port = + vnet_dev_get_port_from_dev_instance (hi->dev_instance); + vnet_dev_t *dev = port->dev; + oct_device_t *cd = vnet_dev_get_data (dev); + struct roc_nix *nix = cd->nix; + struct roc_nix_tm_shaper_profile *profile; + int rc; + + if (roc_nix_tm_shaper_profile_get (nix, params->shaper_id)) + { + rc = -EINVAL; + return oct_roc_err (dev, rc, "oct_nix_tm_shaper_exists"); + } + + profile = plt_zmalloc (sizeof (struct roc_nix_tm_shaper_profile), 0); + if (!profile) + { + rc = -ENOMEM; + return oct_roc_err (dev, rc, "oct_nix_tm_shaper_create_alloc_failed"); + } + profile->id = params->shaper_id; + profile->commit_rate = params->commit.rate; + profile->commit_sz = params->commit.burst_size; + profile->peak_rate = params->peak.rate; + profile->peak_sz = params->peak.burst_size; + /* If Byte mode, then convert to bps */ + if (!params->pkt_mode) + { + profile->commit_rate *= 8; + profile->peak_rate *= 8; + profile->commit_sz *= 8; + profile->peak_sz *= 8; + } + profile->pkt_len_adj = params->pkt_len_adj; + profile->pkt_mode = params->pkt_mode; + profile->free_fn = plt_free; + + rc = roc_nix_tm_shaper_profile_add (nix, profile); + + /* Fill error information based on return value */ + if (rc) + { + plt_free (profile); + return oct_roc_err (dev, rc, "roc_nix_tm_shaper_creation_failed"); + } + + return rc; +} + +int +oct_tm_sys_node_shaper_update (u32 hw_if_idx, u32 node_id, u32 profile_id) +{ + vnet_main_t *vnm = vnet_get_main (); + vnet_hw_interface_t *hi = vnet_get_hw_interface (vnm, hw_if_idx); + vnet_dev_port_t *port = + vnet_dev_get_port_from_dev_instance (hi->dev_instance); + vnet_dev_t *dev = port->dev; + oct_device_t *cd = vnet_dev_get_data (dev); + struct roc_nix *nix = cd->nix; + struct roc_nix_tm_shaper_profile *profile; + struct roc_nix_tm_node *node; + int rc; + + rc = roc_nix_tm_node_shaper_update (nix, node_id, profile_id, false); + if (rc) + { + return oct_roc_err (dev, rc, "oct_nix_tm_node_shaper_update_failed"); + } + + node = roc_nix_tm_node_get (nix, node_id); + if (!node) + { + rc = -EINVAL; + return oct_roc_err (dev, rc, + "oct_nix_tm_node_shaper_update_node_failure"); + } + + profile = roc_nix_tm_shaper_profile_get (nix, profile_id); + roc_nix_tm_shaper_default_red_algo (node, profile); + + return 0; +} +int +oct_tm_sys_shaper_profile_delete (u32 hw_if_idx, u32 shaper_id) +{ + vnet_main_t *vnm = vnet_get_main (); + vnet_hw_interface_t *hi = vnet_get_hw_interface (vnm, hw_if_idx); + vnet_dev_port_t *port = + vnet_dev_get_port_from_dev_instance (hi->dev_instance); + vnet_dev_t *dev = port->dev; + oct_device_t *cd = vnet_dev_get_data (dev); + struct roc_nix *nix = cd->nix; + int rc; + + rc = roc_nix_tm_shaper_profile_delete (nix, shaper_id); + if (rc) + { + return oct_roc_err (dev, rc, "roc_nix_tm_shaper_delete_failed"); + } + + return rc; +} + +int +oct_tm_sys_node_read_stats (u32 hw_if_idx, u32 node_id, + tm_stats_params_t *stats) +{ + vnet_main_t *vnm = vnet_get_main (); + vnet_hw_interface_t *hi = vnet_get_hw_interface (vnm, hw_if_idx); + vnet_dev_port_t *port = + vnet_dev_get_port_from_dev_instance (hi->dev_instance); + vnet_dev_t *dev = port->dev; + oct_device_t *cd = vnet_dev_get_data (dev); + struct roc_nix *nix = cd->nix; + int rc = 0; + int clear = 0; + struct roc_nix_tm_node_stats nix_tm_stats; + struct roc_nix_tm_node *node; + + node = roc_nix_tm_node_get (nix, node_id); + if (!node) + { + goto exit; + } + + if (roc_nix_tm_lvl_is_leaf (nix, node->lvl)) + { + struct roc_nix_stats_queue qstats; + + rc = roc_nix_stats_queue_get (nix, node->id, 0, &qstats); + if (!rc) + { + stats->n_pkts = qstats.tx_pkts; + stats->n_bytes = qstats.tx_octs; + printf (" - STATS for node \n"); + printf (" -- pkts (%" PRIu64 ") bytes (%" PRIu64 ")\n", + stats->n_pkts, stats->n_bytes); + } + goto exit; + } + + rc = roc_nix_tm_node_stats_get (nix, node_id, clear, &nix_tm_stats); + if (!rc) + { + stats->leaf.n_pkts_dropped[TM_COLOR_RED] = + nix_tm_stats.stats[ROC_NIX_TM_NODE_PKTS_DROPPED]; + stats->leaf.n_bytes_dropped[TM_COLOR_RED] = + nix_tm_stats.stats[ROC_NIX_TM_NODE_BYTES_DROPPED]; + } + +exit: + if (rc) + { + return oct_roc_err (dev, rc, "tm_node_read_stats_err"); + } + return rc; +} + +int +oct_tm_sys_start (u32 hw_if_idx) +{ + vnet_main_t *vnm = vnet_get_main (); + vnet_hw_interface_t *hi = vnet_get_hw_interface (vnm, hw_if_idx); + vnet_dev_port_t *port = + vnet_dev_get_port_from_dev_instance (hi->dev_instance); + vnet_dev_t *dev = port->dev; + oct_device_t *cd = vnet_dev_get_data (dev); + struct roc_nix *nix = cd->nix; + int rc = 0; + + if (roc_nix_tm_is_user_hierarchy_enabled (nix)) + { + rc = -EIO; + return oct_roc_err (dev, rc, "oct_nix_tm_hirearchy_exists"); + } + + if (roc_nix_tm_leaf_cnt (nix) < port->intf.num_tx_queues) + { + rc = -EINVAL; + return oct_roc_err (dev, rc, "oct_nix_tm_incomplete hierarchy"); + } + + rc = roc_nix_tm_hierarchy_disable (nix); + if (rc) + { + return oct_roc_err (dev, rc, "oct_nix_tm_hirearchy_exists"); + } + + rc = roc_nix_tm_hierarchy_enable (nix, ROC_NIX_TM_USER, true); + if (rc) + { + return oct_roc_err (dev, rc, "oct_nix_tm_hierarchy_enabled_failed"); + } + return 0; +} + +int +oct_tm_sys_stop (u32 hw_if_idx) +{ + vnet_main_t *vnm = vnet_get_main (); + vnet_hw_interface_t *hi = vnet_get_hw_interface (vnm, hw_if_idx); + vnet_dev_port_t *port = + vnet_dev_get_port_from_dev_instance (hi->dev_instance); + vnet_dev_t *dev = port->dev; + oct_device_t *cd = vnet_dev_get_data (dev); + struct roc_nix *nix = cd->nix; + int rc = 0; + + /* Disable hierarchy */ + rc = roc_nix_tm_hierarchy_disable (nix); + if (rc) + { + rc = -EIO; + return oct_roc_err (dev, rc, "oct_nix_tm_stop_failed"); + } + + return 0; +} + +tm_system_t dev_oct_tm_ops = { + .node_add = oct_tm_sys_node_add, + .node_delete = oct_tm_sys_node_delete, + .node_read_stats = oct_tm_sys_node_read_stats, + .shaper_profile_create = oct_tm_sys_shaper_profile_create, + .node_shaper_update = oct_tm_sys_node_shaper_update, + .shaper_profile_delete = oct_tm_sys_shaper_profile_delete, + .start_tm = oct_tm_sys_start, + .stop_tm = oct_tm_sys_stop, +}; diff --git a/src/plugins/dev_octeon/tm.h b/src/plugins/dev_octeon/tm.h new file mode 100644 index 0000000000..9d7c008b04 --- /dev/null +++ b/src/plugins/dev_octeon/tm.h @@ -0,0 +1,18 @@ +/* + * Copyright (c) 2024 Marvell. + * SPDX-License-Identifier: Apache-2.0 + * https://spdx.org/licenses/Apache-2.0.html + */ + +#ifndef _OCT_TM_H_ +#define _OCT_TM_H_ + +#include +#include +#include + +#define NIX_TM_DFLT_RR_WT 71 +#define OCT_TM_NODE_ID_NULL -1 +#define OCT_TM_INVALID 0 + +#endif /* _OCT_TM_H_ */ From a89c429e68d2ecdb4554d5426d67dc5cf2003145 Mon Sep 17 00:00:00 2001 From: Monendra Singh Kushwaha Date: Wed, 25 Sep 2024 02:04:12 +0530 Subject: [PATCH 129/271] octeon: add inline ipsec outbound support Type: feature Change-Id: I275f922563ee5f5cb16ac849e719a846bf353a46 Signed-off-by: Monendra Singh Kushwaha Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/136270 Tested-by: sa_ip-sw-jenkins Reviewed-by: Nithin Kumar Dabilpuram (cherry picked from commit bc0f07dd10128e1a0ee9197e655156937ad9fe1a) Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/140454 --- src/plugins/dev_octeon/CMakeLists.txt | 1 + src/plugins/dev_octeon/esp_encrypt.c | 611 +++++++++++++ src/plugins/dev_octeon/init.c | 33 +- src/plugins/dev_octeon/ipsec.c | 313 ++++++- src/plugins/dev_octeon/ipsec.h | 32 +- src/plugins/dev_octeon/octeon.h | 44 +- src/plugins/dev_octeon/port.c | 8 + src/plugins/dev_octeon/queue.c | 5 +- src/plugins/dev_octeon/tx_node.c | 1153 +++++++++++++++++++++++++ src/vnet/buffer.h | 3 +- 10 files changed, 2169 insertions(+), 34 deletions(-) create mode 100644 src/plugins/dev_octeon/esp_encrypt.c diff --git a/src/plugins/dev_octeon/CMakeLists.txt b/src/plugins/dev_octeon/CMakeLists.txt index 410317764f..93dec8de6d 100644 --- a/src/plugins/dev_octeon/CMakeLists.txt +++ b/src/plugins/dev_octeon/CMakeLists.txt @@ -43,6 +43,7 @@ add_vpp_plugin(dev_octeon counter.c crypto.c ipsec.c + esp_encrypt.c dpu/dpu.c tm.c diff --git a/src/plugins/dev_octeon/esp_encrypt.c b/src/plugins/dev_octeon/esp_encrypt.c new file mode 100644 index 0000000000..1540f1675b --- /dev/null +++ b/src/plugins/dev_octeon/esp_encrypt.c @@ -0,0 +1,611 @@ +#include +#include +#include +#include + +extern oct_ipsec_main_t oct_ipsec_main; + +#define foreach_oct_esp_handoff_core \ + _ (0) \ + _ (1) \ + _ (2) \ + _ (3) \ + _ (4) \ + _ (5) \ + _ (6) \ + _ (7) \ + _ (8) \ + _ (9) \ + _ (10) \ + _ (11) \ + _ (12) \ + _ (13) \ + _ (14) \ + _ (15) \ + _ (16) \ + _ (17) \ + _ (18) \ + _ (19) \ + _ (20) \ + _ (21) \ + _ (22) \ + _ (23) \ + _ (24) \ + _ (25) \ + _ (26) \ + _ (27) \ + _ (28) \ + _ (29) \ + _ (30) \ + _ (31) \ + _ (32) \ + _ (33) \ + _ (34) \ + _ (35) + +#define OCT_ESP_ENCRYPT_TUN_N_NEXT (OCT_ESP_ENCRYPT_TUN_NEXT_PREP35 + 1) + +#define OCT_ESP_ENCRYPT_TUN_PREP_NODES(x, y) \ + [OCT_ESP_ENCRYPT_TUN_NEXT_PREP##y] = #x #y, + +#define foreach_oct_esp_encrypt_error \ + _ (RX_PKTS, "ESP pkts received") \ + _ (RX_POST_PKTS, "ESP-POST pkts received") \ + _ (NOT_L3PKT, "L3 header offset not valid") \ + _ (CHAINING_NOSUPP, "Packet chainining not supported in IPsec") \ + _ (SEQ_CYCLED, "sequence number cycled (packet dropped)") \ + _ (HANDOFF, "handoff") \ + _ (INVALID_SA, "invalid SA") \ + _ (FRAME_ALLOC, "encrypt ipsec frame alloc failed") \ + _ (UNDEFINED, "undefined encrypt error") + +typedef struct +{ + u32 sa_index; + u32 spi; + u32 seq; + u32 sa_seq_hi; + u32 next_index; + u32 owner_thread; + u32 handoff_thread; + u8 udp_encap; + vlib_error_t error; + ipsec_crypto_alg_t crypto_alg; + ipsec_integ_alg_t integ_alg; + u8 data[256]; + vlib_buffer_t buf; +} oct_esp_encrypt_trace_t; + +#define foreach_oct_esp_encrypt_tun_next \ + _ (DROP4, "ip4-drop") \ + _ (DROP6, "ip6-drop") \ + _ (ADJ_MIDCHAIN_TX, "adj-midchain-tx") + +/* clang-format off */ +typedef enum +{ +#define _(v, s) OCT_ESP_ENCRYPT_TUN_NEXT_##v, + foreach_oct_esp_encrypt_tun_next +#undef _ +#define _(Y) OCT_ESP_ENCRYPT_TUN_NEXT_PREP##Y, + foreach_oct_esp_handoff_core +#undef _ +} oct_esp_encrypt_tun_next_t; +/* clang-format on */ + +/* clang-format off */ +typedef enum +{ +#define _(sym, str) OCT_ESP_ENCRYPT_ERROR_##sym, + foreach_oct_esp_encrypt_error +#undef _ +#define _(sym, str) OCT_ESP_ENCRYPT_CN10K_ERROR_##sym, + foreach_octeon_cn10k_ipsec_ucc +#undef _ +} oct_esp_encrypt_error_t; +/* clang-format on */ + +/* Packet trace format function */ +static u8 * +format_oct_esp_encrypt_trace (u8 *s, va_list *args) +{ + CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *); + CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *); + oct_esp_encrypt_trace_t *t = va_arg (*args, oct_esp_encrypt_trace_t *); + vlib_error_main_t *em = &vm->error_main; + u32 indent = format_get_indent (s); + vlib_error_t e = t->error; + u32 ci; + + s = format (s, "%U %U\n", format_white_space, indent, format_vnet_buffer, + &t->buf); + + if (e) + { + ci = vlib_error_get_code (&vm->node_main, e); + + ci += node->error_heap_index; + + s = format (s, "%UStatus: %s", format_white_space, indent, + em->counters_heap[ci].name); + + if (t->handoff_thread == t->owner_thread) + s = format (s, ", Handoff thread: %u", t->handoff_thread); + + s = format (s, "\n"); + } + + s = format (s, "%USA owner thread: %u\n", format_white_space, indent, + t->owner_thread); + + if (t->next_index != ~0) + s = format (s, "%Unext node: %U\n", format_white_space, indent, + format_vlib_next_node_name, vm, node->index, t->next_index); + + s = format (s, + "%Uesp: sa-index %d spi %u (0x%08x) seq %u sa-seq-hi %u " + "crypto %U integrity %U%s", + format_white_space, indent, t->sa_index, t->spi, t->spi, t->seq, + t->sa_seq_hi, format_ipsec_crypto_alg, t->crypto_alg, + format_ipsec_integ_alg, t->integ_alg, + t->udp_encap ? " udp-encap-enabled" : ""); + + if (vm->trace_main.verbose) + { + s = format (s, "\n%U%U", format_white_space, indent + 4, format_hexdump, + &t->data, 128); + } + return s; +} + +static_always_inline void +oct_esp_encrypt_tun_add_trace (vlib_main_t *vm, vlib_node_runtime_t *node, + vlib_frame_t *frame, vlib_buffer_t *b, + u32 next_index) +{ + oct_esp_encrypt_trace_t *tr; + ipsec_sa_t *sa; + u32 sa_index; + + tr = vlib_add_trace (vm, node, b, sizeof (*tr)); + sa_index = vnet_buffer (b)->ipsec.sad_index; + sa = ipsec_sa_get (sa_index); + tr->next_index = next_index; + tr->sa_index = sa_index; + tr->spi = sa->spi; + tr->seq = sa->seq; + tr->sa_seq_hi = sa->seq_hi; + tr->udp_encap = ipsec_sa_is_set_UDP_ENCAP (sa); + tr->crypto_alg = sa->crypto_alg; + tr->integ_alg = sa->integ_alg; + tr->owner_thread = sa->thread_index; + + clib_memcpy_fast (&tr->buf, b, sizeof b[0] - sizeof b->pre_data); + clib_memcpy_fast (tr->buf.pre_data, b->data, sizeof tr->buf.pre_data); + clib_memcpy_fast (tr->data, vlib_buffer_get_current (b), 256); +} + +static_always_inline u32 +oct_ipsec_sa_index_get (vlib_buffer_t *b, const int is_tun) +{ + u32 sa_index, adj_index; + + if (is_tun) + { + adj_index = vnet_buffer (b)->ip.adj_index[VLIB_TX]; + sa_index = ipsec_tun_protect_get_sa_out (adj_index); + vnet_buffer (b)->ipsec.sad_index = sa_index; + } + else + sa_index = vnet_buffer (b)->ipsec.sad_index; + + return sa_index; +} + +static_always_inline uword +oct_esp_encrypt_tun (vlib_main_t *vm, vlib_node_runtime_t *node, + vlib_frame_t *frame, const int is_ip6) +{ + u32 *from = vlib_frame_vector_args (frame); + u32 n_left = frame->n_vectors; + vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b = bufs; + u16 nexts[VLIB_FRAME_SIZE], *next = nexts; + u32 sa0_index, sa1_index, sa2_index, sa3_index; + u32 current_sa0_index = ~0, current_sa1_index = ~0; + u32 current_sa2_index = ~0, current_sa3_index = ~0; + ipsec_sa_t *sa0 = NULL, *sa1 = NULL, *sa2 = NULL, *sa3 = NULL; + u16 thread0_next = OCT_ESP_ENCRYPT_TUN_NEXT_PREP0; + + vlib_get_buffers (vm, from, b, frame->n_vectors); + + while (n_left > 11) + { + vlib_prefetch_buffer_header (b[8], LOAD); + vlib_prefetch_buffer_header (b[9], LOAD); + vlib_prefetch_buffer_header (b[10], LOAD); + vlib_prefetch_buffer_header (b[11], LOAD); + + sa0_index = oct_ipsec_sa_index_get (b[0], 1); + sa1_index = oct_ipsec_sa_index_get (b[1], 1); + sa2_index = oct_ipsec_sa_index_get (b[2], 1); + sa3_index = oct_ipsec_sa_index_get (b[3], 1); + + if (sa0_index != current_sa0_index) + { + sa0 = ipsec_sa_get (sa0_index); + current_sa0_index = sa0_index; + } + if (sa1_index != current_sa1_index) + { + sa1 = ipsec_sa_get (sa1_index); + current_sa1_index = sa1_index; + } + if (sa2_index != current_sa2_index) + { + sa2 = ipsec_sa_get (sa2_index); + current_sa2_index = sa2_index; + } + if (sa3_index != current_sa3_index) + { + sa3 = ipsec_sa_get (sa3_index); + current_sa3_index = sa3_index; + } + + /* + * If this is the first packet to use this SA, assign thread based + * on SA index. Don't need to do core-handoff on OCTEON as send queue + * is used based on thread index. + */ + if (PREDICT_FALSE (sa0->thread_index == 0xFFFF)) + sa0->thread_index = (sa0_index % vlib_num_workers ()) + 1; + if (PREDICT_FALSE (sa1->thread_index == 0xFFFF)) + sa1->thread_index = (sa1_index % vlib_num_workers ()) + 1; + if (PREDICT_FALSE (sa2->thread_index == 0xFFFF)) + sa2->thread_index = (sa2_index % vlib_num_workers ()) + 1; + if (PREDICT_FALSE (sa3->thread_index == 0xFFFF)) + sa3->thread_index = (sa3_index % vlib_num_workers ()) + 1; + + vnet_buffer (b[0])->ipsec.thread_index = sa0->thread_index; + vnet_buffer (b[1])->ipsec.thread_index = sa1->thread_index; + vnet_buffer (b[2])->ipsec.thread_index = sa2->thread_index; + vnet_buffer (b[3])->ipsec.thread_index = sa3->thread_index; + + next[0] = thread0_next + sa0->thread_index; + next[1] = thread0_next + sa1->thread_index; + next[2] = thread0_next + sa2->thread_index; + next[3] = thread0_next + sa3->thread_index; + + next += 4; + b += 4; + n_left -= 4; + } + + current_sa0_index = ~0; + while (n_left > 0) + { + sa0_index = oct_ipsec_sa_index_get (b[0], 1); + + if (sa0_index != current_sa0_index) + { + sa0 = ipsec_sa_get (sa0_index); + current_sa0_index = sa0_index; + } + + if (PREDICT_FALSE (0XFFFF == sa0->thread_index)) + { + /* + * If this is the first packet to use this SA, claim the SA + * for this thread. Use atomic operation as this could happen + * simultaneously on another thread + */ + clib_atomic_cmp_and_swap (&sa0->thread_index, ~0, + ipsec_sa_assign_thread (vm->thread_index)); + } + + vnet_buffer (b[0])->ipsec.thread_index = sa0->thread_index; + next[0] = thread0_next + sa0->thread_index; + + next++; + b++; + n_left--; + } + + if (PREDICT_FALSE (node->flags & VLIB_NODE_FLAG_TRACE)) + { + n_left = frame->n_vectors; + next = nexts; + b = bufs; + while (n_left > 0) + { + if (PREDICT_FALSE (b[0]->flags & VLIB_BUFFER_IS_TRACED)) + { + oct_esp_encrypt_tun_add_trace (vm, node, frame, b[0], next[0]); + } + + b += 1; + n_left--; + next++; + } + } + + vlib_buffer_enqueue_to_next (vm, node, from, nexts, frame->n_vectors); + + return frame->n_vectors; +} + +/** + * @brief OCTEON ESP4 encryption tunnel node. + * @node oct-esp4-encrypt-tun + * + * This is the OCTEON ESP4 encryption tunnel node. + * + * @param vm vlib_main_t corresponding to the current thread + * @param node vlib_node_runtime_t + * @param frame vlib_frame_t + */ +/* clang-format off */ +VLIB_NODE_FN (oct_esp4_encrypt_tun_node) (vlib_main_t *vm, + vlib_node_runtime_t *node, + vlib_frame_t *frame) +{ + return oct_esp_encrypt_tun ( + vm, node, frame, 0); +} + +VLIB_REGISTER_NODE (oct_esp4_encrypt_tun_node) = { + .name = "oct-esp4-encrypt-tun", + .vector_size = sizeof (u32), + .format_trace = format_oct_esp_encrypt_trace, + .type = VLIB_NODE_TYPE_INTERNAL, + .n_next_nodes = OCT_ESP_ENCRYPT_TUN_N_NEXT, + .next_nodes = { +#define _(next, node) [OCT_ESP_ENCRYPT_TUN_NEXT_##next] = node, + foreach_oct_esp_encrypt_tun_next +#undef _ +#define _(core) \ + OCT_ESP_ENCRYPT_TUN_PREP_NODES (oct-esp4-encrypt-tun-prep, core) + foreach_oct_esp_handoff_core +#undef _ + }, + +}; +/* clang-format on */ + +/** + * @brief OCT ESP6 encryption tunnel node. + * @node oct-esp6-encrypt-tun + * + * This is the ONP ESP6 encryption tunnel node. + * + * @param vm vlib_main_t corresponding to the current thread + * @param node vlib_node_runtime_t + * @param frame vlib_frame_t + */ +/* clang-format off */ +VLIB_NODE_FN (oct_esp6_encrypt_tun_node) (vlib_main_t *vm, + vlib_node_runtime_t *node, + vlib_frame_t *frame) +{ + return oct_esp_encrypt_tun (vm, node, frame, 1); +} + +VLIB_REGISTER_NODE (oct_esp6_encrypt_tun_node) = { + .name = "oct-esp6-encrypt-tun", + .vector_size = sizeof (u32), + .type = VLIB_NODE_TYPE_INTERNAL, + .sibling_of = "oct-esp4-encrypt-tun", +}; +/* clang-format on */ + +static_always_inline i32 +oct_ipsec_rlen_get (oct_ipsec_encap_len_t *encap, uint32_t plen) +{ + uint32_t enc_payload_len; + + enc_payload_len = + round_pow2 (plen + encap->roundup_len, encap->roundup_byte); + + return encap->partial_len + enc_payload_len; +} + +static_always_inline u32 +oct_ipsec_esp_add_footer_and_icv (oct_ipsec_encap_len_t *encap, u32 rlen) +{ + /* plain_text len + pad_bytes + ESP_footer size + icv_len */ + return rlen + encap->icv_len - encap->partial_len; +} + +static_always_inline uword +oct_esp_prep_to_core (vlib_main_t *vm, vlib_node_runtime_t *node, + vlib_frame_t *frame, u16 core_id, const int is_ipv6) +{ + oct_device_t *od; + u32 *from = vlib_frame_vector_args (frame); + u32 n_left = frame->n_vectors; + vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b = bufs; + u16 buffer_data_size = vlib_buffer_get_default_data_size (vm); + u32 sa_index, current_sa_index = ~0, rlen, pkt_len; + oct_ipsec_main_t *im = &oct_ipsec_main; + oct_ipsec_outbound_pkt_meta_t *meta; + oct_ipsec_session_t *sess = NULL; + u16 sa_bytes; + u64 dptr; + + vlib_get_buffers (vm, from, b, frame->n_vectors); + + while (n_left > 0) + { + sa_index = vnet_buffer (b[0])->ipsec.sad_index; + if (sa_index != current_sa_index) + { + sess = pool_elt_at_index (im->inline_ipsec_sessions, sa_index); + if (!sess->inst.w7.s.cptr) + { + od = oct_ipsec_get_oct_device_from_outb_sa (sa_index); + sess->inst.w7.s.cptr = (u64) sess->out_sa[od->nix_idx]; + } + current_sa_index = sa_index; + ALWAYS_ASSERT (current_sa_index < + vec_len (im->inline_ipsec_sessions)); + } + + vnet_buffer (b[0])->oflags |= VNET_BUFFER_OFFLOAD_F_IPSEC_OFFLOAD; + vnet_buffer (b[0])->ipsec.sad_index = current_sa_index; + + /* + * External header buffer is used to store + * the cn10k_ipsec_outbound_pkt_meta_t + */ + + meta = + (oct_ipsec_outbound_pkt_meta_t *) OCT_EXT_HDR_FROM_VLIB_BUFFER (b[0]); + + clib_memset (meta, 0, sizeof (oct_ipsec_outbound_pkt_meta_t)); + + meta->res.cn10k.compcode = CPT_COMP_NOT_DONE; + + pkt_len = b[0]->current_length; + rlen = oct_ipsec_rlen_get (&sess->encap, pkt_len); + meta->core_id = core_id; + + if (is_ipv6) + meta->is_ip6 = 1; + + /* Populate CPT instruction template */ + meta->inst.res_addr = (u64) &meta->res; + meta->inst.w2.u64 = sess->inst.w2.u64; + /* WQE must be aligned on an 64-bit / 8 byte boundary */ + ASSERT (((uintptr_t) b[0] & 0x7ULL) == 0); + meta->inst.w3.u64 = (uintptr_t) b[0]; + /* Enable queue ordering */ + meta->inst.w3.u64 |= 0x1ULL; + + if ((b[0]->flags & VLIB_BUFFER_NEXT_PRESENT) || + (rlen > buffer_data_size)) + { + u16 total_length; + total_length = b[0]->current_length + + b[0]->total_length_not_including_first_buffer; + rlen = oct_ipsec_rlen_get (&sess->encap, total_length); + sa_bytes = oct_ipsec_esp_add_footer_and_icv (&sess->encap, rlen); + meta->sa_bytes = sa_bytes; + + meta->dlen_adj = rlen - total_length; + meta->is_sg_mode = 1; + meta->inst.w4.u64 = sess->inst.w4.u64; + } + else + { + dptr = (u64) vlib_buffer_get_current (b[0]); + sa_bytes = oct_ipsec_esp_add_footer_and_icv (&sess->encap, rlen); + meta->sa_bytes = sa_bytes; + meta->dlen_adj = rlen - pkt_len; + + /* Set w0 nixtx_offset */ + meta->inst.w0.u64 |= + (((int64_t) meta->nixtx - (int64_t) dptr) & 0xFFFFF) << 32; + + /* + * Set nixtx length to 2 dwords. + * NIXTXL + 1 represents the length in dwords + */ + meta->inst.w0.u64 |= 1; + meta->inst.w4.u64 = sess->inst.w4.u64 | pkt_len; + meta->inst.dptr = dptr; + meta->inst.rptr = dptr; + } + + meta->inst.w7.u64 = sess->inst.w7.u64; + + b++; + n_left--; + } + + if (PREDICT_FALSE (node->flags & VLIB_NODE_FLAG_TRACE)) + { + n_left = frame->n_vectors; + b = bufs; + + while (n_left) + { + if (PREDICT_FALSE (b[0]->flags & VLIB_BUFFER_IS_TRACED)) + { + oct_esp_encrypt_tun_add_trace ( + vm, node, frame, b[0], + OCT_ESP_ENCRYPT_TUN_NEXT_ADJ_MIDCHAIN_TX); + } + n_left--; + b++; + } + } + + vlib_buffer_enqueue_to_single_next (vm, node, from, + OCT_ESP_ENCRYPT_TUN_NEXT_ADJ_MIDCHAIN_TX, + frame->n_vectors); + + return frame->n_vectors; +} + +/** + * @brief OCTEON ESP4 encryption tunnel preparation node. + * @node oct-esp4-encrypt-tun-prep + * + * This is the OCTEON ESP4 encryption tunnel preparation node. + * + * @param vm vlib_main_t corresponding to the current thread + * @param node vlib_node_runtime_t + * @param from_frame vlib_frame_t + */ +/* clang-format off */ +#define OCT_DEFINE_ESP4_TUN_PREP_NODE(core) \ + VLIB_NODE_FN (oct_esp4_encrypt_tun_prep##core##_node) \ + (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame) \ + { \ + return oct_esp_prep_to_core (vm, node, frame, core, \ + 0 /* is_ipv6 */); \ + } \ + \ + VLIB_REGISTER_NODE (oct_esp4_encrypt_tun_prep##core##_node) = { \ + .name = "oct-esp4-encrypt-tun-prep"#core, \ + .type = VLIB_NODE_TYPE_INTERNAL, \ + .format_trace = format_oct_esp_encrypt_trace, \ + .vector_size = sizeof (u32), \ + .sibling_of = "oct-esp4-encrypt-tun", \ + }; + +#define _(core) OCT_DEFINE_ESP4_TUN_PREP_NODE (core) +foreach_oct_esp_handoff_core; +#undef _ +/* clang-format on */ + +/** + * @brief OCTEON ESP6 encryption tunnel preparation node. + * @node oct-esp6-encrypt-tun-prep + * + * This is the OCTEON ESP6 encryption tunnel preparation node. + * + * @param vm vlib_main_t corresponding to the current thread + * @param node vlib_node_runtime_t + * @param from_frame vlib_frame_t + */ +/* clang-format off */ +#define OCT_DEFINE_ESP6_TUN_PREP_NODE(core) \ + VLIB_NODE_FN (oct_esp6_encrypt_tun_prep##core##_node) \ + (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame) \ + { \ + return oct_esp_prep_to_core (vm, node, frame, core, \ + 1 /* is_ipv6 */); \ + } \ + \ + VLIB_REGISTER_NODE (oct_esp6_encrypt_tun_prep##core##_node) = { \ + .name = "oct-esp6-encrypt-tun-prep"#core, \ + .type = VLIB_NODE_TYPE_INTERNAL, \ + .format_trace = format_oct_esp_encrypt_trace, \ + .vector_size = sizeof (u32), \ + .sibling_of = "oct-esp6-encrypt-tun", \ + }; + +#define _(core) OCT_DEFINE_ESP6_TUN_PREP_NODE (core) +foreach_oct_esp_handoff_core; +#undef _ +/* clang-format on */ diff --git a/src/plugins/dev_octeon/init.c b/src/plugins/dev_octeon/init.c index 5633694509..5543db8e92 100644 --- a/src/plugins/dev_octeon/init.c +++ b/src/plugins/dev_octeon/init.c @@ -52,6 +52,12 @@ vnet_dev_node_t oct_tx_node = { .n_error_counters = ARRAY_LEN (oct_tx_node_counters), }; +vnet_dev_node_t oct_tx_ipsec_tm_node = { + .format_trace = format_oct_tx_trace, + .error_counters = oct_tx_node_counters, + .n_error_counters = ARRAY_LEN (oct_tx_node_counters), +}; + static struct { u16 device_id; @@ -169,6 +175,7 @@ static vnet_dev_rv_t oct_init_nix (vlib_main_t *vm, vnet_dev_t *dev) { oct_main_t *om = &oct_main; + oct_ipsec_main_t *oim = &oct_ipsec_main; oct_inl_dev_main_t *oidm = &oct_inl_dev_main; oct_device_t *cd = vnet_dev_get_data (dev), **oct_dev = 0; u8 mac_addr[6]; @@ -189,10 +196,6 @@ oct_init_nix (vlib_main_t *vm, vnet_dev_t *dev) if ((rrv = roc_nix_npc_mac_addr_get (cd->nix, mac_addr))) return cnx_return_roc_err (dev, rrv, "roc_nix_npc_mac_addr_get"); - if (oidm->inl_dev) - if ((rv = oct_init_nix_inline_ipsec (vm, oidm->vdev, dev))) - return rv; - vnet_dev_port_add_args_t port_add_args = { .port = { .attr = { @@ -257,6 +260,19 @@ oct_init_nix (vlib_main_t *vm, vnet_dev_t *dev) }, }; + if (oidm->inl_dev) + { + if (oim->inline_ipsec_sessions) + { + log_err (dev, + "device attach not allowed after any IPsec SA addition"); + return VNET_DEV_ERR_NOT_SUPPORTED; + } + if ((rv = oct_init_nix_inline_ipsec (vm, oidm->vdev, dev))) + return rv; + port_add_args.tx_node = &oct_tx_ipsec_tm_node; + } + vnet_dev_set_hw_addr_eth_mac (&port_add_args.port.attr.hw_addr, mac_addr); log_info (dev, "MAC address is %U", format_ethernet_address, mac_addr); @@ -265,7 +281,9 @@ oct_init_nix (vlib_main_t *vm, vnet_dev_t *dev) return rv; pool_get (om->oct_dev, oct_dev); - oct_dev = vnet_dev_get_data (dev); + oct_dev[0] = vnet_dev_get_data (dev); + oct_dev[0]->nix_idx = oct_dev - om->oct_dev; + log_info (dev, "veclen %d %d", vec_len (om->oct_dev), oct_dev[0]->nix_idx); return VNET_DEV_OK; } @@ -550,6 +568,7 @@ oct_early_config (vlib_main_t *vm, unformat_input_t *input) oct_inl_dev_main.in_min_spi = 0; oct_inl_dev_main.in_max_spi = 8192; + oct_inl_dev_main.out_max_sa = 8192; if (!unformat_user (input, unformat_line_input, line_input)) return 0; @@ -564,6 +583,9 @@ oct_early_config (vlib_main_t *vm, unformat_input_t *input) else if (unformat (line_input, "ipsec_in_max_spi %u", &oct_inl_dev_main.in_max_spi)) ; + else if (unformat (line_input, "ipsec_out_max_sa %u", + &oct_inl_dev_main.out_max_sa)) + ; else { error = clib_error_return (0, "unknown input '%U'", @@ -582,3 +604,4 @@ oct_early_config (vlib_main_t *vm, unformat_input_t *input) } VLIB_EARLY_CONFIG_FUNCTION (oct_early_config, "dev_octeon"); +VLIB_BUFFER_SET_EXT_HDR_SIZE (OCT_EXT_HDR_SIZE); diff --git a/src/plugins/dev_octeon/ipsec.c b/src/plugins/dev_octeon/ipsec.c index 5053b7ec27..6d70db1f30 100644 --- a/src/plugins/dev_octeon/ipsec.c +++ b/src/plugins/dev_octeon/ipsec.c @@ -7,6 +7,8 @@ #include #include #include +#include +#include #define OCT_NIX_INL_META_POOL_NAME "OCT_NIX_INL_META_POOL" @@ -70,7 +72,6 @@ oct_ipsec_crypto_inst_w7_get (void *sa) w7.u64 = 0; w7.s.egrp = ROC_CPT_DFLT_ENG_GRP_SE_IE; w7.s.ctx_val = 1; - w7.s.cptr = (u64) sa; return w7.u64; } @@ -380,6 +381,201 @@ oct_ipsec_inb_session_update (oct_ipsec_session_t *sess, ipsec_sa_t *sa) return 0; } +int +oct_ipsec_outb_sa_idx_get (oct_device_t *od, u32 *index, u32 spi) +{ + u32 pos, idx; + u64 slab; + int rc; + + if (!od->outb.sa_bmap) + return -ENOTSUP; + + pos = 0; + slab = 0; + /* Scan from the beginning */ + plt_bitmap_scan_init (od->outb.sa_bmap); + + /* Scan bitmap to get the free sa index */ + rc = plt_bitmap_scan (od->outb.sa_bmap, &pos, &slab); + /* Empty bitmap */ + if (rc == 0) + { + plt_err ("Outbound SA' exhausted, use 'ipsec_out_max_sa' " + "devargs to increase"); + return -ERANGE; + } + + /* Get free SA index */ + idx = pos + (slab ? plt_ctz64 (slab) : 0); + + plt_bitmap_clear (od->outb.sa_bmap, idx); + *index = idx; + return 0; +} + +void * +oct_ipsec_get_oct_device_from_outb_sa (u32 sa_index) +{ + ipsec_sa_t *sa = ipsec_sa_get (sa_index); + vnet_main_t *vnm = vnet_get_main (); + vnet_sw_interface_t *si; + u32 sw_if_index; + vnet_dev_port_t *port; + + sw_if_index = + fib_entry_get_resolving_interface (sa->tunnel.t_fib_entry_index); + si = vnet_get_sw_interface_or_null (vnm, sw_if_index); + port = vnet_dev_get_port_from_hw_if_index (si->hw_if_index); + + return (oct_device_t *) vnet_dev_get_data (port->dev); +} + +static_always_inline i32 +oct_ipsec_outb_session_update (oct_ipsec_session_t *sess, ipsec_sa_t *sa) +{ + oct_main_t *om = &oct_main; + union roc_ot_ipsec_outb_param1 param1; + struct roc_ot_ipsec_outb_sa *out_sa; + oct_ipsec_outb_sa_priv_data_t *outb_priv; + union roc_ot_ipsec_sa_word2 w2; + union cpt_inst_w4 inst_w4; + u32 sa_idx; + u64 *ipv6_addr; + size_t offset; + int rv = 0, i = 0; + + vec_validate_aligned (sess->out_sa, vec_len (om->oct_dev), + CLIB_CACHE_LINE_BYTES); + + pool_foreach_pointer (oct_dev, om->oct_dev) + { + /* Alloc an sa index */ + rv = oct_ipsec_outb_sa_idx_get (oct_dev, &sa_idx, sa->spi); + if (rv) + return rv; + + out_sa = sess->out_sa[i] = + roc_nix_inl_ot_ipsec_outb_sa (oct_dev->outb.sa_base, sa_idx); + + outb_priv = roc_nix_inl_ot_ipsec_outb_sa_sw_rsvd (out_sa); + outb_priv->sa_idx = sa_idx; + + roc_ot_ipsec_outb_sa_init (out_sa); + + w2.u64 = 0; + rv = oct_ipsec_sa_common_param_fill (&w2, out_sa->cipher_key, + out_sa->iv.s.salt, + out_sa->hmac_opad_ipad, sa); + if (rv) + return rv; + + /* Set direction and enable ESN (if needed) */ + w2.s.dir = ROC_IE_SA_DIR_OUTBOUND; + if (ipsec_sa_is_set_USE_ESN (sa)) + out_sa->w0.s.esn_en = 1; + + /* Configure tunnel header generation */ + if (ipsec_sa_is_set_IS_TUNNEL (sa)) + { + if (ipsec_sa_is_set_IS_TUNNEL_V6 (sa)) + { + w2.s.outer_ip_ver = ROC_IE_SA_IP_VERSION_6; + + clib_memcpy (&out_sa->outer_hdr.ipv6.src_addr, + &sa->tunnel.t_src.ip.ip6, sizeof (ip6_address_t)); + clib_memcpy (&out_sa->outer_hdr.ipv6.dst_addr, + &sa->tunnel.t_dst.ip.ip6, sizeof (ip6_address_t)); + + /* Convert host to network byte order of ipv6 address */ + ipv6_addr = (u64 *) &out_sa->outer_hdr.ipv6.src_addr; + *ipv6_addr = clib_host_to_net_u64 (*ipv6_addr); + ipv6_addr++; + *ipv6_addr = clib_host_to_net_u64 (*ipv6_addr); + + ipv6_addr = (u64 *) &out_sa->outer_hdr.ipv6.dst_addr; + *ipv6_addr = clib_host_to_net_u64 (*ipv6_addr); + ipv6_addr++; + *ipv6_addr = clib_host_to_net_u64 (*ipv6_addr); + } + else + { + w2.s.outer_ip_ver = ROC_IE_SA_IP_VERSION_4; + out_sa->outer_hdr.ipv4.src_addr = + clib_host_to_net_u32 (sa->tunnel.t_src.ip.ip4.as_u32); + out_sa->outer_hdr.ipv4.dst_addr = + clib_host_to_net_u32 (sa->tunnel.t_dst.ip.ip4.as_u32); + } + } + + offset = offsetof (struct roc_ot_ipsec_outb_sa, ctx); + out_sa->w0.s.hw_ctx_off = offset / 8; + out_sa->w0.s.ctx_push_size = out_sa->w0.s.hw_ctx_off + 1; + /* Set context size, in number of 128B units following the first 128B */ + out_sa->w0.s.ctx_size = (round_pow2 (offset, 128) >> 7) - 1; + out_sa->w0.s.ctx_hdr_size = 1; + out_sa->w0.s.aop_valid = 1; + + out_sa->w2.u64 = w2.u64; + + if (ipsec_sa_is_set_IS_TUNNEL (sa)) + { + if (sa->tunnel.t_encap_decap_flags & + TUNNEL_ENCAP_DECAP_FLAG_ENCAP_COPY_DF) + out_sa->w2.s.ipv4_df_src_or_ipv6_flw_lbl_src = + ROC_IE_OT_SA_COPY_FROM_INNER_IP_HDR; + if (!sa->tunnel.t_dscp) + out_sa->w2.s.dscp_src = ROC_IE_OT_SA_COPY_FROM_INNER_IP_HDR; + else + { + out_sa->w2.s.dscp_src = ROC_IE_OT_SA_COPY_FROM_SA; + out_sa->w10.s.dscp = sa->tunnel.t_dscp; + } + } + + out_sa->w2.s.ipid_gen = 1; + out_sa->w2.s.iv_src = ROC_IE_OT_SA_IV_SRC_FROM_SA; + out_sa->w2.s.valid = 1; + + asm volatile("dmb oshst" ::: "memory"); + + oct_ipsec_sa_len_precalc (sa, &sess->encap); + + oct_ipsec_common_inst_param_fill (out_sa, sess); + + /* Populate word4 in CPT instruction template */ + inst_w4.u64 = 0; + inst_w4.s.opcode_major = ROC_IE_OT_MAJOR_OP_PROCESS_OUTBOUND_IPSEC; + param1.u16 = 0; + if (sa->tunnel.t_hop_limit) + param1.s.ttl_or_hop_limit = 1; + + /* Enable IP checksum computation by default */ + param1.s.ip_csum_disable = ROC_IE_OT_SA_INNER_PKT_IP_CSUM_ENABLE; + /* Enable L4 checksum computation by default */ + param1.s.l4_csum_disable = ROC_IE_OT_SA_INNER_PKT_L4_CSUM_ENABLE; + + inst_w4.s.param1 = param1.u16; + sess->inst.w4.u64 = inst_w4.u64; + if (ipsec_sa_is_set_UDP_ENCAP (sa)) + { + out_sa->w10.s.udp_dst_port = 4500; + out_sa->w10.s.udp_src_port = 4500; + } + + rv = roc_nix_inl_ctx_write (oct_dev->nix, out_sa, out_sa, false, + sizeof (struct roc_ot_ipsec_outb_sa)); + if (rv) + { + clib_warning ("roc_nix_inl_ctx_write failed with '%s' error", + roc_error_msg_get (rv)); + return -1; + } + i++; + } + return 0; +} + static i32 oct_ipsec_session_create (u32 sa_index) { @@ -396,11 +592,12 @@ oct_ipsec_session_create (u32 sa_index) ASSERT (sa_index == sess_index); if (sa->flags & IPSEC_SA_FLAG_IS_INBOUND) - { - rv = oct_ipsec_inb_session_update (session, sa); - if (rv) - return rv; - } + rv = oct_ipsec_inb_session_update (session, sa); + else + rv = oct_ipsec_outb_session_update (session, sa); + + if (rv) + return rv; /* Initialize the ITF details in ipsec_session for tunnel SAs */ if (ipsec_sa_is_set_IS_TUNNEL (sa)) @@ -411,12 +608,13 @@ oct_ipsec_session_create (u32 sa_index) static i32 oct_ipsec_session_destroy (u32 sa_index) { + oct_main_t *om = &oct_main; oct_ipsec_main_t *oim = &oct_ipsec_main; ipsec_sa_t *sa = ipsec_sa_get (sa_index); oct_ipsec_session_t *session = NULL; struct roc_ot_ipsec_inb_sa *roc_sa; void *sa_dptr = NULL; - int rv; + int rv, i = 0; session = pool_elt_at_index (oim->inline_ipsec_sessions, sa_index); if (pool_is_free (oim->inline_ipsec_sessions, session)) @@ -448,6 +646,28 @@ oct_ipsec_session_destroy (u32 sa_index) plt_free (sa_dptr); } } + else + { + pool_foreach_pointer (oct_dev, om->oct_dev) + { + sa_dptr = plt_zmalloc (sizeof (struct roc_ot_ipsec_outb_sa), 8); + if (sa_dptr != NULL) + { + roc_ot_ipsec_outb_sa_init (sa_dptr); + rv = roc_nix_inl_ctx_write ( + oct_dev->nix, sa_dptr, session->out_sa[i], false, + sizeof (struct roc_ot_ipsec_outb_sa)); + if (rv) + { + clib_warning ( + "Could not write inline outbound session to hardware"); + return rv; + } + plt_free (sa_dptr); + } + i++; + } + } clib_memset (session, 0, sizeof (oct_ipsec_session_t)); return 0; @@ -552,17 +772,18 @@ oct_init_ipsec_backend (vlib_main_t *vm, vnet_dev_t *dev) u32 idx; idx = ipsec_register_esp_backend ( - vm, im, "octeon backend", "esp4-encrypt", "esp4-encrypt-tun", - "esp4-decrypt", "esp4-decrypt-tun", "esp6-encrypt", "esp6-encrypt-tun", + vm, im, "octeon backend", "esp4-encrypt", "oct-esp4-encrypt-tun", + "esp4-decrypt", "esp4-decrypt-tun", "esp6-encrypt", "oct-esp6-encrypt-tun", "esp6-decrypt", "esp6-decrypt-tun", "esp-mpls-encrypt-tun", oct_ipsec_check_support, oct_add_del_session); rv = ipsec_select_esp_backend (im, idx); if (rv) { - log_err (dev, "IPsec ESP backend selection failed"); + log_err (dev, "OCTEON IPsec ESP backend selection failed"); return VNET_DEV_ERR_INTERNAL; } + return VNET_DEV_OK; } @@ -579,9 +800,9 @@ oct_ipsec_inl_dev_inb_cfg (vlib_main_t *vm, vnet_dev_t *dev, if ((rrv = roc_nix_inl_inb_init (cd->nix))) { - log_err (dev, "roc_nix_inl_inb_init: %s [%d]", roc_error_msg_get (rrv), - rrv); - return VNET_DEV_ERR_UNSUPPORTED_DEVICE; + log_err (dev, "roc_nix_inl_inb_init failed - ROC error %s [%d]", + roc_error_msg_get (rrv), rrv); + return VNET_DEV_ERR_INTERNAL; } roc_nix_inb_mode_set (cd->nix, true); @@ -658,6 +879,69 @@ oct_pool_inl_meta_pool_cb (u64 *aura_handle, uintptr_t *mpool, u32 buf_sz, return 0; } + +vnet_dev_rv_t +oct_ipsec_inl_dev_outb_cfg (vnet_dev_t *dev, oct_inl_dev_cfg_t *inl_dev_cfg) +{ + oct_inl_dev_main_t *inl_dev_main = &oct_inl_dev_main; + oct_device_t *cd = vnet_dev_get_data (dev); + struct roc_nix *nix = cd->nix; + struct roc_cpt_lf *cpt_lf; + u64 cpt_io_addr; + struct plt_bitmap *bmap; + size_t bmap_sz; + void *mem; + int rrv, i; + + nix->outb_nb_desc = inl_dev_cfg->outb_nb_desc = 8192; + nix->outb_nb_crypto_qs = inl_dev_cfg->outb_nb_crypto_qs = 1; + nix->ipsec_out_max_sa = cd->outb.max_sa = inl_dev_main->out_max_sa; + nix->ipsec_out_sso_pffunc = false; + + rrv = roc_nix_inl_outb_init (nix); + if (rrv) + { + log_err (dev, "roc_nix_inl_outb_init failed - ROC error '%s [%d]", + roc_error_msg_get (rrv), rrv); + return VNET_DEV_ERR_INTERNAL; + } + + cpt_lf = roc_nix_inl_outb_lf_base_get (nix); + + cpt_io_addr = cpt_lf->io_addr; + cpt_io_addr |= (ROC_CN10K_CPT_INST_DW_M1 << 4); + cd->cpt_io_addr = cpt_io_addr; + + bmap_sz = plt_bitmap_get_memory_footprint (cd->outb.max_sa); + mem = plt_zmalloc (bmap_sz, PLT_CACHE_LINE_SIZE); + if (mem == NULL) + { + log_err (dev, "Outbound SA bmap alloc failed"); + roc_nix_inl_outb_fini (nix); + + return VNET_DEV_ERR_DMA_MEM_ALLOC_FAIL; + } + + bmap = plt_bitmap_init (cd->outb.max_sa, mem, bmap_sz); + if (!bmap) + { + log_err (dev, "Outbound SA bmap init failed"); + roc_nix_inl_outb_fini (nix); + plt_free (mem); + + return VNET_DEV_ERR_DMA_MEM_ALLOC_FAIL; + } + + for (i = 0; i < cd->outb.max_sa; i++) + plt_bitmap_set (bmap, i); + + cd->outb.sa_base = roc_nix_inl_outb_sa_base_get (nix); + cd->outb.sa_bmap_mem = mem; + cd->outb.sa_bmap = bmap; + + return VNET_DEV_OK; +} + vnet_dev_rv_t oct_early_init_inline_ipsec (vlib_main_t *vm, vnet_dev_t *dev) { @@ -698,5 +982,8 @@ oct_init_nix_inline_ipsec (vlib_main_t *vm, vnet_dev_t *inl_dev, if ((rv = oct_ipsec_inl_dev_inb_cfg (vm, dev, &inl_dev_cfg))) return rv; + if ((rv = oct_ipsec_inl_dev_outb_cfg (dev, &inl_dev_cfg))) + return rv; + return VNET_DEV_OK; } diff --git a/src/plugins/dev_octeon/ipsec.h b/src/plugins/dev_octeon/ipsec.h index 483a67f952..02c07375c0 100644 --- a/src/plugins/dev_octeon/ipsec.h +++ b/src/plugins/dev_octeon/ipsec.h @@ -7,8 +7,9 @@ #ifndef _OCTEON_IPSEC_H_ #define _OCTEON_IPSEC_H_ -#define OCT_ROC_SALT_LEN 4 -#define OCT_IPSEC_MAX_SESSION 3600 +#define OCT_ROC_SALT_LEN 4 +#define OCT_EXT_HDR_FROM_VLIB_BUFFER(x) \ + (((oct_ipsec_outbound_pkt_meta_t *) (x)) - 1) #define foreach_octeon10_ipsec_ucc \ _ (SUCCESS, success, INFO, "Packet successfully processed") \ @@ -67,6 +68,21 @@ _ (SUCCESS_PKT_UDP_ZEROCSUM, success_pkt_udp_zerocsum, INFO, \ "Zero UDP checksum") +typedef struct +{ + CLIB_CACHE_LINE_ALIGN_MARK (c0); + struct cpt_inst_s inst; + union cpt_res_s res; + u16 dlen_adj; + u16 sa_bytes; + u8 core_id; + u8 is_ip6; + bool is_sg_mode; + CLIB_CACHE_LINE_ALIGN_MARK (c2); + u64 nixtx[2]; + u8 sg_buffer[128]; +} oct_ipsec_outbound_pkt_meta_t; + typedef struct { uint8_t partial_len; @@ -81,11 +97,17 @@ typedef struct u64 user_data; } oct_ipsec_inb_sa_priv_data_t; +typedef struct +{ + /* SA index */ + u32 sa_idx; +} oct_ipsec_outb_sa_priv_data_t; + typedef struct { CLIB_CACHE_LINE_ALIGN_MARK (cacheline0); /* Outbound SA */ - struct roc_ot_ipsec_outb_sa out_sa; + struct roc_ot_ipsec_outb_sa **out_sa; CLIB_CACHE_LINE_ALIGN_MARK (cacheline1); struct cpt_inst_s inst; uint16_t iv_offset; @@ -104,6 +126,8 @@ typedef struct typedef struct { + u32 outb_nb_desc; + u16 outb_nb_crypto_qs; } oct_inl_dev_cfg_t; typedef struct @@ -117,6 +141,7 @@ typedef struct u8 is_inl_ipsec_flow_enabled; u32 in_min_spi; u32 in_max_spi; + u32 out_max_sa; } oct_inl_dev_main_t; extern oct_ipsec_main_t oct_ipsec_main; @@ -127,6 +152,7 @@ vnet_dev_rv_t oct_init_ipsec_backend (vlib_main_t *vm, vnet_dev_t *dev); vnet_dev_rv_t oct_early_init_inline_ipsec (vlib_main_t *vm, vnet_dev_t *dev); vnet_dev_rv_t oct_init_nix_inline_ipsec (vlib_main_t *vm, vnet_dev_t *inl_dev, vnet_dev_t *dev); +void *oct_ipsec_get_oct_device_from_outb_sa (u32 sa_index); clib_error_t *oct_inl_inb_ipsec_flow_enable (void); diff --git a/src/plugins/dev_octeon/octeon.h b/src/plugins/dev_octeon/octeon.h index 905e10253b..e5498ec4da 100644 --- a/src/plugins/dev_octeon/octeon.h +++ b/src/plugins/dev_octeon/octeon.h @@ -22,6 +22,8 @@ #include #include +#define OCT_EXT_HDR_SIZE \ + PLT_ALIGN (sizeof (oct_ipsec_outbound_pkt_meta_t), ROC_ALIGN) #define OCT_NPA_MAX_POOLS 8192 #define OCT_BATCH_ALLOC_IOVA0_MASK 0xFFFFFFFFFFFFFF80 @@ -51,16 +53,6 @@ typedef enum OCT_DEVICE_TYPE_RVU_INL_VF, } __clib_packed oct_device_type_t; -typedef struct -{ - oct_device_type_t type; - u8 nix_initialized : 1; - u8 status : 1; - u8 full_duplex : 1; - u32 speed; - struct plt_pci_device plt_pci_dev; - struct roc_nix *nix; -} oct_device_t; typedef struct { @@ -110,6 +102,19 @@ typedef union STATIC_ASSERT_SIZEOF (oct_npa_batch_alloc_cl128_t, 128); +struct oct_outb_sa_data +{ + /* SA Bitmap */ + struct plt_bitmap *sa_bmap; + /* SA bitmap memory */ + void *sa_bmap_mem; + + /* SA base */ + u64 sa_base; + + u16 max_sa; +}; + typedef struct { u8 sq_initialized : 1; @@ -124,8 +129,27 @@ typedef struct u8 ba_num_cl; CLIB_CACHE_LINE_ALIGN_MARK (data0); struct roc_nix_sq sq; + i32 cached_pkts; } oct_txq_t; +typedef struct +{ + oct_device_type_t type; + u16 nix_idx; + u8 nix_initialized : 1; + u8 status : 1; + u8 full_duplex : 1; + u32 speed; + struct plt_pci_device plt_pci_dev; + struct roc_nix *nix; + + u32 cached_cpt_pkts; + u64 cpt_io_addr; + oct_txq_t **ctqs; + + struct oct_outb_sa_data outb; +} oct_device_t; + typedef struct { oct_device_t **oct_dev; diff --git a/src/plugins/dev_octeon/port.c b/src/plugins/dev_octeon/port.c index 7274fee162..e38a24b536 100644 --- a/src/plugins/dev_octeon/port.c +++ b/src/plugins/dev_octeon/port.c @@ -234,6 +234,14 @@ oct_port_init (vlib_main_t *vm, vnet_dev_port_t *port) return rv; } + cd->ctqs = clib_mem_alloc_aligned ( + sizeof (oct_txq_t *) * port->intf.num_tx_queues, CLIB_CACHE_LINE_BYTES); + if (!cd->ctqs) + { + oct_port_deinit (vm, port); + return VNET_DEV_ERR_INTERNAL; + } + foreach_vnet_dev_port_tx_queue (q, port) if (q->enabled) if ((rv = oct_txq_init (vm, q))) diff --git a/src/plugins/dev_octeon/queue.c b/src/plugins/dev_octeon/queue.c index b9f19714c9..55b14fb01b 100644 --- a/src/plugins/dev_octeon/queue.c +++ b/src/plugins/dev_octeon/queue.c @@ -96,9 +96,9 @@ oct_rxq_init (vlib_main_t *vm, vnet_dev_rx_queue_t *rxq, u32 total_sz) vlib_get_buffer_pool (vm, vnet_dev_get_rx_queue_buffer_pool_index (rxq)); struct roc_nix *nix = cd->nix; int rrv; - struct npa_aura_s aura = {}; - struct npa_pool_s npapool = { .nat_align = 1 }; + struct npa_pool_s npapool = { .nat_align = 1, + .buf_offset = OCT_EXT_HDR_SIZE / ROC_ALIGN }; if (!om->use_single_rx_aura || !om->aura_handle) { @@ -257,6 +257,7 @@ oct_txq_init (vlib_main_t *vm, vnet_dev_tx_queue_t *txq) ctq->io_addr = ctq->sq.io_addr & ~0x7fULL; ctq->lmt_addr = ctq->sq.lmt_addr; + cd->ctqs[ctq->sq.qid] = ctq; return VNET_DEV_OK; } diff --git a/src/plugins/dev_octeon/tx_node.c b/src/plugins/dev_octeon/tx_node.c index f42f18d989..aa804a3c34 100644 --- a/src/plugins/dev_octeon/tx_node.c +++ b/src/plugins/dev_octeon/tx_node.c @@ -14,6 +14,19 @@ #include #include +#include + +#define OCT_LMT_GET_LINE_ADDR(lmt_addr, lmt_num) \ + (void *) ((u64) (lmt_addr) + ((u64) (lmt_num) << ROC_LMT_LINE_SIZE_LOG2)) + +#define OCT_SEND_HDR_DWORDS 1 + +/* + * Encoded number of segments to number of dwords macro, + * each value of nb_segs is encoded as 4bits. + */ +#define NIX_SEGDW_MAGIC 0x76654432210ULL +#define NIX_NB_SEGS_TO_SEGDW(x) ((NIX_SEGDW_MAGIC >> ((x) << 2)) & 0xF) typedef struct { @@ -413,6 +426,1146 @@ oct_tx_enq16 (vlib_main_t *vm, oct_tx_ctx_t *ctx, vnet_dev_tx_queue_t *txq, return n_pkts; } +static inline u16 +oct_check_fc_nix (struct roc_nix_sq *sq, i32 *fc_cache, u16 pkts) +{ + i32 val, new_val, depth; + u8 retry_count = 32; + + do + { + /* Reduce the cached count */ + val = (i32) __atomic_sub_fetch (fc_cache, pkts, __ATOMIC_RELAXED); + if (val >= 0) + return pkts; + + depth = sq->nb_sqb_bufs_adj - + __atomic_load_n ((u64 *) sq->fc, __ATOMIC_RELAXED); + + if (depth <= 0) + return 0; + + /* Update cached value (fc_cache) when lower than `pkts` */ + new_val = (depth << sq->sqes_per_sqb_log2) - pkts; + if (PREDICT_FALSE (new_val < 0)) + return 0; + + /* Update fc_cache if there is no update done by other cores */ + if (__atomic_compare_exchange_n (fc_cache, &val, new_val, false, + __ATOMIC_RELAXED, __ATOMIC_RELAXED)) + return pkts; + } + while (retry_count--); + + return 0; +} + +static inline u16 +oct_check_fc_cpt (struct roc_cpt_lf *cpt_lf, u32 *fc_cache, u16 pkts) +{ + i32 val, new_val, depth; + u8 retry_count = 32; + + do + { + /* Reduce the cached count */ + val = (i32) __atomic_sub_fetch (fc_cache, pkts, __ATOMIC_RELAXED); + if (val >= 0) + return pkts; + + depth = cpt_lf->nb_desc - clib_atomic_load_relax_n (cpt_lf->fc_addr); + + if (depth <= 0) + return 0; + new_val = depth - pkts; + if (PREDICT_FALSE (new_val < 0)) + return 0; + + /* Update fc_cache if there is no update done by other cores */ + if (__atomic_compare_exchange_n (fc_cache, (u32 *) &val, new_val, false, + __ATOMIC_RELAXED, __ATOMIC_RELAXED)) + return pkts; + } + while (retry_count--); + return 0; +} + +static_always_inline u64 +oct_add_sg_desc (union nix_send_sg_s *sg, int n_segs, vlib_buffer_t *seg1, + vlib_buffer_t *seg2, vlib_buffer_t *seg3) +{ + sg[0].u = 0; + sg[0].segs = n_segs; + sg[0].subdc = NIX_SUBDC_SG; + + switch (n_segs) + { + case 3: + sg[0].seg3_size = seg3->current_length; + sg[3].u = (u64) vlib_buffer_get_current (seg3); + /* Fall through */ + case 2: + sg[0].seg2_size = seg2->current_length; + sg[2].u = (u64) vlib_buffer_get_current (seg2); + /* Fall through */ + case 1: + sg[0].seg1_size = seg1->current_length; + sg[1].u = (u64) vlib_buffer_get_current (seg1); + break; + default: + ASSERT (0); + return 0; + } + + /* Return number of dwords in sub-descriptor */ + return n_segs == 1 ? 1 : 2; +} + +static_always_inline u64 +oct_add_sg_list (union nix_send_sg_s *sg, vlib_buffer_t *b, u64 n_segs) +{ + vlib_main_t *vm = vlib_get_main (); + vlib_buffer_t *seg1, *seg2, *seg3; + u64 n_dwords; + + if (PREDICT_TRUE (!(b->flags & VLIB_BUFFER_NEXT_PRESENT))) + return oct_add_sg_desc (sg, 1, b, NULL, NULL); + + seg1 = b; + n_dwords = 0; + while (n_segs > 2) + { + seg2 = vlib_get_buffer (vm, seg1->next_buffer); + seg3 = vlib_get_buffer (vm, seg2->next_buffer); + + n_dwords += oct_add_sg_desc (sg, 3, seg1, seg2, seg3); + + if (seg3->flags & VLIB_BUFFER_NEXT_PRESENT) + { + seg1 = vlib_get_buffer (vm, seg3->next_buffer); + sg += 4; + } + n_segs -= 3; + } + + if (n_segs == 1) + n_dwords += oct_add_sg_desc (sg, 1, seg1, NULL, NULL); + else if (n_segs == 2) + { + seg2 = vlib_get_buffer (vm, seg1->next_buffer); + n_dwords += oct_add_sg_desc (sg, 2, seg1, seg2, NULL); + } + + return n_dwords; +} + +static_always_inline u64 +oct_add_send_hdr (struct nix_send_hdr_s *hdr, vlib_buffer_t *b, + u64 aura_handle, u64 sq, u64 n_dwords) +{ + vnet_buffer_oflags_t oflags; + + hdr->w0.u = 0; + hdr->w1.u = 0; + hdr->w0.sq = sq; + hdr->w0.aura = roc_npa_aura_handle_to_aura (aura_handle); + hdr->w0.total = b->current_length; + hdr->w0.sizem1 = n_dwords + OCT_SEND_HDR_DWORDS - 1; + + if (b->flags & VLIB_BUFFER_NEXT_PRESENT) + hdr->w0.total = vlib_buffer_length_in_chain (vlib_get_main (), b); + + if (!(b->flags & VNET_BUFFER_F_OFFLOAD)) + return OCT_SEND_HDR_DWORDS; + + if (b->flags & VNET_BUFFER_F_OFFLOAD) + { + oflags = vnet_buffer (b)->oflags; + if (oflags & VNET_BUFFER_OFFLOAD_F_IP_CKSUM) + { + hdr->w1.ol3type = NIX_SENDL3TYPE_IP4_CKSUM; + hdr->w1.ol3ptr = vnet_buffer (b)->l3_hdr_offset - b->current_data; + hdr->w1.ol4ptr = hdr->w1.ol3ptr + sizeof (ip4_header_t); + } + + if (oflags & VNET_BUFFER_OFFLOAD_F_UDP_CKSUM) + { + hdr->w1.ol4type = NIX_SENDL4TYPE_UDP_CKSUM; + hdr->w1.ol4ptr = vnet_buffer (b)->l4_hdr_offset - b->current_data; + } + else if (oflags & VNET_BUFFER_OFFLOAD_F_TCP_CKSUM) + { + hdr->w1.ol4type = NIX_SENDL4TYPE_TCP_CKSUM; + hdr->w1.ol4ptr = vnet_buffer (b)->l4_hdr_offset - b->current_data; + } + } + return OCT_SEND_HDR_DWORDS; +} + +static_always_inline void +oct_ipsec_append_next_buffer (vlib_main_t *vm, vlib_buffer_t *buffer, + uint16_t bytes_to_append) +{ + u32 buffer_index = 0; + vlib_buffer_t *tmp; + + if (vlib_buffer_alloc (vm, &buffer_index, 1) != 1) + { + clib_warning ("buffer allocation failure"); + return; + } + + tmp = vlib_get_buffer (vm, buffer_index); + buffer->next_buffer = buffer_index; + buffer->flags |= VLIB_BUFFER_NEXT_PRESENT; + tmp->current_length += bytes_to_append; +} + +static_always_inline uint32_t +oct_ipsec_fill_sg2_buf (vlib_main_t *vm, struct roc_sg2list_comp *list, int i, + vlib_buffer_t **lb) +{ + struct roc_sg2list_comp *to; + + to = &list[i / 3]; + to->u.s.len[i % 3] = lb[0]->current_length; + to->ptr[i % 3] = (u64) vlib_buffer_get_current (lb[0]); + to->u.s.valid_segs = (i % 3) + 1; + i++; + + while (lb[0]->flags & VLIB_BUFFER_NEXT_PRESENT) + { + to = &list[i / 3]; + lb[0] = vlib_get_buffer (vm, lb[0]->next_buffer); + to->ptr[i % 3] = (u64) vlib_buffer_get_current (lb[0]); + to->u.s.len[i % 3] = lb[0]->current_length; + to->u.s.valid_segs = (i % 3) + 1; + i++; + } + + return i; +} + +static_always_inline int +oct_ipsec_outb_prepare_sg2_list (vlib_main_t *vm, vlib_buffer_t *b, + struct cpt_inst_s *inst, u32 bytes_to_append, + u32 dlen, void *m_data) +{ + u16 buffer_data_size = vlib_buffer_get_default_data_size (vm); + struct roc_sg2list_comp *scatter_comp, *gather_comp; + union cpt_inst_w5 cpt_inst_w5; + union cpt_inst_w6 cpt_inst_w6; + vlib_buffer_t *last_buf = b; + int i; + + /* Input Gather List */ + i = 0; + gather_comp = (struct roc_sg2list_comp *) ((uint8_t *) m_data + 64); + + i = oct_ipsec_fill_sg2_buf (vm, gather_comp, i, &last_buf); + + cpt_inst_w5.s.gather_sz = ((i + 2) / 3); + + if ((bytes_to_append + last_buf->current_length) > buffer_data_size) + { + /* Need an extra buffer */ + oct_ipsec_append_next_buffer (vm, last_buf, bytes_to_append); + } + else + { + vlib_buffer_put_uninit (last_buf, bytes_to_append); + } + + last_buf = b; + + /* Output Gather List */ + i = 0; + scatter_comp = (struct roc_sg2list_comp *) ((uint8_t *) m_data); + + i = oct_ipsec_fill_sg2_buf (vm, scatter_comp, i, &last_buf); + + cpt_inst_w6.s.scatter_sz = ((i + 2) / 3); + cpt_inst_w5.s.dptr = (uint64_t) gather_comp; + + cpt_inst_w6.s.rptr = (uint64_t) scatter_comp; + + inst->w5.u64 = cpt_inst_w5.u64; + inst->w6.u64 = cpt_inst_w6.u64; + inst->w4.s.dlen = dlen; + inst->w4.s.opcode_major &= (~(ROC_IE_OT_INPLACE_BIT)); + + b->total_length_not_including_first_buffer += bytes_to_append; + + return i; +} + +static_always_inline u32 +oct_get_tx_vlib_buf_segs (vlib_main_t *vm, vlib_buffer_t *b) +{ + /* Each buffer will have atleast 1 segment */ + u32 n_segs = 1; + + if (PREDICT_TRUE (!(b->flags & VLIB_BUFFER_NEXT_PRESENT))) + return n_segs; + + do + { + b = vlib_get_buffer (vm, b->next_buffer); + n_segs++; + } + while (b->flags & VLIB_BUFFER_NEXT_PRESENT); + + return n_segs; +} + +void static_always_inline +oct_prepare_ipsec_inst (vlib_main_t *vm, vlib_buffer_t *b, u64 sq_handle, + u64 aura_handle, + oct_ipsec_outbound_pkt_meta_t **pkt_meta, + u64 *n_dwords) +{ + u8 l2_len = sizeof (ethernet_header_t); + struct nix_send_hdr_s *send_hdr; + union nix_send_sg_s *sg; + u64 n_segs; + struct cpt_inst_s *inst; + u16 total_length; + + pkt_meta[0] = + (oct_ipsec_outbound_pkt_meta_t *) OCT_EXT_HDR_FROM_VLIB_BUFFER (b); + + inst = &pkt_meta[0]->inst; + + send_hdr = (struct nix_send_hdr_s *) pkt_meta[0]->nixtx; + sg = (union nix_send_sg_s *) (pkt_meta[0]->nixtx + 2); + + if (pkt_meta[0]->is_sg_mode) + { + total_length = + b->current_length + b->total_length_not_including_first_buffer; + + n_segs = oct_ipsec_outb_prepare_sg2_list ( + vm, b, inst, pkt_meta[0]->dlen_adj, total_length, + (void *) pkt_meta[0]->sg_buffer); + + inst->w0.u64 = (uint64_t) l2_len << 16; + inst->w0.u64 |= NIX_NB_SEGS_TO_SEGDW (n_segs); + inst->w0.u64 |= + (((int64_t) pkt_meta[0]->nixtx - (int64_t) inst->dptr) & 0xFFFFF) + << 32; + n_dwords[0] = (n_segs % 3) + (n_segs / 3) * 2; + sg[0].subdc = NIX_SUBDC_SG; + sg[4].subdc = NIX_SUBDC_SG; + } + else + { + b->current_length += pkt_meta[0]->dlen_adj; + n_segs = oct_get_tx_vlib_buf_segs (vm, b); + n_dwords[0] = oct_add_sg_list (sg, b, n_segs); + } + + oct_add_send_hdr (send_hdr, b, aura_handle, sq_handle, n_dwords[0]); + vlib_increment_combined_counter ( + &ipsec_sa_counters, vlib_get_thread_index (), + vnet_buffer (b)->ipsec.sad_index, 1, pkt_meta[0]->sa_bytes); +} + +void static_always_inline +oct_submit_quad_packets (u64 lmt_arg, oct_device_t *cd, + oct_ipsec_outbound_pkt_meta_t **pkt_meta, + u64 *n_dwords, u64 **lmt_line) +{ + roc_lmt_mov_seg ((void *) lmt_line[0], &pkt_meta[0]->inst, 4); + roc_lmt_mov_seg ((void *) lmt_line[1], &pkt_meta[1]->inst, 4); + roc_lmt_mov_seg ((void *) lmt_line[2], &pkt_meta[2]->inst, 4); + roc_lmt_mov_seg ((void *) lmt_line[3], &pkt_meta[3]->inst, 4); + + /* Count minus one of LMTSTs in the burst */ + lmt_arg |= 3 << 12; + + /* + * Vector of sizes of each LMTST in the burst. Every 3 bits + * represents size - 1 of one LMTST, except first. + */ + lmt_arg |= (n_dwords[1] - 1) << (19 + (3 * 0)); + lmt_arg |= (n_dwords[2] - 1) << (19 + (3 * 1)); + lmt_arg |= (n_dwords[3] - 1) << (19 + (3 * 2)); + + roc_lmt_submit_steorl (lmt_arg, cd->cpt_io_addr); + + asm volatile("dmb oshst" ::: "memory"); +} + +i32 static_always_inline +oct_pkts_send (vlib_main_t *vm, vlib_node_runtime_t *node, oct_tx_ctx_t *ctx, + vnet_dev_tx_queue_t *txq, u16 tx_pkts, vlib_buffer_t **bufs) +{ + oct_txq_t *ctq = vnet_dev_get_tx_queue_data (txq); + u32 desc_sz = 4; + union nix_send_sg_s *sg8, *sg9, *sg10, *sg11, *sg12, *sg13, *sg14, *sg15; + struct nix_send_hdr_s *send_hdr12, *send_hdr13, *send_hdr14, *send_hdr15; + struct nix_send_hdr_s *send_hdr8, *send_hdr9, *send_hdr10, *send_hdr11; + u64 desc12[desc_sz], desc13[desc_sz], desc14[desc_sz], desc15[desc_sz]; + u64 desc8[desc_sz], desc9[desc_sz], desc10[desc_sz], desc11[desc_sz]; + struct nix_send_hdr_s *send_hdr4, *send_hdr5, *send_hdr6, *send_hdr7; + struct nix_send_hdr_s *send_hdr0, *send_hdr1, *send_hdr2, *send_hdr3; + union nix_send_sg_s *sg0, *sg1, *sg2, *sg3, *sg4, *sg5, *sg6, *sg7; + u64 desc0[desc_sz], desc1[desc_sz], desc2[desc_sz], desc3[desc_sz]; + u64 desc4[desc_sz], desc5[desc_sz], desc6[desc_sz], desc7[desc_sz]; + u64 io_addr, sq_handle, n_dwords[16], n_packets; + void *lmt_line12, *lmt_line13, *lmt_line14, *lmt_line15; + void *lmt_line8, *lmt_line9, *lmt_line10, *lmt_line11; + void *lmt_line0, *lmt_line1, *lmt_line2, *lmt_line3; + void *lmt_line4, *lmt_line5, *lmt_line6, *lmt_line7; + u64 n_segs[16], aura_handle; + u64 lmt_arg, core_lmt_base_addr, core_lmt_id; + u16 n_drop = 0; + u32 from[VLIB_FRAME_SIZE]; + struct roc_nix_sq *sq; + vlib_buffer_t **b; + + sq = &ctq->sq; + b = bufs; + io_addr = sq->io_addr; + sq_handle = sq->qid; + aura_handle = ctq->aura_handle; + + if (PREDICT_FALSE (ctq->cached_pkts < tx_pkts)) + { + ctq->cached_pkts = (sq->nb_sqb_bufs_adj - *((u64 *) sq->fc)) + << sq->sqes_per_sqb_log2; + + if (PREDICT_FALSE (ctq->cached_pkts < tx_pkts)) + { + if (ctq->cached_pkts < 0) + { + n_drop = tx_pkts; + tx_pkts = 0; + goto free_pkts; + } + n_drop = tx_pkts - ctq->cached_pkts; + tx_pkts = ctq->cached_pkts; + } + } + + send_hdr0 = (struct nix_send_hdr_s *) &desc0[0]; + send_hdr1 = (struct nix_send_hdr_s *) &desc1[0]; + send_hdr2 = (struct nix_send_hdr_s *) &desc2[0]; + send_hdr3 = (struct nix_send_hdr_s *) &desc3[0]; + send_hdr4 = (struct nix_send_hdr_s *) &desc4[0]; + send_hdr5 = (struct nix_send_hdr_s *) &desc5[0]; + send_hdr6 = (struct nix_send_hdr_s *) &desc6[0]; + send_hdr7 = (struct nix_send_hdr_s *) &desc7[0]; + send_hdr8 = (struct nix_send_hdr_s *) &desc8[0]; + send_hdr9 = (struct nix_send_hdr_s *) &desc9[0]; + send_hdr10 = (struct nix_send_hdr_s *) &desc10[0]; + send_hdr11 = (struct nix_send_hdr_s *) &desc11[0]; + send_hdr12 = (struct nix_send_hdr_s *) &desc12[0]; + send_hdr13 = (struct nix_send_hdr_s *) &desc13[0]; + send_hdr14 = (struct nix_send_hdr_s *) &desc14[0]; + send_hdr15 = (struct nix_send_hdr_s *) &desc15[0]; + + sg0 = (union nix_send_sg_s *) &desc0[2]; + sg1 = (union nix_send_sg_s *) &desc1[2]; + sg2 = (union nix_send_sg_s *) &desc2[2]; + sg3 = (union nix_send_sg_s *) &desc3[2]; + sg4 = (union nix_send_sg_s *) &desc4[2]; + sg5 = (union nix_send_sg_s *) &desc5[2]; + sg6 = (union nix_send_sg_s *) &desc6[2]; + sg7 = (union nix_send_sg_s *) &desc7[2]; + sg8 = (union nix_send_sg_s *) &desc8[2]; + sg9 = (union nix_send_sg_s *) &desc9[2]; + sg10 = (union nix_send_sg_s *) &desc10[2]; + sg11 = (union nix_send_sg_s *) &desc11[2]; + sg12 = (union nix_send_sg_s *) &desc12[2]; + sg13 = (union nix_send_sg_s *) &desc13[2]; + sg14 = (union nix_send_sg_s *) &desc14[2]; + sg15 = (union nix_send_sg_s *) &desc15[2]; + + core_lmt_base_addr = (u64) sq->lmt_addr; + ROC_LMT_BASE_ID_GET (core_lmt_base_addr, core_lmt_id); + + lmt_line0 = OCT_LMT_GET_LINE_ADDR (core_lmt_base_addr, 0); + lmt_line1 = OCT_LMT_GET_LINE_ADDR (core_lmt_base_addr, 1); + lmt_line2 = OCT_LMT_GET_LINE_ADDR (core_lmt_base_addr, 2); + lmt_line3 = OCT_LMT_GET_LINE_ADDR (core_lmt_base_addr, 3); + lmt_line4 = OCT_LMT_GET_LINE_ADDR (core_lmt_base_addr, 4); + lmt_line5 = OCT_LMT_GET_LINE_ADDR (core_lmt_base_addr, 5); + lmt_line6 = OCT_LMT_GET_LINE_ADDR (core_lmt_base_addr, 6); + lmt_line7 = OCT_LMT_GET_LINE_ADDR (core_lmt_base_addr, 7); + lmt_line8 = OCT_LMT_GET_LINE_ADDR (core_lmt_base_addr, 8); + lmt_line9 = OCT_LMT_GET_LINE_ADDR (core_lmt_base_addr, 9); + lmt_line10 = OCT_LMT_GET_LINE_ADDR (core_lmt_base_addr, 10); + lmt_line11 = OCT_LMT_GET_LINE_ADDR (core_lmt_base_addr, 11); + lmt_line12 = OCT_LMT_GET_LINE_ADDR (core_lmt_base_addr, 12); + lmt_line13 = OCT_LMT_GET_LINE_ADDR (core_lmt_base_addr, 13); + lmt_line14 = OCT_LMT_GET_LINE_ADDR (core_lmt_base_addr, 14); + lmt_line15 = OCT_LMT_GET_LINE_ADDR (core_lmt_base_addr, 15); + + n_packets = tx_pkts; + + while (n_packets > 16) + { + n_segs[0] = oct_get_tx_vlib_buf_segs (vm, b[0]); + n_segs[1] = oct_get_tx_vlib_buf_segs (vm, b[1]); + n_segs[2] = oct_get_tx_vlib_buf_segs (vm, b[2]); + n_segs[3] = oct_get_tx_vlib_buf_segs (vm, b[3]); + n_segs[4] = oct_get_tx_vlib_buf_segs (vm, b[4]); + n_segs[5] = oct_get_tx_vlib_buf_segs (vm, b[5]); + n_segs[6] = oct_get_tx_vlib_buf_segs (vm, b[6]); + n_segs[7] = oct_get_tx_vlib_buf_segs (vm, b[7]); + n_segs[8] = oct_get_tx_vlib_buf_segs (vm, b[8]); + n_segs[9] = oct_get_tx_vlib_buf_segs (vm, b[9]); + n_segs[10] = oct_get_tx_vlib_buf_segs (vm, b[10]); + n_segs[11] = oct_get_tx_vlib_buf_segs (vm, b[11]); + n_segs[12] = oct_get_tx_vlib_buf_segs (vm, b[12]); + n_segs[13] = oct_get_tx_vlib_buf_segs (vm, b[13]); + n_segs[14] = oct_get_tx_vlib_buf_segs (vm, b[14]); + n_segs[15] = oct_get_tx_vlib_buf_segs (vm, b[15]); + + n_dwords[0] = oct_add_sg_list (sg0, b[0], n_segs[0]); + n_dwords[1] = oct_add_sg_list (sg1, b[1], n_segs[1]); + n_dwords[2] = oct_add_sg_list (sg2, b[2], n_segs[2]); + n_dwords[3] = oct_add_sg_list (sg3, b[3], n_segs[3]); + n_dwords[4] = oct_add_sg_list (sg4, b[4], n_segs[4]); + n_dwords[5] = oct_add_sg_list (sg5, b[5], n_segs[5]); + n_dwords[6] = oct_add_sg_list (sg6, b[6], n_segs[6]); + n_dwords[7] = oct_add_sg_list (sg7, b[7], n_segs[7]); + n_dwords[8] = oct_add_sg_list (sg8, b[8], n_segs[8]); + n_dwords[9] = oct_add_sg_list (sg9, b[9], n_segs[9]); + n_dwords[10] = oct_add_sg_list (sg10, b[10], n_segs[10]); + n_dwords[11] = oct_add_sg_list (sg11, b[11], n_segs[11]); + n_dwords[12] = oct_add_sg_list (sg12, b[12], n_segs[12]); + n_dwords[13] = oct_add_sg_list (sg13, b[13], n_segs[13]); + n_dwords[14] = oct_add_sg_list (sg14, b[14], n_segs[14]); + n_dwords[15] = oct_add_sg_list (sg15, b[15], n_segs[15]); + + n_dwords[0] += oct_add_send_hdr (send_hdr0, b[0], aura_handle, sq_handle, + n_dwords[0]); + n_dwords[1] += oct_add_send_hdr (send_hdr1, b[1], aura_handle, sq_handle, + n_dwords[1]); + n_dwords[2] += oct_add_send_hdr (send_hdr2, b[2], aura_handle, sq_handle, + n_dwords[2]); + n_dwords[3] += oct_add_send_hdr (send_hdr3, b[3], aura_handle, sq_handle, + n_dwords[3]); + n_dwords[4] += oct_add_send_hdr (send_hdr4, b[4], aura_handle, sq_handle, + n_dwords[4]); + n_dwords[5] += oct_add_send_hdr (send_hdr5, b[5], aura_handle, sq_handle, + n_dwords[5]); + n_dwords[6] += oct_add_send_hdr (send_hdr6, b[6], aura_handle, sq_handle, + n_dwords[6]); + n_dwords[7] += oct_add_send_hdr (send_hdr7, b[7], aura_handle, sq_handle, + n_dwords[7]); + + n_dwords[8] += oct_add_send_hdr (send_hdr8, b[8], aura_handle, sq_handle, + n_dwords[8]); + n_dwords[9] += oct_add_send_hdr (send_hdr9, b[9], aura_handle, sq_handle, + n_dwords[9]); + n_dwords[10] += oct_add_send_hdr (send_hdr10, b[10], aura_handle, + sq_handle, n_dwords[10]); + n_dwords[11] += oct_add_send_hdr (send_hdr11, b[11], aura_handle, + sq_handle, n_dwords[11]); + n_dwords[12] += oct_add_send_hdr (send_hdr12, b[12], aura_handle, + sq_handle, n_dwords[12]); + n_dwords[13] += oct_add_send_hdr (send_hdr13, b[13], aura_handle, + sq_handle, n_dwords[13]); + n_dwords[14] += oct_add_send_hdr (send_hdr14, b[14], aura_handle, + sq_handle, n_dwords[14]); + n_dwords[15] += oct_add_send_hdr (send_hdr15, b[15], aura_handle, + sq_handle, n_dwords[15]); + + /* + * Add a memory barrier so that LMTLINEs from the previous iteration + * can be reused for a subsequent transfer. + */ + asm volatile("dmb oshst" ::: "memory"); + + /* Clear io_addr[6:0] bits */ + io_addr &= ~0x7FULL; + lmt_arg = core_lmt_id; + + /* Set size-1 of first LMTST at io_addr[6:4] */ + io_addr |= (n_dwords[0] - 1) << 4; + + roc_lmt_mov_seg (lmt_line0, desc0, n_dwords[0]); + roc_lmt_mov_seg (lmt_line1, desc1, n_dwords[1]); + roc_lmt_mov_seg (lmt_line2, desc2, n_dwords[2]); + roc_lmt_mov_seg (lmt_line3, desc3, n_dwords[3]); + roc_lmt_mov_seg (lmt_line4, desc4, n_dwords[4]); + roc_lmt_mov_seg (lmt_line5, desc5, n_dwords[5]); + roc_lmt_mov_seg (lmt_line6, desc6, n_dwords[6]); + roc_lmt_mov_seg (lmt_line7, desc7, n_dwords[7]); + roc_lmt_mov_seg (lmt_line8, desc8, n_dwords[8]); + roc_lmt_mov_seg (lmt_line9, desc9, n_dwords[9]); + roc_lmt_mov_seg (lmt_line10, desc10, n_dwords[10]); + roc_lmt_mov_seg (lmt_line11, desc11, n_dwords[11]); + roc_lmt_mov_seg (lmt_line12, desc12, n_dwords[12]); + roc_lmt_mov_seg (lmt_line13, desc13, n_dwords[13]); + roc_lmt_mov_seg (lmt_line14, desc14, n_dwords[14]); + roc_lmt_mov_seg (lmt_line15, desc15, n_dwords[15]); + + /* Set number of LMTSTs, excluding the first */ + lmt_arg |= (16 - 1) << 12; + + /* + * Set vector of sizes of next 15 LMTSTs. + * Every 3 bits represent size-1 of one LMTST + */ + lmt_arg |= (n_dwords[1] - 1) << (19 + (3 * 0)); + lmt_arg |= (n_dwords[2] - 1) << (19 + (3 * 1)); + lmt_arg |= (n_dwords[3] - 1) << (19 + (3 * 2)); + lmt_arg |= (n_dwords[4] - 1) << (19 + (3 * 3)); + lmt_arg |= (n_dwords[5] - 1) << (19 + (3 * 4)); + lmt_arg |= (n_dwords[6] - 1) << (19 + (3 * 5)); + lmt_arg |= (n_dwords[7] - 1) << (19 + (3 * 6)); + lmt_arg |= (n_dwords[8] - 1) << (19 + (3 * 7)); + lmt_arg |= (n_dwords[9] - 1) << (19 + (3 * 8)); + lmt_arg |= (n_dwords[10] - 1) << (19 + (3 * 9)); + lmt_arg |= (n_dwords[11] - 1) << (19 + (3 * 10)); + lmt_arg |= (n_dwords[12] - 1) << (19 + (3 * 11)); + lmt_arg |= (n_dwords[13] - 1) << (19 + (3 * 12)); + lmt_arg |= (n_dwords[14] - 1) << (19 + (3 * 13)); + lmt_arg |= (n_dwords[15] - 1) << (19 + (3 * 14)); + + roc_lmt_submit_steorl (lmt_arg, io_addr); + + n_packets -= 16; + b += 16; + } + + while (n_packets > 8) + { + n_segs[0] = oct_get_tx_vlib_buf_segs (vm, b[0]); + n_segs[1] = oct_get_tx_vlib_buf_segs (vm, b[1]); + n_segs[2] = oct_get_tx_vlib_buf_segs (vm, b[2]); + n_segs[3] = oct_get_tx_vlib_buf_segs (vm, b[3]); + n_segs[4] = oct_get_tx_vlib_buf_segs (vm, b[4]); + n_segs[5] = oct_get_tx_vlib_buf_segs (vm, b[5]); + n_segs[6] = oct_get_tx_vlib_buf_segs (vm, b[6]); + n_segs[7] = oct_get_tx_vlib_buf_segs (vm, b[7]); + + n_dwords[0] = oct_add_sg_list (sg0, b[0], n_segs[0]); + n_dwords[1] = oct_add_sg_list (sg1, b[1], n_segs[1]); + n_dwords[2] = oct_add_sg_list (sg2, b[2], n_segs[2]); + n_dwords[3] = oct_add_sg_list (sg3, b[3], n_segs[3]); + n_dwords[4] = oct_add_sg_list (sg4, b[4], n_segs[4]); + n_dwords[5] = oct_add_sg_list (sg5, b[5], n_segs[5]); + n_dwords[6] = oct_add_sg_list (sg6, b[6], n_segs[6]); + n_dwords[7] = oct_add_sg_list (sg7, b[7], n_segs[7]); + + n_dwords[0] += oct_add_send_hdr (send_hdr0, b[0], aura_handle, sq_handle, + n_dwords[0]); + n_dwords[1] += oct_add_send_hdr (send_hdr1, b[1], aura_handle, sq_handle, + n_dwords[1]); + n_dwords[2] += oct_add_send_hdr (send_hdr2, b[2], aura_handle, sq_handle, + n_dwords[2]); + n_dwords[3] += oct_add_send_hdr (send_hdr3, b[3], aura_handle, sq_handle, + n_dwords[3]); + n_dwords[4] += oct_add_send_hdr (send_hdr4, b[4], aura_handle, sq_handle, + n_dwords[4]); + n_dwords[5] += oct_add_send_hdr (send_hdr5, b[5], aura_handle, sq_handle, + n_dwords[5]); + n_dwords[6] += oct_add_send_hdr (send_hdr6, b[6], aura_handle, sq_handle, + n_dwords[6]); + n_dwords[7] += oct_add_send_hdr (send_hdr7, b[7], aura_handle, sq_handle, + n_dwords[7]); + + /* + * Add a memory barrier so that LMTLINEs from the previous iteration + * can be reused for a subsequent transfer. + */ + asm volatile("dmb oshst" ::: "memory"); + + /* Clear io_addr[6:0] bits */ + io_addr &= ~0x7FULL; + lmt_arg = core_lmt_id; + + /* Set size-1 of first LMTST at io_addr[6:4] */ + io_addr |= (n_dwords[0] - 1) << 4; + + roc_lmt_mov_seg (lmt_line0, desc0, n_dwords[0]); + roc_lmt_mov_seg (lmt_line1, desc1, n_dwords[1]); + roc_lmt_mov_seg (lmt_line2, desc2, n_dwords[2]); + roc_lmt_mov_seg (lmt_line3, desc3, n_dwords[3]); + roc_lmt_mov_seg (lmt_line4, desc4, n_dwords[4]); + roc_lmt_mov_seg (lmt_line5, desc5, n_dwords[5]); + roc_lmt_mov_seg (lmt_line6, desc6, n_dwords[6]); + roc_lmt_mov_seg (lmt_line7, desc7, n_dwords[7]); + + /* Set number of LMTSTs, excluding the first */ + lmt_arg |= (8 - 1) << 12; + + /* + * Set vector of sizes of next 7 LMTSTs. + * Every 3 bits represent size-1 of one LMTST + */ + lmt_arg |= (n_dwords[1] - 1) << (19 + (3 * 0)); + lmt_arg |= (n_dwords[2] - 1) << (19 + (3 * 1)); + lmt_arg |= (n_dwords[3] - 1) << (19 + (3 * 2)); + lmt_arg |= (n_dwords[4] - 1) << (19 + (3 * 3)); + lmt_arg |= (n_dwords[5] - 1) << (19 + (3 * 4)); + lmt_arg |= (n_dwords[6] - 1) << (19 + (3 * 5)); + lmt_arg |= (n_dwords[7] - 1) << (19 + (3 * 6)); + + roc_lmt_submit_steorl (lmt_arg, io_addr); + + n_packets -= 8; + b += 8; + } + + while (n_packets > 4) + { + n_segs[0] = oct_get_tx_vlib_buf_segs (vm, b[0]); + n_segs[1] = oct_get_tx_vlib_buf_segs (vm, b[1]); + n_segs[2] = oct_get_tx_vlib_buf_segs (vm, b[2]); + n_segs[3] = oct_get_tx_vlib_buf_segs (vm, b[3]); + + n_dwords[0] = oct_add_sg_list (sg0, b[0], n_segs[0]); + n_dwords[1] = oct_add_sg_list (sg1, b[1], n_segs[1]); + n_dwords[2] = oct_add_sg_list (sg2, b[2], n_segs[2]); + n_dwords[3] = oct_add_sg_list (sg3, b[3], n_segs[3]); + + n_dwords[0] += oct_add_send_hdr (send_hdr0, b[0], aura_handle, sq_handle, + n_dwords[0]); + n_dwords[1] += oct_add_send_hdr (send_hdr1, b[1], aura_handle, sq_handle, + n_dwords[1]); + n_dwords[2] += oct_add_send_hdr (send_hdr2, b[2], aura_handle, sq_handle, + n_dwords[2]); + n_dwords[3] += oct_add_send_hdr (send_hdr3, b[3], aura_handle, sq_handle, + n_dwords[3]); + + /* + * Add a memory barrier so that LMTLINEs from the previous iteration + * can be reused for a subsequent transfer. + */ + asm volatile("dmb oshst" ::: "memory"); + + /* Clear io_addr[6:0] bits */ + io_addr &= ~0x7FULL; + lmt_arg = core_lmt_id; + + /* Set size-1 of first LMTST at io_addr[6:4] */ + io_addr |= (n_dwords[0] - 1) << 4; + + roc_lmt_mov_seg (lmt_line0, desc0, n_dwords[0]); + roc_lmt_mov_seg (lmt_line1, desc1, n_dwords[1]); + roc_lmt_mov_seg (lmt_line2, desc2, n_dwords[2]); + roc_lmt_mov_seg (lmt_line3, desc3, n_dwords[3]); + + /* Set number of LMTSTs, excluding the first */ + lmt_arg |= (4 - 1) << 12; + + /* + * Set vector of sizes of next 3 LMTSTs. + * Every 3 bits represent size-1 of one LMTST + */ + lmt_arg |= (n_dwords[1] - 1) << (19 + (3 * 0)); + lmt_arg |= (n_dwords[2] - 1) << (19 + (3 * 1)); + lmt_arg |= (n_dwords[3] - 1) << (19 + (3 * 2)); + + roc_lmt_submit_steorl (lmt_arg, io_addr); + + n_packets -= 4; + b += 4; + } + + while (n_packets) + { + lmt_arg = core_lmt_id; + + if (n_packets > 2) + vlib_prefetch_buffer_header (b[2], LOAD); + + n_segs[0] = oct_get_tx_vlib_buf_segs (vm, b[0]); + + n_dwords[0] = oct_add_sg_list (sg0, b[0], n_segs[0]); + n_dwords[0] += oct_add_send_hdr (send_hdr0, b[0], aura_handle, sq_handle, + n_dwords[0]); + + /* Clear io_addr[6:0] bits */ + io_addr &= ~0x7FULL; + + /* Set size-1 of first LMTST at io_addr[6:4] */ + io_addr |= (n_dwords[0] - 1) << 4; + + /* + * Add a memory barrier so that LMTLINEs from the previous iteration + * can be reused for a subsequent transfer. + */ + asm volatile("dmb oshst" ::: "memory"); + + roc_lmt_mov_seg (lmt_line0, desc0, n_dwords[0]); + + roc_lmt_submit_steorl (lmt_arg, io_addr); + + n_packets -= 1; + b += 1; + } + + ctq->cached_pkts -= tx_pkts; + +free_pkts: + if (PREDICT_FALSE (n_drop)) + { + vlib_get_buffer_indices_with_offset (vm, (void **) b, from, n_drop, 0); + vlib_buffer_free (vm, from, n_drop); + } + + return tx_pkts; +} + +i32 static_always_inline +oct_pkts_send_ipsec (vlib_main_t *vm, vlib_node_runtime_t *node, + oct_tx_ctx_t *ctx, vnet_dev_tx_queue_t *txq, u16 tx_pkts, + vlib_buffer_t **bufs) +{ + oct_txq_t *ctq = vnet_dev_get_tx_queue_data (txq); + vnet_dev_t *dev = txq->port->dev; + oct_device_t *cd = vnet_dev_get_data (dev); + u32 current_sq0, current_sq1, current_sq2, current_sq3; + u64 sq_handle0, sq_handle1, sq_handle2, sq_handle3; + u64 aura_handle0, aura_handle1, aura_handle2, aura_handle3; + u64 core_lmt_base_addr, lmt_arg, core_lmt_id; + oct_ipsec_outbound_pkt_meta_t *pkt_meta[4]; + oct_ipsec_outbound_pkt_meta_t *meta_hdr; + u16 n_cpt_fc_drop = 0, n_nix_fc_drop = 0; + u16 n_left0, n_left1, n_left2, n_left3; + u16 n_packets; + struct roc_cpt_lf *cpt_lf = NULL; + u32 failed_buff[VLIB_FRAME_SIZE]; + u32 from[VLIB_FRAME_SIZE]; + u16 sq0, sq1, sq2, sq3; + struct roc_nix_sq *sq; + u32 quad_bit, count; + vlib_buffer_t **b; + u64 *lmt_line[4]; + u64 n_dwords[4]; + + b = bufs; + + sq_handle0 = 0; + sq_handle1 = 0; + sq_handle2 = 0; + sq_handle3 = 0; + + aura_handle0 = 0; + aura_handle1 = 0; + aura_handle2 = 0; + aura_handle3 = 0; + + current_sq0 = ~0; + current_sq1 = ~0; + current_sq2 = ~0; + current_sq3 = ~0; + + core_lmt_base_addr = (uintptr_t) ctq->lmt_addr; + ROC_LMT_BASE_ID_GET (core_lmt_base_addr, core_lmt_id); + + lmt_line[0] = OCT_LMT_GET_LINE_ADDR (core_lmt_base_addr, 0); + lmt_line[1] = OCT_LMT_GET_LINE_ADDR (core_lmt_base_addr, 1); + lmt_line[2] = OCT_LMT_GET_LINE_ADDR (core_lmt_base_addr, 2); + lmt_line[3] = OCT_LMT_GET_LINE_ADDR (core_lmt_base_addr, 3); + + /* Check CPT flow control */ + cpt_lf = roc_nix_inl_outb_lf_base_get (cd->nix); + n_left0 = oct_check_fc_cpt (cpt_lf, (u32 *) &cd->cached_cpt_pkts, tx_pkts); + n_cpt_fc_drop = tx_pkts - n_left0; + + if (!n_left0) + goto cpt_fc_drop; + + /* Process packets up to CPT queue depth */ + n_packets = n_left0; + + n_left0 = 0; + n_left1 = 0; + n_left2 = 0; + n_left3 = 0; + + while (n_packets > 3) + { + meta_hdr = + (oct_ipsec_outbound_pkt_meta_t *) OCT_EXT_HDR_FROM_VLIB_BUFFER (b[0]); + sq0 = meta_hdr->core_id; + meta_hdr = + (oct_ipsec_outbound_pkt_meta_t *) OCT_EXT_HDR_FROM_VLIB_BUFFER (b[1]); + sq1 = meta_hdr->core_id; + meta_hdr = + (oct_ipsec_outbound_pkt_meta_t *) OCT_EXT_HDR_FROM_VLIB_BUFFER (b[2]); + sq2 = meta_hdr->core_id; + meta_hdr = + (oct_ipsec_outbound_pkt_meta_t *) OCT_EXT_HDR_FROM_VLIB_BUFFER (b[3]); + sq3 = meta_hdr->core_id; + + quad_bit = 0; + count = 0; + + if (current_sq0 != sq0) + { + ctq = cd->ctqs[sq0]; + sq = &ctq->sq; + sq_handle0 = sq->qid; + aura_handle0 = ctq->aura_handle; + n_left0 = oct_check_fc_nix (sq, &ctq->cached_pkts, n_packets >> 2); + current_sq0 = sq0; + } + if (current_sq1 != sq1) + { + ctq = cd->ctqs[sq1]; + sq = &ctq->sq; + sq_handle1 = sq->qid; + aura_handle1 = ctq->aura_handle; + n_left1 = oct_check_fc_nix (sq, &ctq->cached_pkts, n_packets >> 2); + current_sq1 = sq1; + } + if (current_sq2 != sq2) + { + ctq = cd->ctqs[sq2]; + sq = &ctq->sq; + sq_handle2 = sq->qid; + aura_handle2 = ctq->aura_handle; + n_left2 = oct_check_fc_nix (sq, &ctq->cached_pkts, n_packets >> 2); + current_sq2 = sq2; + } + if (current_sq3 != sq3) + { + ctq = cd->ctqs[sq3]; + sq = &ctq->sq; + sq_handle3 = sq->qid; + aura_handle3 = ctq->aura_handle; + n_left3 = oct_check_fc_nix (sq, &ctq->cached_pkts, n_packets >> 2); + current_sq3 = sq3; + } + quad_bit |= !(!n_left0) << 0; + quad_bit |= !(!n_left1) << 1; + quad_bit |= !(!n_left2) << 2; + quad_bit |= !(!n_left3) << 3; + + lmt_arg = ROC_CN10K_CPT_LMT_ARG | (uint64_t) core_lmt_id; + if (quad_bit == 0x0F) + { + oct_prepare_ipsec_inst (vm, b[0], sq_handle0, aura_handle0, + &pkt_meta[0], &n_dwords[0]); + oct_prepare_ipsec_inst (vm, b[1], sq_handle1, aura_handle1, + &pkt_meta[1], &n_dwords[1]); + oct_prepare_ipsec_inst (vm, b[2], sq_handle2, aura_handle2, + &pkt_meta[2], &n_dwords[2]); + oct_prepare_ipsec_inst (vm, b[3], sq_handle3, aura_handle3, + &pkt_meta[3], &n_dwords[3]); + + oct_submit_quad_packets (lmt_arg, cd, pkt_meta, n_dwords, lmt_line); + + n_left0 -= 1; + n_left1 -= 1; + n_left2 -= 1; + n_left3 -= 1; + count += 4; + } + else if (quad_bit != 0x0) + { + if (n_left0) + { + oct_prepare_ipsec_inst (vm, b[0], sq_handle0, aura_handle0, + &pkt_meta[0], &n_dwords[0]), + roc_lmt_mov_seg ((void *) lmt_line[count], &pkt_meta[0]->inst, + 4); + count++; + n_left0 -= 1; + } + else + { + failed_buff[n_nix_fc_drop] = vlib_get_buffer_index (vm, b[0]); + n_nix_fc_drop++; + } + if (n_left1) + { + oct_prepare_ipsec_inst (vm, b[1], sq_handle1, aura_handle1, + &pkt_meta[1], &n_dwords[1]); + roc_lmt_mov_seg ((void *) lmt_line[count], &pkt_meta[1]->inst, + 4); + if (count) + lmt_arg |= (n_dwords[1] - 1) << (19 + (3 * (count - 1))); + count++; + n_left1 -= 1; + } + else + { + failed_buff[n_nix_fc_drop] = vlib_get_buffer_index (vm, b[1]); + n_nix_fc_drop++; + } + if (n_left2) + { + oct_prepare_ipsec_inst (vm, b[2], sq_handle2, aura_handle2, + &pkt_meta[2], &n_dwords[2]); + roc_lmt_mov_seg ((void *) lmt_line[count], &pkt_meta[2]->inst, + 4); + if (count) + lmt_arg |= (n_dwords[2] - 1) << (19 + (3 * (count - 1))); + count++; + n_left2 -= 1; + } + else + { + failed_buff[n_nix_fc_drop] = vlib_get_buffer_index (vm, b[2]); + n_nix_fc_drop++; + } + if (n_left3) + { + oct_prepare_ipsec_inst (vm, b[3], sq_handle3, aura_handle3, + &pkt_meta[3], &n_dwords[3]); + roc_lmt_mov_seg ((void *) lmt_line[count], &pkt_meta[3]->inst, + 4); + if (count) + lmt_arg |= (n_dwords[3] - 1) << (19 + (3 * (count - 1))); + count++; + n_left3 -= 1; + } + else + { + failed_buff[n_nix_fc_drop] = vlib_get_buffer_index (vm, b[3]); + n_nix_fc_drop++; + } + if (count == 1) + lmt_arg = ROC_CN10K_CPT_LMT_ARG | core_lmt_id; + else + lmt_arg |= (count - 1) << 12; + roc_lmt_submit_steorl (lmt_arg, cd->cpt_io_addr); + asm volatile("dmb oshst" ::: "memory"); + } + + b += 4; + n_packets -= 4; + } + + current_sq0 = ~0; + sq_handle0 = 0; + n_left0 = 0; + + while (n_packets) + { + meta_hdr = + (oct_ipsec_outbound_pkt_meta_t *) OCT_EXT_HDR_FROM_VLIB_BUFFER (b[0]); + sq0 = meta_hdr->core_id; + + if (current_sq0 != sq0) + { + ctq = cd->ctqs[sq0]; + sq = &ctq->sq; + sq_handle0 = sq->qid; + aura_handle0 = ctq->aura_handle; + n_left0 = oct_check_fc_nix (sq, &ctq->cached_pkts, n_packets); + current_sq0 = sq0; + } + if (!n_left0) + { + failed_buff[n_nix_fc_drop] = vlib_get_buffer_index (vm, b[0]); + n_nix_fc_drop++; + goto next; + } + oct_prepare_ipsec_inst (vm, b[0], sq_handle0, aura_handle0, &pkt_meta[0], + &n_dwords[0]); + + roc_lmt_mov_seg ((void *) lmt_line[0], &pkt_meta[0]->inst, 4); + + lmt_arg = ROC_CN10K_CPT_LMT_ARG | core_lmt_id; + + roc_lmt_submit_steorl (lmt_arg, cd->cpt_io_addr); + + /* + * Add a memory barrier so that LMTLINEs from the previous iteration + * can be reused for a subsequent transfer. + */ + asm volatile("dmb oshst" ::: "memory"); + + n_left0 -= 1; + next: + n_packets -= 1; + b += 1; + } + + /* + * Free packets which failed in nix_fc_check. + * These packet indices are stored in failed_buff, + * as they may not be contiguous when received. + */ + if (PREDICT_FALSE (n_nix_fc_drop)) + vlib_buffer_free (vm, failed_buff, n_nix_fc_drop); + +cpt_fc_drop: + if (PREDICT_FALSE (n_cpt_fc_drop)) + { + vlib_get_buffer_indices_with_offset (vm, (void **) b, from, + n_cpt_fc_drop, 0); + vlib_buffer_free (vm, from, n_cpt_fc_drop); + } + + return tx_pkts - n_cpt_fc_drop - n_nix_fc_drop; +} + +VNET_DEV_NODE_FN (oct_tx_ipsec_tm_node) +(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame) +{ + vnet_dev_tx_node_runtime_t *rt = vnet_dev_get_tx_node_runtime (node); + vnet_dev_tx_queue_t *txq = rt->tx_queue; + oct_txq_t *ctq = vnet_dev_get_tx_queue_data (txq); + vnet_dev_t *dev = txq->port->dev; + oct_device_t *cd = vnet_dev_get_data (dev); + u32 node_index = node->node_index; + u32 n_left, n_pkts = frame->n_vectors; + vlib_buffer_t *buffers[VLIB_FRAME_SIZE + 8], **b = buffers; + vlib_buffer_t *ipsec_buff[VLIB_FRAME_SIZE + 8]; + vlib_buffer_t *buff[VLIB_FRAME_SIZE + 8]; + int ipsec_cnt = 0, pkt_cnt = 0; +#ifdef PLATFORM_OCTEON9 + u64 lmt_id = 0; +#else + u64 lmt_id = vm->thread_index << ROC_LMT_LINES_PER_CORE_LOG2; +#endif + + oct_tx_ctx_t ctx = { + .node = node, + .hdr_w0_teplate = { + .aura = roc_npa_aura_handle_to_aura (cd->ctqs[0]->aura_handle), + .sq = ctq->sq.qid, + .sizem1 = 1, + }, + .max_pkt_len = roc_nix_max_pkt_len (cd->nix), + .lmt_id = lmt_id, + .lmt_ioaddr = ctq->io_addr, + .lmt_lines = ctq->lmt_addr + (lmt_id << ROC_LMT_LINE_SIZE_LOG2), + }; + + oct_batch_free (vm, &ctx, txq); + + vlib_get_buffers (vm, vlib_frame_vector_args (frame), b, n_pkts); + n_left = n_pkts; + while (n_pkts) + { + if (vnet_buffer (b[0])->oflags & VNET_BUFFER_OFFLOAD_F_IPSEC_OFFLOAD) + ipsec_buff[ipsec_cnt++] = b[0]; + else + buff[pkt_cnt++] = b[0]; + + b++; + n_pkts--; + } + + if (ipsec_cnt) + ipsec_cnt = + oct_pkts_send_ipsec (vm, node, &ctx, txq, ipsec_cnt, ipsec_buff); + + if (pkt_cnt) + pkt_cnt = oct_pkts_send (vm, node, &ctx, txq, pkt_cnt, buff); + + if (PREDICT_FALSE (n_left != (ipsec_cnt + pkt_cnt))) + { + vlib_error_count (vm, node_index, OCT_TX_NODE_CTR_NO_FREE_SLOTS, + (n_left - ipsec_cnt - pkt_cnt)); + } + + return (ipsec_cnt + pkt_cnt); +} + VNET_DEV_NODE_FN (oct_tx_node) (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame) { diff --git a/src/vnet/buffer.h b/src/vnet/buffer.h index 144f62ac17..e141e948cd 100644 --- a/src/vnet/buffer.h +++ b/src/vnet/buffer.h @@ -114,7 +114,8 @@ STATIC_ASSERT (((VNET_BUFFER_FLAGS_ALL_AVAIL & VLIB_BUFFER_FLAGS_ALL) == 0), _ (3, OUTER_IP_CKSUM, "offload-outer-ip-cksum", 1) \ _ (4, OUTER_UDP_CKSUM, "offload-outer-udp-cksum", 1) \ _ (5, TNL_VXLAN, "offload-vxlan-tunnel", 1) \ - _ (6, TNL_IPIP, "offload-ipip-tunnel", 1) + _ (6, TNL_IPIP, "offload-ipip-tunnel", 1) \ + _ (7, IPSEC_OFFLOAD, "offload-ipsec-outbound", 1) typedef enum { From e8a346cafcbbc77fc646184af68f60abf07a39e8 Mon Sep 17 00:00:00 2001 From: Monendra Singh Kushwaha Date: Sat, 30 Nov 2024 02:50:57 +0530 Subject: [PATCH 130/271] octeon: fix header offset for alloc and free This patch fixes header offset value which is used in alloc and free. Type: fix Change-Id: I990cc2666fd1fdd8a2b6f6687edb1907d5e4b16d Signed-off-by: Monendra Singh Kushwaha Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/140461 --- src/plugins/dev_octeon/queue.c | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/src/plugins/dev_octeon/queue.c b/src/plugins/dev_octeon/queue.c index 55b14fb01b..d33b131f8b 100644 --- a/src/plugins/dev_octeon/queue.c +++ b/src/plugins/dev_octeon/queue.c @@ -100,6 +100,8 @@ oct_rxq_init (vlib_main_t *vm, vnet_dev_rx_queue_t *rxq, u32 total_sz) struct npa_pool_s npapool = { .nat_align = 1, .buf_offset = OCT_EXT_HDR_SIZE / ROC_ALIGN }; + ASSERT (!(vm->buffer_main->ext_hdr_size % ROC_ALIGN)); + if (!om->use_single_rx_aura || !om->aura_handle) { if ((rrv = roc_npa_pool_create (&crq->aura_handle, bp->alloc_size, @@ -135,15 +137,16 @@ oct_rxq_init (vlib_main_t *vm, vnet_dev_rx_queue_t *rxq, u32 total_sz) log_debug (dev, "CQ %u initialised (qmask 0x%x wdata 0x%lx)", crq->cq.qid, crq->cq.qmask, crq->cq.wdata); - crq->hdr_off = vm->buffer_main->ext_hdr_size; + crq->hdr_off = + vm->buffer_main->ext_hdr_size - (npapool.buf_offset * ROC_ALIGN); crq->rq = (struct roc_nix_rq){ .qid = rxq->queue_id, .cqid = crq->cq.qid, .aura_handle = crq->aura_handle, - .first_skip = sizeof (vlib_buffer_t), - .later_skip = sizeof (vlib_buffer_t), - .lpb_size = bp->data_size + sizeof (vlib_buffer_t), + .first_skip = crq->hdr_off + sizeof (vlib_buffer_t), + .later_skip = crq->hdr_off + sizeof (vlib_buffer_t), + .lpb_size = bp->data_size + crq->hdr_off + sizeof (vlib_buffer_t), .flow_tag_width = 32, }; @@ -215,7 +218,8 @@ oct_txq_init (vlib_main_t *vm, vnet_dev_tx_queue_t *txq) oct_device_t *cd = vnet_dev_get_data (dev); struct roc_nix *nix = cd->nix; struct npa_aura_s aura = {}; - struct npa_pool_s npapool = { .nat_align = 1 }; + struct npa_pool_s npapool = { .nat_align = 1, + .buf_offset = OCT_EXT_HDR_SIZE / ROC_ALIGN }; int rrv; vlib_buffer_pool_t *bp = vlib_get_buffer_pool (vm, 0); @@ -250,7 +254,8 @@ oct_txq_init (vlib_main_t *vm, vnet_dev_tx_queue_t *txq) log_debug (dev, "SQ initialised, qid %u, nb_desc %u, max_sqe_sz %u", ctq->sq.qid, ctq->sq.nb_desc, ctq->sq.max_sqe_sz); - ctq->hdr_off = vm->buffer_main->ext_hdr_size; + ctq->hdr_off = + vm->buffer_main->ext_hdr_size - (npapool.buf_offset * ROC_ALIGN); if (ctq->sq.lmt_addr == 0) ctq->sq.lmt_addr = (void *) nix->lmt_base; From 014f26ba9d9935ec3a383bd3ac86ed92a6d4e37d Mon Sep 17 00:00:00 2001 From: Monendra Singh Kushwaha Date: Sat, 30 Nov 2024 16:21:08 +0530 Subject: [PATCH 131/271] octeon: add port arg to enable MAC pause frame This patch adds port argument to enable MAC pause frame support. Type: refactor Signed-off-by: Monendra Singh Kushwaha Change-Id: I011aa418bd16e521c5259ec6929aa42b0141656a Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/140484 Tested-by: sa_ip-sw-jenkins --- src/plugins/dev_octeon/init.c | 8 ++++++++ src/plugins/dev_octeon/octeon.h | 1 + src/plugins/dev_octeon/port.c | 29 +++++++++++++++++++++-------- 3 files changed, 30 insertions(+), 8 deletions(-) diff --git a/src/plugins/dev_octeon/init.c b/src/plugins/dev_octeon/init.c index 5543db8e92..8356e265ae 100644 --- a/src/plugins/dev_octeon/init.c +++ b/src/plugins/dev_octeon/init.c @@ -93,6 +93,14 @@ static vnet_dev_arg_t oct_port_args[] = { .type = VNET_DEV_ARG_TYPE_BOOL, .default_val.boolean = false, }, + { + .id = OCT_PORT_ARG_EN_ETH_PAUSE_FRAME, + .name = "eth-pause-frame", + .desc = "Enable ethernet pause frame support, applicable to network " + "devices only", + .type = VNET_DEV_ARG_TYPE_BOOL, + .default_val.boolean = false, + }, { .id = OCT_PORT_ARG_END, .name = "end", diff --git a/src/plugins/dev_octeon/octeon.h b/src/plugins/dev_octeon/octeon.h index e5498ec4da..8698d67e7e 100644 --- a/src/plugins/dev_octeon/octeon.h +++ b/src/plugins/dev_octeon/octeon.h @@ -37,6 +37,7 @@ typedef enum { OCT_PORT_ARG_ALLMULTI_MODE = 1, + OCT_PORT_ARG_EN_ETH_PAUSE_FRAME, OCT_PORT_ARG_END } oct_port_args_t; diff --git a/src/plugins/dev_octeon/port.c b/src/plugins/dev_octeon/port.c index e38a24b536..5f63561cd9 100644 --- a/src/plugins/dev_octeon/port.c +++ b/src/plugins/dev_octeon/port.c @@ -145,6 +145,7 @@ oct_port_init (vlib_main_t *vm, vnet_dev_port_t *port) u8 mac_addr[PLT_ETHER_ADDR_LEN]; struct roc_nix *nix = cd->nix; vnet_dev_rv_t rv = -1; + bool is_allmulti_enable = false, is_pause_frame_enable = false; int rrv; log_notice (dev, "port init: port %u", port->port_id); @@ -176,17 +177,27 @@ oct_port_init (vlib_main_t *vm, vnet_dev_port_t *port) } } - /* Enable allmulti mode, if set by arg */ foreach_vnet_dev_port_args (arg, port) { if (arg->id == OCT_PORT_ARG_ALLMULTI_MODE && vnet_dev_arg_get_bool (arg)) + is_allmulti_enable = true; + if (arg->id == OCT_PORT_ARG_EN_ETH_PAUSE_FRAME && + vnet_dev_arg_get_bool (arg)) + is_pause_frame_enable = true; + } + if ((rrv = roc_nix_npc_mcast_config (nix, true, false))) + { + oct_port_deinit (vm, port); + return oct_roc_err (dev, rrv, "roc_nix_mac_addr_set failed"); + } + + /* Enable allmulti mode, if set by arg */ + if (is_allmulti_enable) + { + if ((rrv = roc_nix_npc_mcast_config (nix, true, false))) { - if ((rrv = roc_nix_npc_mcast_config (nix, true, false))) - { - oct_port_deinit (vm, port); - return oct_roc_err (dev, rrv, "roc_nix_mac_addr_set failed"); - } - break; + oct_port_deinit (vm, port); + return oct_roc_err (dev, rrv, "roc_nix_mac_addr_set failed"); } } @@ -257,7 +268,9 @@ oct_port_init (vlib_main_t *vm, vnet_dev_port_t *port) } /* Configure pause frame flow control*/ - if ((rv = oct_port_pause_flow_control_init (vm, port))) + + if (is_pause_frame_enable && + (rv = oct_port_pause_flow_control_init (vm, port))) { oct_port_deinit (vm, port); return rv; From c468a1b4a7f93d238198a6b49d6f515a30b52c93 Mon Sep 17 00:00:00 2001 From: Monendra Singh Kushwaha Date: Sat, 30 Nov 2024 18:47:44 +0530 Subject: [PATCH 132/271] octeon: fix thread index assignment Type: fix Change-Id: I5a18827e20cffcaac0e315966ee1cc1127df3b2f Signed-off-by: Monendra Singh Kushwaha Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/140486 Tested-by: sa_ip-sw-jenkins --- src/plugins/dev_octeon/esp_encrypt.c | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/src/plugins/dev_octeon/esp_encrypt.c b/src/plugins/dev_octeon/esp_encrypt.c index 1540f1675b..2786294f79 100644 --- a/src/plugins/dev_octeon/esp_encrypt.c +++ b/src/plugins/dev_octeon/esp_encrypt.c @@ -291,16 +291,14 @@ oct_esp_encrypt_tun (vlib_main_t *vm, vlib_node_runtime_t *node, current_sa0_index = sa0_index; } + /* + * If this is the first packet to use this SA, assign thread based + * on SA index. Don't need to do core-handoff on OCTEON as send queue + * is used based on thread index. + */ + if (PREDICT_FALSE (0XFFFF == sa0->thread_index)) - { - /* - * If this is the first packet to use this SA, claim the SA - * for this thread. Use atomic operation as this could happen - * simultaneously on another thread - */ - clib_atomic_cmp_and_swap (&sa0->thread_index, ~0, - ipsec_sa_assign_thread (vm->thread_index)); - } + sa0->thread_index = (sa0_index % vlib_num_workers ()) + 1; vnet_buffer (b[0])->ipsec.thread_index = sa0->thread_index; next[0] = thread0_next + sa0->thread_index; From 1f6470157b75b8db360914371fab37f4e9ad34d8 Mon Sep 17 00:00:00 2001 From: Monendra Singh Kushwaha Date: Sun, 1 Dec 2024 15:53:40 +0530 Subject: [PATCH 133/271] octeon: revert default queue size to 1024 Type: fix JIRA: https://essjira.marvell.com/browse/IPBUSW-57256 Signed-off-by: Monendra Singh Kushwaha Change-Id: I50a960fae100d81341cd70754c568ed43947b30f Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/140487 Tested-by: sa_ip-sw-jenkins Reviewed-by: Nithin Kumar Dabilpuram (cherry picked from commit f6c720ace7a3cc422f7aa4c98abea1e9df3fe5f2) Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/140488 Reviewed-by: Devapraba Muthumani --- src/plugins/dev_octeon/init.c | 7 +++---- src/plugins/dev_octeon/queue.c | 4 ++++ 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/plugins/dev_octeon/init.c b/src/plugins/dev_octeon/init.c index 8356e265ae..76d0b86a1c 100644 --- a/src/plugins/dev_octeon/init.c +++ b/src/plugins/dev_octeon/init.c @@ -95,7 +95,7 @@ static vnet_dev_arg_t oct_port_args[] = { }, { .id = OCT_PORT_ARG_EN_ETH_PAUSE_FRAME, - .name = "eth-pause-frame", + .name = "eth_pause_frame", .desc = "Enable ethernet pause frame support, applicable to network " "devices only", .type = VNET_DEV_ARG_TYPE_BOOL, @@ -241,7 +241,7 @@ oct_init_nix (vlib_main_t *vm, vnet_dev_t *dev) .rx_queue = { .config = { .data_size = sizeof (oct_rxq_t), - .default_size = 8192, + .default_size = 1024, .multiplier = 32, .min_size = 256, .max_size = 16384, @@ -255,7 +255,7 @@ oct_init_nix (vlib_main_t *vm, vnet_dev_t *dev) .tx_queue = { .config = { .data_size = sizeof (oct_txq_t), - .default_size = 8192, + .default_size = 1024, .multiplier = 32, .min_size = 256, .max_size = 16384, @@ -291,7 +291,6 @@ oct_init_nix (vlib_main_t *vm, vnet_dev_t *dev) pool_get (om->oct_dev, oct_dev); oct_dev[0] = vnet_dev_get_data (dev); oct_dev[0]->nix_idx = oct_dev - om->oct_dev; - log_info (dev, "veclen %d %d", vec_len (om->oct_dev), oct_dev[0]->nix_idx); return VNET_DEV_OK; } diff --git a/src/plugins/dev_octeon/queue.c b/src/plugins/dev_octeon/queue.c index d33b131f8b..50a519bd58 100644 --- a/src/plugins/dev_octeon/queue.c +++ b/src/plugins/dev_octeon/queue.c @@ -89,6 +89,7 @@ vnet_dev_rv_t oct_rxq_init (vlib_main_t *vm, vnet_dev_rx_queue_t *rxq, u32 total_sz) { oct_main_t *om = &oct_main; + oct_inl_dev_main_t *inl_main = &oct_inl_dev_main; oct_rxq_t *crq = vnet_dev_get_rx_queue_data (rxq); vnet_dev_t *dev = rxq->port->dev; oct_device_t *cd = vnet_dev_get_data (dev); @@ -125,6 +126,9 @@ oct_rxq_init (vlib_main_t *vm, vnet_dev_rx_queue_t *rxq, u32 total_sz) .qid = rxq->queue_id, }; + if (inl_main->inl_dev) + crq->cq.nb_desc = clib_max (crq->cq.nb_desc, 4096); + if ((rrv = roc_nix_cq_init (nix, &crq->cq))) { oct_rxq_deinit (vm, rxq); From 86ec5248c8680df5211bbfa7877caf0a0cccb4e5 Mon Sep 17 00:00:00 2001 From: Bheemappa Agasimundin Date: Wed, 4 Dec 2024 04:51:58 +0000 Subject: [PATCH 134/271] octeon: disable octeon virtio plugin by default This patch adds following changes, 1. Disables OCTEON virtio plugin by default. 2. Adds linking of -lfdt library. Type: fix JIRA: https://essjira.marvell.com/browse/IPBUSW-57292 Signed-off-by: Bheemappa Agasimundin Change-Id: If8f620827cac3dee327a32b0e2dfc2f1315c5902 Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/140722 Tested-by: sa_ip-sw-jenkins Reviewed-by: Monendra Singh Kushwaha (cherry picked from commit bb8afb0ab208691621e2196c39bced86fdfbeb11) Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/141050 --- src/plugins/dev_octeon/dev_octeon_virtio.mk | 2 +- src/plugins/dev_octeon/virtio.c | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/plugins/dev_octeon/dev_octeon_virtio.mk b/src/plugins/dev_octeon/dev_octeon_virtio.mk index 5f84a7f4b4..fff3918d53 100644 --- a/src/plugins/dev_octeon/dev_octeon_virtio.mk +++ b/src/plugins/dev_octeon/dev_octeon_virtio.mk @@ -30,7 +30,7 @@ get_filename_component(DAO_DPDK_LIB_DIR ${DAO_DPDK_LIB} DIRECTORY) link_directories(${DAO_DPDK_LIB_DIR}) string_append(DAO_LINK_FLAGS "-L${DAO_DPDK_LIB_DIR}") -string_append(DAO_LINK_FLAGS "-lnuma -lz -lelf -lpcap -ljansson") +string_append(DAO_LINK_FLAGS "-lnuma -lz -lelf -lpcap -ljansson -lfdt") if(OPENSSL_FOUND) string_append(DAO_LINK_FLAGS "-lssl") string_append(DAO_LINK_FLAGS "-lcrypto") diff --git a/src/plugins/dev_octeon/virtio.c b/src/plugins/dev_octeon/virtio.c index 8279d77e94..a1ceca655c 100644 --- a/src/plugins/dev_octeon/virtio.c +++ b/src/plugins/dev_octeon/virtio.c @@ -432,5 +432,6 @@ VNET_DEV_REGISTER_DRIVER (octeon_virtio) = { VLIB_PLUGIN_REGISTER () = { .version = VPP_BUILD_VER, - .description = "dev_octeon_virtio", + .description = "OCTEON virtio device", + .default_disabled = 1, }; From 6aa7ba903d62f74cbd1cc07bd4e1755668803c5b Mon Sep 17 00:00:00 2001 From: Monendra Singh Kushwaha Date: Thu, 5 Dec 2024 18:49:26 +0530 Subject: [PATCH 135/271] octeon: fix packet check for octeon9 This patch fixes packet check for OCTEON9, as inline IPSec is not supported on OCTEON9. Type: fix JIRA: https://essjira.marvell.com/browse/IPBUSW-57516 Change-Id: I37560face3a50f9fe4e9a923fee13e3695b21b3e Signed-off-by: Monendra Singh Kushwaha Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/141051 Reviewed-by: Devapraba Muthumani Tested-by: sa_ip-sw-jenkins --- src/plugins/dev_octeon/rx_node.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/plugins/dev_octeon/rx_node.c b/src/plugins/dev_octeon/rx_node.c index 22152a9b48..e4beb2fb66 100644 --- a/src/plugins/dev_octeon/rx_node.c +++ b/src/plugins/dev_octeon/rx_node.c @@ -361,7 +361,11 @@ oct_rx_ipsec_update_counters (vlib_main_t *vm, vlib_buffer_t *b, u32 ilen, static_always_inline u8 oct_is_packet_from_cpt (union nix_rx_parse_u *rxp) { +#ifdef PLATFORM_OCTEON9 + return 0; +#else return rxp->chan >> 11; +#endif } static_always_inline uword From 40d4b20946a7583d4aebb7e44613749c5d044420 Mon Sep 17 00:00:00 2001 From: Monendra Singh Kushwaha Date: Sat, 7 Dec 2024 01:59:23 +0530 Subject: [PATCH 136/271] octeon: put deleted session to pool This patch puts deleted session to pool and use single tx aura for all tx queues. Type: fix JIRA: https://essjira.marvell.com/browse/IPBUSW-57575 Change-Id: Ia399a8f1f4f69654ba9381b997b66cd39f9b7578 Signed-off-by: Monendra Singh Kushwaha Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/141057 Tested-by: sa_ip-sw-jenkins --- src/plugins/dev_octeon/init.c | 2 +- src/plugins/dev_octeon/ipsec.c | 2 ++ src/plugins/dev_octeon/octeon.h | 3 ++- src/plugins/dev_octeon/queue.c | 21 ++++++++++++++------- 4 files changed, 19 insertions(+), 9 deletions(-) diff --git a/src/plugins/dev_octeon/init.c b/src/plugins/dev_octeon/init.c index 76d0b86a1c..0d8fb78c9d 100644 --- a/src/plugins/dev_octeon/init.c +++ b/src/plugins/dev_octeon/init.c @@ -373,7 +373,7 @@ oct_init_inl_dev (vlib_main_t *vm, vnet_dev_t *dev) if ((rv = oct_init_ipsec_backend (vm, dev))) return rv; - oct_main.use_single_rx_aura = 1; + oct_main.use_single_rx_tx_aura = 1; oct_main.inl_dev_initialized = 1; return VNET_DEV_OK; diff --git a/src/plugins/dev_octeon/ipsec.c b/src/plugins/dev_octeon/ipsec.c index 6d70db1f30..49817aa8a1 100644 --- a/src/plugins/dev_octeon/ipsec.c +++ b/src/plugins/dev_octeon/ipsec.c @@ -670,6 +670,8 @@ oct_ipsec_session_destroy (u32 sa_index) } clib_memset (session, 0, sizeof (oct_ipsec_session_t)); + pool_put (oim->inline_ipsec_sessions, session); + return 0; } diff --git a/src/plugins/dev_octeon/octeon.h b/src/plugins/dev_octeon/octeon.h index 8698d67e7e..01ae1b810e 100644 --- a/src/plugins/dev_octeon/octeon.h +++ b/src/plugins/dev_octeon/octeon.h @@ -155,8 +155,9 @@ typedef struct { oct_device_t **oct_dev; u8 inl_dev_initialized : 1; - u8 use_single_rx_aura : 1; + u8 use_single_rx_tx_aura : 1; u64 aura_handle; + u64 tx_aura_handle; } oct_main_t; diff --git a/src/plugins/dev_octeon/queue.c b/src/plugins/dev_octeon/queue.c index 50a519bd58..f5792aca1b 100644 --- a/src/plugins/dev_octeon/queue.c +++ b/src/plugins/dev_octeon/queue.c @@ -103,7 +103,7 @@ oct_rxq_init (vlib_main_t *vm, vnet_dev_rx_queue_t *rxq, u32 total_sz) ASSERT (!(vm->buffer_main->ext_hdr_size % ROC_ALIGN)); - if (!om->use_single_rx_aura || !om->aura_handle) + if (!om->use_single_rx_tx_aura || !om->aura_handle) { if ((rrv = roc_npa_pool_create (&crq->aura_handle, bp->alloc_size, total_sz, &aura, &npapool, 0))) @@ -217,6 +217,7 @@ oct_rxq_deinit (vlib_main_t *vm, vnet_dev_rx_queue_t *rxq) vnet_dev_rv_t oct_txq_init (vlib_main_t *vm, vnet_dev_tx_queue_t *txq) { + oct_main_t *om = &oct_main; oct_txq_t *ctq = vnet_dev_get_tx_queue_data (txq); vnet_dev_t *dev = txq->port->dev; oct_device_t *cd = vnet_dev_get_data (dev); @@ -227,14 +228,20 @@ oct_txq_init (vlib_main_t *vm, vnet_dev_tx_queue_t *txq) int rrv; vlib_buffer_pool_t *bp = vlib_get_buffer_pool (vm, 0); - if ((rrv = roc_npa_pool_create ( - &ctq->aura_handle, bp->alloc_size, - txq->size * 6 /* worst case - two SG with 3 segs each = 6 */, &aura, - &npapool, 0))) + if (!om->use_single_rx_tx_aura || !om->tx_aura_handle) { - oct_txq_deinit (vm, txq); - return oct_roc_err (dev, rrv, "roc_npa_pool_create() failed"); + if ((rrv = roc_npa_pool_create ( + &ctq->aura_handle, bp->alloc_size, + txq->size * 6 /* worst case - two SG with 3 segs each = 6 */, + &aura, &npapool, 0))) + { + oct_txq_deinit (vm, txq); + return oct_roc_err (dev, rrv, "roc_npa_pool_create() failed"); + } + om->tx_aura_handle = ctq->aura_handle; } + else + ctq->aura_handle = om->tx_aura_handle; ctq->npa_pool_initialized = 1; log_notice (dev, "NPA pool created, aura_handle = 0x%lx", ctq->aura_handle); From eaee92753dd0fe35f5eb9391bb6b6cdae304b31c Mon Sep 17 00:00:00 2001 From: Monendra Singh Kushwaha Date: Thu, 12 Dec 2024 17:41:30 +0530 Subject: [PATCH 137/271] build: fix dao library installation Type: fix Signed-off-by: Bheemappa Agasimundin --- build/external/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/external/Makefile b/build/external/Makefile index be0b7d2437..e521803628 100644 --- a/build/external/Makefile +++ b/build/external/Makefile @@ -50,7 +50,7 @@ clean: @rm -rf $(B) $(I) .PHONY: install -install: $(if $(ARCH_X86_64), ipsec-mb-install) dpdk-install rdma-core-install quicly-install xdp-tools-install $(if $(AARCH64), octeon-roc-install, octeon-dao-install) +install: $(if $(ARCH_X86_64), ipsec-mb-install) dpdk-install rdma-core-install quicly-install xdp-tools-install $(if $(AARCH64), octeon-roc-install octeon-dao-install) .PHONY: config config: $(if $(ARCH_X86_64), ipsec-mb-config) dpdk-config rdma-core-config quicly-build From 7db6490e66cdc3e2b8ef8914108fa1352ae79832 Mon Sep 17 00:00:00 2001 From: Nagendra T P Date: Fri, 13 Dec 2024 13:31:44 +0530 Subject: [PATCH 138/271] ci: include meson and other dependencies Included meson build-essential pkg-config packages before build Signed-off-by: Nagendra T P --- .github/workflows/build-cn9k.yml | 2 +- .github/workflows/build.yml | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-cn9k.yml b/.github/workflows/build-cn9k.yml index 40b636250a..c3484ce712 100644 --- a/.github/workflows/build-cn9k.yml +++ b/.github/workflows/build-cn9k.yml @@ -128,7 +128,7 @@ jobs: apt-get install -y libconfuse-dev libelf-dev libffi7 libmnl-dev libnl-3-dev libnl-route-3-dev apt-get install -y libnuma-dev libpcap-dev libssl-dev nasm ninja-build python3-all python3-dev apt-get install -y python3-jsonschema python3-pip python3-ply python3-setuptools python3-venv - apt-get install -y python3-virtualenv tshark lsb-release + apt-get install -y python3-virtualenv tshark lsb-release meson build-essential pkg-config run: | source /artifacts/env DISTRO=ubuntu-`lsb_release -rs` diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d603f2f380..82fc970893 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -81,7 +81,7 @@ jobs: shell: /bin/bash install: | apt-get update -q -y - apt-get install -y apt-utils sudo make dialog ccache git + apt-get install -y apt-utils sudo make dialog ccache git build-essential apt-get install -y aspell aspell-en autopoint autotools-dev binfmt-support binutils binutils-aarch64-linux-gnu apt-get install -y binutils-common bsdextrautils bzip2 ca-certificates clang-14 cmake-data cpp cpp-11 cpp-12 dbus apt-get install -y dctrl-tools debugedit dh-autoreconf dh-elpa-helper dh-strip-nondeterminism dictionaries-common dirmngr @@ -127,8 +127,8 @@ jobs: apt-get install -y ethtool exuberant-ctags gcovr git-review iperf iperf3 lcov libapr1-dev apt-get install -y libconfuse-dev libelf-dev libffi7 libmnl-dev libnl-3-dev libnl-route-3-dev apt-get install -y libnuma-dev libpcap-dev libssl-dev nasm ninja-build python3-all python3-dev - apt-get install -y python3-jsonschema python3-pip python3-ply python3-setuptools python3-venv - apt-get install -y python3-virtualenv tshark lsb-release + apt-get install -y python3-jsonschema pkg-config python3-pip python3-ply python3-setuptools python3-venv + apt-get install -y python3-virtualenv tshark lsb-release meson run: | source /artifacts/env DISTRO=ubuntu-`lsb_release -rs` From d8e9c243cd5e87ca6676152b371fad9d1d088546 Mon Sep 17 00:00:00 2001 From: Nagendra T P Date: Fri, 13 Dec 2024 19:03:08 +0530 Subject: [PATCH 139/271] ci: fix errors regarding meson version Signed-off-by: Nagendra T P --- .github/workflows/build-cn9k.yml | 4 +++- .github/workflows/build.yml | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-cn9k.yml b/.github/workflows/build-cn9k.yml index c3484ce712..dfbb42c278 100644 --- a/.github/workflows/build-cn9k.yml +++ b/.github/workflows/build-cn9k.yml @@ -128,7 +128,9 @@ jobs: apt-get install -y libconfuse-dev libelf-dev libffi7 libmnl-dev libnl-3-dev libnl-route-3-dev apt-get install -y libnuma-dev libpcap-dev libssl-dev nasm ninja-build python3-all python3-dev apt-get install -y python3-jsonschema python3-pip python3-ply python3-setuptools python3-venv - apt-get install -y python3-virtualenv tshark lsb-release meson build-essential pkg-config + apt-get install -y python3-virtualenv tshark lsb-release build-essential pkg-config + pip3 install --upgrade pip + pip3 install meson==1.5.2 run: | source /artifacts/env DISTRO=ubuntu-`lsb_release -rs` diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 82fc970893..af9bc47626 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -128,7 +128,9 @@ jobs: apt-get install -y libconfuse-dev libelf-dev libffi7 libmnl-dev libnl-3-dev libnl-route-3-dev apt-get install -y libnuma-dev libpcap-dev libssl-dev nasm ninja-build python3-all python3-dev apt-get install -y python3-jsonschema pkg-config python3-pip python3-ply python3-setuptools python3-venv - apt-get install -y python3-virtualenv tshark lsb-release meson + apt-get install -y python3-virtualenv tshark lsb-release + pip3 install --upgrade pip + pip3 install meson==1.5.2 run: | source /artifacts/env DISTRO=ubuntu-`lsb_release -rs` From 9e9335d78ead771350226602ea927d1ef18ab8b0 Mon Sep 17 00:00:00 2001 From: Nagendra T P Date: Sat, 14 Dec 2024 11:50:37 +0530 Subject: [PATCH 140/271] ci: add doxygen package --- .github/workflows/{build.yml => build-cn10k.yml} | 2 +- .github/workflows/build-cn9k.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) rename .github/workflows/{build.yml => build-cn10k.yml} (99%) diff --git a/.github/workflows/build.yml b/.github/workflows/build-cn10k.yml similarity index 99% rename from .github/workflows/build.yml rename to .github/workflows/build-cn10k.yml index af9bc47626..f79d4d5d9e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build-cn10k.yml @@ -128,7 +128,7 @@ jobs: apt-get install -y libconfuse-dev libelf-dev libffi7 libmnl-dev libnl-3-dev libnl-route-3-dev apt-get install -y libnuma-dev libpcap-dev libssl-dev nasm ninja-build python3-all python3-dev apt-get install -y python3-jsonschema pkg-config python3-pip python3-ply python3-setuptools python3-venv - apt-get install -y python3-virtualenv tshark lsb-release + apt-get install -y python3-virtualenv tshark lsb-release doxygen pip3 install --upgrade pip pip3 install meson==1.5.2 run: | diff --git a/.github/workflows/build-cn9k.yml b/.github/workflows/build-cn9k.yml index dfbb42c278..ae76112a5f 100644 --- a/.github/workflows/build-cn9k.yml +++ b/.github/workflows/build-cn9k.yml @@ -128,7 +128,7 @@ jobs: apt-get install -y libconfuse-dev libelf-dev libffi7 libmnl-dev libnl-3-dev libnl-route-3-dev apt-get install -y libnuma-dev libpcap-dev libssl-dev nasm ninja-build python3-all python3-dev apt-get install -y python3-jsonschema python3-pip python3-ply python3-setuptools python3-venv - apt-get install -y python3-virtualenv tshark lsb-release build-essential pkg-config + apt-get install -y python3-virtualenv tshark lsb-release build-essential pkg-config doxygen pip3 install --upgrade pip pip3 install meson==1.5.2 run: | From 7941de39cf2b7e6fae7150e8ffd29d812aa07f45 Mon Sep 17 00:00:00 2001 From: Nagendra T P Date: Mon, 16 Dec 2024 10:05:23 +0530 Subject: [PATCH 141/271] ci: include packages required for DAO compilation Signed-off-by: Nagendra T P --- .github/workflows/build-cn10k.yml | 21 ++++++++++++--------- .github/workflows/build-cn9k.yml | 23 +++++++++++++---------- 2 files changed, 25 insertions(+), 19 deletions(-) diff --git a/.github/workflows/build-cn10k.yml b/.github/workflows/build-cn10k.yml index f79d4d5d9e..1b72977754 100644 --- a/.github/workflows/build-cn10k.yml +++ b/.github/workflows/build-cn10k.yml @@ -81,7 +81,9 @@ jobs: shell: /bin/bash install: | apt-get update -q -y - apt-get install -y apt-utils sudo make dialog ccache git build-essential + apt-get install -y apt-utils gcc meson sudo make dialog ccache git build-essential software-properties-common + add-apt-repository -y ppa:ubuntu-toolchain-r/test + apt-get update -q -y apt-get install -y aspell aspell-en autopoint autotools-dev binfmt-support binutils binutils-aarch64-linux-gnu apt-get install -y binutils-common bsdextrautils bzip2 ca-certificates clang-14 cmake-data cpp cpp-11 cpp-12 dbus apt-get install -y dctrl-tools debugedit dh-autoreconf dh-elpa-helper dh-strip-nondeterminism dictionaries-common dirmngr @@ -122,15 +124,16 @@ jobs: apt-get install -y python3-pyrsistent python3-requests python3-setuptools-whl python3-six python3-soupsieve python3-tz apt-get install -y python3-urllib3 python3-webencodings python3-wheel python3-wheel-whl python3-zipp python3.10 apt-get install -y python3.10-dev python3.10-minimal python3.10-venv readline-common rpcsvc-proto shared-mime-info tzdata - apt-get install -y ucf uuid-dev wireshark-common xauth xdg-user-dirs xz-utils zlib1g-dev - apt-get install -y check chrpath clang clang-format-11 cmake cscope debhelper dh-python dkms - apt-get install -y ethtool exuberant-ctags gcovr git-review iperf iperf3 lcov libapr1-dev - apt-get install -y libconfuse-dev libelf-dev libffi7 libmnl-dev libnl-3-dev libnl-route-3-dev - apt-get install -y libnuma-dev libpcap-dev libssl-dev nasm ninja-build python3-all python3-dev + apt-get install -y ucf uuid-dev wireshark-common xauth xdg-user-dirs xz-utils zlib1g-dev libgmpxx4ldbl + apt-get install -y check chrpath clang clang-format-11 cmake cscope debhelper dh-python dkms liblz4-dev + apt-get install -y ethtool exuberant-ctags gcovr git-review iperf iperf3 lcov libapr1-dev liblzma-dev wget + apt-get install -y libconfuse-dev libelf-dev libffi7 libmnl-dev libnl-3-dev libnl-route-3-dev libzstd-dev + apt-get install -y libnuma-dev libpcap-dev libssl-dev nasm ninja-build python3-all python3-dev nettle-dev apt-get install -y python3-jsonschema pkg-config python3-pip python3-ply python3-setuptools python3-venv - apt-get install -y python3-virtualenv tshark lsb-release doxygen - pip3 install --upgrade pip - pip3 install meson==1.5.2 + apt-get install -y python3-virtualenv tshark lsb-release doxygen libarchive-dev libbsd-dev libbpf-dev + apt-get install -y libnl-xfrm-3-dev sphinx-common python3-sphinx-rtd-theme libfdt-dev libjansson-dev + apt-get install -y python3-pyelftools gcc-13 bzip2-doc libacl1-dev libattr1-dev libbz2-dev libgmp-dev + pip3 install meson --upgrade run: | source /artifacts/env DISTRO=ubuntu-`lsb_release -rs` diff --git a/.github/workflows/build-cn9k.yml b/.github/workflows/build-cn9k.yml index ae76112a5f..582260fddc 100644 --- a/.github/workflows/build-cn9k.yml +++ b/.github/workflows/build-cn9k.yml @@ -81,7 +81,9 @@ jobs: shell: /bin/bash install: | apt-get update -q -y - apt-get install -y apt-utils sudo make dialog ccache git + apt-get install -y apt-utils gcc meson sudo make dialog ccache git build-essential software-properties-common + add-apt-repository -y ppa:ubuntu-toolchain-r/test + apt-get update -q -y apt-get install -y aspell aspell-en autopoint autotools-dev binfmt-support binutils binutils-aarch64-linux-gnu apt-get install -y binutils-common bsdextrautils bzip2 ca-certificates clang-14 cmake-data cpp cpp-11 cpp-12 dbus apt-get install -y dctrl-tools debugedit dh-autoreconf dh-elpa-helper dh-strip-nondeterminism dictionaries-common dirmngr @@ -122,15 +124,16 @@ jobs: apt-get install -y python3-pyrsistent python3-requests python3-setuptools-whl python3-six python3-soupsieve python3-tz apt-get install -y python3-urllib3 python3-webencodings python3-wheel python3-wheel-whl python3-zipp python3.10 apt-get install -y python3.10-dev python3.10-minimal python3.10-venv readline-common rpcsvc-proto shared-mime-info tzdata - apt-get install -y ucf uuid-dev wireshark-common xauth xdg-user-dirs xz-utils zlib1g-dev - apt-get install -y check chrpath clang clang-format-11 cmake cscope debhelper dh-python dkms - apt-get install -y ethtool exuberant-ctags gcovr git-review iperf iperf3 lcov libapr1-dev - apt-get install -y libconfuse-dev libelf-dev libffi7 libmnl-dev libnl-3-dev libnl-route-3-dev - apt-get install -y libnuma-dev libpcap-dev libssl-dev nasm ninja-build python3-all python3-dev - apt-get install -y python3-jsonschema python3-pip python3-ply python3-setuptools python3-venv - apt-get install -y python3-virtualenv tshark lsb-release build-essential pkg-config doxygen - pip3 install --upgrade pip - pip3 install meson==1.5.2 + apt-get install -y ucf uuid-dev wireshark-common xauth xdg-user-dirs xz-utils zlib1g-dev libgmpxx4ldbl + apt-get install -y check chrpath clang clang-format-11 cmake cscope debhelper dh-python dkms liblz4-dev + apt-get install -y ethtool exuberant-ctags gcovr git-review iperf iperf3 lcov libapr1-dev liblzma-dev wget + apt-get install -y libconfuse-dev libelf-dev libffi7 libmnl-dev libnl-3-dev libnl-route-3-dev libzstd-dev + apt-get install -y libnuma-dev libpcap-dev libssl-dev nasm ninja-build python3-all python3-dev nettle-dev + apt-get install -y python3-jsonschema pkg-config python3-pip python3-ply python3-setuptools python3-venv + apt-get install -y python3-virtualenv tshark lsb-release doxygen libarchive-dev libbsd-dev libbpf-dev + apt-get install -y libnl-xfrm-3-dev sphinx-common python3-sphinx-rtd-theme libfdt-dev libjansson-dev + apt-get install -y python3-pyelftools gcc-13 bzip2-doc libacl1-dev libattr1-dev libbz2-dev libgmp-dev + pip3 install meson --upgrade run: | source /artifacts/env DISTRO=ubuntu-`lsb_release -rs` From 7f51e11be41d45874367432426f7204e153afdec Mon Sep 17 00:00:00 2001 From: Nagendra T P Date: Thu, 19 Dec 2024 10:42:13 +0530 Subject: [PATCH 142/271] doc: bump dpdk version bump dpdk version to 24.11.0 Signed-off-by: Nagendra T P --- DPDK_VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DPDK_VERSION b/DPDK_VERSION index b4fb8b86ab..5930a489ba 100644 --- a/DPDK_VERSION +++ b/DPDK_VERSION @@ -1,2 +1,2 @@ BASE_VERSION=23.11.0 -RELEASE_VERSION=24.08.0 +RELEASE_VERSION=24.11.0 \ No newline at end of file From 325dc6e85ac9b5072a1c5b5f8f094a964c5e45c3 Mon Sep 17 00:00:00 2001 From: Nagendra T P Date: Thu, 19 Dec 2024 10:45:38 +0530 Subject: [PATCH 143/271] doc: bump dao version number Bump version number to 25.01.0. Signed-off-by: Nagendra T P --- MRVL_VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MRVL_VERSION b/MRVL_VERSION index 5480d0d067..4b498c089a 100644 --- a/MRVL_VERSION +++ b/MRVL_VERSION @@ -1 +1 @@ -24.09.0 +25.01.0 \ No newline at end of file From 29480990e657cf6fe2d721c9ad821cfadeb077f0 Mon Sep 17 00:00:00 2001 From: Nagendra T P Date: Mon, 23 Dec 2024 10:29:32 +0530 Subject: [PATCH 144/271] ci: add platform name to workflow tags Added platform name to workflow tags Signed-off-by: Nagendra T P --- .github/workflows/build-cn10k.yml | 2 +- .github/workflows/build-cn9k.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-cn10k.yml b/.github/workflows/build-cn10k.yml index 1b72977754..40f519d495 100644 --- a/.github/workflows/build-cn10k.yml +++ b/.github/workflows/build-cn10k.yml @@ -207,7 +207,7 @@ jobs: -H "X-GitHub-Api-Version: 2022-11-28" \ https://api.github.com/repos/marvellembeddedprocessors/packages/dispatches \ -d '{"event_type":"dispatch-event", "client_payload": {"package" : "vpp", - "tag": "vpp-${{ steps.artifacts.outputs.PKG_VERSION_NAME }}-${{ steps.artifacts.outputs.MRVL_PKG_VERSION }}-${{ steps.artifacts.outputs.DISTRO }}-${{ steps.artifacts.outputs.TAG }}", + "tag": "vpp-${{ steps.artifacts.outputs.PKG_VERSION_NAME }}-cn10k-${{ steps.artifacts.outputs.MRVL_PKG_VERSION }}-${{ steps.artifacts.outputs.DISTRO }}-${{ steps.artifacts.outputs.TAG }}", "dpdk_tag" : "dpdk-cn10k-${{ steps.version.outputs.DPDK_BASE_PKG_VERSION }}_${{ steps.version.outputs.DPDK_PKG_VERSION }}-${{ steps.artifacts.outputs.DISTRO }}-${{ steps.version.outputs.DPDK_PKG_VERSION }}", "has_dpdk" : "true", "distro" : "${{ steps.artifacts.outputs.DISTRO }}", "platform" : "cn10k", diff --git a/.github/workflows/build-cn9k.yml b/.github/workflows/build-cn9k.yml index 582260fddc..e2e805c542 100644 --- a/.github/workflows/build-cn9k.yml +++ b/.github/workflows/build-cn9k.yml @@ -206,7 +206,7 @@ jobs: -H "X-GitHub-Api-Version: 2022-11-28" \ https://api.github.com/repos/marvellembeddedprocessors/packages/dispatches \ -d '{"event_type":"dispatch-event", "client_payload": {"package" : "vpp", - "tag": "vpp-${{ steps.artifacts.outputs.PKG_VERSION_NAME }}-${{ steps.artifacts.outputs.MRVL_PKG_VERSION }}-${{ steps.artifacts.outputs.DISTRO }}-${{ steps.artifacts.outputs.TAG }}", + "tag": "vpp-${{ steps.artifacts.outputs.PKG_VERSION_NAME }}-cn9k-${{ steps.artifacts.outputs.MRVL_PKG_VERSION }}-${{ steps.artifacts.outputs.DISTRO }}-${{ steps.artifacts.outputs.TAG }}", "dpdk_tag" : "dpdk-cn9k-${{ steps.version.outputs.DPDK_BASE_PKG_VERSION }}_${{ steps.version.outputs.DPDK_PKG_VERSION }}-${{ steps.artifacts.outputs.DISTRO }}-${{ steps.version.outputs.DPDK_PKG_VERSION }}", "has_dpdk" : "true", "distro" : "${{ steps.artifacts.outputs.DISTRO }}", "platform" : "cn9k", From 7d2db4ed8f3b1fbfcd5992ec8a33f152deae7151 Mon Sep 17 00:00:00 2001 From: Nagendra T P Date: Mon, 13 Jan 2025 15:28:18 +0530 Subject: [PATCH 145/271] ci: maintain cpt package version Added new file to track CPT debian package version Signed-off-by: Nagendra T P --- .github/workflows/build-cn10k.yml | 3 ++- .github/workflows/build-cn9k.yml | 3 ++- CPT_PKG_VERSION | 1 + 3 files changed, 5 insertions(+), 2 deletions(-) create mode 100644 CPT_PKG_VERSION diff --git a/.github/workflows/build-cn10k.yml b/.github/workflows/build-cn10k.yml index 40f519d495..3664001349 100644 --- a/.github/workflows/build-cn10k.yml +++ b/.github/workflows/build-cn10k.yml @@ -59,6 +59,7 @@ jobs: [ -s /tmp/tags ] && NIGHTLY=false || NIGHTLY=true echo "PKG_VERSION_NAME=`./src/scripts/version | awk -F '-' '{print $1}'`" >> "${PWD}/artifacts/env" echo "MRVL_PKG_VERSION=`cat MRVL_VERSION`" >> "${PWD}/artifacts/env" + echo "CPT_PKG_VERSION=`cat CPT_PKG_VERSION`" >> "${PWD}/artifacts/env" echo "DPDK_PKG_VERSION=`cat DPDK_VERSION | grep RELEASE_VERSION | awk -F'=' '{print $2}'`" >> "${PWD}/artifacts/env" echo "DPDK_BASE_PKG_VERSION=`cat DPDK_VERSION | grep BASE_VERSION | awk -F'=' '{print $2}' | awk -F'.' '{print $1"."$2}'`" >> "${PWD}/artifacts/env" echo "PKG_POSTFIX=${PKG_POSTFIX}" >> "${PWD}/artifacts/env" @@ -149,7 +150,7 @@ jobs: echo 'Package: vpp-'$PKG_VERSION_NAME'-cn10k'$PKG_POSTFIX >> DEBIAN/control echo 'Version: '$MRVL_PKG_VERSION >> DEBIAN/control echo "Maintainer: Jerin Jacob (jerinj@marvell.com)" >> DEBIAN/control - echo 'Depends: python3, python3-ply, dpdk-'$DPDK_BASE_PKG_VERSION'-cn10k (= '$DPDK_PKG_VERSION'), cpt-firmware-cn10k'${FW_PKG_POSTFIX}' (= '$MRVL_PKG_VERSION')' >> DEBIAN/control + echo 'Depends: python3, python3-ply, dpdk-'$DPDK_BASE_PKG_VERSION'-cn10k (= '$DPDK_PKG_VERSION'), cpt-firmware-cn10k'${FW_PKG_POSTFIX}' (= '$CPT_PKG_VERSION')' >> DEBIAN/control echo "Architecture: arm64" >> DEBIAN/control echo "Homepage: https://wiki.fd.io/view/VPP" >> DEBIAN/control echo "Description: Vector Packet Processing (VPP) for Octeon10" >> DEBIAN/control diff --git a/.github/workflows/build-cn9k.yml b/.github/workflows/build-cn9k.yml index e2e805c542..bf8164fc9f 100644 --- a/.github/workflows/build-cn9k.yml +++ b/.github/workflows/build-cn9k.yml @@ -59,6 +59,7 @@ jobs: [ -s /tmp/tags ] && NIGHTLY=false || NIGHTLY=true echo "PKG_VERSION_NAME=`./src/scripts/version | awk -F '-' '{print $1}'`" >> "${PWD}/artifacts/env" echo "MRVL_PKG_VERSION=`cat MRVL_VERSION`" >> "${PWD}/artifacts/env" + echo "CPT_PKG_VERSION=`cat CPT_PKG_VERSION`" >> "${PWD}/artifacts/env" echo "DPDK_PKG_VERSION=`cat DPDK_VERSION | grep RELEASE_VERSION | awk -F'=' '{print $2}'`" >> "${PWD}/artifacts/env" echo "DPDK_BASE_PKG_VERSION=`cat DPDK_VERSION | grep BASE_VERSION | awk -F'=' '{print $2}' | awk -F'.' '{print $1"."$2}'`" >> "${PWD}/artifacts/env" echo "PKG_POSTFIX=${PKG_POSTFIX}" >> "${PWD}/artifacts/env" @@ -149,7 +150,7 @@ jobs: echo 'Package: vpp-'$PKG_VERSION_NAME'-cn9k'$PKG_POSTFIX >> DEBIAN/control echo 'Version: '$MRVL_PKG_VERSION >> DEBIAN/control echo "Maintainer: Jerin Jacob (jerinj@marvell.com)" >> DEBIAN/control - echo 'Depends: python3, python3-ply, dpdk-'$DPDK_BASE_PKG_VERSION'-cn9k (= '$DPDK_PKG_VERSION'), cpt-firmware-cn9k'${FW_PKG_POSTFIX}' (= '$MRVL_PKG_VERSION')' >> DEBIAN/control + echo 'Depends: python3, python3-ply, dpdk-'$DPDK_BASE_PKG_VERSION'-cn9k (= '$DPDK_PKG_VERSION'), cpt-firmware-cn9k'${FW_PKG_POSTFIX}' (= '$CPT_PKG_VERSION')' >> DEBIAN/control echo "Architecture: arm64" >> DEBIAN/control echo "Homepage: https://wiki.fd.io/view/VPP" >> DEBIAN/control echo "Description: Vector Packet Processing (VPP) for Octeon9" >> DEBIAN/control diff --git a/CPT_PKG_VERSION b/CPT_PKG_VERSION new file mode 100644 index 0000000000..c6b958db80 --- /dev/null +++ b/CPT_PKG_VERSION @@ -0,0 +1 @@ +24.09.0 \ No newline at end of file From 92b06c9c9fd18a671c2ac7fbb700bd760211cbb0 Mon Sep 17 00:00:00 2001 From: Nagendra T P Date: Wed, 15 Jan 2025 09:48:35 +0530 Subject: [PATCH 146/271] ci: disable draft releases Added draft: false to disable releases as drafts Signed-off-by: Nagendra T P --- .github/workflows/build-cn10k.yml | 1 + .github/workflows/build-cn9k.yml | 2 ++ 2 files changed, 3 insertions(+) diff --git a/.github/workflows/build-cn10k.yml b/.github/workflows/build-cn10k.yml index 3664001349..96ab439de1 100644 --- a/.github/workflows/build-cn10k.yml +++ b/.github/workflows/build-cn10k.yml @@ -190,6 +190,7 @@ jobs: else echo "Release not found" fi + continue-on-error: true - name: Release VPP cn10k package uses: softprops/action-gh-release@v2.0.4 if: ${{ github.event_name == 'push' }} diff --git a/.github/workflows/build-cn9k.yml b/.github/workflows/build-cn9k.yml index bf8164fc9f..d9d4b846bd 100644 --- a/.github/workflows/build-cn9k.yml +++ b/.github/workflows/build-cn9k.yml @@ -190,10 +190,12 @@ jobs: else echo "Release not found" fi + continue-on-error: true - name: Release VPP cn9k package uses: softprops/action-gh-release@v2.0.4 if: ${{ github.event_name == 'push' }} with: + draft: false tag_name: vpp-${{ steps.artifacts.outputs.PKG_VERSION_NAME }}-cn9k-${{ steps.artifacts.outputs.MRVL_PKG_VERSION }}-${{ steps.artifacts.outputs.DISTRO }}-${{ steps.artifacts.outputs.TAG }} files: | ${{ github.workspace }}/artifacts/vpp-${{ steps.artifacts.outputs.PKG_VERSION_NAME }}-cn9k${{ steps.artifacts.outputs.PKG_POSTFIX }}_${{ steps.artifacts.outputs.MRVL_PKG_VERSION }}_arm64.deb From d9e3b28256327777ae82a0f8e94258aba294ae7e Mon Sep 17 00:00:00 2001 From: Nagendra T P Date: Mon, 3 Feb 2025 15:41:58 +0530 Subject: [PATCH 147/271] ci: switch to native arm runners Use native Arm runners for github actions. Signed-off-by: Nagendra T P --- .github/workflows/build-cn10k.yml | 154 +++++++++++++----------------- .github/workflows/build-cn9k.yml | 154 +++++++++++++----------------- 2 files changed, 136 insertions(+), 172 deletions(-) diff --git a/.github/workflows/build-cn10k.yml b/.github/workflows/build-cn10k.yml index 96ab439de1..9410347b3a 100644 --- a/.github/workflows/build-cn10k.yml +++ b/.github/workflows/build-cn10k.yml @@ -15,27 +15,17 @@ permissions: jobs: ubuntu-cn10k-build: name: ubuntu-cn10k-arm64 - runs-on: ubuntu-latest - - strategy: - fail-fast: true - matrix: - include: - - arch: aarch64 - distro: ubuntu22.04 - compiler: gcc - library: static - + runs-on: ubuntu-22.04-arm steps: - name: Checkout sources - uses: actions/checkout@v4.1.6 + uses: actions/checkout@v4.2.2 with: fetch-depth: 0 fetch-tags: true - name: Generate cache keys id: get_ref_keys run: | - echo 'ccache=ccache-${{ matrix.distro }}-${{ matrix.compiler }}-${{ matrix.arch }}-'$(date -u +%Y-w%W) >> $GITHUB_OUTPUT + echo 'ccache=ccache-'$(date -u +%Y-m%M) >> $GITHUB_OUTPUT - name: Retrieve ccache cache uses: actions/cache@v4 with: @@ -50,7 +40,7 @@ jobs: git tag --points-at HEAD > /tmp/tags [ -s /tmp/tags ] && PKG_POSTFIX= || PKG_POSTFIX=-devel FW_PKG_POSTFIX="" - if [ $PKG_POSTFIX = -devel ]; then + if [ ${PKG_POSTFIX} == "-devel" ]; then FW_PKG_POSTFIX="" else FW_PKG_POSTFIX=$PKG_POSTFIX @@ -67,84 +57,76 @@ jobs: echo "NIGHTLY=${NIGHTLY}" >> $GITHUB_OUTPUT echo "DPDK_PKG_VERSION=${DPDK_PKG_VERSION}" >> $GITHUB_OUTPUT echo "DPDK_BASE_PKG_VERSION=${DPDK_BASE_PKG_VERSION}" >> $GITHUB_OUTPUT - - uses: uraimo/run-on-arch-action@v2.8.1 - name: Build VPP and generate package + - name: Build VPP and generate package id: build - with: - arch: ${{ matrix.arch }} - distro: ${{ matrix.distro }} - githubToken: ${{ github.token }} - setup: | + run: | mkdir -p ~/.ccache - dockerRunArgs: | - --volume "${PWD}/artifacts:/artifacts" - --volume "${HOME}/.ccache:/root/.ccache" - shell: /bin/bash - install: | - apt-get update -q -y - apt-get install -y apt-utils gcc meson sudo make dialog ccache git build-essential software-properties-common - add-apt-repository -y ppa:ubuntu-toolchain-r/test - apt-get update -q -y - apt-get install -y aspell aspell-en autopoint autotools-dev binfmt-support binutils binutils-aarch64-linux-gnu - apt-get install -y binutils-common bsdextrautils bzip2 ca-certificates clang-14 cmake-data cpp cpp-11 cpp-12 dbus - apt-get install -y dctrl-tools debugedit dh-autoreconf dh-elpa-helper dh-strip-nondeterminism dictionaries-common dirmngr - apt-get install -y distro-info-data dpkg-dev dwz emacsen-common fakeroot file fontconfig-config fonts-dejavu-core g++ g++-11 - apt-get install -y gcc gcc-11 gcc-11-base gcc-12 gettext gettext-base git-man gnupg gnupg-l10n gnupg-utils gpg gpg-agent - apt-get install -y gpg-wks-client gpg-wks-server gpgconf gpgsm groff-base hunspell-en-us icu-devtools intltool-debian - apt-get install -y javascript-common kmod less libalgorithm-diff-perl libalgorithm-diff-xs-perl libalgorithm-merge-perl - apt-get install -y libapparmor1 libapr1 libarchive-cpio-perl libarchive-zip-perl libarchive13 libasan6 libasan8 libaspell15 - apt-get install -y libassuan0 libatomic1 libbcg729-0 libbinutils libbrotli1 libbsd0 libc-ares2 libc-dev-bin libc-devtools - apt-get install -y libc6-dev libcap2-bin libcbor0.8 libcc1-0 libclang-common-14-dev libclang-cpp11 libclang-cpp14 - apt-get install -y libclang1-14 libcommon-sense-perl libconfuse-common libconfuse-doc libconfuse2 libcrypt-dev libctf-nobfd0 - apt-get install -y libctf0 libcurl3-gnutls libcurl4 libdbus-1-3 libdbus-1-dev libdebhelper-perl libdeflate0 libdpkg-perl - apt-get install -y libdw1 libedit2 libelf1 libenchant-2-2 liberror-perl libexpat1 libexpat1-dev libfakeroot libfido2-1 - apt-get install -y libfile-fcntllock-perl libfile-stripnondeterminism-perl libfontconfig1 libfreetype6 libgc1 libgcc-11-dev - apt-get install -y libgcc-12-dev libgd-perl libgd3 libgdbm-compat4 libgdbm6 libglib2.0-0 libglib2.0-data libgomp1 - apt-get install -y libhiredis0.14 libhunspell-1.7-0 libhwasan0 libicu-dev libicu70 libiperf0 libisl23 libitm1 libjbig0 - apt-get install -y libjpeg-turbo8 libjpeg8 libjs-jquery libjs-sphinxdoc libjs-underscore libjson-perl libjson-xs-perl - apt-get install -y libjsoncpp25 libkmod2 libksba8 libldap-2.5-0 libldap-common libllvm11 libllvm14 liblocale-gettext-perl - apt-get install -y liblsan0 libltdl-dev libltdl7 liblua5.2-0 libmagic-mgc libmagic1 libmail-sendmail-perl libmaxminddb0 - apt-get install -y libmd0 libmnl0 libmpc3 libmpdec3 libmpfr6 libncurses-dev libnghttp2-14 libnl-3-200 libnl-genl-3-200 - apt-get install -y libnl-route-3-200 libnpth0 libnsl-dev libnuma1 libobjc-11-dev libobjc4 libpam-cap libpcap0.8 - apt-get install -y libpcap0.8-dev libperl5.34 libperlio-gzip-perl libpfm4 libpipeline1 libpng16-16 libpsl5 libpython3-dev - apt-get install -y libpython3-stdlib libpython3.10 libpython3.10-dev libpython3.10-minimal libpython3.10-stdlib libreadline8 - apt-get install -y librhash0 librtmp1 libsasl2-2 libsasl2-modules libsasl2-modules-db libsbc1 libsctp-dev libsctp1 - apt-get install -y libsigsegv2 libsmi2ldbl libsnappy1v5 libspandsp2 libspeexdsp1 libsqlite3-0 libssh-4 libssh-gcrypt-4 - apt-get install -y libstdc++-11-dev libsub-override-perl libsubunit-dev libsubunit0 libsys-hostname-long-perl - apt-get install -y libtext-iconv-perl libtiff5 libtinfo-dev libtirpc-dev libtsan0 libtsan2 libtypes-serialiser-perl - apt-get install -y libubsan1 libuchardet0 libuv1 libwebp7 libwireshark-data libwireshark15 libwiretap12 libwsutil13 libx11-6 - apt-get install -y libx11-data libxau6 libxcb1 libxdmcp6 libxext6 libxml2 libxml2-dev libxmuu1 libxpm4 libxslt1.1 - apt-get install -y libyaml-0-2 libz3-4 libz3-dev linux-headers-5.15.0-107 linux-headers-5.15.0-107-generic - apt-get install -y linux-headers-generic linux-libc-dev llvm-14 llvm-14-dev llvm-14-linker-tools llvm-14-runtime - apt-get install -y llvm-14-tools lsb-release lto-disabled-list m4 man-db manpages manpages-dev media-types netbase - apt-get install -y openssh-client openssl patch perl perl-modules-5.34 pinentry-curses po-debconf publicsuffix - apt-get install -y python-babel-localedata python3 python3-attr python3-babel python3-bs4 python3-certifi python3-chardet - apt-get install -y python3-distlib python3-distutils python3-filelock python3-html5lib python3-idna - apt-get install -y python3-importlib-metadata python3-jinja2 python3-lib2to3 python3-lxml python3-markupsafe python3-minimal - apt-get install -y python3-more-itertools python3-pip-whl python3-pkg-resources python3-platformdirs python3-pygments - apt-get install -y python3-pyrsistent python3-requests python3-setuptools-whl python3-six python3-soupsieve python3-tz - apt-get install -y python3-urllib3 python3-webencodings python3-wheel python3-wheel-whl python3-zipp python3.10 - apt-get install -y python3.10-dev python3.10-minimal python3.10-venv readline-common rpcsvc-proto shared-mime-info tzdata - apt-get install -y ucf uuid-dev wireshark-common xauth xdg-user-dirs xz-utils zlib1g-dev libgmpxx4ldbl - apt-get install -y check chrpath clang clang-format-11 cmake cscope debhelper dh-python dkms liblz4-dev - apt-get install -y ethtool exuberant-ctags gcovr git-review iperf iperf3 lcov libapr1-dev liblzma-dev wget - apt-get install -y libconfuse-dev libelf-dev libffi7 libmnl-dev libnl-3-dev libnl-route-3-dev libzstd-dev - apt-get install -y libnuma-dev libpcap-dev libssl-dev nasm ninja-build python3-all python3-dev nettle-dev - apt-get install -y python3-jsonschema pkg-config python3-pip python3-ply python3-setuptools python3-venv - apt-get install -y python3-virtualenv tshark lsb-release doxygen libarchive-dev libbsd-dev libbpf-dev - apt-get install -y libnl-xfrm-3-dev sphinx-common python3-sphinx-rtd-theme libfdt-dev libjansson-dev - apt-get install -y python3-pyelftools gcc-13 bzip2-doc libacl1-dev libattr1-dev libbz2-dev libgmp-dev - pip3 install meson --upgrade - run: | - source /artifacts/env + sudo apt-get update -q -y + sudo apt-get install -y apt-utils gcc meson sudo make dialog ccache git build-essential software-properties-common + sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test + sudo apt-get update -q -y + sudo apt-get install -y aspell aspell-en autopoint autotools-dev binfmt-support binutils binutils-aarch64-linux-gnu + sudo apt-get install -y binutils-common bsdextrautils bzip2 ca-certificates clang-14 cmake-data cpp cpp-11 cpp-12 dbus + sudo apt-get install -y dctrl-tools debugedit dh-autoreconf dh-elpa-helper dh-strip-nondeterminism dictionaries-common dirmngr + sudo apt-get install -y distro-info-data dpkg-dev dwz emacsen-common fakeroot file fontconfig-config fonts-dejavu-core g++ g++-11 + sudo apt-get install -y gcc gcc-11 gcc-11-base gcc-12 gettext gettext-base git-man gnupg gnupg-l10n gnupg-utils gpg gpg-agent + sudo apt-get install -y gpg-wks-client gpg-wks-server gpgconf gpgsm groff-base hunspell-en-us icu-devtools intltool-debian + sudo apt-get install -y javascript-common kmod less libalgorithm-diff-perl libalgorithm-diff-xs-perl libalgorithm-merge-perl + sudo apt-get install -y libapparmor1 libapr1 libarchive-cpio-perl libarchive-zip-perl libarchive13 libasan6 libasan8 libaspell15 + sudo apt-get install -y libassuan0 libatomic1 libbcg729-0 libbinutils libbrotli1 libbsd0 libc-ares2 libc-dev-bin libc-devtools + sudo apt-get install -y libc6-dev libcap2-bin libcbor0.8 libcc1-0 libclang-common-14-dev libclang-cpp11 libclang-cpp14 + sudo apt-get install -y libclang1-14 libcommon-sense-perl libconfuse-common libconfuse-doc libconfuse2 libcrypt-dev libctf-nobfd0 + sudo apt-get install -y libctf0 libcurl3-gnutls libcurl4 libdbus-1-3 libdbus-1-dev libdebhelper-perl libdeflate0 libdpkg-perl + sudo apt-get install -y libdw1 libedit2 libelf1 libenchant-2-2 liberror-perl libexpat1 libexpat1-dev libfakeroot libfido2-1 + sudo apt-get install -y libfile-fcntllock-perl libfile-stripnondeterminism-perl libfontconfig1 libfreetype6 libgc1 libgcc-11-dev + sudo apt-get install -y libgcc-12-dev libgd-perl libgd3 libgdbm-compat4 libgdbm6 libglib2.0-0 libglib2.0-data libgomp1 + sudo apt-get install -y libhiredis0.14 libhunspell-1.7-0 libhwasan0 libicu-dev libicu70 libiperf0 libisl23 libitm1 libjbig0 + sudo apt-get install -y libjpeg-turbo8 libjpeg8 libjs-jquery libjs-sphinxdoc libjs-underscore libjson-perl libjson-xs-perl + sudo apt-get install -y libjsoncpp25 libkmod2 libksba8 libldap-2.5-0 libldap-common libllvm11 libllvm14 liblocale-gettext-perl + sudo apt-get install -y liblsan0 libltdl-dev libltdl7 liblua5.2-0 libmagic-mgc libmagic1 libmail-sendmail-perl libmaxminddb0 + sudo apt-get install -y libmd0 libmnl0 libmpc3 libmpdec3 libmpfr6 libncurses-dev libnghttp2-14 libnl-3-200 libnl-genl-3-200 + sudo apt-get install -y libnl-route-3-200 libnpth0 libnsl-dev libnuma1 libobjc-11-dev libobjc4 libpam-cap libpcap0.8 + sudo apt-get install -y libpcap0.8-dev libperl5.34 libperlio-gzip-perl libpfm4 libpipeline1 libpng16-16 libpsl5 libpython3-dev + sudo apt-get install -y libpython3-stdlib libpython3.10 libpython3.10-dev libpython3.10-minimal libpython3.10-stdlib libreadline8 + sudo apt-get install -y librhash0 librtmp1 libsasl2-2 libsasl2-modules libsasl2-modules-db libsbc1 libsctp-dev libsctp1 + sudo apt-get install -y libsigsegv2 libsmi2ldbl libsnappy1v5 libspandsp2 libspeexdsp1 libsqlite3-0 libssh-4 libssh-gcrypt-4 + sudo apt-get install -y libstdc++-11-dev libsub-override-perl libsubunit-dev libsubunit0 libsys-hostname-long-perl + sudo apt-get install -y libtext-iconv-perl libtiff5 libtinfo-dev libtirpc-dev libtsan0 libtsan2 libtypes-serialiser-perl + sudo apt-get install -y libubsan1 libuchardet0 libuv1 libwebp7 libwireshark-data libwireshark15 libwiretap12 libwsutil13 libx11-6 + sudo apt-get install -y libx11-data libxau6 libxcb1 libxdmcp6 libxext6 libxml2 libxml2-dev libxmuu1 libxpm4 libxslt1.1 + sudo apt-get install -y libyaml-0-2 libz3-4 libz3-dev linux-headers-5.15.0-107 linux-headers-5.15.0-107-generic + sudo apt-get install -y linux-headers-generic linux-libc-dev llvm-14 llvm-14-dev llvm-14-linker-tools llvm-14-runtime + sudo apt-get install -y llvm-14-tools lsb-release lto-disabled-list m4 man-db manpages manpages-dev media-types netbase + sudo apt-get install -y openssh-client openssl patch perl perl-modules-5.34 pinentry-curses po-debconf publicsuffix + sudo apt-get install -y python-babel-localedata python3 python3-attr python3-babel python3-bs4 python3-certifi python3-chardet + sudo apt-get install -y python3-distlib python3-distutils python3-filelock python3-html5lib python3-idna + sudo apt-get install -y python3-importlib-metadata python3-jinja2 python3-lib2to3 python3-lxml python3-markupsafe python3-minimal + sudo apt-get install -y python3-more-itertools python3-pip-whl python3-pkg-resources python3-platformdirs python3-pygments + sudo apt-get install -y python3-pyrsistent python3-requests python3-setuptools-whl python3-six python3-soupsieve python3-tz + sudo apt-get install -y python3-urllib3 python3-webencodings python3-wheel python3-wheel-whl python3-zipp python3.10 + sudo apt-get install -y python3.10-dev python3.10-minimal python3.10-venv readline-common rpcsvc-proto shared-mime-info tzdata + sudo apt-get install -y ucf uuid-dev wireshark-common xauth xdg-user-dirs xz-utils zlib1g-dev libgmpxx4ldbl + sudo apt-get install -y check chrpath clang clang-format-11 cmake cscope debhelper dh-python dkms liblz4-dev + sudo apt-get install -y ethtool exuberant-ctags gcovr git-review iperf iperf3 lcov libapr1-dev liblzma-dev wget + sudo apt-get install -y libconfuse-dev libelf-dev libffi7 libmnl-dev libnl-3-dev libnl-route-3-dev libzstd-dev + sudo apt-get install -y libnuma-dev libpcap-dev libssl-dev nasm ninja-build python3-all python3-dev nettle-dev + sudo apt-get install -y python3-jsonschema pkg-config python3-pip python3-ply python3-setuptools python3-venv + sudo apt-get install -y python3-virtualenv tshark lsb-release doxygen libarchive-dev libbsd-dev libbpf-dev + sudo apt-get install -y libnl-xfrm-3-dev sphinx-common python3-sphinx-rtd-theme libfdt-dev libjansson-dev + sudo apt-get install -y python3-pyelftools gcc-13 bzip2-doc libacl1-dev libattr1-dev libbz2-dev libgmp-dev + sudo pip3 install meson --upgrade + BASE_DIR=${PWD} + source ${BASE_DIR}/artifacts/env DISTRO=ubuntu-`lsb_release -rs` - echo "DISTRO=${DISTRO}" >> /artifacts/env - echo "cache_dir = /root/.ccache" > /root/.ccache/ccache.conf + echo "DISTRO=${DISTRO}" >> ${BASE_DIR}/artifacts/env + echo "cache_dir = ~/.ccache" > ~/.ccache/ccache.conf ccache -p git config --global --add safe.directory "${PWD}" APT_ARGS='-y -q' make install-deps make build-release VPP_PLATFORM=octeon10 mkdir -p "${PWD}/install/DEBIAN" + mkdir -p "${PWD}/install/usr/share/vpp/api" + cp -r build-root/install-vpp-native/vpp/share/vpp/api/* "${PWD}/install/usr/share/vpp/api" mv build-root/install-vpp-native/vpp/* install/. cd "${PWD}/install" echo 'Package: vpp-'$PKG_VERSION_NAME'-cn10k'$PKG_POSTFIX >> DEBIAN/control @@ -157,7 +139,7 @@ jobs: cd - mv "${PWD}/install" "${PWD}/vpp-${PKG_VERSION_NAME}-cn10k${PKG_POSTFIX}_${MRVL_PKG_VERSION}_arm64" dpkg --build "vpp-${PKG_VERSION_NAME}-cn10k${PKG_POSTFIX}_${MRVL_PKG_VERSION}_arm64" - cp "vpp-${PKG_VERSION_NAME}-cn10k${PKG_POSTFIX}_${MRVL_PKG_VERSION}_arm64.deb" /artifacts/. + cp "vpp-${PKG_VERSION_NAME}-cn10k${PKG_POSTFIX}_${MRVL_PKG_VERSION}_arm64.deb" ${BASE_DIR}/artifacts/. - name: Export version name id: artifacts run: | @@ -213,4 +195,4 @@ jobs: "dpdk_tag" : "dpdk-cn10k-${{ steps.version.outputs.DPDK_BASE_PKG_VERSION }}_${{ steps.version.outputs.DPDK_PKG_VERSION }}-${{ steps.artifacts.outputs.DISTRO }}-${{ steps.version.outputs.DPDK_PKG_VERSION }}", "has_dpdk" : "true", "distro" : "${{ steps.artifacts.outputs.DISTRO }}", "platform" : "cn10k", - "devel": "${{ steps.artifacts.outputs.IS_DEVEL }}"}}' + "devel": "${{ steps.artifacts.outputs.IS_DEVEL }}"}}' \ No newline at end of file diff --git a/.github/workflows/build-cn9k.yml b/.github/workflows/build-cn9k.yml index d9d4b846bd..924c15b2eb 100644 --- a/.github/workflows/build-cn9k.yml +++ b/.github/workflows/build-cn9k.yml @@ -15,27 +15,17 @@ permissions: jobs: ubuntu-cn9k-build: name: ubuntu-cn9k-arm64 - runs-on: ubuntu-latest - - strategy: - fail-fast: true - matrix: - include: - - arch: aarch64 - distro: ubuntu22.04 - compiler: gcc - library: static - + runs-on: ubuntu-22.04-arm steps: - name: Checkout sources - uses: actions/checkout@v4.1.6 + uses: actions/checkout@v4.2.2 with: fetch-depth: 0 fetch-tags: true - name: Generate cache keys id: get_ref_keys run: | - echo 'ccache=ccache-${{ matrix.distro }}-${{ matrix.compiler }}-${{ matrix.arch }}-'$(date -u +%Y-w%W) >> $GITHUB_OUTPUT + echo 'ccache=ccache-'$(date -u +%Y-m%M) >> $GITHUB_OUTPUT - name: Retrieve ccache cache uses: actions/cache@v4 with: @@ -50,7 +40,7 @@ jobs: git tag --points-at HEAD > /tmp/tags [ -s /tmp/tags ] && PKG_POSTFIX= || PKG_POSTFIX=-devel FW_PKG_POSTFIX="" - if [ ${PKG_POSTFIX} == "-devel" ]; then + if [ $PKG_POSTFIX = -devel ]; then FW_PKG_POSTFIX="" else FW_PKG_POSTFIX=$PKG_POSTFIX @@ -67,84 +57,76 @@ jobs: echo "NIGHTLY=${NIGHTLY}" >> $GITHUB_OUTPUT echo "DPDK_PKG_VERSION=${DPDK_PKG_VERSION}" >> $GITHUB_OUTPUT echo "DPDK_BASE_PKG_VERSION=${DPDK_BASE_PKG_VERSION}" >> $GITHUB_OUTPUT - - uses: uraimo/run-on-arch-action@v2.8.1 - name: Build VPP and generate package + - name: Build VPP and generate package id: build - with: - arch: ${{ matrix.arch }} - distro: ${{ matrix.distro }} - githubToken: ${{ github.token }} - setup: | + run: | mkdir -p ~/.ccache - dockerRunArgs: | - --volume "${PWD}/artifacts:/artifacts" - --volume "${HOME}/.ccache:/root/.ccache" - shell: /bin/bash - install: | - apt-get update -q -y - apt-get install -y apt-utils gcc meson sudo make dialog ccache git build-essential software-properties-common - add-apt-repository -y ppa:ubuntu-toolchain-r/test - apt-get update -q -y - apt-get install -y aspell aspell-en autopoint autotools-dev binfmt-support binutils binutils-aarch64-linux-gnu - apt-get install -y binutils-common bsdextrautils bzip2 ca-certificates clang-14 cmake-data cpp cpp-11 cpp-12 dbus - apt-get install -y dctrl-tools debugedit dh-autoreconf dh-elpa-helper dh-strip-nondeterminism dictionaries-common dirmngr - apt-get install -y distro-info-data dpkg-dev dwz emacsen-common fakeroot file fontconfig-config fonts-dejavu-core g++ g++-11 - apt-get install -y gcc gcc-11 gcc-11-base gcc-12 gettext gettext-base git-man gnupg gnupg-l10n gnupg-utils gpg gpg-agent - apt-get install -y gpg-wks-client gpg-wks-server gpgconf gpgsm groff-base hunspell-en-us icu-devtools intltool-debian - apt-get install -y javascript-common kmod less libalgorithm-diff-perl libalgorithm-diff-xs-perl libalgorithm-merge-perl - apt-get install -y libapparmor1 libapr1 libarchive-cpio-perl libarchive-zip-perl libarchive13 libasan6 libasan8 libaspell15 - apt-get install -y libassuan0 libatomic1 libbcg729-0 libbinutils libbrotli1 libbsd0 libc-ares2 libc-dev-bin libc-devtools - apt-get install -y libc6-dev libcap2-bin libcbor0.8 libcc1-0 libclang-common-14-dev libclang-cpp11 libclang-cpp14 - apt-get install -y libclang1-14 libcommon-sense-perl libconfuse-common libconfuse-doc libconfuse2 libcrypt-dev libctf-nobfd0 - apt-get install -y libctf0 libcurl3-gnutls libcurl4 libdbus-1-3 libdbus-1-dev libdebhelper-perl libdeflate0 libdpkg-perl - apt-get install -y libdw1 libedit2 libelf1 libenchant-2-2 liberror-perl libexpat1 libexpat1-dev libfakeroot libfido2-1 - apt-get install -y libfile-fcntllock-perl libfile-stripnondeterminism-perl libfontconfig1 libfreetype6 libgc1 libgcc-11-dev - apt-get install -y libgcc-12-dev libgd-perl libgd3 libgdbm-compat4 libgdbm6 libglib2.0-0 libglib2.0-data libgomp1 - apt-get install -y libhiredis0.14 libhunspell-1.7-0 libhwasan0 libicu-dev libicu70 libiperf0 libisl23 libitm1 libjbig0 - apt-get install -y libjpeg-turbo8 libjpeg8 libjs-jquery libjs-sphinxdoc libjs-underscore libjson-perl libjson-xs-perl - apt-get install -y libjsoncpp25 libkmod2 libksba8 libldap-2.5-0 libldap-common libllvm11 libllvm14 liblocale-gettext-perl - apt-get install -y liblsan0 libltdl-dev libltdl7 liblua5.2-0 libmagic-mgc libmagic1 libmail-sendmail-perl libmaxminddb0 - apt-get install -y libmd0 libmnl0 libmpc3 libmpdec3 libmpfr6 libncurses-dev libnghttp2-14 libnl-3-200 libnl-genl-3-200 - apt-get install -y libnl-route-3-200 libnpth0 libnsl-dev libnuma1 libobjc-11-dev libobjc4 libpam-cap libpcap0.8 - apt-get install -y libpcap0.8-dev libperl5.34 libperlio-gzip-perl libpfm4 libpipeline1 libpng16-16 libpsl5 libpython3-dev - apt-get install -y libpython3-stdlib libpython3.10 libpython3.10-dev libpython3.10-minimal libpython3.10-stdlib libreadline8 - apt-get install -y librhash0 librtmp1 libsasl2-2 libsasl2-modules libsasl2-modules-db libsbc1 libsctp-dev libsctp1 - apt-get install -y libsigsegv2 libsmi2ldbl libsnappy1v5 libspandsp2 libspeexdsp1 libsqlite3-0 libssh-4 libssh-gcrypt-4 - apt-get install -y libstdc++-11-dev libsub-override-perl libsubunit-dev libsubunit0 libsys-hostname-long-perl - apt-get install -y libtext-iconv-perl libtiff5 libtinfo-dev libtirpc-dev libtsan0 libtsan2 libtypes-serialiser-perl - apt-get install -y libubsan1 libuchardet0 libuv1 libwebp7 libwireshark-data libwireshark15 libwiretap12 libwsutil13 libx11-6 - apt-get install -y libx11-data libxau6 libxcb1 libxdmcp6 libxext6 libxml2 libxml2-dev libxmuu1 libxpm4 libxslt1.1 - apt-get install -y libyaml-0-2 libz3-4 libz3-dev linux-headers-5.15.0-107 linux-headers-5.15.0-107-generic - apt-get install -y linux-headers-generic linux-libc-dev llvm-14 llvm-14-dev llvm-14-linker-tools llvm-14-runtime - apt-get install -y llvm-14-tools lsb-release lto-disabled-list m4 man-db manpages manpages-dev media-types netbase - apt-get install -y openssh-client openssl patch perl perl-modules-5.34 pinentry-curses po-debconf publicsuffix - apt-get install -y python-babel-localedata python3 python3-attr python3-babel python3-bs4 python3-certifi python3-chardet - apt-get install -y python3-distlib python3-distutils python3-filelock python3-html5lib python3-idna - apt-get install -y python3-importlib-metadata python3-jinja2 python3-lib2to3 python3-lxml python3-markupsafe python3-minimal - apt-get install -y python3-more-itertools python3-pip-whl python3-pkg-resources python3-platformdirs python3-pygments - apt-get install -y python3-pyrsistent python3-requests python3-setuptools-whl python3-six python3-soupsieve python3-tz - apt-get install -y python3-urllib3 python3-webencodings python3-wheel python3-wheel-whl python3-zipp python3.10 - apt-get install -y python3.10-dev python3.10-minimal python3.10-venv readline-common rpcsvc-proto shared-mime-info tzdata - apt-get install -y ucf uuid-dev wireshark-common xauth xdg-user-dirs xz-utils zlib1g-dev libgmpxx4ldbl - apt-get install -y check chrpath clang clang-format-11 cmake cscope debhelper dh-python dkms liblz4-dev - apt-get install -y ethtool exuberant-ctags gcovr git-review iperf iperf3 lcov libapr1-dev liblzma-dev wget - apt-get install -y libconfuse-dev libelf-dev libffi7 libmnl-dev libnl-3-dev libnl-route-3-dev libzstd-dev - apt-get install -y libnuma-dev libpcap-dev libssl-dev nasm ninja-build python3-all python3-dev nettle-dev - apt-get install -y python3-jsonschema pkg-config python3-pip python3-ply python3-setuptools python3-venv - apt-get install -y python3-virtualenv tshark lsb-release doxygen libarchive-dev libbsd-dev libbpf-dev - apt-get install -y libnl-xfrm-3-dev sphinx-common python3-sphinx-rtd-theme libfdt-dev libjansson-dev - apt-get install -y python3-pyelftools gcc-13 bzip2-doc libacl1-dev libattr1-dev libbz2-dev libgmp-dev - pip3 install meson --upgrade - run: | - source /artifacts/env + sudo apt-get update -q -y + sudo apt-get install -y apt-utils gcc meson sudo make dialog ccache git build-essential software-properties-common + sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test + sudo apt-get update -q -y + sudo apt-get install -y aspell aspell-en autopoint autotools-dev binfmt-support binutils binutils-aarch64-linux-gnu + sudo apt-get install -y binutils-common bsdextrautils bzip2 ca-certificates clang-14 cmake-data cpp cpp-11 cpp-12 dbus + sudo apt-get install -y dctrl-tools debugedit dh-autoreconf dh-elpa-helper dh-strip-nondeterminism dictionaries-common dirmngr + sudo apt-get install -y distro-info-data dpkg-dev dwz emacsen-common fakeroot file fontconfig-config fonts-dejavu-core g++ g++-11 + sudo apt-get install -y gcc gcc-11 gcc-11-base gcc-12 gettext gettext-base git-man gnupg gnupg-l10n gnupg-utils gpg gpg-agent + sudo apt-get install -y gpg-wks-client gpg-wks-server gpgconf gpgsm groff-base hunspell-en-us icu-devtools intltool-debian + sudo apt-get install -y javascript-common kmod less libalgorithm-diff-perl libalgorithm-diff-xs-perl libalgorithm-merge-perl + sudo apt-get install -y libapparmor1 libapr1 libarchive-cpio-perl libarchive-zip-perl libarchive13 libasan6 libasan8 libaspell15 + sudo apt-get install -y libassuan0 libatomic1 libbcg729-0 libbinutils libbrotli1 libbsd0 libc-ares2 libc-dev-bin libc-devtools + sudo apt-get install -y libc6-dev libcap2-bin libcbor0.8 libcc1-0 libclang-common-14-dev libclang-cpp11 libclang-cpp14 + sudo apt-get install -y libclang1-14 libcommon-sense-perl libconfuse-common libconfuse-doc libconfuse2 libcrypt-dev libctf-nobfd0 + sudo apt-get install -y libctf0 libcurl3-gnutls libcurl4 libdbus-1-3 libdbus-1-dev libdebhelper-perl libdeflate0 libdpkg-perl + sudo apt-get install -y libdw1 libedit2 libelf1 libenchant-2-2 liberror-perl libexpat1 libexpat1-dev libfakeroot libfido2-1 + sudo apt-get install -y libfile-fcntllock-perl libfile-stripnondeterminism-perl libfontconfig1 libfreetype6 libgc1 libgcc-11-dev + sudo apt-get install -y libgcc-12-dev libgd-perl libgd3 libgdbm-compat4 libgdbm6 libglib2.0-0 libglib2.0-data libgomp1 + sudo apt-get install -y libhiredis0.14 libhunspell-1.7-0 libhwasan0 libicu-dev libicu70 libiperf0 libisl23 libitm1 libjbig0 + sudo apt-get install -y libjpeg-turbo8 libjpeg8 libjs-jquery libjs-sphinxdoc libjs-underscore libjson-perl libjson-xs-perl + sudo apt-get install -y libjsoncpp25 libkmod2 libksba8 libldap-2.5-0 libldap-common libllvm11 libllvm14 liblocale-gettext-perl + sudo apt-get install -y liblsan0 libltdl-dev libltdl7 liblua5.2-0 libmagic-mgc libmagic1 libmail-sendmail-perl libmaxminddb0 + sudo apt-get install -y libmd0 libmnl0 libmpc3 libmpdec3 libmpfr6 libncurses-dev libnghttp2-14 libnl-3-200 libnl-genl-3-200 + sudo apt-get install -y libnl-route-3-200 libnpth0 libnsl-dev libnuma1 libobjc-11-dev libobjc4 libpam-cap libpcap0.8 + sudo apt-get install -y libpcap0.8-dev libperl5.34 libperlio-gzip-perl libpfm4 libpipeline1 libpng16-16 libpsl5 libpython3-dev + sudo apt-get install -y libpython3-stdlib libpython3.10 libpython3.10-dev libpython3.10-minimal libpython3.10-stdlib libreadline8 + sudo apt-get install -y librhash0 librtmp1 libsasl2-2 libsasl2-modules libsasl2-modules-db libsbc1 libsctp-dev libsctp1 + sudo apt-get install -y libsigsegv2 libsmi2ldbl libsnappy1v5 libspandsp2 libspeexdsp1 libsqlite3-0 libssh-4 libssh-gcrypt-4 + sudo apt-get install -y libstdc++-11-dev libsub-override-perl libsubunit-dev libsubunit0 libsys-hostname-long-perl + sudo apt-get install -y libtext-iconv-perl libtiff5 libtinfo-dev libtirpc-dev libtsan0 libtsan2 libtypes-serialiser-perl + sudo apt-get install -y libubsan1 libuchardet0 libuv1 libwebp7 libwireshark-data libwireshark15 libwiretap12 libwsutil13 libx11-6 + sudo apt-get install -y libx11-data libxau6 libxcb1 libxdmcp6 libxext6 libxml2 libxml2-dev libxmuu1 libxpm4 libxslt1.1 + sudo apt-get install -y libyaml-0-2 libz3-4 libz3-dev linux-headers-5.15.0-107 linux-headers-5.15.0-107-generic + sudo apt-get install -y linux-headers-generic linux-libc-dev llvm-14 llvm-14-dev llvm-14-linker-tools llvm-14-runtime + sudo apt-get install -y llvm-14-tools lsb-release lto-disabled-list m4 man-db manpages manpages-dev media-types netbase + sudo apt-get install -y openssh-client openssl patch perl perl-modules-5.34 pinentry-curses po-debconf publicsuffix + sudo apt-get install -y python-babel-localedata python3 python3-attr python3-babel python3-bs4 python3-certifi python3-chardet + sudo apt-get install -y python3-distlib python3-distutils python3-filelock python3-html5lib python3-idna + sudo apt-get install -y python3-importlib-metadata python3-jinja2 python3-lib2to3 python3-lxml python3-markupsafe python3-minimal + sudo apt-get install -y python3-more-itertools python3-pip-whl python3-pkg-resources python3-platformdirs python3-pygments + sudo apt-get install -y python3-pyrsistent python3-requests python3-setuptools-whl python3-six python3-soupsieve python3-tz + sudo apt-get install -y python3-urllib3 python3-webencodings python3-wheel python3-wheel-whl python3-zipp python3.10 + sudo apt-get install -y python3.10-dev python3.10-minimal python3.10-venv readline-common rpcsvc-proto shared-mime-info tzdata + sudo apt-get install -y ucf uuid-dev wireshark-common xauth xdg-user-dirs xz-utils zlib1g-dev libgmpxx4ldbl + sudo apt-get install -y check chrpath clang clang-format-11 cmake cscope debhelper dh-python dkms liblz4-dev + sudo apt-get install -y ethtool exuberant-ctags gcovr git-review iperf iperf3 lcov libapr1-dev liblzma-dev wget + sudo apt-get install -y libconfuse-dev libelf-dev libffi7 libmnl-dev libnl-3-dev libnl-route-3-dev libzstd-dev + sudo apt-get install -y libnuma-dev libpcap-dev libssl-dev nasm ninja-build python3-all python3-dev nettle-dev + sudo apt-get install -y python3-jsonschema pkg-config python3-pip python3-ply python3-setuptools python3-venv + sudo apt-get install -y python3-virtualenv tshark lsb-release doxygen libarchive-dev libbsd-dev libbpf-dev + sudo apt-get install -y libnl-xfrm-3-dev sphinx-common python3-sphinx-rtd-theme libfdt-dev libjansson-dev + sudo apt-get install -y python3-pyelftools gcc-13 bzip2-doc libacl1-dev libattr1-dev libbz2-dev libgmp-dev + sudo pip3 install meson --upgrade + BASE_DIR=${PWD} + source ${BASE_DIR}/artifacts/env DISTRO=ubuntu-`lsb_release -rs` - echo "DISTRO=${DISTRO}" >> /artifacts/env - echo "cache_dir = /root/.ccache" > /root/.ccache/ccache.conf + echo "DISTRO=${DISTRO}" >> ${BASE_DIR}/artifacts/env + echo "cache_dir = ~/.ccache" > ~/.ccache/ccache.conf ccache -p git config --global --add safe.directory "${PWD}" APT_ARGS='-y -q' make install-deps make build-release VPP_PLATFORM=octeon9 mkdir -p "${PWD}/install/DEBIAN" + mkdir -p "${PWD}/install/usr/share/vpp/api" + cp -r build-root/install-vpp-native/vpp/share/vpp/api/* "${PWD}/install/usr/share/vpp/api" mv build-root/install-vpp-native/vpp/* install/. cd "${PWD}/install" echo 'Package: vpp-'$PKG_VERSION_NAME'-cn9k'$PKG_POSTFIX >> DEBIAN/control @@ -157,7 +139,7 @@ jobs: cd - mv "${PWD}/install" "${PWD}/vpp-${PKG_VERSION_NAME}-cn9k${PKG_POSTFIX}_${MRVL_PKG_VERSION}_arm64" dpkg --build "vpp-${PKG_VERSION_NAME}-cn9k${PKG_POSTFIX}_${MRVL_PKG_VERSION}_arm64" - cp "vpp-${PKG_VERSION_NAME}-cn9k${PKG_POSTFIX}_${MRVL_PKG_VERSION}_arm64.deb" /artifacts/. + cp "vpp-${PKG_VERSION_NAME}-cn9k${PKG_POSTFIX}_${MRVL_PKG_VERSION}_arm64.deb" ${BASE_DIR}/artifacts/. - name: Export version name id: artifacts run: | @@ -213,4 +195,4 @@ jobs: "dpdk_tag" : "dpdk-cn9k-${{ steps.version.outputs.DPDK_BASE_PKG_VERSION }}_${{ steps.version.outputs.DPDK_PKG_VERSION }}-${{ steps.artifacts.outputs.DISTRO }}-${{ steps.version.outputs.DPDK_PKG_VERSION }}", "has_dpdk" : "true", "distro" : "${{ steps.artifacts.outputs.DISTRO }}", "platform" : "cn9k", - "devel": "${{ steps.artifacts.outputs.IS_DEVEL }}"}}' + "devel": "${{ steps.artifacts.outputs.IS_DEVEL }}"}}' \ No newline at end of file From eff1c533916fa28a5ae060377cc28374c03a049d Mon Sep 17 00:00:00 2001 From: Nagendra T P Date: Wed, 5 Feb 2025 12:40:52 +0530 Subject: [PATCH 148/271] ci: bump DPDK version Bump DPDK version to 24.11 and 24.12 Signed-off-by: Nagendra T P --- DPDK_VERSION | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/DPDK_VERSION b/DPDK_VERSION index 5930a489ba..cd9c2e2495 100644 --- a/DPDK_VERSION +++ b/DPDK_VERSION @@ -1,2 +1,2 @@ -BASE_VERSION=23.11.0 -RELEASE_VERSION=24.11.0 \ No newline at end of file +BASE_VERSION=24.11.0 +RELEASE_VERSION=24.12.0 \ No newline at end of file From 389ea6ad0b6167dac81c3ae6a95d481ab6f982d0 Mon Sep 17 00:00:00 2001 From: Monendra Singh Kushwaha Date: Sun, 8 Dec 2024 16:24:09 +0530 Subject: [PATCH 149/271] octeon: enable support for pkts with switch header MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This patch adds support for packet with switch header to receive by OCTEON plugin. Supported switch header types are “chlen24b”, “chlen90b”, “dsa”, “exdsa”, “higig2”, and “vlan_exdsa”. Type: feature JIRA: https://essjira.marvell.com/browse/IPBUSW-57052 Signed-off-by: Monendra Singh Kushwaha Change-Id: Ia8273acfc19fe5e637b0839e7299ed227e2c71de Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/141171 Tested-by: sa_ip-sw-jenkins Reviewed-by: Kiran Kumar Kokkilagadda (cherry picked from commit 11167189f81985c3fe6bff5f6b4aff982a266115) Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/142448 --- src/plugins/dev_octeon/init.c | 7 ++++ src/plugins/dev_octeon/octeon.h | 1 + src/plugins/dev_octeon/port.c | 58 +++++++++++++++++++++++++-------- 3 files changed, 52 insertions(+), 14 deletions(-) diff --git a/src/plugins/dev_octeon/init.c b/src/plugins/dev_octeon/init.c index 0d8fb78c9d..b47905c3cd 100644 --- a/src/plugins/dev_octeon/init.c +++ b/src/plugins/dev_octeon/init.c @@ -101,6 +101,13 @@ static vnet_dev_arg_t oct_port_args[] = { .type = VNET_DEV_ARG_TYPE_BOOL, .default_val.boolean = false, }, + { + .id = OCT_PORT_ARG_SWITCH_HDR_TYPE, + .name = "switch_header", + .desc = "Enable switch header and set specific switch header type, " + "applicable to network devices only", + .type = VNET_DEV_ARG_TYPE_STRING, + }, { .id = OCT_PORT_ARG_END, .name = "end", diff --git a/src/plugins/dev_octeon/octeon.h b/src/plugins/dev_octeon/octeon.h index 01ae1b810e..028e25c575 100644 --- a/src/plugins/dev_octeon/octeon.h +++ b/src/plugins/dev_octeon/octeon.h @@ -38,6 +38,7 @@ typedef enum { OCT_PORT_ARG_ALLMULTI_MODE = 1, OCT_PORT_ARG_EN_ETH_PAUSE_FRAME, + OCT_PORT_ARG_SWITCH_HDR_TYPE, OCT_PORT_ARG_END } oct_port_args_t; diff --git a/src/plugins/dev_octeon/port.c b/src/plugins/dev_octeon/port.c index 5f63561cd9..99527d69a9 100644 --- a/src/plugins/dev_octeon/port.c +++ b/src/plugins/dev_octeon/port.c @@ -136,6 +136,27 @@ oct_port_pause_flow_control_init (vlib_main_t *vm, vnet_dev_port_t *port) return VNET_DEV_OK; } +static u16 +oct_parse_switch_hdr_type (const char *value) +{ + if (strcmp (value, "higig2") == 0) + return ROC_PRIV_FLAGS_HIGIG; + + if (strcmp (value, "dsa") == 0) + return ROC_PRIV_FLAGS_EDSA; + + if (strcmp (value, "chlen90b") == 0) + return ROC_PRIV_FLAGS_LEN_90B; + + if (strcmp (value, "exdsa") == 0) + return ROC_PRIV_FLAGS_EXDSA; + + if (strcmp (value, "vlan_exdsa") == 0) + return ROC_PRIV_FLAGS_VLAN_EXDSA; + + return 0; +} + vnet_dev_rv_t oct_port_init (vlib_main_t *vm, vnet_dev_port_t *port) { @@ -150,6 +171,19 @@ oct_port_init (vlib_main_t *vm, vnet_dev_port_t *port) log_notice (dev, "port init: port %u", port->port_id); + foreach_vnet_dev_port_args (arg, port) + { + if (arg->id == OCT_PORT_ARG_ALLMULTI_MODE && vnet_dev_arg_get_bool (arg)) + is_allmulti_enable = true; + if (arg->id == OCT_PORT_ARG_EN_ETH_PAUSE_FRAME && + vnet_dev_arg_get_bool (arg)) + is_pause_frame_enable = true; + if (arg->id == OCT_PORT_ARG_SWITCH_HDR_TYPE && + vnet_dev_arg_get_string (arg)) + cp->npc.switch_header_type = + oct_parse_switch_hdr_type ((char *) vnet_dev_arg_get_string (arg)); + } + if ((rrv = roc_nix_lf_alloc (nix, port->intf.num_rx_queues, port->intf.num_tx_queues, rxq_cfg))) { @@ -177,20 +211,6 @@ oct_port_init (vlib_main_t *vm, vnet_dev_port_t *port) } } - foreach_vnet_dev_port_args (arg, port) - { - if (arg->id == OCT_PORT_ARG_ALLMULTI_MODE && vnet_dev_arg_get_bool (arg)) - is_allmulti_enable = true; - if (arg->id == OCT_PORT_ARG_EN_ETH_PAUSE_FRAME && - vnet_dev_arg_get_bool (arg)) - is_pause_frame_enable = true; - } - if ((rrv = roc_nix_npc_mcast_config (nix, true, false))) - { - oct_port_deinit (vm, port); - return oct_roc_err (dev, rrv, "roc_nix_mac_addr_set failed"); - } - /* Enable allmulti mode, if set by arg */ if (is_allmulti_enable) { @@ -215,6 +235,13 @@ oct_port_init (vlib_main_t *vm, vnet_dev_port_t *port) return oct_roc_err (dev, rrv, "roc_nix_tm_hierarchy_enable() failed"); } + rrv = roc_nix_switch_hdr_set (nix, cp->npc.switch_header_type, 0, 0, 0); + if (rrv) + { + oct_port_deinit (vm, port); + return oct_roc_err (dev, rrv, "roc_nix_switch_hdr_set() failed"); + } + if ((rrv = roc_nix_rss_default_setup (nix, default_rss_flowkey))) { oct_port_deinit (vm, port); @@ -298,6 +325,9 @@ oct_port_deinit (vlib_main_t *vm, vnet_dev_port_t *port) foreach_vnet_dev_port_tx_queue (q, port) oct_txq_deinit (vm, q); + /* Disable switch hdr pkind */ + roc_nix_switch_hdr_set (nix, 0, 0, 0, 0); + if (cp->npc_initialized) { if ((rrv = roc_npc_fini (&cp->npc))) From d0f19aedf4ef3b13e1cbd5bd4324ee8ddb2c57a2 Mon Sep 17 00:00:00 2001 From: Nithinsen Kaithakadan Date: Wed, 6 Nov 2024 19:21:22 +0530 Subject: [PATCH 150/271] octeon: rework crypto framework Add changes in pending queue structure to include per packet into one inflight request entry. Type: feature Signed-off-by: Nithinsen Kaithakadan Change-Id: I08608a629a861d25cd841d08c55c5c04bd0a545e Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/138689 Tested-by: sa_ip-sw-jenkins Reviewed-by: Tejasree Kondoj Reviewed-by: Monendra Singh Kushwaha (cherry picked from commit ddaf95addb0908fbbefc08edd2ff026cc1c9e4c2) Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/142486 Reviewed-by: Nithinsen Kaithakadan --- src/plugins/dev_octeon/crypto.c | 82 ++++++++++++++++----------------- src/plugins/dev_octeon/crypto.h | 20 ++++---- 2 files changed, 52 insertions(+), 50 deletions(-) diff --git a/src/plugins/dev_octeon/crypto.c b/src/plugins/dev_octeon/crypto.c index a00cd839cc..979ed35784 100644 --- a/src/plugins/dev_octeon/crypto.c +++ b/src/plugins/dev_octeon/crypto.c @@ -1435,22 +1435,24 @@ oct_crypto_enqueue_enc_dec (vlib_main_t *vm, vnet_crypto_async_frame_t *frame, pend_q = &ocm->pend_q[vlib_get_thread_index ()]; - enq_tail = pend_q->enq_tail; - nb_infl_allowed = pend_q->n_desc - pend_q->n_crypto_inflight; - if (PREDICT_FALSE (nb_infl_allowed == 0)) + if (PREDICT_FALSE (nb_infl_allowed < frame->n_elts)) { oct_crypto_update_frame_error_status ( frame, VNET_CRYPTO_OP_STATUS_FAIL_ENGINE_ERR); return -1; } - infl_req = &pend_q->req_queue[enq_tail]; - infl_req->frame = frame; - for (i = 0; i < frame->n_elts; i++) { + enq_tail = pend_q->enq_tail; + infl_req = &pend_q->req_queue[enq_tail]; + infl_req->frame = frame; + infl_req->last_elts = false; + infl_req->index = i; + elts = &frame->elts[i]; + infl_req->fe = elts; buffer_index = frame->buffer_indices[i]; key = vec_elt_at_index (ocm->keys[type], elts->key_index); @@ -1485,7 +1487,7 @@ oct_crypto_enqueue_enc_dec (vlib_main_t *vm, vnet_crypto_async_frame_t *frame, oct_crypto_fill_fc_params ( sess, inst + i, is_aead, aad_len, (u8 *) dptr_start_ptr, elts, - (oct_crypto_scatter_gather_t *) (infl_req->sg_data) + i, + (oct_crypto_scatter_gather_t *) (infl_req->sg_data), crypto_total_length /* cipher_len */, crypto_start_offset /* cipher_offset */, 0 /* auth_len */, integ_start_offset /* auth_off */, buffer, adj_len); @@ -1506,7 +1508,7 @@ oct_crypto_enqueue_enc_dec (vlib_main_t *vm, vnet_crypto_async_frame_t *frame, oct_crypto_fill_fc_params ( sess, inst + i, is_aead, aad_len, (u8 *) dptr_start_ptr, elts, - (oct_crypto_scatter_gather_t *) (infl_req->sg_data) + i, + (oct_crypto_scatter_gather_t *) (infl_req->sg_data), crypto_total_length /* cipher_len */, crypto_start_offset /* cipher_offset */, enc_auth_len /* auth_len */, integ_start_offset /* auth_off */, @@ -1514,19 +1516,20 @@ oct_crypto_enqueue_enc_dec (vlib_main_t *vm, vnet_crypto_async_frame_t *frame, } inst[i].w7.u64 = sess->cpt_inst_w7; - inst[i].res_addr = (u64) &infl_req->res[i]; + inst[i].res_addr = (u64) &infl_req->res; + OCT_MOD_INC (pend_q->enq_tail, pend_q->n_desc); } oct_crypto_burst_submit (crypto_dev, inst, frame->n_elts); - infl_req->elts = frame->n_elts; - OCT_MOD_INC (pend_q->enq_tail, pend_q->n_desc); - pend_q->n_crypto_inflight++; + infl_req->last_elts = true; + + pend_q->n_crypto_inflight += frame->n_elts; + pend_q->n_crypto_frame++; vlib_increment_simple_counter (pend_q->pending_packets, vm->thread_index, 0, frame->n_elts); - vlib_increment_simple_counter (pend_q->crypto_inflight, vm->thread_index, 0, - 1); + vlib_increment_simple_counter (pend_q->crypto_frame, vm->thread_index, 0, 1); return 0; } @@ -1616,22 +1619,22 @@ oct_crypto_frame_dequeue (vlib_main_t *vm, u32 *nb_elts_processed, oct_crypto_pending_queue_t *pend_q; vnet_crypto_async_frame_t *frame; volatile union cpt_res_s *res; - int i; + bool last_elts_processed; pend_q = &ocm->pend_q[vlib_get_thread_index ()]; - if (!pend_q->n_crypto_inflight) + if (!pend_q->n_crypto_frame) return NULL; - deq_head = pend_q->deq_head; - infl_req = &pend_q->req_queue[deq_head]; - frame = infl_req->frame; - - fe = frame->elts; + last_elts_processed = false; - for (i = infl_req->deq_elts; i < infl_req->elts; ++i) + for (; last_elts_processed == false;) { - res = &infl_req->res[i]; + deq_head = pend_q->deq_head; + infl_req = &pend_q->req_queue[deq_head]; + fe = infl_req->fe; + + res = &infl_req->res; if (PREDICT_FALSE (res->cn10k.compcode == CPT_COMP_NOT_DONE)) return NULL; @@ -1639,26 +1642,26 @@ oct_crypto_frame_dequeue (vlib_main_t *vm, u32 *nb_elts_processed, if (PREDICT_FALSE (res->cn10k.uc_compcode)) { if (res->cn10k.uc_compcode == ROC_SE_ERR_GC_ICV_MISCOMPARE) - status = fe[i].status = VNET_CRYPTO_OP_STATUS_FAIL_BAD_HMAC; + status = fe->status = VNET_CRYPTO_OP_STATUS_FAIL_BAD_HMAC; else - status = fe[i].status = VNET_CRYPTO_OP_STATUS_FAIL_ENGINE_ERR; + status = fe->status = VNET_CRYPTO_OP_STATUS_FAIL_ENGINE_ERR; } - infl_req->deq_elts++; + clib_memset ((void *) &infl_req->res, 0, sizeof (union cpt_res_s)); + last_elts_processed = infl_req->last_elts; + OCT_MOD_INC (pend_q->deq_head, pend_q->n_desc); } + frame = infl_req->frame; + vlib_decrement_simple_counter (pend_q->pending_packets, vm->thread_index, 0, - infl_req->elts); + frame->n_elts); vlib_increment_simple_counter (pend_q->success_packets, vm->thread_index, 0, - infl_req->elts); - - clib_memset ((void *) infl_req->res, 0, - sizeof (union cpt_res_s) * VNET_CRYPTO_FRAME_SIZE); + frame->n_elts); - OCT_MOD_INC (pend_q->deq_head, pend_q->n_desc); - pend_q->n_crypto_inflight--; - vlib_decrement_simple_counter (pend_q->crypto_inflight, vm->thread_index, 0, - 1); + pend_q->n_crypto_frame--; + pend_q->n_crypto_inflight -= frame->n_elts; + vlib_decrement_simple_counter (pend_q->crypto_frame, vm->thread_index, 0, 1); frame->state = status == VNET_CRYPTO_OP_STATUS_COMPLETED ? VNET_CRYPTO_FRAME_STATE_SUCCESS : @@ -1667,9 +1670,6 @@ oct_crypto_frame_dequeue (vlib_main_t *vm, u32 *nb_elts_processed, *nb_elts_processed = frame->n_elts; *enqueue_thread_idx = frame->enqueue_thread_index; - infl_req->deq_elts = 0; - infl_req->elts = 0; - return frame; } @@ -1732,8 +1732,7 @@ oct_conf_sw_queue (vlib_main_t *vm, vnet_dev_t *dev) * Each pending queue will get number of cpt desc / number of cores. * And that desc count is shared across inflight entries. */ - n_inflight_req = - (OCT_CPT_LF_MAX_NB_DESC / tm->n_vlib_mains) / VNET_CRYPTO_FRAME_SIZE; + n_inflight_req = (OCT_CPT_LF_MAX_NB_DESC / tm->n_vlib_mains); for (i = 0; i < tm->n_vlib_mains; ++i) { @@ -1754,8 +1753,7 @@ oct_conf_sw_queue (vlib_main_t *vm, vnet_dev_t *dev) infl_req_queue = &ocm->pend_q[i].req_queue[j]; infl_req_queue->sg_data = oct_plt_init_param.oct_plt_zmalloc ( - OCT_SCATTER_GATHER_BUFFER_SIZE * VNET_CRYPTO_FRAME_SIZE, - CLIB_CACHE_LINE_BYTES); + OCT_SCATTER_GATHER_BUFFER_SIZE, CLIB_CACHE_LINE_BYTES); if (infl_req_queue->sg_data == NULL) { log_err (dev, "Failed to allocate crypto scatter gather memory"); diff --git a/src/plugins/dev_octeon/crypto.h b/src/plugins/dev_octeon/crypto.h index 19d37c4caf..0f0df89f69 100644 --- a/src/plugins/dev_octeon/crypto.h +++ b/src/plugins/dev_octeon/crypto.h @@ -18,7 +18,7 @@ /* counter, name, verbose */ #define foreach_crypto_counter \ _ (0, pending_packets, "crypto-pending-packets") \ - _ (1, crypto_inflight, "crypto-inflight-operations") \ + _ (1, crypto_frame, "crypto-pending-frames") \ _ (2, success_packets, "crypto-success-packets") /* CRYPTO_ID, KEY_LENGTH_IN_BYTES, TAG_LEN, AAD_LEN */ @@ -122,17 +122,19 @@ typedef struct oct_crypto_scatter_gather typedef struct { CLIB_CACHE_LINE_ALIGN_MARK (cacheline0); - /** Result data of all entries in the frame */ - volatile union cpt_res_s res[VNET_CRYPTO_FRAME_SIZE]; + /** Result data */ + volatile union cpt_res_s res; /** Scatter gather data */ void *sg_data; /** Frame pointer */ vnet_crypto_async_frame_t *frame; - /** Number of async elements in frame */ - u16 elts; - /** Next read entry in frame, when dequeue */ - u16 deq_elts; -} oct_crypto_inflight_req_t; + /** Async frame element */ + vnet_crypto_async_frame_elt_t *fe; + /** Set if this is last element in frame */ + bool last_elts; + /** Index of element in frame */ + int index; +} __plt_cache_aligned oct_crypto_inflight_req_t; typedef struct { @@ -140,6 +142,8 @@ typedef struct oct_crypto_inflight_req_t *req_queue; /** Number of inflight operations in queue */ u32 n_crypto_inflight; + /** Number of frames in queue */ + u32 n_crypto_frame; /** Tail of queue to be used for enqueue */ u16 enq_tail; /** Head of queue to be used for dequeue */ From 53fadd2bc69fd4b28aa5590191a4967195f76c5e Mon Sep 17 00:00:00 2001 From: Nithinsen Kaithakadan Date: Tue, 3 Dec 2024 14:42:26 +0530 Subject: [PATCH 151/271] octeon: enable context cache for cpt crypto Enable context cache for CPT crypto operations. Type: feature Signed-off-by: Nithinsen Kaithakadan Change-Id: I1788acf639a101317cc7f8f71dd15023f17b24f0 Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/140622 Tested-by: sa_ip-sw-jenkins Reviewed-by: Tejasree Kondoj Reviewed-by: Monendra Singh Kushwaha (cherry picked from commit 47ddd40c472105bb87208023161eca703b1cb357) Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/142487 Reviewed-by: Nithinsen Kaithakadan --- src/plugins/dev_octeon/crypto.c | 7 +++++++ src/plugins/dev_octeon/crypto.h | 7 +++++++ 2 files changed, 14 insertions(+) diff --git a/src/plugins/dev_octeon/crypto.c b/src/plugins/dev_octeon/crypto.c index 979ed35784..fd011a7208 100644 --- a/src/plugins/dev_octeon/crypto.c +++ b/src/plugins/dev_octeon/crypto.c @@ -1165,6 +1165,10 @@ oct_cpt_inst_w7_get (oct_crypto_sess_t *sess, struct roc_cpt *roc_cpt) inst_w7.u64 = 0; inst_w7.s.cptr = (u64) &sess->cpt_ctx.se_ctx.fctx; + + if (oct_hw_ctx_cache_enable ()) + inst_w7.s.ctx_val = 1; + /* Set the engine group */ inst_w7.s.egrp = roc_cpt->eng_grp[CPT_ENG_TYPE_IE]; @@ -1394,6 +1398,9 @@ oct_crypto_session_init (vlib_main_t *vm, oct_crypto_sess_t *session, session->cpt_inst_w7 = oct_cpt_inst_w7_get (session, session->crypto_dev->roc_cpt); + if (oct_hw_ctx_cache_enable ()) + roc_se_ctx_init (&session->cpt_ctx); + session->initialised = 1; return 0; diff --git a/src/plugins/dev_octeon/crypto.h b/src/plugins/dev_octeon/crypto.h index 0f0df89f69..3965169b07 100644 --- a/src/plugins/dev_octeon/crypto.h +++ b/src/plugins/dev_octeon/crypto.h @@ -169,6 +169,13 @@ typedef struct u8 started; } oct_crypto_main_t; +static_always_inline bool +oct_hw_ctx_cache_enable (void) +{ + return roc_errata_cpt_hang_on_mixed_ctx_val () || + roc_model_is_cn10ka_b0 () || roc_model_is_cn10kb_a0 (); +} + extern oct_crypto_main_t oct_crypto_main; void oct_crypto_key_del_handler (vlib_main_t *vm, From bf79a6fb68691a3a6feb94434160904331ea13e9 Mon Sep 17 00:00:00 2001 From: Monendra Singh Kushwaha Date: Fri, 3 Jan 2025 02:16:01 +0530 Subject: [PATCH 152/271] octeon: update roc version Type: feature Signed-off-by: Monendra Singh Kushwaha Change-Id: I2b4e075d6dcb05ef0465684b463a5fbd768c108d Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/142503 Tested-by: sa_ip-sw-jenkins Reviewed-by: Nithin Kumar Dabilpuram (cherry picked from commit 99312d74cec2d94c9bd43bf7ed608fde54245034) Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/142504 --- build/external/packages/octeon-roc.mk | 4 ++-- src/plugins/dev_octeon/init.c | 4 ++-- src/plugins/dev_octeon/ipsec.c | 3 +-- src/plugins/dev_octeon/rx_node.c | 34 +++++++++++++++------------ 4 files changed, 24 insertions(+), 21 deletions(-) diff --git a/build/external/packages/octeon-roc.mk b/build/external/packages/octeon-roc.mk index 5029f780c5..96cd10f9f9 100644 --- a/build/external/packages/octeon-roc.mk +++ b/build/external/packages/octeon-roc.mk @@ -2,9 +2,9 @@ # SPDX-License-Identifier: Apache-2.0 # https://spdx.org/licenses/Apache-2.0.html -octeon-roc_version := octeon-roc-SDK12.24.12 +octeon-roc_version := octeon-roc-SDK12.25.01 octeon-roc_tarball := $(octeon-roc_version).tar.gz -octeon-roc_tarball_md5sum := 821ce30952222e5ef768f89f70f6d902 +octeon-roc_tarball_md5sum := 7f6794507c58dd0e73d3baccca6aff3b octeon-roc_github := https://github.com/MarvellEmbeddedProcessors/marvell-octeon-roc octeon-roc_tarball_strip_dirs := 1 diff --git a/src/plugins/dev_octeon/init.c b/src/plugins/dev_octeon/init.c index b47905c3cd..c476f9c802 100644 --- a/src/plugins/dev_octeon/init.c +++ b/src/plugins/dev_octeon/init.c @@ -319,12 +319,12 @@ oct_conf_cpt (vlib_main_t *vm, vnet_dev_t *dev, oct_crypto_dev_t *ocd, log_err (dev, "Could not add CPT IE engines"); return cnx_return_roc_err (dev, rrv, "roc_cpt_eng_grp_add"); } - if (roc_cpt->eng_grp[CPT_ENG_TYPE_IE] != ROC_CPT_DFLT_ENG_GRP_SE_IE) + if (roc_cpt->eng_grp[CPT_ENG_TYPE_IE] != ROC_LEGACY_CPT_DFLT_ENG_GRP_SE_IE) { log_err (dev, "Invalid CPT IE engine group configuration"); return -1; } - if (roc_cpt->eng_grp[CPT_ENG_TYPE_SE] != ROC_CPT_DFLT_ENG_GRP_SE) + if (roc_cpt->eng_grp[CPT_ENG_TYPE_SE] != ROC_LEGACY_CPT_DFLT_ENG_GRP_SE) { log_err (dev, "Invalid CPT SE engine group configuration"); return -1; diff --git a/src/plugins/dev_octeon/ipsec.c b/src/plugins/dev_octeon/ipsec.c index 49817aa8a1..bba1556379 100644 --- a/src/plugins/dev_octeon/ipsec.c +++ b/src/plugins/dev_octeon/ipsec.c @@ -70,7 +70,7 @@ oct_ipsec_crypto_inst_w7_get (void *sa) union cpt_inst_w7 w7; w7.u64 = 0; - w7.s.egrp = ROC_CPT_DFLT_ENG_GRP_SE_IE; + w7.s.egrp = ROC_LEGACY_CPT_DFLT_ENG_GRP_SE_IE; w7.s.ctx_val = 1; return w7.u64; @@ -960,7 +960,6 @@ oct_early_init_inline_ipsec (vlib_main_t *vm, vnet_dev_t *dev) inl_dev_main->inl_dev->ipsec_in_max_spi = inl_dev_main->in_max_spi; inl_dev_main->inl_dev->wqe_skip = 0; inl_dev_main->inl_dev->nb_meta_bufs = bp->n_buffers; - inl_dev_main->inl_dev->attach_cptlf = true; if ((rrv = roc_nix_inl_dev_init (inl_dev_main->inl_dev)) < 0) { diff --git a/src/plugins/dev_octeon/rx_node.c b/src/plugins/dev_octeon/rx_node.c index e4beb2fb66..dd111a25a2 100644 --- a/src/plugins/dev_octeon/rx_node.c +++ b/src/plugins/dev_octeon/rx_node.c @@ -369,14 +369,14 @@ oct_is_packet_from_cpt (union nix_rx_parse_u *rxp) } static_always_inline uword -oct_ipsec_is_inl_op_success (struct cpt_parse_hdr_s *cpt_hdr) +oct_ipsec_is_inl_op_success (struct cpt_cn10k_parse_hdr_s *cpt_hdr) { return (((1U << cpt_hdr->w3.hw_ccode) & CPT_COMP_HWGOOD_MASK) && roc_ie_ot_ucc_is_success (cpt_hdr->w3.uc_ccode)); } static_always_inline u32 -oct_get_len_from_meta (struct cpt_parse_hdr_s *cpt_hdr, u64 w0, u64 w4) +oct_get_len_from_meta (struct cpt_cn10k_parse_hdr_s *cpt_hdr, u64 w0, u64 w4) { u32 len; uintptr_t ip; @@ -413,14 +413,14 @@ oct_rx_inl_ipsec_vlib_from_cq (vlib_main_t *vm, vlib_node_runtime_t *node, oct_nix_rx_cqe_desc_t *d, vlib_buffer_t **b, oct_rx_node_ctx_t *ctx, vlib_buffer_template_t *bt, - struct cpt_parse_hdr_s *cpt_hdr, + struct cpt_cn10k_parse_hdr_s *cpt_hdr, vlib_buffer_t **buffs, u32 *err_flags) { union nix_rx_parse_u *orig_rxp; u32 is_fail, olen, esp_sz, l2_ol3_sz, idx; u64 *wqe_ptr; - cpt_hdr = (struct cpt_parse_hdr_s *) *(((u64 *) d) + 9); + cpt_hdr = (struct cpt_cn10k_parse_hdr_s *) *(((u64 *) d) + 9); wqe_ptr = (u64 *) clib_net_to_host_u64 (cpt_hdr->wqe_ptr); b[0] = (vlib_buffer_t *) wqe_ptr; @@ -516,8 +516,8 @@ oct_rx_batch (vlib_main_t *vm, vlib_node_runtime_t *node, u32 b2_err_flags = 0, b3_err_flags = 0; u32 n_left, err_flags = 0; oct_nix_rx_cqe_desc_t *d = ctx->next_desc; - struct cpt_parse_hdr_s *cpt_hdr0, *cpt_hdr1; - struct cpt_parse_hdr_s *cpt_hdr2, *cpt_hdr3; + struct cpt_cn10k_parse_hdr_s *cpt_hdr0, *cpt_hdr1; + struct cpt_cn10k_parse_hdr_s *cpt_hdr2, *cpt_hdr3; union nix_rx_parse_u *orig_rxp0, *orig_rxp1; union nix_rx_parse_u *orig_rxp2, *orig_rxp3; u8 is_b0_from_cpt, is_b1_from_cpt; @@ -616,10 +616,10 @@ oct_rx_batch (vlib_main_t *vm, vlib_node_runtime_t *node, else if (n_from_cpt == 4) { /* All packets are from cpt */ - cpt_hdr0 = (struct cpt_parse_hdr_s *) *(((u64 *) &d[0]) + 9); - cpt_hdr1 = (struct cpt_parse_hdr_s *) *(((u64 *) &d[1]) + 9); - cpt_hdr2 = (struct cpt_parse_hdr_s *) *(((u64 *) &d[2]) + 9); - cpt_hdr3 = (struct cpt_parse_hdr_s *) *(((u64 *) &d[3]) + 9); + cpt_hdr0 = (struct cpt_cn10k_parse_hdr_s *) *(((u64 *) &d[0]) + 9); + cpt_hdr1 = (struct cpt_cn10k_parse_hdr_s *) *(((u64 *) &d[1]) + 9); + cpt_hdr2 = (struct cpt_cn10k_parse_hdr_s *) *(((u64 *) &d[2]) + 9); + cpt_hdr3 = (struct cpt_cn10k_parse_hdr_s *) *(((u64 *) &d[3]) + 9); wqe_ptr0 = (u64 *) clib_net_to_host_u64 (cpt_hdr0->wqe_ptr); wqe_ptr1 = (u64 *) clib_net_to_host_u64 (cpt_hdr1->wqe_ptr); @@ -790,7 +790,8 @@ oct_rx_batch (vlib_main_t *vm, vlib_node_runtime_t *node, /* CQ ring contains mix of packets from wire and CPT */ if (is_b0_from_cpt) { - cpt_hdr0 = (struct cpt_parse_hdr_s *) *(((u64 *) &d[0]) + 9); + cpt_hdr0 = + (struct cpt_cn10k_parse_hdr_s *) *(((u64 *) &d[0]) + 9); oct_rx_inl_ipsec_vlib_from_cq (vm, node, &d[0], &b[0], ctx, &bt, cpt_hdr0, buffs, &err_flags); OCT_PUSH_META_TO_FREE ((u64) cpt_hdr0, laddr, &loff); @@ -801,7 +802,8 @@ oct_rx_batch (vlib_main_t *vm, vlib_node_runtime_t *node, if (is_b1_from_cpt) { - cpt_hdr1 = (struct cpt_parse_hdr_s *) *(((u64 *) &d[1]) + 9); + cpt_hdr1 = + (struct cpt_cn10k_parse_hdr_s *) *(((u64 *) &d[1]) + 9); oct_rx_inl_ipsec_vlib_from_cq (vm, node, &d[1], &b[1], ctx, &bt, cpt_hdr1, buffs, &err_flags); OCT_PUSH_META_TO_FREE ((u64) cpt_hdr1, laddr, &loff); @@ -811,7 +813,8 @@ oct_rx_batch (vlib_main_t *vm, vlib_node_runtime_t *node, buffs, &err_flags); if (is_b2_from_cpt) { - cpt_hdr2 = (struct cpt_parse_hdr_s *) *(((u64 *) &d[2]) + 9); + cpt_hdr2 = + (struct cpt_cn10k_parse_hdr_s *) *(((u64 *) &d[2]) + 9); oct_rx_inl_ipsec_vlib_from_cq (vm, node, &d[2], &b[2], ctx, &bt, cpt_hdr2, buffs, &err_flags); OCT_PUSH_META_TO_FREE ((u64) cpt_hdr2, laddr, &loff); @@ -821,7 +824,8 @@ oct_rx_batch (vlib_main_t *vm, vlib_node_runtime_t *node, buffs, &err_flags); if (is_b3_from_cpt) { - cpt_hdr3 = (struct cpt_parse_hdr_s *) *(((u64 *) &d[3]) + 9); + cpt_hdr3 = + (struct cpt_cn10k_parse_hdr_s *) *(((u64 *) &d[3]) + 9); oct_rx_inl_ipsec_vlib_from_cq (vm, node, &d[3], &b[3], ctx, &bt, cpt_hdr3, buffs, &err_flags); OCT_PUSH_META_TO_FREE ((u64) cpt_hdr3, laddr, &loff); @@ -900,7 +904,7 @@ oct_rx_batch (vlib_main_t *vm, vlib_node_runtime_t *node, is_b0_from_cpt = oct_is_packet_from_cpt (&d[0].parse.f); if (is_b0_from_cpt) { - cpt_hdr0 = (struct cpt_parse_hdr_s *) *(((u64 *) &d[0]) + 9); + cpt_hdr0 = (struct cpt_cn10k_parse_hdr_s *) *(((u64 *) &d[0]) + 9); oct_rx_inl_ipsec_vlib_from_cq (vm, node, &d[0], &b[0], ctx, &bt, cpt_hdr0, buffs, &err_flags); OCT_PUSH_META_TO_FREE ((u64) cpt_hdr0, laddr, &loff); From 19085bc08d14f38c9c3f31b81d4b4e97fcec26e1 Mon Sep 17 00:00:00 2001 From: Monendra Singh Kushwaha Date: Fri, 20 Dec 2024 16:23:56 +0530 Subject: [PATCH 153/271] vlib: update input node counts based on state Type: fix Change-Id: I09497ae8d6a685324f8c7d9e0b3208a3ec465f0e Signed-off-by: Monendra Singh Kushwaha Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/142306 Tested-by: sa_ip-sw-jenkins (cherry picked from commit c2453f88925850d62efa6304b49e16862d26b0c1) Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/142505 --- src/vlib/threads.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/vlib/threads.c b/src/vlib/threads.c index 7e6ac25f10..751188f976 100644 --- a/src/vlib/threads.c +++ b/src/vlib/threads.c @@ -1057,6 +1057,13 @@ vlib_worker_thread_node_refork (void) VLIB_NODE_RUNTIME_DATA_SIZE); } + for (j = vec_len (old_rt); + j < vec_len (nm_clone->nodes_by_type[VLIB_NODE_TYPE_INPUT]); j++) + { + rt = &nm_clone->nodes_by_type[VLIB_NODE_TYPE_INPUT][j]; + nm_clone->input_node_counts_by_state[rt->state] += 1; + } + vec_free (old_rt); /* re-clone pre-input nodes */ From bb62b3104120c6f9c3ce38575824cf46f2179eb1 Mon Sep 17 00:00:00 2001 From: Nithinsen Kaithakadan Date: Fri, 3 Jan 2025 17:22:47 +0530 Subject: [PATCH 154/271] octeon: fix scatter gather mem alloc This patch fixes scatter gather memory alloc hang by allocating a block of memory per thread. Type: fix Signed-off-by: Nithinsen Kaithakadan Change-Id: Ia3e3f5289aa72389d3b74d4000bfbf52460cd550 Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/142549 Reviewed-by: Tejasree Kondoj Reviewed-by: Monendra Singh Kushwaha Tested-by: sa_ip-sw-jenkins (cherry picked from commit 88e6a658602376943c504828d7c055be51048cc1) Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/142573 --- src/plugins/dev_octeon/crypto.c | 34 +++++++++++++-------------------- src/plugins/dev_octeon/crypto.h | 4 ++-- 2 files changed, 15 insertions(+), 23 deletions(-) diff --git a/src/plugins/dev_octeon/crypto.c b/src/plugins/dev_octeon/crypto.c index fd011a7208..0cf9931332 100644 --- a/src/plugins/dev_octeon/crypto.c +++ b/src/plugins/dev_octeon/crypto.c @@ -1435,6 +1435,7 @@ oct_crypto_enqueue_enc_dec (vlib_main_t *vm, vnet_crypto_async_frame_t *frame, u32 crypto_total_length; oct_crypto_key_t *key; vlib_buffer_t *buffer; + void *sg_data; u16 adj_len; /* GCM packets having 8 bytes of aad and 8 bytes of iv */ @@ -1450,6 +1451,8 @@ oct_crypto_enqueue_enc_dec (vlib_main_t *vm, vnet_crypto_async_frame_t *frame, return -1; } + sg_data = pend_q->sg_data; + for (i = 0; i < frame->n_elts; i++) { enq_tail = pend_q->enq_tail; @@ -1494,7 +1497,7 @@ oct_crypto_enqueue_enc_dec (vlib_main_t *vm, vnet_crypto_async_frame_t *frame, oct_crypto_fill_fc_params ( sess, inst + i, is_aead, aad_len, (u8 *) dptr_start_ptr, elts, - (oct_crypto_scatter_gather_t *) (infl_req->sg_data), + ((oct_crypto_scatter_gather_t *) (sg_data)) + enq_tail, crypto_total_length /* cipher_len */, crypto_start_offset /* cipher_offset */, 0 /* auth_len */, integ_start_offset /* auth_off */, buffer, adj_len); @@ -1515,7 +1518,7 @@ oct_crypto_enqueue_enc_dec (vlib_main_t *vm, vnet_crypto_async_frame_t *frame, oct_crypto_fill_fc_params ( sess, inst + i, is_aead, aad_len, (u8 *) dptr_start_ptr, elts, - (oct_crypto_scatter_gather_t *) (infl_req->sg_data), + ((oct_crypto_scatter_gather_t *) (sg_data)) + enq_tail, crypto_total_length /* cipher_len */, crypto_start_offset /* cipher_offset */, enc_auth_len /* auth_len */, integ_start_offset /* auth_off */, @@ -1722,9 +1725,8 @@ oct_conf_sw_queue (vlib_main_t *vm, vnet_dev_t *dev) vlib_thread_main_t *tm = vlib_get_thread_main (); extern oct_plt_init_param_t oct_plt_init_param; oct_crypto_main_t *ocm = &oct_crypto_main; - oct_crypto_inflight_req_t *infl_req_queue; u32 n_inflight_req; - int i, j = 0; + int i; ocm->pend_q = oct_plt_init_param.oct_plt_zmalloc ( tm->n_vlib_mains * sizeof (oct_crypto_pending_queue_t), @@ -1755,17 +1757,13 @@ oct_conf_sw_queue (vlib_main_t *vm, vnet_dev_t *dev) goto free; } - for (j = 0; j <= ocm->pend_q[i].n_desc; ++j) + ocm->pend_q[i].sg_data = oct_plt_init_param.oct_plt_zmalloc ( + OCT_SCATTER_GATHER_BUFFER_SIZE * ocm->pend_q[i].n_desc, + CLIB_CACHE_LINE_BYTES); + if (ocm->pend_q[i].sg_data == NULL) { - infl_req_queue = &ocm->pend_q[i].req_queue[j]; - - infl_req_queue->sg_data = oct_plt_init_param.oct_plt_zmalloc ( - OCT_SCATTER_GATHER_BUFFER_SIZE, CLIB_CACHE_LINE_BYTES); - if (infl_req_queue->sg_data == NULL) - { - log_err (dev, "Failed to allocate crypto scatter gather memory"); - goto free; - } + log_err (dev, "Failed to allocate crypto scatter gather memory"); + goto free; } #define _(n, s, d) ocm->pend_q[i].s = &ocm->s##_counter; @@ -1779,15 +1777,9 @@ oct_conf_sw_queue (vlib_main_t *vm, vnet_dev_t *dev) { if (ocm->pend_q[i].req_queue == NULL) continue; - for (; j >= 0; j--) - { - infl_req_queue = &ocm->pend_q[i].req_queue[j]; - if (infl_req_queue->sg_data == NULL) - continue; + oct_plt_init_param.oct_plt_free (ocm->pend_q[i].sg_data); - oct_plt_init_param.oct_plt_free (infl_req_queue->sg_data); - } oct_plt_init_param.oct_plt_free (ocm->pend_q[i].req_queue); } oct_plt_init_param.oct_plt_free (ocm->pend_q); diff --git a/src/plugins/dev_octeon/crypto.h b/src/plugins/dev_octeon/crypto.h index 3965169b07..c24d6d9d24 100644 --- a/src/plugins/dev_octeon/crypto.h +++ b/src/plugins/dev_octeon/crypto.h @@ -124,8 +124,6 @@ typedef struct CLIB_CACHE_LINE_ALIGN_MARK (cacheline0); /** Result data */ volatile union cpt_res_s res; - /** Scatter gather data */ - void *sg_data; /** Frame pointer */ vnet_crypto_async_frame_t *frame; /** Async frame element */ @@ -150,6 +148,8 @@ typedef struct u16 deq_head; /** Number of descriptors */ u16 n_desc; + /** Scatter gather data */ + void **sg_data; /** Crypto counters for pending pkts, inflight operations * and successfully dequeued pkts in queue */ #define _(i, s, d) vlib_simple_counter_main_t *s; From b5837d7d8a7dc5c08bdf95b2938e1449159da36f Mon Sep 17 00:00:00 2001 From: Alok Mishra Date: Mon, 16 Dec 2024 14:44:28 +0500 Subject: [PATCH 155/271] tm: add tm infra for dynamic weight update This patch adds support for dynamically updating traffic management(TM) node weights. Type: feature Signed-off-by: Alok Mishra Change-Id: Iff23faa8e32dd6b679986ca4721c57a30e6f27a2 Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/141703 Tested-by: Monendra Singh Kushwaha Reviewed-by: Monendra Singh Kushwaha (cherry picked from commit 69b7401ce23d18e03da2725aa175f981ba2d05e4) Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/142576 Tested-by: sa_ip-sw-jenkins --- src/vnet/tm/tm.api | 36 ++++++++++++++++++++++++++++++ src/vnet/tm/tm.c | 14 ++++++++++++ src/vnet/tm/tm.h | 12 ++++++++++ src/vnet/tm/tm_api.c | 23 +++++++++++++++++++ src/vnet/tm/tm_test.c | 52 +++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 137 insertions(+) diff --git a/src/vnet/tm/tm.api b/src/vnet/tm/tm.api index 948530384c..dbeb0405f2 100644 --- a/src/vnet/tm/tm.api +++ b/src/vnet/tm/tm.api @@ -260,6 +260,42 @@ define tm_sys_shaper_profile_delete u32 shaper_id; }; +/** + * @brief Reply for updating the scheduling weight of a TM node. + * + * This structure specifies the parameters returned in response to updating the scheduling weight of a TM node. + * + * @param context - Sender context, to match reply with request. + * @param retval - Return value indicating success or failure of the operation. + * @param node_id - Identifier of the TM node whose scheduling weight was updated. + */ +define tm_sys_node_sched_weight_update_reply +{ + u32 context; + i32 retval; + u32 node_id; +}; + +/** + * @brief Update the scheduling weight of a TM node. + * + * This structure outlines the necessary parameters to update the scheduling weight of a TM node. + * + * @param client_index - Opaque cookie to identify the sender. + * @param context - Sender context, to match reply with request. + * @param sw_if_idx - Software interface index. + * @param node_id - Identifier of the TM node to be updated. + * @param weight - New weight to be assigned to the TM node. + */ +define tm_sys_node_sched_weight_update +{ + u32 client_index; + u32 context; + u32 sw_if_idx; + u32 node_id; + u32 weight; +}; + /** * @brief Reply for reading the statistics of a TM node. * diff --git a/src/vnet/tm/tm.c b/src/vnet/tm/tm.c index f5dfec64d8..64cf5cf754 100644 --- a/src/vnet/tm/tm.c +++ b/src/vnet/tm/tm.c @@ -121,6 +121,20 @@ tm_sys_shaper_profile_delete (u32 hw_if_idx, u32 shaper_id) return 0; } +int +tm_sys_node_sched_weight_update (u32 hw_if_idx, u32 node_id, u32 weight) +{ + vnet_main_t *vnm = vnet_get_main (); + vnet_hw_interface_t *hi = vnet_get_hw_interface (vnm, hw_if_idx); + vnet_device_class_t *dev_class = + vnet_get_device_class (vnm, hi->dev_class_index); + + dev_class->tm_sys_impl->node_sched_weight_update (hw_if_idx, node_id, + weight); + + return 0; +} + int tm_sys_node_read_stats (u32 hw_if_idx, u32 node_idx, tm_stats_params_t *param) { diff --git a/src/vnet/tm/tm.h b/src/vnet/tm/tm.h index c29af54632..f57e8cdc8e 100644 --- a/src/vnet/tm/tm.h +++ b/src/vnet/tm/tm.h @@ -167,6 +167,7 @@ typedef struct tm_system_t_ int (*shaper_profile_delete) (u32 hw_if_idx, u32 shaper_id); int (*node_shaper_update) (u32 hw_if_idx, u32 node_id, u32 shaper_profile_id); + int (*node_sched_weight_update) (u32 hw_if_idx, u32 node_id, u32 weight); int (*node_read_stats) (u32 hw_if_idx, u32 node_idx, tm_stats_params_t *param); int (*start_tm) (u32 hw_if_idx); @@ -257,6 +258,17 @@ int tm_sys_node_shaper_update (u32 hw_if_idx, u32 node_id, */ int tm_sys_shaper_profile_delete (u32 hw_if_idx, u32 shaper_id); +/** + * @brief Update the scheduling weight of a TM node. + * + * @param hw_if_idx - Hardware interface index. + * @param node_id - Identifier of the TM node to be updated. + * @param weight - New scheduling weight to be assigned to the node. + * + * @return 0 on success. + */ +int tm_sys_node_sched_weight_update (u32 hw_if_idx, u32 node_id, u32 weight); + /** * @brief Read statistics for a specific traffic management node. * diff --git a/src/vnet/tm/tm_api.c b/src/vnet/tm/tm_api.c index f5f716c838..e0c54619ce 100644 --- a/src/vnet/tm/tm_api.c +++ b/src/vnet/tm/tm_api.c @@ -176,6 +176,29 @@ vl_api_tm_sys_shaper_profile_delete_t_handler ( })); } +void +vl_api_tm_sys_node_sched_weight_update_t_handler ( + vl_api_tm_sys_node_sched_weight_update_t *mp) +{ + vl_api_tm_sys_node_sched_weight_update_reply_t *rmp; + vnet_main_t *vnm = vnet_get_main (); + u32 node_id = 0, weight = 0; + int rv = -1; + + vnet_sw_interface_t *sw = + vnet_get_sup_sw_interface (vnm, clib_net_to_host_u32 (mp->sw_if_idx)); + + node_id = clib_net_to_host_u32 (mp->node_id); + weight = clib_net_to_host_u32 (mp->weight); + + rv = tm_sys_node_sched_weight_update (sw->hw_if_index, node_id, weight); + + REPLY_MACRO2 (VL_API_TM_SYS_NODE_SCHED_WEIGHT_UPDATE_REPLY, ({ + if (!rv) + rmp->node_id = clib_host_to_net_u32 (node_id); + })); +} + void vl_api_tm_sys_node_read_stats_t_handler (vl_api_tm_sys_node_read_stats_t *mp) { diff --git a/src/vnet/tm/tm_test.c b/src/vnet/tm/tm_test.c index 49b733e533..62214ce585 100644 --- a/src/vnet/tm/tm_test.c +++ b/src/vnet/tm/tm_test.c @@ -358,6 +358,49 @@ api_tm_sys_shaper_profile_delete (vat_main_t *vam) return ret; } +static int +api_tm_sys_node_sched_weight_update (vat_main_t *vam) +{ + u8 sw_if_idx_set = 0; + unformat_input_t *i = vam->input; + vl_api_tm_sys_node_sched_weight_update_t *mp; + u32 msg_size = sizeof (*mp); + u32 node_id = 0, weight = 0; + u32 sw_if_idx = 0; + int ret; + + vam->result_ready = 0; + mp = vl_msg_api_alloc_as_if_client (msg_size); + + while (unformat_check_input (i) != UNFORMAT_END_OF_INPUT) + { + if (unformat (i, "sw_if_idx %u", &sw_if_idx)) + sw_if_idx_set = 1; + else if (unformat (i, "node_id %u", &node_id)) + ; + else if (unformat (i, "weight %u", &weight)) + ; + else + { + clib_warning ("Invalid input, unknown parameter"); + return -EINVAL; + } + } + + if (!sw_if_idx_set) + return -EINVAL; + + M (TM_SYS_NODE_SCHED_WEIGHT_UPDATE, mp); + + mp->sw_if_idx = clib_host_to_net_u32 (sw_if_idx); + mp->node_id = clib_host_to_net_u32 (node_id); + mp->weight = clib_host_to_net_u32 (weight); + + S (mp); + W (ret); + return ret; +} + static int api_tm_sys_node_read_stats (vat_main_t *vam) { @@ -539,6 +582,15 @@ vl_api_tm_sys_shaper_profile_delete_reply_t_handler ( vam->result_ready = 1; } +static void +vl_api_tm_sys_node_sched_weight_update_reply_t_handler ( + vl_api_tm_sys_node_sched_weight_update_reply_t *mp) +{ + vat_main_t *vam = tm_test_main.vat_main; + clib_warning ("TM node sched weight updated\n"); + vam->result_ready = 1; +} + static void vl_api_tm_sys_node_read_stats_reply_t_handler ( vl_api_tm_sys_node_read_stats_reply_t *mp) From 3c41ca231b3aefaaa724701ba0842babe390e793 Mon Sep 17 00:00:00 2001 From: Alok Mishra Date: Mon, 16 Dec 2024 16:00:53 +0500 Subject: [PATCH 156/271] octeon: enable dynamic weight update for tm nodes This patch adds support for dynamically updating traffic management(TM) node weights. Type: feature Signed-off-by: Alok Mishra Change-Id: I9458449e863e49241843e040966dc21d252d65e1 Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/141716 Tested-by: sa_ip-sw-jenkins Reviewed-by: Nithin Kumar Dabilpuram Reviewed-by: Satha Koteswara Rao Kottidi (cherry picked from commit 34820c180206f9905fa921494a999362b4fdbc98) Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/142577 Reviewed-by: Monendra Singh Kushwaha --- src/plugins/dev_octeon/tm.c | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/src/plugins/dev_octeon/tm.c b/src/plugins/dev_octeon/tm.c index 3fd1d1c943..8eef54afbd 100644 --- a/src/plugins/dev_octeon/tm.c +++ b/src/plugins/dev_octeon/tm.c @@ -259,6 +259,40 @@ oct_tm_sys_shaper_profile_delete (u32 hw_if_idx, u32 shaper_id) return rc; } +int +oct_tm_sys_node_sched_weight_update (u32 hw_if_idx, u32 node_id, u32 weight) +{ + vnet_main_t *vnm = vnet_get_main (); + vnet_hw_interface_t *hi = vnet_get_hw_interface (vnm, hw_if_idx); + vnet_dev_port_t *port = + vnet_dev_get_port_from_dev_instance (hi->dev_instance); + vnet_dev_t *dev = port->dev; + oct_device_t *cd = vnet_dev_get_data (dev); + struct roc_nix *nix = cd->nix; + struct roc_nix_tm_node *node; + int rc = 0; + u32 parent_id, priority; + + node = roc_nix_tm_node_get (nix, node_id); + if (!node) + { + rc = -EINVAL; + return oct_roc_err (dev, rc, "roc_nix_tm_node_get node_id not found"); + } + + parent_id = node->parent_id; + priority = node->priority; + + rc = + roc_nix_tm_node_parent_update (nix, node_id, parent_id, priority, weight); + if (rc) + { + return oct_roc_err (dev, rc, "roc_nix_tm_node_parent_update failed"); + } + + return rc; +} + int oct_tm_sys_node_read_stats (u32 hw_if_idx, u32 node_id, tm_stats_params_t *stats) @@ -382,6 +416,7 @@ tm_system_t dev_oct_tm_ops = { .shaper_profile_create = oct_tm_sys_shaper_profile_create, .node_shaper_update = oct_tm_sys_node_shaper_update, .shaper_profile_delete = oct_tm_sys_shaper_profile_delete, + .node_sched_weight_update = oct_tm_sys_node_sched_weight_update, .start_tm = oct_tm_sys_start, .stop_tm = oct_tm_sys_stop, }; From 525c9a58a0647f3632970ee7d20c63559c42ee2e Mon Sep 17 00:00:00 2001 From: Monendra Singh Kushwaha Date: Tue, 7 Jan 2025 10:38:49 +0530 Subject: [PATCH 157/271] build: update roc version hash Type: refactor Change-Id: I4390f0c33a72ae82a66bf623d565c01b505ba92c Signed-off-by: Monendra Singh Kushwaha Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/142668 (cherry picked from commit a27ff6314dc0f98329f43085922cdd581daeafb7) Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/142670 Tested-by: sa_ip-sw-jenkins --- build/external/packages/octeon-roc.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/external/packages/octeon-roc.mk b/build/external/packages/octeon-roc.mk index 96cd10f9f9..e69211fd03 100644 --- a/build/external/packages/octeon-roc.mk +++ b/build/external/packages/octeon-roc.mk @@ -4,7 +4,7 @@ octeon-roc_version := octeon-roc-SDK12.25.01 octeon-roc_tarball := $(octeon-roc_version).tar.gz -octeon-roc_tarball_md5sum := 7f6794507c58dd0e73d3baccca6aff3b +octeon-roc_tarball_md5sum := 43cbc76e76cbb182997b58965e500ba1 octeon-roc_github := https://github.com/MarvellEmbeddedProcessors/marvell-octeon-roc octeon-roc_tarball_strip_dirs := 1 From 1845337eb5a95757aa6274b1c6eacea48cedf33d Mon Sep 17 00:00:00 2001 From: Nithinsen Kaithakadan Date: Fri, 10 Jan 2025 10:05:58 +0530 Subject: [PATCH 158/271] octeon: update inline ipsec counter cli This patch disables inline ipsec counters if no inline device is attached to dev-octeon plugin. Type: fix JIRA: https://essjira.marvell.com/browse/IPBUSW-57696 Signed-off-by: Nithinsen Kaithakadan Change-Id: Ib1a8abbd58467c03f1e37427bdf93c4473a9846a Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/142927 Tested-by: sa_ip-sw-jenkins Reviewed-by: Monendra Singh Kushwaha (cherry picked from commit 4c4c26655d182840a20ec32e24dbb52211949015) Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/143059 --- src/plugins/dev_octeon/cli.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/plugins/dev_octeon/cli.c b/src/plugins/dev_octeon/cli.c index b390bb1d1e..3316347d9c 100644 --- a/src/plugins/dev_octeon/cli.c +++ b/src/plugins/dev_octeon/cli.c @@ -212,6 +212,13 @@ oct_ipsec_inline_counters_command_fn (vlib_main_t *vm, unformat_input_t *input, vlib_cli_command_t *cmd) { struct roc_nix_stats stats; + oct_inl_dev_main_t *oim = &oct_inl_dev_main; + + if (!oim->inl_dev) + { + return clib_error_create ( + "No Inline device attached to dev-octeon plugin"); + } roc_nix_inl_dev_stats_get (&stats); From ca35d2397c322878b45cf3b68d4403f45d223f79 Mon Sep 17 00:00:00 2001 From: Monendra Singh Kushwaha Date: Wed, 8 Jan 2025 15:33:59 +0530 Subject: [PATCH 159/271] octeon: update counter name This patch removes unit from counter name. Type: fix JIRA: https://essjira.marvell.com/browse/IPBUSW-58840 Signed-off-by: Monendra Singh Kushwaha Change-Id: I8b8a3a77cc55eb710dc654fe269062439e886c96 Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/142776 Tested-by: sa_ip-sw-jenkins Reviewed-by: Nithin Kumar Dabilpuram (cherry picked from commit 80cd8767711daafd9f8e90d6c5e6198fed5dcb9a) Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/143127 --- src/plugins/dev_octeon/counter.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/plugins/dev_octeon/counter.c b/src/plugins/dev_octeon/counter.c index dd73684c38..b8b283f2b2 100644 --- a/src/plugins/dev_octeon/counter.c +++ b/src/plugins/dev_octeon/counter.c @@ -43,7 +43,7 @@ vnet_dev_counter_t oct_port_counters[] = { VNET_DEV_CTR_RX_BYTES (OCT_PORT_CTR_RX_BYTES), VNET_DEV_CTR_RX_PACKETS (OCT_PORT_CTR_RX_PACKETS), VNET_DEV_CTR_RX_DROPS (OCT_PORT_CTR_RX_DROPS), - VNET_DEV_CTR_VENDOR (OCT_PORT_CTR_RX_DROP_BYTES, RX, BYTES, "drop bytes"), + VNET_DEV_CTR_VENDOR (OCT_PORT_CTR_RX_DROP_BYTES, RX, BYTES, "drop"), VNET_DEV_CTR_VENDOR (OCT_PORT_CTR_RX_UCAST, RX, PACKETS, "unicast"), VNET_DEV_CTR_VENDOR (OCT_PORT_CTR_RX_MCAST, RX, PACKETS, "multicast"), VNET_DEV_CTR_VENDOR (OCT_PORT_CTR_RX_BCAST, RX, PACKETS, "broadcast"), @@ -79,7 +79,7 @@ vnet_dev_counter_t oct_rxq_counters[] = { VNET_DEV_CTR_RX_BYTES (OCT_RXQ_CTR_BYTES), VNET_DEV_CTR_RX_PACKETS (OCT_RXQ_CTR_PKTS), VNET_DEV_CTR_RX_DROPS (OCT_RXQ_CTR_DROPS), - VNET_DEV_CTR_VENDOR (OCT_RXQ_CTR_DROP_BYTES, RX, BYTES, "drop bytes"), + VNET_DEV_CTR_VENDOR (OCT_RXQ_CTR_DROP_BYTES, RX, BYTES, "drop"), VNET_DEV_CTR_VENDOR (OCT_RXQ_CTR_ERR, RX, PACKETS, "error"), }; @@ -95,7 +95,7 @@ vnet_dev_counter_t oct_txq_counters[] = { VNET_DEV_CTR_TX_BYTES (OCT_TXQ_CTR_BYTES), VNET_DEV_CTR_TX_PACKETS (OCT_TXQ_CTR_PKTS), VNET_DEV_CTR_TX_DROPS (OCT_TXQ_CTR_DROPS), - VNET_DEV_CTR_VENDOR (OCT_TXQ_CTR_DROP_BYTES, TX, BYTES, "drop bytes"), + VNET_DEV_CTR_VENDOR (OCT_TXQ_CTR_DROP_BYTES, TX, BYTES, "drop"), }; static vnet_dev_rv_t From 61f28297327878882296f72fcd62071c3f3f5d8b Mon Sep 17 00:00:00 2001 From: Florin Coras Date: Mon, 6 Jan 2025 16:22:20 -0500 Subject: [PATCH 160/271] vcl: fix vls wrk index on fork Type: fix JIRA: https://essjira.marvell.com/browse/IPBUSW-57391 Change-Id: If634dac31fd54466428db0726ac441a1247985b8 Signed-off-by: Florin Coras Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/142842 Tested-by: sa_ip-sw-jenkins Reviewed-by: Venkata Ravichandra Mynidi Reviewed-by: Nithin Kumar Dabilpuram Reviewed-by: Monendra Singh Kushwaha (cherry picked from commit 4c3ae299bf496fdce6495f8d0d1f0f13853a5e4d) Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/143099 --- src/vcl/vcl_locked.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/vcl/vcl_locked.c b/src/vcl/vcl_locked.c index 2a15f6cedf..11283099ef 100644 --- a/src/vcl/vcl_locked.c +++ b/src/vcl/vcl_locked.c @@ -380,7 +380,7 @@ vls_n_workers (void) return pool_elts (vlsm->workers); } -static void +static vls_worker_t * vls_worker_alloc (void) { vls_worker_t *wrk; @@ -391,6 +391,8 @@ vls_worker_alloc (void) wrk->vcl_wrk_index = vcl_get_worker_index (); vec_validate (wrk->pending_vcl_wrk_cleanup, 16); vec_reset_length (wrk->pending_vcl_wrk_cleanup); + + return wrk; } static void @@ -1781,6 +1783,7 @@ static void vls_app_fork_child_handler (void) { vcl_worker_t *parent_wrk; + vls_worker_t *vls_wrk; int parent_wrk_index; parent_wrk_index = vcl_get_worker_index (); @@ -1804,11 +1807,11 @@ vls_app_fork_child_handler (void) /* * Allocate/initialize vls worker and share sessions */ - vls_worker_alloc (); + vls_wrk = vls_worker_alloc (); /* Reset number of threads and set wrk index */ vlsl->vls_mt_n_threads = 0; - vlsl->vls_wrk_index = vcl_get_worker_index (); + vlsl->vls_wrk_index = vls_wrk - vlsm->workers; vlsl->select_mp_check = 0; clib_rwlock_init (&vlsl->vls_pool_lock); vls_mt_locks_init (); From 46eaa58f16ae4e5272dab375da387f346fcdf378 Mon Sep 17 00:00:00 2001 From: Monendra Singh Kushwaha Date: Mon, 25 Nov 2024 16:06:26 +0530 Subject: [PATCH 161/271] octeon: add multi-seg support for inline ipsec Type: feature JIRA: https://essjira.marvell.com/browse/IPBUSW-55591 Signed-off-by: Monendra Singh Kushwaha Change-Id: I2b271324d22b0b68e1667e8df36ae23cd17358d0 Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/140309 Tested-by: sa_ip-sw-jenkins Tested-by: Nithin Kumar Dabilpuram Reviewed-by: Nithin Kumar Dabilpuram Klocwork: Nithin Kumar Dabilpuram (cherry picked from commit db95bf7fa2064967b132b9bb17c28f2ffe7ba3eb) Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/144244 --- src/plugins/dev_octeon/init.c | 10 +++ src/plugins/dev_octeon/ipsec.c | 3 +- src/plugins/dev_octeon/rx_node.c | 121 +++++++++++++++++++++++++++++-- 3 files changed, 128 insertions(+), 6 deletions(-) diff --git a/src/plugins/dev_octeon/init.c b/src/plugins/dev_octeon/init.c index c476f9c802..1ef5f87bf3 100644 --- a/src/plugins/dev_octeon/init.c +++ b/src/plugins/dev_octeon/init.c @@ -370,6 +370,15 @@ oct_init_inl_dev (vlib_main_t *vm, vnet_dev_t *dev) oct_inl_dev_main_t *oidm = &oct_inl_dev_main; vnet_dev_rv_t rv; + if ((STRUCT_SIZE_OF (vlib_buffer_t, pre_data) < 128) || + (STRUCT_OFFSET_OF (vlib_buffer_t, pre_data) % ROC_ALIGN)) + { + log_err (dev, "Failed to initalize inline device: pre_data size should " + "be minimum 128 Bytes and offset of pre_data in vlib " + "should be 128 bytes aligned"); + return VNET_DEV_ERR_NOT_SUPPORTED; + } + oidm->inl_dev = oct_plt_init_param.oct_plt_zmalloc ( sizeof (struct roc_nix_inl_dev), CLIB_CACHE_LINE_BYTES); oidm->inl_dev->pci_dev = &od->plt_pci_dev; @@ -377,6 +386,7 @@ oct_init_inl_dev (vlib_main_t *vm, vnet_dev_t *dev) if ((rv = oct_early_init_inline_ipsec (vm, dev))) return rv; + if ((rv = oct_init_ipsec_backend (vm, dev))) return rv; diff --git a/src/plugins/dev_octeon/ipsec.c b/src/plugins/dev_octeon/ipsec.c index bba1556379..d5edadf307 100644 --- a/src/plugins/dev_octeon/ipsec.c +++ b/src/plugins/dev_octeon/ipsec.c @@ -958,7 +958,8 @@ oct_early_init_inline_ipsec (vlib_main_t *vm, vnet_dev_t *dev) inl_dev_main->inl_dev->ipsec_in_min_spi = inl_dev_main->in_min_spi; inl_dev_main->inl_dev->ipsec_in_max_spi = inl_dev_main->in_max_spi; - inl_dev_main->inl_dev->wqe_skip = 0; + inl_dev_main->inl_dev->wqe_skip = + STRUCT_OFFSET_OF (vlib_buffer_t, pre_data) / ROC_ALIGN; inl_dev_main->inl_dev->nb_meta_bufs = bp->n_buffers; if ((rrv = roc_nix_inl_dev_init (inl_dev_main->inl_dev)) < 0) diff --git a/src/plugins/dev_octeon/rx_node.c b/src/plugins/dev_octeon/rx_node.c index dd111a25a2..009510da81 100644 --- a/src/plugins/dev_octeon/rx_node.c +++ b/src/plugins/dev_octeon/rx_node.c @@ -408,6 +408,105 @@ oct_rx_ipsec_set_error (vlib_main_t *vm, vlib_node_runtime_t *node, } } +#define OCT_SEG_LEN_SHIFT 16 +#define OCT_SEG_LEN_MASK 0xFFFF + +static_always_inline u32 +oct_rx_ipsec_attach_tail (vlib_main_t *vm, const union nix_rx_parse_u *rxp0, + vlib_buffer_template_t *bt, vlib_buffer_t *b) +{ + u32 n_words, n_words_processed, desc_sizem1; + vlib_buffer_t *last_buf, *seg_buf; + u32 n_sg_desc, n_segs, next_seg; + u32 current_desc, bi, sg_len; + vlib_buffer_t *buf = b; + struct nix_rx_sg_s *sg; + u32 total_segs = 1; + u64 seg_len; + i64 len; + + desc_sizem1 = rxp0->desc_sizem1; + if (desc_sizem1 == 0) + return total_segs; + + n_words = desc_sizem1 << 1; + n_sg_desc = (n_words / 4) + 1; + + sg = (struct nix_rx_sg_s *) (((char *) rxp0) + sizeof (*rxp0)); + /* Typecast to u64 to read each seg length swiftly */ + seg_len = *(u64 *) sg; + n_segs = sg->segs; + + /* Start with first descriptor */ + current_desc = 0; + + len = buf->current_length; + /* + * We updated length which is valid in single segment case. + * incase of multi seg, update seg1 length and advance total words processed. + * also, updates total bytes in buffer. + */ + buf->current_length = seg_len & OCT_SEG_LEN_MASK; + len -= buf->current_length; + + /* Process from 2nd segment */ + next_seg = 2; + seg_len = seg_len >> OCT_SEG_LEN_SHIFT; + n_words_processed = 2; + + buf->flags |= VLIB_BUFFER_TOTAL_LENGTH_VALID; + buf->total_length_not_including_first_buffer = 0; + last_buf = buf; + + while (current_desc <= n_sg_desc) + { + while (next_seg <= n_segs) + { + seg_buf = (vlib_buffer_t *) ((*(((u64 *) sg) + n_words_processed)) - + sizeof (vlib_buffer_t)); + seg_buf->template = *bt; + sg_len = seg_len & OCT_SEG_LEN_MASK; + + { + /* + * Adjust last buf data length with negative offset for + * ipsec pkts if needed. + */ + len -= sg_len; + sg_len = (len > 0) ? sg_len : (sg_len + len); + len = (len > 0) ? len : 0; + } + + seg_buf->current_length = sg_len; + bi = vlib_get_buffer_index (vm, seg_buf); + last_buf->flags |= VLIB_BUFFER_NEXT_PRESENT; + last_buf->next_buffer = bi; + last_buf = seg_buf; + seg_len = seg_len >> OCT_SEG_LEN_SHIFT; + buf->total_length_not_including_first_buffer += + seg_buf->current_length; + n_words_processed++; + next_seg++; + total_segs++; + } + current_desc++; + n_sg_desc--; + if (n_sg_desc) + { + struct nix_rx_sg_s *tsg; + + tsg = (struct nix_rx_sg_s *) ((u64 *) sg + n_words_processed); + seg_len = *((u64 *) (tsg)); + n_words_processed++; + /* Start over */ + n_segs = tsg->segs; + next_seg = 1; + } + } + + return total_segs; +} + static_always_inline u32 oct_rx_inl_ipsec_vlib_from_cq (vlib_main_t *vm, vlib_node_runtime_t *node, oct_nix_rx_cqe_desc_t *d, vlib_buffer_t **b, @@ -423,7 +522,7 @@ oct_rx_inl_ipsec_vlib_from_cq (vlib_main_t *vm, vlib_node_runtime_t *node, cpt_hdr = (struct cpt_cn10k_parse_hdr_s *) *(((u64 *) d) + 9); wqe_ptr = (u64 *) clib_net_to_host_u64 (cpt_hdr->wqe_ptr); - b[0] = (vlib_buffer_t *) wqe_ptr; + b[0] = (vlib_buffer_t *) ((u8 *) wqe_ptr - 128); orig_rxp = (union nix_rx_parse_u *) (wqe_ptr + 1); l2_ol3_sz = orig_rxp->leptr - orig_rxp->laptr; olen = orig_rxp->pkt_lenm1 + 1; @@ -449,6 +548,7 @@ oct_rx_inl_ipsec_vlib_from_cq (vlib_main_t *vm, vlib_node_runtime_t *node, oct_get_len_from_meta (cpt_hdr, d[0].parse.w[0], d[0].parse.w[4]); idx = cpt_hdr->w0.cookie; oct_rx_ipsec_update_counters (vm, b[0], esp_sz, 1, idx); + oct_rx_ipsec_attach_tail (vm, orig_rxp, bt, b[0]); } ctx->n_rx_bytes += olen; @@ -456,6 +556,7 @@ oct_rx_inl_ipsec_vlib_from_cq (vlib_main_t *vm, vlib_node_runtime_t *node, return 0; } + static_always_inline u32 oct_rx_vlib_from_cq (vlib_main_t *vm, oct_nix_rx_cqe_desc_t *d, vlib_buffer_t **b, oct_rx_node_ctx_t *ctx, @@ -626,10 +727,10 @@ oct_rx_batch (vlib_main_t *vm, vlib_node_runtime_t *node, wqe_ptr2 = (u64 *) clib_net_to_host_u64 (cpt_hdr2->wqe_ptr); wqe_ptr3 = (u64 *) clib_net_to_host_u64 (cpt_hdr3->wqe_ptr); - b[0] = (vlib_buffer_t *) wqe_ptr0; - b[1] = (vlib_buffer_t *) wqe_ptr1; - b[2] = (vlib_buffer_t *) wqe_ptr2; - b[3] = (vlib_buffer_t *) wqe_ptr3; + b[0] = (vlib_buffer_t *) ((u8 *) wqe_ptr0 - 128); + b[1] = (vlib_buffer_t *) ((u8 *) wqe_ptr1 - 128); + b[2] = (vlib_buffer_t *) ((u8 *) wqe_ptr2 - 128); + b[3] = (vlib_buffer_t *) ((u8 *) wqe_ptr3 - 128); orig_rxp0 = (union nix_rx_parse_u *) (wqe_ptr0 + 1); orig_rxp1 = (union nix_rx_parse_u *) (wqe_ptr1 + 1); @@ -691,6 +792,11 @@ oct_rx_batch (vlib_main_t *vm, vlib_node_runtime_t *node, oct_rx_ipsec_update_counters_x4 ( vm, b[0], esp_sz0, 1, idx0, b[1], esp_sz1, 1, idx1, b[2], esp_sz2, 1, idx2, b[3], esp_sz3, 1, idx3); + + oct_rx_ipsec_attach_tail (vm, orig_rxp0, &bt, b[0]); + oct_rx_ipsec_attach_tail (vm, orig_rxp1, &bt, b[1]); + oct_rx_ipsec_attach_tail (vm, orig_rxp2, &bt, b[2]); + oct_rx_ipsec_attach_tail (vm, orig_rxp3, &bt, b[3]); } else { @@ -708,6 +814,7 @@ oct_rx_batch (vlib_main_t *vm, vlib_node_runtime_t *node, cpt_hdr0, d[0].parse.w[0], d[0].parse.w[4]); idx0 = cpt_hdr0->w0.cookie; oct_rx_ipsec_update_counters (vm, b[0], esp_sz0, 1, idx0); + oct_rx_ipsec_attach_tail (vm, orig_rxp0, &bt, b[0]); } if (is_fail1) @@ -724,6 +831,7 @@ oct_rx_batch (vlib_main_t *vm, vlib_node_runtime_t *node, cpt_hdr1, d[1].parse.w[0], d[1].parse.w[4]); idx1 = cpt_hdr1->w0.cookie; oct_rx_ipsec_update_counters (vm, b[1], esp_sz1, 1, idx1); + oct_rx_ipsec_attach_tail (vm, orig_rxp1, &bt, b[1]); } if (is_fail2) @@ -740,6 +848,7 @@ oct_rx_batch (vlib_main_t *vm, vlib_node_runtime_t *node, cpt_hdr2, d[2].parse.w[0], d[2].parse.w[4]); idx2 = cpt_hdr2->w0.cookie; oct_rx_ipsec_update_counters (vm, b[2], esp_sz2, 1, idx2); + oct_rx_ipsec_attach_tail (vm, orig_rxp2, &bt, b[2]); } if (is_fail3) @@ -756,6 +865,7 @@ oct_rx_batch (vlib_main_t *vm, vlib_node_runtime_t *node, cpt_hdr3, d[3].parse.w[0], d[3].parse.w[4]); idx3 = cpt_hdr3->w0.cookie; oct_rx_ipsec_update_counters (vm, b[3], esp_sz3, 1, idx3); + oct_rx_ipsec_attach_tail (vm, orig_rxp3, &bt, b[3]); } } ctx->n_rx_bytes += olen0 + olen1 + olen2 + olen3; @@ -878,6 +988,7 @@ oct_rx_batch (vlib_main_t *vm, vlib_node_runtime_t *node, laddr = (uintptr_t) LMT_OFF (lbase, lnum, 8); } } + if (loff) { /* 16 LMT Line size m1 */ From 61f0d26e2917a110dc93c96fd80c6baca4dbc861 Mon Sep 17 00:00:00 2001 From: Harish Kumar Malik Date: Mon, 18 Nov 2024 15:46:16 +0500 Subject: [PATCH 162/271] octeon: add tm node suspend and resume support Type: feature Signed-off-by: Harish Kumar Malik Change-Id: I2a1f20164a91c0e88be25323efb8db81064f8724 Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/139430 Tested-by: sa_ip-sw-jenkins Reviewed-by: Nithin Kumar Dabilpuram (cherry picked from commit 7ea1bd724b675c38f9add89c40f2670bfe518264) Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/144245 Reviewed-by: Monendra Singh Kushwaha --- src/plugins/dev_octeon/tm.c | 42 +++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/src/plugins/dev_octeon/tm.c b/src/plugins/dev_octeon/tm.c index 8eef54afbd..a7e05cf0bf 100644 --- a/src/plugins/dev_octeon/tm.c +++ b/src/plugins/dev_octeon/tm.c @@ -111,6 +111,48 @@ oct_tm_sys_node_add (u32 hw_if_idx, u32 node_id, u32 parent_node_id, return 0; } +int +oct_tm_sys_node_suspend (u32 hw_if_idx, u32 node_id) +{ + vnet_main_t *vnm = vnet_get_main (); + vnet_hw_interface_t *hi = vnet_get_hw_interface (vnm, hw_if_idx); + vnet_dev_port_t *port = + vnet_dev_get_port_from_dev_instance (hi->dev_instance); + vnet_dev_t *dev = port->dev; + oct_device_t *cd = vnet_dev_get_data (dev); + struct roc_nix *nix = cd->nix; + int rc; + + rc = roc_nix_tm_node_suspend_resume (nix, node_id, true); + if (rc) + { + return oct_roc_err (dev, rc, "roc_nix_tm_node_suspend_failed"); + } + + return rc; +} + +int +oct_tm_sys_node_resume (u32 hw_if_idx, u32 node_id) +{ + vnet_main_t *vnm = vnet_get_main (); + vnet_hw_interface_t *hi = vnet_get_hw_interface (vnm, hw_if_idx); + vnet_dev_port_t *port = + vnet_dev_get_port_from_dev_instance (hi->dev_instance); + vnet_dev_t *dev = port->dev; + oct_device_t *cd = vnet_dev_get_data (dev); + struct roc_nix *nix = cd->nix; + int rc; + + rc = roc_nix_tm_node_suspend_resume (nix, node_id, false); + if (rc) + { + return oct_roc_err (dev, rc, "roc_nix_tm_node_resume_failed"); + } + + return rc; +} + int oct_tm_sys_node_delete (u32 hw_if_idx, u32 node_id) { From 73fa7efad1a13f08d54e2e3272d8061216e31b21 Mon Sep 17 00:00:00 2001 From: Monendra Singh Kushwaha Date: Wed, 29 Jan 2025 03:09:32 +0530 Subject: [PATCH 163/271] build: update roc version Type: feature Signed-off-by: Monendra Singh Kushwaha Change-Id: I7df36074a34dbbf48d6327a8883741d52f73f3a7 Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/144308 Tested-by: sa_ip-sw-jenkins --- build/external/packages/octeon-roc.mk | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build/external/packages/octeon-roc.mk b/build/external/packages/octeon-roc.mk index e69211fd03..73ee48e617 100644 --- a/build/external/packages/octeon-roc.mk +++ b/build/external/packages/octeon-roc.mk @@ -2,9 +2,9 @@ # SPDX-License-Identifier: Apache-2.0 # https://spdx.org/licenses/Apache-2.0.html -octeon-roc_version := octeon-roc-SDK12.25.01 +octeon-roc_version := octeon-roc-SDK12.25.02 octeon-roc_tarball := $(octeon-roc_version).tar.gz -octeon-roc_tarball_md5sum := 43cbc76e76cbb182997b58965e500ba1 +octeon-roc_tarball_md5sum := 030598a6ce8b5b8e402637bfca85675c octeon-roc_github := https://github.com/MarvellEmbeddedProcessors/marvell-octeon-roc octeon-roc_tarball_strip_dirs := 1 From 7dc46818f5f626c7de40b416f75ec473a31c901a Mon Sep 17 00:00:00 2001 From: Satha Rao Date: Wed, 18 Dec 2024 11:07:26 +0530 Subject: [PATCH 164/271] octeon: enable interrupt mechanism This patch enables the interrupt mechanism to support VFIO MSI-X by fixing the PCI bus master enable in the Octeon plugin. Additionally, it introduces an MSI-X message handler to support all vectors. Note that interrupts for MBOX are disabled, as it requires a parallel thread mechanism for handling. Type: feature Signed-off-by: Satha Rao Change-Id: Ic5cecd4a30526a86c25e88cac84d4bc2a5c8fbd5 Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/139703 Reviewed-by: Nithin Kumar Dabilpuram Tested-by: sa_ip-sw-jenkins (cherry picked from commit af0c55a7a3c3c7e7803f948339ff577c093ca6e8) Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/144309 Reviewed-by: Monendra Singh Kushwaha --- src/plugins/dev_octeon/init.c | 11 ++ src/plugins/dev_octeon/octeon.h | 3 +- src/plugins/dev_octeon/port.c | 14 ++ src/plugins/dev_octeon/roc_helper.c | 248 ++++++++++++++++++++++++++++ 4 files changed, 275 insertions(+), 1 deletion(-) diff --git a/src/plugins/dev_octeon/init.c b/src/plugins/dev_octeon/init.c index 1ef5f87bf3..2f4c3add27 100644 --- a/src/plugins/dev_octeon/init.c +++ b/src/plugins/dev_octeon/init.c @@ -485,6 +485,7 @@ oct_init (vlib_main_t *vm, vnet_dev_t *dev) .id.class_id = pci_hdr.class << 16 | pci_hdr.subclass, .pci_handle = vnet_dev_get_pci_handle (dev), }; + cd->msix_handler = NULL; foreach_int (i, 2, 4) { @@ -494,9 +495,19 @@ oct_init (vlib_main_t *vm, vnet_dev_t *dev) return rv; } + if ((rv = vnet_dev_pci_bus_master_enable (vm, dev))) + return rv; + strncpy ((char *) cd->plt_pci_dev.name, dev->device_id, sizeof (cd->plt_pci_dev.name) - 1); + cd->plt_pci_dev.intr_handle = malloc (sizeof (struct oct_pci_intr_handle)); + if (!cd->plt_pci_dev.intr_handle) + return VNET_DEV_ERR_DMA_MEM_ALLOC_FAIL; + memset (cd->plt_pci_dev.intr_handle, 0x0, + sizeof (struct oct_pci_intr_handle)); + cd->plt_pci_dev.intr_handle->pci_handle = cd->plt_pci_dev.pci_handle; + switch (cd->type) { case OCT_DEVICE_TYPE_RVU_PF: diff --git a/src/plugins/dev_octeon/octeon.h b/src/plugins/dev_octeon/octeon.h index 028e25c575..87b37fa4d4 100644 --- a/src/plugins/dev_octeon/octeon.h +++ b/src/plugins/dev_octeon/octeon.h @@ -55,7 +55,6 @@ typedef enum OCT_DEVICE_TYPE_RVU_INL_VF, } __clib_packed oct_device_type_t; - typedef struct { /* vnet flow index */ @@ -71,6 +70,7 @@ typedef struct u8 lf_allocated : 1; u8 tm_initialized : 1; u8 npc_initialized : 1; + u8 q_intr_enabled : 1; struct roc_npc npc; oct_flow_entry_t *flow_entries; } oct_port_t; @@ -144,6 +144,7 @@ typedef struct u32 speed; struct plt_pci_device plt_pci_dev; struct roc_nix *nix; + oct_msix_handler_info_t *msix_handler; u32 cached_cpt_pkts; u64 cpt_io_addr; diff --git a/src/plugins/dev_octeon/port.c b/src/plugins/dev_octeon/port.c index 99527d69a9..9b966ee8a7 100644 --- a/src/plugins/dev_octeon/port.c +++ b/src/plugins/dev_octeon/port.c @@ -303,6 +303,13 @@ oct_port_init (vlib_main_t *vm, vnet_dev_port_t *port) return rv; } + if (roc_nix_register_queue_irqs (nix)) + { + rv = oct_roc_err (dev, rrv, "roc_nix_register_queue_irqs() failed"); + oct_port_deinit (vm, port); + return rv; + } + cp->q_intr_enabled = 1; oct_port_add_counters (vm, port); oct_init_tm_args (&tm_system_ops); @@ -341,6 +348,13 @@ oct_port_deinit (vlib_main_t *vm, vnet_dev_port_t *port) cp->tm_initialized = 0; } + /* Unregister queue irqs */ + if (cp->q_intr_enabled) + { + roc_nix_unregister_queue_irqs (nix); + cp->q_intr_enabled = 0; + } + if (cp->lf_allocated) { if ((rrv = roc_nix_lf_free (nix))) diff --git a/src/plugins/dev_octeon/roc_helper.c b/src/plugins/dev_octeon/roc_helper.c index c1166b654c..23d51e54dc 100644 --- a/src/plugins/dev_octeon/roc_helper.c +++ b/src/plugins/dev_octeon/roc_helper.c @@ -5,10 +5,13 @@ */ #include +#include +#include #include #include #include #include +#include "octeon.h" static oct_plt_memzone_list_t memzone_list; @@ -170,6 +173,247 @@ oct_plt_memzone_reserve_aligned (const char *name, u64 len, u8 socket, return mem_pool; } +static void +plt_msix_handler (vlib_main_t *vm, vlib_pci_dev_handle_t handle, uint16_t line) +{ + vnet_dev_t *dev = (vnet_dev_t *) vlib_pci_get_private_data (vm, handle); + oct_device_t *cd = vnet_dev_get_data (dev); + + if (cd->msix_handler && cd->msix_handler[line].fn) + cd->msix_handler[line].fn (cd->msix_handler[line].data); +} + +static int +oct_plt_get_num_vectors (oct_pci_dev_handle_t handle) +{ + vlib_main_t *vm = vlib_get_main (); + + return vlib_pci_get_num_msix_interrupts (vm, handle); +} + +static int +oct_plt_intr_enable (oct_pci_dev_handle_t handle, uint16_t start, + uint16_t count, uint8_t enable, + enum oct_msix_rsrc_op_t op) +{ + vlib_main_t *vm = vlib_get_main (); + vnet_dev_t *dev = (vnet_dev_t *) vlib_pci_get_private_data (vm, handle); + oct_device_t *cd = vnet_dev_get_data (dev); + clib_error_t *error = NULL; + + if (op == OCT_MSIX_RSRC_ALLOC) + { + if (cd->msix_handler) + { + clib_warning ("MSIX handlers already allocated\n"); + return -EINVAL; + } + cd->msix_handler = malloc (sizeof (*cd->msix_handler) * (start + count)); + if (!cd->msix_handler) + { + clib_warning ("MSIX handlers alilocation failed\n"); + return -ENOMEM; + } + } + if (enable) + error = vlib_pci_enable_msix_irq (vm, handle, start, count); + else + error = vlib_pci_disable_msix_irq (vm, handle, start, count); + if (error) + { + clib_error_report (error); + return -EINVAL; + } + if (op == OCT_MSIX_RSRC_FREE) + { + if (cd->msix_handler) + free (cd->msix_handler); + } + + return 0; +} + +static int +oct_plt_intr_config (oct_pci_dev_handle_t handle, uint32_t vec, + plt_msix_handler_function_t handler, void *data, + int enable) +{ + vlib_main_t *vm = vlib_get_main (); + vnet_dev_t *dev = (vnet_dev_t *) vlib_pci_get_private_data (vm, handle); + oct_device_t *cd = vnet_dev_get_data (dev); + clib_error_t *error = NULL; + + /* Skip AF_PF_MBOX interrupt FIXME */ + if (vec == RVU_PF_INT_VEC_AFPF_MBOX) + return 0; + + if (enable) + { + error = + vlib_pci_register_msix_handler (vm, handle, vec, 1, plt_msix_handler); + if (error) + { + clib_error_report (error); + return -EINVAL; + } + if (cd->msix_handler) + { + cd->msix_handler[vec].fn = handler; + cd->msix_handler[vec].vec = vec; + cd->msix_handler[vec].data = data; + } + error = vlib_pci_enable_msix_irq (vm, handle, vec, 1); + if (error) + { + clib_error_report (error); + return -EINVAL; + } + } + else + { + error = vlib_pci_disable_msix_irq (vm, handle, vec, 1); + if (error) + { + clib_error_report (error); + return -EINVAL; + } + error = vlib_pci_unregister_msix_handler (vm, handle, vec, 1); + if (error) + { + clib_error_report (error); + return -EINVAL; + } + if (cd->msix_handler) + { + cd->msix_handler[vec].fn = NULL; + cd->msix_handler[vec].data = NULL; + } + } + + return 0; +} + +static inline __attribute__ ((__always_inline__)) int +plt_intr_max_intr_get (const struct plt_intr_handle *intr_handle) +{ + if (!intr_handle) + return -EINVAL; + + return intr_handle->max_intr; +} + +static inline __attribute__ ((__always_inline__)) int +plt_intr_max_intr_set (struct plt_intr_handle *intr_handle, int max_intr) +{ + if (!intr_handle) + return -EINVAL; + + intr_handle->max_intr = max_intr; + + return 0; +} + +static int +irq_get_info (struct plt_intr_handle *intr_handle) +{ + int num_vec; + + num_vec = oct_plt_get_num_vectors (intr_handle->pci_handle); + if (num_vec == 0) + { + plt_err ("HW max=%d > PLT_MAX_RXTX_INTR_VEC_ID: %d", num_vec, + PLT_MAX_RXTX_INTR_VEC_ID); + plt_intr_max_intr_set (intr_handle, PLT_MAX_RXTX_INTR_VEC_ID); + } + else + { + if (plt_intr_max_intr_set (intr_handle, num_vec)) + return -1; + } + + return 0; +} + +static int +irq_init (struct plt_intr_handle *intr_handle) +{ + int rc = oct_plt_intr_enable (intr_handle->pci_handle, 0, + plt_intr_max_intr_get (intr_handle), 0, + OCT_MSIX_RSRC_ALLOC); + + if (rc) + plt_err ("Failed to set irqs vector rc=%d", rc); + + return rc; +} + +static int +oct_plt_irq_register (struct oct_pci_intr_handle *intr_handle, + oct_plt_pci_intr_callback_fn cb, void *data, + unsigned int vec) +{ + /* If no max_intr read from VFIO */ + if (plt_intr_max_intr_get (intr_handle) == 0) + { + irq_get_info (intr_handle); + irq_init (intr_handle); + } + + if (vec > (uint32_t) plt_intr_max_intr_get (intr_handle)) + { + plt_err ("Error registering MSI-X interrupts vec:%d > %d", vec, + plt_intr_max_intr_get (intr_handle)); + return -EINVAL; + } + + oct_plt_intr_config (intr_handle->pci_handle, vec, cb, data, 1); + + return 0; +} + +static void +oct_plt_irq_unregister (struct oct_pci_intr_handle *intr_handle, + oct_plt_pci_intr_callback_fn cb, void *data, + unsigned int vec) +{ + if (vec > (uint32_t) plt_intr_max_intr_get (intr_handle)) + { + plt_err ("Error unregistering MSI-X interrupts vec:%d > %d", vec, + plt_intr_max_intr_get (intr_handle)); + return; + } + + oct_plt_intr_config (intr_handle->pci_handle, vec, cb, data, 0); +} + +static int +oct_plt_irq_disable (struct oct_pci_intr_handle *intr_handle) +{ + int rc = -EINVAL; + + if (!intr_handle) + return rc; + + /* Clear max_intr to indicate re-init next time */ + rc = oct_plt_intr_enable (intr_handle->pci_handle, 0, + plt_intr_max_intr_get (intr_handle), 0, + OCT_MSIX_RSRC_FREE); + plt_intr_max_intr_set (intr_handle, 0); + return rc; +} + +static int +oct_plt_irq_reconfigure (struct oct_pci_intr_handle *intr_handle, + uint16_t max_intr) +{ + /* Disable interrupts if enabled. */ + if (plt_intr_max_intr_get (intr_handle)) + oct_plt_irq_disable (intr_handle); + + plt_intr_max_intr_set (intr_handle, max_intr); + return irq_init (intr_handle); +} + oct_plt_init_param_t oct_plt_init_param = { .oct_plt_log_reg_class = vlib_log_register_class, .oct_plt_log = oct_plt_log, @@ -184,4 +428,8 @@ oct_plt_init_param_t oct_plt_init_param = { .oct_plt_spinlock_trylock = oct_plt_spinlock_trylock, .oct_plt_get_thread_index = oct_plt_get_thread_index, .oct_plt_get_cache_line_size = oct_plt_get_cache_line_size, + .oct_plt_irq_reconfigure = oct_plt_irq_reconfigure, + .oct_plt_irq_register = oct_plt_irq_register, + .oct_plt_irq_unregister = oct_plt_irq_unregister, + .oct_plt_irq_disable = oct_plt_irq_disable }; From 8daa9558325f3c499cb39da9ec1350bbd736dd80 Mon Sep 17 00:00:00 2001 From: Monendra Singh Kushwaha Date: Wed, 8 Jan 2025 18:01:17 +0530 Subject: [PATCH 165/271] octeon: add cli to clear inline device counter Type: feature JIRA: https://essjira.marvell.com/browse/IPBUSW-57533 Signed-off-by: Monendra Singh Kushwaha Change-Id: Ib10a65b1cf3dd420c69898196d42370e5027ed0a Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/142789 Tested-by: sa_ip-sw-jenkins Reviewed-by: Nithin Kumar Dabilpuram (cherry picked from commit 1cc1f9ead28b0784f5707860348bb9728f5df7f5) Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/143128 --- src/plugins/dev_octeon/cli.c | 64 ++++++++++++++++++++++++++++++------ 1 file changed, 54 insertions(+), 10 deletions(-) diff --git a/src/plugins/dev_octeon/cli.c b/src/plugins/dev_octeon/cli.c index 3316347d9c..a70c3f5772 100644 --- a/src/plugins/dev_octeon/cli.c +++ b/src/plugins/dev_octeon/cli.c @@ -222,22 +222,66 @@ oct_ipsec_inline_counters_command_fn (vlib_main_t *vm, unformat_input_t *input, roc_nix_inl_dev_stats_get (&stats); - vlib_cli_output (vm, "rx_ucast %u", stats.rx_ucast); - vlib_cli_output (vm, "rx_bcast %u", stats.rx_bcast); - vlib_cli_output (vm, "rx_mcast %u", stats.rx_mcast); - vlib_cli_output (vm, "rx_drop %u", stats.rx_drop); - vlib_cli_output (vm, "rx_fcs %u", stats.rx_fcs); - vlib_cli_output (vm, "rx_err %u", stats.rx_err); - vlib_cli_output (vm, "rx_drop_bcast %u", stats.rx_drop_bcast); - vlib_cli_output (vm, "rx_drop_mcast %u", stats.rx_drop_mcast); - vlib_cli_output (vm, "rx_drop_l3_bcast %u", stats.rx_drop_l3_bcast); - vlib_cli_output (vm, "rx_drop_l3_bcast %u", stats.rx_drop_l3_mcast); + vlib_cli_output (vm, "%-40s %20Ld", "rx_ucast", stats.rx_ucast); + vlib_cli_output (vm, "%-40s %20Ld", "rx_bcast", stats.rx_bcast); + vlib_cli_output (vm, "%-40s %20Ld", "rx_mcast", stats.rx_mcast); + vlib_cli_output (vm, "%-40s %20Ld", "rx_drop", stats.rx_drop); + vlib_cli_output (vm, "%-40s %20Ld", "rx_fcs", stats.rx_fcs); + vlib_cli_output (vm, "%-40s %20Ld", "rx_err", stats.rx_err); + vlib_cli_output (vm, "%-40s %20Ld", "rx_drop_bcast", stats.rx_drop_bcast); + vlib_cli_output (vm, "%-40s %20Ld", "rx_drop_mcast", stats.rx_drop_mcast); + vlib_cli_output (vm, "%-40s %20Ld", "rx_drop_l3_bcast", + stats.rx_drop_l3_bcast); + vlib_cli_output (vm, "%-40s %20Ld", "rx_drop_l3_bcast", + stats.rx_drop_l3_mcast); return 0; } +/*? + * This command displays OCTEON IPsec inline device counters + * + * @cliexpar + * Example of how to display OCTEON IPsec inline device counters: + * @cliexstart{show octeon ipsec inline counters} + * rx_ucast 10 + * rx_bcast 0 + * rx_mcast 0 + * rx_drop 0 + * rx_fcs 0 + * rx_err 0 + * rx_drop_bcast 0 + * rx_drop_mcast 0 + * rx_drop_l3_bcast 0 + * rx_drop_l3_bcast 0 + * @cliexend +?*/ + VLIB_CLI_COMMAND (oct_ipsec_inline_counters_command, static) = { .path = "show octeon ipsec inline counters", .short_help = "show ipsec inline counters", .function = oct_ipsec_inline_counters_command_fn, }; + +static clib_error_t * +oct_ipsec_inline_counters_clear_command_fn (vlib_main_t *vm, + unformat_input_t *input, + vlib_cli_command_t *cmd) +{ + roc_nix_inl_dev_stats_reset (); + + return 0; +} + +/*? + * This command clears OCTEON IPsec inline device counters + * + * @cliexpar + * @cliexstart{clear octeon ipsec inline counters} + * @cliexend +?*/ +VLIB_CLI_COMMAND (oct_ipsec_inline_counters_clear_command, static) = { + .path = "clear octeon ipsec inline counters", + .short_help = "clear ipsec inline counters", + .function = oct_ipsec_inline_counters_clear_command_fn, +}; From 81e8be1028c0a49f1390062634a327127c3bd3ca Mon Sep 17 00:00:00 2001 From: Monendra Singh Kushwaha Date: Sat, 11 Jan 2025 14:10:28 +0530 Subject: [PATCH 166/271] octeon: add single rx aura handle per pktio This patches fixes single rx aura handle to per pktio interface. Type: fix Change-Id: I66aad05382081717767c689c899bf7839570583a Signed-off-by: Monendra Singh Kushwaha Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/143076 Tested-by: sa_ip-sw-jenkins Reviewed-by: Nithin Kumar Dabilpuram (cherry picked from commit 026373f198502cf15030a67d8c44518871eeb665) Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/144310 --- src/plugins/dev_octeon/octeon.h | 3 +-- src/plugins/dev_octeon/queue.c | 6 +++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/src/plugins/dev_octeon/octeon.h b/src/plugins/dev_octeon/octeon.h index 87b37fa4d4..17042b740f 100644 --- a/src/plugins/dev_octeon/octeon.h +++ b/src/plugins/dev_octeon/octeon.h @@ -146,6 +146,7 @@ typedef struct struct roc_nix *nix; oct_msix_handler_info_t *msix_handler; + u64 aura_handle; u32 cached_cpt_pkts; u64 cpt_io_addr; oct_txq_t **ctqs; @@ -158,9 +159,7 @@ typedef struct oct_device_t **oct_dev; u8 inl_dev_initialized : 1; u8 use_single_rx_tx_aura : 1; - u64 aura_handle; u64 tx_aura_handle; - } oct_main_t; extern oct_main_t oct_main; diff --git a/src/plugins/dev_octeon/queue.c b/src/plugins/dev_octeon/queue.c index f5792aca1b..abf2a46d8b 100644 --- a/src/plugins/dev_octeon/queue.c +++ b/src/plugins/dev_octeon/queue.c @@ -103,7 +103,7 @@ oct_rxq_init (vlib_main_t *vm, vnet_dev_rx_queue_t *rxq, u32 total_sz) ASSERT (!(vm->buffer_main->ext_hdr_size % ROC_ALIGN)); - if (!om->use_single_rx_tx_aura || !om->aura_handle) + if (!om->use_single_rx_tx_aura || !cd->aura_handle) { if ((rrv = roc_npa_pool_create (&crq->aura_handle, bp->alloc_size, total_sz, &aura, &npapool, 0))) @@ -111,11 +111,11 @@ oct_rxq_init (vlib_main_t *vm, vnet_dev_rx_queue_t *rxq, u32 total_sz) oct_rxq_deinit (vm, rxq); return oct_roc_err (dev, rrv, "roc_npa_pool_create() failed"); } - om->aura_handle = crq->aura_handle; + cd->aura_handle = crq->aura_handle; } else { - crq->aura_handle = om->aura_handle; + crq->aura_handle = cd->aura_handle; } crq->npa_pool_initialized = 1; From c44efcd0a761bed32dd934dc7d6262332c9570ea Mon Sep 17 00:00:00 2001 From: Monendra Singh Kushwaha Date: Thu, 16 Jan 2025 22:04:00 +0530 Subject: [PATCH 167/271] octeon: fix meta pool callback Type: fix fixes: 3b72c845 Signed-off-by: Monendra Singh Kushwaha Change-Id: Ic1890d0fefeca04880a0622ab9b3484af57b6173 Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/143427 Tested-by: sa_ip-sw-jenkins Reviewed-by: Nithin Kumar Dabilpuram (cherry picked from commit 0866672af9526a5213c2a676ef99d9241e9e7aea) Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/144311 --- src/plugins/dev_octeon/ipsec.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plugins/dev_octeon/ipsec.c b/src/plugins/dev_octeon/ipsec.c index d5edadf307..9cf35206c8 100644 --- a/src/plugins/dev_octeon/ipsec.c +++ b/src/plugins/dev_octeon/ipsec.c @@ -854,7 +854,7 @@ oct_pool_inl_meta_pool_cb (u64 *aura_handle, uintptr_t *mpool, u32 buf_sz, npapool.nat_align = 1; rv = roc_npa_pool_create (&roc_aura_handle, buf_sz, nb_bufs, &aura, &npapool, - ROC_NPA_ZERO_AURA_F); + mempool_name ? 0 : ROC_NPA_ZERO_AURA_F); if (rv) { clib_warning ("roc_npa_pool_create failed with '%s' error", From d4331ae77e64be5e9433dda89f26b4aad2e6e056 Mon Sep 17 00:00:00 2001 From: Nithinsen Kaithakadan Date: Wed, 4 Dec 2024 14:28:41 +0530 Subject: [PATCH 168/271] octeon: add direct mode changes in crypto datapath This patch introduces support for direct mode crypto submission on CPT. For multi-segmented buffers, scatter-gather submission mode will be utilized. Type: fix JIRA: https://essjira.marvell.com/browse/IPBUSW-51987 Signed-off-by: Nithinsen Kaithakadan Change-Id: Idb99e6c9ea49028e11d3bf530c9559719c988252 Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/140761 Tested-by: sa_ip-sw-jenkins Reviewed-by: Tejasree Kondoj Reviewed-by: Nithinsen Kaithakadan (cherry picked from commit e896a10f428d817bcc4927cb1497f8ef1bf15566) Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/144349 Reviewed-by: Monendra Singh Kushwaha --- src/plugins/dev_octeon/crypto.c | 259 +++++++++++++++++++++++++++----- src/plugins/dev_octeon/crypto.h | 11 ++ 2 files changed, 230 insertions(+), 40 deletions(-) diff --git a/src/plugins/dev_octeon/crypto.c b/src/plugins/dev_octeon/crypto.c index 0cf9931332..7f999dd88f 100644 --- a/src/plugins/dev_octeon/crypto.c +++ b/src/plugins/dev_octeon/crypto.c @@ -1064,12 +1064,11 @@ oct_crypto_cpt_hmac_prep (u32 flags, u64 d_offs, u64 d_lens, } static_always_inline void -oct_crypto_fill_fc_params (oct_crypto_sess_t *sess, struct cpt_inst_s *inst, - const bool is_aead, u8 aad_length, u8 *payload, - vnet_crypto_async_frame_elt_t *elts, void *mdata, - u32 cipher_data_length, u32 cipher_data_offset, - u32 auth_data_length, u32 auth_data_offset, - vlib_buffer_t *b, u16 adj_len) +oct_crypto_scatter_gather_mode ( + oct_crypto_sess_t *sess, struct cpt_inst_s *inst, const bool is_aead, + u8 aad_length, u8 *payload, vnet_crypto_async_frame_elt_t *elts, void *mdata, + u32 cipher_data_length, u32 cipher_data_offset, u32 auth_data_length, + u32 auth_data_offset, vlib_buffer_t *b, u16 adj_len) { struct roc_se_fc_params fc_params = { 0 }; struct roc_se_ctx *ctx = &sess->cpt_ctx; @@ -1307,6 +1306,13 @@ oct_crypto_link_session_update (vlib_main_t *vm, oct_crypto_sess_t *sess, return -1; } + sess->cpt_ctx.template_w4.s.opcode_major = ROC_SE_MAJOR_OP_FC; + + if (sess->cpt_op == VNET_CRYPTO_OP_TYPE_DECRYPT) + sess->cpt_ctx.template_w4.s.opcode_minor |= ROC_SE_FC_MINOR_OP_DECRYPT; + else + sess->cpt_ctx.template_w4.s.opcode_minor |= ROC_SE_FC_MINOR_OP_ENCRYPT; + return 0; } @@ -1363,6 +1369,13 @@ oct_crypto_aead_session_update (vlib_main_t *vm, oct_crypto_sess_t *sess, return -1; } + sess->cpt_ctx.template_w4.s.opcode_major = ROC_SE_MAJOR_OP_FC; + + if (sess->cpt_op == VNET_CRYPTO_OP_TYPE_DECRYPT) + sess->cpt_ctx.template_w4.s.opcode_minor |= ROC_SE_FC_MINOR_OP_DECRYPT; + else + sess->cpt_ctx.template_w4.s.opcode_minor |= ROC_SE_FC_MINOR_OP_ENCRYPT; + if (enc_type == ROC_SE_CHACHA20) sess->cpt_ctx.template_w4.s.opcode_minor |= BIT (5); @@ -1418,6 +1431,138 @@ oct_crypto_update_frame_error_status (vnet_crypto_async_frame_t *f, f->state = VNET_CRYPTO_FRAME_STATE_NOT_PROCESSED; } +static_always_inline void +oct_crypto_direct_mode_linked (vlib_buffer_t *buffer, struct cpt_inst_s *inst, + oct_crypto_sess_t *sess, + oct_crypto_inflight_req_t *infl_req, u8 aad_len) +{ + u32 encr_offset, auth_offset, iv_offset; + vnet_crypto_async_frame_elt_t *elts; + union cpt_inst_w4 cpt_inst_w4; + u64 *offset_control_word; + u32 crypto_total_length; + u32 auth_dlen, enc_dlen; + u32 enc_auth_len; + + elts = infl_req->fe; + enc_auth_len = elts->crypto_total_length + elts->integ_length_adj; + crypto_total_length = elts->crypto_total_length; + + if (sess->cpt_op == VNET_CRYPTO_OP_TYPE_DECRYPT) + { + /* + * Position the offset control word so that it does not + * overlap with the IV. + */ + offset_control_word = (void *) (buffer->data) - ROC_SE_OFF_CTRL_LEN - 4; + + iv_offset = + (void *) elts->iv - (void *) offset_control_word - ROC_SE_OFF_CTRL_LEN; + } + else + { + offset_control_word = (void *) (elts->iv) - ROC_SE_OFF_CTRL_LEN; + iv_offset = 0; + } + + encr_offset = (void *) (buffer->data + elts->crypto_start_offset) - + (void *) offset_control_word - ROC_SE_OFF_CTRL_LEN; + auth_offset = (void *) (buffer->data + elts->integ_start_offset) - + (void *) offset_control_word - ROC_SE_OFF_CTRL_LEN; + *offset_control_word = clib_host_to_net_u64 ( + ((u64) encr_offset << 16) | ((u64) iv_offset << 8) | ((u64) auth_offset)); + + cpt_inst_w4.u64 = sess->cpt_ctx.template_w4.u64; + + cpt_inst_w4.s.param1 = crypto_total_length; + cpt_inst_w4.s.param2 = enc_auth_len; + + auth_dlen = auth_offset + enc_auth_len + ROC_SE_OFF_CTRL_LEN; + enc_dlen = encr_offset + crypto_total_length + ROC_SE_OFF_CTRL_LEN; + + if (sess->cpt_op == VNET_CRYPTO_OP_TYPE_DECRYPT) + cpt_inst_w4.s.dlen = auth_dlen + sess->cpt_ctx.mac_len; + else + { + /* + * In the case of ESN, 4 bytes of the seqhi will be stored at the end of + * the cipher. This data must be overwritten by the digest data during + * the dequeue process. + */ + if (auth_dlen > enc_dlen) + infl_req->esn_enabled = true; + + cpt_inst_w4.s.dlen = auth_dlen; + } + + infl_req->mac_len = sess->cpt_ctx.mac_len; + + inst->dptr = (uint64_t) offset_control_word; + inst->rptr = (uint64_t) ((void *) offset_control_word + ROC_SE_OFF_CTRL_LEN); + inst->w4.u64 = cpt_inst_w4.u64; +} + +static_always_inline void +oct_crypto_direct_mode_aead (vlib_buffer_t *buffer, struct cpt_inst_s *inst, + oct_crypto_sess_t *sess, + oct_crypto_inflight_req_t *infl_req, u8 aad_len) +{ + u32 encr_offset, auth_offset, iv_offset; + u32 auth_copy_offset, iv_copy_offset; + vnet_crypto_async_frame_elt_t *elts; + union cpt_inst_w4 cpt_inst_w4; + u64 *offset_control_word; + u32 crypto_total_length; + + elts = infl_req->fe; + crypto_total_length = elts->crypto_total_length; + + ((u32 *) elts->iv)[3] = clib_host_to_net_u32 (0x1); + + offset_control_word = (void *) (elts->aad) - ROC_SE_OFF_CTRL_LEN; + encr_offset = (void *) (buffer->data + elts->crypto_start_offset) - + (void *) offset_control_word - ROC_SE_OFF_CTRL_LEN; + iv_offset = elts->iv - elts->aad; + auth_offset = encr_offset - aad_len; + + *offset_control_word = clib_host_to_net_u64 ( + ((u64) encr_offset << 16) | ((u64) iv_offset << 8) | ((u64) auth_offset)); + + cpt_inst_w4.u64 = sess->cpt_ctx.template_w4.u64; + + cpt_inst_w4.s.param1 = crypto_total_length; + cpt_inst_w4.s.param2 = crypto_total_length + aad_len; + + if (sess->cpt_op == VNET_CRYPTO_OP_TYPE_DECRYPT) + cpt_inst_w4.s.dlen = encr_offset + elts->crypto_total_length + + ROC_SE_OFF_CTRL_LEN + sess->cpt_ctx.mac_len; + else + cpt_inst_w4.s.dlen = + encr_offset + elts->crypto_total_length + ROC_SE_OFF_CTRL_LEN; + + inst->dptr = (uint64_t) offset_control_word; + inst->rptr = (uint64_t) ((void *) offset_control_word + ROC_SE_OFF_CTRL_LEN); + inst->w4.u64 = cpt_inst_w4.u64; + + /* + * CPT hardware requires the AAD to be followed by the cipher packet. + * Therefore, maintain a copy of the AAD and IV in the inflight request, + * and write the AAD in front of the cipher data before submission. + */ + auth_copy_offset = encr_offset - sess->cpt_ctx.mac_len; + iv_copy_offset = encr_offset - 8; + + clib_memcpy_fast (infl_req->aad, + ((void *) inst->dptr) + auth_copy_offset + 8, 8); + clib_memcpy_fast (infl_req->iv, ((void *) inst->dptr) + iv_copy_offset + 8, + 8); + clib_memcpy_fast (((void *) inst->dptr) + encr_offset + ROC_SE_OFF_CTRL_LEN - + aad_len, + elts->aad, aad_len); + + infl_req->aead_algo = true; +} + static_always_inline int oct_crypto_enqueue_enc_dec (vlib_main_t *vm, vnet_crypto_async_frame_t *frame, const u8 is_aead, u8 aad_len, const u8 type) @@ -1486,43 +1631,58 @@ oct_crypto_enqueue_enc_dec (vlib_main_t *vm, vnet_crypto_async_frame_t *frame, if (is_aead) { - dptr_start_ptr = - (u64) (buffer->data + (elts->crypto_start_offset - aad_iv)); - curr_ptr = (u64) (buffer->data + buffer->current_data); - adj_len = (u16) (dptr_start_ptr - curr_ptr); - - crypto_total_length = elts->crypto_total_length; - crypto_start_offset = aad_iv; - integ_start_offset = 0; - - oct_crypto_fill_fc_params ( - sess, inst + i, is_aead, aad_len, (u8 *) dptr_start_ptr, elts, - ((oct_crypto_scatter_gather_t *) (sg_data)) + enq_tail, - crypto_total_length /* cipher_len */, - crypto_start_offset /* cipher_offset */, 0 /* auth_len */, - integ_start_offset /* auth_off */, buffer, adj_len); + if (buffer->flags & VLIB_BUFFER_NEXT_PRESENT) + { + dptr_start_ptr = + (u64) (buffer->data + (elts->crypto_start_offset - aad_iv)); + curr_ptr = (u64) (buffer->data + buffer->current_data); + adj_len = (u16) (dptr_start_ptr - curr_ptr); + + crypto_total_length = elts->crypto_total_length; + crypto_start_offset = aad_iv; + integ_start_offset = 0; + oct_crypto_scatter_gather_mode ( + sess, inst + i, is_aead, aad_len, (u8 *) dptr_start_ptr, elts, + ((oct_crypto_scatter_gather_t *) (sg_data)) + enq_tail, + crypto_total_length /* cipher_len */, + crypto_start_offset /* cipher_offset */, 0 /* auth_len */, + integ_start_offset /* auth_off */, buffer, adj_len); + } + else + { + oct_crypto_direct_mode_aead (buffer, inst + i, sess, infl_req, + aad_len); + } } else { - dptr_start_ptr = (u64) (buffer->data + elts->integ_start_offset); - - enc_auth_len = elts->crypto_total_length + elts->integ_length_adj; - - curr_ptr = (u64) (buffer->data + buffer->current_data); - adj_len = (u16) (dptr_start_ptr - curr_ptr); - - crypto_total_length = elts->crypto_total_length; - crypto_start_offset = - elts->crypto_start_offset - elts->integ_start_offset; - integ_start_offset = 0; - - oct_crypto_fill_fc_params ( - sess, inst + i, is_aead, aad_len, (u8 *) dptr_start_ptr, elts, - ((oct_crypto_scatter_gather_t *) (sg_data)) + enq_tail, - crypto_total_length /* cipher_len */, - crypto_start_offset /* cipher_offset */, - enc_auth_len /* auth_len */, integ_start_offset /* auth_off */, - buffer, adj_len); + if (buffer->flags & VLIB_BUFFER_NEXT_PRESENT) + { + dptr_start_ptr = (u64) (buffer->data + elts->integ_start_offset); + + curr_ptr = (u64) (buffer->data + buffer->current_data); + adj_len = (u16) (dptr_start_ptr - curr_ptr); + + crypto_start_offset = + elts->crypto_start_offset - elts->integ_start_offset; + integ_start_offset = 0; + enc_auth_len = + elts->crypto_total_length + elts->integ_length_adj; + crypto_total_length = elts->crypto_total_length; + + oct_crypto_scatter_gather_mode ( + sess, inst + i, is_aead, aad_len, (u8 *) dptr_start_ptr, elts, + ((oct_crypto_scatter_gather_t *) (sg_data)) + enq_tail, + crypto_total_length /* cipher_len */, + crypto_start_offset /* cipher_offset */, + enc_auth_len /* auth_len */, integ_start_offset /* auth_off */, + buffer, adj_len); + } + else + { + oct_crypto_direct_mode_linked (buffer, inst + i, sess, infl_req, + aad_len); + } } inst[i].w7.u64 = sess->cpt_inst_w7; @@ -1630,6 +1790,7 @@ oct_crypto_frame_dequeue (vlib_main_t *vm, u32 *nb_elts_processed, vnet_crypto_async_frame_t *frame; volatile union cpt_res_s *res; bool last_elts_processed; + vlib_buffer_t *buffer; pend_q = &ocm->pend_q[vlib_get_thread_index ()]; @@ -1657,6 +1818,24 @@ oct_crypto_frame_dequeue (vlib_main_t *vm, u32 *nb_elts_processed, status = fe->status = VNET_CRYPTO_OP_STATUS_FAIL_ENGINE_ERR; } + buffer = + vlib_get_buffer (vm, infl_req->frame->buffer_indices[infl_req->index]); + + /* + * For AEAD, copy the AAD and IV back to their original positions. + * If ESN is enabled (in case of linked algo), overwrite the ESN + * seqhi at the end of the cipher with the digest data. + */ + if (infl_req->aead_algo) + { + clib_memcpy_fast (buffer->data + fe->crypto_start_offset - 8, + infl_req->iv, 8); + clib_memcpy_fast (buffer->data + fe->crypto_start_offset - 16, + infl_req->aad, 8); + } + else if (infl_req->esn_enabled) + clib_memcpy_fast (fe->digest, fe->digest + 4, infl_req->mac_len); + clib_memset ((void *) &infl_req->res, 0, sizeof (union cpt_res_s)); last_elts_processed = infl_req->last_elts; OCT_MOD_INC (pend_q->deq_head, pend_q->n_desc); diff --git a/src/plugins/dev_octeon/crypto.h b/src/plugins/dev_octeon/crypto.h index c24d6d9d24..aa9d68bc66 100644 --- a/src/plugins/dev_octeon/crypto.h +++ b/src/plugins/dev_octeon/crypto.h @@ -128,6 +128,17 @@ typedef struct vnet_crypto_async_frame_t *frame; /** Async frame element */ vnet_crypto_async_frame_elt_t *fe; + /** AAD meta data */ + u8 aad[8]; + /** IV meta data */ + u8 iv[16]; + /** Digest len */ + u8 mac_len; + /** aead */ + bool aead_algo; + /** Set when encrypting linked algo with esn. + * To move digest data */ + bool esn_enabled; /** Set if this is last element in frame */ bool last_elts; /** Index of element in frame */ From 18cb7bcea64e06b742f0af50b7cc7c434ac31904 Mon Sep 17 00:00:00 2001 From: Monendra Singh Kushwaha Date: Mon, 20 Jan 2025 12:18:26 +0530 Subject: [PATCH 169/271] octeon: update plt memzone and realloc functions This patch adds plt realloc function and updates memzone functions. Type: fix Signed-off-by: Monendra Singh Kushwaha Change-Id: I6aeb5e550b16658e6480926d9dfbe24d762cc11a Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/143579 Tested-by: sa_ip-sw-jenkins Reviewed-by: Nithinsen Kaithakadan (cherry picked from commit cc6b919824616ed7221fb309c4dd2712f83015f4) Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/144350 --- src/plugins/dev_octeon/roc_helper.c | 45 ++++++++++++++--------------- 1 file changed, 22 insertions(+), 23 deletions(-) diff --git a/src/plugins/dev_octeon/roc_helper.c b/src/plugins/dev_octeon/roc_helper.c index 23d51e54dc..7e73d76595 100644 --- a/src/plugins/dev_octeon/roc_helper.c +++ b/src/plugins/dev_octeon/roc_helper.c @@ -117,38 +117,36 @@ oct_plt_zmalloc (u32 size, u32 align) return oct_drv_physmem_alloc (vm, size, align); } -static oct_plt_memzone_t * -memzone_get (u32 index) +static void * +oct_plt_realloc (void *addr, u32 size, u32 align) { - if (index == ((u32) ~0)) - return 0; - - return pool_elt_at_index (memzone_list.mem_pool, index); + if (align) + return clib_mem_realloc_aligned (addr, size, align); + else + return clib_mem_realloc (addr, size); } -static int -oct_plt_memzone_free (const oct_plt_memzone_t *name) +static oct_plt_memzone_t * +oct_plt_memzone_lookup (const char *name) { - uword *p; - p = hash_get_mem (memzone_list.memzone_by_name, name); - - if (p[0] == ((u32) ~0)) - return -EINVAL; - - hash_unset_mem (memzone_list.memzone_by_name, name); + oct_plt_memzone_t *mem_pool; - pool_put_index (memzone_list.mem_pool, p[0]); + pool_foreach (mem_pool, memzone_list.mem_pool) + { + if (!clib_strcmp (mem_pool->name, name)) + return mem_pool; + } return 0; } -static oct_plt_memzone_t * -oct_plt_memzone_lookup (const char *name) +static int +oct_plt_memzone_free (const oct_plt_memzone_t *mz) { - uword *p; - p = hash_get_mem (memzone_list.memzone_by_name, name); - if (p) - return memzone_get (p[0]); + if (!mz || !oct_plt_memzone_lookup (mz->name)) + return -EINVAL; + + pool_put (memzone_list.mem_pool, mz); return 0; } @@ -168,7 +166,7 @@ oct_plt_memzone_reserve_aligned (const char *name, u64 len, u8 socket, mem_pool->addr = p; mem_pool->index = mem_pool - memzone_list.mem_pool; - hash_set_mem (memzone_list.memzone_by_name, name, mem_pool->index); + strcpy (mem_pool->name, name); return mem_pool; } @@ -419,6 +417,7 @@ oct_plt_init_param_t oct_plt_init_param = { .oct_plt_log = oct_plt_log, .oct_plt_free = oct_plt_free, .oct_plt_zmalloc = oct_plt_zmalloc, + .oct_plt_realloc = oct_plt_realloc, .oct_plt_memzone_free = oct_plt_memzone_free, .oct_plt_memzone_lookup = oct_plt_memzone_lookup, .oct_plt_memzone_reserve_aligned = oct_plt_memzone_reserve_aligned, From fa5465605abc73153e13561c2732cbc6dfe537da Mon Sep 17 00:00:00 2001 From: Monendra Singh Kushwaha Date: Tue, 21 Jan 2025 14:20:29 +0530 Subject: [PATCH 170/271] octeon: fix issue with multi-seg support packet Type: fix Change-Id: I1d6e5ed5e1b2d0d26a2ee0da272df84e0ade70b0 Signed-off-by: Monendra Singh Kushwaha Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/143674 Tested-by: sa_ip-sw-jenkins Reviewed-by: Nithin Kumar Dabilpuram (cherry picked from commit 8f49e5fcf041ccde694ced7af014ddbb8f8335d2) Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/144351 --- src/plugins/dev_octeon/port.c | 2 +- src/plugins/dev_octeon/queue.c | 15 +++++++++++---- src/plugins/dev_octeon/rx_node.c | 8 ++++++++ src/plugins/dev_octeon/tx_node.c | 2 +- 4 files changed, 21 insertions(+), 6 deletions(-) diff --git a/src/plugins/dev_octeon/port.c b/src/plugins/dev_octeon/port.c index 9b966ee8a7..6da591802e 100644 --- a/src/plugins/dev_octeon/port.c +++ b/src/plugins/dev_octeon/port.c @@ -167,6 +167,7 @@ oct_port_init (vlib_main_t *vm, vnet_dev_port_t *port) struct roc_nix *nix = cd->nix; vnet_dev_rv_t rv = -1; bool is_allmulti_enable = false, is_pause_frame_enable = false; + u32 total_sz = 0; int rrv; log_notice (dev, "port init: port %u", port->port_id); @@ -260,7 +261,6 @@ oct_port_init (vlib_main_t *vm, vnet_dev_port_t *port) } cp->npc_initialized = 1; - u64 total_sz = 0; foreach_vnet_dev_port_rx_queue (q, port) total_sz += q->size; diff --git a/src/plugins/dev_octeon/queue.c b/src/plugins/dev_octeon/queue.c index abf2a46d8b..fde8f7a928 100644 --- a/src/plugins/dev_octeon/queue.c +++ b/src/plugins/dev_octeon/queue.c @@ -103,6 +103,9 @@ oct_rxq_init (vlib_main_t *vm, vnet_dev_rx_queue_t *rxq, u32 total_sz) ASSERT (!(vm->buffer_main->ext_hdr_size % ROC_ALIGN)); + if (!om->use_single_rx_tx_aura) + total_sz = rxq->size; + if (!om->use_single_rx_tx_aura || !cd->aura_handle) { if ((rrv = roc_npa_pool_create (&crq->aura_handle, bp->alloc_size, @@ -226,14 +229,18 @@ oct_txq_init (vlib_main_t *vm, vnet_dev_tx_queue_t *txq) struct npa_pool_s npapool = { .nat_align = 1, .buf_offset = OCT_EXT_HDR_SIZE / ROC_ALIGN }; int rrv; + u32 n_buffers = 0; vlib_buffer_pool_t *bp = vlib_get_buffer_pool (vm, 0); + if (om->use_single_rx_tx_aura) + n_buffers = bp->n_buffers; + else + n_buffers = txq->size * 6; + if (!om->use_single_rx_tx_aura || !om->tx_aura_handle) { - if ((rrv = roc_npa_pool_create ( - &ctq->aura_handle, bp->alloc_size, - txq->size * 6 /* worst case - two SG with 3 segs each = 6 */, - &aura, &npapool, 0))) + if ((rrv = roc_npa_pool_create (&ctq->aura_handle, bp->alloc_size, + n_buffers, &aura, &npapool, 0))) { oct_txq_deinit (vm, txq); return oct_roc_err (dev, rrv, "roc_npa_pool_create() failed"); diff --git a/src/plugins/dev_octeon/rx_node.c b/src/plugins/dev_octeon/rx_node.c index 009510da81..f827417108 100644 --- a/src/plugins/dev_octeon/rx_node.c +++ b/src/plugins/dev_octeon/rx_node.c @@ -479,6 +479,14 @@ oct_rx_ipsec_attach_tail (vlib_main_t *vm, const union nix_rx_parse_u *rxp0, seg_buf->current_length = sg_len; bi = vlib_get_buffer_index (vm, seg_buf); + + if (seg_buf->current_length == 0) + { + vlib_buffer_free_no_next (vm, &bi, 1); + total_segs++; + return total_segs; + } + last_buf->flags |= VLIB_BUFFER_NEXT_PRESENT; last_buf->next_buffer = bi; last_buf = seg_buf; diff --git a/src/plugins/dev_octeon/tx_node.c b/src/plugins/dev_octeon/tx_node.c index aa804a3c34..a488609576 100644 --- a/src/plugins/dev_octeon/tx_node.c +++ b/src/plugins/dev_octeon/tx_node.c @@ -801,7 +801,7 @@ oct_pkts_send (vlib_main_t *vm, vlib_node_runtime_t *node, oct_tx_ctx_t *ctx, vnet_dev_tx_queue_t *txq, u16 tx_pkts, vlib_buffer_t **bufs) { oct_txq_t *ctq = vnet_dev_get_tx_queue_data (txq); - u32 desc_sz = 4; + u32 desc_sz = 10 /* Worst case - Send hdr + Two SG with 3 segs each */; union nix_send_sg_s *sg8, *sg9, *sg10, *sg11, *sg12, *sg13, *sg14, *sg15; struct nix_send_hdr_s *send_hdr12, *send_hdr13, *send_hdr14, *send_hdr15; struct nix_send_hdr_s *send_hdr8, *send_hdr9, *send_hdr10, *send_hdr11; From fc0ac33697d8bc3d8ebc3619fd9a5237441363b8 Mon Sep 17 00:00:00 2001 From: Florin Coras Date: Thu, 31 Oct 2024 00:55:04 -0700 Subject: [PATCH 171/271] vcl: make ldp workers thread local Multi-threaded apps that do not allocate per-thread workers (multi-thread workers vcl config) ended up sharing worker state like the select bitmaps and time among others. Those should not be shared. To avoid this, make ldp workers thread local variables. Type: fix Signed-off-by: Florin Coras Change-Id: Iabdcc413991dbaafff33f24187f7053a9c5a1270 Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/143835 Tested-by: sa_ip-sw-jenkins Reviewed-by: Venkata Ravichandra Mynidi Reviewed-by: Monendra Singh Kushwaha (cherry picked from commit 53f57fff0c4878c680f232c6526604b333cc1a22) Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/144352 --- src/vcl/ldp.c | 26 +++++--------------------- 1 file changed, 5 insertions(+), 21 deletions(-) diff --git a/src/vcl/ldp.c b/src/vcl/ldp.c index 752aa6b076..be9e5ed5ca 100644 --- a/src/vcl/ldp.c +++ b/src/vcl/ldp.c @@ -105,6 +105,8 @@ typedef struct ldp_worker_ctx_ } ldp_worker_ctx_t; +__thread ldp_worker_ctx_t _ldp_worker = {}; + /* clib_bitmap_t, fd_mask and vcl_si_set are used interchangeably. Make sure * they are the same size */ STATIC_ASSERT (sizeof (clib_bitmap_t) == sizeof (fd_mask), @@ -114,7 +116,6 @@ STATIC_ASSERT (sizeof (vcl_si_set) == sizeof (fd_mask), typedef struct { - ldp_worker_ctx_t *workers; int init; char app_name[LDP_APP_NAME_MAX]; u32 vlsh_bit_val; @@ -154,7 +155,7 @@ static ldp_main_t *ldp = &ldp_main; static inline ldp_worker_ctx_t * ldp_worker_get_current (void) { - return (ldp->workers + vppcom_worker_index ()); + return &_ldp_worker; } /* @@ -190,14 +191,6 @@ ldp_fd_to_vlsh (int fd) return (fd - ldp->vlsh_bit_val); } -static void -ldp_alloc_workers (void) -{ - if (ldp->workers) - return; - ldp->workers = vec_new (ldp_worker_ctx_t, LDP_MAX_NWORKERS); -} - static void ldp_init_cfg (void) { @@ -285,7 +278,6 @@ ldp_init_cfg (void) static int ldp_init (void) { - ldp_worker_ctx_t *ldpw; int rv; if (ldp->init) @@ -311,10 +303,6 @@ ldp_init (void) return rv; } ldp->vcl_needs_real_epoll = 0; - ldp_alloc_workers (); - - vec_foreach (ldpw, ldp->workers) - clib_memset (&ldpw->clib_time, 0, sizeof (ldpw->clib_time)); LDBG (0, "LDP initialization: done!"); @@ -2388,14 +2376,9 @@ epoll_create1 (int flags) if (ldp->vcl_needs_real_epoll || vls_use_real_epoll ()) { - /* Make sure workers have been allocated */ - if (!ldp->workers) - { - ldp_alloc_workers (); - ldpw = ldp_worker_get_current (); - } rv = libc_epoll_create1 (flags); ldp->vcl_needs_real_epoll = 0; + /* Assume this is a request to create the mq epfd */ ldpw->vcl_mq_epfd = rv; LDBG (0, "created vcl epfd %u", rv); return rv; @@ -2656,6 +2639,7 @@ ldp_epoll_pwait_eventfd (int epfd, struct epoll_event *events, if (PREDICT_FALSE (!ldpw->mq_epfd_added)) { struct epoll_event e = { 0 }; + ldpw->vcl_mq_epfd = vppcom_mq_epoll_fd (); e.events = EPOLLIN; e.data.fd = ldpw->vcl_mq_epfd; if (libc_epoll_ctl (libc_epfd, EPOLL_CTL_ADD, ldpw->vcl_mq_epfd, &e) < From 47c5000a42b90a0ce563fcaf28c418483ade6f2f Mon Sep 17 00:00:00 2001 From: Monendra Singh Kushwaha Date: Fri, 29 Nov 2024 12:50:11 +0530 Subject: [PATCH 172/271] octeon: add error handling for inline IPsec This patch handles error for inline inbound and inline outbound IPsec operations and free the packet. Type: feature JIRA: https://essjira.marvell.com/browse/IPBUSW-53992 Signed-off-by: Monendra Singh Kushwaha Change-Id: Id4c76b561004e5a5ef59080cc019513f6c1f739d Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/140366 Tested-by: sa_ip-sw-jenkins Reviewed-by: Nithin Kumar Dabilpuram (cherry picked from commit 956f3285263c43a60eab52346516a07fd075fdf1) Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/144353 --- src/plugins/dev_octeon/ipsec.c | 50 ++++++++++++++++++++++++++++++++++ src/plugins/dev_octeon/ipsec.h | 3 ++ src/plugins/dev_octeon/queue.c | 2 ++ 3 files changed, 55 insertions(+) diff --git a/src/plugins/dev_octeon/ipsec.c b/src/plugins/dev_octeon/ipsec.c index 9cf35206c8..9f842866f9 100644 --- a/src/plugins/dev_octeon/ipsec.c +++ b/src/plugins/dev_octeon/ipsec.c @@ -256,12 +256,17 @@ oct_ipsec_inb_ctx_size (struct roc_ot_ipsec_inb_sa *sa) static_always_inline void oct_ipsec_common_inst_param_fill (void *sa, oct_ipsec_session_t *sess) { + union cpt_inst_w2 w2; union cpt_inst_w3 w3; clib_memset (&sess->inst, 0, sizeof (struct cpt_inst_s)); sess->inst.w7.u64 = oct_ipsec_crypto_inst_w7_get (sa); + w2.u64 = 0; + w2.u64 = ((u64) OCT_EVENT_TYPE_FRM_CPU << 28); + sess->inst.w2.u64 = w2.u64; + /* Populate word3 in CPT instruction template */ w3.u64 = 0; w3.s.qord = 1; @@ -944,6 +949,48 @@ oct_ipsec_inl_dev_outb_cfg (vnet_dev_t *dev, oct_inl_dev_cfg_t *inl_dev_cfg) return VNET_DEV_OK; } +void +oct_ipsec_sso_work_cb (uint64_t *gw, void *args, uint32_t soft_exp_event) +{ + vlib_main_t *vm = vlib_get_main (); + struct roc_ot_ipsec_outb_sa *sa; + oct_ipsec_outb_sa_priv_data_t *outb_priv; + vlib_buffer_t *b; + u32 bi; + + switch ((gw[0] >> 28) & 0xF) + { + case OCT_EVENT_TYPE_FRM_INL_DEV: + /* Event from inbound inline dev due to IPSEC packet bad L4 */ + b = (vlib_buffer_t *) (gw[1] - sizeof (vlib_buffer_t)); + bi = vlib_get_buffer_index (vm, b); + vlib_buffer_free_no_next (vm, &bi, 1); + return; + case OCT_EVENT_TYPE_FRM_CPU: + /* Event from outbound inline error */ + b = (vlib_buffer_t *) gw[1]; + vlib_buffer_free_one (vm, vlib_get_buffer_index (vm, b)); + break; + /* Fall through */ + default: + if (soft_exp_event & 0x1) + { + sa = (struct roc_ot_ipsec_outb_sa *) args; + outb_priv = roc_nix_inl_ot_ipsec_outb_sa_sw_rsvd (sa); + clib_warning ("Soft expiry event received for sa_index %u", + outb_priv->sa_idx); + } + else + { + clib_warning ("Unknown event gw[0] = 0x%016lx, gw[1] = 0x%016lx", + gw[0], gw[1]); + } + return; + } + + return; +} + vnet_dev_rv_t oct_early_init_inline_ipsec (vlib_main_t *vm, vnet_dev_t *dev) { @@ -987,5 +1034,8 @@ oct_init_nix_inline_ipsec (vlib_main_t *vm, vnet_dev_t *inl_dev, if ((rv = oct_ipsec_inl_dev_outb_cfg (dev, &inl_dev_cfg))) return rv; + /* Register callback to handle security error work */ + roc_nix_inl_cb_register (oct_ipsec_sso_work_cb, NULL); + return VNET_DEV_OK; } diff --git a/src/plugins/dev_octeon/ipsec.h b/src/plugins/dev_octeon/ipsec.h index 02c07375c0..fd83fb724d 100644 --- a/src/plugins/dev_octeon/ipsec.h +++ b/src/plugins/dev_octeon/ipsec.h @@ -7,6 +7,9 @@ #ifndef _OCTEON_IPSEC_H_ #define _OCTEON_IPSEC_H_ +#define OCT_EVENT_TYPE_FRM_INL_DEV 0x0 +#define OCT_EVENT_TYPE_FRM_CPU 0x1 + #define OCT_ROC_SALT_LEN 4 #define OCT_EXT_HDR_FROM_VLIB_BUFFER(x) \ (((oct_ipsec_outbound_pkt_meta_t *) (x)) - 1) diff --git a/src/plugins/dev_octeon/queue.c b/src/plugins/dev_octeon/queue.c index fde8f7a928..6eb6ed4019 100644 --- a/src/plugins/dev_octeon/queue.c +++ b/src/plugins/dev_octeon/queue.c @@ -173,6 +173,8 @@ oct_rxq_init (vlib_main_t *vm, vnet_dev_rx_queue_t *rxq, u32 total_sz) log_debug (dev, "RQ %u initialised", crq->cq.qid); /* Configure inline device rq */ + crq->rq.tag_mask = + 0x0FF00000 | ((uint32_t) OCT_EVENT_TYPE_FRM_INL_DEV << 28); rrv = roc_nix_inl_dev_rq_get (&crq->rq, 0 /* disable */); if (rrv) { From 3379eaf61cd5e6976b470e8832adb30e69287265 Mon Sep 17 00:00:00 2001 From: Nawal Kishor Date: Mon, 20 Jan 2025 21:08:32 +0530 Subject: [PATCH 173/271] ci: add ci scripts for enabling CI Added ci scripts that -> * builds deps and VPP. * performs klocwork check. Type: feature Signed-off-by: Nawal Kishor Change-Id: I2753c0cf4f8263e042248d64c66e24303069977a Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/143624 Tested-by: sa_ip-sw-jenkins Tested-by: Nithin Kumar Dabilpuram Reviewed-by: Ashwin Sekhar T K Reviewed-by: Nithin Kumar Dabilpuram (cherry picked from commit 950439cc4759e5e1813187a4339888d28671c0a9) Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/144354 Reviewed-by: Monendra Singh Kushwaha --- MAINTAINERS | 5 ++ ci/build/build-deps.sh | 45 +++++++++++++++ ci/build/build.sh | 86 ++++++++++++++++++++++++++++ ci/checkpatch/run_sanity_check.sh | 5 ++ ci/klocwork/klocwork.sh | 95 +++++++++++++++++++++++++++++++ 5 files changed, 236 insertions(+) create mode 100755 ci/build/build-deps.sh create mode 100755 ci/build/build.sh create mode 100755 ci/checkpatch/run_sanity_check.sh create mode 100755 ci/klocwork/klocwork.sh diff --git a/MAINTAINERS b/MAINTAINERS index 9cc05f2d9f..60007309cb 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -901,6 +901,11 @@ M: Filip Tehlar M: Maros Ondrejicka F: extras/hs-test +CI - Enable VPP dev-ci +I: ci +M: Nawal Kishor +F: ci/ + THE REST I: misc M: vpp-dev Mailing List diff --git a/ci/build/build-deps.sh b/ci/build/build-deps.sh new file mode 100755 index 0000000000..797e8ca70e --- /dev/null +++ b/ci/build/build-deps.sh @@ -0,0 +1,45 @@ +#!/bin/bash +# Marvell CONFIDENTIAL AND PROPRIETARY NOTE +# +# This software contains information confidential and proprietary to +# Marvell. It shall not be reproduced in whole or in +# part, or transferred to other documents, or disclosed to third +# parties, or used for any purpose other than that for which it was +# obtained, without the prior written consent of Marvell. +# +# Copyright (c) 2025 Marvell. If you received this file from Marvell +# and you have entered into a commercial license agreement (a "Commercial License") +# with Marvell, the file is licensed to you under the terms of the applicable Commercial +# License. In the absence of such license agreement the following file is subject to +# Marvell’s standard Limited Use License Agreement. + +# Script syntax: +# build-deps.sh +# +# Script will: +# 1. Create +# 2. Fetch and build dependencies. + +set -euo pipefail +shopt -s extglob + +CROSS_COMPILE=${CROSS_COMPILE:-aarch64-marvell-linux-gnu} +BUILD_ROOT=$(realpath $1) +LIBUUID_DIR=${BUILD_ROOT}/libuuid +DEPS_DIR=${BUILD_ROOT}/deps-prefix + +function build_libuuid { + rm -rf ${LIBUUID_DIR} + mkdir -p ${LIBUUID_DIR} + cd ${LIBUUID_DIR} + wget https://github.com/util-linux/util-linux/archive/refs/tags/v2.38.tar.gz + tar -xvf v2.38.tar.gz + cd util-linux-2.38 + ./autogen.sh + ./configure --target=${CROSS_COMPILE} --host=${CROSS_COMPILE} \ + --build=x86_64-pc-linux-gnu --disable-all-programs --enable-libuuid \ + --prefix ${DEPS_DIR} + make install +} + +build_libuuid diff --git a/ci/build/build.sh b/ci/build/build.sh new file mode 100755 index 0000000000..10690b9c69 --- /dev/null +++ b/ci/build/build.sh @@ -0,0 +1,86 @@ +#!/bin/bash +# Marvell CONFIDENTIAL AND PROPRIETARY NOTE +# +# This software contains information confidential and proprietary to +# Marvell. It shall not be reproduced in whole or in +# part, or transferred to other documents, or disclosed to third +# parties, or used for any purpose other than that for which it was +# obtained, without the prior written consent of Marvell. +# +# Copyright (c) 2025 Marvell. If you received this file from Marvell +# and you have entered into a commercial license agreement (a "Commercial License") +# with Marvell, the file is licensed to you under the terms of the applicable Commercial +# License. In the absence of such license agreement the following file is subject to +# Marvell’s standard Limited Use License Agreement. + +set -euo pipefail + +function help() { + echo "Builds VPP libraries and applications." + echo "" + echo "Usage:" + echo "$SCRIPT_NAME [ARGUMENTS]..." + echo "" + echo "Mandatory Arguments" + echo "===================" + echo "--build-type | -b : Build type; release/debug" + echo "--deps_dir | -d : Deps dir" + echo "" + echo "Optional Arguments" + echo "===================" + echo "--octeon_version | -o : Version(cn10k, cn9k)" + echo "--help | -h : Print this help and exit" +} + +SCRIPT_NAME="$(basename "$0")" +if ! OPTS=$(getopt \ + -o "b:d:oh" \ + -l "build-type:,deps-dir:,octeon_version,help" \ + -n "$SCRIPT_NAME" \ + -- "$@"); then + help + exit 1 +fi + +DEPS_DIR= +BUILD= +export CROSS="aarch64-marvell-linux-gnu-" +export OCTEON_VERSION="cn10k" +export PLATFORM="cnxk" + +eval set -- "$OPTS" +unset OPTS +while [[ $# -gt 1 ]]; do + case $1 in + -b|--build-type) shift; BUILD=$1;; + -d|--deps-dir) shift; DEPS_DIR=$(realpath $1);; + -o|--octeon-version) shift; OCTEON_VERSION=$1;; + -h|--help) help; exit 0;; + *) help; exit 1;; + esac + shift +done + +if [[ -z $BUILD || -z $DEPS_DIR ]]; then + echo "Build_type and Deps directory should be passed as argument !!" + help + exit 1 +fi + +if [[ $BUILD == "debug" ]]; then + BUILD_TYPE=build +elif [[ $BUILD == "release" ]]; then + BUILD_TYPE=build-release +else + echo "Pass build-type (release/debug)" + help + exit 1 +fi + +DEPS_PREFIX=${DEPS_DIR}/deps-prefix +export cnxk_c_flags="-I/${DEPS_PREFIX}/include/ -L/${DEPS_PREFIX}/lib" +export UNATTENDED=y +export DEBIAN_FRONTEND=noninteractive +# FIXME: Remove install-dep command when these deps are installed in docker container. +make install-dep +make $BUILD_TYPE diff --git a/ci/checkpatch/run_sanity_check.sh b/ci/checkpatch/run_sanity_check.sh new file mode 100755 index 0000000000..4185366577 --- /dev/null +++ b/ci/checkpatch/run_sanity_check.sh @@ -0,0 +1,5 @@ +#!/bin/bash +# SPDX-License-Identifier: Marvell-MIT +# Copyright (c) 2025 Marvell. + +# Add when required. diff --git a/ci/klocwork/klocwork.sh b/ci/klocwork/klocwork.sh new file mode 100755 index 0000000000..f485580d13 --- /dev/null +++ b/ci/klocwork/klocwork.sh @@ -0,0 +1,95 @@ +#!/bin/bash +# Marvell CONFIDENTIAL AND PROPRIETARY NOTE +# +# This software contains information confidential and proprietary to +# Marvell. It shall not be reproduced in whole or in +# part, or transferred to other documents, or disclosed to third +# parties, or used for any purpose other than that for which it was +# obtained, without the prior written consent of Marvell. +# +# Copyright (c) 2025 Marvell. If you received this file from Marvell +# and you have entered into a commercial license agreement (a "Commercial License") +# with Marvell, the file is licensed to you under the terms of the applicable Commercial +# License. In the absence of such license agreement the following file is subject to +# Marvell’s standard Limited Use License Agreement. + +set -euo pipefail + +function help() { + echo "Builds VPP libraries and applications with klocwork" + echo "" + echo "Usage:" + echo "$SCRIPT_NAME [ARGUMENTS]..." + echo "" + echo "Mandatory Arguments" + echo "===================" + echo "--build-root | -r : Build root directory" + echo "--deps_dir | -d : Deps dir" + echo "--cnxk_sdk_sysroot | -s : CNXK_SDK_SYSROOT path" + echo "" + echo "Optional Arguments" + echo "===================" + echo "--octeon_version | -o : Version(cn10k, cn9k)" + echo "--help | -h : Print this help and exit" +} + +SCRIPT_NAME="$(basename "$0")" +if ! OPTS=$(getopt \ + -o "r:d:s:oh" \ + -l "build-root:,deps-dir:,cnxk_sdk_sysroot:,octeon_version,help" \ + -n "$SCRIPT_NAME" \ + -- "$@"); then + help + exit 1 +fi + +BUILD_ROOT= +DEPS_DIR= +CNXK_SYSROOT= +export CROSS="aarch64-marvell-linux-gnu-" +export OCTEON_VERSION="cn10k" +export PLATFORM="cnxk" + +eval set -- "$OPTS" +unset OPTS +while [[ $# -gt 1 ]]; do + case $1 in + -r|--build_root) shift; BUILD_ROOT=$(realpath $1);; + -d|--deps-dir) shift; DEPS_DIR=$(realpath $1);; + -s|--cnxk_sdk_sysroot) shift; CNXK_SYSROOT=$1;; + -o|--octeon-version) shift; OCTEON_VERSION=$1;; + -h|--help) help; exit 0;; + *) help; exit 1;; + esac + shift +done + +if [[ -z $DEPS_DIR || -z $BUILD_ROOT || -z $CNXK_SYSROOT ]]; then + echo "Deps directory, build root and cnxk_sdk_sysroot should be passed as argument !!" + help + exit 1 +fi + +DEPS_PREFIX=${DEPS_DIR}/deps-prefix +export CNXK_SDK_SYSROOT=$CNXK_SYSROOT +export cnxk_c_flags="-I/${DEPS_PREFIX}/include/ -L/${DEPS_PREFIX}/lib" +export UNATTENDED=y +export DEBIAN_FRONTEND=noninteractive +# FIXME: Remove install-dep command when these deps are installed in docker container. +make install-dep +rm -rf .kwlp .kwps +kwcheck create +kwcheck set license.host=llic5-01.marvell.com license.port=33138 + +# List of directories to ignore in klocwork checks +IGNORE_FILES="" + +kwinject --ignore-files $IGNORE_FILES -w make build +kwcheck run -r -b kwinject.out -F detailed --report kwreport-detailed.txt +kwcheck list -F scriptable --report kwreport-scritpable.txt +CNXK_ISSUES=$(wc -l kwreport-scritpable.txt | awk '{print $1}') + +echo "#########################################################################" +echo "Klocwork CNXK Issues: $CNXK_ISSUES" +echo "Klocwork Report : $PWD/kwreport-detailed.txt" +echo "#########################################################################" From 7150fe79c5c5b24c6a26fa71a767c4c9108f3c77 Mon Sep 17 00:00:00 2001 From: Monendra Singh Kushwaha Date: Mon, 27 Jan 2025 12:18:13 +0530 Subject: [PATCH 174/271] octeon: fix quad loop processing in rx node Type: fix Signed-off-by: Monendra Singh Kushwaha Change-Id: Ia0b74bec1df2fbf31d9c3ab6a4cee635ef39d1f0 Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/144079 Tested-by: sa_ip-toolkits-Jenkins (cherry picked from commit ee2419a60f704a776e5da92ff8417387c832c773) Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/144355 Tested-by: sa_ip-sw-jenkins --- src/plugins/dev_octeon/rx_node.c | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/plugins/dev_octeon/rx_node.c b/src/plugins/dev_octeon/rx_node.c index f827417108..d6d91a6b79 100644 --- a/src/plugins/dev_octeon/rx_node.c +++ b/src/plugins/dev_octeon/rx_node.c @@ -786,11 +786,11 @@ oct_rx_batch (vlib_main_t *vm, vlib_node_runtime_t *node, b[0]->current_length = oct_get_len_from_meta ( cpt_hdr0, d[0].parse.w[0], d[0].parse.w[4]); b[1]->current_length = oct_get_len_from_meta ( - cpt_hdr0, d[1].parse.w[0], d[1].parse.w[4]); + cpt_hdr1, d[1].parse.w[0], d[1].parse.w[4]); b[2]->current_length = oct_get_len_from_meta ( - cpt_hdr0, d[2].parse.w[0], d[2].parse.w[4]); + cpt_hdr2, d[2].parse.w[0], d[2].parse.w[4]); b[3]->current_length = oct_get_len_from_meta ( - cpt_hdr0, d[3].parse.w[0], d[3].parse.w[4]); + cpt_hdr3, d[3].parse.w[0], d[3].parse.w[4]); idx0 = cpt_hdr0->w0.cookie; idx1 = cpt_hdr1->w0.cookie; @@ -911,46 +911,46 @@ oct_rx_batch (vlib_main_t *vm, vlib_node_runtime_t *node, cpt_hdr0 = (struct cpt_cn10k_parse_hdr_s *) *(((u64 *) &d[0]) + 9); oct_rx_inl_ipsec_vlib_from_cq (vm, node, &d[0], &b[0], ctx, &bt, - cpt_hdr0, buffs, &err_flags); + cpt_hdr0, &buffs[0], &err_flags); OCT_PUSH_META_TO_FREE ((u64) cpt_hdr0, laddr, &loff); } else oct_rx_vlib_from_cq (vm, &d[0], &b[0], ctx, &bt, meta_aura_handle, - buffs, &err_flags); + &buffs[0], &err_flags); if (is_b1_from_cpt) { cpt_hdr1 = (struct cpt_cn10k_parse_hdr_s *) *(((u64 *) &d[1]) + 9); oct_rx_inl_ipsec_vlib_from_cq (vm, node, &d[1], &b[1], ctx, &bt, - cpt_hdr1, buffs, &err_flags); + cpt_hdr1, &buffs[1], &err_flags); OCT_PUSH_META_TO_FREE ((u64) cpt_hdr1, laddr, &loff); } else oct_rx_vlib_from_cq (vm, &d[1], &b[1], ctx, &bt, meta_aura_handle, - buffs, &err_flags); + &buffs[1], &err_flags); if (is_b2_from_cpt) { cpt_hdr2 = (struct cpt_cn10k_parse_hdr_s *) *(((u64 *) &d[2]) + 9); oct_rx_inl_ipsec_vlib_from_cq (vm, node, &d[2], &b[2], ctx, &bt, - cpt_hdr2, buffs, &err_flags); + cpt_hdr2, &buffs[2], &err_flags); OCT_PUSH_META_TO_FREE ((u64) cpt_hdr2, laddr, &loff); } else oct_rx_vlib_from_cq (vm, &d[2], &b[2], ctx, &bt, meta_aura_handle, - buffs, &err_flags); + &buffs[2], &err_flags); if (is_b3_from_cpt) { cpt_hdr3 = (struct cpt_cn10k_parse_hdr_s *) *(((u64 *) &d[3]) + 9); oct_rx_inl_ipsec_vlib_from_cq (vm, node, &d[3], &b[3], ctx, &bt, - cpt_hdr3, buffs, &err_flags); + cpt_hdr3, &buffs[3], &err_flags); OCT_PUSH_META_TO_FREE ((u64) cpt_hdr3, laddr, &loff); } else oct_rx_vlib_from_cq (vm, &d[3], &b[3], ctx, &bt, meta_aura_handle, - buffs, &err_flags); + &buffs[3], &err_flags); } buffs += 4; /* Check if lmtline border is crossed and adjust lnum */ From ebaa4cd60835b0db485e16251eb60d770b15f2a1 Mon Sep 17 00:00:00 2001 From: Nithinsen Kaithakadan Date: Mon, 6 Jan 2025 18:11:22 +0530 Subject: [PATCH 175/271] crypto: add async algo macros for ctr sha2 Add async crypto algo macros for AES_CTR SHA256/384/512. Add support for these in dev octeon plugin. Type: feature Signed-off-by: Nithinsen Kaithakadan Change-Id: I22e81c6ac5a549b2f12556b8c79257a20a5bd47d Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/142634 Tested-by: sa_ip-toolkits-Jenkins Reviewed-by: Monendra Singh Kushwaha (cherry picked from commit d9c8416d86c32418063303c22cfe17d311b60a4a) Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/144356 Tested-by: sa_ip-sw-jenkins --- src/plugins/dev_octeon/crypto.c | 21 +++++++++++++++++++++ src/plugins/dev_octeon/crypto.h | 13 +++++++++++-- src/vnet/crypto/crypto.h | 11 ++++++++++- 3 files changed, 42 insertions(+), 3 deletions(-) diff --git a/src/plugins/dev_octeon/crypto.c b/src/plugins/dev_octeon/crypto.c index 7f999dd88f..3048405842 100644 --- a/src/plugins/dev_octeon/crypto.c +++ b/src/plugins/dev_octeon/crypto.c @@ -1240,6 +1240,27 @@ oct_crypto_link_session_update (vlib_main_t *vm, oct_crypto_sess_t *sess, auth_type = ROC_SE_SHA1_TYPE; digest_len = 12; break; + case VNET_CRYPTO_ALG_AES_128_CTR_SHA256_TAG16: + case VNET_CRYPTO_ALG_AES_192_CTR_SHA256_TAG16: + case VNET_CRYPTO_ALG_AES_256_CTR_SHA256_TAG16: + enc_type = ROC_SE_AES_CTR; + auth_type = ROC_SE_SHA2_SHA256; + digest_len = 16; + break; + case VNET_CRYPTO_ALG_AES_128_CTR_SHA384_TAG24: + case VNET_CRYPTO_ALG_AES_192_CTR_SHA384_TAG24: + case VNET_CRYPTO_ALG_AES_256_CTR_SHA384_TAG24: + enc_type = ROC_SE_AES_CTR; + auth_type = ROC_SE_SHA2_SHA384; + digest_len = 24; + break; + case VNET_CRYPTO_ALG_AES_128_CTR_SHA512_TAG32: + case VNET_CRYPTO_ALG_AES_192_CTR_SHA512_TAG32: + case VNET_CRYPTO_ALG_AES_256_CTR_SHA512_TAG32: + enc_type = ROC_SE_AES_CTR; + auth_type = ROC_SE_SHA2_SHA512; + digest_len = 32; + break; case VNET_CRYPTO_ALG_3DES_CBC_MD5_TAG12: enc_type = ROC_SE_DES3_CBC; auth_type = ROC_SE_MD5_TYPE; diff --git a/src/plugins/dev_octeon/crypto.h b/src/plugins/dev_octeon/crypto.h index aa9d68bc66..2224bc1cd1 100644 --- a/src/plugins/dev_octeon/crypto.h +++ b/src/plugins/dev_octeon/crypto.h @@ -57,11 +57,20 @@ _ (3DES_CBC, SHA512, 24, 32) \ _ (AES_128_CTR, SHA1, 16, 12) \ _ (AES_192_CTR, SHA1, 24, 12) \ - _ (AES_256_CTR, SHA1, 32, 12) + _ (AES_256_CTR, SHA1, 32, 12) \ + _ (AES_128_CTR, SHA256, 16, 16) \ + _ (AES_192_CTR, SHA256, 24, 16) \ + _ (AES_256_CTR, SHA256, 32, 16) \ + _ (AES_128_CTR, SHA384, 16, 24) \ + _ (AES_192_CTR, SHA384, 24, 24) \ + _ (AES_256_CTR, SHA384, 32, 24) \ + _ (AES_128_CTR, SHA512, 16, 32) \ + _ (AES_192_CTR, SHA512, 24, 32) \ + _ (AES_256_CTR, SHA512, 32, 32) #define OCT_MOD_INC(i, l) ((i) == (l - 1) ? (i) = 0 : (i)++) -#define OCT_SCATTER_GATHER_BUFFER_SIZE 1024 +#define OCT_SCATTER_GATHER_BUFFER_SIZE 1024 #define CPT_LMT_SIZE_COPY (sizeof (struct cpt_inst_s) / 16) #define OCT_MAX_LMT_SZ 16 diff --git a/src/vnet/crypto/crypto.h b/src/vnet/crypto/crypto.h index 877eb183ea..1f13f80e58 100644 --- a/src/vnet/crypto/crypto.h +++ b/src/vnet/crypto/crypto.h @@ -130,7 +130,16 @@ typedef enum _ (AES_256_CBC, SHA512, "aes-256-cbc-hmac-sha-512", 32, 32) \ _ (AES_128_CTR, SHA1, "aes-128-ctr-hmac-sha-1", 16, 12) \ _ (AES_192_CTR, SHA1, "aes-192-ctr-hmac-sha-1", 24, 12) \ - _ (AES_256_CTR, SHA1, "aes-256-ctr-hmac-sha-1", 32, 12) + _ (AES_256_CTR, SHA1, "aes-256-ctr-hmac-sha-1", 32, 12) \ + _ (AES_128_CTR, SHA256, "aes-128-ctr-hmac-sha-256", 16, 16) \ + _ (AES_192_CTR, SHA256, "aes-192-ctr-hmac-sha-256", 24, 16) \ + _ (AES_256_CTR, SHA256, "aes-256-ctr-hmac-sha-256", 32, 16) \ + _ (AES_128_CTR, SHA384, "aes-128-ctr-hmac-sha-384", 16, 24) \ + _ (AES_192_CTR, SHA384, "aes-192-ctr-hmac-sha-384", 24, 24) \ + _ (AES_256_CTR, SHA384, "aes-256-ctr-hmac-sha-384", 32, 24) \ + _ (AES_128_CTR, SHA512, "aes-128-ctr-hmac-sha-512", 16, 32) \ + _ (AES_192_CTR, SHA512, "aes-192-ctr-hmac-sha-512", 24, 32) \ + _ (AES_256_CTR, SHA512, "aes-256-ctr-hmac-sha-512", 32, 32) #define foreach_crypto_async_op_type \ _(ENCRYPT, "async-encrypt") \ From 86579f4826f0f2751ab5c02378bafd1b38bbe5ae Mon Sep 17 00:00:00 2001 From: Monendra Singh Kushwaha Date: Thu, 12 Dec 2024 01:20:40 +0530 Subject: [PATCH 176/271] octeon: optimize inline outbound IPsec performance Type: improvement Signed-off-by: Monendra Singh Kushwaha Change-Id: Ieeec0bf394880ce4a56603a24219a78031af2c56 Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/141489 Reviewed-by: Nithin Kumar Dabilpuram Tested-by: sa_ip-toolkits-Jenkins (cherry picked from commit 8359fc377f782648685b521ceeccc813841a2023) Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/144438 Tested-by: sa_ip-sw-jenkins --- src/plugins/dev_octeon/esp_encrypt.c | 289 ++------------------------- src/plugins/dev_octeon/ipsec.h | 24 ++- src/plugins/dev_octeon/queue.c | 5 +- src/plugins/dev_octeon/roc_helper.c | 5 +- src/plugins/dev_octeon/tx_node.c | 245 ++++++++++++++++++----- 5 files changed, 233 insertions(+), 335 deletions(-) diff --git a/src/plugins/dev_octeon/esp_encrypt.c b/src/plugins/dev_octeon/esp_encrypt.c index 2786294f79..78a75988c7 100644 --- a/src/plugins/dev_octeon/esp_encrypt.c +++ b/src/plugins/dev_octeon/esp_encrypt.c @@ -5,49 +5,6 @@ extern oct_ipsec_main_t oct_ipsec_main; -#define foreach_oct_esp_handoff_core \ - _ (0) \ - _ (1) \ - _ (2) \ - _ (3) \ - _ (4) \ - _ (5) \ - _ (6) \ - _ (7) \ - _ (8) \ - _ (9) \ - _ (10) \ - _ (11) \ - _ (12) \ - _ (13) \ - _ (14) \ - _ (15) \ - _ (16) \ - _ (17) \ - _ (18) \ - _ (19) \ - _ (20) \ - _ (21) \ - _ (22) \ - _ (23) \ - _ (24) \ - _ (25) \ - _ (26) \ - _ (27) \ - _ (28) \ - _ (29) \ - _ (30) \ - _ (31) \ - _ (32) \ - _ (33) \ - _ (34) \ - _ (35) - -#define OCT_ESP_ENCRYPT_TUN_N_NEXT (OCT_ESP_ENCRYPT_TUN_NEXT_PREP35 + 1) - -#define OCT_ESP_ENCRYPT_TUN_PREP_NODES(x, y) \ - [OCT_ESP_ENCRYPT_TUN_NEXT_PREP##y] = #x #y, - #define foreach_oct_esp_encrypt_error \ _ (RX_PKTS, "ESP pkts received") \ _ (RX_POST_PKTS, "ESP-POST pkts received") \ @@ -87,9 +44,7 @@ typedef enum #define _(v, s) OCT_ESP_ENCRYPT_TUN_NEXT_##v, foreach_oct_esp_encrypt_tun_next #undef _ -#define _(Y) OCT_ESP_ENCRYPT_TUN_NEXT_PREP##Y, - foreach_oct_esp_handoff_core -#undef _ + OCT_ESP_ENCRYPT_TUN_N_NEXT } oct_esp_encrypt_tun_next_t; /* clang-format on */ @@ -209,12 +164,10 @@ oct_esp_encrypt_tun (vlib_main_t *vm, vlib_node_runtime_t *node, u32 *from = vlib_frame_vector_args (frame); u32 n_left = frame->n_vectors; vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b = bufs; - u16 nexts[VLIB_FRAME_SIZE], *next = nexts; u32 sa0_index, sa1_index, sa2_index, sa3_index; u32 current_sa0_index = ~0, current_sa1_index = ~0; u32 current_sa2_index = ~0, current_sa3_index = ~0; ipsec_sa_t *sa0 = NULL, *sa1 = NULL, *sa2 = NULL, *sa3 = NULL; - u16 thread0_next = OCT_ESP_ENCRYPT_TUN_NEXT_PREP0; vlib_get_buffers (vm, from, b, frame->n_vectors); @@ -270,12 +223,11 @@ oct_esp_encrypt_tun (vlib_main_t *vm, vlib_node_runtime_t *node, vnet_buffer (b[2])->ipsec.thread_index = sa2->thread_index; vnet_buffer (b[3])->ipsec.thread_index = sa3->thread_index; - next[0] = thread0_next + sa0->thread_index; - next[1] = thread0_next + sa1->thread_index; - next[2] = thread0_next + sa2->thread_index; - next[3] = thread0_next + sa3->thread_index; + vnet_buffer (b[0])->oflags |= VNET_BUFFER_OFFLOAD_F_IPSEC_OFFLOAD; + vnet_buffer (b[1])->oflags |= VNET_BUFFER_OFFLOAD_F_IPSEC_OFFLOAD; + vnet_buffer (b[2])->oflags |= VNET_BUFFER_OFFLOAD_F_IPSEC_OFFLOAD; + vnet_buffer (b[3])->oflags |= VNET_BUFFER_OFFLOAD_F_IPSEC_OFFLOAD; - next += 4; b += 4; n_left -= 4; } @@ -301,9 +253,9 @@ oct_esp_encrypt_tun (vlib_main_t *vm, vlib_node_runtime_t *node, sa0->thread_index = (sa0_index % vlib_num_workers ()) + 1; vnet_buffer (b[0])->ipsec.thread_index = sa0->thread_index; - next[0] = thread0_next + sa0->thread_index; - next++; + vnet_buffer (b[0])->oflags |= VNET_BUFFER_OFFLOAD_F_IPSEC_OFFLOAD; + b++; n_left--; } @@ -311,22 +263,24 @@ oct_esp_encrypt_tun (vlib_main_t *vm, vlib_node_runtime_t *node, if (PREDICT_FALSE (node->flags & VLIB_NODE_FLAG_TRACE)) { n_left = frame->n_vectors; - next = nexts; b = bufs; while (n_left > 0) { if (PREDICT_FALSE (b[0]->flags & VLIB_BUFFER_IS_TRACED)) { - oct_esp_encrypt_tun_add_trace (vm, node, frame, b[0], next[0]); + oct_esp_encrypt_tun_add_trace ( + vm, node, frame, b[0], + OCT_ESP_ENCRYPT_TUN_NEXT_ADJ_MIDCHAIN_TX); } b += 1; n_left--; - next++; } } - vlib_buffer_enqueue_to_next (vm, node, from, nexts, frame->n_vectors); + vlib_buffer_enqueue_to_single_next (vm, node, from, + OCT_ESP_ENCRYPT_TUN_NEXT_ADJ_MIDCHAIN_TX, + frame->n_vectors); return frame->n_vectors; } @@ -359,10 +313,6 @@ VLIB_REGISTER_NODE (oct_esp4_encrypt_tun_node) = { .next_nodes = { #define _(next, node) [OCT_ESP_ENCRYPT_TUN_NEXT_##next] = node, foreach_oct_esp_encrypt_tun_next -#undef _ -#define _(core) \ - OCT_ESP_ENCRYPT_TUN_PREP_NODES (oct-esp4-encrypt-tun-prep, core) - foreach_oct_esp_handoff_core #undef _ }, @@ -394,216 +344,3 @@ VLIB_REGISTER_NODE (oct_esp6_encrypt_tun_node) = { .sibling_of = "oct-esp4-encrypt-tun", }; /* clang-format on */ - -static_always_inline i32 -oct_ipsec_rlen_get (oct_ipsec_encap_len_t *encap, uint32_t plen) -{ - uint32_t enc_payload_len; - - enc_payload_len = - round_pow2 (plen + encap->roundup_len, encap->roundup_byte); - - return encap->partial_len + enc_payload_len; -} - -static_always_inline u32 -oct_ipsec_esp_add_footer_and_icv (oct_ipsec_encap_len_t *encap, u32 rlen) -{ - /* plain_text len + pad_bytes + ESP_footer size + icv_len */ - return rlen + encap->icv_len - encap->partial_len; -} - -static_always_inline uword -oct_esp_prep_to_core (vlib_main_t *vm, vlib_node_runtime_t *node, - vlib_frame_t *frame, u16 core_id, const int is_ipv6) -{ - oct_device_t *od; - u32 *from = vlib_frame_vector_args (frame); - u32 n_left = frame->n_vectors; - vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b = bufs; - u16 buffer_data_size = vlib_buffer_get_default_data_size (vm); - u32 sa_index, current_sa_index = ~0, rlen, pkt_len; - oct_ipsec_main_t *im = &oct_ipsec_main; - oct_ipsec_outbound_pkt_meta_t *meta; - oct_ipsec_session_t *sess = NULL; - u16 sa_bytes; - u64 dptr; - - vlib_get_buffers (vm, from, b, frame->n_vectors); - - while (n_left > 0) - { - sa_index = vnet_buffer (b[0])->ipsec.sad_index; - if (sa_index != current_sa_index) - { - sess = pool_elt_at_index (im->inline_ipsec_sessions, sa_index); - if (!sess->inst.w7.s.cptr) - { - od = oct_ipsec_get_oct_device_from_outb_sa (sa_index); - sess->inst.w7.s.cptr = (u64) sess->out_sa[od->nix_idx]; - } - current_sa_index = sa_index; - ALWAYS_ASSERT (current_sa_index < - vec_len (im->inline_ipsec_sessions)); - } - - vnet_buffer (b[0])->oflags |= VNET_BUFFER_OFFLOAD_F_IPSEC_OFFLOAD; - vnet_buffer (b[0])->ipsec.sad_index = current_sa_index; - - /* - * External header buffer is used to store - * the cn10k_ipsec_outbound_pkt_meta_t - */ - - meta = - (oct_ipsec_outbound_pkt_meta_t *) OCT_EXT_HDR_FROM_VLIB_BUFFER (b[0]); - - clib_memset (meta, 0, sizeof (oct_ipsec_outbound_pkt_meta_t)); - - meta->res.cn10k.compcode = CPT_COMP_NOT_DONE; - - pkt_len = b[0]->current_length; - rlen = oct_ipsec_rlen_get (&sess->encap, pkt_len); - meta->core_id = core_id; - - if (is_ipv6) - meta->is_ip6 = 1; - - /* Populate CPT instruction template */ - meta->inst.res_addr = (u64) &meta->res; - meta->inst.w2.u64 = sess->inst.w2.u64; - /* WQE must be aligned on an 64-bit / 8 byte boundary */ - ASSERT (((uintptr_t) b[0] & 0x7ULL) == 0); - meta->inst.w3.u64 = (uintptr_t) b[0]; - /* Enable queue ordering */ - meta->inst.w3.u64 |= 0x1ULL; - - if ((b[0]->flags & VLIB_BUFFER_NEXT_PRESENT) || - (rlen > buffer_data_size)) - { - u16 total_length; - total_length = b[0]->current_length + - b[0]->total_length_not_including_first_buffer; - rlen = oct_ipsec_rlen_get (&sess->encap, total_length); - sa_bytes = oct_ipsec_esp_add_footer_and_icv (&sess->encap, rlen); - meta->sa_bytes = sa_bytes; - - meta->dlen_adj = rlen - total_length; - meta->is_sg_mode = 1; - meta->inst.w4.u64 = sess->inst.w4.u64; - } - else - { - dptr = (u64) vlib_buffer_get_current (b[0]); - sa_bytes = oct_ipsec_esp_add_footer_and_icv (&sess->encap, rlen); - meta->sa_bytes = sa_bytes; - meta->dlen_adj = rlen - pkt_len; - - /* Set w0 nixtx_offset */ - meta->inst.w0.u64 |= - (((int64_t) meta->nixtx - (int64_t) dptr) & 0xFFFFF) << 32; - - /* - * Set nixtx length to 2 dwords. - * NIXTXL + 1 represents the length in dwords - */ - meta->inst.w0.u64 |= 1; - meta->inst.w4.u64 = sess->inst.w4.u64 | pkt_len; - meta->inst.dptr = dptr; - meta->inst.rptr = dptr; - } - - meta->inst.w7.u64 = sess->inst.w7.u64; - - b++; - n_left--; - } - - if (PREDICT_FALSE (node->flags & VLIB_NODE_FLAG_TRACE)) - { - n_left = frame->n_vectors; - b = bufs; - - while (n_left) - { - if (PREDICT_FALSE (b[0]->flags & VLIB_BUFFER_IS_TRACED)) - { - oct_esp_encrypt_tun_add_trace ( - vm, node, frame, b[0], - OCT_ESP_ENCRYPT_TUN_NEXT_ADJ_MIDCHAIN_TX); - } - n_left--; - b++; - } - } - - vlib_buffer_enqueue_to_single_next (vm, node, from, - OCT_ESP_ENCRYPT_TUN_NEXT_ADJ_MIDCHAIN_TX, - frame->n_vectors); - - return frame->n_vectors; -} - -/** - * @brief OCTEON ESP4 encryption tunnel preparation node. - * @node oct-esp4-encrypt-tun-prep - * - * This is the OCTEON ESP4 encryption tunnel preparation node. - * - * @param vm vlib_main_t corresponding to the current thread - * @param node vlib_node_runtime_t - * @param from_frame vlib_frame_t - */ -/* clang-format off */ -#define OCT_DEFINE_ESP4_TUN_PREP_NODE(core) \ - VLIB_NODE_FN (oct_esp4_encrypt_tun_prep##core##_node) \ - (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame) \ - { \ - return oct_esp_prep_to_core (vm, node, frame, core, \ - 0 /* is_ipv6 */); \ - } \ - \ - VLIB_REGISTER_NODE (oct_esp4_encrypt_tun_prep##core##_node) = { \ - .name = "oct-esp4-encrypt-tun-prep"#core, \ - .type = VLIB_NODE_TYPE_INTERNAL, \ - .format_trace = format_oct_esp_encrypt_trace, \ - .vector_size = sizeof (u32), \ - .sibling_of = "oct-esp4-encrypt-tun", \ - }; - -#define _(core) OCT_DEFINE_ESP4_TUN_PREP_NODE (core) -foreach_oct_esp_handoff_core; -#undef _ -/* clang-format on */ - -/** - * @brief OCTEON ESP6 encryption tunnel preparation node. - * @node oct-esp6-encrypt-tun-prep - * - * This is the OCTEON ESP6 encryption tunnel preparation node. - * - * @param vm vlib_main_t corresponding to the current thread - * @param node vlib_node_runtime_t - * @param from_frame vlib_frame_t - */ -/* clang-format off */ -#define OCT_DEFINE_ESP6_TUN_PREP_NODE(core) \ - VLIB_NODE_FN (oct_esp6_encrypt_tun_prep##core##_node) \ - (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame) \ - { \ - return oct_esp_prep_to_core (vm, node, frame, core, \ - 1 /* is_ipv6 */); \ - } \ - \ - VLIB_REGISTER_NODE (oct_esp6_encrypt_tun_prep##core##_node) = { \ - .name = "oct-esp6-encrypt-tun-prep"#core, \ - .type = VLIB_NODE_TYPE_INTERNAL, \ - .format_trace = format_oct_esp_encrypt_trace, \ - .vector_size = sizeof (u32), \ - .sibling_of = "oct-esp6-encrypt-tun", \ - }; - -#define _(core) OCT_DEFINE_ESP6_TUN_PREP_NODE (core) -foreach_oct_esp_handoff_core; -#undef _ -/* clang-format on */ diff --git a/src/plugins/dev_octeon/ipsec.h b/src/plugins/dev_octeon/ipsec.h index fd83fb724d..65fb36e67c 100644 --- a/src/plugins/dev_octeon/ipsec.h +++ b/src/plugins/dev_octeon/ipsec.h @@ -74,18 +74,26 @@ typedef struct { CLIB_CACHE_LINE_ALIGN_MARK (c0); - struct cpt_inst_s inst; - union cpt_res_s res; - u16 dlen_adj; - u16 sa_bytes; - u8 core_id; - u8 is_ip6; - bool is_sg_mode; - CLIB_CACHE_LINE_ALIGN_MARK (c2); u64 nixtx[2]; u8 sg_buffer[128]; } oct_ipsec_outbound_pkt_meta_t; +typedef struct +{ + union cpt_res_s res; + u16 dlen_adj; + u16 sa_bytes; +} oct_ipsec_outb_data_t; + +STATIC_ASSERT (sizeof (oct_ipsec_outb_data_t) <= + STRUCT_SIZE_OF (vnet_buffer_opaque2_t, unused), + "Outbound meta-data too large for vnet_buffer_opaque2_t"); + +#define oct_ipsec_outb_data(b) \ + ((oct_ipsec_outb_data_t *) ((u8 *) (b)->opaque2 + \ + STRUCT_OFFSET_OF (vnet_buffer_opaque2_t, \ + unused))) + typedef struct { uint8_t partial_len; diff --git a/src/plugins/dev_octeon/queue.c b/src/plugins/dev_octeon/queue.c index 6eb6ed4019..0aa814ae2d 100644 --- a/src/plugins/dev_octeon/queue.c +++ b/src/plugins/dev_octeon/queue.c @@ -130,7 +130,10 @@ oct_rxq_init (vlib_main_t *vm, vnet_dev_rx_queue_t *rxq, u32 total_sz) }; if (inl_main->inl_dev) - crq->cq.nb_desc = clib_max (crq->cq.nb_desc, 4096); + { + roc_nix_inl_dev_xaq_realloc (crq->aura_handle); + crq->cq.nb_desc = clib_max (crq->cq.nb_desc, 4096); + } if ((rrv = roc_nix_cq_init (nix, &crq->cq))) { diff --git a/src/plugins/dev_octeon/roc_helper.c b/src/plugins/dev_octeon/roc_helper.c index 7e73d76595..cdd0717037 100644 --- a/src/plugins/dev_octeon/roc_helper.c +++ b/src/plugins/dev_octeon/roc_helper.c @@ -79,7 +79,7 @@ oct_drv_physmem_alloc (vlib_main_t *vm, u32 size, u32 align) if (align) { /* Force ROC align alloc in case alignment is less than ROC align */ - align = align < ROC_ALIGN ? ROC_ALIGN : align; + align = ((align + ROC_ALIGN - 1) & ~(ROC_ALIGN - 1)); mem = vlib_physmem_alloc_aligned_on_numa (vm, size, align, 0); } else @@ -120,6 +120,9 @@ oct_plt_zmalloc (u32 size, u32 align) static void * oct_plt_realloc (void *addr, u32 size, u32 align) { + align = CLIB_CACHE_LINE_ROUND (align); + size = CLIB_CACHE_LINE_ROUND (size); + if (align) return clib_mem_realloc_aligned (addr, size, align); else diff --git a/src/plugins/dev_octeon/tx_node.c b/src/plugins/dev_octeon/tx_node.c index a488609576..64b1270999 100644 --- a/src/plugins/dev_octeon/tx_node.c +++ b/src/plugins/dev_octeon/tx_node.c @@ -718,37 +718,57 @@ oct_get_tx_vlib_buf_segs (vlib_main_t *vm, vlib_buffer_t *b) return n_segs; } +static_always_inline i32 +oct_ipsec_rlen_get (oct_ipsec_encap_len_t *encap, uint32_t plen) +{ + uint32_t enc_payload_len; + + enc_payload_len = + round_pow2 (plen + encap->roundup_len, encap->roundup_byte); + + return encap->partial_len + enc_payload_len; +} + +static_always_inline u32 +oct_ipsec_esp_add_footer_and_icv (oct_ipsec_encap_len_t *encap, u32 rlen) +{ + /* plain_text len + pad_bytes + ESP_footer size + icv_len */ + return rlen + encap->icv_len - encap->partial_len; +} + void static_always_inline oct_prepare_ipsec_inst (vlib_main_t *vm, vlib_buffer_t *b, u64 sq_handle, u64 aura_handle, oct_ipsec_outbound_pkt_meta_t **pkt_meta, - u64 *n_dwords) + struct cpt_inst_s *inst, u64 *n_dwords, + oct_ipsec_session_t *sess) { - u8 l2_len = sizeof (ethernet_header_t); + u16 buffer_data_size = vlib_buffer_get_default_data_size (vm); struct nix_send_hdr_s *send_hdr; union nix_send_sg_s *sg; u64 n_segs; - struct cpt_inst_s *inst; u16 total_length; - - pkt_meta[0] = - (oct_ipsec_outbound_pkt_meta_t *) OCT_EXT_HDR_FROM_VLIB_BUFFER (b); - - inst = &pkt_meta[0]->inst; + u16 l3_hdr_offset = vnet_buffer (b)->l3_hdr_offset; + u32 dlen = b->current_length + b->total_length_not_including_first_buffer - + l3_hdr_offset; + u32 rlen = oct_ipsec_rlen_get (&sess->encap, dlen); + u16 dlen_adj = rlen - dlen; + u32 sa_bytes = oct_ipsec_esp_add_footer_and_icv (&sess->encap, rlen); send_hdr = (struct nix_send_hdr_s *) pkt_meta[0]->nixtx; sg = (union nix_send_sg_s *) (pkt_meta[0]->nixtx + 2); - if (pkt_meta[0]->is_sg_mode) + if (b->flags & VLIB_BUFFER_NEXT_PRESENT || rlen > buffer_data_size) { + ASSERT (0); total_length = b->current_length + b->total_length_not_including_first_buffer; + inst->w4.u64 = sess->inst.w4.u64; n_segs = oct_ipsec_outb_prepare_sg2_list ( - vm, b, inst, pkt_meta[0]->dlen_adj, total_length, - (void *) pkt_meta[0]->sg_buffer); + vm, b, inst, dlen_adj, total_length, (void *) pkt_meta[0]->sg_buffer); - inst->w0.u64 = (uint64_t) l2_len << 16; + inst->w0.u64 = (uint64_t) l3_hdr_offset << 16; inst->w0.u64 |= NIX_NB_SEGS_TO_SEGDW (n_segs); inst->w0.u64 |= (((int64_t) pkt_meta[0]->nixtx - (int64_t) inst->dptr) & 0xFFFFF) @@ -759,7 +779,16 @@ oct_prepare_ipsec_inst (vlib_main_t *vm, vlib_buffer_t *b, u64 sq_handle, } else { - b->current_length += pkt_meta[0]->dlen_adj; + inst->dptr = (u64) ((u8 *) vlib_buffer_get_current (b) + l3_hdr_offset); + inst->rptr = inst->dptr; + /* Set w0 nixtx_offset */ + inst->w0.u64 |= + (((int64_t) pkt_meta[0]->nixtx - (int64_t) inst->dptr) & 0xFFFFF) + << 32; + inst->w0.u64 |= 1; + inst->w4.u64 = sess->inst.w4.u64 | dlen; + + b->current_length += dlen_adj; n_segs = oct_get_tx_vlib_buf_segs (vm, b); n_dwords[0] = oct_add_sg_list (sg, b, n_segs); } @@ -767,18 +796,19 @@ oct_prepare_ipsec_inst (vlib_main_t *vm, vlib_buffer_t *b, u64 sq_handle, oct_add_send_hdr (send_hdr, b, aura_handle, sq_handle, n_dwords[0]); vlib_increment_combined_counter ( &ipsec_sa_counters, vlib_get_thread_index (), - vnet_buffer (b)->ipsec.sad_index, 1, pkt_meta[0]->sa_bytes); + vnet_buffer (b)->ipsec.sad_index, 1, sa_bytes); } void static_always_inline oct_submit_quad_packets (u64 lmt_arg, oct_device_t *cd, - oct_ipsec_outbound_pkt_meta_t **pkt_meta, + struct cpt_inst_s *inst0, struct cpt_inst_s *inst1, + struct cpt_inst_s *inst2, struct cpt_inst_s *inst3, u64 *n_dwords, u64 **lmt_line) { - roc_lmt_mov_seg ((void *) lmt_line[0], &pkt_meta[0]->inst, 4); - roc_lmt_mov_seg ((void *) lmt_line[1], &pkt_meta[1]->inst, 4); - roc_lmt_mov_seg ((void *) lmt_line[2], &pkt_meta[2]->inst, 4); - roc_lmt_mov_seg ((void *) lmt_line[3], &pkt_meta[3]->inst, 4); + roc_lmt_mov_seg ((void *) lmt_line[0], inst0, 4); + roc_lmt_mov_seg ((void *) lmt_line[1], inst1, 4); + roc_lmt_mov_seg ((void *) lmt_line[2], inst2, 4); + roc_lmt_mov_seg ((void *) lmt_line[3], inst3, 4); /* Count minus one of LMTSTs in the burst */ lmt_arg |= 3 << 12; @@ -1219,15 +1249,21 @@ oct_pkts_send_ipsec (vlib_main_t *vm, vlib_node_runtime_t *node, oct_tx_ctx_t *ctx, vnet_dev_tx_queue_t *txq, u16 tx_pkts, vlib_buffer_t **bufs) { + oct_ipsec_main_t *im = &oct_ipsec_main; oct_txq_t *ctq = vnet_dev_get_tx_queue_data (txq); vnet_dev_t *dev = txq->port->dev; oct_device_t *cd = vnet_dev_get_data (dev); u32 current_sq0, current_sq1, current_sq2, current_sq3; u64 sq_handle0, sq_handle1, sq_handle2, sq_handle3; u64 aura_handle0, aura_handle1, aura_handle2, aura_handle3; + u32 sa0_index, sa1_index, sa2_index, sa3_index; + u32 current_sa0_index = ~0, current_sa1_index = ~0; + u32 current_sa2_index = ~0, current_sa3_index = ~0; + oct_ipsec_session_t *sess0 = NULL, *sess1 = NULL; + oct_ipsec_session_t *sess2 = NULL, *sess3 = NULL; + struct cpt_inst_s inst0 = { 0 }, inst1 = { 0 }, inst2 = { 0 }, inst3 = { 0 }; u64 core_lmt_base_addr, lmt_arg, core_lmt_id; oct_ipsec_outbound_pkt_meta_t *pkt_meta[4]; - oct_ipsec_outbound_pkt_meta_t *meta_hdr; u16 n_cpt_fc_drop = 0, n_nix_fc_drop = 0; u16 n_left0, n_left1, n_left2, n_left3; u16 n_packets; @@ -1240,6 +1276,7 @@ oct_pkts_send_ipsec (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_buffer_t **b; u64 *lmt_line[4]; u64 n_dwords[4]; + oct_device_t *od; b = bufs; @@ -1284,18 +1321,105 @@ oct_pkts_send_ipsec (vlib_main_t *vm, vlib_node_runtime_t *node, while (n_packets > 3) { - meta_hdr = + pkt_meta[0] = (oct_ipsec_outbound_pkt_meta_t *) OCT_EXT_HDR_FROM_VLIB_BUFFER (b[0]); - sq0 = meta_hdr->core_id; - meta_hdr = + pkt_meta[1] = (oct_ipsec_outbound_pkt_meta_t *) OCT_EXT_HDR_FROM_VLIB_BUFFER (b[1]); - sq1 = meta_hdr->core_id; - meta_hdr = + pkt_meta[2] = (oct_ipsec_outbound_pkt_meta_t *) OCT_EXT_HDR_FROM_VLIB_BUFFER (b[2]); - sq2 = meta_hdr->core_id; - meta_hdr = + pkt_meta[3] = (oct_ipsec_outbound_pkt_meta_t *) OCT_EXT_HDR_FROM_VLIB_BUFFER (b[3]); - sq3 = meta_hdr->core_id; + + sa0_index = vnet_buffer (b[0])->ipsec.sad_index; + if (sa0_index != current_sa0_index) + { + sess0 = pool_elt_at_index (im->inline_ipsec_sessions, sa0_index); + if (!sess0->inst.w7.s.cptr) + { + od = oct_ipsec_get_oct_device_from_outb_sa (sa0_index); + sess0->inst.w7.s.cptr = (u64) sess0->out_sa[od->nix_idx]; + } + current_sa0_index = sa0_index; + ALWAYS_ASSERT (current_sa0_index < + vec_len (im->inline_ipsec_sessions)); + } + + sa1_index = vnet_buffer (b[1])->ipsec.sad_index; + if (sa1_index != current_sa1_index) + { + sess1 = pool_elt_at_index (im->inline_ipsec_sessions, sa1_index); + if (!sess1->inst.w7.s.cptr) + { + od = oct_ipsec_get_oct_device_from_outb_sa (sa1_index); + sess1->inst.w7.s.cptr = (u64) sess1->out_sa[od->nix_idx]; + } + current_sa1_index = sa1_index; + ALWAYS_ASSERT (current_sa0_index < + vec_len (im->inline_ipsec_sessions)); + } + + sa2_index = vnet_buffer (b[2])->ipsec.sad_index; + if (sa2_index != current_sa2_index) + { + sess2 = pool_elt_at_index (im->inline_ipsec_sessions, sa2_index); + if (!sess2->inst.w7.s.cptr) + { + od = oct_ipsec_get_oct_device_from_outb_sa (sa2_index); + sess2->inst.w7.s.cptr = (u64) sess2->out_sa[od->nix_idx]; + } + current_sa2_index = sa2_index; + ALWAYS_ASSERT (current_sa2_index < + vec_len (im->inline_ipsec_sessions)); + } + + sa3_index = vnet_buffer (b[3])->ipsec.sad_index; + if (sa3_index != current_sa3_index) + { + sess3 = pool_elt_at_index (im->inline_ipsec_sessions, sa3_index); + if (!sess3->inst.w7.s.cptr) + { + od = oct_ipsec_get_oct_device_from_outb_sa (sa3_index); + sess3->inst.w7.s.cptr = (u64) sess3->out_sa[od->nix_idx]; + } + current_sa3_index = sa3_index; + ALWAYS_ASSERT (current_sa3_index < + vec_len (im->inline_ipsec_sessions)); + } + + oct_ipsec_outb_data (b[0])->res.cn10k.compcode = CPT_COMP_NOT_DONE; + oct_ipsec_outb_data (b[1])->res.cn10k.compcode = CPT_COMP_NOT_DONE; + oct_ipsec_outb_data (b[2])->res.cn10k.compcode = CPT_COMP_NOT_DONE; + oct_ipsec_outb_data (b[3])->res.cn10k.compcode = CPT_COMP_NOT_DONE; + + inst0.res_addr = (u64) &oct_ipsec_outb_data (b[0])->res; + inst1.res_addr = (u64) &oct_ipsec_outb_data (b[1])->res; + inst2.res_addr = (u64) &oct_ipsec_outb_data (b[2])->res; + inst3.res_addr = (u64) &oct_ipsec_outb_data (b[3])->res; + + inst0.w2.u64 = sess0->inst.w2.u64; + inst1.w2.u64 = sess1->inst.w2.u64; + inst2.w2.u64 = sess2->inst.w2.u64; + inst3.w2.u64 = sess3->inst.w2.u64; + + inst0.w3.u64 = (uintptr_t) (b[0]); + inst1.w3.u64 = (uintptr_t) (b[1]); + inst2.w3.u64 = (uintptr_t) (b[2]); + inst3.w3.u64 = (uintptr_t) (b[3]); + + inst0.w3.u64 |= 0x1ULL; + inst1.w3.u64 |= 0x1ULL; + inst2.w3.u64 |= 0x1ULL; + inst3.w3.u64 |= 0x1ULL; + + inst0.w7.u64 = sess0->inst.w7.u64; + inst1.w7.u64 = sess1->inst.w7.u64; + inst2.w7.u64 = sess2->inst.w7.u64; + inst3.w7.u64 = sess3->inst.w7.u64; + + sq0 = sa0_index; + sq1 = sa1_index; + sq2 = sa2_index; + sq3 = sa3_index; quad_bit = 0; count = 0; @@ -1345,15 +1469,16 @@ oct_pkts_send_ipsec (vlib_main_t *vm, vlib_node_runtime_t *node, if (quad_bit == 0x0F) { oct_prepare_ipsec_inst (vm, b[0], sq_handle0, aura_handle0, - &pkt_meta[0], &n_dwords[0]); + &pkt_meta[0], &inst0, &n_dwords[0], sess0); oct_prepare_ipsec_inst (vm, b[1], sq_handle1, aura_handle1, - &pkt_meta[1], &n_dwords[1]); + &pkt_meta[1], &inst1, &n_dwords[1], sess1); oct_prepare_ipsec_inst (vm, b[2], sq_handle2, aura_handle2, - &pkt_meta[2], &n_dwords[2]); + &pkt_meta[2], &inst2, &n_dwords[2], sess2); oct_prepare_ipsec_inst (vm, b[3], sq_handle3, aura_handle3, - &pkt_meta[3], &n_dwords[3]); + &pkt_meta[3], &inst3, &n_dwords[3], sess3); - oct_submit_quad_packets (lmt_arg, cd, pkt_meta, n_dwords, lmt_line); + oct_submit_quad_packets (lmt_arg, cd, &inst0, &inst1, &inst2, &inst3, + n_dwords, lmt_line); n_left0 -= 1; n_left1 -= 1; @@ -1366,9 +1491,9 @@ oct_pkts_send_ipsec (vlib_main_t *vm, vlib_node_runtime_t *node, if (n_left0) { oct_prepare_ipsec_inst (vm, b[0], sq_handle0, aura_handle0, - &pkt_meta[0], &n_dwords[0]), - roc_lmt_mov_seg ((void *) lmt_line[count], &pkt_meta[0]->inst, - 4); + &pkt_meta[0], &inst0, &n_dwords[0], + sess0), + roc_lmt_mov_seg ((void *) lmt_line[count], &inst0, 4); count++; n_left0 -= 1; } @@ -1380,9 +1505,9 @@ oct_pkts_send_ipsec (vlib_main_t *vm, vlib_node_runtime_t *node, if (n_left1) { oct_prepare_ipsec_inst (vm, b[1], sq_handle1, aura_handle1, - &pkt_meta[1], &n_dwords[1]); - roc_lmt_mov_seg ((void *) lmt_line[count], &pkt_meta[1]->inst, - 4); + &pkt_meta[1], &inst1, &n_dwords[1], + sess1); + roc_lmt_mov_seg ((void *) lmt_line[count], &inst1, 4); if (count) lmt_arg |= (n_dwords[1] - 1) << (19 + (3 * (count - 1))); count++; @@ -1396,9 +1521,9 @@ oct_pkts_send_ipsec (vlib_main_t *vm, vlib_node_runtime_t *node, if (n_left2) { oct_prepare_ipsec_inst (vm, b[2], sq_handle2, aura_handle2, - &pkt_meta[2], &n_dwords[2]); - roc_lmt_mov_seg ((void *) lmt_line[count], &pkt_meta[2]->inst, - 4); + &pkt_meta[2], &inst2, &n_dwords[2], + sess2); + roc_lmt_mov_seg ((void *) lmt_line[count], &inst2, 4); if (count) lmt_arg |= (n_dwords[2] - 1) << (19 + (3 * (count - 1))); count++; @@ -1412,9 +1537,9 @@ oct_pkts_send_ipsec (vlib_main_t *vm, vlib_node_runtime_t *node, if (n_left3) { oct_prepare_ipsec_inst (vm, b[3], sq_handle3, aura_handle3, - &pkt_meta[3], &n_dwords[3]); - roc_lmt_mov_seg ((void *) lmt_line[count], &pkt_meta[3]->inst, - 4); + &pkt_meta[3], &inst3, &n_dwords[3], + sess3); + roc_lmt_mov_seg ((void *) lmt_line[count], &inst3, 4); if (count) lmt_arg |= (n_dwords[3] - 1) << (19 + (3 * (count - 1))); count++; @@ -1443,9 +1568,30 @@ oct_pkts_send_ipsec (vlib_main_t *vm, vlib_node_runtime_t *node, while (n_packets) { - meta_hdr = + pkt_meta[0] = (oct_ipsec_outbound_pkt_meta_t *) OCT_EXT_HDR_FROM_VLIB_BUFFER (b[0]); - sq0 = meta_hdr->core_id; + sa0_index = vnet_buffer (b[0])->ipsec.sad_index; + if (sa0_index != current_sa0_index) + { + sess0 = pool_elt_at_index (im->inline_ipsec_sessions, sa0_index); + if (!sess0->inst.w7.s.cptr) + { + od = oct_ipsec_get_oct_device_from_outb_sa (sa0_index); + sess0->inst.w7.s.cptr = (u64) sess0->out_sa[od->nix_idx]; + } + current_sa0_index = sa0_index; + ALWAYS_ASSERT (current_sa0_index < + vec_len (im->inline_ipsec_sessions)); + } + + oct_ipsec_outb_data (b[0])->res.cn10k.compcode = CPT_COMP_NOT_DONE; + inst0.res_addr = (u64) &oct_ipsec_outb_data (b[0])->res; + inst0.w2.u64 = sess0->inst.w2.u64; + inst0.w3.u64 = (uintptr_t) (b[0]); + inst0.w3.u64 |= 0x1ULL; + inst0.w7.u64 = sess0->inst.w7.u64; + + sq0 = sa0_index; if (current_sq0 != sq0) { @@ -1462,10 +1608,11 @@ oct_pkts_send_ipsec (vlib_main_t *vm, vlib_node_runtime_t *node, n_nix_fc_drop++; goto next; } + oct_prepare_ipsec_inst (vm, b[0], sq_handle0, aura_handle0, &pkt_meta[0], - &n_dwords[0]); + &inst0, &n_dwords[0], sess0); - roc_lmt_mov_seg ((void *) lmt_line[0], &pkt_meta[0]->inst, 4); + roc_lmt_mov_seg ((void *) lmt_line[0], &inst0, 4); lmt_arg = ROC_CN10K_CPT_LMT_ARG | core_lmt_id; From 0b4d6919fb39c266322d72d862fb978f52e7348b Mon Sep 17 00:00:00 2001 From: Sunil Kumar Kori Date: Wed, 15 Jan 2025 16:59:02 +0530 Subject: [PATCH 177/271] octeon: add support for OCTEON20 platform Enabling VPP on OCTEON20 which contains following changes: - Adding OCTEON20 specific changes in cmake to compile. - Mapping PCI BAR2 only for OCTEON20. Type: feature JIRA: https://essjira.marvell.com/browse/IPBUSW-58201 Signed-off-by: Sunil Kumar Kori Change-Id: If5ac2393005c73d2b29cbdee897c57c17c854bb3 Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/143258 Reviewed-by: Monendra Singh Kushwaha Reviewed-by: Nithin Kumar Dabilpuram Tested-by: sa_ip-sw-jenkins (cherry picked from commit 024ec9aad8a8dc5cf52ba2a3b297b52e2dc83673) Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/144622 --- src/cmake/platform/octeon20.cmake | 5 +++++ src/plugins/dev_octeon/CMakeLists.txt | 2 +- src/plugins/dev_octeon/init.c | 11 +++++++---- 3 files changed, 13 insertions(+), 5 deletions(-) create mode 100644 src/cmake/platform/octeon20.cmake diff --git a/src/cmake/platform/octeon20.cmake b/src/cmake/platform/octeon20.cmake new file mode 100644 index 0000000000..e71b1148e4 --- /dev/null +++ b/src/cmake/platform/octeon20.cmake @@ -0,0 +1,5 @@ + +set(VPP_PLATFORM_CACHE_LINE_SIZE 64) +set(VPP_PLATFORM_MARCH_FLAGS -march=armv8.7-a+sve2) +set(VPP_PLATFORM_BUFFER_ALIGN 128) +set(VPP_PLATFORM_N_PREFETCHES 6) diff --git a/src/plugins/dev_octeon/CMakeLists.txt b/src/plugins/dev_octeon/CMakeLists.txt index 93dec8de6d..f4df8279d9 100644 --- a/src/plugins/dev_octeon/CMakeLists.txt +++ b/src/plugins/dev_octeon/CMakeLists.txt @@ -1,7 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # Copyright(c) 2022 Cisco Systems, Inc. -if (NOT VPP_PLATFORM_NAME STREQUAL "octeon10" AND NOT VPP_PLATFORM_NAME STREQUAL "octeon9") +if (NOT VPP_PLATFORM_NAME STREQUAL "octeon20" AND NOT VPP_PLATFORM_NAME STREQUAL "octeon10" AND NOT VPP_PLATFORM_NAME STREQUAL "octeon9") return() endif() diff --git a/src/plugins/dev_octeon/init.c b/src/plugins/dev_octeon/init.c index 2f4c3add27..f18e27af59 100644 --- a/src/plugins/dev_octeon/init.c +++ b/src/plugins/dev_octeon/init.c @@ -489,10 +489,13 @@ oct_init (vlib_main_t *vm, vnet_dev_t *dev) foreach_int (i, 2, 4) { - rv = vnet_dev_pci_map_region (vm, dev, i, - &cd->plt_pci_dev.mem_resource[i].addr); - if (rv != VNET_DEV_OK) - return rv; + if (i == 2 || !roc_model_is_cn20k ()) + { + rv = vnet_dev_pci_map_region (vm, dev, i, + &cd->plt_pci_dev.mem_resource[i].addr); + if (rv != VNET_DEV_OK) + return rv; + } } if ((rv = vnet_dev_pci_bus_master_enable (vm, dev))) From 0043b5910fa4e7fe979879f5113289e752c7a4fb Mon Sep 17 00:00:00 2001 From: Harish Kumar Malik Date: Thu, 19 Dec 2024 14:13:37 +0500 Subject: [PATCH 178/271] tm: add support for fetching tm capabilities This patch adds support to fetch TM node capabilities and TM level capabilities in the tm infra. Type: feature Signed-off-by: Harish Kumar Malik Change-Id: I907be342299f803964a447f8e4623c20db4d3146 Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/142116 Tested-by: sa_ip-sw-jenkins Reviewed-by: Nithin Kumar Dabilpuram (cherry picked from commit 26336ecf7776c7cac3a588b1c24e307b60cea393) Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/144623 Reviewed-by: Monendra Singh Kushwaha --- src/vnet/tm/tm.api | 62 +++++++ src/vnet/tm/tm.c | 30 ++++ src/vnet/tm/tm.h | 379 ++++++++++++++++++++++++++++++++++++++++++ src/vnet/tm/tm_api.c | 34 ++++ src/vnet/tm/tm_test.c | 92 ++++++++++ 5 files changed, 597 insertions(+) diff --git a/src/vnet/tm/tm.api b/src/vnet/tm/tm.api index dbeb0405f2..16bfb8aa68 100644 --- a/src/vnet/tm/tm.api +++ b/src/vnet/tm/tm.api @@ -296,6 +296,68 @@ define tm_sys_node_sched_weight_update u32 weight; }; +/** + * @brief Reply for getting the capabilities of a TM system. + * + * This structure specifies the parameters returned in response to getting the capabilities of a TM system. + * + * @param context - Sender context, to match reply with request. + * @param retval - Return value indicating success or failure of the operation. + */ +define tm_sys_get_capabilities_reply +{ + u32 context; + i32 retval; +}; + +/** + * @brief Get the capabilities of a TM system. + * + * This structure outlines the necessary parameters to get the capabilities of a TM system. + * + * @param client_index - Opaque cookie to identify the sender. + * @param context - Sender context, to match reply with request. + * @param sw_if_idx - Software interface index. + */ +define tm_sys_get_capabilities + { + u32 client_index; + u32 context; + u32 sw_if_idx; + }; + +/** + * @brief Reply for getting the capabilities of a specific level in a TM system. + * + * This structure specifies the parameters returned in response to getting the capabilities of a specific level in a TM system. + * + * @param context - Sender context, to match reply with request. + * @param retval - Return value indicating success or failure of the operation. + */ +define tm_sys_level_get_capabilities_reply +{ + u32 context; + i32 retval; +}; + +/** + * @brief Get the capabilities of a specific level in a TM system. + * + * This structure outlines the necessary parameters to get the capabilities of a specific level in a TM system. + * + * @param client_index - Opaque cookie to identify the sender. + * @param context - Sender context, to match reply with request. + * @param sw_if_idx - Software interface index. + * @param level - Level in the TM system whose capabilities are to be read. + */ +define tm_sys_level_get_capabilities + { + u32 client_index; + u32 context; + u32 sw_if_idx; + u32 level; + }; + /** * @brief Reply for reading the statistics of a TM node. * diff --git a/src/vnet/tm/tm.c b/src/vnet/tm/tm.c index 64cf5cf754..7f020a6f8e 100644 --- a/src/vnet/tm/tm.c +++ b/src/vnet/tm/tm.c @@ -150,6 +150,36 @@ tm_sys_node_read_stats (u32 hw_if_idx, u32 node_idx, tm_stats_params_t *param) return 0; } +int +tm_sys_get_capabilities (u32 hw_if_idx, tm_capa_params_t *param) +{ + vnet_main_t *vnm = vnet_get_main (); + + vnet_hw_interface_t *hi = vnet_get_hw_interface (vnm, hw_if_idx); + + vnet_device_class_t *dev_class = + vnet_get_device_class (vnm, hi->dev_class_index); + + dev_class->tm_sys_impl->tm_get_capabilities (hw_if_idx, param); + + return 0; +} + +int +tm_sys_level_get_capabilities (u32 hw_if_idx, tm_level_capa_params_t *param, + u32 lvl) +{ + vnet_main_t *vnm = vnet_get_main (); + + vnet_hw_interface_t *hi = vnet_get_hw_interface (vnm, hw_if_idx); + + vnet_device_class_t *dev_class = + vnet_get_device_class (vnm, hi->dev_class_index); + + dev_class->tm_sys_impl->tm_level_get_capabilities (hw_if_idx, param, lvl); + + return 0; +} int tm_sys_start_tm (u32 hw_if_idx) { diff --git a/src/vnet/tm/tm.h b/src/vnet/tm/tm.h index f57e8cdc8e..b284032d9b 100644 --- a/src/vnet/tm/tm.h +++ b/src/vnet/tm/tm.h @@ -118,6 +118,372 @@ typedef enum tm_node_stats_type_t TM_NODE_STATS_MAX, } tm_node_stats_type_t; +/** + * Node Capabilities Params + */ +typedef struct tm_capa_params_ +{ + /** Maximum number of nodes. */ + uint32_t n_nodes_max; + + /** Maximum number of levels (i.e. number of nodes connecting the root + * node with any leaf node, including the root and the leaf). + */ + uint32_t n_levels_max; + + /** When non-zero, this flag indicates that all the non-leaf nodes + * (with the exception of the root node) have identical capability set. + */ + int non_leaf_nodes_identical; + + /** When non-zero, this flag indicates that all the leaf nodes have + * identical capability set. + */ + int leaf_nodes_identical; + + /** Maximum number of shapers, either private or shared. In case the + * implementation does not share any resources between private and + * shared shapers, it is typically equal to the sum of + * *shaper_private_n_max* and *shaper_shared_n_max*. The + * value of zero indicates that traffic shaping is not supported. + */ + uint32_t shaper_n_max; + /** Maximum number of private shapers. Indicates the maximum number of + * nodes that can concurrently have their private shaper enabled. The + * value of zero indicates that private shapers are not supported. + */ + uint32_t shaper_private_n_max; + + /** Maximum number of private shapers that support dual rate shaping. + * Indicates the maximum number of nodes that can concurrently have + * their private shaper enabled with dual rate support. Only valid when + * private shapers are supported. The value of zero indicates that dual + * rate shaping is not available for private shapers. The maximum value + * is *shaper_private_n_max*. + */ + int shaper_private_dual_rate_n_max; + + /** Minimum committed/peak rate (bytes per second) for any private + * shaper. Valid only when private shapers are supported. + */ + uint64_t shaper_private_rate_min; + /** Maximum committed/peak rate (bytes per second) for any private + * shaper. Valid only when private shapers are supported. + */ + uint64_t shaper_private_rate_max; + + /** Shaper private packet mode supported. When non-zero, this parameter + * indicates that there is at least one node that can be configured + * with packet mode in its private shaper. When shaper is configured + * in packet mode, committed/peak rate provided is interpreted + * in packets per second. + */ + int shaper_private_packet_mode_supported; + + /** Shaper private byte mode supported. When non-zero, this parameter + * indicates that there is at least one node that can be configured + * with byte mode in its private shaper. When shaper is configured + * in byte mode, committed/peak rate provided is interpreted in + * bytes per second. + */ + int shaper_private_byte_mode_supported; + /** Minimum value allowed for packet length adjustment for any private + * or shared shaper. + */ + int shaper_pkt_length_adjust_min; + + /** Maximum value allowed for packet length adjustment for any private + * or shared shaper. + */ + int shaper_pkt_length_adjust_max; + + /** Maximum number of children nodes. This parameter indicates that + * there is at least one non-leaf node that can be configured with this + * many children nodes, which might not be true for all the non-leaf + * nodes. + */ + uint32_t sched_n_children_max; + + /** Maximum number of supported priority levels. This parameter + * indicates that there is at least one non-leaf node that can be + * configured with this many priority levels for managing its children + * nodes, which might not be true for all the non-leaf nodes. The value + * of zero is invalid. The value of 1 indicates that only priority 0 is + * supported, which essentially means that Strict Priority (SP) + * algorithm is not supported. + */ + uint32_t sched_sp_n_priorities_max; + /** Maximum number of sibling nodes that can have the same priority at + * any given time, i.e. maximum size of the WFQ sibling node group. This + * parameter indicates there is at least one non-leaf node that meets + * this condition, which might not be true for all the non-leaf nodes. + * The value of zero is invalid. The value of 1 indicates that WFQ + * algorithm is not supported. The maximum value is + * *sched_n_children_max*. + */ + uint32_t sched_wfq_n_children_per_group_max; + + /** Maximum number of priority levels that can have more than one child + * node at any given time, i.e. maximum number of WFQ sibling node + * groups that have two or more members. This parameter indicates there + * is at least one non-leaf node that meets this condition, which might + * not be true for all the non-leaf nodes. The value of zero states that + * WFQ algorithm is not supported. The value of 1 indicates that + * (*sched_sp_n_priorities_max* - 1) priority levels have at most one + * child node, so there can be only one priority level with two or + * more sibling nodes making up a WFQ group. The maximum value is: + * min(floor(*sched_n_children_max* / 2), *sched_sp_n_priorities_max*). + */ + uint32_t sched_wfq_n_groups_max; + + /** Maximum WFQ weight. The value of 1 indicates that all sibling nodes + * with same priority have the same WFQ weight, so WFQ is reduced to FQ. + */ + uint32_t sched_wfq_weight_max; + + /** WFQ packet mode supported. When non-zero, this parameter indicates + * that there is at least one non-leaf node that supports packet mode + * for WFQ among its children. WFQ weights will be applied against + * packet count for scheduling children when a non-leaf node + * is configured appropriately. + */ + int sched_wfq_packet_mode_supported; + + /** WFQ byte mode supported. When non-zero, this parameter indicates + * that there is at least one non-leaf node that supports byte mode + * for WFQ among its children. WFQ weights will be applied against + * bytes for scheduling children when a non-leaf node is configured + * appropriately. + */ + int sched_wfq_byte_mode_supported; + +} tm_capa_params_t; + +/** + * Traffic manager level capabilities + */ +typedef struct tm_level_capa_params_ +{ + /** Maximum number of nodes for the current hierarchy level. */ + uint32_t n_nodes_max; + + /** Maximum number of non-leaf nodes for the current hierarchy level. + * The value of 0 indicates that current level only supports leaf + * nodes. The maximum value is *n_nodes_max*. + */ + uint32_t n_nodes_nonleaf_max; + + /** Maximum number of leaf nodes for the current hierarchy level. The + * value of 0 indicates that current level only supports non-leaf + * nodes. The maximum value is *n_nodes_max*. + */ + uint32_t n_nodes_leaf_max; + + /** When non-zero, this flag indicates that all the non-leaf nodes on + * this level have identical capability set. Valid only when + * *n_nodes_nonleaf_max* is non-zero. + */ + int non_leaf_nodes_identical; + + /** When non-zero, this flag indicates that all the leaf nodes on this + * level have identical capability set. Valid only when + * *n_nodes_leaf_max* is non-zero. + */ + int leaf_nodes_identical; + union + { + /** Items valid only for the non-leaf nodes on this level. */ + struct + { + /** Private shaper support. When non-zero, it indicates + * there is at least one non-leaf node on this level + * with private shaper support, which may not be the + * case for all the non-leaf nodes on this level. + */ + int shaper_private_supported; + + /** Dual rate support for private shaper. Valid only + * when private shaper is supported for the non-leaf + * nodes on the current level. When non-zero, it + * indicates there is at least one non-leaf node on this + * level with dual rate private shaper support, which + * may not be the case for all the non-leaf nodes on + * this level. + */ + int shaper_private_dual_rate_supported; + + /** Minimum committed/peak rate (bytes per second) for + * private shapers of the non-leaf nodes of this level. + * Valid only when private shaper is supported on this + * level. + */ + uint64_t shaper_private_rate_min; + + /** Maximum committed/peak rate (bytes per second) for + * private shapers of the non-leaf nodes on this level. + * Valid only when private shaper is supported on this + * level. + */ + uint64_t shaper_private_rate_max; + + /** Shaper private packet mode supported. When non-zero, + * this parameter indicates there is at least one + * non-leaf node at this level that can be configured + * with packet mode in its private shaper. When private + * shaper is configured in packet mode, committed/peak + * rate provided is interpreted in packets per second. + */ + int shaper_private_packet_mode_supported; + + /** Shaper private byte mode supported. When non-zero, + * this parameter indicates there is at least one + * non-leaf node at this level that can be configured + * with byte mode in its private shaper. When private + * shaper is configured in byte mode, committed/peak + * rate provided is interpreted in bytes per second. + */ + int shaper_private_byte_mode_supported; + + /** Maximum number of children nodes. This parameter + * indicates that there is at least one non-leaf node on + * this level that can be configured with this many + * children nodes, which might not be true for all the + * non-leaf nodes on this level. + */ + uint32_t sched_n_children_max; + /** Maximum number of supported priority levels. This + * parameter indicates that there is at least one + * non-leaf node on this level that can be configured + * with this many priority levels for managing its + * children nodes, which might not be true for all the + * non-leaf nodes on this level. The value of zero is + * invalid. The value of 1 indicates that only priority + * 0 is supported, which essentially means that Strict + * Priority (SP) algorithm is not supported on this + * level. + */ + uint32_t sched_sp_n_priorities_max; + + /** Maximum number of sibling nodes that can have the + * same priority at any given time, i.e. maximum size of + * the WFQ sibling node group. This parameter indicates + * there is at least one non-leaf node on this level + * that meets this condition, which may not be true for + * all the non-leaf nodes on this level. The value of + * zero is invalid. The value of 1 indicates that WFQ + * algorithm is not supported on this level. The maximum + * value is *sched_n_children_max*. + */ + uint32_t sched_wfq_n_children_per_group_max; + + /** Maximum number of priority levels that can have + * more than one child node at any given time, i.e. + * maximum number of WFQ sibling node groups that + * have two or more members. This parameter indicates + * there is at least one non-leaf node on this level + * that meets this condition, which might not be true + * for all the non-leaf nodes. The value of zero states + * that WFQ algorithm is not supported on this level. + * The value of 1 indicates that + * (*sched_sp_n_priorities_max* - 1) priority levels on + * this level have at most one child node, so there can + * be only one priority level with two or more sibling + * nodes making up a WFQ group on this level. The + * maximum value is: + * min(floor(*sched_n_children_max* / 2), + * *sched_sp_n_priorities_max*). + */ + uint32_t sched_wfq_n_groups_max; + /** Maximum WFQ weight. The value of 1 indicates that + * all sibling nodes on this level with same priority + * have the same WFQ weight, so on this level WFQ is + * reduced to FQ. + */ + uint32_t sched_wfq_weight_max; + + /** WFQ packet mode supported. When non-zero, this + * parameter indicates that there is at least one + * non-leaf node at this level that supports packet + * mode for WFQ among its children. WFQ weights will + * be applied against packet count for scheduling + * children when a non-leaf node is configured + * appropriately. + */ + int sched_wfq_packet_mode_supported; + + /** WFQ byte mode supported. When non-zero, this + * parameter indicates that there is at least one + * non-leaf node at this level that supports byte + * mode for WFQ among its children. WFQ weights will + * be applied against bytes for scheduling children + * when a non-leaf node is configured appropriately. + */ + int sched_wfq_byte_mode_supported; + + /** Mask of statistics counter types supported by the + * non-leaf nodes on this level. Every supported + * statistics counter type is supported by at least one + * non-leaf node on this level, which may not be true + * for all the non-leaf nodes on this level. + * @see enum rte_tm_stats_type + */ + uint64_t stats_mask; + } nonleaf; + + /** Items valid only for the leaf nodes on this level. */ + struct + { + /** Private shaper support. When non-zero, it indicates + * there is at least one leaf node on this level with + * private shaper support, which may not be the case for + * all the leaf nodes on this level. + */ + int shaper_private_supported; + + /** Dual rate support for private shaper. Valid only + * when private shaper is supported for the leaf nodes + * on this level. When non-zero, it indicates there is + * at least one leaf node on this level with dual rate + * private shaper support, which may not be the case for + * all the leaf nodes on this level. + */ + int shaper_private_dual_rate_supported; + + /** Minimum committed/peak rate (bytes per second) for + * private shapers of the leaf nodes of this level. + * Valid only when private shaper is supported for the + * leaf nodes on this level. + */ + uint64_t shaper_private_rate_min; + + /** Maximum committed/peak rate (bytes per second) for + * private shapers of the leaf nodes on this level. + * Valid only when private shaper is supported for the + * leaf nodes on this level. + */ + uint64_t shaper_private_rate_max; + + /** Shaper private packet mode supported. When non-zero, + * this parameter indicates there is at least one leaf + * node at this level that can be configured with + * packet mode in its private shaper. When private + * shaper is configured in packet mode, committed/peak + * rate provided is interpreted in packets per second. + */ + int shaper_private_packet_mode_supported; + /** Shaper private byte mode supported. When non-zero, + * this parameter indicates there is at least one leaf + * node at this level that can be configured with + * byte mode in its private shaper. When private shaper + * is configured in byte mode, committed/peak rate + * provided is interpreted in bytes per second. + */ + int shaper_private_byte_mode_supported; + + } leaf; + }; +} tm_level_capa_params_t; + /** * Node statistics counters */ @@ -170,6 +536,9 @@ typedef struct tm_system_t_ int (*node_sched_weight_update) (u32 hw_if_idx, u32 node_id, u32 weight); int (*node_read_stats) (u32 hw_if_idx, u32 node_idx, tm_stats_params_t *param); + int (*tm_get_capabilities) (u32 hw_if_idx, tm_capa_params_t *capa_param); + int (*tm_level_get_capabilities) (u32 hw_if_idx, tm_level_capa_params_t *cap, + u32 lvl); int (*start_tm) (u32 hw_if_idx); int (*stop_tm) (u32 hw_if_idx); } tm_system_t; @@ -280,6 +649,16 @@ int tm_sys_node_sched_weight_update (u32 hw_if_idx, u32 node_id, u32 weight); */ int tm_sys_node_read_stats (u32 hw_if_idx, u32 node_idx, tm_stats_params_t *param); +/** + * @brief Read Capabilities for a specific traffic management system. + */ +int tm_sys_get_capabilities (u32 hw_if_idx, tm_capa_params_t *capa_param); + +/** + * @brief Read level Capabilities for a specific traffic management system. + */ +int tm_sys_level_get_capabilities (u32 hw_if_idx, tm_level_capa_params_t *cap, + u32 lvl); /** * @brief Start the traffic management system. diff --git a/src/vnet/tm/tm_api.c b/src/vnet/tm/tm_api.c index e0c54619ce..b2da4a19a6 100644 --- a/src/vnet/tm/tm_api.c +++ b/src/vnet/tm/tm_api.c @@ -219,6 +219,40 @@ vl_api_tm_sys_node_read_stats_t_handler (vl_api_tm_sys_node_read_stats_t *mp) ({ rmp->node_id = clib_host_to_net_u32 (node_id); })); } +void +vl_api_tm_sys_get_capabilities_t_handler (vl_api_tm_sys_get_capabilities_t *mp) +{ + vl_api_tm_sys_get_capabilities_reply_t *rmp; + vnet_main_t *vnm = vnet_get_main (); + tm_capa_params_t s_p = { 0 }; + int rv = -1; + + vnet_sw_interface_t *sw = + vnet_get_sup_sw_interface (vnm, clib_net_to_host_u32 (mp->sw_if_idx)); + + rv = tm_sys_get_capabilities (sw->hw_if_index, &s_p); + + REPLY_MACRO (VL_API_TM_SYS_GET_CAPABILITIES_REPLY); +} + +void +vl_api_tm_sys_level_get_capabilities_t_handler ( + vl_api_tm_sys_level_get_capabilities_t *mp) +{ + vl_api_tm_sys_level_get_capabilities_reply_t *rmp; + vnet_main_t *vnm = vnet_get_main (); + tm_level_capa_params_t s_p = { 0 }; + int rv = -1; + u32 lvl = 0; + + vnet_sw_interface_t *sw = + vnet_get_sup_sw_interface (vnm, clib_net_to_host_u32 (mp->sw_if_idx)); + + rv = tm_sys_level_get_capabilities (sw->hw_if_index, &s_p, lvl); + + REPLY_MACRO (VL_API_TM_SYS_LEVEL_GET_CAPABILITIES_REPLY); +} + void vl_api_tm_sys_start_tm_t_handler (vl_api_tm_sys_start_tm_t *mp) { diff --git a/src/vnet/tm/tm_test.c b/src/vnet/tm/tm_test.c index 62214ce585..ae3554338b 100644 --- a/src/vnet/tm/tm_test.c +++ b/src/vnet/tm/tm_test.c @@ -435,6 +435,80 @@ api_tm_sys_node_read_stats (vat_main_t *vam) mp->sw_if_idx = clib_host_to_net_u32 (sw_if_idx); mp->node_id = clib_host_to_net_u32 (tm_node_id); + S (mp); + W (ret); + return ret; +} + +static int +api_tm_sys_get_capabilities (vat_main_t *vam) +{ + u8 sw_if_idx_set = 0; + unformat_input_t *i = vam->input; + vl_api_tm_sys_get_capabilities_t *mp; + u32 msg_size = sizeof (*mp); + u32 sw_if_idx = 0; + int ret; + + vam->result_ready = 0; + mp = vl_msg_api_alloc_as_if_client (msg_size); + + while (unformat_check_input (i) != UNFORMAT_END_OF_INPUT) + { + if (unformat (i, "sw_if_idx %u", &sw_if_idx)) + sw_if_idx_set = 1; + else + { + clib_warning ("Invalid input, unknown parameter"); + return -EINVAL; + } + } + + if (!sw_if_idx_set) + return -EINVAL; + + M (TM_SYS_GET_CAPABILITIES, mp); + + mp->sw_if_idx = clib_host_to_net_u32 (sw_if_idx); + + S (mp); + W (ret); + return ret; +} + +static int +api_tm_sys_level_get_capabilities (vat_main_t *vam) +{ + u8 sw_if_idx_set = 0, tm_lvl_idx_set = 0; + unformat_input_t *i = vam->input; + vl_api_tm_sys_level_get_capabilities_t *mp; + u32 msg_size = sizeof (*mp); + u32 tm_lvl, sw_if_idx = 0; + int ret; + + vam->result_ready = 0; + mp = vl_msg_api_alloc_as_if_client (msg_size); + + while (unformat_check_input (i) != UNFORMAT_END_OF_INPUT) + { + if (unformat (i, "sw_if_idx %u", &sw_if_idx)) + sw_if_idx_set = 1; + else if (unformat (i, "tm_level %u", &tm_lvl)) + tm_lvl_idx_set = 1; + else + { + clib_warning ("Invalid input, unknown parameter"); + return -EINVAL; + } + } + + if (!sw_if_idx_set || !tm_lvl_idx_set) + return -EINVAL; + + M (TM_SYS_LEVEL_GET_CAPABILITIES, mp); + + mp->sw_if_idx = clib_host_to_net_u32 (sw_if_idx); + mp->level = clib_host_to_net_u32 (tm_lvl); S (mp); W (ret); @@ -602,6 +676,24 @@ vl_api_tm_sys_node_read_stats_reply_t_handler ( vam->result_ready = 1; } +static void +vl_api_tm_sys_get_capabilities_reply_t_handler ( + vl_api_tm_sys_get_capabilities_reply_t *mp) +{ + vat_main_t *vam = tm_test_main.vat_main; + clib_warning ("TM Capability Passed : %u\n"); + vam->result_ready = 1; +} + +static void +vl_api_tm_sys_level_get_capabilities_reply_t_handler ( + vl_api_tm_sys_level_get_capabilities_reply_t *mp) +{ + vat_main_t *vam = tm_test_main.vat_main; + clib_warning ("TM Level Capability Passed : %u\n"); + vam->result_ready = 1; +} + static void vl_api_tm_sys_start_tm_reply_t_handler (vl_api_tm_sys_start_tm_reply_t *mp) { From 851cff3da2584e400ceec1fe932fa1734c93d06a Mon Sep 17 00:00:00 2001 From: Alok Mishra Date: Tue, 7 Jan 2025 15:25:05 +0500 Subject: [PATCH 179/271] octeon: add support for fetching tm capabilities This patch implements support to fetch TM node capabilities and TM level capabilities in the dev_octeon plugin. Type: feature Signed-off-by: Harish Kumar Malik Signed-off-by: Alok Mishra Change-Id: I08e01a5889044a899164d73fa26ba0a24f12a786 Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/142699 Tested-by: sa_ip-sw-jenkins Reviewed-by: Nithin Kumar Dabilpuram (cherry picked from commit 7a33462af156b04cd2fbf649b65c6dfe6052ae99) Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/144624 Reviewed-by: Monendra Singh Kushwaha --- src/plugins/dev_octeon/tm.c | 147 ++++++++++++++++++++++++++++++++++++ 1 file changed, 147 insertions(+) diff --git a/src/plugins/dev_octeon/tm.c b/src/plugins/dev_octeon/tm.c index a7e05cf0bf..b54662b957 100644 --- a/src/plugins/dev_octeon/tm.c +++ b/src/plugins/dev_octeon/tm.c @@ -335,6 +335,151 @@ oct_tm_sys_node_sched_weight_update (u32 hw_if_idx, u32 node_id, u32 weight) return rc; } +int +oct_tm_sys_get_capabilities (u32 hw_if_idx, tm_capa_params_t *cap) +{ + + vnet_main_t *vnm = vnet_get_main (); + vnet_hw_interface_t *hi = vnet_get_hw_interface (vnm, hw_if_idx); + vnet_dev_port_t *port = + vnet_dev_get_port_from_dev_instance (hi->dev_instance); + vnet_dev_t *dev = port->dev; + oct_device_t *cd = vnet_dev_get_data (dev); + struct roc_nix *nix = cd->nix; + int rc, max_nr_nodes = 0, i, n_lvl; + uint16_t schq[ROC_TM_LVL_MAX]; + + memset (cap, 0, sizeof (*cap)); + + rc = roc_nix_tm_rsrc_count (nix, schq); + if (rc) + { + return oct_roc_err (dev, rc, "oct_tm_sys_get_capabilities failed"); + } + + for (i = 0; i < NIX_TXSCH_LVL_TL1; i++) + max_nr_nodes += schq[i]; + + cap->n_nodes_max = max_nr_nodes + port->intf.num_tx_queues; + + n_lvl = roc_nix_tm_lvl_cnt_get (nix); + /* Consider leaf level */ + cap->n_levels_max = n_lvl + 1; + cap->non_leaf_nodes_identical = 1; + cap->leaf_nodes_identical = 1; + + /* Shaper Capabilities */ + cap->shaper_private_n_max = max_nr_nodes; + cap->shaper_n_max = max_nr_nodes; + cap->shaper_private_dual_rate_n_max = max_nr_nodes; + cap->shaper_private_rate_min = NIX_TM_MIN_SHAPER_RATE / 8; + cap->shaper_private_rate_max = NIX_TM_MAX_SHAPER_RATE / 8; + cap->shaper_private_packet_mode_supported = 1; + cap->shaper_private_byte_mode_supported = 1; + cap->shaper_pkt_length_adjust_min = NIX_TM_LENGTH_ADJUST_MIN; + cap->shaper_pkt_length_adjust_max = NIX_TM_LENGTH_ADJUST_MAX; + + /* Schedule Capabilities */ + cap->sched_n_children_max = schq[n_lvl - 1]; + cap->sched_sp_n_priorities_max = NIX_TM_TLX_SP_PRIO_MAX; + cap->sched_wfq_n_children_per_group_max = cap->sched_n_children_max; + cap->sched_wfq_n_groups_max = 1; + cap->sched_wfq_weight_max = roc_nix_tm_max_sched_wt_get (); + cap->sched_wfq_packet_mode_supported = 1; + cap->sched_wfq_byte_mode_supported = 1; + + return 0; +} + +int +oct_tm_sys_level_get_capabilities (u32 hw_if_idx, tm_level_capa_params_t *cap, + u32 lvl) +{ + vnet_main_t *vnm = vnet_get_main (); + vnet_hw_interface_t *hi = vnet_get_hw_interface (vnm, hw_if_idx); + vnet_dev_port_t *port = + vnet_dev_get_port_from_dev_instance (hi->dev_instance); + vnet_dev_t *dev = port->dev; + oct_device_t *cd = vnet_dev_get_data (dev); + struct roc_nix *nix = cd->nix; + int rc, n_lvl; + uint16_t schq[ROC_TM_LVL_MAX]; + + memset (cap, 0, sizeof (*cap)); + + rc = roc_nix_tm_rsrc_count (nix, schq); + if (rc) + { + return oct_roc_err (dev, rc, "oct_tm_sys_get_capabilities failed"); + } + + n_lvl = roc_nix_tm_lvl_cnt_get (nix); + + if (roc_nix_tm_lvl_is_leaf (nix, lvl)) + { + /* Leaf */ + cap->n_nodes_max = port->intf.num_tx_queues; + cap->n_nodes_leaf_max = port->intf.num_tx_queues; + cap->leaf_nodes_identical = 1; + } + else if (lvl == ROC_TM_LVL_ROOT) + { + /* Root node, a.k.a. TL2(vf)/TL1(pf) */ + cap->n_nodes_max = 1; + cap->n_nodes_nonleaf_max = 1; + cap->non_leaf_nodes_identical = 1; + + cap->nonleaf.shaper_private_supported = true; + cap->nonleaf.shaper_private_dual_rate_supported = + roc_nix_tm_lvl_have_link_access (nix, lvl) ? false : true; + cap->nonleaf.shaper_private_rate_min = NIX_TM_MIN_SHAPER_RATE / 8; + cap->nonleaf.shaper_private_rate_max = NIX_TM_MAX_SHAPER_RATE / 8; + cap->nonleaf.shaper_private_packet_mode_supported = 1; + cap->nonleaf.shaper_private_byte_mode_supported = 1; + + cap->nonleaf.sched_n_children_max = schq[lvl]; + cap->nonleaf.sched_sp_n_priorities_max = + roc_nix_tm_max_prio (nix, lvl) + 1; + cap->nonleaf.sched_wfq_n_groups_max = 1; + cap->nonleaf.sched_wfq_weight_max = roc_nix_tm_max_sched_wt_get (); + cap->nonleaf.sched_wfq_packet_mode_supported = 1; + cap->nonleaf.sched_wfq_byte_mode_supported = 1; + } + else if (lvl < ROC_TM_LVL_MAX) + { + /* TL2, TL3, TL4, MDQ */ + cap->n_nodes_max = schq[lvl]; + cap->n_nodes_nonleaf_max = cap->n_nodes_max; + cap->non_leaf_nodes_identical = 1; + + cap->nonleaf.shaper_private_supported = true; + cap->nonleaf.shaper_private_dual_rate_supported = true; + cap->nonleaf.shaper_private_rate_min = NIX_TM_MIN_SHAPER_RATE / 8; + cap->nonleaf.shaper_private_rate_max = NIX_TM_MAX_SHAPER_RATE / 8; + cap->nonleaf.shaper_private_packet_mode_supported = 1; + cap->nonleaf.shaper_private_byte_mode_supported = 1; + + /* MDQ doesn't support Strict Priority */ + if ((int) lvl == (n_lvl - 1)) + cap->nonleaf.sched_n_children_max = port->intf.num_tx_queues; + else + cap->nonleaf.sched_n_children_max = schq[lvl - 1]; + cap->nonleaf.sched_sp_n_priorities_max = + roc_nix_tm_max_prio (nix, lvl) + 1; + cap->nonleaf.sched_wfq_n_groups_max = 1; + cap->nonleaf.sched_wfq_weight_max = roc_nix_tm_max_sched_wt_get (); + cap->nonleaf.sched_wfq_packet_mode_supported = 1; + cap->nonleaf.sched_wfq_byte_mode_supported = 1; + } + else + { + /* unsupported level */ + return oct_roc_err ( + dev, rc, "oct_tm_sys_get_capabilities unsupported level,failed"); + } + return 0; +} + int oct_tm_sys_node_read_stats (u32 hw_if_idx, u32 node_id, tm_stats_params_t *stats) @@ -455,6 +600,8 @@ tm_system_t dev_oct_tm_ops = { .node_add = oct_tm_sys_node_add, .node_delete = oct_tm_sys_node_delete, .node_read_stats = oct_tm_sys_node_read_stats, + .tm_get_capabilities = oct_tm_sys_get_capabilities, + .tm_level_get_capabilities = oct_tm_sys_level_get_capabilities, .shaper_profile_create = oct_tm_sys_shaper_profile_create, .node_shaper_update = oct_tm_sys_node_shaper_update, .shaper_profile_delete = oct_tm_sys_shaper_profile_delete, From 4ffc3a11ce3c550ed6025cfdb4b9576f3f5562ae Mon Sep 17 00:00:00 2001 From: Alok Mishra Date: Fri, 17 Jan 2025 14:10:28 +0500 Subject: [PATCH 180/271] tm: fix node shaper update api This patch fixes the input parameter reading for "node_shaper_update" TM API. Type: fix Signed-off-by: Alok Mishra Change-Id: Icf595b71c1d4b15cac2171678f4e198a38e5ec51 Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/143469 Tested-by: sa_ip-sw-jenkins Reviewed-by: Nithin Kumar Dabilpuram Reviewed-by: Satha Koteswara Rao Kottidi (cherry picked from commit 8f323729673ef3fabdf116bca89e3a8fe4d8c0e9) Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/144625 Reviewed-by: Monendra Singh Kushwaha --- src/vnet/tm/tm_test.c | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/src/vnet/tm/tm_test.c b/src/vnet/tm/tm_test.c index ae3554338b..457cfb2241 100644 --- a/src/vnet/tm/tm_test.c +++ b/src/vnet/tm/tm_test.c @@ -221,7 +221,7 @@ api_tm_sys_shaper_profile_create (vat_main_t *vam) u64 shaper_peak_rate = 0; u64 shaper_peak_burst = 0; u32 tm_shaper_id = 0; - u8 sw_if_idx_set = 0; + u8 sw_if_idx_set = 0, tm_shaper_id_set = 0; u32 is_packet_mode = 0; u32 sw_if_idx = 0; int ret; @@ -235,7 +235,7 @@ api_tm_sys_shaper_profile_create (vat_main_t *vam) if (unformat (i, "sw_if_idx %u", &sw_if_idx)) sw_if_idx_set = 1; else if (unformat (i, "shaper_id %u", &tm_shaper_id)) - ; + tm_shaper_id_set = 1; else if (unformat (i, "packet_mode %u", &is_packet_mode)) ; else if (unformat (i, "shaper_peak_burst %llu", &shaper_peak_burst)) @@ -255,7 +255,7 @@ api_tm_sys_shaper_profile_create (vat_main_t *vam) } } - if (!sw_if_idx_set) + if (!sw_if_idx_set || !tm_shaper_id_set) return -EINVAL; M (TM_SYS_SHAPER_PROFILE_CREATE, mp); @@ -277,7 +277,7 @@ api_tm_sys_shaper_profile_create (vat_main_t *vam) static int api_tm_sys_node_shaper_update (vat_main_t *vam) { - u8 sw_if_idx_set = 0; + u8 sw_if_idx_set = 0, shaper_profile_set = 0; unformat_input_t *i = vam->input; vl_api_tm_sys_node_shaper_update_t *mp; u32 msg_size = sizeof (*mp); @@ -292,8 +292,8 @@ api_tm_sys_node_shaper_update (vat_main_t *vam) { if (unformat (i, "sw_if_idx %u", &sw_if_idx)) sw_if_idx_set = 1; - else if (unformat (i, "shaper_profile %u", &shaper_profile)) - shaper_profile = 1; + else if (unformat (i, "shaper_profile %d", &shaper_profile)) + shaper_profile_set = 1; else if (unformat (i, "node_id %u", &node_id)) ; else @@ -303,7 +303,7 @@ api_tm_sys_node_shaper_update (vat_main_t *vam) } } - if (!sw_if_idx_set || !shaper_profile) + if (!sw_if_idx_set || !shaper_profile_set) return -EINVAL; M (TM_SYS_NODE_SHAPER_UPDATE, mp); @@ -323,7 +323,7 @@ api_tm_sys_shaper_profile_delete (vat_main_t *vam) vl_api_tm_sys_shaper_profile_delete_t *mp; unformat_input_t *i = vam->input; u32 msg_size = sizeof (*mp); - u8 sw_if_idx_set = 0; + u8 sw_if_idx_set = 0, shaper_id_set = 0; u32 sw_if_idx = 0; u32 shaper_id = 0; int ret; @@ -335,9 +335,8 @@ api_tm_sys_shaper_profile_delete (vat_main_t *vam) { if (unformat (i, "sw_if_idx %u", &sw_if_idx)) sw_if_idx_set = 1; - - if (unformat (i, "shaper_id %u", &shaper_id)) - ; + else if (unformat (i, "shaper_id %u", &shaper_id)) + shaper_id_set = 1; else { clib_warning ("Invalid input, unknown parameter"); @@ -345,7 +344,7 @@ api_tm_sys_shaper_profile_delete (vat_main_t *vam) } } - if (!shaper_id && !sw_if_idx_set) + if (!sw_if_idx_set || !shaper_id_set) return -EINVAL; M (TM_SYS_SHAPER_PROFILE_DELETE, mp); @@ -640,7 +639,7 @@ vl_api_tm_sys_node_shaper_update_reply_t_handler ( vl_api_tm_sys_node_shaper_update_reply_t *mp) { vat_main_t *vam = tm_test_main.vat_main; - clib_warning ("TM node updated shaper id : %u\n", + clib_warning ("TM node updated shaper id : %d\n", clib_net_to_host_u32 (mp->shaper_id)); vam->result_ready = 1; From bac8de2d5f9e478742015c4b67ba82b845ac1bd8 Mon Sep 17 00:00:00 2001 From: Varun Rapelly Date: Wed, 4 Dec 2024 05:58:01 +0000 Subject: [PATCH 181/271] tls: fix async processing issues with engine This patch fixes issues with async processing of events when openssl engine is enabled. Updated async read and write handlers to read/write based on the event and will not add any new async event in the handler. Fixes memory leak issue in handshake event handler. Type: fix JIRA: https://essjira.marvell.com/browse/IPBUSW-58152 Signed-off-by: Varun Rapelly Change-Id: I3dc6aec8125b088c349acf7d6f2e2b9f226654df Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/140728 Tested-by: sa_ip-sw-jenkins Reviewed-by: Venkata Ravichandra Mynidi (cherry picked from commit dcdcc99ad83bb449b63a9d75f04bf7f7248878b1) Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/144443 Reviewed-by: Monendra Singh Kushwaha --- src/plugins/tlsopenssl/tls_async.c | 859 +++++++++++---------------- src/plugins/tlsopenssl/tls_openssl.c | 143 +++-- src/plugins/tlsopenssl/tls_openssl.h | 18 +- src/vnet/tls/tls.c | 6 +- src/vnet/tls/tls.h | 7 +- 5 files changed, 423 insertions(+), 610 deletions(-) diff --git a/src/plugins/tlsopenssl/tls_async.c b/src/plugins/tlsopenssl/tls_async.c index 0fe785bd53..79a2151c2e 100644 --- a/src/plugins/tlsopenssl/tls_async.c +++ b/src/plugins/tlsopenssl/tls_async.c @@ -31,7 +31,7 @@ [SSL_CLIENT_HELLO_CB] = "SSL_CLIENT_HELLO_CB", \ } -static const char *ssl_want[] = SSL_WANT_NAMES; +const char *ssl_want[] = SSL_WANT_NAMES; typedef enum ssl_evt_status_type_ { @@ -39,6 +39,8 @@ typedef enum ssl_evt_status_type_ SSL_ASYNC_INFLIGHT = 1, SSL_ASYNC_READY = 2, SSL_ASYNC_REENTER = 3, + SSL_ASYNC_DEQ_DONE = 4, + SSL_ASYNC_CB_EXECUTED = 5, SSL_ASYNC_MAX_STATUS } ssl_evt_status_type_t; @@ -46,6 +48,8 @@ typedef struct openssl_tls_callback_arg_ { int thread_index; int event_index; + ssl_async_evt_type_t async_evt_type; + openssl_resume_handler *evt_handler; } openssl_tls_callback_arg_t; typedef struct openssl_event_ @@ -53,12 +57,13 @@ typedef struct openssl_event_ u32 ctx_index; int session_index; ssl_evt_status_type_t status; - ssl_async_evt_type_t type; - openssl_resume_handler *handler; + transport_send_params_t *tran_sp; openssl_tls_callback_arg_t cb_args; #define thread_idx cb_args.thread_index -#define event_idx cb_args.event_index +#define event_idx cb_args.event_index +#define async_event_type cb_args.async_evt_type +#define async_evt_handler cb_args.evt_handler int next; } openssl_evt_t; @@ -67,7 +72,6 @@ typedef struct openssl_async_queue_ int evt_run_head; int evt_run_tail; int depth; - int max_depth; } openssl_async_queue_t; typedef struct openssl_async_ @@ -96,10 +100,11 @@ struct engine_polling void qat_init_thread (void *arg); void dpdk_engine_init_thread (void *arg); +void dpdk_engine_polling (); struct engine_polling engine_list[] = { { "qat", qat_polling, qat_pre_init, qat_init_thread }, - { "dpdk_engine", dasync_polling, NULL, dpdk_engine_init_thread }, + { "dpdk_engine", dpdk_engine_polling, NULL, dpdk_engine_init_thread }, { "dasync", dasync_polling, NULL, NULL } }; @@ -133,12 +138,10 @@ evt_pool_init (vlib_main_t * vm) om->queue[i].evt_run_head = -1; om->queue[i].evt_run_tail = -1; om->queue[i].depth = 0; - om->queue[i].max_depth = 0; om->queue_in_init[i].evt_run_head = -1; om->queue_in_init[i].evt_run_tail = -1; om->queue_in_init[i].depth = 0; - om->queue_in_init[i].max_depth = 0; } om->polling = NULL; @@ -270,7 +273,6 @@ openssl_evt_alloc (void) /* In most cases, tls_async_openssl_callback is called by HW to make event active - * When EAGAIN received, VPP will call this callback to retry */ int tls_async_openssl_callback (SSL * s, void *cb_arg) @@ -280,31 +282,41 @@ tls_async_openssl_callback (SSL * s, void *cb_arg) openssl_tls_callback_arg_t *args = (openssl_tls_callback_arg_t *) cb_arg; int thread_index = args->thread_index; int event_index = args->event_index; - int *evt_run_tail = &om->queue[thread_index].evt_run_tail; - int *evt_run_head = &om->queue[thread_index].evt_run_head; + ssl_async_evt_type_t evt_type = args->async_evt_type; + openssl_async_queue_t *queue; + int *evt_run_tail, *evt_run_head; - TLS_DBG (2, "Set event %d to run\n", event_index); event = openssl_evt_get_w_thread (event_index, thread_index); + if (evt_type == SSL_ASYNC_EVT_INIT) + queue = om->queue_in_init; + else + queue = om->queue; + + evt_run_tail = &queue[thread_index].evt_run_tail; + evt_run_head = &queue[thread_index].evt_run_head; + /* Happend when a recursive case, especially in SW simulation */ if (PREDICT_FALSE (event->status == SSL_ASYNC_READY)) { + clib_warning ("EVT: %p ASYNC_ALREADY Set", event); event->status = SSL_ASYNC_REENTER; return 0; } event->status = SSL_ASYNC_READY; event->next = -1; - if (*evt_run_tail >= 0) + if (*evt_run_head < 0) + *evt_run_head = event_index; + else if (*evt_run_tail >= 0) { event_tail = openssl_evt_get_w_thread (*evt_run_tail, thread_index); event_tail->next = event_index; } + + queue[thread_index].depth++; + *evt_run_tail = event_index; - if (*evt_run_head < 0) - { - *evt_run_head = event_index; - } return 1; } @@ -317,59 +329,23 @@ tls_async_openssl_callback (SSL * s, void *cb_arg) * (ASYNC_JOB*), the 'buf' and 'num' params to SSL_write() have * already been set in the initial call, and are meaningless here. * Therefore setting buf=null,num=0, to emphasize the point. - * On successful write, TLS context total_write bytes are updated. + * On successful write, TLS context total_async_write bytes are updated. */ -static int +int openssl_async_write_from_fifo_into_ssl (svm_fifo_t *f, SSL *ssl, - tls_ctx_t *ctx) + openssl_ctx_t *ctx) { int wrote = 0; wrote = SSL_write (ssl, NULL, 0); ossl_check_err_is_fatal (ssl, wrote); - ctx->total_write -= wrote; + ctx->total_async_write -= wrote; svm_fifo_dequeue_drop (f, wrote); return wrote; } -/* - * Perform SSL_write from TX FIFO head. - * On successful write, TLS context total_write bytes are updated. - */ -static_always_inline int -openssl_write_from_fifo_head_into_ssl (svm_fifo_t *f, SSL *ssl, tls_ctx_t *ctx, - u32 max_len) -{ - int wrote = 0, rv, i = 0, len; - u32 n_segs = 2; - svm_fifo_seg_t fs[n_segs]; - - max_len = clib_min (ctx->total_write, max_len); - - len = svm_fifo_segments (f, 0, fs, &n_segs, max_len); - if (len <= 0) - return 0; - - while (wrote < len && i < n_segs) - { - rv = SSL_write (ssl, fs[i].data, fs[i].len); - wrote += (rv > 0) ? rv : 0; - if (rv < (int) fs[i].len) - break; - i++; - } - - if (wrote) - { - ctx->total_write -= wrote; - svm_fifo_dequeue_drop (f, wrote); - } - - return wrote; -} - static int openssl_async_read_from_ssl_into_fifo (svm_fifo_t *f, SSL *ssl) { @@ -384,456 +360,55 @@ openssl_async_read_from_ssl_into_fifo (svm_fifo_t *f, SSL *ssl) return read; } -/* - * Pop the current event from queue and update tail if needed - */ -static void -tls_async_dequeue_update (openssl_evt_t *event, int *evt_run_head, - int *evt_run_tail, int *queue_depth) -{ - /* remove the event from queue head */ - *evt_run_head = event->next; - event->status = SSL_ASYNC_INVALID_STATUS; - event->next = -1; - - (*queue_depth)--; - - if (*evt_run_head < 0) - { - *evt_run_tail = -1; - if (*queue_depth) - ERRORPR ("queue empty but depth:%d\n", *queue_depth); - } -} - -static int -tls_async_dequeue_event (int thread_index) -{ - openssl_evt_t *event; - openssl_async_t *om = &openssl_async_main; - openssl_async_queue_t *queue = om->queue; - int *evt_run_tail = &queue[thread_index].evt_run_tail; - int *evt_run_head = &queue[thread_index].evt_run_head; - int dequeue_cnt = clib_min (queue[thread_index].depth, MAX_VECTOR_ASYNC); - - /* dequeue all pending events, events enqueued during this routine call, - * will be handled next time tls_async_dequeue_event is invoked */ - while (*evt_run_head >= 0 && dequeue_cnt--) - { - session_t *app_session, *tls_session; - openssl_ctx_t *oc; - tls_ctx_t *ctx; - SSL *ssl; - - event = openssl_evt_get_w_thread (*evt_run_head, thread_index); - ctx = openssl_ctx_get_w_thread (event->ctx_index, thread_index); - oc = (openssl_ctx_t *) ctx; - ssl = oc->ssl; - - if (event->type == SSL_ASYNC_EVT_RD) - { - /* read event */ - svm_fifo_t *app_rx_fifo, *tls_rx_fifo; - int read; - - app_session = session_get_from_handle (ctx->app_session_handle); - app_rx_fifo = app_session->rx_fifo; - - tls_session = session_get_from_handle (ctx->tls_session_handle); - tls_rx_fifo = tls_session->rx_fifo; - - /* continue the paused job */ - read = openssl_async_read_from_ssl_into_fifo (app_rx_fifo, ssl); - if (read < 0) - { - if (SSL_want_async (ssl)) - goto handle_later; - - tls_async_dequeue_update (event, evt_run_head, evt_run_tail, - &queue[thread_index].depth); - goto ev_rd_done; - } - - /* read finished or in error, remove the event from queue */ - tls_async_dequeue_update (event, evt_run_head, evt_run_tail, - &queue[thread_index].depth); - - /* Unrecoverable protocol error. Reset connection */ - if (PREDICT_FALSE ((read < 0) && - (SSL_get_error (ssl, read) == SSL_ERROR_SSL))) - { - tls_notify_app_io_error (ctx); - goto ev_rd_done; - } - - /* - * Managed to read some data. If handshake just completed, session - * may still be in accepting state. - */ - if (app_session->session_state >= SESSION_STATE_READY) - tls_notify_app_enqueue (ctx, app_session); - - /* managed to read, try to read more */ - while (read > 0) - { - read = openssl_read_from_ssl_into_fifo (app_rx_fifo, ssl); - if (read < 0) - { - if (SSL_want_async (ssl)) - { - vpp_tls_async_enqueue_event (ctx, SSL_ASYNC_EVT_RD, NULL, - 0); - goto ev_rd_queued; - } - } - - /* Unrecoverable protocol error. Reset connection */ - if (PREDICT_FALSE ((read < 0) && - (SSL_get_error (ssl, read) == SSL_ERROR_SSL))) - { - tls_notify_app_io_error (ctx); - goto ev_rd_done; - } - - /* If handshake just completed, session may still be in accepting - * state */ - if (read >= 0 && - app_session->session_state >= SESSION_STATE_READY) - tls_notify_app_enqueue (ctx, app_session); - } - - ev_rd_done: - /* read done */ - ctx->in_async_read = false; - - if ((SSL_pending (ssl) > 0) || - svm_fifo_max_dequeue_cons (tls_rx_fifo)) - { - tls_add_vpp_q_builtin_rx_evt (tls_session); - } - - ev_rd_queued: - continue; - } - else if (event->type == SSL_ASYNC_EVT_WR) - { - /* write event */ - int wrote, wrote_sum = 0; - u32 space, enq_buf; - svm_fifo_t *app_tx_fifo, *tls_tx_fifo; - transport_send_params_t *sp = - (transport_send_params_t *) event->handler; - - app_session = session_get_from_handle (ctx->app_session_handle); - app_tx_fifo = app_session->tx_fifo; - - /* continue the paused job */ - wrote = - openssl_async_write_from_fifo_into_ssl (app_tx_fifo, ssl, ctx); - if (!wrote) - { - if (SSL_want_async (ssl)) - /* paused job not ready, wait */ - goto handle_later; - ERRORPR ("[wrote:%d want:%s ctx:%d]\n", wrote, - ssl_want[SSL_want (ssl)], oc->openssl_ctx_index); - } - - /* paused job done, remove event, update queue */ - tls_async_dequeue_update (event, evt_run_head, evt_run_tail, - &queue[thread_index].depth); - - /* Unrecoverable protocol error. Reset connection */ - if (PREDICT_FALSE (wrote < 0)) - { - tls_notify_app_io_error (ctx); - ERRORPR ("Unrecoverable protocol error. Reset connection\n"); - goto ev_in_queue; - } - wrote_sum += wrote; - - tls_session = session_get_from_handle (ctx->tls_session_handle); - tls_tx_fifo = tls_session->tx_fifo; - - /* prepare for remaining write(s) */ - space = svm_fifo_max_enqueue_prod (tls_tx_fifo); - /* Leave a bit of extra space for tls ctrl data, if any needed */ - space = clib_max ((int) space - TLSO_CTRL_BYTES, 0); - - /* continue remaining openssl_ctx_write request */ - while (ctx->total_write) - { - int rv; - u32 deq_max = svm_fifo_max_dequeue_cons (app_tx_fifo); - - deq_max = clib_min (deq_max, space); - deq_max = clib_min (deq_max, sp->max_burst_size); - if (!deq_max) - goto check_tls_fifo; - - /* Make sure tcp's tx fifo can actually buffer all bytes to - * be dequeued. If under memory pressure, tls's fifo segment - * might not be able to allocate the chunks needed. This also - * avoids errors from the underlying custom bio to the ssl - * infra which at times can get stuck. */ - if (svm_fifo_provision_chunks (tls_tx_fifo, 0, 0, - deq_max + TLSO_CTRL_BYTES)) - goto check_tls_fifo; - - rv = openssl_write_from_fifo_head_into_ssl (app_tx_fifo, ssl, - ctx, deq_max); - - /* Unrecoverable protocol error. Reset connection */ - if (PREDICT_FALSE (rv < 0)) - { - tls_notify_app_io_error (ctx); - ERRORPR ("Unrecoverable protocol error. Reset connection\n"); - goto ev_in_queue; - } - - if (!rv) - { - if (SSL_want_async (ssl)) - { - /* new paused job, add queue event and wait */ - vpp_tls_async_enqueue_event (ctx, SSL_ASYNC_EVT_WR, sp, - 0); - goto ev_in_queue; - } - ERRORPR ("[rv:%d want:%s ctx:%d]\n", rv, - ssl_want[SSL_want (ssl)], oc->openssl_ctx_index); - break; - } - wrote_sum += rv; - } - - if (svm_fifo_needs_deq_ntf (app_tx_fifo, wrote)) - session_dequeue_notify (app_session); - - check_tls_fifo: - /* we got here, async write is done or not possible */ - ctx->total_write = 0; - - if (PREDICT_FALSE (ctx->app_closed && - BIO_ctrl_pending (oc->rbio) <= 0)) - openssl_confirm_app_close (ctx); - - /* Deschedule and wait for deq notification if fifo is almost full */ - enq_buf = - clib_min (svm_fifo_size (tls_tx_fifo) / 2, TLSO_MIN_ENQ_SPACE); - if (space < wrote_sum + enq_buf) - { - svm_fifo_add_want_deq_ntf (tls_tx_fifo, SVM_FIFO_WANT_DEQ_NOTIF); - transport_connection_deschedule (&ctx->connection); - sp->flags |= TRANSPORT_SND_F_DESCHED; - } - else - { - /* Request tx reschedule of the app session */ - app_session->flags |= SESSION_F_CUSTOM_TX; - transport_connection_reschedule (&ctx->connection); - } - - ev_in_queue: - /* job removed, openssl_ctx_write will resume */ - continue; - } - else - { - /* wrong event type */ - ERRORPR ("goto remove_event [event->type:%d]\n", event->type); - tls_async_dequeue_update (event, evt_run_head, evt_run_tail, - &queue[thread_index].depth); - } - } - -handle_later: - return 1; -} - -static int -tls_async_dequeue_event_in_init (int thread_index) +int +vpp_tls_async_init_event (tls_ctx_t *ctx, openssl_resume_handler *handler, + transport_send_params_t *sp, session_t *session, + ssl_async_evt_type_t evt_type, int wr_size) { - openssl_evt_t *event; - openssl_async_t *om = &openssl_async_main; - openssl_async_queue_t *queue = om->queue_in_init; - int *evt_run_tail = &queue[thread_index].evt_run_tail; - int *evt_run_head = &queue[thread_index].evt_run_head; + u32 eidx; + openssl_evt_t *event = NULL; + openssl_ctx_t *oc = (openssl_ctx_t *) ctx; + u32 thread_id = ctx->c_thread_index; - /* dequeue events if exists */ - while (*evt_run_head >= 0) + if (oc->evt_alloc_flag[evt_type]) { - openssl_ctx_t *oc; - tls_ctx_t *ctx; - int rv, err; - - event = openssl_evt_get_w_thread (*evt_run_head, thread_index); - ctx = openssl_ctx_get_w_thread (event->ctx_index, thread_index); - oc = (openssl_ctx_t *) ctx; - - if (event->type != SSL_ASYNC_EVT_INIT) - { - /* wrong event type */ - ERRORPR ("goto remove_event [event->type:%d]\n", event->type); - goto remove_event; - } - - if (!SSL_in_init (oc->ssl)) - { - ERRORPR ("[!SSL_in_init() != ev->type:%d] th:%d ev:%d\n", - event->type, event->cb_args.thread_index, - event->cb_args.event_index); - goto remove_event; - } - - rv = SSL_do_handshake (oc->ssl); - err = SSL_get_error (oc->ssl, rv); - - /* Do not remove session from tail */ - if (err == SSL_ERROR_WANT_ASYNC) - goto handle_later; - - if (err == SSL_ERROR_SSL) - { - char buf[512]; - ERR_error_string (ERR_get_error (), buf); - ERRORPR ("Err: %s\n", buf); - openssl_handle_handshake_failure (ctx); - goto remove_event; - } - - if (err == SSL_ERROR_WANT_WRITE || err == SSL_ERROR_WANT_READ) - goto handle_later; - - /* client not supported */ - if (!SSL_is_server (oc->ssl)) - { - ERRORPR ("goto remove_event [!SSL_is_server]\n"); - goto remove_event; - } - - /* Need to check transport status */ - if (ctx->is_passive_close) - { - openssl_handle_handshake_failure (ctx); - goto remove_event; - } - - if (tls_notify_app_accept (ctx)) + eidx = oc->evt_index[evt_type]; + if (evt_type == SSL_ASYNC_EVT_WR) { - ctx->c_s_index = SESSION_INVALID_INDEX; - tls_disconnect_transport (ctx); - } - - TLS_DBG (1, "Handshake for %u complete. TLS cipher is %s", - oc->openssl_ctx_index, SSL_get_cipher (oc->ssl)); - - remove_event: - *evt_run_head = event->next; - queue[thread_index].depth--; - - if (*evt_run_head < 0) - { - /* queue empty, bail out */ - *evt_run_tail = -1; - if (queue[thread_index].depth) - ERRORPR ("queue empty but depth:%d\n", queue[thread_index].depth); - break; + event = openssl_evt_get (eidx); + goto update_wr_evnt; } + return 1; } - -handle_later: - return 1; -} - -int -vpp_tls_async_enqueue_event (tls_ctx_t *ctx, int evt_type, - transport_send_params_t *sp, int size) -{ - openssl_evt_t *event; - openssl_async_t *om = &openssl_async_main; - openssl_async_queue_t *queue; - openssl_ctx_t *oc; - int thread_index; - int event_index; - int *evt_run_tail; - int *evt_run_head; - - event = openssl_evt_get (ctx->evt_index[evt_type]); - - thread_index = event->thread_idx; - event_index = event->event_idx; - - oc = (openssl_ctx_t *) ctx; - - /* set queue to be used */ - if (SSL_in_init (oc->ssl)) - queue = om->queue_in_init; else - queue = om->queue; - - evt_run_tail = &queue[thread_index].evt_run_tail; - evt_run_head = &queue[thread_index].evt_run_head; - - event->type = SSL_ASYNC_EVT_INIT; - event->handler = (openssl_resume_handler *) sp; - event->next = -1; - - /* first we enqueue the request */ - if (*evt_run_tail >= 0) { - openssl_evt_t *event_tail; - - /* queue not empty, append to tail event */ - event_tail = openssl_evt_get_w_thread (*evt_run_tail, thread_index); - event_tail->next = event_index; + eidx = openssl_evt_alloc (); + oc->evt_alloc_flag[evt_type] = true; } - /* set tail to use new event index */ - *evt_run_tail = event_index; - - if (*evt_run_head < 0) - /* queue is empty, update head */ - *evt_run_head = event_index; - - queue[thread_index].depth++; - if (queue[thread_index].depth > queue[thread_index].max_depth) - queue[thread_index].max_depth = queue[thread_index].depth; - - return 1; -} - -static int -vpp_tls_async_init_event (tls_ctx_t *ctx, openssl_resume_handler *handler, - session_t *session, ssl_async_evt_type_t evt_type) -{ - u32 eidx; - openssl_evt_t *event; - openssl_ctx_t *oc = (openssl_ctx_t *) ctx; - u32 thread_id = ctx->c_thread_index; - - eidx = openssl_evt_alloc (); event = openssl_evt_get (eidx); event->ctx_index = oc->openssl_ctx_index; + /* async call back args */ event->event_idx = eidx; event->thread_idx = thread_id; - event->handler = NULL; + event->async_event_type = evt_type; + event->async_evt_handler = handler; event->session_index = session->session_index; - event->type = evt_type; event->status = SSL_ASYNC_INVALID_STATUS; - ctx->evt_index[evt_type] = eidx; - - return 1; -} - -int -vpp_tls_async_init_events (tls_ctx_t *ctx, openssl_resume_handler *handler, - session_t *session) -{ - vpp_tls_async_init_event (ctx, handler, session, SSL_ASYNC_EVT_INIT); - vpp_tls_async_init_event (ctx, handler, session, SSL_ASYNC_EVT_RD); - vpp_tls_async_init_event (ctx, handler, session, SSL_ASYNC_EVT_WR); - + oc->evt_index[evt_type] = eidx; + TLS_DBG (2, "EVT: %p ALLOTED_FOR_TYPE_%d: EIDX: %d TID: %d", event, evt_type, + event->event_idx, event->thread_idx); +#ifdef HAVE_OPENSSL_ASYNC + SSL_set_async_callback_arg (oc->ssl, &event->cb_args); +#endif +update_wr_evnt: + if (evt_type == SSL_ASYNC_EVT_WR) + { + transport_connection_deschedule (&ctx->connection); + sp->flags |= TRANSPORT_SND_F_DESCHED; + oc->total_async_write = wr_size; + } + event->tran_sp = sp; return 1; } @@ -842,11 +417,12 @@ vpp_openssl_is_inflight (tls_ctx_t *ctx) { u32 eidx; openssl_evt_t *event; + openssl_ctx_t *oc = (openssl_ctx_t *) ctx; int i; for (i = SSL_ASYNC_EVT_INIT; i < SSL_ASYNC_EVT_MAX; i++) { - eidx = ctx->evt_index[i]; + eidx = oc->evt_index[i]; event = openssl_evt_get (eidx); if (event->status == SSL_ASYNC_INFLIGHT) @@ -856,40 +432,25 @@ vpp_openssl_is_inflight (tls_ctx_t *ctx) return 0; } -int -vpp_tls_async_update_event (tls_ctx_t *ctx, int eagain, - ssl_async_evt_type_t type) -{ - u32 eidx; - openssl_evt_t *event; - - eidx = ctx->evt_index[type]; - event = openssl_evt_get (eidx); - event->status = SSL_ASYNC_INFLIGHT; - if (eagain) - return tls_async_openssl_callback (0, &event->cb_args); - - return 1; -} - void -event_handler (void *tls_async) +async_event_handler (void *tls_async) { openssl_resume_handler *handler; openssl_evt_t *event; session_t *session; int thread_index; - tls_ctx_t *ctx; event = (openssl_evt_t *) tls_async; thread_index = event->thread_idx; - ctx = openssl_ctx_get_w_thread (event->ctx_index, thread_index); - handler = event->handler; + handler = event->async_evt_handler; session = session_get (event->session_index, thread_index); if (handler) { - (*handler) (ctx, session); + (*handler) (event, session); + event->status = SSL_ASYNC_CB_EXECUTED; + TLS_DBG (2, "AFTER_CB_EXECUTION_FOR_EVT: %p for TYPE: %d status: %d", + event, event->async_event_type, event->status); } return; @@ -980,50 +541,93 @@ tls_async_do_job (int eidx, u32 thread_index) if (ctx) { ctx->resume = 1; - session_send_rpc_evt_to_thread (thread_index, event_handler, event); + session_send_rpc_evt_to_thread (thread_index, async_event_handler, + event); } return 1; } int -tls_resume_from_crypto (int thread_index) +handle_async_cb_events (openssl_async_queue_t *queue, int thread_index) { int i; - - openssl_async_t *om = &openssl_async_main; openssl_evt_t *event; - int *evt_run_head = &om->queue[thread_index].evt_run_head; - int *evt_run_tail = &om->queue[thread_index].evt_run_tail; + + int *evt_run_head = &queue[thread_index].evt_run_head; + int *evt_run_tail = &queue[thread_index].evt_run_tail; if (*evt_run_head < 0) return 0; - for (i = 0; i < MAX_VECTOR_ASYNC; i++) + int n_events = queue[thread_index].depth; + for (i = 0; i < n_events; i++) { if (*evt_run_head >= 0) { + TLS_DBG (8, "HEAD_EVT_INDEX: %d CUR_DEPTH: %d", *evt_run_head, + queue[thread_index].depth); event = openssl_evt_get_w_thread (*evt_run_head, thread_index); - tls_async_do_job (*evt_run_head, thread_index); if (PREDICT_FALSE (event->status == SSL_ASYNC_REENTER)) { + TLS_DBG (2, "REENTER_TRIGGERED"); /* recusive event triggered */ - event->status = SSL_ASYNC_READY; - continue; + goto deq_event; } + tls_async_do_job (*evt_run_head, thread_index); - event->status = SSL_ASYNC_INVALID_STATUS; + deq_event: *evt_run_head = event->next; + event->status = SSL_ASYNC_DEQ_DONE; + queue[thread_index].depth--; - if (event->next < 0) + if (*evt_run_head < 0) { *evt_run_tail = -1; + TLS_DBG (8, "EVTS_DEQ_DONE"); break; } } } return 0; +} + +void +resume_handshake_events (int thread_index) +{ + openssl_async_t *om = &openssl_async_main; + openssl_async_queue_t *queue = om->queue_in_init; + handle_async_cb_events (queue, thread_index); +} + +void +resume_read_write_events (int thread_index) +{ + openssl_async_t *om = &openssl_async_main; + + openssl_async_queue_t *queue = om->queue; + handle_async_cb_events (queue, thread_index); +} + +int +tls_resume_from_crypto (int thread_index) +{ + resume_read_write_events (thread_index); + resume_handshake_events (thread_index); + return 0; +} + +void +dpdk_engine_polling () +{ + openssl_async_t *om = &openssl_async_main; + int poll_status = 0; + + if (om->start_polling) + { + ENGINE_ctrl_cmd (om->engine, "POLL", 0, &poll_status, NULL, 0); + } } void @@ -1060,6 +664,207 @@ tls_async_init (vlib_main_t * vm) return 0; } +int +tls_async_handshake_event_handler (void *async_evt, void *unused) +{ + openssl_evt_t *event = (openssl_evt_t *) async_evt; + int thread_index = event->thread_idx; + openssl_ctx_t *oc; + tls_ctx_t *ctx; + int rv, err; + + ctx = openssl_ctx_get_w_thread (event->ctx_index, thread_index); + oc = (openssl_ctx_t *) ctx; + session_t *tls_session = session_get_from_handle (ctx->tls_session_handle); + + if (!SSL_in_init (oc->ssl)) + { + TLS_DBG (8, "[!SSL_in_init]==>CTX: %p EVT: %p EIDX: %d", ctx, event, + event->event_idx); + return 0; + } + + if (ctx->resume) + ctx->resume = 0; + else if (!svm_fifo_max_dequeue_cons (tls_session->rx_fifo)) + return 0; + + rv = SSL_do_handshake (oc->ssl); + err = SSL_get_error (oc->ssl, rv); + + /* Do not remove session from tail */ + if (err == SSL_ERROR_WANT_ASYNC) + return 0; + + if (err == SSL_ERROR_SSL) + { + char buf[512]; + ERR_error_string (ERR_get_error (), buf); + TLS_DBG (2, "[SSL_ERROR_SSL]==>CTX: %p EVT: %p EIDX: %d Buf: %s", ctx, + event, event->event_idx, buf); + openssl_handle_handshake_failure (ctx); + return 0; + } + + if (err == SSL_ERROR_WANT_WRITE || err == SSL_ERROR_WANT_READ) + return 0; + + /* client not supported */ + if (!SSL_is_server (oc->ssl)) + return 0; + + /* Need to check transport status */ + if (ctx->is_passive_close) + { + openssl_handle_handshake_failure (ctx); + return 0; + } + + if (tls_notify_app_accept (ctx)) + { + ctx->c_s_index = SESSION_INVALID_INDEX; + tls_disconnect_transport (ctx); + } + + TLS_DBG (1, + "<=====Handshake for %u complete. TLS cipher is %s EVT: %p =====>", + oc->openssl_ctx_index, SSL_get_cipher (oc->ssl), event); + + return 1; +} + +int +tls_async_read_event_handler (void *async_evt, void *unused) +{ + openssl_evt_t *event = (openssl_evt_t *) async_evt; + int thread_index = event->thread_idx; + session_t *app_session, *tls_session; + openssl_ctx_t *oc; + tls_ctx_t *ctx; + SSL *ssl; + + ctx = openssl_ctx_get_w_thread (event->ctx_index, thread_index); + oc = (openssl_ctx_t *) ctx; + ssl = oc->ssl; + + ctx->in_async_read = true; + /* read event */ + svm_fifo_t *app_rx_fifo, *tls_rx_fifo; + int read, err; + + app_session = session_get_from_handle (ctx->app_session_handle); + app_rx_fifo = app_session->rx_fifo; + + tls_session = session_get_from_handle (ctx->tls_session_handle); + tls_rx_fifo = tls_session->rx_fifo; + + /* continue the paused job */ + read = openssl_async_read_from_ssl_into_fifo (app_rx_fifo, ssl); + err = SSL_get_error (ssl, read); + + /* Unrecoverable protocol error. Reset connection */ + if (PREDICT_FALSE (read <= 0)) + { + if (err == SSL_ERROR_SSL) + tls_notify_app_io_error (ctx); + + if (SSL_want_async (ssl)) + return 0; + + goto ev_rd_done; + } + + /* + * Managed to read some data. If handshake just completed, session + * may still be in accepting state. + */ + if (app_session->session_state >= SESSION_STATE_READY) + tls_notify_app_enqueue (ctx, app_session); + +ev_rd_done: + /* read done */ + ctx->in_async_read = false; + + if ((SSL_pending (ssl) > 0) || svm_fifo_max_dequeue_cons (tls_rx_fifo)) + tls_add_vpp_q_builtin_rx_evt (tls_session); + + return 1; +} + +int +tls_async_write_event_handler (void *async_evt, void *unused) +{ + openssl_evt_t *event = (openssl_evt_t *) async_evt; + int thread_index = event->thread_idx; + session_t *app_session, *tls_session; + openssl_ctx_t *oc; + tls_ctx_t *ctx; + SSL *ssl; + + ctx = openssl_ctx_get_w_thread (event->ctx_index, thread_index); + oc = (openssl_ctx_t *) ctx; + ssl = oc->ssl; + + /* write event */ + int wrote = 0; + u32 space, enq_buf; + svm_fifo_t *app_tx_fifo, *tls_tx_fifo; + transport_send_params_t *sp = event->tran_sp; + + app_session = session_get_from_handle (ctx->app_session_handle); + app_tx_fifo = app_session->tx_fifo; + + /* continue the paused job */ + wrote = openssl_async_write_from_fifo_into_ssl (app_tx_fifo, ssl, oc); + if (!wrote) + { + /* paused job not ready, wait */ + if (SSL_want_async (ssl)) + return 0; + } + + /* Unrecoverable protocol error. Reset connection */ + if (PREDICT_FALSE (wrote < 0)) + { + tls_notify_app_io_error (ctx); + return 0; + } + + tls_session = session_get_from_handle (ctx->tls_session_handle); + tls_tx_fifo = tls_session->tx_fifo; + + /* prepare for remaining write(s) */ + space = svm_fifo_max_enqueue_prod (tls_tx_fifo); + /* Leave a bit of extra space for tls ctrl data, if any needed */ + space = clib_max ((int) space - TLSO_CTRL_BYTES, 0); + + if (svm_fifo_needs_deq_ntf (app_tx_fifo, wrote)) + session_dequeue_notify (app_session); + + /* we got here, async write is done or not possible */ + oc->total_async_write = 0; + + if (PREDICT_FALSE (ctx->app_closed && BIO_ctrl_pending (oc->rbio) <= 0)) + openssl_confirm_app_close (ctx); + + /* Deschedule and wait for deq notification if fifo is almost full */ + enq_buf = clib_min (svm_fifo_size (tls_tx_fifo) / 2, TLSO_MIN_ENQ_SPACE); + if (space < wrote + enq_buf) + { + svm_fifo_add_want_deq_ntf (tls_tx_fifo, SVM_FIFO_WANT_DEQ_NOTIF); + transport_connection_deschedule (&ctx->connection); + sp->flags |= TRANSPORT_SND_F_DESCHED; + } + else + { + /* Request tx reschedule of the app session */ + app_session->flags |= SESSION_F_CUSTOM_TX; + transport_connection_reschedule (&ctx->connection); + } + + return 1; +} + static uword tls_async_process (vlib_main_t * vm, vlib_node_runtime_t * rt, vlib_frame_t * f) @@ -1070,8 +875,8 @@ tls_async_process (vlib_main_t * vm, vlib_node_runtime_t * rt, thread_index = vlib_get_thread_index (); if (pool_elts (om->evt_pool[thread_index]) > 0) { - tls_async_dequeue_event_in_init (thread_index); - tls_async_dequeue_event (thread_index); + openssl_async_polling (); + tls_resume_from_crypto (thread_index); } return 0; diff --git a/src/plugins/tlsopenssl/tls_openssl.c b/src/plugins/tlsopenssl/tls_openssl.c index 44100023a0..9d80b9e6c6 100644 --- a/src/plugins/tlsopenssl/tls_openssl.c +++ b/src/plugins/tlsopenssl/tls_openssl.c @@ -76,11 +76,12 @@ openssl_ctx_free (tls_ctx_t * ctx) if (openssl_main.async) { - openssl_evt_free (ctx->evt_index[SSL_ASYNC_EVT_INIT], + TLS_DBG (2, "FREEING_ASYNC_EVTS: CTX: %p", ctx); + openssl_evt_free (oc->evt_index[SSL_ASYNC_EVT_INIT], ctx->c_thread_index); - openssl_evt_free (ctx->evt_index[SSL_ASYNC_EVT_RD], + openssl_evt_free (oc->evt_index[SSL_ASYNC_EVT_RD], ctx->c_thread_index); - openssl_evt_free (ctx->evt_index[SSL_ASYNC_EVT_WR], + openssl_evt_free (oc->evt_index[SSL_ASYNC_EVT_WR], ctx->c_thread_index); } } @@ -96,6 +97,7 @@ openssl_ctx_detach (tls_ctx_t *ctx) oc_copy = clib_mem_alloc (sizeof (*oc_copy)); clib_memcpy (oc_copy, oc, sizeof (*oc)); + TLS_DBG (2, "CTX: %p detached"); return oc_copy; } @@ -119,6 +121,7 @@ openssl_ctx_attach (u32 thread_index, void *ctx_ptr) sh = (*oc)->ctx.tls_session_handle; BIO_set_data ((*oc)->rbio, uword_to_pointer (sh, void *)); BIO_set_data ((*oc)->wbio, uword_to_pointer (sh, void *)); + TLS_DBG (2, "OSSL_CTX: %p attached", *oc); return ((*oc)->openssl_ctx_index); } @@ -185,6 +188,7 @@ openssl_read_from_ssl_into_fifo (svm_fifo_t *f, SSL *ssl) read = SSL_read (ssl, fs[0].data, fs[0].len); if (read <= 0) { + TLS_DBG (2, "FATAL_ERROR"); ossl_check_err_is_fatal (ssl, read); return read; } @@ -204,6 +208,7 @@ openssl_read_from_ssl_into_fifo (svm_fifo_t *f, SSL *ssl) } } svm_fifo_enqueue_nocopy (f, read); + TLS_DBG (2, "READ_ENQ_NOCOPY: %d", read); return read; } @@ -237,54 +242,22 @@ openssl_write_from_fifo_into_ssl (svm_fifo_t *f, SSL *ssl, u32 max_len) return wrote; } -#ifdef HAVE_OPENSSL_ASYNC -static int -openssl_check_async_status (tls_ctx_t * ctx, openssl_resume_handler * handler, - session_t * session) -{ - openssl_ctx_t *oc = (openssl_ctx_t *) ctx; - int estatus; - - SSL_get_async_status (oc->ssl, &estatus); - if (estatus == ASYNC_STATUS_EAGAIN) - { - vpp_tls_async_update_event (ctx, 1, SSL_ASYNC_EVT_INIT); - vpp_tls_async_update_event (ctx, 1, SSL_ASYNC_EVT_RD); - vpp_tls_async_update_event (ctx, 1, SSL_ASYNC_EVT_WR); - } - else - { - vpp_tls_async_update_event (ctx, 0, SSL_ASYNC_EVT_INIT); - vpp_tls_async_update_event (ctx, 0, SSL_ASYNC_EVT_RD); - vpp_tls_async_update_event (ctx, 0, SSL_ASYNC_EVT_WR); - } - - return 1; - -} - -#endif - -static int +int openssl_handle_want_async (tls_ctx_t *ctx, int evt_type, transport_send_params_t *sp, int size) { - int ret; - - if (evt_type >= SSL_ASYNC_EVT_MAX || evt_type == 0) - { - ERRORPR ("\\- return 0 [illegal evt_type value:%d]\n", evt_type); - return 0; - } - + openssl_ctx_t *oc = (openssl_ctx_t *) ctx; + int ret = 0; if (evt_type == SSL_ASYNC_EVT_WR) { /* de-schedule transport connection */ transport_connection_deschedule (&ctx->connection); sp->flags |= TRANSPORT_SND_F_DESCHED; - ctx->total_write = size; + TLS_DBG (2, "WR_EVT_TOTAL_WRITE: %d", size); + TLS_DBG (2, "CTX_TOTAL_WRITE: %d", oc->total_async_write); + oc->total_async_write = size; } - ret = vpp_tls_async_enqueue_event (ctx, evt_type, sp, size); + TLS_DBG (2, "WANT_ASYNC_EVT_TYPE_%d", evt_type); return ret; } @@ -303,6 +276,7 @@ openssl_handle_handshake_failure (tls_ctx_t *ctx) session_get_if_valid (ctx->c_s_index, ctx->c_thread_index); if (app_session) { + TLS_DBG (2, "FREEING_APP_SESSION: %p", app_session); session_free (app_session); ctx->c_s_index = SESSION_INVALID_INDEX; tls_disconnect_transport (ctx); @@ -314,14 +288,17 @@ openssl_handle_handshake_failure (tls_ctx_t *ctx) /* * Also handles cleanup of the pre-allocated session */ + TLS_DBG (2, "NOTIFY_APP_DISCONNECTED: %p", ctx); tls_notify_app_connected (ctx, SESSION_E_TLS_HANDSHAKE); tls_disconnect_transport (ctx); } } int -openssl_ctx_handshake_rx (tls_ctx_t * ctx, session_t * tls_session) +openssl_ctx_handshake_rx (void *hs_ctx, void *hs_tls_session) { + tls_ctx_t *ctx = (tls_ctx_t *) hs_ctx; + session_t *tls_session = (session_t *) hs_tls_session; openssl_ctx_t *oc = (openssl_ctx_t *) ctx; int rv = 0, err; @@ -339,7 +316,9 @@ openssl_ctx_handshake_rx (tls_ctx_t * ctx, session_t * tls_session) if (openssl_main.async && err == SSL_ERROR_WANT_ASYNC) { - openssl_handle_want_async (ctx, SSL_ASYNC_EVT_INIT, NULL, 0); + TLS_DBG (2, "WANT_ASYNC_HS_RX: CTX: %p", ctx); + vpp_tls_async_init_event (ctx, tls_async_handshake_event_handler, + NULL, tls_session, SSL_ASYNC_EVT_INIT, 0); return -1; } @@ -347,8 +326,7 @@ openssl_ctx_handshake_rx (tls_ctx_t * ctx, session_t * tls_session) { char buf[512]; ERR_error_string (ERR_get_error (), buf); - clib_warning ("Err: %s", buf); - ERRORPR ("Err: %s\n", buf); + TLS_DBG (2, "Err: %s", buf); openssl_handle_handshake_failure (ctx); return -1; } @@ -424,7 +402,10 @@ openssl_confirm_app_close (tls_ctx_t *ctx) /* Shrink FIFOs */ if (app_session) - session_shrink_fifos (app_session); + { + TLS_DBG (2, "SHRINK_FIFOS"); + session_shrink_fifos (app_session); + } } static int @@ -460,9 +441,12 @@ openssl_ctx_write_tls (tls_ctx_t *ctx, session_t *app_session, wrote = openssl_write_from_fifo_into_ssl (f, oc->ssl, deq_max); - /* Unrecoverable protocol error. Reset connection */ if (PREDICT_FALSE (wrote < 0)) { + int err = SSL_get_error (oc->ssl, wrote); + if (err == SSL_ERROR_WANT_WRITE) + return 0; + /* Unrecoverable protocol error. Reset connection */ tls_notify_app_io_error (ctx); return 0; } @@ -471,7 +455,8 @@ openssl_ctx_write_tls (tls_ctx_t *ctx, session_t *app_session, { if (openssl_main.async && SSL_want_async (oc->ssl)) { - openssl_handle_want_async (ctx, SSL_ASYNC_EVT_WR, sp, deq_max); + vpp_tls_async_init_event (ctx, tls_async_write_event_handler, sp, ts, + SSL_ASYNC_EVT_WR, deq_max); return 0; } goto check_tls_fifo; @@ -581,6 +566,7 @@ openssl_ctx_read_tls (tls_ctx_t *ctx, session_t *tls_session) if (PREDICT_FALSE (SSL_in_init (oc->ssl))) { + TLS_DBG (2, "SSL_INIT_FAILED"); if (openssl_ctx_handshake_rx (ctx, tls_session) < 0) return 0; @@ -589,18 +575,23 @@ openssl_ctx_read_tls (tls_ctx_t *ctx, session_t *tls_session) } if (ctx->in_async_read) - return 0; + { + TLS_DBG (2, "ASYNC_READ_ALREADY_SET"); + return 0; + } app_session = session_get_from_handle (ctx->app_session_handle); f = app_session->rx_fifo; read = openssl_read_from_ssl_into_fifo (f, oc->ssl); - if (read < 0) + if (read <= 0) { if (openssl_main.async && SSL_want_async (oc->ssl)) { ctx->in_async_read = true; - openssl_handle_want_async (ctx, SSL_ASYNC_EVT_RD, NULL, 0); + TLS_DBG (2, "WANT_ASYNC_FOR_READ"); + vpp_tls_async_init_event (ctx, tls_async_read_event_handler, NULL, + tls_session, SSL_ASYNC_EVT_RD, 0); return 0; } } @@ -609,13 +600,17 @@ openssl_ctx_read_tls (tls_ctx_t *ctx, session_t *tls_session) if (PREDICT_FALSE ((read < 0) && (SSL_get_error (oc->ssl, read) == SSL_ERROR_SSL))) { + TLS_DBG (2, "APP_IO_ERR"); tls_notify_app_io_error (ctx); return 0; } /* If handshake just completed, session may still be in accepting state */ if (read > 0 && app_session->session_state >= SESSION_STATE_READY) - tls_notify_app_enqueue (ctx, app_session); + { + TLS_DBG (2, "NOTIFY_APP_ENQ"); + tls_notify_app_enqueue (ctx, app_session); + } if ((SSL_pending (oc->ssl) > 0) || svm_fifo_max_dequeue_cons (tls_session->rx_fifo)) @@ -873,7 +868,7 @@ openssl_ctx_init_client (tls_ctx_t * ctx) #ifdef HAVE_OPENSSL_ASYNC session_t *tls_session = session_get_from_handle (ctx->tls_session_handle); - vpp_tls_async_init_events (ctx, openssl_ctx_handshake_rx, tls_session); + openssl_ctx_handshake_rx (ctx, tls_session); #endif while (1) { @@ -882,8 +877,7 @@ openssl_ctx_init_client (tls_ctx_t * ctx) #ifdef HAVE_OPENSSL_ASYNC if (err == SSL_ERROR_WANT_ASYNC) { - openssl_check_async_status (ctx, openssl_ctx_handshake_rx, - tls_session); + TLS_DBG (2, "WANT_ASYNC"); break; } #endif @@ -1108,9 +1102,12 @@ openssl_ctx_init_server (tls_ctx_t * ctx) TLS_DBG (1, "Initiating handshake for [%u]%u", ctx->c_thread_index, oc->openssl_ctx_index); - session_t *tls_session = session_get_from_handle (ctx->tls_session_handle); if (openssl_main.async) - vpp_tls_async_init_events (ctx, openssl_ctx_handshake_rx, tls_session); + { + session_t *tls_session = + session_get_from_handle (ctx->tls_session_handle); + openssl_ctx_handshake_rx (ctx, tls_session); + } while (1) { @@ -1118,15 +1115,14 @@ openssl_ctx_init_server (tls_ctx_t * ctx) err = SSL_get_error (oc->ssl, rv); if (openssl_main.async && err == SSL_ERROR_WANT_ASYNC) { - openssl_handle_want_async (ctx, SSL_ASYNC_EVT_INIT, NULL, 0); - + TLS_DBG (2, "WANT_ASYNC"); break; } if (err != SSL_ERROR_WANT_WRITE) break; } - TLS_DBG (2, "tls state for [%u]%u is su", ctx->c_thread_index, + TLS_DBG (2, "tls state for [%u]%u is %s", ctx->c_thread_index, oc->openssl_ctx_index, SSL_state_string_long (oc->ssl)); return 0; } @@ -1137,6 +1133,7 @@ openssl_handshake_is_over (tls_ctx_t * ctx) openssl_ctx_t *mc = (openssl_ctx_t *) ctx; if (!mc->ssl) return 0; + TLS_DBG (2, "HS_FINISHED"); return SSL_is_init_finished (mc->ssl); } @@ -1144,10 +1141,14 @@ static int openssl_transport_close (tls_ctx_t * ctx) { if (openssl_main.async && vpp_openssl_is_inflight (ctx)) - return 0; + { + TLS_DBG (2, "EVTS_IN_FLIGHT"); + return 0; + } if (!openssl_handshake_is_over (ctx)) { + TLS_DBG (2, "HS_NOT_OVER"); openssl_handle_handshake_failure (ctx); return 0; } @@ -1400,26 +1401,24 @@ tls_openssl_set_tls_fn (vlib_main_t *vm, unformat_input_t *input, vlib_cli_command_t *cmd) { openssl_main_t *om = &openssl_main; - u32 tmp; while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) { - if (unformat (input, "record-size %U", unformat_memory_size, &tmp)) + if (unformat (input, "record-size %U", unformat_memory_size, + &om->record_size)) { - clib_warning ("Using TLS record-size of %d", tmp); - om->record_size = tmp; + clib_warning ("Using TLS record-size of %d", om->record_size); } else if (unformat (input, "record-split-size %U", unformat_memory_size, - &tmp)) + &om->record_split_size)) { - clib_warning ("Using TLS record-split-size of %d", tmp); - om->record_split_size = tmp; + clib_warning ("Using TLS record-split-size of %d", + om->record_split_size); } else if (unformat (input, "max-pipelines %U", unformat_memory_size, - &tmp)) + &om->max_pipelines)) { - clib_warning ("Using TLS max-pipelines of %d", tmp); - om->max_pipelines = tmp; + clib_warning ("Using TLS max-pipelines of %d", om->max_pipelines); } else return clib_error_return (0, "failed: unknown input `%U'", diff --git a/src/plugins/tlsopenssl/tls_openssl.h b/src/plugins/tlsopenssl/tls_openssl.h index 090cc33df6..260e3cf76d 100644 --- a/src/plugins/tlsopenssl/tls_openssl.h +++ b/src/plugins/tlsopenssl/tls_openssl.h @@ -56,6 +56,9 @@ typedef struct tls_ctx_openssl_ tls_ctx_t ctx; /**< First */ u32 openssl_ctx_index; SSL_CTX *client_ssl_ctx; + u32 evt_index[SSL_ASYNC_EVT_MAX]; + bool evt_alloc_flag[SSL_ASYNC_EVT_MAX]; + u32 total_async_write; SSL *ssl; BIO *rbio; BIO *wbio; @@ -90,15 +93,13 @@ typedef struct openssl_main_ u32 max_pipelines; } openssl_main_t; -typedef int openssl_resume_handler (tls_ctx_t * ctx, session_t * tls_session); +typedef int openssl_resume_handler (void *event, void *session); +typedef int (*async_handlers) (void *event, void *session); tls_ctx_t *openssl_ctx_get_w_thread (u32 ctx_index, u8 thread_index); -int vpp_tls_async_init_events (tls_ctx_t *ctx, openssl_resume_handler *handler, - session_t *session); -int vpp_tls_async_update_event (tls_ctx_t *ctx, int eagain, - ssl_async_evt_type_t type); -int vpp_tls_async_enqueue_event (tls_ctx_t *ctx, int evt_type, - transport_send_params_t *sp, int size); +int vpp_tls_async_init_event (tls_ctx_t *ctx, openssl_resume_handler *handler, + transport_send_params_t *sp, session_t *session, + ssl_async_evt_type_t evt_type, int wr_size); int tls_async_openssl_callback (SSL * s, void *evt); int openssl_evt_free (int event_idx, u8 thread_index); void openssl_polling_start (ENGINE * engine); @@ -111,6 +112,9 @@ int openssl_read_from_ssl_into_fifo (svm_fifo_t *f, SSL *ssl); void openssl_handle_handshake_failure (tls_ctx_t *ctx); void openssl_confirm_app_close (tls_ctx_t *ctx); +int tls_async_write_event_handler (void *event, void *session); +int tls_async_read_event_handler (void *event, void *session); +int tls_async_handshake_event_handler (void *event, void *session); #endif /* SRC_PLUGINS_TLSOPENSSL_TLS_OPENSSL_H_ */ /* diff --git a/src/vnet/tls/tls.c b/src/vnet/tls/tls.c index 86c6ac76ed..b52413cd30 100644 --- a/src/vnet/tls/tls.c +++ b/src/vnet/tls/tls.c @@ -695,8 +695,12 @@ tls_app_session_cleanup (session_t * s, session_cleanup_ntf_t ntf) } ctx = tls_ctx_get (s->opaque); + TLS_DBG (1, "SESSION_CLEANUP_CTX: %p", ctx); if (!ctx->no_app_session) - session_transport_delete_notify (&ctx->connection); + { + session_transport_delete_notify (&ctx->connection); + TLS_DBG (1, "NO_APP_SESSION_CLEANUP_CTX: %p", ctx); + } tls_ctx_free (ctx); } diff --git a/src/vnet/tls/tls.h b/src/vnet/tls/tls.h index 741d775b9a..3d3e12fbca 100644 --- a/src/vnet/tls/tls.h +++ b/src/vnet/tls/tls.h @@ -70,7 +70,10 @@ typedef struct tls_cxt_id_ STATIC_ASSERT (sizeof (tls_ctx_id_t) <= TRANSPORT_CONN_ID_LEN, "ctx id must be less than TRANSPORT_CONN_ID_LEN"); -#define foreach_tls_conn_flags _ (HO_DONE, "ho done") +#define foreach_tls_conn_flags \ + _ (HO_DONE, "ho-done") \ + _ (ASYNC_WR, "async-write") \ + _ (ASYNC_RD, "async-read") typedef enum tls_conn_flags_bit_ { @@ -113,8 +116,6 @@ typedef struct tls_ctx_ u8 is_migrated; tls_conn_flags_t flags; u8 *srv_hostname; - u32 evt_index[SSL_ASYNC_EVT_MAX]; - u32 total_write; u32 ckpair_index; transport_proto_t tls_type; bool in_async_read; From 9176e35f8b07ee47120932289f802a7a7d593283 Mon Sep 17 00:00:00 2001 From: Nithinsen Kaithakadan Date: Thu, 16 Jan 2025 18:09:10 +0530 Subject: [PATCH 182/271] octeon: fix crypto failures after session delete This patch fix the crypto failure that occurs when adding a Security Association (SA) with a different algorithm but the same ID after an SA deletion. The issue was caused by reusing the context cache, which has now been fixed by flushing and invalidating the cache during the SA deletion process. Type: fix Signed-off-by: Nithinsen Kaithakadan Change-Id: I2099c0f98cab3787f8581b4c81e305a9b517d8ab Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/144731 Tested-by: sa_ip-sw-jenkins Reviewed-by: Nithinsen Kaithakadan --- src/plugins/dev_octeon/crypto.c | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/plugins/dev_octeon/crypto.c b/src/plugins/dev_octeon/crypto.c index 3048405842..d863cbaa48 100644 --- a/src/plugins/dev_octeon/crypto.c +++ b/src/plugins/dev_octeon/crypto.c @@ -62,6 +62,9 @@ oct_crypto_session_create (vlib_main_t *vm, vnet_crypto_key_index_t key_index, oct_crypto_sess_t *session; vnet_crypto_key_t *key; oct_crypto_key_t *ckey; + oct_crypto_dev_t *ocd; + + ocd = ocm->crypto_dev[op_type]; key = vnet_crypto_get_key (key_index); @@ -88,6 +91,7 @@ oct_crypto_session_create (vlib_main_t *vm, vnet_crypto_key_index_t key_index, session = oct_crypto_session_alloc (vm, op_type); if (session == NULL) return -1; + session->crypto_dev = ocd; } oct_map_keyindex_to_session (session, key_index, op_type); @@ -116,6 +120,12 @@ oct_crypto_key_del_handler (vlib_main_t *vm, vnet_crypto_key_index_t key_index) ocm->keys[VNET_CRYPTO_OP_TYPE_ENCRYPT], ckey->sess->key_index); ckey_linked->sess = NULL; } + + /* Trigger CTX flush + invalidate to remove from CTX_CACHE */ + if (oct_hw_ctx_cache_enable ()) + roc_cpt_lf_ctx_flush (&ckey->sess->crypto_dev->lf, + &ckey->sess->cpt_ctx.se_ctx, true); + oct_plt_init_param.oct_plt_free (ckey->sess); ckey->sess = NULL; } @@ -132,6 +142,12 @@ oct_crypto_key_del_handler (vlib_main_t *vm, vnet_crypto_key_index_t key_index) ocm->keys[VNET_CRYPTO_OP_TYPE_DECRYPT], ckey->sess->key_index); ckey_linked->sess = NULL; } + + /* Trigger CTX flush + invalidate to remove from CTX_CACHE */ + if (oct_hw_ctx_cache_enable ()) + roc_cpt_lf_ctx_flush (&ckey->sess->crypto_dev->lf, + &ckey->sess->cpt_ctx.se_ctx, true); + oct_plt_init_param.oct_plt_free (ckey->sess); ckey->sess = NULL; } From bd9b96937ee4baee8320982b0250f6e57bf2aca2 Mon Sep 17 00:00:00 2001 From: Nithinsen Kaithakadan Date: Thu, 23 Jan 2025 10:34:52 +0530 Subject: [PATCH 183/271] octeon: set cpt descriptor count to 16k Set the number of CPT descriptors to 16384. Add an option to configure cpt descriptor count from startup conf. Type: feature Signed-off-by: Nithinsen Kaithakadan Change-Id: Id1ffbe69ad7d457dbd0b11494be1dc0d4a28c324 Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/144732 Tested-by: sa_ip-sw-jenkins Reviewed-by: Nithinsen Kaithakadan --- src/plugins/dev_octeon/crypto.c | 4 ++-- src/plugins/dev_octeon/crypto.h | 7 ++++++- src/plugins/dev_octeon/init.c | 37 +++++++++++++++++++++++++++++++-- src/plugins/dev_octeon/octeon.h | 5 +++++ 4 files changed, 48 insertions(+), 5 deletions(-) diff --git a/src/plugins/dev_octeon/crypto.c b/src/plugins/dev_octeon/crypto.c index d863cbaa48..3847d2e48a 100644 --- a/src/plugins/dev_octeon/crypto.c +++ b/src/plugins/dev_octeon/crypto.c @@ -1936,7 +1936,7 @@ oct_init_crypto_engine_handlers (vlib_main_t *vm, vnet_dev_t *dev) } int -oct_conf_sw_queue (vlib_main_t *vm, vnet_dev_t *dev) +oct_conf_sw_queue (vlib_main_t *vm, vnet_dev_t *dev, oct_crypto_dev_t *ocd) { vlib_thread_main_t *tm = vlib_get_thread_main (); extern oct_plt_init_param_t oct_plt_init_param; @@ -1957,7 +1957,7 @@ oct_conf_sw_queue (vlib_main_t *vm, vnet_dev_t *dev) * Each pending queue will get number of cpt desc / number of cores. * And that desc count is shared across inflight entries. */ - n_inflight_req = (OCT_CPT_LF_MAX_NB_DESC / tm->n_vlib_mains); + n_inflight_req = (ocd->n_desc / tm->n_vlib_mains); for (i = 0; i < tm->n_vlib_mains; ++i) { diff --git a/src/plugins/dev_octeon/crypto.h b/src/plugins/dev_octeon/crypto.h index 2224bc1cd1..a436f99bbe 100644 --- a/src/plugins/dev_octeon/crypto.h +++ b/src/plugins/dev_octeon/crypto.h @@ -11,6 +11,9 @@ #define OCT_MAX_N_CPT_DEV 2 +#define OCT_CPT_LF_DEF_NB_DESC 16384 + +#define OCT_CPT_LF_MIN_NB_DESC 8192 #define OCT_CPT_LF_MAX_NB_DESC 128000 #define OCT_MAX_CRYPTO_COUNTERS 3 @@ -89,6 +92,7 @@ typedef struct struct roc_cpt_lmtline lmtline; struct roc_cpt_lf lf; vnet_dev_t *dev; + u32 n_desc; } oct_crypto_dev_t; typedef struct @@ -227,5 +231,6 @@ vnet_crypto_async_frame_t *oct_crypto_frame_dequeue (vlib_main_t *vm, u32 *nb_elts_processed, u32 *enqueue_thread_idx); int oct_init_crypto_engine_handlers (vlib_main_t *vm, vnet_dev_t *dev); -int oct_conf_sw_queue (vlib_main_t *vm, vnet_dev_t *dev); +int oct_conf_sw_queue (vlib_main_t *vm, vnet_dev_t *dev, + oct_crypto_dev_t *ocd); #endif /* _CRYPTO_H_ */ diff --git a/src/plugins/dev_octeon/init.c b/src/plugins/dev_octeon/init.c index f18e27af59..d3e036eb96 100644 --- a/src/plugins/dev_octeon/init.c +++ b/src/plugins/dev_octeon/init.c @@ -116,6 +116,16 @@ static vnet_dev_arg_t oct_port_args[] = { }, }; +static vnet_dev_arg_t oct_dev_args[] = { + { + .id = OCT_DEV_ARG_CRYPTO_N_DESC, + .name = "n_desc", + .desc = "number of cpt descriptors", + .type = VNET_DEV_ARG_TYPE_UINT32, + .default_val.uint32 = 16384, + }, +}; + clib_error_t * oct_inl_inb_ipsec_flow_enable (void) { @@ -349,7 +359,7 @@ oct_conf_cpt_queue (vlib_main_t *vm, vnet_dev_t *dev, oct_crypto_dev_t *ocd) cpt_lf = &ocd->lf; cpt_lmtline = &ocd->lmtline; - cpt_lf->nb_desc = OCT_CPT_LF_MAX_NB_DESC; + cpt_lf->nb_desc = ocd->n_desc; cpt_lf->lf_id = 0; if ((rrv = roc_cpt_lf_init (roc_cpt, cpt_lf)) < 0) return cnx_return_roc_err (dev, rrv, "roc_cpt_lf_init"); @@ -403,6 +413,7 @@ oct_init_cpt (vlib_main_t *vm, vnet_dev_t *dev) extern oct_plt_init_param_t oct_plt_init_param; oct_device_t *cd = vnet_dev_get_data (dev); oct_crypto_dev_t *ocd = NULL; + u32 n_desc; int rrv; if (ocm->n_cpt == OCT_MAX_N_CPT_DEV || ocm->started) @@ -416,6 +427,27 @@ oct_init_cpt (vlib_main_t *vm, vnet_dev_t *dev) ocd->roc_cpt->pci_dev = &cd->plt_pci_dev; ocd->dev = dev; + ocd->n_desc = OCT_CPT_LF_DEF_NB_DESC; + + foreach_vnet_dev_args (arg, dev) + { + if (arg->id == OCT_DEV_ARG_CRYPTO_N_DESC && + vnet_dev_arg_get_uint32 (arg)) + { + n_desc = vnet_dev_arg_get_uint32 (arg); + if (n_desc < OCT_CPT_LF_MIN_NB_DESC || + n_desc > OCT_CPT_LF_MAX_NB_DESC) + { + log_err (dev, + "number of cpt descriptors should be within range " + "of %u and %u", + OCT_CPT_LF_MIN_NB_DESC, OCT_CPT_LF_MAX_NB_DESC); + return VNET_DEV_ERR_NOT_SUPPORTED; + } + + ocd->n_desc = vnet_dev_arg_get_uint32 (arg); + } + } if ((rrv = roc_cpt_dev_init (ocd->roc_cpt))) return cnx_return_roc_err (dev, rrv, "roc_cpt_dev_init"); @@ -432,7 +464,7 @@ oct_init_cpt (vlib_main_t *vm, vnet_dev_t *dev) * Initialize s/w queues, which are common across multiple * crypto devices */ - oct_conf_sw_queue (vm, dev); + oct_conf_sw_queue (vm, dev, ocd); ocm->crypto_dev[0] = ocd; /* Initialize counters */ @@ -563,6 +595,7 @@ VNET_DEV_REGISTER_DRIVER (octeon) = { .free = oct_free, .probe = oct_probe, }, + .args = oct_dev_args, }; static int diff --git a/src/plugins/dev_octeon/octeon.h b/src/plugins/dev_octeon/octeon.h index 17042b740f..82e2e47eea 100644 --- a/src/plugins/dev_octeon/octeon.h +++ b/src/plugins/dev_octeon/octeon.h @@ -42,6 +42,11 @@ typedef enum OCT_PORT_ARG_END } oct_port_args_t; +typedef enum +{ + OCT_DEV_ARG_CRYPTO_N_DESC = 1, +} oct_dev_args_t; + typedef enum { OCT_DEVICE_TYPE_UNKNOWN = 0, From e5b8c442ef79d58d7ff9917e5ade385a30f44188 Mon Sep 17 00:00:00 2001 From: Nithinsen Kaithakadan Date: Wed, 5 Feb 2025 18:53:58 +0530 Subject: [PATCH 184/271] octeon: fix octeon device args list Added a fix for the crash caused by the addition of the device arguments list. The solution involves adding VNET_DEV_ARG_END to the newly introduced dev argument list. Type: fix JIRA: https://essjira.marvell.com/browse/IPBUSW-51987 Signed-off-by: Nithinsen Kaithakadan Change-Id: I9d0113c28f9e3ddc07ef0aff463bc07414018219 Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/145039 Tested-by: sa_ip-sw-jenkins Reviewed-by: Nithinsen Kaithakadan --- src/plugins/dev_octeon/init.c | 14 ++++++++++++-- src/plugins/dev_octeon/octeon.h | 1 + 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/src/plugins/dev_octeon/init.c b/src/plugins/dev_octeon/init.c index d3e036eb96..b0b3a517de 100644 --- a/src/plugins/dev_octeon/init.c +++ b/src/plugins/dev_octeon/init.c @@ -120,9 +120,15 @@ static vnet_dev_arg_t oct_dev_args[] = { { .id = OCT_DEV_ARG_CRYPTO_N_DESC, .name = "n_desc", - .desc = "number of cpt descriptors", + .desc = "number of cpt descriptors, applicable to cpt devices only", .type = VNET_DEV_ARG_TYPE_UINT32, - .default_val.uint32 = 16384, + .default_val.uint32 = OCT_CPT_LF_DEF_NB_DESC, + }, + { + .id = OCT_DEV_ARG_END, + .name = "end", + .desc = "Argument end", + .type = VNET_DEV_ARG_END, }, }; @@ -215,6 +221,10 @@ oct_init_nix (vlib_main_t *vm, vnet_dev_t *dev) .hw_vlan_ins = true, }; + if (roc_feature_nix_has_own_meta_aura () && + !roc_feature_nix_has_second_pass_drop ()) + cd->nix->local_meta_aura_ena = true; + if ((rrv = roc_nix_dev_init (cd->nix))) return cnx_return_roc_err (dev, rrv, "roc_nix_dev_init"); diff --git a/src/plugins/dev_octeon/octeon.h b/src/plugins/dev_octeon/octeon.h index 82e2e47eea..e24cbcaf6d 100644 --- a/src/plugins/dev_octeon/octeon.h +++ b/src/plugins/dev_octeon/octeon.h @@ -45,6 +45,7 @@ typedef enum typedef enum { OCT_DEV_ARG_CRYPTO_N_DESC = 1, + OCT_DEV_ARG_END, } oct_dev_args_t; typedef enum From ce5723cb0c83c69dcab546bebe9d41af5a89e40b Mon Sep 17 00:00:00 2001 From: Sunil Kumar Kori Date: Wed, 12 Feb 2025 22:38:06 +0530 Subject: [PATCH 185/271] octeon: fix device BAR4 mapping As per SDK12.25.02, Only BAR2 is exposed for VF devices on cn10k platform. Hence fixed the same for cn10k. Type: fix JIRA: https://essjira.marvell.com/browse/IPBUSW-60666 JIRA: https://essjira.marvell.com/browse/IPBUSW-60688 Signed-off-by: Sunil Kumar Kori Change-Id: Iba4ad89a4c82d57b80c08fb06473e1e8a603697f Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/145941 Reviewed-by: Nithin Kumar Dabilpuram Tested-by: sa_ip-toolkits-Jenkins (cherry picked from commit 1b0738d6d7306d04c4305c4cc2563f9d2b9fee9b) Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/145952 Tested-by: sa_ip-sw-jenkins Reviewed-by: Devapraba Muthumani --- src/plugins/dev_octeon/init.c | 31 ++++++++++++++++++++++++++++++- src/plugins/dev_octeon/octeon.h | 6 ++++++ 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/src/plugins/dev_octeon/init.c b/src/plugins/dev_octeon/init.c index b0b3a517de..8ea982ccfb 100644 --- a/src/plugins/dev_octeon/init.c +++ b/src/plugins/dev_octeon/init.c @@ -496,6 +496,35 @@ oct_init_cpt (vlib_main_t *vm, vnet_dev_t *dev) return VNET_DEV_OK; } +static bool +oct_is_nix_bar_mappable (vnet_dev_t *dev, u32 bar) +{ + oct_device_t *cd = vnet_dev_get_data (dev); + + /* Device-BARs mapping table + * +-----+-------+-------+--------+ + * | | cn9k | cn10k | cn20k | + * +-----+-------+-------+--------+ + * | PF | BAR2 | BAR2 | BAR2 | + * | | BAR4 | BAR4 | | + * +-----+-------+-------+--------+ + * | VF | BAR2 | BAR2 | BAR2 | + * | | BAR4 | | | + * +-----+-------+-------+--------+ + */ + + if (bar == 2) + return true; + + if (roc_model_is_cn20k ()) + return false; + + if (roc_model_is_cn10k () && OCT_DEVTYPE_IS_VF (cd->type)) + return false; + + return true; +} + static vnet_dev_rv_t oct_init (vlib_main_t *vm, vnet_dev_t *dev) { @@ -531,7 +560,7 @@ oct_init (vlib_main_t *vm, vnet_dev_t *dev) foreach_int (i, 2, 4) { - if (i == 2 || !roc_model_is_cn20k ()) + if (oct_is_nix_bar_mappable (dev, i)) { rv = vnet_dev_pci_map_region (vm, dev, i, &cd->plt_pci_dev.mem_resource[i].addr); diff --git a/src/plugins/dev_octeon/octeon.h b/src/plugins/dev_octeon/octeon.h index e24cbcaf6d..3a1333a8e8 100644 --- a/src/plugins/dev_octeon/octeon.h +++ b/src/plugins/dev_octeon/octeon.h @@ -61,6 +61,12 @@ typedef enum OCT_DEVICE_TYPE_RVU_INL_VF, } __clib_packed oct_device_type_t; +#define OCT_DEVTYPE_IS_VF(type) \ + ((type) == OCT_DEVICE_TYPE_RVU_VF || (type) == OCT_DEVICE_TYPE_LBK_VF || \ + (type) == OCT_DEVICE_TYPE_SDP_VF || \ + (type) == OCT_DEVICE_TYPE_O10K_CPT_VF || \ + (type) == OCT_DEVICE_TYPE_RVU_INL_VF) + typedef struct { /* vnet flow index */ From f52d2cc4ed54c2a1c5bc8be4bdf6efb0dfb367ad Mon Sep 17 00:00:00 2001 From: Monendra Singh Kushwaha Date: Sat, 11 Jan 2025 15:40:16 +0530 Subject: [PATCH 186/271] octeon: add local meta aura for Inline IPsec Type: feature JIRA: https://essjira.marvell.com/browse/IPBUSW-58192 Change-Id: I18d148fbd389af45321372f3cad154bd4dd7c669 Signed-off-by: Monendra Singh Kushwaha Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/145868 Tested-by: sa_ip-sw-jenkins --- src/plugins/dev_octeon/queue.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plugins/dev_octeon/queue.c b/src/plugins/dev_octeon/queue.c index 0aa814ae2d..9fb7517f82 100644 --- a/src/plugins/dev_octeon/queue.c +++ b/src/plugins/dev_octeon/queue.c @@ -160,7 +160,7 @@ oct_rxq_init (vlib_main_t *vm, vnet_dev_rx_queue_t *rxq, u32 total_sz) .flow_tag_width = 32, }; - if ((rrv = roc_nix_rq_init (nix, &crq->rq, 1 /* disable */))) + if ((rrv = roc_nix_rq_init (nix, &crq->rq, false /* disable */))) { oct_rxq_deinit (vm, rxq); return oct_roc_err (dev, rrv, "roc_nix_rq_init(qid = %u) failed", From 641233947288cd35698ca23337e2ac5825e15bce Mon Sep 17 00:00:00 2001 From: Monendra Singh Kushwaha Date: Tue, 11 Feb 2025 14:09:13 +0530 Subject: [PATCH 187/271] octeon: fix multi-seg packet processing Type: fix Change-Id: I65b0219361c4a329bf28204f630044283a98545c Signed-off-by: Monendra Singh Kushwaha Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/145721 Tested-by: sa_ip-sw-jenkins Reviewed-by: Nithin Kumar Dabilpuram (cherry picked from commit c0c5abde02a0a7fa4794239ad9c4500b5589375f) Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/145724 --- src/plugins/dev_octeon/rx_node.c | 50 ++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 21 deletions(-) diff --git a/src/plugins/dev_octeon/rx_node.c b/src/plugins/dev_octeon/rx_node.c index d6d91a6b79..89c21e4d88 100644 --- a/src/plugins/dev_octeon/rx_node.c +++ b/src/plugins/dev_octeon/rx_node.c @@ -33,7 +33,8 @@ oct_seg_to_bp (void *p) } static_always_inline void -oct_rx_attach_tail (vlib_main_t *vm, oct_rx_node_ctx_t *ctx, vlib_buffer_t *h, +oct_rx_attach_tail (vlib_main_t *vm, oct_rx_node_ctx_t *ctx, + vlib_buffer_template_t *bt, vlib_buffer_t *h, oct_nix_rx_cqe_desc_t *d) { u32 tail_sz = 0, n_tail_segs = 0; @@ -44,6 +45,7 @@ oct_rx_attach_tail (vlib_main_t *vm, oct_rx_node_ctx_t *ctx, vlib_buffer_t *h, return; b = oct_seg_to_bp (d->segs0[1]); + b->template = *bt; h->next_buffer = vlib_get_buffer_index (vm, b); tail_sz += b->current_length = d->sg0.seg2_size; n_tail_segs++; @@ -54,6 +56,7 @@ oct_rx_attach_tail (vlib_main_t *vm, oct_rx_node_ctx_t *ctx, vlib_buffer_t *h, p = b; p->flags = VLIB_BUFFER_NEXT_PRESENT; b = oct_seg_to_bp (d->segs0[2]); + b->template = *bt; p->next_buffer = vlib_get_buffer_index (vm, b); tail_sz += b->current_length = d->sg0.seg3_size; n_tail_segs++; @@ -68,6 +71,7 @@ oct_rx_attach_tail (vlib_main_t *vm, oct_rx_node_ctx_t *ctx, vlib_buffer_t *h, p = b; p->flags = VLIB_BUFFER_NEXT_PRESENT; b = oct_seg_to_bp (d->segs1[0]); + b->template = *bt; p->next_buffer = vlib_get_buffer_index (vm, b); tail_sz += b->current_length = d->sg1.seg1_size; n_tail_segs++; @@ -78,6 +82,7 @@ oct_rx_attach_tail (vlib_main_t *vm, oct_rx_node_ctx_t *ctx, vlib_buffer_t *h, p = b; p->flags = VLIB_BUFFER_NEXT_PRESENT; b = oct_seg_to_bp (d->segs1[1]); + b->template = *bt; p->next_buffer = vlib_get_buffer_index (vm, b); tail_sz += b->current_length = d->sg1.seg2_size; n_tail_segs++; @@ -88,6 +93,7 @@ oct_rx_attach_tail (vlib_main_t *vm, oct_rx_node_ctx_t *ctx, vlib_buffer_t *h, p = b; p->flags = VLIB_BUFFER_NEXT_PRESENT; b = oct_seg_to_bp (d->segs1[2]); + b->template = *bt; p->next_buffer = vlib_get_buffer_index (vm, b); tail_sz += b->current_length = d->sg1.seg3_size; n_tail_segs++; @@ -411,8 +417,9 @@ oct_rx_ipsec_set_error (vlib_main_t *vm, vlib_node_runtime_t *node, #define OCT_SEG_LEN_SHIFT 16 #define OCT_SEG_LEN_MASK 0xFFFF -static_always_inline u32 -oct_rx_ipsec_attach_tail (vlib_main_t *vm, const union nix_rx_parse_u *rxp0, +static_always_inline void +oct_rx_ipsec_attach_tail (vlib_main_t *vm, oct_rx_node_ctx_t *ctx, + const union nix_rx_parse_u *rxp0, vlib_buffer_template_t *bt, vlib_buffer_t *b) { u32 n_words, n_words_processed, desc_sizem1; @@ -421,13 +428,13 @@ oct_rx_ipsec_attach_tail (vlib_main_t *vm, const union nix_rx_parse_u *rxp0, u32 current_desc, bi, sg_len; vlib_buffer_t *buf = b; struct nix_rx_sg_s *sg; - u32 total_segs = 1; + u32 total_segs = 0; u64 seg_len; i64 len; desc_sizem1 = rxp0->desc_sizem1; if (desc_sizem1 == 0) - return total_segs; + return; n_words = desc_sizem1 << 1; n_sg_desc = (n_words / 4) + 1; @@ -484,7 +491,7 @@ oct_rx_ipsec_attach_tail (vlib_main_t *vm, const union nix_rx_parse_u *rxp0, { vlib_buffer_free_no_next (vm, &bi, 1); total_segs++; - return total_segs; + goto done; } last_buf->flags |= VLIB_BUFFER_NEXT_PRESENT; @@ -512,7 +519,8 @@ oct_rx_ipsec_attach_tail (vlib_main_t *vm, const union nix_rx_parse_u *rxp0, } } - return total_segs; +done: + ctx->n_segs += total_segs; } static_always_inline u32 @@ -556,7 +564,7 @@ oct_rx_inl_ipsec_vlib_from_cq (vlib_main_t *vm, vlib_node_runtime_t *node, oct_get_len_from_meta (cpt_hdr, d[0].parse.w[0], d[0].parse.w[4]); idx = cpt_hdr->w0.cookie; oct_rx_ipsec_update_counters (vm, b[0], esp_sz, 1, idx); - oct_rx_ipsec_attach_tail (vm, orig_rxp, bt, b[0]); + oct_rx_ipsec_attach_tail (vm, ctx, orig_rxp, bt, b[0]); } ctx->n_rx_bytes += olen; @@ -580,7 +588,7 @@ oct_rx_vlib_from_cq (vlib_main_t *vm, oct_nix_rx_cqe_desc_t *d, *err_flags |= ((d[0].parse.w[0] >> 20) & 0xFFF); ctx->n_segs += 1; if (d[0].sg0.segs > 1) - oct_rx_attach_tail (vm, ctx, b[0], d + 0); + oct_rx_attach_tail (vm, ctx, bt, b[0], d + 0); buffs[0] = b[0]; return 0; } @@ -711,10 +719,10 @@ oct_rx_batch (vlib_main_t *vm, vlib_node_runtime_t *node, if (PREDICT_FALSE (segs > 4)) { - oct_rx_attach_tail (vm, ctx, b[0], d + 0); - oct_rx_attach_tail (vm, ctx, b[1], d + 1); - oct_rx_attach_tail (vm, ctx, b[2], d + 2); - oct_rx_attach_tail (vm, ctx, b[3], d + 3); + oct_rx_attach_tail (vm, ctx, &bt, b[0], d + 0); + oct_rx_attach_tail (vm, ctx, &bt, b[1], d + 1); + oct_rx_attach_tail (vm, ctx, &bt, b[2], d + 2); + oct_rx_attach_tail (vm, ctx, &bt, b[3], d + 3); } buffs[0] = b[0]; @@ -801,10 +809,10 @@ oct_rx_batch (vlib_main_t *vm, vlib_node_runtime_t *node, vm, b[0], esp_sz0, 1, idx0, b[1], esp_sz1, 1, idx1, b[2], esp_sz2, 1, idx2, b[3], esp_sz3, 1, idx3); - oct_rx_ipsec_attach_tail (vm, orig_rxp0, &bt, b[0]); - oct_rx_ipsec_attach_tail (vm, orig_rxp1, &bt, b[1]); - oct_rx_ipsec_attach_tail (vm, orig_rxp2, &bt, b[2]); - oct_rx_ipsec_attach_tail (vm, orig_rxp3, &bt, b[3]); + oct_rx_ipsec_attach_tail (vm, ctx, orig_rxp0, &bt, b[0]); + oct_rx_ipsec_attach_tail (vm, ctx, orig_rxp1, &bt, b[1]); + oct_rx_ipsec_attach_tail (vm, ctx, orig_rxp2, &bt, b[2]); + oct_rx_ipsec_attach_tail (vm, ctx, orig_rxp3, &bt, b[3]); } else { @@ -822,7 +830,7 @@ oct_rx_batch (vlib_main_t *vm, vlib_node_runtime_t *node, cpt_hdr0, d[0].parse.w[0], d[0].parse.w[4]); idx0 = cpt_hdr0->w0.cookie; oct_rx_ipsec_update_counters (vm, b[0], esp_sz0, 1, idx0); - oct_rx_ipsec_attach_tail (vm, orig_rxp0, &bt, b[0]); + oct_rx_ipsec_attach_tail (vm, ctx, orig_rxp0, &bt, b[0]); } if (is_fail1) @@ -839,7 +847,7 @@ oct_rx_batch (vlib_main_t *vm, vlib_node_runtime_t *node, cpt_hdr1, d[1].parse.w[0], d[1].parse.w[4]); idx1 = cpt_hdr1->w0.cookie; oct_rx_ipsec_update_counters (vm, b[1], esp_sz1, 1, idx1); - oct_rx_ipsec_attach_tail (vm, orig_rxp1, &bt, b[1]); + oct_rx_ipsec_attach_tail (vm, ctx, orig_rxp1, &bt, b[1]); } if (is_fail2) @@ -856,7 +864,7 @@ oct_rx_batch (vlib_main_t *vm, vlib_node_runtime_t *node, cpt_hdr2, d[2].parse.w[0], d[2].parse.w[4]); idx2 = cpt_hdr2->w0.cookie; oct_rx_ipsec_update_counters (vm, b[2], esp_sz2, 1, idx2); - oct_rx_ipsec_attach_tail (vm, orig_rxp2, &bt, b[2]); + oct_rx_ipsec_attach_tail (vm, ctx, orig_rxp2, &bt, b[2]); } if (is_fail3) @@ -873,7 +881,7 @@ oct_rx_batch (vlib_main_t *vm, vlib_node_runtime_t *node, cpt_hdr3, d[3].parse.w[0], d[3].parse.w[4]); idx3 = cpt_hdr3->w0.cookie; oct_rx_ipsec_update_counters (vm, b[3], esp_sz3, 1, idx3); - oct_rx_ipsec_attach_tail (vm, orig_rxp3, &bt, b[3]); + oct_rx_ipsec_attach_tail (vm, ctx, orig_rxp3, &bt, b[3]); } } ctx->n_rx_bytes += olen0 + olen1 + olen2 + olen3; From fbe8d03cc00747eac92bae672830b52cc5e6242b Mon Sep 17 00:00:00 2001 From: Nagendra T P Date: Mon, 17 Feb 2025 13:25:44 +0530 Subject: [PATCH 188/271] build: ubuntu-24.04 install-deps package list - support for ubuntu-24.04 CI jobs Type: make Jira: https://essjira.marvell.com/browse/IPBUSW-60244 Change-Id: I225d72e22207283de9979cce75d8d208ab99feb5 Signed-off-by: Nagendra T P Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/146175 Tested-by: sa_ip-toolkits-Jenkins Tested-by: sa_ip-sw-jenkins Reviewed-by: Monendra Singh Kushwaha (cherry picked from commit 21daaaf3386ece183a9da237a07a95fe4c24dc45) --- Makefile | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 17d633562a..c4253da768 100644 --- a/Makefile +++ b/Makefile @@ -84,8 +84,12 @@ DEB_DEPENDS += libpcap-dev DEB_DEPENDS += tshark LIBFFI=libffi6 # works on all but 20.04 and debian-testing - -ifeq ($(OS_VERSION_ID),22.04) +ifeq ($(OS_VERSION_ID),24.04) + DEB_DEPENDS += libssl-dev + DEB_DEPENDS += llvm clang clang-format-14 + LIBFFI=libffi8 + DEB_DEPENDS += enchant-2 # for docs +else ifeq ($(OS_VERSION_ID),22.04) DEB_DEPENDS += python3-virtualenv DEB_DEPENDS += libssl-dev DEB_DEPENDS += clang clang-format-11 From d32698f2f9f6f8d9d284be23038491b5870f3036 Mon Sep 17 00:00:00 2001 From: Nagendra T P Date: Tue, 18 Feb 2025 14:38:24 +0530 Subject: [PATCH 189/271] octeon: remove unused variable The variable 'i' was incremented but never used, triggering unused-but-set-variable when building with -Werror. Type: fix Jira: https://essjira.marvell.com/browse/IPBUSW-60244 Signed-off-by: Nagendra T P Change-Id: I6771e543f77161ccf3af0a1ad02d27cb595eb8ab Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/146259 Tested-by: sa_ip-sw-jenkins Reviewed-by: Monendra Singh Kushwaha (cherry picked from commit 345f310ad6a8d681583ab885c699f6bf5e490432) --- src/plugins/dev_octeon/virtio.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/plugins/dev_octeon/virtio.c b/src/plugins/dev_octeon/virtio.c index a1ceca655c..1337decffd 100644 --- a/src/plugins/dev_octeon/virtio.c +++ b/src/plugins/dev_octeon/virtio.c @@ -208,7 +208,6 @@ oct_populate_dma_device_list (u16 *nb_elem, u8 *dma_list) static void oct_virtio_parse_arguments (dao_pal_global_conf_t *conf, vnet_dev_arg_t *args) { - int i = 0; vnet_dev_arg_t *a = args; for (; a < vec_end (args) && a->val_set; a++) @@ -233,7 +232,6 @@ oct_virtio_parse_arguments (dao_pal_global_conf_t *conf, vnet_dev_arg_t *args) log_info ("Invalid virtio device arguments received\n"); } - i++; } } From d472d9ec31c3980c902c07a4a980bfae4cd70113 Mon Sep 17 00:00:00 2001 From: Nagendra T P Date: Wed, 12 Feb 2025 12:32:01 +0530 Subject: [PATCH 190/271] ci: upgrade to ubuntu 24.04 arm runner Accommodate changes to workflow for Ubuntu 24.04. Signed-off-by: Nagendra T P --- .github/workflows/build-cn10k.yml | 55 ++++++++++++++++--------------- .github/workflows/build-cn9k.yml | 51 ++++++++++++++-------------- 2 files changed, 56 insertions(+), 50 deletions(-) diff --git a/.github/workflows/build-cn10k.yml b/.github/workflows/build-cn10k.yml index 9410347b3a..ac22c9175c 100644 --- a/.github/workflows/build-cn10k.yml +++ b/.github/workflows/build-cn10k.yml @@ -15,7 +15,8 @@ permissions: jobs: ubuntu-cn10k-build: name: ubuntu-cn10k-arm64 - runs-on: ubuntu-22.04-arm + runs-on: ubuntu-24.04-arm + steps: - name: Checkout sources uses: actions/checkout@v4.2.2 @@ -40,7 +41,7 @@ jobs: git tag --points-at HEAD > /tmp/tags [ -s /tmp/tags ] && PKG_POSTFIX= || PKG_POSTFIX=-devel FW_PKG_POSTFIX="" - if [ ${PKG_POSTFIX} == "-devel" ]; then + if [ ${PKG_POSTFIX} == "-devel" ]; then FW_PKG_POSTFIX="" else FW_PKG_POSTFIX=$PKG_POSTFIX @@ -61,68 +62,70 @@ jobs: id: build run: | mkdir -p ~/.ccache + BASE_DIR=${PWD} sudo apt-get update -q -y sudo apt-get install -y apt-utils gcc meson sudo make dialog ccache git build-essential software-properties-common sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test sudo apt-get update -q -y + sudo apt-get install -y curl autoconf automake debhelper dkms + sudo apt-get install -y libtool libapr1-dev dh-python libconfuse-dev git-review exuberant-ctags + sudo apt-get install -y cscope pkg-config gcovr lcov chrpath libnuma-dev python3-all + sudo apt-get install -y python3-setuptools check libffi-dev python3-ply libunwind-dev cmake + sudo apt-get install -y ninja-build python3-jsonschema python3-yaml python3-venv python3-dev + sudo apt-get install -y python3-pip libnl-3-dev libnl-route-3-dev libmnl-dev python3-virtualenv + sudo apt-get install -y libssl-dev libelf-dev libpcap-dev iperf3 nasm iperf ethtool tshark jq + sudo apt-get install -y llvm clang clang-format-15 enchant-2 libffi8 sudo apt-get install -y aspell aspell-en autopoint autotools-dev binfmt-support binutils binutils-aarch64-linux-gnu sudo apt-get install -y binutils-common bsdextrautils bzip2 ca-certificates clang-14 cmake-data cpp cpp-11 cpp-12 dbus sudo apt-get install -y dctrl-tools debugedit dh-autoreconf dh-elpa-helper dh-strip-nondeterminism dictionaries-common dirmngr sudo apt-get install -y distro-info-data dpkg-dev dwz emacsen-common fakeroot file fontconfig-config fonts-dejavu-core g++ g++-11 - sudo apt-get install -y gcc gcc-11 gcc-11-base gcc-12 gettext gettext-base git-man gnupg gnupg-l10n gnupg-utils gpg gpg-agent + sudo apt-get install -y gcc gcc-11 gcc-11-base gcc-12 gcc-13 gettext gettext-base git-man gnupg gnupg-l10n gnupg-utils gpg gpg-agent sudo apt-get install -y gpg-wks-client gpg-wks-server gpgconf gpgsm groff-base hunspell-en-us icu-devtools intltool-debian sudo apt-get install -y javascript-common kmod less libalgorithm-diff-perl libalgorithm-diff-xs-perl libalgorithm-merge-perl sudo apt-get install -y libapparmor1 libapr1 libarchive-cpio-perl libarchive-zip-perl libarchive13 libasan6 libasan8 libaspell15 sudo apt-get install -y libassuan0 libatomic1 libbcg729-0 libbinutils libbrotli1 libbsd0 libc-ares2 libc-dev-bin libc-devtools - sudo apt-get install -y libc6-dev libcap2-bin libcbor0.8 libcc1-0 libclang-common-14-dev libclang-cpp11 libclang-cpp14 + sudo apt-get install -y libc6-dev libcap2-bin libcbor0.10 libcc1-0 libclang-common-14-dev libclang-cpp14 sudo apt-get install -y libclang1-14 libcommon-sense-perl libconfuse-common libconfuse-doc libconfuse2 libcrypt-dev libctf-nobfd0 sudo apt-get install -y libctf0 libcurl3-gnutls libcurl4 libdbus-1-3 libdbus-1-dev libdebhelper-perl libdeflate0 libdpkg-perl sudo apt-get install -y libdw1 libedit2 libelf1 libenchant-2-2 liberror-perl libexpat1 libexpat1-dev libfakeroot libfido2-1 sudo apt-get install -y libfile-fcntllock-perl libfile-stripnondeterminism-perl libfontconfig1 libfreetype6 libgc1 libgcc-11-dev sudo apt-get install -y libgcc-12-dev libgd-perl libgd3 libgdbm-compat4 libgdbm6 libglib2.0-0 libglib2.0-data libgomp1 - sudo apt-get install -y libhiredis0.14 libhunspell-1.7-0 libhwasan0 libicu-dev libicu70 libiperf0 libisl23 libitm1 libjbig0 + sudo apt-get install -y libhiredis1.1.0 libhunspell-1.7-0 libhwasan0 libicu-dev libicu74 libiperf0 libisl23 libitm1 libjbig0 sudo apt-get install -y libjpeg-turbo8 libjpeg8 libjs-jquery libjs-sphinxdoc libjs-underscore libjson-perl libjson-xs-perl - sudo apt-get install -y libjsoncpp25 libkmod2 libksba8 libldap-2.5-0 libldap-common libllvm11 libllvm14 liblocale-gettext-perl + sudo apt-get install -y libjsoncpp25 libkmod2 libksba8 libldap2 libldap2-dev libldap-common libllvm14t64 liblocale-gettext-perl sudo apt-get install -y liblsan0 libltdl-dev libltdl7 liblua5.2-0 libmagic-mgc libmagic1 libmail-sendmail-perl libmaxminddb0 - sudo apt-get install -y libmd0 libmnl0 libmpc3 libmpdec3 libmpfr6 libncurses-dev libnghttp2-14 libnl-3-200 libnl-genl-3-200 + sudo apt-get install -y libmd0 libmnl0 libmpc3 libmpfr6 libncurses-dev libnghttp2-14 libnl-3-200 libnl-genl-3-200 sudo apt-get install -y libnl-route-3-200 libnpth0 libnsl-dev libnuma1 libobjc-11-dev libobjc4 libpam-cap libpcap0.8 - sudo apt-get install -y libpcap0.8-dev libperl5.34 libperlio-gzip-perl libpfm4 libpipeline1 libpng16-16 libpsl5 libpython3-dev - sudo apt-get install -y libpython3-stdlib libpython3.10 libpython3.10-dev libpython3.10-minimal libpython3.10-stdlib libreadline8 + sudo apt-get install -y libpcap0.8-dev libperl5.38t64 libperlio-gzip-perl libpfm4 libpipeline1 libpng16-16 libpsl5 libpython3-dev + sudo apt-get install -y libpython3-stdlib libpython3.12t64 libpython3.12-dev libpython3.12-minimal libreadline8 sudo apt-get install -y librhash0 librtmp1 libsasl2-2 libsasl2-modules libsasl2-modules-db libsbc1 libsctp-dev libsctp1 sudo apt-get install -y libsigsegv2 libsmi2ldbl libsnappy1v5 libspandsp2 libspeexdsp1 libsqlite3-0 libssh-4 libssh-gcrypt-4 sudo apt-get install -y libstdc++-11-dev libsub-override-perl libsubunit-dev libsubunit0 libsys-hostname-long-perl - sudo apt-get install -y libtext-iconv-perl libtiff5 libtinfo-dev libtirpc-dev libtsan0 libtsan2 libtypes-serialiser-perl - sudo apt-get install -y libubsan1 libuchardet0 libuv1 libwebp7 libwireshark-data libwireshark15 libwiretap12 libwsutil13 libx11-6 + sudo apt-get install -y libtext-iconv-perl libtiff6 libtinfo-dev libtirpc-dev libtsan0 libtsan2 libtypes-serialiser-perl + sudo apt-get install -y libubsan1 libuchardet0 libuv1 libwebp7 libwireshark-data libwireshark17t64 libwiretap14t64 libwsutil15t64 libx11-6 sudo apt-get install -y libx11-data libxau6 libxcb1 libxdmcp6 libxext6 libxml2 libxml2-dev libxmuu1 libxpm4 libxslt1.1 - sudo apt-get install -y libyaml-0-2 libz3-4 libz3-dev linux-headers-5.15.0-107 linux-headers-5.15.0-107-generic + sudo apt-get install -y libyaml-0-2 libz3-4 libz3-dev linux-headers-6.8.0-51 linux-headers-6.8.0-51-generic sudo apt-get install -y linux-headers-generic linux-libc-dev llvm-14 llvm-14-dev llvm-14-linker-tools llvm-14-runtime sudo apt-get install -y llvm-14-tools lsb-release lto-disabled-list m4 man-db manpages manpages-dev media-types netbase - sudo apt-get install -y openssh-client openssl patch perl perl-modules-5.34 pinentry-curses po-debconf publicsuffix + sudo apt-get install -y openssh-client openssl patch perl perl-modules-5.38 pinentry-curses po-debconf publicsuffix sudo apt-get install -y python-babel-localedata python3 python3-attr python3-babel python3-bs4 python3-certifi python3-chardet - sudo apt-get install -y python3-distlib python3-distutils python3-filelock python3-html5lib python3-idna + sudo apt-get install -y python3-distlib python3-filelock python3-html5lib python3-idna sudo apt-get install -y python3-importlib-metadata python3-jinja2 python3-lib2to3 python3-lxml python3-markupsafe python3-minimal sudo apt-get install -y python3-more-itertools python3-pip-whl python3-pkg-resources python3-platformdirs python3-pygments sudo apt-get install -y python3-pyrsistent python3-requests python3-setuptools-whl python3-six python3-soupsieve python3-tz sudo apt-get install -y python3-urllib3 python3-webencodings python3-wheel python3-wheel-whl python3-zipp python3.10 - sudo apt-get install -y python3.10-dev python3.10-minimal python3.10-venv readline-common rpcsvc-proto shared-mime-info tzdata + sudo apt-get install -y python3-minimal readline-common rpcsvc-proto shared-mime-info tzdata sudo apt-get install -y ucf uuid-dev wireshark-common xauth xdg-user-dirs xz-utils zlib1g-dev libgmpxx4ldbl - sudo apt-get install -y check chrpath clang clang-format-11 cmake cscope debhelper dh-python dkms liblz4-dev - sudo apt-get install -y ethtool exuberant-ctags gcovr git-review iperf iperf3 lcov libapr1-dev liblzma-dev wget - sudo apt-get install -y libconfuse-dev libelf-dev libffi7 libmnl-dev libnl-3-dev libnl-route-3-dev libzstd-dev - sudo apt-get install -y libnuma-dev libpcap-dev libssl-dev nasm ninja-build python3-all python3-dev nettle-dev - sudo apt-get install -y python3-jsonschema pkg-config python3-pip python3-ply python3-setuptools python3-venv - sudo apt-get install -y python3-virtualenv tshark lsb-release doxygen libarchive-dev libbsd-dev libbpf-dev - sudo apt-get install -y libnl-xfrm-3-dev sphinx-common python3-sphinx-rtd-theme libfdt-dev libjansson-dev - sudo apt-get install -y python3-pyelftools gcc-13 bzip2-doc libacl1-dev libattr1-dev libbz2-dev libgmp-dev - sudo pip3 install meson --upgrade - BASE_DIR=${PWD} + sudo apt-get install -y liblz4-dev liblzma-dev wget libzstd-dev nettle-dev lsb-release doxygen libarchive-dev + sudo apt-get install -y libnl-xfrm-3-dev sphinx-common python3-sphinx-rtd-theme libfdt-dev libjansson-dev libbsd-dev + sudo apt-get install -y python3-pyelftools gcc-14 bzip2-doc libacl1-dev libattr1-dev libbz2-dev libgmp-dev libbpf-dev source ${BASE_DIR}/artifacts/env DISTRO=ubuntu-`lsb_release -rs` echo "DISTRO=${DISTRO}" >> ${BASE_DIR}/artifacts/env echo "cache_dir = ~/.ccache" > ~/.ccache/ccache.conf ccache -p git config --global --add safe.directory "${PWD}" - APT_ARGS='-y -q' make install-deps + sudo APT_ARGS='-y -q' make install-deps make build-release VPP_PLATFORM=octeon10 mkdir -p "${PWD}/install/DEBIAN" mkdir -p "${PWD}/install/usr/share/vpp/api" diff --git a/.github/workflows/build-cn9k.yml b/.github/workflows/build-cn9k.yml index 924c15b2eb..22b97b9349 100644 --- a/.github/workflows/build-cn9k.yml +++ b/.github/workflows/build-cn9k.yml @@ -15,7 +15,7 @@ permissions: jobs: ubuntu-cn9k-build: name: ubuntu-cn9k-arm64 - runs-on: ubuntu-22.04-arm + runs-on: ubuntu-24.04-arm steps: - name: Checkout sources uses: actions/checkout@v4.2.2 @@ -61,60 +61,63 @@ jobs: id: build run: | mkdir -p ~/.ccache + BASE_DIR=${PWD} sudo apt-get update -q -y sudo apt-get install -y apt-utils gcc meson sudo make dialog ccache git build-essential software-properties-common sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test sudo apt-get update -q -y + sudo apt-get install -y curl autoconf automake debhelper dkms + sudo apt-get install -y libtool libapr1-dev dh-python libconfuse-dev git-review exuberant-ctags + sudo apt-get install -y cscope pkg-config gcovr lcov chrpath libnuma-dev python3-all + sudo apt-get install -y python3-setuptools check libffi-dev python3-ply libunwind-dev cmake + sudo apt-get install -y ninja-build python3-jsonschema python3-yaml python3-venv python3-dev + sudo apt-get install -y python3-pip libnl-3-dev libnl-route-3-dev libmnl-dev python3-virtualenv + sudo apt-get install -y libssl-dev libelf-dev libpcap-dev iperf3 nasm iperf ethtool tshark jq + sudo apt-get install -y llvm clang clang-format-15 enchant-2 libffi8 sudo apt-get install -y aspell aspell-en autopoint autotools-dev binfmt-support binutils binutils-aarch64-linux-gnu sudo apt-get install -y binutils-common bsdextrautils bzip2 ca-certificates clang-14 cmake-data cpp cpp-11 cpp-12 dbus sudo apt-get install -y dctrl-tools debugedit dh-autoreconf dh-elpa-helper dh-strip-nondeterminism dictionaries-common dirmngr sudo apt-get install -y distro-info-data dpkg-dev dwz emacsen-common fakeroot file fontconfig-config fonts-dejavu-core g++ g++-11 - sudo apt-get install -y gcc gcc-11 gcc-11-base gcc-12 gettext gettext-base git-man gnupg gnupg-l10n gnupg-utils gpg gpg-agent + sudo apt-get install -y gcc gcc-11 gcc-11-base gcc-12 gcc-13 gettext gettext-base git-man gnupg gnupg-l10n gnupg-utils gpg gpg-agent sudo apt-get install -y gpg-wks-client gpg-wks-server gpgconf gpgsm groff-base hunspell-en-us icu-devtools intltool-debian sudo apt-get install -y javascript-common kmod less libalgorithm-diff-perl libalgorithm-diff-xs-perl libalgorithm-merge-perl sudo apt-get install -y libapparmor1 libapr1 libarchive-cpio-perl libarchive-zip-perl libarchive13 libasan6 libasan8 libaspell15 sudo apt-get install -y libassuan0 libatomic1 libbcg729-0 libbinutils libbrotli1 libbsd0 libc-ares2 libc-dev-bin libc-devtools - sudo apt-get install -y libc6-dev libcap2-bin libcbor0.8 libcc1-0 libclang-common-14-dev libclang-cpp11 libclang-cpp14 + sudo apt-get install -y libc6-dev libcap2-bin libcbor0.10 libcc1-0 libclang-common-14-dev libclang-cpp14 sudo apt-get install -y libclang1-14 libcommon-sense-perl libconfuse-common libconfuse-doc libconfuse2 libcrypt-dev libctf-nobfd0 sudo apt-get install -y libctf0 libcurl3-gnutls libcurl4 libdbus-1-3 libdbus-1-dev libdebhelper-perl libdeflate0 libdpkg-perl sudo apt-get install -y libdw1 libedit2 libelf1 libenchant-2-2 liberror-perl libexpat1 libexpat1-dev libfakeroot libfido2-1 sudo apt-get install -y libfile-fcntllock-perl libfile-stripnondeterminism-perl libfontconfig1 libfreetype6 libgc1 libgcc-11-dev sudo apt-get install -y libgcc-12-dev libgd-perl libgd3 libgdbm-compat4 libgdbm6 libglib2.0-0 libglib2.0-data libgomp1 - sudo apt-get install -y libhiredis0.14 libhunspell-1.7-0 libhwasan0 libicu-dev libicu70 libiperf0 libisl23 libitm1 libjbig0 + sudo apt-get install -y libhiredis1.1.0 libhunspell-1.7-0 libhwasan0 libicu-dev libicu74 libiperf0 libisl23 libitm1 libjbig0 sudo apt-get install -y libjpeg-turbo8 libjpeg8 libjs-jquery libjs-sphinxdoc libjs-underscore libjson-perl libjson-xs-perl - sudo apt-get install -y libjsoncpp25 libkmod2 libksba8 libldap-2.5-0 libldap-common libllvm11 libllvm14 liblocale-gettext-perl + sudo apt-get install -y libjsoncpp25 libkmod2 libksba8 libldap2 libldap2-dev libldap-common libllvm14t64 liblocale-gettext-perl sudo apt-get install -y liblsan0 libltdl-dev libltdl7 liblua5.2-0 libmagic-mgc libmagic1 libmail-sendmail-perl libmaxminddb0 - sudo apt-get install -y libmd0 libmnl0 libmpc3 libmpdec3 libmpfr6 libncurses-dev libnghttp2-14 libnl-3-200 libnl-genl-3-200 + sudo apt-get install -y libmd0 libmnl0 libmpc3 libmpfr6 libncurses-dev libnghttp2-14 libnl-3-200 libnl-genl-3-200 sudo apt-get install -y libnl-route-3-200 libnpth0 libnsl-dev libnuma1 libobjc-11-dev libobjc4 libpam-cap libpcap0.8 - sudo apt-get install -y libpcap0.8-dev libperl5.34 libperlio-gzip-perl libpfm4 libpipeline1 libpng16-16 libpsl5 libpython3-dev - sudo apt-get install -y libpython3-stdlib libpython3.10 libpython3.10-dev libpython3.10-minimal libpython3.10-stdlib libreadline8 + sudo apt-get install -y libpcap0.8-dev libperl5.38t64 libperlio-gzip-perl libpfm4 libpipeline1 libpng16-16 libpsl5 libpython3-dev + sudo apt-get install -y libpython3-stdlib libpython3.12t64 libpython3.12-dev libpython3.12-minimal libreadline8 sudo apt-get install -y librhash0 librtmp1 libsasl2-2 libsasl2-modules libsasl2-modules-db libsbc1 libsctp-dev libsctp1 sudo apt-get install -y libsigsegv2 libsmi2ldbl libsnappy1v5 libspandsp2 libspeexdsp1 libsqlite3-0 libssh-4 libssh-gcrypt-4 sudo apt-get install -y libstdc++-11-dev libsub-override-perl libsubunit-dev libsubunit0 libsys-hostname-long-perl - sudo apt-get install -y libtext-iconv-perl libtiff5 libtinfo-dev libtirpc-dev libtsan0 libtsan2 libtypes-serialiser-perl - sudo apt-get install -y libubsan1 libuchardet0 libuv1 libwebp7 libwireshark-data libwireshark15 libwiretap12 libwsutil13 libx11-6 + sudo apt-get install -y libtext-iconv-perl libtiff6 libtinfo-dev libtirpc-dev libtsan0 libtsan2 libtypes-serialiser-perl + sudo apt-get install -y libubsan1 libuchardet0 libuv1 libwebp7 libwireshark-data libwireshark17t64 libwiretap14t64 libwsutil15t64 libx11-6 sudo apt-get install -y libx11-data libxau6 libxcb1 libxdmcp6 libxext6 libxml2 libxml2-dev libxmuu1 libxpm4 libxslt1.1 - sudo apt-get install -y libyaml-0-2 libz3-4 libz3-dev linux-headers-5.15.0-107 linux-headers-5.15.0-107-generic + sudo apt-get install -y libyaml-0-2 libz3-4 libz3-dev linux-headers-6.8.0-51 linux-headers-6.8.0-51-generic sudo apt-get install -y linux-headers-generic linux-libc-dev llvm-14 llvm-14-dev llvm-14-linker-tools llvm-14-runtime sudo apt-get install -y llvm-14-tools lsb-release lto-disabled-list m4 man-db manpages manpages-dev media-types netbase - sudo apt-get install -y openssh-client openssl patch perl perl-modules-5.34 pinentry-curses po-debconf publicsuffix + sudo apt-get install -y openssh-client openssl patch perl perl-modules-5.38 pinentry-curses po-debconf publicsuffix sudo apt-get install -y python-babel-localedata python3 python3-attr python3-babel python3-bs4 python3-certifi python3-chardet - sudo apt-get install -y python3-distlib python3-distutils python3-filelock python3-html5lib python3-idna + sudo apt-get install -y python3-distlib python3-filelock python3-html5lib python3-idna sudo apt-get install -y python3-importlib-metadata python3-jinja2 python3-lib2to3 python3-lxml python3-markupsafe python3-minimal sudo apt-get install -y python3-more-itertools python3-pip-whl python3-pkg-resources python3-platformdirs python3-pygments sudo apt-get install -y python3-pyrsistent python3-requests python3-setuptools-whl python3-six python3-soupsieve python3-tz sudo apt-get install -y python3-urllib3 python3-webencodings python3-wheel python3-wheel-whl python3-zipp python3.10 - sudo apt-get install -y python3.10-dev python3.10-minimal python3.10-venv readline-common rpcsvc-proto shared-mime-info tzdata + sudo apt-get install -y python3-minimal readline-common rpcsvc-proto shared-mime-info tzdata sudo apt-get install -y ucf uuid-dev wireshark-common xauth xdg-user-dirs xz-utils zlib1g-dev libgmpxx4ldbl - sudo apt-get install -y check chrpath clang clang-format-11 cmake cscope debhelper dh-python dkms liblz4-dev - sudo apt-get install -y ethtool exuberant-ctags gcovr git-review iperf iperf3 lcov libapr1-dev liblzma-dev wget - sudo apt-get install -y libconfuse-dev libelf-dev libffi7 libmnl-dev libnl-3-dev libnl-route-3-dev libzstd-dev - sudo apt-get install -y libnuma-dev libpcap-dev libssl-dev nasm ninja-build python3-all python3-dev nettle-dev - sudo apt-get install -y python3-jsonschema pkg-config python3-pip python3-ply python3-setuptools python3-venv - sudo apt-get install -y python3-virtualenv tshark lsb-release doxygen libarchive-dev libbsd-dev libbpf-dev - sudo apt-get install -y libnl-xfrm-3-dev sphinx-common python3-sphinx-rtd-theme libfdt-dev libjansson-dev - sudo apt-get install -y python3-pyelftools gcc-13 bzip2-doc libacl1-dev libattr1-dev libbz2-dev libgmp-dev - sudo pip3 install meson --upgrade + sudo apt-get install -y liblz4-dev liblzma-dev wget libzstd-dev nettle-dev lsb-release doxygen libarchive-dev + sudo apt-get install -y libnl-xfrm-3-dev sphinx-common python3-sphinx-rtd-theme libfdt-dev libjansson-dev libbsd-dev + sudo apt-get install -y python3-pyelftools gcc-14 bzip2-doc libacl1-dev libattr1-dev libbz2-dev libgmp-dev libbpf-dev BASE_DIR=${PWD} source ${BASE_DIR}/artifacts/env DISTRO=ubuntu-`lsb_release -rs` @@ -122,7 +125,7 @@ jobs: echo "cache_dir = ~/.ccache" > ~/.ccache/ccache.conf ccache -p git config --global --add safe.directory "${PWD}" - APT_ARGS='-y -q' make install-deps + sudo APT_ARGS='-y -q' make install-deps make build-release VPP_PLATFORM=octeon9 mkdir -p "${PWD}/install/DEBIAN" mkdir -p "${PWD}/install/usr/share/vpp/api" From c0376d8bdace51ddb0bf8b8b099c82070116bb58 Mon Sep 17 00:00:00 2001 From: Nagendra T P Date: Thu, 20 Feb 2025 11:35:35 +0530 Subject: [PATCH 191/271] doc: bump dpdk version bump dpdk version to 25.01.0 Signed-off-by: Nagendra T P --- DPDK_VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DPDK_VERSION b/DPDK_VERSION index cd9c2e2495..d88b097b2a 100644 --- a/DPDK_VERSION +++ b/DPDK_VERSION @@ -1,2 +1,2 @@ BASE_VERSION=24.11.0 -RELEASE_VERSION=24.12.0 \ No newline at end of file +RELEASE_VERSION=25.01.0 \ No newline at end of file From 0602ddb04fabfadef8f8c91b44b5d092c5588d7b Mon Sep 17 00:00:00 2001 From: Nagendra T P Date: Thu, 6 Mar 2025 17:36:36 +0530 Subject: [PATCH 192/271] ci: add oct-ep-target dependency Add oct-ep-target as a dependency for VPP build and VPP package install. Signed-off-by: Nagendra T P --- .github/workflows/build-cn10k.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-cn10k.yml b/.github/workflows/build-cn10k.yml index ac22c9175c..5cb882e497 100644 --- a/.github/workflows/build-cn10k.yml +++ b/.github/workflows/build-cn10k.yml @@ -118,7 +118,7 @@ jobs: sudo apt-get install -y ucf uuid-dev wireshark-common xauth xdg-user-dirs xz-utils zlib1g-dev libgmpxx4ldbl sudo apt-get install -y liblz4-dev liblzma-dev wget libzstd-dev nettle-dev lsb-release doxygen libarchive-dev sudo apt-get install -y libnl-xfrm-3-dev sphinx-common python3-sphinx-rtd-theme libfdt-dev libjansson-dev libbsd-dev - sudo apt-get install -y python3-pyelftools gcc-14 bzip2-doc libacl1-dev libattr1-dev libbz2-dev libgmp-dev libbpf-dev + sudo apt-get install -y python3-pyelftools gcc-14 bzip2-doc libacl1-dev libattr1-dev libbz2-dev libgmp-dev libbpf-dev libconfig-dev source ${BASE_DIR}/artifacts/env DISTRO=ubuntu-`lsb_release -rs` echo "DISTRO=${DISTRO}" >> ${BASE_DIR}/artifacts/env @@ -126,6 +126,8 @@ jobs: ccache -p git config --global --add safe.directory "${PWD}" sudo APT_ARGS='-y -q' make install-deps + wget "https://github.com/MarvellEmbeddedProcessors/pcie_ep_octeon_target/releases/download/oct-ep-target-cn10k-${MRVL_PKG_VERSION}-${DISTRO}-${MRVL_PKG_VERSION}/oct-ep-target-cn10k_${MRVL_PKG_VERSION}_arm64.deb" + sudo apt-get install -y ./"oct-ep-target-cn10k_${MRVL_PKG_VERSION}_arm64.deb" make build-release VPP_PLATFORM=octeon10 mkdir -p "${PWD}/install/DEBIAN" mkdir -p "${PWD}/install/usr/share/vpp/api" @@ -135,7 +137,7 @@ jobs: echo 'Package: vpp-'$PKG_VERSION_NAME'-cn10k'$PKG_POSTFIX >> DEBIAN/control echo 'Version: '$MRVL_PKG_VERSION >> DEBIAN/control echo "Maintainer: Jerin Jacob (jerinj@marvell.com)" >> DEBIAN/control - echo 'Depends: python3, python3-ply, dpdk-'$DPDK_BASE_PKG_VERSION'-cn10k (= '$DPDK_PKG_VERSION'), cpt-firmware-cn10k'${FW_PKG_POSTFIX}' (= '$CPT_PKG_VERSION')' >> DEBIAN/control + echo 'Depends: python3, python3-ply, dpdk-'$DPDK_BASE_PKG_VERSION'-cn10k (= '$DPDK_PKG_VERSION'), cpt-firmware-cn10k'${FW_PKG_POSTFIX}' (= '$CPT_PKG_VERSION'), oct-ep-target-cn10k (= '$MRVL_PKG_VERSION')' >> DEBIAN/control echo "Architecture: arm64" >> DEBIAN/control echo "Homepage: https://wiki.fd.io/view/VPP" >> DEBIAN/control echo "Description: Vector Packet Processing (VPP) for Octeon10" >> DEBIAN/control From 3f2565c72ced37bae1ed8d3e2b496824b3598dc9 Mon Sep 17 00:00:00 2001 From: Monendra Singh Kushwaha Date: Tue, 18 Feb 2025 13:07:35 +0530 Subject: [PATCH 193/271] octeon: use single rx and tx aura This patch enables single rx and tx aura as default. Type: fix Signed-off-by: Monendra Singh Kushwaha Change-Id: I9f7af1fa3d42138c5f7dba614ca2d03e9886267d Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/146250 Reviewed-by: Nithin Kumar Dabilpuram Tested-by: sa_ip-toolkits-Jenkins (cherry picked from commit 8d195c36cec0b6fe851b7cb18b86283a33b2cdf5) Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/147316 --- src/plugins/dev_octeon/init.c | 15 +++++++++++++++ src/plugins/dev_octeon/octeon.h | 2 +- src/plugins/dev_octeon/queue.c | 5 ++--- 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/src/plugins/dev_octeon/init.c b/src/plugins/dev_octeon/init.c index 8ea982ccfb..c7887f996f 100644 --- a/src/plugins/dev_octeon/init.c +++ b/src/plugins/dev_octeon/init.c @@ -208,6 +208,11 @@ oct_init_nix (vlib_main_t *vm, vnet_dev_t *dev) oct_main_t *om = &oct_main; oct_ipsec_main_t *oim = &oct_ipsec_main; oct_inl_dev_main_t *oidm = &oct_inl_dev_main; + u8 bp_index = vlib_buffer_pool_get_default_for_numa (vm, 0); + vlib_buffer_pool_t *bp = vlib_get_buffer_pool (vm, bp_index); + struct npa_aura_s aura = {}; + struct npa_pool_s npapool = { .nat_align = 1, + .buf_offset = OCT_EXT_HDR_SIZE / ROC_ALIGN }; oct_device_t *cd = vnet_dev_get_data (dev), **oct_dev = 0; u8 mac_addr[6]; int rrv; @@ -295,6 +300,13 @@ oct_init_nix (vlib_main_t *vm, vnet_dev_t *dev) }, }; + if (om->use_single_rx_tx_aura && !om->rx_aura_handle) + { + if ((rrv = roc_npa_pool_create (&om->rx_aura_handle, bp->alloc_size, + bp->n_buffers, &aura, &npapool, 0))) + return cnx_return_roc_err (dev, rrv, "roc_npa_pool_create"); + } + if (oidm->inl_dev) { if (oim->inline_ipsec_sessions) @@ -676,6 +688,7 @@ oct_early_config (vlib_main_t *vm, unformat_input_t *input) unformat_input_t _line_input, *line_input = &_line_input; clib_error_t *error = 0; + oct_main.use_single_rx_tx_aura = 1; oct_inl_dev_main.in_min_spi = 0; oct_inl_dev_main.in_max_spi = 8192; oct_inl_dev_main.out_max_sa = 8192; @@ -687,6 +700,8 @@ oct_early_config (vlib_main_t *vm, unformat_input_t *input) { if (unformat (line_input, "max-pools %u", &oct_npa_max_pools)) ; + if (unformat (line_input, "disable-single-rx-tx-aura")) + oct_main.use_single_rx_tx_aura = 0; else if (unformat (line_input, "ipsec_in_min_spi %u", &oct_inl_dev_main.in_min_spi)) ; diff --git a/src/plugins/dev_octeon/octeon.h b/src/plugins/dev_octeon/octeon.h index 3a1333a8e8..ebcb2ecf0c 100644 --- a/src/plugins/dev_octeon/octeon.h +++ b/src/plugins/dev_octeon/octeon.h @@ -158,7 +158,6 @@ typedef struct struct roc_nix *nix; oct_msix_handler_info_t *msix_handler; - u64 aura_handle; u32 cached_cpt_pkts; u64 cpt_io_addr; oct_txq_t **ctqs; @@ -171,6 +170,7 @@ typedef struct oct_device_t **oct_dev; u8 inl_dev_initialized : 1; u8 use_single_rx_tx_aura : 1; + u64 rx_aura_handle; u64 tx_aura_handle; } oct_main_t; diff --git a/src/plugins/dev_octeon/queue.c b/src/plugins/dev_octeon/queue.c index 9fb7517f82..5d9e7d31f2 100644 --- a/src/plugins/dev_octeon/queue.c +++ b/src/plugins/dev_octeon/queue.c @@ -106,7 +106,7 @@ oct_rxq_init (vlib_main_t *vm, vnet_dev_rx_queue_t *rxq, u32 total_sz) if (!om->use_single_rx_tx_aura) total_sz = rxq->size; - if (!om->use_single_rx_tx_aura || !cd->aura_handle) + if (!om->use_single_rx_tx_aura || !om->rx_aura_handle) { if ((rrv = roc_npa_pool_create (&crq->aura_handle, bp->alloc_size, total_sz, &aura, &npapool, 0))) @@ -114,11 +114,10 @@ oct_rxq_init (vlib_main_t *vm, vnet_dev_rx_queue_t *rxq, u32 total_sz) oct_rxq_deinit (vm, rxq); return oct_roc_err (dev, rrv, "roc_npa_pool_create() failed"); } - cd->aura_handle = crq->aura_handle; } else { - crq->aura_handle = cd->aura_handle; + crq->aura_handle = om->rx_aura_handle; } crq->npa_pool_initialized = 1; From 16bdbf1339fc6835d97b54c83a2f06c92e3e73c4 Mon Sep 17 00:00:00 2001 From: Monendra Singh Kushwaha Date: Thu, 20 Feb 2025 17:54:27 +0530 Subject: [PATCH 194/271] octeon: drop packets if no space in send queue Type: fix Signed-off-by: Monendra Singh Kushwaha Change-Id: Ia068ef371224aada4b5c3e9b790b499e11237eeb Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/146486 Tested-by: sa_ip-toolkits-Jenkins Tested-by: sa_ip-sw-jenkins Reviewed-by: Alok Mishra Reviewed-by: Nithin Kumar Dabilpuram (cherry picked from commit 73800717ccb73907bf3091554418449b56e11f24) Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/147317 --- src/plugins/dev_octeon/tx_node.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/plugins/dev_octeon/tx_node.c b/src/plugins/dev_octeon/tx_node.c index 64b1270999..580f76a9af 100644 --- a/src/plugins/dev_octeon/tx_node.c +++ b/src/plugins/dev_octeon/tx_node.c @@ -1557,6 +1557,14 @@ oct_pkts_send_ipsec (vlib_main_t *vm, vlib_node_runtime_t *node, roc_lmt_submit_steorl (lmt_arg, cd->cpt_io_addr); asm volatile("dmb oshst" ::: "memory"); } + else if (quad_bit == 0x0) + { + failed_buff[n_nix_fc_drop] = vlib_get_buffer_index (vm, b[0]); + failed_buff[n_nix_fc_drop + 1] = vlib_get_buffer_index (vm, b[1]); + failed_buff[n_nix_fc_drop + 2] = vlib_get_buffer_index (vm, b[2]); + failed_buff[n_nix_fc_drop + 3] = vlib_get_buffer_index (vm, b[3]); + n_nix_fc_drop += 4; + } b += 4; n_packets -= 4; From b610fe70e3c6fd197aa34316e6ea662aaf90cdb0 Mon Sep 17 00:00:00 2001 From: Monendra Singh Kushwaha Date: Wed, 29 Jan 2025 15:57:34 +0530 Subject: [PATCH 195/271] octeon: add post IPsec reassembly support Type: feature JIRA: https://essjira.marvell.com/browse/IPBUSW-59811 Signed-off-by: Monendra Singh Kushwaha Change-Id: I01f64ed9a1caa00558bf646dadab54dcb56019fa Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/144368 Tested-by: sa_ip-toolkits-Jenkins Reviewed-by: Nithin Kumar Dabilpuram (cherry picked from commit d0068d44118a48f82d50c46bff1ede78204b8295) Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/147329 --- src/plugins/dev_octeon/format.c | 2 +- src/plugins/dev_octeon/ipsec.c | 17 +- src/plugins/dev_octeon/octeon.h | 12 + src/plugins/dev_octeon/rx_node.c | 719 ++++++++++++++++++++++++++----- src/vnet/ipsec/ipsec_cli.c | 2 + src/vnet/ipsec/ipsec_sa.h | 3 +- 6 files changed, 655 insertions(+), 100 deletions(-) diff --git a/src/plugins/dev_octeon/format.c b/src/plugins/dev_octeon/format.c index dac512b8dd..7cc43c1eec 100644 --- a/src/plugins/dev_octeon/format.c +++ b/src/plugins/dev_octeon/format.c @@ -25,7 +25,7 @@ format_oct_nix_rx_cqe_desc (u8 *s, va_list *args) typeof (d->sg0) *sg0 = &d->sg0; typeof (d->sg0) *sg1 = &d->sg1; - s = format (s, "hdr: cqe_type %u nude %u qid %u tag 0x%x", h->cqe_type, + s = format (s, "hdr: cqe_type %u node %u qid %u tag 0x%x", h->cqe_type, h->node, h->q, h->tag); s = format (s, "\n%Uparse:", format_white_space, indent); #define _(n, f) s = format (s, " " #n " " f, p->n) diff --git a/src/plugins/dev_octeon/ipsec.c b/src/plugins/dev_octeon/ipsec.c index 9f842866f9..73edf831ed 100644 --- a/src/plugins/dev_octeon/ipsec.c +++ b/src/plugins/dev_octeon/ipsec.c @@ -328,10 +328,15 @@ oct_ipsec_inb_session_update (oct_ipsec_session_t *sess, ipsec_sa_t *sa) /* * Default options for pkt_out and pkt_fmt are with - * second pass meta and no defrag. + * second pass meta and defrag. */ roc_sa->w0.s.pkt_format = ROC_IE_OT_SA_PKT_FMT_META; - roc_sa->w0.s.pkt_output = ROC_IE_OT_SA_PKT_OUTPUT_NO_FRAG; + + if (sa->flags & IPSEC_SA_FLAG_IS_INL_REASSEMBLY) + roc_sa->w0.s.pkt_output = ROC_IE_OT_SA_PKT_OUTPUT_HW_BASED_DEFRAG; + else + roc_sa->w0.s.pkt_output = ROC_IE_OT_SA_PKT_OUTPUT_NO_FRAG; + roc_sa->w0.s.pkind = ROC_IE_OT_CPT_PKIND; offset = offsetof (struct roc_ot_ipsec_inb_sa, ctx); @@ -800,6 +805,7 @@ oct_ipsec_inl_dev_inb_cfg (vlib_main_t *vm, vnet_dev_t *dev, { oct_inl_dev_main_t *inl_dev_main = &oct_inl_dev_main; oct_device_t *cd = vnet_dev_get_data (dev); + struct roc_cpt_rxc_time_cfg rxc_cfg = { 0 }; int rrv; cd->nix->ipsec_in_min_spi = inl_dev_main->in_min_spi; @@ -815,6 +821,13 @@ oct_ipsec_inl_dev_inb_cfg (vlib_main_t *vm, vnet_dev_t *dev, roc_nix_inb_mode_set (cd->nix, true); roc_nix_inl_inb_set (cd->nix, true); + if ((rrv = roc_nix_reassembly_configure (&rxc_cfg, 1000))) + { + log_err (dev, "roc_nix_reassembly_configure failed - ROC error %s [%d]", + roc_error_msg_get (rrv), rrv); + return VNET_DEV_ERR_INTERNAL; + } + inl_dev_main->inb_sa_base = roc_nix_inl_inb_sa_base_get (NULL, true); inl_dev_main->inb_sa_sz = roc_nix_inl_inb_sa_sz (NULL, true); diff --git a/src/plugins/dev_octeon/octeon.h b/src/plugins/dev_octeon/octeon.h index ebcb2ecf0c..806c6a27a7 100644 --- a/src/plugins/dev_octeon/octeon.h +++ b/src/plugins/dev_octeon/octeon.h @@ -22,6 +22,7 @@ #include #include +#define OCT_FRAME_SIZE (VLIB_FRAME_SIZE * 4) #define OCT_EXT_HDR_SIZE \ PLT_ALIGN (sizeof (oct_ipsec_outbound_pkt_meta_t), ROC_ALIGN) #define OCT_NPA_MAX_POOLS 8192 @@ -282,4 +283,15 @@ typedef struct extern tm_system_t dev_oct_tm_ops; +#define foreach_oct_fp_flag \ + _ (UNUSED, 0) \ + _ (TRACE_EN, 1) + +typedef enum +{ +#define _(name, bit) OCT_FP_FLAG_##name = (1 << bit), + foreach_oct_fp_flag +#undef _ +} oct_fp_flag_t; + #endif /* _OCTEON_H_ */ diff --git a/src/plugins/dev_octeon/rx_node.c b/src/plugins/dev_octeon/rx_node.c index 89c21e4d88..ed4294936c 100644 --- a/src/plugins/dev_octeon/rx_node.c +++ b/src/plugins/dev_octeon/rx_node.c @@ -24,6 +24,7 @@ typedef struct u32 n_rx_pkts; u32 n_rx_bytes; u32 n_segs; + u16 buffer_start_index; } oct_rx_node_ctx_t; static_always_inline vlib_buffer_t * @@ -32,6 +33,27 @@ oct_seg_to_bp (void *p) return (vlib_buffer_t *) p - 1; } +static_always_inline void +oct_rx_verify_vlib (vlib_main_t *vm, vlib_buffer_t *b) +{ + /* + * Warning: Since this assertion is performed in a critical section, + * with increasing number of worker cores, scaling of packet receive-rates + * will be impacted in debug builds + */ + ASSERT (VLIB_BUFFER_KNOWN_ALLOCATED == + vlib_buffer_is_known (vm, vlib_get_buffer_index (vm, b))); +} + +static_always_inline u32 +oct_rx_n_segs (vlib_main_t *vm, const oct_nix_rx_parse_t *rxp) +{ + struct nix_rx_sg_s *sg; + + sg = (struct nix_rx_sg_s *) (((char *) rxp) + sizeof (rxp->f)); + return sg->segs; +} + static_always_inline void oct_rx_attach_tail (vlib_main_t *vm, oct_rx_node_ctx_t *ctx, vlib_buffer_template_t *bt, vlib_buffer_t *h, @@ -106,6 +128,395 @@ oct_rx_attach_tail (vlib_main_t *vm, oct_rx_node_ctx_t *ctx, ctx->n_segs += n_tail_segs; } +/* + * Read upto 4 fragments in case of successful reassembly. + * Fragments which are further segmented are not + * supported currently. + */ +static_always_inline u8 +oct_rx_ipsec_reassembly_success (vlib_main_t *vm, vlib_buffer_template_t *bt, + struct cpt_cn10k_parse_hdr_s *hdr, + oct_nix_rx_cqe_desc_t *d, vlib_buffer_t *buf, + u32 *olen, u32 *esp_len, u32 l2_ol3_hdr_size) +{ + oct_nix_rx_parse_t *rxp_ptr2, *rxp_ptr3; + oct_nix_rx_parse_t *rxp_ptr, *rxp_ptr1; + u16 frag_size1, frag_size2, frag_size3; + vlib_buffer_t *b0, *b1, *b2, *b3; + struct cpt_frag_info_s *frag_info; + oct_nix_rx_parse_t *rxp_meta = &d->parse; + u32 offset, l2_l3_inner_hdr_size; + u64 *wqe_ptr2, *wqe_ptr3; + u64 *wqe_ptr, *wqe_ptr1; + uint64_t *frag_ptr; + u8 frag_cnt; + + wqe_ptr = (u64 *) clib_net_to_host_u64 (hdr->wqe_ptr); + rxp_ptr = (oct_nix_rx_parse_t *) (wqe_ptr + 1); + ASSERT (oct_rx_n_segs (vm, rxp_ptr) == 1); + + l2_l3_inner_hdr_size = rxp_meta->f.ldptr - rxp_meta->f.laptr; + frag_cnt = hdr->w0.num_frags; + + buf->flags |= VLIB_BUFFER_TOTAL_LENGTH_VALID; + buf->total_length_not_including_first_buffer = 0; + b0 = buf; + + /* + * fi_offset is 8B offset from cpt_parse_hdr_s + fi_pad to frag_info_s. + * fi_offset 0 indicates 256B. + */ + offset = hdr->w2.fi_offset; + offset = (((offset - 1) & 0x1f) + 1) * 8; + frag_info = PLT_PTR_ADD (hdr, offset); + + if (frag_cnt == 2) + { + frag_size1 = clib_net_to_host_u16 (frag_info->w1.frag_size1); + wqe_ptr1 = (u64 *) clib_net_to_host_u64 (hdr->frag1_wqe_ptr); + b1 = (vlib_buffer_t *) ((u8 *) wqe_ptr1 - 128); + rxp_ptr1 = (oct_nix_rx_parse_t *) (wqe_ptr1 + 1); + ASSERT (oct_rx_n_segs (vm, rxp_ptr1) == 1); + + oct_rx_verify_vlib (vm, b1); + b1->template = *bt; + + *olen += rxp_ptr1->f.pkt_lenm1 + 1; + *esp_len += rxp_ptr1->f.pkt_lenm1 + 1 - l2_ol3_hdr_size; + + b1->current_length = frag_size1; + b1->current_data = l2_l3_inner_hdr_size; + + b0->total_length_not_including_first_buffer += b1->current_length; + b0->flags |= VLIB_BUFFER_NEXT_PRESENT; + b0->next_buffer = vlib_get_buffer_index (vm, b1); + + return 2; + } + + if (PREDICT_FALSE (frag_cnt == 3)) + { + frag_ptr = (uint64_t *) (frag_info + 1); + + frag_size1 = clib_net_to_host_u16 (frag_info->w1.frag_size1); + frag_size2 = clib_net_to_host_u16 (frag_info->w1.frag_size2); + + wqe_ptr1 = (u64 *) clib_net_to_host_u64 (hdr->frag1_wqe_ptr); + wqe_ptr2 = (u64 *) clib_net_to_host_u64 (*frag_ptr); + + b1 = (vlib_buffer_t *) ((u8 *) wqe_ptr1 - 128); + b2 = (vlib_buffer_t *) ((u8 *) wqe_ptr2 - 128); + + rxp_ptr1 = (oct_nix_rx_parse_t *) (wqe_ptr1 + 1); + rxp_ptr2 = (oct_nix_rx_parse_t *) (wqe_ptr2 + 1); + + ASSERT (oct_rx_n_segs (vm, rxp_ptr1) == 1); + ASSERT (oct_rx_n_segs (vm, rxp_ptr2) == 1); + + b1->template = *bt; + b2->template = *bt; + + *olen += rxp_ptr1->f.pkt_lenm1 + 1; + *olen += rxp_ptr2->f.pkt_lenm1 + 1; + + *esp_len += rxp_ptr1->f.pkt_lenm1 + 1 - l2_ol3_hdr_size; + *esp_len += rxp_ptr2->f.pkt_lenm1 + 1 - l2_ol3_hdr_size; + + b1->current_length = frag_size1; + b2->current_length = frag_size2; + b1->current_data = l2_l3_inner_hdr_size; + b2->current_data = l2_l3_inner_hdr_size; + + b0->total_length_not_including_first_buffer += b1->current_length; + b0->total_length_not_including_first_buffer += b2->current_length; + + b0->flags |= VLIB_BUFFER_NEXT_PRESENT; + b1->flags |= VLIB_BUFFER_NEXT_PRESENT; + + b0->next_buffer = vlib_get_buffer_index (vm, b1); + b1->next_buffer = vlib_get_buffer_index (vm, b2); + + return 3; + } + + if (PREDICT_FALSE (frag_cnt == 4)) + { + frag_ptr = (uint64_t *) (frag_info + 1); + + frag_size1 = clib_net_to_host_u16 (frag_info->w1.frag_size1); + frag_size2 = clib_net_to_host_u16 (frag_info->w1.frag_size2); + frag_size3 = clib_net_to_host_u16 (frag_info->w1.frag_size3); + + wqe_ptr1 = (u64 *) clib_net_to_host_u64 (hdr->frag1_wqe_ptr); + wqe_ptr2 = (u64 *) clib_net_to_host_u64 (*frag_ptr); + wqe_ptr3 = (u64 *) clib_net_to_host_u64 (*(frag_ptr + 1)); + + b1 = (vlib_buffer_t *) ((u8 *) wqe_ptr1 - 128); + b2 = (vlib_buffer_t *) ((u8 *) wqe_ptr2 - 128); + b3 = (vlib_buffer_t *) ((u8 *) wqe_ptr3 - 128); + + rxp_ptr1 = (oct_nix_rx_parse_t *) (wqe_ptr1 + 1); + rxp_ptr2 = (oct_nix_rx_parse_t *) (wqe_ptr2 + 1); + rxp_ptr3 = (oct_nix_rx_parse_t *) (wqe_ptr3 + 1); + + ASSERT (oct_rx_n_segs (vm, rxp_ptr1) == 1); + ASSERT (oct_rx_n_segs (vm, rxp_ptr2) == 1); + ASSERT (oct_rx_n_segs (vm, rxp_ptr3) == 1); + + b1->template = *bt; + b2->template = *bt; + b3->template = *bt; + + *olen += rxp_ptr1->f.pkt_lenm1 + 1; + *olen += rxp_ptr2->f.pkt_lenm1 + 1; + *olen += rxp_ptr3->f.pkt_lenm1 + 1; + + *esp_len += rxp_ptr1->f.pkt_lenm1 + 1 - l2_ol3_hdr_size; + *esp_len += rxp_ptr2->f.pkt_lenm1 + 1 - l2_ol3_hdr_size; + *esp_len += rxp_ptr3->f.pkt_lenm1 + 1 - l2_ol3_hdr_size; + + b1->current_length = frag_size1; + b2->current_length = frag_size2; + b3->current_length = frag_size3; + b1->current_data = l2_l3_inner_hdr_size; + b2->current_data = l2_l3_inner_hdr_size; + b3->current_data = l2_l3_inner_hdr_size; + + b0->total_length_not_including_first_buffer += b1->current_length; + b0->total_length_not_including_first_buffer += b2->current_length; + b0->total_length_not_including_first_buffer += b3->current_length; + + b0->flags |= VLIB_BUFFER_NEXT_PRESENT; + b1->flags |= VLIB_BUFFER_NEXT_PRESENT; + b2->flags |= VLIB_BUFFER_NEXT_PRESENT; + + b0->next_buffer = vlib_get_buffer_index (vm, b1); + b1->next_buffer = vlib_get_buffer_index (vm, b2); + b2->next_buffer = vlib_get_buffer_index (vm, b3); + + return 4; + } + + return frag_cnt; +} + +/* + * Reassemble failure cases. Read upto 4 fragments. + * Append them to the buffer list. + * Fragments which are further segmented are not + * supported currently. + * */ +static_always_inline u8 +oct_rx_ipsec_reassembly_failure (vlib_main_t *vm, vlib_buffer_template_t *bt, + struct cpt_cn10k_parse_hdr_s *hdr, + oct_nix_rx_cqe_desc_t *d, + vlib_buffer_t **buffs, u16 *next, + u16 *buffer_next_index, u32 *olen, + u32 *esp_len, u32 l2_ol3_hdr_size, + const u64 fp_flags) +{ + oct_nix_rx_parse_t *rxp_ptr2, *rxp_ptr3; + oct_nix_rx_parse_t *rxp_ptr, *rxp_ptr1; + struct cpt_frag_info_s *frag_info; + vlib_buffer_t *b1, *b2, *b3; + u32 l2_sz1, l2_sz2, l2_sz3; + u64 *wqe_ptr2, *wqe_ptr3; + u64 *wqe_ptr, *wqe_ptr1; + u16 rlen1, rlen2, rlen3; + uint64_t *frag_ptr; + uint32_t offset; + u16 next_index = next[*buffer_next_index - 1]; + u8 frag_cnt; + + wqe_ptr = (u64 *) clib_net_to_host_u64 (hdr->wqe_ptr); + rxp_ptr = (oct_nix_rx_parse_t *) (wqe_ptr + 1); + ASSERT (oct_rx_n_segs (vm, rxp_ptr) == 1); + + frag_cnt = hdr->w0.num_frags; + + /* + * fi_offset is 8B offset from cpt_parse_hdr_s + fi_pad to frag_info_s. + * fi_offset 0 indicates 256B. + */ + offset = hdr->w2.fi_offset; + offset = (((offset - 1) & 0x1f) + 1) * 8; + frag_info = PLT_PTR_ADD (hdr, offset); + + if (frag_cnt == 2) + { + wqe_ptr1 = (u64 *) clib_net_to_host_u64 (hdr->frag1_wqe_ptr); + b1 = (vlib_buffer_t *) ((u8 *) wqe_ptr1 - 128); + rxp_ptr1 = (oct_nix_rx_parse_t *) (wqe_ptr1 + 1); + ASSERT (oct_rx_n_segs (vm, rxp_ptr1) == 1); + rlen1 = ((*(wqe_ptr1 + 10)) >> 16) & 0xFFFF; + + oct_rx_verify_vlib (vm, b1); + b1->template = *bt; + + VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b1); + + l2_sz1 = rxp_ptr1->f.lcptr - rxp_ptr1->f.laptr; + b1->current_length = rlen1 + l2_sz1; + b1->current_data = 0; + if (fp_flags & OCT_FP_FLAG_TRACE_EN) + clib_memcpy_fast (b1->pre_data, d, sizeof (oct_nix_rx_cqe_desc_t)); + + *olen += rxp_ptr1->f.pkt_lenm1 + 1; + *esp_len += rxp_ptr1->f.pkt_lenm1 + 1 - l2_ol3_hdr_size; + + buffs[*buffer_next_index] = b1; + next[*buffer_next_index] = next_index; + *buffer_next_index = *buffer_next_index + 1; + + return 2; + } + + if (PREDICT_FALSE (frag_cnt == 3)) + { + frag_ptr = (uint64_t *) (frag_info + 1); + + wqe_ptr1 = (u64 *) clib_net_to_host_u64 (hdr->frag1_wqe_ptr); + wqe_ptr2 = (u64 *) clib_net_to_host_u64 (*frag_ptr); + + b1 = (vlib_buffer_t *) ((u8 *) wqe_ptr1 - 128); + b2 = (vlib_buffer_t *) ((u8 *) wqe_ptr2 - 128); + + rxp_ptr1 = (oct_nix_rx_parse_t *) (wqe_ptr1 + 1); + rxp_ptr2 = (oct_nix_rx_parse_t *) (wqe_ptr2 + 1); + + ASSERT (oct_rx_n_segs (vm, rxp_ptr1) == 1); + ASSERT (oct_rx_n_segs (vm, rxp_ptr2) == 1); + + rlen1 = ((*(wqe_ptr1 + 10)) >> 16) & 0xFFFF; + rlen2 = ((*(wqe_ptr2 + 10)) >> 16) & 0xFFFF; + + oct_rx_verify_vlib (vm, b1); + oct_rx_verify_vlib (vm, b2); + + b1->template = *bt; + b2->template = *bt; + + VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b1); + VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b2); + + l2_sz1 = rxp_ptr1->f.lcptr - rxp_ptr1->f.laptr; + l2_sz2 = rxp_ptr2->f.lcptr - rxp_ptr2->f.laptr; + + b1->current_length = rlen1 + l2_sz1; + b2->current_length = rlen2 + l2_sz2; + b1->current_data = 0; + b2->current_data = 0; + + if (fp_flags & OCT_FP_FLAG_TRACE_EN) + { + clib_memcpy_fast (b1->pre_data, d, sizeof (oct_nix_rx_cqe_desc_t)); + clib_memcpy_fast (b2->pre_data, d, sizeof (oct_nix_rx_cqe_desc_t)); + } + + *olen += rxp_ptr1->f.pkt_lenm1 + 1; + *esp_len += rxp_ptr1->f.pkt_lenm1 + 1 - l2_ol3_hdr_size; + *olen += rxp_ptr2->f.pkt_lenm1 + 1; + *esp_len += rxp_ptr2->f.pkt_lenm1 + 1 - l2_ol3_hdr_size; + + buffs[*buffer_next_index] = b1; + buffs[*buffer_next_index + 1] = b2; + next[*buffer_next_index] = next_index; + next[*buffer_next_index + 1] = next_index; + *buffer_next_index = *buffer_next_index + 2; + + return 3; + } + + if (PREDICT_FALSE (frag_cnt == 4)) + { + frag_ptr = (uint64_t *) (frag_info + 1); + + wqe_ptr1 = (u64 *) clib_net_to_host_u64 (hdr->frag1_wqe_ptr); + wqe_ptr2 = (u64 *) clib_net_to_host_u64 (*frag_ptr); + wqe_ptr3 = (u64 *) clib_net_to_host_u64 (*(frag_ptr + 1)); + b1 = (vlib_buffer_t *) ((u8 *) wqe_ptr1 - 128); + b2 = (vlib_buffer_t *) ((u8 *) wqe_ptr2 - 128); + b3 = (vlib_buffer_t *) ((u8 *) wqe_ptr3 - 128); + rxp_ptr1 = (oct_nix_rx_parse_t *) (wqe_ptr1 + 1); + rxp_ptr2 = (oct_nix_rx_parse_t *) (wqe_ptr2 + 1); + rxp_ptr3 = (oct_nix_rx_parse_t *) (wqe_ptr3 + 1); + ASSERT (oct_rx_n_segs (vm, rxp_ptr1) == 1); + ASSERT (oct_rx_n_segs (vm, rxp_ptr2) == 1); + ASSERT (oct_rx_n_segs (vm, rxp_ptr3) == 1); + rlen1 = ((*(wqe_ptr1 + 10)) >> 16) & 0xFFFF; + rlen2 = ((*(wqe_ptr2 + 10)) >> 16) & 0xFFFF; + rlen3 = ((*(wqe_ptr3 + 10)) >> 16) & 0xFFFF; + + oct_rx_verify_vlib (vm, b1); + oct_rx_verify_vlib (vm, b2); + oct_rx_verify_vlib (vm, b3); + + b1->template = *bt; + b2->template = *bt; + b3->template = *bt; + + VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b1); + VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b2); + VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b3); + + l2_sz1 = rxp_ptr1->f.lcptr - rxp_ptr1->f.laptr; + l2_sz2 = rxp_ptr2->f.lcptr - rxp_ptr2->f.laptr; + l2_sz3 = rxp_ptr3->f.lcptr - rxp_ptr3->f.laptr; + + b1->current_length = rlen1 + l2_sz1; + b2->current_length = rlen2 + l2_sz2; + b3->current_length = rlen3 + l2_sz3; + b1->current_data = 0; + b2->current_data = 0; + b3->current_data = 0; + + if (fp_flags & OCT_FP_FLAG_TRACE_EN) + { + clib_memcpy_fast (b1->pre_data, d, sizeof (oct_nix_rx_cqe_desc_t)); + clib_memcpy_fast (b2->pre_data, d, sizeof (oct_nix_rx_cqe_desc_t)); + clib_memcpy_fast (b3->pre_data, d, sizeof (oct_nix_rx_cqe_desc_t)); + } + + *olen += rxp_ptr1->f.pkt_lenm1 + 1; + *olen += rxp_ptr2->f.pkt_lenm1 + 1; + *olen += rxp_ptr3->f.pkt_lenm1 + 1; + *esp_len += rxp_ptr1->f.pkt_lenm1 + 1 - l2_ol3_hdr_size; + *esp_len += rxp_ptr2->f.pkt_lenm1 + 1 - l2_ol3_hdr_size; + *esp_len += rxp_ptr3->f.pkt_lenm1 + 1 - l2_ol3_hdr_size; + + buffs[*buffer_next_index] = b1; + buffs[*buffer_next_index + 1] = b2; + buffs[*buffer_next_index + 2] = b3; + next[*buffer_next_index] = next_index; + next[*buffer_next_index + 1] = next_index; + next[*buffer_next_index + 2] = next_index; + *buffer_next_index = *buffer_next_index + 3; + + return 4; + } + + return frag_cnt; +} + +static_always_inline u8 +oct_rx_ipsec_reassembly (vlib_main_t *vm, vlib_buffer_template_t *bt, + struct cpt_cn10k_parse_hdr_s *cpt_hdr, + oct_nix_rx_cqe_desc_t *d, vlib_buffer_t *b, + vlib_buffer_t **buf, u16 *next, + u16 *buffer_next_index, u32 *olen, u32 *esp_len, + u32 l2_ol3_hdr_size, const u64 fp_flags) +{ + if ((cpt_hdr->w0.num_frags) && !(cpt_hdr->w0.reas_sts)) + return oct_rx_ipsec_reassembly_success (vm, bt, cpt_hdr, d, b, olen, + esp_len, l2_ol3_hdr_size); + else if (cpt_hdr->w0.reas_sts) + return oct_rx_ipsec_reassembly_failure (vm, bt, cpt_hdr, d, buf, next, + buffer_next_index, olen, esp_len, + l2_ol3_hdr_size, fp_flags); + else + return 1; +} + static_always_inline u32 oct_ipsec_update_itf_sw_idx (oct_ipsec_session_t *session, u32 sa_idx) { @@ -524,17 +935,18 @@ oct_rx_ipsec_attach_tail (vlib_main_t *vm, oct_rx_node_ctx_t *ctx, } static_always_inline u32 -oct_rx_inl_ipsec_vlib_from_cq (vlib_main_t *vm, vlib_node_runtime_t *node, - oct_nix_rx_cqe_desc_t *d, vlib_buffer_t **b, - oct_rx_node_ctx_t *ctx, - vlib_buffer_template_t *bt, - struct cpt_cn10k_parse_hdr_s *cpt_hdr, - vlib_buffer_t **buffs, u32 *err_flags) +oct_rx_inl_ipsec_vlib_from_cq ( + vlib_main_t *vm, vlib_node_runtime_t *node, oct_nix_rx_cqe_desc_t *d, + vlib_buffer_t **b, oct_rx_node_ctx_t *ctx, vlib_buffer_template_t *bt, + struct cpt_cn10k_parse_hdr_s *cpt_hdr, vlib_buffer_t **buffs, u32 *err_flags, + u16 *next, u16 *buffer_next_index, const u64 fp_flags) { - union nix_rx_parse_u *orig_rxp; + union nix_rx_parse_u *orig_rxp, *rxp; u32 is_fail, olen, esp_sz, l2_ol3_sz, idx; u64 *wqe_ptr; + u8 frag_cnt; + rxp = &d->parse.f; cpt_hdr = (struct cpt_cn10k_parse_hdr_s *) *(((u64 *) d) + 9); wqe_ptr = (u64 *) clib_net_to_host_u64 (cpt_hdr->wqe_ptr); @@ -543,7 +955,6 @@ oct_rx_inl_ipsec_vlib_from_cq (vlib_main_t *vm, vlib_node_runtime_t *node, l2_ol3_sz = orig_rxp->leptr - orig_rxp->laptr; olen = orig_rxp->pkt_lenm1 + 1; esp_sz = olen - l2_ol3_sz; - ctx->to_next[0] = vlib_get_buffer_index (vm, b[0]); b[0]->template = *bt; b[0]->flow_id = d[0].parse.w[3] >> 48; *err_flags |= ((d[0].parse.w[0] >> 20) & 0xFFF); @@ -551,24 +962,36 @@ oct_rx_inl_ipsec_vlib_from_cq (vlib_main_t *vm, vlib_node_runtime_t *node, is_fail = !oct_ipsec_is_inl_op_success (cpt_hdr); + buffs[*buffer_next_index] = b[0]; if (PREDICT_FALSE (is_fail)) { b[0]->current_length = olen; - ctx->next[0] = VNET_DEV_ETH_RX_PORT_NEXT_DROP; + clib_memcpy_fast (rxp, orig_rxp, sizeof (oct_nix_rx_parse_t)); + next[*buffer_next_index] = VNET_DEV_ETH_RX_PORT_NEXT_DROP; + *buffer_next_index = *buffer_next_index + 1; oct_rx_ipsec_set_error (vm, node, b[0], cpt_hdr->w3.uc_ccode); + frag_cnt = 1; } else { - ctx->next[0] = ctx->next_index; + next[*buffer_next_index] = ctx->next_index; + *buffer_next_index = *buffer_next_index + 1; b[0]->current_length = oct_get_len_from_meta (cpt_hdr, d[0].parse.w[0], d[0].parse.w[4]); + + frag_cnt = oct_rx_ipsec_reassembly (vm, bt, cpt_hdr, &d[0], b[0], buffs, + next, buffer_next_index, &olen, + &esp_sz, l2_ol3_sz, fp_flags); + idx = cpt_hdr->w0.cookie; - oct_rx_ipsec_update_counters (vm, b[0], esp_sz, 1, idx); + oct_rx_ipsec_update_counters (vm, b[0], esp_sz, frag_cnt, idx); oct_rx_ipsec_attach_tail (vm, ctx, orig_rxp, bt, b[0]); } ctx->n_rx_bytes += olen; + ctx->n_segs += frag_cnt; - buffs[0] = b[0]; + if (fp_flags & OCT_FP_FLAG_TRACE_EN) + clib_memcpy_fast (b[0]->pre_data, d, sizeof (oct_nix_rx_cqe_desc_t)); return 0; } @@ -576,12 +999,11 @@ oct_rx_inl_ipsec_vlib_from_cq (vlib_main_t *vm, vlib_node_runtime_t *node, static_always_inline u32 oct_rx_vlib_from_cq (vlib_main_t *vm, oct_nix_rx_cqe_desc_t *d, vlib_buffer_t **b, oct_rx_node_ctx_t *ctx, - vlib_buffer_template_t *bt, u64 meta_aura_handle, - vlib_buffer_t **buffs, u32 *err_flags) + vlib_buffer_template_t *bt, vlib_buffer_t **buffs, + u32 *err_flags, u16 *next, u16 *buffer_next_index, + const u64 fp_flags) { b[0] = (vlib_buffer_t *) d->segs0[0] - 1; - ctx->to_next[0] = vlib_get_buffer_index (vm, b[0]); - ctx->next[0] = ctx->next_index; b[0]->template = *bt; ctx->n_rx_bytes += b[0]->current_length = d[0].sg0.seg1_size; b[0]->flow_id = d[0].parse.w[3] >> 48; @@ -589,7 +1011,9 @@ oct_rx_vlib_from_cq (vlib_main_t *vm, oct_nix_rx_cqe_desc_t *d, ctx->n_segs += 1; if (d[0].sg0.segs > 1) oct_rx_attach_tail (vm, ctx, bt, b[0], d + 0); - buffs[0] = b[0]; + buffs[*buffer_next_index] = b[0]; + next[*buffer_next_index] = ctx->next_index; + *buffer_next_index = *buffer_next_index + 1; return 0; } @@ -625,7 +1049,7 @@ oct_rx_flush_meta_burst (u16 lmt_id, u64 data, u16 lnum, uintptr_t aura_handle) static_always_inline u32 oct_rx_batch (vlib_main_t *vm, vlib_node_runtime_t *node, oct_rx_node_ctx_t *ctx, vnet_dev_rx_queue_t *rxq, u32 n, - vlib_buffer_t **buffs) + vlib_buffer_t **buffers, const u64 fp_flags) { oct_rxq_t *crq = vnet_dev_get_rx_queue_data (rxq); vlib_buffer_template_t bt = rxq->buffer_template; @@ -635,6 +1059,8 @@ oct_rx_batch (vlib_main_t *vm, vlib_node_runtime_t *node, oct_nix_rx_cqe_desc_t *d = ctx->next_desc; struct cpt_cn10k_parse_hdr_s *cpt_hdr0, *cpt_hdr1; struct cpt_cn10k_parse_hdr_s *cpt_hdr2, *cpt_hdr3; + union nix_rx_parse_u *rxp0, *rxp1; + union nix_rx_parse_u *rxp2, *rxp3; union nix_rx_parse_u *orig_rxp0, *orig_rxp1; union nix_rx_parse_u *orig_rxp2, *orig_rxp3; u8 is_b0_from_cpt, is_b1_from_cpt; @@ -647,11 +1073,16 @@ oct_rx_batch (vlib_main_t *vm, vlib_node_runtime_t *node, u32 l2_ol3_sz0, l2_ol3_sz1, l2_ol3_sz2, l2_ol3_sz3; u32 idx0, idx1, idx2, idx3; vlib_buffer_t *b[4]; + vlib_buffer_t **buffs = buffers + ctx->buffer_start_index; + u16 *next = ctx->next + ctx->buffer_start_index; + u8 frag_cnt0 = 1, frag_cnt1 = 1; + u8 frag_cnt2 = 1, frag_cnt3 = 1; u8 n_from_cpt, n_cpt_err; u64 meta_aura_handle; u64 lbase = crq->lmt_base_addr; u8 loff = 0, lnum = 0, shft = 0; - u16 lmt_id; + u16 lmt_id, buffer_next_index = 0; + u16 cqe_desc_bytes = sizeof (oct_nix_rx_cqe_desc_t); u64 laddr; meta_aura_handle = crq->rq.meta_aura_handle; @@ -659,8 +1090,7 @@ oct_rx_batch (vlib_main_t *vm, vlib_node_runtime_t *node, laddr = lbase; laddr += 8; - for (n_left = n; n_left >= 8; - d += 4, n_left -= 4, ctx->to_next += 4, ctx->next += 4) + for (n_left = n; n_left >= 8; d += 4, n_left -= 4) { u32 segs = 0; clib_prefetch_store (oct_seg_to_bp (d[4].segs0[0])); @@ -672,6 +1102,11 @@ oct_rx_batch (vlib_main_t *vm, vlib_node_runtime_t *node, b[2] = oct_seg_to_bp (d[2].segs0[0]); b[3] = oct_seg_to_bp (d[3].segs0[0]); + rxp0 = &d[0].parse.f; + rxp1 = &d[1].parse.f; + rxp2 = &d[2].parse.f; + rxp3 = &d[3].parse.f; + is_b0_from_cpt = oct_is_packet_from_cpt (&d[0].parse.f); is_b1_from_cpt = oct_is_packet_from_cpt (&d[1].parse.f); is_b2_from_cpt = oct_is_packet_from_cpt (&d[2].parse.f); @@ -681,15 +1116,10 @@ oct_rx_batch (vlib_main_t *vm, vlib_node_runtime_t *node, is_b0_from_cpt + is_b1_from_cpt + is_b2_from_cpt + is_b3_from_cpt; if (n_from_cpt == 0) { - ctx->to_next[0] = vlib_get_buffer_index (vm, b[0]); - ctx->to_next[1] = vlib_get_buffer_index (vm, b[1]); - ctx->to_next[2] = vlib_get_buffer_index (vm, b[2]); - ctx->to_next[3] = vlib_get_buffer_index (vm, b[3]); - - ctx->next[0] = ctx->next_index; - ctx->next[1] = ctx->next_index; - ctx->next[2] = ctx->next_index; - ctx->next[3] = ctx->next_index; + next[buffer_next_index + 0] = ctx->next_index; + next[buffer_next_index + 1] = ctx->next_index; + next[buffer_next_index + 2] = ctx->next_index; + next[buffer_next_index + 3] = ctx->next_index; b[0]->template = bt; b[1]->template = bt; @@ -714,6 +1144,14 @@ oct_rx_batch (vlib_main_t *vm, vlib_node_runtime_t *node, err_flags |= b0_err_flags | b1_err_flags | b2_err_flags | b3_err_flags; + if (fp_flags & OCT_FP_FLAG_TRACE_EN) + { + clib_memcpy_fast (b[0]->pre_data, &d[0], cqe_desc_bytes); + clib_memcpy_fast (b[1]->pre_data, &d[1], cqe_desc_bytes); + clib_memcpy_fast (b[2]->pre_data, &d[2], cqe_desc_bytes); + clib_memcpy_fast (b[3]->pre_data, &d[3], cqe_desc_bytes); + } + ctx->n_segs += 4; segs = d[0].sg0.segs + d[1].sg0.segs + d[2].sg0.segs + d[3].sg0.segs; @@ -725,10 +1163,12 @@ oct_rx_batch (vlib_main_t *vm, vlib_node_runtime_t *node, oct_rx_attach_tail (vm, ctx, &bt, b[3], d + 3); } - buffs[0] = b[0]; - buffs[1] = b[1]; - buffs[2] = b[2]; - buffs[3] = b[3]; + buffs[buffer_next_index + 0] = b[0]; + buffs[buffer_next_index + 1] = b[1]; + buffs[buffer_next_index + 2] = b[2]; + buffs[buffer_next_index + 3] = b[3]; + + buffer_next_index += 4; } else if (n_from_cpt == 4) { @@ -768,11 +1208,6 @@ oct_rx_batch (vlib_main_t *vm, vlib_node_runtime_t *node, esp_sz2 = olen2 - l2_ol3_sz2; esp_sz3 = olen3 - l2_ol3_sz3; - ctx->to_next[0] = vlib_get_buffer_index (vm, b[0]); - ctx->to_next[1] = vlib_get_buffer_index (vm, b[1]); - ctx->to_next[2] = vlib_get_buffer_index (vm, b[2]); - ctx->to_next[3] = vlib_get_buffer_index (vm, b[3]); - b[0]->template = bt; b[1]->template = bt; b[2]->template = bt; @@ -786,11 +1221,6 @@ oct_rx_batch (vlib_main_t *vm, vlib_node_runtime_t *node, if (PREDICT_TRUE (!n_cpt_err)) { - ctx->next[0] = ctx->next_index; - ctx->next[1] = ctx->next_index; - ctx->next[2] = ctx->next_index; - ctx->next[3] = ctx->next_index; - b[0]->current_length = oct_get_len_from_meta ( cpt_hdr0, d[0].parse.w[0], d[0].parse.w[4]); b[1]->current_length = oct_get_len_from_meta ( @@ -800,14 +1230,43 @@ oct_rx_batch (vlib_main_t *vm, vlib_node_runtime_t *node, b[3]->current_length = oct_get_len_from_meta ( cpt_hdr3, d[3].parse.w[0], d[3].parse.w[4]); + next[buffer_next_index] = ctx->next_index; + buffs[buffer_next_index] = b[0]; + buffer_next_index += 1; + frag_cnt0 = oct_rx_ipsec_reassembly ( + vm, &bt, cpt_hdr0, &d[0], b[0], buffs, next, + &buffer_next_index, &olen0, &esp_sz0, l2_ol3_sz0, fp_flags); + + buffs[buffer_next_index] = b[1]; + next[buffer_next_index] = ctx->next_index; + buffer_next_index += 1; + frag_cnt1 = oct_rx_ipsec_reassembly ( + vm, &bt, cpt_hdr1, &d[1], b[1], buffs, next, + &buffer_next_index, &olen1, &esp_sz1, l2_ol3_sz1, fp_flags); + + buffs[buffer_next_index] = b[2]; + next[buffer_next_index] = ctx->next_index; + buffer_next_index += 1; + frag_cnt2 = oct_rx_ipsec_reassembly ( + vm, &bt, cpt_hdr2, &d[2], b[2], buffs, next, + &buffer_next_index, &olen2, &esp_sz2, l2_ol3_sz2, fp_flags); + + buffs[buffer_next_index] = b[3]; + next[buffer_next_index] = ctx->next_index; + buffer_next_index += 1; + frag_cnt3 = oct_rx_ipsec_reassembly ( + vm, &bt, cpt_hdr3, &d[3], b[3], buffs, next, + &buffer_next_index, &olen3, &esp_sz3, l2_ol3_sz3, fp_flags); + idx0 = cpt_hdr0->w0.cookie; idx1 = cpt_hdr1->w0.cookie; idx2 = cpt_hdr2->w0.cookie; idx3 = cpt_hdr3->w0.cookie; oct_rx_ipsec_update_counters_x4 ( - vm, b[0], esp_sz0, 1, idx0, b[1], esp_sz1, 1, idx1, b[2], - esp_sz2, 1, idx2, b[3], esp_sz3, 1, idx3); + vm, b[0], esp_sz0, frag_cnt0, idx0, b[1], esp_sz1, frag_cnt1, + idx1, b[2], esp_sz2, frag_cnt2, idx2, b[3], esp_sz3, frag_cnt3, + idx3); oct_rx_ipsec_attach_tail (vm, ctx, orig_rxp0, &bt, b[0]); oct_rx_ipsec_attach_tail (vm, ctx, orig_rxp1, &bt, b[1]); @@ -816,75 +1275,127 @@ oct_rx_batch (vlib_main_t *vm, vlib_node_runtime_t *node, } else { + buffs[buffer_next_index] = b[0]; + if (is_fail0) { b[0]->current_length = olen0; - ctx->next[0] = VNET_DEV_ETH_RX_PORT_NEXT_DROP; + clib_memcpy_fast (rxp0, orig_rxp0, + sizeof (oct_nix_rx_parse_t)); + next[buffer_next_index] = VNET_DEV_ETH_RX_PORT_NEXT_DROP; + buffer_next_index += 1; oct_rx_ipsec_set_error (vm, node, b[0], cpt_hdr0->w3.uc_ccode); + frag_cnt0 = 1; } else { - ctx->next[0] = ctx->next_index; + next[buffer_next_index] = ctx->next_index; + buffer_next_index += 1; b[0]->current_length = oct_get_len_from_meta ( cpt_hdr0, d[0].parse.w[0], d[0].parse.w[4]); + + frag_cnt0 = oct_rx_ipsec_reassembly ( + vm, &bt, cpt_hdr0, &d[0], b[0], buffs, next, + &buffer_next_index, &olen0, &esp_sz0, l2_ol3_sz0, + fp_flags); + idx0 = cpt_hdr0->w0.cookie; - oct_rx_ipsec_update_counters (vm, b[0], esp_sz0, 1, idx0); + oct_rx_ipsec_update_counters (vm, b[0], esp_sz0, frag_cnt0, + idx0); oct_rx_ipsec_attach_tail (vm, ctx, orig_rxp0, &bt, b[0]); } + buffs[buffer_next_index] = b[1]; + if (is_fail1) { b[1]->current_length = olen1; - ctx->next[1] = VNET_DEV_ETH_RX_PORT_NEXT_DROP; + clib_memcpy_fast (rxp1, orig_rxp1, + sizeof (oct_nix_rx_parse_t)); + next[buffer_next_index] = VNET_DEV_ETH_RX_PORT_NEXT_DROP; + buffer_next_index += 1; oct_rx_ipsec_set_error (vm, node, b[1], cpt_hdr1->w3.uc_ccode); + frag_cnt1 = 1; } else { - ctx->next[1] = ctx->next_index; + next[buffer_next_index] = ctx->next_index; + buffer_next_index += 1; b[1]->current_length = oct_get_len_from_meta ( cpt_hdr1, d[1].parse.w[0], d[1].parse.w[4]); + frag_cnt1 = oct_rx_ipsec_reassembly ( + vm, &bt, cpt_hdr1, &d[1], b[1], buffs, next, + &buffer_next_index, &olen1, &esp_sz1, l2_ol3_sz1, + fp_flags); idx1 = cpt_hdr1->w0.cookie; - oct_rx_ipsec_update_counters (vm, b[1], esp_sz1, 1, idx1); + oct_rx_ipsec_update_counters (vm, b[1], esp_sz1, frag_cnt1, + idx1); oct_rx_ipsec_attach_tail (vm, ctx, orig_rxp1, &bt, b[1]); } + buffs[buffer_next_index] = b[2]; + if (is_fail2) { b[2]->current_length = olen2; - ctx->next[2] = VNET_DEV_ETH_RX_PORT_NEXT_DROP; + clib_memcpy_fast (rxp2, orig_rxp2, + sizeof (oct_nix_rx_parse_t)); + next[buffer_next_index] = VNET_DEV_ETH_RX_PORT_NEXT_DROP; + buffer_next_index += 1; oct_rx_ipsec_set_error (vm, node, b[2], cpt_hdr2->w3.uc_ccode); + frag_cnt2 = 1; } else { - ctx->next[2] = ctx->next_index; + next[buffer_next_index] = ctx->next_index; + buffer_next_index += 1; b[2]->current_length = oct_get_len_from_meta ( cpt_hdr2, d[2].parse.w[0], d[2].parse.w[4]); + frag_cnt2 = oct_rx_ipsec_reassembly ( + vm, &bt, cpt_hdr2, &d[2], b[2], buffs, next, + &buffer_next_index, &olen2, &esp_sz2, l2_ol3_sz2, + fp_flags); idx2 = cpt_hdr2->w0.cookie; - oct_rx_ipsec_update_counters (vm, b[2], esp_sz2, 1, idx2); + oct_rx_ipsec_update_counters (vm, b[2], esp_sz2, frag_cnt2, + idx2); oct_rx_ipsec_attach_tail (vm, ctx, orig_rxp2, &bt, b[2]); } + buffs[buffer_next_index] = b[3]; + if (is_fail3) { b[3]->current_length = olen3; - ctx->next[3] = VNET_DEV_ETH_RX_PORT_NEXT_DROP; + clib_memcpy_fast (rxp3, orig_rxp3, + sizeof (oct_nix_rx_parse_t)); + next[buffer_next_index] = VNET_DEV_ETH_RX_PORT_NEXT_DROP; + buffer_next_index += 1; oct_rx_ipsec_set_error (vm, node, b[3], cpt_hdr3->w3.uc_ccode); + frag_cnt3 = 1; } else { - ctx->next[3] = ctx->next_index; + next[buffer_next_index] = ctx->next_index; + buffer_next_index += 1; b[3]->current_length = oct_get_len_from_meta ( cpt_hdr3, d[3].parse.w[0], d[3].parse.w[4]); + + frag_cnt3 = oct_rx_ipsec_reassembly ( + vm, &bt, cpt_hdr3, &d[3], b[3], buffs, next, + &buffer_next_index, &olen3, &esp_sz3, l2_ol3_sz3, + fp_flags); idx3 = cpt_hdr3->w0.cookie; - oct_rx_ipsec_update_counters (vm, b[3], esp_sz3, 1, idx3); + oct_rx_ipsec_update_counters (vm, b[3], esp_sz3, frag_cnt3, + idx3); oct_rx_ipsec_attach_tail (vm, ctx, orig_rxp3, &bt, b[3]); } } ctx->n_rx_bytes += olen0 + olen1 + olen2 + olen3; + ctx->n_segs += frag_cnt0 + frag_cnt1 + frag_cnt2 + frag_cnt3; b[0]->flow_id = d[0].parse.w[3] >> 48; b[1]->flow_id = d[1].parse.w[3] >> 48; @@ -899,17 +1410,18 @@ oct_rx_batch (vlib_main_t *vm, vlib_node_runtime_t *node, err_flags |= b0_err_flags | b1_err_flags | b2_err_flags | b3_err_flags; - ctx->n_segs += 4; - OCT_PUSH_META_TO_FREE ((u64) cpt_hdr0, laddr, &loff); OCT_PUSH_META_TO_FREE ((u64) cpt_hdr1, laddr, &loff); OCT_PUSH_META_TO_FREE ((u64) cpt_hdr2, laddr, &loff); OCT_PUSH_META_TO_FREE ((u64) cpt_hdr3, laddr, &loff); - buffs[0] = b[0]; - buffs[1] = b[1]; - buffs[2] = b[2]; - buffs[3] = b[3]; + if (fp_flags & OCT_FP_FLAG_TRACE_EN) + { + clib_memcpy_fast (b[0]->pre_data, &d[0], cqe_desc_bytes); + clib_memcpy_fast (b[1]->pre_data, &d[1], cqe_desc_bytes); + clib_memcpy_fast (b[2]->pre_data, &d[2], cqe_desc_bytes); + clib_memcpy_fast (b[3]->pre_data, &d[3], cqe_desc_bytes); + } } else { @@ -919,48 +1431,51 @@ oct_rx_batch (vlib_main_t *vm, vlib_node_runtime_t *node, cpt_hdr0 = (struct cpt_cn10k_parse_hdr_s *) *(((u64 *) &d[0]) + 9); oct_rx_inl_ipsec_vlib_from_cq (vm, node, &d[0], &b[0], ctx, &bt, - cpt_hdr0, &buffs[0], &err_flags); + cpt_hdr0, buffs, &err_flags, next, + &buffer_next_index, fp_flags); OCT_PUSH_META_TO_FREE ((u64) cpt_hdr0, laddr, &loff); } else - oct_rx_vlib_from_cq (vm, &d[0], &b[0], ctx, &bt, meta_aura_handle, - &buffs[0], &err_flags); + oct_rx_vlib_from_cq (vm, &d[0], &b[0], ctx, &bt, buffs, &err_flags, + next, &buffer_next_index, fp_flags); if (is_b1_from_cpt) { cpt_hdr1 = (struct cpt_cn10k_parse_hdr_s *) *(((u64 *) &d[1]) + 9); oct_rx_inl_ipsec_vlib_from_cq (vm, node, &d[1], &b[1], ctx, &bt, - cpt_hdr1, &buffs[1], &err_flags); + cpt_hdr1, buffs, &err_flags, next, + &buffer_next_index, fp_flags); OCT_PUSH_META_TO_FREE ((u64) cpt_hdr1, laddr, &loff); } else - oct_rx_vlib_from_cq (vm, &d[1], &b[1], ctx, &bt, meta_aura_handle, - &buffs[1], &err_flags); + oct_rx_vlib_from_cq (vm, &d[1], &b[1], ctx, &bt, buffs, &err_flags, + next, &buffer_next_index, fp_flags); if (is_b2_from_cpt) { cpt_hdr2 = (struct cpt_cn10k_parse_hdr_s *) *(((u64 *) &d[2]) + 9); oct_rx_inl_ipsec_vlib_from_cq (vm, node, &d[2], &b[2], ctx, &bt, - cpt_hdr2, &buffs[2], &err_flags); + cpt_hdr2, buffs, &err_flags, next, + &buffer_next_index, fp_flags); OCT_PUSH_META_TO_FREE ((u64) cpt_hdr2, laddr, &loff); } else - oct_rx_vlib_from_cq (vm, &d[2], &b[2], ctx, &bt, meta_aura_handle, - &buffs[2], &err_flags); + oct_rx_vlib_from_cq (vm, &d[2], &b[2], ctx, &bt, buffs, &err_flags, + next, &buffer_next_index, fp_flags); if (is_b3_from_cpt) { cpt_hdr3 = (struct cpt_cn10k_parse_hdr_s *) *(((u64 *) &d[3]) + 9); oct_rx_inl_ipsec_vlib_from_cq (vm, node, &d[3], &b[3], ctx, &bt, - cpt_hdr3, &buffs[3], &err_flags); + cpt_hdr3, buffs, &err_flags, next, + &buffer_next_index, fp_flags); OCT_PUSH_META_TO_FREE ((u64) cpt_hdr3, laddr, &loff); } else - oct_rx_vlib_from_cq (vm, &d[3], &b[3], ctx, &bt, meta_aura_handle, - &buffs[3], &err_flags); + oct_rx_vlib_from_cq (vm, &d[3], &b[3], ctx, &bt, buffs, &err_flags, + next, &buffer_next_index, fp_flags); } - buffs += 4; /* Check if lmtline border is crossed and adjust lnum */ if (loff > 15) { @@ -1026,21 +1541,21 @@ oct_rx_batch (vlib_main_t *vm, vlib_node_runtime_t *node, laddr = (uintptr_t) LMT_OFF (lbase, lnum, 8); } - for (; n_left; d += 1, n_left -= 1, ctx->to_next += 1, ctx->next += 1) + for (; n_left; d += 1, n_left -= 1) { is_b0_from_cpt = oct_is_packet_from_cpt (&d[0].parse.f); if (is_b0_from_cpt) { cpt_hdr0 = (struct cpt_cn10k_parse_hdr_s *) *(((u64 *) &d[0]) + 9); oct_rx_inl_ipsec_vlib_from_cq (vm, node, &d[0], &b[0], ctx, &bt, - cpt_hdr0, buffs, &err_flags); + cpt_hdr0, buffs, &err_flags, next, + &buffer_next_index, fp_flags); OCT_PUSH_META_TO_FREE ((u64) cpt_hdr0, laddr, &loff); } else - oct_rx_vlib_from_cq (vm, &d[0], &b[0], ctx, &bt, meta_aura_handle, - buffs, &err_flags); - buffs += 1; + oct_rx_vlib_from_cq (vm, &d[0], &b[0], ctx, &bt, buffs, &err_flags, + next, &buffer_next_index, fp_flags); } if (loff) { @@ -1059,12 +1574,13 @@ oct_rx_batch (vlib_main_t *vm, vlib_node_runtime_t *node, } plt_write64 ((crq->cq.wdata | n), crq->cq.door); - ctx->n_rx_pkts += n; - ctx->n_left_to_next -= n; + ctx->n_rx_pkts += buffer_next_index; + ctx->n_left_to_next -= buffer_next_index; + ctx->buffer_start_index += buffer_next_index; if (err_flags) ctx->parse_w0_or = (err_flags << 20); - return n; + return buffer_next_index; } #ifdef PLATFORM_OCTEON9 @@ -1215,7 +1731,7 @@ oct_rx_trace (vlib_main_t *vm, vlib_node_runtime_t *node, oct_rx_trace_t *tr = vlib_add_trace (vm, node, b, sizeof (*tr)); tr->next_index = ctx->next_index; tr->sw_if_index = ctx->sw_if_index; - tr->desc = d[i]; + tr->desc = *(oct_nix_rx_cqe_desc_t *) b->pre_data; ctx->n_traced++; } i++; @@ -1254,14 +1770,14 @@ oct_rx_node_inline (vlib_main_t *vm, vlib_node_runtime_t *node, vnet_main_t *vnm = vnet_get_main (); u32 thr_idx = vlib_get_thread_index (); oct_rxq_t *crq = vnet_dev_get_rx_queue_data (rxq); - u32 n_desc, head, n, n_enq; + u32 n_desc, head, n, n_enq, n_processed = 0; u32 cq_size = crq->cq.nb_desc; u32 cq_mask = crq->cq.qmask; oct_nix_rx_cqe_desc_t *descs = crq->cq.desc_base; oct_nix_lf_cq_op_status_t status; - u32 to_next[VLIB_FRAME_SIZE]; - u16 next[VLIB_FRAME_SIZE]; - vlib_buffer_t *buffs[256]; + u32 to_next[OCT_FRAME_SIZE]; + u16 next[OCT_FRAME_SIZE]; + vlib_buffer_t *buffs[OCT_FRAME_SIZE]; u8 is_single_next = 1; oct_rx_node_ctx_t _ctx = { .next_index = rxq->next_index, @@ -1269,7 +1785,8 @@ oct_rx_node_inline (vlib_main_t *vm, vlib_node_runtime_t *node, .hw_if_index = port->intf.hw_if_index, .to_next = to_next, .next = next, - .n_left_to_next = VLIB_FRAME_SIZE, + .n_left_to_next = OCT_FRAME_SIZE, + .buffer_start_index = 0, }, *ctx = &_ctx; /* get head and tail from NIX_LF_CQ_OP_STATUS */ @@ -1288,11 +1805,16 @@ oct_rx_node_inline (vlib_main_t *vm, vlib_node_runtime_t *node, while (1) { ctx->next_desc = descs + head; - n = clib_min (cq_size - head, clib_min (n_desc, ctx->n_left_to_next)); - n = oct_rx_batch (vm, node, ctx, rxq, n, buffs); - oct_rx_trace (vm, node, ctx, descs + head, n, buffs); + n = + clib_min (cq_size - head, clib_min (n_desc, ctx->n_left_to_next / 4)); - if (ctx->n_left_to_next == 0) + if (PREDICT_TRUE (ctx->trace_count == 0)) + n_processed += oct_rx_batch (vm, node, ctx, rxq, n, buffs, 0); + else + n_processed += + oct_rx_batch (vm, node, ctx, rxq, n, buffs, OCT_FP_FLAG_TRACE_EN); + + if (n_processed >= 256) break; status.as_u64 = roc_atomic64_add_sync (crq->cq.wdata, crq->cq.status); @@ -1304,9 +1826,14 @@ oct_rx_node_inline (vlib_main_t *vm, vlib_node_runtime_t *node, if (n_desc == 0) break; } - ctx->to_next = to_next; + + vlib_get_buffer_indices_with_offset (vm, (void **) buffs, ctx->to_next, + n_processed, 0); + ctx->next = next; + oct_rx_trace (vm, node, ctx, descs + head, n_processed, buffs); + oct_rx_enq_to_next (vm, node, ctx, &is_single_next); if (ctx->n_traced) diff --git a/src/vnet/ipsec/ipsec_cli.c b/src/vnet/ipsec/ipsec_cli.c index 5aef630a33..0ce186046a 100644 --- a/src/vnet/ipsec/ipsec_cli.c +++ b/src/vnet/ipsec/ipsec_cli.c @@ -168,6 +168,8 @@ ipsec_sa_add_del_command_fn (vlib_main_t * vm, flags |= IPSEC_SA_FLAG_UDP_ENCAP; else if (unformat (line_input, "async")) flags |= IPSEC_SA_FLAG_IS_ASYNC; + else if (unformat (line_input, "inline-reassembly")) + flags |= IPSEC_SA_FLAG_IS_INL_REASSEMBLY; else { error = clib_error_return (0, "parse error: '%U'", diff --git a/src/vnet/ipsec/ipsec_sa.h b/src/vnet/ipsec/ipsec_sa.h index 4f73f1eab0..7e38925692 100644 --- a/src/vnet/ipsec/ipsec_sa.h +++ b/src/vnet/ipsec/ipsec_sa.h @@ -122,7 +122,8 @@ typedef struct ipsec_key_t_ _ (512, IS_ASYNC, "async") \ _ (1024, NO_ALGO_NO_DROP, "no-algo-no-drop") \ _ (2048, IS_NULL_GMAC, "null-gmac") \ - _ (4096, ANTI_REPLAY_HUGE, "anti-replay-huge") + _ (4096, ANTI_REPLAY_HUGE, "anti-replay-huge") \ + _ (8192, IS_INL_REASSEMBLY, "inline-reassembly") typedef enum ipsec_sad_flags_t_ { From f5e13cf531585d7bc6ffb4a861c15412b555065a Mon Sep 17 00:00:00 2001 From: Monendra Singh Kushwaha Date: Tue, 4 Mar 2025 06:14:34 +0530 Subject: [PATCH 196/271] build: update roc version Type: feature Signed-off-by: Monendra Singh Kushwaha Change-Id: Ibdb4455de60e8eb15d5578c7757cb4b3523fb876 Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/147434 --- build/external/packages/octeon-roc.mk | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build/external/packages/octeon-roc.mk b/build/external/packages/octeon-roc.mk index 73ee48e617..457c0b7d69 100644 --- a/build/external/packages/octeon-roc.mk +++ b/build/external/packages/octeon-roc.mk @@ -2,9 +2,9 @@ # SPDX-License-Identifier: Apache-2.0 # https://spdx.org/licenses/Apache-2.0.html -octeon-roc_version := octeon-roc-SDK12.25.02 +octeon-roc_version := octeon-roc-SDK12.25.03 octeon-roc_tarball := $(octeon-roc_version).tar.gz -octeon-roc_tarball_md5sum := 030598a6ce8b5b8e402637bfca85675c +octeon-roc_tarball_md5sum := c7fd294e3fdc170396096c4ef6789597 octeon-roc_github := https://github.com/MarvellEmbeddedProcessors/marvell-octeon-roc octeon-roc_tarball_strip_dirs := 1 From 17482d5a4946799c375bbff49b084185692af961 Mon Sep 17 00:00:00 2001 From: Bheemappa Agasimundin Date: Thu, 5 Sep 2024 09:23:03 +0000 Subject: [PATCH 197/271] octep-cp: update octeon end point control plain doc This patch revises the OCTEON endpoint control plane plugin documentation to incorporate details and configuration options for checksum offload Type: fix JIRA: https://essjira.marvell.com/browse/IPBUSW-62872 Signed-off-by: Bheemappa Agasimundin Change-Id: Ia277d449dc0f581f0a36204c00838ac542d1a116 Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/147605 Reviewed-by: Devapraba Muthumani Reviewed-by: Nithinsen Kaithakadan Tested-by: Devapraba Muthumani --- .../octep_cp/docs/octep_cp_plugin_doc.md | 52 ++++++++++++++++--- 1 file changed, 46 insertions(+), 6 deletions(-) diff --git a/src/plugins/octep_cp/docs/octep_cp_plugin_doc.md b/src/plugins/octep_cp/docs/octep_cp_plugin_doc.md index 8ed49805e1..dc32602f50 100644 --- a/src/plugins/octep_cp/docs/octep_cp_plugin_doc.md +++ b/src/plugins/octep_cp/docs/octep_cp_plugin_doc.md @@ -6,6 +6,10 @@ Marvell OCTEON firmware provides convenience user library liboctep.so to setup and interact with the host over mailbox. This octep_cp plugin uses liboctep.so library to read/send control message from/to host over mailbox. +For the host checksum offload feature, a 24-byte header is added to each +packet. The dev OCTEON plugin’s h2d-input and d2h-output components handle the +checksum computation and verification. + ## Supported SoC - OCTEON CN10KXX @@ -23,18 +27,38 @@ OCTEON connected to host. 1. OCTEON should be connected to host via SDP interface. 2. Determine SDP interface on OCTEON "lspci | grep SDP" OR "dmesg | grep sdp" - 0002:1f:00.0 Ethernet controller: Cavium, Inc. Octeon Tx2 SDP Physical Function (rev 51) - 0002:1f:00.1 Ethernet controller: Cavium, Inc. Octeon Tx2 SDP Virtual Function (rev 51) + 0002:01:00.1 Ethernet controller: Cavium, Inc. Octeon Tx2 SDP Physical Function (rev 51) + 0002:01:00.2 Ethernet controller: Cavium, Inc. Octeon Tx2 SDP Virtual Function (rev 51) + 0002:01:00.3 Ethernet controller: Cavium, Inc. Octeon Tx2 SDP Virtual Function (rev 51) 3. Bind SDP VF to vfio-pci driver - dpdk-devbind.py -b vfio-pci 0002:1f:00.1 + dpdk-devbind.py -b vfio-pci 0002:01:00.1 4. Modify startup.conf - Enable octep_cp plugin plugins { plugin octep_cp_plugin.so { enable } } - - Add SDP VF device under `onp` section - onp { - dev 0002:1f:00.1 + - Device bringup using startup.conf device section + devices { + dev pci/0002:01:00.1 + { + driver octeon + port 0 + { + name eth0 + num-rx-queues 4 + num-tx-queues 4 + } + } + dev pci/0002:01:00.2 + { + driver octeon + port 0 + { + name eth1 + num-rx-queues 5 + num-tx-queues 5 + } + } } 5. Determine SDP interface on HOST side - lspci | grep Cavium @@ -45,3 +69,19 @@ OCTEON connected to host. #### Configuration This plugin uses /usr/bin/cn10kxx.cfg configuration file to configure PCIe end point. + +1. Checksum offload configuration on DPU + - Enable checksum offload in /usr/bin/cn10kxx.cfg file + ``` + pkind=1 + ``` + - CLI + ``` + vppctl set int feature eth1 h2d-input arc port-rx-eth + vppctl set int feature eth1 d2h-output arc interface-output + ``` +2. Checksum offload configuration on host + - Enable checksum offload on SDP VF with ethtool + ``` + ethtool -K rx on tx on + ``` From 950f1ba08e1aac8a1ff700cbc5b40e4dc55115cb Mon Sep 17 00:00:00 2001 From: Sunil Kumar Kori Date: Tue, 4 Mar 2025 20:44:52 +0530 Subject: [PATCH 198/271] dev: fix max ethernet size issue for VFs When user provides maximum ethernet size more than supported size then initially it gets stored in a u32 variable which is further copied to u16 variable. Due to this conversion maximum input value truncated to 0. This trunctation leads to failure in later stage. Type: fix JIRA: https://essjira.marvell.com/browse/IPBUSW-62958 Signed-off-by: Sunil Kumar Kori Change-Id: Id7d7fe1890129ed0de9adb1ae035e848cc9098f5 Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/147519 Reviewed-by: Nithin Kumar Dabilpuram Tested-by: sa_ip-toolkits-Jenkins (cherry picked from commit aded3e60710c9fec40925cbee9cb123fc7f5dd2e) Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/147615 Reviewed-by: Monendra Singh Kushwaha Tested-by: Monendra Singh Kushwaha --- src/vnet/dev/dev.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/vnet/dev/dev.h b/src/vnet/dev/dev.h index eb06eeba34..1885913c39 100644 --- a/src/vnet/dev/dev.h +++ b/src/vnet/dev/dev.h @@ -170,7 +170,7 @@ typedef struct vnet_dev_port_cfg_change_req { u8 promisc : 1; vnet_dev_hw_addr_t addr; - u16 max_rx_frame_size; + u32 max_rx_frame_size; vnet_dev_queue_id_t queue_id; struct { From 131b485f5edf378ba3fb89cdbf0f2a641fbb2880 Mon Sep 17 00:00:00 2001 From: Monendra Singh Kushwaha Date: Sat, 1 Mar 2025 14:50:44 +0530 Subject: [PATCH 199/271] octeon: fix batch free from tx aura Type: fix Signed-off-by: Monendra Singh Kushwaha Change-Id: Ia545cc239f99a7f1c508e4b30a0497886763a7b2 Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/147328 Reviewed-by: Nithin Kumar Dabilpuram Tested-by: sa_ip-toolkits-Jenkins (cherry picked from commit c828509321608ffb6369b42bab480a3b1a8ce120) Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/147526 Tested-by: sa_ip-sw-jenkins --- src/plugins/dev_octeon/tx_node.c | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/src/plugins/dev_octeon/tx_node.c b/src/plugins/dev_octeon/tx_node.c index 580f76a9af..3ca0ee9878 100644 --- a/src/plugins/dev_octeon/tx_node.c +++ b/src/plugins/dev_octeon/tx_node.c @@ -16,6 +16,9 @@ #include #include +#define OCT_TX_NODE (1 << 0) +#define OCT_TX_IPSEC_TM_NODE (1 << 1) + #define OCT_LMT_GET_LINE_ADDR(lmt_addr, lmt_num) \ (void *) ((u64) (lmt_addr) + ((u64) (lmt_num) << ROC_LMT_LINE_SIZE_LOG2)) @@ -47,7 +50,8 @@ typedef struct #ifdef PLATFORM_OCTEON9 static_always_inline u32 -oct_batch_free (vlib_main_t *vm, oct_tx_ctx_t *ctx, vnet_dev_tx_queue_t *txq) +oct_batch_free (vlib_main_t *vm, oct_tx_ctx_t *ctx, vnet_dev_tx_queue_t *txq, + const u64 flags) { oct_txq_t *ctq = vnet_dev_get_tx_queue_data (txq); u16 off = ctq->hdr_off; @@ -61,6 +65,8 @@ oct_batch_free (vlib_main_t *vm, oct_tx_ctx_t *ctx, vnet_dev_tx_queue_t *txq) u64 buffers[n]; u32 bi[n]; + if (flags & OCT_TX_NODE) + n = clib_min (n, ctq->n_enq); n_freed = roc_npa_aura_op_bulk_alloc (ah, buffers, n, 0, 1); vlib_get_buffer_indices_with_offset (vm, (void **) &buffers, bi, n_freed, off); @@ -84,13 +90,19 @@ oct_lmt_copy (void *lmt_addr, u64 io_addr, void *desc, u64 dwords) } #else static_always_inline u32 -oct_batch_free (vlib_main_t *vm, oct_tx_ctx_t *ctx, vnet_dev_tx_queue_t *txq) +oct_batch_free (vlib_main_t *vm, oct_tx_ctx_t *ctx, vnet_dev_tx_queue_t *txq, + const u64 flags) { oct_txq_t *ctq = vnet_dev_get_tx_queue_data (txq); + oct_npa_batch_alloc_cl128_t *cl; + u32 n_freed = 0, n, n_alloc; u8 num_cl; u64 ah; - u32 n_freed = 0, n; - oct_npa_batch_alloc_cl128_t *cl; + + if (flags & OCT_TX_NODE) + n_alloc = clib_min (ctq->n_enq, ROC_CN10K_NPA_BATCH_ALLOC_MAX_PTRS); + else + n_alloc = ROC_CN10K_NPA_BATCH_ALLOC_MAX_PTRS; num_cl = ctq->ba_num_cl; if (num_cl) @@ -162,7 +174,7 @@ oct_batch_free (vlib_main_t *vm, oct_tx_ctx_t *ctx, vnet_dev_tx_queue_t *txq) { u64 addr, res; - n = clib_min (n, ROC_CN10K_NPA_BATCH_ALLOC_MAX_PTRS); + n = clib_min (n, n_alloc); oct_npa_batch_alloc_compare_t cmp = { .compare_s = { .aura = roc_npa_aura_handle_to_aura (ah), @@ -1690,7 +1702,7 @@ VNET_DEV_NODE_FN (oct_tx_ipsec_tm_node) .lmt_lines = ctq->lmt_addr + (lmt_id << ROC_LMT_LINE_SIZE_LOG2), }; - oct_batch_free (vm, &ctx, txq); + oct_batch_free (vm, &ctx, txq, OCT_TX_IPSEC_TM_NODE); vlib_get_buffers (vm, vlib_frame_vector_args (frame), b, n_pkts); n_left = n_pkts; @@ -1759,7 +1771,7 @@ VNET_DEV_NODE_FN (oct_tx_node) vnet_dev_tx_queue_lock_if_needed (txq); n_enq = ctq->n_enq; - n_enq -= oct_batch_free (vm, &ctx, txq); + n_enq -= oct_batch_free (vm, &ctx, txq, OCT_TX_NODE); if (PREDICT_FALSE (node->flags & VLIB_NODE_FLAG_TRACE)) { From ca14795bf74c8fec1d7e925bd001d6aa39a53d5e Mon Sep 17 00:00:00 2001 From: Monendra Singh Kushwaha Date: Thu, 6 Mar 2025 15:57:54 +0530 Subject: [PATCH 200/271] octeon: fix trace for single loop Type: fix JIRA: https://essjira.marvell.com/browse/IPBUSW-63189 Change-Id: I67bcb0c4fdc7bed08b42f19e3cd4bbc68fb61dd5 Signed-off-by: Monendra Singh Kushwaha Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/147750 Tested-by: sa_ip-toolkits-Jenkins Reviewed-by: Nithin Kumar Dabilpuram (cherry picked from commit 77e2781ce06de12bf03c050b4ef95bd20dcf048d) Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/147791 Tested-by: sa_ip-sw-jenkins --- src/plugins/dev_octeon/rx_node.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/plugins/dev_octeon/rx_node.c b/src/plugins/dev_octeon/rx_node.c index ed4294936c..d6d94dc12e 100644 --- a/src/plugins/dev_octeon/rx_node.c +++ b/src/plugins/dev_octeon/rx_node.c @@ -1014,6 +1014,10 @@ oct_rx_vlib_from_cq (vlib_main_t *vm, oct_nix_rx_cqe_desc_t *d, buffs[*buffer_next_index] = b[0]; next[*buffer_next_index] = ctx->next_index; *buffer_next_index = *buffer_next_index + 1; + + if (fp_flags & OCT_FP_FLAG_TRACE_EN) + clib_memcpy_fast (b[0]->pre_data, d, sizeof (oct_nix_rx_cqe_desc_t)); + return 0; } From fe63a692a16d7f871f990c1702eff28eee9239ca Mon Sep 17 00:00:00 2001 From: Monendra Singh Kushwaha Date: Mon, 10 Mar 2025 14:25:02 +0530 Subject: [PATCH 201/271] octeon: use per tx-queue aura Type: fix JIRA: https://essjira.marvell.com/browse/IPBUSW-63283 Signed-off-by: Monendra Singh Kushwaha Change-Id: If52c025a4c0666e2c602114645a0e095d24628ff Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/147984 Tested-by: sa_ip-toolkits-Jenkins Tested-by: sa_ip-sw-jenkins Reviewed-by: Nithin Kumar Dabilpuram (cherry picked from commit ab23bbf5a131a005a4a1b051950f9dd7f1313210) Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/148019 --- src/plugins/dev_octeon/init.c | 10 +++++----- src/plugins/dev_octeon/octeon.h | 3 +-- src/plugins/dev_octeon/queue.c | 27 +++++++-------------------- src/plugins/dev_octeon/tx_node.c | 30 ++++++++++-------------------- 4 files changed, 23 insertions(+), 47 deletions(-) diff --git a/src/plugins/dev_octeon/init.c b/src/plugins/dev_octeon/init.c index c7887f996f..abd97f3138 100644 --- a/src/plugins/dev_octeon/init.c +++ b/src/plugins/dev_octeon/init.c @@ -300,7 +300,7 @@ oct_init_nix (vlib_main_t *vm, vnet_dev_t *dev) }, }; - if (om->use_single_rx_tx_aura && !om->rx_aura_handle) + if (om->use_single_rx_aura && !om->rx_aura_handle) { if ((rrv = roc_npa_pool_create (&om->rx_aura_handle, bp->alloc_size, bp->n_buffers, &aura, &npapool, 0))) @@ -422,7 +422,7 @@ oct_init_inl_dev (vlib_main_t *vm, vnet_dev_t *dev) if ((rv = oct_init_ipsec_backend (vm, dev))) return rv; - oct_main.use_single_rx_tx_aura = 1; + oct_main.use_single_rx_aura = 1; oct_main.inl_dev_initialized = 1; return VNET_DEV_OK; @@ -688,7 +688,7 @@ oct_early_config (vlib_main_t *vm, unformat_input_t *input) unformat_input_t _line_input, *line_input = &_line_input; clib_error_t *error = 0; - oct_main.use_single_rx_tx_aura = 1; + oct_main.use_single_rx_aura = 1; oct_inl_dev_main.in_min_spi = 0; oct_inl_dev_main.in_max_spi = 8192; oct_inl_dev_main.out_max_sa = 8192; @@ -700,8 +700,8 @@ oct_early_config (vlib_main_t *vm, unformat_input_t *input) { if (unformat (line_input, "max-pools %u", &oct_npa_max_pools)) ; - if (unformat (line_input, "disable-single-rx-tx-aura")) - oct_main.use_single_rx_tx_aura = 0; + if (unformat (line_input, "disable-single-rx-aura")) + oct_main.use_single_rx_aura = 0; else if (unformat (line_input, "ipsec_in_min_spi %u", &oct_inl_dev_main.in_min_spi)) ; diff --git a/src/plugins/dev_octeon/octeon.h b/src/plugins/dev_octeon/octeon.h index 806c6a27a7..2815fe4219 100644 --- a/src/plugins/dev_octeon/octeon.h +++ b/src/plugins/dev_octeon/octeon.h @@ -170,9 +170,8 @@ typedef struct { oct_device_t **oct_dev; u8 inl_dev_initialized : 1; - u8 use_single_rx_tx_aura : 1; + u8 use_single_rx_aura : 1; u64 rx_aura_handle; - u64 tx_aura_handle; } oct_main_t; extern oct_main_t oct_main; diff --git a/src/plugins/dev_octeon/queue.c b/src/plugins/dev_octeon/queue.c index 5d9e7d31f2..9323d2b621 100644 --- a/src/plugins/dev_octeon/queue.c +++ b/src/plugins/dev_octeon/queue.c @@ -103,10 +103,10 @@ oct_rxq_init (vlib_main_t *vm, vnet_dev_rx_queue_t *rxq, u32 total_sz) ASSERT (!(vm->buffer_main->ext_hdr_size % ROC_ALIGN)); - if (!om->use_single_rx_tx_aura) + if (!om->use_single_rx_aura) total_sz = rxq->size; - if (!om->use_single_rx_tx_aura || !om->rx_aura_handle) + if (!om->use_single_rx_aura || !om->rx_aura_handle) { if ((rrv = roc_npa_pool_create (&crq->aura_handle, bp->alloc_size, total_sz, &aura, &npapool, 0))) @@ -224,7 +224,6 @@ oct_rxq_deinit (vlib_main_t *vm, vnet_dev_rx_queue_t *rxq) vnet_dev_rv_t oct_txq_init (vlib_main_t *vm, vnet_dev_tx_queue_t *txq) { - oct_main_t *om = &oct_main; oct_txq_t *ctq = vnet_dev_get_tx_queue_data (txq); vnet_dev_t *dev = txq->port->dev; oct_device_t *cd = vnet_dev_get_data (dev); @@ -232,27 +231,15 @@ oct_txq_init (vlib_main_t *vm, vnet_dev_tx_queue_t *txq) struct npa_aura_s aura = {}; struct npa_pool_s npapool = { .nat_align = 1, .buf_offset = OCT_EXT_HDR_SIZE / ROC_ALIGN }; - int rrv; - u32 n_buffers = 0; vlib_buffer_pool_t *bp = vlib_get_buffer_pool (vm, 0); + int rrv; - if (om->use_single_rx_tx_aura) - n_buffers = bp->n_buffers; - else - n_buffers = txq->size * 6; - - if (!om->use_single_rx_tx_aura || !om->tx_aura_handle) + if ((rrv = roc_npa_pool_create (&ctq->aura_handle, bp->alloc_size, + bp->n_buffers, &aura, &npapool, 0))) { - if ((rrv = roc_npa_pool_create (&ctq->aura_handle, bp->alloc_size, - n_buffers, &aura, &npapool, 0))) - { - oct_txq_deinit (vm, txq); - return oct_roc_err (dev, rrv, "roc_npa_pool_create() failed"); - } - om->tx_aura_handle = ctq->aura_handle; + oct_txq_deinit (vm, txq); + return oct_roc_err (dev, rrv, "roc_npa_pool_create() failed"); } - else - ctq->aura_handle = om->tx_aura_handle; ctq->npa_pool_initialized = 1; log_notice (dev, "NPA pool created, aura_handle = 0x%lx", ctq->aura_handle); diff --git a/src/plugins/dev_octeon/tx_node.c b/src/plugins/dev_octeon/tx_node.c index 3ca0ee9878..692de07172 100644 --- a/src/plugins/dev_octeon/tx_node.c +++ b/src/plugins/dev_octeon/tx_node.c @@ -1263,11 +1263,11 @@ oct_pkts_send_ipsec (vlib_main_t *vm, vlib_node_runtime_t *node, { oct_ipsec_main_t *im = &oct_ipsec_main; oct_txq_t *ctq = vnet_dev_get_tx_queue_data (txq); + u64 aura_handle = ctq->aura_handle; vnet_dev_t *dev = txq->port->dev; oct_device_t *cd = vnet_dev_get_data (dev); u32 current_sq0, current_sq1, current_sq2, current_sq3; u64 sq_handle0, sq_handle1, sq_handle2, sq_handle3; - u64 aura_handle0, aura_handle1, aura_handle2, aura_handle3; u32 sa0_index, sa1_index, sa2_index, sa3_index; u32 current_sa0_index = ~0, current_sa1_index = ~0; u32 current_sa2_index = ~0, current_sa3_index = ~0; @@ -1297,11 +1297,6 @@ oct_pkts_send_ipsec (vlib_main_t *vm, vlib_node_runtime_t *node, sq_handle2 = 0; sq_handle3 = 0; - aura_handle0 = 0; - aura_handle1 = 0; - aura_handle2 = 0; - aura_handle3 = 0; - current_sq0 = ~0; current_sq1 = ~0; current_sq2 = ~0; @@ -1441,7 +1436,6 @@ oct_pkts_send_ipsec (vlib_main_t *vm, vlib_node_runtime_t *node, ctq = cd->ctqs[sq0]; sq = &ctq->sq; sq_handle0 = sq->qid; - aura_handle0 = ctq->aura_handle; n_left0 = oct_check_fc_nix (sq, &ctq->cached_pkts, n_packets >> 2); current_sq0 = sq0; } @@ -1450,7 +1444,6 @@ oct_pkts_send_ipsec (vlib_main_t *vm, vlib_node_runtime_t *node, ctq = cd->ctqs[sq1]; sq = &ctq->sq; sq_handle1 = sq->qid; - aura_handle1 = ctq->aura_handle; n_left1 = oct_check_fc_nix (sq, &ctq->cached_pkts, n_packets >> 2); current_sq1 = sq1; } @@ -1459,7 +1452,6 @@ oct_pkts_send_ipsec (vlib_main_t *vm, vlib_node_runtime_t *node, ctq = cd->ctqs[sq2]; sq = &ctq->sq; sq_handle2 = sq->qid; - aura_handle2 = ctq->aura_handle; n_left2 = oct_check_fc_nix (sq, &ctq->cached_pkts, n_packets >> 2); current_sq2 = sq2; } @@ -1468,7 +1460,6 @@ oct_pkts_send_ipsec (vlib_main_t *vm, vlib_node_runtime_t *node, ctq = cd->ctqs[sq3]; sq = &ctq->sq; sq_handle3 = sq->qid; - aura_handle3 = ctq->aura_handle; n_left3 = oct_check_fc_nix (sq, &ctq->cached_pkts, n_packets >> 2); current_sq3 = sq3; } @@ -1480,13 +1471,13 @@ oct_pkts_send_ipsec (vlib_main_t *vm, vlib_node_runtime_t *node, lmt_arg = ROC_CN10K_CPT_LMT_ARG | (uint64_t) core_lmt_id; if (quad_bit == 0x0F) { - oct_prepare_ipsec_inst (vm, b[0], sq_handle0, aura_handle0, + oct_prepare_ipsec_inst (vm, b[0], sq_handle0, aura_handle, &pkt_meta[0], &inst0, &n_dwords[0], sess0); - oct_prepare_ipsec_inst (vm, b[1], sq_handle1, aura_handle1, + oct_prepare_ipsec_inst (vm, b[1], sq_handle1, aura_handle, &pkt_meta[1], &inst1, &n_dwords[1], sess1); - oct_prepare_ipsec_inst (vm, b[2], sq_handle2, aura_handle2, + oct_prepare_ipsec_inst (vm, b[2], sq_handle2, aura_handle, &pkt_meta[2], &inst2, &n_dwords[2], sess2); - oct_prepare_ipsec_inst (vm, b[3], sq_handle3, aura_handle3, + oct_prepare_ipsec_inst (vm, b[3], sq_handle3, aura_handle, &pkt_meta[3], &inst3, &n_dwords[3], sess3); oct_submit_quad_packets (lmt_arg, cd, &inst0, &inst1, &inst2, &inst3, @@ -1502,7 +1493,7 @@ oct_pkts_send_ipsec (vlib_main_t *vm, vlib_node_runtime_t *node, { if (n_left0) { - oct_prepare_ipsec_inst (vm, b[0], sq_handle0, aura_handle0, + oct_prepare_ipsec_inst (vm, b[0], sq_handle0, aura_handle, &pkt_meta[0], &inst0, &n_dwords[0], sess0), roc_lmt_mov_seg ((void *) lmt_line[count], &inst0, 4); @@ -1516,7 +1507,7 @@ oct_pkts_send_ipsec (vlib_main_t *vm, vlib_node_runtime_t *node, } if (n_left1) { - oct_prepare_ipsec_inst (vm, b[1], sq_handle1, aura_handle1, + oct_prepare_ipsec_inst (vm, b[1], sq_handle1, aura_handle, &pkt_meta[1], &inst1, &n_dwords[1], sess1); roc_lmt_mov_seg ((void *) lmt_line[count], &inst1, 4); @@ -1532,7 +1523,7 @@ oct_pkts_send_ipsec (vlib_main_t *vm, vlib_node_runtime_t *node, } if (n_left2) { - oct_prepare_ipsec_inst (vm, b[2], sq_handle2, aura_handle2, + oct_prepare_ipsec_inst (vm, b[2], sq_handle2, aura_handle, &pkt_meta[2], &inst2, &n_dwords[2], sess2); roc_lmt_mov_seg ((void *) lmt_line[count], &inst2, 4); @@ -1548,7 +1539,7 @@ oct_pkts_send_ipsec (vlib_main_t *vm, vlib_node_runtime_t *node, } if (n_left3) { - oct_prepare_ipsec_inst (vm, b[3], sq_handle3, aura_handle3, + oct_prepare_ipsec_inst (vm, b[3], sq_handle3, aura_handle, &pkt_meta[3], &inst3, &n_dwords[3], sess3); roc_lmt_mov_seg ((void *) lmt_line[count], &inst3, 4); @@ -1618,7 +1609,6 @@ oct_pkts_send_ipsec (vlib_main_t *vm, vlib_node_runtime_t *node, ctq = cd->ctqs[sq0]; sq = &ctq->sq; sq_handle0 = sq->qid; - aura_handle0 = ctq->aura_handle; n_left0 = oct_check_fc_nix (sq, &ctq->cached_pkts, n_packets); current_sq0 = sq0; } @@ -1629,7 +1619,7 @@ oct_pkts_send_ipsec (vlib_main_t *vm, vlib_node_runtime_t *node, goto next; } - oct_prepare_ipsec_inst (vm, b[0], sq_handle0, aura_handle0, &pkt_meta[0], + oct_prepare_ipsec_inst (vm, b[0], sq_handle0, aura_handle, &pkt_meta[0], &inst0, &n_dwords[0], sess0); roc_lmt_mov_seg ((void *) lmt_line[0], &inst0, 4); From 8eac5da440aaec5360e9c9c9fe1b913b66f49201 Mon Sep 17 00:00:00 2001 From: Monendra Singh Kushwaha Date: Thu, 6 Mar 2025 11:24:40 +0530 Subject: [PATCH 202/271] octeon: fix buffer free in port stop This patch fixes buffer free in port stop and updates input node to poll on started rx-queue. Type: fix JIRA: https://essjira.marvell.com/browse/IPBUSW-62952 Signed-off-by: Monendra Singh Kushwaha Change-Id: Id6c78eaa0eea597c554074543f27a204cb9629aa Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/147723 Reviewed-by: Nithin Kumar Dabilpuram Tested-by: Nithin Kumar Dabilpuram Klocwork: Nithin Kumar Dabilpuram (cherry picked from commit 6d4d94470d0a7c5622d51611a01d7b0fb90de1aa) Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/148020 Tested-by: sa_ip-sw-jenkins --- src/plugins/dev_octeon/common.h | 5 ++++- src/plugins/dev_octeon/port.c | 17 ++++++----------- src/plugins/dev_octeon/rx_node.c | 4 ++++ 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/src/plugins/dev_octeon/common.h b/src/plugins/dev_octeon/common.h index a7a051526d..9c6dde694f 100644 --- a/src/plugins/dev_octeon/common.h +++ b/src/plugins/dev_octeon/common.h @@ -12,7 +12,8 @@ #include static_always_inline u32 -oct_aura_free_all_buffers (vlib_main_t *vm, u64 aura_handle, u16 hdr_off) +oct_aura_free_all_buffers (vlib_main_t *vm, u64 aura_handle, u16 hdr_off, + u32 num_buffers) { u32 n = 0; u64 iova; @@ -22,6 +23,8 @@ oct_aura_free_all_buffers (vlib_main_t *vm, u64 aura_handle, u16 hdr_off) vlib_buffer_t *b = (void *) iova + hdr_off; vlib_buffer_free_one (vm, vlib_get_buffer_index (vm, b)); n++; + if (num_buffers && n == num_buffers) + break; } return n; } diff --git a/src/plugins/dev_octeon/port.c b/src/plugins/dev_octeon/port.c index 6da591802e..c2b04a1281 100644 --- a/src/plugins/dev_octeon/port.c +++ b/src/plugins/dev_octeon/port.c @@ -475,7 +475,8 @@ oct_rxq_stop (vlib_main_t *vm, vnet_dev_rx_queue_t *rxq) if ((rrv = roc_nix_rq_ena_dis (&crq->rq, 0))) oct_roc_err (dev, rrv, "roc_nix_rq_ena_dis() failed"); - n = oct_aura_free_all_buffers (vm, crq->aura_handle, crq->hdr_off); + n = + oct_aura_free_all_buffers (vm, crq->aura_handle, crq->hdr_off, crq->n_enq); if (crq->n_enq - n > 0) log_err (dev, "%u buffers leaked on rx queue %u stop", crq->n_enq - n, @@ -494,10 +495,7 @@ oct_txq_stop (vlib_main_t *vm, vnet_dev_tx_queue_t *txq) oct_npa_batch_alloc_cl128_t *cl; u32 n, off = ctq->hdr_off; - n = oct_aura_free_all_buffers (vm, ctq->aura_handle, off); - ctq->n_enq -= n; - - if (ctq->n_enq > 0 && ctq->ba_num_cl > 0) + if (ctq->ba_num_cl > 0) for (n = ctq->ba_num_cl, cl = ctq->ba_buffer + ctq->ba_first_cl; n; cl++, n--) { @@ -513,6 +511,9 @@ oct_txq_stop (vlib_main_t *vm, vnet_dev_tx_queue_t *txq) } } + n = oct_aura_free_all_buffers (vm, ctq->aura_handle, off, 0); + ctq->n_enq -= n; + if (ctq->n_enq > 0) log_err (dev, "%u buffers leaked on tx queue %u stop", ctq->n_enq, txq->queue_id); @@ -530,7 +531,6 @@ oct_port_start (vlib_main_t *vm, vnet_dev_port_t *port) oct_device_t *cd = vnet_dev_get_data (dev); oct_port_t *cp = vnet_dev_get_port_data (port); struct roc_nix *nix = cd->nix; - struct roc_nix_eeprom_info eeprom_info = {}; vnet_dev_rv_t rv; int rrv; @@ -569,11 +569,6 @@ oct_port_start (vlib_main_t *vm, vnet_dev_port_t *port) vnet_dev_poll_port_add (vm, port, 0.5, oct_port_poll); - if (roc_nix_eeprom_info_get (nix, &eeprom_info) == 0) - { - log_debug (dev, "sff_id %u data %U", eeprom_info.sff_id, format_hexdump, - eeprom_info.buf, sizeof (eeprom_info.buf)); - } done: if (rv != VNET_DEV_OK) oct_port_stop (vm, port); diff --git a/src/plugins/dev_octeon/rx_node.c b/src/plugins/dev_octeon/rx_node.c index d6d94dc12e..702f253762 100644 --- a/src/plugins/dev_octeon/rx_node.c +++ b/src/plugins/dev_octeon/rx_node.c @@ -1883,6 +1883,10 @@ VNET_DEV_NODE_FN (oct_rx_node) foreach_vnet_dev_rx_queue_runtime (rxq, node) { vnet_dev_port_t *port = rxq->port; + + if (!rxq->started) + continue; + n_rx += oct_rx_node_inline (vm, node, frame, port, rxq, 0); } From 872943e4f5503183d5c92c04bcd7e30a29644582 Mon Sep 17 00:00:00 2001 From: Nithinsen Kaithakadan Date: Sun, 9 Mar 2025 18:46:01 +0530 Subject: [PATCH 203/271] octeon: fix invalid mac length in chacha poly This patch fixes invalid mac length to resolve crash in Chacha Poly. Type: fix JIRA: https://essjira.marvell.com/browse/IPBUSW-62078 Signed-off-by: Nithinsen Kaithakadan Change-Id: Ia940a721b5cb47f7b5fe68adbea78adaaa0f54f0 Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/147957 Reviewed-by: Monendra Singh Kushwaha Tested-by: sa_ip-toolkits-Jenkins (cherry picked from commit 2a43de5e1260df3809aa25a0d02fefc95945b5c6) Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/147963 Tested-by: Monendra Singh Kushwaha --- src/plugins/dev_octeon/crypto.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/plugins/dev_octeon/crypto.c b/src/plugins/dev_octeon/crypto.c index 3847d2e48a..be0d19ff7d 100644 --- a/src/plugins/dev_octeon/crypto.c +++ b/src/plugins/dev_octeon/crypto.c @@ -1362,7 +1362,7 @@ oct_crypto_aead_session_update (vlib_main_t *vm, oct_crypto_sess_t *sess, vnet_crypto_key_t *key = vnet_crypto_get_key (key_index); roc_se_cipher_type enc_type = 0; roc_se_auth_type auth_type = 0; - u32 digest_len = ~0; + u32 digest_len = 16; i32 rv = 0; switch (key->async_alg) @@ -1374,9 +1374,6 @@ oct_crypto_aead_session_update (vlib_main_t *vm, oct_crypto_sess_t *sess, sess->aes_gcm = 1; sess->iv_offset = 0; sess->iv_length = 16; - sess->cpt_ctx.mac_len = 16; - sess->cpt_op = type; - digest_len = 16; break; case VNET_CRYPTO_ALG_CHACHA20_POLY1305: enc_type = ROC_SE_CHACHA20; @@ -1389,6 +1386,9 @@ oct_crypto_aead_session_update (vlib_main_t *vm, oct_crypto_sess_t *sess, return -1; } + sess->cpt_ctx.mac_len = digest_len; + sess->cpt_op = type; + rv = roc_se_ciph_key_set (&sess->cpt_ctx, enc_type, key->data, vec_len (key->data)); if (rv) From 9a907c99b509492395bb3d8954a0c5b0ce9cd626 Mon Sep 17 00:00:00 2001 From: Nagendra T P Date: Mon, 24 Mar 2025 17:44:42 +0530 Subject: [PATCH 204/271] ci: resolve oct-ep dependency Resolve oct-ep dependency issues while package installation. Signed-off-by: Nagendra T P --- .github/workflows/build-cn10k.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-cn10k.yml b/.github/workflows/build-cn10k.yml index 5cb882e497..fefb657dde 100644 --- a/.github/workflows/build-cn10k.yml +++ b/.github/workflows/build-cn10k.yml @@ -137,7 +137,7 @@ jobs: echo 'Package: vpp-'$PKG_VERSION_NAME'-cn10k'$PKG_POSTFIX >> DEBIAN/control echo 'Version: '$MRVL_PKG_VERSION >> DEBIAN/control echo "Maintainer: Jerin Jacob (jerinj@marvell.com)" >> DEBIAN/control - echo 'Depends: python3, python3-ply, dpdk-'$DPDK_BASE_PKG_VERSION'-cn10k (= '$DPDK_PKG_VERSION'), cpt-firmware-cn10k'${FW_PKG_POSTFIX}' (= '$CPT_PKG_VERSION'), oct-ep-target-cn10k (= '$MRVL_PKG_VERSION')' >> DEBIAN/control + echo 'Depends: python3, python3-ply, dpdk-'$DPDK_BASE_PKG_VERSION'-cn10k (= '$DPDK_PKG_VERSION'), cpt-firmware-cn10k'${FW_PKG_POSTFIX}' (= '$CPT_PKG_VERSION'), oct-ep-target-cn10k (>= '$MRVL_PKG_VERSION')' >> DEBIAN/control echo "Architecture: arm64" >> DEBIAN/control echo "Homepage: https://wiki.fd.io/view/VPP" >> DEBIAN/control echo "Description: Vector Packet Processing (VPP) for Octeon10" >> DEBIAN/control From 9a8746bc1e571e68c9f75e521fa2d7965f49c56d Mon Sep 17 00:00:00 2001 From: Nagendra T P Date: Fri, 4 Apr 2025 13:05:15 +0530 Subject: [PATCH 205/271] doc: bump dao version number Bump version number to 25.05.0. Signed-off-by: Nagendra T P --- .github/workflows/build-cn10k.yml | 10 ++++++---- .github/workflows/build-cn9k.yml | 3 ++- CPT_PKG_VERSION | 1 - DEP_PKG_VERSION | 2 ++ MRVL_VERSION | 2 +- 5 files changed, 11 insertions(+), 7 deletions(-) delete mode 100644 CPT_PKG_VERSION create mode 100644 DEP_PKG_VERSION diff --git a/.github/workflows/build-cn10k.yml b/.github/workflows/build-cn10k.yml index fefb657dde..809dd7c943 100644 --- a/.github/workflows/build-cn10k.yml +++ b/.github/workflows/build-cn10k.yml @@ -50,7 +50,8 @@ jobs: [ -s /tmp/tags ] && NIGHTLY=false || NIGHTLY=true echo "PKG_VERSION_NAME=`./src/scripts/version | awk -F '-' '{print $1}'`" >> "${PWD}/artifacts/env" echo "MRVL_PKG_VERSION=`cat MRVL_VERSION`" >> "${PWD}/artifacts/env" - echo "CPT_PKG_VERSION=`cat CPT_PKG_VERSION`" >> "${PWD}/artifacts/env" + echo "CPT_PKG_VERSION=`cat DEP_PKG_VERSION | grep CPT_PKG_VERSION | awk -F'=' '{print $2}'`" >> "${PWD}/artifacts/env" + echo "OCTEP_PKG_VERSION=`cat DEP_PKG_VERSION | grep OCTEP_PKG_VERSION | awk -F'=' '{print $2}'`" >> "${PWD}/artifacts/env" echo "DPDK_PKG_VERSION=`cat DPDK_VERSION | grep RELEASE_VERSION | awk -F'=' '{print $2}'`" >> "${PWD}/artifacts/env" echo "DPDK_BASE_PKG_VERSION=`cat DPDK_VERSION | grep BASE_VERSION | awk -F'=' '{print $2}' | awk -F'.' '{print $1"."$2}'`" >> "${PWD}/artifacts/env" echo "PKG_POSTFIX=${PKG_POSTFIX}" >> "${PWD}/artifacts/env" @@ -61,6 +62,7 @@ jobs: - name: Build VPP and generate package id: build run: | + set -x mkdir -p ~/.ccache BASE_DIR=${PWD} sudo apt-get update -q -y @@ -126,8 +128,8 @@ jobs: ccache -p git config --global --add safe.directory "${PWD}" sudo APT_ARGS='-y -q' make install-deps - wget "https://github.com/MarvellEmbeddedProcessors/pcie_ep_octeon_target/releases/download/oct-ep-target-cn10k-${MRVL_PKG_VERSION}-${DISTRO}-${MRVL_PKG_VERSION}/oct-ep-target-cn10k_${MRVL_PKG_VERSION}_arm64.deb" - sudo apt-get install -y ./"oct-ep-target-cn10k_${MRVL_PKG_VERSION}_arm64.deb" + wget "https://github.com/MarvellEmbeddedProcessors/pcie_ep_octeon_target/releases/download/oct-ep-target-cn10k-${OCTEP_PKG_VERSION}-${DISTRO}-${OCTEP_PKG_VERSION}/oct-ep-target-cn10k_${OCTEP_PKG_VERSION}_arm64.deb" + sudo apt-get install -y ./"oct-ep-target-cn10k_${OCTEP_PKG_VERSION}_arm64.deb" make build-release VPP_PLATFORM=octeon10 mkdir -p "${PWD}/install/DEBIAN" mkdir -p "${PWD}/install/usr/share/vpp/api" @@ -137,7 +139,7 @@ jobs: echo 'Package: vpp-'$PKG_VERSION_NAME'-cn10k'$PKG_POSTFIX >> DEBIAN/control echo 'Version: '$MRVL_PKG_VERSION >> DEBIAN/control echo "Maintainer: Jerin Jacob (jerinj@marvell.com)" >> DEBIAN/control - echo 'Depends: python3, python3-ply, dpdk-'$DPDK_BASE_PKG_VERSION'-cn10k (= '$DPDK_PKG_VERSION'), cpt-firmware-cn10k'${FW_PKG_POSTFIX}' (= '$CPT_PKG_VERSION'), oct-ep-target-cn10k (>= '$MRVL_PKG_VERSION')' >> DEBIAN/control + echo 'Depends: python3, python3-ply, dpdk-'$DPDK_BASE_PKG_VERSION'-cn10k (= '$DPDK_PKG_VERSION'), cpt-firmware-cn10k'${FW_PKG_POSTFIX}' (= '$CPT_PKG_VERSION'), oct-ep-target-cn10k (>= '$OCTEP_PKG_VERSION')' >> DEBIAN/control echo "Architecture: arm64" >> DEBIAN/control echo "Homepage: https://wiki.fd.io/view/VPP" >> DEBIAN/control echo "Description: Vector Packet Processing (VPP) for Octeon10" >> DEBIAN/control diff --git a/.github/workflows/build-cn9k.yml b/.github/workflows/build-cn9k.yml index 22b97b9349..265d624c5a 100644 --- a/.github/workflows/build-cn9k.yml +++ b/.github/workflows/build-cn9k.yml @@ -49,7 +49,7 @@ jobs: [ -s /tmp/tags ] && NIGHTLY=false || NIGHTLY=true echo "PKG_VERSION_NAME=`./src/scripts/version | awk -F '-' '{print $1}'`" >> "${PWD}/artifacts/env" echo "MRVL_PKG_VERSION=`cat MRVL_VERSION`" >> "${PWD}/artifacts/env" - echo "CPT_PKG_VERSION=`cat CPT_PKG_VERSION`" >> "${PWD}/artifacts/env" + echo "CPT_PKG_VERSION=`cat DEP_PKG_VERSION | grep CPT_PKG_VERSION | awk -F'=' '{print $2}'`" >> "${PWD}/artifacts/env" echo "DPDK_PKG_VERSION=`cat DPDK_VERSION | grep RELEASE_VERSION | awk -F'=' '{print $2}'`" >> "${PWD}/artifacts/env" echo "DPDK_BASE_PKG_VERSION=`cat DPDK_VERSION | grep BASE_VERSION | awk -F'=' '{print $2}' | awk -F'.' '{print $1"."$2}'`" >> "${PWD}/artifacts/env" echo "PKG_POSTFIX=${PKG_POSTFIX}" >> "${PWD}/artifacts/env" @@ -60,6 +60,7 @@ jobs: - name: Build VPP and generate package id: build run: | + set -x mkdir -p ~/.ccache BASE_DIR=${PWD} sudo apt-get update -q -y diff --git a/CPT_PKG_VERSION b/CPT_PKG_VERSION deleted file mode 100644 index c6b958db80..0000000000 --- a/CPT_PKG_VERSION +++ /dev/null @@ -1 +0,0 @@ -24.09.0 \ No newline at end of file diff --git a/DEP_PKG_VERSION b/DEP_PKG_VERSION new file mode 100644 index 0000000000..eb532ffd84 --- /dev/null +++ b/DEP_PKG_VERSION @@ -0,0 +1,2 @@ +CPT_PKG_VERSION=24.09.0 +OCTEP_PKG_VERSION=25.03.0 \ No newline at end of file diff --git a/MRVL_VERSION b/MRVL_VERSION index 4b498c089a..39359a17f9 100644 --- a/MRVL_VERSION +++ b/MRVL_VERSION @@ -1 +1 @@ -25.01.0 \ No newline at end of file +25.05.0 \ No newline at end of file From 4569e81b29e257f392514ad9f4bfb33eedf1f8c0 Mon Sep 17 00:00:00 2001 From: Nagendra T P Date: Mon, 14 Apr 2025 12:38:05 +0530 Subject: [PATCH 206/271] doc: bump dpdk version bump dpdk version to 25.03.0 Signed-off-by: Nagendra T P --- DPDK_VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DPDK_VERSION b/DPDK_VERSION index d88b097b2a..9d5efc6763 100644 --- a/DPDK_VERSION +++ b/DPDK_VERSION @@ -1,2 +1,2 @@ BASE_VERSION=24.11.0 -RELEASE_VERSION=25.01.0 \ No newline at end of file +RELEASE_VERSION=25.03.0 \ No newline at end of file From 46f0f0c2f27abc64fc1fec75cb46cef931b0d266 Mon Sep 17 00:00:00 2001 From: Nagendra T P Date: Mon, 28 Apr 2025 15:49:23 +0530 Subject: [PATCH 207/271] ci: remove unused FW_PKG_POSTFIX variable Since firmware builds are always release, dropping firmware postfix variable. Signed-off-by: Nagendra T P --- .github/workflows/build-cn10k.yml | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/.github/workflows/build-cn10k.yml b/.github/workflows/build-cn10k.yml index 809dd7c943..4cef5fb40a 100644 --- a/.github/workflows/build-cn10k.yml +++ b/.github/workflows/build-cn10k.yml @@ -40,13 +40,6 @@ jobs: mkdir -p "${PWD}/artifacts" git tag --points-at HEAD > /tmp/tags [ -s /tmp/tags ] && PKG_POSTFIX= || PKG_POSTFIX=-devel - FW_PKG_POSTFIX="" - if [ ${PKG_POSTFIX} == "-devel" ]; then - FW_PKG_POSTFIX="" - else - FW_PKG_POSTFIX=$PKG_POSTFIX - fi - echo "FW_PKG_POSTFIX=${FW_PKG_POSTFIX}" >> "${PWD}/artifacts/env" [ -s /tmp/tags ] && NIGHTLY=false || NIGHTLY=true echo "PKG_VERSION_NAME=`./src/scripts/version | awk -F '-' '{print $1}'`" >> "${PWD}/artifacts/env" echo "MRVL_PKG_VERSION=`cat MRVL_VERSION`" >> "${PWD}/artifacts/env" @@ -139,7 +132,7 @@ jobs: echo 'Package: vpp-'$PKG_VERSION_NAME'-cn10k'$PKG_POSTFIX >> DEBIAN/control echo 'Version: '$MRVL_PKG_VERSION >> DEBIAN/control echo "Maintainer: Jerin Jacob (jerinj@marvell.com)" >> DEBIAN/control - echo 'Depends: python3, python3-ply, dpdk-'$DPDK_BASE_PKG_VERSION'-cn10k (= '$DPDK_PKG_VERSION'), cpt-firmware-cn10k'${FW_PKG_POSTFIX}' (= '$CPT_PKG_VERSION'), oct-ep-target-cn10k (>= '$OCTEP_PKG_VERSION')' >> DEBIAN/control + echo 'Depends: python3, python3-ply, dpdk-'$DPDK_BASE_PKG_VERSION'-cn10k (= '$DPDK_PKG_VERSION'), cpt-firmware-cn10k (= '$CPT_PKG_VERSION'), oct-ep-target-cn10k (>= '$OCTEP_PKG_VERSION')' >> DEBIAN/control echo "Architecture: arm64" >> DEBIAN/control echo "Homepage: https://wiki.fd.io/view/VPP" >> DEBIAN/control echo "Description: Vector Packet Processing (VPP) for Octeon10" >> DEBIAN/control From abfd190be834135cc51df2919c64d37daeab72c9 Mon Sep 17 00:00:00 2001 From: Nagendra T P Date: Mon, 28 Apr 2025 21:50:34 +0530 Subject: [PATCH 208/271] ci: upgrade action-gh-release to v2.2.2 Upgrade action-gh-release to v2.2.2 to publish releases. Signed-off-by: Nagendra T P --- .github/workflows/build-cn10k.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-cn10k.yml b/.github/workflows/build-cn10k.yml index 4cef5fb40a..5a61f587f8 100644 --- a/.github/workflows/build-cn10k.yml +++ b/.github/workflows/build-cn10k.yml @@ -174,7 +174,7 @@ jobs: fi continue-on-error: true - name: Release VPP cn10k package - uses: softprops/action-gh-release@v2.0.4 + uses: softprops/action-gh-release@v2.2.2 if: ${{ github.event_name == 'push' }} with: draft: false From 1dcb050ea09495cd2c8736cdeb31700297b3bba6 Mon Sep 17 00:00:00 2001 From: Nagendra T P Date: Fri, 2 May 2025 09:56:50 +0530 Subject: [PATCH 209/271] doc: bump pcie-ep version number Bump version number to 25.04.0. Signed-off-by: Nagendra T P --- DEP_PKG_VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DEP_PKG_VERSION b/DEP_PKG_VERSION index eb532ffd84..c7ef5229b6 100644 --- a/DEP_PKG_VERSION +++ b/DEP_PKG_VERSION @@ -1,2 +1,2 @@ CPT_PKG_VERSION=24.09.0 -OCTEP_PKG_VERSION=25.03.0 \ No newline at end of file +OCTEP_PKG_VERSION=25.04.0 \ No newline at end of file From ff246302047c43d4039829a384bbd546b8020fd1 Mon Sep 17 00:00:00 2001 From: Nawal Kishor Date: Tue, 11 Mar 2025 21:20:57 +0530 Subject: [PATCH 210/271] ci: set default val of OCTEON_VERSION Pick OCTEON_VERSION from the env variable and set its default value to cn10k if not set. Type: fix Signed-off-by: Nawal Kishor Change-Id: Ibec570477ccb30d5ffaf44b73c44581c87d69cb4 Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/148094 Reviewed-by: Ashwin Sekhar T K Tested-by: Nithin Kumar Dabilpuram (cherry picked from commit b8a16054efb8321ddb1dab67760ed3130acbe226) Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/149668 Tested-by: sa_ip-sw-jenkins Reviewed-by: Monendra Singh Kushwaha --- ci/build/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/build/build.sh b/ci/build/build.sh index 10690b9c69..cfad2bca6d 100755 --- a/ci/build/build.sh +++ b/ci/build/build.sh @@ -45,7 +45,7 @@ fi DEPS_DIR= BUILD= export CROSS="aarch64-marvell-linux-gnu-" -export OCTEON_VERSION="cn10k" +export OCTEON_VERSION=${OCTEON_VERSION:-"cn10k"} export PLATFORM="cnxk" eval set -- "$OPTS" From ca48c6c8b07c49b6ae47d396437dd8804f95f1d4 Mon Sep 17 00:00:00 2001 From: Nawal Kishor Date: Tue, 11 Mar 2025 09:11:28 +0530 Subject: [PATCH 211/271] ci: add test scripts Added ci test scripts --> * test.sh * ci_runner.py * dpdk-devbind.py Type: feature Signed-off-by: Nawal Kishor Change-Id: Idd4c3b39a5bd2cab82460864100d31d3745f1a61 Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/148045 Reviewed-by: Ashwin Sekhar T K Tested-by: Nithin Kumar Dabilpuram (cherry picked from commit f6278834f669ec10dd109bf8628eb31e210ec78c) Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/149869 Reviewed-by: Monendra Singh Kushwaha Tested-by: Monendra Singh Kushwaha --- ci/test/board/ci_runner.py | 519 +++++++++++++++++++++ ci/test/board/dpdk-devbind.py | 853 ++++++++++++++++++++++++++++++++++ ci/test/test.sh | 63 +++ 3 files changed, 1435 insertions(+) create mode 100755 ci/test/board/ci_runner.py create mode 100755 ci/test/board/dpdk-devbind.py create mode 100755 ci/test/test.sh diff --git a/ci/test/board/ci_runner.py b/ci/test/board/ci_runner.py new file mode 100755 index 0000000000..5679c2cef9 --- /dev/null +++ b/ci/test/board/ci_runner.py @@ -0,0 +1,519 @@ +# Copyright (c) 2025 Marvell. +# SPDX-License-Identifier: Apache-2.0 +# https://spdx.org/licenses/Apache-2.0.html +# +# Copied from /src/plugins/onp/test/ci/ci_runner.py from vpp-24.02. + +import os +import re +import sys +import string +import argparse +import subprocess + +if sys.version_info >= (3, 0): + from configparser import ConfigParser +else: + from ConfigParser import ConfigParser + +MAJOR_NUM = 1 +MINOR_NUM = 0 +REVISION = 0 +devid_lbk_vf = "a0f8" +devid_evt_vf = "a0f9" +devid_inl_pf = "a0f0" + +CPU_PART_CN96xx = "0x0b2" +CPU_PART_CN98xx = "0x0b1" +CPU_PART_CN10xx = "0xd49" + +NUM_EVTDEV = 2 +NUM_CPTDEV = 2 +NUM_LBKDEV = 4 +NUM_INLDEV = 1 + +EVENT_DEV_LIMIT = 50 + +SYS_DRV_PATH = "/sys/bus/pci/drivers" +SYS_DEV_PATH = "/sys/bus/pci/devices" + + +def findSystemArch(): + command = "cat /proc/cpuinfo" + info = subprocess.check_output(command, shell=True).decode().strip() + cpu_part = "" + for line in info.split("\n"): + if "CPU part" in line: + cpu_part = re.sub(".*CPU part.*: ", "", line, 1) + break + if cpu_part == CPU_PART_CN96xx: + return "_96xx" + if cpu_part == CPU_PART_CN98xx: + return "_98xx" + if cpu_part == CPU_PART_CN10xx: + return "_10xx" + + print("cpu part doesnt match 96/98/10xx") + return "_unknown" + + +class CIRunner: + def __init__(self, dir, dpdk_devbind, verb, dry_run): + # Modify the number of VFs and the kernel driver as needed. + # ========================================================= + self.num_evtdev = NUM_EVTDEV + self.num_cptdev = NUM_CPTDEV + self.num_lbkdev = NUM_LBKDEV + self.num_inldev = NUM_INLDEV + self.drv_cptdev = b"rvu_cptpf" + self.drv_lbkdev = b"rvu_nicvf" + self.event_dev_id = devid_evt_vf.encode("UTF-8") + # ========================================================= + self.evtpfbdf = None + self.evtpfdev = None + self.evtpfbdf2 = None + self.evtpfdev2 = None + self.cptpfbdf = None + self.cptpfdev = None + self.test_dir = dir + self.dpdk_devbind = dpdk_devbind + self.drun = dry_run + self.verb = verb + out = subprocess.Popen( + "uname -m".split(), stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) + stdout, stderr = out.communicate() + print("ARCH={}".format(stdout.decode("utf-8"))) + if b"x86" in stdout: + exit("Cannot execute on x86") + else: + print("Starting CI Runner") + + self.arch_model = findSystemArch() + + def parse_through_event_devices(self, line): + device = line.split(b" ")[0].decode("utf-8") + + # if arch is cn10k, then use pf device, as + # kernel support is not available for vf + if self.arch_model == "_10xx": + return device, devid_evt_vf + + totalvfs_filePath = SYS_DEV_PATH + "/" + str(device) + "/sriov_totalvfs" + + if os.path.isfile(totalvfs_filePath): + command = "cat " + totalvfs_filePath + totalvfs = subprocess.check_output(command, shell=True).decode().strip() + if int(totalvfs) > self.num_evtdev: + return device, devid_evt_vf + + return None, None + + def init_pf(self): + print("==== Init PF devices ====") + + cmd = subprocess.Popen( + f"{self.dpdk_devbind} -s", + shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + for line in cmd.stdout: + if self.drv_lbkdev in line: + lnparse = line.split(b"'") + dvparse = lnparse[1].split(b" ") + self.lbkpfdev = dvparse[1].strip() + self.lbkpfbdf = lnparse[0].strip() + + if self.drv_cptdev in line: + lnparse = line.split(b"'") + dvparse = lnparse[1].split(b" ") + self.cptpfdev = dvparse[1].strip() + self.cptpfbdf = lnparse[0].strip() + + if self.evtpfdev is None and self.event_dev_id in line: + self.evtpfbdf, self.evtpfdev = self.parse_through_event_devices(line) + + elif ( + self.arch_model == "_10xx" + and self.evtpfdev2 is None + and self.event_dev_id in line + ): + self.evtpfbdf2, self.evtpfdev2 = self.parse_through_event_devices(line) + + def unbind_all(self): + print("==== Unbind all VF devices bound to vfio-pci ====") + + cmd = subprocess.Popen( + f"{self.dpdk_devbind} -s", + shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + + for line in cmd.stdout: + lnparse = line.split(b" ") + cmdstr = "echo {} > {}/{}/driver/unbind".format( + lnparse[0].decode("utf-8"), SYS_DEV_PATH, lnparse[0].decode("utf-8") + ) + self.run_cmd(cmdstr) + + cmdstr = "echo 0 > {}/{}/sriov_numvfs".format(SYS_DEV_PATH, self.evtpfbdf) + self.run_cmd(cmdstr) + + cmdstr = "echo 0 > {}/{}/sriov_numvfs".format( + SYS_DEV_PATH, self.cptpfbdf.decode("utf-8") + ) + self.run_cmd(cmdstr) + + def set_limits_over_eventdev(self, limit, bdf): + if self.arch_model == "_10xx": + return + self.set_limits(limit, bdf) + + def update_limits_over_eventdev(self): + if self.arch_model == "_10xx": + return + + # Clear limits on SSO and SSOW devices + cmd = subprocess.Popen( + "lspci -d:{}".format(self.evtpfdev.decode("utf-8")), + shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + for line in cmd.stdout: + lnparse = line.split(b" ") + evtbdf = lnparse[0].strip() + limit = 0 + self.set_limits(limit, evtbdf) + + # Set limits on Event devices + limit = EVENT_DEV_LIMIT + self.set_limits_over_eventdev(limit, self.evtpfbdf) + + def bind_driver(self, driver, dev_bdf): + if self.arch_model == "_10xx": + cmdstr = f"{self.dpdk_devbind} -b {driver} {dev_bdf}" + self.run_cmd(cmdstr) + + def bind_evtdev_pf(self): + if self.arch_model == "_10xx": + if self.drun is False: + os.system( + "echo 'event{}_bdf: {}' >> {}/{}".format( + 1, self.evtpfbdf, self.test_dir, "configs/pcie.ini" + ) + ) + os.system( + "echo 'event{}_bdf: {}' >> {}/{}".format( + 2, self.evtpfbdf2, self.test_dir, "configs/pcie.ini" + ) + ) + return + + def bind_evtdev_vf(self): + self.bind_driver("vfio-pci", self.evtpfbdf) + self.update_limits_over_eventdev() + + fnparse = self.evtpfbdf.split(".") + fn = fnparse[1] + dbdparse = fnparse[0] + cmdstr = "echo {} > {}/{}/sriov_numvfs".format( + self.num_evtdev, SYS_DEV_PATH, self.evtpfbdf + ) + self.run_cmd(cmdstr) + + # Resolve Created VFs + lspci = subprocess.Popen( + "lspci | grep {}".format(dbdparse), + shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + + evtcount = 1 + dbdparse = dbdparse.encode("utf-8") + for line in lspci.stdout: + if dbdparse in line: + lnparse = line.split(b" ") + if lnparse[0] != self.evtpfbdf.encode("utf-8"): + limit = 10 + self.set_limits_over_eventdev(limit, lnparse[0]) + cmdstr = "echo {} > {}/{}/driver/unbind".format( + lnparse[0].decode("utf-8"), + SYS_DEV_PATH, + lnparse[0].decode("utf-8"), + ) + self.run_cmd(cmdstr) + + devbdf = lnparse[0].decode("utf-8") + cmdstr = "echo vfio-pci > /sys/bus/pci/devices/%s/driver_override" % devbdf + self.run_cmd(cmdstr) + cmdstr = "echo %s > /sys/bus/pci/drivers/vfio-pci/bind" % devbdf + self.run_cmd(cmdstr) + # cmdstr = "echo %s > /sys/bus/pci/driver_probe" % devbdf + # self.run_cmd(cmdstr) + + if self.drun is False: + os.system( + "echo 'event{}_bdf: {}' >> {}/{}".format( + evtcount, + lnparse[0].decode("utf-8"), + self.test_dir, + "configs/pcie.ini", + ) + ) + evtcount = evtcount + 1 + + def bind_evtdev(self): + print("==== Binding Event devices ====") + print("Using Eventdev PF device ID: {}".format(self.evtpfdev)) + print("Using Eventdev PF BDF: {}".format(self.evtpfbdf)) + self.evtpfbdf2 and print( + "Using Eventdev PF device ID: {}".format(self.evtpfdev) + ) + self.evtpfbdf2 and print("Using Eventdev PF BDF: {}".format(self.evtpfbdf2)) + + if self.arch_model == "_10xx": + self.bind_evtdev_pf() + else: + self.bind_evtdev_vf() + + def bind_cptdev(self): + print("==== Binding CPT devices ====") + print("Using CPT PF device ID: {}".format(self.cptpfdev.decode("utf-8"))) + print("Using CPT PF BDF: {}".format(self.cptpfbdf.decode("utf-8"))) + fnparse = self.cptpfbdf.split(b".") + fn = fnparse[1] + dbdparse = fnparse[0] + cmdstr = "echo {} > {}/{}/sriov_numvfs".format( + self.num_cptdev, SYS_DEV_PATH, self.cptpfbdf.decode("utf-8") + ) + self.run_cmd(cmdstr) + + # Resolve Created VFs + lspci = subprocess.Popen( + "lspci | grep {}".format(dbdparse.decode("utf-8")), + shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + + cptcount = 1 + for line in lspci.stdout: + if dbdparse in line: + lnparse = line.split(b" ") + if lnparse[0] != self.cptpfbdf: + cmdstr = "echo {} > {}/{}/driver/unbind".format( + lnparse[0].decode("utf-8"), + SYS_DEV_PATH, + lnparse[0].decode("utf-8"), + ) + self.run_cmd(cmdstr) + + devbdf = lnparse[0].decode("utf-8") + cmdstr = "echo vfio-pci > /sys/bus/pci/devices/%s/driver_override" % devbdf + self.run_cmd(cmdstr) + cmdstr = "echo %s > /sys/bus/pci/drivers/vfio-pci/bind" % devbdf + self.run_cmd(cmdstr) + # cmdstr = "echo %s > /sys/bus/pci/driver_probe" % devbdf + # self.run_cmd(cmdstr) + + if self.drun is False: + os.system( + "echo 'crypto{}_bdf: {}' >> {}/{}".format( + cptcount, + lnparse[0].decode("utf-8"), + self.test_dir, + "configs/pcie.ini", + ) + ) + cptcount = cptcount + 1 + + def bind_lbkdev(self): + print("==== Binding LBK devices ====") + + # Resolve Pre-created VFs + lspci = subprocess.Popen( + "lspci -d:{}".format(devid_lbk_vf), + shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + lbkcount = 1 + for line in lspci.stdout: + lnparse = line.split(b" ") + if lbkcount <= self.num_lbkdev: + cmdstr = "echo {} > {}/{}/driver/unbind".format( + lnparse[0].decode("utf-8"), SYS_DEV_PATH, lnparse[0].decode("utf-8") + ) + self.run_cmd(cmdstr) + + devbdf = lnparse[0].decode("utf-8") + cmdstr = "echo vfio-pci > /sys/bus/pci/devices/%s/driver_override" % devbdf + self.run_cmd(cmdstr) + cmdstr = "echo %s > /sys/bus/pci/drivers/vfio-pci/bind" % devbdf + self.run_cmd(cmdstr) + # cmdstr = "echo %s > /sys/bus/pci/driver_probe" % devbdf + # self.run_cmd(cmdstr) + + if self.drun is False: + os.system( + "echo 'lbk{}_bdf: {}' >> {}/{}".format( + lbkcount, + lnparse[0].decode("utf-8"), + self.test_dir, + "configs/pcie.ini", + ) + ) + lbkcount = lbkcount + 1 + + def bind_inldev(self): + print("==== Binding Inline devices ====") + + # Resolve Inline device PFs + lspci = subprocess.Popen( + "lspci -d:{}".format(devid_inl_pf), + shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + inlcount = 1 + for line in lspci.stdout: + lnparse = line.split(b" ") + if inlcount <= self.num_inldev: + cmdstr = "echo {} > {}/{}/driver/unbind".format( + lnparse[0].decode("utf-8"), SYS_DEV_PATH, lnparse[0].decode("utf-8") + ) + self.run_cmd(cmdstr) + + devbdf = lnparse[0].decode("utf-8") + cmdstr = "echo vfio-pci > /sys/bus/pci/devices/%s/driver_override" % devbdf + self.run_cmd(cmdstr) + cmdstr = "echo %s > /sys/bus/pci/drivers/vfio-pci/bind" % devbdf + self.run_cmd(cmdstr) + # cmdstr = "echo %s > /sys/bus/pci/driver_probe" % devbdf + # self.run_cmd(cmdstr) + + if self.drun is False: + os.system( + "echo 'inl{}_bdf: {}' >> {}/{}".format( + inlcount, + lnparse[0].decode("utf-8"), + self.test_dir, + "configs/pcie.ini", + ) + ) + inlcount = inlcount + 1 + + def set_limits(self, limit, evt_bdf): + cmdstr = "echo {} > {}/{}/limits/sso".format( + limit, SYS_DEV_PATH, evt_bdf.decode("utf-8") + ) + self.run_cmd(cmdstr) + cmdstr = "echo {} > {}/{}/limits/ssow".format( + limit, SYS_DEV_PATH, evt_bdf.decode("utf-8") + ) + self.run_cmd(cmdstr) + + def run_cmd(self, cmdstr): + if self.drun: + print(cmdstr) + else: + if self.verb: + print(cmdstr) + os.system(cmdstr) + + def enable_sriov(self): + enable_sriov_file = "/sys/module/vfio_pci/parameters/enable_sriov" + if os.path.isfile(enable_sriov_file): + cmdstr = "echo 1 > " + enable_sriov_file + self.run_cmd(cmdstr) + else: + print("File: " + enable_sriov_file + " doesnt exist") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "-b", + "--bind", + action="store_true", + default=False, + help="Bind PCIe addresses to vfio-pci driver", + ) + parser.add_argument( + "-u", + "--unbind", + action="store_true", + default=False, + help="Unbind PCIe addresses from vfio-pci driver", + ) + parser.add_argument( + "-d", + "--dryrun_bind", + action="store_true", + default=False, + help="Dryrun bind PCIe addresses to vfio-pci driver", + ) + parser.add_argument( + "-g", + "--dryrun_unbind", + action="store_true", + default=False, + help="Dryrun unbind PCIe addresses from vfio-pci driver", + ) + parser.add_argument( + "-v", + "--verbose", + action="store_true", + default=False, + help="Optionally used with [-b|-u] for verbose output", + ) + + try: + args = parser.parse_args() + except: + sys.exit("Call 'python ci_runner.py --help' for more info") + + print("Version = {}.{}.{}".format(MAJOR_NUM, MINOR_NUM, REVISION)) + if args.bind or args.unbind: + dryrun = False + elif args.dryrun_bind or args.dryrun_unbind: + dryrun = True + else: + exit(parser.print_help()) + + if args.verbose: + verb = True + else: + verb = False + + # Read the environment variables + test_dir = os.environ.get("TEST_DIR") + dpdk_devbind = os.environ.get("DPDK_DEVBIND") + if test_dir is None or dpdk_devbind is None: + exit("Please set the environment 'TEST_DIR' and 'DPDK_DEVBIND'") + + runner = CIRunner(dir=test_dir, dpdk_devbind=dpdk_devbind, verb=verb, dry_run=dryrun) + + os.system("> {}/configs/pcie.ini".format(test_dir)) + os.system("echo '[default]' >> {}/configs/pcie.ini".format(test_dir)) + + # Init PF devices + runner.init_pf() + + if args.unbind or args.dryrun_unbind: + # Unbind all devices bound to vfio-pci + runner.unbind_all() + else: + runner.enable_sriov() + # Bind event-dev + runner.bind_evtdev() + # Bind cpt-dev + runner.bind_cptdev() + # Bind lbk-dev + runner.bind_lbkdev() + # Bind inline-dev + runner.bind_inldev() diff --git a/ci/test/board/dpdk-devbind.py b/ci/test/board/dpdk-devbind.py new file mode 100755 index 0000000000..9d4828db9f --- /dev/null +++ b/ci/test/board/dpdk-devbind.py @@ -0,0 +1,853 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2010-2014 Intel Corporation +# +# Copied from dpdk-24.11. + +import sys +import os +import subprocess +import argparse +import platform + +from glob import glob +from os.path import exists, basename +from os.path import join as path_join + +# The PCI base class for all devices +network_class = {'Class': '02', 'Vendor': None, 'Device': None, + 'SVendor': None, 'SDevice': None} +acceleration_class = {'Class': '12', 'Vendor': None, 'Device': None, + 'SVendor': None, 'SDevice': None} +ifpga_class = {'Class': '12', 'Vendor': '8086', 'Device': '0b30', + 'SVendor': None, 'SDevice': None} +encryption_class = {'Class': '10', 'Vendor': None, 'Device': None, + 'SVendor': None, 'SDevice': None} +intel_processor_class = {'Class': '0b', 'Vendor': '8086', 'Device': None, + 'SVendor': None, 'SDevice': None} +cavium_sso = {'Class': '08', 'Vendor': '177d', 'Device': 'a04b,a04d', + 'SVendor': None, 'SDevice': None} +cavium_fpa = {'Class': '08', 'Vendor': '177d', 'Device': 'a053', + 'SVendor': None, 'SDevice': None} +cavium_pkx = {'Class': '08', 'Vendor': '177d', 'Device': 'a0dd,a049', + 'SVendor': None, 'SDevice': None} +cavium_tim = {'Class': '08', 'Vendor': '177d', 'Device': 'a051', + 'SVendor': None, 'SDevice': None} +cavium_zip = {'Class': '12', 'Vendor': '177d', 'Device': 'a037', + 'SVendor': None, 'SDevice': None} +avp_vnic = {'Class': '05', 'Vendor': '1af4', 'Device': '1110', + 'SVendor': None, 'SDevice': None} + +cnxk_bphy = {'Class': '08', 'Vendor': '177d', 'Device': 'a089', + 'SVendor': None, 'SDevice': None} +cnxk_bphy_cgx = {'Class': '08', 'Vendor': '177d', 'Device': 'a059,a060', + 'SVendor': None, 'SDevice': None} +cnxk_dma = {'Class': '08', 'Vendor': '177d', 'Device': 'a081', + 'SVendor': None, 'SDevice': None} +cnxk_inl_dev = {'Class': '08', 'Vendor': '177d', 'Device': 'a0f0,a0f1', + 'SVendor': None, 'SDevice': None} + +hisilicon_dma = {'Class': '08', 'Vendor': '19e5', 'Device': 'a122', + 'SVendor': None, 'SDevice': None} +odm_dma = {'Class': '08', 'Vendor': '177d', 'Device': 'a08c', + 'SVendor': None, 'SDevice': None} + +intel_dlb = {'Class': '0b', 'Vendor': '8086', 'Device': '270b,2710,2714', + 'SVendor': None, 'SDevice': None} +intel_ioat_bdw = {'Class': '08', 'Vendor': '8086', + 'Device': '6f20,6f21,6f22,6f23,6f24,6f25,6f26,6f27,6f2e,6f2f', + 'SVendor': None, 'SDevice': None} +intel_ioat_skx = {'Class': '08', 'Vendor': '8086', 'Device': '2021', + 'SVendor': None, 'SDevice': None} +intel_ioat_icx = {'Class': '08', 'Vendor': '8086', 'Device': '0b00', + 'SVendor': None, 'SDevice': None} +intel_idxd_spr = {'Class': '08', 'Vendor': '8086', 'Device': '0b25', + 'SVendor': None, 'SDevice': None} +intel_ntb_skx = {'Class': '06', 'Vendor': '8086', 'Device': '201c', + 'SVendor': None, 'SDevice': None} +intel_ntb_icx = {'Class': '06', 'Vendor': '8086', 'Device': '347e', + 'SVendor': None, 'SDevice': None} + +cnxk_sso = {'Class': '08', 'Vendor': '177d', 'Device': 'a0f9,a0fa', + 'SVendor': None, 'SDevice': None} +cnxk_npa = {'Class': '08', 'Vendor': '177d', 'Device': 'a0fb,a0fc', + 'SVendor': None, 'SDevice': None} +cn9k_ree = {'Class': '08', 'Vendor': '177d', 'Device': 'a0f4', + 'SVendor': None, 'SDevice': None} + +virtio_blk = {'Class': '01', 'Vendor': "1af4", 'Device': '1001,1042', + 'SVendor': None, 'SDevice': None} + +cnxk_ml = {'Class': '08', 'Vendor': '177d', 'Device': 'a092', + 'SVendor': None, 'SDevice': None} + +network_devices = [network_class, cavium_pkx, avp_vnic, ifpga_class] +baseband_devices = [acceleration_class] +crypto_devices = [encryption_class, intel_processor_class] +dma_devices = [cnxk_dma, hisilicon_dma, + intel_idxd_spr, intel_ioat_bdw, intel_ioat_icx, intel_ioat_skx, + odm_dma] +eventdev_devices = [cavium_sso, cavium_tim, intel_dlb, cnxk_sso] +mempool_devices = [cavium_fpa, cnxk_npa] +compress_devices = [cavium_zip] +regex_devices = [cn9k_ree] +ml_devices = [cnxk_ml] +misc_devices = [cnxk_bphy, cnxk_bphy_cgx, cnxk_inl_dev, + intel_ntb_skx, intel_ntb_icx, + virtio_blk] + +# global dict ethernet devices present. Dictionary indexed by PCI address. +# Each device within this is itself a dictionary of device properties +devices = {} +# list of supported DPDK drivers +dpdk_drivers = ["igb_uio", "vfio-pci", "uio_pci_generic"] +# list of currently loaded kernel modules +loaded_modules = None + +# command-line arg flags +b_flag = None +status_flag = False +force_flag = False +noiommu_flag = False +args = [] + + +# check if a specific kernel module is loaded +def module_is_loaded(module): + global loaded_modules + + if module == 'vfio_pci': + module = 'vfio-pci' + + if loaded_modules: + return module in loaded_modules + + # Get list of sysfs modules (both built-in and dynamically loaded) + sysfs_path = '/sys/module/' + + # Get the list of directories in sysfs_path + sysfs_mods = [m for m in os.listdir(sysfs_path) + if os.path.isdir(os.path.join(sysfs_path, m))] + + # special case for vfio_pci (module is named vfio-pci, + # but its .ko is named vfio_pci) + sysfs_mods = [a if a != 'vfio_pci' else 'vfio-pci' for a in sysfs_mods] + + loaded_modules = sysfs_mods + + # add built-in modules as loaded + release = platform.uname().release + filename = os.path.join("/lib/modules/", release, "modules.builtin") + if os.path.exists(filename): + try: + with open(filename) as f: + loaded_modules += [os.path.splitext(os.path.basename(mod))[0] for mod in f] + except IOError: + print("Warning: cannot read list of built-in kernel modules") + + return module in loaded_modules + + +def check_modules(): + '''Checks that igb_uio is loaded''' + global dpdk_drivers + + # list of supported modules + mods = [{"Name": driver, "Found": False} for driver in dpdk_drivers] + + # first check if module is loaded + for mod in mods: + if module_is_loaded(mod["Name"]): + mod["Found"] = True + + # check if we have at least one loaded module + if True not in [mod["Found"] for mod in mods] and b_flag is not None: + print("Warning: no supported DPDK kernel modules are loaded", file=sys.stderr) + + # change DPDK driver list to only contain drivers that are loaded + dpdk_drivers = [mod["Name"] for mod in mods if mod["Found"]] + + +def has_driver(dev_id): + '''return true if a device is assigned to a driver. False otherwise''' + return "Driver_str" in devices[dev_id] + + +def get_pci_device_details(dev_id, probe_lspci): + '''This function gets additional details for a PCI device''' + device = {} + + if probe_lspci: + extra_info = subprocess.check_output(["lspci", "-vmmks", dev_id]).splitlines() + # parse lspci details + for line in extra_info: + if not line: + continue + name, value = line.decode("utf8").split("\t", 1) + name = name.strip(":") + "_str" + device[name] = value + # check for a unix interface name + device["Interface"] = "" + for base, dirs, _ in os.walk("/sys/bus/pci/devices/%s/" % dev_id): + if "net" in dirs: + device["Interface"] = \ + ",".join(os.listdir(os.path.join(base, "net"))) + break + # check if a port is used for ssh connection + device["Ssh_if"] = False + device["Active"] = "" + + return device + + +def clear_data(): + '''This function clears any old data''' + global devices + devices = {} + + +def get_device_details(devices_type): + '''This function populates the "devices" dictionary. The keys used are + the pci addresses (domain:bus:slot.func). The values are themselves + dictionaries - one for each NIC.''' + global devices + global dpdk_drivers + + # first loop through and read details for all devices + # request machine readable format, with numeric IDs and String + dev = {} + dev_lines = subprocess.check_output(["lspci", "-Dvmmnnk"]).splitlines() + for dev_line in dev_lines: + if not dev_line: + if device_type_match(dev, devices_type): + # Replace "Driver" with "Driver_str" to have consistency of + # of dictionary key names + if "Driver" in dev.keys(): + dev["Driver_str"] = dev.pop("Driver") + if "Module" in dev.keys(): + dev["Module_str"] = dev.pop("Module") + # use dict to make copy of dev + devices[dev["Slot"]] = dict(dev) + # Clear previous device's data + dev = {} + else: + name, value = dev_line.decode("utf8").split("\t", 1) + value_list = value.rsplit(' ', 1) + if value_list: + # String stored in _str + dev[name.rstrip(":") + '_str'] = value_list[0] + # Numeric IDs + dev[name.rstrip(":")] = value_list[len(value_list) - 1] \ + .rstrip("]").lstrip("[") + + if devices_type == network_devices: + # check what is the interface if any for an ssh connection if + # any to this host, so we can mark it later. + ssh_if = [] + route = subprocess.check_output(["ip", "-o", "route"]) + # filter out all lines for 169.254 routes + route = "\n".join(filter(lambda ln: not ln.startswith("169.254"), + route.decode().splitlines())) + rt_info = route.split() + for i in range(len(rt_info) - 1): + if rt_info[i] == "dev": + ssh_if.append(rt_info[i + 1]) + + # based on the basic info, get extended text details + for d in devices.keys(): + if not device_type_match(devices[d], devices_type): + continue + + # get additional info and add it to existing data + devices[d] = devices[d].copy() + # No need to probe lspci + devices[d].update(get_pci_device_details(d, False).items()) + + if devices_type == network_devices: + for _if in ssh_if: + if _if in devices[d]["Interface"].split(","): + devices[d]["Ssh_if"] = True + devices[d]["Active"] = "*Active*" + break + + # add igb_uio to list of supporting modules if needed + if "Module_str" in devices[d]: + for driver in dpdk_drivers: + if driver not in devices[d]["Module_str"]: + devices[d]["Module_str"] = \ + devices[d]["Module_str"] + ",%s" % driver + else: + devices[d]["Module_str"] = ",".join(dpdk_drivers) + + # make sure the driver and module strings do not have any duplicates + if has_driver(d): + modules = devices[d]["Module_str"].split(",") + if devices[d]["Driver_str"] in modules: + modules.remove(devices[d]["Driver_str"]) + devices[d]["Module_str"] = ",".join(modules) + + +def device_type_match(dev, devices_type): + for i in range(len(devices_type)): + param_count = len( + [x for x in devices_type[i].values() if x is not None]) + match_count = 0 + if dev["Class"][0:2] == devices_type[i]["Class"]: + match_count = match_count + 1 + for key in devices_type[i].keys(): + if key != 'Class' and devices_type[i][key]: + value_list = devices_type[i][key].split(',') + for value in value_list: + if value.strip(' ') == dev[key]: + match_count = match_count + 1 + # count must be the number of non None parameters to match + if match_count == param_count: + return True + return False + + +def dev_id_from_dev_name(dev_name): + '''Take a device "name" - a string passed in by user to identify a NIC + device, and determine the device id - i.e. the domain:bus:slot.func - for + it, which can then be used to index into the devices array''' + + # check if it's already a suitable index + if dev_name in devices: + return dev_name + # check if it's an index just missing the domain part + if "0000:" + dev_name in devices: + return "0000:" + dev_name + + # check if it's an interface name, e.g. eth1 + for d in devices.keys(): + if dev_name in devices[d]["Interface"].split(","): + return devices[d]["Slot"] + # if nothing else matches - error + raise ValueError("Unknown device: %s. " + "Please specify device in \"bus:slot.func\" format" % dev_name) + + +def unbind_one(dev_id, force): + '''Unbind the device identified by "dev_id" from its current driver''' + dev = devices[dev_id] + if not has_driver(dev_id): + print("Notice: %s %s %s is not currently managed by any driver" % + (dev["Slot"], dev["Device_str"], dev["Interface"]), file=sys.stderr) + return + + # prevent us disconnecting ourselves + if dev["Ssh_if"] and not force: + print("Warning: routing table indicates that interface %s is active. " + "Skipping unbind" % dev_id, file=sys.stderr) + return + + # write to /sys to unbind + filename = "/sys/bus/pci/drivers/%s/unbind" % dev["Driver_str"] + try: + f = open(filename, "a") + except OSError as err: + sys.exit("Error: unbind failed for %s - Cannot open %s: %s" % + (dev_id, filename, err)) + f.write(dev_id) + f.close() + + +def bind_one(dev_id, driver, force): + '''Bind the device given by "dev_id" to the driver "driver". If the device + is already bound to a different driver, it will be unbound first''' + dev = devices[dev_id] + saved_driver = None # used to rollback any unbind in case of failure + + # prevent disconnection of our ssh session + if dev["Ssh_if"] and not force: + print("Warning: routing table indicates that interface %s is active. " + "Not modifying" % dev_id, file=sys.stderr) + return + + # unbind any existing drivers we don't want + if has_driver(dev_id): + if dev["Driver_str"] == driver: + print("Notice: %s already bound to driver %s, skipping" % + (dev_id, driver), file=sys.stderr) + return + saved_driver = dev["Driver_str"] + unbind_one(dev_id, force) + dev["Driver_str"] = "" # clear driver string + + # For kernels >= 3.15 driver_override can be used to specify the driver + # for a device rather than relying on the driver to provide a positive + # match of the device. The existing process of looking up + # the vendor and device ID, adding them to the driver new_id, + # will erroneously bind other devices too which has the additional burden + # of unbinding those devices + if driver in dpdk_drivers: + filename = "/sys/bus/pci/devices/%s/driver_override" % dev_id + if exists(filename): + try: + f = open(filename, "w") + except OSError as err: + print("Error: bind failed for %s - Cannot open %s: %s" + % (dev_id, filename, err), file=sys.stderr) + return + try: + f.write("%s" % driver) + f.close() + except OSError as err: + print("Error: bind failed for %s - Cannot write driver %s to " + "PCI ID: %s" % (dev_id, driver, err), file=sys.stderr) + return + # For kernels < 3.15 use new_id to add PCI id's to the driver + else: + filename = "/sys/bus/pci/drivers/%s/new_id" % driver + try: + f = open(filename, "w") + except OSError as err: + print("Error: bind failed for %s - Cannot open %s: %s" + % (dev_id, filename, err), file=sys.stderr) + return + try: + # Convert Device and Vendor Id to int to write to new_id + f.write("%04x %04x" % (int(dev["Vendor"], 16), + int(dev["Device"], 16))) + f.close() + except OSError as err: + print("Error: bind failed for %s - Cannot write new PCI ID to " + "driver %s: %s" % (dev_id, driver, err), file=sys.stderr) + return + + # do the bind by writing to /sys + filename = "/sys/bus/pci/drivers/%s/bind" % driver + try: + f = open(filename, "a") + except OSError as err: + print("Error: bind failed for %s - Cannot open %s: %s" + % (dev_id, filename, err), file=sys.stderr) + if saved_driver is not None: # restore any previous driver + bind_one(dev_id, saved_driver, force) + return + try: + f.write(dev_id) + f.close() + except OSError as err: + # for some reason, closing dev_id after adding a new PCI ID to new_id + # results in IOError. however, if the device was successfully bound, + # we don't care for any errors and can safely ignore IOError + tmp = get_pci_device_details(dev_id, True) + if "Driver_str" in tmp and tmp["Driver_str"] == driver: + return + print("Error: bind failed for %s - Cannot bind to driver %s: %s" + % (dev_id, driver, err), file=sys.stderr) + if saved_driver is not None: # restore any previous driver + bind_one(dev_id, saved_driver, force) + return + + # For kernels > 3.15 driver_override is used to bind a device to a driver. + # Before unbinding it, overwrite driver_override with empty string so that + # the device can be bound to any other driver + filename = "/sys/bus/pci/devices/%s/driver_override" % dev_id + if exists(filename): + try: + f = open(filename, "w") + except OSError as err: + sys.exit("Error: unbind failed for %s - Cannot open %s: %s" + % (dev_id, filename, err)) + try: + f.write("\00") + f.close() + except OSError as err: + sys.exit("Error: unbind failed for %s - Cannot write %s: %s" + % (dev_id, filename, err)) + + +def unbind_all(dev_list, force=False): + """Unbind method, takes a list of device locations""" + + if dev_list[0] == "dpdk": + for d in devices.keys(): + if "Driver_str" in devices[d]: + if devices[d]["Driver_str"] in dpdk_drivers: + unbind_one(devices[d]["Slot"], force) + return + + try: + dev_list = map(dev_id_from_dev_name, dev_list) + except ValueError as ex: + print(ex) + sys.exit(1) + + for d in dev_list: + unbind_one(d, force) + + +def has_iommu(): + """Check if IOMMU is enabled on system""" + return len(os.listdir("/sys/class/iommu")) > 0 + + +def check_noiommu_mode(): + """Check and enable the noiommu mode for VFIO drivers""" + global noiommu_flag + filename = "/sys/module/vfio/parameters/enable_unsafe_noiommu_mode" + + try: + with open(filename, "r") as f: + value = f.read(1) + if value in ("1", "y" ,"Y"): + return + except OSError as err: + sys.exit(f"Error: failed to check unsafe noiommu mode - Cannot open {filename}: {err}") + + if not noiommu_flag: + sys.exit("Error: IOMMU support is disabled, use --noiommu-mode for binding in noiommu mode") + + try: + with open(filename, "w") as f: + f.write("1") + except OSError as err: + sys.exit(f"Error: failed to enable unsafe noiommu mode - Cannot open {filename}: {err}") + print("Warning: enabling unsafe no IOMMU mode for VFIO drivers") + + +def bind_all(dev_list, driver, force=False): + """Bind method, takes a list of device locations""" + global devices + + # a common user error is to forget to specify the driver the devices need to + # be bound to. check if the driver is a valid device, and if it is, show + # a meaningful error. + try: + dev_id_from_dev_name(driver) + # if we've made it this far, this means that the "driver" was a valid + # device string, so it's probably not a valid driver name. + sys.exit("Error: Driver '%s' does not look like a valid driver. " + "Did you forget to specify the driver to bind devices to?" % driver) + except ValueError: + # driver generated error - it's not a valid device ID, so all is well + pass + + # check if we're attempting to bind to a driver that isn't loaded + if not module_is_loaded(driver.replace('-', '_')): + sys.exit("Error: Driver '%s' is not loaded." % driver) + + try: + dev_list = map(dev_id_from_dev_name, dev_list) + except ValueError as ex: + sys.exit(ex) + + # check for IOMMU support + if driver == "vfio-pci" and not has_iommu(): + check_noiommu_mode() + + for d in dev_list: + bind_one(d, driver, force) + + # For kernels < 3.15 when binding devices to a generic driver + # (i.e. one that doesn't have a PCI ID table) using new_id, some devices + # that are not bound to any other driver could be bound even if no one has + # asked them to. hence, we check the list of drivers again, and see if + # some of the previously-unbound devices were erroneously bound. + if not exists("/sys/bus/pci/devices/%s/driver_override" % d): + for d in devices.keys(): + # skip devices that were already bound or that we know should be bound + if "Driver_str" in devices[d] or d in dev_list: + continue + + # update information about this device + devices[d] = dict(devices[d].items() + + get_pci_device_details(d, True).items()) + + # check if updated information indicates that the device was bound + if "Driver_str" in devices[d]: + unbind_one(d, force) + + +def display_devices(title, dev_list, extra_params=None): + '''Displays to the user the details of a list of devices given in + "dev_list". The "extra_params" parameter, if given, should contain a string + with %()s fields in it for replacement by the named fields in each + device's dictionary.''' + strings = [] # this holds the strings to print. We sort before printing + print("\n%s" % title) + print("=" * len(title)) + if not dev_list: + strings.append("") + else: + for dev in dev_list: + if extra_params is not None: + strings.append("%s '%s %s' %s" % (dev["Slot"], + dev["Device_str"], + dev["Device"], + extra_params % dev)) + else: + strings.append("%s '%s'" % (dev["Slot"], dev["Device_str"])) + # sort before printing, so that the entries appear in PCI order + strings.sort() + print("\n".join(strings)) # print one per line + + +def show_device_status(devices_type, device_name, if_field=False): + global dpdk_drivers + kernel_drv = [] + dpdk_drv = [] + no_drv = [] + + print_numa = True # by default, assume we can print NUMA information + + # split our list of network devices into the three categories above + for d in devices.keys(): + if device_type_match(devices[d], devices_type): + print_numa &= "NUMANode" in devices[d] + if not has_driver(d): + no_drv.append(devices[d]) + continue + if devices[d]["Driver_str"] in dpdk_drivers: + dpdk_drv.append(devices[d]) + else: + kernel_drv.append(devices[d]) + + n_devs = len(dpdk_drv) + len(kernel_drv) + len(no_drv) + + # don't bother displaying anything if there are no devices + if n_devs == 0: + msg = "No '%s' devices detected" % device_name + print("") + print(msg) + print("".join('=' * len(msg))) + return + + # print each category separately, so we can clearly see what's used by DPDK + if dpdk_drv: + extra_param = "drv=%(Driver_str)s unused=%(Module_str)s" + if print_numa: + extra_param = "numa_node=%(NUMANode)s " + extra_param + display_devices("%s devices using DPDK-compatible driver" % device_name, + dpdk_drv, extra_param) + if kernel_drv: + extra_param = "drv=%(Driver_str)s unused=%(Module_str)s" + if if_field: + extra_param = "if=%(Interface)s " + extra_param + if print_numa: + extra_param = "numa_node=%(NUMANode)s " + extra_param + extra_param += " %(Active)s" + display_devices("%s devices using kernel driver" % device_name, + kernel_drv, extra_param) + if no_drv: + extra_param = "unused=%(Module_str)s" + if print_numa: + extra_param = "numa_node=%(NUMANode)s " + extra_param + display_devices("Other %s devices" % device_name, no_drv, extra_param) + + +def show_status(): + '''Function called when the script is passed the "--status" option. + Displays to the user what devices are bound to the igb_uio driver, the + kernel driver or to no driver''' + + if status_dev in ["net", "all"]: + show_device_status(network_devices, "Network", if_field=True) + + if status_dev in ["baseband", "all"]: + show_device_status(baseband_devices, "Baseband") + + if status_dev in ["crypto", "all"]: + show_device_status(crypto_devices, "Crypto") + + if status_dev in ["dma", "all"]: + show_device_status(dma_devices, "DMA") + + if status_dev in ["event", "all"]: + show_device_status(eventdev_devices, "Eventdev") + + if status_dev in ["mempool", "all"]: + show_device_status(mempool_devices, "Mempool") + + if status_dev in ["compress", "all"]: + show_device_status(compress_devices, "Compress") + + if status_dev in ["misc", "all"]: + show_device_status(misc_devices, "Misc (rawdev)") + + if status_dev in ["regex", "all"]: + show_device_status(regex_devices, "Regex") + + if status_dev in ["ml", "all"]: + show_device_status(ml_devices, "ML") + + +def pci_glob(arg): + '''Returns a list containing either: + * List of PCI B:D:F matching arg, using shell wildcards e.g. 80:04.* + * Only the passed arg if matching list is empty''' + sysfs_path = "/sys/bus/pci/devices" + for _glob in [arg, '0000:' + arg]: + paths = [basename(path) for path in glob(path_join(sysfs_path, _glob))] + if paths: + return paths + return [arg] + + +def parse_args(): + '''Parses the command-line arguments given by the user and takes the + appropriate action for each''' + global b_flag + global status_flag + global status_dev + global force_flag + global noiommu_flag + global args + + parser = argparse.ArgumentParser( + description='Utility to bind and unbind devices from Linux kernel', + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Examples: +--------- + +To display current device status: + %(prog)s --status + +To display current network device status: + %(prog)s --status-dev net + +To bind eth1 from the current driver and move to use vfio-pci + %(prog)s --bind=vfio-pci eth1 + +To unbind 0000:01:00.0 from using any driver + %(prog)s -u 0000:01:00.0 + +To bind 0000:02:00.0 and 0000:02:00.1 to the ixgbe kernel driver + %(prog)s -b ixgbe 02:00.0 02:00.1 +""") + + parser.add_argument( + '-s', + '--status', + action='store_true', + help="Print the current status of all known devices.") + parser.add_argument( + '--status-dev', + help="Print the status of given device group.", + choices=['baseband', 'compress', 'crypto', 'dma', 'event', + 'mempool', 'misc', 'net', 'regex', 'ml']) + bind_group = parser.add_mutually_exclusive_group() + bind_group.add_argument( + '-b', + '--bind', + metavar='DRIVER', + help="Select the driver to use or \"none\" to unbind the device") + bind_group.add_argument( + '-u', + '--unbind', + action='store_true', + help="Unbind a device (equivalent to \"-b none\")") + parser.add_argument( + '--noiommu-mode', + action='store_true', + help="If IOMMU is not available, enable no IOMMU mode for VFIO drivers") + parser.add_argument( + '--force', + action='store_true', + help=""" +Override restriction on binding devices in use by Linux" +WARNING: This can lead to loss of network connection and should be used with caution. +""") + parser.add_argument( + 'devices', + metavar='DEVICE', + nargs='*', + help=""" +Device specified as PCI "domain:bus:slot.func" syntax or "bus:slot.func" syntax. +For devices bound to Linux kernel drivers, they may be referred to by interface name. +""") + + opt = parser.parse_args() + + if opt.status_dev: + status_flag = True + status_dev = opt.status_dev + if opt.status: + status_flag = True + status_dev = "all" + if opt.force: + force_flag = True + if opt.noiommu_mode: + noiommu_flag = True + if opt.bind: + b_flag = opt.bind + elif opt.unbind: + b_flag = "none" + args = opt.devices + + if not b_flag and not status_flag: + print("Error: No action specified for devices. " + "Please give a --bind, --ubind or --status option", + file=sys.stderr) + parser.print_usage() + sys.exit(1) + + if b_flag and not args: + print("Error: No devices specified.", file=sys.stderr) + parser.print_usage() + sys.exit(1) + + # resolve any PCI globs in the args + new_args = [] + for arg in args: + new_args.extend(pci_glob(arg)) + args = new_args + + +def do_arg_actions(): + '''do the actual action requested by the user''' + global b_flag + global status_flag + global force_flag + global args + + if b_flag in ["none", "None"]: + unbind_all(args, force_flag) + elif b_flag is not None: + bind_all(args, b_flag, force_flag) + if status_flag: + if b_flag is not None: + clear_data() + # refresh if we have changed anything + get_device_details(network_devices) + get_device_details(baseband_devices) + get_device_details(crypto_devices) + get_device_details(dma_devices) + get_device_details(eventdev_devices) + get_device_details(mempool_devices) + get_device_details(compress_devices) + get_device_details(regex_devices) + get_device_details(ml_devices) + get_device_details(misc_devices) + show_status() + + +def main(): + '''program main function''' + # check if lspci is installed, suppress any output + with open(os.devnull, 'w') as devnull: + ret = subprocess.call(['which', 'lspci'], + stdout=devnull, stderr=devnull) + if ret != 0: + sys.exit("'lspci' not found - please install 'pciutils'") + parse_args() + check_modules() + clear_data() + get_device_details(network_devices) + get_device_details(baseband_devices) + get_device_details(crypto_devices) + get_device_details(dma_devices) + get_device_details(eventdev_devices) + get_device_details(mempool_devices) + get_device_details(compress_devices) + get_device_details(regex_devices) + get_device_details(ml_devices) + get_device_details(misc_devices) + do_arg_actions() + + +if __name__ == "__main__": + main() diff --git a/ci/test/test.sh b/ci/test/test.sh new file mode 100755 index 0000000000..73724a4784 --- /dev/null +++ b/ci/test/test.sh @@ -0,0 +1,63 @@ +#!/bin/bash +# Marvell CONFIDENTIAL AND PROPRIETARY NOTE +# +# This software contains information confidential and proprietary to +# Marvell. It shall not be reproduced in whole or in +# part, or transferred to other documents, or disclosed to third +# parties, or used for any purpose other than that for which it was +# obtained, without the prior written consent of Marvell. +# +# Copyright (c) 2025 Marvell. If you received this file from Marvell +# and you have entered into a commercial license agreement (a "Commercial License") +# with Marvell, the file is licensed to you under the terms of the applicable Commercial +# License. In the absence of such license agreement the following file is subject to +# Marvell’s standard Limited Use License Agreement. + +set -euox pipefail + +function target_board_init() { + echo "Setting up target board for running tests..." + $REMOTE "sudo TEST_DIR=${TEST_DIR} DPDK_DEVBIND=${REMOTE_BUILD_DIR}/ci/test/board/dpdk-devbind.py python3 ${REMOTE_BUILD_DIR}/ci/test/board/ci_runner.py -bv" +} + +function install_packages() { + echo "Enabling internet on target board...." + $REMOTE 'echo "DNS=10.28.116.24 10.31.116.251 10.68.76.63" | sudo tee -a /etc/systemd/resolved.conf' + $REMOTE "sudo systemctl restart systemd-resolved" + sleep 30 + echo "Installing essential packages..." + $REMOTE "sudo apt-get update" + $REMOTE "sudo apt-get install -y python3-venv python3-pip" + $REMOTE "sudo pip3 install --break-system-packages --no-input psutil syslog_rfc5424_parser parameterized noise" +} + +function sync_files() { + echo "Syncing build to target board..." + $REMOTE "rm -rf $REMOTE_DIR" + $REMOTE "mkdir -p $REMOTE_DIR" + # Sync build directory + rsync -e "$TARGET_SSH_CMD" -av $BUILD_DIR/* $TARGET_BOARD:$REMOTE_BUILD_DIR/ + # Sync deps build directory if required + rsync -e "$TARGET_SSH_CMD" -r $DEPS_DIR/* $TARGET_BOARD:$REMOTE_DIR/deps_build +} + +function run_tests() { + echo "Running tests using run_tests.py..." + $REMOTE "python3 ${REMOTE_BUILD_DIR}/test/run_tests.py -d ${TEST_DIR}" +} + +PROJECT_ROOT=${PROJECT_ROOT:-$PWD} +TARGET_BOARD=${TARGET_BOARD:-root@127.0.0.1} +TARGET_SSH_CMD=${TARGET_SSH_CMD:-"ssh"} +REMOTE="$TARGET_SSH_CMD $TARGET_BOARD -n" +REMOTE_DIR=${REMOTE_DIR:-/tmp/vpp} +BUILD_DIR=${BUILD_DIR:-$PWD/build} +DEPS_DIR=${DEPS_DIR:-${PROJECT_ROOT}/deps-prefix} +REMOTE_BUILD_DIR=${REMOTE_DIR}/build + +TEST_DIR=${REMOTE_BUILD_DIR}/src/plugins/onp/test/ + +install_packages +sync_files +target_board_init +run_tests From b8ba87fde5204a9051b2012b258bcad4a31a8792 Mon Sep 17 00:00:00 2001 From: Monendra Singh Kushwaha Date: Tue, 1 Apr 2025 13:08:42 +0530 Subject: [PATCH 212/271] octeon: fix debug mode issue Type: fix Change-Id: I0d5a5cb44406d688a89fea2ac7463f7193400941 Signed-off-by: Monendra Singh Kushwaha Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/149660 Tested-by: sa_ip-toolkits-Jenkins (cherry picked from commit b91b78dd374aef59f816157a5c7ce7a57ac802a2) Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/149671 Tested-by: sa_ip-sw-jenkins --- src/plugins/dev_octeon/tx_node.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/plugins/dev_octeon/tx_node.c b/src/plugins/dev_octeon/tx_node.c index 692de07172..9fe0bed703 100644 --- a/src/plugins/dev_octeon/tx_node.c +++ b/src/plugins/dev_octeon/tx_node.c @@ -772,7 +772,6 @@ oct_prepare_ipsec_inst (vlib_main_t *vm, vlib_buffer_t *b, u64 sq_handle, if (b->flags & VLIB_BUFFER_NEXT_PRESENT || rlen > buffer_data_size) { - ASSERT (0); total_length = b->current_length + b->total_length_not_including_first_buffer; inst->w4.u64 = sess->inst.w4.u64; From 1233ea732ccb274d4c4f15560d5093db7e4a8e88 Mon Sep 17 00:00:00 2001 From: Nithinsen Kaithakadan Date: Thu, 20 Mar 2025 23:00:54 +0530 Subject: [PATCH 213/271] octeon: add model check for cn20k Type: feature Signed-off-by: Nithinsen Kaithakadan Change-Id: I9e63f134085465ff7dfeb4856bfd523b9aedfbfd Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/148890 Reviewed-by: Nithinsen Kaithakadan Reviewed-by: Tejasree Kondoj Tested-by: sa_ip-toolkits-Jenkins (cherry picked from commit abed407a2f8e1cc5df801b6daf4d12ccbb904894) Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/149682 Tested-by: sa_ip-sw-jenkins --- src/plugins/dev_octeon/crypto.c | 5 ++++- src/plugins/dev_octeon/init.c | 20 ++++++++++++-------- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/src/plugins/dev_octeon/crypto.c b/src/plugins/dev_octeon/crypto.c index be0d19ff7d..7c8f70e6bf 100644 --- a/src/plugins/dev_octeon/crypto.c +++ b/src/plugins/dev_octeon/crypto.c @@ -1185,7 +1185,10 @@ oct_cpt_inst_w7_get (oct_crypto_sess_t *sess, struct roc_cpt *roc_cpt) inst_w7.s.ctx_val = 1; /* Set the engine group */ - inst_w7.s.egrp = roc_cpt->eng_grp[CPT_ENG_TYPE_IE]; + if (roc_model_is_cn20k ()) + inst_w7.s.egrp = roc_cpt->eng_grp[CPT_ENG_TYPE_SE]; + else + inst_w7.s.egrp = roc_cpt->eng_grp[CPT_ENG_TYPE_IE]; return inst_w7.u64; } diff --git a/src/plugins/dev_octeon/init.c b/src/plugins/dev_octeon/init.c index abd97f3138..7d61a9865a 100644 --- a/src/plugins/dev_octeon/init.c +++ b/src/plugins/dev_octeon/init.c @@ -346,15 +346,19 @@ oct_conf_cpt (vlib_main_t *vm, vnet_dev_t *dev, oct_crypto_dev_t *ocd, log_err (dev, "Could not add CPT SE engines"); return cnx_return_roc_err (dev, rrv, "roc_cpt_eng_grp_add"); } - if ((rrv = roc_cpt_eng_grp_add (roc_cpt, CPT_ENG_TYPE_IE)) < 0) + if (!roc_model_is_cn20k ()) { - log_err (dev, "Could not add CPT IE engines"); - return cnx_return_roc_err (dev, rrv, "roc_cpt_eng_grp_add"); - } - if (roc_cpt->eng_grp[CPT_ENG_TYPE_IE] != ROC_LEGACY_CPT_DFLT_ENG_GRP_SE_IE) - { - log_err (dev, "Invalid CPT IE engine group configuration"); - return -1; + if ((rrv = roc_cpt_eng_grp_add (roc_cpt, CPT_ENG_TYPE_IE)) < 0) + { + log_err (dev, "Could not add CPT IE engines"); + return cnx_return_roc_err (dev, rrv, "roc_cpt_eng_grp_add"); + } + if (roc_cpt->eng_grp[CPT_ENG_TYPE_IE] != + ROC_LEGACY_CPT_DFLT_ENG_GRP_SE_IE) + { + log_err (dev, "Invalid CPT IE engine group configuration"); + return -1; + } } if (roc_cpt->eng_grp[CPT_ENG_TYPE_SE] != ROC_LEGACY_CPT_DFLT_ENG_GRP_SE) { From e135b9ac0a537c10fd1d0aa754148c4c7a5399ef Mon Sep 17 00:00:00 2001 From: Alok Mishra Date: Thu, 27 Mar 2025 14:29:55 +0500 Subject: [PATCH 214/271] tm: update tm api version Type: fix Signed-off-by: Alok Mishra Change-Id: I4545e56ee5c9e752c0036258ab7b0f9b2cf9ed4d Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/149513 Tested-by: sa_ip-toolkits-Jenkins Reviewed-by: Monendra Singh Kushwaha (cherry picked from commit e05e614c725c6f623e42c595e0500b12678cef42) Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/149855 Tested-by: sa_ip-sw-jenkins --- src/vnet/tm/tm.api | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/vnet/tm/tm.api b/src/vnet/tm/tm.api index 16bfb8aa68..d792b8da89 100644 --- a/src/vnet/tm/tm.api +++ b/src/vnet/tm/tm.api @@ -4,6 +4,8 @@ * https://spdx.org/licenses/Apache-2.0.html */ +option version = "0.1.0"; + /** * @brief Reply for adding a traffic management node. * From 789764d1635cf3f1e74b19d6a33b0083fef3f9ee Mon Sep 17 00:00:00 2001 From: Monendra Singh Kushwaha Date: Thu, 3 Apr 2025 08:49:14 +0530 Subject: [PATCH 215/271] build: update roc version Type: feature Signed-off-by: Monendra Singh Kushwaha Change-Id: If24b300e525276c1865cf2f5b5443c29f4f78ffe Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/149903 Tested-by: sa_ip-sw-jenkins --- build/external/packages/octeon-roc.mk | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build/external/packages/octeon-roc.mk b/build/external/packages/octeon-roc.mk index 457c0b7d69..001e50da26 100644 --- a/build/external/packages/octeon-roc.mk +++ b/build/external/packages/octeon-roc.mk @@ -2,9 +2,9 @@ # SPDX-License-Identifier: Apache-2.0 # https://spdx.org/licenses/Apache-2.0.html -octeon-roc_version := octeon-roc-SDK12.25.03 +octeon-roc_version := octeon-roc-SDK12.25.04 octeon-roc_tarball := $(octeon-roc_version).tar.gz -octeon-roc_tarball_md5sum := c7fd294e3fdc170396096c4ef6789597 +octeon-roc_tarball_md5sum := a678243b30b4feb2346d38fbd3afdf5f octeon-roc_github := https://github.com/MarvellEmbeddedProcessors/marvell-octeon-roc octeon-roc_tarball_strip_dirs := 1 From 6ada8161d0c47c70568e23936b1200498961418c Mon Sep 17 00:00:00 2001 From: root Date: Thu, 3 Apr 2025 13:25:54 +0530 Subject: [PATCH 216/271] ci: update test dir and fix checkstyle Type: fix Signed-off-by: Monendra Singh Kushwaha Change-Id: I1c71126c2ce1f9c7836a2d3879b4c7ee77d9de95 Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/149938 (cherry picked from commit 83ff5ee578c3b32ac6f56caabc15f3e65fe0b6d3) Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/149939 Tested-by: sa_ip-sw-jenkins --- ci/test/board/ci_runner.py | 23 +- ci/test/board/dpdk-devbind.py | 853 ------------------ ci/test/test.sh | 7 +- src/plugins/dev_octeon/test/configs/pcie.ini | 1 + .../dev_octeon/test/configs/unittest.ini | 45 + 5 files changed, 69 insertions(+), 860 deletions(-) delete mode 100755 ci/test/board/dpdk-devbind.py create mode 100644 src/plugins/dev_octeon/test/configs/pcie.ini create mode 100644 src/plugins/dev_octeon/test/configs/unittest.ini diff --git a/ci/test/board/ci_runner.py b/ci/test/board/ci_runner.py index 5679c2cef9..9987b8494c 100755 --- a/ci/test/board/ci_runner.py +++ b/ci/test/board/ci_runner.py @@ -2,7 +2,6 @@ # SPDX-License-Identifier: Apache-2.0 # https://spdx.org/licenses/Apache-2.0.html # -# Copied from /src/plugins/onp/test/ci/ci_runner.py from vpp-24.02. import os import re @@ -248,7 +247,10 @@ def bind_evtdev_vf(self): self.run_cmd(cmdstr) devbdf = lnparse[0].decode("utf-8") - cmdstr = "echo vfio-pci > /sys/bus/pci/devices/%s/driver_override" % devbdf + cmdstr = ( + "echo vfio-pci > /sys/bus/pci/devices/%s/driver_override" + % devbdf + ) self.run_cmd(cmdstr) cmdstr = "echo %s > /sys/bus/pci/drivers/vfio-pci/bind" % devbdf self.run_cmd(cmdstr) @@ -313,7 +315,10 @@ def bind_cptdev(self): self.run_cmd(cmdstr) devbdf = lnparse[0].decode("utf-8") - cmdstr = "echo vfio-pci > /sys/bus/pci/devices/%s/driver_override" % devbdf + cmdstr = ( + "echo vfio-pci > /sys/bus/pci/devices/%s/driver_override" + % devbdf + ) self.run_cmd(cmdstr) cmdstr = "echo %s > /sys/bus/pci/drivers/vfio-pci/bind" % devbdf self.run_cmd(cmdstr) @@ -351,7 +356,9 @@ def bind_lbkdev(self): self.run_cmd(cmdstr) devbdf = lnparse[0].decode("utf-8") - cmdstr = "echo vfio-pci > /sys/bus/pci/devices/%s/driver_override" % devbdf + cmdstr = ( + "echo vfio-pci > /sys/bus/pci/devices/%s/driver_override" % devbdf + ) self.run_cmd(cmdstr) cmdstr = "echo %s > /sys/bus/pci/drivers/vfio-pci/bind" % devbdf self.run_cmd(cmdstr) @@ -389,7 +396,9 @@ def bind_inldev(self): self.run_cmd(cmdstr) devbdf = lnparse[0].decode("utf-8") - cmdstr = "echo vfio-pci > /sys/bus/pci/devices/%s/driver_override" % devbdf + cmdstr = ( + "echo vfio-pci > /sys/bus/pci/devices/%s/driver_override" % devbdf + ) self.run_cmd(cmdstr) cmdstr = "echo %s > /sys/bus/pci/drivers/vfio-pci/bind" % devbdf self.run_cmd(cmdstr) @@ -496,7 +505,9 @@ def enable_sriov(self): if test_dir is None or dpdk_devbind is None: exit("Please set the environment 'TEST_DIR' and 'DPDK_DEVBIND'") - runner = CIRunner(dir=test_dir, dpdk_devbind=dpdk_devbind, verb=verb, dry_run=dryrun) + runner = CIRunner( + dir=test_dir, dpdk_devbind=dpdk_devbind, verb=verb, dry_run=dryrun + ) os.system("> {}/configs/pcie.ini".format(test_dir)) os.system("echo '[default]' >> {}/configs/pcie.ini".format(test_dir)) diff --git a/ci/test/board/dpdk-devbind.py b/ci/test/board/dpdk-devbind.py deleted file mode 100755 index 9d4828db9f..0000000000 --- a/ci/test/board/dpdk-devbind.py +++ /dev/null @@ -1,853 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-License-Identifier: BSD-3-Clause -# Copyright(c) 2010-2014 Intel Corporation -# -# Copied from dpdk-24.11. - -import sys -import os -import subprocess -import argparse -import platform - -from glob import glob -from os.path import exists, basename -from os.path import join as path_join - -# The PCI base class for all devices -network_class = {'Class': '02', 'Vendor': None, 'Device': None, - 'SVendor': None, 'SDevice': None} -acceleration_class = {'Class': '12', 'Vendor': None, 'Device': None, - 'SVendor': None, 'SDevice': None} -ifpga_class = {'Class': '12', 'Vendor': '8086', 'Device': '0b30', - 'SVendor': None, 'SDevice': None} -encryption_class = {'Class': '10', 'Vendor': None, 'Device': None, - 'SVendor': None, 'SDevice': None} -intel_processor_class = {'Class': '0b', 'Vendor': '8086', 'Device': None, - 'SVendor': None, 'SDevice': None} -cavium_sso = {'Class': '08', 'Vendor': '177d', 'Device': 'a04b,a04d', - 'SVendor': None, 'SDevice': None} -cavium_fpa = {'Class': '08', 'Vendor': '177d', 'Device': 'a053', - 'SVendor': None, 'SDevice': None} -cavium_pkx = {'Class': '08', 'Vendor': '177d', 'Device': 'a0dd,a049', - 'SVendor': None, 'SDevice': None} -cavium_tim = {'Class': '08', 'Vendor': '177d', 'Device': 'a051', - 'SVendor': None, 'SDevice': None} -cavium_zip = {'Class': '12', 'Vendor': '177d', 'Device': 'a037', - 'SVendor': None, 'SDevice': None} -avp_vnic = {'Class': '05', 'Vendor': '1af4', 'Device': '1110', - 'SVendor': None, 'SDevice': None} - -cnxk_bphy = {'Class': '08', 'Vendor': '177d', 'Device': 'a089', - 'SVendor': None, 'SDevice': None} -cnxk_bphy_cgx = {'Class': '08', 'Vendor': '177d', 'Device': 'a059,a060', - 'SVendor': None, 'SDevice': None} -cnxk_dma = {'Class': '08', 'Vendor': '177d', 'Device': 'a081', - 'SVendor': None, 'SDevice': None} -cnxk_inl_dev = {'Class': '08', 'Vendor': '177d', 'Device': 'a0f0,a0f1', - 'SVendor': None, 'SDevice': None} - -hisilicon_dma = {'Class': '08', 'Vendor': '19e5', 'Device': 'a122', - 'SVendor': None, 'SDevice': None} -odm_dma = {'Class': '08', 'Vendor': '177d', 'Device': 'a08c', - 'SVendor': None, 'SDevice': None} - -intel_dlb = {'Class': '0b', 'Vendor': '8086', 'Device': '270b,2710,2714', - 'SVendor': None, 'SDevice': None} -intel_ioat_bdw = {'Class': '08', 'Vendor': '8086', - 'Device': '6f20,6f21,6f22,6f23,6f24,6f25,6f26,6f27,6f2e,6f2f', - 'SVendor': None, 'SDevice': None} -intel_ioat_skx = {'Class': '08', 'Vendor': '8086', 'Device': '2021', - 'SVendor': None, 'SDevice': None} -intel_ioat_icx = {'Class': '08', 'Vendor': '8086', 'Device': '0b00', - 'SVendor': None, 'SDevice': None} -intel_idxd_spr = {'Class': '08', 'Vendor': '8086', 'Device': '0b25', - 'SVendor': None, 'SDevice': None} -intel_ntb_skx = {'Class': '06', 'Vendor': '8086', 'Device': '201c', - 'SVendor': None, 'SDevice': None} -intel_ntb_icx = {'Class': '06', 'Vendor': '8086', 'Device': '347e', - 'SVendor': None, 'SDevice': None} - -cnxk_sso = {'Class': '08', 'Vendor': '177d', 'Device': 'a0f9,a0fa', - 'SVendor': None, 'SDevice': None} -cnxk_npa = {'Class': '08', 'Vendor': '177d', 'Device': 'a0fb,a0fc', - 'SVendor': None, 'SDevice': None} -cn9k_ree = {'Class': '08', 'Vendor': '177d', 'Device': 'a0f4', - 'SVendor': None, 'SDevice': None} - -virtio_blk = {'Class': '01', 'Vendor': "1af4", 'Device': '1001,1042', - 'SVendor': None, 'SDevice': None} - -cnxk_ml = {'Class': '08', 'Vendor': '177d', 'Device': 'a092', - 'SVendor': None, 'SDevice': None} - -network_devices = [network_class, cavium_pkx, avp_vnic, ifpga_class] -baseband_devices = [acceleration_class] -crypto_devices = [encryption_class, intel_processor_class] -dma_devices = [cnxk_dma, hisilicon_dma, - intel_idxd_spr, intel_ioat_bdw, intel_ioat_icx, intel_ioat_skx, - odm_dma] -eventdev_devices = [cavium_sso, cavium_tim, intel_dlb, cnxk_sso] -mempool_devices = [cavium_fpa, cnxk_npa] -compress_devices = [cavium_zip] -regex_devices = [cn9k_ree] -ml_devices = [cnxk_ml] -misc_devices = [cnxk_bphy, cnxk_bphy_cgx, cnxk_inl_dev, - intel_ntb_skx, intel_ntb_icx, - virtio_blk] - -# global dict ethernet devices present. Dictionary indexed by PCI address. -# Each device within this is itself a dictionary of device properties -devices = {} -# list of supported DPDK drivers -dpdk_drivers = ["igb_uio", "vfio-pci", "uio_pci_generic"] -# list of currently loaded kernel modules -loaded_modules = None - -# command-line arg flags -b_flag = None -status_flag = False -force_flag = False -noiommu_flag = False -args = [] - - -# check if a specific kernel module is loaded -def module_is_loaded(module): - global loaded_modules - - if module == 'vfio_pci': - module = 'vfio-pci' - - if loaded_modules: - return module in loaded_modules - - # Get list of sysfs modules (both built-in and dynamically loaded) - sysfs_path = '/sys/module/' - - # Get the list of directories in sysfs_path - sysfs_mods = [m for m in os.listdir(sysfs_path) - if os.path.isdir(os.path.join(sysfs_path, m))] - - # special case for vfio_pci (module is named vfio-pci, - # but its .ko is named vfio_pci) - sysfs_mods = [a if a != 'vfio_pci' else 'vfio-pci' for a in sysfs_mods] - - loaded_modules = sysfs_mods - - # add built-in modules as loaded - release = platform.uname().release - filename = os.path.join("/lib/modules/", release, "modules.builtin") - if os.path.exists(filename): - try: - with open(filename) as f: - loaded_modules += [os.path.splitext(os.path.basename(mod))[0] for mod in f] - except IOError: - print("Warning: cannot read list of built-in kernel modules") - - return module in loaded_modules - - -def check_modules(): - '''Checks that igb_uio is loaded''' - global dpdk_drivers - - # list of supported modules - mods = [{"Name": driver, "Found": False} for driver in dpdk_drivers] - - # first check if module is loaded - for mod in mods: - if module_is_loaded(mod["Name"]): - mod["Found"] = True - - # check if we have at least one loaded module - if True not in [mod["Found"] for mod in mods] and b_flag is not None: - print("Warning: no supported DPDK kernel modules are loaded", file=sys.stderr) - - # change DPDK driver list to only contain drivers that are loaded - dpdk_drivers = [mod["Name"] for mod in mods if mod["Found"]] - - -def has_driver(dev_id): - '''return true if a device is assigned to a driver. False otherwise''' - return "Driver_str" in devices[dev_id] - - -def get_pci_device_details(dev_id, probe_lspci): - '''This function gets additional details for a PCI device''' - device = {} - - if probe_lspci: - extra_info = subprocess.check_output(["lspci", "-vmmks", dev_id]).splitlines() - # parse lspci details - for line in extra_info: - if not line: - continue - name, value = line.decode("utf8").split("\t", 1) - name = name.strip(":") + "_str" - device[name] = value - # check for a unix interface name - device["Interface"] = "" - for base, dirs, _ in os.walk("/sys/bus/pci/devices/%s/" % dev_id): - if "net" in dirs: - device["Interface"] = \ - ",".join(os.listdir(os.path.join(base, "net"))) - break - # check if a port is used for ssh connection - device["Ssh_if"] = False - device["Active"] = "" - - return device - - -def clear_data(): - '''This function clears any old data''' - global devices - devices = {} - - -def get_device_details(devices_type): - '''This function populates the "devices" dictionary. The keys used are - the pci addresses (domain:bus:slot.func). The values are themselves - dictionaries - one for each NIC.''' - global devices - global dpdk_drivers - - # first loop through and read details for all devices - # request machine readable format, with numeric IDs and String - dev = {} - dev_lines = subprocess.check_output(["lspci", "-Dvmmnnk"]).splitlines() - for dev_line in dev_lines: - if not dev_line: - if device_type_match(dev, devices_type): - # Replace "Driver" with "Driver_str" to have consistency of - # of dictionary key names - if "Driver" in dev.keys(): - dev["Driver_str"] = dev.pop("Driver") - if "Module" in dev.keys(): - dev["Module_str"] = dev.pop("Module") - # use dict to make copy of dev - devices[dev["Slot"]] = dict(dev) - # Clear previous device's data - dev = {} - else: - name, value = dev_line.decode("utf8").split("\t", 1) - value_list = value.rsplit(' ', 1) - if value_list: - # String stored in _str - dev[name.rstrip(":") + '_str'] = value_list[0] - # Numeric IDs - dev[name.rstrip(":")] = value_list[len(value_list) - 1] \ - .rstrip("]").lstrip("[") - - if devices_type == network_devices: - # check what is the interface if any for an ssh connection if - # any to this host, so we can mark it later. - ssh_if = [] - route = subprocess.check_output(["ip", "-o", "route"]) - # filter out all lines for 169.254 routes - route = "\n".join(filter(lambda ln: not ln.startswith("169.254"), - route.decode().splitlines())) - rt_info = route.split() - for i in range(len(rt_info) - 1): - if rt_info[i] == "dev": - ssh_if.append(rt_info[i + 1]) - - # based on the basic info, get extended text details - for d in devices.keys(): - if not device_type_match(devices[d], devices_type): - continue - - # get additional info and add it to existing data - devices[d] = devices[d].copy() - # No need to probe lspci - devices[d].update(get_pci_device_details(d, False).items()) - - if devices_type == network_devices: - for _if in ssh_if: - if _if in devices[d]["Interface"].split(","): - devices[d]["Ssh_if"] = True - devices[d]["Active"] = "*Active*" - break - - # add igb_uio to list of supporting modules if needed - if "Module_str" in devices[d]: - for driver in dpdk_drivers: - if driver not in devices[d]["Module_str"]: - devices[d]["Module_str"] = \ - devices[d]["Module_str"] + ",%s" % driver - else: - devices[d]["Module_str"] = ",".join(dpdk_drivers) - - # make sure the driver and module strings do not have any duplicates - if has_driver(d): - modules = devices[d]["Module_str"].split(",") - if devices[d]["Driver_str"] in modules: - modules.remove(devices[d]["Driver_str"]) - devices[d]["Module_str"] = ",".join(modules) - - -def device_type_match(dev, devices_type): - for i in range(len(devices_type)): - param_count = len( - [x for x in devices_type[i].values() if x is not None]) - match_count = 0 - if dev["Class"][0:2] == devices_type[i]["Class"]: - match_count = match_count + 1 - for key in devices_type[i].keys(): - if key != 'Class' and devices_type[i][key]: - value_list = devices_type[i][key].split(',') - for value in value_list: - if value.strip(' ') == dev[key]: - match_count = match_count + 1 - # count must be the number of non None parameters to match - if match_count == param_count: - return True - return False - - -def dev_id_from_dev_name(dev_name): - '''Take a device "name" - a string passed in by user to identify a NIC - device, and determine the device id - i.e. the domain:bus:slot.func - for - it, which can then be used to index into the devices array''' - - # check if it's already a suitable index - if dev_name in devices: - return dev_name - # check if it's an index just missing the domain part - if "0000:" + dev_name in devices: - return "0000:" + dev_name - - # check if it's an interface name, e.g. eth1 - for d in devices.keys(): - if dev_name in devices[d]["Interface"].split(","): - return devices[d]["Slot"] - # if nothing else matches - error - raise ValueError("Unknown device: %s. " - "Please specify device in \"bus:slot.func\" format" % dev_name) - - -def unbind_one(dev_id, force): - '''Unbind the device identified by "dev_id" from its current driver''' - dev = devices[dev_id] - if not has_driver(dev_id): - print("Notice: %s %s %s is not currently managed by any driver" % - (dev["Slot"], dev["Device_str"], dev["Interface"]), file=sys.stderr) - return - - # prevent us disconnecting ourselves - if dev["Ssh_if"] and not force: - print("Warning: routing table indicates that interface %s is active. " - "Skipping unbind" % dev_id, file=sys.stderr) - return - - # write to /sys to unbind - filename = "/sys/bus/pci/drivers/%s/unbind" % dev["Driver_str"] - try: - f = open(filename, "a") - except OSError as err: - sys.exit("Error: unbind failed for %s - Cannot open %s: %s" % - (dev_id, filename, err)) - f.write(dev_id) - f.close() - - -def bind_one(dev_id, driver, force): - '''Bind the device given by "dev_id" to the driver "driver". If the device - is already bound to a different driver, it will be unbound first''' - dev = devices[dev_id] - saved_driver = None # used to rollback any unbind in case of failure - - # prevent disconnection of our ssh session - if dev["Ssh_if"] and not force: - print("Warning: routing table indicates that interface %s is active. " - "Not modifying" % dev_id, file=sys.stderr) - return - - # unbind any existing drivers we don't want - if has_driver(dev_id): - if dev["Driver_str"] == driver: - print("Notice: %s already bound to driver %s, skipping" % - (dev_id, driver), file=sys.stderr) - return - saved_driver = dev["Driver_str"] - unbind_one(dev_id, force) - dev["Driver_str"] = "" # clear driver string - - # For kernels >= 3.15 driver_override can be used to specify the driver - # for a device rather than relying on the driver to provide a positive - # match of the device. The existing process of looking up - # the vendor and device ID, adding them to the driver new_id, - # will erroneously bind other devices too which has the additional burden - # of unbinding those devices - if driver in dpdk_drivers: - filename = "/sys/bus/pci/devices/%s/driver_override" % dev_id - if exists(filename): - try: - f = open(filename, "w") - except OSError as err: - print("Error: bind failed for %s - Cannot open %s: %s" - % (dev_id, filename, err), file=sys.stderr) - return - try: - f.write("%s" % driver) - f.close() - except OSError as err: - print("Error: bind failed for %s - Cannot write driver %s to " - "PCI ID: %s" % (dev_id, driver, err), file=sys.stderr) - return - # For kernels < 3.15 use new_id to add PCI id's to the driver - else: - filename = "/sys/bus/pci/drivers/%s/new_id" % driver - try: - f = open(filename, "w") - except OSError as err: - print("Error: bind failed for %s - Cannot open %s: %s" - % (dev_id, filename, err), file=sys.stderr) - return - try: - # Convert Device and Vendor Id to int to write to new_id - f.write("%04x %04x" % (int(dev["Vendor"], 16), - int(dev["Device"], 16))) - f.close() - except OSError as err: - print("Error: bind failed for %s - Cannot write new PCI ID to " - "driver %s: %s" % (dev_id, driver, err), file=sys.stderr) - return - - # do the bind by writing to /sys - filename = "/sys/bus/pci/drivers/%s/bind" % driver - try: - f = open(filename, "a") - except OSError as err: - print("Error: bind failed for %s - Cannot open %s: %s" - % (dev_id, filename, err), file=sys.stderr) - if saved_driver is not None: # restore any previous driver - bind_one(dev_id, saved_driver, force) - return - try: - f.write(dev_id) - f.close() - except OSError as err: - # for some reason, closing dev_id after adding a new PCI ID to new_id - # results in IOError. however, if the device was successfully bound, - # we don't care for any errors and can safely ignore IOError - tmp = get_pci_device_details(dev_id, True) - if "Driver_str" in tmp and tmp["Driver_str"] == driver: - return - print("Error: bind failed for %s - Cannot bind to driver %s: %s" - % (dev_id, driver, err), file=sys.stderr) - if saved_driver is not None: # restore any previous driver - bind_one(dev_id, saved_driver, force) - return - - # For kernels > 3.15 driver_override is used to bind a device to a driver. - # Before unbinding it, overwrite driver_override with empty string so that - # the device can be bound to any other driver - filename = "/sys/bus/pci/devices/%s/driver_override" % dev_id - if exists(filename): - try: - f = open(filename, "w") - except OSError as err: - sys.exit("Error: unbind failed for %s - Cannot open %s: %s" - % (dev_id, filename, err)) - try: - f.write("\00") - f.close() - except OSError as err: - sys.exit("Error: unbind failed for %s - Cannot write %s: %s" - % (dev_id, filename, err)) - - -def unbind_all(dev_list, force=False): - """Unbind method, takes a list of device locations""" - - if dev_list[0] == "dpdk": - for d in devices.keys(): - if "Driver_str" in devices[d]: - if devices[d]["Driver_str"] in dpdk_drivers: - unbind_one(devices[d]["Slot"], force) - return - - try: - dev_list = map(dev_id_from_dev_name, dev_list) - except ValueError as ex: - print(ex) - sys.exit(1) - - for d in dev_list: - unbind_one(d, force) - - -def has_iommu(): - """Check if IOMMU is enabled on system""" - return len(os.listdir("/sys/class/iommu")) > 0 - - -def check_noiommu_mode(): - """Check and enable the noiommu mode for VFIO drivers""" - global noiommu_flag - filename = "/sys/module/vfio/parameters/enable_unsafe_noiommu_mode" - - try: - with open(filename, "r") as f: - value = f.read(1) - if value in ("1", "y" ,"Y"): - return - except OSError as err: - sys.exit(f"Error: failed to check unsafe noiommu mode - Cannot open {filename}: {err}") - - if not noiommu_flag: - sys.exit("Error: IOMMU support is disabled, use --noiommu-mode for binding in noiommu mode") - - try: - with open(filename, "w") as f: - f.write("1") - except OSError as err: - sys.exit(f"Error: failed to enable unsafe noiommu mode - Cannot open {filename}: {err}") - print("Warning: enabling unsafe no IOMMU mode for VFIO drivers") - - -def bind_all(dev_list, driver, force=False): - """Bind method, takes a list of device locations""" - global devices - - # a common user error is to forget to specify the driver the devices need to - # be bound to. check if the driver is a valid device, and if it is, show - # a meaningful error. - try: - dev_id_from_dev_name(driver) - # if we've made it this far, this means that the "driver" was a valid - # device string, so it's probably not a valid driver name. - sys.exit("Error: Driver '%s' does not look like a valid driver. " - "Did you forget to specify the driver to bind devices to?" % driver) - except ValueError: - # driver generated error - it's not a valid device ID, so all is well - pass - - # check if we're attempting to bind to a driver that isn't loaded - if not module_is_loaded(driver.replace('-', '_')): - sys.exit("Error: Driver '%s' is not loaded." % driver) - - try: - dev_list = map(dev_id_from_dev_name, dev_list) - except ValueError as ex: - sys.exit(ex) - - # check for IOMMU support - if driver == "vfio-pci" and not has_iommu(): - check_noiommu_mode() - - for d in dev_list: - bind_one(d, driver, force) - - # For kernels < 3.15 when binding devices to a generic driver - # (i.e. one that doesn't have a PCI ID table) using new_id, some devices - # that are not bound to any other driver could be bound even if no one has - # asked them to. hence, we check the list of drivers again, and see if - # some of the previously-unbound devices were erroneously bound. - if not exists("/sys/bus/pci/devices/%s/driver_override" % d): - for d in devices.keys(): - # skip devices that were already bound or that we know should be bound - if "Driver_str" in devices[d] or d in dev_list: - continue - - # update information about this device - devices[d] = dict(devices[d].items() - + get_pci_device_details(d, True).items()) - - # check if updated information indicates that the device was bound - if "Driver_str" in devices[d]: - unbind_one(d, force) - - -def display_devices(title, dev_list, extra_params=None): - '''Displays to the user the details of a list of devices given in - "dev_list". The "extra_params" parameter, if given, should contain a string - with %()s fields in it for replacement by the named fields in each - device's dictionary.''' - strings = [] # this holds the strings to print. We sort before printing - print("\n%s" % title) - print("=" * len(title)) - if not dev_list: - strings.append("") - else: - for dev in dev_list: - if extra_params is not None: - strings.append("%s '%s %s' %s" % (dev["Slot"], - dev["Device_str"], - dev["Device"], - extra_params % dev)) - else: - strings.append("%s '%s'" % (dev["Slot"], dev["Device_str"])) - # sort before printing, so that the entries appear in PCI order - strings.sort() - print("\n".join(strings)) # print one per line - - -def show_device_status(devices_type, device_name, if_field=False): - global dpdk_drivers - kernel_drv = [] - dpdk_drv = [] - no_drv = [] - - print_numa = True # by default, assume we can print NUMA information - - # split our list of network devices into the three categories above - for d in devices.keys(): - if device_type_match(devices[d], devices_type): - print_numa &= "NUMANode" in devices[d] - if not has_driver(d): - no_drv.append(devices[d]) - continue - if devices[d]["Driver_str"] in dpdk_drivers: - dpdk_drv.append(devices[d]) - else: - kernel_drv.append(devices[d]) - - n_devs = len(dpdk_drv) + len(kernel_drv) + len(no_drv) - - # don't bother displaying anything if there are no devices - if n_devs == 0: - msg = "No '%s' devices detected" % device_name - print("") - print(msg) - print("".join('=' * len(msg))) - return - - # print each category separately, so we can clearly see what's used by DPDK - if dpdk_drv: - extra_param = "drv=%(Driver_str)s unused=%(Module_str)s" - if print_numa: - extra_param = "numa_node=%(NUMANode)s " + extra_param - display_devices("%s devices using DPDK-compatible driver" % device_name, - dpdk_drv, extra_param) - if kernel_drv: - extra_param = "drv=%(Driver_str)s unused=%(Module_str)s" - if if_field: - extra_param = "if=%(Interface)s " + extra_param - if print_numa: - extra_param = "numa_node=%(NUMANode)s " + extra_param - extra_param += " %(Active)s" - display_devices("%s devices using kernel driver" % device_name, - kernel_drv, extra_param) - if no_drv: - extra_param = "unused=%(Module_str)s" - if print_numa: - extra_param = "numa_node=%(NUMANode)s " + extra_param - display_devices("Other %s devices" % device_name, no_drv, extra_param) - - -def show_status(): - '''Function called when the script is passed the "--status" option. - Displays to the user what devices are bound to the igb_uio driver, the - kernel driver or to no driver''' - - if status_dev in ["net", "all"]: - show_device_status(network_devices, "Network", if_field=True) - - if status_dev in ["baseband", "all"]: - show_device_status(baseband_devices, "Baseband") - - if status_dev in ["crypto", "all"]: - show_device_status(crypto_devices, "Crypto") - - if status_dev in ["dma", "all"]: - show_device_status(dma_devices, "DMA") - - if status_dev in ["event", "all"]: - show_device_status(eventdev_devices, "Eventdev") - - if status_dev in ["mempool", "all"]: - show_device_status(mempool_devices, "Mempool") - - if status_dev in ["compress", "all"]: - show_device_status(compress_devices, "Compress") - - if status_dev in ["misc", "all"]: - show_device_status(misc_devices, "Misc (rawdev)") - - if status_dev in ["regex", "all"]: - show_device_status(regex_devices, "Regex") - - if status_dev in ["ml", "all"]: - show_device_status(ml_devices, "ML") - - -def pci_glob(arg): - '''Returns a list containing either: - * List of PCI B:D:F matching arg, using shell wildcards e.g. 80:04.* - * Only the passed arg if matching list is empty''' - sysfs_path = "/sys/bus/pci/devices" - for _glob in [arg, '0000:' + arg]: - paths = [basename(path) for path in glob(path_join(sysfs_path, _glob))] - if paths: - return paths - return [arg] - - -def parse_args(): - '''Parses the command-line arguments given by the user and takes the - appropriate action for each''' - global b_flag - global status_flag - global status_dev - global force_flag - global noiommu_flag - global args - - parser = argparse.ArgumentParser( - description='Utility to bind and unbind devices from Linux kernel', - formatter_class=argparse.RawDescriptionHelpFormatter, - epilog=""" -Examples: ---------- - -To display current device status: - %(prog)s --status - -To display current network device status: - %(prog)s --status-dev net - -To bind eth1 from the current driver and move to use vfio-pci - %(prog)s --bind=vfio-pci eth1 - -To unbind 0000:01:00.0 from using any driver - %(prog)s -u 0000:01:00.0 - -To bind 0000:02:00.0 and 0000:02:00.1 to the ixgbe kernel driver - %(prog)s -b ixgbe 02:00.0 02:00.1 -""") - - parser.add_argument( - '-s', - '--status', - action='store_true', - help="Print the current status of all known devices.") - parser.add_argument( - '--status-dev', - help="Print the status of given device group.", - choices=['baseband', 'compress', 'crypto', 'dma', 'event', - 'mempool', 'misc', 'net', 'regex', 'ml']) - bind_group = parser.add_mutually_exclusive_group() - bind_group.add_argument( - '-b', - '--bind', - metavar='DRIVER', - help="Select the driver to use or \"none\" to unbind the device") - bind_group.add_argument( - '-u', - '--unbind', - action='store_true', - help="Unbind a device (equivalent to \"-b none\")") - parser.add_argument( - '--noiommu-mode', - action='store_true', - help="If IOMMU is not available, enable no IOMMU mode for VFIO drivers") - parser.add_argument( - '--force', - action='store_true', - help=""" -Override restriction on binding devices in use by Linux" -WARNING: This can lead to loss of network connection and should be used with caution. -""") - parser.add_argument( - 'devices', - metavar='DEVICE', - nargs='*', - help=""" -Device specified as PCI "domain:bus:slot.func" syntax or "bus:slot.func" syntax. -For devices bound to Linux kernel drivers, they may be referred to by interface name. -""") - - opt = parser.parse_args() - - if opt.status_dev: - status_flag = True - status_dev = opt.status_dev - if opt.status: - status_flag = True - status_dev = "all" - if opt.force: - force_flag = True - if opt.noiommu_mode: - noiommu_flag = True - if opt.bind: - b_flag = opt.bind - elif opt.unbind: - b_flag = "none" - args = opt.devices - - if not b_flag and not status_flag: - print("Error: No action specified for devices. " - "Please give a --bind, --ubind or --status option", - file=sys.stderr) - parser.print_usage() - sys.exit(1) - - if b_flag and not args: - print("Error: No devices specified.", file=sys.stderr) - parser.print_usage() - sys.exit(1) - - # resolve any PCI globs in the args - new_args = [] - for arg in args: - new_args.extend(pci_glob(arg)) - args = new_args - - -def do_arg_actions(): - '''do the actual action requested by the user''' - global b_flag - global status_flag - global force_flag - global args - - if b_flag in ["none", "None"]: - unbind_all(args, force_flag) - elif b_flag is not None: - bind_all(args, b_flag, force_flag) - if status_flag: - if b_flag is not None: - clear_data() - # refresh if we have changed anything - get_device_details(network_devices) - get_device_details(baseband_devices) - get_device_details(crypto_devices) - get_device_details(dma_devices) - get_device_details(eventdev_devices) - get_device_details(mempool_devices) - get_device_details(compress_devices) - get_device_details(regex_devices) - get_device_details(ml_devices) - get_device_details(misc_devices) - show_status() - - -def main(): - '''program main function''' - # check if lspci is installed, suppress any output - with open(os.devnull, 'w') as devnull: - ret = subprocess.call(['which', 'lspci'], - stdout=devnull, stderr=devnull) - if ret != 0: - sys.exit("'lspci' not found - please install 'pciutils'") - parse_args() - check_modules() - clear_data() - get_device_details(network_devices) - get_device_details(baseband_devices) - get_device_details(crypto_devices) - get_device_details(dma_devices) - get_device_details(eventdev_devices) - get_device_details(mempool_devices) - get_device_details(compress_devices) - get_device_details(regex_devices) - get_device_details(ml_devices) - get_device_details(misc_devices) - do_arg_actions() - - -if __name__ == "__main__": - main() diff --git a/ci/test/test.sh b/ci/test/test.sh index 73724a4784..e0c272446e 100755 --- a/ci/test/test.sh +++ b/ci/test/test.sh @@ -39,6 +39,8 @@ function sync_files() { rsync -e "$TARGET_SSH_CMD" -av $BUILD_DIR/* $TARGET_BOARD:$REMOTE_BUILD_DIR/ # Sync deps build directory if required rsync -e "$TARGET_SSH_CMD" -r $DEPS_DIR/* $TARGET_BOARD:$REMOTE_DIR/deps_build + # Sync dpdk-devbind.py + $TARGET_SSH_CMD $TARGET_BOARD "sudo $TARGET_SCP_CMD $DPDK_DEVBIND_LOCATION ${REMOTE_BUILD_DIR}/ci/test/board/" } function run_tests() { @@ -49,13 +51,16 @@ function run_tests() { PROJECT_ROOT=${PROJECT_ROOT:-$PWD} TARGET_BOARD=${TARGET_BOARD:-root@127.0.0.1} TARGET_SSH_CMD=${TARGET_SSH_CMD:-"ssh"} +TARGET_SCP_CMD=${TARGET_SCP_CMD:-"scp"} REMOTE="$TARGET_SSH_CMD $TARGET_BOARD -n" REMOTE_DIR=${REMOTE_DIR:-/tmp/vpp} BUILD_DIR=${BUILD_DIR:-$PWD/build} DEPS_DIR=${DEPS_DIR:-${PROJECT_ROOT}/deps-prefix} REMOTE_BUILD_DIR=${REMOTE_DIR}/build +PLAT=${PLAT:-cn10k} +DPDK_DEVBIND_LOCATION=${DPDK_DEVBIND_LOCATION:-ci@10.28.36.188:/home/ci/vpp/perf_stage_bins/$PLAT/dpdk-devbind.py} -TEST_DIR=${REMOTE_BUILD_DIR}/src/plugins/onp/test/ +TEST_DIR=${REMOTE_BUILD_DIR}/src/plugins/dev_octeon/test/ install_packages sync_files diff --git a/src/plugins/dev_octeon/test/configs/pcie.ini b/src/plugins/dev_octeon/test/configs/pcie.ini new file mode 100644 index 0000000000..ab109a1720 --- /dev/null +++ b/src/plugins/dev_octeon/test/configs/pcie.ini @@ -0,0 +1 @@ +[default] diff --git a/src/plugins/dev_octeon/test/configs/unittest.ini b/src/plugins/dev_octeon/test/configs/unittest.ini new file mode 100644 index 0000000000..0ce577547a --- /dev/null +++ b/src/plugins/dev_octeon/test/configs/unittest.ini @@ -0,0 +1,45 @@ +[default] +lbk1_ip: 192.168.1.1 +lbk2_ip: 192.168.1.2 +lbk3_ip: 172.66.4.3 +lbk4_ip: 12.0.0.2 +lbk1_intf_name: eth0 +lbk4_intf_name: eth1 +dut_instance: dev_octeon +igw_instance: native_crypto +aes_gcm_128_crypto_key: 4339314b55523947594d6d3547666b45 + +[ipsec] +DEF_IPSEC_TNL_LOCAL_IP4: 192.168.1.1 +DEF_IPSEC_TNL_REMOTE_IP4: 192.168.1.2 +DEF_IPSEC_OUTBOUND_AES_GCM_128_CRYPTO_KEY: C91KUR9GYMm5GfkE +DEF_IPSEC_INBOUND_AES_GCM_128_CRYPTO_KEY: C91KUR9GYMm5GfkE +DEF_IPSEC_OUTBOUND_AES_GCM_192_CRYPTO_KEY: abcdefghijklmnopabcdefgh +DEF_IPSEC_INBOUND_AES_GCM_192_CRYPTO_KEY: abcdefghijklmnopabcdefgh +DEF_IPSEC_OUTBOUND_AES_GCM_256_CRYPTO_KEY: abcdefghijklmnopabcdabcdefghijkl +DEF_IPSEC_INBOUND_AES_GCM_256_CRYPTO_KEY: abcdefghijklmnopabcdabcdefghijkl + +DEF_IPSEC_OUTBOUND_AES_CTR_128_CRYPTO_KEY: abcdefghijklmnop +DEF_IPSEC_INBOUND_AES_CTR_128_CRYPTO_KEY: abcdefghijklmnop +DEF_IPSEC_OUTBOUND_AES_CTR_192_CRYPTO_KEY: abcdefghijklmnopabcdefgh +DEF_IPSEC_INBOUND_AES_CTR_192_CRYPTO_KEY: abcdefghijklmnopabcdefgh +DEF_IPSEC_OUTBOUND_AES_CTR_256_CRYPTO_KEY: abcdefghijklmnopabcdabcdefghijkl +DEF_IPSEC_INBOUND_AES_CTR_256_CRYPTO_KEY: abcdefghijklmnopabcdabcdefghijkl + +DEF_IPSEC_OUTBOUND_AES_CBC_128_CRYPTO_KEY: abcdefghijklmnop +DEF_IPSEC_INBOUND_AES_CBC_128_CRYPTO_KEY: abcdefghijklmnop +DEF_IPSEC_OUTBOUND_AES_CBC_192_CRYPTO_KEY: abcdefghijklmnopabcdefgh +DEF_IPSEC_INBOUND_AES_CBC_192_CRYPTO_KEY: abcdefghijklmnopabcdefgh +DEF_IPSEC_OUTBOUND_AES_CBC_256_CRYPTO_KEY: abcdefghijklmnopabcdefghijklmnop +DEF_IPSEC_INBOUND_AES_CBC_256_CRYPTO_KEY: abcdefghijklmnopabcdefghijklmnop + +DEF_IPSEC_OUTBOUND_SHA1_96_INTEG_KEY: abcdefghijklmnopabcd +DEF_IPSEC_INBOUND_SHA1_96_INTEG_KEY: abcdefghijklmnopabcd +DEF_IPSEC_OUTBOUND_SHA_256_96_INTEG_KEY: abcdefghijklmnopabcdabcdefghijkl +DEF_IPSEC_INBOUND_SHA_256_96_INTEG_KEY: abcdefghijklmnopabcdabcdefghijkl +DEF_IPSEC_OUTBOUND_SHA_256_128_INTEG_KEY: abcdefghijklmnopabcdabcdefghijkl +DEF_IPSEC_INBOUND_SHA_256_128_INTEG_KEY: abcdefghijklmnopabcdabcdefghijkl +DEF_IPSEC_OUTBOUND_SHA_384_192_INTEG_KEY: abcdefghijklmnopabcdabcdefghijklabcdefghijklmnop +DEF_IPSEC_INBOUND_SHA_384_192_INTEG_KEY: abcdefghijklmnopabcdabcdefghijklabcdefghijklmnop +DEF_IPSEC_OUTBOUND_SHA_512_256_INTEG_KEY: abcdefghijklmnopabcdabcdefghijklabcdefghijklmnopabcdabcdefghijkl +DEF_IPSEC_INBOUND_SHA_512_256_INTEG_KEY: abcdefghijklmnopabcdabcdefghijklabcdefghijklmnopabcdabcdefghijkl From 155f8f494da3059c57b7e020dedb125320222283 Mon Sep 17 00:00:00 2001 From: Monendra Singh Kushwaha Date: Mon, 7 Apr 2025 09:38:27 +0000 Subject: [PATCH 217/271] octeon: fix number of packets to refill Type: fix JIRA: https://essjira.marvell.com/browse/IPBUSW-63115 Signed-off-by: Monendra Singh Kushwaha Change-Id: I65e354c170307b026ef65d7ef52261a9ce002c75 Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/150157 Reviewed-by: Nithin Kumar Dabilpuram Tested-by: sa_ip-toolkits-Jenkins (cherry picked from commit 09f9c602ba6f2f43b741dc43f4fdb5895e082d5c) Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/150160 --- src/plugins/dev_octeon/rx_node.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/plugins/dev_octeon/rx_node.c b/src/plugins/dev_octeon/rx_node.c index 702f253762..8e7ef8bebc 100644 --- a/src/plugins/dev_octeon/rx_node.c +++ b/src/plugins/dev_octeon/rx_node.c @@ -958,7 +958,6 @@ oct_rx_inl_ipsec_vlib_from_cq ( b[0]->template = *bt; b[0]->flow_id = d[0].parse.w[3] >> 48; *err_flags |= ((d[0].parse.w[0] >> 20) & 0xFFF); - ctx->n_segs += 1; is_fail = !oct_ipsec_is_inl_op_success (cpt_hdr); From edb6702cd615e008f3d929cf7eaa154a5703e878 Mon Sep 17 00:00:00 2001 From: Monendra Singh Kushwaha Date: Wed, 9 Apr 2025 16:53:36 +0530 Subject: [PATCH 218/271] octeon: use send queue derived from sa_index This patch stores send queue in session derived from sa_index and uses it. Type: fix JIRA: https://essjira.marvell.com/browse/IPBUSW-63115 Signed-off-by: Monendra Singh Kushwaha Change-Id: I17cfe4ca2a10824d861e71a1cb79ccbe748114d7 Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/150376 Tested-by: sa_ip-sw-jenkins Reviewed-by: Devapraba Muthumani --- src/plugins/dev_octeon/ipsec.h | 3 +-- src/plugins/dev_octeon/tx_node.c | 21 ++++++++++++++++----- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/src/plugins/dev_octeon/ipsec.h b/src/plugins/dev_octeon/ipsec.h index 65fb36e67c..dbe3c02b70 100644 --- a/src/plugins/dev_octeon/ipsec.h +++ b/src/plugins/dev_octeon/ipsec.h @@ -121,8 +121,7 @@ typedef struct struct roc_ot_ipsec_outb_sa **out_sa; CLIB_CACHE_LINE_ALIGN_MARK (cacheline1); struct cpt_inst_s inst; - uint16_t iv_offset; - uint8_t iv_length; + u16 sq; u32 itf_sw_idx; /* Packet length for IPsec encapsulation */ oct_ipsec_encap_len_t encap; diff --git a/src/plugins/dev_octeon/tx_node.c b/src/plugins/dev_octeon/tx_node.c index 9fe0bed703..cfca955d2b 100644 --- a/src/plugins/dev_octeon/tx_node.c +++ b/src/plugins/dev_octeon/tx_node.c @@ -1262,6 +1262,7 @@ oct_pkts_send_ipsec (vlib_main_t *vm, vlib_node_runtime_t *node, { oct_ipsec_main_t *im = &oct_ipsec_main; oct_txq_t *ctq = vnet_dev_get_tx_queue_data (txq); + u16 num_tx_queues = txq->port->intf.num_tx_queues; u64 aura_handle = ctq->aura_handle; vnet_dev_t *dev = txq->port->dev; oct_device_t *cd = vnet_dev_get_data (dev); @@ -1344,6 +1345,8 @@ oct_pkts_send_ipsec (vlib_main_t *vm, vlib_node_runtime_t *node, { od = oct_ipsec_get_oct_device_from_outb_sa (sa0_index); sess0->inst.w7.s.cptr = (u64) sess0->out_sa[od->nix_idx]; + sess0->sq = + ((sa0_index % vlib_num_workers ()) + 1) % num_tx_queues; } current_sa0_index = sa0_index; ALWAYS_ASSERT (current_sa0_index < @@ -1358,6 +1361,8 @@ oct_pkts_send_ipsec (vlib_main_t *vm, vlib_node_runtime_t *node, { od = oct_ipsec_get_oct_device_from_outb_sa (sa1_index); sess1->inst.w7.s.cptr = (u64) sess1->out_sa[od->nix_idx]; + sess1->sq = + ((sa1_index % vlib_num_workers ()) + 1) % num_tx_queues; } current_sa1_index = sa1_index; ALWAYS_ASSERT (current_sa0_index < @@ -1372,6 +1377,8 @@ oct_pkts_send_ipsec (vlib_main_t *vm, vlib_node_runtime_t *node, { od = oct_ipsec_get_oct_device_from_outb_sa (sa2_index); sess2->inst.w7.s.cptr = (u64) sess2->out_sa[od->nix_idx]; + sess2->sq = + ((sa2_index % vlib_num_workers ()) + 1) % num_tx_queues; } current_sa2_index = sa2_index; ALWAYS_ASSERT (current_sa2_index < @@ -1386,6 +1393,8 @@ oct_pkts_send_ipsec (vlib_main_t *vm, vlib_node_runtime_t *node, { od = oct_ipsec_get_oct_device_from_outb_sa (sa3_index); sess3->inst.w7.s.cptr = (u64) sess3->out_sa[od->nix_idx]; + sess3->sq = + ((sa3_index % vlib_num_workers ()) + 1) % num_tx_queues; } current_sa3_index = sa3_index; ALWAYS_ASSERT (current_sa3_index < @@ -1422,10 +1431,10 @@ oct_pkts_send_ipsec (vlib_main_t *vm, vlib_node_runtime_t *node, inst2.w7.u64 = sess2->inst.w7.u64; inst3.w7.u64 = sess3->inst.w7.u64; - sq0 = sa0_index; - sq1 = sa1_index; - sq2 = sa2_index; - sq3 = sa3_index; + sq0 = sess0->sq; + sq1 = sess1->sq; + sq2 = sess2->sq; + sq3 = sess3->sq; quad_bit = 0; count = 0; @@ -1588,6 +1597,8 @@ oct_pkts_send_ipsec (vlib_main_t *vm, vlib_node_runtime_t *node, { od = oct_ipsec_get_oct_device_from_outb_sa (sa0_index); sess0->inst.w7.s.cptr = (u64) sess0->out_sa[od->nix_idx]; + sess0->sq = + ((sa0_index % vlib_num_workers ()) + 1) % num_tx_queues; } current_sa0_index = sa0_index; ALWAYS_ASSERT (current_sa0_index < @@ -1601,7 +1612,7 @@ oct_pkts_send_ipsec (vlib_main_t *vm, vlib_node_runtime_t *node, inst0.w3.u64 |= 0x1ULL; inst0.w7.u64 = sess0->inst.w7.u64; - sq0 = sa0_index; + sq0 = sess0->sq; if (current_sq0 != sq0) { From f5ef1783fefd30aac788418e3ad013bd927152ad Mon Sep 17 00:00:00 2001 From: Satha Rao Date: Mon, 10 Mar 2025 22:50:00 +0530 Subject: [PATCH 219/271] octeon: flush CQ buffers on stop This patch ensures that any pending buffers received are flushed while closing the Completion Queue (CQ). Also fixes following corner cases Disable CPT RQ when port stopped. fix: correct memory offset when buffer frees in txq stop Type: feature Change-Id: I0bddc3d697827c1c5f09591aebef42a9f1c50d7f Signed-off-by: Satha Rao Signed-off-by: Monendra Singh Kushwaha Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/139747 Reviewed-by: Nithin Kumar Dabilpuram Tested-by: sa_ip-toolkits-Jenkins (cherry picked from commit 8bbda59fc9dbed9cb6a8ff001ef21818708f53bf) Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/150513 Tested-by: sa_ip-sw-jenkins --- src/plugins/dev_octeon/octeon.h | 1 + src/plugins/dev_octeon/port.c | 20 +++++++- src/plugins/dev_octeon/queue.c | 84 +++++++++++++++++++++++++++++++++ 3 files changed, 103 insertions(+), 2 deletions(-) diff --git a/src/plugins/dev_octeon/octeon.h b/src/plugins/dev_octeon/octeon.h index 2815fe4219..7b94996609 100644 --- a/src/plugins/dev_octeon/octeon.h +++ b/src/plugins/dev_octeon/octeon.h @@ -200,6 +200,7 @@ void oct_tx_queue_free (vlib_main_t *, vnet_dev_tx_queue_t *); vnet_dev_rv_t oct_rxq_init (vlib_main_t *, vnet_dev_rx_queue_t *, u32); vnet_dev_rv_t oct_txq_init (vlib_main_t *, vnet_dev_tx_queue_t *); void oct_rxq_deinit (vlib_main_t *, vnet_dev_rx_queue_t *); +int oct_drain_queue (vlib_main_t *vm, vnet_dev_rx_queue_t *rxq); void oct_txq_deinit (vlib_main_t *, vnet_dev_tx_queue_t *); format_function_t format_oct_rxq_info; format_function_t format_oct_txq_info; diff --git a/src/plugins/dev_octeon/port.c b/src/plugins/dev_octeon/port.c index c2b04a1281..6494f354a8 100644 --- a/src/plugins/dev_octeon/port.c +++ b/src/plugins/dev_octeon/port.c @@ -475,8 +475,9 @@ oct_rxq_stop (vlib_main_t *vm, vnet_dev_rx_queue_t *rxq) if ((rrv = roc_nix_rq_ena_dis (&crq->rq, 0))) oct_roc_err (dev, rrv, "roc_nix_rq_ena_dis() failed"); - n = - oct_aura_free_all_buffers (vm, crq->aura_handle, crq->hdr_off, crq->n_enq); + n = oct_drain_queue (vm, rxq); + n += oct_aura_free_all_buffers (vm, crq->aura_handle, crq->hdr_off, + crq->n_enq - n); if (crq->n_enq - n > 0) log_err (dev, "%u buffers leaked on rx queue %u stop", crq->n_enq - n, @@ -505,6 +506,10 @@ oct_txq_stop (vlib_main_t *vm, vnet_dev_tx_queue_t *txq) if (st.status.ccode != ALLOC_CCODE_INVAL) for (u32 i = 0; i < st.status.count; i++) { +#if (CLIB_DEBUG > 0) + if (!i || (i == 8)) + cl->iova[i] &= OCT_BATCH_ALLOC_IOVA0_MASK; +#endif vlib_buffer_t *b = (vlib_buffer_t *) (cl->iova[i] + off); vlib_buffer_free_one (vm, vlib_get_buffer_index (vm, b)); ctq->n_enq--; @@ -521,6 +526,7 @@ oct_txq_stop (vlib_main_t *vm, vnet_dev_tx_queue_t *txq) log_debug (dev, "%u buffers freed from tx queue %u", n, txq->queue_id); ctq->n_enq = 0; + ctq->ba_num_cl = ctq->ba_first_cl = 0; } vnet_dev_rv_t @@ -578,6 +584,7 @@ oct_port_start (vlib_main_t *vm, vnet_dev_port_t *port) void oct_port_stop (vlib_main_t *vm, vnet_dev_port_t *port) { + oct_inl_dev_main_t *inl_main = &oct_inl_dev_main; vnet_dev_t *dev = port->dev; oct_device_t *cd = vnet_dev_get_data (dev); oct_port_t *cp = vnet_dev_get_port_data (port); @@ -603,6 +610,15 @@ oct_port_stop (vlib_main_t *vm, vnet_dev_port_t *port) return; } + if (inl_main->inl_dev) + { + if ((rrv = roc_nix_inl_rq_ena_dis (nix, false))) + { + oct_roc_err (dev, rrv, "roc_nix_inl_rq_ena_dis failed"); + return; + } + } + foreach_vnet_dev_port_rx_queue (q, port) oct_rxq_stop (vm, q); diff --git a/src/plugins/dev_octeon/queue.c b/src/plugins/dev_octeon/queue.c index 9323d2b621..f6308e8f5f 100644 --- a/src/plugins/dev_octeon/queue.c +++ b/src/plugins/dev_octeon/queue.c @@ -189,6 +189,89 @@ oct_rxq_init (vlib_main_t *vm, vnet_dev_rx_queue_t *rxq, u32 total_sz) return VNET_DEV_OK; } +static_always_inline vlib_buffer_t * +oct_seg_to_bp (void *p) +{ + return (vlib_buffer_t *) p - 1; +} + +static void +oct_multi_seg_free (vlib_main_t *vm, vnet_dev_rx_queue_t *rxq, + oct_nix_rx_cqe_desc_t *d) +{ + vlib_buffer_t *t; + u8 s0 = d->sg0.segs, s1; + + t = oct_seg_to_bp (d->segs0[1]); + vlib_buffer_free_one (vm, vlib_get_buffer_index (vm, t)); + + if (s0 == 2) + return; + t = oct_seg_to_bp (d->segs0[2]); + vlib_buffer_free_one (vm, vlib_get_buffer_index (vm, t)); + + if (d->sg1.subdc != NIX_SUBDC_SG) + return; + + s1 = d->sg1.segs; + if (s1 == 0) + return; + + t = oct_seg_to_bp (d->segs1[0]); + vlib_buffer_free_one (vm, vlib_get_buffer_index (vm, t)); + + if (s1 == 1) + return; + t = oct_seg_to_bp (d->segs1[1]); + vlib_buffer_free_one (vm, vlib_get_buffer_index (vm, t)); + + if (s1 == 2) + return; + t = oct_seg_to_bp (d->segs1[2]); + vlib_buffer_free_one (vm, vlib_get_buffer_index (vm, t)); +} + +int +oct_drain_queue (vlib_main_t *vm, vnet_dev_rx_queue_t *rxq) +{ + oct_rxq_t *crq = vnet_dev_get_rx_queue_data (rxq); + oct_nix_rx_cqe_desc_t *descs = crq->cq.desc_base; + oct_nix_lf_cq_op_status_t status; + u32 cq_size = crq->cq.nb_desc; + u32 cq_mask = crq->cq.qmask; + vlib_buffer_t *b; + u32 i, head, n_desc, n, f_cnt = 0; + + /* Free all CQ entries */ + while (1) + { + /* get head and tail from NIX_LF_CQ_OP_STATUS */ + status.as_u64 = roc_atomic64_add_sync (crq->cq.wdata, crq->cq.status); + if (status.cq_err || status.op_err) + return f_cnt; + + head = status.head; + n_desc = (status.tail - head) & cq_mask; + + if (n_desc == 0) + return f_cnt; + + n = clib_min (cq_size - head, n_desc); + for (i = head; i < n; i++) + { + b = oct_seg_to_bp (descs[i].segs0[0]); + vlib_buffer_free_one (vm, vlib_get_buffer_index (vm, b)); + if (descs[i].sg0.segs > 1) + oct_multi_seg_free (vm, rxq, &descs[i]); + } + f_cnt += n; + plt_write64 ((crq->cq.wdata | n), crq->cq.door); + plt_wmb (); + } + + return f_cnt; +} + void oct_rxq_deinit (vlib_main_t *vm, vnet_dev_rx_queue_t *rxq) { @@ -206,6 +289,7 @@ oct_rxq_deinit (vlib_main_t *vm, vnet_dev_rx_queue_t *rxq) if (crq->cq_initialized) { + oct_drain_queue (vm, rxq); rrv = roc_nix_cq_fini (&crq->cq); if (rrv) oct_roc_err (dev, rrv, "roc_nix_cq_fini() failed"); From 8389bc09bba8af8b3fd8c399ed4ee5d27ae3661a Mon Sep 17 00:00:00 2001 From: Alok Mishra Date: Thu, 3 Apr 2025 11:52:06 +0500 Subject: [PATCH 220/271] tm: refactor tm.api data types Update data types of parent_node_id and shaper_profile_id from u32 to i32. Type: fix JIRA: https://essjira.marvell.com/browse/IPBUSW-60487 Signed-off-by: Alok Mishra Change-Id: Id69581b2d36ad048f0e8fb1fcbe75cf828615bda Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/149925 Tested-by: sa_ip-toolkits-Jenkins Reviewed-by: Nithin Kumar Dabilpuram (cherry picked from commit c9765aa75dc8422b9fe41cdfc2caaac942be35d2) Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/151960 Tested-by: sa_ip-sw-jenkins Reviewed-by: Monendra Singh Kushwaha --- src/plugins/dev_octeon/tm.c | 11 ++++++----- src/vnet/tm/tm.api | 4 ++-- src/vnet/tm/tm.c | 2 +- src/vnet/tm/tm.h | 6 +++--- src/vnet/tm/tm_api.c | 6 +++--- src/vnet/tm/tm_test.c | 9 +++++---- 6 files changed, 20 insertions(+), 18 deletions(-) diff --git a/src/plugins/dev_octeon/tm.c b/src/plugins/dev_octeon/tm.c index b54662b957..a143985297 100644 --- a/src/plugins/dev_octeon/tm.c +++ b/src/plugins/dev_octeon/tm.c @@ -39,7 +39,7 @@ oct_roc_err (vnet_dev_t *dev, int rv, char *fmt, ...) } int -oct_tm_sys_node_add (u32 hw_if_idx, u32 node_id, u32 parent_node_id, +oct_tm_sys_node_add (u32 hw_if_idx, u32 node_id, i32 parent_node_id, u32 priority, u32 weight, u32 lvl, tm_node_params_t *params) @@ -64,7 +64,7 @@ oct_tm_sys_node_add (u32 hw_if_idx, u32 node_id, u32 parent_node_id, } if (parent_node_id) { - parent_node = roc_nix_tm_node_get (nix, parent_node_id); + parent_node = roc_nix_tm_node_get (nix, (u32) parent_node_id); } /* Find the right level */ @@ -91,14 +91,15 @@ oct_tm_sys_node_add (u32 hw_if_idx, u32 node_id, u32 parent_node_id, } tm_node->id = node_id; - tm_node->parent_id = parent_node_id; + tm_node->parent_id = (u32) parent_node_id; tm_node->lvl = lvl; tm_node->priority = priority; tm_node->free_fn = plt_free; tm_node->weight = weight; - tm_node->shaper_profile_id = params->shaper_profile_id; + tm_node->shaper_profile_id = (u32) params->shaper_profile_id; - profile = roc_nix_tm_shaper_profile_get (nix, params->shaper_profile_id); + profile = + roc_nix_tm_shaper_profile_get (nix, (u32) params->shaper_profile_id); rc = roc_nix_tm_node_add (nix, tm_node); if (rc < 0) diff --git a/src/vnet/tm/tm.api b/src/vnet/tm/tm.api index d792b8da89..d3e8e81c07 100644 --- a/src/vnet/tm/tm.api +++ b/src/vnet/tm/tm.api @@ -42,10 +42,10 @@ define tm_sys_node_add u32 client_index; u32 context; u32 sw_if_idx; - u32 parent_node_id; + i32 parent_node_id; u32 node_id; u32 weight; - u32 shaper_id; + i32 shaper_id; u32 lvl; u32 priority; }; diff --git a/src/vnet/tm/tm.c b/src/vnet/tm/tm.c index 7f020a6f8e..f1039d96d9 100644 --- a/src/vnet/tm/tm.c +++ b/src/vnet/tm/tm.c @@ -24,7 +24,7 @@ tm_system_register (tm_system_t *tm_sys, u32 hw_if_idx) } int -tm_sys_node_add (u32 hw_if_idx, u32 node_id, u32 parent_node_id, u32 priority, +tm_sys_node_add (u32 hw_if_idx, u32 node_id, i32 parent_node_id, u32 priority, u32 weight, u32 lvl, tm_node_params_t *params) { vnet_main_t *vnm = vnet_get_main (); diff --git a/src/vnet/tm/tm.h b/src/vnet/tm/tm.h index b284032d9b..57d6b7ee64 100644 --- a/src/vnet/tm/tm.h +++ b/src/vnet/tm/tm.h @@ -16,7 +16,7 @@ typedef struct tm_node_params_ { /* Shaper profile for the node. */ - u32 shaper_profile_id; + i32 shaper_profile_id; union { @@ -522,7 +522,7 @@ typedef struct tm_stats_params_ typedef struct tm_system_t_ { u32 hw_if_idx; - int (*node_add) (u32 hw_if_idx, u32 node_id, u32 parent_node_id, + int (*node_add) (u32 hw_if_idx, u32 node_id, i32 parent_node_id, u32 priority, u32 weight, u32 lvl, tm_node_params_t *params); @@ -558,7 +558,7 @@ typedef struct tm_system_t_ * * @return 0 on success. */ -int tm_sys_node_add (u32 hw_if_idx, u32 node_id, u32 parent_node_id, +int tm_sys_node_add (u32 hw_if_idx, u32 node_id, i32 parent_node_id, u32 priority, u32 weight, u32 lvl, tm_node_params_t *params); diff --git a/src/vnet/tm/tm_api.c b/src/vnet/tm/tm_api.c index b2da4a19a6..cffba4d47c 100644 --- a/src/vnet/tm/tm_api.c +++ b/src/vnet/tm/tm_api.c @@ -28,7 +28,7 @@ vl_api_tm_sys_node_add_t_handler (vl_api_tm_sys_node_add_t *mp) tm_node_params_t n_p; int rv = -1; u32 node_id = 0; - u32 parent_node_id = 0; + i32 parent_node_id = 0; u32 priority = 0; u32 weight = 0; u32 lvl = 0; @@ -36,8 +36,8 @@ vl_api_tm_sys_node_add_t_handler (vl_api_tm_sys_node_add_t *mp) vnet_get_sup_sw_interface (vnm, clib_net_to_host_u32 (mp->sw_if_idx)); node_id = clib_net_to_host_u32 (mp->node_id); - parent_node_id = clib_net_to_host_u32 (mp->parent_node_id); - n_p.shaper_profile_id = clib_net_to_host_u32 (mp->shaper_id); + parent_node_id = clib_net_to_host_i32 (mp->parent_node_id); + n_p.shaper_profile_id = clib_net_to_host_i32 (mp->shaper_id); weight = clib_net_to_host_u32 (mp->weight); priority = clib_net_to_host_u32 (mp->priority); lvl = clib_net_to_host_u32 (mp->lvl); diff --git a/src/vnet/tm/tm_test.c b/src/vnet/tm/tm_test.c index 457cfb2241..12287631bd 100644 --- a/src/vnet/tm/tm_test.c +++ b/src/vnet/tm/tm_test.c @@ -35,14 +35,15 @@ uword unformat_sw_if_index (unformat_input_t *input, va_list *args); static int api_tm_sys_node_add (vat_main_t *vam) { - u32 level, priority, node_id, weight, parent_node_id; + u32 level, priority, node_id, weight; + i32 parent_node_id = 0; + i32 shaper_id = 0; u8 priority_set = 0, level_set = 0; unformat_input_t *i = vam->input; vl_api_tm_sys_node_add_t *mp; u32 msg_size = sizeof (*mp); u8 sw_if_idx_set = 0; u32 sw_if_idx = 0; - u32 shaper_id = 0; int ret; vam->result_ready = 0; @@ -78,8 +79,8 @@ api_tm_sys_node_add (vat_main_t *vam) mp->sw_if_idx = clib_host_to_net_u32 (sw_if_idx); mp->node_id = clib_host_to_net_u32 (node_id); - mp->parent_node_id = clib_host_to_net_u32 (parent_node_id); - mp->shaper_id = clib_host_to_net_u32 (shaper_id); + mp->parent_node_id = clib_host_to_net_i32 (parent_node_id); + mp->shaper_id = clib_host_to_net_i32 (shaper_id); mp->weight = clib_host_to_net_u32 (weight); mp->priority = clib_host_to_net_u32 (priority); mp->lvl = clib_host_to_net_u32 (level); From e73b1a9d63cb8f76a43a7542d6712587ae40d367 Mon Sep 17 00:00:00 2001 From: Monendra Singh Kushwaha Date: Tue, 1 Apr 2025 12:22:14 +0530 Subject: [PATCH 221/271] ci: add l3fwd test script Type: feature Change-Id: I728413213882f9ea2b15eec4917da0b4abbd418c Signed-off-by: Monendra Singh Kushwaha Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/149659 Reviewed-by: Nithin Kumar Dabilpuram Tested-by: Nithin Kumar Dabilpuram (cherry picked from commit 8895bd12b898a4144375e5628fd88eb91de787ee) Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/151961 Tested-by: sa_ip-sw-jenkins --- ci/test/board/board_test_run.sh | 236 +++++++++++++++++++++++ ci/test/board/oct-target-setup.sh | 232 ++++++++++++++++++++++ ci/test/board/oxk-devbind-basic.sh | 63 ++++++ ci/test/common/pcap/pcap.env | 31 +++ ci/test/common/pcap/pcap_len.c | 35 ++++ ci/test/common/pcap/pcap_mac.c | 58 ++++++ ci/test/common/pcap/pcap_pkt_cnt.c | 35 ++++ ci/test/common/test_list_helper_funcs.sh | 138 +++++++++++++ ci/test/common/testpmd/common.env | 180 +++++++++++++++++ ci/test/common/testpmd/pktgen.env | 88 +++++++++ ci/test/common/vpp/vpp.env | 151 +++++++++++++++ ci/test/env/cn10k.env | 90 +++++++++ ci/test/l3fwd/l3fwd.conf | 52 +++++ ci/test/l3fwd/l3fwd.exec | 10 + ci/test/l3fwd/l3fwd.pcap | Bin 0 -> 728837 bytes ci/test/l3fwd/l3fwd.sh | 110 +++++++++++ ci/test/test.list | 1 + ci/test/test.sh | 90 ++++++--- 18 files changed, 1569 insertions(+), 31 deletions(-) create mode 100755 ci/test/board/board_test_run.sh create mode 100755 ci/test/board/oct-target-setup.sh create mode 100755 ci/test/board/oxk-devbind-basic.sh create mode 100644 ci/test/common/pcap/pcap.env create mode 100644 ci/test/common/pcap/pcap_len.c create mode 100644 ci/test/common/pcap/pcap_mac.c create mode 100644 ci/test/common/pcap/pcap_pkt_cnt.c create mode 100644 ci/test/common/test_list_helper_funcs.sh create mode 100644 ci/test/common/testpmd/common.env create mode 100644 ci/test/common/testpmd/pktgen.env create mode 100644 ci/test/common/vpp/vpp.env create mode 100644 ci/test/env/cn10k.env create mode 100644 ci/test/l3fwd/l3fwd.conf create mode 100644 ci/test/l3fwd/l3fwd.exec create mode 100644 ci/test/l3fwd/l3fwd.pcap create mode 100644 ci/test/l3fwd/l3fwd.sh create mode 100644 ci/test/test.list diff --git a/ci/test/board/board_test_run.sh b/ci/test/board/board_test_run.sh new file mode 100755 index 0000000000..5fa6826bff --- /dev/null +++ b/ci/test/board/board_test_run.sh @@ -0,0 +1,236 @@ +#!/bin/bash + +# Copyright (c) 2025 Marvell. +# SPDX-License-Identifier: Apache-2.0 +# https://spdx.org/licenses/Apache-2.0.html + +set -euox pipefail + +source $TEST_ENV_CONF + +SKIP_SYNC=${SKIP_SYNC:-} +SKIP_TARGET_SETUP=${SKIP_TARGET_SETUP:-} +PROJECT_ROOT=${PROJECT_ROOT:-$PWD} +TARGET_BOARD=${TARGET_BOARD:-root@127.0.0.1} +TARGET_SSH_CMD=${TARGET_SSH_CMD:-"ssh"} +TARGET_SCP_CMD=${TARGET_SCP_CMD:-"scp"} +REMOTE="$TARGET_SSH_CMD $TARGET_BOARD -n" +REMOTE_DIR=${REMOTE_DIR:-/tmp/vpp} +REMOTE_BUILD_DIR=${REMOTE_DIR}/build +BUILD_DIR=${BUILD_DIR:-$PWD/build} +DEPS_DIR=${DEPS_DIR:-${PROJECT_ROOT}/deps-prefix} +REBOOT_ON_FAIL=${REBOOT_ON_FAIL:-} +UTILS_LOCATION=${DPDK_DEVBIND_LOCATION:-ci@10.28.36.188:/home/ci/vpp/perf_stage_bins/$PLAT/} + +function save_log() +{ + local logfile=$1 + local save_name=${2:-} + + if [[ -z $RUN_DIR ]] || [[ ! -d $RUN_DIR ]]; then + return + fi + + if [[ -n $save_name ]]; then + cp $logfile $RUN_DIR/$save_name 2>/dev/null || true + else + cp $logfile $RUN_DIR/ 2>/dev/null || true + fi +} + +function test_init() +{ + $REMOTE 'sudo dmesg -c' 2>&1 > /dev/null + $REMOTE 'uname -a' +} + +# Sync the files +function target_sync() +{ + local sync="rsync -azzh --delete" + if [[ -n $SKIP_SYNC ]]; then + return + fi + echo "Syncing files to target board..." + $REMOTE "rm -rf $REMOTE_DIR" + $REMOTE "mkdir -p $REMOTE_DIR" + # Sync build directory + rsync -e "$TARGET_SSH_CMD" -av $BUILD_DIR/* $TARGET_BOARD:$REMOTE_BUILD_DIR/ + rsync -e "$TARGET_SSH_CMD" -av $BUILD_DIR/../* $TARGET_BOARD:$REMOTE_BUILD_DIR/ + # Sync deps build directory if required + #rsync -e "$TARGET_SSH_CMD" -r $DEPS_DIR/* $TARGET_BOARD:$REMOTE_DIR/deps_build + # Sync dpdk-devbind.py + $TARGET_SSH_CMD $TARGET_BOARD "sudo $TARGET_SCP_CMD $UTILS_LOCATION/dpdk-devbind.py ${REMOTE_BUILD_DIR}/ci/test/board/" + # Sync testpmd + $TARGET_SSH_CMD $TARGET_BOARD "sudo $TARGET_SCP_CMD $UTILS_LOCATION/dpdk-testpmd /usr/local/bin" + # Sync pcap utils + $TARGET_SSH_CMD $TARGET_BOARD "sudo $TARGET_SCP_CMD $UTILS_LOCATION/pcap-* ${REMOTE_BUILD_DIR}/ci/test/common/pcap/" +} + +function target_setup() +{ + echo "Setting up target" + # Setup the board + export TARGET_BOARD + export TARGET_SSH_CMD + export REMOTE_DIR + if [[ -n $SKIP_TARGET_SETUP ]]; then + return + fi + $PROJECT_ROOT/ci/test/board/oct-target-setup.sh +} + +function run_test() +{ + local name=$1 + local tmo + local cmd + local curtime + local exec_bin + local res + + exec_bin=$(get_test_exec_bin $name) + binary_name=$(basename $exec_bin) + tmo=$(get_test_timeout $name) + + # Update sig handlers to pass in test name also. + trap "sig_handler INT $binary_name" INT + trap "sig_handler TERM $binary_name" TERM + trap "sig_handler QUIT $binary_name" QUIT + + test_info_print $name + cmd=$(get_test_command $name) + + curtime=$SECONDS + timeout --foreground -v -k 30 -s 3 $tmo $REMOTE "$cmd" + res=$? + echo -en "\n$name completed in $((SECONDS - curtime)) seconds ... " + if [[ $res -eq 0 ]]; then + echo "TEST SUCCESS (ret = $res)" + elif [[ $res -eq 77 ]]; then + echo "TEST SKIPPED (ret = $res)" + else + echo "TEST FAILURE (ret = $res)" + fi + + return $res +} + +function run_all_tests() +{ + local tst + local res + local failed_tests="" + local passed_tests="" + local skipped_tests="" + local test_num=0 + + # Errors will be handled inline. No need for sig handler + set +e + trap - ERR + + # Read the tests info one by one from the test list created by meson test + while [[ true ]]; do + test_num=$((test_num + 1)) + test_enabled $test_num + res=$? + tst=$(get_test_name $test_num) + if [[ $res == 77 ]]; then + skipped_tests="${skipped_tests}${tst}#" + continue + fi + if [[ $res -ne 0 ]]; then + break + fi + + # Run the tests + run_test $tst + res=$? + if [[ $res -ne 0 ]] && [[ $res -ne 77 ]] ; then + failed_tests="${failed_tests}${tst}#" + if [[ -n $CONTINUE_ON_FAILURE ]]; then + echo "FAILURE: Test $tst failed" + else + test_exit -1 "FAILURE: Test $tst failed" + fi + else + passed_tests="${passed_tests}${tst}#" + fi + done + if [[ -n $STATUS_OUTFILE ]] ; then + echo "FAILED: $failed_tests" > $STATUS_OUTFILE + echo "PASSED: $passed_tests" >> $STATUS_OUTFILE + echo "SKIPPED: $skipped_tests" >> $STATUS_OUTFILE + fi + if [[ -n $failed_tests ]]; then + test_exit -1 "FAILURE: Test(s) [$failed_tests] failed" + fi +} + +function test_exit() +{ + local result=$1 + local msg=$2 + local waittime + + set +e + trap - INT + trap - TERM + trap - ERR + trap - QUIT + + $REMOTE 'dmesg; uptime; cat /proc/uptime' > remote_dmesg.log + save_log remote_dmesg.log + + if [[ $result -ne 0 ]]; then + if [[ -n $REBOOT_ON_FAIL ]]; then + echo "Test case failure, rebooting the board." + waittime=300 + while [[ $waittime -gt 0 ]]; do + $REMOTE true 2> /dev/null && break + sleep 10 + waittime=$((waittime - 1)) + done + + if ($REMOTE true 2> /dev/null); then + echo "Rebooting board failed." + fi + fi + fi + echo "###########################################################" + echo "Run time: $((SECONDS / 60)) mins $((SECONDS % 60)) secs" + echo "$msg" + echo "###########################################################" + + exit $result +} + +function sig_handler() +{ + local signame=$1 + local binary_name=$2 + + # Make sure that sig_handler is fully executed. + set +e + trap - INT + trap - TERM + trap - ERR + trap - QUIT + + $REMOTE "sudo killall -SIGINT $binary_name" 2>/dev/null + + test_exit 1 "Error: Caught signal $signame in $0" +} + +trap "sig_handler INT NONE" INT +trap "sig_handler TERM NONE" TERM +trap "sig_handler ERR NONE" ERR +trap "sig_handler QUIT NONE" QUIT + +test_init +target_sync +target_setup + +run_all_tests + +test_exit 0 "SUCCESS: Tests Completed" diff --git a/ci/test/board/oct-target-setup.sh b/ci/test/board/oct-target-setup.sh new file mode 100755 index 0000000000..793167f2e1 --- /dev/null +++ b/ci/test/board/oct-target-setup.sh @@ -0,0 +1,232 @@ +#!/bin/bash -x +# Copyright (c) 2025 Marvell. +# SPDX-License-Identifier: Apache-2.0 +# https://spdx.org/licenses/Apache-2.0.html + +# Script syntax: +# oct-target-setup.sh +# +# Optional environment variables: +# HP How many hugepages of default size to enable. +# NOHP Flag disallowing hugepages allocation +# DEVS Space separated list of PCI devices to bind to VFIO. +# VFIO_DEVBIND Alternative location of oxk-devbind-basic.sh script. +# TARGET_BOARD Optional SSH URL for the target board to setup. If not given, +# all commands are run locally. If it is given the script is +# copied to REMOTE_DIR on the TARGET_BOARD and run from there. +# +# Below options are used only when TARGET_BOARD is set. +# +# TARGET_SSH_CMD ssh cmd used to connect to target. Default is "ssh" +# TARGET_SCP_CMD scp cmd used to connect to target. Default is "scp" +# REMOTE_DIR Directory where build dir is located on the remote target. +# It is used to find oxk-devbind-basic.sh script. +# SUDO This is used only when the command is to run as sudo on the +# remote target. Default set to "sudo" i.e. to run as SUDO. +# +# Script will: +# 1. Mount hugetlbfs and enable HP hugepages of default size. +# 2. Bind each PCI device using VFIO_DEVBIND script +# 3. Sets device configuration. + +set -euo pipefail +shopt -s extglob + +function get_cpu_string() { + local cpu_impl + local cpu_str + local cpu_pn + + cpu_pn=$(grep -m 1 'CPU part' /proc/cpuinfo | awk -F': ' '{print $2}') + cpu_impl=$(grep -m 1 'CPU implementer' /proc/cpuinfo | awk -F': ' '{print $2}') + + if [[ $cpu_impl == 0x43 ]] && [[ $cpu_pn == 0x0b1 ]]; then + cpu_str="98xx" + elif [[ $cpu_impl == 0x43 ]] && [[ $cpu_pn == 0x0b2 ]]; then + cpu_str="96xx" + elif [[ $cpu_impl == 0x43 ]] && [[ $cpu_pn == 0x0b4 ]]; then + cpu_str="95xx" + elif [[ $cpu_impl == 0x41 ]] && [[ $cpu_pn == 0xd49 ]]; then + cpu_str="cn10ka" + compatible=`cat /proc/device-tree/compatible` + IFS=',' read -ra list <<< "$compatible" + if [[ "${list[0]}" = "marvell" ]] + then + cpu_str=${list[1]} + fi + else + echo "Invalid CPU (Implementer=$cpu_impl Part Number=$cpu_pn" + exit 1 + fi + echo $cpu_str +} + +function mount_hugetlbfs() { + # Mount hugetlbfs. + if ! mount | grep -q hugepages; then + mount -t hugetlbfs none /dev/hugepages/ + fi +} + +function setup_hp() { + if [[ -n $NO_HP ]]; then + echo "Skipping huge page setup" + return + fi + # Enable HP hugepages. + echo $HP > /proc/sys/vm/nr_hugepages +} + +function setup_devices() { + local npa_pf + local sso_pf + local dma_pf + local dma_vf + local cpt_pf="" + local cpt_vf="" + local inl_pf + local devs + local nix_lbk_vfs + local nix_pfs + local pcid + + nix_lbk_vfs="0002:01:00.1 0002:01:00.2 0002:01:00.3" + devs=${DEVS:-$nix_lbk_vfs} + + if [[ $CPU == "cn10ka" ]] || [[ $CPU == "cn10kb" ]]; then + cpt_pf="0002:20:00.0" + cpt_vf="0002:20:00.1" + elif [[ $IS_CN9K -eq 1 ]]; then + cpt_pf="0002:10:00.0" + cpt_vf="0002:10:00.1" + fi + + # Set KVF Limits + if [[ $CPU == "cn10kb" ]]; then + echo 8 > /sys/bus/pci/devices/$cpt_pf/kvf_limits + else + echo 24 > /sys/bus/pci/devices/$cpt_pf/kvf_limits + fi + + # Disable existing VFs and enable CPT VFs + if [[ -e /sys/bus/pci/devices/$cpt_pf/sriov_numvfs ]]; then + echo 0 > /sys/bus/pci/devices/$cpt_pf/sriov_numvfs + echo 2 > /sys/bus/pci/devices/$cpt_pf/sriov_numvfs + devlink dev info pci/$cpt_pf + devs+=" $cpt_vf" + fi + + # SSO and NPA devices + sso_pf=${SSO_DEV:-$(lspci -d :a0f9 | tail -1 | awk '{ print $1 }')} + npa_pf=${NPA_DEV:-$(lspci -d :a0fb | tail -1 | awk '{ print $1 }')} + devs+=" $sso_pf" + devs+=" $npa_pf" + + # DMA device + dma_pf=$(lspci -d :a080 | tail -1 | awk '{ print $1 }') + if [[ -e /sys/bus/pci/devices/$dma_pf/sriov_numvfs ]]; then + echo 0 > /sys/bus/pci/devices/$dma_pf/sriov_numvfs + echo 1 > /sys/bus/pci/devices/$dma_pf/sriov_numvfs + dma_vf=$(lspci -d :a081 | tail -1 | awk '{ print $1 }') + devs+=" $dma_vf" + fi + + if [[ $CPU == "cn10ka" ]]; then + inl_pf=${INL_DEV:-$(lspci -d :a0f0 | tail -1 | awk '{ print $1 }')} + devs+=" $inl_pf" + fi + + if [[ $CPU == "cn10kb" ]]; then + nix_pfs=${ETH_DEV:-$(lspci -d :a063 | tail -1 | awk '{ print $1 }')} + devs+=" $nix_pfs" + inl_pf=${INL_DEV:-$(lspci -d :a0f0 | tail -1 | awk '{ print $1 }')} + devs+=" $inl_pf" + fi + + # Unbind all SSO devices first + for d in $(lspci -d :a0f9 | awk '{ print $1 }'); do + $VFIO_DEVBIND -u $d || exit 1 + done + + # Bind devices + for d in $devs; do + $VFIO_DEVBIND -b vfio-pci $d || exit 1 + done + + if [[ $IS_CN9K -eq 0 ]]; then + echo "Skipping limits configuration on 106xx" + return + fi + + # Configure limits + pcid="02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f 10" + if [[ $CPU == "98xx" ]]; then + pcid="$pcid 11 12 13 14 15 16 17 18" + fi + set +euo pipefail + for d in $pcid; do + path=/sys/bus/pci/devices/0002\:$d\:00.0/limits/ + if [[ -d $path ]]; then + echo 0 > ${path}ssow + echo 0 > ${path}sso + fi + done + + path=/sys/bus/pci/devices/$sso_pf/limits/ + if [[ ! -d $path ]]; then + set -euo pipefail + return + fi + echo 256 > ${path}sso + # Max number of available work slots are (2 x num_core) + 4. + # Max limit needs to be set for tests to run in dual workslot mode. + if [[ $CPU == "96xx" ]] || [[ $CPU == "95xx" ]]; then + echo 46 > ${path}ssow + elif [[ $CPU == "98xx" ]]; then + echo 76 > ${path}ssow + fi + echo 8 > ${path}tim + set -euo pipefail +} + +# Environment variables +NO_HP=${NO_HP:-} +HP=${HP:-8} + +if [[ -n ${TARGET_BOARD:-} ]]; then + # Run on remote by copying this script to the remote board + SCRIPTPATH="$(cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" + SCRIPTNAME="$(basename $0)" + SUDO=${SUDO:-"sudo"} + TARGET_SSH_CMD=${TARGET_SSH_CMD:-"ssh"} + TARGET_SCP_CMD=${TARGET_SCP_CMD:-"scp"} + REMOTE_DIR=${REMOTE_DIR:-/tmp/vpp} + REMOTE_BUILD_DIR=${REMOTE_DIR}/build + $TARGET_SSH_CMD $TARGET_BOARD mkdir -p $REMOTE_DIR + $TARGET_SCP_CMD $SCRIPTPATH/$SCRIPTNAME $TARGET_BOARD:$REMOTE_DIR/oct-target-setup.sh + VFIO_DEVBIND=${VFIO_DEVBIND:-$REMOTE_BUILD_DIR/ci/test/board/oxk-devbind-basic.sh} + TARGET_EXPORTS="VFIO_DEVBIND=$VFIO_DEVBIND HP=$HP" + $TARGET_SSH_CMD $TARGET_BOARD \ + "$SUDO $TARGET_EXPORTS $REMOTE_DIR/oct-target-setup.sh" + exit 0 +fi + +VFIO_DEVBIND=${VFIO_DEVBIND:-$(command -v oxk-devbind-basic.sh)} +if [[ ! -x $VFIO_DEVBIND ]]; then + echo "VFIO_DEVBIND Invalid. Set VFIO_DEVBIND to a valid oxk-devbind-basic.sh script." + exit 1 +fi + +# Get CPU +CPU=$(get_cpu_string) +IS_CN9K=0 +if [[ "$CPU" == "98xx" ]] || [[ "$CPU" == "96xx" ]] || [[ "$CPU" == "cnf10ka" ]] +then + IS_CN9K=1 +fi + +mount_hugetlbfs +setup_hp +setup_devices + +set -euo pipefail diff --git a/ci/test/board/oxk-devbind-basic.sh b/ci/test/board/oxk-devbind-basic.sh new file mode 100755 index 0000000000..d2fc61aad9 --- /dev/null +++ b/ci/test/board/oxk-devbind-basic.sh @@ -0,0 +1,63 @@ +#! /bin/sh + +# Copyright (c) 2025 Marvell. +# SPDX-License-Identifier: Apache-2.0 +# https://spdx.org/licenses/Apache-2.0.html + +HELP="usage: oxk-devbind-basic.sh [OPTIONS] devices.. +Script for binding/unbinding devices from Linux kernel drivers. +NOTE: Options -b and -u are exclusive. + OPTIONS: + -b driver - Bind given devices to a given driver + -u - Unbind devices from their driver + devices: Space separated List of DBDF addresses (i.e. 0001:02:00.1)" + +OPTS=$(getopt -u -n $0 -o "b:uh" -- $@) + +driver="" +unbind=0 + +eval set -- "$OPTS" + +while true; do + case "$1" in + -h) echo $HELP; exit 0 ;; + -b) driver=$2; shift 2 ;; + -u) unbind=1; shift ;; + --) shift; break ;; + *) echo "Unknown arguments"; echo $HELP; exit 1 ;; + esac +done + +if [ "x$driver" != "x" -a $unbind -eq 1 ]; then + echo "Cannot have -b and -u" + echo "$HELP" + exit 1 +fi + +if [ "x$driver" = "x" -a $unbind -eq 0 ]; then + echo "Please specify either -b or -u" + echo "$HELP" + exit 1 +fi + +for dbdf in $@; do + ddir="/sys/bus/pci/devices/$dbdf" + cur_drv="$(readlink -n $ddir/driver)" + if [ ! -z "$cur_drv" ]; then + cur_drv="$(basename $cur_drv)" + fi + # If user wants to bind and same driver is bound, skip the device + if [ $unbind -eq 0 -a "x$driver" = "x$cur_drv" ]; then + continue + fi + # Either user wanted to unbind or we have to unbind for re-binding + if [ -e $ddir/driver/unbind ]; then + echo $dbdf > "$ddir/driver/unbind" + fi + # If user specified -b then do try to bind + if [ "x$driver" != "x" ]; then + echo $driver > "$ddir/driver_override" + echo $dbdf > /sys/bus/pci/drivers_probe + fi +done diff --git a/ci/test/common/pcap/pcap.env b/ci/test/common/pcap/pcap.env new file mode 100644 index 0000000000..9ddd3da480 --- /dev/null +++ b/ci/test/common/pcap/pcap.env @@ -0,0 +1,31 @@ +#!/bin/bash +# SPDX-License-Identifier: Apache-2.0 +# Copyright(C) 2025 Marvell. + +PCAP_UTILS_PATH=$(dirname $(readlink -f "${BASH_SOURCE[0]}")) + +if ! [ -e $PCAP_UTILS_PATH/pcap-pkt-cnt ]; then + gcc $PCAP_UTILS_PATH/pcap_len.c -lpcap -o $PCAP_UTILS_PATH/pcap-len + gcc $PCAP_UTILS_PATH/pcap_pkt_cnt.c -lpcap -o $PCAP_UTILS_PATH/pcap-pkt-cnt + gcc $PCAP_UTILS_PATH/pcap_mac.c -lpcap -o $PCAP_UTILS_PATH/pcap-mac +fi + +function pcap_packet_count() +{ + $PCAP_UTILS_PATH/pcap-pkt-cnt $1 +} + +function pcap_length() +{ + $PCAP_UTILS_PATH/pcap-len $1 +} + +function pcap_packet_dmac() +{ + $PCAP_UTILS_PATH/pcap-mac $1 "dst" +} + +function pcap_packet_smac() +{ + $PCAP_UTILS_PATH/pcap-mac $1 "src" +} diff --git a/ci/test/common/pcap/pcap_len.c b/ci/test/common/pcap/pcap_len.c new file mode 100644 index 0000000000..f500c7fc1d --- /dev/null +++ b/ci/test/common/pcap/pcap_len.c @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2025 Marvell. + * SPDX-License-Identifier: Apache-2.0 + * https://spdx.org/licenses/Apache-2.0.html + */ + +#include +#include +#include + +int +main (int argc, char **argv) +{ + unsigned int total_len = 0; + char errbuf[PCAP_ERRBUF_SIZE]; + struct pcap_pkthdr header; + const u_char *packet; + pcap_t *handle; + + if (argc < 2) + return -1; + + handle = pcap_open_offline (argv[1], errbuf); + + if (handle == NULL) + return -2; + + while ((packet = pcap_next (handle, &header))) + total_len += header.len; + + pcap_close (handle); + + printf ("%u\n", total_len); + return 0; +} diff --git a/ci/test/common/pcap/pcap_mac.c b/ci/test/common/pcap/pcap_mac.c new file mode 100644 index 0000000000..e1cdc57a86 --- /dev/null +++ b/ci/test/common/pcap/pcap_mac.c @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2025 Marvell. + * SPDX-License-Identifier: Apache-2.0 + * https://spdx.org/licenses/Apache-2.0.html + */ + +#include +#include +#include +#include +#include + +int +main (int argc, char **argv) +{ + const struct ether_header *eth_hdr; + char errbuf[PCAP_ERRBUF_SIZE]; + struct pcap_pkthdr header; + const u_char *packet; + const u_char *ptr; + int i, smac = 0; + pcap_t *handle; + + if (argc < 2) + return -1; + + if (argc == 3) + { + if (strncmp (argv[2], "src", strlen ("src")) == 0) + smac = 1; + } + + handle = pcap_open_offline (argv[1], errbuf); + + if (handle == NULL) + return -2; + + while ((packet = pcap_next (handle, &header))) + { + eth_hdr = (const struct ether_header *) packet; + if (smac) + ptr = eth_hdr->ether_shost; + else + ptr = eth_hdr->ether_dhost; + + i = ETHER_ADDR_LEN; + do + { + printf ("%s%02x", (i == ETHER_ADDR_LEN) ? " " : ":", *ptr++); + } + while (--i > 0); + printf ("\n"); + } + + pcap_close (handle); + + return 0; +} diff --git a/ci/test/common/pcap/pcap_pkt_cnt.c b/ci/test/common/pcap/pcap_pkt_cnt.c new file mode 100644 index 0000000000..3ed60fae5b --- /dev/null +++ b/ci/test/common/pcap/pcap_pkt_cnt.c @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2025 Marvell. + * SPDX-License-Identifier: Apache-2.0 + * https://spdx.org/licenses/Apache-2.0.html + */ + +#include +#include +#include + +int +main (int argc, char **argv) +{ + unsigned int packet_count = 0; + char errbuf[PCAP_ERRBUF_SIZE]; + struct pcap_pkthdr header; + const u_char *packet; + pcap_t *handle; + + if (argc < 2) + return -1; + + handle = pcap_open_offline (argv[1], errbuf); + + if (handle == NULL) + return -2; + + while ((packet = pcap_next (handle, &header))) + packet_count++; + + pcap_close (handle); + + printf ("%u\n", packet_count); + return 0; +} diff --git a/ci/test/common/test_list_helper_funcs.sh b/ci/test/common/test_list_helper_funcs.sh new file mode 100644 index 0000000000..cc06a38f95 --- /dev/null +++ b/ci/test/common/test_list_helper_funcs.sh @@ -0,0 +1,138 @@ +#!/bin/bash + +# Copyright (c) 2025 Marvell. +# SPDX-License-Identifier: Apache-2.0 +# https://spdx.org/licenses/Apache-2.0.html + +# Functions required to manipulate the test.list file. + +TEST_LIST=$BUILD_ROOT/ci/test/test.list + +function get_test_name() +{ + local test_num=$1 + local num=1 + local info="LIST_END" + while read -r testinfo; do + if [[ $num == $test_num ]]; then + info=$testinfo + break + fi + num=$((num + 1)) + done <$TEST_LIST + echo $info | awk -F'#' '{print $1}' +} + +function get_test_info() +{ + local test_name=$1 + local name + local info="LIST_END" + while read -r testinfo; do + name=$(echo $testinfo | awk -F'#' '{print $1}') + if [[ $name == $test_name ]]; then + info=$testinfo + break + fi + done <$TEST_LIST + echo $info +} + +function get_test_timeout() +{ + local tmo=${DEFAULT_CMD_TIMEOUT:-5m} + local tst=$1 + + for t in ${CMD_TIMEOUTS:-}; do + if [ "${t%=*}" == "$tst" ]; then + tmo=${t#*=} + break + fi + done + echo $tmo +} + +function test_enabled() +{ + local test_num=$1 + local tst=$(get_test_name $test_num) + + if [[ $tst == LIST_END ]]; then + return 1 + fi + + echo -e "\n\n#################### Test $test_num: $tst ########################" + + # Check the SKIP_TESTS and RUN_TESTS and make sure that test need indeed be run + if [[ -n $RUN_TESTS ]]; then + if ! (echo "$RUN_TESTS" | grep -q "$tst"); then + echo "Skipping $tst as not on RUN_TESTS list !!" + echo "$test_num: $tst [RUN_TESTS]" >> $RUN_DIR/skip.list + return 77 + fi + elif $(echo "$SKIP_TESTS" | grep -q "$tst"); then + echo "Skipping $tst on SKIP_TESTS list !!" + echo "$test_num: $tst [SKIP_TESTS]" >> $RUN_DIR/skip.list + return 77 + fi + + if [[ $test_num -lt ${START_TEST_NUM} ]] || [[ $test_num -gt ${END_TEST_NUM} ]]; then + echo "Skipping $tst as test num not within given test num range ($START_TEST_NUM-$END_TEST_NUM) !!" + echo "$test_num: $tst [TEST_NUM_OUT_OF_RANGE $START_TEST_NUM-$END_TEST_NUM]" >> $RUN_DIR/skip.list + return 77 + fi + + echo "$test_num: $tst" >> $RUN_DIR/run.list + return 0 +} + +function test_info_print() +{ + local name=$1 + local exec_bin + local args= + local defargs + local envs + local tmo + local cmd + local test_dir + local extra_args= + + exec_bin=$(get_test_exec_bin $name) + test_dir=$(get_test_dir $name) + defargs=$(get_test_args $name) + envs=$(get_test_env $name) + tmo=$(get_test_timeout $name) + cmd=$(get_test_command $name) + extra_args=$(get_test_extra_args $name) + echo "Test Binary/script -> $exec_bin" + echo "Test Timeout -> $tmo" + echo "Test Environment -> $envs" + echo "Test Directory -> $test_dir" + + # Remove unnecessary arguments from command line + echo "Default arguments -> '$defargs'" + eval set -- "$defargs" + while [[ $# -gt 0 ]]; do + case $1 in + -l) shift; shift;; + --no-huge) shift;; + -m) shift; shift;; + *) args="$args $1"; shift;; + esac + done + echo "Modified arguments -> '$args $extra_args'" + echo "Test Command -> $cmd" +} + +function get_test_command() +{ + local name=$1 + local cmd + local test_dir + + test_dir=$(get_test_dir $name) + + cmd="cd $REMOTE_BUILD_DIR/ci/test/$name && $TARGET_SUDO bash ${REMOTE_BUILD_DIR}/ci/test/$name/$name.sh" + echo "$cmd" +} diff --git a/ci/test/common/testpmd/common.env b/ci/test/common/testpmd/common.env new file mode 100644 index 0000000000..e2b5dec99e --- /dev/null +++ b/ci/test/common/testpmd/common.env @@ -0,0 +1,180 @@ +#!/bin/bash +# SPDX-License-Identifier: Apache-2.0 +# Copyright(C) 2025 Marvell. + +TESTPMD_SCRIPT_PATH=$(dirname $(readlink -f "${BASH_SOURCE[0]}")) + +TESTPMD=$(which dpdk-testpmd) + +if [[ -z $TESTPMD ]]; then + echo "dpdk-testpmd not found !!" + exit 1 +fi + +function testpmd_cleanup() +{ + local prefix=$1 + + # Issue kill + ps -eo "pid,args" | grep testpmd | grep $prefix | \ + awk '{print $1}' | xargs -I[] -n1 kill -9 [] 2>/dev/null || true + + # Wait until the process is killed + while (ps -ef | grep testpmd | grep -q $prefix); do + continue + done +} + +function testpmd_prompt() +{ + local prefix=$1 + local refresh=${2:-} + local skip_bytes=${3:-} + local in=testpmd.in.$prefix + local out=testpmd.out.$prefix + + local cmd="tail -n1 $out" + + if [[ "$skip_bytes" != "" ]] + then + cmd="tail -c +$skip_bytes $out" + fi + + start_ts=`date +%s` + start_ts=$((start_ts + 60)) + while ! ($cmd | grep -q "^testpmd> $"); do + if [ "$refresh" == "yes" ] + then + sleep 0.01 + echo "" >>$in + fi + #Link change may break this logic, timeout loop to continue again + ts=`date +%s` + if (( $ts > $start_ts )) + then + break; + fi + continue; + done +} + +function testpmd_launch() +{ + local prefix=$1 + local eal_args=$2 + local testpmd_args=$3 + local out=testpmd.out.$prefix + local in=testpmd.in.$prefix + local unbuffer="stdbuf -o0" + + testpmd_cleanup $prefix + rm -f $out + rm -f $in + touch $in + tail -f $in | \ + ($unbuffer $TESTPMD $eal_args --file-prefix $prefix -- \ + $testpmd_args -i &>$out) & + # Wait till out file is created + while [[ ! -f $out ]]; do + continue + done + # Wait till testpmd prompt comes up + testpmd_prompt $prefix +} + +function testpmd_cmd() +{ + local prefix=$1 + local cmd=$2 + local in=testpmd.in.$prefix + local skip_bytes=$(stat -c %s testpmd.out.$prefix) + + echo "$cmd" >> $in + testpmd_prompt $prefix "no" $skip_bytes +} + +function testpmd_cmd_refresh() +{ + local prefix=$1 + local cmd=$2 + local in=testpmd.in.$prefix + + echo "$cmd" >> $in + testpmd_prompt $prefix "yes" +} + +function testpmd_quit() +{ + local prefix=$1 + local in=testpmd.in.$prefix + + echo "quit" >> $in + while (ps -ef | grep dpdk-testpmd | grep -q $prefix); do + continue + done +} + +function testpmd_port_stats() +{ + local prefix=$1 + local port=$2 + local in=testpmd.in.$prefix + local out=testpmd.out.$prefix + + echo "show port stats $port" >> $in + sleep 0.5 + testpmd_prompt $prefix + cat $out | tail -n10 | head -n4 +} + +function testpmd_port_rx_count() +{ + local stats=$(testpmd_port_stats $1 $2) + + echo $stats | awk '{print $2}' +} + +function testpmd_port_rx_bytes() +{ + local stats=$(testpmd_port_stats $1 $2) + + echo $stats | awk '{print $6}' +} + +function testpmd_port_tx_count() +{ + local stats=$(testpmd_port_stats $1 $2) + + echo $stats | awk '{print $12}' +} + +function testpmd_port_tx_bytes() +{ + local stats=$(testpmd_port_stats $1 $2) + + echo $stats | awk '{print $16}' +} + +function testpmd_log() +{ + local prefix=$1 + local out=testpmd.out.$prefix + cat $out +} + +function testpmd_log_off() +{ + local prefix=$1 + local offset=$2 + local out=testpmd.out.$prefix + + dd if=$out skip=$offset bs=1 status=none +} + +function testpmd_log_sz() +{ + local prefix=$1 + local out=testpmd.out.$prefix + + stat -c %s $out +} diff --git a/ci/test/common/testpmd/pktgen.env b/ci/test/common/testpmd/pktgen.env new file mode 100644 index 0000000000..6955147639 --- /dev/null +++ b/ci/test/common/testpmd/pktgen.env @@ -0,0 +1,88 @@ +#!/bin/bash +# SPDX-License-Identifier: Apache-2.0 +# Copyright(C) 2025 Marvell. + +PKTGEN_SCRIPT_PATH=$(dirname $(readlink -f "${BASH_SOURCE[0]}")) + +source $PKTGEN_SCRIPT_PATH/common.env + +function pktgen_cleanup() +{ + testpmd_cleanup "pktgen" +} + +function pktgen_prompt() +{ + testpmd_prompt "pktgen" +} + +function pktgen_launch() +{ + local opts + local coremask="0x3" + local port="0002:01:00.1" + local pcapin=$PKTGEN_SCRIPT_PATH/../pcap/sample.pcap + + if ! opts=$(getopt \ + -o "i:p:c:" \ + -l "in-pcap:,port:,coremask:" \ + -- "$@"); then + echo "Failed to parse pktgen arguments" + exit 1 + fi + + eval set -- "$opts" + while [[ $# -gt 1 ]]; do + case $1 in + -i|--in-pcap) shift; pcapin=$1;; + -p|--port) shift; port=$1;; + -c|--coremask) shift; coremask=$1;; + *) echo "Unknown pktgen argument"; exit 1;; + esac + shift + done + + testpmd_launch "pktgen" \ + "-c $coremask -a $port --vdev eth_pcap0,rx_pcap=$pcapin,tx_pcap=out.pcap" \ + "--port-topology=paired --portlist=0,1 --no-flush-rx" +} + +function pktgen_start() +{ + testpmd_cmd "pktgen" "start" +} + +function pktgen_quit() +{ + testpmd_quit "pktgen" +} + +function pktgen_stats() +{ + testpmd_port_stats "pktgen" "0" +} + +function pktgen_rx_count() +{ + testpmd_port_rx_count "pktgen" "0" +} + +function pktgen_rx_bytes() +{ + testpmd_port_rx_bytes "pktgen" "0" +} + +function pktgen_tx_count() +{ + testpmd_port_tx_count "pktgen" "0" +} + +function pktgen_tx_bytes() +{ + testpmd_port_tx_bytes "pktgen" "0" +} + +function pktgen_log() +{ + testpmd_log "pktgen" | awk '{print "PKTGEN# "$0}' +} diff --git a/ci/test/common/vpp/vpp.env b/ci/test/common/vpp/vpp.env new file mode 100644 index 0000000000..8429a99b09 --- /dev/null +++ b/ci/test/common/vpp/vpp.env @@ -0,0 +1,151 @@ +#!/bin/bash +# SPDX-License-Identifier: Apache-2.0 +# Copyright(C) 2025 Marvell. + +VPP_SCRIPT_PATH=$(dirname $(readlink -f "${BASH_SOURCE[0]}")) +source $VPP_SCRIPT_PATH/../testpmd/common.env + +if [[ -f $VPP_SCRIPT_PATH/../../../../build-root/install-cnxk-aarch64/vpp/bin/vpp ]]; then + # This is running from build directory + VPP=$VPP_SCRIPT_PATH/../../../../build-root/install-cnxk-aarch64/vpp/bin/vpp + export LD_LIBRARY_PATH=$VPP_SCRIPT_PATH/../../../../build-root/install-cnxk-aarch64/vpp/lib +elif [[ -f $VPP_SCRIPT_PATH/../../../../build-root/install-cnxk_debug-aarch64/vpp/bin/vpp ]]; then + VPP=$VPP_SCRIPT_PATH/../../../../build-root/install-cnxk_debug-aarch64/vpp/bin/vpp + export LD_LIBRARY_PATH=$VPP_SCRIPT_PATH/../../../../build-root/install-cnxk_debug-aarch64/vpp/lib +elif [[ -f $VPP_SCRIPT_PATH/../../../../build-root/install-vpp-native/vpp/bin/vpp ]]; then + VPP=$VPP_SCRIPT_PATH/../../../../build-root/install-vpp-native/vpp/bin/vpp + export LD_LIBRARY_PATH=$VPP_SCRIPT_PATH/../../../../build-root/install-vpp-native/vpp/lib +elif [[ -f $VPP_SCRIPT_PATH/../../../../build-root/install-vpp_debug-native/vpp/bin/vpp ]]; then + VPP=$VPP_SCRIPT_PATH/../../../../build-root/install-vpp_debug-native/vpp/bin/vpp + export LD_LIBRARY_PATH=$VPP_SCRIPT_PATH/../../../../build-root/install-vpp_debug-native/vpp/lib +else + VPP=$(which vpp) +fi + +if [[ -z $VPP ]]; then + echo "vpp not found !!" + exit 1 +fi + +if [[ -f $VPP_SCRIPT_PATH/../../../../build-root/install-cnxk-aarch64/vpp/bin/vppctl ]]; then + # This is running from build directory + VPPCTL=$VPP_SCRIPT_PATH/../../../../build-root/install-cnxk-aarch64/vpp/bin/vppctl +elif [[ -f $VPP_SCRIPT_PATH/../../../../build-root/install-cnxk_debug-aarch64/vpp/bin/vppctl ]]; then + VPPCTL=$VPP_SCRIPT_PATH/../../../../build-root/install-cnxk_debug-aarch64/vpp/bin/vppctl +elif [[ -f $VPP_SCRIPT_PATH/../../../../build-root/install-vpp-native/vpp/bin/vppctl ]]; then + VPPCTL=$VPP_SCRIPT_PATH/../../../../build-root/install-vpp-native/vpp/bin/vppctl +elif [[ -f $VPP_SCRIPT_PATH/../../../../build-root/install-vpp_debug-native/vpp/bin/vppctl ]]; then + VPPCTL=$VPP_SCRIPT_PATH/../../../../build-root/install-vpp_debug-native/vpp/bin/vppctl +else + VPPCTL=$(which vppctl) +fi + +if [[ -z $VPPCTL ]]; then + echo "vppctl not found !!" + exit 1 +fi + +function vpp_cleanup() +{ + local startup_conf=$1 + + # Issue kill + ps -eo "pid,args" | grep "vpp \-c" | grep $startup_conf | \ + awk '{print $1}' | xargs -I[] -n1 kill -9 [] 2>/dev/null || true + + # Wait until the process is killed + while (ps -ef | grep "vpp \-c" | grep -q $startup_conf); do + continue + done +} + +function vpp_launch() +{ + local startup_conf=$1 + + echo "$VPP -c $startup_conf.conf" + vpp_cleanup $1 + echo "$VPP -c $startup_conf.conf" + $VPP -c $startup_conf.conf & >1.log + sleep 2 +} + +function vpp_start() +{ + local startup_conf=$1 + set +e +pidof vpp + $VPPCTL -s /tmp/$startup_conf/cli.sock exec /tmp/$startup_conf/$startup_conf.exec + sleep 2 +pidof vpp + $VPPCTL -s /tmp/$startup_conf/cli.sock clear interfaces + sleep 2 +pidof vpp + set -e +} + +function vpp_trace() +{ + local startup_conf=$1 + local port=$2 + + $VPPCTL -s /tmp/$startup_conf/cli.sock trace add $2-rx 100 +} + +function vpp_port_down() +{ + local startup_conf=$1 + local port=$2 + + set +e + $VPPCTL -s /tmp/$startup_conf/cli.sock set int state $2 down + set -e +} + +function vpp_stats_all() +{ + set +e + local test=$1 + echo $VPPCTL -s /tmp/$test/cli.sock show interface > /tmp/$test/stats_all.log + $VPPCTL -s /tmp/$test/cli.sock show interface > /tmp/$test/stats_all.log + cat /tmp/$test/stats_all.log + set -e +} + +function vpp_stats() +{ + set +e + local test=$1 + local port=$2 + echo $VPPCTL -s /tmp/$test/cli.sock show interface $port> /tmp/$test/stats.log + $VPPCTL -s /tmp/$test/cli.sock show interface $port> /tmp/$test/stats.log + cat /tmp/$test/stats.log + set -e +} + +function vpp_rx_count() +{ + set +e + $VPPCTL -s /tmp/$1/cli.sock show interface | grep "rx packets" | awk '{print $7}' | tr -d '\r' + set -e +} + +function vpp_rx_bytes() +{ + set +e + $VPPCTL -s /tmp/$1/cli.sock show interface | grep "rx bytes" | awk '{print $3}' | tr -d '\r' + set -e +} +function vpp_tx_count() +{ + set +e + $VPPCTL -s /tmp/$1/cli.sock show interface | grep "tx packets" | awk '{print $3}' | tr -d '\r' + set -e +} + +function vpp_tx_bytes() +{ + set +e + $VPPCTL -s /tmp/$1/cli.sock show interface | grep "tx bytes" | awk '{print $3}' | tr -d '\r' + set -e +} diff --git a/ci/test/env/cn10k.env b/ci/test/env/cn10k.env new file mode 100644 index 0000000000..3244edbaf2 --- /dev/null +++ b/ci/test/env/cn10k.env @@ -0,0 +1,90 @@ +#!/bin/bash +# Copyright (c) 2024 Marvell. +# SPDX-License-Identifier: Apache-2.0 +# https://spdx.org/licenses/Apache-2.0.html + +# Directory to keep run specific data on build machine +RUN_DIR=${RUN_DIR:-$BUILD_DIR} + +# Test run command +TEST_RUN_CMD=$PROJECT_ROOT/ci/test/board/board_test_run.sh + +# Skip syncing DPDK build directory to target +SKIP_SYNC=${SKIP_SYNC:-} + +# Skip setting up target. Useful when repeatedly running tests. +SKIP_TARGET_SETUP=${SKIP_TARGET_SETUP:-} + +# Reboot the target if DPDK tests fail +REBOOT_ON_FAIL=${REBOOT_ON_FAIL:-} + +# Platform +PLAT=${PLAT:-"cn10k"} + +# Target board user@IP. The user is expected to have passwordless ssh. +TARGET_BOARD=${TARGET_BOARD:-root@127.0.0.1} + +# Target directory where the VPP build is to be synced +REMOTE_DIR=${REMOTE_DIR:-/tmp/vpp} + +# Directory from where the tests will eventually run. +TARGET_RUN_DIR=${TARGET_RUN_DIR:-$REMOTE_DIR} + +# Dependency install directory on build machine +DEPS_INSTALL_DIR=${DEPS_INSTALL_DIR:-$BUILD_ROOT/build/deps} + +# Sudo command used when running the tests +TARGET_SUDO=sudo + +# SSH command used to access board +TARGET_SSH_CMD=${TARGET_SSH_CMD:-"ssh -o LogLevel=ERROR -o ServerAliveInterval=30 -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"} + +# SCP command used to copy files on board +TARGET_SCP_CMD=${TARGET_SCP_CMD:-"scp -o LogLevel=ERROR -o ServerAliveInterval=30 -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"} + +# Extra env to be set when running the tests on the target +EXTRA_TARGET_ENV=${EXTRA_TARGET_ENV:-} + +# Default timeout to be applied to the test commands +DEFAULT_CMD_TIMEOUT=${DEFAULT_CMD_TIMEOUT:-30m} + +# Test specific command timeouts. To be given as a string of test=timeout entries. +# Eg: CMD_TIMEOUTS="dump_memzone=20m dump_physmem=30m" +CMD_TIMEOUTS=${CMD_TIMEOUTS:-} + +# Additional arguments to be passed for specific tests. +CMD_EXTRA_ARGS=" +" + +# List of tests to be run. If list is empty all tests are run except those in SKIP_TESTS. +RUN_TESTS=${RUN_TESTS:-} + +# Flag to enable target setup needed for perf stage. +PERF_STAGE=${PERF_STAGE:-} + +# Continue testing regardless of failure +CONTINUE_ON_FAILURE=${CONTINUE_ON_FAILURE:-} + +# File to save status into +STATUS_OUTFILE=${STATUS_OUTFILE:-} + +FIXME_SKIP_TESTS=" +" + +DEFAULT_SKIP_TESTS=" + l3_fwd + ${FIXME_SKIP_TESTS} +" + +# Tests to skipped. +SKIP_TESTS=${SKIP_TESTS:-$DEFAULT_SKIP_TESTS} + +# Run tests within this test num range +START_TEST_NUM=${START_TEST_NUM:-1} +END_TEST_NUM=${END_TEST_NUM:-999} + +# Functions required to work with test list file. +source $PROJECT_ROOT/ci/test/common/test_list_helper_funcs.sh + +# Export the path to this conf so that other scripts can source this conf. +export TEST_ENV_CONF=$PROJECT_ROOT/ci/test/env/cn10k.env diff --git a/ci/test/l3fwd/l3fwd.conf b/ci/test/l3fwd/l3fwd.conf new file mode 100644 index 0000000000..704eb936eb --- /dev/null +++ b/ci/test/l3fwd/l3fwd.conf @@ -0,0 +1,52 @@ +unix { + log /tmp/l3fwd/vpp.log + cli-listen /tmp/l3fwd/cli.sock + #full-coredump + #interactive + nodaemon +} + +api-trace { + on +} + +logging { + default-syslog-log-level info +} + +cpu { + main-core 2 + corelist-workers 3 +} + +session +{ + event-queue-length 10 +} +#tcp { no-csum-offload } + +buffers { + ## Increase number of buffers allocated, needed only in scenarios with + ## large number of interfaces and worker threads. Value is per numa node. + ## Default is 16384 (8192 if running unprivileged) +# buffers-per-numa 5000 + + ## Size of buffer data area + ## Default is 2048 + default data-size 2048 +# naturally-aligned +# enable-nat-alignment +} + +devices { + dev pci/0002:01:00.2 { + driver octeon + port 0 { + name eth0 + } + } +} +plugins { + plugin dpdk_plugin.so { disable } + plugin onp_plugin.so { disable } +} diff --git a/ci/test/l3fwd/l3fwd.exec b/ci/test/l3fwd/l3fwd.exec new file mode 100644 index 0000000000..233446b3df --- /dev/null +++ b/ci/test/l3fwd/l3fwd.exec @@ -0,0 +1,10 @@ +set int mac address eth0 d6:bd:c6:81:0f:b1 + +set int ip address eth0 2.0.0.1/24 + +set ip neighbor eth0 3.0.0.2 0a:5f:b6:10:a4:17 + +ip route add 3.0.0.2/24 via eth0 + +set int state eth0 up + diff --git a/ci/test/l3fwd/l3fwd.pcap b/ci/test/l3fwd/l3fwd.pcap new file mode 100644 index 0000000000000000000000000000000000000000..98405e6523b467adfead1b1b5d8751f34a2154ea GIT binary patch literal 728837 zcmeEP2bf+()t=qWZg#)jB=lYa7my;bGxysDNR=vR6jVUz5Q<0*O;9O80qKJDrYKDi zl->jcMVd%aIsziScP#wxxpQ*veBbQk4`GvRIXs^D{~EtLbI$v|b7tnu%tL?p?Rkx6 zlqvriWg77RJbCatJ6^M5i&+c*H(6ux7wFFh(`wc>ri=dC>Mi^H9{*Wm6a9q-(^mW? z{4wFQ@#f1L7*p9e>&(`luy1vfwI*crlZR|N+H8csKopo$i++4V{_zV_KYsW5#gDuA$ESRx`1j&Vw_Vp}cEnf6qAzX6Upgc8 zr3)7-zLd+CZ052&t7ctUch-|_jQ^W`R*_E^pMBujikXB@l0~1L$e%qn_1WKFT70%T z<;Szy&-P}0neEK1&8$Nki)D1vOSsY&1_%hIx}a{KKTV{ zU(veX-ap2iftHa)tviNWw_~bx>;I={UC(E1opYJXv&>a9*M+9a&($_{x!!T+LNt>s zYU=6S)Xh^(o$KzRsl5Z)RM(xkp3L=Tt`E(YpO$84m8U@azAYNeRcJF=)b0zo-5aIa z{ozSPyZdHTyYtR0w^b1PwV>}5v7HLH zLtfRvxCSVp_rqS~)hzGI^6o4rtt!7+cBu%9JEzI~1-&7QdUPxIC{Ok1>W(5TP@|dL zqr5lE`?6|hRwY)K-!FUT19sfCqs$ZN8d=o6`?z~crMh?cQ@(rUXfu-n7Iv@dvMPq1 zYE~sxfa*u1PUic1?f4e+68cFN_4RS?>-?#{{=VDyRYtX`4Lk!0&QxEk-C4CK%c{Lu zm2?H4Om+O~{YIPj(P^@%<1cc@J5n8=e=FZ{8A=CL$GbYSE}M0^tSisDs^kZ0XNmQK z$?qNCYMPMkkwsp3&wF9gCn+zyIluRU4B&&=3te4VS9jLclXdlyN2JZAM?52rdbG{V z26vFvyQG0qVWh^B-cGsXn%BKcBxe{ZF6rvax;wLOn{`u)h`cOCoIO`>Hw(iHWRZWy zv45UV`RAXV-anG43`PHR=UI0(>qbuBopl!@rF5r6Q^BvfPVF$u!ckz+CYh;neGXijRi4JIWX^vW^HunhEb{4E?9(e!K3!;e z@6%bD{>}=Y_I75yHtTg+Z=Us1R)WcOG{_VVHkdVU3FneU4xYdcK0oE)C*JZ7mZWz$ zb3X)aor8P3vfl2jw!W2xtdXVSaMY$x^!Ivp&B6$7<6QG!&A)5p(S zG3OvUWKo1p;0SG(iqL5<`UuI~Nh36!5we}wN$Ko{(PR|s>XFVB;kxFscJph5iYyA( z*&MEKq{6iUk2W%4({K$yxNIl3nmVC|SZqh=#JNH=tkcq*W4^h^nClQXvM6?!aO~Dk z#cu8^Jk?~HsIeR1*jd_K#X7sik~88cjt=7J1G)P_W6d8CM6xK5*K#0NNd@v0zqx=t z8y(fA%x%JKyC&tUNj2(6wNNud5{7}aST{#RA+E(fb?Nn*sQv@=tej}fgNP_u6xBO8 zsw<|V`onDsdSWwCqdJqLYH6F4w)?T)Z?Wcs3Jr;ZM7csCUhSKW<{ts{+qJ1+F1(eTXgAoh+6^EGFg%w?st3ofYxE7t12AA=YG3yr1TH&z*|* zH~-}0Epva3_h5*(#Ug;koYP`nh}cVX%GmqhKlImT^DzQX76t!R4t{GY_^Y6b3v*9Y zQ>dQLVm%MZ9tPiHmSmA`+oJxcj8p&wkAxf68*N4-&}0!3KJrM|``;-f9RCN81X<-! z85nFb5F)`M#j{8NEJg*=2MJpQi#0EAHFF{AWc9FUY++cuo`S`ar+Qe(x{AVL$iTv4 z)UXf?3v^hN$^j-4MuAC@LEN)L#ViI0kVP1olQHs43L{T_*JDIha}*;(79$q*lomB> z7WId~i-c70;>VgFw|AHofCsV&Fyk3852XNe)>k}WWDQ9HGb{jOQL$=KvTCtu5AaBM zOYnH~EP{%el>rX22tCU)dj6C`&$f$u^vKGUqG#Bl$D$_KqA1uFRl(4rXrBP9&Cg)1 zV}q(O>j5cb_!s`6KmGBb#+4XK*VBL5-|?V#{#`%`u+{0$5SaedB)#?7O*nFeLKH&D zqAJUx?AoI2+7@-!W#~nu3S1q2SA+RFU_(ZTs=?K|jH^pixY~bik1JWAQ;8bZ5><~Y zi-JECZet`wX*-Uw2@YyO_WssQW?Nv0ECSgk46<`mAlvvp4>DO6RFI8akXam{usA_s zQCkYY2_^(^ez<&j-e$8q07VudZW~72Pg96nb$5?AS&US~DdGrm7Uxwg4y@SX#0qk^ zI9)47)vs1Yn}dKZvIu^=G5mg%g5P}0did#Lby+SuleUznwlkGltRC=et!7jW1uul3 zMM1sA`5=n}LJ&g1N-2aqb9fmC%8mro$RaEr$XGleg~eB|_E?m4SCvAQ!g?tTv1oBx z&EkNE#d$1?16jbPU`Mc7h~)F#74tLTjw}M^5e&>-Q@~sX^IYucVWOi`+L1V={nR2^ z4@`@rf)-~5Esh@%N(D%@DE%H*fPVoHl0_&znNhk`3Z-w{>rpD}%POBLpPxoPYfx%& zPSfI`ro~B3AXY#ni1m}78_yeKE(apXBFO%nA^WuyWH0=chpen>D`XY20|Hr#vz-=a zWGxQK0=@#U624jGS*&%Rxv?=f0iI+Let*OGT|0%}9ry6~m9=!mui|&W@oRC)+v2FF z#Q|ktS-=)x*@OA6iH+tiV3jNa^NkGV-V~UdS1G`ZWme5{HOmdwK2u8pQ_XV2{2>(M z5X=_G?JdsRTby0CI3145-~@aE?n0(NUT&26JAg|TA^Xpa?B!C(z8C)m=5N?LP-H8z zX9cn?E~{Tsu45&AFt+j(EkujOKUOku7t zpVct8xPijr3JUx|X-kR9I?OM$ZL4_;kS2>T{~Tj}_7vvZaSI_yyKIkW2CfaYh$p5VQtl@ND?UvK^)!*$r95hW~mt?Dav44d>6{*&y3M$_8b_;AVrx zEkzdB6j@wTWN~O6>=5({JBms0skm}{KG1@!o*GRRrpC)DY8-Tjr-tl3DK(TDLyH;~ zcSl)V9_5P5qfj5k^IV}xR(TF<`ipF5%#z>;S;UihnI})Cc(U(ao+q;7r94rd3|*c$ z+~?(RpO?dZUKm;g)g@;(Krx>EF#SkOq%;sq&X8e$kQ}j_Suv)N}Az88iz~p zz@9Ex@m!5Syq-Y|e{Ymo4eTL{7-X42e@HRtYj=7E$?l#qNEtK?8RT$Fp~JOz4%gmc zz!10xZ1Mxrn`gF|4M8Qch)%0Aovu#N>BCD4I-zJ;_d;|pWKivegzLiLvRM5>Kseh8 zv$&c8DWsFbEs_p55jxyN=!%;NOZW@DiWyS()2(JR@QN(r)|Z%D7p1s$&5@p4x}C0P zMsPf-JR~>M3Pq-A&t-;k%i(fJhr1zNaW^E+#q_@vvGjy?GZ7>si-@)<6Yb0t(YEwA ziQ$xk4q`fpDbXCR>vXuT)8V>KJj^MykiiYd&}ad0?A~L{-e4SA#JU}rb;qSxch@C; zc+-*s9o}?!!%M9J>m0p`iyKrOZcrr(3P(Z;{u31J#FfYggMMTY1@~eK9-5-yZ?^Fi zlx<>N4AI4qVZInrKYI%^%p?UJZZUTB{B!aCGmgPVX()z7;$ss^N{;~-$s#`fkomYz zijO~i%=1w;rj?J%N9ALYh}H~=As-#RvrW&O(>v$IMZ-9vUeC{hoNLU}Y)%I;$s%$d z&E(uEMb3Rr^W>E6a3!abQ_1=1$mwwRv%}rb4tGBzuN5wpd2PYeyl=F*04ybon0gvB z^;;>XPF&qHRW{+3smfGk>hv?!;WloE+qfNW;|6PmS0!tG_WRONZRSc)mMo&}ub8%9 zPSJMV*F0@y=U-{7v{l*;3~e3m&v&>J-Qj9$P*^w?P}p<$(kI%@t>7VY0wn=u#I8J# z1?TG*X)uq2*<=yB?`3v>F2(NWeLXqu9#eKJyOrIuhTRSqRXSY#=Wu`8##u>5VR}|j zegAWs%!{BoSw!{6nCc6psDApN{Xj0&9GaPHX0Dm}49?8!2l9~W4p+=M+{EQ@PZ4M@ z#4Ks=dH?33E#^INo-E@13(Wg5Dc*18x2xcM|_YU_pIRwSwst#y? zkTlQ$F9KJd+iDs|K>*0=t}JVMmqmd7n|`k>JMwlv_)Fo64*oj$>)<~~2LDh59Bw6X zxN^ba2)V;KZ&C$e?G#lgrUUcg%)>aS0a?8wjB4;03il_k6;c+YTA^B@ zS}~Nh!l8uTp*q*0)(^TN%r14q3&(N~HJWaS23aH=OS5p?lM;@~{$4nw7)gagg+qm7 zxCnoBwYe7NCA{FUk75RNiMIOih3U}h+eG}c_&o2i}i|I9_zsamY5ZpVJsl%+;r9418$DU5^6Kx;$}L~H!~ zX!$>Nm|Y+=WRci>o5kkTl-L~J;>AYl%XF1RS6OtGWjL;~)X$HSV&kAS4sT^Typ;*z z5osueC#$@GjiJrAGiE=i4q2o;JG1iqB&9rS-RqS{s@ha}RC!c+RC!o=9A4>lc)rx( z`BJEm$V8w*UWN`s8QWw?5LqNc`?3uEASFZh9qnaE%IH*vREAWBREGMIA%{nl9Uhc* zcu>|A56VKLqWU^Yjn==d*_;49B8xQXFxI3!Q=0U%30{+=cuzG+HAyu|HE9|&$>EJ( zhc|v*@y0Lmr+R(zNz)xTF?2S>i7XPR<5{4#PYKk2zvBf;N(NP+RG?I#RG>bkKsh|S z45h*(5AQBxk|%mnPV$PCrZ<1kZZ3gRkwt2C7OT}aQ)=}DuW?JEqNFP`kGJ_mOUffvx>UMUx>ULb zO1kpS&Me34j{R;euVY$Yf<_jJ z+U+cAohebf;>AMLuo0kz;93Z-h2UDKITJ-K@9crN^}=ZWS!$37oKM>(Rz{nrp>AZ6 z!u^#ME=ws~<0@X^r1n&&pE~{2>8DOVXTe0-yQf(0=1IxjZ~eVS*ge&RP7^v!=rp05MG0Mv+~pS2S&QkcrT5iJaVj%C zpXyD%ro(&$-6M;%?-ka*wv_hK%ci&}0aYWaeX4z`eX4zfP5W}&4GqMs*V026rBDSr zm{ndv68LSLOKNHM775{pUI_R2Pf7?se%uS8RQBqeSLeJs=hZpyES>X)Nnj|1x$Vnx zhsm+So=R`E4?a8J+Z2ys-mi*s^XAP z#oS>E?J$LQ#S~gZtV|sXiJb4PW-}i0NES)t94wK~rX+IfGrdGg5wl99N~B7pO5~7} z$lPJ3?J(1J#Y|fSt&~eY(VlpDt63geNfznlqO6w>r}XkKw|l*m(rML8)l1b&)ypBR zm$}1)Jnzi#2R+=0QzG@VaXhO*tCTNX+HO{YaFRvh`FR%4J5%C$#rM2;O2M{@r;4YF zr;6wB5zjn#5K&Crb9$Z#N9|%E^-6lsC1cIHP*Ji-N&8qyZ%8TW3Ga9%l@f7PQdLq_ zQdQF7sib+1Loqq#_c`YGc`?5i<&*h+^%je_SeU@N1ePTz%chc0tWlsF8HNbz7rKRBp(wsDt9E8pn^j#_&9iDX zt9E5o+}mF5$*R3swJ+=H%(`rb_q9>!g4eb2t}dR{#jCn_R2L6u;{8lKpNZ%3@H!qI z$HRM9c5EX~0I7G!EDh^R`h>Am09D?Ey6o;VnS5TBWL}ehZ zF$X>HWkkgxDh^R`h>Am0KJ7K;#k0}_agBNLP69oX5ME-Y=%Vu(%6mHZZ_LM zW62_o-Iz7@yp+bCxG8I_l*3Pd?y4G#K!NtEvCvS}SPXm)1E0gd=P>X&uo+sa8XFv| z9y}VGW8lj%@Z}izNN+_BL%sEads5bFc7xcGMS{CE3+_oN!F~0cEVu)3LI{`wqBN;h z!BxQp4ODQ8tHh>ukHz2=+#EBF95am^^S>PPze0gUA4P$Sboc31+RWO<6jrakrpK%A zJ~yS*cagbReakaSltbUKL&a=_A0>-^bT$6bt5QFD@elY%rEC_tdGSxt;vXH`ZZ^jc zlSMziA^-S=sUN@jr~KnmaW>`0YyQ3X(oGk1m>uyIvgk{j@t4j>ed*Yoza+&PG8Lc^ zQT|hj2O3a(_D}bZF_Z8~vgorD`LoBSK6~>e{8`!Nl+R+#B*UUdhH18+1<7BD+VIwV z?l~(|%-@jqZv1klg z)S!d8L6cGq`og!lL9(#+|2K$MZfF$)>k?RUDB36g>%d<{>ppC4G-se?WKrvm;nwY# zYTegced`dF0a=)XE6bm%Z|V)tHJS_2OtPq{r*l&`Pc=33M{#JKYe1SBrd!l(`Prb^ z9srMSKFVB$Hj_o|zJS}kQL5dI=dz1M20t|buxf`zJFE^(Els38$S1-+_)fg{^Ja4! zIzSe6;!5ts>ZwkgbUb&WR9iGC!)j_)L+y}Obts;uPSh>nQlI3DL7xibZ}@nN`3rhO z7WL>>?opoV(XJbFk4mYZ89fS@q^WoE^`Lh?V3oOA%@gPvS=7DzxO+>by0`k0+`V$J znc3Y7SKO(q(vq;NzOOqR*k)crKgpuLKF)oeKh@W@K}?)hrp4|7SBB5JyHL}txZ_tn zTQTpW(_~S{U*wK=q&ohDFEXbU{z2FAx@{HmgS0a6L&4-X*Y7Y*4W_Wj3-5U^O!_3{ zg==^oj=^%Udm-FhB9BOm1CMw{ynNmmGaFn&R_~IAMs~^DDVI!MoEaf$!;Beo>f|~U zw~+nQv)@MUk=BRq@%}k#>2YRZ_=YU<&p7ta^C|!Q;8O1&j7dWYGE>=)guBYl1n#Q5 ziVSwG6&lR4a1>eOtcBQFkEEP6?=RR{k|7N_FimaqmOLjttM}Za6B|tryhavzZW;F6 z-6_w#yA^v*^0Hy&xp3Q?yePd4yqFaL+~ejZ^F_FjEOKKvyYc3f8?VpUjgkruTQ`Q) z1LRWaap+R-(@{$`o3Fy3WRXwTVxL}-^691Luumlc9T7gQtH>Z1ivt1|7oa`u`z>Zm zIF~GP@C0`7`6&nQF_9fCY41pMa9Hv}&K73`&h~I#==ss+JMc7F|xRq36Bpx5Mmff4H42a{Vss`lC{=e`H>Ey`=Xe9q7s0 zE{cPADUQP%4^+(Ihyhs?hy6GX-%rKiG+uqhjCGjgv;#?jsn*z0T*PZ-T#8ZPdndT7(wjlFvdU|ijBbHqi(eyDWKp=z=5T!@6|Pkv1xD0Y`{?AA}k?#Sgib}~`a*bQLpqAG9-rnp*yS&(oPw!r_0 zAd*Fayp{vGN-B`6%*TP0>8S>CMg}r0=ck~G+a;);o)@e<$~=gOl0{LygQL1)Dyr}L zvu{-DuOs2WjD*uwKui&q5R?)2A-?vn&E_8nFIg1g2ROuwr$YQY-a|)ym46nhCx7<(W5|2@@e zK1SflqTs*E!AJd%X>6ju&|uoIja>XQ;k5Cl;TRsJWywKD>6tZ3PtTz{0)vFA1VfRp zHGZRFMkCN<5fVP~NZ9+|DI{Ee2qQrjITQ(l83{GVF9{YBw>ntt_-ebE3sEPlhecxx z!{YT6EKcfWSjf_f!eYq4B0QB#5Rn*`5b=Y^Bd8=<3=klTFfu1&P(8Iq!s zI_gj0k=Tad@#tCNu?Dj;z(E$FXL&}?pHk>qdqqZ%tXnC1h8241FNz>ENvumW6+$%E zSDVawKnhs|rIi>;*QcPg}wyhQE!!YK1~z=kZs)w+zU zOH;VAJ2S3ijZSejVsKS+mkohTKu`~|f6qVKYzqvLMIhURL3VBmWG}DFAd{6r1=+|1 zS$JU*0ZsrB0M5(WUte!Ey8}>U5#qLC#QijdxQn=q3kP>~qBmS7dINez7oktU5u(q- z?+zSgItb_@i{Q5#!|z8a_rsjuKQ)uoOmd(udxfr+hzby!RS2?8O-<%3D|nF z3E#{FXG%ziobjYtVyDr@fiPqdX_jEp+@B&%`{_&?*=p0gOY^RwlXp$iL$kyn!FSA{ z<}=#NYG4mp#30KI`a_CAAJ4`NlFdD3kTPhPF{u8jW@41!y<}7|+iCb=yV(#_B8%v> zD%0ue6rG;Cm+2&1ib^M?(=ew~`0_bXOR!&3%O`t_zB$Hh240ax-1-u8>!K95p4f}I zB^#N_E#=k-;+DhXXbw+^IlQe!JQE%SJj*I?VKw`)H;kDGl95G3+mwlRW{PNkT$qWb z8~RzVE=3HRr3lxV<4HxlV(vQe!aLqYb$EZ;;Vn?t*#}b5tBZIv53eeQM00p%&fzUH zhu6D^bi#>{be?tVE#74I2II&g*6qNoJ1)h#*Z93y9NEwXKxLh>ZVKxh9)@%DoG9My zDkvxfi79x}_RZ#C(2p#l;9g9@LsJwybXOkuWS3YoOwBMyXoeXMd^Hqwc=^rYsWL}T zD&vVK8Wn{wB_)eQwDpwH<`{61EaKx2nUDLV__#9Oi>J!cZrPhwJ}MuTk4Q#?A+es1 z4v&91JUr#_$WaY>N^bgmb=wcy%;_K|SwzmGnVdVN$oVE8qU@F(aV4jcQ^`5)(>#oZOm%o< z$>B{NhgVx_NENcy({|At#+oZZS+a<>zhc^cIYrxbo?_a{=D!ZGI>0Jzr*D7_Y3uL; zkHbqW4i9iRytY6z7Lt`TuDp${lm(7%Ft>ugWD$pd#~fZG#o<@(XAVobfpS{fOwyJsT19j-ig zxOm#(wqIhpFfm}dr}_drJ5!D?oQFc|V zhvU*uqP?&(q`l|;=O?tA_rQ6wi1#lr@5iKgzwg$}d#NH(-Yf5w_k)1<4mYMc+%oB~ zL*cMPK{_BTEp?#C^G`c;jA-#WdHBqGON8^expPtB^c2;D(8X zAPr=ZJd9;|_*Y6EKAy7dFEu)vFlfS{3Byp52ZzEYhjaf92b4)Fgvx=r(W~Wlno4LYfz+LkPY$6}BGd|pgVqk`J{?YGLN`iUhq~c~ zdU)WU43ei2D1gUg)GvSudu%SGNmtv9lX1dsvokBtPg2UW)SaB>NKu&2|B z*U!pCGfGMFsnN=3x0@57M`V#E9mblpXG)Wv;48=RCB6JuFF)4u+hJ5QQ@`~cZd%o> zFo!EZ;iRlSH3~J!^k><036(3WkaYbE+jG}K zw#Xvsx|pSFgOqeF@-3DwsgBe{OA{?kv^3G0-b5>uF4xzW<(-`{3oE9Km^?;FSfOL@ z-`!&FfQFGpI(7}~*cVbd_GBOHn3iGdDKI?+Hmpv8&9oJ*WC~WVV>zBS%<-~ej+YIw zbBM*TNYi|B_Ski8<^c#AStM$=v#52ZL~ZitSk$BtRFgAJ&NMmGKqSJ=Xj91kULSMNbc6}9cNyJ)R9GU_Y}+BJSn;R@HLh@DSB18Q@K;QQ@NXEa+g~N z-NSU&7B|3Rr!=0*W?AKZ>@*yN>l{CV?vX{>_X=xYTT1($+n%*g%4}8pRQpu>RQm>n z_T{z<8i-jhZqLeb5hu-hMXAa(un@v0|JrO?noVJm5Ps-|aF73_gz)ZpSqP=LSA|f8 zP=!#1a8L?iZZSD_m>j#}p+6L*#0g@dig%vdYUYG0lGUqXb2F>r8!1)1?}R8<`Gs+c=Wp&h2s&QGC5!OFCeXV5>tsoji+Jd#BcIR{JRvnh$ZU`Linsbf}& zREboHREZo?5}7;9v>j&Ju6S}1C4zB6$$I&Pm&cmrp_OEjUM|Xd`EW`v?|-MzONRnP z{(|bI>ZR(X>gABt%iQ%sH~X?I&*@U@K-$7wGpl@n?V84`8qG=&PO?ZmKhNTMXG%PG z@V6&9RBo$y>X|k@)23(I2JlSV)C#J@+QXX4oN%kUe!C`Ir3~leVX|2xo_UTVDLE$Y zIVSEoP22;iE5)=>())hfY}SQ}l0{0|$4YubN=X+vo6}IK64x|T(@;%AH4PmeX=uHY z<~io~Ip+5{&F>wK{l)Y9LRMFPy47p~NhOP9bsd(~%TluX=R;Ulr8ZqfAq4m3C5+h(VoMeY?$#{0C#3}U)vvJNN*TTi zt_rRSt_tqR72Ldvi#DoQQ>bE1q1r z@t34rLq;Zwhy15CGZ6CLB{(TD37;j4K0A>=du-~nf4JO#7N>Kkx($golz$~^!&#=q zV7aKUs118_8xBde;i7}M4WD8wl-eVIJ#3H95?)zvv^f?HB8wVyFgIvYszEF7#0`?A zHHk%6)}8fay;)xdcc2A0-G{?{INOKgEjZnR!!0=5f{JcxpZu?b_7zC6hqRhA&@!^9 zb;oe)c1*SIt2iEr7m*#lJJpa)rFAGbN3}W1Dl?Q-*8Eg`Q*T(M&0L6Pl0{8DotwIO zs;P_qotr8N-KTbBO;b?-isB~JHWj5!4sNekDbTWT@s8_M%vESJS=8cmM$a3@N&Ma;82 zt7g-R;-vG#P86Gb7&$Upm!_9q9EcFdr3lOX?{}ELpf_Yuk8b52<*6R+`X%mBDfKg> zM_6a4^`O4YuLgnRaqm3pzPG|y^8~s^7Ip7F?%qtCmZ=0R>>lP} zn1*2%hMa%`DJ`i5=SLWKwyqlU68cFN_4RS?>-?#{u5ks2QU~GVruvHX2iZ@C z?59dc6CD)1pBQHq>i9FeHkkL(X|kx}FLK8_QXPK`N=niKY7K@U*L1wvmvwb!T{i1- z846NrS{VSK@WNY%H=3pfQ&{AM_q-P-eUkFRRok%_WDFlX;ji;T7v!=l>*~(BitRRx zoBbf8aLLn`HJRDq60&-iG*I4ZR&SZ~cFHA(uD~vlv|&iNq^mdU>dU%2v+iOzLr&3; zdpvOt!xhR4!wY1Qf5x$Yo=^Gbdl#{P1}N?MjOsWw;)IY1{nPET?mX+RX5AQoi;)uP zSM06=OgAH?UKWlbi=4F(JL{2@v*zUl2Av%2Dl?7F>h8(9d$aDotf#Y>p_V8oT63`LhQ#_!JCvEYfEtcr;swBBs91ys;P|bgKwwo>CT(Zc)6WGD$ryRWN zwhUTHdq<*!@oFa??Zj)Hcq@;bEzSs>?cr<==rG@br^zC3Z^7O^CFSkarbvY3+9TcD zcrFlc=-@F5T;fii7srI2_pX2BjWK3_xScF={VweKqf)N_Tjqg|Rfyr39)yEjsLpjt z5&}JRJ+7O^g{*kjC@bDIa;2BzIBdQ2ICD5+Ko-SeKaRuqQ*k)u8jgcZDwF~x1#04e z%Ytw<2A*rdc}9y9hu{Ym(4yqI@-b%U7cbIaPDDV+qM%IXpzM(f%4Dph(+nL;2TG3V zKbceG!GJhLgzh=H(VTT@lhS4yX=8ET%%Wu2;)Z>TItz=P zK8wq!DO%#qAX-@wuG>0BnO`GRWKp=z=5T!@6|P-x=Wq?r$>>0iHuWl(3YW#*;TCs? zTWkzc=)|*O=zQ#c@l3P14sj!kVs{D0Zv9m34qt|2Clf`b+GnI%7&}|+B7%~%kBAMu zBADW831&gUC3hNa{)ixwMS;AQ1G!2nkiBzpAZ2>0ft-2s~L7 z{8u^nt*PL*oapr&l?EF8K^T0C)qRWQXNy%)i}g8xp$t_ChC&9m!P3xZ1ez>D!bctn zd;dFygv*X$B*-F%j_k8yWDjLOl!1CASgaCStVrRHVtha)O_UB){*BS$+m|$$xe#@- zdRR2JFf3kA!Q#X(GAwj)YG}X1Ny;HHER`cGFtYCAO=dAbfGonuoQ#oYQW%;19AiWl zbW}QqjC5Em!&$87SX5J5%o;7`6&A@R;YH#Z;Kc*x)eA?N6@Uk_2r%OrFb|~wGx1CY zj4UH5V1_(kEV5LKe3XhmfwMKC2Mqws}X}M zi-K+(&%#KE%68H&0YRx<9%QG@-fp%9hR7n2ZNeZsHwChPt;Hadl|cpB$OM_i`3cK6 zdJ^CS5CPyk;*Pwg!|V<~kwu8xh7tGE6ykn4CnHYQBNcIqxbQXuLZ5&mM4yM>VmFL6 z2LWAV5&U*z`28pazcpX*@Po8!;y)4+|1evt8C65U3*l#R`o-dSkj42R>kkN(U@75P z`484bw_TyZ90{nAMOZwLv3NiViK4k#0z z1#Kmsi~OPM{EE2?SS7>1YV(I18O*&YFu(N;2D2>SE0`6`GZf4g$L%dnbXpt^CtwTm zN??1LUSWd{^LGH3EJF658QIIFko_>^} z2PFyvlzW(OxW!oWPoS7Ag8APV=8L3Ye%C21;Idbt0u@aY!?Cyx z#Nt{O5Tb-@Nr+-NSa0oS(~jkBvWN};^=#PdgA^NH=k09lQz{#j4MTzr7Pk~x+)`w5 zaRf0#pj$G-4-_XXGuq4tT9DOKqp8BwcsWIlQ?KWNLbjhYQ_@UnaAivM14T#;i>sq7 z?vAp!Jc?){Fb-(qdGhv{HnSu+LKg94UgpV@DV}V18uLUpy_6@)lcC8IhfCJz$!>aJ zmN+Aj4msmV^JCOaI}nB}BFz#^n)_3vY3DQVsKe2`OY^QFmv@DOOGp}rE7rgsJXwM} z)`&rZ@0dZKn|q8|4eTL{7-X42e@HRt<5nJaWOGl49UXRt{;*TSActED9j?7|xb}|T z!y-lr-b+Rm3jx3QXsp=~R3eM$v?|l->J*)xyMyT@TZ&32rPDB{6Fu)nPpZ+gWkfB( zeo3vY@;~gk9Qjd$*$liQi@5bA=GH|iZauLJb4$0=hxYv?;oTHdJ1qX0&y0k1`ZeXy zsj9tc-AL8|r*a#ZLLZZ=&gbvqr(t~zHI^je}I?uXuaTm(oU>sS*x*eEx$E8^J z62BLVVq#s((6x-=xt1}l!&}HYdQp+yJao80wV-k8}2+})u^!yq!Rvb#>!K~z_&sQ(4jxnc$m}C(- zk7jc2lp^PQD>6A{M_kFNhR>F;y$fo_J^|-qtn)ViiesI0f=01>_EF$zROz3Kg z(D(e12`xnqN@yjt5_$#`njXBNM@;A~3L342edTEFQ|_m_cFUbmtyyg zi!i&Tv_jdf>{fQqM0V5rAP$c?&|Uk)bYWt^bWe46Z@YOBG$)Iw{uonzffUuxxRd@R|Aa37rQ1_U2ZW`i4pht`@9cQ-Mh&JB_v;l_cjsi&yDS1E5B=U* zedM!V1aSYK7J6x+mlk?WQ=wN_SQ74ChdW~ROY1eua^WmK%=5#+zg`6B+F81x)8YO? zQiPJSb&7Cv$0#!nia=JM7L01}im>N9DMeW43RVQEmeI68(*jKk24PxIqX-UHJkm9H zbh8(!gV48Bhpb2#e);Pbvk;_#ERu(@ED!%m$-^f{vph(Rj>?0|gUZ8Dk_WotjP7ru z`)Eiigvx6>iUnCK9!+V*&6}}SNM(>}g=&Rr#Zc7>x_yW43!#e* zNH>Jmp>BBL`1_^prW>L`7754FEFAZwgk$~1SU98>Nrgj&Lxp2F2nXHPK!?X2j(0m8 zmm?t&nwLUSOjYJvb*xzn3PKjCNEfTf?^7x=+bgUhQUIkYqAH>)G8|Qe4)@WyAv%RX ziXxOR6(y@QV7uh*GaAeU$O%~_DQmK%{5B;iXY%#tcvM5TOLV(Lw@Ze|c1gH68m@kX zGn)FuB$O0772!}wjX(UccmH6N7Ah!zA|fnJ92I-aiP=xUCx z=8VMEoN%@kCP4KvMDs98iJX5>w$zLuIjS^5lP05p_5|n=S)@sau_o=A(xk_hVNH_y zJxz@?HPX~bQ=@)TqfnC^6v^R@Uxzn-F?qxP6{SpBRtVJn$2FO=Ax>nGKpoElwS7vU z?!AEpN~#7`pj4n#pj4nfg+MtxLJp0>BoFT{lUj+G1lg2VtEF~pF_%E8$Rf2mi`D9z zDYg0$-%^C{=@d_=csj+?DPFn8J+&!bm`>HGmBTCX4o_h_JcUilCBhOamzS;!aeCrf z$QD^7T^F--ZIF_#1-{9NmQ+V-qNRzJCR&d$VN|)>F%W}Lx{AH0SFpdBx<*_sCA}9?cj_>O$tF()Kt_|)Kt`Fw5a8s-4M4Pn4f=&0EP3< zN)t9BpM0soJPmatixlp!tZ-RM;l?>uI4M6>g;Rx7g;Rx_nF^P8Vw!3(O||~XTP(cA z`CL}W-R(z?GOt4F$RfFWisf#el-zyzJjm^E?K#8DGRO&kY(;uuO~?l9AKm}xtDa+1md`(1G)z1-)VCbK-Wk}T58 zMOiN&PU+=6Z?IlU^|b1x>ZR(X>gABt%iQ%sH~X?I&*@TYN@J7aIUZ-)R)TPnMdJB+ z7SB6V;UTTsp6^PIXuKO&v7Ir$HYCy#69;Dcages;$BSKk36u=tP2$- zi-U$>hPD`@a~W2GQoHC8oNHC8otWNB<(#bQAfiv?AB_=^rk_Pgpzg8Ti& zo6K$yTe3)Sw`RdTDJ8hCuFHZeW%w$%D!3}RD!3z8aPulI+Nfepp^7zyYEM=uu;`;? zyhXbE+~%Xq+Qt-CufC?otM5KHrPOzk4pv_&{Le_zU;TNr*$6*M7X9dI{G(T;e)OV) z{g2{&>6F6O%>Prgc$php%;xxEvgpS*i z=u4aNm(ECi>F6H*l9X%6$b=CQhsgRF5sS|rac--bgin%1pPk5`JvQ~(-~XCFD;u5i z*{-ZR>&bevz6`FVHppKQOuz}g4X3x6L-2jFs118_8xBde;g|b!8)Sc=|2E+CF#LdK z6i4akKwUpR@eMj`_I7hD8bcN}=wNQpq*Q}e-kKXEOKbmsgK)eBr(1Bi1!r4O(M|r6 z|8?N6!d?Gezr&n?mXSrRJBC}gW2$vTlM4O5%t}9X>rifvYIBrTW+> z?opoV(JuZnO*$>XzdvKbQr9D_v(tJ|U*=bX%CCnJE4ugam5t^Jbd4dNf!0>aqjE4B1bWjwVt$WsG2EW%~!C%=_pxS=8|tx#JzFjz4xTcU(r(nU3Kw$Tc0W_GMk2 zS(nYaT!sT8#k(B8*mHNMJooNb*mIJX4J*&}*sRB8J$Z(=&`IB=mw~=}wvKzh!+a4g zB#YeG&2GFo<;LshWH(AGIBeaB7v=GuJf3^TYtQ6T>2c^%@6%<_n-MXuk4U4K-{^?zHCT`%dqy1sNh z-Z8?1Ah<|BE8aDt3g$S~z+{jF=j{tenZpqSvM3JwaU8y%io+?t;W)^oLgSE#10HO_ zwUM|!4No3eTW0TR#=MLzd*&$wX0OH-NFT*jlx;0Ye*`*x4+CDXx}a7NgpO%XXMQ zB8X&BAg|>>u96C5bv6#9OiwkCGcu4C+tZe|)38lT3>CLaP(3|geq)Sz5D_JdqIw5M zb;VRv-+R|b6+2TpW1WS!dLhDP2+9a&l~&9VE?vTye;~YMQHUSl5HFqz@s-$2q(uoV zK5K|)VTdjEeJwUbE%pX$!V!eq$9s*t8_a8nHCYtzr#aqpr{aA8N*0QE%j{o;dS(f= zrTQHz;6VjL7;lSxP>T&Ci|s9ny@V!=y$}9bb2OQc5qPpF_^)#CTT{VrM8Q%Kd|7hP z;18zYTWr`_Y&}_QoLH>%6Br~^B^V0XKLanujz*x#A|!m|k+Ao_Q%Jb%a7KbGawrl8 zGZHK|B`h`$@P{8CByM%En6zb!nF~=TtA|Bn3&Y~|6f90$g<&B}D+-Gt0}G3lWQ$cx zi-j>#4vAq25kH7r_wH!37$86vVPsCm$TKO7On#g(A`3c-ks*l@izPXWwH=G{L&A&1 zGr)@n%sgkbnH7KsvIsEa888o}05kC<28=8tDPV>?U@Vp#EGDHE6GsA%#5M$vN6(85 z?Pg_wgDgVN@{FE8rO>nXqKqC{w^H;BEA&{*cP!Fei=tqQL5t8Nu`bb6B)yj((P7pD zQph4Gt;A5eJ_V&E?`0^-YMDZ5SVPHTOtX+C>&vcDfE@3%7r0t()v@O5fDKuMt92Py zm!@##cJa7Eb+fLa4U#pqa80FtT3s`l3bW=gBMJw&dWi~gWr3>9BJ>uO?a;0g1f_Om zl{PHYcARL;w!jcs1hP#SWap+p_R4w;GF_ann%sivr=zL+Dd8whu$!)?xJt1R4M znU;mSnFlh9^Ana|*dV|OAOgU7#62^y(d-UDkwu8xh7tGE6yh$Pj}a&9kvd$CnBlTU z)uY^ktO&Fx$ ztUz`HQ2TcPmn=f|pBdT9rI7s~pPRr6hHf2==&hsF8f*O`SGcfHGk*&+<1o#rm+27M z7WW=lTzp{t%?Bk41C)E1kN*59^G~3dEQ0yp80L$lVE&ggy?|qpPGPPvA2iRyL)lA6 zMwtKKnJwlmK$FO{TDj4gd9Q*z1E78(!w^Z0rYV3ZyB}ph%k7Pk~x z+)`vwQANxU=$6dz14T1#^P3OUAgiZFQ-!JVa*7%!@ol}>jZ$hTHHI8DEUu2SxI4GfjxM# z1b3_vg9P6(gT9W(6jlR+$RY+=X3!r}4Eh+mb#!Y4DlarW)AVder)S}?Q^O#KTM8Yn zy>qzs&iQNaO1zhhDh8wjpJ_H5f=Xl&omOQ!U7e!Sb2l-aWJ^&8Bpr~3*??3}Cx@E| z9d06YxQUR462X2+EuZWi@$hJ~8F)n&aqCOWt&39JdTd|MEffdpP^ClF$QY_ZZaLiY z>2Nos!`+bJS;>QdXP#(#ty3`*K{B$4Xqz(8&P);Q_QjZJx}mS9QROu12pYsfqB&f` z>TtK8!`*&FI^je}I?uYtam()BU>sS*x*eEx$E8^JI^U#&0%9HBba)%t!&}HYhZ{8= zJtvAAR0|3UL1GHtbkJCHFz81XQE)G&;GroB9`;=x_+*z@XKy-t(}53zUqC^JTZ|oU zwRQ9iqQBJ^2VQv0Dk)j%!1Dg~2Q`>uz(ul%k3VES?vvuQ^qRfDY=<>a-P#|all2gg40jD+5Fc5RtH!e zV5e_@4QcCeiN3=H=?)j9JAXksj=1+fgMAMdT;B@*l0_W;9dme%6o+5Ag*mL{HM*6e zTPfvM%4Z&W>pARjXMN^yX}$B8*5inO|AaoYI>w-6uCR#Ew=kirDMH_~9}`-N9CT3D zL0JdofghA>2<`BWfWtci4(|w1(k<)@lJ3eFEI4nroiUGt-DDBF?`3v>F2(Nam-Xz% zy&yVM)|s--lm~34Jk{YkWVgdBAr4PFI6UnDrk6|%n4T3>|2v*bd=WG!i>Uq>Q+!B2nHe@0It1fcFlsojANX;gEMZT&hkwAS^9)pcwprowb_AQ4jz!oMH2S z`U_N>-enP>|E94}i3ZbF{3ZM`;k5DQ@TXV=q;5q;Kt(`BU=WIc>%x2hH-S4`r(7t4 zP_|AHDpyv_IH&sg_YiP(@Hh80?DRa4WUL1-=f~n^L|Y z^exrF&l+BMufr?^X&{T_VJyqTzf$t>?;}_qq((>OLFGZ^VJOLi!)>q*H)1+m0!UIJ zR1TyftBl1a!a}&vei`TlS)>&UvQ|8r(u(W9&RU`6cDm=Ud;T+P&p)+^5UwoNuQ1e1 zF2YnI)Cz~I5gqP*bGXNfbVFzz>V_AN-GA9=x*-~5k#H={!f{VZIM!a6g+pqQG|A8; zLz9f5nq-8+;c$hQ!=*zGx1%`!ycw?Qjw_TX{b*gP*{lTxA&XR`i&ffmK8b zpj1UvMN~zGql!3ON8xZ`g!3=X;o|>DQG8`OcBF_999PLxVi$WZGx zO=dDAh%Az!eOZQnkdmQ~_GTH9N;;Jxl_8ZOm7#uQ$YGbtVO`nz2W9b!WV@0P*$^kPNT7~qf!aPLP=CILlP0Me)TBw1CQX_&Y5ISXrcj_9(kCbt zCV6;wnbbYFLGI`lwRE2&jfwNkZGwNka3UbS*~CEnpF zY=@_?Nx4K=Lgn&_R@V(<&9#s%vPilvX6f1>C0+Au%+e**kt$s(T`FBFT>~LquCFi4 z@j7{q*U5ALIypowTGFb_h83;dmu@t7K*Pu)9lM5g>ZSmx7phl70{RB_FzY3`%i{$Ppmb-aU za#wkimDwSLTjE?& zOS88~2tV{fxW|7|LiptTEQC_rt3s$ks6wbhI4FfMx0oC|OpYBrn_b1uMyl!WfNU391iBySHiByRkQWBXv%(NY5+K!%_tfGV_PAIYB zNP4;3fo*1aXeC*smy5DqKAh6aJ6~bFlLp`S)S7aJXMtG zB*nAkfDW?~gp({1&(E`X-kB25EpB1)lzMFyPZduUPZiJMA)a}TBPls1?l~szxu3X; z)RhzW${a{T+i``{x=>NFNJ;xxNpDCg={!GUC6y|1RZ>+_RZ>;b;i;r~j`@9#`F-x^ z_ab|xs%C|(zGs`vCXiIJNLJTjS-mVJtG6D&vMROdDyu51Dyu51!(UeO9LoSX_T{U7 z6+k2~kXWy=7p~Z1wt>EqMH;&?YwUR`jXlo4tyx8FJTEh-#;V4u#;V4SERD^pSS+Yw zv7kzC83%G0>a7>t+i>sUZV+3tNN~4i!96J@xG($bc&a#vq=KvGA@w|@o`;;(=OM!b zLiI)RVQpww1r=`b*KAqUueye71z}Pf3T|G-MH^MDDO9nhQ0>VI1r~i21um8pZrY(@ z);6ZFdi6CuUVZnuDW$#(&*7I8a3pGq`qJ{RJGpo$i++4V{_zV_KYr5*{Nqw{Hs!}_{=N9p0=JGaJK`&3(U&&kFP)M4 z($OpOm!w=nrUEn~%6}^HKm&@;{^X;vW)eP07JYUifA-kaXK%QWKPwxZ^4YGeJL}1M zv%U<|6KaF}m6|rpi3slVi7|)Z`(#lY_U1Mml4`?wXnKV{Ao~OTw*iATkb|>y{wQ7f z*TeQ?6>89>^EH}d(HOF*K?ie#CZ!s*@}}G%Sz7!58-xhKL1+`swxFV$x+DMVpnXN_ zR$9EtoPm~+MXfuATeoAXb?g0yTPL&9Pu)6{o1@wsWtADqDygaRQ}s|@5C$LZ85i@17uMr zuH;Uvp6bMj{u#p|~)z+PUZ%{+mwkwx9RkGr>2s(Y*YS0k!G z%ghW|*ged{Fb%^j3^@S>Qd$yr)%W!rqyjIYpJY*AALqW#pX%$HzvWQMz&2wTRtL^h zUy=SG`^k{~REwjDP?;nvSKGbaypK+kMIC>UJKmA%_+zJX$7M7fJRPt0WnGNVl7^FkNKs;;c7JL@X8 z+sYOP9`THL?z}N(Hn@bW-X#roJI}hSSvRsDKT=9(0(VvBLI(Tz z8^$aPN0CL&T8N$XNXl7rpW~f{G-D|7D+E)Wv$}h-?%u4sFYDOJ@OiH)WQ zUL%V z-R#DjQ*ON0U#(fi_&Xc{7`hQJ%HutGJok*(p2?-s;{ceLs}rdo!K~z~@F!X1)3w;A zSEPJ;G0#d+Q8$u&iU+juq%B^w#hwUWl_VF70|FNp4qolX7PBRsOBOkJ0z3Hpl!L#! z9dlpO-eJrA5VUm;#;cupv=eW3;;lS#wm2hjwukd}Uu!krfv3qLZ*RfgJ|*St)fZxK zORhcAy^V(n@h%~r0K*Gl4=cU z4f__ygDp2V6H>l$fDR?!m(RF6}ulT%dwM*qEhWMQZ0;~ z#n!5&-7nhM#Li|BOmVdYvmoKPrY7@81d%KX&im-&BjIa;!vx~Nv ze;~YMQHUSl5HFqz@#VaSjvYA-@hl9nMYVy&ZoS2RI)z)p5ro^v`<c%WME;jab>ZeVzIJM${{fs#-wZ@G#iva>;5k}@@j69RV$m9nZ zBeI~Q7#WfnvDkUASmm}@NF}^TJOjLBm3gp0S2@v`6@Uk_2r%OrFb|~wv;DaqFev`g zlyeZKob_2p%~%tPN(dN>^<0aEL5rGBfTxUY2p*4~W7lmoD+3&45qg$q^!zD>p0$=^ z^vJrE%FU3Mn-Dz~i&qxQ8y0h3i)ksLNn%~1sSu)Xzt&{d15(H$D6Pa$x;_P^>`{i2 ztd=R1hBcHdW=|GX*VdO^qwbgopb}PIqBi z$l^Gb#d$1}NdZnElLat$-eRoz8E{7y0rLn3=B_DV{+JKCf(bhH7|~OY)EHeak|8iH zP6}Ea6|^|SRiIQ*6ryxqtN@SyhcUkZ2+1Opp3ErSDuvQL_##4RwW3r}%5#@#mCqWK zS{&1~IHzfGP?L035EURcs}N*2|7D}O9Ec=~Ap3KM?AKC|U1upj;v=HGcR%)tXHtU1}OJ1zYyi7{{)K3BAEY;VZKNT=6Cb8R9FmA z0apQ^*#aKI+~TC9#SIkrqc}QMLNdbqa)0VDZvoO|5$2y`%+H>}{N$}z+GU4CrCp_c zmPmUI<}EH>vDnG9xD13c8-Ztte=qvuZy0O-2Yi!71bCAPFe*iWCHV#xEHJ9*tLP6J z(GLk=aUG7ueK;1kfmmG2LWB^wmV~IxhXuvewrDWzO{TDj4gd9Q*z1E78~(-H*;r~< zHYgj01RE@FDYCew$l@jzVunDsWJXp{Jo}Z(?F;U#X$g z7;@CGxH`(>?kJ1PqlhK~Ke%?I<3HV%X#i%7Eslji;uY1;VxZtNE;X_Pd>hBOYB;DJ4O zvIKXmfk7p{V+QTKVVhYE>>-O7WSK#KNHOTc4!@9vt!rhFGH7@*$l;bkhimT~uDx^q z+Pf0(C8LVj&PP9NHyeUVWD%WKWjbA*qSG^fW;*F+wjQgHgFJ(4UATT(EL;c(XFFjQ zS2G~h)5+l`LWi3O9d06|!Ah`SQp+cM7ko0tYzAJDMcn!lbL*lMw;tP#xg{H!y62*M zE+dFr4!3+d+zsh)Hze^)co6U`tIUtp>{CB6W+F&N77=YzCfb=PqTN0}6HPbtbr2hg zgIGv3hf7$eymU;Y6HbJr^Q^mjjwZ7=7)KVdZU<)FaVgfld_1#G_H1={)8TD|4sRjr z94=LK^p-1bP%XwiAxKQYP3CDf2ZMfO5e4^R3LcuG;173Z3d%0AQcx+V6vO}+P|)EP zV~1O99X(C#Z?(lC0bV^UDOn_<%kDJV90M+rMST1r^KqXPAA8qiKFZ#-@=^Jye4IKT z9X)c6i-sL88m5d@$WwCD=c}td+Gb7%G07rw9?j(3DMilLKF8#g9dRY6l2ge!?c{X0 z1>E88XNS9=iKs%PfT*6S*R^+;3&2vch^ePBQ@@pB>J|TGrpi9NGF6$XOr1WaI$YB3 za2vP7ZQR6KAyvp)PunXG8f&fuWyvDi{)%b)dUbp5)#@2kemzY|H zo$72f%zncGHl(e?CHf8*q&r-Y?)(MmCCN$}R~Eol%IY{6a4Yyr7IFA@%;7as9DeB@ z=CGF6=vInurIcGK1I}TGJL@xtOY5D#v>u0_<0UB~^mWHInfpLyvWU>PFrlj{Lf>^5 z6IzNKG?&&~T65{oo=evc$~A;`ct^nD9RY`T1dw#&R)dJ#SutFnv0{sP9Ly$**nKav z`*SIFU%xoBTS_aG-O6rd_e^BB!z&>UPdhj~?LbTyCYE`5=Bd8iwyowx(3~uy`eRJ> z1yWQ$?XOIADdruedn8w&C#D2GL$v8PeYK{+8cW%zNNGS;YGnnD=8+ zyr1-K=Dk#rDDRc`%KJgUdxuAM9Ns^1Sn70m`Gj;pSX$~pk>~IDuMX2V3Iae@cjsi& zyDS3q-!v8~(O}w&zl1*~oHpJZ@)(PN)UBups0gSC3_=lbc!b8`9Dl(pz}*lHvPd|VX5qLeB^(_=Fv$po!{L5ahsz@!F2i&Fd9#w{rH~X;l^^}N&8!6lA&XR`i&fvQEGfTD zNy_Q3u%t-Yl}d_Aib~3GmlTKFZX6x~#2>}f2k$5(^yR_3#+oglEo70te1-J|Z?-13 zV<)q|NU@jdi|UK&%ZSn!hl>gv&OkeysKhc0-fM}qCaWxjWTtt?MzafqhAa}BZ?o8( zni89DtifU<85$i7c(#4_d-J#^w`PWPFI9HSc6*9EW)uYX1NDx^h zL;JD}{U9YnAAN^qNGj=6hE#@BhE#_7ks*hQI)_pj=O2{CTY-@z`PAsJeJbVz=n+|@ zNr$l}?U~Y~$N0)|JhsNaubQNqq?)9f^#5p*!!EkRzL&$=G)>+_Kf=jdku*JX=ooW0 z#EC2tsN-3nwoeJv9lztGNvZ}_pj4n#pj4nfg+MthYD1|o$-~r+)JnvpMy(dcO4IfC z8*>SiiY!vAvskUZnNq7C?8j;)wTh}%s#dC2s#ephRt_^ihwREhvq-r_SVHB>DkNQ- zozrBlg=~>U(seOQ*9Ix+T4I|*y1MX$u+9Xv2wIDvhj9^fSiV?OYZTT3gxhoVi{#UAu~gYh}ui_eSq;=7=3{VdZ{a@Mzen|T0&MizdGoR25DYP8CiSZe}W6-ic|d#Wd9xPu_MRyGzMk^UGt+tB^XfNba6uxtk{?ca;}e z?xg5dyFUW-|;^|M}4s$i>vgm9synoUcyDJ&Ag54{lX z@t>3sJ~=xJp%nM35ULQW5ULOkN+HZGCdUqwV@J?Ws*0+Ls)~bN6?2Cvw8IqI`6;w0SeZ8BKszOlPjq14zA|>r(CA}e~qzj(GN-9<2s-&uuIgv zKks#z-5|DPk>GC4f_qX*a9{osC$&&|+z-mEXf!acP?{z}w_n~!TUhv55UQ5*K=HXM>_!>y5B4h_`-)yjHQCQTvW4LvA6|Dc~1;5^lr*0iwht{F2GDBGn$=KyDk!eONvadYCLczQjMk;;Va+b|HC`_%ke@x;X8wZSkVQSZ zm3x$@dbG>B+@n(JXGV{(&Q9w=eVJbkLKx%T`GDPo`r{|iHL|FC_i^`@N_B5l|7t`R zKr+obSSLHv6R@y*n1^8+hFKVL0t%$GBnIc#ckVDRp`TcE-mE7BiiKN+&0YH>6XB<&~0KFD+5I>x+@PLo9)e~~-hk?Qzk$8*PJG#xx0ulAwh z2?IR>pJ6SE03fXl08lXb&iTffCX7U6kr&?cUYPVr$_u~S%6kDaI2fQ_6aG3cbU`k= zvaar|tJrQUTO4>Kt1O08^FUm3J{#OYR_~IAMs~^DDVH3)EW1R~hCvIA&?Q~HSyx}y z-I;Y4!x;v(eu+c=Sr~U0EDYa}MgAGb{&_y-pZ$Kt{*hc|$oZ$+W!-t!UCp{N02d=A zEyT`xB;~BRPhn?ChBVZj)!mbI_h#LFSx;v%Lq%Cp!gG7R zFv|46Yh;n#1GhTZpmr9Sr0mA$AlYfpjUxh!(BA>3sKD{F4 z(_itd1W7grZAADK4`|~_TfAtCB|5w+NiG%#1THRA^WiaVW=lAiEOPJ!cJTQr2k$&3 zE0MH!Bsv(ccH+@ayxEDj^2piZjKJ9*&c8aTV!i`UlSSU%g1vo8%G;~V#}XmA_DJ_O z9wx-Qgm?lBFMyHf#WA7hz3W$fwcYFwx06M#--TU&RLb>#YiHL>datf8U5~fd@Uju! zHOh*2jk+-N?`M$oaky!TG3Ic@fGmo`ejJDIr{Zw3e}=6K3kxGy(L=!rDNqv!JlKNw zTJT&8o;wvebDBk-@lK1XQ0uq{ZIP_+A&#MFs zNQrc$cL5O)krs;dBoyf#X#!FtROwYfK)RxIkRly|gz`s_BEtWhv%{Rdci+a4`pV?T zJRpR7&)NCDZ)bOBXV^osdQ$mH$X6|GENbanl$*Aw!D>;A(4xY=Ma>!VmW*cLE$^;l zN7b9F;3~4nUFWmAHch!}*SpzWvSd?tO{KdmYKL3g?6$a|NUoES4PEDb_uB6p%x&-+ zS>(Gb*>`KEd^c`!_MI#g)pyg-cNRyhmJVR(91#bJg)?QS#hHa>p4T?OJOC$>MUK3c z9l2u4k!@TR4_lo&t4;Z=7HVcl!pM=9&V=c72B*TrP#Jb{s;B49FAOwK!lPu7SMO)9 zelO+K_a-tuWksy?oSyUyy=v*OpHBPzaX-$F;#ERH+{JfjGOxhBWRZ)XU>7fva`9Ds zhK{o-CGHF+Zs=l*vsH_%35#S2BDA=pBs4N|D7gLk8Wr;%d`%Yl`(^g`94UYAbv093 zR{u)rnL+8$-xirB7WoernGNK7aZTuZ@BHR3Tg(@5JXz%YiR}EQl=HthlASMG4(j}# zrXDCURKZviYa%nrX^)R@`fb+TGm)DL7>yq|)_$tyD~WNSqQq9+SP z6)Y?ctu2mREzSs`97we3CSoLvoQPE7g#iJw2qVK7Bd?|~GVW!@h-~O6MtUViEDky? z&fhFff(S3-&r(tlauQ(9UZue-4?K`XfLVwE^Hd5jJMzIYa=vt~>1}gOC@LXfERL-# zjz}za;R!tA+Yme+J%2&7g%JP;S%jYD7(EZB(6egBawGdzDmT4eZbI}}oD5iOFHh1f>-iO1Gz=ls(Q+lHD?eQlEyB#jcsfR*Lm$*GRNY zL`3KRw#jS)*pNlI8pXJ}I)$sHcVJw}9-ZQ<-{8t(g^uf4mVhudmD?lbt~YS^or?MWk-I zmvM#GRHx+(ol8;$%Nk5vPa?s~Hgb1RNpyJp2y- zdWbm~=pu{Yw>!h{s1*F#Utsvj4yeLU;TOWs0(=(NgDma`S${zY{}zRKl|`|I@!H}|;-03(MNQIGK~#X)tPs}uaKrFg zAd)PC>_rUO4O5U^V*!S&Y-}rJ6|z$eS&JL87FT2~?#L2+1zjP0J$^6Gnhe#tD(YR; zF5a(U{H~tD@7xzMeq}3N@vHcqruenEyFs8Z-q_RgW~o%3byaJ(Ju3>O;#BbsM1>wrpR5uLuzbh-FmYQG^4>dvWRs*W!9aPV%=NE^YkWX zwmQA(^wv+Ow~%!XMRFZ=r$hzSV%`&i#1wq+2Tf)S=tmY&aBrsIxD*8s`YBUTPKlL* zNg%i8B9}VeO`#Rv!#}N<6Tn5Xh>wRcANNi1aV2iri!^Nw zL^TlAK=hkHbdr2@)GVC3g`?0gBG!^T!E{-y5|6uOusH|BB#X#-Jd<;m6gfXy)=!qm zGFNgcIhCB>nw$<*z#VEoJJfzAq6(1$qWYEPGmi{4!G;W5>!ZnE1hrMkCFb{YT4DXlzR>k?I!e+dg~gjEV^Z+V9oW5er%RrjjGJ0`U$ z_NmOtWOB=tY{O1=^czmFA#ELs=sOfhcPNnVe1UY_k&IWc#Ni`wlyVRFOBQkX2IlZ8 zDGtALA9GmJ4Rl9Ica(HTY0^7N;R<$gIqXogK65Bq?|jjE+yw2O(0?}r&0j%gvWU=k zGof2kgnr~8CbT3uD4~_mO6ciKXoo%m4t)e1`Up_aE$j<|?jqz~5DgPw0JF&=c0bDO zULwWr+vj0+OKOF(TiLDbo`LLk=n~@4+QFf<12J8g7%<(huJ`|_VkUs*WD(V$XR6PW zqWaknGu0)bL#eJ*SE|n>sylRXap<<;&}@TfFRTn{?|Hw;PllLJz+j0kw1Nm`xtuQJ%SxABJ zx6|-3ivayzxBYewd4@$m@>Wy?R0LE6dY}k6GgtUfF7l@KoQ7l z!-B>-uLyhnE2Rjl{@yDBN(Si}xt@{h8Tm9jBdiMprY#XlmFgNq*C4tE>4j^M zFvtjn!=V(tLpg4T3dGL0H!Epg3Q4h4S?qK%S)?LutRjC*smPEC9DYawlrB|t zsiNUWUkE>HRm7njtwVWD=R3}!qH~n<=mV1_QPpe%$O%~_DL-IIxjrQ+=e*96B57AD zDJm%{DScg19I7Nblyk!mzw{CQmk~==SqjIo7mcbn+dx~$B7Iq(_2qXdeHnW&>x(3N zslKSbsJ`?oeQ_wQ<_*hq?* zij9hmicP;28;1%h4#gK7>LZZwh%SV}UB9a8xm(sDM+0{sXVZF^SY_ zhaWYVE1^_mky@R{YPDHPt&TsE!zsyC)FqxS@pOr&OT7QF#0$fzDz$Prk8{`>ci5C7 zAZ6!adFk zm!%YLVc*rN9bc#W{JPJt`~156oXUOvaOZ4NVOywh`EV>#EtaXaXn7l`QI*^+gJaD( zj;}KlA$4St-2H>)ZqAh4RbJtMPLf_Vpwoa(13C@pW@JEDC3m@X&^;_?EmeSRN6Z&5 zWwWe8+PBTxjplRc9$BP)@3Qtar?l@AU#F=Z5v7KD8tQ4Nr=i}=4)sFq%WWGp5UXCC z|KvFTAuSZ8D#O4+2zRawG6M&g!XhF3%nRY3U#5icrDhHSCAn9FKn(&l2-G03M+AYj zLYU(qJas1lUa19SNJk;<| z!^55&9)_xzJ1n6cme8(PLYE3whK;;}J^{D37J@vIMG`q1OXO=QiM-_(ERmAOtbwBj zjv6>>;Mn5>$50}3hn2R&O50J($#$H%!~rGra#`#+-i=FS%Rwv2BE4LY_44VIUOqgL z^-{8@RWDU9RWDU9d#zsPt`oZ1m1TKO4S3pdc#{;*Cr)ZGD?m8OBJo_7#q*(*cy7x# zaB(!G>v3I=>v~+*@Flx(3VY?^CjR?}03oxZI@q^9w z&{wiZV>e=ry)dP*zgm|yRGB}f_r*OaR0d~3$CQ$Yf!5} ztp>Fk)b_8SHWb{v6@@lhv8T|AJ%!eetWaRlM~QeVq`vE4S!Y%^rm$M|H631k_q{Wv zzVlaDeI@ZfqYpjyzIuZ|xUlF$NAic>l={%i_u&tf#Mz906;Hh41r26Pd^lP3;p^~+ zUzYmtJ5KW-j{MwjPU*h+^ziS+H{EtwquCkXLKc0~7W_@;roQReoWDuZHDqSOjELOe zZf3;dYe&C2z+eapi@tUz{@N2$Uwg;p{Izn>DPP-`wPzh!XV#S=-depfe<42W% zZt5NtLyAX5Zj(=K!asC_fMST>JG8~zgl8s;o_jn$_qwU){`7QyZjr%h_uR-ZK!O3% z3vfV2LoZ(u3_a6old}e!d+`Ef(M#OOFR^m!B~JBiDBJOc-)I+np5<9ZQRy8QD z!bv#z;Y$>Ue3&^h+Lxxrn!z^)?^L+uoX3Wkzu|4jqBpvS-zZPL(XJ!eEv3}Yh&RGM zJM9N`WqvmZ-b^%hUP-inL)*8?_Yq{G73!#WJhFs#B56Tp$> zlLU?gIG4w){leYGyo2{7i{AAGe%J4&-gVU*_+4dUn?4My0cYx65&j_h$q@at7FQF& z&2D_`UwqTfdh>6*G+FfG6Zpjkr(XQI-|&meY}#{PytON98=ke(6`8qWUH*ChEAai(+9fdZ?Wn0$Pp0yRnZAfS61{uYWyorOh zSurGJwL?-@&qMN2YDmT`!b2iqL$4T;woddI#I-f-+7y71rw;(bit!tydCZToA&bUm z79O8BQ{yxGA|4-!ReIg{v^y{=Yj4fkF##7dC8o?6O$C^qU98D0je#PI25UYZtmjgL zHOEOjSP~)i_Q7iJ$l5!z_O7gBc(Fo7q>&i8dk|fBVARN>kz0yK?$4=_`)F+*If={q z)W~(%tixp;d4^u-r0?>UAzcdqe)UAftcD>Zi-xhChw-k|Fy1-~595ptJ|}B_ubJjT zfe+QJhOJcq42KaN<JlSM zLCXV+mIuTS8MTrhg;U;LrqP@Vhmb{1If9+CXUZw#c44Q;QbwIJnNG3jLt)W}!lHgV zIY>sZbdaabyB7^G7r;AYk%vxU5B((Np+kqThh+7n9-0CVS=80HsIYHQ9h$r)qZxS1 zyX&1_4>DK5Rb-L7&S!URnsV2!x3arr$)@g_N_Sb*k+i6vW>GGPTqh$Ny3YIVl_AaM zHu#M!^4*o}yERk3J9H8Doh%g9chk^!7Dua=4q)hD6DOO6Gi9j7nZ>NO!@d>s0Gvn` zIr3I^FG#|LuyOMX*d-ohRU#uQ$0O5TxPI&5*{Usym~)-^?NC= zzB7U8DJx?2>WuQLrNe$Y?f1w1*j0*G2?cSlz@g_Wa4%Wp;wRX}i=LvQNnO;+oL+-uc@jL+%SWo-A_yM0S2t%K2X(%g&cA2X%f=a=t~%kwwam zMFNaPA_##&TvdXhkby&1X)uG}XtD?ipL-;X{!a=CSB+&P$R>v(p(i83B3H&DZ3I7D z0Sob44J=w8YBaOM>twaCs2|9%cs~V;lRFs}vbCbH=rypgNFK3Bl(RVYC*=?ymJsoi z$nM(?Gz$X)WD!P&F-BfZVPxF1j1k$;QH=CTj94T&SR83voGBAt#Ge6PJYcqKY%*F;Co^DV8%Y7v>j7hNN^Ws@YjKQ7;1S=3;PL2L_p@d*0^lHv(6bz) z=fM*s}>>Ll)s`6yxgZ6s{KEmT@I} zbc(BfgDZ=jS=3g*Oo)9O(k=l(sa;vc0%Xfz<8B9Fh%5rx#tgFGra(3U7b7e9eB?sw zMr{w-s0}ynYWHrcR_x)@EDXEDD5`cI48^MoWEOiumMeNz0U`jLN8F1Y^=1zMiY!9h z_KdjGQivN5ycUR){YaIy{wZrA;=*bMggyaBh&~U$x&PZ>4hFi&BKYmj@I!{1sc)b^ zP-mKP=c)L$!PyI$5w9@(WCv8?r|=8mXA#g@%(oVEqV*SqkX0Xrc*W#<&*KBkF@PFb zgvEmxiwC5z*l;ytQMO$bi;BfbVbKDK7T2*X?qiWm3UC6MEPy%Y=|Se#z#Ulx%%d5Y zyQP465kq#v6lB+!lOZb` z+X`8Q>{LV6;%cYG6Pit zNg?}*&luTqYM{tgWX~{UThu+UD12ajXm_NfXUmykZ zzi>2ypj{8I^zf?3w4;NvmyisU-D7^;TLzi`1Ek3!%)ic<|4s_?M{eh(9TC4`UNPTe zFmF+K#iAC7MKKV8ezvRIU5AwmdTOF|SOQ2Y5UrUfU}WDy(w>)EjPrztkP{R&S9auTGGzDD{pCDOO3 z3T07*3qP=P6HEuSY_Op&h z@)RP6UOI*76w-U9kdP-1Mb;eZc{$YcBF+e;L(X{8+&rq@I1q*`BF&;qn#WS4X+DZc zBS&pY8YNBdCyhf9Jg^5XOHgBt7$o?P8MG}fg^vV#$RY+=X3(Ef4Epp-W{@22DT9QZlND;;vq5kXZ**B8%wseWugRDLTD+8`DXS6qQa& zr#?<6hf0JFl?WXw5z=HO*e|K&gS`z8YBpPdS7Z^l)@E*9p5oT?d=nD~_qvmyI|;pd zCm~!RRRw#sQ&q?vSmB3DT9E2ubHzcP0uAj0WS#BG&zsS$9&3b?=_yLq}xoDC?AU z%DQh@=TIcqQ9Dso)GR0{1gWNAEB4?g-fhen(2p#l;NDEZaVZLp*_A0Mr^LE?)74u) zTD^r4W;pRxQP82PtwU8?M_qk=Ra<14#=9xR$JT}ha{{4 zaw4weRB|dgC!d@SRlpr;KReWZCZY!oNr;z_2h9R4e9 zm9|RTsi&<&5q*aO=?(?boiC7%Wa;i#u$Qz9Hur$PWD$pNU=FX6;_y3nFoz}GKsl@& zRt`^l4m;GW&m4-@J72UOHy^qu^sm1jV*Uy;lSPESn+e^TBJ?Apnb4BtpoCUJE1{<| zp&j}NIP?*4=p#Tux3Dh=x+`ti(^~OEV_pEW$s%??%Isbu#qQf@V|Ghwg|b`Ot?ZtG z>~`o9;?UZ`p|t}sU6>d!Ju9d_4o&hVfaYWo)t_go&y%A1nNNGFd=AGp>JTJ2twH!MR?$~ zikSs!Kvt^=jdflT_WD;!5k_9YiXhoCstBqGst7$@5gghCIkX9KXcI*7g3z~A2fu1K zqjiXx57Iyu$-_{VhqqGl@Y#_p50ay!@}Tmd^3a>)!J&(oL+2)keoZ75LghdzvPwG+ z5stjhn5Cc-WRX_P%UbdGlvdobA!~(X2B}u4R;X6=R;_U8^5)Qs$f2PN>4wlc)D15j zv!jevJ4Ayl5{@NVIR26njvvg!!XddxDjX^tDja=5I2^j?I5Zk@Xv5%qdo$EsjWd)e z{P_841I(&W5VA-`+E_*Ym{O7EH&{g^0ZLUwRYX;!FRF+`fq#dZ=FWGVL#f(GQG8(X z@W+G929Oi7NK$^ll5%}YQqKA2xJ1&fR8mw@R8so7q&QT&cBtNqAAad0{128s#g54W zn^w#=&=#^tU)E=R`CUq1#_q!*izIugzNo&azVs`7aVW{^P&?A0t{rW&h!zA|39u$t+Mi zrUdG5cd|f9)}RWM3X}?z3e>j{D2M9_P%13)kVHsoC1Mh()$k2k%#~0ovPiAYW3}2W zrB+9eWwnxAMO7b>zL$Zs*b6Ssg9|RO%EN*vGJ8-Ihhyz;ycjmI?}YP5VgIQZ8T3n z(8wZDyN^X}cuLg9EykiIiJ&TKDrzcfDr(bP)N+JrIbppQvsn&WNfznlf~=QMr}Xmv39OfrJ*|4FdZ~J;df99BGIyQO&8{rV zb85iTflbS#c;0aIV6y^*lPnU?Wm!BQN{Q!|d;=F-qWldio+_Ryo+_SwLp<{wS5k5; z+;c44bH8vGsVf)ml@$<%u7mTQQBYB`NJ+a`N&k>i(qX4@7%Exfs-&u-*fV??v`XRm}=n-SdJ5voR!9Ov?_eiuL_FpyZUvD=+6&}ON$6L8vxKKFkdbv!KEw{;DIZ+FjRh zuOJL+L&42kQD~zTdkU@CQ)unTNP$azl!&+3Qdstqido&5!fMsmba?gM_s*31&NnNo zuO$9wBv7}V%2 zEP95~{0w7L&v5=;{0wqF(ET&uz6i3TahJ|tr9+BKw^0H;&;z&EnG^9qWYGhS;Ro6$ z^*~)4^8?A&+W&qaoD}0I7nfUbw*~PFjhFo2!FW~Z={~x#-kggkBa5Ey1b(`mQ%|@0 z=lpcCD*e_^2YMsh9BGvq(kkgu<)dmJwH^g_F2iGzMUQ$8KkAmLN1gpI{HPMpP2Hn{ z20$Sk?^By_P~Qy#iXr~-jswh1cxJNbxySQ!ubX=AFVFVRjU<5S8YW`s%dZHA9@HkB ztcT5od+`Ef(M#OOFR^m!B~Ct?U!qi7_&m$A)~t%!A+2gqT!oWx@BtVMKAxRU`RI@j zGe<`I(nY5i{ATQz;*Gu@)@1&Mw;_w(=pKHfJoQGqtj%v!O8ty@BkZ%&eo$BDcZ2Z0 z@!NTay>@J~c@eKh7QNnI`Sli0z21uR^Xrw9&5V9Mti!Mj!zv6h0fAGVBnId9f2x>w z@SbGRyS~8h`rXvKj=YB7RVKD+rwo$-XXwiCT@n5u`pFRev=&zrf#z;}?45i>%V6_w zyfj(#;uHAA2d7^AxzqW@Wj5_OFW%afwGGeOY}V#7ER6^N@{|Dp3MT(y(IKXx&J-4n z!Y6(d_W3$B3OB6Jqabs5j~#_J$YopB)}FN$$8Au!Zjez7$w%i7HM3$!$ZChAuAYbF zqtuX$nTLl&!iHWkByF8pTUXXTJZmqmuYq9Q3{O^B5z|@SJ;uzBu_24bXBHlxH&f%Y z$N4-y607vO@o9Hid!DtoX6=}O{Y)t@6Y`4&s}t4mm&QPmMT0dT57u+3!J6Y&JXjJT z_4dJP@5tIa(MK1>x{DPmrsEiB1%WP*$;=L{BXv_><3v?s2>u3K*JXFYeBmfv^=nAc|iP-Q7idTIAzQ0hL}^~5VFWA zN3c`&OgUxTH?x2&Wz;E?=@g4T6c&9bEb6zDgJc9t2W6F&K$}x>dVc}DLl$}H6!y?h zQXV=K|96ExUsg}*p(*f?MTLEf3i}qtsH;XZ@RoPi+~^&06!^ES>(uD*^w)z z9N9h_J5rXXO2TPK!pM;pht!si({L(GPL*L7r+RvRxZptZBs@wMdG&tw>i1G!eg8f7 zs;r3Bt24^0mJa*rwBH~1HXk@F|A^P5u6|IbnEeA#kP=l3M%TjU;DBp+Gi7+7S$ z5E#T&B^U}BxPFVlW)K`r79rttkA%_xNg?5?0~iUi$)QN-$w;tBhO$U}!Vf<`h~H{p z@uOFUnAzcVvRYWw4`f)ppMu57!xGj45=BqX74MPT&nyT;Kc)G&gbjQ^1uUG1ek>wFi)ib zvm?oD+${ynBl$QN_m_0((XW>tVN^J|z_gG{3us!vr$DKoC`9Q< z92s;hUuP}`2+1Opp1~;naSElo|A^&Nj|}+c%k)k23$w$*JfbkkBuqM}J$J4;Z48f< zrs>*cico5CP1E9@ro}}~(p5oJfY_`;klp`Z_2yb2k}QJkMGV;uQ;=O_C_`2@wspqW z8GouFYjH!?;)<-r9a(~}peuy0$L~`QH5#gQRn)seF$j<1I@5&mSoPO1epgT7cTTRn z1e$1Ipn<`RC`AZ7muM^TT!`_Hw;O040anTIS2dXLU@&*4!2I5(UW`E=1+#*AMu6Gk zy1m8KWsA$<1Z+WG32ZOZpP=T{vj8qxgzSeI*~_Gm{dglITaS)3GL~KaX&Po&)Z(zX zkZ(~&!lI5uh-{0x2Ns17tgn24z$Dr|L)JM()5kyDY~BEh$s(9P!!TbU1@ni_V3^BU zg+|jFP0tjVTimv@s6c@q#nmYU;wjAkJ%CZn41g<3^Dk~!dng?m~Ee+lxHvHGKVed~&gZ`9a(7&5JgK!d}(~eF%eP-IJ zVvs|XLWi<<4rTA0FMC(wy<}9e+F4?WidhF#B8%wseWugRDLTD;KhsH$6m>$<38{}w zNVRlws6^;ciO``EAtoyvJMvUgQp*Q>XQ2Jf7T^_G#I3cNTbHM}_1vz^Ejh?kZYj6= z5w{$wd^*&Ibf^tUJQE%SJo7}evkWymfn;P6(KctI{U$}Ud*@)H$?2&QO^Mb|iRMs* z^_xz|L^|O_NV=@D3eJ#@zRsA@U>sS*x}P%ZPD-)v?Gu=Fa%QWnQ`RZ#z`ic@n?;C; z3aZ7tCj^Npc;bEaW(?>@7Ey3-rr@{~1rOedDJZAJNg%i8 zBIPbx@)c9!LZ3F86Tn5Xh>wRcANNi1v27&tQO>57kIF~or;>B>$>~r9+@bcfL+xiGst_q4 zDz7YYL1R2vN)|EoY-Z|KDW?AZ3udaOmgqgCnR^duvU7F6^(pzJ4kf~k({MkhYGqj) z)`rt)$W(_S?GDwr9jb8?YlT!HYdvjGSz)lb5tJp1XnQHscD)pBJGd4%Qu+00UXSMc z$I*OvmN?yL>rh1Bp;DtmrACKJjYMN1SxMs}NPg&%q2?a&mn`D&4b0(HQXGE!AwTuv zCW3NUIjkK1<|t*Vr`}o)JJhVt9E#REU$nktTtMg_;FRWowx26dF z&|yqyNpjFtudaG^)jMUY-Y}M)B%vMp2src+aOfj|*&3JnqW$KqLhL?a*GBUKm`xV3 z`%z~15-E1yx&X6VQY)0*%5G)%3}m-Mmk@{64i2pyi0Q(_avH}4cB1+Ll|g0#XigSU z{duPPJSnQ5`6yFe5;~OXN_D0BOrp9&7Z-wd9S=z-uD3BJ2csG=zrp{P2$ifnRGx{TIxVC`MDuNr9DAsS?na4gBf@t2fve1A?B4#`DQ;ZWgF;phv(;n0Q8p`n>WS19M( zo1vg&l*CXhRi0R}*{li$A&XR`jaB52DHWOPqe4YGQG-uY4mIUaQx0cL%3+wH6y`95 z2b;AA994Up;Z8=lR0+e6S`~3=FOvdU^W%9*%Zok5y+ zVUgHu&0=#_N^Eu+!NH8Am}xMh!Hfno{VJFV#m1o)v_sKahk}|UJfaJs@Ob69azvxq zAId`(DbLSXd5%jdPy1c0Jd)I=%A?Ao%A?8y<#DJi>d?N_p_ZB}x|epM|3kFgDgvN= zar5N}NDx^hL;JA|9h#D%7GLGC6Rjn5wWX^qU2XOE)m9h))yj}VRVs%XH?C+X8)!xl z92J_h`9aO*6zCCIq)CUfChe8dq=~~gG?M&14UIH3($Gjlqi#Z@P?H?$d^nT?aJVQ; zi#O4aK%aQ+Ht%3_KE#PE5~!0|pmt0N)Squ-fs(926(|)b6(|*`Zy`_)*P5YJSmdGa zGO3k_Nu*Y%T{+ZT38f;7)apD|tIbksb;Kd8R+6izYNcwWYNcv5rE2AHv&!MhghTo> zDVGRKs9afPbsWzfwq(7z6|zMZN!JxDU2CPJYwk^0x+FVNrAwttrAwu2T1b~e(sho^ z=p3m`dC{G`6a8T0U3Sv3J>PCL_d~_Q)LnMnW!GJH-DNM+ z3?{Y99`0yWEf~WPtX9WzWG&=4FU*U6@tvjU1mRf`oE>%PAoBzSjVuzi`&iV5r$p^n zOS7m+BB%yu8k}ixroq|t2+l%L%dxYcW4|oN$`k8PbQ_O?Hk1XWua2%iYV14lxrUb!3s; z{e$Ih&Xn9Wy}@!PNv|q*Dt9V(Dt9wY?sDs(dsxm|ssP)Gf9fXE%PMOi)cf;=#(WOl zBa5`}UDm$ll=l5=OV&O~vsLX=?NjYj?duWRm)kaIAXdE=t6u9@y`ofQ7+4758GAOE zfdfookr007g>cU=Q$qN{5Eeqc5TN~GwLdHp9Y8q;%1z%Ubu+zUO!E5mZ6Kq zSFdW#8wP>3LYP}DjvW@qj@o8-LMGz?u~5b5*BfYtK^4hrRdK)oR>coes`%LTtcsFS ztl^=ChZ-KLD)z|mFjU3dVF~T9gm!)jEeckKjW{f!74+Dzn#@9wN3uvFXJd(cEhUk+ zY{L>MdCV%2Dv>IYDv`ZPB6Ek8w!=!>QOn6r1pjeB$$FVR+G3W2R+2?}xghK1(BO!{QatnHhnf{2oMe%BF3aNiP)a;E z5 zuT<5nkkzN3Z8RH0QpqA&U4v!y_bFNZ(?Kk&lAEppss^YUplX2HHv`mAR`VR&06EU( zTZ>%)oXmH#3s7k6YHtoQ+e2T;B8}aMHTJ@k#vbi^Yj)y*OG8%;T{U#o(6!%$uA#=} zt=KGR#b!Y(^)e3RFbHXJWJP-gNByZ{c8A!KMS{B>3-0MD!JROQgIY<$*PvE|S`BJ7 zsO?`tZ78^TD++D2Vo#wJdkU=`#SKklTh&Iqh59amgn`wKDXdm~O@~+CeeX=E?|g$< zeI@ZfBS|0i)?l+PK9nr_(2@M1H>E!GviDAHEKM_+_aN zzvDRmaLGCQ=EJN0Uil$jw(}2!GvFy*mSMcg|2r73;^`j2<(6~tWMt9Poxo4GbL#2V`ih@UR;B-YxR6zj zyEjM$!htqUoUvz!H7!=18Hi4gDj!w*sKNftfoIsB+wrXKaXkMpBWhxIKz zwtQ^x*d74C-+z#~3C~OxJ@MN$@WvcO|74afyJFzECS z@)hAb_?Nig{U&oSUVtomi5vMPR!+Ud$z%8>O0|X0vpj3fs;C{(YEn2!y+m=yhnXXz zed(gHCjL2QW`X=oA8s~(!`qNWZ*&j8QJ#9ET~_5cDy4o#yb<=YBYW!gH4&IY2 zde;~DUB8=p*AZ9pyUN5ieHc~)&d`&08UvbNz_o6Xu>hP@~PfIMXYfXbSPc3uP*8|u78 zqwt9zg?+wGjlvDQ4hN~GbcU^(!zNRkreqY_AeU`fTYJ`49JiHE9E?a-49TGf*O^%{ zBxJQiQdiGI@=10vI#hNP`CYwOC|hiC1@bXGonI6QuQ#@$zM=EvBOMdLFI zkI$Q_@!98M9v_KSdX8e$5GRC8I6mzzYtOUx)~p>9a4}OtgJPX2!1U5Gjb>>K6j?M_ z^YLIkml~{Lr}1D(gw)#ytGy#@@66h}vX0@!3Ki2-V&vApcYx`@sF6h@w-k@upHn0E z;rcvs5|{Oq*$jCZAm@n-ITg*c+O zs#y(Ns{j}dBRa~XpFG+tN=}%*d z2HI$8i;lL~qC;0n8e$oMV2BIV{Mi~6vn>XfEE?bqcz`cS4e(A|GxsIz?X%nuL0dDx z=-P>)epCv4yZn!;Q7idTSqp2^=kGS= zR5*kza>^0xls!{U8MB*r3YNm^lyB!0i#`+?y{&IZc#hjqOvZzPDVC#o%h{bpEjD? z;5V|!cUQ9S)=c^CFg}*VcA$#kl#5|Vwa|AKN2`{$dg)*jC!2*cWvIoOg=RjB`cDtQ ziDZ!@Z)HcWm~x~W$|RKKsgiISk}z_l#UZt&<20PR5<_L!#i^d2iz9*fNqCej^6LHU z)$gUe`cIs`PymS?YNh84pl9e+OULPSo{j_cDi@R<_AcH7z2skkd&wdfKfx|uB<14E z`3xN=5bENY=wgclCyT5Ji?j)Hx40v4xA*thm0Qev@HJWF@0Z!%bEN#e`xQ)SS^cX} z&nTe|DINOTB2C8PjL{;CgM2Tp34QOK|KA4&n=jyavdH-p+4)%gnED3#19heu$H>L6 z4bEQ3d^(PuFIx`k{GQ}|iyR({Bp!<_9*Z;{0)x1!1Va(8ZHvsqL2xt~{=zT%S!bG% zJz4j;N5bg;(0}X>30LjKNYJgJ-dDw#U?F}BV6iqfr;gajnAzcVG6F;hi~4~Ki}zEo zIB`V}3mmv>G|)q$fpB)H6$pz&KZ{I1{GilTOe08+vq&KWL`ol)5XmZpk(b}8GYbO( zWD!P&F-BfZVPwn;j1k$;(MhByPa;W-SmfVXb2aH9gnngO6MUD}HM|>NChvfzpB1QllWD$CnWAr?jLeDA- zGJ0g+O3~A&&|{HnWsxalk=kOBdqHRtUzcbqX0GikgUk@HjBJuYO7FVy%!umyP~u)R!wwdP>+EPzW!n65(h!;I`@QpkSnOO|OlHBgyWnVzmP z9U|MJ?tw+&1M4dvK+L1vGn71}{j>A1m-_}#OcufX8HV`+DVX1P3d3B^Dir1l^O*v3 zi;@Ty6)5ndxH^U4r8~^83(PKlw~182wY1-6l^$Y#{s4V%Tuz54gd9Q*!$BI8(!w)Y^*7i4a$aI z!3K*eMHW?xEYkLf83Nss8GfR80`;)x1}(^HsnJkjYWy=rjZ<&*krI~LN)4q(ucL-V z4IGOiI2KiKK$8;20Zlwl1|HXJ76V7fBA(2}Jb5X_lWn-{Di-a^6Xi*-=7~eKD2IAp z4)wfF^1V=2-MI+iCFIcn2oxGuwc)-t?y6%xi>;p7sM z#-Ruv*n`#|sIf*25`4!D8u61MW+d1{7BR>&gZ`9a5OpG;DnQsH)@etlojx<|R58e* zN})s9JBPA&&X>I_@m?~jvL3d@E=P@pbwDMuh)&;UI^CS2)5~`JO0na?q7JsPG>;#gLMMT@2iT0Zm(eCBDfjC3em{4QF{t**~lUPVJ zha#-sw3a8*2`56*dDgji2b$4f99hJ=pEB!CO0n+sV|aR#Gh3bBbb9Nj(_6?oM_p&B zWgjZ27W1AEB&Ohn^EaC@pdVR8!M&M+<5Cp-#p!H^oOgg83+C)5=HXqw;ame00<-oVtah z&@e@;LY|VFK3;wIu_5Lh5R)t-=kZL=T~g%yU@1>dBw6VyQCEq&O8jP(ILXN}B&S0a zaEBtl4n=-J)RIU6QM1bWIFy)mmO3*YEG3JWdNwn4s}xgj_?nq2=kPj>>NKj;=;TkM zAyXZyaXVDwcBsZptQAs)to5{AZ;=LbBPdH2(e_fN?RqKNuJ{zwRu2D_wn|&2?bOrO zp@_aifpmug>CP8O#}$Ei1xr)!8;1`t_kh1-5r=PJ4zH5p@Y{DYhc&$hb+V9Uj`fIg zSUEiHIqXogK65Bq?|jjE-1q68&<9{6QQe`7 zi$k{!hh`f@dtqfrd(Zo)=bpuU0?v~~ynmZ{KP1Kby*FUqOBRXpUU{#)?*Y7bXtLwb z|HR=e(V^=->431b)Pc$d*p^!c^(5;XApm4ln6LQhYfd(N%pyR)*KN(o1D|9Okh~QY z0TlrifgUIVt_|yfHmnEGIlWK>p=^yJTzhMynFVSGkZc)M z1XTo8gr2Sl4sC)Q+5|bY38HvG=v%5oRs;->EH%i?2WcRS;;YW_!fpR>V=&w;s9iNw1q`7)ICK$n=-lMcxoM*;pmHD; zUMo&~xY;ZPogj;}VqVsYzo)d~o~>CcBr`}u2@NGQl;};NM5q-GUEUm;X*o30BHa*L zhq~c~W1X`Gn|6o>StJ}wvT*z*B^>K4%EBSJNGcpE94Z`rK{y<`@HzB~a_AN1&?}0B zL}*?LN##e_+dOuFF{?sB$RZVKV-@*hN=1gf&nhAbP^u!TBB~;NQAHd&&N=j%b4ACw zE>sPSavrlnQl3KPr41k_WRax&fFYNm5qX5QkPbJZ8)(&?B-)lMZK1+AF0=6I)o5B!5peNi|6|Nj2$zXp%$K zJck-w4h4K@@h18aF5U`(I%09lM2fY^9Wrz1Yb*COMg^W2$4SW2$4*L&tKYe&^^HpBMe&yU-{!(lj5O z9r*lE^8^HqEE2W*Sk#86MD6g!S=1yER7FiiO+`&bZF-AZjC2@yF?AZh(60u-)4 zD;wb;aujyOUWU4nMGE&gD_oXRxCNJFg_HDCRX9~RRXA0+8L4nN&M|YG4&>NpE96ep zD3ZI|+Xk44kUFwR?*74YH)l%jnqFkNlcZObJC!??JC(bcCU-eD)vbHsQQ>4`Fh z(w7yXUgIZ&%;(TOvPk>hW$kNDY2Uv#WbKnQTh%_*KGia~8= zD@s*rVIhQTUQ#gwQ8k7v62i~C5bpV9N(f)T|5YKWmE>L(LKQ+4LKVUuDTKMj;@Dwv z?5J&apjxFU`tWe%sYA>#Xd79rDh?RHs`x=l6(75TRZ&ukRTWhgRTWhgd%P;<4ohf< zCA9NPXi>00A+yTHI4l{ne4SYc@<61L|IB> ziY`TjX~wm=dSQ-BcpO}HSQ73Y)~=w#z%i7_++n5du+nzaapSE{NHP&b=nh}jsDN*2lL8Z4{7Ps!>Z_h(s^+;o*yl~t8hmDRp4t9g!XfE?%Y zt$r6kBruTJtg;DCoPPPWG2265$sh&A@Yl*S&h7Hg~|#H+@t#;V4u#`Z6b&0Dcq z(2C81)}og&4zRl2btS?5=%og;JH(bO65Q=ra8FMO?u3ynxRQpif~$h7f~$hte+4&h zMWKyW>?yQjPocG=xS@&kfm#JF!rg;W32k*_3aeFL)8W;3-#b(4J6{v4uO$9wBW1O{ovPY;XS1GTlG(zltXwf57ubF zHl}00R5pdwe}9`g^EbQ=S@cHt@Ehf+H`-+czfmdmGvbZ#I zU0>jL{ch@ASH9N2D~5EkHGdj_GjwJ6t_Xh+{bY!KT8pcR64fKdd#It`TB!tb1At9?BlDc{x zl8;hDa_9m)Boa3CSYU)h($<-^b!F|tv-V;-gLHH=JbrwZ+@ZQ9N=Im-VTU>#$je z%R2H5z0gVDlpJSxSqS4-lNBhjwXn%hmue1m`b+n@lc#M2GFyYAip|$;*(tJ=QKw9%Q!M&WSoEQ=sNYTwk`XK&{}G0CU41T2Hx`SdTv;gxeBf#i`;cSyKB>wyLP#e-6cyl zb=Oq7%c6F;MeT5l%DUt_8QIWv-goDn(`;^o-^e20UCF*%Gv&L37G&SaLQ#D;4Si>e zV@-%A9cA4gxX*>y!l0{y<$Nks{k5>r= zao^f)sCfnMC5v4A1iN^Vl#74IXXw~v*JX0FOs-n1)vgi3a5M~d!r3!Y;E=eXi!Bbr zEwUyo(k6(|;*OHgIJVy$+ajObZOnV{HCg2Em)YNQr2IYl8c%5)si?nuh`%k;WGwO@ zEV57F`_eU`@4fSH+Ns`r0mqX?&Y#H6Z%R4;%U`kcWy?YJd}io*NcPbA7P&_j$ww9m zFcxV%1O{8O9iSHHDEePclYiLq{>vD=}h`PHB<3Ws!17coBaFc=3Q)&w!C_Bn3>b2aHAXrbQm3MGhZNC z$D`-r_KFz+aF9jlS&q^3UGl+q7QUaMB)eq_r9KTMiv%=_j4A8Wu94&y1PZIRb0DUShYtKZU%%4y)bUCjl9C?vP#xM#D&!ifW8tOA^JT0PQ)FqgMlux z2!6XW{EkY&uk$5_pX`9D;Hlt+@Uu7+wKyrWI5DyQf)FksL?K=wg}dxHz#IdpkwsWM zh_QG;3X6u@Tj6qpN})>Ow=AbkGAY0bWU>I}g~V#kQTrW{btjnAR@1xD~0VpUQuMG9VWNgk%v)&tR1PIEB*P zHer;?7PF#MQ94B^wOC(UEO{&fPeQC9D$rFA*#l6d>*0M#NGu#j4Uub?Z0ugC9`&kQzH>#C@CRh*^~it9`h%45}E!}wi2h2J^P zV9AxObj7dYcbekYLVPU{Y;ix?;({{aS#Fe7`J6tW-t zn2{}~28wJ&_6$R|Mco67!Uxt@J^<$9^`D3Nv)eV8H-KWY2 zCO~6~02xjfM(S}*L5f}$F@jO}T)FEauaD*)4$z05nmr^|0;v`;C z$f1|6qIDHL^H(P#<(6`*A92f}%BMqZNQc^x#53VRz%x&@Ly^(F6G%oD5p8oO+HX=syBGbD3!=&C zsS-_z)=!D%P=wW?T&F|1P9mLfA|#z>-4m}>%xExX#``MxPGZ9sY6c9Bln7TT;l8pyT$s(qn%}m`Y z#nfy5!%UTPcx9?GRhc?vOm!&I?of@}p&B=_R!9}H*31 z=VMG;Is8}JDs7dvQ%_rmBKi&m(j5wBA5re4$U@*_QJ}L_F3is0cJ!dr~m)} literal 0 HcmV?d00001 diff --git a/ci/test/l3fwd/l3fwd.sh b/ci/test/l3fwd/l3fwd.sh new file mode 100644 index 0000000000..04e836ad98 --- /dev/null +++ b/ci/test/l3fwd/l3fwd.sh @@ -0,0 +1,110 @@ +#!/bin/bash + +# Copyright (c) 2025 Marvell. +# SPDX-License-Identifier: Apache-2.0 +# https://spdx.org/licenses/Apache-2.0.html + +#set -e +set -euox pipefail + +OCTEONTESTPATH="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )/.." + +source $OCTEONTESTPATH/common/vpp/vpp.env +source $OCTEONTESTPATH/common/testpmd/pktgen.env +source $OCTEONTESTPATH/common/pcap/pcap.env + +NO_HP=${NO_HP:-} +HP=${HP:-8} + +function mount_hugetlbfs() { + # Mount hugetlbfs. + if ! mount | grep -q hugepages; then + mount -t hugetlbfs none /dev/hugepages/ + fi +} + +function setup_hp() { + if [[ -n $NO_HP ]]; then + echo "Skipping huge page setup" + return + fi + # Enable HP hugepages. + echo $HP > /proc/sys/vm/nr_hugepages +} + +function sig_handler() +{ + local status=$? + set +e + trap - ERR + trap - INT + trap - QUIT + trap - EXIT + if [[ $status -ne 0 ]]; then + vpp_stats_all l3fwd + echo $status + echo "$1 Handler" + fi + pktgen_quit + pktgen_cleanup + vpp_cleanup l3fwd + exit $status +} + +PKTGEN_PCAP="l3fwd.pcap" +PKTGEN_PORT="0002:01:00.1" +PKTGEN_COREMASK="0xf0" +L3FWD_PORT="0002:01:00.2" +L3FWD_MAINCORE="0x2" +L3FWD_WORKER_COREMASK="0x4" + + +trap "sig_handler ERR" ERR +trap "sig_handler INT" INT +trap "sig_handler QUIT" QUIT +trap "sig_handler EXIT" EXIT + +mount_hugetlbfs +setup_hp + +PCAP_CNT=$(pcap_packet_count $PKTGEN_PCAP) +PCAP_LEN=$(pcap_length $PKTGEN_PCAP) + +echo "Starting l3fwd with Port=$L3FWD_PORT, Worker_Coremask=$L3FWD_WORKER_COREMASK" +rm -rf /tmp/l3fwd +mkdir -p /tmp/l3fwd +cp l3fwd.exec /tmp/l3fwd/ +vpp_launch l3fwd +vpp_start l3fwd +echo "Starting pktgen with Port=$PKTGEN_PORT, Coremask=$PKTGEN_COREMASK, Pcap=$PKTGEN_PCAP" +pktgen_launch -c $PKTGEN_COREMASK -p $PKTGEN_PORT -i $PKTGEN_PCAP +echo "pktgen start" +pktgen_start +sleep 5 +vpp_port_down l3fwd eth0 + +vpp_stats_all l3fwd > /dev/null +pktgen_stats > /dev/null + +echo "-------------------- L3FWD LOGS ---------------------" +#vpp_log 'L3FWD' +echo "-------------------- PKTGEN LOGS --------------------" +pktgen_log + +VPP_RX_COUNT=$(vpp_rx_count l3fwd eth0) +VPP_TX_COUNT=$(vpp_tx_count l3fwd eth0) +VPP_RX_BYTES=$(vpp_rx_bytes l3fwd eth0) +VPP_TX_BYTES=$(vpp_tx_bytes l3fwd eth0) + +if [[ $VPP_RX_COUNT -ne $PCAP_CNT ]] || + [[ $VPP_TX_COUNT -ne $PCAP_CNT ]] || + [[ $VPP_RX_BYTES -ne $PCAP_LEN ]] || + [[ $VPP_TX_BYTES -ne $PCAP_LEN ]]; then + echo "FAILURE: Error in l3fwd" + exit 1 +fi + +echo "SUCCESS: l3fwd completed" + +pktgen_quit +pktgen_cleanup diff --git a/ci/test/test.list b/ci/test/test.list new file mode 100644 index 0000000000..08ad3d08dc --- /dev/null +++ b/ci/test/test.list @@ -0,0 +1 @@ +l3fwd# diff --git a/ci/test/test.sh b/ci/test/test.sh index e0c272446e..9494370431 100755 --- a/ci/test/test.sh +++ b/ci/test/test.sh @@ -15,11 +15,6 @@ set -euox pipefail -function target_board_init() { - echo "Setting up target board for running tests..." - $REMOTE "sudo TEST_DIR=${TEST_DIR} DPDK_DEVBIND=${REMOTE_BUILD_DIR}/ci/test/board/dpdk-devbind.py python3 ${REMOTE_BUILD_DIR}/ci/test/board/ci_runner.py -bv" -} - function install_packages() { echo "Enabling internet on target board...." $REMOTE 'echo "DNS=10.28.116.24 10.31.116.251 10.68.76.63" | sudo tee -a /etc/systemd/resolved.conf' @@ -31,38 +26,71 @@ function install_packages() { $REMOTE "sudo pip3 install --break-system-packages --no-input psutil syslog_rfc5424_parser parameterized noise" } -function sync_files() { - echo "Syncing build to target board..." - $REMOTE "rm -rf $REMOTE_DIR" - $REMOTE "mkdir -p $REMOTE_DIR" - # Sync build directory - rsync -e "$TARGET_SSH_CMD" -av $BUILD_DIR/* $TARGET_BOARD:$REMOTE_BUILD_DIR/ - # Sync deps build directory if required - rsync -e "$TARGET_SSH_CMD" -r $DEPS_DIR/* $TARGET_BOARD:$REMOTE_DIR/deps_build - # Sync dpdk-devbind.py - $TARGET_SSH_CMD $TARGET_BOARD "sudo $TARGET_SCP_CMD $DPDK_DEVBIND_LOCATION ${REMOTE_BUILD_DIR}/ci/test/board/" +function help() { + set +x + echo "" + echo "Usage:" + echo "$SCRIPT_NAME [ARGUMENTS]..." + echo "" + echo "Mandatory Arguments" + echo "===================" + echo "--build-root | -r : Build root directory" + echo "--test-env | -t : Test Environment" + echo "" + echo "Optional Arguments" + echo "===================" + echo "--run-dir | -d : Run directory [Default=Build Root]" + echo "--project-root | -p : VPP Project root [Default: PWD]" + echo "--help | -h : Print this help and exit" + set -x } -function run_tests() { - echo "Running tests using run_tests.py..." - $REMOTE "python3 ${REMOTE_BUILD_DIR}/test/run_tests.py -d ${TEST_DIR}" -} +SCRIPT_NAME="$(basename "$0")" +if ! OPTS=$(getopt \ + -o "r:d:t:p:h" \ + -l "build-root:,run-dir:,test-env:,project-root:,help" \ + -n "$SCRIPT_NAME" \ + -- "$@"); then + help + exit 1 +fi -PROJECT_ROOT=${PROJECT_ROOT:-$PWD} +BUILD_ROOT= +TEST_ENV_CONF= +PROJECT_ROOT="$PWD" TARGET_BOARD=${TARGET_BOARD:-root@127.0.0.1} TARGET_SSH_CMD=${TARGET_SSH_CMD:-"ssh"} -TARGET_SCP_CMD=${TARGET_SCP_CMD:-"scp"} REMOTE="$TARGET_SSH_CMD $TARGET_BOARD -n" -REMOTE_DIR=${REMOTE_DIR:-/tmp/vpp} -BUILD_DIR=${BUILD_DIR:-$PWD/build} -DEPS_DIR=${DEPS_DIR:-${PROJECT_ROOT}/deps-prefix} -REMOTE_BUILD_DIR=${REMOTE_DIR}/build -PLAT=${PLAT:-cn10k} -DPDK_DEVBIND_LOCATION=${DPDK_DEVBIND_LOCATION:-ci@10.28.36.188:/home/ci/vpp/perf_stage_bins/$PLAT/dpdk-devbind.py} -TEST_DIR=${REMOTE_BUILD_DIR}/src/plugins/dev_octeon/test/ +eval set -- "$OPTS" +unset OPTS +while [[ $# -gt 1 ]]; do + case $1 in + -r|--build-root) shift; BUILD_ROOT=$1;; + -d|--run-dir) shift; RUN_DIR=$1;; + -t|--test-env) shift; TEST_ENV_CONF=$(realpath $1);; + -p|--project-root) shift; PROJECT_ROOT=$1;; + -h|--help) help; exit 0;; + *) help; exit 1;; + esac + shift +done + +if [[ -z $BUILD_ROOT || -z $TEST_ENV_CONF ]]; then + echo "Build root directory and test env should be given !!" + help + exit 1 +fi + +export PROJECT_ROOT=$(realpath $PROJECT_ROOT) +mkdir -p $BUILD_ROOT +export BUILD_ROOT=$(realpath $BUILD_ROOT) +export BUILD_DIR=$BUILD_ROOT/build +export RUN_DIR=${RUN_DIR:-$BUILD_DIR} +mkdir -p $RUN_DIR + +source $TEST_ENV_CONF install_packages -sync_files -target_board_init -run_tests +# Run the tests +$TEST_RUN_CMD From caa18be84b3a8128874813da18e17a1b710e13a6 Mon Sep 17 00:00:00 2001 From: Monendra Singh Kushwaha Date: Wed, 16 Apr 2025 06:34:23 +0000 Subject: [PATCH 222/271] ci: add vpp log for test case Type: feature Change-Id: I383e7bf8d692c87a9357bfb98f0dd989609f5492 Signed-off-by: Monendra Singh Kushwaha Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/150891 Tested-by: sa_ip-toolkits-Jenkins Reviewed-by: Nithin Kumar Dabilpuram (cherry picked from commit 7d83fced8e2a37cb001909f324cbc93713d29739) Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/152188 --- ci/test/common/test_list_helper_funcs.sh | 37 ++++++++++++++++++++++++ ci/test/common/vpp/vpp.env | 21 ++++++++++---- ci/test/env/cn10k.env | 5 ++-- ci/test/l3fwd/l3fwd.sh | 5 ++-- 4 files changed, 56 insertions(+), 12 deletions(-) diff --git a/ci/test/common/test_list_helper_funcs.sh b/ci/test/common/test_list_helper_funcs.sh index cc06a38f95..a3f97ab735 100644 --- a/ci/test/common/test_list_helper_funcs.sh +++ b/ci/test/common/test_list_helper_funcs.sh @@ -38,6 +38,43 @@ function get_test_info() echo $info } +function get_test_exec_bin() +{ + echo "vpp" +} + +function get_test_dir() +{ + echo "ci/test/$1" +} + +function get_test_args() +{ + get_test_info $1 | awk -F'#' '{print $4}' +} + +function get_test_extra_args() +{ + local tst=$1 + local args= + + tst="${tst%% }" + IFS=$'\n' + for t in ${CMD_EXTRA_ARGS:-}; do + if [ "${t%,*}" == "$tst" ]; then + args=${t#*,} + break + fi + done + echo $args + IFS=' ' +} + +function get_test_env() +{ + get_test_info $1 | awk -F'#' '{print $5}' +} + function get_test_timeout() { local tmo=${DEFAULT_CMD_TIMEOUT:-5m} diff --git a/ci/test/common/vpp/vpp.env b/ci/test/common/vpp/vpp.env index 8429a99b09..f5471424d0 100644 --- a/ci/test/common/vpp/vpp.env +++ b/ci/test/common/vpp/vpp.env @@ -63,9 +63,7 @@ function vpp_launch() { local startup_conf=$1 - echo "$VPP -c $startup_conf.conf" - vpp_cleanup $1 - echo "$VPP -c $startup_conf.conf" + vpp_cleanup $1 $VPP -c $startup_conf.conf & >1.log sleep 2 } @@ -74,13 +72,10 @@ function vpp_start() { local startup_conf=$1 set +e -pidof vpp $VPPCTL -s /tmp/$startup_conf/cli.sock exec /tmp/$startup_conf/$startup_conf.exec sleep 2 -pidof vpp $VPPCTL -s /tmp/$startup_conf/cli.sock clear interfaces sleep 2 -pidof vpp set -e } @@ -149,3 +144,17 @@ function vpp_tx_bytes() $VPPCTL -s /tmp/$1/cli.sock show interface | grep "tx bytes" | awk '{print $3}' | tr -d '\r' set -e } + +function vpp_log() +{ + set +e + local test=$1 + $VPPCTL -s /tmp/$test/cli.sock show log > /tmp/$test/vpp.log + $VPPCTL -s /tmp/$test/cli.sock show hardware-interfaces >> /tmp/$test/vpp.log + $VPPCTL -s /tmp/$test/cli.sock show device counters >> /tmp/$test/vpp.log + $VPPCTL -s /tmp/$test/cli.sock show run >> /tmp/$test/vpp.log + $VPPCTL -s /tmp/$test/cli.sock show error >> /tmp/$test/vpp.log + $VPPCTL -s /tmp/$test/cli.sock show int >> /tmp/$test/vpp.log + cat /tmp/$test/vpp.log + set -e +} diff --git a/ci/test/env/cn10k.env b/ci/test/env/cn10k.env index 3244edbaf2..fcbc011f94 100644 --- a/ci/test/env/cn10k.env +++ b/ci/test/env/cn10k.env @@ -9,13 +9,13 @@ RUN_DIR=${RUN_DIR:-$BUILD_DIR} # Test run command TEST_RUN_CMD=$PROJECT_ROOT/ci/test/board/board_test_run.sh -# Skip syncing DPDK build directory to target +# Skip syncing build directory to target SKIP_SYNC=${SKIP_SYNC:-} # Skip setting up target. Useful when repeatedly running tests. SKIP_TARGET_SETUP=${SKIP_TARGET_SETUP:-} -# Reboot the target if DPDK tests fail +# Reboot the target if tests fail REBOOT_ON_FAIL=${REBOOT_ON_FAIL:-} # Platform @@ -72,7 +72,6 @@ FIXME_SKIP_TESTS=" " DEFAULT_SKIP_TESTS=" - l3_fwd ${FIXME_SKIP_TESTS} " diff --git a/ci/test/l3fwd/l3fwd.sh b/ci/test/l3fwd/l3fwd.sh index 04e836ad98..c011526f46 100644 --- a/ci/test/l3fwd/l3fwd.sh +++ b/ci/test/l3fwd/l3fwd.sh @@ -4,8 +4,7 @@ # SPDX-License-Identifier: Apache-2.0 # https://spdx.org/licenses/Apache-2.0.html -#set -e -set -euox pipefail +set -e OCTEONTESTPATH="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )/.." @@ -87,7 +86,7 @@ vpp_stats_all l3fwd > /dev/null pktgen_stats > /dev/null echo "-------------------- L3FWD LOGS ---------------------" -#vpp_log 'L3FWD' +vpp_log l3fwd echo "-------------------- PKTGEN LOGS --------------------" pktgen_log From 849572aee556ba20db8eb2d14c7bcf300b8698d3 Mon Sep 17 00:00:00 2001 From: Monendra Singh Kushwaha Date: Sun, 20 Apr 2025 12:30:06 +0000 Subject: [PATCH 223/271] ci: add inline ipsec test case Type: feature JIRA: https://essjira.marvell.com/browse/IPBUSW-64943 Change-Id: Id0fdd2893319697549c260bedfbb7759f0e48784 Signed-off-by: Monendra Singh Kushwaha Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/151143 Reviewed-by: Nithin Kumar Dabilpuram Tested-by: Nithin Kumar Dabilpuram (cherry picked from commit 4843104e99f2a082234ff067286ad5452b4422e6) Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/152189 Tested-by: sa_ip-sw-jenkins --- ci/test/common/vpp/vpp.env | 14 +- ci/test/inl_ipsec/inl_ipsec.conf | 66 +++++++ ci/test/inl_ipsec/inl_ipsec.exec | 21 ++ ci/test/inl_ipsec/inl_ipsec.sh | 325 +++++++++++++++++++++++++++++++ ci/test/test.list | 1 + 5 files changed, 426 insertions(+), 1 deletion(-) create mode 100644 ci/test/inl_ipsec/inl_ipsec.conf create mode 100644 ci/test/inl_ipsec/inl_ipsec.exec create mode 100755 ci/test/inl_ipsec/inl_ipsec.sh diff --git a/ci/test/common/vpp/vpp.env b/ci/test/common/vpp/vpp.env index f5471424d0..9a35a7e72a 100644 --- a/ci/test/common/vpp/vpp.env +++ b/ci/test/common/vpp/vpp.env @@ -79,12 +79,24 @@ function vpp_start() set -e } -function vpp_trace() +function vpp_add_trace() { + set +e local startup_conf=$1 local port=$2 $VPPCTL -s /tmp/$startup_conf/cli.sock trace add $2-rx 100 + set -e +} + +function vpp_show_trace() +{ + set +e + local startup_conf=$1 + + $VPPCTL -s /tmp/$startup_conf/cli.sock show trace > /tmp/$startup_conf/trace.log + cat /tmp/$startup_conf/trace.log + set -e } function vpp_port_down() diff --git a/ci/test/inl_ipsec/inl_ipsec.conf b/ci/test/inl_ipsec/inl_ipsec.conf new file mode 100644 index 0000000000..5a71174c31 --- /dev/null +++ b/ci/test/inl_ipsec/inl_ipsec.conf @@ -0,0 +1,66 @@ +unix { + log /tmp/inl_ipsec/vpp.log + cli-listen /tmp/inl_ipsec/cli.sock + #full-coredump + #interactive + nodaemon +} + +api-trace { + on +} + +logging { + default-syslog-log-level info +} + +cpu { + main-core 2 + corelist-workers 3 +} + +session +{ + event-queue-length 10 +} +#tcp { no-csum-offload } + +buffers { + ## Increase number of buffers allocated, needed only in scenarios with + ## large number of interfaces and worker threads. Value is per numa node. + ## Default is 16384 (8192 if running unprivileged) +# buffers-per-numa 5000 + + ## Size of buffer data area + ## Default is 2048 + default data-size 2048 +# naturally-aligned +# enable-nat-alignment +} + +devices { + dev pci/0002:20:00.1 + { + driver octeon + } + dev pci/0002:1d:00.0 + { + driver octeon + } + dev pci/0002:01:00.6 { + driver octeon + port 0 { + name eth0 + } + } + dev pci/0002:01:01.0 { + driver octeon + port 0 { + name eth1 + } + } +} +plugins { + plugin dpdk_plugin.so { disable } + plugin onp_plugin.so { disable } +} diff --git a/ci/test/inl_ipsec/inl_ipsec.exec b/ci/test/inl_ipsec/inl_ipsec.exec new file mode 100644 index 0000000000..0a279ffad4 --- /dev/null +++ b/ci/test/inl_ipsec/inl_ipsec.exec @@ -0,0 +1,21 @@ +set int ip address eth0 12.168.101.1/24 +set int state eth0 up + +set int ip address eth1 192.168.1.1/24 +set int state eth1 up + +set ipsec async mode on + +ipsec sa add 1 spi 1 esp crypto-key a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0 crypto-alg aes-cbc-128 integ-alg sha1-96 integ-key a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0 tunnel src 1.1.1.1 dst 1.1.1.2 inbound +ipsec sa add 11 spi 101 esp crypto-key a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0 crypto-alg aes-cbc-128 integ-alg sha1-96 integ-key a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0 tunnel src 1.1.1.2 dst 1.1.1.1 + +ipsec itf create +ipsec tunnel protect sa-out 11 sa-in 1 ipsec0 +set int state ipsec0 up + +ip route add 192.168.101.0/24 via ipsec0 +set ip neighbor eth0 1.1.1.1 00:16:3e:7e:94:9a + +ip route add 1.1.1.0/24 via eth0 +ip route add 192.168.1.0/24 via eth1 + diff --git a/ci/test/inl_ipsec/inl_ipsec.sh b/ci/test/inl_ipsec/inl_ipsec.sh new file mode 100755 index 0000000000..4632eadae1 --- /dev/null +++ b/ci/test/inl_ipsec/inl_ipsec.sh @@ -0,0 +1,325 @@ +#!/bin/bash +# Copyright (c) 2025 Marvell. +# SPDX-License-Identifier: Apache-2.0 +# https://spdx.org/licenses/Apache-2.0.html + +#set -e +set -euox pipefail + +OCTEONTESTPATH="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )/.." + +source $OCTEONTESTPATH/common/vpp/vpp.env + +PKTLOSS_ALLOWED_P=0 + +if [[ -d /sys/bus/pci/drivers/octeontx2-nicvf ]]; then + NICVF="octeontx2-nicvf" +else + NICVF="rvu_nicvf" +fi + +function sig_handler() +{ + local status=$? + set +e + trap - ERR + trap - INT + trap - QUIT + trap - EXIT + if [[ $status -ne 0 ]]; then + echo "$1 Handler" + awk ' { print FILENAME": " $0 } ' $APP_FULL_LOG + awk ' { print FILENAME": " $0 } ' $APP_LOG + awk ' { print FILENAME": " $0 } ' $APP_RESULT + awk ' { print FILENAME": " $0 } ' $PING_LOG + fi + vpp_log inl_ipsec + vpp_cleanup inl_ipsec + ip netns exec vm0 ip xfrm state + ip netns exec vm0 ip xfrm state deleteall + ip netns exec vm0 ip xfrm policy deleteall + cleanup_interfaces + ps -eo "pid,args" | grep tcpdump | awk ' { print $1 }' | xargs -I[] -n1 kill -9 [] 2 >/dev/null || true + exit $status +} + +# Display results for ping test +function print_result() +{ + local na + local passed + local failed + local partial + + if [[ "$1" == "0%" ]]; then + echo -e "\tPASS: no packet loss" + echo -e "\t case "$2"\tpacket-size "$3"Bytes\tNo packet loss ----- PASSED" >> $APP_RESULT + #passed=$((passed + 1)) + set +e + elif [[ "$1" == "100%" ]]; then + echo -e "\t$1"" ERROR: packets loss" + echo -e "\t case "$2"\tpacket-size "$3"Bytes\t""$1"" packets loss ----- FAILED" >> $APP_RESULT + #failed=$((failed + 1)) + elif [[ -z "$1" ]]; then + echo -e "\tERROR: Unable to capture Results" + echo -e "\tcase "$2"\tunable to capture Results ----- N/A" >> $APP_RESULT + #na=$((na + 1)) + else + echo -e "\t$1"" packets loss" + echo -e "\t case "$2"\tpacket-size "$3"Bytes\t""$1"" packets loss ----- PARTIAL PASSED" >> $APP_RESULT +# partial=$((partial + 1)) + fi +} + +# Start ping packets from interfaces for all test cases. +# This would need reconfiguration of interfaces. +function start_ping_test() +{ + local errp + + echo -e "ping_pkts :-------updated ip: 192.168.$Y.2" + while [ $Y -le $MAX_Y ]; do + if [[ $1 == "1" ]]; then + if [[ $Y -gt 4 && $Y -lt 8 ]]; then + ((++Y)) + ((++X)) + continue + fi + fi + if [[ $1 == "2" ]] || [[ $1 == "3" ]]; then + if [[ $Y -gt 1 && $Y -lt 8 ]]; then + ((++Y)) + ((++X)) + continue + fi + fi + reconfigure_interfaces + ip netns exec vm0 ip xfrm state list + ip netns exec vm0 ip xfrm policy list + ip netns exec vm0 tcpdump -nexi enP2p1s0v4 >/tmp/inl_ipsec/4 & + ip netns exec vm0 tcpdump -nexi enP2p1s0v4:1 >/tmp/inl_ipsec/4.1 & + ip netns exec vm2 tcpdump -nexi enP2p1s0v6 >/tmp/inl_ipsec/6 & + for pkt_size in $PKT_LIST + do + local itr=0 + echo -e "ping_pkts :-------" "$pkt_size" "$Y" + + while [ $itr -le $PING_RETRY ]; do + ip netns exec vm0 ping 192.168.$Y.2 \ + -i $PKT_GAP -c $PING_PKTS -s \ + $pkt_size $PING_ARGS | tee -a $PING_LOG + + RESULT=`tail -n 3 $PING_LOG | \ + grep -o "\w*\.\w*%\|\w*%"` + + print_result "$RESULT" "$Y" "$pkt_size" + errp=$(echo $RESULT | cut -c 1) + + # Break if success + check=$(echo "$errp <= $PKTLOSS_ALLOWED_P" | bc) + if [ $check -eq 1 ]; then break; fi + ((++itr)) + done + # Wait until the process is killed +# while (ps -ef | grep ping); do +# continue +# done + + if (( $(echo "$errp > $PKTLOSS_ALLOWED_P" | bc) )); then + echo -e "Test Failed as packets loss $RESULT > $PKTLOSS_ALLOWED_P%" + vpp_cleanup inl_ipsec + interfaces_cleanup + exit 1 + fi + done + interfaces_cleanup + ((++Y)) + ((++X)) + done +} + +function run_test() +{ + echo "Starting ping test" | tee $PING_LOG + start_ping_test $1 +} + +function run_inline_ipsec() +{ + X=101 + Y=1 + echo -e "" + echo -e "Inline protocol IPsec" + echo -e "---------------------" + rm -rf /tmp/inl_ipsec + mkdir -p /tmp/inl_ipsec + cp inl_ipsec.exec /tmp/inl_ipsec + vpp_launch inl_ipsec + vpp_start inl_ipsec + + sleep 2 + run_test 2 +} + +#configure vm0 +function configure_vm0() +{ + ip netns exec vm0 ip addr add 192.168.$X.2/24 dev $LBK1 + ip netns exec vm0 ip addr add 1.1.$Y.1/24 dev $LBK1:1 + ip netns exec vm0 ip link set $LBK1 up + ip netns exec vm0 ip link set $LBK1 address $VM0_MAC + ip netns exec vm0 ip route add 192.168.$Y.0/24 via 192.168.$X.1 + ip netns exec vm0 arp -s 192.168.$X.1 $VM0_MAC + ip netns exec vm0 arp -s 1.1.$Y.2 $VM0_MAC + ip netns exec vm0 ip xfrm state add src 1.1.$Y.1 dst 1.1.$Y.2 proto esp spi $Y reqid 0 mode tunnel ${CASE[$Y]} + ip netns exec vm0 ip xfrm state add src 1.1.$Y.2 dst 1.1.$Y.1 proto esp spi $X reqid 0 mode tunnel ${CASE[$Y]} + ip netns exec vm0 ip xfrm policy add src 192.168.$X.2 dst 192.168.$Y.2 dir out tmpl src 1.1.$Y.1 dst 1.1.$Y.2 proto esp spi $Y reqid 0 mode tunnel + ip netns exec vm0 ip xfrm policy add src 192.168.$Y.2 dst 192.168.$X.2 dir in tmpl src 1.1.$Y.2 dst 1.1.$Y.1 proto esp spi $X reqid 0 mode tunnel +} + +#configure vm2 +function configure_vm2() +{ + ip netns exec vm2 ip addr add 192.168.$Y.2/24 dev $LBK3 + ip netns exec vm2 ip link set $LBK3 up + ip netns exec vm2 ip link set lo up + ip netns exec vm2 ip link set $LBK3 address $VM2_MAC + ip netns exec vm2 ip route add 192.168.$X.0/24 via 192.168.$Y.1 + ip netns exec vm2 arp -s 192.168.$Y.1 $VM2_MAC +} + +# Configure interfaces +function setup_interfaces() +{ + echo -e "Create namespaces" + ip netns add vm0 + ip netns add vm2 + + echo -e "dev bind $LIF1 $LIF2 $LIF3 $LIF4" + $VFIO_DEVBIND -b $NICVF $LIF1 + #$VFIO_DEVBIND -b $NICVF $LIF2 + $VFIO_DEVBIND -b $NICVF $LIF3 + #$VFIO_DEVBIND -b $NICVF $LIF4 + + echo -e "Bind LBK devices required to act as LBK pairs b/w VPP and Linux" + $VFIO_DEVBIND -b vfio-pci $LIF2 + $VFIO_DEVBIND -b vfio-pci $LIF4 + + $VFIO_DEVBIND -b vfio-pci $INLINE_DEV + + LBK1=`ls /sys/bus/pci/devices/$LIF1/net/` + LBK3=`ls /sys/bus/pci/devices/$LIF3/net/` + + echo -e "Add devices in namespaces $LBK1 $LBK3" + ip link set dev $LBK1 netns vm0 + ip link set dev $LBK3 netns vm2 +} + +function interfaces_cleanup() +{ + echo -e "\ninterfaces_cleanup" + ip netns exec vm0 ip xfrm state + ip netns exec vm0 ip xfrm state deleteall + ip netns exec vm0 ip xfrm policy deleteall + ip netns exec vm0 arp -d 192.168.$X.1 + ip netns exec vm0 arp -d 1.1.$Y.2 + ip netns exec vm0 ip route del 192.168.$Y.0/24 + ip netns exec vm0 ip addr del 192.168.$X.2/24 dev $LBK1 + ip netns exec vm0 ip addr del 1.1.$Y.1/24 dev $LBK1:1 + ip netns exec vm0 ip link set $LBK1 down + + ip netns exec vm2 ip route del 192.168.$X.0/24 + ip netns exec vm2 arp -d 192.168.$Y.1 + ip netns exec vm2 ip addr del 192.168.$Y.2/24 dev $LBK3 + ip netns exec vm2 ip link set $LBK3 down + ip netns exec vm2 ip link set lo down +} + +function reconfigure_interfaces() +{ + echo -e "\nreconfigure_interfaces" + + # Configure vm0 + configure_vm0 + # Configure vm2 + configure_vm2 +} + +function cleanup_interfaces() +{ + ip netns del vm0 + ip netns del vm2 + + # Bind the LIF2 device back to nicvf + $VFIO_DEVBIND -b $NICVF $LIF2 + $VFIO_DEVBIND -b $NICVF $LIF4 +} + +function main() +{ + setup_interfaces + run_inline_ipsec +} + +trap "sig_handler ERR" ERR +trap "sig_handler INT" INT +trap "sig_handler QUIT" QUIT +trap "sig_handler EXIT" EXIT + +# script's starting point +CASE=( + "" + "enc cbc(aes) 0xa0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0 auth sha1 0xa0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0" + "enc cbc(aes) 0xa0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0 auth-trunc sha256 0xa0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0 128" +) + +PING_RETRY=1 +PING_PKTS=320 +PKT_LIST="64 380 1410" +PKT_GAP="0.001" +PING_ARGS="" + +#VM0_MAC and VM2_MAC are taken from hard coded destination MAC addresses of ipsec-secgw app +VM0_MAC=00:16:3e:7e:94:9a +VM2_MAC=00:16:3e:22:a1:d9 + +LIF1=0002:01:00.5 +LIF2=0002:01:00.6 +LIF3=0002:01:00.7 +LIF4=0002:01:01.0 + +CDEV_PF=$(lspci -d :a0fd | head -1 | awk '{ print $1 }') +CDEV_VF=$(lspci -d :a0fe | head -1 | awk '{ print $1 }') +if [ -z "$CDEV_PF" ] +then + CDEV_PF=$(lspci -d :a0f2 | head -1 | awk '{ print $1 }') + CDEV_VF=$(lspci -d :a0f3 | head -1 | awk '{ print $1 }') + if [ -z "$CDEV_PF" ] + then + echo "Error: CPTPF not found" + exit 1; + fi +fi + +INLINE_DEV=0002:1d:00.0 + +LBK1="" +LBK3="" + +APP_LOG=app.log +APP_FULL_LOG=app_full.log +APP_RESULT=app_result.log +PING_LOG=ping.log +VFIO_DEVBIND="$OCTEONTESTPATH/board/oxk-devbind-basic.sh" +if ! [[ -f $VFIO_DEVBIND ]] +then +VFIO_DEVBIND=$(which oxk-devbind-basic.sh) +fi + +rm -f $APP_LOG $APP_FULL_LOG $APP_RESULT $PING_LOG + +MAX_X=110 +MAX_Y=2 + +main +exit 0 diff --git a/ci/test/test.list b/ci/test/test.list index 08ad3d08dc..ecf95a0a46 100644 --- a/ci/test/test.list +++ b/ci/test/test.list @@ -1 +1,2 @@ l3fwd# +inl_ipsec# From d13995069a9abd52771a36a38f480c30b888cc57 Mon Sep 17 00:00:00 2001 From: Nithinsen Kaithakadan Date: Fri, 2 May 2025 12:17:34 +0530 Subject: [PATCH 224/271] octeon: set cpt min descriptor count to 1024 Set the minimum number of CPT descriptors to 1024. Type: fix Signed-off-by: Nithinsen Kaithakadan Change-Id: I1913613f3433fa2716647e26544c6bdebcbc7911 Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/152192 Tested-by: sa_ip-toolkits-Jenkins Reviewed-by: Monendra Singh Kushwaha (cherry picked from commit 90b171a00e2ed635113756361e0dc9465bcc1554) Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/152194 Tested-by: Monendra Singh Kushwaha --- src/plugins/dev_octeon/crypto.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plugins/dev_octeon/crypto.h b/src/plugins/dev_octeon/crypto.h index a436f99bbe..4b5e58b41a 100644 --- a/src/plugins/dev_octeon/crypto.h +++ b/src/plugins/dev_octeon/crypto.h @@ -13,7 +13,7 @@ #define OCT_CPT_LF_DEF_NB_DESC 16384 -#define OCT_CPT_LF_MIN_NB_DESC 8192 +#define OCT_CPT_LF_MIN_NB_DESC 1024 #define OCT_CPT_LF_MAX_NB_DESC 128000 #define OCT_MAX_CRYPTO_COUNTERS 3 From 25207eb122a5a9dfbc8c1489b340021a7b67dfe0 Mon Sep 17 00:00:00 2001 From: Monendra Singh Kushwaha Date: Sat, 3 May 2025 21:00:16 +0530 Subject: [PATCH 225/271] build: update roc version Type: feature Change-Id: I41bf66c98ca14f89194500ed2a08994cad46a58e Signed-off-by: Monendra Singh Kushwaha Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/152344 Tested-by: sa_ip-sw-jenkins --- build/external/packages/octeon-roc.mk | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build/external/packages/octeon-roc.mk b/build/external/packages/octeon-roc.mk index 001e50da26..2e1a029893 100644 --- a/build/external/packages/octeon-roc.mk +++ b/build/external/packages/octeon-roc.mk @@ -2,9 +2,9 @@ # SPDX-License-Identifier: Apache-2.0 # https://spdx.org/licenses/Apache-2.0.html -octeon-roc_version := octeon-roc-SDK12.25.04 +octeon-roc_version := octeon-roc-SDK1X.25.05 octeon-roc_tarball := $(octeon-roc_version).tar.gz -octeon-roc_tarball_md5sum := a678243b30b4feb2346d38fbd3afdf5f +octeon-roc_tarball_md5sum := 7ca465369902f9a6f5c1b78cff140331 octeon-roc_github := https://github.com/MarvellEmbeddedProcessors/marvell-octeon-roc octeon-roc_tarball_strip_dirs := 1 From 493ae7aae591c8aea086b796641b843bc2dddada Mon Sep 17 00:00:00 2001 From: Nawal Kishor Date: Thu, 1 May 2025 23:01:56 +0530 Subject: [PATCH 226/271] ci: remove dynamic installation of packages Removed the packages that were installed on target board on the fly, These apt packages are now part of initramfs. Type: fix Signed-off-by: Nawal Kishor Change-Id: Id3f065de0c80ace9e93dae7cbd204579eaa5d6c8 Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/152346 Tested-by: sa_ip-toolkits-Jenkins Reviewed-by: Ashwin Sekhar T K --- ci/test/test.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ci/test/test.sh b/ci/test/test.sh index 9494370431..4de38cc46f 100755 --- a/ci/test/test.sh +++ b/ci/test/test.sh @@ -22,8 +22,7 @@ function install_packages() { sleep 30 echo "Installing essential packages..." $REMOTE "sudo apt-get update" - $REMOTE "sudo apt-get install -y python3-venv python3-pip" - $REMOTE "sudo pip3 install --break-system-packages --no-input psutil syslog_rfc5424_parser parameterized noise" + $REMOTE "sudo pip3 install --break-system-packages --no-input syslog_rfc5424_parser noise" } function help() { From 9636544c25cad9d5a23634eccd4f0aef9ea781bd Mon Sep 17 00:00:00 2001 From: Monendra Singh Kushwaha Date: Mon, 5 May 2025 20:33:45 +0530 Subject: [PATCH 227/271] octeon: configure CPT result offset This patch configures CPT result offset relative to wqe address. Type: feature Signed-off-by: Monendra Singh Kushwaha Change-Id: Id63b7032f3a0df311264efd0b67348c7eec1f267 Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/152436 Tested-by: sa_ip-sw-jenkins (cherry picked from commit 5622c7deb77150ee6df540976d9fb17a44546e09) Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/152442 --- src/plugins/dev_octeon/ipsec.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/plugins/dev_octeon/ipsec.c b/src/plugins/dev_octeon/ipsec.c index 73edf831ed..1b4ffbc7dc 100644 --- a/src/plugins/dev_octeon/ipsec.c +++ b/src/plugins/dev_octeon/ipsec.c @@ -1021,6 +1021,7 @@ oct_early_init_inline_ipsec (vlib_main_t *vm, vnet_dev_t *dev) inl_dev_main->inl_dev->wqe_skip = STRUCT_OFFSET_OF (vlib_buffer_t, pre_data) / ROC_ALIGN; inl_dev_main->inl_dev->nb_meta_bufs = bp->n_buffers; + inl_dev_main->inl_dev->res_addr_offset = -1; if ((rrv = roc_nix_inl_dev_init (inl_dev_main->inl_dev)) < 0) { From a8fc379f9f8268afb66e13be7e8649d5059f45f5 Mon Sep 17 00:00:00 2001 From: Nagendra T P Date: Thu, 5 Jun 2025 15:41:19 +0530 Subject: [PATCH 228/271] doc: bump dao version number Start a new release cycle 25.09.0. Signed-off-by: Nagendra T P --- DEP_PKG_VERSION | 2 +- MRVL_VERSION | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/DEP_PKG_VERSION b/DEP_PKG_VERSION index c7ef5229b6..f450043529 100644 --- a/DEP_PKG_VERSION +++ b/DEP_PKG_VERSION @@ -1,2 +1,2 @@ CPT_PKG_VERSION=24.09.0 -OCTEP_PKG_VERSION=25.04.0 \ No newline at end of file +OCTEP_PKG_VERSION=25.05.0 \ No newline at end of file diff --git a/MRVL_VERSION b/MRVL_VERSION index 39359a17f9..cf7b6a32c3 100644 --- a/MRVL_VERSION +++ b/MRVL_VERSION @@ -1 +1 @@ -25.05.0 \ No newline at end of file +25.09.0 \ No newline at end of file From 36e961c854a5c47faf85c4f391e7b2153b3cedbe Mon Sep 17 00:00:00 2001 From: Nagendra T P Date: Thu, 17 Jul 2025 09:29:07 +0530 Subject: [PATCH 229/271] doc: bump dpdk version bump dpdk version to 25.07.0 Signed-off-by: Nagendra T P --- DPDK_VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DPDK_VERSION b/DPDK_VERSION index 9d5efc6763..6ed68f9774 100644 --- a/DPDK_VERSION +++ b/DPDK_VERSION @@ -1,2 +1,2 @@ BASE_VERSION=24.11.0 -RELEASE_VERSION=25.03.0 \ No newline at end of file +RELEASE_VERSION=25.07.0 \ No newline at end of file From c78a62f2fe019a2b3351db1990978da604d20f91 Mon Sep 17 00:00:00 2001 From: Monendra Singh Kushwaha Date: Thu, 15 May 2025 16:17:02 +0530 Subject: [PATCH 230/271] dev: fix default number of rx queues Type: fix Change-Id: I54b79b4a639ac07f901c98641ff2857813347d65 Signed-off-by: Monendra Singh Kushwaha Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/154767 Tested-by: sa_ip-toolkits-Jenkins (cherry picked from commit 20a919d2d58221ccb6d661baa2132b4a3918e181) Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/154818 --- src/vnet/dev/api.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/vnet/dev/api.c b/src/vnet/dev/api.c index 114b63d666..f891b5df80 100644 --- a/src/vnet/dev/api.c +++ b/src/vnet/dev/api.c @@ -205,7 +205,7 @@ vnet_dev_api_create_port_if (vlib_main_t *vm, port->intf.num_rx_queues = args->num_rx_queues; } else - port->intf.num_rx_queues = clib_min (port->attr.max_tx_queues, 1); + port->intf.num_rx_queues = 1; if (args->num_tx_queues) { From 8ec405a1e02d6a089138cdde47fba2a903e02507 Mon Sep 17 00:00:00 2001 From: Monendra Singh Kushwaha Date: Wed, 14 May 2025 17:26:16 +0530 Subject: [PATCH 231/271] dev: add support to configure driver arguments This patch adds support to configure driver specific global arguments/options which are global for driver or not specific to interface/port. Driver specific arguments can be configures via startup config devices { driver { args 'ARG1= ARG2=' } .. } Type: feature Change-Id: I6186531abda386aa02e4dc8784b4a5a02665aa20 Signed-off-by: Monendra Singh Kushwaha Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/154768 Tested-by: sa_ip-toolkits-Jenkins (cherry picked from commit e902f56c7ae6b71ff1203efc8bb65dddb22b326b) Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/154819 --- src/vnet/dev/config.c | 68 +++++++++++++++++++++++++++++++++++++++++++ src/vnet/dev/dev.h | 7 ++++- 2 files changed, 74 insertions(+), 1 deletion(-) diff --git a/src/vnet/dev/config.c b/src/vnet/dev/config.c index 8883e727ac..1e428a544c 100644 --- a/src/vnet/dev/config.c +++ b/src/vnet/dev/config.c @@ -53,6 +53,66 @@ vnet_dev_config_one_interface (vlib_main_t *vm, unformat_input_t *input, } return err; } + +static clib_error_t * +vnet_dev_config_driver_args (vlib_main_t *vm, unformat_input_t *input, + char *driver_name) +{ + vnet_dev_main_t *dm = &vnet_dev_main; + clib_error_t *err = 0; + u8 *args; + + while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) + { + if (unformat (input, "args %U", unformat_single_quoted_string, &args)) + ; + else + { + err = clib_error_return (0, "unknown input '%U'", + format_unformat_error, input); + break; + } + } + + if (err == 0) + { + vnet_dev_driver_t *driver; + vnet_dev_rv_t rv = VNET_DEV_OK; + + vec_foreach (driver, dm->drivers) + { + if (driver_name[0] && + strcmp (driver_name, driver->registration->name)) + continue; + if (driver->registration->drv_args) + { + for (vnet_dev_arg_t *a = driver->registration->drv_args; + a->type != VNET_DEV_ARG_END; a++) + vec_add1 (driver->args, *a); + + if (args) + { + rv = vnet_dev_arg_parse (vm, NULL, driver->args, args); + if (rv != VNET_DEV_OK) + goto done; + + if (driver->ops.config_args) + rv = driver->ops.config_args (vm, driver); + break; + } + } + } + done: + vec_free (args); + + if (rv != VNET_DEV_OK) + err = clib_error_return (0, "error: %U for driver '%s'", + format_vnet_dev_rv, rv, driver_name); + } + + return err; +} + static clib_error_t * vnet_dev_config_one_device (vlib_main_t *vm, unformat_input_t *input, char *device_id) @@ -128,6 +188,7 @@ dev_config_process_node_fn (vlib_main_t *vm, vlib_node_runtime_t *rt, vlib_frame_t *f) { vnet_dev_main_t *dm = &vnet_dev_main; + vnet_dev_driver_name_t driver_name; unformat_input_t input; clib_error_t *err = 0; @@ -156,6 +217,13 @@ dev_config_process_node_fn (vlib_main_t *vm, vlib_node_runtime_t *rt, err = vnet_dev_config_one_device (vm, &no_input, device_id); unformat_free (&no_input); } + else if (unformat (&input, "driver %U %U", unformat_c_string_array, + driver_name, sizeof (driver_name), + unformat_vlib_cli_sub_input, &sub_input)) + { + err = vnet_dev_config_driver_args (vm, &sub_input, driver_name); + unformat_free (&sub_input); + } else err = clib_error_return (0, "unknown input '%U'", format_unformat_error, &input); diff --git a/src/vnet/dev/dev.h b/src/vnet/dev/dev.h index 1885913c39..50d752b996 100644 --- a/src/vnet/dev/dev.h +++ b/src/vnet/dev/dev.h @@ -80,6 +80,7 @@ typedef union typedef struct vnet_dev_bus_registration vnet_dev_bus_registration_t; typedef struct vnet_dev_driver_registration vnet_dev_driver_registration_t; +typedef struct vnet_dev_driver vnet_dev_driver_t; typedef struct vnet_dev vnet_dev_t; typedef struct vnet_dev_port vnet_dev_port_t; typedef struct vnet_dev_rx_queue vnet_dev_rx_queue_t; @@ -90,6 +91,7 @@ typedef struct vnet_dev_counter vnet_dev_counter_t; typedef struct vnet_dev_counter_main vnet_dev_counter_main_t; typedef struct vnet_dev_port_cfg_change_req vnet_dev_port_cfg_change_req_t; +typedef vnet_dev_rv_t (vnet_dev_drv_op_t) (vlib_main_t *, vnet_dev_driver_t *); typedef vnet_dev_rv_t (vnet_dev_op_t) (vlib_main_t *, vnet_dev_t *); typedef vnet_dev_rv_t (vnet_dev_port_op_t) (vlib_main_t *, vnet_dev_port_t *); typedef vnet_dev_rv_t (vnet_dev_port_cfg_change_op_t) ( @@ -228,6 +230,7 @@ typedef struct typedef struct { + vnet_dev_drv_op_t *config_args; vnet_dev_op_t *alloc; vnet_dev_op_t *init; vnet_dev_op_no_rv_t *deinit; @@ -435,6 +438,7 @@ struct vnet_dev_driver_registration int priority; vnet_dev_ops_t ops; vnet_dev_arg_t *args; + vnet_dev_arg_t *drv_args; }; typedef struct @@ -444,7 +448,7 @@ typedef struct vnet_dev_bus_ops_t ops; } vnet_dev_bus_t; -typedef struct +typedef struct vnet_dev_driver { u32 index; void *dev_data; @@ -452,6 +456,7 @@ typedef struct u32 dev_class_index; vnet_dev_bus_index_t bus_index; vnet_dev_ops_t ops; + vnet_dev_arg_t *args; } vnet_dev_driver_t; typedef struct From ad609ca2c091bcb29c957c4e297240145f041147 Mon Sep 17 00:00:00 2001 From: Monendra Singh Kushwaha Date: Tue, 3 Jun 2025 05:47:59 +0000 Subject: [PATCH 232/271] octeon: configure max npa pools using driver arg This patch register callback to set max npa pools and use driver argument to get user specific value for max npa pools. Type: feature Signed-off-by: Monendra Singh Kushwaha Change-Id: Id6cc97ca30af97ea192a924676ebeb22bd80bb38 Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/154810 Tested-by: sa_ip-toolkits-Jenkins --- src/plugins/dev_octeon/init.c | 75 +++++++++++++++++++++++++++++---- src/plugins/dev_octeon/octeon.h | 16 +++++-- 2 files changed, 79 insertions(+), 12 deletions(-) diff --git a/src/plugins/dev_octeon/init.c b/src/plugins/dev_octeon/init.c index 7d61a9865a..05b6dba3c2 100644 --- a/src/plugins/dev_octeon/init.c +++ b/src/plugins/dev_octeon/init.c @@ -17,7 +17,6 @@ #include struct roc_model oct_model; -u32 oct_npa_max_pools = OCT_NPA_MAX_POOLS; oct_main_t oct_main; extern oct_crypto_main_t oct_crypto_main; extern oct_inl_dev_main_t oct_inl_dev_main; @@ -85,6 +84,22 @@ static struct #undef _ }; +static vnet_dev_arg_t oct_drv_args[] = { + { + .id = OCT_DRV_ARG_NPA_MAX_POOLS, + .name = "npa_max_pools", + .desc = "Max NPA pools", + .type = VNET_DEV_ARG_TYPE_UINT32, + .default_val.uint32 = 128, + }, + { + .id = OCT_DRV_ARG_END, + .name = "end", + .desc = "Argument end", + .type = VNET_DEV_ARG_END, + }, +}; + static vnet_dev_arg_t oct_port_args[] = { { .id = OCT_PORT_ARG_ALLMULTI_MODE, @@ -193,6 +208,42 @@ cnx_return_roc_err (vnet_dev_t *dev, int rrv, char *fmt, ...) return VNET_DEV_ERR_UNSUPPORTED_DEVICE; } +static vnet_dev_rv_t +oct_config_args (vlib_main_t *vm, vnet_dev_driver_t *drv) +{ + if (!oct_main.is_config_done) + { + foreach_vnet_dev_port_args (arg, drv) + { + if (arg->id == OCT_DRV_ARG_NPA_MAX_POOLS && + vnet_dev_arg_get_uint32 (arg)) + { + oct_main.npa_max_pools = vnet_dev_arg_get_uint32 (arg); + + if (oct_main.npa_max_pools < 128 || + (oct_main.npa_max_pools > BIT_ULL (20))) + { + log_err ( + NULL, + "Invalid max-pools value (%u), should be in range of " + "(128 - %u)\n", + oct_main.npa_max_pools, BIT_ULL (20)); + return VNET_DEV_ERR_UNSUPPORTED_CONFIG; + } + } + } + oct_main.is_config_done = 1; + } + else + { + log_err (NULL, "Driver config arguments are already initialized or " + "devices are already initialized"); + return VNET_DEV_ERR_UNSUPPORTED_CONFIG; + } + + return 0; +} + static vnet_dev_rv_t oct_alloc (vlib_main_t *vm, vnet_dev_t *dev) { @@ -548,6 +599,13 @@ oct_init (vlib_main_t *vm, vnet_dev_t *dev) vlib_pci_config_hdr_t pci_hdr; vnet_dev_rv_t rv; + /* + * Drivers config arguments should be initialized by this time + * otherwise don't allow to set after device init + */ + if (!oct_main.is_config_done) + oct_main.is_config_done = 1; + rv = vnet_dev_pci_read_config_header (vm, dev, &pci_hdr); if (rv != VNET_DEV_OK) return rv; @@ -644,6 +702,7 @@ VNET_DEV_REGISTER_DRIVER (octeon) = { .bus = "pci", .device_data_sz = sizeof (oct_device_t), .ops = { + .config_args = oct_config_args, .alloc = oct_alloc, .init = oct_init, .deinit = oct_deinit, @@ -651,12 +710,13 @@ VNET_DEV_REGISTER_DRIVER (octeon) = { .probe = oct_probe, }, .args = oct_dev_args, + .drv_args = oct_drv_args, }; static int oct_npa_max_pools_set_cb (struct plt_pci_device *pci_dev) { - roc_idev_npa_maxpools_set (oct_npa_max_pools); + roc_idev_npa_maxpools_set (oct_main.npa_max_pools); return 0; } @@ -676,6 +736,11 @@ oct_plugin_init (vlib_main_t *vm) roc_npa_lf_init_cb_register (oct_npa_max_pools_set_cb); + /* set default values in oct_main */ + oct_main.npa_max_pools = OCT_NPA_MAX_POOLS; + + roc_npa_lf_init_cb_register (oct_npa_max_pools_set_cb); + return 0; } @@ -702,8 +767,6 @@ oct_early_config (vlib_main_t *vm, unformat_input_t *input) while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT) { - if (unformat (line_input, "max-pools %u", &oct_npa_max_pools)) - ; if (unformat (line_input, "disable-single-rx-aura")) oct_main.use_single_rx_aura = 0; else if (unformat (line_input, "ipsec_in_min_spi %u", @@ -723,10 +786,6 @@ oct_early_config (vlib_main_t *vm, unformat_input_t *input) } } - if (oct_npa_max_pools < 128 || (oct_npa_max_pools > BIT_ULL (20))) - error = clib_error_return ( - 0, "Invalid max-pools value (%u), should be in range of (128 - %u)\n", - oct_npa_max_pools, BIT_ULL (20)); done: unformat_free (line_input); return error; diff --git a/src/plugins/dev_octeon/octeon.h b/src/plugins/dev_octeon/octeon.h index 7b94996609..9568e057e1 100644 --- a/src/plugins/dev_octeon/octeon.h +++ b/src/plugins/dev_octeon/octeon.h @@ -25,7 +25,7 @@ #define OCT_FRAME_SIZE (VLIB_FRAME_SIZE * 4) #define OCT_EXT_HDR_SIZE \ PLT_ALIGN (sizeof (oct_ipsec_outbound_pkt_meta_t), ROC_ALIGN) -#define OCT_NPA_MAX_POOLS 8192 +#define OCT_NPA_MAX_POOLS 128 #define OCT_BATCH_ALLOC_IOVA0_MASK 0xFFFFFFFFFFFFFF80 /* @@ -37,8 +37,14 @@ typedef enum { - OCT_PORT_ARG_ALLMULTI_MODE = 1, - OCT_PORT_ARG_EN_ETH_PAUSE_FRAME, + OCT_DRV_ARG_NPA_MAX_POOLS = 1, + OCT_DRV_ARG_END, +} oct_drv_args_t; + +typedef enum +{ + OCT_PORT_ARG_EN_ETH_PAUSE_FRAME = 1, + OCT_PORT_ARG_ALLMULTI_MODE, OCT_PORT_ARG_SWITCH_HDR_TYPE, OCT_PORT_ARG_END } oct_port_args_t; @@ -168,10 +174,12 @@ typedef struct typedef struct { - oct_device_t **oct_dev; u8 inl_dev_initialized : 1; u8 use_single_rx_aura : 1; + u8 is_config_done; + u32 npa_max_pools; u64 rx_aura_handle; + oct_device_t **oct_dev; } oct_main_t; extern oct_main_t oct_main; From 57420a51481c596097a18a8b76f971fec6bbd004 Mon Sep 17 00:00:00 2001 From: Monendra Singh Kushwaha Date: Tue, 3 Jun 2025 06:38:55 +0000 Subject: [PATCH 233/271] octeon: move global config option to driver args This patch removes early_config from OCTEON plugin and moves global options to driver arguments. Type: refactor JIRA: https://essjira.marvell.com/browse/IPBUSW-68962 Signed-off-by: Monendra Singh Kushwaha Change-Id: I3df16d144670274c4e5bd69f27438156446881c1 Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/154835 Tested-by: sa_ip-toolkits-Jenkins (cherry picked from commit 5c935b59ca0745480fd96b0fc9885d7fc417f037) Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/154917 --- src/plugins/dev_octeon/init.c | 91 +++++++++++++++++---------------- src/plugins/dev_octeon/octeon.h | 4 ++ 2 files changed, 52 insertions(+), 43 deletions(-) diff --git a/src/plugins/dev_octeon/init.c b/src/plugins/dev_octeon/init.c index 05b6dba3c2..8b611afe3f 100644 --- a/src/plugins/dev_octeon/init.c +++ b/src/plugins/dev_octeon/init.c @@ -92,6 +92,34 @@ static vnet_dev_arg_t oct_drv_args[] = { .type = VNET_DEV_ARG_TYPE_UINT32, .default_val.uint32 = 128, }, + { + .id = OCT_DRV_ARG_USE_SINGLE_RX_AURA, + .name = "use_single_rx_aura", + .desc = "Use single rx aura", + .type = VNET_DEV_ARG_TYPE_BOOL, + .default_val.boolean = true, + }, + { + .id = OCT_DRV_ARG_IPSEC_IN_MIN_SPI, + .name = "ipsec_in_min_spi", + .desc = "Inline IPsec inbound minimum spi value", + .type = VNET_DEV_ARG_TYPE_UINT32, + .default_val.uint32 = 0, + }, + { + .id = OCT_DRV_ARG_IPSEC_IN_MAX_SPI, + .name = "ipsec_in_max_spi", + .desc = "Inline IPsec inbound maximum spi value", + .type = VNET_DEV_ARG_TYPE_UINT32, + .default_val.uint32 = 8192, + }, + { + .id = OCT_DRV_ARG_IPSEC_OUT_MAX_SA, + .name = "ipsec_out_max_sa", + .desc = "Inline IPsec outbound maximum sa", + .type = VNET_DEV_ARG_TYPE_UINT32, + .default_val.uint32 = 8192, + }, { .id = OCT_DRV_ARG_END, .name = "end", @@ -215,8 +243,10 @@ oct_config_args (vlib_main_t *vm, vnet_dev_driver_t *drv) { foreach_vnet_dev_port_args (arg, drv) { - if (arg->id == OCT_DRV_ARG_NPA_MAX_POOLS && - vnet_dev_arg_get_uint32 (arg)) + if (!arg->val_set) + continue; + + if (arg->id == OCT_DRV_ARG_NPA_MAX_POOLS) { oct_main.npa_max_pools = vnet_dev_arg_get_uint32 (arg); @@ -231,6 +261,18 @@ oct_config_args (vlib_main_t *vm, vnet_dev_driver_t *drv) return VNET_DEV_ERR_UNSUPPORTED_CONFIG; } } + + if (arg->id == OCT_DRV_ARG_USE_SINGLE_RX_AURA) + oct_main.use_single_rx_aura = vnet_dev_arg_get_bool (arg); + + if (arg->id == OCT_DRV_ARG_IPSEC_IN_MIN_SPI) + oct_inl_dev_main.in_min_spi = vnet_dev_arg_get_uint32 (arg); + + if (arg->id == OCT_DRV_ARG_IPSEC_IN_MAX_SPI) + oct_inl_dev_main.in_max_spi = vnet_dev_arg_get_uint32 (arg); + + if (arg->id == OCT_DRV_ARG_IPSEC_OUT_MAX_SA) + oct_inl_dev_main.out_max_sa = vnet_dev_arg_get_uint32 (arg); } oct_main.is_config_done = 1; } @@ -738,6 +780,10 @@ oct_plugin_init (vlib_main_t *vm) /* set default values in oct_main */ oct_main.npa_max_pools = OCT_NPA_MAX_POOLS; + oct_main.use_single_rx_aura = 1; + oct_inl_dev_main.in_min_spi = 0; + oct_inl_dev_main.in_max_spi = 8192; + oct_inl_dev_main.out_max_sa = 8192; roc_npa_lf_init_cb_register (oct_npa_max_pools_set_cb); @@ -751,45 +797,4 @@ VLIB_PLUGIN_REGISTER () = { .description = "dev_octeon", }; -static clib_error_t * -oct_early_config (vlib_main_t *vm, unformat_input_t *input) -{ - unformat_input_t _line_input, *line_input = &_line_input; - clib_error_t *error = 0; - - oct_main.use_single_rx_aura = 1; - oct_inl_dev_main.in_min_spi = 0; - oct_inl_dev_main.in_max_spi = 8192; - oct_inl_dev_main.out_max_sa = 8192; - - if (!unformat_user (input, unformat_line_input, line_input)) - return 0; - - while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT) - { - if (unformat (line_input, "disable-single-rx-aura")) - oct_main.use_single_rx_aura = 0; - else if (unformat (line_input, "ipsec_in_min_spi %u", - &oct_inl_dev_main.in_min_spi)) - ; - else if (unformat (line_input, "ipsec_in_max_spi %u", - &oct_inl_dev_main.in_max_spi)) - ; - else if (unformat (line_input, "ipsec_out_max_sa %u", - &oct_inl_dev_main.out_max_sa)) - ; - else - { - error = clib_error_return (0, "unknown input '%U'", - format_unformat_error, line_input); - goto done; - } - } - -done: - unformat_free (line_input); - return error; -} - -VLIB_EARLY_CONFIG_FUNCTION (oct_early_config, "dev_octeon"); VLIB_BUFFER_SET_EXT_HDR_SIZE (OCT_EXT_HDR_SIZE); diff --git a/src/plugins/dev_octeon/octeon.h b/src/plugins/dev_octeon/octeon.h index 9568e057e1..de91736e13 100644 --- a/src/plugins/dev_octeon/octeon.h +++ b/src/plugins/dev_octeon/octeon.h @@ -38,6 +38,10 @@ typedef enum { OCT_DRV_ARG_NPA_MAX_POOLS = 1, + OCT_DRV_ARG_USE_SINGLE_RX_AURA, + OCT_DRV_ARG_IPSEC_IN_MIN_SPI, + OCT_DRV_ARG_IPSEC_IN_MAX_SPI, + OCT_DRV_ARG_IPSEC_OUT_MAX_SA, OCT_DRV_ARG_END, } oct_drv_args_t; From f268cc405fc99217ca2a09adf3594aefe50122c3 Mon Sep 17 00:00:00 2001 From: Monendra Singh Kushwaha Date: Wed, 4 Jun 2025 16:02:09 +0530 Subject: [PATCH 234/271] build: update roc version Type: feature Signed-off-by: Monendra Singh Kushwaha Change-Id: I2c911c325f0772536addaeb87629b1a785cfd7c9 Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/155019 Tested-by: sa_ip-toolkits-Jenkins (cherry picked from commit f4dbd21a6737d7d959cfebd25590629bb252800e) Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/155023 --- build/external/packages/octeon-roc.mk | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/build/external/packages/octeon-roc.mk b/build/external/packages/octeon-roc.mk index 2e1a029893..3567352393 100644 --- a/build/external/packages/octeon-roc.mk +++ b/build/external/packages/octeon-roc.mk @@ -2,9 +2,9 @@ # SPDX-License-Identifier: Apache-2.0 # https://spdx.org/licenses/Apache-2.0.html -octeon-roc_version := octeon-roc-SDK1X.25.05 -octeon-roc_tarball := $(octeon-roc_version).tar.gz -octeon-roc_tarball_md5sum := 7ca465369902f9a6f5c1b78cff140331 +octeon-roc_version := 25.06 +octeon-roc_tarball := v$(octeon-roc_version).tar.gz +octeon-roc_tarball_md5sum := 0498a68e9bd7b0b49a6454f7ec3915fb octeon-roc_github := https://github.com/MarvellEmbeddedProcessors/marvell-octeon-roc octeon-roc_tarball_strip_dirs := 1 From 55c4e42e52b595efaf25b89f83bd40aca6939c01 Mon Sep 17 00:00:00 2001 From: Monendra Singh Kushwaha Date: Wed, 21 May 2025 10:01:29 +0000 Subject: [PATCH 235/271] ci: add l3fwd performance test case Type: feature JIRA: https://essjira.marvell.com/browse/IPBUSW-66706 Change-Id: Idcb7384610c0214ab5c46a8719e6fc182ee3d623 Signed-off-by: Monendra Singh Kushwaha Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/153632 Tested-by: sa_ip-toolkits-Jenkins Reviewed-by: Nithin Kumar Dabilpuram (cherry picked from commit 1444849bddd6a295c62b525eb1bb4d55804c99bf) Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/155071 --- ci/test/board/board_test_run.sh | 18 + ci/test/common/test_list_helper_funcs.sh | 2 +- ci/test/common/vpp/vpp.env | 9 + ci/test/env/cn10k-perf.env | 23 ++ ci/test/env/cn10k.env | 4 + ci/test/l3fwd_perf/l3fwd_gen.sh | 90 +++++ ci/test/l3fwd_perf/l3fwd_perf.exec | 8 + ci/test/l3fwd_perf/l3fwd_perf.sh | 382 ++++++++++++++++++ ci/test/l3fwd_perf/l3fwd_perf_1.conf | 52 +++ ci/test/l3fwd_perf/l3fwd_perf_2.conf | 52 +++ ci/test/l3fwd_perf/l3fwd_perf_4.conf | 52 +++ ci/test/l3fwd_perf/l3fwd_perf_8.conf | 52 +++ .../ref_numbers/rclk2000_sclk1000.cn106.l3fwd | 7 + .../ref_numbers/rclk2200_sclk1000.cn106.l3fwd | 4 + .../ref_numbers/rclk2500_sclk1000.cn106.l3fwd | 7 + .../ref_numbers/rclk2500_sclk1100.cn106.l3fwd | 7 + ci/test/test.list | 1 + 17 files changed, 769 insertions(+), 1 deletion(-) create mode 100644 ci/test/env/cn10k-perf.env create mode 100755 ci/test/l3fwd_perf/l3fwd_gen.sh create mode 100644 ci/test/l3fwd_perf/l3fwd_perf.exec create mode 100755 ci/test/l3fwd_perf/l3fwd_perf.sh create mode 100644 ci/test/l3fwd_perf/l3fwd_perf_1.conf create mode 100644 ci/test/l3fwd_perf/l3fwd_perf_2.conf create mode 100644 ci/test/l3fwd_perf/l3fwd_perf_4.conf create mode 100644 ci/test/l3fwd_perf/l3fwd_perf_8.conf create mode 100644 ci/test/l3fwd_perf/ref_numbers/rclk2000_sclk1000.cn106.l3fwd create mode 100644 ci/test/l3fwd_perf/ref_numbers/rclk2200_sclk1000.cn106.l3fwd create mode 100644 ci/test/l3fwd_perf/ref_numbers/rclk2500_sclk1000.cn106.l3fwd create mode 100644 ci/test/l3fwd_perf/ref_numbers/rclk2500_sclk1100.cn106.l3fwd diff --git a/ci/test/board/board_test_run.sh b/ci/test/board/board_test_run.sh index 5fa6826bff..283b05ca72 100755 --- a/ci/test/board/board_test_run.sh +++ b/ci/test/board/board_test_run.sh @@ -65,6 +65,17 @@ function target_sync() $TARGET_SSH_CMD $TARGET_BOARD "sudo $TARGET_SCP_CMD $UTILS_LOCATION/dpdk-testpmd /usr/local/bin" # Sync pcap utils $TARGET_SSH_CMD $TARGET_BOARD "sudo $TARGET_SCP_CMD $UTILS_LOCATION/pcap-* ${REMOTE_BUILD_DIR}/ci/test/common/pcap/" +# + if [[ -n $GENERATOR_BOARD ]]; then + $TARGET_SSH_CMD $GENERATOR_BOARD mkdir -p $TARGET_RUN_DIR/deps + rsync -e "$TARGET_SSH_CMD" -av $BUILD_DIR/* $GENERATOR_BOARD:$REMOTE_BUILD_DIR/ + rsync -e "$TARGET_SSH_CMD" -av $BUILD_DIR/../* $GENERATOR_BOARD:$REMOTE_BUILD_DIR/ + # Sync testpmd + $TARGET_SSH_CMD $GENERATOR_BOARD "sudo $TARGET_SCP_CMD $UTILS_LOCATION/dpdk-testpmd /usr/local/bin" + $sync -e "$TARGET_SSH_CMD" \ + $PROJECT_ROOT/ci/test/board/oxk-devbind-basic.sh \ + $GENERATOR_BOARD:$REMOTE_DIR + fi } function target_setup() @@ -78,6 +89,13 @@ function target_setup() return fi $PROJECT_ROOT/ci/test/board/oct-target-setup.sh + + if [[ -n $GENERATOR_BOARD ]]; then + # Setup Generator Board also + TARGET_BOARD=$GENERATOR_BOARD VFIO_DEVBIND=$REMOTE_DIR/oxk-devbind-basic.sh \ + $PROJECT_ROOT/ci/test/board/oct-target-setup.sh + fi + } function run_test() diff --git a/ci/test/common/test_list_helper_funcs.sh b/ci/test/common/test_list_helper_funcs.sh index a3f97ab735..caf78c261c 100644 --- a/ci/test/common/test_list_helper_funcs.sh +++ b/ci/test/common/test_list_helper_funcs.sh @@ -170,6 +170,6 @@ function get_test_command() test_dir=$(get_test_dir $name) - cmd="cd $REMOTE_BUILD_DIR/ci/test/$name && $TARGET_SUDO bash ${REMOTE_BUILD_DIR}/ci/test/$name/$name.sh" + cmd="cd $REMOTE_BUILD_DIR/ci/test/$name && $TARGET_SUDO TARGET_BOARD=$TARGET_BOARD GENERATOR_BOARD=$GENERATOR_BOARD bash ${REMOTE_BUILD_DIR}/ci/test/$name/$name.sh" echo "$cmd" } diff --git a/ci/test/common/vpp/vpp.env b/ci/test/common/vpp/vpp.env index 9a35a7e72a..91676cabbf 100644 --- a/ci/test/common/vpp/vpp.env +++ b/ci/test/common/vpp/vpp.env @@ -68,6 +68,15 @@ function vpp_launch() sleep 2 } +function vpp_exec_cmd() +{ + local startup_conf=$1 + local cmd=$2 + set +e + $VPPCTL -s /tmp/$startup_conf/cli.sock $cmd + set -e +} + function vpp_start() { local startup_conf=$1 diff --git a/ci/test/env/cn10k-perf.env b/ci/test/env/cn10k-perf.env new file mode 100644 index 0000000000..988822c990 --- /dev/null +++ b/ci/test/env/cn10k-perf.env @@ -0,0 +1,23 @@ +#!/bin/bash +# Copyright (c) 2024 Marvell. +# SPDX-License-Identifier: Apache-2.0 +# https://spdx.org/licenses/Apache-2.0.html + +source $PROJECT_ROOT/ci/test/env/cn10k.env + +# List of perf tests to be run. +RUN_TESTS=" + l3fwd_perf +" + +# Update command timeout +CMD_TIMEOUTS="l3fwd_perf=10m $CMD_TIMEOUTS" + +# Perf stage flag +PERF_STAGE=1 + +# Continue testing regardless of failure +CONTINUE_ON_FAILURE=1 + +# Export the path to this conf so that other scripts can source this conf. +export TEST_ENV_CONF=$PROJECT_ROOT/ci/test/env/cn10k-perf.env diff --git a/ci/test/env/cn10k.env b/ci/test/env/cn10k.env index fcbc011f94..eb25518de9 100644 --- a/ci/test/env/cn10k.env +++ b/ci/test/env/cn10k.env @@ -24,6 +24,9 @@ PLAT=${PLAT:-"cn10k"} # Target board user@IP. The user is expected to have passwordless ssh. TARGET_BOARD=${TARGET_BOARD:-root@127.0.0.1} +# Generator board user@IP. The user is expected to have passwordless ssh. +GENERATOR_BOARD=${GENERATOR_BOARD:-} + # Target directory where the VPP build is to be synced REMOTE_DIR=${REMOTE_DIR:-/tmp/vpp} @@ -72,6 +75,7 @@ FIXME_SKIP_TESTS=" " DEFAULT_SKIP_TESTS=" + l3fwd_perf ${FIXME_SKIP_TESTS} " diff --git a/ci/test/l3fwd_perf/l3fwd_gen.sh b/ci/test/l3fwd_perf/l3fwd_gen.sh new file mode 100755 index 0000000000..ea64ab2c4f --- /dev/null +++ b/ci/test/l3fwd_perf/l3fwd_gen.sh @@ -0,0 +1,90 @@ +#!/bin/bash + +# Copyright (c) 2025 Marvell. +# SPDX-License-Identifier: Apache-2.0 +# https://spdx.org/licenses/Apache-2.0.html + +TEST_OP=${TEST_OP:-} +set -e + +OCTEONTESTPATH="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )/.." + +source $OCTEONTESTPATH/common/testpmd/common.env + +PRFX="fwd-gen" +PORT0="${PORT0:-0002:02:00.0}" + +VFIO_DEVBIND="$OCTEONTESTPATH/board/oxk-devbind-basic.sh" + +function sig_handler() +{ + local status=$? + set +e + trap - ERR + trap - INT + if [[ $status -ne 0 ]]; then + echo "$1 Handler" + # Dump error logs + testpmd_log $PRFX + fi + + testpmd_cleanup $PRFX + exit $status +} + +trap "sig_handler ERR" ERR +trap "sig_handler INT" INT + +case $TEST_OP in + launch) + $VFIO_DEVBIND -b vfio-pci $PORT0 + num_cores=$(grep -c ^processor /proc/cpuinfo) + ((num_cores-=1)) + num_cores=${GEN_CORES:-$num_cores} + ((fwd_cores=num_cores-1)) + num_flows=${GEN_FLOWS} + + # Limit the number forwarding cores on cn10k. + # Tx rate peaks (99 MPPS) after 10 cores and drop after 18. + fwd_cores=$(( fwd_cores < 12 ? fwd_cores : 12 )) + + testpmd_launch $PRFX \ + "-l 1-$num_cores -a $PORT0" \ + "--no-flush-rx --nb-cores=$fwd_cores --forward-mode=flowgen \ + -i --txq=$fwd_cores --rxq=$fwd_cores \ + --flowgen-flows=$num_flows --eth-peer=0,00:01:02:03:04:01" /dev/null + testpmd_cmd $PRFX "port stop 0" + testpmd_cmd $PRFX "set flow_ctrl rx off 0" + testpmd_cmd $PRFX "set flow_ctrl tx off 0" + testpmd_cmd $PRFX "port start 0" + ;; + start) + testpmd_cmd $PRFX "start tx_first 256" + testpmd_cmd $PRFX "show port stats all" + ;; + stop) + testpmd_cmd $PRFX "show port stats all" + testpmd_cmd $PRFX "stop" + ;; + rx_pps) + testpmd_cmd $PRFX "show port stats all" + val=`testpmd_log $PRFX | tail -4 | grep -ao 'Rx-pps: .*' | \ + awk '{print $2}'` + echo $val + ;; + tx_pps) + testpmd_cmd $PRFX "show port stats all" + cut -f 2 -d ":" + val=`testpmd_log $PRFX | tail -4 | grep -ao 'Tx-pps: .*' | \ + awk '{print $2}'` + echo $val + ;; + cleanup) + testpmd_cleanup $PRFX + ;; + log) + testpmd_log $PRFX + ;; +esac + +exit 0 diff --git a/ci/test/l3fwd_perf/l3fwd_perf.exec b/ci/test/l3fwd_perf/l3fwd_perf.exec new file mode 100644 index 0000000000..4c8bc50160 --- /dev/null +++ b/ci/test/l3fwd_perf/l3fwd_perf.exec @@ -0,0 +1,8 @@ +set int mac address eth0 00:01:02:03:04:01 + +set int ip address eth0 2.0.0.1/24 + +ip route add 0.0.0.0/0 via eth0 + +set int state eth0 up + diff --git a/ci/test/l3fwd_perf/l3fwd_perf.sh b/ci/test/l3fwd_perf/l3fwd_perf.sh new file mode 100755 index 0000000000..b0995af5c2 --- /dev/null +++ b/ci/test/l3fwd_perf/l3fwd_perf.sh @@ -0,0 +1,382 @@ +#!/bin/bash + +# Copyright (c) 2025 Marvell. +# SPDX-License-Identifier: Apache-2.0 +# https://spdx.org/licenses/Apache-2.0.html + + +set -e +#set -euox pipefail + +GENERATOR_BOARD=${GENERATOR_BOARD:-} +PLAT=${PLAT:-} +OCTEONTESTPATH="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )/.." +VFIO_DEVBIND="$OCTEONTESTPATH/board/oxk-devbind-basic.sh" + +source $OCTEONTESTPATH/common/testpmd/common.env +source $OCTEONTESTPATH/common/vpp/vpp.env + +# Find the dpdk-testpmd application + +TESTPMD_BIN=$(which dpdk-testpmd) +if [[ -z $TESTPMD_BIN ]]; then + echo "dpdk-testpmd not found !!" + exit 1 +fi +echo $TESTPMD_BIN +declare -i num_tests +declare -a test_name +declare -a test_lbk +SUDO="sudo" +remote_ssh="${TARGET_SSH_CMD:-"ssh -o LogLevel=ERROR -o ServerAliveInterval=30 -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"} $GENERATOR_BOARD" +gen=$(realpath ./l3fwd_gen.sh) +MAX_RETRY=${MAX_RETRY:-5} +WITH_GEN_BOARD=0 +GEN_ARG= +G_ENV= +TOLERANCE=${TOLERANCE:-6} + +FWD_PERF_IN=fwd_perf.in +FWD_PERF_OUT=fwd_perf.out +FWD_PERF_OUT_FULL=fwd_perf.out.full +GEN_LOG_FULL=gen.out.full + +START_STR=">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>" +END_STR="<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<" + +LIF0=0002:01:00.1 +LIF1=0002:01:00.2 + +! $(cat /proc/device-tree/compatible | grep -q "cn10k") +IS_CN10K=$? + +if [[ -z "$GENERATOR_BOARD" ]]; then + echo "Generator board details missing!!" + WITH_GEN_BOARD=0 +else + echo "Found Generator board details $GENERATOR_BOARD" + if [[ $IS_CN10K -ne 0 ]]; then + WITH_GEN_BOARD=1 + fi +fi + +if [[ $WITH_GEN_BOARD -eq 0 ]] +then + IF0=$LIF0 + IF1=$LIF1 + remote_ssh="sh -c " + GEN_PORT=$IF1 + G_ENV="GEN_CORES=6" + SUDO="" + echo "Running locally without generator board" +else + IF0=0002:02:00.0 + GEN_PORT=$IF0 + $VFIO_DEVBIND -b vfio-pci $IF0 + # Dummy whitelist device + IF1=0008:08:08.0 + echo "Running with generator board" +fi + +rm -rf $FWD_PERF_IN $FWD_PERF_OUT $FWD_PERF_OUT_FULL $GEN_LOG_FULL + +function sig_handler() +{ + local status=$? + set +e + trap - ERR + trap - INT + trap - QUIT + trap - EXIT + if [[ $status -ne 0 ]]; then + echo "$1 Handler" + fi + + awk ' { print FILENAME": " $0 } ' $FWD_PERF_OUT_FULL + awk ' { print FILENAME": " $0 } ' $FWD_PERF_OUT + awk ' { print FILENAME": " $0 } ' $GEN_LOG_FULL + + killall -9 dpdk-testpmd + $remote_ssh "sudo killall -9 dpdk-testpmd" + vpp_log l3fwd_perf + vpp_stats_all l3fwd_perf + vpp_cleanup l3fwd_perf + exit $status +} + +trap "sig_handler ERR" ERR +trap "sig_handler INT" INT +trap "sig_handler QUIT" QUIT +trap "sig_handler EXIT" EXIT + +# Get CPU PART NUMBER +PARTNUM_106XX=0xd49 +PARTNUM=$(grep -m 1 'CPU part' /proc/cpuinfo | awk -F': ' '{print $2}') +DTC=$(tr -d '\0' >$GEN_LOG_FULL + $remote_ssh "$SUDO PORT0=$GEN_PORT TEST_OP=launch $G_ENV GEN_FLOWS=$CORES $gen $GEN_ARG" +} + +start_gen() { + $remote_ssh "$SUDO PLAT=$PLAT PORT0=$GEN_PORT TEST_OP=start $gen" +} + +stop_gen() { + $remote_ssh "$SUDO PLAT=$PLAT PORT0=$GEN_PORT TEST_OP=stop $gen" +} + +cleanup_gen() { + $remote_ssh "$SUDO PLAT=$PLAT PORT0=$GEN_PORT TEST_OP=log $gen" >>$GEN_LOG_FULL + echo $END_STR ${test_name[$idx]} >>$GEN_LOG_FULL + + $remote_ssh "$SUDO PLAT=$PLAT PORT0=$GEN_PORT TEST_OP=cleanup $gen" +} + +testpmd_pps_local() { + local rx_pps=0 + + echo "show port stats all" >>$FWD_PERF_IN + sleep 1 + echo "show port stats all" >>$FWD_PERF_IN + sleep 1 + echo "show port stats all" >>$FWD_PERF_IN + while ! (tail -n1 $FWD_PERF_OUT | grep -q "testpmd> $") + do + sleep 0.1 + continue; + done + + pps=`cat $FWD_PERF_OUT | \ + grep "Rx-pps:" | awk '{print $2}' | tail -2` + for i in $pps + do + rx_pps=$((rx_pps + i)) + done + echo $rx_pps +} + +check_pps() { + idx=$1 + pass_pps=$(expected_pps $idx) + ref_pps=$(ref_pps $idx) + local retry=3 + + while [[ retry -ne 0 ]] + do + + rx_pps=$($remote_ssh "$SUDO TEST_OP=rx_pps $gen") + + if [[ rx_pps -lt pass_pps ]]; then + echo -n "Low PPS for ${test_name[$idx]} ($rx_pps < $pass_pps)" + echo " (Ref $ref_pps, tolerance $TOLERANCE%)" + else + echo -n "Rx PPS $rx_pps as expected $pass_pps" + echo " (Ref $ref_pps, tolerance $TOLERANCE%)" + return 0 + fi + + sleep 1 + ((retry-=1)) + done + + return 1 +} + +cleanup_one() { + local idx=$1 + + vpp_log l3fwd_perf + vpp_stats_all l3fwd_perf + vpp_cleanup l3fwd_perf + + stop_gen + cleanup_gen $idx + + cat $FWD_PERF_OUT >> $FWD_PERF_OUT_FULL + echo $END_STR ${test_name[$idx]} >>$FWD_PERF_OUT_FULL +} + +run_one() { + unbuffer="$(command -v stdbuf) -o 0" || unbuffer= + local in=$FWD_PERF_IN + local out=$FWD_PERF_OUT + idx=$1 + + echo $START_STR ${test_name[$idx]} >>$FWD_PERF_OUT_FULL + + rm -rf $in $out + touch $in $out + + CORES=${test_ncores[$idx]} + + echo -n "Starting l3fwd with 'n_cores=$CORES port=$IF0 " + rm -rf /tmp/l3fwd_perf + mkdir -p /tmp/l3fwd_perf + cp l3fwd_perf_$CORES.conf /tmp/l3fwd_perf/ + cp l3fwd_perf.exec /tmp/l3fwd_perf + vpp_launch l3fwd_perf_$CORES + vpp_exec_cmd l3fwd_perf "device attach pci/$IF0 driver octeon" + vpp_exec_cmd l3fwd_perf "device create-interface pci/$IF0 port 0 name eth0 num-rx-queues $CORES tx-queues-size 16384" + vpp_start l3fwd_perf + for (( i=0; i<$CORES; i++ )) + do + vpp_exec_cmd l3fwd_perf "set ip neighbor eth0 10.253.0.$i 00:01:02:03:04:00" + vpp_exec_cmd l3fwd_perf "test flow add dst-ip 10.253.0.$i/255.255.255.255 proto 17 redirect-to-queue $i" + vpp_exec_cmd l3fwd_perf "test flow enable index $i eth0" + done + + launch_gen $idx + start_gen +} + +run_fwd_tests() { + + get_system_info + + idx=0 + ret=0 + REF_WITH_GEN_BOARD=$WITH_GEN_BOARD + REF_IF0=$IF0 + REF_IF1=$IF1 + local retry_count=$MAX_RETRY + while [[ idx -lt num_tests ]]; do + + if [[ ${test_lbk[$idx]} -eq 1 ]]; then + # Forcing change to run on LBK interface only + WITH_GEN_BOARD=0 + IF0=$LIF0 + IF1=$LIF1 + else + # Restore for other cases + WITH_GEN_BOARD=$REF_WITH_GEN_BOARD + IF0=$REF_IF0 + IF1=$REF_IF1 + fi + + run_one $idx + + sleep 3 + + set +e + check_pps $idx + local k=$? + set -e + + if [[ k -eq 0 ]]; then + cleanup_one $idx + + ((idx+=1)) + retry_count=$MAX_RETRY + continue + fi + ((retry_count-=1)) || true + + if [[ retry_count -eq 0 ]]; then + echo "FAIL: ${test_name[$idx]}" + cleanup_one $idx + + ((ret+=1)) + ((idx+=1)) + retry_count=$MAX_RETRY + else + echo "Re-run ${test_name[$idx]} $retry_count" + cleanup_one $idx + fi + done + + exit $ret +} + +num_tests=0 + +# Register fwd performance tests. +# Format: + +register_fwd_test "L3FWD_1C" "1" "0" +register_fwd_test "L3FWD_2C" "2" "0" +register_fwd_test "L3FWD_4C" "4" "0" +register_fwd_test "L3FWD_8C" "8" "0" + +run_fwd_tests + +cleanup_gen diff --git a/ci/test/l3fwd_perf/l3fwd_perf_1.conf b/ci/test/l3fwd_perf/l3fwd_perf_1.conf new file mode 100644 index 0000000000..ea0bc279ec --- /dev/null +++ b/ci/test/l3fwd_perf/l3fwd_perf_1.conf @@ -0,0 +1,52 @@ +unix { + log /tmp/l3fwd_perf/vpp.log + cli-listen /tmp/l3fwd_perf/cli.sock + #full-coredump + #interactive + nodaemon +} + +api-trace { + on +} + +logging { + default-syslog-log-level info +} + +cpu { + main-core 23 + corelist-workers 22 +} + +session +{ + event-queue-length 10 +} +#tcp { no-csum-offload } + +buffers { + ## Increase number of buffers allocated, needed only in scenarios with + ## large number of interfaces and worker threads. Value is per numa node. + ## Default is 16384 (8192 if running unprivileged) +# buffers-per-numa 5000 + + ## Size of buffer data area + ## Default is 2048 + default data-size 2048 +# naturally-aligned +# enable-nat-alignment +} + +#devices { +# dev pci/0002:01:00.1 { +# driver octeon +# port 0 { +# name eth0 +# } +# } +#} +plugins { + plugin dpdk_plugin.so { disable } + plugin onp_plugin.so { disable } +} diff --git a/ci/test/l3fwd_perf/l3fwd_perf_2.conf b/ci/test/l3fwd_perf/l3fwd_perf_2.conf new file mode 100644 index 0000000000..b06c084291 --- /dev/null +++ b/ci/test/l3fwd_perf/l3fwd_perf_2.conf @@ -0,0 +1,52 @@ +unix { + log /tmp/l3fwd_perf/vpp.log + cli-listen /tmp/l3fwd_perf/cli.sock + #full-coredump + #interactive + nodaemon +} + +api-trace { + on +} + +logging { + default-syslog-log-level info +} + +cpu { + main-core 23 + corelist-workers 21-22 +} + +session +{ + event-queue-length 10 +} +#tcp { no-csum-offload } + +buffers { + ## Increase number of buffers allocated, needed only in scenarios with + ## large number of interfaces and worker threads. Value is per numa node. + ## Default is 16384 (8192 if running unprivileged) +# buffers-per-numa 5000 + + ## Size of buffer data area + ## Default is 2048 + default data-size 2048 +# naturally-aligned +# enable-nat-alignment +} + +#devices { +# dev pci/0002:01:00.1 { +# driver octeon +# port 0 { +# name eth0 +# } +# } +#} +plugins { + plugin dpdk_plugin.so { disable } + plugin onp_plugin.so { disable } +} diff --git a/ci/test/l3fwd_perf/l3fwd_perf_4.conf b/ci/test/l3fwd_perf/l3fwd_perf_4.conf new file mode 100644 index 0000000000..63e6f13c14 --- /dev/null +++ b/ci/test/l3fwd_perf/l3fwd_perf_4.conf @@ -0,0 +1,52 @@ +unix { + log /tmp/l3fwd_perf/vpp.log + cli-listen /tmp/l3fwd_perf/cli.sock + #full-coredump + #interactive + nodaemon +} + +api-trace { + on +} + +logging { + default-syslog-log-level info +} + +cpu { + main-core 23 + corelist-workers 19-22 +} + +session +{ + event-queue-length 10 +} +#tcp { no-csum-offload } + +buffers { + ## Increase number of buffers allocated, needed only in scenarios with + ## large number of interfaces and worker threads. Value is per numa node. + ## Default is 16384 (8192 if running unprivileged) +# buffers-per-numa 5000 + + ## Size of buffer data area + ## Default is 2048 + default data-size 2048 +# naturally-aligned +# enable-nat-alignment +} + +#devices { +# dev pci/0002:01:00.1 { +# driver octeon +# port 0 { +# name eth0 +# } +# } +#} +plugins { + plugin dpdk_plugin.so { disable } + plugin onp_plugin.so { disable } +} diff --git a/ci/test/l3fwd_perf/l3fwd_perf_8.conf b/ci/test/l3fwd_perf/l3fwd_perf_8.conf new file mode 100644 index 0000000000..cc3aa45d16 --- /dev/null +++ b/ci/test/l3fwd_perf/l3fwd_perf_8.conf @@ -0,0 +1,52 @@ +unix { + log /tmp/l3fwd_perf/vpp.log + cli-listen /tmp/l3fwd_perf/cli.sock + #full-coredump + #interactive + nodaemon +} + +api-trace { + on +} + +logging { + default-syslog-log-level info +} + +cpu { + main-core 23 + corelist-workers 15-22 +} + +session +{ + event-queue-length 10 +} +#tcp { no-csum-offload } + +buffers { + ## Increase number of buffers allocated, needed only in scenarios with + ## large number of interfaces and worker threads. Value is per numa node. + ## Default is 16384 (8192 if running unprivileged) +# buffers-per-numa 5000 + + ## Size of buffer data area + ## Default is 2048 + default data-size 2048 +# naturally-aligned +# enable-nat-alignment +} + +#devices { +# dev pci/0002:01:00.1 { +# driver octeon +# port 0 { +# name eth0 +# } +# } +#} +plugins { + plugin dpdk_plugin.so { disable } + plugin onp_plugin.so { disable } +} diff --git a/ci/test/l3fwd_perf/ref_numbers/rclk2000_sclk1000.cn106.l3fwd b/ci/test/l3fwd_perf/ref_numbers/rclk2000_sclk1000.cn106.l3fwd new file mode 100644 index 0000000000..f0ca89d57f --- /dev/null +++ b/ci/test/l3fwd_perf/ref_numbers/rclk2000_sclk1000.cn106.l3fwd @@ -0,0 +1,7 @@ +PPS of tests on 106xx and multiple RCLK +--------------------------------------- + +L3FWD_1C 12550000 +L3FWD_2C 24550000 +L3FWD_4C 44000000 +L3FWD_8C 74000000 diff --git a/ci/test/l3fwd_perf/ref_numbers/rclk2200_sclk1000.cn106.l3fwd b/ci/test/l3fwd_perf/ref_numbers/rclk2200_sclk1000.cn106.l3fwd new file mode 100644 index 0000000000..6274989d16 --- /dev/null +++ b/ci/test/l3fwd_perf/ref_numbers/rclk2200_sclk1000.cn106.l3fwd @@ -0,0 +1,4 @@ +PPS of tests on 106xx and multiple RCLK +--------------------------------------- + +L3FWD_1C 12550000 diff --git a/ci/test/l3fwd_perf/ref_numbers/rclk2500_sclk1000.cn106.l3fwd b/ci/test/l3fwd_perf/ref_numbers/rclk2500_sclk1000.cn106.l3fwd new file mode 100644 index 0000000000..d3d887b22b --- /dev/null +++ b/ci/test/l3fwd_perf/ref_numbers/rclk2500_sclk1000.cn106.l3fwd @@ -0,0 +1,7 @@ +PPS of tests on 106xx and multiple RCLK +--------------------------------------- + +L3FWD_1C 14494473 +L3FWD_2C 27358193 +L3FWD_4C 50342188 +L3FWD_8C 83399916 diff --git a/ci/test/l3fwd_perf/ref_numbers/rclk2500_sclk1100.cn106.l3fwd b/ci/test/l3fwd_perf/ref_numbers/rclk2500_sclk1100.cn106.l3fwd new file mode 100644 index 0000000000..d3d887b22b --- /dev/null +++ b/ci/test/l3fwd_perf/ref_numbers/rclk2500_sclk1100.cn106.l3fwd @@ -0,0 +1,7 @@ +PPS of tests on 106xx and multiple RCLK +--------------------------------------- + +L3FWD_1C 14494473 +L3FWD_2C 27358193 +L3FWD_4C 50342188 +L3FWD_8C 83399916 diff --git a/ci/test/test.list b/ci/test/test.list index ecf95a0a46..a434ec0a0c 100644 --- a/ci/test/test.list +++ b/ci/test/test.list @@ -1,2 +1,3 @@ l3fwd# inl_ipsec# +l3fwd_perf# From c1af631595c5644aca55df0a2d8a40b2032632ac Mon Sep 17 00:00:00 2001 From: Monendra Singh Kushwaha Date: Sun, 15 Jun 2025 16:45:06 +0530 Subject: [PATCH 236/271] octeon: fix length in inline inbound multi-seg This patch fix vlib buffer length in inline inbound multi-seg case when last segment doesn't have any valid data. Type: fix JIRA: https://essjira.marvell.com/browse/IPBUSW-70580 Change-Id: I7271304eb5d448cc848c12cfb9c5b1c9b901556c Signed-off-by: Monendra Singh Kushwaha Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/155719 Tested-by: sa_ip-sw-jenkins Reviewed-by: Devapraba Muthumani --- src/plugins/dev_octeon/rx_node.c | 41 +++++++++++++++++++------------- 1 file changed, 24 insertions(+), 17 deletions(-) diff --git a/src/plugins/dev_octeon/rx_node.c b/src/plugins/dev_octeon/rx_node.c index 8e7ef8bebc..33545d2290 100644 --- a/src/plugins/dev_octeon/rx_node.c +++ b/src/plugins/dev_octeon/rx_node.c @@ -864,8 +864,14 @@ oct_rx_ipsec_attach_tail (vlib_main_t *vm, oct_rx_node_ctx_t *ctx, * incase of multi seg, update seg1 length and advance total words processed. * also, updates total bytes in buffer. */ - buf->current_length = seg_len & OCT_SEG_LEN_MASK; - len -= buf->current_length; + sg_len = seg_len & OCT_SEG_LEN_MASK; + len -= sg_len; + if (len < 0) + { + sg_len = sg_len + len; + len = 0; + } + buf->current_length = sg_len; /* Process from 2nd segment */ next_seg = 2; @@ -884,16 +890,16 @@ oct_rx_ipsec_attach_tail (vlib_main_t *vm, oct_rx_node_ctx_t *ctx, sizeof (vlib_buffer_t)); seg_buf->template = *bt; sg_len = seg_len & OCT_SEG_LEN_MASK; - - { - /* - * Adjust last buf data length with negative offset for - * ipsec pkts if needed. - */ - len -= sg_len; - sg_len = (len > 0) ? sg_len : (sg_len + len); - len = (len > 0) ? len : 0; - } + /* + * Adjust last buf data length with negative offset for + * ipsec pkts if needed. + */ + len -= sg_len; + if (len < 0) + { + sg_len = sg_len + len; + len = 0; + } seg_buf->current_length = sg_len; bi = vlib_get_buffer_index (vm, seg_buf); @@ -984,8 +990,8 @@ oct_rx_inl_ipsec_vlib_from_cq ( idx = cpt_hdr->w0.cookie; oct_rx_ipsec_update_counters (vm, b[0], esp_sz, frag_cnt, idx); - oct_rx_ipsec_attach_tail (vm, ctx, orig_rxp, bt, b[0]); } + oct_rx_ipsec_attach_tail (vm, ctx, orig_rxp, bt, b[0]); ctx->n_rx_bytes += olen; ctx->n_segs += frag_cnt; @@ -1306,8 +1312,9 @@ oct_rx_batch (vlib_main_t *vm, vlib_node_runtime_t *node, idx0 = cpt_hdr0->w0.cookie; oct_rx_ipsec_update_counters (vm, b[0], esp_sz0, frag_cnt0, idx0); - oct_rx_ipsec_attach_tail (vm, ctx, orig_rxp0, &bt, b[0]); } + /* Success and Failure both cases can be multi seg */ + oct_rx_ipsec_attach_tail (vm, ctx, orig_rxp0, &bt, b[0]); buffs[buffer_next_index] = b[1]; @@ -1335,8 +1342,8 @@ oct_rx_batch (vlib_main_t *vm, vlib_node_runtime_t *node, idx1 = cpt_hdr1->w0.cookie; oct_rx_ipsec_update_counters (vm, b[1], esp_sz1, frag_cnt1, idx1); - oct_rx_ipsec_attach_tail (vm, ctx, orig_rxp1, &bt, b[1]); } + oct_rx_ipsec_attach_tail (vm, ctx, orig_rxp1, &bt, b[1]); buffs[buffer_next_index] = b[2]; @@ -1364,8 +1371,8 @@ oct_rx_batch (vlib_main_t *vm, vlib_node_runtime_t *node, idx2 = cpt_hdr2->w0.cookie; oct_rx_ipsec_update_counters (vm, b[2], esp_sz2, frag_cnt2, idx2); - oct_rx_ipsec_attach_tail (vm, ctx, orig_rxp2, &bt, b[2]); } + oct_rx_ipsec_attach_tail (vm, ctx, orig_rxp2, &bt, b[2]); buffs[buffer_next_index] = b[3]; @@ -1394,8 +1401,8 @@ oct_rx_batch (vlib_main_t *vm, vlib_node_runtime_t *node, idx3 = cpt_hdr3->w0.cookie; oct_rx_ipsec_update_counters (vm, b[3], esp_sz3, frag_cnt3, idx3); - oct_rx_ipsec_attach_tail (vm, ctx, orig_rxp3, &bt, b[3]); } + oct_rx_ipsec_attach_tail (vm, ctx, orig_rxp3, &bt, b[3]); } ctx->n_rx_bytes += olen0 + olen1 + olen2 + olen3; ctx->n_segs += frag_cnt0 + frag_cnt1 + frag_cnt2 + frag_cnt3; From 4c33ae3b656aefb170423e50eeb3225e217fc7e7 Mon Sep 17 00:00:00 2001 From: Monendra Singh Kushwaha Date: Sat, 7 Jun 2025 23:41:25 +0530 Subject: [PATCH 237/271] octeon: fix ipsec interface counter This patch fixes IPSec interface counter and drops packets if no matching tunnel found. Type: fix JIRA: https://essjira.marvell.com/browse/IPBUSW-69494 Signed-off-by: Monendra Singh Kushwaha Change-Id: Id315e126a65f7cdd60c7dc6b525301c6f77bfe77 Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/155244 Tested-by: sa_ip-toolkits-Jenkins Reviewed-by: Nithin Kumar Dabilpuram (cherry picked from commit 4f7dcd90a6002756cff73eed4c8eb36eff40206e) Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/156437 Tested-by: sa_ip-sw-jenkins --- src/plugins/dev_octeon/octeon.h | 1 + src/plugins/dev_octeon/rx_node.c | 296 ++++++++----------------------- 2 files changed, 71 insertions(+), 226 deletions(-) diff --git a/src/plugins/dev_octeon/octeon.h b/src/plugins/dev_octeon/octeon.h index de91736e13..2b6624f4c2 100644 --- a/src/plugins/dev_octeon/octeon.h +++ b/src/plugins/dev_octeon/octeon.h @@ -252,6 +252,7 @@ vnet_dev_rv_t oct_txq_get_stats (vlib_main_t *, vnet_dev_port_t *, format_vnet_dev_addr, (dev), ##__VA_ARGS__) #define foreach_oct_rx_node_counter \ + _ (ERR_NO_TUNNEL, err_no_tunnel, ERROR, "no matching IPsec tunnel") \ _ (ERR_UNDEFINED, err_undefined, ERROR, "undefined decrypt error") /* clang-format off */ diff --git a/src/plugins/dev_octeon/rx_node.c b/src/plugins/dev_octeon/rx_node.c index 33545d2290..c0e9b8ca58 100644 --- a/src/plugins/dev_octeon/rx_node.c +++ b/src/plugins/dev_octeon/rx_node.c @@ -498,25 +498,6 @@ oct_rx_ipsec_reassembly_failure (vlib_main_t *vm, vlib_buffer_template_t *bt, return frag_cnt; } -static_always_inline u8 -oct_rx_ipsec_reassembly (vlib_main_t *vm, vlib_buffer_template_t *bt, - struct cpt_cn10k_parse_hdr_s *cpt_hdr, - oct_nix_rx_cqe_desc_t *d, vlib_buffer_t *b, - vlib_buffer_t **buf, u16 *next, - u16 *buffer_next_index, u32 *olen, u32 *esp_len, - u32 l2_ol3_hdr_size, const u64 fp_flags) -{ - if ((cpt_hdr->w0.num_frags) && !(cpt_hdr->w0.reas_sts)) - return oct_rx_ipsec_reassembly_success (vm, bt, cpt_hdr, d, b, olen, - esp_len, l2_ol3_hdr_size); - else if (cpt_hdr->w0.reas_sts) - return oct_rx_ipsec_reassembly_failure (vm, bt, cpt_hdr, d, buf, next, - buffer_next_index, olen, esp_len, - l2_ol3_hdr_size, fp_flags); - else - return 1; -} - static_always_inline u32 oct_ipsec_update_itf_sw_idx (oct_ipsec_session_t *session, u32 sa_idx) { @@ -542,7 +523,8 @@ oct_ipsec_update_itf_sw_idx (oct_ipsec_session_t *session, u32 sa_idx) ipsec4_tunnel_mk_key (key40, &ip_addr->ip.ip4, clib_host_to_net_u32 (sa->spi)); rv = clib_bihash_search_inline_8_16 (&ipm->tun4_protect_by_key, &bkey40); - ASSERT (!rv); + if (PREDICT_FALSE (rv)) + return ~0; res = (ipsec_tun_lkup_result_t *) &bkey40.value; } @@ -556,7 +538,8 @@ oct_ipsec_update_itf_sw_idx (oct_ipsec_session_t *session, u32 sa_idx) rv = clib_bihash_search_inline_24_16 (&ipm->tun6_protect_by_key, &bkey60); - ASSERT (!rv); + if (PREDICT_FALSE (rv)) + return ~0; res = (ipsec_tun_lkup_result_t *) &bkey60.value; } @@ -569,43 +552,41 @@ oct_ipsec_update_itf_sw_idx (oct_ipsec_session_t *session, u32 sa_idx) } static_always_inline void -oct_rx_ipsec_update_sa_counters_x4 (vlib_main_t *vm, vlib_buffer_t *b0, - vlib_buffer_t *b1, vlib_buffer_t *b2, - vlib_buffer_t *b3, u32 ilen, u8 frag_cnt, - u32 oct_sa_idx) +oct_rx_ipsec_update_counters (vlib_main_t *vm, vlib_node_runtime_t *node, + vlib_buffer_t **buffs, u16 *next, + u16 *buffer_next_index, u32 ilen, u8 frag_cnt, + u32 idx, const u8 reass_fail) { vlib_combined_counter_main_t *rx_counter; ipsec_main_t *im = &ipsec_main; oct_ipsec_session_t *session; struct roc_ot_ipsec_inb_sa *roc_sa; + oct_ipsec_inb_sa_priv_data_t *inb_sa_priv; + vlib_buffer_t *b = buffs[*buffer_next_index - 1]; + u32 sa_idx, itf_sw_idx; vnet_interface_main_t *vim; oct_ipsec_main_t *oim = &oct_ipsec_main; oct_inl_dev_main_t *oidm = &oct_inl_dev_main; - oct_ipsec_inb_sa_priv_data_t *inb_sa_priv; - u32 sa_idx, itf_sw_idx; vnet_main_t *vnm; + int i; vnm = im->vnet_main; vim = &vnm->interface_main; rx_counter = vim->combined_sw_if_counters + VNET_INTERFACE_COUNTER_RX; - roc_sa = roc_nix_inl_ot_ipsec_inb_sa (oidm->inb_sa_base, oct_sa_idx); + roc_sa = roc_nix_inl_ot_ipsec_inb_sa (oidm->inb_sa_base, idx); inb_sa_priv = roc_nix_inl_ot_ipsec_inb_sa_sw_rsvd (roc_sa); sa_idx = (u32) inb_sa_priv->user_data; - vnet_buffer (b0)->ipsec.sad_index = sa_idx; - vnet_buffer (b1)->ipsec.sad_index = sa_idx; - vnet_buffer (b2)->ipsec.sad_index = sa_idx; - vnet_buffer (b3)->ipsec.sad_index = sa_idx; - + vnet_buffer (b)->ipsec.sad_index = sa_idx; ASSERT (sa_idx < vec_len (oim->inline_ipsec_sessions)); session = pool_elt_at_index (oim->inline_ipsec_sessions, sa_idx); itf_sw_idx = session->itf_sw_idx; /* * Check if itf_sw_idx is populated already. First packet on the SA - * populates the itf_sw_idx in the SA session + * populates the itf_sw_idx in the SA session. */ if (PREDICT_FALSE (itf_sw_idx == ~0)) itf_sw_idx = oct_ipsec_update_itf_sw_idx (session, sa_idx); @@ -614,165 +595,55 @@ oct_rx_ipsec_update_sa_counters_x4 (vlib_main_t *vm, vlib_buffer_t *b0, vlib_increment_combined_counter (&ipsec_sa_counters, vm->thread_index, sa_idx, frag_cnt, ilen); - /* Update ITF counters with inner IP length */ - vlib_increment_combined_counter (rx_counter, vm->thread_index, itf_sw_idx, - frag_cnt, ilen); -} -static_always_inline void -oct_rx_ipsec_update_counters_x4 (vlib_main_t *vm, vlib_buffer_t *b0, u32 ilen0, - u8 frag_cnt0, u32 idx0, vlib_buffer_t *b1, - u32 ilen1, u8 frag_cnt1, u32 idx1, - vlib_buffer_t *b2, u32 ilen2, u8 frag_cnt2, - u32 idx2, vlib_buffer_t *b3, u32 ilen3, - u8 frag_cnt3, u32 idx3) -{ - vlib_combined_counter_main_t *rx_counter; - oct_inl_dev_main_t *oidm = &oct_inl_dev_main; - oct_ipsec_main_t *oim = &oct_ipsec_main; - struct roc_ot_ipsec_inb_sa *roc_sa0, *roc_sa1; - struct roc_ot_ipsec_inb_sa *roc_sa2, *roc_sa3; - oct_ipsec_inb_sa_priv_data_t *inb_sa_priv; - ipsec_main_t *im = &ipsec_main; - oct_ipsec_session_t *session; - vnet_interface_main_t *vim; - u32 sa_idx0, itf_sw_idx0; - u32 sa_idx1, itf_sw_idx1; - u32 sa_idx2, itf_sw_idx2; - u32 sa_idx3, itf_sw_idx3; - vnet_main_t *vnm; - u32 idx_xor; - - idx_xor = idx0 ^ idx1; - idx_xor += idx1 ^ idx2; - idx_xor += idx2 ^ idx3; - - if (!idx_xor) + if (PREDICT_FALSE (itf_sw_idx == ~0)) { - oct_rx_ipsec_update_sa_counters_x4 ( - vm, b0, b1, b2, b3, ilen0 + ilen1 + ilen2 + ilen3, - frag_cnt0 + frag_cnt1 + frag_cnt2 + frag_cnt3, idx0); - return; - } - - vnm = im->vnet_main; - vim = &vnm->interface_main; - rx_counter = vim->combined_sw_if_counters + VNET_INTERFACE_COUNTER_RX; - - roc_sa0 = roc_nix_inl_ot_ipsec_inb_sa (oidm->inb_sa_base, idx0); - inb_sa_priv = roc_nix_inl_ot_ipsec_inb_sa_sw_rsvd (roc_sa0); - - sa_idx0 = (u32) inb_sa_priv->user_data; - vnet_buffer (b0)->ipsec.sad_index = sa_idx0; - - roc_sa1 = roc_nix_inl_ot_ipsec_inb_sa (oidm->inb_sa_base, idx1); - inb_sa_priv = roc_nix_inl_ot_ipsec_inb_sa_sw_rsvd (roc_sa1); - - sa_idx1 = (u32) inb_sa_priv->user_data; - vnet_buffer (b1)->ipsec.sad_index = sa_idx1; + b->error = node->errors[OCT_RX_NODE_CTR_ERR_NO_TUNNEL]; + next[*buffer_next_index - 1] = VNET_DEV_ETH_RX_PORT_NEXT_DROP; - roc_sa2 = roc_nix_inl_ot_ipsec_inb_sa (oidm->inb_sa_base, idx2); - inb_sa_priv = roc_nix_inl_ot_ipsec_inb_sa_sw_rsvd (roc_sa2); - - sa_idx2 = (u32) inb_sa_priv->user_data; - vnet_buffer (b2)->ipsec.sad_index = sa_idx2; - - roc_sa3 = roc_nix_inl_ot_ipsec_inb_sa (oidm->inb_sa_base, idx3); - inb_sa_priv = roc_nix_inl_ot_ipsec_inb_sa_sw_rsvd (roc_sa3); - - sa_idx3 = (u32) inb_sa_priv->user_data; - vnet_buffer (b3)->ipsec.sad_index = sa_idx3; - - ASSERT (sa_idx0 < vec_len (oim->inline_ipsec_sessions)); - ASSERT (sa_idx1 < vec_len (oim->inline_ipsec_sessions)); - ASSERT (sa_idx2 < vec_len (oim->inline_ipsec_sessions)); - ASSERT (sa_idx3 < vec_len (oim->inline_ipsec_sessions)); - - session = pool_elt_at_index (oim->inline_ipsec_sessions, sa_idx0); - itf_sw_idx0 = session->itf_sw_idx; - /* Check if itf_sw_idx is populated already. First packet on the SA - populates the itf_sw_idx in the SA session */ - if (PREDICT_FALSE (itf_sw_idx0 == ~0)) - itf_sw_idx0 = oct_ipsec_update_itf_sw_idx (session, sa_idx0); - - session = pool_elt_at_index (oim->inline_ipsec_sessions, sa_idx1); - itf_sw_idx1 = session->itf_sw_idx; - if (PREDICT_FALSE (itf_sw_idx1 == ~0)) - itf_sw_idx1 = oct_ipsec_update_itf_sw_idx (session, sa_idx1); - - session = pool_elt_at_index (oim->inline_ipsec_sessions, sa_idx2); - itf_sw_idx2 = session->itf_sw_idx; - if (PREDICT_FALSE (itf_sw_idx2 == ~0)) - itf_sw_idx2 = oct_ipsec_update_itf_sw_idx (session, sa_idx2); - - session = pool_elt_at_index (oim->inline_ipsec_sessions, sa_idx3); - itf_sw_idx3 = session->itf_sw_idx; - if (PREDICT_FALSE (itf_sw_idx3 == ~0)) - itf_sw_idx3 = oct_ipsec_update_itf_sw_idx (session, sa_idx3); - - /* Update IPsec counters with outer IP length */ - vlib_increment_combined_counter (&ipsec_sa_counters, vm->thread_index, - sa_idx0, frag_cnt0, ilen0); - vlib_increment_combined_counter (&ipsec_sa_counters, vm->thread_index, - sa_idx1, frag_cnt1, ilen1); - vlib_increment_combined_counter (&ipsec_sa_counters, vm->thread_index, - sa_idx2, frag_cnt2, ilen2); - vlib_increment_combined_counter (&ipsec_sa_counters, vm->thread_index, - sa_idx3, frag_cnt3, ilen3); - - /* Update ITF counters with inner IP length */ - vlib_increment_combined_counter (rx_counter, vm->thread_index, itf_sw_idx0, - frag_cnt0, ilen0); - vlib_increment_combined_counter (rx_counter, vm->thread_index, itf_sw_idx1, - frag_cnt1, ilen1); - vlib_increment_combined_counter (rx_counter, vm->thread_index, itf_sw_idx2, - frag_cnt2, ilen2); - vlib_increment_combined_counter (rx_counter, vm->thread_index, itf_sw_idx3, - frag_cnt3, ilen3); + if (reass_fail) + { + for (i = frag_cnt; i > 1; i--) + { + buffs[*buffer_next_index - frag_cnt]->error = + node->errors[OCT_RX_NODE_CTR_ERR_NO_TUNNEL]; + next[*buffer_next_index - frag_cnt] = + VNET_DEV_ETH_RX_PORT_NEXT_DROP; + } + } + } + else + /* Update ITF counters with inner IP length */ + vlib_increment_combined_counter (rx_counter, vm->thread_index, itf_sw_idx, + frag_cnt, ilen); } -static_always_inline void -oct_rx_ipsec_update_counters (vlib_main_t *vm, vlib_buffer_t *b, u32 ilen, - u8 frag_cnt, u32 idx) +static_always_inline u8 +oct_rx_ipsec_reassembly (vlib_main_t *vm, vlib_node_runtime_t *node, + vlib_buffer_template_t *bt, + struct cpt_cn10k_parse_hdr_s *cpt_hdr, + oct_nix_rx_cqe_desc_t *d, vlib_buffer_t *b, + vlib_buffer_t **buf, u16 *next, + u16 *buffer_next_index, u32 *olen, u32 *esp_len, + u32 l2_ol3_hdr_size, const u64 fp_flags) { - vlib_combined_counter_main_t *rx_counter; - ipsec_main_t *im = &ipsec_main; - oct_ipsec_session_t *session; - struct roc_ot_ipsec_inb_sa *roc_sa; - oct_ipsec_inb_sa_priv_data_t *inb_sa_priv; - u32 sa_idx, itf_sw_idx; - vnet_interface_main_t *vim; - oct_ipsec_main_t *oim = &oct_ipsec_main; - oct_inl_dev_main_t *oidm = &oct_inl_dev_main; - vnet_main_t *vnm; - - vnm = im->vnet_main; - vim = &vnm->interface_main; - rx_counter = vim->combined_sw_if_counters + VNET_INTERFACE_COUNTER_RX; - - roc_sa = roc_nix_inl_ot_ipsec_inb_sa (oidm->inb_sa_base, idx); - inb_sa_priv = roc_nix_inl_ot_ipsec_inb_sa_sw_rsvd (roc_sa); - - sa_idx = (u32) inb_sa_priv->user_data; - - vnet_buffer (b)->ipsec.sad_index = sa_idx; - ASSERT (sa_idx < vec_len (oim->inline_ipsec_sessions)); - - session = pool_elt_at_index (oim->inline_ipsec_sessions, sa_idx); - itf_sw_idx = session->itf_sw_idx; - /* - * Check if itf_sw_idx is populated already. First packet on the SA - * populates the itf_sw_idx in the SA session. - */ - if (PREDICT_FALSE (itf_sw_idx == ~0)) - itf_sw_idx = oct_ipsec_update_itf_sw_idx (session, sa_idx); + u8 frag_cnt = 1; + u32 idx = cpt_hdr->w0.cookie; - /* Update IPsec counters with inner IP length */ - vlib_increment_combined_counter (&ipsec_sa_counters, vm->thread_index, - sa_idx, frag_cnt, ilen); - - /* Update ITF counters with inner IP length */ - vlib_increment_combined_counter (rx_counter, vm->thread_index, itf_sw_idx, - frag_cnt, ilen); + if ((cpt_hdr->w0.num_frags) && !(cpt_hdr->w0.reas_sts)) + frag_cnt = oct_rx_ipsec_reassembly_success (vm, bt, cpt_hdr, d, b, olen, + esp_len, l2_ol3_hdr_size); + else if (cpt_hdr->w0.reas_sts) + { + frag_cnt = oct_rx_ipsec_reassembly_failure ( + vm, bt, cpt_hdr, d, buf, next, buffer_next_index, olen, esp_len, + l2_ol3_hdr_size, fp_flags); + oct_rx_ipsec_update_counters (vm, node, buf, next, buffer_next_index, + *esp_len, frag_cnt, idx, 1); + return frag_cnt; + } + oct_rx_ipsec_update_counters (vm, node, buf, next, buffer_next_index, + *esp_len, frag_cnt, idx, 0); + return frag_cnt; } static_always_inline u8 @@ -948,7 +819,7 @@ oct_rx_inl_ipsec_vlib_from_cq ( u16 *next, u16 *buffer_next_index, const u64 fp_flags) { union nix_rx_parse_u *orig_rxp, *rxp; - u32 is_fail, olen, esp_sz, l2_ol3_sz, idx; + u32 is_fail, olen, esp_sz, l2_ol3_sz; u64 *wqe_ptr; u8 frag_cnt; @@ -984,12 +855,9 @@ oct_rx_inl_ipsec_vlib_from_cq ( b[0]->current_length = oct_get_len_from_meta (cpt_hdr, d[0].parse.w[0], d[0].parse.w[4]); - frag_cnt = oct_rx_ipsec_reassembly (vm, bt, cpt_hdr, &d[0], b[0], buffs, - next, buffer_next_index, &olen, - &esp_sz, l2_ol3_sz, fp_flags); - - idx = cpt_hdr->w0.cookie; - oct_rx_ipsec_update_counters (vm, b[0], esp_sz, frag_cnt, idx); + frag_cnt = oct_rx_ipsec_reassembly (vm, node, bt, cpt_hdr, &d[0], b[0], + buffs, next, buffer_next_index, + &olen, &esp_sz, l2_ol3_sz, fp_flags); } oct_rx_ipsec_attach_tail (vm, ctx, orig_rxp, bt, b[0]); ctx->n_rx_bytes += olen; @@ -1080,7 +948,6 @@ oct_rx_batch (vlib_main_t *vm, vlib_node_runtime_t *node, u32 olen0, olen1, olen2, olen3; u32 esp_sz0, esp_sz1, esp_sz2, esp_sz3; u32 l2_ol3_sz0, l2_ol3_sz1, l2_ol3_sz2, l2_ol3_sz3; - u32 idx0, idx1, idx2, idx3; vlib_buffer_t *b[4]; vlib_buffer_t **buffs = buffers + ctx->buffer_start_index; u16 *next = ctx->next + ctx->buffer_start_index; @@ -1243,40 +1110,30 @@ oct_rx_batch (vlib_main_t *vm, vlib_node_runtime_t *node, buffs[buffer_next_index] = b[0]; buffer_next_index += 1; frag_cnt0 = oct_rx_ipsec_reassembly ( - vm, &bt, cpt_hdr0, &d[0], b[0], buffs, next, + vm, node, &bt, cpt_hdr0, &d[0], b[0], buffs, next, &buffer_next_index, &olen0, &esp_sz0, l2_ol3_sz0, fp_flags); buffs[buffer_next_index] = b[1]; next[buffer_next_index] = ctx->next_index; buffer_next_index += 1; frag_cnt1 = oct_rx_ipsec_reassembly ( - vm, &bt, cpt_hdr1, &d[1], b[1], buffs, next, + vm, node, &bt, cpt_hdr1, &d[1], b[1], buffs, next, &buffer_next_index, &olen1, &esp_sz1, l2_ol3_sz1, fp_flags); buffs[buffer_next_index] = b[2]; next[buffer_next_index] = ctx->next_index; buffer_next_index += 1; frag_cnt2 = oct_rx_ipsec_reassembly ( - vm, &bt, cpt_hdr2, &d[2], b[2], buffs, next, + vm, node, &bt, cpt_hdr2, &d[2], b[2], buffs, next, &buffer_next_index, &olen2, &esp_sz2, l2_ol3_sz2, fp_flags); buffs[buffer_next_index] = b[3]; next[buffer_next_index] = ctx->next_index; buffer_next_index += 1; frag_cnt3 = oct_rx_ipsec_reassembly ( - vm, &bt, cpt_hdr3, &d[3], b[3], buffs, next, + vm, node, &bt, cpt_hdr3, &d[3], b[3], buffs, next, &buffer_next_index, &olen3, &esp_sz3, l2_ol3_sz3, fp_flags); - idx0 = cpt_hdr0->w0.cookie; - idx1 = cpt_hdr1->w0.cookie; - idx2 = cpt_hdr2->w0.cookie; - idx3 = cpt_hdr3->w0.cookie; - - oct_rx_ipsec_update_counters_x4 ( - vm, b[0], esp_sz0, frag_cnt0, idx0, b[1], esp_sz1, frag_cnt1, - idx1, b[2], esp_sz2, frag_cnt2, idx2, b[3], esp_sz3, frag_cnt3, - idx3); - oct_rx_ipsec_attach_tail (vm, ctx, orig_rxp0, &bt, b[0]); oct_rx_ipsec_attach_tail (vm, ctx, orig_rxp1, &bt, b[1]); oct_rx_ipsec_attach_tail (vm, ctx, orig_rxp2, &bt, b[2]); @@ -1305,13 +1162,9 @@ oct_rx_batch (vlib_main_t *vm, vlib_node_runtime_t *node, cpt_hdr0, d[0].parse.w[0], d[0].parse.w[4]); frag_cnt0 = oct_rx_ipsec_reassembly ( - vm, &bt, cpt_hdr0, &d[0], b[0], buffs, next, + vm, node, &bt, cpt_hdr0, &d[0], b[0], buffs, next, &buffer_next_index, &olen0, &esp_sz0, l2_ol3_sz0, fp_flags); - - idx0 = cpt_hdr0->w0.cookie; - oct_rx_ipsec_update_counters (vm, b[0], esp_sz0, frag_cnt0, - idx0); } /* Success and Failure both cases can be multi seg */ oct_rx_ipsec_attach_tail (vm, ctx, orig_rxp0, &bt, b[0]); @@ -1336,12 +1189,9 @@ oct_rx_batch (vlib_main_t *vm, vlib_node_runtime_t *node, b[1]->current_length = oct_get_len_from_meta ( cpt_hdr1, d[1].parse.w[0], d[1].parse.w[4]); frag_cnt1 = oct_rx_ipsec_reassembly ( - vm, &bt, cpt_hdr1, &d[1], b[1], buffs, next, + vm, node, &bt, cpt_hdr1, &d[1], b[1], buffs, next, &buffer_next_index, &olen1, &esp_sz1, l2_ol3_sz1, fp_flags); - idx1 = cpt_hdr1->w0.cookie; - oct_rx_ipsec_update_counters (vm, b[1], esp_sz1, frag_cnt1, - idx1); } oct_rx_ipsec_attach_tail (vm, ctx, orig_rxp1, &bt, b[1]); @@ -1365,12 +1215,9 @@ oct_rx_batch (vlib_main_t *vm, vlib_node_runtime_t *node, b[2]->current_length = oct_get_len_from_meta ( cpt_hdr2, d[2].parse.w[0], d[2].parse.w[4]); frag_cnt2 = oct_rx_ipsec_reassembly ( - vm, &bt, cpt_hdr2, &d[2], b[2], buffs, next, + vm, node, &bt, cpt_hdr2, &d[2], b[2], buffs, next, &buffer_next_index, &olen2, &esp_sz2, l2_ol3_sz2, fp_flags); - idx2 = cpt_hdr2->w0.cookie; - oct_rx_ipsec_update_counters (vm, b[2], esp_sz2, frag_cnt2, - idx2); } oct_rx_ipsec_attach_tail (vm, ctx, orig_rxp2, &bt, b[2]); @@ -1395,12 +1242,9 @@ oct_rx_batch (vlib_main_t *vm, vlib_node_runtime_t *node, cpt_hdr3, d[3].parse.w[0], d[3].parse.w[4]); frag_cnt3 = oct_rx_ipsec_reassembly ( - vm, &bt, cpt_hdr3, &d[3], b[3], buffs, next, + vm, node, &bt, cpt_hdr3, &d[3], b[3], buffs, next, &buffer_next_index, &olen3, &esp_sz3, l2_ol3_sz3, fp_flags); - idx3 = cpt_hdr3->w0.cookie; - oct_rx_ipsec_update_counters (vm, b[3], esp_sz3, frag_cnt3, - idx3); } oct_rx_ipsec_attach_tail (vm, ctx, orig_rxp3, &bt, b[3]); } From 81b74032572465e7ebcf465e1d6e4237a9302abf Mon Sep 17 00:00:00 2001 From: Monendra Singh Kushwaha Date: Tue, 17 Jun 2025 12:29:06 +0530 Subject: [PATCH 238/271] octeon: add debug cli to set aura available count Type: feature Signed-off-by: Monendra Singh Kushwaha Change-Id: I0a33d4182bb47dd3d79b369d5af0e6686d145835 Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/155857 Tested-by: sa_ip-toolkits-Jenkins Reviewed-by: Nithin Kumar Dabilpuram (cherry picked from commit cc430c2f5fcfd9e559a8ce2889850c079b024712) Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/156438 Tested-by: sa_ip-sw-jenkins --- src/plugins/dev_octeon/cli.c | 70 ++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/src/plugins/dev_octeon/cli.c b/src/plugins/dev_octeon/cli.c index a70c3f5772..e1bb648d22 100644 --- a/src/plugins/dev_octeon/cli.c +++ b/src/plugins/dev_octeon/cli.c @@ -285,3 +285,73 @@ VLIB_CLI_COMMAND (oct_ipsec_inline_counters_clear_command, static) = { .short_help = "clear ipsec inline counters", .function = oct_ipsec_inline_counters_clear_command_fn, }; + +static clib_error_t * +oct_aura_available_command_fn (vlib_main_t *vm, unformat_input_t *input, + vlib_cli_command_t *cmd) +{ + vnet_dev_main_t *dm = &vnet_dev_main; + oct_rxq_t *crq; + int i; + + if (oct_main.use_single_rx_aura && oct_main.rx_aura_handle) + vlib_cli_output (vm, "rx queue aura 0x%llx avl_count %llu\n\n", + oct_main.rx_aura_handle, + roc_npa_aura_op_available (oct_main.rx_aura_handle)); + + pool_foreach_pointer (dev, dm->devices) + { + oct_device_t *od = vnet_dev_get_data (dev); + + if (od->type == OCT_DEVICE_TYPE_RVU_PF || + od->type == OCT_DEVICE_TYPE_RVU_VF || + od->type == OCT_DEVICE_TYPE_SDP_VF || + od->type == OCT_DEVICE_TYPE_LBK_VF) + { + vlib_cli_output (vm, "Interface: %U", format_vnet_dev_log, dev, 0); + vlib_cli_output (vm, "%-.25s", ul); + if (!oct_main.use_single_rx_aura) + { + for (i = 0; i < dev->ports[0]->intf.num_rx_queues; i++) + { + crq = + vnet_dev_get_rx_queue_data (dev->ports[0]->rx_queues[i]); + vlib_cli_output ( + vm, "rx queue %d aura 0x%llx avl_count %llu\n", i, + crq->aura_handle, + roc_npa_aura_op_available (crq->aura_handle)); + } + } + for (i = 0; i < dev->ports[0]->intf.num_tx_queues; i++) + { + vlib_cli_output ( + vm, "tx queue %d aura %x avl_count %d\n", i, + od->ctqs[i]->aura_handle, + roc_npa_aura_op_available (od->ctqs[i]->aura_handle)); + } + if (oct_main.inl_dev_initialized && roc_model_is_cn10k ()) + { + crq = vnet_dev_get_rx_queue_data (dev->ports[0]->rx_queues[0]); + vlib_cli_output ( + vm, "meta_aura_handle %x avl_count %d\n", + crq->rq.meta_aura_handle, + roc_npa_aura_op_available (crq->rq.meta_aura_handle)); + } + vlib_cli_output (vm, "\n"); + } + } + return 0; +} + +/*? + * This command displays OCTEON aura avaialbe counts + * + * @cliexpar + * @cliexstart{show octeon aura available} + * @cliexend +?*/ +VLIB_CLI_COMMAND (oct_aura_available_command, static) = { + .path = "show octeon aura available", + .short_help = "show octeon aura available", + .function = oct_aura_available_command_fn, +}; From 28893a1fadd74e6318e70dfc59b435bda7d7f400 Mon Sep 17 00:00:00 2001 From: Monendra Singh Kushwaha Date: Mon, 23 Jun 2025 14:23:35 +0530 Subject: [PATCH 239/271] octeon: update dlen for non-chained packets Type: fix JIRA: https://essjira.marvell.com/browse/IPBUSW-70894 Signed-off-by: Monendra Singh Kushwaha Change-Id: Ibbe10a2710d0e0690c4af57749cd1359d409967b Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/156259 Reviewed-by: Nithin Kumar Dabilpuram Tested-by: sa_ip-toolkits-Jenkins (cherry picked from commit be3d3ac89ea040dc44327d72ec0eb2d69b263049) Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/156817 Tested-by: sa_ip-sw-jenkins --- src/plugins/dev_octeon/tx_node.c | 103 +++++++++++++++++++------------ 1 file changed, 64 insertions(+), 39 deletions(-) diff --git a/src/plugins/dev_octeon/tx_node.c b/src/plugins/dev_octeon/tx_node.c index cfca955d2b..86b2a298e6 100644 --- a/src/plugins/dev_octeon/tx_node.c +++ b/src/plugins/dev_octeon/tx_node.c @@ -630,6 +630,7 @@ oct_ipsec_append_next_buffer (vlib_main_t *vm, vlib_buffer_t *buffer, tmp = vlib_get_buffer (vm, buffer_index); buffer->next_buffer = buffer_index; buffer->flags |= VLIB_BUFFER_NEXT_PRESENT; + buffer->total_length_not_including_first_buffer = 0; tmp->current_length += bytes_to_append; } @@ -661,22 +662,26 @@ oct_ipsec_fill_sg2_buf (vlib_main_t *vm, struct roc_sg2list_comp *list, int i, static_always_inline int oct_ipsec_outb_prepare_sg2_list (vlib_main_t *vm, vlib_buffer_t *b, struct cpt_inst_s *inst, u32 bytes_to_append, - u32 dlen, void *m_data) + u32 dlen, + oct_ipsec_outbound_pkt_meta_t **pkt_meta, + u64 *n_dwords, oct_ipsec_session_t *sess) { u16 buffer_data_size = vlib_buffer_get_default_data_size (vm); struct roc_sg2list_comp *scatter_comp, *gather_comp; + void *m_data = (void *) pkt_meta[0]->sg_buffer; + union nix_send_sg_s *sg; union cpt_inst_w5 cpt_inst_w5; union cpt_inst_w6 cpt_inst_w6; vlib_buffer_t *last_buf = b; - int i; + int n_segs; /* Input Gather List */ - i = 0; + n_segs = 0; gather_comp = (struct roc_sg2list_comp *) ((uint8_t *) m_data + 64); - i = oct_ipsec_fill_sg2_buf (vm, gather_comp, i, &last_buf); + n_segs = oct_ipsec_fill_sg2_buf (vm, gather_comp, n_segs, &last_buf); - cpt_inst_w5.s.gather_sz = ((i + 2) / 3); + cpt_inst_w5.s.gather_sz = ((n_segs + 2) / 3); if ((bytes_to_append + last_buf->current_length) > buffer_data_size) { @@ -691,12 +696,12 @@ oct_ipsec_outb_prepare_sg2_list (vlib_main_t *vm, vlib_buffer_t *b, last_buf = b; /* Output Gather List */ - i = 0; + n_segs = 0; scatter_comp = (struct roc_sg2list_comp *) ((uint8_t *) m_data); - i = oct_ipsec_fill_sg2_buf (vm, scatter_comp, i, &last_buf); + n_segs = oct_ipsec_fill_sg2_buf (vm, scatter_comp, n_segs, &last_buf); - cpt_inst_w6.s.scatter_sz = ((i + 2) / 3); + cpt_inst_w6.s.scatter_sz = ((n_segs + 2) / 3); cpt_inst_w5.s.dptr = (uint64_t) gather_comp; cpt_inst_w6.s.rptr = (uint64_t) scatter_comp; @@ -708,7 +713,16 @@ oct_ipsec_outb_prepare_sg2_list (vlib_main_t *vm, vlib_buffer_t *b, b->total_length_not_including_first_buffer += bytes_to_append; - return i; + sg = (union nix_send_sg_s *) (pkt_meta[0]->nixtx + 2); + inst->w0.u64 = (uint64_t) vnet_buffer (b)->l3_hdr_offset << 16; + inst->w0.u64 |= NIX_NB_SEGS_TO_SEGDW (n_segs); + inst->w0.u64 |= + (((int64_t) pkt_meta[0]->nixtx - (int64_t) inst->dptr) & 0xFFFFF) << 32; + n_dwords[0] = (n_segs % 3) + (n_segs / 3) * 2; + sg[0].subdc = NIX_SUBDC_SG; + sg[4].subdc = NIX_SUBDC_SG; + + return n_segs; } static_always_inline u32 @@ -759,52 +773,63 @@ oct_prepare_ipsec_inst (vlib_main_t *vm, vlib_buffer_t *b, u64 sq_handle, struct nix_send_hdr_s *send_hdr; union nix_send_sg_s *sg; u64 n_segs; - u16 total_length; + u16 total_length, dlen_adj; u16 l3_hdr_offset = vnet_buffer (b)->l3_hdr_offset; - u32 dlen = b->current_length + b->total_length_not_including_first_buffer - - l3_hdr_offset; - u32 rlen = oct_ipsec_rlen_get (&sess->encap, dlen); - u16 dlen_adj = rlen - dlen; - u32 sa_bytes = oct_ipsec_esp_add_footer_and_icv (&sess->encap, rlen); + u32 dlen, rlen, sa_bytes; send_hdr = (struct nix_send_hdr_s *) pkt_meta[0]->nixtx; - sg = (union nix_send_sg_s *) (pkt_meta[0]->nixtx + 2); - if (b->flags & VLIB_BUFFER_NEXT_PRESENT || rlen > buffer_data_size) + if (b->flags & VLIB_BUFFER_NEXT_PRESENT) { total_length = b->current_length + b->total_length_not_including_first_buffer; + dlen = total_length - l3_hdr_offset; + rlen = oct_ipsec_rlen_get (&sess->encap, dlen); + dlen_adj = rlen - dlen; + inst->w4.u64 = sess->inst.w4.u64; n_segs = oct_ipsec_outb_prepare_sg2_list ( - vm, b, inst, dlen_adj, total_length, (void *) pkt_meta[0]->sg_buffer); - - inst->w0.u64 = (uint64_t) l3_hdr_offset << 16; - inst->w0.u64 |= NIX_NB_SEGS_TO_SEGDW (n_segs); - inst->w0.u64 |= - (((int64_t) pkt_meta[0]->nixtx - (int64_t) inst->dptr) & 0xFFFFF) - << 32; - n_dwords[0] = (n_segs % 3) + (n_segs / 3) * 2; - sg[0].subdc = NIX_SUBDC_SG; - sg[4].subdc = NIX_SUBDC_SG; + vm, b, inst, dlen_adj, total_length, pkt_meta, n_dwords, sess); } else { - inst->dptr = (u64) ((u8 *) vlib_buffer_get_current (b) + l3_hdr_offset); - inst->rptr = inst->dptr; - /* Set w0 nixtx_offset */ - inst->w0.u64 |= - (((int64_t) pkt_meta[0]->nixtx - (int64_t) inst->dptr) & 0xFFFFF) - << 32; - inst->w0.u64 |= 1; - inst->w4.u64 = sess->inst.w4.u64 | dlen; - - b->current_length += dlen_adj; - n_segs = oct_get_tx_vlib_buf_segs (vm, b); - n_dwords[0] = oct_add_sg_list (sg, b, n_segs); + dlen = b->current_length - l3_hdr_offset; + + rlen = oct_ipsec_rlen_get (&sess->encap, dlen); + dlen_adj = rlen - dlen; + + if (rlen > buffer_data_size) + { + inst->w4.u64 = sess->inst.w4.u64; + + n_segs = oct_ipsec_outb_prepare_sg2_list (vm, b, inst, dlen_adj, + b->current_length, + pkt_meta, n_dwords, sess); + } + else + { + sg = (union nix_send_sg_s *) (pkt_meta[0]->nixtx + 2); + + inst->dptr = + (u64) ((u8 *) vlib_buffer_get_current (b) + l3_hdr_offset); + inst->rptr = inst->dptr; + /* Set w0 nixtx_offset */ + inst->w0.u64 |= + (((int64_t) pkt_meta[0]->nixtx - (int64_t) inst->dptr) & 0xFFFFF) + << 32; + inst->w0.u64 |= 1; + inst->w4.u64 = sess->inst.w4.u64 | dlen; + + b->current_length += dlen_adj; + n_segs = oct_get_tx_vlib_buf_segs (vm, b); + n_dwords[0] = oct_add_sg_list (sg, b, n_segs); + } } oct_add_send_hdr (send_hdr, b, aura_handle, sq_handle, n_dwords[0]); + + sa_bytes = oct_ipsec_esp_add_footer_and_icv (&sess->encap, rlen); vlib_increment_combined_counter ( &ipsec_sa_counters, vlib_get_thread_index (), vnet_buffer (b)->ipsec.sad_index, 1, sa_bytes); From 63c151b2ab965257d9ebd351daaf1e6dc0421387 Mon Sep 17 00:00:00 2001 From: Alok Mishra Date: Tue, 10 Jun 2025 11:08:32 +0000 Subject: [PATCH 240/271] octeon: enable 3des-cbc crypto algorithm This patch adds support for 3des-cbc algorithm for Inline IPSec processing in dev_octeon plugin. Type: feature Signed-off-by: Alok Mishra Change-Id: I00b38556a673c4b123cc3ade0d1b2594e770b3f9 Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/155536 Tested-by: sa_ip-toolkits-Jenkins Reviewed-by: Monendra Singh Kushwaha (cherry picked from commit f4f08e9a46eeee4a1c311b809d640fccfd427664) Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/156912 Tested-by: sa_ip-sw-jenkins --- src/plugins/dev_octeon/ipsec.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/plugins/dev_octeon/ipsec.c b/src/plugins/dev_octeon/ipsec.c index 1b4ffbc7dc..fbb7b71507 100644 --- a/src/plugins/dev_octeon/ipsec.c +++ b/src/plugins/dev_octeon/ipsec.c @@ -131,6 +131,9 @@ oct_ipsec_sa_common_param_fill (union roc_ot_ipsec_sa_word2 *w2, case IPSEC_CRYPTO_ALG_AES_CBC_256: w2->s.enc_type = ROC_IE_SA_ENC_AES_CBC; break; + case IPSEC_CRYPTO_ALG_3DES_CBC: + w2->s.enc_type = ROC_IE_SA_ENC_3DES_CBC; + break; default: clib_warning ("Unsupported encryption algorithm"); return -1; @@ -743,6 +746,9 @@ oct_ipsec_check_support (ipsec_sa_t *sa) case IPSEC_CRYPTO_ALG_AES_CTR_256: is_cipher_algo_supported = hw_caps.aes; break; + case IPSEC_CRYPTO_ALG_3DES_CBC: + is_cipher_algo_supported = hw_caps.des; + break; default: is_cipher_algo_supported = 0; break; From 589778561b6211ce31550c572c365cdc1b7f13b7 Mon Sep 17 00:00:00 2001 From: Monendra Singh Kushwaha Date: Mon, 30 Jun 2025 18:23:17 +0000 Subject: [PATCH 241/271] octeon: set backpressure config for inline ipsec Type: fix JIRA: https://essjira.marvell.com/browse/IPBUSW-71340 Signed-off-by: Monendra Singh Kushwaha Change-Id: Iff1be0bcfdfd67ca32032f60f910a46f54eaa377 Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/156883 Tested-by: sa_ip-toolkits-Jenkins Reviewed-by: Nithin Kumar Dabilpuram (cherry picked from commit aef87b3b8edf8aad3bb34db1be2f5fe363fc41a1) Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/156915 Tested-by: sa_ip-sw-jenkins --- src/plugins/dev_octeon/port.c | 33 ++++++++++++++++++++++++--------- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/src/plugins/dev_octeon/port.c b/src/plugins/dev_octeon/port.c index 6494f354a8..28b72b7d23 100644 --- a/src/plugins/dev_octeon/port.c +++ b/src/plugins/dev_octeon/port.c @@ -160,6 +160,7 @@ oct_parse_switch_hdr_type (const char *value) vnet_dev_rv_t oct_port_init (vlib_main_t *vm, vnet_dev_port_t *port) { + oct_inl_dev_main_t *inl_main = &oct_inl_dev_main; vnet_dev_t *dev = port->dev; oct_device_t *cd = vnet_dev_get_data (dev); oct_port_t *cp = vnet_dev_get_port_data (port); @@ -261,6 +262,29 @@ oct_port_init (vlib_main_t *vm, vnet_dev_port_t *port) } cp->npc_initialized = 1; + /* Configure pause frame flow control*/ + if (is_pause_frame_enable && + (rv = oct_port_pause_flow_control_init (vm, port))) + { + oct_port_deinit (vm, port); + return rv; + } + + if (inl_main->inl_dev) + { + struct roc_nix_fc_cfg fc_cfg; + + fc_cfg.type = ROC_NIX_FC_RXCHAN_CFG; + fc_cfg.rxchan_cfg.enable = true; + rrv = roc_nix_fc_config_set (nix, &fc_cfg); + if (rrv) + { + rv = oct_roc_err (dev, rrv, "roc_nix_fc_config_set failed"); + oct_port_deinit (vm, port); + return rv; + } + } + foreach_vnet_dev_port_rx_queue (q, port) total_sz += q->size; @@ -294,15 +318,6 @@ oct_port_init (vlib_main_t *vm, vnet_dev_port_t *port) return rv; } - /* Configure pause frame flow control*/ - - if (is_pause_frame_enable && - (rv = oct_port_pause_flow_control_init (vm, port))) - { - oct_port_deinit (vm, port); - return rv; - } - if (roc_nix_register_queue_irqs (nix)) { rv = oct_roc_err (dev, rrv, "roc_nix_register_queue_irqs() failed"); From 9bf7e8383d817f604035d94d0f26ba249ab6b806 Mon Sep 17 00:00:00 2001 From: Monendra Singh Kushwaha Date: Wed, 2 Jul 2025 10:00:44 +0000 Subject: [PATCH 242/271] build: update roc version Type: feature Signed-off-by: Monendra Singh Kushwaha Change-Id: Ifc7dd3be3fb34862b50d2b57f55fffe749420ca0 Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/157067 Tested-by: sa_ip-sw-jenkins --- build/external/packages/octeon-roc.mk | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build/external/packages/octeon-roc.mk b/build/external/packages/octeon-roc.mk index 3567352393..4cbfea9a61 100644 --- a/build/external/packages/octeon-roc.mk +++ b/build/external/packages/octeon-roc.mk @@ -2,9 +2,9 @@ # SPDX-License-Identifier: Apache-2.0 # https://spdx.org/licenses/Apache-2.0.html -octeon-roc_version := 25.06 +octeon-roc_version := 25.07 octeon-roc_tarball := v$(octeon-roc_version).tar.gz -octeon-roc_tarball_md5sum := 0498a68e9bd7b0b49a6454f7ec3915fb +octeon-roc_tarball_md5sum := a83cb082df2beb6dc975b23aac58d3f9 octeon-roc_github := https://github.com/MarvellEmbeddedProcessors/marvell-octeon-roc octeon-roc_tarball_strip_dirs := 1 From 96d2d181aeb8026659dd3de3b4bbc00b7e63ec37 Mon Sep 17 00:00:00 2001 From: Monendra Singh Kushwaha Date: Fri, 30 May 2025 11:21:13 +0530 Subject: [PATCH 243/271] octeon: add inline IPsec support for o20 Type: feature JIRA: https://essjira.marvell.com/browse/IPBUSW-69498 Signed-off-by: Monendra Singh Kushwaha Change-Id: Id8959034abb83d991657e287a9be6c9c6bcb6eeb Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/157108 --- src/plugins/dev_octeon/init.c | 11 +- src/plugins/dev_octeon/ipsec.c | 517 +++++++++++++++++++++++++++++-- src/plugins/dev_octeon/ipsec.h | 4 +- src/plugins/dev_octeon/octeon.h | 5 +- src/plugins/dev_octeon/queue.c | 22 +- src/plugins/dev_octeon/rx_node.c | 215 +++++++++---- 6 files changed, 675 insertions(+), 99 deletions(-) diff --git a/src/plugins/dev_octeon/init.c b/src/plugins/dev_octeon/init.c index 8b611afe3f..8511bd0b4c 100644 --- a/src/plugins/dev_octeon/init.c +++ b/src/plugins/dev_octeon/init.c @@ -31,7 +31,7 @@ VLIB_REGISTER_LOG_CLASS (oct_log, static) = { vlib_error_desc_t oct_rx_node_counters[] = { /* clang-format off */ - foreach_octeon10_ipsec_ucc + foreach_octeon_ipsec_ucc foreach_oct_rx_node_counter /* clang-format on */ }; @@ -45,6 +45,12 @@ vnet_dev_node_t oct_rx_node = { .n_error_counters = ARRAY_LEN (oct_rx_node_counters), }; +vnet_dev_node_t oct_o20_rx_node = { + .format_trace = format_oct_rx_trace, + .error_counters = oct_rx_node_counters, + .n_error_counters = ARRAY_LEN (oct_rx_node_counters), +}; + vnet_dev_node_t oct_tx_node = { .format_trace = format_oct_tx_trace, .error_counters = oct_tx_node_counters, @@ -413,6 +419,9 @@ oct_init_nix (vlib_main_t *vm, vnet_dev_t *dev) port_add_args.tx_node = &oct_tx_ipsec_tm_node; } + if (roc_model_is_cn20k ()) + port_add_args.rx_node = &oct_o20_rx_node; + vnet_dev_set_hw_addr_eth_mac (&port_add_args.port.attr.hw_addr, mac_addr); log_info (dev, "MAC address is %U", format_ethernet_address, mac_addr); diff --git a/src/plugins/dev_octeon/ipsec.c b/src/plugins/dev_octeon/ipsec.c index fbb7b71507..ea48f2c742 100644 --- a/src/plugins/dev_octeon/ipsec.c +++ b/src/plugins/dev_octeon/ipsec.c @@ -20,8 +20,55 @@ VLIB_REGISTER_LOG_CLASS (oct_log, static) = { .subclass_name = "ipsec", }; +static inline union cpt_eng_caps +oct_cpt_get_eng_caps (oct_crypto_main_t *ocm) +{ + oct_crypto_dev_t *ocd = ocm->crypto_dev[0]; + + if (roc_model_is_cn20k ()) + return ocd->roc_cpt->hw_caps[CPT_ENG_TYPE_SE]; + else + return ocd->roc_cpt->hw_caps[CPT_ENG_TYPE_IE]; +} + +static inline u32 +oct_ipsec_get_inb_sa_sz () +{ + if (roc_model_is_cn20k ()) + return (sizeof (struct roc_ow_ipsec_inb_sa)); + return (sizeof (struct roc_ot_ipsec_inb_sa)); +} + +static inline u32 +oct_ipsec_get_outb_sa_sz () +{ + if (roc_model_is_cn20k ()) + return (sizeof (struct roc_ow_ipsec_outb_sa)); + return (sizeof (struct roc_ot_ipsec_outb_sa)); +} + +static inline void +oct_ipsec_sa_init (void *sa_dptr, bool is_inb) +{ + if (roc_model_is_cn20k ()) + { + if (is_inb) + roc_ow_ipsec_inb_sa_init (sa_dptr); + else + roc_ow_ipsec_outb_sa_init (sa_dptr); + return; + } + + if (is_inb) + roc_ot_ipsec_inb_sa_init (sa_dptr); + else + roc_ot_ipsec_outb_sa_init (sa_dptr); + + return; +} + static void -oct_cn10k_ipsec_hmac_opad_ipad_gen (ipsec_sa_t *sa, u8 *hmac_opad_ipad) +oct_ipsec_hmac_opad_ipad_gen (ipsec_sa_t *sa, u8 *hmac_opad_ipad) { u8 opad[128] = { [0 ... 127] = 0x5c }; u8 ipad[128] = { [0 ... 127] = 0x36 }; @@ -70,9 +117,13 @@ oct_ipsec_crypto_inst_w7_get (void *sa) union cpt_inst_w7 w7; w7.u64 = 0; - w7.s.egrp = ROC_LEGACY_CPT_DFLT_ENG_GRP_SE_IE; w7.s.ctx_val = 1; + if (roc_model_is_cn20k ()) + w7.s.egrp = ROC_CPT_DFLT_ENG_GRP_SE; + else + w7.s.egrp = ROC_LEGACY_CPT_DFLT_ENG_GRP_SE_IE; + return w7.u64; } @@ -187,7 +238,140 @@ oct_ipsec_sa_common_param_fill (union roc_ot_ipsec_sa_word2 *w2, } } - oct_cn10k_ipsec_hmac_opad_ipad_gen (sa, hmac_opad_ipad); + oct_ipsec_hmac_opad_ipad_gen (sa, hmac_opad_ipad); + + tmp_key = (u64 *) hmac_opad_ipad; + for (i = 0; i < (int) (ROC_CTX_MAX_OPAD_IPAD_LEN / sizeof (u64)); i++) + tmp_key[i] = clib_net_to_host_u64 (tmp_key[i]); + + if (ipsec_sa_is_set_IS_AEAD (sa)) + { + if (IPSEC_CRYPTO_ALG_IS_GCM (sa->crypto_alg)) + clib_memcpy (salt_key, &sa->salt, OCT_ROC_SALT_LEN); + tmp_salt = (u32 *) salt_key; + *tmp_salt = clib_net_to_host_u32 (*tmp_salt); + } + + /* Populate encryption key */ + clib_memcpy (cipher_key, sa->crypto_key.data, sa->crypto_key.len); + tmp_key = (u64 *) cipher_key; + for (i = 0; i < (int) (ROC_CTX_MAX_CKEY_LEN / sizeof (u64)); i++) + tmp_key[i] = clib_net_to_host_u64 (tmp_key[i]); + + w2->s.spi = sa->spi; + + return 0; +} + +static_always_inline i32 +oct_o20_ipsec_sa_common_param_fill (union roc_ow_ipsec_sa_word2 *w2, + u8 *cipher_key, u8 *salt_key, + u8 *hmac_opad_ipad, ipsec_sa_t *sa) +{ + u32 *tmp_salt; + u64 *tmp_key; + int i; + + if (ipsec_sa_is_set_UDP_ENCAP (sa)) + w2->s.encap_type = ROC_IE_OW_SA_ENCAP_UDP; + + /* Set protocol - ESP vs AH */ + if (sa->protocol == IPSEC_PROTOCOL_ESP) + w2->s.protocol = ROC_IE_SA_PROTOCOL_ESP; + else + w2->s.protocol = ROC_IE_SA_PROTOCOL_AH; + + /* Set mode - transport vs tunnel */ + if (ipsec_sa_is_set_IS_TUNNEL (sa)) + w2->s.mode = ROC_IE_SA_MODE_TUNNEL; + else + w2->s.mode = ROC_IE_SA_MODE_TRANSPORT; + + if (ipsec_sa_is_set_IS_CTR (sa)) + { + if (ipsec_sa_is_set_IS_AEAD (sa)) + { + /* AEAD is set for AES_GCM */ + if (IPSEC_CRYPTO_ALG_IS_GCM (sa->crypto_alg)) + { + w2->s.enc_type = ROC_IE_SA_ENC_AES_GCM; + w2->s.auth_type = ROC_IE_SA_AUTH_NULL; + } + else + { + clib_warning ("Unsupported AEAD algorithm"); + return -1; + } + } + else + w2->s.enc_type = ROC_IE_SA_ENC_AES_CTR; + } + else + { + switch (sa->crypto_alg) + { + case IPSEC_CRYPTO_ALG_NONE: + w2->s.enc_type = ROC_IE_SA_ENC_NULL; + break; + case IPSEC_CRYPTO_ALG_AES_CBC_128: + case IPSEC_CRYPTO_ALG_AES_CBC_192: + case IPSEC_CRYPTO_ALG_AES_CBC_256: + w2->s.enc_type = ROC_IE_SA_ENC_AES_CBC; + break; + default: + clib_warning ("Unsupported encryption algorithm"); + return -1; + } + } + + switch (sa->crypto_alg) + { + case IPSEC_CRYPTO_ALG_AES_GCM_128: + case IPSEC_CRYPTO_ALG_AES_CBC_128: + case IPSEC_CRYPTO_ALG_AES_CTR_128: + w2->s.aes_key_len = ROC_IE_SA_AES_KEY_LEN_128; + break; + case IPSEC_CRYPTO_ALG_AES_GCM_192: + case IPSEC_CRYPTO_ALG_AES_CBC_192: + case IPSEC_CRYPTO_ALG_AES_CTR_192: + w2->s.aes_key_len = ROC_IE_SA_AES_KEY_LEN_192; + break; + case IPSEC_CRYPTO_ALG_AES_GCM_256: + case IPSEC_CRYPTO_ALG_AES_CBC_256: + case IPSEC_CRYPTO_ALG_AES_CTR_256: + w2->s.aes_key_len = ROC_IE_SA_AES_KEY_LEN_256; + break; + default: + break; + } + + if (!ipsec_sa_is_set_IS_AEAD (sa)) + { + switch (sa->integ_alg) + { + case IPSEC_INTEG_ALG_NONE: + w2->s.auth_type = ROC_IE_SA_AUTH_NULL; + break; + case IPSEC_INTEG_ALG_SHA1_96: + w2->s.auth_type = ROC_IE_SA_AUTH_SHA1; + break; + case IPSEC_INTEG_ALG_SHA_256_96: + case IPSEC_INTEG_ALG_SHA_256_128: + w2->s.auth_type = ROC_IE_SA_AUTH_SHA2_256; + break; + case IPSEC_INTEG_ALG_SHA_384_192: + w2->s.auth_type = ROC_IE_SA_AUTH_SHA2_384; + break; + case IPSEC_INTEG_ALG_SHA_512_256: + w2->s.auth_type = ROC_IE_SA_AUTH_SHA2_512; + break; + default: + clib_warning ("Unsupported authentication algorithm"); + return -1; + } + } + + oct_ipsec_hmac_opad_ipad_gen (sa, hmac_opad_ipad); tmp_key = (u64 *) hmac_opad_ipad; for (i = 0; i < (int) (ROC_CTX_MAX_OPAD_IPAD_LEN / sizeof (u64)); i++) @@ -256,6 +440,21 @@ oct_ipsec_inb_ctx_size (struct roc_ot_ipsec_inb_sa *sa) return size; } +static size_t +oct_o20_ipsec_inb_ctx_size (struct roc_ow_ipsec_inb_sa *sa) +{ + size_t size; + + /* Variable based on anti-replay window */ + size = offsetof (struct roc_ow_ipsec_inb_sa, ctx) + + offsetof (struct roc_ow_ipsec_inb_ctx_update_reg, ar_winbits); + + if (sa->w0.s.ar_win) + size += (1 << (sa->w0.s.ar_win - 1)) * sizeof (u64); + + return size; +} + static_always_inline void oct_ipsec_common_inst_param_fill (void *sa, oct_ipsec_session_t *sess) { @@ -277,7 +476,7 @@ oct_ipsec_common_inst_param_fill (void *sa, oct_ipsec_session_t *sess) } static i32 -oct_ipsec_inb_session_update (oct_ipsec_session_t *sess, ipsec_sa_t *sa) +oct_o10_ipsec_inb_session_update (oct_ipsec_session_t *sess, ipsec_sa_t *sa) { union roc_ot_ipsec_sa_word2 w2; u32 min_spi, max_spi, spi_mask; @@ -394,6 +593,132 @@ oct_ipsec_inb_session_update (oct_ipsec_session_t *sess, ipsec_sa_t *sa) return 0; } +static i32 +oct_o20_ipsec_inb_session_update (oct_ipsec_session_t *sess, ipsec_sa_t *sa) +{ + union roc_ow_ipsec_sa_word2 w2; + u32 min_spi, max_spi, spi_mask; + struct roc_ow_ipsec_inb_sa *roc_sa; + oct_ipsec_inb_sa_priv_data_t *inb_sa_priv; + union cpt_inst_w4 inst_w4; + union roc_ow_ipsec_inb_param1 param1; + size_t offset; + + /* Ensure SPI is within the range supported by inline pktio device */ + spi_mask = roc_nix_inl_inb_spi_range (NULL, true, &min_spi, &max_spi); + if (sa->spi < min_spi || sa->spi > max_spi) + { + clib_warning ("SPI %u is not within supported range %u-%u", sa->spi, + min_spi, max_spi); + return -1; + } + + roc_sa = (struct roc_ow_ipsec_inb_sa *) roc_nix_inl_inb_sa_get (NULL, true, + sa->spi); + if (!roc_sa) + { + clib_warning ("Failed to create inbound sa session"); + return -1; + } + + inb_sa_priv = roc_nix_inl_ow_ipsec_inb_sa_sw_rsvd (roc_sa); + inb_sa_priv->user_data = sa->stat_index; + + if (ipsec_sa_is_set_UDP_ENCAP (sa)) + { + roc_sa->w10.s.udp_dst_port = 4500; + roc_sa->w10.s.udp_src_port = 4500; + } + + w2.u64 = 0; + int rv = oct_o20_ipsec_sa_common_param_fill ( + &w2, roc_sa->cipher_key, roc_sa->w8.s.salt, roc_sa->hmac_opad_ipad, sa); + if (rv) + return rv; + + oct_ipsec_sa_len_precalc (sa, &sess->encap); + + if (sa->flags & IPSEC_SA_FLAG_USE_ANTI_REPLAY) + roc_sa->w0.s.ar_win = max_log2 (IPSEC_SA_ANTI_REPLAY_WINDOW_SIZE (sa)) - 5; + + /* Set direction and enable ESN (if needed) */ + w2.s.dir = ROC_IE_SA_DIR_INBOUND; + if (ipsec_sa_is_set_USE_ESN (sa)) + w2.s.esn_en = 1; + + /* + * Default options for pkt_out and pkt_fmt are with + * second pass meta and defrag. + */ + roc_sa->w0.s.pkt_format = ROC_IE_OW_SA_PKT_FMT_META; + + if (sa->flags & IPSEC_SA_FLAG_IS_INL_REASSEMBLY) + roc_sa->w0.s.pkt_output = ROC_IE_OW_SA_PKT_OUTPUT_HW_BASED_DEFRAG; + else + roc_sa->w0.s.pkt_output = ROC_IE_OW_SA_PKT_OUTPUT_NO_FRAG; + + roc_sa->w0.s.pkind = ROC_IE_OW_CPT_PKIND; + + offset = offsetof (struct roc_ow_ipsec_inb_sa, ctx); + roc_sa->w0.s.hw_ctx_off = offset / 8; + roc_sa->w0.s.ctx_push_size = roc_sa->w0.s.hw_ctx_off + 1; + + /* Set context size, in number of 128B units following the first 128B */ + roc_sa->w0.s.ctx_size = + (round_pow2 (oct_o20_ipsec_inb_ctx_size (roc_sa), 128) >> 7) - 1; + + /* Save SA index/SPI in cookie for now */ + roc_sa->w1.s.cookie = sa->spi & spi_mask; + + /* Enable SA */ + w2.s.valid = 1; + roc_sa->w2.u64 = w2.u64; + + asm volatile ("dmb oshst" ::: "memory"); + + oct_ipsec_common_inst_param_fill (roc_sa, sess); + + /* Populate word4 in CPT instruction template */ + inst_w4.u64 = 0; + inst_w4.s.opcode_major = ROC_IE_OW_MAJOR_OP_PROCESS_INBOUND_IPSEC; + param1.u16 = 0; + /* Disable IP checksum verification by default */ + param1.s.ip_csum_disable = ROC_IE_OW_SA_INNER_PKT_IP_CSUM_DISABLE; + /* Disable L4 checksum verification by default */ + param1.s.l4_csum_disable = ROC_IE_OW_SA_INNER_PKT_L4_CSUM_DISABLE; + param1.s.esp_trailer_disable = 0; + inst_w4.s.param1 = param1.u16; + sess->inst.w4.u64 = inst_w4.u64; + + rv = roc_nix_inl_ctx_write (NULL, roc_sa, roc_sa, true, + sizeof (struct roc_ow_ipsec_inb_sa)); + if (rv) + { + clib_warning ("roc_nix_inl_ctx_write failed with '%s' error", + roc_error_msg_get (rv)); + return rv; + } + + rv = roc_nix_inl_sa_sync (NULL, roc_sa, true, ROC_NIX_INL_SA_OP_FLUSH); + if (rv) + { + clib_warning ( + "roc_nix_inl_sa_sync flush operation failed with '%s' error", + roc_error_msg_get (rv)); + return rv; + } + + return 0; +} + +static i32 +oct_ipsec_inb_session_update (oct_ipsec_session_t *sess, ipsec_sa_t *sa) +{ + if (roc_model_is_cn20k ()) + return oct_o20_ipsec_inb_session_update (sess, sa); + return oct_o10_ipsec_inb_session_update (sess, sa); +} + int oct_ipsec_outb_sa_idx_get (oct_device_t *od, u32 *index, u32 spi) { @@ -445,7 +770,7 @@ oct_ipsec_get_oct_device_from_outb_sa (u32 sa_index) } static_always_inline i32 -oct_ipsec_outb_session_update (oct_ipsec_session_t *sess, ipsec_sa_t *sa) +oct_o10_ipsec_outb_session_update (oct_ipsec_session_t *sess, ipsec_sa_t *sa) { oct_main_t *om = &oct_main; union roc_ot_ipsec_outb_param1 param1; @@ -589,6 +914,159 @@ oct_ipsec_outb_session_update (oct_ipsec_session_t *sess, ipsec_sa_t *sa) return 0; } +static_always_inline i32 +oct_o20_ipsec_outb_session_update (oct_ipsec_session_t *sess, ipsec_sa_t *sa) +{ + oct_main_t *om = &oct_main; + union roc_ow_ipsec_outb_param1 param1; + struct roc_ow_ipsec_outb_sa *out_sa; + oct_ipsec_outb_sa_priv_data_t *outb_priv; + union roc_ow_ipsec_sa_word2 w2; + union cpt_inst_w4 inst_w4; + u32 sa_idx; + u64 *ipv6_addr; + size_t offset; + int rv = 0, i = 0; + + vec_validate_aligned (sess->out_sa, vec_len (om->oct_dev), + CLIB_CACHE_LINE_BYTES); + + pool_foreach_pointer (oct_dev, om->oct_dev) + { + /* Alloc an sa index */ + rv = oct_ipsec_outb_sa_idx_get (oct_dev, &sa_idx, sa->spi); + if (rv) + return rv; + + out_sa = sess->out_sa[i] = + roc_nix_inl_ow_ipsec_outb_sa (oct_dev->outb.sa_base, sa_idx); + + outb_priv = roc_nix_inl_ow_ipsec_outb_sa_sw_rsvd (out_sa); + outb_priv->sa_idx = sa_idx; + + roc_ow_ipsec_outb_sa_init (out_sa); + + w2.u64 = 0; + rv = oct_o20_ipsec_sa_common_param_fill (&w2, out_sa->cipher_key, + out_sa->iv.s.salt, + out_sa->hmac_opad_ipad, sa); + if (rv) + return rv; + + /* Set direction and enable ESN (if needed) */ + w2.s.dir = ROC_IE_SA_DIR_OUTBOUND; + if (ipsec_sa_is_set_USE_ESN (sa)) + out_sa->w0.s.esn_en = 1; + + /* Configure tunnel header generation */ + if (ipsec_sa_is_set_IS_TUNNEL (sa)) + { + if (ipsec_sa_is_set_IS_TUNNEL_V6 (sa)) + { + w2.s.outer_ip_ver = ROC_IE_SA_IP_VERSION_6; + + clib_memcpy (&out_sa->outer_hdr.ipv6.src_addr, + &sa->tunnel.t_src.ip.ip6, sizeof (ip6_address_t)); + clib_memcpy (&out_sa->outer_hdr.ipv6.dst_addr, + &sa->tunnel.t_dst.ip.ip6, sizeof (ip6_address_t)); + + /* Convert host to network byte order of ipv6 address */ + ipv6_addr = (u64 *) &out_sa->outer_hdr.ipv6.src_addr; + *ipv6_addr = clib_host_to_net_u64 (*ipv6_addr); + ipv6_addr++; + *ipv6_addr = clib_host_to_net_u64 (*ipv6_addr); + + ipv6_addr = (u64 *) &out_sa->outer_hdr.ipv6.dst_addr; + *ipv6_addr = clib_host_to_net_u64 (*ipv6_addr); + ipv6_addr++; + *ipv6_addr = clib_host_to_net_u64 (*ipv6_addr); + } + else + { + w2.s.outer_ip_ver = ROC_IE_SA_IP_VERSION_4; + out_sa->outer_hdr.ipv4.src_addr = + clib_host_to_net_u32 (sa->tunnel.t_src.ip.ip4.as_u32); + out_sa->outer_hdr.ipv4.dst_addr = + clib_host_to_net_u32 (sa->tunnel.t_dst.ip.ip4.as_u32); + } + } + + offset = offsetof (struct roc_ow_ipsec_outb_sa, ctx); + out_sa->w0.s.hw_ctx_off = offset / 8; + out_sa->w0.s.ctx_push_size = out_sa->w0.s.hw_ctx_off + 1; + /* Set context size, in number of 128B units following the first 128B */ + out_sa->w0.s.ctx_size = (round_pow2 (offset, 128) >> 7) - 1; + out_sa->w0.s.ctx_hdr_size = 1; + out_sa->w0.s.aop_valid = 1; + + out_sa->w2.u64 = w2.u64; + + if (ipsec_sa_is_set_IS_TUNNEL (sa)) + { + if (sa->tunnel.t_encap_decap_flags & + TUNNEL_ENCAP_DECAP_FLAG_ENCAP_COPY_DF) + out_sa->w2.s.ipv4_df_src_or_ipv6_flw_lbl_src = + ROC_IE_OW_SA_COPY_FROM_INNER_IP_HDR; + if (!sa->tunnel.t_dscp) + out_sa->w2.s.dscp_src = ROC_IE_OW_SA_COPY_FROM_INNER_IP_HDR; + else + { + out_sa->w2.s.dscp_src = ROC_IE_OW_SA_COPY_FROM_SA; + out_sa->w10.s.dscp = sa->tunnel.t_dscp; + } + } + + out_sa->w2.s.ipid_gen = 1; + out_sa->w2.s.iv_src = ROC_IE_OW_SA_IV_SRC_FROM_SA; + out_sa->w2.s.valid = 1; + + asm volatile ("dmb oshst" ::: "memory"); + + oct_ipsec_sa_len_precalc (sa, &sess->encap); + + oct_ipsec_common_inst_param_fill (out_sa, sess); + + /* Populate word4 in CPT instruction template */ + inst_w4.u64 = 0; + inst_w4.s.opcode_major = ROC_IE_OW_MAJOR_OP_PROCESS_OUTBOUND_IPSEC; + param1.u16 = 0; + if (sa->tunnel.t_hop_limit) + param1.s.ttl_or_hop_limit = 1; + + /* Enable IP checksum computation by default */ + param1.s.ip_csum_disable = ROC_IE_OW_SA_INNER_PKT_IP_CSUM_ENABLE; + /* Enable L4 checksum computation by default */ + param1.s.l4_csum_disable = ROC_IE_OW_SA_INNER_PKT_L4_CSUM_ENABLE; + + inst_w4.s.param1 = param1.u16; + sess->inst.w4.u64 = inst_w4.u64; + if (ipsec_sa_is_set_UDP_ENCAP (sa)) + { + out_sa->w10.s.udp_dst_port = 4500; + out_sa->w10.s.udp_src_port = 4500; + } + + rv = roc_nix_inl_ctx_write (oct_dev->nix, out_sa, out_sa, false, + sizeof (struct roc_ow_ipsec_outb_sa)); + if (rv) + { + clib_warning ("roc_nix_inl_ctx_write failed with '%s' error", + roc_error_msg_get (rv)); + return -1; + } + i++; + } + return 0; +} + +static i32 +oct_ipsec_outb_session_update (oct_ipsec_session_t *sess, ipsec_sa_t *sa) +{ + if (roc_model_is_cn20k ()) + return oct_o20_ipsec_outb_session_update (sess, sa); + return oct_o10_ipsec_outb_session_update (sess, sa); +} + static i32 oct_ipsec_session_create (u32 sa_index) { @@ -625,7 +1103,7 @@ oct_ipsec_session_destroy (u32 sa_index) oct_ipsec_main_t *oim = &oct_ipsec_main; ipsec_sa_t *sa = ipsec_sa_get (sa_index); oct_ipsec_session_t *session = NULL; - struct roc_ot_ipsec_inb_sa *roc_sa; + void *roc_sa; void *sa_dptr = NULL; int rv, i = 0; @@ -635,8 +1113,7 @@ oct_ipsec_session_destroy (u32 sa_index) if (sa->flags & IPSEC_SA_FLAG_IS_INBOUND) { - roc_sa = (struct roc_ot_ipsec_inb_sa *) roc_nix_inl_inb_sa_get ( - NULL, true, sa->spi); + roc_sa = (void *) roc_nix_inl_inb_sa_get (NULL, true, sa->spi); if (!roc_sa) { clib_warning ("roc_nix_inl_inb_sa_get failed to get SA for spi %u", @@ -644,12 +1121,12 @@ oct_ipsec_session_destroy (u32 sa_index) return -1; } - sa_dptr = plt_zmalloc (sizeof (struct roc_ot_ipsec_inb_sa), 8); + sa_dptr = plt_zmalloc (oct_ipsec_get_inb_sa_sz (), 8); if (sa_dptr != NULL) { - roc_ot_ipsec_inb_sa_init (sa_dptr); + oct_ipsec_sa_init (sa_dptr, true); rv = roc_nix_inl_ctx_write (NULL, sa_dptr, roc_sa, true, - sizeof (struct roc_ot_ipsec_inb_sa)); + oct_ipsec_get_inb_sa_sz ()); if (rv) { clib_warning ("roc_nix_inl_ctx_write failed - ROC error %s (%d)", @@ -663,13 +1140,13 @@ oct_ipsec_session_destroy (u32 sa_index) { pool_foreach_pointer (oct_dev, om->oct_dev) { - sa_dptr = plt_zmalloc (sizeof (struct roc_ot_ipsec_outb_sa), 8); + sa_dptr = plt_zmalloc (oct_ipsec_get_outb_sa_sz (), 8); if (sa_dptr != NULL) { - roc_ot_ipsec_outb_sa_init (sa_dptr); - rv = roc_nix_inl_ctx_write ( - oct_dev->nix, sa_dptr, session->out_sa[i], false, - sizeof (struct roc_ot_ipsec_outb_sa)); + oct_ipsec_sa_init (sa_dptr, false); + rv = roc_nix_inl_ctx_write (oct_dev->nix, sa_dptr, + session->out_sa[i], false, + oct_ipsec_get_outb_sa_sz ()); if (rv) { clib_warning ( @@ -721,8 +1198,7 @@ static clib_error_t * oct_ipsec_check_support (ipsec_sa_t *sa) { oct_crypto_main_t *ocm = &oct_crypto_main; - oct_crypto_dev_t *ocd = ocm->crypto_dev[0]; - union cpt_eng_caps hw_caps = ocd->roc_cpt->hw_caps[CPT_ENG_TYPE_IE]; + union cpt_eng_caps hw_caps = oct_cpt_get_eng_caps (ocm); u8 is_cipher_algo_supported; u8 is_auth_algo_supported; @@ -825,7 +1301,8 @@ oct_ipsec_inl_dev_inb_cfg (vlib_main_t *vm, vnet_dev_t *dev, } roc_nix_inb_mode_set (cd->nix, true); - roc_nix_inl_inb_set (cd->nix, true); + if (roc_model_is_cn10k ()) + roc_nix_inl_inb_set (cd->nix, true); if ((rrv = roc_nix_reassembly_configure (&rxc_cfg, 1000))) { @@ -1028,6 +1505,8 @@ oct_early_init_inline_ipsec (vlib_main_t *vm, vnet_dev_t *dev) STRUCT_OFFSET_OF (vlib_buffer_t, pre_data) / ROC_ALIGN; inl_dev_main->inl_dev->nb_meta_bufs = bp->n_buffers; inl_dev_main->inl_dev->res_addr_offset = -1; + if (roc_feature_nix_has_inl_multi_queue ()) + inl_dev_main->inl_dev->nb_inb_cptlfs = 1; if ((rrv = roc_nix_inl_dev_init (inl_dev_main->inl_dev)) < 0) { diff --git a/src/plugins/dev_octeon/ipsec.h b/src/plugins/dev_octeon/ipsec.h index dbe3c02b70..b03d5c914b 100644 --- a/src/plugins/dev_octeon/ipsec.h +++ b/src/plugins/dev_octeon/ipsec.h @@ -14,7 +14,7 @@ #define OCT_EXT_HDR_FROM_VLIB_BUFFER(x) \ (((oct_ipsec_outbound_pkt_meta_t *) (x)) - 1) -#define foreach_octeon10_ipsec_ucc \ +#define foreach_octeon_ipsec_ucc \ _ (SUCCESS, success, INFO, "Packet successfully processed") \ _ (ERR_SA_INVAL, err_sa_inval, ERROR, "SA invalid") \ _ (ERR_SA_EXPIRED, err_sa_expired, ERROR, "SA hard-expired") \ @@ -118,7 +118,7 @@ typedef struct { CLIB_CACHE_LINE_ALIGN_MARK (cacheline0); /* Outbound SA */ - struct roc_ot_ipsec_outb_sa **out_sa; + void **out_sa; CLIB_CACHE_LINE_ALIGN_MARK (cacheline1); struct cpt_inst_s inst; u16 sq; diff --git a/src/plugins/dev_octeon/octeon.h b/src/plugins/dev_octeon/octeon.h index 2b6624f4c2..d10f7f8646 100644 --- a/src/plugins/dev_octeon/octeon.h +++ b/src/plugins/dev_octeon/octeon.h @@ -259,7 +259,7 @@ vnet_dev_rv_t oct_txq_get_stats (vlib_main_t *, vnet_dev_port_t *, typedef enum { #define _(f, n, s, d) OCT_RX_NODE_CTR_##f, - foreach_octeon10_ipsec_ucc + foreach_octeon_ipsec_ucc foreach_oct_rx_node_counter #undef _ } oct_rx_node_counter_t; @@ -298,7 +298,8 @@ extern tm_system_t dev_oct_tm_ops; #define foreach_oct_fp_flag \ _ (UNUSED, 0) \ - _ (TRACE_EN, 1) + _ (TRACE_EN, 1) \ + _ (O20, 2) typedef enum { diff --git a/src/plugins/dev_octeon/queue.c b/src/plugins/dev_octeon/queue.c index f6308e8f5f..da069369d4 100644 --- a/src/plugins/dev_octeon/queue.c +++ b/src/plugins/dev_octeon/queue.c @@ -174,16 +174,22 @@ oct_rxq_init (vlib_main_t *vm, vnet_dev_rx_queue_t *rxq, u32 total_sz) (0x7 << 4); log_debug (dev, "RQ %u initialised", crq->cq.qid); - /* Configure inline device rq */ - crq->rq.tag_mask = - 0x0FF00000 | ((uint32_t) OCT_EVENT_TYPE_FRM_INL_DEV << 28); - rrv = roc_nix_inl_dev_rq_get (&crq->rq, 0 /* disable */); - if (rrv) + + if (inl_main->inl_dev) { - clib_warning ("roc_nix_inl_dev_rq_get failed with '%s' error", - roc_error_msg_get (rrv)); + /* Configure inline device rq */ + crq->rq.tag_mask = + 0x0FF00000 | ((uint32_t) OCT_EVENT_TYPE_FRM_INL_DEV << 28); + rrv = roc_nix_inl_dev_rq_get (&crq->rq, 0 /* disable */); + if (rrv) + { + clib_warning ("roc_nix_inl_dev_rq_get failed with '%s' error", + roc_error_msg_get (rrv)); - return -1; + return -1; + } + if (!crq->rq.meta_aura_handle && roc_model_is_cn20k ()) + crq->rq.meta_aura_handle = crq->rq.aura_handle; } return VNET_DEV_OK; diff --git a/src/plugins/dev_octeon/rx_node.c b/src/plugins/dev_octeon/rx_node.c index c0e9b8ca58..58b4c549eb 100644 --- a/src/plugins/dev_octeon/rx_node.c +++ b/src/plugins/dev_octeon/rx_node.c @@ -27,6 +27,31 @@ typedef struct u16 buffer_start_index; } oct_rx_node_ctx_t; +static_always_inline u64 +oct_get_wqe_from_cpt_hdr (union cpt_parse_hdr_u *cpt_hdr, const u64 fp_flags) +{ + if (fp_flags & OCT_FP_FLAG_O20) + return cpt_hdr->u64[1]; + return clib_net_to_host_u64 (cpt_hdr->u64[1]); +} + +static_always_inline void * +oct_ipsec_inb_sa_priv (u32 idx, const u64 fp_flags) +{ + oct_inl_dev_main_t *oidm = &oct_inl_dev_main; + struct roc_ot_ipsec_inb_sa *roc_sa; + + if (fp_flags & OCT_FP_FLAG_O20) + { + struct roc_ow_ipsec_inb_sa *roc_sa; + roc_sa = roc_nix_inl_ow_ipsec_inb_sa (oidm->inb_sa_base, idx); + return roc_nix_inl_ow_ipsec_inb_sa_sw_rsvd (roc_sa); + } + + roc_sa = roc_nix_inl_ot_ipsec_inb_sa (oidm->inb_sa_base, idx); + return roc_nix_inl_ot_ipsec_inb_sa_sw_rsvd (roc_sa); +} + static_always_inline vlib_buffer_t * oct_seg_to_bp (void *p) { @@ -555,18 +580,16 @@ static_always_inline void oct_rx_ipsec_update_counters (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_buffer_t **buffs, u16 *next, u16 *buffer_next_index, u32 ilen, u8 frag_cnt, - u32 idx, const u8 reass_fail) + u32 idx, const u8 reass_fail, const u64 fp_flags) { vlib_combined_counter_main_t *rx_counter; ipsec_main_t *im = &ipsec_main; oct_ipsec_session_t *session; - struct roc_ot_ipsec_inb_sa *roc_sa; oct_ipsec_inb_sa_priv_data_t *inb_sa_priv; vlib_buffer_t *b = buffs[*buffer_next_index - 1]; u32 sa_idx, itf_sw_idx; vnet_interface_main_t *vim; oct_ipsec_main_t *oim = &oct_ipsec_main; - oct_inl_dev_main_t *oidm = &oct_inl_dev_main; vnet_main_t *vnm; int i; @@ -574,8 +597,7 @@ oct_rx_ipsec_update_counters (vlib_main_t *vm, vlib_node_runtime_t *node, vim = &vnm->interface_main; rx_counter = vim->combined_sw_if_counters + VNET_INTERFACE_COUNTER_RX; - roc_sa = roc_nix_inl_ot_ipsec_inb_sa (oidm->inb_sa_base, idx); - inb_sa_priv = roc_nix_inl_ot_ipsec_inb_sa_sw_rsvd (roc_sa); + inb_sa_priv = oct_ipsec_inb_sa_priv (idx, fp_flags); sa_idx = (u32) inb_sa_priv->user_data; @@ -620,29 +642,40 @@ oct_rx_ipsec_update_counters (vlib_main_t *vm, vlib_node_runtime_t *node, static_always_inline u8 oct_rx_ipsec_reassembly (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_buffer_template_t *bt, - struct cpt_cn10k_parse_hdr_s *cpt_hdr, + union cpt_parse_hdr_u *cpt_hdr_u, oct_nix_rx_cqe_desc_t *d, vlib_buffer_t *b, vlib_buffer_t **buf, u16 *next, u16 *buffer_next_index, u32 *olen, u32 *esp_len, u32 l2_ol3_hdr_size, const u64 fp_flags) { u8 frag_cnt = 1; - u32 idx = cpt_hdr->w0.cookie; + u32 idx; - if ((cpt_hdr->w0.num_frags) && !(cpt_hdr->w0.reas_sts)) - frag_cnt = oct_rx_ipsec_reassembly_success (vm, bt, cpt_hdr, d, b, olen, - esp_len, l2_ol3_hdr_size); - else if (cpt_hdr->w0.reas_sts) + if (fp_flags & OCT_FP_FLAG_O20) { - frag_cnt = oct_rx_ipsec_reassembly_failure ( - vm, bt, cpt_hdr, d, buf, next, buffer_next_index, olen, esp_len, - l2_ol3_hdr_size, fp_flags); - oct_rx_ipsec_update_counters (vm, node, buf, next, buffer_next_index, - *esp_len, frag_cnt, idx, 1); - return frag_cnt; + idx = cpt_hdr_u->s.w0.cookie; + } + else + { + struct cpt_cn10k_parse_hdr_s *cpt_hdr = + (struct cpt_cn10k_parse_hdr_s *) cpt_hdr_u; + + idx = cpt_hdr->w0.cookie; + if ((cpt_hdr->w0.num_frags) && !(cpt_hdr->w0.reas_sts)) + frag_cnt = oct_rx_ipsec_reassembly_success ( + vm, bt, cpt_hdr, d, b, olen, esp_len, l2_ol3_hdr_size); + else if (cpt_hdr->w0.reas_sts) + { + frag_cnt = oct_rx_ipsec_reassembly_failure ( + vm, bt, cpt_hdr, d, buf, next, buffer_next_index, olen, esp_len, + l2_ol3_hdr_size, fp_flags); + oct_rx_ipsec_update_counters (vm, node, buf, next, buffer_next_index, + *esp_len, frag_cnt, idx, 1, fp_flags); + return frag_cnt; + } } oct_rx_ipsec_update_counters (vm, node, buf, next, buffer_next_index, - *esp_len, frag_cnt, idx, 0); + *esp_len, frag_cnt, idx, 0, fp_flags); return frag_cnt; } @@ -657,14 +690,29 @@ oct_is_packet_from_cpt (union nix_rx_parse_u *rxp) } static_always_inline uword -oct_ipsec_is_inl_op_success (struct cpt_cn10k_parse_hdr_s *cpt_hdr) +oct_ipsec_is_inl_op_success (union cpt_parse_hdr_u *cpt_hdr, + const u64 fp_flags) { - return (((1U << cpt_hdr->w3.hw_ccode) & CPT_COMP_HWGOOD_MASK) && - roc_ie_ot_ucc_is_success (cpt_hdr->w3.uc_ccode)); + if (fp_flags & OCT_FP_FLAG_O20) + { + u8 hw_ccode = cpt_hdr->s.w3.hw_ccode; + u8 uc_ccode = cpt_hdr->s.w3.uc_ccode; + + return (((1U << hw_ccode) & CPT_COMP_HWGOOD_MASK) && + roc_ie_ow_ucc_is_success (uc_ccode)); + } + else + { + u8 hw_ccode = cpt_hdr->cn10k.w3.hw_ccode; + u8 uc_ccode = cpt_hdr->cn10k.w3.uc_ccode; + + return (((1U << hw_ccode) & CPT_COMP_HWGOOD_MASK) && + roc_ie_ot_ucc_is_success (uc_ccode)); + } } static_always_inline u32 -oct_get_len_from_meta (struct cpt_cn10k_parse_hdr_s *cpt_hdr, u64 w0, u64 w4) +oct_get_len_from_meta (union cpt_parse_hdr_u *cpt_hdr, u64 w0, u64 w4) { u32 len; uintptr_t ip; @@ -679,21 +727,44 @@ oct_get_len_from_meta (struct cpt_cn10k_parse_hdr_s *cpt_hdr, u64 w0, u64 w4) static_always_inline void oct_rx_ipsec_set_error (vlib_main_t *vm, vlib_node_runtime_t *node, - vlib_buffer_t *b, u16 uc_err) + vlib_buffer_t *b, union cpt_parse_hdr_u *cpt_hdr, + const u64 fp_flags) { - switch (uc_err) + u8 uc_err; + if (fp_flags & OCT_FP_FLAG_O20) + { + uc_err = cpt_hdr->s.w3.uc_ccode; + switch (uc_err) + { + /* clang-format off */ +#define _(f, n, s, d) \ + case ROC_IE_OW_UCC_##f: \ + b->error = node->errors[OCT_RX_NODE_CTR_##f]; \ + break; + foreach_octeon_ipsec_ucc; +#undef _ + /* clang-format on */ + default: + b->error = node->errors[OCT_RX_NODE_CTR_ERR_UNDEFINED]; + } + } + else { - /* clang-format off */ + uc_err = cpt_hdr->cn10k.w3.uc_ccode; + switch (uc_err) + { + /* clang-format off */ #define _(f, n, s, d) \ case ROC_IE_OT_UCC_##f: \ b->error = node->errors[OCT_RX_NODE_CTR_##f]; \ break; - foreach_octeon10_ipsec_ucc; + foreach_octeon_ipsec_ucc; #undef _ /* clang-format on */ default: b->error = node->errors[OCT_RX_NODE_CTR_ERR_UNDEFINED]; } + } } #define OCT_SEG_LEN_SHIFT 16 @@ -815,7 +886,7 @@ static_always_inline u32 oct_rx_inl_ipsec_vlib_from_cq ( vlib_main_t *vm, vlib_node_runtime_t *node, oct_nix_rx_cqe_desc_t *d, vlib_buffer_t **b, oct_rx_node_ctx_t *ctx, vlib_buffer_template_t *bt, - struct cpt_cn10k_parse_hdr_s *cpt_hdr, vlib_buffer_t **buffs, u32 *err_flags, + union cpt_parse_hdr_u *cpt_hdr, vlib_buffer_t **buffs, u32 *err_flags, u16 *next, u16 *buffer_next_index, const u64 fp_flags) { union nix_rx_parse_u *orig_rxp, *rxp; @@ -824,8 +895,8 @@ oct_rx_inl_ipsec_vlib_from_cq ( u8 frag_cnt; rxp = &d->parse.f; - cpt_hdr = (struct cpt_cn10k_parse_hdr_s *) *(((u64 *) d) + 9); - wqe_ptr = (u64 *) clib_net_to_host_u64 (cpt_hdr->wqe_ptr); + cpt_hdr = (union cpt_parse_hdr_u *) *(((u64 *) d) + 9); + wqe_ptr = (u64 *) oct_get_wqe_from_cpt_hdr (cpt_hdr, fp_flags); b[0] = (vlib_buffer_t *) ((u8 *) wqe_ptr - 128); orig_rxp = (union nix_rx_parse_u *) (wqe_ptr + 1); @@ -836,7 +907,7 @@ oct_rx_inl_ipsec_vlib_from_cq ( b[0]->flow_id = d[0].parse.w[3] >> 48; *err_flags |= ((d[0].parse.w[0] >> 20) & 0xFFF); - is_fail = !oct_ipsec_is_inl_op_success (cpt_hdr); + is_fail = !oct_ipsec_is_inl_op_success (cpt_hdr, fp_flags); buffs[*buffer_next_index] = b[0]; if (PREDICT_FALSE (is_fail)) @@ -845,7 +916,7 @@ oct_rx_inl_ipsec_vlib_from_cq ( clib_memcpy_fast (rxp, orig_rxp, sizeof (oct_nix_rx_parse_t)); next[*buffer_next_index] = VNET_DEV_ETH_RX_PORT_NEXT_DROP; *buffer_next_index = *buffer_next_index + 1; - oct_rx_ipsec_set_error (vm, node, b[0], cpt_hdr->w3.uc_ccode); + oct_rx_ipsec_set_error (vm, node, b[0], cpt_hdr, fp_flags); frag_cnt = 1; } else @@ -934,8 +1005,8 @@ oct_rx_batch (vlib_main_t *vm, vlib_node_runtime_t *node, u32 b2_err_flags = 0, b3_err_flags = 0; u32 n_left, err_flags = 0; oct_nix_rx_cqe_desc_t *d = ctx->next_desc; - struct cpt_cn10k_parse_hdr_s *cpt_hdr0, *cpt_hdr1; - struct cpt_cn10k_parse_hdr_s *cpt_hdr2, *cpt_hdr3; + union cpt_parse_hdr_u *cpt_hdr0, *cpt_hdr1; + union cpt_parse_hdr_u *cpt_hdr2, *cpt_hdr3; union nix_rx_parse_u *rxp0, *rxp1; union nix_rx_parse_u *rxp2, *rxp3; union nix_rx_parse_u *orig_rxp0, *orig_rxp1; @@ -1049,15 +1120,15 @@ oct_rx_batch (vlib_main_t *vm, vlib_node_runtime_t *node, else if (n_from_cpt == 4) { /* All packets are from cpt */ - cpt_hdr0 = (struct cpt_cn10k_parse_hdr_s *) *(((u64 *) &d[0]) + 9); - cpt_hdr1 = (struct cpt_cn10k_parse_hdr_s *) *(((u64 *) &d[1]) + 9); - cpt_hdr2 = (struct cpt_cn10k_parse_hdr_s *) *(((u64 *) &d[2]) + 9); - cpt_hdr3 = (struct cpt_cn10k_parse_hdr_s *) *(((u64 *) &d[3]) + 9); + cpt_hdr0 = (union cpt_parse_hdr_u *) *(((u64 *) &d[0]) + 9); + cpt_hdr1 = (union cpt_parse_hdr_u *) *(((u64 *) &d[1]) + 9); + cpt_hdr2 = (union cpt_parse_hdr_u *) *(((u64 *) &d[2]) + 9); + cpt_hdr3 = (union cpt_parse_hdr_u *) *(((u64 *) &d[3]) + 9); - wqe_ptr0 = (u64 *) clib_net_to_host_u64 (cpt_hdr0->wqe_ptr); - wqe_ptr1 = (u64 *) clib_net_to_host_u64 (cpt_hdr1->wqe_ptr); - wqe_ptr2 = (u64 *) clib_net_to_host_u64 (cpt_hdr2->wqe_ptr); - wqe_ptr3 = (u64 *) clib_net_to_host_u64 (cpt_hdr3->wqe_ptr); + wqe_ptr0 = (u64 *) oct_get_wqe_from_cpt_hdr (cpt_hdr0, fp_flags); + wqe_ptr1 = (u64 *) oct_get_wqe_from_cpt_hdr (cpt_hdr1, fp_flags); + wqe_ptr2 = (u64 *) oct_get_wqe_from_cpt_hdr (cpt_hdr2, fp_flags); + wqe_ptr3 = (u64 *) oct_get_wqe_from_cpt_hdr (cpt_hdr3, fp_flags); b[0] = (vlib_buffer_t *) ((u8 *) wqe_ptr0 - 128); b[1] = (vlib_buffer_t *) ((u8 *) wqe_ptr1 - 128); @@ -1089,10 +1160,10 @@ oct_rx_batch (vlib_main_t *vm, vlib_node_runtime_t *node, b[2]->template = bt; b[3]->template = bt; - is_fail0 = !oct_ipsec_is_inl_op_success (cpt_hdr0); - is_fail1 = !oct_ipsec_is_inl_op_success (cpt_hdr1); - is_fail2 = !oct_ipsec_is_inl_op_success (cpt_hdr2); - is_fail3 = !oct_ipsec_is_inl_op_success (cpt_hdr3); + is_fail0 = !oct_ipsec_is_inl_op_success (cpt_hdr0, fp_flags); + is_fail1 = !oct_ipsec_is_inl_op_success (cpt_hdr1, fp_flags); + is_fail2 = !oct_ipsec_is_inl_op_success (cpt_hdr2, fp_flags); + is_fail3 = !oct_ipsec_is_inl_op_success (cpt_hdr3, fp_flags); n_cpt_err = is_fail0 + is_fail1 + is_fail2 + is_fail3; if (PREDICT_TRUE (!n_cpt_err)) @@ -1150,8 +1221,7 @@ oct_rx_batch (vlib_main_t *vm, vlib_node_runtime_t *node, sizeof (oct_nix_rx_parse_t)); next[buffer_next_index] = VNET_DEV_ETH_RX_PORT_NEXT_DROP; buffer_next_index += 1; - oct_rx_ipsec_set_error (vm, node, b[0], - cpt_hdr0->w3.uc_ccode); + oct_rx_ipsec_set_error (vm, node, b[0], cpt_hdr0, fp_flags); frag_cnt0 = 1; } else @@ -1178,8 +1248,7 @@ oct_rx_batch (vlib_main_t *vm, vlib_node_runtime_t *node, sizeof (oct_nix_rx_parse_t)); next[buffer_next_index] = VNET_DEV_ETH_RX_PORT_NEXT_DROP; buffer_next_index += 1; - oct_rx_ipsec_set_error (vm, node, b[1], - cpt_hdr1->w3.uc_ccode); + oct_rx_ipsec_set_error (vm, node, b[1], cpt_hdr1, fp_flags); frag_cnt1 = 1; } else @@ -1204,8 +1273,7 @@ oct_rx_batch (vlib_main_t *vm, vlib_node_runtime_t *node, sizeof (oct_nix_rx_parse_t)); next[buffer_next_index] = VNET_DEV_ETH_RX_PORT_NEXT_DROP; buffer_next_index += 1; - oct_rx_ipsec_set_error (vm, node, b[2], - cpt_hdr2->w3.uc_ccode); + oct_rx_ipsec_set_error (vm, node, b[2], cpt_hdr2, fp_flags); frag_cnt2 = 1; } else @@ -1230,8 +1298,7 @@ oct_rx_batch (vlib_main_t *vm, vlib_node_runtime_t *node, sizeof (oct_nix_rx_parse_t)); next[buffer_next_index] = VNET_DEV_ETH_RX_PORT_NEXT_DROP; buffer_next_index += 1; - oct_rx_ipsec_set_error (vm, node, b[3], - cpt_hdr3->w3.uc_ccode); + oct_rx_ipsec_set_error (vm, node, b[3], cpt_hdr3, fp_flags); frag_cnt3 = 1; } else @@ -1282,8 +1349,7 @@ oct_rx_batch (vlib_main_t *vm, vlib_node_runtime_t *node, /* CQ ring contains mix of packets from wire and CPT */ if (is_b0_from_cpt) { - cpt_hdr0 = - (struct cpt_cn10k_parse_hdr_s *) *(((u64 *) &d[0]) + 9); + cpt_hdr0 = (union cpt_parse_hdr_u *) *(((u64 *) &d[0]) + 9); oct_rx_inl_ipsec_vlib_from_cq (vm, node, &d[0], &b[0], ctx, &bt, cpt_hdr0, buffs, &err_flags, next, &buffer_next_index, fp_flags); @@ -1295,8 +1361,7 @@ oct_rx_batch (vlib_main_t *vm, vlib_node_runtime_t *node, if (is_b1_from_cpt) { - cpt_hdr1 = - (struct cpt_cn10k_parse_hdr_s *) *(((u64 *) &d[1]) + 9); + cpt_hdr1 = (union cpt_parse_hdr_u *) *(((u64 *) &d[1]) + 9); oct_rx_inl_ipsec_vlib_from_cq (vm, node, &d[1], &b[1], ctx, &bt, cpt_hdr1, buffs, &err_flags, next, &buffer_next_index, fp_flags); @@ -1307,8 +1372,7 @@ oct_rx_batch (vlib_main_t *vm, vlib_node_runtime_t *node, next, &buffer_next_index, fp_flags); if (is_b2_from_cpt) { - cpt_hdr2 = - (struct cpt_cn10k_parse_hdr_s *) *(((u64 *) &d[2]) + 9); + cpt_hdr2 = (union cpt_parse_hdr_u *) *(((u64 *) &d[2]) + 9); oct_rx_inl_ipsec_vlib_from_cq (vm, node, &d[2], &b[2], ctx, &bt, cpt_hdr2, buffs, &err_flags, next, &buffer_next_index, fp_flags); @@ -1319,8 +1383,7 @@ oct_rx_batch (vlib_main_t *vm, vlib_node_runtime_t *node, next, &buffer_next_index, fp_flags); if (is_b3_from_cpt) { - cpt_hdr3 = - (struct cpt_cn10k_parse_hdr_s *) *(((u64 *) &d[3]) + 9); + cpt_hdr3 = (union cpt_parse_hdr_u *) *(((u64 *) &d[3]) + 9); oct_rx_inl_ipsec_vlib_from_cq (vm, node, &d[3], &b[3], ctx, &bt, cpt_hdr3, buffs, &err_flags, next, &buffer_next_index, fp_flags); @@ -1400,7 +1463,7 @@ oct_rx_batch (vlib_main_t *vm, vlib_node_runtime_t *node, is_b0_from_cpt = oct_is_packet_from_cpt (&d[0].parse.f); if (is_b0_from_cpt) { - cpt_hdr0 = (struct cpt_cn10k_parse_hdr_s *) *(((u64 *) &d[0]) + 9); + cpt_hdr0 = (union cpt_parse_hdr_u *) *(((u64 *) &d[0]) + 9); oct_rx_inl_ipsec_vlib_from_cq (vm, node, &d[0], &b[0], ctx, &bt, cpt_hdr0, buffs, &err_flags, next, &buffer_next_index, fp_flags); @@ -1619,7 +1682,7 @@ oct_rx_enq_to_next (vlib_main_t *vm, vlib_node_runtime_t *node, static_always_inline uword oct_rx_node_inline (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame, vnet_dev_port_t *port, - vnet_dev_rx_queue_t *rxq, int with_flows) + vnet_dev_rx_queue_t *rxq, int with_flows, const u64 flags) { vnet_main_t *vnm = vnet_get_main (); u32 thr_idx = vlib_get_thread_index (); @@ -1663,10 +1726,10 @@ oct_rx_node_inline (vlib_main_t *vm, vlib_node_runtime_t *node, clib_min (cq_size - head, clib_min (n_desc, ctx->n_left_to_next / 4)); if (PREDICT_TRUE (ctx->trace_count == 0)) - n_processed += oct_rx_batch (vm, node, ctx, rxq, n, buffs, 0); + n_processed += oct_rx_batch (vm, node, ctx, rxq, n, buffs, flags); else - n_processed += - oct_rx_batch (vm, node, ctx, rxq, n, buffs, OCT_FP_FLAG_TRACE_EN); + n_processed += oct_rx_batch (vm, node, ctx, rxq, n, buffs, + OCT_FP_FLAG_TRACE_EN | flags); if (n_processed >= 256) break; @@ -1737,7 +1800,25 @@ VNET_DEV_NODE_FN (oct_rx_node) if (!rxq->started) continue; - n_rx += oct_rx_node_inline (vm, node, frame, port, rxq, 0); + n_rx += oct_rx_node_inline (vm, node, frame, port, rxq, 0, 0); + } + + return n_rx; +} + +VNET_DEV_NODE_FN (oct_o20_rx_node) +(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame) +{ + u32 n_rx = 0; + foreach_vnet_dev_rx_queue_runtime (rxq, node) + { + vnet_dev_port_t *port = rxq->port; + + if (!rxq->started) + continue; + + n_rx += + oct_rx_node_inline (vm, node, frame, port, rxq, 0, OCT_FP_FLAG_O20); } return n_rx; From 592ddab70843ad7889a279b9bb49ae4f68ef32b8 Mon Sep 17 00:00:00 2001 From: Monendra Singh Kushwaha Date: Wed, 9 Jul 2025 18:25:55 +0530 Subject: [PATCH 244/271] octeon: fix ipsec interface rx counter Type: fix JIRA: https://essjira.marvell.com/browse/IPBUSW-72689 Signed-off-by: Monendra Singh Kushwaha Change-Id: Ia2622d3c68a551e09ee2bc708ddcf2a0cd7abc21 Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/157427 Tested-by: sa_ip-sw-jenkins --- src/plugins/dev_octeon/rx_node.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/plugins/dev_octeon/rx_node.c b/src/plugins/dev_octeon/rx_node.c index 58b4c549eb..10a872b18f 100644 --- a/src/plugins/dev_octeon/rx_node.c +++ b/src/plugins/dev_octeon/rx_node.c @@ -528,7 +528,7 @@ oct_ipsec_update_itf_sw_idx (oct_ipsec_session_t *session, u32 sa_idx) { clib_bihash_kv_24_16_t bkey60 = { 0 }; clib_bihash_kv_8_16_t bkey40 = { 0 }; - ipsec_tun_lkup_result_t *res; + ipsec_tun_lkup_result_t res; ipsec4_tunnel_kv_t *key40; ipsec6_tunnel_kv_t *key60; ip_address_t *ip_addr; @@ -551,7 +551,7 @@ oct_ipsec_update_itf_sw_idx (oct_ipsec_session_t *session, u32 sa_idx) if (PREDICT_FALSE (rv)) return ~0; - res = (ipsec_tun_lkup_result_t *) &bkey40.value; + clib_memcpy_fast (&res, &bkey40.value, sizeof (res)); } else { @@ -566,14 +566,14 @@ oct_ipsec_update_itf_sw_idx (oct_ipsec_session_t *session, u32 sa_idx) if (PREDICT_FALSE (rv)) return ~0; - res = (ipsec_tun_lkup_result_t *) &bkey60.value; + clib_memcpy_fast (&res, &bkey60.value, sizeof (res)); } /* Store the ITF sw_if_index in the SA session to avoid duplicate lookups for each packet */ - session->itf_sw_idx = res->sw_if_index; + session->itf_sw_idx = res.sw_if_index; - return res->sw_if_index; + return res.sw_if_index; } static_always_inline void From 5c4004fa75283c065f97a52e2e6f308d4a37fb40 Mon Sep 17 00:00:00 2001 From: Monendra Singh Kushwaha Date: Thu, 10 Jul 2025 15:01:30 +0530 Subject: [PATCH 245/271] octeon: fix pause frame support Type: fix JIRA: https://essjira.marvell.com/browse/IPBUSW-71340 fixes: aef87b3 Signed-off-by: Monendra Singh Kushwaha Change-Id: Iad9da587f25fbf930fae02318f9df81f5cfcf93c Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/157490 Tested-by: sa_ip-toolkits-Jenkins (cherry picked from commit edf4f7a4b78b6e00c3ede18ebe8b0ac865caaf12) Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/157503 Tested-by: sa_ip-sw-jenkins --- src/plugins/dev_octeon/port.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/plugins/dev_octeon/port.c b/src/plugins/dev_octeon/port.c index 28b72b7d23..cb4a6f59b6 100644 --- a/src/plugins/dev_octeon/port.c +++ b/src/plugins/dev_octeon/port.c @@ -262,14 +262,6 @@ oct_port_init (vlib_main_t *vm, vnet_dev_port_t *port) } cp->npc_initialized = 1; - /* Configure pause frame flow control*/ - if (is_pause_frame_enable && - (rv = oct_port_pause_flow_control_init (vm, port))) - { - oct_port_deinit (vm, port); - return rv; - } - if (inl_main->inl_dev) { struct roc_nix_fc_cfg fc_cfg; @@ -318,6 +310,14 @@ oct_port_init (vlib_main_t *vm, vnet_dev_port_t *port) return rv; } + /* Configure pause frame flow control*/ + if (is_pause_frame_enable && + (rv = oct_port_pause_flow_control_init (vm, port))) + { + oct_port_deinit (vm, port); + return rv; + } + if (roc_nix_register_queue_irqs (nix)) { rv = oct_roc_err (dev, rrv, "roc_nix_register_queue_irqs() failed"); From 9bad0d15c23df84d0e7baa105c159b0d47cd4e32 Mon Sep 17 00:00:00 2001 From: Monendra Singh Kushwaha Date: Sun, 29 Jun 2025 22:40:54 +0530 Subject: [PATCH 246/271] ci: add inline outbound ipsec perf test case Type: feature JIRA: https://essjira.marvell.com/browse/IPBUSW-69495 Change-Id: Ia284b751e64286c6a654e9de89c0850228412676 Signed-off-by: Monendra Singh Kushwaha Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/156768 Reviewed-by: Nithin Kumar Dabilpuram Tested-by: Nithin Kumar Dabilpuram (cherry picked from commit 394c2f7518dbf53aee0bc55f637f3cfdbabf7d3d) Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/159001 Tested-by: sa_ip-sw-jenkins --- ci/test/common/vpp/vpp.env | 9 + ci/test/env/cn10k-perf.env | 1 + ci/test/env/cn10k.env | 1 + ci/test/inl_ipsec_perf/aes_cbc_sha1_hmac.cfg | 20 + .../inl_ipsec_perf/aes_cbc_sha1_hmac_ib.cfg | 19 + ci/test/inl_ipsec_perf/aes_gcm.cfg | 20 + ci/test/inl_ipsec_perf/aes_gcm_ib.cfg | 18 + ci/test/inl_ipsec_perf/inl_ipsec_gen.sh | 113 +++ ci/test/inl_ipsec_perf/inl_ipsec_perf.conf | 55 ++ ci/test/inl_ipsec_perf/inl_ipsec_perf.exec | 21 + ci/test/inl_ipsec_perf/inl_ipsec_perf.sh | 832 ++++++++++++++++++ .../pcap/enc_aes-cbc_sha1-hmac_1410.pcap | Bin 0 -> 1518 bytes .../pcap/enc_aes-cbc_sha1-hmac_380.pcap | Bin 0 -> 478 bytes .../pcap/enc_aes-cbc_sha1-hmac_64.pcap | Bin 0 -> 174 bytes .../inl_ipsec_perf/pcap/enc_aes-gcm_1410.pcap | Bin 0 -> 1506 bytes .../inl_ipsec_perf/pcap/enc_aes-gcm_380.pcap | Bin 0 -> 474 bytes .../inl_ipsec_perf/pcap/enc_aes-gcm_64.pcap | Bin 0 -> 158 bytes .../pcap/enc_aes-gcm_msns_1410.pcap | Bin 0 -> 12080 bytes .../pcap/enc_aes-gcm_msns_380.pcap | Bin 0 -> 3888 bytes .../pcap/enc_aes-gcm_msns_64.pcap | Bin 0 -> 1456 bytes .../cn10k/rclk2500_sclk1000.103xx.ip.inb | 10 + .../cn10k/rclk2500_sclk1000.103xx.ip.outb | 10 + .../cn10k/rclk2500_sclk1000.106xx.ip.inb | 10 + .../cn10k/rclk2500_sclk1000.106xx.ip.outb | 10 + .../cn10k/rclk2500_sclk1100.106xx.ip.inb | 10 + .../cn10k/rclk2500_sclk1100.106xx.ip.outb | 10 + ci/test/test.list | 1 + 27 files changed, 1170 insertions(+) create mode 100644 ci/test/inl_ipsec_perf/aes_cbc_sha1_hmac.cfg create mode 100644 ci/test/inl_ipsec_perf/aes_cbc_sha1_hmac_ib.cfg create mode 100644 ci/test/inl_ipsec_perf/aes_gcm.cfg create mode 100644 ci/test/inl_ipsec_perf/aes_gcm_ib.cfg create mode 100755 ci/test/inl_ipsec_perf/inl_ipsec_gen.sh create mode 100644 ci/test/inl_ipsec_perf/inl_ipsec_perf.conf create mode 100644 ci/test/inl_ipsec_perf/inl_ipsec_perf.exec create mode 100755 ci/test/inl_ipsec_perf/inl_ipsec_perf.sh create mode 100644 ci/test/inl_ipsec_perf/pcap/enc_aes-cbc_sha1-hmac_1410.pcap create mode 100644 ci/test/inl_ipsec_perf/pcap/enc_aes-cbc_sha1-hmac_380.pcap create mode 100644 ci/test/inl_ipsec_perf/pcap/enc_aes-cbc_sha1-hmac_64.pcap create mode 100644 ci/test/inl_ipsec_perf/pcap/enc_aes-gcm_1410.pcap create mode 100644 ci/test/inl_ipsec_perf/pcap/enc_aes-gcm_380.pcap create mode 100644 ci/test/inl_ipsec_perf/pcap/enc_aes-gcm_64.pcap create mode 100644 ci/test/inl_ipsec_perf/pcap/enc_aes-gcm_msns_1410.pcap create mode 100644 ci/test/inl_ipsec_perf/pcap/enc_aes-gcm_msns_380.pcap create mode 100644 ci/test/inl_ipsec_perf/pcap/enc_aes-gcm_msns_64.pcap create mode 100644 ci/test/inl_ipsec_perf/ref_numbers/cn10k/rclk2500_sclk1000.103xx.ip.inb create mode 100644 ci/test/inl_ipsec_perf/ref_numbers/cn10k/rclk2500_sclk1000.103xx.ip.outb create mode 100644 ci/test/inl_ipsec_perf/ref_numbers/cn10k/rclk2500_sclk1000.106xx.ip.inb create mode 100644 ci/test/inl_ipsec_perf/ref_numbers/cn10k/rclk2500_sclk1000.106xx.ip.outb create mode 100644 ci/test/inl_ipsec_perf/ref_numbers/cn10k/rclk2500_sclk1100.106xx.ip.inb create mode 100644 ci/test/inl_ipsec_perf/ref_numbers/cn10k/rclk2500_sclk1100.106xx.ip.outb diff --git a/ci/test/common/vpp/vpp.env b/ci/test/common/vpp/vpp.env index 91676cabbf..70ad1cb2db 100644 --- a/ci/test/common/vpp/vpp.env +++ b/ci/test/common/vpp/vpp.env @@ -77,6 +77,15 @@ function vpp_exec_cmd() set -e } +function vpp_exec_file() +{ + local startup_conf=$1 + local file=$2 + set +e + $VPPCTL -s /tmp/$startup_conf/cli.sock exec $file + set -e +} + function vpp_start() { local startup_conf=$1 diff --git a/ci/test/env/cn10k-perf.env b/ci/test/env/cn10k-perf.env index 988822c990..e5b34b779f 100644 --- a/ci/test/env/cn10k-perf.env +++ b/ci/test/env/cn10k-perf.env @@ -8,6 +8,7 @@ source $PROJECT_ROOT/ci/test/env/cn10k.env # List of perf tests to be run. RUN_TESTS=" l3fwd_perf + inl_ipsec_perf " # Update command timeout diff --git a/ci/test/env/cn10k.env b/ci/test/env/cn10k.env index eb25518de9..9411149217 100644 --- a/ci/test/env/cn10k.env +++ b/ci/test/env/cn10k.env @@ -76,6 +76,7 @@ FIXME_SKIP_TESTS=" DEFAULT_SKIP_TESTS=" l3fwd_perf + inl_ipsec_perf ${FIXME_SKIP_TESTS} " diff --git a/ci/test/inl_ipsec_perf/aes_cbc_sha1_hmac.cfg b/ci/test/inl_ipsec_perf/aes_cbc_sha1_hmac.cfg new file mode 100644 index 0000000000..ed6579594b --- /dev/null +++ b/ci/test/inl_ipsec_perf/aes_cbc_sha1_hmac.cfg @@ -0,0 +1,20 @@ +set int ip address eth0 12.168.101.1/24 +set int state eth0 up + +set int ip address eth1 13.168.1.1/24 +set int state eth1 up + +set ipsec async mode on + +ipsec sa add 1 spi 1 esp crypto-key a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0 crypto-alg aes-cbc-128 integ-alg sha1-96 integ-key a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0 tunnel src 1.1.1.1 dst 1.1.1.2 inbound +ipsec sa add 11 spi 101 esp crypto-key a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0 crypto-alg aes-cbc-128 integ-alg sha1-96 integ-key a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0 tunnel src 1.1.1.2 dst 1.1.1.1 + +ipsec itf create +ipsec tunnel protect sa-out 11 sa-in 1 ipsec0 +set int state ipsec0 up + +ip route add 192.168.1.2/24 via ipsec0 +set ip neighbor eth0 1.1.1.1 00:16:3e:7e:94:9a + +ip route add 1.1.1.0/24 via eth0 + diff --git a/ci/test/inl_ipsec_perf/aes_cbc_sha1_hmac_ib.cfg b/ci/test/inl_ipsec_perf/aes_cbc_sha1_hmac_ib.cfg new file mode 100644 index 0000000000..99842a6a27 --- /dev/null +++ b/ci/test/inl_ipsec_perf/aes_cbc_sha1_hmac_ib.cfg @@ -0,0 +1,19 @@ +set int ip address eth0 12.168.101.1/24 +set int mac address eth0 00:16:3e:22:a1:d9 +set int state eth0 up + +set int ip address eth1 13.168.1.1/24 +set int state eth1 up + +set ipsec async mode on + +ipsec sa add 1 spi 1 esp crypto-key a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0 crypto-alg aes-cbc-128 integ-alg sha1-96 integ-key a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0 tunnel src 1.1.1.1 dst 1.1.1.2 inbound +ipsec sa add 11 spi 101 esp crypto-key a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0 crypto-alg aes-cbc-128 integ-alg sha1-96 integ-key a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0 tunnel src 1.1.1.2 dst 1.1.1.1 + +ipsec itf create +ipsec tunnel protect sa-out 11 sa-in 1 ipsec0 +set int state ipsec0 up + +ip route add 192.168.1.2/24 via eth0 +set ip neighbor eth0 192.168.1.2 00:16:3e:7e:94:9a + diff --git a/ci/test/inl_ipsec_perf/aes_gcm.cfg b/ci/test/inl_ipsec_perf/aes_gcm.cfg new file mode 100644 index 0000000000..02137abd55 --- /dev/null +++ b/ci/test/inl_ipsec_perf/aes_gcm.cfg @@ -0,0 +1,20 @@ +set int ip address eth0 12.168.101.1/24 +set int state eth0 up + +set int ip address eth1 13.168.1.1/24 +set int state eth1 up + +set ipsec async mode on + +ipsec sa add 1 spi 1 esp crypto-key a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0 crypto-alg aes-gcm-128 tunnel src 1.1.1.1 dst 1.1.1.2 inbound +ipsec sa add 11 spi 101 esp crypto-key a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0 crypto-alg aes-gcm-128 tunnel src 1.1.1.2 dst 1.1.1.1 + +ipsec itf create +ipsec tunnel protect sa-out 11 sa-in 1 ipsec0 +set int state ipsec0 up + +ip route add 192.168.2.2/24 via ipsec0 +set ip neighbor eth0 1.1.1.1 00:16:3e:7e:94:9a + +ip route add 1.1.1.0/24 via eth0 + diff --git a/ci/test/inl_ipsec_perf/aes_gcm_ib.cfg b/ci/test/inl_ipsec_perf/aes_gcm_ib.cfg new file mode 100644 index 0000000000..c8f13bbdcc --- /dev/null +++ b/ci/test/inl_ipsec_perf/aes_gcm_ib.cfg @@ -0,0 +1,18 @@ +set int ip address eth0 12.168.101.1/24 +set int mac address eth0 00:01:02:03:04:01 +set int state eth0 up + +set int ip address eth1 13.168.1.1/24 +set int state eth1 up + +set ipsec async mode on + +ipsec sa add 1 spi 1 esp crypto-key a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0 crypto-alg aes-gcm-128 tunnel src 1.1.1.1 dst 1.1.1.2 inbound +ipsec sa add 11 spi 101 esp crypto-key a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0 crypto-alg aes-gcm-128 tunnel src 1.1.1.2 dst 1.1.1.1 + +ipsec itf create +ipsec tunnel protect sa-out 11 sa-in 1 ipsec0 +set int state ipsec0 up + +ip route add 192.168.2.2/24 via eth0 +set ip neighbor eth0 192.168.2.2 00:16:3e:7e:94:9a diff --git a/ci/test/inl_ipsec_perf/inl_ipsec_gen.sh b/ci/test/inl_ipsec_perf/inl_ipsec_gen.sh new file mode 100755 index 0000000000..ce22d2eb5c --- /dev/null +++ b/ci/test/inl_ipsec_perf/inl_ipsec_gen.sh @@ -0,0 +1,113 @@ +#!/bin/bash +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (C) 2022 Marvell. + +set -eou pipefail +OCTEONTESTPATH="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )/.." +source $OCTEONTESTPATH/common/testpmd/common.env + +TX_PRFX="tpmd_tx" +RX_PRFX="tpmd_rx" +PORT0="${PORT0:-0002:02:00.0}" +PORT1="${PORT1:-0002:03:00.0}" + +function sig_handler() +{ + local status=$? + set +e + trap - ERR + trap - INT + if [[ $status -ne 0 ]]; then + echo "$1 Handler" + # Dump error logs + testpmd_log $TX_PRFX + testpmd_log $RX_PRFX + fi + + testpmd_cleanup $TX_PRFX + testpmd_cleanup $RX_PRFX + exit $status +} + +trap "sig_handler ERR" ERR +trap "sig_handler INT" INT + +launch_testpmd_rx() +{ + #local out=testpmd.out.$1 + testpmd_launch $1 \ + "-c 0x700 -a $PORT1,disable_xqe_drop=1" \ + "--nb-cores=1 --forward-mode=rxonly" \ + /dev/null & + sleep 1 + testpmd_cmd $1 "port stop 0" + testpmd_cmd $1 "set flow_ctrl rx off 0" + testpmd_cmd $1 "set flow_ctrl tx off 0" + testpmd_cmd $1 "port start 0" +} + +launch_testpmd_tx_outb() +{ + testpmd_launch $1 \ + "-c 0xF800 -a $PORT0,disable_xqe_drop=1" \ + "--nb-cores=3 --forward-mode=txonly --tx-ip=192.168.$2.1,192.168.$2.2 --txq=3 --rxq=3 --eth-peer=0,00:01:02:03:04:01" \ + /dev/null & + sleep 1 + testpmd_cmd $1 "port stop 0" + testpmd_cmd $1 "set flow_ctrl rx off 0" + testpmd_cmd $1 "set flow_ctrl tx off 0" + testpmd_cmd $1 "port start 0" +} + +launch_testpmd_tx_inb() +{ + testpmd_launch $1 \ + "-c 0x3F800 --vdev net_pcap0,rx_pcap=$2,rx_pcap=$2,rx_pcap=$2,rx_pcap=$2,rx_pcap=$2,infinite_rx=1 -a $PORT0,disable_xqe_drop=1" \ + "--nb-cores=5 --txq=5 --rxq=5 --no-flush-rx --eth-peer=0,00:01:02:03:04:01" \ + /dev/null & + sleep 1 + testpmd_cmd $1 "port stop 0" + testpmd_cmd $1 "set flow_ctrl rx off 0" + testpmd_cmd $1 "set flow_ctrl tx off 0" + testpmd_cmd $1 "port start 0" +} + +case $TESTPMD_OP in + launch_tx_outb) + launch_testpmd_tx_outb $1 $2 + ;; + launch_tx_inb) + launch_testpmd_tx_inb $1 $2 + ;; + launch_rx) + launch_testpmd_rx $1 + ;; + start) + testpmd_cmd $1 "start tx_first 64" + testpmd_cmd $1 "show port stats all" + ;; + stop) + testpmd_cmd $1 "stop" + ;; + rx_pps) + prev=$(testpmd_log_sz $1) + curr=$prev + testpmd_cmd $1 "show port stats $2" + + while [ $prev -eq $curr ]; do sleep 0.1; curr=$(testpmd_log_sz $1); done + testpmd_prompt $1 + val=`testpmd_log $1 | tail -4 | grep -ao 'Rx-pps: .*' | \ + awk '{print $2}'` + echo $val + ;; + pktsize) + testpmd_cmd $1 "set txpkts $2" + ;; + quit) + testpmd_quit $1 + ;; + log) + testpmd_log $1 + ;; +esac +exit 0 diff --git a/ci/test/inl_ipsec_perf/inl_ipsec_perf.conf b/ci/test/inl_ipsec_perf/inl_ipsec_perf.conf new file mode 100644 index 0000000000..99b4521193 --- /dev/null +++ b/ci/test/inl_ipsec_perf/inl_ipsec_perf.conf @@ -0,0 +1,55 @@ +unix { + log /tmp/inl_ipsec_perf/vpp.log + cli-listen /tmp/inl_ipsec_perf/cli.sock + #full-coredump + #interactive + nodaemon +} + +api-trace { + on +} + +logging { + default-syslog-log-level info +} + +cpu { + main-core 2 + corelist-workers 3 +} + +session +{ + event-queue-length 10 +} +#tcp { no-csum-offload } + +buffers { + ## Increase number of buffers allocated, needed only in scenarios with + ## large number of interfaces and worker threads. Value is per numa node. + ## Default is 16384 (8192 if running unprivileged) +# buffers-per-numa 5000 + + ## Size of buffer data area + ## Default is 2048 + default data-size 2048 +# naturally-aligned +# enable-nat-alignment +} + +devices { + dev pci/0002:20:00.1 + { + driver octeon + } + dev pci/0002:1d:00.0 + { + driver octeon + } +} + +plugins { + plugin dpdk_plugin.so { disable } + plugin onp_plugin.so { disable } +} diff --git a/ci/test/inl_ipsec_perf/inl_ipsec_perf.exec b/ci/test/inl_ipsec_perf/inl_ipsec_perf.exec new file mode 100644 index 0000000000..e9ae479c82 --- /dev/null +++ b/ci/test/inl_ipsec_perf/inl_ipsec_perf.exec @@ -0,0 +1,21 @@ +set int ip address eth0 12.168.101.1/24 +set int state eth0 up + +set int ip address eth1 13.168.1.1/24 +set int state eth1 up + +set ipsec async mode on + +ipsec sa add 1 spi 1 esp crypto-key a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0 crypto-alg aes-cbc-128 integ-alg sha1-96 integ-key a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0 tunnel src 1.1.1.1 dst 1.1.1.2 inbound +ipsec sa add 11 spi 101 esp crypto-key a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0 crypto-alg aes-cbc-128 integ-alg sha1-96 integ-key a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0 tunnel src 1.1.1.2 dst 1.1.1.1 + +ipsec itf create +ipsec tunnel protect sa-out 11 sa-in 1 ipsec0 +set int state ipsec0 up + +ip route add 192.168.1.2/24 via ipsec0 +set ip neighbor eth0 1.1.1.1 00:16:3e:7e:94:9a + +ip route add 1.1.1.0/24 via eth0 +set int promiscuous on eth0 + diff --git a/ci/test/inl_ipsec_perf/inl_ipsec_perf.sh b/ci/test/inl_ipsec_perf/inl_ipsec_perf.sh new file mode 100755 index 0000000000..7ede6e8a32 --- /dev/null +++ b/ci/test/inl_ipsec_perf/inl_ipsec_perf.sh @@ -0,0 +1,832 @@ +#!/bin/bash +# Copyright (c) 2025 Marvell. +# SPDX-License-Identifier: Apache-2.0 +# https://spdx.org/licenses/Apache-2.0.html + +#set -e +set -euox pipefail + +GENERATOR_BOARD=${GENERATOR_BOARD:-} +REMOTE_DIR=${REMOTE_DIR:-$(pwd | cut -d/ -f 1-3)} +OCTEONTESTPATH="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )/.." +PKT_LIST="64 380 1410" +NUM_CAPTURE=3 +MAX_TRY_CNT=5 +CORES=(1) +COREMASK="0x10000" +TXWAIT=15 +RXWAIT=5 +WS=2 +IS_RXPPS_TXTPMD=0 +TARGET_SSH_CMD=${TARGET_SSH_CMD:-"ssh -o LogLevel=ERROR -o ServerAliveInterval=30 \ + -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"} +TARGET_SSH_CMD="$TARGET_SSH_CMD -n" +GENERATOR_SCRIPT=${GENERATOR_SCRIPT:-inl_ipsec_gen.sh} +WITH_GEN_BOARD=0 + +source $OCTEONTESTPATH/common/testpmd/pktgen.env +source $OCTEONTESTPATH/common/testpmd/common.env +source $OCTEONTESTPATH/common/vpp/vpp.env + +TPMD_RX_PREFIX="tpmd_rx" +TPMD_TX_PREFIX="tpmd_tx" + +declare -i SCLK +declare -i RCLK +declare -i CPTCLK +declare -A PASS_PPS_TABLE +SUDO="sudo" + +! $(cat /proc/device-tree/compatible | grep -q "cn10k") +IS_CN10K=$? +DTC=$(tr -d '\0' > $in + + while [ $prev -eq $curr ]; do sleep 0.1; curr=$(testpmd_log_sz $prefix); done + testpmd_prompt $prefix + done + fi +} + +function rx_stats() +{ + local prefix=$1 + local port=$2 + local in=testpmd.in.$prefix + local out=testpmd.out.$prefix + + if [[ $WITH_GEN_BOARD -eq 1 ]]; then + rxpps=$(exec_testpmd_cmd_gen "rx_pps" $prefix $port) + echo $rxpps + else + prev=$(testpmd_log_sz $prefix) + curr=$prev + + echo "show port stats $port" >> $in + while [ $prev -eq $curr ]; do sleep 0.1; curr=$(testpmd_log_sz $prefix); done + testpmd_prompt $prefix + cat $out | tail -n4 | head -n1 + fi +} + +function capture_rx_pps() +{ + local stats + if [[ $IS_RXPPS_TXTPMD -ne 0 ]]; then + # Specific case of Inline Protocol Single-SA configuration. + # Packets are routed back to originating port. + stats=$(rx_stats $TPMD_TX_PREFIX "0") + else + stats=$(rx_stats $TPMD_RX_PREFIX "0") + fi + + if [[ $WITH_GEN_BOARD -eq 1 ]]; then + echo $stats + else + echo $stats | awk '{print $2}' + fi +} + +# Configure interfaces +function setup_interfaces() +{ + echo -e "dev bind $LIF1 $LIF2 $LIF3 $LIF4" + + $VFIO_DEVBIND -b vfio-pci $LIF1 + $VFIO_DEVBIND -b vfio-pci $LIF2 + $VFIO_DEVBIND -b vfio-pci $LIF3 + $VFIO_DEVBIND -b vfio-pci $LIF4 +} + +function cleanup_interfaces() +{ + # Bind the vfio-pci binded devices back to nicvf + $VFIO_DEVBIND -b $NICVF $LIF1 + $VFIO_DEVBIND -b $NICVF $LIF2 + $VFIO_DEVBIND -b $NICVF $LIF3 + $VFIO_DEVBIND -b $NICVF $LIF4 +} + +function start_testpmd() +{ + if [[ $WITH_GEN_BOARD -eq 1 ]]; then + exec_testpmd_cmd_gen "start" $TPMD_TX_PREFIX "NOP" + else + testpmd_cmd "$TPMD_RX_PREFIX" "start" + testpmd_cmd "$TPMD_TX_PREFIX" "start" + fi +} + +function stop_testpmd() +{ + if [[ $WITH_GEN_BOARD -eq 1 ]]; then + exec_testpmd_cmd_gen "stop" $TPMD_TX_PREFIX "NOP" + else + testpmd_cmd "$TPMD_TX_PREFIX" "stop" + testpmd_cmd "$TPMD_RX_PREFIX" "stop" + fi +} + +function set_pktsize_testpmd() +{ + if [[ $WITH_GEN_BOARD -eq 1 ]]; then + exec_testpmd_cmd_gen "pktsize" "$TPMD_TX_PREFIX" $1 + else + testpmd_cmd "$TPMD_TX_PREFIX" "set txpkts $1" + fi +} + +function quit_testpmd() +{ + if [[ $WITH_GEN_BOARD -eq 1 ]]; then + if [[ $1 == $TPMD_TX_PREFIX ]]; then + exec_testpmd_cmd_gen "log" $1 "NOP" >testpmd.out.$1 + exec_testpmd_cmd_gen "quit" $1 "NOP" + fi + else + testpmd_quit $1 + fi +} + +function outb_perf() +{ + local rx_pps + local avg_pps + local pktsz + local tcnt + local algo + local rn + local i + + [[ $X = 1 ]] && algo="aes-cbc_sha1-hmac" || algo="aes-gcm" + + rn=0 + for pktsz in ${PKT_LIST[@]} + do + set_pktsize_testpmd $pktsz + + tcnt=1 + while [ $tcnt -le $MAX_TRY_CNT ]; do + echo "Try $tcnt" + i=1 + rx_pps=0 + if [[ $tcnt -gt 1 ]]; then + # Restart vpp + vpp_show_trace inl_ipsec_perf + vpp_log inl_ipsec_perf + vpp_stats_all inl_ipsec_perf + vpp_cleanup inl_ipsec_perf + echo "Restart vpp" + run_vpp_ipsec + fi + start_testpmd + pmd_rx_dry_run + # Wait for few seconds for traffic to stabilize + sleep $TXWAIT + while [ $i -le $NUM_CAPTURE ]; do + rx_pps=$rx_pps+$(capture_rx_pps) + ((++i)) + sleep $RXWAIT + done + stop_testpmd + avg_pps=$(echo "(($rx_pps) / $NUM_CAPTURE)" | bc) + p=${PASS_PPS_TABLE[$rn,$2]} + echo "pktsize: $pktsz avg_pps: $avg_pps" + echo "pass_pps $p" + if (( $(echo "$avg_pps < $p" | bc) )); then + echo "$1:Low numbers for packet size $pktsz " \ + "($avg_pps < $p) for $3 cores">&2 + else + echo "Test Passed" + break + fi + ((++tcnt)) + sleep $WS + done + if [[ $tcnt -gt $MAX_TRY_CNT ]]; then + echo "Test Failed" + Failed_tests="$Failed_tests \"${TN[$Y]} outbound $algo pktsize:$pktsz\"" + fi + ((++rn)) + done +} + +function inb_perf() +{ + local rx_pps + local avg_pps + local pktsz + local tcnt + local algo + local rn + local i + + [[ $X = 1 ]] && algo="aes-cbc_sha1-hmac" || algo="aes-gcm" + + rn=0 + for pktsz in ${PKT_LIST[@]} + do + sleep $WS + pmd_tx_launch_for_inb $1 $pktsz + + tcnt=1 + while [ $tcnt -le $MAX_TRY_CNT ]; do + echo "Try $tcnt" + i=1 + rx_pps=0 + if [[ $tcnt -gt 1 ]]; then + # Restart vpp + vpp_log inl_ipsec_perf + vpp_stats_all inl_ipsec_perf + vpp_cleanup inl_ipsec_perf + echo "Restart vpp" + run_vpp_ipsec_inb + fi + start_testpmd + pmd_rx_dry_run + # Wait for few seconds for traffic to stabilize + sleep $TXWAIT + while [ $i -le $NUM_CAPTURE ]; do + rx_pps=$rx_pps+$(capture_rx_pps) + ((++i)) + sleep $RXWAIT + done + stop_testpmd + avg_pps=$(echo "(($rx_pps) / $NUM_CAPTURE)" | bc) + p=${PASS_PPS_TABLE[$rn,$2]} + echo "pktsize: $pktsz avg_pps: $avg_pps" + echo "pass_pps $p" + if (( $(echo "$avg_pps < $p" | bc) )); then + echo "$1:Low numbers for packet size $pktsz " \ + "($avg_pps < $p) for $3 cores">&2 + else + echo "Test Passed" + quit_testpmd "$TPMD_TX_PREFIX" + break + fi + ((++tcnt)) + sleep $WS + done + if [[ $tcnt -gt $MAX_TRY_CNT ]]; then + echo "Test Failed" + quit_testpmd "$TPMD_TX_PREFIX" + Failed_tests="$Failed_tests \"${TN[$Y]} inbound $algo pktsize:$pktsz\"" + fi + ((++rn)) + done +} + +function get_ref_mops() +{ + local ref_mops + ref_mops=$(awk -v pat=$1 '$0~pat','/end/' \ + $FPATH.$3 | grep $2: | tr -s ' ') + echo $ref_mops +} + +function populate_pass_mops() +{ + local rn=0 + local cn + + for i in ${PKT_LIST[@]} + do + cn=0 + ref_mops=$(get_ref_mops $1 $i $2) + for j in ${CORES[@]} + do + tmp=$(( $cn + 2 )) + ref_n=$(echo "$ref_mops" | cut -d " " -f $tmp) + PASS_PPS_TABLE[$rn,$cn]=$(echo "($ref_n * .97)" | bc) + ((++cn)) + done + ((++rn)) + done +} + +function aes_cbc_sha1_hmac_outb() +{ + local cipher="aes-cbc" + local auth="sha1-hmac" + local algo_str="${cipher}_${auth}" + local cn + + echo "Outbound Perf Test: $algo_str" + populate_pass_mops $algo_str "${TYPE[$Y]}.outb" + + cn=0 + for j in ${CORES[@]} + do + outb_perf $algo_str $cn $j + ((++cn)) + done +} + +function aes_cbc_sha1_hmac_inb() +{ + local cipher="aes-cbc" + local auth="sha1-hmac" + local algo_str="${cipher}_${auth}" + local cn + + echo "Inbound Perf Test: $algo_str" + populate_pass_mops $algo_str "${TYPE[$Y]}.inb" + + cn=0 + for j in ${CORES[@]} + do + inb_perf $algo_str $cn $j + ((++cn)) + done +} + +function aes_gcm_outb() +{ + local cipher="aes-gcm" + local algo_str="${cipher}" + local cn + + echo "Outbound Perf Test: $algo_str" + populate_pass_mops $algo_str "${TYPE[$Y]}.outb" + + cn=0 + for j in ${CORES[@]} + do + outb_perf $algo_str $cn $j + ((++cn)) + done +} + +function aes_gcm_inb() +{ + local cipher="aes-gcm" + local algo_str="${cipher}" + local cn + + echo "Inbound Perf Test: $algo_str" + populate_pass_mops $algo_str "${TYPE[$Y]}.inb" + + cn=0 + for j in ${CORES[@]} + do + inb_perf $algo_str $cn $j + ((++cn)) + done +} + +get_system_info + +if [[ $IS_CN10K -ne 0 ]]; then + FNAME="rclk"${RCLK}"_sclk"${SCLK}"."${HW} + FPATH="$OCTEONTESTPATH/inl_ipsec_perf/ref_numbers/cn10k/$FNAME" +else + FNAME="rclk"${RCLK}"_sclk"${SCLK}"_cptclk"${CPTCLK}"."${HW} + FPATH="$OCTEONTESTPATH/inl_ipsec_perf/ref_numbers/cn9k/$FNAME" +fi + +function check_ref_files() +{ + local outb + local inb + + for type in "${TYPE[@]}"; do + if [[ $IS_CN10K -eq 0 ]] && ! supported_by_9k $type; then + continue + fi + inb="$FPATH.$type.inb" + if [[ ! -f $inb ]]; then + echo "File $inb not present" + exit 1 + fi + + if [[ $type = "ip_p_msns" ]]; then + continue + fi + + outb="$FPATH.$type.outb" + if [[ ! -f $outb ]]; then + echo "File $outb not present" + exit 1 + fi + done +} + +check_ref_files + +trap "sig_handler ERR" ERR +trap "sig_handler INT" INT +trap "sig_handler QUIT" QUIT +trap "sig_handler EXIT" EXIT + +SSO_DEV=${SSO_DEV:-$(lspci -d :a0f9 | tail -1 | awk '{ print $1 }')} +EVENT_VF=$SSO_DEV + +setup_interfaces +exec_genboard_cleanup + +Y=0 + +echo "" +echo "Test: ${TN[$Y]}" +echo "----------------------" +sleep $WS + +# Outbound +# aes-cbc sha1-hmac + +X=1 +Y=0 +run_vpp_ipsec + +pmd_rx_launch +pmd_tx_launch +aes_cbc_sha1_hmac_outb +quit_testpmd "$TPMD_TX_PREFIX" +quit_testpmd "$TPMD_RX_PREFIX" + +sleep $WS + +echo "" +# aes-gcm + +X=2 +Y=1 +vpp_log inl_ipsec_perf +vpp_stats_all inl_ipsec_perf +vpp_cleanup inl_ipsec_perf +run_vpp_ipsec + +pmd_rx_launch +pmd_tx_launch +aes_gcm_outb +quit_testpmd "$TPMD_TX_PREFIX" +quit_testpmd "$TPMD_RX_PREFIX" +vpp_log inl_ipsec_perf +vpp_stats_all inl_ipsec_perf +vpp_cleanup inl_ipsec_perf +# +echo "" +# Inbound +#X=1 +#Y=0 +#run_vpp_ipsec_inb +#pmd_rx_launch +#aes_cbc_sha1_hmac_inb +#quit_testpmd "$TPMD_RX_PREFIX" +# +#sleep $WS + +#echo "" +#X=2 +#Y=1 +#pmd_rx_launch +#aes_gcm_inb +#quit_testpmd "$TPMD_RX_PREFIX" +vpp_log inl_ipsec_perf +vpp_stats_all inl_ipsec_perf +vpp_cleanup inl_ipsec_perf + +echo "" +if [[ -n $Failed_tests ]]; then + echo "FAILURE: Test(s) [$Failed_tests] failed" + exit 1 +fi + +exit 0 diff --git a/ci/test/inl_ipsec_perf/pcap/enc_aes-cbc_sha1-hmac_1410.pcap b/ci/test/inl_ipsec_perf/pcap/enc_aes-cbc_sha1-hmac_1410.pcap new file mode 100644 index 0000000000000000000000000000000000000000..c676a133d552b49817160954a61579660c64d0ba GIT binary patch literal 1518 zcmV2Q8;4O`vD_h~ajEDappr3ts28(0na1%8X+$;E*_(HY(zWUe*yRK}1X|VaVgId{ zbsa-nHH*vVPAeG|gV^H{{k?yti~K5_cHr|R^`gdvO`k2D@@;cF&3r=8tS)bfa0 zK1OXcBkvXjsiBMU+pHLOw>x_9mF0tb@nI2kFcE z7Pq0IFz#4I^ESG_C+dw@Qfi)|Wt{=6kOr`J{DJ&<_|AwnsAO=wy6+Bh76Nl!S-+cWJun&hY;uHRS zi1i{{_2!j%T+Er8X=0*PzRIw`Uv}jL!Gd=IY^npnNcI$DHB7AL7tAsr>6Sq%M|{-O z2m?X#s@cp+E@{(3Kt;?$%6RGxM(G6R1phUDBl%Fk7j>_2IIwfZ2?)~)i)^;POdn$oko zH1Qyu4QR|%5lnm&Ra|}wvT*7Ng6;2`x#_S8zkP4Dsz&c;6Ro)CZm_M@+qOEpB>JOz zM}B#_FQaC(Ofl~k@izoQ4j@NgZ*Y>enl@N1m9n@xw6jQIh0YSRV2@vT*jIfPu<^l4 z`i`NAgg#{OC{WiylgHkeQ%#Fw)sKQ)&o;7#AXbiTsg1D*BE0NN=E1bEi4*cTJ>5Y( z2xxkOfAeoQ(I9w@->fsd@{Fy(MAk}JB*$5l=|}<~9$rF5@)?^LyU{m|ptRqKe^oHD ztKw@6FdZ;HGHUuYJi#G!j0VRXF zv2N=sHZr2q4hP3%^JpKe>}d4Krm%>#G0{hsE3Q6N+e%jEN`Z~O(;)NCM=quoOKIg6 zj61wU%zDQ$p;Ry~(qv9>c8oH5gY{k*`$ESY+xQ$G#h<7n0 zw31K4+tuw7(1&k?-bl0T?`8xPx^?L8`yp}%W zkdh1{O4ODQa0IZAs*~T5G%QLU^F!56`2GWG`s6pc8n&*{Jns%uIjhBIA3{(F4aWOz zO70B61f3i`lQ$Z(M{Lg>dxh-)0C^1xKLlmE908vVIW@S}1I0hQ`1Hg=81=DS@BX6D zk`K2U3c!M-5Xuzq2rk)m4vz1?0QfyF-O})HJAL=nWX0Hhu}rz|)J%Je~RJ62xWA zlS;83Q9=aq3ChL2C=eJuGRYt1gKlN#rv9oVgKr^G-`KH6Z<;E3AB8B=6n%K6wloP; z%A0?WYY>zM{tWi*pQjy@6yu@ypF@0f4oYM^PUS_z&<>vsyHth1U;yRQyK#;GvatSv UPvx}djU`lue!0(s6U|Ucsb5sxEdT%j literal 0 HcmV?d00001 diff --git a/ci/test/inl_ipsec_perf/pcap/enc_aes-cbc_sha1-hmac_380.pcap b/ci/test/inl_ipsec_perf/pcap/enc_aes-cbc_sha1-hmac_380.pcap new file mode 100644 index 0000000000000000000000000000000000000000..75852cd6d4f1ed8e824fc62c30f800b903d5ea9e GIT binary patch literal 478 zcmV<40U`d>!?K|Q00aO4000000000000aO50000LeJo*!ZVUjn0RRBD0RR917Cs`O z*~Zo6=ttuS07U=+r~m)}06;QzAOQgZ0RaI40ssI20RTwnySo}Z`Y-6jG*LBZu^aWt zF1v2-3>Q$|nr{X@&X|JW>u zJkvuvcJu7Mj8p$Y&qVcxH&fYYFDaliRm6B+VCm2+Ty%`TxJ0>~B55m;*2YEo^!4gk zq*m~YqW}IyxgUJ4@`7@)+G09z979crX;9Nv-O8Gb_IV(%oeGwm2Z#{d2H8*z+&<3M z$XYZELU{Vmwq4Nt^A!Y_Eo%L&R`76jByQc;r?L0qq#fM9i)u5gA9cK(ZflIfEo6!* zL|PX`fM`0Bp~cfQEZfyu5FgXG0IL))G`c#&XEohY<>Q#ep=c+sz^p<2W zj8-k(Bf;YNqNGFcvj=rVEcqK9kIx{MRqTY&6ZtU$W8X;CCc0D4782Yxo9d=p5Ss0Y Udoo?LfhU$~MS=;`eD5Ec>zFRux&QzG literal 0 HcmV?d00001 diff --git a/ci/test/inl_ipsec_perf/pcap/enc_aes-cbc_sha1-hmac_64.pcap b/ci/test/inl_ipsec_perf/pcap/enc_aes-cbc_sha1-hmac_64.pcap new file mode 100644 index 0000000000000000000000000000000000000000..1de5de67c895849e8a3b5dc70645dc8787293be7 GIT binary patch literal 174 zcmca|c+)~A1{MYcU}0bcas-mp6CMQoq6RqcS#OJyjYi&0a$f$n9Cv-c3DIvmIl4UpVv?B)v zs$X0R{lN1polL1poj57Cs`O z*@`#&_Q?te07U==tN;K206;Qt9036W0RaI50ssI20s)%lIsgCw00EljI(>106Y>JF z98xD0gB2dMfc1dA!LS(>%zhV^c+t*vhB|0HuI#>tgp8T}AEv9>&Y zPgA8GHraLU^6)s$D9+nl_I%ic`Q7V@WKl4JkhtvBCFpP;2NH$2Uj1vP z7A(SrS`~aMINRa7Wgar-T{M)e{LdE{}QZ`Fpgf zzVr-lL2NEgxRJ4)gt?v;blaN!r1)<33sGys-hGC)P_xb&ap`;A22FMa zg+^bT)kKOzF1#sQU^~8YTTcDkesBIx*1#d)9I_Ffwxn~ovU5TClapz>huO(m$+3v3 zMNghF0U|QwJ{~QK;vRB_j#^}H8Wnn?MIMQtD8N+6HLrP8bkNBH+HGX{#X8@-o*@MV ze8G(BF!#Lc6GyQ2C@e?2+m&UaeiLKL_AO6#aY&b&2T**nSO%|W*q6h9)i*SrKPy`1 zzDdRH797Q!Z##y<;evJ~x!RZ1T@Tw=>ppzAtDw1c%F#qgsKFjJ+2~;+k($?0)Xv7S{S6Cpi|pwKO01Y8bpT4Sij*n<$QlBrkO2u{l)Be-!8@PrI1+~-Ljngw zGh8koOYQq8N5R)RVhXK-d4Ve^BQtRV!!X*gE0fT)K^&^}R*<_@H8GYTpuM3l8|{qb z3pJOn*B&3}06$D7VVcti`ZGZi8i85cHDLWG8-yh>+!|~Z)Ovm)Sn?bXFqbZ!^axst zAWl4r_%5SlO#BKh*kSr{jQYw!p}t%E1u97_18mR!1#bc~>ggAws*Uh7*hPWm-odo% zG~VeRl*F`UAR(0}@rO9O_kU`6Kx_7+?Pjy3)ePH|El_3-_$(Cbb6uRU2g3!XI?A0h zEXaEO&~+D=F#A)pw@gCY*hED4gvbPYao#fVMfN!GI8v|7`ZF;VZLWr`*+}x(w>v1a zq!4kg864d>l`^XkzZMJzY+VWd43nM(#Lyt*b>EjQ3t@=JnUu}TD0L}8n-Oj3fyrMR z&2yvTPe=l>ykDToI|#@dJrb$K+E-NnmX-8tj_?F|AAYivz35(B*dp;Nm|})b#2;3} zJHxSqnOW$Ytt+Y@WX3W_3^&+$la89^=s*< zRkBbaJPfN_yCg!GTBD=ZI+MfM`kEgJ7@bLZ7bTQ6$7)0}wkP>4Z6V{0`=>NG#2C@h zrKZ+a9p13yij|+zXzx3Bqt+aR zvhJyj7=I09{#uC&1SR&CT66F-MJ;DLpqvwv{U~Ud7Qe$6AyRN-jKEtW!s$2zeU>4~IjZC+UjsmV5ASUnO}l*}F-MaOLN1=RK6%6__fXmD{yc26wcV5I&=%vjou zyha1ThEd_7O^h8=dhD|k>LhDyDcSn`?xttx4|3cPh`HX_6vFQalX?b#Sgm{+Ox}zD zZC7kJAnt(;kyFcprOE#bLawv`*m8K2sKWg!&=g#L$y>L!<2D~IK^|la`b)9dRQ1)| IX~zX}d{ZXI#{d8T literal 0 HcmV?d00001 diff --git a/ci/test/inl_ipsec_perf/pcap/enc_aes-gcm_380.pcap b/ci/test/inl_ipsec_perf/pcap/enc_aes-gcm_380.pcap new file mode 100644 index 0000000000000000000000000000000000000000..62c4f1e40157c912243d4e19447e9e56a9c03a2a GIT binary patch literal 474 zcmca|c+)~A1{MYcU}0bca*~4`5}V#~GHe2JK$tjpd8aOpLPAeH>vZcR$H>(lt0_Vp{?U)>GrGpq`lm)e+NUX-1U0i zv;EJgi%%)H-4q>nX!7a4NJm>Gt^!Ulwhi7uSU(4B@2+>~OEwuUF#%QZ7 z_G$A4y)0(Td7C_i^xzRR|Kx4Gv$54;ty znJ1>abIy-OA``$p$bgjtDA&gEQ#Tt|7HKY&+X@qXD^tkKwr#t*OMlt6ZQEV8ZQHhOn|H0V@6(NQANGhCGc#jGG=h=rb+o`r}~!rY0-gtg{GZfsZKW6XsUaUq*Wj8=nRt4WT?g3Wp9SJl$=T^^VM<%VT+q2! zRF`j?#^iWGX+COcdDkMOrIvNbVDs-O_V&DSrLT}+MstS)J!gH%`innf@suj8ve{#( zpdN@CY%_`>2nQXEe!v*G?h&>yRvJclSaQH7@OLC+BPSIV&+{bs%BsaK^H!_C5dGZ_SC*$I*>zcPslJ1h<)cfj!;5Si#XsidM@K|M4Y0ivqyI=W( zuB{-dL@2Xd6HBwtU!#1i2P=NBD&>sxZ$}vGzIKbcLiJtha>q>abzWnYvS1(yq$}da?dQ})H@)ZIdfFQ2lc*F=MLQiT}sk7_AXq6J^ z@6wiNlB92gNx}2Epq$v$dNhzqm9$#t)ab`UHJlF{tPwE${&)T^19MFxJS9=C9aPx@gpGQ^~x*BbF| zNg3@A&Z>K$bGYnYRi#q=L(_8$@hQY#>3+8w!os1F4{}9Z-Q(pwz|G5_>D}5lXs&9n zF5Dt^Q?%KVu{anApI&!_D!w9eAeL6>vFH7DVt+vU>$%;;1h8UVgFJ~owf-1hq0YRk z-t(AAN6*3b2M@MDlx0(BXs9+EO)4S8E}kU@yt{FtjbU2>FMg%XC@b>lR-L$;rY$)q z{E87F-atKSs)5A{IFDLM3#Z8cYFu+@k2a2U>QudSi1( zAxbJ~s_3aX%hO4#hUYThN@6y0k+HUz{nBu=@Bt>ku~xrdSj(UdKm*trKB4Zw*Ofjv zP8$kUnxGrFb!3Ugy5RpoP7nS_)^x#gbGG4@M$bf4-@6XI6+na0le@F&VkE$?^4P(; zIN4JU4xGw1n*xc_**(4OUYJHtL$Lzf#$#HuFhNtrjqP-Gl?bh0v7wR>MZv!Aes>-Q;o#OP=>Mx-xU;jgK9 zV%He;2gQ@}m#`rwEBfB~C3$DWOG-=*c~rAxh3a1vl!FUgDp6k%gN+isAJQtvXw}9@ zqi9>XEXj50KZ67E)?{;1$Kxi>HOL({xprIw*Esg3>kQrSlngYY0FoPR(AlFb?+sXW zmhm!|3lgG}#dg^*vug)C@E%~5YKc-B{@?5zHW#=P(K&z=HsTH3Pf%Br-O5DXntxrz#4~BZa7?L7In#+~;^^fs!UM9bNdeI zP(2_UPI@ss33eT@p#rG)CX@{+qfZvYmY2)q5ZnQ#Ja83w@~gzy3saRS1WSyHhxSwv z1{^t$OiK{P!lE(B5z3|3<$?NGw$zEg%V}4kjK0#*J1;~@Y;Q6dKWJ%`3KKCqN3PR_oGGuXXmlL-%xfn=^}qJr#Vup0N$1sojTT43^k#V}wMO>B zL;Ahdd{vjlx?GRG&{XL}tg#LkuBa7im-4dseI%jiFO9s5-Jr*yTYel zQ%8lMPL;yg6z(a zf0da;1>~PX*g0`F<|u9jINL5EQdXB$`aYxCsw`&1Pd#^3;Re<15&_+Kxoj%LKF^yl z86CYG+J=}*fXz>jHIX>re}p5vj9&-WBVcsF&sKfTP+Cvpb_`|1tO`qdDU_pi#+oX zeTjUbRUbwoi|uabh^iE@PwB9QJ@ny2Tin~(A-8J2yA$f>ZfdL>5FzmlIo?Azwk3a) zU3midhd>4Wk+43r0ON4xwKW*uVjFTp<>o2+cL-hZ5ye%OIFJzSyV#!}Z>WEiE;YR> zho+IA)Jd^Oc?&98Z38N0@c40VtL(3&92OG^-OlAy=^x;Je5j8unR|$pvjW$PQxoLH zX_R)=Ww?Kl%}fVAWw`~V#EO_Y^2!sG6*H*N?lU1KurK~Rx!?gN$s>By*1czKWvwp} zm5P-cTO^?-<%@c!6!d(n#Tz;vJWDoPjwxv&IIB_oN zP?O@x1X~|`LXukmY+OEt4P&KWW>^@SEXK6nGk=$VJb~hI9;rc`W9m^6;kl6DXBvA#)mk$CmiYET0AeKp)u^wF*rC`4s4fz93_BGt4#Q zu+6V0k>x>|GZ$B}ofs*gqB;<@OXDK`+{{S-TB1B5?R_+T0&zcQt>jyyc)r!yQ|*@s z==l0m_l}cr8Sa!nY|m@AWf_a?G<3IeJ5+iys;(BMjtBYl+qVA(!+j=8n%g=ZkO9je zN21?7Mu@$fJmQs0e4CX!q@fdgf@&}8X&iW0=WutJgJ{QcV z7ic^x&jv$QI=j{~@LpJzE?|{2c3t=bvZuV9lQ7U!{ecRB?FU^WCt898Wyc`eSA-7S z00<(5p+pFJ=Uhm{jyNhx&;ANnJ=Z3kf_m{XQHMH>A3zVfc_(7Wmx{X8IZPJ)J{k>c zivHhCr@mbN?@R~%*Yx3q5OmnnggvoOk(J-e8%|BjGS0y?FA-sKglg*E`51P>EAT;J zK};&*>ersqA#kdVSV9&f{2^$$KzWHF*~tXaUst&DF`(K%BX{TPAs|hCA5>2bze2a+ zz1WATe?ON9Jm~m!@+D{Pw{*F2MbatZ1lE;?I!_~&eY8cHH^z4{0z>Dh;h7|=p0w)@ ze#go%{h`3NGq}eLlw0(VMBT4mx?ysap|&WUbpMq-)2cdS0uq#GY{*qxzoE#HHJI%f z1j_3ZaYnIt4XD@m0si_7Jr zx-TOQqiPA(+o6C`x&H{d!LZk$K=9s_GtE3WqzTRjQiIK=!;G4uAwAT(_}qBUmi>tO z$S*8)FUwvd2ZK9MIE5K}>Uec%#i83&VFC9Dt3QZ7kx9)Z=tCos&!+eBev%X;Z zaBs`U%V%JeE%{4-a)>4|kBC_offvNOmWfd6u(hA=~>5KI0 zie#ZLnCnl!VXY&kz04L~FtAr+KROsF?bt#ndf^6=$9`#pi8bd(NDfX{-_~d847ri0 za!h`a6w(?47q&HF`GA&?96@faNE;YlZn9y6{TZk|1qzyaqzpwtFAXX7tcL3f${fIZ zEb`HT#zQ3;=%4kPA%uRjRVODlK`8ec?h&*uEzav-@=+&ds_|~olMapN96dC$a@V;X zbL{rq(g;OBzwrEbv@>*PeBr9}sYqmKeBLV;$Hl4cfPV({8c^)YCNx_{i_y{RhP}Ay zn3u2Xs~Zwq=H@;>GQ&oQLTU3gKVh{nt!oL!WB0DH^GKeeIZ2-p5&6A63dCS9r6R0M zj6JlaM8y89Ny%3ou&OI2QxH`$8YUy6T##-~t!R~BnJ8(#UJxKIseM=33x2rkH>og> z5kPctfd@85(z{tdlq&`!!;yK8x<88^vrxrq46Hj=?mF6OdTkWs?`oXw1GGlvO0J1@ zpFX`j(8Kq03bMH&GA4HjoYw-)b-^IPx%Kqz1!rSN%6tuDXR%%5K^`qRz{tFBS7qsU zJv96RBPZ4K7vqc;i))1`vSOX7&->2+_GhnG`94W@G+tg{>XFyPafr`6MY?kAducj( z&|CN!d+BjYWaCq~hFlMaZ@$Q-lYzJ8KvP{|fN?o~>Y4Arnme6PK@sWe$}TB8Vuux- z(p?Ib4K1PAJk43fWaY>E~MGk6&->d6YyXX$SVxHGAF z+n48Dq0}8{LN?_+Ec8MJ4u|bdr&>9zjwC)H=%;njWr7Q*}2b={VaG7~LG9AZ*zwM}#C* zW7gz$**)zz55A;v@e=iM9u2n0+_=jtTox84tQETvfOj!^LTfZEIJ3jHA^VMYgs^tE zW}hGfvKPeBsz6wLk-BR?^6nap#-K}HSInsmPwOeYGfl#f&G#pDCFj<4=IER+#(}V9 zygZ*0TAGagsdCp?G~O;LoF$5qe9ICPPoVQd_{~xo7JW4=Rjf;I`J^3%em>-*AIstU zLfG1EE6f{eQ*Y&_W6w5D7z=!4{J<*>M{3;t7^kQMcV-4Bh!P7W-|Lj2kgby6{Z0kP zPuJ-Z_ix*+T4$@Eg`Pc9P0hpspO4UmL|EXWqKcWT9#M>e43)*~-YSG4HEFHf&?9S(I{XU z$670bA>M+SdK0K7d;?!q|Y@<}VfLBsPoY?X^rMCx&9~K|)@+0M90k>^Z9jHbP)T zL6Aw+)S_$6J!Q^|u9pzI&F;8v4iDJXan9t_?k!;P01;;or^l<+Nb-aq zytdRp${}>-pGVZVENdIr?-i>1lM#L&#F!wkPNl@6QUcH!-G@6>Lh#uLXy!(d5a0p0 zn6^uH?i8D-S~XU)gNCwUUZiD*4aLcfEsK7;1gkyc=C;cn;ag^TE#=Rm_7QG{M?QBt zMY)J@xd-FBif#&UjYdgjGQFgSSfq~`^W5i_U9v{4Wg>iMKIzr;w%4yDt{<(Rzp2s0 z9tXFuwb;e|~3sK$=zw5_bBzN?@^j1D{O!E~@DoKdV(MB}%`rc0)=|db&UOC*-zLGiLpX zG3p?4m>3OpSAD3Q{dLbZz`{LktRnnZd@|e9<^jz9L>ne}{`1}K+`fpo( zjkDFo=cI3~3cu8|iH33Z#J$wwRBQF95Ov6D$RJ({3wy%c(=PqBM_ciNqvxYeP_7Ub z;*WVRL*USXG+E$sriq(xpNWMw(EiSn2i}W5qM;Z}jD%4M2s?)+lpWDNR8t5^WE>4Gy9!gR^ zf~~xBySCNPCWDC%ZXiv*?z{IbX!*>rgR4vgK;!yezmCMv_?8HZ zo90OL)$Jf^m)akL+OXE7PA_+6&xSntj0r>8kP_D~8XBYtnNBVH!yg7qfpatqNHh)} zr|teBc3S<*rqIBvt;*WVNDvD@h}nHmEUuL-<)p8-XSMylR8V`H>n;)!^g9+N8-l1%krFx>GX9%77zbcz50Ojix>cD`dNdjUh@#i238m79Xjx zLky&kjrSa`e>-5WJEgBZGr{Odmy&!&rj)Eyo&bG9kRPV*R;AMF=!fiO1hY(!YrxT< z8_)J)>|~Wx_Ssnb8Cyil;Ute^*ZSR&Hwt$8a7;96P|P-)68V;?aq>cGGa&=#>n@H- z5sO{$@twqUIvlEd;_poDAc1f(nslIV3(lv9v^*Gy7Lq?gvWq9%CER02MkK;i^jzh- ze1jTKX?7A8!J0Y|tRHCh1jd)&TP#SW6bM6IUNZ1lAuW15_TrA8;w8U~Er%pB58b@j zh|b9(K!LFDQ^l8E`J8VplD5zaS*BOOHCL>4rJ<7YWhKTkhR#UXE3c(SO|ya+cVt(Y z(v|SC0vGl0Lz(4%bBGNCR*xZ_Q;T{*jG@pAo8crsRzo&tqCwo%3I0xgp)CiPpagbz zxep{Xz%1)R+&;X(KCXhL_4?O2E8&u_;nrLLnaQ9IzT`t?>c*@gc74UL@7#9cl4C61U{>NVF?2PV?8^5ppNl1cMAsrwm1(peM zn3MHP_d?KUd%|V_{gMv)zyO+K>gdNO)4msI|Cji3-YyJXg)96FBg;W7WESGZZ(F@r z;JAJ~9cN0f*Tt5H2PF#L_n|@1>;YSG(e|;;e50@e^!QlKN0DyOq4rK7(Z+dKxZOnU;-sMVPgXyL$h5DJ$b%eNTFbrp?vrCBVucflWUxo0E%-*aPMv^N@*hb>|Lo2FK0o273m* z%c~zfmuP?@H-@vx<_iTwmkAK#wh=Or$pqtd98)yng$nF-j(P@x^`ygC-F4E++^Oe+ zgzV$-bA4%D3YC1sruTARQ%bYc(BB;?4%zhHGxA28BuTh^s-lr4kp4Mw!WKJ z%s7pr-pw>=bl{}VxOe3z68i}Tg;{!g79Mvd?BT%Kj@!H7@jZ4skn!aRZJE`UkC8(p zz7eFKqM{wmqKOW+U&{(P(2&p7d~vlT#|IxXHh8L2gI^NF%Ig%X>xUxWd|avr0;#vQ z?4whJ!4NF=o4R3E?uEf`+?yB2*H;an*_TLW;!{q1*If@K5l33IDytut2;v<)^@omK ziMq-@6u=X5H=hs54DodAwt`O36r*yc$ifdi@KR~`q?JKA=TH>?G0S%}amG)YNjk?1 zN*!fO)XeP?Vg->{2}gbzFX9Yl)1#yL=V{JPB*dOgw@7=1%t>)iW}4iuP-5(XvowQX z=9+txh9w*qMEXSF`+jWWgME;EKwjo)Cob^?Y&ShPfKFEd;ey#848jwYgl0k#M+KfK zd=IXAl;_Iz9B*_ue??JxJf9hYBD{4ua~o3}Q5CDR*wZLLe`)mZkj5~RJo4s3wY993 zfhZg;TA=`rA=De*g_G*RZyr$FhfQ>GXrf;5j)%gDKf+^`4TOJ5=T)CvYZG{&dfz!= zFiHS&O-M4k@@^J~EWPlS9}8ot$K*(q-^qye-GD$`1G+3b+EP0}!UiXJxFk`3#}Bud z+nYKN(=L{VJ+t`aDZ9uQ#4`Td*jAhdo7y^;??KD)Ql}Wsu2m+;A6NR!E&4u`r0P%k z39Im=EPK%6Fnrh8z>Q=mIUc?PZ~h}H@9^q%`~Z1mriu7gEMPck6tU|q9t^y&AJ*wi_z{cOxE37j~Z52lq|BF;km@L z0FKo78ZudrUae%$7alG9AM(~E%DahO*|$~sW!}_w=Q3`1lPi^UIBr~gpWQ?iTQdXM zak%X-K0VqvHiPOSqc|^#a5q+ee3WNAzApRei&-MeSfZUVghuTeHe+@xfqFO?gWdc! z*Z}YivOHL}-k^Ps@GNPXUj}-abEm>WG2y>9I`~xRH>G2Ze{a=(eYD{U_dRNhs1B|= zIap;Iejwtt4j?AD`Z;EYO%6CwvRF9~O#<-Fzn=yN?>$e?$D7~Mwpj%ia!jI8iL)Il z#U?8*5MB)Od&6|WyXa=#W#R?~F@oKiuA0){fwUk3a4r)&~-A?@h zHf?yzgv#~S^d!MDFRsHf@za!CwjmLvV~i(ZjC16+vFksFC7Q$A>O_H*^DXZE&?$y* z-aB87aCK>26D!*8)c1g7jmhcPT&?J6lwowVcpjUn0lFsX%Q6bC&>itDomC|?Svn(w zMFvFt!{cK=)(J%covgb3*XvuU3ve0{S+&4PNvSZ9-M^GpCvM!jS5bz~1H$x#c=*r{ z9{>9}{axhvzcU^3U(<^{x+)wt_E5Cc`O3&AX6CcZo&GRknJ{f2^Vug+?f6}_MD0Ti z1r$d%S9ZY&Ar#wltGx(Q!zM{OQn%PfT9uE~3tnZ}g&mIft>g*W2YH&M{SrPGVI-S= z&eHxH^NY$EJC!HEw>L$m@uI5vbN93_wpb@HPYQHos$A-4{*fknAV~OC)xHVk9m8}5 zshQIipByBYaPR6DUntZPR?WrQPixr&=M)V-*9;6S2u`@z8Z++rD>oYdg-2tPTRSdf zJJ%S_EQaG5bY%BcBj`L7Mq}lMUNWO@Qt}Q0Klp{B#d0c>??;?pw zxa=!-JSg_)AzaYK`z)ZKW@tK=63gTz`>ywEQp}T8ZpUvx!T=}ps`vLfJPl*yUVY(Z zlF0pmb^8(G!?n`-T7c7#h0mcInwsUip%!6H1o-n#Txukt#+|=VD zN*M}mSkCZ=xd4LVf#bvZuu|Kf!YR+V&Ub`HJ&mC>&KB$@QEWo#EHG!s z1dB_I4nnX^!u+tfEa-jjG-)nif%pF3-S>(%tsCRs*SN?CYXkh?k2AX{e@@U2q8u?h zZh|$FZ!RG>7LsqR7)HGqs(?Yz%&ZG}+8O1{ybLNnVD{iF`2u^cJiTZon#fr@zr!)r zXtqV)V)AeRq}UiHFaUi$UIDBh=*0|pDyylOA_%oF&TNS#ZLLrHq`YUItZ7W^y7Z15 zi?{>mk*(G+8Ua3g-JH13&Xqo^@4CxC7*VFPOxQ0U{IgUDe&_T*CeGMc=j*1_6Y31Pd&I5p7J<6b7B4-KJ>^Bb8oVM?E5(8j!tm@6mh9 z`jxeJ_j3EXSO>egokFnEwg=s5XItE`o(GQMfSYTpt1}Cw8QGSSx!L>TM%BCAPIF$# zE2M#b;>3DEn7=pB?9~0Tk!c4c5&3d;-f@%EAIK0bo|p=OIoOB$S?%!C2kM<>%pRK5 z?*0)UqeU%ru=;A6x)^=9VOMlqfnFu-ZWztCYMDMO*UV!G>#v^ib42UI-Z#G*FZv#i zFQ}fUg?QJNl+Qo}AK7bEJ@Yh(0@){|Bme|UW$1fI!~&C`Xltgy9nV(i{K0wQDA?US zdmWKVtj6GMdAn^A3<)EG)>|U++1=Uo06)P0ZhA1}`G03R)W4<&wxKe)5mSlbYor~@ z8R@DIWJZrvzG)~nj@(Du`22v7qgM<*olP1xPmn)EdEpYNM0J0fG90x1J@d5+C1)1j zIrBLiRVSr>5s&T@5$;Bbwg^NfnIfUHh&<}E9Ttw!sZ%Tw={dY3^1MrIN$?^SB%ILTxf)1;;p}l zoz*UPfn4QSxoUU+c{+ToZv+wC9YvW7jR!UZS0cHR0_Vr%OCUT@gsP=I9)W!wYDO`M z@NAVTfgvae{6`n}@cBtc>J1MB2k#XoQQFVeQS|CJsBlKWMq4-iM|`EPugd@> z1NlA_Yo`C-2cqqui4wW%NY`P->aT7~u*Rc%)IC&^g_->l+Z}29FPe#6Lo_SvhQwgz z(rfVEr&eb6&LSN)r@z*$AT}BAP$uBjB5;$q@_&6ZT%{}Mke@s_H_ZG7;|j13K9ovn z&uTJB%=x>5^E0_@mh{N1{eoy+au?zvH(NMV+pH!s;?lV+@f!lsJ{`EIHuP^=azE(= z%KMU(l=8M+c!(_~x;TOff3ll&T&bb4r?T8H40**bRD&Dkvt$NneD2F)e|%Bv>@=ft z99C`zbOu1sG-(X)bNhF4c{GoF+f(zBt|8ELD(l2hxH&%95x(Gpj2y+2XMM_vJXLyY zz|h<_bHq4qv#wX~=TB}!lA>C6waEA8t@V7%OQah029u%!kW~m>+mMeN_cC#RUKUNW zby~lv78BC`);||}$uU2Xn-%~VaSI9GT&}wKl}@Iy_))8U$m^2~fHR9tjGS=|Wm~V% z3BGY4_1I>g{ObR9rh9%D&dbgqM$W!I$5U~kHP{GTlddGAYiD`NbysZL>(q8U-0aZ4 ze4nXc9#r2%|6r^^$b^SGgGmNNxkZP#4Bk8(pIFW7)mcERk#&Eu-F0vtko)eJto zLO1QpZ=Z&2Bm<)wE%7TM#i3Dy64bSW>cqQa^vyBYg;0zNaY|?rmY=yOF5r(-DNCc! zwf0&gb__%7are-=N(SYPz%2ba5`K{DIK)c&3Sw5!vw=;VZX*d@QuSMcTiAI2q}H#S(LrN z?sVn3QV7pJnf-C39p;PxmH7k1!dCB%$GX+0Xa}O}o2VdvzUHV6kRjmv)&8<=un;rx zNbr~Dw)9@j_n{(Mx!`uGcZX^1bAA7TFjoF6f|;h(gq{6!7MZH8efV^U0#0bM>sK)& znOb`JzH-(D1PPGlg@%xIB>jQz>;V3*1Vp7@wd=M&hiZN?hQiW&d(WqX-%UVmJ9Lys z^PwXn?E2ci#`H8hx)H#+2I}3T4fzr;mhE%3Zw5N6+wjX#^^B2Z>m=D|8*$)bxT2bGhxTL1t6 literal 0 HcmV?d00001 diff --git a/ci/test/inl_ipsec_perf/pcap/enc_aes-gcm_msns_380.pcap b/ci/test/inl_ipsec_perf/pcap/enc_aes-gcm_msns_380.pcap new file mode 100644 index 0000000000000000000000000000000000000000..43ee4e8100d698ebe60a6dd17fa2eda61fdd9879 GIT binary patch literal 3888 zcmb8wcQ_RO9{}*P$vSdp#)UFM#u?eWv-eIy&LQg#$>>}enOS9iQMg0a*{gIYTSj&v zS=pnU5q|mh{pKcDyec|V`e`;XTjp9}Q#7eW94fPvyQ4w7Hz*?&O>UrLYYPByzVmz^%YWzD+3%Y2SO4*R@GC|+C;_5Z z-bGaX>d*7yzrp-(Py@68Bn9VBs7n4Ad0Rw6`YX@x{hK!ya22xoKRytUMVmZ?YrF2r zTE%IP_w%xk=3fhz)3|tdIK8xbLKh6AFBIV4#qyTb+1mrmFAMmtouL+&?!KbNiq;Lt z07c*2|^r!v6;2@Yhs#x7OY1Mi}h^J-)w02ub{udwvaH3xt z93-UUei7$=om*wc4&Z`&QZe6U<$1s72DRjk8sOIg>C{D%bm+n-Axd0-*Xb$q{O}w6 zKl%%uUg>j?HCb_>l+zdZuQfK;oot&*Or+^cxM|6R%|nTh?h&~U^dqwpnxC+=*w9(y)M%S~fxUF6qGOBkDSyA)0&9rMPqky}s%u?N{^!k0Ui@o188 zUa2Uq*yu-g>t=SBEMcAeprw8M@U*0VFMV_!1|M~5XkT6pcms`*f1Eo%$8~)Usn9$h zBw_lLE;M8Hdx-9+Nb*xAYMUDYJG{xh&TsU>nD59&XjB7}cpVD9`B?TaJfzi6K!ua= zYW-9~zo5H3y~<(tAqKmoTkU<7WM*k`;FXCtv0{epic0BH96sCDA{L7etO+Pq_6^YI zJiH0_?4bObrZdMj9HfQ|t;m&v4J!CAMzoe>&e|iPiSpaz+D2lx`to8*hVO9^LD$B# zoqlRSVpt|yA}1O5-*_)~Hs4vwF`k7~5RTZPYW?pS3`y_dwCAqG2yZy{u*A=?ieO~_ zY&tsL_jjgb=ca7m^_EE}!(-@nU7eiXk14E=q{+FFI@=DcS^CH9Z*)uNGQU)~-xEoy zaIF+(8dA?~A)^Es;}eoC6c&cGr~~tpHOLw)W`As=mXl1d*6#6~2ODop`Z?}U*Sk~s zbl!A zSj1Pw5BH`g#Um{1s?g`jkMFQz@5)@zlR9lgCB3)K&Xmf8GGW$2QrTq3lj zE*i=)7O_oHluxhK5HoO_x)b*s@J(@G!GP-QFRhynSG45i!sHq9N_tX$P=NSN6ObQ; z=VZGxb%-DknWdrbxVReGlQs(ZLW8N>05H-&scfg+DrwD*{z&(!V5Uk5Or!IbY(!ze zFixHyZ^w7|{Dd#1LHLw|5`W|v(!wYkaWv)V@jPWvVu$ckTKCVUL%6}eGbKMaWgO43 z`R6KQh@O8pO)%(mM4d}j)MkX)*zf859jZx=s3N*nZ_0J`OMNx1v@Hdx7u-N^p;-5d z<*D+ZBqLnVbaPl}=&=C}i3W`}CUsqj_ArBDUr)Fg^5ZK+Epf-m8*pLeq_wib^&u_% zN>Qa3EBU*mb*w;HoRq^?aJJcXbLAG+6^SWV0gHd}+Y#Ys5^~+B$!SddsMISNrV?eqfQd1$8BJY7o`EZy$*m-p;M|8qd1Knmu^^ zg-(FlTS?W*TR(cKvoyY;L`Dz`{H%ew8hPI`y%F!A#bgjrh?U=AI%+Br*u@a4YHanVBv6MRpX$MzLPZa3~&~Putl)R zzKz{RXRYfKn&4S%0?q9G@3>F(T)*@ng?5h?ED#0BXWbPp7uiI+ngM=nRLGfGqMh#> z`}DZa+ANr@DSF_=HO|v-3au{A6F1YqjtZcf>9wklQk7*~{Ku!^9FG?O8h!pMjNSe7Kb*(m z0S|7`T#B`lmN5sdmQd@<^x(fF8#j!9ABAz(!!Ts^i_&?B-I3VagYQ1jH)OpZ1c$!Q z)ny(TX6I~9!>L&uzM<(-Cjt7SXtE4 ze82kZ@7%F&JVxmM*_2Cd;_pn)KV%X-`i|+DtzEU;&%xpu4Qac*!HnZaeGcEN%NX%G zQQ?jJDK`dK;9pjkRdU&SOwa7?S#&rYf`!bXjuf~^U5P4B7Gnz?t1%HaLv)8sWa z>g-xFF5mNkKA$dk5I<>Qh|=canDSoWa~VFK(#Vz+38C0@lYgB3$KT{Y-g)lrU9! zk~VGz(Py!ZP#pJSnUs$N`{z+qqSA97M^QFdtJ@y7*^AI`=5_$*J|8X6JRuyWF#Xvy z=jx}wGo?B=m4I`BoFiQW-3%9=b4o2GuURmQPSDg{4ZzppOroSw3MT zXtm+tmoXzL>P^D39HVmM1Ws=TA>eEj?*x89Q2q4zmP-0j+oj=l0H znuTrqrWOU!Zv5dZOD{^5`m3k9OALKaqbs(Kxh3O?piS^Yt6QymzA480^g*?Ak#L9L z;C^OWGZPFULhIP~I_ID)HZioni|VsVnYO%tc^J58I-_gpMfqHGyi#F>le?DU;nx6u zmme!NjRU57C5O_+t!*_UUMi^aQXqUu{f50vHpP)!mtL6u55hGjBJj_q)Vs5PXG(o;x{%0J z2IBWnWax#yqPc3v)SZ}wdc^f9E7$vZT|2hgK*rha1^67~OWZO!hE%beYe%?322t4|8njk^zk=y|dRcq-MzmI=> zyeGC2WJ+Vz!YS@Mc`1hTLETyseYFk1nzN4vWZcBry0fsoP(tg(n#~-VbmqjyY<82j zs@-BZQ8{4}Ev&%-RN}?9W0omwVh&@Dd~J8Fl1=ZV1TIv>pG{`T98Hs{(RKjs5jpMl zR&Zx#>kkHVFu?=&7fM1RB|S!f^dY7JjQ-MYp9qoZR(FT`v+rDb^Lo(=DqnIQPgR;< zx1D*ggoiGxDhPLV9!@}B#sq_A zc+4{KiaZ(}@7O0o-RF`*t&lGw#U5nrYBE$)5Npr{7`pt6xeq7G(1`uve0mBntZBex eCV_`iRV&G;Sl-KqtC`>?piRE=f^zM*>Hh#lmz)m( literal 0 HcmV?d00001 diff --git a/ci/test/inl_ipsec_perf/pcap/enc_aes-gcm_msns_64.pcap b/ci/test/inl_ipsec_perf/pcap/enc_aes-gcm_msns_64.pcap new file mode 100644 index 0000000000000000000000000000000000000000..c72971114476a99124a39e4c719c0817682979cb GIT binary patch literal 1456 zcmd<$<>jhjU|{gI(UxKa(*L1=nL(VvCo`|KLcvte&{WSr*V0foJvA@2C^MOXg~6J^ zH?=4|H94_BK_fh~D782vu_#+p!C22o&sagjJ+nli48kr*OwLYBPfbxsEQZMH8fpS< zVPF8+39?TFO8@^4VY31GOMn;zu1c{nykYgPZ341ESQre1z<`6nm4TrIWQ2p!cjf~t z7@2?=WFQ#pFy?>tP8Nz^<8+~$W8c~0CzGFREZqJf**oyr#T(y5?zP&P3eRZ2BAs{J zJwfE~#V0#k5*sG9KNUW>y+~dEl}=*RD|@leOVc;q_;<{A`n=bz%+J;xjZ2c3gt!ge zjh8P4k>*AwkUR5r^L)Q5xf;pIiA;1tt-pE?!Skz{jY#DDGQuH|E+r767*0W=G zH1Iu5|+QQhAua>i@mdBChRZvDf43!tB$#z||gySW^Ec*N(CTNcaJ&8~A6-tM!< z?#7t%DAL@>0&-*K-;3$z{l6@qm(23M^hbmr(}w1p$|G<7tb8j||Np`Ey=vd!vuZdC~6H+X2aodKyPNu;?El%N@3uXecA{IHgP+t;uYnLN=;19u0i9(Z}`-o&hD?_Y{~EBgp`iZ1NA zCwSJbZQZiBeuc;T`S%}`jlW=|IA{ICuDKV2c5LEYb3ZdsqW11VhIa-kH80A-GIlEG zVs|6cyHwKL2u#-uj1C=>SPa;A9aWbTnO>2#cXs|)y}xs$r}(uTzMfHJ-R!{<(Yijm zPWy5~n%dQ)C%#k&8O$t?x7d{B5_NO~f9WD-J-eR57Z)tWRtGx>H?8{m)XyPmVN;7? z6?Qj9ZObIhjT|61+U>oc5x6P9m1$A=KNiOgt-tNHiZ`zJcX^|Fdxw~8%E=w(tC_{! zeCD1yrYH1l@~2lT-d>ve;N_x3)is~{txcZaJ^zifvB}x>e?#Eh($F; +aes-gcm +64: 5101840 +380: 5120769 +1410: 4194633 + diff --git a/ci/test/inl_ipsec_perf/ref_numbers/cn10k/rclk2500_sclk1000.103xx.ip.outb b/ci/test/inl_ipsec_perf/ref_numbers/cn10k/rclk2500_sclk1000.103xx.ip.outb new file mode 100644 index 0000000000..f26e338455 --- /dev/null +++ b/ci/test/inl_ipsec_perf/ref_numbers/cn10k/rclk2500_sclk1000.103xx.ip.outb @@ -0,0 +1,10 @@ +aes-cbc_sha1-hmac +64: 5186596 +380: 5165938 +1410: 4161094 + +aes-gcm +64: 5164575 +380: 5165459 +1410: 4194624 + diff --git a/ci/test/inl_ipsec_perf/ref_numbers/cn10k/rclk2500_sclk1000.106xx.ip.inb b/ci/test/inl_ipsec_perf/ref_numbers/cn10k/rclk2500_sclk1000.106xx.ip.inb new file mode 100644 index 0000000000..2b6811e397 --- /dev/null +++ b/ci/test/inl_ipsec_perf/ref_numbers/cn10k/rclk2500_sclk1000.106xx.ip.inb @@ -0,0 +1,10 @@ +aes-cbc_sha1-hmac +64: 5102561 +380: 5121262 +1410: 4161023 + +aes-gcm +64: 5101840 +380: 5120769 +1410: 4194633 + diff --git a/ci/test/inl_ipsec_perf/ref_numbers/cn10k/rclk2500_sclk1000.106xx.ip.outb b/ci/test/inl_ipsec_perf/ref_numbers/cn10k/rclk2500_sclk1000.106xx.ip.outb new file mode 100644 index 0000000000..7dd8275b95 --- /dev/null +++ b/ci/test/inl_ipsec_perf/ref_numbers/cn10k/rclk2500_sclk1000.106xx.ip.outb @@ -0,0 +1,10 @@ +aes-cbc_sha1-hmac +64: 7257102 +380: 7238439 +1410: 7256166 + +aes-gcm +64: 7253390 +380: 7234528 +1410: 7260576 + diff --git a/ci/test/inl_ipsec_perf/ref_numbers/cn10k/rclk2500_sclk1100.106xx.ip.inb b/ci/test/inl_ipsec_perf/ref_numbers/cn10k/rclk2500_sclk1100.106xx.ip.inb new file mode 100644 index 0000000000..2b6811e397 --- /dev/null +++ b/ci/test/inl_ipsec_perf/ref_numbers/cn10k/rclk2500_sclk1100.106xx.ip.inb @@ -0,0 +1,10 @@ +aes-cbc_sha1-hmac +64: 5102561 +380: 5121262 +1410: 4161023 + +aes-gcm +64: 5101840 +380: 5120769 +1410: 4194633 + diff --git a/ci/test/inl_ipsec_perf/ref_numbers/cn10k/rclk2500_sclk1100.106xx.ip.outb b/ci/test/inl_ipsec_perf/ref_numbers/cn10k/rclk2500_sclk1100.106xx.ip.outb new file mode 100644 index 0000000000..7dd8275b95 --- /dev/null +++ b/ci/test/inl_ipsec_perf/ref_numbers/cn10k/rclk2500_sclk1100.106xx.ip.outb @@ -0,0 +1,10 @@ +aes-cbc_sha1-hmac +64: 7257102 +380: 7238439 +1410: 7256166 + +aes-gcm +64: 7253390 +380: 7234528 +1410: 7260576 + diff --git a/ci/test/test.list b/ci/test/test.list index a434ec0a0c..d2491f2881 100644 --- a/ci/test/test.list +++ b/ci/test/test.list @@ -1,3 +1,4 @@ l3fwd# inl_ipsec# l3fwd_perf# +inl_ipsec_perf# From eff0e59ed327c9c6a98595a441eb500014474f6d Mon Sep 17 00:00:00 2001 From: Monendra Singh Kushwaha Date: Fri, 4 Jul 2025 10:43:13 +0000 Subject: [PATCH 247/271] ci: add inline inbound ipsec perf test case Type: feature JIRA: ://essjira.marvell.com/browse/IPBUSW-71966 Signed-off-by: Monendra Singh Kushwaha Change-Id: I2c01e619b948ad9941f1f968a885242920bc6638 Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/157237 Tested-by: Nithin Kumar Dabilpuram Reviewed-by: Nithin Kumar Dabilpuram (cherry picked from commit 5766a016b77d27c827e37b9ab2d17908033a1424) Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/159002 Tested-by: sa_ip-sw-jenkins --- ci/test/inl_ipsec_perf/aes_gcm_ib.cfg | 8 ++-- ci/test/inl_ipsec_perf/inl_ipsec_perf.exec | 2 +- ci/test/inl_ipsec_perf/inl_ipsec_perf.sh | 35 ++++++++++-------- .../inl_ipsec_perf/pcap/enc_aes-gcm_1410.pcap | Bin 1506 -> 1506 bytes .../inl_ipsec_perf/pcap/enc_aes-gcm_380.pcap | Bin 474 -> 474 bytes .../inl_ipsec_perf/pcap/enc_aes-gcm_64.pcap | Bin 158 -> 158 bytes .../pcap/enc_aes-gcm_msns_1410.pcap | Bin 12080 -> 0 bytes .../pcap/enc_aes-gcm_msns_380.pcap | Bin 3888 -> 0 bytes .../pcap/enc_aes-gcm_msns_64.pcap | Bin 1456 -> 0 bytes .../cn10k/rclk2500_sclk1000.106xx.ip.inb | 12 +++--- .../cn10k/rclk2500_sclk1100.106xx.ip.inb | 12 +++--- 11 files changed, 37 insertions(+), 32 deletions(-) delete mode 100644 ci/test/inl_ipsec_perf/pcap/enc_aes-gcm_msns_1410.pcap delete mode 100644 ci/test/inl_ipsec_perf/pcap/enc_aes-gcm_msns_380.pcap delete mode 100644 ci/test/inl_ipsec_perf/pcap/enc_aes-gcm_msns_64.pcap diff --git a/ci/test/inl_ipsec_perf/aes_gcm_ib.cfg b/ci/test/inl_ipsec_perf/aes_gcm_ib.cfg index c8f13bbdcc..97e15dd626 100644 --- a/ci/test/inl_ipsec_perf/aes_gcm_ib.cfg +++ b/ci/test/inl_ipsec_perf/aes_gcm_ib.cfg @@ -1,5 +1,5 @@ set int ip address eth0 12.168.101.1/24 -set int mac address eth0 00:01:02:03:04:01 +set int mac address eth0 00:16:3e:22:a1:d9 set int state eth0 up set int ip address eth1 13.168.1.1/24 @@ -7,11 +7,11 @@ set int state eth1 up set ipsec async mode on -ipsec sa add 1 spi 1 esp crypto-key a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0 crypto-alg aes-gcm-128 tunnel src 1.1.1.1 dst 1.1.1.2 inbound -ipsec sa add 11 spi 101 esp crypto-key a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0 crypto-alg aes-gcm-128 tunnel src 1.1.1.2 dst 1.1.1.1 +ipsec sa add 2 spi 2 esp crypto-key a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0 crypto-alg aes-gcm-128 tunnel src 1.1.2.1 dst 1.1.2.2 inbound +ipsec sa add 11 spi 101 esp crypto-key a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0 crypto-alg aes-gcm-128 tunnel src 1.1.2.2 dst 1.1.2.1 ipsec itf create -ipsec tunnel protect sa-out 11 sa-in 1 ipsec0 +ipsec tunnel protect sa-out 11 sa-in 2 ipsec0 set int state ipsec0 up ip route add 192.168.2.2/24 via eth0 diff --git a/ci/test/inl_ipsec_perf/inl_ipsec_perf.exec b/ci/test/inl_ipsec_perf/inl_ipsec_perf.exec index e9ae479c82..3955b7315e 100644 --- a/ci/test/inl_ipsec_perf/inl_ipsec_perf.exec +++ b/ci/test/inl_ipsec_perf/inl_ipsec_perf.exec @@ -1,4 +1,5 @@ set int ip address eth0 12.168.101.1/24 +set int mac address eth0 00:01:02:03:04:01 set int state eth0 up set int ip address eth1 13.168.1.1/24 @@ -17,5 +18,4 @@ ip route add 192.168.1.2/24 via ipsec0 set ip neighbor eth0 1.1.1.1 00:16:3e:7e:94:9a ip route add 1.1.1.0/24 via eth0 -set int promiscuous on eth0 diff --git a/ci/test/inl_ipsec_perf/inl_ipsec_perf.sh b/ci/test/inl_ipsec_perf/inl_ipsec_perf.sh index 7ede6e8a32..7fe63f3e3a 100755 --- a/ci/test/inl_ipsec_perf/inl_ipsec_perf.sh +++ b/ci/test/inl_ipsec_perf/inl_ipsec_perf.sh @@ -801,24 +801,29 @@ quit_testpmd "$TPMD_RX_PREFIX" vpp_log inl_ipsec_perf vpp_stats_all inl_ipsec_perf vpp_cleanup inl_ipsec_perf + # echo "" # Inbound -#X=1 -#Y=0 -#run_vpp_ipsec_inb -#pmd_rx_launch -#aes_cbc_sha1_hmac_inb -#quit_testpmd "$TPMD_RX_PREFIX" -# -#sleep $WS - -#echo "" -#X=2 -#Y=1 -#pmd_rx_launch -#aes_gcm_inb -#quit_testpmd "$TPMD_RX_PREFIX" +X=1 +Y=0 +run_vpp_ipsec_inb +pmd_rx_launch +aes_cbc_sha1_hmac_inb +quit_testpmd "$TPMD_RX_PREFIX" + +sleep $WS + +echo "" +X=2 +Y=1 +vpp_log inl_ipsec_perf +vpp_stats_all inl_ipsec_perf +vpp_cleanup inl_ipsec_perf +run_vpp_ipsec_inb +pmd_rx_launch +aes_gcm_inb +quit_testpmd "$TPMD_RX_PREFIX" vpp_log inl_ipsec_perf vpp_stats_all inl_ipsec_perf vpp_cleanup inl_ipsec_perf diff --git a/ci/test/inl_ipsec_perf/pcap/enc_aes-gcm_1410.pcap b/ci/test/inl_ipsec_perf/pcap/enc_aes-gcm_1410.pcap index 23c7e795e656dc3f088f0a5df1d8b61673d5e03e..0c480a4c66d0ab5f71e59049a78d2a5e6ebddb68 100644 GIT binary patch literal 1506 zcmV<81s(d-!?K|Q00aO40000000030{{R31000020000L*bV@?1polL1poj57Cs`O z*#H(kBB9v`07U==tN;Q406;Qt8UXXdQTmf~ zQ)Pb8wXV;Q)P&wt2p$>@MA7uu63JSBDIm3#0Ui01PPeIv3lPxrdUVX$iC@YaB82MhzLc3HCq2Re;ryw3m!amc zfrVt4Mg9mS;&~jUkC^5O1RJxuBPc!>TtxAeI;{^X;)wg*Rv3#M80v;|*4;c)V_#4r z;Fj5zLJ;)|^weu0#jrJ4Hu8E_OJ5k)!{I3qODF!Y`n8?mnq_gARU{!XRVIk)WCmRu z<6W;wrAm7u8nXqDqMCkPl(tCAuCvnDA(W3~i_;8><)L5c6d%3pbIk%VcgK&Dm&(vo zTEA4z9v@!=Iq>(r2YxH|NagidEpU;#SPgp;d5VK* zBTS(_Hdt_H9YSU%TCHrwsSSnO2-2iJk8#g(i51(*(P)w*&tGTR*qq!=3n6G3GPHE+ z05t=^xBP0kT4p^?%!wQ%wC`i~Ef z<1g}oZo8g@;xKj|g+B_x^uN3oZ7|U>#MK~-SqC)7&L^R;l^wbowq-)b!PoK(l!v*y z)Kv%<5hIHOHSMientsN}w^R;VHp$woC|so03%PjWK~F&PUI=C!hyIpmk2L;1Np&hazVM6U_o`_ zW!v7`y{JX)PU9CHz|#)2*^RLc7e^!a@o92?42smvS1Gr3G>@#)7*ua2G-MVn91wjX z)aO6a_g<^ppWF`i8mNdVc$YSdPLXWuFnSOCpbN`f4T}M3gh@W4tZWu>hZ!4HEuSMX z^`h60rG>-N)17sjNR389L{#vWZ!G^;I>oFpRwZhovYcY=P@4js(k~Qp3!TgtFW#6tlTtQUXOClR_A3iRJ-(#I4?t+El`=q4b(1Ft{mp2H zdYT?9ik-X#!X+v45MdDh$%aelKLcRLP#Q~w;!}+WP-hLN-k=#nDucgTPD9~&4q5e* z8tyk!d(6&{liA(FC`#rkwiBfyp^<%XN|UBi=?p^)lF=90kiUva%IH;zi%(B|4$8D^ z{w>BUu@%n*nmqnZp7(oFQQXh*LS$!8avP9v$Z7Q+>ncP^4QnSUoY#y{amq!kK$m*G IHB%gkg>5&s$p8QV literal 1506 zcmV<81s(d-!?K|Q00aO4000000000000aO50000X0R{lN1polL1poj57Cs`O z*@`#&_Q?te07U==tN;K206;Qt9036W0RaI50ssI20s)%lIsgCw00EljI(>106Y>JF z98xD0gB2dMfc1dA!LS(>%zhV^c+t*vhB|0HuI#>tgp8T}AEv9>&Y zPgA8GHraLU^6)s$D9+nl_I%ic`Q7V@WKl4JkhtvBCFpP;2NH$2Uj1vP z7A(SrS`~aMINRa7Wgar-T{M)e{LdE{}QZ`Fpgf zzVr-lL2NEgxRJ4)gt?v;blaN!r1)<33sGys-hGC)P_xb&ap`;A22FMa zg+^bT)kKOzF1#sQU^~8YTTcDkesBIx*1#d)9I_Ffwxn~ovU5TClapz>huO(m$+3v3 zMNghF0U|QwJ{~QK;vRB_j#^}H8Wnn?MIMQtD8N+6HLrP8bkNBH+HGX{#X8@-o*@MV ze8G(BF!#Lc6GyQ2C@e?2+m&UaeiLKL_AO6#aY&b&2T**nSO%|W*q6h9)i*SrKPy`1 zzDdRH797Q!Z##y<;evJ~x!RZ1T@Tw=>ppzAtDw1c%F#qgsKFjJ+2~;+k($?0)Xv7S{S6Cpi|pwKO01Y8bpT4Sij*n<$QlBrkO2u{l)Be-!8@PrI1+~-Ljngw zGh8koOYQq8N5R)RVhXK-d4Ve^BQtRV!!X*gE0fT)K^&^}R*<_@H8GYTpuM3l8|{qb z3pJOn*B&3}06$D7VVcti`ZGZi8i85cHDLWG8-yh>+!|~Z)Ovm)Sn?bXFqbZ!^axst zAWl4r_%5SlO#BKh*kSr{jQYw!p}t%E1u97_18mR!1#bc~>ggAws*Uh7*hPWm-odo% zG~VeRl*F`UAR(0}@rO9O_kU`6Kx_7+?Pjy3)ePH|El_3-_$(Cbb6uRU2g3!XI?A0h zEXaEO&~+D=F#A)pw@gCY*hED4gvbPYao#fVMfN!GI8v|7`ZF;VZLWr`*+}x(w>v1a zq!4kg864d>l`^XkzZMJzY+VWd43nM(#Lyt*b>EjQ3t@=JnUu}TD0L}8n-Oj3fyrMR z&2yvTPe=l>ykDToI|#@dJrb$K+E-NnmX-8tj_?F|AAYivz35(B*dp;Nm|})b#2;3} zJHxSqnOW$Ytt+Y@WX3W_3^&+$la89^=s*< zRkBbaJPfN_yCg!GTBD=ZI+MfM`kEgJ7@bLZ7bTQ6$7)0}wkP>4Z6V{0`=>NG#2C@h zrKZ+a9p13yij|+zXzx3Bqt+aR zvhJyj7=I09{#uC&1SR&CT66F-MJ;DLpqvwv{U~Ud7Qe$6AyRN-jKEtW!s$2zeU>4~IjZC+UjsmV5ASUnO}l*}F-MaOLN1=RK6%6__fXmD{yc26wcV5I&=%vjou zyha1ThEd_7O^h8=dhD|k>LhDyDcSn`?xttx4|3cPh`HX_6vFQalX?b#Sgm{+Ox}zD zZC7kJAnt(;kyFcprOE#bLawv`*m8K2sKWg!&=g#L$y>L!<2D~IK^|la`b)9dRQ1)| IX~zX}d{ZXI#{d8T diff --git a/ci/test/inl_ipsec_perf/pcap/enc_aes-gcm_380.pcap b/ci/test/inl_ipsec_perf/pcap/enc_aes-gcm_380.pcap index 62c4f1e40157c912243d4e19447e9e56a9c03a2a..df5212600e8feb58f2c57d0894707ceed47a377c 100644 GIT binary patch literal 474 zcmca|c+)~A1{MYw`2U}Qff2|7iP-hBGHe2JK$t&-T4VuTJD8#$Vu_KTs^mG#mT68O@*yf8D)P? z_S?R)lUv}z=c|cN z+{N?s=l6`a`?!SE6aHU+^P7pIz}!{F#dth)WbSiN==f|pLueK^Vi)K zUdU8*Pig7b8wVEuchp(O^^NN_1DLoT=C`6dP`#)$8Q${IwVAbl(X6`I zDcd|wuiJd-y5f}nljpd8aOpLPAeH>vZcR$H>(lt0_Vp{?U)>GrGpq`lm)e+NUX-1U0i zv;EJgi%%)H-4q>nX!7a4NJm>Gt^!Ulwhi7uSU(4B@2+>~OEwuUF#%QZ7 z_G$A4y)0(Td7C_i^xzRR|Kx4Gv$54;ty znJ1>abIy-Oro1NpNc2)$i-hPq@-@FN8x*ir3{zm-yY2QT3V%TcA``$p$bgjtDA&gEQ#Tt|7HKY&+X@qXD^tkKwr#t*OMlt6ZQEV8ZQHhOn|H0V@6(NQANGhCGc#jGG=h=rb+o`r}~!rY0-gtg{GZfsZKW6XsUaUq*Wj8=nRt4WT?g3Wp9SJl$=T^^VM<%VT+q2! zRF`j?#^iWGX+COcdDkMOrIvNbVDs-O_V&DSrLT}+MstS)J!gH%`innf@suj8ve{#( zpdN@CY%_`>2nQXEe!v*G?h&>yRvJclSaQH7@OLC+BPSIV&+{bs%BsaK^H!_C5dGZ_SC*$I*>zcPslJ1h<)cfj!;5Si#XsidM@K|M4Y0ivqyI=W( zuB{-dL@2Xd6HBwtU!#1i2P=NBD&>sxZ$}vGzIKbcLiJtha>q>abzWnYvS1(yq$}da?dQ})H@)ZIdfFQ2lc*F=MLQiT}sk7_AXq6J^ z@6wiNlB92gNx}2Epq$v$dNhzqm9$#t)ab`UHJlF{tPwE${&)T^19MFxJS9=C9aPx@gpGQ^~x*BbF| zNg3@A&Z>K$bGYnYRi#q=L(_8$@hQY#>3+8w!os1F4{}9Z-Q(pwz|G5_>D}5lXs&9n zF5Dt^Q?%KVu{anApI&!_D!w9eAeL6>vFH7DVt+vU>$%;;1h8UVgFJ~owf-1hq0YRk z-t(AAN6*3b2M@MDlx0(BXs9+EO)4S8E}kU@yt{FtjbU2>FMg%XC@b>lR-L$;rY$)q z{E87F-atKSs)5A{IFDLM3#Z8cYFu+@k2a2U>QudSi1( zAxbJ~s_3aX%hO4#hUYThN@6y0k+HUz{nBu=@Bt>ku~xrdSj(UdKm*trKB4Zw*Ofjv zP8$kUnxGrFb!3Ugy5RpoP7nS_)^x#gbGG4@M$bf4-@6XI6+na0le@F&VkE$?^4P(; zIN4JU4xGw1n*xc_**(4OUYJHtL$Lzf#$#HuFhNtrjqP-Gl?bh0v7wR>MZv!Aes>-Q;o#OP=>Mx-xU;jgK9 zV%He;2gQ@}m#`rwEBfB~C3$DWOG-=*c~rAxh3a1vl!FUgDp6k%gN+isAJQtvXw}9@ zqi9>XEXj50KZ67E)?{;1$Kxi>HOL({xprIw*Esg3>kQrSlngYY0FoPR(AlFb?+sXW zmhm!|3lgG}#dg^*vug)C@E%~5YKc-B{@?5zHW#=P(K&z=HsTH3Pf%Br-O5DXntxrz#4~BZa7?L7In#+~;^^fs!UM9bNdeI zP(2_UPI@ss33eT@p#rG)CX@{+qfZvYmY2)q5ZnQ#Ja83w@~gzy3saRS1WSyHhxSwv z1{^t$OiK{P!lE(B5z3|3<$?NGw$zEg%V}4kjK0#*J1;~@Y;Q6dKWJ%`3KKCqN3PR_oGGuXXmlL-%xfn=^}qJr#Vup0N$1sojTT43^k#V}wMO>B zL;Ahdd{vjlx?GRG&{XL}tg#LkuBa7im-4dseI%jiFO9s5-Jr*yTYel zQ%8lMPL;yg6z(a zf0da;1>~PX*g0`F<|u9jINL5EQdXB$`aYxCsw`&1Pd#^3;Re<15&_+Kxoj%LKF^yl z86CYG+J=}*fXz>jHIX>re}p5vj9&-WBVcsF&sKfTP+Cvpb_`|1tO`qdDU_pi#+oX zeTjUbRUbwoi|uabh^iE@PwB9QJ@ny2Tin~(A-8J2yA$f>ZfdL>5FzmlIo?Azwk3a) zU3midhd>4Wk+43r0ON4xwKW*uVjFTp<>o2+cL-hZ5ye%OIFJzSyV#!}Z>WEiE;YR> zho+IA)Jd^Oc?&98Z38N0@c40VtL(3&92OG^-OlAy=^x;Je5j8unR|$pvjW$PQxoLH zX_R)=Ww?Kl%}fVAWw`~V#EO_Y^2!sG6*H*N?lU1KurK~Rx!?gN$s>By*1czKWvwp} zm5P-cTO^?-<%@c!6!d(n#Tz;vJWDoPjwxv&IIB_oN zP?O@x1X~|`LXukmY+OEt4P&KWW>^@SEXK6nGk=$VJb~hI9;rc`W9m^6;kl6DXBvA#)mk$CmiYET0AeKp)u^wF*rC`4s4fz93_BGt4#Q zu+6V0k>x>|GZ$B}ofs*gqB;<@OXDK`+{{S-TB1B5?R_+T0&zcQt>jyyc)r!yQ|*@s z==l0m_l}cr8Sa!nY|m@AWf_a?G<3IeJ5+iys;(BMjtBYl+qVA(!+j=8n%g=ZkO9je zN21?7Mu@$fJmQs0e4CX!q@fdgf@&}8X&iW0=WutJgJ{QcV z7ic^x&jv$QI=j{~@LpJzE?|{2c3t=bvZuV9lQ7U!{ecRB?FU^WCt898Wyc`eSA-7S z00<(5p+pFJ=Uhm{jyNhx&;ANnJ=Z3kf_m{XQHMH>A3zVfc_(7Wmx{X8IZPJ)J{k>c zivHhCr@mbN?@R~%*Yx3q5OmnnggvoOk(J-e8%|BjGS0y?FA-sKglg*E`51P>EAT;J zK};&*>ersqA#kdVSV9&f{2^$$KzWHF*~tXaUst&DF`(K%BX{TPAs|hCA5>2bze2a+ zz1WATe?ON9Jm~m!@+D{Pw{*F2MbatZ1lE;?I!_~&eY8cHH^z4{0z>Dh;h7|=p0w)@ ze#go%{h`3NGq}eLlw0(VMBT4mx?ysap|&WUbpMq-)2cdS0uq#GY{*qxzoE#HHJI%f z1j_3ZaYnIt4XD@m0si_7Jr zx-TOQqiPA(+o6C`x&H{d!LZk$K=9s_GtE3WqzTRjQiIK=!;G4uAwAT(_}qBUmi>tO z$S*8)FUwvd2ZK9MIE5K}>Uec%#i83&VFC9Dt3QZ7kx9)Z=tCos&!+eBev%X;Z zaBs`U%V%JeE%{4-a)>4|kBC_offvNOmWfd6u(hA=~>5KI0 zie#ZLnCnl!VXY&kz04L~FtAr+KROsF?bt#ndf^6=$9`#pi8bd(NDfX{-_~d847ri0 za!h`a6w(?47q&HF`GA&?96@faNE;YlZn9y6{TZk|1qzyaqzpwtFAXX7tcL3f${fIZ zEb`HT#zQ3;=%4kPA%uRjRVODlK`8ec?h&*uEzav-@=+&ds_|~olMapN96dC$a@V;X zbL{rq(g;OBzwrEbv@>*PeBr9}sYqmKeBLV;$Hl4cfPV({8c^)YCNx_{i_y{RhP}Ay zn3u2Xs~Zwq=H@;>GQ&oQLTU3gKVh{nt!oL!WB0DH^GKeeIZ2-p5&6A63dCS9r6R0M zj6JlaM8y89Ny%3ou&OI2QxH`$8YUy6T##-~t!R~BnJ8(#UJxKIseM=33x2rkH>og> z5kPctfd@85(z{tdlq&`!!;yK8x<88^vrxrq46Hj=?mF6OdTkWs?`oXw1GGlvO0J1@ zpFX`j(8Kq03bMH&GA4HjoYw-)b-^IPx%Kqz1!rSN%6tuDXR%%5K^`qRz{tFBS7qsU zJv96RBPZ4K7vqc;i))1`vSOX7&->2+_GhnG`94W@G+tg{>XFyPafr`6MY?kAducj( z&|CN!d+BjYWaCq~hFlMaZ@$Q-lYzJ8KvP{|fN?o~>Y4Arnme6PK@sWe$}TB8Vuux- z(p?Ib4K1PAJk43fWaY>E~MGk6&->d6YyXX$SVxHGAF z+n48Dq0}8{LN?_+Ec8MJ4u|bdr&>9zjwC)H=%;njWr7Q*}2b={VaG7~LG9AZ*zwM}#C* zW7gz$**)zz55A;v@e=iM9u2n0+_=jtTox84tQETvfOj!^LTfZEIJ3jHA^VMYgs^tE zW}hGfvKPeBsz6wLk-BR?^6nap#-K}HSInsmPwOeYGfl#f&G#pDCFj<4=IER+#(}V9 zygZ*0TAGagsdCp?G~O;LoF$5qe9ICPPoVQd_{~xo7JW4=Rjf;I`J^3%em>-*AIstU zLfG1EE6f{eQ*Y&_W6w5D7z=!4{J<*>M{3;t7^kQMcV-4Bh!P7W-|Lj2kgby6{Z0kP zPuJ-Z_ix*+T4$@Eg`Pc9P0hpspO4UmL|EXWqKcWT9#M>e43)*~-YSG4HEFHf&?9S(I{XU z$670bA>M+SdK0K7d;?!q|Y@<}VfLBsPoY?X^rMCx&9~K|)@+0M90k>^Z9jHbP)T zL6Aw+)S_$6J!Q^|u9pzI&F;8v4iDJXan9t_?k!;P01;;or^l<+Nb-aq zytdRp${}>-pGVZVENdIr?-i>1lM#L&#F!wkPNl@6QUcH!-G@6>Lh#uLXy!(d5a0p0 zn6^uH?i8D-S~XU)gNCwUUZiD*4aLcfEsK7;1gkyc=C;cn;ag^TE#=Rm_7QG{M?QBt zMY)J@xd-FBif#&UjYdgjGQFgSSfq~`^W5i_U9v{4Wg>iMKIzr;w%4yDt{<(Rzp2s0 z9tXFuwb;e|~3sK$=zw5_bBzN?@^j1D{O!E~@DoKdV(MB}%`rc0)=|db&UOC*-zLGiLpX zG3p?4m>3OpSAD3Q{dLbZz`{LktRnnZd@|e9<^jz9L>ne}{`1}K+`fpo( zjkDFo=cI3~3cu8|iH33Z#J$wwRBQF95Ov6D$RJ({3wy%c(=PqBM_ciNqvxYeP_7Ub z;*WVRL*USXG+E$sriq(xpNWMw(EiSn2i}W5qM;Z}jD%4M2s?)+lpWDNR8t5^WE>4Gy9!gR^ zf~~xBySCNPCWDC%ZXiv*?z{IbX!*>rgR4vgK;!yezmCMv_?8HZ zo90OL)$Jf^m)akL+OXE7PA_+6&xSntj0r>8kP_D~8XBYtnNBVH!yg7qfpatqNHh)} zr|teBc3S<*rqIBvt;*WVNDvD@h}nHmEUuL-<)p8-XSMylR8V`H>n;)!^g9+N8-l1%krFx>GX9%77zbcz50Ojix>cD`dNdjUh@#i238m79Xjx zLky&kjrSa`e>-5WJEgBZGr{Odmy&!&rj)Eyo&bG9kRPV*R;AMF=!fiO1hY(!YrxT< z8_)J)>|~Wx_Ssnb8Cyil;Ute^*ZSR&Hwt$8a7;96P|P-)68V;?aq>cGGa&=#>n@H- z5sO{$@twqUIvlEd;_poDAc1f(nslIV3(lv9v^*Gy7Lq?gvWq9%CER02MkK;i^jzh- ze1jTKX?7A8!J0Y|tRHCh1jd)&TP#SW6bM6IUNZ1lAuW15_TrA8;w8U~Er%pB58b@j zh|b9(K!LFDQ^l8E`J8VplD5zaS*BOOHCL>4rJ<7YWhKTkhR#UXE3c(SO|ya+cVt(Y z(v|SC0vGl0Lz(4%bBGNCR*xZ_Q;T{*jG@pAo8crsRzo&tqCwo%3I0xgp)CiPpagbz zxep{Xz%1)R+&;X(KCXhL_4?O2E8&u_;nrLLnaQ9IzT`t?>c*@gc74UL@7#9cl4C61U{>NVF?2PV?8^5ppNl1cMAsrwm1(peM zn3MHP_d?KUd%|V_{gMv)zyO+K>gdNO)4msI|Cji3-YyJXg)96FBg;W7WESGZZ(F@r z;JAJ~9cN0f*Tt5H2PF#L_n|@1>;YSG(e|;;e50@e^!QlKN0DyOq4rK7(Z+dKxZOnU;-sMVPgXyL$h5DJ$b%eNTFbrp?vrCBVucflWUxo0E%-*aPMv^N@*hb>|Lo2FK0o273m* z%c~zfmuP?@H-@vx<_iTwmkAK#wh=Or$pqtd98)yng$nF-j(P@x^`ygC-F4E++^Oe+ zgzV$-bA4%D3YC1sruTARQ%bYc(BB;?4%zhHGxA28BuTh^s-lr4kp4Mw!WKJ z%s7pr-pw>=bl{}VxOe3z68i}Tg;{!g79Mvd?BT%Kj@!H7@jZ4skn!aRZJE`UkC8(p zz7eFKqM{wmqKOW+U&{(P(2&p7d~vlT#|IxXHh8L2gI^NF%Ig%X>xUxWd|avr0;#vQ z?4whJ!4NF=o4R3E?uEf`+?yB2*H;an*_TLW;!{q1*If@K5l33IDytut2;v<)^@omK ziMq-@6u=X5H=hs54DodAwt`O36r*yc$ifdi@KR~`q?JKA=TH>?G0S%}amG)YNjk?1 zN*!fO)XeP?Vg->{2}gbzFX9Yl)1#yL=V{JPB*dOgw@7=1%t>)iW}4iuP-5(XvowQX z=9+txh9w*qMEXSF`+jWWgME;EKwjo)Cob^?Y&ShPfKFEd;ey#848jwYgl0k#M+KfK zd=IXAl;_Iz9B*_ue??JxJf9hYBD{4ua~o3}Q5CDR*wZLLe`)mZkj5~RJo4s3wY993 zfhZg;TA=`rA=De*g_G*RZyr$FhfQ>GXrf;5j)%gDKf+^`4TOJ5=T)CvYZG{&dfz!= zFiHS&O-M4k@@^J~EWPlS9}8ot$K*(q-^qye-GD$`1G+3b+EP0}!UiXJxFk`3#}Bud z+nYKN(=L{VJ+t`aDZ9uQ#4`Td*jAhdo7y^;??KD)Ql}Wsu2m+;A6NR!E&4u`r0P%k z39Im=EPK%6Fnrh8z>Q=mIUc?PZ~h}H@9^q%`~Z1mriu7gEMPck6tU|q9t^y&AJ*wi_z{cOxE37j~Z52lq|BF;km@L z0FKo78ZudrUae%$7alG9AM(~E%DahO*|$~sW!}_w=Q3`1lPi^UIBr~gpWQ?iTQdXM zak%X-K0VqvHiPOSqc|^#a5q+ee3WNAzApRei&-MeSfZUVghuTeHe+@xfqFO?gWdc! z*Z}YivOHL}-k^Ps@GNPXUj}-abEm>WG2y>9I`~xRH>G2Ze{a=(eYD{U_dRNhs1B|= zIap;Iejwtt4j?AD`Z;EYO%6CwvRF9~O#<-Fzn=yN?>$e?$D7~Mwpj%ia!jI8iL)Il z#U?8*5MB)Od&6|WyXa=#W#R?~F@oKiuA0){fwUk3a4r)&~-A?@h zHf?yzgv#~S^d!MDFRsHf@za!CwjmLvV~i(ZjC16+vFksFC7Q$A>O_H*^DXZE&?$y* z-aB87aCK>26D!*8)c1g7jmhcPT&?J6lwowVcpjUn0lFsX%Q6bC&>itDomC|?Svn(w zMFvFt!{cK=)(J%covgb3*XvuU3ve0{S+&4PNvSZ9-M^GpCvM!jS5bz~1H$x#c=*r{ z9{>9}{axhvzcU^3U(<^{x+)wt_E5Cc`O3&AX6CcZo&GRknJ{f2^Vug+?f6}_MD0Ti z1r$d%S9ZY&Ar#wltGx(Q!zM{OQn%PfT9uE~3tnZ}g&mIft>g*W2YH&M{SrPGVI-S= z&eHxH^NY$EJC!HEw>L$m@uI5vbN93_wpb@HPYQHos$A-4{*fknAV~OC)xHVk9m8}5 zshQIipByBYaPR6DUntZPR?WrQPixr&=M)V-*9;6S2u`@z8Z++rD>oYdg-2tPTRSdf zJJ%S_EQaG5bY%BcBj`L7Mq}lMUNWO@Qt}Q0Klp{B#d0c>??;?pw zxa=!-JSg_)AzaYK`z)ZKW@tK=63gTz`>ywEQp}T8ZpUvx!T=}ps`vLfJPl*yUVY(Z zlF0pmb^8(G!?n`-T7c7#h0mcInwsUip%!6H1o-n#Txukt#+|=VD zN*M}mSkCZ=xd4LVf#bvZuu|Kf!YR+V&Ub`HJ&mC>&KB$@QEWo#EHG!s z1dB_I4nnX^!u+tfEa-jjG-)nif%pF3-S>(%tsCRs*SN?CYXkh?k2AX{e@@U2q8u?h zZh|$FZ!RG>7LsqR7)HGqs(?Yz%&ZG}+8O1{ybLNnVD{iF`2u^cJiTZon#fr@zr!)r zXtqV)V)AeRq}UiHFaUi$UIDBh=*0|pDyylOA_%oF&TNS#ZLLrHq`YUItZ7W^y7Z15 zi?{>mk*(G+8Ua3g-JH13&Xqo^@4CxC7*VFPOxQ0U{IgUDe&_T*CeGMc=j*1_6Y31Pd&I5p7J<6b7B4-KJ>^Bb8oVM?E5(8j!tm@6mh9 z`jxeJ_j3EXSO>egokFnEwg=s5XItE`o(GQMfSYTpt1}Cw8QGSSx!L>TM%BCAPIF$# zE2M#b;>3DEn7=pB?9~0Tk!c4c5&3d;-f@%EAIK0bo|p=OIoOB$S?%!C2kM<>%pRK5 z?*0)UqeU%ru=;A6x)^=9VOMlqfnFu-ZWztCYMDMO*UV!G>#v^ib42UI-Z#G*FZv#i zFQ}fUg?QJNl+Qo}AK7bEJ@Yh(0@){|Bme|UW$1fI!~&C`Xltgy9nV(i{K0wQDA?US zdmWKVtj6GMdAn^A3<)EG)>|U++1=Uo06)P0ZhA1}`G03R)W4<&wxKe)5mSlbYor~@ z8R@DIWJZrvzG)~nj@(Du`22v7qgM<*olP1xPmn)EdEpYNM0J0fG90x1J@d5+C1)1j zIrBLiRVSr>5s&T@5$;Bbwg^NfnIfUHh&<}E9Ttw!sZ%Tw={dY3^1MrIN$?^SB%ILTxf)1;;p}l zoz*UPfn4QSxoUU+c{+ToZv+wC9YvW7jR!UZS0cHR0_Vr%OCUT@gsP=I9)W!wYDO`M z@NAVTfgvae{6`n}@cBtc>J1MB2k#XoQQFVeQS|CJsBlKWMq4-iM|`EPugd@> z1NlA_Yo`C-2cqqui4wW%NY`P->aT7~u*Rc%)IC&^g_->l+Z}29FPe#6Lo_SvhQwgz z(rfVEr&eb6&LSN)r@z*$AT}BAP$uBjB5;$q@_&6ZT%{}Mke@s_H_ZG7;|j13K9ovn z&uTJB%=x>5^E0_@mh{N1{eoy+au?zvH(NMV+pH!s;?lV+@f!lsJ{`EIHuP^=azE(= z%KMU(l=8M+c!(_~x;TOff3ll&T&bb4r?T8H40**bRD&Dkvt$NneD2F)e|%Bv>@=ft z99C`zbOu1sG-(X)bNhF4c{GoF+f(zBt|8ELD(l2hxH&%95x(Gpj2y+2XMM_vJXLyY zz|h<_bHq4qv#wX~=TB}!lA>C6waEA8t@V7%OQah029u%!kW~m>+mMeN_cC#RUKUNW zby~lv78BC`);||}$uU2Xn-%~VaSI9GT&}wKl}@Iy_))8U$m^2~fHR9tjGS=|Wm~V% z3BGY4_1I>g{ObR9rh9%D&dbgqM$W!I$5U~kHP{GTlddGAYiD`NbysZL>(q8U-0aZ4 ze4nXc9#r2%|6r^^$b^SGgGmNNxkZP#4Bk8(pIFW7)mcERk#&Eu-F0vtko)eJto zLO1QpZ=Z&2Bm<)wE%7TM#i3Dy64bSW>cqQa^vyBYg;0zNaY|?rmY=yOF5r(-DNCc! zwf0&gb__%7are-=N(SYPz%2ba5`K{DIK)c&3Sw5!vw=;VZX*d@QuSMcTiAI2q}H#S(LrN z?sVn3QV7pJnf-C39p;PxmH7k1!dCB%$GX+0Xa}O}o2VdvzUHV6kRjmv)&8<=un;rx zNbr~Dw)9@j_n{(Mx!`uGcZX^1bAA7TFjoF6f|;h(gq{6!7MZH8efV^U0#0bM>sK)& znOb`JzH-(D1PPGlg@%xIB>jQz>;V3*1Vp7@wd=M&hiZN?hQiW&d(WqX-%UVmJ9Lys z^PwXn?E2ci#`H8hx)H#+2I}3T4fzr;mhE%3Zw5N6+wjX#^^B2Z>m=D|8*$)bxT2bGhxTL1t6 diff --git a/ci/test/inl_ipsec_perf/pcap/enc_aes-gcm_msns_380.pcap b/ci/test/inl_ipsec_perf/pcap/enc_aes-gcm_msns_380.pcap deleted file mode 100644 index 43ee4e8100d698ebe60a6dd17fa2eda61fdd9879..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 3888 zcmb8wcQ_RO9{}*P$vSdp#)UFM#u?eWv-eIy&LQg#$>>}enOS9iQMg0a*{gIYTSj&v zS=pnU5q|mh{pKcDyec|V`e`;XTjp9}Q#7eW94fPvyQ4w7Hz*?&O>UrLYYPByzVmz^%YWzD+3%Y2SO4*R@GC|+C;_5Z z-bGaX>d*7yzrp-(Py@68Bn9VBs7n4Ad0Rw6`YX@x{hK!ya22xoKRytUMVmZ?YrF2r zTE%IP_w%xk=3fhz)3|tdIK8xbLKh6AFBIV4#qyTb+1mrmFAMmtouL+&?!KbNiq;Lt z07c*2|^r!v6;2@Yhs#x7OY1Mi}h^J-)w02ub{udwvaH3xt z93-UUei7$=om*wc4&Z`&QZe6U<$1s72DRjk8sOIg>C{D%bm+n-Axd0-*Xb$q{O}w6 zKl%%uUg>j?HCb_>l+zdZuQfK;oot&*Or+^cxM|6R%|nTh?h&~U^dqwpnxC+=*w9(y)M%S~fxUF6qGOBkDSyA)0&9rMPqky}s%u?N{^!k0Ui@o188 zUa2Uq*yu-g>t=SBEMcAeprw8M@U*0VFMV_!1|M~5XkT6pcms`*f1Eo%$8~)Usn9$h zBw_lLE;M8Hdx-9+Nb*xAYMUDYJG{xh&TsU>nD59&XjB7}cpVD9`B?TaJfzi6K!ua= zYW-9~zo5H3y~<(tAqKmoTkU<7WM*k`;FXCtv0{epic0BH96sCDA{L7etO+Pq_6^YI zJiH0_?4bObrZdMj9HfQ|t;m&v4J!CAMzoe>&e|iPiSpaz+D2lx`to8*hVO9^LD$B# zoqlRSVpt|yA}1O5-*_)~Hs4vwF`k7~5RTZPYW?pS3`y_dwCAqG2yZy{u*A=?ieO~_ zY&tsL_jjgb=ca7m^_EE}!(-@nU7eiXk14E=q{+FFI@=DcS^CH9Z*)uNGQU)~-xEoy zaIF+(8dA?~A)^Es;}eoC6c&cGr~~tpHOLw)W`As=mXl1d*6#6~2ODop`Z?}U*Sk~s zbl!A zSj1Pw5BH`g#Um{1s?g`jkMFQz@5)@zlR9lgCB3)K&Xmf8GGW$2QrTq3lj zE*i=)7O_oHluxhK5HoO_x)b*s@J(@G!GP-QFRhynSG45i!sHq9N_tX$P=NSN6ObQ; z=VZGxb%-DknWdrbxVReGlQs(ZLW8N>05H-&scfg+DrwD*{z&(!V5Uk5Or!IbY(!ze zFixHyZ^w7|{Dd#1LHLw|5`W|v(!wYkaWv)V@jPWvVu$ckTKCVUL%6}eGbKMaWgO43 z`R6KQh@O8pO)%(mM4d}j)MkX)*zf859jZx=s3N*nZ_0J`OMNx1v@Hdx7u-N^p;-5d z<*D+ZBqLnVbaPl}=&=C}i3W`}CUsqj_ArBDUr)Fg^5ZK+Epf-m8*pLeq_wib^&u_% zN>Qa3EBU*mb*w;HoRq^?aJJcXbLAG+6^SWV0gHd}+Y#Ys5^~+B$!SddsMISNrV?eqfQd1$8BJY7o`EZy$*m-p;M|8qd1Knmu^^ zg-(FlTS?W*TR(cKvoyY;L`Dz`{H%ew8hPI`y%F!A#bgjrh?U=AI%+Br*u@a4YHanVBv6MRpX$MzLPZa3~&~Putl)R zzKz{RXRYfKn&4S%0?q9G@3>F(T)*@ng?5h?ED#0BXWbPp7uiI+ngM=nRLGfGqMh#> z`}DZa+ANr@DSF_=HO|v-3au{A6F1YqjtZcf>9wklQk7*~{Ku!^9FG?O8h!pMjNSe7Kb*(m z0S|7`T#B`lmN5sdmQd@<^x(fF8#j!9ABAz(!!Ts^i_&?B-I3VagYQ1jH)OpZ1c$!Q z)ny(TX6I~9!>L&uzM<(-Cjt7SXtE4 ze82kZ@7%F&JVxmM*_2Cd;_pn)KV%X-`i|+DtzEU;&%xpu4Qac*!HnZaeGcEN%NX%G zQQ?jJDK`dK;9pjkRdU&SOwa7?S#&rYf`!bXjuf~^U5P4B7Gnz?t1%HaLv)8sWa z>g-xFF5mNkKA$dk5I<>Qh|=canDSoWa~VFK(#Vz+38C0@lYgB3$KT{Y-g)lrU9! zk~VGz(Py!ZP#pJSnUs$N`{z+qqSA97M^QFdtJ@y7*^AI`=5_$*J|8X6JRuyWF#Xvy z=jx}wGo?B=m4I`BoFiQW-3%9=b4o2GuURmQPSDg{4ZzppOroSw3MT zXtm+tmoXzL>P^D39HVmM1Ws=TA>eEj?*x89Q2q4zmP-0j+oj=l0H znuTrqrWOU!Zv5dZOD{^5`m3k9OALKaqbs(Kxh3O?piS^Yt6QymzA480^g*?Ak#L9L z;C^OWGZPFULhIP~I_ID)HZioni|VsVnYO%tc^J58I-_gpMfqHGyi#F>le?DU;nx6u zmme!NjRU57C5O_+t!*_UUMi^aQXqUu{f50vHpP)!mtL6u55hGjBJj_q)Vs5PXG(o;x{%0J z2IBWnWax#yqPc3v)SZ}wdc^f9E7$vZT|2hgK*rha1^67~OWZO!hE%beYe%?322t4|8njk^zk=y|dRcq-MzmI=> zyeGC2WJ+Vz!YS@Mc`1hTLETyseYFk1nzN4vWZcBry0fsoP(tg(n#~-VbmqjyY<82j zs@-BZQ8{4}Ev&%-RN}?9W0omwVh&@Dd~J8Fl1=ZV1TIv>pG{`T98Hs{(RKjs5jpMl zR&Zx#>kkHVFu?=&7fM1RB|S!f^dY7JjQ-MYp9qoZR(FT`v+rDb^Lo(=DqnIQPgR;< zx1D*ggoiGxDhPLV9!@}B#sq_A zc+4{KiaZ(}@7O0o-RF`*t&lGw#U5nrYBE$)5Npr{7`pt6xeq7G(1`uve0mBntZBex eCV_`iRV&G;Sl-KqtC`>?piRE=f^zM*>Hh#lmz)m( diff --git a/ci/test/inl_ipsec_perf/pcap/enc_aes-gcm_msns_64.pcap b/ci/test/inl_ipsec_perf/pcap/enc_aes-gcm_msns_64.pcap deleted file mode 100644 index c72971114476a99124a39e4c719c0817682979cb..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1456 zcmd<$<>jhjU|{gI(UxKa(*L1=nL(VvCo`|KLcvte&{WSr*V0foJvA@2C^MOXg~6J^ zH?=4|H94_BK_fh~D782vu_#+p!C22o&sagjJ+nli48kr*OwLYBPfbxsEQZMH8fpS< zVPF8+39?TFO8@^4VY31GOMn;zu1c{nykYgPZ341ESQre1z<`6nm4TrIWQ2p!cjf~t z7@2?=WFQ#pFy?>tP8Nz^<8+~$W8c~0CzGFREZqJf**oyr#T(y5?zP&P3eRZ2BAs{J zJwfE~#V0#k5*sG9KNUW>y+~dEl}=*RD|@leOVc;q_;<{A`n=bz%+J;xjZ2c3gt!ge zjh8P4k>*AwkUR5r^L)Q5xf;pIiA;1tt-pE?!Skz{jY#DDGQuH|E+r767*0W=G zH1Iu5|+QQhAua>i@mdBChRZvDf43!tB$#z||gySW^Ec*N(CTNcaJ&8~A6-tM!< z?#7t%DAL@>0&-*K-;3$z{l6@qm(23M^hbmr(}w1p$|G<7tb8j||Np`Ey=vd!vuZdC~6H+X2aodKyPNu;?El%N@3uXecA{IHgP+t;uYnLN=;19u0i9(Z}`-o&hD?_Y{~EBgp`iZ1NA zCwSJbZQZiBeuc;T`S%}`jlW=|IA{ICuDKV2c5LEYb3ZdsqW11VhIa-kH80A-GIlEG zVs|6cyHwKL2u#-uj1C=>SPa;A9aWbTnO>2#cXs|)y}xs$r}(uTzMfHJ-R!{<(Yijm zPWy5~n%dQ)C%#k&8O$t?x7d{B5_NO~f9WD-J-eR57Z)tWRtGx>H?8{m)XyPmVN;7? z6?Qj9ZObIhjT|61+U>oc5x6P9m1$A=KNiOgt-tNHiZ`zJcX^|Fdxw~8%E=w(tC_{! zeCD1yrYH1l@~2lT-d>ve;N_x3)is~{txcZaJ^zifvB}x>e?#Eh($F; aes-gcm -64: 5101840 -380: 5120769 -1410: 4194633 +64: 8176186 +380: 8173173 +1410: 8141213 diff --git a/ci/test/inl_ipsec_perf/ref_numbers/cn10k/rclk2500_sclk1100.106xx.ip.inb b/ci/test/inl_ipsec_perf/ref_numbers/cn10k/rclk2500_sclk1100.106xx.ip.inb index 2b6811e397..f679676176 100644 --- a/ci/test/inl_ipsec_perf/ref_numbers/cn10k/rclk2500_sclk1100.106xx.ip.inb +++ b/ci/test/inl_ipsec_perf/ref_numbers/cn10k/rclk2500_sclk1100.106xx.ip.inb @@ -1,10 +1,10 @@ aes-cbc_sha1-hmac -64: 5102561 -380: 5121262 -1410: 4161023 +64: 8187281 +380: 8177745 +1410: 8196843 aes-gcm -64: 5101840 -380: 5120769 -1410: 4194633 +64: 8176186 +380: 8173173 +1410: 8141213 From 3932329bc756b15fea738ed0e96027e93219e646 Mon Sep 17 00:00:00 2001 From: Monendra Singh Kushwaha Date: Fri, 25 Jul 2025 08:23:40 +0000 Subject: [PATCH 248/271] ci: fix skip test search Type: fix Signed-off-by: Monendra Singh Kushwaha Change-Id: I366f032a65b8dc93fd0d2e0b31f7a5eae5b63998 Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/158647 Tested-by: sa_ip-toolkits-Jenkins Reviewed-by: Nithin Kumar Dabilpuram (cherry picked from commit 453c0de71c5f6f1f915f4b6501c63dc917eb8b80) Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/159003 Tested-by: sa_ip-sw-jenkins --- ci/test/common/test_list_helper_funcs.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/test/common/test_list_helper_funcs.sh b/ci/test/common/test_list_helper_funcs.sh index caf78c261c..3b0a587946 100644 --- a/ci/test/common/test_list_helper_funcs.sh +++ b/ci/test/common/test_list_helper_funcs.sh @@ -107,7 +107,7 @@ function test_enabled() echo "$test_num: $tst [RUN_TESTS]" >> $RUN_DIR/skip.list return 77 fi - elif $(echo "$SKIP_TESTS" | grep -q "$tst"); then + elif $(echo "$SKIP_TESTS" | grep -qw "$tst"); then echo "Skipping $tst on SKIP_TESTS list !!" echo "$test_num: $tst [SKIP_TESTS]" >> $RUN_DIR/skip.list return 77 From b36fb33e8c41ae8adfd5cc99f9b2c1e6a1b3849f Mon Sep 17 00:00:00 2001 From: Monendra Singh Kushwaha Date: Mon, 28 Jul 2025 06:19:20 +0000 Subject: [PATCH 249/271] dpdk: change default behaviour Type: refactor JIRA: https://essjira.marvell.com/browse/IPBUSW-73628 Signed-off-by: Monendra Singh Kushwaha Change-Id: Ibe95ebc47c2bef966bc01430bee54c49f67b7e65 Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/158746 Tested-by: sa_ip-toolkits-Jenkins (cherry picked from commit ec1200b69bbdb0e0f463a37cb3b6e2ab461b6ca9) Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/159005 Tested-by: sa_ip-sw-jenkins --- src/plugins/dpdk/main.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/plugins/dpdk/main.c b/src/plugins/dpdk/main.c index ec0e8fb7ff..9d36a51e1b 100644 --- a/src/plugins/dpdk/main.c +++ b/src/plugins/dpdk/main.c @@ -94,7 +94,8 @@ VLIB_INIT_FUNCTION (dpdk_main_init) = }; VLIB_PLUGIN_REGISTER () = { - .version = VPP_BUILD_VER, - .description = "Data Plane Development Kit (DPDK)", + .version = VPP_BUILD_VER, + .description = "Data Plane Development Kit (DPDK)", + .default_disabled = 1, }; /* *INDENT-ON* */ From 30aeda3d1c4303b66bc6e851524fe95dd959c49c Mon Sep 17 00:00:00 2001 From: Alok Mishra Date: Mon, 14 Jul 2025 10:30:09 +0000 Subject: [PATCH 250/271] octeon: update inline ipsec integrity algos MD5-96 integrity algorithm is not supported in Inline IPsec. Type: fix JIRA: https://essjira.marvell.com/browse/IPBUSW-72664 Signed-off-by: Alok Mishra Change-Id: I2f6a0e953df0026220042fe0be3a82ed6a3952e9 Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/157688 Tested-by: sa_ip-toolkits-Jenkins Reviewed-by: Monendra Singh Kushwaha (cherry picked from commit 93d6b0200a377cff040b9f997f8a00e04f7cffce) Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/159004 Tested-by: sa_ip-sw-jenkins --- src/plugins/dev_octeon/ipsec.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/plugins/dev_octeon/ipsec.c b/src/plugins/dev_octeon/ipsec.c index ea48f2c742..455ef9beab 100644 --- a/src/plugins/dev_octeon/ipsec.c +++ b/src/plugins/dev_octeon/ipsec.c @@ -1235,7 +1235,6 @@ oct_ipsec_check_support (ipsec_sa_t *sa) case IPSEC_INTEG_ALG_NONE: is_auth_algo_supported = 1; break; - case IPSEC_INTEG_ALG_MD5_96: case IPSEC_INTEG_ALG_SHA1_96: case IPSEC_INTEG_ALG_SHA_256_128: case IPSEC_INTEG_ALG_SHA_384_192: From 8d6e1071c86a1001220265c29d487207b08d6d20 Mon Sep 17 00:00:00 2001 From: Monendra Singh Kushwaha Date: Mon, 28 Jul 2025 07:12:54 +0000 Subject: [PATCH 251/271] octeon: fix buffer free in multi-seg cases Type: fix Signed-off-by: Monendra Singh Kushwaha Change-Id: Iea47d4be2f75b3558390c59c1ca8ccdb9ac7dc07 Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/158753 Tested-by: sa_ip-toolkits-Jenkins Reviewed-by: Nithin Kumar Dabilpuram (cherry picked from commit 58e6bf926748f21b607bc253ff8e6fcb9a1c8737) Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/159254 Tested-by: sa_ip-sw-jenkins --- src/plugins/dev_octeon/common.h | 3 ++- src/plugins/dev_octeon/port.c | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/plugins/dev_octeon/common.h b/src/plugins/dev_octeon/common.h index 9c6dde694f..a2cda69dc9 100644 --- a/src/plugins/dev_octeon/common.h +++ b/src/plugins/dev_octeon/common.h @@ -21,7 +21,8 @@ oct_aura_free_all_buffers (vlib_main_t *vm, u64 aura_handle, u16 hdr_off, while ((iova = roc_npa_aura_op_alloc (aura_handle, 0))) { vlib_buffer_t *b = (void *) iova + hdr_off; - vlib_buffer_free_one (vm, vlib_get_buffer_index (vm, b)); + u32 bi = vlib_get_buffer_index (vm, b); + vlib_buffer_free_no_next (vm, &bi, 1); n++; if (num_buffers && n == num_buffers) break; diff --git a/src/plugins/dev_octeon/port.c b/src/plugins/dev_octeon/port.c index cb4a6f59b6..4a42b73803 100644 --- a/src/plugins/dev_octeon/port.c +++ b/src/plugins/dev_octeon/port.c @@ -526,7 +526,8 @@ oct_txq_stop (vlib_main_t *vm, vnet_dev_tx_queue_t *txq) cl->iova[i] &= OCT_BATCH_ALLOC_IOVA0_MASK; #endif vlib_buffer_t *b = (vlib_buffer_t *) (cl->iova[i] + off); - vlib_buffer_free_one (vm, vlib_get_buffer_index (vm, b)); + u32 bi = vlib_get_buffer_index (vm, b); + vlib_buffer_free_no_next (vm, &bi, 1); ctq->n_enq--; } } From 24532105739562b5b85812b9b01de8f47068c6d5 Mon Sep 17 00:00:00 2001 From: Alok Mishra Date: Wed, 28 May 2025 11:28:59 +0000 Subject: [PATCH 252/271] ci: add tx_cksum test script Type: feature Signed-off-by: Alok Mishra Change-Id: I21a5a06e7166272edac5a85a002ba8e3c16de1b7 Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/154237 Tested-by: sa_ip-toolkits-Jenkins Reviewed-by: Nithin Kumar Dabilpuram Reviewed-by: Monendra Singh Kushwaha (cherry picked from commit b0149519bc3e9756a49cc0c3409a0d82e527837f) Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/159255 Tested-by: sa_ip-sw-jenkins --- ci/test/common/testpmd/pktgen.env | 8 +- ci/test/test.list | 1 + ci/test/tx_cksum/in.pcap | Bin 0 -> 728837 bytes ci/test/tx_cksum/in_mseg.pcap | Bin 0 -> 1487874 bytes ci/test/tx_cksum/out.pcap | Bin 0 -> 728837 bytes ci/test/tx_cksum/out_mseg.pcap | Bin 0 -> 1487874 bytes ci/test/tx_cksum/tx_cksum.conf | 51 +++++++ ci/test/tx_cksum/tx_cksum.exec | 10 ++ ci/test/tx_cksum/tx_cksum.sh | 184 ++++++++++++++++++++++++++ ci/test/tx_cksum/tx_cksum_inline.conf | 57 ++++++++ 10 files changed, 308 insertions(+), 3 deletions(-) create mode 100644 ci/test/tx_cksum/in.pcap create mode 100644 ci/test/tx_cksum/in_mseg.pcap create mode 100644 ci/test/tx_cksum/out.pcap create mode 100644 ci/test/tx_cksum/out_mseg.pcap create mode 100644 ci/test/tx_cksum/tx_cksum.conf create mode 100644 ci/test/tx_cksum/tx_cksum.exec create mode 100644 ci/test/tx_cksum/tx_cksum.sh create mode 100644 ci/test/tx_cksum/tx_cksum_inline.conf diff --git a/ci/test/common/testpmd/pktgen.env b/ci/test/common/testpmd/pktgen.env index 6955147639..2fac32a63b 100644 --- a/ci/test/common/testpmd/pktgen.env +++ b/ci/test/common/testpmd/pktgen.env @@ -22,10 +22,11 @@ function pktgen_launch() local coremask="0x3" local port="0002:01:00.1" local pcapin=$PKTGEN_SCRIPT_PATH/../pcap/sample.pcap + local pcapout="out.pcap" if ! opts=$(getopt \ - -o "i:p:c:" \ - -l "in-pcap:,port:,coremask:" \ + -o "i:o:p:c:" \ + -l "in-pcap:,out-pcap:,port:,coremask:" \ -- "$@"); then echo "Failed to parse pktgen arguments" exit 1 @@ -35,6 +36,7 @@ function pktgen_launch() while [[ $# -gt 1 ]]; do case $1 in -i|--in-pcap) shift; pcapin=$1;; + -o|--out-pcap) shift; pcapout=$1;; -p|--port) shift; port=$1;; -c|--coremask) shift; coremask=$1;; *) echo "Unknown pktgen argument"; exit 1;; @@ -43,7 +45,7 @@ function pktgen_launch() done testpmd_launch "pktgen" \ - "-c $coremask -a $port --vdev eth_pcap0,rx_pcap=$pcapin,tx_pcap=out.pcap" \ + "-c $coremask -a $port --vdev eth_pcap0,rx_pcap=$pcapin,tx_pcap=$pcapout" \ "--port-topology=paired --portlist=0,1 --no-flush-rx" } diff --git a/ci/test/test.list b/ci/test/test.list index d2491f2881..491c1e612d 100644 --- a/ci/test/test.list +++ b/ci/test/test.list @@ -2,3 +2,4 @@ l3fwd# inl_ipsec# l3fwd_perf# inl_ipsec_perf# +tx_cksum# diff --git a/ci/test/tx_cksum/in.pcap b/ci/test/tx_cksum/in.pcap new file mode 100644 index 0000000000000000000000000000000000000000..98405e6523b467adfead1b1b5d8751f34a2154ea GIT binary patch literal 728837 zcmeEP2bf+()t=qWZg#)jB=lYa7my;bGxysDNR=vR6jVUz5Q<0*O;9O80qKJDrYKDi zl->jcMVd%aIsziScP#wxxpQ*veBbQk4`GvRIXs^D{~EtLbI$v|b7tnu%tL?p?Rkx6 zlqvriWg77RJbCatJ6^M5i&+c*H(6ux7wFFh(`wc>ri=dC>Mi^H9{*Wm6a9q-(^mW? z{4wFQ@#f1L7*p9e>&(`luy1vfwI*crlZR|N+H8csKopo$i++4V{_zV_KYsW5#gDuA$ESRx`1j&Vw_Vp}cEnf6qAzX6Upgc8 zr3)7-zLd+CZ052&t7ctUch-|_jQ^W`R*_E^pMBujikXB@l0~1L$e%qn_1WKFT70%T z<;Szy&-P}0neEK1&8$Nki)D1vOSsY&1_%hIx}a{KKTV{ zU(veX-ap2iftHa)tviNWw_~bx>;I={UC(E1opYJXv&>a9*M+9a&($_{x!!T+LNt>s zYU=6S)Xh^(o$KzRsl5Z)RM(xkp3L=Tt`E(YpO$84m8U@azAYNeRcJF=)b0zo-5aIa z{ozSPyZdHTyYtR0w^b1PwV>}5v7HLH zLtfRvxCSVp_rqS~)hzGI^6o4rtt!7+cBu%9JEzI~1-&7QdUPxIC{Ok1>W(5TP@|dL zqr5lE`?6|hRwY)K-!FUT19sfCqs$ZN8d=o6`?z~crMh?cQ@(rUXfu-n7Iv@dvMPq1 zYE~sxfa*u1PUic1?f4e+68cFN_4RS?>-?#{{=VDyRYtX`4Lk!0&QxEk-C4CK%c{Lu zm2?H4Om+O~{YIPj(P^@%<1cc@J5n8=e=FZ{8A=CL$GbYSE}M0^tSisDs^kZ0XNmQK z$?qNCYMPMkkwsp3&wF9gCn+zyIluRU4B&&=3te4VS9jLclXdlyN2JZAM?52rdbG{V z26vFvyQG0qVWh^B-cGsXn%BKcBxe{ZF6rvax;wLOn{`u)h`cOCoIO`>Hw(iHWRZWy zv45UV`RAXV-anG43`PHR=UI0(>qbuBopl!@rF5r6Q^BvfPVF$u!ckz+CYh;neGXijRi4JIWX^vW^HunhEb{4E?9(e!K3!;e z@6%bD{>}=Y_I75yHtTg+Z=Us1R)WcOG{_VVHkdVU3FneU4xYdcK0oE)C*JZ7mZWz$ zb3X)aor8P3vfl2jw!W2xtdXVSaMY$x^!Ivp&B6$7<6QG!&A)5p(S zG3OvUWKo1p;0SG(iqL5<`UuI~Nh36!5we}wN$Ko{(PR|s>XFVB;kxFscJph5iYyA( z*&MEKq{6iUk2W%4({K$yxNIl3nmVC|SZqh=#JNH=tkcq*W4^h^nClQXvM6?!aO~Dk z#cu8^Jk?~HsIeR1*jd_K#X7sik~88cjt=7J1G)P_W6d8CM6xK5*K#0NNd@v0zqx=t z8y(fA%x%JKyC&tUNj2(6wNNud5{7}aST{#RA+E(fb?Nn*sQv@=tej}fgNP_u6xBO8 zsw<|V`onDsdSWwCqdJqLYH6F4w)?T)Z?Wcs3Jr;ZM7csCUhSKW<{ts{+qJ1+F1(eTXgAoh+6^EGFg%w?st3ofYxE7t12AA=YG3yr1TH&z*|* zH~-}0Epva3_h5*(#Ug;koYP`nh}cVX%GmqhKlImT^DzQX76t!R4t{GY_^Y6b3v*9Y zQ>dQLVm%MZ9tPiHmSmA`+oJxcj8p&wkAxf68*N4-&}0!3KJrM|``;-f9RCN81X<-! z85nFb5F)`M#j{8NEJg*=2MJpQi#0EAHFF{AWc9FUY++cuo`S`ar+Qe(x{AVL$iTv4 z)UXf?3v^hN$^j-4MuAC@LEN)L#ViI0kVP1olQHs43L{T_*JDIha}*;(79$q*lomB> z7WId~i-c70;>VgFw|AHofCsV&Fyk3852XNe)>k}WWDQ9HGb{jOQL$=KvTCtu5AaBM zOYnH~EP{%el>rX22tCU)dj6C`&$f$u^vKGUqG#Bl$D$_KqA1uFRl(4rXrBP9&Cg)1 zV}q(O>j5cb_!s`6KmGBb#+4XK*VBL5-|?V#{#`%`u+{0$5SaedB)#?7O*nFeLKH&D zqAJUx?AoI2+7@-!W#~nu3S1q2SA+RFU_(ZTs=?K|jH^pixY~bik1JWAQ;8bZ5><~Y zi-JECZet`wX*-Uw2@YyO_WssQW?Nv0ECSgk46<`mAlvvp4>DO6RFI8akXam{usA_s zQCkYY2_^(^ez<&j-e$8q07VudZW~72Pg96nb$5?AS&US~DdGrm7Uxwg4y@SX#0qk^ zI9)47)vs1Yn}dKZvIu^=G5mg%g5P}0did#Lby+SuleUznwlkGltRC=et!7jW1uul3 zMM1sA`5=n}LJ&g1N-2aqb9fmC%8mro$RaEr$XGleg~eB|_E?m4SCvAQ!g?tTv1oBx z&EkNE#d$1?16jbPU`Mc7h~)F#74tLTjw}M^5e&>-Q@~sX^IYucVWOi`+L1V={nR2^ z4@`@rf)-~5Esh@%N(D%@DE%H*fPVoHl0_&znNhk`3Z-w{>rpD}%POBLpPxoPYfx%& zPSfI`ro~B3AXY#ni1m}78_yeKE(apXBFO%nA^WuyWH0=chpen>D`XY20|Hr#vz-=a zWGxQK0=@#U624jGS*&%Rxv?=f0iI+Let*OGT|0%}9ry6~m9=!mui|&W@oRC)+v2FF z#Q|ktS-=)x*@OA6iH+tiV3jNa^NkGV-V~UdS1G`ZWme5{HOmdwK2u8pQ_XV2{2>(M z5X=_G?JdsRTby0CI3145-~@aE?n0(NUT&26JAg|TA^Xpa?B!C(z8C)m=5N?LP-H8z zX9cn?E~{Tsu45&AFt+j(EkujOKUOku7t zpVct8xPijr3JUx|X-kR9I?OM$ZL4_;kS2>T{~Tj}_7vvZaSI_yyKIkW2CfaYh$p5VQtl@ND?UvK^)!*$r95hW~mt?Dav44d>6{*&y3M$_8b_;AVrx zEkzdB6j@wTWN~O6>=5({JBms0skm}{KG1@!o*GRRrpC)DY8-Tjr-tl3DK(TDLyH;~ zcSl)V9_5P5qfj5k^IV}xR(TF<`ipF5%#z>;S;UihnI})Cc(U(ao+q;7r94rd3|*c$ z+~?(RpO?dZUKm;g)g@;(Krx>EF#SkOq%;sq&X8e$kQ}j_Suv)N}Az88iz~p zz@9Ex@m!5Syq-Y|e{Ymo4eTL{7-X42e@HRtYj=7E$?l#qNEtK?8RT$Fp~JOz4%gmc zz!10xZ1Mxrn`gF|4M8Qch)%0Aovu#N>BCD4I-zJ;_d;|pWKivegzLiLvRM5>Kseh8 zv$&c8DWsFbEs_p55jxyN=!%;NOZW@DiWyS()2(JR@QN(r)|Z%D7p1s$&5@p4x}C0P zMsPf-JR~>M3Pq-A&t-;k%i(fJhr1zNaW^E+#q_@vvGjy?GZ7>si-@)<6Yb0t(YEwA ziQ$xk4q`fpDbXCR>vXuT)8V>KJj^MykiiYd&}ad0?A~L{-e4SA#JU}rb;qSxch@C; zc+-*s9o}?!!%M9J>m0p`iyKrOZcrr(3P(Z;{u31J#FfYggMMTY1@~eK9-5-yZ?^Fi zlx<>N4AI4qVZInrKYI%^%p?UJZZUTB{B!aCGmgPVX()z7;$ss^N{;~-$s#`fkomYz zijO~i%=1w;rj?J%N9ALYh}H~=As-#RvrW&O(>v$IMZ-9vUeC{hoNLU}Y)%I;$s%$d z&E(uEMb3Rr^W>E6a3!abQ_1=1$mwwRv%}rb4tGBzuN5wpd2PYeyl=F*04ybon0gvB z^;;>XPF&qHRW{+3smfGk>hv?!;WloE+qfNW;|6PmS0!tG_WRONZRSc)mMo&}ub8%9 zPSJMV*F0@y=U-{7v{l*;3~e3m&v&>J-Qj9$P*^w?P}p<$(kI%@t>7VY0wn=u#I8J# z1?TG*X)uq2*<=yB?`3v>F2(NWeLXqu9#eKJyOrIuhTRSqRXSY#=Wu`8##u>5VR}|j zegAWs%!{BoSw!{6nCc6psDApN{Xj0&9GaPHX0Dm}49?8!2l9~W4p+=M+{EQ@PZ4M@ z#4Ks=dH?33E#^INo-E@13(Wg5Dc*18x2xcM|_YU_pIRwSwst#y? zkTlQ$F9KJd+iDs|K>*0=t}JVMmqmd7n|`k>JMwlv_)Fo64*oj$>)<~~2LDh59Bw6X zxN^ba2)V;KZ&C$e?G#lgrUUcg%)>aS0a?8wjB4;03il_k6;c+YTA^B@ zS}~Nh!l8uTp*q*0)(^TN%r14q3&(N~HJWaS23aH=OS5p?lM;@~{$4nw7)gagg+qm7 zxCnoBwYe7NCA{FUk75RNiMIOih3U}h+eG}c_&o2i}i|I9_zsamY5ZpVJsl%+;r9418$DU5^6Kx;$}L~H!~ zX!$>Nm|Y+=WRci>o5kkTl-L~J;>AYl%XF1RS6OtGWjL;~)X$HSV&kAS4sT^Typ;*z z5osueC#$@GjiJrAGiE=i4q2o;JG1iqB&9rS-RqS{s@ha}RC!c+RC!o=9A4>lc)rx( z`BJEm$V8w*UWN`s8QWw?5LqNc`?3uEASFZh9qnaE%IH*vREAWBREGMIA%{nl9Uhc* zcu>|A56VKLqWU^Yjn==d*_;49B8xQXFxI3!Q=0U%30{+=cuzG+HAyu|HE9|&$>EJ( zhc|v*@y0Lmr+R(zNz)xTF?2S>i7XPR<5{4#PYKk2zvBf;N(NP+RG?I#RG>bkKsh|S z45h*(5AQBxk|%mnPV$PCrZ<1kZZ3gRkwt2C7OT}aQ)=}DuW?JEqNFP`kGJ_mOUffvx>UMUx>ULb zO1kpS&Me34j{R;euVY$Yf<_jJ z+U+cAohebf;>AMLuo0kz;93Z-h2UDKITJ-K@9crN^}=ZWS!$37oKM>(Rz{nrp>AZ6 z!u^#ME=ws~<0@X^r1n&&pE~{2>8DOVXTe0-yQf(0=1IxjZ~eVS*ge&RP7^v!=rp05MG0Mv+~pS2S&QkcrT5iJaVj%C zpXyD%ro(&$-6M;%?-ka*wv_hK%ci&}0aYWaeX4z`eX4zfP5W}&4GqMs*V026rBDSr zm{ndv68LSLOKNHM775{pUI_R2Pf7?se%uS8RQBqeSLeJs=hZpyES>X)Nnj|1x$Vnx zhsm+So=R`E4?a8J+Z2ys-mi*s^XAP z#oS>E?J$LQ#S~gZtV|sXiJb4PW-}i0NES)t94wK~rX+IfGrdGg5wl99N~B7pO5~7} z$lPJ3?J(1J#Y|fSt&~eY(VlpDt63geNfznlqO6w>r}XkKw|l*m(rML8)l1b&)ypBR zm$}1)Jnzi#2R+=0QzG@VaXhO*tCTNX+HO{YaFRvh`FR%4J5%C$#rM2;O2M{@r;4YF zr;6wB5zjn#5K&Crb9$Z#N9|%E^-6lsC1cIHP*Ji-N&8qyZ%8TW3Ga9%l@f7PQdLq_ zQdQF7sib+1Loqq#_c`YGc`?5i<&*h+^%je_SeU@N1ePTz%chc0tWlsF8HNbz7rKRBp(wsDt9E8pn^j#_&9iDX zt9E5o+}mF5$*R3swJ+=H%(`rb_q9>!g4eb2t}dR{#jCn_R2L6u;{8lKpNZ%3@H!qI z$HRM9c5EX~0I7G!EDh^R`h>Am09D?Ey6o;VnS5TBWL}ehZ zF$X>HWkkgxDh^R`h>Am0KJ7K;#k0}_agBNLP69oX5ME-Y=%Vu(%6mHZZ_LM zW62_o-Iz7@yp+bCxG8I_l*3Pd?y4G#K!NtEvCvS}SPXm)1E0gd=P>X&uo+sa8XFv| z9y}VGW8lj%@Z}izNN+_BL%sEads5bFc7xcGMS{CE3+_oN!F~0cEVu)3LI{`wqBN;h z!BxQp4ODQ8tHh>ukHz2=+#EBF95am^^S>PPze0gUA4P$Sboc31+RWO<6jrakrpK%A zJ~yS*cagbReakaSltbUKL&a=_A0>-^bT$6bt5QFD@elY%rEC_tdGSxt;vXH`ZZ^jc zlSMziA^-S=sUN@jr~KnmaW>`0YyQ3X(oGk1m>uyIvgk{j@t4j>ed*Yoza+&PG8Lc^ zQT|hj2O3a(_D}bZF_Z8~vgorD`LoBSK6~>e{8`!Nl+R+#B*UUdhH18+1<7BD+VIwV z?l~(|%-@jqZv1klg z)S!d8L6cGq`og!lL9(#+|2K$MZfF$)>k?RUDB36g>%d<{>ppC4G-se?WKrvm;nwY# zYTegced`dF0a=)XE6bm%Z|V)tHJS_2OtPq{r*l&`Pc=33M{#JKYe1SBrd!l(`Prb^ z9srMSKFVB$Hj_o|zJS}kQL5dI=dz1M20t|buxf`zJFE^(Els38$S1-+_)fg{^Ja4! zIzSe6;!5ts>ZwkgbUb&WR9iGC!)j_)L+y}Obts;uPSh>nQlI3DL7xibZ}@nN`3rhO z7WL>>?opoV(XJbFk4mYZ89fS@q^WoE^`Lh?V3oOA%@gPvS=7DzxO+>by0`k0+`V$J znc3Y7SKO(q(vq;NzOOqR*k)crKgpuLKF)oeKh@W@K}?)hrp4|7SBB5JyHL}txZ_tn zTQTpW(_~S{U*wK=q&ohDFEXbU{z2FAx@{HmgS0a6L&4-X*Y7Y*4W_Wj3-5U^O!_3{ zg==^oj=^%Udm-FhB9BOm1CMw{ynNmmGaFn&R_~IAMs~^DDVI!MoEaf$!;Beo>f|~U zw~+nQv)@MUk=BRq@%}k#>2YRZ_=YU<&p7ta^C|!Q;8O1&j7dWYGE>=)guBYl1n#Q5 ziVSwG6&lR4a1>eOtcBQFkEEP6?=RR{k|7N_FimaqmOLjttM}Za6B|tryhavzZW;F6 z-6_w#yA^v*^0Hy&xp3Q?yePd4yqFaL+~ejZ^F_FjEOKKvyYc3f8?VpUjgkruTQ`Q) z1LRWaap+R-(@{$`o3Fy3WRXwTVxL}-^691Luumlc9T7gQtH>Z1ivt1|7oa`u`z>Zm zIF~GP@C0`7`6&nQF_9fCY41pMa9Hv}&K73`&h~I#==ss+JMc7F|xRq36Bpx5Mmff4H42a{Vss`lC{=e`H>Ey`=Xe9q7s0 zE{cPADUQP%4^+(Ihyhs?hy6GX-%rKiG+uqhjCGjgv;#?jsn*z0T*PZ-T#8ZPdndT7(wjlFvdU|ijBbHqi(eyDWKp=z=5T!@6|Pkv1xD0Y`{?AA}k?#Sgib}~`a*bQLpqAG9-rnp*yS&(oPw!r_0 zAd*Fayp{vGN-B`6%*TP0>8S>CMg}r0=ck~G+a;);o)@e<$~=gOl0{LygQL1)Dyr}L zvu{-DuOs2WjD*uwKui&q5R?)2A-?vn&E_8nFIg1g2ROuwr$YQY-a|)ym46nhCx7<(W5|2@@e zK1SflqTs*E!AJd%X>6ju&|uoIja>XQ;k5Cl;TRsJWywKD>6tZ3PtTz{0)vFA1VfRp zHGZRFMkCN<5fVP~NZ9+|DI{Ee2qQrjITQ(l83{GVF9{YBw>ntt_-ebE3sEPlhecxx z!{YT6EKcfWSjf_f!eYq4B0QB#5Rn*`5b=Y^Bd8=<3=klTFfu1&P(8Iq!s zI_gj0k=Tad@#tCNu?Dj;z(E$FXL&}?pHk>qdqqZ%tXnC1h8241FNz>ENvumW6+$%E zSDVawKnhs|rIi>;*QcPg}wyhQE!!YK1~z=kZs)w+zU zOH;VAJ2S3ijZSejVsKS+mkohTKu`~|f6qVKYzqvLMIhURL3VBmWG}DFAd{6r1=+|1 zS$JU*0ZsrB0M5(WUte!Ey8}>U5#qLC#QijdxQn=q3kP>~qBmS7dINez7oktU5u(q- z?+zSgItb_@i{Q5#!|z8a_rsjuKQ)uoOmd(udxfr+hzby!RS2?8O-<%3D|nF z3E#{FXG%ziobjYtVyDr@fiPqdX_jEp+@B&%`{_&?*=p0gOY^RwlXp$iL$kyn!FSA{ z<}=#NYG4mp#30KI`a_CAAJ4`NlFdD3kTPhPF{u8jW@41!y<}7|+iCb=yV(#_B8%v> zD%0ue6rG;Cm+2&1ib^M?(=ew~`0_bXOR!&3%O`t_zB$Hh240ax-1-u8>!K95p4f}I zB^#N_E#=k-;+DhXXbw+^IlQe!JQE%SJj*I?VKw`)H;kDGl95G3+mwlRW{PNkT$qWb z8~RzVE=3HRr3lxV<4HxlV(vQe!aLqYb$EZ;;Vn?t*#}b5tBZIv53eeQM00p%&fzUH zhu6D^bi#>{be?tVE#74I2II&g*6qNoJ1)h#*Z93y9NEwXKxLh>ZVKxh9)@%DoG9My zDkvxfi79x}_RZ#C(2p#l;9g9@LsJwybXOkuWS3YoOwBMyXoeXMd^Hqwc=^rYsWL}T zD&vVK8Wn{wB_)eQwDpwH<`{61EaKx2nUDLV__#9Oi>J!cZrPhwJ}MuTk4Q#?A+es1 z4v&91JUr#_$WaY>N^bgmb=wcy%;_K|SwzmGnVdVN$oVE8qU@F(aV4jcQ^`5)(>#oZOm%o< z$>B{NhgVx_NENcy({|At#+oZZS+a<>zhc^cIYrxbo?_a{=D!ZGI>0Jzr*D7_Y3uL; zkHbqW4i9iRytY6z7Lt`TuDp${lm(7%Ft>ugWD$pd#~fZG#o<@(XAVobfpS{fOwyJsT19j-ig zxOm#(wqIhpFfm}dr}_drJ5!D?oQFc|V zhvU*uqP?&(q`l|;=O?tA_rQ6wi1#lr@5iKgzwg$}d#NH(-Yf5w_k)1<4mYMc+%oB~ zL*cMPK{_BTEp?#C^G`c;jA-#WdHBqGON8^expPtB^c2;D(8X zAPr=ZJd9;|_*Y6EKAy7dFEu)vFlfS{3Byp52ZzEYhjaf92b4)Fgvx=r(W~Wlno4LYfz+LkPY$6}BGd|pgVqk`J{?YGLN`iUhq~c~ zdU)WU43ei2D1gUg)GvSudu%SGNmtv9lX1dsvokBtPg2UW)SaB>NKu&2|B z*U!pCGfGMFsnN=3x0@57M`V#E9mblpXG)Wv;48=RCB6JuFF)4u+hJ5QQ@`~cZd%o> zFo!EZ;iRlSH3~J!^k><036(3WkaYbE+jG}K zw#Xvsx|pSFgOqeF@-3DwsgBe{OA{?kv^3G0-b5>uF4xzW<(-`{3oE9Km^?;FSfOL@ z-`!&FfQFGpI(7}~*cVbd_GBOHn3iGdDKI?+Hmpv8&9oJ*WC~WVV>zBS%<-~ej+YIw zbBM*TNYi|B_Ski8<^c#AStM$=v#52ZL~ZitSk$BtRFgAJ&NMmGKqSJ=Xj91kULSMNbc6}9cNyJ)R9GU_Y}+BJSn;R@HLh@DSB18Q@K;QQ@NXEa+g~N z-NSU&7B|3Rr!=0*W?AKZ>@*yN>l{CV?vX{>_X=xYTT1($+n%*g%4}8pRQpu>RQm>n z_T{z<8i-jhZqLeb5hu-hMXAa(un@v0|JrO?noVJm5Ps-|aF73_gz)ZpSqP=LSA|f8 zP=!#1a8L?iZZSD_m>j#}p+6L*#0g@dig%vdYUYG0lGUqXb2F>r8!1)1?}R8<`Gs+c=Wp&h2s&QGC5!OFCeXV5>tsoji+Jd#BcIR{JRvnh$ZU`Linsbf}& zREboHREZo?5}7;9v>j&Ju6S}1C4zB6$$I&Pm&cmrp_OEjUM|Xd`EW`v?|-MzONRnP z{(|bI>ZR(X>gABt%iQ%sH~X?I&*@U@K-$7wGpl@n?V84`8qG=&PO?ZmKhNTMXG%PG z@V6&9RBo$y>X|k@)23(I2JlSV)C#J@+QXX4oN%kUe!C`Ir3~leVX|2xo_UTVDLE$Y zIVSEoP22;iE5)=>())hfY}SQ}l0{0|$4YubN=X+vo6}IK64x|T(@;%AH4PmeX=uHY z<~io~Ip+5{&F>wK{l)Y9LRMFPy47p~NhOP9bsd(~%TluX=R;Ulr8ZqfAq4m3C5+h(VoMeY?$#{0C#3}U)vvJNN*TTi zt_rRSt_tqR72Ldvi#DoQQ>bE1q1r z@t34rLq;Zwhy15CGZ6CLB{(TD37;j4K0A>=du-~nf4JO#7N>Kkx($golz$~^!&#=q zV7aKUs118_8xBde;i7}M4WD8wl-eVIJ#3H95?)zvv^f?HB8wVyFgIvYszEF7#0`?A zHHk%6)}8fay;)xdcc2A0-G{?{INOKgEjZnR!!0=5f{JcxpZu?b_7zC6hqRhA&@!^9 zb;oe)c1*SIt2iEr7m*#lJJpa)rFAGbN3}W1Dl?Q-*8Eg`Q*T(M&0L6Pl0{8DotwIO zs;P_qotr8N-KTbBO;b?-isB~JHWj5!4sNekDbTWT@s8_M%vESJS=8cmM$a3@N&Ma;82 zt7g-R;-vG#P86Gb7&$Upm!_9q9EcFdr3lOX?{}ELpf_Yuk8b52<*6R+`X%mBDfKg> zM_6a4^`O4YuLgnRaqm3pzPG|y^8~s^7Ip7F?%qtCmZ=0R>>lP} zn1*2%hMa%`DJ`i5=SLWKwyqlU68cFN_4RS?>-?#{u5ks2QU~GVruvHX2iZ@C z?59dc6CD)1pBQHq>i9FeHkkL(X|kx}FLK8_QXPK`N=niKY7K@U*L1wvmvwb!T{i1- z846NrS{VSK@WNY%H=3pfQ&{AM_q-P-eUkFRRok%_WDFlX;ji;T7v!=l>*~(BitRRx zoBbf8aLLn`HJRDq60&-iG*I4ZR&SZ~cFHA(uD~vlv|&iNq^mdU>dU%2v+iOzLr&3; zdpvOt!xhR4!wY1Qf5x$Yo=^Gbdl#{P1}N?MjOsWw;)IY1{nPET?mX+RX5AQoi;)uP zSM06=OgAH?UKWlbi=4F(JL{2@v*zUl2Av%2Dl?7F>h8(9d$aDotf#Y>p_V8oT63`LhQ#_!JCvEYfEtcr;swBBs91ys;P|bgKwwo>CT(Zc)6WGD$ryRWN zwhUTHdq<*!@oFa??Zj)Hcq@;bEzSs>?cr<==rG@br^zC3Z^7O^CFSkarbvY3+9TcD zcrFlc=-@F5T;fii7srI2_pX2BjWK3_xScF={VweKqf)N_Tjqg|Rfyr39)yEjsLpjt z5&}JRJ+7O^g{*kjC@bDIa;2BzIBdQ2ICD5+Ko-SeKaRuqQ*k)u8jgcZDwF~x1#04e z%Ytw<2A*rdc}9y9hu{Ym(4yqI@-b%U7cbIaPDDV+qM%IXpzM(f%4Dph(+nL;2TG3V zKbceG!GJhLgzh=H(VTT@lhS4yX=8ET%%Wu2;)Z>TItz=P zK8wq!DO%#qAX-@wuG>0BnO`GRWKp=z=5T!@6|P-x=Wq?r$>>0iHuWl(3YW#*;TCs? zTWkzc=)|*O=zQ#c@l3P14sj!kVs{D0Zv9m34qt|2Clf`b+GnI%7&}|+B7%~%kBAMu zBADW831&gUC3hNa{)ixwMS;AQ1G!2nkiBzpAZ2>0ft-2s~L7 z{8u^nt*PL*oapr&l?EF8K^T0C)qRWQXNy%)i}g8xp$t_ChC&9m!P3xZ1ez>D!bctn zd;dFygv*X$B*-F%j_k8yWDjLOl!1CASgaCStVrRHVtha)O_UB){*BS$+m|$$xe#@- zdRR2JFf3kA!Q#X(GAwj)YG}X1Ny;HHER`cGFtYCAO=dAbfGonuoQ#oYQW%;19AiWl zbW}QqjC5Em!&$87SX5J5%o;7`6&A@R;YH#Z;Kc*x)eA?N6@Uk_2r%OrFb|~wGx1CY zj4UH5V1_(kEV5LKe3XhmfwMKC2Mqws}X}M zi-K+(&%#KE%68H&0YRx<9%QG@-fp%9hR7n2ZNeZsHwChPt;Hadl|cpB$OM_i`3cK6 zdJ^CS5CPyk;*Pwg!|V<~kwu8xh7tGE6ykn4CnHYQBNcIqxbQXuLZ5&mM4yM>VmFL6 z2LWAV5&U*z`28pazcpX*@Po8!;y)4+|1evt8C65U3*l#R`o-dSkj42R>kkN(U@75P z`484bw_TyZ90{nAMOZwLv3NiViK4k#0z z1#Kmsi~OPM{EE2?SS7>1YV(I18O*&YFu(N;2D2>SE0`6`GZf4g$L%dnbXpt^CtwTm zN??1LUSWd{^LGH3EJF658QIIFko_>^} z2PFyvlzW(OxW!oWPoS7Ag8APV=8L3Ye%C21;Idbt0u@aY!?Cyx z#Nt{O5Tb-@Nr+-NSa0oS(~jkBvWN};^=#PdgA^NH=k09lQz{#j4MTzr7Pk~x+)`w5 zaRf0#pj$G-4-_XXGuq4tT9DOKqp8BwcsWIlQ?KWNLbjhYQ_@UnaAivM14T#;i>sq7 z?vAp!Jc?){Fb-(qdGhv{HnSu+LKg94UgpV@DV}V18uLUpy_6@)lcC8IhfCJz$!>aJ zmN+Aj4msmV^JCOaI}nB}BFz#^n)_3vY3DQVsKe2`OY^QFmv@DOOGp}rE7rgsJXwM} z)`&rZ@0dZKn|q8|4eTL{7-X42e@HRt<5nJaWOGl49UXRt{;*TSActED9j?7|xb}|T z!y-lr-b+Rm3jx3QXsp=~R3eM$v?|l->J*)xyMyT@TZ&32rPDB{6Fu)nPpZ+gWkfB( zeo3vY@;~gk9Qjd$*$liQi@5bA=GH|iZauLJb4$0=hxYv?;oTHdJ1qX0&y0k1`ZeXy zsj9tc-AL8|r*a#ZLLZZ=&gbvqr(t~zHI^je}I?uXuaTm(oU>sS*x*eEx$E8^J z62BLVVq#s((6x-=xt1}l!&}HYdQp+yJao80wV-k8}2+})u^!yq!Rvb#>!K~z_&sQ(4jxnc$m}C(- zk7jc2lp^PQD>6A{M_kFNhR>F;y$fo_J^|-qtn)ViiesI0f=01>_EF$zROz3Kg z(D(e12`xnqN@yjt5_$#`njXBNM@;A~3L342edTEFQ|_m_cFUbmtyyg zi!i&Tv_jdf>{fQqM0V5rAP$c?&|Uk)bYWt^bWe46Z@YOBG$)Iw{uonzffUuxxRd@R|Aa37rQ1_U2ZW`i4pht`@9cQ-Mh&JB_v;l_cjsi&yDS1E5B=U* zedM!V1aSYK7J6x+mlk?WQ=wN_SQ74ChdW~ROY1eua^WmK%=5#+zg`6B+F81x)8YO? zQiPJSb&7Cv$0#!nia=JM7L01}im>N9DMeW43RVQEmeI68(*jKk24PxIqX-UHJkm9H zbh8(!gV48Bhpb2#e);Pbvk;_#ERu(@ED!%m$-^f{vph(Rj>?0|gUZ8Dk_WotjP7ru z`)Eiigvx6>iUnCK9!+V*&6}}SNM(>}g=&Rr#Zc7>x_yW43!#e* zNH>Jmp>BBL`1_^prW>L`7754FEFAZwgk$~1SU98>Nrgj&Lxp2F2nXHPK!?X2j(0m8 zmm?t&nwLUSOjYJvb*xzn3PKjCNEfTf?^7x=+bgUhQUIkYqAH>)G8|Qe4)@WyAv%RX ziXxOR6(y@QV7uh*GaAeU$O%~_DQmK%{5B;iXY%#tcvM5TOLV(Lw@Ze|c1gH68m@kX zGn)FuB$O0772!}wjX(UccmH6N7Ah!zA|fnJ92I-aiP=xUCx z=8VMEoN%@kCP4KvMDs98iJX5>w$zLuIjS^5lP05p_5|n=S)@sau_o=A(xk_hVNH_y zJxz@?HPX~bQ=@)TqfnC^6v^R@Uxzn-F?qxP6{SpBRtVJn$2FO=Ax>nGKpoElwS7vU z?!AEpN~#7`pj4n#pj4nfg+MtxLJp0>BoFT{lUj+G1lg2VtEF~pF_%E8$Rf2mi`D9z zDYg0$-%^C{=@d_=csj+?DPFn8J+&!bm`>HGmBTCX4o_h_JcUilCBhOamzS;!aeCrf z$QD^7T^F--ZIF_#1-{9NmQ+V-qNRzJCR&d$VN|)>F%W}Lx{AH0SFpdBx<*_sCA}9?cj_>O$tF()Kt_|)Kt`Fw5a8s-4M4Pn4f=&0EP3< zN)t9BpM0soJPmatixlp!tZ-RM;l?>uI4M6>g;Rx7g;Rx_nF^P8Vw!3(O||~XTP(cA z`CL}W-R(z?GOt4F$RfFWisf#el-zyzJjm^E?K#8DGRO&kY(;uuO~?l9AKm}xtDa+1md`(1G)z1-)VCbK-Wk}T58 zMOiN&PU+=6Z?IlU^|b1x>ZR(X>gABt%iQ%sH~X?I&*@TYN@J7aIUZ-)R)TPnMdJB+ z7SB6V;UTTsp6^PIXuKO&v7Ir$HYCy#69;Dcages;$BSKk36u=tP2$- zi-U$>hPD`@a~W2GQoHC8oNHC8otWNB<(#bQAfiv?AB_=^rk_Pgpzg8Ti& zo6K$yTe3)Sw`RdTDJ8hCuFHZeW%w$%D!3}RD!3z8aPulI+Nfepp^7zyYEM=uu;`;? zyhXbE+~%Xq+Qt-CufC?otM5KHrPOzk4pv_&{Le_zU;TNr*$6*M7X9dI{G(T;e)OV) z{g2{&>6F6O%>Prgc$php%;xxEvgpS*i z=u4aNm(ECi>F6H*l9X%6$b=CQhsgRF5sS|rac--bgin%1pPk5`JvQ~(-~XCFD;u5i z*{-ZR>&bevz6`FVHppKQOuz}g4X3x6L-2jFs118_8xBde;g|b!8)Sc=|2E+CF#LdK z6i4akKwUpR@eMj`_I7hD8bcN}=wNQpq*Q}e-kKXEOKbmsgK)eBr(1Bi1!r4O(M|r6 z|8?N6!d?Gezr&n?mXSrRJBC}gW2$vTlM4O5%t}9X>rifvYIBrTW+> z?opoV(JuZnO*$>XzdvKbQr9D_v(tJ|U*=bX%CCnJE4ugam5t^Jbd4dNf!0>aqjE4B1bWjwVt$WsG2EW%~!C%=_pxS=8|tx#JzFjz4xTcU(r(nU3Kw$Tc0W_GMk2 zS(nYaT!sT8#k(B8*mHNMJooNb*mIJX4J*&}*sRB8J$Z(=&`IB=mw~=}wvKzh!+a4g zB#YeG&2GFo<;LshWH(AGIBeaB7v=GuJf3^TYtQ6T>2c^%@6%<_n-MXuk4U4K-{^?zHCT`%dqy1sNh z-Z8?1Ah<|BE8aDt3g$S~z+{jF=j{tenZpqSvM3JwaU8y%io+?t;W)^oLgSE#10HO_ zwUM|!4No3eTW0TR#=MLzd*&$wX0OH-NFT*jlx;0Ye*`*x4+CDXx}a7NgpO%XXMQ zB8X&BAg|>>u96C5bv6#9OiwkCGcu4C+tZe|)38lT3>CLaP(3|geq)Sz5D_JdqIw5M zb;VRv-+R|b6+2TpW1WS!dLhDP2+9a&l~&9VE?vTye;~YMQHUSl5HFqz@s-$2q(uoV zK5K|)VTdjEeJwUbE%pX$!V!eq$9s*t8_a8nHCYtzr#aqpr{aA8N*0QE%j{o;dS(f= zrTQHz;6VjL7;lSxP>T&Ci|s9ny@V!=y$}9bb2OQc5qPpF_^)#CTT{VrM8Q%Kd|7hP z;18zYTWr`_Y&}_QoLH>%6Br~^B^V0XKLanujz*x#A|!m|k+Ao_Q%Jb%a7KbGawrl8 zGZHK|B`h`$@P{8CByM%En6zb!nF~=TtA|Bn3&Y~|6f90$g<&B}D+-Gt0}G3lWQ$cx zi-j>#4vAq25kH7r_wH!37$86vVPsCm$TKO7On#g(A`3c-ks*l@izPXWwH=G{L&A&1 zGr)@n%sgkbnH7KsvIsEa888o}05kC<28=8tDPV>?U@Vp#EGDHE6GsA%#5M$vN6(85 z?Pg_wgDgVN@{FE8rO>nXqKqC{w^H;BEA&{*cP!Fei=tqQL5t8Nu`bb6B)yj((P7pD zQph4Gt;A5eJ_V&E?`0^-YMDZ5SVPHTOtX+C>&vcDfE@3%7r0t()v@O5fDKuMt92Py zm!@##cJa7Eb+fLa4U#pqa80FtT3s`l3bW=gBMJw&dWi~gWr3>9BJ>uO?a;0g1f_Om zl{PHYcARL;w!jcs1hP#SWap+p_R4w;GF_ann%sivr=zL+Dd8whu$!)?xJt1R4M znU;mSnFlh9^Ana|*dV|OAOgU7#62^y(d-UDkwu8xh7tGE6yh$Pj}a&9kvd$CnBlTU z)uY^ktO&Fx$ ztUz`HQ2TcPmn=f|pBdT9rI7s~pPRr6hHf2==&hsF8f*O`SGcfHGk*&+<1o#rm+27M z7WW=lTzp{t%?Bk41C)E1kN*59^G~3dEQ0yp80L$lVE&ggy?|qpPGPPvA2iRyL)lA6 zMwtKKnJwlmK$FO{TDj4gd9Q*z1E78(!w^Z0rYV3ZyB}ph%k7Pk~x z+)`vwQANxU=$6dz14T1#^P3OUAgiZFQ-!JVa*7%!@ol}>jZ$hTHHI8DEUu2SxI4GfjxM# z1b3_vg9P6(gT9W(6jlR+$RY+=X3!r}4Eh+mb#!Y4DlarW)AVder)S}?Q^O#KTM8Yn zy>qzs&iQNaO1zhhDh8wjpJ_H5f=Xl&omOQ!U7e!Sb2l-aWJ^&8Bpr~3*??3}Cx@E| z9d06YxQUR462X2+EuZWi@$hJ~8F)n&aqCOWt&39JdTd|MEffdpP^ClF$QY_ZZaLiY z>2Nos!`+bJS;>QdXP#(#ty3`*K{B$4Xqz(8&P);Q_QjZJx}mS9QROu12pYsfqB&f` z>TtK8!`*&FI^je}I?uYtam()BU>sS*x*eEx$E8^JI^U#&0%9HBba)%t!&}HYhZ{8= zJtvAAR0|3UL1GHtbkJCHFz81XQE)G&;GroB9`;=x_+*z@XKy-t(}53zUqC^JTZ|oU zwRQ9iqQBJ^2VQv0Dk)j%!1Dg~2Q`>uz(ul%k3VES?vvuQ^qRfDY=<>a-P#|all2gg40jD+5Fc5RtH!e zV5e_@4QcCeiN3=H=?)j9JAXksj=1+fgMAMdT;B@*l0_W;9dme%6o+5Ag*mL{HM*6e zTPfvM%4Z&W>pARjXMN^yX}$B8*5inO|AaoYI>w-6uCR#Ew=kirDMH_~9}`-N9CT3D zL0JdofghA>2<`BWfWtci4(|w1(k<)@lJ3eFEI4nroiUGt-DDBF?`3v>F2(Nam-Xz% zy&yVM)|s--lm~34Jk{YkWVgdBAr4PFI6UnDrk6|%n4T3>|2v*bd=WG!i>Uq>Q+!B2nHe@0It1fcFlsojANX;gEMZT&hkwAS^9)pcwprowb_AQ4jz!oMH2S z`U_N>-enP>|E94}i3ZbF{3ZM`;k5DQ@TXV=q;5q;Kt(`BU=WIc>%x2hH-S4`r(7t4 zP_|AHDpyv_IH&sg_YiP(@Hh80?DRa4WUL1-=f~n^L|Y z^exrF&l+BMufr?^X&{T_VJyqTzf$t>?;}_qq((>OLFGZ^VJOLi!)>q*H)1+m0!UIJ zR1TyftBl1a!a}&vei`TlS)>&UvQ|8r(u(W9&RU`6cDm=Ud;T+P&p)+^5UwoNuQ1e1 zF2YnI)Cz~I5gqP*bGXNfbVFzz>V_AN-GA9=x*-~5k#H={!f{VZIM!a6g+pqQG|A8; zLz9f5nq-8+;c$hQ!=*zGx1%`!ycw?Qjw_TX{b*gP*{lTxA&XR`i&ffmK8b zpj1UvMN~zGql!3ON8xZ`g!3=X;o|>DQG8`OcBF_999PLxVi$WZGx zO=dDAh%Az!eOZQnkdmQ~_GTH9N;;Jxl_8ZOm7#uQ$YGbtVO`nz2W9b!WV@0P*$^kPNT7~qf!aPLP=CILlP0Me)TBw1CQX_&Y5ISXrcj_9(kCbt zCV6;wnbbYFLGI`lwRE2&jfwNkZGwNka3UbS*~CEnpF zY=@_?Nx4K=Lgn&_R@V(<&9#s%vPilvX6f1>C0+Au%+e**kt$s(T`FBFT>~LquCFi4 z@j7{q*U5ALIypowTGFb_h83;dmu@t7K*Pu)9lM5g>ZSmx7phl70{RB_FzY3`%i{$Ppmb-aU za#wkimDwSLTjE?& zOS88~2tV{fxW|7|LiptTEQC_rt3s$ks6wbhI4FfMx0oC|OpYBrn_b1uMyl!WfNU391iBySHiByRkQWBXv%(NY5+K!%_tfGV_PAIYB zNP4;3fo*1aXeC*smy5DqKAh6aJ6~bFlLp`S)S7aJXMtG zB*nAkfDW?~gp({1&(E`X-kB25EpB1)lzMFyPZduUPZiJMA)a}TBPls1?l~szxu3X; z)RhzW${a{T+i``{x=>NFNJ;xxNpDCg={!GUC6y|1RZ>+_RZ>;b;i;r~j`@9#`F-x^ z_ab|xs%C|(zGs`vCXiIJNLJTjS-mVJtG6D&vMROdDyu51Dyu51!(UeO9LoSX_T{U7 z6+k2~kXWy=7p~Z1wt>EqMH;&?YwUR`jXlo4tyx8FJTEh-#;V4u#;V4SERD^pSS+Yw zv7kzC83%G0>a7>t+i>sUZV+3tNN~4i!96J@xG($bc&a#vq=KvGA@w|@o`;;(=OM!b zLiI)RVQpww1r=`b*KAqUueye71z}Pf3T|G-MH^MDDO9nhQ0>VI1r~i21um8pZrY(@ z);6ZFdi6CuUVZnuDW$#(&*7I8a3pGq`qJ{RJGpo$i++4V{_zV_KYr5*{Nqw{Hs!}_{=N9p0=JGaJK`&3(U&&kFP)M4 z($OpOm!w=nrUEn~%6}^HKm&@;{^X;vW)eP07JYUifA-kaXK%QWKPwxZ^4YGeJL}1M zv%U<|6KaF}m6|rpi3slVi7|)Z`(#lY_U1Mml4`?wXnKV{Ao~OTw*iATkb|>y{wQ7f z*TeQ?6>89>^EH}d(HOF*K?ie#CZ!s*@}}G%Sz7!58-xhKL1+`swxFV$x+DMVpnXN_ zR$9EtoPm~+MXfuATeoAXb?g0yTPL&9Pu)6{o1@wsWtADqDygaRQ}s|@5C$LZ85i@17uMr zuH;Uvp6bMj{u#p|~)z+PUZ%{+mwkwx9RkGr>2s(Y*YS0k!G z%ghW|*ged{Fb%^j3^@S>Qd$yr)%W!rqyjIYpJY*AALqW#pX%$HzvWQMz&2wTRtL^h zUy=SG`^k{~REwjDP?;nvSKGbaypK+kMIC>UJKmA%_+zJX$7M7fJRPt0WnGNVl7^FkNKs;;c7JL@X8 z+sYOP9`THL?z}N(Hn@bW-X#roJI}hSSvRsDKT=9(0(VvBLI(Tz z8^$aPN0CL&T8N$XNXl7rpW~f{G-D|7D+E)Wv$}h-?%u4sFYDOJ@OiH)WQ zUL%V z-R#DjQ*ON0U#(fi_&Xc{7`hQJ%HutGJok*(p2?-s;{ceLs}rdo!K~z~@F!X1)3w;A zSEPJ;G0#d+Q8$u&iU+juq%B^w#hwUWl_VF70|FNp4qolX7PBRsOBOkJ0z3Hpl!L#! z9dlpO-eJrA5VUm;#;cupv=eW3;;lS#wm2hjwukd}Uu!krfv3qLZ*RfgJ|*St)fZxK zORhcAy^V(n@h%~r0K*Gl4=cU z4f__ygDp2V6H>l$fDR?!m(RF6}ulT%dwM*qEhWMQZ0;~ z#n!5&-7nhM#Li|BOmVdYvmoKPrY7@81d%KX&im-&BjIa;!vx~Nv ze;~YMQHUSl5HFqz@#VaSjvYA-@hl9nMYVy&ZoS2RI)z)p5ro^v`<c%WME;jab>ZeVzIJM${{fs#-wZ@G#iva>;5k}@@j69RV$m9nZ zBeI~Q7#WfnvDkUASmm}@NF}^TJOjLBm3gp0S2@v`6@Uk_2r%OrFb|~wv;DaqFev`g zlyeZKob_2p%~%tPN(dN>^<0aEL5rGBfTxUY2p*4~W7lmoD+3&45qg$q^!zD>p0$=^ z^vJrE%FU3Mn-Dz~i&qxQ8y0h3i)ksLNn%~1sSu)Xzt&{d15(H$D6Pa$x;_P^>`{i2 ztd=R1hBcHdW=|GX*VdO^qwbgopb}PIqBi z$l^Gb#d$1}NdZnElLat$-eRoz8E{7y0rLn3=B_DV{+JKCf(bhH7|~OY)EHeak|8iH zP6}Ea6|^|SRiIQ*6ryxqtN@SyhcUkZ2+1Opp3ErSDuvQL_##4RwW3r}%5#@#mCqWK zS{&1~IHzfGP?L035EURcs}N*2|7D}O9Ec=~Ap3KM?AKC|U1upj;v=HGcR%)tXHtU1}OJ1zYyi7{{)K3BAEY;VZKNT=6Cb8R9FmA z0apQ^*#aKI+~TC9#SIkrqc}QMLNdbqa)0VDZvoO|5$2y`%+H>}{N$}z+GU4CrCp_c zmPmUI<}EH>vDnG9xD13c8-Ztte=qvuZy0O-2Yi!71bCAPFe*iWCHV#xEHJ9*tLP6J z(GLk=aUG7ueK;1kfmmG2LWB^wmV~IxhXuvewrDWzO{TDj4gd9Q*z1E78~(-H*;r~< zHYgj01RE@FDYCew$l@jzVunDsWJXp{Jo}Z(?F;U#X$g z7;@CGxH`(>?kJ1PqlhK~Ke%?I<3HV%X#i%7Eslji;uY1;VxZtNE;X_Pd>hBOYB;DJ4O zvIKXmfk7p{V+QTKVVhYE>>-O7WSK#KNHOTc4!@9vt!rhFGH7@*$l;bkhimT~uDx^q z+Pf0(C8LVj&PP9NHyeUVWD%WKWjbA*qSG^fW;*F+wjQgHgFJ(4UATT(EL;c(XFFjQ zS2G~h)5+l`LWi3O9d06|!Ah`SQp+cM7ko0tYzAJDMcn!lbL*lMw;tP#xg{H!y62*M zE+dFr4!3+d+zsh)Hze^)co6U`tIUtp>{CB6W+F&N77=YzCfb=PqTN0}6HPbtbr2hg zgIGv3hf7$eymU;Y6HbJr^Q^mjjwZ7=7)KVdZU<)FaVgfld_1#G_H1={)8TD|4sRjr z94=LK^p-1bP%XwiAxKQYP3CDf2ZMfO5e4^R3LcuG;173Z3d%0AQcx+V6vO}+P|)EP zV~1O99X(C#Z?(lC0bV^UDOn_<%kDJV90M+rMST1r^KqXPAA8qiKFZ#-@=^Jye4IKT z9X)c6i-sL88m5d@$WwCD=c}td+Gb7%G07rw9?j(3DMilLKF8#g9dRY6l2ge!?c{X0 z1>E88XNS9=iKs%PfT*6S*R^+;3&2vch^ePBQ@@pB>J|TGrpi9NGF6$XOr1WaI$YB3 za2vP7ZQR6KAyvp)PunXG8f&fuWyvDi{)%b)dUbp5)#@2kemzY|H zo$72f%zncGHl(e?CHf8*q&r-Y?)(MmCCN$}R~Eol%IY{6a4Yyr7IFA@%;7as9DeB@ z=CGF6=vInurIcGK1I}TGJL@xtOY5D#v>u0_<0UB~^mWHInfpLyvWU>PFrlj{Lf>^5 z6IzNKG?&&~T65{oo=evc$~A;`ct^nD9RY`T1dw#&R)dJ#SutFnv0{sP9Ly$**nKav z`*SIFU%xoBTS_aG-O6rd_e^BB!z&>UPdhj~?LbTyCYE`5=Bd8iwyowx(3~uy`eRJ> z1yWQ$?XOIADdruedn8w&C#D2GL$v8PeYK{+8cW%zNNGS;YGnnD=8+ zyr1-K=Dk#rDDRc`%KJgUdxuAM9Ns^1Sn70m`Gj;pSX$~pk>~IDuMX2V3Iae@cjsi& zyDS3q-!v8~(O}w&zl1*~oHpJZ@)(PN)UBups0gSC3_=lbc!b8`9Dl(pz}*lHvPd|VX5qLeB^(_=Fv$po!{L5ahsz@!F2i&Fd9#w{rH~X;l^^}N&8!6lA&XR`i&fvQEGfTD zNy_Q3u%t-Yl}d_Aib~3GmlTKFZX6x~#2>}f2k$5(^yR_3#+oglEo70te1-J|Z?-13 zV<)q|NU@jdi|UK&%ZSn!hl>gv&OkeysKhc0-fM}qCaWxjWTtt?MzafqhAa}BZ?o8( zni89DtifU<85$i7c(#4_d-J#^w`PWPFI9HSc6*9EW)uYX1NDx^h zL;JD}{U9YnAAN^qNGj=6hE#@BhE#_7ks*hQI)_pj=O2{CTY-@z`PAsJeJbVz=n+|@ zNr$l}?U~Y~$N0)|JhsNaubQNqq?)9f^#5p*!!EkRzL&$=G)>+_Kf=jdku*JX=ooW0 z#EC2tsN-3nwoeJv9lztGNvZ}_pj4n#pj4nfg+MthYD1|o$-~r+)JnvpMy(dcO4IfC z8*>SiiY!vAvskUZnNq7C?8j;)wTh}%s#dC2s#ephRt_^ihwREhvq-r_SVHB>DkNQ- zozrBlg=~>U(seOQ*9Ix+T4I|*y1MX$u+9Xv2wIDvhj9^fSiV?OYZTT3gxhoVi{#UAu~gYh}ui_eSq;=7=3{VdZ{a@Mzen|T0&MizdGoR25DYP8CiSZe}W6-ic|d#Wd9xPu_MRyGzMk^UGt+tB^XfNba6uxtk{?ca;}e z?xg5dyFUW-|;^|M}4s$i>vgm9synoUcyDJ&Ag54{lX z@t>3sJ~=xJp%nM35ULQW5ULOkN+HZGCdUqwV@J?Ws*0+Ls)~bN6?2Cvw8IqI`6;w0SeZ8BKszOlPjq14zA|>r(CA}e~qzj(GN-9<2s-&uuIgv zKks#z-5|DPk>GC4f_qX*a9{osC$&&|+z-mEXf!acP?{z}w_n~!TUhv55UQ5*K=HXM>_!>y5B4h_`-)yjHQCQTvW4LvA6|Dc~1;5^lr*0iwht{F2GDBGn$=KyDk!eONvadYCLczQjMk;;Va+b|HC`_%ke@x;X8wZSkVQSZ zm3x$@dbG>B+@n(JXGV{(&Q9w=eVJbkLKx%T`GDPo`r{|iHL|FC_i^`@N_B5l|7t`R zKr+obSSLHv6R@y*n1^8+hFKVL0t%$GBnIc#ckVDRp`TcE-mE7BiiKN+&0YH>6XB<&~0KFD+5I>x+@PLo9)e~~-hk?Qzk$8*PJG#xx0ulAwh z2?IR>pJ6SE03fXl08lXb&iTffCX7U6kr&?cUYPVr$_u~S%6kDaI2fQ_6aG3cbU`k= zvaar|tJrQUTO4>Kt1O08^FUm3J{#OYR_~IAMs~^DDVH3)EW1R~hCvIA&?Q~HSyx}y z-I;Y4!x;v(eu+c=Sr~U0EDYa}MgAGb{&_y-pZ$Kt{*hc|$oZ$+W!-t!UCp{N02d=A zEyT`xB;~BRPhn?ChBVZj)!mbI_h#LFSx;v%Lq%Cp!gG7R zFv|46Yh;n#1GhTZpmr9Sr0mA$AlYfpjUxh!(BA>3sKD{F4 z(_itd1W7grZAADK4`|~_TfAtCB|5w+NiG%#1THRA^WiaVW=lAiEOPJ!cJTQr2k$&3 zE0MH!Bsv(ccH+@ayxEDj^2piZjKJ9*&c8aTV!i`UlSSU%g1vo8%G;~V#}XmA_DJ_O z9wx-Qgm?lBFMyHf#WA7hz3W$fwcYFwx06M#--TU&RLb>#YiHL>datf8U5~fd@Uju! zHOh*2jk+-N?`M$oaky!TG3Ic@fGmo`ejJDIr{Zw3e}=6K3kxGy(L=!rDNqv!JlKNw zTJT&8o;wvebDBk-@lK1XQ0uq{ZIP_+A&#MFs zNQrc$cL5O)krs;dBoyf#X#!FtROwYfK)RxIkRly|gz`s_BEtWhv%{Rdci+a4`pV?T zJRpR7&)NCDZ)bOBXV^osdQ$mH$X6|GENbanl$*Aw!D>;A(4xY=Ma>!VmW*cLE$^;l zN7b9F;3~4nUFWmAHch!}*SpzWvSd?tO{KdmYKL3g?6$a|NUoES4PEDb_uB6p%x&-+ zS>(Gb*>`KEd^c`!_MI#g)pyg-cNRyhmJVR(91#bJg)?QS#hHa>p4T?OJOC$>MUK3c z9l2u4k!@TR4_lo&t4;Z=7HVcl!pM=9&V=c72B*TrP#Jb{s;B49FAOwK!lPu7SMO)9 zelO+K_a-tuWksy?oSyUyy=v*OpHBPzaX-$F;#ERH+{JfjGOxhBWRZ)XU>7fva`9Ds zhK{o-CGHF+Zs=l*vsH_%35#S2BDA=pBs4N|D7gLk8Wr;%d`%Yl`(^g`94UYAbv093 zR{u)rnL+8$-xirB7WoernGNK7aZTuZ@BHR3Tg(@5JXz%YiR}EQl=HthlASMG4(j}# zrXDCURKZviYa%nrX^)R@`fb+TGm)DL7>yq|)_$tyD~WNSqQq9+SP z6)Y?ctu2mREzSs`97we3CSoLvoQPE7g#iJw2qVK7Bd?|~GVW!@h-~O6MtUViEDky? z&fhFff(S3-&r(tlauQ(9UZue-4?K`XfLVwE^Hd5jJMzIYa=vt~>1}gOC@LXfERL-# zjz}za;R!tA+Yme+J%2&7g%JP;S%jYD7(EZB(6egBawGdzDmT4eZbI}}oD5iOFHh1f>-iO1Gz=ls(Q+lHD?eQlEyB#jcsfR*Lm$*GRNY zL`3KRw#jS)*pNlI8pXJ}I)$sHcVJw}9-ZQ<-{8t(g^uf4mVhudmD?lbt~YS^or?MWk-I zmvM#GRHx+(ol8;$%Nk5vPa?s~Hgb1RNpyJp2y- zdWbm~=pu{Yw>!h{s1*F#Utsvj4yeLU;TOWs0(=(NgDma`S${zY{}zRKl|`|I@!H}|;-03(MNQIGK~#X)tPs}uaKrFg zAd)PC>_rUO4O5U^V*!S&Y-}rJ6|z$eS&JL87FT2~?#L2+1zjP0J$^6Gnhe#tD(YR; zF5a(U{H~tD@7xzMeq}3N@vHcqruenEyFs8Z-q_RgW~o%3byaJ(Ju3>O;#BbsM1>wrpR5uLuzbh-FmYQG^4>dvWRs*W!9aPV%=NE^YkWX zwmQA(^wv+Ow~%!XMRFZ=r$hzSV%`&i#1wq+2Tf)S=tmY&aBrsIxD*8s`YBUTPKlL* zNg%i8B9}VeO`#Rv!#}N<6Tn5Xh>wRcANNi1aV2iri!^Nw zL^TlAK=hkHbdr2@)GVC3g`?0gBG!^T!E{-y5|6uOusH|BB#X#-Jd<;m6gfXy)=!qm zGFNgcIhCB>nw$<*z#VEoJJfzAq6(1$qWYEPGmi{4!G;W5>!ZnE1hrMkCFb{YT4DXlzR>k?I!e+dg~gjEV^Z+V9oW5er%RrjjGJ0`U$ z_NmOtWOB=tY{O1=^czmFA#ELs=sOfhcPNnVe1UY_k&IWc#Ni`wlyVRFOBQkX2IlZ8 zDGtALA9GmJ4Rl9Ica(HTY0^7N;R<$gIqXogK65Bq?|jjE+yw2O(0?}r&0j%gvWU=k zGof2kgnr~8CbT3uD4~_mO6ciKXoo%m4t)e1`Up_aE$j<|?jqz~5DgPw0JF&=c0bDO zULwWr+vj0+OKOF(TiLDbo`LLk=n~@4+QFf<12J8g7%<(huJ`|_VkUs*WD(V$XR6PW zqWaknGu0)bL#eJ*SE|n>sylRXap<<;&}@TfFRTn{?|Hw;PllLJz+j0kw1Nm`xtuQJ%SxABJ zx6|-3ivayzxBYewd4@$m@>Wy?R0LE6dY}k6GgtUfF7l@KoQ7l z!-B>-uLyhnE2Rjl{@yDBN(Si}xt@{h8Tm9jBdiMprY#XlmFgNq*C4tE>4j^M zFvtjn!=V(tLpg4T3dGL0H!Epg3Q4h4S?qK%S)?LutRjC*smPEC9DYawlrB|t zsiNUWUkE>HRm7njtwVWD=R3}!qH~n<=mV1_QPpe%$O%~_DL-IIxjrQ+=e*96B57AD zDJm%{DScg19I7Nblyk!mzw{CQmk~==SqjIo7mcbn+dx~$B7Iq(_2qXdeHnW&>x(3N zslKSbsJ`?oeQ_wQ<_*hq?* zij9hmicP;28;1%h4#gK7>LZZwh%SV}UB9a8xm(sDM+0{sXVZF^SY_ zhaWYVE1^_mky@R{YPDHPt&TsE!zsyC)FqxS@pOr&OT7QF#0$fzDz$Prk8{`>ci5C7 zAZ6!adFk zm!%YLVc*rN9bc#W{JPJt`~156oXUOvaOZ4NVOywh`EV>#EtaXaXn7l`QI*^+gJaD( zj;}KlA$4St-2H>)ZqAh4RbJtMPLf_Vpwoa(13C@pW@JEDC3m@X&^;_?EmeSRN6Z&5 zWwWe8+PBTxjplRc9$BP)@3Qtar?l@AU#F=Z5v7KD8tQ4Nr=i}=4)sFq%WWGp5UXCC z|KvFTAuSZ8D#O4+2zRawG6M&g!XhF3%nRY3U#5icrDhHSCAn9FKn(&l2-G03M+AYj zLYU(qJas1lUa19SNJk;<| z!^55&9)_xzJ1n6cme8(PLYE3whK;;}J^{D37J@vIMG`q1OXO=QiM-_(ERmAOtbwBj zjv6>>;Mn5>$50}3hn2R&O50J($#$H%!~rGra#`#+-i=FS%Rwv2BE4LY_44VIUOqgL z^-{8@RWDU9RWDU9d#zsPt`oZ1m1TKO4S3pdc#{;*Cr)ZGD?m8OBJo_7#q*(*cy7x# zaB(!G>v3I=>v~+*@Flx(3VY?^CjR?}03oxZI@q^9w z&{wiZV>e=ry)dP*zgm|yRGB}f_r*OaR0d~3$CQ$Yf!5} ztp>Fk)b_8SHWb{v6@@lhv8T|AJ%!eetWaRlM~QeVq`vE4S!Y%^rm$M|H631k_q{Wv zzVlaDeI@ZfqYpjyzIuZ|xUlF$NAic>l={%i_u&tf#Mz906;Hh41r26Pd^lP3;p^~+ zUzYmtJ5KW-j{MwjPU*h+^ziS+H{EtwquCkXLKc0~7W_@;roQReoWDuZHDqSOjELOe zZf3;dYe&C2z+eapi@tUz{@N2$Uwg;p{Izn>DPP-`wPzh!XV#S=-depfe<42W% zZt5NtLyAX5Zj(=K!asC_fMST>JG8~zgl8s;o_jn$_qwU){`7QyZjr%h_uR-ZK!O3% z3vfV2LoZ(u3_a6old}e!d+`Ef(M#OOFR^m!B~JBiDBJOc-)I+np5<9ZQRy8QD z!bv#z;Y$>Ue3&^h+Lxxrn!z^)?^L+uoX3Wkzu|4jqBpvS-zZPL(XJ!eEv3}Yh&RGM zJM9N`WqvmZ-b^%hUP-inL)*8?_Yq{G73!#WJhFs#B56Tp$> zlLU?gIG4w){leYGyo2{7i{AAGe%J4&-gVU*_+4dUn?4My0cYx65&j_h$q@at7FQF& z&2D_`UwqTfdh>6*G+FfG6Zpjkr(XQI-|&meY}#{PytON98=ke(6`8qWUH*ChEAai(+9fdZ?Wn0$Pp0yRnZAfS61{uYWyorOh zSurGJwL?-@&qMN2YDmT`!b2iqL$4T;woddI#I-f-+7y71rw;(bit!tydCZToA&bUm z79O8BQ{yxGA|4-!ReIg{v^y{=Yj4fkF##7dC8o?6O$C^qU98D0je#PI25UYZtmjgL zHOEOjSP~)i_Q7iJ$l5!z_O7gBc(Fo7q>&i8dk|fBVARN>kz0yK?$4=_`)F+*If={q z)W~(%tixp;d4^u-r0?>UAzcdqe)UAftcD>Zi-xhChw-k|Fy1-~595ptJ|}B_ubJjT zfe+QJhOJcq42KaN<JlSM zLCXV+mIuTS8MTrhg;U;LrqP@Vhmb{1If9+CXUZw#c44Q;QbwIJnNG3jLt)W}!lHgV zIY>sZbdaabyB7^G7r;AYk%vxU5B((Np+kqThh+7n9-0CVS=80HsIYHQ9h$r)qZxS1 zyX&1_4>DK5Rb-L7&S!URnsV2!x3arr$)@g_N_Sb*k+i6vW>GGPTqh$Ny3YIVl_AaM zHu#M!^4*o}yERk3J9H8Doh%g9chk^!7Dua=4q)hD6DOO6Gi9j7nZ>NO!@d>s0Gvn` zIr3I^FG#|LuyOMX*d-ohRU#uQ$0O5TxPI&5*{Usym~)-^?NC= zzB7U8DJx?2>WuQLrNe$Y?f1w1*j0*G2?cSlz@g_Wa4%Wp;wRX}i=LvQNnO;+oL+-uc@jL+%SWo-A_yM0S2t%K2X(%g&cA2X%f=a=t~%kwwam zMFNaPA_##&TvdXhkby&1X)uG}XtD?ipL-;X{!a=CSB+&P$R>v(p(i83B3H&DZ3I7D z0Sob44J=w8YBaOM>twaCs2|9%cs~V;lRFs}vbCbH=rypgNFK3Bl(RVYC*=?ymJsoi z$nM(?Gz$X)WD!P&F-BfZVPxF1j1k$;QH=CTj94T&SR83voGBAt#Ge6PJYcqKY%*F;Co^DV8%Y7v>j7hNN^Ws@YjKQ7;1S=3;PL2L_p@d*0^lHv(6bz) z=fM*s}>>Ll)s`6yxgZ6s{KEmT@I} zbc(BfgDZ=jS=3g*Oo)9O(k=l(sa;vc0%Xfz<8B9Fh%5rx#tgFGra(3U7b7e9eB?sw zMr{w-s0}ynYWHrcR_x)@EDXEDD5`cI48^MoWEOiumMeNz0U`jLN8F1Y^=1zMiY!9h z_KdjGQivN5ycUR){YaIy{wZrA;=*bMggyaBh&~U$x&PZ>4hFi&BKYmj@I!{1sc)b^ zP-mKP=c)L$!PyI$5w9@(WCv8?r|=8mXA#g@%(oVEqV*SqkX0Xrc*W#<&*KBkF@PFb zgvEmxiwC5z*l;ytQMO$bi;BfbVbKDK7T2*X?qiWm3UC6MEPy%Y=|Se#z#Ulx%%d5Y zyQP465kq#v6lB+!lOZb` z+X`8Q>{LV6;%cYG6Pit zNg?}*&luTqYM{tgWX~{UThu+UD12ajXm_NfXUmykZ zzi>2ypj{8I^zf?3w4;NvmyisU-D7^;TLzi`1Ek3!%)ic<|4s_?M{eh(9TC4`UNPTe zFmF+K#iAC7MKKV8ezvRIU5AwmdTOF|SOQ2Y5UrUfU}WDy(w>)EjPrztkP{R&S9auTGGzDD{pCDOO3 z3T07*3qP=P6HEuSY_Op&h z@)RP6UOI*76w-U9kdP-1Mb;eZc{$YcBF+e;L(X{8+&rq@I1q*`BF&;qn#WS4X+DZc zBS&pY8YNBdCyhf9Jg^5XOHgBt7$o?P8MG}fg^vV#$RY+=X3(Ef4Epp-W{@22DT9QZlND;;vq5kXZ**B8%wseWugRDLTD+8`DXS6qQa& zr#?<6hf0JFl?WXw5z=HO*e|K&gS`z8YBpPdS7Z^l)@E*9p5oT?d=nD~_qvmyI|;pd zCm~!RRRw#sQ&q?vSmB3DT9E2ubHzcP0uAj0WS#BG&zsS$9&3b?=_yLq}xoDC?AU z%DQh@=TIcqQ9Dso)GR0{1gWNAEB4?g-fhen(2p#l;NDEZaVZLp*_A0Mr^LE?)74u) zTD^r4W;pRxQP82PtwU8?M_qk=Ra<14#=9xR$JT}ha{{4 zaw4weRB|dgC!d@SRlpr;KReWZCZY!oNr;z_2h9R4e9 zm9|RTsi&<&5q*aO=?(?boiC7%Wa;i#u$Qz9Hur$PWD$pNU=FX6;_y3nFoz}GKsl@& zRt`^l4m;GW&m4-@J72UOHy^qu^sm1jV*Uy;lSPESn+e^TBJ?Apnb4BtpoCUJE1{<| zp&j}NIP?*4=p#Tux3Dh=x+`ti(^~OEV_pEW$s%??%Isbu#qQf@V|Ghwg|b`Ot?ZtG z>~`o9;?UZ`p|t}sU6>d!Ju9d_4o&hVfaYWo)t_go&y%A1nNNGFd=AGp>JTJ2twH!MR?$~ zikSs!Kvt^=jdflT_WD;!5k_9YiXhoCstBqGst7$@5gghCIkX9KXcI*7g3z~A2fu1K zqjiXx57Iyu$-_{VhqqGl@Y#_p50ay!@}Tmd^3a>)!J&(oL+2)keoZ75LghdzvPwG+ z5stjhn5Cc-WRX_P%UbdGlvdobA!~(X2B}u4R;X6=R;_U8^5)Qs$f2PN>4wlc)D15j zv!jevJ4Ayl5{@NVIR26njvvg!!XddxDjX^tDja=5I2^j?I5Zk@Xv5%qdo$EsjWd)e z{P_841I(&W5VA-`+E_*Ym{O7EH&{g^0ZLUwRYX;!FRF+`fq#dZ=FWGVL#f(GQG8(X z@W+G929Oi7NK$^ll5%}YQqKA2xJ1&fR8mw@R8so7q&QT&cBtNqAAad0{128s#g54W zn^w#=&=#^tU)E=R`CUq1#_q!*izIugzNo&azVs`7aVW{^P&?A0t{rW&h!zA|39u$t+Mi zrUdG5cd|f9)}RWM3X}?z3e>j{D2M9_P%13)kVHsoC1Mh()$k2k%#~0ovPiAYW3}2W zrB+9eWwnxAMO7b>zL$Zs*b6Ssg9|RO%EN*vGJ8-Ihhyz;ycjmI?}YP5VgIQZ8T3n z(8wZDyN^X}cuLg9EykiIiJ&TKDrzcfDr(bP)N+JrIbppQvsn&WNfznlf~=QMr}Xmv39OfrJ*|4FdZ~J;df99BGIyQO&8{rV zb85iTflbS#c;0aIV6y^*lPnU?Wm!BQN{Q!|d;=F-qWldio+_Ryo+_SwLp<{wS5k5; z+;c44bH8vGsVf)ml@$<%u7mTQQBYB`NJ+a`N&k>i(qX4@7%Exfs-&u-*fV??v`XRm}=n-SdJ5voR!9Ov?_eiuL_FpyZUvD=+6&}ON$6L8vxKKFkdbv!KEw{;DIZ+FjRh zuOJL+L&42kQD~zTdkU@CQ)unTNP$azl!&+3Qdstqido&5!fMsmba?gM_s*31&NnNo zuO$9wBv7}V%2 zEP95~{0w7L&v5=;{0wqF(ET&uz6i3TahJ|tr9+BKw^0H;&;z&EnG^9qWYGhS;Ro6$ z^*~)4^8?A&+W&qaoD}0I7nfUbw*~PFjhFo2!FW~Z={~x#-kggkBa5Ey1b(`mQ%|@0 z=lpcCD*e_^2YMsh9BGvq(kkgu<)dmJwH^g_F2iGzMUQ$8KkAmLN1gpI{HPMpP2Hn{ z20$Sk?^By_P~Qy#iXr~-jswh1cxJNbxySQ!ubX=AFVFVRjU<5S8YW`s%dZHA9@HkB ztcT5od+`Ef(M#OOFR^m!B~Ct?U!qi7_&m$A)~t%!A+2gqT!oWx@BtVMKAxRU`RI@j zGe<`I(nY5i{ATQz;*Gu@)@1&Mw;_w(=pKHfJoQGqtj%v!O8ty@BkZ%&eo$BDcZ2Z0 z@!NTay>@J~c@eKh7QNnI`Sli0z21uR^Xrw9&5V9Mti!Mj!zv6h0fAGVBnId9f2x>w z@SbGRyS~8h`rXvKj=YB7RVKD+rwo$-XXwiCT@n5u`pFRev=&zrf#z;}?45i>%V6_w zyfj(#;uHAA2d7^AxzqW@Wj5_OFW%afwGGeOY}V#7ER6^N@{|Dp3MT(y(IKXx&J-4n z!Y6(d_W3$B3OB6Jqabs5j~#_J$YopB)}FN$$8Au!Zjez7$w%i7HM3$!$ZChAuAYbF zqtuX$nTLl&!iHWkByF8pTUXXTJZmqmuYq9Q3{O^B5z|@SJ;uzBu_24bXBHlxH&f%Y z$N4-y607vO@o9Hid!DtoX6=}O{Y)t@6Y`4&s}t4mm&QPmMT0dT57u+3!J6Y&JXjJT z_4dJP@5tIa(MK1>x{DPmrsEiB1%WP*$;=L{BXv_><3v?s2>u3K*JXFYeBmfv^=nAc|iP-Q7idTIAzQ0hL}^~5VFWA zN3c`&OgUxTH?x2&Wz;E?=@g4T6c&9bEb6zDgJc9t2W6F&K$}x>dVc}DLl$}H6!y?h zQXV=K|96ExUsg}*p(*f?MTLEf3i}qtsH;XZ@RoPi+~^&06!^ES>(uD*^w)z z9N9h_J5rXXO2TPK!pM;pht!si({L(GPL*L7r+RvRxZptZBs@wMdG&tw>i1G!eg8f7 zs;r3Bt24^0mJa*rwBH~1HXk@F|A^P5u6|IbnEeA#kP=l3M%TjU;DBp+Gi7+7S$ z5E#T&B^U}BxPFVlW)K`r79rttkA%_xNg?5?0~iUi$)QN-$w;tBhO$U}!Vf<`h~H{p z@uOFUnAzcVvRYWw4`f)ppMu57!xGj45=BqX74MPT&nyT;Kc)G&gbjQ^1uUG1ek>wFi)ib zvm?oD+${ynBl$QN_m_0((XW>tVN^J|z_gG{3us!vr$DKoC`9Q< z92s;hUuP}`2+1Opp1~;naSElo|A^&Nj|}+c%k)k23$w$*JfbkkBuqM}J$J4;Z48f< zrs>*cico5CP1E9@ro}}~(p5oJfY_`;klp`Z_2yb2k}QJkMGV;uQ;=O_C_`2@wspqW z8GouFYjH!?;)<-r9a(~}peuy0$L~`QH5#gQRn)seF$j<1I@5&mSoPO1epgT7cTTRn z1e$1Ipn<`RC`AZ7muM^TT!`_Hw;O040anTIS2dXLU@&*4!2I5(UW`E=1+#*AMu6Gk zy1m8KWsA$<1Z+WG32ZOZpP=T{vj8qxgzSeI*~_Gm{dglITaS)3GL~KaX&Po&)Z(zX zkZ(~&!lI5uh-{0x2Ns17tgn24z$Dr|L)JM()5kyDY~BEh$s(9P!!TbU1@ni_V3^BU zg+|jFP0tjVTimv@s6c@q#nmYU;wjAkJ%CZn41g<3^Dk~!dng?m~Ee+lxHvHGKVed~&gZ`9a(7&5JgK!d}(~eF%eP-IJ zVvs|XLWi<<4rTA0FMC(wy<}9e+F4?WidhF#B8%wseWugRDLTD;KhsH$6m>$<38{}w zNVRlws6^;ciO``EAtoyvJMvUgQp*Q>XQ2Jf7T^_G#I3cNTbHM}_1vz^Ejh?kZYj6= z5w{$wd^*&Ibf^tUJQE%SJo7}evkWymfn;P6(KctI{U$}Ud*@)H$?2&QO^Mb|iRMs* z^_xz|L^|O_NV=@D3eJ#@zRsA@U>sS*x}P%ZPD-)v?Gu=Fa%QWnQ`RZ#z`ic@n?;C; z3aZ7tCj^Npc;bEaW(?>@7Ey3-rr@{~1rOedDJZAJNg%i8 zBIPbx@)c9!LZ3F86Tn5Xh>wRcANNi1v27&tQO>57kIF~or;>B>$>~r9+@bcfL+xiGst_q4 zDz7YYL1R2vN)|EoY-Z|KDW?AZ3udaOmgqgCnR^duvU7F6^(pzJ4kf~k({MkhYGqj) z)`rt)$W(_S?GDwr9jb8?YlT!HYdvjGSz)lb5tJp1XnQHscD)pBJGd4%Qu+00UXSMc z$I*OvmN?yL>rh1Bp;DtmrACKJjYMN1SxMs}NPg&%q2?a&mn`D&4b0(HQXGE!AwTuv zCW3NUIjkK1<|t*Vr`}o)JJhVt9E#REU$nktTtMg_;FRWowx26dF z&|yqyNpjFtudaG^)jMUY-Y}M)B%vMp2src+aOfj|*&3JnqW$KqLhL?a*GBUKm`xV3 z`%z~15-E1yx&X6VQY)0*%5G)%3}m-Mmk@{64i2pyi0Q(_avH}4cB1+Ll|g0#XigSU z{duPPJSnQ5`6yFe5;~OXN_D0BOrp9&7Z-wd9S=z-uD3BJ2csG=zrp{P2$ifnRGx{TIxVC`MDuNr9DAsS?na4gBf@t2fve1A?B4#`DQ;ZWgF;phv(;n0Q8p`n>WS19M( zo1vg&l*CXhRi0R}*{li$A&XR`jaB52DHWOPqe4YGQG-uY4mIUaQx0cL%3+wH6y`95 z2b;AA994Up;Z8=lR0+e6S`~3=FOvdU^W%9*%Zok5y+ zVUgHu&0=#_N^Eu+!NH8Am}xMh!Hfno{VJFV#m1o)v_sKahk}|UJfaJs@Ob69azvxq zAId`(DbLSXd5%jdPy1c0Jd)I=%A?Ao%A?8y<#DJi>d?N_p_ZB}x|epM|3kFgDgvN= zar5N}NDx^hL;JA|9h#D%7GLGC6Rjn5wWX^qU2XOE)m9h))yj}VRVs%XH?C+X8)!xl z92J_h`9aO*6zCCIq)CUfChe8dq=~~gG?M&14UIH3($Gjlqi#Z@P?H?$d^nT?aJVQ; zi#O4aK%aQ+Ht%3_KE#PE5~!0|pmt0N)Squ-fs(926(|)b6(|*`Zy`_)*P5YJSmdGa zGO3k_Nu*Y%T{+ZT38f;7)apD|tIbksb;Kd8R+6izYNcwWYNcv5rE2AHv&!MhghTo> zDVGRKs9afPbsWzfwq(7z6|zMZN!JxDU2CPJYwk^0x+FVNrAwttrAwu2T1b~e(sho^ z=p3m`dC{G`6a8T0U3Sv3J>PCL_d~_Q)LnMnW!GJH-DNM+ z3?{Y99`0yWEf~WPtX9WzWG&=4FU*U6@tvjU1mRf`oE>%PAoBzSjVuzi`&iV5r$p^n zOS7m+BB%yu8k}ixroq|t2+l%L%dxYcW4|oN$`k8PbQ_O?Hk1XWua2%iYV14lxrUb!3s; z{e$Ih&Xn9Wy}@!PNv|q*Dt9V(Dt9wY?sDs(dsxm|ssP)Gf9fXE%PMOi)cf;=#(WOl zBa5`}UDm$ll=l5=OV&O~vsLX=?NjYj?duWRm)kaIAXdE=t6u9@y`ofQ7+4758GAOE zfdfookr007g>cU=Q$qN{5Eeqc5TN~GwLdHp9Y8q;%1z%Ubu+zUO!E5mZ6Kq zSFdW#8wP>3LYP}DjvW@qj@o8-LMGz?u~5b5*BfYtK^4hrRdK)oR>coes`%LTtcsFS ztl^=ChZ-KLD)z|mFjU3dVF~T9gm!)jEeckKjW{f!74+Dzn#@9wN3uvFXJd(cEhUk+ zY{L>MdCV%2Dv>IYDv`ZPB6Ek8w!=!>QOn6r1pjeB$$FVR+G3W2R+2?}xghK1(BO!{QatnHhnf{2oMe%BF3aNiP)a;E z5 zuT<5nkkzN3Z8RH0QpqA&U4v!y_bFNZ(?Kk&lAEppss^YUplX2HHv`mAR`VR&06EU( zTZ>%)oXmH#3s7k6YHtoQ+e2T;B8}aMHTJ@k#vbi^Yj)y*OG8%;T{U#o(6!%$uA#=} zt=KGR#b!Y(^)e3RFbHXJWJP-gNByZ{c8A!KMS{B>3-0MD!JROQgIY<$*PvE|S`BJ7 zsO?`tZ78^TD++D2Vo#wJdkU=`#SKklTh&Iqh59amgn`wKDXdm~O@~+CeeX=E?|g$< zeI@ZfBS|0i)?l+PK9nr_(2@M1H>E!GviDAHEKM_+_aN zzvDRmaLGCQ=EJN0Uil$jw(}2!GvFy*mSMcg|2r73;^`j2<(6~tWMt9Poxo4GbL#2V`ih@UR;B-YxR6zj zyEjM$!htqUoUvz!H7!=18Hi4gDj!w*sKNftfoIsB+wrXKaXkMpBWhxIKz zwtQ^x*d74C-+z#~3C~OxJ@MN$@WvcO|74afyJFzECS z@)hAb_?Nig{U&oSUVtomi5vMPR!+Ud$z%8>O0|X0vpj3fs;C{(YEn2!y+m=yhnXXz zed(gHCjL2QW`X=oA8s~(!`qNWZ*&j8QJ#9ET~_5cDy4o#yb<=YBYW!gH4&IY2 zde;~DUB8=p*AZ9pyUN5ieHc~)&d`&08UvbNz_o6Xu>hP@~PfIMXYfXbSPc3uP*8|u78 zqwt9zg?+wGjlvDQ4hN~GbcU^(!zNRkreqY_AeU`fTYJ`49JiHE9E?a-49TGf*O^%{ zBxJQiQdiGI@=10vI#hNP`CYwOC|hiC1@bXGonI6QuQ#@$zM=EvBOMdLFI zkI$Q_@!98M9v_KSdX8e$5GRC8I6mzzYtOUx)~p>9a4}OtgJPX2!1U5Gjb>>K6j?M_ z^YLIkml~{Lr}1D(gw)#ytGy#@@66h}vX0@!3Ki2-V&vApcYx`@sF6h@w-k@upHn0E z;rcvs5|{Oq*$jCZAm@n-ITg*c+O zs#y(Ns{j}dBRa~XpFG+tN=}%*d z2HI$8i;lL~qC;0n8e$oMV2BIV{Mi~6vn>XfEE?bqcz`cS4e(A|GxsIz?X%nuL0dDx z=-P>)epCv4yZn!;Q7idTSqp2^=kGS= zR5*kza>^0xls!{U8MB*r3YNm^lyB!0i#`+?y{&IZc#hjqOvZzPDVC#o%h{bpEjD? z;5V|!cUQ9S)=c^CFg}*VcA$#kl#5|Vwa|AKN2`{$dg)*jC!2*cWvIoOg=RjB`cDtQ ziDZ!@Z)HcWm~x~W$|RKKsgiISk}z_l#UZt&<20PR5<_L!#i^d2iz9*fNqCej^6LHU z)$gUe`cIs`PymS?YNh84pl9e+OULPSo{j_cDi@R<_AcH7z2skkd&wdfKfx|uB<14E z`3xN=5bENY=wgclCyT5Ji?j)Hx40v4xA*thm0Qev@HJWF@0Z!%bEN#e`xQ)SS^cX} z&nTe|DINOTB2C8PjL{;CgM2Tp34QOK|KA4&n=jyavdH-p+4)%gnED3#19heu$H>L6 z4bEQ3d^(PuFIx`k{GQ}|iyR({Bp!<_9*Z;{0)x1!1Va(8ZHvsqL2xt~{=zT%S!bG% zJz4j;N5bg;(0}X>30LjKNYJgJ-dDw#U?F}BV6iqfr;gajnAzcVG6F;hi~4~Ki}zEo zIB`V}3mmv>G|)q$fpB)H6$pz&KZ{I1{GilTOe08+vq&KWL`ol)5XmZpk(b}8GYbO( zWD!P&F-BfZVPwn;j1k$;(MhByPa;W-SmfVXb2aH9gnngO6MUD}HM|>NChvfzpB1QllWD$CnWAr?jLeDA- zGJ0g+O3~A&&|{HnWsxalk=kOBdqHRtUzcbqX0GikgUk@HjBJuYO7FVy%!umyP~u)R!wwdP>+EPzW!n65(h!;I`@QpkSnOO|OlHBgyWnVzmP z9U|MJ?tw+&1M4dvK+L1vGn71}{j>A1m-_}#OcufX8HV`+DVX1P3d3B^Dir1l^O*v3 zi;@Ty6)5ndxH^U4r8~^83(PKlw~182wY1-6l^$Y#{s4V%Tuz54gd9Q*!$BI8(!w)Y^*7i4a$aI z!3K*eMHW?xEYkLf83Nss8GfR80`;)x1}(^HsnJkjYWy=rjZ<&*krI~LN)4q(ucL-V z4IGOiI2KiKK$8;20Zlwl1|HXJ76V7fBA(2}Jb5X_lWn-{Di-a^6Xi*-=7~eKD2IAp z4)wfF^1V=2-MI+iCFIcn2oxGuwc)-t?y6%xi>;p7sM z#-Ruv*n`#|sIf*25`4!D8u61MW+d1{7BR>&gZ`9a5OpG;DnQsH)@etlojx<|R58e* zN})s9JBPA&&X>I_@m?~jvL3d@E=P@pbwDMuh)&;UI^CS2)5~`JO0na?q7JsPG>;#gLMMT@2iT0Zm(eCBDfjC3em{4QF{t**~lUPVJ zha#-sw3a8*2`56*dDgji2b$4f99hJ=pEB!CO0n+sV|aR#Gh3bBbb9Nj(_6?oM_p&B zWgjZ27W1AEB&Ohn^EaC@pdVR8!M&M+<5Cp-#p!H^oOgg83+C)5=HXqw;ame00<-oVtah z&@e@;LY|VFK3;wIu_5Lh5R)t-=kZL=T~g%yU@1>dBw6VyQCEq&O8jP(ILXN}B&S0a zaEBtl4n=-J)RIU6QM1bWIFy)mmO3*YEG3JWdNwn4s}xgj_?nq2=kPj>>NKj;=;TkM zAyXZyaXVDwcBsZptQAs)to5{AZ;=LbBPdH2(e_fN?RqKNuJ{zwRu2D_wn|&2?bOrO zp@_aifpmug>CP8O#}$Ei1xr)!8;1`t_kh1-5r=PJ4zH5p@Y{DYhc&$hb+V9Uj`fIg zSUEiHIqXogK65Bq?|jjE-1q68&<9{6QQe`7 zi$k{!hh`f@dtqfrd(Zo)=bpuU0?v~~ynmZ{KP1Kby*FUqOBRXpUU{#)?*Y7bXtLwb z|HR=e(V^=->431b)Pc$d*p^!c^(5;XApm4ln6LQhYfd(N%pyR)*KN(o1D|9Okh~QY z0TlrifgUIVt_|yfHmnEGIlWK>p=^yJTzhMynFVSGkZc)M z1XTo8gr2Sl4sC)Q+5|bY38HvG=v%5oRs;->EH%i?2WcRS;;YW_!fpR>V=&w;s9iNw1q`7)ICK$n=-lMcxoM*;pmHD; zUMo&~xY;ZPogj;}VqVsYzo)d~o~>CcBr`}u2@NGQl;};NM5q-GUEUm;X*o30BHa*L zhq~c~W1X`Gn|6o>StJ}wvT*z*B^>K4%EBSJNGcpE94Z`rK{y<`@HzB~a_AN1&?}0B zL}*?LN##e_+dOuFF{?sB$RZVKV-@*hN=1gf&nhAbP^u!TBB~;NQAHd&&N=j%b4ACw zE>sPSavrlnQl3KPr41k_WRax&fFYNm5qX5QkPbJZ8)(&?B-)lMZK1+AF0=6I)o5B!5peNi|6|Nj2$zXp%$K zJck-w4h4K@@h18aF5U`(I%09lM2fY^9Wrz1Yb*COMg^W2$4SW2$4*L&tKYe&^^HpBMe&yU-{!(lj5O z9r*lE^8^HqEE2W*Sk#86MD6g!S=1yER7FiiO+`&bZF-AZjC2@yF?AZh(60u-)4 zD;wb;aujyOUWU4nMGE&gD_oXRxCNJFg_HDCRX9~RRXA0+8L4nN&M|YG4&>NpE96ep zD3ZI|+Xk44kUFwR?*74YH)l%jnqFkNlcZObJC!??JC(bcCU-eD)vbHsQQ>4`Fh z(w7yXUgIZ&%;(TOvPk>hW$kNDY2Uv#WbKnQTh%_*KGia~8= zD@s*rVIhQTUQ#gwQ8k7v62i~C5bpV9N(f)T|5YKWmE>L(LKQ+4LKVUuDTKMj;@Dwv z?5J&apjxFU`tWe%sYA>#Xd79rDh?RHs`x=l6(75TRZ&ukRTWhgRTWhgd%P;<4ohf< zCA9NPXi>00A+yTHI4l{ne4SYc@<61L|IB> ziY`TjX~wm=dSQ-BcpO}HSQ73Y)~=w#z%i7_++n5du+nzaapSE{NHP&b=nh}jsDN*2lL8Z4{7Ps!>Z_h(s^+;o*yl~t8hmDRp4t9g!XfE?%Y zt$r6kBruTJtg;DCoPPPWG2265$sh&A@Yl*S&h7Hg~|#H+@t#;V4u#`Z6b&0Dcq z(2C81)}og&4zRl2btS?5=%og;JH(bO65Q=ra8FMO?u3ynxRQpif~$h7f~$hte+4&h zMWKyW>?yQjPocG=xS@&kfm#JF!rg;W32k*_3aeFL)8W;3-#b(4J6{v4uO$9wBW1O{ovPY;XS1GTlG(zltXwf57ubF zHl}00R5pdwe}9`g^EbQ=S@cHt@Ehf+H`-+czfmdmGvbZ#I zU0>jL{ch@ASH9N2D~5EkHGdj_GjwJ6t_Xh+{bY!KT8pcR64fKdd#It`TB!tb1At9?BlDc{x zl8;hDa_9m)Boa3CSYU)h($<-^b!F|tv-V;-gLHH=JbrwZ+@ZQ9N=Im-VTU>#$je z%R2H5z0gVDlpJSxSqS4-lNBhjwXn%hmue1m`b+n@lc#M2GFyYAip|$;*(tJ=QKw9%Q!M&WSoEQ=sNYTwk`XK&{}G0CU41T2Hx`SdTv;gxeBf#i`;cSyKB>wyLP#e-6cyl zb=Oq7%c6F;MeT5l%DUt_8QIWv-goDn(`;^o-^e20UCF*%Gv&L37G&SaLQ#D;4Si>e zV@-%A9cA4gxX*>y!l0{y<$Nks{k5>r= zao^f)sCfnMC5v4A1iN^Vl#74IXXw~v*JX0FOs-n1)vgi3a5M~d!r3!Y;E=eXi!Bbr zEwUyo(k6(|;*OHgIJVy$+ajObZOnV{HCg2Em)YNQr2IYl8c%5)si?nuh`%k;WGwO@ zEV57F`_eU`@4fSH+Ns`r0mqX?&Y#H6Z%R4;%U`kcWy?YJd}io*NcPbA7P&_j$ww9m zFcxV%1O{8O9iSHHDEePclYiLq{>vD=}h`PHB<3Ws!17coBaFc=3Q)&w!C_Bn3>b2aHAXrbQm3MGhZNC z$D`-r_KFz+aF9jlS&q^3UGl+q7QUaMB)eq_r9KTMiv%=_j4A8Wu94&y1PZIRb0DUShYtKZU%%4y)bUCjl9C?vP#xM#D&!ifW8tOA^JT0PQ)FqgMlux z2!6XW{EkY&uk$5_pX`9D;Hlt+@Uu7+wKyrWI5DyQf)FksL?K=wg}dxHz#IdpkwsWM zh_QG;3X6u@Tj6qpN})>Ow=AbkGAY0bWU>I}g~V#kQTrW{btjnAR@1xD~0VpUQuMG9VWNgk%v)&tR1PIEB*P zHer;?7PF#MQ94B^wOC(UEO{&fPeQC9D$rFA*#l6d>*0M#NGu#j4Uub?Z0ugC9`&kQzH>#C@CRh*^~it9`h%45}E!}wi2h2J^P zV9AxObj7dYcbekYLVPU{Y;ix?;({{aS#Fe7`J6tW-t zn2{}~28wJ&_6$R|Mco67!Uxt@J^<$9^`D3Nv)eV8H-KWY2 zCO~6~02xjfM(S}*L5f}$F@jO}T)FEauaD*)4$z05nmr^|0;v`;C z$f1|6qIDHL^H(P#<(6`*A92f}%BMqZNQc^x#53VRz%x&@Ly^(F6G%oD5p8oO+HX=syBGbD3!=&C zsS-_z)=!D%P=wW?T&F|1P9mLfA|#z>-4m}>%xExX#``MxPGZ9sY6c9Bln7TT;l8pyT$s(qn%}m`Y z#nfy5!%UTPcx9?GRhc?vOm!&I?of@}p&B=_R!9}H*31 z=VMG;Is8}JDs7dvQ%_rmBKi&m(j5wBA5re4$U@*_QJ}L_F3is0cJ!dr~m)} literal 0 HcmV?d00001 diff --git a/ci/test/tx_cksum/in_mseg.pcap b/ci/test/tx_cksum/in_mseg.pcap new file mode 100644 index 0000000000000000000000000000000000000000..d552d6cc0a1849aa614459afd59c8bdddef3d0be GIT binary patch literal 1487874 zcmeFa3D_oAS*Ksw({Iwvj;ILe5k%SE^PH_p7Qp}-6lv@XI|va$#$|+ou(_dsWJCoW z1to}x3!(#v2q?mUvIwH0Kv+~ZQB)QeP!R;=e?RrsS6`=V$_tYK>GaR*I>$uM_ocgj z?^5?&&w0*8uYCS9PVP?a_WtKnpX$1k_?K7R{QPhIxeL$ke*3I$94C(pd_Ji=v%6h) z*01@pTc7^l@4)}<dRa`-9N>u!3_uG<_5+i>pT3-WpA91by9NFj$3YG|Pk zLx|fDV~7@F3NeRRLaZUS5c?2^VB28)j0H;}SPoW#)nF}HA8ZI|8&V93-mwnG4vMt6#5+c68ai?{=$9e zhcIlz5W`?$NMXofC?O0r3@r>?5WgZ^k}cO{%SG98RkmD~E!Snsh1qgtwp^Mm>&8pR zI=^^+^|*XYEc7erm(Z^v7cp`bBbPC99U~Vqav>vEGIA*+*D`W3BUdwWIV0CIazU9` z$Tf{z)W}thT-L~Sja=Bsm5p55$hD1J+{o39T;9m_ja=Z!1&&Z%9k6iyq0FeYD5kxYGgb+y~5fKFP>fKFP?&f*l8TXvk&2Ouk&5w(Hc~NCF;X!?F+wpyF+wqaW&2Bed~I{Fja1BE z-v0Xb7r4K|{UuH+Mk+=sMk+=qMkq!oMkq!oMkq!oMkq`yBo!kSBNZbRBNZbRBNZbR zBNZbQBNQVPBNQVPBNQVPBNQeUl8TXvk&2Ouk&2Ouk&2Ouk%|$D5sDFt5sDFt5sDFt z5egFvNySLTNX1CSNX1CSNX1CSNW}=n2*n7+2*n7+2*n7+2!)A-q++CEq++C!NF|X< zB9%lciBuAyBtl7qk_aUcN+Og*D2Y&*SV$^~R1&EqQc0weNF|X;@2!)A- zq>@M_kxC+!L@J3?5~(CoNu-hpB@s#@ltd_rP!gdeLP>@M_kxC+!L@J3? z5~(CoNraLJB@s#@ltd_rP!gdeLSbSdsU%WKq>@M_kxC+!L@J3?5~(CYNraLJB@s#@ zluRg@P%@z~@!3l%nN%{VWKzkbl1U|#N+y*|Dw$9+p=3hIgpvs*6G|qOOejqLQO-hA z$)u7=C6h`fl}sv`R5Gb#Ldk@Zd8=eX$%K*#B@;>}6ebq>SA%~y_?N@@?Z86+e()~{ zQpx@eK`5C}GNEKb$%K*#B@;>}6ebpuN+y*|Dw$L=sbo^gq>@P`lS(F(OemR9GNEKb z$%K*#B@+r03rQuDN+y*|Dw$L=sbo^gq>@P`6G|qOOemR9GNEKb$%K*#g^7ivl1U|# zN+y*|Dw$LYsT5Kvq*4f_5K1AGLMVk$3ZWE2DTKnrLQ*NDQb?tcN+Fd(Duq-EsT5Kv zgi;8l5K1AGLMVk$3ZWE2VPYYv6jCXqQb?tcN+Fd(Duq-EsT4vfgi;8l5K1AG!ds;f zN+A>`7LrOKl|m|oR0^pSQYoZTNTrZUA(TQWg-{Bi6hbM4QV69G3KI)SrI1P?l|m|o zR0^pSQYoZTNTm=;A(TQWg-{Bi6hbM4QV4~Kg``qQrI1P?l|m|oR0^pSQYoZT2&E88 zA(TQWg-{Bi6hf(l!o+&_q*6(xl1e3&N-C99DydXLsf1Dqr4mXdlu9U-P%5D? zv5-_MsZ>&_q*6(xl1e3&N-C99Dxp+Dsf1Dqr4mXdlu9U-P?%UqDwR|!sZ>&_q*6(x zl1e3&N-C95Dxp+Dsf1Dqr4mXdlu9T}EF_gmDwR|!sZ>&_q*6(xl1e3&N+^|3Dxp+D zsf1Dqr4mXd6ebpuN+p#_DwR|!sZ>&_q*6(xl1e3%N+^|3Dxp+Dsf1Dqr4kAg3rVGt zN+p#_DwR|!sZ>&_q|!*G5lSPJMktL?8lf~oX@t@Ug^7iv(nzI|N+Xppouh@w$)KeZ21Dbsw+$_)Byj-=M1oT{Y^26&;}{W^seRyTgC>ZyoC1#ofJ+ z#U1_|raSzwc8C8~%5x~cNqfItclf%)*B!p@@O6i;@*7G=bce4yeBI%z{DyKHmET+@ zH2dMp0CwD5_Btb<{oip3>j^K2H` zyPtKtuPPl?+jE;wp*R(IsX+)5>es!$ zRKHZeRKIrGYNxGs+G?lmWw_J!lHB_@wm&zvKR32NAE5`%u^bRKui+uUp;w*Moql#V zjs*yObO!0#SjeKvW>eE4?l}j$>wyP87Km`w2@&42h6vAo;D`wQ z-ftHo2oZz`LIfd#5J89_L=Ykf5rhcG10pm|P-vW>&^SS%ae~4T4%#{f88)w_(CBp! zI<>nAtl(q83)h`^;mvEjaDE(lVc7fC;sx=7ctN}%UJx&c7sLzV1@VG-;dtbQ#))f< z53t68YezVu_ZW5vBRo9#;hXL&0S6xoc)0Nd53gN=hfm&mga;dS2O}#eD0( zhZZANT>q|TbmxH-J{DMU`w1(ayM`5SyY3Dv=9Afk6~YQ(g|I?cA*>Kq2rGmY!U|!< z6&5R6AK?q?p!(#9F=N0UV(cDWyzt=%yZgZxJ{F8|WMYhGtTD#p{$|G*I-fb{z&0J& z23zRBHsf>0#28`>F@_jJj3LGlW1J9-(RxmX?x~jIP8i;pkznSHuz4M|Qdc_v)b2Zh z4Ic~OxbFlUKeYxN*L~>@IOY@R^iV?&HS|zJ4>bfF0uBL(fJ49`;1F<}aKO=e{>nW? zRsG=bG2_8LeC+7Mp1kQE4SD!j(8qU7^zpbg`uL0Q+tJ5-?w#mE^db5XeTY6pAEFP@ zhv-A}A^JF>>7xy_tQ{!58q7&#MunqDBy9eW2O;17q?5Y;1%~)opva>p6nWGdirnoM zI~17@+Y^ciMT8V5%0@v(rEpPC@$zH5;3`_Ux!wFEQsY96R{jwBbFyV zX-6#cQHCNG5sQdL#3Eu5v4~hiEFu;Wi-<+U@?RyEew!ed%qYHHW#)^0dfDO2!+!Iu z?xmoMj|ILwcfyz3ui?uRZaU(N%||E-UxY8h7vYQWMff6o5xxjtgfGGu;mf7LmwsEp zn8xTqnSItc3T49Pk9idH!FQkB{Q(@~W5F^nnOJ66W0{9PW5+V{nT=u@v5Z(oEF+c? z%ZO#fGGZCAj95l2b6K%Wza2s!DKg`j_?~kdVVjvfX0{o@=AZ9+pnD5Y<6{9fubsf= z#%o~nO}E>D&3sa%z(!yruo2h@Yy>s}8-b0$Mqneb5!igDz^3QW$ey!Xdb%b9IWvdM zkP|kq=kd**p3rrF1>yKuP|lkt%DL_uca$@qPbta~<%n`bIieg- zKmR!K&uMG?^VSFK_-8(gQ~V?T5&wvP#6RL6@sIdN{3HGm|A>D+r~K1X@YWj@a6G4h zg=SbeiiN`FPpE&p#z#)=HU~i(#sY>uv4f#+{pcDDeeGB6U}!%5Q!peL5)2831Ve%$ z!H{4`FeDff3<-uVH!#%0LcP)3&8U;Ju}*G=ml=*mGP?24oYj3EROI7WGCFu*BBKwi zk-Nx~#yk}ye_Buo+} z9T%9?C#voeRd*=|FT0zm<``!B^%ouJBD~~d!AxOdrZ=oH(+xhfW2X6dRWXy8Nz5c> z5;KXJ#7tr)F_V}{%p_(y{+OvxROcnC^HOtunh9s0pTg!1JaoD+p48n6c=E9TsOQfI$QN)+G zz|=nN90gN5n)>;VoY8$9B;{j4Q@5OG>bKTt%9}&b5S%JO(G=v$wlXgJ5Qc5wW!S$8#${ta->z{NGm25((OIc z>^;)#J<|Cw()lpb`7qM?BhvXJ()lB@*)6i!Ewb4y@-#9+)R7lLB&^7@N+uTaurk8k z5$=v~cjRRgdD%o>HZij>^9wWUGV3zyGV3zyGV3zy3SSnnu&}VOu&}VOu&}VOu&}VO zu(0wAE9)xjD(foiD(foiD(h;`FRCnTENm=nEaYw^HWoG(HWoG(Hhy7aUC+9nb=`Dn=?sDn=?sDn=+qC`KqoC`KqoC`KqoC`>FQ6(bcR6(bcR6(bcR6(bcR6(bZQ z6eAQP6eAQP6eAQP6ebpuijj(uijj(uijj(uijj(uiV=ztiV=ztiV=ztiV=zt3KI)S z#Yn|S#Yn|S#Yn|S#Yn|S#R$a+#R$a+#R$a+#R$a+g^7ivVx(fEVx(fCDbbW@N;D;! z5>1JwL{nEBn(Da$>$w5zxdH3B0Xv$0X1W?5RNd^44|R6~t9&fb>JAfH{n{E@z2)b3 zXf+>Rs|Qs=E1{LpN@yjt5?TqZgjPZ;p_S0;ii=h~Zo9{A_qc5^YbK!?v%=<$6a+s0 z{-<}}0%Q4DaMoQX&U)G!XT5T}_1)(l>>dJe`B*^L{U_-9@iplBhsW=rE30mhdNtJzQX@!J zp6UiwH%NgY)tp8(r%}ymygQ7yhf&OF6muHIoW=*SQOs!+a~j2*Of00D)2QY&syU5n z&O|k5qM9>N&6%j?OjL6wia8U-oQY!2L@{Tgm@`q#nJDH=6mv4MkZR6EHD{ulGf~Z% zsOC&mb0(@e6V;rFYR*J4XQG%hQOubr=1dfGCW<)|#hi&^P9_%Os}i4|iE7S7HD{ul zGf~Z%sOC&mb0(@e6SV}1T7pC^L86u*A-n`#f-XUqpi9st=n{0Dc+l0eaieGB22*kZ zn-S?~0(%pWy*3{@wfjEk%g2Ji9y}4)53Ui|Q}3}Ou=#XgJ@(RLFA15W^$qYQ3@oC1h z%|)=;i+=Hx?kRAYj|H3k$i!w3T4S?Uet5@b^C`$;GqIW2Ol&4L6Pt<6#AaeMv6-0Ld%8@+X){XAoEAnxC0reaQ1hmaqwGVA!^JX5eee{th zcfSd-`B+ffFHh8Vr!{Ij?WsFzn-6>zwTaq9ZK5_&o2X6HCTbJ4iP}VMqBc?6f0^2P z4sq={#I@%TR|sxKtfL5Shj8Ef1L6;l6qe;U4*t5#i`)remPh&QZre z>lkPq1Fd$BA=6|t(_}N#WHZxbGt*=<(_=H!V>8oZGt*-;(_=H!V>8oZGt*;}iG?)T z%rx1|G}+8F*~~QA%rx1|G}+8F*~~QA%=FmI^w`Yw*vxdv%IwX~YyijpQdSJG1MXiG}Pl z$m}!7>@&#JvSeymGPNujMVe8h8AY1;vdw(iX1;7QC#GjkOwXK{o`rBiI3b)6P6#K2 zyHp6*^Q}U+cuFbxS~mF}ijOX6MBe}2p-jljNhUI)Lc<$vB&y_WvyTc84JQw$V zzaGiyk(?gM>5-fs$?1_Ca3h|}je{q01N<7`*8sl;_%*Ig5BT+fA8(b+Nfw!tEHWoq zWKOckoMe$X$s%)-Mdl=n%R7MYVQGACJNPO>PRWKlTDqHvN$ z;UtT~Nfw2ZED9%C6i%`zKz0GL3!yNv5Xvr4c7d`BlwF|g0%aE{yFl3m$}T{50kR8_ zU4ZNYWEUX20NDk|W?~_fU7+j&Wfv&BK-mS#E>L!XvI~@5fb7D3NCC17kX?Z60%R8; zy8zitEQGQPlwF|g0%aE{yFl3m$}UiLfwBvbU4ZNYWEUX20NDk|EFQl}aj=R4S=dQmLd;Nu`pCcuqVgp1bUL zuIB@+=L4+g1FYu*48EKBYv#L+!`+uY=gjWU0XrWHxcj3C?ryjSch~*h9o)^wX$$TI zcY-^?o#0MzC%DtYI>DXbPH-o<6Wo2~;I79`_1LK%I|b;?EH;B)*t`|e`;F&z-QPla zJ{F|+XA|jt;Tq|kbe|pR&4+l4^hA0hJ&~SBPoyW(6X}WcM0z4Uk)BBJvqO44bqGCm z2t7AoP~XgD`_#9?zEeMZM)xlupN|Fh{mq1Zht{y~qLX&mH=i6X>=X70`-FYMK4G7* zPuM5y6ZQ%Fgnh!k&p!6`RG;@$pZ8RsgMc%m9fg2927K~YUa33jP&bYR1OD^GfG4dn z;I(eFW5D^maxtJ7Pz)#r6a$I@#eiZ!F`yVw3@8Q^1AcB9uxAHy&kp3C9mo!ZGq7C@ z2;W96Xu8`O-64R`$FV@@oY-}@KK+6JvIc}t{<|Fz&L^Y`2nB=!LII(GP(Uak6c7ps z1%v`Z0il5Kas-4uJ*j$nQuU4n>ETV>;~VSlrXGxTj+=q&UOeOp0Oi zXQ1Mh-+Ov@Lom_D0u{etLd8E{L&bM~?+z8`)8K`QLPepXP*JETR1_)-6@`jIMWLcl zQK)#FpkmMAHa&;i^c-#jHqKzTkBvKSeBF^lU4a>WEV%K@CT{%WHEw*uc{^^L&!87K ziW|j^;zn_!xKZ3FZWK3)8^w*{MseeD$c;URq7EF2I&dZ`3^_yIQ4AS2f6l|l7yj7k z-B$yUJ{Ay}Cy4yrHHf_OtwxCC$kKBUUy#o`=dd7B5GjZhL<%AWk%CA;q##ldDTowA z3L=klh#a;COeRKVcs=Vr^vzH?66O7_b*Q@&H0fhOl((6P@;BFr@+ZD(N0d=S>79tY z5s`>eL@A;aQHm%?EMk+=sMk*$}6kZB1 zg_puh;id3WcqzQRBID)2ZSlZu@xW~{s5#T%jGAHNtoh>~J=i@6&h)Wh&3jI)`II%* z{H}-ZSTl(=#hPMGv8GrvApjBrK&&a&6l;n##hPMGv8Gt_L}AT=M=%4AUy0YSb*oZPT={YYv6getM0%v3p@p$0#AXbz*FGaLZ%)dBR(_Y zGb27TUq+cPqs*64rXCH#wK08A{TARto@kf{gA)B|Md0W$RfnRsdWaJO{q(2fphEzUzU|nG%nJ&>e;T z^npX&4?>_m78Lrhi9)|`jY4nr=pBU?QK%?X6ewEN>2f%$S_7?WJY0BMq~607E(h{sUfJ;5L9XiDm4U^ z=L(hQ3YF&yl{-2n7II5hxuvVr5L9XiDm4U^8iGmBA*qm5NGc?~IFb&0ehqwn4Saq% zpUwz*aXx+B?GANMgH3%b`1FYrpMJ|4pWgUMJ3ejVQ}LleZulG0pdk$!(x4#?8q%O44VYNS&V$CzgT~H- z#?FJr&V$CzgT~H-#s^qq=RsrVL8B94qZ47H6JetfVWSaYqY+`_tDcF4bRul@hivqR zZ1jh0+zB_@LpIt&Hrhir+Cw(+srd9i;nM-ZHXzsr-YT%_jFU5~hRr*9nEL9soZkH! zpz31*t3Nxz>fP30_0>~-Z!cICtO`~ItAbU*s$f;HDp(b)>R~E;Dp(b)CO|a{RzFQx z9q>8>UT47T0IoAy?t|-&Tp#z$L)~vdtv(jy`l}PUe%%_mKH|>1$E<_MRpcsi6}gIB zMXn-Ok*mm6d~p6b^**H{mbB z--E&m>MAI!psIqR3UB5}OGP>=(ooSGIns`iPKz{Gq(LJ+7l%M01$7jZQSd^-fKWq0 z2?dX$fqLI`!JZ4YT=-rOc5AQ?gVL5j*#uP+UNk|8L-a}yiZ>|BpyGmB3d$rXfS~pv zaJx>S1a%UWNl+z0k%TZ-7%Pkw#tLJFvBFqktT0v>D~uJ!3S)O|elIn~4)EUq01gn~ z00JIyf}5~&#@dl-uYcVub#H)YeJq&vl@rs(HKx7rlpWK?yfruT5=z)kIhzNp+A&4k~h$Dzd0wxynI*GhaBCnIk z>m=d|BC;T23nIE8;tL|eAYu$6${^wlBGMot4Ih&d3`ifP5PVp=h+m{v?H zrWMnQX~nc++RqTv4zS7qs|>ITd^;oP%(r3lZXVix^_$P=-U;0LSOD&yPJsKRYk>Q& zH`xKM32^n$R)8zO72pbR1-JrS0j>a7fGfZi;0ka*3xGTDFmB*s9FrF*Q;B=*)7-H6 zOKA5QPwu)8K)OB_wEK>Uc0X^8cHgr*aEBsal_=Lh7dM)?>2WT#;E`hRNHut*96VAF z9w{e?D3{0!G4gVW6op5s!Xstjk-G3mVK@^DsSS_ZEk=sNBh}%N^6*G~c%(o)QXw8G z5s%b}N4P3dB_1ggkJO1r3dJLp;t}h>#6qHoWD$9`9(lMPdAc5XydHVJ9(lkXdB7fd z!XA0V9(l%YJYzSWu^Z3Wjc4q}Gj=8_=6T3&JY+W>vKtTCjfd<;(Y^7I-FV1uJY+YX zu^Z3Wjc4q}Gj{Xhd#}D9%kOy?8V}jcEAYJp-)r!_2;ZyldB|=&WH%nN8xPryXY9r^ zcHPg#%_2k!&@2Niiw51uZ_30@sQnk$ZkAjHy*MZ57~`}?8ZZO;~Bg0jNK^D zH|q0^P>fKFP>fKRSV$^HDn=?sDn=?sDn=?sDn=?sC`KqoC`KqoC`KqoC`KquEF={p z6%*}>c163QUD2*+SF|hI743?4KijlBa9ccZTRd=E4EfG%y8oPahrU1Y^fSAUg1tT# z===T&eK%|9`+K`4Q`|8NeTBYIR)n&m&{yaSBO@dXAz=s!Lr54t!tfEs*P-Df3?E_m z2*XDhK7xsb2nj<-7(&7j5(y!Z5E2O?kq{CIA(8MA2_KR05eXlW@DT|gk?;`-AHl>z zghWC}B!omlNF; z-2uEkfcFRR1_20(gpf%1h=h+w_z0n|&{yay^cDIFeTBY4U!m{k5`71r2n{?D8h9cE z2G5*&6b6UQdw2wV?{7P)JMBu{I2IiIZ#xctz$e!@_$j}?D|jyAU==*8;8~A=^$1uT zEDjb2i-X0%;$U&GI9MEfdEwxJ%E*Dr$brhpfoFDb@yx9=7mrZ<(jPj|T@@(yaV!*H z>C_2||7i`1A9nv86jwp9pjc2WC>9h8iUq}jVnMN>SWqk|78DDLFRxHMP{=w^$U3lH z5I~+8b_V3I`70h9pa0D#cl3fD$AXZrG7<7$tr7A!y?;l@O@u5$79op}MaUv#5wZwb zge*c9A&Zbj$Rgz9fsh9p>I^j08R(}2Ezdl=Ps=-8{*k9{x*LPbJ{Guqoe7uUvWCl# zJhH>(Ubrk=7A^~yh0DTa;j(a9xGY>2E(@21%fjX34VMSn&vMQ!<=FI}g`H=(J(`{W znn%rF@WZEc1KjLm!Ol0D*!d6E*!c$6+p+T?b{0E}oyE>#XR))`S?nx!7CVcb#m-`9 zvGZ}w&I3)Z2ReKW>}H0eXV5*0qr=8Q`qA$=tGhiA?PCGb{RGl4T?6T#{?G{090sG7 z*J^pKme)G%iB{H}!DXDmrIy#Uzos=d4Huc{jZLSCM5l>Fr-{T{C9$a{v8g7psU}e} zlqeZWlnf;tXhMM|6lmgOJn=D}_!v*@yJ2D>S~T&Ip7_{JFk9jbuEZH!i8HtoXK*FX z;7T09l{kVcaRgVwZX`lUgp%k(p6El)#6sj>q7iwb5qaX4C((#J(TF_Jh&<7VJkf|e z(T67*;(gSy3 z12gJ{qH0mKs9IDls=mTf^#D{4eAf>=)_|~Q2%ZT$jEH^FaIm{Cc z*xxuI_Ajg<_6L4)hu9{>7Gevrh1f!DA+``(h%LkxVhgc_*g|X}_6dR51C@gVm4gG7 zgW&cI#QV4%Hh;q-?ThYmdiNc$wvPpGf78U*cI6cx%;%)J^cw4+J-WG3*x5eAy zZSl5vTf8ma7H^+$ygjf}Z(yh1z)n4f@0pH|h3~t+_E7gIz}v?HzQ28f@Bg(1->2PY z_pmz)z6IZcZ^5_VTktLT7JLi71>b^i!MEUB@O@&#_duVVf!pGNJ~@E+Ov*=rc-Z_c zkHH^u%9-8cpl}}x5`W}G;*VY<@i)9~N8&{!E)o}si^N6ZB5{$pNL(Z?5*LY!#6{vF zagq2Qi4SzF9q3p)(6M%)VJ&1nlk-gGo4*6gKX}`dx}O5ceJrs2hbAol&NVFmrSo@K zUWMhta$&i!Tv#qF7nTdlh2_F>VY#qeSS~CVmak#?05lGqBQtQ042V8c^o;0XWb{wH z=0Nuh_}s^W(SLGc^!u$b`VW3&$LLLrE=Culi_yjCVstUO7+s7mMi--t(Z%RubTRs; z!{`HVm4UNf2hMth*Jq;M=k>689}m?ZeVtRf=K||K7Qp`V6R@AR2JA;J+yQ$pU>C3p z*ahqYb^*J9UBE727qAQ11?&QL0lR?xzY6TbKu!EWP5h7@-ehQAhNpF}1>=1zQ2uu&l>f>#lz-zHBg%6Ff9~N6@_FYR4l!6r zA%_xbXrT{7h}#fjh!$cBF^5<}tRc1#`w)j?hyF=Pvw|6~rigj_>zA>=;fA(U+>F%;vzA{72eC6pRU3&p=? zL*0fNL$y#-s5w;r#5L3wY9HzlI4PbRGCqsFbv{k|=nlZ{0}2|@B3w@X# zVE1O2-p7LLUo&z2P1d;n-|x5M`Y5g!*Nf}L_2PPQy|`XnFRmBYi|fVp;(Bqtxc)Q4 z^}s#P%ePyem~XeqF@MJJeazo{fQR=lx%U~}djNbN3&_7{g8b{PLH@lSxr2NYjysZ z{J4S`y!Rok4|AY(LZEd5jT6Sr79I-2D>A$wgB}K>sc}(58-p=48X0WalQA|rV4b%* z@LmTrJ78j=_dD>02j203mIslZ2a%=+k*)`kwg-{62a&!9k;Vs+&Igg!2a(&@H z?gx?f2TUxa145()LZk;mqzOW#3qqs~LZlBuq!B`-6GEgFLZlZ$q!~h_8$zTVLZlr+ zq#r`0Ap#Q%>4_lx7yb+Xh5y2T;lJ=-_%Hky{tN$w|HA*zE&hX>klNN^a!cPl*8wxH z@9BWe-%|>3yIY>p9k_Bgj)e>O_|64<%g5GSz+b+3R|-(%0#phh7a$iP7a$iP7a$iP z7a$iP7a$iP7a$iP7jXG<0T=-u+-`Zej1`#4ea{Mbk${VVTYUN1-8B&dK8_UwS3YwR z10P%y10TyfF;FE2BnBh~BnBh~BnBh~BnBh~BnBh~BnBh~BnBh~F0Wz$JK&83yzG2S z-8r6M2Km_&Z2kdDaHDsf)&&fKkA)?;#$*ZJwPp$4{>hysXtD&d1hNFO1hNFO1hNFO z1hNFO1hNFO1hNFO1hNFji6uZ4uuFW){&al7jP$cF@WKLD2G@P_p>B&V@Uc(^Uot6! zH?Apzn}7aJ8T3*HQU+26QU+26QU+26QU+26QU+26QU+26QU+26$DK038+gM4Z)&%t zs~rYmCj5Pau=z*)!P9=?%C$=w|g2|gAw z;TDrj`0X{B@b^DI$^=^fPbV$tCH{-Ffu(7bQu(7bQu(7c53mfae*4VR-6Fws+eMH)OMw)p< zqz)4c@n?}2edNU%IhZ1HGDYNQipbd%k;5q>hf_pOr-&R+5r-id=TjKxQyAw{80S+M z=Tk5hMrAZc&tM@vMU9rCMiUXEhlsIL$k-@k>=QByfQFQ6(bcR6(bcR z6(bcR6(bcR6(bZQ6eAQP6eAQP6eAQP6ebpuijj&*CP*eoCP*eoCP*eoCP*eoCP*eo zCS1W}0!9IT+JZ57g?;wl^9pQpaI^5>ho05l6|3N5VHVDv%)Lgf>mEmBrY~Vr*qGwz3#T&e+Lf>|`sPPL@OsSfT_hp=lGAHX&&fAD&DsWGhQz zD@$T4OJXZaVk=8xD@$T4OJXZaVk=8xCre@{OJXOBoP(T$oP(T$oP(T$oP(T$oP(Uh z3FsUU4{TE3a^uY8O&hq&KUP6(K8%3)_J^I>eJ}FC$3j4S*CZhRzcm5zCC}Rlh`jg9 zB_JdqBp@UpBp?#4G!m^e60J0tFctLDNc7T3^wLQ5(n$2uNc7T3^wLQ5(n$2uNVL*O zw9-hl(nz$@NVL*Ow9-hl(nz$@U}7P?G!nfu61_AMtefbik?5t7=%tb9rIF~Rk!Yon zXr+;8rIDyaNi;Y~gp#mA2`j|JLbOmq3njEr;#M)Cg%VmQp@kA(7zr(uutEtdl(0ex zE0nN887q{rLK!Q>lo**&7?mLbAps!)Aps!)Aps!)Aps!)ApwDf@J0_jyhBCIw&190 zh_Lx*Y{bnUd~)|AmC6^({G=4`*ho<**?AY z;Wj$$(`cVQ`?T4o$^L)hGXlN!X{}FZeWVgu`1OkP)sOVmCul}yxPEf`4A(Sz$5yUt^bRTBAcYGXeGd|sHhN)?*Y$99qYEfKFP>fKRcr}uW zk&2Ouk&2Ouk&2Ouk&2Ou5sDFt5sDFt5sDFt5sDEC6AS&-?JsYCeftaCU*Y}|ClwgkpqZgkpqZgkpqZgkprk#6nUr zQZZ7oOXxFrtMeN#gM&%=wBWPiGB}u&4;(&vE`x(f$AdvGi*rHvt7JmSgpzryWZo*7 zw@T)%l6k9S-YQHi^lt|LYVhv{|8f|=9a!k!54=^fe?j;+1aFnhTP5>W$-GrEZ{0oGH;d4TP5>W$-GrEZk%hcf3U8Id z2Uy_)tndL=_y8+>fE7N#3ZWE2DTGo8r4ULXltL(lP?%UqDuq-EsT5Kvq*6$wkV+wy zLMnw&3ZWE2DTGo8r4UNt^Q#aF6AMYDkV^3{ZvW=?uWtYD_Al@8+nZnL-{1ZP?%&{q zQV69GN+Fa&C`|lDQYoZTNTrZUA(cWZg;WY3V1*B`LMVk$3ZWE2DTGo8r4ULX6ebpu zN+Fd(Duq-EsT5Kvq*6$wkV+wxLMVk$3ZWE2DTGo8r4R~t8Qf-&N+p#_DwR|!sZ>&_ zq*6(xl1e3%N+^|3Dxp+Dsf1Dqr4kAg3rVGtN+p#_DwR|!sZ>&_q*6(x5=teMN+^|3 zDxp+Dsf1Dqg^7ivQc0zfN+p#_DwR|!sZ>&_q*4i`5=teMN+^|3Dxp+Dsf5DBLf$Kt zR4S=dQmKAp<#$$oYvuP=eskq_SL5v!>jvyN+T2|7LrOM zl}0L!R2r!?QfZ{pNTrcVBa}ucjZhk)G(u^F(g>vy3KI)SrIAV_l}0L!R2r!?QfZ{p zNTm@Y25=?SGLl%7y}Lg@*mCzPI0m{>?EJ*o7h(vwP0Dm|(6q|%d0 zPbxj3^n}tAN>3;~q4b2(6G~4gOe`doo>Y2L=}Dz0m7Y|3Qt3&hCzYO1dP3<5r6-i0 zP3_1sr01MlS)r0J)!i3(i2KgC_SO{gwhjA=t-p~ zm7Y|3Qt3&hCzYO5dQ$00r6-l1P3_1sr01MlS)r2 zJ*o7h(vwP0C_SMJgfbAyKqv#D41_We3KK*_Dg&tuq%x4oKq>>N45TuU%0Maup$vpF z5XwL(1ECCrG7!o@C`>FQm4Q?SQW;2PAeDhs22vSFWgwM-PzFL72xTCYflvlQ83<({ z6ebpu%0MausSKntkjg+R1E~z8GLXtZCE-p#Nh^Apht;LV`^G zBV@?rKSGL3{v+fFD*JzgB#-dFX7zBC+Ss{`XB|A){TW`v$HHy=;p8^He9dhf{)WkI zM7a(B@ZclCKM{DZ^ET!U%6pG@6|ZrBU+^lI+lYKzL_RJe9~Y62i^#`CLKxea%l7zRGV6QAIT5Aek2cjDta@#&rT@J@VoCqB9ppWKPhZMhAu4PUMQ z*_o$u8wdfuDCIWba_(^CHsm(AwUpbC+mPFk+mPFk+c-|#27<$F2%d_`n`-dq-ZK&V zs$=tS6mk6EJWpiH;-xJc$n5`}Hc~km!hr(TEt0h|!1`jfl~R z7mavPi4Ft?@>4||)ND`{f($flAd`wXc&jAdDv7sB;;oW!f(a*>aDoXZ$fP0;o)M^s zLq#0CRaC@5p#qQ9RK%eo4i#~zh(kpjD&p|UhZCuY;~3GwzmCNvu%|nZSxW3X54Ns+ z6zlPwzj0Fc&lnFM3+wUT$$DIA&3e4|6_fQyvK}h$(AE`gU7>Q)sGKw^CymNUqjJ)y zoHX7YM)i|X{$$iY83jfWL59qQh3O!tmYTY=CQ z)2#{R)GkA2Td^-gyqoFAC<%G^i%;tg;zN8ae8{JEKIH!Yy5>WkfAz_S6!{SO5cv@K z5cv@K5IhHcSTcQBGJRMweOTl})XkL6W$I?iz7_7q)Xh}gOx4X)-AvWZRNYKz0_lw) zX;SK)BE40l_loppk=`wm?coFMBAL|9^#9S#^b+_Gq=+{O@urhoc9^>!ndQY%dSvtO z$dSJ&UH1iu5g*6Ok*l0N$&tTXlOt`fOGuR*ksOg6ksOg6ksOg6p|CmA{yo$ForwY& z`oHU_7s(N7b|pu2)QgUK(NQln^`MSlN4)_4I_gDoWP6E@db#v+1ViHI$@rlhTTbP0 zS2DYd*_Di@~$zEhD3+r>QG$Hzty3*IusW{LaBtxNK{60DGtTG%uNZN#Cwr& zewI6v*=o$r#CP9(oC1@7xbW02qfC4(w8;%7ZSwjxZF12ACvDP8n@F2Tn@F2Tn@F2T zn@F2Tn{ZIhK%bq;f#K37C=(j1NSg#{6KNA#MM|4Un{ZCS05od%%{HJYE*}LZ``QF) zGXA-^mE#|6)F`v%m^I4iP`>FAhq~L~PkbyKN}3$X@2xqM-@ftWPzE^^ITSe*ITSe* zITSe*ITSe*Ih4v?@yb^5%1&`Pls+^YZ1^K*l4hK#M z4V;A#cm@%WufZ=_perDUBydIoecx&O&H)LWkHGN=oQ}ZR2)w(f>k8C$vH#6DQj`-# z{XkL95e;b@*dt2+tduxwf|E1@^}&fFHTyssWP!}e>@Vj($>wtQmRH|r&N%}i(}Hdr z{HQ_W?v5SToH-ackdu!y&JJS7Ax(2>nd@KreCP9(?@vtvJD;fLp_a#$6!*pGC@v(I zk$vlpzflW~Z$`E&ZUcv*auzB_p>h%`f>A<6LPbJFLPbJFLPbJFLPbJFLWNsMZe|;I zvyIzX2^H>~B~*F|6$uquy{af>dt3??tco{E!3HpmjZ*gY$G%_Ld=k6z#MhkC-4nCo zV_{eBIN6n7S+gq_eE(!uqU?(7itLK)itLK)itLK)itLK)itLK)3a5xPP7!IGBGNcT zM0Ta}BvxB~RIVbsf}FS_*cB9uH&>zJf;tw;NTxEAeb3^BEUsF*R~+oV70u#fp<3=S zsg?`ZRLe)dZBi|^_g_#UOYk$QIHJmH--4e{mFZAT5g7aE{17XSsUjFZc-psZ`7ujg za729%M=W{kU_xVfc&r<1fLP~MK$HP7vCu1lycEc5fxH-qYM@9tP~_t)QVV8)n!r!(|%pD#L~uGR%-+h72=gm?6W=x5oM2IKE(*Sm@j1e1Dv8knx1*Rc>AX4o*ph8Z@@uwjM`Gi;b) z!wef{$S^~O88XcH);GTOjZlnGj8K?ZNGe7uMk+=sMk+=sMk+=sMk+=qMkq!oMkq!o zMkq!oMkq`yBo!kSBNdZ(k#~`Ik#~`Ik#~`Ik#~`I!L!J_a9ccZTRd=EJaAhq@1hMv zP}dc##N`<8!oNbBTj$cPgXs}UBxX;tR{^v66cXl9cRZ#00R)VXg@k$NBw@aPO~U-> z+b0Q=C1JGBNX0LbFp@BmFp@BmFp@BmFp@BmFp@BmFqHjE!m#gwf*+n351{z~nXCBa z1d}kR7#{m?sSj`+GaHjV9pl@P+{j$(n-6wR#K!no7@5aRM&^I68JX|7!(?QNjEszo zjEszojEszojEszojEszojEszojEszojLg7ov5bt@!MKy~wijFPdvRiw#q1jy{>lIL zzGq|0rVSS~vp|^z&8EA>`QQ4S+n?F}ENaHbLeD&5(lZZO(=&&!H|d!wJtI9MJtI9M zJtI9MJtI9MJtI9MJtI9MJtI9MJtIAXilI@2^vqZ=bBvzB(a=<5OIHmSHM2^YMU9uz zoP?`+#P*c#SMfAH7Ov*0ldHM=nyY#DKTWQt$<@f!$koWz$koWz$koWz$koWz$koWz z$koWz$koWz$kll5%(S^_)zu(ul3N>ZN<$M=S2wd+*;hBCxcU8u9qe9&wDGYJH_w{H z&DXDqo9DfJ5;wiXjl_+_jl_+_jl_+_jl_+_jl_+_jl_+_jl_+_jl_+_%_R^w*c(?p zoL}P#XLc<63dhTDPNw|ku@617dnE?P$HL;gaI!dGwPtZ1a{gp-23Z_g99bM$99bM$ z99bM$99bM$99bM$99bM$99bM$99f)8WpPkB-us4gh-2g+;?e5N&Sl@~Y)(PxJpKDm z>HZX*<71(8UNI>hTT?oB{KlQq;TX2*OmLmqtuwoIW;bew^Q(1cH)nWrxHkvZ8N$XSk<0H)Y1WSB?B~DULoTQ#ONj-6pdg37U z#6jwbgVYlTsV5FnPaLG4I7pp|g`AC-Xvm)E`IXpCny5=mi0wowiBu9NsV5FnPaLG4 zI7mHlkb2@E^~6EyiG$P=2dRg|N$QD{)DtJECr(mNoTQ%E!kcKYmgujRXs?zyNj-6p zdg37U#6jwbgVd#Tq;#Zoq;#Zoq;#Zoq;#Zoq;#Zoq;xJ*rGwa^VNL{!DB0P!FZ*)G zOLpAv^lv@by#u%7W8ruHWb!-LU-LU(`=^uNiSj!t*-^=kN_JGTBflfRBflfRBflfR zBflfRBflfRBflfR^BM9xNFJK(L>_xHjZJoVcs&){9Ho0Ury_eEdd&mf`w=}p7P9B< zlkB9>Zed0 zRHzOrR0kERg9_^k>niIi>niIi>niJNp}eTFu(GhSu(GhR5U-FL3mXd?3md<%v97VM zv97TW1DAT%5lSKyCKgg+l@Qd4601asRpPchQDT)Su}YL!B}%Lk6;_D~t3-uWqQWXs zVU?(`%2Zfoo+dCEBQY|gFe*c`N3ut}F>7vpEg(bIseH+Wi~W$H&6_yl*l;XRVo^7e00}KS}0CWj-qNQJIg*d{pKm z^CR;k^CR;k^CR;k^CR;k^CR;k^Yb||KjM4^G`p}vay2qi(2^BdZ zIUzYAIUzYAIUzYAIUzYAIUzYAIUzYAIUzYAIUzZr%a;>E40+EWj=&l9(Clya^$`m)$rd%)BH1F@BH1F@BH1F@BH1F@BH1F@BH1F@BH1F@BH1F@qT|IDp^Usa5vS05 zbD~-7?0X~cPUPz7>+XJ5*Q1SmEY#6gOzP-$YwGAlUpuLzUg}8dNa{%HNa{%HNa{%H zNa{%HNa{%HNa{%HNa{%HNb2bLQ%CqCZ)jxR(1-@tZjxrtbCgNi9PkZmuQ|QD9S+II z!XvfGBmLhskMzrbFnOdw9!VZa9!VZa9!VZa9!VZa9!VZa9!VZa9!VZa9!VZa9_hIE zNXR5_f@H;~I%=g^`^;Ktvq37o=8h+K--uB1v5-o)outz9)}+#7p0$%oX)CEDsU)c+ zsU)c+sU)c+sU)c+sU)c+sU)c+sU)c+sU)d%MUzUHCGVR=y%JSR`vPdzOTPWh4b$IV z`C#`=*d-qe!*u7#F#Y11VY<^dOol1SFv&2Vp4~DYgKz0g&x~)M-(rg+argAbKg-eDVt7zPVN3PTQIC}H5^xP<{T z=V}f&w?)ovv2$DW+?Keuq_xG+ZBcYv9NpII*xccbZQngfFKP2!JysUF%X69MhYODVU!4?Mc`*xvhxW36J8JGDuKyW0xrR4{wq73F!}@+3ZqfD_>pt`-~31R zT7~mZ{JFfV6O;eMyyuR8^#6+Mg^PB`YN>B`YN>B`YN>B`YN>B`YN>B`YN>B`bwg zl9k%CQm82xSzi0%#V;D|wxr+h3F)&HIp)%k0RYBNlVs!CBw zQAtrrQAtrrQAtrrQAtrrQAtrrQAtrrQAtrrQAtrrQF+bOR5*2vqQX;osg>{0@-13! ztY)J$8>`Kg@Kv|^zLUG>;HrEqeAUw@U-jSCeATt~Hu^RBD)}n;D)}n;D)}n;D)}n; zD)}n;D)}n;D)}n;D)}n;D*38U+gBm2#tn#c6>@bo%cWUe(JtNP)rA)x?0yGvC8<0uB>Y)3QB zY%~scG!Az(j$trPVK5GNG|pf!j$klOU@#6~Fb;P#PG2w%UocKsFdEnyTY`<+O2bJT z@8Mv)&y4#(tN*eJv_$}kO)ZFp=$V;k>%{5tgkprk#6nUr zQZZ67QZZ67QZZ67QZZ67LNP)yLNP)yLNP)yLNP*NVj-y*sTir4w3f7%w3f7%w3f7% zw3f7%w3f7%w3f7%w3f8iWvaDsTk!l?NfW`_Bf-1BPZDb1a&4}P9Oqj=nc}$4Mgm_Gd$AoxHh{uF@Oo+#Xcw}NB<}qO& z6Xr2t9uwvLE-&hG;q_O~KD+w|WS5VH@Ot+oyso|`yk7sClkhSLFYQ&-UPTpk zsi;fBOTtUSOTtUSOTtUSOTtUSOTtUSOTtUS>$4-guwLxc8aI>qZZh{@vzw~CHp1&CHp1&CHp1&CHp1&CHp1&CHp1& zCHp1&CHp1&CHwU`v0u@fDe+&XagU3zSyUY*!YBm0Ix6f#?>MzP^J?8V7AoxDcPi}s ze_vB!-+7Zsg=MKQsW7Q9sW7Q9sW7Q9sW7Q9sW7Q9sW7Q9sW7Q9sW7Q9sj$zr3d4ta zLon|x?#;#BjLqKaC^NRX243u%sIP0_#e5v=#jbY75~^L@?!F0@?!F0 z@?!F0@?!F0@?!F0@?!F0@?!F0@?!F0@?!F0moqPh9P=h(e(au~yXTs0)>yM98ztE- zAAWZCB?vMf3rTkMNs_&LO_JT{yCz9iCCMboB*`SnB*`SnB*`SnB*`SnB*`SnB*`Sn zB*`SnB*`SnF5i+2Q`WmK+ch4Wh1RUgyzl-sDJ%Q%^Uv&VhAs24FlJvo8M8O78MBw4 zI~lVkV+m=5oH}On1 z)|<_KYhR#kJ`a8NtDiizy9MgZ$3maobkb+PzoyTA^4gO=>!r`6&!o?!&!o?!&!o?! z&!o?!&!o?!&!o?!&!o?!&!o?!&yG`lhC}oIW^LEsY&Ko{BF#(Du7yi`#hp&-&c&no zSh%#CPcH4nYcA~@UoyG0K`u=$O)gC?O)gC?O)gC?O)gC?O)gC?O)gC?O)gC?O)gC? z?YMVo2({t|YBq0*)^<(NX7jc0)<&^*;YZKz?t)bFu@GxtJBhVtuZgwmK5Zw~%2r}c zVohRAVohRAVohRAVohRAVohRAVohRAVohRAVohRAV(khi*05{dNUe>H)b<_NzGCz8 zwa=$~?dtbG(0wz8&BwyB{kO@oJ$=oxUHh!bvPD@oSvFZVSvFZVSvFZVSvFZVSvFZV zSvFZVSvFZVSvFZVS+*;%Ws4E;W%A~0{n&i1%u4Ji>9+X-l-oz%bz1jb=r$h<<#zt0 z-2V5Pa(mK;C*@{RZc=ViZc=ViZc=ViZc=ViZc=ViZc=ViZc=ViZc=ViZc=V1j&j4h zdBe6o`&9h+gEAYk*}!eS5dZcs_ddD%9^9Lcg@5}WlYe{cntyxf^C$n7d&HWI`?7mZGA>KTNybUWNybUWNybUWNybUWNybUWNybUWNybUW zNybUWNybUWxsmg}aC|GqIBdgMCO5mYS)jhXx9i#FX)W957-%-j!8X71b8%-rU> zlbI_rb24)>b24)>b24)>b24)>b24)>b24)>b24)>b24)>b24)>a|<(vpYvvN!>*a! zY|&;%=LK|Mgr@tI??0paB@~^Hg{FJ*r0MRnrs+QL)JfA-X*y{-X*y{-X*y{-X*y{- zX*y{-X*y{-X*y{-X*y{-X*y{-X}bSJ)8Xp80o~vY=!RVbx>>62Tf5D5aCWaxr*^-A zuk*2RcE30|yF0HryVu-na&}G5PR>rwPR>rwPR>rwPR>rwPR>rwPR>rwPR>rwPR>rw zPR>rw?$dX6h&yjuH|(0$&3bL$-g(j8brE=X{dm{C40-2cA@H6z3B22`3A}S}F$uh0 z0#5=@0#5=@0#5=@0#5=@0#5=@0#5=@0#5=@0#5=@0#5=@0`HOtJS?6!x*KS8XWQ-A z=We!bNBO+b=H;^vc7KS;^Rcjb|97%^ZO!K0=65HXH^}D6=E>&C=E>&C=E>&C=E>&C z=E>&C=E>&C=E>&C=E>&C=E>$=1~w0+M{hjanontzd$W9-<=*Cc6zbjOZw_>CL+kli zsJ+)sYVRx7)ZY95Y^V0>R%%abPijwUPijwUPijwUPijwUPijwUPijwUPijwUPijwU zPipV7S9|z9`svxWj7Qb(d$`&1u_5=1@q90T<>}pfaeO`&p6{)b=lh~H&-b1?O`b2x z^U3qc^U3qc^U3qc^U3qc^U3qc^U3qc^U3qc^U3qc^U3qc^L<7=AGVJse70>q^O$7X+Z_I3M0aMXc!zw*KE?5lUxk0%>xk0%>xk0%> zxk0%>xk0%>xk0%>xk0%>xk0(X%b6QQ5PDC+=sgAFuBYHE@{Ur4o9iPA-}ZH9bT>c} z`dEm0%6>AFd>t8%6#9j(f3Q-DC3Q-DC3Q-DC3Q-DC z3Q-DC3Q-DC3Q-DC3Q-DC3Q-F2cvXmaL~nR##iu%UJ)Di-Y!pYIch6e3^s(@X zUp4u}7q0oln|;sZ69@T3`9%3d`9%3d`9%3d`9%3d`9%3d`9%3d`9%3d`9%3d`9%4| zD~L}-DtePd+cim?-QR2${Wz5yQmpvUr<~Q@1F`61As4@Xl8et;lZ&5!&7E9qTggSq zMaf0UMaf0UMaf0UMaf0UMaf0UMaf0UMaf0UMaf0UMaji0oLs~(dSk`3^(lH4q zca57b$28vOU(W0vh-LJ#FpYPgOyg76OyeitHJQdJ(Ii;HN z%RX@I42RCpJCe?kfyR;S`DVvA`@N~oq&5?IiaJG{@|WSSl5$8Tq#9BSsSjxgc^h&J z*+S+&nL{oi*AOz?+lM@avJEAMVlYrB{ELBSEO!P-CbTY6>-n%AdG~+CuF^ z9Ri{@aN(MTmO{&+mC*Pbwb1&|hJaq_W9TjPDfF3Jiy-YN?I`Ui?I`Ui?I`Ui?I`Ui z?I`Ui?I`Ui?I`Ui?I`Ui?Res7N8F?LZ^YO6JT`Ddr7bk%=0-Tk554}B?lJgB9}5Th zKPLzIm^BCa#CuN;GVT3(Imp0m*RTyk41 zc1vDc-0T)PyT#6K(X(6Djrx{#E^slpOe}PX>lW8Fu6x|^{BW;w9ghL>$Qk|u@cx~w z8+Y5Y&dXuF8WyL^M&xe9?nd-(#P3D~Z^ZCM6mP`wMjUTM z@Mrm>}pa!hi|b-r@|H{3q}RO#Tz@4qP7i%#V!zU=#?WK^PUn=nzJUFj@qD zhIKoS;6E{%1TO~LsRUet|0DmEolY2if(wO6Z5WgPh@UJxN`=uXj9Ovz3Zqzvl#22H z^`F2k`0PL8c45>DqhA;W!)O>r#V|UCQ8J8{VblzxXBb7pXc}BK@EcqG9Q8`5F%lKEWSER;_iR%@qGh_0fpw`T1{}Bq#nEY=~(HWEf2!&@%{v#Bh zG5L>BfX3uMfjDFGpP&wnY5a3hjK)9uk5G`t6TKe@?wRQ$EZsF)pwk;r{AUJtQBWz>8Ik{6mR+VyB+9Wh_m#u z@Rq+adCNPldCTv-&*UweyrsOQyrsOQyrsOQyrsOQyrsOQyrsOQyrsOQyrsOQyrsOQ zyrsOQH?jQmye0C|dszC>Cq^$)*O;?q+}D_!n;|h@c-1qyS0FHbEF|U&CW(3LHHrC1 z{vR9v^}QsfB&H;$B&H;$B&H;$B&H;$B&H;$B&H;$B&H;$B&H;$B&H;$B<3ZSm>lOc z9{0nEN*5E3`#zJh%q>RqS#LPC`x9)YkA>0vy~${%HKTd8+Yr(=q!>6|owL{!QMaKy%7o%#Lu-P!je=Q$%NkoMzw_IkdM zvvTiu-@DdX=l6Tp`%)Q{x}a1A<)S^;>}#@B$yFs&6~qhQR4*Jw$m7ia-&?{C-If!R2tv&H>T-5 zE$?Y~PrG}X-HSKHo8nFJrg&4lDc%%siZ{iZ;!W|UcvHM7-V|?&H^rOc%{KzxgmI?8 zCTNdL&QlV33OZ9^v%orU@yM$Bb2w+71J?OlE9<=dG1mD@cV1a%BkPoP$~tA8vQAm2 ztW(x0>y&lMI%S=*PFbg{Q`RZ#ly%BFWu0$S)(P-TQBG1B8J41)qH)N`rlww}AXp8I;{?SHIi zvR;$+O5W?9#MeEUuX|Eo_vF6rA=P`bU-!grPw4i%Zpn}3b;*z=MV5gVCQCMHvdNQ8 zqAZ!R4XLsXxv~w(vXpM+;dP|THss4TB+NEs%r>OVHss7UB+W7;=av_eINOjp+mJfj zkUQIuJlmx7E9tWh`LhiPv<(@w4JouunpKfR+oa@cD!!6NOUf=UB$KvDT5+<9Nv3TA z-C^!cehu?m$S+|-N^L_Lip!<~Y(ti9aDKz@bwi?UL#Ay*s%=BAEdwu{cww545sQXkG(@8z91Zbk z2uMRf8Y0pVl7^Tx1f>v^LQo1pDFlUqlw+b&h)N+Ug{TywQiw_+ZMhJYLR1PtDFmeu zltNGnK`BXJPX01MVc>;HVNMQnl9-dloHXX-F%y+SR0>fk1f>v^LQo1pDFmeultNHa zkCz;1UPx35Q7J^F5S6qLO9!zO;!Pdil;KSQi$YM6>z!2ZWO~PhM1*8-VBm#Br4W@u zR0>fkM5PdwLR1P-DMY0ZltNGnK`8{K5R^ht3PC9Zg@G3ml|obsQ7M8?!KdI;@G1Bd zd zsQY|K-RD#4KI|ayAR|*2ey{+9g&-^lVPOahMBoeLC7E)<5i(;T3At^1f)e=Hnm_VY zPH@7)69N<#qOc%^g()mhVWA2OR#>>g0u~msu%Lyo1?KTNmVv*@f)^IPumFaIFf52+ zVGIjoSSZ7S85Yj4fQE%M1U2wq2y9qr!-5;a8-O4DycXcF5QhahEX-kn4hwZyu*1R~ z7Vr@Az?86u1wJhFVZjdze^>xS2n7E&Kd&H&g+VM3VxbTVhFCa6Kt%p^zOS%|1x741 zV!;s$k63`jLL?R>5hekq$&5tamgxV%)RtjqRrxa8Q zDg~8-NKMH&?Px5*;)KB0>GC9ttCTiNzqzz zw3Z~TB};2b(@LHeWI9wY{5z>y4AAW4YB7Y)CS8jmATR(t8C!7dOj$}z27o7Piy>2z zOqawhhRg{6PVN@ad!dadIdz$`WYsNcbxU5|l32H7)-9=Z$*oHo7el_U@b6@DG31Mo z%at)-C819~7el5b%`O>T44INlyCiinfyViRQU3620{~D-!+NV@UM(zH~*Rok&z9DiRfmibO@CB2kg3NK_;$ z5*3MxL`9+^QIV)fR3sW2DiTfhX;OkkqCn5<2Z_Q(7gU=|$+`a;M7j~o4Uz78_~`pQ z>4NHQV54~s_~@l8AN}KFeDp^yU-@V+AC-^FN9CjPQTeERR6Z&nm5<6t<)iXZ`KWwU zJ}MuTkIF}bM&+Z|d&=l_`6&OKR90#xgp_V9b0eimCLMsKw|nOK)m;Fkc@ALdTd%P6 zH;%#5Z~f^NmJY&FVX3fGSSl+3Q^~31RB|dgm7GdWC8v^8$*JU2aw<8MoJvk5r#CKg3O-FSQ@T+n|8#?#rwla- zs1q3V&VPBPx(5g~&jCgqR*d?i$1v)-k6JP6Bt{jZic!U=VpK7z7*&iaMirxqQN^fY zR57X;Rg5Y|6{Ct##i%z{j0#IlmJCFx5L&)rZ1U^BgeM%T}iP zJ;#{pPhPk()mf%0Q_F?STa{P26`=HT>$HEKkt(2LBQ2K2Y~g86|g@07+}5TuTKE$yce(vSOu&C zRspMkRlq7>6|f3e1*`&A0jq#jz$#!BunJfOtOC}XC}0J%rqJrN6k0u{q1U3;ln8ha z(pf+4UoNg50cp*1KwBTU($=p%Mq8hH-<7sD(pG7!v{l+FZI!l4TcxeiR%xrWRoW_T zm9|P-rLEFdX{)qV+Io|xtsvJFXPvqXeezuA4UTT`b@!g2*YCgouKEnHYn}u2`tTLK ze(5pv`ha&@(Q6UCie5#pqF2$Y=vDM8dKJBjUPZ5>SJA8JRrD%)6}^gHMX#dQn;UwC zzowuo4YC4ZH%z)g*rdR|7aaE7mt0sq2?m?zfWvE7Me_J_hD~FZC%3-o4T>&%LdPG{PYyhCT;d^5ZYV)`1#c{ptE@n2<=y_ zg!W^P5!y#QawW9AgjPZ;p_R}|XeG1~S_!R$RzfSGmC#CPCA1P+39W=yLMx$_&`M|z z2rZbl4XaJ9*}1P5V!QFwQ)0V&A8_rbeb{-`3qZAb4sh)^t+@7n$8hcZ9xJXL#I@pD zajm#kTq~{>*NSV!wc=WFt+-ZPE3OsSifhHS;#zU7xK>>I@8VjZZ3@)RCk1LZvU&<` zlWP0Eu-hN}%X6w9fZOIdV7K4CvfIm#vD{fOwyOrI(9_$u?o1fRrly-ko)OG`{XQABP`vG!q_jPBgmjH3| z9Dv*xtswWkj)C0ozh(uwlOR`+E65e(3UURxf?PqaAXktp$Q9%Yas|19TtTiNSCA{n z732zX|KlJRnoA+vc3(2IOKpXXyKY=}cXz1nyI)yVzYNjMb3k=pyi(n}9;3S7@W7So z&Qe{eu2fg5E7g_iN_C~WQeCO8R9C7i)s^Z>b)~veU8$~ASE?)3eS=Y5&@RPu+kLg$H;5M$jC?zt{|xL93EU6-~lU0wRR40RdnGSy|S z+kM?Ob<2OVt=q0{`??+KcC6c}Zs)q&*IiS0rSABWUETF{H`H|pW0>l0uKRu6H+5h5 zyjk~rOMTrBbwAd9K7Q51z8;!-DD}|RLst)c$3s1g^)S`LT#tLs=S@A9dTi^ltH-_` z`5TS(IMw4^gR-Wko=QEn_0(~Ut3|vbUJ#4F+z@rrmw zydqu^uZZ`JfOz4&5Y=`MQ*HMX#P=*9drEwFm%)3>oiC~W3f7zFfcO61%6o5pjQ3vt z%9Zyv@?LqbyjR{U@0It;d*!|IUU{#)SKce{mG{bf<-PJ=d9S=z-Yf5YQ3< z6$}6GV_5i^Z?|IMPAn`I77L4o#lm7?v9MTJEG!ll3yX!t!eU{uuvl0uEEX0Ei-pC) zHw-Kc8%_b?G-YdP%N9z!G2F8#@$UU$#Gm?i7glc%AI@{Yh~IK$#Q*CUBYw)WS4P~+ zh-JhwVi~cFSVk-(mJ!Q{WyCUK8L^C5Ml2(i5zB~W#4=(Tv5fcz$%p~Pd|z$ZXJ{5= zydm8U8Sg#-VEm;QTv)w3usF{FFn+rg82`#KVEmBJS%L8&FcugKj0MI5V}Y^2SYRwL z78nbR1;zqnfw90?U@R~e7z>OA#scFT12Bdfm!RX5I>j5@-OzDLQd|x_{;B4I>OCRH zc@F6DyRG#2r;gF%cX;QO9*@#v>9O=!dMrJb9!rm<$I@fzvGiDaEIpPUOOK_;(qrkd z^jLZX^^6mpc$#;6dEvgRyBj-6l$?vhEzq&6>InM!Ce*cv#f5$Pde9cQ%t~|??<;rqpxw2eYt}IuU zE6bJT%5r77vRqlNELWB*%a!HIa%H))T=}NLm4VCrRoXU1l-rXc${Phf#mu`80y2O5 z@0?dX1i+l<05V^-Lgr^4gUk>A?i0w|?1juiW+AhXS;#D87BUN&h0H=`A+wNK$Shh~lmN-kCCC(COiL=C6;w*8NI7^%*&Jt&dv&32AEOC}N-{gri z*g3_T+djimtGU_m@L2$Q_aWft5B{=qtIq~K=Q+U7k6Q8b7aqgUzstYaT?uq6;%D)* z_*wiceilEApT*DOXYsT6S^O-17C(!h#n0kr@w50@{49RHx#4FRbm}{|M80k3bPhbF z#T$*@-3JzZpT9Y$`Vu&Fo&y&BxRphJ<}nuicVEA<=vEdji5snpoa$?U z(s>TR^cSyS`jd`<>BoKU3Z^^3v|w5=EtnQe3#J9rf@#6DU|KLOm=;V6rUlc2X~DE$ zS}-k`7EBAKuLaZ4=@gRg+6>*2KD}||jZ^PF6iWS8e}AU>76^5o14@0(N~u5k7^QyT z`>m9^mr_fqrPNYtDYcYZN-d?9QcJ0&)KY3GwUk;)Ev1%HOR1&QQfeu+l==vz234o{ zbazsGdLzmkuTJyB4+B|0^NDAw?*>=rIY8FWT9NgK9YfaTr&eS=h^$4{B5RSg$XaAA zvKCp3tVPx$Ymv3cT4XJ<7Fmm|Mb;u~k+sNLWc}|VYq)g^R(JfewB1R;>J2cT671cT z@ay+|+cVXVz^?Ng@aykd`Sm@I@$1`s`O2?H`L+C7el5S2U(2uM*Ya!mwftIsEx(pu z%dh3v@@x6E{91l3zm{LiuU}t&-L(B4lHH%wxZY6nDalSU_Eo^_TfMxheg=@8=KyB^ z@Cvi1V=()fk6dB)B+M3O3$umU!fau-Fk6@{%ob(~vxV8hY+<%ATbM1(7G?{xh1tSv zVfKFLbh-Fq^kOLsCJ$Mvi;L5+5WG`$o6x0E7_hU z+mda`wq#qfE!mcAOSUE3l5NShWLvT=*_Lcewk6w=ZOOJ|Te2P%c<-wEBM^6<1I+zvE9QRNW0?CcuR6ipWiRFybBnpf++uDqx0qYZ zE#?+;i@C+zVs0_Fm|M&(<`#2{xy9UKZZY>819QW=Q-HhY_p9xf0^Fxe`Yh_b`*4`| z+rzs5W_=Er_aCmz`&P%8_uD^oW!{a0Q|gT z0Q_&q6@V83xBy%LE&vyR3%~{70&oGi09*hr02hD@zy;s}Z~?dgTmUWr7k~@E-}C?+ z3Z6pW!&2zGY_NJm;pt)d5u}4Z>Q0wd7vHv8o&y^GuO~G8LI39%4gb9JR~p_*!=>TU zaA~+STpBJ7mxfEjrQywCE(^gL2%gN>Ba&kGjoLo*WCzq4U$>rp7ayhx2Tuv@0my^rM z<>Yd5Ik}u%PJV;rS%Nq4^**7n%#rh2}zYp}EjpXf8AtnhVW^=0bC!xzJo_E;JXK3(bY*LUW<{jRKlO z&QrL2JSkkh@$FN5p0x7^LeQUftEze*=y{$4g5InI{eK=K=s)wqm7tFjbP2iyU4kw_ zm!M0~CFl}#3AzMbf-XUqpi9st=n`}Zx&&QkU7RjX7pIHU#p&X7ak@DD#*Ncq>iiP7BXMk@U02qnZ1j7h>${JJt>5+Ys_Oo5 z^*je`{oX5E|F&an{Vw-i+4?M7m#xdzW$Uta*}809wk}(jt;^PB>#}v(x@=vxE?bwa z%hqMpk2@|Xcx2#+6C=`c0s$KUC=IQ7qko71?@Lc&<<@+ z9rmfq(APZc;9_Y(%awuO!cV{_dEyG{o_{Z{$ zOWmdJQg^Al)LrT>b(gwJ-KFkQcd5J7UFt4%m%2;cZzj|o_MYPH(^9;>-3a-{-;?xb=;NkH6-nX2En|59bzPB4NKPB?JkA=s-`ZZ^&{|1ZC zbHL*tzw-FU9^>(kxn$+>tvp^HFOQeU%j4zo@_2c?JYF6zkC(^GLvA( zdP%*cUQ#cqm()w@CH0beNxh_AQZK2O)Jy6m^^$r?y`=sKsR!4m7X3VCm=-)E|BgR!ZuQ?m_IVDl{c~4r{~^b){bTR9V*5dCFSZxki|xhsVtcW@*j{Wewinxr z?Zx(Dd$GOPUTiP67u$>N#r9%*vHk0Y?P2%)vUGcXURTzo-5C2u@F|k>2{8O$yx)1% zPr&c<95DRvTN(cQ9b@>P_?s)kA7%J5d>OtBUxqKkm*LCsW%x3D8NLi(hA+dH;mh!4 z_%eJMz6@W6FT09)u&K?w?8SAzrptn?Wd>h!=U>g`YY#GzXjRPb3pfhex>_&K1TOn@wk=l&(eMA zzI0!@FWr~!OZTPw(tYW^bYHqJ-Iwl5_oe&Ned)e*U%D^dm+njV-*9vv#7}YkZclOj zZhul-e`E2duz&Y(Q2*neeT(YP!2Ed*Q2#4d)PIL#sQ=p@euDbDy{KQ*FX|Wdi~2?V zqJB}os9)4C>KFBk`bGVseo?=uU(_$^7xjz!Mg5}wHyY}P_w)PJHTx9rIi>Qa&_5~u zkAVAM^f?z){{ZvPbHM%obmjh+9OM4a`lFTmH*$ZuzuaH$FZY-G%l+m4a(}tM++Xf5 z_m}(2{pJ30f4RThU+ypWm;1~8-vrzr_@5g7O=|czOAY^SgY;)5*UR|$~+@0@ei4l0<-yJgo<)Nz)Xf*;F0gZr0KqH_L&(efjkF#;LTP&@F&Ogz>nT@)dQV+Ks}%yP!FgF)C1}P^?-UnJ)j;?52y#!1L^_w zfOVX@Q9>5XsE7_HOhGw}=u+jX@7VJJ5SMYnEa&h%8c!E3!uHcTV zEBMW0uHf0XTwOu0E6^3_3Umd!0$qWwKv$qE&=u$kbOpKsU4gDZSD-7<73d0d1-b%V zfv(^N>8I(T5 zu-rb_F#oAX*gYDHaMjOVQeB2Y$a7#3`qd)**fERnG523B!l*^iB4`n`2wDU!f)+uG zpheIkXc4psS_CbE7D0=kMbIK>5wr+e1TBIV;l^kYPzmXMP)>RuY{382C*)$nV^9h& zz3rvd73hRK2TI}Yt5W#hV@lx;7q3cTQYokuR0=8um4Zq^rJzz!DX0`w3MvJaf=WT9 zpi)pNs1#HRDg~8-Ngsc(26{=|uF z==aJ7WrMOo*`RDtHYgjE4ax>(gR(){plnb!C>xXw$_8bFvO(FPY*02R8*Zwy0ppOy z3~d@Sv`b@#Q!{Xud)Qr#d3gFSpId!8)*;VM4WWimL#QEco*JU*x;;XoTlzorn=;shM2a$f22SFie&80> zSKuS^95{(5tWM(5$DG8k{qE`{TAhSWLMNe<&`IbdbP_rVorF$8C!v$jN$4bW5;_T; zgib;yp_9-_=p=L!H+Lt2m`KlvuFcRbcP2K2aH=SFpNXJ&+DBYieG_sb&w-%$@>Nhg z^q8Rdl_#u%qEk>PC=?V53I&CNLP4RRP*5l+6ch>y1%-k_L7|{fP$(!A6bcFjg@Qss zp`dszL4l>{>$r-OMiZM%ICT|Vr}!*v#kc;|Mb)=sD)JoIif>$P#fKlW6%YB4)mHS{ z3T=h9LR+D&&{k+Gv=!P4ZH2Z%TcNGcR%k1<71|1Ig|wXXe+c8$7}`4BF!l} z_KB`LX-=`Zg;RO4dmQTG7r*x6>P2XaJO}FH`K!A4z+>uSs#kR}s4i3&steVH>Oys) zx=>xHE>st)3)O|{LUp0KP+h1lR2QlX)rIOpb)mXYUA&I!qUriHw4k?kI&5#2;Z$Oz zqs3?AG5+yhmsBstVdOdR7%yBs#(N+082{zztH&7i7^=vn@pEr; zY4vi1MxFzy@sq36*dLP`&wI=&H72EoQbVbs)KF?DHIy1k4W))sL#d(EP--YOlp0D6 zrG`>NsiD+RYA7|78cL1-tkl44^wAqbhH<&Nu{ns%aqJ$C;rQ1VoT+{fyOHO>aJ+0a z9Q857@udA~IA#rphC{=l;m~ksI5Zp@4h@HfL&Krr&~RutG#nZZ4Tpw9!=d5OaA-I* z92$-{u;D;+u(NdiNw1B~Mr^JlRjNJ@-EsG8&aGaF>d14TJAQZ79k)5AJEm8k=#F8p z?ofBAJJcQO4t0mRL*1e7PnfpM1q^ZZSH1$A;TzhU(o4Sm3nd&mv?Y?fC zy5+yw)@@g}eccXqJJ#(~w{zX?>#nK!p>}-9uI~D}8|u1)#7%WK*R*S=SvxIGXmCP% z6Y79b1&kVC+>haQ3~q=Y44#Oe6hEhd?|7((u^y&+nCp?{3eOr#J+}4O)ni|e{EfzX zoa%9|0lrgHPospc9pTAm>zdmL~ zUiPZhinQ1M=d~g=9|Y%p%`g63>bb4wuCC|4p80Gr)-#$Yo(NI2#}w^RMSEP)9$B;} zWP2jE#~1B+^VsvcMJVyQC?%W{122qLidTwQidjM}HMpe)xzu2n8thVoUTW}54T7n` zFf}Nq2FKJOnHnq;LnIY1#56UirUuv4Ae$O&Q-f}5@J$WEsabw|W3zZ2(uv)ECgXe2n$14AOc?? zFUgb>j*uA(Nl1F?2}*z^jQPGo8y4IU-T?gI z=d}Qbg*Yt8p&{dy|Bs(nlis%td9Mul%ai-cUvBxnQYxhZgE8c*kOs?;DWPNyl#C%$ zNQmV}kuSm_o*^^Hi)F|Ra$_@Q3JJ0dnL?5*ASP2tlx4^il4Th(g@jp#%t+*$M7~Mn z8zP@EQxf_np>GoUCZTT<`X-@o68a{gFX2WdkuM=fCFrPx9TnhFMy8bTqms~FN})<6Sg9oP zC6O-yOC@BffR-}mtAs9<;H472R05bv2vZ4SDhYi_=u0S5N#si+Ujmv+B43Dn#!Lxp zDx^9yWJ*F`68e(RmxR6~^eKwNkQs@5N#si+Ux<9hOiAcVLSGX4lF*lgz9jS|p)U!2 zA@mHHk;s=sz9jO6$Y;!yguW#7C7~|~eM#s`LSGX4lF*kd`jW_(M7|{QC6O;gK4YdN z^d+G$34NQ;w+Vfl(6Ch~0}-zM@ckfG1Ch~0}-xB$ZnUc`A34OZ&#f2y)^z8{!%#;NxE>v;BiVIg}-**Xpm(X_!eV5R834NE)cL{x$ z(02)amqp(t@?9d|CGuS&-!1!nm(X_!eV0|=CG=fF-zD^2Lf<9y-2#;ts=Q$3>)(p- z?vn3|(5HIVCNR#vH&XAOh$8u}=U!aB9eO0sfg*W}RgwI`F-3BpUtbkTry^lJuo_qk ztOV8ptAI7Y3gGu&MZ#SnmSA?QY*^W^vR!4j%4Wr0#lSCYwyL~GWuMA6m0c>ERQ9NB zQQ4ugL1ll+_LSWzn^X3tY)#9~l#MC-Qnsb+O4*dMCuK{@j+6~4`%$)|>_*v)*o$)h z$nhhmj~qT2xW>dTl#@pe9yxd9*pX954jqNi1oFW)_A@3wPL*xjN6GXO&oF8(0$l)Pp zha4Spa>&6U=Y|{`a%#w-A!h~#UPx4OV#t9Z=Y^aWa#+Y&AxDLr6mn3=IU&b{oDy`awf=;ASZ$x2yz@?;Dtmb zhk=|0aumo(AP0e*19A+=DIkY{oB?tK$O$0t|9Sh*yMNyNxv9nriAvu3TcVOT{+6h; zM5QGvEm3KSO5XDGhM)KQyxprvR3t2wj-}GER2aC?$Xe-GDjiFuW2tm3m5!y-u~a&i zO2<;^SSlS$rDLgdER~L>(y>%JmP*G`Vc>u$~R9X9Mfm!1~e;^-cp)2D&j!}Bf%R-)JsVih2G$dmo~ZOhr6(voLFoxfPf&V-(i4=Pp!5WV zffo{$o~ZOhr6(#qQR#_FPgHuM(i4@Qp!5W#Cn!BZ=?O|tP z%0N^GqB0PafuIZoWgsX6K^X|jKu`vP!oUlO%0N^GqB0Pbfv5~bWgsd8Q5lHJKu`vP zG7yx3pbP|MASeSt83+mkFC;1hQ5lHJKvV{zG7y!4sHjL(BrKJYr82TqMwZIRQW;q) zBTHpusf;WYhTO%@Wo&L^vs5OQ%EVHcSSk}sWn!sJER~6+GO<)9mdeCZnOG_lOJ!oI zOe_@!UdURRSSk}sWn!sJER~6+GO<)9mdeCZnOG_lOJ!oIOe~d&r82QpCYH*?QeohQ ztd)tSGO<)9mdeCZnOG_lOJ!oIOe~d&r82QpCYH*?QkhsP6H8@csZ1;t242WonOG_l zOJ!oIOe~d&r82QpCYH*?QkmGmCN{8%{cB?Xn%KW4_OFTkYhwT6qXjQy1Dn{uCN{8% z4Qyfqo7li6Hn52eY+?hO*uW+>u!;R^V*i@hzb5vtiT!J0|6<^UL}el>6H%Fn%0yHq zqB0SciKt9OWg;jOL7554Oi*TmG82@Upv(k?0rf#tW}-3^m6@o_L}ex_Gf|m|%1l&d zf-)17nV`%BWhN*yL7554Oi&njAyJu$%1l&dqB0YenW)S}WhN>!QJD$KOi*TmG82@U zpv(kiCMYvOVc>;CWhN>!QJIO#OjKr~G82`VsLVuVCMYvOnF-2FP-cQM6O@^t%mjsW zHeN_nW}-3^m6@o_L}ex_Gf|m|%1l&df-)17nV`%BWhN*yL7554Oi&njAyHA0s7Npz ztU(4oa;iwce{g?RE(4#

7%Ybgm*nhd?J(Bq|aWiHd|hGU`S}(s61-o%0saac*N^ zlk7OTG4PJMHA_c&EI%)fZ~QS+IKVMv3P(7GOyLm60Qij}!NT)C)NznwIO#XC`HZuS z$?i$`lKZ^$lIq=XC3y~f$(>eT@~g*u$t|y5eMzq`$p;cXX5>=_pC9;Wz@eY>JSTO| z%{gmxa`sAzz62-02Sa@cIz(TRoQPyZBq1XC5J`tfHbjykk_(Ykh-5+}5rTmivePFG zB3Tegf=CKPG9Z!wk^G0GKP3Ah$q&hWNa{l}ACmZxyoaPcFz`YkN^%~O@{o*&Bs?VF zA?Xguc1W^AavhTDkW7apIwa2_X%2Y{&l@-c2p@=&42L8*B)=i)4asguazk<(lG>2W zh9ov5uOVp-Noq(=LsA-&(Lk+4UI;`#E=|@ zq%b6dAqfo0Utr*cd=~5YDAw^wtmA`N$LFvPhSI@MIv7d^L+M~B9So&|p>!~myc_4u zIPb-ID`wz@L?v&;9Z|{Ka9VifJveW{c?ZrLaNd9O_M3O#y!qz6H*dZA6666<(U%|( z^d$pJWnifcER}(!GO$zzmde0VVZgw!femb60~^@D1~#yP4QyZo8`!`GHn4#WY+wT$ z*uefZuzwBgUjzHs!2UI`e^DTn7qWp3Y+wT$*uVxhuz?M1U;`W2zy>z3fsJfnBS9J2 zzee`2k^O6A{~Fo9M)ofT5I!5&$ObmDfsJfnBOBPr1~#&Rjci~e8`#JOHnM?@>|Z1M z*U0`gvVV>2UnBb$121F)8`;1{Hn5QmY-9r)*}z6Nu#pXHWCI&nDkB@%$o@65e~s*4 zBm39L{x!0HG4Mh*u#pXHWCI)7z(zK(kqvBQ0~^`EMmDgK4Qyls8`-}`_OFrsYh?c# z*}q2iF9u%71~#&RjYMT6DkD)DiONV+Mxrtjm64!~1Z5;BBS9Gn%1BT~f-({mD$epk zqB0Vdk*JJBWh5#iQ5lKKNK{6mqA$Tl5EXrizJ#SRvs7l5%FI%kSt>J2WoD_&`P`Tf zj``%6kB<55$o@66e=+dFe12pDo7up)SHMU7JzE$9P+-p{#sC!9vyCx?%(Ia(00s8! zWZXdrN7%EOF=Ptc8AGP9p)q6%TN*>Au&HrlAzy@TjUhAG*cdV+TN`7hu(>g03fmi6 zBvaVn7&3(|jv-Un3{W5^WtJci5& zMA-+T>;qBuIk_=rN+`;n&5t2d*!~zYB@ksFh_Vkv*|P;QWCoicLuRlIaw}u`zSs%* zW2UedGGq$7Aw#CHA2MVLJ0cn+UxY1@Av1_HLuL@`jG02X88U@{bK@gZ2suNh5Oju2 zA?ys9LEITKBatVX{I>{D2Bz>+%)k`>rWu&Rx08V>%wZt(44IM0r#b~gW)S&|nUc_R z*&#!wB=p=d$dD-sJ*oW}@C-1t-lb0ToC$G5sDo+OG2`3}YMVyLq2v(lt0Gsn`j;}er z=J1-cYmTltx#r-Sb8C*RoLU)pVGgaFS-ot+^|-K_P&eyw zS&xg`eK1$+aUHvCa$R}CKj-H~oN_TO0gZ#Nv=w#sX21-5kIa8i9)j3m~GqpLrhT=68 zG8Qe{#eEglielj``@fwPq zP3&yKohTA=?QCLa6FZx5r!RLma_!L0CY0mK&6T9>zuslLj++$zO@73=ahjs*5i6Vc z*u={2OVBDG`{eVht57O=4z$X>R;}`#$F#~dk6*RQq*hU@s8!S|Y8AVoQ0j_zc&mF+ z;fu07c0;in3ODTRhGKOut9#iE1w+KWZM&gR`L+A!XR@ivS{p-|w*hVgIcZYZy_R^eSK?uNA32$oHO zY=R|MSH2X-^1?s4p!yj6N}dD9a^>n+zWJDAd53$ij%C)d=vZ_tIu;#^jzz~}2Nk@E z9aJcIj$5&V3I#vyph7zl9gB`d$3kxgJI>od1!`^w6&jpSq{$8{cATegO*+n`d5#_D zOYE8*=j}Le$9W`5I#bzk{(9@6f^%7mfbplZudbXr7XFl`D`}S2(JU>>WiukD%4PRt zgv+fyYF9l9*^=i#xIB0jE?<32xct>!PJ|1cGA_U3>h7w`uBd54T~isprcY>1@>b2g z8jch((bW9o`=JL1-8E>eQKvoHTyy-UF&YU@G?=JKZz8*ierR+fhyang>HZk%ZYN<&Y2b<%~C zCY;nRs4-gjEPm4bocSp$Tof(}7ln($Md6}wQMlxcp>P>;xKg;-vYw+fJ((3Q3KxY7 z?dcUR3Kv_}+p^x4^|q|HWxXxyn|6ka+p_)!*s}ih7cN+rC>oluqh3zk$fEd-4QgglT;&m79x1{X~Dbx&JcbUmgFEhJj zA&IMLx%9xmQfc?eW8bHN{hps!o4QhVYOzxbH)8l@u~SROtqOh)Z-P!O_?5M!7=JqZ zy3473;ZF%l@+>;u=@%Qbxsp>GvwJEk=9f`0U%WmCD(16R74uogRLp07#j0Y8Dn=Ef zic!U=VpK7z7*&iaMirxq;Vw0|x>YeK7%#$1T*RMLF|!w8ya#)~jsgz+Mb z7h${z<3$)`MFeb@ZeCrwd3BVsqRkfyvP(;+8&T!4<0=LrlfMcj0dhrVvnHofCRb#> z3@>ws=bTergO|y3;AOsO^)in<=4C$NHLI6t^)h-Hy^LN)FQb>y%jjkFGI|-kj9v!z zIFcKtmq`(0y$ltp^)jtqMlZwd7`=>MMlYk6(aY#%^fKs*o0FG8&g522I)HW5ri-T8 zB+6NuX7}YtnkU`&{OVZpfVyUF5XC)1SMoE(rW?r6A(kN+^G)fvJjgm%5bMuxon3~)iO7X&u z;)QWGn^W1G&F(8OHut*ah1GXqYw{czn{Qr?&6UTD%@2LmYHWIqjmAb}qp{K0Xlyh# z8XJv`#zteKvC-IQYz9ot$TMkdrlPUgw_pj4jn``EH$yF(rlSCiooDPkW9J!~%=C24 zD1o=d&NHcb)u!fEo0?Z`YF@Q{+U}47!;rQ++*HoU&#UcuS84l9=`)4?BMg~B|4W8U z88T%^r=B4*hRhf;W5^5|v}eo|3Q;j+3f!C_Q^rggGiA(_F;k#j44J_jJ40sBRE}Z! zzCgnKF;miiq>Z6((`l?t&8wE0R}7hwyA>^=XUL31zD>`eHa&+rdJbh|N|&ZXU78Mc zX*$%U=}@<4O1Ec7fYH|iVp zjrvA?GjlAS!9272Cguj#q`pz#sBiFA!Ff$^UK5hrgyc3MxlM3h6P(wCkZi+#BIl3HO4)O+8wGY+R<)Cq3 z+IF}hdmKyQRUBPT>7swS=$|gaD7`L9dXSgg$0)gvQMiwhG5RMLGDAK6*@xcg|G&AXgx9WLoygV=fhyPc_i4XKmoKG zOzbqzT*c0$*iq~#b`(2`9mS4fN3o;WQS2yo6g!F?#g1Y}v7^|D$$|0i!*~@tfXp}@ z#g3PBy0{I+POfKJV-3Z_CfUa6wbod(##)#1dR=U87n|F~=612UU2JX_o7=_acCooP z#TskJ#SZ_5zkQ!enLU>><9Rj>bLx3^UxVej>${#;{UL@Y&w=In)z$Lc>6qoY{XTWU|~&G?>8$ej9~bl>?}l+PdC<$~%}=$bCCcz8>nare7vU9p2fH)~H&-%6C<#<|ps{ zyz|dBwLU5INtw@@DxcK&q{1ilJ*n={2${>I$V6c4Y*OS+UKFJ;jSX zZHo1Iv8N69Z_{$C^Ge-It|pR{e!|4_+|tniX%O8 z^xV9O4%&3hrh|4*M+QCr!!N0JcR=az9LS)5IgvpRc+D{x^ssMQWl&KDDT911ob{9L-co@-h%c$jd-n z@9;7ZmEhPVJIZ;{xW_I(x!JSN&7M8`#Or?<=sGfpf8?@RPbs!&p-tRuS}0}Wz8*7l z$Ge@WZi^Mlb6qoZhl^G-^q0rX(EjeL8EQ2{njy`QW=J!n8PW`ChBQN(A-X#v|LHPu zN6irbB430?JsF$D6m24BGex^^KokAJW6!JJ5k-{eKoi|&)kMF0OcR|)4Z2-CRHr6V z6RC;RL~0^6k(x+Nq$W}msfpA?Y9cj}nn+EgCQ=iriPS`DqTHTR6SXNM*s6(gcdSh# zsdmooDK$~2CQ2iz>q8UaigHb5OznIwPHt&PZpZGtwF9jC4jiBb|}XNN1!o(i!QDbVfR(TxHN1 zwW&AQ>Wp%!tfSmuMlM$AjJoUJ86l3+w5X(MQ8{T^v>BdLg|z!71k$Hoba8bzii(k5w>v`N||ZIU)A#S67bn4?_Ay7AZ~loI<_-=+god(sLR7oRR7XfsW_XW^M1ai?2U zABSVgbKsdieDzGq-WAI-2^=o0wA5S`u3y=(7;a0Iw_r$PD&@GlhR4)q;yg`DV>x~N++e0 z(n;y0bW%Dgoo?FF2^WyYO)dL<-_E)Abn1vU2eo@P2I{;IyQF$N_9@SSfqL|6pq_Bd zK;83WR|C~(pfpe#C=HYbN&}^V(m-jTG*B8S4U`5-1Eqn|Kxv>fP#P!=lm5U~Uw*~a-Bp)evAL+43sKKSNBzNz&Qwo9MddlrQJ=f&sH>0Z zsCWJSRY#R;zd+s8rEcr4?(3l*>#1h<#~R8*QL~T zY3tI}rLW6SmyyDONVJ-kRWz)krXUpssVB%SFRpmu+AwW+HvW$M4f)mX_>x`S^>sJY zbq5cf>Ta(4ecd;8UtnW(&$raq{ZRK~-3Jxb!@eGxdMNeK)L_)TI!Ya-j#5XdqtsFAD0S4$R7c^Y(jW?9!>}}nnm2iLYNvMJikte0 zN1tCk9WRyVz)d}QbyE*J=B9Svvbw2OH>I1>P3fj|Q@SbLlx|8lrJK@C>85m3x+&e1 zZb~<$o6=3`rgT%fDczKAN;jpOikm`EaaXD5N>S?hZ5rv+QsolW^AJ@pz1v0AbCFbe z4n)<{R#Ej)$3)c=9>0pJPEn<(QdB9b6jh2UMU|pTQKhI-R4J+yRf;M_m7+>frKnO= zDXJ7ziYi5wqDoPvsCsQtg{?{-D$3t6ob;jEtkPNDYWHnet7qNnlIr^~R(TGr)pJ&B zb)RF_>g(TpwN|~>N^7OH(pqV)v{qUxt(DeFYo)c)T4}AcR$42qmDWmYrM1#pX|1$Y zS}U!U)=F#j|Fu?4Pva&87QLHt@oE!HX9+AWUVS?X>ow8y)b@yWm z>vn&?Dy%_;rNUBSsjyU7Dl8S23QL8h!ct+WuvAzoEESdtONFJvQemmER9Gr36_yH1 zg{8uJT@@A{D-EyuG`#9h8eVO_=`5qQdjUS{{hoh*_4Bx_JO@7O$5x-!AM;t~K4kS- zqdrTYrO(o5>9h1%`Ye5xK1-ja&(deqS3&QS}PMR-Oa7^>eG-dgo(u>lg02%B@McrQA|( zDYukc$}Q!Va!a|T+){2Sx0GATE#;PSOSz@oQf?`?lv~Oz<(6_wxux8C!^kZR7jG(k zf6`cM6H%MyswqqOotUnVeDNjKpJKW49GI@(Tus*3Tz(uBIRMIIp3}P?q~$ Lo0mEjU%USY9gLz% literal 0 HcmV?d00001 diff --git a/ci/test/tx_cksum/out.pcap b/ci/test/tx_cksum/out.pcap new file mode 100644 index 0000000000000000000000000000000000000000..a031cfc0d68ab9217053d59e7d4ce6920015ab2a GIT binary patch literal 728837 zcmeEP2bf+()!u{*NKrsQ1VX(OLy?`i-+sFbNRedwQUpbMl_mm#(0-8`no^`FQbbS` zkS0o(PH0LIPjA>|SqQAD@ z+y~w|yrE(ETrm9ad8ho!CTA_zuu&KOtfx|kA8gY&7;j$_+hf>$9wq4FHZgVpD!$a+{HgW*d8~D^q?vtNmH$)Yyw&uus|)rJcW zDB938J#DagW-FO(%4`+wk$a8*Mz3Z z&($_{5}JB3nn@Nl^=xkHR;i{g`sbplEq&QkSIt~==2|k>ie}4COS3%y#@{~LT!S`~ zMeV+b+ufUL_rK35+TA*<+MPFMxy^DAJVzhoGi4uqClXSApars=ppomEb7ti+@n0z zqiYu^!U8p#%{|Invb;5`G-efIb@~0WcRpavcaApCplf7N_a5Z#t(@xKG0*w#m7~ox z#JDd57Iv@VvI>TsN>(9Nfa*u1PUic%&Vg&_o*CkSYy|wE5Dx=!82Amaf;-6ST{4VPVWh?fyqj{#b#Hl>NX{@&T+-Bo3=fk^4 zo2B6evdBLRvVUGl`RCy;YLh^=3uH8 z<}x+od)+V>3Vf($b!@ExVCcqX%+Q;&=9a9PQirn7flIwlufA-I*%baHi+s8v`}C@m zPe-odeL6?e-#Ovamd32bW-Tsj$+H&9N-(*O2AM)Nn=c<@wu5uYA_s564!$tu;Ah_P z4wj^MFmpcyZJmQ#nzEK^*3z7{P{M=xU*v2L=XtIeW4;SdlSSU%mc4yu%G=jnLGrVk z-;FW9Ky=8W2%X9i+Bp@WvtIKNlDU&cXa*x>8?lqpScTDK6zl4d&K2Rh=&CX1GK7jO z3fFlYu5YEn)y1QYOxQGBeGo3&h^?kZs38{H5jt_M5RKPqNB@3|xdCw_i(+>v$F4IK zyTz{ZRFi3<#;%WJXK8a4>+BXw&WNKpI*6lB`JO}|??w>GqCj5Hfm|mQ$TR)s0`_cl zRGTul3A62*l&>b$s2|lr%?wEx2GU~P91(@M7W>qt*Q2ObCX6wUBBEqbRPW-bu9=GJ z;X4-e#Ac#Kbv8%U(l#k=_hY}`V$BB?8WIJGaxr=y_t6;hcZ8QL3h^Tx;uTUM-eox- zVwr`j0?$GPt|4xHh%MHgES5toCgupYL`1^vg}UeCG3HIgnkHUZPXR-UokyPsW&!5qPpF_!Bw!pGyUQ9aM2)?ulv&)$=*5 z=ONj{;9Ja+EYfXT)E||R3V`5|a2b#=4+2dVA>kvBgmIswknrO>JrZPJd656;6*|z zc=2P+#Q%&jUj-h>BET%ifca|*FhAee14h=66flDVFcuZ77A30|oAv;Ygtr8bM^ESE zF=lOmgDgVN8jPNMQs~)f1&ECh4uuZo-i(6rvDH7FAgmW!DyE*S4s;E<-OORb)gj z&);Ug3D}TDxN2uyU7o_#!HaoZ$pW29)S#BAdR$o){Go6gBOyxLag0rHPz$n23$&S? zfFZI7WScX{&QF1Cvj;uMWLZ!_HgrK|ae%_&1cgOyDF7#!5WxA&>x9p@nSB5#vIucI zGU858A#UA$JmO?AQW2+!Bg9#pSFt#-Vv7?i$lc;}tr%7J`a+vI6zC$0;I}u!@7NUl zzPzf3pDtFH<+8JBOLn%3KF`)_M%7U8Likw})LWbnvN#|FAr!2XLdY|Ru?x1D z696@`2#eomEFO}=;>2q`7G>R4rBJ1?UJ64jTAWt1IN)J%9?RlD7O*MU5o{JB*||`g z`6+Nm76J2k2IgKVV2;8y7}(LnL`SEzLvc#`nMJZ5m=;F`EzSyB96utI3Xp11y1|ld z=2rkAS%lJ`FiN*ip>)y%9;LFrtn#Vy`B~(%2Bj9~G%XHlTAb7bVg*!!SU>sMW~nyw zJ0OxQg6uCDvfoHS_Tt}o$jYj=LRKN$Cy=!`+i7t|*5Z&X;41(t;p_3c$H+EwGvG-U z;rB|$@0b*RciY$FSJu)Mzlz^J$FIdHZ;PXz76+7pWdU1&We?^BmTohD0#?Z)FyF*r zZb^aJtW$s)%dDE^YL*+UeWsQIrkUl2`9mniA($Z0OaXK8A!3p>T+{OIu zA|U&50GBL6_I-@()l$fQfc`D~xNI4yOsh=KQkkwtw#CH<7B?T*;_8DEh=Ou2;PZT? z&Abe-l0`6ojA6b^3g)Z%wOnkEser41&j|q!VQz5)g~b&V_=D1x5|wqBzY3Ut2aqO< zF#i%`e&H16=f^FCB<-?2qSCI?KF6fJ2J;p-uUPCQS=&Z8QG`zR4m2 zyv+m{ks`p;CwT(Mri>Cm3DExtU~wOg#f3N)XLLaZL2Ez;p2M!zX6DC!3t7a5|9Cd+ z|6z&^7k<&RLAHUE4a$c8%?68GiY%@vvbd(m;?O$SA?Ou$6qDc)tGAgigBE1<)R=2N zrp7;0)Hw7mPYv07QfeqQ1{O6e?vAp!JjxZ9N1;9{Dv2t1(z8aJSqU5=i+Hj)^W@nS zPY&GA^F(&MlqbrQfy)zz`@9_P^K!V)3qy;bT80)b-CD4@%{UN-EF#T{Oq#!>NOLZ3 zkf&+5?6WCplr)2bG!B>Gfjv#I;<*}ucs+yGY-uy=fjwjqgDf-X&J=^baj$2P?CvRp zltF`#K@PVRI$V3_aP1ui41s&VCchAHN^6_x0hP!iI<3oex;90pe_vkE2}R4g7ovM1 z{c0~HTo(?P#p)LV!r4xk#nlW*A)OpK?ugD^9 zbu+gvNpb7C6Fj$cJ6+F=;CNDbNN%;$RY~v z#}qs&MZqg~^c0kBVqFZ;#gIY17*aob3p30l1s!fNcJ%yn@%}T8!A5B)4~a|e(`HTr z7s(<%9?pC`AjQYipYnW^jcMhh@=^I%B%(D#V#r5F?`+dE=k(5banUf2sMqteSdQ6p z-!^kLh)EWa^F$`+?kRE}aF!>hY=&qhv%yPqBIes;L~8F{U6smyB&rY<|S z&0GYQl0{5CiN1 zP?jvB?QfX28>ML5{-&p`?EEWjm9|RTzM-wd{rL`eqB~q|4GIg#0t$N$KmPqTb36D; z7IF9w%;61E96sf1p2Jdapd3~XD~J1@!wwgVJKO+`Kj=U|ZV{@V@@Bjw_0FMf=0T8| zEF$!6Oz29A&}%GS5E?fW>E5XBjq2X$bS)9rFG+>7-Y}OA2jv<QvWVReFuPYtvHKNYPma6Cl-`z!(tDG@kff*<^)a771y9sG6h?-DjX^tDjb7F zI2@`B9SV~iY8N0P!ty{wyoz-Gw9RY?1tE)6q={AJ)|84oP5%~tUn-zfMN~ypMFyja zIMhZsY?L}|l(^!>xsvswD#hx@N05|FAPi)Yqn`xc%pXs9NR4vvtw`2QMUmPYy4k?U-%s^{I4n%8sesn>b z*%LxT7KzO^EH*z&iOn%{d$E!FGMyjk{7C0VLt%bYlgxx-9@=fhT6?#M#Y0}ZGNn=x*^wUkeCQ0$0YLaS_YLaTwbZC;p8@~>3{JP?eU*u2q z`s9J1!|X+K>g>tUZA98Pz6c_N(D*<>N5(I!?Vj!Dopb5?lLBM zq9^4fuh_7^8ftYZl!`1;tDm!4ZIM!|XLyZU3KdnYRIOC4RIO%4tsGv7cX$fh;VEn= zmuO3=Tt3knaebS)9#y1G)*b@v@U(US5=l`fSol`fU8zLKuIu`$c>I(d%Q z$suE+Gl7hG9eW5mb{8~^EYh*-SjWDW(y>2m=ygoXu(hU1YnrsCiE5g{GJ|lluYQ$1 zT+ynTFor2usAG9!4n3>DnqbAmQglajt(aHdbVHkY1cF8uiP{7fwZ@dFUG-WaYS;+S zLU1hv*FtbD)SQi?mNzy-+*)8X|132~1kR^zl{?$a^H4XkNZ}r4h09V3H+&tha8i4! z(@&j#>hx2mpR;26S)aCr3YS|cws`Uudz@3>)|_`jo0$lyBa7tjIhMOGrR47S z{$3;Oo@zp;37sZ%n$XRmgsw*Ja*OG##dOxv`)Z{)m6@JT_3nZ8eFWVji?r_z*1maD z+D9*&;-Umpji~mi_Nn%%_VqXI%WV}Jh*__thcHT^3Uts5;XV(xnYm3v16d@5|Mo&S z_S2LQe*Cl-LaFT4Ij_!nb< zYBP&K5y|RR(HK_6Nhwvl;cBmnQctX^sH&){sH!+1R55p$LOV>MT``3g5i3(iUuL{& zyIBtMNES)t7g-`-OiAPp=X!~hB4(9Hl}MFHmB;}nk-5W6+hL~dikY?uS}B*j;<#G7 zSp!;07U|`(te1aF>E%Ncyk1J_wCbhmrRt^X<$%`9++jkVH)i;Q9`3{`k^0#g{GN2q#%2o~yHX-kTE7tNzc6rxa|fc&d1+c&c~~9`Vd`2NA`@J*Vf1aMUgqQlExS zUZdT#Lq*9VC2eIT{bNc=PkqlTsg#JTlB$xblB$vpP9@EA9E!;?zt1th&x`rJD4)#l zi`CERa{odvq~kaG-Vas+g@qTDlJ*1HEU|jnrw#mwNdGU z*R}DkE}qrJtGakp7Y}IS{Y*TciRba~IvyU!!+TeF?h3D6;msO6S%VjAGQ2f`rzY^y z1Rk2erSP~D9yh|{!e-ppjH6AsKo#i0orSm`3|H{rUJD!$$Kguc8-_}KRIj3n3k3{x z`voq)z}**^CSZWWQF8K^#d%DN^OzRrm@LjQS)60CILBmB@^4Y{Z&C7ZQ4niU5NlBo zYf&JEKkye(A!bn}VNoSvQ6*uqIcTvtXt6nHu{mhrM+-+>QkoNVD` zI}-&(nT6Xe{BGfR3&&e{-oo`3zPE6`h4(GoZ{dH709Zu8A_NvOun2-h6fDAkKkye3 z35!rz#KIyN7SXT>hebRr0%8#mi-=f+#3CjZL2(F*Lr@%o;t&-40W=~i4pDK4ibGT! zqT&!0hp0G2#UUyVL2(F*Lr|txP?R}Dr7x~A2R-m*M8zR04pDK4ibGUB>ow-Zv(kNW zjd}4-0zH$Ex>X&415j)p97nrrK+*PvFiS#u{j3590OmDfsgc7!7Z*5o7O!R{Znvr%rtV$G;+-U za?Jk<1r~i21um8pMs~EDF%1n3WcBLX(CpRsz?)O*yUe1jzQGx#PpZ3LjUVmB50XVc zx*q@NHK`x{?P2_*QZ|d+y!fYRaeHUG*$O{Q7X5e+|M_&!N2c0v;X&L6*&&#E8)&rz z^L$M6X+MD$Ql_$@;GOb&H0WeBhAe8(5!|40sRn&*8*Y#+to`o>(aH_2VqjeYOAbZ* zQYSc81l`klG=w>QFpg zov2&Dr9R0QgFY3=e+E5z2)!YTdUQMYC{Oihug$nerPRa4v%E*)k~H;Bz8>_>2W-cU z+s!lR8d=o62f2GIr@FWPO5DA2u$kT63s>B!tJ0FNtG=%bZ_;jFM?cA;zCO);T_V-j zF(4*RE7M|kpDV-6?k?0cEAIGp==i_TX|kx}uW`p0NOk-fUt~@z{Qa)ub=xZB2We&C z2cLFq+HU3=*3du}dEo=^g##w1yl@@Q!!cO)cQ1sSOXLx0ao`cph;_GXHw(cXWc4l? zHk@7ZZptO&S71g++AwRzoI1G<#Vut2jO@3Od!+TDd%S-p!#_*IH)N527G(dtlJd`C zmwW$UOd3d#nZ|x3+*Niaa96>v6Sr?StHM!ak+Vj!vz|yfYw=&Pvm`?raA2C&<}GdUFv;$+>Y&LQ}~lC^67@`)2mWGz5EyKQ%OLFgiq@# zGRVc^fWXCtYF2h?H`~FvWRZh6VFzEBa`3*pvV$e<9f}SPOJ2y?;*7xA9?p~C?eD_V zWRbVGWpAIE^7e+yvbX!;6re5>4s*b14CsAXU__o5$Aq5uuD^Qcc5^V?P8PX-Pj>we zQ?7qvady3=_d^}%$=WW8gLo;9!&R5In`00IvM3G*aU6b-io;pF`idFrAjxU_k^6_A2M62{>5G^0B5m&UE%MdEEC|u`pxW1JN*IxH? zxMa$vgqi_Dr7DRjbmG}CbUt?1B6c?*Ze&sHF6G#Dreb%(8XP;BC~EBbFm_QDI0aK& zEx{~CwY{%wH+Lh5WKkfm=RmHL3gkLp=0M8yR0BCH0~wa{Q&7e25>!vm<$l+09z{gS zqNv`*QC%|?)%X3`H!Ahlk+5$@!Wk+crU*+2$_V=qKZg+i9pNR5Li`Aac!gAm|G<0b zsISrx&%qFf=Wr<85{@9;KHleD)o$KItjVHyKhN=AEEVsA{Y8CrYDeSUPvRY(yQ0`j zXu{b0;O}#5yZIP_CyRnVk%RxaRPcwLgnx^^AWIH9O3$uQdPWZ2)r6`9Ly@noa$CEZ z2Z1JwknoX5!njXTNci26j09QaP$cwcB-9+gBv?q?>R|CWV6iBoPF4?#;d3)A-b%sZ zj24E4EUhRk1`I61Q@I2YiD3y5KZxvid%Ia45Fm>%vIt}3g%n1{zu_^0(@vT}^ur8d zhE8q+FJ(Lfym-K@a!0%QD)2xS0cJS{%wJQ0+4VvOj4UIms0>I^Ngeek@JMV!@ObpR z3iPZEaF9jlS%cAYPYOL_)@1a^x|O15P@$*(q6k8h#JWUNk@TK$XS?YDQph4Gt;JBf zF$JZSo@FS>YMDZ5P(vxabA(_eaW7%zC2I2t?dF?+4OxV%cE;7^DO}k-7+11Jr??t2 zxT?9!hCn7Ds0Z1@e`z;60YhXF$Tnw?ou2~PKie5(vNEV38=4>sFH9o92_OQ%d09L6 z;dZkR07VudZbwGk=_$lr!ev}IxT_Pr!7|b7(<{0NeFBaUeI9;0KGJRu1-i&0`0dT` zJ2nNsuf52Er;F8kL)>)T5Jzwnu#|8tW)90f+HOt&)W{+%exI>;ND7PQO2(oryXxSp zgYWd-0!z3Q;FP%ZBKb0a`BUJIECS~749vY!z&znv2Bs|bDliq8(+fdX!%E*LL$OfRHRg=}#D?+ow>v@Ao`PapR|ElA1~O)n%_hY>B74u~X+Ap1*(>^D-7?O2r|D+}8ytSYQCB&@0Xi3z@ft`NQ+zdOI!Zs@KnvIxId zGJeOT@Vg|RXUFBrD!D4TeI>cmy8D&zENCn7Tui^d@>09`6R=7af%zr|b4v=$?`+3n zEDQK5#wx~rD#p{gES`WZ$SZ;EW%@ZF`*8r5EJF5ujO^7?$o|^`jBMF8P-H8zXB)D^ zXAlVGg2Djh9_EL=+-_b5ipe6FKgKX$CI#~caTg&K9^)Mi%|S)6(DY>Fql{oD^Bcz#lOqC6SUJPF^- z1ZPS}hn(@GnfEIl#(^+o5ouOr()=Ywn)%OW(#Teu=3Sb14V=7dx*nP(1_{1n2HgY( ztq1myMGUgcpgU6x`gkE`kZkTLgOovoj6wBJH4~!*?Sk_TlH%4g`!TmL`b*H3Q_a?s=iz6Gl0H_Oq%DO46 zb9fle(Q~4Bx2vF_5G1DHxN3(v0`wz`D7YU}@Te38kJ^g|KG`MK3{x{q9r!T#1r&66 z`OV>}GDlA;cbxJVZ9@o?tj0VzJNjkn^dva~9D)5=HXqw;au zd~|sH%i-ZEhewWT$WwCD=d1UDoM(fWWDz+}WODAFBInzDh_Wg>;z~{>r;>B}$?5RO zmBZ6X4sY-fQH4kWQF%RQbceYJEG3JWdKNQv>l9OOnb!}a&01Qb>p57r$zi&lgY}$g zrnTWP8Zy=4ktK&Wc^qDC0c%TAg{<|o-DXUOxf+xui)i~BrtL;4+O|E%w3W?&9bk2U z)d6CUOh=N^<*@`~NoXFi(ToWD&a`V0N#PV)soeGrOgo4 zxJ23Eidu(DKZ*9j%8>S+_xFMKAAs{@5$|7R-hV#D`vZ4i-b)pU@?Lqbyzd9RcepXt z;g(5<9SVmX3eo{#X{iInvfOT)c9`KK8XCwV5qO_Ppdlp!hmnv5vPd4j z!1C~qlstSqW!Yb9bTnbmgh3OA{+=*|^5Ae#&EcS$!$CEY3ZZf!6<#Zj-oC?(f=-Y{ zTCo&s#gi$mxMN$NO0;OXou(3+N({(UBGd|pL#+oC-D8;}Ze(EFquh*CMJv+>{&=#^t zUp8ia`E^QPjy|4K7Af{peNlZ;eHl{v;!up?uz%yQFiW|NXhEPgKAD*evDp(sLl%k6 zHY_$jONq_5`BIS*dFKMb!W-zfIN5)Ng%gTLhTcp(Lhevz0f;ehArRkKWQ<_ed zYuwYDPSvQDYegM*E9Qc&#dV3Lu!PFRGeJn#^^h&HNV5}S5O|&%8 z(nL!Wtr<h#wx_E8RqAoB0%B%vq;+xeX7Ge4|OAp6z*YGxGbe`3$DcqC*`NA zaH?>saH?>#Q{nPPOj9kUsn$Pvi!FmVpYsXbZclfZiI6(7Nba6vx%*N|?*9EI%bgUx zs@$pEsobgD%`v<%L4iI%|s?V6js=73uTZ_p4Vr%tz2YvPk>hVC|bXrF}2$!rCWg zwyJ%qeX4z`ef>iFa@zzA#H<$=aprie5=WD;2ODW&G3VXuA01|H)6hT`3E{uJ5RUyc zC4_%ooP|(|dsPTk2vrDG2>YcF<`$DE?J$LQehMuL7HyWS`EG|<4)RDAN#qw*e24 zdij_43cYkFK;$o|UaDTIUaDRWSiQ_$3v{zJ%krEqwGN~$%r(7uj(@MitOemDi^Owv z7SDTA;`tqady+%twu+~oY11=pdZw)p&$QJa3<(RW!`j1|%A9bkx_-MRT%`=><6*K{ zBc6GVBPls1?l~szIZfOHsVl{_*zZ}Qq0_WOMad#1ZDl3>V@gSvIgis&sS?*TRMSvR zLp2Q@9BF91lIA()_c`YGInD1Kj{U{+`$ASHK~^`1q>@Fl+QzbaMM_rhJCbEpYSUF# zRaRA2RaOVTtmZkE0dnliSNtk~NMM<`@(SvS!#d55&{wiZV>e@sy&$ErCvVOgD+TeY zv8u7Ev8u5{OJnm277Hp^EU3`KUk)d~}Pm)ES-IYIka_X~p{?30Er*o&d4T(0Ce^DfkBPDW$Mq6Qto4H}nf(Av9mgJfwe>-Sj5wdY;-8pC(S=73dxOKawTDK`3 z52W9hS?OnP9m>s7ZH}_a3}uxyKULq<$!O}uXeL?I)U&y%Tcw)1?Bm>2N$6&-sVD$N zaT98riqa+rw@)QcEL?ob!ky+Cw3#ew_eI?9-c-B)i^vo%78&fjcB8@o1qLWDzy=w) zUOo}Ho~ivII&lX&Ko)i4YVO4PsZN}6Ja-~c+YC8&R>`Ip#Wgqyoga3h*yO{=k#RUkVQSZoqLq0dbC$J_o$S5pm>(|2g&_o*CkSY-QX$?r3`GdhGBK!O!XD%53-*O*-wRzCORniRATIdyvs73=3nSE zS=8~@xZ?|?I{p-tl%xfb>JLG#>3F3zYii7zY}Vv56r|L&G5|oqfvom`^#_;|V{yHx-K`xuJrfSwyY`0Z|M$NgD=)OPX4;rq-<5m{p763^~PA+~bM!5d5<=d_xxbXF>MQD=Gi{-zDsy zK1zG0b2ytW#h3)3V*{4^fe0s?gQcD6lBz%eowDF`ZUbMv$ z9bT0r7mEV|7Z^B-bA5-o|r*ctZz|QQ#7H^1L`E^t^Zdr*Qqj za64J#`aRk8KTNs)vCIP2n+nQ!tfbQn9ZLsFju}6hQ{%yaI7NhRK!kpQ=#WJbI+Y`|b1Fhd zEWr_y*^|;{I%#8Z-prz8*y4tLi#iL78}=^f7i zJ7yHeP9};%!qqWUNzN)|=+E{^J&si?j^iK8krVvXwTimIi} ze%kH#+x=K5jOnvTwRIvOTB(^TM)xM7Gb zcI+&6e=N2*h|m&_lF+9Q>^Ez?`$@bl_Gv7394yw$ zDfSYYF!rA8&EM@bA0zN&QSc{n@IRLd{@kZ|Jx8U12EQK$-(q#&V)@x(Rn%gA4qzxl zRf3_&*X{=r=0TvzA|!m|kudI)6cVmDiIE_S96GYkiIF{&{ZIz#kzlb(Xt5%NKZ@}I zl{8U0P{3mQy*kaJh&ow4EQZg`uy`v4i_=CkEOc>-wdJ1vB zUW5@R>ye5$MO=8B0ijR85u(q-@9?8L&7nXSSp>hm8Ggs6;P>@cJ^Uc8n)nZe#6Qf| zYDU#i@Iv@m9O$q(9%OMo$od09C0I&07Bh#D<2%g>fErnZ#qTo~4@qHhzN;CFvh1o- zs8TpBELt3su{e%paUKgYiQF(T^*9Z{{3&op76J2k2IgKVU>?|6 zNkNOFf)X24%y5V1vaiMHaUdSsZpDW(agk zX83_(hub^Nmq81%dTPuyA5-I>DQf)eMjj|+`$;n;&6N6Arc^&rgw(LOI?CeiD2vOZ zh$aH#fF_0XQ zwBwzfW<9WnEMkyl2Hlxr(8r(SVMjLiblA~hXW$PzH4JjNrO@HpJBMrU=shfAl;FK& zRIv~+?}Sd%11gb4bXu3`bZv@GFWtp-k}XB0lhSFB(}|vUqbJqq*)pP*V85i6Pxc;q zpwny#UXexI>Sk_TlH%4gdos6lJAGi^UlQI;F|EVmGkscLehEGO$O`s2jj>h)_sRrcS?$Nuk(AcC??jm3|-3@ zoNF1=JG_OgqZbwF%|nM9R0|3UL1GI2;-OA+1n5T=QE)$|;87_G9yitxe5mDD3MvJa zf>Q>*8VWkxV(f6Mt)piU{jIhoVMj1`B{crYuu>GRcjpXxMcgP3FyIZtGA?w%s&2Wv7p zWk+1espM30PCq&6l|FhQj$R`pq6(1$qI#yD1*TpEmXbwGJ&T#Tb&9Ds8fL2O!z)vj zsmjzDV=6uFM-QIS%UQ%)Ayvp)PumTi?lf0}vSbl$f5WuhC`H?^KgG0_&3~n>(pG6Z z^R#uiM4z73a=0L!UgIGe3&~0v7cFImLlKQM0VL5EUZsjgI?LsX}i4(Psl zhfBVR_QJ}L_MZ36uXUOazGo980byyW1H~-LBzj)xpmi=6SEvjD$3hMe^_k zmWO|&iMHhR{0H4KEyj{X&*lfdM^RtxLeEVxGHd11yVxwZCVl!mLhDu6kq<{+pJ zS)@FBu=4yUr93O&#>yi_ZK^z~JgPjZJWw9mSE40hTEubw^-{dyIQ1i}$8XwY#zTV0 zA{jc6W$3V!41M%{FGF}8PgiqvHAh!-hT>{YINJ&npn4gic^IWc&OazyYDSP86`NKa z-{>-@LXXHIO*)!2X>3Z9o*KoPB=vim8fj{zsgb5eQ%Q|NO>$5qhc|v5-uT7j4gW1l znKD0jJ8PRRa~{NrEE1?6vq0^V5~v6M$O0u*gDOxeP%2O=P@h4d93CNuMq!eNcb7@6 zL`;Hg%B$7R+jf~tp;Tm%TK$~WYKxRw9m}^A;d?s8(|L`xGbO|)h- z(F&!@wYFwCUMJ7-I(hD2Cr3IKO&N<7t;%j)<}PR$S)^mvv5tK$rDIPvv5rYirs|mL znCh77*sRd891q6ljTM-mSc>Qb=~*FalOSr3K+wn{QJcV`)|e8tBQh2>DFjtfQ&Ce< zQ&F4MqLw#SA#TkuKmQZ~3IxtiKd;)o%RCQtBa0O7VOF>-rEm*6RyZj?RfSW9Q-xE7 zo1F@mH)5JS+CKGl1Czb^9;bdN03zBgF==1pneOWUyaNtvx`pK70K zpK4#f(7xO@K?5=CwV3r-{YLPyy7OCHI4Yw-7RlS-yZ2CXw_ok8mix?j$q!^AO^$lPJ3?J(1J^yDO!1twP< zmp!b@tO2bgi}Z3?*2}-8^z!~mtd~+ft$L|?sd}k;IbiiNcP-G()-20&y40E`_et^m zJH&G>2q#%2o~yHX-kTE79Vf7OO1-vdJ|G zvEQ@e@m;1JDoPe9X)7z~A5%)Y;%Bsq$%Ie^k)jY>CK#qO+ieCi~2@E9G zYwV&w>@qt-U&$hk-HbK%f|SPoa1+*8DTr5%RgG1RRgE258k<+JSWv-YL4_XvqJxpi zRoBN)>oR*oY{??Q-GK%7jFjL`Y-hohGJF+W6GxX!h!R;LR!ZU1kARUn%_0>?hUT7d@lP^x_A}q90w4 zfApHvk6v|9H)R6kqxn zU)l{{A&b7WC4cFh)R#_d<}XRPhKx)Y5pjrYDkEa?+3kPQWyax?WYK4L<bDQ$E|2RkP--C2P&#T55y*6~P3YP_*GwwBbm6pDb#_{@jKmQ*HS5!Q2MfADDU@ zaC#ViKr@P?babF@Dn9WI+VQ7d=43R6ENajZ+@Nu(2CcmVH%OM&{&$0LyalIQaJU6$ zTTszW{*r$?@K@ok7tp$M&@!^9btiG_c1yLcpb7oH%t}9V>rifvYIBrTW+fU8 zxD$ceX2`L#N>)SdkXCgluE9y@{ICkZU?#Y0a7% zvnHE0xeNzH$Pd!W000Hlhdte8<{H+}Ko)u71Mh_cCa1h`&DQJ%8N>Un7n&fKO<7Yl zYbv(epn_9DM&XiE;F5*l4zhZe3>(fac{k;fqgG~@NZK$UT+-B%HMM5d#;jTlXBeKR z;vTOUBcJIqOT#y0k$)Cs|Gbj&&w;;Y|MXGXGo8a(DE`w+oDedhf2uC4=2^9pRWSe; zBPCLq*j)vfE`zgHg`>zKXN_cMJ&|(O;-|B-Btsf-V5)OgwK=P{WYyNJxv`j`A_Gi# zuIJe<(+sbXMV=eQp8IpkbMJ4=o|C+6P~!=LLio5G)Dkxw^dpI(*n z=@pzhVD7-s9}+&r1KN1f7BAXji4Lzy!o{Tn0v8vmIpX;)vmKmE7CCqmcJPHM2amFPiSR6RDI5$NB5)YOE^71uqVwd>^qC*x%=v0o-&Z!6;u`owSW=|TS8HkX@4f_^1 z>|0z$P02GR22`V`^1451>6!gU^p>szUC?R67}OQviZu9*y%MTv~X-QgBD>r&{% zvtj6b>`r;J%iMstkwvk)lw;SKirtYbaqMKGsIlwA*ja3?TH1i2jZN%q7Qqx(OE8O3 zZTYvl%-skgSro|YIgsn50$Evz11Zx}4dkp0q{a5MrR_9q(-K3)?GjW^&zlg{M-fr7 zD5`gHRM$*J^@I0)RIxKPgqH+TbEuj`zCM8rbCqx;45c-qXBbsWoBg!g@3;HQ5R?)2 zA>MRSm-#!wOBRLr5f1SRsSsa{%|u$1z~ZxpxSw3yRuhgO+&e(gKA*C($`7JiUE%t&f_CYN+j4ZadDE1PXF!nz9EgyB6j}dsX zDEJdO_@7G!e>e)3ir~wVg9g7p1>a)B&SLAyV&lYOrJukcp(?>pNeT&99K%SEMGi$me@24Eri8`D0se3WEF^Aqu-N0{F0&}2PF4?#;d3)A-b%sZ zv~?I3vb3VG7%;G~SV^{6wX|3mBju17mJsoS$UL8PndJcivIrxKFh*WTVPyQ%j1gJT zQH%^oj94tmS*-0?lphjaB%T3YJYWt1V7>}GkVSx5jsf%66kv8eg8?JUND7z%4;YIj z2a8Fm#l(@oBe4y^qv*s`(3J17)i3)LL zfvU_R^cI!v(5?~$rFQvoaKt>_W+z~XECSi)46^f6AbX>OL8c3}Ts;=nGtIuW_|(GP z>;sv_`3cJ}Y!KiC5CPyk;#OR>+w22Ckwu8xkr8)#3UR;vG9ymbBXzhOGQ(w!tc8dR zZ!;kD2{=OZdHB5y_#Fy#kwx&^o8fnC3V!Rp#PE|9P=%kuFNB}P=@*ORK^EtOtUn+G zS&UM=BEvg;wQh3)phgy9@%xO$LsD3r`*(~*S$0({Di)`OMT-MQ7RRwH&SQ~G3UC6M zEPy$3^=|W1;EpT;=J5>7y;8tD?nVZtEcPle6`0ctOpB9(7Dojw4ssMI6%^H@^ah~x zR{$YdgwmfdO1Dp;bl+_mrLx4VC{>is5K1kMXRRMV5PAbGP{uuu2wz`6dQ) zOA5^IY{X!e1$+gwf_avL+2Xjp#nEMp!{G#ML0$=LFVly&beqQkT(St+_c5|pOCkGF zJ~x3C4Ba{!(pyKVHP-q?u5e+YX8smt#$lRKFVi8iE$%(AxcI>On-5A91}OJ1U%s{5 zybKhRMKFJiVZKZX<`13g1sscX3Uh^dzj+=W%3eY;!u$ka{vAM?EW-RtjQNFAm_L4J z#=Pv1Xwsrd%WO+pLTR@+!e(*n3jRQDQcX-xAkH-N~xjL7;x0ExH`(>?kJ1PqlhK~q^>Tm*J(l_$!R0nHPKOV%9j^K!V)i#Q{Y4mrcKkhX5)Kp3)!G%GS`{*ofi{6ECM zMPHDuHl2p+G`xRJ!|P`uVcr!EE+J_gF2MtP@MH<@SgXNz%%FwayUluF4_U+@%M7|R z#h{NHm_f3+rwmdC4KfBf+*0Up?VZE5cg|mXhe{((vr0x43jwX0bekSfi7cYix=g2Q zQ*?UiW~P&DDJq?mPJ^6I4mS}x+(hVb6Cn*&g8hq#jU3f z^xQ&mpw5tVhBRnqNHxh`{ZJKh%i)$!hr1yi?uG=k0LW5XHG>1!A9j@zixUQ2(C!7dL=UHdJ(QWnzFiAhJ`8>V1s!g+b-2~m(KCqtR$Cl+nfhwVh1+zSlfXr?h>wRe9}h_J zah*2iqwGy9AC-^F$7%D?(Ie-$XxQPRVaiyAJi%~TEOmBn+ilJUG07rwp2+0fJw?uU zR%3F?j<}Lj$*JU=esVh80`744v%}rbL{uSCKvX}od>>4`2rMOwn0gj7b?X#UuciCs z=>xJ4uS`{@DpO~SsScO4JKV%q(w)B`9Y@@!K7+me|GLfX;4fLk;Xg2k zH%M{#joX;RQf{C_uMWLB^v?XyThC#KJL@xtOY5D#v>r$Nr%vdx-|sdLg3M$Qp>JbC zS5kz&{~#u`6geoNmC#D)Sxjh$cLW^X5pZ}%fRb)uUyyVcYg)@5+HIZ&v&kZMKfvr> zCB^O=SM}`1y&$>=t9!7z2Ro(mq0dw9n&CQRx5FzT4o^EcJnaCcmrM+p?q}DJgX*t= z=427opJJ*nnWFkxPcqe|phE|89msVcpN#`~NOgx-TpV88aCl;aXfLb`Y43Tz-!a|h z18|-!;{B`4`_HF%KW-Q1y;PAX@0It;`+mTChu2OV-kfmAyBsc6Cmj%$mO4=6`J;~Q zHp53WG>}Ci@IH$`LrMgWd5%Ru>Q+<)R0LE6`k@H8Cd>zL6S%{5%7r2bW$P5-g`2z0 zf=~mpdPNv9%qzlv@1+#s>zA`4NVSYAf+~V4LVs5Thg+!~F7S1@-jwnMp>L@Ue%A2E zTe{6iNCR0U4_{z;_(w_}J~^J{L27hV9#kGw9tM&;INS#7a3iL}C4eLqLghdzyjC21 zYquE%ogj;}Vky>&CsSH+<2P9=wA@bXcC>D17S`=d*Cs-^vRJ>uP&2s*Q;AS39Ii%m zxcANB9xKufp>?PmUN{o!%023aH=Ut!_6KP4PvmS*9QS|m*}G|A8;W1uD(p>Q}{ zQRi^!ki+dL&OdL4tGcINp)~r>-DX252w9{eO{^lfrc`8sS6M}*07_LvRYX-}Fsg{d zbrcR4MmYcC94`Kk6vZbdlOZXaKu*XaN%=ZU%I{N>a_;jiDN=T&lA@BLk}}vO#o_F* z!?8sC;io>r|6uAc|%i+|9Lvc5jSxPMkw8keh`#jKX_Jq)oMPjoJi_OncV)Ly|78@xs z(_}`I8BJz}R5BBajYEyOL&2#-u^9=E=t3wwUU?RMyxSZE)ggI>;|t*u#(*U59dPM-VM$suadl2)sXC@QraGoNHY;>2$Aj@X-iy!ud+|upB2Dwj*^_T} zn@1pMWRa*%U{PyKiQ1vdvZzTRsEV43nu?l=+N>6}ys-*#YlivxrwC9u|18qBpT5;? zo`<@TMGE&YD_oXRxaHR=6t03CQ8yxWBT_dabtAH@4xQeO$eMNjaOJFi`WdEep~B^j zn5J4xQ*H6&ZJf)&^mvpT8QMdUBxePqk0APqnXK zXkTudpn;h6TFiPa&3Y@?u!yv}C?_-`+SV?Rv^;j>Gy5K3{c3ZV+2 z3ZV*NzZAmUVsh*-Id=4Hb_F{hktq7Tj&ncmHj6+J$?8?n7*@qeDOG&r7FI&M4l_Mw4QM4~8c&^Ukd2dQQx4n(UQ|h%P zNmWTzNmWS)r;_G5=Jz?~_qm_ni|m!E>NkvTU8u)w4oM}8WVMZD^@^0N-hK$ns??^d ztg5W4tg5UIep$_PECb}&m#_F$0Fl5zV!g&*y>O4&5&B9NY3ydKu@|H?_7wlNW(BqJ zyv(2)s~W2ss~S7BG&Zkbv7my*f(pH59LQm)w_b4PUaH6J4Y4JQ1a}7(+%r;w`%ixz zPXz~&RB-h?q@IV=^N@4;JY;x4sJ=)(tPKsTpu#Qwnk}pPRo8H>AWUjQ!ObhUXrqEP zg$mXbD$QA;z@m?$z{N`KgHYcw4Gj%s_3GQu?A7Te)N(<`A4O2HsgPa7N3tF-wHoW7X5e+|M(Pww% z&z_w6>>n@Y&&o!pe6}g8X3bek)|z2@LT!-0618E)ZF|g-_&!N2c0v0h&G^ zeL(gHrrrh&+CUD@()puwc5Ft1S zZNk|WRCH5!WvGn7?Q zQ{|`Xn|c+R3Wzk2MNK`Mo4Qr1smnd#n~LuV;WMOpStYAMR~XqkN7ghI1)wNyLT!^T zZ7P2@XtoDH=k`738nl@#YWGFl?%q_pC!fph78#s57l-Xeg#ii-P+ouyGPoXbVAq6w z@SS)Dowx%XAd5P2HFskDR3}dJ&rntn=010R*ok734(cbnOYp^@PX+RK+o8ui zgx-)vJ-VHHl&5;M=f)hCQtE-?zJb#;dW3a$S`TVPIY`kv`FhYhAF#>j-ZSVLS=7A; zxqB<8y0@->HKGEv%+7#?-NQT#(=g1!kP}cKr6plkeP2)AvB$iQev(CfeVY5aM5?b} z|2>CN2DVwlusU$2`ik@i*-wV-r&1hEgvumYx#CVe=3nSES=8~@xZ?|?I{wtpxZ^UK z_MeVdTC=9ctjT6gF2mG_03fXl08lXbYItF;VGRvrkrzJjUN~TK$_v-+R$%-5xS(Q zC2MNUs*PFI4`Rd(YoDK}p4uhy(!{2hz{4BdzqFtTjyZB+KERy@n$F9$|GlsGXiJ(0sT&R`@8ToS>)|)+1qEP zyuJQN_O|5OL*3hWm=Nz0;t4Rk07jk{$Aq5uuHSOM9&<3weQ?7q(5q7<# z_v-r6^>}*?FB{=qqpWz>sDed}sSJ`n4zD5(#~=n|Q5+89IQ$?LhchqYI1H?5(DbHh zsZnwof)i4pCJuO_1y38`CU87?U~!!-@k6{;@}n39#_r!^PD4P*qM(fDpzNCp%J^M4 zC^D5%awN!69~6u0?JTZ!vbe>B0wf+R1LSElVqA~;1)@V1Md(zH(9Wp{9WgIQNM=tO zp&5vf#SQxw$Ac|Sxly#ln?baExXwkmE<>ouqHvwZ;rdo8TzlT@!-eJ3A)L2{Y1MSA zTu7)eTo!kSTihLPakDN$S9&%KosZq>2lSX55I3?Yc9(MOI#aPbc2$m@Oca%BGfA~D zb{1Q!mUh2rV-q`@MKHzH63k*$y9t538$l$C0(m_La-CElTNmR%%JftNIV%Hcu}N)d zFB7}M#87d&1l7}X(*t|VqlhS36xF*ps%xgA`VL=sjU5}6);Uk8OQU3=3@k& zEDHWa4*us-!T;|G9DG@F(BSu{;9G2nTWkSal#5uD@emj!R3#Yv1`3cc4+2dVA>kvB zgmIswkZ{Eh7zwh-p-AY@NU+$DwAfC=AAWq0xYfa8^rby!QAC}r9u~vrW>~zHg2ibS zhJ`GxC@cmHEG#yzEcR0@R`y9bB!(qK{2;R8Wj$tjK!7a5$Rdo97g87*|0rWb7IYLN z0}>+^I}aAC+!hO|gcpftfEN##2>{Gjfd{e(Fv~Gu{+a^JF6Vo|p!iFt82x055vH8= zSx3!S6N*X*7>o5>i-kdpnofYHjBN-WkDl!>?=fow9ApuC)?oD9lS0pit1)_H-Ad(V zz{^dD9*e~*i{%ZAIj_ashtMRkF40sZy^{f@4j_drg3?+Hr5jUF%ARB>$!eKGX;4GS zV)kTFb!~mwHR_Ie04ibSCF;~GddxQg8?p#j?To9-Q@Hvnzm0{WW*y)L;{aDLQ6a7@ z@}|sU{IgIk(k=l(J;>I%vd8QM43R}3+nhmmehOr-j%M+al|dD+AtPQjAhXah%V*08 zZ~}+`a9-A~yP?PI13-~Qh})47cX|qOzgd(KC+m@lI7M7|eGH*bz!9R)!*9zQd(5Fg z7g+?qy%~PTrr@{M^9(;(0af@Z{6hFyR0>)g53)EPWc>jl9Hfd;ykh3?KCpNKphgy9 z@%xO$LsD2Aei>s?mR%K#ip6PR(c*xS#c?c)^H?O40-Qi53t*o0#~$-j;EpT;=J5>7 zy;8tDjt{zm2|D!{(o>HxE1X_nTAUQLI4Wpyh^s)Upr{_D>)q62egzPcMJW9VqjdWe zO84c92%*)AQbj4xU1kWS7RNL#&S_d4)FfRMLK4k#0z1#Kmsi!67+dwR^DfK{>x%r`NZTT)=2gzMGl z3NPf;D#j|tvqFsP!EABd-r^*o#o_Q8T;t7T2;6Aq1`^A&NQdJTLW_`R8hAAdA@WAJ2yUKTNUVAH1E7rFLb5 zvSC25!Qz%8i(85;Zek&32y{zk@FW=2_%digR!@z&=3{F7GewOb^Cl*i_LUk+jR8ju zi>sq7?vAp!Jc?){Fb-(qd9uaJJ!U0vge>C8;>?q0Q#{$4pNqium-0k;GN5_laLJm( zeO?atc@bv>(jjL&X+8pJ90)@ek!D3E&0kWanU~-1#(uGqMoBYhNaJt`9@v8?OK`^; z7*ygrX3zz%^qBR)9~O2Cqo;}et+qHM!0YcNCH(=|PkPKr;38SX$HSSA z2c-De@^$8;>`f~lm5<8DY4g$1Bj>ni*x{mK%2d-{FFEhYQl3zaYINSxMt! z27AI9z2_34tLgP4wu$De`!4qKTmy0 zs`abA=0T8|EF$!6Oz29A(0@9b2`xnqnoDagt-18f=hF3qat)y!-VtzkN5J770VLhH z)gU6bPq`<8-A{wrWD&a`V0N#PV)udR9`Yh^|Kyks!KtKQeCO8RG&jscX-9c;k6Bi$2N%e!pe~L zp7-~(_L>jCd9sN2uQKmHpW^+vZJ75`MWVb{-Yf6>0q-3i*>QOP#9^t^;pG$30byyW z1I4o3;cNAp;UgLv$RZJVpGBY{B?3o2#UdbeD=Gpi0xAOiPy`$vp>cTm!{L>MLJ@?r zb&9aw+P!8$r~z5MB8(X36=A>kQi`xHU#X1yvvjjfH`{cxZOWO4S-VgluIq&}|8P1` zpBB_8g2Uq|4i5!5+_p~ng3z~A2S00gAM!8~(m)o;!xvZ{{*jW0565%DAT>IgFlfS{ z2}6HR7(#h)xWL}wmU)MpkVz_p%7IjPt+;E{}A$4A1%J%}Sb=LQ<@4j%x2U8$v}f2k$5(^yM|^%eK%KvPfSxW_|f}N?(o~&-x<8 zUaBvuFRCv?N?#l*IH31OWHY7w*+K@Z|0=&*dTrq(E|1qiq2>KDnw)v0hMSd(al(&ZprIiw}W z>*RUyI(Z;8rIZyD#<9Egn!BK3WRZ?t$2#`4l#ac$9_yIYWNHegDVU~Unu7I33RbUU zIUbD9@m_phycgdDbxSPMS9b3;k3i7KB2k;bqSlxawI8m*q9%o)DrzcfDrzcfvs%>h z#wx_E8AkKZCZuk$!1?Lt24j28^H4XkNZ}r4h09V3w|s>aPRdVJ;Z)&N;Z)&fr^4lp zn5J4xQ*H6&Z4GP@H1Zdw!&^@w9``%#fn>VF>A8g0kCuO#(eX4z`eX4!^Li=*t z1P#Qj*J9Rd{j67%s&1>|{_ppixlKa@StNx2_Ch%J)07ZCyD$r(6!)qSst~FWsu1=| zA{>Q&JgR>esvRea=HRz)c(R#jA0R8>?} z?DwjeJ4~S+rqIq$p+&*Uw2=?AkNiQeSq}0@7D?n6St4IdN#qSvYW1X!StU{>QYBI) zazIIB?l9AKm}xtDa%i^anFm1dntA0#JyPAJpA}x(+(9SiW?1iHJd|H$s$>8V_Cf-C98KI#uWZ{Ql)THKHPDtSwXE|1PD{t zP-F8777Hp^EU3_1#(^A`3k6f|tZ#p)*X#|kC5r@i2Nv8jQiA)>Zcb{Y3}2I4O=>l% z)ueW4CAFd8<`rDDQNfx*1#1eG=B!X)(MM6>Vo71-)4gU)Lqh{uz4|sZd-Xl==9Ky_ z{RLKEDg4juC)M3g!H@Rh2g#xzU5|hCn$(Y8axnj>6wYF(DE=v0{2_jPEBr87^y5AJ z;}@rX{N|JR$ED_M%8%FlXYr+R&-9wz@D;M?OIz}n&Pjdg#8vo9Qm!E*6OD-S|4Ka2 zfa0^C;> zENa94+=e4lZTQV0z6~gXp6*!?+Ef1Zusz->lhL4)(HOF*K}T?d#-$q6x(hc*me!{3 z6fC`AYKX}j&i3JW3r@G-a0@~OPf`2i-wxVWgzWI=dJQ%$8pxv7oy4u%E!Db>hH>j; zR{EJ+2iKu>D67m+R!L2jpQ>-_u;+Wt#b_p3)YP-NsavI*I`XgF)LAgUrDn^|2F>;W z*bD8x25lyb+IFmhzHE=><>HleTa zdP#x&L;l`t9zt)(q8{DOJ<3x(+OwT|R7yQiJj;88b#_`0YR&v=5W*Pu&IfG73%%wU zbd4pJ6SE03fXl08lXbI(T6&j6`IS7e4S_IAC(h3xC+& zdjT@oAD~_n{yHx-K`xuJrfSwyY`2vy4m{$4vHeTEW+AwPtllNVhOauE{ zRV!H)18^}?LY`uGd49dzYgUD$$RcNrWM@5*a@JyJva=*Z8tBfdHfPnAtlFA2Hx@Hg zlocgBx6UiQrWsx%i##`qJ@@C7=ic3nJtukDpz>U^&6-`-oM(6o9r})}E%u^UvqL8J zn$d6}S>(nlyYZHk8?RrS-6*Nxpmif&l*fDWc@a@4+_99Rp{pv-tt4h2X& zSO&<`rv1HM^9w|WEQ-*n9HE_45jt!!j*!frRK61ORZkm>yYwxto3^-v)#5TjiyQVW z?wp}$i8q62`EWgsa9xH_kwxJ;kHht?RJiuMjl(5VHVxNIhRfpaaEp`O76%k5bmG}C zbUte8WVhJL&grg)hSLE2QY-HX;#Vl zY{;wZ_Y>I-*IZ$Ym$QaLDbtAJpAV3yjWD&;53n`2o^*m!l7IYLN0}>+^ z8=V&WZx%a2gcpftDJh(CUevac`6}>076E2C2FzbmfZ2sNmQnMiW6eMtYeG>80b{Xk zWwAwKu?kP%k=Tad@j2^y?Hif30S>YVJ!>#}?n$9%!;Ixd)~!@-2E5#a=&{%tuvlWY zCyv6Nzc*)SS2VV!s zb^?aTB9Lv)AUi(=vX`bz&17XzK{hl&W|65|zRZIFCx8e5=Mi^Y=SF590E#R^+>VU6 z(^H81T`p$PI=BP}vd|Sp-4^L=Zt1WeXr{SOOSiRd!I2 zfDu6xFt`Ci1mV7Qx>BdVucu?kI!oQ>DX#zhWxjK&-uFG#-PKj!OTn))jo~Ldpb9^Q zUkE=7@L60BvbY~){RJWXzoHbcSUJ>pwV2ZYHL?hcM>7_`lfvS%H+n4Mhja_0FK%Jf zE<(eVLsf{XFek(^gxTHF-0xGHFI{jET$pr{t5U0-i8mji@k5lYWvl8Sk}rLx7WC{>is z7fLOzXs^t3c5o0di-9nON*giS7Z@>Z(#gxm%{Jbmok23D_!xc_+6;@wYcPMaXs1M zezL^{Wx}(dt;BPYR1?InZFqmZnU%{+kUZh~QxNdK8H{9ZK zI00LbR|4D1^lTveIRKX|LiQt!?9Zi;{S0;ap}#Mu28wJ&_F_Y}Mc)I9#s}7SJ}6Nb zpxnd!qWxOTt3WYX1oPi9%-2uB{GoFh=5kh{Fjtr_5tv)t2e;@zfqxWNr?B-A?HF^} z+gAIxn12G&WD(|HVa%_Z!u+^B81r&MqL^3AFL}&cG+wc2gkaGOgfb6-XQ=sJ^rwLU z9{}HE5dr?e1Q?Jaz((AG1-mtRXrhNET(Quz2w>3;$D$vOMK=(Owk$*lfon;KVhy|Z z0WD_5W$Nn4A~yWjvti6TDK`A|RsT7_zNNB3+0ZN4V9}+>qDzrQpARsjgl@?U|Djm@ z|60u2paoeiHI`X{squP>8s|-BYRK`EW=fhV_3RHtNDYhDQ5L z$Rg5g%%piTMVjR?q=R-?ap0zTm*!o)F7FCIE+J_gTC9OR82y1BYgPD;8MM}6EoKPV zLl!Z}GJ}4VV$iz-m_c&5r=K1D?DYQ6P8EY3x)eIJy>n=L=X~2c9B)TE!^K9xM9^s` zP>C#})7DI<+fsCTc`DOMjue$nN~b$1m~FUX*ETBC3g~CZcl^(Yf-`(Xeou7LG>4$XHAA1fR=dJ7&jUw3rJ)OtOfa zXD~TOrpWoP&Hcv`Rpv@gC8v_}Ba_pi3%En?XNTU;L{uSCKvci79B^Zcxe_cTi^JobR@_0Bh~$4$`g2|fC*7V|jBOcoLP zmrUsT6rq28G!t5i9F)*XXeIO_CbYvG0f#vP4s!%3=@#|{Nq4cQHE2qUnFeN)MeKfz z**z%5?mO3Ic1vl6vRm1$>|TuQc9;_4FxtUkv;#3+m>4kKudZJN)n|g{WD(V0V5+a1 zqWT4oFx91?L#eJ*SE?@|syj?^ahSH@Fl>WpFRTn{&t+D3x0tuVd9sN2Gnn@)r+7cM zi+L|qB+7f`z4E>X@ZMp_j>G&DhZ72iF=3J6ebKn$Az0Lr3SR=Ful62S3wd+8`%nk)&+Hl5$f@QZ9UjB}K}vR8mw@ zR8so7q&Rd*bZF;>fB2=3@W0GhyuN%0ec21zLKf*uE9=Y8Qu;Ff7}ggl_ELROeNlbs zSNh`6TFYTDqeC+h+GY_g2(-qF&BT{l%pnjOvPf+9WU={CN^Ev%X0eeHGZh;Z8x@;= zD>e=tQXHBuIP^y#;SpU3g~uyT>mORoQBWPSNO=xs<@tU}c{ZNH$|FT>sywPZsywPZ zP#%XH=nj`w9qzU{KfM%FhobGAVuScK$k0iUAhJk?j$|1+E+s>=4`3ORN;;Jxl_8ZO zm7#8A$l)fALlwUB17$G;Es`Xk8cmqqVkSb5$RbTTku_;_N|R<~g(g)nI7*Mc_2^rV zzWeaecX()3wZk0l0ELUP+SDl2B!`M$hf+R=vYgH=(2sEORwPXmX117%Ax>nGK%LD3 zbzn-Mp7DcyDwt}g0;K|_0;K{~?i$UjKsl8BK&i0E!`x+3D-n}Oty*7eG1o$=$Rf4+ z39HqvDYZJ|WKO4~R#BIDy2R5Zo-XnJ#}Y40r>fM-;XKY^YusT|ij+%)B~&iI5?tx^ z7IQmfi!73^YgoFznv$*!_hRYNB51ump|>ac)a{A6l`k&Tm0(i395%3WY%=9Yz4Br@ zc?F}Eqb;ps!T5tWTg(H{FtSL;CbN!xDWzl6w`3iYnoLc>GzHTXOjEFhk%HCgSPm7- zF)uzZ=EYa=pCV23$=Qx?wV07Bd~{MiwdDQ><`VO5ry0Q>`laIo;>ieSY2N*M0s4+~*H> z&gPc3g$kDs#xm7nnQDuXw}Bc(a(BfCEoK&^jx3V9Kd{`bnUcE|{>TZP6uoLfrwN@V zbehmD&V;T??sDs(dsxm|>Hu3o&KECbGryR9zS3%DL-)uc?fWZh-|{K#d)xPEsvx7( zR8LbqP4zU@ThggssC~I@fCggKi|$!Dx@VCVic*znV6o=Cys_1M%+%G9MMC(l7sAo+ zr-bmOfh&<;yzS1h4R1uN4=UO}(f+-f$2 zJd#Bc`AL?@Kcysc^8c|!N*%K%j+!`X;;4yZk53#!iOd~V+72siMv3I=_r~>jm~2*wXP)CqN{)qlj)i;f z7w#f;<-)z#+59_{bO)#?S)`=RtfY6Plyt50ISrL6aZN)t4b?PM)6l+=hSn-+o@0HV zV|}0d^}Wbmsj9x7=%V&k(*;Q-i)3|smeuQ1vU=ZlSyrVsU1e2eRb^FWweQPno?{yz z$GLpH-vtl}3?$ZT?0y}sW$-7gs~H8cC5r@iUl!bR zQ-b^YFcw@X!`Gx%lUhw`HL2}iNo^>&c|96!)MHPf9(xM)jai|GJWUp3(M7WMD`lCbmkKUU4qgRjRKPrW@ z8UIi5#FOyHcgG(li~jgd{Kv0K{qZU1_&<*N+&QOobN+hxzl&d5YwuPw0>45Q{nBv$ zr5~q$>GYg`Ny;_k%Y-i?YJ4XQ;TNXORDjU;;iUo?*~Ft!6xapDcQYG5ieUQ_pbO5&R5tKG6L$pppgWu{bdG zSLsj@*o{y815LmKorQ-XiymkkKhW6J12yl-4k_;+e^!=f0Akd*{@1 zzjH1>x5(hadu~)1puhm-1vnrh*UL`?u4h_3f|s}tFF+Q(#4Y?1UrxQm5BwO)3Vv{o zb|L0jp4Den)DCG?gW@Wjgw79NqB!Kkmm{NnX&S5<{9^D!jXw*vRplk zAp6OX{nQs%6T!`HV(h;yGw|a7#!Hh$FFuoBe5KTjKYtOwxO`1}&WqPKXAOh12Aegw z3%Pcy%&!7Fy)1tcV#cgH@wGsp#gH)kTq1YhT^yl z-bi)9(+R<`DYdO&!1EN8FLx?M{<>3=bwrLqq0hUR>23j_)_AN8KbEH)BSMPXW=Na z$XNr~SFfwMiFr^DOdgr~_OZ|}w4K0oE{q3f}?CD-nca1OnV z!GxGgh!J3z07jk{$Aq5uu0ME6tNAwEP8PZT5O)3PDc3){7Q0^3dv$&3ddyzKWFyQq z%KTg-tcc@OgLB&dAPy%Z24qnjj^a2Rn~KAE*K!x9 z+c6cp<37!?lZB$jZXw3b;%L><0Sp~%;$*W3rnp*yS$x%AKp-DP5XquI-p+yCDiugO zM>A!4s)1aTfwVZJwsf3^Q(zg-lq`zs102;YQc-;S|2wjGw zBs8xH&waGjyn$GgMe&}_@m?bp?;~&Gc+2WvDZL~pZBhJSQ6*zhCSy@1V^RNLQGG(O zm(YZ<_jt2s~L7{8=3Qv(p(i83qE^PDYy|&s1uP_PHLz%Xyw$9Z zsFT&gV(E`DEdG{)#o0{^3)xyxSo9iLSQL*~6v|l~`;&4=3`>akkH`#QWFtU;EW*fY zjFFd97&+lN#)xd_C`NiEMl6aPERM7-&Xfr+63+lH9x%H;(Q3W`Jdj0z*^mMAYzi<3 zp3Q)fZ6pOuuLq39DY?brt;I1Sfk$E+g2$uhQJ`l_fP*YT&*vFE52esE>{E;$*|$>k z^eOaM9DrIJwppC#Sez0Nnk3dGnu?_N{@=HnuK+1z5tP2jP`Wb(rR))glI)f#l=?K3 zEKV*gs;;dsyGHp{Vh?S^3$5nsfDKuMs~s3u*QIc^>E4Vh*`rfj^&4DS?98IK0=|UU zw;}Bk5R}^Gzk@FWWcvd{WD&@^7-T<9fovu&MXrFqk6LKmsO=#ewc*BH?cPn*ialJK zg=u$~Mb&->L-DEtnZ=%v<&K_JfCvER5qHwGR&y8tMHV4$KSta+Da2j51|v@PBURS= zr>upD3%eN*`UD&y`aJvwzSwGx0lLT{_>E%teJ=&SE&s^ylO0fnpTaMMpG87v@x8V9 zCR%?%2vzk_is!e>fyL7RHL?hcM>7_`lfvRM*D)4l+f}isSezRcEs$t&9n0cA7RjUl zCy>bkm?QquYJLdZkww5fm4SI^3YaI~#=w-#UInHCb6$aIaZ}LZs-VT~qXMOZqFR)$ zG^5pA4iJ(>C_Rr+x=#wFqlPm|Ws6x+swkZ=lv-TVw792faZ!_WRS*>*)@Prm0J6UT zBFQ4iUdE8^NI`b{H5sz9v8|9*$SyEsEv|N2T#>c7BTMiVbcOKs_#HH})ljc1vIxI7 zFn+g7;dku|7{9WWuJ~2_E>!$lT=KTKN^5aH+2VpS;aSjD;rpS4X>NTy=i%o2}+mpqMOz`R^F!>!)D;C}$%` z+V${C53hR6I65eM3CTd&y|l0LR;&3ZAWarw{uRdjswvE$yswvbWc-SG#e9#!yhY;` zi(ViW%|HnM0?!cto&fFZwV4lqZ?cF0|6l?PND*LD?!bcOt`a~A&?5<8(GJI=AC5&= z7K^qlL)9~oofI2p@NqUaev}Q$hF-x2i!MbLU5YH) z;}A0hx+OFGhvJI$+sxXa1z9aMmRW(R@p_6H6S-S2_LGzvN{wDe4U5)M7QLe^nnw{$ z1jYePJWpCTXfvCDBV-Xz)?%K#l;X*rKfu35KOl!*x>=-~MN4F}$fB{JMIT4}1DpTB zilR253O^wsPaK-8IrQ^#=;u`h>5wy?H1C5n4um0#NV74M=E)Rkmj51;MvmH)G)kJ@ zPa20Ncwi4kmY~NPF-Y(oGw9h3+sqKKhb&@{Wd{8!#h`cIX9mgPo-#-o)W;a)(529! z?VUs0JLlWp;Ygd8BPFAXjez$*+h%qGmB=DGZOwGLEk&o7?_fH~k)qN`>D0&R=i1D0@QN(r*4LO@SEso30^h{M!M*Mz=uSef-bn~oNL9&R z?WZc_mP40Mhu)A5y&;Kb!h?Wko@ia0x0!=LGO~zhyD`x&N)hcoKHEWcj1on5aF_t%L&bwt&UvQAm2tee9+hbFm> z#)+b%W?`Mrc9kbraClfcj(6L(2bi|E2Iip%jX?UZRQqGmMo&} z&zZI@DcWxN4AWK)|Mdf_A6TXBg45QaiM~UFbcY7%&NoO$rCawa*a<`1%vA7~EaLFZ z%;BLa4!<^qIV|M{%3`KI+KTkW3E+iufl9tWApB0~R?30mk1@LkrPzJv zCz;(+TA}P#b}PFVBfA}@ggA_La2V}COcy2wO!uqnsl(dLOwgPxqWTL=^>tHJKmWI$ z>bN7U|p|zGP)cd*=O4 zZRTxoo-E@14CejHDc&E>omgvcM5qOJ5pe`i>CrsmyztpX0p09bn=J`t~&ksNTwIbj!y3}C;rNi97LJ@?r zHHt8>rOm7YH6W{1gaJ!=ML7J;lp+kdh801oWmFMV5mXU+x*|A?333<{<+MI0LZJM=Vne&QTj)kccq6O)~G zZ!>L>6S7EBwqZ%RDJ3Zv&N(iTvMZGol@yhfzAh;a-L4%PQsW{AD{ znZ2MbWRbqKvcCK*r7z>ha>^pbUaBvuFRCy7N?#mWvO4sRbm(hG+bp65f!6qB=ClLb z%pnjOvPf+9WU={CN^C|BE5xP|J<7EJObfuY0IcU0fK}CggeSe>QBJr$Tf55=u8*pc znNVyTdiptZ-E-(-)R_gk5DJf1p3i=x%^U^QA&ZpfU{;>*r1CbC>5xWAW#n16QERBAHrc>#Hg08aSM# zOR6JPx>UMUx>ULrhIBcU734Vi&#||j7t_fbF&j48(kgb@2VCD~9)O0CMLIT_b?i$i z9h=t3Iwm!ls$;5Ss$;5Si$ce8Y<%TdPUgkD_(sgSjx^0DXNTU|W}b$ikwv0*Ka1Mn zl&GDs35%K(f~u&gsHv!_s4Z$y%aNw#khOfUUw?`Kh3n5EZQJ~=HZvXSMiwdDQ><`V zO5rxzm=#XSPgUVm;Z)&N;TEUD<%6+IwOFRwV&rWj{-1F^=M%cuAa}DMb!3s;{ek6f z&6M0NH_OW%ev5xVk-UEx)_O~PY>66sLiYf zMI@_LMPpbM|DICCC+}ocl#*gqMO8&rMODQfuZp?D653%2?fep26f9gq7m4E;FSnTu zA&+E{M1GPb@=qy=ocs-zNU391iBySHiByT~RT7yyth60g+Kxs}He%l~PAK^jS^J;b z%;%w%WRYHOzs0P6K!vv1yqU&sAS( zGhc*ol11XVIg97RDe>H$Z{T7}lz&0RQ^ixoQ^m7yh-aSTN=lA}dya*B?icPNb>+gn zShk-(tIg~H6(x(5w3(Iku9T9lb`GbZQYEfRs!FO#s!G~7l{C+>zR$6~&;9ydWUo|J ze_%BHjW*K-NhOP9b$gc8>r=9N&yg&vQk$-_sos=SzqOhDps!?+#&)vCUYgR_6L(~dm4bNHSk+k7Sk>76rLlQEHVf*pSx`^2j2lrA zmE3hb@SQd@3SvtZ3GTiuxaXz>cV?>>T-00XuB%SV&}kVuEu);4F}G&XVPn>+zK3Ba zvD$V#VH=C^#&^|y$nb(tZIOIf8yZ$Yg-85VM^?4FuHjxmnAC=Xo7bb!Mm_cv>anL# z-W(e`r}h3@E@0&vpIjf>VFr% zblB`RGXlRt7X8w2{-qzMe(Chj@h?fahAagr9`b*cc;EqwpM4HLi-B`>WYNza#6Nph z>Su5N8UL&tbjr^*WRb-c>tMHg)(W73-kGgy6QCEMIA5{{%1$$J`04PM@eQ6U8>bpTe;o@E2Z!@>z znaQH(zLKAN=hSn*e}R8)6ag&KG!d>Ze=Hvgjpl;g|Sw>Lt!Tm0u!I z+YC8&R-aW-JET<&imPxEIv;?6^YQF-%14KM_;O^lFI`M}!T&LiOA(eG|J!Dsz}t{T zZ#0$PC{Ml7$glAml~NBBFY-6SK0ECPHD`V|2=5!eo!8=9@Om%e)ySgPdz@cy)70y2 zwI07-`LS8tuZMLQmSI?hAtxYk%9F(4+~|Wg^BUfhEPB^z{I2Vy-gU?g{I2q0TX@Pa z7jTB54Br*$53-*O*-w3OH4$j;CdNL%Pcy%&!7Fy)1tTiFZp4ezmDXnzKXANX$J)d&c8mF+cBtz=$&Z;zKl_t#5MYHZ=g^JH{477rVD}A}$G{S3S zk>@sJ&;2Ilxqs~HJ%?{S|NTDYxkig@AG)4l7CQ93e9HiTo~@U{joZS7WRV*y?8bXi zZoG{rVBsU%*WHMT@|Y)&anG3cOfHo-4qfVfy2a3T(++=AUifGlM+DDxN;i#Ze)b0{qOw^M+`gJpocd~M#^ZZ1J| z$f5{MNEXpQ`&=QW4(7Yx*t)tz%fmoA8@t)4{ULzIn!>{Lf%j#d_y`|%OcQZ5fWy5B#eD8g@j4pVI;^Vha#aTBf+8=%A)WI|M1@jiCYaU z2JO{uR!7vyYGJYT#~2oWOTpso!3+!8T2WZ^8dzA=Q(2T2S(Md~a!3qIi1?4lRA6Ky zK!7a5$ZCv{ms1!y0o8T1J%-W@U9tdS(DUTLK(p5qdt)=y@oGo}s)&h#D|OPhUij zMZJYZ8Gyxsw8dd4&{W2{L{pLUcJ0${z5=9>MNs-8L+Q>Gls4kM4%8AVl==gdEY5H( zjtQ+VyDsBi!pck3>if2vuLCw@5w3P%TwRyK)#nbtzeTW>Jvz;rdu7&aQGRVvV{CCD zBmBVCN>qp|i_@{p;#9?A6TS)rwIDkKAln}pB8x!Q#UT4>3S_SiWsu3vpn|M_g3RJ9 z!1CR40-OLM037G-hqRl+04TBuar-gi&PgHe@>LmevLC64Q^bYsV+ef$ju3qwes2MO z#{gYq5&T9m{JxihUn8!$6<<}^0af@Z{6hFyY(ZJ9JT2A>)?X09)fZ0fOE?xQhY8Uw=WDypRW-NXug~bnXLAk)9Y`ZEJ6^nDjqQzps;?ry)kR+1=oD!E_B<-Q?=7+!? zSp>{e8JLHrfO#?>=i>g7E@}JQlD2B;5oU$+3QP;Rw1B1sdu4_`QMgyIl&uYjWo$&_ojhO$-)CD?;G8L|cjHV)->2 zVE#3*N)~~63WK>R1?D$)@nQ_}D3}$@iv!FS*X=E?E?Zm)kp0vEMz$UuX=W_D`U^G9u;|5MaUtKLjf6!Xi4fTqeGe=eA6Vb{0EtPo zd&X7gL(gwFuL8wn5zK$bFke3f^M}u4n9Es(X49HYFAj~G#?;l2y{zk_(kwG zm$#d>K?|~4YAmw?Q{(j%HO}O2y;z(mHIy2?jv5xNqbzzySu~F#nh1;ons}bf08cgn zN5~?cti?QeDaDiBFmHgWP_Q+nJW-zXYMwYWS##*;<IX?bNPX;sR7)p^PJ|Af2pu{R;$wwlNB)$Q)bh#R zJK)xE@QN(r*4LO@SEso3{2|OOImlFQDYyC&w;Z~BI`oEg=nY9c6CMOS^F+J!hIVrh zNJbVBZ8s*`MJb})w+0hUPEVC+O0<4TG>0avb0!@V>4Xy@={)Q9yrbQW0prLb)_sFn zHzCEk8D}!<s-ffql)GH;WV#9aM|&o)9Fa;7WJ4n{l8YSwz9ZnSv*zD0s|4 zOhGv%RthQwm4ek2bm(I2(ACz_RA1lK7A1GllCRiJc@BI$6I>*V_;@_?@rV>38-_3+ zIKZyJyJ}){y)rAEiKV| zNK5t}(mdDde&kc~xgAP`8>iuZPSwh?HmwamqajlrnzTD~<96uAO{^7Cg{<|oZGWKM z+ycsyMYR1n)3zl=+eYrijZ%I+n%ATG{&6%Po+U0a+B!7RcW4Ld&<@n09VpRQNLJFg zNRpR&u-!}rf5{>a-^?5yn&R+`hyAA)HxZP>%3`KI+H z<7xPmH``VykL!xR^XX&Vm1Hi-7Z z%8>T{6W{z)yLlU&CyRJLgL%JliuZ@_#=Ms*66L+}UU}aGc<(S|$6@}7!#0V-pk&ek zVQHxY#kSlmh``e5uSXV%z*{T=btw@z?l~3#sasJIP!UiO=z${OFeKYGU_F45w}m1I zWos1SqNm%}E|@Nt%hzog{ht??`mQlq2tpz@&d(3|AJVTzc;*bF*B7HP$&SSx;?(u$i~SSzG5NVP(>Lbal|YK6m;H-}+b4#TuaH-y%q zZg}CC@KU>}Ks3lA;rI*-$D=9X*m_MC4yi>_;ZWgF;phv(;V^~IVW64AR4C`io1vj( zRK!s1ZH`#0!wiFhkVPuez$$WYN=4TCSD_+J=)tEYhgx!|C5MZr>Ns)XrBt%^8|E^-*`;);oL0p)|#q>z-&*X}TFkQ1^=RWG6DafrB4%D`bHUxU-(RizU&2UA&c~-mG$LkDSbJ| zxAAR4yI@UOG-c70rJtlMRr=!4THm4jy+gBO(i+i%AeZrCbICd#<`4)CStK@lve^76 zB{n0s>}HX067lk<{;LYNV-=rbe0?b(0!}n&i;u!=W93!$o0Qyor7U`oy(4 zgF4K`5GS%opw4E2IxrL;vLyQb9Yq+?mFq*hVYO4Ul$O4Vw9)ym;!mBW<@hw^7qE)kYcxqPDa zA*Abe$QD^7UDvR5eKjRrYk!@kOR6JPx>UMUx>ULrhIBa;UFWEb&QZ#g7t_g`Fb}r- zUG_V+=`asK!^k2Xo6I`)rIe2SaVxK5n2o8s?7GXYyX?BlUX~fmZI?aV(W+W7hACLB zj^(IY$Z=kn7xUtqO3?|@vtpb6%AGpQ(-1VWNYw6UQ5&2RwNpOJq9%o)nw)8JrpcKm zXNw{^3q>u*&VG*lvK%Wc}Fw`vc3}nkl(k?p2mMDSB18 zQ@K;QQ@LAWa+g~N-NSO$QU};3{HJa*J+FPw?%H8yL-)uc?fWZh-|{K#dvkZzJ}I+R z?NjYj?NjaR5!#pA252Bwy%wup>sP&^RAm}i?CPDrTZj3WsjDN4gz#N2grnb23E{Mr zSqSw)fX;{2`LHN-0OcGgw`hmdE%A;q$?MnidLdwek1E1ly{a{Dm;}}eVQ#TFc32!c z8k^k&nT!*}LKSx!-eFdQB9hgrqA{$Be^066lQ*#{N=dP%hngO0dZ?<{Bh$lB6?2Cr zw8IkG`6aX{SeZ8RVaap*cbE+!k7SWVev&2fPbrCszje*LO97H@!Xun^Wl_u?!q^4vHIs>J{l z=eUxRW8t1-;hy`2yGUKRa4&W?yZ*1k>;M%diH^|LjbaKNRhtER4+x@zj$Z&KG#WAl1!7Sv<2pq^$K2XYvsw0vaM_3aKb z3SvtZ3GTiuxaXz>cjgY9)Jhq?CbgQ>YEr97ZU0JYL&44K(P*O{dkXc~Q>bq&ZfK&~ zsy5>-(%p5w(_yx&tE(fcRo}Wsuf9j#ol@U{E3x`Y;eTe|tBf3vKe{viAX)TBhwvZ0 zHT6fYI+FjW6wYS+KgAQzz#qrWs5-LfkMG2P{HoL+pYnbF<5F`r=Z{zY@8XwQzuRF( z;8)0^UmDK8^yAboou2V8Nx6o6ndpls|JU$~Sp4i{{Onl#Bw6&c2l3CImHOFRF6Ez< zgHHL`hOCk`W=&ahhO4{u4Dw%zo?)Y-JIr|eK3Vh(WB3`yr=H=$|Kn$n^MUT40k?;7 znGQGT{8c&})7D~?f>Jo;QasREco?$ifyVIzjZHmJ&t6;HS9 zF&*Z|crvo+>CWV*83CBB?`iL=M? zO9X10A;-?@vnpzbw3-`EQZG>)^5M&o(Y|ysSQGy_W@dr>d+|n3;BCmFH=4?Cl&9Wk z4tQtE-?MgB(EXQ%z3=FIN~;Wgv8^8su9Rfl;IuSOQV-sAjwo2FiGt2Oxb%8$+B zem$(iunfa03^@UTQ=TLS=LYw8nAh;0WYN1$<9A&r^{!i9%kL^5wnf9R8gPc84Br*$ z53-*O*-w3OH4*>4Zer|%d;wnk-*{=V=*4I9i?5V=@#oLr7niST&w26s=B#0G)?l*+ zmtil803c5p0H9#<1`l+YWtOU|Ba6K7w)esjAEvx;Gq1xzs`>rGR(-?fQJdx|{54)^ zfLu0Y4VA2+IBqMSIPi!E#+7i%C*TgUT9+)fG`r+qDVH3#sRsr`rZ>5yp($%<&MJem zO7S@>pFVVt_s<3oc9`|x8?wkhtFV9mobu1u%h^AYtMpjngpdjSQ*l`(&nop<1s~wz zO9>5%b*A9gqj1(|;V81mSp(Tw&!?QV+Bxhj$&h-xvnq{Qr75d4XN`l46)HYc3C|t> zP={%R*T^EzZN{GaP0DlsXl2hyUe>2P*J!gwmo?@YW}!ph@uiQw=-K)q+_)`VNEW%V z!fw1L<;L510v7U!-l}FbY^?%d=tfMG$2@tAd&ab9a;dy=tbh|A?lA4}Ct2jvVeHcz zQ$D?%S0zYR`jbyFKpP`%G0_%VbeJkhE*1v_E-qAat4BJ_-f%8i7)*${gct#a31H-TaZKoW@A@fl{kP$EvdHy^u( zOg6$?qs-4W!jiC?kED;oPQUFiCnE-AQ5=rqI2@ab!+F>NT!A9pyOu$7$!X{5D>)az zNyY&KTQIK$<61ECz+&VNS4i%B6ClR_*y>|1o$ zx9G5M(Ttj+CEg68<-_$M!Zit@B8$RxF^6lHRJcY?;c&^4O~bW-;j-u*ZqYm3qO&fA zPCOfi&d2Wb=Q_+Ch#OfHyK6aiJEmfHJReJ9J5a@N{>3n)S{OTvqg6`>Fm$kqlg%QS z;%W(I@l_l6e1~}uK_rU;c{>Mkt5hJ}$C-q(JXI1dL=r}Uv^b=;bex8BS7NBRU4rWA zc?+WY3?fPvMfCxW>K3V}zK+uuN+7XAt@Kh#U<1srNekzl*w3}F&sMM#+KkudhX6cQ$l zW+dp=Q19zvOt6r+)xcuHD;;KaM4hY_7E6DOVez*VEY8}>!vY8Hnho@jY#{tP)Cz<} zp`S&iAO1n5tN4tdIL@Mk3=k<}SVF{qLWKr5; zQF}pXl315$Dw5t+R_Zig0aC~!D1DKkbY}`m8$HHQlHD?eQlEyBMRkcqorCpd*C>tT zthR)em#9a9tFHq#WD%}*U|e07!quRC7+1PY3hq*2&pe;Arw$C~dWw;JmC2_;{x|41gkw5Vs#A?wl0jE?JooC;O3#I7M98&4AD+ z;0V#@;dcq(cMQ-)7Qt^6!|!`3_~p+t{A34I;ivEm;b*ZqZm|<;vDIb$1tDDGh*G>_ z<*@B4o#r$^jV!|A(Tv6Kq_Ftj6^upMc2z7Y7UzaViyaY*b*#mrgJe>G6Ubx%%%Pv? zG(QCH$Rc2#%D_A{1C_Rr+x=#wFhw|tv+-p*lDoPhtBSJ!~ASytt&pt=2+i89QM3P02y^JB-k%H{D z>$0%Q#YBl~kH zWIy>n%e0&ts7$L&FH)Hfk!{iUz@qVi^_>qO=F#pMw>*3eFn<*&CW~PHJBIoCDVX0s zkzp=p6$*2O`4WM-MN0&W4ixxDadisGOA7PDH|;e41fg_yVNsxA(F}w#8-Zt#*|6v@JE+rq0DO~01o#INU_goh>vIPdWKx>7Yu3KF zv-U8zsS^E=02b|VEc)SCv^B73%R+<@xR!({*07Vnh835ot0Rlp@L$h{G4G_weO8!WmMS#&9~=<`9$5a^c7@QdK~&vcr#K?|~4YAmw?Q{(j%HGXi5&y=v# zR%$3UdL1<^T1Q#*j)g z203&obZC3$(Du&xws$4oOGXtN0Z)KVJAq1M5uLVXI^C9{)AYOfgCs|aN++dLAE%Q; zCqjo#gbtku=~E)uFRA5|y|MM3W;l377IEuq%&n_a+ru)&zqdt>gP>AZ~gT17P8LK z)L9zYhYqU6cTWfsQ}9Dja2)7I7E$nUrr-%F3VwS|hAF4SN#K+^Ak4L2VXt(4~qMS`DAC-^F$GP*-(Xeou z7LG>4l(7nVN^bgmwe`!L=0XsYEF$L_OwN%ha{hfYPfiqB=_*lIiMmRhvr3%n$1)_R zLlRp(XH-9<^ZpqP znd;Du+o2n`LpN?>t&l2Yt*7nrU+pxvfU;x}ZGX;bRoW_T7o4^Z zP4pcaq&qZ7cfLV7t_XC$g5CIQon|WdOBQkXX6EqF6o+U0k~ys9HRzLtDs!wyl*7v5 zh0kG!p7ohS(|YHd*5kfU_k_Lzgnk@kCW{FDOD1%EiqH=q#e|k32mL53p_R~!n9vS$ z1RUlFILr~Cq+8e*B;Cb^^MJ2+nrUD*S;X$gnB9X??7n?XX1A19D7%&2%I?L;Zigu$ z4x=3$MmrGGg^2;v{pxzBvpUU8(3~uy`U_0;byHM7{{g1D6m%%nmFi0MB}8?HDJ~Av zHXMd+5bcGPA?-cyhfe4;Z-et>5$|U(?^jOo{_r;Dy;PAX@0It;`yRl1hao!-^G_Vk z5*?=AlMV<=OC2ck{P!UOOAn~4Ba1}fEf#^gln8wH85RMlTTu~E5l|86fg<1`ES zK@MYrC|?lzmg?YF4R1~AGy@?GWRX04oaNy!DS24>>_Q%z(bz(Z-Siw!i`^Ee*exuf zsXdFYI>!sQ{lksHaJQg#)lii%g!14pMa*Gxlf&eu&McsEAQfIKp17{lYzCbmi?rfX ztQEgcX~on%Su3P6NK*+-B{Y@jO{qkv6%JG09ENE*4AUar5L$=2;e})T^_`{y(IAV2 z<1;KAkEVoUr;S-Sq!vkqLxn?yqb~@D!xTP;Sy2wNq8w&Lk&p<@OCc%tHk)tgG{c}E zWRZ$Au!`K9QjyjE#wsEOP^u!TBB~;NQAHdk&N5rS_=jKm2>;8Bg?CJ#FMB~-$Rd4d zWqtWsN?*=7lJ!N3y;NURUsPZEmA*I(D{`36;xLJVR!5=*f!27jIpxkya|ncnEE1bN zS!{lk5}T1Zi;a|+so1F4sMz#dv2mE`;V`4Xq02f6kLW@uJYIP=yQ|Y21=S&ol;>bp zp6{oWr*a!Bj}*13@~HBt@~HAac^o>@JG9PrXzS~W>7~u+SRZZY6dS}hKG11Sf&`I8 zGIS)%&~YglT5)fdA*rNO8B!Ti8B!VQMur?(HaPTbaK%8`W(?4XB*~{nV;}4^6QM_B zktUtUnlw74NwZdDO_KUO)g;v<)g;xV|Dj0^UGp4za5*&ap~ai%N4R(^lBW8HI?crp zC$dPO&Srr+FeOk=-s1%dqfWE{SPOu)0JxVI0Eb0=wY53nse08BShz(U?i$su-NK|P z6ex$LL{KU$^3c`*YE_C!q*mX3s?%Hxr6P;e>L;vLyQb9Yv=dmZq*hVWDNUy|ozisb zBcxMRYUR+8!QnQx!zgT0E)kYcx%^6S$kUzXcE}c4Bwg3AbbU1?UF+@6(k0cADqSjF zDqSjF3q!gbE{Nv1#F^t3M_x=PZ^qEdXiKYDFwTaKJpc_Oi*#%<>)4l4I`(1{>zLGJ zs*b6Ssg9|REeajWQTm-@UVL86i*LrD&`8sKayI#yPV+PbjVuzi`&rZmr$p_- za8iD%3a1LE3a1LUI2A6(IcAR2fgJm6h1`i6MRK>#A3Dt}NF7-ucYk2HTQenh%e}~Q zCq=I+cPe)(cPe*FOzv`Q$mLkT=E(6V(-UP1rOz*B*P7mGW<&SLBJKMtYv1xI?R&F> zwNJ`yRr^%?RQpu>dW81nwgDQ5RjER@(;%d+~vRYL%hE?(J zDOG&(8dgOqDOOcfRa8|}RqXMqm^&!nmrYf7mprKXgcQueBp zGSti5H9r=9N?@=tP zQk$-_soxWvXzYH_SF%WBJ6U5dO=;{&UuBJz zf_T+f)mYV7)!6=}v3Wf<3+l01P+!b4#sOAx*Yz}{v!fukWRc+R%Yu7uN^oZmVZoI$ zd=*?3Toqgu-2N-Lc|96!)MHPf9(xM)jl~U3ln>M@u)mGILYLXDuC9)(R(-@-0x5@W)Mb8*CnJlV?o58V z5vix!{y+S5vMT+^PluWyoNVJfCPP^zJ*xbv+DCl{k9rjzlPr4F3;9uZPd(}ykMW~Q zLbqU#iULsNh7c!T+JueJT9pDX3m2dA=`M3Co|!Cq?koAZcTPR`TNC-YMFtn%bECol z1qL8A_(ZOkp9oyfFR@bAW$wcZkVP+X3%|sdQ!jD$xA`Rkwat)YXZ2Zt(1aO9-4Br*$53-*O*-w3OHL-lG_{9(1 zq|5vpFHIJ`_)LEBl~OPMyC3n3%hz<#NJ|YX=jp}ko3n<&S%b|QT!w-a002IUv0gBF z;HF(>nWgIL$RaPi?Y(fshbb@I*vVe_2%r4L75*A8G(avJvW805P#m{`9>tThdYykNF015Or9P|R16+J5A+xc&3Vuz8vpx$)kwwlL$j*8` z<*e1d&(4wzskb|;(wJ46vPyH-IJj7$La`H`8}XSg(+IDTMV{M?J@=cG=l-z+drtDQ zKIOSan>D(uG0!jyo%CJaGW^aJ0DK>A+!ih*i`-aYH{O$S<83?v3tvA0fRt0c9aX?rIVSWx=bCW+x8ZiO$n}S?>rYR){^`}& z^^)HA3w5qhk`U;a35k)An7)SLYnZ);$wruK1Z)xmupnZ9#&OuTq05|%7?4GAIEv$N zY$^`tT*+~eMTN#85eE!x!MqlXYr)6^i;)Ku7x7vdm*Ok%EQ0a_1cWRK%1IoQZ>54V z?m!NTEM+t(^B5G1ITRLiC@lK7Q-H*SWq`bV9a`x!mmoT1QG_OPgbqkW=$Pd>Lb7_& z2+c=?EIRC4blA6OMorNYZwArw;hK$bO+u*1qHtZz;o2n?u93HJxMazu;ab3OS@aIK z=pAm+S(id5o()6iV|PYlm$?IRBa32pEyr%hRP2u4fMX{MMUCA;jGZlxH6faGu!)n+ zBADW831;zC+n}k-JcuBYMS;AX1G!ZykY8AV11ZZ>4dkK>q{Sh%r7dfm3KK)c?GjW^ z&w<-?nP(7DvM8z#a8$QQMfER#=BUbwSfjeQqH5`|pHBPzaX&V~<5faI+{p;>9}!-% zD8x^5h(DbQ@z3}S9lPwhOpcbxRcp1{HA0wstJp-2_m$F zqa-vR+mG0}%e;YDlST2K&hcI&74I=OcuM0)MdRH=;%!kTV^RNLQGJ5gm!Sz`@AZ7! zmM-%j1fDDk{wxmua;f0Ie+ma*wj5N?mxP{&WDkRHQF~-jd}L7oV^PLKV31IiU?^n& zDIj541ez>D!fcO(vG1jjaLuud1lir6kVSymkOA{-3NZVh z!GMu%Bn3>b2aHAWrbQj2MGYT;M`9a-$D^mQy~}I~aF9jl`8=cNp%i+CuEpq)eJe#z zpF)pC&7(zOpGCErMcoylNn%~1smN{~1(d!5q>x2W`XWQ=&J>h3dVrxMyJZTcJ`E*{ z0yK+?DeKFwQREjT&0eC;-J{EV9k3ybaJ2*D>bexJvOO4AvPY-5>NmKus9?%0s*EfO zIY_$%1f_O)kk#+mW%dV#$Rdz+G01+J0@;hr3^LgnRFL&gkXclwSnlXafD=Fjfb)oZ z3Wz%lfFg?!w;v9U#<}^T!EW+Z^jK%Myuvo|At#G+PrBJ2tBev5fnH1mzGFbrgDgg6` zz#Ulx%u^Ydho*pe;!P})ve~O5sUkTyk+e8EvN#*C*zdL2wzJr5u~-??+65Q4BK7p2 z@<&E=nacq}vIwQ;F-rGIq4dzNGfHKPSy8GeoiCJHtgkJWJQj&3AyyC-=&Fb8Q6sy| zFMvq02(p(kWIIxj-FD96M>e(O(iOjo--U`_3-Pr;u*LmkiwnwxXF*$u=OW9!17Q9&uu2wzc?yHM zDFx;?wr4QQ2EKw>!MsSpY;oP*;zp;%U*L{yt#-Do{)o!Tfg&^Yv3O|K*tsb2+O} zm@CYe2+S?+odq_mxJ+FgS;U6_dNz!C zC&h+W`8XTdma;+F&@0$r(WS_uOOeIZS7L@hw`7K29ZWo;%d8DrkkwLSnH88CucxSS z7I*8#B1Ea7)aZ58uxK4+(L2hbc@)t^U>wlI^Q8XBF0%| z4)>Hn%Ah{RAcro64sGuo+TJI2mbRu*ZXy(v~kUmy|{gPTf*_#Y*4F|8tB5r++xpj4lThC(-U%@Ro z$W(4AxB3yc9J+iu^oDfk4M{u`9t1q|L>oQ6%Nzufkwrw?jfr+qifH#4CYqd{D$$f^ z{gh}9O;{b;bvm@`B+>~dLehEGy?lC?83V?VMXdVqTsQZ zzg$pIPKouyryoB3@ZsYZP|%@^u|ro|M^k-$S6dWWbibQ2`N}SHCb&oz@$q=(;}I!7 zR=&)9l(T8&qw-PtICnle8gh2oRM2|ky_R_Exey3BX zCQME_5m$05IhCCAPELm|;10c?9eO_#QH4kWQT@ttqpQ2jm0&4Z#MBFzse7cDdc%9n zR4pyx*_6qVeT!`j#N0Z=FaBOnwas>teIyoHvHJX!mMq+rWo=p;envy4Iy7l_=*I2P zjhk32qzYN-ER zLAvt|(oqQA{R;N@Yr4!-@Ruy&@XgHOp(zf}xScsHm9Ir8kkKMvHLM*_n;KJZ~r*6TS_aG-O6rd_hMwX!;}z*(GCuy9f;||#DM93 zb$!KcU1la|P8L!91*ZDCDXO1;H&a~-I+W^4b*1_eqPoKr7l&yZ4#PHx_QJ}L_Sp*m E51e8kQUCw| literal 0 HcmV?d00001 diff --git a/ci/test/tx_cksum/out_mseg.pcap b/ci/test/tx_cksum/out_mseg.pcap new file mode 100644 index 0000000000000000000000000000000000000000..322279fb15c3bad60c09c86ebf5478ac9167ecf6 GIT binary patch literal 1487874 zcmeFa3A`mqS*KlqxIin2ATo-eA;>7giHOX|Ok7YAhepxHYL{^V#RZkoQAbg9TyWo} z9T!|+6mf8L9NbWGL2*G*P!SXn6a-OR02fe@RleuVlQ(a5rRj&=4c*oC`1?gn_Ymh+ zSN_jAH=ifpc;gvQeC(rcc;?N{oc!PGUw`Jz4fvNkF@ELcmz?>zo1U4+P454=n?L&2 z&-|q)J@jTj`IcwydFG}+63g!X<`25}4bPmp;f)yl`&*s-l*_*7_Gj+$ac9olm zOD}tHclkXoOF5NPQX_=3xPqVPBfHe~scWelQtHOkO{qVm zKBvB58)M{#7YWx?s)P~f?)TT5X(vVXcN*Zb!x-|4@ z;BPdfVNAo6#zPu&8cP~$8oM<1X>4iaFFdAkO4A`tIZY)^HBDWb`jn=YrXfvSkhmgT zk^|S|z(qN5RSsO11J~uig*k9#4qTc8>*hZ@D&s_gZ0GR|b5o9vRgpf%g6GJ8k zCKeJ!CW}lMnKUwSWb()al1U_!NG6g@CYewQp%g+Xgi;8l5DF7-M^Y)IQb?tcN+FfP zJGzicA(cWZg-{Bi6hbM4QsSK*Z|(WL&BZRHQsV6$@9%hn$2&aU;-peYrI1P?l|m?m zPzs?GLMen&2&E88ArvMSl1d?!LMnw+3aJ!QDWp&_q*6(xl1e3&N-C99Dj$_fD3wqup;SVtgi;Bm#upEvFtKp{ z@?l|o{lxcAd;!HbP<#cEN+p#_D3wqup;SVtgi;Bm5=tc$CKi%PC6!7ll~gLJR8py= zQc0zfN+py^D3wqup;SVtgi;Bm5(*OwNu`oXC6!7ll~gLJR8py=Qc0x}N+py^D3wqu zp;SVtgi;BGiG`$6Nu`oXC6!7ll~gLJR8py=QVFFJN+py^D3wq;Lg@&lBNQgSdr74u zm5x+8Qt3#gBbAO+I#TIKr6ZJ%P&z{C2&E&Gj!-&6=?H}>zRFohDjlhGq|%W}M=Bku zbfnUeN=GOip>%vyIzs6Pr6ZJ%P&z_kVqyGhh~EwI%VGZQz{2?b5WgTurHkJXgwhd8 zM<^YkbcE6oN=GOip)j$KR60`WNTnl{j#N5Q=}4s`m5x+8Lg@&lBb1I%Izs6Pr6ZJ% zP?%UqDjlhGq|%W}M=BkubfnUeN=GUkp>%}O5lTlW9ieoD(h*8WC`>FQm5x+8Qt3#g zBbAO+dQ$00r6-l1P3_1sr01MlS)r2J*o7h(vwP0 zC_SO{gwhjAPbfX1^n}tA3KI)Sr6-l1RC-eBNu?*1o>Y2L=}Dz0l%7y}Lg@*mCzPI# zN>3;~p)j$KRC-eBNu?*1o>Y2L=}Dz0m7Y|3Lg@*mCzPI0dP3<5r6-i0P?%UqDm|(6 zq|%d0PbxjB^rX_0N>3_1q4b2(6G~4gJ)!i3(i2KgC`>FQm7Y|3Qt3&hCzYO5dQ$00 zr6-l1PU*H}o+!H}${1+|(cTZt7oq#e7p=PJXy@ z-kW;mymB6mqMTRGE9aH-m}=!bH}Uvl<-BtKYRdU@Z|VmgnG8HK8Bj%MDT>+L;2-bs zf1Nx0=W%x*V{?c9^yLme?cL%35#>3Q-_(%*{}OKXV{C5s|8%+CfBU`L z{h$8c`F6kacHb%;tL?eXr%;>?-x7yxndxgH$_E)>w{tdm| zAJ_shumxgZ3&dG=zn(Aq=X?L(49@T2OMjDo(u@Q`6XxO zu>pZs9zozUU%LkaU-^s~1lq|D7X$s1uA$us36AqRB*|S7b^JsJu0~Cqh=}?PJXylAQeajQh`(;6-WhAfm9$B zNCnqbDj3+GG0+}$pgrnY`sQy4;rQU-w?Ty4g9kA-h;WMq5nix|2#*$NkWp4 zBqRw*LXwaqBne4El8_{>FC;Mx{7nYxn+EEeM&8+=#d_<1f)+DYJRPjK9HfY`!HN%F zu;Rz|u;N*_J7UFpGMiW-R)`g1g;*h0h!tXmSRq!371vv=7{&}=PzN>EK#UawP7vex z=;9Axj8B3uVr&@W?7|q2+GC8b`^zI^cs_ITz%~zTgDpI;t?<25WDFTY#*i^&3>ibl zxF{H77&#ewq*{tQVR&Ojf|WOp!0`-#EjQfk1v5dVr=N+vljaJ zx;^^%lP^Eg$9nFa^dWsnAJT{PA$>?6(uedReMlb{HGK>dEo&!Auci_tvZBH{ByvQN zXM!SM4~E3ppvV_5DDuU7DDpAyb3~E#usu;k6cI&45m7`G5k*81QA89GMMRMWMTUuP zxDypx6BSz1T%on1!-^*JuJdQXB@cs1Vr;nN>lQBgtUWIIe`k+evL5j#m&he@iCiL= z$R%=#Tq2jqC31;evco0A#HE?W#57lOtsHS>NO?Ap@&f=$j18oG#{wyzxCbfUdFBWy z>!E@|ijX3t2q{8}kRqfADME^nBBTf@ZwRD}heH|2 zh~-fWv7Fr_mT!63kyzHF3?&wcMPiXyBo>K9Vv$%R7Kue-ky!q{#4;W#t=f${bndd9cir;g}d3mU;ZbGSeQ*eBq;xEVG{3D9gw)vWzSv z%g8daj4UI|$TG5wEOSk<%y^j6m?<*jnD~)%9ATT4Jyy1vx9&e5uz4m>6JrB5zqNqP zo%g`z{`WkB&3aO#U?bQFHiC^{BiINwf{kD!*a$X)%^M4BMh=Y}IlE<~Yci0ta>xof z$HzB+2Ic%YgcD;!Ie)lN&h7Rn=b=A%q@4A9N-0Omk#eLQDM!kYa-X$RqNIJR;AV9C=0>4vg&78L4RFiO&isr}%Sxf8-zeNB)t2u3JblOB4rpM1+R_dIjSYmYGW8L!-fp?lr^2t(`XpTdwZBn$~d z!jLc|3<*QRkT4_+2}9Qz7#d-rvC!MCP$yUBI=K~IRydmbAN&<$bZe+6#`$FQ)^D|t z(Tn%U=+(D6lF@qPsAMD=Nk)>9WF#3$Mv{?aBpFFYlF@aRjK)GaU8S6^&gFC~$gGrf ze1P=VVA8vRNHI2;^!5uT{plViwFewAX+7vvOcImCBr!=$5|hLvF-c4klf)!3>4Lzd zu~Kzcsk*COin6G$l<*Q__?)B~3|F(v&nMO-WPNADSAw z3{_F<*HP=&MTfmp`dR5}ZX)r_S6p)DW5B8y8??Ijf>uAWhgQ%0{v%qghu4Z$qLpYR zT8UPom1reeiB_VOXeC-*f6;2}s0i$+2<)OFko{Fl$=-@t$46Mtg0nsq#)`4wtdCwe z>wEV&>xqXWXRRmZ%2{%joF!+;S#p+~C1=T5a+aJWXI&heHFgvZb`%cwbK&4hMW@Vl zgsx`;U7rVV#n?dCr!3I*Klh;P@4oQ}UF$i$LYL4bbO~KTm(V4230*>$&?R&UT^Ap8 zjXm{+eMw9;o_ei_^sf=vmqTAMHU##$3xR#r9)UgLV~+&3o(?R5NnjF~1SWw=U=o-F zCV@#{5|{*bkrUY1caRu$hD`f-IZKX0n-VCY#A-vYBiqo5^OfnQSJT?Xua}sA_C< zO>A>f2Wp@6)FY^^M=uL%f|{Tvs0nIw(Wwo75(?No`V_)F!n_ZBm=mCbdazQro|q+QvqSWTQl~O|g4zMXYlO?uc+N0O5WG ze2cL`xW_IC_knu|_mIcW2*+8|ONV|DPJ|QTL^u&ngcIRJI1x^S6X8TS5$Qw2S#QTMrCx2p3xWsV`Q){vo5nPvo5nPcZrFGnT45!nT45! zg@uKMg@uKMg&$a0S6EkAS6EkAS6Rm?^H&`htITv=FISXl`DXWlWH^UE{m zmuJo|&zxVLIb<($$X@1{mK#U z*5kB&SSQ>Gcfy@;C)^2l!kus@+zEHWopATYgS&Cyq0+!pr6CG4c^q>-=pD;4UkK^_ z6_giaLwe6%Nbeo@Nbd&sJCfddh_|FC=}CH$o}?$~NqUl=q$lY~dXk=`_ogAeaXe5; zNcko=U{K%6W#^P{&JXxr1or(C$QNUSeSf)N-`nhA-!pD-#J=_9aIsJ96Z^zIu}|z1 z`@}x6PwW%>#6Gd_O^@lIf3j14OO zhXob?cn=l-^vjN@xSj?tDvFAtqNpe;ii)D5s3_Ozs-*1LUjwfHv z%Mc=kNFh>)6e5L4AySAGB85mHQiv2HFK~#Q4h5L3jE)iYtf$boLgl>kF^KZRp~)B< zqWpk`D1T~?D8KdYN21IUWppCqDKLjp2FoWtLhLxDq?n3g3W^g?iFruOGh$FbgZdfN z&!m(|DU(trrF>|NfiW^zmsyutmsyutm%GHo!py?V!py?V!otGB!otGB!om+MtShW5 ztShW5tgEc6tgD3|R9RSASXo$ESXo$ESXtPyuwx--Yi3Sb$+X(dbXUr3pv{yA*mEnDWpg zUW%9ErFbb`ikH`CyqtLIGV#=9;;9R$xzgZ@n#bbUSHPN|0cXb8u;#}ttoc2Atob=# zcx25gYs#9krmQJz%9=y!(3>5#5ubo~D5c4yME$#{lxEjxN{}LQ|>&1=PLov2LqikHsJZ`3wVC> z9(aDtEso&X37&$d;3;?to`NS7CU^>-J$D^V@Dw}+Pr+00ycofALNZTC<_XCRgszl$ z4ul>l^i@#kS3#gLHWc~=3x$699)-T&mmDdymqMjbDO3uTLZwhC)W@IDrxZ$mfJT1+ zrpzde%IFw9qcH}?$Y3Fr0*y+6#?ynw(}Tv-gT_r<<0h_g6W6E|U}7Pq0*y+6Mx{Wb zQlL>O(5Mt>R0=dI1satCjY@$=r9h)npiwE%s1#^a3N%7#r~xJxQYz4>6lhcmG%5ue zl>&`Qfkvf3qf($zDbT1CXjBR`Dg_#q0*y+6Mx{XGu}0&rl8J?s3N$JO8kGW#N`XeD zK%-KiQ7O=<6p%utP$~3cr_hNnu!--liSMt8?=PryrN)&?=jKAM0ZIQSI2vPvq+hup z>F4bs>DTRh01EOF+6by)h0Z}j@3I;^M zfG8Na*&px(1D;^O6AXBQ0Z%aC2?jg?6ARf=FtDXyU`xTkmV$vT1p`|O2DTIoe1Q#Y zDHzyNFwjVFppoD}Bf)__f&+a72l@yO{HkYSA&mqF+B*)kcN}Q%IB+LC(A{yMyW>E2 z$ARvSgGefpUKvRzl-h(+n^0=Orz=8UnNME}pMEcF8e_w!-?H%Or|$9Toga4O(?LF! zPvukjR6dnYWD1hXr}AlqO*{GY-{8}UnvaRk%7n^+Racx`S@l@o`!B%i zj{vGMHn94?7Fhk5Jy`vXrM`C*R)tkzRag~Pg;ilySQS=K!SzV4uZLWJ7HW;LA=m%Akn4x+k?R-T_xPA~l3XQM z$yIWdTqRe@RdSVFC0EH+a+O>q*JFRAt3<9774s7n^Ai>GQ0$7C=TPhsV{h=!mz?=^ z5H`jJV;{F*?ECFu?91*lV=R^OmtOYZ?(%zFmU1eI?)!;-_i0F=bfR+rjRWW#K-&Pi z2GBHso&gX!8*bTe%XV9K+fvy~VRNFena9RFHRho)7!HL4;rvbbi}3fL@SnPW%KoYP zr|3UU5ulept^DcaA14TK4gf9w>FUqP05tob5{3TM`BUbP0^xBzHU890^C%jqk3Imj z0iX*&v>%{>0IdPodXXsWr>Z}S`YFNBu@iyK1#BB&M*y|{l<`xbPi=nUcAY|g>i8++ zr;48z6=TI%F;Tmkn@0Jswm<0c-)F-4IwmAL0W%{|iYjiKEaL%K0GwENtJc5l5$yDvB% zxI>YzCCV+)Wt=ACKF*~UJW~vwsRqxKgJ z*-KR5M+tt^;71XDRN?cGz3`B|@Q}UmkiGDXz3_~^@Ql6ijJ@!Tz2L11-m2iOm{`c? zy6~|sJY+9CWG_5qFFa%~JY+9CWG_5qFFa!}JYz4E=NIbp3!xN3DTGo8g^7ivQb?tc zN+Fd(Duq-EsT5Kvq*4f_5K1AGLMVk$3ZWE2DTKnrLQ*NDQlwpJSK5_!rCn)P+Ld;t zU1|4CO}i7f#S^#16Su{X@5-j9&v}pNdz)8Ya^{s_Z;TE4zGy+;o9v)lViHeY@2#Jc2s0fLQ zkf;cWO3{DCM^t=7#Ya?pM8!u`d_;}50Qd+d7S62!SQyO#qB}se2Z;Uv(I5aJQ4taq zA5rlU6(1q`ioT++=qvh)zM`+_EBaoW=sWR5XyS>`#1kPfc;(b{F!;#9?+6Fq_-1G3 zvEkr-}9tn!E-MMTkvebvyXs%1S|*3!E&%1ECc%m|L zqB3%#GIHXX9bCL}>&nG*o2%Ob#kT;8W1J7gH+!=MivM8`iof7fj!@i$Vxd?l7K(*p zp;#yuiiKjKSSS{Xg<_%jx(dY;g{%{WtP|S>0pyioS3o{KHhyOa`F7B8j13{b?Lx?Z zzDLOSf6!p;-i9Am@I@42w^llR#9j<-Lu^CUaV&a$)YEIZ52 zva{?gJIl_pv+OK8%gz@xJ5MyZp6Kv3v6~r=UP1R9jy^twem6k+gMsK68;~9skp9Iz zkp8Zh&LGWUFt)t5<+UxZJ?)8B)||mrID^ZU*R;Q;H8u?wndps8r-@3ZiAtx5%15QL zsiv~2rn0G~QZiI28LE^FRUBwVfmRe~FVj)_z@|9ls+O05K zywZrg(ulm$h`iE=ywZof(ucg#hrH5+`=3(|tL zAT3B=Z$NtD4s7BEY~luN;s)$&ItAYqtmj@~cYvzTLew!fRDI8ds(*Zss_*moBUNXq z+Q-vAp7!yykEeY+Emcd^QngeqRZG=UwN!n*rRoW&p7>oq@mK@GULklT?Bj#$cL%XQ z5xkDELF|uM5c>!A5c|d7enjjdVvE=!wumiai`XKzh%I7^*dn%wEnc*oel_kUU7`|J1M`^NV>KJ4y< zZ{b_`7QTgV;am6?zJ+h$Tlf~fg>T{e;)d^uJ~&- zkFg=~hb$!iC3__P^rsw2yqCl!aYjogv2L0)=qS+ zo#5NmEdT61EPwO^j#%EraJ>*uA7peHT}GGDWpo)`MwiiLbQxVnm(gW(8C^zyLl}MHqcU;U>%>{F@cK&Br@Vdy z_6)H97_c5=1MJ_w0Q=>8z<&1OM_?ZXc7a`B7uW@Mfn8u1*adchU0@g31$Kd5VE^|5 z`!rD#KT#7ubph`yU7rK*M|!W&`{N<^7#n*3(S_ds(;mG)^qEI`pQLx`U3!<^rFZFF zdY9g%cj;Yvm)@m!>0NriI`lqGJrv&((-0`XqV@*mJ5c_&!1x#&l>dbVN_WhtkUYU)y-S{l-rrj!pU=afszHRUelKINA3kn))Fl*$4ANu{Jx zQz~65eJU-LA(b(eDb+)&IaPjUO|?t4Pqn2wq&lWLrS6croVt=a{>UzMed=23hLpN7 zbyMmOsn4k|+*hQY|D-2RS ze46&r9e~{j6f~ekxSafrY3D}|0a^&qL4XDVqP!?C%8T-%yeKcqi}IqpC@;#3@}j&b ze~qF1G(pr;F!kX8*Pn9thUIY4vj_z2liyl zjSg5Btq!8s0nHAWSQz~dqTxYwJfP)4rsqMX=|QIJL8k3NrtLwd??I;VL8kLTru9Ll z_d%xlL8kjbru_jE3+aH6X@QXGfsko}km-VuX@ijIgOF*2km-bwX@!vKg^+26km-hy zX@`($hmh%qkZFj(#6o%^i2vfh_%Hs8|Kh*+FaC@F;=lMW{)_+DF8+g?klG<&@<89b z&;cv2pXh+O6yUva0dIBlGxONEfY%&dz^A=x&jtM1A0A5qdR>5}0J;EOfG$86pbO9i z=mK;Bx&U2(EF0d3%=NIj=iff5z`h6T~o?m$;N48ltIrv_mzIru>Q z!TaJ4Vr=}uT^E1w8+-oXfuDBt2b2Clf1p3mALtMC2l@m3f&M^$pg+(b=nwP<`hyGI zA0QE;$3gVPKhPK-ov;G`iB6bh!aa})_eLbd*vN$US!BY`?a73{{r*`d(E5KlX~Clq zJPINA?3(F0Gohy%rsvFr9sT5Kvq*6$wkV+wyLMnw+3ZWE2 zDTGo8r4ULXltL(lP?%UqDuq;vGC`T3Oi(5$6O;+c1Z9FUL7AXTxSq)bi~{_00Auh9 zr|f^?6_!I5KM1q%(O88T8?$ifVix}2o>_SFmmSSQ)+|`2V3~quL9?J)&@5;cGz*#q z&4Ok@v!GefEL?xh0)Byqvj?7$GEE^GRuym}8s-9q4@NV53W_1dMl*cWq8Yw#PcuCK zsYlIF)C_6{HG`T#&7fvbGpHHV3~B~7gPK9jpk`1rTr`>?A1KCViUtf6j4!(yPK3ir z2giHi96lG{5M$#UK6!Bt-@4}kV2_d!GadPmm8G(krLvW!vX!N>m8G(k zrLvQyvXiA!16C;kt7zJarL9QX%9ke-3)#w2*~(Jc%2L_NQrXH<*~(Jc%2L_NQrXH< z*~wDb$x_+LqI1wW=p1woItQJD&Ozs(bI>_lgw6r+z$WzrH_lAaw1K<)^A*Hg^l&Kx z;>(Z^F*X9?a~1*d)q4WsU4P;zAi9$uu7FTLC?FIN3W!Q8jY=zxN-GT}Oa;9(D!nu+ zy)-JlG%CF`D!nu+y)-JlG%CF`Dy=jstu!jFG%Bq$Dy=jstu!jFG%Bq$m{>?JjY=<# zN-vEH>sEScRC;MtdTCU8X;gY?R9b0NT4_{TX;dmvDh*C5p;W9;#R@U85G_>ELKQ7k zxmB!ap^6r&Xran4jEWYjSfPp)s#u|l6{=XF4lC4Qg*vPdQ)P6Fp3xWz2nB=!LII(G zP(Uak6c7ps1q2o%8a?pv4i&N5f^(`N(rhC>1RL>9n1~n~8}X%!jrhVn8}WXxIogO` z8{rlVw_vyh!z~z=Luezk5!wiCgf>DOp^eZ+Xd|={r#6Crirx?u6En@dAy#>CDko+& zaUaygccUd@Y}CXzE^6X4_teDC_eV9+R1>NR)r4w7HKCeNO{gYR6RHW-gla-Hp_))l zs3vyRL_V<7nJLdlWvm|I)KV;mWqcT(;sn1jhM*bgHLOJ$>qFQy+cw z>4;CWd)nC3p`IS}iG4B@pHO;2;Ryx@f0NKR{MGr3(}A_D6emd~e zfS>;RwBM)uKF#;(y-(|XI`7kXpSJsS-KXh3J@*s0rL^3q<30`d>9Sy}u6Eve^gnkP96n#n}Io5I7 zV&=5P%xR06!xouX$Z3n2(-t$QEoM$z%$&BEIc+g>+G6Ik#msfgT*%Co%v{RMwai@1 z%+<_sR4vz&iG^`!4A(SA$5yUtjt(i&AcYH?V-FISHb-Gk)b((6a|p8VV_e@H?=7xy z%3R~jMb2F1tK&0L=CWt5d*;GtE_~+7XD)r_+Gj3)=IUoIf9CpU0>~tgi6E0fCWK51 znHZQ@$YBSW!wxdPPBVucWDYyX9Cna7>>$&YHWNrDkxV3+OfsPqLMen&2&E7T6YoY+ zDWpJxdsj%}O@lomc zsC0Z(x^sL6|H48(D;*z|j*m*mN2TMV((zI0_yX(r0_*q!>jO^dcMGVzQB6Ez3_1q4b2(6G~4gJ)!i3()0b*6ABXxNu?*1K7Mh> zZ|?Zj9lyKdm-qbJn;#gzzvCBp{01kKo=|#1=?SGL6efNmsr01MlS)r2J*o7h(vwQh z7g*01SWhTDq4b2(6G~4gJ)!i3(h~|33rVFXm7Y|3Qt3&hCzYO5dQ$00r6-i0PrIAV_l}0L!R2r!?QfZ{pNTm@gwTgSIxVehEtNHed zb%fFgr4dRaltw5_EF_giDveYcsWeh)q|!*GkxC<#MktL?8lf~oX@t@Ur4dRa6z>N45TuU%0MausSKnt5XwL(1ECCrG7!o@C%19_9p^Stw63R#@BcY6hG7<_C3rS@p zm623NQW;5QB$bg=Mp79`Wh9i5P)0%-31uXdkx)iL83`qgq%xAqNGc<#jHEJ>%1A0B zsf?sDlFCRZBcY6hG7`#2C?lbagfbEe6AMXYB$bg=Mp79`Wh9l6R7O%6No6FIkx(W= znFwVfl!;I#LYWAK38EpDiBu+1nMh?Km5Ed)Qkh6)B9)0yCPJAAWg?V`P$oi|2xTG^ zCKi&)L@E=hOr$cA%0wy?sZ69Yk;+6U6QN9mG7-u|C=;PfgfbBd6AMXYB9)0$CQ_M5 zWg?Y{R3=iHNM$0FiBKj&nFwVfl!;I#LYW8!R%0QlOr$cA%0wy?sZ69Yk;+6W6RAvu zG7-u|C=;PfgfbDzL?{!XFtLzSuCC7j8u^K&GLgzeDi9=8b08NcxaL4IOpwiiY?xr1 z1K}`1HwWT@Hz8W)cnJ|P#Y+f@DPBTMOz{$eVv3g#6;r%|u$bZ%q{YM=fH_`5VodQ8 zGJ|I0C8Wj_FCjOkcnQfd#Vg2;DPBQ(uVtpFd|hO|E;3&inXd~b-no2V=r-^ea6VrJx{Yv~7zVz= zE8pOiFYwCucjfE5^6g#u@~(V$SH8L{-`th&ZQTafhF`7!)|sce4TJ!{D0LffId?d^ z4c!K}mbwkyhHgW*q1(`HTp(@(!4WnDPsJ2XHTds7F%hS#W8RbU)rgM2L~_K~h>mA1 zqT`+RM90~GTtr7X`SBKUC^|A?G$TecVl*R0Gh#I3MKfMh(Sg7~epL+z2mj~E;4@xsM=+-%Q!yzti+ z>ru5HmUp;y#jPt;P8KRB3zd_F%E?0IWTA4h@aZU2KNZTK3iVHg0w}FVY{jD*gD>fd zAgo#swx_T?1C(>?id$DeIk&F3b;YeKZe5|`z^yA(kGplntt)O_L2fKtSN5$3|4Zz4 ziD&Kyo+O3@Sslfx1epszz5xYt=55c+W1~P`wkVJr?kSKbJbY0goeD$+q5@HYs6fu| zMr!vCI#jS-v|}$t$7a!vy`mjkMLTwic5D>w*e8neprC_oqAC#7huu5u-huj{@dO_g zyLZ^V!|oj_5W9ESy~FMu=XLLRgB1vUF~gcrP8~92wH2o_q}PY&L-Zl~5PgU~1kXVqmX1Cw9er3j`mpFj>}E>mGP{|wZ-u)t zyP4X})NZDBGqszk-Ari$8I2%mQW~8iqg7<|ii~EF(Jhkg;S=p5ne1lzuXHoL3O)oW z5=}y)>EwYO=AlPcd2x;&nTtif89DMF5F;_pmm_a`$s$ModQXlFCtX6Cazr_z98r!a zN0cMV5el0-+P`-1lQ7<0#;!!U&^`MU5qh0`ik9tv#9Inz) zFIQiVU`XOTnK+c=z^NSJN>-P#x{}$HT!AThcPvSajVZa!VoLsG&y?KyraX=m@DxEK--#r05J&cF3gTo1)XkWeaN8Hr^i zSL0CJYuuFJNun1C=Vyg8S*^zEOip&+d<)v7Lz%?bXp=iG+T^Kw+Ty}K0A#A!__7z6B?_iO_JI~Z9=O^wTaq?vRaN+qs*lz--bi^0Q^aejYFx6L;2M`hw^iGUL4A#L(!q= zP;@9d6dj5VMTeq8(V;ZO znx3%ZI*L+O-?7@1lcJPwN2q)lG9|`FsJ#CoRDOC-sC>^8jzWdbf0thN;O_E!T$VT? zG;tO};u%CjzNWZjiLQVglE4`W^nIu8I|n3iJ_5%ha5@5KBk<{>t}9X3#s0U#k)oU^ z8V8DUj%ccf#2!)lXVuDC6P%=(s1L3jsW~RnAPZzx$NqBuNH&+Vx4gwZbIut^9WCe% zDUKQ}+}*L`nllFz2XgXt#@RvaIHYMVEpy|SKELz%l^;({5<8!$=Ao7+loa>H=qN5E zmyvzz1An6-4g6+gyW$~n7%FF>aug~jp&}R+Dhd^aib6%9qEJz&C{z?G3Kec4xtSff zn;p2FRj6?1tWX&hDhd@^y;_uVxFCfJRwWvxU;~)uMk%NIDF@1YePs%QG6g}Iil9tMQ06g8 z<}pgVq-`LODJFlnCY6@J5kPrb;MNCX}fY z$`lG^3WYM2LYYz_%92<}wNR#9NY$chQMIUAR4u9&Rg0=c)uL)qwV+n0b*GMgq>z53 zl3vvU8M!maLYC{BY607U;R737nC5;gaQoK0%q1=l$Gdz1&LzgiyF765F8^cCyZquM zi+8CfKVR>ncgg&M$^3%Jxctm!t;}w%%yzBJeytoQddGp@Y}v|jVpSYi#jY(T7P4_G zvvVuMEt#!bIX1zuc`HZ7RupVeua!e6;uCYc$Kx%|J3OOgV5n&k-mN8)&6_#;f85vfPVFejhkYNQG#>93uY*@jD6>M0+h81jB z!G;xFX5mv+uwexmR*+!@8CH;C1sPUiYh3J&;};AQ3uAj+?2n5LaP}lXW z#N|Bi!oNbBhv3pffazIEBvwyyQUNndm`5OCz7hcwVu_s}^`8kV(>69?; zGqU(a38REj!YE;sFiIFDj1oo(ql8hyQ1-8cVc!DK=TPQxA^5ElQ5_l9{V4t z4+tHz8j}+pbFv-j`!F)!f{lr>F)|NbjLd)AGcsR#@5RXU8X1j@Mn)r}kKXNn zdPY5iilI@2dS)(|IZw~vXlSZ&psPlRnpLH&qUNNO=KFCq{})daW8-QbvACN5yyt43 z{|}3+8FV$e8eNU9MpvV&(bec`bTzsfU5&0rSEH-Z)#z$;HBmdWY;M|hH3*v;)+U_PXLT&63g@K!=FwQ3Ct`47Y%I>tEEebP zdlu*O9Mfq6;Rv@D*102CIC5 zRZdc`oTOekNxgEAdgUPX%0cRtgVZYrsaFnCuNuHPM$@(2jb}ZSkWXF;n z{f>S|zoXyL@91~*JNg~{j($hKqu+TW`5hz=O?EPmy_x1FJ3PEzifzu(J##V74nSg4ENm=nENmu>Pm@KrNpXoTV5%#s+3q&N~|g+R+S2? zN`+OW!m3hXRjIJ5R9JOXSam#2U@DBt=ometF_b;Z9%YZRN7ENAsik(fnwBG(Xph`9c3gGaq_f#CEUM(yR{XWVhFkqJiG* z7H8(M(Lk>_YM@W~mpu*i1GicITT|fh;F_JqQBS^M0b0{B8ZxTNI|3^QV=PK6hsOl1(AYCL8Ks35GjZhL<%AW zk%H(tD~QlU(P)Tcl9`T;hE|<(>WgMu^jK`sdt!=WY;4hO7hCj4d$#BfpSakfL0hCP z(iUlpv_;w?ZIQM}Tcj=07HNyLMcN{5k+w)%bfMTHlu}f zMPU5@I#L~}j#Njg zqYF0@fMR$8^ss+DG`^f;u_ zMOrZbR8lG_m6S?KC8d&5NvWh%QYtBxluAk^rIJ!f zsdRmlN|>eSn?$`5RZFJ=Xw^$6+uxq_>PybtAG;J|W0>x{7^eTdXP7?x!xqDoHB1^N z4U>jR!=z!-Flm@HOd2K)lZHvdq+!x9X_z!j*I&bgW{O@*Sf-fsY@Ajhv>K!%ms3|##~<0Hu1{S{sRMq-)J>^B zq&}y7H~?x+mR}?n(Egd(u7Wo^(&TC*9M< z=$;Tz(bNfR!!$Q_LgQ>y)LaDhyNIZ-MMA~ch^Q}GMAU!Z6H!Qb8eH1TyjqzN+@Y7RGdK+YYoa|iU?fw&H&b->UaP;>_z-687O!r{$r z-y_K=X^UJVRu+cK3z-*(3!?Mp{_xQr9-%ik^@^TeIK6rEC01`PjKk|iRUA`zK^Ddv zCEh2zO)`EjBlt3gFQfP}jxQtmGL|o+`7)j_BlKmLTq0~_bH+jQHm%< zlp;zIrHE2QDWViniYP^tB1#dZhzhC)o)6>9_u%t^??xyeem)=zm>}o_!hk7s0M39Z zUV%4YidTd?2zd~5Xo2ttvp|>)!mJQxhcHWo*&^^7mK;4oydvs>LM1STO28$=%%5`9 z3A0ZKp)ebTD<3(>|6jc1q*XZo#D7f13=F*a7}YZfc@U-zukL+`s-sa`8(d6ZU4E2WjvN@=CEQd%jklvYYB zrIpf3X{EGMS}CNGR_eq`p{7D)MeR!zzl5Y(71BA9YF1QFM^Sx0dMd_7QGMH@s2;GV zsBSVXimIuoR8%S|6_tugMWv!rQK_g@R4OVJm5NG5rJ_<%si;&`Q8Tp^PMxQy@KjN1 z6+5(Ii&hw`)hMmTYW7vnz*qe!t}4dHS3Pp^RUf(MtKQ*cqu-#f(pTxL^i}#QeU-jS zU!||oSLv(tRr)G@mA*<}rLWRgy}`Z;X*F*^q^nSM9-g13nXZ^$Uor7#n%@ zm_=UQYfoN%-;XWDR-^Jtd8NEkUMa7XSIR5pmGVk?rMyyJDX)}Q$}8oS@=AGiRpb>O zE9$eNDRy+l4wtpMrqyN5WmwO`Wc?-N1IRT;PXo4QM+M`x`nzJ?Dj*Wv18rxo5 zWpE@xV_joi8#v6Mv9PglU?EtT2Nn)299TH8aA4uU4;)xGunvKeN7jw38(BB9Zj7Qt z{9=x?8b%g!SC!+ShB&7oj%kQf8sdU5;lj@0LLp|M46`7z3m&_mu?wI5!biVwwqxm8*RzgL3ZWE2VPYYv6jCXq zQb?tcN+Fd(Duq-EsT4vfgi;8l5K1AGLMVk$3ZXEukW>n(6jCW_Ewz?fORc5WQfsNT z)LLpSwU$~-t)Ju$BKBYh({(CVje5zv0@%8 z=CNWPE9S9c9xLXtVje5vu_7KT;;|wgE8?*t9xLLpA|6x4JXXwO#XMHbW5qmH%wxqo zR?K6?JXXwO<-4^a9xLLpA|7>IIxZcTj!Va-ZorSMXCDZJh^!VBxgPOW(}S?neY|FznwbNts_?)7}^*K06eF*f$= zZx{RZ7JK&VB_Fidud4mherdn7U)nG2m-b8hrTx-=X}`2z+Ar;w_DlPv{nCD2EA}f# zGbR3H8ux?UaZ%P>BaP7dNIA2UQ92h7t@RB#q?r&F};{xOfRMv z(~IfFu47&dITlUC;@G`7cP}*AsBFWwrK^9{p$!@twvghwfviJO)MUpio znUYLNrX*96Dan*%N-`yxl1xdaBvX0-C$v*x-?zd1@F=jYJD82IYd*m;n);yHD9N0Z7x##JH*;YA=P4R#M-?UvG$lfv39%f zJ&LvdpjcC^Db^HgiZ#WWVokB8SW~Pi))Z@sHN~1@O|hm}Q>re)KzY1y=F*JI0;GvLb<&DX}c`C7j!v2&!`T-f$fl-uW^+hS~#+XEKm_HBF0 z?O`ullv`1`soYdpP35L?Q@N?!E*|BEcZ-H?V;55& z#~<{oAzKaH?BD(g|MsQ0w-_7$_FotO_BDI{?eibI__wNm)4%E8^l$n%{hR(x|E7P_ zzv#wp{JamqMloH9-s7e+4n!tq-% z=jo80ZWsp-^oYC1KYnodoprc=|Y>C|-phNi>S zMFYAi8qiJ026U@bJGFLmY29mZc0Y-)i?MNb|9f$E_uX@LzxnZtvm11FIy;@6&Q52i zv(wq>>~wZIJDr`*PG_gH)7k0lbapyBo!uMm>=1X+v~D^!ty}flsl7WXx_cc0?+M7e z7#o526N|w6pgnH91KZ&ctZ@Dz9oJO!QtPl2bvQ{XA^6nF|e1)c&=fv3Pz;3@DF zcvnT>Vez8T-9)3iayZO=?pE7&j?bIR@m`P3`)y2KjE&9vrN!nAdp7R_er2(FlQvJA zr_IymY4fys+B|KZHcy+U&C}*-^R#)|JZ+vfPn)OBy9R6?N{`-n%?LB2td+)KQ_FnYtquOf+)t+ikwWr!s?Wy)ud#XLvo@!6Er`l8P zsrFQRsy)@7YEQLy&8t299{u#np`VYcJ@s&_rzrR?d-_7@=-xJaz{fg30>8JEl`YHXCeo8;3pVCk1 zr}R_$DgBgwNOgg%I#3;` z4paxK1J!}*Ky{!xP#vfaR0pa9)q(0jb>MYE2jT>yL16CZjl!$BI~9W`rGl@=4c-Pf z7~_05c*~nEZt!pR+~DmmU)*4?8`KTz26cnFLEWHkP&cR>)D7web%VM=-JotzH>exb z4eAE3V{Q;Z7(E4Z^c2j;o`S2$J4Y4HMT9qa?ImaKh$M`$5ryxth{EUWiNdFR^dbtI zqEJz&C{z?G3KfNlLPepXP*JETR1_)-6@`jIMWLclQK%@qZi_-}Ve}y^hvV_XtJ6Ej z8J-+Kj5Vwn!x$TDc!$Lr{@$K7eBv_}YdB~PwT4h2_}1a8-QgNW{7D@Xb(&?~gu=u~CR+QHZ~`rw~8% z=NE-Ist{F(Dnu2c3Q>iqLR2BD5LJjOL=~b6QH7{NR3WMmRfsCY3soWF5u@Q@>0@eh z*TdELtwwS7iEn{Vybmrh#>OY!eesDuv*#1<`lX9cob-wMM17(@QJ<(!)F^Ee&Z*_)qL-bcam2rp%^pwsdn! zHRYFM;@BAuouPLmog)*CBiZxKj&JsRQ=Lg|Ch`<@ia6yj!(XL#srIS1REJc@RHxJ( zQkPR#QpX?JrLIq1OR0l<$J9-!KcqgVzQ91K=Rc`WeM|k2`apuz4yonTN@_K=E;as( zTWUjUV`@`E)Fv+6P|{G-(50bI1An6-4PzRngkBkQ8cP~$8ar++lG;)2sCHC4svXsi zYDcxB+EMMOc2qm69o3F%N42BcQSEr~Xh+;*^l!x1#XL7~M5S#s;XA@z|J1fvj^7A`j&Mea51<{EDVVo7B@6*c-;B?@ThW~j{%Cv8Sw^) z{++CwciXcr%3-4#7N^S;N|=T5PT`G`d7EU^ZpQ6qL7KHI!GO)4pIlHgVaIlAa#&BNFAgOQU@7S4?G{nneV~p z1K*8MKKy(@7BE532ZR9=uzkQ8FvTnI22Amaa0ej|V&)~YKbQr=Y!GIJFgt`JASvh3t%lKEWSEj~{iR+cAGh>QZP-_-*yo7=?ruf@abjB1fq410;UPAF1 zQ@n%%G^Th3;*2R?K^+>?{NF_}8vhtCp&*SZUIO?s#Y-qmV~UqhoW>M?NF<*l`8>ue zBKb`6F~>_H`W(^ch(1U3Iik-IeU9jJL|?)il}NsXIVz!!O1PtfJc^l@l<-F-1X79U zOGIDh8o-FYMD!&rQVESzBKZ;`sYLQ+R!Jn!KVIojY5p}{a&jUJ{}eBY=&!uVnzu>( zAt9E|bCCRdyac^;mioh07jBCp-wq*p1u` z3Q2{eLQ)~AkW@%2Bo&ehNrj|BQX#33#3w2w!%8V6FKQu)l?+iD6>o)-xA2pzJv_%x z&gGJ~#7;f}GZ|xJC;!J{Cm*N$sR|Qah=g)J|$AwUgRO?WA^6 zJE@)2PHHE$lOZOxlTrJ4F_%l8XD3mVQDGTdatnKMLsqUbag~*`s=PI-@`ursF*d65 z;ft#L*gaMGyek$}*;JLPN>!z*QdOy{R8^`fRh6nrRi&y@RjH~}RjMjgm8wcrWmrm8 z<&_F4&r_BBYwS2@Pk%_uRV%L2^5j&U+u$vK24@*#<1K$^@s{`5^Om1|zr|Y)dP}{f z-coO=x71teE%laqOTDGuQg5lZ)LZH;^_F@|y`|n#Zy8N2-!N~9yo?@}ar8-{7innB z)iR!H%(?jT9ghQN%mk(fWdNX+-&lb8?Tf4E8f_)&?e#8hG`F_oA~OeLlgQ;Dg> zRAMSIm6%FQC8iQniK)a?Vk$AOvc%*#r}?-aPE@**a6I*yDa}UnwiwOd!)C_V7|mZ@ zjAq?4nwLC#F`AP`Q=_TT)M#onHJTbtjiyFZqp8uGbQhTY8WTqhallX<-JaqMc8*hLH$11WQ&GxabxNGN&XMb-q*Q9+v~|u}FD-G- zI;X6cDb+)&IdQr=XRFsP)jrjhXb;PQ>J+bXb`d8R(HWG!pmYUgqdnW~6VGWBEw~a~hrUmJOZ|}gK#$Z8spZs4YBjYkHU5iRYC~#cYEv4}!%*r_5~sFvFcR%X z*}PBTE5)fCe9zyQL+?5Ao&)bW?w-T$)tl-~^`?4Ly{X<*Z>l%do9a#Vrg~Gosoqp? zsyEe}>P_|L8$oa4IHSQPYL80JQxkbAI!`)mz6-ANZ}6NkHm>te7T0;3J=giPcVAp* z)^+MSb)C9SU8k;7*Qx8&b?Q2Gow`n4r>;}isq55r>N<6uy3RMM>qK})Q%*`5ndYXP z)FUoMnCGa^xyUmiK3{?KjIj})f4zv$OZLR)ecoXapXKD|TjZIz*P9M$%4sTTs%h#{ zn))>H>wicSRj*O@O5N*$!q)?puLnwB57fRMFx3aDUk_w=Ai4wV=K5IHMTIOSvP>+D zD%mKLjXK#Vl%-NOQ!1OOmCY2((z=zwI?81;^|G0Q*-XW3rero#Gn*-zWr~_x7E(By zshrJ}&Sq+7GsUwxTE9|0o2j486wqcWXfq|WIS#9$h&D&d*Xa0487(EdETob)M_F-H z6;n)`L%QSKbG!}XUC5g-Q&O9$sm&DCW~yp4Wwn{Q+Du_>rm~iag%sCjs%taiAX8qO zsjtlx*k&qh<9w`WE*l+SGgY>s^E2=3OrdS2(l%3So2j*BV&TZbIDCvOGGS!W$i$J! zBNIp_kW3<(NHUpZLMen&2&E88ArvM`j!C7EN+Fd(Duq-EsT9hV3#k-RDTGo8r4ULX zltL&a%9o>lnNXNm7$wY6!yHA-QNosDuWrVxQ4z@uUdj~y?WTTO-lkE$)Ecu5q~Md21xydwN} z)NUcYXWMwJsf(A)RdsV&-CS2U7uL;{b#rN5)Ye5A7gPLqh5wF9E~fZHsO5?|{*;J5 z>baQWB~fM<6jq0s9r<#DUATz%-tj`%yG zL~o7~jd8vbz10mDCHk^GCHlN?T$E_15><(+L{*|HQI)7lR3)ksRf(!ZRiY|Um8eQo zC8`otiG~eTiAMEllmx3pk)GF%62*(osx~$y$Ntw)q^rSP73o|udMEtoZE>S9Hh%PG ziywXAo*(_COBX-d>qqsY`ceI;epElIAJvcQNA;unQT?cXR6nX8)sN~&^`rXHP^0?M zYrSOjd_T&+M=2{k6T+0PE^{@dC&i@ifh>IwL}`qTEPba%mOg7wmj2L_7g^errOHxe zsj^gAsw`EODod56%2H*ivQ$~BELD~&OO>U{Qe~;K^g6M>eqWZ-IXCwG!*5K-#-yv- zJawmYndzM|rvuhB#>Slf$Hkof_MSQYx+@lQI%rNcr|szz0#s!`RbYE(6<8dZ&|MpdJ#QPrqwR5hv^RgJ1fy-+nOE;SCG z8lwN{aO{7&I?r>g>Rg&yaH{vkr^eVg)q5;X^(Xh7>aV}$;#4P{s!mm>s#Dde>Qr^A zI#r#jPF1I>Q`M>JRCTI4Rh_C%Ri~;`y`DH##A^Ia22T1aD37^vb#{>wP#ze_{hb!W^Jps zRokj<)wXI|wXND#ZL79b+p2BVwrX3ot=d*?tF~3!s%^dA+g6loG-n;VnEI%5omM%z z%GbHjbws^>5!yAzM!kOKqF%poPrZKJ9TxRkRIjR6)vM}N^{RSRy{cYSuc}wotLjzt zs(Mwus$NyEs#n#k>h)q%ulUz!=*j_EAz@cdx=PrS64<-pVZR0k8)M^Pzi{!eU%KaE zpZzb3hpl>8J**y9537gO!|Gx6uzFZMtR7YmtB2LY>S6V;dRRTI9##)~@p@QfZ2aZ9 zA=i=D+Maq5~LN>-m%6|1CWj|+6%0A|4iQdTLelvTRNTJx>jAQu2t8nYt^;tT6L|uR$Z&ERoAL()wTb>x)y2s|Fw7KQTuLX zT|bT~LJlO8Ek(BFkmhsmeUFy51VN9MoaF(B6h(womctVBlwzXdR8pcsNe&AIXLBOO z6bDw~tSeAKD3Khp6cvXIa;VSue((2wf1GFC1$sE+=Z|}@AMkA6;d=M=?C0~@*S>N= z?Q(HJ?M_y&!0qKx+XumJ{{!4M&l$V@){Wg>f5dLz`I|O&JIQWkx3XK=t?X8IE4!85 z%5G)1vRm1$>{fOwyOrI_Ze_Q!TiLDb_O)QQ0Nnh(7N)e*#ih18XuSsIu7_|R0?7R_ zAa0&Bko$cb$o;!VAoqKox`EtTkSoX)b)~veU8$~ASE?)3mFh}$rMgmGsjgI4sw>r%>PmH`x>DU+jOv1RxjeU> zhL!K`1a{}Ump3px6vX>7@NS+n#QTB`@jl=P@&2nZt5TQpc_h8YH6h} zLmA6d=E6^O;xC(n#C$uYIp7ZYAvOPt{XQEBZjxY0uq0R#ED4qbOM)fAl3+=&Bv=wG36=y)f+fL{U`enfSQ0D=zAZ^G zOgNVTbMtPP@D6RSnDFIo!2b{|{MW$3d2Wt{@AbwT7XIBMEd2EQZ&8E2{8WKz~Ve-!1(?fF#d%j zVEm|mwgKZ&U@R~e7z>OA#sXu3vA|ehEHD-r3ycNE0%L)(z*t}`FcugKj0MJb1Yis` zu0h8a_Z07NcSpyUS5iC@di-}F$9c}^@w;vG_{Wdv@dMszqsNo9O=!dMrJb9!rm<$I@fzvGiDaEIpPUOONkZ^caMkyN&BmGLGv(9 z_X8v6IYY_sv7zMeKSIfW@FyEeo<+%`WKpsxS(GeF7A1?4MaiOMQL-pmlq^aXC5w_p z$)aRYvM5=Ue8)z~@Z?;GTrVy}-m&ivl`oGdKL)P+aF}wQGp_u;8(02!M_l=-Keln@ zMXoGYmMhDZ<;rqpxw2eYt}IuUE6bJT%5r77vRqlNELWB*%a!HIcNVS;T;@k_ zi%XPu3Vel`*Tc&n3uJy2fH}_@WPaENnV)_HnIHR|7m&F-37Lh=LS`YekXgtqWEL_D znT5{-g~*f58!cekuRM-EIWB zRq?aQvpl&>!r}$4vu#$UCIik9|xHJ*Ffn! zXJGn^HZc7OM_~H#pS6MMUN9|~7EBAK1=E6Q!L(pnFfEuCObez3(}HQiv|w5=EtnQe z3#J9rg6T^z4V}(K(tXR&ug9l%j=Xc~l~R8yl=>ME>O5zZ`l%bG{-`5L{fPJ8DD@zv zmQqWprPNYtDYcYZN-d?9QcJ0&)KY3GwUk;)Ev1%HOR1&QQfevn0i_02=kn?P;_~U8 zDDS-bviadnAnWf0SLZoH)?c?F>km3Y*7e6XWIc+kMb;u~k+sNLWG%85S&OVi)*@?> zwa8jzEwUC_i>yV~B5RSg$XaCm|3%ht>s(mf^T*QqiwmoFzm4;;k?hOE*q;H+{wY9qo->&JoDF8rM=<;8pSZ#7S(q)% z7G?{xh1tSvVYV<^m@Ui}W(%{0*}`mLwlG_mEzA~X3$umU!tB2|%!X*^UhEd--oFp;&U41Re{EylZ+XPL-~Q1X z^X_EcGH;o;%vK%cv0Q_?S;C~MM&T|I9U$Fu3H#`F1e=uzTyb8bt-~w;~xBy%LE&vyR z3%~{70&oGi09*hr02hD@zy;s}Z~?dgTmXLS18^vKF8UtVi@xg)t9KNB*~9Ylq2X_K z@9Wm*jE4W&1r7hG|8+#eKl_a~8s4Pg(r{_GG+Y`k4VQ*X!=>TUaA~+STpBJ7mxfEj zrQydD{x9IBa&kGjoLo*WCzq4U$>rp7ayhx2Tuy$wxO67rpDUvcyG zaQTyf<`rO`=M0*^!v@X&(-AcPjrZB0`6x6OnhVW^=0bC!xzJo_E;JXK3(bY*LUW+bP2iyU4kw_m!M0~?_dNS zjGoKRr(AYEU0im)hSR_I2&cc_mu)zG7N?8T#p&X7 zak@BNoGwlmr;F3Y>Ed*8x;R~&E>0Jxi_^vF;&gGkIQ@=|(_!lT5x0prHfYz4rPiH( z?{s}->ra8Le;8an&ly|)fQ_wx^ATHrmxpg`eUYup)@AFmb=kUXUA8V;m#xdzW$Uta z*}809wk}(jt;^PB>#}v(x@`SU#nu7rxl?_pk2@| zXup$!c4&L>D=xl+_w{)DlmGng*L^C)JfTA+ zrS4L9sk_u&>MnJcx=Y=q?oxNDyVPCkE_Ii>OWmdJQg^Al)LrU+7oqO3_gvmSua~#C zosjSR{qo@Zmx16v8~mQ<48cEcL-3z_gy6sY`VGNX5xfXq1TTUY!HeKU@FI8-ya-+d zFM=1ri{M4@B6tzJ2wns)f)~My;O}w>9uCjnePb`&wPn5Vz3ssKipXEw_zsW%m$3Le zXFUE18;}3ABOd>_du%+u$>Zhm@_2c?JYF6zkC(^GhtTI@`6U<2 zFT;AT{7%xZVfE`V`lmtazY$8G=Zw@pZ6oy`enje@|L~2}50ZLGy`)}JFR7Q*OX?-{ zl6pzKq+U`lsh8AC>LvA(dP%*cUQ#cqm()w@4@f<@K6lYCQ-*m3_d8hM!TpNuzY1*s z??Lu?&anMAY}oz-kFfoxeej0uN3p%wUTiP67u$>N#r9%*vAx({Y%jJK+l%eR_F{Xn zz1UuCFSZxki|xhsuNk(7-SfxNpZI;OtBUxqKkm*LCsW%x3D8NLi(hA+dH;mh!4_%eJMz6@W6FTkRe7`}zCoyHhUjxuWt{(Esv?|2M<^|0m2p&l&gs&5irt)+?P^*NIOf3!(}>y9MA`~1!(0jd%}37`Z}0w@8L z07?KQfD%9npaf6?C;^lJN&qE*5ar1EMVvL*Z6=^ zRwMB37=bs(2;{lB5xCFmT^NCn{KF$7P#?V+fu<4A2xtT}0vZ90fJQ(gpb^jrXaqC@ z8Uc-fMnEH=5zq)|1T+E~0gZr0;5Ib^c!4wv=+Z2pTbl*+JHEe05UhIOJJADw9W{{W zOb`6kO%MFHBR%j#58m`ZuO3hjs0Y*o>H+nDdO$s(9#9Xc2h;=V0rh}-Ks}%yP!FgF z)C1}P^?-Wd_M``J1pG<%^^~Do%L#UxzuSV<6+8=9@XmOGJZG-pZ8lf%;v-k^%)4){ zV9*ul3Umd!0$qWwKv$qE&=u$kbOpKsU4gDZSD-7<73d0d1-b%Vfv!MTa65Jd2!k{w zs25EMcCNo$gUi;$--9?9kp_9r#KAjl;^1eF#KGr2brT1p;y`hrI8YoY4ipE91I2;j zKyjcrP#h=@6bFg}#ew2LaiBO*94HPH2a1C`ggC$+q>(`#GK_2W!H)T_Ji=Oj@N6u? zgD?nr&Md;PS%e=xvIrmjVVgymvU%<=?gZ$u;>@`3;G59f__22 zpkL50=oj=0`UU-henG#WU(he;7xWAI1^t44LBF71xO4dhBtx1kw2LMSy9&564VT3X zKZI;}ETSRLnQZuoO*VYRk!<+9=U&K$;iPO(HYgjE4ax>(gR(){plnb!C>xXw$_8bF zvO(FPY*02R8Fb-+V(9)Qpt&JJ3%)mA7VJ&lbF6QB%U>)+DnTL6aqhWJr5#OI+P@|G#cSw|Y;A#bv2h^mHA zL#QFt5NZfDgc?E(p@vXHs3Fu4Y6vxi8bS@BhEPMOA=D6R2sMNn;;yM7y1qXlB>J`g z!>}uZT}WJB%Jkzni7&-RB2v>??Et>dA1jW}N zC-R&LiYIS^;?YNf;uoH{35s4pp`cJuC@2&Z3JL{y1%-m*)dU5WVkmJH7mX%%nQ-MQN?C2iPh%^-6;qMt%vOB$W-C7A$W}b+ z12tOys)x=>xH zE>st)3)O|{LUp0KP+h1lR2QlX)rIOpb)mXYU8pWp7pjZbP+fHWkcJlY)=r1*-7;KB zjLVJ|&&Ojt4~LQG%ws%z^BC`W9MW=`r*edJH{=9z&0z$IxTwG4vRE3_XS( zLyw`y&|~N^^cZ>!J%%1bkDQr>cIj}9)>w-(ejcgu0)$4M zGpX^To76ZRNsVuM+$J?NsiD+RYA7|78cGeNhEhYRq0~@nC^eKCN)4rkQbVbs z)KF?DHIy1kjlZnaz-$cB8)JrPE#251#O^p&!|{t4j$grU@w7pdR*CwU4N{l z9xuUtybAA;=gfWl_U1m`^vHeO_xm>Y(dj;PAG#0Shwel7q5IH%=st8Gx)0rl?nC#X z`_O&pK6D?t58a3EL-(Qk(0$x$?gIgmrXJ(k)B_=M{S7zu4|~uJy9K!{0{LY`$Q$4H zy7f5|A+Nj;A&>aeBN6h-ZWAH(@(Yx%)Y3{{hBB6^%!PGN({zjsjYjevm5$rExQlBl z9Z7L1^#naGwU$y_sXdL=u{NbnrOu@;rJYLaO5^{tmDZOwls1+&l{S~Ql>SuuuJB9k z`I3F#JI&f@aYBO=+M93>7`K3N2N?A+RL9_k_{HFf_)YP98u^aL zGEQZj%ea(@j;zS@U$I${cKLs=6)AiWTuy~Q{H2zqm8CCb8Op+E zgQ+ZNqIe=i(Fs#@LKU5GMJHs@iIAO$*a=^B;?3j4>sFz}>!Or!N({U(S}9&BVku?` zwbbF3I^y6Fgbx0@vo_Kg2+Nr}k;kbAq>Z!v$X+^XmS`n>?RzxeJ714@lMYJMX z5v_<;L@S~d$)|cgJfCJhJ)dSiKA*^qe9GtZ^C^_<1TJ6*4FDN1@M-%5HDJgL$N>X0 z0uM4WW#tDeKv)UFiV#+YutEgBKwgq57aSooR+5n8sTU{#mXLq)RW5MC$`b+BX zjAEQ4%bPe_qg} zOj&u-3Y1o&v?8UIDXmZ`RH+6l)kMA~@-<+ohAdUkQpS9h(4`u@RKu5Q08mK zL_TAtB=j|*uL*rk=xah>6Z)Fa*MzGL&OXyod-@*-BBHt4EmdLk6 zz7hG1nX(>9Pv~1h-xB(k(6@xXCG;(!Z^4Kyk#7NsEs<}Dd?WH1GbN#K34L3E;z|?~ z`gVa7Gi8N}D^*;v;>s0Qu$ag`j*JIM7|~R zEs<|TK4YdN^ev%p34KfGTSDIw`j*hQguZ3bw?w`r@-2~XiG0gBc*aag=vzYH68e_V zw}ieW^ev%p34P0=Z;56yyi(;AE8q53gm;&GUxYrlXYB&x+PaalDw3Z-k=!3WlIKj3yw#>ie)UL^JoH7I zBI#8mtOr&DYk`%(I$#yB23P_7{i{f*6=De{V^HLA$ZjK>jY`(Z zPJ@9Ll5;94r;>3h38#{8D(R+@ZL-721|$26Y%j9A$mSwji|j12vBzGkPSoj3)wAXvyi<)whGxPWTTLMLbeIn zC1jJ3Jwmn!*SLZXuGL3Rh(9As~htwDAM*%)MBkZnPB1=$p2PmnD^b_CfFWIMpX z3yDfL1KA5?E0CQ)HUilPWE+rOKsEu{17r)39YEgy^Y)*2|GfEAsm2S5O5XY#QOO&B zBPxxkG@{aoN+T+H%g-Br-tY5vuOd;AuvB`MO3zYZprVnr(z8^0mP*f3=~*g0OQmP2 z^emO0rP8xhdX`GhQt4SLJxir$sq`$Bo~6RT3t1~YOQmP2^emO0rP8xhdX~z-QW;n( z150IKsSGTYfu%CAR0fvHz)~4lDil+uppqA|R0fvHz)~4lDg#SpV5tl&m4T%)uv7+? z%D_??SSkZcWnifcER}(!!oUkzD+5bqV5tl&m4T%)uv7+?%D_??SSkZcWnifcER}(! zGO$zzmdZf>8uDp)Ab|}euz>_NkiZ5K*gygsNMJ*%K1pB$32Y#N4J5FE{56oj2J+WH z{u;<%1Nn=A7m~mR64*cj8%SUS32Y#N4Mb%iDg#j&2+BZE27)pWl!2fO1Z5y713_Wn zg+yf_Dg#j&h{`}z2BI<$m4T=XL}ef-13?)G%0N&Cf-(@4fuIZoh4NJDQ;Et*R7RpQ z5|xptj6`K5DkD)DiONV&MuIXDl#!r}1Z5;BBS9Gn3Ii`BDkD)DiONV+Mxrtjm6527 zL}er@BS9Gn%1BT~f-(}6k)VtOWh5vJypX7jL}er@BT*TN%1Bg3qB0Vdk*JIWWh5vg zK^Y0kNKi(CG7^-LpfK=4qB0Vdk*JJBWh5#iQ5lJfibO@iQkhsP6H8@csZ1=DiKQ~J zR3?_n#8P2MEq02rsm5lh%q*3er82WrW|qp#QkhvQGfQP=smv^unWZwbRA!dS%u<!QJIO#OjKr~G82`V zsLVuVCMYvOSqREPP!@u+5R`?WEChuC^+8k?qOuT`g{UklypX6Y zL}ei=3sG5!%0g5YqOuT`g{Uk9Wg#dFL0JgOLQochvJjMopfK=4qM{;EkzhDjgA9D+ zRFQ!H;Qp*!20oiZ-7@g$Tt$KofljDMR3s`A6$v>q>PAJ*r;dvkG*~l?m^c&fI#x=%d^(8-r zFL^gyNuD!b@^+gq`Nbn&a`&4yUoz-R@_~ep8TpjK=LbF-u<2(%&rY3vbN1TooGmHQ zm*52WV5l!ahv-XkCL$*yav&n-A#xldry+6}B4;6T6e1@fau9-n7n1373?io>atI`m>a?Bxb;dujR0O12sa>5}89CE%P#~X6GA%`1swjoCwa#2N`mXA*UE}h#_Yf za)co#7;=Ci=NA}wA)m#1K8p2x66^UO*7G^6hoST^lpcoC!%%t{N)JQnVJJNeCGW<0 zGtPT)-ijG`AyLU2aZgn8Hk=k-c@NH8aNdFQ2Aucby#40gH*daq@6B7Uz65zdRP-gt z1AWQJQW;q)BTHpusf;X@k)<-SR2VQYB(RYLHj=RjU=#<1U8bu zM)KE4{u;?&Bl&A2e~siXE=c8tB(RYLHj=RjU=#%1U36H%Fn%0yHqf-(`5iJ(jbWg;jOL7523L{PYKmKPG0iKt9OWg;pQ zQJIL!L{uiCG7%Mh2{wYL=u7k^ER}_&vanPZmde6XSy(CyOJ&LD#(Z$hC&zqr%x6dP z*Fyec;D!18NCI0(VAK`x5&uLAV*m=ANMZ~?ffH$rA!MFJ#sCyJk;$k*2uC=P%os9- zbjFY=Bs7LhA*C^73Q3KMg?tgx8bfA~*cdV+sf{sHNNx<7LV81sWC{t6AyY_k44Fcb zW5^8B97ARVnxDvY88ao+{1j?_BHJ-!3i*y9Q^nL^HE$c#XgQy|JI5apDe8)K$~ zqMS&644FduW5|?1lv5zeDG=pE3S`I(k|0B7kOryBSidhaA^*%2av?*ekPR6!g?z}6 zDP%-6NWKUuks&jPG(%<(>x`K~xEV5qfK&02DTJIMQwTakrVw_9%pmRznUTm7P5!?K zPzI*(Tg<=|e$os~;oHf;6y`7xdWOtMW zDM6R>77;qbz*|K43eqHKbdC)vPe|C;SLxMts)Z7aK0 z240v=D|=RpO(-rwvasK@*aV)>2L+2wlqbp)ib~Kne73gO#9|YmhQ%hz6Xl7;CKj8( z-Z`d*asa(sY@$3-o+wYekBiHk;?2B|%lo*nnou|I1 zi*pjuD7)#n)F_wz-k*<2d0#9_o->oOY$oLgj!eq)zHKuplO{!zVrLVt?cyMo*LHDr z7quIlpUVxM3>0sm)N?;)uIJ3{oVlDccXL{XVi^i27QMEMdKK1+*LHC;7lMQjJzm>o z8H!~nT+_u(Qk<;z+Ago{;z+yKc3Fl(Ifi8@mZ8|$#LgzvL~$V3&L(y?v9k#^eW~3@ z*`b|HxQ;88D>=6RT3_3BY*P4N@+(foX)axlSlPwLE>_k8m0v)sJPf6h=S-_SWYa3& zexy~N`h-oZ%xV?2idsdjqE@jR3Rhk64)5(=-0;P- zyba@R7>`2Xf*%{k-_qSsUSqAoyHeZ@$6_N`b_KEvmdnzW7vWew8o!d~%(2|KIhLr*BO<&ZK#c9p`K8njPouIB&;!BuY9{*>V0h>!5;j*)9R& zU)PqdTsap0m8L6cme$iOEy`s#B3H^~Eoyl&!sU~YEqTs_%Of}8@{}Xt@;mQxAzbK` zas3T9^$&Z{4TUz;g`43EeL@R|w+eMNY$;-*x$}?jhaMbs*PyXRNqe-Vu>Gbn8V8(c zFi|+(#OWsbq0tSEUTAzkr4JfS&}e~112k&Ip*Xanpz#@P&(fHf_P}%p=CT#OWO@zL zH8{tJxqO9-=?f*rIB>4!;btC+3h8G@FA$ED(Fue`AoKyDponrJnt%ie@Exa9D))HP z)|7^&H07ipC+#@t#+f#tH1wobCtWyc!pYqQ1*3(};y2CjncuR)Md6}wQMf2v6fO!E zg-iAr3YRgPD}{?K>)A@vlUd=Sa8bC>o?hXiaIs~*E$eMrZ_9dH*4whaYYSZ5mi4#8 zmi4c_aKXAn(a?k)^>XD#uGGtAxyygSzC0fDlIP66eBx$bzVOJteA9C``_gG&v@hBh z?ThwB`=Wi(zGz>xFJn%&XkRcdoY~xJh7mj~ zr0ohvs2MDGSva3w7qVpK5LeeIdSGCwv{TNppVGko#P6%+UMV}Z*r|nz82(u7 z)Y4N`!SCT#=+uH=*{&4hU)PqqT-g`?m7wIDMbA6^YGZa+a%E%ILYQAh#e5MOCeN9Q z`HW4)e8!QA`SdT{R7_RHsA5zxsu)#_Dn=Efic!U=VpK8IQd8Bfib26x2s2ZNKdWLE z3t=pT!NRCwEQGNT#zGhiVJw8P5XM3nWJLsQpKe}#x_R|nWks7W6l9;4PIsW2$BtDD zLMA^7R|2F&X169+Qs%Nm=4E)9r{ZMtoOziq+`P;u9(kFMedXq5nqEdPqnFXk=w9w*L*>1A?>v0jE7sr52VFQb>CIz}&}m(k1UW%M$78NCd;;;!Un zkTa>ONe8f=yXm57c8PM0rdf+;egjGKbqJa~XOiYAo1}Tg%cV4yJdDU{~RU6WFha)fyX}d$Eaz=h%ZQxy{4Vf}z3jIeI zGKKz^44E=!%9u_)V`hw*F=ob?88m3mm?>OD#gHj*bB0WrGG)q?DO09Qfp#%u25;;P znL$%IhV}ab3G>fPN&k@+L*LSAtmV$D#+_FTnUdOyM(7zbBav_EIn>f~sHf*pMyB*> zI@G7>P@kqleVPvSC#Li#rr3F=PvmdcooCQ7iDN?l>SZYJDwSd;ojeWSj?TLtHJ!FgRsZWof+#BImjDW)hSe0mxIQEY1`q3 z)8kkQOL25LwU7SkqksAcqx8C{=|Nsok5N;PQK`qs82yuijGBUsnu3fpnnhnzk5N;PQDgCHES@e$m!r#}5all1 z;pVUu=VmU4|43#T;_jn$cJFefbxK*q&hH|2UWnAmb0&6vViP;>dL(wbr*C3sR_rKt z6g!F?#g1Y}v7^{g>?n2=JBl5}j$%i#qu5dG#N@zuPhq@@9YAKBj$+3mojz_uv6J#F z?^r|ekR)52Uh|GM?^x?|y zD084FGoEMHFjt;uwLHI%<@q%XPo6W&^NXA1dAlRabKgg8mS@rOXnC|eS{^NrmPgB@ z<oOjZB zjV%u^dp9`UC3Pdp3Ck#e~9vV1-d8CnezG7P5Hdlk@9)whg~QiK=1W8+|)np zK{u4H)Y3{{hBB6^&@Yp%4)1J8Yg8>^<-01m^ON^}-uaipU7uX^$u*yaTRyqtlN&y{ z-;>)tx!aSQJ-OGDTRpkclN&v`&y%umioV%6ux((|z`+peeK{Gxxd6&NDfZ;AreK9} z!c=)81yObSNmBWTPU+bry<7WTAU)@NZ)3-@nnxz#M}v9RaWyYC=& zE^ln(pVt=lTqz&C4{s_%jLU#?!2n(PnQIKtTJrP9_@6(+{p30GKfky6pEo-4KTrGV z&Hr@zAN`O1NB^V$(f{aw^gsF^{g3`f|D*rW|LA}8Kl&g2528c=leR7TA8vBvRCrCZ zCQE)S`LX23k{?Tc_zm15B|kUwKm5liqtx9DRNh1f?Yd^yL8}aU6*A~<_e1IA`Ka%G zi)Y^Zg+KUXpY!J5^(NOn_`3W4>4gmXh*uuTppXBWO$JqEkTOUaqzqC9DT9;Qqa1~p$&@OIvEp&NJ+@D~E?u`}7b8|Cv zzc<~?&~G1^q0<95Gt@Lgnjy`QW=J!n8PW`ChBQN(AJsG>z z6zw8sH$`g!(VwG<-T_6F=S&mbYtuw8J<>#Pm^FmPjvb5;+ue--}9lj{fnKLSzGkX4!GkVI)H)k~HjC4jiBb|}XNN1!o z(i!QDbVfQOosrH+XQVUI8R?94Mmi&%kuW_{|- z`Xv|h-R=Yu7AcL6>P4fY-Su3lq|0)oH^wG?046EVnN51n%_cqj$R^!5ZZ>JsCTWwj zN!lcBk~T@3q)pN$X_K@`+9Yj~Hc6YLP0}W5le9_NByEy5Nt>ii%H@UHB+OBYvF8I5A zNru*!8%v#c6SSMA)id24&-54^Q=T)=^dXyP`syRk^s?{XJkz3Q(lhCq^h|mtJ(HeE z&!lJ4GwGT1OnN3glb%V>q-WAI>6!FQdL})So=MN7XSx%5CImn_V-4-13DYixcI8yc zDxKaO>GU{+Q=T*F^yp1GJ^4sFJ@TtBq|x~N++e0 z(n;y0bW%Dgos>>WC#93pN$I3?Qaatar4ue7jhhSjC-!+$hL>f>e1?;)QF&wBgzK8TkqM)9(3_ed&kN zkEQhR(7E(W8BS&B%1~irW#C&H$}pB;Dnn3F8Bb;G%2>GGPg4KeB>I1>P3fj|Q@W|RDFhX@N&_WDxzBIcNLQBXvWV(Fh^lWuQsp@lRbR1*s*gAl zRZo1vCaQWxm7+>frKnO=DXJ7ziYi5wqDoPvs8UoZsuWdQb#QdB9b6jh2U zMU|rJ)kPJyDt)N9{*K|I57lm!uJKlDdDYust)7Lk%5!F|zJ9Y-4?VJ0U-=fBwHmZm zS}U!U)=F!owbEK?t+ZBJE3K8*N^7OH(pqV)v{qUxt(DeFYo)c)T4}AcR$8n7r?u(^ z8aE-Z=-rgUt6eZ%Be1saRd0*J`VsV1o->8@-J8OC;E}?*&mV0HYgA#WuvAzoEESdt zONFJvQemmER9Gr36_yH1g{8t$VX3fGSSlb_;uZD|; zSG#Yz#%Qf2S@*|h{VXmk&zaBq;mv0aM?UKf9<}+bNuQ<9(r4+j^jZ2WeU?5;pQX>z zXX&%_S^6w}mOe|LrO(o5>9h1%`Ye5xK1-ja&-x4cETmSNWDR46X)V~=_0z7mR=M>6 zz$6|*3Z5BCbwqgmU2tErQA|(DYukc$}Q!Va!a|T+){2Sx0GAT zE#;PSOSz@oQf?`?lv~Oz<(6{mmXTW+F5Xmz;i9qDE~0kLRkmvh-|hA8aoulXx$>Nu zt`~2n>uruq*GInLX1W$lm!?b8rRmaiX}UCBnl4S3rc2YM>C$v*x-?yyE=`xFOVg$4 l(sXIMG+ml5O_!$YmNs48FrIK;W0#?>^53>LO literal 0 HcmV?d00001 diff --git a/ci/test/tx_cksum/tx_cksum.conf b/ci/test/tx_cksum/tx_cksum.conf new file mode 100644 index 0000000000..111ddb8f60 --- /dev/null +++ b/ci/test/tx_cksum/tx_cksum.conf @@ -0,0 +1,51 @@ +unix { + log /tmp/tx_cksum/vpp.log + cli-listen /tmp/tx_cksum/cli.sock + #full-coredump + #interactive + nodaemon +} + +api-trace { + on +} + +logging { + default-syslog-log-level info +} + +cpu { + main-core 2 + corelist-workers 3 +} + +session +{ + event-queue-length 10 +} +#tcp { no-csum-offload } + +buffers { + ## Increase number of buffers allocated, needed only in scenarios with + ## large number of interfaces and worker threads. Value is per numa node. + ## Default is 16384 (8192 if running unprivileged) +# buffers-per-numa 5000 + + ## Size of buffer data area + ## Default is 2048 + default data-size 2048 +} + +devices { + dev pci/0002:01:00.2 { + driver octeon + port 0 { + name eth0 + num-rx-queues 4 + } + } +} +plugins { + plugin dpdk_plugin.so { disable } + plugin onp_plugin.so { disable } +} diff --git a/ci/test/tx_cksum/tx_cksum.exec b/ci/test/tx_cksum/tx_cksum.exec new file mode 100644 index 0000000000..233446b3df --- /dev/null +++ b/ci/test/tx_cksum/tx_cksum.exec @@ -0,0 +1,10 @@ +set int mac address eth0 d6:bd:c6:81:0f:b1 + +set int ip address eth0 2.0.0.1/24 + +set ip neighbor eth0 3.0.0.2 0a:5f:b6:10:a4:17 + +ip route add 3.0.0.2/24 via eth0 + +set int state eth0 up + diff --git a/ci/test/tx_cksum/tx_cksum.sh b/ci/test/tx_cksum/tx_cksum.sh new file mode 100644 index 0000000000..836f729988 --- /dev/null +++ b/ci/test/tx_cksum/tx_cksum.sh @@ -0,0 +1,184 @@ +#!/bin/bash + +# Copyright (c) 2025 Marvell. +# SPDX-License-Identifier: Apache-2.0 +# https://spdx.org/licenses/Apache-2.0.html + +#set -e +set -euox pipefail + +PRFX="tx_cksum" +OCTEONTESTPATH="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )/.." + +source $OCTEONTESTPATH/common/vpp/vpp.env +source $OCTEONTESTPATH/common/testpmd/pktgen.env +source $OCTEONTESTPATH/common/pcap/pcap.env + +TX_PCAP="in.pcap" +EXPECTED_PCAP="out.pcap" +TX_MSEG_PCAP="in_mseg.pcap" +EXPECTED_MSEG_PCAP="out_mseg.pcap" +RECV_PCAP="recv.pcap" +PKTGEN_PORT="0002:01:00.1" +PKTGEN_COREMASK="0xf0" +PORT0="0002:01:00.2" +PORT1="0002:01:00.3" +MAINCORE="0x2" +WORKER_COREMASK="0x4" +CONF_FILE="tx_cksum.conf" +INLINE_CONF_FILE="tx_cksum_inline.conf" + +TMP_DIR=/tmp/$PRFX +rm -rf $TMP_DIR +mkdir -p $TMP_DIR + +NO_HP=${NO_HP:-} +HP=${HP:-8} + +function mount_hugetlbfs() { + # Mount hugetlbfs. + if ! mount | grep -q hugepages; then + mount -t hugetlbfs none /dev/hugepages/ + fi +} + +function setup_hp() { + if [[ -n $NO_HP ]]; then + echo "Skipping huge page setup" + return + fi + # Enable HP hugepages. + echo $HP > /proc/sys/vm/nr_hugepages +} + +function sig_handler() +{ + local status=$? + set +e + trap - ERR + trap - INT + trap - QUIT + trap - EXIT + if [[ $status -ne 0 ]]; then + vpp_stats_all tx_cksum + echo $status + echo "$1 Handler" + fi + pktgen_quit + pktgen_cleanup + vpp_cleanup tx_cksum + exit $status +} + + +trap "sig_handler ERR" ERR +trap "sig_handler INT" INT +trap "sig_handler QUIT" QUIT +trap "sig_handler EXIT" EXIT + +mount_hugetlbfs +setup_hp + +run_tx_cksum_test() { + local CONF_PRFX=$1 + CONF_FILE="${CONF_PRFX}.conf" + echo "Starting VPP with Port0=$PORT0, Conf: $CONF_FILE" + cp tx_cksum.exec /tmp/tx_cksum/ + vpp_launch $CONF_PRFX + vpp_start $PRFX + sleep 2 + + echo "Starting pktgen with Port=$PKTGEN_PORT, Coremask=$PKTGEN_COREMASK, In-pcap=$TX_PCAP, received-pcap=$RECV_PCAP" + pktgen_launch -c $PKTGEN_COREMASK -p $PKTGEN_PORT -i $TX_PCAP -o $RECV_PCAP + sleep 2 + testpmd_cmd "pktgen" "port stop all" + testpmd_cmd "pktgen" "port config mtu 0 9000" + testpmd_cmd "pktgen" "port start all" + sleep 2 + echo "pktgen start" + pktgen_start + sleep 5 + + vpp_port_down $PRFX eth0 + + vpp_stats_all $PRFX > /dev/null + pktgen_stats > /dev/null + + echo "-------------------- TX_CKSUM VPP LOGS ---------------------" + vpp_log $PRFX + echo "-------------------- TX_CKSUM PKTGEN LOGS --------------------" + pktgen_log + + echo "Verifying tx_cksum test" + + VPP_RX_COUNT=$(vpp_rx_count $PRFX eth0) + VPP_TX_COUNT=$(vpp_tx_count $PRFX eth0) + VPP_RX_BYTES=$(vpp_rx_bytes $PRFX eth0) + VPP_TX_BYTES=$(vpp_tx_bytes $PRFX eth0) + + if [[ $VPP_RX_COUNT -ne $PCAP_CNT ]] || + [[ $VPP_TX_COUNT -ne $PCAP_CNT ]] || + [[ $VPP_RX_BYTES -ne $PCAP_LEN ]] || + [[ $VPP_TX_BYTES -ne $PCAP_LEN ]]; then + echo "FAILURE: Error in tx_cksum" + exit 1 + fi + + tcpdump -nr $EXPECTED_PCAP -xvve -t >$TMP_DIR/expect.txt + tcpdump -nr $RECV_PCAP -xvve -t >$TMP_DIR/recv.txt + + # Compare received and expected + diff -sqad $TMP_DIR/recv.txt $TMP_DIR/expect.txt + + pktgen_quit + echo "########## SUCCESS: tx_cksum test completed ##########" + echo " Used TX_PCAP: $TX_PCAP" + echo " Used EXPECTED_PCAP: $EXPECTED_PCAP" + echo " Used CONF_FILE: $CONF_FILE" + echo "######################################################" +} + +run_all_tests() { + local TX_PCAP=$1 + local EXPECTED_PCAP=$2 + + PCAP_CNT=$(pcap_packet_count $TX_PCAP) + PCAP_LEN=$(pcap_length $TX_PCAP) + + export TX_PCAP + export EXPECTED_PCAP + export PCAP_CNT + export PCAP_LEN + + # TEST-1: Run without inline device in startup.conf + if [[ -f "$CONF_FILE" ]]; then + run_tx_cksum_test "tx_cksum" + sleep 1 + vpp_cleanup tx_cksum + sleep 1 + pktgen_cleanup + else + echo "Startup config file $CONF_FILE not found!" + fi + + sleep 10 + + # TEST-2: Run with inline device in startup.conf + if [[ -f "$INLINE_CONF_FILE" ]]; then + run_tx_cksum_test "tx_cksum_inline" + sleep 1 + vpp_cleanup tx_cksum_inline + sleep 1 + pktgen_cleanup + else + echo "Inline config file $INLINE_CONF_FILE not found!" + fi +} + +#Run with single-seg pcap +run_all_tests "$TX_PCAP" "$EXPECTED_PCAP" + +sleep 10 + +#Run with multi-seg pcap +run_all_tests "$TX_MSEG_PCAP" "$EXPECTED_MSEG_PCAP" diff --git a/ci/test/tx_cksum/tx_cksum_inline.conf b/ci/test/tx_cksum/tx_cksum_inline.conf new file mode 100644 index 0000000000..0ce6c21af7 --- /dev/null +++ b/ci/test/tx_cksum/tx_cksum_inline.conf @@ -0,0 +1,57 @@ +unix { + log /tmp/tx_cksum/vpp.log + cli-listen /tmp/tx_cksum/cli.sock + #full-coredump + #interactive + nodaemon +} + +api-trace { + on +} + +logging { + default-syslog-log-level info +} + +cpu { + main-core 2 + corelist-workers 3 +} + +session +{ + event-queue-length 10 +} +#tcp { no-csum-offload } + +buffers { + ## Increase number of buffers allocated, needed only in scenarios with + ## large number of interfaces and worker threads. Value is per numa node. + ## Default is 16384 (8192 if running unprivileged) +# buffers-per-numa 5000 + + ## Size of buffer data area + ## Default is 2048 + default data-size 2048 +} + +devices { + dev pci/0002:20:00.1 { + driver octeon + } + dev pci/0002:1d:00.0 { + driver octeon + } + dev pci/0002:01:00.2 { + driver octeon + port 0 { + name eth0 + num-rx-queues 4 + } + } +} +plugins { + plugin dpdk_plugin.so { disable } + plugin onp_plugin.so { disable } +} From 4ad96d0d4dcf14f767b545a2e502526a3e64b0ec Mon Sep 17 00:00:00 2001 From: Alok Mishra Date: Fri, 25 Jul 2025 06:37:47 +0000 Subject: [PATCH 253/271] ci: add multi-seg test with inline ipsec This patch adds multi-seg test cases in Inline IPsec functional test script. Type: feature JIRA: https://essjira.marvell.com/browse/IPBUSW-72039 Signed-off-by: Alok Mishra Change-Id: Id31573270b166bb46772603d7c580038dc75166a Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/158635 Tested-by: sa_ip-toolkits-Jenkins Reviewed-by: Nithin Kumar Dabilpuram (cherry picked from commit ae812b25f43bfebb25f08195e514620ee97bf6e8) Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/159256 Tested-by: sa_ip-sw-jenkins Reviewed-by: Monendra Singh Kushwaha --- ci/test/inl_ipsec/inl_ipsec.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ci/test/inl_ipsec/inl_ipsec.sh b/ci/test/inl_ipsec/inl_ipsec.sh index 4632eadae1..16984d524f 100755 --- a/ci/test/inl_ipsec/inl_ipsec.sh +++ b/ci/test/inl_ipsec/inl_ipsec.sh @@ -166,6 +166,7 @@ function configure_vm0() { ip netns exec vm0 ip addr add 192.168.$X.2/24 dev $LBK1 ip netns exec vm0 ip addr add 1.1.$Y.1/24 dev $LBK1:1 + ip netns exec vm0 ip link set $LBK1 mtu 9000 ip netns exec vm0 ip link set $LBK1 up ip netns exec vm0 ip link set $LBK1 address $VM0_MAC ip netns exec vm0 ip route add 192.168.$Y.0/24 via 192.168.$X.1 @@ -181,6 +182,7 @@ function configure_vm0() function configure_vm2() { ip netns exec vm2 ip addr add 192.168.$Y.2/24 dev $LBK3 + ip netns exec vm2 ip link set $LBK3 mtu 9000 ip netns exec vm2 ip link set $LBK3 up ip netns exec vm2 ip link set lo up ip netns exec vm2 ip link set $LBK3 address $VM2_MAC @@ -275,7 +277,7 @@ CASE=( PING_RETRY=1 PING_PKTS=320 -PKT_LIST="64 380 1410" +PKT_LIST="64 380 1410 4000 8000" PKT_GAP="0.001" PING_ARGS="" From 9a6882c971d2455e960ba7345b5d3d0f43ac9afb Mon Sep 17 00:00:00 2001 From: Monendra Singh Kushwaha Date: Mon, 4 Aug 2025 09:15:29 +0000 Subject: [PATCH 254/271] build: update roc version Type: feature Change-Id: I48fb2039a066a686105385f5e3e83db6472f0e2f Signed-off-by: Monendra Singh Kushwaha Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/159344 Tested-by: sa_ip-sw-jenkins --- build/external/packages/octeon-roc.mk | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build/external/packages/octeon-roc.mk b/build/external/packages/octeon-roc.mk index 4cbfea9a61..8054e22c7e 100644 --- a/build/external/packages/octeon-roc.mk +++ b/build/external/packages/octeon-roc.mk @@ -2,9 +2,9 @@ # SPDX-License-Identifier: Apache-2.0 # https://spdx.org/licenses/Apache-2.0.html -octeon-roc_version := 25.07 +octeon-roc_version := 25.08 octeon-roc_tarball := v$(octeon-roc_version).tar.gz -octeon-roc_tarball_md5sum := a83cb082df2beb6dc975b23aac58d3f9 +octeon-roc_tarball_md5sum := c6dd2e885c332e32f64ddc819d05bd90 octeon-roc_github := https://github.com/MarvellEmbeddedProcessors/marvell-octeon-roc octeon-roc_tarball_strip_dirs := 1 From 7186f64d77c1744f14d9d246b63d92d3fb926ab7 Mon Sep 17 00:00:00 2001 From: Nithinsen Kaithakadan Date: Wed, 25 Jun 2025 08:06:58 +0000 Subject: [PATCH 255/271] octeon: align cptr to 256B in cn20k Change the alignment of cptr to 256 bytes from 128 bytes in cn20k. Type: feature Signed-off-by: Nithinsen Kaithakadan Change-Id: I97df27dbae0e11306d35d24df09e6b118aa76ad4 Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/156424 Tested-by: sa_ip-toolkits-Jenkins Reviewed-by: Tejasree Kondoj Reviewed-by: Anoob Joseph (cherry picked from commit 871e8db13ff39e6664b0ca0ecd7c01e96c2303e7) Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/159211 Reviewed-by: Monendra Singh Kushwaha (cherry picked from commit 4fbc36675a26549d8ff83fd4e664996f89cce295) Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/159257 Tested-by: sa_ip-sw-jenkins --- src/plugins/dev_octeon/crypto.c | 2 +- src/plugins/dev_octeon/crypto.h | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/plugins/dev_octeon/crypto.c b/src/plugins/dev_octeon/crypto.c index 7c8f70e6bf..6bc36bea0b 100644 --- a/src/plugins/dev_octeon/crypto.c +++ b/src/plugins/dev_octeon/crypto.c @@ -44,7 +44,7 @@ oct_crypto_session_alloc (vlib_main_t *vm, u8 type) size = sizeof (oct_crypto_sess_t); - addr = oct_plt_init_param.oct_plt_zmalloc (size, CLIB_CACHE_LINE_BYTES); + addr = oct_plt_init_param.oct_plt_zmalloc (size, ROC_CPTR_CACHE_LINE_SZ); if (addr == NULL) { log_err (ocd->dev, "Failed to allocate crypto session memory"); diff --git a/src/plugins/dev_octeon/crypto.h b/src/plugins/dev_octeon/crypto.h index 4b5e58b41a..b529142ad6 100644 --- a/src/plugins/dev_octeon/crypto.h +++ b/src/plugins/dev_octeon/crypto.h @@ -98,6 +98,8 @@ typedef struct typedef struct { CLIB_CACHE_LINE_ALIGN_MARK (cacheline0); + /** ROC CPT context */ + struct roc_se_ctx cpt_ctx; /** CPT opcode */ u16 cpt_op : 4; /** Flag for AES GCM */ @@ -117,7 +119,6 @@ typedef struct /* store link key index in case of linked algo */ vnet_crypto_key_index_t key_index; oct_crypto_dev_t *crypto_dev; - struct roc_se_ctx cpt_ctx; } oct_crypto_sess_t; typedef struct From b9a69c6520ed505727cd4b489e114ad47c5c27f9 Mon Sep 17 00:00:00 2001 From: Nagendra Puthane Date: Thu, 4 Sep 2025 11:56:26 +0530 Subject: [PATCH 256/271] accept devel oct-ep package --- .github/workflows/build-cn10k.yml | 8 ++++---- DEP_PKG_VERSION | 1 - 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build-cn10k.yml b/.github/workflows/build-cn10k.yml index 5a61f587f8..16695057f8 100644 --- a/.github/workflows/build-cn10k.yml +++ b/.github/workflows/build-cn10k.yml @@ -44,7 +44,6 @@ jobs: echo "PKG_VERSION_NAME=`./src/scripts/version | awk -F '-' '{print $1}'`" >> "${PWD}/artifacts/env" echo "MRVL_PKG_VERSION=`cat MRVL_VERSION`" >> "${PWD}/artifacts/env" echo "CPT_PKG_VERSION=`cat DEP_PKG_VERSION | grep CPT_PKG_VERSION | awk -F'=' '{print $2}'`" >> "${PWD}/artifacts/env" - echo "OCTEP_PKG_VERSION=`cat DEP_PKG_VERSION | grep OCTEP_PKG_VERSION | awk -F'=' '{print $2}'`" >> "${PWD}/artifacts/env" echo "DPDK_PKG_VERSION=`cat DPDK_VERSION | grep RELEASE_VERSION | awk -F'=' '{print $2}'`" >> "${PWD}/artifacts/env" echo "DPDK_BASE_PKG_VERSION=`cat DPDK_VERSION | grep BASE_VERSION | awk -F'=' '{print $2}' | awk -F'.' '{print $1"."$2}'`" >> "${PWD}/artifacts/env" echo "PKG_POSTFIX=${PKG_POSTFIX}" >> "${PWD}/artifacts/env" @@ -121,8 +120,9 @@ jobs: ccache -p git config --global --add safe.directory "${PWD}" sudo APT_ARGS='-y -q' make install-deps - wget "https://github.com/MarvellEmbeddedProcessors/pcie_ep_octeon_target/releases/download/oct-ep-target-cn10k-${OCTEP_PKG_VERSION}-${DISTRO}-${OCTEP_PKG_VERSION}/oct-ep-target-cn10k_${OCTEP_PKG_VERSION}_arm64.deb" - sudo apt-get install -y ./"oct-ep-target-cn10k_${OCTEP_PKG_VERSION}_arm64.deb" + [[ "$PKG_POSTFIX" == "-devel" ]] && TAG=devel || TAG=${MRVL_PKG_VERSION} + wget "https://github.com/MarvellEmbeddedProcessors/dao/releases/download/oct-ep-target-cn10k-${MRVL_PKG_VERSION}-${DISTRO}-${TAG}/oct-ep-target-cn10k${PKG_POSTFIX}_${MRVL_PKG_VERSION}_arm64.deb" + sudo apt-get install -y ./"oct-ep-target-cn10k${PKG_POSTFIX}_${MRVL_PKG_VERSION}_arm64.deb" make build-release VPP_PLATFORM=octeon10 mkdir -p "${PWD}/install/DEBIAN" mkdir -p "${PWD}/install/usr/share/vpp/api" @@ -132,7 +132,7 @@ jobs: echo 'Package: vpp-'$PKG_VERSION_NAME'-cn10k'$PKG_POSTFIX >> DEBIAN/control echo 'Version: '$MRVL_PKG_VERSION >> DEBIAN/control echo "Maintainer: Jerin Jacob (jerinj@marvell.com)" >> DEBIAN/control - echo 'Depends: python3, python3-ply, dpdk-'$DPDK_BASE_PKG_VERSION'-cn10k (= '$DPDK_PKG_VERSION'), cpt-firmware-cn10k (= '$CPT_PKG_VERSION'), oct-ep-target-cn10k (>= '$OCTEP_PKG_VERSION')' >> DEBIAN/control + echo 'Depends: python3, python3-ply, dpdk-'$DPDK_BASE_PKG_VERSION'-cn10k (= '$DPDK_PKG_VERSION'), cpt-firmware-cn10k (= '$CPT_PKG_VERSION'), oct-ep-target-cn10k'$PKG_POSTFIX' (>= '$MRVL_PKG_VERSION')' >> DEBIAN/control echo "Architecture: arm64" >> DEBIAN/control echo "Homepage: https://wiki.fd.io/view/VPP" >> DEBIAN/control echo "Description: Vector Packet Processing (VPP) for Octeon10" >> DEBIAN/control diff --git a/DEP_PKG_VERSION b/DEP_PKG_VERSION index f450043529..46e4a66aed 100644 --- a/DEP_PKG_VERSION +++ b/DEP_PKG_VERSION @@ -1,2 +1 @@ CPT_PKG_VERSION=24.09.0 -OCTEP_PKG_VERSION=25.05.0 \ No newline at end of file From 58ead80b55b3904e8b869ea3b0ead6fa6c2eb0e9 Mon Sep 17 00:00:00 2001 From: Monendra Singh Kushwaha Date: Wed, 23 Jul 2025 10:00:05 +0000 Subject: [PATCH 257/271] ci: add multicore support for inline IPsec Type: feature JIRA: https://essjira.marvell.com/browse/IPBUSW-72233 Signed-off-by: Monendra Singh Kushwaha Change-Id: I77bcacd9a9b22c9b3d45b2c7011f505571807bbc Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/158454 Reviewed-by: Nithin Kumar Dabilpuram Tested-by: Nithin Kumar Dabilpuram (cherry picked from commit ba96308ccdd2ff8eb66eecf5c43eb91377879d48) Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/159382 Tested-by: sa_ip-sw-jenkins --- ci/test/common/vpp/vpp.env | 4 +- ci/test/env/cn10k-perf.env | 2 +- ci/test/inl_ipsec_perf/aes_cbc_sha1_hmac.cfg | 25 +- .../inl_ipsec_perf/aes_cbc_sha1_hmac_ib.cfg | 20 +- ci/test/inl_ipsec_perf/aes_gcm.cfg | 25 +- ci/test/inl_ipsec_perf/aes_gcm_ib.cfg | 18 +- ci/test/inl_ipsec_perf/inl_ipsec_gen.sh | 2 +- ci/test/inl_ipsec_perf/inl_ipsec_perf.exec | 21 -- ci/test/inl_ipsec_perf/inl_ipsec_perf.sh | 230 ++++++++++-------- ci/test/inl_ipsec_perf/inl_ipsec_perf_1.conf | 55 +++++ ci/test/inl_ipsec_perf/inl_ipsec_perf_2.conf | 55 +++++ ci/test/inl_ipsec_perf/inl_ipsec_perf_4.conf | 55 +++++ .../pcap/enc_aes-cbc_sha1-hmac_1410.pcap | Bin 1518 -> 0 bytes .../pcap/enc_aes-cbc_sha1-hmac_1410_1.pcap | Bin 0 -> 1518 bytes .../pcap/enc_aes-cbc_sha1-hmac_1410_2.pcap | Bin 0 -> 3012 bytes .../pcap/enc_aes-cbc_sha1-hmac_1410_4.pcap | Bin 0 -> 6000 bytes .../pcap/enc_aes-cbc_sha1-hmac_380.pcap | Bin 478 -> 0 bytes .../pcap/enc_aes-cbc_sha1-hmac_380_1.pcap | Bin 0 -> 478 bytes .../pcap/enc_aes-cbc_sha1-hmac_380_2.pcap | Bin 0 -> 932 bytes .../pcap/enc_aes-cbc_sha1-hmac_380_4.pcap | Bin 0 -> 1840 bytes .../pcap/enc_aes-cbc_sha1-hmac_64.pcap | Bin 174 -> 0 bytes .../pcap/enc_aes-cbc_sha1-hmac_64_1.pcap | Bin 0 -> 174 bytes .../pcap/enc_aes-cbc_sha1-hmac_64_2.pcap | Bin 0 -> 324 bytes .../pcap/enc_aes-cbc_sha1-hmac_64_4.pcap | Bin 0 -> 624 bytes ...-gcm_1410.pcap => enc_aes-gcm_1410_1.pcap} | Bin 1506 -> 1506 bytes .../pcap/enc_aes-gcm_1410_2.pcap | Bin 0 -> 2988 bytes .../pcap/enc_aes-gcm_1410_4.pcap | Bin 0 -> 5952 bytes ...es-gcm_380.pcap => enc_aes-gcm_380_1.pcap} | Bin 474 -> 474 bytes .../pcap/enc_aes-gcm_380_2.pcap | Bin 0 -> 924 bytes .../pcap/enc_aes-gcm_380_4.pcap | Bin 0 -> 1824 bytes .../inl_ipsec_perf/pcap/enc_aes-gcm_64.pcap | Bin 158 -> 0 bytes .../inl_ipsec_perf/pcap/enc_aes-gcm_64_1.pcap | Bin 0 -> 158 bytes .../inl_ipsec_perf/pcap/enc_aes-gcm_64_2.pcap | Bin 0 -> 292 bytes .../inl_ipsec_perf/pcap/enc_aes-gcm_64_4.pcap | Bin 0 -> 560 bytes .../cn10k/rclk2500_sclk1000.103xx.ip.inb | 10 - .../cn10k/rclk2500_sclk1000.103xx.ip.outb | 10 - .../cn10k/rclk2500_sclk1000.106xx.ip.inb | 12 +- .../cn10k/rclk2500_sclk1000.106xx.ip.outb | 12 +- .../cn10k/rclk2500_sclk1100.106xx.ip.inb | 12 +- .../cn10k/rclk2500_sclk1100.106xx.ip.outb | 12 +- 40 files changed, 369 insertions(+), 211 deletions(-) delete mode 100644 ci/test/inl_ipsec_perf/inl_ipsec_perf.exec create mode 100644 ci/test/inl_ipsec_perf/inl_ipsec_perf_1.conf create mode 100644 ci/test/inl_ipsec_perf/inl_ipsec_perf_2.conf create mode 100644 ci/test/inl_ipsec_perf/inl_ipsec_perf_4.conf delete mode 100644 ci/test/inl_ipsec_perf/pcap/enc_aes-cbc_sha1-hmac_1410.pcap create mode 100644 ci/test/inl_ipsec_perf/pcap/enc_aes-cbc_sha1-hmac_1410_1.pcap create mode 100644 ci/test/inl_ipsec_perf/pcap/enc_aes-cbc_sha1-hmac_1410_2.pcap create mode 100644 ci/test/inl_ipsec_perf/pcap/enc_aes-cbc_sha1-hmac_1410_4.pcap delete mode 100644 ci/test/inl_ipsec_perf/pcap/enc_aes-cbc_sha1-hmac_380.pcap create mode 100644 ci/test/inl_ipsec_perf/pcap/enc_aes-cbc_sha1-hmac_380_1.pcap create mode 100644 ci/test/inl_ipsec_perf/pcap/enc_aes-cbc_sha1-hmac_380_2.pcap create mode 100644 ci/test/inl_ipsec_perf/pcap/enc_aes-cbc_sha1-hmac_380_4.pcap delete mode 100644 ci/test/inl_ipsec_perf/pcap/enc_aes-cbc_sha1-hmac_64.pcap create mode 100644 ci/test/inl_ipsec_perf/pcap/enc_aes-cbc_sha1-hmac_64_1.pcap create mode 100644 ci/test/inl_ipsec_perf/pcap/enc_aes-cbc_sha1-hmac_64_2.pcap create mode 100644 ci/test/inl_ipsec_perf/pcap/enc_aes-cbc_sha1-hmac_64_4.pcap rename ci/test/inl_ipsec_perf/pcap/{enc_aes-gcm_1410.pcap => enc_aes-gcm_1410_1.pcap} (84%) create mode 100644 ci/test/inl_ipsec_perf/pcap/enc_aes-gcm_1410_2.pcap create mode 100644 ci/test/inl_ipsec_perf/pcap/enc_aes-gcm_1410_4.pcap rename ci/test/inl_ipsec_perf/pcap/{enc_aes-gcm_380.pcap => enc_aes-gcm_380_1.pcap} (63%) create mode 100644 ci/test/inl_ipsec_perf/pcap/enc_aes-gcm_380_2.pcap create mode 100644 ci/test/inl_ipsec_perf/pcap/enc_aes-gcm_380_4.pcap delete mode 100644 ci/test/inl_ipsec_perf/pcap/enc_aes-gcm_64.pcap create mode 100644 ci/test/inl_ipsec_perf/pcap/enc_aes-gcm_64_1.pcap create mode 100644 ci/test/inl_ipsec_perf/pcap/enc_aes-gcm_64_2.pcap create mode 100644 ci/test/inl_ipsec_perf/pcap/enc_aes-gcm_64_4.pcap delete mode 100644 ci/test/inl_ipsec_perf/ref_numbers/cn10k/rclk2500_sclk1000.103xx.ip.inb delete mode 100644 ci/test/inl_ipsec_perf/ref_numbers/cn10k/rclk2500_sclk1000.103xx.ip.outb diff --git a/ci/test/common/vpp/vpp.env b/ci/test/common/vpp/vpp.env index 70ad1cb2db..66cbd324bb 100644 --- a/ci/test/common/vpp/vpp.env +++ b/ci/test/common/vpp/vpp.env @@ -185,6 +185,8 @@ function vpp_log() $VPPCTL -s /tmp/$test/cli.sock show run >> /tmp/$test/vpp.log $VPPCTL -s /tmp/$test/cli.sock show error >> /tmp/$test/vpp.log $VPPCTL -s /tmp/$test/cli.sock show int >> /tmp/$test/vpp.log - cat /tmp/$test/vpp.log + $VPPCTL -s /tmp/$test/cli.sock show ipsec sa detail >> /tmp/$test/vpp.log + $VPPCTL -s /tmp/$test/cli.sock show ip fib >> /tmp/$test/vpp.log + awk ' { print FILENAME": " $0 } ' /tmp/$test/vpp.log set -e } diff --git a/ci/test/env/cn10k-perf.env b/ci/test/env/cn10k-perf.env index e5b34b779f..9a76eb2a0d 100644 --- a/ci/test/env/cn10k-perf.env +++ b/ci/test/env/cn10k-perf.env @@ -12,7 +12,7 @@ RUN_TESTS=" " # Update command timeout -CMD_TIMEOUTS="l3fwd_perf=10m $CMD_TIMEOUTS" +CMD_TIMEOUTS="l3fwd_perf=10m inl_ipsec_perf=60m $CMD_TIMEOUTS" # Perf stage flag PERF_STAGE=1 diff --git a/ci/test/inl_ipsec_perf/aes_cbc_sha1_hmac.cfg b/ci/test/inl_ipsec_perf/aes_cbc_sha1_hmac.cfg index ed6579594b..5f92996661 100644 --- a/ci/test/inl_ipsec_perf/aes_cbc_sha1_hmac.cfg +++ b/ci/test/inl_ipsec_perf/aes_cbc_sha1_hmac.cfg @@ -1,3 +1,4 @@ +set int mac address eth0 00:01:02:03:04:01 set int ip address eth0 12.168.101.1/24 set int state eth0 up @@ -6,15 +7,15 @@ set int state eth1 up set ipsec async mode on -ipsec sa add 1 spi 1 esp crypto-key a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0 crypto-alg aes-cbc-128 integ-alg sha1-96 integ-key a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0 tunnel src 1.1.1.1 dst 1.1.1.2 inbound -ipsec sa add 11 spi 101 esp crypto-key a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0 crypto-alg aes-cbc-128 integ-alg sha1-96 integ-key a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0 tunnel src 1.1.1.2 dst 1.1.1.1 - -ipsec itf create -ipsec tunnel protect sa-out 11 sa-in 1 ipsec0 -set int state ipsec0 up - -ip route add 192.168.1.2/24 via ipsec0 -set ip neighbor eth0 1.1.1.1 00:16:3e:7e:94:9a - -ip route add 1.1.1.0/24 via eth0 - +#ipsec sa add 1 spi 1 esp crypto-key a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0 crypto-alg aes-cbc-128 integ-alg sha1-96 integ-key a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0 tunnel src 1.1.1.1 dst 1.1.1.2 inbound +#ipsec sa add 11 spi 101 esp crypto-key a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0 crypto-alg aes-cbc-128 integ-alg sha1-96 integ-key a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0 tunnel src 1.1.1.2 dst 1.1.1.1 +# +#ipsec itf create +#ipsec tunnel protect sa-out 11 sa-in 1 ipsec0 +#set int state ipsec0 up +# +#ip route add 192.168.1.2/24 via ipsec0 +#set ip neighbor eth0 1.1.1.1 00:16:3e:7e:94:9a +# +#ip route add 1.1.1.0/24 via eth0 +# diff --git a/ci/test/inl_ipsec_perf/aes_cbc_sha1_hmac_ib.cfg b/ci/test/inl_ipsec_perf/aes_cbc_sha1_hmac_ib.cfg index 99842a6a27..7ad9d59606 100644 --- a/ci/test/inl_ipsec_perf/aes_cbc_sha1_hmac_ib.cfg +++ b/ci/test/inl_ipsec_perf/aes_cbc_sha1_hmac_ib.cfg @@ -7,13 +7,13 @@ set int state eth1 up set ipsec async mode on -ipsec sa add 1 spi 1 esp crypto-key a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0 crypto-alg aes-cbc-128 integ-alg sha1-96 integ-key a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0 tunnel src 1.1.1.1 dst 1.1.1.2 inbound -ipsec sa add 11 spi 101 esp crypto-key a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0 crypto-alg aes-cbc-128 integ-alg sha1-96 integ-key a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0 tunnel src 1.1.1.2 dst 1.1.1.1 - -ipsec itf create -ipsec tunnel protect sa-out 11 sa-in 1 ipsec0 -set int state ipsec0 up - -ip route add 192.168.1.2/24 via eth0 -set ip neighbor eth0 192.168.1.2 00:16:3e:7e:94:9a - +#ipsec sa add 1 spi 1 esp crypto-key a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0 crypto-alg aes-cbc-128 integ-alg sha1-96 integ-key a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0 tunnel src 1.1.1.1 dst 1.1.1.2 inbound +#ipsec sa add 11 spi 101 esp crypto-key a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0 crypto-alg aes-cbc-128 integ-alg sha1-96 integ-key a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0 tunnel src 1.1.1.2 dst 1.1.1.1 +# +#ipsec itf create +#ipsec tunnel protect sa-out 11 sa-in 1 ipsec0 +#set int state ipsec0 up +# +#ip route add 192.168.1.2/24 via eth0 +#set ip neighbor eth0 192.168.1.2 00:16:3e:7e:94:9a +# diff --git a/ci/test/inl_ipsec_perf/aes_gcm.cfg b/ci/test/inl_ipsec_perf/aes_gcm.cfg index 02137abd55..d089248e3b 100644 --- a/ci/test/inl_ipsec_perf/aes_gcm.cfg +++ b/ci/test/inl_ipsec_perf/aes_gcm.cfg @@ -1,3 +1,4 @@ +set int mac address eth0 00:01:02:03:04:01 set int ip address eth0 12.168.101.1/24 set int state eth0 up @@ -6,15 +7,15 @@ set int state eth1 up set ipsec async mode on -ipsec sa add 1 spi 1 esp crypto-key a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0 crypto-alg aes-gcm-128 tunnel src 1.1.1.1 dst 1.1.1.2 inbound -ipsec sa add 11 spi 101 esp crypto-key a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0 crypto-alg aes-gcm-128 tunnel src 1.1.1.2 dst 1.1.1.1 - -ipsec itf create -ipsec tunnel protect sa-out 11 sa-in 1 ipsec0 -set int state ipsec0 up - -ip route add 192.168.2.2/24 via ipsec0 -set ip neighbor eth0 1.1.1.1 00:16:3e:7e:94:9a - -ip route add 1.1.1.0/24 via eth0 - +#ipsec sa add 1 spi 1 esp crypto-key a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0 crypto-alg aes-gcm-128 tunnel src 1.1.1.1 dst 1.1.1.2 inbound +#ipsec sa add 11 spi 101 esp crypto-key a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0 crypto-alg aes-gcm-128 tunnel src 1.1.1.2 dst 1.1.1.1 +# +#ipsec itf create +#ipsec tunnel protect sa-out 11 sa-in 1 ipsec0 +#set int state ipsec0 up +# +#ip route add 192.168.2.2/24 via ipsec0 +#set ip neighbor eth0 1.1.1.1 00:16:3e:7e:94:9a +# +#ip route add 1.1.1.0/24 via eth0 +# diff --git a/ci/test/inl_ipsec_perf/aes_gcm_ib.cfg b/ci/test/inl_ipsec_perf/aes_gcm_ib.cfg index 97e15dd626..8582cd473d 100644 --- a/ci/test/inl_ipsec_perf/aes_gcm_ib.cfg +++ b/ci/test/inl_ipsec_perf/aes_gcm_ib.cfg @@ -7,12 +7,12 @@ set int state eth1 up set ipsec async mode on -ipsec sa add 2 spi 2 esp crypto-key a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0 crypto-alg aes-gcm-128 tunnel src 1.1.2.1 dst 1.1.2.2 inbound -ipsec sa add 11 spi 101 esp crypto-key a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0 crypto-alg aes-gcm-128 tunnel src 1.1.2.2 dst 1.1.2.1 - -ipsec itf create -ipsec tunnel protect sa-out 11 sa-in 2 ipsec0 -set int state ipsec0 up - -ip route add 192.168.2.2/24 via eth0 -set ip neighbor eth0 192.168.2.2 00:16:3e:7e:94:9a +#ipsec sa add 2 spi 2 esp crypto-key a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0 crypto-alg aes-gcm-128 tunnel src 1.1.2.1 dst 1.1.2.2 inbound +#ipsec sa add 11 spi 101 esp crypto-key a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0 crypto-alg aes-gcm-128 tunnel src 1.1.2.2 dst 1.1.2.1 +# +#ipsec itf create +#ipsec tunnel protect sa-out 11 sa-in 2 ipsec0 +#set int state ipsec0 up +# +#ip route add 192.168.2.2/24 via eth0 +#set ip neighbor eth0 192.168.2.2 00:16:3e:7e:94:9a diff --git a/ci/test/inl_ipsec_perf/inl_ipsec_gen.sh b/ci/test/inl_ipsec_perf/inl_ipsec_gen.sh index ce22d2eb5c..7992f770d3 100755 --- a/ci/test/inl_ipsec_perf/inl_ipsec_gen.sh +++ b/ci/test/inl_ipsec_perf/inl_ipsec_gen.sh @@ -50,7 +50,7 @@ launch_testpmd_tx_outb() { testpmd_launch $1 \ "-c 0xF800 -a $PORT0,disable_xqe_drop=1" \ - "--nb-cores=3 --forward-mode=txonly --tx-ip=192.168.$2.1,192.168.$2.2 --txq=3 --rxq=3 --eth-peer=0,00:01:02:03:04:01" \ + "--nb-cores=3 --forward-mode=flowgen --flowgen-flows=$2 --txq=3 --rxq=3 --eth-peer=0,00:01:02:03:04:01" \ /dev/null & sleep 1 testpmd_cmd $1 "port stop 0" diff --git a/ci/test/inl_ipsec_perf/inl_ipsec_perf.exec b/ci/test/inl_ipsec_perf/inl_ipsec_perf.exec deleted file mode 100644 index 3955b7315e..0000000000 --- a/ci/test/inl_ipsec_perf/inl_ipsec_perf.exec +++ /dev/null @@ -1,21 +0,0 @@ -set int ip address eth0 12.168.101.1/24 -set int mac address eth0 00:01:02:03:04:01 -set int state eth0 up - -set int ip address eth1 13.168.1.1/24 -set int state eth1 up - -set ipsec async mode on - -ipsec sa add 1 spi 1 esp crypto-key a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0 crypto-alg aes-cbc-128 integ-alg sha1-96 integ-key a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0 tunnel src 1.1.1.1 dst 1.1.1.2 inbound -ipsec sa add 11 spi 101 esp crypto-key a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0 crypto-alg aes-cbc-128 integ-alg sha1-96 integ-key a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0 tunnel src 1.1.1.2 dst 1.1.1.1 - -ipsec itf create -ipsec tunnel protect sa-out 11 sa-in 1 ipsec0 -set int state ipsec0 up - -ip route add 192.168.1.2/24 via ipsec0 -set ip neighbor eth0 1.1.1.1 00:16:3e:7e:94:9a - -ip route add 1.1.1.0/24 via eth0 - diff --git a/ci/test/inl_ipsec_perf/inl_ipsec_perf.sh b/ci/test/inl_ipsec_perf/inl_ipsec_perf.sh index 7fe63f3e3a..510f748091 100755 --- a/ci/test/inl_ipsec_perf/inl_ipsec_perf.sh +++ b/ci/test/inl_ipsec_perf/inl_ipsec_perf.sh @@ -12,7 +12,7 @@ OCTEONTESTPATH="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )/.." PKT_LIST="64 380 1410" NUM_CAPTURE=3 MAX_TRY_CNT=5 -CORES=(1) +CORES=(1 2 4) COREMASK="0x10000" TXWAIT=15 RXWAIT=5 @@ -208,41 +208,87 @@ function supported_by_9k() function run_vpp_ipsec() { - echo "vpp outb" + local i + n_CORES=$1 + echo "vpp outb num_cores:$n_CORES" IS_RXPPS_TXTPMD=1 rm -rf /tmp/inl_ipsec_perf mkdir -p /tmp/inl_ipsec_perf - cp inl_ipsec_perf.conf /tmp/inl_ipsec_perf/ + cp inl_ipsec_perf_$n_CORES.conf /tmp/inl_ipsec_perf/ cp ${CFG[$Y]} /tmp/inl_ipsec_perf/ # Inline Protocol - vpp_launch inl_ipsec_perf + vpp_launch inl_ipsec_perf_$n_CORES vpp_exec_cmd inl_ipsec_perf "device attach pci/$IF0 driver octeon" vpp_exec_cmd inl_ipsec_perf "device create-interface pci/$IF0 port 0 name eth0 num-rx-queues 8 tx-queues-size 16384" vpp_exec_cmd inl_ipsec_perf "device attach pci/$IF1 driver octeon" vpp_exec_cmd inl_ipsec_perf "device create-interface pci/$IF1 port 0 name eth1 num-rx-queues 8 tx-queues-size 16384" vpp_exec_file inl_ipsec_perf /tmp/inl_ipsec_perf/${CFG[$Y]} - vpp_add_trace inl_ipsec_perf eth0 + for (( i=0; i<$n_CORES; i++ )) + do + if [[ $Y == 0 ]]; then + vpp_exec_cmd inl_ipsec_perf "ipsec sa add $((i+1)) spi $((i+1)) esp crypto-key a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0 crypto-alg aes-cbc-128 integ-alg sha1-96 integ-key a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0 tunnel src $((i+1)).1.1.1 dst $((i+1)).1.1.2 inbound" + vpp_exec_cmd inl_ipsec_perf "ipsec sa add $((i+11)) spi $((i+101)) esp crypto-key a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0 crypto-alg aes-cbc-128 integ-alg sha1-96 integ-key a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0 tunnel src $((i+1)).1.1.2 dst $((i+1)).1.1.1" + else + vpp_exec_cmd inl_ipsec_perf "ipsec sa add $((i+1)) spi $((i+1)) esp crypto-key a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0 crypto-alg aes-gcm-128 tunnel src $((i+1)).1.1.1 dst $((i+1)).1.1.2 inbound" + vpp_exec_cmd inl_ipsec_perf "ipsec sa add $((i+11)) spi $((i+101)) esp crypto-key a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0 crypto-alg aes-gcm-128 tunnel src $((i+1)).1.1.2 dst $((i+1)).1.1.1" + fi + + vpp_exec_cmd inl_ipsec_perf "ipsec itf create" + vpp_exec_cmd inl_ipsec_perf "ipsec tunnel protect sa-out $((i+11)) sa-in $((i+1)) ipsec$i" + vpp_exec_cmd inl_ipsec_perf "set int state ipsec$i up" + + vpp_exec_cmd inl_ipsec_perf "ip route add 10.253.0.$i/32 via ipsec$i" + vpp_exec_cmd inl_ipsec_perf "set ip neighbor eth0 $((i+1)).1.1.1 00:16:3e:7e:94:9a" + + vpp_exec_cmd inl_ipsec_perf "ip route add $((i+1)).1.1.0/24 via eth0" + + vpp_exec_cmd inl_ipsec_perf "test flow add dst-ip 10.253.0.$i/255.255.255.255 proto 17 redirect-to-queue $i" + vpp_exec_cmd inl_ipsec_perf "test flow enable index $((i+2)) eth0" + done + sleep $WS } function run_vpp_ipsec_inb() { - echo "vpp inb" + local i + n_CORES=$1 + echo "vpp inb num_cores:$n_CORES" IS_RXPPS_TXTPMD=1 rm -rf /tmp/inl_ipsec_perf mkdir -p /tmp/inl_ipsec_perf - cp inl_ipsec_perf.conf /tmp/inl_ipsec_perf/ + cp inl_ipsec_perf_$n_CORES.conf /tmp/inl_ipsec_perf/ cp ${IP_IB_CFG[$Y]} /tmp/inl_ipsec_perf/ # Inline Protocol - vpp_launch inl_ipsec_perf + vpp_launch inl_ipsec_perf_$n_CORES vpp_exec_cmd inl_ipsec_perf "device attach pci/$IF0 driver octeon" vpp_exec_cmd inl_ipsec_perf "device create-interface pci/$IF0 port 0 name eth0 num-rx-queues 8 tx-queues-size 16384" vpp_exec_cmd inl_ipsec_perf "device attach pci/$IF1 driver octeon" vpp_exec_cmd inl_ipsec_perf "device create-interface pci/$IF1 port 0 name eth1 num-rx-queues 8 tx-queues-size 16384" vpp_exec_file inl_ipsec_perf /tmp/inl_ipsec_perf/${IP_IB_CFG[$Y]} + for (( i=0; i<$n_CORES; i++ )) + do + if [[ $Y == 0 ]]; then + vpp_exec_cmd inl_ipsec_perf "ipsec sa add $((i+1)) spi $((i+1)) esp crypto-key a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0 crypto-alg aes-cbc-128 integ-alg sha1-96 integ-key a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0 tunnel src $((i+1)).1.1.1 dst $((i+1)).1.1.2 inbound" + vpp_exec_cmd inl_ipsec_perf "ipsec sa add $((i+11)) spi $((i+101)) esp crypto-key a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0 crypto-alg aes-cbc-128 integ-alg sha1-96 integ-key a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0 tunnel src $((i+1)).1.1.2 dst $((i+1)).1.1.1" + else + vpp_exec_cmd inl_ipsec_perf "ipsec sa add $((i+1)) spi $((i+1)) esp crypto-key a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0 crypto-alg aes-gcm-128 tunnel src $((i+1)).1.2.1 dst $((i+1)).1.2.2 inbound" + vpp_exec_cmd inl_ipsec_perf "ipsec sa add $((i+11)) spi $((i+101)) esp crypto-key a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0 crypto-alg aes-gcm-128 tunnel src $((i+1)).1.2.2 dst $((i+1)).1.2.1" + fi + + vpp_exec_cmd inl_ipsec_perf "ipsec itf create" + vpp_exec_cmd inl_ipsec_perf "ipsec tunnel protect sa-in $((i+1)) sa-out $((i+11)) ipsec$i" + vpp_exec_cmd inl_ipsec_perf "set int state ipsec$i up" + + vpp_exec_cmd inl_ipsec_perf "test flow add dst-ip 192.168.2.$((i+2))/255.255.255.255 proto 17 redirect-to-queue $i" + vpp_exec_cmd inl_ipsec_perf "test flow enable index $((i+2)) eth0" + vpp_exec_cmd inl_ipsec_perf "ip route add 192.168.2.$((i+2))/32 via eth0" + vpp_exec_cmd inl_ipsec_perf "set ip neighbor eth0 192.168.2.$((i+2)) 00:16:3e:7e:94:9a" + done + sleep $WS } @@ -261,7 +307,6 @@ function sig_handler() quit_testpmd "$TPMD_RX_PREFIX" if [[ $status -ne 0 ]]; then echo "$1 Handler" - ps -ef fi awk ' { print FILENAME": " $0 } ' testpmd.out.$TPMD_TX_PREFIX cleanup_interfaces @@ -298,7 +343,7 @@ function pmd_tx_launch() C_MSK="0x3800" fi if [[ $WITH_GEN_BOARD -eq 1 ]]; then - exec_testpmd_cmd_gen "launch_tx_outb" $TPMD_TX_PREFIX $X + exec_testpmd_cmd_gen "launch_tx_outb" $TPMD_TX_PREFIX $1 else testpmd_launch "$TPMD_TX_PREFIX" \ "-c $C_MSK -a $LIF1" \ @@ -322,7 +367,7 @@ function pmd_tx_launch_for_inb() C_MSK="0x3800" fi - local pcap=$OCTEONTESTPATH/inl_ipsec_perf/pcap/enc_$1_$2.pcap + local pcap=$OCTEONTESTPATH/inl_ipsec_perf/pcap/enc_$1_$2_$3.pcap if [[ $WITH_GEN_BOARD -eq 1 ]]; then exec_testpmd_cmd_gen "launch_tx_inb" $TPMD_TX_PREFIX $pcap @@ -509,12 +554,11 @@ function outb_perf() rx_pps=0 if [[ $tcnt -gt 1 ]]; then # Restart vpp - vpp_show_trace inl_ipsec_perf vpp_log inl_ipsec_perf vpp_stats_all inl_ipsec_perf vpp_cleanup inl_ipsec_perf echo "Restart vpp" - run_vpp_ipsec + run_vpp_ipsec $3 fi start_testpmd pmd_rx_dry_run @@ -542,7 +586,7 @@ function outb_perf() done if [[ $tcnt -gt $MAX_TRY_CNT ]]; then echo "Test Failed" - Failed_tests="$Failed_tests \"${TN[$Y]} outbound $algo pktsize:$pktsz\"" + Failed_tests="$Failed_tests \"${TN[$Y]} outbound $algo pktsize:$pktsz num_cores:$3\"" fi ((++rn)) done @@ -564,7 +608,7 @@ function inb_perf() for pktsz in ${PKT_LIST[@]} do sleep $WS - pmd_tx_launch_for_inb $1 $pktsz + pmd_tx_launch_for_inb $1 $pktsz $3 tcnt=1 while [ $tcnt -le $MAX_TRY_CNT ]; do @@ -577,7 +621,7 @@ function inb_perf() vpp_stats_all inl_ipsec_perf vpp_cleanup inl_ipsec_perf echo "Restart vpp" - run_vpp_ipsec_inb + run_vpp_ipsec_inb $3 fi start_testpmd pmd_rx_dry_run @@ -607,7 +651,7 @@ function inb_perf() if [[ $tcnt -gt $MAX_TRY_CNT ]]; then echo "Test Failed" quit_testpmd "$TPMD_TX_PREFIX" - Failed_tests="$Failed_tests \"${TN[$Y]} inbound $algo pktsize:$pktsz\"" + Failed_tests="$Failed_tests \"${TN[$Y]} inbound $algo pktsize:$pktsz num_cores:$3\"" fi ((++rn)) done @@ -646,17 +690,11 @@ function aes_cbc_sha1_hmac_outb() local cipher="aes-cbc" local auth="sha1-hmac" local algo_str="${cipher}_${auth}" - local cn echo "Outbound Perf Test: $algo_str" populate_pass_mops $algo_str "${TYPE[$Y]}.outb" - cn=0 - for j in ${CORES[@]} - do - outb_perf $algo_str $cn $j - ((++cn)) - done + outb_perf $algo_str $1 $2 } function aes_cbc_sha1_hmac_inb() @@ -669,12 +707,7 @@ function aes_cbc_sha1_hmac_inb() echo "Inbound Perf Test: $algo_str" populate_pass_mops $algo_str "${TYPE[$Y]}.inb" - cn=0 - for j in ${CORES[@]} - do - inb_perf $algo_str $cn $j - ((++cn)) - done + inb_perf $algo_str $1 $2 } function aes_gcm_outb() @@ -686,12 +719,7 @@ function aes_gcm_outb() echo "Outbound Perf Test: $algo_str" populate_pass_mops $algo_str "${TYPE[$Y]}.outb" - cn=0 - for j in ${CORES[@]} - do - outb_perf $algo_str $cn $j - ((++cn)) - done + outb_perf $algo_str $1 $2 } function aes_gcm_inb() @@ -703,12 +731,7 @@ function aes_gcm_inb() echo "Inbound Perf Test: $algo_str" populate_pass_mops $algo_str "${TYPE[$Y]}.inb" - cn=0 - for j in ${CORES[@]} - do - inb_perf $algo_str $cn $j - ((++cn)) - done + inb_perf $algo_str $1 $2 } get_system_info @@ -761,72 +784,79 @@ EVENT_VF=$SSO_DEV setup_interfaces exec_genboard_cleanup -Y=0 +count=0 +for c in ${CORES[@]} +do + Y=0 + echo "" + echo "Test: ${TN[$Y]} Num_cores: $c" + echo "----------------------" + sleep $WS -echo "" -echo "Test: ${TN[$Y]}" -echo "----------------------" -sleep $WS + # Outbound + # aes-cbc sha1-hmac -# Outbound -# aes-cbc sha1-hmac + X=1 + Y=0 + run_vpp_ipsec "$c" -X=1 -Y=0 -run_vpp_ipsec + pmd_rx_launch + pmd_tx_launch $c + aes_cbc_sha1_hmac_outb $count $c + quit_testpmd "$TPMD_TX_PREFIX" + quit_testpmd "$TPMD_RX_PREFIX" + awk ' { print FILENAME": " $0 } ' testpmd.out.$TPMD_TX_PREFIX + vpp_log inl_ipsec_perf + vpp_stats_all inl_ipsec_perf + vpp_cleanup inl_ipsec_perf + sleep $WS -pmd_rx_launch -pmd_tx_launch -aes_cbc_sha1_hmac_outb -quit_testpmd "$TPMD_TX_PREFIX" -quit_testpmd "$TPMD_RX_PREFIX" + echo "" + # aes-gcm -sleep $WS + X=2 + Y=1 + run_vpp_ipsec "$c" -echo "" -# aes-gcm - -X=2 -Y=1 -vpp_log inl_ipsec_perf -vpp_stats_all inl_ipsec_perf -vpp_cleanup inl_ipsec_perf -run_vpp_ipsec - -pmd_rx_launch -pmd_tx_launch -aes_gcm_outb -quit_testpmd "$TPMD_TX_PREFIX" -quit_testpmd "$TPMD_RX_PREFIX" -vpp_log inl_ipsec_perf -vpp_stats_all inl_ipsec_perf -vpp_cleanup inl_ipsec_perf - -# -echo "" -# Inbound -X=1 -Y=0 -run_vpp_ipsec_inb -pmd_rx_launch -aes_cbc_sha1_hmac_inb -quit_testpmd "$TPMD_RX_PREFIX" + pmd_rx_launch + pmd_tx_launch $c + aes_gcm_outb $count $c + quit_testpmd "$TPMD_TX_PREFIX" + quit_testpmd "$TPMD_RX_PREFIX" + awk ' { print FILENAME": " $0 } ' testpmd.out.$TPMD_TX_PREFIX + vpp_log inl_ipsec_perf + vpp_stats_all inl_ipsec_perf + vpp_cleanup inl_ipsec_perf -sleep $WS + # + echo "" + # Inbound + X=1 + Y=0 + run_vpp_ipsec_inb $c + pmd_rx_launch + aes_cbc_sha1_hmac_inb $count $c + quit_testpmd "$TPMD_RX_PREFIX" + awk ' { print FILENAME": " $0 } ' testpmd.out.$TPMD_TX_PREFIX + vpp_log inl_ipsec_perf + vpp_stats_all inl_ipsec_perf + vpp_cleanup inl_ipsec_perf -echo "" -X=2 -Y=1 -vpp_log inl_ipsec_perf -vpp_stats_all inl_ipsec_perf -vpp_cleanup inl_ipsec_perf -run_vpp_ipsec_inb -pmd_rx_launch -aes_gcm_inb -quit_testpmd "$TPMD_RX_PREFIX" -vpp_log inl_ipsec_perf -vpp_stats_all inl_ipsec_perf -vpp_cleanup inl_ipsec_perf + sleep $WS + + echo "" + X=2 + Y=1 + run_vpp_ipsec_inb $c + pmd_rx_launch + aes_gcm_inb $count $c + quit_testpmd "$TPMD_RX_PREFIX" + awk ' { print FILENAME": " $0 } ' testpmd.out.$TPMD_TX_PREFIX + vpp_log inl_ipsec_perf + vpp_stats_all inl_ipsec_perf + vpp_cleanup inl_ipsec_perf + ((++count)) +done echo "" if [[ -n $Failed_tests ]]; then diff --git a/ci/test/inl_ipsec_perf/inl_ipsec_perf_1.conf b/ci/test/inl_ipsec_perf/inl_ipsec_perf_1.conf new file mode 100644 index 0000000000..99b4521193 --- /dev/null +++ b/ci/test/inl_ipsec_perf/inl_ipsec_perf_1.conf @@ -0,0 +1,55 @@ +unix { + log /tmp/inl_ipsec_perf/vpp.log + cli-listen /tmp/inl_ipsec_perf/cli.sock + #full-coredump + #interactive + nodaemon +} + +api-trace { + on +} + +logging { + default-syslog-log-level info +} + +cpu { + main-core 2 + corelist-workers 3 +} + +session +{ + event-queue-length 10 +} +#tcp { no-csum-offload } + +buffers { + ## Increase number of buffers allocated, needed only in scenarios with + ## large number of interfaces and worker threads. Value is per numa node. + ## Default is 16384 (8192 if running unprivileged) +# buffers-per-numa 5000 + + ## Size of buffer data area + ## Default is 2048 + default data-size 2048 +# naturally-aligned +# enable-nat-alignment +} + +devices { + dev pci/0002:20:00.1 + { + driver octeon + } + dev pci/0002:1d:00.0 + { + driver octeon + } +} + +plugins { + plugin dpdk_plugin.so { disable } + plugin onp_plugin.so { disable } +} diff --git a/ci/test/inl_ipsec_perf/inl_ipsec_perf_2.conf b/ci/test/inl_ipsec_perf/inl_ipsec_perf_2.conf new file mode 100644 index 0000000000..965e287b3a --- /dev/null +++ b/ci/test/inl_ipsec_perf/inl_ipsec_perf_2.conf @@ -0,0 +1,55 @@ +unix { + log /tmp/inl_ipsec_perf/vpp.log + cli-listen /tmp/inl_ipsec_perf/cli.sock + #full-coredump + #interactive + nodaemon +} + +api-trace { + on +} + +logging { + default-syslog-log-level info +} + +cpu { + main-core 2 + corelist-workers 3-4 +} + +session +{ + event-queue-length 10 +} +#tcp { no-csum-offload } + +buffers { + ## Increase number of buffers allocated, needed only in scenarios with + ## large number of interfaces and worker threads. Value is per numa node. + ## Default is 16384 (8192 if running unprivileged) +# buffers-per-numa 5000 + + ## Size of buffer data area + ## Default is 2048 + default data-size 2048 +# naturally-aligned +# enable-nat-alignment +} + +devices { + dev pci/0002:20:00.1 + { + driver octeon + } + dev pci/0002:1d:00.0 + { + driver octeon + } +} + +plugins { + plugin dpdk_plugin.so { disable } + plugin onp_plugin.so { disable } +} diff --git a/ci/test/inl_ipsec_perf/inl_ipsec_perf_4.conf b/ci/test/inl_ipsec_perf/inl_ipsec_perf_4.conf new file mode 100644 index 0000000000..eaadfa3512 --- /dev/null +++ b/ci/test/inl_ipsec_perf/inl_ipsec_perf_4.conf @@ -0,0 +1,55 @@ +unix { + log /tmp/inl_ipsec_perf/vpp.log + cli-listen /tmp/inl_ipsec_perf/cli.sock + #full-coredump + #interactive + nodaemon +} + +api-trace { + on +} + +logging { + default-syslog-log-level info +} + +cpu { + main-core 2 + corelist-workers 3-6 +} + +session +{ + event-queue-length 10 +} +#tcp { no-csum-offload } + +buffers { + ## Increase number of buffers allocated, needed only in scenarios with + ## large number of interfaces and worker threads. Value is per numa node. + ## Default is 16384 (8192 if running unprivileged) +# buffers-per-numa 5000 + + ## Size of buffer data area + ## Default is 2048 + default data-size 2048 +# naturally-aligned +# enable-nat-alignment +} + +devices { + dev pci/0002:20:00.1 + { + driver octeon + } + dev pci/0002:1d:00.0 + { + driver octeon + } +} + +plugins { + plugin dpdk_plugin.so { disable } + plugin onp_plugin.so { disable } +} diff --git a/ci/test/inl_ipsec_perf/pcap/enc_aes-cbc_sha1-hmac_1410.pcap b/ci/test/inl_ipsec_perf/pcap/enc_aes-cbc_sha1-hmac_1410.pcap deleted file mode 100644 index c676a133d552b49817160954a61579660c64d0ba..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1518 zcmV2Q8;4O`vD_h~ajEDappr3ts28(0na1%8X+$;E*_(HY(zWUe*yRK}1X|VaVgId{ zbsa-nHH*vVPAeG|gV^H{{k?yti~K5_cHr|R^`gdvO`k2D@@;cF&3r=8tS)bfa0 zK1OXcBkvXjsiBMU+pHLOw>x_9mF0tb@nI2kFcE z7Pq0IFz#4I^ESG_C+dw@Qfi)|Wt{=6kOr`J{DJ&<_|AwnsAO=wy6+Bh76Nl!S-+cWJun&hY;uHRS zi1i{{_2!j%T+Er8X=0*PzRIw`Uv}jL!Gd=IY^npnNcI$DHB7AL7tAsr>6Sq%M|{-O z2m?X#s@cp+E@{(3Kt;?$%6RGxM(G6R1phUDBl%Fk7j>_2IIwfZ2?)~)i)^;POdn$oko zH1Qyu4QR|%5lnm&Ra|}wvT*7Ng6;2`x#_S8zkP4Dsz&c;6Ro)CZm_M@+qOEpB>JOz zM}B#_FQaC(Ofl~k@izoQ4j@NgZ*Y>enl@N1m9n@xw6jQIh0YSRV2@vT*jIfPu<^l4 z`i`NAgg#{OC{WiylgHkeQ%#Fw)sKQ)&o;7#AXbiTsg1D*BE0NN=E1bEi4*cTJ>5Y( z2xxkOfAeoQ(I9w@->fsd@{Fy(MAk}JB*$5l=|}<~9$rF5@)?^LyU{m|ptRqKe^oHD ztKw@6FdZ;HGHUuYJi#G!j0VRXF zv2N=sHZr2q4hP3%^JpKe>}d4Krm%>#G0{hsE3Q6N+e%jEN`Z~O(;)NCM=quoOKIg6 zj61wU%zDQ$p;Ry~(qv9>c8oH5gY{k*`$ESY+xQ$G#h<7n0 zw31K4+tuw7(1&k?-bl0T?`8xPx^?L8`yp}%W zkdh1{O4ODQa0IZAs*~T5G%QLU^F!56`2GWG`s6pc8n&*{Jns%uIjhBIA3{(F4aWOz zO70B61f3i`lQ$Z(M{Lg>dxh-)0C^1xKLlmE908vVIW@S}1I0hQ`1Hg=81=DS@BX6D zk`K2U3c!M-5Xuzq2rk)m4vz1?0QfyF-O})HJAL=nWX0Hhu}rz|)J%Je~RJ62xWA zlS;83Q9=aq3ChL2C=eJuGRYt1gKlN#rv9oVgKr^G-`KH6Z<;E3AB8B=6n%K6wloP; z%A0?WYY>zM{tWi*pQjy@6yu@ypF@0f4oYM^PUS_z&<>vsyHth1U;yRQyK#;GvatSv UPvx}djU`lue!0(s6U|Ucsb5sxEdT%j diff --git a/ci/test/inl_ipsec_perf/pcap/enc_aes-cbc_sha1-hmac_1410_1.pcap b/ci/test/inl_ipsec_perf/pcap/enc_aes-cbc_sha1-hmac_1410_1.pcap new file mode 100644 index 0000000000000000000000000000000000000000..6b83eeb285a409735032a3219f801f1892317256 GIT binary patch literal 1518 zcmVxJ0RaI50ssI20RR910RR91000000000000000 z040uRHdrtGJ3Jj+`{>Nk8XO!ieBJjT4f?@|BkC4cE~=BNbvv~T?Fc=oKxdK|PhV$_ zyuFd68^lV3oCx3iLTHVJ6^oUf>qoZ>LDAv>UOqRaXq^kwFGvl@S5W}@;>= znz=yQ|Nrtyh%i=9fH%NpzFuY_l*kAIM#8N`&TQ!^N{`yu4&9)%>+gZ8?v$tRDZ@HE z!){!E3z&STL^N+=_A(o3BB=O7awE94z)+u3&&`abeD==V6o{r(1;yC3S&z?-z5zer zyJF=qp8X?5ifKBp0rk~WCgFZdW|-h9!l9g|bUM7bE@1?p0&h_RIH#Fe22OH?1VlT9 z%jpEu+K9n5z#II^n@RxEn%Daby7OYwp8(hR@;|gf=zOp!rb?ni ztys&T#@Mq;9^igT<`^-CTf@W-MlQMDiTTJ?x3&k3=U2Hwj;4Ei-N(s}X9FHhNRqrf z)G<@F|5T_VS`{E!#&GJwoSzNiS_gj-Lo{5W(Ooq>E^4j)@n`j8Mk@Q681O@B1js)h zKR57)W<-Dwq)L$s-luCl@r*u8^fKd-?*izfT*4>qs4*$=-i}6`GJ-B%GplHiy)w?sr zB*r!S8+KC`aHn<8ZHY7sr65x@1XqmeFW%VGvsSqM=fIv|+GvnLB~Ixji)*tPoq`zQ8Kx{plX|vkt;gCpmqn_1=LOy^qr! z90ce8ndcv6L*!fC-#|Kly~7(O9+AH{9L^9;~)A z{hYLy93Xiil0Jl%lke*Ea5F-oOqW8N+sK@Ph`txi&Qk4ec);0HsasDOjtcVXxI+e! zPySn;v`iEp3b#n1urT@xga!xJhLRQHel)!R_52aEM1ZQaz=F0oE;^_EX&HNMNFaDy zAsgh}N{uDfA?F|oSzGK?@OkBf~@c0>kOEnp{v02*%>R;S`5I3E^&a$%f z{`KH)eO`Z1qwI|A$(j{A{xuUMd=H8vUnZ8ym0Dk0lzd8GbmsPPv=X^S5$5P)x-jsG z*X+E)`6-mJA8toDmQur7l+wXpF~tfO$V>}?LOge_wBO4+9g})xjIz(tNL>*LNqcq{dXyI)eV`;h@MjPAN}4uP`AhA4stc{ivAT=LB|0)CvF%}`4bbDf3G&>T!* z3(}SG5`&w&1VHS!Lg^twW&b7FrUC+{D~X-4JY}F zSS1|WO>lR%*19zwDTR5`1MqUexRAtr;x|9r5rj`fMqthtm{kqBiLXla+wzi`*1&97 UzaXuwC_^V%>G&O3rc8uC3QGz10h?H72T3fxS8MMSIO|@o?7FDZ?h!uOZ zHdT98?Y%Ylyx;eGFW#H)Ilt$e^ISZ)|DooxEJ^@1;2)iz0{|2kf#$-tS`2`;i|20x z*i`wlM*p{I0SEw16@U@|P?vUNq@bXrxbQ{#Z}IQ@FA2ms$eBrym6Wex{w#I%b3q^y z0h5P33@43|{3~px;svh@o+(w(uhXi0Qg=w?FtBontF24;z}L0&=gNQf#w+f|q!87|&d+yrA|)RfgvdA8*IPd3 z#dpwBYBd$Xd!80yBF&1*&90bW;Jbi4ILyuFK$_c$FaJo> zlfSB>!SFq*w>#!zz(LQ%)yUlYG_51$=JCC8^%QC|H8u;98K?XRoReY+#r_Xt!Py|) z$M0rDnzGVzy%lST;?~sfDcy{y6!MbIFWvKupoS|&d|##>9ER zhrS#P)`VfQ`X9swWIFw2J`E8LKLey#P;@AH}0kqYPoC#W6g;D+Ky@2EwrXy zw9M5SQ5B>XrJM$~K+1ZLOSy3OqWk6^sS#BOx*}p_Q=PS&`^_kY=U2i%Wu)eAkGLbf zCExknh>fNJVYs0qQ>aU%YdrV-h~ukwz_~U<6qt9ZJ?7N2-5ZubW7A;o5-Rp;$TPeC zFG%t_ki!&P&5=!}WOb`?Ev)lYsq#I@bo}n{qeiCnMw{#(ZXaVd_q=DlE>SSxM;>Ds zwQ08L77tbrrONXir&5TrO+Cp ziY#X#$YQR(LG<_OGW6gpW0RrK?ZEu~sZjR1_`z!s>c#Wq#q0K(zbq!F)fI#5nz;p` z2_$()4@iH(E{GK$`1mY&5A|@-3@SQ8r?!@XW-&fZD}M{&@#9TY4NrQ#xw7XjqmiZe zRwH%1BMlo_&)(gGT6gtr7&ZQ6VZa$jx3f~Ec`3o*%p#*)?<$mzpq*7Jc|sR{>GJT4 zMAo?=SRG)W3@(R<6qGk$D;2~Q^G=;OpSx)D_*(FC|C&I?3Jmiu@?0^uSVM)!ySzA#jJIvuqvI&vfKz@ z!tq-PCVhK@wz9wnAg#QAA9$2A6Scs5rujc){}%?Y_RE?>-Ly>wcuKtdd*Y6!o_935?O8bM*Bh)KFQBCa`q+h|dpwJlB-h6WCd z7OW+I1xscK{;*)m;A<<79#>^)x$LAC43UZ2MXU&)l}INZu4PL6Zp8xiDmJ)+f`7IX z_7Wnn5l-f}IBxk9T}ua3YEP2l@@s~kSDNg~bV)b*PyXD@kcZuK9>|%Pc3z z)c7Yq~YDr`ynoWRNV613dW{EL{Z)i`z zlcJyroe<*j!ak?Z_W+}4hDDhv%`4kBum}qM?(ACw9${ zLv2V;t^;Lgx!;|2GX!mfi7meN2hCgt-VP3nMhMJBKN&zY7?bFdA@~&bY$KP~?Z2QR zTMxO1na|)af4q3OiV6V=v5tAMiO=6Z;TJz8LIKeQbJ+p%EKP|fLLc`zbJN4t)?M<*kciy>-C=BDE_Egdewk}Wgzp)4{911ecWe8Gp+F0npu z_mLF%ol;LrW{*?@Cq)zAywMlReC;S)jPMUPFIL9|8TjWiuZ;bI;^#d%$=geIC3ia> zBF=r7sDJ^b-i5h(0Kd}BLZ(O5Io98V?Kh_FV${@#ZE?~x-(h)p+o8GA$X!}Q>sem} zDS<~qe0}2qkIzr^bjWrrAc^AgK;2Mr?51AEz`7a$4a$J%r*;qQ(AMKNT_Y=G(uWFdVd_8niFyI6aYf; zORvLXTW(`~eaKklr@M0FekmV{bHfAWoq^wfud%S$HVJCV+#1$HIVBFG7QC433?7{? zs_F851k3v_`PV!_*=GdL@JIa0x0T2}W%m|0hzy(K$PUW?@!>o{)39jk@yw9T*qdsg z$NOx{=qa+U)5|^%+Z+4!Z!uZ=@tKSKr#B9FdbKnP0=B4>_7;7AF0|z**=psyoS&!! zvjn}iI|;W1%M~MTnX*afN7uX27L1{feCBbedsz)~5B$!m+67UgT?CKUy%V6-Ib+Oq zEG$HE$b8~jZN2Of-r#7H-4U|YlodewATP${P85L~nQ}TZt|pDy>`8`I*f(!~(31m^ zKYKe`mBI{#{I&U~90amSufj5DiVnJ8!I%R@U^?|?JSTdN=hcuUCmqB~xu?AOI;Ja3 zZ$Gfn?LiFIa5j|);n-x2C+#)g-YsD?Q@H4jRZj8qPX=xXrTGwWFf_N^SdR1SuNK!; z!lHw2=VVqzuDsjynC$He6dTS+oVXfOF@!J1MQj&TE7;h~DD*Nb?ntQ|dyvj)RL{%A zL|5Zv7d2_6Cjw_w4Y!8Ihf4({&m8r-%)~&6& z`*xk~>iz*wpPBCJTxb9+;D6@r4FG_8zaYM+g8&ZD2M7390Jy@Gxr_h3BLJiTaE$)e3mP=kzt;QhpO=5-{rKMz=tmPm@v<0e@EDa*hEzz4PKmlV=Z)DB=@|sd^{$*X zvo&iwS(l0ajA#=xcHO;{z?*STueZDnxL1p%Wt5QJkoPRTpryMbMgmRE3?$z1Zd2X+ z70aGsL7o#eiiXMM<2NZfWlelh#_6C0l@?uJyJIX*SNvJX=GwWP*W-VoIUKQS!(!L4 zQ5wT+uSniava@R5#|!Y`c}&T*36xnw4TNw=);W1ST%|x)FLZ}Ln68hQ;S&GD+ag^gAj-%PV z3mrhyALwEs$AH&RUZF$rnbIQQ%hOfyw7(4F8OuXN%-NjEka{tOce??wJf-g2XOW0p zFV|zx*p-=OYY_e1r<2?;3?O-gAmU&E1JlQ~2x*bzXL&tb8cdp{vSp5av~+AB}&b(w{wGmR?!X9Qx+i+#Mfhoj?>|CF@qS!2tGXUT~Vge$W^WiAOSiWaBx& ztlf?bnFE0+^>q~0_cVxUFTo4A@d267FDO!qW-HQeN~%M>9Vv}CF&!^Y^-DtkUs8s1 zL!vp%SIHlXGVORl)g+nSG5%r6j9KxFAwo;MFxSG)#Y;SE7iW)KQ}R2bW` zZhLp2X!6I8q)jXamktvqj|_QY=-RHzVgOl}- zXgPsL3oah0KYw)xynnNVdWg}dVRax3= zBaN!KjWmLrJJfvw<^EJ=9_MJcUK$IJFq!8Deq@(8fi$%A4XVl;nPflsW#orE+5>=U zRO~05KYRELJlvlH1?{i6mp#)9%8P=1cx;h}caO2K%)4kMI6loOs@cTPsqMI;TPT~n z?TIK*7Gm&t?0Yrasaa)dckvJRt^i=!q4+9nJsvWlTax;x7wI7TS{s| zcFxW-id=hEu=z28Z%%#FmST{r6qB<@^+J7jIW8?;>U|C)aJ}a-ztdNeY%ZPuV|%3@ z1IstT;xX79gS%YnlQu4kQe>MWLg|v;z58~A+UML3ZY|H(#(rA0#gPW*gz**N(U!Ua zSs_&5xAvg5H{l_if~)esJ4Ul*nlx zqNfz#B3%>^*49S??tr$n z;zZWE9H>r;U$9a=x=E_?O2qyd%oTa;t>_QnvI>TDB4_LZf_Ih;YEb@@>hJiAvfH7< zysQ46e?*6Q&p%(nzs+LkMHQkO@C}}58z^5~F|10^28|pWylwK+6eWU4wFB!799(Ne zw^lUSZ9*S3(0Q~`MSl!ld0gX5%|P}Tg%@1wxV8s@lILHi@a_5# z27ARwIVW>7SqyzOMe_$B18GN#T}I7HI6mcUQ_9;KnkkMG$uN4-;_5)kE^-r&i9=10 zXf_|1V&%?tO(-Pe8+%d$=!C5u>5k=xn&n3%sIjY6Hhdjru z!`K~W=SS{Sl1r(|6HLZ0jBf?wB97kyhC7vuMcAicr)*xY>scZ?nJ+UHRP2gw#!Kj8 zERz170n$3!a0#l;E-lhsUmM&cWik;7H%4QSLWdK4f>an;(st7a*TIcmNPmoeNS80F zOQz_U;j9>lXIG((80q%MV>HH?U|Lsm{v_WKP0)3jPQF5qPQIOL6}nEe{}S`fOK&ZY zHaB*xnJBMK4>232g{;`KF|$VXWIP-mw)KG0k!4IdvO9U$bU){q#vtj^j)Dim1#@Iz zCVT!%{SSljlN4+5QH4#mtfGGT(_g2=7i?0Xb?`)KI6E|fIGVp8nxf?_d)F8WlBPdU zU`*+ea&G3c#z1AYUMt1SubEjLr;99_;wfrsc%6d^M{r=GQ`7H;8kJHbb#@P3eV%5~^vgfalNhxBLVniGwR+13!f(CYdzXpoTV<0XI4~m*x3Rh#c37v6vF_ zgut*O?-8@vRh~`pyyH^T=tNQdFE&bwBG`d|V;iiat;z&4+st4xN`4Itmy2mE$0SM3 z%Tx2Xd4858B_-;?Tje0h3SYDj7xLH(z+4|pLS#g-%#ZTYe3b?%z+uh5eId&wF>eoS zFdmwzq=3{eb=y(Em;2z=IzL|CbMRm9i9OugQnv)yIdkaaT<-c}aOD|rDVzD%*_Cj= zB}+>UpI2WS9{GuSx)+Dw?-0T>tMd#3fJN)w3E*DTjl-~(jq0AVke||HSvObS4GQ!{ zff0eX=V|KAavD_<7LSjk#YDWX1^Q5HyGP{;qG);38lf)`EqBLG{j$03OT5NMNQ_^N zxEQ*&I*DB3PgkaWpzRXrSIBK9^U>A$3RlVMxz8{fB>p7(D*13YPTuqrqj2f}e%C zyx3M6F_;BjsL(Ci0uKD#(w}OE5z*{`;67cgXURL7>3hVqD~bahZzk_{5Ug=*Mg^)C zff^b&ZIos@pRDZCLT1zLWn`E9)(Wa8tIgUHq}j?!`9YNCXj{^jf=U`T?!k8sn#9PK z6g_Nno#-3=L(QqUW2D2g&0fEp^fvCz23$bMSUw21$je-vQ=w6djqTS?kvieC&rRw| zS1qC1)&U}{yVq=I`YlVcTMMK?%H|VMzIxa3)*sVftlIu=HtT_nGyhsz`5EP0EN0X7Ca!A3aoHohDyp z$&YRoKAq{73PItxN^@?I;4RE~a60|0#v3P8vKEkp;}r{Cg;KA5*Sp@;iZ7(7_3kSU zwK^ish}=F;Bkrdl*|&JZx^T~9dT6=}Ay<8_1l!Tk#TUeTzi?GC-p~!If&q8CzGH+U zxzlpp<-IpKxKA9l%^zk_FX%Fh4$R`?in=&w1k(Mal?v7JJn8sL5EZ~`fn9FDpR%8p zp_~LKrvUz4#mWQMOz1AME3&m?WXImvv!PBo1~u&mg!++q%?Tvk}z*&N2}`mLaa5l8E0x~PAS zdy0r2EGp6TUe|6muy=(kSw$6%I`@L5yxqQs@8=N0D`4fSV0c@4^+6rKmp@Z8QUJFh z8=Uc2uaGaLfR?etI1`l8#~IrR?os)K?&!MW&&V*lkk;vMsC!SJ;j5WcC_%usjBW;_ zmkOf4RI*l@v+e{kBVUZO_;$_iyz?CxM@>$^Ti+|kP)Ck0Cp)%TWU;r=DtccYyr z2%qXi9lYo}wnr7|l-h&^VW{}^f6iCa8Z3|qOcbC+0N2e5s39}_#)#eB@4djV1cMa# zLV~%L?}*6wW0GF@?I5_dwf-wIYhT$pI_^hN)GAcIxd8hr^DU=H#)G$(51Ar3@K>Xe z&2>T$v~qK@D({U#=|hTGF-hFF`N5W0J|X)peo;ppT5a%>w)|>noEqe zvFR1(D`#T$|vA_}sK0$3$|%<5|wlvxAwJEPB} zqKq2hhvaqma{Fp9@3-=oG1lk!knBlE*)zHKCJhBwLS>-ly?Mw=R-do)9-{n(+smr1 z`?NS1gOEX1pzRQ@ku%^bqsdrG<2L(;0Ka_dV6u(<_r&V0lZco%-RF7Yb-F>u0i17H zIj8b=!aq8K>jL6O>HH-RIhh4ulA~jIN@tyfPQ(1b9fK&^sB#<97uOCtnj@*=WuWEx z^_)ff^c;P@mETUN^pcRhx1WAL6-~R+@|$U;5Kfvl&k$w_o_pc#!VHw7zU|W(bw3QCqDZuG$a+B2sK@qA8>1(NnD5Zy9S}H*h5K#piXlkOZHq% zS!GKMQxl=YJTT!Hh$M8Q%r(n5_vszz=o{$&r22a^7iD)ug?d;0J^!fwkLdsO^?w_h zXlrgAmS+J$5`(Axkx4>gVixb!J95;=&M>M&Ty5skgwn49btoqYLcc_<(An#pCLGrNts774aj1XTL2?roVUQ?;9;wFemsW%Dk>c! zy7cyTDHq17rR>ksVcBM6MUgmw04?2BCOKw*z#t#9w1BpnQ+sq4PB4_@34xUq2pK+r zon8tXy%+YI6&bd;A#q`&GYQbb#gfsnl|vu0pF!at@yroC4G^6o za8BPsjXddwulz1PTd00_51KSHWG(z*K(nF#!_Pf!h*)&d_wTFygVEM7q)M{!ErK&_ z>UD1#C6szNw z=Rd3@{m~2Li?Sw9(gcYvYx8u&B_wx<9=uS%57%tYLrNNw@lQvT2!qz{>C!0F2 zG8$S4&1iq>Kqh5D;Jh9LZC3=$Ch_pP_8x)!4qPATHfQ)ENvASRUFVFDMk%B+NPSnU zsiV`!aNh5T!N0OHiY^wM@!T}A4NEg=B;M+;aKYK!!kHj#F%x0hrQZvSM zVK+xljgZx~{oE>G@S2FfT1-~SKc67uYX>&r=RHH?I5s={X>}25^tjCd4O`DPbM#X8 zaXk^atK&Kqe;xl4&WT(|uFeUo9d*d2H#M>d`RP_4rg#{R4Lp?U5#dB)x*ELcoTFbO zepE08>OJP=s(r6Dt#nBf2xx`~aS{9h>Tk7mFz&t_klxmyb*a$t`IOwO1>m*DOp_49 z?!Be$ky%Ro)n8o~<@amtCZZ*cYv&;z11jxU_~In`SiPoQs~PU;kTNbHi5zF?VX41> zYg35HJt^XagOd}^4zHa!#B&u>#50x$z=2_Hc%x%#Ck`GRi|W7hl?kGeE)N?e;Nz5E z^|NXfu}P5{WOFN7ACZFLKg@;3<81kIwgPURF={3KOKUp<>v@^^i(hOAzJHM=_t2*x z`nw|klV*-$j~r1~|5z>`HXRM#-EpP0$d+S__36aLw-8XMyJrAEh zZs~epNUS7tkl{^M>()!0>NIBSMZz0UL=F%a{8nDOzWJJeRD=*)!5e6eYun)BSd6nH z2b?wGW+#4=y-}D!CB$-{a7(_c-wVBulIII)CmrOM1hGnzXqe`6Zgzn-9m=x}I(yq| z@TXqsh_jm&-5&2g)>{RkqVtXe2t6mgnF4 zxBIkN4Wmz5XnN7PDnuEZ`*b}_PB$1H(2S;z4Lgqm*iGGzh=fzMg~=1w$hEY(dN`3& zk65iSq$&9=mcpm8{xwFlsKJPew7*NPMoN3EI~JK_e#{+41ASPtJ4~L#i5^ku9wK#U zh?kX|zSYk)agj2Fcuv8byL2|j_IPjch%93SD@#h7jNzqfqBezB$lokq#;477*z2^D Um;Iryh5eWxXvfyM6=0S6UnmhaDF6Tf literal 0 HcmV?d00001 diff --git a/ci/test/inl_ipsec_perf/pcap/enc_aes-cbc_sha1-hmac_380.pcap b/ci/test/inl_ipsec_perf/pcap/enc_aes-cbc_sha1-hmac_380.pcap deleted file mode 100644 index 75852cd6d4f1ed8e824fc62c30f800b903d5ea9e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 478 zcmV<40U`d>!?K|Q00aO4000000000000aO50000LeJo*!ZVUjn0RRBD0RR917Cs`O z*~Zo6=ttuS07U=+r~m)}06;QzAOQgZ0RaI40ssI20RTwnySo}Z`Y-6jG*LBZu^aWt zF1v2-3>Q$|nr{X@&X|JW>u zJkvuvcJu7Mj8p$Y&qVcxH&fYYFDaliRm6B+VCm2+Ty%`TxJ0>~B55m;*2YEo^!4gk zq*m~YqW}IyxgUJ4@`7@)+G09z979crX;9Nv-O8Gb_IV(%oeGwm2Z#{d2H8*z+&<3M z$XYZELU{Vmwq4Nt^A!Y_Eo%L&R`76jByQc;r?L0qq#fM9i)u5gA9cK(ZflIfEo6!* zL|PX`fM`0Bp~cfQEZfyu5FgXG0IL))G`c#&XEohY<>Q#ep=c+sz^p<2W zj8-k(Bf;YNqNGFcvj=rVEcqK9kIx{MRqTY&6ZtU$W8X;CCc0D4782Yxo9d=p5Ss0Y Udoo?LfhU$~MS=;`eD5Ec>zFRux&QzG diff --git a/ci/test/inl_ipsec_perf/pcap/enc_aes-cbc_sha1-hmac_380_1.pcap b/ci/test/inl_ipsec_perf/pcap/enc_aes-cbc_sha1-hmac_380_1.pcap new file mode 100644 index 0000000000000000000000000000000000000000..c3358018cba997677412154324a33eba9981728e GIT binary patch literal 478 zcmca|c+)~A1{MYw`2U}Qff2~z0b)&2Him6L4hS=d*(oi&2_ZQcTp1WwFt9T)I2aX6 zGBPqT0x=U%4M-hO6cwECFy}gx?Y(%z8=+Tv{-*bCEnWBS+(W5-$w>?Ng7c?mTgEds zMARQ$`KVyZytpFGi|Q+NQa()l!ek{Ay8CRxuTm+#xt8unf}{7Zb6xl*>HPjR4bwKM z^Xh&|%&GACXuW-Iz$Uja_ifesk*n8oh;c9VYjT#?npKJ;`Fbo=<$_ptoVyhh=^XcC68kE}ddoc1B$QNfgXaYdRJ)mQ4Ie3SZC+p6^=SFhy|<6i37^dhv5P1x+-;oWOb#O*0|tD3fNhPSO- zjJr{lNlCu`B|Y1S`481{cV1)Mb!*`UwIxeWpO9f#rXRH3?4HUoF0IcW89JHRJ}4xv zFS!!)A^GF%xMq%~6{ZPs&u9N}NoJ0@BD*(JHSoZa1t&AI`rX&7FLJn1`=NA|Q0CTe zQZG1qlErpCpZLf@Z^Nz+%lsDXSfd?XI>oZ=jK2E&h*;4Wdx3A7CqG`=c1XYT(XZMV z_H1G9D-T|EuYaGgFW0iiCGeiY`B^8sXYaeVh-Kl1XokQ5xhf;C!WZZ6uXJ%<_QQR# z)~tT3?Jw08S2ILw>y+2{u9A;+mgqOxx>EJS*S+2*##)uviWwaCX5Oj4)3M4TUzr~m z71#bS;)_LAP%IWoGXY}}2tlz3!az|}P^1)Rb#M04TT|HU`IOfORpozi5vXd|*ZUw* z;?Vz488@ejO^nlz=4PhLwM%`zy!hyYdxp1M`xl~JukM|{ z%As33Rq^hI(q+b+3q2=PZu@Jvi>-gkx-ibYAzEkt)t#|Z{=_xgd@1|YX}{+LT|a#D zWa1^&VBgxQn{M6AyR$OByT(#`imaJGZ`GzlzGhFj58u8t$vS7JU3y7CP}9Ta6BGYl z{qgJW!R=dq>B_nKR0{hvqZB??r_W9jo6sTyK z8`2T{H}|G#?3U@h^J?Xf&J{hoWc^bGM%CnvbFPWLx?X$!#V>7Jo<_-QZ_l5fwAXmw z?lvaVlUlp-?Po+vTS&$Cto50yxrFgkwY|$?u~kZOUNdgp%8$Eqp~bRfuEW6>hL`4@ P%HI$*BX!rQ{pTtHYyXs& literal 0 HcmV?d00001 diff --git a/ci/test/inl_ipsec_perf/pcap/enc_aes-cbc_sha1-hmac_380_4.pcap b/ci/test/inl_ipsec_perf/pcap/enc_aes-cbc_sha1-hmac_380_4.pcap new file mode 100644 index 0000000000000000000000000000000000000000..438b1a3403671661de3d618cf449b5cad5dd79af GIT binary patch literal 1840 zcmajfYg7_w8V7JdQ!{gEJd&wsiB}kPye4=}5i`phnihl+bsRNvQfLe_%SbeH&1+7K zCJ9bxrlHBE19)w9BJqNln0VQ?sL{Nn=1}Is?%5BWJ!kfL&;LE=dCvRocb-?;!()K} z+JOJo))oK&R5x1cj1qMKZR$o70L0IVHSwQY4-gCh-UsLa00FM&O+mnaACUTuChFN+_tkqZ(VHLbZOTqDtmZzyC4QG{&HFD z2V2o+?&s)^Rebt6dc0?1fH-1!#{j%N)xYwXnU`t8&Bnv!4s75`ra!tCQg1~#R4JFI5?&6`k(8={E~U$e znF1ZdPJZzs5k=Z%tfo+Q0{=`=4ISW`3e{38s-qFq&};Hl^)#qs8{iI13D>kp}D(9Szi2eOR$aO>M zDl|?P{nP9&3c9njlt05-by{Oae%GLXN8o&5KHHx1W_&GaB{O?U3T*GS-pq?cRUd&Q zu*POvCuNx9JqejUp;o+1N$+M^>#bMA*`6>P=wMhY>9y0T76TJ&~dI;hE~W$cfeq*26|jsi7zwd(_6Xc`E;>m9Z8|zQt+UTO@9Zb^~RYj1*@; zJ@;)bUyMX$VN9e?J(w+`j5z~fvM`xi4f&`as1xT&#>bo7eJ>El!5s?bOVby_c`zYC zXp`vs$@Y?|ut(%Vh(Q_@>F1m22B^FgmcDSu-Q|=+xnM41y?Qgd0?d}cKorEYvDI}- zLl;@#9qdqIa16z0BCUp1GwbvEsM@Sv)CS${sUF5(q<^9M^l57VwWdU+1dzRbBHHS1Q!q5+aspSQWWi4iT5C$*jd-h~GboU!I_>f-~) zpPTul;^aXna}3@HF@GMtZbnqw1SF=u;`h`4J6a;&5E& z-cff!>UJadt%OpOMciYyt5bWMlQylJ&xwLfyZ(J*UTg^teMgKqY<(n3n_d(mGpv`cGz0 z|KWv@?O@L>sJpau*rw9*)ob&Mg;H;^TbgIS>n|eD1nE<>JpWVlu`=P9wkmfLK z$Nl#7Qt>JXWggq{{G^I%YFLE~>BZti4})3rrzD~WS1NdRI~`YuXEP2jd-JOzd6=Ui zZJEx;8XEN=J2;^^%Gk>~MxKhEj>dk%ujv?Cg|ITr7lRLRrMXj5okyH}_OgLIbm5}PPe{1R!INw}t$bKm16;SKXMnxuu7 zLIp`SQJw6Xf$x)xLYnOE1-!^wqSk+t^myG=sdtkEX@65Z7vR*~zEl@p^S1{G9JLI2 zLF{a|7=OYT?+;#g7+4-@>vqncQ)S_GQ;c>?4CBO z83%}pw4k{jIMQoq6RqcS#OJyjYi&0a$f$n9Cv-c3DIvmIl4UpVv?B)v zs$q_-g}v^Y1lB~r_Tc3OZ_)1@0d;W zeXwax%Q~5p3)dxnHs>*15psRs&ABUV4!k|*_wRFW{kI*ne|7G7<#BR}2+-)Gku3OJ z#SC&)kuMX_RX_-G6$k@GQ9dV)S{cKaJmU!K8doh23 yz0AeHce}Ogj`J5@u$6nbUgW6u+UD?6O}|Zx&pXY3T4|D@uY0G2<2_e>NjCt6JZPZ+ literal 0 HcmV?d00001 diff --git a/ci/test/inl_ipsec_perf/pcap/enc_aes-cbc_sha1-hmac_64_4.pcap b/ci/test/inl_ipsec_perf/pcap/enc_aes-cbc_sha1-hmac_64_4.pcap new file mode 100644 index 0000000000000000000000000000000000000000..2b45271d33fdc731cf14d02cd201defa793a2f50 GIT binary patch literal 624 zcmca|c+)~A1{MYw`2U}Qff2}I1!5Uf7KSz;4Z;j!c1jCxLP!ncg8kGZJV5h`Lh!l@=mrO) z0zYP;tAH5fB4!{4(I_CkX!nX2OZSL}?7S`b>SeUrWE(YK=5unl?+e9A3D(Ry|FG7; zqjme4+JfnfHrW#KJkxr+j$(WV=1gIb1zNCuHC|V- zfx@c9ixKE5AO^Vx6!kz+R4_$*hY)k!`ww?Eu89-XI~I85y_fl#h7A*b`YhnR)PJ+` zj@d-t2b<=!tdlvpa9!eOa~{JLA=mfaoV&v2z}s_v|32qzxu^W=W2lbwjhM3lsF}Un literal 0 HcmV?d00001 diff --git a/ci/test/inl_ipsec_perf/pcap/enc_aes-gcm_1410.pcap b/ci/test/inl_ipsec_perf/pcap/enc_aes-gcm_1410_1.pcap similarity index 84% rename from ci/test/inl_ipsec_perf/pcap/enc_aes-gcm_1410.pcap rename to ci/test/inl_ipsec_perf/pcap/enc_aes-gcm_1410_1.pcap index 0c480a4c66d0ab5f71e59049a78d2a5e6ebddb68..90fde9c52ba114ae05673ab3c89c45f01d9410fd 100644 GIT binary patch delta 62 zcmaFF{fK*l1Unl81B3e)u8Hz8;y|W@QN9=>BNHPKGXW(SftYP$Py?&LY@=zf`_H_1 NZMMFiwM2{i4gi}R56J)k delta 62 zcmaFF{fK*l1UnN01B2)dzKQZO;y|W@QN9!-BNHPKgCxLsV^9OD!1bPh!c(qm9Hv+8 KH4T>OYy|*<%nq#p diff --git a/ci/test/inl_ipsec_perf/pcap/enc_aes-gcm_1410_2.pcap b/ci/test/inl_ipsec_perf/pcap/enc_aes-gcm_1410_2.pcap new file mode 100644 index 0000000000000000000000000000000000000000..d42647be94efcb801cb0de739bbdc8f5302f31b6 GIT binary patch literal 2988 zcmeIydpOg390%~<#PV2&QSz9O9Z+>-=^8^!`5Y&+~jg&-3~BB@~qNz<@mPPZkyc z0JQi4FP7*H2dW{!-vu=6(LB-j2?YoML3spwAiJCrNhPT_jtQnd(E?B zy+i2F@7?aq6edf2pn6D{OV?DdYHPp=9O_;+NOz7+s=Aq`gn0frF}7VaOtEoIlwwTV zcM3)1A$|$HxvxerNGN=XVpnp&DEQnFH%qy6<&n8r+IXF!DmG}L_v05ZG|HNwWBHQ9 zP{Y>Rm)a)9Vtv!(<1F`6N8O;BDs=H`9ynTp=1;$BxH<>>EZj>B?Ib>}av$J#Z42EN`*%;k-ioni!Y z2r{U_V1f>x#Ti{GuU%2CZ)mT%+eMmfQ=+nAF!hVrSBt5mHFhTErFe_?fk01me~u`} z2{ElWB@9P3ms$DaClh_$1GQcX25<;>tgLMQUj6_#V_ew`TwlR6f>VvqLFo>Djd z_G4&d1%!2lOW$+8lKAv?MR)JI^Q>TIk8=75FYvwE+S-pX&%kE!k6G;8w&y+p^**mF zk1EO`sn+F9cM*|o!UM|uBb@vh7{N1mvQkxylN#uXB=!F$J*dpAVLKz~+lca-eElkd- z6IErqP;~|ui_AqUHzGIT`?IfvRjNFsTo)I~#8dsXkhj-TS543FIxt@AvT$9GnWrJA zmC^#A#5lM`ITrpwAHW^o=buS|-&wPNOGt4Y`J^e5Hf0qgdTc?$7n|x;o+U_v(k=J) z^p<{nl~=IBM7;iVd6=5=!>;%kBl5|}5>l%I-Drk5GVO01b*|bUc1$%PJ&C5jgJ+NT zkD}^29mE8bL^R)oG5Vn0h4!q&ozZFPeQjN|Ed98u}~H*;yt&6ug@8fRE&KPlrk#SrKXh99r)mEpDNRMVYVmmMw3WPt|v z%??HkHyY!vmu6K-Z*DER;BGy>&ByCVQRC|dbWH)|gKhwtaOnALt|nrhrNSCAnLPE& z!%NA7rbqM`TNN9p>T5I(nRb~ydVxB-OTqGS2bNcsty_((3~_2{?449TpYq_LkOI|Q zrO%YJ8ZYtV(p#QX`M?A4ZLL!y%Cc*%p#hr{ntVgJ;{$m*+p!UY35Q6`Y#2 zaXJb~^{u+2ajZY{loAnuzDIR1g_unm#+5dmx{uF=nk2~0HAX}`KHvm7#tnw_NnY39 zAdHiSG<6$#;CDqCcVL>X`m@r|Xu5K`@R=X3D$bI1t4GTx5@Q*xVXT8VjuyV%+C96s zxbN~C_zdz!D(+}5o=G|v@xe4vy8C^Xs4{>rTgjl(Hxv?<1y_06WQ^s@L2 zL3dAn+(lx>HYXRKPdSkr7yN9RZ<+ZTgz?Jr_Ky#7_GR%s`%%T*6G(w7U3Srv`}>?S z#nz7I!Au^5;LKlK>y!?0W;0(H=91L;W7Q=du_R2qyTTB)hz)%HHHv^Z3D9n^JTHRt zyG_)hmGav)O(Yy!HW}|nWgQ5gF;a52t1E0#aiA4vn>%>Lv9Su@p3zbgxq3$F`8ALx z1KcEHA41l8#{Jz^xuD0MNcZGH?+gV`D(t$XV2$&Jr1}7o^FRW8*K`(g!rJ>-dk2dx z5(^Bsk8Y@3%}4Xj9go`1zD9np?5vd4r76m+Pj_p3M@eIPcpQhf-3^x+w;0_~@090m zk?p)RdyMS$rc>$^O!bJ?&Ww5zI<5C%gNqw19E;1_CktHllt{FZ$JiB+85Q+PiyYti z0{$<_S1d_>NCUhm`QlrWe5s4u@A;SHQn0jjx-#E3mGLYClmv#5;P(*0=xB%QfB9yuUy@Yv1Ma$HLq(GQrxzqH%8MX%2PwE*#Y) zeKusvG5yJ7Rc`!4=Zp|F5Y{a&^VpSI*YEGu3=HqO!V1Y6RLh*^1-{YR+4L^<83>o~ zgvBoCe(n>{;`6HZ>^3lhW>eixDQJz$^EWqbM5&lD<0xrWu5U-}zRlnN-`=)+$!Wgz zR`b$GEtu{6884;8SwA9u-)ol-BDSzb68(!*^UDUH{S<>7BTPrm|33D)zRPT3ow0yc z0rkXAMMYmsMUB(Es>PVB#UEBkRR`)Z(mwF za&oT8dA-Cit5655o7M?kFmZ5;aV-9sIgULa?4Lu0-q$;EAhe>Ha?uh&U)&oceqv1~ zR9G6;enT1!%ETWZ9Ikrzvhd~>3(4-Usw1@29``52no%xBSCYGwnP$tb(@Xy5F_#-f+YVWP#{n_mDz{ErcCh@ayUpBZK3-?a+g`OAHQt20MFtp=PCZ{K zfWy{U8mvi+g-c&NzMe8+dB!mH;I_8KmPVaZmi@TzUZB1?qKtpii{T0KkPV31P-17> z@PgWvv`3FcR2BGk<1Fyrxk^7Sv-4@44>W+#-L*KaCcD!W9`I>myKg9WZoJUo5GHD3 zPHF6Ht%;;*3#V~syn%9ZOBZq`p7m>?Y7z`!?9mWHb61=!R&rjL7Lc|1vkjFVab%MOcjZZ1IUDf) z%NP>uB7k1X^1KEr8nDn}s226WEk-%EYzo1T#yT0fY^LgD*IeAG;Xtp*vvTl?XJeFq ze@0JB;u@M|7BxcJ(bxr8Fih6$q?IxkB=fFL^TV zYW}`3vt8t9f6*KC5K3E)@(&hU{$Ecn@75ixPQgKP$>n21@{P!M|EmA#$*VUd-?}OJ z4u#|j$rX|-Bv(lO@g*;#l(eSSqd@~`N=W{pm<|0;Pj94L2Oo(;t}zGePN r@`sy}t87ZHsgPVDxk7S<J>qL1OaYhCP2cu$TMn)z^AZ7wefH336pbAETs0UO3aQNC^ Nz27Tr-M2jFGXQoP4@Lk0 diff --git a/ci/test/inl_ipsec_perf/pcap/enc_aes-gcm_380_2.pcap b/ci/test/inl_ipsec_perf/pcap/enc_aes-gcm_380_2.pcap new file mode 100644 index 0000000000000000000000000000000000000000..3e1ed56b28052907abcbd3b30e579850f79a1f3c GIT binary patch literal 924 zcmca|c+)~A1{MYw`2U}Qff2}I2jUwc+zgw591vy@vr}4l6GCz@xH2#F*Cp2!CU2X;q`>#H z%+rLqi|6Uj?-_6RaS5p>{J;L@HxoyJxvPxHwHd7<>U-_hTQ`@ehjp@)nk1z^^W+fc zue&R}kg4dN($cRt4lMr3`}s)A;mc}-Y>4vPcSnoM~ zD__F6zMMa+GyascQp>A%`=-oPQMWzB`0&rls_6@#ZD?#wnc@16L-kRm%+mfD&p276 zH}6u`uoI7Q`8d^TEx*>Ij^B5~B)VlJUbPipyK57i93P4`6b(bwI$ou)#Y#VpUGKTnt6E8FGi(2% zS#_~fwt1XhxB1d_#VP$M-IsZ~o-T}kDI&l3P4QVK!}8<(uV1g-F;~x3iu=?_#n^W( z77O+W0Ha~$3NC^%Sg6PZj6om-#UKb{i@_bx803V;;L6b$q<##(O!~IsYyVH~i}!`5 Jw`aN}0RVatkc$8S literal 0 HcmV?d00001 diff --git a/ci/test/inl_ipsec_perf/pcap/enc_aes-gcm_380_4.pcap b/ci/test/inl_ipsec_perf/pcap/enc_aes-gcm_380_4.pcap new file mode 100644 index 0000000000000000000000000000000000000000..48968b1ff1efa2b53ef5fe2f03e81dc4648275b0 GIT binary patch literal 1824 zcmca|c+)~A1{MYw`2U}Qff2|7iBz(1Gi(BKK$tVr$$r~c zc5(|`_*_+T=H`=7EBC68Y6fR~uiOlJymW2d(c60tKXP4iO=0r38B7X%Kg&E#n7eqM z{`{Wtb|06JdcyzfZ+^jO)w! zvpVBXNh`Iydbe-NOciz8LyQmqtgM>8@Y#mO)|45p|2R}1Rmv>wpYe>7MSAltWeq#= z7?+Pzt=95uJ?i*R7KG+RITGx3R|r7sJ7t^4 z>2;egU00mapVEDqr|ap$_?IH`d*2kFWil*3-v9cwkK98U%>`VR&pS;Tn$M}v;{`^> z!B>0)W3X6(5g3C&42nHi)B~9e3`jA!8ybTg&=_1X8iUl2L5?F`vu2w9`0>BI`e(!Z z!w&MmI9R}gHxB~i%)uyMi5VD!Kn#jOW*`R9V1N{ZJE1Yi2aUmPqcKSR7*t;T!!T>@ z&aVwQ*_VsbuYS-0#)57-qy)uLih^S>SBV7}gFp<5IZz3TEe5wiV~`&jgWE@Akoqxr Udq!92$xg#(`pw46vz61d0Ym#58UO$Q literal 0 HcmV?d00001 diff --git a/ci/test/inl_ipsec_perf/pcap/enc_aes-gcm_64.pcap b/ci/test/inl_ipsec_perf/pcap/enc_aes-gcm_64.pcap deleted file mode 100644 index 0affa2601b5f882d2386489c546d83f637fc30af..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 158 zcmca|c+)~A1{MYw`2U}Qff2}I1!CQ1c7`$_4Z;j!c1jCxLP!nro1NpNc2)$i-hPq@-@FN8x*ir3{zm-yY2QT3V%TcTuyrb< m?9a)5+gEmS3taeIRdQzGj8!hPES^t$*L}NAe%bYKffWEN0xz5Z literal 0 HcmV?d00001 diff --git a/ci/test/inl_ipsec_perf/pcap/enc_aes-gcm_64_2.pcap b/ci/test/inl_ipsec_perf/pcap/enc_aes-gcm_64_2.pcap new file mode 100644 index 0000000000000000000000000000000000000000..fb292e4c47360b4686df3570dd7ac5b976b47c2b GIT binary patch literal 292 zcmca|c+)~A1{MYw`2U}Qff2~D17gb;j0|N!8iX0d?35PXgpeEzt_%zr3~USx4n`$W zjEqc-K+FVG1JVc-g@9mpBk^C8OM+ADu6|#4e!`WOdm$WhQoJr#y2S6EjH=gE*gBO_ z_UB~3?JGOE1ulH9Dmk-o#umMGb%z%oPdk?2X#2F!nn#P1f#JS7FFscQ?Q}3Iif014 r0ti8_fVl+h3?x^4fx3bd>WW*qTv6t_^kMJ+H@tloSG4&X4`%}aH$GSs literal 0 HcmV?d00001 diff --git a/ci/test/inl_ipsec_perf/pcap/enc_aes-gcm_64_4.pcap b/ci/test/inl_ipsec_perf/pcap/enc_aes-gcm_64_4.pcap new file mode 100644 index 0000000000000000000000000000000000000000..d2bc6828771c8c483895200f1ffcfb2a8b971dfc GIT binary patch literal 560 zcmca|c+)~A1{MYw`2U}Qff2~z0^<4WIT*@-Gzc?@*(oi&2_ZQcTp1WL7?>Cs9E^(M zm>9tj$ODOhF-Sc_u)C4?ugN9BsdZPstUEv9O3S?vPB|%Fmn&W3w@yaYYbtD=$|(DD zvfuWVo!kN!K3A2TSvVs@f8z3$CyqXttk$)`HsZy3DWExjxft=e0_a!=qmp<=peukF zT=5<13J$0%?&5NV&7G?T)9)5P{~mU-X<~P&x(U#v`37tRT~QFv40Ht$ zgIvK3#MoT%73vB;s4H&ca>czTEOFm;ZcLjgrp_z)*7}44(7>Fhkl4TxBjB*ei)R74 p0*FDb07Wb|SA2rHf* -aes-gcm -64: 5101840 -380: 5120769 -1410: 4194633 - diff --git a/ci/test/inl_ipsec_perf/ref_numbers/cn10k/rclk2500_sclk1000.103xx.ip.outb b/ci/test/inl_ipsec_perf/ref_numbers/cn10k/rclk2500_sclk1000.103xx.ip.outb deleted file mode 100644 index f26e338455..0000000000 --- a/ci/test/inl_ipsec_perf/ref_numbers/cn10k/rclk2500_sclk1000.103xx.ip.outb +++ /dev/null @@ -1,10 +0,0 @@ -aes-cbc_sha1-hmac -64: 5186596 -380: 5165938 -1410: 4161094 - -aes-gcm -64: 5164575 -380: 5165459 -1410: 4194624 - diff --git a/ci/test/inl_ipsec_perf/ref_numbers/cn10k/rclk2500_sclk1000.106xx.ip.inb b/ci/test/inl_ipsec_perf/ref_numbers/cn10k/rclk2500_sclk1000.106xx.ip.inb index f679676176..b90c42ec7e 100644 --- a/ci/test/inl_ipsec_perf/ref_numbers/cn10k/rclk2500_sclk1000.106xx.ip.inb +++ b/ci/test/inl_ipsec_perf/ref_numbers/cn10k/rclk2500_sclk1000.106xx.ip.inb @@ -1,10 +1,10 @@ aes-cbc_sha1-hmac -64: 8187281 -380: 8177745 -1410: 8196843 +64: 8157035 17676380 34471600 +380: 8155538 17645993 21956102 +1410: 8148850 8322770 8322761 aes-gcm -64: 8176186 -380: 8173173 -1410: 8141213 +64: 8198632 17730833 36100815 +380: 8200873 17572093 21955849 +1410: 8172678 8389795 8389801 diff --git a/ci/test/inl_ipsec_perf/ref_numbers/cn10k/rclk2500_sclk1000.106xx.ip.outb b/ci/test/inl_ipsec_perf/ref_numbers/cn10k/rclk2500_sclk1000.106xx.ip.outb index 7dd8275b95..6df2ae9ada 100644 --- a/ci/test/inl_ipsec_perf/ref_numbers/cn10k/rclk2500_sclk1000.106xx.ip.outb +++ b/ci/test/inl_ipsec_perf/ref_numbers/cn10k/rclk2500_sclk1000.106xx.ip.outb @@ -1,10 +1,10 @@ aes-cbc_sha1-hmac -64: 7257102 -380: 7238439 -1410: 7256166 +64: 7221905 15248204 31554795 +380: 7220807 15195168 27058018 +1410: 7237801 8322772 8322771 aes-gcm -64: 7253390 -380: 7234528 -1410: 7260576 +64: 7203667 15349844 31428190 +380: 7200141 15354581 27534812 +1410: 7228652 8412384 8412385 diff --git a/ci/test/inl_ipsec_perf/ref_numbers/cn10k/rclk2500_sclk1100.106xx.ip.inb b/ci/test/inl_ipsec_perf/ref_numbers/cn10k/rclk2500_sclk1100.106xx.ip.inb index f679676176..b90c42ec7e 100644 --- a/ci/test/inl_ipsec_perf/ref_numbers/cn10k/rclk2500_sclk1100.106xx.ip.inb +++ b/ci/test/inl_ipsec_perf/ref_numbers/cn10k/rclk2500_sclk1100.106xx.ip.inb @@ -1,10 +1,10 @@ aes-cbc_sha1-hmac -64: 8187281 -380: 8177745 -1410: 8196843 +64: 8157035 17676380 34471600 +380: 8155538 17645993 21956102 +1410: 8148850 8322770 8322761 aes-gcm -64: 8176186 -380: 8173173 -1410: 8141213 +64: 8198632 17730833 36100815 +380: 8200873 17572093 21955849 +1410: 8172678 8389795 8389801 diff --git a/ci/test/inl_ipsec_perf/ref_numbers/cn10k/rclk2500_sclk1100.106xx.ip.outb b/ci/test/inl_ipsec_perf/ref_numbers/cn10k/rclk2500_sclk1100.106xx.ip.outb index 7dd8275b95..6df2ae9ada 100644 --- a/ci/test/inl_ipsec_perf/ref_numbers/cn10k/rclk2500_sclk1100.106xx.ip.outb +++ b/ci/test/inl_ipsec_perf/ref_numbers/cn10k/rclk2500_sclk1100.106xx.ip.outb @@ -1,10 +1,10 @@ aes-cbc_sha1-hmac -64: 7257102 -380: 7238439 -1410: 7256166 +64: 7221905 15248204 31554795 +380: 7220807 15195168 27058018 +1410: 7237801 8322772 8322771 aes-gcm -64: 7253390 -380: 7234528 -1410: 7260576 +64: 7203667 15349844 31428190 +380: 7200141 15354581 27534812 +1410: 7228652 8412384 8412385 From a6847fa8e1589aaa81b9dd2c54f78b00dac376cf Mon Sep 17 00:00:00 2001 From: Alok Mishra Date: Wed, 20 Aug 2025 11:22:49 +0000 Subject: [PATCH 258/271] octeon: enable tranport mode ipsec support Type: feature Signed-off-by: Alok Mishra Change-Id: Iaa1c891b1cdd019741567ca8e39a24a70eef8750 Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/144544 Reviewed-by: Monendra Singh Kushwaha Reviewed-by: Nithin Kumar Dabilpuram Tested-by: sa_ip-toolkits-Jenkins (cherry picked from commit 4868f5ecc3d1f8be78772789751ebf0393a6a020) Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/161648 Tested-by: Monendra Singh Kushwaha --- src/plugins/dev_octeon/ipsec.c | 20 +++++++++++++------- src/plugins/dev_octeon/ipsec.h | 1 + src/plugins/dev_octeon/tx_node.c | 22 ++++++++-------------- 3 files changed, 22 insertions(+), 21 deletions(-) diff --git a/src/plugins/dev_octeon/ipsec.c b/src/plugins/dev_octeon/ipsec.c index 455ef9beab..a8864aa7d8 100644 --- a/src/plugins/dev_octeon/ipsec.c +++ b/src/plugins/dev_octeon/ipsec.c @@ -399,10 +399,20 @@ oct_o20_ipsec_sa_common_param_fill (union roc_ow_ipsec_sa_word2 *w2, static_always_inline void oct_ipsec_sa_len_precalc (ipsec_sa_t *sa, oct_ipsec_encap_len_t *encap) { - if (ipsec_sa_is_set_IS_TUNNEL_V6 (sa)) - encap->partial_len = ROC_CPT_TUNNEL_IPV6_HDR_LEN; + encap->adj_len = 0; + + if (ipsec_sa_is_set_IS_TUNNEL (sa)) + { + if (ipsec_sa_is_set_IS_TUNNEL_V6 (sa)) + encap->partial_len = ROC_CPT_TUNNEL_IPV6_HDR_LEN; + else + encap->partial_len = ROC_CPT_TUNNEL_IPV4_HDR_LEN; + } else - encap->partial_len = ROC_CPT_TUNNEL_IPV4_HDR_LEN; + { + encap->partial_len = 0; + encap->adj_len = ROC_CPT_TUNNEL_IPV4_HDR_LEN; + } if (sa->protocol == IPSEC_PROTOCOL_ESP) { @@ -1202,10 +1212,6 @@ oct_ipsec_check_support (ipsec_sa_t *sa) u8 is_cipher_algo_supported; u8 is_auth_algo_supported; - if (!ipsec_sa_is_set_IS_TUNNEL (sa)) - return clib_error_create ( - "Transport mode SA is not supported in Inline IPsec operation"); - switch (sa->crypto_alg) { case IPSEC_CRYPTO_ALG_NONE: diff --git a/src/plugins/dev_octeon/ipsec.h b/src/plugins/dev_octeon/ipsec.h index b03d5c914b..18402cf3f3 100644 --- a/src/plugins/dev_octeon/ipsec.h +++ b/src/plugins/dev_octeon/ipsec.h @@ -101,6 +101,7 @@ typedef struct uint8_t footer_len; uint8_t roundup_byte; uint8_t icv_len; + uint8_t adj_len; } oct_ipsec_encap_len_t; typedef struct diff --git a/src/plugins/dev_octeon/tx_node.c b/src/plugins/dev_octeon/tx_node.c index 86b2a298e6..f2846e82e7 100644 --- a/src/plugins/dev_octeon/tx_node.c +++ b/src/plugins/dev_octeon/tx_node.c @@ -749,10 +749,10 @@ oct_ipsec_rlen_get (oct_ipsec_encap_len_t *encap, uint32_t plen) { uint32_t enc_payload_len; - enc_payload_len = - round_pow2 (plen + encap->roundup_len, encap->roundup_byte); + enc_payload_len = round_pow2 (plen + encap->roundup_len - encap->adj_len, + encap->roundup_byte); - return encap->partial_len + enc_payload_len; + return encap->partial_len + enc_payload_len + encap->adj_len; } static_always_inline u32 @@ -1313,7 +1313,6 @@ oct_pkts_send_ipsec (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_buffer_t **b; u64 *lmt_line[4]; u64 n_dwords[4]; - oct_device_t *od; b = bufs; @@ -1368,8 +1367,7 @@ oct_pkts_send_ipsec (vlib_main_t *vm, vlib_node_runtime_t *node, sess0 = pool_elt_at_index (im->inline_ipsec_sessions, sa0_index); if (!sess0->inst.w7.s.cptr) { - od = oct_ipsec_get_oct_device_from_outb_sa (sa0_index); - sess0->inst.w7.s.cptr = (u64) sess0->out_sa[od->nix_idx]; + sess0->inst.w7.s.cptr = (u64) sess0->out_sa[cd->nix_idx]; sess0->sq = ((sa0_index % vlib_num_workers ()) + 1) % num_tx_queues; } @@ -1384,10 +1382,9 @@ oct_pkts_send_ipsec (vlib_main_t *vm, vlib_node_runtime_t *node, sess1 = pool_elt_at_index (im->inline_ipsec_sessions, sa1_index); if (!sess1->inst.w7.s.cptr) { - od = oct_ipsec_get_oct_device_from_outb_sa (sa1_index); - sess1->inst.w7.s.cptr = (u64) sess1->out_sa[od->nix_idx]; sess1->sq = ((sa1_index % vlib_num_workers ()) + 1) % num_tx_queues; + sess1->inst.w7.s.cptr = (u64) sess1->out_sa[cd->nix_idx]; } current_sa1_index = sa1_index; ALWAYS_ASSERT (current_sa0_index < @@ -1400,10 +1397,9 @@ oct_pkts_send_ipsec (vlib_main_t *vm, vlib_node_runtime_t *node, sess2 = pool_elt_at_index (im->inline_ipsec_sessions, sa2_index); if (!sess2->inst.w7.s.cptr) { - od = oct_ipsec_get_oct_device_from_outb_sa (sa2_index); - sess2->inst.w7.s.cptr = (u64) sess2->out_sa[od->nix_idx]; sess2->sq = ((sa2_index % vlib_num_workers ()) + 1) % num_tx_queues; + sess2->inst.w7.s.cptr = (u64) sess2->out_sa[cd->nix_idx]; } current_sa2_index = sa2_index; ALWAYS_ASSERT (current_sa2_index < @@ -1416,10 +1412,9 @@ oct_pkts_send_ipsec (vlib_main_t *vm, vlib_node_runtime_t *node, sess3 = pool_elt_at_index (im->inline_ipsec_sessions, sa3_index); if (!sess3->inst.w7.s.cptr) { - od = oct_ipsec_get_oct_device_from_outb_sa (sa3_index); - sess3->inst.w7.s.cptr = (u64) sess3->out_sa[od->nix_idx]; sess3->sq = ((sa3_index % vlib_num_workers ()) + 1) % num_tx_queues; + sess3->inst.w7.s.cptr = (u64) sess3->out_sa[cd->nix_idx]; } current_sa3_index = sa3_index; ALWAYS_ASSERT (current_sa3_index < @@ -1620,10 +1615,9 @@ oct_pkts_send_ipsec (vlib_main_t *vm, vlib_node_runtime_t *node, sess0 = pool_elt_at_index (im->inline_ipsec_sessions, sa0_index); if (!sess0->inst.w7.s.cptr) { - od = oct_ipsec_get_oct_device_from_outb_sa (sa0_index); - sess0->inst.w7.s.cptr = (u64) sess0->out_sa[od->nix_idx]; sess0->sq = ((sa0_index % vlib_num_workers ()) + 1) % num_tx_queues; + sess0->inst.w7.s.cptr = (u64) sess0->out_sa[cd->nix_idx]; } current_sa0_index = sa0_index; ALWAYS_ASSERT (current_sa0_index < From 7c86331cd835ee37cc90710c6ce0ed2836b36b9f Mon Sep 17 00:00:00 2001 From: Monendra Singh Kushwaha Date: Thu, 21 Aug 2025 06:14:52 +0000 Subject: [PATCH 259/271] octeon: add L4 checksum flags This patch adds L4 checksum related flags in vlib buffer. Type: feature Signed-off-by: Monendra Singh Kushwaha Change-Id: I8e52587ffdb9b25c13613f3c4b56abe1cfd39076 Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/160937 Tested-by: sa_ip-toolkits-Jenkins Reviewed-by: Nithin Kumar Dabilpuram (cherry picked from commit 8161a10d7ed0d58ce02048745012a4eef158c6de) Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/161650 Tested-by: sa_ip-sw-jenkins --- src/plugins/dev_octeon/rx_node.c | 65 +++++++++++++++++++++++++++++--- 1 file changed, 60 insertions(+), 5 deletions(-) diff --git a/src/plugins/dev_octeon/rx_node.c b/src/plugins/dev_octeon/rx_node.c index 10a872b18f..a01ac75d90 100644 --- a/src/plugins/dev_octeon/rx_node.c +++ b/src/plugins/dev_octeon/rx_node.c @@ -892,6 +892,7 @@ oct_rx_inl_ipsec_vlib_from_cq ( union nix_rx_parse_u *orig_rxp, *rxp; u32 is_fail, olen, esp_sz, l2_ol3_sz; u64 *wqe_ptr; + u32 err_flag; u8 frag_cnt; rxp = &d->parse.f; @@ -905,7 +906,13 @@ oct_rx_inl_ipsec_vlib_from_cq ( esp_sz = olen - l2_ol3_sz; b[0]->template = *bt; b[0]->flow_id = d[0].parse.w[3] >> 48; - *err_flags |= ((d[0].parse.w[0] >> 20) & 0xFFF); + err_flag = ((d[0].parse.w[0] >> 20) & 0xFFF); + if (PREDICT_FALSE (err_flag)) + { + b[0]->flags &= ~(VNET_BUFFER_F_L4_CHECKSUM_CORRECT | + VNET_BUFFER_F_L4_CHECKSUM_COMPUTED); + *err_flags |= err_flag; + } is_fail = !oct_ipsec_is_inl_op_success (cpt_hdr, fp_flags); @@ -947,11 +954,20 @@ oct_rx_vlib_from_cq (vlib_main_t *vm, oct_nix_rx_cqe_desc_t *d, u32 *err_flags, u16 *next, u16 *buffer_next_index, const u64 fp_flags) { + u32 err_flag; + b[0] = (vlib_buffer_t *) d->segs0[0] - 1; b[0]->template = *bt; ctx->n_rx_bytes += b[0]->current_length = d[0].sg0.seg1_size; b[0]->flow_id = d[0].parse.w[3] >> 48; - *err_flags |= ((d[0].parse.w[0] >> 20) & 0xFFF); + err_flag = ((d[0].parse.w[0] >> 20) & 0xFFF); + if (PREDICT_FALSE (err_flag)) + { + b[0]->flags &= ~(VNET_BUFFER_F_L4_CHECKSUM_CORRECT | + VNET_BUFFER_F_L4_CHECKSUM_COMPUTED); + *err_flags |= err_flag; + } + ctx->n_segs += 1; if (d[0].sg0.segs > 1) oct_rx_attach_tail (vm, ctx, bt, b[0], d + 0); @@ -1003,7 +1019,7 @@ oct_rx_batch (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_buffer_template_t bt = rxq->buffer_template; u32 b0_err_flags = 0, b1_err_flags = 0; u32 b2_err_flags = 0, b3_err_flags = 0; - u32 n_left, err_flags = 0; + u32 n_left, err_flags = 0, err_flags_x4 = 0; oct_nix_rx_cqe_desc_t *d = ctx->next_desc; union cpt_parse_hdr_u *cpt_hdr0, *cpt_hdr1; union cpt_parse_hdr_u *cpt_hdr2, *cpt_hdr3; @@ -1037,6 +1053,9 @@ oct_rx_batch (vlib_main_t *vm, vlib_node_runtime_t *node, laddr = lbase; laddr += 8; + bt.flags |= + (VNET_BUFFER_F_L4_CHECKSUM_COMPUTED | VNET_BUFFER_F_L4_CHECKSUM_CORRECT); + for (n_left = n; n_left >= 8; d += 4, n_left -= 4) { u32 segs = 0; @@ -1088,9 +1107,27 @@ oct_rx_batch (vlib_main_t *vm, vlib_node_runtime_t *node, b2_err_flags = (d[2].parse.w[0] >> 20) & 0xFFF; b3_err_flags = (d[3].parse.w[0] >> 20) & 0xFFF; - err_flags |= + err_flags_x4 = b0_err_flags | b1_err_flags | b2_err_flags | b3_err_flags; + if (PREDICT_FALSE (err_flags_x4)) + { + err_flags |= err_flags_x4; + + if (b0_err_flags) + b[0]->flags &= ~(VNET_BUFFER_F_L4_CHECKSUM_CORRECT | + VNET_BUFFER_F_L4_CHECKSUM_COMPUTED); + if (b1_err_flags) + b[1]->flags &= ~(VNET_BUFFER_F_L4_CHECKSUM_CORRECT | + VNET_BUFFER_F_L4_CHECKSUM_COMPUTED); + if (b2_err_flags) + b[2]->flags &= ~(VNET_BUFFER_F_L4_CHECKSUM_CORRECT | + VNET_BUFFER_F_L4_CHECKSUM_COMPUTED); + if (b3_err_flags) + b[3]->flags &= ~(VNET_BUFFER_F_L4_CHECKSUM_CORRECT | + VNET_BUFFER_F_L4_CHECKSUM_COMPUTED); + } + if (fp_flags & OCT_FP_FLAG_TRACE_EN) { clib_memcpy_fast (b[0]->pre_data, &d[0], cqe_desc_bytes); @@ -1328,9 +1365,27 @@ oct_rx_batch (vlib_main_t *vm, vlib_node_runtime_t *node, b2_err_flags = (d[2].parse.w[0] >> 20) & 0xFFF; b3_err_flags = (d[3].parse.w[0] >> 20) & 0xFFF; - err_flags |= + err_flags_x4 = b0_err_flags | b1_err_flags | b2_err_flags | b3_err_flags; + if (PREDICT_FALSE (err_flags_x4)) + { + err_flags |= err_flags_x4; + + if (b0_err_flags) + b[0]->flags &= ~(VNET_BUFFER_F_L4_CHECKSUM_CORRECT | + VNET_BUFFER_F_L4_CHECKSUM_COMPUTED); + if (b1_err_flags) + b[1]->flags &= ~(VNET_BUFFER_F_L4_CHECKSUM_CORRECT | + VNET_BUFFER_F_L4_CHECKSUM_COMPUTED); + if (b2_err_flags) + b[2]->flags &= ~(VNET_BUFFER_F_L4_CHECKSUM_CORRECT | + VNET_BUFFER_F_L4_CHECKSUM_COMPUTED); + if (b3_err_flags) + b[3]->flags &= ~(VNET_BUFFER_F_L4_CHECKSUM_CORRECT | + VNET_BUFFER_F_L4_CHECKSUM_COMPUTED); + } + OCT_PUSH_META_TO_FREE ((u64) cpt_hdr0, laddr, &loff); OCT_PUSH_META_TO_FREE ((u64) cpt_hdr1, laddr, &loff); OCT_PUSH_META_TO_FREE ((u64) cpt_hdr2, laddr, &loff); From 8943b4a1bb9ff8290117a3175a147e7cdbf023b3 Mon Sep 17 00:00:00 2001 From: Monendra Singh Kushwaha Date: Mon, 1 Sep 2025 12:00:48 +0000 Subject: [PATCH 260/271] build: update roc version Type: feature Signed-off-by: Monendra Singh Kushwaha Change-Id: I7b63b52d6f533e3d9fc578cb7657e49bbf50d580 Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/161707 Tested-by: sa_ip-sw-jenkins --- build/external/packages/octeon-roc.mk | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build/external/packages/octeon-roc.mk b/build/external/packages/octeon-roc.mk index 8054e22c7e..c5b8752388 100644 --- a/build/external/packages/octeon-roc.mk +++ b/build/external/packages/octeon-roc.mk @@ -2,9 +2,9 @@ # SPDX-License-Identifier: Apache-2.0 # https://spdx.org/licenses/Apache-2.0.html -octeon-roc_version := 25.08 +octeon-roc_version := 25.09 octeon-roc_tarball := v$(octeon-roc_version).tar.gz -octeon-roc_tarball_md5sum := c6dd2e885c332e32f64ddc819d05bd90 +octeon-roc_tarball_md5sum := 593be96859fe9d8660f87d3bcdb1b008 octeon-roc_github := https://github.com/MarvellEmbeddedProcessors/marvell-octeon-roc octeon-roc_tarball_strip_dirs := 1 From aa2698c5c17c17ca0af37c0328f6d733ffa82839 Mon Sep 17 00:00:00 2001 From: Damjan Marion Date: Tue, 8 Oct 2024 23:14:49 +0200 Subject: [PATCH 261/271] dev: add helper functions Type: improvement Change-Id: I7c9e882b1cdf141b34e84dbfed46b392624d1f62 Signed-off-by: Damjan Marion Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/161759 Reviewed-by: Monendra Singh Kushwaha Tested-by: sa_ip-sw-jenkins --- src/plugins/dev_ena/rx_node.c | 9 ++++---- src/plugins/dev_iavf/port.c | 2 +- src/plugins/dev_iavf/rx_node.c | 8 +++---- src/vnet/dev/dev_funcs.h | 40 +++++++++++++++++++++++++++------- src/vnet/dev/port.c | 2 +- 5 files changed, 42 insertions(+), 19 deletions(-) diff --git a/src/plugins/dev_ena/rx_node.c b/src/plugins/dev_ena/rx_node.c index 41fc5b8c94..51c6dbce84 100644 --- a/src/plugins/dev_ena/rx_node.c +++ b/src/plugins/dev_ena/rx_node.c @@ -251,7 +251,6 @@ ena_device_input_inline (vlib_main_t *vm, vlib_node_runtime_t *node, vnet_dev_rx_queue_t *rxq) { ena_rxq_t *q = vnet_dev_get_rx_queue_data (rxq); - vnet_dev_port_t *port = rxq->port; vnet_main_t *vnm = vnet_get_main (); vlib_buffer_t *buffers[VLIB_FRAME_SIZE], **b; ena_rx_cdesc_status_t statuses[VLIB_FRAME_SIZE + 8]; @@ -260,13 +259,13 @@ ena_device_input_inline (vlib_main_t *vm, vlib_node_runtime_t *node, u16 *csi; uword n_rx_packets = 0, n_rx_bytes = 0; vlib_frame_bitmap_t head_bmp = {}; - u32 sw_if_index = port->intf.sw_if_index; - u32 hw_if_index = port->intf.hw_if_index; + u32 sw_if_index = vnet_dev_get_rx_queue_if_sw_if_index (rxq); + u32 hw_if_index = vnet_dev_get_rx_queue_if_hw_if_index (rxq); u32 n_trace, n_deq, n_left; u32 cq_next = q->cq_next; - u32 next_index = rxq->next_index; + u32 next_index = vnet_dev_get_rx_queue_if_next_index (rxq); vlib_frame_t *next_frame; - vlib_buffer_template_t bt = rxq->buffer_template; + vlib_buffer_template_t bt = vnet_dev_get_rx_queue_if_buffer_template (rxq); u32 *bi; int maybe_chained; diff --git a/src/plugins/dev_iavf/port.c b/src/plugins/dev_iavf/port.c index 982436d9b4..033685722b 100644 --- a/src/plugins/dev_iavf/port.c +++ b/src/plugins/dev_iavf/port.c @@ -257,7 +257,7 @@ avf_msix_n_handler (vlib_main_t *vm, vnet_dev_t *dev, u16 line) iavf_reg_write (ad, IAVF_VFINT_DYN_CTLN (line), dyn_ctln_enabled.as_u32); vlib_node_set_interrupt_pending (vlib_get_main_by_index (line), - port->intf.rx_node_index); + vnet_dev_get_port_rx_node_idex (port)); } vnet_dev_rv_t diff --git a/src/plugins/dev_iavf/rx_node.c b/src/plugins/dev_iavf/rx_node.c index ee6d7e8def..bf650f9bfb 100644 --- a/src/plugins/dev_iavf/rx_node.c +++ b/src/plugins/dev_iavf/rx_node.c @@ -249,14 +249,14 @@ iavf_device_input_inline (vlib_main_t *vm, vlib_node_runtime_t *node, u32 thr_idx = vlib_get_thread_index (); iavf_rt_data_t *rtd = vnet_dev_get_rt_temp_space (vm); iavf_rxq_t *arq = vnet_dev_get_rx_queue_data (rxq); - vlib_buffer_template_t bt = rxq->buffer_template; + vlib_buffer_template_t bt = vnet_dev_get_rx_queue_if_buffer_template (rxq); u32 n_trace, n_rx_packets = 0, n_rx_bytes = 0; u16 n_tail_desc = 0; u64 or_qw1 = 0; u32 *bi, *to_next, n_left_to_next; - u32 next_index = rxq->next_index; - u32 sw_if_index = port->intf.sw_if_index; - u32 hw_if_index = port->intf.hw_if_index; + u32 next_index = vnet_dev_get_rx_queue_if_next_index (rxq); + u32 sw_if_index = vnet_dev_get_rx_queue_if_sw_if_index (rxq); + u32 hw_if_index = vnet_dev_get_rx_queue_if_hw_if_index (rxq); u16 next = arq->next; u16 size = rxq->size; u16 mask = size - 1; diff --git a/src/vnet/dev/dev_funcs.h b/src/vnet/dev/dev_funcs.h index 521157abbe..5a3cd00ec8 100644 --- a/src/vnet/dev/dev_funcs.h +++ b/src/vnet/dev/dev_funcs.h @@ -74,6 +74,24 @@ vnet_dev_get_port_from_hw_if_index (u32 hw_if_index) return port; } +static_always_inline u32 +vnet_dev_get_rx_queue_if_sw_if_index (vnet_dev_rx_queue_t *rxq) +{ + return rxq->port->intf.sw_if_index; +} + +static_always_inline u32 +vnet_dev_get_rx_queue_if_hw_if_index (vnet_dev_rx_queue_t *rxq) +{ + return rxq->port->intf.hw_if_index; +} + +static_always_inline u32 +vnet_dev_get_port_rx_node_idex (vnet_dev_port_t *port) +{ + return port->intf.rx_node_index; +} + static_always_inline vnet_dev_t * vnet_dev_by_index (u32 index) { @@ -128,12 +146,6 @@ vnet_dev_port_validate (vlib_main_t *vm, vnet_dev_port_t *port) ASSERT (vm->thread_index == 0); } -static_always_inline u32 -vnet_dev_port_get_sw_if_index (vnet_dev_port_t *port) -{ - return port->intf.sw_if_index; -} - static_always_inline vnet_dev_port_t * vnet_dev_get_port_by_id (vnet_dev_t *dev, vnet_dev_port_id_t port_id) { @@ -144,7 +156,7 @@ vnet_dev_get_port_by_id (vnet_dev_t *dev, vnet_dev_port_id_t port_id) } static_always_inline vnet_dev_rx_queue_t * -vnet_dev_port_get_rx_queue_by_id (vnet_dev_port_t *port, +vnet_dev_get_port_rx_queue_by_id (vnet_dev_port_t *port, vnet_dev_queue_id_t queue_id) { foreach_vnet_dev_port_rx_queue (q, port) @@ -154,7 +166,7 @@ vnet_dev_port_get_rx_queue_by_id (vnet_dev_port_t *port, } static_always_inline vnet_dev_tx_queue_t * -vnet_dev_port_get_tx_queue_by_id (vnet_dev_port_t *port, +vnet_dev_get_port_tx_queue_by_id (vnet_dev_port_t *port, vnet_dev_queue_id_t queue_id) { foreach_vnet_dev_port_tx_queue (q, port) @@ -199,6 +211,18 @@ vnet_dev_tx_queue_unlock_if_needed (vnet_dev_tx_queue_t *txq) __atomic_store_n (&txq->lock, 0, __ATOMIC_RELEASE); } +static_always_inline vlib_buffer_template_t +vnet_dev_get_rx_queue_if_buffer_template (vnet_dev_rx_queue_t *rxq) +{ + return rxq->buffer_template; +} + +static_always_inline u16 +vnet_dev_get_rx_queue_if_next_index (vnet_dev_rx_queue_t *rxq) +{ + return rxq->next_index; +} + static_always_inline u8 vnet_dev_get_rx_queue_buffer_pool_index (vnet_dev_rx_queue_t *rxq) { diff --git a/src/vnet/dev/port.c b/src/vnet/dev/port.c index a56cb86ea7..32ce52c7a4 100644 --- a/src/vnet/dev/port.c +++ b/src/vnet/dev/port.c @@ -370,7 +370,7 @@ vnet_dev_port_cfg_change (vlib_main_t *vm, vnet_dev_port_t *port, { if (req->all_queues == 0) { - rxq = vnet_dev_port_get_rx_queue_by_id (port, req->queue_id); + rxq = vnet_dev_get_port_rx_queue_by_id (port, req->queue_id); if (rxq == 0) return VNET_DEV_ERR_BUG; } From 01d7ac48d3b180bb55330484b5a4656fee71d85c Mon Sep 17 00:00:00 2001 From: Sunil Kumar Kori Date: Wed, 21 May 2025 16:01:49 +0530 Subject: [PATCH 262/271] pfc: add framework for priority flow control Type: feature 1. What is the patch about? The patch adds changes to VNET library in VPP which adds optional HW Priority Flow Control(PFC) support to VNET device. When a user configures PFC on given queue with respective TC and pause time on a VNET device, then a. Packets sent on any of the send queues of the net device can be paused if PFC frame are received with configured TC. b. When NIC receives frames at a very high rate and CQ threshold reaches to limit then NIC starts sending PFC frames to peer with configured TC and pause time. c. Patch exposes new binary APIs for following purposes: - To get PFC capability. - As pause and PFC flow control are mutual exclusive and can't be enabled together. Hence API to disable preconfigured pause frame flow control. - Configure PFC on a given queue for particular TC and pause time. 2. Need for the PFC framework: Priority-Based Flow Control (PFC) is needed in networking, particularly in Ethernet networks, to manage network traffic effectively and prevent packet loss, especially during congestion. It helps ensuring reliable transmission of critical data by prioritizing different traffic types and managing their flow. a. Prevents Packet Loss: PFC uses a mechanism to manage the flow of data and prevents packet loss by controlling the flow of traffic on the network. b. Prioritizes Traffic: PFC allows for the prioritization of different types of traffic based on their importance or service requirements. c. Ensures Reliable Transmission: By prioritizing and managing traffic flow, PFC helps ensure that critical data is transmitted reliably, especially in scenarios where congestion is likely. d. Improves Network Performance: PFC can improve network performance by minimizing congestion and ensuring that critical traffic gets the necessary resources. e. Suitable for Lossless Networks: PFC is a key technology in establishing lossless networks, where packet loss is not acceptable. Signed-off-by: Sunil Kumar Kori Change-Id: I4a4dfd7bd8de2f9a838d529fe79cd844e5361445 Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/154173 Tested-by: sa_ip-toolkits-Jenkins Reviewed-by: Nithin Kumar Dabilpuram Reviewed-by: Monendra Singh Kushwaha Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/161760 Tested-by: sa_ip-sw-jenkins --- MAINTAINERS | 4 + src/vnet/CMakeLists.txt | 16 ++++ src/vnet/dev/dev.h | 1 + src/vnet/interface.h | 2 + src/vnet/pfc/pfc.api | 115 +++++++++++++++++++++++ src/vnet/pfc/pfc.c | 70 ++++++++++++++ src/vnet/pfc/pfc.h | 122 +++++++++++++++++++++++++ src/vnet/pfc/pfc_api.c | 91 ++++++++++++++++++ src/vnet/pfc/pfc_test.c | 198 ++++++++++++++++++++++++++++++++++++++++ 9 files changed, 619 insertions(+) create mode 100644 src/vnet/pfc/pfc.api create mode 100644 src/vnet/pfc/pfc.c create mode 100644 src/vnet/pfc/pfc.h create mode 100644 src/vnet/pfc/pfc_api.c create mode 100644 src/vnet/pfc/pfc_test.c diff --git a/MAINTAINERS b/MAINTAINERS index 60007309cb..be3a5b28f6 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -102,6 +102,10 @@ VNET Traffic Management I: tm F: src/vnet/tm/ +VNET Priority Flow Control +I: pfc +F: src/vnet/pfc/ + VNET New Device Drivers Infra I: dev M: Damjan Marion diff --git a/src/vnet/CMakeLists.txt b/src/vnet/CMakeLists.txt index b7963f5c83..129af6dbf9 100644 --- a/src/vnet/CMakeLists.txt +++ b/src/vnet/CMakeLists.txt @@ -1425,6 +1425,21 @@ list(APPEND VNET_HEADERS list(APPEND VNET_API_FILES tm/tm.api) +############################################################################## +# PFC +############################################################################## + +list(APPEND VNET_SOURCES + pfc/pfc.c + pfc/pfc_api.c +) + +list(APPEND VNET_HEADERS + pfc/pfc.h +) + +list(APPEND VNET_API_FILES pfc/pfc.api) + ############################################################################## # ARP/ND ############################################################################## @@ -1493,6 +1508,7 @@ add_vat_test_library(vnet l2/l2_test.c ipsec/ipsec_test.c tm/tm_test.c + pfc/pfc_test.c ) ############################################################################## diff --git a/src/vnet/dev/dev.h b/src/vnet/dev/dev.h index 50d752b996..7fcfd12e92 100644 --- a/src/vnet/dev/dev.h +++ b/src/vnet/dev/dev.h @@ -280,6 +280,7 @@ typedef struct vnet_dev_rx_queue u8 enabled : 1; u8 started : 1; u8 suspended : 1; + u8 tc : 4; vnet_dev_queue_id_t queue_id; u16 size; u16 next_index; diff --git a/src/vnet/interface.h b/src/vnet/interface.h index bcbae3ba24..596c3cb564 100644 --- a/src/vnet/interface.h +++ b/src/vnet/interface.h @@ -45,6 +45,7 @@ #include #include #include +#include #include struct vnet_main_t; @@ -292,6 +293,7 @@ typedef struct _vnet_device_class vnet_interface_rss_queues_set_t *set_rss_queues_function; tm_system_t *tm_sys_impl; + pfc_system_t *pfc_sys_impl; } vnet_device_class_t; u32 vnet_register_device_class (vlib_main_t *, vnet_device_class_t *); diff --git a/src/vnet/pfc/pfc.api b/src/vnet/pfc/pfc.api new file mode 100644 index 0000000000..1e43ca77cb --- /dev/null +++ b/src/vnet/pfc/pfc.api @@ -0,0 +1,115 @@ +/* + * Copyright (c) 2025 Marvell. + * SPDX-License-Identifier: Apache-2.0 + * https://spdx.org/licenses/Apache-2.0.html + */ + +option version = "0.1.0"; + +/** + * @brief Reply for configuring PFC. + * + * This structure specifies the parameters returned in response for configuring the PFC. + * + * @param context - Sender context, to match reply with request. + * @param retval - Return value indicating success or failure of the operation. + */ +define pfc_sys_configure_reply +{ + u32 context; + i32 retval; +}; + +/** + * @brief Configure PFC on given Rx/Tx queues with given TC and time quanta. + * + * This structure outlines the necessary parameters to configure PFC for given TC and time quanta. + * + * @param client_index - Opaque cookie to identify the sender. + * @param context - Sender context, to match reply with request. + * @param sw_if_idx - Software interface index. + * @param mode - PFC mode to be configured. + * @param txq - Tx queue to be paused for given TC. + * @param tx_tc - Traffic class to be applied PFC frame processing. + * @param pause_time - Pause time to filled in PFC frames. + * @param rxq - Rx queue which will trigger PFC frames with given TC and pause time if congested. + * @param rx_tc - Traffic class to filled in PFC frames. + */ +define pfc_sys_configure +{ + u32 client_index; + u32 context; + u32 sw_if_idx; + u32 mode; + u32 txq; + u32 tx_tc; + u32 pause_time; + u32 rxq; + u32 rx_tc; +}; + +/** + * @brief Reply for getting the capabilities of a PFC system. + * + * This structure specifies the parameters returned in response for getting the capabilities of a PFC system. + * + * @param context - Sender context, to match reply with request. + * @param retval - Return value indicating success or failure of the operation. + * @param tc_max - Represents maximum number of traffic classes supported by H/W. + * @param mode - Represents supported mode of operation i.e. RX_PAUSE, TX_PAUSE or FULL. + */ +define pfc_sys_get_capabilities_reply +{ + u32 context; + i32 retval; + u32 tc_max; + u32 mode; +}; + +/** + * @brief Get the capabilities of a PFC system. + * + * This structure outlines the necessary parameters to get the capabilities of a PFC system. + * + * @param client_index - Opaque cookie to identify the sender. + * @param context - Sender context, to match reply with request. + * @param sw_if_idx - Software interface index. + */ +define pfc_sys_get_capabilities + { + u32 client_index; + u32 context; + u32 sw_if_idx; + }; + +/** + * @brief Reply for disabling priority flow control. + * + * This structure specifies the parameters returned in response to disable PFC. + * + * @param context - Sender context, to match reply with request. + * @param retval - Return value indicating success or failure of the operation. + */ +define pfc_sys_disable_pause_frame_flow_ctrl_reply +{ + u32 context; + i32 retval; +}; + +/** + * @brief Disable priority flow control. + * + * This structure outlines the necessary parameters to disable applied flow control mechanism. + * + * @param client_index - Opaque cookie to identify the sender. + * @param context - Sender context, to match reply with request. + * @param sw_if_idx - Software interface index. + * @param disable - Disables priority flow control. + */ +define pfc_sys_disable_pause_frame_flow_ctrl + { + u32 client_index; + u32 context; + u32 sw_if_idx; + u32 disable; + }; diff --git a/src/vnet/pfc/pfc.c b/src/vnet/pfc/pfc.c new file mode 100644 index 0000000000..c5b12e9b54 --- /dev/null +++ b/src/vnet/pfc/pfc.c @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2025 Marvell. + * SPDX-License-Identifier: Apache-2.0 + * https://spdx.org/licenses/Apache-2.0.html + */ + +#include + +pfc_system_t pfc_system_main; + +int +pfc_system_register (pfc_system_t *pfc_sys, u32 hw_if_idx) +{ + vnet_main_t *vnm = vnet_get_main (); + + vnet_hw_interface_t *hi = vnet_get_hw_interface (vnm, hw_if_idx); + + vnet_device_class_t *dev_class = + vnet_get_device_class (vnm, hi->dev_class_index); + + dev_class->pfc_sys_impl = pfc_sys; + + return 0; +} + +int +pfc_sys_configure (u32 hw_if_idx, pfc_params_t *params) +{ + vnet_main_t *vnm = vnet_get_main (); + + vnet_hw_interface_t *hi = vnet_get_hw_interface (vnm, hw_if_idx); + + vnet_device_class_t *dev_class = + vnet_get_device_class (vnm, hi->dev_class_index); + + dev_class->pfc_sys_impl->pfc_configure (hw_if_idx, params); + + return 0; +} + +int +pfc_sys_get_capabilities (u32 hw_if_idx, pfc_capa_params_t *capa) +{ + vnet_main_t *vnm = vnet_get_main (); + + vnet_hw_interface_t *hi = vnet_get_hw_interface (vnm, hw_if_idx); + + vnet_device_class_t *dev_class = + vnet_get_device_class (vnm, hi->dev_class_index); + + dev_class->pfc_sys_impl->pfc_get_capabilities (hw_if_idx, capa); + + return 0; +} + +int +pfc_sys_disable_pause_frame_flow_ctrl (u32 hw_if_idx, u32 disable) +{ + vnet_main_t *vnm = vnet_get_main (); + + vnet_hw_interface_t *hi = vnet_get_hw_interface (vnm, hw_if_idx); + + vnet_device_class_t *dev_class = + vnet_get_device_class (vnm, hi->dev_class_index); + + dev_class->pfc_sys_impl->pfc_disable_pause_frame_flow_ctrl (hw_if_idx, + disable); + + return 0; +} diff --git a/src/vnet/pfc/pfc.h b/src/vnet/pfc/pfc.h new file mode 100644 index 0000000000..edb1371546 --- /dev/null +++ b/src/vnet/pfc/pfc.h @@ -0,0 +1,122 @@ +/* + * Copyright (c) 2025 Marvell. + * SPDX-License-Identifier: Apache-2.0 + * https://spdx.org/licenses/Apache-2.0.html + */ + +#ifndef _PFC_H_ +#define _PFC_H_ + +#include +#include +#include +#include +#include + +/** + * This enum indicates the flow control mode + */ +typedef enum +{ + /** Disable flow control. */ + PFC_ETH_FC_NONE = 0, + /** Rx pause frame, enable flowctrl on Tx side. */ + PFC_ETH_FC_RX_PAUSE, + /** Tx pause frame, enable flowctrl on Rx side. */ + PFC_ETH_FC_TX_PAUSE, + /**< Enable flow control on both side. */ + PFC_ETH_FC_FULL +} pfc_mode_t; + +/** + * PFC Capabilities params + */ +typedef struct pfc_capa_params_ +{ + /** Maximum supported traffic class as per PFC (802.1Qbb) specification. */ + uint8_t tc_max; + /** PFC mode capabilities. */ + pfc_mode_t mode; +} pfc_capa_params_t; + +/** + * A structure used to configure priority flow control on + * ethernet device for given Rx/Tx queues. + */ +typedef struct pfc_params_ +{ + /** Flow control mode */ + pfc_mode_t mode; + /** Structure shall be used to configure given tx_qid with corresponding tc. + * When device receives PFC frame with mentioned tc, traffic will be paused + * on tx_qid for that tc. + * Valid when (mode == PFC_ETH_FC_RX_PAUSE || mode == PFC_ETH_FC_FULL) + */ + struct + { + /** Tx queue ID */ + uint16_t txq; + /** Traffic class as per PFC (802.1Qbb) spec. The value must be + * in the range [0, max_tx_queues - 1] + */ + uint8_t tc; + } rx_pause; + /** + * Structure shall be used to configure pfc on given rx_qid. + * When rx_qid is congested, PFC frames are generated with tc + * and pause_time to the peer. + * Valid when (mode == PFC_ETH_FC_TX_PAUSE || mode == PFC_ETH_FC_FULL) + */ + struct + { + /** Pause quota in the Pause frame */ + uint16_t pause_time; + /** Rx queue ID */ + uint16_t rxq; + /** Traffic class as per PFC (802.1Qbb) spec. The value must be + * in the range [0, max_rx_queues - 1] + */ + uint8_t tc; + } tx_pause; +} pfc_params_t; + +typedef struct pfc_system_t_ +{ + u32 hw_if_idx; + int (*pfc_configure) (u32 hw_if_idx, pfc_params_t *params); + int (*pfc_get_capabilities) (u32 hw_if_idx, pfc_capa_params_t *capa_param); + int (*pfc_disable_pause_frame_flow_ctrl) (u32 hw_if_idx, u32 disable); +} pfc_system_t; + +/** + * @brief Configure priority flow control on given device. + * @param hw_if_idx - Hardware interface index. + * @param capa_param - Pointer to structure containing pfc parameters. + */ +int pfc_sys_configure (u32 hw_if_idx, pfc_params_t *params); + +/** + * @brief Read capabilities for a pfc system. + * @param hw_if_idx - Hardware interface index. + * @param capa_param - Pointer to structure where capabilities are to be + * filled. + */ +int pfc_sys_get_capabilities (u32 hw_if_idx, pfc_capa_params_t *capa_param); + +/** + * @brief Disable pause flow control. + * @param hw_if_idx - Hardware interface index. + * @param disable - Flag to toggle pause flow control. + */ +int pfc_sys_disable_pause_frame_flow_ctrl (u32 hw_if_idx, u32 disable); + +/** + * @brief Register the Priority Flow Control (PFC) system. + * + * @param pfc_sys - Pointer to the PFC system structure to be registered. + * @param hw_if_idx - Hardware interface index. + * + * @return 0 on success. + */ +int pfc_system_register (pfc_system_t *pfc_sys, u32 hw_if_idx); +#endif diff --git a/src/vnet/pfc/pfc_api.c b/src/vnet/pfc/pfc_api.c new file mode 100644 index 0000000000..248ff0ce99 --- /dev/null +++ b/src/vnet/pfc/pfc_api.c @@ -0,0 +1,91 @@ +/* + * Copyright (c) 2025 Marvell. + * SPDX-License-Identifier: Apache-2.0 + * https://spdx.org/licenses/Apache-2.0.html + */ + +#include +#include +#include +#include +#include +#include +#include + +/** + * Base message ID fot the plugin + */ +static u32 pfc_base_msg_id; +#define REPLY_MSG_ID_BASE pfc_base_msg_id + +#include + +void +vl_api_pfc_sys_configure_t_handler (vl_api_pfc_sys_configure_t *mp) +{ + vnet_main_t *vnm = vnet_get_main (); + vnet_sw_interface_t *sw = + vnet_get_sup_sw_interface (vnm, clib_net_to_host_u32 (mp->sw_if_idx)); + vl_api_pfc_sys_configure_reply_t *rmp; + pfc_params_t params = { 0 }; + int rv = -1; + + params.mode = clib_net_to_host_u32 (mp->mode); + params.rx_pause.txq = clib_net_to_host_u32 (mp->txq); + params.rx_pause.tc = clib_net_to_host_u32 (mp->tx_tc); + params.tx_pause.pause_time = clib_net_to_host_u32 (mp->pause_time); + params.tx_pause.rxq = clib_net_to_host_u32 (mp->rxq); + params.tx_pause.tc = clib_net_to_host_u32 (mp->rx_tc); + + rv = pfc_sys_configure (sw->hw_if_index, ¶ms); + + REPLY_MACRO (VL_API_PFC_SYS_CONFIGURE_REPLY); +} + +void +vl_api_pfc_sys_get_capabilities_t_handler ( + vl_api_pfc_sys_get_capabilities_t *mp) +{ + vnet_main_t *vnm = vnet_get_main (); + vnet_sw_interface_t *sw = + vnet_get_sup_sw_interface (vnm, clib_net_to_host_u32 (mp->sw_if_idx)); + vl_api_pfc_sys_get_capabilities_reply_t *rmp; + pfc_capa_params_t capa = { 0 }; + int rv = -1; + + rv = pfc_sys_get_capabilities (sw->hw_if_index, &capa); + + REPLY_MACRO2 (VL_API_PFC_SYS_GET_CAPABILITIES_REPLY, ({ + rmp->mode = clib_host_to_net_u32 (capa.mode); + rmp->tc_max = clib_host_to_net_u32 (capa.tc_max); + })); +} + +void +vl_api_pfc_sys_disable_pause_frame_flow_ctrl_t_handler ( + vl_api_pfc_sys_disable_pause_frame_flow_ctrl_t *mp) +{ + vnet_main_t *vnm = vnet_get_main (); + vnet_sw_interface_t *sw = + vnet_get_sup_sw_interface (vnm, clib_net_to_host_u32 (mp->sw_if_idx)); + vl_api_pfc_sys_disable_pause_frame_flow_ctrl_reply_t *rmp; + int rv = -1; + + rv = pfc_sys_disable_pause_frame_flow_ctrl ( + sw->hw_if_index, clib_host_to_net_u32 (mp->disable)); + + REPLY_MACRO (VL_API_PFC_SYS_DISABLE_PAUSE_FRAME_FLOW_CTRL_REPLY); +} + +#include + +static clib_error_t * +pfc_api_init (vlib_main_t *vm) +{ + /* Ask for a correctly-sized block of API message decode slots */ + pfc_base_msg_id = setup_message_id_table (); + + return 0; +} + +VLIB_INIT_FUNCTION (pfc_api_init); diff --git a/src/vnet/pfc/pfc_test.c b/src/vnet/pfc/pfc_test.c new file mode 100644 index 0000000000..bef899dc33 --- /dev/null +++ b/src/vnet/pfc/pfc_test.c @@ -0,0 +1,198 @@ +/* + * Copyright (c) 2024 Marvell. + * SPDX-License-Identifier: Apache-2.0 + * https://spdx.org/licenses/Apache-2.0.html + */ + +#include +#include +#include +#include + +#include + +#include + +typedef struct +{ + /* API message ID base */ + u16 msg_id_base; + u32 ping_id; + vat_main_t *vat_main; +} pfc_test_main_t; + +pfc_test_main_t pfc_test_main; + +#define __plugin_msg_base pfc_test_main.msg_id_base +#include +uword unformat_sw_if_index (unformat_input_t *input, va_list *args); + +/* Declare message IDs */ +#include +#include +#include + +static int +api_pfc_sys_configure (vat_main_t *vam) +{ + u32 pause_time = 0, rxq = 0, rx_tc = 0; + u32 mode = 0, txq = 0, tx_tc = 0; + unformat_input_t *i = vam->input; + vl_api_pfc_sys_configure_t *mp; + u32 msg_size = sizeof (*mp); + u8 sw_if_idx_set = 0; + u32 sw_if_idx = 0; + int ret; + + vam->result_ready = 0; + mp = vl_msg_api_alloc_as_if_client (msg_size); + + while (unformat_check_input (i) != UNFORMAT_END_OF_INPUT) + { + if (unformat (i, "sw_if_idx %u", &sw_if_idx)) + sw_if_idx_set = 1; + else if (unformat (i, "mode %u", &mode)) + ; + else if (unformat (i, "txq %u", &txq)) + ; + else if (unformat (i, "tx_tc %u", &tx_tc)) + ; + else if (unformat (i, "pause_time %u", &pause_time)) + ; + else if (unformat (i, "rxq %u", &rxq)) + ; + else if (unformat (i, "rx_tc %u", &rx_tc)) + ; + else + { + clib_warning ("Invalid input, unknown parameter"); + return -EINVAL; + } + } + + if (!sw_if_idx_set) + return -EINVAL; + + M (PFC_SYS_CONFIGURE, mp); + + mp->sw_if_idx = clib_host_to_net_u32 (sw_if_idx); + mp->mode = clib_host_to_net_u32 (mode); + mp->txq = clib_host_to_net_u32 (txq); + mp->tx_tc = clib_host_to_net_u32 (tx_tc); + mp->pause_time = clib_host_to_net_u32 (pause_time); + mp->rxq = clib_host_to_net_u32 (rxq); + mp->rx_tc = clib_host_to_net_u32 (rx_tc); + + S (mp); + W (ret); + return ret; +} + +static int +api_pfc_sys_get_capabilities (vat_main_t *vam) +{ + unformat_input_t *i = vam->input; + vl_api_pfc_sys_get_capabilities_t *mp; + u32 msg_size = sizeof (*mp); + u8 sw_if_idx_set = 0; + u32 sw_if_idx = 0; + int ret; + + vam->result_ready = 0; + mp = vl_msg_api_alloc_as_if_client (msg_size); + + while (unformat_check_input (i) != UNFORMAT_END_OF_INPUT) + { + if (unformat (i, "sw_if_idx %u", &sw_if_idx)) + sw_if_idx_set = 1; + else + { + clib_warning ("Invalid input, unknown parameter"); + return -EINVAL; + } + } + + if (!sw_if_idx_set) + return -EINVAL; + + M (PFC_SYS_GET_CAPABILITIES, mp); + + mp->sw_if_idx = clib_host_to_net_u32 (sw_if_idx); + + S (mp); + W (ret); + return ret; +} + +static int +api_pfc_sys_disable_pause_frame_flow_ctrl (vat_main_t *vam) +{ + vl_api_pfc_sys_disable_pause_frame_flow_ctrl_t *mp; + unformat_input_t *i = vam->input; + u32 sw_if_idx = 0, disable = 0; + u32 msg_size = sizeof (*mp); + u8 sw_if_idx_set = 0; + int ret; + + vam->result_ready = 0; + mp = vl_msg_api_alloc_as_if_client (msg_size); + + while (unformat_check_input (i) != UNFORMAT_END_OF_INPUT) + { + if (unformat (i, "sw_if_idx %u", &sw_if_idx)) + sw_if_idx_set = 1; + else if (unformat (i, "disable %u", &disable)) + ; + else + { + clib_warning ("Invalid input, unknown parameter"); + return -EINVAL; + } + } + + if (!sw_if_idx_set) + return -EINVAL; + + M (PFC_SYS_DISABLE_PAUSE_FRAME_FLOW_CTRL, mp); + + mp->sw_if_idx = clib_host_to_net_u32 (sw_if_idx); + mp->disable = clib_host_to_net_u32 (disable); + + S (mp); + W (ret); + return ret; +} + +static void +vl_api_pfc_sys_configure_reply_t_handler (vl_api_pfc_sys_configure_reply_t *mp) +{ + vat_main_t *vam = pfc_test_main.vat_main; + if (mp->retval < 0) + clib_warning ("PFC configure failed: %d", mp->retval); + + vam->result_ready = 1; +} + +static void +vl_api_pfc_sys_get_capabilities_reply_t_handler ( + vl_api_pfc_sys_get_capabilities_reply_t *mp) +{ + vat_main_t *vam = pfc_test_main.vat_main; + if (mp->retval < 0) + clib_warning ("PFC capability get failed: %d", mp->retval); + + vam->result_ready = 1; +} + +static void +vl_api_pfc_sys_disable_pause_frame_flow_ctrl_reply_t_handler ( + vl_api_pfc_sys_disable_pause_frame_flow_ctrl_reply_t *mp) +{ + vat_main_t *vam = pfc_test_main.vat_main; + if (mp->retval < 0) + clib_warning ("Pause frame disable failed: %d", mp->retval); + + vam->result_ready = 1; +} + +#include From 39be74207a9fbe0b45445fbe097dbeecd513ab51 Mon Sep 17 00:00:00 2001 From: Sunil Kumar Kori Date: Fri, 30 May 2025 12:37:10 +0530 Subject: [PATCH 263/271] octeon: add PFC support Type: feature Signed-off-by: Sunil Kumar Kori Change-Id: I55033852a1960fb5ac47c9ec56b1d5c2779515fd Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/154174 Tested-by: sa_ip-toolkits-Jenkins Reviewed-by: Nithin Kumar Dabilpuram Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/161761 Reviewed-by: Monendra Singh Kushwaha Tested-by: sa_ip-sw-jenkins --- src/plugins/dev_octeon/CMakeLists.txt | 1 + src/plugins/dev_octeon/init.c | 8 +- src/plugins/dev_octeon/octeon.h | 9 +- src/plugins/dev_octeon/pfc.c | 375 ++++++++++++++++++++++++++ src/plugins/dev_octeon/port.c | 20 +- 5 files changed, 400 insertions(+), 13 deletions(-) create mode 100644 src/plugins/dev_octeon/pfc.c diff --git a/src/plugins/dev_octeon/CMakeLists.txt b/src/plugins/dev_octeon/CMakeLists.txt index f4df8279d9..f6490883e2 100644 --- a/src/plugins/dev_octeon/CMakeLists.txt +++ b/src/plugins/dev_octeon/CMakeLists.txt @@ -46,6 +46,7 @@ add_vpp_plugin(dev_octeon esp_encrypt.c dpu/dpu.c tm.c + pfc.c MULTIARCH_SOURCES rx_node.c diff --git a/src/plugins/dev_octeon/init.c b/src/plugins/dev_octeon/init.c index 8511bd0b4c..181f9fce14 100644 --- a/src/plugins/dev_octeon/init.c +++ b/src/plugins/dev_octeon/init.c @@ -143,10 +143,10 @@ static vnet_dev_arg_t oct_port_args[] = { .default_val.boolean = false, }, { - .id = OCT_PORT_ARG_EN_ETH_PAUSE_FRAME, - .name = "eth_pause_frame", - .desc = "Enable ethernet pause frame support, applicable to network " - "devices only", + .id = OCT_PORT_ARG_EN_ETH_FLOW_CTRL, + .name = "eth_flow_ctrl", + .desc = "Enable Ethernet flow control support," + "applicable to network devices only", .type = VNET_DEV_ARG_TYPE_BOOL, .default_val.boolean = false, }, diff --git a/src/plugins/dev_octeon/octeon.h b/src/plugins/dev_octeon/octeon.h index d10f7f8646..90ded967a0 100644 --- a/src/plugins/dev_octeon/octeon.h +++ b/src/plugins/dev_octeon/octeon.h @@ -47,7 +47,7 @@ typedef enum typedef enum { - OCT_PORT_ARG_EN_ETH_PAUSE_FRAME = 1, + OCT_PORT_ARG_EN_ETH_FLOW_CTRL = 1, OCT_PORT_ARG_ALLMULTI_MODE, OCT_PORT_ARG_SWITCH_HDR_TYPE, OCT_PORT_ARG_END @@ -164,6 +164,10 @@ typedef struct u8 nix_initialized : 1; u8 status : 1; u8 full_duplex : 1; + u8 mode : 2; + u8 class_en; + u8 rx_pause_en; + u8 tx_pause_en; u32 speed; struct plt_pci_device plt_pci_dev; struct roc_nix *nix; @@ -226,6 +230,9 @@ vnet_dev_rv_t oct_flow_validate_params (vlib_main_t *, vnet_dev_port_t *, vnet_dev_rv_t oct_flow_query (vlib_main_t *, vnet_dev_port_t *, u32, uword, u64 *); +/* pfc.c */ +int oct_pfc_sys_init_args (pfc_system_t *pfc); + /* counter.c */ void oct_port_add_counters (vlib_main_t *, vnet_dev_port_t *); void oct_port_clear_counters (vlib_main_t *, vnet_dev_port_t *); diff --git a/src/plugins/dev_octeon/pfc.c b/src/plugins/dev_octeon/pfc.c new file mode 100644 index 0000000000..cff3a2c7fd --- /dev/null +++ b/src/plugins/dev_octeon/pfc.c @@ -0,0 +1,375 @@ +/* + * Copyright (c) 2025 Marvell. + * SPDX-License-Identifier: Apache-2.0 + * https://spdx.org/licenses/Apache-2.0.html + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +pfc_system_t dev_oct_pfc_ops; + +VLIB_REGISTER_LOG_CLASS (oct_log, static) = { + .class_name = "octeon", + .subclass_name = "pfc", +}; + +static vnet_dev_rv_t +oct_roc_err (vnet_dev_t *dev, int rv, char *fmt, ...) +{ + u8 *s = 0; + va_list va; + + va_start (va, fmt); + s = va_format (s, fmt, &va); + va_end (va); + + log_err (dev, "%v - ROC error %s (%d)", s, roc_error_msg_get (rv), rv); + + vec_free (s); + return VNET_DEV_ERR_INTERNAL; +} + +static int +oct_nix_pfc_rq_conf (vnet_dev_port_t *port, uint16_t qid, uint8_t tx_pause, + uint8_t tc) +{ + vnet_dev_t *dev = port->dev; + oct_device_t *cd = vnet_dev_get_data (dev); + struct roc_nix_pfc_cfg pfc_cfg; + struct roc_nix *nix = cd->nix; + struct roc_nix_fc_cfg fc_cfg; + enum roc_nix_fc_mode mode; + vnet_dev_rx_queue_t *rxq; + struct roc_nix_rq *rq; + struct roc_nix_cq *cq; + oct_rxq_t *crq; + int rc; + + if (port->rx_queues == NULL) + return -EINVAL; + + if (qid >= port->attr.max_rx_queues) + return -ENOTSUP; + + /* Configure RQ */ + rxq = vnet_dev_get_port_rx_queue_by_id (port, qid); + if (rxq == 0) + return -ENODEV; + + crq = vnet_dev_get_rx_queue_data (rxq); + rq = &crq->rq; + cq = &crq->cq; + + memset (&fc_cfg, 0, sizeof (struct roc_nix_fc_cfg)); + fc_cfg.type = ROC_NIX_FC_RQ_CFG; + fc_cfg.rq_cfg.tc = tc; + fc_cfg.rq_cfg.enable = !!tx_pause; + fc_cfg.rq_cfg.rq = rq->qid; + fc_cfg.rq_cfg.pool = rq->aura_handle; + fc_cfg.rq_cfg.spb_pool = rq->spb_aura_handle; + fc_cfg.rq_cfg.cq_drop = cq->drop_thresh; + fc_cfg.rq_cfg.cq_bp = cq->bp_thresh; + fc_cfg.rq_cfg.pool_drop_pct = ROC_NIX_AURA_THRESH; + rc = roc_nix_fc_config_set (nix, &fc_cfg); + if (rc) + return rc; + + rxq->tc = tc; + /* Recheck number of RQ's that have PFC enabled */ + cd->tx_pause_en = 0; + foreach_vnet_dev_port_rx_queue (q, port) + { + /* Skip if RQ does not exist */ + if (!q->enabled) + continue; + + oct_rxq_t *crq = vnet_dev_get_rx_queue_data (q); + rq = &crq->rq; + if (rq->tc != ROC_NIX_PFC_CLASS_INVALID) + cd->tx_pause_en++; + } + + /* Skip if PFC already enabled in mac */ + if (cd->tx_pause_en > 1) + return 0; + + /* Configure MAC block */ + cd->class_en = cd->tx_pause_en ? 0xFF : 0x0; + + if (cd->rx_pause_en) + mode = cd->tx_pause_en ? ROC_NIX_FC_FULL : ROC_NIX_FC_RX; + else + mode = cd->tx_pause_en ? ROC_NIX_FC_TX : ROC_NIX_FC_NONE; + + memset (&pfc_cfg, 0, sizeof (struct roc_nix_pfc_cfg)); + pfc_cfg.mode = mode; + pfc_cfg.tc = cd->class_en; + return roc_nix_pfc_mode_set (nix, &pfc_cfg); +} + +static int +oct_nix_pfc_sq_conf (vnet_dev_port_t *port, uint16_t qid, uint8_t rx_pause, + uint8_t tc) +{ + vnet_dev_t *dev = port->dev; + oct_device_t *cd = vnet_dev_get_data (dev); + struct roc_nix_pfc_cfg pfc_cfg; + struct roc_nix *nix = cd->nix; + struct roc_nix_fc_cfg fc_cfg; + enum roc_nix_fc_mode mode; + vnet_dev_tx_queue_t *txq; + struct roc_nix_sq *sq; + oct_txq_t *ctq; + int rc; + + if (port->tx_queues == NULL) + return -EINVAL; + + if (qid >= port->attr.max_tx_queues) + return -ENOTSUP; + + /* Check if RX pause frame is enabled or not and + * confirm user requested for PFC. + */ + if (!cd->rx_pause_en && rx_pause) + { + if ((roc_nix_tm_tree_type_get (nix) == ROC_NIX_TM_DEFAULT) && + port->attr.max_tx_queues > 1) + { + /* + * Disabled xmit will be enabled when new topology is available. + */ + rc = roc_nix_tm_hierarchy_disable (nix); + if (rc) + goto exit; + + rc = roc_nix_tm_pfc_prepare_tree (nix); + if (rc) + goto exit; + + rc = roc_nix_tm_hierarchy_enable (nix, ROC_NIX_TM_PFC, true); + if (rc) + goto exit; + } + } + + txq = vnet_dev_get_port_tx_queue_by_id (port, qid); + if (txq == 0) + { + rc = -ENODEV; + goto exit; + } + ctq = vnet_dev_get_tx_queue_data (txq); + sq = &ctq->sq; + + memset (&fc_cfg, 0, sizeof (struct roc_nix_fc_cfg)); + fc_cfg.type = ROC_NIX_FC_TM_CFG; + fc_cfg.tm_cfg.sq = sq->qid; + fc_cfg.tm_cfg.tc = tc; + fc_cfg.tm_cfg.enable = !!rx_pause; + rc = roc_nix_fc_config_set (nix, &fc_cfg); + if (rc) + return rc; + + /* Recheck number of SQ's that have PFC enabled */ + cd->rx_pause_en = 0; + foreach_vnet_dev_port_tx_queue (q, port) + { + /* Skip if RQ does not exist */ + if (!q->enabled) + continue; + + oct_txq_t *ctq = vnet_dev_get_tx_queue_data (q); + sq = &ctq->sq; + if (sq->tc != ROC_NIX_PFC_CLASS_INVALID) + cd->rx_pause_en++; + } + + if (cd->rx_pause_en > 1) + goto exit; + + if (cd->tx_pause_en) + mode = cd->rx_pause_en ? ROC_NIX_FC_FULL : ROC_NIX_FC_TX; + else + mode = cd->rx_pause_en ? ROC_NIX_FC_RX : ROC_NIX_FC_NONE; + + memset (&pfc_cfg, 0, sizeof (struct roc_nix_pfc_cfg)); + pfc_cfg.mode = mode; + pfc_cfg.tc = cd->class_en; + rc = roc_nix_pfc_mode_set (nix, &pfc_cfg); +exit: + return rc; +} + +int +oct_pfc_sys_init_args (pfc_system_t *pfc) +{ + memset (pfc, 0, sizeof (pfc_system_t)); + memcpy (pfc, &dev_oct_pfc_ops, sizeof (pfc_system_t)); + return 0; +} + +int +oct_pfc_sys_configure (u32 hw_if_idx, pfc_params_t *params) +{ + vnet_main_t *vnm = vnet_get_main (); + vnet_hw_interface_t *hi = vnet_get_hw_interface (vnm, hw_if_idx); + vnet_dev_port_t *port = + vnet_dev_get_port_from_dev_instance (hi->dev_instance); + vnet_dev_t *dev = port->dev; + oct_device_t *cd = vnet_dev_get_data (dev); + struct roc_nix *nix = cd->nix; + uint8_t en, tc, mode; + uint16_t qid; + int rc = 0; + + if (cd->mode != PFC_ETH_FC_NONE) + { + return oct_roc_err ( + dev, -ENOTSUP, + "Disable pause frame flow control before configuring PFC"); + } + + if (roc_nix_is_sdp (nix) || roc_nix_is_lbk (nix)) + return oct_roc_err (dev, -ENOTSUP, + "Prio flow ctrl config is not allowed on SDP/LBK"); + + /* Disallow flow control changes when device is in started state */ + if (port->started) + return oct_roc_err (dev, -EBUSY, "Stop the port=%d for setting PFC", + port->port_id); + + mode = params->mode; + + /* Perform Tx pause configuration on RQ */ + qid = params->tx_pause.rxq; + if (qid < port->attr.max_rx_queues) + { + en = (mode == PFC_ETH_FC_FULL) || (mode == PFC_ETH_FC_TX_PAUSE); + tc = params->tx_pause.tc; + rc = oct_nix_pfc_rq_conf (port, qid, en, tc); + } + + /* Perform Rx pause configuration on SQ */ + qid = params->rx_pause.txq; + if (qid < port->attr.max_tx_queues) + { + en = (mode == PFC_ETH_FC_FULL) || (mode == PFC_ETH_FC_RX_PAUSE); + tc = params->rx_pause.tc; + rc |= oct_nix_pfc_sq_conf (port, qid, en, tc); + } + + log_debug (dev, "hw_if_idx %d\n", hw_if_idx); + log_debug (dev, "mode %x\n", params->mode); + log_debug (dev, "rx_pause.txq %d\n", params->rx_pause.txq); + log_debug (dev, "rx_pause.tc %d\n", params->rx_pause.tc); + log_debug (dev, "tx_pause.pause_time %d\n", params->tx_pause.pause_time); + log_debug (dev, "tx_pause.rxq %d\n", params->tx_pause.rxq); + log_debug (dev, "tx_pause.tc %d\n", params->tx_pause.tc); + return rc; +} + +int +oct_pfc_sys_get_capabilities (u32 hw_if_idx, pfc_capa_params_t *cap) +{ + vnet_main_t *vnm = vnet_get_main (); + vnet_hw_interface_t *hi = vnet_get_hw_interface (vnm, hw_if_idx); + vnet_dev_port_t *port = + vnet_dev_get_port_from_dev_instance (hi->dev_instance); + vnet_dev_t *dev = port->dev; + oct_device_t *cd = vnet_dev_get_data (dev); + struct roc_nix *nix = cd->nix; + + memset (cap, 0, sizeof (*cap)); + cap->tc_max = roc_nix_chan_count_get (nix); + cap->mode = PFC_ETH_FC_FULL; + + log_debug (dev, "Max TC %d Supported mode %d", cap->tc_max, cap->mode); + return 0; +} + +int +oct_pfc_sys_disable_pause_frame_flow_ctrl (u32 hw_if_idx, u32 disable) +{ + vnet_main_t *vnm = vnet_get_main (); + vnet_hw_interface_t *hi = vnet_get_hw_interface (vnm, hw_if_idx); + vnet_dev_port_t *port = + vnet_dev_get_port_from_dev_instance (hi->dev_instance); + vnet_dev_t *dev = port->dev; + oct_device_t *cd = vnet_dev_get_data (dev); + struct roc_nix *nix = cd->nix; + struct roc_nix_sq *sq; + struct roc_nix_cq *cq; + struct roc_nix_rq *rq; + int rc = 0; + + foreach_vnet_dev_port_rx_queue (q, port) + { + struct roc_nix_fc_cfg fc_cfg; + + /* Skip if RQ does not exist */ + if (!q->enabled) + continue; + + oct_rxq_t *crq = vnet_dev_get_rx_queue_data (q); + rq = &crq->rq; + cq = &crq->cq; + + memset (&fc_cfg, 0, sizeof (struct roc_nix_fc_cfg)); + fc_cfg.type = ROC_NIX_FC_RQ_CFG; + fc_cfg.rq_cfg.rq = rq->qid; + fc_cfg.rq_cfg.pool = rq->aura_handle; + fc_cfg.rq_cfg.spb_pool = rq->spb_aura_handle; + fc_cfg.rq_cfg.cq_drop = cq->drop_thresh; + fc_cfg.rq_cfg.cq_bp = cq->bp_thresh; + fc_cfg.rq_cfg.pool_drop_pct = ROC_NIX_AURA_THRESH; + + rc = roc_nix_fc_config_set (nix, &fc_cfg); + if (rc) + return oct_roc_err ( + dev, rc, "Failed to disable flow control on Rx queue %u", rq->qid); + } + + foreach_vnet_dev_port_tx_queue (q, port) + { + struct roc_nix_fc_cfg fc_cfg; + + /* Skip if SQ does not exist */ + if (!q->enabled) + continue; + + oct_txq_t *ctq = vnet_dev_get_tx_queue_data (q); + sq = &ctq->sq; + + memset (&fc_cfg, 0, sizeof (struct roc_nix_fc_cfg)); + fc_cfg.type = ROC_NIX_FC_TM_CFG; + fc_cfg.tm_cfg.sq = sq->qid; + rc = roc_nix_fc_config_set (nix, &fc_cfg); + if (rc && rc != -EEXIST) + return oct_roc_err ( + dev, rc, "Failed to disable flow control on Tx queue %u", sq->qid); + } + + rc = roc_nix_fc_mode_set (nix, ROC_NIX_FC_NONE); + if (rc) + return oct_roc_err (dev, rc, "Failed to disable flow control on MAC"); + + cd->mode = PFC_ETH_FC_NONE; + return rc; +} + +pfc_system_t dev_oct_pfc_ops = { + .pfc_configure = oct_pfc_sys_configure, + .pfc_get_capabilities = oct_pfc_sys_get_capabilities, + .pfc_disable_pause_frame_flow_ctrl = + oct_pfc_sys_disable_pause_frame_flow_ctrl, +}; diff --git a/src/plugins/dev_octeon/port.c b/src/plugins/dev_octeon/port.c index 4a42b73803..428c5b3341 100644 --- a/src/plugins/dev_octeon/port.c +++ b/src/plugins/dev_octeon/port.c @@ -17,6 +17,7 @@ extern oct_inl_dev_main_t oct_inl_dev_main; tm_system_t tm_system_ops; +pfc_system_t pfc_system_ops; VLIB_REGISTER_LOG_CLASS (oct_log, static) = { .class_name = "octeon", @@ -66,7 +67,7 @@ oct_roc_err (vnet_dev_t *dev, int rv, char *fmt, ...) } vnet_dev_rv_t -oct_port_pause_flow_control_init (vlib_main_t *vm, vnet_dev_port_t *port) +oct_port_flow_control_init (vlib_main_t *vm, vnet_dev_port_t *port) { vnet_dev_t *dev = port->dev; oct_device_t *cd = vnet_dev_get_data (dev); @@ -77,7 +78,7 @@ oct_port_pause_flow_control_init (vlib_main_t *vm, vnet_dev_port_t *port) struct roc_nix_rq *rq; int rrv; - /* pause flow control is not supported on SDP/LBK devices */ + /* Flow control is not supported on SDP/LBK devices */ if (roc_nix_is_sdp (nix) || roc_nix_is_lbk (nix)) { log_notice (dev, @@ -133,6 +134,7 @@ oct_port_pause_flow_control_init (vlib_main_t *vm, vnet_dev_port_t *port) if (rrv) return oct_roc_err (dev, rrv, "roc_nix_fc_mode_set failed"); + cd->mode = PFC_ETH_FC_FULL; return VNET_DEV_OK; } @@ -167,7 +169,7 @@ oct_port_init (vlib_main_t *vm, vnet_dev_port_t *port) u8 mac_addr[PLT_ETHER_ADDR_LEN]; struct roc_nix *nix = cd->nix; vnet_dev_rv_t rv = -1; - bool is_allmulti_enable = false, is_pause_frame_enable = false; + bool is_allmulti_enable = false, is_flow_ctrl_enable = false; u32 total_sz = 0; int rrv; @@ -177,9 +179,9 @@ oct_port_init (vlib_main_t *vm, vnet_dev_port_t *port) { if (arg->id == OCT_PORT_ARG_ALLMULTI_MODE && vnet_dev_arg_get_bool (arg)) is_allmulti_enable = true; - if (arg->id == OCT_PORT_ARG_EN_ETH_PAUSE_FRAME && + if (arg->id == OCT_PORT_ARG_EN_ETH_FLOW_CTRL && vnet_dev_arg_get_bool (arg)) - is_pause_frame_enable = true; + is_flow_ctrl_enable = true; if (arg->id == OCT_PORT_ARG_SWITCH_HDR_TYPE && vnet_dev_arg_get_string (arg)) cp->npc.switch_header_type = @@ -310,9 +312,8 @@ oct_port_init (vlib_main_t *vm, vnet_dev_port_t *port) return rv; } - /* Configure pause frame flow control*/ - if (is_pause_frame_enable && - (rv = oct_port_pause_flow_control_init (vm, port))) + /* Configure flow control if requested */ + if (is_flow_ctrl_enable && (rv = oct_port_flow_control_init (vm, port))) { oct_port_deinit (vm, port); return rv; @@ -330,6 +331,9 @@ oct_port_init (vlib_main_t *vm, vnet_dev_port_t *port) oct_init_tm_args (&tm_system_ops); tm_system_register (&tm_system_ops, port->intf.hw_if_index); + oct_pfc_sys_init_args (&pfc_system_ops); + pfc_system_register (&pfc_system_ops, port->intf.hw_if_index); + return VNET_DEV_OK; } From 12cef4e0a1147660a026718ea349612c1c6ed054 Mon Sep 17 00:00:00 2001 From: Monendra Singh Kushwaha Date: Tue, 2 Sep 2025 06:56:37 +0000 Subject: [PATCH 264/271] octeon: revert pause frame port argument naming Type: fix Signed-off-by: Monendra Singh Kushwaha Change-Id: I5797b5959c84838fc15a79e5cd4a9552823ec190 Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/161762 Tested-by: sa_ip-sw-jenkins --- src/plugins/dev_octeon/init.c | 6 +++--- src/plugins/dev_octeon/octeon.h | 2 +- src/plugins/dev_octeon/port.c | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/plugins/dev_octeon/init.c b/src/plugins/dev_octeon/init.c index 181f9fce14..baa026825a 100644 --- a/src/plugins/dev_octeon/init.c +++ b/src/plugins/dev_octeon/init.c @@ -143,9 +143,9 @@ static vnet_dev_arg_t oct_port_args[] = { .default_val.boolean = false, }, { - .id = OCT_PORT_ARG_EN_ETH_FLOW_CTRL, - .name = "eth_flow_ctrl", - .desc = "Enable Ethernet flow control support," + .id = OCT_PORT_ARG_EN_ETH_PAUSE_FRAME, + .name = "eth_pause_frame", + .desc = "Enable ethernet pause frame (flow control) support, " "applicable to network devices only", .type = VNET_DEV_ARG_TYPE_BOOL, .default_val.boolean = false, diff --git a/src/plugins/dev_octeon/octeon.h b/src/plugins/dev_octeon/octeon.h index 90ded967a0..7289cde3a4 100644 --- a/src/plugins/dev_octeon/octeon.h +++ b/src/plugins/dev_octeon/octeon.h @@ -47,7 +47,7 @@ typedef enum typedef enum { - OCT_PORT_ARG_EN_ETH_FLOW_CTRL = 1, + OCT_PORT_ARG_EN_ETH_PAUSE_FRAME = 1, OCT_PORT_ARG_ALLMULTI_MODE, OCT_PORT_ARG_SWITCH_HDR_TYPE, OCT_PORT_ARG_END diff --git a/src/plugins/dev_octeon/port.c b/src/plugins/dev_octeon/port.c index 428c5b3341..4264d33144 100644 --- a/src/plugins/dev_octeon/port.c +++ b/src/plugins/dev_octeon/port.c @@ -179,7 +179,7 @@ oct_port_init (vlib_main_t *vm, vnet_dev_port_t *port) { if (arg->id == OCT_PORT_ARG_ALLMULTI_MODE && vnet_dev_arg_get_bool (arg)) is_allmulti_enable = true; - if (arg->id == OCT_PORT_ARG_EN_ETH_FLOW_CTRL && + if (arg->id == OCT_PORT_ARG_EN_ETH_PAUSE_FRAME && vnet_dev_arg_get_bool (arg)) is_flow_ctrl_enable = true; if (arg->id == OCT_PORT_ARG_SWITCH_HDR_TYPE && From ad2464ebe052e6e0ce15c02f14fca165ab0f72ef Mon Sep 17 00:00:00 2001 From: Damjan Marion Date: Mon, 15 Sep 2025 14:03:02 +0200 Subject: [PATCH 265/271] octeon: fix issue with RSS and promisc mode Type: fix Change-Id: I64c4cb8655ff957940a6d7ec38a82a8bf0dd4053 Signed-off-by: Damjan Marion Signed-off-by: Monendra Singh Kushwaha Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/162760 Tested-by: sa_ip-toolkits-Jenkins (cherry picked from commit 15f8ecd470905f9c754568cdf7c5b0e46896ed3f) --- src/plugins/dev_octeon/port.c | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/src/plugins/dev_octeon/port.c b/src/plugins/dev_octeon/port.c index 4264d33144..3b791aac06 100644 --- a/src/plugins/dev_octeon/port.c +++ b/src/plugins/dev_octeon/port.c @@ -593,6 +593,28 @@ oct_port_start (vlib_main_t *vm, vnet_dev_port_t *port) goto done; } + if (!(roc_nix_is_sdp (nix) || roc_nix_is_lbk (nix))) + { + + rv = roc_nix_npc_promisc_ena_dis (nix, port->promisc); + if (rv) + { + return oct_roc_err (dev, rv, "roc_nix_npc_promisc_ena_dis failed"); + } + + if (roc_nix_is_pf (nix)) + { + + rv = roc_nix_mac_promisc_mode_enable (nix, port->promisc); + if (rv) + { + return oct_roc_err (dev, rv, + "roc_nix_mac_promisc_mode_enable(%s) failed", + port->promisc ? "true" : "false"); + } + } + } + vnet_dev_poll_port_add (vm, port, 0.5, oct_port_poll); done: From fb6e3402292965105f17df7bc23cb0e7bac1b1c0 Mon Sep 17 00:00:00 2001 From: Damjan Marion Date: Mon, 15 Sep 2025 15:24:08 +0200 Subject: [PATCH 266/271] octeon: add option to specify RSS flowkey bitmap Type: improvement Change-Id: I0fa660c96a44ede0fcb435af560544bbc0f8da04 Signed-off-by: Damjan Marion Signed-off-by: Monendra Singh Kushwaha Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/162761 Tested-by: sa_ip-toolkits-Jenkins (cherry picked from commit 06e502302bb859ec7cdec28764bcc914f3d34d68) --- src/plugins/dev_octeon/init.c | 23 ++++++++++++++++------- src/plugins/dev_octeon/octeon.h | 2 ++ src/plugins/dev_octeon/port.c | 19 +++++++++---------- 3 files changed, 27 insertions(+), 17 deletions(-) diff --git a/src/plugins/dev_octeon/init.c b/src/plugins/dev_octeon/init.c index baa026825a..8faed6b250 100644 --- a/src/plugins/dev_octeon/init.c +++ b/src/plugins/dev_octeon/init.c @@ -135,13 +135,6 @@ static vnet_dev_arg_t oct_drv_args[] = { }; static vnet_dev_arg_t oct_port_args[] = { - { - .id = OCT_PORT_ARG_ALLMULTI_MODE, - .name = "allmulti", - .desc = "Set allmulti mode, applicable to network devices only", - .type = VNET_DEV_ARG_TYPE_BOOL, - .default_val.boolean = false, - }, { .id = OCT_PORT_ARG_EN_ETH_PAUSE_FRAME, .name = "eth_pause_frame", @@ -150,6 +143,22 @@ static vnet_dev_arg_t oct_port_args[] = { .type = VNET_DEV_ARG_TYPE_BOOL, .default_val.boolean = false, }, + { + .id = OCT_PORT_ARG_RSS_FLOW_KEY, + .name = "rss_flow_key", + .desc = "RSS Flow Key Bitmap, applicable to network devices only", + .type = VNET_DEV_ARG_TYPE_UINT32, + .default_val.uint32 = FLOW_KEY_TYPE_IPV4 | FLOW_KEY_TYPE_IPV6 | + FLOW_KEY_TYPE_TCP | FLOW_KEY_TYPE_UDP | + FLOW_KEY_TYPE_SCTP, + }, + { + .id = OCT_PORT_ARG_ALLMULTI_MODE, + .name = "allmulti", + .desc = "Set allmulti mode, applicable to network devices only", + .type = VNET_DEV_ARG_TYPE_BOOL, + .default_val.boolean = false, + }, { .id = OCT_PORT_ARG_SWITCH_HDR_TYPE, .name = "switch_header", diff --git a/src/plugins/dev_octeon/octeon.h b/src/plugins/dev_octeon/octeon.h index 7289cde3a4..54b075cbfd 100644 --- a/src/plugins/dev_octeon/octeon.h +++ b/src/plugins/dev_octeon/octeon.h @@ -48,6 +48,7 @@ typedef enum typedef enum { OCT_PORT_ARG_EN_ETH_PAUSE_FRAME = 1, + OCT_PORT_ARG_RSS_FLOW_KEY = 2, OCT_PORT_ARG_ALLMULTI_MODE, OCT_PORT_ARG_SWITCH_HDR_TYPE, OCT_PORT_ARG_END @@ -96,6 +97,7 @@ typedef struct u8 q_intr_enabled : 1; struct roc_npc npc; oct_flow_entry_t *flow_entries; + u32 rss_flowkey; } oct_port_t; typedef struct diff --git a/src/plugins/dev_octeon/port.c b/src/plugins/dev_octeon/port.c index 3b791aac06..5366c7b0f2 100644 --- a/src/plugins/dev_octeon/port.c +++ b/src/plugins/dev_octeon/port.c @@ -31,10 +31,6 @@ static const u8 default_rss_key[] = { 0x56, 0xbc, 0x78, 0x9a, 0x9a, 0x78, 0xbc, 0x56, 0xcd, 0x34, 0xef, 0x12 }; -static const u32 default_rss_flowkey = - (FLOW_KEY_TYPE_IPV4 | FLOW_KEY_TYPE_IPV6 | FLOW_KEY_TYPE_TCP | - FLOW_KEY_TYPE_UDP | FLOW_KEY_TYPE_SCTP); - static const u64 rxq_cfg = ROC_NIX_LF_RX_CFG_DIS_APAD | ROC_NIX_LF_RX_CFG_IP6_UDP_OPT | ROC_NIX_LF_RX_CFG_L2_LEN_ERR | ROC_NIX_LF_RX_CFG_DROP_RE | @@ -179,13 +175,15 @@ oct_port_init (vlib_main_t *vm, vnet_dev_port_t *port) { if (arg->id == OCT_PORT_ARG_ALLMULTI_MODE && vnet_dev_arg_get_bool (arg)) is_allmulti_enable = true; - if (arg->id == OCT_PORT_ARG_EN_ETH_PAUSE_FRAME && - vnet_dev_arg_get_bool (arg)) + else if (arg->id == OCT_PORT_ARG_EN_ETH_PAUSE_FRAME && + vnet_dev_arg_get_bool (arg)) is_flow_ctrl_enable = true; - if (arg->id == OCT_PORT_ARG_SWITCH_HDR_TYPE && - vnet_dev_arg_get_string (arg)) + else if (arg->id == OCT_PORT_ARG_SWITCH_HDR_TYPE && + vnet_dev_arg_get_string (arg)) cp->npc.switch_header_type = oct_parse_switch_hdr_type ((char *) vnet_dev_arg_get_string (arg)); + else if (arg->id == OCT_PORT_ARG_RSS_FLOW_KEY) + cp->rss_flowkey = vnet_dev_arg_get_uint32 (arg); } if ((rrv = roc_nix_lf_alloc (nix, port->intf.num_rx_queues, @@ -246,7 +244,7 @@ oct_port_init (vlib_main_t *vm, vnet_dev_port_t *port) return oct_roc_err (dev, rrv, "roc_nix_switch_hdr_set() failed"); } - if ((rrv = roc_nix_rss_default_setup (nix, default_rss_flowkey))) + if ((rrv = roc_nix_rss_default_setup (nix, cp->rss_flowkey))) { oct_port_deinit (vm, port); return oct_roc_err (dev, rrv, "roc_nix_rss_default_setup() failed"); @@ -728,6 +726,7 @@ oct_port_add_del_eth_addr (vlib_main_t *vm, vnet_dev_port_t *port, { vnet_dev_t *dev = port->dev; oct_device_t *cd = vnet_dev_get_data (dev); + oct_port_t *cp = vnet_dev_get_port_data (port); struct roc_nix *nix = cd->nix; vnet_dev_rv_t rv = VNET_DEV_OK; i32 rrv; @@ -754,7 +753,7 @@ oct_port_add_del_eth_addr (vlib_main_t *vm, vnet_dev_port_t *port, } } - rrv = roc_nix_rss_default_setup (nix, default_rss_flowkey); + rrv = roc_nix_rss_default_setup (nix, cp->rss_flowkey); if (rrv) rv = oct_roc_err (dev, rrv, "roc_nix_rss_default_setup() failed"); } From 8a8f6b1f99e4b5c72f4066f6dc0eba371400aed9 Mon Sep 17 00:00:00 2001 From: Damjan Marion Date: Mon, 15 Sep 2025 19:35:48 +0200 Subject: [PATCH 267/271] dev: add support for changing RSS key Type: improvement Change-Id: I40350be4c362d20f60b94c82faf564f06d8e86f6 Signed-off-by: Damjan Marion Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/162762 Tested-by: sa_ip-toolkits-Jenkins Reviewed-by: Monendra Singh Kushwaha (cherry picked from commit 988275c9700d7c397f9809ea3c8273e84c03aa23) --- src/vnet/dev/api.c | 40 ++++++++++++++++++++++++ src/vnet/dev/api.h | 11 +++++++ src/vnet/dev/args.c | 2 +- src/vnet/dev/cli.c | 67 ++++++++++++++++++++++++++++++++++++++++ src/vnet/dev/config.c | 57 ++++++++++++++++++++++------------ src/vnet/dev/dev.h | 7 ++++- src/vnet/dev/dev_funcs.h | 13 ++++++++ src/vnet/dev/format.c | 26 ++++++++++++++++ src/vnet/dev/port.c | 12 +++++++ src/vnet/dev/types.h | 5 +++ 10 files changed, 218 insertions(+), 22 deletions(-) diff --git a/src/vnet/dev/api.c b/src/vnet/dev/api.c index f891b5df80..18bfb88ffe 100644 --- a/src/vnet/dev/api.c +++ b/src/vnet/dev/api.c @@ -273,3 +273,43 @@ vnet_dev_api_remove_port_if (vlib_main_t *vm, return vnet_dev_process_call_port_op (vm, port, vnet_dev_port_if_remove); } + +vnet_dev_rv_t +vnet_dev_api_port_set_rss_key (vlib_main_t *vm, + vnet_dev_api_port_set_rss_key_args_t *args) +{ + vnet_dev_port_t *port = 0; + vnet_dev_t *dev = vnet_dev_by_index (args->dev_index); + vnet_dev_rv_t rv = VNET_DEV_OK; + vnet_dev_port_cfg_change_req_t req = { + .type = VNET_DEV_PORT_CFG_SET_RSS_KEY, + .rss_key = args->rss_key, + }; + + if (!dev) + return VNET_DEV_ERR_UNKNOWN_DEVICE; + + log_debug (dev, "port %u rss_key %U", args->port_id, + format_hex_bytes_no_wrap, args->rss_key.key, + args->rss_key.length); + + port = vnet_dev_get_port_by_id (dev, args->port_id); + if (!port) + return VNET_DEV_ERR_UNKNOWN_DEVICE; + + rv = vnet_dev_port_cfg_change_req_validate (vm, port, &req); + if (rv != VNET_DEV_OK) + { + log_err (dev, "RSS key cannot be set"); + return rv; + } + + rv = vnet_dev_process_port_cfg_change_req (vm, port, &req); + if (rv != VNET_DEV_OK) + { + log_err (dev, "device failed to set RSS key"); + return rv; + } + + return rv; +} diff --git a/src/vnet/dev/api.h b/src/vnet/dev/api.h index 1b7bf27d62..92752bfb12 100644 --- a/src/vnet/dev/api.h +++ b/src/vnet/dev/api.h @@ -65,4 +65,15 @@ vnet_dev_rv_t vnet_dev_api_remove_port_if (vlib_main_t *, vnet_dev_api_remove_port_if_args_t *); +typedef struct +{ + u32 dev_index; + vnet_dev_port_id_t port_id; + vnet_dev_rss_key_t rss_key; +} vnet_dev_api_port_set_rss_key_args_t; + +vnet_dev_rv_t +vnet_dev_api_port_set_rss_key (vlib_main_t *, + vnet_dev_api_port_set_rss_key_args_t *); + #endif /* _VNET_DEV_API_H_ */ diff --git a/src/vnet/dev/args.c b/src/vnet/dev/args.c index e302517cc6..07d1078761 100644 --- a/src/vnet/dev/args.c +++ b/src/vnet/dev/args.c @@ -82,7 +82,7 @@ vnet_dev_arg_parse (vlib_main_t *vm, vnet_dev_t *dev, vnet_dev_arg_t *args, else if (a->type == VNET_DEV_ARG_TYPE_UINT32) { u32 val, min = 0, max = CLIB_U32_MAX; - if (!unformat (&in, "%u", &val)) + if (!unformat (&in, "0x%x", &val) && !unformat (&in, "%u", &val)) { err = format (0, "unsigned integer in range %u - %u expected for " diff --git a/src/vnet/dev/cli.c b/src/vnet/dev/cli.c index 53be448318..aa7659e738 100644 --- a/src/vnet/dev/cli.c +++ b/src/vnet/dev/cli.c @@ -329,3 +329,70 @@ VLIB_CLI_COMMAND (show_device_counters_cmd, static) = { .function = show_device_counters_cmd_fn, .is_mp_safe = 1, }; + +static clib_error_t * +device_set_rss_key_cmd_fn (vlib_main_t *vm, unformat_input_t *input, + vlib_cli_command_t *cmd) +{ + vnet_dev_api_port_set_rss_key_args_t a = {}; + vnet_dev_rv_t rv; + int device_id_set = 0; + int sw_if_index_set = 0; + vnet_dev_device_id_t device_id = {}; + uint32_t sw_if_index, n; + + while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) + { + if (unformat (input, "port %u", &n)) + a.port_id = n; + else if (unformat (input, "dev %U", unformat_c_string_array, &device_id, + sizeof (device_id))) + device_id_set = 1; + else if (unformat (input, "key %U", unformat_vnet_dev_rss_key, + &a.rss_key)) + ; + else if (unformat (input, "%U", unformat_vnet_sw_interface, + vnet_get_main (), &sw_if_index)) + sw_if_index_set = 1; + else + return clib_error_return (0, "unknown input `%U'", + format_unformat_error, input); + } + + if (sw_if_index_set == device_id_set) + return clib_error_return ( + 0, "please specify either interface name or port id"); + + if (sw_if_index_set) + { + vnet_dev_port_t *port = vnet_dev_get_port_from_sw_if_index (sw_if_index); + + if (port == 0) + return clib_error_return (0, "unsupported interface"); + a.port_id = port->port_id; + a.dev_index = port->dev->index; + } + else + { + vnet_dev_t *dev = vnet_dev_by_id (device_id); + if (!dev) + return clib_error_return (0, "please specify valid device id"); + a.dev_index = dev->index; + } + + rv = vnet_dev_api_port_set_rss_key (vm, &a); + + if (rv != VNET_DEV_OK) + return clib_error_return (0, "unable to set_rss_key: %U", + format_vnet_dev_rv, rv); + + return 0; +} + +VLIB_CLI_COMMAND (device_set_rss_key_cmd, static) = { + .path = "device set-rss-key", + .short_help = "device set-rss-key [] [port ] [dev " + "] [key ]", + .function = device_set_rss_key_cmd_fn, + .is_mp_safe = 1, +}; diff --git a/src/vnet/dev/config.c b/src/vnet/dev/config.c index 1e428a544c..9cb8e94683 100644 --- a/src/vnet/dev/config.c +++ b/src/vnet/dev/config.c @@ -15,34 +15,44 @@ VLIB_REGISTER_LOG_CLASS (dev_log, static) = { .subclass_name = "config", }; +typedef struct +{ + vnet_dev_api_create_port_if_args_t intf; + vnet_dev_api_port_set_rss_key_args_t rss_key_args; +} port_config_t; + static clib_error_t * vnet_dev_config_one_interface (vlib_main_t *vm, unformat_input_t *input, - vnet_dev_api_create_port_if_args_t *args) + port_config_t *args) { clib_error_t *err = 0; - log_debug (0, "port %u %U", args->port_id, format_unformat_input, input); + log_debug (0, "port %u %U", args->intf.port_id, format_unformat_input, + input); while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) { u32 n; - if (unformat (input, "name %U", unformat_c_string_array, args->intf_name, - sizeof (args->intf_name))) + if (unformat (input, "name %U", unformat_c_string_array, + args->intf.intf_name, sizeof (args->intf.intf_name))) ; else if (unformat (input, "num-rx-queues %u", &n)) - args->num_rx_queues = n; + args->intf.num_rx_queues = n; else if (unformat (input, "num-tx-queues %u", &n)) - args->num_tx_queues = n; + args->intf.num_tx_queues = n; else if (unformat (input, "rx-queue-size %u", &n)) - args->rx_queue_size = n; + args->intf.rx_queue_size = n; else if (unformat (input, "tx-queue-size %u", &n)) - args->tx_queue_size = n; + args->intf.tx_queue_size = n; + else if (unformat (input, "rss-key %U", unformat_vnet_dev_rss_key, + &args->rss_key_args.rss_key)) + ; else if (unformat (input, "flags %U", unformat_vnet_dev_port_flags, - &args->flags)) + &args->intf.flags)) ; else if (unformat (input, "args %U", unformat_single_quoted_string, - &args->args)) + &args->intf.args)) ; else { @@ -112,7 +122,6 @@ vnet_dev_config_driver_args (vlib_main_t *vm, unformat_input_t *input, return err; } - static clib_error_t * vnet_dev_config_one_device (vlib_main_t *vm, unformat_input_t *input, char *device_id) @@ -120,7 +129,7 @@ vnet_dev_config_one_device (vlib_main_t *vm, unformat_input_t *input, log_debug (0, "device %s %U", device_id, format_unformat_input, input); clib_error_t *err = 0; vnet_dev_api_attach_args_t args = {}; - vnet_dev_api_create_port_if_args_t *if_args_vec = 0, *if_args; + port_config_t *ports = 0, *p; while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) { @@ -139,10 +148,9 @@ vnet_dev_config_one_device (vlib_main_t *vm, unformat_input_t *input, else if (unformat (input, "port %u %U", &n, unformat_vlib_cli_sub_input, &sub_input)) { - vnet_dev_api_create_port_if_args_t *if_args; - vec_add2 (if_args_vec, if_args, 1); - if_args->port_id = n; - err = vnet_dev_config_one_interface (vm, &sub_input, if_args); + vec_add2 (ports, p, 1); + p->intf.port_id = n; + err = vnet_dev_config_one_interface (vm, &sub_input, p); unformat_free (&sub_input); if (err) break; @@ -165,12 +173,21 @@ vnet_dev_config_one_device (vlib_main_t *vm, unformat_input_t *input, if (rv == VNET_DEV_OK) { - vec_foreach (if_args, if_args_vec) + vec_foreach (p, ports) { - if_args->dev_index = args.dev_index; - rv = vnet_dev_api_create_port_if (vm, if_args); + p->intf.dev_index = args.dev_index; + rv = vnet_dev_api_create_port_if (vm, &p->intf); if (rv != VNET_DEV_OK) break; + if (p->rss_key_args.rss_key.length) + { + vnet_dev_api_port_set_rss_key_args_t *rka = &p->rss_key_args; + rka->port_id = p->intf.port_id; + rka->dev_index = p->intf.dev_index; + rv = vnet_dev_api_port_set_rss_key (vm, rka); + if (rv != VNET_DEV_OK) + break; + } } } @@ -179,7 +196,7 @@ vnet_dev_config_one_device (vlib_main_t *vm, unformat_input_t *input, format_vnet_dev_rv, rv, device_id); } - vec_free (if_args_vec); + vec_free (ports); return err; } diff --git a/src/vnet/dev/dev.h b/src/vnet/dev/dev.h index 7fcfd12e92..be6fa3ed40 100644 --- a/src/vnet/dev/dev.h +++ b/src/vnet/dev/dev.h @@ -152,7 +152,8 @@ typedef struct _ (ADD_RX_FLOW) \ _ (DEL_RX_FLOW) \ _ (GET_RX_FLOW_COUNTER) \ - _ (RESET_RX_FLOW_COUNTER) + _ (RESET_RX_FLOW_COUNTER) \ + _ (SET_RSS_KEY) typedef enum { @@ -174,6 +175,7 @@ typedef struct vnet_dev_port_cfg_change_req vnet_dev_hw_addr_t addr; u32 max_rx_frame_size; vnet_dev_queue_id_t queue_id; + vnet_dev_rss_key_t rss_key; struct { u32 flow_index; @@ -363,6 +365,7 @@ typedef struct vnet_dev_port u16 txq_sz; u16 rxq_sz; } intf; + vnet_dev_rss_key_t rss_key; CLIB_CACHE_LINE_ALIGN_MARK (data0); u8 data[]; @@ -490,6 +493,7 @@ typedef struct vnet_dev_arg_t *args; u16 data_size; void *initial_data; + vnet_dev_rss_key_t default_rss_key; } port; vnet_dev_node_t *rx_node; @@ -677,6 +681,7 @@ format_function_t format_vnet_dev_tx_queue_info; format_function_t format_vnet_dev_flow; unformat_function_t unformat_vnet_dev_flags; unformat_function_t unformat_vnet_dev_port_flags; +unformat_function_t unformat_vnet_dev_rss_key; typedef struct { diff --git a/src/vnet/dev/dev_funcs.h b/src/vnet/dev/dev_funcs.h index 5a3cd00ec8..d636f08d4a 100644 --- a/src/vnet/dev/dev_funcs.h +++ b/src/vnet/dev/dev_funcs.h @@ -92,6 +92,19 @@ vnet_dev_get_port_rx_node_idex (vnet_dev_port_t *port) return port->intf.rx_node_index; } +static_always_inline vnet_dev_port_t * +vnet_dev_get_port_from_sw_if_index (u32 sw_if_index) +{ + vnet_main_t *vnm = vnet_get_main (); + vnet_sw_interface_t *si; + + si = vnet_get_sw_interface_or_null (vnm, sw_if_index); + if (!si) + return 0; + + return vnet_dev_get_port_from_hw_if_index (si->hw_if_index); +} + static_always_inline vnet_dev_t * vnet_dev_by_index (u32 index) { diff --git a/src/vnet/dev/format.c b/src/vnet/dev/format.c index f599c0f8b8..ad739e4d8a 100644 --- a/src/vnet/dev/format.c +++ b/src/vnet/dev/format.c @@ -119,6 +119,10 @@ format_vnet_dev_port_info (u8 *s, va_list *args) s = format (s, " %U", format_vnet_dev_hw_addr, a); } } + if (port->rss_key.length) + s = format (s, "\n%URSS Key is %U", format_white_space, indent, + format_hex_bytes_no_wrap, port->rss_key.key, + port->rss_key.length); s = format (s, "\n%UMax RX frame size is %u (max supported %u)", format_white_space, indent, port->max_rx_frame_size, port->attr.max_supported_rx_frame_size); @@ -505,3 +509,25 @@ format_vnet_dev_flow (u8 *s, va_list *args) return s; } + +uword +unformat_vnet_dev_rss_key (unformat_input_t *input, va_list *args) +{ + vnet_dev_rss_key_t *k = va_arg (*args, vnet_dev_rss_key_t *); + u8 *v; + u32 len; + + if (!(unformat_user (input, unformat_hex_string, &v))) + return 0; + + len = vec_len (v); + if (len > sizeof (k->key)) + { + vec_free (v); + return 0; + } + + clib_memcpy (k->key, v, len); + k->length = len; + return 1; +} diff --git a/src/vnet/dev/port.c b/src/vnet/dev/port.c index 32ce52c7a4..9cffac2277 100644 --- a/src/vnet/dev/port.c +++ b/src/vnet/dev/port.c @@ -267,6 +267,9 @@ vnet_dev_port_add (vlib_main_t *vm, vnet_dev_t *dev, vnet_dev_port_id_t id, port->rx_node = *args->rx_node; port->tx_node = *args->tx_node; + if (port->attr.caps.rss && args->port.default_rss_key.length) + port->rss_key = args->port.default_rss_key; + if (args->port.args) for (vnet_dev_arg_t *a = args->port.args; a->type != VNET_DEV_ARG_END; a++) vec_add1 (port->args, *a); @@ -337,6 +340,11 @@ vnet_dev_port_cfg_change_req_validate (vlib_main_t *vm, vnet_dev_port_t *port, return VNET_DEV_ERR_NO_SUCH_ENTRY; break; + case VNET_DEV_PORT_CFG_SET_RSS_KEY: + if (!port->attr.caps.rss) + return VNET_DEV_ERR_NOT_SUPPORTED; + break; + default: break; } @@ -453,6 +461,10 @@ vnet_dev_port_cfg_change (vlib_main_t *vm, vnet_dev_port_t *port, } break; + case VNET_DEV_PORT_CFG_SET_RSS_KEY: + port->rss_key = req->rss_key; + break; + default: break; } diff --git a/src/vnet/dev/types.h b/src/vnet/dev/types.h index 006d18e5bc..c4301a135f 100644 --- a/src/vnet/dev/types.h +++ b/src/vnet/dev/types.h @@ -17,6 +17,11 @@ typedef struct vnet_dev vnet_dev_t; typedef struct vnet_dev_port vnet_dev_port_t; typedef struct vnet_dev_rx_queue vnet_dev_rx_queue_t; typedef struct vnet_dev_tx_queue vnet_dev_tx_queue_t; +typedef struct +{ + u8 key[48]; + u8 length; +} vnet_dev_rss_key_t; typedef enum { From 8c320255610f48d0728d6d8ff4f7ae1ba18f7215 Mon Sep 17 00:00:00 2001 From: Damjan Marion Date: Mon, 15 Sep 2025 19:36:24 +0200 Subject: [PATCH 268/271] octeon: add support for changing RSS key Change-Id: I855c394262b275d1f98d9bce1a3cd4a2411d88f7 Type: improvement Signed-off-by: Damjan Marion Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/toolkits/vpp/+/162763 Tested-by: sa_ip-toolkits-Jenkins Reviewed-by: Monendra Singh Kushwaha (cherry picked from commit 5a458f4fc5857461d6a54ca28679e630a9c3a60d) --- src/plugins/dev_octeon/init.c | 10 ++++++++++ src/plugins/dev_octeon/port.c | 27 +++++++++++++++++++-------- 2 files changed, 29 insertions(+), 8 deletions(-) diff --git a/src/plugins/dev_octeon/init.c b/src/plugins/dev_octeon/init.c index 8faed6b250..2c71888c0d 100644 --- a/src/plugins/dev_octeon/init.c +++ b/src/plugins/dev_octeon/init.c @@ -360,6 +360,16 @@ oct_init_nix (vlib_main_t *vm, vnet_dev_t *dev) .tx_offloads = { .ip4_cksum = 1, }, + + }, + .default_rss_key = { + .key = { + 0xfe, 0xed, 0x0b, 0xad, 0xfe, 0xed, 0x0b, 0xad, 0xad, 0x0b, 0xed, 0xfe, + 0xad, 0x0b, 0xed, 0xfe, 0x13, 0x57, 0x9b, 0xef, 0x24, 0x68, 0xac, 0x0e, + 0x91, 0x72, 0x53, 0x11, 0x82, 0x64, 0x20, 0x44, 0x12, 0xef, 0x34, 0xcd, + 0x56, 0xbc, 0x78, 0x9a, 0x9a, 0x78, 0xbc, 0x56, 0xcd, 0x34, 0xef, 0x12, + }, + .length = 48, }, .ops = { .init = oct_port_init, diff --git a/src/plugins/dev_octeon/port.c b/src/plugins/dev_octeon/port.c index 5366c7b0f2..dd976569ce 100644 --- a/src/plugins/dev_octeon/port.c +++ b/src/plugins/dev_octeon/port.c @@ -24,13 +24,6 @@ VLIB_REGISTER_LOG_CLASS (oct_log, static) = { .subclass_name = "port", }; -static const u8 default_rss_key[] = { - 0xfe, 0xed, 0x0b, 0xad, 0xfe, 0xed, 0x0b, 0xad, 0xad, 0x0b, 0xed, 0xfe, - 0xad, 0x0b, 0xed, 0xfe, 0x13, 0x57, 0x9b, 0xef, 0x24, 0x68, 0xac, 0x0e, - 0x91, 0x72, 0x53, 0x11, 0x82, 0x64, 0x20, 0x44, 0x12, 0xef, 0x34, 0xcd, - 0x56, 0xbc, 0x78, 0x9a, 0x9a, 0x78, 0xbc, 0x56, 0xcd, 0x34, 0xef, 0x12 -}; - static const u64 rxq_cfg = ROC_NIX_LF_RX_CFG_DIS_APAD | ROC_NIX_LF_RX_CFG_IP6_UDP_OPT | ROC_NIX_LF_RX_CFG_L2_LEN_ERR | ROC_NIX_LF_RX_CFG_DROP_RE | @@ -250,7 +243,7 @@ oct_port_init (vlib_main_t *vm, vnet_dev_port_t *port) return oct_roc_err (dev, rrv, "roc_nix_rss_default_setup() failed"); } - roc_nix_rss_key_set (nix, default_rss_key); + roc_nix_rss_key_set (nix, port->rss_key.key); cp->npc.roc_nix = nix; cp->npc.flow_prealloc_size = OCT_FLOW_PREALLOC_SIZE; @@ -815,6 +808,19 @@ oct_op_config_max_rx_len (vlib_main_t *vm, vnet_dev_port_t *port, return rv; } +vnet_dev_rv_t +oct_op_config_set_rss_key (vlib_main_t *vm, vnet_dev_port_t *port, + vnet_dev_rss_key_t *k) +{ + vnet_dev_t *dev = port->dev; + oct_device_t *cd = vnet_dev_get_data (dev); + vnet_dev_rv_t rv = VNET_DEV_OK; + + roc_nix_rss_key_set (cd->nix, k->key); + + return rv; +} + vnet_dev_rv_t oct_port_cfg_change_validate (vlib_main_t *vm, vnet_dev_port_t *port, vnet_dev_port_cfg_change_req_t *req) @@ -833,6 +839,7 @@ oct_port_cfg_change_validate (vlib_main_t *vm, vnet_dev_port_t *port, case VNET_DEV_PORT_CFG_CHANGE_PRIMARY_HW_ADDR: case VNET_DEV_PORT_CFG_ADD_SECONDARY_HW_ADDR: case VNET_DEV_PORT_CFG_REMOVE_SECONDARY_HW_ADDR: + case VNET_DEV_PORT_CFG_SET_RSS_KEY: break; case VNET_DEV_PORT_CFG_ADD_RX_FLOW: @@ -878,6 +885,10 @@ oct_port_cfg_change (vlib_main_t *vm, vnet_dev_port_t *port, rv = oct_op_config_max_rx_len (vm, port, req->max_rx_frame_size); break; + case VNET_DEV_PORT_CFG_SET_RSS_KEY: + rv = oct_op_config_set_rss_key (vm, port, &req->rss_key); + break; + case VNET_DEV_PORT_CFG_ADD_RX_FLOW: case VNET_DEV_PORT_CFG_DEL_RX_FLOW: case VNET_DEV_PORT_CFG_GET_RX_FLOW_COUNTER: From b1ad2381089437a11f1fae6038ed42a6c065365d Mon Sep 17 00:00:00 2001 From: Nagendra Puthane Date: Thu, 11 Sep 2025 11:33:26 +0530 Subject: [PATCH 269/271] ci: move package dispatch to dao repo Move Package dispatch to DAO repo. Run only CI for pushes to VPP repo. Signed-off-by: Nagendra Puthane --- .github/workflows/build-cn10k.yml | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/.github/workflows/build-cn10k.yml b/.github/workflows/build-cn10k.yml index 16695057f8..76511b1e5c 100644 --- a/.github/workflows/build-cn10k.yml +++ b/.github/workflows/build-cn10k.yml @@ -180,19 +180,4 @@ jobs: draft: false tag_name: vpp-${{ steps.artifacts.outputs.PKG_VERSION_NAME }}-cn10k-${{ steps.artifacts.outputs.MRVL_PKG_VERSION }}-${{ steps.artifacts.outputs.DISTRO }}-${{ steps.artifacts.outputs.TAG }} files: | - ${{ github.workspace }}/artifacts/vpp-${{ steps.artifacts.outputs.PKG_VERSION_NAME }}-cn10k${{ steps.artifacts.outputs.PKG_POSTFIX }}_${{ steps.artifacts.outputs.MRVL_PKG_VERSION }}_arm64.deb - - name: Dispatch package update event - if: ${{ github.event_name == 'push' }} - run: | - curl -L \ - -X POST \ - -H "Accept: application/vnd.github+json" \ - -H "Authorization: Bearer ${{ secrets.PPA_REPO_SECRET }}" \ - -H "X-GitHub-Api-Version: 2022-11-28" \ - https://api.github.com/repos/marvellembeddedprocessors/packages/dispatches \ - -d '{"event_type":"dispatch-event", "client_payload": {"package" : "vpp", - "tag": "vpp-${{ steps.artifacts.outputs.PKG_VERSION_NAME }}-cn10k-${{ steps.artifacts.outputs.MRVL_PKG_VERSION }}-${{ steps.artifacts.outputs.DISTRO }}-${{ steps.artifacts.outputs.TAG }}", - "dpdk_tag" : "dpdk-cn10k-${{ steps.version.outputs.DPDK_BASE_PKG_VERSION }}_${{ steps.version.outputs.DPDK_PKG_VERSION }}-${{ steps.artifacts.outputs.DISTRO }}-${{ steps.version.outputs.DPDK_PKG_VERSION }}", - "has_dpdk" : "true", "distro" : "${{ steps.artifacts.outputs.DISTRO }}", - "platform" : "cn10k", - "devel": "${{ steps.artifacts.outputs.IS_DEVEL }}"}}' \ No newline at end of file + ${{ github.workspace }}/artifacts/vpp-${{ steps.artifacts.outputs.PKG_VERSION_NAME }}-cn10k${{ steps.artifacts.outputs.PKG_POSTFIX }}_${{ steps.artifacts.outputs.MRVL_PKG_VERSION }}_arm64.deb \ No newline at end of file From 09335d7c60386c5964078a94e7573098800e999f Mon Sep 17 00:00:00 2001 From: Nagendra T P Date: Thu, 25 Sep 2025 15:48:34 +0530 Subject: [PATCH 270/271] ci: fix cron schedule for devel runners - VPP scheduled cron job only picks stable version name. - Treat HEAD as "release" only if a release tag (YY.MM.0) points at it. Signed-off-by: Nagendra T P --- .github/workflows/build-cn10k.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-cn10k.yml b/.github/workflows/build-cn10k.yml index 76511b1e5c..70576be7c0 100644 --- a/.github/workflows/build-cn10k.yml +++ b/.github/workflows/build-cn10k.yml @@ -39,8 +39,14 @@ jobs: run: | mkdir -p "${PWD}/artifacts" git tag --points-at HEAD > /tmp/tags - [ -s /tmp/tags ] && PKG_POSTFIX= || PKG_POSTFIX=-devel - [ -s /tmp/tags ] && NIGHTLY=false || NIGHTLY=true + # Treat HEAD as "release" only if a release tag (YY.MM.0) points at it. + if git tag --points-at HEAD --list '[0-9][0-9].[0-9][0-9].0' | grep -q .; then + PKG_POSTFIX= + NIGHTLY=false + else + PKG_POSTFIX=-devel + NIGHTLY=true + fi echo "PKG_VERSION_NAME=`./src/scripts/version | awk -F '-' '{print $1}'`" >> "${PWD}/artifacts/env" echo "MRVL_PKG_VERSION=`cat MRVL_VERSION`" >> "${PWD}/artifacts/env" echo "CPT_PKG_VERSION=`cat DEP_PKG_VERSION | grep CPT_PKG_VERSION | awk -F'=' '{print $2}'`" >> "${PWD}/artifacts/env" From 9bcbd7c3778f0704fd33cf11a9f8aac007bcaf9d Mon Sep 17 00:00:00 2001 From: like-a-bus Date: Tue, 12 May 2026 09:25:00 +0000 Subject: [PATCH 271/271] linux-cp: fix interface ipip parser in xfrm-nl plugin The interface type parser in lcp_xfrm_itf_pair_config() only handled 'interface ipsec' and silently ignored 'interface ipip', causing the plugin to always create ipsec-itf even when ipip was requested in startup.conf. Additionally, the original parser used 'unformat ... %s' which returns a vec without null-terminator, making clib_strcmp() comparisons unreliable. This patch replaces the parser with direct unformat literals for both ipsec and ipip variants. This fixes the missing ipip handling and removes the vec/cstring ambiguity. Unused 'tunnel_name' variable is also dropped. Tested with strongSwan 5.9.13 + linux-xfrm-nl in route-based mode. IPIP tunnel is correctly created on NEWSA notification and IPsec encryption verified via packet trace (esp4-encrypt-tun node) and pcap capture on the WAN interface. Without this fix, 'interface ipip' in startup.conf is silently ignored and route-mode falls back to ipsec-itf creation, which has separate unresolved issues with inner-protocol mapping in decrypt path. --- src/plugins/linux-cp/lcp_xfrm_nl.c | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/src/plugins/linux-cp/lcp_xfrm_nl.c b/src/plugins/linux-cp/lcp_xfrm_nl.c index 4eabf40e6a..f17ff54325 100644 --- a/src/plugins/linux-cp/lcp_xfrm_nl.c +++ b/src/plugins/linux-cp/lcp_xfrm_nl.c @@ -530,7 +530,6 @@ static clib_error_t * lcp_xfrm_itf_pair_config (vlib_main_t *vm, unformat_input_t *input) { u32 buf_size, batch_size, batch_delay_ms; - char *tunnel_name = NULL; while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) { @@ -542,13 +541,10 @@ lcp_xfrm_itf_pair_config (vlib_main_t *vm, unformat_input_t *input) lcp_xfrm_nl_set_batch_size (batch_size); else if (unformat (input, "nl-batch-delay-ms %u", &batch_delay_ms)) lcp_xfrm_nl_set_batch_delay (batch_delay_ms); - else if (unformat (input, "interface %s", tunnel_name)) - { - if (!clib_strcmp (tunnel_name, "ipsec")) - nm->interface_type = NL_INTERFACE_TYPE_IPSEC; - - vec_free (tunnel_name); - } + else if (unformat (input, "interface ipsec")) + nm->interface_type = NL_INTERFACE_TYPE_IPSEC; + else if (unformat (input, "interface ipip")) + nm->interface_type = NL_INTERFACE_TYPE_IPIP; else return clib_error_return (0, "invalid netlink option: %U", format_unformat_error, input);