199a2dd95SBruce Richardson /* SPDX-License-Identifier: BSD-3-Clause 299a2dd95SBruce Richardson * Copyright(c) 2010-2018 Intel Corporation 399a2dd95SBruce Richardson */ 499a2dd95SBruce Richardson #include <stdlib.h> 599a2dd95SBruce Richardson #include <string.h> 699a2dd95SBruce Richardson 799a2dd95SBruce Richardson #include <rte_common.h> 899a2dd95SBruce Richardson #include <rte_byteorder.h> 999a2dd95SBruce Richardson #include <rte_cycles.h> 1099a2dd95SBruce Richardson #include <rte_malloc.h> 1199a2dd95SBruce Richardson #include <rte_memcpy.h> 1299a2dd95SBruce Richardson #include <rte_ether.h> 1399a2dd95SBruce Richardson #include <rte_ip.h> 1499a2dd95SBruce Richardson #include <rte_tcp.h> 1599a2dd95SBruce Richardson #include <rte_udp.h> 1699a2dd95SBruce Richardson #include <rte_vxlan.h> 1799a2dd95SBruce Richardson #include <rte_cryptodev.h> 1899a2dd95SBruce Richardson 1999a2dd95SBruce Richardson #include "rte_table_action.h" 2099a2dd95SBruce Richardson 2199a2dd95SBruce Richardson #define rte_htons rte_cpu_to_be_16 2299a2dd95SBruce Richardson #define rte_htonl rte_cpu_to_be_32 2399a2dd95SBruce Richardson 2499a2dd95SBruce Richardson #define rte_ntohs rte_be_to_cpu_16 2599a2dd95SBruce Richardson #define rte_ntohl rte_be_to_cpu_32 2699a2dd95SBruce Richardson 2799a2dd95SBruce Richardson /** 2899a2dd95SBruce Richardson * RTE_TABLE_ACTION_FWD 2999a2dd95SBruce Richardson */ 3099a2dd95SBruce Richardson #define fwd_data rte_pipeline_table_entry 3199a2dd95SBruce Richardson 3299a2dd95SBruce Richardson static int 3399a2dd95SBruce Richardson fwd_apply(struct fwd_data *data, 3499a2dd95SBruce Richardson struct rte_table_action_fwd_params *p) 3599a2dd95SBruce Richardson { 3699a2dd95SBruce Richardson data->action = p->action; 3799a2dd95SBruce Richardson 3899a2dd95SBruce Richardson if (p->action == RTE_PIPELINE_ACTION_PORT) 3999a2dd95SBruce Richardson data->port_id = p->id; 4099a2dd95SBruce Richardson 4199a2dd95SBruce Richardson if (p->action == RTE_PIPELINE_ACTION_TABLE) 4299a2dd95SBruce Richardson data->table_id = p->id; 4399a2dd95SBruce Richardson 4499a2dd95SBruce Richardson return 0; 4599a2dd95SBruce Richardson } 4699a2dd95SBruce Richardson 4799a2dd95SBruce Richardson /** 4899a2dd95SBruce Richardson * RTE_TABLE_ACTION_LB 4999a2dd95SBruce Richardson */ 5099a2dd95SBruce Richardson static int 5199a2dd95SBruce Richardson lb_cfg_check(struct rte_table_action_lb_config *cfg) 5299a2dd95SBruce Richardson { 5399a2dd95SBruce Richardson if ((cfg == NULL) || 5499a2dd95SBruce Richardson (cfg->key_size < RTE_TABLE_ACTION_LB_KEY_SIZE_MIN) || 5599a2dd95SBruce Richardson (cfg->key_size > RTE_TABLE_ACTION_LB_KEY_SIZE_MAX) || 5699a2dd95SBruce Richardson (!rte_is_power_of_2(cfg->key_size)) || 5799a2dd95SBruce Richardson (cfg->f_hash == NULL)) 5899a2dd95SBruce Richardson return -1; 5999a2dd95SBruce Richardson 6099a2dd95SBruce Richardson return 0; 6199a2dd95SBruce Richardson } 6299a2dd95SBruce Richardson 63*fba98755SAndre Muezerie struct __rte_packed_begin lb_data { 6499a2dd95SBruce Richardson uint32_t out[RTE_TABLE_ACTION_LB_TABLE_SIZE]; 65*fba98755SAndre Muezerie } __rte_packed_end; 6699a2dd95SBruce Richardson 6799a2dd95SBruce Richardson static int 6899a2dd95SBruce Richardson lb_apply(struct lb_data *data, 6999a2dd95SBruce Richardson struct rte_table_action_lb_params *p) 7099a2dd95SBruce Richardson { 7199a2dd95SBruce Richardson memcpy(data->out, p->out, sizeof(data->out)); 7299a2dd95SBruce Richardson 7399a2dd95SBruce Richardson return 0; 7499a2dd95SBruce Richardson } 7599a2dd95SBruce Richardson 7699a2dd95SBruce Richardson static __rte_always_inline void 7799a2dd95SBruce Richardson pkt_work_lb(struct rte_mbuf *mbuf, 7899a2dd95SBruce Richardson struct lb_data *data, 7999a2dd95SBruce Richardson struct rte_table_action_lb_config *cfg) 8099a2dd95SBruce Richardson { 8199a2dd95SBruce Richardson uint8_t *pkt_key = RTE_MBUF_METADATA_UINT8_PTR(mbuf, cfg->key_offset); 8299a2dd95SBruce Richardson uint32_t *out = RTE_MBUF_METADATA_UINT32_PTR(mbuf, cfg->out_offset); 8399a2dd95SBruce Richardson uint64_t digest, pos; 8499a2dd95SBruce Richardson uint32_t out_val; 8599a2dd95SBruce Richardson 8699a2dd95SBruce Richardson digest = cfg->f_hash(pkt_key, 8799a2dd95SBruce Richardson cfg->key_mask, 8899a2dd95SBruce Richardson cfg->key_size, 8999a2dd95SBruce Richardson cfg->seed); 9099a2dd95SBruce Richardson pos = digest & (RTE_TABLE_ACTION_LB_TABLE_SIZE - 1); 9199a2dd95SBruce Richardson out_val = data->out[pos]; 9299a2dd95SBruce Richardson 9399a2dd95SBruce Richardson *out = out_val; 9499a2dd95SBruce Richardson } 9599a2dd95SBruce Richardson 9699a2dd95SBruce Richardson /** 9799a2dd95SBruce Richardson * RTE_TABLE_ACTION_MTR 9899a2dd95SBruce Richardson */ 9999a2dd95SBruce Richardson static int 10099a2dd95SBruce Richardson mtr_cfg_check(struct rte_table_action_mtr_config *mtr) 10199a2dd95SBruce Richardson { 10299a2dd95SBruce Richardson if ((mtr->alg == RTE_TABLE_ACTION_METER_SRTCM) || 10399a2dd95SBruce Richardson ((mtr->n_tc != 1) && (mtr->n_tc != 4)) || 10499a2dd95SBruce Richardson (mtr->n_bytes_enabled != 0)) 10599a2dd95SBruce Richardson return -ENOTSUP; 10699a2dd95SBruce Richardson return 0; 10799a2dd95SBruce Richardson } 10899a2dd95SBruce Richardson 10999a2dd95SBruce Richardson struct mtr_trtcm_data { 11099a2dd95SBruce Richardson struct rte_meter_trtcm trtcm; 11199a2dd95SBruce Richardson uint64_t stats[RTE_COLORS]; 112eca9f4f8SBruce Richardson }; 11399a2dd95SBruce Richardson 11499a2dd95SBruce Richardson #define MTR_TRTCM_DATA_METER_PROFILE_ID_GET(data) \ 11599a2dd95SBruce Richardson (((data)->stats[RTE_COLOR_GREEN] & 0xF8LLU) >> 3) 11699a2dd95SBruce Richardson 11799a2dd95SBruce Richardson static void 11899a2dd95SBruce Richardson mtr_trtcm_data_meter_profile_id_set(struct mtr_trtcm_data *data, 11999a2dd95SBruce Richardson uint32_t profile_id) 12099a2dd95SBruce Richardson { 12199a2dd95SBruce Richardson data->stats[RTE_COLOR_GREEN] &= ~0xF8LLU; 12299a2dd95SBruce Richardson data->stats[RTE_COLOR_GREEN] |= (profile_id % 32) << 3; 12399a2dd95SBruce Richardson } 12499a2dd95SBruce Richardson 12599a2dd95SBruce Richardson #define MTR_TRTCM_DATA_POLICER_ACTION_DROP_GET(data, color)\ 12699a2dd95SBruce Richardson (((data)->stats[(color)] & 4LLU) >> 2) 12799a2dd95SBruce Richardson 12899a2dd95SBruce Richardson #define MTR_TRTCM_DATA_POLICER_ACTION_COLOR_GET(data, color)\ 12999a2dd95SBruce Richardson ((enum rte_color)((data)->stats[(color)] & 3LLU)) 13099a2dd95SBruce Richardson 13199a2dd95SBruce Richardson static void 13299a2dd95SBruce Richardson mtr_trtcm_data_policer_action_set(struct mtr_trtcm_data *data, 13399a2dd95SBruce Richardson enum rte_color color, 13499a2dd95SBruce Richardson enum rte_table_action_policer action) 13599a2dd95SBruce Richardson { 13699a2dd95SBruce Richardson if (action == RTE_TABLE_ACTION_POLICER_DROP) { 13799a2dd95SBruce Richardson data->stats[color] |= 4LLU; 13899a2dd95SBruce Richardson } else { 13999a2dd95SBruce Richardson data->stats[color] &= ~7LLU; 14099a2dd95SBruce Richardson data->stats[color] |= color & 3LLU; 14199a2dd95SBruce Richardson } 14299a2dd95SBruce Richardson } 14399a2dd95SBruce Richardson 14499a2dd95SBruce Richardson static uint64_t 14599a2dd95SBruce Richardson mtr_trtcm_data_stats_get(struct mtr_trtcm_data *data, 14699a2dd95SBruce Richardson enum rte_color color) 14799a2dd95SBruce Richardson { 14899a2dd95SBruce Richardson return data->stats[color] >> 8; 14999a2dd95SBruce Richardson } 15099a2dd95SBruce Richardson 15199a2dd95SBruce Richardson static void 15299a2dd95SBruce Richardson mtr_trtcm_data_stats_reset(struct mtr_trtcm_data *data, 15399a2dd95SBruce Richardson enum rte_color color) 15499a2dd95SBruce Richardson { 15599a2dd95SBruce Richardson data->stats[color] &= 0xFFLU; 15699a2dd95SBruce Richardson } 15799a2dd95SBruce Richardson 15899a2dd95SBruce Richardson #define MTR_TRTCM_DATA_STATS_INC(data, color) \ 15999a2dd95SBruce Richardson ((data)->stats[(color)] += (1LLU << 8)) 16099a2dd95SBruce Richardson 16199a2dd95SBruce Richardson static size_t 16299a2dd95SBruce Richardson mtr_data_size(struct rte_table_action_mtr_config *mtr) 16399a2dd95SBruce Richardson { 16499a2dd95SBruce Richardson return mtr->n_tc * sizeof(struct mtr_trtcm_data); 16599a2dd95SBruce Richardson } 16699a2dd95SBruce Richardson 16799a2dd95SBruce Richardson struct dscp_table_entry_data { 16899a2dd95SBruce Richardson enum rte_color color; 16999a2dd95SBruce Richardson uint16_t tc; 17099a2dd95SBruce Richardson uint16_t tc_queue; 17199a2dd95SBruce Richardson }; 17299a2dd95SBruce Richardson 17399a2dd95SBruce Richardson struct dscp_table_data { 17499a2dd95SBruce Richardson struct dscp_table_entry_data entry[64]; 17599a2dd95SBruce Richardson }; 17699a2dd95SBruce Richardson 17799a2dd95SBruce Richardson struct meter_profile_data { 17899a2dd95SBruce Richardson struct rte_meter_trtcm_profile profile; 17999a2dd95SBruce Richardson uint32_t profile_id; 18099a2dd95SBruce Richardson int valid; 18199a2dd95SBruce Richardson }; 18299a2dd95SBruce Richardson 18399a2dd95SBruce Richardson static struct meter_profile_data * 18499a2dd95SBruce Richardson meter_profile_data_find(struct meter_profile_data *mp, 18599a2dd95SBruce Richardson uint32_t mp_size, 18699a2dd95SBruce Richardson uint32_t profile_id) 18799a2dd95SBruce Richardson { 18899a2dd95SBruce Richardson uint32_t i; 18999a2dd95SBruce Richardson 19099a2dd95SBruce Richardson for (i = 0; i < mp_size; i++) { 19199a2dd95SBruce Richardson struct meter_profile_data *mp_data = &mp[i]; 19299a2dd95SBruce Richardson 19399a2dd95SBruce Richardson if (mp_data->valid && (mp_data->profile_id == profile_id)) 19499a2dd95SBruce Richardson return mp_data; 19599a2dd95SBruce Richardson } 19699a2dd95SBruce Richardson 19799a2dd95SBruce Richardson return NULL; 19899a2dd95SBruce Richardson } 19999a2dd95SBruce Richardson 20099a2dd95SBruce Richardson static struct meter_profile_data * 20199a2dd95SBruce Richardson meter_profile_data_find_unused(struct meter_profile_data *mp, 20299a2dd95SBruce Richardson uint32_t mp_size) 20399a2dd95SBruce Richardson { 20499a2dd95SBruce Richardson uint32_t i; 20599a2dd95SBruce Richardson 20699a2dd95SBruce Richardson for (i = 0; i < mp_size; i++) { 20799a2dd95SBruce Richardson struct meter_profile_data *mp_data = &mp[i]; 20899a2dd95SBruce Richardson 20999a2dd95SBruce Richardson if (!mp_data->valid) 21099a2dd95SBruce Richardson return mp_data; 21199a2dd95SBruce Richardson } 21299a2dd95SBruce Richardson 21399a2dd95SBruce Richardson return NULL; 21499a2dd95SBruce Richardson } 21599a2dd95SBruce Richardson 21699a2dd95SBruce Richardson static int 21799a2dd95SBruce Richardson mtr_apply_check(struct rte_table_action_mtr_params *p, 21899a2dd95SBruce Richardson struct rte_table_action_mtr_config *cfg, 21999a2dd95SBruce Richardson struct meter_profile_data *mp, 22099a2dd95SBruce Richardson uint32_t mp_size) 22199a2dd95SBruce Richardson { 22299a2dd95SBruce Richardson uint32_t i; 22399a2dd95SBruce Richardson 22499a2dd95SBruce Richardson if (p->tc_mask > RTE_LEN2MASK(cfg->n_tc, uint32_t)) 22599a2dd95SBruce Richardson return -EINVAL; 22699a2dd95SBruce Richardson 22799a2dd95SBruce Richardson for (i = 0; i < RTE_TABLE_ACTION_TC_MAX; i++) { 22899a2dd95SBruce Richardson struct rte_table_action_mtr_tc_params *p_tc = &p->mtr[i]; 22999a2dd95SBruce Richardson struct meter_profile_data *mp_data; 23099a2dd95SBruce Richardson 23199a2dd95SBruce Richardson if ((p->tc_mask & (1LLU << i)) == 0) 23299a2dd95SBruce Richardson continue; 23399a2dd95SBruce Richardson 23499a2dd95SBruce Richardson mp_data = meter_profile_data_find(mp, 23599a2dd95SBruce Richardson mp_size, 23699a2dd95SBruce Richardson p_tc->meter_profile_id); 23799a2dd95SBruce Richardson if (!mp_data) 23899a2dd95SBruce Richardson return -EINVAL; 23999a2dd95SBruce Richardson } 24099a2dd95SBruce Richardson 24199a2dd95SBruce Richardson return 0; 24299a2dd95SBruce Richardson } 24399a2dd95SBruce Richardson 24499a2dd95SBruce Richardson static int 24599a2dd95SBruce Richardson mtr_apply(struct mtr_trtcm_data *data, 24699a2dd95SBruce Richardson struct rte_table_action_mtr_params *p, 24799a2dd95SBruce Richardson struct rte_table_action_mtr_config *cfg, 24899a2dd95SBruce Richardson struct meter_profile_data *mp, 24999a2dd95SBruce Richardson uint32_t mp_size) 25099a2dd95SBruce Richardson { 25199a2dd95SBruce Richardson uint32_t i; 25299a2dd95SBruce Richardson int status; 25399a2dd95SBruce Richardson 25499a2dd95SBruce Richardson /* Check input arguments */ 25599a2dd95SBruce Richardson status = mtr_apply_check(p, cfg, mp, mp_size); 25699a2dd95SBruce Richardson if (status) 25799a2dd95SBruce Richardson return status; 25899a2dd95SBruce Richardson 25999a2dd95SBruce Richardson /* Apply */ 26099a2dd95SBruce Richardson for (i = 0; i < RTE_TABLE_ACTION_TC_MAX; i++) { 26199a2dd95SBruce Richardson struct rte_table_action_mtr_tc_params *p_tc = &p->mtr[i]; 26299a2dd95SBruce Richardson struct mtr_trtcm_data *data_tc = &data[i]; 26399a2dd95SBruce Richardson struct meter_profile_data *mp_data; 26499a2dd95SBruce Richardson 26599a2dd95SBruce Richardson if ((p->tc_mask & (1LLU << i)) == 0) 26699a2dd95SBruce Richardson continue; 26799a2dd95SBruce Richardson 26899a2dd95SBruce Richardson /* Find profile */ 26999a2dd95SBruce Richardson mp_data = meter_profile_data_find(mp, 27099a2dd95SBruce Richardson mp_size, 27199a2dd95SBruce Richardson p_tc->meter_profile_id); 27299a2dd95SBruce Richardson if (!mp_data) 27399a2dd95SBruce Richardson return -EINVAL; 27499a2dd95SBruce Richardson 27599a2dd95SBruce Richardson memset(data_tc, 0, sizeof(*data_tc)); 27699a2dd95SBruce Richardson 27799a2dd95SBruce Richardson /* Meter object */ 27899a2dd95SBruce Richardson status = rte_meter_trtcm_config(&data_tc->trtcm, 27999a2dd95SBruce Richardson &mp_data->profile); 28099a2dd95SBruce Richardson if (status) 28199a2dd95SBruce Richardson return status; 28299a2dd95SBruce Richardson 28399a2dd95SBruce Richardson /* Meter profile */ 28499a2dd95SBruce Richardson mtr_trtcm_data_meter_profile_id_set(data_tc, 28599a2dd95SBruce Richardson mp_data - mp); 28699a2dd95SBruce Richardson 28799a2dd95SBruce Richardson /* Policer actions */ 28899a2dd95SBruce Richardson mtr_trtcm_data_policer_action_set(data_tc, 28999a2dd95SBruce Richardson RTE_COLOR_GREEN, 29099a2dd95SBruce Richardson p_tc->policer[RTE_COLOR_GREEN]); 29199a2dd95SBruce Richardson 29299a2dd95SBruce Richardson mtr_trtcm_data_policer_action_set(data_tc, 29399a2dd95SBruce Richardson RTE_COLOR_YELLOW, 29499a2dd95SBruce Richardson p_tc->policer[RTE_COLOR_YELLOW]); 29599a2dd95SBruce Richardson 29699a2dd95SBruce Richardson mtr_trtcm_data_policer_action_set(data_tc, 29799a2dd95SBruce Richardson RTE_COLOR_RED, 29899a2dd95SBruce Richardson p_tc->policer[RTE_COLOR_RED]); 29999a2dd95SBruce Richardson } 30099a2dd95SBruce Richardson 30199a2dd95SBruce Richardson return 0; 30299a2dd95SBruce Richardson } 30399a2dd95SBruce Richardson 30499a2dd95SBruce Richardson static __rte_always_inline uint64_t 30599a2dd95SBruce Richardson pkt_work_mtr(struct rte_mbuf *mbuf, 30699a2dd95SBruce Richardson struct mtr_trtcm_data *data, 30799a2dd95SBruce Richardson struct dscp_table_data *dscp_table, 30899a2dd95SBruce Richardson struct meter_profile_data *mp, 30999a2dd95SBruce Richardson uint64_t time, 31099a2dd95SBruce Richardson uint32_t dscp, 31199a2dd95SBruce Richardson uint16_t total_length) 31299a2dd95SBruce Richardson { 31399a2dd95SBruce Richardson uint64_t drop_mask; 31499a2dd95SBruce Richardson struct dscp_table_entry_data *dscp_entry = &dscp_table->entry[dscp]; 31599a2dd95SBruce Richardson enum rte_color color_in, color_meter, color_policer; 31699a2dd95SBruce Richardson uint32_t tc, mp_id; 31799a2dd95SBruce Richardson 31899a2dd95SBruce Richardson tc = dscp_entry->tc; 31999a2dd95SBruce Richardson color_in = dscp_entry->color; 32099a2dd95SBruce Richardson data += tc; 32199a2dd95SBruce Richardson mp_id = MTR_TRTCM_DATA_METER_PROFILE_ID_GET(data); 32299a2dd95SBruce Richardson 32399a2dd95SBruce Richardson /* Meter */ 32499a2dd95SBruce Richardson color_meter = rte_meter_trtcm_color_aware_check( 32599a2dd95SBruce Richardson &data->trtcm, 32699a2dd95SBruce Richardson &mp[mp_id].profile, 32799a2dd95SBruce Richardson time, 32899a2dd95SBruce Richardson total_length, 32999a2dd95SBruce Richardson color_in); 33099a2dd95SBruce Richardson 33199a2dd95SBruce Richardson /* Stats */ 33299a2dd95SBruce Richardson MTR_TRTCM_DATA_STATS_INC(data, color_meter); 33399a2dd95SBruce Richardson 33499a2dd95SBruce Richardson /* Police */ 33599a2dd95SBruce Richardson drop_mask = MTR_TRTCM_DATA_POLICER_ACTION_DROP_GET(data, color_meter); 33699a2dd95SBruce Richardson color_policer = 33799a2dd95SBruce Richardson MTR_TRTCM_DATA_POLICER_ACTION_COLOR_GET(data, color_meter); 33899a2dd95SBruce Richardson rte_mbuf_sched_color_set(mbuf, (uint8_t)color_policer); 33999a2dd95SBruce Richardson 34099a2dd95SBruce Richardson return drop_mask; 34199a2dd95SBruce Richardson } 34299a2dd95SBruce Richardson 34399a2dd95SBruce Richardson /** 34499a2dd95SBruce Richardson * RTE_TABLE_ACTION_TM 34599a2dd95SBruce Richardson */ 34699a2dd95SBruce Richardson static int 34799a2dd95SBruce Richardson tm_cfg_check(struct rte_table_action_tm_config *tm) 34899a2dd95SBruce Richardson { 34999a2dd95SBruce Richardson if ((tm->n_subports_per_port == 0) || 35099a2dd95SBruce Richardson (rte_is_power_of_2(tm->n_subports_per_port) == 0) || 35199a2dd95SBruce Richardson (tm->n_subports_per_port > UINT16_MAX) || 35299a2dd95SBruce Richardson (tm->n_pipes_per_subport == 0) || 35399a2dd95SBruce Richardson (rte_is_power_of_2(tm->n_pipes_per_subport) == 0)) 35499a2dd95SBruce Richardson return -ENOTSUP; 35599a2dd95SBruce Richardson 35699a2dd95SBruce Richardson return 0; 35799a2dd95SBruce Richardson } 35899a2dd95SBruce Richardson 359*fba98755SAndre Muezerie struct __rte_packed_begin tm_data { 36099a2dd95SBruce Richardson uint32_t queue_id; 36199a2dd95SBruce Richardson uint32_t reserved; 362*fba98755SAndre Muezerie } __rte_packed_end; 36399a2dd95SBruce Richardson 36499a2dd95SBruce Richardson static int 36599a2dd95SBruce Richardson tm_apply_check(struct rte_table_action_tm_params *p, 36699a2dd95SBruce Richardson struct rte_table_action_tm_config *cfg) 36799a2dd95SBruce Richardson { 36899a2dd95SBruce Richardson if ((p->subport_id >= cfg->n_subports_per_port) || 36999a2dd95SBruce Richardson (p->pipe_id >= cfg->n_pipes_per_subport)) 37099a2dd95SBruce Richardson return -EINVAL; 37199a2dd95SBruce Richardson 37299a2dd95SBruce Richardson return 0; 37399a2dd95SBruce Richardson } 37499a2dd95SBruce Richardson 37599a2dd95SBruce Richardson static int 37699a2dd95SBruce Richardson tm_apply(struct tm_data *data, 37799a2dd95SBruce Richardson struct rte_table_action_tm_params *p, 37899a2dd95SBruce Richardson struct rte_table_action_tm_config *cfg) 37999a2dd95SBruce Richardson { 38099a2dd95SBruce Richardson int status; 38199a2dd95SBruce Richardson 38299a2dd95SBruce Richardson /* Check input arguments */ 38399a2dd95SBruce Richardson status = tm_apply_check(p, cfg); 38499a2dd95SBruce Richardson if (status) 38599a2dd95SBruce Richardson return status; 38699a2dd95SBruce Richardson 38799a2dd95SBruce Richardson /* Apply */ 38899a2dd95SBruce Richardson data->queue_id = p->subport_id << 3893d4e27fdSDavid Marchand (rte_ctz32(cfg->n_pipes_per_subport) + 4) | 39099a2dd95SBruce Richardson p->pipe_id << 4; 39199a2dd95SBruce Richardson 39299a2dd95SBruce Richardson return 0; 39399a2dd95SBruce Richardson } 39499a2dd95SBruce Richardson 39599a2dd95SBruce Richardson static __rte_always_inline void 39699a2dd95SBruce Richardson pkt_work_tm(struct rte_mbuf *mbuf, 39799a2dd95SBruce Richardson struct tm_data *data, 39899a2dd95SBruce Richardson struct dscp_table_data *dscp_table, 39999a2dd95SBruce Richardson uint32_t dscp) 40099a2dd95SBruce Richardson { 40199a2dd95SBruce Richardson struct dscp_table_entry_data *dscp_entry = &dscp_table->entry[dscp]; 40299a2dd95SBruce Richardson uint32_t queue_id = data->queue_id | 40399a2dd95SBruce Richardson dscp_entry->tc_queue; 40499a2dd95SBruce Richardson rte_mbuf_sched_set(mbuf, queue_id, dscp_entry->tc, 40599a2dd95SBruce Richardson (uint8_t)dscp_entry->color); 40699a2dd95SBruce Richardson } 40799a2dd95SBruce Richardson 40899a2dd95SBruce Richardson /** 40999a2dd95SBruce Richardson * RTE_TABLE_ACTION_ENCAP 41099a2dd95SBruce Richardson */ 41199a2dd95SBruce Richardson static int 41299a2dd95SBruce Richardson encap_valid(enum rte_table_action_encap_type encap) 41399a2dd95SBruce Richardson { 41499a2dd95SBruce Richardson switch (encap) { 41599a2dd95SBruce Richardson case RTE_TABLE_ACTION_ENCAP_ETHER: 41699a2dd95SBruce Richardson case RTE_TABLE_ACTION_ENCAP_VLAN: 41799a2dd95SBruce Richardson case RTE_TABLE_ACTION_ENCAP_QINQ: 41899a2dd95SBruce Richardson case RTE_TABLE_ACTION_ENCAP_MPLS: 41999a2dd95SBruce Richardson case RTE_TABLE_ACTION_ENCAP_PPPOE: 42099a2dd95SBruce Richardson case RTE_TABLE_ACTION_ENCAP_VXLAN: 42199a2dd95SBruce Richardson case RTE_TABLE_ACTION_ENCAP_QINQ_PPPOE: 42299a2dd95SBruce Richardson return 1; 42399a2dd95SBruce Richardson default: 42499a2dd95SBruce Richardson return 0; 42599a2dd95SBruce Richardson } 42699a2dd95SBruce Richardson } 42799a2dd95SBruce Richardson 42899a2dd95SBruce Richardson static int 42999a2dd95SBruce Richardson encap_cfg_check(struct rte_table_action_encap_config *encap) 43099a2dd95SBruce Richardson { 43199a2dd95SBruce Richardson if ((encap->encap_mask == 0) || 4323d4e27fdSDavid Marchand (rte_popcount64(encap->encap_mask) != 1)) 43399a2dd95SBruce Richardson return -ENOTSUP; 43499a2dd95SBruce Richardson 43599a2dd95SBruce Richardson return 0; 43699a2dd95SBruce Richardson } 43799a2dd95SBruce Richardson 43899a2dd95SBruce Richardson struct encap_ether_data { 43999a2dd95SBruce Richardson struct rte_ether_hdr ether; 44099a2dd95SBruce Richardson }; 44199a2dd95SBruce Richardson 44299a2dd95SBruce Richardson #define VLAN(pcp, dei, vid) \ 44399a2dd95SBruce Richardson ((uint16_t)((((uint64_t)(pcp)) & 0x7LLU) << 13) | \ 44499a2dd95SBruce Richardson ((((uint64_t)(dei)) & 0x1LLU) << 12) | \ 44599a2dd95SBruce Richardson (((uint64_t)(vid)) & 0xFFFLLU)) \ 44699a2dd95SBruce Richardson 44799a2dd95SBruce Richardson struct encap_vlan_data { 44899a2dd95SBruce Richardson struct rte_ether_hdr ether; 44999a2dd95SBruce Richardson struct rte_vlan_hdr vlan; 45099a2dd95SBruce Richardson }; 45199a2dd95SBruce Richardson 45299a2dd95SBruce Richardson struct encap_qinq_data { 45399a2dd95SBruce Richardson struct rte_ether_hdr ether; 45499a2dd95SBruce Richardson struct rte_vlan_hdr svlan; 45599a2dd95SBruce Richardson struct rte_vlan_hdr cvlan; 45699a2dd95SBruce Richardson }; 45799a2dd95SBruce Richardson 45899a2dd95SBruce Richardson #define ETHER_TYPE_MPLS_UNICAST 0x8847 45999a2dd95SBruce Richardson 46099a2dd95SBruce Richardson #define ETHER_TYPE_MPLS_MULTICAST 0x8848 46199a2dd95SBruce Richardson 46299a2dd95SBruce Richardson #define MPLS(label, tc, s, ttl) \ 46399a2dd95SBruce Richardson ((uint32_t)(((((uint64_t)(label)) & 0xFFFFFLLU) << 12) |\ 46499a2dd95SBruce Richardson ((((uint64_t)(tc)) & 0x7LLU) << 9) | \ 46599a2dd95SBruce Richardson ((((uint64_t)(s)) & 0x1LLU) << 8) | \ 46699a2dd95SBruce Richardson (((uint64_t)(ttl)) & 0xFFLLU))) 46799a2dd95SBruce Richardson 468*fba98755SAndre Muezerie struct __rte_aligned(2) __rte_packed_begin encap_mpls_data { 46999a2dd95SBruce Richardson struct rte_ether_hdr ether; 47099a2dd95SBruce Richardson uint32_t mpls[RTE_TABLE_ACTION_MPLS_LABELS_MAX]; 47199a2dd95SBruce Richardson uint32_t mpls_count; 472*fba98755SAndre Muezerie } __rte_packed_end; 47399a2dd95SBruce Richardson 47499a2dd95SBruce Richardson #define PPP_PROTOCOL_IP 0x0021 47599a2dd95SBruce Richardson 47699a2dd95SBruce Richardson struct pppoe_ppp_hdr { 47799a2dd95SBruce Richardson uint16_t ver_type_code; 47899a2dd95SBruce Richardson uint16_t session_id; 47999a2dd95SBruce Richardson uint16_t length; 48099a2dd95SBruce Richardson uint16_t protocol; 48199a2dd95SBruce Richardson }; 48299a2dd95SBruce Richardson 48399a2dd95SBruce Richardson struct encap_pppoe_data { 48499a2dd95SBruce Richardson struct rte_ether_hdr ether; 48599a2dd95SBruce Richardson struct pppoe_ppp_hdr pppoe_ppp; 48699a2dd95SBruce Richardson }; 48799a2dd95SBruce Richardson 48899a2dd95SBruce Richardson #define IP_PROTO_UDP 17 48999a2dd95SBruce Richardson 490*fba98755SAndre Muezerie struct __rte_aligned(2) __rte_packed_begin encap_vxlan_ipv4_data { 49199a2dd95SBruce Richardson struct rte_ether_hdr ether; 49299a2dd95SBruce Richardson struct rte_ipv4_hdr ipv4; 49399a2dd95SBruce Richardson struct rte_udp_hdr udp; 49499a2dd95SBruce Richardson struct rte_vxlan_hdr vxlan; 495*fba98755SAndre Muezerie } __rte_packed_end; 49699a2dd95SBruce Richardson 497*fba98755SAndre Muezerie struct __rte_aligned(2) __rte_packed_begin encap_vxlan_ipv4_vlan_data { 49899a2dd95SBruce Richardson struct rte_ether_hdr ether; 49999a2dd95SBruce Richardson struct rte_vlan_hdr vlan; 50099a2dd95SBruce Richardson struct rte_ipv4_hdr ipv4; 50199a2dd95SBruce Richardson struct rte_udp_hdr udp; 50299a2dd95SBruce Richardson struct rte_vxlan_hdr vxlan; 503*fba98755SAndre Muezerie } __rte_packed_end; 50499a2dd95SBruce Richardson 505*fba98755SAndre Muezerie struct __rte_aligned(2) __rte_packed_begin encap_vxlan_ipv6_data { 50699a2dd95SBruce Richardson struct rte_ether_hdr ether; 50799a2dd95SBruce Richardson struct rte_ipv6_hdr ipv6; 50899a2dd95SBruce Richardson struct rte_udp_hdr udp; 50999a2dd95SBruce Richardson struct rte_vxlan_hdr vxlan; 510*fba98755SAndre Muezerie } __rte_packed_end; 51199a2dd95SBruce Richardson 512*fba98755SAndre Muezerie struct __rte_aligned(2) __rte_packed_begin encap_vxlan_ipv6_vlan_data { 51399a2dd95SBruce Richardson struct rte_ether_hdr ether; 51499a2dd95SBruce Richardson struct rte_vlan_hdr vlan; 51599a2dd95SBruce Richardson struct rte_ipv6_hdr ipv6; 51699a2dd95SBruce Richardson struct rte_udp_hdr udp; 51799a2dd95SBruce Richardson struct rte_vxlan_hdr vxlan; 518*fba98755SAndre Muezerie } __rte_packed_end; 51999a2dd95SBruce Richardson 520*fba98755SAndre Muezerie struct __rte_aligned(2) __rte_packed_begin encap_qinq_pppoe_data { 52199a2dd95SBruce Richardson struct rte_ether_hdr ether; 52299a2dd95SBruce Richardson struct rte_vlan_hdr svlan; 52399a2dd95SBruce Richardson struct rte_vlan_hdr cvlan; 52499a2dd95SBruce Richardson struct pppoe_ppp_hdr pppoe_ppp; 525*fba98755SAndre Muezerie } __rte_packed_end; 52699a2dd95SBruce Richardson 52799a2dd95SBruce Richardson static size_t 52899a2dd95SBruce Richardson encap_data_size(struct rte_table_action_encap_config *encap) 52999a2dd95SBruce Richardson { 53099a2dd95SBruce Richardson switch (encap->encap_mask) { 53199a2dd95SBruce Richardson case 1LLU << RTE_TABLE_ACTION_ENCAP_ETHER: 53299a2dd95SBruce Richardson return sizeof(struct encap_ether_data); 53399a2dd95SBruce Richardson 53499a2dd95SBruce Richardson case 1LLU << RTE_TABLE_ACTION_ENCAP_VLAN: 53599a2dd95SBruce Richardson return sizeof(struct encap_vlan_data); 53699a2dd95SBruce Richardson 53799a2dd95SBruce Richardson case 1LLU << RTE_TABLE_ACTION_ENCAP_QINQ: 53899a2dd95SBruce Richardson return sizeof(struct encap_qinq_data); 53999a2dd95SBruce Richardson 54099a2dd95SBruce Richardson case 1LLU << RTE_TABLE_ACTION_ENCAP_MPLS: 54199a2dd95SBruce Richardson return sizeof(struct encap_mpls_data); 54299a2dd95SBruce Richardson 54399a2dd95SBruce Richardson case 1LLU << RTE_TABLE_ACTION_ENCAP_PPPOE: 54499a2dd95SBruce Richardson return sizeof(struct encap_pppoe_data); 54599a2dd95SBruce Richardson 54699a2dd95SBruce Richardson case 1LLU << RTE_TABLE_ACTION_ENCAP_VXLAN: 54799a2dd95SBruce Richardson if (encap->vxlan.ip_version) 54899a2dd95SBruce Richardson if (encap->vxlan.vlan) 54999a2dd95SBruce Richardson return sizeof(struct encap_vxlan_ipv4_vlan_data); 55099a2dd95SBruce Richardson else 55199a2dd95SBruce Richardson return sizeof(struct encap_vxlan_ipv4_data); 55299a2dd95SBruce Richardson else 55399a2dd95SBruce Richardson if (encap->vxlan.vlan) 55499a2dd95SBruce Richardson return sizeof(struct encap_vxlan_ipv6_vlan_data); 55599a2dd95SBruce Richardson else 55699a2dd95SBruce Richardson return sizeof(struct encap_vxlan_ipv6_data); 55799a2dd95SBruce Richardson 55899a2dd95SBruce Richardson case 1LLU << RTE_TABLE_ACTION_ENCAP_QINQ_PPPOE: 55999a2dd95SBruce Richardson return sizeof(struct encap_qinq_pppoe_data); 56099a2dd95SBruce Richardson 56199a2dd95SBruce Richardson default: 56299a2dd95SBruce Richardson return 0; 56399a2dd95SBruce Richardson } 56499a2dd95SBruce Richardson } 56599a2dd95SBruce Richardson 56699a2dd95SBruce Richardson static int 56799a2dd95SBruce Richardson encap_apply_check(struct rte_table_action_encap_params *p, 56899a2dd95SBruce Richardson struct rte_table_action_encap_config *cfg) 56999a2dd95SBruce Richardson { 57099a2dd95SBruce Richardson if ((encap_valid(p->type) == 0) || 57199a2dd95SBruce Richardson ((cfg->encap_mask & (1LLU << p->type)) == 0)) 57299a2dd95SBruce Richardson return -EINVAL; 57399a2dd95SBruce Richardson 57499a2dd95SBruce Richardson switch (p->type) { 57599a2dd95SBruce Richardson case RTE_TABLE_ACTION_ENCAP_ETHER: 57699a2dd95SBruce Richardson return 0; 57799a2dd95SBruce Richardson 57899a2dd95SBruce Richardson case RTE_TABLE_ACTION_ENCAP_VLAN: 57999a2dd95SBruce Richardson return 0; 58099a2dd95SBruce Richardson 58199a2dd95SBruce Richardson case RTE_TABLE_ACTION_ENCAP_QINQ: 58299a2dd95SBruce Richardson return 0; 58399a2dd95SBruce Richardson 58499a2dd95SBruce Richardson case RTE_TABLE_ACTION_ENCAP_MPLS: 58599a2dd95SBruce Richardson if ((p->mpls.mpls_count == 0) || 58699a2dd95SBruce Richardson (p->mpls.mpls_count > RTE_TABLE_ACTION_MPLS_LABELS_MAX)) 58799a2dd95SBruce Richardson return -EINVAL; 58899a2dd95SBruce Richardson 58999a2dd95SBruce Richardson return 0; 59099a2dd95SBruce Richardson 59199a2dd95SBruce Richardson case RTE_TABLE_ACTION_ENCAP_PPPOE: 59299a2dd95SBruce Richardson return 0; 59399a2dd95SBruce Richardson 59499a2dd95SBruce Richardson case RTE_TABLE_ACTION_ENCAP_VXLAN: 59599a2dd95SBruce Richardson return 0; 59699a2dd95SBruce Richardson 59799a2dd95SBruce Richardson case RTE_TABLE_ACTION_ENCAP_QINQ_PPPOE: 59899a2dd95SBruce Richardson return 0; 59999a2dd95SBruce Richardson 60099a2dd95SBruce Richardson default: 60199a2dd95SBruce Richardson return -EINVAL; 60299a2dd95SBruce Richardson } 60399a2dd95SBruce Richardson } 60499a2dd95SBruce Richardson 60599a2dd95SBruce Richardson static int 60699a2dd95SBruce Richardson encap_ether_apply(void *data, 60799a2dd95SBruce Richardson struct rte_table_action_encap_params *p, 60899a2dd95SBruce Richardson struct rte_table_action_common_config *common_cfg) 60999a2dd95SBruce Richardson { 61099a2dd95SBruce Richardson struct encap_ether_data *d = data; 61199a2dd95SBruce Richardson uint16_t ethertype = (common_cfg->ip_version) ? 61299a2dd95SBruce Richardson RTE_ETHER_TYPE_IPV4 : 61399a2dd95SBruce Richardson RTE_ETHER_TYPE_IPV6; 61499a2dd95SBruce Richardson 61599a2dd95SBruce Richardson /* Ethernet */ 61604d43857SDmitry Kozlyuk rte_ether_addr_copy(&p->ether.ether.da, &d->ether.dst_addr); 61704d43857SDmitry Kozlyuk rte_ether_addr_copy(&p->ether.ether.sa, &d->ether.src_addr); 61899a2dd95SBruce Richardson d->ether.ether_type = rte_htons(ethertype); 61999a2dd95SBruce Richardson 62099a2dd95SBruce Richardson return 0; 62199a2dd95SBruce Richardson } 62299a2dd95SBruce Richardson 62399a2dd95SBruce Richardson static int 62499a2dd95SBruce Richardson encap_vlan_apply(void *data, 62599a2dd95SBruce Richardson struct rte_table_action_encap_params *p, 62699a2dd95SBruce Richardson struct rte_table_action_common_config *common_cfg) 62799a2dd95SBruce Richardson { 62899a2dd95SBruce Richardson struct encap_vlan_data *d = data; 62999a2dd95SBruce Richardson uint16_t ethertype = (common_cfg->ip_version) ? 63099a2dd95SBruce Richardson RTE_ETHER_TYPE_IPV4 : 63199a2dd95SBruce Richardson RTE_ETHER_TYPE_IPV6; 63299a2dd95SBruce Richardson 63399a2dd95SBruce Richardson /* Ethernet */ 63404d43857SDmitry Kozlyuk rte_ether_addr_copy(&p->vlan.ether.da, &d->ether.dst_addr); 63504d43857SDmitry Kozlyuk rte_ether_addr_copy(&p->vlan.ether.sa, &d->ether.src_addr); 63699a2dd95SBruce Richardson d->ether.ether_type = rte_htons(RTE_ETHER_TYPE_VLAN); 63799a2dd95SBruce Richardson 63899a2dd95SBruce Richardson /* VLAN */ 63999a2dd95SBruce Richardson d->vlan.vlan_tci = rte_htons(VLAN(p->vlan.vlan.pcp, 64099a2dd95SBruce Richardson p->vlan.vlan.dei, 64199a2dd95SBruce Richardson p->vlan.vlan.vid)); 64299a2dd95SBruce Richardson d->vlan.eth_proto = rte_htons(ethertype); 64399a2dd95SBruce Richardson 64499a2dd95SBruce Richardson return 0; 64599a2dd95SBruce Richardson } 64699a2dd95SBruce Richardson 64799a2dd95SBruce Richardson static int 64899a2dd95SBruce Richardson encap_qinq_apply(void *data, 64999a2dd95SBruce Richardson struct rte_table_action_encap_params *p, 65099a2dd95SBruce Richardson struct rte_table_action_common_config *common_cfg) 65199a2dd95SBruce Richardson { 65299a2dd95SBruce Richardson struct encap_qinq_data *d = data; 65399a2dd95SBruce Richardson uint16_t ethertype = (common_cfg->ip_version) ? 65499a2dd95SBruce Richardson RTE_ETHER_TYPE_IPV4 : 65599a2dd95SBruce Richardson RTE_ETHER_TYPE_IPV6; 65699a2dd95SBruce Richardson 65799a2dd95SBruce Richardson /* Ethernet */ 65804d43857SDmitry Kozlyuk rte_ether_addr_copy(&p->qinq.ether.da, &d->ether.dst_addr); 65904d43857SDmitry Kozlyuk rte_ether_addr_copy(&p->qinq.ether.sa, &d->ether.src_addr); 66099a2dd95SBruce Richardson d->ether.ether_type = rte_htons(RTE_ETHER_TYPE_QINQ); 66199a2dd95SBruce Richardson 66299a2dd95SBruce Richardson /* SVLAN */ 66399a2dd95SBruce Richardson d->svlan.vlan_tci = rte_htons(VLAN(p->qinq.svlan.pcp, 66499a2dd95SBruce Richardson p->qinq.svlan.dei, 66599a2dd95SBruce Richardson p->qinq.svlan.vid)); 66699a2dd95SBruce Richardson d->svlan.eth_proto = rte_htons(RTE_ETHER_TYPE_VLAN); 66799a2dd95SBruce Richardson 66899a2dd95SBruce Richardson /* CVLAN */ 66999a2dd95SBruce Richardson d->cvlan.vlan_tci = rte_htons(VLAN(p->qinq.cvlan.pcp, 67099a2dd95SBruce Richardson p->qinq.cvlan.dei, 67199a2dd95SBruce Richardson p->qinq.cvlan.vid)); 67299a2dd95SBruce Richardson d->cvlan.eth_proto = rte_htons(ethertype); 67399a2dd95SBruce Richardson 67499a2dd95SBruce Richardson return 0; 67599a2dd95SBruce Richardson } 67699a2dd95SBruce Richardson 67799a2dd95SBruce Richardson static int 67899a2dd95SBruce Richardson encap_qinq_pppoe_apply(void *data, 67999a2dd95SBruce Richardson struct rte_table_action_encap_params *p) 68099a2dd95SBruce Richardson { 68199a2dd95SBruce Richardson struct encap_qinq_pppoe_data *d = data; 68299a2dd95SBruce Richardson 68399a2dd95SBruce Richardson /* Ethernet */ 68404d43857SDmitry Kozlyuk rte_ether_addr_copy(&p->qinq.ether.da, &d->ether.dst_addr); 68504d43857SDmitry Kozlyuk rte_ether_addr_copy(&p->qinq.ether.sa, &d->ether.src_addr); 68699a2dd95SBruce Richardson d->ether.ether_type = rte_htons(RTE_ETHER_TYPE_VLAN); 68799a2dd95SBruce Richardson 68899a2dd95SBruce Richardson /* SVLAN */ 68999a2dd95SBruce Richardson d->svlan.vlan_tci = rte_htons(VLAN(p->qinq.svlan.pcp, 69099a2dd95SBruce Richardson p->qinq.svlan.dei, 69199a2dd95SBruce Richardson p->qinq.svlan.vid)); 69299a2dd95SBruce Richardson d->svlan.eth_proto = rte_htons(RTE_ETHER_TYPE_VLAN); 69399a2dd95SBruce Richardson 69499a2dd95SBruce Richardson /* CVLAN */ 69599a2dd95SBruce Richardson d->cvlan.vlan_tci = rte_htons(VLAN(p->qinq.cvlan.pcp, 69699a2dd95SBruce Richardson p->qinq.cvlan.dei, 69799a2dd95SBruce Richardson p->qinq.cvlan.vid)); 69899a2dd95SBruce Richardson d->cvlan.eth_proto = rte_htons(RTE_ETHER_TYPE_PPPOE_SESSION); 69999a2dd95SBruce Richardson 70099a2dd95SBruce Richardson /* PPPoE and PPP*/ 70199a2dd95SBruce Richardson d->pppoe_ppp.ver_type_code = rte_htons(0x1100); 70299a2dd95SBruce Richardson d->pppoe_ppp.session_id = rte_htons(p->qinq_pppoe.pppoe.session_id); 70399a2dd95SBruce Richardson d->pppoe_ppp.length = 0; /* not pre-computed */ 70499a2dd95SBruce Richardson d->pppoe_ppp.protocol = rte_htons(PPP_PROTOCOL_IP); 70599a2dd95SBruce Richardson 70699a2dd95SBruce Richardson return 0; 70799a2dd95SBruce Richardson } 70899a2dd95SBruce Richardson 70999a2dd95SBruce Richardson static int 71099a2dd95SBruce Richardson encap_mpls_apply(void *data, 71199a2dd95SBruce Richardson struct rte_table_action_encap_params *p) 71299a2dd95SBruce Richardson { 71399a2dd95SBruce Richardson struct encap_mpls_data *d = data; 71499a2dd95SBruce Richardson uint16_t ethertype = (p->mpls.unicast) ? 71599a2dd95SBruce Richardson ETHER_TYPE_MPLS_UNICAST : 71699a2dd95SBruce Richardson ETHER_TYPE_MPLS_MULTICAST; 71799a2dd95SBruce Richardson uint32_t i; 71899a2dd95SBruce Richardson 71999a2dd95SBruce Richardson /* Ethernet */ 72004d43857SDmitry Kozlyuk rte_ether_addr_copy(&p->mpls.ether.da, &d->ether.dst_addr); 72104d43857SDmitry Kozlyuk rte_ether_addr_copy(&p->mpls.ether.sa, &d->ether.src_addr); 72299a2dd95SBruce Richardson d->ether.ether_type = rte_htons(ethertype); 72399a2dd95SBruce Richardson 72499a2dd95SBruce Richardson /* MPLS */ 72599a2dd95SBruce Richardson for (i = 0; i < p->mpls.mpls_count - 1; i++) 72699a2dd95SBruce Richardson d->mpls[i] = rte_htonl(MPLS(p->mpls.mpls[i].label, 72799a2dd95SBruce Richardson p->mpls.mpls[i].tc, 72899a2dd95SBruce Richardson 0, 72999a2dd95SBruce Richardson p->mpls.mpls[i].ttl)); 73099a2dd95SBruce Richardson 73199a2dd95SBruce Richardson d->mpls[i] = rte_htonl(MPLS(p->mpls.mpls[i].label, 73299a2dd95SBruce Richardson p->mpls.mpls[i].tc, 73399a2dd95SBruce Richardson 1, 73499a2dd95SBruce Richardson p->mpls.mpls[i].ttl)); 73599a2dd95SBruce Richardson 73699a2dd95SBruce Richardson d->mpls_count = p->mpls.mpls_count; 73799a2dd95SBruce Richardson return 0; 73899a2dd95SBruce Richardson } 73999a2dd95SBruce Richardson 74099a2dd95SBruce Richardson static int 74199a2dd95SBruce Richardson encap_pppoe_apply(void *data, 74299a2dd95SBruce Richardson struct rte_table_action_encap_params *p) 74399a2dd95SBruce Richardson { 74499a2dd95SBruce Richardson struct encap_pppoe_data *d = data; 74599a2dd95SBruce Richardson 74699a2dd95SBruce Richardson /* Ethernet */ 74704d43857SDmitry Kozlyuk rte_ether_addr_copy(&p->pppoe.ether.da, &d->ether.dst_addr); 74804d43857SDmitry Kozlyuk rte_ether_addr_copy(&p->pppoe.ether.sa, &d->ether.src_addr); 74999a2dd95SBruce Richardson d->ether.ether_type = rte_htons(RTE_ETHER_TYPE_PPPOE_SESSION); 75099a2dd95SBruce Richardson 75199a2dd95SBruce Richardson /* PPPoE and PPP*/ 75299a2dd95SBruce Richardson d->pppoe_ppp.ver_type_code = rte_htons(0x1100); 75399a2dd95SBruce Richardson d->pppoe_ppp.session_id = rte_htons(p->pppoe.pppoe.session_id); 75499a2dd95SBruce Richardson d->pppoe_ppp.length = 0; /* not pre-computed */ 75599a2dd95SBruce Richardson d->pppoe_ppp.protocol = rte_htons(PPP_PROTOCOL_IP); 75699a2dd95SBruce Richardson 75799a2dd95SBruce Richardson return 0; 75899a2dd95SBruce Richardson } 75999a2dd95SBruce Richardson 76099a2dd95SBruce Richardson static int 76199a2dd95SBruce Richardson encap_vxlan_apply(void *data, 76299a2dd95SBruce Richardson struct rte_table_action_encap_params *p, 76399a2dd95SBruce Richardson struct rte_table_action_encap_config *cfg) 76499a2dd95SBruce Richardson { 76599a2dd95SBruce Richardson if ((p->vxlan.vxlan.vni > 0xFFFFFF) || 76699a2dd95SBruce Richardson (cfg->vxlan.ip_version && (p->vxlan.ipv4.dscp > 0x3F)) || 76799a2dd95SBruce Richardson (!cfg->vxlan.ip_version && (p->vxlan.ipv6.flow_label > 0xFFFFF)) || 76899a2dd95SBruce Richardson (!cfg->vxlan.ip_version && (p->vxlan.ipv6.dscp > 0x3F)) || 76999a2dd95SBruce Richardson (cfg->vxlan.vlan && (p->vxlan.vlan.vid > 0xFFF))) 77099a2dd95SBruce Richardson return -1; 77199a2dd95SBruce Richardson 77299a2dd95SBruce Richardson if (cfg->vxlan.ip_version) 77399a2dd95SBruce Richardson if (cfg->vxlan.vlan) { 77499a2dd95SBruce Richardson struct encap_vxlan_ipv4_vlan_data *d = data; 77599a2dd95SBruce Richardson 77699a2dd95SBruce Richardson /* Ethernet */ 77799a2dd95SBruce Richardson rte_ether_addr_copy(&p->vxlan.ether.da, 77804d43857SDmitry Kozlyuk &d->ether.dst_addr); 77999a2dd95SBruce Richardson rte_ether_addr_copy(&p->vxlan.ether.sa, 78004d43857SDmitry Kozlyuk &d->ether.src_addr); 78199a2dd95SBruce Richardson d->ether.ether_type = rte_htons(RTE_ETHER_TYPE_VLAN); 78299a2dd95SBruce Richardson 78399a2dd95SBruce Richardson /* VLAN */ 78499a2dd95SBruce Richardson d->vlan.vlan_tci = rte_htons(VLAN(p->vxlan.vlan.pcp, 78599a2dd95SBruce Richardson p->vxlan.vlan.dei, 78699a2dd95SBruce Richardson p->vxlan.vlan.vid)); 78799a2dd95SBruce Richardson d->vlan.eth_proto = rte_htons(RTE_ETHER_TYPE_IPV4); 78899a2dd95SBruce Richardson 78999a2dd95SBruce Richardson /* IPv4*/ 79099a2dd95SBruce Richardson d->ipv4.version_ihl = 0x45; 79199a2dd95SBruce Richardson d->ipv4.type_of_service = p->vxlan.ipv4.dscp << 2; 79299a2dd95SBruce Richardson d->ipv4.total_length = 0; /* not pre-computed */ 79399a2dd95SBruce Richardson d->ipv4.packet_id = 0; 79499a2dd95SBruce Richardson d->ipv4.fragment_offset = 0; 79599a2dd95SBruce Richardson d->ipv4.time_to_live = p->vxlan.ipv4.ttl; 79699a2dd95SBruce Richardson d->ipv4.next_proto_id = IP_PROTO_UDP; 79799a2dd95SBruce Richardson d->ipv4.hdr_checksum = 0; 79899a2dd95SBruce Richardson d->ipv4.src_addr = rte_htonl(p->vxlan.ipv4.sa); 79999a2dd95SBruce Richardson d->ipv4.dst_addr = rte_htonl(p->vxlan.ipv4.da); 80099a2dd95SBruce Richardson 80199a2dd95SBruce Richardson d->ipv4.hdr_checksum = rte_ipv4_cksum(&d->ipv4); 80299a2dd95SBruce Richardson 80399a2dd95SBruce Richardson /* UDP */ 80499a2dd95SBruce Richardson d->udp.src_port = rte_htons(p->vxlan.udp.sp); 80599a2dd95SBruce Richardson d->udp.dst_port = rte_htons(p->vxlan.udp.dp); 80699a2dd95SBruce Richardson d->udp.dgram_len = 0; /* not pre-computed */ 80799a2dd95SBruce Richardson d->udp.dgram_cksum = 0; 80899a2dd95SBruce Richardson 80999a2dd95SBruce Richardson /* VXLAN */ 81099a2dd95SBruce Richardson d->vxlan.vx_flags = rte_htonl(0x08000000); 81199a2dd95SBruce Richardson d->vxlan.vx_vni = rte_htonl(p->vxlan.vxlan.vni << 8); 81299a2dd95SBruce Richardson 81399a2dd95SBruce Richardson return 0; 81499a2dd95SBruce Richardson } else { 81599a2dd95SBruce Richardson struct encap_vxlan_ipv4_data *d = data; 81699a2dd95SBruce Richardson 81799a2dd95SBruce Richardson /* Ethernet */ 81899a2dd95SBruce Richardson rte_ether_addr_copy(&p->vxlan.ether.da, 81904d43857SDmitry Kozlyuk &d->ether.dst_addr); 82099a2dd95SBruce Richardson rte_ether_addr_copy(&p->vxlan.ether.sa, 82104d43857SDmitry Kozlyuk &d->ether.src_addr); 82299a2dd95SBruce Richardson d->ether.ether_type = rte_htons(RTE_ETHER_TYPE_IPV4); 82399a2dd95SBruce Richardson 82499a2dd95SBruce Richardson /* IPv4*/ 82599a2dd95SBruce Richardson d->ipv4.version_ihl = 0x45; 82699a2dd95SBruce Richardson d->ipv4.type_of_service = p->vxlan.ipv4.dscp << 2; 82799a2dd95SBruce Richardson d->ipv4.total_length = 0; /* not pre-computed */ 82899a2dd95SBruce Richardson d->ipv4.packet_id = 0; 82999a2dd95SBruce Richardson d->ipv4.fragment_offset = 0; 83099a2dd95SBruce Richardson d->ipv4.time_to_live = p->vxlan.ipv4.ttl; 83199a2dd95SBruce Richardson d->ipv4.next_proto_id = IP_PROTO_UDP; 83299a2dd95SBruce Richardson d->ipv4.hdr_checksum = 0; 83399a2dd95SBruce Richardson d->ipv4.src_addr = rte_htonl(p->vxlan.ipv4.sa); 83499a2dd95SBruce Richardson d->ipv4.dst_addr = rte_htonl(p->vxlan.ipv4.da); 83599a2dd95SBruce Richardson 83699a2dd95SBruce Richardson d->ipv4.hdr_checksum = rte_ipv4_cksum(&d->ipv4); 83799a2dd95SBruce Richardson 83899a2dd95SBruce Richardson /* UDP */ 83999a2dd95SBruce Richardson d->udp.src_port = rte_htons(p->vxlan.udp.sp); 84099a2dd95SBruce Richardson d->udp.dst_port = rte_htons(p->vxlan.udp.dp); 84199a2dd95SBruce Richardson d->udp.dgram_len = 0; /* not pre-computed */ 84299a2dd95SBruce Richardson d->udp.dgram_cksum = 0; 84399a2dd95SBruce Richardson 84499a2dd95SBruce Richardson /* VXLAN */ 84599a2dd95SBruce Richardson d->vxlan.vx_flags = rte_htonl(0x08000000); 84699a2dd95SBruce Richardson d->vxlan.vx_vni = rte_htonl(p->vxlan.vxlan.vni << 8); 84799a2dd95SBruce Richardson 84899a2dd95SBruce Richardson return 0; 84999a2dd95SBruce Richardson } 85099a2dd95SBruce Richardson else 85199a2dd95SBruce Richardson if (cfg->vxlan.vlan) { 85299a2dd95SBruce Richardson struct encap_vxlan_ipv6_vlan_data *d = data; 85399a2dd95SBruce Richardson 85499a2dd95SBruce Richardson /* Ethernet */ 85599a2dd95SBruce Richardson rte_ether_addr_copy(&p->vxlan.ether.da, 85604d43857SDmitry Kozlyuk &d->ether.dst_addr); 85799a2dd95SBruce Richardson rte_ether_addr_copy(&p->vxlan.ether.sa, 85804d43857SDmitry Kozlyuk &d->ether.src_addr); 85999a2dd95SBruce Richardson d->ether.ether_type = rte_htons(RTE_ETHER_TYPE_VLAN); 86099a2dd95SBruce Richardson 86199a2dd95SBruce Richardson /* VLAN */ 86299a2dd95SBruce Richardson d->vlan.vlan_tci = rte_htons(VLAN(p->vxlan.vlan.pcp, 86399a2dd95SBruce Richardson p->vxlan.vlan.dei, 86499a2dd95SBruce Richardson p->vxlan.vlan.vid)); 86599a2dd95SBruce Richardson d->vlan.eth_proto = rte_htons(RTE_ETHER_TYPE_IPV6); 86699a2dd95SBruce Richardson 86799a2dd95SBruce Richardson /* IPv6*/ 86899a2dd95SBruce Richardson d->ipv6.vtc_flow = rte_htonl((6 << 28) | 86999a2dd95SBruce Richardson (p->vxlan.ipv6.dscp << 22) | 87099a2dd95SBruce Richardson p->vxlan.ipv6.flow_label); 87199a2dd95SBruce Richardson d->ipv6.payload_len = 0; /* not pre-computed */ 87299a2dd95SBruce Richardson d->ipv6.proto = IP_PROTO_UDP; 87399a2dd95SBruce Richardson d->ipv6.hop_limits = p->vxlan.ipv6.hop_limit; 8745ac1abddSRobin Jarry d->ipv6.src_addr = p->vxlan.ipv6.sa; 8755ac1abddSRobin Jarry d->ipv6.dst_addr = p->vxlan.ipv6.da; 87699a2dd95SBruce Richardson 87799a2dd95SBruce Richardson /* UDP */ 87899a2dd95SBruce Richardson d->udp.src_port = rte_htons(p->vxlan.udp.sp); 87999a2dd95SBruce Richardson d->udp.dst_port = rte_htons(p->vxlan.udp.dp); 88099a2dd95SBruce Richardson d->udp.dgram_len = 0; /* not pre-computed */ 88199a2dd95SBruce Richardson d->udp.dgram_cksum = 0; 88299a2dd95SBruce Richardson 88399a2dd95SBruce Richardson /* VXLAN */ 88499a2dd95SBruce Richardson d->vxlan.vx_flags = rte_htonl(0x08000000); 88599a2dd95SBruce Richardson d->vxlan.vx_vni = rte_htonl(p->vxlan.vxlan.vni << 8); 88699a2dd95SBruce Richardson 88799a2dd95SBruce Richardson return 0; 88899a2dd95SBruce Richardson } else { 88999a2dd95SBruce Richardson struct encap_vxlan_ipv6_data *d = data; 89099a2dd95SBruce Richardson 89199a2dd95SBruce Richardson /* Ethernet */ 89299a2dd95SBruce Richardson rte_ether_addr_copy(&p->vxlan.ether.da, 89304d43857SDmitry Kozlyuk &d->ether.dst_addr); 89499a2dd95SBruce Richardson rte_ether_addr_copy(&p->vxlan.ether.sa, 89504d43857SDmitry Kozlyuk &d->ether.src_addr); 89699a2dd95SBruce Richardson d->ether.ether_type = rte_htons(RTE_ETHER_TYPE_IPV6); 89799a2dd95SBruce Richardson 89899a2dd95SBruce Richardson /* IPv6*/ 89999a2dd95SBruce Richardson d->ipv6.vtc_flow = rte_htonl((6 << 28) | 90099a2dd95SBruce Richardson (p->vxlan.ipv6.dscp << 22) | 90199a2dd95SBruce Richardson p->vxlan.ipv6.flow_label); 90299a2dd95SBruce Richardson d->ipv6.payload_len = 0; /* not pre-computed */ 90399a2dd95SBruce Richardson d->ipv6.proto = IP_PROTO_UDP; 90499a2dd95SBruce Richardson d->ipv6.hop_limits = p->vxlan.ipv6.hop_limit; 9055ac1abddSRobin Jarry d->ipv6.src_addr = p->vxlan.ipv6.sa; 9065ac1abddSRobin Jarry d->ipv6.dst_addr = p->vxlan.ipv6.da; 90799a2dd95SBruce Richardson 90899a2dd95SBruce Richardson /* UDP */ 90999a2dd95SBruce Richardson d->udp.src_port = rte_htons(p->vxlan.udp.sp); 91099a2dd95SBruce Richardson d->udp.dst_port = rte_htons(p->vxlan.udp.dp); 91199a2dd95SBruce Richardson d->udp.dgram_len = 0; /* not pre-computed */ 91299a2dd95SBruce Richardson d->udp.dgram_cksum = 0; 91399a2dd95SBruce Richardson 91499a2dd95SBruce Richardson /* VXLAN */ 91599a2dd95SBruce Richardson d->vxlan.vx_flags = rte_htonl(0x08000000); 91699a2dd95SBruce Richardson d->vxlan.vx_vni = rte_htonl(p->vxlan.vxlan.vni << 8); 91799a2dd95SBruce Richardson 91899a2dd95SBruce Richardson return 0; 91999a2dd95SBruce Richardson } 92099a2dd95SBruce Richardson } 92199a2dd95SBruce Richardson 92299a2dd95SBruce Richardson static int 92399a2dd95SBruce Richardson encap_apply(void *data, 92499a2dd95SBruce Richardson struct rte_table_action_encap_params *p, 92599a2dd95SBruce Richardson struct rte_table_action_encap_config *cfg, 92699a2dd95SBruce Richardson struct rte_table_action_common_config *common_cfg) 92799a2dd95SBruce Richardson { 92899a2dd95SBruce Richardson int status; 92999a2dd95SBruce Richardson 93099a2dd95SBruce Richardson /* Check input arguments */ 93199a2dd95SBruce Richardson status = encap_apply_check(p, cfg); 93299a2dd95SBruce Richardson if (status) 93399a2dd95SBruce Richardson return status; 93499a2dd95SBruce Richardson 93599a2dd95SBruce Richardson switch (p->type) { 93699a2dd95SBruce Richardson case RTE_TABLE_ACTION_ENCAP_ETHER: 93799a2dd95SBruce Richardson return encap_ether_apply(data, p, common_cfg); 93899a2dd95SBruce Richardson 93999a2dd95SBruce Richardson case RTE_TABLE_ACTION_ENCAP_VLAN: 94099a2dd95SBruce Richardson return encap_vlan_apply(data, p, common_cfg); 94199a2dd95SBruce Richardson 94299a2dd95SBruce Richardson case RTE_TABLE_ACTION_ENCAP_QINQ: 94399a2dd95SBruce Richardson return encap_qinq_apply(data, p, common_cfg); 94499a2dd95SBruce Richardson 94599a2dd95SBruce Richardson case RTE_TABLE_ACTION_ENCAP_MPLS: 94699a2dd95SBruce Richardson return encap_mpls_apply(data, p); 94799a2dd95SBruce Richardson 94899a2dd95SBruce Richardson case RTE_TABLE_ACTION_ENCAP_PPPOE: 94999a2dd95SBruce Richardson return encap_pppoe_apply(data, p); 95099a2dd95SBruce Richardson 95199a2dd95SBruce Richardson case RTE_TABLE_ACTION_ENCAP_VXLAN: 95299a2dd95SBruce Richardson return encap_vxlan_apply(data, p, cfg); 95399a2dd95SBruce Richardson 95499a2dd95SBruce Richardson case RTE_TABLE_ACTION_ENCAP_QINQ_PPPOE: 95599a2dd95SBruce Richardson return encap_qinq_pppoe_apply(data, p); 95699a2dd95SBruce Richardson 95799a2dd95SBruce Richardson default: 95899a2dd95SBruce Richardson return -EINVAL; 95999a2dd95SBruce Richardson } 96099a2dd95SBruce Richardson } 96199a2dd95SBruce Richardson 96299a2dd95SBruce Richardson static __rte_always_inline uint16_t 96399a2dd95SBruce Richardson encap_vxlan_ipv4_checksum_update(uint16_t cksum0, 96499a2dd95SBruce Richardson uint16_t total_length) 96599a2dd95SBruce Richardson { 96699a2dd95SBruce Richardson int32_t cksum1; 96799a2dd95SBruce Richardson 96899a2dd95SBruce Richardson cksum1 = cksum0; 96999a2dd95SBruce Richardson cksum1 = ~cksum1 & 0xFFFF; 97099a2dd95SBruce Richardson 97199a2dd95SBruce Richardson /* Add total length (one's complement logic) */ 97299a2dd95SBruce Richardson cksum1 += total_length; 97399a2dd95SBruce Richardson cksum1 = (cksum1 & 0xFFFF) + (cksum1 >> 16); 97499a2dd95SBruce Richardson cksum1 = (cksum1 & 0xFFFF) + (cksum1 >> 16); 97599a2dd95SBruce Richardson 97699a2dd95SBruce Richardson return (uint16_t)(~cksum1); 97799a2dd95SBruce Richardson } 97899a2dd95SBruce Richardson 97999a2dd95SBruce Richardson static __rte_always_inline void * 98099a2dd95SBruce Richardson encap(void *dst, const void *src, size_t n) 98199a2dd95SBruce Richardson { 98299a2dd95SBruce Richardson dst = ((uint8_t *) dst) - n; 98399a2dd95SBruce Richardson return rte_memcpy(dst, src, n); 98499a2dd95SBruce Richardson } 98599a2dd95SBruce Richardson 98699a2dd95SBruce Richardson static __rte_always_inline void 98799a2dd95SBruce Richardson pkt_work_encap_vxlan_ipv4(struct rte_mbuf *mbuf, 98899a2dd95SBruce Richardson struct encap_vxlan_ipv4_data *vxlan_tbl, 98999a2dd95SBruce Richardson struct rte_table_action_encap_config *cfg) 99099a2dd95SBruce Richardson { 99199a2dd95SBruce Richardson uint32_t ether_offset = cfg->vxlan.data_offset; 99299a2dd95SBruce Richardson void *ether = RTE_MBUF_METADATA_UINT32_PTR(mbuf, ether_offset); 99399a2dd95SBruce Richardson struct encap_vxlan_ipv4_data *vxlan_pkt; 99499a2dd95SBruce Richardson uint16_t ether_length, ipv4_total_length, ipv4_hdr_cksum, udp_length; 99599a2dd95SBruce Richardson 99699a2dd95SBruce Richardson ether_length = (uint16_t)mbuf->pkt_len; 99799a2dd95SBruce Richardson ipv4_total_length = ether_length + 99899a2dd95SBruce Richardson (sizeof(struct rte_vxlan_hdr) + 99999a2dd95SBruce Richardson sizeof(struct rte_udp_hdr) + 100099a2dd95SBruce Richardson sizeof(struct rte_ipv4_hdr)); 100199a2dd95SBruce Richardson ipv4_hdr_cksum = encap_vxlan_ipv4_checksum_update(vxlan_tbl->ipv4.hdr_checksum, 100299a2dd95SBruce Richardson rte_htons(ipv4_total_length)); 100399a2dd95SBruce Richardson udp_length = ether_length + 100499a2dd95SBruce Richardson (sizeof(struct rte_vxlan_hdr) + 100599a2dd95SBruce Richardson sizeof(struct rte_udp_hdr)); 100699a2dd95SBruce Richardson 100799a2dd95SBruce Richardson vxlan_pkt = encap(ether, vxlan_tbl, sizeof(*vxlan_tbl)); 100899a2dd95SBruce Richardson vxlan_pkt->ipv4.total_length = rte_htons(ipv4_total_length); 100999a2dd95SBruce Richardson vxlan_pkt->ipv4.hdr_checksum = ipv4_hdr_cksum; 101099a2dd95SBruce Richardson vxlan_pkt->udp.dgram_len = rte_htons(udp_length); 101199a2dd95SBruce Richardson 101299a2dd95SBruce Richardson mbuf->data_off = ether_offset - (sizeof(struct rte_mbuf) + sizeof(*vxlan_pkt)); 101399a2dd95SBruce Richardson mbuf->pkt_len = mbuf->data_len = ether_length + sizeof(*vxlan_pkt); 101499a2dd95SBruce Richardson } 101599a2dd95SBruce Richardson 101699a2dd95SBruce Richardson static __rte_always_inline void 101799a2dd95SBruce Richardson pkt_work_encap_vxlan_ipv4_vlan(struct rte_mbuf *mbuf, 101899a2dd95SBruce Richardson struct encap_vxlan_ipv4_vlan_data *vxlan_tbl, 101999a2dd95SBruce Richardson struct rte_table_action_encap_config *cfg) 102099a2dd95SBruce Richardson { 102199a2dd95SBruce Richardson uint32_t ether_offset = cfg->vxlan.data_offset; 102299a2dd95SBruce Richardson void *ether = RTE_MBUF_METADATA_UINT32_PTR(mbuf, ether_offset); 102399a2dd95SBruce Richardson struct encap_vxlan_ipv4_vlan_data *vxlan_pkt; 102499a2dd95SBruce Richardson uint16_t ether_length, ipv4_total_length, ipv4_hdr_cksum, udp_length; 102599a2dd95SBruce Richardson 102699a2dd95SBruce Richardson ether_length = (uint16_t)mbuf->pkt_len; 102799a2dd95SBruce Richardson ipv4_total_length = ether_length + 102899a2dd95SBruce Richardson (sizeof(struct rte_vxlan_hdr) + 102999a2dd95SBruce Richardson sizeof(struct rte_udp_hdr) + 103099a2dd95SBruce Richardson sizeof(struct rte_ipv4_hdr)); 103199a2dd95SBruce Richardson ipv4_hdr_cksum = encap_vxlan_ipv4_checksum_update(vxlan_tbl->ipv4.hdr_checksum, 103299a2dd95SBruce Richardson rte_htons(ipv4_total_length)); 103399a2dd95SBruce Richardson udp_length = ether_length + 103499a2dd95SBruce Richardson (sizeof(struct rte_vxlan_hdr) + 103599a2dd95SBruce Richardson sizeof(struct rte_udp_hdr)); 103699a2dd95SBruce Richardson 103799a2dd95SBruce Richardson vxlan_pkt = encap(ether, vxlan_tbl, sizeof(*vxlan_tbl)); 103899a2dd95SBruce Richardson vxlan_pkt->ipv4.total_length = rte_htons(ipv4_total_length); 103999a2dd95SBruce Richardson vxlan_pkt->ipv4.hdr_checksum = ipv4_hdr_cksum; 104099a2dd95SBruce Richardson vxlan_pkt->udp.dgram_len = rte_htons(udp_length); 104199a2dd95SBruce Richardson 104299a2dd95SBruce Richardson mbuf->data_off = ether_offset - (sizeof(struct rte_mbuf) + sizeof(*vxlan_pkt)); 104399a2dd95SBruce Richardson mbuf->pkt_len = mbuf->data_len = ether_length + sizeof(*vxlan_pkt); 104499a2dd95SBruce Richardson } 104599a2dd95SBruce Richardson 104699a2dd95SBruce Richardson static __rte_always_inline void 104799a2dd95SBruce Richardson pkt_work_encap_vxlan_ipv6(struct rte_mbuf *mbuf, 104899a2dd95SBruce Richardson struct encap_vxlan_ipv6_data *vxlan_tbl, 104999a2dd95SBruce Richardson struct rte_table_action_encap_config *cfg) 105099a2dd95SBruce Richardson { 105199a2dd95SBruce Richardson uint32_t ether_offset = cfg->vxlan.data_offset; 105299a2dd95SBruce Richardson void *ether = RTE_MBUF_METADATA_UINT32_PTR(mbuf, ether_offset); 105399a2dd95SBruce Richardson struct encap_vxlan_ipv6_data *vxlan_pkt; 105499a2dd95SBruce Richardson uint16_t ether_length, ipv6_payload_length, udp_length; 105599a2dd95SBruce Richardson 105699a2dd95SBruce Richardson ether_length = (uint16_t)mbuf->pkt_len; 105799a2dd95SBruce Richardson ipv6_payload_length = ether_length + 105899a2dd95SBruce Richardson (sizeof(struct rte_vxlan_hdr) + 105999a2dd95SBruce Richardson sizeof(struct rte_udp_hdr)); 106099a2dd95SBruce Richardson udp_length = ether_length + 106199a2dd95SBruce Richardson (sizeof(struct rte_vxlan_hdr) + 106299a2dd95SBruce Richardson sizeof(struct rte_udp_hdr)); 106399a2dd95SBruce Richardson 106499a2dd95SBruce Richardson vxlan_pkt = encap(ether, vxlan_tbl, sizeof(*vxlan_tbl)); 106599a2dd95SBruce Richardson vxlan_pkt->ipv6.payload_len = rte_htons(ipv6_payload_length); 106699a2dd95SBruce Richardson vxlan_pkt->udp.dgram_len = rte_htons(udp_length); 106799a2dd95SBruce Richardson 106899a2dd95SBruce Richardson mbuf->data_off = ether_offset - (sizeof(struct rte_mbuf) + sizeof(*vxlan_pkt)); 106999a2dd95SBruce Richardson mbuf->pkt_len = mbuf->data_len = ether_length + sizeof(*vxlan_pkt); 107099a2dd95SBruce Richardson } 107199a2dd95SBruce Richardson 107299a2dd95SBruce Richardson static __rte_always_inline void 107399a2dd95SBruce Richardson pkt_work_encap_vxlan_ipv6_vlan(struct rte_mbuf *mbuf, 107499a2dd95SBruce Richardson struct encap_vxlan_ipv6_vlan_data *vxlan_tbl, 107599a2dd95SBruce Richardson struct rte_table_action_encap_config *cfg) 107699a2dd95SBruce Richardson { 107799a2dd95SBruce Richardson uint32_t ether_offset = cfg->vxlan.data_offset; 107899a2dd95SBruce Richardson void *ether = RTE_MBUF_METADATA_UINT32_PTR(mbuf, ether_offset); 107999a2dd95SBruce Richardson struct encap_vxlan_ipv6_vlan_data *vxlan_pkt; 108099a2dd95SBruce Richardson uint16_t ether_length, ipv6_payload_length, udp_length; 108199a2dd95SBruce Richardson 108299a2dd95SBruce Richardson ether_length = (uint16_t)mbuf->pkt_len; 108399a2dd95SBruce Richardson ipv6_payload_length = ether_length + 108499a2dd95SBruce Richardson (sizeof(struct rte_vxlan_hdr) + 108599a2dd95SBruce Richardson sizeof(struct rte_udp_hdr)); 108699a2dd95SBruce Richardson udp_length = ether_length + 108799a2dd95SBruce Richardson (sizeof(struct rte_vxlan_hdr) + 108899a2dd95SBruce Richardson sizeof(struct rte_udp_hdr)); 108999a2dd95SBruce Richardson 109099a2dd95SBruce Richardson vxlan_pkt = encap(ether, vxlan_tbl, sizeof(*vxlan_tbl)); 109199a2dd95SBruce Richardson vxlan_pkt->ipv6.payload_len = rte_htons(ipv6_payload_length); 109299a2dd95SBruce Richardson vxlan_pkt->udp.dgram_len = rte_htons(udp_length); 109399a2dd95SBruce Richardson 109499a2dd95SBruce Richardson mbuf->data_off = ether_offset - (sizeof(struct rte_mbuf) + sizeof(*vxlan_pkt)); 109599a2dd95SBruce Richardson mbuf->pkt_len = mbuf->data_len = ether_length + sizeof(*vxlan_pkt); 109699a2dd95SBruce Richardson } 109799a2dd95SBruce Richardson 109899a2dd95SBruce Richardson static __rte_always_inline void 109999a2dd95SBruce Richardson pkt_work_encap(struct rte_mbuf *mbuf, 110099a2dd95SBruce Richardson void *data, 110199a2dd95SBruce Richardson struct rte_table_action_encap_config *cfg, 110299a2dd95SBruce Richardson void *ip, 110399a2dd95SBruce Richardson uint16_t total_length, 110499a2dd95SBruce Richardson uint32_t ip_offset) 110599a2dd95SBruce Richardson { 110699a2dd95SBruce Richardson switch (cfg->encap_mask) { 110799a2dd95SBruce Richardson case 1LLU << RTE_TABLE_ACTION_ENCAP_ETHER: 110899a2dd95SBruce Richardson encap(ip, data, sizeof(struct encap_ether_data)); 110999a2dd95SBruce Richardson mbuf->data_off = ip_offset - (sizeof(struct rte_mbuf) + 111099a2dd95SBruce Richardson sizeof(struct encap_ether_data)); 111199a2dd95SBruce Richardson mbuf->pkt_len = mbuf->data_len = total_length + 111299a2dd95SBruce Richardson sizeof(struct encap_ether_data); 111399a2dd95SBruce Richardson break; 111499a2dd95SBruce Richardson 111599a2dd95SBruce Richardson case 1LLU << RTE_TABLE_ACTION_ENCAP_VLAN: 111699a2dd95SBruce Richardson encap(ip, data, sizeof(struct encap_vlan_data)); 111799a2dd95SBruce Richardson mbuf->data_off = ip_offset - (sizeof(struct rte_mbuf) + 111899a2dd95SBruce Richardson sizeof(struct encap_vlan_data)); 111999a2dd95SBruce Richardson mbuf->pkt_len = mbuf->data_len = total_length + 112099a2dd95SBruce Richardson sizeof(struct encap_vlan_data); 112199a2dd95SBruce Richardson break; 112299a2dd95SBruce Richardson 112399a2dd95SBruce Richardson case 1LLU << RTE_TABLE_ACTION_ENCAP_QINQ: 112499a2dd95SBruce Richardson encap(ip, data, sizeof(struct encap_qinq_data)); 112599a2dd95SBruce Richardson mbuf->data_off = ip_offset - (sizeof(struct rte_mbuf) + 112699a2dd95SBruce Richardson sizeof(struct encap_qinq_data)); 112799a2dd95SBruce Richardson mbuf->pkt_len = mbuf->data_len = total_length + 112899a2dd95SBruce Richardson sizeof(struct encap_qinq_data); 112999a2dd95SBruce Richardson break; 113099a2dd95SBruce Richardson 113199a2dd95SBruce Richardson case 1LLU << RTE_TABLE_ACTION_ENCAP_MPLS: 113299a2dd95SBruce Richardson { 113399a2dd95SBruce Richardson struct encap_mpls_data *mpls = data; 113499a2dd95SBruce Richardson size_t size = sizeof(struct rte_ether_hdr) + 113599a2dd95SBruce Richardson mpls->mpls_count * 4; 113699a2dd95SBruce Richardson 113799a2dd95SBruce Richardson encap(ip, data, size); 113899a2dd95SBruce Richardson mbuf->data_off = ip_offset - (sizeof(struct rte_mbuf) + size); 113999a2dd95SBruce Richardson mbuf->pkt_len = mbuf->data_len = total_length + size; 114099a2dd95SBruce Richardson break; 114199a2dd95SBruce Richardson } 114299a2dd95SBruce Richardson 114399a2dd95SBruce Richardson case 1LLU << RTE_TABLE_ACTION_ENCAP_PPPOE: 114499a2dd95SBruce Richardson { 114599a2dd95SBruce Richardson struct encap_pppoe_data *pppoe = 114699a2dd95SBruce Richardson encap(ip, data, sizeof(struct encap_pppoe_data)); 114799a2dd95SBruce Richardson pppoe->pppoe_ppp.length = rte_htons(total_length + 2); 114899a2dd95SBruce Richardson mbuf->data_off = ip_offset - (sizeof(struct rte_mbuf) + 114999a2dd95SBruce Richardson sizeof(struct encap_pppoe_data)); 115099a2dd95SBruce Richardson mbuf->pkt_len = mbuf->data_len = total_length + 115199a2dd95SBruce Richardson sizeof(struct encap_pppoe_data); 115299a2dd95SBruce Richardson break; 115399a2dd95SBruce Richardson } 115499a2dd95SBruce Richardson 115599a2dd95SBruce Richardson case 1LLU << RTE_TABLE_ACTION_ENCAP_QINQ_PPPOE: 115699a2dd95SBruce Richardson { 115799a2dd95SBruce Richardson struct encap_qinq_pppoe_data *qinq_pppoe = 115899a2dd95SBruce Richardson encap(ip, data, sizeof(struct encap_qinq_pppoe_data)); 115999a2dd95SBruce Richardson qinq_pppoe->pppoe_ppp.length = rte_htons(total_length + 2); 116099a2dd95SBruce Richardson mbuf->data_off = ip_offset - (sizeof(struct rte_mbuf) + 116199a2dd95SBruce Richardson sizeof(struct encap_qinq_pppoe_data)); 116299a2dd95SBruce Richardson mbuf->pkt_len = mbuf->data_len = total_length + 116399a2dd95SBruce Richardson sizeof(struct encap_qinq_pppoe_data); 116499a2dd95SBruce Richardson break; 116599a2dd95SBruce Richardson } 116699a2dd95SBruce Richardson 116799a2dd95SBruce Richardson case 1LLU << RTE_TABLE_ACTION_ENCAP_VXLAN: 116899a2dd95SBruce Richardson { 116999a2dd95SBruce Richardson if (cfg->vxlan.ip_version) 117099a2dd95SBruce Richardson if (cfg->vxlan.vlan) 117199a2dd95SBruce Richardson pkt_work_encap_vxlan_ipv4_vlan(mbuf, data, cfg); 117299a2dd95SBruce Richardson else 117399a2dd95SBruce Richardson pkt_work_encap_vxlan_ipv4(mbuf, data, cfg); 117499a2dd95SBruce Richardson else 117599a2dd95SBruce Richardson if (cfg->vxlan.vlan) 117699a2dd95SBruce Richardson pkt_work_encap_vxlan_ipv6_vlan(mbuf, data, cfg); 117799a2dd95SBruce Richardson else 117899a2dd95SBruce Richardson pkt_work_encap_vxlan_ipv6(mbuf, data, cfg); 117999a2dd95SBruce Richardson } 118099a2dd95SBruce Richardson 118199a2dd95SBruce Richardson default: 118299a2dd95SBruce Richardson break; 118399a2dd95SBruce Richardson } 118499a2dd95SBruce Richardson } 118599a2dd95SBruce Richardson 118699a2dd95SBruce Richardson /** 118799a2dd95SBruce Richardson * RTE_TABLE_ACTION_NAT 118899a2dd95SBruce Richardson */ 118999a2dd95SBruce Richardson static int 119099a2dd95SBruce Richardson nat_cfg_check(struct rte_table_action_nat_config *nat) 119199a2dd95SBruce Richardson { 119299a2dd95SBruce Richardson if ((nat->proto != 0x06) && 119399a2dd95SBruce Richardson (nat->proto != 0x11)) 119499a2dd95SBruce Richardson return -ENOTSUP; 119599a2dd95SBruce Richardson 119699a2dd95SBruce Richardson return 0; 119799a2dd95SBruce Richardson } 119899a2dd95SBruce Richardson 1199*fba98755SAndre Muezerie struct __rte_packed_begin nat_ipv4_data { 120099a2dd95SBruce Richardson uint32_t addr; 120199a2dd95SBruce Richardson uint16_t port; 1202*fba98755SAndre Muezerie } __rte_packed_end; 120399a2dd95SBruce Richardson 1204*fba98755SAndre Muezerie struct __rte_packed_begin nat_ipv6_data { 12055ac1abddSRobin Jarry struct rte_ipv6_addr addr; 120699a2dd95SBruce Richardson uint16_t port; 1207*fba98755SAndre Muezerie } __rte_packed_end; 120899a2dd95SBruce Richardson 120999a2dd95SBruce Richardson static size_t 121099a2dd95SBruce Richardson nat_data_size(struct rte_table_action_nat_config *nat __rte_unused, 121199a2dd95SBruce Richardson struct rte_table_action_common_config *common) 121299a2dd95SBruce Richardson { 121399a2dd95SBruce Richardson int ip_version = common->ip_version; 121499a2dd95SBruce Richardson 121599a2dd95SBruce Richardson return (ip_version) ? 121699a2dd95SBruce Richardson sizeof(struct nat_ipv4_data) : 121799a2dd95SBruce Richardson sizeof(struct nat_ipv6_data); 121899a2dd95SBruce Richardson } 121999a2dd95SBruce Richardson 122099a2dd95SBruce Richardson static int 122199a2dd95SBruce Richardson nat_apply_check(struct rte_table_action_nat_params *p, 122299a2dd95SBruce Richardson struct rte_table_action_common_config *cfg) 122399a2dd95SBruce Richardson { 122499a2dd95SBruce Richardson if ((p->ip_version && (cfg->ip_version == 0)) || 122599a2dd95SBruce Richardson ((p->ip_version == 0) && cfg->ip_version)) 122699a2dd95SBruce Richardson return -EINVAL; 122799a2dd95SBruce Richardson 122899a2dd95SBruce Richardson return 0; 122999a2dd95SBruce Richardson } 123099a2dd95SBruce Richardson 123199a2dd95SBruce Richardson static int 123299a2dd95SBruce Richardson nat_apply(void *data, 123399a2dd95SBruce Richardson struct rte_table_action_nat_params *p, 123499a2dd95SBruce Richardson struct rte_table_action_common_config *cfg) 123599a2dd95SBruce Richardson { 123699a2dd95SBruce Richardson int status; 123799a2dd95SBruce Richardson 123899a2dd95SBruce Richardson /* Check input arguments */ 123999a2dd95SBruce Richardson status = nat_apply_check(p, cfg); 124099a2dd95SBruce Richardson if (status) 124199a2dd95SBruce Richardson return status; 124299a2dd95SBruce Richardson 124399a2dd95SBruce Richardson /* Apply */ 124499a2dd95SBruce Richardson if (p->ip_version) { 124599a2dd95SBruce Richardson struct nat_ipv4_data *d = data; 124699a2dd95SBruce Richardson 124799a2dd95SBruce Richardson d->addr = rte_htonl(p->addr.ipv4); 124899a2dd95SBruce Richardson d->port = rte_htons(p->port); 124999a2dd95SBruce Richardson } else { 125099a2dd95SBruce Richardson struct nat_ipv6_data *d = data; 125199a2dd95SBruce Richardson 12525ac1abddSRobin Jarry d->addr = p->addr.ipv6; 125399a2dd95SBruce Richardson d->port = rte_htons(p->port); 125499a2dd95SBruce Richardson } 125599a2dd95SBruce Richardson 125699a2dd95SBruce Richardson return 0; 125799a2dd95SBruce Richardson } 125899a2dd95SBruce Richardson 125999a2dd95SBruce Richardson static __rte_always_inline uint16_t 126099a2dd95SBruce Richardson nat_ipv4_checksum_update(uint16_t cksum0, 126199a2dd95SBruce Richardson uint32_t ip0, 126299a2dd95SBruce Richardson uint32_t ip1) 126399a2dd95SBruce Richardson { 126499a2dd95SBruce Richardson int32_t cksum1; 126599a2dd95SBruce Richardson 126699a2dd95SBruce Richardson cksum1 = cksum0; 126799a2dd95SBruce Richardson cksum1 = ~cksum1 & 0xFFFF; 126899a2dd95SBruce Richardson 126999a2dd95SBruce Richardson /* Subtract ip0 (one's complement logic) */ 127099a2dd95SBruce Richardson cksum1 -= (ip0 >> 16) + (ip0 & 0xFFFF); 127199a2dd95SBruce Richardson cksum1 = (cksum1 & 0xFFFF) + (cksum1 >> 16); 127299a2dd95SBruce Richardson cksum1 = (cksum1 & 0xFFFF) + (cksum1 >> 16); 127399a2dd95SBruce Richardson 127499a2dd95SBruce Richardson /* Add ip1 (one's complement logic) */ 127599a2dd95SBruce Richardson cksum1 += (ip1 >> 16) + (ip1 & 0xFFFF); 127699a2dd95SBruce Richardson cksum1 = (cksum1 & 0xFFFF) + (cksum1 >> 16); 127799a2dd95SBruce Richardson cksum1 = (cksum1 & 0xFFFF) + (cksum1 >> 16); 127899a2dd95SBruce Richardson 127999a2dd95SBruce Richardson return (uint16_t)(~cksum1); 128099a2dd95SBruce Richardson } 128199a2dd95SBruce Richardson 128299a2dd95SBruce Richardson static __rte_always_inline uint16_t 128399a2dd95SBruce Richardson nat_ipv4_tcp_udp_checksum_update(uint16_t cksum0, 128499a2dd95SBruce Richardson uint32_t ip0, 128599a2dd95SBruce Richardson uint32_t ip1, 128699a2dd95SBruce Richardson uint16_t port0, 128799a2dd95SBruce Richardson uint16_t port1) 128899a2dd95SBruce Richardson { 128999a2dd95SBruce Richardson int32_t cksum1; 129099a2dd95SBruce Richardson 129199a2dd95SBruce Richardson cksum1 = cksum0; 129299a2dd95SBruce Richardson cksum1 = ~cksum1 & 0xFFFF; 129399a2dd95SBruce Richardson 129499a2dd95SBruce Richardson /* Subtract ip0 and port 0 (one's complement logic) */ 129599a2dd95SBruce Richardson cksum1 -= (ip0 >> 16) + (ip0 & 0xFFFF) + port0; 129699a2dd95SBruce Richardson cksum1 = (cksum1 & 0xFFFF) + (cksum1 >> 16); 129799a2dd95SBruce Richardson cksum1 = (cksum1 & 0xFFFF) + (cksum1 >> 16); 129899a2dd95SBruce Richardson 129999a2dd95SBruce Richardson /* Add ip1 and port1 (one's complement logic) */ 130099a2dd95SBruce Richardson cksum1 += (ip1 >> 16) + (ip1 & 0xFFFF) + port1; 130199a2dd95SBruce Richardson cksum1 = (cksum1 & 0xFFFF) + (cksum1 >> 16); 130299a2dd95SBruce Richardson cksum1 = (cksum1 & 0xFFFF) + (cksum1 >> 16); 130399a2dd95SBruce Richardson 130499a2dd95SBruce Richardson return (uint16_t)(~cksum1); 130599a2dd95SBruce Richardson } 130699a2dd95SBruce Richardson 130799a2dd95SBruce Richardson static __rte_always_inline uint16_t 130899a2dd95SBruce Richardson nat_ipv6_tcp_udp_checksum_update(uint16_t cksum0, 130999a2dd95SBruce Richardson uint16_t *ip0, 131099a2dd95SBruce Richardson uint16_t *ip1, 131199a2dd95SBruce Richardson uint16_t port0, 131299a2dd95SBruce Richardson uint16_t port1) 131399a2dd95SBruce Richardson { 131499a2dd95SBruce Richardson int32_t cksum1; 131599a2dd95SBruce Richardson 131699a2dd95SBruce Richardson cksum1 = cksum0; 131799a2dd95SBruce Richardson cksum1 = ~cksum1 & 0xFFFF; 131899a2dd95SBruce Richardson 131999a2dd95SBruce Richardson /* Subtract ip0 and port 0 (one's complement logic) */ 132099a2dd95SBruce Richardson cksum1 -= ip0[0] + ip0[1] + ip0[2] + ip0[3] + 132199a2dd95SBruce Richardson ip0[4] + ip0[5] + ip0[6] + ip0[7] + port0; 132299a2dd95SBruce Richardson cksum1 = (cksum1 & 0xFFFF) + (cksum1 >> 16); 132399a2dd95SBruce Richardson cksum1 = (cksum1 & 0xFFFF) + (cksum1 >> 16); 132499a2dd95SBruce Richardson 132599a2dd95SBruce Richardson /* Add ip1 and port1 (one's complement logic) */ 132699a2dd95SBruce Richardson cksum1 += ip1[0] + ip1[1] + ip1[2] + ip1[3] + 132799a2dd95SBruce Richardson ip1[4] + ip1[5] + ip1[6] + ip1[7] + port1; 132899a2dd95SBruce Richardson cksum1 = (cksum1 & 0xFFFF) + (cksum1 >> 16); 132999a2dd95SBruce Richardson cksum1 = (cksum1 & 0xFFFF) + (cksum1 >> 16); 133099a2dd95SBruce Richardson 133199a2dd95SBruce Richardson return (uint16_t)(~cksum1); 133299a2dd95SBruce Richardson } 133399a2dd95SBruce Richardson 133499a2dd95SBruce Richardson static __rte_always_inline void 133599a2dd95SBruce Richardson pkt_ipv4_work_nat(struct rte_ipv4_hdr *ip, 133699a2dd95SBruce Richardson struct nat_ipv4_data *data, 133799a2dd95SBruce Richardson struct rte_table_action_nat_config *cfg) 133899a2dd95SBruce Richardson { 133999a2dd95SBruce Richardson if (cfg->source_nat) { 134099a2dd95SBruce Richardson if (cfg->proto == 0x6) { 134199a2dd95SBruce Richardson struct rte_tcp_hdr *tcp = (struct rte_tcp_hdr *) &ip[1]; 134299a2dd95SBruce Richardson uint16_t ip_cksum, tcp_cksum; 134399a2dd95SBruce Richardson 134499a2dd95SBruce Richardson ip_cksum = nat_ipv4_checksum_update(ip->hdr_checksum, 134599a2dd95SBruce Richardson ip->src_addr, 134699a2dd95SBruce Richardson data->addr); 134799a2dd95SBruce Richardson 134899a2dd95SBruce Richardson tcp_cksum = nat_ipv4_tcp_udp_checksum_update(tcp->cksum, 134999a2dd95SBruce Richardson ip->src_addr, 135099a2dd95SBruce Richardson data->addr, 135199a2dd95SBruce Richardson tcp->src_port, 135299a2dd95SBruce Richardson data->port); 135399a2dd95SBruce Richardson 135499a2dd95SBruce Richardson ip->src_addr = data->addr; 135599a2dd95SBruce Richardson ip->hdr_checksum = ip_cksum; 135699a2dd95SBruce Richardson tcp->src_port = data->port; 135799a2dd95SBruce Richardson tcp->cksum = tcp_cksum; 135899a2dd95SBruce Richardson } else { 135999a2dd95SBruce Richardson struct rte_udp_hdr *udp = (struct rte_udp_hdr *) &ip[1]; 136099a2dd95SBruce Richardson uint16_t ip_cksum, udp_cksum; 136199a2dd95SBruce Richardson 136299a2dd95SBruce Richardson ip_cksum = nat_ipv4_checksum_update(ip->hdr_checksum, 136399a2dd95SBruce Richardson ip->src_addr, 136499a2dd95SBruce Richardson data->addr); 136599a2dd95SBruce Richardson 136699a2dd95SBruce Richardson udp_cksum = nat_ipv4_tcp_udp_checksum_update(udp->dgram_cksum, 136799a2dd95SBruce Richardson ip->src_addr, 136899a2dd95SBruce Richardson data->addr, 136999a2dd95SBruce Richardson udp->src_port, 137099a2dd95SBruce Richardson data->port); 137199a2dd95SBruce Richardson 137299a2dd95SBruce Richardson ip->src_addr = data->addr; 137399a2dd95SBruce Richardson ip->hdr_checksum = ip_cksum; 137499a2dd95SBruce Richardson udp->src_port = data->port; 137599a2dd95SBruce Richardson if (udp->dgram_cksum) 137699a2dd95SBruce Richardson udp->dgram_cksum = udp_cksum; 137799a2dd95SBruce Richardson } 137899a2dd95SBruce Richardson } else { 137999a2dd95SBruce Richardson if (cfg->proto == 0x6) { 138099a2dd95SBruce Richardson struct rte_tcp_hdr *tcp = (struct rte_tcp_hdr *) &ip[1]; 138199a2dd95SBruce Richardson uint16_t ip_cksum, tcp_cksum; 138299a2dd95SBruce Richardson 138399a2dd95SBruce Richardson ip_cksum = nat_ipv4_checksum_update(ip->hdr_checksum, 138499a2dd95SBruce Richardson ip->dst_addr, 138599a2dd95SBruce Richardson data->addr); 138699a2dd95SBruce Richardson 138799a2dd95SBruce Richardson tcp_cksum = nat_ipv4_tcp_udp_checksum_update(tcp->cksum, 138899a2dd95SBruce Richardson ip->dst_addr, 138999a2dd95SBruce Richardson data->addr, 139099a2dd95SBruce Richardson tcp->dst_port, 139199a2dd95SBruce Richardson data->port); 139299a2dd95SBruce Richardson 139399a2dd95SBruce Richardson ip->dst_addr = data->addr; 139499a2dd95SBruce Richardson ip->hdr_checksum = ip_cksum; 139599a2dd95SBruce Richardson tcp->dst_port = data->port; 139699a2dd95SBruce Richardson tcp->cksum = tcp_cksum; 139799a2dd95SBruce Richardson } else { 139899a2dd95SBruce Richardson struct rte_udp_hdr *udp = (struct rte_udp_hdr *) &ip[1]; 139999a2dd95SBruce Richardson uint16_t ip_cksum, udp_cksum; 140099a2dd95SBruce Richardson 140199a2dd95SBruce Richardson ip_cksum = nat_ipv4_checksum_update(ip->hdr_checksum, 140299a2dd95SBruce Richardson ip->dst_addr, 140399a2dd95SBruce Richardson data->addr); 140499a2dd95SBruce Richardson 140599a2dd95SBruce Richardson udp_cksum = nat_ipv4_tcp_udp_checksum_update(udp->dgram_cksum, 140699a2dd95SBruce Richardson ip->dst_addr, 140799a2dd95SBruce Richardson data->addr, 140899a2dd95SBruce Richardson udp->dst_port, 140999a2dd95SBruce Richardson data->port); 141099a2dd95SBruce Richardson 141199a2dd95SBruce Richardson ip->dst_addr = data->addr; 141299a2dd95SBruce Richardson ip->hdr_checksum = ip_cksum; 141399a2dd95SBruce Richardson udp->dst_port = data->port; 141499a2dd95SBruce Richardson if (udp->dgram_cksum) 141599a2dd95SBruce Richardson udp->dgram_cksum = udp_cksum; 141699a2dd95SBruce Richardson } 141799a2dd95SBruce Richardson } 141899a2dd95SBruce Richardson } 141999a2dd95SBruce Richardson 142099a2dd95SBruce Richardson static __rte_always_inline void 142199a2dd95SBruce Richardson pkt_ipv6_work_nat(struct rte_ipv6_hdr *ip, 142299a2dd95SBruce Richardson struct nat_ipv6_data *data, 142399a2dd95SBruce Richardson struct rte_table_action_nat_config *cfg) 142499a2dd95SBruce Richardson { 142599a2dd95SBruce Richardson if (cfg->source_nat) { 142699a2dd95SBruce Richardson if (cfg->proto == 0x6) { 142799a2dd95SBruce Richardson struct rte_tcp_hdr *tcp = (struct rte_tcp_hdr *) &ip[1]; 142899a2dd95SBruce Richardson uint16_t tcp_cksum; 142999a2dd95SBruce Richardson 143099a2dd95SBruce Richardson tcp_cksum = nat_ipv6_tcp_udp_checksum_update(tcp->cksum, 143189b5642dSRobin Jarry (uint16_t *)&ip->src_addr, 14325ac1abddSRobin Jarry (uint16_t *)&data->addr, 143399a2dd95SBruce Richardson tcp->src_port, 143499a2dd95SBruce Richardson data->port); 143599a2dd95SBruce Richardson 14365ac1abddSRobin Jarry ip->src_addr = data->addr; 143799a2dd95SBruce Richardson tcp->src_port = data->port; 143899a2dd95SBruce Richardson tcp->cksum = tcp_cksum; 143999a2dd95SBruce Richardson } else { 144099a2dd95SBruce Richardson struct rte_udp_hdr *udp = (struct rte_udp_hdr *) &ip[1]; 144199a2dd95SBruce Richardson uint16_t udp_cksum; 144299a2dd95SBruce Richardson 144399a2dd95SBruce Richardson udp_cksum = nat_ipv6_tcp_udp_checksum_update(udp->dgram_cksum, 144489b5642dSRobin Jarry (uint16_t *)&ip->src_addr, 14455ac1abddSRobin Jarry (uint16_t *)&data->addr, 144699a2dd95SBruce Richardson udp->src_port, 144799a2dd95SBruce Richardson data->port); 144899a2dd95SBruce Richardson 14495ac1abddSRobin Jarry ip->src_addr = data->addr; 145099a2dd95SBruce Richardson udp->src_port = data->port; 145199a2dd95SBruce Richardson udp->dgram_cksum = udp_cksum; 145299a2dd95SBruce Richardson } 145399a2dd95SBruce Richardson } else { 145499a2dd95SBruce Richardson if (cfg->proto == 0x6) { 145599a2dd95SBruce Richardson struct rte_tcp_hdr *tcp = (struct rte_tcp_hdr *) &ip[1]; 145699a2dd95SBruce Richardson uint16_t tcp_cksum; 145799a2dd95SBruce Richardson 145899a2dd95SBruce Richardson tcp_cksum = nat_ipv6_tcp_udp_checksum_update(tcp->cksum, 145989b5642dSRobin Jarry (uint16_t *)&ip->dst_addr, 14605ac1abddSRobin Jarry (uint16_t *)&data->addr, 146199a2dd95SBruce Richardson tcp->dst_port, 146299a2dd95SBruce Richardson data->port); 146399a2dd95SBruce Richardson 14645ac1abddSRobin Jarry ip->dst_addr = data->addr; 146599a2dd95SBruce Richardson tcp->dst_port = data->port; 146699a2dd95SBruce Richardson tcp->cksum = tcp_cksum; 146799a2dd95SBruce Richardson } else { 146899a2dd95SBruce Richardson struct rte_udp_hdr *udp = (struct rte_udp_hdr *) &ip[1]; 146999a2dd95SBruce Richardson uint16_t udp_cksum; 147099a2dd95SBruce Richardson 147199a2dd95SBruce Richardson udp_cksum = nat_ipv6_tcp_udp_checksum_update(udp->dgram_cksum, 147289b5642dSRobin Jarry (uint16_t *)&ip->dst_addr, 14735ac1abddSRobin Jarry (uint16_t *)&data->addr, 147499a2dd95SBruce Richardson udp->dst_port, 147599a2dd95SBruce Richardson data->port); 147699a2dd95SBruce Richardson 14775ac1abddSRobin Jarry ip->dst_addr = data->addr; 147899a2dd95SBruce Richardson udp->dst_port = data->port; 147999a2dd95SBruce Richardson udp->dgram_cksum = udp_cksum; 148099a2dd95SBruce Richardson } 148199a2dd95SBruce Richardson } 148299a2dd95SBruce Richardson } 148399a2dd95SBruce Richardson 148499a2dd95SBruce Richardson /** 148599a2dd95SBruce Richardson * RTE_TABLE_ACTION_TTL 148699a2dd95SBruce Richardson */ 148799a2dd95SBruce Richardson static int 148899a2dd95SBruce Richardson ttl_cfg_check(struct rte_table_action_ttl_config *ttl) 148999a2dd95SBruce Richardson { 149099a2dd95SBruce Richardson if (ttl->drop == 0) 149199a2dd95SBruce Richardson return -ENOTSUP; 149299a2dd95SBruce Richardson 149399a2dd95SBruce Richardson return 0; 149499a2dd95SBruce Richardson } 149599a2dd95SBruce Richardson 1496*fba98755SAndre Muezerie struct __rte_packed_begin ttl_data { 149799a2dd95SBruce Richardson uint32_t n_packets; 1498*fba98755SAndre Muezerie } __rte_packed_end; 149999a2dd95SBruce Richardson 150099a2dd95SBruce Richardson #define TTL_INIT(data, decrement) \ 150199a2dd95SBruce Richardson ((data)->n_packets = (decrement) ? 1 : 0) 150299a2dd95SBruce Richardson 150399a2dd95SBruce Richardson #define TTL_DEC_GET(data) \ 150499a2dd95SBruce Richardson ((uint8_t)((data)->n_packets & 1)) 150599a2dd95SBruce Richardson 150699a2dd95SBruce Richardson #define TTL_STATS_RESET(data) \ 150799a2dd95SBruce Richardson ((data)->n_packets = ((data)->n_packets & 1)) 150899a2dd95SBruce Richardson 150999a2dd95SBruce Richardson #define TTL_STATS_READ(data) \ 151099a2dd95SBruce Richardson ((data)->n_packets >> 1) 151199a2dd95SBruce Richardson 151299a2dd95SBruce Richardson #define TTL_STATS_ADD(data, value) \ 151399a2dd95SBruce Richardson ((data)->n_packets = \ 151499a2dd95SBruce Richardson (((((data)->n_packets >> 1) + (value)) << 1) | \ 151599a2dd95SBruce Richardson ((data)->n_packets & 1))) 151699a2dd95SBruce Richardson 151799a2dd95SBruce Richardson static int 151899a2dd95SBruce Richardson ttl_apply(void *data, 151999a2dd95SBruce Richardson struct rte_table_action_ttl_params *p) 152099a2dd95SBruce Richardson { 152199a2dd95SBruce Richardson struct ttl_data *d = data; 152299a2dd95SBruce Richardson 152399a2dd95SBruce Richardson TTL_INIT(d, p->decrement); 152499a2dd95SBruce Richardson 152599a2dd95SBruce Richardson return 0; 152699a2dd95SBruce Richardson } 152799a2dd95SBruce Richardson 152899a2dd95SBruce Richardson static __rte_always_inline uint64_t 152999a2dd95SBruce Richardson pkt_ipv4_work_ttl(struct rte_ipv4_hdr *ip, 153099a2dd95SBruce Richardson struct ttl_data *data) 153199a2dd95SBruce Richardson { 153299a2dd95SBruce Richardson uint32_t drop; 153399a2dd95SBruce Richardson uint16_t cksum = ip->hdr_checksum; 153499a2dd95SBruce Richardson uint8_t ttl = ip->time_to_live; 153599a2dd95SBruce Richardson uint8_t ttl_diff = TTL_DEC_GET(data); 153699a2dd95SBruce Richardson 153799a2dd95SBruce Richardson cksum += ttl_diff; 153899a2dd95SBruce Richardson ttl -= ttl_diff; 153999a2dd95SBruce Richardson 154099a2dd95SBruce Richardson ip->hdr_checksum = cksum; 154199a2dd95SBruce Richardson ip->time_to_live = ttl; 154299a2dd95SBruce Richardson 154399a2dd95SBruce Richardson drop = (ttl == 0) ? 1 : 0; 154499a2dd95SBruce Richardson TTL_STATS_ADD(data, drop); 154599a2dd95SBruce Richardson 154699a2dd95SBruce Richardson return drop; 154799a2dd95SBruce Richardson } 154899a2dd95SBruce Richardson 154999a2dd95SBruce Richardson static __rte_always_inline uint64_t 155099a2dd95SBruce Richardson pkt_ipv6_work_ttl(struct rte_ipv6_hdr *ip, 155199a2dd95SBruce Richardson struct ttl_data *data) 155299a2dd95SBruce Richardson { 155399a2dd95SBruce Richardson uint32_t drop; 155499a2dd95SBruce Richardson uint8_t ttl = ip->hop_limits; 155599a2dd95SBruce Richardson uint8_t ttl_diff = TTL_DEC_GET(data); 155699a2dd95SBruce Richardson 155799a2dd95SBruce Richardson ttl -= ttl_diff; 155899a2dd95SBruce Richardson 155999a2dd95SBruce Richardson ip->hop_limits = ttl; 156099a2dd95SBruce Richardson 156199a2dd95SBruce Richardson drop = (ttl == 0) ? 1 : 0; 156299a2dd95SBruce Richardson TTL_STATS_ADD(data, drop); 156399a2dd95SBruce Richardson 156499a2dd95SBruce Richardson return drop; 156599a2dd95SBruce Richardson } 156699a2dd95SBruce Richardson 156799a2dd95SBruce Richardson /** 156899a2dd95SBruce Richardson * RTE_TABLE_ACTION_STATS 156999a2dd95SBruce Richardson */ 157099a2dd95SBruce Richardson static int 157199a2dd95SBruce Richardson stats_cfg_check(struct rte_table_action_stats_config *stats) 157299a2dd95SBruce Richardson { 157399a2dd95SBruce Richardson if ((stats->n_packets_enabled == 0) && (stats->n_bytes_enabled == 0)) 157499a2dd95SBruce Richardson return -EINVAL; 157599a2dd95SBruce Richardson 157699a2dd95SBruce Richardson return 0; 157799a2dd95SBruce Richardson } 157899a2dd95SBruce Richardson 1579*fba98755SAndre Muezerie struct __rte_packed_begin stats_data { 158099a2dd95SBruce Richardson uint64_t n_packets; 158199a2dd95SBruce Richardson uint64_t n_bytes; 1582*fba98755SAndre Muezerie } __rte_packed_end; 158399a2dd95SBruce Richardson 158499a2dd95SBruce Richardson static int 158599a2dd95SBruce Richardson stats_apply(struct stats_data *data, 158699a2dd95SBruce Richardson struct rte_table_action_stats_params *p) 158799a2dd95SBruce Richardson { 158899a2dd95SBruce Richardson data->n_packets = p->n_packets; 158999a2dd95SBruce Richardson data->n_bytes = p->n_bytes; 159099a2dd95SBruce Richardson 159199a2dd95SBruce Richardson return 0; 159299a2dd95SBruce Richardson } 159399a2dd95SBruce Richardson 159499a2dd95SBruce Richardson static __rte_always_inline void 159599a2dd95SBruce Richardson pkt_work_stats(struct stats_data *data, 159699a2dd95SBruce Richardson uint16_t total_length) 159799a2dd95SBruce Richardson { 159899a2dd95SBruce Richardson data->n_packets++; 159999a2dd95SBruce Richardson data->n_bytes += total_length; 160099a2dd95SBruce Richardson } 160199a2dd95SBruce Richardson 160299a2dd95SBruce Richardson /** 160399a2dd95SBruce Richardson * RTE_TABLE_ACTION_TIME 160499a2dd95SBruce Richardson */ 1605*fba98755SAndre Muezerie struct __rte_packed_begin time_data { 160699a2dd95SBruce Richardson uint64_t time; 1607*fba98755SAndre Muezerie } __rte_packed_end; 160899a2dd95SBruce Richardson 160999a2dd95SBruce Richardson static int 161099a2dd95SBruce Richardson time_apply(struct time_data *data, 161199a2dd95SBruce Richardson struct rte_table_action_time_params *p) 161299a2dd95SBruce Richardson { 161399a2dd95SBruce Richardson data->time = p->time; 161499a2dd95SBruce Richardson return 0; 161599a2dd95SBruce Richardson } 161699a2dd95SBruce Richardson 161799a2dd95SBruce Richardson static __rte_always_inline void 161899a2dd95SBruce Richardson pkt_work_time(struct time_data *data, 161999a2dd95SBruce Richardson uint64_t time) 162099a2dd95SBruce Richardson { 162199a2dd95SBruce Richardson data->time = time; 162299a2dd95SBruce Richardson } 162399a2dd95SBruce Richardson 162499a2dd95SBruce Richardson 162599a2dd95SBruce Richardson /** 162699a2dd95SBruce Richardson * RTE_TABLE_ACTION_CRYPTO 162799a2dd95SBruce Richardson */ 162899a2dd95SBruce Richardson 162999a2dd95SBruce Richardson #define CRYPTO_OP_MASK_CIPHER 0x1 163099a2dd95SBruce Richardson #define CRYPTO_OP_MASK_AUTH 0x2 163199a2dd95SBruce Richardson #define CRYPTO_OP_MASK_AEAD 0x4 163299a2dd95SBruce Richardson 163399a2dd95SBruce Richardson struct crypto_op_sym_iv_aad { 163499a2dd95SBruce Richardson struct rte_crypto_op op; 163599a2dd95SBruce Richardson struct rte_crypto_sym_op sym_op; 163699a2dd95SBruce Richardson union { 163799a2dd95SBruce Richardson struct { 163899a2dd95SBruce Richardson uint8_t cipher_iv[ 163999a2dd95SBruce Richardson RTE_TABLE_ACTION_SYM_CRYPTO_IV_SIZE_MAX]; 164099a2dd95SBruce Richardson uint8_t auth_iv[ 164199a2dd95SBruce Richardson RTE_TABLE_ACTION_SYM_CRYPTO_IV_SIZE_MAX]; 164299a2dd95SBruce Richardson } cipher_auth; 164399a2dd95SBruce Richardson 164499a2dd95SBruce Richardson struct { 164599a2dd95SBruce Richardson uint8_t iv[RTE_TABLE_ACTION_SYM_CRYPTO_IV_SIZE_MAX]; 164699a2dd95SBruce Richardson uint8_t aad[RTE_TABLE_ACTION_SYM_CRYPTO_AAD_SIZE_MAX]; 164799a2dd95SBruce Richardson } aead_iv_aad; 164899a2dd95SBruce Richardson 164999a2dd95SBruce Richardson } iv_aad; 165099a2dd95SBruce Richardson }; 165199a2dd95SBruce Richardson 1652*fba98755SAndre Muezerie struct __rte_packed_begin sym_crypto_data { 165399a2dd95SBruce Richardson 165499a2dd95SBruce Richardson union { 165599a2dd95SBruce Richardson struct { 165699a2dd95SBruce Richardson 165799a2dd95SBruce Richardson /** Length of cipher iv. */ 165899a2dd95SBruce Richardson uint16_t cipher_iv_len; 165999a2dd95SBruce Richardson 166099a2dd95SBruce Richardson /** Offset from start of IP header to the cipher iv. */ 166199a2dd95SBruce Richardson uint16_t cipher_iv_data_offset; 166299a2dd95SBruce Richardson 166399a2dd95SBruce Richardson /** Length of cipher iv to be updated in the mbuf. */ 166499a2dd95SBruce Richardson uint16_t cipher_iv_update_len; 166599a2dd95SBruce Richardson 166699a2dd95SBruce Richardson /** Offset from start of IP header to the auth iv. */ 166799a2dd95SBruce Richardson uint16_t auth_iv_data_offset; 166899a2dd95SBruce Richardson 166999a2dd95SBruce Richardson /** Length of auth iv in the mbuf. */ 167099a2dd95SBruce Richardson uint16_t auth_iv_len; 167199a2dd95SBruce Richardson 167299a2dd95SBruce Richardson /** Length of auth iv to be updated in the mbuf. */ 167399a2dd95SBruce Richardson uint16_t auth_iv_update_len; 167499a2dd95SBruce Richardson 167599a2dd95SBruce Richardson } cipher_auth; 167699a2dd95SBruce Richardson struct { 167799a2dd95SBruce Richardson 167899a2dd95SBruce Richardson /** Length of iv. */ 167999a2dd95SBruce Richardson uint16_t iv_len; 168099a2dd95SBruce Richardson 168199a2dd95SBruce Richardson /** Offset from start of IP header to the aead iv. */ 168299a2dd95SBruce Richardson uint16_t iv_data_offset; 168399a2dd95SBruce Richardson 168499a2dd95SBruce Richardson /** Length of iv to be updated in the mbuf. */ 168599a2dd95SBruce Richardson uint16_t iv_update_len; 168699a2dd95SBruce Richardson 168799a2dd95SBruce Richardson /** Length of aad */ 168899a2dd95SBruce Richardson uint16_t aad_len; 168999a2dd95SBruce Richardson 169099a2dd95SBruce Richardson /** Offset from start of IP header to the aad. */ 169199a2dd95SBruce Richardson uint16_t aad_data_offset; 169299a2dd95SBruce Richardson 169399a2dd95SBruce Richardson /** Length of aad to updated in the mbuf. */ 169499a2dd95SBruce Richardson uint16_t aad_update_len; 169599a2dd95SBruce Richardson 169699a2dd95SBruce Richardson } aead; 169799a2dd95SBruce Richardson }; 169899a2dd95SBruce Richardson 169999a2dd95SBruce Richardson /** Offset from start of IP header to the data. */ 170099a2dd95SBruce Richardson uint16_t data_offset; 170199a2dd95SBruce Richardson 170299a2dd95SBruce Richardson /** Digest length. */ 170399a2dd95SBruce Richardson uint16_t digest_len; 170499a2dd95SBruce Richardson 170599a2dd95SBruce Richardson /** block size */ 170699a2dd95SBruce Richardson uint16_t block_size; 170799a2dd95SBruce Richardson 170899a2dd95SBruce Richardson /** Mask of crypto operation */ 170999a2dd95SBruce Richardson uint16_t op_mask; 171099a2dd95SBruce Richardson 171199a2dd95SBruce Richardson /** Session pointer. */ 171299a2dd95SBruce Richardson struct rte_cryptodev_sym_session *session; 171399a2dd95SBruce Richardson 171499a2dd95SBruce Richardson /** Direction of crypto, encrypt or decrypt */ 171599a2dd95SBruce Richardson uint16_t direction; 171699a2dd95SBruce Richardson 171799a2dd95SBruce Richardson /** Private data size to store cipher iv / aad. */ 171899a2dd95SBruce Richardson uint8_t iv_aad_data[32]; 171999a2dd95SBruce Richardson 1720*fba98755SAndre Muezerie } __rte_packed_end; 172199a2dd95SBruce Richardson 172299a2dd95SBruce Richardson static int 172399a2dd95SBruce Richardson sym_crypto_cfg_check(struct rte_table_action_sym_crypto_config *cfg) 172499a2dd95SBruce Richardson { 1725e74abd48SAkhil Goyal if (!rte_cryptodev_is_valid_dev(cfg->cryptodev_id)) 172699a2dd95SBruce Richardson return -EINVAL; 172799a2dd95SBruce Richardson if (cfg->mp_create == NULL || cfg->mp_init == NULL) 172899a2dd95SBruce Richardson return -EINVAL; 172999a2dd95SBruce Richardson 173099a2dd95SBruce Richardson return 0; 173199a2dd95SBruce Richardson } 173299a2dd95SBruce Richardson 173399a2dd95SBruce Richardson static int 173499a2dd95SBruce Richardson get_block_size(const struct rte_crypto_sym_xform *xform, uint8_t cdev_id) 173599a2dd95SBruce Richardson { 173699a2dd95SBruce Richardson struct rte_cryptodev_info dev_info; 173799a2dd95SBruce Richardson const struct rte_cryptodev_capabilities *cap; 173899a2dd95SBruce Richardson uint32_t i; 173999a2dd95SBruce Richardson 174099a2dd95SBruce Richardson rte_cryptodev_info_get(cdev_id, &dev_info); 174199a2dd95SBruce Richardson 174299a2dd95SBruce Richardson for (i = 0; dev_info.capabilities[i].op != RTE_CRYPTO_OP_TYPE_UNDEFINED; 174399a2dd95SBruce Richardson i++) { 174499a2dd95SBruce Richardson cap = &dev_info.capabilities[i]; 174599a2dd95SBruce Richardson 174699a2dd95SBruce Richardson if (cap->sym.xform_type != xform->type) 174799a2dd95SBruce Richardson continue; 174899a2dd95SBruce Richardson 174999a2dd95SBruce Richardson if ((xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) && 175099a2dd95SBruce Richardson (cap->sym.cipher.algo == xform->cipher.algo)) 175199a2dd95SBruce Richardson return cap->sym.cipher.block_size; 175299a2dd95SBruce Richardson 175399a2dd95SBruce Richardson if ((xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) && 175499a2dd95SBruce Richardson (cap->sym.aead.algo == xform->aead.algo)) 175599a2dd95SBruce Richardson return cap->sym.aead.block_size; 175699a2dd95SBruce Richardson 175799a2dd95SBruce Richardson if (xform->type == RTE_CRYPTO_SYM_XFORM_NOT_SPECIFIED) 175899a2dd95SBruce Richardson break; 175999a2dd95SBruce Richardson } 176099a2dd95SBruce Richardson 176199a2dd95SBruce Richardson return -1; 176299a2dd95SBruce Richardson } 176399a2dd95SBruce Richardson 176499a2dd95SBruce Richardson static int 176599a2dd95SBruce Richardson sym_crypto_apply(struct sym_crypto_data *data, 176699a2dd95SBruce Richardson struct rte_table_action_sym_crypto_config *cfg, 176799a2dd95SBruce Richardson struct rte_table_action_sym_crypto_params *p) 176899a2dd95SBruce Richardson { 176999a2dd95SBruce Richardson const struct rte_crypto_cipher_xform *cipher_xform = NULL; 177099a2dd95SBruce Richardson const struct rte_crypto_auth_xform *auth_xform = NULL; 177199a2dd95SBruce Richardson const struct rte_crypto_aead_xform *aead_xform = NULL; 177299a2dd95SBruce Richardson struct rte_crypto_sym_xform *xform = p->xform; 177399a2dd95SBruce Richardson struct rte_cryptodev_sym_session *session; 177499a2dd95SBruce Richardson int ret; 177599a2dd95SBruce Richardson 177699a2dd95SBruce Richardson memset(data, 0, sizeof(*data)); 177799a2dd95SBruce Richardson 177899a2dd95SBruce Richardson while (xform) { 177999a2dd95SBruce Richardson if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) { 178099a2dd95SBruce Richardson cipher_xform = &xform->cipher; 178199a2dd95SBruce Richardson 178299a2dd95SBruce Richardson if (cipher_xform->iv.length > 178399a2dd95SBruce Richardson RTE_TABLE_ACTION_SYM_CRYPTO_IV_SIZE_MAX) 178499a2dd95SBruce Richardson return -ENOMEM; 178599a2dd95SBruce Richardson if (cipher_xform->iv.offset != 178699a2dd95SBruce Richardson RTE_TABLE_ACTION_SYM_CRYPTO_IV_OFFSET) 178799a2dd95SBruce Richardson return -EINVAL; 178899a2dd95SBruce Richardson 178999a2dd95SBruce Richardson ret = get_block_size(xform, cfg->cryptodev_id); 179099a2dd95SBruce Richardson if (ret < 0) 179199a2dd95SBruce Richardson return -1; 179299a2dd95SBruce Richardson data->block_size = (uint16_t)ret; 179399a2dd95SBruce Richardson data->op_mask |= CRYPTO_OP_MASK_CIPHER; 179499a2dd95SBruce Richardson 179599a2dd95SBruce Richardson data->cipher_auth.cipher_iv_len = 179699a2dd95SBruce Richardson cipher_xform->iv.length; 179799a2dd95SBruce Richardson data->cipher_auth.cipher_iv_data_offset = (uint16_t) 179899a2dd95SBruce Richardson p->cipher_auth.cipher_iv_update.offset; 179999a2dd95SBruce Richardson data->cipher_auth.cipher_iv_update_len = (uint16_t) 180099a2dd95SBruce Richardson p->cipher_auth.cipher_iv_update.length; 180199a2dd95SBruce Richardson 180299a2dd95SBruce Richardson rte_memcpy(data->iv_aad_data, 180399a2dd95SBruce Richardson p->cipher_auth.cipher_iv.val, 180499a2dd95SBruce Richardson p->cipher_auth.cipher_iv.length); 180599a2dd95SBruce Richardson 180699a2dd95SBruce Richardson data->direction = cipher_xform->op; 180799a2dd95SBruce Richardson 180899a2dd95SBruce Richardson } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) { 180999a2dd95SBruce Richardson auth_xform = &xform->auth; 181099a2dd95SBruce Richardson if (auth_xform->iv.length > 181199a2dd95SBruce Richardson RTE_TABLE_ACTION_SYM_CRYPTO_IV_SIZE_MAX) 181299a2dd95SBruce Richardson return -ENOMEM; 181399a2dd95SBruce Richardson data->op_mask |= CRYPTO_OP_MASK_AUTH; 181499a2dd95SBruce Richardson 181599a2dd95SBruce Richardson data->cipher_auth.auth_iv_len = auth_xform->iv.length; 181699a2dd95SBruce Richardson data->cipher_auth.auth_iv_data_offset = (uint16_t) 181799a2dd95SBruce Richardson p->cipher_auth.auth_iv_update.offset; 181899a2dd95SBruce Richardson data->cipher_auth.auth_iv_update_len = (uint16_t) 181999a2dd95SBruce Richardson p->cipher_auth.auth_iv_update.length; 182099a2dd95SBruce Richardson data->digest_len = auth_xform->digest_length; 182199a2dd95SBruce Richardson 182299a2dd95SBruce Richardson data->direction = (auth_xform->op == 182399a2dd95SBruce Richardson RTE_CRYPTO_AUTH_OP_GENERATE) ? 182499a2dd95SBruce Richardson RTE_CRYPTO_CIPHER_OP_ENCRYPT : 182599a2dd95SBruce Richardson RTE_CRYPTO_CIPHER_OP_DECRYPT; 182699a2dd95SBruce Richardson 182799a2dd95SBruce Richardson } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) { 182899a2dd95SBruce Richardson aead_xform = &xform->aead; 182999a2dd95SBruce Richardson 183099a2dd95SBruce Richardson if ((aead_xform->iv.length > 183199a2dd95SBruce Richardson RTE_TABLE_ACTION_SYM_CRYPTO_IV_SIZE_MAX) || ( 183299a2dd95SBruce Richardson aead_xform->aad_length > 183399a2dd95SBruce Richardson RTE_TABLE_ACTION_SYM_CRYPTO_AAD_SIZE_MAX)) 183499a2dd95SBruce Richardson return -EINVAL; 183599a2dd95SBruce Richardson if (aead_xform->iv.offset != 183699a2dd95SBruce Richardson RTE_TABLE_ACTION_SYM_CRYPTO_IV_OFFSET) 183799a2dd95SBruce Richardson return -EINVAL; 183899a2dd95SBruce Richardson 183999a2dd95SBruce Richardson ret = get_block_size(xform, cfg->cryptodev_id); 184099a2dd95SBruce Richardson if (ret < 0) 184199a2dd95SBruce Richardson return -1; 184299a2dd95SBruce Richardson data->block_size = (uint16_t)ret; 184399a2dd95SBruce Richardson data->op_mask |= CRYPTO_OP_MASK_AEAD; 184499a2dd95SBruce Richardson 184599a2dd95SBruce Richardson data->digest_len = aead_xform->digest_length; 184699a2dd95SBruce Richardson data->aead.iv_len = aead_xform->iv.length; 184799a2dd95SBruce Richardson data->aead.aad_len = aead_xform->aad_length; 184899a2dd95SBruce Richardson 184999a2dd95SBruce Richardson data->aead.iv_data_offset = (uint16_t) 185099a2dd95SBruce Richardson p->aead.iv_update.offset; 185199a2dd95SBruce Richardson data->aead.iv_update_len = (uint16_t) 185299a2dd95SBruce Richardson p->aead.iv_update.length; 185399a2dd95SBruce Richardson data->aead.aad_data_offset = (uint16_t) 185499a2dd95SBruce Richardson p->aead.aad_update.offset; 185599a2dd95SBruce Richardson data->aead.aad_update_len = (uint16_t) 185699a2dd95SBruce Richardson p->aead.aad_update.length; 185799a2dd95SBruce Richardson 185899a2dd95SBruce Richardson rte_memcpy(data->iv_aad_data, 185999a2dd95SBruce Richardson p->aead.iv.val, 186099a2dd95SBruce Richardson p->aead.iv.length); 186199a2dd95SBruce Richardson 186299a2dd95SBruce Richardson rte_memcpy(data->iv_aad_data + p->aead.iv.length, 186399a2dd95SBruce Richardson p->aead.aad.val, 186499a2dd95SBruce Richardson p->aead.aad.length); 186599a2dd95SBruce Richardson 186699a2dd95SBruce Richardson data->direction = (aead_xform->op == 186799a2dd95SBruce Richardson RTE_CRYPTO_AEAD_OP_ENCRYPT) ? 186899a2dd95SBruce Richardson RTE_CRYPTO_CIPHER_OP_ENCRYPT : 186999a2dd95SBruce Richardson RTE_CRYPTO_CIPHER_OP_DECRYPT; 187099a2dd95SBruce Richardson } else 187199a2dd95SBruce Richardson return -EINVAL; 187299a2dd95SBruce Richardson 187399a2dd95SBruce Richardson xform = xform->next; 187499a2dd95SBruce Richardson } 187599a2dd95SBruce Richardson 187699a2dd95SBruce Richardson if (auth_xform && auth_xform->iv.length) { 187799a2dd95SBruce Richardson if (cipher_xform) { 187899a2dd95SBruce Richardson if (auth_xform->iv.offset != 187999a2dd95SBruce Richardson RTE_TABLE_ACTION_SYM_CRYPTO_IV_OFFSET + 188099a2dd95SBruce Richardson cipher_xform->iv.length) 188199a2dd95SBruce Richardson return -EINVAL; 188299a2dd95SBruce Richardson 188399a2dd95SBruce Richardson rte_memcpy(data->iv_aad_data + cipher_xform->iv.length, 188499a2dd95SBruce Richardson p->cipher_auth.auth_iv.val, 188599a2dd95SBruce Richardson p->cipher_auth.auth_iv.length); 188699a2dd95SBruce Richardson } else { 188799a2dd95SBruce Richardson rte_memcpy(data->iv_aad_data, 188899a2dd95SBruce Richardson p->cipher_auth.auth_iv.val, 188999a2dd95SBruce Richardson p->cipher_auth.auth_iv.length); 189099a2dd95SBruce Richardson } 189199a2dd95SBruce Richardson } 189299a2dd95SBruce Richardson 1893bdce2564SAkhil Goyal session = rte_cryptodev_sym_session_create(cfg->cryptodev_id, 1894bdce2564SAkhil Goyal p->xform, cfg->mp_create); 189599a2dd95SBruce Richardson if (!session) 189699a2dd95SBruce Richardson return -ENOMEM; 189799a2dd95SBruce Richardson 189899a2dd95SBruce Richardson data->data_offset = (uint16_t)p->data_offset; 189999a2dd95SBruce Richardson data->session = session; 190099a2dd95SBruce Richardson 190199a2dd95SBruce Richardson return 0; 190299a2dd95SBruce Richardson } 190399a2dd95SBruce Richardson 190499a2dd95SBruce Richardson static __rte_always_inline uint64_t 190599a2dd95SBruce Richardson pkt_work_sym_crypto(struct rte_mbuf *mbuf, struct sym_crypto_data *data, 190699a2dd95SBruce Richardson struct rte_table_action_sym_crypto_config *cfg, 190799a2dd95SBruce Richardson uint16_t ip_offset) 190899a2dd95SBruce Richardson { 190999a2dd95SBruce Richardson struct crypto_op_sym_iv_aad *crypto_op = (struct crypto_op_sym_iv_aad *) 191099a2dd95SBruce Richardson RTE_MBUF_METADATA_UINT8_PTR(mbuf, cfg->op_offset); 191199a2dd95SBruce Richardson struct rte_crypto_op *op = &crypto_op->op; 191299a2dd95SBruce Richardson struct rte_crypto_sym_op *sym = op->sym; 191399a2dd95SBruce Richardson uint32_t pkt_offset = sizeof(*mbuf) + mbuf->data_off; 191499a2dd95SBruce Richardson uint32_t payload_len = pkt_offset + mbuf->data_len - data->data_offset; 191599a2dd95SBruce Richardson 191699a2dd95SBruce Richardson op->type = RTE_CRYPTO_OP_TYPE_SYMMETRIC; 191799a2dd95SBruce Richardson op->sess_type = RTE_CRYPTO_OP_WITH_SESSION; 1918e811e2d7SShijith Thotton op->phys_addr = rte_mbuf_iova_get(mbuf) + cfg->op_offset - sizeof(*mbuf); 191999a2dd95SBruce Richardson op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 192099a2dd95SBruce Richardson sym->m_src = mbuf; 192199a2dd95SBruce Richardson sym->m_dst = NULL; 192299a2dd95SBruce Richardson sym->session = data->session; 192399a2dd95SBruce Richardson 192499a2dd95SBruce Richardson /** pad the packet */ 192599a2dd95SBruce Richardson if (data->direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { 192699a2dd95SBruce Richardson uint32_t append_len = RTE_ALIGN_CEIL(payload_len, 192799a2dd95SBruce Richardson data->block_size) - payload_len; 192899a2dd95SBruce Richardson 192999a2dd95SBruce Richardson if (unlikely(rte_pktmbuf_append(mbuf, append_len + 193099a2dd95SBruce Richardson data->digest_len) == NULL)) 193199a2dd95SBruce Richardson return 1; 193299a2dd95SBruce Richardson 193399a2dd95SBruce Richardson payload_len += append_len; 193499a2dd95SBruce Richardson } else 193599a2dd95SBruce Richardson payload_len -= data->digest_len; 193699a2dd95SBruce Richardson 193799a2dd95SBruce Richardson if (data->op_mask & CRYPTO_OP_MASK_CIPHER) { 193899a2dd95SBruce Richardson /** prepare cipher op */ 193999a2dd95SBruce Richardson uint8_t *iv = crypto_op->iv_aad.cipher_auth.cipher_iv; 194099a2dd95SBruce Richardson 194199a2dd95SBruce Richardson sym->cipher.data.length = payload_len; 194299a2dd95SBruce Richardson sym->cipher.data.offset = data->data_offset - pkt_offset; 194399a2dd95SBruce Richardson 194499a2dd95SBruce Richardson if (data->cipher_auth.cipher_iv_update_len) { 194599a2dd95SBruce Richardson uint8_t *pkt_iv = RTE_MBUF_METADATA_UINT8_PTR(mbuf, 194699a2dd95SBruce Richardson data->cipher_auth.cipher_iv_data_offset 194799a2dd95SBruce Richardson + ip_offset); 194899a2dd95SBruce Richardson 194999a2dd95SBruce Richardson /** For encryption, update the pkt iv field, otherwise 195099a2dd95SBruce Richardson * update the iv_aad_field 19513e4c5be9SThomas Monjalon */ 195299a2dd95SBruce Richardson if (data->direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) 195399a2dd95SBruce Richardson rte_memcpy(pkt_iv, data->iv_aad_data, 195499a2dd95SBruce Richardson data->cipher_auth.cipher_iv_update_len); 195599a2dd95SBruce Richardson else 195699a2dd95SBruce Richardson rte_memcpy(data->iv_aad_data, pkt_iv, 195799a2dd95SBruce Richardson data->cipher_auth.cipher_iv_update_len); 195899a2dd95SBruce Richardson } 195999a2dd95SBruce Richardson 196099a2dd95SBruce Richardson /** write iv */ 196199a2dd95SBruce Richardson rte_memcpy(iv, data->iv_aad_data, 196299a2dd95SBruce Richardson data->cipher_auth.cipher_iv_len); 196399a2dd95SBruce Richardson } 196499a2dd95SBruce Richardson 196599a2dd95SBruce Richardson if (data->op_mask & CRYPTO_OP_MASK_AUTH) { 196699a2dd95SBruce Richardson /** authentication always start from IP header. */ 196799a2dd95SBruce Richardson sym->auth.data.offset = ip_offset - pkt_offset; 196899a2dd95SBruce Richardson sym->auth.data.length = mbuf->data_len - sym->auth.data.offset - 196999a2dd95SBruce Richardson data->digest_len; 197099a2dd95SBruce Richardson sym->auth.digest.data = rte_pktmbuf_mtod_offset(mbuf, 197199a2dd95SBruce Richardson uint8_t *, rte_pktmbuf_pkt_len(mbuf) - 197299a2dd95SBruce Richardson data->digest_len); 197399a2dd95SBruce Richardson sym->auth.digest.phys_addr = rte_pktmbuf_iova_offset(mbuf, 197499a2dd95SBruce Richardson rte_pktmbuf_pkt_len(mbuf) - data->digest_len); 197599a2dd95SBruce Richardson 197699a2dd95SBruce Richardson if (data->cipher_auth.auth_iv_update_len) { 197799a2dd95SBruce Richardson uint8_t *pkt_iv = RTE_MBUF_METADATA_UINT8_PTR(mbuf, 197899a2dd95SBruce Richardson data->cipher_auth.auth_iv_data_offset 197999a2dd95SBruce Richardson + ip_offset); 198099a2dd95SBruce Richardson uint8_t *data_iv = data->iv_aad_data + 198199a2dd95SBruce Richardson data->cipher_auth.cipher_iv_len; 198299a2dd95SBruce Richardson 198399a2dd95SBruce Richardson if (data->direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) 198499a2dd95SBruce Richardson rte_memcpy(pkt_iv, data_iv, 198599a2dd95SBruce Richardson data->cipher_auth.auth_iv_update_len); 198699a2dd95SBruce Richardson else 198799a2dd95SBruce Richardson rte_memcpy(data_iv, pkt_iv, 198899a2dd95SBruce Richardson data->cipher_auth.auth_iv_update_len); 198999a2dd95SBruce Richardson } 199099a2dd95SBruce Richardson 199199a2dd95SBruce Richardson if (data->cipher_auth.auth_iv_len) { 199299a2dd95SBruce Richardson /** prepare cipher op */ 199399a2dd95SBruce Richardson uint8_t *iv = crypto_op->iv_aad.cipher_auth.auth_iv; 199499a2dd95SBruce Richardson 199599a2dd95SBruce Richardson rte_memcpy(iv, data->iv_aad_data + 199699a2dd95SBruce Richardson data->cipher_auth.cipher_iv_len, 199799a2dd95SBruce Richardson data->cipher_auth.auth_iv_len); 199899a2dd95SBruce Richardson } 199999a2dd95SBruce Richardson } 200099a2dd95SBruce Richardson 200199a2dd95SBruce Richardson if (data->op_mask & CRYPTO_OP_MASK_AEAD) { 200299a2dd95SBruce Richardson uint8_t *iv = crypto_op->iv_aad.aead_iv_aad.iv; 200399a2dd95SBruce Richardson uint8_t *aad = crypto_op->iv_aad.aead_iv_aad.aad; 200499a2dd95SBruce Richardson 200599a2dd95SBruce Richardson sym->aead.aad.data = aad; 200699a2dd95SBruce Richardson sym->aead.aad.phys_addr = rte_pktmbuf_iova_offset(mbuf, 200799a2dd95SBruce Richardson aad - rte_pktmbuf_mtod(mbuf, uint8_t *)); 200899a2dd95SBruce Richardson sym->aead.digest.data = rte_pktmbuf_mtod_offset(mbuf, 200999a2dd95SBruce Richardson uint8_t *, rte_pktmbuf_pkt_len(mbuf) - 201099a2dd95SBruce Richardson data->digest_len); 201199a2dd95SBruce Richardson sym->aead.digest.phys_addr = rte_pktmbuf_iova_offset(mbuf, 201299a2dd95SBruce Richardson rte_pktmbuf_pkt_len(mbuf) - data->digest_len); 201399a2dd95SBruce Richardson sym->aead.data.offset = data->data_offset - pkt_offset; 201499a2dd95SBruce Richardson sym->aead.data.length = payload_len; 201599a2dd95SBruce Richardson 201699a2dd95SBruce Richardson if (data->aead.iv_update_len) { 201799a2dd95SBruce Richardson uint8_t *pkt_iv = RTE_MBUF_METADATA_UINT8_PTR(mbuf, 201899a2dd95SBruce Richardson data->aead.iv_data_offset + ip_offset); 201999a2dd95SBruce Richardson uint8_t *data_iv = data->iv_aad_data; 202099a2dd95SBruce Richardson 202199a2dd95SBruce Richardson if (data->direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) 202299a2dd95SBruce Richardson rte_memcpy(pkt_iv, data_iv, 202399a2dd95SBruce Richardson data->aead.iv_update_len); 202499a2dd95SBruce Richardson else 202599a2dd95SBruce Richardson rte_memcpy(data_iv, pkt_iv, 202699a2dd95SBruce Richardson data->aead.iv_update_len); 202799a2dd95SBruce Richardson } 202899a2dd95SBruce Richardson 202999a2dd95SBruce Richardson rte_memcpy(iv, data->iv_aad_data, data->aead.iv_len); 203099a2dd95SBruce Richardson 203199a2dd95SBruce Richardson if (data->aead.aad_update_len) { 203299a2dd95SBruce Richardson uint8_t *pkt_aad = RTE_MBUF_METADATA_UINT8_PTR(mbuf, 203399a2dd95SBruce Richardson data->aead.aad_data_offset + ip_offset); 203499a2dd95SBruce Richardson uint8_t *data_aad = data->iv_aad_data + 203599a2dd95SBruce Richardson data->aead.iv_len; 203699a2dd95SBruce Richardson 203799a2dd95SBruce Richardson if (data->direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) 203899a2dd95SBruce Richardson rte_memcpy(pkt_aad, data_aad, 203999a2dd95SBruce Richardson data->aead.iv_update_len); 204099a2dd95SBruce Richardson else 204199a2dd95SBruce Richardson rte_memcpy(data_aad, pkt_aad, 204299a2dd95SBruce Richardson data->aead.iv_update_len); 204399a2dd95SBruce Richardson } 204499a2dd95SBruce Richardson 204599a2dd95SBruce Richardson rte_memcpy(aad, data->iv_aad_data + data->aead.iv_len, 204699a2dd95SBruce Richardson data->aead.aad_len); 204799a2dd95SBruce Richardson } 204899a2dd95SBruce Richardson 204999a2dd95SBruce Richardson return 0; 205099a2dd95SBruce Richardson } 205199a2dd95SBruce Richardson 205299a2dd95SBruce Richardson /** 205399a2dd95SBruce Richardson * RTE_TABLE_ACTION_TAG 205499a2dd95SBruce Richardson */ 2055*fba98755SAndre Muezerie struct __rte_packed_begin tag_data { 205699a2dd95SBruce Richardson uint32_t tag; 2057*fba98755SAndre Muezerie } __rte_packed_end; 205899a2dd95SBruce Richardson 205999a2dd95SBruce Richardson static int 206099a2dd95SBruce Richardson tag_apply(struct tag_data *data, 206199a2dd95SBruce Richardson struct rte_table_action_tag_params *p) 206299a2dd95SBruce Richardson { 206399a2dd95SBruce Richardson data->tag = p->tag; 206499a2dd95SBruce Richardson return 0; 206599a2dd95SBruce Richardson } 206699a2dd95SBruce Richardson 206799a2dd95SBruce Richardson static __rte_always_inline void 206899a2dd95SBruce Richardson pkt_work_tag(struct rte_mbuf *mbuf, 206999a2dd95SBruce Richardson struct tag_data *data) 207099a2dd95SBruce Richardson { 207199a2dd95SBruce Richardson mbuf->hash.fdir.hi = data->tag; 2072daa02b5cSOlivier Matz mbuf->ol_flags |= RTE_MBUF_F_RX_FDIR | RTE_MBUF_F_RX_FDIR_ID; 207399a2dd95SBruce Richardson } 207499a2dd95SBruce Richardson 207599a2dd95SBruce Richardson static __rte_always_inline void 207699a2dd95SBruce Richardson pkt4_work_tag(struct rte_mbuf *mbuf0, 207799a2dd95SBruce Richardson struct rte_mbuf *mbuf1, 207899a2dd95SBruce Richardson struct rte_mbuf *mbuf2, 207999a2dd95SBruce Richardson struct rte_mbuf *mbuf3, 208099a2dd95SBruce Richardson struct tag_data *data0, 208199a2dd95SBruce Richardson struct tag_data *data1, 208299a2dd95SBruce Richardson struct tag_data *data2, 208399a2dd95SBruce Richardson struct tag_data *data3) 208499a2dd95SBruce Richardson { 208599a2dd95SBruce Richardson mbuf0->hash.fdir.hi = data0->tag; 208699a2dd95SBruce Richardson mbuf1->hash.fdir.hi = data1->tag; 208799a2dd95SBruce Richardson mbuf2->hash.fdir.hi = data2->tag; 208899a2dd95SBruce Richardson mbuf3->hash.fdir.hi = data3->tag; 208999a2dd95SBruce Richardson 2090daa02b5cSOlivier Matz mbuf0->ol_flags |= RTE_MBUF_F_RX_FDIR | RTE_MBUF_F_RX_FDIR_ID; 2091daa02b5cSOlivier Matz mbuf1->ol_flags |= RTE_MBUF_F_RX_FDIR | RTE_MBUF_F_RX_FDIR_ID; 2092daa02b5cSOlivier Matz mbuf2->ol_flags |= RTE_MBUF_F_RX_FDIR | RTE_MBUF_F_RX_FDIR_ID; 2093daa02b5cSOlivier Matz mbuf3->ol_flags |= RTE_MBUF_F_RX_FDIR | RTE_MBUF_F_RX_FDIR_ID; 209499a2dd95SBruce Richardson } 209599a2dd95SBruce Richardson 209699a2dd95SBruce Richardson /** 209799a2dd95SBruce Richardson * RTE_TABLE_ACTION_DECAP 209899a2dd95SBruce Richardson */ 2099*fba98755SAndre Muezerie struct __rte_packed_begin decap_data { 210099a2dd95SBruce Richardson uint16_t n; 2101*fba98755SAndre Muezerie } __rte_packed_end; 210299a2dd95SBruce Richardson 210399a2dd95SBruce Richardson static int 210499a2dd95SBruce Richardson decap_apply(struct decap_data *data, 210599a2dd95SBruce Richardson struct rte_table_action_decap_params *p) 210699a2dd95SBruce Richardson { 210799a2dd95SBruce Richardson data->n = p->n; 210899a2dd95SBruce Richardson return 0; 210999a2dd95SBruce Richardson } 211099a2dd95SBruce Richardson 211199a2dd95SBruce Richardson static __rte_always_inline void 211299a2dd95SBruce Richardson pkt_work_decap(struct rte_mbuf *mbuf, 211399a2dd95SBruce Richardson struct decap_data *data) 211499a2dd95SBruce Richardson { 211599a2dd95SBruce Richardson uint16_t data_off = mbuf->data_off; 211699a2dd95SBruce Richardson uint16_t data_len = mbuf->data_len; 211799a2dd95SBruce Richardson uint32_t pkt_len = mbuf->pkt_len; 211899a2dd95SBruce Richardson uint16_t n = data->n; 211999a2dd95SBruce Richardson 212099a2dd95SBruce Richardson mbuf->data_off = data_off + n; 212199a2dd95SBruce Richardson mbuf->data_len = data_len - n; 212299a2dd95SBruce Richardson mbuf->pkt_len = pkt_len - n; 212399a2dd95SBruce Richardson } 212499a2dd95SBruce Richardson 212599a2dd95SBruce Richardson static __rte_always_inline void 212699a2dd95SBruce Richardson pkt4_work_decap(struct rte_mbuf *mbuf0, 212799a2dd95SBruce Richardson struct rte_mbuf *mbuf1, 212899a2dd95SBruce Richardson struct rte_mbuf *mbuf2, 212999a2dd95SBruce Richardson struct rte_mbuf *mbuf3, 213099a2dd95SBruce Richardson struct decap_data *data0, 213199a2dd95SBruce Richardson struct decap_data *data1, 213299a2dd95SBruce Richardson struct decap_data *data2, 213399a2dd95SBruce Richardson struct decap_data *data3) 213499a2dd95SBruce Richardson { 213599a2dd95SBruce Richardson uint16_t data_off0 = mbuf0->data_off; 213699a2dd95SBruce Richardson uint16_t data_len0 = mbuf0->data_len; 213799a2dd95SBruce Richardson uint32_t pkt_len0 = mbuf0->pkt_len; 213899a2dd95SBruce Richardson 213999a2dd95SBruce Richardson uint16_t data_off1 = mbuf1->data_off; 214099a2dd95SBruce Richardson uint16_t data_len1 = mbuf1->data_len; 214199a2dd95SBruce Richardson uint32_t pkt_len1 = mbuf1->pkt_len; 214299a2dd95SBruce Richardson 214399a2dd95SBruce Richardson uint16_t data_off2 = mbuf2->data_off; 214499a2dd95SBruce Richardson uint16_t data_len2 = mbuf2->data_len; 214599a2dd95SBruce Richardson uint32_t pkt_len2 = mbuf2->pkt_len; 214699a2dd95SBruce Richardson 214799a2dd95SBruce Richardson uint16_t data_off3 = mbuf3->data_off; 214899a2dd95SBruce Richardson uint16_t data_len3 = mbuf3->data_len; 214999a2dd95SBruce Richardson uint32_t pkt_len3 = mbuf3->pkt_len; 215099a2dd95SBruce Richardson 215199a2dd95SBruce Richardson uint16_t n0 = data0->n; 215299a2dd95SBruce Richardson uint16_t n1 = data1->n; 215399a2dd95SBruce Richardson uint16_t n2 = data2->n; 215499a2dd95SBruce Richardson uint16_t n3 = data3->n; 215599a2dd95SBruce Richardson 215699a2dd95SBruce Richardson mbuf0->data_off = data_off0 + n0; 215799a2dd95SBruce Richardson mbuf0->data_len = data_len0 - n0; 215899a2dd95SBruce Richardson mbuf0->pkt_len = pkt_len0 - n0; 215999a2dd95SBruce Richardson 216099a2dd95SBruce Richardson mbuf1->data_off = data_off1 + n1; 216199a2dd95SBruce Richardson mbuf1->data_len = data_len1 - n1; 216299a2dd95SBruce Richardson mbuf1->pkt_len = pkt_len1 - n1; 216399a2dd95SBruce Richardson 216499a2dd95SBruce Richardson mbuf2->data_off = data_off2 + n2; 216599a2dd95SBruce Richardson mbuf2->data_len = data_len2 - n2; 216699a2dd95SBruce Richardson mbuf2->pkt_len = pkt_len2 - n2; 216799a2dd95SBruce Richardson 216899a2dd95SBruce Richardson mbuf3->data_off = data_off3 + n3; 216999a2dd95SBruce Richardson mbuf3->data_len = data_len3 - n3; 217099a2dd95SBruce Richardson mbuf3->pkt_len = pkt_len3 - n3; 217199a2dd95SBruce Richardson } 217299a2dd95SBruce Richardson 217399a2dd95SBruce Richardson /** 217499a2dd95SBruce Richardson * Action profile 217599a2dd95SBruce Richardson */ 217699a2dd95SBruce Richardson static int 217799a2dd95SBruce Richardson action_valid(enum rte_table_action_type action) 217899a2dd95SBruce Richardson { 217999a2dd95SBruce Richardson switch (action) { 218099a2dd95SBruce Richardson case RTE_TABLE_ACTION_FWD: 218199a2dd95SBruce Richardson case RTE_TABLE_ACTION_LB: 218299a2dd95SBruce Richardson case RTE_TABLE_ACTION_MTR: 218399a2dd95SBruce Richardson case RTE_TABLE_ACTION_TM: 218499a2dd95SBruce Richardson case RTE_TABLE_ACTION_ENCAP: 218599a2dd95SBruce Richardson case RTE_TABLE_ACTION_NAT: 218699a2dd95SBruce Richardson case RTE_TABLE_ACTION_TTL: 218799a2dd95SBruce Richardson case RTE_TABLE_ACTION_STATS: 218899a2dd95SBruce Richardson case RTE_TABLE_ACTION_TIME: 218999a2dd95SBruce Richardson case RTE_TABLE_ACTION_SYM_CRYPTO: 219099a2dd95SBruce Richardson case RTE_TABLE_ACTION_TAG: 219199a2dd95SBruce Richardson case RTE_TABLE_ACTION_DECAP: 219299a2dd95SBruce Richardson return 1; 219399a2dd95SBruce Richardson default: 219499a2dd95SBruce Richardson return 0; 219599a2dd95SBruce Richardson } 219699a2dd95SBruce Richardson } 219799a2dd95SBruce Richardson 219899a2dd95SBruce Richardson 219999a2dd95SBruce Richardson #define RTE_TABLE_ACTION_MAX 64 220099a2dd95SBruce Richardson 220199a2dd95SBruce Richardson struct ap_config { 220299a2dd95SBruce Richardson uint64_t action_mask; 220399a2dd95SBruce Richardson struct rte_table_action_common_config common; 220499a2dd95SBruce Richardson struct rte_table_action_lb_config lb; 220599a2dd95SBruce Richardson struct rte_table_action_mtr_config mtr; 220699a2dd95SBruce Richardson struct rte_table_action_tm_config tm; 220799a2dd95SBruce Richardson struct rte_table_action_encap_config encap; 220899a2dd95SBruce Richardson struct rte_table_action_nat_config nat; 220999a2dd95SBruce Richardson struct rte_table_action_ttl_config ttl; 221099a2dd95SBruce Richardson struct rte_table_action_stats_config stats; 221199a2dd95SBruce Richardson struct rte_table_action_sym_crypto_config sym_crypto; 221299a2dd95SBruce Richardson }; 221399a2dd95SBruce Richardson 221499a2dd95SBruce Richardson static size_t 221599a2dd95SBruce Richardson action_cfg_size(enum rte_table_action_type action) 221699a2dd95SBruce Richardson { 221799a2dd95SBruce Richardson switch (action) { 221899a2dd95SBruce Richardson case RTE_TABLE_ACTION_LB: 221999a2dd95SBruce Richardson return sizeof(struct rte_table_action_lb_config); 222099a2dd95SBruce Richardson case RTE_TABLE_ACTION_MTR: 222199a2dd95SBruce Richardson return sizeof(struct rte_table_action_mtr_config); 222299a2dd95SBruce Richardson case RTE_TABLE_ACTION_TM: 222399a2dd95SBruce Richardson return sizeof(struct rte_table_action_tm_config); 222499a2dd95SBruce Richardson case RTE_TABLE_ACTION_ENCAP: 222599a2dd95SBruce Richardson return sizeof(struct rte_table_action_encap_config); 222699a2dd95SBruce Richardson case RTE_TABLE_ACTION_NAT: 222799a2dd95SBruce Richardson return sizeof(struct rte_table_action_nat_config); 222899a2dd95SBruce Richardson case RTE_TABLE_ACTION_TTL: 222999a2dd95SBruce Richardson return sizeof(struct rte_table_action_ttl_config); 223099a2dd95SBruce Richardson case RTE_TABLE_ACTION_STATS: 223199a2dd95SBruce Richardson return sizeof(struct rte_table_action_stats_config); 223299a2dd95SBruce Richardson case RTE_TABLE_ACTION_SYM_CRYPTO: 223399a2dd95SBruce Richardson return sizeof(struct rte_table_action_sym_crypto_config); 223499a2dd95SBruce Richardson default: 223599a2dd95SBruce Richardson return 0; 223699a2dd95SBruce Richardson } 223799a2dd95SBruce Richardson } 223899a2dd95SBruce Richardson 223999a2dd95SBruce Richardson static void* 224099a2dd95SBruce Richardson action_cfg_get(struct ap_config *ap_config, 224199a2dd95SBruce Richardson enum rte_table_action_type type) 224299a2dd95SBruce Richardson { 224399a2dd95SBruce Richardson switch (type) { 224499a2dd95SBruce Richardson case RTE_TABLE_ACTION_LB: 224599a2dd95SBruce Richardson return &ap_config->lb; 224699a2dd95SBruce Richardson 224799a2dd95SBruce Richardson case RTE_TABLE_ACTION_MTR: 224899a2dd95SBruce Richardson return &ap_config->mtr; 224999a2dd95SBruce Richardson 225099a2dd95SBruce Richardson case RTE_TABLE_ACTION_TM: 225199a2dd95SBruce Richardson return &ap_config->tm; 225299a2dd95SBruce Richardson 225399a2dd95SBruce Richardson case RTE_TABLE_ACTION_ENCAP: 225499a2dd95SBruce Richardson return &ap_config->encap; 225599a2dd95SBruce Richardson 225699a2dd95SBruce Richardson case RTE_TABLE_ACTION_NAT: 225799a2dd95SBruce Richardson return &ap_config->nat; 225899a2dd95SBruce Richardson 225999a2dd95SBruce Richardson case RTE_TABLE_ACTION_TTL: 226099a2dd95SBruce Richardson return &ap_config->ttl; 226199a2dd95SBruce Richardson 226299a2dd95SBruce Richardson case RTE_TABLE_ACTION_STATS: 226399a2dd95SBruce Richardson return &ap_config->stats; 226499a2dd95SBruce Richardson 226599a2dd95SBruce Richardson case RTE_TABLE_ACTION_SYM_CRYPTO: 226699a2dd95SBruce Richardson return &ap_config->sym_crypto; 226799a2dd95SBruce Richardson default: 226899a2dd95SBruce Richardson return NULL; 226999a2dd95SBruce Richardson } 227099a2dd95SBruce Richardson } 227199a2dd95SBruce Richardson 227299a2dd95SBruce Richardson static void 227399a2dd95SBruce Richardson action_cfg_set(struct ap_config *ap_config, 227499a2dd95SBruce Richardson enum rte_table_action_type type, 227599a2dd95SBruce Richardson void *action_cfg) 227699a2dd95SBruce Richardson { 227799a2dd95SBruce Richardson void *dst = action_cfg_get(ap_config, type); 227899a2dd95SBruce Richardson 227999a2dd95SBruce Richardson if (dst) 228099a2dd95SBruce Richardson memcpy(dst, action_cfg, action_cfg_size(type)); 228199a2dd95SBruce Richardson 228299a2dd95SBruce Richardson ap_config->action_mask |= 1LLU << type; 228399a2dd95SBruce Richardson } 228499a2dd95SBruce Richardson 228599a2dd95SBruce Richardson struct ap_data { 228699a2dd95SBruce Richardson size_t offset[RTE_TABLE_ACTION_MAX]; 228799a2dd95SBruce Richardson size_t total_size; 228899a2dd95SBruce Richardson }; 228999a2dd95SBruce Richardson 229099a2dd95SBruce Richardson static size_t 229199a2dd95SBruce Richardson action_data_size(enum rte_table_action_type action, 229299a2dd95SBruce Richardson struct ap_config *ap_config) 229399a2dd95SBruce Richardson { 229499a2dd95SBruce Richardson switch (action) { 229599a2dd95SBruce Richardson case RTE_TABLE_ACTION_FWD: 229699a2dd95SBruce Richardson return sizeof(struct fwd_data); 229799a2dd95SBruce Richardson 229899a2dd95SBruce Richardson case RTE_TABLE_ACTION_LB: 229999a2dd95SBruce Richardson return sizeof(struct lb_data); 230099a2dd95SBruce Richardson 230199a2dd95SBruce Richardson case RTE_TABLE_ACTION_MTR: 230299a2dd95SBruce Richardson return mtr_data_size(&ap_config->mtr); 230399a2dd95SBruce Richardson 230499a2dd95SBruce Richardson case RTE_TABLE_ACTION_TM: 230599a2dd95SBruce Richardson return sizeof(struct tm_data); 230699a2dd95SBruce Richardson 230799a2dd95SBruce Richardson case RTE_TABLE_ACTION_ENCAP: 230899a2dd95SBruce Richardson return encap_data_size(&ap_config->encap); 230999a2dd95SBruce Richardson 231099a2dd95SBruce Richardson case RTE_TABLE_ACTION_NAT: 231199a2dd95SBruce Richardson return nat_data_size(&ap_config->nat, 231299a2dd95SBruce Richardson &ap_config->common); 231399a2dd95SBruce Richardson 231499a2dd95SBruce Richardson case RTE_TABLE_ACTION_TTL: 231599a2dd95SBruce Richardson return sizeof(struct ttl_data); 231699a2dd95SBruce Richardson 231799a2dd95SBruce Richardson case RTE_TABLE_ACTION_STATS: 231899a2dd95SBruce Richardson return sizeof(struct stats_data); 231999a2dd95SBruce Richardson 232099a2dd95SBruce Richardson case RTE_TABLE_ACTION_TIME: 232199a2dd95SBruce Richardson return sizeof(struct time_data); 232299a2dd95SBruce Richardson 232399a2dd95SBruce Richardson case RTE_TABLE_ACTION_SYM_CRYPTO: 232499a2dd95SBruce Richardson return (sizeof(struct sym_crypto_data)); 232599a2dd95SBruce Richardson 232699a2dd95SBruce Richardson case RTE_TABLE_ACTION_TAG: 232799a2dd95SBruce Richardson return sizeof(struct tag_data); 232899a2dd95SBruce Richardson 232999a2dd95SBruce Richardson case RTE_TABLE_ACTION_DECAP: 233099a2dd95SBruce Richardson return sizeof(struct decap_data); 233199a2dd95SBruce Richardson 233299a2dd95SBruce Richardson default: 233399a2dd95SBruce Richardson return 0; 233499a2dd95SBruce Richardson } 233599a2dd95SBruce Richardson } 233699a2dd95SBruce Richardson 233799a2dd95SBruce Richardson 233899a2dd95SBruce Richardson static void 233999a2dd95SBruce Richardson action_data_offset_set(struct ap_data *ap_data, 234099a2dd95SBruce Richardson struct ap_config *ap_config) 234199a2dd95SBruce Richardson { 234299a2dd95SBruce Richardson uint64_t action_mask = ap_config->action_mask; 234399a2dd95SBruce Richardson size_t offset; 234499a2dd95SBruce Richardson uint32_t action; 234599a2dd95SBruce Richardson 234699a2dd95SBruce Richardson memset(ap_data->offset, 0, sizeof(ap_data->offset)); 234799a2dd95SBruce Richardson 234899a2dd95SBruce Richardson offset = 0; 234999a2dd95SBruce Richardson for (action = 0; action < RTE_TABLE_ACTION_MAX; action++) 235099a2dd95SBruce Richardson if (action_mask & (1LLU << action)) { 235199a2dd95SBruce Richardson ap_data->offset[action] = offset; 235299a2dd95SBruce Richardson offset += action_data_size((enum rte_table_action_type)action, 235399a2dd95SBruce Richardson ap_config); 235499a2dd95SBruce Richardson } 235599a2dd95SBruce Richardson 235699a2dd95SBruce Richardson ap_data->total_size = offset; 235799a2dd95SBruce Richardson } 235899a2dd95SBruce Richardson 235999a2dd95SBruce Richardson struct rte_table_action_profile { 236099a2dd95SBruce Richardson struct ap_config cfg; 236199a2dd95SBruce Richardson struct ap_data data; 236299a2dd95SBruce Richardson int frozen; 236399a2dd95SBruce Richardson }; 236499a2dd95SBruce Richardson 236599a2dd95SBruce Richardson struct rte_table_action_profile * 236699a2dd95SBruce Richardson rte_table_action_profile_create(struct rte_table_action_common_config *common) 236799a2dd95SBruce Richardson { 236899a2dd95SBruce Richardson struct rte_table_action_profile *ap; 236999a2dd95SBruce Richardson 237099a2dd95SBruce Richardson /* Check input arguments */ 237199a2dd95SBruce Richardson if (common == NULL) 237299a2dd95SBruce Richardson return NULL; 237399a2dd95SBruce Richardson 237499a2dd95SBruce Richardson /* Memory allocation */ 237599a2dd95SBruce Richardson ap = calloc(1, sizeof(struct rte_table_action_profile)); 237699a2dd95SBruce Richardson if (ap == NULL) 237799a2dd95SBruce Richardson return NULL; 237899a2dd95SBruce Richardson 237999a2dd95SBruce Richardson /* Initialization */ 238099a2dd95SBruce Richardson memcpy(&ap->cfg.common, common, sizeof(*common)); 238199a2dd95SBruce Richardson 238299a2dd95SBruce Richardson return ap; 238399a2dd95SBruce Richardson } 238499a2dd95SBruce Richardson 238599a2dd95SBruce Richardson 238699a2dd95SBruce Richardson int 238799a2dd95SBruce Richardson rte_table_action_profile_action_register(struct rte_table_action_profile *profile, 238899a2dd95SBruce Richardson enum rte_table_action_type type, 238999a2dd95SBruce Richardson void *action_config) 239099a2dd95SBruce Richardson { 239199a2dd95SBruce Richardson int status; 239299a2dd95SBruce Richardson 239399a2dd95SBruce Richardson /* Check input arguments */ 239499a2dd95SBruce Richardson if ((profile == NULL) || 239599a2dd95SBruce Richardson profile->frozen || 239699a2dd95SBruce Richardson (action_valid(type) == 0) || 239799a2dd95SBruce Richardson (profile->cfg.action_mask & (1LLU << type)) || 239899a2dd95SBruce Richardson ((action_cfg_size(type) == 0) && action_config) || 239999a2dd95SBruce Richardson (action_cfg_size(type) && (action_config == NULL))) 240099a2dd95SBruce Richardson return -EINVAL; 240199a2dd95SBruce Richardson 240299a2dd95SBruce Richardson switch (type) { 240399a2dd95SBruce Richardson case RTE_TABLE_ACTION_LB: 240499a2dd95SBruce Richardson status = lb_cfg_check(action_config); 240599a2dd95SBruce Richardson break; 240699a2dd95SBruce Richardson 240799a2dd95SBruce Richardson case RTE_TABLE_ACTION_MTR: 240899a2dd95SBruce Richardson status = mtr_cfg_check(action_config); 240999a2dd95SBruce Richardson break; 241099a2dd95SBruce Richardson 241199a2dd95SBruce Richardson case RTE_TABLE_ACTION_TM: 241299a2dd95SBruce Richardson status = tm_cfg_check(action_config); 241399a2dd95SBruce Richardson break; 241499a2dd95SBruce Richardson 241599a2dd95SBruce Richardson case RTE_TABLE_ACTION_ENCAP: 241699a2dd95SBruce Richardson status = encap_cfg_check(action_config); 241799a2dd95SBruce Richardson break; 241899a2dd95SBruce Richardson 241999a2dd95SBruce Richardson case RTE_TABLE_ACTION_NAT: 242099a2dd95SBruce Richardson status = nat_cfg_check(action_config); 242199a2dd95SBruce Richardson break; 242299a2dd95SBruce Richardson 242399a2dd95SBruce Richardson case RTE_TABLE_ACTION_TTL: 242499a2dd95SBruce Richardson status = ttl_cfg_check(action_config); 242599a2dd95SBruce Richardson break; 242699a2dd95SBruce Richardson 242799a2dd95SBruce Richardson case RTE_TABLE_ACTION_STATS: 242899a2dd95SBruce Richardson status = stats_cfg_check(action_config); 242999a2dd95SBruce Richardson break; 243099a2dd95SBruce Richardson 243199a2dd95SBruce Richardson case RTE_TABLE_ACTION_SYM_CRYPTO: 243299a2dd95SBruce Richardson status = sym_crypto_cfg_check(action_config); 243399a2dd95SBruce Richardson break; 243499a2dd95SBruce Richardson 243599a2dd95SBruce Richardson default: 243699a2dd95SBruce Richardson status = 0; 243799a2dd95SBruce Richardson break; 243899a2dd95SBruce Richardson } 243999a2dd95SBruce Richardson 244099a2dd95SBruce Richardson if (status) 244199a2dd95SBruce Richardson return status; 244299a2dd95SBruce Richardson 244399a2dd95SBruce Richardson /* Action enable */ 244499a2dd95SBruce Richardson action_cfg_set(&profile->cfg, type, action_config); 244599a2dd95SBruce Richardson 244699a2dd95SBruce Richardson return 0; 244799a2dd95SBruce Richardson } 244899a2dd95SBruce Richardson 244999a2dd95SBruce Richardson int 245099a2dd95SBruce Richardson rte_table_action_profile_freeze(struct rte_table_action_profile *profile) 245199a2dd95SBruce Richardson { 245299a2dd95SBruce Richardson if (profile->frozen) 245399a2dd95SBruce Richardson return -EBUSY; 245499a2dd95SBruce Richardson 245599a2dd95SBruce Richardson profile->cfg.action_mask |= 1LLU << RTE_TABLE_ACTION_FWD; 245699a2dd95SBruce Richardson action_data_offset_set(&profile->data, &profile->cfg); 245799a2dd95SBruce Richardson profile->frozen = 1; 245899a2dd95SBruce Richardson 245999a2dd95SBruce Richardson return 0; 246099a2dd95SBruce Richardson } 246199a2dd95SBruce Richardson 246299a2dd95SBruce Richardson int 246399a2dd95SBruce Richardson rte_table_action_profile_free(struct rte_table_action_profile *profile) 246499a2dd95SBruce Richardson { 246599a2dd95SBruce Richardson if (profile == NULL) 246699a2dd95SBruce Richardson return 0; 246799a2dd95SBruce Richardson 246899a2dd95SBruce Richardson free(profile); 246999a2dd95SBruce Richardson return 0; 247099a2dd95SBruce Richardson } 247199a2dd95SBruce Richardson 247299a2dd95SBruce Richardson /** 247399a2dd95SBruce Richardson * Action 247499a2dd95SBruce Richardson */ 247599a2dd95SBruce Richardson #define METER_PROFILES_MAX 32 247699a2dd95SBruce Richardson 247799a2dd95SBruce Richardson struct rte_table_action { 247899a2dd95SBruce Richardson struct ap_config cfg; 247999a2dd95SBruce Richardson struct ap_data data; 248099a2dd95SBruce Richardson struct dscp_table_data dscp_table; 248199a2dd95SBruce Richardson struct meter_profile_data mp[METER_PROFILES_MAX]; 248299a2dd95SBruce Richardson }; 248399a2dd95SBruce Richardson 248499a2dd95SBruce Richardson struct rte_table_action * 248599a2dd95SBruce Richardson rte_table_action_create(struct rte_table_action_profile *profile, 248699a2dd95SBruce Richardson uint32_t socket_id) 248799a2dd95SBruce Richardson { 248899a2dd95SBruce Richardson struct rte_table_action *action; 248999a2dd95SBruce Richardson 249099a2dd95SBruce Richardson /* Check input arguments */ 249199a2dd95SBruce Richardson if ((profile == NULL) || 249299a2dd95SBruce Richardson (profile->frozen == 0)) 249399a2dd95SBruce Richardson return NULL; 249499a2dd95SBruce Richardson 249599a2dd95SBruce Richardson /* Memory allocation */ 249699a2dd95SBruce Richardson action = rte_zmalloc_socket(NULL, 249799a2dd95SBruce Richardson sizeof(struct rte_table_action), 249899a2dd95SBruce Richardson RTE_CACHE_LINE_SIZE, 249999a2dd95SBruce Richardson socket_id); 250099a2dd95SBruce Richardson if (action == NULL) 250199a2dd95SBruce Richardson return NULL; 250299a2dd95SBruce Richardson 250399a2dd95SBruce Richardson /* Initialization */ 250499a2dd95SBruce Richardson memcpy(&action->cfg, &profile->cfg, sizeof(profile->cfg)); 250599a2dd95SBruce Richardson memcpy(&action->data, &profile->data, sizeof(profile->data)); 250699a2dd95SBruce Richardson 250799a2dd95SBruce Richardson return action; 250899a2dd95SBruce Richardson } 250999a2dd95SBruce Richardson 251099a2dd95SBruce Richardson static __rte_always_inline void * 251199a2dd95SBruce Richardson action_data_get(void *data, 251299a2dd95SBruce Richardson struct rte_table_action *action, 251399a2dd95SBruce Richardson enum rte_table_action_type type) 251499a2dd95SBruce Richardson { 251599a2dd95SBruce Richardson size_t offset = action->data.offset[type]; 251699a2dd95SBruce Richardson uint8_t *data_bytes = data; 251799a2dd95SBruce Richardson 251899a2dd95SBruce Richardson return &data_bytes[offset]; 251999a2dd95SBruce Richardson } 252099a2dd95SBruce Richardson 252199a2dd95SBruce Richardson int 252299a2dd95SBruce Richardson rte_table_action_apply(struct rte_table_action *action, 252399a2dd95SBruce Richardson void *data, 252499a2dd95SBruce Richardson enum rte_table_action_type type, 252599a2dd95SBruce Richardson void *action_params) 252699a2dd95SBruce Richardson { 252799a2dd95SBruce Richardson void *action_data; 252899a2dd95SBruce Richardson 252999a2dd95SBruce Richardson /* Check input arguments */ 253099a2dd95SBruce Richardson if ((action == NULL) || 253199a2dd95SBruce Richardson (data == NULL) || 253299a2dd95SBruce Richardson (action_valid(type) == 0) || 253399a2dd95SBruce Richardson ((action->cfg.action_mask & (1LLU << type)) == 0) || 253499a2dd95SBruce Richardson (action_params == NULL)) 253599a2dd95SBruce Richardson return -EINVAL; 253699a2dd95SBruce Richardson 253799a2dd95SBruce Richardson /* Data update */ 253899a2dd95SBruce Richardson action_data = action_data_get(data, action, type); 253999a2dd95SBruce Richardson 254099a2dd95SBruce Richardson switch (type) { 254199a2dd95SBruce Richardson case RTE_TABLE_ACTION_FWD: 254299a2dd95SBruce Richardson return fwd_apply(action_data, 254399a2dd95SBruce Richardson action_params); 254499a2dd95SBruce Richardson 254599a2dd95SBruce Richardson case RTE_TABLE_ACTION_LB: 254699a2dd95SBruce Richardson return lb_apply(action_data, 254799a2dd95SBruce Richardson action_params); 254899a2dd95SBruce Richardson 254999a2dd95SBruce Richardson case RTE_TABLE_ACTION_MTR: 255099a2dd95SBruce Richardson return mtr_apply(action_data, 255199a2dd95SBruce Richardson action_params, 255299a2dd95SBruce Richardson &action->cfg.mtr, 255399a2dd95SBruce Richardson action->mp, 255499a2dd95SBruce Richardson RTE_DIM(action->mp)); 255599a2dd95SBruce Richardson 255699a2dd95SBruce Richardson case RTE_TABLE_ACTION_TM: 255799a2dd95SBruce Richardson return tm_apply(action_data, 255899a2dd95SBruce Richardson action_params, 255999a2dd95SBruce Richardson &action->cfg.tm); 256099a2dd95SBruce Richardson 256199a2dd95SBruce Richardson case RTE_TABLE_ACTION_ENCAP: 256299a2dd95SBruce Richardson return encap_apply(action_data, 256399a2dd95SBruce Richardson action_params, 256499a2dd95SBruce Richardson &action->cfg.encap, 256599a2dd95SBruce Richardson &action->cfg.common); 256699a2dd95SBruce Richardson 256799a2dd95SBruce Richardson case RTE_TABLE_ACTION_NAT: 256899a2dd95SBruce Richardson return nat_apply(action_data, 256999a2dd95SBruce Richardson action_params, 257099a2dd95SBruce Richardson &action->cfg.common); 257199a2dd95SBruce Richardson 257299a2dd95SBruce Richardson case RTE_TABLE_ACTION_TTL: 257399a2dd95SBruce Richardson return ttl_apply(action_data, 257499a2dd95SBruce Richardson action_params); 257599a2dd95SBruce Richardson 257699a2dd95SBruce Richardson case RTE_TABLE_ACTION_STATS: 257799a2dd95SBruce Richardson return stats_apply(action_data, 257899a2dd95SBruce Richardson action_params); 257999a2dd95SBruce Richardson 258099a2dd95SBruce Richardson case RTE_TABLE_ACTION_TIME: 258199a2dd95SBruce Richardson return time_apply(action_data, 258299a2dd95SBruce Richardson action_params); 258399a2dd95SBruce Richardson 258499a2dd95SBruce Richardson case RTE_TABLE_ACTION_SYM_CRYPTO: 258599a2dd95SBruce Richardson return sym_crypto_apply(action_data, 258699a2dd95SBruce Richardson &action->cfg.sym_crypto, 258799a2dd95SBruce Richardson action_params); 258899a2dd95SBruce Richardson 258999a2dd95SBruce Richardson case RTE_TABLE_ACTION_TAG: 259099a2dd95SBruce Richardson return tag_apply(action_data, 259199a2dd95SBruce Richardson action_params); 259299a2dd95SBruce Richardson 259399a2dd95SBruce Richardson case RTE_TABLE_ACTION_DECAP: 259499a2dd95SBruce Richardson return decap_apply(action_data, 259599a2dd95SBruce Richardson action_params); 259699a2dd95SBruce Richardson 259799a2dd95SBruce Richardson default: 259899a2dd95SBruce Richardson return -EINVAL; 259999a2dd95SBruce Richardson } 260099a2dd95SBruce Richardson } 260199a2dd95SBruce Richardson 260299a2dd95SBruce Richardson int 260399a2dd95SBruce Richardson rte_table_action_dscp_table_update(struct rte_table_action *action, 260499a2dd95SBruce Richardson uint64_t dscp_mask, 260599a2dd95SBruce Richardson struct rte_table_action_dscp_table *table) 260699a2dd95SBruce Richardson { 260799a2dd95SBruce Richardson uint32_t i; 260899a2dd95SBruce Richardson 260999a2dd95SBruce Richardson /* Check input arguments */ 261099a2dd95SBruce Richardson if ((action == NULL) || 261199a2dd95SBruce Richardson ((action->cfg.action_mask & ((1LLU << RTE_TABLE_ACTION_MTR) | 261299a2dd95SBruce Richardson (1LLU << RTE_TABLE_ACTION_TM))) == 0) || 261399a2dd95SBruce Richardson (dscp_mask == 0) || 261499a2dd95SBruce Richardson (table == NULL)) 261599a2dd95SBruce Richardson return -EINVAL; 261699a2dd95SBruce Richardson 261799a2dd95SBruce Richardson for (i = 0; i < RTE_DIM(table->entry); i++) { 261899a2dd95SBruce Richardson struct dscp_table_entry_data *data = 261999a2dd95SBruce Richardson &action->dscp_table.entry[i]; 262099a2dd95SBruce Richardson struct rte_table_action_dscp_table_entry *entry = 262199a2dd95SBruce Richardson &table->entry[i]; 262299a2dd95SBruce Richardson 262399a2dd95SBruce Richardson if ((dscp_mask & (1LLU << i)) == 0) 262499a2dd95SBruce Richardson continue; 262599a2dd95SBruce Richardson 262699a2dd95SBruce Richardson data->color = entry->color; 262799a2dd95SBruce Richardson data->tc = entry->tc_id; 262899a2dd95SBruce Richardson data->tc_queue = entry->tc_queue_id; 262999a2dd95SBruce Richardson } 263099a2dd95SBruce Richardson 263199a2dd95SBruce Richardson return 0; 263299a2dd95SBruce Richardson } 263399a2dd95SBruce Richardson 263499a2dd95SBruce Richardson int 263599a2dd95SBruce Richardson rte_table_action_meter_profile_add(struct rte_table_action *action, 263699a2dd95SBruce Richardson uint32_t meter_profile_id, 263799a2dd95SBruce Richardson struct rte_table_action_meter_profile *profile) 263899a2dd95SBruce Richardson { 263999a2dd95SBruce Richardson struct meter_profile_data *mp_data; 264099a2dd95SBruce Richardson uint32_t status; 264199a2dd95SBruce Richardson 264299a2dd95SBruce Richardson /* Check input arguments */ 264399a2dd95SBruce Richardson if ((action == NULL) || 264499a2dd95SBruce Richardson ((action->cfg.action_mask & (1LLU << RTE_TABLE_ACTION_MTR)) == 0) || 264599a2dd95SBruce Richardson (profile == NULL)) 264699a2dd95SBruce Richardson return -EINVAL; 264799a2dd95SBruce Richardson 264899a2dd95SBruce Richardson if (profile->alg != RTE_TABLE_ACTION_METER_TRTCM) 264999a2dd95SBruce Richardson return -ENOTSUP; 265099a2dd95SBruce Richardson 265199a2dd95SBruce Richardson mp_data = meter_profile_data_find(action->mp, 265299a2dd95SBruce Richardson RTE_DIM(action->mp), 265399a2dd95SBruce Richardson meter_profile_id); 265499a2dd95SBruce Richardson if (mp_data) 265599a2dd95SBruce Richardson return -EEXIST; 265699a2dd95SBruce Richardson 265799a2dd95SBruce Richardson mp_data = meter_profile_data_find_unused(action->mp, 265899a2dd95SBruce Richardson RTE_DIM(action->mp)); 265999a2dd95SBruce Richardson if (!mp_data) 266099a2dd95SBruce Richardson return -ENOSPC; 266199a2dd95SBruce Richardson 266299a2dd95SBruce Richardson /* Install new profile */ 266399a2dd95SBruce Richardson status = rte_meter_trtcm_profile_config(&mp_data->profile, 266499a2dd95SBruce Richardson &profile->trtcm); 266599a2dd95SBruce Richardson if (status) 266699a2dd95SBruce Richardson return status; 266799a2dd95SBruce Richardson 266899a2dd95SBruce Richardson mp_data->profile_id = meter_profile_id; 266999a2dd95SBruce Richardson mp_data->valid = 1; 267099a2dd95SBruce Richardson 267199a2dd95SBruce Richardson return 0; 267299a2dd95SBruce Richardson } 267399a2dd95SBruce Richardson 267499a2dd95SBruce Richardson int 267599a2dd95SBruce Richardson rte_table_action_meter_profile_delete(struct rte_table_action *action, 267699a2dd95SBruce Richardson uint32_t meter_profile_id) 267799a2dd95SBruce Richardson { 267899a2dd95SBruce Richardson struct meter_profile_data *mp_data; 267999a2dd95SBruce Richardson 268099a2dd95SBruce Richardson /* Check input arguments */ 268199a2dd95SBruce Richardson if ((action == NULL) || 268299a2dd95SBruce Richardson ((action->cfg.action_mask & (1LLU << RTE_TABLE_ACTION_MTR)) == 0)) 268399a2dd95SBruce Richardson return -EINVAL; 268499a2dd95SBruce Richardson 268599a2dd95SBruce Richardson mp_data = meter_profile_data_find(action->mp, 268699a2dd95SBruce Richardson RTE_DIM(action->mp), 268799a2dd95SBruce Richardson meter_profile_id); 268899a2dd95SBruce Richardson if (!mp_data) 268999a2dd95SBruce Richardson return 0; 269099a2dd95SBruce Richardson 269199a2dd95SBruce Richardson /* Uninstall profile */ 269299a2dd95SBruce Richardson mp_data->valid = 0; 269399a2dd95SBruce Richardson 269499a2dd95SBruce Richardson return 0; 269599a2dd95SBruce Richardson } 269699a2dd95SBruce Richardson 269799a2dd95SBruce Richardson int 269899a2dd95SBruce Richardson rte_table_action_meter_read(struct rte_table_action *action, 269999a2dd95SBruce Richardson void *data, 270099a2dd95SBruce Richardson uint32_t tc_mask, 270199a2dd95SBruce Richardson struct rte_table_action_mtr_counters *stats, 270299a2dd95SBruce Richardson int clear) 270399a2dd95SBruce Richardson { 270499a2dd95SBruce Richardson struct mtr_trtcm_data *mtr_data; 270599a2dd95SBruce Richardson uint32_t i; 270699a2dd95SBruce Richardson 270799a2dd95SBruce Richardson /* Check input arguments */ 270899a2dd95SBruce Richardson if ((action == NULL) || 270999a2dd95SBruce Richardson ((action->cfg.action_mask & (1LLU << RTE_TABLE_ACTION_MTR)) == 0) || 271099a2dd95SBruce Richardson (data == NULL) || 271199a2dd95SBruce Richardson (tc_mask > RTE_LEN2MASK(action->cfg.mtr.n_tc, uint32_t))) 271299a2dd95SBruce Richardson return -EINVAL; 271399a2dd95SBruce Richardson 271499a2dd95SBruce Richardson mtr_data = action_data_get(data, action, RTE_TABLE_ACTION_MTR); 271599a2dd95SBruce Richardson 271699a2dd95SBruce Richardson /* Read */ 271799a2dd95SBruce Richardson if (stats) { 271899a2dd95SBruce Richardson for (i = 0; i < RTE_TABLE_ACTION_TC_MAX; i++) { 271999a2dd95SBruce Richardson struct rte_table_action_mtr_counters_tc *dst = 272099a2dd95SBruce Richardson &stats->stats[i]; 272199a2dd95SBruce Richardson struct mtr_trtcm_data *src = &mtr_data[i]; 272299a2dd95SBruce Richardson 272399a2dd95SBruce Richardson if ((tc_mask & (1 << i)) == 0) 272499a2dd95SBruce Richardson continue; 272599a2dd95SBruce Richardson 272699a2dd95SBruce Richardson dst->n_packets[RTE_COLOR_GREEN] = 272799a2dd95SBruce Richardson mtr_trtcm_data_stats_get(src, RTE_COLOR_GREEN); 272899a2dd95SBruce Richardson 272999a2dd95SBruce Richardson dst->n_packets[RTE_COLOR_YELLOW] = 273099a2dd95SBruce Richardson mtr_trtcm_data_stats_get(src, RTE_COLOR_YELLOW); 273199a2dd95SBruce Richardson 273299a2dd95SBruce Richardson dst->n_packets[RTE_COLOR_RED] = 273399a2dd95SBruce Richardson mtr_trtcm_data_stats_get(src, RTE_COLOR_RED); 273499a2dd95SBruce Richardson 273599a2dd95SBruce Richardson dst->n_packets_valid = 1; 273699a2dd95SBruce Richardson dst->n_bytes_valid = 0; 273799a2dd95SBruce Richardson } 273899a2dd95SBruce Richardson 273999a2dd95SBruce Richardson stats->tc_mask = tc_mask; 274099a2dd95SBruce Richardson } 274199a2dd95SBruce Richardson 274299a2dd95SBruce Richardson /* Clear */ 274399a2dd95SBruce Richardson if (clear) 274499a2dd95SBruce Richardson for (i = 0; i < RTE_TABLE_ACTION_TC_MAX; i++) { 274599a2dd95SBruce Richardson struct mtr_trtcm_data *src = &mtr_data[i]; 274699a2dd95SBruce Richardson 274799a2dd95SBruce Richardson if ((tc_mask & (1 << i)) == 0) 274899a2dd95SBruce Richardson continue; 274999a2dd95SBruce Richardson 275099a2dd95SBruce Richardson mtr_trtcm_data_stats_reset(src, RTE_COLOR_GREEN); 275199a2dd95SBruce Richardson mtr_trtcm_data_stats_reset(src, RTE_COLOR_YELLOW); 275299a2dd95SBruce Richardson mtr_trtcm_data_stats_reset(src, RTE_COLOR_RED); 275399a2dd95SBruce Richardson } 275499a2dd95SBruce Richardson 275599a2dd95SBruce Richardson 275699a2dd95SBruce Richardson return 0; 275799a2dd95SBruce Richardson } 275899a2dd95SBruce Richardson 275999a2dd95SBruce Richardson int 276099a2dd95SBruce Richardson rte_table_action_ttl_read(struct rte_table_action *action, 276199a2dd95SBruce Richardson void *data, 276299a2dd95SBruce Richardson struct rte_table_action_ttl_counters *stats, 276399a2dd95SBruce Richardson int clear) 276499a2dd95SBruce Richardson { 276599a2dd95SBruce Richardson struct ttl_data *ttl_data; 276699a2dd95SBruce Richardson 276799a2dd95SBruce Richardson /* Check input arguments */ 276899a2dd95SBruce Richardson if ((action == NULL) || 276999a2dd95SBruce Richardson ((action->cfg.action_mask & 277099a2dd95SBruce Richardson (1LLU << RTE_TABLE_ACTION_TTL)) == 0) || 277199a2dd95SBruce Richardson (data == NULL)) 277299a2dd95SBruce Richardson return -EINVAL; 277399a2dd95SBruce Richardson 277499a2dd95SBruce Richardson ttl_data = action_data_get(data, action, RTE_TABLE_ACTION_TTL); 277599a2dd95SBruce Richardson 277699a2dd95SBruce Richardson /* Read */ 277799a2dd95SBruce Richardson if (stats) 277899a2dd95SBruce Richardson stats->n_packets = TTL_STATS_READ(ttl_data); 277999a2dd95SBruce Richardson 278099a2dd95SBruce Richardson /* Clear */ 278199a2dd95SBruce Richardson if (clear) 278299a2dd95SBruce Richardson TTL_STATS_RESET(ttl_data); 278399a2dd95SBruce Richardson 278499a2dd95SBruce Richardson return 0; 278599a2dd95SBruce Richardson } 278699a2dd95SBruce Richardson 278799a2dd95SBruce Richardson int 278899a2dd95SBruce Richardson rte_table_action_stats_read(struct rte_table_action *action, 278999a2dd95SBruce Richardson void *data, 279099a2dd95SBruce Richardson struct rte_table_action_stats_counters *stats, 279199a2dd95SBruce Richardson int clear) 279299a2dd95SBruce Richardson { 279399a2dd95SBruce Richardson struct stats_data *stats_data; 279499a2dd95SBruce Richardson 279599a2dd95SBruce Richardson /* Check input arguments */ 279699a2dd95SBruce Richardson if ((action == NULL) || 279799a2dd95SBruce Richardson ((action->cfg.action_mask & 279899a2dd95SBruce Richardson (1LLU << RTE_TABLE_ACTION_STATS)) == 0) || 279999a2dd95SBruce Richardson (data == NULL)) 280099a2dd95SBruce Richardson return -EINVAL; 280199a2dd95SBruce Richardson 280299a2dd95SBruce Richardson stats_data = action_data_get(data, action, 280399a2dd95SBruce Richardson RTE_TABLE_ACTION_STATS); 280499a2dd95SBruce Richardson 280599a2dd95SBruce Richardson /* Read */ 280699a2dd95SBruce Richardson if (stats) { 280799a2dd95SBruce Richardson stats->n_packets = stats_data->n_packets; 280899a2dd95SBruce Richardson stats->n_bytes = stats_data->n_bytes; 280999a2dd95SBruce Richardson stats->n_packets_valid = 1; 281099a2dd95SBruce Richardson stats->n_bytes_valid = 1; 281199a2dd95SBruce Richardson } 281299a2dd95SBruce Richardson 281399a2dd95SBruce Richardson /* Clear */ 281499a2dd95SBruce Richardson if (clear) { 281599a2dd95SBruce Richardson stats_data->n_packets = 0; 281699a2dd95SBruce Richardson stats_data->n_bytes = 0; 281799a2dd95SBruce Richardson } 281899a2dd95SBruce Richardson 281999a2dd95SBruce Richardson return 0; 282099a2dd95SBruce Richardson } 282199a2dd95SBruce Richardson 282299a2dd95SBruce Richardson int 282399a2dd95SBruce Richardson rte_table_action_time_read(struct rte_table_action *action, 282499a2dd95SBruce Richardson void *data, 282599a2dd95SBruce Richardson uint64_t *timestamp) 282699a2dd95SBruce Richardson { 282799a2dd95SBruce Richardson struct time_data *time_data; 282899a2dd95SBruce Richardson 282999a2dd95SBruce Richardson /* Check input arguments */ 283099a2dd95SBruce Richardson if ((action == NULL) || 283199a2dd95SBruce Richardson ((action->cfg.action_mask & 283299a2dd95SBruce Richardson (1LLU << RTE_TABLE_ACTION_TIME)) == 0) || 283399a2dd95SBruce Richardson (data == NULL) || 283499a2dd95SBruce Richardson (timestamp == NULL)) 283599a2dd95SBruce Richardson return -EINVAL; 283699a2dd95SBruce Richardson 283799a2dd95SBruce Richardson time_data = action_data_get(data, action, RTE_TABLE_ACTION_TIME); 283899a2dd95SBruce Richardson 283999a2dd95SBruce Richardson /* Read */ 284099a2dd95SBruce Richardson *timestamp = time_data->time; 284199a2dd95SBruce Richardson 284299a2dd95SBruce Richardson return 0; 284399a2dd95SBruce Richardson } 284499a2dd95SBruce Richardson 284599a2dd95SBruce Richardson struct rte_cryptodev_sym_session * 284699a2dd95SBruce Richardson rte_table_action_crypto_sym_session_get(struct rte_table_action *action, 284799a2dd95SBruce Richardson void *data) 284899a2dd95SBruce Richardson { 284999a2dd95SBruce Richardson struct sym_crypto_data *sym_crypto_data; 285099a2dd95SBruce Richardson 285199a2dd95SBruce Richardson /* Check input arguments */ 285299a2dd95SBruce Richardson if ((action == NULL) || 285399a2dd95SBruce Richardson ((action->cfg.action_mask & 285499a2dd95SBruce Richardson (1LLU << RTE_TABLE_ACTION_SYM_CRYPTO)) == 0) || 285599a2dd95SBruce Richardson (data == NULL)) 285699a2dd95SBruce Richardson return NULL; 285799a2dd95SBruce Richardson 285899a2dd95SBruce Richardson sym_crypto_data = action_data_get(data, action, 285999a2dd95SBruce Richardson RTE_TABLE_ACTION_SYM_CRYPTO); 286099a2dd95SBruce Richardson 286199a2dd95SBruce Richardson return sym_crypto_data->session; 286299a2dd95SBruce Richardson } 286399a2dd95SBruce Richardson 286499a2dd95SBruce Richardson static __rte_always_inline uint64_t 286599a2dd95SBruce Richardson pkt_work(struct rte_mbuf *mbuf, 286699a2dd95SBruce Richardson struct rte_pipeline_table_entry *table_entry, 286799a2dd95SBruce Richardson uint64_t time, 286899a2dd95SBruce Richardson struct rte_table_action *action, 286999a2dd95SBruce Richardson struct ap_config *cfg) 287099a2dd95SBruce Richardson { 287199a2dd95SBruce Richardson uint64_t drop_mask = 0; 287299a2dd95SBruce Richardson 287399a2dd95SBruce Richardson uint32_t ip_offset = action->cfg.common.ip_offset; 287499a2dd95SBruce Richardson void *ip = RTE_MBUF_METADATA_UINT32_PTR(mbuf, ip_offset); 287599a2dd95SBruce Richardson 287699a2dd95SBruce Richardson uint32_t dscp; 287799a2dd95SBruce Richardson uint16_t total_length; 287899a2dd95SBruce Richardson 287999a2dd95SBruce Richardson if (cfg->common.ip_version) { 288099a2dd95SBruce Richardson struct rte_ipv4_hdr *hdr = ip; 288199a2dd95SBruce Richardson 288299a2dd95SBruce Richardson dscp = hdr->type_of_service >> 2; 288399a2dd95SBruce Richardson total_length = rte_ntohs(hdr->total_length); 288499a2dd95SBruce Richardson } else { 288599a2dd95SBruce Richardson struct rte_ipv6_hdr *hdr = ip; 288699a2dd95SBruce Richardson 288799a2dd95SBruce Richardson dscp = (rte_ntohl(hdr->vtc_flow) & 0x0F600000) >> 18; 288899a2dd95SBruce Richardson total_length = rte_ntohs(hdr->payload_len) + 288999a2dd95SBruce Richardson sizeof(struct rte_ipv6_hdr); 289099a2dd95SBruce Richardson } 289199a2dd95SBruce Richardson 289299a2dd95SBruce Richardson if (cfg->action_mask & (1LLU << RTE_TABLE_ACTION_LB)) { 289399a2dd95SBruce Richardson void *data = 289499a2dd95SBruce Richardson action_data_get(table_entry, action, RTE_TABLE_ACTION_LB); 289599a2dd95SBruce Richardson 289699a2dd95SBruce Richardson pkt_work_lb(mbuf, 289799a2dd95SBruce Richardson data, 289899a2dd95SBruce Richardson &cfg->lb); 289999a2dd95SBruce Richardson } 290099a2dd95SBruce Richardson if (cfg->action_mask & (1LLU << RTE_TABLE_ACTION_MTR)) { 290199a2dd95SBruce Richardson void *data = 290299a2dd95SBruce Richardson action_data_get(table_entry, action, RTE_TABLE_ACTION_MTR); 290399a2dd95SBruce Richardson 290499a2dd95SBruce Richardson drop_mask |= pkt_work_mtr(mbuf, 290599a2dd95SBruce Richardson data, 290699a2dd95SBruce Richardson &action->dscp_table, 290799a2dd95SBruce Richardson action->mp, 290899a2dd95SBruce Richardson time, 290999a2dd95SBruce Richardson dscp, 291099a2dd95SBruce Richardson total_length); 291199a2dd95SBruce Richardson } 291299a2dd95SBruce Richardson 291399a2dd95SBruce Richardson if (cfg->action_mask & (1LLU << RTE_TABLE_ACTION_TM)) { 291499a2dd95SBruce Richardson void *data = 291599a2dd95SBruce Richardson action_data_get(table_entry, action, RTE_TABLE_ACTION_TM); 291699a2dd95SBruce Richardson 291799a2dd95SBruce Richardson pkt_work_tm(mbuf, 291899a2dd95SBruce Richardson data, 291999a2dd95SBruce Richardson &action->dscp_table, 292099a2dd95SBruce Richardson dscp); 292199a2dd95SBruce Richardson } 292299a2dd95SBruce Richardson 292399a2dd95SBruce Richardson if (cfg->action_mask & (1LLU << RTE_TABLE_ACTION_DECAP)) { 292499a2dd95SBruce Richardson void *data = action_data_get(table_entry, 292599a2dd95SBruce Richardson action, 292699a2dd95SBruce Richardson RTE_TABLE_ACTION_DECAP); 292799a2dd95SBruce Richardson 292899a2dd95SBruce Richardson pkt_work_decap(mbuf, data); 292999a2dd95SBruce Richardson } 293099a2dd95SBruce Richardson 293199a2dd95SBruce Richardson if (cfg->action_mask & (1LLU << RTE_TABLE_ACTION_ENCAP)) { 293299a2dd95SBruce Richardson void *data = 293399a2dd95SBruce Richardson action_data_get(table_entry, action, RTE_TABLE_ACTION_ENCAP); 293499a2dd95SBruce Richardson 293599a2dd95SBruce Richardson pkt_work_encap(mbuf, 293699a2dd95SBruce Richardson data, 293799a2dd95SBruce Richardson &cfg->encap, 293899a2dd95SBruce Richardson ip, 293999a2dd95SBruce Richardson total_length, 294099a2dd95SBruce Richardson ip_offset); 294199a2dd95SBruce Richardson } 294299a2dd95SBruce Richardson 294399a2dd95SBruce Richardson if (cfg->action_mask & (1LLU << RTE_TABLE_ACTION_NAT)) { 294499a2dd95SBruce Richardson void *data = 294599a2dd95SBruce Richardson action_data_get(table_entry, action, RTE_TABLE_ACTION_NAT); 294699a2dd95SBruce Richardson 294799a2dd95SBruce Richardson if (cfg->common.ip_version) 294899a2dd95SBruce Richardson pkt_ipv4_work_nat(ip, data, &cfg->nat); 294999a2dd95SBruce Richardson else 295099a2dd95SBruce Richardson pkt_ipv6_work_nat(ip, data, &cfg->nat); 295199a2dd95SBruce Richardson } 295299a2dd95SBruce Richardson 295399a2dd95SBruce Richardson if (cfg->action_mask & (1LLU << RTE_TABLE_ACTION_TTL)) { 295499a2dd95SBruce Richardson void *data = 295599a2dd95SBruce Richardson action_data_get(table_entry, action, RTE_TABLE_ACTION_TTL); 295699a2dd95SBruce Richardson 295799a2dd95SBruce Richardson if (cfg->common.ip_version) 295899a2dd95SBruce Richardson drop_mask |= pkt_ipv4_work_ttl(ip, data); 295999a2dd95SBruce Richardson else 296099a2dd95SBruce Richardson drop_mask |= pkt_ipv6_work_ttl(ip, data); 296199a2dd95SBruce Richardson } 296299a2dd95SBruce Richardson 296399a2dd95SBruce Richardson if (cfg->action_mask & (1LLU << RTE_TABLE_ACTION_STATS)) { 296499a2dd95SBruce Richardson void *data = 296599a2dd95SBruce Richardson action_data_get(table_entry, action, RTE_TABLE_ACTION_STATS); 296699a2dd95SBruce Richardson 296799a2dd95SBruce Richardson pkt_work_stats(data, total_length); 296899a2dd95SBruce Richardson } 296999a2dd95SBruce Richardson 297099a2dd95SBruce Richardson if (cfg->action_mask & (1LLU << RTE_TABLE_ACTION_TIME)) { 297199a2dd95SBruce Richardson void *data = 297299a2dd95SBruce Richardson action_data_get(table_entry, action, RTE_TABLE_ACTION_TIME); 297399a2dd95SBruce Richardson 297499a2dd95SBruce Richardson pkt_work_time(data, time); 297599a2dd95SBruce Richardson } 297699a2dd95SBruce Richardson 297799a2dd95SBruce Richardson if (cfg->action_mask & (1LLU << RTE_TABLE_ACTION_SYM_CRYPTO)) { 297899a2dd95SBruce Richardson void *data = action_data_get(table_entry, action, 297999a2dd95SBruce Richardson RTE_TABLE_ACTION_SYM_CRYPTO); 298099a2dd95SBruce Richardson 298199a2dd95SBruce Richardson drop_mask |= pkt_work_sym_crypto(mbuf, data, &cfg->sym_crypto, 298299a2dd95SBruce Richardson ip_offset); 298399a2dd95SBruce Richardson } 298499a2dd95SBruce Richardson 298599a2dd95SBruce Richardson if (cfg->action_mask & (1LLU << RTE_TABLE_ACTION_TAG)) { 298699a2dd95SBruce Richardson void *data = action_data_get(table_entry, 298799a2dd95SBruce Richardson action, 298899a2dd95SBruce Richardson RTE_TABLE_ACTION_TAG); 298999a2dd95SBruce Richardson 299099a2dd95SBruce Richardson pkt_work_tag(mbuf, data); 299199a2dd95SBruce Richardson } 299299a2dd95SBruce Richardson 299399a2dd95SBruce Richardson return drop_mask; 299499a2dd95SBruce Richardson } 299599a2dd95SBruce Richardson 299699a2dd95SBruce Richardson static __rte_always_inline uint64_t 299799a2dd95SBruce Richardson pkt4_work(struct rte_mbuf **mbufs, 299899a2dd95SBruce Richardson struct rte_pipeline_table_entry **table_entries, 299999a2dd95SBruce Richardson uint64_t time, 300099a2dd95SBruce Richardson struct rte_table_action *action, 300199a2dd95SBruce Richardson struct ap_config *cfg) 300299a2dd95SBruce Richardson { 300399a2dd95SBruce Richardson uint64_t drop_mask0 = 0; 300499a2dd95SBruce Richardson uint64_t drop_mask1 = 0; 300599a2dd95SBruce Richardson uint64_t drop_mask2 = 0; 300699a2dd95SBruce Richardson uint64_t drop_mask3 = 0; 300799a2dd95SBruce Richardson 300899a2dd95SBruce Richardson struct rte_mbuf *mbuf0 = mbufs[0]; 300999a2dd95SBruce Richardson struct rte_mbuf *mbuf1 = mbufs[1]; 301099a2dd95SBruce Richardson struct rte_mbuf *mbuf2 = mbufs[2]; 301199a2dd95SBruce Richardson struct rte_mbuf *mbuf3 = mbufs[3]; 301299a2dd95SBruce Richardson 301399a2dd95SBruce Richardson struct rte_pipeline_table_entry *table_entry0 = table_entries[0]; 301499a2dd95SBruce Richardson struct rte_pipeline_table_entry *table_entry1 = table_entries[1]; 301599a2dd95SBruce Richardson struct rte_pipeline_table_entry *table_entry2 = table_entries[2]; 301699a2dd95SBruce Richardson struct rte_pipeline_table_entry *table_entry3 = table_entries[3]; 301799a2dd95SBruce Richardson 301899a2dd95SBruce Richardson uint32_t ip_offset = action->cfg.common.ip_offset; 301999a2dd95SBruce Richardson void *ip0 = RTE_MBUF_METADATA_UINT32_PTR(mbuf0, ip_offset); 302099a2dd95SBruce Richardson void *ip1 = RTE_MBUF_METADATA_UINT32_PTR(mbuf1, ip_offset); 302199a2dd95SBruce Richardson void *ip2 = RTE_MBUF_METADATA_UINT32_PTR(mbuf2, ip_offset); 302299a2dd95SBruce Richardson void *ip3 = RTE_MBUF_METADATA_UINT32_PTR(mbuf3, ip_offset); 302399a2dd95SBruce Richardson 302499a2dd95SBruce Richardson uint32_t dscp0, dscp1, dscp2, dscp3; 302599a2dd95SBruce Richardson uint16_t total_length0, total_length1, total_length2, total_length3; 302699a2dd95SBruce Richardson 302799a2dd95SBruce Richardson if (cfg->common.ip_version) { 302899a2dd95SBruce Richardson struct rte_ipv4_hdr *hdr0 = ip0; 302999a2dd95SBruce Richardson struct rte_ipv4_hdr *hdr1 = ip1; 303099a2dd95SBruce Richardson struct rte_ipv4_hdr *hdr2 = ip2; 303199a2dd95SBruce Richardson struct rte_ipv4_hdr *hdr3 = ip3; 303299a2dd95SBruce Richardson 303399a2dd95SBruce Richardson dscp0 = hdr0->type_of_service >> 2; 303499a2dd95SBruce Richardson dscp1 = hdr1->type_of_service >> 2; 303599a2dd95SBruce Richardson dscp2 = hdr2->type_of_service >> 2; 303699a2dd95SBruce Richardson dscp3 = hdr3->type_of_service >> 2; 303799a2dd95SBruce Richardson 303899a2dd95SBruce Richardson total_length0 = rte_ntohs(hdr0->total_length); 303999a2dd95SBruce Richardson total_length1 = rte_ntohs(hdr1->total_length); 304099a2dd95SBruce Richardson total_length2 = rte_ntohs(hdr2->total_length); 304199a2dd95SBruce Richardson total_length3 = rte_ntohs(hdr3->total_length); 304299a2dd95SBruce Richardson } else { 304399a2dd95SBruce Richardson struct rte_ipv6_hdr *hdr0 = ip0; 304499a2dd95SBruce Richardson struct rte_ipv6_hdr *hdr1 = ip1; 304599a2dd95SBruce Richardson struct rte_ipv6_hdr *hdr2 = ip2; 304699a2dd95SBruce Richardson struct rte_ipv6_hdr *hdr3 = ip3; 304799a2dd95SBruce Richardson 304899a2dd95SBruce Richardson dscp0 = (rte_ntohl(hdr0->vtc_flow) & 0x0F600000) >> 18; 304999a2dd95SBruce Richardson dscp1 = (rte_ntohl(hdr1->vtc_flow) & 0x0F600000) >> 18; 305099a2dd95SBruce Richardson dscp2 = (rte_ntohl(hdr2->vtc_flow) & 0x0F600000) >> 18; 305199a2dd95SBruce Richardson dscp3 = (rte_ntohl(hdr3->vtc_flow) & 0x0F600000) >> 18; 305299a2dd95SBruce Richardson 305399a2dd95SBruce Richardson total_length0 = rte_ntohs(hdr0->payload_len) + 305499a2dd95SBruce Richardson sizeof(struct rte_ipv6_hdr); 305599a2dd95SBruce Richardson total_length1 = rte_ntohs(hdr1->payload_len) + 305699a2dd95SBruce Richardson sizeof(struct rte_ipv6_hdr); 305799a2dd95SBruce Richardson total_length2 = rte_ntohs(hdr2->payload_len) + 305899a2dd95SBruce Richardson sizeof(struct rte_ipv6_hdr); 305999a2dd95SBruce Richardson total_length3 = rte_ntohs(hdr3->payload_len) + 306099a2dd95SBruce Richardson sizeof(struct rte_ipv6_hdr); 306199a2dd95SBruce Richardson } 306299a2dd95SBruce Richardson 306399a2dd95SBruce Richardson if (cfg->action_mask & (1LLU << RTE_TABLE_ACTION_LB)) { 306499a2dd95SBruce Richardson void *data0 = 306599a2dd95SBruce Richardson action_data_get(table_entry0, action, RTE_TABLE_ACTION_LB); 306699a2dd95SBruce Richardson void *data1 = 306799a2dd95SBruce Richardson action_data_get(table_entry1, action, RTE_TABLE_ACTION_LB); 306899a2dd95SBruce Richardson void *data2 = 306999a2dd95SBruce Richardson action_data_get(table_entry2, action, RTE_TABLE_ACTION_LB); 307099a2dd95SBruce Richardson void *data3 = 307199a2dd95SBruce Richardson action_data_get(table_entry3, action, RTE_TABLE_ACTION_LB); 307299a2dd95SBruce Richardson 307399a2dd95SBruce Richardson pkt_work_lb(mbuf0, 307499a2dd95SBruce Richardson data0, 307599a2dd95SBruce Richardson &cfg->lb); 307699a2dd95SBruce Richardson 307799a2dd95SBruce Richardson pkt_work_lb(mbuf1, 307899a2dd95SBruce Richardson data1, 307999a2dd95SBruce Richardson &cfg->lb); 308099a2dd95SBruce Richardson 308199a2dd95SBruce Richardson pkt_work_lb(mbuf2, 308299a2dd95SBruce Richardson data2, 308399a2dd95SBruce Richardson &cfg->lb); 308499a2dd95SBruce Richardson 308599a2dd95SBruce Richardson pkt_work_lb(mbuf3, 308699a2dd95SBruce Richardson data3, 308799a2dd95SBruce Richardson &cfg->lb); 308899a2dd95SBruce Richardson } 308999a2dd95SBruce Richardson 309099a2dd95SBruce Richardson if (cfg->action_mask & (1LLU << RTE_TABLE_ACTION_MTR)) { 309199a2dd95SBruce Richardson void *data0 = 309299a2dd95SBruce Richardson action_data_get(table_entry0, action, RTE_TABLE_ACTION_MTR); 309399a2dd95SBruce Richardson void *data1 = 309499a2dd95SBruce Richardson action_data_get(table_entry1, action, RTE_TABLE_ACTION_MTR); 309599a2dd95SBruce Richardson void *data2 = 309699a2dd95SBruce Richardson action_data_get(table_entry2, action, RTE_TABLE_ACTION_MTR); 309799a2dd95SBruce Richardson void *data3 = 309899a2dd95SBruce Richardson action_data_get(table_entry3, action, RTE_TABLE_ACTION_MTR); 309999a2dd95SBruce Richardson 310099a2dd95SBruce Richardson drop_mask0 |= pkt_work_mtr(mbuf0, 310199a2dd95SBruce Richardson data0, 310299a2dd95SBruce Richardson &action->dscp_table, 310399a2dd95SBruce Richardson action->mp, 310499a2dd95SBruce Richardson time, 310599a2dd95SBruce Richardson dscp0, 310699a2dd95SBruce Richardson total_length0); 310799a2dd95SBruce Richardson 310899a2dd95SBruce Richardson drop_mask1 |= pkt_work_mtr(mbuf1, 310999a2dd95SBruce Richardson data1, 311099a2dd95SBruce Richardson &action->dscp_table, 311199a2dd95SBruce Richardson action->mp, 311299a2dd95SBruce Richardson time, 311399a2dd95SBruce Richardson dscp1, 311499a2dd95SBruce Richardson total_length1); 311599a2dd95SBruce Richardson 311699a2dd95SBruce Richardson drop_mask2 |= pkt_work_mtr(mbuf2, 311799a2dd95SBruce Richardson data2, 311899a2dd95SBruce Richardson &action->dscp_table, 311999a2dd95SBruce Richardson action->mp, 312099a2dd95SBruce Richardson time, 312199a2dd95SBruce Richardson dscp2, 312299a2dd95SBruce Richardson total_length2); 312399a2dd95SBruce Richardson 312499a2dd95SBruce Richardson drop_mask3 |= pkt_work_mtr(mbuf3, 312599a2dd95SBruce Richardson data3, 312699a2dd95SBruce Richardson &action->dscp_table, 312799a2dd95SBruce Richardson action->mp, 312899a2dd95SBruce Richardson time, 312999a2dd95SBruce Richardson dscp3, 313099a2dd95SBruce Richardson total_length3); 313199a2dd95SBruce Richardson } 313299a2dd95SBruce Richardson 313399a2dd95SBruce Richardson if (cfg->action_mask & (1LLU << RTE_TABLE_ACTION_TM)) { 313499a2dd95SBruce Richardson void *data0 = 313599a2dd95SBruce Richardson action_data_get(table_entry0, action, RTE_TABLE_ACTION_TM); 313699a2dd95SBruce Richardson void *data1 = 313799a2dd95SBruce Richardson action_data_get(table_entry1, action, RTE_TABLE_ACTION_TM); 313899a2dd95SBruce Richardson void *data2 = 313999a2dd95SBruce Richardson action_data_get(table_entry2, action, RTE_TABLE_ACTION_TM); 314099a2dd95SBruce Richardson void *data3 = 314199a2dd95SBruce Richardson action_data_get(table_entry3, action, RTE_TABLE_ACTION_TM); 314299a2dd95SBruce Richardson 314399a2dd95SBruce Richardson pkt_work_tm(mbuf0, 314499a2dd95SBruce Richardson data0, 314599a2dd95SBruce Richardson &action->dscp_table, 314699a2dd95SBruce Richardson dscp0); 314799a2dd95SBruce Richardson 314899a2dd95SBruce Richardson pkt_work_tm(mbuf1, 314999a2dd95SBruce Richardson data1, 315099a2dd95SBruce Richardson &action->dscp_table, 315199a2dd95SBruce Richardson dscp1); 315299a2dd95SBruce Richardson 315399a2dd95SBruce Richardson pkt_work_tm(mbuf2, 315499a2dd95SBruce Richardson data2, 315599a2dd95SBruce Richardson &action->dscp_table, 315699a2dd95SBruce Richardson dscp2); 315799a2dd95SBruce Richardson 315899a2dd95SBruce Richardson pkt_work_tm(mbuf3, 315999a2dd95SBruce Richardson data3, 316099a2dd95SBruce Richardson &action->dscp_table, 316199a2dd95SBruce Richardson dscp3); 316299a2dd95SBruce Richardson } 316399a2dd95SBruce Richardson 316499a2dd95SBruce Richardson if (cfg->action_mask & (1LLU << RTE_TABLE_ACTION_DECAP)) { 316599a2dd95SBruce Richardson void *data0 = action_data_get(table_entry0, 316699a2dd95SBruce Richardson action, 316799a2dd95SBruce Richardson RTE_TABLE_ACTION_DECAP); 316899a2dd95SBruce Richardson void *data1 = action_data_get(table_entry1, 316999a2dd95SBruce Richardson action, 317099a2dd95SBruce Richardson RTE_TABLE_ACTION_DECAP); 317199a2dd95SBruce Richardson void *data2 = action_data_get(table_entry2, 317299a2dd95SBruce Richardson action, 317399a2dd95SBruce Richardson RTE_TABLE_ACTION_DECAP); 317499a2dd95SBruce Richardson void *data3 = action_data_get(table_entry3, 317599a2dd95SBruce Richardson action, 317699a2dd95SBruce Richardson RTE_TABLE_ACTION_DECAP); 317799a2dd95SBruce Richardson 317899a2dd95SBruce Richardson pkt4_work_decap(mbuf0, mbuf1, mbuf2, mbuf3, 317999a2dd95SBruce Richardson data0, data1, data2, data3); 318099a2dd95SBruce Richardson } 318199a2dd95SBruce Richardson 318299a2dd95SBruce Richardson if (cfg->action_mask & (1LLU << RTE_TABLE_ACTION_ENCAP)) { 318399a2dd95SBruce Richardson void *data0 = 318499a2dd95SBruce Richardson action_data_get(table_entry0, action, RTE_TABLE_ACTION_ENCAP); 318599a2dd95SBruce Richardson void *data1 = 318699a2dd95SBruce Richardson action_data_get(table_entry1, action, RTE_TABLE_ACTION_ENCAP); 318799a2dd95SBruce Richardson void *data2 = 318899a2dd95SBruce Richardson action_data_get(table_entry2, action, RTE_TABLE_ACTION_ENCAP); 318999a2dd95SBruce Richardson void *data3 = 319099a2dd95SBruce Richardson action_data_get(table_entry3, action, RTE_TABLE_ACTION_ENCAP); 319199a2dd95SBruce Richardson 319299a2dd95SBruce Richardson pkt_work_encap(mbuf0, 319399a2dd95SBruce Richardson data0, 319499a2dd95SBruce Richardson &cfg->encap, 319599a2dd95SBruce Richardson ip0, 319699a2dd95SBruce Richardson total_length0, 319799a2dd95SBruce Richardson ip_offset); 319899a2dd95SBruce Richardson 319999a2dd95SBruce Richardson pkt_work_encap(mbuf1, 320099a2dd95SBruce Richardson data1, 320199a2dd95SBruce Richardson &cfg->encap, 320299a2dd95SBruce Richardson ip1, 320399a2dd95SBruce Richardson total_length1, 320499a2dd95SBruce Richardson ip_offset); 320599a2dd95SBruce Richardson 320699a2dd95SBruce Richardson pkt_work_encap(mbuf2, 320799a2dd95SBruce Richardson data2, 320899a2dd95SBruce Richardson &cfg->encap, 320999a2dd95SBruce Richardson ip2, 321099a2dd95SBruce Richardson total_length2, 321199a2dd95SBruce Richardson ip_offset); 321299a2dd95SBruce Richardson 321399a2dd95SBruce Richardson pkt_work_encap(mbuf3, 321499a2dd95SBruce Richardson data3, 321599a2dd95SBruce Richardson &cfg->encap, 321699a2dd95SBruce Richardson ip3, 321799a2dd95SBruce Richardson total_length3, 321899a2dd95SBruce Richardson ip_offset); 321999a2dd95SBruce Richardson } 322099a2dd95SBruce Richardson 322199a2dd95SBruce Richardson if (cfg->action_mask & (1LLU << RTE_TABLE_ACTION_NAT)) { 322299a2dd95SBruce Richardson void *data0 = 322399a2dd95SBruce Richardson action_data_get(table_entry0, action, RTE_TABLE_ACTION_NAT); 322499a2dd95SBruce Richardson void *data1 = 322599a2dd95SBruce Richardson action_data_get(table_entry1, action, RTE_TABLE_ACTION_NAT); 322699a2dd95SBruce Richardson void *data2 = 322799a2dd95SBruce Richardson action_data_get(table_entry2, action, RTE_TABLE_ACTION_NAT); 322899a2dd95SBruce Richardson void *data3 = 322999a2dd95SBruce Richardson action_data_get(table_entry3, action, RTE_TABLE_ACTION_NAT); 323099a2dd95SBruce Richardson 323199a2dd95SBruce Richardson if (cfg->common.ip_version) { 323299a2dd95SBruce Richardson pkt_ipv4_work_nat(ip0, data0, &cfg->nat); 323399a2dd95SBruce Richardson pkt_ipv4_work_nat(ip1, data1, &cfg->nat); 323499a2dd95SBruce Richardson pkt_ipv4_work_nat(ip2, data2, &cfg->nat); 323599a2dd95SBruce Richardson pkt_ipv4_work_nat(ip3, data3, &cfg->nat); 323699a2dd95SBruce Richardson } else { 323799a2dd95SBruce Richardson pkt_ipv6_work_nat(ip0, data0, &cfg->nat); 323899a2dd95SBruce Richardson pkt_ipv6_work_nat(ip1, data1, &cfg->nat); 323999a2dd95SBruce Richardson pkt_ipv6_work_nat(ip2, data2, &cfg->nat); 324099a2dd95SBruce Richardson pkt_ipv6_work_nat(ip3, data3, &cfg->nat); 324199a2dd95SBruce Richardson } 324299a2dd95SBruce Richardson } 324399a2dd95SBruce Richardson 324499a2dd95SBruce Richardson if (cfg->action_mask & (1LLU << RTE_TABLE_ACTION_TTL)) { 324599a2dd95SBruce Richardson void *data0 = 324699a2dd95SBruce Richardson action_data_get(table_entry0, action, RTE_TABLE_ACTION_TTL); 324799a2dd95SBruce Richardson void *data1 = 324899a2dd95SBruce Richardson action_data_get(table_entry1, action, RTE_TABLE_ACTION_TTL); 324999a2dd95SBruce Richardson void *data2 = 325099a2dd95SBruce Richardson action_data_get(table_entry2, action, RTE_TABLE_ACTION_TTL); 325199a2dd95SBruce Richardson void *data3 = 325299a2dd95SBruce Richardson action_data_get(table_entry3, action, RTE_TABLE_ACTION_TTL); 325399a2dd95SBruce Richardson 325499a2dd95SBruce Richardson if (cfg->common.ip_version) { 325599a2dd95SBruce Richardson drop_mask0 |= pkt_ipv4_work_ttl(ip0, data0); 325699a2dd95SBruce Richardson drop_mask1 |= pkt_ipv4_work_ttl(ip1, data1); 325799a2dd95SBruce Richardson drop_mask2 |= pkt_ipv4_work_ttl(ip2, data2); 325899a2dd95SBruce Richardson drop_mask3 |= pkt_ipv4_work_ttl(ip3, data3); 325999a2dd95SBruce Richardson } else { 326099a2dd95SBruce Richardson drop_mask0 |= pkt_ipv6_work_ttl(ip0, data0); 326199a2dd95SBruce Richardson drop_mask1 |= pkt_ipv6_work_ttl(ip1, data1); 326299a2dd95SBruce Richardson drop_mask2 |= pkt_ipv6_work_ttl(ip2, data2); 326399a2dd95SBruce Richardson drop_mask3 |= pkt_ipv6_work_ttl(ip3, data3); 326499a2dd95SBruce Richardson } 326599a2dd95SBruce Richardson } 326699a2dd95SBruce Richardson 326799a2dd95SBruce Richardson if (cfg->action_mask & (1LLU << RTE_TABLE_ACTION_STATS)) { 326899a2dd95SBruce Richardson void *data0 = 326999a2dd95SBruce Richardson action_data_get(table_entry0, action, RTE_TABLE_ACTION_STATS); 327099a2dd95SBruce Richardson void *data1 = 327199a2dd95SBruce Richardson action_data_get(table_entry1, action, RTE_TABLE_ACTION_STATS); 327299a2dd95SBruce Richardson void *data2 = 327399a2dd95SBruce Richardson action_data_get(table_entry2, action, RTE_TABLE_ACTION_STATS); 327499a2dd95SBruce Richardson void *data3 = 327599a2dd95SBruce Richardson action_data_get(table_entry3, action, RTE_TABLE_ACTION_STATS); 327699a2dd95SBruce Richardson 327799a2dd95SBruce Richardson pkt_work_stats(data0, total_length0); 327899a2dd95SBruce Richardson pkt_work_stats(data1, total_length1); 327999a2dd95SBruce Richardson pkt_work_stats(data2, total_length2); 328099a2dd95SBruce Richardson pkt_work_stats(data3, total_length3); 328199a2dd95SBruce Richardson } 328299a2dd95SBruce Richardson 328399a2dd95SBruce Richardson if (cfg->action_mask & (1LLU << RTE_TABLE_ACTION_TIME)) { 328499a2dd95SBruce Richardson void *data0 = 328599a2dd95SBruce Richardson action_data_get(table_entry0, action, RTE_TABLE_ACTION_TIME); 328699a2dd95SBruce Richardson void *data1 = 328799a2dd95SBruce Richardson action_data_get(table_entry1, action, RTE_TABLE_ACTION_TIME); 328899a2dd95SBruce Richardson void *data2 = 328999a2dd95SBruce Richardson action_data_get(table_entry2, action, RTE_TABLE_ACTION_TIME); 329099a2dd95SBruce Richardson void *data3 = 329199a2dd95SBruce Richardson action_data_get(table_entry3, action, RTE_TABLE_ACTION_TIME); 329299a2dd95SBruce Richardson 329399a2dd95SBruce Richardson pkt_work_time(data0, time); 329499a2dd95SBruce Richardson pkt_work_time(data1, time); 329599a2dd95SBruce Richardson pkt_work_time(data2, time); 329699a2dd95SBruce Richardson pkt_work_time(data3, time); 329799a2dd95SBruce Richardson } 329899a2dd95SBruce Richardson 329999a2dd95SBruce Richardson if (cfg->action_mask & (1LLU << RTE_TABLE_ACTION_SYM_CRYPTO)) { 330099a2dd95SBruce Richardson void *data0 = action_data_get(table_entry0, action, 330199a2dd95SBruce Richardson RTE_TABLE_ACTION_SYM_CRYPTO); 330299a2dd95SBruce Richardson void *data1 = action_data_get(table_entry1, action, 330399a2dd95SBruce Richardson RTE_TABLE_ACTION_SYM_CRYPTO); 330499a2dd95SBruce Richardson void *data2 = action_data_get(table_entry2, action, 330599a2dd95SBruce Richardson RTE_TABLE_ACTION_SYM_CRYPTO); 330699a2dd95SBruce Richardson void *data3 = action_data_get(table_entry3, action, 330799a2dd95SBruce Richardson RTE_TABLE_ACTION_SYM_CRYPTO); 330899a2dd95SBruce Richardson 330999a2dd95SBruce Richardson drop_mask0 |= pkt_work_sym_crypto(mbuf0, data0, &cfg->sym_crypto, 331099a2dd95SBruce Richardson ip_offset); 331199a2dd95SBruce Richardson drop_mask1 |= pkt_work_sym_crypto(mbuf1, data1, &cfg->sym_crypto, 331299a2dd95SBruce Richardson ip_offset); 331399a2dd95SBruce Richardson drop_mask2 |= pkt_work_sym_crypto(mbuf2, data2, &cfg->sym_crypto, 331499a2dd95SBruce Richardson ip_offset); 331599a2dd95SBruce Richardson drop_mask3 |= pkt_work_sym_crypto(mbuf3, data3, &cfg->sym_crypto, 331699a2dd95SBruce Richardson ip_offset); 331799a2dd95SBruce Richardson } 331899a2dd95SBruce Richardson 331999a2dd95SBruce Richardson if (cfg->action_mask & (1LLU << RTE_TABLE_ACTION_TAG)) { 332099a2dd95SBruce Richardson void *data0 = action_data_get(table_entry0, 332199a2dd95SBruce Richardson action, 332299a2dd95SBruce Richardson RTE_TABLE_ACTION_TAG); 332399a2dd95SBruce Richardson void *data1 = action_data_get(table_entry1, 332499a2dd95SBruce Richardson action, 332599a2dd95SBruce Richardson RTE_TABLE_ACTION_TAG); 332699a2dd95SBruce Richardson void *data2 = action_data_get(table_entry2, 332799a2dd95SBruce Richardson action, 332899a2dd95SBruce Richardson RTE_TABLE_ACTION_TAG); 332999a2dd95SBruce Richardson void *data3 = action_data_get(table_entry3, 333099a2dd95SBruce Richardson action, 333199a2dd95SBruce Richardson RTE_TABLE_ACTION_TAG); 333299a2dd95SBruce Richardson 333399a2dd95SBruce Richardson pkt4_work_tag(mbuf0, mbuf1, mbuf2, mbuf3, 333499a2dd95SBruce Richardson data0, data1, data2, data3); 333599a2dd95SBruce Richardson } 333699a2dd95SBruce Richardson 333799a2dd95SBruce Richardson return drop_mask0 | 333899a2dd95SBruce Richardson (drop_mask1 << 1) | 333999a2dd95SBruce Richardson (drop_mask2 << 2) | 334099a2dd95SBruce Richardson (drop_mask3 << 3); 334199a2dd95SBruce Richardson } 334299a2dd95SBruce Richardson 334399a2dd95SBruce Richardson static __rte_always_inline int 334499a2dd95SBruce Richardson ah(struct rte_pipeline *p, 334599a2dd95SBruce Richardson struct rte_mbuf **pkts, 334699a2dd95SBruce Richardson uint64_t pkts_mask, 334799a2dd95SBruce Richardson struct rte_pipeline_table_entry **entries, 334899a2dd95SBruce Richardson struct rte_table_action *action, 334999a2dd95SBruce Richardson struct ap_config *cfg) 335099a2dd95SBruce Richardson { 335199a2dd95SBruce Richardson uint64_t pkts_drop_mask = 0; 335299a2dd95SBruce Richardson uint64_t time = 0; 335399a2dd95SBruce Richardson 335499a2dd95SBruce Richardson if (cfg->action_mask & ((1LLU << RTE_TABLE_ACTION_MTR) | 335599a2dd95SBruce Richardson (1LLU << RTE_TABLE_ACTION_TIME))) 335699a2dd95SBruce Richardson time = rte_rdtsc(); 335799a2dd95SBruce Richardson 335899a2dd95SBruce Richardson if ((pkts_mask & (pkts_mask + 1)) == 0) { 33593d4e27fdSDavid Marchand uint64_t n_pkts = rte_popcount64(pkts_mask); 336099a2dd95SBruce Richardson uint32_t i; 336199a2dd95SBruce Richardson 336299a2dd95SBruce Richardson for (i = 0; i < (n_pkts & (~0x3LLU)); i += 4) { 336399a2dd95SBruce Richardson uint64_t drop_mask; 336499a2dd95SBruce Richardson 336599a2dd95SBruce Richardson drop_mask = pkt4_work(&pkts[i], 336699a2dd95SBruce Richardson &entries[i], 336799a2dd95SBruce Richardson time, 336899a2dd95SBruce Richardson action, 336999a2dd95SBruce Richardson cfg); 337099a2dd95SBruce Richardson 337199a2dd95SBruce Richardson pkts_drop_mask |= drop_mask << i; 337299a2dd95SBruce Richardson } 337399a2dd95SBruce Richardson 337499a2dd95SBruce Richardson for ( ; i < n_pkts; i++) { 337599a2dd95SBruce Richardson uint64_t drop_mask; 337699a2dd95SBruce Richardson 337799a2dd95SBruce Richardson drop_mask = pkt_work(pkts[i], 337899a2dd95SBruce Richardson entries[i], 337999a2dd95SBruce Richardson time, 338099a2dd95SBruce Richardson action, 338199a2dd95SBruce Richardson cfg); 338299a2dd95SBruce Richardson 338399a2dd95SBruce Richardson pkts_drop_mask |= drop_mask << i; 338499a2dd95SBruce Richardson } 338599a2dd95SBruce Richardson } else 338699a2dd95SBruce Richardson for ( ; pkts_mask; ) { 33873d4e27fdSDavid Marchand uint32_t pos = rte_ctz64(pkts_mask); 338899a2dd95SBruce Richardson uint64_t pkt_mask = 1LLU << pos; 338999a2dd95SBruce Richardson uint64_t drop_mask; 339099a2dd95SBruce Richardson 339199a2dd95SBruce Richardson drop_mask = pkt_work(pkts[pos], 339299a2dd95SBruce Richardson entries[pos], 339399a2dd95SBruce Richardson time, 339499a2dd95SBruce Richardson action, 339599a2dd95SBruce Richardson cfg); 339699a2dd95SBruce Richardson 339799a2dd95SBruce Richardson pkts_mask &= ~pkt_mask; 339899a2dd95SBruce Richardson pkts_drop_mask |= drop_mask << pos; 339999a2dd95SBruce Richardson } 340099a2dd95SBruce Richardson 340199a2dd95SBruce Richardson rte_pipeline_ah_packet_drop(p, pkts_drop_mask); 340299a2dd95SBruce Richardson 340399a2dd95SBruce Richardson return 0; 340499a2dd95SBruce Richardson } 340599a2dd95SBruce Richardson 340699a2dd95SBruce Richardson static int 340799a2dd95SBruce Richardson ah_default(struct rte_pipeline *p, 340899a2dd95SBruce Richardson struct rte_mbuf **pkts, 340999a2dd95SBruce Richardson uint64_t pkts_mask, 341099a2dd95SBruce Richardson struct rte_pipeline_table_entry **entries, 341199a2dd95SBruce Richardson void *arg) 341299a2dd95SBruce Richardson { 341399a2dd95SBruce Richardson struct rte_table_action *action = arg; 341499a2dd95SBruce Richardson 341599a2dd95SBruce Richardson return ah(p, 341699a2dd95SBruce Richardson pkts, 341799a2dd95SBruce Richardson pkts_mask, 341899a2dd95SBruce Richardson entries, 341999a2dd95SBruce Richardson action, 342099a2dd95SBruce Richardson &action->cfg); 342199a2dd95SBruce Richardson } 342299a2dd95SBruce Richardson 342399a2dd95SBruce Richardson static rte_pipeline_table_action_handler_hit 342499a2dd95SBruce Richardson ah_selector(struct rte_table_action *action) 342599a2dd95SBruce Richardson { 342699a2dd95SBruce Richardson if (action->cfg.action_mask == (1LLU << RTE_TABLE_ACTION_FWD)) 342799a2dd95SBruce Richardson return NULL; 342899a2dd95SBruce Richardson 342999a2dd95SBruce Richardson return ah_default; 343099a2dd95SBruce Richardson } 343199a2dd95SBruce Richardson 343299a2dd95SBruce Richardson int 343399a2dd95SBruce Richardson rte_table_action_table_params_get(struct rte_table_action *action, 343499a2dd95SBruce Richardson struct rte_pipeline_table_params *params) 343599a2dd95SBruce Richardson { 343699a2dd95SBruce Richardson rte_pipeline_table_action_handler_hit f_action_hit; 343799a2dd95SBruce Richardson uint32_t total_size; 343899a2dd95SBruce Richardson 343999a2dd95SBruce Richardson /* Check input arguments */ 344099a2dd95SBruce Richardson if ((action == NULL) || 344199a2dd95SBruce Richardson (params == NULL)) 344299a2dd95SBruce Richardson return -EINVAL; 344399a2dd95SBruce Richardson 344499a2dd95SBruce Richardson f_action_hit = ah_selector(action); 344599a2dd95SBruce Richardson total_size = rte_align32pow2(action->data.total_size); 344699a2dd95SBruce Richardson 344799a2dd95SBruce Richardson /* Fill in params */ 344899a2dd95SBruce Richardson params->f_action_hit = f_action_hit; 344999a2dd95SBruce Richardson params->f_action_miss = NULL; 345099a2dd95SBruce Richardson params->arg_ah = (f_action_hit) ? action : NULL; 345199a2dd95SBruce Richardson params->action_data_size = total_size - 345299a2dd95SBruce Richardson sizeof(struct rte_pipeline_table_entry); 345399a2dd95SBruce Richardson 345499a2dd95SBruce Richardson return 0; 345599a2dd95SBruce Richardson } 345699a2dd95SBruce Richardson 345799a2dd95SBruce Richardson int 345899a2dd95SBruce Richardson rte_table_action_free(struct rte_table_action *action) 345999a2dd95SBruce Richardson { 346099a2dd95SBruce Richardson if (action == NULL) 346199a2dd95SBruce Richardson return 0; 346299a2dd95SBruce Richardson 346399a2dd95SBruce Richardson rte_free(action); 346499a2dd95SBruce Richardson 346599a2dd95SBruce Richardson return 0; 346699a2dd95SBruce Richardson } 3467