1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2018 Intel Corporation 3 */ 4 #include <stdlib.h> 5 #include <string.h> 6 7 #include <rte_common.h> 8 #include <rte_byteorder.h> 9 #include <rte_cycles.h> 10 #include <rte_malloc.h> 11 #include <rte_memcpy.h> 12 #include <rte_ether.h> 13 #include <rte_ip.h> 14 #include <rte_tcp.h> 15 #include <rte_udp.h> 16 #include <rte_vxlan.h> 17 #include <rte_cryptodev.h> 18 19 #include "rte_table_action.h" 20 21 #define rte_htons rte_cpu_to_be_16 22 #define rte_htonl rte_cpu_to_be_32 23 24 #define rte_ntohs rte_be_to_cpu_16 25 #define rte_ntohl rte_be_to_cpu_32 26 27 /** 28 * RTE_TABLE_ACTION_FWD 29 */ 30 #define fwd_data rte_pipeline_table_entry 31 32 static int 33 fwd_apply(struct fwd_data *data, 34 struct rte_table_action_fwd_params *p) 35 { 36 data->action = p->action; 37 38 if (p->action == RTE_PIPELINE_ACTION_PORT) 39 data->port_id = p->id; 40 41 if (p->action == RTE_PIPELINE_ACTION_TABLE) 42 data->table_id = p->id; 43 44 return 0; 45 } 46 47 /** 48 * RTE_TABLE_ACTION_LB 49 */ 50 static int 51 lb_cfg_check(struct rte_table_action_lb_config *cfg) 52 { 53 if ((cfg == NULL) || 54 (cfg->key_size < RTE_TABLE_ACTION_LB_KEY_SIZE_MIN) || 55 (cfg->key_size > RTE_TABLE_ACTION_LB_KEY_SIZE_MAX) || 56 (!rte_is_power_of_2(cfg->key_size)) || 57 (cfg->f_hash == NULL)) 58 return -1; 59 60 return 0; 61 } 62 63 struct lb_data { 64 uint32_t out[RTE_TABLE_ACTION_LB_TABLE_SIZE]; 65 } __rte_packed; 66 67 static int 68 lb_apply(struct lb_data *data, 69 struct rte_table_action_lb_params *p) 70 { 71 memcpy(data->out, p->out, sizeof(data->out)); 72 73 return 0; 74 } 75 76 static __rte_always_inline void 77 pkt_work_lb(struct rte_mbuf *mbuf, 78 struct lb_data *data, 79 struct rte_table_action_lb_config *cfg) 80 { 81 uint8_t *pkt_key = RTE_MBUF_METADATA_UINT8_PTR(mbuf, cfg->key_offset); 82 uint32_t *out = RTE_MBUF_METADATA_UINT32_PTR(mbuf, cfg->out_offset); 83 uint64_t digest, pos; 84 uint32_t out_val; 85 86 digest = cfg->f_hash(pkt_key, 87 cfg->key_mask, 88 cfg->key_size, 89 cfg->seed); 90 pos = digest & (RTE_TABLE_ACTION_LB_TABLE_SIZE - 1); 91 out_val = data->out[pos]; 92 93 *out = out_val; 94 } 95 96 /** 97 * RTE_TABLE_ACTION_MTR 98 */ 99 static int 100 mtr_cfg_check(struct rte_table_action_mtr_config *mtr) 101 { 102 if ((mtr->alg == RTE_TABLE_ACTION_METER_SRTCM) || 103 ((mtr->n_tc != 1) && (mtr->n_tc != 4)) || 104 (mtr->n_bytes_enabled != 0)) 105 return -ENOTSUP; 106 return 0; 107 } 108 109 struct mtr_trtcm_data { 110 struct rte_meter_trtcm trtcm; 111 uint64_t stats[RTE_COLORS]; 112 } __rte_packed; 113 114 #define MTR_TRTCM_DATA_METER_PROFILE_ID_GET(data) \ 115 (((data)->stats[RTE_COLOR_GREEN] & 0xF8LLU) >> 3) 116 117 static void 118 mtr_trtcm_data_meter_profile_id_set(struct mtr_trtcm_data *data, 119 uint32_t profile_id) 120 { 121 data->stats[RTE_COLOR_GREEN] &= ~0xF8LLU; 122 data->stats[RTE_COLOR_GREEN] |= (profile_id % 32) << 3; 123 } 124 125 #define MTR_TRTCM_DATA_POLICER_ACTION_DROP_GET(data, color)\ 126 (((data)->stats[(color)] & 4LLU) >> 2) 127 128 #define MTR_TRTCM_DATA_POLICER_ACTION_COLOR_GET(data, color)\ 129 ((enum rte_color)((data)->stats[(color)] & 3LLU)) 130 131 static void 132 mtr_trtcm_data_policer_action_set(struct mtr_trtcm_data *data, 133 enum rte_color color, 134 enum rte_table_action_policer action) 135 { 136 if (action == RTE_TABLE_ACTION_POLICER_DROP) { 137 data->stats[color] |= 4LLU; 138 } else { 139 data->stats[color] &= ~7LLU; 140 data->stats[color] |= color & 3LLU; 141 } 142 } 143 144 static uint64_t 145 mtr_trtcm_data_stats_get(struct mtr_trtcm_data *data, 146 enum rte_color color) 147 { 148 return data->stats[color] >> 8; 149 } 150 151 static void 152 mtr_trtcm_data_stats_reset(struct mtr_trtcm_data *data, 153 enum rte_color color) 154 { 155 data->stats[color] &= 0xFFLU; 156 } 157 158 #define MTR_TRTCM_DATA_STATS_INC(data, color) \ 159 ((data)->stats[(color)] += (1LLU << 8)) 160 161 static size_t 162 mtr_data_size(struct rte_table_action_mtr_config *mtr) 163 { 164 return mtr->n_tc * sizeof(struct mtr_trtcm_data); 165 } 166 167 struct dscp_table_entry_data { 168 enum rte_color color; 169 uint16_t tc; 170 uint16_t tc_queue; 171 }; 172 173 struct dscp_table_data { 174 struct dscp_table_entry_data entry[64]; 175 }; 176 177 struct meter_profile_data { 178 struct rte_meter_trtcm_profile profile; 179 uint32_t profile_id; 180 int valid; 181 }; 182 183 static struct meter_profile_data * 184 meter_profile_data_find(struct meter_profile_data *mp, 185 uint32_t mp_size, 186 uint32_t profile_id) 187 { 188 uint32_t i; 189 190 for (i = 0; i < mp_size; i++) { 191 struct meter_profile_data *mp_data = &mp[i]; 192 193 if (mp_data->valid && (mp_data->profile_id == profile_id)) 194 return mp_data; 195 } 196 197 return NULL; 198 } 199 200 static struct meter_profile_data * 201 meter_profile_data_find_unused(struct meter_profile_data *mp, 202 uint32_t mp_size) 203 { 204 uint32_t i; 205 206 for (i = 0; i < mp_size; i++) { 207 struct meter_profile_data *mp_data = &mp[i]; 208 209 if (!mp_data->valid) 210 return mp_data; 211 } 212 213 return NULL; 214 } 215 216 static int 217 mtr_apply_check(struct rte_table_action_mtr_params *p, 218 struct rte_table_action_mtr_config *cfg, 219 struct meter_profile_data *mp, 220 uint32_t mp_size) 221 { 222 uint32_t i; 223 224 if (p->tc_mask > RTE_LEN2MASK(cfg->n_tc, uint32_t)) 225 return -EINVAL; 226 227 for (i = 0; i < RTE_TABLE_ACTION_TC_MAX; i++) { 228 struct rte_table_action_mtr_tc_params *p_tc = &p->mtr[i]; 229 struct meter_profile_data *mp_data; 230 231 if ((p->tc_mask & (1LLU << i)) == 0) 232 continue; 233 234 mp_data = meter_profile_data_find(mp, 235 mp_size, 236 p_tc->meter_profile_id); 237 if (!mp_data) 238 return -EINVAL; 239 } 240 241 return 0; 242 } 243 244 static int 245 mtr_apply(struct mtr_trtcm_data *data, 246 struct rte_table_action_mtr_params *p, 247 struct rte_table_action_mtr_config *cfg, 248 struct meter_profile_data *mp, 249 uint32_t mp_size) 250 { 251 uint32_t i; 252 int status; 253 254 /* Check input arguments */ 255 status = mtr_apply_check(p, cfg, mp, mp_size); 256 if (status) 257 return status; 258 259 /* Apply */ 260 for (i = 0; i < RTE_TABLE_ACTION_TC_MAX; i++) { 261 struct rte_table_action_mtr_tc_params *p_tc = &p->mtr[i]; 262 struct mtr_trtcm_data *data_tc = &data[i]; 263 struct meter_profile_data *mp_data; 264 265 if ((p->tc_mask & (1LLU << i)) == 0) 266 continue; 267 268 /* Find profile */ 269 mp_data = meter_profile_data_find(mp, 270 mp_size, 271 p_tc->meter_profile_id); 272 if (!mp_data) 273 return -EINVAL; 274 275 memset(data_tc, 0, sizeof(*data_tc)); 276 277 /* Meter object */ 278 status = rte_meter_trtcm_config(&data_tc->trtcm, 279 &mp_data->profile); 280 if (status) 281 return status; 282 283 /* Meter profile */ 284 mtr_trtcm_data_meter_profile_id_set(data_tc, 285 mp_data - mp); 286 287 /* Policer actions */ 288 mtr_trtcm_data_policer_action_set(data_tc, 289 RTE_COLOR_GREEN, 290 p_tc->policer[RTE_COLOR_GREEN]); 291 292 mtr_trtcm_data_policer_action_set(data_tc, 293 RTE_COLOR_YELLOW, 294 p_tc->policer[RTE_COLOR_YELLOW]); 295 296 mtr_trtcm_data_policer_action_set(data_tc, 297 RTE_COLOR_RED, 298 p_tc->policer[RTE_COLOR_RED]); 299 } 300 301 return 0; 302 } 303 304 static __rte_always_inline uint64_t 305 pkt_work_mtr(struct rte_mbuf *mbuf, 306 struct mtr_trtcm_data *data, 307 struct dscp_table_data *dscp_table, 308 struct meter_profile_data *mp, 309 uint64_t time, 310 uint32_t dscp, 311 uint16_t total_length) 312 { 313 uint64_t drop_mask; 314 struct dscp_table_entry_data *dscp_entry = &dscp_table->entry[dscp]; 315 enum rte_color color_in, color_meter, color_policer; 316 uint32_t tc, mp_id; 317 318 tc = dscp_entry->tc; 319 color_in = dscp_entry->color; 320 data += tc; 321 mp_id = MTR_TRTCM_DATA_METER_PROFILE_ID_GET(data); 322 323 /* Meter */ 324 color_meter = rte_meter_trtcm_color_aware_check( 325 &data->trtcm, 326 &mp[mp_id].profile, 327 time, 328 total_length, 329 color_in); 330 331 /* Stats */ 332 MTR_TRTCM_DATA_STATS_INC(data, color_meter); 333 334 /* Police */ 335 drop_mask = MTR_TRTCM_DATA_POLICER_ACTION_DROP_GET(data, color_meter); 336 color_policer = 337 MTR_TRTCM_DATA_POLICER_ACTION_COLOR_GET(data, color_meter); 338 rte_mbuf_sched_color_set(mbuf, (uint8_t)color_policer); 339 340 return drop_mask; 341 } 342 343 /** 344 * RTE_TABLE_ACTION_TM 345 */ 346 static int 347 tm_cfg_check(struct rte_table_action_tm_config *tm) 348 { 349 if ((tm->n_subports_per_port == 0) || 350 (rte_is_power_of_2(tm->n_subports_per_port) == 0) || 351 (tm->n_subports_per_port > UINT16_MAX) || 352 (tm->n_pipes_per_subport == 0) || 353 (rte_is_power_of_2(tm->n_pipes_per_subport) == 0)) 354 return -ENOTSUP; 355 356 return 0; 357 } 358 359 struct tm_data { 360 uint32_t queue_id; 361 uint32_t reserved; 362 } __rte_packed; 363 364 static int 365 tm_apply_check(struct rte_table_action_tm_params *p, 366 struct rte_table_action_tm_config *cfg) 367 { 368 if ((p->subport_id >= cfg->n_subports_per_port) || 369 (p->pipe_id >= cfg->n_pipes_per_subport)) 370 return -EINVAL; 371 372 return 0; 373 } 374 375 static int 376 tm_apply(struct tm_data *data, 377 struct rte_table_action_tm_params *p, 378 struct rte_table_action_tm_config *cfg) 379 { 380 int status; 381 382 /* Check input arguments */ 383 status = tm_apply_check(p, cfg); 384 if (status) 385 return status; 386 387 /* Apply */ 388 data->queue_id = p->subport_id << 389 (__builtin_ctz(cfg->n_pipes_per_subport) + 4) | 390 p->pipe_id << 4; 391 392 return 0; 393 } 394 395 static __rte_always_inline void 396 pkt_work_tm(struct rte_mbuf *mbuf, 397 struct tm_data *data, 398 struct dscp_table_data *dscp_table, 399 uint32_t dscp) 400 { 401 struct dscp_table_entry_data *dscp_entry = &dscp_table->entry[dscp]; 402 uint32_t queue_id = data->queue_id | 403 dscp_entry->tc_queue; 404 rte_mbuf_sched_set(mbuf, queue_id, dscp_entry->tc, 405 (uint8_t)dscp_entry->color); 406 } 407 408 /** 409 * RTE_TABLE_ACTION_ENCAP 410 */ 411 static int 412 encap_valid(enum rte_table_action_encap_type encap) 413 { 414 switch (encap) { 415 case RTE_TABLE_ACTION_ENCAP_ETHER: 416 case RTE_TABLE_ACTION_ENCAP_VLAN: 417 case RTE_TABLE_ACTION_ENCAP_QINQ: 418 case RTE_TABLE_ACTION_ENCAP_MPLS: 419 case RTE_TABLE_ACTION_ENCAP_PPPOE: 420 case RTE_TABLE_ACTION_ENCAP_VXLAN: 421 case RTE_TABLE_ACTION_ENCAP_QINQ_PPPOE: 422 return 1; 423 default: 424 return 0; 425 } 426 } 427 428 static int 429 encap_cfg_check(struct rte_table_action_encap_config *encap) 430 { 431 if ((encap->encap_mask == 0) || 432 (__builtin_popcountll(encap->encap_mask) != 1)) 433 return -ENOTSUP; 434 435 return 0; 436 } 437 438 struct encap_ether_data { 439 struct rte_ether_hdr ether; 440 }; 441 442 #define VLAN(pcp, dei, vid) \ 443 ((uint16_t)((((uint64_t)(pcp)) & 0x7LLU) << 13) | \ 444 ((((uint64_t)(dei)) & 0x1LLU) << 12) | \ 445 (((uint64_t)(vid)) & 0xFFFLLU)) \ 446 447 struct encap_vlan_data { 448 struct rte_ether_hdr ether; 449 struct rte_vlan_hdr vlan; 450 }; 451 452 struct encap_qinq_data { 453 struct rte_ether_hdr ether; 454 struct rte_vlan_hdr svlan; 455 struct rte_vlan_hdr cvlan; 456 }; 457 458 #define ETHER_TYPE_MPLS_UNICAST 0x8847 459 460 #define ETHER_TYPE_MPLS_MULTICAST 0x8848 461 462 #define MPLS(label, tc, s, ttl) \ 463 ((uint32_t)(((((uint64_t)(label)) & 0xFFFFFLLU) << 12) |\ 464 ((((uint64_t)(tc)) & 0x7LLU) << 9) | \ 465 ((((uint64_t)(s)) & 0x1LLU) << 8) | \ 466 (((uint64_t)(ttl)) & 0xFFLLU))) 467 468 struct encap_mpls_data { 469 struct rte_ether_hdr ether; 470 uint32_t mpls[RTE_TABLE_ACTION_MPLS_LABELS_MAX]; 471 uint32_t mpls_count; 472 } __rte_packed __rte_aligned(2); 473 474 #define PPP_PROTOCOL_IP 0x0021 475 476 struct pppoe_ppp_hdr { 477 uint16_t ver_type_code; 478 uint16_t session_id; 479 uint16_t length; 480 uint16_t protocol; 481 }; 482 483 struct encap_pppoe_data { 484 struct rte_ether_hdr ether; 485 struct pppoe_ppp_hdr pppoe_ppp; 486 }; 487 488 #define IP_PROTO_UDP 17 489 490 struct encap_vxlan_ipv4_data { 491 struct rte_ether_hdr ether; 492 struct rte_ipv4_hdr ipv4; 493 struct rte_udp_hdr udp; 494 struct rte_vxlan_hdr vxlan; 495 } __rte_packed __rte_aligned(2); 496 497 struct encap_vxlan_ipv4_vlan_data { 498 struct rte_ether_hdr ether; 499 struct rte_vlan_hdr vlan; 500 struct rte_ipv4_hdr ipv4; 501 struct rte_udp_hdr udp; 502 struct rte_vxlan_hdr vxlan; 503 } __rte_packed __rte_aligned(2); 504 505 struct encap_vxlan_ipv6_data { 506 struct rte_ether_hdr ether; 507 struct rte_ipv6_hdr ipv6; 508 struct rte_udp_hdr udp; 509 struct rte_vxlan_hdr vxlan; 510 } __rte_packed __rte_aligned(2); 511 512 struct encap_vxlan_ipv6_vlan_data { 513 struct rte_ether_hdr ether; 514 struct rte_vlan_hdr vlan; 515 struct rte_ipv6_hdr ipv6; 516 struct rte_udp_hdr udp; 517 struct rte_vxlan_hdr vxlan; 518 } __rte_packed __rte_aligned(2); 519 520 struct encap_qinq_pppoe_data { 521 struct rte_ether_hdr ether; 522 struct rte_vlan_hdr svlan; 523 struct rte_vlan_hdr cvlan; 524 struct pppoe_ppp_hdr pppoe_ppp; 525 } __rte_packed __rte_aligned(2); 526 527 static size_t 528 encap_data_size(struct rte_table_action_encap_config *encap) 529 { 530 switch (encap->encap_mask) { 531 case 1LLU << RTE_TABLE_ACTION_ENCAP_ETHER: 532 return sizeof(struct encap_ether_data); 533 534 case 1LLU << RTE_TABLE_ACTION_ENCAP_VLAN: 535 return sizeof(struct encap_vlan_data); 536 537 case 1LLU << RTE_TABLE_ACTION_ENCAP_QINQ: 538 return sizeof(struct encap_qinq_data); 539 540 case 1LLU << RTE_TABLE_ACTION_ENCAP_MPLS: 541 return sizeof(struct encap_mpls_data); 542 543 case 1LLU << RTE_TABLE_ACTION_ENCAP_PPPOE: 544 return sizeof(struct encap_pppoe_data); 545 546 case 1LLU << RTE_TABLE_ACTION_ENCAP_VXLAN: 547 if (encap->vxlan.ip_version) 548 if (encap->vxlan.vlan) 549 return sizeof(struct encap_vxlan_ipv4_vlan_data); 550 else 551 return sizeof(struct encap_vxlan_ipv4_data); 552 else 553 if (encap->vxlan.vlan) 554 return sizeof(struct encap_vxlan_ipv6_vlan_data); 555 else 556 return sizeof(struct encap_vxlan_ipv6_data); 557 558 case 1LLU << RTE_TABLE_ACTION_ENCAP_QINQ_PPPOE: 559 return sizeof(struct encap_qinq_pppoe_data); 560 561 default: 562 return 0; 563 } 564 } 565 566 static int 567 encap_apply_check(struct rte_table_action_encap_params *p, 568 struct rte_table_action_encap_config *cfg) 569 { 570 if ((encap_valid(p->type) == 0) || 571 ((cfg->encap_mask & (1LLU << p->type)) == 0)) 572 return -EINVAL; 573 574 switch (p->type) { 575 case RTE_TABLE_ACTION_ENCAP_ETHER: 576 return 0; 577 578 case RTE_TABLE_ACTION_ENCAP_VLAN: 579 return 0; 580 581 case RTE_TABLE_ACTION_ENCAP_QINQ: 582 return 0; 583 584 case RTE_TABLE_ACTION_ENCAP_MPLS: 585 if ((p->mpls.mpls_count == 0) || 586 (p->mpls.mpls_count > RTE_TABLE_ACTION_MPLS_LABELS_MAX)) 587 return -EINVAL; 588 589 return 0; 590 591 case RTE_TABLE_ACTION_ENCAP_PPPOE: 592 return 0; 593 594 case RTE_TABLE_ACTION_ENCAP_VXLAN: 595 return 0; 596 597 case RTE_TABLE_ACTION_ENCAP_QINQ_PPPOE: 598 return 0; 599 600 default: 601 return -EINVAL; 602 } 603 } 604 605 static int 606 encap_ether_apply(void *data, 607 struct rte_table_action_encap_params *p, 608 struct rte_table_action_common_config *common_cfg) 609 { 610 struct encap_ether_data *d = data; 611 uint16_t ethertype = (common_cfg->ip_version) ? 612 RTE_ETHER_TYPE_IPV4 : 613 RTE_ETHER_TYPE_IPV6; 614 615 /* Ethernet */ 616 rte_ether_addr_copy(&p->ether.ether.da, &d->ether.dst_addr); 617 rte_ether_addr_copy(&p->ether.ether.sa, &d->ether.src_addr); 618 d->ether.ether_type = rte_htons(ethertype); 619 620 return 0; 621 } 622 623 static int 624 encap_vlan_apply(void *data, 625 struct rte_table_action_encap_params *p, 626 struct rte_table_action_common_config *common_cfg) 627 { 628 struct encap_vlan_data *d = data; 629 uint16_t ethertype = (common_cfg->ip_version) ? 630 RTE_ETHER_TYPE_IPV4 : 631 RTE_ETHER_TYPE_IPV6; 632 633 /* Ethernet */ 634 rte_ether_addr_copy(&p->vlan.ether.da, &d->ether.dst_addr); 635 rte_ether_addr_copy(&p->vlan.ether.sa, &d->ether.src_addr); 636 d->ether.ether_type = rte_htons(RTE_ETHER_TYPE_VLAN); 637 638 /* VLAN */ 639 d->vlan.vlan_tci = rte_htons(VLAN(p->vlan.vlan.pcp, 640 p->vlan.vlan.dei, 641 p->vlan.vlan.vid)); 642 d->vlan.eth_proto = rte_htons(ethertype); 643 644 return 0; 645 } 646 647 static int 648 encap_qinq_apply(void *data, 649 struct rte_table_action_encap_params *p, 650 struct rte_table_action_common_config *common_cfg) 651 { 652 struct encap_qinq_data *d = data; 653 uint16_t ethertype = (common_cfg->ip_version) ? 654 RTE_ETHER_TYPE_IPV4 : 655 RTE_ETHER_TYPE_IPV6; 656 657 /* Ethernet */ 658 rte_ether_addr_copy(&p->qinq.ether.da, &d->ether.dst_addr); 659 rte_ether_addr_copy(&p->qinq.ether.sa, &d->ether.src_addr); 660 d->ether.ether_type = rte_htons(RTE_ETHER_TYPE_QINQ); 661 662 /* SVLAN */ 663 d->svlan.vlan_tci = rte_htons(VLAN(p->qinq.svlan.pcp, 664 p->qinq.svlan.dei, 665 p->qinq.svlan.vid)); 666 d->svlan.eth_proto = rte_htons(RTE_ETHER_TYPE_VLAN); 667 668 /* CVLAN */ 669 d->cvlan.vlan_tci = rte_htons(VLAN(p->qinq.cvlan.pcp, 670 p->qinq.cvlan.dei, 671 p->qinq.cvlan.vid)); 672 d->cvlan.eth_proto = rte_htons(ethertype); 673 674 return 0; 675 } 676 677 static int 678 encap_qinq_pppoe_apply(void *data, 679 struct rte_table_action_encap_params *p) 680 { 681 struct encap_qinq_pppoe_data *d = data; 682 683 /* Ethernet */ 684 rte_ether_addr_copy(&p->qinq.ether.da, &d->ether.dst_addr); 685 rte_ether_addr_copy(&p->qinq.ether.sa, &d->ether.src_addr); 686 d->ether.ether_type = rte_htons(RTE_ETHER_TYPE_VLAN); 687 688 /* SVLAN */ 689 d->svlan.vlan_tci = rte_htons(VLAN(p->qinq.svlan.pcp, 690 p->qinq.svlan.dei, 691 p->qinq.svlan.vid)); 692 d->svlan.eth_proto = rte_htons(RTE_ETHER_TYPE_VLAN); 693 694 /* CVLAN */ 695 d->cvlan.vlan_tci = rte_htons(VLAN(p->qinq.cvlan.pcp, 696 p->qinq.cvlan.dei, 697 p->qinq.cvlan.vid)); 698 d->cvlan.eth_proto = rte_htons(RTE_ETHER_TYPE_PPPOE_SESSION); 699 700 /* PPPoE and PPP*/ 701 d->pppoe_ppp.ver_type_code = rte_htons(0x1100); 702 d->pppoe_ppp.session_id = rte_htons(p->qinq_pppoe.pppoe.session_id); 703 d->pppoe_ppp.length = 0; /* not pre-computed */ 704 d->pppoe_ppp.protocol = rte_htons(PPP_PROTOCOL_IP); 705 706 return 0; 707 } 708 709 static int 710 encap_mpls_apply(void *data, 711 struct rte_table_action_encap_params *p) 712 { 713 struct encap_mpls_data *d = data; 714 uint16_t ethertype = (p->mpls.unicast) ? 715 ETHER_TYPE_MPLS_UNICAST : 716 ETHER_TYPE_MPLS_MULTICAST; 717 uint32_t i; 718 719 /* Ethernet */ 720 rte_ether_addr_copy(&p->mpls.ether.da, &d->ether.dst_addr); 721 rte_ether_addr_copy(&p->mpls.ether.sa, &d->ether.src_addr); 722 d->ether.ether_type = rte_htons(ethertype); 723 724 /* MPLS */ 725 for (i = 0; i < p->mpls.mpls_count - 1; i++) 726 d->mpls[i] = rte_htonl(MPLS(p->mpls.mpls[i].label, 727 p->mpls.mpls[i].tc, 728 0, 729 p->mpls.mpls[i].ttl)); 730 731 d->mpls[i] = rte_htonl(MPLS(p->mpls.mpls[i].label, 732 p->mpls.mpls[i].tc, 733 1, 734 p->mpls.mpls[i].ttl)); 735 736 d->mpls_count = p->mpls.mpls_count; 737 return 0; 738 } 739 740 static int 741 encap_pppoe_apply(void *data, 742 struct rte_table_action_encap_params *p) 743 { 744 struct encap_pppoe_data *d = data; 745 746 /* Ethernet */ 747 rte_ether_addr_copy(&p->pppoe.ether.da, &d->ether.dst_addr); 748 rte_ether_addr_copy(&p->pppoe.ether.sa, &d->ether.src_addr); 749 d->ether.ether_type = rte_htons(RTE_ETHER_TYPE_PPPOE_SESSION); 750 751 /* PPPoE and PPP*/ 752 d->pppoe_ppp.ver_type_code = rte_htons(0x1100); 753 d->pppoe_ppp.session_id = rte_htons(p->pppoe.pppoe.session_id); 754 d->pppoe_ppp.length = 0; /* not pre-computed */ 755 d->pppoe_ppp.protocol = rte_htons(PPP_PROTOCOL_IP); 756 757 return 0; 758 } 759 760 static int 761 encap_vxlan_apply(void *data, 762 struct rte_table_action_encap_params *p, 763 struct rte_table_action_encap_config *cfg) 764 { 765 if ((p->vxlan.vxlan.vni > 0xFFFFFF) || 766 (cfg->vxlan.ip_version && (p->vxlan.ipv4.dscp > 0x3F)) || 767 (!cfg->vxlan.ip_version && (p->vxlan.ipv6.flow_label > 0xFFFFF)) || 768 (!cfg->vxlan.ip_version && (p->vxlan.ipv6.dscp > 0x3F)) || 769 (cfg->vxlan.vlan && (p->vxlan.vlan.vid > 0xFFF))) 770 return -1; 771 772 if (cfg->vxlan.ip_version) 773 if (cfg->vxlan.vlan) { 774 struct encap_vxlan_ipv4_vlan_data *d = data; 775 776 /* Ethernet */ 777 rte_ether_addr_copy(&p->vxlan.ether.da, 778 &d->ether.dst_addr); 779 rte_ether_addr_copy(&p->vxlan.ether.sa, 780 &d->ether.src_addr); 781 d->ether.ether_type = rte_htons(RTE_ETHER_TYPE_VLAN); 782 783 /* VLAN */ 784 d->vlan.vlan_tci = rte_htons(VLAN(p->vxlan.vlan.pcp, 785 p->vxlan.vlan.dei, 786 p->vxlan.vlan.vid)); 787 d->vlan.eth_proto = rte_htons(RTE_ETHER_TYPE_IPV4); 788 789 /* IPv4*/ 790 d->ipv4.version_ihl = 0x45; 791 d->ipv4.type_of_service = p->vxlan.ipv4.dscp << 2; 792 d->ipv4.total_length = 0; /* not pre-computed */ 793 d->ipv4.packet_id = 0; 794 d->ipv4.fragment_offset = 0; 795 d->ipv4.time_to_live = p->vxlan.ipv4.ttl; 796 d->ipv4.next_proto_id = IP_PROTO_UDP; 797 d->ipv4.hdr_checksum = 0; 798 d->ipv4.src_addr = rte_htonl(p->vxlan.ipv4.sa); 799 d->ipv4.dst_addr = rte_htonl(p->vxlan.ipv4.da); 800 801 d->ipv4.hdr_checksum = rte_ipv4_cksum(&d->ipv4); 802 803 /* UDP */ 804 d->udp.src_port = rte_htons(p->vxlan.udp.sp); 805 d->udp.dst_port = rte_htons(p->vxlan.udp.dp); 806 d->udp.dgram_len = 0; /* not pre-computed */ 807 d->udp.dgram_cksum = 0; 808 809 /* VXLAN */ 810 d->vxlan.vx_flags = rte_htonl(0x08000000); 811 d->vxlan.vx_vni = rte_htonl(p->vxlan.vxlan.vni << 8); 812 813 return 0; 814 } else { 815 struct encap_vxlan_ipv4_data *d = data; 816 817 /* Ethernet */ 818 rte_ether_addr_copy(&p->vxlan.ether.da, 819 &d->ether.dst_addr); 820 rte_ether_addr_copy(&p->vxlan.ether.sa, 821 &d->ether.src_addr); 822 d->ether.ether_type = rte_htons(RTE_ETHER_TYPE_IPV4); 823 824 /* IPv4*/ 825 d->ipv4.version_ihl = 0x45; 826 d->ipv4.type_of_service = p->vxlan.ipv4.dscp << 2; 827 d->ipv4.total_length = 0; /* not pre-computed */ 828 d->ipv4.packet_id = 0; 829 d->ipv4.fragment_offset = 0; 830 d->ipv4.time_to_live = p->vxlan.ipv4.ttl; 831 d->ipv4.next_proto_id = IP_PROTO_UDP; 832 d->ipv4.hdr_checksum = 0; 833 d->ipv4.src_addr = rte_htonl(p->vxlan.ipv4.sa); 834 d->ipv4.dst_addr = rte_htonl(p->vxlan.ipv4.da); 835 836 d->ipv4.hdr_checksum = rte_ipv4_cksum(&d->ipv4); 837 838 /* UDP */ 839 d->udp.src_port = rte_htons(p->vxlan.udp.sp); 840 d->udp.dst_port = rte_htons(p->vxlan.udp.dp); 841 d->udp.dgram_len = 0; /* not pre-computed */ 842 d->udp.dgram_cksum = 0; 843 844 /* VXLAN */ 845 d->vxlan.vx_flags = rte_htonl(0x08000000); 846 d->vxlan.vx_vni = rte_htonl(p->vxlan.vxlan.vni << 8); 847 848 return 0; 849 } 850 else 851 if (cfg->vxlan.vlan) { 852 struct encap_vxlan_ipv6_vlan_data *d = data; 853 854 /* Ethernet */ 855 rte_ether_addr_copy(&p->vxlan.ether.da, 856 &d->ether.dst_addr); 857 rte_ether_addr_copy(&p->vxlan.ether.sa, 858 &d->ether.src_addr); 859 d->ether.ether_type = rte_htons(RTE_ETHER_TYPE_VLAN); 860 861 /* VLAN */ 862 d->vlan.vlan_tci = rte_htons(VLAN(p->vxlan.vlan.pcp, 863 p->vxlan.vlan.dei, 864 p->vxlan.vlan.vid)); 865 d->vlan.eth_proto = rte_htons(RTE_ETHER_TYPE_IPV6); 866 867 /* IPv6*/ 868 d->ipv6.vtc_flow = rte_htonl((6 << 28) | 869 (p->vxlan.ipv6.dscp << 22) | 870 p->vxlan.ipv6.flow_label); 871 d->ipv6.payload_len = 0; /* not pre-computed */ 872 d->ipv6.proto = IP_PROTO_UDP; 873 d->ipv6.hop_limits = p->vxlan.ipv6.hop_limit; 874 memcpy(d->ipv6.src_addr, 875 p->vxlan.ipv6.sa, 876 sizeof(p->vxlan.ipv6.sa)); 877 memcpy(d->ipv6.dst_addr, 878 p->vxlan.ipv6.da, 879 sizeof(p->vxlan.ipv6.da)); 880 881 /* UDP */ 882 d->udp.src_port = rte_htons(p->vxlan.udp.sp); 883 d->udp.dst_port = rte_htons(p->vxlan.udp.dp); 884 d->udp.dgram_len = 0; /* not pre-computed */ 885 d->udp.dgram_cksum = 0; 886 887 /* VXLAN */ 888 d->vxlan.vx_flags = rte_htonl(0x08000000); 889 d->vxlan.vx_vni = rte_htonl(p->vxlan.vxlan.vni << 8); 890 891 return 0; 892 } else { 893 struct encap_vxlan_ipv6_data *d = data; 894 895 /* Ethernet */ 896 rte_ether_addr_copy(&p->vxlan.ether.da, 897 &d->ether.dst_addr); 898 rte_ether_addr_copy(&p->vxlan.ether.sa, 899 &d->ether.src_addr); 900 d->ether.ether_type = rte_htons(RTE_ETHER_TYPE_IPV6); 901 902 /* IPv6*/ 903 d->ipv6.vtc_flow = rte_htonl((6 << 28) | 904 (p->vxlan.ipv6.dscp << 22) | 905 p->vxlan.ipv6.flow_label); 906 d->ipv6.payload_len = 0; /* not pre-computed */ 907 d->ipv6.proto = IP_PROTO_UDP; 908 d->ipv6.hop_limits = p->vxlan.ipv6.hop_limit; 909 memcpy(d->ipv6.src_addr, 910 p->vxlan.ipv6.sa, 911 sizeof(p->vxlan.ipv6.sa)); 912 memcpy(d->ipv6.dst_addr, 913 p->vxlan.ipv6.da, 914 sizeof(p->vxlan.ipv6.da)); 915 916 /* UDP */ 917 d->udp.src_port = rte_htons(p->vxlan.udp.sp); 918 d->udp.dst_port = rte_htons(p->vxlan.udp.dp); 919 d->udp.dgram_len = 0; /* not pre-computed */ 920 d->udp.dgram_cksum = 0; 921 922 /* VXLAN */ 923 d->vxlan.vx_flags = rte_htonl(0x08000000); 924 d->vxlan.vx_vni = rte_htonl(p->vxlan.vxlan.vni << 8); 925 926 return 0; 927 } 928 } 929 930 static int 931 encap_apply(void *data, 932 struct rte_table_action_encap_params *p, 933 struct rte_table_action_encap_config *cfg, 934 struct rte_table_action_common_config *common_cfg) 935 { 936 int status; 937 938 /* Check input arguments */ 939 status = encap_apply_check(p, cfg); 940 if (status) 941 return status; 942 943 switch (p->type) { 944 case RTE_TABLE_ACTION_ENCAP_ETHER: 945 return encap_ether_apply(data, p, common_cfg); 946 947 case RTE_TABLE_ACTION_ENCAP_VLAN: 948 return encap_vlan_apply(data, p, common_cfg); 949 950 case RTE_TABLE_ACTION_ENCAP_QINQ: 951 return encap_qinq_apply(data, p, common_cfg); 952 953 case RTE_TABLE_ACTION_ENCAP_MPLS: 954 return encap_mpls_apply(data, p); 955 956 case RTE_TABLE_ACTION_ENCAP_PPPOE: 957 return encap_pppoe_apply(data, p); 958 959 case RTE_TABLE_ACTION_ENCAP_VXLAN: 960 return encap_vxlan_apply(data, p, cfg); 961 962 case RTE_TABLE_ACTION_ENCAP_QINQ_PPPOE: 963 return encap_qinq_pppoe_apply(data, p); 964 965 default: 966 return -EINVAL; 967 } 968 } 969 970 static __rte_always_inline uint16_t 971 encap_vxlan_ipv4_checksum_update(uint16_t cksum0, 972 uint16_t total_length) 973 { 974 int32_t cksum1; 975 976 cksum1 = cksum0; 977 cksum1 = ~cksum1 & 0xFFFF; 978 979 /* Add total length (one's complement logic) */ 980 cksum1 += total_length; 981 cksum1 = (cksum1 & 0xFFFF) + (cksum1 >> 16); 982 cksum1 = (cksum1 & 0xFFFF) + (cksum1 >> 16); 983 984 return (uint16_t)(~cksum1); 985 } 986 987 static __rte_always_inline void * 988 encap(void *dst, const void *src, size_t n) 989 { 990 dst = ((uint8_t *) dst) - n; 991 return rte_memcpy(dst, src, n); 992 } 993 994 static __rte_always_inline void 995 pkt_work_encap_vxlan_ipv4(struct rte_mbuf *mbuf, 996 struct encap_vxlan_ipv4_data *vxlan_tbl, 997 struct rte_table_action_encap_config *cfg) 998 { 999 uint32_t ether_offset = cfg->vxlan.data_offset; 1000 void *ether = RTE_MBUF_METADATA_UINT32_PTR(mbuf, ether_offset); 1001 struct encap_vxlan_ipv4_data *vxlan_pkt; 1002 uint16_t ether_length, ipv4_total_length, ipv4_hdr_cksum, udp_length; 1003 1004 ether_length = (uint16_t)mbuf->pkt_len; 1005 ipv4_total_length = ether_length + 1006 (sizeof(struct rte_vxlan_hdr) + 1007 sizeof(struct rte_udp_hdr) + 1008 sizeof(struct rte_ipv4_hdr)); 1009 ipv4_hdr_cksum = encap_vxlan_ipv4_checksum_update(vxlan_tbl->ipv4.hdr_checksum, 1010 rte_htons(ipv4_total_length)); 1011 udp_length = ether_length + 1012 (sizeof(struct rte_vxlan_hdr) + 1013 sizeof(struct rte_udp_hdr)); 1014 1015 vxlan_pkt = encap(ether, vxlan_tbl, sizeof(*vxlan_tbl)); 1016 vxlan_pkt->ipv4.total_length = rte_htons(ipv4_total_length); 1017 vxlan_pkt->ipv4.hdr_checksum = ipv4_hdr_cksum; 1018 vxlan_pkt->udp.dgram_len = rte_htons(udp_length); 1019 1020 mbuf->data_off = ether_offset - (sizeof(struct rte_mbuf) + sizeof(*vxlan_pkt)); 1021 mbuf->pkt_len = mbuf->data_len = ether_length + sizeof(*vxlan_pkt); 1022 } 1023 1024 static __rte_always_inline void 1025 pkt_work_encap_vxlan_ipv4_vlan(struct rte_mbuf *mbuf, 1026 struct encap_vxlan_ipv4_vlan_data *vxlan_tbl, 1027 struct rte_table_action_encap_config *cfg) 1028 { 1029 uint32_t ether_offset = cfg->vxlan.data_offset; 1030 void *ether = RTE_MBUF_METADATA_UINT32_PTR(mbuf, ether_offset); 1031 struct encap_vxlan_ipv4_vlan_data *vxlan_pkt; 1032 uint16_t ether_length, ipv4_total_length, ipv4_hdr_cksum, udp_length; 1033 1034 ether_length = (uint16_t)mbuf->pkt_len; 1035 ipv4_total_length = ether_length + 1036 (sizeof(struct rte_vxlan_hdr) + 1037 sizeof(struct rte_udp_hdr) + 1038 sizeof(struct rte_ipv4_hdr)); 1039 ipv4_hdr_cksum = encap_vxlan_ipv4_checksum_update(vxlan_tbl->ipv4.hdr_checksum, 1040 rte_htons(ipv4_total_length)); 1041 udp_length = ether_length + 1042 (sizeof(struct rte_vxlan_hdr) + 1043 sizeof(struct rte_udp_hdr)); 1044 1045 vxlan_pkt = encap(ether, vxlan_tbl, sizeof(*vxlan_tbl)); 1046 vxlan_pkt->ipv4.total_length = rte_htons(ipv4_total_length); 1047 vxlan_pkt->ipv4.hdr_checksum = ipv4_hdr_cksum; 1048 vxlan_pkt->udp.dgram_len = rte_htons(udp_length); 1049 1050 mbuf->data_off = ether_offset - (sizeof(struct rte_mbuf) + sizeof(*vxlan_pkt)); 1051 mbuf->pkt_len = mbuf->data_len = ether_length + sizeof(*vxlan_pkt); 1052 } 1053 1054 static __rte_always_inline void 1055 pkt_work_encap_vxlan_ipv6(struct rte_mbuf *mbuf, 1056 struct encap_vxlan_ipv6_data *vxlan_tbl, 1057 struct rte_table_action_encap_config *cfg) 1058 { 1059 uint32_t ether_offset = cfg->vxlan.data_offset; 1060 void *ether = RTE_MBUF_METADATA_UINT32_PTR(mbuf, ether_offset); 1061 struct encap_vxlan_ipv6_data *vxlan_pkt; 1062 uint16_t ether_length, ipv6_payload_length, udp_length; 1063 1064 ether_length = (uint16_t)mbuf->pkt_len; 1065 ipv6_payload_length = ether_length + 1066 (sizeof(struct rte_vxlan_hdr) + 1067 sizeof(struct rte_udp_hdr)); 1068 udp_length = ether_length + 1069 (sizeof(struct rte_vxlan_hdr) + 1070 sizeof(struct rte_udp_hdr)); 1071 1072 vxlan_pkt = encap(ether, vxlan_tbl, sizeof(*vxlan_tbl)); 1073 vxlan_pkt->ipv6.payload_len = rte_htons(ipv6_payload_length); 1074 vxlan_pkt->udp.dgram_len = rte_htons(udp_length); 1075 1076 mbuf->data_off = ether_offset - (sizeof(struct rte_mbuf) + sizeof(*vxlan_pkt)); 1077 mbuf->pkt_len = mbuf->data_len = ether_length + sizeof(*vxlan_pkt); 1078 } 1079 1080 static __rte_always_inline void 1081 pkt_work_encap_vxlan_ipv6_vlan(struct rte_mbuf *mbuf, 1082 struct encap_vxlan_ipv6_vlan_data *vxlan_tbl, 1083 struct rte_table_action_encap_config *cfg) 1084 { 1085 uint32_t ether_offset = cfg->vxlan.data_offset; 1086 void *ether = RTE_MBUF_METADATA_UINT32_PTR(mbuf, ether_offset); 1087 struct encap_vxlan_ipv6_vlan_data *vxlan_pkt; 1088 uint16_t ether_length, ipv6_payload_length, udp_length; 1089 1090 ether_length = (uint16_t)mbuf->pkt_len; 1091 ipv6_payload_length = ether_length + 1092 (sizeof(struct rte_vxlan_hdr) + 1093 sizeof(struct rte_udp_hdr)); 1094 udp_length = ether_length + 1095 (sizeof(struct rte_vxlan_hdr) + 1096 sizeof(struct rte_udp_hdr)); 1097 1098 vxlan_pkt = encap(ether, vxlan_tbl, sizeof(*vxlan_tbl)); 1099 vxlan_pkt->ipv6.payload_len = rte_htons(ipv6_payload_length); 1100 vxlan_pkt->udp.dgram_len = rte_htons(udp_length); 1101 1102 mbuf->data_off = ether_offset - (sizeof(struct rte_mbuf) + sizeof(*vxlan_pkt)); 1103 mbuf->pkt_len = mbuf->data_len = ether_length + sizeof(*vxlan_pkt); 1104 } 1105 1106 static __rte_always_inline void 1107 pkt_work_encap(struct rte_mbuf *mbuf, 1108 void *data, 1109 struct rte_table_action_encap_config *cfg, 1110 void *ip, 1111 uint16_t total_length, 1112 uint32_t ip_offset) 1113 { 1114 switch (cfg->encap_mask) { 1115 case 1LLU << RTE_TABLE_ACTION_ENCAP_ETHER: 1116 encap(ip, data, sizeof(struct encap_ether_data)); 1117 mbuf->data_off = ip_offset - (sizeof(struct rte_mbuf) + 1118 sizeof(struct encap_ether_data)); 1119 mbuf->pkt_len = mbuf->data_len = total_length + 1120 sizeof(struct encap_ether_data); 1121 break; 1122 1123 case 1LLU << RTE_TABLE_ACTION_ENCAP_VLAN: 1124 encap(ip, data, sizeof(struct encap_vlan_data)); 1125 mbuf->data_off = ip_offset - (sizeof(struct rte_mbuf) + 1126 sizeof(struct encap_vlan_data)); 1127 mbuf->pkt_len = mbuf->data_len = total_length + 1128 sizeof(struct encap_vlan_data); 1129 break; 1130 1131 case 1LLU << RTE_TABLE_ACTION_ENCAP_QINQ: 1132 encap(ip, data, sizeof(struct encap_qinq_data)); 1133 mbuf->data_off = ip_offset - (sizeof(struct rte_mbuf) + 1134 sizeof(struct encap_qinq_data)); 1135 mbuf->pkt_len = mbuf->data_len = total_length + 1136 sizeof(struct encap_qinq_data); 1137 break; 1138 1139 case 1LLU << RTE_TABLE_ACTION_ENCAP_MPLS: 1140 { 1141 struct encap_mpls_data *mpls = data; 1142 size_t size = sizeof(struct rte_ether_hdr) + 1143 mpls->mpls_count * 4; 1144 1145 encap(ip, data, size); 1146 mbuf->data_off = ip_offset - (sizeof(struct rte_mbuf) + size); 1147 mbuf->pkt_len = mbuf->data_len = total_length + size; 1148 break; 1149 } 1150 1151 case 1LLU << RTE_TABLE_ACTION_ENCAP_PPPOE: 1152 { 1153 struct encap_pppoe_data *pppoe = 1154 encap(ip, data, sizeof(struct encap_pppoe_data)); 1155 pppoe->pppoe_ppp.length = rte_htons(total_length + 2); 1156 mbuf->data_off = ip_offset - (sizeof(struct rte_mbuf) + 1157 sizeof(struct encap_pppoe_data)); 1158 mbuf->pkt_len = mbuf->data_len = total_length + 1159 sizeof(struct encap_pppoe_data); 1160 break; 1161 } 1162 1163 case 1LLU << RTE_TABLE_ACTION_ENCAP_QINQ_PPPOE: 1164 { 1165 struct encap_qinq_pppoe_data *qinq_pppoe = 1166 encap(ip, data, sizeof(struct encap_qinq_pppoe_data)); 1167 qinq_pppoe->pppoe_ppp.length = rte_htons(total_length + 2); 1168 mbuf->data_off = ip_offset - (sizeof(struct rte_mbuf) + 1169 sizeof(struct encap_qinq_pppoe_data)); 1170 mbuf->pkt_len = mbuf->data_len = total_length + 1171 sizeof(struct encap_qinq_pppoe_data); 1172 break; 1173 } 1174 1175 case 1LLU << RTE_TABLE_ACTION_ENCAP_VXLAN: 1176 { 1177 if (cfg->vxlan.ip_version) 1178 if (cfg->vxlan.vlan) 1179 pkt_work_encap_vxlan_ipv4_vlan(mbuf, data, cfg); 1180 else 1181 pkt_work_encap_vxlan_ipv4(mbuf, data, cfg); 1182 else 1183 if (cfg->vxlan.vlan) 1184 pkt_work_encap_vxlan_ipv6_vlan(mbuf, data, cfg); 1185 else 1186 pkt_work_encap_vxlan_ipv6(mbuf, data, cfg); 1187 } 1188 1189 default: 1190 break; 1191 } 1192 } 1193 1194 /** 1195 * RTE_TABLE_ACTION_NAT 1196 */ 1197 static int 1198 nat_cfg_check(struct rte_table_action_nat_config *nat) 1199 { 1200 if ((nat->proto != 0x06) && 1201 (nat->proto != 0x11)) 1202 return -ENOTSUP; 1203 1204 return 0; 1205 } 1206 1207 struct nat_ipv4_data { 1208 uint32_t addr; 1209 uint16_t port; 1210 } __rte_packed; 1211 1212 struct nat_ipv6_data { 1213 uint8_t addr[16]; 1214 uint16_t port; 1215 } __rte_packed; 1216 1217 static size_t 1218 nat_data_size(struct rte_table_action_nat_config *nat __rte_unused, 1219 struct rte_table_action_common_config *common) 1220 { 1221 int ip_version = common->ip_version; 1222 1223 return (ip_version) ? 1224 sizeof(struct nat_ipv4_data) : 1225 sizeof(struct nat_ipv6_data); 1226 } 1227 1228 static int 1229 nat_apply_check(struct rte_table_action_nat_params *p, 1230 struct rte_table_action_common_config *cfg) 1231 { 1232 if ((p->ip_version && (cfg->ip_version == 0)) || 1233 ((p->ip_version == 0) && cfg->ip_version)) 1234 return -EINVAL; 1235 1236 return 0; 1237 } 1238 1239 static int 1240 nat_apply(void *data, 1241 struct rte_table_action_nat_params *p, 1242 struct rte_table_action_common_config *cfg) 1243 { 1244 int status; 1245 1246 /* Check input arguments */ 1247 status = nat_apply_check(p, cfg); 1248 if (status) 1249 return status; 1250 1251 /* Apply */ 1252 if (p->ip_version) { 1253 struct nat_ipv4_data *d = data; 1254 1255 d->addr = rte_htonl(p->addr.ipv4); 1256 d->port = rte_htons(p->port); 1257 } else { 1258 struct nat_ipv6_data *d = data; 1259 1260 memcpy(d->addr, p->addr.ipv6, sizeof(d->addr)); 1261 d->port = rte_htons(p->port); 1262 } 1263 1264 return 0; 1265 } 1266 1267 static __rte_always_inline uint16_t 1268 nat_ipv4_checksum_update(uint16_t cksum0, 1269 uint32_t ip0, 1270 uint32_t ip1) 1271 { 1272 int32_t cksum1; 1273 1274 cksum1 = cksum0; 1275 cksum1 = ~cksum1 & 0xFFFF; 1276 1277 /* Subtract ip0 (one's complement logic) */ 1278 cksum1 -= (ip0 >> 16) + (ip0 & 0xFFFF); 1279 cksum1 = (cksum1 & 0xFFFF) + (cksum1 >> 16); 1280 cksum1 = (cksum1 & 0xFFFF) + (cksum1 >> 16); 1281 1282 /* Add ip1 (one's complement logic) */ 1283 cksum1 += (ip1 >> 16) + (ip1 & 0xFFFF); 1284 cksum1 = (cksum1 & 0xFFFF) + (cksum1 >> 16); 1285 cksum1 = (cksum1 & 0xFFFF) + (cksum1 >> 16); 1286 1287 return (uint16_t)(~cksum1); 1288 } 1289 1290 static __rte_always_inline uint16_t 1291 nat_ipv4_tcp_udp_checksum_update(uint16_t cksum0, 1292 uint32_t ip0, 1293 uint32_t ip1, 1294 uint16_t port0, 1295 uint16_t port1) 1296 { 1297 int32_t cksum1; 1298 1299 cksum1 = cksum0; 1300 cksum1 = ~cksum1 & 0xFFFF; 1301 1302 /* Subtract ip0 and port 0 (one's complement logic) */ 1303 cksum1 -= (ip0 >> 16) + (ip0 & 0xFFFF) + port0; 1304 cksum1 = (cksum1 & 0xFFFF) + (cksum1 >> 16); 1305 cksum1 = (cksum1 & 0xFFFF) + (cksum1 >> 16); 1306 1307 /* Add ip1 and port1 (one's complement logic) */ 1308 cksum1 += (ip1 >> 16) + (ip1 & 0xFFFF) + port1; 1309 cksum1 = (cksum1 & 0xFFFF) + (cksum1 >> 16); 1310 cksum1 = (cksum1 & 0xFFFF) + (cksum1 >> 16); 1311 1312 return (uint16_t)(~cksum1); 1313 } 1314 1315 static __rte_always_inline uint16_t 1316 nat_ipv6_tcp_udp_checksum_update(uint16_t cksum0, 1317 uint16_t *ip0, 1318 uint16_t *ip1, 1319 uint16_t port0, 1320 uint16_t port1) 1321 { 1322 int32_t cksum1; 1323 1324 cksum1 = cksum0; 1325 cksum1 = ~cksum1 & 0xFFFF; 1326 1327 /* Subtract ip0 and port 0 (one's complement logic) */ 1328 cksum1 -= ip0[0] + ip0[1] + ip0[2] + ip0[3] + 1329 ip0[4] + ip0[5] + ip0[6] + ip0[7] + port0; 1330 cksum1 = (cksum1 & 0xFFFF) + (cksum1 >> 16); 1331 cksum1 = (cksum1 & 0xFFFF) + (cksum1 >> 16); 1332 1333 /* Add ip1 and port1 (one's complement logic) */ 1334 cksum1 += ip1[0] + ip1[1] + ip1[2] + ip1[3] + 1335 ip1[4] + ip1[5] + ip1[6] + ip1[7] + port1; 1336 cksum1 = (cksum1 & 0xFFFF) + (cksum1 >> 16); 1337 cksum1 = (cksum1 & 0xFFFF) + (cksum1 >> 16); 1338 1339 return (uint16_t)(~cksum1); 1340 } 1341 1342 static __rte_always_inline void 1343 pkt_ipv4_work_nat(struct rte_ipv4_hdr *ip, 1344 struct nat_ipv4_data *data, 1345 struct rte_table_action_nat_config *cfg) 1346 { 1347 if (cfg->source_nat) { 1348 if (cfg->proto == 0x6) { 1349 struct rte_tcp_hdr *tcp = (struct rte_tcp_hdr *) &ip[1]; 1350 uint16_t ip_cksum, tcp_cksum; 1351 1352 ip_cksum = nat_ipv4_checksum_update(ip->hdr_checksum, 1353 ip->src_addr, 1354 data->addr); 1355 1356 tcp_cksum = nat_ipv4_tcp_udp_checksum_update(tcp->cksum, 1357 ip->src_addr, 1358 data->addr, 1359 tcp->src_port, 1360 data->port); 1361 1362 ip->src_addr = data->addr; 1363 ip->hdr_checksum = ip_cksum; 1364 tcp->src_port = data->port; 1365 tcp->cksum = tcp_cksum; 1366 } else { 1367 struct rte_udp_hdr *udp = (struct rte_udp_hdr *) &ip[1]; 1368 uint16_t ip_cksum, udp_cksum; 1369 1370 ip_cksum = nat_ipv4_checksum_update(ip->hdr_checksum, 1371 ip->src_addr, 1372 data->addr); 1373 1374 udp_cksum = nat_ipv4_tcp_udp_checksum_update(udp->dgram_cksum, 1375 ip->src_addr, 1376 data->addr, 1377 udp->src_port, 1378 data->port); 1379 1380 ip->src_addr = data->addr; 1381 ip->hdr_checksum = ip_cksum; 1382 udp->src_port = data->port; 1383 if (udp->dgram_cksum) 1384 udp->dgram_cksum = udp_cksum; 1385 } 1386 } else { 1387 if (cfg->proto == 0x6) { 1388 struct rte_tcp_hdr *tcp = (struct rte_tcp_hdr *) &ip[1]; 1389 uint16_t ip_cksum, tcp_cksum; 1390 1391 ip_cksum = nat_ipv4_checksum_update(ip->hdr_checksum, 1392 ip->dst_addr, 1393 data->addr); 1394 1395 tcp_cksum = nat_ipv4_tcp_udp_checksum_update(tcp->cksum, 1396 ip->dst_addr, 1397 data->addr, 1398 tcp->dst_port, 1399 data->port); 1400 1401 ip->dst_addr = data->addr; 1402 ip->hdr_checksum = ip_cksum; 1403 tcp->dst_port = data->port; 1404 tcp->cksum = tcp_cksum; 1405 } else { 1406 struct rte_udp_hdr *udp = (struct rte_udp_hdr *) &ip[1]; 1407 uint16_t ip_cksum, udp_cksum; 1408 1409 ip_cksum = nat_ipv4_checksum_update(ip->hdr_checksum, 1410 ip->dst_addr, 1411 data->addr); 1412 1413 udp_cksum = nat_ipv4_tcp_udp_checksum_update(udp->dgram_cksum, 1414 ip->dst_addr, 1415 data->addr, 1416 udp->dst_port, 1417 data->port); 1418 1419 ip->dst_addr = data->addr; 1420 ip->hdr_checksum = ip_cksum; 1421 udp->dst_port = data->port; 1422 if (udp->dgram_cksum) 1423 udp->dgram_cksum = udp_cksum; 1424 } 1425 } 1426 } 1427 1428 static __rte_always_inline void 1429 pkt_ipv6_work_nat(struct rte_ipv6_hdr *ip, 1430 struct nat_ipv6_data *data, 1431 struct rte_table_action_nat_config *cfg) 1432 { 1433 if (cfg->source_nat) { 1434 if (cfg->proto == 0x6) { 1435 struct rte_tcp_hdr *tcp = (struct rte_tcp_hdr *) &ip[1]; 1436 uint16_t tcp_cksum; 1437 1438 tcp_cksum = nat_ipv6_tcp_udp_checksum_update(tcp->cksum, 1439 (uint16_t *)ip->src_addr, 1440 (uint16_t *)data->addr, 1441 tcp->src_port, 1442 data->port); 1443 1444 rte_memcpy(ip->src_addr, data->addr, 16); 1445 tcp->src_port = data->port; 1446 tcp->cksum = tcp_cksum; 1447 } else { 1448 struct rte_udp_hdr *udp = (struct rte_udp_hdr *) &ip[1]; 1449 uint16_t udp_cksum; 1450 1451 udp_cksum = nat_ipv6_tcp_udp_checksum_update(udp->dgram_cksum, 1452 (uint16_t *)ip->src_addr, 1453 (uint16_t *)data->addr, 1454 udp->src_port, 1455 data->port); 1456 1457 rte_memcpy(ip->src_addr, data->addr, 16); 1458 udp->src_port = data->port; 1459 udp->dgram_cksum = udp_cksum; 1460 } 1461 } else { 1462 if (cfg->proto == 0x6) { 1463 struct rte_tcp_hdr *tcp = (struct rte_tcp_hdr *) &ip[1]; 1464 uint16_t tcp_cksum; 1465 1466 tcp_cksum = nat_ipv6_tcp_udp_checksum_update(tcp->cksum, 1467 (uint16_t *)ip->dst_addr, 1468 (uint16_t *)data->addr, 1469 tcp->dst_port, 1470 data->port); 1471 1472 rte_memcpy(ip->dst_addr, data->addr, 16); 1473 tcp->dst_port = data->port; 1474 tcp->cksum = tcp_cksum; 1475 } else { 1476 struct rte_udp_hdr *udp = (struct rte_udp_hdr *) &ip[1]; 1477 uint16_t udp_cksum; 1478 1479 udp_cksum = nat_ipv6_tcp_udp_checksum_update(udp->dgram_cksum, 1480 (uint16_t *)ip->dst_addr, 1481 (uint16_t *)data->addr, 1482 udp->dst_port, 1483 data->port); 1484 1485 rte_memcpy(ip->dst_addr, data->addr, 16); 1486 udp->dst_port = data->port; 1487 udp->dgram_cksum = udp_cksum; 1488 } 1489 } 1490 } 1491 1492 /** 1493 * RTE_TABLE_ACTION_TTL 1494 */ 1495 static int 1496 ttl_cfg_check(struct rte_table_action_ttl_config *ttl) 1497 { 1498 if (ttl->drop == 0) 1499 return -ENOTSUP; 1500 1501 return 0; 1502 } 1503 1504 struct ttl_data { 1505 uint32_t n_packets; 1506 } __rte_packed; 1507 1508 #define TTL_INIT(data, decrement) \ 1509 ((data)->n_packets = (decrement) ? 1 : 0) 1510 1511 #define TTL_DEC_GET(data) \ 1512 ((uint8_t)((data)->n_packets & 1)) 1513 1514 #define TTL_STATS_RESET(data) \ 1515 ((data)->n_packets = ((data)->n_packets & 1)) 1516 1517 #define TTL_STATS_READ(data) \ 1518 ((data)->n_packets >> 1) 1519 1520 #define TTL_STATS_ADD(data, value) \ 1521 ((data)->n_packets = \ 1522 (((((data)->n_packets >> 1) + (value)) << 1) | \ 1523 ((data)->n_packets & 1))) 1524 1525 static int 1526 ttl_apply(void *data, 1527 struct rte_table_action_ttl_params *p) 1528 { 1529 struct ttl_data *d = data; 1530 1531 TTL_INIT(d, p->decrement); 1532 1533 return 0; 1534 } 1535 1536 static __rte_always_inline uint64_t 1537 pkt_ipv4_work_ttl(struct rte_ipv4_hdr *ip, 1538 struct ttl_data *data) 1539 { 1540 uint32_t drop; 1541 uint16_t cksum = ip->hdr_checksum; 1542 uint8_t ttl = ip->time_to_live; 1543 uint8_t ttl_diff = TTL_DEC_GET(data); 1544 1545 cksum += ttl_diff; 1546 ttl -= ttl_diff; 1547 1548 ip->hdr_checksum = cksum; 1549 ip->time_to_live = ttl; 1550 1551 drop = (ttl == 0) ? 1 : 0; 1552 TTL_STATS_ADD(data, drop); 1553 1554 return drop; 1555 } 1556 1557 static __rte_always_inline uint64_t 1558 pkt_ipv6_work_ttl(struct rte_ipv6_hdr *ip, 1559 struct ttl_data *data) 1560 { 1561 uint32_t drop; 1562 uint8_t ttl = ip->hop_limits; 1563 uint8_t ttl_diff = TTL_DEC_GET(data); 1564 1565 ttl -= ttl_diff; 1566 1567 ip->hop_limits = ttl; 1568 1569 drop = (ttl == 0) ? 1 : 0; 1570 TTL_STATS_ADD(data, drop); 1571 1572 return drop; 1573 } 1574 1575 /** 1576 * RTE_TABLE_ACTION_STATS 1577 */ 1578 static int 1579 stats_cfg_check(struct rte_table_action_stats_config *stats) 1580 { 1581 if ((stats->n_packets_enabled == 0) && (stats->n_bytes_enabled == 0)) 1582 return -EINVAL; 1583 1584 return 0; 1585 } 1586 1587 struct stats_data { 1588 uint64_t n_packets; 1589 uint64_t n_bytes; 1590 } __rte_packed; 1591 1592 static int 1593 stats_apply(struct stats_data *data, 1594 struct rte_table_action_stats_params *p) 1595 { 1596 data->n_packets = p->n_packets; 1597 data->n_bytes = p->n_bytes; 1598 1599 return 0; 1600 } 1601 1602 static __rte_always_inline void 1603 pkt_work_stats(struct stats_data *data, 1604 uint16_t total_length) 1605 { 1606 data->n_packets++; 1607 data->n_bytes += total_length; 1608 } 1609 1610 /** 1611 * RTE_TABLE_ACTION_TIME 1612 */ 1613 struct time_data { 1614 uint64_t time; 1615 } __rte_packed; 1616 1617 static int 1618 time_apply(struct time_data *data, 1619 struct rte_table_action_time_params *p) 1620 { 1621 data->time = p->time; 1622 return 0; 1623 } 1624 1625 static __rte_always_inline void 1626 pkt_work_time(struct time_data *data, 1627 uint64_t time) 1628 { 1629 data->time = time; 1630 } 1631 1632 1633 /** 1634 * RTE_TABLE_ACTION_CRYPTO 1635 */ 1636 1637 #define CRYPTO_OP_MASK_CIPHER 0x1 1638 #define CRYPTO_OP_MASK_AUTH 0x2 1639 #define CRYPTO_OP_MASK_AEAD 0x4 1640 1641 struct crypto_op_sym_iv_aad { 1642 struct rte_crypto_op op; 1643 struct rte_crypto_sym_op sym_op; 1644 union { 1645 struct { 1646 uint8_t cipher_iv[ 1647 RTE_TABLE_ACTION_SYM_CRYPTO_IV_SIZE_MAX]; 1648 uint8_t auth_iv[ 1649 RTE_TABLE_ACTION_SYM_CRYPTO_IV_SIZE_MAX]; 1650 } cipher_auth; 1651 1652 struct { 1653 uint8_t iv[RTE_TABLE_ACTION_SYM_CRYPTO_IV_SIZE_MAX]; 1654 uint8_t aad[RTE_TABLE_ACTION_SYM_CRYPTO_AAD_SIZE_MAX]; 1655 } aead_iv_aad; 1656 1657 } iv_aad; 1658 }; 1659 1660 struct sym_crypto_data { 1661 1662 union { 1663 struct { 1664 1665 /** Length of cipher iv. */ 1666 uint16_t cipher_iv_len; 1667 1668 /** Offset from start of IP header to the cipher iv. */ 1669 uint16_t cipher_iv_data_offset; 1670 1671 /** Length of cipher iv to be updated in the mbuf. */ 1672 uint16_t cipher_iv_update_len; 1673 1674 /** Offset from start of IP header to the auth iv. */ 1675 uint16_t auth_iv_data_offset; 1676 1677 /** Length of auth iv in the mbuf. */ 1678 uint16_t auth_iv_len; 1679 1680 /** Length of auth iv to be updated in the mbuf. */ 1681 uint16_t auth_iv_update_len; 1682 1683 } cipher_auth; 1684 struct { 1685 1686 /** Length of iv. */ 1687 uint16_t iv_len; 1688 1689 /** Offset from start of IP header to the aead iv. */ 1690 uint16_t iv_data_offset; 1691 1692 /** Length of iv to be updated in the mbuf. */ 1693 uint16_t iv_update_len; 1694 1695 /** Length of aad */ 1696 uint16_t aad_len; 1697 1698 /** Offset from start of IP header to the aad. */ 1699 uint16_t aad_data_offset; 1700 1701 /** Length of aad to updated in the mbuf. */ 1702 uint16_t aad_update_len; 1703 1704 } aead; 1705 }; 1706 1707 /** Offset from start of IP header to the data. */ 1708 uint16_t data_offset; 1709 1710 /** Digest length. */ 1711 uint16_t digest_len; 1712 1713 /** block size */ 1714 uint16_t block_size; 1715 1716 /** Mask of crypto operation */ 1717 uint16_t op_mask; 1718 1719 /** Session pointer. */ 1720 struct rte_cryptodev_sym_session *session; 1721 1722 /** Direction of crypto, encrypt or decrypt */ 1723 uint16_t direction; 1724 1725 /** Private data size to store cipher iv / aad. */ 1726 uint8_t iv_aad_data[32]; 1727 1728 } __rte_packed; 1729 1730 static int 1731 sym_crypto_cfg_check(struct rte_table_action_sym_crypto_config *cfg) 1732 { 1733 if (!rte_cryptodev_is_valid_dev(cfg->cryptodev_id)) 1734 return -EINVAL; 1735 if (cfg->mp_create == NULL || cfg->mp_init == NULL) 1736 return -EINVAL; 1737 1738 return 0; 1739 } 1740 1741 static int 1742 get_block_size(const struct rte_crypto_sym_xform *xform, uint8_t cdev_id) 1743 { 1744 struct rte_cryptodev_info dev_info; 1745 const struct rte_cryptodev_capabilities *cap; 1746 uint32_t i; 1747 1748 rte_cryptodev_info_get(cdev_id, &dev_info); 1749 1750 for (i = 0; dev_info.capabilities[i].op != RTE_CRYPTO_OP_TYPE_UNDEFINED; 1751 i++) { 1752 cap = &dev_info.capabilities[i]; 1753 1754 if (cap->sym.xform_type != xform->type) 1755 continue; 1756 1757 if ((xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) && 1758 (cap->sym.cipher.algo == xform->cipher.algo)) 1759 return cap->sym.cipher.block_size; 1760 1761 if ((xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) && 1762 (cap->sym.aead.algo == xform->aead.algo)) 1763 return cap->sym.aead.block_size; 1764 1765 if (xform->type == RTE_CRYPTO_SYM_XFORM_NOT_SPECIFIED) 1766 break; 1767 } 1768 1769 return -1; 1770 } 1771 1772 static int 1773 sym_crypto_apply(struct sym_crypto_data *data, 1774 struct rte_table_action_sym_crypto_config *cfg, 1775 struct rte_table_action_sym_crypto_params *p) 1776 { 1777 const struct rte_crypto_cipher_xform *cipher_xform = NULL; 1778 const struct rte_crypto_auth_xform *auth_xform = NULL; 1779 const struct rte_crypto_aead_xform *aead_xform = NULL; 1780 struct rte_crypto_sym_xform *xform = p->xform; 1781 struct rte_cryptodev_sym_session *session; 1782 int ret; 1783 1784 memset(data, 0, sizeof(*data)); 1785 1786 while (xform) { 1787 if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) { 1788 cipher_xform = &xform->cipher; 1789 1790 if (cipher_xform->iv.length > 1791 RTE_TABLE_ACTION_SYM_CRYPTO_IV_SIZE_MAX) 1792 return -ENOMEM; 1793 if (cipher_xform->iv.offset != 1794 RTE_TABLE_ACTION_SYM_CRYPTO_IV_OFFSET) 1795 return -EINVAL; 1796 1797 ret = get_block_size(xform, cfg->cryptodev_id); 1798 if (ret < 0) 1799 return -1; 1800 data->block_size = (uint16_t)ret; 1801 data->op_mask |= CRYPTO_OP_MASK_CIPHER; 1802 1803 data->cipher_auth.cipher_iv_len = 1804 cipher_xform->iv.length; 1805 data->cipher_auth.cipher_iv_data_offset = (uint16_t) 1806 p->cipher_auth.cipher_iv_update.offset; 1807 data->cipher_auth.cipher_iv_update_len = (uint16_t) 1808 p->cipher_auth.cipher_iv_update.length; 1809 1810 rte_memcpy(data->iv_aad_data, 1811 p->cipher_auth.cipher_iv.val, 1812 p->cipher_auth.cipher_iv.length); 1813 1814 data->direction = cipher_xform->op; 1815 1816 } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) { 1817 auth_xform = &xform->auth; 1818 if (auth_xform->iv.length > 1819 RTE_TABLE_ACTION_SYM_CRYPTO_IV_SIZE_MAX) 1820 return -ENOMEM; 1821 data->op_mask |= CRYPTO_OP_MASK_AUTH; 1822 1823 data->cipher_auth.auth_iv_len = auth_xform->iv.length; 1824 data->cipher_auth.auth_iv_data_offset = (uint16_t) 1825 p->cipher_auth.auth_iv_update.offset; 1826 data->cipher_auth.auth_iv_update_len = (uint16_t) 1827 p->cipher_auth.auth_iv_update.length; 1828 data->digest_len = auth_xform->digest_length; 1829 1830 data->direction = (auth_xform->op == 1831 RTE_CRYPTO_AUTH_OP_GENERATE) ? 1832 RTE_CRYPTO_CIPHER_OP_ENCRYPT : 1833 RTE_CRYPTO_CIPHER_OP_DECRYPT; 1834 1835 } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) { 1836 aead_xform = &xform->aead; 1837 1838 if ((aead_xform->iv.length > 1839 RTE_TABLE_ACTION_SYM_CRYPTO_IV_SIZE_MAX) || ( 1840 aead_xform->aad_length > 1841 RTE_TABLE_ACTION_SYM_CRYPTO_AAD_SIZE_MAX)) 1842 return -EINVAL; 1843 if (aead_xform->iv.offset != 1844 RTE_TABLE_ACTION_SYM_CRYPTO_IV_OFFSET) 1845 return -EINVAL; 1846 1847 ret = get_block_size(xform, cfg->cryptodev_id); 1848 if (ret < 0) 1849 return -1; 1850 data->block_size = (uint16_t)ret; 1851 data->op_mask |= CRYPTO_OP_MASK_AEAD; 1852 1853 data->digest_len = aead_xform->digest_length; 1854 data->aead.iv_len = aead_xform->iv.length; 1855 data->aead.aad_len = aead_xform->aad_length; 1856 1857 data->aead.iv_data_offset = (uint16_t) 1858 p->aead.iv_update.offset; 1859 data->aead.iv_update_len = (uint16_t) 1860 p->aead.iv_update.length; 1861 data->aead.aad_data_offset = (uint16_t) 1862 p->aead.aad_update.offset; 1863 data->aead.aad_update_len = (uint16_t) 1864 p->aead.aad_update.length; 1865 1866 rte_memcpy(data->iv_aad_data, 1867 p->aead.iv.val, 1868 p->aead.iv.length); 1869 1870 rte_memcpy(data->iv_aad_data + p->aead.iv.length, 1871 p->aead.aad.val, 1872 p->aead.aad.length); 1873 1874 data->direction = (aead_xform->op == 1875 RTE_CRYPTO_AEAD_OP_ENCRYPT) ? 1876 RTE_CRYPTO_CIPHER_OP_ENCRYPT : 1877 RTE_CRYPTO_CIPHER_OP_DECRYPT; 1878 } else 1879 return -EINVAL; 1880 1881 xform = xform->next; 1882 } 1883 1884 if (auth_xform && auth_xform->iv.length) { 1885 if (cipher_xform) { 1886 if (auth_xform->iv.offset != 1887 RTE_TABLE_ACTION_SYM_CRYPTO_IV_OFFSET + 1888 cipher_xform->iv.length) 1889 return -EINVAL; 1890 1891 rte_memcpy(data->iv_aad_data + cipher_xform->iv.length, 1892 p->cipher_auth.auth_iv.val, 1893 p->cipher_auth.auth_iv.length); 1894 } else { 1895 rte_memcpy(data->iv_aad_data, 1896 p->cipher_auth.auth_iv.val, 1897 p->cipher_auth.auth_iv.length); 1898 } 1899 } 1900 1901 session = rte_cryptodev_sym_session_create(cfg->mp_create); 1902 if (!session) 1903 return -ENOMEM; 1904 1905 ret = rte_cryptodev_sym_session_init(cfg->cryptodev_id, session, 1906 p->xform, cfg->mp_init); 1907 if (ret < 0) { 1908 rte_cryptodev_sym_session_free(session); 1909 return ret; 1910 } 1911 1912 data->data_offset = (uint16_t)p->data_offset; 1913 data->session = session; 1914 1915 return 0; 1916 } 1917 1918 static __rte_always_inline uint64_t 1919 pkt_work_sym_crypto(struct rte_mbuf *mbuf, struct sym_crypto_data *data, 1920 struct rte_table_action_sym_crypto_config *cfg, 1921 uint16_t ip_offset) 1922 { 1923 struct crypto_op_sym_iv_aad *crypto_op = (struct crypto_op_sym_iv_aad *) 1924 RTE_MBUF_METADATA_UINT8_PTR(mbuf, cfg->op_offset); 1925 struct rte_crypto_op *op = &crypto_op->op; 1926 struct rte_crypto_sym_op *sym = op->sym; 1927 uint32_t pkt_offset = sizeof(*mbuf) + mbuf->data_off; 1928 uint32_t payload_len = pkt_offset + mbuf->data_len - data->data_offset; 1929 1930 op->type = RTE_CRYPTO_OP_TYPE_SYMMETRIC; 1931 op->sess_type = RTE_CRYPTO_OP_WITH_SESSION; 1932 op->phys_addr = mbuf->buf_iova + cfg->op_offset - sizeof(*mbuf); 1933 op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 1934 sym->m_src = mbuf; 1935 sym->m_dst = NULL; 1936 sym->session = data->session; 1937 1938 /** pad the packet */ 1939 if (data->direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { 1940 uint32_t append_len = RTE_ALIGN_CEIL(payload_len, 1941 data->block_size) - payload_len; 1942 1943 if (unlikely(rte_pktmbuf_append(mbuf, append_len + 1944 data->digest_len) == NULL)) 1945 return 1; 1946 1947 payload_len += append_len; 1948 } else 1949 payload_len -= data->digest_len; 1950 1951 if (data->op_mask & CRYPTO_OP_MASK_CIPHER) { 1952 /** prepare cipher op */ 1953 uint8_t *iv = crypto_op->iv_aad.cipher_auth.cipher_iv; 1954 1955 sym->cipher.data.length = payload_len; 1956 sym->cipher.data.offset = data->data_offset - pkt_offset; 1957 1958 if (data->cipher_auth.cipher_iv_update_len) { 1959 uint8_t *pkt_iv = RTE_MBUF_METADATA_UINT8_PTR(mbuf, 1960 data->cipher_auth.cipher_iv_data_offset 1961 + ip_offset); 1962 1963 /** For encryption, update the pkt iv field, otherwise 1964 * update the iv_aad_field 1965 **/ 1966 if (data->direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) 1967 rte_memcpy(pkt_iv, data->iv_aad_data, 1968 data->cipher_auth.cipher_iv_update_len); 1969 else 1970 rte_memcpy(data->iv_aad_data, pkt_iv, 1971 data->cipher_auth.cipher_iv_update_len); 1972 } 1973 1974 /** write iv */ 1975 rte_memcpy(iv, data->iv_aad_data, 1976 data->cipher_auth.cipher_iv_len); 1977 } 1978 1979 if (data->op_mask & CRYPTO_OP_MASK_AUTH) { 1980 /** authentication always start from IP header. */ 1981 sym->auth.data.offset = ip_offset - pkt_offset; 1982 sym->auth.data.length = mbuf->data_len - sym->auth.data.offset - 1983 data->digest_len; 1984 sym->auth.digest.data = rte_pktmbuf_mtod_offset(mbuf, 1985 uint8_t *, rte_pktmbuf_pkt_len(mbuf) - 1986 data->digest_len); 1987 sym->auth.digest.phys_addr = rte_pktmbuf_iova_offset(mbuf, 1988 rte_pktmbuf_pkt_len(mbuf) - data->digest_len); 1989 1990 if (data->cipher_auth.auth_iv_update_len) { 1991 uint8_t *pkt_iv = RTE_MBUF_METADATA_UINT8_PTR(mbuf, 1992 data->cipher_auth.auth_iv_data_offset 1993 + ip_offset); 1994 uint8_t *data_iv = data->iv_aad_data + 1995 data->cipher_auth.cipher_iv_len; 1996 1997 if (data->direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) 1998 rte_memcpy(pkt_iv, data_iv, 1999 data->cipher_auth.auth_iv_update_len); 2000 else 2001 rte_memcpy(data_iv, pkt_iv, 2002 data->cipher_auth.auth_iv_update_len); 2003 } 2004 2005 if (data->cipher_auth.auth_iv_len) { 2006 /** prepare cipher op */ 2007 uint8_t *iv = crypto_op->iv_aad.cipher_auth.auth_iv; 2008 2009 rte_memcpy(iv, data->iv_aad_data + 2010 data->cipher_auth.cipher_iv_len, 2011 data->cipher_auth.auth_iv_len); 2012 } 2013 } 2014 2015 if (data->op_mask & CRYPTO_OP_MASK_AEAD) { 2016 uint8_t *iv = crypto_op->iv_aad.aead_iv_aad.iv; 2017 uint8_t *aad = crypto_op->iv_aad.aead_iv_aad.aad; 2018 2019 sym->aead.aad.data = aad; 2020 sym->aead.aad.phys_addr = rte_pktmbuf_iova_offset(mbuf, 2021 aad - rte_pktmbuf_mtod(mbuf, uint8_t *)); 2022 sym->aead.digest.data = rte_pktmbuf_mtod_offset(mbuf, 2023 uint8_t *, rte_pktmbuf_pkt_len(mbuf) - 2024 data->digest_len); 2025 sym->aead.digest.phys_addr = rte_pktmbuf_iova_offset(mbuf, 2026 rte_pktmbuf_pkt_len(mbuf) - data->digest_len); 2027 sym->aead.data.offset = data->data_offset - pkt_offset; 2028 sym->aead.data.length = payload_len; 2029 2030 if (data->aead.iv_update_len) { 2031 uint8_t *pkt_iv = RTE_MBUF_METADATA_UINT8_PTR(mbuf, 2032 data->aead.iv_data_offset + ip_offset); 2033 uint8_t *data_iv = data->iv_aad_data; 2034 2035 if (data->direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) 2036 rte_memcpy(pkt_iv, data_iv, 2037 data->aead.iv_update_len); 2038 else 2039 rte_memcpy(data_iv, pkt_iv, 2040 data->aead.iv_update_len); 2041 } 2042 2043 rte_memcpy(iv, data->iv_aad_data, data->aead.iv_len); 2044 2045 if (data->aead.aad_update_len) { 2046 uint8_t *pkt_aad = RTE_MBUF_METADATA_UINT8_PTR(mbuf, 2047 data->aead.aad_data_offset + ip_offset); 2048 uint8_t *data_aad = data->iv_aad_data + 2049 data->aead.iv_len; 2050 2051 if (data->direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) 2052 rte_memcpy(pkt_aad, data_aad, 2053 data->aead.iv_update_len); 2054 else 2055 rte_memcpy(data_aad, pkt_aad, 2056 data->aead.iv_update_len); 2057 } 2058 2059 rte_memcpy(aad, data->iv_aad_data + data->aead.iv_len, 2060 data->aead.aad_len); 2061 } 2062 2063 return 0; 2064 } 2065 2066 /** 2067 * RTE_TABLE_ACTION_TAG 2068 */ 2069 struct tag_data { 2070 uint32_t tag; 2071 } __rte_packed; 2072 2073 static int 2074 tag_apply(struct tag_data *data, 2075 struct rte_table_action_tag_params *p) 2076 { 2077 data->tag = p->tag; 2078 return 0; 2079 } 2080 2081 static __rte_always_inline void 2082 pkt_work_tag(struct rte_mbuf *mbuf, 2083 struct tag_data *data) 2084 { 2085 mbuf->hash.fdir.hi = data->tag; 2086 mbuf->ol_flags |= RTE_MBUF_F_RX_FDIR | RTE_MBUF_F_RX_FDIR_ID; 2087 } 2088 2089 static __rte_always_inline void 2090 pkt4_work_tag(struct rte_mbuf *mbuf0, 2091 struct rte_mbuf *mbuf1, 2092 struct rte_mbuf *mbuf2, 2093 struct rte_mbuf *mbuf3, 2094 struct tag_data *data0, 2095 struct tag_data *data1, 2096 struct tag_data *data2, 2097 struct tag_data *data3) 2098 { 2099 mbuf0->hash.fdir.hi = data0->tag; 2100 mbuf1->hash.fdir.hi = data1->tag; 2101 mbuf2->hash.fdir.hi = data2->tag; 2102 mbuf3->hash.fdir.hi = data3->tag; 2103 2104 mbuf0->ol_flags |= RTE_MBUF_F_RX_FDIR | RTE_MBUF_F_RX_FDIR_ID; 2105 mbuf1->ol_flags |= RTE_MBUF_F_RX_FDIR | RTE_MBUF_F_RX_FDIR_ID; 2106 mbuf2->ol_flags |= RTE_MBUF_F_RX_FDIR | RTE_MBUF_F_RX_FDIR_ID; 2107 mbuf3->ol_flags |= RTE_MBUF_F_RX_FDIR | RTE_MBUF_F_RX_FDIR_ID; 2108 } 2109 2110 /** 2111 * RTE_TABLE_ACTION_DECAP 2112 */ 2113 struct decap_data { 2114 uint16_t n; 2115 } __rte_packed; 2116 2117 static int 2118 decap_apply(struct decap_data *data, 2119 struct rte_table_action_decap_params *p) 2120 { 2121 data->n = p->n; 2122 return 0; 2123 } 2124 2125 static __rte_always_inline void 2126 pkt_work_decap(struct rte_mbuf *mbuf, 2127 struct decap_data *data) 2128 { 2129 uint16_t data_off = mbuf->data_off; 2130 uint16_t data_len = mbuf->data_len; 2131 uint32_t pkt_len = mbuf->pkt_len; 2132 uint16_t n = data->n; 2133 2134 mbuf->data_off = data_off + n; 2135 mbuf->data_len = data_len - n; 2136 mbuf->pkt_len = pkt_len - n; 2137 } 2138 2139 static __rte_always_inline void 2140 pkt4_work_decap(struct rte_mbuf *mbuf0, 2141 struct rte_mbuf *mbuf1, 2142 struct rte_mbuf *mbuf2, 2143 struct rte_mbuf *mbuf3, 2144 struct decap_data *data0, 2145 struct decap_data *data1, 2146 struct decap_data *data2, 2147 struct decap_data *data3) 2148 { 2149 uint16_t data_off0 = mbuf0->data_off; 2150 uint16_t data_len0 = mbuf0->data_len; 2151 uint32_t pkt_len0 = mbuf0->pkt_len; 2152 2153 uint16_t data_off1 = mbuf1->data_off; 2154 uint16_t data_len1 = mbuf1->data_len; 2155 uint32_t pkt_len1 = mbuf1->pkt_len; 2156 2157 uint16_t data_off2 = mbuf2->data_off; 2158 uint16_t data_len2 = mbuf2->data_len; 2159 uint32_t pkt_len2 = mbuf2->pkt_len; 2160 2161 uint16_t data_off3 = mbuf3->data_off; 2162 uint16_t data_len3 = mbuf3->data_len; 2163 uint32_t pkt_len3 = mbuf3->pkt_len; 2164 2165 uint16_t n0 = data0->n; 2166 uint16_t n1 = data1->n; 2167 uint16_t n2 = data2->n; 2168 uint16_t n3 = data3->n; 2169 2170 mbuf0->data_off = data_off0 + n0; 2171 mbuf0->data_len = data_len0 - n0; 2172 mbuf0->pkt_len = pkt_len0 - n0; 2173 2174 mbuf1->data_off = data_off1 + n1; 2175 mbuf1->data_len = data_len1 - n1; 2176 mbuf1->pkt_len = pkt_len1 - n1; 2177 2178 mbuf2->data_off = data_off2 + n2; 2179 mbuf2->data_len = data_len2 - n2; 2180 mbuf2->pkt_len = pkt_len2 - n2; 2181 2182 mbuf3->data_off = data_off3 + n3; 2183 mbuf3->data_len = data_len3 - n3; 2184 mbuf3->pkt_len = pkt_len3 - n3; 2185 } 2186 2187 /** 2188 * Action profile 2189 */ 2190 static int 2191 action_valid(enum rte_table_action_type action) 2192 { 2193 switch (action) { 2194 case RTE_TABLE_ACTION_FWD: 2195 case RTE_TABLE_ACTION_LB: 2196 case RTE_TABLE_ACTION_MTR: 2197 case RTE_TABLE_ACTION_TM: 2198 case RTE_TABLE_ACTION_ENCAP: 2199 case RTE_TABLE_ACTION_NAT: 2200 case RTE_TABLE_ACTION_TTL: 2201 case RTE_TABLE_ACTION_STATS: 2202 case RTE_TABLE_ACTION_TIME: 2203 case RTE_TABLE_ACTION_SYM_CRYPTO: 2204 case RTE_TABLE_ACTION_TAG: 2205 case RTE_TABLE_ACTION_DECAP: 2206 return 1; 2207 default: 2208 return 0; 2209 } 2210 } 2211 2212 2213 #define RTE_TABLE_ACTION_MAX 64 2214 2215 struct ap_config { 2216 uint64_t action_mask; 2217 struct rte_table_action_common_config common; 2218 struct rte_table_action_lb_config lb; 2219 struct rte_table_action_mtr_config mtr; 2220 struct rte_table_action_tm_config tm; 2221 struct rte_table_action_encap_config encap; 2222 struct rte_table_action_nat_config nat; 2223 struct rte_table_action_ttl_config ttl; 2224 struct rte_table_action_stats_config stats; 2225 struct rte_table_action_sym_crypto_config sym_crypto; 2226 }; 2227 2228 static size_t 2229 action_cfg_size(enum rte_table_action_type action) 2230 { 2231 switch (action) { 2232 case RTE_TABLE_ACTION_LB: 2233 return sizeof(struct rte_table_action_lb_config); 2234 case RTE_TABLE_ACTION_MTR: 2235 return sizeof(struct rte_table_action_mtr_config); 2236 case RTE_TABLE_ACTION_TM: 2237 return sizeof(struct rte_table_action_tm_config); 2238 case RTE_TABLE_ACTION_ENCAP: 2239 return sizeof(struct rte_table_action_encap_config); 2240 case RTE_TABLE_ACTION_NAT: 2241 return sizeof(struct rte_table_action_nat_config); 2242 case RTE_TABLE_ACTION_TTL: 2243 return sizeof(struct rte_table_action_ttl_config); 2244 case RTE_TABLE_ACTION_STATS: 2245 return sizeof(struct rte_table_action_stats_config); 2246 case RTE_TABLE_ACTION_SYM_CRYPTO: 2247 return sizeof(struct rte_table_action_sym_crypto_config); 2248 default: 2249 return 0; 2250 } 2251 } 2252 2253 static void* 2254 action_cfg_get(struct ap_config *ap_config, 2255 enum rte_table_action_type type) 2256 { 2257 switch (type) { 2258 case RTE_TABLE_ACTION_LB: 2259 return &ap_config->lb; 2260 2261 case RTE_TABLE_ACTION_MTR: 2262 return &ap_config->mtr; 2263 2264 case RTE_TABLE_ACTION_TM: 2265 return &ap_config->tm; 2266 2267 case RTE_TABLE_ACTION_ENCAP: 2268 return &ap_config->encap; 2269 2270 case RTE_TABLE_ACTION_NAT: 2271 return &ap_config->nat; 2272 2273 case RTE_TABLE_ACTION_TTL: 2274 return &ap_config->ttl; 2275 2276 case RTE_TABLE_ACTION_STATS: 2277 return &ap_config->stats; 2278 2279 case RTE_TABLE_ACTION_SYM_CRYPTO: 2280 return &ap_config->sym_crypto; 2281 default: 2282 return NULL; 2283 } 2284 } 2285 2286 static void 2287 action_cfg_set(struct ap_config *ap_config, 2288 enum rte_table_action_type type, 2289 void *action_cfg) 2290 { 2291 void *dst = action_cfg_get(ap_config, type); 2292 2293 if (dst) 2294 memcpy(dst, action_cfg, action_cfg_size(type)); 2295 2296 ap_config->action_mask |= 1LLU << type; 2297 } 2298 2299 struct ap_data { 2300 size_t offset[RTE_TABLE_ACTION_MAX]; 2301 size_t total_size; 2302 }; 2303 2304 static size_t 2305 action_data_size(enum rte_table_action_type action, 2306 struct ap_config *ap_config) 2307 { 2308 switch (action) { 2309 case RTE_TABLE_ACTION_FWD: 2310 return sizeof(struct fwd_data); 2311 2312 case RTE_TABLE_ACTION_LB: 2313 return sizeof(struct lb_data); 2314 2315 case RTE_TABLE_ACTION_MTR: 2316 return mtr_data_size(&ap_config->mtr); 2317 2318 case RTE_TABLE_ACTION_TM: 2319 return sizeof(struct tm_data); 2320 2321 case RTE_TABLE_ACTION_ENCAP: 2322 return encap_data_size(&ap_config->encap); 2323 2324 case RTE_TABLE_ACTION_NAT: 2325 return nat_data_size(&ap_config->nat, 2326 &ap_config->common); 2327 2328 case RTE_TABLE_ACTION_TTL: 2329 return sizeof(struct ttl_data); 2330 2331 case RTE_TABLE_ACTION_STATS: 2332 return sizeof(struct stats_data); 2333 2334 case RTE_TABLE_ACTION_TIME: 2335 return sizeof(struct time_data); 2336 2337 case RTE_TABLE_ACTION_SYM_CRYPTO: 2338 return (sizeof(struct sym_crypto_data)); 2339 2340 case RTE_TABLE_ACTION_TAG: 2341 return sizeof(struct tag_data); 2342 2343 case RTE_TABLE_ACTION_DECAP: 2344 return sizeof(struct decap_data); 2345 2346 default: 2347 return 0; 2348 } 2349 } 2350 2351 2352 static void 2353 action_data_offset_set(struct ap_data *ap_data, 2354 struct ap_config *ap_config) 2355 { 2356 uint64_t action_mask = ap_config->action_mask; 2357 size_t offset; 2358 uint32_t action; 2359 2360 memset(ap_data->offset, 0, sizeof(ap_data->offset)); 2361 2362 offset = 0; 2363 for (action = 0; action < RTE_TABLE_ACTION_MAX; action++) 2364 if (action_mask & (1LLU << action)) { 2365 ap_data->offset[action] = offset; 2366 offset += action_data_size((enum rte_table_action_type)action, 2367 ap_config); 2368 } 2369 2370 ap_data->total_size = offset; 2371 } 2372 2373 struct rte_table_action_profile { 2374 struct ap_config cfg; 2375 struct ap_data data; 2376 int frozen; 2377 }; 2378 2379 struct rte_table_action_profile * 2380 rte_table_action_profile_create(struct rte_table_action_common_config *common) 2381 { 2382 struct rte_table_action_profile *ap; 2383 2384 /* Check input arguments */ 2385 if (common == NULL) 2386 return NULL; 2387 2388 /* Memory allocation */ 2389 ap = calloc(1, sizeof(struct rte_table_action_profile)); 2390 if (ap == NULL) 2391 return NULL; 2392 2393 /* Initialization */ 2394 memcpy(&ap->cfg.common, common, sizeof(*common)); 2395 2396 return ap; 2397 } 2398 2399 2400 int 2401 rte_table_action_profile_action_register(struct rte_table_action_profile *profile, 2402 enum rte_table_action_type type, 2403 void *action_config) 2404 { 2405 int status; 2406 2407 /* Check input arguments */ 2408 if ((profile == NULL) || 2409 profile->frozen || 2410 (action_valid(type) == 0) || 2411 (profile->cfg.action_mask & (1LLU << type)) || 2412 ((action_cfg_size(type) == 0) && action_config) || 2413 (action_cfg_size(type) && (action_config == NULL))) 2414 return -EINVAL; 2415 2416 switch (type) { 2417 case RTE_TABLE_ACTION_LB: 2418 status = lb_cfg_check(action_config); 2419 break; 2420 2421 case RTE_TABLE_ACTION_MTR: 2422 status = mtr_cfg_check(action_config); 2423 break; 2424 2425 case RTE_TABLE_ACTION_TM: 2426 status = tm_cfg_check(action_config); 2427 break; 2428 2429 case RTE_TABLE_ACTION_ENCAP: 2430 status = encap_cfg_check(action_config); 2431 break; 2432 2433 case RTE_TABLE_ACTION_NAT: 2434 status = nat_cfg_check(action_config); 2435 break; 2436 2437 case RTE_TABLE_ACTION_TTL: 2438 status = ttl_cfg_check(action_config); 2439 break; 2440 2441 case RTE_TABLE_ACTION_STATS: 2442 status = stats_cfg_check(action_config); 2443 break; 2444 2445 case RTE_TABLE_ACTION_SYM_CRYPTO: 2446 status = sym_crypto_cfg_check(action_config); 2447 break; 2448 2449 default: 2450 status = 0; 2451 break; 2452 } 2453 2454 if (status) 2455 return status; 2456 2457 /* Action enable */ 2458 action_cfg_set(&profile->cfg, type, action_config); 2459 2460 return 0; 2461 } 2462 2463 int 2464 rte_table_action_profile_freeze(struct rte_table_action_profile *profile) 2465 { 2466 if (profile->frozen) 2467 return -EBUSY; 2468 2469 profile->cfg.action_mask |= 1LLU << RTE_TABLE_ACTION_FWD; 2470 action_data_offset_set(&profile->data, &profile->cfg); 2471 profile->frozen = 1; 2472 2473 return 0; 2474 } 2475 2476 int 2477 rte_table_action_profile_free(struct rte_table_action_profile *profile) 2478 { 2479 if (profile == NULL) 2480 return 0; 2481 2482 free(profile); 2483 return 0; 2484 } 2485 2486 /** 2487 * Action 2488 */ 2489 #define METER_PROFILES_MAX 32 2490 2491 struct rte_table_action { 2492 struct ap_config cfg; 2493 struct ap_data data; 2494 struct dscp_table_data dscp_table; 2495 struct meter_profile_data mp[METER_PROFILES_MAX]; 2496 }; 2497 2498 struct rte_table_action * 2499 rte_table_action_create(struct rte_table_action_profile *profile, 2500 uint32_t socket_id) 2501 { 2502 struct rte_table_action *action; 2503 2504 /* Check input arguments */ 2505 if ((profile == NULL) || 2506 (profile->frozen == 0)) 2507 return NULL; 2508 2509 /* Memory allocation */ 2510 action = rte_zmalloc_socket(NULL, 2511 sizeof(struct rte_table_action), 2512 RTE_CACHE_LINE_SIZE, 2513 socket_id); 2514 if (action == NULL) 2515 return NULL; 2516 2517 /* Initialization */ 2518 memcpy(&action->cfg, &profile->cfg, sizeof(profile->cfg)); 2519 memcpy(&action->data, &profile->data, sizeof(profile->data)); 2520 2521 return action; 2522 } 2523 2524 static __rte_always_inline void * 2525 action_data_get(void *data, 2526 struct rte_table_action *action, 2527 enum rte_table_action_type type) 2528 { 2529 size_t offset = action->data.offset[type]; 2530 uint8_t *data_bytes = data; 2531 2532 return &data_bytes[offset]; 2533 } 2534 2535 int 2536 rte_table_action_apply(struct rte_table_action *action, 2537 void *data, 2538 enum rte_table_action_type type, 2539 void *action_params) 2540 { 2541 void *action_data; 2542 2543 /* Check input arguments */ 2544 if ((action == NULL) || 2545 (data == NULL) || 2546 (action_valid(type) == 0) || 2547 ((action->cfg.action_mask & (1LLU << type)) == 0) || 2548 (action_params == NULL)) 2549 return -EINVAL; 2550 2551 /* Data update */ 2552 action_data = action_data_get(data, action, type); 2553 2554 switch (type) { 2555 case RTE_TABLE_ACTION_FWD: 2556 return fwd_apply(action_data, 2557 action_params); 2558 2559 case RTE_TABLE_ACTION_LB: 2560 return lb_apply(action_data, 2561 action_params); 2562 2563 case RTE_TABLE_ACTION_MTR: 2564 return mtr_apply(action_data, 2565 action_params, 2566 &action->cfg.mtr, 2567 action->mp, 2568 RTE_DIM(action->mp)); 2569 2570 case RTE_TABLE_ACTION_TM: 2571 return tm_apply(action_data, 2572 action_params, 2573 &action->cfg.tm); 2574 2575 case RTE_TABLE_ACTION_ENCAP: 2576 return encap_apply(action_data, 2577 action_params, 2578 &action->cfg.encap, 2579 &action->cfg.common); 2580 2581 case RTE_TABLE_ACTION_NAT: 2582 return nat_apply(action_data, 2583 action_params, 2584 &action->cfg.common); 2585 2586 case RTE_TABLE_ACTION_TTL: 2587 return ttl_apply(action_data, 2588 action_params); 2589 2590 case RTE_TABLE_ACTION_STATS: 2591 return stats_apply(action_data, 2592 action_params); 2593 2594 case RTE_TABLE_ACTION_TIME: 2595 return time_apply(action_data, 2596 action_params); 2597 2598 case RTE_TABLE_ACTION_SYM_CRYPTO: 2599 return sym_crypto_apply(action_data, 2600 &action->cfg.sym_crypto, 2601 action_params); 2602 2603 case RTE_TABLE_ACTION_TAG: 2604 return tag_apply(action_data, 2605 action_params); 2606 2607 case RTE_TABLE_ACTION_DECAP: 2608 return decap_apply(action_data, 2609 action_params); 2610 2611 default: 2612 return -EINVAL; 2613 } 2614 } 2615 2616 int 2617 rte_table_action_dscp_table_update(struct rte_table_action *action, 2618 uint64_t dscp_mask, 2619 struct rte_table_action_dscp_table *table) 2620 { 2621 uint32_t i; 2622 2623 /* Check input arguments */ 2624 if ((action == NULL) || 2625 ((action->cfg.action_mask & ((1LLU << RTE_TABLE_ACTION_MTR) | 2626 (1LLU << RTE_TABLE_ACTION_TM))) == 0) || 2627 (dscp_mask == 0) || 2628 (table == NULL)) 2629 return -EINVAL; 2630 2631 for (i = 0; i < RTE_DIM(table->entry); i++) { 2632 struct dscp_table_entry_data *data = 2633 &action->dscp_table.entry[i]; 2634 struct rte_table_action_dscp_table_entry *entry = 2635 &table->entry[i]; 2636 2637 if ((dscp_mask & (1LLU << i)) == 0) 2638 continue; 2639 2640 data->color = entry->color; 2641 data->tc = entry->tc_id; 2642 data->tc_queue = entry->tc_queue_id; 2643 } 2644 2645 return 0; 2646 } 2647 2648 int 2649 rte_table_action_meter_profile_add(struct rte_table_action *action, 2650 uint32_t meter_profile_id, 2651 struct rte_table_action_meter_profile *profile) 2652 { 2653 struct meter_profile_data *mp_data; 2654 uint32_t status; 2655 2656 /* Check input arguments */ 2657 if ((action == NULL) || 2658 ((action->cfg.action_mask & (1LLU << RTE_TABLE_ACTION_MTR)) == 0) || 2659 (profile == NULL)) 2660 return -EINVAL; 2661 2662 if (profile->alg != RTE_TABLE_ACTION_METER_TRTCM) 2663 return -ENOTSUP; 2664 2665 mp_data = meter_profile_data_find(action->mp, 2666 RTE_DIM(action->mp), 2667 meter_profile_id); 2668 if (mp_data) 2669 return -EEXIST; 2670 2671 mp_data = meter_profile_data_find_unused(action->mp, 2672 RTE_DIM(action->mp)); 2673 if (!mp_data) 2674 return -ENOSPC; 2675 2676 /* Install new profile */ 2677 status = rte_meter_trtcm_profile_config(&mp_data->profile, 2678 &profile->trtcm); 2679 if (status) 2680 return status; 2681 2682 mp_data->profile_id = meter_profile_id; 2683 mp_data->valid = 1; 2684 2685 return 0; 2686 } 2687 2688 int 2689 rte_table_action_meter_profile_delete(struct rte_table_action *action, 2690 uint32_t meter_profile_id) 2691 { 2692 struct meter_profile_data *mp_data; 2693 2694 /* Check input arguments */ 2695 if ((action == NULL) || 2696 ((action->cfg.action_mask & (1LLU << RTE_TABLE_ACTION_MTR)) == 0)) 2697 return -EINVAL; 2698 2699 mp_data = meter_profile_data_find(action->mp, 2700 RTE_DIM(action->mp), 2701 meter_profile_id); 2702 if (!mp_data) 2703 return 0; 2704 2705 /* Uninstall profile */ 2706 mp_data->valid = 0; 2707 2708 return 0; 2709 } 2710 2711 int 2712 rte_table_action_meter_read(struct rte_table_action *action, 2713 void *data, 2714 uint32_t tc_mask, 2715 struct rte_table_action_mtr_counters *stats, 2716 int clear) 2717 { 2718 struct mtr_trtcm_data *mtr_data; 2719 uint32_t i; 2720 2721 /* Check input arguments */ 2722 if ((action == NULL) || 2723 ((action->cfg.action_mask & (1LLU << RTE_TABLE_ACTION_MTR)) == 0) || 2724 (data == NULL) || 2725 (tc_mask > RTE_LEN2MASK(action->cfg.mtr.n_tc, uint32_t))) 2726 return -EINVAL; 2727 2728 mtr_data = action_data_get(data, action, RTE_TABLE_ACTION_MTR); 2729 2730 /* Read */ 2731 if (stats) { 2732 for (i = 0; i < RTE_TABLE_ACTION_TC_MAX; i++) { 2733 struct rte_table_action_mtr_counters_tc *dst = 2734 &stats->stats[i]; 2735 struct mtr_trtcm_data *src = &mtr_data[i]; 2736 2737 if ((tc_mask & (1 << i)) == 0) 2738 continue; 2739 2740 dst->n_packets[RTE_COLOR_GREEN] = 2741 mtr_trtcm_data_stats_get(src, RTE_COLOR_GREEN); 2742 2743 dst->n_packets[RTE_COLOR_YELLOW] = 2744 mtr_trtcm_data_stats_get(src, RTE_COLOR_YELLOW); 2745 2746 dst->n_packets[RTE_COLOR_RED] = 2747 mtr_trtcm_data_stats_get(src, RTE_COLOR_RED); 2748 2749 dst->n_packets_valid = 1; 2750 dst->n_bytes_valid = 0; 2751 } 2752 2753 stats->tc_mask = tc_mask; 2754 } 2755 2756 /* Clear */ 2757 if (clear) 2758 for (i = 0; i < RTE_TABLE_ACTION_TC_MAX; i++) { 2759 struct mtr_trtcm_data *src = &mtr_data[i]; 2760 2761 if ((tc_mask & (1 << i)) == 0) 2762 continue; 2763 2764 mtr_trtcm_data_stats_reset(src, RTE_COLOR_GREEN); 2765 mtr_trtcm_data_stats_reset(src, RTE_COLOR_YELLOW); 2766 mtr_trtcm_data_stats_reset(src, RTE_COLOR_RED); 2767 } 2768 2769 2770 return 0; 2771 } 2772 2773 int 2774 rte_table_action_ttl_read(struct rte_table_action *action, 2775 void *data, 2776 struct rte_table_action_ttl_counters *stats, 2777 int clear) 2778 { 2779 struct ttl_data *ttl_data; 2780 2781 /* Check input arguments */ 2782 if ((action == NULL) || 2783 ((action->cfg.action_mask & 2784 (1LLU << RTE_TABLE_ACTION_TTL)) == 0) || 2785 (data == NULL)) 2786 return -EINVAL; 2787 2788 ttl_data = action_data_get(data, action, RTE_TABLE_ACTION_TTL); 2789 2790 /* Read */ 2791 if (stats) 2792 stats->n_packets = TTL_STATS_READ(ttl_data); 2793 2794 /* Clear */ 2795 if (clear) 2796 TTL_STATS_RESET(ttl_data); 2797 2798 return 0; 2799 } 2800 2801 int 2802 rte_table_action_stats_read(struct rte_table_action *action, 2803 void *data, 2804 struct rte_table_action_stats_counters *stats, 2805 int clear) 2806 { 2807 struct stats_data *stats_data; 2808 2809 /* Check input arguments */ 2810 if ((action == NULL) || 2811 ((action->cfg.action_mask & 2812 (1LLU << RTE_TABLE_ACTION_STATS)) == 0) || 2813 (data == NULL)) 2814 return -EINVAL; 2815 2816 stats_data = action_data_get(data, action, 2817 RTE_TABLE_ACTION_STATS); 2818 2819 /* Read */ 2820 if (stats) { 2821 stats->n_packets = stats_data->n_packets; 2822 stats->n_bytes = stats_data->n_bytes; 2823 stats->n_packets_valid = 1; 2824 stats->n_bytes_valid = 1; 2825 } 2826 2827 /* Clear */ 2828 if (clear) { 2829 stats_data->n_packets = 0; 2830 stats_data->n_bytes = 0; 2831 } 2832 2833 return 0; 2834 } 2835 2836 int 2837 rte_table_action_time_read(struct rte_table_action *action, 2838 void *data, 2839 uint64_t *timestamp) 2840 { 2841 struct time_data *time_data; 2842 2843 /* Check input arguments */ 2844 if ((action == NULL) || 2845 ((action->cfg.action_mask & 2846 (1LLU << RTE_TABLE_ACTION_TIME)) == 0) || 2847 (data == NULL) || 2848 (timestamp == NULL)) 2849 return -EINVAL; 2850 2851 time_data = action_data_get(data, action, RTE_TABLE_ACTION_TIME); 2852 2853 /* Read */ 2854 *timestamp = time_data->time; 2855 2856 return 0; 2857 } 2858 2859 struct rte_cryptodev_sym_session * 2860 rte_table_action_crypto_sym_session_get(struct rte_table_action *action, 2861 void *data) 2862 { 2863 struct sym_crypto_data *sym_crypto_data; 2864 2865 /* Check input arguments */ 2866 if ((action == NULL) || 2867 ((action->cfg.action_mask & 2868 (1LLU << RTE_TABLE_ACTION_SYM_CRYPTO)) == 0) || 2869 (data == NULL)) 2870 return NULL; 2871 2872 sym_crypto_data = action_data_get(data, action, 2873 RTE_TABLE_ACTION_SYM_CRYPTO); 2874 2875 return sym_crypto_data->session; 2876 } 2877 2878 static __rte_always_inline uint64_t 2879 pkt_work(struct rte_mbuf *mbuf, 2880 struct rte_pipeline_table_entry *table_entry, 2881 uint64_t time, 2882 struct rte_table_action *action, 2883 struct ap_config *cfg) 2884 { 2885 uint64_t drop_mask = 0; 2886 2887 uint32_t ip_offset = action->cfg.common.ip_offset; 2888 void *ip = RTE_MBUF_METADATA_UINT32_PTR(mbuf, ip_offset); 2889 2890 uint32_t dscp; 2891 uint16_t total_length; 2892 2893 if (cfg->common.ip_version) { 2894 struct rte_ipv4_hdr *hdr = ip; 2895 2896 dscp = hdr->type_of_service >> 2; 2897 total_length = rte_ntohs(hdr->total_length); 2898 } else { 2899 struct rte_ipv6_hdr *hdr = ip; 2900 2901 dscp = (rte_ntohl(hdr->vtc_flow) & 0x0F600000) >> 18; 2902 total_length = rte_ntohs(hdr->payload_len) + 2903 sizeof(struct rte_ipv6_hdr); 2904 } 2905 2906 if (cfg->action_mask & (1LLU << RTE_TABLE_ACTION_LB)) { 2907 void *data = 2908 action_data_get(table_entry, action, RTE_TABLE_ACTION_LB); 2909 2910 pkt_work_lb(mbuf, 2911 data, 2912 &cfg->lb); 2913 } 2914 if (cfg->action_mask & (1LLU << RTE_TABLE_ACTION_MTR)) { 2915 void *data = 2916 action_data_get(table_entry, action, RTE_TABLE_ACTION_MTR); 2917 2918 drop_mask |= pkt_work_mtr(mbuf, 2919 data, 2920 &action->dscp_table, 2921 action->mp, 2922 time, 2923 dscp, 2924 total_length); 2925 } 2926 2927 if (cfg->action_mask & (1LLU << RTE_TABLE_ACTION_TM)) { 2928 void *data = 2929 action_data_get(table_entry, action, RTE_TABLE_ACTION_TM); 2930 2931 pkt_work_tm(mbuf, 2932 data, 2933 &action->dscp_table, 2934 dscp); 2935 } 2936 2937 if (cfg->action_mask & (1LLU << RTE_TABLE_ACTION_DECAP)) { 2938 void *data = action_data_get(table_entry, 2939 action, 2940 RTE_TABLE_ACTION_DECAP); 2941 2942 pkt_work_decap(mbuf, data); 2943 } 2944 2945 if (cfg->action_mask & (1LLU << RTE_TABLE_ACTION_ENCAP)) { 2946 void *data = 2947 action_data_get(table_entry, action, RTE_TABLE_ACTION_ENCAP); 2948 2949 pkt_work_encap(mbuf, 2950 data, 2951 &cfg->encap, 2952 ip, 2953 total_length, 2954 ip_offset); 2955 } 2956 2957 if (cfg->action_mask & (1LLU << RTE_TABLE_ACTION_NAT)) { 2958 void *data = 2959 action_data_get(table_entry, action, RTE_TABLE_ACTION_NAT); 2960 2961 if (cfg->common.ip_version) 2962 pkt_ipv4_work_nat(ip, data, &cfg->nat); 2963 else 2964 pkt_ipv6_work_nat(ip, data, &cfg->nat); 2965 } 2966 2967 if (cfg->action_mask & (1LLU << RTE_TABLE_ACTION_TTL)) { 2968 void *data = 2969 action_data_get(table_entry, action, RTE_TABLE_ACTION_TTL); 2970 2971 if (cfg->common.ip_version) 2972 drop_mask |= pkt_ipv4_work_ttl(ip, data); 2973 else 2974 drop_mask |= pkt_ipv6_work_ttl(ip, data); 2975 } 2976 2977 if (cfg->action_mask & (1LLU << RTE_TABLE_ACTION_STATS)) { 2978 void *data = 2979 action_data_get(table_entry, action, RTE_TABLE_ACTION_STATS); 2980 2981 pkt_work_stats(data, total_length); 2982 } 2983 2984 if (cfg->action_mask & (1LLU << RTE_TABLE_ACTION_TIME)) { 2985 void *data = 2986 action_data_get(table_entry, action, RTE_TABLE_ACTION_TIME); 2987 2988 pkt_work_time(data, time); 2989 } 2990 2991 if (cfg->action_mask & (1LLU << RTE_TABLE_ACTION_SYM_CRYPTO)) { 2992 void *data = action_data_get(table_entry, action, 2993 RTE_TABLE_ACTION_SYM_CRYPTO); 2994 2995 drop_mask |= pkt_work_sym_crypto(mbuf, data, &cfg->sym_crypto, 2996 ip_offset); 2997 } 2998 2999 if (cfg->action_mask & (1LLU << RTE_TABLE_ACTION_TAG)) { 3000 void *data = action_data_get(table_entry, 3001 action, 3002 RTE_TABLE_ACTION_TAG); 3003 3004 pkt_work_tag(mbuf, data); 3005 } 3006 3007 return drop_mask; 3008 } 3009 3010 static __rte_always_inline uint64_t 3011 pkt4_work(struct rte_mbuf **mbufs, 3012 struct rte_pipeline_table_entry **table_entries, 3013 uint64_t time, 3014 struct rte_table_action *action, 3015 struct ap_config *cfg) 3016 { 3017 uint64_t drop_mask0 = 0; 3018 uint64_t drop_mask1 = 0; 3019 uint64_t drop_mask2 = 0; 3020 uint64_t drop_mask3 = 0; 3021 3022 struct rte_mbuf *mbuf0 = mbufs[0]; 3023 struct rte_mbuf *mbuf1 = mbufs[1]; 3024 struct rte_mbuf *mbuf2 = mbufs[2]; 3025 struct rte_mbuf *mbuf3 = mbufs[3]; 3026 3027 struct rte_pipeline_table_entry *table_entry0 = table_entries[0]; 3028 struct rte_pipeline_table_entry *table_entry1 = table_entries[1]; 3029 struct rte_pipeline_table_entry *table_entry2 = table_entries[2]; 3030 struct rte_pipeline_table_entry *table_entry3 = table_entries[3]; 3031 3032 uint32_t ip_offset = action->cfg.common.ip_offset; 3033 void *ip0 = RTE_MBUF_METADATA_UINT32_PTR(mbuf0, ip_offset); 3034 void *ip1 = RTE_MBUF_METADATA_UINT32_PTR(mbuf1, ip_offset); 3035 void *ip2 = RTE_MBUF_METADATA_UINT32_PTR(mbuf2, ip_offset); 3036 void *ip3 = RTE_MBUF_METADATA_UINT32_PTR(mbuf3, ip_offset); 3037 3038 uint32_t dscp0, dscp1, dscp2, dscp3; 3039 uint16_t total_length0, total_length1, total_length2, total_length3; 3040 3041 if (cfg->common.ip_version) { 3042 struct rte_ipv4_hdr *hdr0 = ip0; 3043 struct rte_ipv4_hdr *hdr1 = ip1; 3044 struct rte_ipv4_hdr *hdr2 = ip2; 3045 struct rte_ipv4_hdr *hdr3 = ip3; 3046 3047 dscp0 = hdr0->type_of_service >> 2; 3048 dscp1 = hdr1->type_of_service >> 2; 3049 dscp2 = hdr2->type_of_service >> 2; 3050 dscp3 = hdr3->type_of_service >> 2; 3051 3052 total_length0 = rte_ntohs(hdr0->total_length); 3053 total_length1 = rte_ntohs(hdr1->total_length); 3054 total_length2 = rte_ntohs(hdr2->total_length); 3055 total_length3 = rte_ntohs(hdr3->total_length); 3056 } else { 3057 struct rte_ipv6_hdr *hdr0 = ip0; 3058 struct rte_ipv6_hdr *hdr1 = ip1; 3059 struct rte_ipv6_hdr *hdr2 = ip2; 3060 struct rte_ipv6_hdr *hdr3 = ip3; 3061 3062 dscp0 = (rte_ntohl(hdr0->vtc_flow) & 0x0F600000) >> 18; 3063 dscp1 = (rte_ntohl(hdr1->vtc_flow) & 0x0F600000) >> 18; 3064 dscp2 = (rte_ntohl(hdr2->vtc_flow) & 0x0F600000) >> 18; 3065 dscp3 = (rte_ntohl(hdr3->vtc_flow) & 0x0F600000) >> 18; 3066 3067 total_length0 = rte_ntohs(hdr0->payload_len) + 3068 sizeof(struct rte_ipv6_hdr); 3069 total_length1 = rte_ntohs(hdr1->payload_len) + 3070 sizeof(struct rte_ipv6_hdr); 3071 total_length2 = rte_ntohs(hdr2->payload_len) + 3072 sizeof(struct rte_ipv6_hdr); 3073 total_length3 = rte_ntohs(hdr3->payload_len) + 3074 sizeof(struct rte_ipv6_hdr); 3075 } 3076 3077 if (cfg->action_mask & (1LLU << RTE_TABLE_ACTION_LB)) { 3078 void *data0 = 3079 action_data_get(table_entry0, action, RTE_TABLE_ACTION_LB); 3080 void *data1 = 3081 action_data_get(table_entry1, action, RTE_TABLE_ACTION_LB); 3082 void *data2 = 3083 action_data_get(table_entry2, action, RTE_TABLE_ACTION_LB); 3084 void *data3 = 3085 action_data_get(table_entry3, action, RTE_TABLE_ACTION_LB); 3086 3087 pkt_work_lb(mbuf0, 3088 data0, 3089 &cfg->lb); 3090 3091 pkt_work_lb(mbuf1, 3092 data1, 3093 &cfg->lb); 3094 3095 pkt_work_lb(mbuf2, 3096 data2, 3097 &cfg->lb); 3098 3099 pkt_work_lb(mbuf3, 3100 data3, 3101 &cfg->lb); 3102 } 3103 3104 if (cfg->action_mask & (1LLU << RTE_TABLE_ACTION_MTR)) { 3105 void *data0 = 3106 action_data_get(table_entry0, action, RTE_TABLE_ACTION_MTR); 3107 void *data1 = 3108 action_data_get(table_entry1, action, RTE_TABLE_ACTION_MTR); 3109 void *data2 = 3110 action_data_get(table_entry2, action, RTE_TABLE_ACTION_MTR); 3111 void *data3 = 3112 action_data_get(table_entry3, action, RTE_TABLE_ACTION_MTR); 3113 3114 drop_mask0 |= pkt_work_mtr(mbuf0, 3115 data0, 3116 &action->dscp_table, 3117 action->mp, 3118 time, 3119 dscp0, 3120 total_length0); 3121 3122 drop_mask1 |= pkt_work_mtr(mbuf1, 3123 data1, 3124 &action->dscp_table, 3125 action->mp, 3126 time, 3127 dscp1, 3128 total_length1); 3129 3130 drop_mask2 |= pkt_work_mtr(mbuf2, 3131 data2, 3132 &action->dscp_table, 3133 action->mp, 3134 time, 3135 dscp2, 3136 total_length2); 3137 3138 drop_mask3 |= pkt_work_mtr(mbuf3, 3139 data3, 3140 &action->dscp_table, 3141 action->mp, 3142 time, 3143 dscp3, 3144 total_length3); 3145 } 3146 3147 if (cfg->action_mask & (1LLU << RTE_TABLE_ACTION_TM)) { 3148 void *data0 = 3149 action_data_get(table_entry0, action, RTE_TABLE_ACTION_TM); 3150 void *data1 = 3151 action_data_get(table_entry1, action, RTE_TABLE_ACTION_TM); 3152 void *data2 = 3153 action_data_get(table_entry2, action, RTE_TABLE_ACTION_TM); 3154 void *data3 = 3155 action_data_get(table_entry3, action, RTE_TABLE_ACTION_TM); 3156 3157 pkt_work_tm(mbuf0, 3158 data0, 3159 &action->dscp_table, 3160 dscp0); 3161 3162 pkt_work_tm(mbuf1, 3163 data1, 3164 &action->dscp_table, 3165 dscp1); 3166 3167 pkt_work_tm(mbuf2, 3168 data2, 3169 &action->dscp_table, 3170 dscp2); 3171 3172 pkt_work_tm(mbuf3, 3173 data3, 3174 &action->dscp_table, 3175 dscp3); 3176 } 3177 3178 if (cfg->action_mask & (1LLU << RTE_TABLE_ACTION_DECAP)) { 3179 void *data0 = action_data_get(table_entry0, 3180 action, 3181 RTE_TABLE_ACTION_DECAP); 3182 void *data1 = action_data_get(table_entry1, 3183 action, 3184 RTE_TABLE_ACTION_DECAP); 3185 void *data2 = action_data_get(table_entry2, 3186 action, 3187 RTE_TABLE_ACTION_DECAP); 3188 void *data3 = action_data_get(table_entry3, 3189 action, 3190 RTE_TABLE_ACTION_DECAP); 3191 3192 pkt4_work_decap(mbuf0, mbuf1, mbuf2, mbuf3, 3193 data0, data1, data2, data3); 3194 } 3195 3196 if (cfg->action_mask & (1LLU << RTE_TABLE_ACTION_ENCAP)) { 3197 void *data0 = 3198 action_data_get(table_entry0, action, RTE_TABLE_ACTION_ENCAP); 3199 void *data1 = 3200 action_data_get(table_entry1, action, RTE_TABLE_ACTION_ENCAP); 3201 void *data2 = 3202 action_data_get(table_entry2, action, RTE_TABLE_ACTION_ENCAP); 3203 void *data3 = 3204 action_data_get(table_entry3, action, RTE_TABLE_ACTION_ENCAP); 3205 3206 pkt_work_encap(mbuf0, 3207 data0, 3208 &cfg->encap, 3209 ip0, 3210 total_length0, 3211 ip_offset); 3212 3213 pkt_work_encap(mbuf1, 3214 data1, 3215 &cfg->encap, 3216 ip1, 3217 total_length1, 3218 ip_offset); 3219 3220 pkt_work_encap(mbuf2, 3221 data2, 3222 &cfg->encap, 3223 ip2, 3224 total_length2, 3225 ip_offset); 3226 3227 pkt_work_encap(mbuf3, 3228 data3, 3229 &cfg->encap, 3230 ip3, 3231 total_length3, 3232 ip_offset); 3233 } 3234 3235 if (cfg->action_mask & (1LLU << RTE_TABLE_ACTION_NAT)) { 3236 void *data0 = 3237 action_data_get(table_entry0, action, RTE_TABLE_ACTION_NAT); 3238 void *data1 = 3239 action_data_get(table_entry1, action, RTE_TABLE_ACTION_NAT); 3240 void *data2 = 3241 action_data_get(table_entry2, action, RTE_TABLE_ACTION_NAT); 3242 void *data3 = 3243 action_data_get(table_entry3, action, RTE_TABLE_ACTION_NAT); 3244 3245 if (cfg->common.ip_version) { 3246 pkt_ipv4_work_nat(ip0, data0, &cfg->nat); 3247 pkt_ipv4_work_nat(ip1, data1, &cfg->nat); 3248 pkt_ipv4_work_nat(ip2, data2, &cfg->nat); 3249 pkt_ipv4_work_nat(ip3, data3, &cfg->nat); 3250 } else { 3251 pkt_ipv6_work_nat(ip0, data0, &cfg->nat); 3252 pkt_ipv6_work_nat(ip1, data1, &cfg->nat); 3253 pkt_ipv6_work_nat(ip2, data2, &cfg->nat); 3254 pkt_ipv6_work_nat(ip3, data3, &cfg->nat); 3255 } 3256 } 3257 3258 if (cfg->action_mask & (1LLU << RTE_TABLE_ACTION_TTL)) { 3259 void *data0 = 3260 action_data_get(table_entry0, action, RTE_TABLE_ACTION_TTL); 3261 void *data1 = 3262 action_data_get(table_entry1, action, RTE_TABLE_ACTION_TTL); 3263 void *data2 = 3264 action_data_get(table_entry2, action, RTE_TABLE_ACTION_TTL); 3265 void *data3 = 3266 action_data_get(table_entry3, action, RTE_TABLE_ACTION_TTL); 3267 3268 if (cfg->common.ip_version) { 3269 drop_mask0 |= pkt_ipv4_work_ttl(ip0, data0); 3270 drop_mask1 |= pkt_ipv4_work_ttl(ip1, data1); 3271 drop_mask2 |= pkt_ipv4_work_ttl(ip2, data2); 3272 drop_mask3 |= pkt_ipv4_work_ttl(ip3, data3); 3273 } else { 3274 drop_mask0 |= pkt_ipv6_work_ttl(ip0, data0); 3275 drop_mask1 |= pkt_ipv6_work_ttl(ip1, data1); 3276 drop_mask2 |= pkt_ipv6_work_ttl(ip2, data2); 3277 drop_mask3 |= pkt_ipv6_work_ttl(ip3, data3); 3278 } 3279 } 3280 3281 if (cfg->action_mask & (1LLU << RTE_TABLE_ACTION_STATS)) { 3282 void *data0 = 3283 action_data_get(table_entry0, action, RTE_TABLE_ACTION_STATS); 3284 void *data1 = 3285 action_data_get(table_entry1, action, RTE_TABLE_ACTION_STATS); 3286 void *data2 = 3287 action_data_get(table_entry2, action, RTE_TABLE_ACTION_STATS); 3288 void *data3 = 3289 action_data_get(table_entry3, action, RTE_TABLE_ACTION_STATS); 3290 3291 pkt_work_stats(data0, total_length0); 3292 pkt_work_stats(data1, total_length1); 3293 pkt_work_stats(data2, total_length2); 3294 pkt_work_stats(data3, total_length3); 3295 } 3296 3297 if (cfg->action_mask & (1LLU << RTE_TABLE_ACTION_TIME)) { 3298 void *data0 = 3299 action_data_get(table_entry0, action, RTE_TABLE_ACTION_TIME); 3300 void *data1 = 3301 action_data_get(table_entry1, action, RTE_TABLE_ACTION_TIME); 3302 void *data2 = 3303 action_data_get(table_entry2, action, RTE_TABLE_ACTION_TIME); 3304 void *data3 = 3305 action_data_get(table_entry3, action, RTE_TABLE_ACTION_TIME); 3306 3307 pkt_work_time(data0, time); 3308 pkt_work_time(data1, time); 3309 pkt_work_time(data2, time); 3310 pkt_work_time(data3, time); 3311 } 3312 3313 if (cfg->action_mask & (1LLU << RTE_TABLE_ACTION_SYM_CRYPTO)) { 3314 void *data0 = action_data_get(table_entry0, action, 3315 RTE_TABLE_ACTION_SYM_CRYPTO); 3316 void *data1 = action_data_get(table_entry1, action, 3317 RTE_TABLE_ACTION_SYM_CRYPTO); 3318 void *data2 = action_data_get(table_entry2, action, 3319 RTE_TABLE_ACTION_SYM_CRYPTO); 3320 void *data3 = action_data_get(table_entry3, action, 3321 RTE_TABLE_ACTION_SYM_CRYPTO); 3322 3323 drop_mask0 |= pkt_work_sym_crypto(mbuf0, data0, &cfg->sym_crypto, 3324 ip_offset); 3325 drop_mask1 |= pkt_work_sym_crypto(mbuf1, data1, &cfg->sym_crypto, 3326 ip_offset); 3327 drop_mask2 |= pkt_work_sym_crypto(mbuf2, data2, &cfg->sym_crypto, 3328 ip_offset); 3329 drop_mask3 |= pkt_work_sym_crypto(mbuf3, data3, &cfg->sym_crypto, 3330 ip_offset); 3331 } 3332 3333 if (cfg->action_mask & (1LLU << RTE_TABLE_ACTION_TAG)) { 3334 void *data0 = action_data_get(table_entry0, 3335 action, 3336 RTE_TABLE_ACTION_TAG); 3337 void *data1 = action_data_get(table_entry1, 3338 action, 3339 RTE_TABLE_ACTION_TAG); 3340 void *data2 = action_data_get(table_entry2, 3341 action, 3342 RTE_TABLE_ACTION_TAG); 3343 void *data3 = action_data_get(table_entry3, 3344 action, 3345 RTE_TABLE_ACTION_TAG); 3346 3347 pkt4_work_tag(mbuf0, mbuf1, mbuf2, mbuf3, 3348 data0, data1, data2, data3); 3349 } 3350 3351 return drop_mask0 | 3352 (drop_mask1 << 1) | 3353 (drop_mask2 << 2) | 3354 (drop_mask3 << 3); 3355 } 3356 3357 static __rte_always_inline int 3358 ah(struct rte_pipeline *p, 3359 struct rte_mbuf **pkts, 3360 uint64_t pkts_mask, 3361 struct rte_pipeline_table_entry **entries, 3362 struct rte_table_action *action, 3363 struct ap_config *cfg) 3364 { 3365 uint64_t pkts_drop_mask = 0; 3366 uint64_t time = 0; 3367 3368 if (cfg->action_mask & ((1LLU << RTE_TABLE_ACTION_MTR) | 3369 (1LLU << RTE_TABLE_ACTION_TIME))) 3370 time = rte_rdtsc(); 3371 3372 if ((pkts_mask & (pkts_mask + 1)) == 0) { 3373 uint64_t n_pkts = __builtin_popcountll(pkts_mask); 3374 uint32_t i; 3375 3376 for (i = 0; i < (n_pkts & (~0x3LLU)); i += 4) { 3377 uint64_t drop_mask; 3378 3379 drop_mask = pkt4_work(&pkts[i], 3380 &entries[i], 3381 time, 3382 action, 3383 cfg); 3384 3385 pkts_drop_mask |= drop_mask << i; 3386 } 3387 3388 for ( ; i < n_pkts; i++) { 3389 uint64_t drop_mask; 3390 3391 drop_mask = pkt_work(pkts[i], 3392 entries[i], 3393 time, 3394 action, 3395 cfg); 3396 3397 pkts_drop_mask |= drop_mask << i; 3398 } 3399 } else 3400 for ( ; pkts_mask; ) { 3401 uint32_t pos = __builtin_ctzll(pkts_mask); 3402 uint64_t pkt_mask = 1LLU << pos; 3403 uint64_t drop_mask; 3404 3405 drop_mask = pkt_work(pkts[pos], 3406 entries[pos], 3407 time, 3408 action, 3409 cfg); 3410 3411 pkts_mask &= ~pkt_mask; 3412 pkts_drop_mask |= drop_mask << pos; 3413 } 3414 3415 rte_pipeline_ah_packet_drop(p, pkts_drop_mask); 3416 3417 return 0; 3418 } 3419 3420 static int 3421 ah_default(struct rte_pipeline *p, 3422 struct rte_mbuf **pkts, 3423 uint64_t pkts_mask, 3424 struct rte_pipeline_table_entry **entries, 3425 void *arg) 3426 { 3427 struct rte_table_action *action = arg; 3428 3429 return ah(p, 3430 pkts, 3431 pkts_mask, 3432 entries, 3433 action, 3434 &action->cfg); 3435 } 3436 3437 static rte_pipeline_table_action_handler_hit 3438 ah_selector(struct rte_table_action *action) 3439 { 3440 if (action->cfg.action_mask == (1LLU << RTE_TABLE_ACTION_FWD)) 3441 return NULL; 3442 3443 return ah_default; 3444 } 3445 3446 int 3447 rte_table_action_table_params_get(struct rte_table_action *action, 3448 struct rte_pipeline_table_params *params) 3449 { 3450 rte_pipeline_table_action_handler_hit f_action_hit; 3451 uint32_t total_size; 3452 3453 /* Check input arguments */ 3454 if ((action == NULL) || 3455 (params == NULL)) 3456 return -EINVAL; 3457 3458 f_action_hit = ah_selector(action); 3459 total_size = rte_align32pow2(action->data.total_size); 3460 3461 /* Fill in params */ 3462 params->f_action_hit = f_action_hit; 3463 params->f_action_miss = NULL; 3464 params->arg_ah = (f_action_hit) ? action : NULL; 3465 params->action_data_size = total_size - 3466 sizeof(struct rte_pipeline_table_entry); 3467 3468 return 0; 3469 } 3470 3471 int 3472 rte_table_action_free(struct rte_table_action *action) 3473 { 3474 if (action == NULL) 3475 return 0; 3476 3477 rte_free(action); 3478 3479 return 0; 3480 } 3481