1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2018 Intel Corporation 3 */ 4 #include <stdlib.h> 5 #include <string.h> 6 7 #include <rte_common.h> 8 #include <rte_byteorder.h> 9 #include <rte_cycles.h> 10 #include <rte_malloc.h> 11 #include <rte_memcpy.h> 12 #include <rte_ether.h> 13 #include <rte_ip.h> 14 #include <rte_tcp.h> 15 #include <rte_udp.h> 16 #include <rte_vxlan.h> 17 #include <rte_cryptodev.h> 18 19 #include "rte_table_action.h" 20 21 #define rte_htons rte_cpu_to_be_16 22 #define rte_htonl rte_cpu_to_be_32 23 24 #define rte_ntohs rte_be_to_cpu_16 25 #define rte_ntohl rte_be_to_cpu_32 26 27 /** 28 * RTE_TABLE_ACTION_FWD 29 */ 30 #define fwd_data rte_pipeline_table_entry 31 32 static int 33 fwd_apply(struct fwd_data *data, 34 struct rte_table_action_fwd_params *p) 35 { 36 data->action = p->action; 37 38 if (p->action == RTE_PIPELINE_ACTION_PORT) 39 data->port_id = p->id; 40 41 if (p->action == RTE_PIPELINE_ACTION_TABLE) 42 data->table_id = p->id; 43 44 return 0; 45 } 46 47 /** 48 * RTE_TABLE_ACTION_LB 49 */ 50 static int 51 lb_cfg_check(struct rte_table_action_lb_config *cfg) 52 { 53 if ((cfg == NULL) || 54 (cfg->key_size < RTE_TABLE_ACTION_LB_KEY_SIZE_MIN) || 55 (cfg->key_size > RTE_TABLE_ACTION_LB_KEY_SIZE_MAX) || 56 (!rte_is_power_of_2(cfg->key_size)) || 57 (cfg->f_hash == NULL)) 58 return -1; 59 60 return 0; 61 } 62 63 struct lb_data { 64 uint32_t out[RTE_TABLE_ACTION_LB_TABLE_SIZE]; 65 } __rte_packed; 66 67 static int 68 lb_apply(struct lb_data *data, 69 struct rte_table_action_lb_params *p) 70 { 71 memcpy(data->out, p->out, sizeof(data->out)); 72 73 return 0; 74 } 75 76 static __rte_always_inline void 77 pkt_work_lb(struct rte_mbuf *mbuf, 78 struct lb_data *data, 79 struct rte_table_action_lb_config *cfg) 80 { 81 uint8_t *pkt_key = RTE_MBUF_METADATA_UINT8_PTR(mbuf, cfg->key_offset); 82 uint32_t *out = RTE_MBUF_METADATA_UINT32_PTR(mbuf, cfg->out_offset); 83 uint64_t digest, pos; 84 uint32_t out_val; 85 86 digest = cfg->f_hash(pkt_key, 87 cfg->key_mask, 88 cfg->key_size, 89 cfg->seed); 90 pos = digest & (RTE_TABLE_ACTION_LB_TABLE_SIZE - 1); 91 out_val = data->out[pos]; 92 93 *out = out_val; 94 } 95 96 /** 97 * RTE_TABLE_ACTION_MTR 98 */ 99 static int 100 mtr_cfg_check(struct rte_table_action_mtr_config *mtr) 101 { 102 if ((mtr->alg == RTE_TABLE_ACTION_METER_SRTCM) || 103 ((mtr->n_tc != 1) && (mtr->n_tc != 4)) || 104 (mtr->n_bytes_enabled != 0)) 105 return -ENOTSUP; 106 return 0; 107 } 108 109 struct mtr_trtcm_data { 110 struct rte_meter_trtcm trtcm; 111 uint64_t stats[RTE_COLORS]; 112 } __rte_packed; 113 114 #define MTR_TRTCM_DATA_METER_PROFILE_ID_GET(data) \ 115 (((data)->stats[RTE_COLOR_GREEN] & 0xF8LLU) >> 3) 116 117 static void 118 mtr_trtcm_data_meter_profile_id_set(struct mtr_trtcm_data *data, 119 uint32_t profile_id) 120 { 121 data->stats[RTE_COLOR_GREEN] &= ~0xF8LLU; 122 data->stats[RTE_COLOR_GREEN] |= (profile_id % 32) << 3; 123 } 124 125 #define MTR_TRTCM_DATA_POLICER_ACTION_DROP_GET(data, color)\ 126 (((data)->stats[(color)] & 4LLU) >> 2) 127 128 #define MTR_TRTCM_DATA_POLICER_ACTION_COLOR_GET(data, color)\ 129 ((enum rte_color)((data)->stats[(color)] & 3LLU)) 130 131 static void 132 mtr_trtcm_data_policer_action_set(struct mtr_trtcm_data *data, 133 enum rte_color color, 134 enum rte_table_action_policer action) 135 { 136 if (action == RTE_TABLE_ACTION_POLICER_DROP) { 137 data->stats[color] |= 4LLU; 138 } else { 139 data->stats[color] &= ~7LLU; 140 data->stats[color] |= color & 3LLU; 141 } 142 } 143 144 static uint64_t 145 mtr_trtcm_data_stats_get(struct mtr_trtcm_data *data, 146 enum rte_color color) 147 { 148 return data->stats[color] >> 8; 149 } 150 151 static void 152 mtr_trtcm_data_stats_reset(struct mtr_trtcm_data *data, 153 enum rte_color color) 154 { 155 data->stats[color] &= 0xFFLU; 156 } 157 158 #define MTR_TRTCM_DATA_STATS_INC(data, color) \ 159 ((data)->stats[(color)] += (1LLU << 8)) 160 161 static size_t 162 mtr_data_size(struct rte_table_action_mtr_config *mtr) 163 { 164 return mtr->n_tc * sizeof(struct mtr_trtcm_data); 165 } 166 167 struct dscp_table_entry_data { 168 enum rte_color color; 169 uint16_t tc; 170 uint16_t tc_queue; 171 }; 172 173 struct dscp_table_data { 174 struct dscp_table_entry_data entry[64]; 175 }; 176 177 struct meter_profile_data { 178 struct rte_meter_trtcm_profile profile; 179 uint32_t profile_id; 180 int valid; 181 }; 182 183 static struct meter_profile_data * 184 meter_profile_data_find(struct meter_profile_data *mp, 185 uint32_t mp_size, 186 uint32_t profile_id) 187 { 188 uint32_t i; 189 190 for (i = 0; i < mp_size; i++) { 191 struct meter_profile_data *mp_data = &mp[i]; 192 193 if (mp_data->valid && (mp_data->profile_id == profile_id)) 194 return mp_data; 195 } 196 197 return NULL; 198 } 199 200 static struct meter_profile_data * 201 meter_profile_data_find_unused(struct meter_profile_data *mp, 202 uint32_t mp_size) 203 { 204 uint32_t i; 205 206 for (i = 0; i < mp_size; i++) { 207 struct meter_profile_data *mp_data = &mp[i]; 208 209 if (!mp_data->valid) 210 return mp_data; 211 } 212 213 return NULL; 214 } 215 216 static int 217 mtr_apply_check(struct rte_table_action_mtr_params *p, 218 struct rte_table_action_mtr_config *cfg, 219 struct meter_profile_data *mp, 220 uint32_t mp_size) 221 { 222 uint32_t i; 223 224 if (p->tc_mask > RTE_LEN2MASK(cfg->n_tc, uint32_t)) 225 return -EINVAL; 226 227 for (i = 0; i < RTE_TABLE_ACTION_TC_MAX; i++) { 228 struct rte_table_action_mtr_tc_params *p_tc = &p->mtr[i]; 229 struct meter_profile_data *mp_data; 230 231 if ((p->tc_mask & (1LLU << i)) == 0) 232 continue; 233 234 mp_data = meter_profile_data_find(mp, 235 mp_size, 236 p_tc->meter_profile_id); 237 if (!mp_data) 238 return -EINVAL; 239 } 240 241 return 0; 242 } 243 244 static int 245 mtr_apply(struct mtr_trtcm_data *data, 246 struct rte_table_action_mtr_params *p, 247 struct rte_table_action_mtr_config *cfg, 248 struct meter_profile_data *mp, 249 uint32_t mp_size) 250 { 251 uint32_t i; 252 int status; 253 254 /* Check input arguments */ 255 status = mtr_apply_check(p, cfg, mp, mp_size); 256 if (status) 257 return status; 258 259 /* Apply */ 260 for (i = 0; i < RTE_TABLE_ACTION_TC_MAX; i++) { 261 struct rte_table_action_mtr_tc_params *p_tc = &p->mtr[i]; 262 struct mtr_trtcm_data *data_tc = &data[i]; 263 struct meter_profile_data *mp_data; 264 265 if ((p->tc_mask & (1LLU << i)) == 0) 266 continue; 267 268 /* Find profile */ 269 mp_data = meter_profile_data_find(mp, 270 mp_size, 271 p_tc->meter_profile_id); 272 if (!mp_data) 273 return -EINVAL; 274 275 memset(data_tc, 0, sizeof(*data_tc)); 276 277 /* Meter object */ 278 status = rte_meter_trtcm_config(&data_tc->trtcm, 279 &mp_data->profile); 280 if (status) 281 return status; 282 283 /* Meter profile */ 284 mtr_trtcm_data_meter_profile_id_set(data_tc, 285 mp_data - mp); 286 287 /* Policer actions */ 288 mtr_trtcm_data_policer_action_set(data_tc, 289 RTE_COLOR_GREEN, 290 p_tc->policer[RTE_COLOR_GREEN]); 291 292 mtr_trtcm_data_policer_action_set(data_tc, 293 RTE_COLOR_YELLOW, 294 p_tc->policer[RTE_COLOR_YELLOW]); 295 296 mtr_trtcm_data_policer_action_set(data_tc, 297 RTE_COLOR_RED, 298 p_tc->policer[RTE_COLOR_RED]); 299 } 300 301 return 0; 302 } 303 304 static __rte_always_inline uint64_t 305 pkt_work_mtr(struct rte_mbuf *mbuf, 306 struct mtr_trtcm_data *data, 307 struct dscp_table_data *dscp_table, 308 struct meter_profile_data *mp, 309 uint64_t time, 310 uint32_t dscp, 311 uint16_t total_length) 312 { 313 uint64_t drop_mask; 314 struct dscp_table_entry_data *dscp_entry = &dscp_table->entry[dscp]; 315 enum rte_color color_in, color_meter, color_policer; 316 uint32_t tc, mp_id; 317 318 tc = dscp_entry->tc; 319 color_in = dscp_entry->color; 320 data += tc; 321 mp_id = MTR_TRTCM_DATA_METER_PROFILE_ID_GET(data); 322 323 /* Meter */ 324 color_meter = rte_meter_trtcm_color_aware_check( 325 &data->trtcm, 326 &mp[mp_id].profile, 327 time, 328 total_length, 329 color_in); 330 331 /* Stats */ 332 MTR_TRTCM_DATA_STATS_INC(data, color_meter); 333 334 /* Police */ 335 drop_mask = MTR_TRTCM_DATA_POLICER_ACTION_DROP_GET(data, color_meter); 336 color_policer = 337 MTR_TRTCM_DATA_POLICER_ACTION_COLOR_GET(data, color_meter); 338 rte_mbuf_sched_color_set(mbuf, (uint8_t)color_policer); 339 340 return drop_mask; 341 } 342 343 /** 344 * RTE_TABLE_ACTION_TM 345 */ 346 static int 347 tm_cfg_check(struct rte_table_action_tm_config *tm) 348 { 349 if ((tm->n_subports_per_port == 0) || 350 (rte_is_power_of_2(tm->n_subports_per_port) == 0) || 351 (tm->n_subports_per_port > UINT16_MAX) || 352 (tm->n_pipes_per_subport == 0) || 353 (rte_is_power_of_2(tm->n_pipes_per_subport) == 0)) 354 return -ENOTSUP; 355 356 return 0; 357 } 358 359 struct tm_data { 360 uint32_t queue_id; 361 uint32_t reserved; 362 } __rte_packed; 363 364 static int 365 tm_apply_check(struct rte_table_action_tm_params *p, 366 struct rte_table_action_tm_config *cfg) 367 { 368 if ((p->subport_id >= cfg->n_subports_per_port) || 369 (p->pipe_id >= cfg->n_pipes_per_subport)) 370 return -EINVAL; 371 372 return 0; 373 } 374 375 static int 376 tm_apply(struct tm_data *data, 377 struct rte_table_action_tm_params *p, 378 struct rte_table_action_tm_config *cfg) 379 { 380 int status; 381 382 /* Check input arguments */ 383 status = tm_apply_check(p, cfg); 384 if (status) 385 return status; 386 387 /* Apply */ 388 data->queue_id = p->subport_id << 389 (rte_ctz32(cfg->n_pipes_per_subport) + 4) | 390 p->pipe_id << 4; 391 392 return 0; 393 } 394 395 static __rte_always_inline void 396 pkt_work_tm(struct rte_mbuf *mbuf, 397 struct tm_data *data, 398 struct dscp_table_data *dscp_table, 399 uint32_t dscp) 400 { 401 struct dscp_table_entry_data *dscp_entry = &dscp_table->entry[dscp]; 402 uint32_t queue_id = data->queue_id | 403 dscp_entry->tc_queue; 404 rte_mbuf_sched_set(mbuf, queue_id, dscp_entry->tc, 405 (uint8_t)dscp_entry->color); 406 } 407 408 /** 409 * RTE_TABLE_ACTION_ENCAP 410 */ 411 static int 412 encap_valid(enum rte_table_action_encap_type encap) 413 { 414 switch (encap) { 415 case RTE_TABLE_ACTION_ENCAP_ETHER: 416 case RTE_TABLE_ACTION_ENCAP_VLAN: 417 case RTE_TABLE_ACTION_ENCAP_QINQ: 418 case RTE_TABLE_ACTION_ENCAP_MPLS: 419 case RTE_TABLE_ACTION_ENCAP_PPPOE: 420 case RTE_TABLE_ACTION_ENCAP_VXLAN: 421 case RTE_TABLE_ACTION_ENCAP_QINQ_PPPOE: 422 return 1; 423 default: 424 return 0; 425 } 426 } 427 428 static int 429 encap_cfg_check(struct rte_table_action_encap_config *encap) 430 { 431 if ((encap->encap_mask == 0) || 432 (rte_popcount64(encap->encap_mask) != 1)) 433 return -ENOTSUP; 434 435 return 0; 436 } 437 438 struct encap_ether_data { 439 struct rte_ether_hdr ether; 440 }; 441 442 #define VLAN(pcp, dei, vid) \ 443 ((uint16_t)((((uint64_t)(pcp)) & 0x7LLU) << 13) | \ 444 ((((uint64_t)(dei)) & 0x1LLU) << 12) | \ 445 (((uint64_t)(vid)) & 0xFFFLLU)) \ 446 447 struct encap_vlan_data { 448 struct rte_ether_hdr ether; 449 struct rte_vlan_hdr vlan; 450 }; 451 452 struct encap_qinq_data { 453 struct rte_ether_hdr ether; 454 struct rte_vlan_hdr svlan; 455 struct rte_vlan_hdr cvlan; 456 }; 457 458 #define ETHER_TYPE_MPLS_UNICAST 0x8847 459 460 #define ETHER_TYPE_MPLS_MULTICAST 0x8848 461 462 #define MPLS(label, tc, s, ttl) \ 463 ((uint32_t)(((((uint64_t)(label)) & 0xFFFFFLLU) << 12) |\ 464 ((((uint64_t)(tc)) & 0x7LLU) << 9) | \ 465 ((((uint64_t)(s)) & 0x1LLU) << 8) | \ 466 (((uint64_t)(ttl)) & 0xFFLLU))) 467 468 struct __rte_aligned(2) encap_mpls_data { 469 struct rte_ether_hdr ether; 470 uint32_t mpls[RTE_TABLE_ACTION_MPLS_LABELS_MAX]; 471 uint32_t mpls_count; 472 } __rte_packed; 473 474 #define PPP_PROTOCOL_IP 0x0021 475 476 struct pppoe_ppp_hdr { 477 uint16_t ver_type_code; 478 uint16_t session_id; 479 uint16_t length; 480 uint16_t protocol; 481 }; 482 483 struct encap_pppoe_data { 484 struct rte_ether_hdr ether; 485 struct pppoe_ppp_hdr pppoe_ppp; 486 }; 487 488 #define IP_PROTO_UDP 17 489 490 struct __rte_aligned(2) encap_vxlan_ipv4_data { 491 struct rte_ether_hdr ether; 492 struct rte_ipv4_hdr ipv4; 493 struct rte_udp_hdr udp; 494 struct rte_vxlan_hdr vxlan; 495 } __rte_packed; 496 497 struct __rte_aligned(2) encap_vxlan_ipv4_vlan_data { 498 struct rte_ether_hdr ether; 499 struct rte_vlan_hdr vlan; 500 struct rte_ipv4_hdr ipv4; 501 struct rte_udp_hdr udp; 502 struct rte_vxlan_hdr vxlan; 503 } __rte_packed; 504 505 struct __rte_aligned(2) encap_vxlan_ipv6_data { 506 struct rte_ether_hdr ether; 507 struct rte_ipv6_hdr ipv6; 508 struct rte_udp_hdr udp; 509 struct rte_vxlan_hdr vxlan; 510 } __rte_packed; 511 512 struct __rte_aligned(2) encap_vxlan_ipv6_vlan_data { 513 struct rte_ether_hdr ether; 514 struct rte_vlan_hdr vlan; 515 struct rte_ipv6_hdr ipv6; 516 struct rte_udp_hdr udp; 517 struct rte_vxlan_hdr vxlan; 518 } __rte_packed; 519 520 struct __rte_aligned(2) encap_qinq_pppoe_data { 521 struct rte_ether_hdr ether; 522 struct rte_vlan_hdr svlan; 523 struct rte_vlan_hdr cvlan; 524 struct pppoe_ppp_hdr pppoe_ppp; 525 } __rte_packed; 526 527 static size_t 528 encap_data_size(struct rte_table_action_encap_config *encap) 529 { 530 switch (encap->encap_mask) { 531 case 1LLU << RTE_TABLE_ACTION_ENCAP_ETHER: 532 return sizeof(struct encap_ether_data); 533 534 case 1LLU << RTE_TABLE_ACTION_ENCAP_VLAN: 535 return sizeof(struct encap_vlan_data); 536 537 case 1LLU << RTE_TABLE_ACTION_ENCAP_QINQ: 538 return sizeof(struct encap_qinq_data); 539 540 case 1LLU << RTE_TABLE_ACTION_ENCAP_MPLS: 541 return sizeof(struct encap_mpls_data); 542 543 case 1LLU << RTE_TABLE_ACTION_ENCAP_PPPOE: 544 return sizeof(struct encap_pppoe_data); 545 546 case 1LLU << RTE_TABLE_ACTION_ENCAP_VXLAN: 547 if (encap->vxlan.ip_version) 548 if (encap->vxlan.vlan) 549 return sizeof(struct encap_vxlan_ipv4_vlan_data); 550 else 551 return sizeof(struct encap_vxlan_ipv4_data); 552 else 553 if (encap->vxlan.vlan) 554 return sizeof(struct encap_vxlan_ipv6_vlan_data); 555 else 556 return sizeof(struct encap_vxlan_ipv6_data); 557 558 case 1LLU << RTE_TABLE_ACTION_ENCAP_QINQ_PPPOE: 559 return sizeof(struct encap_qinq_pppoe_data); 560 561 default: 562 return 0; 563 } 564 } 565 566 static int 567 encap_apply_check(struct rte_table_action_encap_params *p, 568 struct rte_table_action_encap_config *cfg) 569 { 570 if ((encap_valid(p->type) == 0) || 571 ((cfg->encap_mask & (1LLU << p->type)) == 0)) 572 return -EINVAL; 573 574 switch (p->type) { 575 case RTE_TABLE_ACTION_ENCAP_ETHER: 576 return 0; 577 578 case RTE_TABLE_ACTION_ENCAP_VLAN: 579 return 0; 580 581 case RTE_TABLE_ACTION_ENCAP_QINQ: 582 return 0; 583 584 case RTE_TABLE_ACTION_ENCAP_MPLS: 585 if ((p->mpls.mpls_count == 0) || 586 (p->mpls.mpls_count > RTE_TABLE_ACTION_MPLS_LABELS_MAX)) 587 return -EINVAL; 588 589 return 0; 590 591 case RTE_TABLE_ACTION_ENCAP_PPPOE: 592 return 0; 593 594 case RTE_TABLE_ACTION_ENCAP_VXLAN: 595 return 0; 596 597 case RTE_TABLE_ACTION_ENCAP_QINQ_PPPOE: 598 return 0; 599 600 default: 601 return -EINVAL; 602 } 603 } 604 605 static int 606 encap_ether_apply(void *data, 607 struct rte_table_action_encap_params *p, 608 struct rte_table_action_common_config *common_cfg) 609 { 610 struct encap_ether_data *d = data; 611 uint16_t ethertype = (common_cfg->ip_version) ? 612 RTE_ETHER_TYPE_IPV4 : 613 RTE_ETHER_TYPE_IPV6; 614 615 /* Ethernet */ 616 rte_ether_addr_copy(&p->ether.ether.da, &d->ether.dst_addr); 617 rte_ether_addr_copy(&p->ether.ether.sa, &d->ether.src_addr); 618 d->ether.ether_type = rte_htons(ethertype); 619 620 return 0; 621 } 622 623 static int 624 encap_vlan_apply(void *data, 625 struct rte_table_action_encap_params *p, 626 struct rte_table_action_common_config *common_cfg) 627 { 628 struct encap_vlan_data *d = data; 629 uint16_t ethertype = (common_cfg->ip_version) ? 630 RTE_ETHER_TYPE_IPV4 : 631 RTE_ETHER_TYPE_IPV6; 632 633 /* Ethernet */ 634 rte_ether_addr_copy(&p->vlan.ether.da, &d->ether.dst_addr); 635 rte_ether_addr_copy(&p->vlan.ether.sa, &d->ether.src_addr); 636 d->ether.ether_type = rte_htons(RTE_ETHER_TYPE_VLAN); 637 638 /* VLAN */ 639 d->vlan.vlan_tci = rte_htons(VLAN(p->vlan.vlan.pcp, 640 p->vlan.vlan.dei, 641 p->vlan.vlan.vid)); 642 d->vlan.eth_proto = rte_htons(ethertype); 643 644 return 0; 645 } 646 647 static int 648 encap_qinq_apply(void *data, 649 struct rte_table_action_encap_params *p, 650 struct rte_table_action_common_config *common_cfg) 651 { 652 struct encap_qinq_data *d = data; 653 uint16_t ethertype = (common_cfg->ip_version) ? 654 RTE_ETHER_TYPE_IPV4 : 655 RTE_ETHER_TYPE_IPV6; 656 657 /* Ethernet */ 658 rte_ether_addr_copy(&p->qinq.ether.da, &d->ether.dst_addr); 659 rte_ether_addr_copy(&p->qinq.ether.sa, &d->ether.src_addr); 660 d->ether.ether_type = rte_htons(RTE_ETHER_TYPE_QINQ); 661 662 /* SVLAN */ 663 d->svlan.vlan_tci = rte_htons(VLAN(p->qinq.svlan.pcp, 664 p->qinq.svlan.dei, 665 p->qinq.svlan.vid)); 666 d->svlan.eth_proto = rte_htons(RTE_ETHER_TYPE_VLAN); 667 668 /* CVLAN */ 669 d->cvlan.vlan_tci = rte_htons(VLAN(p->qinq.cvlan.pcp, 670 p->qinq.cvlan.dei, 671 p->qinq.cvlan.vid)); 672 d->cvlan.eth_proto = rte_htons(ethertype); 673 674 return 0; 675 } 676 677 static int 678 encap_qinq_pppoe_apply(void *data, 679 struct rte_table_action_encap_params *p) 680 { 681 struct encap_qinq_pppoe_data *d = data; 682 683 /* Ethernet */ 684 rte_ether_addr_copy(&p->qinq.ether.da, &d->ether.dst_addr); 685 rte_ether_addr_copy(&p->qinq.ether.sa, &d->ether.src_addr); 686 d->ether.ether_type = rte_htons(RTE_ETHER_TYPE_VLAN); 687 688 /* SVLAN */ 689 d->svlan.vlan_tci = rte_htons(VLAN(p->qinq.svlan.pcp, 690 p->qinq.svlan.dei, 691 p->qinq.svlan.vid)); 692 d->svlan.eth_proto = rte_htons(RTE_ETHER_TYPE_VLAN); 693 694 /* CVLAN */ 695 d->cvlan.vlan_tci = rte_htons(VLAN(p->qinq.cvlan.pcp, 696 p->qinq.cvlan.dei, 697 p->qinq.cvlan.vid)); 698 d->cvlan.eth_proto = rte_htons(RTE_ETHER_TYPE_PPPOE_SESSION); 699 700 /* PPPoE and PPP*/ 701 d->pppoe_ppp.ver_type_code = rte_htons(0x1100); 702 d->pppoe_ppp.session_id = rte_htons(p->qinq_pppoe.pppoe.session_id); 703 d->pppoe_ppp.length = 0; /* not pre-computed */ 704 d->pppoe_ppp.protocol = rte_htons(PPP_PROTOCOL_IP); 705 706 return 0; 707 } 708 709 static int 710 encap_mpls_apply(void *data, 711 struct rte_table_action_encap_params *p) 712 { 713 struct encap_mpls_data *d = data; 714 uint16_t ethertype = (p->mpls.unicast) ? 715 ETHER_TYPE_MPLS_UNICAST : 716 ETHER_TYPE_MPLS_MULTICAST; 717 uint32_t i; 718 719 /* Ethernet */ 720 rte_ether_addr_copy(&p->mpls.ether.da, &d->ether.dst_addr); 721 rte_ether_addr_copy(&p->mpls.ether.sa, &d->ether.src_addr); 722 d->ether.ether_type = rte_htons(ethertype); 723 724 /* MPLS */ 725 for (i = 0; i < p->mpls.mpls_count - 1; i++) 726 d->mpls[i] = rte_htonl(MPLS(p->mpls.mpls[i].label, 727 p->mpls.mpls[i].tc, 728 0, 729 p->mpls.mpls[i].ttl)); 730 731 d->mpls[i] = rte_htonl(MPLS(p->mpls.mpls[i].label, 732 p->mpls.mpls[i].tc, 733 1, 734 p->mpls.mpls[i].ttl)); 735 736 d->mpls_count = p->mpls.mpls_count; 737 return 0; 738 } 739 740 static int 741 encap_pppoe_apply(void *data, 742 struct rte_table_action_encap_params *p) 743 { 744 struct encap_pppoe_data *d = data; 745 746 /* Ethernet */ 747 rte_ether_addr_copy(&p->pppoe.ether.da, &d->ether.dst_addr); 748 rte_ether_addr_copy(&p->pppoe.ether.sa, &d->ether.src_addr); 749 d->ether.ether_type = rte_htons(RTE_ETHER_TYPE_PPPOE_SESSION); 750 751 /* PPPoE and PPP*/ 752 d->pppoe_ppp.ver_type_code = rte_htons(0x1100); 753 d->pppoe_ppp.session_id = rte_htons(p->pppoe.pppoe.session_id); 754 d->pppoe_ppp.length = 0; /* not pre-computed */ 755 d->pppoe_ppp.protocol = rte_htons(PPP_PROTOCOL_IP); 756 757 return 0; 758 } 759 760 static int 761 encap_vxlan_apply(void *data, 762 struct rte_table_action_encap_params *p, 763 struct rte_table_action_encap_config *cfg) 764 { 765 if ((p->vxlan.vxlan.vni > 0xFFFFFF) || 766 (cfg->vxlan.ip_version && (p->vxlan.ipv4.dscp > 0x3F)) || 767 (!cfg->vxlan.ip_version && (p->vxlan.ipv6.flow_label > 0xFFFFF)) || 768 (!cfg->vxlan.ip_version && (p->vxlan.ipv6.dscp > 0x3F)) || 769 (cfg->vxlan.vlan && (p->vxlan.vlan.vid > 0xFFF))) 770 return -1; 771 772 if (cfg->vxlan.ip_version) 773 if (cfg->vxlan.vlan) { 774 struct encap_vxlan_ipv4_vlan_data *d = data; 775 776 /* Ethernet */ 777 rte_ether_addr_copy(&p->vxlan.ether.da, 778 &d->ether.dst_addr); 779 rte_ether_addr_copy(&p->vxlan.ether.sa, 780 &d->ether.src_addr); 781 d->ether.ether_type = rte_htons(RTE_ETHER_TYPE_VLAN); 782 783 /* VLAN */ 784 d->vlan.vlan_tci = rte_htons(VLAN(p->vxlan.vlan.pcp, 785 p->vxlan.vlan.dei, 786 p->vxlan.vlan.vid)); 787 d->vlan.eth_proto = rte_htons(RTE_ETHER_TYPE_IPV4); 788 789 /* IPv4*/ 790 d->ipv4.version_ihl = 0x45; 791 d->ipv4.type_of_service = p->vxlan.ipv4.dscp << 2; 792 d->ipv4.total_length = 0; /* not pre-computed */ 793 d->ipv4.packet_id = 0; 794 d->ipv4.fragment_offset = 0; 795 d->ipv4.time_to_live = p->vxlan.ipv4.ttl; 796 d->ipv4.next_proto_id = IP_PROTO_UDP; 797 d->ipv4.hdr_checksum = 0; 798 d->ipv4.src_addr = rte_htonl(p->vxlan.ipv4.sa); 799 d->ipv4.dst_addr = rte_htonl(p->vxlan.ipv4.da); 800 801 d->ipv4.hdr_checksum = rte_ipv4_cksum(&d->ipv4); 802 803 /* UDP */ 804 d->udp.src_port = rte_htons(p->vxlan.udp.sp); 805 d->udp.dst_port = rte_htons(p->vxlan.udp.dp); 806 d->udp.dgram_len = 0; /* not pre-computed */ 807 d->udp.dgram_cksum = 0; 808 809 /* VXLAN */ 810 d->vxlan.vx_flags = rte_htonl(0x08000000); 811 d->vxlan.vx_vni = rte_htonl(p->vxlan.vxlan.vni << 8); 812 813 return 0; 814 } else { 815 struct encap_vxlan_ipv4_data *d = data; 816 817 /* Ethernet */ 818 rte_ether_addr_copy(&p->vxlan.ether.da, 819 &d->ether.dst_addr); 820 rte_ether_addr_copy(&p->vxlan.ether.sa, 821 &d->ether.src_addr); 822 d->ether.ether_type = rte_htons(RTE_ETHER_TYPE_IPV4); 823 824 /* IPv4*/ 825 d->ipv4.version_ihl = 0x45; 826 d->ipv4.type_of_service = p->vxlan.ipv4.dscp << 2; 827 d->ipv4.total_length = 0; /* not pre-computed */ 828 d->ipv4.packet_id = 0; 829 d->ipv4.fragment_offset = 0; 830 d->ipv4.time_to_live = p->vxlan.ipv4.ttl; 831 d->ipv4.next_proto_id = IP_PROTO_UDP; 832 d->ipv4.hdr_checksum = 0; 833 d->ipv4.src_addr = rte_htonl(p->vxlan.ipv4.sa); 834 d->ipv4.dst_addr = rte_htonl(p->vxlan.ipv4.da); 835 836 d->ipv4.hdr_checksum = rte_ipv4_cksum(&d->ipv4); 837 838 /* UDP */ 839 d->udp.src_port = rte_htons(p->vxlan.udp.sp); 840 d->udp.dst_port = rte_htons(p->vxlan.udp.dp); 841 d->udp.dgram_len = 0; /* not pre-computed */ 842 d->udp.dgram_cksum = 0; 843 844 /* VXLAN */ 845 d->vxlan.vx_flags = rte_htonl(0x08000000); 846 d->vxlan.vx_vni = rte_htonl(p->vxlan.vxlan.vni << 8); 847 848 return 0; 849 } 850 else 851 if (cfg->vxlan.vlan) { 852 struct encap_vxlan_ipv6_vlan_data *d = data; 853 854 /* Ethernet */ 855 rte_ether_addr_copy(&p->vxlan.ether.da, 856 &d->ether.dst_addr); 857 rte_ether_addr_copy(&p->vxlan.ether.sa, 858 &d->ether.src_addr); 859 d->ether.ether_type = rte_htons(RTE_ETHER_TYPE_VLAN); 860 861 /* VLAN */ 862 d->vlan.vlan_tci = rte_htons(VLAN(p->vxlan.vlan.pcp, 863 p->vxlan.vlan.dei, 864 p->vxlan.vlan.vid)); 865 d->vlan.eth_proto = rte_htons(RTE_ETHER_TYPE_IPV6); 866 867 /* IPv6*/ 868 d->ipv6.vtc_flow = rte_htonl((6 << 28) | 869 (p->vxlan.ipv6.dscp << 22) | 870 p->vxlan.ipv6.flow_label); 871 d->ipv6.payload_len = 0; /* not pre-computed */ 872 d->ipv6.proto = IP_PROTO_UDP; 873 d->ipv6.hop_limits = p->vxlan.ipv6.hop_limit; 874 memcpy(d->ipv6.src_addr, 875 p->vxlan.ipv6.sa, 876 sizeof(p->vxlan.ipv6.sa)); 877 memcpy(d->ipv6.dst_addr, 878 p->vxlan.ipv6.da, 879 sizeof(p->vxlan.ipv6.da)); 880 881 /* UDP */ 882 d->udp.src_port = rte_htons(p->vxlan.udp.sp); 883 d->udp.dst_port = rte_htons(p->vxlan.udp.dp); 884 d->udp.dgram_len = 0; /* not pre-computed */ 885 d->udp.dgram_cksum = 0; 886 887 /* VXLAN */ 888 d->vxlan.vx_flags = rte_htonl(0x08000000); 889 d->vxlan.vx_vni = rte_htonl(p->vxlan.vxlan.vni << 8); 890 891 return 0; 892 } else { 893 struct encap_vxlan_ipv6_data *d = data; 894 895 /* Ethernet */ 896 rte_ether_addr_copy(&p->vxlan.ether.da, 897 &d->ether.dst_addr); 898 rte_ether_addr_copy(&p->vxlan.ether.sa, 899 &d->ether.src_addr); 900 d->ether.ether_type = rte_htons(RTE_ETHER_TYPE_IPV6); 901 902 /* IPv6*/ 903 d->ipv6.vtc_flow = rte_htonl((6 << 28) | 904 (p->vxlan.ipv6.dscp << 22) | 905 p->vxlan.ipv6.flow_label); 906 d->ipv6.payload_len = 0; /* not pre-computed */ 907 d->ipv6.proto = IP_PROTO_UDP; 908 d->ipv6.hop_limits = p->vxlan.ipv6.hop_limit; 909 memcpy(d->ipv6.src_addr, 910 p->vxlan.ipv6.sa, 911 sizeof(p->vxlan.ipv6.sa)); 912 memcpy(d->ipv6.dst_addr, 913 p->vxlan.ipv6.da, 914 sizeof(p->vxlan.ipv6.da)); 915 916 /* UDP */ 917 d->udp.src_port = rte_htons(p->vxlan.udp.sp); 918 d->udp.dst_port = rte_htons(p->vxlan.udp.dp); 919 d->udp.dgram_len = 0; /* not pre-computed */ 920 d->udp.dgram_cksum = 0; 921 922 /* VXLAN */ 923 d->vxlan.vx_flags = rte_htonl(0x08000000); 924 d->vxlan.vx_vni = rte_htonl(p->vxlan.vxlan.vni << 8); 925 926 return 0; 927 } 928 } 929 930 static int 931 encap_apply(void *data, 932 struct rte_table_action_encap_params *p, 933 struct rte_table_action_encap_config *cfg, 934 struct rte_table_action_common_config *common_cfg) 935 { 936 int status; 937 938 /* Check input arguments */ 939 status = encap_apply_check(p, cfg); 940 if (status) 941 return status; 942 943 switch (p->type) { 944 case RTE_TABLE_ACTION_ENCAP_ETHER: 945 return encap_ether_apply(data, p, common_cfg); 946 947 case RTE_TABLE_ACTION_ENCAP_VLAN: 948 return encap_vlan_apply(data, p, common_cfg); 949 950 case RTE_TABLE_ACTION_ENCAP_QINQ: 951 return encap_qinq_apply(data, p, common_cfg); 952 953 case RTE_TABLE_ACTION_ENCAP_MPLS: 954 return encap_mpls_apply(data, p); 955 956 case RTE_TABLE_ACTION_ENCAP_PPPOE: 957 return encap_pppoe_apply(data, p); 958 959 case RTE_TABLE_ACTION_ENCAP_VXLAN: 960 return encap_vxlan_apply(data, p, cfg); 961 962 case RTE_TABLE_ACTION_ENCAP_QINQ_PPPOE: 963 return encap_qinq_pppoe_apply(data, p); 964 965 default: 966 return -EINVAL; 967 } 968 } 969 970 static __rte_always_inline uint16_t 971 encap_vxlan_ipv4_checksum_update(uint16_t cksum0, 972 uint16_t total_length) 973 { 974 int32_t cksum1; 975 976 cksum1 = cksum0; 977 cksum1 = ~cksum1 & 0xFFFF; 978 979 /* Add total length (one's complement logic) */ 980 cksum1 += total_length; 981 cksum1 = (cksum1 & 0xFFFF) + (cksum1 >> 16); 982 cksum1 = (cksum1 & 0xFFFF) + (cksum1 >> 16); 983 984 return (uint16_t)(~cksum1); 985 } 986 987 static __rte_always_inline void * 988 encap(void *dst, const void *src, size_t n) 989 { 990 dst = ((uint8_t *) dst) - n; 991 return rte_memcpy(dst, src, n); 992 } 993 994 static __rte_always_inline void 995 pkt_work_encap_vxlan_ipv4(struct rte_mbuf *mbuf, 996 struct encap_vxlan_ipv4_data *vxlan_tbl, 997 struct rte_table_action_encap_config *cfg) 998 { 999 uint32_t ether_offset = cfg->vxlan.data_offset; 1000 void *ether = RTE_MBUF_METADATA_UINT32_PTR(mbuf, ether_offset); 1001 struct encap_vxlan_ipv4_data *vxlan_pkt; 1002 uint16_t ether_length, ipv4_total_length, ipv4_hdr_cksum, udp_length; 1003 1004 ether_length = (uint16_t)mbuf->pkt_len; 1005 ipv4_total_length = ether_length + 1006 (sizeof(struct rte_vxlan_hdr) + 1007 sizeof(struct rte_udp_hdr) + 1008 sizeof(struct rte_ipv4_hdr)); 1009 ipv4_hdr_cksum = encap_vxlan_ipv4_checksum_update(vxlan_tbl->ipv4.hdr_checksum, 1010 rte_htons(ipv4_total_length)); 1011 udp_length = ether_length + 1012 (sizeof(struct rte_vxlan_hdr) + 1013 sizeof(struct rte_udp_hdr)); 1014 1015 vxlan_pkt = encap(ether, vxlan_tbl, sizeof(*vxlan_tbl)); 1016 vxlan_pkt->ipv4.total_length = rte_htons(ipv4_total_length); 1017 vxlan_pkt->ipv4.hdr_checksum = ipv4_hdr_cksum; 1018 vxlan_pkt->udp.dgram_len = rte_htons(udp_length); 1019 1020 mbuf->data_off = ether_offset - (sizeof(struct rte_mbuf) + sizeof(*vxlan_pkt)); 1021 mbuf->pkt_len = mbuf->data_len = ether_length + sizeof(*vxlan_pkt); 1022 } 1023 1024 static __rte_always_inline void 1025 pkt_work_encap_vxlan_ipv4_vlan(struct rte_mbuf *mbuf, 1026 struct encap_vxlan_ipv4_vlan_data *vxlan_tbl, 1027 struct rte_table_action_encap_config *cfg) 1028 { 1029 uint32_t ether_offset = cfg->vxlan.data_offset; 1030 void *ether = RTE_MBUF_METADATA_UINT32_PTR(mbuf, ether_offset); 1031 struct encap_vxlan_ipv4_vlan_data *vxlan_pkt; 1032 uint16_t ether_length, ipv4_total_length, ipv4_hdr_cksum, udp_length; 1033 1034 ether_length = (uint16_t)mbuf->pkt_len; 1035 ipv4_total_length = ether_length + 1036 (sizeof(struct rte_vxlan_hdr) + 1037 sizeof(struct rte_udp_hdr) + 1038 sizeof(struct rte_ipv4_hdr)); 1039 ipv4_hdr_cksum = encap_vxlan_ipv4_checksum_update(vxlan_tbl->ipv4.hdr_checksum, 1040 rte_htons(ipv4_total_length)); 1041 udp_length = ether_length + 1042 (sizeof(struct rte_vxlan_hdr) + 1043 sizeof(struct rte_udp_hdr)); 1044 1045 vxlan_pkt = encap(ether, vxlan_tbl, sizeof(*vxlan_tbl)); 1046 vxlan_pkt->ipv4.total_length = rte_htons(ipv4_total_length); 1047 vxlan_pkt->ipv4.hdr_checksum = ipv4_hdr_cksum; 1048 vxlan_pkt->udp.dgram_len = rte_htons(udp_length); 1049 1050 mbuf->data_off = ether_offset - (sizeof(struct rte_mbuf) + sizeof(*vxlan_pkt)); 1051 mbuf->pkt_len = mbuf->data_len = ether_length + sizeof(*vxlan_pkt); 1052 } 1053 1054 static __rte_always_inline void 1055 pkt_work_encap_vxlan_ipv6(struct rte_mbuf *mbuf, 1056 struct encap_vxlan_ipv6_data *vxlan_tbl, 1057 struct rte_table_action_encap_config *cfg) 1058 { 1059 uint32_t ether_offset = cfg->vxlan.data_offset; 1060 void *ether = RTE_MBUF_METADATA_UINT32_PTR(mbuf, ether_offset); 1061 struct encap_vxlan_ipv6_data *vxlan_pkt; 1062 uint16_t ether_length, ipv6_payload_length, udp_length; 1063 1064 ether_length = (uint16_t)mbuf->pkt_len; 1065 ipv6_payload_length = ether_length + 1066 (sizeof(struct rte_vxlan_hdr) + 1067 sizeof(struct rte_udp_hdr)); 1068 udp_length = ether_length + 1069 (sizeof(struct rte_vxlan_hdr) + 1070 sizeof(struct rte_udp_hdr)); 1071 1072 vxlan_pkt = encap(ether, vxlan_tbl, sizeof(*vxlan_tbl)); 1073 vxlan_pkt->ipv6.payload_len = rte_htons(ipv6_payload_length); 1074 vxlan_pkt->udp.dgram_len = rte_htons(udp_length); 1075 1076 mbuf->data_off = ether_offset - (sizeof(struct rte_mbuf) + sizeof(*vxlan_pkt)); 1077 mbuf->pkt_len = mbuf->data_len = ether_length + sizeof(*vxlan_pkt); 1078 } 1079 1080 static __rte_always_inline void 1081 pkt_work_encap_vxlan_ipv6_vlan(struct rte_mbuf *mbuf, 1082 struct encap_vxlan_ipv6_vlan_data *vxlan_tbl, 1083 struct rte_table_action_encap_config *cfg) 1084 { 1085 uint32_t ether_offset = cfg->vxlan.data_offset; 1086 void *ether = RTE_MBUF_METADATA_UINT32_PTR(mbuf, ether_offset); 1087 struct encap_vxlan_ipv6_vlan_data *vxlan_pkt; 1088 uint16_t ether_length, ipv6_payload_length, udp_length; 1089 1090 ether_length = (uint16_t)mbuf->pkt_len; 1091 ipv6_payload_length = ether_length + 1092 (sizeof(struct rte_vxlan_hdr) + 1093 sizeof(struct rte_udp_hdr)); 1094 udp_length = ether_length + 1095 (sizeof(struct rte_vxlan_hdr) + 1096 sizeof(struct rte_udp_hdr)); 1097 1098 vxlan_pkt = encap(ether, vxlan_tbl, sizeof(*vxlan_tbl)); 1099 vxlan_pkt->ipv6.payload_len = rte_htons(ipv6_payload_length); 1100 vxlan_pkt->udp.dgram_len = rte_htons(udp_length); 1101 1102 mbuf->data_off = ether_offset - (sizeof(struct rte_mbuf) + sizeof(*vxlan_pkt)); 1103 mbuf->pkt_len = mbuf->data_len = ether_length + sizeof(*vxlan_pkt); 1104 } 1105 1106 static __rte_always_inline void 1107 pkt_work_encap(struct rte_mbuf *mbuf, 1108 void *data, 1109 struct rte_table_action_encap_config *cfg, 1110 void *ip, 1111 uint16_t total_length, 1112 uint32_t ip_offset) 1113 { 1114 switch (cfg->encap_mask) { 1115 case 1LLU << RTE_TABLE_ACTION_ENCAP_ETHER: 1116 encap(ip, data, sizeof(struct encap_ether_data)); 1117 mbuf->data_off = ip_offset - (sizeof(struct rte_mbuf) + 1118 sizeof(struct encap_ether_data)); 1119 mbuf->pkt_len = mbuf->data_len = total_length + 1120 sizeof(struct encap_ether_data); 1121 break; 1122 1123 case 1LLU << RTE_TABLE_ACTION_ENCAP_VLAN: 1124 encap(ip, data, sizeof(struct encap_vlan_data)); 1125 mbuf->data_off = ip_offset - (sizeof(struct rte_mbuf) + 1126 sizeof(struct encap_vlan_data)); 1127 mbuf->pkt_len = mbuf->data_len = total_length + 1128 sizeof(struct encap_vlan_data); 1129 break; 1130 1131 case 1LLU << RTE_TABLE_ACTION_ENCAP_QINQ: 1132 encap(ip, data, sizeof(struct encap_qinq_data)); 1133 mbuf->data_off = ip_offset - (sizeof(struct rte_mbuf) + 1134 sizeof(struct encap_qinq_data)); 1135 mbuf->pkt_len = mbuf->data_len = total_length + 1136 sizeof(struct encap_qinq_data); 1137 break; 1138 1139 case 1LLU << RTE_TABLE_ACTION_ENCAP_MPLS: 1140 { 1141 struct encap_mpls_data *mpls = data; 1142 size_t size = sizeof(struct rte_ether_hdr) + 1143 mpls->mpls_count * 4; 1144 1145 encap(ip, data, size); 1146 mbuf->data_off = ip_offset - (sizeof(struct rte_mbuf) + size); 1147 mbuf->pkt_len = mbuf->data_len = total_length + size; 1148 break; 1149 } 1150 1151 case 1LLU << RTE_TABLE_ACTION_ENCAP_PPPOE: 1152 { 1153 struct encap_pppoe_data *pppoe = 1154 encap(ip, data, sizeof(struct encap_pppoe_data)); 1155 pppoe->pppoe_ppp.length = rte_htons(total_length + 2); 1156 mbuf->data_off = ip_offset - (sizeof(struct rte_mbuf) + 1157 sizeof(struct encap_pppoe_data)); 1158 mbuf->pkt_len = mbuf->data_len = total_length + 1159 sizeof(struct encap_pppoe_data); 1160 break; 1161 } 1162 1163 case 1LLU << RTE_TABLE_ACTION_ENCAP_QINQ_PPPOE: 1164 { 1165 struct encap_qinq_pppoe_data *qinq_pppoe = 1166 encap(ip, data, sizeof(struct encap_qinq_pppoe_data)); 1167 qinq_pppoe->pppoe_ppp.length = rte_htons(total_length + 2); 1168 mbuf->data_off = ip_offset - (sizeof(struct rte_mbuf) + 1169 sizeof(struct encap_qinq_pppoe_data)); 1170 mbuf->pkt_len = mbuf->data_len = total_length + 1171 sizeof(struct encap_qinq_pppoe_data); 1172 break; 1173 } 1174 1175 case 1LLU << RTE_TABLE_ACTION_ENCAP_VXLAN: 1176 { 1177 if (cfg->vxlan.ip_version) 1178 if (cfg->vxlan.vlan) 1179 pkt_work_encap_vxlan_ipv4_vlan(mbuf, data, cfg); 1180 else 1181 pkt_work_encap_vxlan_ipv4(mbuf, data, cfg); 1182 else 1183 if (cfg->vxlan.vlan) 1184 pkt_work_encap_vxlan_ipv6_vlan(mbuf, data, cfg); 1185 else 1186 pkt_work_encap_vxlan_ipv6(mbuf, data, cfg); 1187 } 1188 1189 default: 1190 break; 1191 } 1192 } 1193 1194 /** 1195 * RTE_TABLE_ACTION_NAT 1196 */ 1197 static int 1198 nat_cfg_check(struct rte_table_action_nat_config *nat) 1199 { 1200 if ((nat->proto != 0x06) && 1201 (nat->proto != 0x11)) 1202 return -ENOTSUP; 1203 1204 return 0; 1205 } 1206 1207 struct nat_ipv4_data { 1208 uint32_t addr; 1209 uint16_t port; 1210 } __rte_packed; 1211 1212 struct nat_ipv6_data { 1213 uint8_t addr[16]; 1214 uint16_t port; 1215 } __rte_packed; 1216 1217 static size_t 1218 nat_data_size(struct rte_table_action_nat_config *nat __rte_unused, 1219 struct rte_table_action_common_config *common) 1220 { 1221 int ip_version = common->ip_version; 1222 1223 return (ip_version) ? 1224 sizeof(struct nat_ipv4_data) : 1225 sizeof(struct nat_ipv6_data); 1226 } 1227 1228 static int 1229 nat_apply_check(struct rte_table_action_nat_params *p, 1230 struct rte_table_action_common_config *cfg) 1231 { 1232 if ((p->ip_version && (cfg->ip_version == 0)) || 1233 ((p->ip_version == 0) && cfg->ip_version)) 1234 return -EINVAL; 1235 1236 return 0; 1237 } 1238 1239 static int 1240 nat_apply(void *data, 1241 struct rte_table_action_nat_params *p, 1242 struct rte_table_action_common_config *cfg) 1243 { 1244 int status; 1245 1246 /* Check input arguments */ 1247 status = nat_apply_check(p, cfg); 1248 if (status) 1249 return status; 1250 1251 /* Apply */ 1252 if (p->ip_version) { 1253 struct nat_ipv4_data *d = data; 1254 1255 d->addr = rte_htonl(p->addr.ipv4); 1256 d->port = rte_htons(p->port); 1257 } else { 1258 struct nat_ipv6_data *d = data; 1259 1260 memcpy(d->addr, p->addr.ipv6, sizeof(d->addr)); 1261 d->port = rte_htons(p->port); 1262 } 1263 1264 return 0; 1265 } 1266 1267 static __rte_always_inline uint16_t 1268 nat_ipv4_checksum_update(uint16_t cksum0, 1269 uint32_t ip0, 1270 uint32_t ip1) 1271 { 1272 int32_t cksum1; 1273 1274 cksum1 = cksum0; 1275 cksum1 = ~cksum1 & 0xFFFF; 1276 1277 /* Subtract ip0 (one's complement logic) */ 1278 cksum1 -= (ip0 >> 16) + (ip0 & 0xFFFF); 1279 cksum1 = (cksum1 & 0xFFFF) + (cksum1 >> 16); 1280 cksum1 = (cksum1 & 0xFFFF) + (cksum1 >> 16); 1281 1282 /* Add ip1 (one's complement logic) */ 1283 cksum1 += (ip1 >> 16) + (ip1 & 0xFFFF); 1284 cksum1 = (cksum1 & 0xFFFF) + (cksum1 >> 16); 1285 cksum1 = (cksum1 & 0xFFFF) + (cksum1 >> 16); 1286 1287 return (uint16_t)(~cksum1); 1288 } 1289 1290 static __rte_always_inline uint16_t 1291 nat_ipv4_tcp_udp_checksum_update(uint16_t cksum0, 1292 uint32_t ip0, 1293 uint32_t ip1, 1294 uint16_t port0, 1295 uint16_t port1) 1296 { 1297 int32_t cksum1; 1298 1299 cksum1 = cksum0; 1300 cksum1 = ~cksum1 & 0xFFFF; 1301 1302 /* Subtract ip0 and port 0 (one's complement logic) */ 1303 cksum1 -= (ip0 >> 16) + (ip0 & 0xFFFF) + port0; 1304 cksum1 = (cksum1 & 0xFFFF) + (cksum1 >> 16); 1305 cksum1 = (cksum1 & 0xFFFF) + (cksum1 >> 16); 1306 1307 /* Add ip1 and port1 (one's complement logic) */ 1308 cksum1 += (ip1 >> 16) + (ip1 & 0xFFFF) + port1; 1309 cksum1 = (cksum1 & 0xFFFF) + (cksum1 >> 16); 1310 cksum1 = (cksum1 & 0xFFFF) + (cksum1 >> 16); 1311 1312 return (uint16_t)(~cksum1); 1313 } 1314 1315 static __rte_always_inline uint16_t 1316 nat_ipv6_tcp_udp_checksum_update(uint16_t cksum0, 1317 uint16_t *ip0, 1318 uint16_t *ip1, 1319 uint16_t port0, 1320 uint16_t port1) 1321 { 1322 int32_t cksum1; 1323 1324 cksum1 = cksum0; 1325 cksum1 = ~cksum1 & 0xFFFF; 1326 1327 /* Subtract ip0 and port 0 (one's complement logic) */ 1328 cksum1 -= ip0[0] + ip0[1] + ip0[2] + ip0[3] + 1329 ip0[4] + ip0[5] + ip0[6] + ip0[7] + port0; 1330 cksum1 = (cksum1 & 0xFFFF) + (cksum1 >> 16); 1331 cksum1 = (cksum1 & 0xFFFF) + (cksum1 >> 16); 1332 1333 /* Add ip1 and port1 (one's complement logic) */ 1334 cksum1 += ip1[0] + ip1[1] + ip1[2] + ip1[3] + 1335 ip1[4] + ip1[5] + ip1[6] + ip1[7] + port1; 1336 cksum1 = (cksum1 & 0xFFFF) + (cksum1 >> 16); 1337 cksum1 = (cksum1 & 0xFFFF) + (cksum1 >> 16); 1338 1339 return (uint16_t)(~cksum1); 1340 } 1341 1342 static __rte_always_inline void 1343 pkt_ipv4_work_nat(struct rte_ipv4_hdr *ip, 1344 struct nat_ipv4_data *data, 1345 struct rte_table_action_nat_config *cfg) 1346 { 1347 if (cfg->source_nat) { 1348 if (cfg->proto == 0x6) { 1349 struct rte_tcp_hdr *tcp = (struct rte_tcp_hdr *) &ip[1]; 1350 uint16_t ip_cksum, tcp_cksum; 1351 1352 ip_cksum = nat_ipv4_checksum_update(ip->hdr_checksum, 1353 ip->src_addr, 1354 data->addr); 1355 1356 tcp_cksum = nat_ipv4_tcp_udp_checksum_update(tcp->cksum, 1357 ip->src_addr, 1358 data->addr, 1359 tcp->src_port, 1360 data->port); 1361 1362 ip->src_addr = data->addr; 1363 ip->hdr_checksum = ip_cksum; 1364 tcp->src_port = data->port; 1365 tcp->cksum = tcp_cksum; 1366 } else { 1367 struct rte_udp_hdr *udp = (struct rte_udp_hdr *) &ip[1]; 1368 uint16_t ip_cksum, udp_cksum; 1369 1370 ip_cksum = nat_ipv4_checksum_update(ip->hdr_checksum, 1371 ip->src_addr, 1372 data->addr); 1373 1374 udp_cksum = nat_ipv4_tcp_udp_checksum_update(udp->dgram_cksum, 1375 ip->src_addr, 1376 data->addr, 1377 udp->src_port, 1378 data->port); 1379 1380 ip->src_addr = data->addr; 1381 ip->hdr_checksum = ip_cksum; 1382 udp->src_port = data->port; 1383 if (udp->dgram_cksum) 1384 udp->dgram_cksum = udp_cksum; 1385 } 1386 } else { 1387 if (cfg->proto == 0x6) { 1388 struct rte_tcp_hdr *tcp = (struct rte_tcp_hdr *) &ip[1]; 1389 uint16_t ip_cksum, tcp_cksum; 1390 1391 ip_cksum = nat_ipv4_checksum_update(ip->hdr_checksum, 1392 ip->dst_addr, 1393 data->addr); 1394 1395 tcp_cksum = nat_ipv4_tcp_udp_checksum_update(tcp->cksum, 1396 ip->dst_addr, 1397 data->addr, 1398 tcp->dst_port, 1399 data->port); 1400 1401 ip->dst_addr = data->addr; 1402 ip->hdr_checksum = ip_cksum; 1403 tcp->dst_port = data->port; 1404 tcp->cksum = tcp_cksum; 1405 } else { 1406 struct rte_udp_hdr *udp = (struct rte_udp_hdr *) &ip[1]; 1407 uint16_t ip_cksum, udp_cksum; 1408 1409 ip_cksum = nat_ipv4_checksum_update(ip->hdr_checksum, 1410 ip->dst_addr, 1411 data->addr); 1412 1413 udp_cksum = nat_ipv4_tcp_udp_checksum_update(udp->dgram_cksum, 1414 ip->dst_addr, 1415 data->addr, 1416 udp->dst_port, 1417 data->port); 1418 1419 ip->dst_addr = data->addr; 1420 ip->hdr_checksum = ip_cksum; 1421 udp->dst_port = data->port; 1422 if (udp->dgram_cksum) 1423 udp->dgram_cksum = udp_cksum; 1424 } 1425 } 1426 } 1427 1428 static __rte_always_inline void 1429 pkt_ipv6_work_nat(struct rte_ipv6_hdr *ip, 1430 struct nat_ipv6_data *data, 1431 struct rte_table_action_nat_config *cfg) 1432 { 1433 if (cfg->source_nat) { 1434 if (cfg->proto == 0x6) { 1435 struct rte_tcp_hdr *tcp = (struct rte_tcp_hdr *) &ip[1]; 1436 uint16_t tcp_cksum; 1437 1438 tcp_cksum = nat_ipv6_tcp_udp_checksum_update(tcp->cksum, 1439 (uint16_t *)ip->src_addr, 1440 (uint16_t *)data->addr, 1441 tcp->src_port, 1442 data->port); 1443 1444 rte_memcpy(ip->src_addr, data->addr, 16); 1445 tcp->src_port = data->port; 1446 tcp->cksum = tcp_cksum; 1447 } else { 1448 struct rte_udp_hdr *udp = (struct rte_udp_hdr *) &ip[1]; 1449 uint16_t udp_cksum; 1450 1451 udp_cksum = nat_ipv6_tcp_udp_checksum_update(udp->dgram_cksum, 1452 (uint16_t *)ip->src_addr, 1453 (uint16_t *)data->addr, 1454 udp->src_port, 1455 data->port); 1456 1457 rte_memcpy(ip->src_addr, data->addr, 16); 1458 udp->src_port = data->port; 1459 udp->dgram_cksum = udp_cksum; 1460 } 1461 } else { 1462 if (cfg->proto == 0x6) { 1463 struct rte_tcp_hdr *tcp = (struct rte_tcp_hdr *) &ip[1]; 1464 uint16_t tcp_cksum; 1465 1466 tcp_cksum = nat_ipv6_tcp_udp_checksum_update(tcp->cksum, 1467 (uint16_t *)ip->dst_addr, 1468 (uint16_t *)data->addr, 1469 tcp->dst_port, 1470 data->port); 1471 1472 rte_memcpy(ip->dst_addr, data->addr, 16); 1473 tcp->dst_port = data->port; 1474 tcp->cksum = tcp_cksum; 1475 } else { 1476 struct rte_udp_hdr *udp = (struct rte_udp_hdr *) &ip[1]; 1477 uint16_t udp_cksum; 1478 1479 udp_cksum = nat_ipv6_tcp_udp_checksum_update(udp->dgram_cksum, 1480 (uint16_t *)ip->dst_addr, 1481 (uint16_t *)data->addr, 1482 udp->dst_port, 1483 data->port); 1484 1485 rte_memcpy(ip->dst_addr, data->addr, 16); 1486 udp->dst_port = data->port; 1487 udp->dgram_cksum = udp_cksum; 1488 } 1489 } 1490 } 1491 1492 /** 1493 * RTE_TABLE_ACTION_TTL 1494 */ 1495 static int 1496 ttl_cfg_check(struct rte_table_action_ttl_config *ttl) 1497 { 1498 if (ttl->drop == 0) 1499 return -ENOTSUP; 1500 1501 return 0; 1502 } 1503 1504 struct ttl_data { 1505 uint32_t n_packets; 1506 } __rte_packed; 1507 1508 #define TTL_INIT(data, decrement) \ 1509 ((data)->n_packets = (decrement) ? 1 : 0) 1510 1511 #define TTL_DEC_GET(data) \ 1512 ((uint8_t)((data)->n_packets & 1)) 1513 1514 #define TTL_STATS_RESET(data) \ 1515 ((data)->n_packets = ((data)->n_packets & 1)) 1516 1517 #define TTL_STATS_READ(data) \ 1518 ((data)->n_packets >> 1) 1519 1520 #define TTL_STATS_ADD(data, value) \ 1521 ((data)->n_packets = \ 1522 (((((data)->n_packets >> 1) + (value)) << 1) | \ 1523 ((data)->n_packets & 1))) 1524 1525 static int 1526 ttl_apply(void *data, 1527 struct rte_table_action_ttl_params *p) 1528 { 1529 struct ttl_data *d = data; 1530 1531 TTL_INIT(d, p->decrement); 1532 1533 return 0; 1534 } 1535 1536 static __rte_always_inline uint64_t 1537 pkt_ipv4_work_ttl(struct rte_ipv4_hdr *ip, 1538 struct ttl_data *data) 1539 { 1540 uint32_t drop; 1541 uint16_t cksum = ip->hdr_checksum; 1542 uint8_t ttl = ip->time_to_live; 1543 uint8_t ttl_diff = TTL_DEC_GET(data); 1544 1545 cksum += ttl_diff; 1546 ttl -= ttl_diff; 1547 1548 ip->hdr_checksum = cksum; 1549 ip->time_to_live = ttl; 1550 1551 drop = (ttl == 0) ? 1 : 0; 1552 TTL_STATS_ADD(data, drop); 1553 1554 return drop; 1555 } 1556 1557 static __rte_always_inline uint64_t 1558 pkt_ipv6_work_ttl(struct rte_ipv6_hdr *ip, 1559 struct ttl_data *data) 1560 { 1561 uint32_t drop; 1562 uint8_t ttl = ip->hop_limits; 1563 uint8_t ttl_diff = TTL_DEC_GET(data); 1564 1565 ttl -= ttl_diff; 1566 1567 ip->hop_limits = ttl; 1568 1569 drop = (ttl == 0) ? 1 : 0; 1570 TTL_STATS_ADD(data, drop); 1571 1572 return drop; 1573 } 1574 1575 /** 1576 * RTE_TABLE_ACTION_STATS 1577 */ 1578 static int 1579 stats_cfg_check(struct rte_table_action_stats_config *stats) 1580 { 1581 if ((stats->n_packets_enabled == 0) && (stats->n_bytes_enabled == 0)) 1582 return -EINVAL; 1583 1584 return 0; 1585 } 1586 1587 struct stats_data { 1588 uint64_t n_packets; 1589 uint64_t n_bytes; 1590 } __rte_packed; 1591 1592 static int 1593 stats_apply(struct stats_data *data, 1594 struct rte_table_action_stats_params *p) 1595 { 1596 data->n_packets = p->n_packets; 1597 data->n_bytes = p->n_bytes; 1598 1599 return 0; 1600 } 1601 1602 static __rte_always_inline void 1603 pkt_work_stats(struct stats_data *data, 1604 uint16_t total_length) 1605 { 1606 data->n_packets++; 1607 data->n_bytes += total_length; 1608 } 1609 1610 /** 1611 * RTE_TABLE_ACTION_TIME 1612 */ 1613 struct time_data { 1614 uint64_t time; 1615 } __rte_packed; 1616 1617 static int 1618 time_apply(struct time_data *data, 1619 struct rte_table_action_time_params *p) 1620 { 1621 data->time = p->time; 1622 return 0; 1623 } 1624 1625 static __rte_always_inline void 1626 pkt_work_time(struct time_data *data, 1627 uint64_t time) 1628 { 1629 data->time = time; 1630 } 1631 1632 1633 /** 1634 * RTE_TABLE_ACTION_CRYPTO 1635 */ 1636 1637 #define CRYPTO_OP_MASK_CIPHER 0x1 1638 #define CRYPTO_OP_MASK_AUTH 0x2 1639 #define CRYPTO_OP_MASK_AEAD 0x4 1640 1641 struct crypto_op_sym_iv_aad { 1642 struct rte_crypto_op op; 1643 struct rte_crypto_sym_op sym_op; 1644 union { 1645 struct { 1646 uint8_t cipher_iv[ 1647 RTE_TABLE_ACTION_SYM_CRYPTO_IV_SIZE_MAX]; 1648 uint8_t auth_iv[ 1649 RTE_TABLE_ACTION_SYM_CRYPTO_IV_SIZE_MAX]; 1650 } cipher_auth; 1651 1652 struct { 1653 uint8_t iv[RTE_TABLE_ACTION_SYM_CRYPTO_IV_SIZE_MAX]; 1654 uint8_t aad[RTE_TABLE_ACTION_SYM_CRYPTO_AAD_SIZE_MAX]; 1655 } aead_iv_aad; 1656 1657 } iv_aad; 1658 }; 1659 1660 struct sym_crypto_data { 1661 1662 union { 1663 struct { 1664 1665 /** Length of cipher iv. */ 1666 uint16_t cipher_iv_len; 1667 1668 /** Offset from start of IP header to the cipher iv. */ 1669 uint16_t cipher_iv_data_offset; 1670 1671 /** Length of cipher iv to be updated in the mbuf. */ 1672 uint16_t cipher_iv_update_len; 1673 1674 /** Offset from start of IP header to the auth iv. */ 1675 uint16_t auth_iv_data_offset; 1676 1677 /** Length of auth iv in the mbuf. */ 1678 uint16_t auth_iv_len; 1679 1680 /** Length of auth iv to be updated in the mbuf. */ 1681 uint16_t auth_iv_update_len; 1682 1683 } cipher_auth; 1684 struct { 1685 1686 /** Length of iv. */ 1687 uint16_t iv_len; 1688 1689 /** Offset from start of IP header to the aead iv. */ 1690 uint16_t iv_data_offset; 1691 1692 /** Length of iv to be updated in the mbuf. */ 1693 uint16_t iv_update_len; 1694 1695 /** Length of aad */ 1696 uint16_t aad_len; 1697 1698 /** Offset from start of IP header to the aad. */ 1699 uint16_t aad_data_offset; 1700 1701 /** Length of aad to updated in the mbuf. */ 1702 uint16_t aad_update_len; 1703 1704 } aead; 1705 }; 1706 1707 /** Offset from start of IP header to the data. */ 1708 uint16_t data_offset; 1709 1710 /** Digest length. */ 1711 uint16_t digest_len; 1712 1713 /** block size */ 1714 uint16_t block_size; 1715 1716 /** Mask of crypto operation */ 1717 uint16_t op_mask; 1718 1719 /** Session pointer. */ 1720 struct rte_cryptodev_sym_session *session; 1721 1722 /** Direction of crypto, encrypt or decrypt */ 1723 uint16_t direction; 1724 1725 /** Private data size to store cipher iv / aad. */ 1726 uint8_t iv_aad_data[32]; 1727 1728 } __rte_packed; 1729 1730 static int 1731 sym_crypto_cfg_check(struct rte_table_action_sym_crypto_config *cfg) 1732 { 1733 if (!rte_cryptodev_is_valid_dev(cfg->cryptodev_id)) 1734 return -EINVAL; 1735 if (cfg->mp_create == NULL || cfg->mp_init == NULL) 1736 return -EINVAL; 1737 1738 return 0; 1739 } 1740 1741 static int 1742 get_block_size(const struct rte_crypto_sym_xform *xform, uint8_t cdev_id) 1743 { 1744 struct rte_cryptodev_info dev_info; 1745 const struct rte_cryptodev_capabilities *cap; 1746 uint32_t i; 1747 1748 rte_cryptodev_info_get(cdev_id, &dev_info); 1749 1750 for (i = 0; dev_info.capabilities[i].op != RTE_CRYPTO_OP_TYPE_UNDEFINED; 1751 i++) { 1752 cap = &dev_info.capabilities[i]; 1753 1754 if (cap->sym.xform_type != xform->type) 1755 continue; 1756 1757 if ((xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) && 1758 (cap->sym.cipher.algo == xform->cipher.algo)) 1759 return cap->sym.cipher.block_size; 1760 1761 if ((xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) && 1762 (cap->sym.aead.algo == xform->aead.algo)) 1763 return cap->sym.aead.block_size; 1764 1765 if (xform->type == RTE_CRYPTO_SYM_XFORM_NOT_SPECIFIED) 1766 break; 1767 } 1768 1769 return -1; 1770 } 1771 1772 static int 1773 sym_crypto_apply(struct sym_crypto_data *data, 1774 struct rte_table_action_sym_crypto_config *cfg, 1775 struct rte_table_action_sym_crypto_params *p) 1776 { 1777 const struct rte_crypto_cipher_xform *cipher_xform = NULL; 1778 const struct rte_crypto_auth_xform *auth_xform = NULL; 1779 const struct rte_crypto_aead_xform *aead_xform = NULL; 1780 struct rte_crypto_sym_xform *xform = p->xform; 1781 struct rte_cryptodev_sym_session *session; 1782 int ret; 1783 1784 memset(data, 0, sizeof(*data)); 1785 1786 while (xform) { 1787 if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) { 1788 cipher_xform = &xform->cipher; 1789 1790 if (cipher_xform->iv.length > 1791 RTE_TABLE_ACTION_SYM_CRYPTO_IV_SIZE_MAX) 1792 return -ENOMEM; 1793 if (cipher_xform->iv.offset != 1794 RTE_TABLE_ACTION_SYM_CRYPTO_IV_OFFSET) 1795 return -EINVAL; 1796 1797 ret = get_block_size(xform, cfg->cryptodev_id); 1798 if (ret < 0) 1799 return -1; 1800 data->block_size = (uint16_t)ret; 1801 data->op_mask |= CRYPTO_OP_MASK_CIPHER; 1802 1803 data->cipher_auth.cipher_iv_len = 1804 cipher_xform->iv.length; 1805 data->cipher_auth.cipher_iv_data_offset = (uint16_t) 1806 p->cipher_auth.cipher_iv_update.offset; 1807 data->cipher_auth.cipher_iv_update_len = (uint16_t) 1808 p->cipher_auth.cipher_iv_update.length; 1809 1810 rte_memcpy(data->iv_aad_data, 1811 p->cipher_auth.cipher_iv.val, 1812 p->cipher_auth.cipher_iv.length); 1813 1814 data->direction = cipher_xform->op; 1815 1816 } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) { 1817 auth_xform = &xform->auth; 1818 if (auth_xform->iv.length > 1819 RTE_TABLE_ACTION_SYM_CRYPTO_IV_SIZE_MAX) 1820 return -ENOMEM; 1821 data->op_mask |= CRYPTO_OP_MASK_AUTH; 1822 1823 data->cipher_auth.auth_iv_len = auth_xform->iv.length; 1824 data->cipher_auth.auth_iv_data_offset = (uint16_t) 1825 p->cipher_auth.auth_iv_update.offset; 1826 data->cipher_auth.auth_iv_update_len = (uint16_t) 1827 p->cipher_auth.auth_iv_update.length; 1828 data->digest_len = auth_xform->digest_length; 1829 1830 data->direction = (auth_xform->op == 1831 RTE_CRYPTO_AUTH_OP_GENERATE) ? 1832 RTE_CRYPTO_CIPHER_OP_ENCRYPT : 1833 RTE_CRYPTO_CIPHER_OP_DECRYPT; 1834 1835 } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) { 1836 aead_xform = &xform->aead; 1837 1838 if ((aead_xform->iv.length > 1839 RTE_TABLE_ACTION_SYM_CRYPTO_IV_SIZE_MAX) || ( 1840 aead_xform->aad_length > 1841 RTE_TABLE_ACTION_SYM_CRYPTO_AAD_SIZE_MAX)) 1842 return -EINVAL; 1843 if (aead_xform->iv.offset != 1844 RTE_TABLE_ACTION_SYM_CRYPTO_IV_OFFSET) 1845 return -EINVAL; 1846 1847 ret = get_block_size(xform, cfg->cryptodev_id); 1848 if (ret < 0) 1849 return -1; 1850 data->block_size = (uint16_t)ret; 1851 data->op_mask |= CRYPTO_OP_MASK_AEAD; 1852 1853 data->digest_len = aead_xform->digest_length; 1854 data->aead.iv_len = aead_xform->iv.length; 1855 data->aead.aad_len = aead_xform->aad_length; 1856 1857 data->aead.iv_data_offset = (uint16_t) 1858 p->aead.iv_update.offset; 1859 data->aead.iv_update_len = (uint16_t) 1860 p->aead.iv_update.length; 1861 data->aead.aad_data_offset = (uint16_t) 1862 p->aead.aad_update.offset; 1863 data->aead.aad_update_len = (uint16_t) 1864 p->aead.aad_update.length; 1865 1866 rte_memcpy(data->iv_aad_data, 1867 p->aead.iv.val, 1868 p->aead.iv.length); 1869 1870 rte_memcpy(data->iv_aad_data + p->aead.iv.length, 1871 p->aead.aad.val, 1872 p->aead.aad.length); 1873 1874 data->direction = (aead_xform->op == 1875 RTE_CRYPTO_AEAD_OP_ENCRYPT) ? 1876 RTE_CRYPTO_CIPHER_OP_ENCRYPT : 1877 RTE_CRYPTO_CIPHER_OP_DECRYPT; 1878 } else 1879 return -EINVAL; 1880 1881 xform = xform->next; 1882 } 1883 1884 if (auth_xform && auth_xform->iv.length) { 1885 if (cipher_xform) { 1886 if (auth_xform->iv.offset != 1887 RTE_TABLE_ACTION_SYM_CRYPTO_IV_OFFSET + 1888 cipher_xform->iv.length) 1889 return -EINVAL; 1890 1891 rte_memcpy(data->iv_aad_data + cipher_xform->iv.length, 1892 p->cipher_auth.auth_iv.val, 1893 p->cipher_auth.auth_iv.length); 1894 } else { 1895 rte_memcpy(data->iv_aad_data, 1896 p->cipher_auth.auth_iv.val, 1897 p->cipher_auth.auth_iv.length); 1898 } 1899 } 1900 1901 session = rte_cryptodev_sym_session_create(cfg->cryptodev_id, 1902 p->xform, cfg->mp_create); 1903 if (!session) 1904 return -ENOMEM; 1905 1906 data->data_offset = (uint16_t)p->data_offset; 1907 data->session = session; 1908 1909 return 0; 1910 } 1911 1912 static __rte_always_inline uint64_t 1913 pkt_work_sym_crypto(struct rte_mbuf *mbuf, struct sym_crypto_data *data, 1914 struct rte_table_action_sym_crypto_config *cfg, 1915 uint16_t ip_offset) 1916 { 1917 struct crypto_op_sym_iv_aad *crypto_op = (struct crypto_op_sym_iv_aad *) 1918 RTE_MBUF_METADATA_UINT8_PTR(mbuf, cfg->op_offset); 1919 struct rte_crypto_op *op = &crypto_op->op; 1920 struct rte_crypto_sym_op *sym = op->sym; 1921 uint32_t pkt_offset = sizeof(*mbuf) + mbuf->data_off; 1922 uint32_t payload_len = pkt_offset + mbuf->data_len - data->data_offset; 1923 1924 op->type = RTE_CRYPTO_OP_TYPE_SYMMETRIC; 1925 op->sess_type = RTE_CRYPTO_OP_WITH_SESSION; 1926 op->phys_addr = rte_mbuf_iova_get(mbuf) + cfg->op_offset - sizeof(*mbuf); 1927 op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 1928 sym->m_src = mbuf; 1929 sym->m_dst = NULL; 1930 sym->session = data->session; 1931 1932 /** pad the packet */ 1933 if (data->direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { 1934 uint32_t append_len = RTE_ALIGN_CEIL(payload_len, 1935 data->block_size) - payload_len; 1936 1937 if (unlikely(rte_pktmbuf_append(mbuf, append_len + 1938 data->digest_len) == NULL)) 1939 return 1; 1940 1941 payload_len += append_len; 1942 } else 1943 payload_len -= data->digest_len; 1944 1945 if (data->op_mask & CRYPTO_OP_MASK_CIPHER) { 1946 /** prepare cipher op */ 1947 uint8_t *iv = crypto_op->iv_aad.cipher_auth.cipher_iv; 1948 1949 sym->cipher.data.length = payload_len; 1950 sym->cipher.data.offset = data->data_offset - pkt_offset; 1951 1952 if (data->cipher_auth.cipher_iv_update_len) { 1953 uint8_t *pkt_iv = RTE_MBUF_METADATA_UINT8_PTR(mbuf, 1954 data->cipher_auth.cipher_iv_data_offset 1955 + ip_offset); 1956 1957 /** For encryption, update the pkt iv field, otherwise 1958 * update the iv_aad_field 1959 */ 1960 if (data->direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) 1961 rte_memcpy(pkt_iv, data->iv_aad_data, 1962 data->cipher_auth.cipher_iv_update_len); 1963 else 1964 rte_memcpy(data->iv_aad_data, pkt_iv, 1965 data->cipher_auth.cipher_iv_update_len); 1966 } 1967 1968 /** write iv */ 1969 rte_memcpy(iv, data->iv_aad_data, 1970 data->cipher_auth.cipher_iv_len); 1971 } 1972 1973 if (data->op_mask & CRYPTO_OP_MASK_AUTH) { 1974 /** authentication always start from IP header. */ 1975 sym->auth.data.offset = ip_offset - pkt_offset; 1976 sym->auth.data.length = mbuf->data_len - sym->auth.data.offset - 1977 data->digest_len; 1978 sym->auth.digest.data = rte_pktmbuf_mtod_offset(mbuf, 1979 uint8_t *, rte_pktmbuf_pkt_len(mbuf) - 1980 data->digest_len); 1981 sym->auth.digest.phys_addr = rte_pktmbuf_iova_offset(mbuf, 1982 rte_pktmbuf_pkt_len(mbuf) - data->digest_len); 1983 1984 if (data->cipher_auth.auth_iv_update_len) { 1985 uint8_t *pkt_iv = RTE_MBUF_METADATA_UINT8_PTR(mbuf, 1986 data->cipher_auth.auth_iv_data_offset 1987 + ip_offset); 1988 uint8_t *data_iv = data->iv_aad_data + 1989 data->cipher_auth.cipher_iv_len; 1990 1991 if (data->direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) 1992 rte_memcpy(pkt_iv, data_iv, 1993 data->cipher_auth.auth_iv_update_len); 1994 else 1995 rte_memcpy(data_iv, pkt_iv, 1996 data->cipher_auth.auth_iv_update_len); 1997 } 1998 1999 if (data->cipher_auth.auth_iv_len) { 2000 /** prepare cipher op */ 2001 uint8_t *iv = crypto_op->iv_aad.cipher_auth.auth_iv; 2002 2003 rte_memcpy(iv, data->iv_aad_data + 2004 data->cipher_auth.cipher_iv_len, 2005 data->cipher_auth.auth_iv_len); 2006 } 2007 } 2008 2009 if (data->op_mask & CRYPTO_OP_MASK_AEAD) { 2010 uint8_t *iv = crypto_op->iv_aad.aead_iv_aad.iv; 2011 uint8_t *aad = crypto_op->iv_aad.aead_iv_aad.aad; 2012 2013 sym->aead.aad.data = aad; 2014 sym->aead.aad.phys_addr = rte_pktmbuf_iova_offset(mbuf, 2015 aad - rte_pktmbuf_mtod(mbuf, uint8_t *)); 2016 sym->aead.digest.data = rte_pktmbuf_mtod_offset(mbuf, 2017 uint8_t *, rte_pktmbuf_pkt_len(mbuf) - 2018 data->digest_len); 2019 sym->aead.digest.phys_addr = rte_pktmbuf_iova_offset(mbuf, 2020 rte_pktmbuf_pkt_len(mbuf) - data->digest_len); 2021 sym->aead.data.offset = data->data_offset - pkt_offset; 2022 sym->aead.data.length = payload_len; 2023 2024 if (data->aead.iv_update_len) { 2025 uint8_t *pkt_iv = RTE_MBUF_METADATA_UINT8_PTR(mbuf, 2026 data->aead.iv_data_offset + ip_offset); 2027 uint8_t *data_iv = data->iv_aad_data; 2028 2029 if (data->direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) 2030 rte_memcpy(pkt_iv, data_iv, 2031 data->aead.iv_update_len); 2032 else 2033 rte_memcpy(data_iv, pkt_iv, 2034 data->aead.iv_update_len); 2035 } 2036 2037 rte_memcpy(iv, data->iv_aad_data, data->aead.iv_len); 2038 2039 if (data->aead.aad_update_len) { 2040 uint8_t *pkt_aad = RTE_MBUF_METADATA_UINT8_PTR(mbuf, 2041 data->aead.aad_data_offset + ip_offset); 2042 uint8_t *data_aad = data->iv_aad_data + 2043 data->aead.iv_len; 2044 2045 if (data->direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) 2046 rte_memcpy(pkt_aad, data_aad, 2047 data->aead.iv_update_len); 2048 else 2049 rte_memcpy(data_aad, pkt_aad, 2050 data->aead.iv_update_len); 2051 } 2052 2053 rte_memcpy(aad, data->iv_aad_data + data->aead.iv_len, 2054 data->aead.aad_len); 2055 } 2056 2057 return 0; 2058 } 2059 2060 /** 2061 * RTE_TABLE_ACTION_TAG 2062 */ 2063 struct tag_data { 2064 uint32_t tag; 2065 } __rte_packed; 2066 2067 static int 2068 tag_apply(struct tag_data *data, 2069 struct rte_table_action_tag_params *p) 2070 { 2071 data->tag = p->tag; 2072 return 0; 2073 } 2074 2075 static __rte_always_inline void 2076 pkt_work_tag(struct rte_mbuf *mbuf, 2077 struct tag_data *data) 2078 { 2079 mbuf->hash.fdir.hi = data->tag; 2080 mbuf->ol_flags |= RTE_MBUF_F_RX_FDIR | RTE_MBUF_F_RX_FDIR_ID; 2081 } 2082 2083 static __rte_always_inline void 2084 pkt4_work_tag(struct rte_mbuf *mbuf0, 2085 struct rte_mbuf *mbuf1, 2086 struct rte_mbuf *mbuf2, 2087 struct rte_mbuf *mbuf3, 2088 struct tag_data *data0, 2089 struct tag_data *data1, 2090 struct tag_data *data2, 2091 struct tag_data *data3) 2092 { 2093 mbuf0->hash.fdir.hi = data0->tag; 2094 mbuf1->hash.fdir.hi = data1->tag; 2095 mbuf2->hash.fdir.hi = data2->tag; 2096 mbuf3->hash.fdir.hi = data3->tag; 2097 2098 mbuf0->ol_flags |= RTE_MBUF_F_RX_FDIR | RTE_MBUF_F_RX_FDIR_ID; 2099 mbuf1->ol_flags |= RTE_MBUF_F_RX_FDIR | RTE_MBUF_F_RX_FDIR_ID; 2100 mbuf2->ol_flags |= RTE_MBUF_F_RX_FDIR | RTE_MBUF_F_RX_FDIR_ID; 2101 mbuf3->ol_flags |= RTE_MBUF_F_RX_FDIR | RTE_MBUF_F_RX_FDIR_ID; 2102 } 2103 2104 /** 2105 * RTE_TABLE_ACTION_DECAP 2106 */ 2107 struct decap_data { 2108 uint16_t n; 2109 } __rte_packed; 2110 2111 static int 2112 decap_apply(struct decap_data *data, 2113 struct rte_table_action_decap_params *p) 2114 { 2115 data->n = p->n; 2116 return 0; 2117 } 2118 2119 static __rte_always_inline void 2120 pkt_work_decap(struct rte_mbuf *mbuf, 2121 struct decap_data *data) 2122 { 2123 uint16_t data_off = mbuf->data_off; 2124 uint16_t data_len = mbuf->data_len; 2125 uint32_t pkt_len = mbuf->pkt_len; 2126 uint16_t n = data->n; 2127 2128 mbuf->data_off = data_off + n; 2129 mbuf->data_len = data_len - n; 2130 mbuf->pkt_len = pkt_len - n; 2131 } 2132 2133 static __rte_always_inline void 2134 pkt4_work_decap(struct rte_mbuf *mbuf0, 2135 struct rte_mbuf *mbuf1, 2136 struct rte_mbuf *mbuf2, 2137 struct rte_mbuf *mbuf3, 2138 struct decap_data *data0, 2139 struct decap_data *data1, 2140 struct decap_data *data2, 2141 struct decap_data *data3) 2142 { 2143 uint16_t data_off0 = mbuf0->data_off; 2144 uint16_t data_len0 = mbuf0->data_len; 2145 uint32_t pkt_len0 = mbuf0->pkt_len; 2146 2147 uint16_t data_off1 = mbuf1->data_off; 2148 uint16_t data_len1 = mbuf1->data_len; 2149 uint32_t pkt_len1 = mbuf1->pkt_len; 2150 2151 uint16_t data_off2 = mbuf2->data_off; 2152 uint16_t data_len2 = mbuf2->data_len; 2153 uint32_t pkt_len2 = mbuf2->pkt_len; 2154 2155 uint16_t data_off3 = mbuf3->data_off; 2156 uint16_t data_len3 = mbuf3->data_len; 2157 uint32_t pkt_len3 = mbuf3->pkt_len; 2158 2159 uint16_t n0 = data0->n; 2160 uint16_t n1 = data1->n; 2161 uint16_t n2 = data2->n; 2162 uint16_t n3 = data3->n; 2163 2164 mbuf0->data_off = data_off0 + n0; 2165 mbuf0->data_len = data_len0 - n0; 2166 mbuf0->pkt_len = pkt_len0 - n0; 2167 2168 mbuf1->data_off = data_off1 + n1; 2169 mbuf1->data_len = data_len1 - n1; 2170 mbuf1->pkt_len = pkt_len1 - n1; 2171 2172 mbuf2->data_off = data_off2 + n2; 2173 mbuf2->data_len = data_len2 - n2; 2174 mbuf2->pkt_len = pkt_len2 - n2; 2175 2176 mbuf3->data_off = data_off3 + n3; 2177 mbuf3->data_len = data_len3 - n3; 2178 mbuf3->pkt_len = pkt_len3 - n3; 2179 } 2180 2181 /** 2182 * Action profile 2183 */ 2184 static int 2185 action_valid(enum rte_table_action_type action) 2186 { 2187 switch (action) { 2188 case RTE_TABLE_ACTION_FWD: 2189 case RTE_TABLE_ACTION_LB: 2190 case RTE_TABLE_ACTION_MTR: 2191 case RTE_TABLE_ACTION_TM: 2192 case RTE_TABLE_ACTION_ENCAP: 2193 case RTE_TABLE_ACTION_NAT: 2194 case RTE_TABLE_ACTION_TTL: 2195 case RTE_TABLE_ACTION_STATS: 2196 case RTE_TABLE_ACTION_TIME: 2197 case RTE_TABLE_ACTION_SYM_CRYPTO: 2198 case RTE_TABLE_ACTION_TAG: 2199 case RTE_TABLE_ACTION_DECAP: 2200 return 1; 2201 default: 2202 return 0; 2203 } 2204 } 2205 2206 2207 #define RTE_TABLE_ACTION_MAX 64 2208 2209 struct ap_config { 2210 uint64_t action_mask; 2211 struct rte_table_action_common_config common; 2212 struct rte_table_action_lb_config lb; 2213 struct rte_table_action_mtr_config mtr; 2214 struct rte_table_action_tm_config tm; 2215 struct rte_table_action_encap_config encap; 2216 struct rte_table_action_nat_config nat; 2217 struct rte_table_action_ttl_config ttl; 2218 struct rte_table_action_stats_config stats; 2219 struct rte_table_action_sym_crypto_config sym_crypto; 2220 }; 2221 2222 static size_t 2223 action_cfg_size(enum rte_table_action_type action) 2224 { 2225 switch (action) { 2226 case RTE_TABLE_ACTION_LB: 2227 return sizeof(struct rte_table_action_lb_config); 2228 case RTE_TABLE_ACTION_MTR: 2229 return sizeof(struct rte_table_action_mtr_config); 2230 case RTE_TABLE_ACTION_TM: 2231 return sizeof(struct rte_table_action_tm_config); 2232 case RTE_TABLE_ACTION_ENCAP: 2233 return sizeof(struct rte_table_action_encap_config); 2234 case RTE_TABLE_ACTION_NAT: 2235 return sizeof(struct rte_table_action_nat_config); 2236 case RTE_TABLE_ACTION_TTL: 2237 return sizeof(struct rte_table_action_ttl_config); 2238 case RTE_TABLE_ACTION_STATS: 2239 return sizeof(struct rte_table_action_stats_config); 2240 case RTE_TABLE_ACTION_SYM_CRYPTO: 2241 return sizeof(struct rte_table_action_sym_crypto_config); 2242 default: 2243 return 0; 2244 } 2245 } 2246 2247 static void* 2248 action_cfg_get(struct ap_config *ap_config, 2249 enum rte_table_action_type type) 2250 { 2251 switch (type) { 2252 case RTE_TABLE_ACTION_LB: 2253 return &ap_config->lb; 2254 2255 case RTE_TABLE_ACTION_MTR: 2256 return &ap_config->mtr; 2257 2258 case RTE_TABLE_ACTION_TM: 2259 return &ap_config->tm; 2260 2261 case RTE_TABLE_ACTION_ENCAP: 2262 return &ap_config->encap; 2263 2264 case RTE_TABLE_ACTION_NAT: 2265 return &ap_config->nat; 2266 2267 case RTE_TABLE_ACTION_TTL: 2268 return &ap_config->ttl; 2269 2270 case RTE_TABLE_ACTION_STATS: 2271 return &ap_config->stats; 2272 2273 case RTE_TABLE_ACTION_SYM_CRYPTO: 2274 return &ap_config->sym_crypto; 2275 default: 2276 return NULL; 2277 } 2278 } 2279 2280 static void 2281 action_cfg_set(struct ap_config *ap_config, 2282 enum rte_table_action_type type, 2283 void *action_cfg) 2284 { 2285 void *dst = action_cfg_get(ap_config, type); 2286 2287 if (dst) 2288 memcpy(dst, action_cfg, action_cfg_size(type)); 2289 2290 ap_config->action_mask |= 1LLU << type; 2291 } 2292 2293 struct ap_data { 2294 size_t offset[RTE_TABLE_ACTION_MAX]; 2295 size_t total_size; 2296 }; 2297 2298 static size_t 2299 action_data_size(enum rte_table_action_type action, 2300 struct ap_config *ap_config) 2301 { 2302 switch (action) { 2303 case RTE_TABLE_ACTION_FWD: 2304 return sizeof(struct fwd_data); 2305 2306 case RTE_TABLE_ACTION_LB: 2307 return sizeof(struct lb_data); 2308 2309 case RTE_TABLE_ACTION_MTR: 2310 return mtr_data_size(&ap_config->mtr); 2311 2312 case RTE_TABLE_ACTION_TM: 2313 return sizeof(struct tm_data); 2314 2315 case RTE_TABLE_ACTION_ENCAP: 2316 return encap_data_size(&ap_config->encap); 2317 2318 case RTE_TABLE_ACTION_NAT: 2319 return nat_data_size(&ap_config->nat, 2320 &ap_config->common); 2321 2322 case RTE_TABLE_ACTION_TTL: 2323 return sizeof(struct ttl_data); 2324 2325 case RTE_TABLE_ACTION_STATS: 2326 return sizeof(struct stats_data); 2327 2328 case RTE_TABLE_ACTION_TIME: 2329 return sizeof(struct time_data); 2330 2331 case RTE_TABLE_ACTION_SYM_CRYPTO: 2332 return (sizeof(struct sym_crypto_data)); 2333 2334 case RTE_TABLE_ACTION_TAG: 2335 return sizeof(struct tag_data); 2336 2337 case RTE_TABLE_ACTION_DECAP: 2338 return sizeof(struct decap_data); 2339 2340 default: 2341 return 0; 2342 } 2343 } 2344 2345 2346 static void 2347 action_data_offset_set(struct ap_data *ap_data, 2348 struct ap_config *ap_config) 2349 { 2350 uint64_t action_mask = ap_config->action_mask; 2351 size_t offset; 2352 uint32_t action; 2353 2354 memset(ap_data->offset, 0, sizeof(ap_data->offset)); 2355 2356 offset = 0; 2357 for (action = 0; action < RTE_TABLE_ACTION_MAX; action++) 2358 if (action_mask & (1LLU << action)) { 2359 ap_data->offset[action] = offset; 2360 offset += action_data_size((enum rte_table_action_type)action, 2361 ap_config); 2362 } 2363 2364 ap_data->total_size = offset; 2365 } 2366 2367 struct rte_table_action_profile { 2368 struct ap_config cfg; 2369 struct ap_data data; 2370 int frozen; 2371 }; 2372 2373 struct rte_table_action_profile * 2374 rte_table_action_profile_create(struct rte_table_action_common_config *common) 2375 { 2376 struct rte_table_action_profile *ap; 2377 2378 /* Check input arguments */ 2379 if (common == NULL) 2380 return NULL; 2381 2382 /* Memory allocation */ 2383 ap = calloc(1, sizeof(struct rte_table_action_profile)); 2384 if (ap == NULL) 2385 return NULL; 2386 2387 /* Initialization */ 2388 memcpy(&ap->cfg.common, common, sizeof(*common)); 2389 2390 return ap; 2391 } 2392 2393 2394 int 2395 rte_table_action_profile_action_register(struct rte_table_action_profile *profile, 2396 enum rte_table_action_type type, 2397 void *action_config) 2398 { 2399 int status; 2400 2401 /* Check input arguments */ 2402 if ((profile == NULL) || 2403 profile->frozen || 2404 (action_valid(type) == 0) || 2405 (profile->cfg.action_mask & (1LLU << type)) || 2406 ((action_cfg_size(type) == 0) && action_config) || 2407 (action_cfg_size(type) && (action_config == NULL))) 2408 return -EINVAL; 2409 2410 switch (type) { 2411 case RTE_TABLE_ACTION_LB: 2412 status = lb_cfg_check(action_config); 2413 break; 2414 2415 case RTE_TABLE_ACTION_MTR: 2416 status = mtr_cfg_check(action_config); 2417 break; 2418 2419 case RTE_TABLE_ACTION_TM: 2420 status = tm_cfg_check(action_config); 2421 break; 2422 2423 case RTE_TABLE_ACTION_ENCAP: 2424 status = encap_cfg_check(action_config); 2425 break; 2426 2427 case RTE_TABLE_ACTION_NAT: 2428 status = nat_cfg_check(action_config); 2429 break; 2430 2431 case RTE_TABLE_ACTION_TTL: 2432 status = ttl_cfg_check(action_config); 2433 break; 2434 2435 case RTE_TABLE_ACTION_STATS: 2436 status = stats_cfg_check(action_config); 2437 break; 2438 2439 case RTE_TABLE_ACTION_SYM_CRYPTO: 2440 status = sym_crypto_cfg_check(action_config); 2441 break; 2442 2443 default: 2444 status = 0; 2445 break; 2446 } 2447 2448 if (status) 2449 return status; 2450 2451 /* Action enable */ 2452 action_cfg_set(&profile->cfg, type, action_config); 2453 2454 return 0; 2455 } 2456 2457 int 2458 rte_table_action_profile_freeze(struct rte_table_action_profile *profile) 2459 { 2460 if (profile->frozen) 2461 return -EBUSY; 2462 2463 profile->cfg.action_mask |= 1LLU << RTE_TABLE_ACTION_FWD; 2464 action_data_offset_set(&profile->data, &profile->cfg); 2465 profile->frozen = 1; 2466 2467 return 0; 2468 } 2469 2470 int 2471 rte_table_action_profile_free(struct rte_table_action_profile *profile) 2472 { 2473 if (profile == NULL) 2474 return 0; 2475 2476 free(profile); 2477 return 0; 2478 } 2479 2480 /** 2481 * Action 2482 */ 2483 #define METER_PROFILES_MAX 32 2484 2485 struct rte_table_action { 2486 struct ap_config cfg; 2487 struct ap_data data; 2488 struct dscp_table_data dscp_table; 2489 struct meter_profile_data mp[METER_PROFILES_MAX]; 2490 }; 2491 2492 struct rte_table_action * 2493 rte_table_action_create(struct rte_table_action_profile *profile, 2494 uint32_t socket_id) 2495 { 2496 struct rte_table_action *action; 2497 2498 /* Check input arguments */ 2499 if ((profile == NULL) || 2500 (profile->frozen == 0)) 2501 return NULL; 2502 2503 /* Memory allocation */ 2504 action = rte_zmalloc_socket(NULL, 2505 sizeof(struct rte_table_action), 2506 RTE_CACHE_LINE_SIZE, 2507 socket_id); 2508 if (action == NULL) 2509 return NULL; 2510 2511 /* Initialization */ 2512 memcpy(&action->cfg, &profile->cfg, sizeof(profile->cfg)); 2513 memcpy(&action->data, &profile->data, sizeof(profile->data)); 2514 2515 return action; 2516 } 2517 2518 static __rte_always_inline void * 2519 action_data_get(void *data, 2520 struct rte_table_action *action, 2521 enum rte_table_action_type type) 2522 { 2523 size_t offset = action->data.offset[type]; 2524 uint8_t *data_bytes = data; 2525 2526 return &data_bytes[offset]; 2527 } 2528 2529 int 2530 rte_table_action_apply(struct rte_table_action *action, 2531 void *data, 2532 enum rte_table_action_type type, 2533 void *action_params) 2534 { 2535 void *action_data; 2536 2537 /* Check input arguments */ 2538 if ((action == NULL) || 2539 (data == NULL) || 2540 (action_valid(type) == 0) || 2541 ((action->cfg.action_mask & (1LLU << type)) == 0) || 2542 (action_params == NULL)) 2543 return -EINVAL; 2544 2545 /* Data update */ 2546 action_data = action_data_get(data, action, type); 2547 2548 switch (type) { 2549 case RTE_TABLE_ACTION_FWD: 2550 return fwd_apply(action_data, 2551 action_params); 2552 2553 case RTE_TABLE_ACTION_LB: 2554 return lb_apply(action_data, 2555 action_params); 2556 2557 case RTE_TABLE_ACTION_MTR: 2558 return mtr_apply(action_data, 2559 action_params, 2560 &action->cfg.mtr, 2561 action->mp, 2562 RTE_DIM(action->mp)); 2563 2564 case RTE_TABLE_ACTION_TM: 2565 return tm_apply(action_data, 2566 action_params, 2567 &action->cfg.tm); 2568 2569 case RTE_TABLE_ACTION_ENCAP: 2570 return encap_apply(action_data, 2571 action_params, 2572 &action->cfg.encap, 2573 &action->cfg.common); 2574 2575 case RTE_TABLE_ACTION_NAT: 2576 return nat_apply(action_data, 2577 action_params, 2578 &action->cfg.common); 2579 2580 case RTE_TABLE_ACTION_TTL: 2581 return ttl_apply(action_data, 2582 action_params); 2583 2584 case RTE_TABLE_ACTION_STATS: 2585 return stats_apply(action_data, 2586 action_params); 2587 2588 case RTE_TABLE_ACTION_TIME: 2589 return time_apply(action_data, 2590 action_params); 2591 2592 case RTE_TABLE_ACTION_SYM_CRYPTO: 2593 return sym_crypto_apply(action_data, 2594 &action->cfg.sym_crypto, 2595 action_params); 2596 2597 case RTE_TABLE_ACTION_TAG: 2598 return tag_apply(action_data, 2599 action_params); 2600 2601 case RTE_TABLE_ACTION_DECAP: 2602 return decap_apply(action_data, 2603 action_params); 2604 2605 default: 2606 return -EINVAL; 2607 } 2608 } 2609 2610 int 2611 rte_table_action_dscp_table_update(struct rte_table_action *action, 2612 uint64_t dscp_mask, 2613 struct rte_table_action_dscp_table *table) 2614 { 2615 uint32_t i; 2616 2617 /* Check input arguments */ 2618 if ((action == NULL) || 2619 ((action->cfg.action_mask & ((1LLU << RTE_TABLE_ACTION_MTR) | 2620 (1LLU << RTE_TABLE_ACTION_TM))) == 0) || 2621 (dscp_mask == 0) || 2622 (table == NULL)) 2623 return -EINVAL; 2624 2625 for (i = 0; i < RTE_DIM(table->entry); i++) { 2626 struct dscp_table_entry_data *data = 2627 &action->dscp_table.entry[i]; 2628 struct rte_table_action_dscp_table_entry *entry = 2629 &table->entry[i]; 2630 2631 if ((dscp_mask & (1LLU << i)) == 0) 2632 continue; 2633 2634 data->color = entry->color; 2635 data->tc = entry->tc_id; 2636 data->tc_queue = entry->tc_queue_id; 2637 } 2638 2639 return 0; 2640 } 2641 2642 int 2643 rte_table_action_meter_profile_add(struct rte_table_action *action, 2644 uint32_t meter_profile_id, 2645 struct rte_table_action_meter_profile *profile) 2646 { 2647 struct meter_profile_data *mp_data; 2648 uint32_t status; 2649 2650 /* Check input arguments */ 2651 if ((action == NULL) || 2652 ((action->cfg.action_mask & (1LLU << RTE_TABLE_ACTION_MTR)) == 0) || 2653 (profile == NULL)) 2654 return -EINVAL; 2655 2656 if (profile->alg != RTE_TABLE_ACTION_METER_TRTCM) 2657 return -ENOTSUP; 2658 2659 mp_data = meter_profile_data_find(action->mp, 2660 RTE_DIM(action->mp), 2661 meter_profile_id); 2662 if (mp_data) 2663 return -EEXIST; 2664 2665 mp_data = meter_profile_data_find_unused(action->mp, 2666 RTE_DIM(action->mp)); 2667 if (!mp_data) 2668 return -ENOSPC; 2669 2670 /* Install new profile */ 2671 status = rte_meter_trtcm_profile_config(&mp_data->profile, 2672 &profile->trtcm); 2673 if (status) 2674 return status; 2675 2676 mp_data->profile_id = meter_profile_id; 2677 mp_data->valid = 1; 2678 2679 return 0; 2680 } 2681 2682 int 2683 rte_table_action_meter_profile_delete(struct rte_table_action *action, 2684 uint32_t meter_profile_id) 2685 { 2686 struct meter_profile_data *mp_data; 2687 2688 /* Check input arguments */ 2689 if ((action == NULL) || 2690 ((action->cfg.action_mask & (1LLU << RTE_TABLE_ACTION_MTR)) == 0)) 2691 return -EINVAL; 2692 2693 mp_data = meter_profile_data_find(action->mp, 2694 RTE_DIM(action->mp), 2695 meter_profile_id); 2696 if (!mp_data) 2697 return 0; 2698 2699 /* Uninstall profile */ 2700 mp_data->valid = 0; 2701 2702 return 0; 2703 } 2704 2705 int 2706 rte_table_action_meter_read(struct rte_table_action *action, 2707 void *data, 2708 uint32_t tc_mask, 2709 struct rte_table_action_mtr_counters *stats, 2710 int clear) 2711 { 2712 struct mtr_trtcm_data *mtr_data; 2713 uint32_t i; 2714 2715 /* Check input arguments */ 2716 if ((action == NULL) || 2717 ((action->cfg.action_mask & (1LLU << RTE_TABLE_ACTION_MTR)) == 0) || 2718 (data == NULL) || 2719 (tc_mask > RTE_LEN2MASK(action->cfg.mtr.n_tc, uint32_t))) 2720 return -EINVAL; 2721 2722 mtr_data = action_data_get(data, action, RTE_TABLE_ACTION_MTR); 2723 2724 /* Read */ 2725 if (stats) { 2726 for (i = 0; i < RTE_TABLE_ACTION_TC_MAX; i++) { 2727 struct rte_table_action_mtr_counters_tc *dst = 2728 &stats->stats[i]; 2729 struct mtr_trtcm_data *src = &mtr_data[i]; 2730 2731 if ((tc_mask & (1 << i)) == 0) 2732 continue; 2733 2734 dst->n_packets[RTE_COLOR_GREEN] = 2735 mtr_trtcm_data_stats_get(src, RTE_COLOR_GREEN); 2736 2737 dst->n_packets[RTE_COLOR_YELLOW] = 2738 mtr_trtcm_data_stats_get(src, RTE_COLOR_YELLOW); 2739 2740 dst->n_packets[RTE_COLOR_RED] = 2741 mtr_trtcm_data_stats_get(src, RTE_COLOR_RED); 2742 2743 dst->n_packets_valid = 1; 2744 dst->n_bytes_valid = 0; 2745 } 2746 2747 stats->tc_mask = tc_mask; 2748 } 2749 2750 /* Clear */ 2751 if (clear) 2752 for (i = 0; i < RTE_TABLE_ACTION_TC_MAX; i++) { 2753 struct mtr_trtcm_data *src = &mtr_data[i]; 2754 2755 if ((tc_mask & (1 << i)) == 0) 2756 continue; 2757 2758 mtr_trtcm_data_stats_reset(src, RTE_COLOR_GREEN); 2759 mtr_trtcm_data_stats_reset(src, RTE_COLOR_YELLOW); 2760 mtr_trtcm_data_stats_reset(src, RTE_COLOR_RED); 2761 } 2762 2763 2764 return 0; 2765 } 2766 2767 int 2768 rte_table_action_ttl_read(struct rte_table_action *action, 2769 void *data, 2770 struct rte_table_action_ttl_counters *stats, 2771 int clear) 2772 { 2773 struct ttl_data *ttl_data; 2774 2775 /* Check input arguments */ 2776 if ((action == NULL) || 2777 ((action->cfg.action_mask & 2778 (1LLU << RTE_TABLE_ACTION_TTL)) == 0) || 2779 (data == NULL)) 2780 return -EINVAL; 2781 2782 ttl_data = action_data_get(data, action, RTE_TABLE_ACTION_TTL); 2783 2784 /* Read */ 2785 if (stats) 2786 stats->n_packets = TTL_STATS_READ(ttl_data); 2787 2788 /* Clear */ 2789 if (clear) 2790 TTL_STATS_RESET(ttl_data); 2791 2792 return 0; 2793 } 2794 2795 int 2796 rte_table_action_stats_read(struct rte_table_action *action, 2797 void *data, 2798 struct rte_table_action_stats_counters *stats, 2799 int clear) 2800 { 2801 struct stats_data *stats_data; 2802 2803 /* Check input arguments */ 2804 if ((action == NULL) || 2805 ((action->cfg.action_mask & 2806 (1LLU << RTE_TABLE_ACTION_STATS)) == 0) || 2807 (data == NULL)) 2808 return -EINVAL; 2809 2810 stats_data = action_data_get(data, action, 2811 RTE_TABLE_ACTION_STATS); 2812 2813 /* Read */ 2814 if (stats) { 2815 stats->n_packets = stats_data->n_packets; 2816 stats->n_bytes = stats_data->n_bytes; 2817 stats->n_packets_valid = 1; 2818 stats->n_bytes_valid = 1; 2819 } 2820 2821 /* Clear */ 2822 if (clear) { 2823 stats_data->n_packets = 0; 2824 stats_data->n_bytes = 0; 2825 } 2826 2827 return 0; 2828 } 2829 2830 int 2831 rte_table_action_time_read(struct rte_table_action *action, 2832 void *data, 2833 uint64_t *timestamp) 2834 { 2835 struct time_data *time_data; 2836 2837 /* Check input arguments */ 2838 if ((action == NULL) || 2839 ((action->cfg.action_mask & 2840 (1LLU << RTE_TABLE_ACTION_TIME)) == 0) || 2841 (data == NULL) || 2842 (timestamp == NULL)) 2843 return -EINVAL; 2844 2845 time_data = action_data_get(data, action, RTE_TABLE_ACTION_TIME); 2846 2847 /* Read */ 2848 *timestamp = time_data->time; 2849 2850 return 0; 2851 } 2852 2853 struct rte_cryptodev_sym_session * 2854 rte_table_action_crypto_sym_session_get(struct rte_table_action *action, 2855 void *data) 2856 { 2857 struct sym_crypto_data *sym_crypto_data; 2858 2859 /* Check input arguments */ 2860 if ((action == NULL) || 2861 ((action->cfg.action_mask & 2862 (1LLU << RTE_TABLE_ACTION_SYM_CRYPTO)) == 0) || 2863 (data == NULL)) 2864 return NULL; 2865 2866 sym_crypto_data = action_data_get(data, action, 2867 RTE_TABLE_ACTION_SYM_CRYPTO); 2868 2869 return sym_crypto_data->session; 2870 } 2871 2872 static __rte_always_inline uint64_t 2873 pkt_work(struct rte_mbuf *mbuf, 2874 struct rte_pipeline_table_entry *table_entry, 2875 uint64_t time, 2876 struct rte_table_action *action, 2877 struct ap_config *cfg) 2878 { 2879 uint64_t drop_mask = 0; 2880 2881 uint32_t ip_offset = action->cfg.common.ip_offset; 2882 void *ip = RTE_MBUF_METADATA_UINT32_PTR(mbuf, ip_offset); 2883 2884 uint32_t dscp; 2885 uint16_t total_length; 2886 2887 if (cfg->common.ip_version) { 2888 struct rte_ipv4_hdr *hdr = ip; 2889 2890 dscp = hdr->type_of_service >> 2; 2891 total_length = rte_ntohs(hdr->total_length); 2892 } else { 2893 struct rte_ipv6_hdr *hdr = ip; 2894 2895 dscp = (rte_ntohl(hdr->vtc_flow) & 0x0F600000) >> 18; 2896 total_length = rte_ntohs(hdr->payload_len) + 2897 sizeof(struct rte_ipv6_hdr); 2898 } 2899 2900 if (cfg->action_mask & (1LLU << RTE_TABLE_ACTION_LB)) { 2901 void *data = 2902 action_data_get(table_entry, action, RTE_TABLE_ACTION_LB); 2903 2904 pkt_work_lb(mbuf, 2905 data, 2906 &cfg->lb); 2907 } 2908 if (cfg->action_mask & (1LLU << RTE_TABLE_ACTION_MTR)) { 2909 void *data = 2910 action_data_get(table_entry, action, RTE_TABLE_ACTION_MTR); 2911 2912 drop_mask |= pkt_work_mtr(mbuf, 2913 data, 2914 &action->dscp_table, 2915 action->mp, 2916 time, 2917 dscp, 2918 total_length); 2919 } 2920 2921 if (cfg->action_mask & (1LLU << RTE_TABLE_ACTION_TM)) { 2922 void *data = 2923 action_data_get(table_entry, action, RTE_TABLE_ACTION_TM); 2924 2925 pkt_work_tm(mbuf, 2926 data, 2927 &action->dscp_table, 2928 dscp); 2929 } 2930 2931 if (cfg->action_mask & (1LLU << RTE_TABLE_ACTION_DECAP)) { 2932 void *data = action_data_get(table_entry, 2933 action, 2934 RTE_TABLE_ACTION_DECAP); 2935 2936 pkt_work_decap(mbuf, data); 2937 } 2938 2939 if (cfg->action_mask & (1LLU << RTE_TABLE_ACTION_ENCAP)) { 2940 void *data = 2941 action_data_get(table_entry, action, RTE_TABLE_ACTION_ENCAP); 2942 2943 pkt_work_encap(mbuf, 2944 data, 2945 &cfg->encap, 2946 ip, 2947 total_length, 2948 ip_offset); 2949 } 2950 2951 if (cfg->action_mask & (1LLU << RTE_TABLE_ACTION_NAT)) { 2952 void *data = 2953 action_data_get(table_entry, action, RTE_TABLE_ACTION_NAT); 2954 2955 if (cfg->common.ip_version) 2956 pkt_ipv4_work_nat(ip, data, &cfg->nat); 2957 else 2958 pkt_ipv6_work_nat(ip, data, &cfg->nat); 2959 } 2960 2961 if (cfg->action_mask & (1LLU << RTE_TABLE_ACTION_TTL)) { 2962 void *data = 2963 action_data_get(table_entry, action, RTE_TABLE_ACTION_TTL); 2964 2965 if (cfg->common.ip_version) 2966 drop_mask |= pkt_ipv4_work_ttl(ip, data); 2967 else 2968 drop_mask |= pkt_ipv6_work_ttl(ip, data); 2969 } 2970 2971 if (cfg->action_mask & (1LLU << RTE_TABLE_ACTION_STATS)) { 2972 void *data = 2973 action_data_get(table_entry, action, RTE_TABLE_ACTION_STATS); 2974 2975 pkt_work_stats(data, total_length); 2976 } 2977 2978 if (cfg->action_mask & (1LLU << RTE_TABLE_ACTION_TIME)) { 2979 void *data = 2980 action_data_get(table_entry, action, RTE_TABLE_ACTION_TIME); 2981 2982 pkt_work_time(data, time); 2983 } 2984 2985 if (cfg->action_mask & (1LLU << RTE_TABLE_ACTION_SYM_CRYPTO)) { 2986 void *data = action_data_get(table_entry, action, 2987 RTE_TABLE_ACTION_SYM_CRYPTO); 2988 2989 drop_mask |= pkt_work_sym_crypto(mbuf, data, &cfg->sym_crypto, 2990 ip_offset); 2991 } 2992 2993 if (cfg->action_mask & (1LLU << RTE_TABLE_ACTION_TAG)) { 2994 void *data = action_data_get(table_entry, 2995 action, 2996 RTE_TABLE_ACTION_TAG); 2997 2998 pkt_work_tag(mbuf, data); 2999 } 3000 3001 return drop_mask; 3002 } 3003 3004 static __rte_always_inline uint64_t 3005 pkt4_work(struct rte_mbuf **mbufs, 3006 struct rte_pipeline_table_entry **table_entries, 3007 uint64_t time, 3008 struct rte_table_action *action, 3009 struct ap_config *cfg) 3010 { 3011 uint64_t drop_mask0 = 0; 3012 uint64_t drop_mask1 = 0; 3013 uint64_t drop_mask2 = 0; 3014 uint64_t drop_mask3 = 0; 3015 3016 struct rte_mbuf *mbuf0 = mbufs[0]; 3017 struct rte_mbuf *mbuf1 = mbufs[1]; 3018 struct rte_mbuf *mbuf2 = mbufs[2]; 3019 struct rte_mbuf *mbuf3 = mbufs[3]; 3020 3021 struct rte_pipeline_table_entry *table_entry0 = table_entries[0]; 3022 struct rte_pipeline_table_entry *table_entry1 = table_entries[1]; 3023 struct rte_pipeline_table_entry *table_entry2 = table_entries[2]; 3024 struct rte_pipeline_table_entry *table_entry3 = table_entries[3]; 3025 3026 uint32_t ip_offset = action->cfg.common.ip_offset; 3027 void *ip0 = RTE_MBUF_METADATA_UINT32_PTR(mbuf0, ip_offset); 3028 void *ip1 = RTE_MBUF_METADATA_UINT32_PTR(mbuf1, ip_offset); 3029 void *ip2 = RTE_MBUF_METADATA_UINT32_PTR(mbuf2, ip_offset); 3030 void *ip3 = RTE_MBUF_METADATA_UINT32_PTR(mbuf3, ip_offset); 3031 3032 uint32_t dscp0, dscp1, dscp2, dscp3; 3033 uint16_t total_length0, total_length1, total_length2, total_length3; 3034 3035 if (cfg->common.ip_version) { 3036 struct rte_ipv4_hdr *hdr0 = ip0; 3037 struct rte_ipv4_hdr *hdr1 = ip1; 3038 struct rte_ipv4_hdr *hdr2 = ip2; 3039 struct rte_ipv4_hdr *hdr3 = ip3; 3040 3041 dscp0 = hdr0->type_of_service >> 2; 3042 dscp1 = hdr1->type_of_service >> 2; 3043 dscp2 = hdr2->type_of_service >> 2; 3044 dscp3 = hdr3->type_of_service >> 2; 3045 3046 total_length0 = rte_ntohs(hdr0->total_length); 3047 total_length1 = rte_ntohs(hdr1->total_length); 3048 total_length2 = rte_ntohs(hdr2->total_length); 3049 total_length3 = rte_ntohs(hdr3->total_length); 3050 } else { 3051 struct rte_ipv6_hdr *hdr0 = ip0; 3052 struct rte_ipv6_hdr *hdr1 = ip1; 3053 struct rte_ipv6_hdr *hdr2 = ip2; 3054 struct rte_ipv6_hdr *hdr3 = ip3; 3055 3056 dscp0 = (rte_ntohl(hdr0->vtc_flow) & 0x0F600000) >> 18; 3057 dscp1 = (rte_ntohl(hdr1->vtc_flow) & 0x0F600000) >> 18; 3058 dscp2 = (rte_ntohl(hdr2->vtc_flow) & 0x0F600000) >> 18; 3059 dscp3 = (rte_ntohl(hdr3->vtc_flow) & 0x0F600000) >> 18; 3060 3061 total_length0 = rte_ntohs(hdr0->payload_len) + 3062 sizeof(struct rte_ipv6_hdr); 3063 total_length1 = rte_ntohs(hdr1->payload_len) + 3064 sizeof(struct rte_ipv6_hdr); 3065 total_length2 = rte_ntohs(hdr2->payload_len) + 3066 sizeof(struct rte_ipv6_hdr); 3067 total_length3 = rte_ntohs(hdr3->payload_len) + 3068 sizeof(struct rte_ipv6_hdr); 3069 } 3070 3071 if (cfg->action_mask & (1LLU << RTE_TABLE_ACTION_LB)) { 3072 void *data0 = 3073 action_data_get(table_entry0, action, RTE_TABLE_ACTION_LB); 3074 void *data1 = 3075 action_data_get(table_entry1, action, RTE_TABLE_ACTION_LB); 3076 void *data2 = 3077 action_data_get(table_entry2, action, RTE_TABLE_ACTION_LB); 3078 void *data3 = 3079 action_data_get(table_entry3, action, RTE_TABLE_ACTION_LB); 3080 3081 pkt_work_lb(mbuf0, 3082 data0, 3083 &cfg->lb); 3084 3085 pkt_work_lb(mbuf1, 3086 data1, 3087 &cfg->lb); 3088 3089 pkt_work_lb(mbuf2, 3090 data2, 3091 &cfg->lb); 3092 3093 pkt_work_lb(mbuf3, 3094 data3, 3095 &cfg->lb); 3096 } 3097 3098 if (cfg->action_mask & (1LLU << RTE_TABLE_ACTION_MTR)) { 3099 void *data0 = 3100 action_data_get(table_entry0, action, RTE_TABLE_ACTION_MTR); 3101 void *data1 = 3102 action_data_get(table_entry1, action, RTE_TABLE_ACTION_MTR); 3103 void *data2 = 3104 action_data_get(table_entry2, action, RTE_TABLE_ACTION_MTR); 3105 void *data3 = 3106 action_data_get(table_entry3, action, RTE_TABLE_ACTION_MTR); 3107 3108 drop_mask0 |= pkt_work_mtr(mbuf0, 3109 data0, 3110 &action->dscp_table, 3111 action->mp, 3112 time, 3113 dscp0, 3114 total_length0); 3115 3116 drop_mask1 |= pkt_work_mtr(mbuf1, 3117 data1, 3118 &action->dscp_table, 3119 action->mp, 3120 time, 3121 dscp1, 3122 total_length1); 3123 3124 drop_mask2 |= pkt_work_mtr(mbuf2, 3125 data2, 3126 &action->dscp_table, 3127 action->mp, 3128 time, 3129 dscp2, 3130 total_length2); 3131 3132 drop_mask3 |= pkt_work_mtr(mbuf3, 3133 data3, 3134 &action->dscp_table, 3135 action->mp, 3136 time, 3137 dscp3, 3138 total_length3); 3139 } 3140 3141 if (cfg->action_mask & (1LLU << RTE_TABLE_ACTION_TM)) { 3142 void *data0 = 3143 action_data_get(table_entry0, action, RTE_TABLE_ACTION_TM); 3144 void *data1 = 3145 action_data_get(table_entry1, action, RTE_TABLE_ACTION_TM); 3146 void *data2 = 3147 action_data_get(table_entry2, action, RTE_TABLE_ACTION_TM); 3148 void *data3 = 3149 action_data_get(table_entry3, action, RTE_TABLE_ACTION_TM); 3150 3151 pkt_work_tm(mbuf0, 3152 data0, 3153 &action->dscp_table, 3154 dscp0); 3155 3156 pkt_work_tm(mbuf1, 3157 data1, 3158 &action->dscp_table, 3159 dscp1); 3160 3161 pkt_work_tm(mbuf2, 3162 data2, 3163 &action->dscp_table, 3164 dscp2); 3165 3166 pkt_work_tm(mbuf3, 3167 data3, 3168 &action->dscp_table, 3169 dscp3); 3170 } 3171 3172 if (cfg->action_mask & (1LLU << RTE_TABLE_ACTION_DECAP)) { 3173 void *data0 = action_data_get(table_entry0, 3174 action, 3175 RTE_TABLE_ACTION_DECAP); 3176 void *data1 = action_data_get(table_entry1, 3177 action, 3178 RTE_TABLE_ACTION_DECAP); 3179 void *data2 = action_data_get(table_entry2, 3180 action, 3181 RTE_TABLE_ACTION_DECAP); 3182 void *data3 = action_data_get(table_entry3, 3183 action, 3184 RTE_TABLE_ACTION_DECAP); 3185 3186 pkt4_work_decap(mbuf0, mbuf1, mbuf2, mbuf3, 3187 data0, data1, data2, data3); 3188 } 3189 3190 if (cfg->action_mask & (1LLU << RTE_TABLE_ACTION_ENCAP)) { 3191 void *data0 = 3192 action_data_get(table_entry0, action, RTE_TABLE_ACTION_ENCAP); 3193 void *data1 = 3194 action_data_get(table_entry1, action, RTE_TABLE_ACTION_ENCAP); 3195 void *data2 = 3196 action_data_get(table_entry2, action, RTE_TABLE_ACTION_ENCAP); 3197 void *data3 = 3198 action_data_get(table_entry3, action, RTE_TABLE_ACTION_ENCAP); 3199 3200 pkt_work_encap(mbuf0, 3201 data0, 3202 &cfg->encap, 3203 ip0, 3204 total_length0, 3205 ip_offset); 3206 3207 pkt_work_encap(mbuf1, 3208 data1, 3209 &cfg->encap, 3210 ip1, 3211 total_length1, 3212 ip_offset); 3213 3214 pkt_work_encap(mbuf2, 3215 data2, 3216 &cfg->encap, 3217 ip2, 3218 total_length2, 3219 ip_offset); 3220 3221 pkt_work_encap(mbuf3, 3222 data3, 3223 &cfg->encap, 3224 ip3, 3225 total_length3, 3226 ip_offset); 3227 } 3228 3229 if (cfg->action_mask & (1LLU << RTE_TABLE_ACTION_NAT)) { 3230 void *data0 = 3231 action_data_get(table_entry0, action, RTE_TABLE_ACTION_NAT); 3232 void *data1 = 3233 action_data_get(table_entry1, action, RTE_TABLE_ACTION_NAT); 3234 void *data2 = 3235 action_data_get(table_entry2, action, RTE_TABLE_ACTION_NAT); 3236 void *data3 = 3237 action_data_get(table_entry3, action, RTE_TABLE_ACTION_NAT); 3238 3239 if (cfg->common.ip_version) { 3240 pkt_ipv4_work_nat(ip0, data0, &cfg->nat); 3241 pkt_ipv4_work_nat(ip1, data1, &cfg->nat); 3242 pkt_ipv4_work_nat(ip2, data2, &cfg->nat); 3243 pkt_ipv4_work_nat(ip3, data3, &cfg->nat); 3244 } else { 3245 pkt_ipv6_work_nat(ip0, data0, &cfg->nat); 3246 pkt_ipv6_work_nat(ip1, data1, &cfg->nat); 3247 pkt_ipv6_work_nat(ip2, data2, &cfg->nat); 3248 pkt_ipv6_work_nat(ip3, data3, &cfg->nat); 3249 } 3250 } 3251 3252 if (cfg->action_mask & (1LLU << RTE_TABLE_ACTION_TTL)) { 3253 void *data0 = 3254 action_data_get(table_entry0, action, RTE_TABLE_ACTION_TTL); 3255 void *data1 = 3256 action_data_get(table_entry1, action, RTE_TABLE_ACTION_TTL); 3257 void *data2 = 3258 action_data_get(table_entry2, action, RTE_TABLE_ACTION_TTL); 3259 void *data3 = 3260 action_data_get(table_entry3, action, RTE_TABLE_ACTION_TTL); 3261 3262 if (cfg->common.ip_version) { 3263 drop_mask0 |= pkt_ipv4_work_ttl(ip0, data0); 3264 drop_mask1 |= pkt_ipv4_work_ttl(ip1, data1); 3265 drop_mask2 |= pkt_ipv4_work_ttl(ip2, data2); 3266 drop_mask3 |= pkt_ipv4_work_ttl(ip3, data3); 3267 } else { 3268 drop_mask0 |= pkt_ipv6_work_ttl(ip0, data0); 3269 drop_mask1 |= pkt_ipv6_work_ttl(ip1, data1); 3270 drop_mask2 |= pkt_ipv6_work_ttl(ip2, data2); 3271 drop_mask3 |= pkt_ipv6_work_ttl(ip3, data3); 3272 } 3273 } 3274 3275 if (cfg->action_mask & (1LLU << RTE_TABLE_ACTION_STATS)) { 3276 void *data0 = 3277 action_data_get(table_entry0, action, RTE_TABLE_ACTION_STATS); 3278 void *data1 = 3279 action_data_get(table_entry1, action, RTE_TABLE_ACTION_STATS); 3280 void *data2 = 3281 action_data_get(table_entry2, action, RTE_TABLE_ACTION_STATS); 3282 void *data3 = 3283 action_data_get(table_entry3, action, RTE_TABLE_ACTION_STATS); 3284 3285 pkt_work_stats(data0, total_length0); 3286 pkt_work_stats(data1, total_length1); 3287 pkt_work_stats(data2, total_length2); 3288 pkt_work_stats(data3, total_length3); 3289 } 3290 3291 if (cfg->action_mask & (1LLU << RTE_TABLE_ACTION_TIME)) { 3292 void *data0 = 3293 action_data_get(table_entry0, action, RTE_TABLE_ACTION_TIME); 3294 void *data1 = 3295 action_data_get(table_entry1, action, RTE_TABLE_ACTION_TIME); 3296 void *data2 = 3297 action_data_get(table_entry2, action, RTE_TABLE_ACTION_TIME); 3298 void *data3 = 3299 action_data_get(table_entry3, action, RTE_TABLE_ACTION_TIME); 3300 3301 pkt_work_time(data0, time); 3302 pkt_work_time(data1, time); 3303 pkt_work_time(data2, time); 3304 pkt_work_time(data3, time); 3305 } 3306 3307 if (cfg->action_mask & (1LLU << RTE_TABLE_ACTION_SYM_CRYPTO)) { 3308 void *data0 = action_data_get(table_entry0, action, 3309 RTE_TABLE_ACTION_SYM_CRYPTO); 3310 void *data1 = action_data_get(table_entry1, action, 3311 RTE_TABLE_ACTION_SYM_CRYPTO); 3312 void *data2 = action_data_get(table_entry2, action, 3313 RTE_TABLE_ACTION_SYM_CRYPTO); 3314 void *data3 = action_data_get(table_entry3, action, 3315 RTE_TABLE_ACTION_SYM_CRYPTO); 3316 3317 drop_mask0 |= pkt_work_sym_crypto(mbuf0, data0, &cfg->sym_crypto, 3318 ip_offset); 3319 drop_mask1 |= pkt_work_sym_crypto(mbuf1, data1, &cfg->sym_crypto, 3320 ip_offset); 3321 drop_mask2 |= pkt_work_sym_crypto(mbuf2, data2, &cfg->sym_crypto, 3322 ip_offset); 3323 drop_mask3 |= pkt_work_sym_crypto(mbuf3, data3, &cfg->sym_crypto, 3324 ip_offset); 3325 } 3326 3327 if (cfg->action_mask & (1LLU << RTE_TABLE_ACTION_TAG)) { 3328 void *data0 = action_data_get(table_entry0, 3329 action, 3330 RTE_TABLE_ACTION_TAG); 3331 void *data1 = action_data_get(table_entry1, 3332 action, 3333 RTE_TABLE_ACTION_TAG); 3334 void *data2 = action_data_get(table_entry2, 3335 action, 3336 RTE_TABLE_ACTION_TAG); 3337 void *data3 = action_data_get(table_entry3, 3338 action, 3339 RTE_TABLE_ACTION_TAG); 3340 3341 pkt4_work_tag(mbuf0, mbuf1, mbuf2, mbuf3, 3342 data0, data1, data2, data3); 3343 } 3344 3345 return drop_mask0 | 3346 (drop_mask1 << 1) | 3347 (drop_mask2 << 2) | 3348 (drop_mask3 << 3); 3349 } 3350 3351 static __rte_always_inline int 3352 ah(struct rte_pipeline *p, 3353 struct rte_mbuf **pkts, 3354 uint64_t pkts_mask, 3355 struct rte_pipeline_table_entry **entries, 3356 struct rte_table_action *action, 3357 struct ap_config *cfg) 3358 { 3359 uint64_t pkts_drop_mask = 0; 3360 uint64_t time = 0; 3361 3362 if (cfg->action_mask & ((1LLU << RTE_TABLE_ACTION_MTR) | 3363 (1LLU << RTE_TABLE_ACTION_TIME))) 3364 time = rte_rdtsc(); 3365 3366 if ((pkts_mask & (pkts_mask + 1)) == 0) { 3367 uint64_t n_pkts = rte_popcount64(pkts_mask); 3368 uint32_t i; 3369 3370 for (i = 0; i < (n_pkts & (~0x3LLU)); i += 4) { 3371 uint64_t drop_mask; 3372 3373 drop_mask = pkt4_work(&pkts[i], 3374 &entries[i], 3375 time, 3376 action, 3377 cfg); 3378 3379 pkts_drop_mask |= drop_mask << i; 3380 } 3381 3382 for ( ; i < n_pkts; i++) { 3383 uint64_t drop_mask; 3384 3385 drop_mask = pkt_work(pkts[i], 3386 entries[i], 3387 time, 3388 action, 3389 cfg); 3390 3391 pkts_drop_mask |= drop_mask << i; 3392 } 3393 } else 3394 for ( ; pkts_mask; ) { 3395 uint32_t pos = rte_ctz64(pkts_mask); 3396 uint64_t pkt_mask = 1LLU << pos; 3397 uint64_t drop_mask; 3398 3399 drop_mask = pkt_work(pkts[pos], 3400 entries[pos], 3401 time, 3402 action, 3403 cfg); 3404 3405 pkts_mask &= ~pkt_mask; 3406 pkts_drop_mask |= drop_mask << pos; 3407 } 3408 3409 rte_pipeline_ah_packet_drop(p, pkts_drop_mask); 3410 3411 return 0; 3412 } 3413 3414 static int 3415 ah_default(struct rte_pipeline *p, 3416 struct rte_mbuf **pkts, 3417 uint64_t pkts_mask, 3418 struct rte_pipeline_table_entry **entries, 3419 void *arg) 3420 { 3421 struct rte_table_action *action = arg; 3422 3423 return ah(p, 3424 pkts, 3425 pkts_mask, 3426 entries, 3427 action, 3428 &action->cfg); 3429 } 3430 3431 static rte_pipeline_table_action_handler_hit 3432 ah_selector(struct rte_table_action *action) 3433 { 3434 if (action->cfg.action_mask == (1LLU << RTE_TABLE_ACTION_FWD)) 3435 return NULL; 3436 3437 return ah_default; 3438 } 3439 3440 int 3441 rte_table_action_table_params_get(struct rte_table_action *action, 3442 struct rte_pipeline_table_params *params) 3443 { 3444 rte_pipeline_table_action_handler_hit f_action_hit; 3445 uint32_t total_size; 3446 3447 /* Check input arguments */ 3448 if ((action == NULL) || 3449 (params == NULL)) 3450 return -EINVAL; 3451 3452 f_action_hit = ah_selector(action); 3453 total_size = rte_align32pow2(action->data.total_size); 3454 3455 /* Fill in params */ 3456 params->f_action_hit = f_action_hit; 3457 params->f_action_miss = NULL; 3458 params->arg_ah = (f_action_hit) ? action : NULL; 3459 params->action_data_size = total_size - 3460 sizeof(struct rte_pipeline_table_entry); 3461 3462 return 0; 3463 } 3464 3465 int 3466 rte_table_action_free(struct rte_table_action *action) 3467 { 3468 if (action == NULL) 3469 return 0; 3470 3471 rte_free(action); 3472 3473 return 0; 3474 } 3475