1 /* 2 * SPDX-License-Identifier: BSD-3-Clause 3 * Copyright(c) 2023 Napatech A/S 4 */ 5 #include "ntlog.h" 6 #include "nt_util.h" 7 8 #include "flow_api_engine.h" 9 #include "flow_api_nic_setup.h" 10 #include "ntnic_mod_reg.h" 11 12 #include "flow_api.h" 13 #include "flow_filter.h" 14 15 #define RSS_TO_STRING(name) \ 16 { \ 17 name, #name \ 18 } 19 20 const char *dbg_res_descr[] = { 21 /* RES_QUEUE */ "RES_QUEUE", 22 /* RES_CAT_CFN */ "RES_CAT_CFN", 23 /* RES_CAT_COT */ "RES_CAT_COT", 24 /* RES_CAT_EXO */ "RES_CAT_EXO", 25 /* RES_CAT_LEN */ "RES_CAT_LEN", 26 /* RES_KM_FLOW_TYPE */ "RES_KM_FLOW_TYPE", 27 /* RES_KM_CATEGORY */ "RES_KM_CATEGORY", 28 /* RES_HSH_RCP */ "RES_HSH_RCP", 29 /* RES_PDB_RCP */ "RES_PDB_RCP", 30 /* RES_QSL_RCP */ "RES_QSL_RCP", 31 /* RES_QSL_LTX */ "RES_QSL_LTX", 32 /* RES_QSL_QST */ "RES_QSL_QST", 33 /* RES_SLC_LR_RCP */ "RES_SLC_LR_RCP", 34 /* RES_FLM_FLOW_TYPE */ "RES_FLM_FLOW_TYPE", 35 /* RES_FLM_RCP */ "RES_FLM_RCP", 36 /* RES_TPE_RCP */ "RES_TPE_RCP", 37 /* RES_TPE_EXT */ "RES_TPE_EXT", 38 /* RES_TPE_RPL */ "RES_TPE_RPL", 39 /* RES_COUNT */ "RES_COUNT", 40 /* RES_INVALID */ "RES_INVALID" 41 }; 42 43 static struct flow_nic_dev *dev_base; 44 static pthread_mutex_t base_mtx = PTHREAD_MUTEX_INITIALIZER; 45 46 /* 47 * Error handling 48 */ 49 50 static const struct { 51 const char *message; 52 } err_msg[] = { 53 /* 00 */ { "Operation successfully completed" }, 54 /* 01 */ { "Operation failed" }, 55 /* 02 */ { "Memory allocation failed" }, 56 /* 03 */ { "Too many output destinations" }, 57 /* 04 */ { "Too many output queues for RSS" }, 58 /* 05 */ { "The VLAN TPID specified is not supported" }, 59 /* 06 */ { "The VxLan Push header specified is not accepted" }, 60 /* 07 */ { "While interpreting VxLan Pop action, could not find a destination port" }, 61 /* 08 */ { "Failed in creating a HW-internal VTEP port" }, 62 /* 09 */ { "Too many VLAN tag matches" }, 63 /* 10 */ { "IPv6 invalid header specified" }, 64 /* 11 */ { "Too many tunnel ports. HW limit reached" }, 65 /* 12 */ { "Unknown or unsupported flow match element received" }, 66 /* 13 */ { "Match failed because of HW limitations" }, 67 /* 14 */ { "Match failed because of HW resource limitations" }, 68 /* 15 */ { "Match failed because of too complex element definitions" }, 69 /* 16 */ { "Action failed. To too many output destinations" }, 70 /* 17 */ { "Action Output failed, due to HW resource exhaustion" }, 71 /* 18 */ { "Push Tunnel Header action cannot output to multiple destination queues" }, 72 /* 19 */ { "Inline action HW resource exhaustion" }, 73 /* 20 */ { "Action retransmit/recirculate HW resource exhaustion" }, 74 /* 21 */ { "Flow counter HW resource exhaustion" }, 75 /* 22 */ { "Internal HW resource exhaustion to handle Actions" }, 76 /* 23 */ { "Internal HW QSL compare failed" }, 77 /* 24 */ { "Internal CAT CFN reuse failed" }, 78 /* 25 */ { "Match variations too complex" }, 79 /* 26 */ { "Match failed because of CAM/TCAM full" }, 80 /* 27 */ { "Internal creation of a tunnel end point port failed" }, 81 /* 28 */ { "Unknown or unsupported flow action received" }, 82 /* 29 */ { "Removing flow failed" }, 83 }; 84 85 void flow_nic_set_error(enum flow_nic_err_msg_e msg, struct rte_flow_error *error) 86 { 87 assert(msg < ERR_MSG_NO_MSG); 88 89 if (error) { 90 error->message = err_msg[msg].message; 91 error->type = (msg == ERR_SUCCESS) ? RTE_FLOW_ERROR_TYPE_NONE : 92 RTE_FLOW_ERROR_TYPE_UNSPECIFIED; 93 } 94 } 95 96 /* 97 * Resources 98 */ 99 100 int flow_nic_alloc_resource(struct flow_nic_dev *ndev, enum res_type_e res_type, 101 uint32_t alignment) 102 { 103 for (unsigned int i = 0; i < ndev->res[res_type].resource_count; i += alignment) { 104 if (!flow_nic_is_resource_used(ndev, res_type, i)) { 105 flow_nic_mark_resource_used(ndev, res_type, i); 106 ndev->res[res_type].ref[i] = 1; 107 return i; 108 } 109 } 110 111 return -1; 112 } 113 114 int flow_nic_alloc_resource_config(struct flow_nic_dev *ndev, enum res_type_e res_type, 115 unsigned int num, uint32_t alignment) 116 { 117 unsigned int idx_offs; 118 119 for (unsigned int res_idx = 0; res_idx < ndev->res[res_type].resource_count - (num - 1); 120 res_idx += alignment) { 121 if (!flow_nic_is_resource_used(ndev, res_type, res_idx)) { 122 for (idx_offs = 1; idx_offs < num; idx_offs++) 123 if (flow_nic_is_resource_used(ndev, res_type, res_idx + idx_offs)) 124 break; 125 126 if (idx_offs < num) 127 continue; 128 129 /* found a contiguous number of "num" res_type elements - allocate them */ 130 for (idx_offs = 0; idx_offs < num; idx_offs++) { 131 flow_nic_mark_resource_used(ndev, res_type, res_idx + idx_offs); 132 ndev->res[res_type].ref[res_idx + idx_offs] = 1; 133 } 134 135 return res_idx; 136 } 137 } 138 139 return -1; 140 } 141 142 void flow_nic_free_resource(struct flow_nic_dev *ndev, enum res_type_e res_type, int idx) 143 { 144 flow_nic_mark_resource_unused(ndev, res_type, idx); 145 } 146 147 int flow_nic_ref_resource(struct flow_nic_dev *ndev, enum res_type_e res_type, int index) 148 { 149 NT_LOG(DBG, FILTER, "Reference resource %s idx %i (before ref cnt %i)", 150 dbg_res_descr[res_type], index, ndev->res[res_type].ref[index]); 151 assert(flow_nic_is_resource_used(ndev, res_type, index)); 152 153 if (ndev->res[res_type].ref[index] == (uint32_t)-1) 154 return -1; 155 156 ndev->res[res_type].ref[index]++; 157 return 0; 158 } 159 160 int flow_nic_deref_resource(struct flow_nic_dev *ndev, enum res_type_e res_type, int index) 161 { 162 NT_LOG(DBG, FILTER, "De-reference resource %s idx %i (before ref cnt %i)", 163 dbg_res_descr[res_type], index, ndev->res[res_type].ref[index]); 164 assert(flow_nic_is_resource_used(ndev, res_type, index)); 165 assert(ndev->res[res_type].ref[index]); 166 /* deref */ 167 ndev->res[res_type].ref[index]--; 168 169 if (!ndev->res[res_type].ref[index]) 170 flow_nic_free_resource(ndev, res_type, index); 171 172 return !!ndev->res[res_type].ref[index];/* if 0 resource has been freed */ 173 } 174 175 /* 176 * Nic port/adapter lookup 177 */ 178 179 static struct flow_eth_dev *nic_and_port_to_eth_dev(uint8_t adapter_no, uint8_t port) 180 { 181 struct flow_nic_dev *nic_dev = dev_base; 182 183 while (nic_dev) { 184 if (nic_dev->adapter_no == adapter_no) 185 break; 186 187 nic_dev = nic_dev->next; 188 } 189 190 if (!nic_dev) 191 return NULL; 192 193 struct flow_eth_dev *dev = nic_dev->eth_base; 194 195 while (dev) { 196 if (port == dev->port) 197 return dev; 198 199 dev = dev->next; 200 } 201 202 return NULL; 203 } 204 205 static struct flow_nic_dev *get_nic_dev_from_adapter_no(uint8_t adapter_no) 206 { 207 struct flow_nic_dev *ndev = dev_base; 208 209 while (ndev) { 210 if (adapter_no == ndev->adapter_no) 211 break; 212 213 ndev = ndev->next; 214 } 215 216 return ndev; 217 } 218 /* 219 * Flow API 220 */ 221 222 static struct flow_handle *flow_create(struct flow_eth_dev *dev __rte_unused, 223 const struct rte_flow_attr *attr __rte_unused, 224 uint16_t forced_vlan_vid __rte_unused, 225 uint16_t caller_id __rte_unused, 226 const struct rte_flow_item item[] __rte_unused, 227 const struct rte_flow_action action[] __rte_unused, 228 struct rte_flow_error *error __rte_unused) 229 { 230 const struct profile_inline_ops *profile_inline_ops = get_profile_inline_ops(); 231 232 if (profile_inline_ops == NULL) { 233 NT_LOG(ERR, FILTER, "%s: profile_inline module uninitialized", __func__); 234 return NULL; 235 } 236 237 return profile_inline_ops->flow_create_profile_inline(dev, attr, 238 forced_vlan_vid, caller_id, item, action, error); 239 } 240 241 static int flow_destroy(struct flow_eth_dev *dev __rte_unused, 242 struct flow_handle *flow __rte_unused, struct rte_flow_error *error __rte_unused) 243 { 244 const struct profile_inline_ops *profile_inline_ops = get_profile_inline_ops(); 245 246 if (profile_inline_ops == NULL) { 247 NT_LOG(ERR, FILTER, "%s: profile_inline module uninitialized", __func__); 248 return -1; 249 } 250 251 return profile_inline_ops->flow_destroy_profile_inline(dev, flow, error); 252 } 253 254 /* 255 * Device Management API 256 */ 257 258 static void nic_insert_eth_port_dev(struct flow_nic_dev *ndev, struct flow_eth_dev *dev) 259 { 260 dev->next = ndev->eth_base; 261 ndev->eth_base = dev; 262 } 263 264 static int nic_remove_eth_port_dev(struct flow_nic_dev *ndev, struct flow_eth_dev *eth_dev) 265 { 266 struct flow_eth_dev *dev = ndev->eth_base, *prev = NULL; 267 268 while (dev) { 269 if (dev == eth_dev) { 270 if (prev) 271 prev->next = dev->next; 272 273 else 274 ndev->eth_base = dev->next; 275 276 return 0; 277 } 278 279 prev = dev; 280 dev = dev->next; 281 } 282 283 return -1; 284 } 285 286 static void flow_ndev_reset(struct flow_nic_dev *ndev) 287 { 288 const struct profile_inline_ops *profile_inline_ops = get_profile_inline_ops(); 289 290 if (profile_inline_ops == NULL) { 291 NT_LOG(ERR, FILTER, "%s: profile_inline module uninitialized", __func__); 292 return; 293 } 294 295 /* Delete all eth-port devices created on this NIC device */ 296 while (ndev->eth_base) 297 flow_delete_eth_dev(ndev->eth_base); 298 299 /* Error check */ 300 while (ndev->flow_base) { 301 NT_LOG(ERR, FILTER, 302 "ERROR : Flows still defined but all eth-ports deleted. Flow %p", 303 ndev->flow_base); 304 305 profile_inline_ops->flow_destroy_profile_inline(ndev->flow_base->dev, 306 ndev->flow_base, NULL); 307 } 308 309 profile_inline_ops->done_flow_management_of_ndev_profile_inline(ndev); 310 311 km_free_ndev_resource_management(&ndev->km_res_handle); 312 kcc_free_ndev_resource_management(&ndev->kcc_res_handle); 313 314 ndev->flow_unique_id_counter = 0; 315 316 #ifdef FLOW_DEBUG 317 /* 318 * free all resources default allocated, initially for this NIC DEV 319 * Is not really needed since the bitmap will be freed in a sec. Therefore 320 * only in debug mode 321 */ 322 323 /* Check if all resources has been released */ 324 NT_LOG(DBG, FILTER, "Delete NIC DEV Adaptor %i", ndev->adapter_no); 325 326 for (unsigned int i = 0; i < RES_COUNT; i++) { 327 int err = 0; 328 #if defined(FLOW_DEBUG) 329 NT_LOG(DBG, FILTER, "RES state for: %s", dbg_res_descr[i]); 330 #endif 331 332 for (unsigned int ii = 0; ii < ndev->res[i].resource_count; ii++) { 333 int ref = ndev->res[i].ref[ii]; 334 int used = flow_nic_is_resource_used(ndev, i, ii); 335 336 if (ref || used) { 337 NT_LOG(DBG, FILTER, " [%i]: ref cnt %i, used %i", ii, ref, 338 used); 339 err = 1; 340 } 341 } 342 343 if (err) 344 NT_LOG(DBG, FILTER, "ERROR - some resources not freed"); 345 } 346 347 #endif 348 } 349 350 int flow_delete_eth_dev(struct flow_eth_dev *eth_dev) 351 { 352 const struct profile_inline_ops *profile_inline_ops = get_profile_inline_ops(); 353 354 if (profile_inline_ops == NULL) { 355 NT_LOG(ERR, FILTER, "%s: profile_inline module uninitialized", __func__); 356 return -1; 357 } 358 359 struct flow_nic_dev *ndev = eth_dev->ndev; 360 361 if (!ndev) { 362 /* Error invalid nic device */ 363 return -1; 364 } 365 366 NT_LOG(DBG, FILTER, "Delete eth-port device %p, port %i", eth_dev, eth_dev->port); 367 368 #ifdef FLOW_DEBUG 369 ndev->be.iface->set_debug_mode(ndev->be.be_dev, FLOW_BACKEND_DEBUG_MODE_WRITE); 370 #endif 371 372 /* delete all created flows from this device */ 373 pthread_mutex_lock(&ndev->mtx); 374 375 struct flow_handle *flow = ndev->flow_base; 376 377 while (flow) { 378 if (flow->dev == eth_dev) { 379 struct flow_handle *flow_next = flow->next; 380 profile_inline_ops->flow_destroy_locked_profile_inline(eth_dev, flow, 381 NULL); 382 flow = flow_next; 383 384 } else { 385 flow = flow->next; 386 } 387 } 388 389 /* 390 * remove unmatched queue if setup in QSL 391 * remove exception queue setting in QSL UNM 392 */ 393 hw_mod_qsl_unmq_set(&ndev->be, HW_QSL_UNMQ_DEST_QUEUE, eth_dev->port, 0); 394 hw_mod_qsl_unmq_set(&ndev->be, HW_QSL_UNMQ_EN, eth_dev->port, 0); 395 hw_mod_qsl_unmq_flush(&ndev->be, eth_dev->port, 1); 396 397 if (ndev->flow_profile == FLOW_ETH_DEV_PROFILE_INLINE) { 398 for (int i = 0; i < eth_dev->num_queues; ++i) { 399 uint32_t qen_value = 0; 400 uint32_t queue_id = (uint32_t)eth_dev->rx_queue[i].hw_id; 401 402 hw_mod_qsl_qen_get(&ndev->be, HW_QSL_QEN_EN, queue_id / 4, &qen_value); 403 hw_mod_qsl_qen_set(&ndev->be, HW_QSL_QEN_EN, queue_id / 4, 404 qen_value & ~(1U << (queue_id % 4))); 405 hw_mod_qsl_qen_flush(&ndev->be, queue_id / 4, 1); 406 } 407 } 408 409 #ifdef FLOW_DEBUG 410 ndev->be.iface->set_debug_mode(ndev->be.be_dev, FLOW_BACKEND_DEBUG_MODE_NONE); 411 #endif 412 413 /* take eth_dev out of ndev list */ 414 if (nic_remove_eth_port_dev(ndev, eth_dev) != 0) 415 NT_LOG(ERR, FILTER, "ERROR : eth_dev %p not found", eth_dev); 416 417 pthread_mutex_unlock(&ndev->mtx); 418 419 /* free eth_dev */ 420 free(eth_dev); 421 422 return 0; 423 } 424 425 /* 426 * Flow API NIC Setup 427 * Flow backend creation function - register and initialize common backend API to FPA modules 428 */ 429 430 static int init_resource_elements(struct flow_nic_dev *ndev, enum res_type_e res_type, 431 uint32_t count) 432 { 433 assert(ndev->res[res_type].alloc_bm == NULL); 434 /* allocate bitmap and ref counter */ 435 ndev->res[res_type].alloc_bm = 436 calloc(1, BIT_CONTAINER_8_ALIGN(count) + count * sizeof(uint32_t)); 437 438 if (ndev->res[res_type].alloc_bm) { 439 ndev->res[res_type].ref = 440 (uint32_t *)&ndev->res[res_type].alloc_bm[BIT_CONTAINER_8_ALIGN(count)]; 441 ndev->res[res_type].resource_count = count; 442 return 0; 443 } 444 445 return -1; 446 } 447 448 static void done_resource_elements(struct flow_nic_dev *ndev, enum res_type_e res_type) 449 { 450 assert(ndev); 451 452 if (ndev->res[res_type].alloc_bm) 453 free(ndev->res[res_type].alloc_bm); 454 } 455 456 static void list_insert_flow_nic(struct flow_nic_dev *ndev) 457 { 458 pthread_mutex_lock(&base_mtx); 459 ndev->next = dev_base; 460 dev_base = ndev; 461 pthread_mutex_unlock(&base_mtx); 462 } 463 464 static int list_remove_flow_nic(struct flow_nic_dev *ndev) 465 { 466 pthread_mutex_lock(&base_mtx); 467 struct flow_nic_dev *nic_dev = dev_base, *prev = NULL; 468 469 while (nic_dev) { 470 if (nic_dev == ndev) { 471 if (prev) 472 prev->next = nic_dev->next; 473 474 else 475 dev_base = nic_dev->next; 476 477 pthread_mutex_unlock(&base_mtx); 478 return 0; 479 } 480 481 prev = nic_dev; 482 nic_dev = nic_dev->next; 483 } 484 485 pthread_mutex_unlock(&base_mtx); 486 return -1; 487 } 488 489 /* 490 * adapter_no physical adapter no 491 * port_no local port no 492 * alloc_rx_queues number of rx-queues to allocate for this eth_dev 493 */ 494 static struct flow_eth_dev *flow_get_eth_dev(uint8_t adapter_no, uint8_t port_no, uint32_t port_id, 495 int alloc_rx_queues, struct flow_queue_id_s queue_ids[], 496 int *rss_target_id, enum flow_eth_dev_profile flow_profile, 497 uint32_t exception_path) 498 { 499 const struct profile_inline_ops *profile_inline_ops = get_profile_inline_ops(); 500 501 if (profile_inline_ops == NULL) 502 NT_LOG(ERR, FILTER, "%s: profile_inline module uninitialized", __func__); 503 504 int i; 505 struct flow_eth_dev *eth_dev = NULL; 506 507 NT_LOG(DBG, FILTER, 508 "Get eth-port adapter %i, port %i, port_id %u, rx queues %i, profile %i", 509 adapter_no, port_no, port_id, alloc_rx_queues, flow_profile); 510 511 if (MAX_OUTPUT_DEST < FLOW_MAX_QUEUES) { 512 assert(0); 513 NT_LOG(ERR, FILTER, 514 "ERROR: Internal array for multiple queues too small for API"); 515 } 516 517 pthread_mutex_lock(&base_mtx); 518 struct flow_nic_dev *ndev = get_nic_dev_from_adapter_no(adapter_no); 519 520 if (!ndev) { 521 /* Error - no flow api found on specified adapter */ 522 NT_LOG(ERR, FILTER, "ERROR: no flow interface registered for adapter %d", 523 adapter_no); 524 pthread_mutex_unlock(&base_mtx); 525 return NULL; 526 } 527 528 if (ndev->ports < ((uint16_t)port_no + 1)) { 529 NT_LOG(ERR, FILTER, "ERROR: port exceeds supported port range for adapter"); 530 pthread_mutex_unlock(&base_mtx); 531 return NULL; 532 } 533 534 if ((alloc_rx_queues - 1) > FLOW_MAX_QUEUES) { /* 0th is exception so +1 */ 535 NT_LOG(ERR, FILTER, 536 "ERROR: Exceeds supported number of rx queues per eth device"); 537 pthread_mutex_unlock(&base_mtx); 538 return NULL; 539 } 540 541 /* don't accept multiple eth_dev's on same NIC and same port */ 542 eth_dev = nic_and_port_to_eth_dev(adapter_no, port_no); 543 544 if (eth_dev) { 545 NT_LOG(DBG, FILTER, "Re-opening existing NIC port device: NIC DEV: %i Port %i", 546 adapter_no, port_no); 547 pthread_mutex_unlock(&base_mtx); 548 flow_delete_eth_dev(eth_dev); 549 eth_dev = NULL; 550 } 551 552 eth_dev = calloc(1, sizeof(struct flow_eth_dev)); 553 554 if (!eth_dev) { 555 NT_LOG(ERR, FILTER, "ERROR: calloc failed"); 556 goto err_exit1; 557 } 558 559 pthread_mutex_lock(&ndev->mtx); 560 561 eth_dev->ndev = ndev; 562 eth_dev->port = port_no; 563 eth_dev->port_id = port_id; 564 565 /* First time then NIC is initialized */ 566 if (!ndev->flow_mgnt_prepared) { 567 ndev->flow_profile = flow_profile; 568 569 /* Initialize modules if needed - recipe 0 is used as no-match and must be setup */ 570 if (profile_inline_ops != NULL && 571 profile_inline_ops->initialize_flow_management_of_ndev_profile_inline(ndev)) 572 goto err_exit0; 573 574 } else { 575 /* check if same flow type is requested, otherwise fail */ 576 if (ndev->flow_profile != flow_profile) { 577 NT_LOG(ERR, FILTER, 578 "ERROR: Different flow types requested on same NIC device. Not supported."); 579 goto err_exit0; 580 } 581 } 582 583 /* Allocate the requested queues in HW for this dev */ 584 585 for (i = 0; i < alloc_rx_queues; i++) { 586 eth_dev->rx_queue[i] = queue_ids[i]; 587 588 if (i == 0 && (flow_profile == FLOW_ETH_DEV_PROFILE_INLINE && exception_path)) { 589 /* 590 * Init QSL UNM - unmatched - redirects otherwise discarded 591 * packets in QSL 592 */ 593 if (hw_mod_qsl_unmq_set(&ndev->be, HW_QSL_UNMQ_DEST_QUEUE, eth_dev->port, 594 eth_dev->rx_queue[0].hw_id) < 0) 595 goto err_exit0; 596 597 if (hw_mod_qsl_unmq_set(&ndev->be, HW_QSL_UNMQ_EN, eth_dev->port, 1) < 0) 598 goto err_exit0; 599 600 if (hw_mod_qsl_unmq_flush(&ndev->be, eth_dev->port, 1) < 0) 601 goto err_exit0; 602 } 603 604 eth_dev->num_queues++; 605 } 606 607 eth_dev->rss_target_id = -1; 608 609 if (flow_profile == FLOW_ETH_DEV_PROFILE_INLINE) { 610 for (i = 0; i < eth_dev->num_queues; i++) { 611 uint32_t qen_value = 0; 612 uint32_t queue_id = (uint32_t)eth_dev->rx_queue[i].hw_id; 613 614 hw_mod_qsl_qen_get(&ndev->be, HW_QSL_QEN_EN, queue_id / 4, &qen_value); 615 hw_mod_qsl_qen_set(&ndev->be, HW_QSL_QEN_EN, queue_id / 4, 616 qen_value | (1 << (queue_id % 4))); 617 hw_mod_qsl_qen_flush(&ndev->be, queue_id / 4, 1); 618 } 619 } 620 621 *rss_target_id = eth_dev->rss_target_id; 622 623 nic_insert_eth_port_dev(ndev, eth_dev); 624 625 pthread_mutex_unlock(&ndev->mtx); 626 pthread_mutex_unlock(&base_mtx); 627 return eth_dev; 628 629 err_exit0: 630 pthread_mutex_unlock(&ndev->mtx); 631 pthread_mutex_unlock(&base_mtx); 632 633 err_exit1: 634 if (eth_dev) 635 free(eth_dev); 636 637 #ifdef FLOW_DEBUG 638 ndev->be.iface->set_debug_mode(ndev->be.be_dev, FLOW_BACKEND_DEBUG_MODE_NONE); 639 #endif 640 641 NT_LOG(DBG, FILTER, "ERR in %s", __func__); 642 return NULL; /* Error exit */ 643 } 644 645 struct flow_nic_dev *flow_api_create(uint8_t adapter_no, const struct flow_api_backend_ops *be_if, 646 void *be_dev) 647 { 648 (void)adapter_no; 649 650 if (!be_if || be_if->version != 1) { 651 NT_LOG(DBG, FILTER, "ERR: %s", __func__); 652 return NULL; 653 } 654 655 struct flow_nic_dev *ndev = calloc(1, sizeof(struct flow_nic_dev)); 656 657 if (!ndev) { 658 NT_LOG(ERR, FILTER, "ERROR: calloc failed"); 659 return NULL; 660 } 661 662 /* 663 * To dump module initialization writes use 664 * FLOW_BACKEND_DEBUG_MODE_WRITE 665 * then remember to set it ...NONE afterwards again 666 */ 667 be_if->set_debug_mode(be_dev, FLOW_BACKEND_DEBUG_MODE_NONE); 668 669 if (flow_api_backend_init(&ndev->be, be_if, be_dev) != 0) 670 goto err_exit; 671 672 ndev->adapter_no = adapter_no; 673 674 ndev->ports = (uint16_t)((ndev->be.num_rx_ports > 256) ? 256 : ndev->be.num_rx_ports); 675 676 /* 677 * Free resources in NIC must be managed by this module 678 * Get resource sizes and create resource manager elements 679 */ 680 if (init_resource_elements(ndev, RES_QUEUE, ndev->be.max_queues)) 681 goto err_exit; 682 683 if (init_resource_elements(ndev, RES_CAT_CFN, ndev->be.cat.nb_cat_funcs)) 684 goto err_exit; 685 686 if (init_resource_elements(ndev, RES_CAT_COT, ndev->be.max_categories)) 687 goto err_exit; 688 689 if (init_resource_elements(ndev, RES_CAT_EXO, ndev->be.cat.nb_pm_ext)) 690 goto err_exit; 691 692 if (init_resource_elements(ndev, RES_CAT_LEN, ndev->be.cat.nb_len)) 693 goto err_exit; 694 695 if (init_resource_elements(ndev, RES_KM_FLOW_TYPE, ndev->be.cat.nb_flow_types)) 696 goto err_exit; 697 698 if (init_resource_elements(ndev, RES_KM_CATEGORY, ndev->be.km.nb_categories)) 699 goto err_exit; 700 701 if (init_resource_elements(ndev, RES_HSH_RCP, ndev->be.hsh.nb_rcp)) 702 goto err_exit; 703 704 if (init_resource_elements(ndev, RES_PDB_RCP, ndev->be.pdb.nb_pdb_rcp_categories)) 705 goto err_exit; 706 707 if (init_resource_elements(ndev, RES_QSL_RCP, ndev->be.qsl.nb_rcp_categories)) 708 goto err_exit; 709 710 if (init_resource_elements(ndev, RES_QSL_QST, ndev->be.qsl.nb_qst_entries)) 711 goto err_exit; 712 713 if (init_resource_elements(ndev, RES_SLC_LR_RCP, ndev->be.max_categories)) 714 goto err_exit; 715 716 if (init_resource_elements(ndev, RES_FLM_FLOW_TYPE, ndev->be.cat.nb_flow_types)) 717 goto err_exit; 718 719 if (init_resource_elements(ndev, RES_FLM_RCP, ndev->be.flm.nb_categories)) 720 goto err_exit; 721 722 if (init_resource_elements(ndev, RES_TPE_RCP, ndev->be.tpe.nb_rcp_categories)) 723 goto err_exit; 724 725 if (init_resource_elements(ndev, RES_TPE_EXT, ndev->be.tpe.nb_rpl_ext_categories)) 726 goto err_exit; 727 728 if (init_resource_elements(ndev, RES_TPE_RPL, ndev->be.tpe.nb_rpl_depth)) 729 goto err_exit; 730 731 if (init_resource_elements(ndev, RES_SCRUB_RCP, ndev->be.flm.nb_scrub_profiles)) 732 goto err_exit; 733 734 /* may need IPF, COR */ 735 736 /* check all defined has been initialized */ 737 for (int i = 0; i < RES_COUNT; i++) 738 assert(ndev->res[i].alloc_bm); 739 740 pthread_mutex_init(&ndev->mtx, NULL); 741 list_insert_flow_nic(ndev); 742 743 return ndev; 744 745 err_exit: 746 747 if (ndev) 748 flow_api_done(ndev); 749 750 NT_LOG(DBG, FILTER, "ERR: %s", __func__); 751 return NULL; 752 } 753 754 int flow_api_done(struct flow_nic_dev *ndev) 755 { 756 NT_LOG(DBG, FILTER, "FLOW API DONE"); 757 758 if (ndev) { 759 flow_ndev_reset(ndev); 760 761 /* delete resource management allocations for this ndev */ 762 for (int i = 0; i < RES_COUNT; i++) 763 done_resource_elements(ndev, i); 764 765 flow_api_backend_done(&ndev->be); 766 list_remove_flow_nic(ndev); 767 free(ndev); 768 } 769 770 return 0; 771 } 772 773 void *flow_api_get_be_dev(struct flow_nic_dev *ndev) 774 { 775 if (!ndev) { 776 NT_LOG(DBG, FILTER, "ERR: %s", __func__); 777 return NULL; 778 } 779 780 return ndev->be.be_dev; 781 } 782 783 /* Information for a given RSS type. */ 784 struct rss_type_info { 785 uint64_t rss_type; 786 const char *str; 787 }; 788 789 static struct rss_type_info rss_to_string[] = { 790 /* RTE_BIT64(2) IPv4 dst + IPv4 src */ 791 RSS_TO_STRING(RTE_ETH_RSS_IPV4), 792 /* RTE_BIT64(3) IPv4 dst + IPv4 src + Identification of group of fragments */ 793 RSS_TO_STRING(RTE_ETH_RSS_FRAG_IPV4), 794 /* RTE_BIT64(4) IPv4 dst + IPv4 src + L4 protocol */ 795 RSS_TO_STRING(RTE_ETH_RSS_NONFRAG_IPV4_TCP), 796 /* RTE_BIT64(5) IPv4 dst + IPv4 src + L4 protocol */ 797 RSS_TO_STRING(RTE_ETH_RSS_NONFRAG_IPV4_UDP), 798 /* RTE_BIT64(6) IPv4 dst + IPv4 src + L4 protocol */ 799 RSS_TO_STRING(RTE_ETH_RSS_NONFRAG_IPV4_SCTP), 800 /* RTE_BIT64(7) IPv4 dst + IPv4 src + L4 protocol */ 801 RSS_TO_STRING(RTE_ETH_RSS_NONFRAG_IPV4_OTHER), 802 /* 803 * RTE_BIT64(14) 128-bits of L2 payload starting after src MAC, i.e. including optional 804 * VLAN tag and ethertype. Overrides all L3 and L4 flags at the same level, but inner 805 * L2 payload can be combined with outer S-VLAN and GTPU TEID flags. 806 */ 807 RSS_TO_STRING(RTE_ETH_RSS_L2_PAYLOAD), 808 /* RTE_BIT64(18) L4 dst + L4 src + L4 protocol - see comment of RTE_ETH_RSS_L4_CHKSUM */ 809 RSS_TO_STRING(RTE_ETH_RSS_PORT), 810 /* RTE_BIT64(19) Not supported */ 811 RSS_TO_STRING(RTE_ETH_RSS_VXLAN), 812 /* RTE_BIT64(20) Not supported */ 813 RSS_TO_STRING(RTE_ETH_RSS_GENEVE), 814 /* RTE_BIT64(21) Not supported */ 815 RSS_TO_STRING(RTE_ETH_RSS_NVGRE), 816 /* RTE_BIT64(23) GTP TEID - always from outer GTPU header */ 817 RSS_TO_STRING(RTE_ETH_RSS_GTPU), 818 /* RTE_BIT64(24) MAC dst + MAC src */ 819 RSS_TO_STRING(RTE_ETH_RSS_ETH), 820 /* RTE_BIT64(25) outermost VLAN ID + L4 protocol */ 821 RSS_TO_STRING(RTE_ETH_RSS_S_VLAN), 822 /* RTE_BIT64(26) innermost VLAN ID + L4 protocol */ 823 RSS_TO_STRING(RTE_ETH_RSS_C_VLAN), 824 /* RTE_BIT64(27) Not supported */ 825 RSS_TO_STRING(RTE_ETH_RSS_ESP), 826 /* RTE_BIT64(28) Not supported */ 827 RSS_TO_STRING(RTE_ETH_RSS_AH), 828 /* RTE_BIT64(29) Not supported */ 829 RSS_TO_STRING(RTE_ETH_RSS_L2TPV3), 830 /* RTE_BIT64(30) Not supported */ 831 RSS_TO_STRING(RTE_ETH_RSS_PFCP), 832 /* RTE_BIT64(31) Not supported */ 833 RSS_TO_STRING(RTE_ETH_RSS_PPPOE), 834 /* RTE_BIT64(32) Not supported */ 835 RSS_TO_STRING(RTE_ETH_RSS_ECPRI), 836 /* RTE_BIT64(33) Not supported */ 837 RSS_TO_STRING(RTE_ETH_RSS_MPLS), 838 /* RTE_BIT64(34) IPv4 Header checksum + L4 protocol */ 839 RSS_TO_STRING(RTE_ETH_RSS_IPV4_CHKSUM), 840 841 /* 842 * if combined with RTE_ETH_RSS_NONFRAG_IPV4_[TCP|UDP|SCTP] then 843 * L4 protocol + chosen protocol header Checksum 844 * else 845 * error 846 */ 847 /* RTE_BIT64(35) */ 848 RSS_TO_STRING(RTE_ETH_RSS_L4_CHKSUM), 849 #ifndef ANDROMEDA_DPDK_21_11 850 /* RTE_BIT64(36) Not supported */ 851 RSS_TO_STRING(RTE_ETH_RSS_L2TPV2), 852 #endif 853 854 { RTE_BIT64(37), "unknown_RTE_BIT64(37)" }, 855 { RTE_BIT64(38), "unknown_RTE_BIT64(38)" }, 856 { RTE_BIT64(39), "unknown_RTE_BIT64(39)" }, 857 { RTE_BIT64(40), "unknown_RTE_BIT64(40)" }, 858 { RTE_BIT64(41), "unknown_RTE_BIT64(41)" }, 859 { RTE_BIT64(42), "unknown_RTE_BIT64(42)" }, 860 { RTE_BIT64(43), "unknown_RTE_BIT64(43)" }, 861 { RTE_BIT64(44), "unknown_RTE_BIT64(44)" }, 862 { RTE_BIT64(45), "unknown_RTE_BIT64(45)" }, 863 { RTE_BIT64(46), "unknown_RTE_BIT64(46)" }, 864 { RTE_BIT64(47), "unknown_RTE_BIT64(47)" }, 865 { RTE_BIT64(48), "unknown_RTE_BIT64(48)" }, 866 { RTE_BIT64(49), "unknown_RTE_BIT64(49)" }, 867 868 /* RTE_BIT64(50) outermost encapsulation */ 869 RSS_TO_STRING(RTE_ETH_RSS_LEVEL_OUTERMOST), 870 /* RTE_BIT64(51) innermost encapsulation */ 871 RSS_TO_STRING(RTE_ETH_RSS_LEVEL_INNERMOST), 872 873 /* RTE_BIT64(52) Not supported */ 874 RSS_TO_STRING(RTE_ETH_RSS_L3_PRE96), 875 /* RTE_BIT64(53) Not supported */ 876 RSS_TO_STRING(RTE_ETH_RSS_L3_PRE64), 877 /* RTE_BIT64(54) Not supported */ 878 RSS_TO_STRING(RTE_ETH_RSS_L3_PRE56), 879 /* RTE_BIT64(55) Not supported */ 880 RSS_TO_STRING(RTE_ETH_RSS_L3_PRE48), 881 /* RTE_BIT64(56) Not supported */ 882 RSS_TO_STRING(RTE_ETH_RSS_L3_PRE40), 883 /* RTE_BIT64(57) Not supported */ 884 RSS_TO_STRING(RTE_ETH_RSS_L3_PRE32), 885 886 /* RTE_BIT64(58) */ 887 RSS_TO_STRING(RTE_ETH_RSS_L2_DST_ONLY), 888 /* RTE_BIT64(59) */ 889 RSS_TO_STRING(RTE_ETH_RSS_L2_SRC_ONLY), 890 /* RTE_BIT64(60) */ 891 RSS_TO_STRING(RTE_ETH_RSS_L4_DST_ONLY), 892 /* RTE_BIT64(61) */ 893 RSS_TO_STRING(RTE_ETH_RSS_L4_SRC_ONLY), 894 /* RTE_BIT64(62) */ 895 RSS_TO_STRING(RTE_ETH_RSS_L3_DST_ONLY), 896 /* RTE_BIT64(63) */ 897 RSS_TO_STRING(RTE_ETH_RSS_L3_SRC_ONLY), 898 }; 899 900 int sprint_nt_rss_mask(char *str, uint16_t str_len, const char *prefix, uint64_t hash_mask) 901 { 902 if (str == NULL || str_len == 0) 903 return -1; 904 905 memset(str, 0x0, str_len); 906 uint16_t str_end = 0; 907 const struct rss_type_info *start = rss_to_string; 908 909 for (const struct rss_type_info *p = start; p != start + ARRAY_SIZE(rss_to_string); ++p) { 910 if (p->rss_type & hash_mask) { 911 if (strlen(prefix) + strlen(p->str) < (size_t)(str_len - str_end)) { 912 snprintf(str + str_end, str_len - str_end, "%s", prefix); 913 str_end += strlen(prefix); 914 snprintf(str + str_end, str_len - str_end, "%s", p->str); 915 str_end += strlen(p->str); 916 917 } else { 918 return -1; 919 } 920 } 921 } 922 923 return 0; 924 } 925 926 /* 927 * Hash 928 */ 929 930 int flow_nic_set_hasher(struct flow_nic_dev *ndev, int hsh_idx, enum flow_nic_hash_e algorithm) 931 { 932 hw_mod_hsh_rcp_set(&ndev->be, HW_HSH_RCP_PRESET_ALL, hsh_idx, 0, 0); 933 934 switch (algorithm) { 935 case HASH_ALGO_5TUPLE: 936 /* need to create an IPv6 hashing and enable the adaptive ip mask bit */ 937 hw_mod_hsh_rcp_set(&ndev->be, HW_HSH_RCP_LOAD_DIST_TYPE, hsh_idx, 0, 2); 938 hw_mod_hsh_rcp_set(&ndev->be, HW_HSH_RCP_QW0_PE, hsh_idx, 0, DYN_FINAL_IP_DST); 939 hw_mod_hsh_rcp_set(&ndev->be, HW_HSH_RCP_QW0_OFS, hsh_idx, 0, -16); 940 hw_mod_hsh_rcp_set(&ndev->be, HW_HSH_RCP_QW4_PE, hsh_idx, 0, DYN_FINAL_IP_DST); 941 hw_mod_hsh_rcp_set(&ndev->be, HW_HSH_RCP_QW4_OFS, hsh_idx, 0, 0); 942 hw_mod_hsh_rcp_set(&ndev->be, HW_HSH_RCP_W8_PE, hsh_idx, 0, DYN_L4); 943 hw_mod_hsh_rcp_set(&ndev->be, HW_HSH_RCP_W8_OFS, hsh_idx, 0, 0); 944 hw_mod_hsh_rcp_set(&ndev->be, HW_HSH_RCP_W9_PE, hsh_idx, 0, 0); 945 hw_mod_hsh_rcp_set(&ndev->be, HW_HSH_RCP_W9_OFS, hsh_idx, 0, 0); 946 hw_mod_hsh_rcp_set(&ndev->be, HW_HSH_RCP_W9_P, hsh_idx, 0, 0); 947 hw_mod_hsh_rcp_set(&ndev->be, HW_HSH_RCP_P_MASK, hsh_idx, 0, 1); 948 hw_mod_hsh_rcp_set(&ndev->be, HW_HSH_RCP_WORD_MASK, hsh_idx, 0, 0xffffffff); 949 hw_mod_hsh_rcp_set(&ndev->be, HW_HSH_RCP_WORD_MASK, hsh_idx, 1, 0xffffffff); 950 hw_mod_hsh_rcp_set(&ndev->be, HW_HSH_RCP_WORD_MASK, hsh_idx, 2, 0xffffffff); 951 hw_mod_hsh_rcp_set(&ndev->be, HW_HSH_RCP_WORD_MASK, hsh_idx, 3, 0xffffffff); 952 hw_mod_hsh_rcp_set(&ndev->be, HW_HSH_RCP_WORD_MASK, hsh_idx, 4, 0xffffffff); 953 hw_mod_hsh_rcp_set(&ndev->be, HW_HSH_RCP_WORD_MASK, hsh_idx, 5, 0xffffffff); 954 hw_mod_hsh_rcp_set(&ndev->be, HW_HSH_RCP_WORD_MASK, hsh_idx, 6, 0xffffffff); 955 hw_mod_hsh_rcp_set(&ndev->be, HW_HSH_RCP_WORD_MASK, hsh_idx, 7, 0xffffffff); 956 hw_mod_hsh_rcp_set(&ndev->be, HW_HSH_RCP_WORD_MASK, hsh_idx, 8, 0xffffffff); 957 hw_mod_hsh_rcp_set(&ndev->be, HW_HSH_RCP_WORD_MASK, hsh_idx, 9, 0); 958 hw_mod_hsh_rcp_set(&ndev->be, HW_HSH_RCP_SEED, hsh_idx, 0, 0xffffffff); 959 hw_mod_hsh_rcp_set(&ndev->be, HW_HSH_RCP_HSH_VALID, hsh_idx, 0, 1); 960 hw_mod_hsh_rcp_set(&ndev->be, HW_HSH_RCP_HSH_TYPE, hsh_idx, 0, HASH_5TUPLE); 961 hw_mod_hsh_rcp_set(&ndev->be, HW_HSH_RCP_AUTO_IPV4_MASK, hsh_idx, 0, 1); 962 963 NT_LOG(DBG, FILTER, "Set IPv6 5-tuple hasher with adaptive IPv4 hashing"); 964 break; 965 966 default: 967 case HASH_ALGO_ROUND_ROBIN: 968 /* zero is round-robin */ 969 break; 970 } 971 972 return 0; 973 } 974 975 static int flow_dev_dump(struct flow_eth_dev *dev, 976 struct flow_handle *flow, 977 uint16_t caller_id, 978 FILE *file, 979 struct rte_flow_error *error) 980 { 981 const struct profile_inline_ops *profile_inline_ops = get_profile_inline_ops(); 982 983 if (profile_inline_ops == NULL) { 984 NT_LOG(ERR, FILTER, "%s: profile_inline module uninitialized", __func__); 985 return -1; 986 } 987 988 return profile_inline_ops->flow_dev_dump_profile_inline(dev, flow, caller_id, file, error); 989 } 990 991 int flow_nic_set_hasher_fields(struct flow_nic_dev *ndev, int hsh_idx, 992 struct nt_eth_rss_conf rss_conf) 993 { 994 const struct profile_inline_ops *profile_inline_ops = get_profile_inline_ops(); 995 996 if (profile_inline_ops == NULL) { 997 NT_LOG(ERR, FILTER, "%s: profile_inline module uninitialized", __func__); 998 return -1; 999 } 1000 1001 return profile_inline_ops->flow_nic_set_hasher_fields_inline(ndev, hsh_idx, rss_conf); 1002 } 1003 1004 static const struct flow_filter_ops ops = { 1005 .flow_filter_init = flow_filter_init, 1006 .flow_filter_done = flow_filter_done, 1007 /* 1008 * Device Management API 1009 */ 1010 .flow_get_eth_dev = flow_get_eth_dev, 1011 /* 1012 * NT Flow API 1013 */ 1014 .flow_create = flow_create, 1015 .flow_destroy = flow_destroy, 1016 .flow_dev_dump = flow_dev_dump, 1017 }; 1018 1019 void init_flow_filter(void) 1020 { 1021 register_flow_filter_ops(&ops); 1022 } 1023