1 /* 2 * SPDX-License-Identifier: BSD-3-Clause 3 * Copyright(c) 2023 Napatech A/S 4 */ 5 #include "rte_spinlock.h" 6 #include "ntlog.h" 7 #include "nt_util.h" 8 9 #include "flow_api_engine.h" 10 #include "flow_api_nic_setup.h" 11 #include "ntnic_mod_reg.h" 12 13 #include "flow_api.h" 14 #include "flow_filter.h" 15 16 #define RSS_TO_STRING(name) \ 17 { \ 18 name, #name \ 19 } 20 21 const char *dbg_res_descr[] = { 22 /* RES_QUEUE */ "RES_QUEUE", 23 /* RES_CAT_CFN */ "RES_CAT_CFN", 24 /* RES_CAT_COT */ "RES_CAT_COT", 25 /* RES_CAT_EXO */ "RES_CAT_EXO", 26 /* RES_CAT_LEN */ "RES_CAT_LEN", 27 /* RES_KM_FLOW_TYPE */ "RES_KM_FLOW_TYPE", 28 /* RES_KM_CATEGORY */ "RES_KM_CATEGORY", 29 /* RES_HSH_RCP */ "RES_HSH_RCP", 30 /* RES_PDB_RCP */ "RES_PDB_RCP", 31 /* RES_QSL_RCP */ "RES_QSL_RCP", 32 /* RES_QSL_LTX */ "RES_QSL_LTX", 33 /* RES_QSL_QST */ "RES_QSL_QST", 34 /* RES_SLC_LR_RCP */ "RES_SLC_LR_RCP", 35 /* RES_FLM_FLOW_TYPE */ "RES_FLM_FLOW_TYPE", 36 /* RES_FLM_RCP */ "RES_FLM_RCP", 37 /* RES_TPE_RCP */ "RES_TPE_RCP", 38 /* RES_TPE_EXT */ "RES_TPE_EXT", 39 /* RES_TPE_RPL */ "RES_TPE_RPL", 40 /* RES_COUNT */ "RES_COUNT", 41 /* RES_INVALID */ "RES_INVALID" 42 }; 43 44 static struct flow_nic_dev *dev_base; 45 static rte_spinlock_t base_mtx = RTE_SPINLOCK_INITIALIZER; 46 47 /* 48 * Error handling 49 */ 50 51 static const struct { 52 const char *message; 53 } err_msg[] = { 54 /* 00 */ { "Operation successfully completed" }, 55 /* 01 */ { "Operation failed" }, 56 /* 02 */ { "Memory allocation failed" }, 57 /* 03 */ { "Too many output destinations" }, 58 /* 04 */ { "Too many output queues for RSS" }, 59 /* 05 */ { "The VLAN TPID specified is not supported" }, 60 /* 06 */ { "The VxLan Push header specified is not accepted" }, 61 /* 07 */ { "While interpreting VxLan Pop action, could not find a destination port" }, 62 /* 08 */ { "Failed in creating a HW-internal VTEP port" }, 63 /* 09 */ { "Too many VLAN tag matches" }, 64 /* 10 */ { "IPv6 invalid header specified" }, 65 /* 11 */ { "Too many tunnel ports. HW limit reached" }, 66 /* 12 */ { "Unknown or unsupported flow match element received" }, 67 /* 13 */ { "Match failed because of HW limitations" }, 68 /* 14 */ { "Match failed because of HW resource limitations" }, 69 /* 15 */ { "Match failed because of too complex element definitions" }, 70 /* 16 */ { "Action failed. To too many output destinations" }, 71 /* 17 */ { "Action Output failed, due to HW resource exhaustion" }, 72 /* 18 */ { "Push Tunnel Header action cannot output to multiple destination queues" }, 73 /* 19 */ { "Inline action HW resource exhaustion" }, 74 /* 20 */ { "Action retransmit/recirculate HW resource exhaustion" }, 75 /* 21 */ { "Flow counter HW resource exhaustion" }, 76 /* 22 */ { "Internal HW resource exhaustion to handle Actions" }, 77 /* 23 */ { "Internal HW QSL compare failed" }, 78 /* 24 */ { "Internal CAT CFN reuse failed" }, 79 /* 25 */ { "Match variations too complex" }, 80 /* 26 */ { "Match failed because of CAM/TCAM full" }, 81 /* 27 */ { "Internal creation of a tunnel end point port failed" }, 82 /* 28 */ { "Unknown or unsupported flow action received" }, 83 /* 29 */ { "Removing flow failed" }, 84 }; 85 86 void flow_nic_set_error(enum flow_nic_err_msg_e msg, struct rte_flow_error *error) 87 { 88 assert(msg < ERR_MSG_NO_MSG); 89 90 if (error) { 91 error->message = err_msg[msg].message; 92 error->type = (msg == ERR_SUCCESS) ? RTE_FLOW_ERROR_TYPE_NONE : 93 RTE_FLOW_ERROR_TYPE_UNSPECIFIED; 94 } 95 } 96 97 /* 98 * Resources 99 */ 100 101 int flow_nic_alloc_resource(struct flow_nic_dev *ndev, enum res_type_e res_type, 102 uint32_t alignment) 103 { 104 for (unsigned int i = 0; i < ndev->res[res_type].resource_count; i += alignment) { 105 if (!flow_nic_is_resource_used(ndev, res_type, i)) { 106 flow_nic_mark_resource_used(ndev, res_type, i); 107 ndev->res[res_type].ref[i] = 1; 108 return i; 109 } 110 } 111 112 return -1; 113 } 114 115 int flow_nic_alloc_resource_config(struct flow_nic_dev *ndev, enum res_type_e res_type, 116 unsigned int num, uint32_t alignment) 117 { 118 unsigned int idx_offs; 119 120 for (unsigned int res_idx = 0; res_idx < ndev->res[res_type].resource_count - (num - 1); 121 res_idx += alignment) { 122 if (!flow_nic_is_resource_used(ndev, res_type, res_idx)) { 123 for (idx_offs = 1; idx_offs < num; idx_offs++) 124 if (flow_nic_is_resource_used(ndev, res_type, res_idx + idx_offs)) 125 break; 126 127 if (idx_offs < num) 128 continue; 129 130 /* found a contiguous number of "num" res_type elements - allocate them */ 131 for (idx_offs = 0; idx_offs < num; idx_offs++) { 132 flow_nic_mark_resource_used(ndev, res_type, res_idx + idx_offs); 133 ndev->res[res_type].ref[res_idx + idx_offs] = 1; 134 } 135 136 return res_idx; 137 } 138 } 139 140 return -1; 141 } 142 143 void flow_nic_free_resource(struct flow_nic_dev *ndev, enum res_type_e res_type, int idx) 144 { 145 flow_nic_mark_resource_unused(ndev, res_type, idx); 146 } 147 148 int flow_nic_ref_resource(struct flow_nic_dev *ndev, enum res_type_e res_type, int index) 149 { 150 NT_LOG(DBG, FILTER, "Reference resource %s idx %i (before ref cnt %i)", 151 dbg_res_descr[res_type], index, ndev->res[res_type].ref[index]); 152 assert(flow_nic_is_resource_used(ndev, res_type, index)); 153 154 if (ndev->res[res_type].ref[index] == (uint32_t)-1) 155 return -1; 156 157 ndev->res[res_type].ref[index]++; 158 return 0; 159 } 160 161 int flow_nic_deref_resource(struct flow_nic_dev *ndev, enum res_type_e res_type, int index) 162 { 163 NT_LOG(DBG, FILTER, "De-reference resource %s idx %i (before ref cnt %i)", 164 dbg_res_descr[res_type], index, ndev->res[res_type].ref[index]); 165 assert(flow_nic_is_resource_used(ndev, res_type, index)); 166 assert(ndev->res[res_type].ref[index]); 167 /* deref */ 168 ndev->res[res_type].ref[index]--; 169 170 if (!ndev->res[res_type].ref[index]) 171 flow_nic_free_resource(ndev, res_type, index); 172 173 return !!ndev->res[res_type].ref[index];/* if 0 resource has been freed */ 174 } 175 176 /* 177 * Nic port/adapter lookup 178 */ 179 180 static struct flow_eth_dev *nic_and_port_to_eth_dev(uint8_t adapter_no, uint8_t port) 181 { 182 struct flow_nic_dev *nic_dev = dev_base; 183 184 while (nic_dev) { 185 if (nic_dev->adapter_no == adapter_no) 186 break; 187 188 nic_dev = nic_dev->next; 189 } 190 191 if (!nic_dev) 192 return NULL; 193 194 struct flow_eth_dev *dev = nic_dev->eth_base; 195 196 while (dev) { 197 if (port == dev->port) 198 return dev; 199 200 dev = dev->next; 201 } 202 203 return NULL; 204 } 205 206 static struct flow_nic_dev *get_nic_dev_from_adapter_no(uint8_t adapter_no) 207 { 208 struct flow_nic_dev *ndev = dev_base; 209 210 while (ndev) { 211 if (adapter_no == ndev->adapter_no) 212 break; 213 214 ndev = ndev->next; 215 } 216 217 return ndev; 218 } 219 /* 220 * Flow API 221 */ 222 223 static struct flow_handle *flow_create(struct flow_eth_dev *dev __rte_unused, 224 const struct rte_flow_attr *attr __rte_unused, 225 uint16_t forced_vlan_vid __rte_unused, 226 uint16_t caller_id __rte_unused, 227 const struct rte_flow_item item[] __rte_unused, 228 const struct rte_flow_action action[] __rte_unused, 229 struct rte_flow_error *error __rte_unused) 230 { 231 const struct profile_inline_ops *profile_inline_ops = get_profile_inline_ops(); 232 233 if (profile_inline_ops == NULL) { 234 NT_LOG(ERR, FILTER, "%s: profile_inline module uninitialized", __func__); 235 return NULL; 236 } 237 238 return profile_inline_ops->flow_create_profile_inline(dev, attr, 239 forced_vlan_vid, caller_id, item, action, error); 240 } 241 242 static int flow_destroy(struct flow_eth_dev *dev __rte_unused, 243 struct flow_handle *flow __rte_unused, struct rte_flow_error *error __rte_unused) 244 { 245 const struct profile_inline_ops *profile_inline_ops = get_profile_inline_ops(); 246 247 if (profile_inline_ops == NULL) { 248 NT_LOG(ERR, FILTER, "%s: profile_inline module uninitialized", __func__); 249 return -1; 250 } 251 252 return profile_inline_ops->flow_destroy_profile_inline(dev, flow, error); 253 } 254 255 static int flow_flush(struct flow_eth_dev *dev, uint16_t caller_id, struct rte_flow_error *error) 256 { 257 const struct profile_inline_ops *profile_inline_ops = get_profile_inline_ops(); 258 259 if (profile_inline_ops == NULL) { 260 NT_LOG_DBGX(ERR, FILTER, "profile_inline module uninitialized"); 261 return -1; 262 } 263 264 return profile_inline_ops->flow_flush_profile_inline(dev, caller_id, error); 265 } 266 267 static int flow_actions_update(struct flow_eth_dev *dev, 268 struct flow_handle *flow, 269 const struct rte_flow_action action[], 270 struct rte_flow_error *error) 271 { 272 const struct profile_inline_ops *profile_inline_ops = get_profile_inline_ops(); 273 274 if (profile_inline_ops == NULL) { 275 NT_LOG_DBGX(ERR, FILTER, "profile_inline module uninitialized"); 276 return -1; 277 } 278 279 return profile_inline_ops->flow_actions_update_profile_inline(dev, flow, action, error); 280 } 281 282 /* 283 * Device Management API 284 */ 285 286 static void nic_insert_eth_port_dev(struct flow_nic_dev *ndev, struct flow_eth_dev *dev) 287 { 288 dev->next = ndev->eth_base; 289 ndev->eth_base = dev; 290 } 291 292 static int nic_remove_eth_port_dev(struct flow_nic_dev *ndev, struct flow_eth_dev *eth_dev) 293 { 294 struct flow_eth_dev *dev = ndev->eth_base, *prev = NULL; 295 296 while (dev) { 297 if (dev == eth_dev) { 298 if (prev) 299 prev->next = dev->next; 300 301 else 302 ndev->eth_base = dev->next; 303 304 return 0; 305 } 306 307 prev = dev; 308 dev = dev->next; 309 } 310 311 return -1; 312 } 313 314 static void flow_ndev_reset(struct flow_nic_dev *ndev) 315 { 316 const struct profile_inline_ops *profile_inline_ops = get_profile_inline_ops(); 317 318 if (profile_inline_ops == NULL) { 319 NT_LOG(ERR, FILTER, "%s: profile_inline module uninitialized", __func__); 320 return; 321 } 322 323 /* Delete all eth-port devices created on this NIC device */ 324 while (ndev->eth_base) 325 flow_delete_eth_dev(ndev->eth_base); 326 327 /* Error check */ 328 while (ndev->flow_base) { 329 NT_LOG(ERR, FILTER, 330 "ERROR : Flows still defined but all eth-ports deleted. Flow %p", 331 ndev->flow_base); 332 333 profile_inline_ops->flow_destroy_profile_inline(ndev->flow_base->dev, 334 ndev->flow_base, NULL); 335 } 336 337 profile_inline_ops->done_flow_management_of_ndev_profile_inline(ndev); 338 339 km_free_ndev_resource_management(&ndev->km_res_handle); 340 kcc_free_ndev_resource_management(&ndev->kcc_res_handle); 341 342 ndev->flow_unique_id_counter = 0; 343 344 /* 345 * free all resources default allocated, initially for this NIC DEV 346 * Is not really needed since the bitmap will be freed in a sec. Therefore 347 * only in debug mode 348 */ 349 350 /* Check if all resources has been released */ 351 NT_LOG(DBG, FILTER, "Delete NIC DEV Adaptor %i", ndev->adapter_no); 352 353 for (unsigned int i = 0; i < RES_COUNT; i++) { 354 int err = 0; 355 NT_LOG(DBG, FILTER, "RES state for: %s", dbg_res_descr[i]); 356 357 for (unsigned int ii = 0; ii < ndev->res[i].resource_count; ii++) { 358 int ref = ndev->res[i].ref[ii]; 359 int used = flow_nic_is_resource_used(ndev, i, ii); 360 361 if (ref || used) { 362 NT_LOG(DBG, FILTER, " [%i]: ref cnt %i, used %i", ii, ref, 363 used); 364 err = 1; 365 } 366 } 367 368 if (err) 369 NT_LOG(DBG, FILTER, "ERROR - some resources not freed"); 370 } 371 372 } 373 374 int flow_delete_eth_dev(struct flow_eth_dev *eth_dev) 375 { 376 const struct profile_inline_ops *profile_inline_ops = get_profile_inline_ops(); 377 378 if (profile_inline_ops == NULL) { 379 NT_LOG(ERR, FILTER, "%s: profile_inline module uninitialized", __func__); 380 return -1; 381 } 382 383 struct flow_nic_dev *ndev = eth_dev->ndev; 384 385 if (!ndev) { 386 /* Error invalid nic device */ 387 return -1; 388 } 389 390 NT_LOG(DBG, FILTER, "Delete eth-port device %p, port %i", eth_dev, eth_dev->port); 391 392 #ifdef FLOW_DEBUG 393 ndev->be.iface->set_debug_mode(ndev->be.be_dev, FLOW_BACKEND_DEBUG_MODE_WRITE); 394 #endif 395 396 /* delete all created flows from this device */ 397 rte_spinlock_lock(&ndev->mtx); 398 399 struct flow_handle *flow = ndev->flow_base; 400 401 while (flow) { 402 if (flow->dev == eth_dev) { 403 struct flow_handle *flow_next = flow->next; 404 profile_inline_ops->flow_destroy_locked_profile_inline(eth_dev, flow, 405 NULL); 406 flow = flow_next; 407 408 } else { 409 flow = flow->next; 410 } 411 } 412 413 /* 414 * remove unmatched queue if setup in QSL 415 * remove exception queue setting in QSL UNM 416 */ 417 hw_mod_qsl_unmq_set(&ndev->be, HW_QSL_UNMQ_DEST_QUEUE, eth_dev->port, 0); 418 hw_mod_qsl_unmq_set(&ndev->be, HW_QSL_UNMQ_EN, eth_dev->port, 0); 419 hw_mod_qsl_unmq_flush(&ndev->be, eth_dev->port, 1); 420 421 if (ndev->flow_profile == FLOW_ETH_DEV_PROFILE_INLINE) { 422 for (int i = 0; i < eth_dev->num_queues; ++i) { 423 uint32_t qen_value = 0; 424 uint32_t queue_id = (uint32_t)eth_dev->rx_queue[i].hw_id; 425 426 hw_mod_qsl_qen_get(&ndev->be, HW_QSL_QEN_EN, queue_id / 4, &qen_value); 427 hw_mod_qsl_qen_set(&ndev->be, HW_QSL_QEN_EN, queue_id / 4, 428 qen_value & ~(1U << (queue_id % 4))); 429 hw_mod_qsl_qen_flush(&ndev->be, queue_id / 4, 1); 430 } 431 } 432 433 #ifdef FLOW_DEBUG 434 ndev->be.iface->set_debug_mode(ndev->be.be_dev, FLOW_BACKEND_DEBUG_MODE_NONE); 435 #endif 436 437 /* take eth_dev out of ndev list */ 438 if (nic_remove_eth_port_dev(ndev, eth_dev) != 0) 439 NT_LOG(ERR, FILTER, "ERROR : eth_dev %p not found", eth_dev); 440 441 rte_spinlock_unlock(&ndev->mtx); 442 443 /* free eth_dev */ 444 free(eth_dev); 445 446 return 0; 447 } 448 449 /* 450 * Flow API NIC Setup 451 * Flow backend creation function - register and initialize common backend API to FPA modules 452 */ 453 454 static int init_resource_elements(struct flow_nic_dev *ndev, enum res_type_e res_type, 455 uint32_t count) 456 { 457 assert(ndev->res[res_type].alloc_bm == NULL); 458 /* allocate bitmap and ref counter */ 459 ndev->res[res_type].alloc_bm = 460 calloc(1, BIT_CONTAINER_8_ALIGN(count) + count * sizeof(uint32_t)); 461 462 if (ndev->res[res_type].alloc_bm) { 463 ndev->res[res_type].ref = 464 (uint32_t *)&ndev->res[res_type].alloc_bm[BIT_CONTAINER_8_ALIGN(count)]; 465 ndev->res[res_type].resource_count = count; 466 return 0; 467 } 468 469 return -1; 470 } 471 472 static void done_resource_elements(struct flow_nic_dev *ndev, enum res_type_e res_type) 473 { 474 assert(ndev); 475 476 free(ndev->res[res_type].alloc_bm); 477 } 478 479 static void list_insert_flow_nic(struct flow_nic_dev *ndev) 480 { 481 rte_spinlock_lock(&base_mtx); 482 ndev->next = dev_base; 483 dev_base = ndev; 484 rte_spinlock_unlock(&base_mtx); 485 } 486 487 static int list_remove_flow_nic(struct flow_nic_dev *ndev) 488 { 489 rte_spinlock_lock(&base_mtx); 490 struct flow_nic_dev *nic_dev = dev_base, *prev = NULL; 491 492 while (nic_dev) { 493 if (nic_dev == ndev) { 494 if (prev) 495 prev->next = nic_dev->next; 496 497 else 498 dev_base = nic_dev->next; 499 500 rte_spinlock_unlock(&base_mtx); 501 return 0; 502 } 503 504 prev = nic_dev; 505 nic_dev = nic_dev->next; 506 } 507 508 rte_spinlock_unlock(&base_mtx); 509 return -1; 510 } 511 512 /* 513 * adapter_no physical adapter no 514 * port_no local port no 515 * alloc_rx_queues number of rx-queues to allocate for this eth_dev 516 */ 517 static struct flow_eth_dev *flow_get_eth_dev(uint8_t adapter_no, uint8_t port_no, uint32_t port_id, 518 int alloc_rx_queues, struct flow_queue_id_s queue_ids[], 519 int *rss_target_id, enum flow_eth_dev_profile flow_profile, 520 uint32_t exception_path) 521 { 522 const struct profile_inline_ops *profile_inline_ops = get_profile_inline_ops(); 523 524 if (profile_inline_ops == NULL) 525 NT_LOG(ERR, FILTER, "%s: profile_inline module uninitialized", __func__); 526 527 int i; 528 struct flow_eth_dev *eth_dev = NULL; 529 530 NT_LOG(DBG, FILTER, 531 "Get eth-port adapter %i, port %i, port_id %u, rx queues %i, profile %i", 532 adapter_no, port_no, port_id, alloc_rx_queues, flow_profile); 533 534 if (MAX_OUTPUT_DEST < FLOW_MAX_QUEUES) { 535 assert(0); 536 NT_LOG(ERR, FILTER, 537 "ERROR: Internal array for multiple queues too small for API"); 538 } 539 540 rte_spinlock_lock(&base_mtx); 541 struct flow_nic_dev *ndev = get_nic_dev_from_adapter_no(adapter_no); 542 543 if (!ndev) { 544 /* Error - no flow api found on specified adapter */ 545 NT_LOG(ERR, FILTER, "ERROR: no flow interface registered for adapter %d", 546 adapter_no); 547 rte_spinlock_unlock(&base_mtx); 548 return NULL; 549 } 550 551 if (ndev->ports < ((uint16_t)port_no + 1)) { 552 NT_LOG(ERR, FILTER, "ERROR: port exceeds supported port range for adapter"); 553 rte_spinlock_unlock(&base_mtx); 554 return NULL; 555 } 556 557 if ((alloc_rx_queues - 1) > FLOW_MAX_QUEUES) { /* 0th is exception so +1 */ 558 NT_LOG(ERR, FILTER, 559 "ERROR: Exceeds supported number of rx queues per eth device"); 560 rte_spinlock_unlock(&base_mtx); 561 return NULL; 562 } 563 564 /* don't accept multiple eth_dev's on same NIC and same port */ 565 eth_dev = nic_and_port_to_eth_dev(adapter_no, port_no); 566 567 if (eth_dev) { 568 NT_LOG(DBG, FILTER, "Re-opening existing NIC port device: NIC DEV: %i Port %i", 569 adapter_no, port_no); 570 flow_delete_eth_dev(eth_dev); 571 eth_dev = NULL; 572 } 573 574 rte_spinlock_lock(&ndev->mtx); 575 576 eth_dev = calloc(1, sizeof(struct flow_eth_dev)); 577 578 if (!eth_dev) { 579 NT_LOG(ERR, FILTER, "ERROR: calloc failed"); 580 goto err_exit0; 581 } 582 583 eth_dev->ndev = ndev; 584 eth_dev->port = port_no; 585 eth_dev->port_id = port_id; 586 587 /* First time then NIC is initialized */ 588 if (!ndev->flow_mgnt_prepared) { 589 ndev->flow_profile = flow_profile; 590 591 /* Initialize modules if needed - recipe 0 is used as no-match and must be setup */ 592 if (profile_inline_ops != NULL && 593 profile_inline_ops->initialize_flow_management_of_ndev_profile_inline(ndev)) 594 goto err_exit0; 595 596 } else { 597 /* check if same flow type is requested, otherwise fail */ 598 if (ndev->flow_profile != flow_profile) { 599 NT_LOG(ERR, FILTER, 600 "ERROR: Different flow types requested on same NIC device. Not supported."); 601 goto err_exit0; 602 } 603 } 604 605 /* Allocate the requested queues in HW for this dev */ 606 607 for (i = 0; i < alloc_rx_queues; i++) { 608 eth_dev->rx_queue[i] = queue_ids[i]; 609 610 if (i == 0 && (flow_profile == FLOW_ETH_DEV_PROFILE_INLINE && exception_path)) { 611 /* 612 * Init QSL UNM - unmatched - redirects otherwise discarded 613 * packets in QSL 614 */ 615 if (hw_mod_qsl_unmq_set(&ndev->be, HW_QSL_UNMQ_DEST_QUEUE, eth_dev->port, 616 eth_dev->rx_queue[0].hw_id) < 0) 617 goto err_exit0; 618 619 if (hw_mod_qsl_unmq_set(&ndev->be, HW_QSL_UNMQ_EN, eth_dev->port, 1) < 0) 620 goto err_exit0; 621 622 if (hw_mod_qsl_unmq_flush(&ndev->be, eth_dev->port, 1) < 0) 623 goto err_exit0; 624 } 625 626 eth_dev->num_queues++; 627 } 628 629 eth_dev->rss_target_id = -1; 630 631 if (flow_profile == FLOW_ETH_DEV_PROFILE_INLINE) { 632 for (i = 0; i < eth_dev->num_queues; i++) { 633 uint32_t qen_value = 0; 634 uint32_t queue_id = (uint32_t)eth_dev->rx_queue[i].hw_id; 635 636 hw_mod_qsl_qen_get(&ndev->be, HW_QSL_QEN_EN, queue_id / 4, &qen_value); 637 hw_mod_qsl_qen_set(&ndev->be, HW_QSL_QEN_EN, queue_id / 4, 638 qen_value | (1 << (queue_id % 4))); 639 hw_mod_qsl_qen_flush(&ndev->be, queue_id / 4, 1); 640 } 641 } 642 643 *rss_target_id = eth_dev->rss_target_id; 644 645 nic_insert_eth_port_dev(ndev, eth_dev); 646 647 rte_spinlock_unlock(&ndev->mtx); 648 rte_spinlock_unlock(&base_mtx); 649 return eth_dev; 650 651 err_exit0: 652 rte_spinlock_unlock(&ndev->mtx); 653 rte_spinlock_unlock(&base_mtx); 654 655 free(eth_dev); 656 657 #ifdef FLOW_DEBUG 658 ndev->be.iface->set_debug_mode(ndev->be.be_dev, FLOW_BACKEND_DEBUG_MODE_NONE); 659 #endif 660 661 NT_LOG(DBG, FILTER, "ERR in %s", __func__); 662 return NULL; /* Error exit */ 663 } 664 665 struct flow_nic_dev *flow_api_create(uint8_t adapter_no, const struct flow_api_backend_ops *be_if, 666 void *be_dev) 667 { 668 (void)adapter_no; 669 670 if (!be_if || be_if->version != 1) { 671 NT_LOG(DBG, FILTER, "ERR: %s", __func__); 672 return NULL; 673 } 674 675 struct flow_nic_dev *ndev = calloc(1, sizeof(struct flow_nic_dev)); 676 677 if (!ndev) { 678 NT_LOG(ERR, FILTER, "ERROR: calloc failed"); 679 return NULL; 680 } 681 682 /* 683 * To dump module initialization writes use 684 * FLOW_BACKEND_DEBUG_MODE_WRITE 685 * then remember to set it ...NONE afterwards again 686 */ 687 be_if->set_debug_mode(be_dev, FLOW_BACKEND_DEBUG_MODE_NONE); 688 689 if (flow_api_backend_init(&ndev->be, be_if, be_dev) != 0) 690 goto err_exit; 691 692 ndev->adapter_no = adapter_no; 693 694 ndev->ports = (uint16_t)((ndev->be.num_rx_ports > 256) ? 256 : ndev->be.num_rx_ports); 695 696 /* 697 * Free resources in NIC must be managed by this module 698 * Get resource sizes and create resource manager elements 699 */ 700 if (init_resource_elements(ndev, RES_QUEUE, ndev->be.max_queues)) 701 goto err_exit; 702 703 if (init_resource_elements(ndev, RES_CAT_CFN, ndev->be.cat.nb_cat_funcs)) 704 goto err_exit; 705 706 if (init_resource_elements(ndev, RES_CAT_COT, ndev->be.max_categories)) 707 goto err_exit; 708 709 if (init_resource_elements(ndev, RES_CAT_EXO, ndev->be.cat.nb_pm_ext)) 710 goto err_exit; 711 712 if (init_resource_elements(ndev, RES_CAT_LEN, ndev->be.cat.nb_len)) 713 goto err_exit; 714 715 if (init_resource_elements(ndev, RES_KM_FLOW_TYPE, ndev->be.cat.nb_flow_types)) 716 goto err_exit; 717 718 if (init_resource_elements(ndev, RES_KM_CATEGORY, ndev->be.km.nb_categories)) 719 goto err_exit; 720 721 if (init_resource_elements(ndev, RES_HSH_RCP, ndev->be.hsh.nb_rcp)) 722 goto err_exit; 723 724 if (init_resource_elements(ndev, RES_PDB_RCP, ndev->be.pdb.nb_pdb_rcp_categories)) 725 goto err_exit; 726 727 if (init_resource_elements(ndev, RES_QSL_RCP, ndev->be.qsl.nb_rcp_categories)) 728 goto err_exit; 729 730 if (init_resource_elements(ndev, RES_QSL_QST, ndev->be.qsl.nb_qst_entries)) 731 goto err_exit; 732 733 if (init_resource_elements(ndev, RES_SLC_LR_RCP, ndev->be.max_categories)) 734 goto err_exit; 735 736 if (init_resource_elements(ndev, RES_FLM_FLOW_TYPE, ndev->be.cat.nb_flow_types)) 737 goto err_exit; 738 739 if (init_resource_elements(ndev, RES_FLM_RCP, ndev->be.flm.nb_categories)) 740 goto err_exit; 741 742 if (init_resource_elements(ndev, RES_TPE_RCP, ndev->be.tpe.nb_rcp_categories)) 743 goto err_exit; 744 745 if (init_resource_elements(ndev, RES_TPE_EXT, ndev->be.tpe.nb_rpl_ext_categories)) 746 goto err_exit; 747 748 if (init_resource_elements(ndev, RES_TPE_RPL, ndev->be.tpe.nb_rpl_depth)) 749 goto err_exit; 750 751 if (init_resource_elements(ndev, RES_SCRUB_RCP, ndev->be.flm.nb_scrub_profiles)) 752 goto err_exit; 753 754 /* may need IPF, COR */ 755 756 /* check all defined has been initialized */ 757 for (int i = 0; i < RES_COUNT; i++) 758 assert(ndev->res[i].alloc_bm); 759 760 rte_spinlock_init(&ndev->mtx); 761 list_insert_flow_nic(ndev); 762 763 return ndev; 764 765 err_exit: 766 767 if (ndev) 768 flow_api_done(ndev); 769 770 NT_LOG(DBG, FILTER, "ERR: %s", __func__); 771 return NULL; 772 } 773 774 int flow_api_done(struct flow_nic_dev *ndev) 775 { 776 NT_LOG(DBG, FILTER, "FLOW API DONE"); 777 778 if (ndev) { 779 flow_ndev_reset(ndev); 780 781 /* delete resource management allocations for this ndev */ 782 for (int i = 0; i < RES_COUNT; i++) 783 done_resource_elements(ndev, i); 784 785 flow_api_backend_done(&ndev->be); 786 list_remove_flow_nic(ndev); 787 free(ndev); 788 } 789 790 return 0; 791 } 792 793 void *flow_api_get_be_dev(struct flow_nic_dev *ndev) 794 { 795 if (!ndev) { 796 NT_LOG(DBG, FILTER, "ERR: %s", __func__); 797 return NULL; 798 } 799 800 return ndev->be.be_dev; 801 } 802 803 /* Information for a given RSS type. */ 804 struct rss_type_info { 805 uint64_t rss_type; 806 const char *str; 807 }; 808 809 static struct rss_type_info rss_to_string[] = { 810 /* RTE_BIT64(2) IPv4 dst + IPv4 src */ 811 RSS_TO_STRING(RTE_ETH_RSS_IPV4), 812 /* RTE_BIT64(3) IPv4 dst + IPv4 src + Identification of group of fragments */ 813 RSS_TO_STRING(RTE_ETH_RSS_FRAG_IPV4), 814 /* RTE_BIT64(4) IPv4 dst + IPv4 src + L4 protocol */ 815 RSS_TO_STRING(RTE_ETH_RSS_NONFRAG_IPV4_TCP), 816 /* RTE_BIT64(5) IPv4 dst + IPv4 src + L4 protocol */ 817 RSS_TO_STRING(RTE_ETH_RSS_NONFRAG_IPV4_UDP), 818 /* RTE_BIT64(6) IPv4 dst + IPv4 src + L4 protocol */ 819 RSS_TO_STRING(RTE_ETH_RSS_NONFRAG_IPV4_SCTP), 820 /* RTE_BIT64(7) IPv4 dst + IPv4 src + L4 protocol */ 821 RSS_TO_STRING(RTE_ETH_RSS_NONFRAG_IPV4_OTHER), 822 /* 823 * RTE_BIT64(14) 128-bits of L2 payload starting after src MAC, i.e. including optional 824 * VLAN tag and ethertype. Overrides all L3 and L4 flags at the same level, but inner 825 * L2 payload can be combined with outer S-VLAN and GTPU TEID flags. 826 */ 827 RSS_TO_STRING(RTE_ETH_RSS_L2_PAYLOAD), 828 /* RTE_BIT64(18) L4 dst + L4 src + L4 protocol - see comment of RTE_ETH_RSS_L4_CHKSUM */ 829 RSS_TO_STRING(RTE_ETH_RSS_PORT), 830 /* RTE_BIT64(19) Not supported */ 831 RSS_TO_STRING(RTE_ETH_RSS_VXLAN), 832 /* RTE_BIT64(20) Not supported */ 833 RSS_TO_STRING(RTE_ETH_RSS_GENEVE), 834 /* RTE_BIT64(21) Not supported */ 835 RSS_TO_STRING(RTE_ETH_RSS_NVGRE), 836 /* RTE_BIT64(23) GTP TEID - always from outer GTPU header */ 837 RSS_TO_STRING(RTE_ETH_RSS_GTPU), 838 /* RTE_BIT64(24) MAC dst + MAC src */ 839 RSS_TO_STRING(RTE_ETH_RSS_ETH), 840 /* RTE_BIT64(25) outermost VLAN ID + L4 protocol */ 841 RSS_TO_STRING(RTE_ETH_RSS_S_VLAN), 842 /* RTE_BIT64(26) innermost VLAN ID + L4 protocol */ 843 RSS_TO_STRING(RTE_ETH_RSS_C_VLAN), 844 /* RTE_BIT64(27) Not supported */ 845 RSS_TO_STRING(RTE_ETH_RSS_ESP), 846 /* RTE_BIT64(28) Not supported */ 847 RSS_TO_STRING(RTE_ETH_RSS_AH), 848 /* RTE_BIT64(29) Not supported */ 849 RSS_TO_STRING(RTE_ETH_RSS_L2TPV3), 850 /* RTE_BIT64(30) Not supported */ 851 RSS_TO_STRING(RTE_ETH_RSS_PFCP), 852 /* RTE_BIT64(31) Not supported */ 853 RSS_TO_STRING(RTE_ETH_RSS_PPPOE), 854 /* RTE_BIT64(32) Not supported */ 855 RSS_TO_STRING(RTE_ETH_RSS_ECPRI), 856 /* RTE_BIT64(33) Not supported */ 857 RSS_TO_STRING(RTE_ETH_RSS_MPLS), 858 /* RTE_BIT64(34) IPv4 Header checksum + L4 protocol */ 859 RSS_TO_STRING(RTE_ETH_RSS_IPV4_CHKSUM), 860 861 /* 862 * if combined with RTE_ETH_RSS_NONFRAG_IPV4_[TCP|UDP|SCTP] then 863 * L4 protocol + chosen protocol header Checksum 864 * else 865 * error 866 */ 867 /* RTE_BIT64(35) */ 868 RSS_TO_STRING(RTE_ETH_RSS_L4_CHKSUM), 869 #ifndef ANDROMEDA_DPDK_21_11 870 /* RTE_BIT64(36) Not supported */ 871 RSS_TO_STRING(RTE_ETH_RSS_L2TPV2), 872 #endif 873 874 { RTE_BIT64(37), "unknown_RTE_BIT64(37)" }, 875 { RTE_BIT64(38), "unknown_RTE_BIT64(38)" }, 876 { RTE_BIT64(39), "unknown_RTE_BIT64(39)" }, 877 { RTE_BIT64(40), "unknown_RTE_BIT64(40)" }, 878 { RTE_BIT64(41), "unknown_RTE_BIT64(41)" }, 879 { RTE_BIT64(42), "unknown_RTE_BIT64(42)" }, 880 { RTE_BIT64(43), "unknown_RTE_BIT64(43)" }, 881 { RTE_BIT64(44), "unknown_RTE_BIT64(44)" }, 882 { RTE_BIT64(45), "unknown_RTE_BIT64(45)" }, 883 { RTE_BIT64(46), "unknown_RTE_BIT64(46)" }, 884 { RTE_BIT64(47), "unknown_RTE_BIT64(47)" }, 885 { RTE_BIT64(48), "unknown_RTE_BIT64(48)" }, 886 { RTE_BIT64(49), "unknown_RTE_BIT64(49)" }, 887 888 /* RTE_BIT64(50) outermost encapsulation */ 889 RSS_TO_STRING(RTE_ETH_RSS_LEVEL_OUTERMOST), 890 /* RTE_BIT64(51) innermost encapsulation */ 891 RSS_TO_STRING(RTE_ETH_RSS_LEVEL_INNERMOST), 892 893 /* RTE_BIT64(52) Not supported */ 894 RSS_TO_STRING(RTE_ETH_RSS_L3_PRE96), 895 /* RTE_BIT64(53) Not supported */ 896 RSS_TO_STRING(RTE_ETH_RSS_L3_PRE64), 897 /* RTE_BIT64(54) Not supported */ 898 RSS_TO_STRING(RTE_ETH_RSS_L3_PRE56), 899 /* RTE_BIT64(55) Not supported */ 900 RSS_TO_STRING(RTE_ETH_RSS_L3_PRE48), 901 /* RTE_BIT64(56) Not supported */ 902 RSS_TO_STRING(RTE_ETH_RSS_L3_PRE40), 903 /* RTE_BIT64(57) Not supported */ 904 RSS_TO_STRING(RTE_ETH_RSS_L3_PRE32), 905 906 /* RTE_BIT64(58) */ 907 RSS_TO_STRING(RTE_ETH_RSS_L2_DST_ONLY), 908 /* RTE_BIT64(59) */ 909 RSS_TO_STRING(RTE_ETH_RSS_L2_SRC_ONLY), 910 /* RTE_BIT64(60) */ 911 RSS_TO_STRING(RTE_ETH_RSS_L4_DST_ONLY), 912 /* RTE_BIT64(61) */ 913 RSS_TO_STRING(RTE_ETH_RSS_L4_SRC_ONLY), 914 /* RTE_BIT64(62) */ 915 RSS_TO_STRING(RTE_ETH_RSS_L3_DST_ONLY), 916 /* RTE_BIT64(63) */ 917 RSS_TO_STRING(RTE_ETH_RSS_L3_SRC_ONLY), 918 }; 919 920 int sprint_nt_rss_mask(char *str, uint16_t str_len, const char *prefix, uint64_t hash_mask) 921 { 922 if (str == NULL || str_len == 0) 923 return -1; 924 925 memset(str, 0x0, str_len); 926 uint16_t str_end = 0; 927 const struct rss_type_info *start = rss_to_string; 928 929 for (const struct rss_type_info *p = start; p != start + ARRAY_SIZE(rss_to_string); ++p) { 930 if (p->rss_type & hash_mask) { 931 if (strlen(prefix) + strlen(p->str) < (size_t)(str_len - str_end)) { 932 snprintf(str + str_end, str_len - str_end, "%s", prefix); 933 str_end += strlen(prefix); 934 snprintf(str + str_end, str_len - str_end, "%s", p->str); 935 str_end += strlen(p->str); 936 937 } else { 938 return -1; 939 } 940 } 941 } 942 943 return 0; 944 } 945 946 /* 947 * Hash 948 */ 949 950 int flow_nic_set_hasher(struct flow_nic_dev *ndev, int hsh_idx, enum flow_nic_hash_e algorithm) 951 { 952 hw_mod_hsh_rcp_set(&ndev->be, HW_HSH_RCP_PRESET_ALL, hsh_idx, 0, 0); 953 954 switch (algorithm) { 955 case HASH_ALGO_5TUPLE: 956 /* need to create an IPv6 hashing and enable the adaptive ip mask bit */ 957 hw_mod_hsh_rcp_set(&ndev->be, HW_HSH_RCP_LOAD_DIST_TYPE, hsh_idx, 0, 2); 958 hw_mod_hsh_rcp_set(&ndev->be, HW_HSH_RCP_QW0_PE, hsh_idx, 0, DYN_FINAL_IP_DST); 959 hw_mod_hsh_rcp_set(&ndev->be, HW_HSH_RCP_QW0_OFS, hsh_idx, 0, -16); 960 hw_mod_hsh_rcp_set(&ndev->be, HW_HSH_RCP_QW4_PE, hsh_idx, 0, DYN_FINAL_IP_DST); 961 hw_mod_hsh_rcp_set(&ndev->be, HW_HSH_RCP_QW4_OFS, hsh_idx, 0, 0); 962 hw_mod_hsh_rcp_set(&ndev->be, HW_HSH_RCP_W8_PE, hsh_idx, 0, DYN_L4); 963 hw_mod_hsh_rcp_set(&ndev->be, HW_HSH_RCP_W8_OFS, hsh_idx, 0, 0); 964 hw_mod_hsh_rcp_set(&ndev->be, HW_HSH_RCP_W9_PE, hsh_idx, 0, 0); 965 hw_mod_hsh_rcp_set(&ndev->be, HW_HSH_RCP_W9_OFS, hsh_idx, 0, 0); 966 hw_mod_hsh_rcp_set(&ndev->be, HW_HSH_RCP_W9_P, hsh_idx, 0, 0); 967 hw_mod_hsh_rcp_set(&ndev->be, HW_HSH_RCP_P_MASK, hsh_idx, 0, 1); 968 hw_mod_hsh_rcp_set(&ndev->be, HW_HSH_RCP_WORD_MASK, hsh_idx, 0, 0xffffffff); 969 hw_mod_hsh_rcp_set(&ndev->be, HW_HSH_RCP_WORD_MASK, hsh_idx, 1, 0xffffffff); 970 hw_mod_hsh_rcp_set(&ndev->be, HW_HSH_RCP_WORD_MASK, hsh_idx, 2, 0xffffffff); 971 hw_mod_hsh_rcp_set(&ndev->be, HW_HSH_RCP_WORD_MASK, hsh_idx, 3, 0xffffffff); 972 hw_mod_hsh_rcp_set(&ndev->be, HW_HSH_RCP_WORD_MASK, hsh_idx, 4, 0xffffffff); 973 hw_mod_hsh_rcp_set(&ndev->be, HW_HSH_RCP_WORD_MASK, hsh_idx, 5, 0xffffffff); 974 hw_mod_hsh_rcp_set(&ndev->be, HW_HSH_RCP_WORD_MASK, hsh_idx, 6, 0xffffffff); 975 hw_mod_hsh_rcp_set(&ndev->be, HW_HSH_RCP_WORD_MASK, hsh_idx, 7, 0xffffffff); 976 hw_mod_hsh_rcp_set(&ndev->be, HW_HSH_RCP_WORD_MASK, hsh_idx, 8, 0xffffffff); 977 hw_mod_hsh_rcp_set(&ndev->be, HW_HSH_RCP_WORD_MASK, hsh_idx, 9, 0); 978 hw_mod_hsh_rcp_set(&ndev->be, HW_HSH_RCP_SEED, hsh_idx, 0, 0xffffffff); 979 hw_mod_hsh_rcp_set(&ndev->be, HW_HSH_RCP_HSH_VALID, hsh_idx, 0, 1); 980 hw_mod_hsh_rcp_set(&ndev->be, HW_HSH_RCP_HSH_TYPE, hsh_idx, 0, HASH_5TUPLE); 981 hw_mod_hsh_rcp_set(&ndev->be, HW_HSH_RCP_AUTO_IPV4_MASK, hsh_idx, 0, 1); 982 983 NT_LOG(DBG, FILTER, "Set IPv6 5-tuple hasher with adaptive IPv4 hashing"); 984 break; 985 986 default: 987 case HASH_ALGO_ROUND_ROBIN: 988 /* zero is round-robin */ 989 break; 990 } 991 992 return 0; 993 } 994 995 static int flow_dev_dump(struct flow_eth_dev *dev, 996 struct flow_handle *flow, 997 uint16_t caller_id, 998 FILE *file, 999 struct rte_flow_error *error) 1000 { 1001 const struct profile_inline_ops *profile_inline_ops = get_profile_inline_ops(); 1002 1003 if (profile_inline_ops == NULL) { 1004 NT_LOG(ERR, FILTER, "%s: profile_inline module uninitialized", __func__); 1005 return -1; 1006 } 1007 1008 return profile_inline_ops->flow_dev_dump_profile_inline(dev, flow, caller_id, file, error); 1009 } 1010 1011 int flow_nic_set_hasher_fields(struct flow_nic_dev *ndev, int hsh_idx, 1012 struct nt_eth_rss_conf rss_conf) 1013 { 1014 const struct profile_inline_ops *profile_inline_ops = get_profile_inline_ops(); 1015 1016 if (profile_inline_ops == NULL) { 1017 NT_LOG(ERR, FILTER, "%s: profile_inline module uninitialized", __func__); 1018 return -1; 1019 } 1020 1021 return profile_inline_ops->flow_nic_set_hasher_fields_inline(ndev, hsh_idx, rss_conf); 1022 } 1023 1024 static int flow_get_aged_flows(struct flow_eth_dev *dev, 1025 uint16_t caller_id, 1026 void **context, 1027 uint32_t nb_contexts, 1028 struct rte_flow_error *error) 1029 { 1030 const struct profile_inline_ops *profile_inline_ops = get_profile_inline_ops(); 1031 1032 if (profile_inline_ops == NULL) { 1033 NT_LOG_DBGX(ERR, FILTER, "profile_inline_ops uninitialized"); 1034 return -1; 1035 } 1036 1037 if (nb_contexts > 0 && !context) { 1038 error->type = RTE_FLOW_ERROR_TYPE_UNSPECIFIED; 1039 error->message = "rte_flow_get_aged_flows - empty context"; 1040 return -1; 1041 } 1042 1043 return profile_inline_ops->flow_get_aged_flows_profile_inline(dev, caller_id, context, 1044 nb_contexts, error); 1045 } 1046 1047 static int flow_info_get(struct flow_eth_dev *dev, uint8_t caller_id, 1048 struct rte_flow_port_info *port_info, struct rte_flow_queue_info *queue_info, 1049 struct rte_flow_error *error) 1050 { 1051 const struct profile_inline_ops *profile_inline_ops = get_profile_inline_ops(); 1052 1053 if (profile_inline_ops == NULL) { 1054 NT_LOG_DBGX(ERR, FILTER, "profile_inline module uninitialized"); 1055 return -1; 1056 } 1057 1058 return profile_inline_ops->flow_info_get_profile_inline(dev, caller_id, port_info, 1059 queue_info, error); 1060 } 1061 1062 static int flow_configure(struct flow_eth_dev *dev, uint8_t caller_id, 1063 const struct rte_flow_port_attr *port_attr, uint16_t nb_queue, 1064 const struct rte_flow_queue_attr *queue_attr[], struct rte_flow_error *error) 1065 { 1066 const struct profile_inline_ops *profile_inline_ops = get_profile_inline_ops(); 1067 1068 if (profile_inline_ops == NULL) { 1069 NT_LOG_DBGX(ERR, FILTER, "profile_inline module uninitialized"); 1070 return -1; 1071 } 1072 1073 return profile_inline_ops->flow_configure_profile_inline(dev, caller_id, port_attr, 1074 nb_queue, queue_attr, error); 1075 } 1076 1077 /* 1078 * Flow Asynchronous operation API 1079 */ 1080 1081 static struct flow_pattern_template * 1082 flow_pattern_template_create(struct flow_eth_dev *dev, 1083 const struct rte_flow_pattern_template_attr *template_attr, uint16_t caller_id, 1084 const struct rte_flow_item pattern[], struct rte_flow_error *error) 1085 { 1086 const struct profile_inline_ops *profile_inline_ops = get_profile_inline_ops(); 1087 1088 if (profile_inline_ops == NULL) { 1089 NT_LOG_DBGX(ERR, FILTER, "profile_inline module uninitialized"); 1090 return NULL; 1091 } 1092 1093 return profile_inline_ops->flow_pattern_template_create_profile_inline(dev, template_attr, 1094 caller_id, pattern, error); 1095 } 1096 1097 static int flow_pattern_template_destroy(struct flow_eth_dev *dev, 1098 struct flow_pattern_template *pattern_template, 1099 struct rte_flow_error *error) 1100 { 1101 const struct profile_inline_ops *profile_inline_ops = get_profile_inline_ops(); 1102 1103 if (profile_inline_ops == NULL) { 1104 NT_LOG_DBGX(ERR, FILTER, "profile_inline module uninitialized"); 1105 return -1; 1106 } 1107 1108 return profile_inline_ops->flow_pattern_template_destroy_profile_inline(dev, 1109 pattern_template, 1110 error); 1111 } 1112 1113 static struct flow_actions_template * 1114 flow_actions_template_create(struct flow_eth_dev *dev, 1115 const struct rte_flow_actions_template_attr *template_attr, uint16_t caller_id, 1116 const struct rte_flow_action actions[], const struct rte_flow_action masks[], 1117 struct rte_flow_error *error) 1118 { 1119 const struct profile_inline_ops *profile_inline_ops = get_profile_inline_ops(); 1120 1121 if (profile_inline_ops == NULL) { 1122 NT_LOG_DBGX(ERR, FILTER, "profile_inline module uninitialized"); 1123 return NULL; 1124 } 1125 1126 return profile_inline_ops->flow_actions_template_create_profile_inline(dev, template_attr, 1127 caller_id, actions, masks, error); 1128 } 1129 1130 static int flow_actions_template_destroy(struct flow_eth_dev *dev, 1131 struct flow_actions_template *actions_template, 1132 struct rte_flow_error *error) 1133 { 1134 const struct profile_inline_ops *profile_inline_ops = get_profile_inline_ops(); 1135 1136 if (profile_inline_ops == NULL) { 1137 NT_LOG_DBGX(ERR, FILTER, "profile_inline module uninitialized"); 1138 return -1; 1139 } 1140 1141 return profile_inline_ops->flow_actions_template_destroy_profile_inline(dev, 1142 actions_template, 1143 error); 1144 } 1145 1146 static struct flow_template_table *flow_template_table_create(struct flow_eth_dev *dev, 1147 const struct rte_flow_template_table_attr *table_attr, uint16_t forced_vlan_vid, 1148 uint16_t caller_id, struct flow_pattern_template *pattern_templates[], 1149 uint8_t nb_pattern_templates, struct flow_actions_template *actions_templates[], 1150 uint8_t nb_actions_templates, struct rte_flow_error *error) 1151 { 1152 const struct profile_inline_ops *profile_inline_ops = get_profile_inline_ops(); 1153 1154 if (profile_inline_ops == NULL) { 1155 NT_LOG_DBGX(ERR, FILTER, "profile_inline module uninitialized"); 1156 return NULL; 1157 } 1158 1159 return profile_inline_ops->flow_template_table_create_profile_inline(dev, table_attr, 1160 forced_vlan_vid, caller_id, pattern_templates, nb_pattern_templates, 1161 actions_templates, nb_actions_templates, error); 1162 } 1163 1164 static int flow_template_table_destroy(struct flow_eth_dev *dev, 1165 struct flow_template_table *template_table, 1166 struct rte_flow_error *error) 1167 { 1168 const struct profile_inline_ops *profile_inline_ops = get_profile_inline_ops(); 1169 1170 if (profile_inline_ops == NULL) { 1171 NT_LOG_DBGX(ERR, FILTER, "profile_inline module uninitialized"); 1172 return -1; 1173 } 1174 1175 return profile_inline_ops->flow_template_table_destroy_profile_inline(dev, template_table, 1176 error); 1177 } 1178 1179 static struct flow_handle * 1180 flow_async_create(struct flow_eth_dev *dev, uint32_t queue_id, 1181 const struct rte_flow_op_attr *op_attr, struct flow_template_table *template_table, 1182 const struct rte_flow_item pattern[], uint8_t pattern_template_index, 1183 const struct rte_flow_action actions[], uint8_t actions_template_index, void *user_data, 1184 struct rte_flow_error *error) 1185 { 1186 const struct profile_inline_ops *profile_inline_ops = get_profile_inline_ops(); 1187 1188 if (profile_inline_ops == NULL) { 1189 NT_LOG_DBGX(ERR, FILTER, "profile_inline module uninitialized"); 1190 return NULL; 1191 } 1192 1193 return profile_inline_ops->flow_async_create_profile_inline(dev, queue_id, op_attr, 1194 template_table, pattern, pattern_template_index, actions, 1195 actions_template_index, user_data, error); 1196 } 1197 1198 static int flow_async_destroy(struct flow_eth_dev *dev, uint32_t queue_id, 1199 const struct rte_flow_op_attr *op_attr, struct flow_handle *flow, 1200 void *user_data, struct rte_flow_error *error) 1201 { 1202 const struct profile_inline_ops *profile_inline_ops = get_profile_inline_ops(); 1203 1204 if (profile_inline_ops == NULL) { 1205 NT_LOG_DBGX(ERR, FILTER, "profile_inline module uninitialized"); 1206 return -1; 1207 } 1208 1209 return profile_inline_ops->flow_async_destroy_profile_inline(dev, queue_id, op_attr, flow, 1210 user_data, error); 1211 } 1212 int flow_get_flm_stats(struct flow_nic_dev *ndev, uint64_t *data, uint64_t size) 1213 { 1214 const struct profile_inline_ops *profile_inline_ops = get_profile_inline_ops(); 1215 1216 if (profile_inline_ops == NULL) 1217 return -1; 1218 1219 if (ndev->flow_profile == FLOW_ETH_DEV_PROFILE_INLINE) 1220 return profile_inline_ops->flow_get_flm_stats_profile_inline(ndev, data, size); 1221 1222 return -1; 1223 } 1224 1225 static const struct flow_filter_ops ops = { 1226 .flow_filter_init = flow_filter_init, 1227 .flow_filter_done = flow_filter_done, 1228 /* 1229 * Device Management API 1230 */ 1231 .flow_get_eth_dev = flow_get_eth_dev, 1232 /* 1233 * NT Flow API 1234 */ 1235 .flow_create = flow_create, 1236 .flow_destroy = flow_destroy, 1237 .flow_flush = flow_flush, 1238 .flow_actions_update = flow_actions_update, 1239 .flow_dev_dump = flow_dev_dump, 1240 .flow_get_flm_stats = flow_get_flm_stats, 1241 .flow_get_aged_flows = flow_get_aged_flows, 1242 1243 /* 1244 * NT Flow asynchronous operations API 1245 */ 1246 .flow_info_get = flow_info_get, 1247 .flow_configure = flow_configure, 1248 .flow_pattern_template_create = flow_pattern_template_create, 1249 .flow_pattern_template_destroy = flow_pattern_template_destroy, 1250 .flow_actions_template_create = flow_actions_template_create, 1251 .flow_actions_template_destroy = flow_actions_template_destroy, 1252 .flow_template_table_create = flow_template_table_create, 1253 .flow_template_table_destroy = flow_template_table_destroy, 1254 .flow_async_create = flow_async_create, 1255 .flow_async_destroy = flow_async_destroy, 1256 1257 /* 1258 * Other 1259 */ 1260 .hw_mod_hsh_rcp_flush = hw_mod_hsh_rcp_flush, 1261 .flow_nic_set_hasher_fields = flow_nic_set_hasher_fields, 1262 }; 1263 1264 void init_flow_filter(void) 1265 { 1266 register_flow_filter_ops(&ops); 1267 } 1268