1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2018 6WIND S.A. 3 * Copyright 2018 Mellanox Technologies, Ltd 4 */ 5 6 #include <errno.h> 7 #include <stdalign.h> 8 #include <stddef.h> 9 #include <stdint.h> 10 #include <stdlib.h> 11 #include <rte_memory.h> 12 /* 13 * Not needed by this file; included to work around the lack of off_t 14 * definition for mlx5dv.h with unpatched rdma-core versions. 15 */ 16 #include <sys/types.h> 17 18 #include "mlx5_glue.h" 19 #include "../mlx5_malloc.h" 20 21 static int 22 mlx5_glue_fork_init(void) 23 { 24 return ibv_fork_init(); 25 } 26 27 static struct ibv_pd * 28 mlx5_glue_alloc_pd(struct ibv_context *context) 29 { 30 return ibv_alloc_pd(context); 31 } 32 33 static int 34 mlx5_glue_dealloc_pd(struct ibv_pd *pd) 35 { 36 return ibv_dealloc_pd(pd); 37 } 38 39 static struct ibv_device ** 40 mlx5_glue_get_device_list(int *num_devices) 41 { 42 return ibv_get_device_list(num_devices); 43 } 44 45 static void 46 mlx5_glue_free_device_list(struct ibv_device **list) 47 { 48 ibv_free_device_list(list); 49 } 50 51 static struct ibv_context * 52 mlx5_glue_open_device(struct ibv_device *device) 53 { 54 return ibv_open_device(device); 55 } 56 57 static int 58 mlx5_glue_close_device(struct ibv_context *context) 59 { 60 return ibv_close_device(context); 61 } 62 63 static int 64 mlx5_glue_query_device(struct ibv_context *context, 65 struct ibv_device_attr *device_attr) 66 { 67 return ibv_query_device(context, device_attr); 68 } 69 70 static int 71 mlx5_glue_query_device_ex(struct ibv_context *context, 72 const struct ibv_query_device_ex_input *input, 73 struct ibv_device_attr_ex *attr) 74 { 75 return ibv_query_device_ex(context, input, attr); 76 } 77 78 static int 79 mlx5_glue_query_rt_values_ex(struct ibv_context *context, 80 struct ibv_values_ex *values) 81 { 82 return ibv_query_rt_values_ex(context, values); 83 } 84 85 static int 86 mlx5_glue_query_port(struct ibv_context *context, uint8_t port_num, 87 struct ibv_port_attr *port_attr) 88 { 89 return ibv_query_port(context, port_num, port_attr); 90 } 91 92 static struct ibv_comp_channel * 93 mlx5_glue_create_comp_channel(struct ibv_context *context) 94 { 95 return ibv_create_comp_channel(context); 96 } 97 98 static int 99 mlx5_glue_destroy_comp_channel(struct ibv_comp_channel *channel) 100 { 101 return ibv_destroy_comp_channel(channel); 102 } 103 104 static struct ibv_cq * 105 mlx5_glue_create_cq(struct ibv_context *context, int cqe, void *cq_context, 106 struct ibv_comp_channel *channel, int comp_vector) 107 { 108 return ibv_create_cq(context, cqe, cq_context, channel, comp_vector); 109 } 110 111 static int 112 mlx5_glue_destroy_cq(struct ibv_cq *cq) 113 { 114 return ibv_destroy_cq(cq); 115 } 116 117 static int 118 mlx5_glue_get_cq_event(struct ibv_comp_channel *channel, struct ibv_cq **cq, 119 void **cq_context) 120 { 121 return ibv_get_cq_event(channel, cq, cq_context); 122 } 123 124 static void 125 mlx5_glue_ack_cq_events(struct ibv_cq *cq, unsigned int nevents) 126 { 127 ibv_ack_cq_events(cq, nevents); 128 } 129 130 static struct ibv_rwq_ind_table * 131 mlx5_glue_create_rwq_ind_table(struct ibv_context *context, 132 struct ibv_rwq_ind_table_init_attr *init_attr) 133 { 134 return ibv_create_rwq_ind_table(context, init_attr); 135 } 136 137 static int 138 mlx5_glue_destroy_rwq_ind_table(struct ibv_rwq_ind_table *rwq_ind_table) 139 { 140 return ibv_destroy_rwq_ind_table(rwq_ind_table); 141 } 142 143 static struct ibv_wq * 144 mlx5_glue_create_wq(struct ibv_context *context, 145 struct ibv_wq_init_attr *wq_init_attr) 146 { 147 return ibv_create_wq(context, wq_init_attr); 148 } 149 150 static int 151 mlx5_glue_destroy_wq(struct ibv_wq *wq) 152 { 153 return ibv_destroy_wq(wq); 154 } 155 static int 156 mlx5_glue_modify_wq(struct ibv_wq *wq, struct ibv_wq_attr *wq_attr) 157 { 158 return ibv_modify_wq(wq, wq_attr); 159 } 160 161 static struct ibv_flow * 162 mlx5_glue_create_flow(struct ibv_qp *qp, struct ibv_flow_attr *flow) 163 { 164 return ibv_create_flow(qp, flow); 165 } 166 167 static int 168 mlx5_glue_destroy_flow(struct ibv_flow *flow_id) 169 { 170 return ibv_destroy_flow(flow_id); 171 } 172 173 static int 174 mlx5_glue_destroy_flow_action(void *action) 175 { 176 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 177 #ifdef HAVE_MLX5DV_DR 178 return mlx5dv_dr_action_destroy(action); 179 #else 180 struct mlx5dv_flow_action_attr *attr = action; 181 int res = 0; 182 switch (attr->type) { 183 case MLX5DV_FLOW_ACTION_TAG: 184 break; 185 default: 186 res = ibv_destroy_flow_action(attr->action); 187 break; 188 } 189 mlx5_free(action); 190 return res; 191 #endif 192 #else 193 (void)action; 194 return -ENOTSUP; 195 #endif 196 } 197 198 static struct ibv_qp * 199 mlx5_glue_create_qp(struct ibv_pd *pd, struct ibv_qp_init_attr *qp_init_attr) 200 { 201 return ibv_create_qp(pd, qp_init_attr); 202 } 203 204 static struct ibv_qp * 205 mlx5_glue_create_qp_ex(struct ibv_context *context, 206 struct ibv_qp_init_attr_ex *qp_init_attr_ex) 207 { 208 return ibv_create_qp_ex(context, qp_init_attr_ex); 209 } 210 211 static int 212 mlx5_glue_destroy_qp(struct ibv_qp *qp) 213 { 214 return ibv_destroy_qp(qp); 215 } 216 217 static int 218 mlx5_glue_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, int attr_mask) 219 { 220 return ibv_modify_qp(qp, attr, attr_mask); 221 } 222 223 static struct ibv_mr * 224 mlx5_glue_reg_mr(struct ibv_pd *pd, void *addr, size_t length, int access) 225 { 226 return ibv_reg_mr(pd, addr, length, access); 227 } 228 229 static struct ibv_mr * 230 mlx5_glue_alloc_null_mr(struct ibv_pd *pd) 231 { 232 #ifdef HAVE_IBV_DEVX_OBJ 233 return ibv_alloc_null_mr(pd); 234 #else 235 (void)pd; 236 errno = ENOTSUP; 237 return NULL; 238 #endif 239 } 240 241 static int 242 mlx5_glue_dereg_mr(struct ibv_mr *mr) 243 { 244 return ibv_dereg_mr(mr); 245 } 246 247 static struct ibv_counter_set * 248 mlx5_glue_create_counter_set(struct ibv_context *context, 249 struct ibv_counter_set_init_attr *init_attr) 250 { 251 #ifndef HAVE_IBV_DEVICE_COUNTERS_SET_V42 252 (void)context; 253 (void)init_attr; 254 return NULL; 255 #else 256 return ibv_create_counter_set(context, init_attr); 257 #endif 258 } 259 260 static int 261 mlx5_glue_destroy_counter_set(struct ibv_counter_set *cs) 262 { 263 #ifndef HAVE_IBV_DEVICE_COUNTERS_SET_V42 264 (void)cs; 265 return -ENOTSUP; 266 #else 267 return ibv_destroy_counter_set(cs); 268 #endif 269 } 270 271 static int 272 mlx5_glue_describe_counter_set(struct ibv_context *context, 273 uint16_t counter_set_id, 274 struct ibv_counter_set_description *cs_desc) 275 { 276 #ifndef HAVE_IBV_DEVICE_COUNTERS_SET_V42 277 (void)context; 278 (void)counter_set_id; 279 (void)cs_desc; 280 return -ENOTSUP; 281 #else 282 return ibv_describe_counter_set(context, counter_set_id, cs_desc); 283 #endif 284 } 285 286 static int 287 mlx5_glue_query_counter_set(struct ibv_query_counter_set_attr *query_attr, 288 struct ibv_counter_set_data *cs_data) 289 { 290 #ifndef HAVE_IBV_DEVICE_COUNTERS_SET_V42 291 (void)query_attr; 292 (void)cs_data; 293 return -ENOTSUP; 294 #else 295 return ibv_query_counter_set(query_attr, cs_data); 296 #endif 297 } 298 299 static struct ibv_counters * 300 mlx5_glue_create_counters(struct ibv_context *context, 301 struct ibv_counters_init_attr *init_attr) 302 { 303 #ifndef HAVE_IBV_DEVICE_COUNTERS_SET_V45 304 (void)context; 305 (void)init_attr; 306 errno = ENOTSUP; 307 return NULL; 308 #else 309 return ibv_create_counters(context, init_attr); 310 #endif 311 } 312 313 static int 314 mlx5_glue_destroy_counters(struct ibv_counters *counters) 315 { 316 #ifndef HAVE_IBV_DEVICE_COUNTERS_SET_V45 317 (void)counters; 318 return -ENOTSUP; 319 #else 320 return ibv_destroy_counters(counters); 321 #endif 322 } 323 324 static int 325 mlx5_glue_attach_counters(struct ibv_counters *counters, 326 struct ibv_counter_attach_attr *attr, 327 struct ibv_flow *flow) 328 { 329 #ifndef HAVE_IBV_DEVICE_COUNTERS_SET_V45 330 (void)counters; 331 (void)attr; 332 (void)flow; 333 return -ENOTSUP; 334 #else 335 return ibv_attach_counters_point_flow(counters, attr, flow); 336 #endif 337 } 338 339 static int 340 mlx5_glue_query_counters(struct ibv_counters *counters, 341 uint64_t *counters_value, 342 uint32_t ncounters, 343 uint32_t flags) 344 { 345 #ifndef HAVE_IBV_DEVICE_COUNTERS_SET_V45 346 (void)counters; 347 (void)counters_value; 348 (void)ncounters; 349 (void)flags; 350 return -ENOTSUP; 351 #else 352 return ibv_read_counters(counters, counters_value, ncounters, flags); 353 #endif 354 } 355 356 static void 357 mlx5_glue_ack_async_event(struct ibv_async_event *event) 358 { 359 ibv_ack_async_event(event); 360 } 361 362 static int 363 mlx5_glue_get_async_event(struct ibv_context *context, 364 struct ibv_async_event *event) 365 { 366 return ibv_get_async_event(context, event); 367 } 368 369 static const char * 370 mlx5_glue_port_state_str(enum ibv_port_state port_state) 371 { 372 return ibv_port_state_str(port_state); 373 } 374 375 static struct ibv_cq * 376 mlx5_glue_cq_ex_to_cq(struct ibv_cq_ex *cq) 377 { 378 return ibv_cq_ex_to_cq(cq); 379 } 380 381 static void * 382 mlx5_glue_dr_create_flow_action_dest_flow_tbl(void *tbl) 383 { 384 #ifdef HAVE_MLX5DV_DR 385 return mlx5dv_dr_action_create_dest_table(tbl); 386 #else 387 (void)tbl; 388 errno = ENOTSUP; 389 return NULL; 390 #endif 391 } 392 393 static void * 394 mlx5_glue_dr_create_flow_action_dest_port(void *domain, uint32_t port) 395 { 396 #ifdef HAVE_MLX5DV_DR_DEVX_PORT 397 return mlx5dv_dr_action_create_dest_ib_port(domain, port); 398 #else 399 #ifdef HAVE_MLX5DV_DR_ESWITCH 400 return mlx5dv_dr_action_create_dest_vport(domain, port); 401 #else 402 (void)domain; 403 (void)port; 404 errno = ENOTSUP; 405 return NULL; 406 #endif 407 #endif 408 } 409 410 static void * 411 mlx5_glue_dr_create_flow_action_drop(void) 412 { 413 #ifdef HAVE_MLX5DV_DR_ESWITCH 414 return mlx5dv_dr_action_create_drop(); 415 #else 416 errno = ENOTSUP; 417 return NULL; 418 #endif 419 } 420 421 static void * 422 mlx5_glue_dr_create_flow_action_push_vlan(struct mlx5dv_dr_domain *domain, 423 rte_be32_t vlan_tag) 424 { 425 #ifdef HAVE_MLX5DV_DR_VLAN 426 return mlx5dv_dr_action_create_push_vlan(domain, vlan_tag); 427 #else 428 (void)domain; 429 (void)vlan_tag; 430 errno = ENOTSUP; 431 return NULL; 432 #endif 433 } 434 435 static void * 436 mlx5_glue_dr_create_flow_action_pop_vlan(void) 437 { 438 #ifdef HAVE_MLX5DV_DR_VLAN 439 return mlx5dv_dr_action_create_pop_vlan(); 440 #else 441 errno = ENOTSUP; 442 return NULL; 443 #endif 444 } 445 446 static void * 447 mlx5_glue_dr_create_flow_tbl(void *domain, uint32_t level) 448 { 449 #ifdef HAVE_MLX5DV_DR 450 return mlx5dv_dr_table_create(domain, level); 451 #else 452 (void)domain; 453 (void)level; 454 errno = ENOTSUP; 455 return NULL; 456 #endif 457 } 458 459 static int 460 mlx5_glue_dr_destroy_flow_tbl(void *tbl) 461 { 462 #ifdef HAVE_MLX5DV_DR 463 return mlx5dv_dr_table_destroy(tbl); 464 #else 465 (void)tbl; 466 errno = ENOTSUP; 467 return errno; 468 #endif 469 } 470 471 static void * 472 mlx5_glue_dr_create_domain(struct ibv_context *ctx, 473 enum mlx5dv_dr_domain_type domain) 474 { 475 #ifdef HAVE_MLX5DV_DR 476 return mlx5dv_dr_domain_create(ctx, domain); 477 #else 478 (void)ctx; 479 (void)domain; 480 errno = ENOTSUP; 481 return NULL; 482 #endif 483 } 484 485 static int 486 mlx5_glue_dr_destroy_domain(void *domain) 487 { 488 #ifdef HAVE_MLX5DV_DR 489 return mlx5dv_dr_domain_destroy(domain); 490 #else 491 (void)domain; 492 errno = ENOTSUP; 493 return errno; 494 #endif 495 } 496 497 static struct ibv_cq_ex * 498 mlx5_glue_dv_create_cq(struct ibv_context *context, 499 struct ibv_cq_init_attr_ex *cq_attr, 500 struct mlx5dv_cq_init_attr *mlx5_cq_attr) 501 { 502 return mlx5dv_create_cq(context, cq_attr, mlx5_cq_attr); 503 } 504 505 static struct ibv_wq * 506 mlx5_glue_dv_create_wq(struct ibv_context *context, 507 struct ibv_wq_init_attr *wq_attr, 508 struct mlx5dv_wq_init_attr *mlx5_wq_attr) 509 { 510 #ifndef HAVE_IBV_DEVICE_STRIDING_RQ_SUPPORT 511 (void)context; 512 (void)wq_attr; 513 (void)mlx5_wq_attr; 514 errno = ENOTSUP; 515 return NULL; 516 #else 517 return mlx5dv_create_wq(context, wq_attr, mlx5_wq_attr); 518 #endif 519 } 520 521 static int 522 mlx5_glue_dv_query_device(struct ibv_context *ctx, 523 struct mlx5dv_context *attrs_out) 524 { 525 return mlx5dv_query_device(ctx, attrs_out); 526 } 527 528 static int 529 mlx5_glue_dv_set_context_attr(struct ibv_context *ibv_ctx, 530 enum mlx5dv_set_ctx_attr_type type, void *attr) 531 { 532 return mlx5dv_set_context_attr(ibv_ctx, type, attr); 533 } 534 535 static int 536 mlx5_glue_dv_init_obj(struct mlx5dv_obj *obj, uint64_t obj_type) 537 { 538 return mlx5dv_init_obj(obj, obj_type); 539 } 540 541 static struct ibv_qp * 542 mlx5_glue_dv_create_qp(struct ibv_context *context, 543 struct ibv_qp_init_attr_ex *qp_init_attr_ex, 544 struct mlx5dv_qp_init_attr *dv_qp_init_attr) 545 { 546 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 547 return mlx5dv_create_qp(context, qp_init_attr_ex, dv_qp_init_attr); 548 #else 549 (void)context; 550 (void)qp_init_attr_ex; 551 (void)dv_qp_init_attr; 552 errno = ENOTSUP; 553 return NULL; 554 #endif 555 } 556 557 static void * 558 mlx5_glue_dv_create_flow_matcher(struct ibv_context *context, 559 struct mlx5dv_flow_matcher_attr *matcher_attr, 560 void *tbl) 561 { 562 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 563 #ifdef HAVE_MLX5DV_DR 564 (void)context; 565 return mlx5dv_dr_matcher_create(tbl, matcher_attr->priority, 566 matcher_attr->match_criteria_enable, 567 matcher_attr->match_mask); 568 #else 569 (void)tbl; 570 return mlx5dv_create_flow_matcher(context, matcher_attr); 571 #endif 572 #else 573 (void)context; 574 (void)matcher_attr; 575 (void)tbl; 576 errno = ENOTSUP; 577 return NULL; 578 #endif 579 } 580 581 static void * 582 mlx5_glue_dv_create_flow(void *matcher, 583 void *match_value, 584 size_t num_actions, 585 void *actions[]) 586 { 587 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 588 #ifdef HAVE_MLX5DV_DR 589 return mlx5dv_dr_rule_create(matcher, match_value, num_actions, 590 (struct mlx5dv_dr_action **)actions); 591 #else 592 size_t i; 593 struct mlx5dv_flow_action_attr actions_attr[8]; 594 595 if (num_actions > 8) 596 return NULL; 597 for (i = 0; i < num_actions; i++) 598 actions_attr[i] = 599 *((struct mlx5dv_flow_action_attr *)(actions[i])); 600 return mlx5dv_create_flow(matcher, match_value, 601 num_actions, actions_attr); 602 #endif 603 #else 604 (void)matcher; 605 (void)match_value; 606 (void)num_actions; 607 (void)actions; 608 return NULL; 609 #endif 610 } 611 612 static void * 613 mlx5_glue_dv_create_flow_action_counter(void *counter_obj, uint32_t offset) 614 { 615 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 616 #ifdef HAVE_MLX5DV_DR 617 return mlx5dv_dr_action_create_flow_counter(counter_obj, offset); 618 #else 619 struct mlx5dv_flow_action_attr *action; 620 621 (void)offset; 622 action = mlx5_malloc(0, sizeof(*action), 0, SOCKET_ID_ANY); 623 if (!action) 624 return NULL; 625 action->type = MLX5DV_FLOW_ACTION_COUNTERS_DEVX; 626 action->obj = counter_obj; 627 return action; 628 #endif 629 #else 630 (void)counter_obj; 631 (void)offset; 632 errno = ENOTSUP; 633 return NULL; 634 #endif 635 } 636 637 static void * 638 mlx5_glue_dv_create_flow_action_dest_ibv_qp(void *qp) 639 { 640 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 641 #ifdef HAVE_MLX5DV_DR 642 return mlx5dv_dr_action_create_dest_ibv_qp(qp); 643 #else 644 struct mlx5dv_flow_action_attr *action; 645 646 action = mlx5_malloc(0, sizeof(*action), 0, SOCKET_ID_ANY); 647 if (!action) 648 return NULL; 649 action->type = MLX5DV_FLOW_ACTION_DEST_IBV_QP; 650 action->obj = qp; 651 return action; 652 #endif 653 #else 654 (void)qp; 655 errno = ENOTSUP; 656 return NULL; 657 #endif 658 } 659 660 static void * 661 mlx5_glue_dv_create_flow_action_dest_devx_tir(void *tir) 662 { 663 #ifdef HAVE_MLX5DV_DR_ACTION_DEST_DEVX_TIR 664 return mlx5dv_dr_action_create_dest_devx_tir(tir); 665 #else 666 (void)tir; 667 errno = ENOTSUP; 668 return NULL; 669 #endif 670 } 671 672 static void * 673 mlx5_glue_dv_create_flow_action_modify_header 674 (struct ibv_context *ctx, 675 enum mlx5dv_flow_table_type ft_type, 676 void *domain, uint64_t flags, 677 size_t actions_sz, 678 uint64_t actions[]) 679 { 680 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 681 #ifdef HAVE_MLX5DV_DR 682 (void)ctx; 683 (void)ft_type; 684 return mlx5dv_dr_action_create_modify_header(domain, flags, actions_sz, 685 (__be64 *)actions); 686 #else 687 struct mlx5dv_flow_action_attr *action; 688 689 (void)domain; 690 (void)flags; 691 action = mlx5_malloc(0, sizeof(*action), 0, SOCKET_ID_ANY); 692 if (!action) 693 return NULL; 694 action->type = MLX5DV_FLOW_ACTION_IBV_FLOW_ACTION; 695 action->action = mlx5dv_create_flow_action_modify_header 696 (ctx, actions_sz, actions, ft_type); 697 return action; 698 #endif 699 #else 700 (void)ctx; 701 (void)ft_type; 702 (void)domain; 703 (void)flags; 704 (void)actions_sz; 705 (void)actions; 706 errno = ENOTSUP; 707 return NULL; 708 #endif 709 } 710 711 static void * 712 mlx5_glue_dv_create_flow_action_packet_reformat 713 (struct ibv_context *ctx, 714 enum mlx5dv_flow_action_packet_reformat_type reformat_type, 715 enum mlx5dv_flow_table_type ft_type, 716 struct mlx5dv_dr_domain *domain, 717 uint32_t flags, size_t data_sz, void *data) 718 { 719 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 720 #ifdef HAVE_MLX5DV_DR 721 (void)ctx; 722 (void)ft_type; 723 return mlx5dv_dr_action_create_packet_reformat(domain, flags, 724 reformat_type, data_sz, 725 data); 726 #else 727 (void)domain; 728 (void)flags; 729 struct mlx5dv_flow_action_attr *action; 730 731 action = mlx5_malloc(0, sizeof(*action), 0, SOCKET_ID_ANY); 732 if (!action) 733 return NULL; 734 action->type = MLX5DV_FLOW_ACTION_IBV_FLOW_ACTION; 735 action->action = mlx5dv_create_flow_action_packet_reformat 736 (ctx, data_sz, data, reformat_type, ft_type); 737 return action; 738 #endif 739 #else 740 (void)ctx; 741 (void)reformat_type; 742 (void)ft_type; 743 (void)domain; 744 (void)flags; 745 (void)data_sz; 746 (void)data; 747 errno = ENOTSUP; 748 return NULL; 749 #endif 750 } 751 752 static void * 753 mlx5_glue_dv_create_flow_action_tag(uint32_t tag) 754 { 755 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 756 #ifdef HAVE_MLX5DV_DR 757 return mlx5dv_dr_action_create_tag(tag); 758 #else /* HAVE_MLX5DV_DR */ 759 struct mlx5dv_flow_action_attr *action; 760 761 action = mlx5_malloc(0, sizeof(*action), 0, SOCKET_ID_ANY); 762 if (!action) 763 return NULL; 764 action->type = MLX5DV_FLOW_ACTION_TAG; 765 action->tag_value = tag; 766 return action; 767 #endif /* HAVE_MLX5DV_DR */ 768 #else /* HAVE_IBV_FLOW_DV_SUPPORT */ 769 (void)tag; 770 errno = ENOTSUP; 771 return NULL; 772 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */ 773 } 774 775 static void * 776 mlx5_glue_dv_create_flow_action_meter(struct mlx5dv_dr_flow_meter_attr *attr) 777 { 778 #if defined(HAVE_MLX5DV_DR) && defined(HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER) 779 return mlx5dv_dr_action_create_flow_meter(attr); 780 #else 781 (void)attr; 782 errno = ENOTSUP; 783 return NULL; 784 #endif 785 } 786 787 static int 788 mlx5_glue_dv_modify_flow_action_meter(void *action, 789 struct mlx5dv_dr_flow_meter_attr *attr, 790 uint64_t modify_bits) 791 { 792 #if defined(HAVE_MLX5DV_DR) && defined(HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER) 793 return mlx5dv_dr_action_modify_flow_meter(action, attr, modify_bits); 794 #else 795 (void)action; 796 (void)attr; 797 (void)modify_bits; 798 errno = ENOTSUP; 799 return errno; 800 #endif 801 } 802 803 static void * 804 mlx5_glue_dr_create_flow_action_default_miss(void) 805 { 806 #if defined(HAVE_MLX5DV_DR) && defined(HAVE_MLX5_DR_CREATE_ACTION_DEFAULT_MISS) 807 return mlx5dv_dr_action_create_default_miss(); 808 #else 809 errno = ENOTSUP; 810 return NULL; 811 #endif 812 } 813 814 static int 815 mlx5_glue_dv_destroy_flow(void *flow_id) 816 { 817 #ifdef HAVE_MLX5DV_DR 818 return mlx5dv_dr_rule_destroy(flow_id); 819 #else 820 return ibv_destroy_flow(flow_id); 821 #endif 822 } 823 824 static int 825 mlx5_glue_dv_destroy_flow_matcher(void *matcher) 826 { 827 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 828 #ifdef HAVE_MLX5DV_DR 829 return mlx5dv_dr_matcher_destroy(matcher); 830 #else 831 return mlx5dv_destroy_flow_matcher(matcher); 832 #endif 833 #else 834 (void)matcher; 835 errno = ENOTSUP; 836 return errno; 837 #endif 838 } 839 840 static struct ibv_context * 841 mlx5_glue_dv_open_device(struct ibv_device *device) 842 { 843 #ifdef HAVE_IBV_DEVX_OBJ 844 return mlx5dv_open_device(device, 845 &(struct mlx5dv_context_attr){ 846 .flags = MLX5DV_CONTEXT_FLAGS_DEVX, 847 }); 848 #else 849 (void)device; 850 errno = ENOTSUP; 851 return NULL; 852 #endif 853 } 854 855 static struct mlx5dv_devx_obj * 856 mlx5_glue_devx_obj_create(struct ibv_context *ctx, 857 const void *in, size_t inlen, 858 void *out, size_t outlen) 859 { 860 #ifdef HAVE_IBV_DEVX_OBJ 861 return mlx5dv_devx_obj_create(ctx, in, inlen, out, outlen); 862 #else 863 (void)ctx; 864 (void)in; 865 (void)inlen; 866 (void)out; 867 (void)outlen; 868 errno = ENOTSUP; 869 return NULL; 870 #endif 871 } 872 873 static int 874 mlx5_glue_devx_obj_destroy(struct mlx5dv_devx_obj *obj) 875 { 876 #ifdef HAVE_IBV_DEVX_OBJ 877 return mlx5dv_devx_obj_destroy(obj); 878 #else 879 (void)obj; 880 return -ENOTSUP; 881 #endif 882 } 883 884 static int 885 mlx5_glue_devx_obj_query(struct mlx5dv_devx_obj *obj, 886 const void *in, size_t inlen, 887 void *out, size_t outlen) 888 { 889 #ifdef HAVE_IBV_DEVX_OBJ 890 return mlx5dv_devx_obj_query(obj, in, inlen, out, outlen); 891 #else 892 (void)obj; 893 (void)in; 894 (void)inlen; 895 (void)out; 896 (void)outlen; 897 return -ENOTSUP; 898 #endif 899 } 900 901 static int 902 mlx5_glue_devx_obj_modify(struct mlx5dv_devx_obj *obj, 903 const void *in, size_t inlen, 904 void *out, size_t outlen) 905 { 906 #ifdef HAVE_IBV_DEVX_OBJ 907 return mlx5dv_devx_obj_modify(obj, in, inlen, out, outlen); 908 #else 909 (void)obj; 910 (void)in; 911 (void)inlen; 912 (void)out; 913 (void)outlen; 914 return -ENOTSUP; 915 #endif 916 } 917 918 static int 919 mlx5_glue_devx_general_cmd(struct ibv_context *ctx, 920 const void *in, size_t inlen, 921 void *out, size_t outlen) 922 { 923 #ifdef HAVE_IBV_DEVX_OBJ 924 return mlx5dv_devx_general_cmd(ctx, in, inlen, out, outlen); 925 #else 926 (void)ctx; 927 (void)in; 928 (void)inlen; 929 (void)out; 930 (void)outlen; 931 return -ENOTSUP; 932 #endif 933 } 934 935 static struct mlx5dv_devx_cmd_comp * 936 mlx5_glue_devx_create_cmd_comp(struct ibv_context *ctx) 937 { 938 #ifdef HAVE_IBV_DEVX_ASYNC 939 return mlx5dv_devx_create_cmd_comp(ctx); 940 #else 941 (void)ctx; 942 errno = -ENOTSUP; 943 return NULL; 944 #endif 945 } 946 947 static void 948 mlx5_glue_devx_destroy_cmd_comp(struct mlx5dv_devx_cmd_comp *cmd_comp) 949 { 950 #ifdef HAVE_IBV_DEVX_ASYNC 951 mlx5dv_devx_destroy_cmd_comp(cmd_comp); 952 #else 953 (void)cmd_comp; 954 errno = -ENOTSUP; 955 #endif 956 } 957 958 static int 959 mlx5_glue_devx_obj_query_async(struct mlx5dv_devx_obj *obj, const void *in, 960 size_t inlen, size_t outlen, uint64_t wr_id, 961 struct mlx5dv_devx_cmd_comp *cmd_comp) 962 { 963 #ifdef HAVE_IBV_DEVX_ASYNC 964 return mlx5dv_devx_obj_query_async(obj, in, inlen, outlen, wr_id, 965 cmd_comp); 966 #else 967 (void)obj; 968 (void)in; 969 (void)inlen; 970 (void)outlen; 971 (void)wr_id; 972 (void)cmd_comp; 973 return -ENOTSUP; 974 #endif 975 } 976 977 static int 978 mlx5_glue_devx_get_async_cmd_comp(struct mlx5dv_devx_cmd_comp *cmd_comp, 979 struct mlx5dv_devx_async_cmd_hdr *cmd_resp, 980 size_t cmd_resp_len) 981 { 982 #ifdef HAVE_IBV_DEVX_ASYNC 983 return mlx5dv_devx_get_async_cmd_comp(cmd_comp, cmd_resp, 984 cmd_resp_len); 985 #else 986 (void)cmd_comp; 987 (void)cmd_resp; 988 (void)cmd_resp_len; 989 return -ENOTSUP; 990 #endif 991 } 992 993 static struct mlx5dv_devx_umem * 994 mlx5_glue_devx_umem_reg(struct ibv_context *context, void *addr, size_t size, 995 uint32_t access) 996 { 997 #ifdef HAVE_IBV_DEVX_OBJ 998 return mlx5dv_devx_umem_reg(context, addr, size, access); 999 #else 1000 (void)context; 1001 (void)addr; 1002 (void)size; 1003 (void)access; 1004 errno = -ENOTSUP; 1005 return NULL; 1006 #endif 1007 } 1008 1009 static int 1010 mlx5_glue_devx_umem_dereg(struct mlx5dv_devx_umem *dv_devx_umem) 1011 { 1012 #ifdef HAVE_IBV_DEVX_OBJ 1013 return mlx5dv_devx_umem_dereg(dv_devx_umem); 1014 #else 1015 (void)dv_devx_umem; 1016 return -ENOTSUP; 1017 #endif 1018 } 1019 1020 static int 1021 mlx5_glue_devx_qp_query(struct ibv_qp *qp, 1022 const void *in, size_t inlen, 1023 void *out, size_t outlen) 1024 { 1025 #ifdef HAVE_IBV_DEVX_QP 1026 return mlx5dv_devx_qp_query(qp, in, inlen, out, outlen); 1027 #else 1028 (void)qp; 1029 (void)in; 1030 (void)inlen; 1031 (void)out; 1032 (void)outlen; 1033 errno = ENOTSUP; 1034 return errno; 1035 #endif 1036 } 1037 1038 static int 1039 mlx5_glue_devx_port_query(struct ibv_context *ctx, 1040 uint32_t port_num, 1041 struct mlx5dv_devx_port *mlx5_devx_port) 1042 { 1043 #ifdef HAVE_MLX5DV_DR_DEVX_PORT 1044 return mlx5dv_query_devx_port(ctx, port_num, mlx5_devx_port); 1045 #else 1046 (void)ctx; 1047 (void)port_num; 1048 (void)mlx5_devx_port; 1049 errno = ENOTSUP; 1050 return errno; 1051 #endif 1052 } 1053 1054 static int 1055 mlx5_glue_dr_dump_domain(FILE *file, void *domain) 1056 { 1057 #ifdef HAVE_MLX5_DR_FLOW_DUMP 1058 return mlx5dv_dump_dr_domain(file, domain); 1059 #else 1060 RTE_SET_USED(file); 1061 RTE_SET_USED(domain); 1062 return -ENOTSUP; 1063 #endif 1064 } 1065 1066 static int 1067 mlx5_glue_devx_query_eqn(struct ibv_context *ctx, uint32_t cpus, 1068 uint32_t *eqn) 1069 { 1070 #ifdef HAVE_IBV_DEVX_OBJ 1071 return mlx5dv_devx_query_eqn(ctx, cpus, eqn); 1072 #else 1073 (void)ctx; 1074 (void)cpus; 1075 (void)eqn; 1076 return -ENOTSUP; 1077 #endif 1078 } 1079 1080 static struct mlx5dv_devx_event_channel * 1081 mlx5_glue_devx_create_event_channel(struct ibv_context *ctx, int flags) 1082 { 1083 #ifdef HAVE_IBV_DEVX_EVENT 1084 return mlx5dv_devx_create_event_channel(ctx, flags); 1085 #else 1086 (void)ctx; 1087 (void)flags; 1088 errno = ENOTSUP; 1089 return NULL; 1090 #endif 1091 } 1092 1093 static void 1094 mlx5_glue_devx_destroy_event_channel(struct mlx5dv_devx_event_channel *eventc) 1095 { 1096 #ifdef HAVE_IBV_DEVX_EVENT 1097 mlx5dv_devx_destroy_event_channel(eventc); 1098 #else 1099 (void)eventc; 1100 #endif 1101 } 1102 1103 static int 1104 mlx5_glue_devx_subscribe_devx_event(struct mlx5dv_devx_event_channel *eventc, 1105 struct mlx5dv_devx_obj *obj, 1106 uint16_t events_sz, uint16_t events_num[], 1107 uint64_t cookie) 1108 { 1109 #ifdef HAVE_IBV_DEVX_EVENT 1110 return mlx5dv_devx_subscribe_devx_event(eventc, obj, events_sz, 1111 events_num, cookie); 1112 #else 1113 (void)eventc; 1114 (void)obj; 1115 (void)events_sz; 1116 (void)events_num; 1117 (void)cookie; 1118 return -ENOTSUP; 1119 #endif 1120 } 1121 1122 static int 1123 mlx5_glue_devx_subscribe_devx_event_fd(struct mlx5dv_devx_event_channel *eventc, 1124 int fd, struct mlx5dv_devx_obj *obj, 1125 uint16_t event_num) 1126 { 1127 #ifdef HAVE_IBV_DEVX_EVENT 1128 return mlx5dv_devx_subscribe_devx_event_fd(eventc, fd, obj, event_num); 1129 #else 1130 (void)eventc; 1131 (void)fd; 1132 (void)obj; 1133 (void)event_num; 1134 return -ENOTSUP; 1135 #endif 1136 } 1137 1138 static ssize_t 1139 mlx5_glue_devx_get_event(struct mlx5dv_devx_event_channel *eventc, 1140 struct mlx5dv_devx_async_event_hdr *event_data, 1141 size_t event_resp_len) 1142 { 1143 #ifdef HAVE_IBV_DEVX_EVENT 1144 return mlx5dv_devx_get_event(eventc, event_data, event_resp_len); 1145 #else 1146 (void)eventc; 1147 (void)event_data; 1148 (void)event_resp_len; 1149 errno = ENOTSUP; 1150 return -1; 1151 #endif 1152 } 1153 1154 static struct mlx5dv_devx_uar * 1155 mlx5_glue_devx_alloc_uar(struct ibv_context *context, uint32_t flags) 1156 { 1157 #ifdef HAVE_IBV_DEVX_OBJ 1158 return mlx5dv_devx_alloc_uar(context, flags); 1159 #else 1160 (void)context; 1161 (void)flags; 1162 errno = ENOTSUP; 1163 return NULL; 1164 #endif 1165 } 1166 1167 static void 1168 mlx5_glue_devx_free_uar(struct mlx5dv_devx_uar *devx_uar) 1169 { 1170 #ifdef HAVE_IBV_DEVX_OBJ 1171 mlx5dv_devx_free_uar(devx_uar); 1172 #else 1173 (void)devx_uar; 1174 #endif 1175 } 1176 1177 static struct mlx5dv_var * 1178 mlx5_glue_dv_alloc_var(struct ibv_context *context, uint32_t flags) 1179 { 1180 #ifdef HAVE_IBV_VAR 1181 return mlx5dv_alloc_var(context, flags); 1182 #else 1183 (void)context; 1184 (void)flags; 1185 errno = ENOTSUP; 1186 return NULL; 1187 #endif 1188 } 1189 1190 static void 1191 mlx5_glue_dv_free_var(struct mlx5dv_var *var) 1192 { 1193 #ifdef HAVE_IBV_VAR 1194 mlx5dv_free_var(var); 1195 #else 1196 (void)var; 1197 errno = ENOTSUP; 1198 #endif 1199 } 1200 1201 static void 1202 mlx5_glue_dr_reclaim_domain_memory(void *domain, uint32_t enable) 1203 { 1204 #ifdef HAVE_MLX5DV_DR_MEM_RECLAIM 1205 mlx5dv_dr_domain_set_reclaim_device_memory(domain, enable); 1206 #else 1207 (void)(enable); 1208 (void)(domain); 1209 #endif 1210 } 1211 1212 static struct mlx5dv_pp * 1213 mlx5_glue_dv_alloc_pp(struct ibv_context *context, 1214 size_t pp_context_sz, 1215 const void *pp_context, 1216 uint32_t flags) 1217 { 1218 #ifdef HAVE_MLX5DV_PP_ALLOC 1219 return mlx5dv_pp_alloc(context, pp_context_sz, pp_context, flags); 1220 #else 1221 RTE_SET_USED(context); 1222 RTE_SET_USED(pp_context_sz); 1223 RTE_SET_USED(pp_context); 1224 RTE_SET_USED(flags); 1225 errno = ENOTSUP; 1226 return NULL; 1227 #endif 1228 } 1229 1230 static void 1231 mlx5_glue_dv_free_pp(struct mlx5dv_pp *pp) 1232 { 1233 #ifdef HAVE_MLX5DV_PP_ALLOC 1234 mlx5dv_pp_free(pp); 1235 #else 1236 RTE_SET_USED(pp); 1237 #endif 1238 } 1239 1240 __rte_cache_aligned 1241 const struct mlx5_glue *mlx5_glue = &(const struct mlx5_glue) { 1242 .version = MLX5_GLUE_VERSION, 1243 .fork_init = mlx5_glue_fork_init, 1244 .alloc_pd = mlx5_glue_alloc_pd, 1245 .dealloc_pd = mlx5_glue_dealloc_pd, 1246 .get_device_list = mlx5_glue_get_device_list, 1247 .free_device_list = mlx5_glue_free_device_list, 1248 .open_device = mlx5_glue_open_device, 1249 .close_device = mlx5_glue_close_device, 1250 .query_device = mlx5_glue_query_device, 1251 .query_device_ex = mlx5_glue_query_device_ex, 1252 .query_rt_values_ex = mlx5_glue_query_rt_values_ex, 1253 .query_port = mlx5_glue_query_port, 1254 .create_comp_channel = mlx5_glue_create_comp_channel, 1255 .destroy_comp_channel = mlx5_glue_destroy_comp_channel, 1256 .create_cq = mlx5_glue_create_cq, 1257 .destroy_cq = mlx5_glue_destroy_cq, 1258 .get_cq_event = mlx5_glue_get_cq_event, 1259 .ack_cq_events = mlx5_glue_ack_cq_events, 1260 .create_rwq_ind_table = mlx5_glue_create_rwq_ind_table, 1261 .destroy_rwq_ind_table = mlx5_glue_destroy_rwq_ind_table, 1262 .create_wq = mlx5_glue_create_wq, 1263 .destroy_wq = mlx5_glue_destroy_wq, 1264 .modify_wq = mlx5_glue_modify_wq, 1265 .create_flow = mlx5_glue_create_flow, 1266 .destroy_flow = mlx5_glue_destroy_flow, 1267 .destroy_flow_action = mlx5_glue_destroy_flow_action, 1268 .create_qp = mlx5_glue_create_qp, 1269 .create_qp_ex = mlx5_glue_create_qp_ex, 1270 .destroy_qp = mlx5_glue_destroy_qp, 1271 .modify_qp = mlx5_glue_modify_qp, 1272 .reg_mr = mlx5_glue_reg_mr, 1273 .alloc_null_mr = mlx5_glue_alloc_null_mr, 1274 .dereg_mr = mlx5_glue_dereg_mr, 1275 .create_counter_set = mlx5_glue_create_counter_set, 1276 .destroy_counter_set = mlx5_glue_destroy_counter_set, 1277 .describe_counter_set = mlx5_glue_describe_counter_set, 1278 .query_counter_set = mlx5_glue_query_counter_set, 1279 .create_counters = mlx5_glue_create_counters, 1280 .destroy_counters = mlx5_glue_destroy_counters, 1281 .attach_counters = mlx5_glue_attach_counters, 1282 .query_counters = mlx5_glue_query_counters, 1283 .ack_async_event = mlx5_glue_ack_async_event, 1284 .get_async_event = mlx5_glue_get_async_event, 1285 .port_state_str = mlx5_glue_port_state_str, 1286 .cq_ex_to_cq = mlx5_glue_cq_ex_to_cq, 1287 .dr_create_flow_action_dest_flow_tbl = 1288 mlx5_glue_dr_create_flow_action_dest_flow_tbl, 1289 .dr_create_flow_action_dest_port = 1290 mlx5_glue_dr_create_flow_action_dest_port, 1291 .dr_create_flow_action_drop = 1292 mlx5_glue_dr_create_flow_action_drop, 1293 .dr_create_flow_action_push_vlan = 1294 mlx5_glue_dr_create_flow_action_push_vlan, 1295 .dr_create_flow_action_pop_vlan = 1296 mlx5_glue_dr_create_flow_action_pop_vlan, 1297 .dr_create_flow_tbl = mlx5_glue_dr_create_flow_tbl, 1298 .dr_destroy_flow_tbl = mlx5_glue_dr_destroy_flow_tbl, 1299 .dr_create_domain = mlx5_glue_dr_create_domain, 1300 .dr_destroy_domain = mlx5_glue_dr_destroy_domain, 1301 .dv_create_cq = mlx5_glue_dv_create_cq, 1302 .dv_create_wq = mlx5_glue_dv_create_wq, 1303 .dv_query_device = mlx5_glue_dv_query_device, 1304 .dv_set_context_attr = mlx5_glue_dv_set_context_attr, 1305 .dv_init_obj = mlx5_glue_dv_init_obj, 1306 .dv_create_qp = mlx5_glue_dv_create_qp, 1307 .dv_create_flow_matcher = mlx5_glue_dv_create_flow_matcher, 1308 .dv_create_flow = mlx5_glue_dv_create_flow, 1309 .dv_create_flow_action_counter = 1310 mlx5_glue_dv_create_flow_action_counter, 1311 .dv_create_flow_action_dest_ibv_qp = 1312 mlx5_glue_dv_create_flow_action_dest_ibv_qp, 1313 .dv_create_flow_action_dest_devx_tir = 1314 mlx5_glue_dv_create_flow_action_dest_devx_tir, 1315 .dv_create_flow_action_modify_header = 1316 mlx5_glue_dv_create_flow_action_modify_header, 1317 .dv_create_flow_action_packet_reformat = 1318 mlx5_glue_dv_create_flow_action_packet_reformat, 1319 .dv_create_flow_action_tag = mlx5_glue_dv_create_flow_action_tag, 1320 .dv_create_flow_action_meter = mlx5_glue_dv_create_flow_action_meter, 1321 .dv_modify_flow_action_meter = mlx5_glue_dv_modify_flow_action_meter, 1322 .dr_create_flow_action_default_miss = 1323 mlx5_glue_dr_create_flow_action_default_miss, 1324 .dv_destroy_flow = mlx5_glue_dv_destroy_flow, 1325 .dv_destroy_flow_matcher = mlx5_glue_dv_destroy_flow_matcher, 1326 .dv_open_device = mlx5_glue_dv_open_device, 1327 .devx_obj_create = mlx5_glue_devx_obj_create, 1328 .devx_obj_destroy = mlx5_glue_devx_obj_destroy, 1329 .devx_obj_query = mlx5_glue_devx_obj_query, 1330 .devx_obj_modify = mlx5_glue_devx_obj_modify, 1331 .devx_general_cmd = mlx5_glue_devx_general_cmd, 1332 .devx_create_cmd_comp = mlx5_glue_devx_create_cmd_comp, 1333 .devx_destroy_cmd_comp = mlx5_glue_devx_destroy_cmd_comp, 1334 .devx_obj_query_async = mlx5_glue_devx_obj_query_async, 1335 .devx_get_async_cmd_comp = mlx5_glue_devx_get_async_cmd_comp, 1336 .devx_umem_reg = mlx5_glue_devx_umem_reg, 1337 .devx_umem_dereg = mlx5_glue_devx_umem_dereg, 1338 .devx_qp_query = mlx5_glue_devx_qp_query, 1339 .devx_port_query = mlx5_glue_devx_port_query, 1340 .dr_dump_domain = mlx5_glue_dr_dump_domain, 1341 .dr_reclaim_domain_memory = mlx5_glue_dr_reclaim_domain_memory, 1342 .devx_query_eqn = mlx5_glue_devx_query_eqn, 1343 .devx_create_event_channel = mlx5_glue_devx_create_event_channel, 1344 .devx_destroy_event_channel = mlx5_glue_devx_destroy_event_channel, 1345 .devx_subscribe_devx_event = mlx5_glue_devx_subscribe_devx_event, 1346 .devx_subscribe_devx_event_fd = mlx5_glue_devx_subscribe_devx_event_fd, 1347 .devx_get_event = mlx5_glue_devx_get_event, 1348 .devx_alloc_uar = mlx5_glue_devx_alloc_uar, 1349 .devx_free_uar = mlx5_glue_devx_free_uar, 1350 .dv_alloc_var = mlx5_glue_dv_alloc_var, 1351 .dv_free_var = mlx5_glue_dv_free_var, 1352 .dv_alloc_pp = mlx5_glue_dv_alloc_pp, 1353 .dv_free_pp = mlx5_glue_dv_free_pp, 1354 }; 1355