1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2018 6WIND S.A. 3 * Copyright 2018 Mellanox Technologies, Ltd 4 */ 5 6 #include <errno.h> 7 #include <stdalign.h> 8 #include <stddef.h> 9 #include <stdint.h> 10 #include <stdlib.h> 11 /* 12 * Not needed by this file; included to work around the lack of off_t 13 * definition for mlx5dv.h with unpatched rdma-core versions. 14 */ 15 #include <sys/types.h> 16 17 #include "mlx5_glue.h" 18 19 static int 20 mlx5_glue_fork_init(void) 21 { 22 return ibv_fork_init(); 23 } 24 25 static struct ibv_pd * 26 mlx5_glue_alloc_pd(struct ibv_context *context) 27 { 28 return ibv_alloc_pd(context); 29 } 30 31 static int 32 mlx5_glue_dealloc_pd(struct ibv_pd *pd) 33 { 34 return ibv_dealloc_pd(pd); 35 } 36 37 static struct ibv_device ** 38 mlx5_glue_get_device_list(int *num_devices) 39 { 40 return ibv_get_device_list(num_devices); 41 } 42 43 static void 44 mlx5_glue_free_device_list(struct ibv_device **list) 45 { 46 ibv_free_device_list(list); 47 } 48 49 static struct ibv_context * 50 mlx5_glue_open_device(struct ibv_device *device) 51 { 52 return ibv_open_device(device); 53 } 54 55 static int 56 mlx5_glue_close_device(struct ibv_context *context) 57 { 58 return ibv_close_device(context); 59 } 60 61 static int 62 mlx5_glue_query_device(struct ibv_context *context, 63 struct ibv_device_attr *device_attr) 64 { 65 return ibv_query_device(context, device_attr); 66 } 67 68 static int 69 mlx5_glue_query_device_ex(struct ibv_context *context, 70 const struct ibv_query_device_ex_input *input, 71 struct ibv_device_attr_ex *attr) 72 { 73 return ibv_query_device_ex(context, input, attr); 74 } 75 76 static int 77 mlx5_glue_query_rt_values_ex(struct ibv_context *context, 78 struct ibv_values_ex *values) 79 { 80 return ibv_query_rt_values_ex(context, values); 81 } 82 83 static int 84 mlx5_glue_query_port(struct ibv_context *context, uint8_t port_num, 85 struct ibv_port_attr *port_attr) 86 { 87 return ibv_query_port(context, port_num, port_attr); 88 } 89 90 static struct ibv_comp_channel * 91 mlx5_glue_create_comp_channel(struct ibv_context *context) 92 { 93 return ibv_create_comp_channel(context); 94 } 95 96 static int 97 mlx5_glue_destroy_comp_channel(struct ibv_comp_channel *channel) 98 { 99 return ibv_destroy_comp_channel(channel); 100 } 101 102 static struct ibv_cq * 103 mlx5_glue_create_cq(struct ibv_context *context, int cqe, void *cq_context, 104 struct ibv_comp_channel *channel, int comp_vector) 105 { 106 return ibv_create_cq(context, cqe, cq_context, channel, comp_vector); 107 } 108 109 static int 110 mlx5_glue_destroy_cq(struct ibv_cq *cq) 111 { 112 return ibv_destroy_cq(cq); 113 } 114 115 static int 116 mlx5_glue_get_cq_event(struct ibv_comp_channel *channel, struct ibv_cq **cq, 117 void **cq_context) 118 { 119 return ibv_get_cq_event(channel, cq, cq_context); 120 } 121 122 static void 123 mlx5_glue_ack_cq_events(struct ibv_cq *cq, unsigned int nevents) 124 { 125 ibv_ack_cq_events(cq, nevents); 126 } 127 128 static struct ibv_rwq_ind_table * 129 mlx5_glue_create_rwq_ind_table(struct ibv_context *context, 130 struct ibv_rwq_ind_table_init_attr *init_attr) 131 { 132 return ibv_create_rwq_ind_table(context, init_attr); 133 } 134 135 static int 136 mlx5_glue_destroy_rwq_ind_table(struct ibv_rwq_ind_table *rwq_ind_table) 137 { 138 return ibv_destroy_rwq_ind_table(rwq_ind_table); 139 } 140 141 static struct ibv_wq * 142 mlx5_glue_create_wq(struct ibv_context *context, 143 struct ibv_wq_init_attr *wq_init_attr) 144 { 145 return ibv_create_wq(context, wq_init_attr); 146 } 147 148 static int 149 mlx5_glue_destroy_wq(struct ibv_wq *wq) 150 { 151 return ibv_destroy_wq(wq); 152 } 153 static int 154 mlx5_glue_modify_wq(struct ibv_wq *wq, struct ibv_wq_attr *wq_attr) 155 { 156 return ibv_modify_wq(wq, wq_attr); 157 } 158 159 static struct ibv_flow * 160 mlx5_glue_create_flow(struct ibv_qp *qp, struct ibv_flow_attr *flow) 161 { 162 return ibv_create_flow(qp, flow); 163 } 164 165 static int 166 mlx5_glue_destroy_flow(struct ibv_flow *flow_id) 167 { 168 return ibv_destroy_flow(flow_id); 169 } 170 171 static int 172 mlx5_glue_destroy_flow_action(void *action) 173 { 174 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 175 #ifdef HAVE_MLX5DV_DR 176 return mlx5dv_dr_action_destroy(action); 177 #else 178 struct mlx5dv_flow_action_attr *attr = action; 179 int res = 0; 180 switch (attr->type) { 181 case MLX5DV_FLOW_ACTION_TAG: 182 break; 183 default: 184 res = ibv_destroy_flow_action(attr->action); 185 break; 186 } 187 free(action); 188 return res; 189 #endif 190 #else 191 (void)action; 192 return -ENOTSUP; 193 #endif 194 } 195 196 static struct ibv_qp * 197 mlx5_glue_create_qp(struct ibv_pd *pd, struct ibv_qp_init_attr *qp_init_attr) 198 { 199 return ibv_create_qp(pd, qp_init_attr); 200 } 201 202 static struct ibv_qp * 203 mlx5_glue_create_qp_ex(struct ibv_context *context, 204 struct ibv_qp_init_attr_ex *qp_init_attr_ex) 205 { 206 return ibv_create_qp_ex(context, qp_init_attr_ex); 207 } 208 209 static int 210 mlx5_glue_destroy_qp(struct ibv_qp *qp) 211 { 212 return ibv_destroy_qp(qp); 213 } 214 215 static int 216 mlx5_glue_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, int attr_mask) 217 { 218 return ibv_modify_qp(qp, attr, attr_mask); 219 } 220 221 static struct ibv_mr * 222 mlx5_glue_reg_mr(struct ibv_pd *pd, void *addr, size_t length, int access) 223 { 224 return ibv_reg_mr(pd, addr, length, access); 225 } 226 227 static struct ibv_mr * 228 mlx5_glue_alloc_null_mr(struct ibv_pd *pd) 229 { 230 #ifdef HAVE_IBV_DEVX_OBJ 231 return ibv_alloc_null_mr(pd); 232 #else 233 (void)pd; 234 errno = ENOTSUP; 235 return NULL; 236 #endif 237 } 238 239 static int 240 mlx5_glue_dereg_mr(struct ibv_mr *mr) 241 { 242 return ibv_dereg_mr(mr); 243 } 244 245 static struct ibv_counter_set * 246 mlx5_glue_create_counter_set(struct ibv_context *context, 247 struct ibv_counter_set_init_attr *init_attr) 248 { 249 #ifndef HAVE_IBV_DEVICE_COUNTERS_SET_V42 250 (void)context; 251 (void)init_attr; 252 return NULL; 253 #else 254 return ibv_create_counter_set(context, init_attr); 255 #endif 256 } 257 258 static int 259 mlx5_glue_destroy_counter_set(struct ibv_counter_set *cs) 260 { 261 #ifndef HAVE_IBV_DEVICE_COUNTERS_SET_V42 262 (void)cs; 263 return -ENOTSUP; 264 #else 265 return ibv_destroy_counter_set(cs); 266 #endif 267 } 268 269 static int 270 mlx5_glue_describe_counter_set(struct ibv_context *context, 271 uint16_t counter_set_id, 272 struct ibv_counter_set_description *cs_desc) 273 { 274 #ifndef HAVE_IBV_DEVICE_COUNTERS_SET_V42 275 (void)context; 276 (void)counter_set_id; 277 (void)cs_desc; 278 return -ENOTSUP; 279 #else 280 return ibv_describe_counter_set(context, counter_set_id, cs_desc); 281 #endif 282 } 283 284 static int 285 mlx5_glue_query_counter_set(struct ibv_query_counter_set_attr *query_attr, 286 struct ibv_counter_set_data *cs_data) 287 { 288 #ifndef HAVE_IBV_DEVICE_COUNTERS_SET_V42 289 (void)query_attr; 290 (void)cs_data; 291 return -ENOTSUP; 292 #else 293 return ibv_query_counter_set(query_attr, cs_data); 294 #endif 295 } 296 297 static struct ibv_counters * 298 mlx5_glue_create_counters(struct ibv_context *context, 299 struct ibv_counters_init_attr *init_attr) 300 { 301 #ifndef HAVE_IBV_DEVICE_COUNTERS_SET_V45 302 (void)context; 303 (void)init_attr; 304 errno = ENOTSUP; 305 return NULL; 306 #else 307 return ibv_create_counters(context, init_attr); 308 #endif 309 } 310 311 static int 312 mlx5_glue_destroy_counters(struct ibv_counters *counters) 313 { 314 #ifndef HAVE_IBV_DEVICE_COUNTERS_SET_V45 315 (void)counters; 316 return -ENOTSUP; 317 #else 318 return ibv_destroy_counters(counters); 319 #endif 320 } 321 322 static int 323 mlx5_glue_attach_counters(struct ibv_counters *counters, 324 struct ibv_counter_attach_attr *attr, 325 struct ibv_flow *flow) 326 { 327 #ifndef HAVE_IBV_DEVICE_COUNTERS_SET_V45 328 (void)counters; 329 (void)attr; 330 (void)flow; 331 return -ENOTSUP; 332 #else 333 return ibv_attach_counters_point_flow(counters, attr, flow); 334 #endif 335 } 336 337 static int 338 mlx5_glue_query_counters(struct ibv_counters *counters, 339 uint64_t *counters_value, 340 uint32_t ncounters, 341 uint32_t flags) 342 { 343 #ifndef HAVE_IBV_DEVICE_COUNTERS_SET_V45 344 (void)counters; 345 (void)counters_value; 346 (void)ncounters; 347 (void)flags; 348 return -ENOTSUP; 349 #else 350 return ibv_read_counters(counters, counters_value, ncounters, flags); 351 #endif 352 } 353 354 static void 355 mlx5_glue_ack_async_event(struct ibv_async_event *event) 356 { 357 ibv_ack_async_event(event); 358 } 359 360 static int 361 mlx5_glue_get_async_event(struct ibv_context *context, 362 struct ibv_async_event *event) 363 { 364 return ibv_get_async_event(context, event); 365 } 366 367 static const char * 368 mlx5_glue_port_state_str(enum ibv_port_state port_state) 369 { 370 return ibv_port_state_str(port_state); 371 } 372 373 static struct ibv_cq * 374 mlx5_glue_cq_ex_to_cq(struct ibv_cq_ex *cq) 375 { 376 return ibv_cq_ex_to_cq(cq); 377 } 378 379 static void * 380 mlx5_glue_dr_create_flow_action_dest_flow_tbl(void *tbl) 381 { 382 #ifdef HAVE_MLX5DV_DR 383 return mlx5dv_dr_action_create_dest_table(tbl); 384 #else 385 (void)tbl; 386 errno = ENOTSUP; 387 return NULL; 388 #endif 389 } 390 391 static void * 392 mlx5_glue_dr_create_flow_action_dest_port(void *domain, uint32_t port) 393 { 394 #ifdef HAVE_MLX5DV_DR_DEVX_PORT 395 return mlx5dv_dr_action_create_dest_ib_port(domain, port); 396 #else 397 #ifdef HAVE_MLX5DV_DR_ESWITCH 398 return mlx5dv_dr_action_create_dest_vport(domain, port); 399 #else 400 (void)domain; 401 (void)port; 402 errno = ENOTSUP; 403 return NULL; 404 #endif 405 #endif 406 } 407 408 static void * 409 mlx5_glue_dr_create_flow_action_drop(void) 410 { 411 #ifdef HAVE_MLX5DV_DR_ESWITCH 412 return mlx5dv_dr_action_create_drop(); 413 #else 414 errno = ENOTSUP; 415 return NULL; 416 #endif 417 } 418 419 static void * 420 mlx5_glue_dr_create_flow_action_push_vlan(struct mlx5dv_dr_domain *domain, 421 rte_be32_t vlan_tag) 422 { 423 #ifdef HAVE_MLX5DV_DR_VLAN 424 return mlx5dv_dr_action_create_push_vlan(domain, vlan_tag); 425 #else 426 (void)domain; 427 (void)vlan_tag; 428 errno = ENOTSUP; 429 return NULL; 430 #endif 431 } 432 433 static void * 434 mlx5_glue_dr_create_flow_action_pop_vlan(void) 435 { 436 #ifdef HAVE_MLX5DV_DR_VLAN 437 return mlx5dv_dr_action_create_pop_vlan(); 438 #else 439 errno = ENOTSUP; 440 return NULL; 441 #endif 442 } 443 444 static void * 445 mlx5_glue_dr_create_flow_tbl(void *domain, uint32_t level) 446 { 447 #ifdef HAVE_MLX5DV_DR 448 return mlx5dv_dr_table_create(domain, level); 449 #else 450 (void)domain; 451 (void)level; 452 errno = ENOTSUP; 453 return NULL; 454 #endif 455 } 456 457 static int 458 mlx5_glue_dr_destroy_flow_tbl(void *tbl) 459 { 460 #ifdef HAVE_MLX5DV_DR 461 return mlx5dv_dr_table_destroy(tbl); 462 #else 463 (void)tbl; 464 errno = ENOTSUP; 465 return errno; 466 #endif 467 } 468 469 static void * 470 mlx5_glue_dr_create_domain(struct ibv_context *ctx, 471 enum mlx5dv_dr_domain_type domain) 472 { 473 #ifdef HAVE_MLX5DV_DR 474 return mlx5dv_dr_domain_create(ctx, domain); 475 #else 476 (void)ctx; 477 (void)domain; 478 errno = ENOTSUP; 479 return NULL; 480 #endif 481 } 482 483 static int 484 mlx5_glue_dr_destroy_domain(void *domain) 485 { 486 #ifdef HAVE_MLX5DV_DR 487 return mlx5dv_dr_domain_destroy(domain); 488 #else 489 (void)domain; 490 errno = ENOTSUP; 491 return errno; 492 #endif 493 } 494 495 static struct ibv_cq_ex * 496 mlx5_glue_dv_create_cq(struct ibv_context *context, 497 struct ibv_cq_init_attr_ex *cq_attr, 498 struct mlx5dv_cq_init_attr *mlx5_cq_attr) 499 { 500 return mlx5dv_create_cq(context, cq_attr, mlx5_cq_attr); 501 } 502 503 static struct ibv_wq * 504 mlx5_glue_dv_create_wq(struct ibv_context *context, 505 struct ibv_wq_init_attr *wq_attr, 506 struct mlx5dv_wq_init_attr *mlx5_wq_attr) 507 { 508 #ifndef HAVE_IBV_DEVICE_STRIDING_RQ_SUPPORT 509 (void)context; 510 (void)wq_attr; 511 (void)mlx5_wq_attr; 512 errno = ENOTSUP; 513 return NULL; 514 #else 515 return mlx5dv_create_wq(context, wq_attr, mlx5_wq_attr); 516 #endif 517 } 518 519 static int 520 mlx5_glue_dv_query_device(struct ibv_context *ctx, 521 struct mlx5dv_context *attrs_out) 522 { 523 return mlx5dv_query_device(ctx, attrs_out); 524 } 525 526 static int 527 mlx5_glue_dv_set_context_attr(struct ibv_context *ibv_ctx, 528 enum mlx5dv_set_ctx_attr_type type, void *attr) 529 { 530 return mlx5dv_set_context_attr(ibv_ctx, type, attr); 531 } 532 533 static int 534 mlx5_glue_dv_init_obj(struct mlx5dv_obj *obj, uint64_t obj_type) 535 { 536 return mlx5dv_init_obj(obj, obj_type); 537 } 538 539 static struct ibv_qp * 540 mlx5_glue_dv_create_qp(struct ibv_context *context, 541 struct ibv_qp_init_attr_ex *qp_init_attr_ex, 542 struct mlx5dv_qp_init_attr *dv_qp_init_attr) 543 { 544 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 545 return mlx5dv_create_qp(context, qp_init_attr_ex, dv_qp_init_attr); 546 #else 547 (void)context; 548 (void)qp_init_attr_ex; 549 (void)dv_qp_init_attr; 550 errno = ENOTSUP; 551 return NULL; 552 #endif 553 } 554 555 static void * 556 mlx5_glue_dv_create_flow_matcher(struct ibv_context *context, 557 struct mlx5dv_flow_matcher_attr *matcher_attr, 558 void *tbl) 559 { 560 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 561 #ifdef HAVE_MLX5DV_DR 562 (void)context; 563 return mlx5dv_dr_matcher_create(tbl, matcher_attr->priority, 564 matcher_attr->match_criteria_enable, 565 matcher_attr->match_mask); 566 #else 567 (void)tbl; 568 return mlx5dv_create_flow_matcher(context, matcher_attr); 569 #endif 570 #else 571 (void)context; 572 (void)matcher_attr; 573 (void)tbl; 574 errno = ENOTSUP; 575 return NULL; 576 #endif 577 } 578 579 static void * 580 mlx5_glue_dv_create_flow(void *matcher, 581 void *match_value, 582 size_t num_actions, 583 void *actions[]) 584 { 585 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 586 #ifdef HAVE_MLX5DV_DR 587 return mlx5dv_dr_rule_create(matcher, match_value, num_actions, 588 (struct mlx5dv_dr_action **)actions); 589 #else 590 size_t i; 591 struct mlx5dv_flow_action_attr actions_attr[8]; 592 593 if (num_actions > 8) 594 return NULL; 595 for (i = 0; i < num_actions; i++) 596 actions_attr[i] = 597 *((struct mlx5dv_flow_action_attr *)(actions[i])); 598 return mlx5dv_create_flow(matcher, match_value, 599 num_actions, actions_attr); 600 #endif 601 #else 602 (void)matcher; 603 (void)match_value; 604 (void)num_actions; 605 (void)actions; 606 return NULL; 607 #endif 608 } 609 610 static void * 611 mlx5_glue_dv_create_flow_action_counter(void *counter_obj, uint32_t offset) 612 { 613 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 614 #ifdef HAVE_MLX5DV_DR 615 return mlx5dv_dr_action_create_flow_counter(counter_obj, offset); 616 #else 617 struct mlx5dv_flow_action_attr *action; 618 619 (void)offset; 620 action = malloc(sizeof(*action)); 621 if (!action) 622 return NULL; 623 action->type = MLX5DV_FLOW_ACTION_COUNTERS_DEVX; 624 action->obj = counter_obj; 625 return action; 626 #endif 627 #else 628 (void)counter_obj; 629 (void)offset; 630 errno = ENOTSUP; 631 return NULL; 632 #endif 633 } 634 635 static void * 636 mlx5_glue_dv_create_flow_action_dest_ibv_qp(void *qp) 637 { 638 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 639 #ifdef HAVE_MLX5DV_DR 640 return mlx5dv_dr_action_create_dest_ibv_qp(qp); 641 #else 642 struct mlx5dv_flow_action_attr *action; 643 644 action = malloc(sizeof(*action)); 645 if (!action) 646 return NULL; 647 action->type = MLX5DV_FLOW_ACTION_DEST_IBV_QP; 648 action->obj = qp; 649 return action; 650 #endif 651 #else 652 (void)qp; 653 errno = ENOTSUP; 654 return NULL; 655 #endif 656 } 657 658 static void * 659 mlx5_glue_dv_create_flow_action_dest_devx_tir(void *tir) 660 { 661 #ifdef HAVE_MLX5DV_DR_ACTION_DEST_DEVX_TIR 662 return mlx5dv_dr_action_create_dest_devx_tir(tir); 663 #else 664 (void)tir; 665 errno = ENOTSUP; 666 return NULL; 667 #endif 668 } 669 670 static void * 671 mlx5_glue_dv_create_flow_action_modify_header 672 (struct ibv_context *ctx, 673 enum mlx5dv_flow_table_type ft_type, 674 void *domain, uint64_t flags, 675 size_t actions_sz, 676 uint64_t actions[]) 677 { 678 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 679 #ifdef HAVE_MLX5DV_DR 680 (void)ctx; 681 (void)ft_type; 682 return mlx5dv_dr_action_create_modify_header(domain, flags, actions_sz, 683 (__be64 *)actions); 684 #else 685 struct mlx5dv_flow_action_attr *action; 686 687 (void)domain; 688 (void)flags; 689 action = malloc(sizeof(*action)); 690 if (!action) 691 return NULL; 692 action->type = MLX5DV_FLOW_ACTION_IBV_FLOW_ACTION; 693 action->action = mlx5dv_create_flow_action_modify_header 694 (ctx, actions_sz, actions, ft_type); 695 return action; 696 #endif 697 #else 698 (void)ctx; 699 (void)ft_type; 700 (void)domain; 701 (void)flags; 702 (void)actions_sz; 703 (void)actions; 704 errno = ENOTSUP; 705 return NULL; 706 #endif 707 } 708 709 static void * 710 mlx5_glue_dv_create_flow_action_packet_reformat 711 (struct ibv_context *ctx, 712 enum mlx5dv_flow_action_packet_reformat_type reformat_type, 713 enum mlx5dv_flow_table_type ft_type, 714 struct mlx5dv_dr_domain *domain, 715 uint32_t flags, size_t data_sz, void *data) 716 { 717 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 718 #ifdef HAVE_MLX5DV_DR 719 (void)ctx; 720 (void)ft_type; 721 return mlx5dv_dr_action_create_packet_reformat(domain, flags, 722 reformat_type, data_sz, 723 data); 724 #else 725 (void)domain; 726 (void)flags; 727 struct mlx5dv_flow_action_attr *action; 728 729 action = malloc(sizeof(*action)); 730 if (!action) 731 return NULL; 732 action->type = MLX5DV_FLOW_ACTION_IBV_FLOW_ACTION; 733 action->action = mlx5dv_create_flow_action_packet_reformat 734 (ctx, data_sz, data, reformat_type, ft_type); 735 return action; 736 #endif 737 #else 738 (void)ctx; 739 (void)reformat_type; 740 (void)ft_type; 741 (void)domain; 742 (void)flags; 743 (void)data_sz; 744 (void)data; 745 errno = ENOTSUP; 746 return NULL; 747 #endif 748 } 749 750 static void * 751 mlx5_glue_dv_create_flow_action_tag(uint32_t tag) 752 { 753 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 754 #ifdef HAVE_MLX5DV_DR 755 return mlx5dv_dr_action_create_tag(tag); 756 #else /* HAVE_MLX5DV_DR */ 757 struct mlx5dv_flow_action_attr *action; 758 action = malloc(sizeof(*action)); 759 if (!action) 760 return NULL; 761 action->type = MLX5DV_FLOW_ACTION_TAG; 762 action->tag_value = tag; 763 return action; 764 #endif /* HAVE_MLX5DV_DR */ 765 #else /* HAVE_IBV_FLOW_DV_SUPPORT */ 766 (void)tag; 767 errno = ENOTSUP; 768 return NULL; 769 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */ 770 } 771 772 static void * 773 mlx5_glue_dv_create_flow_action_meter(struct mlx5dv_dr_flow_meter_attr *attr) 774 { 775 #if defined(HAVE_MLX5DV_DR) && defined(HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER) 776 return mlx5dv_dr_action_create_flow_meter(attr); 777 #else 778 (void)attr; 779 errno = ENOTSUP; 780 return NULL; 781 #endif 782 } 783 784 static int 785 mlx5_glue_dv_modify_flow_action_meter(void *action, 786 struct mlx5dv_dr_flow_meter_attr *attr, 787 uint64_t modify_bits) 788 { 789 #if defined(HAVE_MLX5DV_DR) && defined(HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER) 790 return mlx5dv_dr_action_modify_flow_meter(action, attr, modify_bits); 791 #else 792 (void)action; 793 (void)attr; 794 (void)modify_bits; 795 errno = ENOTSUP; 796 return errno; 797 #endif 798 } 799 800 static void * 801 mlx5_glue_dr_create_flow_action_default_miss(void) 802 { 803 #if defined(HAVE_MLX5DV_DR) && defined(HAVE_MLX5_DR_CREATE_ACTION_DEFAULT_MISS) 804 return mlx5dv_dr_action_create_default_miss(); 805 #else 806 errno = ENOTSUP; 807 return NULL; 808 #endif 809 } 810 811 static int 812 mlx5_glue_dv_destroy_flow(void *flow_id) 813 { 814 #ifdef HAVE_MLX5DV_DR 815 return mlx5dv_dr_rule_destroy(flow_id); 816 #else 817 return ibv_destroy_flow(flow_id); 818 #endif 819 } 820 821 static int 822 mlx5_glue_dv_destroy_flow_matcher(void *matcher) 823 { 824 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 825 #ifdef HAVE_MLX5DV_DR 826 return mlx5dv_dr_matcher_destroy(matcher); 827 #else 828 return mlx5dv_destroy_flow_matcher(matcher); 829 #endif 830 #else 831 (void)matcher; 832 errno = ENOTSUP; 833 return errno; 834 #endif 835 } 836 837 static struct ibv_context * 838 mlx5_glue_dv_open_device(struct ibv_device *device) 839 { 840 #ifdef HAVE_IBV_DEVX_OBJ 841 return mlx5dv_open_device(device, 842 &(struct mlx5dv_context_attr){ 843 .flags = MLX5DV_CONTEXT_FLAGS_DEVX, 844 }); 845 #else 846 (void)device; 847 errno = ENOTSUP; 848 return NULL; 849 #endif 850 } 851 852 static struct mlx5dv_devx_obj * 853 mlx5_glue_devx_obj_create(struct ibv_context *ctx, 854 const void *in, size_t inlen, 855 void *out, size_t outlen) 856 { 857 #ifdef HAVE_IBV_DEVX_OBJ 858 return mlx5dv_devx_obj_create(ctx, in, inlen, out, outlen); 859 #else 860 (void)ctx; 861 (void)in; 862 (void)inlen; 863 (void)out; 864 (void)outlen; 865 errno = ENOTSUP; 866 return NULL; 867 #endif 868 } 869 870 static int 871 mlx5_glue_devx_obj_destroy(struct mlx5dv_devx_obj *obj) 872 { 873 #ifdef HAVE_IBV_DEVX_OBJ 874 return mlx5dv_devx_obj_destroy(obj); 875 #else 876 (void)obj; 877 return -ENOTSUP; 878 #endif 879 } 880 881 static int 882 mlx5_glue_devx_obj_query(struct mlx5dv_devx_obj *obj, 883 const void *in, size_t inlen, 884 void *out, size_t outlen) 885 { 886 #ifdef HAVE_IBV_DEVX_OBJ 887 return mlx5dv_devx_obj_query(obj, in, inlen, out, outlen); 888 #else 889 (void)obj; 890 (void)in; 891 (void)inlen; 892 (void)out; 893 (void)outlen; 894 return -ENOTSUP; 895 #endif 896 } 897 898 static int 899 mlx5_glue_devx_obj_modify(struct mlx5dv_devx_obj *obj, 900 const void *in, size_t inlen, 901 void *out, size_t outlen) 902 { 903 #ifdef HAVE_IBV_DEVX_OBJ 904 return mlx5dv_devx_obj_modify(obj, in, inlen, out, outlen); 905 #else 906 (void)obj; 907 (void)in; 908 (void)inlen; 909 (void)out; 910 (void)outlen; 911 return -ENOTSUP; 912 #endif 913 } 914 915 static int 916 mlx5_glue_devx_general_cmd(struct ibv_context *ctx, 917 const void *in, size_t inlen, 918 void *out, size_t outlen) 919 { 920 #ifdef HAVE_IBV_DEVX_OBJ 921 return mlx5dv_devx_general_cmd(ctx, in, inlen, out, outlen); 922 #else 923 (void)ctx; 924 (void)in; 925 (void)inlen; 926 (void)out; 927 (void)outlen; 928 return -ENOTSUP; 929 #endif 930 } 931 932 static struct mlx5dv_devx_cmd_comp * 933 mlx5_glue_devx_create_cmd_comp(struct ibv_context *ctx) 934 { 935 #ifdef HAVE_IBV_DEVX_ASYNC 936 return mlx5dv_devx_create_cmd_comp(ctx); 937 #else 938 (void)ctx; 939 errno = -ENOTSUP; 940 return NULL; 941 #endif 942 } 943 944 static void 945 mlx5_glue_devx_destroy_cmd_comp(struct mlx5dv_devx_cmd_comp *cmd_comp) 946 { 947 #ifdef HAVE_IBV_DEVX_ASYNC 948 mlx5dv_devx_destroy_cmd_comp(cmd_comp); 949 #else 950 (void)cmd_comp; 951 errno = -ENOTSUP; 952 #endif 953 } 954 955 static int 956 mlx5_glue_devx_obj_query_async(struct mlx5dv_devx_obj *obj, const void *in, 957 size_t inlen, size_t outlen, uint64_t wr_id, 958 struct mlx5dv_devx_cmd_comp *cmd_comp) 959 { 960 #ifdef HAVE_IBV_DEVX_ASYNC 961 return mlx5dv_devx_obj_query_async(obj, in, inlen, outlen, wr_id, 962 cmd_comp); 963 #else 964 (void)obj; 965 (void)in; 966 (void)inlen; 967 (void)outlen; 968 (void)wr_id; 969 (void)cmd_comp; 970 return -ENOTSUP; 971 #endif 972 } 973 974 static int 975 mlx5_glue_devx_get_async_cmd_comp(struct mlx5dv_devx_cmd_comp *cmd_comp, 976 struct mlx5dv_devx_async_cmd_hdr *cmd_resp, 977 size_t cmd_resp_len) 978 { 979 #ifdef HAVE_IBV_DEVX_ASYNC 980 return mlx5dv_devx_get_async_cmd_comp(cmd_comp, cmd_resp, 981 cmd_resp_len); 982 #else 983 (void)cmd_comp; 984 (void)cmd_resp; 985 (void)cmd_resp_len; 986 return -ENOTSUP; 987 #endif 988 } 989 990 static struct mlx5dv_devx_umem * 991 mlx5_glue_devx_umem_reg(struct ibv_context *context, void *addr, size_t size, 992 uint32_t access) 993 { 994 #ifdef HAVE_IBV_DEVX_OBJ 995 return mlx5dv_devx_umem_reg(context, addr, size, access); 996 #else 997 (void)context; 998 (void)addr; 999 (void)size; 1000 (void)access; 1001 errno = -ENOTSUP; 1002 return NULL; 1003 #endif 1004 } 1005 1006 static int 1007 mlx5_glue_devx_umem_dereg(struct mlx5dv_devx_umem *dv_devx_umem) 1008 { 1009 #ifdef HAVE_IBV_DEVX_OBJ 1010 return mlx5dv_devx_umem_dereg(dv_devx_umem); 1011 #else 1012 (void)dv_devx_umem; 1013 return -ENOTSUP; 1014 #endif 1015 } 1016 1017 static int 1018 mlx5_glue_devx_qp_query(struct ibv_qp *qp, 1019 const void *in, size_t inlen, 1020 void *out, size_t outlen) 1021 { 1022 #ifdef HAVE_IBV_DEVX_QP 1023 return mlx5dv_devx_qp_query(qp, in, inlen, out, outlen); 1024 #else 1025 (void)qp; 1026 (void)in; 1027 (void)inlen; 1028 (void)out; 1029 (void)outlen; 1030 errno = ENOTSUP; 1031 return errno; 1032 #endif 1033 } 1034 1035 static int 1036 mlx5_glue_devx_port_query(struct ibv_context *ctx, 1037 uint32_t port_num, 1038 struct mlx5dv_devx_port *mlx5_devx_port) 1039 { 1040 #ifdef HAVE_MLX5DV_DR_DEVX_PORT 1041 return mlx5dv_query_devx_port(ctx, port_num, mlx5_devx_port); 1042 #else 1043 (void)ctx; 1044 (void)port_num; 1045 (void)mlx5_devx_port; 1046 errno = ENOTSUP; 1047 return errno; 1048 #endif 1049 } 1050 1051 static int 1052 mlx5_glue_dr_dump_domain(FILE *file, void *domain) 1053 { 1054 #ifdef HAVE_MLX5_DR_FLOW_DUMP 1055 return mlx5dv_dump_dr_domain(file, domain); 1056 #else 1057 RTE_SET_USED(file); 1058 RTE_SET_USED(domain); 1059 return -ENOTSUP; 1060 #endif 1061 } 1062 1063 static int 1064 mlx5_glue_devx_query_eqn(struct ibv_context *ctx, uint32_t cpus, 1065 uint32_t *eqn) 1066 { 1067 #ifdef HAVE_IBV_DEVX_OBJ 1068 return mlx5dv_devx_query_eqn(ctx, cpus, eqn); 1069 #else 1070 (void)ctx; 1071 (void)cpus; 1072 (void)eqn; 1073 return -ENOTSUP; 1074 #endif 1075 } 1076 1077 static struct mlx5dv_devx_event_channel * 1078 mlx5_glue_devx_create_event_channel(struct ibv_context *ctx, int flags) 1079 { 1080 #ifdef HAVE_IBV_DEVX_EVENT 1081 return mlx5dv_devx_create_event_channel(ctx, flags); 1082 #else 1083 (void)ctx; 1084 (void)flags; 1085 errno = ENOTSUP; 1086 return NULL; 1087 #endif 1088 } 1089 1090 static void 1091 mlx5_glue_devx_destroy_event_channel(struct mlx5dv_devx_event_channel *eventc) 1092 { 1093 #ifdef HAVE_IBV_DEVX_EVENT 1094 mlx5dv_devx_destroy_event_channel(eventc); 1095 #else 1096 (void)eventc; 1097 #endif 1098 } 1099 1100 static int 1101 mlx5_glue_devx_subscribe_devx_event(struct mlx5dv_devx_event_channel *eventc, 1102 struct mlx5dv_devx_obj *obj, 1103 uint16_t events_sz, uint16_t events_num[], 1104 uint64_t cookie) 1105 { 1106 #ifdef HAVE_IBV_DEVX_EVENT 1107 return mlx5dv_devx_subscribe_devx_event(eventc, obj, events_sz, 1108 events_num, cookie); 1109 #else 1110 (void)eventc; 1111 (void)obj; 1112 (void)events_sz; 1113 (void)events_num; 1114 (void)cookie; 1115 return -ENOTSUP; 1116 #endif 1117 } 1118 1119 static int 1120 mlx5_glue_devx_subscribe_devx_event_fd(struct mlx5dv_devx_event_channel *eventc, 1121 int fd, struct mlx5dv_devx_obj *obj, 1122 uint16_t event_num) 1123 { 1124 #ifdef HAVE_IBV_DEVX_EVENT 1125 return mlx5dv_devx_subscribe_devx_event_fd(eventc, fd, obj, event_num); 1126 #else 1127 (void)eventc; 1128 (void)fd; 1129 (void)obj; 1130 (void)event_num; 1131 return -ENOTSUP; 1132 #endif 1133 } 1134 1135 static ssize_t 1136 mlx5_glue_devx_get_event(struct mlx5dv_devx_event_channel *eventc, 1137 struct mlx5dv_devx_async_event_hdr *event_data, 1138 size_t event_resp_len) 1139 { 1140 #ifdef HAVE_IBV_DEVX_EVENT 1141 return mlx5dv_devx_get_event(eventc, event_data, event_resp_len); 1142 #else 1143 (void)eventc; 1144 (void)event_data; 1145 (void)event_resp_len; 1146 errno = ENOTSUP; 1147 return -1; 1148 #endif 1149 } 1150 1151 static struct mlx5dv_devx_uar * 1152 mlx5_glue_devx_alloc_uar(struct ibv_context *context, uint32_t flags) 1153 { 1154 #ifdef HAVE_IBV_DEVX_OBJ 1155 return mlx5dv_devx_alloc_uar(context, flags); 1156 #else 1157 (void)context; 1158 (void)flags; 1159 errno = ENOTSUP; 1160 return NULL; 1161 #endif 1162 } 1163 1164 static void 1165 mlx5_glue_devx_free_uar(struct mlx5dv_devx_uar *devx_uar) 1166 { 1167 #ifdef HAVE_IBV_DEVX_OBJ 1168 mlx5dv_devx_free_uar(devx_uar); 1169 #else 1170 (void)devx_uar; 1171 #endif 1172 } 1173 1174 static struct mlx5dv_var * 1175 mlx5_glue_dv_alloc_var(struct ibv_context *context, uint32_t flags) 1176 { 1177 #ifdef HAVE_IBV_VAR 1178 return mlx5dv_alloc_var(context, flags); 1179 #else 1180 (void)context; 1181 (void)flags; 1182 errno = ENOTSUP; 1183 return NULL; 1184 #endif 1185 } 1186 1187 static void 1188 mlx5_glue_dv_free_var(struct mlx5dv_var *var) 1189 { 1190 #ifdef HAVE_IBV_VAR 1191 mlx5dv_free_var(var); 1192 #else 1193 (void)var; 1194 errno = ENOTSUP; 1195 #endif 1196 } 1197 1198 1199 static void 1200 mlx5_glue_dr_reclaim_domain_memory(void *domain, uint32_t enable) 1201 { 1202 #ifdef HAVE_MLX5DV_DR_MEM_RECLAIM 1203 mlx5dv_dr_domain_set_reclaim_device_memory(domain, enable); 1204 #else 1205 (void)(enable); 1206 (void)(domain); 1207 #endif 1208 } 1209 1210 __rte_cache_aligned 1211 const struct mlx5_glue *mlx5_glue = &(const struct mlx5_glue) { 1212 .version = MLX5_GLUE_VERSION, 1213 .fork_init = mlx5_glue_fork_init, 1214 .alloc_pd = mlx5_glue_alloc_pd, 1215 .dealloc_pd = mlx5_glue_dealloc_pd, 1216 .get_device_list = mlx5_glue_get_device_list, 1217 .free_device_list = mlx5_glue_free_device_list, 1218 .open_device = mlx5_glue_open_device, 1219 .close_device = mlx5_glue_close_device, 1220 .query_device = mlx5_glue_query_device, 1221 .query_device_ex = mlx5_glue_query_device_ex, 1222 .query_rt_values_ex = mlx5_glue_query_rt_values_ex, 1223 .query_port = mlx5_glue_query_port, 1224 .create_comp_channel = mlx5_glue_create_comp_channel, 1225 .destroy_comp_channel = mlx5_glue_destroy_comp_channel, 1226 .create_cq = mlx5_glue_create_cq, 1227 .destroy_cq = mlx5_glue_destroy_cq, 1228 .get_cq_event = mlx5_glue_get_cq_event, 1229 .ack_cq_events = mlx5_glue_ack_cq_events, 1230 .create_rwq_ind_table = mlx5_glue_create_rwq_ind_table, 1231 .destroy_rwq_ind_table = mlx5_glue_destroy_rwq_ind_table, 1232 .create_wq = mlx5_glue_create_wq, 1233 .destroy_wq = mlx5_glue_destroy_wq, 1234 .modify_wq = mlx5_glue_modify_wq, 1235 .create_flow = mlx5_glue_create_flow, 1236 .destroy_flow = mlx5_glue_destroy_flow, 1237 .destroy_flow_action = mlx5_glue_destroy_flow_action, 1238 .create_qp = mlx5_glue_create_qp, 1239 .create_qp_ex = mlx5_glue_create_qp_ex, 1240 .destroy_qp = mlx5_glue_destroy_qp, 1241 .modify_qp = mlx5_glue_modify_qp, 1242 .reg_mr = mlx5_glue_reg_mr, 1243 .alloc_null_mr = mlx5_glue_alloc_null_mr, 1244 .dereg_mr = mlx5_glue_dereg_mr, 1245 .create_counter_set = mlx5_glue_create_counter_set, 1246 .destroy_counter_set = mlx5_glue_destroy_counter_set, 1247 .describe_counter_set = mlx5_glue_describe_counter_set, 1248 .query_counter_set = mlx5_glue_query_counter_set, 1249 .create_counters = mlx5_glue_create_counters, 1250 .destroy_counters = mlx5_glue_destroy_counters, 1251 .attach_counters = mlx5_glue_attach_counters, 1252 .query_counters = mlx5_glue_query_counters, 1253 .ack_async_event = mlx5_glue_ack_async_event, 1254 .get_async_event = mlx5_glue_get_async_event, 1255 .port_state_str = mlx5_glue_port_state_str, 1256 .cq_ex_to_cq = mlx5_glue_cq_ex_to_cq, 1257 .dr_create_flow_action_dest_flow_tbl = 1258 mlx5_glue_dr_create_flow_action_dest_flow_tbl, 1259 .dr_create_flow_action_dest_port = 1260 mlx5_glue_dr_create_flow_action_dest_port, 1261 .dr_create_flow_action_drop = 1262 mlx5_glue_dr_create_flow_action_drop, 1263 .dr_create_flow_action_push_vlan = 1264 mlx5_glue_dr_create_flow_action_push_vlan, 1265 .dr_create_flow_action_pop_vlan = 1266 mlx5_glue_dr_create_flow_action_pop_vlan, 1267 .dr_create_flow_tbl = mlx5_glue_dr_create_flow_tbl, 1268 .dr_destroy_flow_tbl = mlx5_glue_dr_destroy_flow_tbl, 1269 .dr_create_domain = mlx5_glue_dr_create_domain, 1270 .dr_destroy_domain = mlx5_glue_dr_destroy_domain, 1271 .dv_create_cq = mlx5_glue_dv_create_cq, 1272 .dv_create_wq = mlx5_glue_dv_create_wq, 1273 .dv_query_device = mlx5_glue_dv_query_device, 1274 .dv_set_context_attr = mlx5_glue_dv_set_context_attr, 1275 .dv_init_obj = mlx5_glue_dv_init_obj, 1276 .dv_create_qp = mlx5_glue_dv_create_qp, 1277 .dv_create_flow_matcher = mlx5_glue_dv_create_flow_matcher, 1278 .dv_create_flow = mlx5_glue_dv_create_flow, 1279 .dv_create_flow_action_counter = 1280 mlx5_glue_dv_create_flow_action_counter, 1281 .dv_create_flow_action_dest_ibv_qp = 1282 mlx5_glue_dv_create_flow_action_dest_ibv_qp, 1283 .dv_create_flow_action_dest_devx_tir = 1284 mlx5_glue_dv_create_flow_action_dest_devx_tir, 1285 .dv_create_flow_action_modify_header = 1286 mlx5_glue_dv_create_flow_action_modify_header, 1287 .dv_create_flow_action_packet_reformat = 1288 mlx5_glue_dv_create_flow_action_packet_reformat, 1289 .dv_create_flow_action_tag = mlx5_glue_dv_create_flow_action_tag, 1290 .dv_create_flow_action_meter = mlx5_glue_dv_create_flow_action_meter, 1291 .dv_modify_flow_action_meter = mlx5_glue_dv_modify_flow_action_meter, 1292 .dr_create_flow_action_default_miss = 1293 mlx5_glue_dr_create_flow_action_default_miss, 1294 .dv_destroy_flow = mlx5_glue_dv_destroy_flow, 1295 .dv_destroy_flow_matcher = mlx5_glue_dv_destroy_flow_matcher, 1296 .dv_open_device = mlx5_glue_dv_open_device, 1297 .devx_obj_create = mlx5_glue_devx_obj_create, 1298 .devx_obj_destroy = mlx5_glue_devx_obj_destroy, 1299 .devx_obj_query = mlx5_glue_devx_obj_query, 1300 .devx_obj_modify = mlx5_glue_devx_obj_modify, 1301 .devx_general_cmd = mlx5_glue_devx_general_cmd, 1302 .devx_create_cmd_comp = mlx5_glue_devx_create_cmd_comp, 1303 .devx_destroy_cmd_comp = mlx5_glue_devx_destroy_cmd_comp, 1304 .devx_obj_query_async = mlx5_glue_devx_obj_query_async, 1305 .devx_get_async_cmd_comp = mlx5_glue_devx_get_async_cmd_comp, 1306 .devx_umem_reg = mlx5_glue_devx_umem_reg, 1307 .devx_umem_dereg = mlx5_glue_devx_umem_dereg, 1308 .devx_qp_query = mlx5_glue_devx_qp_query, 1309 .devx_port_query = mlx5_glue_devx_port_query, 1310 .dr_dump_domain = mlx5_glue_dr_dump_domain, 1311 .dr_reclaim_domain_memory = mlx5_glue_dr_reclaim_domain_memory, 1312 .devx_query_eqn = mlx5_glue_devx_query_eqn, 1313 .devx_create_event_channel = mlx5_glue_devx_create_event_channel, 1314 .devx_destroy_event_channel = mlx5_glue_devx_destroy_event_channel, 1315 .devx_subscribe_devx_event = mlx5_glue_devx_subscribe_devx_event, 1316 .devx_subscribe_devx_event_fd = mlx5_glue_devx_subscribe_devx_event_fd, 1317 .devx_get_event = mlx5_glue_devx_get_event, 1318 .devx_alloc_uar = mlx5_glue_devx_alloc_uar, 1319 .devx_free_uar = mlx5_glue_devx_free_uar, 1320 .dv_alloc_var = mlx5_glue_dv_alloc_var, 1321 .dv_free_var = mlx5_glue_dv_free_var, 1322 }; 1323