1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2019 Mellanox Technologies, Ltd 3 */ 4 5 #ifndef RTE_PMD_MLX5_VDPA_H_ 6 #define RTE_PMD_MLX5_VDPA_H_ 7 8 #include <linux/virtio_net.h> 9 #include <sys/queue.h> 10 11 #ifdef PEDANTIC 12 #pragma GCC diagnostic ignored "-Wpedantic" 13 #endif 14 #include <rte_vdpa.h> 15 #include <vdpa_driver.h> 16 #include <rte_vhost.h> 17 #ifdef PEDANTIC 18 #pragma GCC diagnostic error "-Wpedantic" 19 #endif 20 #include <rte_spinlock.h> 21 #include <rte_interrupts.h> 22 #include <rte_thread.h> 23 24 #include <mlx5_glue.h> 25 #include <mlx5_devx_cmds.h> 26 #include <mlx5_common_devx.h> 27 #include <mlx5_prm.h> 28 29 30 #define MLX5_VDPA_INTR_RETRIES 256 31 #define MLX5_VDPA_INTR_RETRIES_USEC 1000 32 33 #ifndef VIRTIO_F_ORDER_PLATFORM 34 #define VIRTIO_F_ORDER_PLATFORM 36 35 #endif 36 37 #ifndef VIRTIO_F_RING_PACKED 38 #define VIRTIO_F_RING_PACKED 34 39 #endif 40 41 #define MLX5_VDPA_DEFAULT_TIMER_DELAY_US 0u 42 #define MLX5_VDPA_DEFAULT_TIMER_STEP_US 1u 43 44 struct mlx5_vdpa_cq { 45 uint16_t log_desc_n; 46 uint32_t cq_ci:24; 47 uint32_t arm_sn:2; 48 uint32_t armed:1; 49 int callfd; 50 rte_spinlock_t sl; 51 struct mlx5_devx_cq cq_obj; 52 uint64_t errors; 53 }; 54 55 struct mlx5_vdpa_event_qp { 56 struct mlx5_vdpa_cq cq; 57 struct mlx5_devx_obj *fw_qp; 58 struct mlx5_devx_qp sw_qp; 59 uint16_t qp_pi; 60 }; 61 62 struct mlx5_vdpa_query_mr { 63 union { 64 struct ibv_mr *mr; 65 struct mlx5_devx_obj *mkey; 66 }; 67 int is_indirect; 68 }; 69 70 enum { 71 MLX5_VDPA_NOTIFIER_STATE_DISABLED, 72 MLX5_VDPA_NOTIFIER_STATE_ENABLED, 73 MLX5_VDPA_NOTIFIER_STATE_ERR 74 }; 75 76 #define MLX5_VDPA_USED_RING_LEN(size) \ 77 ((size) * sizeof(struct vring_used_elem) + sizeof(uint16_t) * 3) 78 #define MLX5_VDPA_MAX_C_THRD 256 79 #define MLX5_VDPA_MAX_TASKS_PER_THRD 4096 80 #define MLX5_VDPA_TASKS_PER_DEV 64 81 #define MLX5_VDPA_MAX_MRS 0xFFFF 82 83 /* Vdpa task types. */ 84 enum mlx5_vdpa_task_type { 85 MLX5_VDPA_TASK_REG_MR = 1, 86 MLX5_VDPA_TASK_SETUP_VIRTQ, 87 MLX5_VDPA_TASK_STOP_VIRTQ, 88 MLX5_VDPA_TASK_DEV_CLOSE_NOWAIT, 89 MLX5_VDPA_TASK_PREPARE_VIRTQ, 90 }; 91 92 /* Generic task information and size must be multiple of 4B. */ 93 struct __rte_aligned(4) __rte_packed_begin mlx5_vdpa_task { 94 struct mlx5_vdpa_priv *priv; 95 enum mlx5_vdpa_task_type type; 96 RTE_ATOMIC(uint32_t) *remaining_cnt; 97 RTE_ATOMIC(uint32_t) *err_cnt; 98 uint32_t idx; 99 } __rte_packed_end; 100 101 /* Generic mlx5_vdpa_c_thread information. */ 102 struct mlx5_vdpa_c_thread { 103 rte_thread_t tid; 104 struct rte_ring *rng; 105 pthread_cond_t c_cond; 106 }; 107 108 struct mlx5_vdpa_conf_thread_mng { 109 void *initializer_priv; 110 RTE_ATOMIC(uint32_t) refcnt; 111 uint32_t max_thrds; 112 pthread_mutex_t cthrd_lock; 113 struct mlx5_vdpa_c_thread cthrd[MLX5_VDPA_MAX_C_THRD]; 114 }; 115 extern struct mlx5_vdpa_conf_thread_mng conf_thread_mng; 116 117 struct mlx5_vdpa_vmem_info { 118 struct rte_vhost_memory *vmem; 119 uint32_t entries_num; 120 uint64_t gcd; 121 uint64_t size; 122 uint8_t mode; 123 }; 124 125 struct mlx5_vdpa_virtq { 126 SLIST_ENTRY(mlx5_vdpa_virtq) next; 127 uint16_t index; 128 uint16_t vq_size; 129 uint8_t notifier_state; 130 uint32_t configured:1; 131 uint32_t enable:1; 132 uint32_t stopped:1; 133 uint32_t rx_csum:1; 134 uint32_t virtio_version_1_0:1; 135 uint32_t event_mode:3; 136 uint32_t version; 137 pthread_mutex_t virtq_lock; 138 struct mlx5_vdpa_priv *priv; 139 struct mlx5_devx_obj *virtq; 140 struct mlx5_devx_obj *counters; 141 struct mlx5_vdpa_event_qp eqp; 142 struct { 143 struct mlx5dv_devx_umem *obj; 144 void *buf; 145 uint32_t size; 146 } umems[3]; 147 struct rte_intr_handle *intr_handle; 148 uint64_t err_time[3]; /* RDTSC time of recent errors. */ 149 uint32_t n_retry; 150 struct mlx5_devx_virtio_q_couners_attr stats; 151 struct mlx5_devx_virtio_q_couners_attr reset; 152 }; 153 154 struct mlx5_vdpa_steer { 155 struct mlx5_devx_obj *rqt; 156 void *domain; 157 void *tbl; 158 struct { 159 struct mlx5dv_flow_matcher *matcher; 160 struct mlx5_devx_obj *tir; 161 void *tir_action; 162 void *flow; 163 } rss[7]; 164 }; 165 166 enum { 167 MLX5_VDPA_EVENT_MODE_DYNAMIC_TIMER, 168 MLX5_VDPA_EVENT_MODE_FIXED_TIMER, 169 MLX5_VDPA_EVENT_MODE_ONLY_INTERRUPT 170 }; 171 172 enum mlx5_dev_state { 173 MLX5_VDPA_STATE_PROBED = 0, 174 MLX5_VDPA_STATE_CONFIGURED, 175 MLX5_VDPA_STATE_IN_PROGRESS /* Shutting down. */ 176 }; 177 178 struct mlx5_vdpa_priv { 179 TAILQ_ENTRY(mlx5_vdpa_priv) next; 180 bool connected; 181 bool use_c_thread; 182 enum mlx5_dev_state state; 183 rte_spinlock_t db_lock; 184 pthread_mutex_t steer_update_lock; 185 uint64_t no_traffic_counter; 186 rte_thread_t timer_tid; 187 int event_mode; 188 int event_core; /* Event thread cpu affinity core. */ 189 uint32_t event_us; 190 uint32_t timer_delay_us; 191 uint32_t no_traffic_max; 192 uint8_t hw_latency_mode; /* Hardware CQ moderation mode. */ 193 uint16_t hw_max_latency_us; /* Hardware CQ moderation period in usec. */ 194 uint16_t hw_max_pending_comp; /* Hardware CQ moderation counter. */ 195 uint16_t queue_size; /* virtq depth for pre-creating virtq resource */ 196 uint16_t queues; /* Max virtq pair for pre-creating virtq resource */ 197 struct rte_vdpa_device *vdev; /* vDPA device. */ 198 struct mlx5_common_device *cdev; /* Backend mlx5 device. */ 199 int vid; /* vhost device id. */ 200 struct mlx5_hca_vdpa_attr caps; 201 uint32_t gpa_mkey_index; 202 struct ibv_mr *null_mr; 203 struct mlx5_vdpa_vmem_info vmem_info; 204 struct mlx5dv_devx_event_channel *eventc; 205 struct mlx5dv_devx_event_channel *err_chnl; 206 struct mlx5_uar uar; 207 struct rte_intr_handle *err_intr_handle; 208 struct mlx5_devx_obj *td; 209 struct mlx5_devx_obj *tiss[16]; /* TIS list for each LAG port. */ 210 uint16_t nr_virtqs; 211 uint8_t num_lag_ports; 212 uint64_t features; /* Negotiated features. */ 213 uint16_t log_max_rqt_size; 214 uint16_t last_c_thrd_idx; 215 RTE_ATOMIC(uint16_t) dev_close_progress; 216 uint16_t num_mrs; /* Number of memory regions. */ 217 struct mlx5_vdpa_steer steer; 218 struct mlx5dv_var *var; 219 void *virtq_db_addr; 220 struct mlx5_pmd_wrapped_mr lm_mr; 221 struct mlx5_vdpa_query_mr **mrs; 222 struct mlx5_vdpa_virtq virtqs[]; 223 }; 224 225 enum { 226 MLX5_VDPA_STATS_RECEIVED_DESCRIPTORS, 227 MLX5_VDPA_STATS_COMPLETED_DESCRIPTORS, 228 MLX5_VDPA_STATS_BAD_DESCRIPTOR_ERRORS, 229 MLX5_VDPA_STATS_EXCEED_MAX_CHAIN, 230 MLX5_VDPA_STATS_INVALID_BUFFER, 231 MLX5_VDPA_STATS_COMPLETION_ERRORS, 232 MLX5_VDPA_STATS_MAX 233 }; 234 235 /* 236 * Check whether virtq is for traffic receive. 237 * According to VIRTIO_NET Spec the virtqueues index identity its type by: 238 * 0 receiveq1 239 * 1 transmitq1 240 * ... 241 * 2(N-1) receiveqN 242 * 2(N-1)+1 transmitqN 243 * 2N controlq 244 */ 245 static inline uint8_t 246 is_virtq_recvq(int virtq_index, int nr_vring) 247 { 248 if (virtq_index % 2 == 0 && virtq_index != nr_vring - 1) 249 return 1; 250 return 0; 251 } 252 253 /** 254 * Release all the prepared memory regions and all their related resources. 255 * 256 * @param[in] priv 257 * The vdpa driver private structure. 258 */ 259 void mlx5_vdpa_mem_dereg(struct mlx5_vdpa_priv *priv); 260 261 /** 262 * Register all the memory regions of the virtio device to the HW and allocate 263 * all their related resources. 264 * 265 * @param[in] priv 266 * The vdpa driver private structure. 267 * 268 * @return 269 * 0 on success, a negative errno value otherwise and rte_errno is set. 270 */ 271 int mlx5_vdpa_mem_register(struct mlx5_vdpa_priv *priv); 272 273 274 /** 275 * Create an event QP and all its related resources. 276 * 277 * @param[in] priv 278 * The vdpa driver private structure. 279 * @param[in] desc_n 280 * Number of descriptors. 281 * @param[in] callfd 282 * The guest notification file descriptor. 283 * @param[in/out] virtq 284 * Pointer to the virt-queue structure. 285 * @param[in] reset 286 * If true, it will reset event qp. 287 * 288 * @return 289 * 0 on success, -1 otherwise and rte_errno is set. 290 */ 291 int 292 mlx5_vdpa_event_qp_prepare(struct mlx5_vdpa_priv *priv, uint16_t desc_n, 293 int callfd, struct mlx5_vdpa_virtq *virtq, bool reset); 294 295 /** 296 * Destroy an event QP and all its related resources. 297 * 298 * @param[in/out] eqp 299 * Pointer to the event QP structure. 300 */ 301 void mlx5_vdpa_event_qp_destroy(struct mlx5_vdpa_event_qp *eqp); 302 303 /** 304 * Create all the event global resources. 305 * 306 * @param[in] priv 307 * The vdpa driver private structure. 308 */ 309 int 310 mlx5_vdpa_event_qp_global_prepare(struct mlx5_vdpa_priv *priv); 311 312 /** 313 * Release all the event global resources. 314 * 315 * @param[in] priv 316 * The vdpa driver private structure. 317 */ 318 void mlx5_vdpa_event_qp_global_release(struct mlx5_vdpa_priv *priv); 319 320 /** 321 * Setup CQE event. 322 * 323 * @param[in] priv 324 * The vdpa driver private structure. 325 * 326 * @return 327 * 0 on success, a negative errno value otherwise and rte_errno is set. 328 */ 329 int mlx5_vdpa_cqe_event_setup(struct mlx5_vdpa_priv *priv); 330 331 /** 332 * Unset CQE event . 333 * 334 * @param[in] priv 335 * The vdpa driver private structure. 336 */ 337 void mlx5_vdpa_cqe_event_unset(struct mlx5_vdpa_priv *priv); 338 339 /** 340 * Setup error interrupt handler. 341 * 342 * @param[in] priv 343 * The vdpa driver private structure. 344 * 345 * @return 346 * 0 on success, a negative errno value otherwise and rte_errno is set. 347 */ 348 int mlx5_vdpa_err_event_setup(struct mlx5_vdpa_priv *priv); 349 350 /** 351 * Unset error event handler. 352 * 353 * @param[in] priv 354 * The vdpa driver private structure. 355 */ 356 void mlx5_vdpa_err_event_unset(struct mlx5_vdpa_priv *priv); 357 358 /** 359 * Release virtqs and resources except that to be reused. 360 * 361 * @param[in] priv 362 * The vdpa driver private structure. 363 * @param[in] release_resource 364 * The vdpa driver release resource without prepare resource. 365 */ 366 void 367 mlx5_vdpa_virtqs_release(struct mlx5_vdpa_priv *priv, 368 bool release_resource); 369 370 /** 371 * Cleanup cached resources of all virtqs. 372 * 373 * @param[in] priv 374 * The vdpa driver private structure. 375 */ 376 void mlx5_vdpa_virtqs_cleanup(struct mlx5_vdpa_priv *priv); 377 378 /** 379 * Create all the HW virtqs resources and all their related resources. 380 * 381 * @param[in] priv 382 * The vdpa driver private structure. 383 * 384 * @return 385 * 0 on success, a negative errno value otherwise and rte_errno is set. 386 */ 387 int mlx5_vdpa_virtqs_prepare(struct mlx5_vdpa_priv *priv); 388 389 /** 390 * Enable\Disable virtq.. 391 * 392 * @param[in] priv 393 * The vdpa driver private structure. 394 * @param[in] index 395 * The virtq index. 396 * @param[in] enable 397 * Set to enable, otherwise disable. 398 * 399 * @return 400 * 0 on success, a negative value otherwise. 401 */ 402 int mlx5_vdpa_virtq_enable(struct mlx5_vdpa_priv *priv, int index, int enable); 403 404 /** 405 * Unset steering - stop traffic. 406 * 407 * @param[in] priv 408 * The vdpa driver private structure. 409 */ 410 void mlx5_vdpa_steer_unset(struct mlx5_vdpa_priv *priv); 411 412 /** 413 * Update steering according to the received queues status. 414 * 415 * @param[in] priv 416 * The vdpa driver private structure. 417 * @param[in] is_dummy 418 * If set, it is updated with dummy queue for prepare resource. 419 * 420 * @return 421 * 0 on success, a negative value otherwise. 422 */ 423 int mlx5_vdpa_steer_update(struct mlx5_vdpa_priv *priv, bool is_dummy); 424 425 /** 426 * Setup steering and all its related resources to enable RSS traffic from the 427 * device to all the Rx host queues. 428 * 429 * @param[in] priv 430 * The vdpa driver private structure. 431 * 432 * @return 433 * 0 on success, a negative value otherwise. 434 */ 435 int mlx5_vdpa_steer_setup(struct mlx5_vdpa_priv *priv); 436 437 /** 438 * Enable\Disable live migration logging. 439 * 440 * @param[in] priv 441 * The vdpa driver private structure. 442 * @param[in] enable 443 * Set for enable, unset for disable. 444 * 445 * @return 446 * 0 on success, a negative value otherwise. 447 */ 448 int mlx5_vdpa_logging_enable(struct mlx5_vdpa_priv *priv, int enable); 449 450 /** 451 * Set dirty bitmap logging to allow live migration. 452 * 453 * @param[in] priv 454 * The vdpa driver private structure. 455 * @param[in] log_base 456 * Vhost log base. 457 * @param[in] log_size 458 * Vhost log size. 459 * 460 * @return 461 * 0 on success, a negative value otherwise. 462 */ 463 int mlx5_vdpa_dirty_bitmap_set(struct mlx5_vdpa_priv *priv, uint64_t log_base, 464 uint64_t log_size); 465 466 /** 467 * Log all virtqs information for live migration. 468 * 469 * @param[in] priv 470 * The vdpa driver private structure. 471 * @param[in] enable 472 * Set for enable, unset for disable. 473 * 474 * @return 475 * 0 on success, a negative value otherwise. 476 */ 477 int mlx5_vdpa_lm_log(struct mlx5_vdpa_priv *priv); 478 479 /** 480 * Modify virtq state to be ready or suspend. 481 * 482 * @param[in] virtq 483 * The vdpa driver private virtq structure. 484 * @param[in] state 485 * Set for ready, otherwise suspend. 486 * 487 * @return 488 * 0 on success, a negative value otherwise. 489 */ 490 int mlx5_vdpa_virtq_modify(struct mlx5_vdpa_virtq *virtq, int state); 491 492 /** 493 * Stop virtq before destroying it. 494 * 495 * @param[in] priv 496 * The vdpa driver private structure. 497 * @param[in] index 498 * The virtq index. 499 * 500 * @return 501 * 0 on success, a negative value otherwise. 502 */ 503 int mlx5_vdpa_virtq_stop(struct mlx5_vdpa_priv *priv, int index); 504 505 /** 506 * Query virtq information. 507 * 508 * @param[in] priv 509 * The vdpa driver private structure. 510 * @param[in] index 511 * The virtq index. 512 * 513 * @return 514 * 0 on success, a negative value otherwise. 515 */ 516 int mlx5_vdpa_virtq_query(struct mlx5_vdpa_priv *priv, int index); 517 518 /** 519 * Get virtq statistics. 520 * 521 * @param[in] priv 522 * The vdpa driver private structure. 523 * @param[in] qid 524 * The virtq index. 525 * @param stats 526 * The virtq statistics array to fill. 527 * @param n 528 * The number of elements in @p stats array. 529 * 530 * @return 531 * A negative value on error, otherwise the number of entries filled in the 532 * @p stats array. 533 */ 534 int 535 mlx5_vdpa_virtq_stats_get(struct mlx5_vdpa_priv *priv, int qid, 536 struct rte_vdpa_stat *stats, unsigned int n); 537 538 /** 539 * Reset virtq statistics. 540 * 541 * @param[in] priv 542 * The vdpa driver private structure. 543 * @param[in] qid 544 * The virtq index. 545 * 546 * @return 547 * A negative value on error, otherwise 0. 548 */ 549 int 550 mlx5_vdpa_virtq_stats_reset(struct mlx5_vdpa_priv *priv, int qid); 551 552 /** 553 * Drain virtq CQ CQE. 554 * 555 * @param[in] priv 556 * The vdpa driver private structure. 557 */ 558 void 559 mlx5_vdpa_drain_cq(struct mlx5_vdpa_priv *priv); 560 561 bool 562 mlx5_vdpa_is_modify_virtq_supported(struct mlx5_vdpa_priv *priv); 563 564 /** 565 * Create configuration multi-threads resource 566 * 567 * @return 568 * 0 on success, a negative value otherwise. 569 */ 570 int 571 mlx5_vdpa_mult_threads_create(void); 572 573 /** 574 * Destroy configuration multi-threads resource 575 * 576 */ 577 void 578 mlx5_vdpa_mult_threads_destroy(bool need_unlock); 579 580 bool 581 mlx5_vdpa_task_add(struct mlx5_vdpa_priv *priv, 582 uint32_t thrd_idx, 583 enum mlx5_vdpa_task_type task_type, 584 RTE_ATOMIC(uint32_t) *remaining_cnt, RTE_ATOMIC(uint32_t) *err_cnt, 585 void **task_data, uint32_t num); 586 int 587 mlx5_vdpa_register_mr(struct mlx5_vdpa_priv *priv, uint32_t idx); 588 bool 589 mlx5_vdpa_c_thread_wait_bulk_tasks_done(RTE_ATOMIC(uint32_t) *remaining_cnt, 590 RTE_ATOMIC(uint32_t) *err_cnt, uint32_t sleep_time); 591 int 592 mlx5_vdpa_virtq_setup(struct mlx5_vdpa_priv *priv, int index, bool reg_kick); 593 void 594 mlx5_vdpa_dev_cache_clean(struct mlx5_vdpa_priv *priv); 595 void 596 mlx5_vdpa_virtq_unreg_intr_handle_all(struct mlx5_vdpa_priv *priv); 597 bool 598 mlx5_vdpa_virtq_single_resource_prepare(struct mlx5_vdpa_priv *priv, 599 int index); 600 int 601 mlx5_vdpa_qps2rst2rts(struct mlx5_vdpa_event_qp *eqp); 602 void 603 mlx5_vdpa_virtq_unset(struct mlx5_vdpa_virtq *virtq); 604 void 605 mlx5_vdpa_prepare_virtq_destroy(struct mlx5_vdpa_priv *priv); 606 #endif /* RTE_PMD_MLX5_VDPA_H_ */ 607