1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2021 Mellanox Technologies, Ltd 3 */ 4 5 #include <rte_malloc.h> 6 #include <rte_log.h> 7 #include <rte_errno.h> 8 #include <rte_pci.h> 9 #include <rte_spinlock.h> 10 #include <rte_comp.h> 11 #include <rte_compressdev.h> 12 #include <rte_compressdev_pmd.h> 13 14 #include <mlx5_glue.h> 15 #include <mlx5_common.h> 16 #include <mlx5_common_pci.h> 17 #include <mlx5_devx_cmds.h> 18 #include <mlx5_common_os.h> 19 #include <mlx5_common_devx.h> 20 #include <mlx5_common_mr.h> 21 #include <mlx5_prm.h> 22 23 #include "mlx5_compress_utils.h" 24 25 #define MLX5_COMPRESS_DRIVER_NAME mlx5_compress 26 #define MLX5_COMPRESS_LOG_NAME pmd.compress.mlx5 27 #define MLX5_COMPRESS_MAX_QPS 1024 28 #define MLX5_COMP_MAX_WIN_SIZE_CONF 6u 29 30 struct mlx5_compress_xform { 31 LIST_ENTRY(mlx5_compress_xform) next; 32 enum rte_comp_xform_type type; 33 enum rte_comp_checksum_type csum_type; 34 uint32_t opcode; 35 uint32_t gga_ctrl1; /* BE. */ 36 }; 37 38 struct mlx5_compress_priv { 39 TAILQ_ENTRY(mlx5_compress_priv) next; 40 struct ibv_context *ctx; /* Device context. */ 41 struct rte_pci_device *pci_dev; 42 struct rte_compressdev *cdev; 43 void *uar; 44 uint32_t pdn; /* Protection Domain number. */ 45 uint8_t min_block_size; 46 /* Minimum huffman block size supported by the device. */ 47 struct ibv_pd *pd; 48 struct rte_compressdev_config dev_config; 49 LIST_HEAD(xform_list, mlx5_compress_xform) xform_list; 50 rte_spinlock_t xform_sl; 51 struct mlx5_mr_share_cache mr_scache; /* Global shared MR cache. */ 52 volatile uint64_t *uar_addr; 53 #ifndef RTE_ARCH_64 54 rte_spinlock_t uar32_sl; 55 #endif /* RTE_ARCH_64 */ 56 }; 57 58 struct mlx5_compress_qp { 59 uint16_t qp_id; 60 uint16_t entries_n; 61 uint16_t pi; 62 uint16_t ci; 63 struct mlx5_mr_ctrl mr_ctrl; 64 int socket_id; 65 struct mlx5_devx_cq cq; 66 struct mlx5_devx_sq sq; 67 struct mlx5_pmd_mr opaque_mr; 68 struct rte_comp_op **ops; 69 struct mlx5_compress_priv *priv; 70 struct rte_compressdev_stats stats; 71 }; 72 73 TAILQ_HEAD(mlx5_compress_privs, mlx5_compress_priv) mlx5_compress_priv_list = 74 TAILQ_HEAD_INITIALIZER(mlx5_compress_priv_list); 75 static pthread_mutex_t priv_list_lock = PTHREAD_MUTEX_INITIALIZER; 76 77 int mlx5_compress_logtype; 78 79 const struct rte_compressdev_capabilities mlx5_caps[RTE_COMP_ALGO_LIST_END]; 80 81 82 static void 83 mlx5_compress_dev_info_get(struct rte_compressdev *dev, 84 struct rte_compressdev_info *info) 85 { 86 RTE_SET_USED(dev); 87 if (info != NULL) { 88 info->max_nb_queue_pairs = MLX5_COMPRESS_MAX_QPS; 89 info->feature_flags = RTE_COMPDEV_FF_HW_ACCELERATED; 90 info->capabilities = mlx5_caps; 91 } 92 } 93 94 static int 95 mlx5_compress_dev_configure(struct rte_compressdev *dev, 96 struct rte_compressdev_config *config) 97 { 98 struct mlx5_compress_priv *priv; 99 100 if (dev == NULL || config == NULL) 101 return -EINVAL; 102 priv = dev->data->dev_private; 103 priv->dev_config = *config; 104 return 0; 105 } 106 107 static int 108 mlx5_compress_dev_close(struct rte_compressdev *dev) 109 { 110 RTE_SET_USED(dev); 111 return 0; 112 } 113 114 static int 115 mlx5_compress_qp_release(struct rte_compressdev *dev, uint16_t qp_id) 116 { 117 struct mlx5_compress_qp *qp = dev->data->queue_pairs[qp_id]; 118 119 if (qp->sq.sq != NULL) 120 mlx5_devx_sq_destroy(&qp->sq); 121 if (qp->cq.cq != NULL) 122 mlx5_devx_cq_destroy(&qp->cq); 123 if (qp->opaque_mr.obj != NULL) { 124 void *opaq = qp->opaque_mr.addr; 125 126 mlx5_common_verbs_dereg_mr(&qp->opaque_mr); 127 if (opaq != NULL) 128 rte_free(opaq); 129 } 130 mlx5_mr_btree_free(&qp->mr_ctrl.cache_bh); 131 rte_free(qp); 132 dev->data->queue_pairs[qp_id] = NULL; 133 return 0; 134 } 135 136 static void 137 mlx5_compress_init_sq(struct mlx5_compress_qp *qp) 138 { 139 volatile struct mlx5_gga_wqe *restrict wqe = 140 (volatile struct mlx5_gga_wqe *)qp->sq.wqes; 141 volatile struct mlx5_gga_compress_opaque *opaq = qp->opaque_mr.addr; 142 const uint32_t sq_ds = rte_cpu_to_be_32((qp->sq.sq->id << 8) | 4u); 143 const uint32_t flags = RTE_BE32(MLX5_COMP_ALWAYS << 144 MLX5_COMP_MODE_OFFSET); 145 const uint32_t opaq_lkey = rte_cpu_to_be_32(qp->opaque_mr.lkey); 146 int i; 147 148 /* All the next fields state should stay constant. */ 149 for (i = 0; i < qp->entries_n; ++i, ++wqe) { 150 wqe->sq_ds = sq_ds; 151 wqe->flags = flags; 152 wqe->opaque_lkey = opaq_lkey; 153 wqe->opaque_vaddr = rte_cpu_to_be_64 154 ((uint64_t)(uintptr_t)&opaq[i]); 155 } 156 } 157 158 static int 159 mlx5_compress_qp_setup(struct rte_compressdev *dev, uint16_t qp_id, 160 uint32_t max_inflight_ops, int socket_id) 161 { 162 struct mlx5_compress_priv *priv = dev->data->dev_private; 163 struct mlx5_compress_qp *qp; 164 struct mlx5_devx_cq_attr cq_attr = { 165 .uar_page_id = mlx5_os_get_devx_uar_page_id(priv->uar), 166 }; 167 struct mlx5_devx_create_sq_attr sq_attr = { 168 .user_index = qp_id, 169 .wq_attr = (struct mlx5_devx_wq_attr){ 170 .pd = priv->pdn, 171 .uar_page = mlx5_os_get_devx_uar_page_id(priv->uar), 172 }, 173 }; 174 struct mlx5_devx_modify_sq_attr modify_attr = { 175 .state = MLX5_SQC_STATE_RDY, 176 }; 177 uint32_t log_ops_n = rte_log2_u32(max_inflight_ops); 178 uint32_t alloc_size = sizeof(*qp); 179 void *opaq_buf; 180 int ret; 181 182 alloc_size = RTE_ALIGN(alloc_size, RTE_CACHE_LINE_SIZE); 183 alloc_size += sizeof(struct rte_comp_op *) * (1u << log_ops_n); 184 qp = rte_zmalloc_socket(__func__, alloc_size, RTE_CACHE_LINE_SIZE, 185 socket_id); 186 if (qp == NULL) { 187 DRV_LOG(ERR, "Failed to allocate qp memory."); 188 rte_errno = ENOMEM; 189 return -rte_errno; 190 } 191 dev->data->queue_pairs[qp_id] = qp; 192 opaq_buf = rte_calloc(__func__, 1u << log_ops_n, 193 sizeof(struct mlx5_gga_compress_opaque), 194 sizeof(struct mlx5_gga_compress_opaque)); 195 if (opaq_buf == NULL) { 196 DRV_LOG(ERR, "Failed to allocate opaque memory."); 197 rte_errno = ENOMEM; 198 goto err; 199 } 200 if (mlx5_mr_btree_init(&qp->mr_ctrl.cache_bh, MLX5_MR_BTREE_CACHE_N, 201 priv->dev_config.socket_id)) { 202 DRV_LOG(ERR, "Cannot allocate MR Btree for qp %u.", 203 (uint32_t)qp_id); 204 rte_errno = ENOMEM; 205 goto err; 206 } 207 qp->entries_n = 1 << log_ops_n; 208 qp->socket_id = socket_id; 209 qp->qp_id = qp_id; 210 qp->priv = priv; 211 qp->ops = (struct rte_comp_op **)RTE_ALIGN((uintptr_t)(qp + 1), 212 RTE_CACHE_LINE_SIZE); 213 if (mlx5_common_verbs_reg_mr(priv->pd, opaq_buf, qp->entries_n * 214 sizeof(struct mlx5_gga_compress_opaque), 215 &qp->opaque_mr) != 0) { 216 rte_free(opaq_buf); 217 DRV_LOG(ERR, "Failed to register opaque MR."); 218 rte_errno = ENOMEM; 219 goto err; 220 } 221 ret = mlx5_devx_cq_create(priv->ctx, &qp->cq, log_ops_n, &cq_attr, 222 socket_id); 223 if (ret != 0) { 224 DRV_LOG(ERR, "Failed to create CQ."); 225 goto err; 226 } 227 sq_attr.cqn = qp->cq.cq->id; 228 ret = mlx5_devx_sq_create(priv->ctx, &qp->sq, log_ops_n, &sq_attr, 229 socket_id); 230 if (ret != 0) { 231 DRV_LOG(ERR, "Failed to create SQ."); 232 goto err; 233 } 234 mlx5_compress_init_sq(qp); 235 ret = mlx5_devx_cmd_modify_sq(qp->sq.sq, &modify_attr); 236 if (ret != 0) { 237 DRV_LOG(ERR, "Can't change SQ state to ready."); 238 goto err; 239 } 240 DRV_LOG(INFO, "QP %u: SQN=0x%X CQN=0x%X entries num = %u\n", 241 (uint32_t)qp_id, qp->sq.sq->id, qp->cq.cq->id, qp->entries_n); 242 return 0; 243 err: 244 mlx5_compress_qp_release(dev, qp_id); 245 return -1; 246 } 247 248 static int 249 mlx5_compress_xform_free(struct rte_compressdev *dev, void *xform) 250 { 251 struct mlx5_compress_priv *priv = dev->data->dev_private; 252 253 rte_spinlock_lock(&priv->xform_sl); 254 LIST_REMOVE((struct mlx5_compress_xform *)xform, next); 255 rte_spinlock_unlock(&priv->xform_sl); 256 rte_free(xform); 257 return 0; 258 } 259 260 static int 261 mlx5_compress_xform_create(struct rte_compressdev *dev, 262 const struct rte_comp_xform *xform, 263 void **private_xform) 264 { 265 struct mlx5_compress_priv *priv = dev->data->dev_private; 266 struct mlx5_compress_xform *xfrm; 267 uint32_t size; 268 269 if (xform->type == RTE_COMP_COMPRESS && xform->compress.level == 270 RTE_COMP_LEVEL_NONE) { 271 DRV_LOG(ERR, "Non-compressed block is not supported."); 272 return -ENOTSUP; 273 } 274 if ((xform->type == RTE_COMP_COMPRESS && xform->compress.hash_algo != 275 RTE_COMP_HASH_ALGO_NONE) || (xform->type == RTE_COMP_DECOMPRESS && 276 xform->decompress.hash_algo != RTE_COMP_HASH_ALGO_NONE)) { 277 DRV_LOG(ERR, "SHA is not supported."); 278 return -ENOTSUP; 279 } 280 xfrm = rte_zmalloc_socket(__func__, sizeof(*xfrm), 0, 281 priv->dev_config.socket_id); 282 if (xfrm == NULL) 283 return -ENOMEM; 284 xfrm->opcode = MLX5_OPCODE_MMO; 285 xfrm->type = xform->type; 286 switch (xform->type) { 287 case RTE_COMP_COMPRESS: 288 switch (xform->compress.algo) { 289 case RTE_COMP_ALGO_NULL: 290 xfrm->opcode += MLX5_OPC_MOD_MMO_DMA << 291 WQE_CSEG_OPC_MOD_OFFSET; 292 break; 293 case RTE_COMP_ALGO_DEFLATE: 294 size = 1 << xform->compress.window_size; 295 size /= MLX5_GGA_COMP_WIN_SIZE_UNITS; 296 xfrm->gga_ctrl1 += RTE_MIN(rte_log2_u32(size), 297 MLX5_COMP_MAX_WIN_SIZE_CONF) << 298 WQE_GGA_COMP_WIN_SIZE_OFFSET; 299 if (xform->compress.level == RTE_COMP_LEVEL_PMD_DEFAULT) 300 size = MLX5_GGA_COMP_LOG_BLOCK_SIZE_MAX; 301 else 302 size = priv->min_block_size - 1 + 303 xform->compress.level; 304 xfrm->gga_ctrl1 += RTE_MIN(size, 305 MLX5_GGA_COMP_LOG_BLOCK_SIZE_MAX) << 306 WQE_GGA_COMP_BLOCK_SIZE_OFFSET; 307 xfrm->opcode += MLX5_OPC_MOD_MMO_COMP << 308 WQE_CSEG_OPC_MOD_OFFSET; 309 size = xform->compress.deflate.huffman == 310 RTE_COMP_HUFFMAN_DYNAMIC ? 311 MLX5_GGA_COMP_LOG_DYNAMIC_SIZE_MAX : 312 MLX5_GGA_COMP_LOG_DYNAMIC_SIZE_MIN; 313 xfrm->gga_ctrl1 += size << 314 WQE_GGA_COMP_DYNAMIC_SIZE_OFFSET; 315 break; 316 default: 317 goto err; 318 } 319 xfrm->csum_type = xform->compress.chksum; 320 break; 321 case RTE_COMP_DECOMPRESS: 322 switch (xform->decompress.algo) { 323 case RTE_COMP_ALGO_NULL: 324 xfrm->opcode += MLX5_OPC_MOD_MMO_DMA << 325 WQE_CSEG_OPC_MOD_OFFSET; 326 break; 327 case RTE_COMP_ALGO_DEFLATE: 328 xfrm->opcode += MLX5_OPC_MOD_MMO_DECOMP << 329 WQE_CSEG_OPC_MOD_OFFSET; 330 break; 331 default: 332 goto err; 333 } 334 xfrm->csum_type = xform->decompress.chksum; 335 break; 336 default: 337 DRV_LOG(ERR, "Algorithm %u is not supported.", xform->type); 338 goto err; 339 } 340 DRV_LOG(DEBUG, "New xform: gga ctrl1 = 0x%08X opcode = 0x%08X csum " 341 "type = %d.", xfrm->gga_ctrl1, xfrm->opcode, xfrm->csum_type); 342 xfrm->gga_ctrl1 = rte_cpu_to_be_32(xfrm->gga_ctrl1); 343 rte_spinlock_lock(&priv->xform_sl); 344 LIST_INSERT_HEAD(&priv->xform_list, xfrm, next); 345 rte_spinlock_unlock(&priv->xform_sl); 346 *private_xform = xfrm; 347 return 0; 348 err: 349 rte_free(xfrm); 350 return -ENOTSUP; 351 } 352 353 static void 354 mlx5_compress_dev_stop(struct rte_compressdev *dev) 355 { 356 RTE_SET_USED(dev); 357 } 358 359 static int 360 mlx5_compress_dev_start(struct rte_compressdev *dev) 361 { 362 RTE_SET_USED(dev); 363 return 0; 364 } 365 366 static void 367 mlx5_compress_stats_get(struct rte_compressdev *dev, 368 struct rte_compressdev_stats *stats) 369 { 370 int qp_id; 371 372 for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) { 373 struct mlx5_compress_qp *qp = dev->data->queue_pairs[qp_id]; 374 375 stats->enqueued_count += qp->stats.enqueued_count; 376 stats->dequeued_count += qp->stats.dequeued_count; 377 stats->enqueue_err_count += qp->stats.enqueue_err_count; 378 stats->dequeue_err_count += qp->stats.dequeue_err_count; 379 } 380 } 381 382 static void 383 mlx5_compress_stats_reset(struct rte_compressdev *dev) 384 { 385 int qp_id; 386 387 for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) { 388 struct mlx5_compress_qp *qp = dev->data->queue_pairs[qp_id]; 389 390 memset(&qp->stats, 0, sizeof(qp->stats)); 391 } 392 } 393 394 static struct rte_compressdev_ops mlx5_compress_ops = { 395 .dev_configure = mlx5_compress_dev_configure, 396 .dev_start = mlx5_compress_dev_start, 397 .dev_stop = mlx5_compress_dev_stop, 398 .dev_close = mlx5_compress_dev_close, 399 .dev_infos_get = mlx5_compress_dev_info_get, 400 .stats_get = mlx5_compress_stats_get, 401 .stats_reset = mlx5_compress_stats_reset, 402 .queue_pair_setup = mlx5_compress_qp_setup, 403 .queue_pair_release = mlx5_compress_qp_release, 404 .private_xform_create = mlx5_compress_xform_create, 405 .private_xform_free = mlx5_compress_xform_free, 406 .stream_create = NULL, 407 .stream_free = NULL, 408 }; 409 410 static __rte_always_inline uint32_t 411 mlx5_compress_dseg_set(struct mlx5_compress_qp *qp, 412 volatile struct mlx5_wqe_dseg *restrict dseg, 413 struct rte_mbuf *restrict mbuf, 414 uint32_t offset, uint32_t len) 415 { 416 uintptr_t addr = rte_pktmbuf_mtod_offset(mbuf, uintptr_t, offset); 417 418 dseg->bcount = rte_cpu_to_be_32(len); 419 dseg->lkey = mlx5_mr_addr2mr_bh(qp->priv->pd, 0, &qp->priv->mr_scache, 420 &qp->mr_ctrl, addr, 421 !!(mbuf->ol_flags & EXT_ATTACHED_MBUF)); 422 dseg->pbuf = rte_cpu_to_be_64(addr); 423 return dseg->lkey; 424 } 425 426 /* 427 * Provide safe 64bit store operation to mlx5 UAR region for both 32bit and 428 * 64bit architectures. 429 */ 430 static __rte_always_inline void 431 mlx5_compress_uar_write(uint64_t val, struct mlx5_compress_priv *priv) 432 { 433 #ifdef RTE_ARCH_64 434 *priv->uar_addr = val; 435 #else /* !RTE_ARCH_64 */ 436 rte_spinlock_lock(&priv->uar32_sl); 437 *(volatile uint32_t *)priv->uar_addr = val; 438 rte_io_wmb(); 439 *((volatile uint32_t *)priv->uar_addr + 1) = val >> 32; 440 rte_spinlock_unlock(&priv->uar32_sl); 441 #endif 442 } 443 444 static uint16_t 445 mlx5_compress_enqueue_burst(void *queue_pair, struct rte_comp_op **ops, 446 uint16_t nb_ops) 447 { 448 struct mlx5_compress_qp *qp = queue_pair; 449 volatile struct mlx5_gga_wqe *wqes = (volatile struct mlx5_gga_wqe *) 450 qp->sq.wqes, *wqe; 451 struct mlx5_compress_xform *xform; 452 struct rte_comp_op *op; 453 uint16_t mask = qp->entries_n - 1; 454 uint16_t remain = qp->entries_n - (qp->pi - qp->ci); 455 uint16_t idx; 456 bool invalid; 457 458 if (remain < nb_ops) 459 nb_ops = remain; 460 else 461 remain = nb_ops; 462 if (unlikely(remain == 0)) 463 return 0; 464 do { 465 idx = qp->pi & mask; 466 wqe = &wqes[idx]; 467 rte_prefetch0(&wqes[(qp->pi + 1) & mask]); 468 op = *ops++; 469 xform = op->private_xform; 470 /* 471 * Check operation arguments and error cases: 472 * - Operation type must be state-less. 473 * - Compress operation flush flag must be FULL or FINAL. 474 * - Source and destination buffers must be mapped internally. 475 */ 476 invalid = op->op_type != RTE_COMP_OP_STATELESS || 477 (xform->type == RTE_COMP_COMPRESS && 478 op->flush_flag < RTE_COMP_FLUSH_FULL); 479 if (unlikely(invalid || 480 (mlx5_compress_dseg_set(qp, &wqe->gather, 481 op->m_src, 482 op->src.offset, 483 op->src.length) == 484 UINT32_MAX) || 485 (mlx5_compress_dseg_set(qp, &wqe->scatter, 486 op->m_dst, 487 op->dst.offset, 488 rte_pktmbuf_pkt_len(op->m_dst) - 489 op->dst.offset) == 490 UINT32_MAX))) { 491 op->status = invalid ? RTE_COMP_OP_STATUS_INVALID_ARGS : 492 RTE_COMP_OP_STATUS_ERROR; 493 nb_ops -= remain; 494 if (unlikely(nb_ops == 0)) 495 return 0; 496 break; 497 } 498 wqe->gga_ctrl1 = xform->gga_ctrl1; 499 wqe->opcode = rte_cpu_to_be_32(xform->opcode + (qp->pi << 8)); 500 qp->ops[idx] = op; 501 qp->pi++; 502 } while (--remain); 503 qp->stats.enqueued_count += nb_ops; 504 rte_io_wmb(); 505 qp->sq.db_rec[MLX5_SND_DBR] = rte_cpu_to_be_32(qp->pi); 506 rte_wmb(); 507 mlx5_compress_uar_write(*(volatile uint64_t *)wqe, qp->priv); 508 rte_wmb(); 509 return nb_ops; 510 } 511 512 static void 513 mlx5_compress_dump_err_objs(volatile uint32_t *cqe, volatile uint32_t *wqe, 514 volatile uint32_t *opaq) 515 { 516 size_t i; 517 518 DRV_LOG(ERR, "Error cqe:"); 519 for (i = 0; i < sizeof(struct mlx5_err_cqe) >> 2; i += 4) 520 DRV_LOG(ERR, "%08X %08X %08X %08X", cqe[i], cqe[i + 1], 521 cqe[i + 2], cqe[i + 3]); 522 DRV_LOG(ERR, "\nError wqe:"); 523 for (i = 0; i < sizeof(struct mlx5_gga_wqe) >> 2; i += 4) 524 DRV_LOG(ERR, "%08X %08X %08X %08X", wqe[i], wqe[i + 1], 525 wqe[i + 2], wqe[i + 3]); 526 DRV_LOG(ERR, "\nError opaq:"); 527 for (i = 0; i < sizeof(struct mlx5_gga_compress_opaque) >> 2; i += 4) 528 DRV_LOG(ERR, "%08X %08X %08X %08X", opaq[i], opaq[i + 1], 529 opaq[i + 2], opaq[i + 3]); 530 } 531 532 static void 533 mlx5_compress_cqe_err_handle(struct mlx5_compress_qp *qp, 534 struct rte_comp_op *op) 535 { 536 const uint32_t idx = qp->ci & (qp->entries_n - 1); 537 volatile struct mlx5_err_cqe *cqe = (volatile struct mlx5_err_cqe *) 538 &qp->cq.cqes[idx]; 539 volatile struct mlx5_gga_wqe *wqes = (volatile struct mlx5_gga_wqe *) 540 qp->sq.wqes; 541 volatile struct mlx5_gga_compress_opaque *opaq = qp->opaque_mr.addr; 542 543 op->status = RTE_COMP_OP_STATUS_ERROR; 544 op->consumed = 0; 545 op->produced = 0; 546 op->output_chksum = 0; 547 op->debug_status = rte_be_to_cpu_32(opaq[idx].syndrom) | 548 ((uint64_t)rte_be_to_cpu_32(cqe->syndrome) << 32); 549 mlx5_compress_dump_err_objs((volatile uint32_t *)cqe, 550 (volatile uint32_t *)&wqes[idx], 551 (volatile uint32_t *)&opaq[idx]); 552 qp->stats.dequeue_err_count++; 553 } 554 555 static uint16_t 556 mlx5_compress_dequeue_burst(void *queue_pair, struct rte_comp_op **ops, 557 uint16_t nb_ops) 558 { 559 struct mlx5_compress_qp *qp = queue_pair; 560 volatile struct mlx5_compress_xform *restrict xform; 561 volatile struct mlx5_cqe *restrict cqe; 562 volatile struct mlx5_gga_compress_opaque *opaq = qp->opaque_mr.addr; 563 struct rte_comp_op *restrict op; 564 const unsigned int cq_size = qp->entries_n; 565 const unsigned int mask = cq_size - 1; 566 uint32_t idx; 567 uint32_t next_idx = qp->ci & mask; 568 const uint16_t max = RTE_MIN((uint16_t)(qp->pi - qp->ci), nb_ops); 569 uint16_t i = 0; 570 int ret; 571 572 if (unlikely(max == 0)) 573 return 0; 574 do { 575 idx = next_idx; 576 next_idx = (qp->ci + 1) & mask; 577 rte_prefetch0(&qp->cq.cqes[next_idx]); 578 rte_prefetch0(qp->ops[next_idx]); 579 op = qp->ops[idx]; 580 cqe = &qp->cq.cqes[idx]; 581 ret = check_cqe(cqe, cq_size, qp->ci); 582 /* 583 * Be sure owner read is done before any other cookie field or 584 * opaque field. 585 */ 586 rte_io_rmb(); 587 if (unlikely(ret != MLX5_CQE_STATUS_SW_OWN)) { 588 if (likely(ret == MLX5_CQE_STATUS_HW_OWN)) 589 break; 590 mlx5_compress_cqe_err_handle(qp, op); 591 } else { 592 xform = op->private_xform; 593 op->status = RTE_COMP_OP_STATUS_SUCCESS; 594 op->consumed = op->src.length; 595 op->produced = rte_be_to_cpu_32(cqe->byte_cnt); 596 MLX5_ASSERT(cqe->byte_cnt == 597 qp->opaque_buf[idx].scattered_length); 598 switch (xform->csum_type) { 599 case RTE_COMP_CHECKSUM_CRC32: 600 op->output_chksum = (uint64_t)rte_be_to_cpu_32 601 (opaq[idx].crc32); 602 break; 603 case RTE_COMP_CHECKSUM_ADLER32: 604 op->output_chksum = (uint64_t)rte_be_to_cpu_32 605 (opaq[idx].adler32) << 32; 606 break; 607 case RTE_COMP_CHECKSUM_CRC32_ADLER32: 608 op->output_chksum = (uint64_t)rte_be_to_cpu_32 609 (opaq[idx].crc32) | 610 ((uint64_t)rte_be_to_cpu_32 611 (opaq[idx].adler32) << 32); 612 break; 613 default: 614 break; 615 } 616 } 617 ops[i++] = op; 618 qp->ci++; 619 } while (i < max); 620 if (likely(i != 0)) { 621 rte_io_wmb(); 622 qp->cq.db_rec[0] = rte_cpu_to_be_32(qp->ci); 623 qp->stats.dequeued_count += i; 624 } 625 return i; 626 } 627 628 static struct ibv_device * 629 mlx5_compress_get_ib_device_match(struct rte_pci_addr *addr) 630 { 631 int n; 632 struct ibv_device **ibv_list = mlx5_glue->get_device_list(&n); 633 struct ibv_device *ibv_match = NULL; 634 635 if (ibv_list == NULL) { 636 rte_errno = ENOSYS; 637 return NULL; 638 } 639 while (n-- > 0) { 640 struct rte_pci_addr paddr; 641 642 DRV_LOG(DEBUG, "Checking device \"%s\"..", ibv_list[n]->name); 643 if (mlx5_dev_to_pci_addr(ibv_list[n]->ibdev_path, &paddr) != 0) 644 continue; 645 if (rte_pci_addr_cmp(addr, &paddr) != 0) 646 continue; 647 ibv_match = ibv_list[n]; 648 break; 649 } 650 if (ibv_match == NULL) 651 rte_errno = ENOENT; 652 mlx5_glue->free_device_list(ibv_list); 653 return ibv_match; 654 } 655 656 static void 657 mlx5_compress_hw_global_release(struct mlx5_compress_priv *priv) 658 { 659 if (priv->pd != NULL) { 660 claim_zero(mlx5_glue->dealloc_pd(priv->pd)); 661 priv->pd = NULL; 662 } 663 if (priv->uar != NULL) { 664 mlx5_glue->devx_free_uar(priv->uar); 665 priv->uar = NULL; 666 } 667 } 668 669 static int 670 mlx5_compress_pd_create(struct mlx5_compress_priv *priv) 671 { 672 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 673 struct mlx5dv_obj obj; 674 struct mlx5dv_pd pd_info; 675 int ret; 676 677 priv->pd = mlx5_glue->alloc_pd(priv->ctx); 678 if (priv->pd == NULL) { 679 DRV_LOG(ERR, "Failed to allocate PD."); 680 return errno ? -errno : -ENOMEM; 681 } 682 obj.pd.in = priv->pd; 683 obj.pd.out = &pd_info; 684 ret = mlx5_glue->dv_init_obj(&obj, MLX5DV_OBJ_PD); 685 if (ret != 0) { 686 DRV_LOG(ERR, "Fail to get PD object info."); 687 mlx5_glue->dealloc_pd(priv->pd); 688 priv->pd = NULL; 689 return -errno; 690 } 691 priv->pdn = pd_info.pdn; 692 return 0; 693 #else 694 (void)priv; 695 DRV_LOG(ERR, "Cannot get pdn - no DV support."); 696 return -ENOTSUP; 697 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */ 698 } 699 700 static int 701 mlx5_compress_hw_global_prepare(struct mlx5_compress_priv *priv) 702 { 703 if (mlx5_compress_pd_create(priv) != 0) 704 return -1; 705 priv->uar = mlx5_devx_alloc_uar(priv->ctx, -1); 706 if (priv->uar == NULL || mlx5_os_get_devx_uar_reg_addr(priv->uar) == 707 NULL) { 708 rte_errno = errno; 709 claim_zero(mlx5_glue->dealloc_pd(priv->pd)); 710 DRV_LOG(ERR, "Failed to allocate UAR."); 711 return -1; 712 } 713 priv->uar_addr = mlx5_os_get_devx_uar_reg_addr(priv->uar); 714 MLX5_ASSERT(qp->uar_addr); 715 #ifndef RTE_ARCH_64 716 rte_spinlock_init(&priv->uar32_sl); 717 #endif /* RTE_ARCH_64 */ 718 return 0; 719 } 720 721 /** 722 * DPDK callback to register a PCI device. 723 * 724 * This function spawns compress device out of a given PCI device. 725 * 726 * @param[in] pci_drv 727 * PCI driver structure (mlx5_compress_driver). 728 * @param[in] pci_dev 729 * PCI device information. 730 * 731 * @return 732 * 0 on success, 1 to skip this driver, a negative errno value otherwise 733 * and rte_errno is set. 734 */ 735 static int 736 mlx5_compress_pci_probe(struct rte_pci_driver *pci_drv, 737 struct rte_pci_device *pci_dev) 738 { 739 struct ibv_device *ibv; 740 struct rte_compressdev *cdev; 741 struct ibv_context *ctx; 742 struct mlx5_compress_priv *priv; 743 struct mlx5_hca_attr att = { 0 }; 744 struct rte_compressdev_pmd_init_params init_params = { 745 .name = "", 746 .socket_id = pci_dev->device.numa_node, 747 }; 748 749 RTE_SET_USED(pci_drv); 750 if (rte_eal_process_type() != RTE_PROC_PRIMARY) { 751 DRV_LOG(ERR, "Non-primary process type is not supported."); 752 rte_errno = ENOTSUP; 753 return -rte_errno; 754 } 755 ibv = mlx5_compress_get_ib_device_match(&pci_dev->addr); 756 if (ibv == NULL) { 757 DRV_LOG(ERR, "No matching IB device for PCI slot " 758 PCI_PRI_FMT ".", pci_dev->addr.domain, 759 pci_dev->addr.bus, pci_dev->addr.devid, 760 pci_dev->addr.function); 761 return -rte_errno; 762 } 763 DRV_LOG(INFO, "PCI information matches for device \"%s\".", ibv->name); 764 ctx = mlx5_glue->dv_open_device(ibv); 765 if (ctx == NULL) { 766 DRV_LOG(ERR, "Failed to open IB device \"%s\".", ibv->name); 767 rte_errno = ENODEV; 768 return -rte_errno; 769 } 770 if (mlx5_devx_cmd_query_hca_attr(ctx, &att) != 0 || 771 att.mmo_compress_en == 0 || att.mmo_decompress_en == 0 || 772 att.mmo_dma_en == 0) { 773 DRV_LOG(ERR, "Not enough capabilities to support compress " 774 "operations, maybe old FW/OFED version?"); 775 claim_zero(mlx5_glue->close_device(ctx)); 776 rte_errno = ENOTSUP; 777 return -ENOTSUP; 778 } 779 cdev = rte_compressdev_pmd_create(ibv->name, &pci_dev->device, 780 sizeof(*priv), &init_params); 781 if (cdev == NULL) { 782 DRV_LOG(ERR, "Failed to create device \"%s\".", ibv->name); 783 claim_zero(mlx5_glue->close_device(ctx)); 784 return -ENODEV; 785 } 786 DRV_LOG(INFO, 787 "Compress device %s was created successfully.", ibv->name); 788 cdev->dev_ops = &mlx5_compress_ops; 789 cdev->dequeue_burst = mlx5_compress_dequeue_burst; 790 cdev->enqueue_burst = mlx5_compress_enqueue_burst; 791 cdev->feature_flags = RTE_COMPDEV_FF_HW_ACCELERATED; 792 priv = cdev->data->dev_private; 793 priv->ctx = ctx; 794 priv->pci_dev = pci_dev; 795 priv->cdev = cdev; 796 priv->min_block_size = att.compress_min_block_size; 797 if (mlx5_compress_hw_global_prepare(priv) != 0) { 798 rte_compressdev_pmd_destroy(priv->cdev); 799 claim_zero(mlx5_glue->close_device(priv->ctx)); 800 return -1; 801 } 802 if (mlx5_mr_btree_init(&priv->mr_scache.cache, 803 MLX5_MR_BTREE_CACHE_N * 2, rte_socket_id()) != 0) { 804 DRV_LOG(ERR, "Failed to allocate shared cache MR memory."); 805 mlx5_compress_hw_global_release(priv); 806 rte_compressdev_pmd_destroy(priv->cdev); 807 claim_zero(mlx5_glue->close_device(priv->ctx)); 808 rte_errno = ENOMEM; 809 return -rte_errno; 810 } 811 priv->mr_scache.reg_mr_cb = mlx5_common_verbs_reg_mr; 812 priv->mr_scache.dereg_mr_cb = mlx5_common_verbs_dereg_mr; 813 pthread_mutex_lock(&priv_list_lock); 814 TAILQ_INSERT_TAIL(&mlx5_compress_priv_list, priv, next); 815 pthread_mutex_unlock(&priv_list_lock); 816 return 0; 817 } 818 819 /** 820 * DPDK callback to remove a PCI device. 821 * 822 * This function removes all compress devices belong to a given PCI device. 823 * 824 * @param[in] pci_dev 825 * Pointer to the PCI device. 826 * 827 * @return 828 * 0 on success, the function cannot fail. 829 */ 830 static int 831 mlx5_compress_pci_remove(struct rte_pci_device *pdev) 832 { 833 struct mlx5_compress_priv *priv = NULL; 834 835 pthread_mutex_lock(&priv_list_lock); 836 TAILQ_FOREACH(priv, &mlx5_compress_priv_list, next) 837 if (rte_pci_addr_cmp(&priv->pci_dev->addr, &pdev->addr) != 0) 838 break; 839 if (priv) 840 TAILQ_REMOVE(&mlx5_compress_priv_list, priv, next); 841 pthread_mutex_unlock(&priv_list_lock); 842 if (priv) { 843 mlx5_mr_release_cache(&priv->mr_scache); 844 mlx5_compress_hw_global_release(priv); 845 rte_compressdev_pmd_destroy(priv->cdev); 846 claim_zero(mlx5_glue->close_device(priv->ctx)); 847 } 848 return 0; 849 } 850 851 static const struct rte_pci_id mlx5_compress_pci_id_map[] = { 852 { 853 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 854 PCI_DEVICE_ID_MELLANOX_CONNECTX6DXBF) 855 }, 856 { 857 .vendor_id = 0 858 } 859 }; 860 861 static struct mlx5_pci_driver mlx5_compress_driver = { 862 .driver_class = MLX5_CLASS_COMPRESS, 863 .pci_driver = { 864 .driver = { 865 .name = RTE_STR(MLX5_COMPRESS_DRIVER_NAME), 866 }, 867 .id_table = mlx5_compress_pci_id_map, 868 .probe = mlx5_compress_pci_probe, 869 .remove = mlx5_compress_pci_remove, 870 .drv_flags = 0, 871 }, 872 }; 873 874 RTE_INIT(rte_mlx5_compress_init) 875 { 876 mlx5_common_init(); 877 if (mlx5_glue != NULL) 878 mlx5_pci_driver_register(&mlx5_compress_driver); 879 } 880 881 RTE_LOG_REGISTER(mlx5_compress_logtype, MLX5_COMPRESS_LOG_NAME, NOTICE) 882 RTE_PMD_EXPORT_NAME(MLX5_COMPRESS_DRIVER_NAME, __COUNTER__); 883 RTE_PMD_REGISTER_PCI_TABLE(MLX5_COMPRESS_DRIVER_NAME, mlx5_compress_pci_id_map); 884 RTE_PMD_REGISTER_KMOD_DEP(MLX5_COMPRESS_DRIVER_NAME, "* ib_uverbs & mlx5_core & mlx5_ib"); 885