1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2020 Mellanox Technologies, Ltd 3 */ 4 #include <rte_ether.h> 5 #include <rte_ethdev_driver.h> 6 #include <rte_interrupts.h> 7 #include <rte_alarm.h> 8 #include <rte_malloc.h> 9 10 #include "mlx5.h" 11 #include "mlx5_rxtx.h" 12 #include "mlx5_common_os.h" 13 14 /* Destroy Event Queue Notification Channel. */ 15 static void 16 mlx5_txpp_destroy_eqn(struct mlx5_dev_ctx_shared *sh) 17 { 18 if (sh->txpp.echan) { 19 mlx5_glue->devx_destroy_event_channel(sh->txpp.echan); 20 sh->txpp.echan = NULL; 21 } 22 sh->txpp.eqn = 0; 23 } 24 25 /* Create Event Queue Notification Channel. */ 26 static int 27 mlx5_txpp_create_eqn(struct mlx5_dev_ctx_shared *sh) 28 { 29 uint32_t lcore; 30 31 MLX5_ASSERT(!sh->txpp.echan); 32 lcore = (uint32_t)rte_lcore_to_cpu_id(-1); 33 if (mlx5_glue->devx_query_eqn(sh->ctx, lcore, &sh->txpp.eqn)) { 34 rte_errno = errno; 35 DRV_LOG(ERR, "Failed to query EQ number %d.", rte_errno); 36 sh->txpp.eqn = 0; 37 return -rte_errno; 38 } 39 sh->txpp.echan = mlx5_glue->devx_create_event_channel(sh->ctx, 40 MLX5DV_DEVX_CREATE_EVENT_CHANNEL_FLAGS_OMIT_EV_DATA); 41 if (!sh->txpp.echan) { 42 sh->txpp.eqn = 0; 43 rte_errno = errno; 44 DRV_LOG(ERR, "Failed to create event channel %d.", 45 rte_errno); 46 return -rte_errno; 47 } 48 return 0; 49 } 50 51 static void 52 mlx5_txpp_destroy_send_queue(struct mlx5_txpp_wq *wq) 53 { 54 if (wq->sq) 55 claim_zero(mlx5_devx_cmd_destroy(wq->sq)); 56 if (wq->sq_umem) 57 claim_zero(mlx5_glue->devx_umem_dereg(wq->sq_umem)); 58 if (wq->sq_buf) 59 rte_free((void *)(uintptr_t)wq->sq_buf); 60 if (wq->cq) 61 claim_zero(mlx5_devx_cmd_destroy(wq->cq)); 62 if (wq->cq_umem) 63 claim_zero(mlx5_glue->devx_umem_dereg(wq->cq_umem)); 64 if (wq->cq_buf) 65 rte_free((void *)(uintptr_t)wq->cq_buf); 66 memset(wq, 0, sizeof(*wq)); 67 } 68 69 static void 70 mlx5_txpp_destroy_rearm_queue(struct mlx5_dev_ctx_shared *sh) 71 { 72 struct mlx5_txpp_wq *wq = &sh->txpp.rearm_queue; 73 74 mlx5_txpp_destroy_send_queue(wq); 75 } 76 77 static void 78 mlx5_txpp_destroy_clock_queue(struct mlx5_dev_ctx_shared *sh) 79 { 80 struct mlx5_txpp_wq *wq = &sh->txpp.clock_queue; 81 82 mlx5_txpp_destroy_send_queue(wq); 83 } 84 85 static void 86 mlx5_txpp_fill_cqe_rearm_queue(struct mlx5_dev_ctx_shared *sh) 87 { 88 struct mlx5_txpp_wq *wq = &sh->txpp.rearm_queue; 89 struct mlx5_cqe *cqe = (struct mlx5_cqe *)(uintptr_t)wq->cqes; 90 uint32_t i; 91 92 for (i = 0; i < MLX5_TXPP_REARM_CQ_SIZE; i++) { 93 cqe->op_own = (MLX5_CQE_INVALID << 4) | MLX5_CQE_OWNER_MASK; 94 ++cqe; 95 } 96 } 97 98 static void 99 mlx5_txpp_fill_wqe_rearm_queue(struct mlx5_dev_ctx_shared *sh) 100 { 101 struct mlx5_txpp_wq *wq = &sh->txpp.rearm_queue; 102 struct mlx5_wqe *wqe = (struct mlx5_wqe *)(uintptr_t)wq->wqes; 103 uint32_t i; 104 105 for (i = 0; i < wq->sq_size; i += 2) { 106 struct mlx5_wqe_cseg *cs; 107 struct mlx5_wqe_qseg *qs; 108 uint32_t index; 109 110 /* Build SEND_EN request with slave WQE index. */ 111 cs = &wqe[i + 0].cseg; 112 cs->opcode = RTE_BE32(MLX5_OPCODE_SEND_EN | 0); 113 cs->sq_ds = rte_cpu_to_be_32((wq->sq->id << 8) | 2); 114 cs->flags = RTE_BE32(MLX5_COMP_ALWAYS << 115 MLX5_COMP_MODE_OFFSET); 116 cs->misc = RTE_BE32(0); 117 qs = RTE_PTR_ADD(cs, sizeof(struct mlx5_wqe_cseg)); 118 index = (i * MLX5_TXPP_REARM / 2 + MLX5_TXPP_REARM) & 119 ((1 << MLX5_WQ_INDEX_WIDTH) - 1); 120 qs->max_index = rte_cpu_to_be_32(index); 121 qs->qpn_cqn = rte_cpu_to_be_32(sh->txpp.clock_queue.sq->id); 122 /* Build WAIT request with slave CQE index. */ 123 cs = &wqe[i + 1].cseg; 124 cs->opcode = RTE_BE32(MLX5_OPCODE_WAIT | 0); 125 cs->sq_ds = rte_cpu_to_be_32((wq->sq->id << 8) | 2); 126 cs->flags = RTE_BE32(MLX5_COMP_ONLY_ERR << 127 MLX5_COMP_MODE_OFFSET); 128 cs->misc = RTE_BE32(0); 129 qs = RTE_PTR_ADD(cs, sizeof(struct mlx5_wqe_cseg)); 130 index = (i * MLX5_TXPP_REARM / 2 + MLX5_TXPP_REARM / 2) & 131 ((1 << MLX5_CQ_INDEX_WIDTH) - 1); 132 qs->max_index = rte_cpu_to_be_32(index); 133 qs->qpn_cqn = rte_cpu_to_be_32(sh->txpp.clock_queue.cq->id); 134 } 135 } 136 137 /* Creates the Rearm Queue to fire the requests to Clock Queue in realtime. */ 138 static int 139 mlx5_txpp_create_rearm_queue(struct mlx5_dev_ctx_shared *sh) 140 { 141 struct mlx5_devx_create_sq_attr sq_attr = { 0 }; 142 struct mlx5_devx_modify_sq_attr msq_attr = { 0 }; 143 struct mlx5_devx_cq_attr cq_attr = { 0 }; 144 struct mlx5_txpp_wq *wq = &sh->txpp.rearm_queue; 145 size_t page_size = sysconf(_SC_PAGESIZE); 146 uint32_t umem_size, umem_dbrec; 147 int ret; 148 149 /* Allocate memory buffer for CQEs and doorbell record. */ 150 umem_size = sizeof(struct mlx5_cqe) * MLX5_TXPP_REARM_CQ_SIZE; 151 umem_dbrec = RTE_ALIGN(umem_size, MLX5_DBR_SIZE); 152 umem_size += MLX5_DBR_SIZE; 153 wq->cq_buf = rte_zmalloc_socket(__func__, umem_size, 154 page_size, sh->numa_node); 155 if (!wq->cq_buf) { 156 DRV_LOG(ERR, "Failed to allocate memory for Rearm Queue."); 157 return -ENOMEM; 158 } 159 /* Register allocated buffer in user space with DevX. */ 160 wq->cq_umem = mlx5_glue->devx_umem_reg(sh->ctx, 161 (void *)(uintptr_t)wq->cq_buf, 162 umem_size, 163 IBV_ACCESS_LOCAL_WRITE); 164 if (!wq->cq_umem) { 165 rte_errno = errno; 166 DRV_LOG(ERR, "Failed to register umem for Rearm Queue."); 167 goto error; 168 } 169 /* Create completion queue object for Rearm Queue. */ 170 cq_attr.cqe_size = (sizeof(struct mlx5_cqe) == 128) ? 171 MLX5_CQE_SIZE_128B : MLX5_CQE_SIZE_64B; 172 cq_attr.uar_page_id = sh->tx_uar->page_id; 173 cq_attr.eqn = sh->txpp.eqn; 174 cq_attr.q_umem_valid = 1; 175 cq_attr.q_umem_offset = 0; 176 cq_attr.q_umem_id = mlx5_os_get_umem_id(wq->cq_umem); 177 cq_attr.db_umem_valid = 1; 178 cq_attr.db_umem_offset = umem_dbrec; 179 cq_attr.db_umem_id = mlx5_os_get_umem_id(wq->cq_umem); 180 cq_attr.log_cq_size = rte_log2_u32(MLX5_TXPP_REARM_CQ_SIZE); 181 cq_attr.log_page_size = rte_log2_u32(page_size); 182 wq->cq = mlx5_devx_cmd_create_cq(sh->ctx, &cq_attr); 183 if (!wq->cq) { 184 rte_errno = errno; 185 DRV_LOG(ERR, "Failed to create CQ for Rearm Queue."); 186 goto error; 187 } 188 wq->cq_dbrec = RTE_PTR_ADD(wq->cq_buf, umem_dbrec); 189 wq->cq_ci = 0; 190 wq->arm_sn = 0; 191 /* Mark all CQEs initially as invalid. */ 192 mlx5_txpp_fill_cqe_rearm_queue(sh); 193 /* 194 * Allocate memory buffer for Send Queue WQEs. 195 * There should be no WQE leftovers in the cyclic queue. 196 */ 197 wq->sq_size = MLX5_TXPP_REARM_SQ_SIZE; 198 MLX5_ASSERT(wq->sq_size == (1 << log2above(wq->sq_size))); 199 umem_size = MLX5_WQE_SIZE * wq->sq_size; 200 umem_dbrec = RTE_ALIGN(umem_size, MLX5_DBR_SIZE); 201 umem_size += MLX5_DBR_SIZE; 202 wq->sq_buf = rte_zmalloc_socket(__func__, umem_size, 203 page_size, sh->numa_node); 204 if (!wq->sq_buf) { 205 DRV_LOG(ERR, "Failed to allocate memory for Rearm Queue."); 206 rte_errno = ENOMEM; 207 goto error; 208 } 209 /* Register allocated buffer in user space with DevX. */ 210 wq->sq_umem = mlx5_glue->devx_umem_reg(sh->ctx, 211 (void *)(uintptr_t)wq->sq_buf, 212 umem_size, 213 IBV_ACCESS_LOCAL_WRITE); 214 if (!wq->sq_umem) { 215 rte_errno = errno; 216 DRV_LOG(ERR, "Failed to register umem for Rearm Queue."); 217 goto error; 218 } 219 /* Create send queue object for Rearm Queue. */ 220 sq_attr.state = MLX5_SQC_STATE_RST; 221 sq_attr.tis_lst_sz = 1; 222 sq_attr.tis_num = sh->tis->id; 223 sq_attr.cqn = wq->cq->id; 224 sq_attr.cd_master = 1; 225 sq_attr.wq_attr.uar_page = sh->tx_uar->page_id; 226 sq_attr.wq_attr.wq_type = MLX5_WQ_TYPE_CYCLIC; 227 sq_attr.wq_attr.pd = sh->pdn; 228 sq_attr.wq_attr.log_wq_stride = rte_log2_u32(MLX5_WQE_SIZE); 229 sq_attr.wq_attr.log_wq_sz = rte_log2_u32(wq->sq_size); 230 sq_attr.wq_attr.dbr_umem_valid = 1; 231 sq_attr.wq_attr.dbr_addr = umem_dbrec; 232 sq_attr.wq_attr.dbr_umem_id = mlx5_os_get_umem_id(wq->sq_umem); 233 sq_attr.wq_attr.wq_umem_valid = 1; 234 sq_attr.wq_attr.wq_umem_id = mlx5_os_get_umem_id(wq->sq_umem); 235 sq_attr.wq_attr.wq_umem_offset = 0; 236 wq->sq = mlx5_devx_cmd_create_sq(sh->ctx, &sq_attr); 237 if (!wq->sq) { 238 rte_errno = errno; 239 DRV_LOG(ERR, "Failed to create SQ for Rearm Queue."); 240 goto error; 241 } 242 wq->sq_dbrec = RTE_PTR_ADD(wq->sq_buf, umem_dbrec + 243 MLX5_SND_DBR * sizeof(uint32_t)); 244 /* Build the WQEs in the Send Queue before goto Ready state. */ 245 mlx5_txpp_fill_wqe_rearm_queue(sh); 246 /* Change queue state to ready. */ 247 msq_attr.sq_state = MLX5_SQC_STATE_RST; 248 msq_attr.state = MLX5_SQC_STATE_RDY; 249 ret = mlx5_devx_cmd_modify_sq(wq->sq, &msq_attr); 250 if (ret) { 251 DRV_LOG(ERR, "Failed to set SQ ready state Rearm Queue."); 252 goto error; 253 } 254 return 0; 255 error: 256 ret = -rte_errno; 257 mlx5_txpp_destroy_rearm_queue(sh); 258 rte_errno = -ret; 259 return ret; 260 } 261 262 static void 263 mlx5_txpp_fill_wqe_clock_queue(struct mlx5_dev_ctx_shared *sh) 264 { 265 struct mlx5_txpp_wq *wq = &sh->txpp.clock_queue; 266 struct mlx5_wqe *wqe = (struct mlx5_wqe *)(uintptr_t)wq->wqes; 267 struct mlx5_wqe_cseg *cs = &wqe->cseg; 268 uint32_t wqe_size, opcode, i; 269 uint8_t *dst; 270 271 /* For test purposes fill the WQ with SEND inline packet. */ 272 if (sh->txpp.test) { 273 wqe_size = RTE_ALIGN(MLX5_TXPP_TEST_PKT_SIZE + 274 MLX5_WQE_CSEG_SIZE + 275 2 * MLX5_WQE_ESEG_SIZE - 276 MLX5_ESEG_MIN_INLINE_SIZE, 277 MLX5_WSEG_SIZE); 278 opcode = MLX5_OPCODE_SEND; 279 } else { 280 wqe_size = MLX5_WSEG_SIZE; 281 opcode = MLX5_OPCODE_NOP; 282 } 283 cs->opcode = rte_cpu_to_be_32(opcode | 0); /* Index is ignored. */ 284 cs->sq_ds = rte_cpu_to_be_32((wq->sq->id << 8) | 285 (wqe_size / MLX5_WSEG_SIZE)); 286 cs->flags = RTE_BE32(MLX5_COMP_ALWAYS << MLX5_COMP_MODE_OFFSET); 287 cs->misc = RTE_BE32(0); 288 wqe_size = RTE_ALIGN(wqe_size, MLX5_WQE_SIZE); 289 if (sh->txpp.test) { 290 struct mlx5_wqe_eseg *es = &wqe->eseg; 291 struct rte_ether_hdr *eth_hdr; 292 struct rte_ipv4_hdr *ip_hdr; 293 struct rte_udp_hdr *udp_hdr; 294 295 /* Build the inline test packet pattern. */ 296 MLX5_ASSERT(wqe_size <= MLX5_WQE_SIZE_MAX); 297 MLX5_ASSERT(MLX5_TXPP_TEST_PKT_SIZE >= 298 (sizeof(struct rte_ether_hdr) + 299 sizeof(struct rte_ipv4_hdr))); 300 es->flags = 0; 301 es->cs_flags = MLX5_ETH_WQE_L3_CSUM | MLX5_ETH_WQE_L4_CSUM; 302 es->swp_offs = 0; 303 es->metadata = 0; 304 es->swp_flags = 0; 305 es->mss = 0; 306 es->inline_hdr_sz = RTE_BE16(MLX5_TXPP_TEST_PKT_SIZE); 307 /* Build test packet L2 header (Ethernet). */ 308 dst = (uint8_t *)&es->inline_data; 309 eth_hdr = (struct rte_ether_hdr *)dst; 310 rte_eth_random_addr(ð_hdr->d_addr.addr_bytes[0]); 311 rte_eth_random_addr(ð_hdr->s_addr.addr_bytes[0]); 312 eth_hdr->ether_type = rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV4); 313 /* Build test packet L3 header (IP v4). */ 314 dst += sizeof(struct rte_ether_hdr); 315 ip_hdr = (struct rte_ipv4_hdr *)dst; 316 ip_hdr->version_ihl = RTE_IPV4_VHL_DEF; 317 ip_hdr->type_of_service = 0; 318 ip_hdr->fragment_offset = 0; 319 ip_hdr->time_to_live = 64; 320 ip_hdr->next_proto_id = IPPROTO_UDP; 321 ip_hdr->packet_id = 0; 322 ip_hdr->total_length = RTE_BE16(MLX5_TXPP_TEST_PKT_SIZE - 323 sizeof(struct rte_ether_hdr)); 324 /* use RFC5735 / RFC2544 reserved network test addresses */ 325 ip_hdr->src_addr = RTE_BE32((198U << 24) | (18 << 16) | 326 (0 << 8) | 1); 327 ip_hdr->dst_addr = RTE_BE32((198U << 24) | (18 << 16) | 328 (0 << 8) | 2); 329 if (MLX5_TXPP_TEST_PKT_SIZE < 330 (sizeof(struct rte_ether_hdr) + 331 sizeof(struct rte_ipv4_hdr) + 332 sizeof(struct rte_udp_hdr))) 333 goto wcopy; 334 /* Build test packet L4 header (UDP). */ 335 dst += sizeof(struct rte_ipv4_hdr); 336 udp_hdr = (struct rte_udp_hdr *)dst; 337 udp_hdr->src_port = RTE_BE16(9); /* RFC863 Discard. */ 338 udp_hdr->dst_port = RTE_BE16(9); 339 udp_hdr->dgram_len = RTE_BE16(MLX5_TXPP_TEST_PKT_SIZE - 340 sizeof(struct rte_ether_hdr) - 341 sizeof(struct rte_ipv4_hdr)); 342 udp_hdr->dgram_cksum = 0; 343 /* Fill the test packet data. */ 344 dst += sizeof(struct rte_udp_hdr); 345 for (i = sizeof(struct rte_ether_hdr) + 346 sizeof(struct rte_ipv4_hdr) + 347 sizeof(struct rte_udp_hdr); 348 i < MLX5_TXPP_TEST_PKT_SIZE; i++) 349 *dst++ = (uint8_t)(i & 0xFF); 350 } 351 wcopy: 352 /* Duplicate the pattern to the next WQEs. */ 353 dst = (uint8_t *)(uintptr_t)wq->sq_buf; 354 for (i = 1; i < MLX5_TXPP_CLKQ_SIZE; i++) { 355 dst += wqe_size; 356 rte_memcpy(dst, (void *)(uintptr_t)wq->sq_buf, wqe_size); 357 } 358 } 359 360 /* Creates the Clock Queue for packet pacing, returns zero on success. */ 361 static int 362 mlx5_txpp_create_clock_queue(struct mlx5_dev_ctx_shared *sh) 363 { 364 struct mlx5_devx_create_sq_attr sq_attr = { 0 }; 365 struct mlx5_devx_modify_sq_attr msq_attr = { 0 }; 366 struct mlx5_devx_cq_attr cq_attr = { 0 }; 367 struct mlx5_txpp_wq *wq = &sh->txpp.clock_queue; 368 size_t page_size = sysconf(_SC_PAGESIZE); 369 uint32_t umem_size, umem_dbrec; 370 int ret; 371 372 /* Allocate memory buffer for CQEs and doorbell record. */ 373 umem_size = sizeof(struct mlx5_cqe) * MLX5_TXPP_CLKQ_SIZE; 374 umem_dbrec = RTE_ALIGN(umem_size, MLX5_DBR_SIZE); 375 umem_size += MLX5_DBR_SIZE; 376 wq->cq_buf = rte_zmalloc_socket(__func__, umem_size, 377 page_size, sh->numa_node); 378 if (!wq->cq_buf) { 379 DRV_LOG(ERR, "Failed to allocate memory for Clock Queue."); 380 return -ENOMEM; 381 } 382 /* Register allocated buffer in user space with DevX. */ 383 wq->cq_umem = mlx5_glue->devx_umem_reg(sh->ctx, 384 (void *)(uintptr_t)wq->cq_buf, 385 umem_size, 386 IBV_ACCESS_LOCAL_WRITE); 387 if (!wq->cq_umem) { 388 rte_errno = errno; 389 DRV_LOG(ERR, "Failed to register umem for Clock Queue."); 390 goto error; 391 } 392 /* Create completion queue object for Clock Queue. */ 393 cq_attr.cqe_size = (sizeof(struct mlx5_cqe) == 128) ? 394 MLX5_CQE_SIZE_128B : MLX5_CQE_SIZE_64B; 395 cq_attr.use_first_only = 1; 396 cq_attr.overrun_ignore = 1; 397 cq_attr.uar_page_id = sh->tx_uar->page_id; 398 cq_attr.eqn = sh->txpp.eqn; 399 cq_attr.q_umem_valid = 1; 400 cq_attr.q_umem_offset = 0; 401 cq_attr.q_umem_id = wq->cq_umem->umem_id; 402 cq_attr.db_umem_valid = 1; 403 cq_attr.db_umem_offset = umem_dbrec; 404 cq_attr.db_umem_id = wq->cq_umem->umem_id; 405 cq_attr.log_cq_size = rte_log2_u32(MLX5_TXPP_CLKQ_SIZE); 406 cq_attr.log_page_size = rte_log2_u32(page_size); 407 wq->cq = mlx5_devx_cmd_create_cq(sh->ctx, &cq_attr); 408 if (!wq->cq) { 409 rte_errno = errno; 410 DRV_LOG(ERR, "Failed to create CQ for Clock Queue."); 411 goto error; 412 } 413 wq->cq_dbrec = RTE_PTR_ADD(wq->cq_buf, umem_dbrec); 414 wq->cq_ci = 0; 415 /* Allocate memory buffer for Send Queue WQEs. */ 416 if (sh->txpp.test) { 417 wq->sq_size = RTE_ALIGN(MLX5_TXPP_TEST_PKT_SIZE + 418 MLX5_WQE_CSEG_SIZE + 419 2 * MLX5_WQE_ESEG_SIZE - 420 MLX5_ESEG_MIN_INLINE_SIZE, 421 MLX5_WQE_SIZE) / MLX5_WQE_SIZE; 422 wq->sq_size *= MLX5_TXPP_CLKQ_SIZE; 423 } else { 424 wq->sq_size = MLX5_TXPP_CLKQ_SIZE; 425 } 426 /* There should not be WQE leftovers in the cyclic queue. */ 427 MLX5_ASSERT(wq->sq_size == (1 << log2above(wq->sq_size))); 428 umem_size = MLX5_WQE_SIZE * wq->sq_size; 429 umem_dbrec = RTE_ALIGN(umem_size, MLX5_DBR_SIZE); 430 umem_size += MLX5_DBR_SIZE; 431 wq->sq_buf = rte_zmalloc_socket(__func__, umem_size, 432 page_size, sh->numa_node); 433 if (!wq->sq_buf) { 434 DRV_LOG(ERR, "Failed to allocate memory for Clock Queue."); 435 rte_errno = ENOMEM; 436 goto error; 437 } 438 /* Register allocated buffer in user space with DevX. */ 439 wq->sq_umem = mlx5_glue->devx_umem_reg(sh->ctx, 440 (void *)(uintptr_t)wq->sq_buf, 441 umem_size, 442 IBV_ACCESS_LOCAL_WRITE); 443 if (!wq->sq_umem) { 444 rte_errno = errno; 445 DRV_LOG(ERR, "Failed to register umem for Clock Queue."); 446 goto error; 447 } 448 /* Create send queue object for Clock Queue. */ 449 if (sh->txpp.test) { 450 sq_attr.tis_lst_sz = 1; 451 sq_attr.tis_num = sh->tis->id; 452 sq_attr.non_wire = 0; 453 sq_attr.static_sq_wq = 1; 454 } else { 455 sq_attr.non_wire = 1; 456 sq_attr.static_sq_wq = 1; 457 } 458 sq_attr.state = MLX5_SQC_STATE_RST; 459 sq_attr.cqn = wq->cq->id; 460 sq_attr.wq_attr.cd_slave = 1; 461 sq_attr.wq_attr.uar_page = sh->tx_uar->page_id; 462 sq_attr.wq_attr.wq_type = MLX5_WQ_TYPE_CYCLIC; 463 sq_attr.wq_attr.pd = sh->pdn; 464 sq_attr.wq_attr.log_wq_stride = rte_log2_u32(MLX5_WQE_SIZE); 465 sq_attr.wq_attr.log_wq_sz = rte_log2_u32(wq->sq_size); 466 sq_attr.wq_attr.dbr_umem_valid = 1; 467 sq_attr.wq_attr.dbr_addr = umem_dbrec; 468 sq_attr.wq_attr.dbr_umem_id = wq->sq_umem->umem_id; 469 sq_attr.wq_attr.wq_umem_valid = 1; 470 sq_attr.wq_attr.wq_umem_id = wq->sq_umem->umem_id; 471 /* umem_offset must be zero for static_sq_wq queue. */ 472 sq_attr.wq_attr.wq_umem_offset = 0; 473 wq->sq = mlx5_devx_cmd_create_sq(sh->ctx, &sq_attr); 474 if (!wq->sq) { 475 rte_errno = errno; 476 DRV_LOG(ERR, "Failed to create SQ for Clock Queue."); 477 goto error; 478 } 479 wq->sq_dbrec = RTE_PTR_ADD(wq->sq_buf, umem_dbrec + 480 MLX5_SND_DBR * sizeof(uint32_t)); 481 /* Build the WQEs in the Send Queue before goto Ready state. */ 482 mlx5_txpp_fill_wqe_clock_queue(sh); 483 /* Change queue state to ready. */ 484 msq_attr.sq_state = MLX5_SQC_STATE_RST; 485 msq_attr.state = MLX5_SQC_STATE_RDY; 486 wq->sq_ci = 0; 487 ret = mlx5_devx_cmd_modify_sq(wq->sq, &msq_attr); 488 if (ret) { 489 DRV_LOG(ERR, "Failed to set SQ ready state Clock Queue."); 490 goto error; 491 } 492 return 0; 493 error: 494 ret = -rte_errno; 495 mlx5_txpp_destroy_clock_queue(sh); 496 rte_errno = -ret; 497 return ret; 498 } 499 500 /* 501 * The routine initializes the packet pacing infrastructure: 502 * - allocates PP context 503 * - Clock CQ/SQ 504 * - Rearm CQ/SQ 505 * - attaches rearm interrupt handler 506 * 507 * Returns 0 on success, negative otherwise 508 */ 509 static int 510 mlx5_txpp_create(struct mlx5_dev_ctx_shared *sh, struct mlx5_priv *priv) 511 { 512 int tx_pp = priv->config.tx_pp; 513 int ret; 514 515 /* Store the requested pacing parameters. */ 516 sh->txpp.tick = tx_pp >= 0 ? tx_pp : -tx_pp; 517 sh->txpp.test = !!(tx_pp < 0); 518 sh->txpp.skew = priv->config.tx_skew; 519 sh->txpp.freq = priv->config.hca_attr.dev_freq_khz; 520 ret = mlx5_txpp_create_eqn(sh); 521 if (ret) 522 goto exit; 523 ret = mlx5_txpp_create_clock_queue(sh); 524 if (ret) 525 goto exit; 526 ret = mlx5_txpp_create_rearm_queue(sh); 527 if (ret) 528 goto exit; 529 exit: 530 if (ret) { 531 mlx5_txpp_destroy_rearm_queue(sh); 532 mlx5_txpp_destroy_clock_queue(sh); 533 mlx5_txpp_destroy_eqn(sh); 534 sh->txpp.tick = 0; 535 sh->txpp.test = 0; 536 sh->txpp.skew = 0; 537 } 538 return ret; 539 } 540 541 /* 542 * The routine destroys the packet pacing infrastructure: 543 * - detaches rearm interrupt handler 544 * - Rearm CQ/SQ 545 * - Clock CQ/SQ 546 * - PP context 547 */ 548 static void 549 mlx5_txpp_destroy(struct mlx5_dev_ctx_shared *sh) 550 { 551 mlx5_txpp_destroy_rearm_queue(sh); 552 mlx5_txpp_destroy_clock_queue(sh); 553 mlx5_txpp_destroy_eqn(sh); 554 sh->txpp.tick = 0; 555 sh->txpp.test = 0; 556 sh->txpp.skew = 0; 557 } 558 559 /** 560 * Creates and starts packet pacing infrastructure on specified device. 561 * 562 * @param dev 563 * Pointer to Ethernet device structure. 564 * 565 * @return 566 * 0 on success, a negative errno value otherwise and rte_errno is set. 567 */ 568 int 569 mlx5_txpp_start(struct rte_eth_dev *dev) 570 { 571 struct mlx5_priv *priv = dev->data->dev_private; 572 struct mlx5_dev_ctx_shared *sh = priv->sh; 573 int err = 0; 574 int ret; 575 576 if (!priv->config.tx_pp) { 577 /* Packet pacing is not requested for the device. */ 578 MLX5_ASSERT(priv->txpp_en == 0); 579 return 0; 580 } 581 if (priv->txpp_en) { 582 /* Packet pacing is already enabled for the device. */ 583 MLX5_ASSERT(sh->txpp.refcnt); 584 return 0; 585 } 586 if (priv->config.tx_pp > 0) { 587 ret = rte_mbuf_dynflag_lookup 588 (RTE_MBUF_DYNFLAG_TX_TIMESTAMP_NAME, NULL); 589 if (ret < 0) 590 return 0; 591 } 592 ret = pthread_mutex_lock(&sh->txpp.mutex); 593 MLX5_ASSERT(!ret); 594 RTE_SET_USED(ret); 595 if (sh->txpp.refcnt) { 596 priv->txpp_en = 1; 597 ++sh->txpp.refcnt; 598 } else { 599 err = mlx5_txpp_create(sh, priv); 600 if (!err) { 601 MLX5_ASSERT(sh->txpp.tick); 602 priv->txpp_en = 1; 603 sh->txpp.refcnt = 1; 604 } else { 605 rte_errno = -err; 606 } 607 } 608 ret = pthread_mutex_unlock(&sh->txpp.mutex); 609 MLX5_ASSERT(!ret); 610 RTE_SET_USED(ret); 611 return err; 612 } 613 614 /** 615 * Stops and destroys packet pacing infrastructure on specified device. 616 * 617 * @param dev 618 * Pointer to Ethernet device structure. 619 * 620 * @return 621 * 0 on success, a negative errno value otherwise and rte_errno is set. 622 */ 623 void 624 mlx5_txpp_stop(struct rte_eth_dev *dev) 625 { 626 struct mlx5_priv *priv = dev->data->dev_private; 627 struct mlx5_dev_ctx_shared *sh = priv->sh; 628 int ret; 629 630 if (!priv->txpp_en) { 631 /* Packet pacing is already disabled for the device. */ 632 return; 633 } 634 priv->txpp_en = 0; 635 ret = pthread_mutex_lock(&sh->txpp.mutex); 636 MLX5_ASSERT(!ret); 637 RTE_SET_USED(ret); 638 MLX5_ASSERT(sh->txpp.refcnt); 639 if (!sh->txpp.refcnt || --sh->txpp.refcnt) 640 return; 641 /* No references any more, do actual destroy. */ 642 mlx5_txpp_destroy(sh); 643 ret = pthread_mutex_unlock(&sh->txpp.mutex); 644 MLX5_ASSERT(!ret); 645 RTE_SET_USED(ret); 646 } 647