1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2020 Mellanox Technologies, Ltd 3 */ 4 #include <fcntl.h> 5 #include <stdint.h> 6 7 #include <rte_ether.h> 8 #include <rte_ethdev_driver.h> 9 #include <rte_interrupts.h> 10 #include <rte_alarm.h> 11 #include <rte_malloc.h> 12 #include <rte_cycles.h> 13 14 #include "mlx5.h" 15 #include "mlx5_rxtx.h" 16 #include "mlx5_common_os.h" 17 18 /* Destroy Event Queue Notification Channel. */ 19 static void 20 mlx5_txpp_destroy_eqn(struct mlx5_dev_ctx_shared *sh) 21 { 22 if (sh->txpp.echan) { 23 mlx5_glue->devx_destroy_event_channel(sh->txpp.echan); 24 sh->txpp.echan = NULL; 25 } 26 sh->txpp.eqn = 0; 27 } 28 29 /* Create Event Queue Notification Channel. */ 30 static int 31 mlx5_txpp_create_eqn(struct mlx5_dev_ctx_shared *sh) 32 { 33 uint32_t lcore; 34 35 MLX5_ASSERT(!sh->txpp.echan); 36 lcore = (uint32_t)rte_lcore_to_cpu_id(-1); 37 if (mlx5_glue->devx_query_eqn(sh->ctx, lcore, &sh->txpp.eqn)) { 38 rte_errno = errno; 39 DRV_LOG(ERR, "Failed to query EQ number %d.", rte_errno); 40 sh->txpp.eqn = 0; 41 return -rte_errno; 42 } 43 sh->txpp.echan = mlx5_glue->devx_create_event_channel(sh->ctx, 44 MLX5DV_DEVX_CREATE_EVENT_CHANNEL_FLAGS_OMIT_EV_DATA); 45 if (!sh->txpp.echan) { 46 sh->txpp.eqn = 0; 47 rte_errno = errno; 48 DRV_LOG(ERR, "Failed to create event channel %d.", 49 rte_errno); 50 return -rte_errno; 51 } 52 return 0; 53 } 54 55 static void 56 mlx5_txpp_free_pp_index(struct mlx5_dev_ctx_shared *sh) 57 { 58 if (sh->txpp.pp) { 59 mlx5_glue->dv_free_pp(sh->txpp.pp); 60 sh->txpp.pp = NULL; 61 sh->txpp.pp_id = 0; 62 } 63 } 64 65 /* Allocate Packet Pacing index from kernel via mlx5dv call. */ 66 static int 67 mlx5_txpp_alloc_pp_index(struct mlx5_dev_ctx_shared *sh) 68 { 69 #ifdef HAVE_MLX5DV_PP_ALLOC 70 uint32_t pp[MLX5_ST_SZ_DW(set_pp_rate_limit_context)]; 71 uint64_t rate; 72 73 MLX5_ASSERT(!sh->txpp.pp); 74 memset(&pp, 0, sizeof(pp)); 75 rate = NS_PER_S / sh->txpp.tick; 76 if (rate * sh->txpp.tick != NS_PER_S) 77 DRV_LOG(WARNING, "Packet pacing frequency is not precise."); 78 if (sh->txpp.test) { 79 uint32_t len; 80 81 len = RTE_MAX(MLX5_TXPP_TEST_PKT_SIZE, 82 (size_t)RTE_ETHER_MIN_LEN); 83 MLX5_SET(set_pp_rate_limit_context, &pp, 84 burst_upper_bound, len); 85 MLX5_SET(set_pp_rate_limit_context, &pp, 86 typical_packet_size, len); 87 /* Convert packets per second into kilobits. */ 88 rate = (rate * len) / (1000ul / CHAR_BIT); 89 DRV_LOG(INFO, "Packet pacing rate set to %" PRIu64, rate); 90 } 91 MLX5_SET(set_pp_rate_limit_context, &pp, rate_limit, rate); 92 MLX5_SET(set_pp_rate_limit_context, &pp, rate_mode, 93 sh->txpp.test ? MLX5_DATA_RATE : MLX5_WQE_RATE); 94 sh->txpp.pp = mlx5_glue->dv_alloc_pp 95 (sh->ctx, sizeof(pp), &pp, 96 MLX5DV_PP_ALLOC_FLAGS_DEDICATED_INDEX); 97 if (sh->txpp.pp == NULL) { 98 DRV_LOG(ERR, "Failed to allocate packet pacing index."); 99 rte_errno = errno; 100 return -errno; 101 } 102 if (!sh->txpp.pp->index) { 103 DRV_LOG(ERR, "Zero packet pacing index allocated."); 104 mlx5_txpp_free_pp_index(sh); 105 rte_errno = ENOTSUP; 106 return -ENOTSUP; 107 } 108 sh->txpp.pp_id = sh->txpp.pp->index; 109 return 0; 110 #else 111 RTE_SET_USED(sh); 112 DRV_LOG(ERR, "Allocating pacing index is not supported."); 113 rte_errno = ENOTSUP; 114 return -ENOTSUP; 115 #endif 116 } 117 118 static void 119 mlx5_txpp_destroy_send_queue(struct mlx5_txpp_wq *wq) 120 { 121 if (wq->sq) 122 claim_zero(mlx5_devx_cmd_destroy(wq->sq)); 123 if (wq->sq_umem) 124 claim_zero(mlx5_glue->devx_umem_dereg(wq->sq_umem)); 125 if (wq->sq_buf) 126 rte_free((void *)(uintptr_t)wq->sq_buf); 127 if (wq->cq) 128 claim_zero(mlx5_devx_cmd_destroy(wq->cq)); 129 if (wq->cq_umem) 130 claim_zero(mlx5_glue->devx_umem_dereg(wq->cq_umem)); 131 if (wq->cq_buf) 132 rte_free((void *)(uintptr_t)wq->cq_buf); 133 memset(wq, 0, sizeof(*wq)); 134 } 135 136 static void 137 mlx5_txpp_destroy_rearm_queue(struct mlx5_dev_ctx_shared *sh) 138 { 139 struct mlx5_txpp_wq *wq = &sh->txpp.rearm_queue; 140 141 mlx5_txpp_destroy_send_queue(wq); 142 } 143 144 static void 145 mlx5_txpp_destroy_clock_queue(struct mlx5_dev_ctx_shared *sh) 146 { 147 struct mlx5_txpp_wq *wq = &sh->txpp.clock_queue; 148 149 mlx5_txpp_destroy_send_queue(wq); 150 if (sh->txpp.tsa) { 151 rte_free(sh->txpp.tsa); 152 sh->txpp.tsa = NULL; 153 } 154 } 155 156 static void 157 mlx5_txpp_doorbell_rearm_queue(struct mlx5_dev_ctx_shared *sh, uint16_t ci) 158 { 159 struct mlx5_txpp_wq *wq = &sh->txpp.rearm_queue; 160 union { 161 uint32_t w32[2]; 162 uint64_t w64; 163 } cs; 164 165 wq->sq_ci = ci + 1; 166 cs.w32[0] = rte_cpu_to_be_32(rte_be_to_cpu_32 167 (wq->wqes[ci & (wq->sq_size - 1)].ctrl[0]) | (ci - 1) << 8); 168 cs.w32[1] = wq->wqes[ci & (wq->sq_size - 1)].ctrl[1]; 169 /* Update SQ doorbell record with new SQ ci. */ 170 rte_compiler_barrier(); 171 *wq->sq_dbrec = rte_cpu_to_be_32(wq->sq_ci); 172 /* Make sure the doorbell record is updated. */ 173 rte_wmb(); 174 /* Write to doorbel register to start processing. */ 175 __mlx5_uar_write64_relaxed(cs.w64, sh->tx_uar->reg_addr, NULL); 176 rte_wmb(); 177 } 178 179 static void 180 mlx5_txpp_fill_cqe_rearm_queue(struct mlx5_dev_ctx_shared *sh) 181 { 182 struct mlx5_txpp_wq *wq = &sh->txpp.rearm_queue; 183 struct mlx5_cqe *cqe = (struct mlx5_cqe *)(uintptr_t)wq->cqes; 184 uint32_t i; 185 186 for (i = 0; i < MLX5_TXPP_REARM_CQ_SIZE; i++) { 187 cqe->op_own = (MLX5_CQE_INVALID << 4) | MLX5_CQE_OWNER_MASK; 188 ++cqe; 189 } 190 } 191 192 static void 193 mlx5_txpp_fill_wqe_rearm_queue(struct mlx5_dev_ctx_shared *sh) 194 { 195 struct mlx5_txpp_wq *wq = &sh->txpp.rearm_queue; 196 struct mlx5_wqe *wqe = (struct mlx5_wqe *)(uintptr_t)wq->wqes; 197 uint32_t i; 198 199 for (i = 0; i < wq->sq_size; i += 2) { 200 struct mlx5_wqe_cseg *cs; 201 struct mlx5_wqe_qseg *qs; 202 uint32_t index; 203 204 /* Build SEND_EN request with slave WQE index. */ 205 cs = &wqe[i + 0].cseg; 206 cs->opcode = RTE_BE32(MLX5_OPCODE_SEND_EN | 0); 207 cs->sq_ds = rte_cpu_to_be_32((wq->sq->id << 8) | 2); 208 cs->flags = RTE_BE32(MLX5_COMP_ALWAYS << 209 MLX5_COMP_MODE_OFFSET); 210 cs->misc = RTE_BE32(0); 211 qs = RTE_PTR_ADD(cs, sizeof(struct mlx5_wqe_cseg)); 212 index = (i * MLX5_TXPP_REARM / 2 + MLX5_TXPP_REARM) & 213 ((1 << MLX5_WQ_INDEX_WIDTH) - 1); 214 qs->max_index = rte_cpu_to_be_32(index); 215 qs->qpn_cqn = rte_cpu_to_be_32(sh->txpp.clock_queue.sq->id); 216 /* Build WAIT request with slave CQE index. */ 217 cs = &wqe[i + 1].cseg; 218 cs->opcode = RTE_BE32(MLX5_OPCODE_WAIT | 0); 219 cs->sq_ds = rte_cpu_to_be_32((wq->sq->id << 8) | 2); 220 cs->flags = RTE_BE32(MLX5_COMP_ONLY_ERR << 221 MLX5_COMP_MODE_OFFSET); 222 cs->misc = RTE_BE32(0); 223 qs = RTE_PTR_ADD(cs, sizeof(struct mlx5_wqe_cseg)); 224 index = (i * MLX5_TXPP_REARM / 2 + MLX5_TXPP_REARM / 2) & 225 ((1 << MLX5_CQ_INDEX_WIDTH) - 1); 226 qs->max_index = rte_cpu_to_be_32(index); 227 qs->qpn_cqn = rte_cpu_to_be_32(sh->txpp.clock_queue.cq->id); 228 } 229 } 230 231 /* Creates the Rearm Queue to fire the requests to Clock Queue in realtime. */ 232 static int 233 mlx5_txpp_create_rearm_queue(struct mlx5_dev_ctx_shared *sh) 234 { 235 struct mlx5_devx_create_sq_attr sq_attr = { 0 }; 236 struct mlx5_devx_modify_sq_attr msq_attr = { 0 }; 237 struct mlx5_devx_cq_attr cq_attr = { 0 }; 238 struct mlx5_txpp_wq *wq = &sh->txpp.rearm_queue; 239 size_t page_size = sysconf(_SC_PAGESIZE); 240 uint32_t umem_size, umem_dbrec; 241 int ret; 242 243 /* Allocate memory buffer for CQEs and doorbell record. */ 244 umem_size = sizeof(struct mlx5_cqe) * MLX5_TXPP_REARM_CQ_SIZE; 245 umem_dbrec = RTE_ALIGN(umem_size, MLX5_DBR_SIZE); 246 umem_size += MLX5_DBR_SIZE; 247 wq->cq_buf = rte_zmalloc_socket(__func__, umem_size, 248 page_size, sh->numa_node); 249 if (!wq->cq_buf) { 250 DRV_LOG(ERR, "Failed to allocate memory for Rearm Queue."); 251 return -ENOMEM; 252 } 253 /* Register allocated buffer in user space with DevX. */ 254 wq->cq_umem = mlx5_glue->devx_umem_reg(sh->ctx, 255 (void *)(uintptr_t)wq->cq_buf, 256 umem_size, 257 IBV_ACCESS_LOCAL_WRITE); 258 if (!wq->cq_umem) { 259 rte_errno = errno; 260 DRV_LOG(ERR, "Failed to register umem for Rearm Queue."); 261 goto error; 262 } 263 /* Create completion queue object for Rearm Queue. */ 264 cq_attr.cqe_size = (sizeof(struct mlx5_cqe) == 128) ? 265 MLX5_CQE_SIZE_128B : MLX5_CQE_SIZE_64B; 266 cq_attr.uar_page_id = sh->tx_uar->page_id; 267 cq_attr.eqn = sh->txpp.eqn; 268 cq_attr.q_umem_valid = 1; 269 cq_attr.q_umem_offset = 0; 270 cq_attr.q_umem_id = mlx5_os_get_umem_id(wq->cq_umem); 271 cq_attr.db_umem_valid = 1; 272 cq_attr.db_umem_offset = umem_dbrec; 273 cq_attr.db_umem_id = mlx5_os_get_umem_id(wq->cq_umem); 274 cq_attr.log_cq_size = rte_log2_u32(MLX5_TXPP_REARM_CQ_SIZE); 275 cq_attr.log_page_size = rte_log2_u32(page_size); 276 wq->cq = mlx5_devx_cmd_create_cq(sh->ctx, &cq_attr); 277 if (!wq->cq) { 278 rte_errno = errno; 279 DRV_LOG(ERR, "Failed to create CQ for Rearm Queue."); 280 goto error; 281 } 282 wq->cq_dbrec = RTE_PTR_ADD(wq->cq_buf, umem_dbrec); 283 wq->cq_ci = 0; 284 wq->arm_sn = 0; 285 /* Mark all CQEs initially as invalid. */ 286 mlx5_txpp_fill_cqe_rearm_queue(sh); 287 /* 288 * Allocate memory buffer for Send Queue WQEs. 289 * There should be no WQE leftovers in the cyclic queue. 290 */ 291 wq->sq_size = MLX5_TXPP_REARM_SQ_SIZE; 292 MLX5_ASSERT(wq->sq_size == (1 << log2above(wq->sq_size))); 293 umem_size = MLX5_WQE_SIZE * wq->sq_size; 294 umem_dbrec = RTE_ALIGN(umem_size, MLX5_DBR_SIZE); 295 umem_size += MLX5_DBR_SIZE; 296 wq->sq_buf = rte_zmalloc_socket(__func__, umem_size, 297 page_size, sh->numa_node); 298 if (!wq->sq_buf) { 299 DRV_LOG(ERR, "Failed to allocate memory for Rearm Queue."); 300 rte_errno = ENOMEM; 301 goto error; 302 } 303 /* Register allocated buffer in user space with DevX. */ 304 wq->sq_umem = mlx5_glue->devx_umem_reg(sh->ctx, 305 (void *)(uintptr_t)wq->sq_buf, 306 umem_size, 307 IBV_ACCESS_LOCAL_WRITE); 308 if (!wq->sq_umem) { 309 rte_errno = errno; 310 DRV_LOG(ERR, "Failed to register umem for Rearm Queue."); 311 goto error; 312 } 313 /* Create send queue object for Rearm Queue. */ 314 sq_attr.state = MLX5_SQC_STATE_RST; 315 sq_attr.tis_lst_sz = 1; 316 sq_attr.tis_num = sh->tis->id; 317 sq_attr.cqn = wq->cq->id; 318 sq_attr.cd_master = 1; 319 sq_attr.wq_attr.uar_page = sh->tx_uar->page_id; 320 sq_attr.wq_attr.wq_type = MLX5_WQ_TYPE_CYCLIC; 321 sq_attr.wq_attr.pd = sh->pdn; 322 sq_attr.wq_attr.log_wq_stride = rte_log2_u32(MLX5_WQE_SIZE); 323 sq_attr.wq_attr.log_wq_sz = rte_log2_u32(wq->sq_size); 324 sq_attr.wq_attr.dbr_umem_valid = 1; 325 sq_attr.wq_attr.dbr_addr = umem_dbrec; 326 sq_attr.wq_attr.dbr_umem_id = mlx5_os_get_umem_id(wq->sq_umem); 327 sq_attr.wq_attr.wq_umem_valid = 1; 328 sq_attr.wq_attr.wq_umem_id = mlx5_os_get_umem_id(wq->sq_umem); 329 sq_attr.wq_attr.wq_umem_offset = 0; 330 wq->sq = mlx5_devx_cmd_create_sq(sh->ctx, &sq_attr); 331 if (!wq->sq) { 332 rte_errno = errno; 333 DRV_LOG(ERR, "Failed to create SQ for Rearm Queue."); 334 goto error; 335 } 336 wq->sq_dbrec = RTE_PTR_ADD(wq->sq_buf, umem_dbrec + 337 MLX5_SND_DBR * sizeof(uint32_t)); 338 /* Build the WQEs in the Send Queue before goto Ready state. */ 339 mlx5_txpp_fill_wqe_rearm_queue(sh); 340 /* Change queue state to ready. */ 341 msq_attr.sq_state = MLX5_SQC_STATE_RST; 342 msq_attr.state = MLX5_SQC_STATE_RDY; 343 ret = mlx5_devx_cmd_modify_sq(wq->sq, &msq_attr); 344 if (ret) { 345 DRV_LOG(ERR, "Failed to set SQ ready state Rearm Queue."); 346 goto error; 347 } 348 return 0; 349 error: 350 ret = -rte_errno; 351 mlx5_txpp_destroy_rearm_queue(sh); 352 rte_errno = -ret; 353 return ret; 354 } 355 356 static void 357 mlx5_txpp_fill_wqe_clock_queue(struct mlx5_dev_ctx_shared *sh) 358 { 359 struct mlx5_txpp_wq *wq = &sh->txpp.clock_queue; 360 struct mlx5_wqe *wqe = (struct mlx5_wqe *)(uintptr_t)wq->wqes; 361 struct mlx5_wqe_cseg *cs = &wqe->cseg; 362 uint32_t wqe_size, opcode, i; 363 uint8_t *dst; 364 365 /* For test purposes fill the WQ with SEND inline packet. */ 366 if (sh->txpp.test) { 367 wqe_size = RTE_ALIGN(MLX5_TXPP_TEST_PKT_SIZE + 368 MLX5_WQE_CSEG_SIZE + 369 2 * MLX5_WQE_ESEG_SIZE - 370 MLX5_ESEG_MIN_INLINE_SIZE, 371 MLX5_WSEG_SIZE); 372 opcode = MLX5_OPCODE_SEND; 373 } else { 374 wqe_size = MLX5_WSEG_SIZE; 375 opcode = MLX5_OPCODE_NOP; 376 } 377 cs->opcode = rte_cpu_to_be_32(opcode | 0); /* Index is ignored. */ 378 cs->sq_ds = rte_cpu_to_be_32((wq->sq->id << 8) | 379 (wqe_size / MLX5_WSEG_SIZE)); 380 cs->flags = RTE_BE32(MLX5_COMP_ALWAYS << MLX5_COMP_MODE_OFFSET); 381 cs->misc = RTE_BE32(0); 382 wqe_size = RTE_ALIGN(wqe_size, MLX5_WQE_SIZE); 383 if (sh->txpp.test) { 384 struct mlx5_wqe_eseg *es = &wqe->eseg; 385 struct rte_ether_hdr *eth_hdr; 386 struct rte_ipv4_hdr *ip_hdr; 387 struct rte_udp_hdr *udp_hdr; 388 389 /* Build the inline test packet pattern. */ 390 MLX5_ASSERT(wqe_size <= MLX5_WQE_SIZE_MAX); 391 MLX5_ASSERT(MLX5_TXPP_TEST_PKT_SIZE >= 392 (sizeof(struct rte_ether_hdr) + 393 sizeof(struct rte_ipv4_hdr))); 394 es->flags = 0; 395 es->cs_flags = MLX5_ETH_WQE_L3_CSUM | MLX5_ETH_WQE_L4_CSUM; 396 es->swp_offs = 0; 397 es->metadata = 0; 398 es->swp_flags = 0; 399 es->mss = 0; 400 es->inline_hdr_sz = RTE_BE16(MLX5_TXPP_TEST_PKT_SIZE); 401 /* Build test packet L2 header (Ethernet). */ 402 dst = (uint8_t *)&es->inline_data; 403 eth_hdr = (struct rte_ether_hdr *)dst; 404 rte_eth_random_addr(ð_hdr->d_addr.addr_bytes[0]); 405 rte_eth_random_addr(ð_hdr->s_addr.addr_bytes[0]); 406 eth_hdr->ether_type = rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV4); 407 /* Build test packet L3 header (IP v4). */ 408 dst += sizeof(struct rte_ether_hdr); 409 ip_hdr = (struct rte_ipv4_hdr *)dst; 410 ip_hdr->version_ihl = RTE_IPV4_VHL_DEF; 411 ip_hdr->type_of_service = 0; 412 ip_hdr->fragment_offset = 0; 413 ip_hdr->time_to_live = 64; 414 ip_hdr->next_proto_id = IPPROTO_UDP; 415 ip_hdr->packet_id = 0; 416 ip_hdr->total_length = RTE_BE16(MLX5_TXPP_TEST_PKT_SIZE - 417 sizeof(struct rte_ether_hdr)); 418 /* use RFC5735 / RFC2544 reserved network test addresses */ 419 ip_hdr->src_addr = RTE_BE32((198U << 24) | (18 << 16) | 420 (0 << 8) | 1); 421 ip_hdr->dst_addr = RTE_BE32((198U << 24) | (18 << 16) | 422 (0 << 8) | 2); 423 if (MLX5_TXPP_TEST_PKT_SIZE < 424 (sizeof(struct rte_ether_hdr) + 425 sizeof(struct rte_ipv4_hdr) + 426 sizeof(struct rte_udp_hdr))) 427 goto wcopy; 428 /* Build test packet L4 header (UDP). */ 429 dst += sizeof(struct rte_ipv4_hdr); 430 udp_hdr = (struct rte_udp_hdr *)dst; 431 udp_hdr->src_port = RTE_BE16(9); /* RFC863 Discard. */ 432 udp_hdr->dst_port = RTE_BE16(9); 433 udp_hdr->dgram_len = RTE_BE16(MLX5_TXPP_TEST_PKT_SIZE - 434 sizeof(struct rte_ether_hdr) - 435 sizeof(struct rte_ipv4_hdr)); 436 udp_hdr->dgram_cksum = 0; 437 /* Fill the test packet data. */ 438 dst += sizeof(struct rte_udp_hdr); 439 for (i = sizeof(struct rte_ether_hdr) + 440 sizeof(struct rte_ipv4_hdr) + 441 sizeof(struct rte_udp_hdr); 442 i < MLX5_TXPP_TEST_PKT_SIZE; i++) 443 *dst++ = (uint8_t)(i & 0xFF); 444 } 445 wcopy: 446 /* Duplicate the pattern to the next WQEs. */ 447 dst = (uint8_t *)(uintptr_t)wq->sq_buf; 448 for (i = 1; i < MLX5_TXPP_CLKQ_SIZE; i++) { 449 dst += wqe_size; 450 rte_memcpy(dst, (void *)(uintptr_t)wq->sq_buf, wqe_size); 451 } 452 } 453 454 /* Creates the Clock Queue for packet pacing, returns zero on success. */ 455 static int 456 mlx5_txpp_create_clock_queue(struct mlx5_dev_ctx_shared *sh) 457 { 458 struct mlx5_devx_create_sq_attr sq_attr = { 0 }; 459 struct mlx5_devx_modify_sq_attr msq_attr = { 0 }; 460 struct mlx5_devx_cq_attr cq_attr = { 0 }; 461 struct mlx5_txpp_wq *wq = &sh->txpp.clock_queue; 462 size_t page_size = sysconf(_SC_PAGESIZE); 463 uint32_t umem_size, umem_dbrec; 464 int ret; 465 466 sh->txpp.tsa = rte_zmalloc_socket(__func__, 467 MLX5_TXPP_REARM_SQ_SIZE * 468 sizeof(struct mlx5_txpp_ts), 469 0, sh->numa_node); 470 if (!sh->txpp.tsa) { 471 DRV_LOG(ERR, "Failed to allocate memory for CQ stats."); 472 return -ENOMEM; 473 } 474 sh->txpp.ts_p = 0; 475 sh->txpp.ts_n = 0; 476 /* Allocate memory buffer for CQEs and doorbell record. */ 477 umem_size = sizeof(struct mlx5_cqe) * MLX5_TXPP_CLKQ_SIZE; 478 umem_dbrec = RTE_ALIGN(umem_size, MLX5_DBR_SIZE); 479 umem_size += MLX5_DBR_SIZE; 480 wq->cq_buf = rte_zmalloc_socket(__func__, umem_size, 481 page_size, sh->numa_node); 482 if (!wq->cq_buf) { 483 DRV_LOG(ERR, "Failed to allocate memory for Clock Queue."); 484 return -ENOMEM; 485 } 486 /* Register allocated buffer in user space with DevX. */ 487 wq->cq_umem = mlx5_glue->devx_umem_reg(sh->ctx, 488 (void *)(uintptr_t)wq->cq_buf, 489 umem_size, 490 IBV_ACCESS_LOCAL_WRITE); 491 if (!wq->cq_umem) { 492 rte_errno = errno; 493 DRV_LOG(ERR, "Failed to register umem for Clock Queue."); 494 goto error; 495 } 496 /* Create completion queue object for Clock Queue. */ 497 cq_attr.cqe_size = (sizeof(struct mlx5_cqe) == 128) ? 498 MLX5_CQE_SIZE_128B : MLX5_CQE_SIZE_64B; 499 cq_attr.use_first_only = 1; 500 cq_attr.overrun_ignore = 1; 501 cq_attr.uar_page_id = sh->tx_uar->page_id; 502 cq_attr.eqn = sh->txpp.eqn; 503 cq_attr.q_umem_valid = 1; 504 cq_attr.q_umem_offset = 0; 505 cq_attr.q_umem_id = wq->cq_umem->umem_id; 506 cq_attr.db_umem_valid = 1; 507 cq_attr.db_umem_offset = umem_dbrec; 508 cq_attr.db_umem_id = wq->cq_umem->umem_id; 509 cq_attr.log_cq_size = rte_log2_u32(MLX5_TXPP_CLKQ_SIZE); 510 cq_attr.log_page_size = rte_log2_u32(page_size); 511 wq->cq = mlx5_devx_cmd_create_cq(sh->ctx, &cq_attr); 512 if (!wq->cq) { 513 rte_errno = errno; 514 DRV_LOG(ERR, "Failed to create CQ for Clock Queue."); 515 goto error; 516 } 517 wq->cq_dbrec = RTE_PTR_ADD(wq->cq_buf, umem_dbrec); 518 wq->cq_ci = 0; 519 /* Allocate memory buffer for Send Queue WQEs. */ 520 if (sh->txpp.test) { 521 wq->sq_size = RTE_ALIGN(MLX5_TXPP_TEST_PKT_SIZE + 522 MLX5_WQE_CSEG_SIZE + 523 2 * MLX5_WQE_ESEG_SIZE - 524 MLX5_ESEG_MIN_INLINE_SIZE, 525 MLX5_WQE_SIZE) / MLX5_WQE_SIZE; 526 wq->sq_size *= MLX5_TXPP_CLKQ_SIZE; 527 } else { 528 wq->sq_size = MLX5_TXPP_CLKQ_SIZE; 529 } 530 /* There should not be WQE leftovers in the cyclic queue. */ 531 MLX5_ASSERT(wq->sq_size == (1 << log2above(wq->sq_size))); 532 umem_size = MLX5_WQE_SIZE * wq->sq_size; 533 umem_dbrec = RTE_ALIGN(umem_size, MLX5_DBR_SIZE); 534 umem_size += MLX5_DBR_SIZE; 535 wq->sq_buf = rte_zmalloc_socket(__func__, umem_size, 536 page_size, sh->numa_node); 537 if (!wq->sq_buf) { 538 DRV_LOG(ERR, "Failed to allocate memory for Clock Queue."); 539 rte_errno = ENOMEM; 540 goto error; 541 } 542 /* Register allocated buffer in user space with DevX. */ 543 wq->sq_umem = mlx5_glue->devx_umem_reg(sh->ctx, 544 (void *)(uintptr_t)wq->sq_buf, 545 umem_size, 546 IBV_ACCESS_LOCAL_WRITE); 547 if (!wq->sq_umem) { 548 rte_errno = errno; 549 DRV_LOG(ERR, "Failed to register umem for Clock Queue."); 550 goto error; 551 } 552 /* Create send queue object for Clock Queue. */ 553 if (sh->txpp.test) { 554 sq_attr.tis_lst_sz = 1; 555 sq_attr.tis_num = sh->tis->id; 556 sq_attr.non_wire = 0; 557 sq_attr.static_sq_wq = 1; 558 } else { 559 sq_attr.non_wire = 1; 560 sq_attr.static_sq_wq = 1; 561 } 562 sq_attr.state = MLX5_SQC_STATE_RST; 563 sq_attr.cqn = wq->cq->id; 564 sq_attr.packet_pacing_rate_limit_index = sh->txpp.pp_id; 565 sq_attr.wq_attr.cd_slave = 1; 566 sq_attr.wq_attr.uar_page = sh->tx_uar->page_id; 567 sq_attr.wq_attr.wq_type = MLX5_WQ_TYPE_CYCLIC; 568 sq_attr.wq_attr.pd = sh->pdn; 569 sq_attr.wq_attr.log_wq_stride = rte_log2_u32(MLX5_WQE_SIZE); 570 sq_attr.wq_attr.log_wq_sz = rte_log2_u32(wq->sq_size); 571 sq_attr.wq_attr.dbr_umem_valid = 1; 572 sq_attr.wq_attr.dbr_addr = umem_dbrec; 573 sq_attr.wq_attr.dbr_umem_id = wq->sq_umem->umem_id; 574 sq_attr.wq_attr.wq_umem_valid = 1; 575 sq_attr.wq_attr.wq_umem_id = wq->sq_umem->umem_id; 576 /* umem_offset must be zero for static_sq_wq queue. */ 577 sq_attr.wq_attr.wq_umem_offset = 0; 578 wq->sq = mlx5_devx_cmd_create_sq(sh->ctx, &sq_attr); 579 if (!wq->sq) { 580 rte_errno = errno; 581 DRV_LOG(ERR, "Failed to create SQ for Clock Queue."); 582 goto error; 583 } 584 wq->sq_dbrec = RTE_PTR_ADD(wq->sq_buf, umem_dbrec + 585 MLX5_SND_DBR * sizeof(uint32_t)); 586 /* Build the WQEs in the Send Queue before goto Ready state. */ 587 mlx5_txpp_fill_wqe_clock_queue(sh); 588 /* Change queue state to ready. */ 589 msq_attr.sq_state = MLX5_SQC_STATE_RST; 590 msq_attr.state = MLX5_SQC_STATE_RDY; 591 wq->sq_ci = 0; 592 ret = mlx5_devx_cmd_modify_sq(wq->sq, &msq_attr); 593 if (ret) { 594 DRV_LOG(ERR, "Failed to set SQ ready state Clock Queue."); 595 goto error; 596 } 597 return 0; 598 error: 599 ret = -rte_errno; 600 mlx5_txpp_destroy_clock_queue(sh); 601 rte_errno = -ret; 602 return ret; 603 } 604 605 /* Enable notification from the Rearm Queue CQ. */ 606 static inline void 607 mlx5_txpp_cq_arm(struct mlx5_dev_ctx_shared *sh) 608 { 609 struct mlx5_txpp_wq *aq = &sh->txpp.rearm_queue; 610 uint32_t arm_sn = aq->arm_sn << MLX5_CQ_SQN_OFFSET; 611 uint32_t db_hi = arm_sn | MLX5_CQ_DBR_CMD_ALL | aq->cq_ci; 612 uint64_t db_be = rte_cpu_to_be_64(((uint64_t)db_hi << 32) | aq->cq->id); 613 uint32_t *addr = RTE_PTR_ADD(sh->tx_uar->base_addr, MLX5_CQ_DOORBELL); 614 615 rte_compiler_barrier(); 616 aq->cq_dbrec[MLX5_CQ_ARM_DB] = rte_cpu_to_be_32(db_hi); 617 rte_wmb(); 618 #ifdef RTE_ARCH_64 619 *(uint64_t *)addr = db_be; 620 #else 621 *(uint32_t *)addr = db_be; 622 rte_io_wmb(); 623 *((uint32_t *)addr + 1) = db_be >> 32; 624 #endif 625 aq->arm_sn++; 626 } 627 628 static inline void 629 mlx5_atomic_read_cqe(rte_int128_t *from, rte_int128_t *ts) 630 { 631 /* 632 * The only CQE of Clock Queue is being continuously 633 * update by hardware with soecified rate. We have to 634 * read timestump and WQE completion index atomically. 635 */ 636 #if defined(RTE_ARCH_X86_64) || defined(RTE_ARCH_ARM64) 637 rte_int128_t src; 638 639 memset(&src, 0, sizeof(src)); 640 *ts = src; 641 /* if (*from == *ts) *from = *src else *ts = *from; */ 642 rte_atomic128_cmp_exchange(from, ts, &src, 0, 643 __ATOMIC_RELAXED, __ATOMIC_RELAXED); 644 #else 645 rte_atomic64_t *cqe = (rte_atomic64_t *)from; 646 647 /* Power architecture does not support 16B compare-and-swap. */ 648 for (;;) { 649 int64_t tm, op; 650 int64_t *ps; 651 652 rte_compiler_barrier(); 653 tm = rte_atomic64_read(cqe + 0); 654 op = rte_atomic64_read(cqe + 1); 655 rte_compiler_barrier(); 656 if (tm != rte_atomic64_read(cqe + 0)) 657 continue; 658 if (op != rte_atomic64_read(cqe + 1)) 659 continue; 660 ps = (int64_t *)ts; 661 ps[0] = tm; 662 ps[1] = op; 663 return; 664 } 665 #endif 666 } 667 668 /* Stores timestamp in the cache structure to share data with datapath. */ 669 static inline void 670 mlx5_txpp_cache_timestamp(struct mlx5_dev_ctx_shared *sh, 671 uint64_t ts, uint64_t ci) 672 { 673 ci = ci << (64 - MLX5_CQ_INDEX_WIDTH); 674 ci |= (ts << MLX5_CQ_INDEX_WIDTH) >> MLX5_CQ_INDEX_WIDTH; 675 rte_compiler_barrier(); 676 rte_atomic64_set(&sh->txpp.ts.ts, ts); 677 rte_atomic64_set(&sh->txpp.ts.ci_ts, ci); 678 rte_wmb(); 679 } 680 681 /* Reads timestamp from Clock Queue CQE and stores in the cache. */ 682 static inline void 683 mlx5_txpp_update_timestamp(struct mlx5_dev_ctx_shared *sh) 684 { 685 struct mlx5_txpp_wq *wq = &sh->txpp.clock_queue; 686 struct mlx5_cqe *cqe = (struct mlx5_cqe *)(uintptr_t)wq->cqes; 687 union { 688 rte_int128_t u128; 689 struct mlx5_cqe_ts cts; 690 } to; 691 uint64_t ts; 692 uint16_t ci; 693 694 static_assert(sizeof(struct mlx5_cqe_ts) == sizeof(rte_int128_t), 695 "Wrong timestamp CQE part size"); 696 mlx5_atomic_read_cqe((rte_int128_t *)&cqe->timestamp, &to.u128); 697 if (to.cts.op_own >> 4) { 698 DRV_LOG(DEBUG, "Clock Queue error sync lost."); 699 rte_atomic32_inc(&sh->txpp.err_clock_queue); 700 sh->txpp.sync_lost = 1; 701 return; 702 } 703 ci = rte_be_to_cpu_16(to.cts.wqe_counter); 704 ts = rte_be_to_cpu_64(to.cts.timestamp); 705 ts = mlx5_txpp_convert_rx_ts(sh, ts); 706 wq->cq_ci += (ci - wq->sq_ci) & UINT16_MAX; 707 wq->sq_ci = ci; 708 mlx5_txpp_cache_timestamp(sh, ts, wq->cq_ci); 709 } 710 711 /* Waits for the first completion on Clock Queue to init timestamp. */ 712 static inline void 713 mlx5_txpp_init_timestamp(struct mlx5_dev_ctx_shared *sh) 714 { 715 struct mlx5_txpp_wq *wq = &sh->txpp.clock_queue; 716 uint32_t wait; 717 718 sh->txpp.ts_p = 0; 719 sh->txpp.ts_n = 0; 720 for (wait = 0; wait < MLX5_TXPP_WAIT_INIT_TS; wait++) { 721 struct timespec onems; 722 723 mlx5_txpp_update_timestamp(sh); 724 if (wq->sq_ci) 725 return; 726 /* Wait one millisecond and try again. */ 727 onems.tv_sec = 0; 728 onems.tv_nsec = NS_PER_S / MS_PER_S; 729 nanosleep(&onems, 0); 730 } 731 DRV_LOG(ERR, "Unable to initialize timestamp."); 732 sh->txpp.sync_lost = 1; 733 } 734 735 #ifdef HAVE_IBV_DEVX_EVENT 736 /* Gather statistics for timestamp from Clock Queue CQE. */ 737 static inline void 738 mlx5_txpp_gather_timestamp(struct mlx5_dev_ctx_shared *sh) 739 { 740 /* Check whether we have a valid timestamp. */ 741 if (!sh->txpp.clock_queue.sq_ci && !sh->txpp.ts_n) 742 return; 743 MLX5_ASSERT(sh->txpp.ts_p < MLX5_TXPP_REARM_SQ_SIZE); 744 sh->txpp.tsa[sh->txpp.ts_p] = sh->txpp.ts; 745 if (++sh->txpp.ts_p >= MLX5_TXPP_REARM_SQ_SIZE) 746 sh->txpp.ts_p = 0; 747 if (sh->txpp.ts_n < MLX5_TXPP_REARM_SQ_SIZE) 748 ++sh->txpp.ts_n; 749 } 750 751 /* Handles Rearm Queue completions in periodic service. */ 752 static __rte_always_inline void 753 mlx5_txpp_handle_rearm_queue(struct mlx5_dev_ctx_shared *sh) 754 { 755 struct mlx5_txpp_wq *wq = &sh->txpp.rearm_queue; 756 uint32_t cq_ci = wq->cq_ci; 757 bool error = false; 758 int ret; 759 760 do { 761 volatile struct mlx5_cqe *cqe; 762 763 cqe = &wq->cqes[cq_ci & (MLX5_TXPP_REARM_CQ_SIZE - 1)]; 764 ret = check_cqe(cqe, MLX5_TXPP_REARM_CQ_SIZE, cq_ci); 765 switch (ret) { 766 case MLX5_CQE_STATUS_ERR: 767 error = true; 768 ++cq_ci; 769 break; 770 case MLX5_CQE_STATUS_SW_OWN: 771 wq->sq_ci += 2; 772 ++cq_ci; 773 break; 774 case MLX5_CQE_STATUS_HW_OWN: 775 break; 776 default: 777 MLX5_ASSERT(false); 778 break; 779 } 780 } while (ret != MLX5_CQE_STATUS_HW_OWN); 781 if (likely(cq_ci != wq->cq_ci)) { 782 /* Check whether we have missed interrupts. */ 783 if (cq_ci - wq->cq_ci != 1) { 784 DRV_LOG(DEBUG, "Rearm Queue missed interrupt."); 785 rte_atomic32_inc(&sh->txpp.err_miss_int); 786 /* Check sync lost on wqe index. */ 787 if (cq_ci - wq->cq_ci >= 788 (((1UL << MLX5_WQ_INDEX_WIDTH) / 789 MLX5_TXPP_REARM) - 1)) 790 error = 1; 791 } 792 /* Update doorbell record to notify hardware. */ 793 rte_compiler_barrier(); 794 *wq->cq_dbrec = rte_cpu_to_be_32(cq_ci); 795 rte_wmb(); 796 wq->cq_ci = cq_ci; 797 /* Fire new requests to Rearm Queue. */ 798 if (error) { 799 DRV_LOG(DEBUG, "Rearm Queue error sync lost."); 800 rte_atomic32_inc(&sh->txpp.err_rearm_queue); 801 sh->txpp.sync_lost = 1; 802 } 803 } 804 } 805 806 /* Handles Clock Queue completions in periodic service. */ 807 static __rte_always_inline void 808 mlx5_txpp_handle_clock_queue(struct mlx5_dev_ctx_shared *sh) 809 { 810 mlx5_txpp_update_timestamp(sh); 811 mlx5_txpp_gather_timestamp(sh); 812 } 813 #endif 814 815 /* Invoked periodically on Rearm Queue completions. */ 816 void 817 mlx5_txpp_interrupt_handler(void *cb_arg) 818 { 819 #ifndef HAVE_IBV_DEVX_EVENT 820 RTE_SET_USED(cb_arg); 821 return; 822 #else 823 struct mlx5_dev_ctx_shared *sh = cb_arg; 824 union { 825 struct mlx5dv_devx_async_event_hdr event_resp; 826 uint8_t buf[sizeof(struct mlx5dv_devx_async_event_hdr) + 128]; 827 } out; 828 829 MLX5_ASSERT(rte_eal_process_type() == RTE_PROC_PRIMARY); 830 /* Process events in the loop. Only rearm completions are expected. */ 831 while (mlx5_glue->devx_get_event 832 (sh->txpp.echan, 833 &out.event_resp, 834 sizeof(out.buf)) >= 835 (ssize_t)sizeof(out.event_resp.cookie)) { 836 mlx5_txpp_handle_rearm_queue(sh); 837 mlx5_txpp_handle_clock_queue(sh); 838 mlx5_txpp_cq_arm(sh); 839 mlx5_txpp_doorbell_rearm_queue 840 (sh, sh->txpp.rearm_queue.sq_ci - 1); 841 } 842 #endif /* HAVE_IBV_DEVX_ASYNC */ 843 } 844 845 static void 846 mlx5_txpp_stop_service(struct mlx5_dev_ctx_shared *sh) 847 { 848 if (!sh->txpp.intr_handle.fd) 849 return; 850 mlx5_intr_callback_unregister(&sh->txpp.intr_handle, 851 mlx5_txpp_interrupt_handler, sh); 852 sh->txpp.intr_handle.fd = 0; 853 } 854 855 /* Attach interrupt handler and fires first request to Rearm Queue. */ 856 static int 857 mlx5_txpp_start_service(struct mlx5_dev_ctx_shared *sh) 858 { 859 uint16_t event_nums[1] = {0}; 860 int flags; 861 int ret; 862 863 rte_atomic32_set(&sh->txpp.err_miss_int, 0); 864 rte_atomic32_set(&sh->txpp.err_rearm_queue, 0); 865 rte_atomic32_set(&sh->txpp.err_clock_queue, 0); 866 rte_atomic32_set(&sh->txpp.err_ts_past, 0); 867 rte_atomic32_set(&sh->txpp.err_ts_future, 0); 868 /* Attach interrupt handler to process Rearm Queue completions. */ 869 flags = fcntl(sh->txpp.echan->fd, F_GETFL); 870 ret = fcntl(sh->txpp.echan->fd, F_SETFL, flags | O_NONBLOCK); 871 if (ret) { 872 DRV_LOG(ERR, "Failed to change event channel FD."); 873 rte_errno = errno; 874 return -rte_errno; 875 } 876 memset(&sh->txpp.intr_handle, 0, sizeof(sh->txpp.intr_handle)); 877 sh->txpp.intr_handle.fd = sh->txpp.echan->fd; 878 sh->txpp.intr_handle.type = RTE_INTR_HANDLE_EXT; 879 if (rte_intr_callback_register(&sh->txpp.intr_handle, 880 mlx5_txpp_interrupt_handler, sh)) { 881 sh->txpp.intr_handle.fd = 0; 882 DRV_LOG(ERR, "Failed to register CQE interrupt %d.", rte_errno); 883 return -rte_errno; 884 } 885 /* Subscribe CQ event to the event channel controlled by the driver. */ 886 ret = mlx5_glue->devx_subscribe_devx_event(sh->txpp.echan, 887 sh->txpp.rearm_queue.cq->obj, 888 sizeof(event_nums), 889 event_nums, 0); 890 if (ret) { 891 DRV_LOG(ERR, "Failed to subscribe CQE event."); 892 rte_errno = errno; 893 return -errno; 894 } 895 /* Enable interrupts in the CQ. */ 896 mlx5_txpp_cq_arm(sh); 897 /* Fire the first request on Rearm Queue. */ 898 mlx5_txpp_doorbell_rearm_queue(sh, sh->txpp.rearm_queue.sq_size - 1); 899 mlx5_txpp_init_timestamp(sh); 900 return 0; 901 } 902 903 /* 904 * The routine initializes the packet pacing infrastructure: 905 * - allocates PP context 906 * - Clock CQ/SQ 907 * - Rearm CQ/SQ 908 * - attaches rearm interrupt handler 909 * - starts Clock Queue 910 * 911 * Returns 0 on success, negative otherwise 912 */ 913 static int 914 mlx5_txpp_create(struct mlx5_dev_ctx_shared *sh, struct mlx5_priv *priv) 915 { 916 int tx_pp = priv->config.tx_pp; 917 int ret; 918 919 /* Store the requested pacing parameters. */ 920 sh->txpp.tick = tx_pp >= 0 ? tx_pp : -tx_pp; 921 sh->txpp.test = !!(tx_pp < 0); 922 sh->txpp.skew = priv->config.tx_skew; 923 sh->txpp.freq = priv->config.hca_attr.dev_freq_khz; 924 ret = mlx5_txpp_create_eqn(sh); 925 if (ret) 926 goto exit; 927 ret = mlx5_txpp_alloc_pp_index(sh); 928 if (ret) 929 goto exit; 930 ret = mlx5_txpp_create_clock_queue(sh); 931 if (ret) 932 goto exit; 933 ret = mlx5_txpp_create_rearm_queue(sh); 934 if (ret) 935 goto exit; 936 ret = mlx5_txpp_start_service(sh); 937 if (ret) 938 goto exit; 939 exit: 940 if (ret) { 941 mlx5_txpp_stop_service(sh); 942 mlx5_txpp_destroy_rearm_queue(sh); 943 mlx5_txpp_destroy_clock_queue(sh); 944 mlx5_txpp_free_pp_index(sh); 945 mlx5_txpp_destroy_eqn(sh); 946 sh->txpp.tick = 0; 947 sh->txpp.test = 0; 948 sh->txpp.skew = 0; 949 } 950 return ret; 951 } 952 953 /* 954 * The routine destroys the packet pacing infrastructure: 955 * - detaches rearm interrupt handler 956 * - Rearm CQ/SQ 957 * - Clock CQ/SQ 958 * - PP context 959 */ 960 static void 961 mlx5_txpp_destroy(struct mlx5_dev_ctx_shared *sh) 962 { 963 mlx5_txpp_stop_service(sh); 964 mlx5_txpp_destroy_rearm_queue(sh); 965 mlx5_txpp_destroy_clock_queue(sh); 966 mlx5_txpp_free_pp_index(sh); 967 mlx5_txpp_destroy_eqn(sh); 968 sh->txpp.tick = 0; 969 sh->txpp.test = 0; 970 sh->txpp.skew = 0; 971 } 972 973 /** 974 * Creates and starts packet pacing infrastructure on specified device. 975 * 976 * @param dev 977 * Pointer to Ethernet device structure. 978 * 979 * @return 980 * 0 on success, a negative errno value otherwise and rte_errno is set. 981 */ 982 int 983 mlx5_txpp_start(struct rte_eth_dev *dev) 984 { 985 struct mlx5_priv *priv = dev->data->dev_private; 986 struct mlx5_dev_ctx_shared *sh = priv->sh; 987 int err = 0; 988 int ret; 989 990 if (!priv->config.tx_pp) { 991 /* Packet pacing is not requested for the device. */ 992 MLX5_ASSERT(priv->txpp_en == 0); 993 return 0; 994 } 995 if (priv->txpp_en) { 996 /* Packet pacing is already enabled for the device. */ 997 MLX5_ASSERT(sh->txpp.refcnt); 998 return 0; 999 } 1000 if (priv->config.tx_pp > 0) { 1001 ret = rte_mbuf_dynflag_lookup 1002 (RTE_MBUF_DYNFLAG_TX_TIMESTAMP_NAME, NULL); 1003 if (ret < 0) 1004 return 0; 1005 } 1006 ret = pthread_mutex_lock(&sh->txpp.mutex); 1007 MLX5_ASSERT(!ret); 1008 RTE_SET_USED(ret); 1009 if (sh->txpp.refcnt) { 1010 priv->txpp_en = 1; 1011 ++sh->txpp.refcnt; 1012 } else { 1013 err = mlx5_txpp_create(sh, priv); 1014 if (!err) { 1015 MLX5_ASSERT(sh->txpp.tick); 1016 priv->txpp_en = 1; 1017 sh->txpp.refcnt = 1; 1018 } else { 1019 rte_errno = -err; 1020 } 1021 } 1022 ret = pthread_mutex_unlock(&sh->txpp.mutex); 1023 MLX5_ASSERT(!ret); 1024 RTE_SET_USED(ret); 1025 return err; 1026 } 1027 1028 /** 1029 * Stops and destroys packet pacing infrastructure on specified device. 1030 * 1031 * @param dev 1032 * Pointer to Ethernet device structure. 1033 * 1034 * @return 1035 * 0 on success, a negative errno value otherwise and rte_errno is set. 1036 */ 1037 void 1038 mlx5_txpp_stop(struct rte_eth_dev *dev) 1039 { 1040 struct mlx5_priv *priv = dev->data->dev_private; 1041 struct mlx5_dev_ctx_shared *sh = priv->sh; 1042 int ret; 1043 1044 if (!priv->txpp_en) { 1045 /* Packet pacing is already disabled for the device. */ 1046 return; 1047 } 1048 priv->txpp_en = 0; 1049 ret = pthread_mutex_lock(&sh->txpp.mutex); 1050 MLX5_ASSERT(!ret); 1051 RTE_SET_USED(ret); 1052 MLX5_ASSERT(sh->txpp.refcnt); 1053 if (!sh->txpp.refcnt || --sh->txpp.refcnt) 1054 return; 1055 /* No references any more, do actual destroy. */ 1056 mlx5_txpp_destroy(sh); 1057 ret = pthread_mutex_unlock(&sh->txpp.mutex); 1058 MLX5_ASSERT(!ret); 1059 RTE_SET_USED(ret); 1060 } 1061 1062 /* 1063 * Read the current clock counter of an Ethernet device 1064 * 1065 * This returns the current raw clock value of an Ethernet device. It is 1066 * a raw amount of ticks, with no given time reference. 1067 * The value returned here is from the same clock than the one 1068 * filling timestamp field of Rx/Tx packets when using hardware timestamp 1069 * offload. Therefore it can be used to compute a precise conversion of 1070 * the device clock to the real time. 1071 * 1072 * @param dev 1073 * Pointer to Ethernet device structure. 1074 * @param clock 1075 * Pointer to the uint64_t that holds the raw clock value. 1076 * 1077 * @return 1078 * - 0: Success. 1079 * - -ENOTSUP: The function is not supported in this mode. Requires 1080 * packet pacing module configured and started (tx_pp devarg) 1081 */ 1082 int 1083 mlx5_txpp_read_clock(struct rte_eth_dev *dev, uint64_t *timestamp) 1084 { 1085 struct mlx5_priv *priv = dev->data->dev_private; 1086 struct mlx5_dev_ctx_shared *sh = priv->sh; 1087 int ret; 1088 1089 if (sh->txpp.refcnt) { 1090 struct mlx5_txpp_wq *wq = &sh->txpp.clock_queue; 1091 struct mlx5_cqe *cqe = (struct mlx5_cqe *)(uintptr_t)wq->cqes; 1092 union { 1093 rte_int128_t u128; 1094 struct mlx5_cqe_ts cts; 1095 } to; 1096 uint64_t ts; 1097 1098 mlx5_atomic_read_cqe((rte_int128_t *)&cqe->timestamp, &to.u128); 1099 if (to.cts.op_own >> 4) { 1100 DRV_LOG(DEBUG, "Clock Queue error sync lost."); 1101 rte_atomic32_inc(&sh->txpp.err_clock_queue); 1102 sh->txpp.sync_lost = 1; 1103 return -EIO; 1104 } 1105 ts = rte_be_to_cpu_64(to.cts.timestamp); 1106 ts = mlx5_txpp_convert_rx_ts(sh, ts); 1107 *timestamp = ts; 1108 return 0; 1109 } 1110 /* Not supported in isolated mode - kernel does not see the CQEs. */ 1111 if (priv->isolated || rte_eal_process_type() != RTE_PROC_PRIMARY) 1112 return -ENOTSUP; 1113 ret = mlx5_read_clock(dev, timestamp); 1114 return ret; 1115 } 1116