1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2020 Mellanox Technologies, Ltd 3 */ 4 #include <fcntl.h> 5 #include <stdint.h> 6 7 #include <rte_ether.h> 8 #include <rte_ethdev_driver.h> 9 #include <rte_interrupts.h> 10 #include <rte_alarm.h> 11 #include <rte_malloc.h> 12 #include <rte_cycles.h> 13 #include <rte_eal_paging.h> 14 15 #include <mlx5_malloc.h> 16 #include <mlx5_common_devx.h> 17 18 #include "mlx5.h" 19 #include "mlx5_rxtx.h" 20 #include "mlx5_common_os.h" 21 22 static_assert(sizeof(struct mlx5_cqe_ts) == sizeof(rte_int128_t), 23 "Wrong timestamp CQE part size"); 24 25 static const char * const mlx5_txpp_stat_names[] = { 26 "tx_pp_missed_interrupt_errors", /* Missed service interrupt. */ 27 "tx_pp_rearm_queue_errors", /* Rearm Queue errors. */ 28 "tx_pp_clock_queue_errors", /* Clock Queue errors. */ 29 "tx_pp_timestamp_past_errors", /* Timestamp in the past. */ 30 "tx_pp_timestamp_future_errors", /* Timestamp in the distant future. */ 31 "tx_pp_jitter", /* Timestamp jitter (one Clock Queue completion). */ 32 "tx_pp_wander", /* Timestamp wander (half of Clock Queue CQEs). */ 33 "tx_pp_sync_lost", /* Scheduling synchronization lost. */ 34 }; 35 36 /* Destroy Event Queue Notification Channel. */ 37 static void 38 mlx5_txpp_destroy_event_channel(struct mlx5_dev_ctx_shared *sh) 39 { 40 if (sh->txpp.echan) { 41 mlx5_os_devx_destroy_event_channel(sh->txpp.echan); 42 sh->txpp.echan = NULL; 43 } 44 } 45 46 /* Create Event Queue Notification Channel. */ 47 static int 48 mlx5_txpp_create_event_channel(struct mlx5_dev_ctx_shared *sh) 49 { 50 MLX5_ASSERT(!sh->txpp.echan); 51 sh->txpp.echan = mlx5_os_devx_create_event_channel(sh->ctx, 52 MLX5DV_DEVX_CREATE_EVENT_CHANNEL_FLAGS_OMIT_EV_DATA); 53 if (!sh->txpp.echan) { 54 rte_errno = errno; 55 DRV_LOG(ERR, "Failed to create event channel %d.", rte_errno); 56 return -rte_errno; 57 } 58 return 0; 59 } 60 61 static void 62 mlx5_txpp_free_pp_index(struct mlx5_dev_ctx_shared *sh) 63 { 64 #ifdef HAVE_MLX5DV_PP_ALLOC 65 if (sh->txpp.pp) { 66 mlx5_glue->dv_free_pp(sh->txpp.pp); 67 sh->txpp.pp = NULL; 68 sh->txpp.pp_id = 0; 69 } 70 #else 71 RTE_SET_USED(sh); 72 DRV_LOG(ERR, "Freeing pacing index is not supported."); 73 #endif 74 } 75 76 /* Allocate Packet Pacing index from kernel via mlx5dv call. */ 77 static int 78 mlx5_txpp_alloc_pp_index(struct mlx5_dev_ctx_shared *sh) 79 { 80 #ifdef HAVE_MLX5DV_PP_ALLOC 81 uint32_t pp[MLX5_ST_SZ_DW(set_pp_rate_limit_context)]; 82 uint64_t rate; 83 84 MLX5_ASSERT(!sh->txpp.pp); 85 memset(&pp, 0, sizeof(pp)); 86 rate = NS_PER_S / sh->txpp.tick; 87 if (rate * sh->txpp.tick != NS_PER_S) 88 DRV_LOG(WARNING, "Packet pacing frequency is not precise."); 89 if (sh->txpp.test) { 90 uint32_t len; 91 92 len = RTE_MAX(MLX5_TXPP_TEST_PKT_SIZE, 93 (size_t)RTE_ETHER_MIN_LEN); 94 MLX5_SET(set_pp_rate_limit_context, &pp, 95 burst_upper_bound, len); 96 MLX5_SET(set_pp_rate_limit_context, &pp, 97 typical_packet_size, len); 98 /* Convert packets per second into kilobits. */ 99 rate = (rate * len) / (1000ul / CHAR_BIT); 100 DRV_LOG(INFO, "Packet pacing rate set to %" PRIu64, rate); 101 } 102 MLX5_SET(set_pp_rate_limit_context, &pp, rate_limit, rate); 103 MLX5_SET(set_pp_rate_limit_context, &pp, rate_mode, 104 sh->txpp.test ? MLX5_DATA_RATE : MLX5_WQE_RATE); 105 sh->txpp.pp = mlx5_glue->dv_alloc_pp 106 (sh->ctx, sizeof(pp), &pp, 107 MLX5DV_PP_ALLOC_FLAGS_DEDICATED_INDEX); 108 if (sh->txpp.pp == NULL) { 109 DRV_LOG(ERR, "Failed to allocate packet pacing index."); 110 rte_errno = errno; 111 return -errno; 112 } 113 if (!((struct mlx5dv_pp *)sh->txpp.pp)->index) { 114 DRV_LOG(ERR, "Zero packet pacing index allocated."); 115 mlx5_txpp_free_pp_index(sh); 116 rte_errno = ENOTSUP; 117 return -ENOTSUP; 118 } 119 sh->txpp.pp_id = ((struct mlx5dv_pp *)(sh->txpp.pp))->index; 120 return 0; 121 #else 122 RTE_SET_USED(sh); 123 DRV_LOG(ERR, "Allocating pacing index is not supported."); 124 rte_errno = ENOTSUP; 125 return -ENOTSUP; 126 #endif 127 } 128 129 static void 130 mlx5_txpp_destroy_send_queue(struct mlx5_txpp_wq *wq) 131 { 132 mlx5_devx_sq_destroy(&wq->sq_obj); 133 mlx5_devx_cq_destroy(&wq->cq_obj); 134 memset(wq, 0, sizeof(*wq)); 135 } 136 137 static void 138 mlx5_txpp_destroy_rearm_queue(struct mlx5_dev_ctx_shared *sh) 139 { 140 struct mlx5_txpp_wq *wq = &sh->txpp.rearm_queue; 141 142 mlx5_txpp_destroy_send_queue(wq); 143 } 144 145 static void 146 mlx5_txpp_destroy_clock_queue(struct mlx5_dev_ctx_shared *sh) 147 { 148 struct mlx5_txpp_wq *wq = &sh->txpp.clock_queue; 149 150 mlx5_txpp_destroy_send_queue(wq); 151 if (sh->txpp.tsa) { 152 mlx5_free(sh->txpp.tsa); 153 sh->txpp.tsa = NULL; 154 } 155 } 156 157 static void 158 mlx5_txpp_doorbell_rearm_queue(struct mlx5_dev_ctx_shared *sh, uint16_t ci) 159 { 160 struct mlx5_txpp_wq *wq = &sh->txpp.rearm_queue; 161 struct mlx5_wqe *wqe = (struct mlx5_wqe *)(uintptr_t)wq->sq_obj.wqes; 162 union { 163 uint32_t w32[2]; 164 uint64_t w64; 165 } cs; 166 void *reg_addr; 167 168 wq->sq_ci = ci + 1; 169 cs.w32[0] = rte_cpu_to_be_32(rte_be_to_cpu_32 170 (wqe[ci & (wq->sq_size - 1)].ctrl[0]) | (ci - 1) << 8); 171 cs.w32[1] = wqe[ci & (wq->sq_size - 1)].ctrl[1]; 172 /* Update SQ doorbell record with new SQ ci. */ 173 rte_compiler_barrier(); 174 *wq->sq_obj.db_rec = rte_cpu_to_be_32(wq->sq_ci); 175 /* Make sure the doorbell record is updated. */ 176 rte_wmb(); 177 /* Write to doorbel register to start processing. */ 178 reg_addr = mlx5_os_get_devx_uar_reg_addr(sh->tx_uar); 179 __mlx5_uar_write64_relaxed(cs.w64, reg_addr, NULL); 180 rte_wmb(); 181 } 182 183 static void 184 mlx5_txpp_fill_wqe_rearm_queue(struct mlx5_dev_ctx_shared *sh) 185 { 186 struct mlx5_txpp_wq *wq = &sh->txpp.rearm_queue; 187 struct mlx5_wqe *wqe = (struct mlx5_wqe *)(uintptr_t)wq->sq_obj.wqes; 188 uint32_t i; 189 190 for (i = 0; i < wq->sq_size; i += 2) { 191 struct mlx5_wqe_cseg *cs; 192 struct mlx5_wqe_qseg *qs; 193 uint32_t index; 194 195 /* Build SEND_EN request with slave WQE index. */ 196 cs = &wqe[i + 0].cseg; 197 cs->opcode = RTE_BE32(MLX5_OPCODE_SEND_EN | 0); 198 cs->sq_ds = rte_cpu_to_be_32((wq->sq_obj.sq->id << 8) | 2); 199 cs->flags = RTE_BE32(MLX5_COMP_ALWAYS << 200 MLX5_COMP_MODE_OFFSET); 201 cs->misc = RTE_BE32(0); 202 qs = RTE_PTR_ADD(cs, sizeof(struct mlx5_wqe_cseg)); 203 index = (i * MLX5_TXPP_REARM / 2 + MLX5_TXPP_REARM) & 204 ((1 << MLX5_WQ_INDEX_WIDTH) - 1); 205 qs->max_index = rte_cpu_to_be_32(index); 206 qs->qpn_cqn = 207 rte_cpu_to_be_32(sh->txpp.clock_queue.sq_obj.sq->id); 208 /* Build WAIT request with slave CQE index. */ 209 cs = &wqe[i + 1].cseg; 210 cs->opcode = RTE_BE32(MLX5_OPCODE_WAIT | 0); 211 cs->sq_ds = rte_cpu_to_be_32((wq->sq_obj.sq->id << 8) | 2); 212 cs->flags = RTE_BE32(MLX5_COMP_ONLY_ERR << 213 MLX5_COMP_MODE_OFFSET); 214 cs->misc = RTE_BE32(0); 215 qs = RTE_PTR_ADD(cs, sizeof(struct mlx5_wqe_cseg)); 216 index = (i * MLX5_TXPP_REARM / 2 + MLX5_TXPP_REARM / 2) & 217 ((1 << MLX5_CQ_INDEX_WIDTH) - 1); 218 qs->max_index = rte_cpu_to_be_32(index); 219 qs->qpn_cqn = 220 rte_cpu_to_be_32(sh->txpp.clock_queue.cq_obj.cq->id); 221 } 222 } 223 224 /* Creates the Rearm Queue to fire the requests to Clock Queue in realtime. */ 225 static int 226 mlx5_txpp_create_rearm_queue(struct mlx5_dev_ctx_shared *sh) 227 { 228 struct mlx5_devx_create_sq_attr sq_attr = { 229 .cd_master = 1, 230 .state = MLX5_SQC_STATE_RST, 231 .tis_lst_sz = 1, 232 .tis_num = sh->tis->id, 233 .wq_attr = (struct mlx5_devx_wq_attr){ 234 .pd = sh->pdn, 235 .uar_page = mlx5_os_get_devx_uar_page_id(sh->tx_uar), 236 }, 237 }; 238 struct mlx5_devx_modify_sq_attr msq_attr = { 0 }; 239 struct mlx5_devx_cq_attr cq_attr = { 240 .uar_page_id = mlx5_os_get_devx_uar_page_id(sh->tx_uar), 241 }; 242 struct mlx5_txpp_wq *wq = &sh->txpp.rearm_queue; 243 int ret; 244 245 /* Create completion queue object for Rearm Queue. */ 246 ret = mlx5_devx_cq_create(sh->ctx, &wq->cq_obj, 247 log2above(MLX5_TXPP_REARM_CQ_SIZE), &cq_attr, 248 sh->numa_node); 249 if (ret) { 250 DRV_LOG(ERR, "Failed to create CQ for Rearm Queue."); 251 return ret; 252 } 253 wq->cq_ci = 0; 254 wq->arm_sn = 0; 255 wq->sq_size = MLX5_TXPP_REARM_SQ_SIZE; 256 MLX5_ASSERT(wq->sq_size == (1 << log2above(wq->sq_size))); 257 /* Create send queue object for Rearm Queue. */ 258 sq_attr.cqn = wq->cq_obj.cq->id; 259 /* There should be no WQE leftovers in the cyclic queue. */ 260 ret = mlx5_devx_sq_create(sh->ctx, &wq->sq_obj, 261 log2above(MLX5_TXPP_REARM_SQ_SIZE), &sq_attr, 262 sh->numa_node); 263 if (ret) { 264 rte_errno = errno; 265 DRV_LOG(ERR, "Failed to create SQ for Rearm Queue."); 266 goto error; 267 } 268 /* Build the WQEs in the Send Queue before goto Ready state. */ 269 mlx5_txpp_fill_wqe_rearm_queue(sh); 270 /* Change queue state to ready. */ 271 msq_attr.sq_state = MLX5_SQC_STATE_RST; 272 msq_attr.state = MLX5_SQC_STATE_RDY; 273 ret = mlx5_devx_cmd_modify_sq(wq->sq_obj.sq, &msq_attr); 274 if (ret) { 275 DRV_LOG(ERR, "Failed to set SQ ready state Rearm Queue."); 276 goto error; 277 } 278 return 0; 279 error: 280 ret = -rte_errno; 281 mlx5_txpp_destroy_rearm_queue(sh); 282 rte_errno = -ret; 283 return ret; 284 } 285 286 static void 287 mlx5_txpp_fill_wqe_clock_queue(struct mlx5_dev_ctx_shared *sh) 288 { 289 struct mlx5_txpp_wq *wq = &sh->txpp.clock_queue; 290 struct mlx5_wqe *wqe = (struct mlx5_wqe *)(uintptr_t)wq->sq_obj.wqes; 291 struct mlx5_wqe_cseg *cs = &wqe->cseg; 292 uint32_t wqe_size, opcode, i; 293 uint8_t *dst; 294 295 /* For test purposes fill the WQ with SEND inline packet. */ 296 if (sh->txpp.test) { 297 wqe_size = RTE_ALIGN(MLX5_TXPP_TEST_PKT_SIZE + 298 MLX5_WQE_CSEG_SIZE + 299 2 * MLX5_WQE_ESEG_SIZE - 300 MLX5_ESEG_MIN_INLINE_SIZE, 301 MLX5_WSEG_SIZE); 302 opcode = MLX5_OPCODE_SEND; 303 } else { 304 wqe_size = MLX5_WSEG_SIZE; 305 opcode = MLX5_OPCODE_NOP; 306 } 307 cs->opcode = rte_cpu_to_be_32(opcode | 0); /* Index is ignored. */ 308 cs->sq_ds = rte_cpu_to_be_32((wq->sq_obj.sq->id << 8) | 309 (wqe_size / MLX5_WSEG_SIZE)); 310 cs->flags = RTE_BE32(MLX5_COMP_ALWAYS << MLX5_COMP_MODE_OFFSET); 311 cs->misc = RTE_BE32(0); 312 wqe_size = RTE_ALIGN(wqe_size, MLX5_WQE_SIZE); 313 if (sh->txpp.test) { 314 struct mlx5_wqe_eseg *es = &wqe->eseg; 315 struct rte_ether_hdr *eth_hdr; 316 struct rte_ipv4_hdr *ip_hdr; 317 struct rte_udp_hdr *udp_hdr; 318 319 /* Build the inline test packet pattern. */ 320 MLX5_ASSERT(wqe_size <= MLX5_WQE_SIZE_MAX); 321 MLX5_ASSERT(MLX5_TXPP_TEST_PKT_SIZE >= 322 (sizeof(struct rte_ether_hdr) + 323 sizeof(struct rte_ipv4_hdr))); 324 es->flags = 0; 325 es->cs_flags = MLX5_ETH_WQE_L3_CSUM | MLX5_ETH_WQE_L4_CSUM; 326 es->swp_offs = 0; 327 es->metadata = 0; 328 es->swp_flags = 0; 329 es->mss = 0; 330 es->inline_hdr_sz = RTE_BE16(MLX5_TXPP_TEST_PKT_SIZE); 331 /* Build test packet L2 header (Ethernet). */ 332 dst = (uint8_t *)&es->inline_data; 333 eth_hdr = (struct rte_ether_hdr *)dst; 334 rte_eth_random_addr(ð_hdr->d_addr.addr_bytes[0]); 335 rte_eth_random_addr(ð_hdr->s_addr.addr_bytes[0]); 336 eth_hdr->ether_type = rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV4); 337 /* Build test packet L3 header (IP v4). */ 338 dst += sizeof(struct rte_ether_hdr); 339 ip_hdr = (struct rte_ipv4_hdr *)dst; 340 ip_hdr->version_ihl = RTE_IPV4_VHL_DEF; 341 ip_hdr->type_of_service = 0; 342 ip_hdr->fragment_offset = 0; 343 ip_hdr->time_to_live = 64; 344 ip_hdr->next_proto_id = IPPROTO_UDP; 345 ip_hdr->packet_id = 0; 346 ip_hdr->total_length = RTE_BE16(MLX5_TXPP_TEST_PKT_SIZE - 347 sizeof(struct rte_ether_hdr)); 348 /* use RFC5735 / RFC2544 reserved network test addresses */ 349 ip_hdr->src_addr = RTE_BE32((198U << 24) | (18 << 16) | 350 (0 << 8) | 1); 351 ip_hdr->dst_addr = RTE_BE32((198U << 24) | (18 << 16) | 352 (0 << 8) | 2); 353 if (MLX5_TXPP_TEST_PKT_SIZE < 354 (sizeof(struct rte_ether_hdr) + 355 sizeof(struct rte_ipv4_hdr) + 356 sizeof(struct rte_udp_hdr))) 357 goto wcopy; 358 /* Build test packet L4 header (UDP). */ 359 dst += sizeof(struct rte_ipv4_hdr); 360 udp_hdr = (struct rte_udp_hdr *)dst; 361 udp_hdr->src_port = RTE_BE16(9); /* RFC863 Discard. */ 362 udp_hdr->dst_port = RTE_BE16(9); 363 udp_hdr->dgram_len = RTE_BE16(MLX5_TXPP_TEST_PKT_SIZE - 364 sizeof(struct rte_ether_hdr) - 365 sizeof(struct rte_ipv4_hdr)); 366 udp_hdr->dgram_cksum = 0; 367 /* Fill the test packet data. */ 368 dst += sizeof(struct rte_udp_hdr); 369 for (i = sizeof(struct rte_ether_hdr) + 370 sizeof(struct rte_ipv4_hdr) + 371 sizeof(struct rte_udp_hdr); 372 i < MLX5_TXPP_TEST_PKT_SIZE; i++) 373 *dst++ = (uint8_t)(i & 0xFF); 374 } 375 wcopy: 376 /* Duplicate the pattern to the next WQEs. */ 377 dst = (uint8_t *)(uintptr_t)wq->sq_obj.umem_buf; 378 for (i = 1; i < MLX5_TXPP_CLKQ_SIZE; i++) { 379 dst += wqe_size; 380 rte_memcpy(dst, (void *)(uintptr_t)wq->sq_obj.umem_buf, 381 wqe_size); 382 } 383 } 384 385 /* Creates the Clock Queue for packet pacing, returns zero on success. */ 386 static int 387 mlx5_txpp_create_clock_queue(struct mlx5_dev_ctx_shared *sh) 388 { 389 struct mlx5_devx_create_sq_attr sq_attr = { 0 }; 390 struct mlx5_devx_modify_sq_attr msq_attr = { 0 }; 391 struct mlx5_devx_cq_attr cq_attr = { 392 .use_first_only = 1, 393 .overrun_ignore = 1, 394 .uar_page_id = mlx5_os_get_devx_uar_page_id(sh->tx_uar), 395 }; 396 struct mlx5_txpp_wq *wq = &sh->txpp.clock_queue; 397 int ret; 398 399 sh->txpp.tsa = mlx5_malloc(MLX5_MEM_RTE | MLX5_MEM_ZERO, 400 MLX5_TXPP_REARM_SQ_SIZE * 401 sizeof(struct mlx5_txpp_ts), 402 0, sh->numa_node); 403 if (!sh->txpp.tsa) { 404 DRV_LOG(ERR, "Failed to allocate memory for CQ stats."); 405 return -ENOMEM; 406 } 407 sh->txpp.ts_p = 0; 408 sh->txpp.ts_n = 0; 409 /* Create completion queue object for Clock Queue. */ 410 ret = mlx5_devx_cq_create(sh->ctx, &wq->cq_obj, 411 log2above(MLX5_TXPP_CLKQ_SIZE), &cq_attr, 412 sh->numa_node); 413 if (ret) { 414 DRV_LOG(ERR, "Failed to create CQ for Clock Queue."); 415 goto error; 416 } 417 wq->cq_ci = 0; 418 /* Allocate memory buffer for Send Queue WQEs. */ 419 if (sh->txpp.test) { 420 wq->sq_size = RTE_ALIGN(MLX5_TXPP_TEST_PKT_SIZE + 421 MLX5_WQE_CSEG_SIZE + 422 2 * MLX5_WQE_ESEG_SIZE - 423 MLX5_ESEG_MIN_INLINE_SIZE, 424 MLX5_WQE_SIZE) / MLX5_WQE_SIZE; 425 wq->sq_size *= MLX5_TXPP_CLKQ_SIZE; 426 } else { 427 wq->sq_size = MLX5_TXPP_CLKQ_SIZE; 428 } 429 /* There should not be WQE leftovers in the cyclic queue. */ 430 MLX5_ASSERT(wq->sq_size == (1 << log2above(wq->sq_size))); 431 /* Create send queue object for Clock Queue. */ 432 if (sh->txpp.test) { 433 sq_attr.tis_lst_sz = 1; 434 sq_attr.tis_num = sh->tis->id; 435 sq_attr.non_wire = 0; 436 sq_attr.static_sq_wq = 1; 437 } else { 438 sq_attr.non_wire = 1; 439 sq_attr.static_sq_wq = 1; 440 } 441 sq_attr.cqn = wq->cq_obj.cq->id; 442 sq_attr.packet_pacing_rate_limit_index = sh->txpp.pp_id; 443 sq_attr.wq_attr.cd_slave = 1; 444 sq_attr.wq_attr.uar_page = mlx5_os_get_devx_uar_page_id(sh->tx_uar); 445 sq_attr.wq_attr.pd = sh->pdn; 446 ret = mlx5_devx_sq_create(sh->ctx, &wq->sq_obj, log2above(wq->sq_size), 447 &sq_attr, sh->numa_node); 448 if (ret) { 449 rte_errno = errno; 450 DRV_LOG(ERR, "Failed to create SQ for Clock Queue."); 451 goto error; 452 } 453 /* Build the WQEs in the Send Queue before goto Ready state. */ 454 mlx5_txpp_fill_wqe_clock_queue(sh); 455 /* Change queue state to ready. */ 456 msq_attr.sq_state = MLX5_SQC_STATE_RST; 457 msq_attr.state = MLX5_SQC_STATE_RDY; 458 wq->sq_ci = 0; 459 ret = mlx5_devx_cmd_modify_sq(wq->sq_obj.sq, &msq_attr); 460 if (ret) { 461 DRV_LOG(ERR, "Failed to set SQ ready state Clock Queue."); 462 goto error; 463 } 464 return 0; 465 error: 466 ret = -rte_errno; 467 mlx5_txpp_destroy_clock_queue(sh); 468 rte_errno = -ret; 469 return ret; 470 } 471 472 /* Enable notification from the Rearm Queue CQ. */ 473 static inline void 474 mlx5_txpp_cq_arm(struct mlx5_dev_ctx_shared *sh) 475 { 476 void *base_addr; 477 478 struct mlx5_txpp_wq *aq = &sh->txpp.rearm_queue; 479 uint32_t arm_sn = aq->arm_sn << MLX5_CQ_SQN_OFFSET; 480 uint32_t db_hi = arm_sn | MLX5_CQ_DBR_CMD_ALL | aq->cq_ci; 481 uint64_t db_be = 482 rte_cpu_to_be_64(((uint64_t)db_hi << 32) | aq->cq_obj.cq->id); 483 base_addr = mlx5_os_get_devx_uar_base_addr(sh->tx_uar); 484 uint32_t *addr = RTE_PTR_ADD(base_addr, MLX5_CQ_DOORBELL); 485 486 rte_compiler_barrier(); 487 aq->cq_obj.db_rec[MLX5_CQ_ARM_DB] = rte_cpu_to_be_32(db_hi); 488 rte_wmb(); 489 #ifdef RTE_ARCH_64 490 *(uint64_t *)addr = db_be; 491 #else 492 *(uint32_t *)addr = db_be; 493 rte_io_wmb(); 494 *((uint32_t *)addr + 1) = db_be >> 32; 495 #endif 496 aq->arm_sn++; 497 } 498 499 #if defined(RTE_ARCH_X86_64) 500 static inline int 501 mlx5_atomic128_compare_exchange(rte_int128_t *dst, 502 rte_int128_t *exp, 503 const rte_int128_t *src) 504 { 505 uint8_t res; 506 507 asm volatile (MPLOCKED 508 "cmpxchg16b %[dst];" 509 " sete %[res]" 510 : [dst] "=m" (dst->val[0]), 511 "=a" (exp->val[0]), 512 "=d" (exp->val[1]), 513 [res] "=r" (res) 514 : "b" (src->val[0]), 515 "c" (src->val[1]), 516 "a" (exp->val[0]), 517 "d" (exp->val[1]), 518 "m" (dst->val[0]) 519 : "memory"); 520 521 return res; 522 } 523 #endif 524 525 static inline void 526 mlx5_atomic_read_cqe(rte_int128_t *from, rte_int128_t *ts) 527 { 528 /* 529 * The only CQE of Clock Queue is being continuously 530 * update by hardware with soecified rate. We have to 531 * read timestump and WQE completion index atomically. 532 */ 533 #if defined(RTE_ARCH_X86_64) 534 rte_int128_t src; 535 536 memset(&src, 0, sizeof(src)); 537 *ts = src; 538 /* if (*from == *ts) *from = *src else *ts = *from; */ 539 mlx5_atomic128_compare_exchange(from, ts, &src); 540 #else 541 uint64_t *cqe = (uint64_t *)from; 542 543 /* 544 * Power architecture does not support 16B compare-and-swap. 545 * ARM implements it in software, code below is more relevant. 546 */ 547 for (;;) { 548 uint64_t tm, op; 549 uint64_t *ps; 550 551 rte_compiler_barrier(); 552 tm = __atomic_load_n(cqe + 0, __ATOMIC_RELAXED); 553 op = __atomic_load_n(cqe + 1, __ATOMIC_RELAXED); 554 rte_compiler_barrier(); 555 if (tm != __atomic_load_n(cqe + 0, __ATOMIC_RELAXED)) 556 continue; 557 if (op != __atomic_load_n(cqe + 1, __ATOMIC_RELAXED)) 558 continue; 559 ps = (uint64_t *)ts; 560 ps[0] = tm; 561 ps[1] = op; 562 return; 563 } 564 #endif 565 } 566 567 /* Stores timestamp in the cache structure to share data with datapath. */ 568 static inline void 569 mlx5_txpp_cache_timestamp(struct mlx5_dev_ctx_shared *sh, 570 uint64_t ts, uint64_t ci) 571 { 572 ci = ci << (64 - MLX5_CQ_INDEX_WIDTH); 573 ci |= (ts << MLX5_CQ_INDEX_WIDTH) >> MLX5_CQ_INDEX_WIDTH; 574 rte_compiler_barrier(); 575 __atomic_store_n(&sh->txpp.ts.ts, ts, __ATOMIC_RELAXED); 576 __atomic_store_n(&sh->txpp.ts.ci_ts, ci, __ATOMIC_RELAXED); 577 rte_wmb(); 578 } 579 580 /* Reads timestamp from Clock Queue CQE and stores in the cache. */ 581 static inline void 582 mlx5_txpp_update_timestamp(struct mlx5_dev_ctx_shared *sh) 583 { 584 struct mlx5_txpp_wq *wq = &sh->txpp.clock_queue; 585 struct mlx5_cqe *cqe = (struct mlx5_cqe *)(uintptr_t)wq->cq_obj.cqes; 586 union { 587 rte_int128_t u128; 588 struct mlx5_cqe_ts cts; 589 } to; 590 uint64_t ts; 591 uint16_t ci; 592 593 mlx5_atomic_read_cqe((rte_int128_t *)&cqe->timestamp, &to.u128); 594 if (to.cts.op_own >> 4) { 595 DRV_LOG(DEBUG, "Clock Queue error sync lost."); 596 __atomic_fetch_add(&sh->txpp.err_clock_queue, 597 1, __ATOMIC_RELAXED); 598 sh->txpp.sync_lost = 1; 599 return; 600 } 601 ci = rte_be_to_cpu_16(to.cts.wqe_counter); 602 ts = rte_be_to_cpu_64(to.cts.timestamp); 603 ts = mlx5_txpp_convert_rx_ts(sh, ts); 604 wq->cq_ci += (ci - wq->sq_ci) & UINT16_MAX; 605 wq->sq_ci = ci; 606 mlx5_txpp_cache_timestamp(sh, ts, wq->cq_ci); 607 } 608 609 /* Waits for the first completion on Clock Queue to init timestamp. */ 610 static inline void 611 mlx5_txpp_init_timestamp(struct mlx5_dev_ctx_shared *sh) 612 { 613 struct mlx5_txpp_wq *wq = &sh->txpp.clock_queue; 614 uint32_t wait; 615 616 sh->txpp.ts_p = 0; 617 sh->txpp.ts_n = 0; 618 for (wait = 0; wait < MLX5_TXPP_WAIT_INIT_TS; wait++) { 619 mlx5_txpp_update_timestamp(sh); 620 if (wq->sq_ci) 621 return; 622 /* Wait one millisecond and try again. */ 623 rte_delay_us_sleep(US_PER_S / MS_PER_S); 624 } 625 DRV_LOG(ERR, "Unable to initialize timestamp."); 626 sh->txpp.sync_lost = 1; 627 } 628 629 #ifdef HAVE_IBV_DEVX_EVENT 630 /* Gather statistics for timestamp from Clock Queue CQE. */ 631 static inline void 632 mlx5_txpp_gather_timestamp(struct mlx5_dev_ctx_shared *sh) 633 { 634 /* Check whether we have a valid timestamp. */ 635 if (!sh->txpp.clock_queue.sq_ci && !sh->txpp.ts_n) 636 return; 637 MLX5_ASSERT(sh->txpp.ts_p < MLX5_TXPP_REARM_SQ_SIZE); 638 __atomic_store_n(&sh->txpp.tsa[sh->txpp.ts_p].ts, 639 sh->txpp.ts.ts, __ATOMIC_RELAXED); 640 __atomic_store_n(&sh->txpp.tsa[sh->txpp.ts_p].ci_ts, 641 sh->txpp.ts.ci_ts, __ATOMIC_RELAXED); 642 if (++sh->txpp.ts_p >= MLX5_TXPP_REARM_SQ_SIZE) 643 sh->txpp.ts_p = 0; 644 if (sh->txpp.ts_n < MLX5_TXPP_REARM_SQ_SIZE) 645 ++sh->txpp.ts_n; 646 } 647 648 /* Handles Rearm Queue completions in periodic service. */ 649 static __rte_always_inline void 650 mlx5_txpp_handle_rearm_queue(struct mlx5_dev_ctx_shared *sh) 651 { 652 struct mlx5_txpp_wq *wq = &sh->txpp.rearm_queue; 653 uint32_t cq_ci = wq->cq_ci; 654 bool error = false; 655 int ret; 656 657 do { 658 volatile struct mlx5_cqe *cqe; 659 660 cqe = &wq->cq_obj.cqes[cq_ci & (MLX5_TXPP_REARM_CQ_SIZE - 1)]; 661 ret = check_cqe(cqe, MLX5_TXPP_REARM_CQ_SIZE, cq_ci); 662 switch (ret) { 663 case MLX5_CQE_STATUS_ERR: 664 error = true; 665 ++cq_ci; 666 break; 667 case MLX5_CQE_STATUS_SW_OWN: 668 wq->sq_ci += 2; 669 ++cq_ci; 670 break; 671 case MLX5_CQE_STATUS_HW_OWN: 672 break; 673 default: 674 MLX5_ASSERT(false); 675 break; 676 } 677 } while (ret != MLX5_CQE_STATUS_HW_OWN); 678 if (likely(cq_ci != wq->cq_ci)) { 679 /* Check whether we have missed interrupts. */ 680 if (cq_ci - wq->cq_ci != 1) { 681 DRV_LOG(DEBUG, "Rearm Queue missed interrupt."); 682 __atomic_fetch_add(&sh->txpp.err_miss_int, 683 1, __ATOMIC_RELAXED); 684 /* Check sync lost on wqe index. */ 685 if (cq_ci - wq->cq_ci >= 686 (((1UL << MLX5_WQ_INDEX_WIDTH) / 687 MLX5_TXPP_REARM) - 1)) 688 error = 1; 689 } 690 /* Update doorbell record to notify hardware. */ 691 rte_compiler_barrier(); 692 *wq->cq_obj.db_rec = rte_cpu_to_be_32(cq_ci); 693 rte_wmb(); 694 wq->cq_ci = cq_ci; 695 /* Fire new requests to Rearm Queue. */ 696 if (error) { 697 DRV_LOG(DEBUG, "Rearm Queue error sync lost."); 698 __atomic_fetch_add(&sh->txpp.err_rearm_queue, 699 1, __ATOMIC_RELAXED); 700 sh->txpp.sync_lost = 1; 701 } 702 } 703 } 704 705 /* Handles Clock Queue completions in periodic service. */ 706 static __rte_always_inline void 707 mlx5_txpp_handle_clock_queue(struct mlx5_dev_ctx_shared *sh) 708 { 709 mlx5_txpp_update_timestamp(sh); 710 mlx5_txpp_gather_timestamp(sh); 711 } 712 #endif 713 714 /* Invoked periodically on Rearm Queue completions. */ 715 void 716 mlx5_txpp_interrupt_handler(void *cb_arg) 717 { 718 #ifndef HAVE_IBV_DEVX_EVENT 719 RTE_SET_USED(cb_arg); 720 return; 721 #else 722 struct mlx5_dev_ctx_shared *sh = cb_arg; 723 union { 724 struct mlx5dv_devx_async_event_hdr event_resp; 725 uint8_t buf[sizeof(struct mlx5dv_devx_async_event_hdr) + 128]; 726 } out; 727 728 MLX5_ASSERT(rte_eal_process_type() == RTE_PROC_PRIMARY); 729 /* Process events in the loop. Only rearm completions are expected. */ 730 while (mlx5_glue->devx_get_event 731 (sh->txpp.echan, 732 &out.event_resp, 733 sizeof(out.buf)) >= 734 (ssize_t)sizeof(out.event_resp.cookie)) { 735 mlx5_txpp_handle_rearm_queue(sh); 736 mlx5_txpp_handle_clock_queue(sh); 737 mlx5_txpp_cq_arm(sh); 738 mlx5_txpp_doorbell_rearm_queue 739 (sh, sh->txpp.rearm_queue.sq_ci - 1); 740 } 741 #endif /* HAVE_IBV_DEVX_ASYNC */ 742 } 743 744 static void 745 mlx5_txpp_stop_service(struct mlx5_dev_ctx_shared *sh) 746 { 747 if (!sh->txpp.intr_handle.fd) 748 return; 749 mlx5_intr_callback_unregister(&sh->txpp.intr_handle, 750 mlx5_txpp_interrupt_handler, sh); 751 sh->txpp.intr_handle.fd = 0; 752 } 753 754 /* Attach interrupt handler and fires first request to Rearm Queue. */ 755 static int 756 mlx5_txpp_start_service(struct mlx5_dev_ctx_shared *sh) 757 { 758 uint16_t event_nums[1] = {0}; 759 int ret; 760 int fd; 761 762 sh->txpp.err_miss_int = 0; 763 sh->txpp.err_rearm_queue = 0; 764 sh->txpp.err_clock_queue = 0; 765 sh->txpp.err_ts_past = 0; 766 sh->txpp.err_ts_future = 0; 767 /* Attach interrupt handler to process Rearm Queue completions. */ 768 fd = mlx5_os_get_devx_channel_fd(sh->txpp.echan); 769 ret = mlx5_os_set_nonblock_channel_fd(fd); 770 if (ret) { 771 DRV_LOG(ERR, "Failed to change event channel FD."); 772 rte_errno = errno; 773 return -rte_errno; 774 } 775 memset(&sh->txpp.intr_handle, 0, sizeof(sh->txpp.intr_handle)); 776 fd = mlx5_os_get_devx_channel_fd(sh->txpp.echan); 777 sh->txpp.intr_handle.fd = fd; 778 sh->txpp.intr_handle.type = RTE_INTR_HANDLE_EXT; 779 if (rte_intr_callback_register(&sh->txpp.intr_handle, 780 mlx5_txpp_interrupt_handler, sh)) { 781 sh->txpp.intr_handle.fd = 0; 782 DRV_LOG(ERR, "Failed to register CQE interrupt %d.", rte_errno); 783 return -rte_errno; 784 } 785 /* Subscribe CQ event to the event channel controlled by the driver. */ 786 ret = mlx5_os_devx_subscribe_devx_event(sh->txpp.echan, 787 sh->txpp.rearm_queue.cq_obj.cq->obj, 788 sizeof(event_nums), event_nums, 0); 789 if (ret) { 790 DRV_LOG(ERR, "Failed to subscribe CQE event."); 791 rte_errno = errno; 792 return -errno; 793 } 794 /* Enable interrupts in the CQ. */ 795 mlx5_txpp_cq_arm(sh); 796 /* Fire the first request on Rearm Queue. */ 797 mlx5_txpp_doorbell_rearm_queue(sh, sh->txpp.rearm_queue.sq_size - 1); 798 mlx5_txpp_init_timestamp(sh); 799 return 0; 800 } 801 802 /* 803 * The routine initializes the packet pacing infrastructure: 804 * - allocates PP context 805 * - Clock CQ/SQ 806 * - Rearm CQ/SQ 807 * - attaches rearm interrupt handler 808 * - starts Clock Queue 809 * 810 * Returns 0 on success, negative otherwise 811 */ 812 static int 813 mlx5_txpp_create(struct mlx5_dev_ctx_shared *sh, struct mlx5_priv *priv) 814 { 815 int tx_pp = priv->config.tx_pp; 816 int ret; 817 818 /* Store the requested pacing parameters. */ 819 sh->txpp.tick = tx_pp >= 0 ? tx_pp : -tx_pp; 820 sh->txpp.test = !!(tx_pp < 0); 821 sh->txpp.skew = priv->config.tx_skew; 822 sh->txpp.freq = priv->config.hca_attr.dev_freq_khz; 823 ret = mlx5_txpp_create_event_channel(sh); 824 if (ret) 825 goto exit; 826 ret = mlx5_txpp_alloc_pp_index(sh); 827 if (ret) 828 goto exit; 829 ret = mlx5_txpp_create_clock_queue(sh); 830 if (ret) 831 goto exit; 832 ret = mlx5_txpp_create_rearm_queue(sh); 833 if (ret) 834 goto exit; 835 ret = mlx5_txpp_start_service(sh); 836 if (ret) 837 goto exit; 838 exit: 839 if (ret) { 840 mlx5_txpp_stop_service(sh); 841 mlx5_txpp_destroy_rearm_queue(sh); 842 mlx5_txpp_destroy_clock_queue(sh); 843 mlx5_txpp_free_pp_index(sh); 844 mlx5_txpp_destroy_event_channel(sh); 845 sh->txpp.tick = 0; 846 sh->txpp.test = 0; 847 sh->txpp.skew = 0; 848 } 849 return ret; 850 } 851 852 /* 853 * The routine destroys the packet pacing infrastructure: 854 * - detaches rearm interrupt handler 855 * - Rearm CQ/SQ 856 * - Clock CQ/SQ 857 * - PP context 858 */ 859 static void 860 mlx5_txpp_destroy(struct mlx5_dev_ctx_shared *sh) 861 { 862 mlx5_txpp_stop_service(sh); 863 mlx5_txpp_destroy_rearm_queue(sh); 864 mlx5_txpp_destroy_clock_queue(sh); 865 mlx5_txpp_free_pp_index(sh); 866 mlx5_txpp_destroy_event_channel(sh); 867 sh->txpp.tick = 0; 868 sh->txpp.test = 0; 869 sh->txpp.skew = 0; 870 } 871 872 /** 873 * Creates and starts packet pacing infrastructure on specified device. 874 * 875 * @param dev 876 * Pointer to Ethernet device structure. 877 * 878 * @return 879 * 0 on success, a negative errno value otherwise and rte_errno is set. 880 */ 881 int 882 mlx5_txpp_start(struct rte_eth_dev *dev) 883 { 884 struct mlx5_priv *priv = dev->data->dev_private; 885 struct mlx5_dev_ctx_shared *sh = priv->sh; 886 int err = 0; 887 int ret; 888 889 if (!priv->config.tx_pp) { 890 /* Packet pacing is not requested for the device. */ 891 MLX5_ASSERT(priv->txpp_en == 0); 892 return 0; 893 } 894 if (priv->txpp_en) { 895 /* Packet pacing is already enabled for the device. */ 896 MLX5_ASSERT(sh->txpp.refcnt); 897 return 0; 898 } 899 if (priv->config.tx_pp > 0) { 900 ret = rte_mbuf_dynflag_lookup 901 (RTE_MBUF_DYNFLAG_TX_TIMESTAMP_NAME, NULL); 902 if (ret < 0) 903 return 0; 904 } 905 ret = pthread_mutex_lock(&sh->txpp.mutex); 906 MLX5_ASSERT(!ret); 907 RTE_SET_USED(ret); 908 if (sh->txpp.refcnt) { 909 priv->txpp_en = 1; 910 ++sh->txpp.refcnt; 911 } else { 912 err = mlx5_txpp_create(sh, priv); 913 if (!err) { 914 MLX5_ASSERT(sh->txpp.tick); 915 priv->txpp_en = 1; 916 sh->txpp.refcnt = 1; 917 } else { 918 rte_errno = -err; 919 } 920 } 921 ret = pthread_mutex_unlock(&sh->txpp.mutex); 922 MLX5_ASSERT(!ret); 923 RTE_SET_USED(ret); 924 return err; 925 } 926 927 /** 928 * Stops and destroys packet pacing infrastructure on specified device. 929 * 930 * @param dev 931 * Pointer to Ethernet device structure. 932 * 933 * @return 934 * 0 on success, a negative errno value otherwise and rte_errno is set. 935 */ 936 void 937 mlx5_txpp_stop(struct rte_eth_dev *dev) 938 { 939 struct mlx5_priv *priv = dev->data->dev_private; 940 struct mlx5_dev_ctx_shared *sh = priv->sh; 941 int ret; 942 943 if (!priv->txpp_en) { 944 /* Packet pacing is already disabled for the device. */ 945 return; 946 } 947 priv->txpp_en = 0; 948 ret = pthread_mutex_lock(&sh->txpp.mutex); 949 MLX5_ASSERT(!ret); 950 RTE_SET_USED(ret); 951 MLX5_ASSERT(sh->txpp.refcnt); 952 if (!sh->txpp.refcnt || --sh->txpp.refcnt) 953 return; 954 /* No references any more, do actual destroy. */ 955 mlx5_txpp_destroy(sh); 956 ret = pthread_mutex_unlock(&sh->txpp.mutex); 957 MLX5_ASSERT(!ret); 958 RTE_SET_USED(ret); 959 } 960 961 /* 962 * Read the current clock counter of an Ethernet device 963 * 964 * This returns the current raw clock value of an Ethernet device. It is 965 * a raw amount of ticks, with no given time reference. 966 * The value returned here is from the same clock than the one 967 * filling timestamp field of Rx/Tx packets when using hardware timestamp 968 * offload. Therefore it can be used to compute a precise conversion of 969 * the device clock to the real time. 970 * 971 * @param dev 972 * Pointer to Ethernet device structure. 973 * @param clock 974 * Pointer to the uint64_t that holds the raw clock value. 975 * 976 * @return 977 * - 0: Success. 978 * - -ENOTSUP: The function is not supported in this mode. Requires 979 * packet pacing module configured and started (tx_pp devarg) 980 */ 981 int 982 mlx5_txpp_read_clock(struct rte_eth_dev *dev, uint64_t *timestamp) 983 { 984 struct mlx5_priv *priv = dev->data->dev_private; 985 struct mlx5_dev_ctx_shared *sh = priv->sh; 986 int ret; 987 988 if (sh->txpp.refcnt) { 989 struct mlx5_txpp_wq *wq = &sh->txpp.clock_queue; 990 struct mlx5_cqe *cqe = 991 (struct mlx5_cqe *)(uintptr_t)wq->cq_obj.cqes; 992 union { 993 rte_int128_t u128; 994 struct mlx5_cqe_ts cts; 995 } to; 996 uint64_t ts; 997 998 mlx5_atomic_read_cqe((rte_int128_t *)&cqe->timestamp, &to.u128); 999 if (to.cts.op_own >> 4) { 1000 DRV_LOG(DEBUG, "Clock Queue error sync lost."); 1001 __atomic_fetch_add(&sh->txpp.err_clock_queue, 1002 1, __ATOMIC_RELAXED); 1003 sh->txpp.sync_lost = 1; 1004 return -EIO; 1005 } 1006 ts = rte_be_to_cpu_64(to.cts.timestamp); 1007 ts = mlx5_txpp_convert_rx_ts(sh, ts); 1008 *timestamp = ts; 1009 return 0; 1010 } 1011 /* Not supported in isolated mode - kernel does not see the CQEs. */ 1012 if (priv->isolated || rte_eal_process_type() != RTE_PROC_PRIMARY) 1013 return -ENOTSUP; 1014 ret = mlx5_read_clock(dev, timestamp); 1015 return ret; 1016 } 1017 1018 /** 1019 * DPDK callback to clear device extended statistics. 1020 * 1021 * @param dev 1022 * Pointer to Ethernet device structure. 1023 * 1024 * @return 1025 * 0 on success and stats is reset, negative errno value otherwise and 1026 * rte_errno is set. 1027 */ 1028 int mlx5_txpp_xstats_reset(struct rte_eth_dev *dev) 1029 { 1030 struct mlx5_priv *priv = dev->data->dev_private; 1031 struct mlx5_dev_ctx_shared *sh = priv->sh; 1032 1033 __atomic_store_n(&sh->txpp.err_miss_int, 0, __ATOMIC_RELAXED); 1034 __atomic_store_n(&sh->txpp.err_rearm_queue, 0, __ATOMIC_RELAXED); 1035 __atomic_store_n(&sh->txpp.err_clock_queue, 0, __ATOMIC_RELAXED); 1036 __atomic_store_n(&sh->txpp.err_ts_past, 0, __ATOMIC_RELAXED); 1037 __atomic_store_n(&sh->txpp.err_ts_future, 0, __ATOMIC_RELAXED); 1038 return 0; 1039 } 1040 1041 /** 1042 * Routine to retrieve names of extended device statistics 1043 * for packet send scheduling. It appends the specific stats names 1044 * after the parts filled by preceding modules (eth stats, etc.) 1045 * 1046 * @param dev 1047 * Pointer to Ethernet device structure. 1048 * @param[out] xstats_names 1049 * Buffer to insert names into. 1050 * @param n 1051 * Number of names. 1052 * @param n_used 1053 * Number of names filled by preceding statistics modules. 1054 * 1055 * @return 1056 * Number of xstats names. 1057 */ 1058 int mlx5_txpp_xstats_get_names(struct rte_eth_dev *dev __rte_unused, 1059 struct rte_eth_xstat_name *xstats_names, 1060 unsigned int n, unsigned int n_used) 1061 { 1062 unsigned int n_txpp = RTE_DIM(mlx5_txpp_stat_names); 1063 unsigned int i; 1064 1065 if (n >= n_used + n_txpp && xstats_names) { 1066 for (i = 0; i < n_txpp; ++i) { 1067 strncpy(xstats_names[i + n_used].name, 1068 mlx5_txpp_stat_names[i], 1069 RTE_ETH_XSTATS_NAME_SIZE); 1070 xstats_names[i + n_used].name 1071 [RTE_ETH_XSTATS_NAME_SIZE - 1] = 0; 1072 } 1073 } 1074 return n_used + n_txpp; 1075 } 1076 1077 static inline void 1078 mlx5_txpp_read_tsa(struct mlx5_dev_txpp *txpp, 1079 struct mlx5_txpp_ts *tsa, uint16_t idx) 1080 { 1081 do { 1082 uint64_t ts, ci; 1083 1084 ts = __atomic_load_n(&txpp->tsa[idx].ts, __ATOMIC_RELAXED); 1085 ci = __atomic_load_n(&txpp->tsa[idx].ci_ts, __ATOMIC_RELAXED); 1086 rte_compiler_barrier(); 1087 if ((ci ^ ts) << MLX5_CQ_INDEX_WIDTH != 0) 1088 continue; 1089 if (__atomic_load_n(&txpp->tsa[idx].ts, 1090 __ATOMIC_RELAXED) != ts) 1091 continue; 1092 if (__atomic_load_n(&txpp->tsa[idx].ci_ts, 1093 __ATOMIC_RELAXED) != ci) 1094 continue; 1095 tsa->ts = ts; 1096 tsa->ci_ts = ci; 1097 return; 1098 } while (true); 1099 } 1100 1101 /* 1102 * Jitter reflects the clock change between 1103 * neighbours Clock Queue completions. 1104 */ 1105 static uint64_t 1106 mlx5_txpp_xstats_jitter(struct mlx5_dev_txpp *txpp) 1107 { 1108 struct mlx5_txpp_ts tsa0, tsa1; 1109 int64_t dts, dci; 1110 uint16_t ts_p; 1111 1112 if (txpp->ts_n < 2) { 1113 /* No gathered enough reports yet. */ 1114 return 0; 1115 } 1116 do { 1117 int ts_0, ts_1; 1118 1119 ts_p = txpp->ts_p; 1120 rte_compiler_barrier(); 1121 ts_0 = ts_p - 2; 1122 if (ts_0 < 0) 1123 ts_0 += MLX5_TXPP_REARM_SQ_SIZE; 1124 ts_1 = ts_p - 1; 1125 if (ts_1 < 0) 1126 ts_1 += MLX5_TXPP_REARM_SQ_SIZE; 1127 mlx5_txpp_read_tsa(txpp, &tsa0, ts_0); 1128 mlx5_txpp_read_tsa(txpp, &tsa1, ts_1); 1129 rte_compiler_barrier(); 1130 } while (ts_p != txpp->ts_p); 1131 /* We have two neighbor reports, calculate the jitter. */ 1132 dts = tsa1.ts - tsa0.ts; 1133 dci = (tsa1.ci_ts >> (64 - MLX5_CQ_INDEX_WIDTH)) - 1134 (tsa0.ci_ts >> (64 - MLX5_CQ_INDEX_WIDTH)); 1135 if (dci < 0) 1136 dci += 1 << MLX5_CQ_INDEX_WIDTH; 1137 dci *= txpp->tick; 1138 return (dts > dci) ? dts - dci : dci - dts; 1139 } 1140 1141 /* 1142 * Wander reflects the long-term clock change 1143 * over the entire length of all Clock Queue completions. 1144 */ 1145 static uint64_t 1146 mlx5_txpp_xstats_wander(struct mlx5_dev_txpp *txpp) 1147 { 1148 struct mlx5_txpp_ts tsa0, tsa1; 1149 int64_t dts, dci; 1150 uint16_t ts_p; 1151 1152 if (txpp->ts_n < MLX5_TXPP_REARM_SQ_SIZE) { 1153 /* No gathered enough reports yet. */ 1154 return 0; 1155 } 1156 do { 1157 int ts_0, ts_1; 1158 1159 ts_p = txpp->ts_p; 1160 rte_compiler_barrier(); 1161 ts_0 = ts_p - MLX5_TXPP_REARM_SQ_SIZE / 2 - 1; 1162 if (ts_0 < 0) 1163 ts_0 += MLX5_TXPP_REARM_SQ_SIZE; 1164 ts_1 = ts_p - 1; 1165 if (ts_1 < 0) 1166 ts_1 += MLX5_TXPP_REARM_SQ_SIZE; 1167 mlx5_txpp_read_tsa(txpp, &tsa0, ts_0); 1168 mlx5_txpp_read_tsa(txpp, &tsa1, ts_1); 1169 rte_compiler_barrier(); 1170 } while (ts_p != txpp->ts_p); 1171 /* We have two neighbor reports, calculate the jitter. */ 1172 dts = tsa1.ts - tsa0.ts; 1173 dci = (tsa1.ci_ts >> (64 - MLX5_CQ_INDEX_WIDTH)) - 1174 (tsa0.ci_ts >> (64 - MLX5_CQ_INDEX_WIDTH)); 1175 dci += 1 << MLX5_CQ_INDEX_WIDTH; 1176 dci *= txpp->tick; 1177 return (dts > dci) ? dts - dci : dci - dts; 1178 } 1179 1180 /** 1181 * Routine to retrieve extended device statistics 1182 * for packet send scheduling. It appends the specific statistics 1183 * after the parts filled by preceding modules (eth stats, etc.) 1184 * 1185 * @param dev 1186 * Pointer to Ethernet device. 1187 * @param[out] stats 1188 * Pointer to rte extended stats table. 1189 * @param n 1190 * The size of the stats table. 1191 * @param n_used 1192 * Number of stats filled by preceding statistics modules. 1193 * 1194 * @return 1195 * Number of extended stats on success and stats is filled, 1196 * negative on error and rte_errno is set. 1197 */ 1198 int 1199 mlx5_txpp_xstats_get(struct rte_eth_dev *dev, 1200 struct rte_eth_xstat *stats, 1201 unsigned int n, unsigned int n_used) 1202 { 1203 unsigned int n_txpp = RTE_DIM(mlx5_txpp_stat_names); 1204 1205 if (n >= n_used + n_txpp && stats) { 1206 struct mlx5_priv *priv = dev->data->dev_private; 1207 struct mlx5_dev_ctx_shared *sh = priv->sh; 1208 unsigned int i; 1209 1210 for (i = 0; i < n_txpp; ++i) 1211 stats[n_used + i].id = n_used + i; 1212 stats[n_used + 0].value = 1213 __atomic_load_n(&sh->txpp.err_miss_int, 1214 __ATOMIC_RELAXED); 1215 stats[n_used + 1].value = 1216 __atomic_load_n(&sh->txpp.err_rearm_queue, 1217 __ATOMIC_RELAXED); 1218 stats[n_used + 2].value = 1219 __atomic_load_n(&sh->txpp.err_clock_queue, 1220 __ATOMIC_RELAXED); 1221 stats[n_used + 3].value = 1222 __atomic_load_n(&sh->txpp.err_ts_past, 1223 __ATOMIC_RELAXED); 1224 stats[n_used + 4].value = 1225 __atomic_load_n(&sh->txpp.err_ts_future, 1226 __ATOMIC_RELAXED); 1227 stats[n_used + 5].value = mlx5_txpp_xstats_jitter(&sh->txpp); 1228 stats[n_used + 6].value = mlx5_txpp_xstats_wander(&sh->txpp); 1229 stats[n_used + 7].value = sh->txpp.sync_lost; 1230 } 1231 return n_used + n_txpp; 1232 } 1233