1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2020 Mellanox Technologies, Ltd 3 */ 4 #include <fcntl.h> 5 #include <stdint.h> 6 7 #include <rte_ether.h> 8 #include <ethdev_driver.h> 9 #include <rte_interrupts.h> 10 #include <rte_alarm.h> 11 #include <rte_malloc.h> 12 #include <rte_cycles.h> 13 #include <rte_eal_paging.h> 14 15 #include <mlx5_malloc.h> 16 #include <mlx5_common_devx.h> 17 18 #include "mlx5.h" 19 #include "mlx5_rxtx.h" 20 #include "mlx5_common_os.h" 21 22 static_assert(sizeof(struct mlx5_cqe_ts) == sizeof(rte_int128_t), 23 "Wrong timestamp CQE part size"); 24 25 static const char * const mlx5_txpp_stat_names[] = { 26 "tx_pp_missed_interrupt_errors", /* Missed service interrupt. */ 27 "tx_pp_rearm_queue_errors", /* Rearm Queue errors. */ 28 "tx_pp_clock_queue_errors", /* Clock Queue errors. */ 29 "tx_pp_timestamp_past_errors", /* Timestamp in the past. */ 30 "tx_pp_timestamp_future_errors", /* Timestamp in the distant future. */ 31 "tx_pp_jitter", /* Timestamp jitter (one Clock Queue completion). */ 32 "tx_pp_wander", /* Timestamp wander (half of Clock Queue CQEs). */ 33 "tx_pp_sync_lost", /* Scheduling synchronization lost. */ 34 }; 35 36 /* Destroy Event Queue Notification Channel. */ 37 static void 38 mlx5_txpp_destroy_event_channel(struct mlx5_dev_ctx_shared *sh) 39 { 40 if (sh->txpp.echan) { 41 mlx5_os_devx_destroy_event_channel(sh->txpp.echan); 42 sh->txpp.echan = NULL; 43 } 44 } 45 46 /* Create Event Queue Notification Channel. */ 47 static int 48 mlx5_txpp_create_event_channel(struct mlx5_dev_ctx_shared *sh) 49 { 50 MLX5_ASSERT(!sh->txpp.echan); 51 sh->txpp.echan = mlx5_os_devx_create_event_channel(sh->ctx, 52 MLX5DV_DEVX_CREATE_EVENT_CHANNEL_FLAGS_OMIT_EV_DATA); 53 if (!sh->txpp.echan) { 54 rte_errno = errno; 55 DRV_LOG(ERR, "Failed to create event channel %d.", rte_errno); 56 return -rte_errno; 57 } 58 return 0; 59 } 60 61 static void 62 mlx5_txpp_free_pp_index(struct mlx5_dev_ctx_shared *sh) 63 { 64 #ifdef HAVE_MLX5DV_PP_ALLOC 65 if (sh->txpp.pp) { 66 mlx5_glue->dv_free_pp(sh->txpp.pp); 67 sh->txpp.pp = NULL; 68 sh->txpp.pp_id = 0; 69 } 70 #else 71 RTE_SET_USED(sh); 72 DRV_LOG(ERR, "Freeing pacing index is not supported."); 73 #endif 74 } 75 76 /* Allocate Packet Pacing index from kernel via mlx5dv call. */ 77 static int 78 mlx5_txpp_alloc_pp_index(struct mlx5_dev_ctx_shared *sh) 79 { 80 #ifdef HAVE_MLX5DV_PP_ALLOC 81 uint32_t pp[MLX5_ST_SZ_DW(set_pp_rate_limit_context)]; 82 uint64_t rate; 83 84 MLX5_ASSERT(!sh->txpp.pp); 85 memset(&pp, 0, sizeof(pp)); 86 rate = NS_PER_S / sh->txpp.tick; 87 if (rate * sh->txpp.tick != NS_PER_S) 88 DRV_LOG(WARNING, "Packet pacing frequency is not precise."); 89 if (sh->txpp.test) { 90 uint32_t len; 91 92 len = RTE_MAX(MLX5_TXPP_TEST_PKT_SIZE, 93 (size_t)RTE_ETHER_MIN_LEN); 94 MLX5_SET(set_pp_rate_limit_context, &pp, 95 burst_upper_bound, len); 96 MLX5_SET(set_pp_rate_limit_context, &pp, 97 typical_packet_size, len); 98 /* Convert packets per second into kilobits. */ 99 rate = (rate * len) / (1000ul / CHAR_BIT); 100 DRV_LOG(INFO, "Packet pacing rate set to %" PRIu64, rate); 101 } 102 MLX5_SET(set_pp_rate_limit_context, &pp, rate_limit, rate); 103 MLX5_SET(set_pp_rate_limit_context, &pp, rate_mode, 104 sh->txpp.test ? MLX5_DATA_RATE : MLX5_WQE_RATE); 105 sh->txpp.pp = mlx5_glue->dv_alloc_pp 106 (sh->ctx, sizeof(pp), &pp, 107 MLX5DV_PP_ALLOC_FLAGS_DEDICATED_INDEX); 108 if (sh->txpp.pp == NULL) { 109 DRV_LOG(ERR, "Failed to allocate packet pacing index."); 110 rte_errno = errno; 111 return -errno; 112 } 113 if (!((struct mlx5dv_pp *)sh->txpp.pp)->index) { 114 DRV_LOG(ERR, "Zero packet pacing index allocated."); 115 mlx5_txpp_free_pp_index(sh); 116 rte_errno = ENOTSUP; 117 return -ENOTSUP; 118 } 119 sh->txpp.pp_id = ((struct mlx5dv_pp *)(sh->txpp.pp))->index; 120 return 0; 121 #else 122 RTE_SET_USED(sh); 123 DRV_LOG(ERR, "Allocating pacing index is not supported."); 124 rte_errno = ENOTSUP; 125 return -ENOTSUP; 126 #endif 127 } 128 129 static void 130 mlx5_txpp_destroy_send_queue(struct mlx5_txpp_wq *wq) 131 { 132 mlx5_devx_sq_destroy(&wq->sq_obj); 133 mlx5_devx_cq_destroy(&wq->cq_obj); 134 memset(wq, 0, sizeof(*wq)); 135 } 136 137 static void 138 mlx5_txpp_destroy_rearm_queue(struct mlx5_dev_ctx_shared *sh) 139 { 140 struct mlx5_txpp_wq *wq = &sh->txpp.rearm_queue; 141 142 mlx5_txpp_destroy_send_queue(wq); 143 } 144 145 static void 146 mlx5_txpp_destroy_clock_queue(struct mlx5_dev_ctx_shared *sh) 147 { 148 struct mlx5_txpp_wq *wq = &sh->txpp.clock_queue; 149 150 mlx5_txpp_destroy_send_queue(wq); 151 if (sh->txpp.tsa) { 152 mlx5_free(sh->txpp.tsa); 153 sh->txpp.tsa = NULL; 154 } 155 } 156 157 static void 158 mlx5_txpp_doorbell_rearm_queue(struct mlx5_dev_ctx_shared *sh, uint16_t ci) 159 { 160 struct mlx5_txpp_wq *wq = &sh->txpp.rearm_queue; 161 struct mlx5_wqe *wqe = (struct mlx5_wqe *)(uintptr_t)wq->sq_obj.wqes; 162 union { 163 uint32_t w32[2]; 164 uint64_t w64; 165 } cs; 166 void *reg_addr; 167 168 wq->sq_ci = ci + 1; 169 cs.w32[0] = rte_cpu_to_be_32(rte_be_to_cpu_32 170 (wqe[ci & (wq->sq_size - 1)].ctrl[0]) | (ci - 1) << 8); 171 cs.w32[1] = wqe[ci & (wq->sq_size - 1)].ctrl[1]; 172 /* Update SQ doorbell record with new SQ ci. */ 173 rte_compiler_barrier(); 174 *wq->sq_obj.db_rec = rte_cpu_to_be_32(wq->sq_ci); 175 /* Make sure the doorbell record is updated. */ 176 rte_wmb(); 177 /* Write to doorbel register to start processing. */ 178 reg_addr = mlx5_os_get_devx_uar_reg_addr(sh->tx_uar); 179 __mlx5_uar_write64_relaxed(cs.w64, reg_addr, NULL); 180 rte_wmb(); 181 } 182 183 static void 184 mlx5_txpp_fill_wqe_rearm_queue(struct mlx5_dev_ctx_shared *sh) 185 { 186 struct mlx5_txpp_wq *wq = &sh->txpp.rearm_queue; 187 struct mlx5_wqe *wqe = (struct mlx5_wqe *)(uintptr_t)wq->sq_obj.wqes; 188 uint32_t i; 189 190 for (i = 0; i < wq->sq_size; i += 2) { 191 struct mlx5_wqe_cseg *cs; 192 struct mlx5_wqe_qseg *qs; 193 uint32_t index; 194 195 /* Build SEND_EN request with slave WQE index. */ 196 cs = &wqe[i + 0].cseg; 197 cs->opcode = RTE_BE32(MLX5_OPCODE_SEND_EN | 0); 198 cs->sq_ds = rte_cpu_to_be_32((wq->sq_obj.sq->id << 8) | 2); 199 cs->flags = RTE_BE32(MLX5_COMP_ALWAYS << 200 MLX5_COMP_MODE_OFFSET); 201 cs->misc = RTE_BE32(0); 202 qs = RTE_PTR_ADD(cs, sizeof(struct mlx5_wqe_cseg)); 203 index = (i * MLX5_TXPP_REARM / 2 + MLX5_TXPP_REARM) & 204 ((1 << MLX5_WQ_INDEX_WIDTH) - 1); 205 qs->max_index = rte_cpu_to_be_32(index); 206 qs->qpn_cqn = 207 rte_cpu_to_be_32(sh->txpp.clock_queue.sq_obj.sq->id); 208 /* Build WAIT request with slave CQE index. */ 209 cs = &wqe[i + 1].cseg; 210 cs->opcode = RTE_BE32(MLX5_OPCODE_WAIT | 0); 211 cs->sq_ds = rte_cpu_to_be_32((wq->sq_obj.sq->id << 8) | 2); 212 cs->flags = RTE_BE32(MLX5_COMP_ONLY_ERR << 213 MLX5_COMP_MODE_OFFSET); 214 cs->misc = RTE_BE32(0); 215 qs = RTE_PTR_ADD(cs, sizeof(struct mlx5_wqe_cseg)); 216 index = (i * MLX5_TXPP_REARM / 2 + MLX5_TXPP_REARM / 2) & 217 ((1 << MLX5_CQ_INDEX_WIDTH) - 1); 218 qs->max_index = rte_cpu_to_be_32(index); 219 qs->qpn_cqn = 220 rte_cpu_to_be_32(sh->txpp.clock_queue.cq_obj.cq->id); 221 } 222 } 223 224 /* Creates the Rearm Queue to fire the requests to Clock Queue in realtime. */ 225 static int 226 mlx5_txpp_create_rearm_queue(struct mlx5_dev_ctx_shared *sh) 227 { 228 struct mlx5_devx_create_sq_attr sq_attr = { 229 .cd_master = 1, 230 .state = MLX5_SQC_STATE_RST, 231 .tis_lst_sz = 1, 232 .tis_num = sh->tis->id, 233 .wq_attr = (struct mlx5_devx_wq_attr){ 234 .pd = sh->pdn, 235 .uar_page = mlx5_os_get_devx_uar_page_id(sh->tx_uar), 236 }, 237 .ts_format = mlx5_ts_format_conv(sh->sq_ts_format), 238 }; 239 struct mlx5_devx_modify_sq_attr msq_attr = { 0 }; 240 struct mlx5_devx_cq_attr cq_attr = { 241 .uar_page_id = mlx5_os_get_devx_uar_page_id(sh->tx_uar), 242 }; 243 struct mlx5_txpp_wq *wq = &sh->txpp.rearm_queue; 244 int ret; 245 246 /* Create completion queue object for Rearm Queue. */ 247 ret = mlx5_devx_cq_create(sh->ctx, &wq->cq_obj, 248 log2above(MLX5_TXPP_REARM_CQ_SIZE), &cq_attr, 249 sh->numa_node); 250 if (ret) { 251 DRV_LOG(ERR, "Failed to create CQ for Rearm Queue."); 252 return ret; 253 } 254 wq->cq_ci = 0; 255 wq->arm_sn = 0; 256 wq->sq_size = MLX5_TXPP_REARM_SQ_SIZE; 257 MLX5_ASSERT(wq->sq_size == (1 << log2above(wq->sq_size))); 258 /* Create send queue object for Rearm Queue. */ 259 sq_attr.cqn = wq->cq_obj.cq->id; 260 /* There should be no WQE leftovers in the cyclic queue. */ 261 ret = mlx5_devx_sq_create(sh->ctx, &wq->sq_obj, 262 log2above(MLX5_TXPP_REARM_SQ_SIZE), &sq_attr, 263 sh->numa_node); 264 if (ret) { 265 rte_errno = errno; 266 DRV_LOG(ERR, "Failed to create SQ for Rearm Queue."); 267 goto error; 268 } 269 /* Build the WQEs in the Send Queue before goto Ready state. */ 270 mlx5_txpp_fill_wqe_rearm_queue(sh); 271 /* Change queue state to ready. */ 272 msq_attr.sq_state = MLX5_SQC_STATE_RST; 273 msq_attr.state = MLX5_SQC_STATE_RDY; 274 ret = mlx5_devx_cmd_modify_sq(wq->sq_obj.sq, &msq_attr); 275 if (ret) { 276 DRV_LOG(ERR, "Failed to set SQ ready state Rearm Queue."); 277 goto error; 278 } 279 return 0; 280 error: 281 ret = -rte_errno; 282 mlx5_txpp_destroy_rearm_queue(sh); 283 rte_errno = -ret; 284 return ret; 285 } 286 287 static void 288 mlx5_txpp_fill_wqe_clock_queue(struct mlx5_dev_ctx_shared *sh) 289 { 290 struct mlx5_txpp_wq *wq = &sh->txpp.clock_queue; 291 struct mlx5_wqe *wqe = (struct mlx5_wqe *)(uintptr_t)wq->sq_obj.wqes; 292 struct mlx5_wqe_cseg *cs = &wqe->cseg; 293 uint32_t wqe_size, opcode, i; 294 uint8_t *dst; 295 296 /* For test purposes fill the WQ with SEND inline packet. */ 297 if (sh->txpp.test) { 298 wqe_size = RTE_ALIGN(MLX5_TXPP_TEST_PKT_SIZE + 299 MLX5_WQE_CSEG_SIZE + 300 2 * MLX5_WQE_ESEG_SIZE - 301 MLX5_ESEG_MIN_INLINE_SIZE, 302 MLX5_WSEG_SIZE); 303 opcode = MLX5_OPCODE_SEND; 304 } else { 305 wqe_size = MLX5_WSEG_SIZE; 306 opcode = MLX5_OPCODE_NOP; 307 } 308 cs->opcode = rte_cpu_to_be_32(opcode | 0); /* Index is ignored. */ 309 cs->sq_ds = rte_cpu_to_be_32((wq->sq_obj.sq->id << 8) | 310 (wqe_size / MLX5_WSEG_SIZE)); 311 cs->flags = RTE_BE32(MLX5_COMP_ALWAYS << MLX5_COMP_MODE_OFFSET); 312 cs->misc = RTE_BE32(0); 313 wqe_size = RTE_ALIGN(wqe_size, MLX5_WQE_SIZE); 314 if (sh->txpp.test) { 315 struct mlx5_wqe_eseg *es = &wqe->eseg; 316 struct rte_ether_hdr *eth_hdr; 317 struct rte_ipv4_hdr *ip_hdr; 318 struct rte_udp_hdr *udp_hdr; 319 320 /* Build the inline test packet pattern. */ 321 MLX5_ASSERT(wqe_size <= MLX5_WQE_SIZE_MAX); 322 MLX5_ASSERT(MLX5_TXPP_TEST_PKT_SIZE >= 323 (sizeof(struct rte_ether_hdr) + 324 sizeof(struct rte_ipv4_hdr))); 325 es->flags = 0; 326 es->cs_flags = MLX5_ETH_WQE_L3_CSUM | MLX5_ETH_WQE_L4_CSUM; 327 es->swp_offs = 0; 328 es->metadata = 0; 329 es->swp_flags = 0; 330 es->mss = 0; 331 es->inline_hdr_sz = RTE_BE16(MLX5_TXPP_TEST_PKT_SIZE); 332 /* Build test packet L2 header (Ethernet). */ 333 dst = (uint8_t *)&es->inline_data; 334 eth_hdr = (struct rte_ether_hdr *)dst; 335 rte_eth_random_addr(ð_hdr->d_addr.addr_bytes[0]); 336 rte_eth_random_addr(ð_hdr->s_addr.addr_bytes[0]); 337 eth_hdr->ether_type = rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV4); 338 /* Build test packet L3 header (IP v4). */ 339 dst += sizeof(struct rte_ether_hdr); 340 ip_hdr = (struct rte_ipv4_hdr *)dst; 341 ip_hdr->version_ihl = RTE_IPV4_VHL_DEF; 342 ip_hdr->type_of_service = 0; 343 ip_hdr->fragment_offset = 0; 344 ip_hdr->time_to_live = 64; 345 ip_hdr->next_proto_id = IPPROTO_UDP; 346 ip_hdr->packet_id = 0; 347 ip_hdr->total_length = RTE_BE16(MLX5_TXPP_TEST_PKT_SIZE - 348 sizeof(struct rte_ether_hdr)); 349 /* use RFC5735 / RFC2544 reserved network test addresses */ 350 ip_hdr->src_addr = RTE_BE32((198U << 24) | (18 << 16) | 351 (0 << 8) | 1); 352 ip_hdr->dst_addr = RTE_BE32((198U << 24) | (18 << 16) | 353 (0 << 8) | 2); 354 if (MLX5_TXPP_TEST_PKT_SIZE < 355 (sizeof(struct rte_ether_hdr) + 356 sizeof(struct rte_ipv4_hdr) + 357 sizeof(struct rte_udp_hdr))) 358 goto wcopy; 359 /* Build test packet L4 header (UDP). */ 360 dst += sizeof(struct rte_ipv4_hdr); 361 udp_hdr = (struct rte_udp_hdr *)dst; 362 udp_hdr->src_port = RTE_BE16(9); /* RFC863 Discard. */ 363 udp_hdr->dst_port = RTE_BE16(9); 364 udp_hdr->dgram_len = RTE_BE16(MLX5_TXPP_TEST_PKT_SIZE - 365 sizeof(struct rte_ether_hdr) - 366 sizeof(struct rte_ipv4_hdr)); 367 udp_hdr->dgram_cksum = 0; 368 /* Fill the test packet data. */ 369 dst += sizeof(struct rte_udp_hdr); 370 for (i = sizeof(struct rte_ether_hdr) + 371 sizeof(struct rte_ipv4_hdr) + 372 sizeof(struct rte_udp_hdr); 373 i < MLX5_TXPP_TEST_PKT_SIZE; i++) 374 *dst++ = (uint8_t)(i & 0xFF); 375 } 376 wcopy: 377 /* Duplicate the pattern to the next WQEs. */ 378 dst = (uint8_t *)(uintptr_t)wq->sq_obj.umem_buf; 379 for (i = 1; i < MLX5_TXPP_CLKQ_SIZE; i++) { 380 dst += wqe_size; 381 rte_memcpy(dst, (void *)(uintptr_t)wq->sq_obj.umem_buf, 382 wqe_size); 383 } 384 } 385 386 /* Creates the Clock Queue for packet pacing, returns zero on success. */ 387 static int 388 mlx5_txpp_create_clock_queue(struct mlx5_dev_ctx_shared *sh) 389 { 390 struct mlx5_devx_create_sq_attr sq_attr = { 0 }; 391 struct mlx5_devx_modify_sq_attr msq_attr = { 0 }; 392 struct mlx5_devx_cq_attr cq_attr = { 393 .use_first_only = 1, 394 .overrun_ignore = 1, 395 .uar_page_id = mlx5_os_get_devx_uar_page_id(sh->tx_uar), 396 }; 397 struct mlx5_txpp_wq *wq = &sh->txpp.clock_queue; 398 int ret; 399 400 sh->txpp.tsa = mlx5_malloc(MLX5_MEM_RTE | MLX5_MEM_ZERO, 401 MLX5_TXPP_REARM_SQ_SIZE * 402 sizeof(struct mlx5_txpp_ts), 403 0, sh->numa_node); 404 if (!sh->txpp.tsa) { 405 DRV_LOG(ERR, "Failed to allocate memory for CQ stats."); 406 return -ENOMEM; 407 } 408 sh->txpp.ts_p = 0; 409 sh->txpp.ts_n = 0; 410 /* Create completion queue object for Clock Queue. */ 411 ret = mlx5_devx_cq_create(sh->ctx, &wq->cq_obj, 412 log2above(MLX5_TXPP_CLKQ_SIZE), &cq_attr, 413 sh->numa_node); 414 if (ret) { 415 DRV_LOG(ERR, "Failed to create CQ for Clock Queue."); 416 goto error; 417 } 418 wq->cq_ci = 0; 419 /* Allocate memory buffer for Send Queue WQEs. */ 420 if (sh->txpp.test) { 421 wq->sq_size = RTE_ALIGN(MLX5_TXPP_TEST_PKT_SIZE + 422 MLX5_WQE_CSEG_SIZE + 423 2 * MLX5_WQE_ESEG_SIZE - 424 MLX5_ESEG_MIN_INLINE_SIZE, 425 MLX5_WQE_SIZE) / MLX5_WQE_SIZE; 426 wq->sq_size *= MLX5_TXPP_CLKQ_SIZE; 427 } else { 428 wq->sq_size = MLX5_TXPP_CLKQ_SIZE; 429 } 430 /* There should not be WQE leftovers in the cyclic queue. */ 431 MLX5_ASSERT(wq->sq_size == (1 << log2above(wq->sq_size))); 432 /* Create send queue object for Clock Queue. */ 433 if (sh->txpp.test) { 434 sq_attr.tis_lst_sz = 1; 435 sq_attr.tis_num = sh->tis->id; 436 sq_attr.non_wire = 0; 437 sq_attr.static_sq_wq = 1; 438 } else { 439 sq_attr.non_wire = 1; 440 sq_attr.static_sq_wq = 1; 441 } 442 sq_attr.cqn = wq->cq_obj.cq->id; 443 sq_attr.packet_pacing_rate_limit_index = sh->txpp.pp_id; 444 sq_attr.wq_attr.cd_slave = 1; 445 sq_attr.wq_attr.uar_page = mlx5_os_get_devx_uar_page_id(sh->tx_uar); 446 sq_attr.wq_attr.pd = sh->pdn; 447 sq_attr.ts_format = mlx5_ts_format_conv(sh->sq_ts_format); 448 ret = mlx5_devx_sq_create(sh->ctx, &wq->sq_obj, log2above(wq->sq_size), 449 &sq_attr, sh->numa_node); 450 if (ret) { 451 rte_errno = errno; 452 DRV_LOG(ERR, "Failed to create SQ for Clock Queue."); 453 goto error; 454 } 455 /* Build the WQEs in the Send Queue before goto Ready state. */ 456 mlx5_txpp_fill_wqe_clock_queue(sh); 457 /* Change queue state to ready. */ 458 msq_attr.sq_state = MLX5_SQC_STATE_RST; 459 msq_attr.state = MLX5_SQC_STATE_RDY; 460 wq->sq_ci = 0; 461 ret = mlx5_devx_cmd_modify_sq(wq->sq_obj.sq, &msq_attr); 462 if (ret) { 463 DRV_LOG(ERR, "Failed to set SQ ready state Clock Queue."); 464 goto error; 465 } 466 return 0; 467 error: 468 ret = -rte_errno; 469 mlx5_txpp_destroy_clock_queue(sh); 470 rte_errno = -ret; 471 return ret; 472 } 473 474 /* Enable notification from the Rearm Queue CQ. */ 475 static inline void 476 mlx5_txpp_cq_arm(struct mlx5_dev_ctx_shared *sh) 477 { 478 void *base_addr; 479 480 struct mlx5_txpp_wq *aq = &sh->txpp.rearm_queue; 481 uint32_t arm_sn = aq->arm_sn << MLX5_CQ_SQN_OFFSET; 482 uint32_t db_hi = arm_sn | MLX5_CQ_DBR_CMD_ALL | aq->cq_ci; 483 uint64_t db_be = 484 rte_cpu_to_be_64(((uint64_t)db_hi << 32) | aq->cq_obj.cq->id); 485 base_addr = mlx5_os_get_devx_uar_base_addr(sh->tx_uar); 486 uint32_t *addr = RTE_PTR_ADD(base_addr, MLX5_CQ_DOORBELL); 487 488 rte_compiler_barrier(); 489 aq->cq_obj.db_rec[MLX5_CQ_ARM_DB] = rte_cpu_to_be_32(db_hi); 490 rte_wmb(); 491 #ifdef RTE_ARCH_64 492 *(uint64_t *)addr = db_be; 493 #else 494 *(uint32_t *)addr = db_be; 495 rte_io_wmb(); 496 *((uint32_t *)addr + 1) = db_be >> 32; 497 #endif 498 aq->arm_sn++; 499 } 500 501 #if defined(RTE_ARCH_X86_64) 502 static inline int 503 mlx5_atomic128_compare_exchange(rte_int128_t *dst, 504 rte_int128_t *exp, 505 const rte_int128_t *src) 506 { 507 uint8_t res; 508 509 asm volatile (MPLOCKED 510 "cmpxchg16b %[dst];" 511 " sete %[res]" 512 : [dst] "=m" (dst->val[0]), 513 "=a" (exp->val[0]), 514 "=d" (exp->val[1]), 515 [res] "=r" (res) 516 : "b" (src->val[0]), 517 "c" (src->val[1]), 518 "a" (exp->val[0]), 519 "d" (exp->val[1]), 520 "m" (dst->val[0]) 521 : "memory"); 522 523 return res; 524 } 525 #endif 526 527 static inline void 528 mlx5_atomic_read_cqe(rte_int128_t *from, rte_int128_t *ts) 529 { 530 /* 531 * The only CQE of Clock Queue is being continuously 532 * update by hardware with soecified rate. We have to 533 * read timestump and WQE completion index atomically. 534 */ 535 #if defined(RTE_ARCH_X86_64) 536 rte_int128_t src; 537 538 memset(&src, 0, sizeof(src)); 539 *ts = src; 540 /* if (*from == *ts) *from = *src else *ts = *from; */ 541 mlx5_atomic128_compare_exchange(from, ts, &src); 542 #else 543 uint64_t *cqe = (uint64_t *)from; 544 545 /* 546 * Power architecture does not support 16B compare-and-swap. 547 * ARM implements it in software, code below is more relevant. 548 */ 549 for (;;) { 550 uint64_t tm, op; 551 uint64_t *ps; 552 553 rte_compiler_barrier(); 554 tm = __atomic_load_n(cqe + 0, __ATOMIC_RELAXED); 555 op = __atomic_load_n(cqe + 1, __ATOMIC_RELAXED); 556 rte_compiler_barrier(); 557 if (tm != __atomic_load_n(cqe + 0, __ATOMIC_RELAXED)) 558 continue; 559 if (op != __atomic_load_n(cqe + 1, __ATOMIC_RELAXED)) 560 continue; 561 ps = (uint64_t *)ts; 562 ps[0] = tm; 563 ps[1] = op; 564 return; 565 } 566 #endif 567 } 568 569 /* Stores timestamp in the cache structure to share data with datapath. */ 570 static inline void 571 mlx5_txpp_cache_timestamp(struct mlx5_dev_ctx_shared *sh, 572 uint64_t ts, uint64_t ci) 573 { 574 ci = ci << (64 - MLX5_CQ_INDEX_WIDTH); 575 ci |= (ts << MLX5_CQ_INDEX_WIDTH) >> MLX5_CQ_INDEX_WIDTH; 576 rte_compiler_barrier(); 577 __atomic_store_n(&sh->txpp.ts.ts, ts, __ATOMIC_RELAXED); 578 __atomic_store_n(&sh->txpp.ts.ci_ts, ci, __ATOMIC_RELAXED); 579 rte_wmb(); 580 } 581 582 /* Reads timestamp from Clock Queue CQE and stores in the cache. */ 583 static inline void 584 mlx5_txpp_update_timestamp(struct mlx5_dev_ctx_shared *sh) 585 { 586 struct mlx5_txpp_wq *wq = &sh->txpp.clock_queue; 587 struct mlx5_cqe *cqe = (struct mlx5_cqe *)(uintptr_t)wq->cq_obj.cqes; 588 union { 589 rte_int128_t u128; 590 struct mlx5_cqe_ts cts; 591 } to; 592 uint64_t ts; 593 uint16_t ci; 594 595 mlx5_atomic_read_cqe((rte_int128_t *)&cqe->timestamp, &to.u128); 596 if (to.cts.op_own >> 4) { 597 DRV_LOG(DEBUG, "Clock Queue error sync lost."); 598 __atomic_fetch_add(&sh->txpp.err_clock_queue, 599 1, __ATOMIC_RELAXED); 600 sh->txpp.sync_lost = 1; 601 return; 602 } 603 ci = rte_be_to_cpu_16(to.cts.wqe_counter); 604 ts = rte_be_to_cpu_64(to.cts.timestamp); 605 ts = mlx5_txpp_convert_rx_ts(sh, ts); 606 wq->cq_ci += (ci - wq->sq_ci) & UINT16_MAX; 607 wq->sq_ci = ci; 608 mlx5_txpp_cache_timestamp(sh, ts, wq->cq_ci); 609 } 610 611 /* Waits for the first completion on Clock Queue to init timestamp. */ 612 static inline void 613 mlx5_txpp_init_timestamp(struct mlx5_dev_ctx_shared *sh) 614 { 615 struct mlx5_txpp_wq *wq = &sh->txpp.clock_queue; 616 uint32_t wait; 617 618 sh->txpp.ts_p = 0; 619 sh->txpp.ts_n = 0; 620 for (wait = 0; wait < MLX5_TXPP_WAIT_INIT_TS; wait++) { 621 mlx5_txpp_update_timestamp(sh); 622 if (wq->sq_ci) 623 return; 624 /* Wait one millisecond and try again. */ 625 rte_delay_us_sleep(US_PER_S / MS_PER_S); 626 } 627 DRV_LOG(ERR, "Unable to initialize timestamp."); 628 sh->txpp.sync_lost = 1; 629 } 630 631 #ifdef HAVE_IBV_DEVX_EVENT 632 /* Gather statistics for timestamp from Clock Queue CQE. */ 633 static inline void 634 mlx5_txpp_gather_timestamp(struct mlx5_dev_ctx_shared *sh) 635 { 636 /* Check whether we have a valid timestamp. */ 637 if (!sh->txpp.clock_queue.sq_ci && !sh->txpp.ts_n) 638 return; 639 MLX5_ASSERT(sh->txpp.ts_p < MLX5_TXPP_REARM_SQ_SIZE); 640 __atomic_store_n(&sh->txpp.tsa[sh->txpp.ts_p].ts, 641 sh->txpp.ts.ts, __ATOMIC_RELAXED); 642 __atomic_store_n(&sh->txpp.tsa[sh->txpp.ts_p].ci_ts, 643 sh->txpp.ts.ci_ts, __ATOMIC_RELAXED); 644 if (++sh->txpp.ts_p >= MLX5_TXPP_REARM_SQ_SIZE) 645 sh->txpp.ts_p = 0; 646 if (sh->txpp.ts_n < MLX5_TXPP_REARM_SQ_SIZE) 647 ++sh->txpp.ts_n; 648 } 649 650 /* Handles Rearm Queue completions in periodic service. */ 651 static __rte_always_inline void 652 mlx5_txpp_handle_rearm_queue(struct mlx5_dev_ctx_shared *sh) 653 { 654 struct mlx5_txpp_wq *wq = &sh->txpp.rearm_queue; 655 uint32_t cq_ci = wq->cq_ci; 656 bool error = false; 657 int ret; 658 659 do { 660 volatile struct mlx5_cqe *cqe; 661 662 cqe = &wq->cq_obj.cqes[cq_ci & (MLX5_TXPP_REARM_CQ_SIZE - 1)]; 663 ret = check_cqe(cqe, MLX5_TXPP_REARM_CQ_SIZE, cq_ci); 664 switch (ret) { 665 case MLX5_CQE_STATUS_ERR: 666 error = true; 667 ++cq_ci; 668 break; 669 case MLX5_CQE_STATUS_SW_OWN: 670 wq->sq_ci += 2; 671 ++cq_ci; 672 break; 673 case MLX5_CQE_STATUS_HW_OWN: 674 break; 675 default: 676 MLX5_ASSERT(false); 677 break; 678 } 679 } while (ret != MLX5_CQE_STATUS_HW_OWN); 680 if (likely(cq_ci != wq->cq_ci)) { 681 /* Check whether we have missed interrupts. */ 682 if (cq_ci - wq->cq_ci != 1) { 683 DRV_LOG(DEBUG, "Rearm Queue missed interrupt."); 684 __atomic_fetch_add(&sh->txpp.err_miss_int, 685 1, __ATOMIC_RELAXED); 686 /* Check sync lost on wqe index. */ 687 if (cq_ci - wq->cq_ci >= 688 (((1UL << MLX5_WQ_INDEX_WIDTH) / 689 MLX5_TXPP_REARM) - 1)) 690 error = 1; 691 } 692 /* Update doorbell record to notify hardware. */ 693 rte_compiler_barrier(); 694 *wq->cq_obj.db_rec = rte_cpu_to_be_32(cq_ci); 695 rte_wmb(); 696 wq->cq_ci = cq_ci; 697 /* Fire new requests to Rearm Queue. */ 698 if (error) { 699 DRV_LOG(DEBUG, "Rearm Queue error sync lost."); 700 __atomic_fetch_add(&sh->txpp.err_rearm_queue, 701 1, __ATOMIC_RELAXED); 702 sh->txpp.sync_lost = 1; 703 } 704 } 705 } 706 707 /* Handles Clock Queue completions in periodic service. */ 708 static __rte_always_inline void 709 mlx5_txpp_handle_clock_queue(struct mlx5_dev_ctx_shared *sh) 710 { 711 mlx5_txpp_update_timestamp(sh); 712 mlx5_txpp_gather_timestamp(sh); 713 } 714 #endif 715 716 /* Invoked periodically on Rearm Queue completions. */ 717 void 718 mlx5_txpp_interrupt_handler(void *cb_arg) 719 { 720 #ifndef HAVE_IBV_DEVX_EVENT 721 RTE_SET_USED(cb_arg); 722 return; 723 #else 724 struct mlx5_dev_ctx_shared *sh = cb_arg; 725 union { 726 struct mlx5dv_devx_async_event_hdr event_resp; 727 uint8_t buf[sizeof(struct mlx5dv_devx_async_event_hdr) + 128]; 728 } out; 729 730 MLX5_ASSERT(rte_eal_process_type() == RTE_PROC_PRIMARY); 731 /* Process events in the loop. Only rearm completions are expected. */ 732 while (mlx5_glue->devx_get_event 733 (sh->txpp.echan, 734 &out.event_resp, 735 sizeof(out.buf)) >= 736 (ssize_t)sizeof(out.event_resp.cookie)) { 737 mlx5_txpp_handle_rearm_queue(sh); 738 mlx5_txpp_handle_clock_queue(sh); 739 mlx5_txpp_cq_arm(sh); 740 mlx5_txpp_doorbell_rearm_queue 741 (sh, sh->txpp.rearm_queue.sq_ci - 1); 742 } 743 #endif /* HAVE_IBV_DEVX_ASYNC */ 744 } 745 746 static void 747 mlx5_txpp_stop_service(struct mlx5_dev_ctx_shared *sh) 748 { 749 if (!sh->txpp.intr_handle.fd) 750 return; 751 mlx5_intr_callback_unregister(&sh->txpp.intr_handle, 752 mlx5_txpp_interrupt_handler, sh); 753 sh->txpp.intr_handle.fd = 0; 754 } 755 756 /* Attach interrupt handler and fires first request to Rearm Queue. */ 757 static int 758 mlx5_txpp_start_service(struct mlx5_dev_ctx_shared *sh) 759 { 760 uint16_t event_nums[1] = {0}; 761 int ret; 762 int fd; 763 764 sh->txpp.err_miss_int = 0; 765 sh->txpp.err_rearm_queue = 0; 766 sh->txpp.err_clock_queue = 0; 767 sh->txpp.err_ts_past = 0; 768 sh->txpp.err_ts_future = 0; 769 /* Attach interrupt handler to process Rearm Queue completions. */ 770 fd = mlx5_os_get_devx_channel_fd(sh->txpp.echan); 771 ret = mlx5_os_set_nonblock_channel_fd(fd); 772 if (ret) { 773 DRV_LOG(ERR, "Failed to change event channel FD."); 774 rte_errno = errno; 775 return -rte_errno; 776 } 777 memset(&sh->txpp.intr_handle, 0, sizeof(sh->txpp.intr_handle)); 778 fd = mlx5_os_get_devx_channel_fd(sh->txpp.echan); 779 sh->txpp.intr_handle.fd = fd; 780 sh->txpp.intr_handle.type = RTE_INTR_HANDLE_EXT; 781 if (rte_intr_callback_register(&sh->txpp.intr_handle, 782 mlx5_txpp_interrupt_handler, sh)) { 783 sh->txpp.intr_handle.fd = 0; 784 DRV_LOG(ERR, "Failed to register CQE interrupt %d.", rte_errno); 785 return -rte_errno; 786 } 787 /* Subscribe CQ event to the event channel controlled by the driver. */ 788 ret = mlx5_os_devx_subscribe_devx_event(sh->txpp.echan, 789 sh->txpp.rearm_queue.cq_obj.cq->obj, 790 sizeof(event_nums), event_nums, 0); 791 if (ret) { 792 DRV_LOG(ERR, "Failed to subscribe CQE event."); 793 rte_errno = errno; 794 return -errno; 795 } 796 /* Enable interrupts in the CQ. */ 797 mlx5_txpp_cq_arm(sh); 798 /* Fire the first request on Rearm Queue. */ 799 mlx5_txpp_doorbell_rearm_queue(sh, sh->txpp.rearm_queue.sq_size - 1); 800 mlx5_txpp_init_timestamp(sh); 801 return 0; 802 } 803 804 /* 805 * The routine initializes the packet pacing infrastructure: 806 * - allocates PP context 807 * - Clock CQ/SQ 808 * - Rearm CQ/SQ 809 * - attaches rearm interrupt handler 810 * - starts Clock Queue 811 * 812 * Returns 0 on success, negative otherwise 813 */ 814 static int 815 mlx5_txpp_create(struct mlx5_dev_ctx_shared *sh, struct mlx5_priv *priv) 816 { 817 int tx_pp = priv->config.tx_pp; 818 int ret; 819 820 /* Store the requested pacing parameters. */ 821 sh->txpp.tick = tx_pp >= 0 ? tx_pp : -tx_pp; 822 sh->txpp.test = !!(tx_pp < 0); 823 sh->txpp.skew = priv->config.tx_skew; 824 sh->txpp.freq = priv->config.hca_attr.dev_freq_khz; 825 ret = mlx5_txpp_create_event_channel(sh); 826 if (ret) 827 goto exit; 828 ret = mlx5_txpp_alloc_pp_index(sh); 829 if (ret) 830 goto exit; 831 ret = mlx5_txpp_create_clock_queue(sh); 832 if (ret) 833 goto exit; 834 ret = mlx5_txpp_create_rearm_queue(sh); 835 if (ret) 836 goto exit; 837 ret = mlx5_txpp_start_service(sh); 838 if (ret) 839 goto exit; 840 exit: 841 if (ret) { 842 mlx5_txpp_stop_service(sh); 843 mlx5_txpp_destroy_rearm_queue(sh); 844 mlx5_txpp_destroy_clock_queue(sh); 845 mlx5_txpp_free_pp_index(sh); 846 mlx5_txpp_destroy_event_channel(sh); 847 sh->txpp.tick = 0; 848 sh->txpp.test = 0; 849 sh->txpp.skew = 0; 850 } 851 return ret; 852 } 853 854 /* 855 * The routine destroys the packet pacing infrastructure: 856 * - detaches rearm interrupt handler 857 * - Rearm CQ/SQ 858 * - Clock CQ/SQ 859 * - PP context 860 */ 861 static void 862 mlx5_txpp_destroy(struct mlx5_dev_ctx_shared *sh) 863 { 864 mlx5_txpp_stop_service(sh); 865 mlx5_txpp_destroy_rearm_queue(sh); 866 mlx5_txpp_destroy_clock_queue(sh); 867 mlx5_txpp_free_pp_index(sh); 868 mlx5_txpp_destroy_event_channel(sh); 869 sh->txpp.tick = 0; 870 sh->txpp.test = 0; 871 sh->txpp.skew = 0; 872 } 873 874 /** 875 * Creates and starts packet pacing infrastructure on specified device. 876 * 877 * @param dev 878 * Pointer to Ethernet device structure. 879 * 880 * @return 881 * 0 on success, a negative errno value otherwise and rte_errno is set. 882 */ 883 int 884 mlx5_txpp_start(struct rte_eth_dev *dev) 885 { 886 struct mlx5_priv *priv = dev->data->dev_private; 887 struct mlx5_dev_ctx_shared *sh = priv->sh; 888 int err = 0; 889 int ret; 890 891 if (!priv->config.tx_pp) { 892 /* Packet pacing is not requested for the device. */ 893 MLX5_ASSERT(priv->txpp_en == 0); 894 return 0; 895 } 896 if (priv->txpp_en) { 897 /* Packet pacing is already enabled for the device. */ 898 MLX5_ASSERT(sh->txpp.refcnt); 899 return 0; 900 } 901 if (priv->config.tx_pp > 0) { 902 ret = rte_mbuf_dynflag_lookup 903 (RTE_MBUF_DYNFLAG_TX_TIMESTAMP_NAME, NULL); 904 if (ret < 0) 905 return 0; 906 } 907 ret = pthread_mutex_lock(&sh->txpp.mutex); 908 MLX5_ASSERT(!ret); 909 RTE_SET_USED(ret); 910 if (sh->txpp.refcnt) { 911 priv->txpp_en = 1; 912 ++sh->txpp.refcnt; 913 } else { 914 err = mlx5_txpp_create(sh, priv); 915 if (!err) { 916 MLX5_ASSERT(sh->txpp.tick); 917 priv->txpp_en = 1; 918 sh->txpp.refcnt = 1; 919 } else { 920 rte_errno = -err; 921 } 922 } 923 ret = pthread_mutex_unlock(&sh->txpp.mutex); 924 MLX5_ASSERT(!ret); 925 RTE_SET_USED(ret); 926 return err; 927 } 928 929 /** 930 * Stops and destroys packet pacing infrastructure on specified device. 931 * 932 * @param dev 933 * Pointer to Ethernet device structure. 934 * 935 * @return 936 * 0 on success, a negative errno value otherwise and rte_errno is set. 937 */ 938 void 939 mlx5_txpp_stop(struct rte_eth_dev *dev) 940 { 941 struct mlx5_priv *priv = dev->data->dev_private; 942 struct mlx5_dev_ctx_shared *sh = priv->sh; 943 int ret; 944 945 if (!priv->txpp_en) { 946 /* Packet pacing is already disabled for the device. */ 947 return; 948 } 949 priv->txpp_en = 0; 950 ret = pthread_mutex_lock(&sh->txpp.mutex); 951 MLX5_ASSERT(!ret); 952 RTE_SET_USED(ret); 953 MLX5_ASSERT(sh->txpp.refcnt); 954 if (!sh->txpp.refcnt || --sh->txpp.refcnt) 955 return; 956 /* No references any more, do actual destroy. */ 957 mlx5_txpp_destroy(sh); 958 ret = pthread_mutex_unlock(&sh->txpp.mutex); 959 MLX5_ASSERT(!ret); 960 RTE_SET_USED(ret); 961 } 962 963 /* 964 * Read the current clock counter of an Ethernet device 965 * 966 * This returns the current raw clock value of an Ethernet device. It is 967 * a raw amount of ticks, with no given time reference. 968 * The value returned here is from the same clock than the one 969 * filling timestamp field of Rx/Tx packets when using hardware timestamp 970 * offload. Therefore it can be used to compute a precise conversion of 971 * the device clock to the real time. 972 * 973 * @param dev 974 * Pointer to Ethernet device structure. 975 * @param clock 976 * Pointer to the uint64_t that holds the raw clock value. 977 * 978 * @return 979 * - 0: Success. 980 * - -ENOTSUP: The function is not supported in this mode. Requires 981 * packet pacing module configured and started (tx_pp devarg) 982 */ 983 int 984 mlx5_txpp_read_clock(struct rte_eth_dev *dev, uint64_t *timestamp) 985 { 986 struct mlx5_priv *priv = dev->data->dev_private; 987 struct mlx5_dev_ctx_shared *sh = priv->sh; 988 int ret; 989 990 if (sh->txpp.refcnt) { 991 struct mlx5_txpp_wq *wq = &sh->txpp.clock_queue; 992 struct mlx5_cqe *cqe = 993 (struct mlx5_cqe *)(uintptr_t)wq->cq_obj.cqes; 994 union { 995 rte_int128_t u128; 996 struct mlx5_cqe_ts cts; 997 } to; 998 uint64_t ts; 999 1000 mlx5_atomic_read_cqe((rte_int128_t *)&cqe->timestamp, &to.u128); 1001 if (to.cts.op_own >> 4) { 1002 DRV_LOG(DEBUG, "Clock Queue error sync lost."); 1003 __atomic_fetch_add(&sh->txpp.err_clock_queue, 1004 1, __ATOMIC_RELAXED); 1005 sh->txpp.sync_lost = 1; 1006 return -EIO; 1007 } 1008 ts = rte_be_to_cpu_64(to.cts.timestamp); 1009 ts = mlx5_txpp_convert_rx_ts(sh, ts); 1010 *timestamp = ts; 1011 return 0; 1012 } 1013 /* Not supported in isolated mode - kernel does not see the CQEs. */ 1014 if (priv->isolated || rte_eal_process_type() != RTE_PROC_PRIMARY) 1015 return -ENOTSUP; 1016 ret = mlx5_read_clock(dev, timestamp); 1017 return ret; 1018 } 1019 1020 /** 1021 * DPDK callback to clear device extended statistics. 1022 * 1023 * @param dev 1024 * Pointer to Ethernet device structure. 1025 * 1026 * @return 1027 * 0 on success and stats is reset, negative errno value otherwise and 1028 * rte_errno is set. 1029 */ 1030 int mlx5_txpp_xstats_reset(struct rte_eth_dev *dev) 1031 { 1032 struct mlx5_priv *priv = dev->data->dev_private; 1033 struct mlx5_dev_ctx_shared *sh = priv->sh; 1034 1035 __atomic_store_n(&sh->txpp.err_miss_int, 0, __ATOMIC_RELAXED); 1036 __atomic_store_n(&sh->txpp.err_rearm_queue, 0, __ATOMIC_RELAXED); 1037 __atomic_store_n(&sh->txpp.err_clock_queue, 0, __ATOMIC_RELAXED); 1038 __atomic_store_n(&sh->txpp.err_ts_past, 0, __ATOMIC_RELAXED); 1039 __atomic_store_n(&sh->txpp.err_ts_future, 0, __ATOMIC_RELAXED); 1040 return 0; 1041 } 1042 1043 /** 1044 * Routine to retrieve names of extended device statistics 1045 * for packet send scheduling. It appends the specific stats names 1046 * after the parts filled by preceding modules (eth stats, etc.) 1047 * 1048 * @param dev 1049 * Pointer to Ethernet device structure. 1050 * @param[out] xstats_names 1051 * Buffer to insert names into. 1052 * @param n 1053 * Number of names. 1054 * @param n_used 1055 * Number of names filled by preceding statistics modules. 1056 * 1057 * @return 1058 * Number of xstats names. 1059 */ 1060 int mlx5_txpp_xstats_get_names(struct rte_eth_dev *dev __rte_unused, 1061 struct rte_eth_xstat_name *xstats_names, 1062 unsigned int n, unsigned int n_used) 1063 { 1064 unsigned int n_txpp = RTE_DIM(mlx5_txpp_stat_names); 1065 unsigned int i; 1066 1067 if (n >= n_used + n_txpp && xstats_names) { 1068 for (i = 0; i < n_txpp; ++i) { 1069 strncpy(xstats_names[i + n_used].name, 1070 mlx5_txpp_stat_names[i], 1071 RTE_ETH_XSTATS_NAME_SIZE); 1072 xstats_names[i + n_used].name 1073 [RTE_ETH_XSTATS_NAME_SIZE - 1] = 0; 1074 } 1075 } 1076 return n_used + n_txpp; 1077 } 1078 1079 static inline void 1080 mlx5_txpp_read_tsa(struct mlx5_dev_txpp *txpp, 1081 struct mlx5_txpp_ts *tsa, uint16_t idx) 1082 { 1083 do { 1084 uint64_t ts, ci; 1085 1086 ts = __atomic_load_n(&txpp->tsa[idx].ts, __ATOMIC_RELAXED); 1087 ci = __atomic_load_n(&txpp->tsa[idx].ci_ts, __ATOMIC_RELAXED); 1088 rte_compiler_barrier(); 1089 if ((ci ^ ts) << MLX5_CQ_INDEX_WIDTH != 0) 1090 continue; 1091 if (__atomic_load_n(&txpp->tsa[idx].ts, 1092 __ATOMIC_RELAXED) != ts) 1093 continue; 1094 if (__atomic_load_n(&txpp->tsa[idx].ci_ts, 1095 __ATOMIC_RELAXED) != ci) 1096 continue; 1097 tsa->ts = ts; 1098 tsa->ci_ts = ci; 1099 return; 1100 } while (true); 1101 } 1102 1103 /* 1104 * Jitter reflects the clock change between 1105 * neighbours Clock Queue completions. 1106 */ 1107 static uint64_t 1108 mlx5_txpp_xstats_jitter(struct mlx5_dev_txpp *txpp) 1109 { 1110 struct mlx5_txpp_ts tsa0, tsa1; 1111 int64_t dts, dci; 1112 uint16_t ts_p; 1113 1114 if (txpp->ts_n < 2) { 1115 /* No gathered enough reports yet. */ 1116 return 0; 1117 } 1118 do { 1119 int ts_0, ts_1; 1120 1121 ts_p = txpp->ts_p; 1122 rte_compiler_barrier(); 1123 ts_0 = ts_p - 2; 1124 if (ts_0 < 0) 1125 ts_0 += MLX5_TXPP_REARM_SQ_SIZE; 1126 ts_1 = ts_p - 1; 1127 if (ts_1 < 0) 1128 ts_1 += MLX5_TXPP_REARM_SQ_SIZE; 1129 mlx5_txpp_read_tsa(txpp, &tsa0, ts_0); 1130 mlx5_txpp_read_tsa(txpp, &tsa1, ts_1); 1131 rte_compiler_barrier(); 1132 } while (ts_p != txpp->ts_p); 1133 /* We have two neighbor reports, calculate the jitter. */ 1134 dts = tsa1.ts - tsa0.ts; 1135 dci = (tsa1.ci_ts >> (64 - MLX5_CQ_INDEX_WIDTH)) - 1136 (tsa0.ci_ts >> (64 - MLX5_CQ_INDEX_WIDTH)); 1137 if (dci < 0) 1138 dci += 1 << MLX5_CQ_INDEX_WIDTH; 1139 dci *= txpp->tick; 1140 return (dts > dci) ? dts - dci : dci - dts; 1141 } 1142 1143 /* 1144 * Wander reflects the long-term clock change 1145 * over the entire length of all Clock Queue completions. 1146 */ 1147 static uint64_t 1148 mlx5_txpp_xstats_wander(struct mlx5_dev_txpp *txpp) 1149 { 1150 struct mlx5_txpp_ts tsa0, tsa1; 1151 int64_t dts, dci; 1152 uint16_t ts_p; 1153 1154 if (txpp->ts_n < MLX5_TXPP_REARM_SQ_SIZE) { 1155 /* No gathered enough reports yet. */ 1156 return 0; 1157 } 1158 do { 1159 int ts_0, ts_1; 1160 1161 ts_p = txpp->ts_p; 1162 rte_compiler_barrier(); 1163 ts_0 = ts_p - MLX5_TXPP_REARM_SQ_SIZE / 2 - 1; 1164 if (ts_0 < 0) 1165 ts_0 += MLX5_TXPP_REARM_SQ_SIZE; 1166 ts_1 = ts_p - 1; 1167 if (ts_1 < 0) 1168 ts_1 += MLX5_TXPP_REARM_SQ_SIZE; 1169 mlx5_txpp_read_tsa(txpp, &tsa0, ts_0); 1170 mlx5_txpp_read_tsa(txpp, &tsa1, ts_1); 1171 rte_compiler_barrier(); 1172 } while (ts_p != txpp->ts_p); 1173 /* We have two neighbor reports, calculate the jitter. */ 1174 dts = tsa1.ts - tsa0.ts; 1175 dci = (tsa1.ci_ts >> (64 - MLX5_CQ_INDEX_WIDTH)) - 1176 (tsa0.ci_ts >> (64 - MLX5_CQ_INDEX_WIDTH)); 1177 dci += 1 << MLX5_CQ_INDEX_WIDTH; 1178 dci *= txpp->tick; 1179 return (dts > dci) ? dts - dci : dci - dts; 1180 } 1181 1182 /** 1183 * Routine to retrieve extended device statistics 1184 * for packet send scheduling. It appends the specific statistics 1185 * after the parts filled by preceding modules (eth stats, etc.) 1186 * 1187 * @param dev 1188 * Pointer to Ethernet device. 1189 * @param[out] stats 1190 * Pointer to rte extended stats table. 1191 * @param n 1192 * The size of the stats table. 1193 * @param n_used 1194 * Number of stats filled by preceding statistics modules. 1195 * 1196 * @return 1197 * Number of extended stats on success and stats is filled, 1198 * negative on error and rte_errno is set. 1199 */ 1200 int 1201 mlx5_txpp_xstats_get(struct rte_eth_dev *dev, 1202 struct rte_eth_xstat *stats, 1203 unsigned int n, unsigned int n_used) 1204 { 1205 unsigned int n_txpp = RTE_DIM(mlx5_txpp_stat_names); 1206 1207 if (n >= n_used + n_txpp && stats) { 1208 struct mlx5_priv *priv = dev->data->dev_private; 1209 struct mlx5_dev_ctx_shared *sh = priv->sh; 1210 unsigned int i; 1211 1212 for (i = 0; i < n_txpp; ++i) 1213 stats[n_used + i].id = n_used + i; 1214 stats[n_used + 0].value = 1215 __atomic_load_n(&sh->txpp.err_miss_int, 1216 __ATOMIC_RELAXED); 1217 stats[n_used + 1].value = 1218 __atomic_load_n(&sh->txpp.err_rearm_queue, 1219 __ATOMIC_RELAXED); 1220 stats[n_used + 2].value = 1221 __atomic_load_n(&sh->txpp.err_clock_queue, 1222 __ATOMIC_RELAXED); 1223 stats[n_used + 3].value = 1224 __atomic_load_n(&sh->txpp.err_ts_past, 1225 __ATOMIC_RELAXED); 1226 stats[n_used + 4].value = 1227 __atomic_load_n(&sh->txpp.err_ts_future, 1228 __ATOMIC_RELAXED); 1229 stats[n_used + 5].value = mlx5_txpp_xstats_jitter(&sh->txpp); 1230 stats[n_used + 6].value = mlx5_txpp_xstats_wander(&sh->txpp); 1231 stats[n_used + 7].value = sh->txpp.sync_lost; 1232 } 1233 return n_used + n_txpp; 1234 } 1235