1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2015 6WIND S.A. 3 * Copyright 2015 Mellanox Technologies, Ltd 4 */ 5 6 #include <fcntl.h> 7 #include <inttypes.h> 8 #include <linux/sockios.h> 9 #include <linux/ethtool.h> 10 #include <stdint.h> 11 #include <stdio.h> 12 #include <unistd.h> 13 14 #include <rte_ethdev_driver.h> 15 #include <rte_common.h> 16 #include <rte_malloc.h> 17 18 #include <mlx5_common.h> 19 20 #include "mlx5_defs.h" 21 #include "mlx5.h" 22 #include "mlx5_rxtx.h" 23 24 25 static const struct mlx5_counter_ctrl mlx5_counters_init[] = { 26 { 27 .dpdk_name = "rx_port_unicast_bytes", 28 .ctr_name = "rx_vport_unicast_bytes", 29 }, 30 { 31 .dpdk_name = "rx_port_multicast_bytes", 32 .ctr_name = "rx_vport_multicast_bytes", 33 }, 34 { 35 .dpdk_name = "rx_port_broadcast_bytes", 36 .ctr_name = "rx_vport_broadcast_bytes", 37 }, 38 { 39 .dpdk_name = "rx_port_unicast_packets", 40 .ctr_name = "rx_vport_unicast_packets", 41 }, 42 { 43 .dpdk_name = "rx_port_multicast_packets", 44 .ctr_name = "rx_vport_multicast_packets", 45 }, 46 { 47 .dpdk_name = "rx_port_broadcast_packets", 48 .ctr_name = "rx_vport_broadcast_packets", 49 }, 50 { 51 .dpdk_name = "tx_port_unicast_bytes", 52 .ctr_name = "tx_vport_unicast_bytes", 53 }, 54 { 55 .dpdk_name = "tx_port_multicast_bytes", 56 .ctr_name = "tx_vport_multicast_bytes", 57 }, 58 { 59 .dpdk_name = "tx_port_broadcast_bytes", 60 .ctr_name = "tx_vport_broadcast_bytes", 61 }, 62 { 63 .dpdk_name = "tx_port_unicast_packets", 64 .ctr_name = "tx_vport_unicast_packets", 65 }, 66 { 67 .dpdk_name = "tx_port_multicast_packets", 68 .ctr_name = "tx_vport_multicast_packets", 69 }, 70 { 71 .dpdk_name = "tx_port_broadcast_packets", 72 .ctr_name = "tx_vport_broadcast_packets", 73 }, 74 { 75 .dpdk_name = "rx_wqe_err", 76 .ctr_name = "rx_wqe_err", 77 }, 78 { 79 .dpdk_name = "rx_crc_errors_phy", 80 .ctr_name = "rx_crc_errors_phy", 81 }, 82 { 83 .dpdk_name = "rx_in_range_len_errors_phy", 84 .ctr_name = "rx_in_range_len_errors_phy", 85 }, 86 { 87 .dpdk_name = "rx_symbol_err_phy", 88 .ctr_name = "rx_symbol_err_phy", 89 }, 90 { 91 .dpdk_name = "tx_errors_phy", 92 .ctr_name = "tx_errors_phy", 93 }, 94 { 95 .dpdk_name = "rx_out_of_buffer", 96 .ctr_name = "out_of_buffer", 97 .ib = 1, 98 }, 99 { 100 .dpdk_name = "tx_packets_phy", 101 .ctr_name = "tx_packets_phy", 102 }, 103 { 104 .dpdk_name = "rx_packets_phy", 105 .ctr_name = "rx_packets_phy", 106 }, 107 { 108 .dpdk_name = "tx_discards_phy", 109 .ctr_name = "tx_discards_phy", 110 }, 111 { 112 .dpdk_name = "rx_discards_phy", 113 .ctr_name = "rx_discards_phy", 114 }, 115 { 116 .dpdk_name = "tx_bytes_phy", 117 .ctr_name = "tx_bytes_phy", 118 }, 119 { 120 .dpdk_name = "rx_bytes_phy", 121 .ctr_name = "rx_bytes_phy", 122 }, 123 /* Representor only */ 124 { 125 .dpdk_name = "rx_packets", 126 .ctr_name = "vport_rx_packets", 127 }, 128 { 129 .dpdk_name = "rx_bytes", 130 .ctr_name = "vport_rx_bytes", 131 }, 132 { 133 .dpdk_name = "tx_packets", 134 .ctr_name = "vport_tx_packets", 135 }, 136 { 137 .dpdk_name = "tx_bytes", 138 .ctr_name = "vport_tx_bytes", 139 }, 140 }; 141 142 static const unsigned int xstats_n = RTE_DIM(mlx5_counters_init); 143 144 static inline int 145 mlx5_read_ib_stat(struct mlx5_priv *priv, const char *ctr_name, uint64_t *stat) 146 { 147 int fd; 148 149 if (priv->sh) { 150 MKSTR(path, "%s/ports/%d/hw_counters/%s", 151 priv->sh->ibdev_path, 152 priv->ibv_port, 153 ctr_name); 154 fd = open(path, O_RDONLY); 155 if (fd != -1) { 156 char buf[21] = {'\0'}; 157 ssize_t n = read(fd, buf, sizeof(buf)); 158 159 close(fd); 160 if (n != -1) { 161 *stat = strtoull(buf, NULL, 10); 162 return 0; 163 } 164 } 165 } 166 *stat = 0; 167 return 1; 168 } 169 170 /** 171 * Read device counters table. 172 * 173 * @param dev 174 * Pointer to Ethernet device. 175 * @param[out] stats 176 * Counters table output buffer. 177 * 178 * @return 179 * 0 on success and stats is filled, negative errno value otherwise and 180 * rte_errno is set. 181 */ 182 static int 183 mlx5_read_dev_counters(struct rte_eth_dev *dev, uint64_t *stats) 184 { 185 struct mlx5_priv *priv = dev->data->dev_private; 186 struct mlx5_xstats_ctrl *xstats_ctrl = &priv->xstats_ctrl; 187 unsigned int i; 188 struct ifreq ifr; 189 unsigned int stats_sz = xstats_ctrl->stats_n * sizeof(uint64_t); 190 unsigned char et_stat_buf[sizeof(struct ethtool_stats) + stats_sz]; 191 struct ethtool_stats *et_stats = (struct ethtool_stats *)et_stat_buf; 192 int ret; 193 194 et_stats->cmd = ETHTOOL_GSTATS; 195 et_stats->n_stats = xstats_ctrl->stats_n; 196 ifr.ifr_data = (caddr_t)et_stats; 197 ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr); 198 if (ret) { 199 DRV_LOG(WARNING, 200 "port %u unable to read statistic values from device", 201 dev->data->port_id); 202 return ret; 203 } 204 for (i = 0; i != xstats_ctrl->mlx5_stats_n; ++i) { 205 if (xstats_ctrl->info[i].ib) { 206 ret = mlx5_read_ib_stat(priv, 207 xstats_ctrl->info[i].ctr_name, 208 &stats[i]); 209 /* return last xstats counter if fail to read. */ 210 if (ret == 0) 211 xstats_ctrl->xstats[i] = stats[i]; 212 else 213 stats[i] = xstats_ctrl->xstats[i]; 214 } else { 215 stats[i] = (uint64_t) 216 et_stats->data[xstats_ctrl->dev_table_idx[i]]; 217 } 218 } 219 return 0; 220 } 221 222 /** 223 * Query the number of statistics provided by ETHTOOL. 224 * 225 * @param dev 226 * Pointer to Ethernet device. 227 * 228 * @return 229 * Number of statistics on success, negative errno value otherwise and 230 * rte_errno is set. 231 */ 232 static int 233 mlx5_ethtool_get_stats_n(struct rte_eth_dev *dev) { 234 struct ethtool_drvinfo drvinfo; 235 struct ifreq ifr; 236 int ret; 237 238 drvinfo.cmd = ETHTOOL_GDRVINFO; 239 ifr.ifr_data = (caddr_t)&drvinfo; 240 ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr); 241 if (ret) { 242 DRV_LOG(WARNING, "port %u unable to query number of statistics", 243 dev->data->port_id); 244 return ret; 245 } 246 return drvinfo.n_stats; 247 } 248 249 /** 250 * Init the structures to read device counters. 251 * 252 * @param dev 253 * Pointer to Ethernet device. 254 */ 255 void 256 mlx5_stats_init(struct rte_eth_dev *dev) 257 { 258 struct mlx5_priv *priv = dev->data->dev_private; 259 struct mlx5_xstats_ctrl *xstats_ctrl = &priv->xstats_ctrl; 260 struct mlx5_stats_ctrl *stats_ctrl = &priv->stats_ctrl; 261 unsigned int i; 262 unsigned int j; 263 struct ifreq ifr; 264 struct ethtool_gstrings *strings = NULL; 265 unsigned int dev_stats_n; 266 unsigned int str_sz; 267 int ret; 268 269 /* So that it won't aggregate for each init. */ 270 xstats_ctrl->mlx5_stats_n = 0; 271 ret = mlx5_ethtool_get_stats_n(dev); 272 if (ret < 0) { 273 DRV_LOG(WARNING, "port %u no extended statistics available", 274 dev->data->port_id); 275 return; 276 } 277 dev_stats_n = ret; 278 /* Allocate memory to grab stat names and values. */ 279 str_sz = dev_stats_n * ETH_GSTRING_LEN; 280 strings = (struct ethtool_gstrings *) 281 rte_malloc("xstats_strings", 282 str_sz + sizeof(struct ethtool_gstrings), 0); 283 if (!strings) { 284 DRV_LOG(WARNING, "port %u unable to allocate memory for xstats", 285 dev->data->port_id); 286 return; 287 } 288 strings->cmd = ETHTOOL_GSTRINGS; 289 strings->string_set = ETH_SS_STATS; 290 strings->len = dev_stats_n; 291 ifr.ifr_data = (caddr_t)strings; 292 ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr); 293 if (ret) { 294 DRV_LOG(WARNING, "port %u unable to get statistic names", 295 dev->data->port_id); 296 goto free; 297 } 298 for (i = 0; i != dev_stats_n; ++i) { 299 const char *curr_string = (const char *) 300 &strings->data[i * ETH_GSTRING_LEN]; 301 302 for (j = 0; j != xstats_n; ++j) { 303 if (!strcmp(mlx5_counters_init[j].ctr_name, 304 curr_string)) { 305 unsigned int idx = xstats_ctrl->mlx5_stats_n++; 306 307 xstats_ctrl->dev_table_idx[idx] = i; 308 xstats_ctrl->info[idx] = mlx5_counters_init[j]; 309 break; 310 } 311 } 312 } 313 /* Add IB counters. */ 314 for (i = 0; i != xstats_n; ++i) { 315 if (mlx5_counters_init[i].ib) { 316 unsigned int idx = xstats_ctrl->mlx5_stats_n++; 317 318 xstats_ctrl->info[idx] = mlx5_counters_init[i]; 319 xstats_ctrl->hw_stats[idx] = 0; 320 } 321 } 322 MLX5_ASSERT(xstats_ctrl->mlx5_stats_n <= MLX5_MAX_XSTATS); 323 xstats_ctrl->stats_n = dev_stats_n; 324 /* Copy to base at first time. */ 325 ret = mlx5_read_dev_counters(dev, xstats_ctrl->base); 326 if (ret) 327 DRV_LOG(ERR, "port %u cannot read device counters: %s", 328 dev->data->port_id, strerror(rte_errno)); 329 mlx5_read_ib_stat(priv, "out_of_buffer", &stats_ctrl->imissed_base); 330 stats_ctrl->imissed = 0; 331 free: 332 rte_free(strings); 333 } 334 335 /** 336 * DPDK callback to get extended device statistics. 337 * 338 * @param dev 339 * Pointer to Ethernet device. 340 * @param[out] stats 341 * Pointer to rte extended stats table. 342 * @param n 343 * The size of the stats table. 344 * 345 * @return 346 * Number of extended stats on success and stats is filled, 347 * negative on error and rte_errno is set. 348 */ 349 int 350 mlx5_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *stats, 351 unsigned int n) 352 { 353 struct mlx5_priv *priv = dev->data->dev_private; 354 unsigned int i; 355 uint64_t counters[n]; 356 struct mlx5_xstats_ctrl *xstats_ctrl = &priv->xstats_ctrl; 357 uint16_t mlx5_stats_n = xstats_ctrl->mlx5_stats_n; 358 359 if (n >= mlx5_stats_n && stats) { 360 int stats_n; 361 int ret; 362 363 stats_n = mlx5_ethtool_get_stats_n(dev); 364 if (stats_n < 0) 365 return stats_n; 366 if (xstats_ctrl->stats_n != stats_n) 367 mlx5_stats_init(dev); 368 ret = mlx5_read_dev_counters(dev, counters); 369 if (ret) 370 return ret; 371 for (i = 0; i != mlx5_stats_n; ++i) { 372 stats[i].id = i; 373 if (xstats_ctrl->info[i].ib) { 374 uint64_t wrap_n; 375 uint64_t hw_stat = xstats_ctrl->hw_stats[i]; 376 377 stats[i].value = (counters[i] - 378 xstats_ctrl->base[i]) & 379 (uint64_t)UINT32_MAX; 380 wrap_n = hw_stat >> 32; 381 if (stats[i].value < 382 (hw_stat & (uint64_t)UINT32_MAX)) 383 wrap_n++; 384 stats[i].value |= (wrap_n) << 32; 385 xstats_ctrl->hw_stats[i] = stats[i].value; 386 } else { 387 stats[i].value = 388 (counters[i] - xstats_ctrl->base[i]); 389 } 390 } 391 } 392 return mlx5_stats_n; 393 } 394 395 /** 396 * DPDK callback to get device statistics. 397 * 398 * @param dev 399 * Pointer to Ethernet device structure. 400 * @param[out] stats 401 * Stats structure output buffer. 402 * 403 * @return 404 * 0 on success and stats is filled, negative errno value otherwise and 405 * rte_errno is set. 406 */ 407 int 408 mlx5_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats) 409 { 410 struct mlx5_priv *priv = dev->data->dev_private; 411 struct mlx5_stats_ctrl *stats_ctrl = &priv->stats_ctrl; 412 struct rte_eth_stats tmp; 413 unsigned int i; 414 unsigned int idx; 415 uint64_t wrap_n; 416 int ret; 417 418 memset(&tmp, 0, sizeof(tmp)); 419 /* Add software counters. */ 420 for (i = 0; (i != priv->rxqs_n); ++i) { 421 struct mlx5_rxq_data *rxq = (*priv->rxqs)[i]; 422 423 if (rxq == NULL) 424 continue; 425 idx = rxq->idx; 426 if (idx < RTE_ETHDEV_QUEUE_STAT_CNTRS) { 427 #ifdef MLX5_PMD_SOFT_COUNTERS 428 tmp.q_ipackets[idx] += rxq->stats.ipackets; 429 tmp.q_ibytes[idx] += rxq->stats.ibytes; 430 #endif 431 tmp.q_errors[idx] += (rxq->stats.idropped + 432 rxq->stats.rx_nombuf); 433 } 434 #ifdef MLX5_PMD_SOFT_COUNTERS 435 tmp.ipackets += rxq->stats.ipackets; 436 tmp.ibytes += rxq->stats.ibytes; 437 #endif 438 tmp.ierrors += rxq->stats.idropped; 439 tmp.rx_nombuf += rxq->stats.rx_nombuf; 440 } 441 for (i = 0; (i != priv->txqs_n); ++i) { 442 struct mlx5_txq_data *txq = (*priv->txqs)[i]; 443 444 if (txq == NULL) 445 continue; 446 idx = txq->idx; 447 if (idx < RTE_ETHDEV_QUEUE_STAT_CNTRS) { 448 #ifdef MLX5_PMD_SOFT_COUNTERS 449 tmp.q_opackets[idx] += txq->stats.opackets; 450 tmp.q_obytes[idx] += txq->stats.obytes; 451 #endif 452 } 453 #ifdef MLX5_PMD_SOFT_COUNTERS 454 tmp.opackets += txq->stats.opackets; 455 tmp.obytes += txq->stats.obytes; 456 #endif 457 tmp.oerrors += txq->stats.oerrors; 458 } 459 ret = mlx5_read_ib_stat(priv, "out_of_buffer", &tmp.imissed); 460 if (ret == 0) { 461 tmp.imissed = (tmp.imissed - stats_ctrl->imissed_base) & 462 (uint64_t)UINT32_MAX; 463 wrap_n = stats_ctrl->imissed >> 32; 464 if (tmp.imissed < (stats_ctrl->imissed & (uint64_t)UINT32_MAX)) 465 wrap_n++; 466 tmp.imissed |= (wrap_n) << 32; 467 stats_ctrl->imissed = tmp.imissed; 468 } else { 469 tmp.imissed = stats_ctrl->imissed; 470 } 471 #ifndef MLX5_PMD_SOFT_COUNTERS 472 /* FIXME: retrieve and add hardware counters. */ 473 #endif 474 *stats = tmp; 475 return 0; 476 } 477 478 /** 479 * DPDK callback to clear device statistics. 480 * 481 * @param dev 482 * Pointer to Ethernet device structure. 483 * 484 * @return 485 * always 0 on success and stats is reset 486 */ 487 int 488 mlx5_stats_reset(struct rte_eth_dev *dev) 489 { 490 struct mlx5_priv *priv = dev->data->dev_private; 491 struct mlx5_stats_ctrl *stats_ctrl = &priv->stats_ctrl; 492 unsigned int i; 493 494 for (i = 0; (i != priv->rxqs_n); ++i) { 495 if ((*priv->rxqs)[i] == NULL) 496 continue; 497 memset(&(*priv->rxqs)[i]->stats, 0, 498 sizeof(struct mlx5_rxq_stats)); 499 } 500 for (i = 0; (i != priv->txqs_n); ++i) { 501 if ((*priv->txqs)[i] == NULL) 502 continue; 503 memset(&(*priv->txqs)[i]->stats, 0, 504 sizeof(struct mlx5_txq_stats)); 505 } 506 mlx5_read_ib_stat(priv, "out_of_buffer", &stats_ctrl->imissed_base); 507 stats_ctrl->imissed = 0; 508 #ifndef MLX5_PMD_SOFT_COUNTERS 509 /* FIXME: reset hardware counters. */ 510 #endif 511 512 return 0; 513 } 514 515 /** 516 * DPDK callback to clear device extended statistics. 517 * 518 * @param dev 519 * Pointer to Ethernet device structure. 520 * 521 * @return 522 * 0 on success and stats is reset, negative errno value otherwise and 523 * rte_errno is set. 524 */ 525 int 526 mlx5_xstats_reset(struct rte_eth_dev *dev) 527 { 528 struct mlx5_priv *priv = dev->data->dev_private; 529 struct mlx5_xstats_ctrl *xstats_ctrl = &priv->xstats_ctrl; 530 int stats_n; 531 unsigned int i; 532 unsigned int n = xstats_ctrl->mlx5_stats_n; 533 uint64_t counters[n]; 534 int ret; 535 536 stats_n = mlx5_ethtool_get_stats_n(dev); 537 if (stats_n < 0) { 538 DRV_LOG(ERR, "port %u cannot get stats: %s", dev->data->port_id, 539 strerror(-stats_n)); 540 return stats_n; 541 } 542 if (xstats_ctrl->stats_n != stats_n) 543 mlx5_stats_init(dev); 544 ret = mlx5_read_dev_counters(dev, counters); 545 if (ret) { 546 DRV_LOG(ERR, "port %u cannot read device counters: %s", 547 dev->data->port_id, strerror(rte_errno)); 548 return ret; 549 } 550 for (i = 0; i != n; ++i) { 551 xstats_ctrl->base[i] = counters[i]; 552 xstats_ctrl->hw_stats[i] = 0; 553 } 554 555 return 0; 556 } 557 558 /** 559 * DPDK callback to retrieve names of extended device statistics 560 * 561 * @param dev 562 * Pointer to Ethernet device structure. 563 * @param[out] xstats_names 564 * Buffer to insert names into. 565 * @param n 566 * Number of names. 567 * 568 * @return 569 * Number of xstats names. 570 */ 571 int 572 mlx5_xstats_get_names(struct rte_eth_dev *dev __rte_unused, 573 struct rte_eth_xstat_name *xstats_names, unsigned int n) 574 { 575 unsigned int i; 576 struct mlx5_priv *priv = dev->data->dev_private; 577 struct mlx5_xstats_ctrl *xstats_ctrl = &priv->xstats_ctrl; 578 unsigned int mlx5_xstats_n = xstats_ctrl->mlx5_stats_n; 579 580 if (n >= mlx5_xstats_n && xstats_names) { 581 for (i = 0; i != mlx5_xstats_n; ++i) { 582 strncpy(xstats_names[i].name, 583 xstats_ctrl->info[i].dpdk_name, 584 RTE_ETH_XSTATS_NAME_SIZE); 585 xstats_names[i].name[RTE_ETH_XSTATS_NAME_SIZE - 1] = 0; 586 } 587 } 588 return mlx5_xstats_n; 589 } 590