1 /*- 2 * BSD LICENSE 3 * 4 * Copyright(c) 2010-2015 Intel Corporation. All rights reserved. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include "rte_eth_ring.h" 35 #include <rte_mbuf.h> 36 #include <rte_ethdev.h> 37 #include <rte_malloc.h> 38 #include <rte_memcpy.h> 39 #include <rte_memzone.h> 40 #include <rte_string_fns.h> 41 #include <rte_vdev.h> 42 #include <rte_kvargs.h> 43 #include <rte_errno.h> 44 45 #define ETH_RING_NUMA_NODE_ACTION_ARG "nodeaction" 46 #define ETH_RING_ACTION_CREATE "CREATE" 47 #define ETH_RING_ACTION_ATTACH "ATTACH" 48 49 static const char *valid_arguments[] = { 50 ETH_RING_NUMA_NODE_ACTION_ARG, 51 NULL 52 }; 53 54 enum dev_action { 55 DEV_CREATE, 56 DEV_ATTACH 57 }; 58 59 struct ring_queue { 60 struct rte_ring *rng; 61 rte_atomic64_t rx_pkts; 62 rte_atomic64_t tx_pkts; 63 rte_atomic64_t err_pkts; 64 }; 65 66 struct pmd_internals { 67 unsigned max_rx_queues; 68 unsigned max_tx_queues; 69 70 struct ring_queue rx_ring_queues[RTE_PMD_RING_MAX_RX_RINGS]; 71 struct ring_queue tx_ring_queues[RTE_PMD_RING_MAX_TX_RINGS]; 72 73 struct ether_addr address; 74 enum dev_action action; 75 }; 76 77 78 static const char *drivername = "Rings PMD"; 79 static struct rte_eth_link pmd_link = { 80 .link_speed = ETH_SPEED_NUM_10G, 81 .link_duplex = ETH_LINK_FULL_DUPLEX, 82 .link_status = ETH_LINK_DOWN, 83 .link_autoneg = ETH_LINK_SPEED_AUTONEG 84 }; 85 86 static uint16_t 87 eth_ring_rx(void *q, struct rte_mbuf **bufs, uint16_t nb_bufs) 88 { 89 void **ptrs = (void *)&bufs[0]; 90 struct ring_queue *r = q; 91 const uint16_t nb_rx = (uint16_t)rte_ring_dequeue_burst(r->rng, 92 ptrs, nb_bufs); 93 if (r->rng->flags & RING_F_SC_DEQ) 94 r->rx_pkts.cnt += nb_rx; 95 else 96 rte_atomic64_add(&(r->rx_pkts), nb_rx); 97 return nb_rx; 98 } 99 100 static uint16_t 101 eth_ring_tx(void *q, struct rte_mbuf **bufs, uint16_t nb_bufs) 102 { 103 void **ptrs = (void *)&bufs[0]; 104 struct ring_queue *r = q; 105 const uint16_t nb_tx = (uint16_t)rte_ring_enqueue_burst(r->rng, 106 ptrs, nb_bufs); 107 if (r->rng->flags & RING_F_SP_ENQ) { 108 r->tx_pkts.cnt += nb_tx; 109 r->err_pkts.cnt += nb_bufs - nb_tx; 110 } else { 111 rte_atomic64_add(&(r->tx_pkts), nb_tx); 112 rte_atomic64_add(&(r->err_pkts), nb_bufs - nb_tx); 113 } 114 return nb_tx; 115 } 116 117 static int 118 eth_dev_configure(struct rte_eth_dev *dev __rte_unused) { return 0; } 119 120 static int 121 eth_dev_start(struct rte_eth_dev *dev) 122 { 123 dev->data->dev_link.link_status = ETH_LINK_UP; 124 return 0; 125 } 126 127 static void 128 eth_dev_stop(struct rte_eth_dev *dev) 129 { 130 dev->data->dev_link.link_status = ETH_LINK_DOWN; 131 } 132 133 static int 134 eth_dev_set_link_down(struct rte_eth_dev *dev) 135 { 136 dev->data->dev_link.link_status = ETH_LINK_DOWN; 137 return 0; 138 } 139 140 static int 141 eth_dev_set_link_up(struct rte_eth_dev *dev) 142 { 143 dev->data->dev_link.link_status = ETH_LINK_UP; 144 return 0; 145 } 146 147 static int 148 eth_rx_queue_setup(struct rte_eth_dev *dev, uint16_t rx_queue_id, 149 uint16_t nb_rx_desc __rte_unused, 150 unsigned int socket_id __rte_unused, 151 const struct rte_eth_rxconf *rx_conf __rte_unused, 152 struct rte_mempool *mb_pool __rte_unused) 153 { 154 struct pmd_internals *internals = dev->data->dev_private; 155 dev->data->rx_queues[rx_queue_id] = &internals->rx_ring_queues[rx_queue_id]; 156 return 0; 157 } 158 159 static int 160 eth_tx_queue_setup(struct rte_eth_dev *dev, uint16_t tx_queue_id, 161 uint16_t nb_tx_desc __rte_unused, 162 unsigned int socket_id __rte_unused, 163 const struct rte_eth_txconf *tx_conf __rte_unused) 164 { 165 struct pmd_internals *internals = dev->data->dev_private; 166 dev->data->tx_queues[tx_queue_id] = &internals->tx_ring_queues[tx_queue_id]; 167 return 0; 168 } 169 170 171 static void 172 eth_dev_info(struct rte_eth_dev *dev, 173 struct rte_eth_dev_info *dev_info) 174 { 175 struct pmd_internals *internals = dev->data->dev_private; 176 dev_info->driver_name = drivername; 177 dev_info->max_mac_addrs = 1; 178 dev_info->max_rx_pktlen = (uint32_t)-1; 179 dev_info->max_rx_queues = (uint16_t)internals->max_rx_queues; 180 dev_info->max_tx_queues = (uint16_t)internals->max_tx_queues; 181 dev_info->min_rx_bufsize = 0; 182 dev_info->pci_dev = NULL; 183 } 184 185 static void 186 eth_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats) 187 { 188 unsigned i; 189 unsigned long rx_total = 0, tx_total = 0, tx_err_total = 0; 190 const struct pmd_internals *internal = dev->data->dev_private; 191 192 for (i = 0; i < RTE_ETHDEV_QUEUE_STAT_CNTRS && 193 i < dev->data->nb_rx_queues; i++) { 194 stats->q_ipackets[i] = internal->rx_ring_queues[i].rx_pkts.cnt; 195 rx_total += stats->q_ipackets[i]; 196 } 197 198 for (i = 0; i < RTE_ETHDEV_QUEUE_STAT_CNTRS && 199 i < dev->data->nb_tx_queues; i++) { 200 stats->q_opackets[i] = internal->tx_ring_queues[i].tx_pkts.cnt; 201 stats->q_errors[i] = internal->tx_ring_queues[i].err_pkts.cnt; 202 tx_total += stats->q_opackets[i]; 203 tx_err_total += stats->q_errors[i]; 204 } 205 206 stats->ipackets = rx_total; 207 stats->opackets = tx_total; 208 stats->oerrors = tx_err_total; 209 } 210 211 static void 212 eth_stats_reset(struct rte_eth_dev *dev) 213 { 214 unsigned i; 215 struct pmd_internals *internal = dev->data->dev_private; 216 for (i = 0; i < dev->data->nb_rx_queues; i++) 217 internal->rx_ring_queues[i].rx_pkts.cnt = 0; 218 for (i = 0; i < dev->data->nb_tx_queues; i++) { 219 internal->tx_ring_queues[i].tx_pkts.cnt = 0; 220 internal->tx_ring_queues[i].err_pkts.cnt = 0; 221 } 222 } 223 224 static void 225 eth_mac_addr_remove(struct rte_eth_dev *dev __rte_unused, 226 uint32_t index __rte_unused) 227 { 228 } 229 230 static void 231 eth_mac_addr_add(struct rte_eth_dev *dev __rte_unused, 232 struct ether_addr *mac_addr __rte_unused, 233 uint32_t index __rte_unused, 234 uint32_t vmdq __rte_unused) 235 { 236 } 237 238 static void 239 eth_queue_release(void *q __rte_unused) { ; } 240 static int 241 eth_link_update(struct rte_eth_dev *dev __rte_unused, 242 int wait_to_complete __rte_unused) { return 0; } 243 244 static const struct eth_dev_ops ops = { 245 .dev_start = eth_dev_start, 246 .dev_stop = eth_dev_stop, 247 .dev_set_link_up = eth_dev_set_link_up, 248 .dev_set_link_down = eth_dev_set_link_down, 249 .dev_configure = eth_dev_configure, 250 .dev_infos_get = eth_dev_info, 251 .rx_queue_setup = eth_rx_queue_setup, 252 .tx_queue_setup = eth_tx_queue_setup, 253 .rx_queue_release = eth_queue_release, 254 .tx_queue_release = eth_queue_release, 255 .link_update = eth_link_update, 256 .stats_get = eth_stats_get, 257 .stats_reset = eth_stats_reset, 258 .mac_addr_remove = eth_mac_addr_remove, 259 .mac_addr_add = eth_mac_addr_add, 260 }; 261 262 static int 263 do_eth_dev_ring_create(const char *name, 264 struct rte_ring * const rx_queues[], const unsigned nb_rx_queues, 265 struct rte_ring *const tx_queues[], const unsigned nb_tx_queues, 266 const unsigned numa_node, enum dev_action action) 267 { 268 struct rte_eth_dev_data *data = NULL; 269 struct pmd_internals *internals = NULL; 270 struct rte_eth_dev *eth_dev = NULL; 271 unsigned i; 272 273 RTE_LOG(INFO, PMD, "Creating rings-backed ethdev on numa socket %u\n", 274 numa_node); 275 276 /* now do all data allocation - for eth_dev structure, dummy pci driver 277 * and internal (private) data 278 */ 279 data = rte_zmalloc_socket(name, sizeof(*data), 0, numa_node); 280 if (data == NULL) { 281 rte_errno = ENOMEM; 282 goto error; 283 } 284 285 data->rx_queues = rte_zmalloc_socket(name, 286 sizeof(void *) * nb_rx_queues, 0, numa_node); 287 if (data->rx_queues == NULL) { 288 rte_errno = ENOMEM; 289 goto error; 290 } 291 292 data->tx_queues = rte_zmalloc_socket(name, 293 sizeof(void *) * nb_tx_queues, 0, numa_node); 294 if (data->tx_queues == NULL) { 295 rte_errno = ENOMEM; 296 goto error; 297 } 298 299 internals = rte_zmalloc_socket(name, sizeof(*internals), 0, numa_node); 300 if (internals == NULL) { 301 rte_errno = ENOMEM; 302 goto error; 303 } 304 305 /* reserve an ethdev entry */ 306 eth_dev = rte_eth_dev_allocate(name); 307 if (eth_dev == NULL) { 308 rte_errno = ENOSPC; 309 goto error; 310 } 311 312 /* now put it all together 313 * - store queue data in internals, 314 * - store numa_node info in eth_dev_data 315 * - point eth_dev_data to internals 316 * - and point eth_dev structure to new eth_dev_data structure 317 */ 318 /* NOTE: we'll replace the data element, of originally allocated eth_dev 319 * so the rings are local per-process */ 320 321 internals->action = action; 322 internals->max_rx_queues = nb_rx_queues; 323 internals->max_tx_queues = nb_tx_queues; 324 for (i = 0; i < nb_rx_queues; i++) { 325 internals->rx_ring_queues[i].rng = rx_queues[i]; 326 data->rx_queues[i] = &internals->rx_ring_queues[i]; 327 } 328 for (i = 0; i < nb_tx_queues; i++) { 329 internals->tx_ring_queues[i].rng = tx_queues[i]; 330 data->tx_queues[i] = &internals->tx_ring_queues[i]; 331 } 332 333 data->dev_private = internals; 334 data->port_id = eth_dev->data->port_id; 335 memmove(data->name, eth_dev->data->name, sizeof(data->name)); 336 data->nb_rx_queues = (uint16_t)nb_rx_queues; 337 data->nb_tx_queues = (uint16_t)nb_tx_queues; 338 data->dev_link = pmd_link; 339 data->mac_addrs = &internals->address; 340 341 eth_dev->data = data; 342 eth_dev->driver = NULL; 343 eth_dev->dev_ops = &ops; 344 data->dev_flags = RTE_ETH_DEV_DETACHABLE; 345 data->kdrv = RTE_KDRV_NONE; 346 data->drv_name = drivername; 347 data->numa_node = numa_node; 348 349 /* finally assign rx and tx ops */ 350 eth_dev->rx_pkt_burst = eth_ring_rx; 351 eth_dev->tx_pkt_burst = eth_ring_tx; 352 353 return data->port_id; 354 355 error: 356 if (data) { 357 rte_free(data->rx_queues); 358 rte_free(data->tx_queues); 359 } 360 rte_free(data); 361 rte_free(internals); 362 363 return -1; 364 } 365 366 int 367 rte_eth_from_rings(const char *name, struct rte_ring *const rx_queues[], 368 const unsigned nb_rx_queues, 369 struct rte_ring *const tx_queues[], 370 const unsigned nb_tx_queues, 371 const unsigned numa_node) 372 { 373 /* do some parameter checking */ 374 if (rx_queues == NULL && nb_rx_queues > 0) { 375 rte_errno = EINVAL; 376 return -1; 377 } 378 if (tx_queues == NULL && nb_tx_queues > 0) { 379 rte_errno = EINVAL; 380 return -1; 381 } 382 if (nb_rx_queues > RTE_PMD_RING_MAX_RX_RINGS) { 383 rte_errno = EINVAL; 384 return -1; 385 } 386 387 return do_eth_dev_ring_create(name, rx_queues, nb_rx_queues, 388 tx_queues, nb_tx_queues, numa_node, DEV_ATTACH); 389 } 390 391 int 392 rte_eth_from_ring(struct rte_ring *r) 393 { 394 return rte_eth_from_rings(r->name, &r, 1, &r, 1, 395 r->memzone ? r->memzone->socket_id : SOCKET_ID_ANY); 396 } 397 398 static int 399 eth_dev_ring_create(const char *name, const unsigned numa_node, 400 enum dev_action action) 401 { 402 /* rx and tx are so-called from point of view of first port. 403 * They are inverted from the point of view of second port 404 */ 405 struct rte_ring *rxtx[RTE_PMD_RING_MAX_RX_RINGS]; 406 unsigned i; 407 char rng_name[RTE_RING_NAMESIZE]; 408 unsigned num_rings = RTE_MIN(RTE_PMD_RING_MAX_RX_RINGS, 409 RTE_PMD_RING_MAX_TX_RINGS); 410 411 for (i = 0; i < num_rings; i++) { 412 snprintf(rng_name, sizeof(rng_name), "ETH_RXTX%u_%s", i, name); 413 rxtx[i] = (action == DEV_CREATE) ? 414 rte_ring_create(rng_name, 1024, numa_node, 415 RING_F_SP_ENQ|RING_F_SC_DEQ) : 416 rte_ring_lookup(rng_name); 417 if (rxtx[i] == NULL) 418 return -1; 419 } 420 421 if (do_eth_dev_ring_create(name, rxtx, num_rings, rxtx, num_rings, 422 numa_node, action) < 0) 423 return -1; 424 425 return 0; 426 } 427 428 struct node_action_pair { 429 char name[PATH_MAX]; 430 unsigned node; 431 enum dev_action action; 432 }; 433 434 struct node_action_list { 435 unsigned total; 436 unsigned count; 437 struct node_action_pair *list; 438 }; 439 440 static int parse_kvlist (const char *key __rte_unused, const char *value, void *data) 441 { 442 struct node_action_list *info = data; 443 int ret; 444 char *name; 445 char *action; 446 char *node; 447 char *end; 448 449 name = strdup(value); 450 451 ret = -EINVAL; 452 453 if (!name) { 454 RTE_LOG(WARNING, PMD, "command line paramter is empty for ring pmd!\n"); 455 goto out; 456 } 457 458 node = strchr(name, ':'); 459 if (!node) { 460 RTE_LOG(WARNING, PMD, "could not parse node value from %s", name); 461 goto out; 462 } 463 464 *node = '\0'; 465 node++; 466 467 action = strchr(node, ':'); 468 if (!action) { 469 RTE_LOG(WARNING, PMD, "could not action value from %s", node); 470 goto out; 471 } 472 473 *action = '\0'; 474 action++; 475 476 /* 477 * Need to do some sanity checking here 478 */ 479 480 if (strcmp(action, ETH_RING_ACTION_ATTACH) == 0) 481 info->list[info->count].action = DEV_ATTACH; 482 else if (strcmp(action, ETH_RING_ACTION_CREATE) == 0) 483 info->list[info->count].action = DEV_CREATE; 484 else 485 goto out; 486 487 errno = 0; 488 info->list[info->count].node = strtol(node, &end, 10); 489 490 if ((errno != 0) || (*end != '\0')) { 491 RTE_LOG(WARNING, PMD, "node value %s is unparseable as a number\n", node); 492 goto out; 493 } 494 495 snprintf(info->list[info->count].name, sizeof(info->list[info->count].name), "%s", name); 496 497 info->count++; 498 499 ret = 0; 500 out: 501 free(name); 502 return ret; 503 } 504 505 static int 506 rte_pmd_ring_probe(const char *name, const char *params) 507 { 508 struct rte_kvargs *kvlist = NULL; 509 int ret = 0; 510 struct node_action_list *info = NULL; 511 512 RTE_LOG(INFO, PMD, "Initializing pmd_ring for %s\n", name); 513 514 if (params == NULL || params[0] == '\0') { 515 ret = eth_dev_ring_create(name, rte_socket_id(), DEV_CREATE); 516 if (ret == -1) { 517 RTE_LOG(INFO, PMD, 518 "Attach to pmd_ring for %s\n", name); 519 ret = eth_dev_ring_create(name, rte_socket_id(), 520 DEV_ATTACH); 521 } 522 } 523 else { 524 kvlist = rte_kvargs_parse(params, valid_arguments); 525 526 if (!kvlist) { 527 RTE_LOG(INFO, PMD, "Ignoring unsupported parameters when creating" 528 " rings-backed ethernet device\n"); 529 ret = eth_dev_ring_create(name, rte_socket_id(), 530 DEV_CREATE); 531 if (ret == -1) { 532 RTE_LOG(INFO, PMD, 533 "Attach to pmd_ring for %s\n", 534 name); 535 ret = eth_dev_ring_create(name, rte_socket_id(), 536 DEV_ATTACH); 537 } 538 return ret; 539 } else { 540 ret = rte_kvargs_count(kvlist, ETH_RING_NUMA_NODE_ACTION_ARG); 541 info = rte_zmalloc("struct node_action_list", 542 sizeof(struct node_action_list) + 543 (sizeof(struct node_action_pair) * ret), 544 0); 545 if (!info) 546 goto out_free; 547 548 info->total = ret; 549 info->list = (struct node_action_pair*)(info + 1); 550 551 ret = rte_kvargs_process(kvlist, ETH_RING_NUMA_NODE_ACTION_ARG, 552 parse_kvlist, info); 553 554 if (ret < 0) 555 goto out_free; 556 557 for (info->count = 0; info->count < info->total; info->count++) { 558 ret = eth_dev_ring_create(info->list[info->count].name, 559 info->list[info->count].node, 560 info->list[info->count].action); 561 if ((ret == -1) && 562 (info->list[info->count].action == DEV_CREATE)) { 563 RTE_LOG(INFO, PMD, 564 "Attach to pmd_ring for %s\n", 565 name); 566 ret = eth_dev_ring_create(name, 567 info->list[info->count].node, 568 DEV_ATTACH); 569 } 570 } 571 } 572 } 573 574 out_free: 575 rte_kvargs_free(kvlist); 576 rte_free(info); 577 return ret; 578 } 579 580 static int 581 rte_pmd_ring_remove(const char *name) 582 { 583 struct rte_eth_dev *eth_dev = NULL; 584 struct pmd_internals *internals = NULL; 585 struct ring_queue *r = NULL; 586 uint16_t i; 587 588 RTE_LOG(INFO, PMD, "Un-Initializing pmd_ring for %s\n", name); 589 590 if (name == NULL) 591 return -EINVAL; 592 593 /* find an ethdev entry */ 594 eth_dev = rte_eth_dev_allocated(name); 595 if (eth_dev == NULL) 596 return -ENODEV; 597 598 eth_dev_stop(eth_dev); 599 600 internals = eth_dev->data->dev_private; 601 if (internals->action == DEV_CREATE) { 602 /* 603 * it is only necessary to delete the rings in rx_queues because 604 * they are the same used in tx_queues 605 */ 606 for (i = 0; i < eth_dev->data->nb_rx_queues; i++) { 607 r = eth_dev->data->rx_queues[i]; 608 rte_ring_free(r->rng); 609 } 610 } 611 612 rte_free(eth_dev->data->rx_queues); 613 rte_free(eth_dev->data->tx_queues); 614 rte_free(eth_dev->data->dev_private); 615 616 rte_free(eth_dev->data); 617 618 rte_eth_dev_release_port(eth_dev); 619 return 0; 620 } 621 622 static struct rte_vdev_driver pmd_ring_drv = { 623 .probe = rte_pmd_ring_probe, 624 .remove = rte_pmd_ring_remove, 625 }; 626 627 RTE_PMD_REGISTER_VDEV(net_ring, pmd_ring_drv); 628 RTE_PMD_REGISTER_ALIAS(net_ring, eth_ring); 629 RTE_PMD_REGISTER_PARAM_STRING(net_ring, 630 ETH_RING_NUMA_NODE_ACTION_ARG "=name:node:action(ATTACH|CREATE)"); 631