1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2016 Intel Corporation 3 */ 4 #include <string.h> 5 #include <stdint.h> 6 #include <unistd.h> 7 8 #include <rte_mbuf.h> 9 #include <rte_malloc.h> 10 11 #include "rte_port_fd.h" 12 13 #include "port_log.h" 14 15 /* 16 * Port FD Reader 17 */ 18 #ifdef RTE_PORT_STATS_COLLECT 19 20 #define RTE_PORT_FD_READER_STATS_PKTS_IN_ADD(port, val) \ 21 do { port->stats.n_pkts_in += val; } while (0) 22 #define RTE_PORT_FD_READER_STATS_PKTS_DROP_ADD(port, val) \ 23 do { port->stats.n_pkts_drop += val; } while (0) 24 25 #else 26 27 #define RTE_PORT_FD_READER_STATS_PKTS_IN_ADD(port, val) 28 #define RTE_PORT_FD_READER_STATS_PKTS_DROP_ADD(port, val) 29 30 #endif 31 32 struct rte_port_fd_reader { 33 struct rte_port_in_stats stats; 34 int fd; 35 uint32_t mtu; 36 struct rte_mempool *mempool; 37 }; 38 39 static void * 40 rte_port_fd_reader_create(void *params, int socket_id) 41 { 42 struct rte_port_fd_reader_params *conf = 43 params; 44 struct rte_port_fd_reader *port; 45 46 /* Check input parameters */ 47 if (conf == NULL) { 48 PORT_LOG(ERR, "%s: params is NULL", __func__); 49 return NULL; 50 } 51 if (conf->fd < 0) { 52 PORT_LOG(ERR, "%s: Invalid file descriptor", __func__); 53 return NULL; 54 } 55 if (conf->mtu == 0) { 56 PORT_LOG(ERR, "%s: Invalid MTU", __func__); 57 return NULL; 58 } 59 if (conf->mempool == NULL) { 60 PORT_LOG(ERR, "%s: Invalid mempool", __func__); 61 return NULL; 62 } 63 64 /* Memory allocation */ 65 port = rte_zmalloc_socket("PORT", sizeof(*port), 66 RTE_CACHE_LINE_SIZE, socket_id); 67 if (port == NULL) { 68 PORT_LOG(ERR, "%s: Failed to allocate port", __func__); 69 return NULL; 70 } 71 72 /* Initialization */ 73 port->fd = conf->fd; 74 port->mtu = conf->mtu; 75 port->mempool = conf->mempool; 76 77 return port; 78 } 79 80 static int 81 rte_port_fd_reader_rx(void *port, struct rte_mbuf **pkts, uint32_t n_pkts) 82 { 83 struct rte_port_fd_reader *p = port; 84 uint32_t i, j; 85 86 if (rte_pktmbuf_alloc_bulk(p->mempool, pkts, n_pkts) != 0) 87 return 0; 88 89 for (i = 0; i < n_pkts; i++) { 90 struct rte_mbuf *pkt = pkts[i]; 91 void *pkt_data = rte_pktmbuf_mtod(pkt, void *); 92 ssize_t n_bytes; 93 94 n_bytes = read(p->fd, pkt_data, (size_t) p->mtu); 95 if (n_bytes <= 0) 96 break; 97 98 pkt->data_len = n_bytes; 99 pkt->pkt_len = n_bytes; 100 } 101 102 for (j = i; j < n_pkts; j++) 103 rte_pktmbuf_free(pkts[j]); 104 105 RTE_PORT_FD_READER_STATS_PKTS_IN_ADD(p, i); 106 107 return i; 108 } 109 110 static int 111 rte_port_fd_reader_free(void *port) 112 { 113 if (port == NULL) { 114 PORT_LOG(ERR, "%s: port is NULL", __func__); 115 return -EINVAL; 116 } 117 118 rte_free(port); 119 120 return 0; 121 } 122 123 static int rte_port_fd_reader_stats_read(void *port, 124 struct rte_port_in_stats *stats, int clear) 125 { 126 struct rte_port_fd_reader *p = 127 port; 128 129 if (stats != NULL) 130 memcpy(stats, &p->stats, sizeof(p->stats)); 131 132 if (clear) 133 memset(&p->stats, 0, sizeof(p->stats)); 134 135 return 0; 136 } 137 138 /* 139 * Port FD Writer 140 */ 141 #ifdef RTE_PORT_STATS_COLLECT 142 143 #define RTE_PORT_FD_WRITER_STATS_PKTS_IN_ADD(port, val) \ 144 do { port->stats.n_pkts_in += val; } while (0) 145 #define RTE_PORT_FD_WRITER_STATS_PKTS_DROP_ADD(port, val) \ 146 do { port->stats.n_pkts_drop += val; } while (0) 147 148 #else 149 150 #define RTE_PORT_FD_WRITER_STATS_PKTS_IN_ADD(port, val) 151 #define RTE_PORT_FD_WRITER_STATS_PKTS_DROP_ADD(port, val) 152 153 #endif 154 155 struct rte_port_fd_writer { 156 struct rte_port_out_stats stats; 157 158 struct rte_mbuf *tx_buf[2 * RTE_PORT_IN_BURST_SIZE_MAX]; 159 uint32_t tx_burst_sz; 160 uint16_t tx_buf_count; 161 uint32_t fd; 162 }; 163 164 static void * 165 rte_port_fd_writer_create(void *params, int socket_id) 166 { 167 struct rte_port_fd_writer_params *conf = 168 params; 169 struct rte_port_fd_writer *port; 170 171 /* Check input parameters */ 172 if ((conf == NULL) || 173 (conf->tx_burst_sz == 0) || 174 (conf->tx_burst_sz > RTE_PORT_IN_BURST_SIZE_MAX) || 175 (!rte_is_power_of_2(conf->tx_burst_sz))) { 176 PORT_LOG(ERR, "%s: Invalid input parameters", __func__); 177 return NULL; 178 } 179 180 /* Memory allocation */ 181 port = rte_zmalloc_socket("PORT", sizeof(*port), 182 RTE_CACHE_LINE_SIZE, socket_id); 183 if (port == NULL) { 184 PORT_LOG(ERR, "%s: Failed to allocate port", __func__); 185 return NULL; 186 } 187 188 /* Initialization */ 189 port->fd = conf->fd; 190 port->tx_burst_sz = conf->tx_burst_sz; 191 port->tx_buf_count = 0; 192 193 return port; 194 } 195 196 static inline void 197 send_burst(struct rte_port_fd_writer *p) 198 { 199 uint32_t i; 200 201 for (i = 0; i < p->tx_buf_count; i++) { 202 struct rte_mbuf *pkt = p->tx_buf[i]; 203 void *pkt_data = rte_pktmbuf_mtod(pkt, void*); 204 size_t n_bytes = rte_pktmbuf_data_len(pkt); 205 ssize_t ret; 206 207 ret = write(p->fd, pkt_data, n_bytes); 208 if (ret < 0) 209 break; 210 } 211 212 RTE_PORT_FD_WRITER_STATS_PKTS_DROP_ADD(p, p->tx_buf_count - i); 213 214 for (i = 0; i < p->tx_buf_count; i++) 215 rte_pktmbuf_free(p->tx_buf[i]); 216 217 p->tx_buf_count = 0; 218 } 219 220 static int 221 rte_port_fd_writer_tx(void *port, struct rte_mbuf *pkt) 222 { 223 struct rte_port_fd_writer *p = 224 port; 225 226 p->tx_buf[p->tx_buf_count++] = pkt; 227 RTE_PORT_FD_WRITER_STATS_PKTS_IN_ADD(p, 1); 228 if (p->tx_buf_count >= p->tx_burst_sz) 229 send_burst(p); 230 231 return 0; 232 } 233 234 static int 235 rte_port_fd_writer_tx_bulk(void *port, 236 struct rte_mbuf **pkts, 237 uint64_t pkts_mask) 238 { 239 struct rte_port_fd_writer *p = 240 port; 241 uint32_t tx_buf_count = p->tx_buf_count; 242 243 if ((pkts_mask & (pkts_mask + 1)) == 0) { 244 uint64_t n_pkts = rte_popcount64(pkts_mask); 245 uint32_t i; 246 247 for (i = 0; i < n_pkts; i++) 248 p->tx_buf[tx_buf_count++] = pkts[i]; 249 RTE_PORT_FD_WRITER_STATS_PKTS_IN_ADD(p, n_pkts); 250 } else 251 for ( ; pkts_mask; ) { 252 uint32_t pkt_index = rte_ctz64(pkts_mask); 253 uint64_t pkt_mask = 1LLU << pkt_index; 254 struct rte_mbuf *pkt = pkts[pkt_index]; 255 256 p->tx_buf[tx_buf_count++] = pkt; 257 RTE_PORT_FD_WRITER_STATS_PKTS_IN_ADD(p, 1); 258 pkts_mask &= ~pkt_mask; 259 } 260 261 p->tx_buf_count = tx_buf_count; 262 if (tx_buf_count >= p->tx_burst_sz) 263 send_burst(p); 264 265 return 0; 266 } 267 268 static int 269 rte_port_fd_writer_flush(void *port) 270 { 271 struct rte_port_fd_writer *p = 272 port; 273 274 if (p->tx_buf_count > 0) 275 send_burst(p); 276 277 return 0; 278 } 279 280 static int 281 rte_port_fd_writer_free(void *port) 282 { 283 if (port == NULL) { 284 PORT_LOG(ERR, "%s: Port is NULL", __func__); 285 return -EINVAL; 286 } 287 288 rte_port_fd_writer_flush(port); 289 rte_free(port); 290 291 return 0; 292 } 293 294 static int rte_port_fd_writer_stats_read(void *port, 295 struct rte_port_out_stats *stats, int clear) 296 { 297 struct rte_port_fd_writer *p = 298 port; 299 300 if (stats != NULL) 301 memcpy(stats, &p->stats, sizeof(p->stats)); 302 303 if (clear) 304 memset(&p->stats, 0, sizeof(p->stats)); 305 306 return 0; 307 } 308 309 /* 310 * Port FD Writer Nodrop 311 */ 312 #ifdef RTE_PORT_STATS_COLLECT 313 314 #define RTE_PORT_FD_WRITER_NODROP_STATS_PKTS_IN_ADD(port, val) \ 315 do { port->stats.n_pkts_in += val; } while (0) 316 #define RTE_PORT_FD_WRITER_NODROP_STATS_PKTS_DROP_ADD(port, val) \ 317 do { port->stats.n_pkts_drop += val; } while (0) 318 319 #else 320 321 #define RTE_PORT_FD_WRITER_NODROP_STATS_PKTS_IN_ADD(port, val) 322 #define RTE_PORT_FD_WRITER_NODROP_STATS_PKTS_DROP_ADD(port, val) 323 324 #endif 325 326 struct rte_port_fd_writer_nodrop { 327 struct rte_port_out_stats stats; 328 329 struct rte_mbuf *tx_buf[2 * RTE_PORT_IN_BURST_SIZE_MAX]; 330 uint32_t tx_burst_sz; 331 uint16_t tx_buf_count; 332 uint64_t n_retries; 333 uint32_t fd; 334 }; 335 336 static void * 337 rte_port_fd_writer_nodrop_create(void *params, int socket_id) 338 { 339 struct rte_port_fd_writer_nodrop_params *conf = 340 params; 341 struct rte_port_fd_writer_nodrop *port; 342 343 /* Check input parameters */ 344 if ((conf == NULL) || 345 (conf->fd < 0) || 346 (conf->tx_burst_sz == 0) || 347 (conf->tx_burst_sz > RTE_PORT_IN_BURST_SIZE_MAX) || 348 (!rte_is_power_of_2(conf->tx_burst_sz))) { 349 PORT_LOG(ERR, "%s: Invalid input parameters", __func__); 350 return NULL; 351 } 352 353 /* Memory allocation */ 354 port = rte_zmalloc_socket("PORT", sizeof(*port), 355 RTE_CACHE_LINE_SIZE, socket_id); 356 if (port == NULL) { 357 PORT_LOG(ERR, "%s: Failed to allocate port", __func__); 358 return NULL; 359 } 360 361 /* Initialization */ 362 port->fd = conf->fd; 363 port->tx_burst_sz = conf->tx_burst_sz; 364 port->tx_buf_count = 0; 365 366 /* 367 * When n_retries is 0 it means that we should wait for every packet to 368 * send no matter how many retries should it take. To limit number of 369 * branches in fast path, we use UINT64_MAX instead of branching. 370 */ 371 port->n_retries = (conf->n_retries == 0) ? UINT64_MAX : conf->n_retries; 372 373 return port; 374 } 375 376 static inline void 377 send_burst_nodrop(struct rte_port_fd_writer_nodrop *p) 378 { 379 uint64_t n_retries; 380 uint32_t i; 381 382 n_retries = 0; 383 for (i = 0; (i < p->tx_buf_count) && (n_retries < p->n_retries); i++) { 384 struct rte_mbuf *pkt = p->tx_buf[i]; 385 void *pkt_data = rte_pktmbuf_mtod(pkt, void*); 386 size_t n_bytes = rte_pktmbuf_data_len(pkt); 387 388 for ( ; n_retries < p->n_retries; n_retries++) { 389 ssize_t ret; 390 391 ret = write(p->fd, pkt_data, n_bytes); 392 if (ret) 393 break; 394 } 395 } 396 397 RTE_PORT_FD_WRITER_NODROP_STATS_PKTS_DROP_ADD(p, p->tx_buf_count - i); 398 399 for (i = 0; i < p->tx_buf_count; i++) 400 rte_pktmbuf_free(p->tx_buf[i]); 401 402 p->tx_buf_count = 0; 403 } 404 405 static int 406 rte_port_fd_writer_nodrop_tx(void *port, struct rte_mbuf *pkt) 407 { 408 struct rte_port_fd_writer_nodrop *p = 409 port; 410 411 p->tx_buf[p->tx_buf_count++] = pkt; 412 RTE_PORT_FD_WRITER_NODROP_STATS_PKTS_IN_ADD(p, 1); 413 if (p->tx_buf_count >= p->tx_burst_sz) 414 send_burst_nodrop(p); 415 416 return 0; 417 } 418 419 static int 420 rte_port_fd_writer_nodrop_tx_bulk(void *port, 421 struct rte_mbuf **pkts, 422 uint64_t pkts_mask) 423 { 424 struct rte_port_fd_writer_nodrop *p = 425 port; 426 uint32_t tx_buf_count = p->tx_buf_count; 427 428 if ((pkts_mask & (pkts_mask + 1)) == 0) { 429 uint64_t n_pkts = rte_popcount64(pkts_mask); 430 uint32_t i; 431 432 for (i = 0; i < n_pkts; i++) 433 p->tx_buf[tx_buf_count++] = pkts[i]; 434 RTE_PORT_FD_WRITER_NODROP_STATS_PKTS_IN_ADD(p, n_pkts); 435 } else 436 for ( ; pkts_mask; ) { 437 uint32_t pkt_index = rte_ctz64(pkts_mask); 438 uint64_t pkt_mask = 1LLU << pkt_index; 439 struct rte_mbuf *pkt = pkts[pkt_index]; 440 441 p->tx_buf[tx_buf_count++] = pkt; 442 RTE_PORT_FD_WRITER_NODROP_STATS_PKTS_IN_ADD(p, 1); 443 pkts_mask &= ~pkt_mask; 444 } 445 446 p->tx_buf_count = tx_buf_count; 447 if (tx_buf_count >= p->tx_burst_sz) 448 send_burst_nodrop(p); 449 450 return 0; 451 } 452 453 static int 454 rte_port_fd_writer_nodrop_flush(void *port) 455 { 456 struct rte_port_fd_writer_nodrop *p = 457 port; 458 459 if (p->tx_buf_count > 0) 460 send_burst_nodrop(p); 461 462 return 0; 463 } 464 465 static int 466 rte_port_fd_writer_nodrop_free(void *port) 467 { 468 if (port == NULL) { 469 PORT_LOG(ERR, "%s: Port is NULL", __func__); 470 return -EINVAL; 471 } 472 473 rte_port_fd_writer_nodrop_flush(port); 474 rte_free(port); 475 476 return 0; 477 } 478 479 static int rte_port_fd_writer_nodrop_stats_read(void *port, 480 struct rte_port_out_stats *stats, int clear) 481 { 482 struct rte_port_fd_writer_nodrop *p = 483 port; 484 485 if (stats != NULL) 486 memcpy(stats, &p->stats, sizeof(p->stats)); 487 488 if (clear) 489 memset(&p->stats, 0, sizeof(p->stats)); 490 491 return 0; 492 } 493 494 /* 495 * Summary of port operations 496 */ 497 struct rte_port_in_ops rte_port_fd_reader_ops = { 498 .f_create = rte_port_fd_reader_create, 499 .f_free = rte_port_fd_reader_free, 500 .f_rx = rte_port_fd_reader_rx, 501 .f_stats = rte_port_fd_reader_stats_read, 502 }; 503 504 struct rte_port_out_ops rte_port_fd_writer_ops = { 505 .f_create = rte_port_fd_writer_create, 506 .f_free = rte_port_fd_writer_free, 507 .f_tx = rte_port_fd_writer_tx, 508 .f_tx_bulk = rte_port_fd_writer_tx_bulk, 509 .f_flush = rte_port_fd_writer_flush, 510 .f_stats = rte_port_fd_writer_stats_read, 511 }; 512 513 struct rte_port_out_ops rte_port_fd_writer_nodrop_ops = { 514 .f_create = rte_port_fd_writer_nodrop_create, 515 .f_free = rte_port_fd_writer_nodrop_free, 516 .f_tx = rte_port_fd_writer_nodrop_tx, 517 .f_tx_bulk = rte_port_fd_writer_nodrop_tx_bulk, 518 .f_flush = rte_port_fd_writer_nodrop_flush, 519 .f_stats = rte_port_fd_writer_nodrop_stats_read, 520 }; 521