1 /*- 2 * BSD LICENSE 3 * 4 * Copyright 2015 6WIND S.A. 5 * Copyright 2015 Mellanox. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of 6WIND S.A. nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include <stddef.h> 35 #include <assert.h> 36 #include <errno.h> 37 #include <string.h> 38 #include <stdint.h> 39 40 /* Verbs header. */ 41 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */ 42 #ifdef PEDANTIC 43 #pragma GCC diagnostic ignored "-pedantic" 44 #endif 45 #include <infiniband/verbs.h> 46 #ifdef PEDANTIC 47 #pragma GCC diagnostic error "-pedantic" 48 #endif 49 50 /* DPDK headers don't like -pedantic. */ 51 #ifdef PEDANTIC 52 #pragma GCC diagnostic ignored "-pedantic" 53 #endif 54 #include <rte_mbuf.h> 55 #include <rte_malloc.h> 56 #include <rte_ethdev.h> 57 #include <rte_common.h> 58 #ifdef PEDANTIC 59 #pragma GCC diagnostic error "-pedantic" 60 #endif 61 62 #include "mlx5_utils.h" 63 #include "mlx5_defs.h" 64 #include "mlx5.h" 65 #include "mlx5_rxtx.h" 66 #include "mlx5_autoconf.h" 67 #include "mlx5_defs.h" 68 69 /** 70 * Allocate TX queue elements. 71 * 72 * @param txq_ctrl 73 * Pointer to TX queue structure. 74 * @param elts_n 75 * Number of elements to allocate. 76 */ 77 static void 78 txq_alloc_elts(struct txq_ctrl *txq_ctrl, unsigned int elts_n) 79 { 80 unsigned int i; 81 82 for (i = 0; (i != elts_n); ++i) 83 (*txq_ctrl->txq.elts)[i] = NULL; 84 for (i = 0; (i != txq_ctrl->txq.wqe_n); ++i) { 85 volatile union mlx5_wqe *wqe = &(*txq_ctrl->txq.wqes)[i]; 86 87 memset((void *)(uintptr_t)wqe, 0x0, sizeof(*wqe)); 88 } 89 DEBUG("%p: allocated and configured %u WRs", (void *)txq_ctrl, elts_n); 90 txq_ctrl->txq.elts_head = 0; 91 txq_ctrl->txq.elts_tail = 0; 92 txq_ctrl->txq.elts_comp = 0; 93 } 94 95 /** 96 * Free TX queue elements. 97 * 98 * @param txq_ctrl 99 * Pointer to TX queue structure. 100 */ 101 static void 102 txq_free_elts(struct txq_ctrl *txq_ctrl) 103 { 104 unsigned int elts_n = txq_ctrl->txq.elts_n; 105 unsigned int elts_head = txq_ctrl->txq.elts_head; 106 unsigned int elts_tail = txq_ctrl->txq.elts_tail; 107 struct rte_mbuf *(*elts)[elts_n] = txq_ctrl->txq.elts; 108 109 DEBUG("%p: freeing WRs", (void *)txq_ctrl); 110 txq_ctrl->txq.elts_head = 0; 111 txq_ctrl->txq.elts_tail = 0; 112 txq_ctrl->txq.elts_comp = 0; 113 114 while (elts_tail != elts_head) { 115 struct rte_mbuf *elt = (*elts)[elts_tail]; 116 117 assert(elt != NULL); 118 rte_pktmbuf_free(elt); 119 #ifndef NDEBUG 120 /* Poisoning. */ 121 memset(&(*elts)[elts_tail], 122 0x77, 123 sizeof((*elts)[elts_tail])); 124 #endif 125 if (++elts_tail == elts_n) 126 elts_tail = 0; 127 } 128 } 129 130 /** 131 * Clean up a TX queue. 132 * 133 * Destroy objects, free allocated memory and reset the structure for reuse. 134 * 135 * @param txq_ctrl 136 * Pointer to TX queue structure. 137 */ 138 void 139 txq_cleanup(struct txq_ctrl *txq_ctrl) 140 { 141 struct ibv_exp_release_intf_params params; 142 size_t i; 143 144 DEBUG("cleaning up %p", (void *)txq_ctrl); 145 txq_free_elts(txq_ctrl); 146 if (txq_ctrl->if_qp != NULL) { 147 assert(txq_ctrl->priv != NULL); 148 assert(txq_ctrl->priv->ctx != NULL); 149 assert(txq_ctrl->qp != NULL); 150 params = (struct ibv_exp_release_intf_params){ 151 .comp_mask = 0, 152 }; 153 claim_zero(ibv_exp_release_intf(txq_ctrl->priv->ctx, 154 txq_ctrl->if_qp, 155 ¶ms)); 156 } 157 if (txq_ctrl->if_cq != NULL) { 158 assert(txq_ctrl->priv != NULL); 159 assert(txq_ctrl->priv->ctx != NULL); 160 assert(txq_ctrl->cq != NULL); 161 params = (struct ibv_exp_release_intf_params){ 162 .comp_mask = 0, 163 }; 164 claim_zero(ibv_exp_release_intf(txq_ctrl->priv->ctx, 165 txq_ctrl->if_cq, 166 ¶ms)); 167 } 168 if (txq_ctrl->qp != NULL) 169 claim_zero(ibv_destroy_qp(txq_ctrl->qp)); 170 if (txq_ctrl->cq != NULL) 171 claim_zero(ibv_destroy_cq(txq_ctrl->cq)); 172 if (txq_ctrl->rd != NULL) { 173 struct ibv_exp_destroy_res_domain_attr attr = { 174 .comp_mask = 0, 175 }; 176 177 assert(txq_ctrl->priv != NULL); 178 assert(txq_ctrl->priv->ctx != NULL); 179 claim_zero(ibv_exp_destroy_res_domain(txq_ctrl->priv->ctx, 180 txq_ctrl->rd, 181 &attr)); 182 } 183 for (i = 0; (i != RTE_DIM(txq_ctrl->txq.mp2mr)); ++i) { 184 if (txq_ctrl->txq.mp2mr[i].mp == NULL) 185 break; 186 assert(txq_ctrl->txq.mp2mr[i].mr != NULL); 187 claim_zero(ibv_dereg_mr(txq_ctrl->txq.mp2mr[i].mr)); 188 } 189 memset(txq_ctrl, 0, sizeof(*txq_ctrl)); 190 } 191 192 /** 193 * Initialize TX queue. 194 * 195 * @param tmpl 196 * Pointer to TX queue control template. 197 * @param txq_ctrl 198 * Pointer to TX queue control. 199 * 200 * @return 201 * 0 on success, errno value on failure. 202 */ 203 static inline int 204 txq_setup(struct txq_ctrl *tmpl, struct txq_ctrl *txq_ctrl) 205 { 206 struct mlx5_qp *qp = to_mqp(tmpl->qp); 207 struct ibv_cq *ibcq = tmpl->cq; 208 struct mlx5_cq *cq = to_mxxx(cq, cq); 209 210 if (cq->cqe_sz != RTE_CACHE_LINE_SIZE) { 211 ERROR("Wrong MLX5_CQE_SIZE environment variable value: " 212 "it should be set to %u", RTE_CACHE_LINE_SIZE); 213 return EINVAL; 214 } 215 tmpl->txq.cqe_n = ibcq->cqe + 1; 216 tmpl->txq.qp_num_8s = qp->ctrl_seg.qp_num << 8; 217 tmpl->txq.wqes = 218 (volatile union mlx5_wqe (*)[]) 219 (uintptr_t)qp->gen_data.sqstart; 220 tmpl->txq.wqe_n = qp->sq.wqe_cnt; 221 tmpl->txq.qp_db = &qp->gen_data.db[MLX5_SND_DBR]; 222 tmpl->txq.bf_reg = qp->gen_data.bf->reg; 223 tmpl->txq.bf_offset = qp->gen_data.bf->offset; 224 tmpl->txq.bf_buf_size = qp->gen_data.bf->buf_size; 225 tmpl->txq.cq_db = cq->dbrec; 226 tmpl->txq.cqes = 227 (volatile struct mlx5_cqe (*)[]) 228 (uintptr_t)cq->active_buf->buf; 229 tmpl->txq.elts = 230 (struct rte_mbuf *(*)[tmpl->txq.elts_n]) 231 ((uintptr_t)txq_ctrl + sizeof(*txq_ctrl)); 232 return 0; 233 } 234 235 /** 236 * Configure a TX queue. 237 * 238 * @param dev 239 * Pointer to Ethernet device structure. 240 * @param txq_ctrl 241 * Pointer to TX queue structure. 242 * @param desc 243 * Number of descriptors to configure in queue. 244 * @param socket 245 * NUMA socket on which memory must be allocated. 246 * @param[in] conf 247 * Thresholds parameters. 248 * 249 * @return 250 * 0 on success, errno value on failure. 251 */ 252 int 253 txq_ctrl_setup(struct rte_eth_dev *dev, struct txq_ctrl *txq_ctrl, 254 uint16_t desc, unsigned int socket, 255 const struct rte_eth_txconf *conf) 256 { 257 struct priv *priv = mlx5_get_priv(dev); 258 struct txq_ctrl tmpl = { 259 .priv = priv, 260 .socket = socket, 261 }; 262 union { 263 struct ibv_exp_query_intf_params params; 264 struct ibv_exp_qp_init_attr init; 265 struct ibv_exp_res_domain_init_attr rd; 266 struct ibv_exp_cq_init_attr cq; 267 struct ibv_exp_qp_attr mod; 268 struct ibv_exp_cq_attr cq_attr; 269 } attr; 270 enum ibv_exp_query_intf_status status; 271 int ret = 0; 272 273 if (mlx5_getenv_int("MLX5_ENABLE_CQE_COMPRESSION")) { 274 ret = ENOTSUP; 275 ERROR("MLX5_ENABLE_CQE_COMPRESSION must never be set"); 276 goto error; 277 } 278 (void)conf; /* Thresholds configuration (ignored). */ 279 assert(desc > MLX5_TX_COMP_THRESH); 280 tmpl.txq.elts_n = desc; 281 /* MRs will be registered in mp2mr[] later. */ 282 attr.rd = (struct ibv_exp_res_domain_init_attr){ 283 .comp_mask = (IBV_EXP_RES_DOMAIN_THREAD_MODEL | 284 IBV_EXP_RES_DOMAIN_MSG_MODEL), 285 .thread_model = IBV_EXP_THREAD_SINGLE, 286 .msg_model = IBV_EXP_MSG_HIGH_BW, 287 }; 288 tmpl.rd = ibv_exp_create_res_domain(priv->ctx, &attr.rd); 289 if (tmpl.rd == NULL) { 290 ret = ENOMEM; 291 ERROR("%p: RD creation failure: %s", 292 (void *)dev, strerror(ret)); 293 goto error; 294 } 295 attr.cq = (struct ibv_exp_cq_init_attr){ 296 .comp_mask = IBV_EXP_CQ_INIT_ATTR_RES_DOMAIN, 297 .res_domain = tmpl.rd, 298 }; 299 tmpl.cq = ibv_exp_create_cq(priv->ctx, 300 (((desc / MLX5_TX_COMP_THRESH) - 1) ? 301 ((desc / MLX5_TX_COMP_THRESH) - 1) : 1), 302 NULL, NULL, 0, &attr.cq); 303 if (tmpl.cq == NULL) { 304 ret = ENOMEM; 305 ERROR("%p: CQ creation failure: %s", 306 (void *)dev, strerror(ret)); 307 goto error; 308 } 309 DEBUG("priv->device_attr.max_qp_wr is %d", 310 priv->device_attr.max_qp_wr); 311 DEBUG("priv->device_attr.max_sge is %d", 312 priv->device_attr.max_sge); 313 attr.init = (struct ibv_exp_qp_init_attr){ 314 /* CQ to be associated with the send queue. */ 315 .send_cq = tmpl.cq, 316 /* CQ to be associated with the receive queue. */ 317 .recv_cq = tmpl.cq, 318 .cap = { 319 /* Max number of outstanding WRs. */ 320 .max_send_wr = ((priv->device_attr.max_qp_wr < desc) ? 321 priv->device_attr.max_qp_wr : 322 desc), 323 /* 324 * Max number of scatter/gather elements in a WR, 325 * must be 1 to prevent libmlx5 from trying to affect 326 * too much memory. TX gather is not impacted by the 327 * priv->device_attr.max_sge limit and will still work 328 * properly. 329 */ 330 .max_send_sge = 1, 331 }, 332 .qp_type = IBV_QPT_RAW_PACKET, 333 /* Do *NOT* enable this, completions events are managed per 334 * TX burst. */ 335 .sq_sig_all = 0, 336 .pd = priv->pd, 337 .res_domain = tmpl.rd, 338 .comp_mask = (IBV_EXP_QP_INIT_ATTR_PD | 339 IBV_EXP_QP_INIT_ATTR_RES_DOMAIN), 340 }; 341 if (priv->txq_inline && priv->txqs_n >= priv->txqs_inline) { 342 tmpl.txq.max_inline = priv->txq_inline; 343 attr.init.cap.max_inline_data = tmpl.txq.max_inline; 344 } 345 tmpl.qp = ibv_exp_create_qp(priv->ctx, &attr.init); 346 if (tmpl.qp == NULL) { 347 ret = (errno ? errno : EINVAL); 348 ERROR("%p: QP creation failure: %s", 349 (void *)dev, strerror(ret)); 350 goto error; 351 } 352 DEBUG("TX queue capabilities: max_send_wr=%u, max_send_sge=%u," 353 " max_inline_data=%u", 354 attr.init.cap.max_send_wr, 355 attr.init.cap.max_send_sge, 356 attr.init.cap.max_inline_data); 357 attr.mod = (struct ibv_exp_qp_attr){ 358 /* Move the QP to this state. */ 359 .qp_state = IBV_QPS_INIT, 360 /* Primary port number. */ 361 .port_num = priv->port 362 }; 363 ret = ibv_exp_modify_qp(tmpl.qp, &attr.mod, 364 (IBV_EXP_QP_STATE | IBV_EXP_QP_PORT)); 365 if (ret) { 366 ERROR("%p: QP state to IBV_QPS_INIT failed: %s", 367 (void *)dev, strerror(ret)); 368 goto error; 369 } 370 ret = txq_setup(&tmpl, txq_ctrl); 371 if (ret) { 372 ERROR("%p: cannot initialize TX queue structure: %s", 373 (void *)dev, strerror(ret)); 374 goto error; 375 } 376 txq_alloc_elts(&tmpl, desc); 377 attr.mod = (struct ibv_exp_qp_attr){ 378 .qp_state = IBV_QPS_RTR 379 }; 380 ret = ibv_exp_modify_qp(tmpl.qp, &attr.mod, IBV_EXP_QP_STATE); 381 if (ret) { 382 ERROR("%p: QP state to IBV_QPS_RTR failed: %s", 383 (void *)dev, strerror(ret)); 384 goto error; 385 } 386 attr.mod.qp_state = IBV_QPS_RTS; 387 ret = ibv_exp_modify_qp(tmpl.qp, &attr.mod, IBV_EXP_QP_STATE); 388 if (ret) { 389 ERROR("%p: QP state to IBV_QPS_RTS failed: %s", 390 (void *)dev, strerror(ret)); 391 goto error; 392 } 393 attr.params = (struct ibv_exp_query_intf_params){ 394 .intf_scope = IBV_EXP_INTF_GLOBAL, 395 .intf = IBV_EXP_INTF_CQ, 396 .obj = tmpl.cq, 397 }; 398 tmpl.if_cq = ibv_exp_query_intf(priv->ctx, &attr.params, &status); 399 if (tmpl.if_cq == NULL) { 400 ret = EINVAL; 401 ERROR("%p: CQ interface family query failed with status %d", 402 (void *)dev, status); 403 goto error; 404 } 405 attr.params = (struct ibv_exp_query_intf_params){ 406 .intf_scope = IBV_EXP_INTF_GLOBAL, 407 .intf = IBV_EXP_INTF_QP_BURST, 408 .intf_version = 1, 409 .obj = tmpl.qp, 410 /* Enable multi-packet send if supported. */ 411 .family_flags = 412 ((priv->mps && !priv->sriov) ? 413 IBV_EXP_QP_BURST_CREATE_ENABLE_MULTI_PACKET_SEND_WR : 414 0), 415 }; 416 tmpl.if_qp = ibv_exp_query_intf(priv->ctx, &attr.params, &status); 417 if (tmpl.if_qp == NULL) { 418 ret = EINVAL; 419 ERROR("%p: QP interface family query failed with status %d", 420 (void *)dev, status); 421 goto error; 422 } 423 /* Clean up txq in case we're reinitializing it. */ 424 DEBUG("%p: cleaning-up old txq just in case", (void *)txq_ctrl); 425 txq_cleanup(txq_ctrl); 426 *txq_ctrl = tmpl; 427 DEBUG("%p: txq updated with %p", (void *)txq_ctrl, (void *)&tmpl); 428 /* Pre-register known mempools. */ 429 rte_mempool_walk(txq_mp2mr_iter, txq_ctrl); 430 assert(ret == 0); 431 return 0; 432 error: 433 txq_cleanup(&tmpl); 434 assert(ret > 0); 435 return ret; 436 } 437 438 /** 439 * DPDK callback to configure a TX queue. 440 * 441 * @param dev 442 * Pointer to Ethernet device structure. 443 * @param idx 444 * TX queue index. 445 * @param desc 446 * Number of descriptors to configure in queue. 447 * @param socket 448 * NUMA socket on which memory must be allocated. 449 * @param[in] conf 450 * Thresholds parameters. 451 * 452 * @return 453 * 0 on success, negative errno value on failure. 454 */ 455 int 456 mlx5_tx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc, 457 unsigned int socket, const struct rte_eth_txconf *conf) 458 { 459 struct priv *priv = dev->data->dev_private; 460 struct txq *txq = (*priv->txqs)[idx]; 461 struct txq_ctrl *txq_ctrl = container_of(txq, struct txq_ctrl, txq); 462 int ret; 463 464 if (mlx5_is_secondary()) 465 return -E_RTE_SECONDARY; 466 467 priv_lock(priv); 468 if (desc <= MLX5_TX_COMP_THRESH) { 469 WARN("%p: number of descriptors requested for TX queue %u" 470 " must be higher than MLX5_TX_COMP_THRESH, using" 471 " %u instead of %u", 472 (void *)dev, idx, MLX5_TX_COMP_THRESH + 1, desc); 473 desc = MLX5_TX_COMP_THRESH + 1; 474 } 475 if (!rte_is_power_of_2(desc)) { 476 desc = 1 << log2above(desc); 477 WARN("%p: increased number of descriptors in TX queue %u" 478 " to the next power of two (%d)", 479 (void *)dev, idx, desc); 480 } 481 DEBUG("%p: configuring queue %u for %u descriptors", 482 (void *)dev, idx, desc); 483 if (idx >= priv->txqs_n) { 484 ERROR("%p: queue index out of range (%u >= %u)", 485 (void *)dev, idx, priv->txqs_n); 486 priv_unlock(priv); 487 return -EOVERFLOW; 488 } 489 if (txq != NULL) { 490 DEBUG("%p: reusing already allocated queue index %u (%p)", 491 (void *)dev, idx, (void *)txq); 492 if (priv->started) { 493 priv_unlock(priv); 494 return -EEXIST; 495 } 496 (*priv->txqs)[idx] = NULL; 497 txq_cleanup(txq_ctrl); 498 } else { 499 txq_ctrl = 500 rte_calloc_socket("TXQ", 1, 501 sizeof(*txq_ctrl) + 502 desc * sizeof(struct rte_mbuf *), 503 0, socket); 504 if (txq_ctrl == NULL) { 505 ERROR("%p: unable to allocate queue index %u", 506 (void *)dev, idx); 507 priv_unlock(priv); 508 return -ENOMEM; 509 } 510 } 511 ret = txq_ctrl_setup(dev, txq_ctrl, desc, socket, conf); 512 if (ret) 513 rte_free(txq_ctrl); 514 else { 515 txq_ctrl->txq.stats.idx = idx; 516 DEBUG("%p: adding TX queue %p to list", 517 (void *)dev, (void *)txq_ctrl); 518 (*priv->txqs)[idx] = &txq_ctrl->txq; 519 /* Update send callback. */ 520 priv_select_tx_function(priv); 521 } 522 priv_unlock(priv); 523 return -ret; 524 } 525 526 /** 527 * DPDK callback to release a TX queue. 528 * 529 * @param dpdk_txq 530 * Generic TX queue pointer. 531 */ 532 void 533 mlx5_tx_queue_release(void *dpdk_txq) 534 { 535 struct txq *txq = (struct txq *)dpdk_txq; 536 struct txq_ctrl *txq_ctrl; 537 struct priv *priv; 538 unsigned int i; 539 540 if (mlx5_is_secondary()) 541 return; 542 543 if (txq == NULL) 544 return; 545 txq_ctrl = container_of(txq, struct txq_ctrl, txq); 546 priv = txq_ctrl->priv; 547 priv_lock(priv); 548 for (i = 0; (i != priv->txqs_n); ++i) 549 if ((*priv->txqs)[i] == txq) { 550 DEBUG("%p: removing TX queue %p from list", 551 (void *)priv->dev, (void *)txq_ctrl); 552 (*priv->txqs)[i] = NULL; 553 break; 554 } 555 txq_cleanup(txq_ctrl); 556 rte_free(txq_ctrl); 557 priv_unlock(priv); 558 } 559 560 /** 561 * DPDK callback for TX in secondary processes. 562 * 563 * This function configures all queues from primary process information 564 * if necessary before reverting to the normal TX burst callback. 565 * 566 * @param dpdk_txq 567 * Generic pointer to TX queue structure. 568 * @param[in] pkts 569 * Packets to transmit. 570 * @param pkts_n 571 * Number of packets in array. 572 * 573 * @return 574 * Number of packets successfully transmitted (<= pkts_n). 575 */ 576 uint16_t 577 mlx5_tx_burst_secondary_setup(void *dpdk_txq, struct rte_mbuf **pkts, 578 uint16_t pkts_n) 579 { 580 struct txq *txq = dpdk_txq; 581 struct txq_ctrl *txq_ctrl = container_of(txq, struct txq_ctrl, txq); 582 struct priv *priv = mlx5_secondary_data_setup(txq_ctrl->priv); 583 struct priv *primary_priv; 584 unsigned int index; 585 586 if (priv == NULL) 587 return 0; 588 primary_priv = 589 mlx5_secondary_data[priv->dev->data->port_id].primary_priv; 590 /* Look for queue index in both private structures. */ 591 for (index = 0; index != priv->txqs_n; ++index) 592 if (((*primary_priv->txqs)[index] == txq) || 593 ((*priv->txqs)[index] == txq)) 594 break; 595 if (index == priv->txqs_n) 596 return 0; 597 txq = (*priv->txqs)[index]; 598 return priv->dev->tx_pkt_burst(txq, pkts, pkts_n); 599 } 600