1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (c) 2015-2021 Atomic Rules LLC 3 */ 4 5 #include <unistd.h> 6 7 #include "ark_ethdev_tx.h" 8 #include "ark_global.h" 9 #include "ark_mpu.h" 10 #include "ark_ddm.h" 11 #include "ark_logs.h" 12 13 #define ARK_TX_META_SIZE 32 14 #define ARK_TX_META_OFFSET (RTE_PKTMBUF_HEADROOM - ARK_TX_META_SIZE) 15 #define ARK_TX_MAX_NOCHAIN (RTE_MBUF_DEFAULT_DATAROOM) 16 17 #ifndef RTE_LIBRTE_ARK_MIN_TX_PKTLEN 18 #define ARK_MIN_TX_PKTLEN 0 19 #else 20 #define ARK_MIN_TX_PKTLEN RTE_LIBRTE_ARK_MIN_TX_PKTLEN 21 #endif 22 23 /* ************************************************************************* */ 24 struct ark_tx_queue { 25 union ark_tx_meta *meta_q; 26 struct rte_mbuf **bufs; 27 28 /* handles for hw objects */ 29 struct ark_mpu_t *mpu; 30 struct ark_ddm_t *ddm; 31 32 /* Stats HW tracks bytes and packets, need to count send errors */ 33 uint64_t tx_errors; 34 35 tx_user_meta_hook_fn tx_user_meta_hook; 36 void *ext_user_data; 37 38 uint32_t queue_size; 39 uint32_t queue_mask; 40 41 /* 3 indexes to the paired data rings. */ 42 int32_t prod_index; /* where to put the next one */ 43 int32_t free_index; /* mbuf has been freed */ 44 45 /* The queue Id is used to identify the HW Q */ 46 uint16_t phys_qid; 47 /* The queue Index within the dpdk device structures */ 48 uint16_t queue_index; 49 50 /* next cache line - fields written by device */ 51 RTE_MARKER cacheline1 __rte_cache_min_aligned; 52 volatile int32_t cons_index; /* hw is done, can be freed */ 53 } __rte_cache_aligned; 54 55 /* Forward declarations */ 56 static int eth_ark_tx_jumbo(struct ark_tx_queue *queue, 57 struct rte_mbuf *mbuf, 58 uint32_t *user_meta, uint8_t meta_cnt); 59 static int eth_ark_tx_hw_queue_config(struct ark_tx_queue *queue); 60 static void free_completed_tx(struct ark_tx_queue *queue); 61 62 static inline void 63 ark_tx_hw_queue_stop(struct ark_tx_queue *queue) 64 { 65 ark_mpu_stop(queue->mpu); 66 } 67 68 /* ************************************************************************* */ 69 static inline void 70 eth_ark_tx_desc_fill(struct ark_tx_queue *queue, 71 struct rte_mbuf *mbuf, 72 uint8_t flags, 73 uint32_t *user_meta, 74 uint8_t meta_cnt /* 0 to 5 */ 75 ) 76 { 77 uint32_t tx_idx; 78 union ark_tx_meta *meta; 79 uint8_t m; 80 81 /* Header */ 82 tx_idx = queue->prod_index & queue->queue_mask; 83 meta = &queue->meta_q[tx_idx]; 84 meta->data_len = rte_pktmbuf_data_len(mbuf); 85 meta->flags = flags; 86 meta->meta_cnt = meta_cnt / 2; 87 meta->user1 = meta_cnt ? (*user_meta++) : 0; 88 queue->prod_index++; 89 90 queue->bufs[tx_idx] = mbuf; 91 92 /* 1 or 2 user meta data entries, user words 1,2 and 3,4 */ 93 for (m = 1; m < meta_cnt; m += 2) { 94 tx_idx = queue->prod_index & queue->queue_mask; 95 meta = &queue->meta_q[tx_idx]; 96 meta->usermeta0 = *user_meta++; 97 meta->usermeta1 = *user_meta++; 98 queue->prod_index++; 99 } 100 101 tx_idx = queue->prod_index & queue->queue_mask; 102 meta = &queue->meta_q[tx_idx]; 103 meta->physaddr = rte_mbuf_data_iova(mbuf); 104 queue->prod_index++; 105 } 106 107 108 /* ************************************************************************* */ 109 uint16_t 110 eth_ark_xmit_pkts(void *vtxq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) 111 { 112 struct ark_tx_queue *queue; 113 struct rte_mbuf *mbuf; 114 uint32_t user_meta[5]; 115 116 int stat; 117 int32_t prod_index_limit; 118 uint16_t nb; 119 uint8_t user_len = 0; 120 const uint32_t min_pkt_len = ARK_MIN_TX_PKTLEN; 121 tx_user_meta_hook_fn tx_user_meta_hook; 122 123 queue = (struct ark_tx_queue *)vtxq; 124 tx_user_meta_hook = queue->tx_user_meta_hook; 125 126 /* free any packets after the HW is done with them */ 127 free_completed_tx(queue); 128 129 /* leave 4 elements mpu data */ 130 prod_index_limit = queue->queue_size + queue->free_index - 4; 131 132 for (nb = 0; 133 (nb < nb_pkts) && (prod_index_limit - queue->prod_index) > 0; 134 ++nb) { 135 mbuf = tx_pkts[nb]; 136 137 if (min_pkt_len && 138 unlikely(rte_pktmbuf_pkt_len(mbuf) < min_pkt_len)) { 139 /* this packet even if it is small can be split, 140 * be sure to add to the end mbuf 141 */ 142 uint16_t to_add = min_pkt_len - 143 rte_pktmbuf_pkt_len(mbuf); 144 char *appended = 145 rte_pktmbuf_append(mbuf, to_add); 146 147 if (appended == 0) { 148 /* This packet is in error, 149 * we cannot send it so just 150 * count it and delete it. 151 */ 152 queue->tx_errors += 1; 153 rte_pktmbuf_free(mbuf); 154 continue; 155 } 156 memset(appended, 0, to_add); 157 } 158 159 if (tx_user_meta_hook) 160 tx_user_meta_hook(mbuf, user_meta, &user_len, 161 queue->ext_user_data); 162 if (unlikely(mbuf->nb_segs != 1)) { 163 stat = eth_ark_tx_jumbo(queue, mbuf, 164 user_meta, user_len); 165 if (unlikely(stat != 0)) 166 break; /* Queue is full */ 167 } else { 168 eth_ark_tx_desc_fill(queue, mbuf, 169 ARK_DDM_SOP | ARK_DDM_EOP, 170 user_meta, user_len); 171 } 172 } 173 174 if (ARK_DEBUG_CORE && nb != nb_pkts) { 175 ARK_PMD_LOG(DEBUG, "TX: Failure to send:" 176 " req: %" PRIU32 177 " sent: %" PRIU32 178 " prod: %" PRIU32 179 " cons: %" PRIU32 180 " free: %" PRIU32 "\n", 181 nb_pkts, nb, 182 queue->prod_index, 183 queue->cons_index, 184 queue->free_index); 185 ark_mpu_dump(queue->mpu, 186 "TX Failure MPU: ", 187 queue->phys_qid); 188 } 189 190 /* let FPGA know producer index. */ 191 if (likely(nb != 0)) 192 ark_mpu_set_producer(queue->mpu, queue->prod_index); 193 194 return nb; 195 } 196 197 /* ************************************************************************* */ 198 static int 199 eth_ark_tx_jumbo(struct ark_tx_queue *queue, struct rte_mbuf *mbuf, 200 uint32_t *user_meta, uint8_t meta_cnt) 201 { 202 struct rte_mbuf *next; 203 int32_t free_queue_space; 204 uint8_t flags = ARK_DDM_SOP; 205 206 free_queue_space = queue->queue_mask - 207 (queue->prod_index - queue->free_index); 208 /* We need up to 4 mbufs for first header and 2 for subsequent ones */ 209 if (unlikely(free_queue_space < (2 + (2 * mbuf->nb_segs)))) 210 return -1; 211 212 while (mbuf != NULL) { 213 next = mbuf->next; 214 flags |= (next == NULL) ? ARK_DDM_EOP : 0; 215 216 eth_ark_tx_desc_fill(queue, mbuf, flags, user_meta, meta_cnt); 217 218 flags &= ~ARK_DDM_SOP; /* drop SOP flags */ 219 meta_cnt = 0; /* Meta only on SOP */ 220 mbuf = next; 221 } 222 223 return 0; 224 } 225 226 /* ************************************************************************* */ 227 int 228 eth_ark_tx_queue_setup(struct rte_eth_dev *dev, 229 uint16_t queue_idx, 230 uint16_t nb_desc, 231 unsigned int socket_id, 232 const struct rte_eth_txconf *tx_conf __rte_unused) 233 { 234 struct ark_adapter *ark = dev->data->dev_private; 235 struct ark_tx_queue *queue; 236 int status; 237 238 int qidx = queue_idx; 239 240 if (!rte_is_power_of_2(nb_desc)) { 241 ARK_PMD_LOG(ERR, 242 "DPDK Arkville configuration queue size" 243 " must be power of two %u (%s)\n", 244 nb_desc, __func__); 245 return -1; 246 } 247 248 /* Each packet requires at least 2 mpu elements - double desc count */ 249 nb_desc = 2 * nb_desc; 250 251 /* Allocate queue struct */ 252 queue = rte_zmalloc_socket("Ark_txqueue", 253 sizeof(struct ark_tx_queue), 254 64, 255 socket_id); 256 if (queue == 0) { 257 ARK_PMD_LOG(ERR, "Failed to allocate tx " 258 "queue memory in %s\n", 259 __func__); 260 return -ENOMEM; 261 } 262 263 /* we use zmalloc no need to initialize fields */ 264 queue->queue_size = nb_desc; 265 queue->queue_mask = nb_desc - 1; 266 queue->phys_qid = qidx; 267 queue->queue_index = queue_idx; 268 dev->data->tx_queues[queue_idx] = queue; 269 queue->tx_user_meta_hook = ark->user_ext.tx_user_meta_hook; 270 queue->ext_user_data = ark->user_data[dev->data->port_id]; 271 272 queue->meta_q = 273 rte_zmalloc_socket("Ark_txqueue meta", 274 nb_desc * sizeof(union ark_tx_meta), 275 64, 276 socket_id); 277 queue->bufs = 278 rte_zmalloc_socket("Ark_txqueue bufs", 279 nb_desc * sizeof(struct rte_mbuf *), 280 64, 281 socket_id); 282 283 if (queue->meta_q == 0 || queue->bufs == 0) { 284 ARK_PMD_LOG(ERR, "Failed to allocate " 285 "queue memory in %s\n", __func__); 286 rte_free(queue->meta_q); 287 rte_free(queue->bufs); 288 rte_free(queue); 289 return -ENOMEM; 290 } 291 292 queue->ddm = RTE_PTR_ADD(ark->ddm.v, qidx * ARK_DDM_QOFFSET); 293 queue->mpu = RTE_PTR_ADD(ark->mputx.v, qidx * ARK_MPU_QOFFSET); 294 295 status = eth_ark_tx_hw_queue_config(queue); 296 297 if (unlikely(status != 0)) { 298 rte_free(queue->meta_q); 299 rte_free(queue->bufs); 300 rte_free(queue); 301 return -1; /* ERROR CODE */ 302 } 303 304 return 0; 305 } 306 307 /* ************************************************************************* */ 308 static int 309 eth_ark_tx_hw_queue_config(struct ark_tx_queue *queue) 310 { 311 rte_iova_t queue_base, ring_base, cons_index_addr; 312 uint32_t write_interval_ns; 313 314 /* Verify HW -- MPU */ 315 if (ark_mpu_verify(queue->mpu, sizeof(union ark_tx_meta))) 316 return -1; 317 318 queue_base = rte_malloc_virt2iova(queue); 319 ring_base = rte_malloc_virt2iova(queue->meta_q); 320 cons_index_addr = 321 queue_base + offsetof(struct ark_tx_queue, cons_index); 322 323 ark_mpu_stop(queue->mpu); 324 ark_mpu_reset(queue->mpu); 325 326 /* Stop and Reset and configure MPU */ 327 ark_mpu_configure(queue->mpu, ring_base, queue->queue_size, 1); 328 329 /* 330 * Adjust the write interval based on queue size -- 331 * increase pcie traffic when low mbuf count 332 * Queue sizes less than 128 are not allowed 333 */ 334 switch (queue->queue_size) { 335 case 128: 336 write_interval_ns = 500; 337 break; 338 case 256: 339 write_interval_ns = 500; 340 break; 341 case 512: 342 write_interval_ns = 1000; 343 break; 344 default: 345 write_interval_ns = 2000; 346 break; 347 } 348 349 /* Completion address in UDM */ 350 ark_ddm_setup(queue->ddm, cons_index_addr, write_interval_ns); 351 352 return 0; 353 } 354 355 /* ************************************************************************* */ 356 void 357 eth_ark_tx_queue_release(void *vtx_queue) 358 { 359 struct ark_tx_queue *queue; 360 361 queue = (struct ark_tx_queue *)vtx_queue; 362 363 ark_tx_hw_queue_stop(queue); 364 365 queue->cons_index = queue->prod_index; 366 free_completed_tx(queue); 367 368 rte_free(queue->meta_q); 369 rte_free(queue->bufs); 370 rte_free(queue); 371 } 372 373 /* ************************************************************************* */ 374 int 375 eth_ark_tx_queue_stop(struct rte_eth_dev *dev, uint16_t queue_id) 376 { 377 struct ark_tx_queue *queue; 378 int cnt = 0; 379 380 queue = dev->data->tx_queues[queue_id]; 381 382 /* Wait for DDM to send out all packets. */ 383 while (queue->cons_index != queue->prod_index) { 384 usleep(100); 385 if (cnt++ > 10000) 386 return -1; 387 } 388 389 ark_mpu_stop(queue->mpu); 390 free_completed_tx(queue); 391 392 dev->data->tx_queue_state[queue_id] = RTE_ETH_QUEUE_STATE_STOPPED; 393 394 return 0; 395 } 396 397 int 398 eth_ark_tx_queue_start(struct rte_eth_dev *dev, uint16_t queue_id) 399 { 400 struct ark_tx_queue *queue; 401 402 queue = dev->data->tx_queues[queue_id]; 403 if (dev->data->tx_queue_state[queue_id] == RTE_ETH_QUEUE_STATE_STARTED) 404 return 0; 405 406 ark_mpu_start(queue->mpu); 407 dev->data->tx_queue_state[queue_id] = RTE_ETH_QUEUE_STATE_STARTED; 408 409 return 0; 410 } 411 412 /* ************************************************************************* */ 413 static void 414 free_completed_tx(struct ark_tx_queue *queue) 415 { 416 struct rte_mbuf *mbuf; 417 union ark_tx_meta *meta; 418 int32_t top_index; 419 420 top_index = queue->cons_index; /* read once */ 421 while ((top_index - queue->free_index) > 0) { 422 meta = &queue->meta_q[queue->free_index & queue->queue_mask]; 423 if (likely((meta->flags & ARK_DDM_SOP) != 0)) { 424 mbuf = queue->bufs[queue->free_index & 425 queue->queue_mask]; 426 /* ref count of the mbuf is checked in this call. */ 427 rte_pktmbuf_free(mbuf); 428 } 429 queue->free_index += (meta->meta_cnt + 2); 430 } 431 } 432 433 /* ************************************************************************* */ 434 void 435 eth_tx_queue_stats_get(void *vqueue, struct rte_eth_stats *stats) 436 { 437 struct ark_tx_queue *queue; 438 struct ark_ddm_t *ddm; 439 uint64_t bytes, pkts; 440 441 queue = vqueue; 442 ddm = queue->ddm; 443 444 bytes = ark_ddm_queue_byte_count(ddm); 445 pkts = ark_ddm_queue_pkt_count(ddm); 446 447 stats->q_opackets[queue->queue_index] = pkts; 448 stats->q_obytes[queue->queue_index] = bytes; 449 stats->opackets += pkts; 450 stats->obytes += bytes; 451 stats->oerrors += queue->tx_errors; 452 } 453 454 void 455 eth_tx_queue_stats_reset(void *vqueue) 456 { 457 struct ark_tx_queue *queue; 458 struct ark_ddm_t *ddm; 459 460 queue = vqueue; 461 ddm = queue->ddm; 462 463 ark_ddm_queue_reset_stats(ddm); 464 queue->tx_errors = 0; 465 } 466