1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (c) 2015-2018 Atomic Rules LLC 3 */ 4 5 #include <unistd.h> 6 7 #include "ark_ethdev_tx.h" 8 #include "ark_global.h" 9 #include "ark_mpu.h" 10 #include "ark_ddm.h" 11 #include "ark_logs.h" 12 13 #define ARK_TX_META_SIZE 32 14 #define ARK_TX_META_OFFSET (RTE_PKTMBUF_HEADROOM - ARK_TX_META_SIZE) 15 #define ARK_TX_MAX_NOCHAIN (RTE_MBUF_DEFAULT_DATAROOM) 16 17 #ifndef RTE_LIBRTE_ARK_MIN_TX_PKTLEN 18 #define ARK_MIN_TX_PKTLEN 0 19 #else 20 #define ARK_MIN_TX_PKTLEN RTE_LIBRTE_ARK_MIN_TX_PKTLEN 21 #endif 22 23 /* ************************************************************************* */ 24 struct ark_tx_queue { 25 struct ark_tx_meta *meta_q; 26 struct rte_mbuf **bufs; 27 28 /* handles for hw objects */ 29 struct ark_mpu_t *mpu; 30 struct ark_ddm_t *ddm; 31 32 /* Stats HW tracks bytes and packets, need to count send errors */ 33 uint64_t tx_errors; 34 35 uint32_t queue_size; 36 uint32_t queue_mask; 37 38 /* 3 indexes to the paired data rings. */ 39 uint32_t prod_index; /* where to put the next one */ 40 uint32_t free_index; /* mbuf has been freed */ 41 42 /* The queue Id is used to identify the HW Q */ 43 uint16_t phys_qid; 44 /* The queue Index within the dpdk device structures */ 45 uint16_t queue_index; 46 47 uint32_t pad[1]; 48 49 /* second cache line - fields only used in slow path */ 50 RTE_MARKER cacheline1 __rte_cache_min_aligned; 51 uint32_t cons_index; /* hw is done, can be freed */ 52 } __rte_cache_aligned; 53 54 /* Forward declarations */ 55 static uint32_t eth_ark_tx_jumbo(struct ark_tx_queue *queue, 56 struct rte_mbuf *mbuf); 57 static int eth_ark_tx_hw_queue_config(struct ark_tx_queue *queue); 58 static void free_completed_tx(struct ark_tx_queue *queue); 59 60 static inline void 61 ark_tx_hw_queue_stop(struct ark_tx_queue *queue) 62 { 63 ark_mpu_stop(queue->mpu); 64 } 65 66 /* ************************************************************************* */ 67 static inline void 68 eth_ark_tx_meta_from_mbuf(struct ark_tx_meta *meta, 69 const struct rte_mbuf *mbuf, 70 uint8_t flags) 71 { 72 meta->physaddr = rte_mbuf_data_iova(mbuf); 73 meta->user1 = (uint32_t)mbuf->udata64; 74 meta->data_len = rte_pktmbuf_data_len(mbuf); 75 meta->flags = flags; 76 } 77 78 /* ************************************************************************* */ 79 uint16_t 80 eth_ark_xmit_pkts_noop(void *vtxq __rte_unused, 81 struct rte_mbuf **tx_pkts __rte_unused, 82 uint16_t nb_pkts __rte_unused) 83 { 84 return 0; 85 } 86 87 /* ************************************************************************* */ 88 uint16_t 89 eth_ark_xmit_pkts(void *vtxq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) 90 { 91 struct ark_tx_queue *queue; 92 struct rte_mbuf *mbuf; 93 struct ark_tx_meta *meta; 94 95 uint32_t idx; 96 uint32_t prod_index_limit; 97 int stat; 98 uint16_t nb; 99 const uint32_t min_pkt_len = ARK_MIN_TX_PKTLEN; 100 101 queue = (struct ark_tx_queue *)vtxq; 102 103 /* free any packets after the HW is done with them */ 104 free_completed_tx(queue); 105 106 prod_index_limit = queue->queue_size + queue->free_index; 107 108 for (nb = 0; 109 (nb < nb_pkts) && (queue->prod_index != prod_index_limit); 110 ++nb) { 111 mbuf = tx_pkts[nb]; 112 113 if (min_pkt_len && 114 unlikely(rte_pktmbuf_pkt_len(mbuf) < min_pkt_len)) { 115 /* this packet even if it is small can be split, 116 * be sure to add to the end mbuf 117 */ 118 uint16_t to_add = min_pkt_len - 119 rte_pktmbuf_pkt_len(mbuf); 120 char *appended = 121 rte_pktmbuf_append(mbuf, to_add); 122 123 if (appended == 0) { 124 /* This packet is in error, 125 * we cannot send it so just 126 * count it and delete it. 127 */ 128 queue->tx_errors += 1; 129 rte_pktmbuf_free(mbuf); 130 continue; 131 } 132 memset(appended, 0, to_add); 133 } 134 135 if (unlikely(mbuf->nb_segs != 1)) { 136 stat = eth_ark_tx_jumbo(queue, mbuf); 137 if (unlikely(stat != 0)) 138 break; /* Queue is full */ 139 } else { 140 idx = queue->prod_index & queue->queue_mask; 141 queue->bufs[idx] = mbuf; 142 meta = &queue->meta_q[idx]; 143 eth_ark_tx_meta_from_mbuf(meta, 144 mbuf, 145 ARK_DDM_SOP | 146 ARK_DDM_EOP); 147 queue->prod_index++; 148 } 149 } 150 151 if (ARK_DEBUG_CORE && nb != nb_pkts) { 152 ARK_PMD_LOG(DEBUG, "TX: Failure to send:" 153 " req: %" PRIU32 154 " sent: %" PRIU32 155 " prod: %" PRIU32 156 " cons: %" PRIU32 157 " free: %" PRIU32 "\n", 158 nb_pkts, nb, 159 queue->prod_index, 160 queue->cons_index, 161 queue->free_index); 162 ark_mpu_dump(queue->mpu, 163 "TX Failure MPU: ", 164 queue->phys_qid); 165 } 166 167 /* let FPGA know producer index. */ 168 if (likely(nb != 0)) 169 ark_mpu_set_producer(queue->mpu, queue->prod_index); 170 171 return nb; 172 } 173 174 /* ************************************************************************* */ 175 static uint32_t 176 eth_ark_tx_jumbo(struct ark_tx_queue *queue, struct rte_mbuf *mbuf) 177 { 178 struct rte_mbuf *next; 179 struct ark_tx_meta *meta; 180 uint32_t free_queue_space; 181 uint32_t idx; 182 uint8_t flags = ARK_DDM_SOP; 183 184 free_queue_space = queue->queue_mask - 185 (queue->prod_index - queue->free_index); 186 if (unlikely(free_queue_space < mbuf->nb_segs)) 187 return -1; 188 189 while (mbuf != NULL) { 190 next = mbuf->next; 191 192 idx = queue->prod_index & queue->queue_mask; 193 queue->bufs[idx] = mbuf; 194 meta = &queue->meta_q[idx]; 195 196 flags |= (next == NULL) ? ARK_DDM_EOP : 0; 197 eth_ark_tx_meta_from_mbuf(meta, mbuf, flags); 198 queue->prod_index++; 199 200 flags &= ~ARK_DDM_SOP; /* drop SOP flags */ 201 mbuf = next; 202 } 203 204 return 0; 205 } 206 207 /* ************************************************************************* */ 208 int 209 eth_ark_tx_queue_setup(struct rte_eth_dev *dev, 210 uint16_t queue_idx, 211 uint16_t nb_desc, 212 unsigned int socket_id, 213 const struct rte_eth_txconf *tx_conf __rte_unused) 214 { 215 struct ark_adapter *ark = dev->data->dev_private; 216 struct ark_tx_queue *queue; 217 int status; 218 219 int qidx = queue_idx; 220 221 if (!rte_is_power_of_2(nb_desc)) { 222 ARK_PMD_LOG(ERR, 223 "DPDK Arkville configuration queue size" 224 " must be power of two %u (%s)\n", 225 nb_desc, __func__); 226 return -1; 227 } 228 229 /* Allocate queue struct */ 230 queue = rte_zmalloc_socket("Ark_txqueue", 231 sizeof(struct ark_tx_queue), 232 64, 233 socket_id); 234 if (queue == 0) { 235 ARK_PMD_LOG(ERR, "Failed to allocate tx " 236 "queue memory in %s\n", 237 __func__); 238 return -ENOMEM; 239 } 240 241 /* we use zmalloc no need to initialize fields */ 242 queue->queue_size = nb_desc; 243 queue->queue_mask = nb_desc - 1; 244 queue->phys_qid = qidx; 245 queue->queue_index = queue_idx; 246 dev->data->tx_queues[queue_idx] = queue; 247 248 queue->meta_q = 249 rte_zmalloc_socket("Ark_txqueue meta", 250 nb_desc * sizeof(struct ark_tx_meta), 251 64, 252 socket_id); 253 queue->bufs = 254 rte_zmalloc_socket("Ark_txqueue bufs", 255 nb_desc * sizeof(struct rte_mbuf *), 256 64, 257 socket_id); 258 259 if (queue->meta_q == 0 || queue->bufs == 0) { 260 ARK_PMD_LOG(ERR, "Failed to allocate " 261 "queue memory in %s\n", __func__); 262 rte_free(queue->meta_q); 263 rte_free(queue->bufs); 264 rte_free(queue); 265 return -ENOMEM; 266 } 267 268 queue->ddm = RTE_PTR_ADD(ark->ddm.v, qidx * ARK_DDM_QOFFSET); 269 queue->mpu = RTE_PTR_ADD(ark->mputx.v, qidx * ARK_MPU_QOFFSET); 270 271 status = eth_ark_tx_hw_queue_config(queue); 272 273 if (unlikely(status != 0)) { 274 rte_free(queue->meta_q); 275 rte_free(queue->bufs); 276 rte_free(queue); 277 return -1; /* ERROR CODE */ 278 } 279 280 return 0; 281 } 282 283 /* ************************************************************************* */ 284 static int 285 eth_ark_tx_hw_queue_config(struct ark_tx_queue *queue) 286 { 287 rte_iova_t queue_base, ring_base, cons_index_addr; 288 uint32_t write_interval_ns; 289 290 /* Verify HW -- MPU */ 291 if (ark_mpu_verify(queue->mpu, sizeof(struct ark_tx_meta))) 292 return -1; 293 294 queue_base = rte_malloc_virt2iova(queue); 295 ring_base = rte_malloc_virt2iova(queue->meta_q); 296 cons_index_addr = 297 queue_base + offsetof(struct ark_tx_queue, cons_index); 298 299 ark_mpu_stop(queue->mpu); 300 ark_mpu_reset(queue->mpu); 301 302 /* Stop and Reset and configure MPU */ 303 ark_mpu_configure(queue->mpu, ring_base, queue->queue_size, 1); 304 305 /* 306 * Adjust the write interval based on queue size -- 307 * increase pcie traffic when low mbuf count 308 * Queue sizes less than 128 are not allowed 309 */ 310 switch (queue->queue_size) { 311 case 128: 312 write_interval_ns = 500; 313 break; 314 case 256: 315 write_interval_ns = 500; 316 break; 317 case 512: 318 write_interval_ns = 1000; 319 break; 320 default: 321 write_interval_ns = 2000; 322 break; 323 } 324 325 /* Completion address in UDM */ 326 ark_ddm_setup(queue->ddm, cons_index_addr, write_interval_ns); 327 328 return 0; 329 } 330 331 /* ************************************************************************* */ 332 void 333 eth_ark_tx_queue_release(void *vtx_queue) 334 { 335 struct ark_tx_queue *queue; 336 337 queue = (struct ark_tx_queue *)vtx_queue; 338 339 ark_tx_hw_queue_stop(queue); 340 341 queue->cons_index = queue->prod_index; 342 free_completed_tx(queue); 343 344 rte_free(queue->meta_q); 345 rte_free(queue->bufs); 346 rte_free(queue); 347 } 348 349 /* ************************************************************************* */ 350 int 351 eth_ark_tx_queue_stop(struct rte_eth_dev *dev, uint16_t queue_id) 352 { 353 struct ark_tx_queue *queue; 354 int cnt = 0; 355 356 queue = dev->data->tx_queues[queue_id]; 357 358 /* Wait for DDM to send out all packets. */ 359 while (queue->cons_index != queue->prod_index) { 360 usleep(100); 361 if (cnt++ > 10000) 362 return -1; 363 } 364 365 ark_mpu_stop(queue->mpu); 366 free_completed_tx(queue); 367 368 dev->data->tx_queue_state[queue_id] = RTE_ETH_QUEUE_STATE_STOPPED; 369 370 return 0; 371 } 372 373 int 374 eth_ark_tx_queue_start(struct rte_eth_dev *dev, uint16_t queue_id) 375 { 376 struct ark_tx_queue *queue; 377 378 queue = dev->data->tx_queues[queue_id]; 379 if (dev->data->tx_queue_state[queue_id] == RTE_ETH_QUEUE_STATE_STARTED) 380 return 0; 381 382 ark_mpu_start(queue->mpu); 383 dev->data->tx_queue_state[queue_id] = RTE_ETH_QUEUE_STATE_STARTED; 384 385 return 0; 386 } 387 388 /* ************************************************************************* */ 389 static void 390 free_completed_tx(struct ark_tx_queue *queue) 391 { 392 struct rte_mbuf *mbuf; 393 struct ark_tx_meta *meta; 394 uint32_t top_index; 395 396 top_index = queue->cons_index; /* read once */ 397 while (queue->free_index != top_index) { 398 meta = &queue->meta_q[queue->free_index & queue->queue_mask]; 399 mbuf = queue->bufs[queue->free_index & queue->queue_mask]; 400 401 if (likely((meta->flags & ARK_DDM_SOP) != 0)) { 402 /* ref count of the mbuf is checked in this call. */ 403 rte_pktmbuf_free(mbuf); 404 } 405 queue->free_index++; 406 } 407 } 408 409 /* ************************************************************************* */ 410 void 411 eth_tx_queue_stats_get(void *vqueue, struct rte_eth_stats *stats) 412 { 413 struct ark_tx_queue *queue; 414 struct ark_ddm_t *ddm; 415 uint64_t bytes, pkts; 416 417 queue = vqueue; 418 ddm = queue->ddm; 419 420 bytes = ark_ddm_queue_byte_count(ddm); 421 pkts = ark_ddm_queue_pkt_count(ddm); 422 423 stats->q_opackets[queue->queue_index] = pkts; 424 stats->q_obytes[queue->queue_index] = bytes; 425 stats->opackets += pkts; 426 stats->obytes += bytes; 427 stats->oerrors += queue->tx_errors; 428 } 429 430 void 431 eth_tx_queue_stats_reset(void *vqueue) 432 { 433 struct ark_tx_queue *queue; 434 struct ark_ddm_t *ddm; 435 436 queue = vqueue; 437 ddm = queue->ddm; 438 439 ark_ddm_queue_reset_stats(ddm); 440 queue->tx_errors = 0; 441 } 442