1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (c) 2015-2018 Atomic Rules LLC 3 */ 4 5 #include <unistd.h> 6 7 #include "rte_pmd_ark.h" 8 #include "ark_ethdev_tx.h" 9 #include "ark_global.h" 10 #include "ark_mpu.h" 11 #include "ark_ddm.h" 12 #include "ark_logs.h" 13 14 #define ARK_TX_META_SIZE 32 15 #define ARK_TX_META_OFFSET (RTE_PKTMBUF_HEADROOM - ARK_TX_META_SIZE) 16 #define ARK_TX_MAX_NOCHAIN (RTE_MBUF_DEFAULT_DATAROOM) 17 18 #ifndef RTE_LIBRTE_ARK_MIN_TX_PKTLEN 19 #define ARK_MIN_TX_PKTLEN 0 20 #else 21 #define ARK_MIN_TX_PKTLEN RTE_LIBRTE_ARK_MIN_TX_PKTLEN 22 #endif 23 24 /* ************************************************************************* */ 25 struct ark_tx_queue { 26 struct ark_tx_meta *meta_q; 27 struct rte_mbuf **bufs; 28 29 /* handles for hw objects */ 30 struct ark_mpu_t *mpu; 31 struct ark_ddm_t *ddm; 32 33 /* Stats HW tracks bytes and packets, need to count send errors */ 34 uint64_t tx_errors; 35 36 uint32_t queue_size; 37 uint32_t queue_mask; 38 39 /* 3 indexes to the paired data rings. */ 40 uint32_t prod_index; /* where to put the next one */ 41 uint32_t free_index; /* mbuf has been freed */ 42 43 /* The queue Id is used to identify the HW Q */ 44 uint16_t phys_qid; 45 /* The queue Index within the dpdk device structures */ 46 uint16_t queue_index; 47 48 uint32_t pad[1]; 49 50 /* second cache line - fields only used in slow path */ 51 RTE_MARKER cacheline1 __rte_cache_min_aligned; 52 uint32_t cons_index; /* hw is done, can be freed */ 53 } __rte_cache_aligned; 54 55 /* Forward declarations */ 56 static uint32_t eth_ark_tx_jumbo(struct ark_tx_queue *queue, 57 struct rte_mbuf *mbuf); 58 static int eth_ark_tx_hw_queue_config(struct ark_tx_queue *queue); 59 static void free_completed_tx(struct ark_tx_queue *queue); 60 61 static inline void 62 ark_tx_hw_queue_stop(struct ark_tx_queue *queue) 63 { 64 ark_mpu_stop(queue->mpu); 65 } 66 67 /* ************************************************************************* */ 68 static inline void 69 eth_ark_tx_meta_from_mbuf(struct ark_tx_meta *meta, 70 const struct rte_mbuf *mbuf, 71 uint8_t flags) 72 { 73 meta->physaddr = rte_mbuf_data_iova(mbuf); 74 meta->user1 = rte_pmd_ark_mbuf_tx_userdata_get(mbuf); 75 meta->data_len = rte_pktmbuf_data_len(mbuf); 76 meta->flags = flags; 77 } 78 79 /* ************************************************************************* */ 80 uint16_t 81 eth_ark_xmit_pkts_noop(void *vtxq __rte_unused, 82 struct rte_mbuf **tx_pkts __rte_unused, 83 uint16_t nb_pkts __rte_unused) 84 { 85 return 0; 86 } 87 88 /* ************************************************************************* */ 89 uint16_t 90 eth_ark_xmit_pkts(void *vtxq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) 91 { 92 struct ark_tx_queue *queue; 93 struct rte_mbuf *mbuf; 94 struct ark_tx_meta *meta; 95 96 uint32_t idx; 97 uint32_t prod_index_limit; 98 int stat; 99 uint16_t nb; 100 const uint32_t min_pkt_len = ARK_MIN_TX_PKTLEN; 101 102 queue = (struct ark_tx_queue *)vtxq; 103 104 /* free any packets after the HW is done with them */ 105 free_completed_tx(queue); 106 107 prod_index_limit = queue->queue_size + queue->free_index; 108 109 for (nb = 0; 110 (nb < nb_pkts) && (queue->prod_index != prod_index_limit); 111 ++nb) { 112 mbuf = tx_pkts[nb]; 113 114 if (min_pkt_len && 115 unlikely(rte_pktmbuf_pkt_len(mbuf) < min_pkt_len)) { 116 /* this packet even if it is small can be split, 117 * be sure to add to the end mbuf 118 */ 119 uint16_t to_add = min_pkt_len - 120 rte_pktmbuf_pkt_len(mbuf); 121 char *appended = 122 rte_pktmbuf_append(mbuf, to_add); 123 124 if (appended == 0) { 125 /* This packet is in error, 126 * we cannot send it so just 127 * count it and delete it. 128 */ 129 queue->tx_errors += 1; 130 rte_pktmbuf_free(mbuf); 131 continue; 132 } 133 memset(appended, 0, to_add); 134 } 135 136 if (unlikely(mbuf->nb_segs != 1)) { 137 stat = eth_ark_tx_jumbo(queue, mbuf); 138 if (unlikely(stat != 0)) 139 break; /* Queue is full */ 140 } else { 141 idx = queue->prod_index & queue->queue_mask; 142 queue->bufs[idx] = mbuf; 143 meta = &queue->meta_q[idx]; 144 eth_ark_tx_meta_from_mbuf(meta, 145 mbuf, 146 ARK_DDM_SOP | 147 ARK_DDM_EOP); 148 queue->prod_index++; 149 } 150 } 151 152 if (ARK_DEBUG_CORE && nb != nb_pkts) { 153 ARK_PMD_LOG(DEBUG, "TX: Failure to send:" 154 " req: %" PRIU32 155 " sent: %" PRIU32 156 " prod: %" PRIU32 157 " cons: %" PRIU32 158 " free: %" PRIU32 "\n", 159 nb_pkts, nb, 160 queue->prod_index, 161 queue->cons_index, 162 queue->free_index); 163 ark_mpu_dump(queue->mpu, 164 "TX Failure MPU: ", 165 queue->phys_qid); 166 } 167 168 /* let FPGA know producer index. */ 169 if (likely(nb != 0)) 170 ark_mpu_set_producer(queue->mpu, queue->prod_index); 171 172 return nb; 173 } 174 175 /* ************************************************************************* */ 176 static uint32_t 177 eth_ark_tx_jumbo(struct ark_tx_queue *queue, struct rte_mbuf *mbuf) 178 { 179 struct rte_mbuf *next; 180 struct ark_tx_meta *meta; 181 uint32_t free_queue_space; 182 uint32_t idx; 183 uint8_t flags = ARK_DDM_SOP; 184 185 free_queue_space = queue->queue_mask - 186 (queue->prod_index - queue->free_index); 187 if (unlikely(free_queue_space < mbuf->nb_segs)) 188 return -1; 189 190 while (mbuf != NULL) { 191 next = mbuf->next; 192 193 idx = queue->prod_index & queue->queue_mask; 194 queue->bufs[idx] = mbuf; 195 meta = &queue->meta_q[idx]; 196 197 flags |= (next == NULL) ? ARK_DDM_EOP : 0; 198 eth_ark_tx_meta_from_mbuf(meta, mbuf, flags); 199 queue->prod_index++; 200 201 flags &= ~ARK_DDM_SOP; /* drop SOP flags */ 202 mbuf = next; 203 } 204 205 return 0; 206 } 207 208 /* ************************************************************************* */ 209 int 210 eth_ark_tx_queue_setup(struct rte_eth_dev *dev, 211 uint16_t queue_idx, 212 uint16_t nb_desc, 213 unsigned int socket_id, 214 const struct rte_eth_txconf *tx_conf __rte_unused) 215 { 216 struct ark_adapter *ark = dev->data->dev_private; 217 struct ark_tx_queue *queue; 218 int status; 219 220 int qidx = queue_idx; 221 222 if (!rte_is_power_of_2(nb_desc)) { 223 ARK_PMD_LOG(ERR, 224 "DPDK Arkville configuration queue size" 225 " must be power of two %u (%s)\n", 226 nb_desc, __func__); 227 return -1; 228 } 229 230 /* Allocate queue struct */ 231 queue = rte_zmalloc_socket("Ark_txqueue", 232 sizeof(struct ark_tx_queue), 233 64, 234 socket_id); 235 if (queue == 0) { 236 ARK_PMD_LOG(ERR, "Failed to allocate tx " 237 "queue memory in %s\n", 238 __func__); 239 return -ENOMEM; 240 } 241 242 /* we use zmalloc no need to initialize fields */ 243 queue->queue_size = nb_desc; 244 queue->queue_mask = nb_desc - 1; 245 queue->phys_qid = qidx; 246 queue->queue_index = queue_idx; 247 dev->data->tx_queues[queue_idx] = queue; 248 249 queue->meta_q = 250 rte_zmalloc_socket("Ark_txqueue meta", 251 nb_desc * sizeof(struct ark_tx_meta), 252 64, 253 socket_id); 254 queue->bufs = 255 rte_zmalloc_socket("Ark_txqueue bufs", 256 nb_desc * sizeof(struct rte_mbuf *), 257 64, 258 socket_id); 259 260 if (queue->meta_q == 0 || queue->bufs == 0) { 261 ARK_PMD_LOG(ERR, "Failed to allocate " 262 "queue memory in %s\n", __func__); 263 rte_free(queue->meta_q); 264 rte_free(queue->bufs); 265 rte_free(queue); 266 return -ENOMEM; 267 } 268 269 queue->ddm = RTE_PTR_ADD(ark->ddm.v, qidx * ARK_DDM_QOFFSET); 270 queue->mpu = RTE_PTR_ADD(ark->mputx.v, qidx * ARK_MPU_QOFFSET); 271 272 status = eth_ark_tx_hw_queue_config(queue); 273 274 if (unlikely(status != 0)) { 275 rte_free(queue->meta_q); 276 rte_free(queue->bufs); 277 rte_free(queue); 278 return -1; /* ERROR CODE */ 279 } 280 281 return 0; 282 } 283 284 /* ************************************************************************* */ 285 static int 286 eth_ark_tx_hw_queue_config(struct ark_tx_queue *queue) 287 { 288 rte_iova_t queue_base, ring_base, cons_index_addr; 289 uint32_t write_interval_ns; 290 291 /* Verify HW -- MPU */ 292 if (ark_mpu_verify(queue->mpu, sizeof(struct ark_tx_meta))) 293 return -1; 294 295 queue_base = rte_malloc_virt2iova(queue); 296 ring_base = rte_malloc_virt2iova(queue->meta_q); 297 cons_index_addr = 298 queue_base + offsetof(struct ark_tx_queue, cons_index); 299 300 ark_mpu_stop(queue->mpu); 301 ark_mpu_reset(queue->mpu); 302 303 /* Stop and Reset and configure MPU */ 304 ark_mpu_configure(queue->mpu, ring_base, queue->queue_size, 1); 305 306 /* 307 * Adjust the write interval based on queue size -- 308 * increase pcie traffic when low mbuf count 309 * Queue sizes less than 128 are not allowed 310 */ 311 switch (queue->queue_size) { 312 case 128: 313 write_interval_ns = 500; 314 break; 315 case 256: 316 write_interval_ns = 500; 317 break; 318 case 512: 319 write_interval_ns = 1000; 320 break; 321 default: 322 write_interval_ns = 2000; 323 break; 324 } 325 326 /* Completion address in UDM */ 327 ark_ddm_setup(queue->ddm, cons_index_addr, write_interval_ns); 328 329 return 0; 330 } 331 332 /* ************************************************************************* */ 333 void 334 eth_ark_tx_queue_release(void *vtx_queue) 335 { 336 struct ark_tx_queue *queue; 337 338 queue = (struct ark_tx_queue *)vtx_queue; 339 340 ark_tx_hw_queue_stop(queue); 341 342 queue->cons_index = queue->prod_index; 343 free_completed_tx(queue); 344 345 rte_free(queue->meta_q); 346 rte_free(queue->bufs); 347 rte_free(queue); 348 } 349 350 /* ************************************************************************* */ 351 int 352 eth_ark_tx_queue_stop(struct rte_eth_dev *dev, uint16_t queue_id) 353 { 354 struct ark_tx_queue *queue; 355 int cnt = 0; 356 357 queue = dev->data->tx_queues[queue_id]; 358 359 /* Wait for DDM to send out all packets. */ 360 while (queue->cons_index != queue->prod_index) { 361 usleep(100); 362 if (cnt++ > 10000) 363 return -1; 364 } 365 366 ark_mpu_stop(queue->mpu); 367 free_completed_tx(queue); 368 369 dev->data->tx_queue_state[queue_id] = RTE_ETH_QUEUE_STATE_STOPPED; 370 371 return 0; 372 } 373 374 int 375 eth_ark_tx_queue_start(struct rte_eth_dev *dev, uint16_t queue_id) 376 { 377 struct ark_tx_queue *queue; 378 379 queue = dev->data->tx_queues[queue_id]; 380 if (dev->data->tx_queue_state[queue_id] == RTE_ETH_QUEUE_STATE_STARTED) 381 return 0; 382 383 ark_mpu_start(queue->mpu); 384 dev->data->tx_queue_state[queue_id] = RTE_ETH_QUEUE_STATE_STARTED; 385 386 return 0; 387 } 388 389 /* ************************************************************************* */ 390 static void 391 free_completed_tx(struct ark_tx_queue *queue) 392 { 393 struct rte_mbuf *mbuf; 394 struct ark_tx_meta *meta; 395 uint32_t top_index; 396 397 top_index = queue->cons_index; /* read once */ 398 while (queue->free_index != top_index) { 399 meta = &queue->meta_q[queue->free_index & queue->queue_mask]; 400 mbuf = queue->bufs[queue->free_index & queue->queue_mask]; 401 402 if (likely((meta->flags & ARK_DDM_SOP) != 0)) { 403 /* ref count of the mbuf is checked in this call. */ 404 rte_pktmbuf_free(mbuf); 405 } 406 queue->free_index++; 407 } 408 } 409 410 /* ************************************************************************* */ 411 void 412 eth_tx_queue_stats_get(void *vqueue, struct rte_eth_stats *stats) 413 { 414 struct ark_tx_queue *queue; 415 struct ark_ddm_t *ddm; 416 uint64_t bytes, pkts; 417 418 queue = vqueue; 419 ddm = queue->ddm; 420 421 bytes = ark_ddm_queue_byte_count(ddm); 422 pkts = ark_ddm_queue_pkt_count(ddm); 423 424 stats->q_opackets[queue->queue_index] = pkts; 425 stats->q_obytes[queue->queue_index] = bytes; 426 stats->opackets += pkts; 427 stats->obytes += bytes; 428 stats->oerrors += queue->tx_errors; 429 } 430 431 void 432 eth_tx_queue_stats_reset(void *vqueue) 433 { 434 struct ark_tx_queue *queue; 435 struct ark_ddm_t *ddm; 436 437 queue = vqueue; 438 ddm = queue->ddm; 439 440 ark_ddm_queue_reset_stats(ddm); 441 queue->tx_errors = 0; 442 } 443