1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (c) 2015-2018 Atomic Rules LLC 3 */ 4 5 #include <unistd.h> 6 7 #include "ark_ethdev_tx.h" 8 #include "ark_global.h" 9 #include "ark_mpu.h" 10 #include "ark_ddm.h" 11 #include "ark_logs.h" 12 13 #define ARK_TX_META_SIZE 32 14 #define ARK_TX_META_OFFSET (RTE_PKTMBUF_HEADROOM - ARK_TX_META_SIZE) 15 #define ARK_TX_MAX_NOCHAIN (RTE_MBUF_DEFAULT_DATAROOM) 16 17 18 /* ************************************************************************* */ 19 struct ark_tx_queue { 20 struct ark_tx_meta *meta_q; 21 struct rte_mbuf **bufs; 22 23 /* handles for hw objects */ 24 struct ark_mpu_t *mpu; 25 struct ark_ddm_t *ddm; 26 27 /* Stats HW tracks bytes and packets, need to count send errors */ 28 uint64_t tx_errors; 29 30 uint32_t queue_size; 31 uint32_t queue_mask; 32 33 /* 3 indexes to the paired data rings. */ 34 uint32_t prod_index; /* where to put the next one */ 35 uint32_t free_index; /* mbuf has been freed */ 36 37 /* The queue Id is used to identify the HW Q */ 38 uint16_t phys_qid; 39 /* The queue Index within the dpdk device structures */ 40 uint16_t queue_index; 41 42 uint32_t pad[1]; 43 44 /* second cache line - fields only used in slow path */ 45 MARKER cacheline1 __rte_cache_min_aligned; 46 uint32_t cons_index; /* hw is done, can be freed */ 47 } __rte_cache_aligned; 48 49 /* Forward declarations */ 50 static uint32_t eth_ark_tx_jumbo(struct ark_tx_queue *queue, 51 struct rte_mbuf *mbuf); 52 static int eth_ark_tx_hw_queue_config(struct ark_tx_queue *queue); 53 static void free_completed_tx(struct ark_tx_queue *queue); 54 55 static inline void 56 ark_tx_hw_queue_stop(struct ark_tx_queue *queue) 57 { 58 ark_mpu_stop(queue->mpu); 59 } 60 61 /* ************************************************************************* */ 62 static inline void 63 eth_ark_tx_meta_from_mbuf(struct ark_tx_meta *meta, 64 const struct rte_mbuf *mbuf, 65 uint8_t flags) 66 { 67 meta->physaddr = rte_mbuf_data_iova(mbuf); 68 meta->user1 = (uint32_t)mbuf->udata64; 69 meta->data_len = rte_pktmbuf_data_len(mbuf); 70 meta->flags = flags; 71 } 72 73 /* ************************************************************************* */ 74 uint16_t 75 eth_ark_xmit_pkts_noop(void *vtxq __rte_unused, 76 struct rte_mbuf **tx_pkts __rte_unused, 77 uint16_t nb_pkts __rte_unused) 78 { 79 return 0; 80 } 81 82 /* ************************************************************************* */ 83 uint16_t 84 eth_ark_xmit_pkts(void *vtxq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) 85 { 86 struct ark_tx_queue *queue; 87 struct rte_mbuf *mbuf; 88 struct ark_tx_meta *meta; 89 90 uint32_t idx; 91 uint32_t prod_index_limit; 92 int stat; 93 uint16_t nb; 94 95 queue = (struct ark_tx_queue *)vtxq; 96 97 /* free any packets after the HW is done with them */ 98 free_completed_tx(queue); 99 100 prod_index_limit = queue->queue_size + queue->free_index; 101 102 for (nb = 0; 103 (nb < nb_pkts) && (queue->prod_index != prod_index_limit); 104 ++nb) { 105 mbuf = tx_pkts[nb]; 106 107 if (ARK_TX_PAD_TO_60) { 108 if (unlikely(rte_pktmbuf_pkt_len(mbuf) < 60)) { 109 /* this packet even if it is small can be split, 110 * be sure to add to the end mbuf 111 */ 112 uint16_t to_add = 113 60 - rte_pktmbuf_pkt_len(mbuf); 114 char *appended = 115 rte_pktmbuf_append(mbuf, to_add); 116 117 if (appended == 0) { 118 /* This packet is in error, 119 * we cannot send it so just 120 * count it and delete it. 121 */ 122 queue->tx_errors += 1; 123 rte_pktmbuf_free(mbuf); 124 continue; 125 } 126 memset(appended, 0, to_add); 127 } 128 } 129 130 if (unlikely(mbuf->nb_segs != 1)) { 131 stat = eth_ark_tx_jumbo(queue, mbuf); 132 if (unlikely(stat != 0)) 133 break; /* Queue is full */ 134 } else { 135 idx = queue->prod_index & queue->queue_mask; 136 queue->bufs[idx] = mbuf; 137 meta = &queue->meta_q[idx]; 138 eth_ark_tx_meta_from_mbuf(meta, 139 mbuf, 140 ARK_DDM_SOP | 141 ARK_DDM_EOP); 142 queue->prod_index++; 143 } 144 } 145 146 if (ARK_TX_DEBUG && (nb != nb_pkts)) { 147 PMD_TX_LOG(DEBUG, "TX: Failure to send:" 148 " req: %" PRIU32 149 " sent: %" PRIU32 150 " prod: %" PRIU32 151 " cons: %" PRIU32 152 " free: %" PRIU32 "\n", 153 nb_pkts, nb, 154 queue->prod_index, 155 queue->cons_index, 156 queue->free_index); 157 ark_mpu_dump(queue->mpu, 158 "TX Failure MPU: ", 159 queue->phys_qid); 160 } 161 162 /* let FPGA know producer index. */ 163 if (likely(nb != 0)) 164 ark_mpu_set_producer(queue->mpu, queue->prod_index); 165 166 return nb; 167 } 168 169 /* ************************************************************************* */ 170 static uint32_t 171 eth_ark_tx_jumbo(struct ark_tx_queue *queue, struct rte_mbuf *mbuf) 172 { 173 struct rte_mbuf *next; 174 struct ark_tx_meta *meta; 175 uint32_t free_queue_space; 176 uint32_t idx; 177 uint8_t flags = ARK_DDM_SOP; 178 179 free_queue_space = queue->queue_mask - 180 (queue->prod_index - queue->free_index); 181 if (unlikely(free_queue_space < mbuf->nb_segs)) 182 return -1; 183 184 while (mbuf != NULL) { 185 next = mbuf->next; 186 187 idx = queue->prod_index & queue->queue_mask; 188 queue->bufs[idx] = mbuf; 189 meta = &queue->meta_q[idx]; 190 191 flags |= (next == NULL) ? ARK_DDM_EOP : 0; 192 eth_ark_tx_meta_from_mbuf(meta, mbuf, flags); 193 queue->prod_index++; 194 195 flags &= ~ARK_DDM_SOP; /* drop SOP flags */ 196 mbuf = next; 197 } 198 199 return 0; 200 } 201 202 /* ************************************************************************* */ 203 int 204 eth_ark_tx_queue_setup(struct rte_eth_dev *dev, 205 uint16_t queue_idx, 206 uint16_t nb_desc, 207 unsigned int socket_id, 208 const struct rte_eth_txconf *tx_conf __rte_unused) 209 { 210 struct ark_adapter *ark = (struct ark_adapter *)dev->data->dev_private; 211 struct ark_tx_queue *queue; 212 int status; 213 214 /* Future: divide the Q's evenly with multi-ports */ 215 int port = dev->data->port_id; 216 int qidx = port + queue_idx; 217 218 if (!rte_is_power_of_2(nb_desc)) { 219 PMD_DRV_LOG(ERR, 220 "DPDK Arkville configuration queue size" 221 " must be power of two %u (%s)\n", 222 nb_desc, __func__); 223 return -1; 224 } 225 226 /* Allocate queue struct */ 227 queue = rte_zmalloc_socket("Ark_txqueue", 228 sizeof(struct ark_tx_queue), 229 64, 230 socket_id); 231 if (queue == 0) { 232 PMD_DRV_LOG(ERR, "Failed to allocate tx " 233 "queue memory in %s\n", 234 __func__); 235 return -ENOMEM; 236 } 237 238 /* we use zmalloc no need to initialize fields */ 239 queue->queue_size = nb_desc; 240 queue->queue_mask = nb_desc - 1; 241 queue->phys_qid = qidx; 242 queue->queue_index = queue_idx; 243 dev->data->tx_queues[queue_idx] = queue; 244 245 queue->meta_q = 246 rte_zmalloc_socket("Ark_txqueue meta", 247 nb_desc * sizeof(struct ark_tx_meta), 248 64, 249 socket_id); 250 queue->bufs = 251 rte_zmalloc_socket("Ark_txqueue bufs", 252 nb_desc * sizeof(struct rte_mbuf *), 253 64, 254 socket_id); 255 256 if (queue->meta_q == 0 || queue->bufs == 0) { 257 PMD_DRV_LOG(ERR, "Failed to allocate " 258 "queue memory in %s\n", __func__); 259 rte_free(queue->meta_q); 260 rte_free(queue->bufs); 261 rte_free(queue); 262 return -ENOMEM; 263 } 264 265 queue->ddm = RTE_PTR_ADD(ark->ddm.v, qidx * ARK_DDM_QOFFSET); 266 queue->mpu = RTE_PTR_ADD(ark->mputx.v, qidx * ARK_MPU_QOFFSET); 267 268 status = eth_ark_tx_hw_queue_config(queue); 269 270 if (unlikely(status != 0)) { 271 rte_free(queue->meta_q); 272 rte_free(queue->bufs); 273 rte_free(queue); 274 return -1; /* ERROR CODE */ 275 } 276 277 return 0; 278 } 279 280 /* ************************************************************************* */ 281 static int 282 eth_ark_tx_hw_queue_config(struct ark_tx_queue *queue) 283 { 284 rte_iova_t queue_base, ring_base, cons_index_addr; 285 uint32_t write_interval_ns; 286 287 /* Verify HW -- MPU */ 288 if (ark_mpu_verify(queue->mpu, sizeof(struct ark_tx_meta))) 289 return -1; 290 291 queue_base = rte_malloc_virt2iova(queue); 292 ring_base = rte_malloc_virt2iova(queue->meta_q); 293 cons_index_addr = 294 queue_base + offsetof(struct ark_tx_queue, cons_index); 295 296 ark_mpu_stop(queue->mpu); 297 ark_mpu_reset(queue->mpu); 298 299 /* Stop and Reset and configure MPU */ 300 ark_mpu_configure(queue->mpu, ring_base, queue->queue_size, 1); 301 302 /* 303 * Adjust the write interval based on queue size -- 304 * increase pcie traffic when low mbuf count 305 * Queue sizes less than 128 are not allowed 306 */ 307 switch (queue->queue_size) { 308 case 128: 309 write_interval_ns = 500; 310 break; 311 case 256: 312 write_interval_ns = 500; 313 break; 314 case 512: 315 write_interval_ns = 1000; 316 break; 317 default: 318 write_interval_ns = 2000; 319 break; 320 } 321 322 /* Completion address in UDM */ 323 ark_ddm_setup(queue->ddm, cons_index_addr, write_interval_ns); 324 325 return 0; 326 } 327 328 /* ************************************************************************* */ 329 void 330 eth_ark_tx_queue_release(void *vtx_queue) 331 { 332 struct ark_tx_queue *queue; 333 334 queue = (struct ark_tx_queue *)vtx_queue; 335 336 ark_tx_hw_queue_stop(queue); 337 338 queue->cons_index = queue->prod_index; 339 free_completed_tx(queue); 340 341 rte_free(queue->meta_q); 342 rte_free(queue->bufs); 343 rte_free(queue); 344 } 345 346 /* ************************************************************************* */ 347 int 348 eth_ark_tx_queue_stop(struct rte_eth_dev *dev, uint16_t queue_id) 349 { 350 struct ark_tx_queue *queue; 351 int cnt = 0; 352 353 queue = dev->data->tx_queues[queue_id]; 354 355 /* Wait for DDM to send out all packets. */ 356 while (queue->cons_index != queue->prod_index) { 357 usleep(100); 358 if (cnt++ > 10000) 359 return -1; 360 } 361 362 ark_mpu_stop(queue->mpu); 363 free_completed_tx(queue); 364 365 dev->data->tx_queue_state[queue_id] = RTE_ETH_QUEUE_STATE_STOPPED; 366 367 return 0; 368 } 369 370 int 371 eth_ark_tx_queue_start(struct rte_eth_dev *dev, uint16_t queue_id) 372 { 373 struct ark_tx_queue *queue; 374 375 queue = dev->data->tx_queues[queue_id]; 376 if (dev->data->tx_queue_state[queue_id] == RTE_ETH_QUEUE_STATE_STARTED) 377 return 0; 378 379 ark_mpu_start(queue->mpu); 380 dev->data->tx_queue_state[queue_id] = RTE_ETH_QUEUE_STATE_STARTED; 381 382 return 0; 383 } 384 385 /* ************************************************************************* */ 386 static void 387 free_completed_tx(struct ark_tx_queue *queue) 388 { 389 struct rte_mbuf *mbuf; 390 struct ark_tx_meta *meta; 391 uint32_t top_index; 392 393 top_index = queue->cons_index; /* read once */ 394 while (queue->free_index != top_index) { 395 meta = &queue->meta_q[queue->free_index & queue->queue_mask]; 396 mbuf = queue->bufs[queue->free_index & queue->queue_mask]; 397 398 if (likely((meta->flags & ARK_DDM_SOP) != 0)) { 399 /* ref count of the mbuf is checked in this call. */ 400 rte_pktmbuf_free(mbuf); 401 } 402 queue->free_index++; 403 } 404 } 405 406 /* ************************************************************************* */ 407 void 408 eth_tx_queue_stats_get(void *vqueue, struct rte_eth_stats *stats) 409 { 410 struct ark_tx_queue *queue; 411 struct ark_ddm_t *ddm; 412 uint64_t bytes, pkts; 413 414 queue = vqueue; 415 ddm = queue->ddm; 416 417 bytes = ark_ddm_queue_byte_count(ddm); 418 pkts = ark_ddm_queue_pkt_count(ddm); 419 420 stats->q_opackets[queue->queue_index] = pkts; 421 stats->q_obytes[queue->queue_index] = bytes; 422 stats->opackets += pkts; 423 stats->obytes += bytes; 424 stats->oerrors += queue->tx_errors; 425 } 426 427 void 428 eth_tx_queue_stats_reset(void *vqueue) 429 { 430 struct ark_tx_queue *queue; 431 struct ark_ddm_t *ddm; 432 433 queue = vqueue; 434 ddm = queue->ddm; 435 436 ark_ddm_queue_reset_stats(ddm); 437 queue->tx_errors = 0; 438 } 439