1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (c) 2015-2021 Atomic Rules LLC 3 */ 4 5 #include <unistd.h> 6 7 #include "ark_ethdev_tx.h" 8 #include "ark_global.h" 9 #include "ark_mpu.h" 10 #include "ark_ddm.h" 11 #include "ark_logs.h" 12 13 #define ARK_TX_META_SIZE 32 14 #define ARK_TX_META_OFFSET (RTE_PKTMBUF_HEADROOM - ARK_TX_META_SIZE) 15 #define ARK_TX_MAX_NOCHAIN (RTE_MBUF_DEFAULT_DATAROOM) 16 17 #ifndef RTE_LIBRTE_ARK_MIN_TX_PKTLEN 18 #define ARK_MIN_TX_PKTLEN 0 19 #else 20 #define ARK_MIN_TX_PKTLEN RTE_LIBRTE_ARK_MIN_TX_PKTLEN 21 #endif 22 23 /* ************************************************************************* */ 24 struct __rte_cache_aligned ark_tx_queue { 25 union ark_tx_meta *meta_q; 26 struct rte_mbuf **bufs; 27 28 /* handles for hw objects */ 29 struct ark_mpu_t *mpu; 30 struct ark_ddm_t *ddm; 31 32 /* Stats HW tracks bytes and packets, need to count send errors */ 33 uint64_t tx_errors; 34 35 tx_user_meta_hook_fn tx_user_meta_hook; 36 void *ext_user_data; 37 38 uint32_t queue_size; 39 uint32_t queue_mask; 40 41 /* 3 indexes to the paired data rings. */ 42 uint32_t prod_index; /* where to put the next one */ 43 uint32_t free_index; /* mbuf has been freed */ 44 45 /* The queue Id is used to identify the HW Q */ 46 uint16_t phys_qid; 47 /* The queue Index within the dpdk device structures */ 48 uint16_t queue_index; 49 50 /* next cache line - fields written by device */ 51 alignas(RTE_CACHE_LINE_MIN_SIZE) RTE_MARKER cacheline1; 52 volatile uint32_t cons_index; /* hw is done, can be freed */ 53 }; 54 55 /* Forward declarations */ 56 static int eth_ark_tx_jumbo(struct ark_tx_queue *queue, 57 struct rte_mbuf *mbuf, 58 uint32_t *user_meta, uint8_t meta_cnt); 59 static int eth_ark_tx_hw_queue_config(struct ark_tx_queue *queue); 60 static void free_completed_tx(struct ark_tx_queue *queue); 61 62 /* ************************************************************************* */ 63 static inline void 64 eth_ark_tx_desc_fill(struct ark_tx_queue *queue, 65 struct rte_mbuf *mbuf, 66 uint8_t flags, 67 uint32_t *user_meta, 68 uint8_t meta_cnt /* 0 to 5 */ 69 ) 70 { 71 uint32_t tx_idx; 72 union ark_tx_meta *meta; 73 uint8_t m; 74 75 /* Header */ 76 tx_idx = queue->prod_index & queue->queue_mask; 77 meta = &queue->meta_q[tx_idx]; 78 meta->data_len = rte_pktmbuf_data_len(mbuf); 79 meta->flags = flags; 80 meta->meta_cnt = meta_cnt / 2; 81 meta->user1 = meta_cnt ? (*user_meta++) : 0; 82 queue->prod_index++; 83 84 queue->bufs[tx_idx] = mbuf; 85 86 /* 1 or 2 user meta data entries, user words 1,2 and 3,4 */ 87 for (m = 1; m < meta_cnt; m += 2) { 88 tx_idx = queue->prod_index & queue->queue_mask; 89 meta = &queue->meta_q[tx_idx]; 90 meta->usermeta0 = *user_meta++; 91 meta->usermeta1 = *user_meta++; 92 queue->prod_index++; 93 } 94 95 tx_idx = queue->prod_index & queue->queue_mask; 96 meta = &queue->meta_q[tx_idx]; 97 meta->physaddr = rte_mbuf_data_iova(mbuf); 98 queue->prod_index++; 99 } 100 101 102 /* ************************************************************************* */ 103 uint16_t 104 eth_ark_xmit_pkts(void *vtxq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) 105 { 106 struct ark_tx_queue *queue; 107 struct rte_mbuf *mbuf; 108 uint32_t user_meta[5]; 109 110 int stat; 111 uint32_t prod_index_limit; 112 uint16_t nb; 113 uint8_t user_len = 0; 114 const uint32_t min_pkt_len = ARK_MIN_TX_PKTLEN; 115 tx_user_meta_hook_fn tx_user_meta_hook; 116 117 queue = (struct ark_tx_queue *)vtxq; 118 tx_user_meta_hook = queue->tx_user_meta_hook; 119 120 /* free any packets after the HW is done with them */ 121 free_completed_tx(queue); 122 123 /* leave 4 elements mpu data */ 124 prod_index_limit = queue->queue_size + queue->free_index - 4; 125 126 /* Populate the buffer bringing prod_index up to or slightly beyond 127 * prod_index_limit. Prod_index will increment by 2 or more each 128 * iteration. Note: indexes are uint32_t, cast to (signed) int32_t 129 * to catch the slight overage case; e.g. (200 - 201) 130 */ 131 for (nb = 0; 132 (nb < nb_pkts) && (int32_t)(prod_index_limit - queue->prod_index) > 0; 133 ++nb) { 134 mbuf = tx_pkts[nb]; 135 136 if (min_pkt_len && 137 unlikely(rte_pktmbuf_pkt_len(mbuf) < min_pkt_len)) { 138 /* this packet even if it is small can be split, 139 * be sure to add to the end mbuf 140 */ 141 uint16_t to_add = min_pkt_len - 142 rte_pktmbuf_pkt_len(mbuf); 143 char *appended = 144 rte_pktmbuf_append(mbuf, to_add); 145 146 if (appended == 0) { 147 /* This packet is in error, 148 * we cannot send it so just 149 * count it and delete it. 150 */ 151 queue->tx_errors += 1; 152 rte_pktmbuf_free(mbuf); 153 continue; 154 } 155 memset(appended, 0, to_add); 156 } 157 158 if (tx_user_meta_hook) 159 tx_user_meta_hook(mbuf, user_meta, &user_len, 160 queue->ext_user_data); 161 if (unlikely(mbuf->nb_segs != 1)) { 162 stat = eth_ark_tx_jumbo(queue, mbuf, 163 user_meta, user_len); 164 if (unlikely(stat != 0)) 165 break; /* Queue is full */ 166 } else { 167 eth_ark_tx_desc_fill(queue, mbuf, 168 ARK_DDM_SOP | ARK_DDM_EOP, 169 user_meta, user_len); 170 } 171 } 172 173 if (ARK_DEBUG_CORE && nb != nb_pkts) { 174 ARK_PMD_LOG(DEBUG, "TX: Failure to send:" 175 " req: %" PRIU32 176 " sent: %" PRIU32 177 " prod: %" PRIU32 178 " cons: %" PRIU32 179 " free: %" PRIU32 "\n", 180 nb_pkts, nb, 181 queue->prod_index, 182 queue->cons_index, 183 queue->free_index); 184 ark_mpu_dump(queue->mpu, 185 "TX Failure MPU: ", 186 queue->phys_qid); 187 } 188 189 /* let FPGA know producer index. */ 190 if (likely(nb != 0)) 191 ark_mpu_set_producer(queue->mpu, queue->prod_index); 192 193 return nb; 194 } 195 196 /* ************************************************************************* */ 197 static int 198 eth_ark_tx_jumbo(struct ark_tx_queue *queue, struct rte_mbuf *mbuf, 199 uint32_t *user_meta, uint8_t meta_cnt) 200 { 201 struct rte_mbuf *next; 202 uint32_t free_queue_space; 203 uint8_t flags = ARK_DDM_SOP; 204 205 free_queue_space = queue->queue_mask - 206 (queue->prod_index - queue->free_index); 207 /* We need up to 4 mbufs for first header and 2 for subsequent ones */ 208 if (unlikely(free_queue_space < (2U + (2U * mbuf->nb_segs)))) 209 return -1; 210 211 while (mbuf != NULL) { 212 next = mbuf->next; 213 flags |= (next == NULL) ? ARK_DDM_EOP : 0; 214 215 eth_ark_tx_desc_fill(queue, mbuf, flags, user_meta, meta_cnt); 216 217 flags &= ~ARK_DDM_SOP; /* drop SOP flags */ 218 meta_cnt = 0; /* Meta only on SOP */ 219 mbuf = next; 220 } 221 222 return 0; 223 } 224 225 /* ************************************************************************* */ 226 int 227 eth_ark_tx_queue_setup(struct rte_eth_dev *dev, 228 uint16_t queue_idx, 229 uint16_t nb_desc, 230 unsigned int socket_id, 231 const struct rte_eth_txconf *tx_conf __rte_unused) 232 { 233 struct ark_adapter *ark = dev->data->dev_private; 234 struct ark_tx_queue *queue; 235 int status; 236 237 int qidx = ark->qbase + queue_idx; 238 239 if (!rte_is_power_of_2(nb_desc)) { 240 ARK_PMD_LOG(ERR, 241 "DPDK Arkville configuration queue size" 242 " must be power of two %u (%s)\n", 243 nb_desc, __func__); 244 return -1; 245 } 246 247 /* Each packet requires at least 2 mpu elements - double desc count */ 248 nb_desc = 2 * nb_desc; 249 250 /* Allocate queue struct */ 251 queue = rte_zmalloc_socket("Ark_txqueue", 252 sizeof(struct ark_tx_queue), 253 64, 254 socket_id); 255 if (queue == 0) { 256 ARK_PMD_LOG(ERR, "Failed to allocate tx " 257 "queue memory in %s\n", 258 __func__); 259 return -ENOMEM; 260 } 261 262 /* we use zmalloc no need to initialize fields */ 263 queue->queue_size = nb_desc; 264 queue->queue_mask = nb_desc - 1; 265 queue->phys_qid = qidx; 266 queue->queue_index = queue_idx; 267 dev->data->tx_queues[queue_idx] = queue; 268 queue->tx_user_meta_hook = ark->user_ext.tx_user_meta_hook; 269 queue->ext_user_data = ark->user_data[dev->data->port_id]; 270 271 queue->meta_q = 272 rte_zmalloc_socket("Ark_txqueue meta", 273 nb_desc * sizeof(union ark_tx_meta), 274 64, 275 socket_id); 276 queue->bufs = 277 rte_zmalloc_socket("Ark_txqueue bufs", 278 nb_desc * sizeof(struct rte_mbuf *), 279 64, 280 socket_id); 281 282 if (queue->meta_q == 0 || queue->bufs == 0) { 283 ARK_PMD_LOG(ERR, "Failed to allocate " 284 "queue memory in %s\n", __func__); 285 rte_free(queue->meta_q); 286 rte_free(queue->bufs); 287 rte_free(queue); 288 return -ENOMEM; 289 } 290 291 queue->ddm = RTE_PTR_ADD(ark->ddm.v, qidx * ARK_DDM_QOFFSET); 292 queue->mpu = RTE_PTR_ADD(ark->mputx.v, qidx * ARK_MPU_QOFFSET); 293 294 status = eth_ark_tx_hw_queue_config(queue); 295 296 if (unlikely(status != 0)) { 297 rte_free(queue->meta_q); 298 rte_free(queue->bufs); 299 rte_free(queue); 300 return -1; /* ERROR CODE */ 301 } 302 303 return 0; 304 } 305 306 /* ************************************************************************* */ 307 static int 308 eth_ark_tx_hw_queue_config(struct ark_tx_queue *queue) 309 { 310 rte_iova_t queue_base, ring_base, cons_index_addr; 311 312 /* Verify HW -- MPU */ 313 if (ark_mpu_verify(queue->mpu, sizeof(union ark_tx_meta))) 314 return -1; 315 316 queue_base = rte_malloc_virt2iova(queue); 317 ring_base = rte_malloc_virt2iova(queue->meta_q); 318 cons_index_addr = 319 queue_base + offsetof(struct ark_tx_queue, cons_index); 320 321 ark_mpu_stop(queue->mpu); 322 ark_mpu_reset(queue->mpu); 323 324 /* Stop and Reset and configure MPU */ 325 ark_mpu_configure(queue->mpu, ring_base, queue->queue_size, 1); 326 327 /* Completion address in UDM */ 328 ark_ddm_queue_setup(queue->ddm, cons_index_addr); 329 ark_ddm_queue_reset_stats(queue->ddm); 330 331 return 0; 332 } 333 334 /* ************************************************************************* */ 335 void 336 eth_ark_tx_queue_release(void *vtx_queue) 337 { 338 struct ark_tx_queue *queue; 339 340 queue = (struct ark_tx_queue *)vtx_queue; 341 342 ark_ddm_queue_enable(queue->ddm, 0); 343 ark_mpu_stop(queue->mpu); 344 345 queue->cons_index = queue->prod_index; 346 free_completed_tx(queue); 347 348 rte_free(queue->meta_q); 349 rte_free(queue->bufs); 350 rte_free(queue); 351 } 352 353 /* ************************************************************************* */ 354 int 355 eth_ark_tx_queue_stop(struct rte_eth_dev *dev, uint16_t queue_id) 356 { 357 struct ark_tx_queue *queue; 358 int cnt = 0; 359 360 queue = dev->data->tx_queues[queue_id]; 361 362 /* Wait for DDM to send out all packets. */ 363 while (queue->cons_index != queue->prod_index) { 364 usleep(100); 365 if (cnt++ > 10000) 366 return -1; 367 } 368 369 ark_ddm_queue_enable(queue->ddm, 0); 370 ark_mpu_stop(queue->mpu); 371 free_completed_tx(queue); 372 373 dev->data->tx_queue_state[queue_id] = RTE_ETH_QUEUE_STATE_STOPPED; 374 375 return 0; 376 } 377 378 int 379 eth_ark_tx_queue_start(struct rte_eth_dev *dev, uint16_t queue_id) 380 { 381 struct ark_tx_queue *queue; 382 383 queue = dev->data->tx_queues[queue_id]; 384 if (dev->data->tx_queue_state[queue_id] == RTE_ETH_QUEUE_STATE_STARTED) 385 return 0; 386 387 ark_mpu_start(queue->mpu); 388 ark_ddm_queue_enable(queue->ddm, 1); 389 dev->data->tx_queue_state[queue_id] = RTE_ETH_QUEUE_STATE_STARTED; 390 391 return 0; 392 } 393 394 /* ************************************************************************* */ 395 static void 396 free_completed_tx(struct ark_tx_queue *queue) 397 { 398 struct rte_mbuf *mbuf; 399 union ark_tx_meta *meta; 400 uint32_t top_index; 401 402 top_index = queue->cons_index; /* read once */ 403 404 while ((int32_t)(top_index - queue->free_index) > 0) { 405 meta = &queue->meta_q[queue->free_index & queue->queue_mask]; 406 if (likely((meta->flags & ARK_DDM_SOP) != 0)) { 407 mbuf = queue->bufs[queue->free_index & 408 queue->queue_mask]; 409 /* ref count of the mbuf is checked in this call. */ 410 rte_pktmbuf_free(mbuf); 411 } 412 queue->free_index += (meta->meta_cnt + 2); 413 } 414 } 415 416 /* ************************************************************************* */ 417 void 418 eth_tx_queue_stats_get(void *vqueue, struct rte_eth_stats *stats) 419 { 420 struct ark_tx_queue *queue; 421 struct ark_ddm_t *ddm; 422 uint64_t bytes, pkts; 423 424 queue = vqueue; 425 ddm = queue->ddm; 426 427 bytes = ark_ddm_queue_byte_count(ddm); 428 pkts = ark_ddm_queue_pkt_count(ddm); 429 430 stats->q_opackets[queue->queue_index] = pkts; 431 stats->q_obytes[queue->queue_index] = bytes; 432 stats->opackets += pkts; 433 stats->obytes += bytes; 434 stats->oerrors += queue->tx_errors; 435 } 436 437 void 438 eth_tx_queue_stats_reset(void *vqueue) 439 { 440 struct ark_tx_queue *queue; 441 struct ark_ddm_t *ddm; 442 443 queue = vqueue; 444 ddm = queue->ddm; 445 446 ark_ddm_queue_reset_stats(ddm); 447 queue->tx_errors = 0; 448 } 449