1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (c) 2015-2021 Atomic Rules LLC 3 */ 4 5 #include <unistd.h> 6 7 #include "ark_ethdev_tx.h" 8 #include "ark_global.h" 9 #include "ark_mpu.h" 10 #include "ark_ddm.h" 11 #include "ark_logs.h" 12 13 #define ARK_TX_META_SIZE 32 14 #define ARK_TX_META_OFFSET (RTE_PKTMBUF_HEADROOM - ARK_TX_META_SIZE) 15 #define ARK_TX_MAX_NOCHAIN (RTE_MBUF_DEFAULT_DATAROOM) 16 17 #ifndef RTE_LIBRTE_ARK_MIN_TX_PKTLEN 18 #define ARK_MIN_TX_PKTLEN 0 19 #else 20 #define ARK_MIN_TX_PKTLEN RTE_LIBRTE_ARK_MIN_TX_PKTLEN 21 #endif 22 23 /* ************************************************************************* */ 24 struct ark_tx_queue { 25 union ark_tx_meta *meta_q; 26 struct rte_mbuf **bufs; 27 28 /* handles for hw objects */ 29 struct ark_mpu_t *mpu; 30 struct ark_ddm_t *ddm; 31 32 /* Stats HW tracks bytes and packets, need to count send errors */ 33 uint64_t tx_errors; 34 35 tx_user_meta_hook_fn tx_user_meta_hook; 36 void *ext_user_data; 37 38 uint32_t queue_size; 39 uint32_t queue_mask; 40 41 /* 3 indexes to the paired data rings. */ 42 int32_t prod_index; /* where to put the next one */ 43 int32_t free_index; /* mbuf has been freed */ 44 45 /* The queue Id is used to identify the HW Q */ 46 uint16_t phys_qid; 47 /* The queue Index within the dpdk device structures */ 48 uint16_t queue_index; 49 50 /* next cache line - fields written by device */ 51 RTE_MARKER cacheline1 __rte_cache_min_aligned; 52 volatile int32_t cons_index; /* hw is done, can be freed */ 53 } __rte_cache_aligned; 54 55 /* Forward declarations */ 56 static int eth_ark_tx_jumbo(struct ark_tx_queue *queue, 57 struct rte_mbuf *mbuf, 58 uint32_t *user_meta, uint8_t meta_cnt); 59 static int eth_ark_tx_hw_queue_config(struct ark_tx_queue *queue); 60 static void free_completed_tx(struct ark_tx_queue *queue); 61 62 /* ************************************************************************* */ 63 static inline void 64 eth_ark_tx_desc_fill(struct ark_tx_queue *queue, 65 struct rte_mbuf *mbuf, 66 uint8_t flags, 67 uint32_t *user_meta, 68 uint8_t meta_cnt /* 0 to 5 */ 69 ) 70 { 71 uint32_t tx_idx; 72 union ark_tx_meta *meta; 73 uint8_t m; 74 75 /* Header */ 76 tx_idx = queue->prod_index & queue->queue_mask; 77 meta = &queue->meta_q[tx_idx]; 78 meta->data_len = rte_pktmbuf_data_len(mbuf); 79 meta->flags = flags; 80 meta->meta_cnt = meta_cnt / 2; 81 meta->user1 = meta_cnt ? (*user_meta++) : 0; 82 queue->prod_index++; 83 84 queue->bufs[tx_idx] = mbuf; 85 86 /* 1 or 2 user meta data entries, user words 1,2 and 3,4 */ 87 for (m = 1; m < meta_cnt; m += 2) { 88 tx_idx = queue->prod_index & queue->queue_mask; 89 meta = &queue->meta_q[tx_idx]; 90 meta->usermeta0 = *user_meta++; 91 meta->usermeta1 = *user_meta++; 92 queue->prod_index++; 93 } 94 95 tx_idx = queue->prod_index & queue->queue_mask; 96 meta = &queue->meta_q[tx_idx]; 97 meta->physaddr = rte_mbuf_data_iova(mbuf); 98 queue->prod_index++; 99 } 100 101 102 /* ************************************************************************* */ 103 uint16_t 104 eth_ark_xmit_pkts(void *vtxq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) 105 { 106 struct ark_tx_queue *queue; 107 struct rte_mbuf *mbuf; 108 uint32_t user_meta[5]; 109 110 int stat; 111 int32_t prod_index_limit; 112 uint16_t nb; 113 uint8_t user_len = 0; 114 const uint32_t min_pkt_len = ARK_MIN_TX_PKTLEN; 115 tx_user_meta_hook_fn tx_user_meta_hook; 116 117 queue = (struct ark_tx_queue *)vtxq; 118 tx_user_meta_hook = queue->tx_user_meta_hook; 119 120 /* free any packets after the HW is done with them */ 121 free_completed_tx(queue); 122 123 /* leave 4 elements mpu data */ 124 prod_index_limit = queue->queue_size + queue->free_index - 4; 125 126 for (nb = 0; 127 (nb < nb_pkts) && (prod_index_limit - queue->prod_index) > 0; 128 ++nb) { 129 mbuf = tx_pkts[nb]; 130 131 if (min_pkt_len && 132 unlikely(rte_pktmbuf_pkt_len(mbuf) < min_pkt_len)) { 133 /* this packet even if it is small can be split, 134 * be sure to add to the end mbuf 135 */ 136 uint16_t to_add = min_pkt_len - 137 rte_pktmbuf_pkt_len(mbuf); 138 char *appended = 139 rte_pktmbuf_append(mbuf, to_add); 140 141 if (appended == 0) { 142 /* This packet is in error, 143 * we cannot send it so just 144 * count it and delete it. 145 */ 146 queue->tx_errors += 1; 147 rte_pktmbuf_free(mbuf); 148 continue; 149 } 150 memset(appended, 0, to_add); 151 } 152 153 if (tx_user_meta_hook) 154 tx_user_meta_hook(mbuf, user_meta, &user_len, 155 queue->ext_user_data); 156 if (unlikely(mbuf->nb_segs != 1)) { 157 stat = eth_ark_tx_jumbo(queue, mbuf, 158 user_meta, user_len); 159 if (unlikely(stat != 0)) 160 break; /* Queue is full */ 161 } else { 162 eth_ark_tx_desc_fill(queue, mbuf, 163 ARK_DDM_SOP | ARK_DDM_EOP, 164 user_meta, user_len); 165 } 166 } 167 168 if (ARK_DEBUG_CORE && nb != nb_pkts) { 169 ARK_PMD_LOG(DEBUG, "TX: Failure to send:" 170 " req: %" PRIU32 171 " sent: %" PRIU32 172 " prod: %" PRIU32 173 " cons: %" PRIU32 174 " free: %" PRIU32 "\n", 175 nb_pkts, nb, 176 queue->prod_index, 177 queue->cons_index, 178 queue->free_index); 179 ark_mpu_dump(queue->mpu, 180 "TX Failure MPU: ", 181 queue->phys_qid); 182 } 183 184 /* let FPGA know producer index. */ 185 if (likely(nb != 0)) 186 ark_mpu_set_producer(queue->mpu, queue->prod_index); 187 188 return nb; 189 } 190 191 /* ************************************************************************* */ 192 static int 193 eth_ark_tx_jumbo(struct ark_tx_queue *queue, struct rte_mbuf *mbuf, 194 uint32_t *user_meta, uint8_t meta_cnt) 195 { 196 struct rte_mbuf *next; 197 int32_t free_queue_space; 198 uint8_t flags = ARK_DDM_SOP; 199 200 free_queue_space = queue->queue_mask - 201 (queue->prod_index - queue->free_index); 202 /* We need up to 4 mbufs for first header and 2 for subsequent ones */ 203 if (unlikely(free_queue_space < (2 + (2 * mbuf->nb_segs)))) 204 return -1; 205 206 while (mbuf != NULL) { 207 next = mbuf->next; 208 flags |= (next == NULL) ? ARK_DDM_EOP : 0; 209 210 eth_ark_tx_desc_fill(queue, mbuf, flags, user_meta, meta_cnt); 211 212 flags &= ~ARK_DDM_SOP; /* drop SOP flags */ 213 meta_cnt = 0; /* Meta only on SOP */ 214 mbuf = next; 215 } 216 217 return 0; 218 } 219 220 /* ************************************************************************* */ 221 int 222 eth_ark_tx_queue_setup(struct rte_eth_dev *dev, 223 uint16_t queue_idx, 224 uint16_t nb_desc, 225 unsigned int socket_id, 226 const struct rte_eth_txconf *tx_conf __rte_unused) 227 { 228 struct ark_adapter *ark = dev->data->dev_private; 229 struct ark_tx_queue *queue; 230 int status; 231 232 int qidx = queue_idx; 233 234 if (!rte_is_power_of_2(nb_desc)) { 235 ARK_PMD_LOG(ERR, 236 "DPDK Arkville configuration queue size" 237 " must be power of two %u (%s)\n", 238 nb_desc, __func__); 239 return -1; 240 } 241 242 /* Each packet requires at least 2 mpu elements - double desc count */ 243 nb_desc = 2 * nb_desc; 244 245 /* Allocate queue struct */ 246 queue = rte_zmalloc_socket("Ark_txqueue", 247 sizeof(struct ark_tx_queue), 248 64, 249 socket_id); 250 if (queue == 0) { 251 ARK_PMD_LOG(ERR, "Failed to allocate tx " 252 "queue memory in %s\n", 253 __func__); 254 return -ENOMEM; 255 } 256 257 /* we use zmalloc no need to initialize fields */ 258 queue->queue_size = nb_desc; 259 queue->queue_mask = nb_desc - 1; 260 queue->phys_qid = qidx; 261 queue->queue_index = queue_idx; 262 dev->data->tx_queues[queue_idx] = queue; 263 queue->tx_user_meta_hook = ark->user_ext.tx_user_meta_hook; 264 queue->ext_user_data = ark->user_data[dev->data->port_id]; 265 266 queue->meta_q = 267 rte_zmalloc_socket("Ark_txqueue meta", 268 nb_desc * sizeof(union ark_tx_meta), 269 64, 270 socket_id); 271 queue->bufs = 272 rte_zmalloc_socket("Ark_txqueue bufs", 273 nb_desc * sizeof(struct rte_mbuf *), 274 64, 275 socket_id); 276 277 if (queue->meta_q == 0 || queue->bufs == 0) { 278 ARK_PMD_LOG(ERR, "Failed to allocate " 279 "queue memory in %s\n", __func__); 280 rte_free(queue->meta_q); 281 rte_free(queue->bufs); 282 rte_free(queue); 283 return -ENOMEM; 284 } 285 286 queue->ddm = RTE_PTR_ADD(ark->ddm.v, qidx * ARK_DDM_QOFFSET); 287 queue->mpu = RTE_PTR_ADD(ark->mputx.v, qidx * ARK_MPU_QOFFSET); 288 289 status = eth_ark_tx_hw_queue_config(queue); 290 291 if (unlikely(status != 0)) { 292 rte_free(queue->meta_q); 293 rte_free(queue->bufs); 294 rte_free(queue); 295 return -1; /* ERROR CODE */ 296 } 297 298 return 0; 299 } 300 301 /* ************************************************************************* */ 302 static int 303 eth_ark_tx_hw_queue_config(struct ark_tx_queue *queue) 304 { 305 rte_iova_t queue_base, ring_base, cons_index_addr; 306 307 /* Verify HW -- MPU */ 308 if (ark_mpu_verify(queue->mpu, sizeof(union ark_tx_meta))) 309 return -1; 310 311 queue_base = rte_malloc_virt2iova(queue); 312 ring_base = rte_malloc_virt2iova(queue->meta_q); 313 cons_index_addr = 314 queue_base + offsetof(struct ark_tx_queue, cons_index); 315 316 ark_mpu_stop(queue->mpu); 317 ark_mpu_reset(queue->mpu); 318 319 /* Stop and Reset and configure MPU */ 320 ark_mpu_configure(queue->mpu, ring_base, queue->queue_size, 1); 321 322 /* Completion address in UDM */ 323 ark_ddm_queue_setup(queue->ddm, cons_index_addr); 324 ark_ddm_queue_reset_stats(queue->ddm); 325 326 return 0; 327 } 328 329 /* ************************************************************************* */ 330 void 331 eth_ark_tx_queue_release(void *vtx_queue) 332 { 333 struct ark_tx_queue *queue; 334 335 queue = (struct ark_tx_queue *)vtx_queue; 336 337 ark_ddm_queue_enable(queue->ddm, 0); 338 ark_mpu_stop(queue->mpu); 339 340 queue->cons_index = queue->prod_index; 341 free_completed_tx(queue); 342 343 rte_free(queue->meta_q); 344 rte_free(queue->bufs); 345 rte_free(queue); 346 } 347 348 /* ************************************************************************* */ 349 int 350 eth_ark_tx_queue_stop(struct rte_eth_dev *dev, uint16_t queue_id) 351 { 352 struct ark_tx_queue *queue; 353 int cnt = 0; 354 355 queue = dev->data->tx_queues[queue_id]; 356 357 /* Wait for DDM to send out all packets. */ 358 while (queue->cons_index != queue->prod_index) { 359 usleep(100); 360 if (cnt++ > 10000) 361 return -1; 362 } 363 364 ark_ddm_queue_enable(queue->ddm, 0); 365 ark_mpu_stop(queue->mpu); 366 free_completed_tx(queue); 367 368 dev->data->tx_queue_state[queue_id] = RTE_ETH_QUEUE_STATE_STOPPED; 369 370 return 0; 371 } 372 373 int 374 eth_ark_tx_queue_start(struct rte_eth_dev *dev, uint16_t queue_id) 375 { 376 struct ark_tx_queue *queue; 377 378 queue = dev->data->tx_queues[queue_id]; 379 if (dev->data->tx_queue_state[queue_id] == RTE_ETH_QUEUE_STATE_STARTED) 380 return 0; 381 382 ark_mpu_start(queue->mpu); 383 ark_ddm_queue_enable(queue->ddm, 1); 384 dev->data->tx_queue_state[queue_id] = RTE_ETH_QUEUE_STATE_STARTED; 385 386 return 0; 387 } 388 389 /* ************************************************************************* */ 390 static void 391 free_completed_tx(struct ark_tx_queue *queue) 392 { 393 struct rte_mbuf *mbuf; 394 union ark_tx_meta *meta; 395 int32_t top_index; 396 397 top_index = queue->cons_index; /* read once */ 398 while ((top_index - queue->free_index) > 0) { 399 meta = &queue->meta_q[queue->free_index & queue->queue_mask]; 400 if (likely((meta->flags & ARK_DDM_SOP) != 0)) { 401 mbuf = queue->bufs[queue->free_index & 402 queue->queue_mask]; 403 /* ref count of the mbuf is checked in this call. */ 404 rte_pktmbuf_free(mbuf); 405 } 406 queue->free_index += (meta->meta_cnt + 2); 407 } 408 } 409 410 /* ************************************************************************* */ 411 void 412 eth_tx_queue_stats_get(void *vqueue, struct rte_eth_stats *stats) 413 { 414 struct ark_tx_queue *queue; 415 struct ark_ddm_t *ddm; 416 uint64_t bytes, pkts; 417 418 queue = vqueue; 419 ddm = queue->ddm; 420 421 bytes = ark_ddm_queue_byte_count(ddm); 422 pkts = ark_ddm_queue_pkt_count(ddm); 423 424 stats->q_opackets[queue->queue_index] = pkts; 425 stats->q_obytes[queue->queue_index] = bytes; 426 stats->opackets += pkts; 427 stats->obytes += bytes; 428 stats->oerrors += queue->tx_errors; 429 } 430 431 void 432 eth_tx_queue_stats_reset(void *vqueue) 433 { 434 struct ark_tx_queue *queue; 435 struct ark_ddm_t *ddm; 436 437 queue = vqueue; 438 ddm = queue->ddm; 439 440 ark_ddm_queue_reset_stats(ddm); 441 queue->tx_errors = 0; 442 } 443