xref: /dpdk/drivers/net/ark/ark_ethdev_tx.c (revision 4e30ead5e7ca886535e2b30632b2948d2aac1681) 
1 /*-
2  * BSD LICENSE
3  *
4  * Copyright (c) 2015-2017 Atomic Rules LLC
5  * All rights reserved.
6  *
7  * Redistribution and use in source and binary forms, with or without
8  * modification, are permitted provided that the following conditions
9  * are met:
10  *
11  * * Redistributions of source code must retain the above copyright
12  * notice, this list of conditions and the following disclaimer.
13  * * Redistributions in binary form must reproduce the above copyright
14  * notice, this list of conditions and the following disclaimer in
15  * the documentation and/or other materials provided with the
16  * distribution.
17  * * Neither the name of copyright holder nor the names of its
18  * contributors may be used to endorse or promote products derived
19  * from this software without specific prior written permission.
20  *
21  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25  * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32  */
33 
34 #include <unistd.h>
35 
36 #include "ark_ethdev_tx.h"
37 #include "ark_global.h"
38 #include "ark_mpu.h"
39 #include "ark_ddm.h"
40 #include "ark_ethdev.h"
41 #include "ark_logs.h"
42 
43 #define ARK_TX_META_SIZE   32
44 #define ARK_TX_META_OFFSET (RTE_PKTMBUF_HEADROOM - ARK_TX_META_SIZE)
45 #define ARK_TX_MAX_NOCHAIN (RTE_MBUF_DEFAULT_DATAROOM)
46 
47 
48 /* ************************************************************************* */
49 struct ark_tx_queue {
50 	struct ark_tx_meta *meta_q;
51 	struct rte_mbuf **bufs;
52 
53 	/* handles for hw objects */
54 	struct ark_mpu_t *mpu;
55 	struct ark_ddm_t *ddm;
56 
57 	/* Stats HW tracks bytes and packets, need to count send errors */
58 	uint64_t tx_errors;
59 
60 	uint32_t queue_size;
61 	uint32_t queue_mask;
62 
63 	/* 3 indexes to the paired data rings. */
64 	uint32_t prod_index;		/* where to put the next one */
65 	uint32_t free_index;		/* mbuf has been freed */
66 
67 	/* The queue Id is used to identify the HW Q */
68 	uint16_t phys_qid;
69 	/* The queue Index within the dpdk device structures */
70 	uint16_t queue_index;
71 
72 	uint32_t pad[1];
73 
74 	/* second cache line - fields only used in slow path */
75 	MARKER cacheline1 __rte_cache_min_aligned;
76 	uint32_t cons_index;		/* hw is done, can be freed */
77 } __rte_cache_aligned;
78 
79 /* Forward declarations */
80 static uint32_t eth_ark_tx_jumbo(struct ark_tx_queue *queue,
81 				 struct rte_mbuf *mbuf);
82 static int eth_ark_tx_hw_queue_config(struct ark_tx_queue *queue);
83 static void free_completed_tx(struct ark_tx_queue *queue);
84 
85 static inline void
86 ark_tx_hw_queue_stop(struct ark_tx_queue *queue)
87 {
88 	ark_mpu_stop(queue->mpu);
89 }
90 
91 /* ************************************************************************* */
92 static inline void
93 eth_ark_tx_meta_from_mbuf(struct ark_tx_meta *meta,
94 			  const struct rte_mbuf *mbuf,
95 			  uint8_t flags)
96 {
97 	meta->physaddr = rte_mbuf_data_dma_addr(mbuf);
98 	meta->delta_ns = 0;
99 	meta->data_len = rte_pktmbuf_data_len(mbuf);
100 	meta->flags = flags;
101 }
102 
103 /* ************************************************************************* */
104 uint16_t
105 eth_ark_xmit_pkts_noop(void *vtxq __rte_unused,
106 		       struct rte_mbuf **tx_pkts __rte_unused,
107 		       uint16_t nb_pkts __rte_unused)
108 {
109 	return 0;
110 }
111 
112 /* ************************************************************************* */
113 uint16_t
114 eth_ark_xmit_pkts(void *vtxq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
115 {
116 	struct ark_tx_queue *queue;
117 	struct rte_mbuf *mbuf;
118 	struct ark_tx_meta *meta;
119 
120 	uint32_t idx;
121 	uint32_t prod_index_limit;
122 	int stat;
123 	uint16_t nb;
124 
125 	queue = (struct ark_tx_queue *)vtxq;
126 
127 	/* free any packets after the HW is done with them */
128 	free_completed_tx(queue);
129 
130 	prod_index_limit = queue->queue_size + queue->free_index;
131 
132 	for (nb = 0;
133 	     (nb < nb_pkts) && (queue->prod_index != prod_index_limit);
134 	     ++nb) {
135 		mbuf = tx_pkts[nb];
136 
137 		if (ARK_TX_PAD_TO_60) {
138 			if (unlikely(rte_pktmbuf_pkt_len(mbuf) < 60)) {
139 				/* this packet even if it is small can be split,
140 				 * be sure to add to the end mbuf
141 				 */
142 				uint16_t to_add =
143 					60 - rte_pktmbuf_pkt_len(mbuf);
144 				char *appended =
145 					rte_pktmbuf_append(mbuf, to_add);
146 
147 				if (appended == 0) {
148 					/* This packet is in error,
149 					 * we cannot send it so just
150 					 * count it and delete it.
151 					 */
152 					queue->tx_errors += 1;
153 					rte_pktmbuf_free(mbuf);
154 					continue;
155 				}
156 				memset(appended, 0, to_add);
157 			}
158 		}
159 
160 		if (unlikely(mbuf->nb_segs != 1)) {
161 			stat = eth_ark_tx_jumbo(queue, mbuf);
162 			if (unlikely(stat != 0))
163 				break;		/* Queue is full */
164 		} else {
165 			idx = queue->prod_index & queue->queue_mask;
166 			queue->bufs[idx] = mbuf;
167 			meta = &queue->meta_q[idx];
168 			eth_ark_tx_meta_from_mbuf(meta,
169 						  mbuf,
170 						  ARK_DDM_SOP |
171 						  ARK_DDM_EOP);
172 			queue->prod_index++;
173 		}
174 	}
175 
176 	if (ARK_TX_DEBUG && (nb != nb_pkts)) {
177 		PMD_TX_LOG(DEBUG, "TX: Failure to send:"
178 			   " req: %" PRIU32
179 			   " sent: %" PRIU32
180 			   " prod: %" PRIU32
181 			   " cons: %" PRIU32
182 			   " free: %" PRIU32 "\n",
183 			   nb_pkts, nb,
184 			   queue->prod_index,
185 			   queue->cons_index,
186 			   queue->free_index);
187 		ark_mpu_dump(queue->mpu,
188 			     "TX Failure MPU: ",
189 			     queue->phys_qid);
190 	}
191 
192 	/* let FPGA know producer index.  */
193 	if (likely(nb != 0))
194 		ark_mpu_set_producer(queue->mpu, queue->prod_index);
195 
196 	return nb;
197 }
198 
199 /* ************************************************************************* */
200 static uint32_t
201 eth_ark_tx_jumbo(struct ark_tx_queue *queue, struct rte_mbuf *mbuf)
202 {
203 	struct rte_mbuf *next;
204 	struct ark_tx_meta *meta;
205 	uint32_t free_queue_space;
206 	uint32_t idx;
207 	uint8_t flags = ARK_DDM_SOP;
208 
209 	free_queue_space = queue->queue_mask -
210 		(queue->prod_index - queue->free_index);
211 	if (unlikely(free_queue_space < mbuf->nb_segs))
212 		return -1;
213 
214 	while (mbuf != NULL) {
215 		next = mbuf->next;
216 
217 		idx = queue->prod_index & queue->queue_mask;
218 		queue->bufs[idx] = mbuf;
219 		meta = &queue->meta_q[idx];
220 
221 		flags |= (next == NULL) ? ARK_DDM_EOP : 0;
222 		eth_ark_tx_meta_from_mbuf(meta, mbuf, flags);
223 		queue->prod_index++;
224 
225 		flags &= ~ARK_DDM_SOP;	/* drop SOP flags */
226 		mbuf = next;
227 	}
228 
229 	return 0;
230 }
231 
232 /* ************************************************************************* */
233 int
234 eth_ark_tx_queue_setup(struct rte_eth_dev *dev,
235 		       uint16_t queue_idx,
236 		       uint16_t nb_desc,
237 		       unsigned int socket_id,
238 		       const struct rte_eth_txconf *tx_conf __rte_unused)
239 {
240 	struct ark_adapter *ark = (struct ark_adapter *)dev->data->dev_private;
241 	struct ark_tx_queue *queue;
242 	int status;
243 
244 	/* Future: divide the Q's evenly with multi-ports */
245 	int port = dev->data->port_id;
246 	int qidx = port + queue_idx;
247 
248 	if (!rte_is_power_of_2(nb_desc)) {
249 		PMD_DRV_LOG(ERR,
250 			    "DPDK Arkville configuration queue size"
251 			    " must be power of two %u (%s)\n",
252 			    nb_desc, __func__);
253 		return -1;
254 	}
255 
256 	/* Allocate queue struct */
257 	queue =	rte_zmalloc_socket("Ark_txqueue",
258 				   sizeof(struct ark_tx_queue),
259 				   64,
260 				   socket_id);
261 	if (queue == 0) {
262 		PMD_DRV_LOG(ERR, "Failed to allocate tx "
263 			    "queue memory in %s\n",
264 			    __func__);
265 		return -ENOMEM;
266 	}
267 
268 	/* we use zmalloc no need to initialize fields */
269 	queue->queue_size = nb_desc;
270 	queue->queue_mask = nb_desc - 1;
271 	queue->phys_qid = qidx;
272 	queue->queue_index = queue_idx;
273 	dev->data->tx_queues[queue_idx] = queue;
274 
275 	queue->meta_q =
276 		rte_zmalloc_socket("Ark_txqueue meta",
277 				   nb_desc * sizeof(struct ark_tx_meta),
278 				   64,
279 				   socket_id);
280 	queue->bufs =
281 		rte_zmalloc_socket("Ark_txqueue bufs",
282 				   nb_desc * sizeof(struct rte_mbuf *),
283 				   64,
284 				   socket_id);
285 
286 	if (queue->meta_q == 0 || queue->bufs == 0) {
287 		PMD_DRV_LOG(ERR, "Failed to allocate "
288 			    "queue memory in %s\n", __func__);
289 		rte_free(queue->meta_q);
290 		rte_free(queue->bufs);
291 		rte_free(queue);
292 		return -ENOMEM;
293 	}
294 
295 	queue->ddm = RTE_PTR_ADD(ark->ddm.v, qidx * ARK_DDM_QOFFSET);
296 	queue->mpu = RTE_PTR_ADD(ark->mputx.v, qidx * ARK_MPU_QOFFSET);
297 
298 	status = eth_ark_tx_hw_queue_config(queue);
299 
300 	if (unlikely(status != 0)) {
301 		rte_free(queue->meta_q);
302 		rte_free(queue->bufs);
303 		rte_free(queue);
304 		return -1;		/* ERROR CODE */
305 	}
306 
307 	return 0;
308 }
309 
310 /* ************************************************************************* */
311 static int
312 eth_ark_tx_hw_queue_config(struct ark_tx_queue *queue)
313 {
314 	phys_addr_t queue_base, ring_base, cons_index_addr;
315 	uint32_t write_interval_ns;
316 
317 	/* Verify HW -- MPU */
318 	if (ark_mpu_verify(queue->mpu, sizeof(struct ark_tx_meta)))
319 		return -1;
320 
321 	queue_base = rte_malloc_virt2phy(queue);
322 	ring_base = rte_malloc_virt2phy(queue->meta_q);
323 	cons_index_addr =
324 		queue_base + offsetof(struct ark_tx_queue, cons_index);
325 
326 	ark_mpu_stop(queue->mpu);
327 	ark_mpu_reset(queue->mpu);
328 
329 	/* Stop and Reset and configure MPU */
330 	ark_mpu_configure(queue->mpu, ring_base, queue->queue_size, 1);
331 
332 	/*
333 	 * Adjust the write interval based on queue size --
334 	 * increase pcie traffic  when low mbuf count
335 	 * Queue sizes less than 128 are not allowed
336 	 */
337 	switch (queue->queue_size) {
338 	case 128:
339 		write_interval_ns = 500;
340 		break;
341 	case 256:
342 		write_interval_ns = 500;
343 		break;
344 	case 512:
345 		write_interval_ns = 1000;
346 		break;
347 	default:
348 		write_interval_ns = 2000;
349 		break;
350 	}
351 
352 	/* Completion address in UDM */
353 	ark_ddm_setup(queue->ddm, cons_index_addr, write_interval_ns);
354 
355 	return 0;
356 }
357 
358 /* ************************************************************************* */
359 void
360 eth_ark_tx_queue_release(void *vtx_queue)
361 {
362 	struct ark_tx_queue *queue;
363 
364 	queue = (struct ark_tx_queue *)vtx_queue;
365 
366 	ark_tx_hw_queue_stop(queue);
367 
368 	queue->cons_index = queue->prod_index;
369 	free_completed_tx(queue);
370 
371 	rte_free(queue->meta_q);
372 	rte_free(queue->bufs);
373 	rte_free(queue);
374 }
375 
376 /* ************************************************************************* */
377 int
378 eth_ark_tx_queue_stop(struct rte_eth_dev *dev, uint16_t queue_id)
379 {
380 	struct ark_tx_queue *queue;
381 	int cnt = 0;
382 
383 	queue = dev->data->tx_queues[queue_id];
384 
385 	/* Wait for DDM to send out all packets. */
386 	while (queue->cons_index != queue->prod_index) {
387 		usleep(100);
388 		if (cnt++ > 10000)
389 			return -1;
390 	}
391 
392 	ark_mpu_stop(queue->mpu);
393 	free_completed_tx(queue);
394 
395 	dev->data->tx_queue_state[queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
396 
397 	return 0;
398 }
399 
400 int
401 eth_ark_tx_queue_start(struct rte_eth_dev *dev, uint16_t queue_id)
402 {
403 	struct ark_tx_queue *queue;
404 
405 	queue = dev->data->tx_queues[queue_id];
406 	if (dev->data->tx_queue_state[queue_id] == RTE_ETH_QUEUE_STATE_STARTED)
407 		return 0;
408 
409 	ark_mpu_start(queue->mpu);
410 	dev->data->tx_queue_state[queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
411 
412 	return 0;
413 }
414 
415 /* ************************************************************************* */
416 static void
417 free_completed_tx(struct ark_tx_queue *queue)
418 {
419 	struct rte_mbuf *mbuf;
420 	struct ark_tx_meta *meta;
421 	uint32_t top_index;
422 
423 	top_index = queue->cons_index;	/* read once */
424 	while (queue->free_index != top_index) {
425 		meta = &queue->meta_q[queue->free_index & queue->queue_mask];
426 		mbuf = queue->bufs[queue->free_index & queue->queue_mask];
427 
428 		if (likely((meta->flags & ARK_DDM_SOP) != 0)) {
429 			/* ref count of the mbuf is checked in this call. */
430 			rte_pktmbuf_free(mbuf);
431 		}
432 		queue->free_index++;
433 	}
434 }
435 
436 /* ************************************************************************* */
437 void
438 eth_tx_queue_stats_get(void *vqueue, struct rte_eth_stats *stats)
439 {
440 	struct ark_tx_queue *queue;
441 	struct ark_ddm_t *ddm;
442 	uint64_t bytes, pkts;
443 
444 	queue = vqueue;
445 	ddm = queue->ddm;
446 
447 	bytes = ark_ddm_queue_byte_count(ddm);
448 	pkts = ark_ddm_queue_pkt_count(ddm);
449 
450 	stats->q_opackets[queue->queue_index] = pkts;
451 	stats->q_obytes[queue->queue_index] = bytes;
452 	stats->opackets += pkts;
453 	stats->obytes += bytes;
454 	stats->oerrors += queue->tx_errors;
455 }
456 
457 void
458 eth_tx_queue_stats_reset(void *vqueue)
459 {
460 	struct ark_tx_queue *queue;
461 	struct ark_ddm_t *ddm;
462 
463 	queue = vqueue;
464 	ddm = queue->ddm;
465 
466 	ark_ddm_queue_reset_stats(ddm);
467 	queue->tx_errors = 0;
468 }
469