xref: /dpdk/drivers/net/ark/ark_ethdev_tx.c (revision 3e0ceb9f17fff027fc6c8f18de35e11719ffa61e) 
1 /*-
2  * BSD LICENSE
3  *
4  * Copyright (c) 2015-2017 Atomic Rules LLC
5  * All rights reserved.
6  *
7  * Redistribution and use in source and binary forms, with or without
8  * modification, are permitted provided that the following conditions
9  * are met:
10  *
11  * * Redistributions of source code must retain the above copyright
12  * notice, this list of conditions and the following disclaimer.
13  * * Redistributions in binary form must reproduce the above copyright
14  * notice, this list of conditions and the following disclaimer in
15  * the documentation and/or other materials provided with the
16  * distribution.
17  * * Neither the name of copyright holder nor the names of its
18  * contributors may be used to endorse or promote products derived
19  * from this software without specific prior written permission.
20  *
21  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25  * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32  */
33 
34 #include <unistd.h>
35 
36 #include "ark_ethdev_tx.h"
37 #include "ark_global.h"
38 #include "ark_mpu.h"
39 #include "ark_ddm.h"
40 #include "ark_logs.h"
41 
42 #define ARK_TX_META_SIZE   32
43 #define ARK_TX_META_OFFSET (RTE_PKTMBUF_HEADROOM - ARK_TX_META_SIZE)
44 #define ARK_TX_MAX_NOCHAIN (RTE_MBUF_DEFAULT_DATAROOM)
45 
46 
47 /* ************************************************************************* */
48 struct ark_tx_queue {
49 	struct ark_tx_meta *meta_q;
50 	struct rte_mbuf **bufs;
51 
52 	/* handles for hw objects */
53 	struct ark_mpu_t *mpu;
54 	struct ark_ddm_t *ddm;
55 
56 	/* Stats HW tracks bytes and packets, need to count send errors */
57 	uint64_t tx_errors;
58 
59 	uint32_t queue_size;
60 	uint32_t queue_mask;
61 
62 	/* 3 indexes to the paired data rings. */
63 	uint32_t prod_index;		/* where to put the next one */
64 	uint32_t free_index;		/* mbuf has been freed */
65 
66 	/* The queue Id is used to identify the HW Q */
67 	uint16_t phys_qid;
68 	/* The queue Index within the dpdk device structures */
69 	uint16_t queue_index;
70 
71 	uint32_t pad[1];
72 
73 	/* second cache line - fields only used in slow path */
74 	MARKER cacheline1 __rte_cache_min_aligned;
75 	uint32_t cons_index;		/* hw is done, can be freed */
76 } __rte_cache_aligned;
77 
78 /* Forward declarations */
79 static uint32_t eth_ark_tx_jumbo(struct ark_tx_queue *queue,
80 				 struct rte_mbuf *mbuf);
81 static int eth_ark_tx_hw_queue_config(struct ark_tx_queue *queue);
82 static void free_completed_tx(struct ark_tx_queue *queue);
83 
84 static inline void
85 ark_tx_hw_queue_stop(struct ark_tx_queue *queue)
86 {
87 	ark_mpu_stop(queue->mpu);
88 }
89 
90 /* ************************************************************************* */
91 static inline void
92 eth_ark_tx_meta_from_mbuf(struct ark_tx_meta *meta,
93 			  const struct rte_mbuf *mbuf,
94 			  uint8_t flags)
95 {
96 	meta->physaddr = rte_mbuf_data_iova(mbuf);
97 	meta->delta_ns = 0;
98 	meta->data_len = rte_pktmbuf_data_len(mbuf);
99 	meta->flags = flags;
100 }
101 
102 /* ************************************************************************* */
103 uint16_t
104 eth_ark_xmit_pkts_noop(void *vtxq __rte_unused,
105 		       struct rte_mbuf **tx_pkts __rte_unused,
106 		       uint16_t nb_pkts __rte_unused)
107 {
108 	return 0;
109 }
110 
111 /* ************************************************************************* */
112 uint16_t
113 eth_ark_xmit_pkts(void *vtxq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
114 {
115 	struct ark_tx_queue *queue;
116 	struct rte_mbuf *mbuf;
117 	struct ark_tx_meta *meta;
118 
119 	uint32_t idx;
120 	uint32_t prod_index_limit;
121 	int stat;
122 	uint16_t nb;
123 
124 	queue = (struct ark_tx_queue *)vtxq;
125 
126 	/* free any packets after the HW is done with them */
127 	free_completed_tx(queue);
128 
129 	prod_index_limit = queue->queue_size + queue->free_index;
130 
131 	for (nb = 0;
132 	     (nb < nb_pkts) && (queue->prod_index != prod_index_limit);
133 	     ++nb) {
134 		mbuf = tx_pkts[nb];
135 
136 		if (ARK_TX_PAD_TO_60) {
137 			if (unlikely(rte_pktmbuf_pkt_len(mbuf) < 60)) {
138 				/* this packet even if it is small can be split,
139 				 * be sure to add to the end mbuf
140 				 */
141 				uint16_t to_add =
142 					60 - rte_pktmbuf_pkt_len(mbuf);
143 				char *appended =
144 					rte_pktmbuf_append(mbuf, to_add);
145 
146 				if (appended == 0) {
147 					/* This packet is in error,
148 					 * we cannot send it so just
149 					 * count it and delete it.
150 					 */
151 					queue->tx_errors += 1;
152 					rte_pktmbuf_free(mbuf);
153 					continue;
154 				}
155 				memset(appended, 0, to_add);
156 			}
157 		}
158 
159 		if (unlikely(mbuf->nb_segs != 1)) {
160 			stat = eth_ark_tx_jumbo(queue, mbuf);
161 			if (unlikely(stat != 0))
162 				break;		/* Queue is full */
163 		} else {
164 			idx = queue->prod_index & queue->queue_mask;
165 			queue->bufs[idx] = mbuf;
166 			meta = &queue->meta_q[idx];
167 			eth_ark_tx_meta_from_mbuf(meta,
168 						  mbuf,
169 						  ARK_DDM_SOP |
170 						  ARK_DDM_EOP);
171 			queue->prod_index++;
172 		}
173 	}
174 
175 	if (ARK_TX_DEBUG && (nb != nb_pkts)) {
176 		PMD_TX_LOG(DEBUG, "TX: Failure to send:"
177 			   " req: %" PRIU32
178 			   " sent: %" PRIU32
179 			   " prod: %" PRIU32
180 			   " cons: %" PRIU32
181 			   " free: %" PRIU32 "\n",
182 			   nb_pkts, nb,
183 			   queue->prod_index,
184 			   queue->cons_index,
185 			   queue->free_index);
186 		ark_mpu_dump(queue->mpu,
187 			     "TX Failure MPU: ",
188 			     queue->phys_qid);
189 	}
190 
191 	/* let FPGA know producer index.  */
192 	if (likely(nb != 0))
193 		ark_mpu_set_producer(queue->mpu, queue->prod_index);
194 
195 	return nb;
196 }
197 
198 /* ************************************************************************* */
199 static uint32_t
200 eth_ark_tx_jumbo(struct ark_tx_queue *queue, struct rte_mbuf *mbuf)
201 {
202 	struct rte_mbuf *next;
203 	struct ark_tx_meta *meta;
204 	uint32_t free_queue_space;
205 	uint32_t idx;
206 	uint8_t flags = ARK_DDM_SOP;
207 
208 	free_queue_space = queue->queue_mask -
209 		(queue->prod_index - queue->free_index);
210 	if (unlikely(free_queue_space < mbuf->nb_segs))
211 		return -1;
212 
213 	while (mbuf != NULL) {
214 		next = mbuf->next;
215 
216 		idx = queue->prod_index & queue->queue_mask;
217 		queue->bufs[idx] = mbuf;
218 		meta = &queue->meta_q[idx];
219 
220 		flags |= (next == NULL) ? ARK_DDM_EOP : 0;
221 		eth_ark_tx_meta_from_mbuf(meta, mbuf, flags);
222 		queue->prod_index++;
223 
224 		flags &= ~ARK_DDM_SOP;	/* drop SOP flags */
225 		mbuf = next;
226 	}
227 
228 	return 0;
229 }
230 
231 /* ************************************************************************* */
232 int
233 eth_ark_tx_queue_setup(struct rte_eth_dev *dev,
234 		       uint16_t queue_idx,
235 		       uint16_t nb_desc,
236 		       unsigned int socket_id,
237 		       const struct rte_eth_txconf *tx_conf __rte_unused)
238 {
239 	struct ark_adapter *ark = (struct ark_adapter *)dev->data->dev_private;
240 	struct ark_tx_queue *queue;
241 	int status;
242 
243 	/* Future: divide the Q's evenly with multi-ports */
244 	int port = dev->data->port_id;
245 	int qidx = port + queue_idx;
246 
247 	if (!rte_is_power_of_2(nb_desc)) {
248 		PMD_DRV_LOG(ERR,
249 			    "DPDK Arkville configuration queue size"
250 			    " must be power of two %u (%s)\n",
251 			    nb_desc, __func__);
252 		return -1;
253 	}
254 
255 	/* Allocate queue struct */
256 	queue =	rte_zmalloc_socket("Ark_txqueue",
257 				   sizeof(struct ark_tx_queue),
258 				   64,
259 				   socket_id);
260 	if (queue == 0) {
261 		PMD_DRV_LOG(ERR, "Failed to allocate tx "
262 			    "queue memory in %s\n",
263 			    __func__);
264 		return -ENOMEM;
265 	}
266 
267 	/* we use zmalloc no need to initialize fields */
268 	queue->queue_size = nb_desc;
269 	queue->queue_mask = nb_desc - 1;
270 	queue->phys_qid = qidx;
271 	queue->queue_index = queue_idx;
272 	dev->data->tx_queues[queue_idx] = queue;
273 
274 	queue->meta_q =
275 		rte_zmalloc_socket("Ark_txqueue meta",
276 				   nb_desc * sizeof(struct ark_tx_meta),
277 				   64,
278 				   socket_id);
279 	queue->bufs =
280 		rte_zmalloc_socket("Ark_txqueue bufs",
281 				   nb_desc * sizeof(struct rte_mbuf *),
282 				   64,
283 				   socket_id);
284 
285 	if (queue->meta_q == 0 || queue->bufs == 0) {
286 		PMD_DRV_LOG(ERR, "Failed to allocate "
287 			    "queue memory in %s\n", __func__);
288 		rte_free(queue->meta_q);
289 		rte_free(queue->bufs);
290 		rte_free(queue);
291 		return -ENOMEM;
292 	}
293 
294 	queue->ddm = RTE_PTR_ADD(ark->ddm.v, qidx * ARK_DDM_QOFFSET);
295 	queue->mpu = RTE_PTR_ADD(ark->mputx.v, qidx * ARK_MPU_QOFFSET);
296 
297 	status = eth_ark_tx_hw_queue_config(queue);
298 
299 	if (unlikely(status != 0)) {
300 		rte_free(queue->meta_q);
301 		rte_free(queue->bufs);
302 		rte_free(queue);
303 		return -1;		/* ERROR CODE */
304 	}
305 
306 	return 0;
307 }
308 
309 /* ************************************************************************* */
310 static int
311 eth_ark_tx_hw_queue_config(struct ark_tx_queue *queue)
312 {
313 	rte_iova_t queue_base, ring_base, cons_index_addr;
314 	uint32_t write_interval_ns;
315 
316 	/* Verify HW -- MPU */
317 	if (ark_mpu_verify(queue->mpu, sizeof(struct ark_tx_meta)))
318 		return -1;
319 
320 	queue_base = rte_malloc_virt2iova(queue);
321 	ring_base = rte_malloc_virt2iova(queue->meta_q);
322 	cons_index_addr =
323 		queue_base + offsetof(struct ark_tx_queue, cons_index);
324 
325 	ark_mpu_stop(queue->mpu);
326 	ark_mpu_reset(queue->mpu);
327 
328 	/* Stop and Reset and configure MPU */
329 	ark_mpu_configure(queue->mpu, ring_base, queue->queue_size, 1);
330 
331 	/*
332 	 * Adjust the write interval based on queue size --
333 	 * increase pcie traffic  when low mbuf count
334 	 * Queue sizes less than 128 are not allowed
335 	 */
336 	switch (queue->queue_size) {
337 	case 128:
338 		write_interval_ns = 500;
339 		break;
340 	case 256:
341 		write_interval_ns = 500;
342 		break;
343 	case 512:
344 		write_interval_ns = 1000;
345 		break;
346 	default:
347 		write_interval_ns = 2000;
348 		break;
349 	}
350 
351 	/* Completion address in UDM */
352 	ark_ddm_setup(queue->ddm, cons_index_addr, write_interval_ns);
353 
354 	return 0;
355 }
356 
357 /* ************************************************************************* */
358 void
359 eth_ark_tx_queue_release(void *vtx_queue)
360 {
361 	struct ark_tx_queue *queue;
362 
363 	queue = (struct ark_tx_queue *)vtx_queue;
364 
365 	ark_tx_hw_queue_stop(queue);
366 
367 	queue->cons_index = queue->prod_index;
368 	free_completed_tx(queue);
369 
370 	rte_free(queue->meta_q);
371 	rte_free(queue->bufs);
372 	rte_free(queue);
373 }
374 
375 /* ************************************************************************* */
376 int
377 eth_ark_tx_queue_stop(struct rte_eth_dev *dev, uint16_t queue_id)
378 {
379 	struct ark_tx_queue *queue;
380 	int cnt = 0;
381 
382 	queue = dev->data->tx_queues[queue_id];
383 
384 	/* Wait for DDM to send out all packets. */
385 	while (queue->cons_index != queue->prod_index) {
386 		usleep(100);
387 		if (cnt++ > 10000)
388 			return -1;
389 	}
390 
391 	ark_mpu_stop(queue->mpu);
392 	free_completed_tx(queue);
393 
394 	dev->data->tx_queue_state[queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
395 
396 	return 0;
397 }
398 
399 int
400 eth_ark_tx_queue_start(struct rte_eth_dev *dev, uint16_t queue_id)
401 {
402 	struct ark_tx_queue *queue;
403 
404 	queue = dev->data->tx_queues[queue_id];
405 	if (dev->data->tx_queue_state[queue_id] == RTE_ETH_QUEUE_STATE_STARTED)
406 		return 0;
407 
408 	ark_mpu_start(queue->mpu);
409 	dev->data->tx_queue_state[queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
410 
411 	return 0;
412 }
413 
414 /* ************************************************************************* */
415 static void
416 free_completed_tx(struct ark_tx_queue *queue)
417 {
418 	struct rte_mbuf *mbuf;
419 	struct ark_tx_meta *meta;
420 	uint32_t top_index;
421 
422 	top_index = queue->cons_index;	/* read once */
423 	while (queue->free_index != top_index) {
424 		meta = &queue->meta_q[queue->free_index & queue->queue_mask];
425 		mbuf = queue->bufs[queue->free_index & queue->queue_mask];
426 
427 		if (likely((meta->flags & ARK_DDM_SOP) != 0)) {
428 			/* ref count of the mbuf is checked in this call. */
429 			rte_pktmbuf_free(mbuf);
430 		}
431 		queue->free_index++;
432 	}
433 }
434 
435 /* ************************************************************************* */
436 void
437 eth_tx_queue_stats_get(void *vqueue, struct rte_eth_stats *stats)
438 {
439 	struct ark_tx_queue *queue;
440 	struct ark_ddm_t *ddm;
441 	uint64_t bytes, pkts;
442 
443 	queue = vqueue;
444 	ddm = queue->ddm;
445 
446 	bytes = ark_ddm_queue_byte_count(ddm);
447 	pkts = ark_ddm_queue_pkt_count(ddm);
448 
449 	stats->q_opackets[queue->queue_index] = pkts;
450 	stats->q_obytes[queue->queue_index] = bytes;
451 	stats->opackets += pkts;
452 	stats->obytes += bytes;
453 	stats->oerrors += queue->tx_errors;
454 }
455 
456 void
457 eth_tx_queue_stats_reset(void *vqueue)
458 {
459 	struct ark_tx_queue *queue;
460 	struct ark_ddm_t *ddm;
461 
462 	queue = vqueue;
463 	ddm = queue->ddm;
464 
465 	ark_ddm_queue_reset_stats(ddm);
466 	queue->tx_errors = 0;
467 }
468