23 static void ark_ethdev_rx_dump(const char *name, struct ark_rx_queue *queue);
24 static uint32_t eth_ark_rx_jumbo(struct ark_rx_queue *queue,
28 static inline int eth_ark_rx_seed_mbufs(struct ark_rx_queue *queue);
71 		    struct ark_rx_queue *queue,  in eth_ark_rx_hw_setup()  argument
78 	queue_base = rte_malloc_virt2iova(queue);  in eth_ark_rx_hw_setup()
82 	phys_addr_q_base = rte_malloc_virt2iova(queue->paddress_q);  in eth_ark_rx_hw_setup()
85 	if (ark_mpu_verify(queue->mpu, sizeof(rte_iova_t))) {  in eth_ark_rx_hw_setup()
91 	ark_mpu_configure(queue->mpu, phys_addr_q_base, queue->queue_size, 0);  in eth_ark_rx_hw_setup()
93 	ark_udm_write_addr(queue->udm, phys_addr_prod_index);  in eth_ark_rx_hw_setup()
96 	ark_mpu_set_producer(queue->mpu, queue->seed_index);  in eth_ark_rx_hw_setup()
103 eth_ark_rx_update_cons_index(struct ark_rx_queue *queue, uint32_t cons_index)  in eth_ark_rx_update_cons_index()  argument
105 	queue->cons_index = cons_index;  in eth_ark_rx_update_cons_index()
106 	if ((cons_index + queue->queue_size - queue->seed_index) >= ARK_RX_MPU_CHUNK) {  in eth_ark_rx_update_cons_index()
107 		eth_ark_rx_seed_mbufs(queue);  in eth_ark_rx_update_cons_index()
108 		ark_mpu_set_producer(queue->mpu, queue->seed_index);  in eth_ark_rx_update_cons_index()
124 	struct ark_rx_queue *queue;  in eth_ark_dev_rx_queue_setup()  local
157 	queue = rte_zmalloc_socket("Ark_rxqueue",  in eth_ark_dev_rx_queue_setup()
161 	if (queue == 0) {  in eth_ark_dev_rx_queue_setup()
167 	queue->mb_pool = mb_pool;  in eth_ark_dev_rx_queue_setup()
168 	queue->dataroom = rte_pktmbuf_data_room_size(mb_pool) -  in eth_ark_dev_rx_queue_setup()
175 			queue->dataroom = pool_priv->dataroom;  in eth_ark_dev_rx_queue_setup()
177 	queue->headroom = RTE_PKTMBUF_HEADROOM;  in eth_ark_dev_rx_queue_setup()
178 	queue->phys_qid = qidx;  in eth_ark_dev_rx_queue_setup()
179 	queue->queue_index = queue_idx;  in eth_ark_dev_rx_queue_setup()
180 	queue->queue_size = nb_desc;  in eth_ark_dev_rx_queue_setup()
181 	queue->queue_mask = nb_desc - 1;  in eth_ark_dev_rx_queue_setup()
182 	queue->rx_user_meta_hook = ark->user_ext.rx_user_meta_hook;  in eth_ark_dev_rx_queue_setup()
183 	queue->ext_user_data = ark->user_data[dev->data->port_id];  in eth_ark_dev_rx_queue_setup()
185 	queue->reserve_q =  in eth_ark_dev_rx_queue_setup()
190 	queue->paddress_q =  in eth_ark_dev_rx_queue_setup()
196 	if (queue->reserve_q == 0 || queue->paddress_q == 0) {  in eth_ark_dev_rx_queue_setup()
200 		rte_free(queue->reserve_q);  in eth_ark_dev_rx_queue_setup()
201 		rte_free(queue->paddress_q);  in eth_ark_dev_rx_queue_setup()
202 		rte_free(queue);  in eth_ark_dev_rx_queue_setup()
206 	dev->data->rx_queues[queue_idx] = queue;  in eth_ark_dev_rx_queue_setup()
207 	queue->udm = RTE_PTR_ADD(ark->udm.v, qidx * ARK_UDM_QOFFSET);  in eth_ark_dev_rx_queue_setup()
208 	queue->mpu = RTE_PTR_ADD(ark->mpurx.v, qidx * ARK_MPU_QOFFSET);  in eth_ark_dev_rx_queue_setup()
211 	ark_udm_configure(queue->udm,  in eth_ark_dev_rx_queue_setup()
213 			  queue->dataroom);  in eth_ark_dev_rx_queue_setup()
214 	ark_udm_queue_stats_reset(queue->udm);  in eth_ark_dev_rx_queue_setup()
217 	status = eth_ark_rx_seed_mbufs(queue);  in eth_ark_dev_rx_queue_setup()
219 	if (queue->seed_index != nb_desc) {  in eth_ark_dev_rx_queue_setup()
226 		status = eth_ark_rx_hw_setup(dev, queue, queue_idx);  in eth_ark_dev_rx_queue_setup()
235 		for (i = 0, mbuf = queue->reserve_q;  in eth_ark_dev_rx_queue_setup()
236 		     i < queue->seed_index; ++i, mbuf++) {  in eth_ark_dev_rx_queue_setup()
239 		rte_free(queue->reserve_q);  in eth_ark_dev_rx_queue_setup()
240 		rte_free(queue->paddress_q);  in eth_ark_dev_rx_queue_setup()
241 		rte_free(queue);  in eth_ark_dev_rx_queue_setup()
254 	struct ark_rx_queue *queue;  in eth_ark_recv_pkts()  local
263 	queue = (struct ark_rx_queue *)rx_queue;  in eth_ark_recv_pkts()
264 	if (unlikely(queue == 0))  in eth_ark_recv_pkts()
268 	prod_index = queue->prod_index;  in eth_ark_recv_pkts()
269 	cons_index = queue->cons_index;  in eth_ark_recv_pkts()
275 		mbuf = queue->reserve_q[cons_index & queue->queue_mask];  in eth_ark_recv_pkts()
295 					   queue->phys_qid,  in eth_ark_recv_pkts()
297 					   queue->prod_index,  in eth_ark_recv_pkts()
298 					   queue->seed_index);  in eth_ark_recv_pkts()
304 					   queue->udm->rt_cfg.prod_idx,  in eth_ark_recv_pkts()
306 				ark_mpu_dump(queue->mpu,  in eth_ark_recv_pkts()
308 					     queue->phys_qid);  in eth_ark_recv_pkts()
316 		if (unlikely(meta->pkt_len > queue->dataroom))  in eth_ark_recv_pkts()
318 				(queue, meta, mbuf, cons_index + 1);  in eth_ark_recv_pkts()
328 	rx_user_meta_hook = queue->rx_user_meta_hook;  in eth_ark_recv_pkts()
332 		rx_user_meta_hook(mbuf, meta->user_meta, queue->ext_user_data);  in eth_ark_recv_pkts()
335 	eth_ark_rx_update_cons_index(queue, cons_index);  in eth_ark_recv_pkts()
342 eth_ark_rx_jumbo(struct ark_rx_queue *queue,  in eth_ark_rx_jumbo()  argument
357 	data_len = RTE_MIN(meta->pkt_len, queue->dataroom);  in eth_ark_rx_jumbo()
364 				   queue->dataroom);  in eth_ark_rx_jumbo()
369 		mbuf = queue->reserve_q[cons_index & queue->queue_mask];  in eth_ark_rx_jumbo()
383 eth_ark_rx_queue_drain(struct ark_rx_queue *queue)  in eth_ark_rx_queue_drain()  argument
388 	cons_index = queue->cons_index;  in eth_ark_rx_queue_drain()
391 	while ((cons_index ^ queue->prod_index) & queue->queue_mask) {  in eth_ark_rx_queue_drain()
392 		mbuf = queue->reserve_q[cons_index & queue->queue_mask];  in eth_ark_rx_queue_drain()
395 		eth_ark_rx_update_cons_index(queue, cons_index);  in eth_ark_rx_queue_drain()
402 	struct ark_rx_queue *queue;  in eth_ark_dev_rx_queue_count()  local
404 	queue = rx_queue;  in eth_ark_dev_rx_queue_count()
405 	return (queue->prod_index - queue->cons_index);	/* mod arith */  in eth_ark_dev_rx_queue_count()
412 	struct ark_rx_queue *queue;  in eth_ark_rx_start_queue()  local
414 	queue = dev->data->rx_queues[queue_id];  in eth_ark_rx_start_queue()
415 	if (queue == 0)  in eth_ark_rx_start_queue()
420 	ark_mpu_set_producer(queue->mpu, queue->seed_index);  in eth_ark_rx_start_queue()
421 	ark_mpu_start(queue->mpu);  in eth_ark_rx_start_queue()
423 	ark_udm_queue_enable(queue->udm, 1);  in eth_ark_rx_start_queue()
435 	struct ark_rx_queue *queue;  in eth_ark_rx_stop_queue()  local
437 	queue = dev->data->rx_queues[queue_id];  in eth_ark_rx_stop_queue()
438 	if (queue == 0)  in eth_ark_rx_stop_queue()
441 	ark_udm_queue_enable(queue->udm, 0);  in eth_ark_rx_stop_queue()
450 eth_ark_rx_seed_mbufs(struct ark_rx_queue *queue)  in eth_ark_rx_seed_mbufs()  argument
452 	uint32_t limit = (queue->cons_index & ~(ARK_RX_MPU_CHUNK - 1)) +  in eth_ark_rx_seed_mbufs()
453 		queue->queue_size;  in eth_ark_rx_seed_mbufs()
454 	uint32_t seed_index = queue->seed_index;  in eth_ark_rx_seed_mbufs()
457 	uint32_t seed_m = queue->seed_index & queue->queue_mask;  in eth_ark_rx_seed_mbufs()
462 	if (unlikely(seed_m + nb > queue->queue_size))  in eth_ark_rx_seed_mbufs()
463 		nb = queue->queue_size - seed_m;  in eth_ark_rx_seed_mbufs()
465 	struct rte_mbuf **mbufs = &queue->reserve_q[seed_m];  in eth_ark_rx_seed_mbufs()
466 	int status = rte_pktmbuf_alloc_bulk(queue->mb_pool, mbufs, nb);  in eth_ark_rx_seed_mbufs()
473 			    nb, queue->queue_index,  in eth_ark_rx_seed_mbufs()
474 			    queue->seed_index - queue->cons_index);  in eth_ark_rx_seed_mbufs()
481 				queue->reserve_q[seed_m + count];  in eth_ark_rx_seed_mbufs()
487 				queue->phys_qid;  in eth_ark_rx_seed_mbufs()
492 	queue->seed_index += nb;  in eth_ark_rx_seed_mbufs()
498 			queue->paddress_q[seed_m++] =  in eth_ark_rx_seed_mbufs()
503 		queue->paddress_q[seed_m++] =  in eth_ark_rx_seed_mbufs()
508 		queue->paddress_q[seed_m++] =  in eth_ark_rx_seed_mbufs()
513 		queue->paddress_q[seed_m++] =  in eth_ark_rx_seed_mbufs()
528 	struct ark_rx_queue *queue;  in eth_ark_rx_dump_queue()  local
530 	queue = dev->data->rx_queues[queue_id];  in eth_ark_rx_dump_queue()
532 	ark_ethdev_rx_dump(msg, queue);  in eth_ark_rx_dump_queue()
540 	struct ark_rx_queue *queue;  in eth_ark_dev_rx_queue_release()  local
543 	queue = (struct ark_rx_queue *)vqueue;  in eth_ark_dev_rx_queue_release()
544 	if (queue == 0)  in eth_ark_dev_rx_queue_release()
547 	ark_udm_queue_enable(queue->udm, 0);  in eth_ark_dev_rx_queue_release()
549 	ark_mpu_stop(queue->mpu);  in eth_ark_dev_rx_queue_release()
552 	eth_ark_rx_queue_drain(queue);  in eth_ark_dev_rx_queue_release()
554 	for (i = 0; i < queue->queue_size; ++i)  in eth_ark_dev_rx_queue_release()
555 		rte_pktmbuf_free(queue->reserve_q[i]);  in eth_ark_dev_rx_queue_release()
557 	rte_free(queue->reserve_q);  in eth_ark_dev_rx_queue_release()
558 	rte_free(queue->paddress_q);  in eth_ark_dev_rx_queue_release()
559 	rte_free(queue);  in eth_ark_dev_rx_queue_release()
565 	struct ark_rx_queue *queue;  in eth_rx_queue_stats_get()  local
568 	queue = vqueue;  in eth_rx_queue_stats_get()
569 	if (queue == 0)  in eth_rx_queue_stats_get()
571 	udm = queue->udm;  in eth_rx_queue_stats_get()
575 	uint64_t idropped = ark_udm_dropped(queue->udm);  in eth_rx_queue_stats_get()
577 	stats->q_ipackets[queue->queue_index] = ipackets;  in eth_rx_queue_stats_get()
578 	stats->q_ibytes[queue->queue_index] = ibytes;  in eth_rx_queue_stats_get()
579 	stats->q_errors[queue->queue_index] = idropped;  in eth_rx_queue_stats_get()
588 	struct ark_rx_queue *queue;  in eth_rx_queue_stats_reset()  local
590 	queue = vqueue;  in eth_rx_queue_stats_reset()
591 	if (queue == 0)  in eth_rx_queue_stats_reset()
594 	ark_udm_queue_stats_reset(queue->udm);  in eth_rx_queue_stats_reset()
598 ark_ethdev_rx_dump(const char *name, struct ark_rx_queue *queue)  in ark_ethdev_rx_dump()  argument
600 	if (queue == NULL)  in ark_ethdev_rx_dump()
602 	ARK_PMD_LOG(DEBUG, "RX QUEUE %d -- %s", queue->phys_qid, name);  in ark_ethdev_rx_dump()
604 			"queue_size", queue->queue_size,  in ark_ethdev_rx_dump()
605 			"seed_index", queue->seed_index,  in ark_ethdev_rx_dump()
606 			"prod_index", queue->prod_index,  in ark_ethdev_rx_dump()
607 			"cons_index", queue->cons_index);  in ark_ethdev_rx_dump()
609 	ark_mpu_dump(queue->mpu, name, queue->phys_qid);  in ark_ethdev_rx_dump()
610 	ark_mpu_dump_setup(queue->mpu, queue->phys_qid);  in ark_ethdev_rx_dump()
611 	ark_udm_dump_setup(queue->udm, queue->phys_qid);  in ark_ethdev_rx_dump()