xref: /dpdk/drivers/net/mlx5/mlx5_flow_aso.c (revision 3cddeba0ca38b00c7dc646277484d08a4cb2d862)
129efa63aSLi Zhang /* SPDX-License-Identifier: BSD-3-Clause
229efa63aSLi Zhang  * Copyright 2020 Mellanox Technologies, Ltd
329efa63aSLi Zhang  */
429efa63aSLi Zhang #include <mlx5_prm.h>
529efa63aSLi Zhang #include <rte_malloc.h>
629efa63aSLi Zhang #include <rte_cycles.h>
729efa63aSLi Zhang #include <rte_eal_paging.h>
829efa63aSLi Zhang 
929efa63aSLi Zhang #include <mlx5_malloc.h>
1029efa63aSLi Zhang #include <mlx5_common_os.h>
1129efa63aSLi Zhang #include <mlx5_common_devx.h>
1229efa63aSLi Zhang 
1329efa63aSLi Zhang #include "mlx5.h"
1429efa63aSLi Zhang #include "mlx5_flow.h"
154d368e1dSXiaoyu Min #include "mlx5_hws_cnt.h"
164d368e1dSXiaoyu Min 
174d368e1dSXiaoyu Min #define MLX5_ASO_CNT_QUEUE_LOG_DESC 14
1829efa63aSLi Zhang 
1929efa63aSLi Zhang /**
2029efa63aSLi Zhang  * Free MR resources.
2129efa63aSLi Zhang  *
229f1d636fSMichael Baum  * @param[in] cdev
239f1d636fSMichael Baum  *   Pointer to the mlx5 common device.
2429efa63aSLi Zhang  * @param[in] mr
2529efa63aSLi Zhang  *   MR to free.
2629efa63aSLi Zhang  */
2729efa63aSLi Zhang static void
289f1d636fSMichael Baum mlx5_aso_dereg_mr(struct mlx5_common_device *cdev, struct mlx5_pmd_mr *mr)
2929efa63aSLi Zhang {
30cd414f81SMichael Baum 	void *addr = mr->addr;
31cd414f81SMichael Baum 
329f1d636fSMichael Baum 	cdev->mr_scache.dereg_mr_cb(mr);
33cd414f81SMichael Baum 	mlx5_free(addr);
3429efa63aSLi Zhang 	memset(mr, 0, sizeof(*mr));
3529efa63aSLi Zhang }
3629efa63aSLi Zhang 
3729efa63aSLi Zhang /**
3829efa63aSLi Zhang  * Register Memory Region.
3929efa63aSLi Zhang  *
409f1d636fSMichael Baum  * @param[in] cdev
419f1d636fSMichael Baum  *   Pointer to the mlx5 common device.
4229efa63aSLi Zhang  * @param[in] length
4329efa63aSLi Zhang  *   Size of MR buffer.
4429efa63aSLi Zhang  * @param[in/out] mr
4529efa63aSLi Zhang  *   Pointer to MR to create.
4629efa63aSLi Zhang  *
4729efa63aSLi Zhang  * @return
4829efa63aSLi Zhang  *   0 on success, a negative errno value otherwise and rte_errno is set.
4929efa63aSLi Zhang  */
5029efa63aSLi Zhang static int
519f1d636fSMichael Baum mlx5_aso_reg_mr(struct mlx5_common_device *cdev, size_t length,
52147f6fb4SMichael Baum 		struct mlx5_pmd_mr *mr)
5329efa63aSLi Zhang {
54cd414f81SMichael Baum 	int ret;
55cd414f81SMichael Baum 
56cd414f81SMichael Baum 	mr->addr = mlx5_malloc(MLX5_MEM_RTE | MLX5_MEM_ZERO, length, 4096,
57147f6fb4SMichael Baum 			       SOCKET_ID_ANY);
58cd414f81SMichael Baum 	if (!mr->addr) {
59cd414f81SMichael Baum 		DRV_LOG(ERR, "Failed to create ASO bits mem for MR.");
6029efa63aSLi Zhang 		return -1;
6129efa63aSLi Zhang 	}
629f1d636fSMichael Baum 	ret = cdev->mr_scache.reg_mr_cb(cdev->pd, mr->addr, length, mr);
63cd414f81SMichael Baum 	if (ret) {
6429efa63aSLi Zhang 		DRV_LOG(ERR, "Failed to create direct Mkey.");
65cd414f81SMichael Baum 		mlx5_free(mr->addr);
6629efa63aSLi Zhang 		return -1;
6729efa63aSLi Zhang 	}
68cd414f81SMichael Baum 	return 0;
69cd414f81SMichael Baum }
7029efa63aSLi Zhang 
7129efa63aSLi Zhang /**
7229efa63aSLi Zhang  * Destroy Send Queue used for ASO access.
7329efa63aSLi Zhang  *
7429efa63aSLi Zhang  * @param[in] sq
7529efa63aSLi Zhang  *   ASO SQ to destroy.
7629efa63aSLi Zhang  */
7715896eafSGregory Etelson void
7829efa63aSLi Zhang mlx5_aso_destroy_sq(struct mlx5_aso_sq *sq)
7929efa63aSLi Zhang {
8029efa63aSLi Zhang 	mlx5_devx_sq_destroy(&sq->sq_obj);
81147f6fb4SMichael Baum 	mlx5_devx_cq_destroy(&sq->cq.cq_obj);
8229efa63aSLi Zhang 	memset(sq, 0, sizeof(*sq));
8329efa63aSLi Zhang }
8429efa63aSLi Zhang 
8529efa63aSLi Zhang /**
864d368e1dSXiaoyu Min  * Initialize Send Queue used for ASO access counter.
874d368e1dSXiaoyu Min  *
884d368e1dSXiaoyu Min  * @param[in] sq
894d368e1dSXiaoyu Min  *   ASO SQ to initialize.
904d368e1dSXiaoyu Min  */
914d368e1dSXiaoyu Min static void
924d368e1dSXiaoyu Min mlx5_aso_cnt_init_sq(struct mlx5_aso_sq *sq)
934d368e1dSXiaoyu Min {
944d368e1dSXiaoyu Min 	volatile struct mlx5_aso_wqe *restrict wqe;
954d368e1dSXiaoyu Min 	int i;
964d368e1dSXiaoyu Min 	int size = 1 << sq->log_desc_n;
974d368e1dSXiaoyu Min 
984d368e1dSXiaoyu Min 	/* All the next fields state should stay constant. */
994d368e1dSXiaoyu Min 	for (i = 0, wqe = &sq->sq_obj.aso_wqes[0]; i < size; ++i, ++wqe) {
1004d368e1dSXiaoyu Min 		wqe->general_cseg.sq_ds = rte_cpu_to_be_32((sq->sqn << 8) |
1014d368e1dSXiaoyu Min 							  (sizeof(*wqe) >> 4));
1024d368e1dSXiaoyu Min 		wqe->aso_cseg.operand_masks = rte_cpu_to_be_32
1034d368e1dSXiaoyu Min 			(0u |
1044d368e1dSXiaoyu Min 			 (ASO_OPER_LOGICAL_OR << ASO_CSEG_COND_OPER_OFFSET) |
1054d368e1dSXiaoyu Min 			 (ASO_OP_ALWAYS_FALSE << ASO_CSEG_COND_1_OPER_OFFSET) |
1064d368e1dSXiaoyu Min 			 (ASO_OP_ALWAYS_FALSE << ASO_CSEG_COND_0_OPER_OFFSET) |
1074d368e1dSXiaoyu Min 			 (BYTEWISE_64BYTE << ASO_CSEG_DATA_MASK_MODE_OFFSET));
1084d368e1dSXiaoyu Min 		wqe->aso_cseg.data_mask = RTE_BE64(UINT64_MAX);
1094d368e1dSXiaoyu Min 	}
1104d368e1dSXiaoyu Min }
1114d368e1dSXiaoyu Min 
1124d368e1dSXiaoyu Min /**
11329efa63aSLi Zhang  * Initialize Send Queue used for ASO access.
11429efa63aSLi Zhang  *
11529efa63aSLi Zhang  * @param[in] sq
11629efa63aSLi Zhang  *   ASO SQ to initialize.
11729efa63aSLi Zhang  */
11829efa63aSLi Zhang static void
11929efa63aSLi Zhang mlx5_aso_age_init_sq(struct mlx5_aso_sq *sq)
12029efa63aSLi Zhang {
12129efa63aSLi Zhang 	volatile struct mlx5_aso_wqe *restrict wqe;
12229efa63aSLi Zhang 	int i;
12329efa63aSLi Zhang 	int size = 1 << sq->log_desc_n;
12429efa63aSLi Zhang 	uint64_t addr;
12529efa63aSLi Zhang 
12629efa63aSLi Zhang 	/* All the next fields state should stay constant. */
12729efa63aSLi Zhang 	for (i = 0, wqe = &sq->sq_obj.aso_wqes[0]; i < size; ++i, ++wqe) {
12829efa63aSLi Zhang 		wqe->general_cseg.sq_ds = rte_cpu_to_be_32((sq->sqn << 8) |
12929efa63aSLi Zhang 							  (sizeof(*wqe) >> 4));
130cd414f81SMichael Baum 		wqe->aso_cseg.lkey = rte_cpu_to_be_32(sq->mr.lkey);
131cd414f81SMichael Baum 		addr = (uint64_t)((uint64_t *)sq->mr.addr + i *
13229efa63aSLi Zhang 					    MLX5_ASO_AGE_ACTIONS_PER_POOL / 64);
13329efa63aSLi Zhang 		wqe->aso_cseg.va_h = rte_cpu_to_be_32((uint32_t)(addr >> 32));
13429efa63aSLi Zhang 		wqe->aso_cseg.va_l_r = rte_cpu_to_be_32((uint32_t)addr | 1u);
13529efa63aSLi Zhang 		wqe->aso_cseg.operand_masks = rte_cpu_to_be_32
13629efa63aSLi Zhang 			(0u |
13729efa63aSLi Zhang 			 (ASO_OPER_LOGICAL_OR << ASO_CSEG_COND_OPER_OFFSET) |
13829efa63aSLi Zhang 			 (ASO_OP_ALWAYS_TRUE << ASO_CSEG_COND_1_OPER_OFFSET) |
13929efa63aSLi Zhang 			 (ASO_OP_ALWAYS_TRUE << ASO_CSEG_COND_0_OPER_OFFSET) |
14029efa63aSLi Zhang 			 (BYTEWISE_64BYTE << ASO_CSEG_DATA_MASK_MODE_OFFSET));
14129efa63aSLi Zhang 		wqe->aso_cseg.data_mask = RTE_BE64(UINT64_MAX);
14229efa63aSLi Zhang 	}
14329efa63aSLi Zhang }
14429efa63aSLi Zhang 
14529efa63aSLi Zhang /**
14629efa63aSLi Zhang  * Initialize Send Queue used for ASO flow meter access.
14729efa63aSLi Zhang  *
14829efa63aSLi Zhang  * @param[in] sq
14929efa63aSLi Zhang  *   ASO SQ to initialize.
15029efa63aSLi Zhang  */
15115896eafSGregory Etelson void
15229efa63aSLi Zhang mlx5_aso_mtr_init_sq(struct mlx5_aso_sq *sq)
15329efa63aSLi Zhang {
15429efa63aSLi Zhang 	volatile struct mlx5_aso_wqe *restrict wqe;
15529efa63aSLi Zhang 	int i;
15629efa63aSLi Zhang 	int size = 1 << sq->log_desc_n;
15729efa63aSLi Zhang 
15829efa63aSLi Zhang 	/* All the next fields state should stay constant. */
15929efa63aSLi Zhang 	for (i = 0, wqe = &sq->sq_obj.aso_wqes[0]; i < size; ++i, ++wqe) {
16029efa63aSLi Zhang 		wqe->general_cseg.sq_ds = rte_cpu_to_be_32((sq->sqn << 8) |
16129efa63aSLi Zhang 							  (sizeof(*wqe) >> 4));
16229efa63aSLi Zhang 		wqe->aso_cseg.operand_masks = RTE_BE32(0u |
16329efa63aSLi Zhang 			 (ASO_OPER_LOGICAL_OR << ASO_CSEG_COND_OPER_OFFSET) |
16429efa63aSLi Zhang 			 (ASO_OP_ALWAYS_TRUE << ASO_CSEG_COND_1_OPER_OFFSET) |
16529efa63aSLi Zhang 			 (ASO_OP_ALWAYS_TRUE << ASO_CSEG_COND_0_OPER_OFFSET) |
16629efa63aSLi Zhang 			 (BYTEWISE_64BYTE << ASO_CSEG_DATA_MASK_MODE_OFFSET));
16729efa63aSLi Zhang 		wqe->general_cseg.flags = RTE_BE32(MLX5_COMP_ALWAYS <<
16829efa63aSLi Zhang 							 MLX5_COMP_MODE_OFFSET);
16929efa63aSLi Zhang 	}
17029efa63aSLi Zhang }
17129efa63aSLi Zhang 
172ee9e5fadSBing Zhao /*
173ee9e5fadSBing Zhao  * Initialize Send Queue used for ASO connection tracking.
174ee9e5fadSBing Zhao  *
175ee9e5fadSBing Zhao  * @param[in] sq
176ee9e5fadSBing Zhao  *   ASO SQ to initialize.
177ee9e5fadSBing Zhao  */
178ee9e5fadSBing Zhao static void
179ee9e5fadSBing Zhao mlx5_aso_ct_init_sq(struct mlx5_aso_sq *sq)
180ee9e5fadSBing Zhao {
181ee9e5fadSBing Zhao 	volatile struct mlx5_aso_wqe *restrict wqe;
182ee9e5fadSBing Zhao 	int i;
183ee9e5fadSBing Zhao 	int size = 1 << sq->log_desc_n;
184ee9e5fadSBing Zhao 	uint64_t addr;
185ee9e5fadSBing Zhao 
186ee9e5fadSBing Zhao 	/* All the next fields state should stay constant. */
187ee9e5fadSBing Zhao 	for (i = 0, wqe = &sq->sq_obj.aso_wqes[0]; i < size; ++i, ++wqe) {
188ee9e5fadSBing Zhao 		wqe->general_cseg.sq_ds = rte_cpu_to_be_32((sq->sqn << 8) |
189ee9e5fadSBing Zhao 							  (sizeof(*wqe) >> 4));
190ee9e5fadSBing Zhao 		/* One unique MR for the query data. */
191ee9e5fadSBing Zhao 		wqe->aso_cseg.lkey = rte_cpu_to_be_32(sq->mr.lkey);
192ee9e5fadSBing Zhao 		/* Magic number 64 represents the length of a ASO CT obj. */
193ee9e5fadSBing Zhao 		addr = (uint64_t)((uintptr_t)sq->mr.addr + i * 64);
194ee9e5fadSBing Zhao 		wqe->aso_cseg.va_h = rte_cpu_to_be_32((uint32_t)(addr >> 32));
195ee9e5fadSBing Zhao 		wqe->aso_cseg.va_l_r = rte_cpu_to_be_32((uint32_t)addr | 1u);
196ee9e5fadSBing Zhao 		/*
197ee9e5fadSBing Zhao 		 * The values of operand_masks are different for modify
198ee9e5fadSBing Zhao 		 * and query.
199ee9e5fadSBing Zhao 		 * And data_mask may be different for each modification. In
200ee9e5fadSBing Zhao 		 * query, it could be zero and ignored.
201ee9e5fadSBing Zhao 		 * CQE generation is always needed, in order to decide when
202ee9e5fadSBing Zhao 		 * it is available to create the flow or read the data.
203ee9e5fadSBing Zhao 		 */
204ee9e5fadSBing Zhao 		wqe->general_cseg.flags = RTE_BE32(MLX5_COMP_ALWAYS <<
205ee9e5fadSBing Zhao 						   MLX5_COMP_MODE_OFFSET);
206ee9e5fadSBing Zhao 	}
207ee9e5fadSBing Zhao }
208ee9e5fadSBing Zhao 
20929efa63aSLi Zhang /**
21029efa63aSLi Zhang  * Create Send Queue used for ASO access.
21129efa63aSLi Zhang  *
212147f6fb4SMichael Baum  * @param[in] cdev
213147f6fb4SMichael Baum  *   Pointer to the mlx5 common device.
21429efa63aSLi Zhang  * @param[in/out] sq
21529efa63aSLi Zhang  *   Pointer to SQ to create.
21629efa63aSLi Zhang  * @param[in] uar
21729efa63aSLi Zhang  *   User Access Region object.
21829efa63aSLi Zhang  *
21929efa63aSLi Zhang  * @return
22029efa63aSLi Zhang  *   0 on success, a negative errno value otherwise and rte_errno is set.
22129efa63aSLi Zhang  */
22215896eafSGregory Etelson int
223147f6fb4SMichael Baum mlx5_aso_sq_create(struct mlx5_common_device *cdev, struct mlx5_aso_sq *sq,
2244d368e1dSXiaoyu Min 		   void *uar, uint16_t log_desc_n)
22529efa63aSLi Zhang {
226147f6fb4SMichael Baum 	struct mlx5_devx_cq_attr cq_attr = {
227147f6fb4SMichael Baum 		.uar_page_id = mlx5_os_get_devx_uar_page_id(uar),
228147f6fb4SMichael Baum 	};
229147f6fb4SMichael Baum 	struct mlx5_devx_create_sq_attr sq_attr = {
23029efa63aSLi Zhang 		.user_index = 0xFFFF,
23129efa63aSLi Zhang 		.wq_attr = (struct mlx5_devx_wq_attr){
232147f6fb4SMichael Baum 			.pd = cdev->pdn,
23329efa63aSLi Zhang 			.uar_page = mlx5_os_get_devx_uar_page_id(uar),
23429efa63aSLi Zhang 		},
235147f6fb4SMichael Baum 		.ts_format =
236147f6fb4SMichael Baum 			mlx5_ts_format_conv(cdev->config.hca_attr.sq_ts_format),
23729efa63aSLi Zhang 	};
23829efa63aSLi Zhang 	struct mlx5_devx_modify_sq_attr modify_attr = {
23929efa63aSLi Zhang 		.state = MLX5_SQC_STATE_RDY,
24029efa63aSLi Zhang 	};
24129efa63aSLi Zhang 	uint16_t log_wqbb_n;
24229efa63aSLi Zhang 	int ret;
24329efa63aSLi Zhang 
244147f6fb4SMichael Baum 	if (mlx5_devx_cq_create(cdev->ctx, &sq->cq.cq_obj,
2454d368e1dSXiaoyu Min 				log_desc_n, &cq_attr,
246147f6fb4SMichael Baum 				SOCKET_ID_ANY))
24729efa63aSLi Zhang 		goto error;
248147f6fb4SMichael Baum 	sq->cq.cq_ci = 0;
2494d368e1dSXiaoyu Min 	sq->cq.log_desc_n = log_desc_n;
2504d368e1dSXiaoyu Min 	sq->log_desc_n = log_desc_n;
251147f6fb4SMichael Baum 	sq_attr.cqn = sq->cq.cq_obj.cq->id;
25229efa63aSLi Zhang 	/* for mlx5_aso_wqe that is twice the size of mlx5_wqe */
253147f6fb4SMichael Baum 	log_wqbb_n = sq->log_desc_n + 1;
254147f6fb4SMichael Baum 	ret = mlx5_devx_sq_create(cdev->ctx, &sq->sq_obj, log_wqbb_n, &sq_attr,
255147f6fb4SMichael Baum 				  SOCKET_ID_ANY);
25629efa63aSLi Zhang 	if (ret) {
25729efa63aSLi Zhang 		DRV_LOG(ERR, "Can't create SQ object.");
25829efa63aSLi Zhang 		rte_errno = ENOMEM;
25929efa63aSLi Zhang 		goto error;
26029efa63aSLi Zhang 	}
26129efa63aSLi Zhang 	ret = mlx5_devx_cmd_modify_sq(sq->sq_obj.sq, &modify_attr);
26229efa63aSLi Zhang 	if (ret) {
26329efa63aSLi Zhang 		DRV_LOG(ERR, "Can't change SQ state to ready.");
26429efa63aSLi Zhang 		rte_errno = ENOMEM;
26529efa63aSLi Zhang 		goto error;
26629efa63aSLi Zhang 	}
26729efa63aSLi Zhang 	sq->pi = 0;
26829efa63aSLi Zhang 	sq->head = 0;
26929efa63aSLi Zhang 	sq->tail = 0;
27029efa63aSLi Zhang 	sq->sqn = sq->sq_obj.sq->id;
271cfd2037cSLi Zhang 	rte_spinlock_init(&sq->sqsl);
27229efa63aSLi Zhang 	return 0;
27329efa63aSLi Zhang error:
27429efa63aSLi Zhang 	mlx5_aso_destroy_sq(sq);
27529efa63aSLi Zhang 	return -1;
27629efa63aSLi Zhang }
27729efa63aSLi Zhang 
27848fbb0e9SAlexander Kozyrev void
27948fbb0e9SAlexander Kozyrev mlx5_aso_mtr_queue_uninit(struct mlx5_dev_ctx_shared *sh __rte_unused,
28048fbb0e9SAlexander Kozyrev 			  struct mlx5_aso_mtr_pool *hws_pool,
28148fbb0e9SAlexander Kozyrev 			  struct mlx5_aso_mtr_pools_mng *pool_mng)
28248fbb0e9SAlexander Kozyrev {
28348fbb0e9SAlexander Kozyrev 	uint32_t i;
28448fbb0e9SAlexander Kozyrev 
28548fbb0e9SAlexander Kozyrev 	if (hws_pool) {
28648fbb0e9SAlexander Kozyrev 		for (i = 0; i < hws_pool->nb_sq; i++)
28748fbb0e9SAlexander Kozyrev 			mlx5_aso_destroy_sq(hws_pool->sq + i);
28848fbb0e9SAlexander Kozyrev 		mlx5_free(hws_pool->sq);
28948fbb0e9SAlexander Kozyrev 		return;
29048fbb0e9SAlexander Kozyrev 	}
29148fbb0e9SAlexander Kozyrev 	if (pool_mng)
29248fbb0e9SAlexander Kozyrev 		mlx5_aso_destroy_sq(&pool_mng->sq);
29348fbb0e9SAlexander Kozyrev }
29448fbb0e9SAlexander Kozyrev 
29548fbb0e9SAlexander Kozyrev int
29648fbb0e9SAlexander Kozyrev mlx5_aso_mtr_queue_init(struct mlx5_dev_ctx_shared *sh,
29748fbb0e9SAlexander Kozyrev 				struct mlx5_aso_mtr_pool *hws_pool,
29848fbb0e9SAlexander Kozyrev 				struct mlx5_aso_mtr_pools_mng *pool_mng,
29948fbb0e9SAlexander Kozyrev 				uint32_t nb_queues)
30048fbb0e9SAlexander Kozyrev {
30148fbb0e9SAlexander Kozyrev 	struct mlx5_common_device *cdev = sh->cdev;
30248fbb0e9SAlexander Kozyrev 	struct mlx5_aso_sq *sq;
30348fbb0e9SAlexander Kozyrev 	uint32_t i;
30448fbb0e9SAlexander Kozyrev 
30548fbb0e9SAlexander Kozyrev 	if (hws_pool) {
30648fbb0e9SAlexander Kozyrev 		sq = mlx5_malloc(MLX5_MEM_ZERO,
30748fbb0e9SAlexander Kozyrev 			sizeof(struct mlx5_aso_sq) * nb_queues,
30848fbb0e9SAlexander Kozyrev 			RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
30948fbb0e9SAlexander Kozyrev 		if (!sq)
31048fbb0e9SAlexander Kozyrev 			return -1;
31148fbb0e9SAlexander Kozyrev 		hws_pool->sq = sq;
31248fbb0e9SAlexander Kozyrev 		for (i = 0; i < nb_queues; i++) {
31348fbb0e9SAlexander Kozyrev 			if (mlx5_aso_sq_create(cdev, hws_pool->sq + i,
31448fbb0e9SAlexander Kozyrev 					       sh->tx_uar.obj,
31548fbb0e9SAlexander Kozyrev 					       MLX5_ASO_QUEUE_LOG_DESC))
31648fbb0e9SAlexander Kozyrev 				goto error;
31748fbb0e9SAlexander Kozyrev 			mlx5_aso_mtr_init_sq(hws_pool->sq + i);
31848fbb0e9SAlexander Kozyrev 		}
31948fbb0e9SAlexander Kozyrev 		hws_pool->nb_sq = nb_queues;
32048fbb0e9SAlexander Kozyrev 	}
32148fbb0e9SAlexander Kozyrev 	if (pool_mng) {
32248fbb0e9SAlexander Kozyrev 		if (mlx5_aso_sq_create(cdev, &pool_mng->sq,
32348fbb0e9SAlexander Kozyrev 				       sh->tx_uar.obj,
32448fbb0e9SAlexander Kozyrev 				       MLX5_ASO_QUEUE_LOG_DESC))
32548fbb0e9SAlexander Kozyrev 			return -1;
32648fbb0e9SAlexander Kozyrev 		mlx5_aso_mtr_init_sq(&pool_mng->sq);
32748fbb0e9SAlexander Kozyrev 	}
32848fbb0e9SAlexander Kozyrev 	return 0;
32948fbb0e9SAlexander Kozyrev error:
33048fbb0e9SAlexander Kozyrev 	do {
33148fbb0e9SAlexander Kozyrev 		mlx5_aso_destroy_sq(hws_pool->sq + i);
33248fbb0e9SAlexander Kozyrev 	} while (i--);
33348fbb0e9SAlexander Kozyrev 	return -1;
33448fbb0e9SAlexander Kozyrev }
33548fbb0e9SAlexander Kozyrev 
33629efa63aSLi Zhang /**
33729efa63aSLi Zhang  * API to create and initialize Send Queue used for ASO access.
33829efa63aSLi Zhang  *
33929efa63aSLi Zhang  * @param[in] sh
34029efa63aSLi Zhang  *   Pointer to shared device context.
341cd414f81SMichael Baum  * @param[in] aso_opc_mod
342cd414f81SMichael Baum  *   Mode of ASO feature.
34348fbb0e9SAlexander Kozyrev  * @param[in] nb_queues
34448fbb0e9SAlexander Kozyrev  *   Number of Send Queues to create.
34529efa63aSLi Zhang  *
34629efa63aSLi Zhang  * @return
34729efa63aSLi Zhang  *   0 on success, a negative errno value otherwise and rte_errno is set.
34829efa63aSLi Zhang  */
34929efa63aSLi Zhang int
35029efa63aSLi Zhang mlx5_aso_queue_init(struct mlx5_dev_ctx_shared *sh,
35148fbb0e9SAlexander Kozyrev 		    enum mlx5_access_aso_opc_mod aso_opc_mod,
35248fbb0e9SAlexander Kozyrev 			uint32_t nb_queues)
35329efa63aSLi Zhang {
35429efa63aSLi Zhang 	uint32_t sq_desc_n = 1 << MLX5_ASO_QUEUE_LOG_DESC;
355ca1418ceSMichael Baum 	struct mlx5_common_device *cdev = sh->cdev;
35629efa63aSLi Zhang 
35729efa63aSLi Zhang 	switch (aso_opc_mod) {
35829efa63aSLi Zhang 	case ASO_OPC_MOD_FLOW_HIT:
3599f1d636fSMichael Baum 		if (mlx5_aso_reg_mr(cdev, (MLX5_ASO_AGE_ACTIONS_PER_POOL / 8) *
360147f6fb4SMichael Baum 				    sq_desc_n, &sh->aso_age_mng->aso_sq.mr))
36129efa63aSLi Zhang 			return -1;
362147f6fb4SMichael Baum 		if (mlx5_aso_sq_create(cdev, &sh->aso_age_mng->aso_sq,
3634d368e1dSXiaoyu Min 				       sh->tx_uar.obj,
3644d368e1dSXiaoyu Min 				       MLX5_ASO_QUEUE_LOG_DESC)) {
3659f1d636fSMichael Baum 			mlx5_aso_dereg_mr(cdev, &sh->aso_age_mng->aso_sq.mr);
36629efa63aSLi Zhang 			return -1;
36729efa63aSLi Zhang 		}
36829efa63aSLi Zhang 		mlx5_aso_age_init_sq(&sh->aso_age_mng->aso_sq);
36929efa63aSLi Zhang 		break;
37029efa63aSLi Zhang 	case ASO_OPC_MOD_POLICER:
37148fbb0e9SAlexander Kozyrev 		if (mlx5_aso_mtr_queue_init(sh, NULL,
37248fbb0e9SAlexander Kozyrev 					    &sh->mtrmng->pools_mng, nb_queues))
37329efa63aSLi Zhang 			return -1;
37429efa63aSLi Zhang 		break;
375ee9e5fadSBing Zhao 	case ASO_OPC_MOD_CONNECTION_TRACKING:
376463170a7SSuanming Mou 		if (mlx5_aso_ct_queue_init(sh, sh->ct_mng, MLX5_ASO_CT_SQ_NUM))
377ee9e5fadSBing Zhao 			return -1;
378ee9e5fadSBing Zhao 		break;
37929efa63aSLi Zhang 	default:
38029efa63aSLi Zhang 		DRV_LOG(ERR, "Unknown ASO operation mode");
38129efa63aSLi Zhang 		return -1;
38229efa63aSLi Zhang 	}
38329efa63aSLi Zhang 	return 0;
38429efa63aSLi Zhang }
38529efa63aSLi Zhang 
38629efa63aSLi Zhang /**
38729efa63aSLi Zhang  * API to destroy Send Queue used for ASO access.
38829efa63aSLi Zhang  *
38929efa63aSLi Zhang  * @param[in] sh
39029efa63aSLi Zhang  *   Pointer to shared device context.
391cd414f81SMichael Baum  * @param[in] aso_opc_mod
392cd414f81SMichael Baum  *   Mode of ASO feature.
39329efa63aSLi Zhang  */
39429efa63aSLi Zhang void
39529efa63aSLi Zhang mlx5_aso_queue_uninit(struct mlx5_dev_ctx_shared *sh,
39629efa63aSLi Zhang 		      enum mlx5_access_aso_opc_mod aso_opc_mod)
39729efa63aSLi Zhang {
398463170a7SSuanming Mou 	struct mlx5_aso_sq *sq = NULL;
39929efa63aSLi Zhang 
40029efa63aSLi Zhang 	switch (aso_opc_mod) {
40129efa63aSLi Zhang 	case ASO_OPC_MOD_FLOW_HIT:
4029f1d636fSMichael Baum 		mlx5_aso_dereg_mr(sh->cdev, &sh->aso_age_mng->aso_sq.mr);
40329efa63aSLi Zhang 		sq = &sh->aso_age_mng->aso_sq;
40429efa63aSLi Zhang 		break;
40529efa63aSLi Zhang 	case ASO_OPC_MOD_POLICER:
40648fbb0e9SAlexander Kozyrev 		mlx5_aso_mtr_queue_uninit(sh, NULL, &sh->mtrmng->pools_mng);
40729efa63aSLi Zhang 		break;
4080af8a229SBing Zhao 	case ASO_OPC_MOD_CONNECTION_TRACKING:
409463170a7SSuanming Mou 		mlx5_aso_ct_queue_uninit(sh, sh->ct_mng);
4100af8a229SBing Zhao 		break;
41129efa63aSLi Zhang 	default:
41229efa63aSLi Zhang 		DRV_LOG(ERR, "Unknown ASO operation mode");
41329efa63aSLi Zhang 		return;
41429efa63aSLi Zhang 	}
415463170a7SSuanming Mou 	if (sq)
41629efa63aSLi Zhang 		mlx5_aso_destroy_sq(sq);
41729efa63aSLi Zhang }
41829efa63aSLi Zhang 
41929efa63aSLi Zhang /**
42029efa63aSLi Zhang  * Write a burst of WQEs to ASO SQ.
42129efa63aSLi Zhang  *
4225dfa003dSMichael Baum  * @param[in] sh
4235dfa003dSMichael Baum  *   Pointer to shared device context.
42429efa63aSLi Zhang  * @param[in] n
42529efa63aSLi Zhang  *   Index of the last valid pool.
42629efa63aSLi Zhang  *
42729efa63aSLi Zhang  * @return
42829efa63aSLi Zhang  *   Number of WQEs in burst.
42929efa63aSLi Zhang  */
43029efa63aSLi Zhang static uint16_t
4315dfa003dSMichael Baum mlx5_aso_sq_enqueue_burst(struct mlx5_dev_ctx_shared *sh, uint16_t n)
43229efa63aSLi Zhang {
4335dfa003dSMichael Baum 	struct mlx5_aso_age_mng *mng = sh->aso_age_mng;
43429efa63aSLi Zhang 	volatile struct mlx5_aso_wqe *wqe;
43529efa63aSLi Zhang 	struct mlx5_aso_sq *sq = &mng->aso_sq;
43629efa63aSLi Zhang 	struct mlx5_aso_age_pool *pool;
43729efa63aSLi Zhang 	uint16_t size = 1 << sq->log_desc_n;
43829efa63aSLi Zhang 	uint16_t mask = size - 1;
43929efa63aSLi Zhang 	uint16_t max;
44029efa63aSLi Zhang 	uint16_t start_head = sq->head;
44129efa63aSLi Zhang 
44229efa63aSLi Zhang 	max = RTE_MIN(size - (uint16_t)(sq->head - sq->tail), n - sq->next);
44329efa63aSLi Zhang 	if (unlikely(!max))
44429efa63aSLi Zhang 		return 0;
44529efa63aSLi Zhang 	sq->elts[start_head & mask].burst_size = max;
44629efa63aSLi Zhang 	do {
44729efa63aSLi Zhang 		wqe = &sq->sq_obj.aso_wqes[sq->head & mask];
44829efa63aSLi Zhang 		rte_prefetch0(&sq->sq_obj.aso_wqes[(sq->head + 1) & mask]);
44929efa63aSLi Zhang 		/* Fill next WQE. */
4507cf2d15aSJiawei Wang 		rte_rwlock_read_lock(&mng->resize_rwl);
45129efa63aSLi Zhang 		pool = mng->pools[sq->next];
4527cf2d15aSJiawei Wang 		rte_rwlock_read_unlock(&mng->resize_rwl);
45329efa63aSLi Zhang 		sq->elts[sq->head & mask].pool = pool;
45429efa63aSLi Zhang 		wqe->general_cseg.misc =
45529efa63aSLi Zhang 				rte_cpu_to_be_32(((struct mlx5_devx_obj *)
45629efa63aSLi Zhang 						 (pool->flow_hit_aso_obj))->id);
45729efa63aSLi Zhang 		wqe->general_cseg.flags = RTE_BE32(MLX5_COMP_ONLY_FIRST_ERR <<
45829efa63aSLi Zhang 							 MLX5_COMP_MODE_OFFSET);
45929efa63aSLi Zhang 		wqe->general_cseg.opcode = rte_cpu_to_be_32
46029efa63aSLi Zhang 						(MLX5_OPCODE_ACCESS_ASO |
46129efa63aSLi Zhang 						 (ASO_OPC_MOD_FLOW_HIT <<
46229efa63aSLi Zhang 						  WQE_CSEG_OPC_MOD_OFFSET) |
46329efa63aSLi Zhang 						 (sq->pi <<
46429efa63aSLi Zhang 						  WQE_CSEG_WQE_INDEX_OFFSET));
46529efa63aSLi Zhang 		sq->pi += 2; /* Each WQE contains 2 WQEBB's. */
46629efa63aSLi Zhang 		sq->head++;
46729efa63aSLi Zhang 		sq->next++;
46829efa63aSLi Zhang 		max--;
46929efa63aSLi Zhang 	} while (max);
47029efa63aSLi Zhang 	wqe->general_cseg.flags = RTE_BE32(MLX5_COMP_ALWAYS <<
47129efa63aSLi Zhang 							 MLX5_COMP_MODE_OFFSET);
4725dfa003dSMichael Baum 	mlx5_doorbell_ring(&sh->tx_uar.bf_db, *(volatile uint64_t *)wqe,
4735dfa003dSMichael Baum 			   sq->pi, &sq->sq_obj.db_rec[MLX5_SND_DBR],
4745dfa003dSMichael Baum 			   !sh->tx_uar.dbnc);
47529efa63aSLi Zhang 	return sq->elts[start_head & mask].burst_size;
47629efa63aSLi Zhang }
47729efa63aSLi Zhang 
47829efa63aSLi Zhang /**
47929efa63aSLi Zhang  * Debug utility function. Dump contents of error CQE and WQE.
48029efa63aSLi Zhang  *
48129efa63aSLi Zhang  * @param[in] cqe
48229efa63aSLi Zhang  *   Error CQE to dump.
48329efa63aSLi Zhang  * @param[in] wqe
48429efa63aSLi Zhang  *   Error WQE to dump.
48529efa63aSLi Zhang  */
48629efa63aSLi Zhang static void
48729efa63aSLi Zhang mlx5_aso_dump_err_objs(volatile uint32_t *cqe, volatile uint32_t *wqe)
48829efa63aSLi Zhang {
48929efa63aSLi Zhang 	int i;
49029efa63aSLi Zhang 
49129efa63aSLi Zhang 	DRV_LOG(ERR, "Error cqe:");
492*3cddeba0SAlexander Kozyrev 	for (i = 0; i < (int)sizeof(struct mlx5_error_cqe) / 4; i += 4)
49329efa63aSLi Zhang 		DRV_LOG(ERR, "%08X %08X %08X %08X", cqe[i], cqe[i + 1],
49429efa63aSLi Zhang 			cqe[i + 2], cqe[i + 3]);
49529efa63aSLi Zhang 	DRV_LOG(ERR, "\nError wqe:");
49629efa63aSLi Zhang 	for (i = 0; i < (int)sizeof(struct mlx5_aso_wqe) / 4; i += 4)
49729efa63aSLi Zhang 		DRV_LOG(ERR, "%08X %08X %08X %08X", wqe[i], wqe[i + 1],
49829efa63aSLi Zhang 			wqe[i + 2], wqe[i + 3]);
49929efa63aSLi Zhang }
50029efa63aSLi Zhang 
50129efa63aSLi Zhang /**
50229efa63aSLi Zhang  * Handle case of error CQE.
50329efa63aSLi Zhang  *
50429efa63aSLi Zhang  * @param[in] sq
50529efa63aSLi Zhang  *   ASO SQ to use.
50629efa63aSLi Zhang  */
50715896eafSGregory Etelson void
50829efa63aSLi Zhang mlx5_aso_cqe_err_handle(struct mlx5_aso_sq *sq)
50929efa63aSLi Zhang {
51029efa63aSLi Zhang 	struct mlx5_aso_cq *cq = &sq->cq;
51129efa63aSLi Zhang 	uint32_t idx = cq->cq_ci & ((1 << cq->log_desc_n) - 1);
512*3cddeba0SAlexander Kozyrev 	volatile struct mlx5_error_cqe *cqe =
513*3cddeba0SAlexander Kozyrev 			(volatile struct mlx5_error_cqe *)&cq->cq_obj.cqes[idx];
51429efa63aSLi Zhang 
51529efa63aSLi Zhang 	cq->errors++;
51629efa63aSLi Zhang 	idx = rte_be_to_cpu_16(cqe->wqe_counter) & (1u << sq->log_desc_n);
51729efa63aSLi Zhang 	mlx5_aso_dump_err_objs((volatile uint32_t *)cqe,
51829efa63aSLi Zhang 			       (volatile uint32_t *)&sq->sq_obj.aso_wqes[idx]);
51929efa63aSLi Zhang }
52029efa63aSLi Zhang 
521478ba4bbSSuanming Mou int
522478ba4bbSSuanming Mou mlx5_aso_pull_completion(struct mlx5_aso_sq *sq,
523478ba4bbSSuanming Mou 			 struct rte_flow_op_result res[],
524478ba4bbSSuanming Mou 			 uint16_t n_res)
525478ba4bbSSuanming Mou {
526478ba4bbSSuanming Mou 	struct mlx5_aso_cq *cq = &sq->cq;
527478ba4bbSSuanming Mou 	volatile struct mlx5_cqe *restrict cqe;
528478ba4bbSSuanming Mou 	const uint32_t cq_size = 1 << cq->log_desc_n;
529478ba4bbSSuanming Mou 	const uint32_t mask = cq_size - 1;
530478ba4bbSSuanming Mou 	uint32_t idx;
531478ba4bbSSuanming Mou 	uint32_t next_idx;
532478ba4bbSSuanming Mou 	uint16_t max;
533478ba4bbSSuanming Mou 	uint16_t n = 0;
534478ba4bbSSuanming Mou 	int ret;
535478ba4bbSSuanming Mou 
536478ba4bbSSuanming Mou 	max = (uint16_t)(sq->head - sq->tail);
537478ba4bbSSuanming Mou 	if (unlikely(!max || !n_res))
538478ba4bbSSuanming Mou 		return 0;
539478ba4bbSSuanming Mou 	next_idx = cq->cq_ci & mask;
540478ba4bbSSuanming Mou 	do {
541478ba4bbSSuanming Mou 		idx = next_idx;
542478ba4bbSSuanming Mou 		next_idx = (cq->cq_ci + 1) & mask;
543478ba4bbSSuanming Mou 		/* Need to confirm the position of the prefetch. */
544478ba4bbSSuanming Mou 		rte_prefetch0(&cq->cq_obj.cqes[next_idx]);
545478ba4bbSSuanming Mou 		cqe = &cq->cq_obj.cqes[idx];
546478ba4bbSSuanming Mou 		ret = check_cqe(cqe, cq_size, cq->cq_ci);
547478ba4bbSSuanming Mou 		/*
548478ba4bbSSuanming Mou 		 * Be sure owner read is done before any other cookie field or
549478ba4bbSSuanming Mou 		 * opaque field.
550478ba4bbSSuanming Mou 		 */
551478ba4bbSSuanming Mou 		rte_io_rmb();
552478ba4bbSSuanming Mou 		if (ret == MLX5_CQE_STATUS_HW_OWN)
553478ba4bbSSuanming Mou 			break;
554478ba4bbSSuanming Mou 		res[n].user_data = sq->elts[(uint16_t)((sq->tail + n) & mask)].user_data;
555478ba4bbSSuanming Mou 		if (unlikely(ret != MLX5_CQE_STATUS_SW_OWN)) {
556478ba4bbSSuanming Mou 			mlx5_aso_cqe_err_handle(sq);
557478ba4bbSSuanming Mou 			res[n].status = RTE_FLOW_OP_ERROR;
558478ba4bbSSuanming Mou 		} else {
559478ba4bbSSuanming Mou 			res[n].status = RTE_FLOW_OP_SUCCESS;
560478ba4bbSSuanming Mou 		}
561478ba4bbSSuanming Mou 		cq->cq_ci++;
562478ba4bbSSuanming Mou 		if (++n == n_res)
563478ba4bbSSuanming Mou 			break;
564478ba4bbSSuanming Mou 	} while (1);
565478ba4bbSSuanming Mou 	if (likely(n)) {
566478ba4bbSSuanming Mou 		sq->tail += n;
567478ba4bbSSuanming Mou 		rte_io_wmb();
568478ba4bbSSuanming Mou 		cq->cq_obj.db_rec[0] = rte_cpu_to_be_32(cq->cq_ci);
569478ba4bbSSuanming Mou 	}
570478ba4bbSSuanming Mou 	return n;
571478ba4bbSSuanming Mou }
572478ba4bbSSuanming Mou 
573478ba4bbSSuanming Mou void
574478ba4bbSSuanming Mou mlx5_aso_push_wqe(struct mlx5_dev_ctx_shared *sh,
575478ba4bbSSuanming Mou 		  struct mlx5_aso_sq *sq)
576478ba4bbSSuanming Mou {
577478ba4bbSSuanming Mou 	if (sq->db_pi == sq->pi)
578478ba4bbSSuanming Mou 		return;
579478ba4bbSSuanming Mou 	mlx5_doorbell_ring(&sh->tx_uar.bf_db, *(volatile uint64_t *)sq->db,
580478ba4bbSSuanming Mou 			   sq->pi, &sq->sq_obj.db_rec[MLX5_SND_DBR],
581478ba4bbSSuanming Mou 			   !sh->tx_uar.dbnc);
582478ba4bbSSuanming Mou 	sq->db_pi = sq->pi;
583478ba4bbSSuanming Mou }
584478ba4bbSSuanming Mou 
58529efa63aSLi Zhang /**
58629efa63aSLi Zhang  * Update ASO objects upon completion.
58729efa63aSLi Zhang  *
58829efa63aSLi Zhang  * @param[in] sh
58929efa63aSLi Zhang  *   Shared device context.
59029efa63aSLi Zhang  * @param[in] n
59129efa63aSLi Zhang  *   Number of completed ASO objects.
59229efa63aSLi Zhang  */
59329efa63aSLi Zhang static void
59429efa63aSLi Zhang mlx5_aso_age_action_update(struct mlx5_dev_ctx_shared *sh, uint16_t n)
59529efa63aSLi Zhang {
59629efa63aSLi Zhang 	struct mlx5_aso_age_mng *mng = sh->aso_age_mng;
59729efa63aSLi Zhang 	struct mlx5_aso_sq *sq = &mng->aso_sq;
59829efa63aSLi Zhang 	struct mlx5_age_info *age_info;
59929efa63aSLi Zhang 	const uint16_t size = 1 << sq->log_desc_n;
60029efa63aSLi Zhang 	const uint16_t mask = size - 1;
60129efa63aSLi Zhang 	const uint64_t curr = MLX5_CURR_TIME_SEC;
60229efa63aSLi Zhang 	uint16_t expected = AGE_CANDIDATE;
60329efa63aSLi Zhang 	uint16_t i;
60429efa63aSLi Zhang 
60529efa63aSLi Zhang 	for (i = 0; i < n; ++i) {
60629efa63aSLi Zhang 		uint16_t idx = (sq->tail + i) & mask;
60729efa63aSLi Zhang 		struct mlx5_aso_age_pool *pool = sq->elts[idx].pool;
60829efa63aSLi Zhang 		uint64_t diff = curr - pool->time_of_last_age_check;
609cd414f81SMichael Baum 		uint64_t *addr = sq->mr.addr;
61029efa63aSLi Zhang 		int j;
61129efa63aSLi Zhang 
61229efa63aSLi Zhang 		addr += idx * MLX5_ASO_AGE_ACTIONS_PER_POOL / 64;
61329efa63aSLi Zhang 		pool->time_of_last_age_check = curr;
61429efa63aSLi Zhang 		for (j = 0; j < MLX5_ASO_AGE_ACTIONS_PER_POOL; j++) {
61529efa63aSLi Zhang 			struct mlx5_aso_age_action *act = &pool->actions[j];
61629efa63aSLi Zhang 			struct mlx5_age_param *ap = &act->age_params;
61729efa63aSLi Zhang 			uint8_t byte;
61829efa63aSLi Zhang 			uint8_t offset;
61929efa63aSLi Zhang 			uint8_t *u8addr;
62029efa63aSLi Zhang 			uint8_t hit;
62129efa63aSLi Zhang 
622e12a0166STyler Retzlaff 			if (rte_atomic_load_explicit(&ap->state, rte_memory_order_relaxed) !=
62329efa63aSLi Zhang 					    AGE_CANDIDATE)
62429efa63aSLi Zhang 				continue;
62529efa63aSLi Zhang 			byte = 63 - (j / 8);
62629efa63aSLi Zhang 			offset = j % 8;
62729efa63aSLi Zhang 			u8addr = (uint8_t *)addr;
62829efa63aSLi Zhang 			hit = (u8addr[byte] >> offset) & 0x1;
62929efa63aSLi Zhang 			if (hit) {
630e12a0166STyler Retzlaff 				rte_atomic_store_explicit(&ap->sec_since_last_hit, 0,
631e12a0166STyler Retzlaff 						 rte_memory_order_relaxed);
63229efa63aSLi Zhang 			} else {
63329efa63aSLi Zhang 				struct mlx5_priv *priv;
63429efa63aSLi Zhang 
635e12a0166STyler Retzlaff 				rte_atomic_fetch_add_explicit(&ap->sec_since_last_hit,
636e12a0166STyler Retzlaff 						   diff, rte_memory_order_relaxed);
63729efa63aSLi Zhang 				/* If timeout passed add to aged-out list. */
63829efa63aSLi Zhang 				if (ap->sec_since_last_hit <= ap->timeout)
63929efa63aSLi Zhang 					continue;
64029efa63aSLi Zhang 				priv =
64129efa63aSLi Zhang 				rte_eth_devices[ap->port_id].data->dev_private;
64229efa63aSLi Zhang 				age_info = GET_PORT_AGE_INFO(priv);
64329efa63aSLi Zhang 				rte_spinlock_lock(&age_info->aged_sl);
644e12a0166STyler Retzlaff 				if (rte_atomic_compare_exchange_strong_explicit(&ap->state,
64529efa63aSLi Zhang 								&expected,
64629efa63aSLi Zhang 								AGE_TMOUT,
647e12a0166STyler Retzlaff 							       rte_memory_order_relaxed,
648e12a0166STyler Retzlaff 							    rte_memory_order_relaxed)) {
64929efa63aSLi Zhang 					LIST_INSERT_HEAD(&age_info->aged_aso,
65029efa63aSLi Zhang 							 act, next);
65129efa63aSLi Zhang 					MLX5_AGE_SET(age_info,
65229efa63aSLi Zhang 						     MLX5_AGE_EVENT_NEW);
65329efa63aSLi Zhang 				}
65429efa63aSLi Zhang 				rte_spinlock_unlock(&age_info->aged_sl);
65529efa63aSLi Zhang 			}
65629efa63aSLi Zhang 		}
65729efa63aSLi Zhang 	}
65829efa63aSLi Zhang 	mlx5_age_event_prepare(sh);
65929efa63aSLi Zhang }
66029efa63aSLi Zhang 
66129efa63aSLi Zhang /**
66229efa63aSLi Zhang  * Handle completions from WQEs sent to ASO SQ.
66329efa63aSLi Zhang  *
66429efa63aSLi Zhang  * @param[in] sh
66529efa63aSLi Zhang  *   Shared device context.
66629efa63aSLi Zhang  *
66729efa63aSLi Zhang  * @return
66829efa63aSLi Zhang  *   Number of CQEs handled.
66929efa63aSLi Zhang  */
67029efa63aSLi Zhang static uint16_t
67129efa63aSLi Zhang mlx5_aso_completion_handle(struct mlx5_dev_ctx_shared *sh)
67229efa63aSLi Zhang {
67329efa63aSLi Zhang 	struct mlx5_aso_age_mng *mng = sh->aso_age_mng;
67429efa63aSLi Zhang 	struct mlx5_aso_sq *sq = &mng->aso_sq;
67529efa63aSLi Zhang 	struct mlx5_aso_cq *cq = &sq->cq;
67629efa63aSLi Zhang 	volatile struct mlx5_cqe *restrict cqe;
67729efa63aSLi Zhang 	const unsigned int cq_size = 1 << cq->log_desc_n;
67829efa63aSLi Zhang 	const unsigned int mask = cq_size - 1;
67929efa63aSLi Zhang 	uint32_t idx;
68029efa63aSLi Zhang 	uint32_t next_idx = cq->cq_ci & mask;
68129efa63aSLi Zhang 	const uint16_t max = (uint16_t)(sq->head - sq->tail);
68229efa63aSLi Zhang 	uint16_t i = 0;
68329efa63aSLi Zhang 	int ret;
68429efa63aSLi Zhang 	if (unlikely(!max))
68529efa63aSLi Zhang 		return 0;
68629efa63aSLi Zhang 	do {
68729efa63aSLi Zhang 		idx = next_idx;
68829efa63aSLi Zhang 		next_idx = (cq->cq_ci + 1) & mask;
68929efa63aSLi Zhang 		rte_prefetch0(&cq->cq_obj.cqes[next_idx]);
69029efa63aSLi Zhang 		cqe = &cq->cq_obj.cqes[idx];
69129efa63aSLi Zhang 		ret = check_cqe(cqe, cq_size, cq->cq_ci);
69229efa63aSLi Zhang 		/*
69329efa63aSLi Zhang 		 * Be sure owner read is done before any other cookie field or
69429efa63aSLi Zhang 		 * opaque field.
69529efa63aSLi Zhang 		 */
69629efa63aSLi Zhang 		rte_io_rmb();
69729efa63aSLi Zhang 		if (unlikely(ret != MLX5_CQE_STATUS_SW_OWN)) {
69829efa63aSLi Zhang 			if (likely(ret == MLX5_CQE_STATUS_HW_OWN))
69929efa63aSLi Zhang 				break;
70029efa63aSLi Zhang 			mlx5_aso_cqe_err_handle(sq);
70129efa63aSLi Zhang 		} else {
70229efa63aSLi Zhang 			i += sq->elts[(sq->tail + i) & mask].burst_size;
70329efa63aSLi Zhang 		}
70429efa63aSLi Zhang 		cq->cq_ci++;
70529efa63aSLi Zhang 	} while (1);
70629efa63aSLi Zhang 	if (likely(i)) {
70729efa63aSLi Zhang 		mlx5_aso_age_action_update(sh, i);
70829efa63aSLi Zhang 		sq->tail += i;
70929efa63aSLi Zhang 		rte_io_wmb();
71029efa63aSLi Zhang 		cq->cq_obj.db_rec[0] = rte_cpu_to_be_32(cq->cq_ci);
71129efa63aSLi Zhang 	}
71229efa63aSLi Zhang 	return i;
71329efa63aSLi Zhang }
71429efa63aSLi Zhang 
71529efa63aSLi Zhang /**
71629efa63aSLi Zhang  * Periodically read CQEs and send WQEs to ASO SQ.
71729efa63aSLi Zhang  *
71829efa63aSLi Zhang  * @param[in] arg
71929efa63aSLi Zhang  *   Shared device context containing the ASO SQ.
72029efa63aSLi Zhang  */
72129efa63aSLi Zhang static void
72229efa63aSLi Zhang mlx5_flow_aso_alarm(void *arg)
72329efa63aSLi Zhang {
72429efa63aSLi Zhang 	struct mlx5_dev_ctx_shared *sh = arg;
72529efa63aSLi Zhang 	struct mlx5_aso_sq *sq = &sh->aso_age_mng->aso_sq;
72629efa63aSLi Zhang 	uint32_t us = 100u;
72729efa63aSLi Zhang 	uint16_t n;
72829efa63aSLi Zhang 
7297cf2d15aSJiawei Wang 	rte_rwlock_read_lock(&sh->aso_age_mng->resize_rwl);
73029efa63aSLi Zhang 	n = sh->aso_age_mng->next;
7317cf2d15aSJiawei Wang 	rte_rwlock_read_unlock(&sh->aso_age_mng->resize_rwl);
73229efa63aSLi Zhang 	mlx5_aso_completion_handle(sh);
73329efa63aSLi Zhang 	if (sq->next == n) {
73429efa63aSLi Zhang 		/* End of loop: wait 1 second. */
73529efa63aSLi Zhang 		us = US_PER_S;
73629efa63aSLi Zhang 		sq->next = 0;
73729efa63aSLi Zhang 	}
7385dfa003dSMichael Baum 	mlx5_aso_sq_enqueue_burst(sh, n);
73929efa63aSLi Zhang 	if (rte_eal_alarm_set(us, mlx5_flow_aso_alarm, sh))
74029efa63aSLi Zhang 		DRV_LOG(ERR, "Cannot reinitialize aso alarm.");
74129efa63aSLi Zhang }
74229efa63aSLi Zhang 
74329efa63aSLi Zhang /**
74429efa63aSLi Zhang  * API to start ASO access using ASO SQ.
74529efa63aSLi Zhang  *
74629efa63aSLi Zhang  * @param[in] sh
74729efa63aSLi Zhang  *   Pointer to shared device context.
74829efa63aSLi Zhang  *
74929efa63aSLi Zhang  * @return
75029efa63aSLi Zhang  *   0 on success, a negative errno value otherwise and rte_errno is set.
75129efa63aSLi Zhang  */
75229efa63aSLi Zhang int
75329efa63aSLi Zhang mlx5_aso_flow_hit_queue_poll_start(struct mlx5_dev_ctx_shared *sh)
75429efa63aSLi Zhang {
75529efa63aSLi Zhang 	if (rte_eal_alarm_set(US_PER_S, mlx5_flow_aso_alarm, sh)) {
75629efa63aSLi Zhang 		DRV_LOG(ERR, "Cannot reinitialize ASO age alarm.");
75729efa63aSLi Zhang 		return -rte_errno;
75829efa63aSLi Zhang 	}
75929efa63aSLi Zhang 	return 0;
76029efa63aSLi Zhang }
76129efa63aSLi Zhang 
76229efa63aSLi Zhang /**
76329efa63aSLi Zhang  * API to stop ASO access using ASO SQ.
76429efa63aSLi Zhang  *
76529efa63aSLi Zhang  * @param[in] sh
76629efa63aSLi Zhang  *   Pointer to shared device context.
76729efa63aSLi Zhang  *
76829efa63aSLi Zhang  * @return
76929efa63aSLi Zhang  *   0 on success, a negative errno value otherwise and rte_errno is set.
77029efa63aSLi Zhang  */
77129efa63aSLi Zhang int
77229efa63aSLi Zhang mlx5_aso_flow_hit_queue_poll_stop(struct mlx5_dev_ctx_shared *sh)
77329efa63aSLi Zhang {
77429efa63aSLi Zhang 	int retries = 1024;
77529efa63aSLi Zhang 
77629efa63aSLi Zhang 	if (!sh->aso_age_mng->aso_sq.sq_obj.sq)
77729efa63aSLi Zhang 		return -EINVAL;
77829efa63aSLi Zhang 	rte_errno = 0;
77929efa63aSLi Zhang 	while (--retries) {
78029efa63aSLi Zhang 		rte_eal_alarm_cancel(mlx5_flow_aso_alarm, sh);
78129efa63aSLi Zhang 		if (rte_errno != EINPROGRESS)
78229efa63aSLi Zhang 			break;
78329efa63aSLi Zhang 		rte_pause();
78429efa63aSLi Zhang 	}
78529efa63aSLi Zhang 	return -rte_errno;
78629efa63aSLi Zhang }
787e93c58daSLi Zhang 
788e93c58daSLi Zhang static uint16_t
7895dfa003dSMichael Baum mlx5_aso_mtr_sq_enqueue_single(struct mlx5_dev_ctx_shared *sh,
7905dfa003dSMichael Baum 			       struct mlx5_aso_sq *sq,
79124865366SAlexander Kozyrev 			       struct mlx5_aso_mtr *aso_mtr,
79248fbb0e9SAlexander Kozyrev 			       struct mlx5_mtr_bulk *bulk,
793478ba4bbSSuanming Mou 			       bool need_lock,
7944359d9d1SGregory Etelson 			       struct mlx5_hw_q_job *job,
795478ba4bbSSuanming Mou 			       bool push)
796e93c58daSLi Zhang {
797e93c58daSLi Zhang 	volatile struct mlx5_aso_wqe *wqe = NULL;
798e93c58daSLi Zhang 	struct mlx5_flow_meter_info *fm = NULL;
799aa065a9cSLi Zhang 	struct mlx5_flow_meter_profile *fmp;
800e93c58daSLi Zhang 	uint16_t size = 1 << sq->log_desc_n;
801e93c58daSLi Zhang 	uint16_t mask = size - 1;
802cfd2037cSLi Zhang 	uint16_t res;
803e93c58daSLi Zhang 	uint32_t dseg_idx = 0;
804e93c58daSLi Zhang 	struct mlx5_aso_mtr_pool *pool = NULL;
8056b838de3SShun Hao 	uint32_t param_le;
80624865366SAlexander Kozyrev 	int id;
807e93c58daSLi Zhang 
80848fbb0e9SAlexander Kozyrev 	if (need_lock)
809cfd2037cSLi Zhang 		rte_spinlock_lock(&sq->sqsl);
810cfd2037cSLi Zhang 	res = size - (uint16_t)(sq->head - sq->tail);
811e93c58daSLi Zhang 	if (unlikely(!res)) {
812e93c58daSLi Zhang 		DRV_LOG(ERR, "Fail: SQ is full and no free WQE to send");
81348fbb0e9SAlexander Kozyrev 		if (need_lock)
814cfd2037cSLi Zhang 			rte_spinlock_unlock(&sq->sqsl);
815e93c58daSLi Zhang 		return 0;
816e93c58daSLi Zhang 	}
817e93c58daSLi Zhang 	wqe = &sq->sq_obj.aso_wqes[sq->head & mask];
818e93c58daSLi Zhang 	rte_prefetch0(&sq->sq_obj.aso_wqes[(sq->head + 1) & mask]);
819e93c58daSLi Zhang 	/* Fill next WQE. */
820e93c58daSLi Zhang 	fm = &aso_mtr->fm;
8214359d9d1SGregory Etelson 	sq->elts[sq->head & mask].user_data = job ? job : (void *)aso_mtr;
82224865366SAlexander Kozyrev 	if (aso_mtr->type == ASO_METER_INDIRECT) {
82348fbb0e9SAlexander Kozyrev 		if (likely(sh->config.dv_flow_en == 2))
82448fbb0e9SAlexander Kozyrev 			pool = aso_mtr->pool;
82548fbb0e9SAlexander Kozyrev 		else
826e93c58daSLi Zhang 			pool = container_of(aso_mtr, struct mlx5_aso_mtr_pool,
827e93c58daSLi Zhang 					    mtrs[aso_mtr->offset]);
82824865366SAlexander Kozyrev 		id = pool->devx_obj->id;
82924865366SAlexander Kozyrev 	} else {
83024865366SAlexander Kozyrev 		id = bulk->devx_obj->id;
83124865366SAlexander Kozyrev 	}
83224865366SAlexander Kozyrev 	wqe->general_cseg.misc = rte_cpu_to_be_32(id +
833e93c58daSLi Zhang 						  (aso_mtr->offset >> 1));
83424865366SAlexander Kozyrev 	wqe->general_cseg.opcode =
83524865366SAlexander Kozyrev 		rte_cpu_to_be_32(MLX5_OPCODE_ACCESS_ASO |
83624865366SAlexander Kozyrev 			(ASO_OPC_MOD_POLICER << WQE_CSEG_OPC_MOD_OFFSET) |
837e93c58daSLi Zhang 			 sq->pi << WQE_CSEG_WQE_INDEX_OFFSET);
838e93c58daSLi Zhang 	/* There are 2 meters in one ASO cache line. */
839e93c58daSLi Zhang 	dseg_idx = aso_mtr->offset & 0x1;
840e93c58daSLi Zhang 	wqe->aso_cseg.data_mask =
841e93c58daSLi Zhang 		RTE_BE64(MLX5_IFC_FLOW_METER_PARAM_MASK << (32 * !dseg_idx));
842e93c58daSLi Zhang 	if (fm->is_enable) {
843e93c58daSLi Zhang 		wqe->aso_dseg.mtrs[dseg_idx].cbs_cir =
844e93c58daSLi Zhang 			fm->profile->srtcm_prm.cbs_cir;
845e93c58daSLi Zhang 		wqe->aso_dseg.mtrs[dseg_idx].ebs_eir =
846e93c58daSLi Zhang 			fm->profile->srtcm_prm.ebs_eir;
847e93c58daSLi Zhang 	} else {
848e93c58daSLi Zhang 		wqe->aso_dseg.mtrs[dseg_idx].cbs_cir =
849e93c58daSLi Zhang 			RTE_BE32(MLX5_IFC_FLOW_METER_DISABLE_CBS_CIR_VAL);
850e93c58daSLi Zhang 		wqe->aso_dseg.mtrs[dseg_idx].ebs_eir = 0;
851e93c58daSLi Zhang 	}
852aa065a9cSLi Zhang 	fmp = fm->profile;
8536b838de3SShun Hao 	param_le = (1 << ASO_DSEG_VALID_OFFSET);
8546b838de3SShun Hao 	if (fm->color_aware)
8556b838de3SShun Hao 		param_le |= (MLX5_FLOW_COLOR_UNDEFINED << ASO_DSEG_SC_OFFSET);
856aa065a9cSLi Zhang 	else
8576b838de3SShun Hao 		param_le |= (MLX5_FLOW_COLOR_GREEN << ASO_DSEG_SC_OFFSET);
8586b838de3SShun Hao 	if (fmp->profile.packet_mode)
8596b838de3SShun Hao 		param_le |= (MLX5_METER_MODE_PKT << ASO_DSEG_MTR_MODE);
8606b838de3SShun Hao 	wqe->aso_dseg.mtrs[dseg_idx].v_bo_sc_bbog_mm = RTE_BE32(param_le);
86133a7493cSBing Zhao 	switch (fmp->profile.alg) {
86233a7493cSBing Zhao 	case RTE_MTR_SRTCM_RFC2697:
8639b5463dfSBing Zhao 		/* Only needed for RFC2697. */
8649b5463dfSBing Zhao 		if (fm->profile->srtcm_prm.ebs_eir)
8659b5463dfSBing Zhao 			wqe->aso_dseg.mtrs[dseg_idx].v_bo_sc_bbog_mm |=
8669b5463dfSBing Zhao 					RTE_BE32(1 << ASO_DSEG_BO_OFFSET);
86733a7493cSBing Zhao 		break;
86833a7493cSBing Zhao 	case RTE_MTR_TRTCM_RFC2698:
86933a7493cSBing Zhao 		wqe->aso_dseg.mtrs[dseg_idx].v_bo_sc_bbog_mm |=
87033a7493cSBing Zhao 				RTE_BE32(1 << ASO_DSEG_BBOG_OFFSET);
87133a7493cSBing Zhao 		break;
87233a7493cSBing Zhao 	case RTE_MTR_TRTCM_RFC4115:
8736b838de3SShun Hao 		wqe->aso_dseg.mtrs[dseg_idx].v_bo_sc_bbog_mm |=
8746b838de3SShun Hao 				RTE_BE32(1 << ASO_DSEG_BO_OFFSET);
8756b838de3SShun Hao 		break;
87633a7493cSBing Zhao 	default:
87733a7493cSBing Zhao 		break;
87833a7493cSBing Zhao 	}
87933a7493cSBing Zhao 	/*
88033a7493cSBing Zhao 	 * Note:
88133a7493cSBing Zhao 	 * Due to software performance reason, the token fields will not be
88233a7493cSBing Zhao 	 * set when posting the WQE to ASO SQ. It will be filled by the HW
88333a7493cSBing Zhao 	 * automatically.
88433a7493cSBing Zhao 	 */
885e93c58daSLi Zhang 	sq->head++;
886e93c58daSLi Zhang 	sq->pi += 2;/* Each WQE contains 2 WQEBB's. */
887478ba4bbSSuanming Mou 	if (push) {
8885dfa003dSMichael Baum 		mlx5_doorbell_ring(&sh->tx_uar.bf_db, *(volatile uint64_t *)wqe,
8895dfa003dSMichael Baum 			   sq->pi, &sq->sq_obj.db_rec[MLX5_SND_DBR],
8905dfa003dSMichael Baum 			   !sh->tx_uar.dbnc);
891478ba4bbSSuanming Mou 		sq->db_pi = sq->pi;
892478ba4bbSSuanming Mou 	}
893478ba4bbSSuanming Mou 	sq->db = wqe;
89448fbb0e9SAlexander Kozyrev 	if (need_lock)
895cfd2037cSLi Zhang 		rte_spinlock_unlock(&sq->sqsl);
896e93c58daSLi Zhang 	return 1;
897e93c58daSLi Zhang }
898e93c58daSLi Zhang 
899e93c58daSLi Zhang static void
90048fbb0e9SAlexander Kozyrev mlx5_aso_mtr_completion_handle(struct mlx5_aso_sq *sq, bool need_lock)
901e93c58daSLi Zhang {
902e93c58daSLi Zhang 	struct mlx5_aso_cq *cq = &sq->cq;
903e93c58daSLi Zhang 	volatile struct mlx5_cqe *restrict cqe;
904e93c58daSLi Zhang 	const unsigned int cq_size = 1 << cq->log_desc_n;
905e93c58daSLi Zhang 	const unsigned int mask = cq_size - 1;
906e93c58daSLi Zhang 	uint32_t idx;
907e93c58daSLi Zhang 	uint32_t next_idx = cq->cq_ci & mask;
908cfd2037cSLi Zhang 	uint16_t max;
9094359d9d1SGregory Etelson 	uint16_t i, n = 0;
910e93c58daSLi Zhang 	int ret;
911e93c58daSLi Zhang 
91248fbb0e9SAlexander Kozyrev 	if (need_lock)
913cfd2037cSLi Zhang 		rte_spinlock_lock(&sq->sqsl);
914cfd2037cSLi Zhang 	max = (uint16_t)(sq->head - sq->tail);
915cfd2037cSLi Zhang 	if (unlikely(!max)) {
91629f99df4SWeiguo Li 		if (need_lock)
917cfd2037cSLi Zhang 			rte_spinlock_unlock(&sq->sqsl);
918e93c58daSLi Zhang 		return;
919cfd2037cSLi Zhang 	}
920e93c58daSLi Zhang 	do {
921e93c58daSLi Zhang 		idx = next_idx;
922e93c58daSLi Zhang 		next_idx = (cq->cq_ci + 1) & mask;
923e93c58daSLi Zhang 		rte_prefetch0(&cq->cq_obj.cqes[next_idx]);
924e93c58daSLi Zhang 		cqe = &cq->cq_obj.cqes[idx];
925e93c58daSLi Zhang 		ret = check_cqe(cqe, cq_size, cq->cq_ci);
926e93c58daSLi Zhang 		/*
927e93c58daSLi Zhang 		 * Be sure owner read is done before any other cookie field or
928e93c58daSLi Zhang 		 * opaque field.
929e93c58daSLi Zhang 		 */
930e93c58daSLi Zhang 		rte_io_rmb();
931e93c58daSLi Zhang 		if (ret != MLX5_CQE_STATUS_SW_OWN) {
932e93c58daSLi Zhang 			if (likely(ret == MLX5_CQE_STATUS_HW_OWN))
933e93c58daSLi Zhang 				break;
934e93c58daSLi Zhang 			mlx5_aso_cqe_err_handle(sq);
935e93c58daSLi Zhang 		} else {
936e93c58daSLi Zhang 			n++;
937e93c58daSLi Zhang 		}
938e93c58daSLi Zhang 		cq->cq_ci++;
939e93c58daSLi Zhang 	} while (1);
940e93c58daSLi Zhang 	if (likely(n)) {
9414359d9d1SGregory Etelson 		uint8_t exp_state = ASO_METER_WAIT;
9424359d9d1SGregory Etelson 		struct mlx5_aso_mtr *aso_mtr;
9434359d9d1SGregory Etelson 		__rte_unused bool verdict;
9444359d9d1SGregory Etelson 
9454359d9d1SGregory Etelson 		for (i = 0; i < n; ++i) {
9464359d9d1SGregory Etelson 			aso_mtr = sq->elts[(sq->tail + i) & mask].mtr;
9474359d9d1SGregory Etelson 			MLX5_ASSERT(aso_mtr);
948e12a0166STyler Retzlaff 			verdict = rte_atomic_compare_exchange_strong_explicit(&aso_mtr->state,
9494359d9d1SGregory Etelson 						    &exp_state, ASO_METER_READY,
950e12a0166STyler Retzlaff 						    rte_memory_order_relaxed,
951e12a0166STyler Retzlaff 						    rte_memory_order_relaxed);
9524359d9d1SGregory Etelson 			MLX5_ASSERT(verdict);
9534359d9d1SGregory Etelson 		}
954e93c58daSLi Zhang 		sq->tail += n;
955e93c58daSLi Zhang 		rte_io_wmb();
956e93c58daSLi Zhang 		cq->cq_obj.db_rec[0] = rte_cpu_to_be_32(cq->cq_ci);
957e93c58daSLi Zhang 	}
95848fbb0e9SAlexander Kozyrev 	if (need_lock)
959cfd2037cSLi Zhang 		rte_spinlock_unlock(&sq->sqsl);
960e93c58daSLi Zhang }
961e93c58daSLi Zhang 
9624359d9d1SGregory Etelson static __rte_always_inline struct mlx5_aso_sq *
9634359d9d1SGregory Etelson mlx5_aso_mtr_select_sq(struct mlx5_dev_ctx_shared *sh, uint32_t queue,
9644359d9d1SGregory Etelson 		       struct mlx5_aso_mtr *mtr, bool *need_lock)
9654359d9d1SGregory Etelson {
9664359d9d1SGregory Etelson 	struct mlx5_aso_sq *sq;
9674359d9d1SGregory Etelson 
9684359d9d1SGregory Etelson 	if (likely(sh->config.dv_flow_en == 2) &&
9694359d9d1SGregory Etelson 	    mtr->type == ASO_METER_INDIRECT) {
9704359d9d1SGregory Etelson 		if (queue == MLX5_HW_INV_QUEUE) {
9714359d9d1SGregory Etelson 			sq = &mtr->pool->sq[mtr->pool->nb_sq - 1];
9724359d9d1SGregory Etelson 			*need_lock = true;
9734359d9d1SGregory Etelson 		} else {
9744359d9d1SGregory Etelson 			sq = &mtr->pool->sq[queue];
9754359d9d1SGregory Etelson 			*need_lock = false;
9764359d9d1SGregory Etelson 		}
9774359d9d1SGregory Etelson 	} else {
9784359d9d1SGregory Etelson 		sq = &sh->mtrmng->pools_mng.sq;
9794359d9d1SGregory Etelson 		*need_lock = true;
9804359d9d1SGregory Etelson 	}
9814359d9d1SGregory Etelson 	return sq;
9824359d9d1SGregory Etelson }
9834359d9d1SGregory Etelson 
9844359d9d1SGregory Etelson #if defined(HAVE_MLX5_HWS_SUPPORT)
9854359d9d1SGregory Etelson static void
9864359d9d1SGregory Etelson mlx5_aso_poll_cq_mtr_hws(struct mlx5_priv *priv, struct mlx5_aso_sq *sq)
9874359d9d1SGregory Etelson {
9884359d9d1SGregory Etelson #define MLX5_HWS_MTR_CMPL_NUM 4
9894359d9d1SGregory Etelson 
9904359d9d1SGregory Etelson 	int i, ret;
9914359d9d1SGregory Etelson 	struct mlx5_aso_mtr *mtr;
9924359d9d1SGregory Etelson 	uint8_t exp_state = ASO_METER_WAIT;
9934359d9d1SGregory Etelson 	struct rte_flow_op_result res[MLX5_HWS_MTR_CMPL_NUM];
9944359d9d1SGregory Etelson 	__rte_unused bool verdict;
9954359d9d1SGregory Etelson 
9964359d9d1SGregory Etelson 	rte_spinlock_lock(&sq->sqsl);
9974359d9d1SGregory Etelson repeat:
9984359d9d1SGregory Etelson 	ret = mlx5_aso_pull_completion(sq, res, MLX5_HWS_MTR_CMPL_NUM);
9994359d9d1SGregory Etelson 	if (ret) {
10004359d9d1SGregory Etelson 		for (i = 0; i < ret; i++) {
10014359d9d1SGregory Etelson 			struct mlx5_hw_q_job *job = res[i].user_data;
10024359d9d1SGregory Etelson 
10034359d9d1SGregory Etelson 			MLX5_ASSERT(job);
10044359d9d1SGregory Etelson 			mtr = mlx5_ipool_get(priv->hws_mpool->idx_pool,
10054359d9d1SGregory Etelson 					     MLX5_INDIRECT_ACTION_IDX_GET(job->action));
10064359d9d1SGregory Etelson 			MLX5_ASSERT(mtr);
1007e12a0166STyler Retzlaff 			verdict = rte_atomic_compare_exchange_strong_explicit(&mtr->state,
10084359d9d1SGregory Etelson 						    &exp_state, ASO_METER_READY,
1009e12a0166STyler Retzlaff 						    rte_memory_order_relaxed,
1010e12a0166STyler Retzlaff 						    rte_memory_order_relaxed);
10114359d9d1SGregory Etelson 			MLX5_ASSERT(verdict);
10124359d9d1SGregory Etelson 			flow_hw_job_put(priv, job, CTRL_QUEUE_ID(priv));
10134359d9d1SGregory Etelson 		}
10144359d9d1SGregory Etelson 		if (ret == MLX5_HWS_MTR_CMPL_NUM)
10154359d9d1SGregory Etelson 			goto repeat;
10164359d9d1SGregory Etelson 	}
10174359d9d1SGregory Etelson 	rte_spinlock_unlock(&sq->sqsl);
10184359d9d1SGregory Etelson 
10194359d9d1SGregory Etelson #undef MLX5_HWS_MTR_CMPL_NUM
10204359d9d1SGregory Etelson }
10214359d9d1SGregory Etelson #else
10224359d9d1SGregory Etelson static void
10234359d9d1SGregory Etelson mlx5_aso_poll_cq_mtr_hws(__rte_unused struct mlx5_priv *priv, __rte_unused struct mlx5_aso_sq *sq)
10244359d9d1SGregory Etelson {
10254359d9d1SGregory Etelson 	MLX5_ASSERT(false);
10264359d9d1SGregory Etelson }
10274359d9d1SGregory Etelson #endif
10284359d9d1SGregory Etelson 
10294359d9d1SGregory Etelson static void
10304359d9d1SGregory Etelson mlx5_aso_poll_cq_mtr_sws(__rte_unused struct mlx5_priv *priv,
10314359d9d1SGregory Etelson 			 struct mlx5_aso_sq *sq)
10324359d9d1SGregory Etelson {
10334359d9d1SGregory Etelson 	mlx5_aso_mtr_completion_handle(sq, true);
10344359d9d1SGregory Etelson }
10354359d9d1SGregory Etelson 
10364359d9d1SGregory Etelson typedef void (*poll_cq_t)(struct mlx5_priv *, struct mlx5_aso_sq *);
10374359d9d1SGregory Etelson 
1038e93c58daSLi Zhang /**
1039e93c58daSLi Zhang  * Update meter parameter by send WQE.
1040e93c58daSLi Zhang  *
1041e93c58daSLi Zhang  * @param[in] dev
1042e93c58daSLi Zhang  *   Pointer to Ethernet device.
1043e93c58daSLi Zhang  * @param[in] priv
1044e93c58daSLi Zhang  *   Pointer to mlx5 private data structure.
1045e93c58daSLi Zhang  * @param[in] fm
1046e93c58daSLi Zhang  *   Pointer to flow meter to be modified.
1047e93c58daSLi Zhang  *
1048e93c58daSLi Zhang  * @return
1049e93c58daSLi Zhang  *   0 on success, a negative errno value otherwise and rte_errno is set.
1050e93c58daSLi Zhang  */
1051e93c58daSLi Zhang int
10524359d9d1SGregory Etelson mlx5_aso_meter_update_by_wqe(struct mlx5_priv *priv, uint32_t queue,
105324865366SAlexander Kozyrev 			     struct mlx5_aso_mtr *mtr,
1054478ba4bbSSuanming Mou 			     struct mlx5_mtr_bulk *bulk,
10554359d9d1SGregory Etelson 			     struct mlx5_hw_q_job *job, bool push)
1056e93c58daSLi Zhang {
105748fbb0e9SAlexander Kozyrev 	bool need_lock;
10584359d9d1SGregory Etelson 	struct mlx5_dev_ctx_shared *sh = priv->sh;
10594359d9d1SGregory Etelson 	struct mlx5_aso_sq *sq =
10604359d9d1SGregory Etelson 		mlx5_aso_mtr_select_sq(sh, queue, mtr, &need_lock);
10614359d9d1SGregory Etelson 	uint32_t poll_wqe_times = MLX5_MTR_POLL_WQE_CQE_TIMES;
10624359d9d1SGregory Etelson 	poll_cq_t poll_mtr_cq =
10634359d9d1SGregory Etelson 		job ? mlx5_aso_poll_cq_mtr_hws : mlx5_aso_poll_cq_mtr_sws;
1064478ba4bbSSuanming Mou 	int ret;
1065e93c58daSLi Zhang 
1066478ba4bbSSuanming Mou 	if (queue != MLX5_HW_INV_QUEUE) {
1067478ba4bbSSuanming Mou 		ret = mlx5_aso_mtr_sq_enqueue_single(sh, sq, mtr, bulk,
10684359d9d1SGregory Etelson 						     need_lock, job, push);
1069478ba4bbSSuanming Mou 		return ret > 0 ? 0 : -1;
1070478ba4bbSSuanming Mou 	}
1071e93c58daSLi Zhang 	do {
10724359d9d1SGregory Etelson 		poll_mtr_cq(priv, sq);
1073478ba4bbSSuanming Mou 		if (mlx5_aso_mtr_sq_enqueue_single(sh, sq, mtr, bulk,
10744359d9d1SGregory Etelson 						   need_lock, job, true))
1075e93c58daSLi Zhang 			return 0;
1076e93c58daSLi Zhang 		/* Waiting for wqe resource. */
1077e93c58daSLi Zhang 		rte_delay_us_sleep(MLX5_ASO_WQE_CQE_RESPONSE_DELAY);
1078e93c58daSLi Zhang 	} while (--poll_wqe_times);
10795f0d54f3SLi Zhang 	DRV_LOG(ERR, "Fail to send WQE for ASO meter offset %d",
10805f0d54f3SLi Zhang 			mtr->offset);
1081e93c58daSLi Zhang 	return -1;
1082e93c58daSLi Zhang }
1083e93c58daSLi Zhang 
1084e93c58daSLi Zhang /**
1085e93c58daSLi Zhang  * Wait for meter to be ready.
1086e93c58daSLi Zhang  *
1087e93c58daSLi Zhang  * @param[in] dev
1088e93c58daSLi Zhang  *   Pointer to Ethernet device.
1089e93c58daSLi Zhang  * @param[in] priv
1090e93c58daSLi Zhang  *   Pointer to mlx5 private data structure.
1091e93c58daSLi Zhang  * @param[in] fm
1092e93c58daSLi Zhang  *   Pointer to flow meter to be modified.
1093e93c58daSLi Zhang  *
1094e93c58daSLi Zhang  * @return
1095e93c58daSLi Zhang  *   0 on success, a negative errno value otherwise and rte_errno is set.
1096e93c58daSLi Zhang  */
1097e93c58daSLi Zhang int
10984359d9d1SGregory Etelson mlx5_aso_mtr_wait(struct mlx5_priv *priv,
10994359d9d1SGregory Etelson 		  struct mlx5_aso_mtr *mtr, bool is_tmpl_api)
1100e93c58daSLi Zhang {
110148fbb0e9SAlexander Kozyrev 	bool need_lock;
11024359d9d1SGregory Etelson 	struct mlx5_aso_sq *sq;
11034359d9d1SGregory Etelson 	struct mlx5_dev_ctx_shared *sh = priv->sh;
11044359d9d1SGregory Etelson 	uint32_t poll_cqe_times = MLX5_MTR_POLL_WQE_CQE_TIMES;
1105e12a0166STyler Retzlaff 	uint8_t state = rte_atomic_load_explicit(&mtr->state, rte_memory_order_relaxed);
11064359d9d1SGregory Etelson 	poll_cq_t poll_mtr_cq =
11074359d9d1SGregory Etelson 		is_tmpl_api ? mlx5_aso_poll_cq_mtr_hws : mlx5_aso_poll_cq_mtr_sws;
1108e93c58daSLi Zhang 
1109478ba4bbSSuanming Mou 	if (state == ASO_METER_READY || state == ASO_METER_WAIT_ASYNC)
1110e93c58daSLi Zhang 		return 0;
11114359d9d1SGregory Etelson 	sq = mlx5_aso_mtr_select_sq(sh, MLX5_HW_INV_QUEUE, mtr, &need_lock);
1112e93c58daSLi Zhang 	do {
11134359d9d1SGregory Etelson 		poll_mtr_cq(priv, sq);
1114e12a0166STyler Retzlaff 		if (rte_atomic_load_explicit(&mtr->state, rte_memory_order_relaxed) ==
1115e93c58daSLi Zhang 					    ASO_METER_READY)
1116e93c58daSLi Zhang 			return 0;
1117e93c58daSLi Zhang 		/* Waiting for CQE ready. */
1118e93c58daSLi Zhang 		rte_delay_us_sleep(MLX5_ASO_WQE_CQE_RESPONSE_DELAY);
1119e93c58daSLi Zhang 	} while (--poll_cqe_times);
11205f0d54f3SLi Zhang 	DRV_LOG(ERR, "Fail to poll CQE ready for ASO meter offset %d",
11215f0d54f3SLi Zhang 			mtr->offset);
1122e93c58daSLi Zhang 	return -1;
1123e93c58daSLi Zhang }
1124ebaf1b31SBing Zhao 
1125463170a7SSuanming Mou static inline struct mlx5_aso_sq*
1126463170a7SSuanming Mou __mlx5_aso_ct_get_sq_in_hws(uint32_t queue,
1127463170a7SSuanming Mou 			    struct mlx5_aso_ct_pool *pool)
1128463170a7SSuanming Mou {
1129463170a7SSuanming Mou 	return (queue == MLX5_HW_INV_QUEUE) ?
1130463170a7SSuanming Mou 		pool->shared_sq : &pool->sq[queue];
1131463170a7SSuanming Mou }
1132463170a7SSuanming Mou 
1133463170a7SSuanming Mou static inline struct mlx5_aso_sq*
1134463170a7SSuanming Mou __mlx5_aso_ct_get_sq_in_sws(struct mlx5_dev_ctx_shared *sh,
1135463170a7SSuanming Mou 			    struct mlx5_aso_ct_action *ct)
1136463170a7SSuanming Mou {
1137463170a7SSuanming Mou 	return &sh->ct_mng->aso_sqs[ct->offset & (MLX5_ASO_CT_SQ_NUM - 1)];
1138463170a7SSuanming Mou }
1139463170a7SSuanming Mou 
1140463170a7SSuanming Mou static inline struct mlx5_aso_ct_pool*
1141463170a7SSuanming Mou __mlx5_aso_ct_get_pool(struct mlx5_dev_ctx_shared *sh,
1142463170a7SSuanming Mou 		       struct mlx5_aso_ct_action *ct)
1143463170a7SSuanming Mou {
1144463170a7SSuanming Mou 	if (likely(sh->config.dv_flow_en == 2))
1145463170a7SSuanming Mou 		return ct->pool;
1146463170a7SSuanming Mou 	return container_of(ct, struct mlx5_aso_ct_pool, actions[ct->offset]);
1147463170a7SSuanming Mou }
1148463170a7SSuanming Mou 
1149463170a7SSuanming Mou int
1150463170a7SSuanming Mou mlx5_aso_ct_queue_uninit(struct mlx5_dev_ctx_shared *sh,
1151463170a7SSuanming Mou 			 struct mlx5_aso_ct_pools_mng *ct_mng)
1152463170a7SSuanming Mou {
1153463170a7SSuanming Mou 	uint32_t i;
1154463170a7SSuanming Mou 
1155463170a7SSuanming Mou 	/* 64B per object for query. */
1156463170a7SSuanming Mou 	for (i = 0; i < ct_mng->nb_sq; i++) {
1157463170a7SSuanming Mou 		if (ct_mng->aso_sqs[i].mr.addr)
1158463170a7SSuanming Mou 			mlx5_aso_dereg_mr(sh->cdev, &ct_mng->aso_sqs[i].mr);
1159463170a7SSuanming Mou 		mlx5_aso_destroy_sq(&ct_mng->aso_sqs[i]);
1160463170a7SSuanming Mou 	}
1161463170a7SSuanming Mou 	return 0;
1162463170a7SSuanming Mou }
1163463170a7SSuanming Mou 
1164463170a7SSuanming Mou /**
1165463170a7SSuanming Mou  * API to create and initialize CT Send Queue used for ASO access.
1166463170a7SSuanming Mou  *
1167463170a7SSuanming Mou  * @param[in] sh
1168463170a7SSuanming Mou  *   Pointer to shared device context.
1169463170a7SSuanming Mou  * @param[in] ct_mng
1170463170a7SSuanming Mou  *   Pointer to the CT management struct.
1171463170a7SSuanming Mou  * *param[in] nb_queues
1172463170a7SSuanming Mou  *   Number of queues to be allocated.
1173463170a7SSuanming Mou  *
1174463170a7SSuanming Mou  * @return
1175463170a7SSuanming Mou  *   0 on success, a negative errno value otherwise and rte_errno is set.
1176463170a7SSuanming Mou  */
1177463170a7SSuanming Mou int
1178463170a7SSuanming Mou mlx5_aso_ct_queue_init(struct mlx5_dev_ctx_shared *sh,
1179463170a7SSuanming Mou 		       struct mlx5_aso_ct_pools_mng *ct_mng,
1180463170a7SSuanming Mou 		       uint32_t nb_queues)
1181463170a7SSuanming Mou {
1182463170a7SSuanming Mou 	uint32_t i;
1183463170a7SSuanming Mou 
1184463170a7SSuanming Mou 	/* 64B per object for query. */
1185463170a7SSuanming Mou 	for (i = 0; i < nb_queues; i++) {
1186463170a7SSuanming Mou 		if (mlx5_aso_reg_mr(sh->cdev, 64 * (1 << MLX5_ASO_QUEUE_LOG_DESC),
1187463170a7SSuanming Mou 				    &ct_mng->aso_sqs[i].mr))
1188463170a7SSuanming Mou 			goto error;
1189463170a7SSuanming Mou 		if (mlx5_aso_sq_create(sh->cdev, &ct_mng->aso_sqs[i],
1190463170a7SSuanming Mou 				       sh->tx_uar.obj,
1191463170a7SSuanming Mou 				       MLX5_ASO_QUEUE_LOG_DESC))
1192463170a7SSuanming Mou 			goto error;
1193463170a7SSuanming Mou 		mlx5_aso_ct_init_sq(&ct_mng->aso_sqs[i]);
1194463170a7SSuanming Mou 	}
1195463170a7SSuanming Mou 	ct_mng->nb_sq = nb_queues;
1196463170a7SSuanming Mou 	return 0;
1197463170a7SSuanming Mou error:
1198463170a7SSuanming Mou 	do {
1199463170a7SSuanming Mou 		if (ct_mng->aso_sqs[i].mr.addr)
1200463170a7SSuanming Mou 			mlx5_aso_dereg_mr(sh->cdev, &ct_mng->aso_sqs[i].mr);
1201463170a7SSuanming Mou 		mlx5_aso_destroy_sq(&ct_mng->aso_sqs[i]);
1202463170a7SSuanming Mou 	} while (i--);
1203463170a7SSuanming Mou 	ct_mng->nb_sq = 0;
1204463170a7SSuanming Mou 	return -1;
1205463170a7SSuanming Mou }
1206463170a7SSuanming Mou 
1207ebaf1b31SBing Zhao /*
1208ebaf1b31SBing Zhao  * Post a WQE to the ASO CT SQ to modify the context.
1209ebaf1b31SBing Zhao  *
12105dfa003dSMichael Baum  * @param[in] sh
12115dfa003dSMichael Baum  *   Pointer to shared device context.
1212ebaf1b31SBing Zhao  * @param[in] ct
1213ebaf1b31SBing Zhao  *   Pointer to the generic CT structure related to the context.
1214ebaf1b31SBing Zhao  * @param[in] profile
1215ebaf1b31SBing Zhao  *   Pointer to configuration profile.
1216ebaf1b31SBing Zhao  *
1217ebaf1b31SBing Zhao  * @return
1218ebaf1b31SBing Zhao  *   1 on success (WQE number), 0 on failure.
1219ebaf1b31SBing Zhao  */
1220ebaf1b31SBing Zhao static uint16_t
12215dfa003dSMichael Baum mlx5_aso_ct_sq_enqueue_single(struct mlx5_dev_ctx_shared *sh,
1222463170a7SSuanming Mou 			      struct mlx5_aso_sq *sq,
1223ebaf1b31SBing Zhao 			      struct mlx5_aso_ct_action *ct,
1224463170a7SSuanming Mou 			      const struct rte_flow_action_conntrack *profile,
1225478ba4bbSSuanming Mou 			      bool need_lock,
1226478ba4bbSSuanming Mou 			      void *user_data,
1227478ba4bbSSuanming Mou 			      bool push)
1228ebaf1b31SBing Zhao {
1229ebaf1b31SBing Zhao 	volatile struct mlx5_aso_wqe *wqe = NULL;
1230ebaf1b31SBing Zhao 	uint16_t size = 1 << sq->log_desc_n;
1231ebaf1b31SBing Zhao 	uint16_t mask = size - 1;
1232ebaf1b31SBing Zhao 	uint16_t res;
1233ebaf1b31SBing Zhao 	struct mlx5_aso_ct_pool *pool;
1234ebaf1b31SBing Zhao 	void *desg;
1235ebaf1b31SBing Zhao 	void *orig_dir;
1236ebaf1b31SBing Zhao 	void *reply_dir;
1237ebaf1b31SBing Zhao 
1238463170a7SSuanming Mou 	if (need_lock)
1239ebaf1b31SBing Zhao 		rte_spinlock_lock(&sq->sqsl);
1240ebaf1b31SBing Zhao 	/* Prevent other threads to update the index. */
1241ebaf1b31SBing Zhao 	res = size - (uint16_t)(sq->head - sq->tail);
1242ebaf1b31SBing Zhao 	if (unlikely(!res)) {
1243463170a7SSuanming Mou 		if (need_lock)
1244ebaf1b31SBing Zhao 			rte_spinlock_unlock(&sq->sqsl);
1245ebaf1b31SBing Zhao 		DRV_LOG(ERR, "Fail: SQ is full and no free WQE to send");
1246ebaf1b31SBing Zhao 		return 0;
1247ebaf1b31SBing Zhao 	}
1248ebaf1b31SBing Zhao 	wqe = &sq->sq_obj.aso_wqes[sq->head & mask];
1249ebaf1b31SBing Zhao 	rte_prefetch0(&sq->sq_obj.aso_wqes[(sq->head + 1) & mask]);
1250ebaf1b31SBing Zhao 	/* Fill next WQE. */
1251478ba4bbSSuanming Mou 	MLX5_ASO_CT_UPDATE_STATE(ct,
1252478ba4bbSSuanming Mou 			user_data ? ASO_CONNTRACK_WAIT_ASYNC : ASO_CONNTRACK_WAIT);
1253478ba4bbSSuanming Mou 	if (user_data) {
1254478ba4bbSSuanming Mou 		sq->elts[sq->head & mask].user_data = user_data;
1255478ba4bbSSuanming Mou 	} else {
1256ebaf1b31SBing Zhao 		sq->elts[sq->head & mask].ct = ct;
1257cf756556SBing Zhao 		sq->elts[sq->head & mask].query_data = NULL;
1258478ba4bbSSuanming Mou 	}
1259463170a7SSuanming Mou 	pool = __mlx5_aso_ct_get_pool(sh, ct);
1260478ba4bbSSuanming Mou 
1261ebaf1b31SBing Zhao 	/* Each WQE will have a single CT object. */
1262ebaf1b31SBing Zhao 	wqe->general_cseg.misc = rte_cpu_to_be_32(pool->devx_obj->id +
1263ebaf1b31SBing Zhao 						  ct->offset);
1264ebaf1b31SBing Zhao 	wqe->general_cseg.opcode = rte_cpu_to_be_32(MLX5_OPCODE_ACCESS_ASO |
1265ebaf1b31SBing Zhao 			(ASO_OPC_MOD_CONNECTION_TRACKING <<
1266ebaf1b31SBing Zhao 			 WQE_CSEG_OPC_MOD_OFFSET) |
1267ebaf1b31SBing Zhao 			sq->pi << WQE_CSEG_WQE_INDEX_OFFSET);
1268ebaf1b31SBing Zhao 	wqe->aso_cseg.operand_masks = rte_cpu_to_be_32
1269ebaf1b31SBing Zhao 			(0u |
1270ebaf1b31SBing Zhao 			 (ASO_OPER_LOGICAL_OR << ASO_CSEG_COND_OPER_OFFSET) |
1271ebaf1b31SBing Zhao 			 (ASO_OP_ALWAYS_TRUE << ASO_CSEG_COND_1_OPER_OFFSET) |
1272ebaf1b31SBing Zhao 			 (ASO_OP_ALWAYS_TRUE << ASO_CSEG_COND_0_OPER_OFFSET) |
1273ebaf1b31SBing Zhao 			 (BYTEWISE_64BYTE << ASO_CSEG_DATA_MASK_MODE_OFFSET));
1274ebaf1b31SBing Zhao 	wqe->aso_cseg.data_mask = UINT64_MAX;
1275ebaf1b31SBing Zhao 	/* To make compiler happy. */
1276ebaf1b31SBing Zhao 	desg = (void *)(uintptr_t)wqe->aso_dseg.data;
1277ebaf1b31SBing Zhao 	MLX5_SET(conn_track_aso, desg, valid, 1);
1278ebaf1b31SBing Zhao 	MLX5_SET(conn_track_aso, desg, state, profile->state);
1279ebaf1b31SBing Zhao 	MLX5_SET(conn_track_aso, desg, freeze_track, !profile->enable);
1280ebaf1b31SBing Zhao 	MLX5_SET(conn_track_aso, desg, connection_assured,
1281ebaf1b31SBing Zhao 		 profile->live_connection);
1282ebaf1b31SBing Zhao 	MLX5_SET(conn_track_aso, desg, sack_permitted, profile->selective_ack);
1283ebaf1b31SBing Zhao 	MLX5_SET(conn_track_aso, desg, challenged_acked,
1284ebaf1b31SBing Zhao 		 profile->challenge_ack_passed);
1285ebaf1b31SBing Zhao 	/* Heartbeat, retransmission_counter, retranmission_limit_exceeded: 0 */
1286ebaf1b31SBing Zhao 	MLX5_SET(conn_track_aso, desg, heartbeat, 0);
1287ebaf1b31SBing Zhao 	MLX5_SET(conn_track_aso, desg, max_ack_window,
1288ebaf1b31SBing Zhao 		 profile->max_ack_window);
1289ebaf1b31SBing Zhao 	MLX5_SET(conn_track_aso, desg, retransmission_counter, 0);
1290ebaf1b31SBing Zhao 	MLX5_SET(conn_track_aso, desg, retranmission_limit_exceeded, 0);
1291ebaf1b31SBing Zhao 	MLX5_SET(conn_track_aso, desg, retranmission_limit,
1292ebaf1b31SBing Zhao 		 profile->retransmission_limit);
1293ebaf1b31SBing Zhao 	MLX5_SET(conn_track_aso, desg, reply_direction_tcp_scale,
1294ebaf1b31SBing Zhao 		 profile->reply_dir.scale);
1295ebaf1b31SBing Zhao 	MLX5_SET(conn_track_aso, desg, reply_direction_tcp_close_initiated,
1296ebaf1b31SBing Zhao 		 profile->reply_dir.close_initiated);
1297ebaf1b31SBing Zhao 	/* Both directions will use the same liberal mode. */
1298ebaf1b31SBing Zhao 	MLX5_SET(conn_track_aso, desg, reply_direction_tcp_liberal_enabled,
1299ebaf1b31SBing Zhao 		 profile->liberal_mode);
1300ebaf1b31SBing Zhao 	MLX5_SET(conn_track_aso, desg, reply_direction_tcp_data_unacked,
1301ebaf1b31SBing Zhao 		 profile->reply_dir.data_unacked);
1302ebaf1b31SBing Zhao 	MLX5_SET(conn_track_aso, desg, reply_direction_tcp_max_ack,
1303ebaf1b31SBing Zhao 		 profile->reply_dir.last_ack_seen);
1304ebaf1b31SBing Zhao 	MLX5_SET(conn_track_aso, desg, original_direction_tcp_scale,
1305ebaf1b31SBing Zhao 		 profile->original_dir.scale);
1306ebaf1b31SBing Zhao 	MLX5_SET(conn_track_aso, desg, original_direction_tcp_close_initiated,
1307ebaf1b31SBing Zhao 		 profile->original_dir.close_initiated);
1308ebaf1b31SBing Zhao 	MLX5_SET(conn_track_aso, desg, original_direction_tcp_liberal_enabled,
1309ebaf1b31SBing Zhao 		 profile->liberal_mode);
1310ebaf1b31SBing Zhao 	MLX5_SET(conn_track_aso, desg, original_direction_tcp_data_unacked,
1311ebaf1b31SBing Zhao 		 profile->original_dir.data_unacked);
1312ebaf1b31SBing Zhao 	MLX5_SET(conn_track_aso, desg, original_direction_tcp_max_ack,
1313ebaf1b31SBing Zhao 		 profile->original_dir.last_ack_seen);
1314ebaf1b31SBing Zhao 	MLX5_SET(conn_track_aso, desg, last_win, profile->last_window);
1315ebaf1b31SBing Zhao 	MLX5_SET(conn_track_aso, desg, last_dir, profile->last_direction);
1316ebaf1b31SBing Zhao 	MLX5_SET(conn_track_aso, desg, last_index, profile->last_index);
1317ebaf1b31SBing Zhao 	MLX5_SET(conn_track_aso, desg, last_seq, profile->last_seq);
1318ebaf1b31SBing Zhao 	MLX5_SET(conn_track_aso, desg, last_ack, profile->last_ack);
1319ebaf1b31SBing Zhao 	MLX5_SET(conn_track_aso, desg, last_end, profile->last_end);
1320ebaf1b31SBing Zhao 	orig_dir = MLX5_ADDR_OF(conn_track_aso, desg, original_dir);
1321ebaf1b31SBing Zhao 	MLX5_SET(tcp_window_params, orig_dir, sent_end,
1322ebaf1b31SBing Zhao 		 profile->original_dir.sent_end);
1323ebaf1b31SBing Zhao 	MLX5_SET(tcp_window_params, orig_dir, reply_end,
1324ebaf1b31SBing Zhao 		 profile->original_dir.reply_end);
1325ebaf1b31SBing Zhao 	MLX5_SET(tcp_window_params, orig_dir, max_win,
1326ebaf1b31SBing Zhao 		 profile->original_dir.max_win);
1327ebaf1b31SBing Zhao 	MLX5_SET(tcp_window_params, orig_dir, max_ack,
1328ebaf1b31SBing Zhao 		 profile->original_dir.max_ack);
1329ebaf1b31SBing Zhao 	reply_dir = MLX5_ADDR_OF(conn_track_aso, desg, reply_dir);
1330ebaf1b31SBing Zhao 	MLX5_SET(tcp_window_params, reply_dir, sent_end,
1331ebaf1b31SBing Zhao 		 profile->reply_dir.sent_end);
1332ebaf1b31SBing Zhao 	MLX5_SET(tcp_window_params, reply_dir, reply_end,
1333ebaf1b31SBing Zhao 		 profile->reply_dir.reply_end);
1334ebaf1b31SBing Zhao 	MLX5_SET(tcp_window_params, reply_dir, max_win,
1335ebaf1b31SBing Zhao 		 profile->reply_dir.max_win);
1336ebaf1b31SBing Zhao 	MLX5_SET(tcp_window_params, reply_dir, max_ack,
1337ebaf1b31SBing Zhao 		 profile->reply_dir.max_ack);
1338ebaf1b31SBing Zhao 	sq->head++;
1339ebaf1b31SBing Zhao 	sq->pi += 2; /* Each WQE contains 2 WQEBB's. */
1340478ba4bbSSuanming Mou 	if (push) {
13415dfa003dSMichael Baum 		mlx5_doorbell_ring(&sh->tx_uar.bf_db, *(volatile uint64_t *)wqe,
13425dfa003dSMichael Baum 				   sq->pi, &sq->sq_obj.db_rec[MLX5_SND_DBR],
13435dfa003dSMichael Baum 				   !sh->tx_uar.dbnc);
1344478ba4bbSSuanming Mou 		sq->db_pi = sq->pi;
1345478ba4bbSSuanming Mou 	}
1346478ba4bbSSuanming Mou 	sq->db = wqe;
1347463170a7SSuanming Mou 	if (need_lock)
1348ebaf1b31SBing Zhao 		rte_spinlock_unlock(&sq->sqsl);
1349ebaf1b31SBing Zhao 	return 1;
1350ebaf1b31SBing Zhao }
1351ebaf1b31SBing Zhao 
1352ebaf1b31SBing Zhao /*
1353ebaf1b31SBing Zhao  * Update the status field of CTs to indicate ready to be used by flows.
1354ebaf1b31SBing Zhao  * A continuous number of CTs since last update.
1355ebaf1b31SBing Zhao  *
1356ebaf1b31SBing Zhao  * @param[in] sq
1357ebaf1b31SBing Zhao  *   Pointer to ASO CT SQ.
1358ebaf1b31SBing Zhao  * @param[in] num
1359ebaf1b31SBing Zhao  *   Number of CT structures to be updated.
1360ebaf1b31SBing Zhao  *
1361ebaf1b31SBing Zhao  * @return
1362ebaf1b31SBing Zhao  *   0 on success, a negative value.
1363ebaf1b31SBing Zhao  */
1364ebaf1b31SBing Zhao static void
1365ebaf1b31SBing Zhao mlx5_aso_ct_status_update(struct mlx5_aso_sq *sq, uint16_t num)
1366ebaf1b31SBing Zhao {
1367ebaf1b31SBing Zhao 	uint16_t size = 1 << sq->log_desc_n;
1368ebaf1b31SBing Zhao 	uint16_t mask = size - 1;
1369ebaf1b31SBing Zhao 	uint16_t i;
1370ebaf1b31SBing Zhao 	struct mlx5_aso_ct_action *ct = NULL;
1371ebaf1b31SBing Zhao 	uint16_t idx;
1372ebaf1b31SBing Zhao 
1373ebaf1b31SBing Zhao 	for (i = 0; i < num; i++) {
1374ebaf1b31SBing Zhao 		idx = (uint16_t)((sq->tail + i) & mask);
1375ebaf1b31SBing Zhao 		ct = sq->elts[idx].ct;
1376ebaf1b31SBing Zhao 		MLX5_ASSERT(ct);
1377ebaf1b31SBing Zhao 		MLX5_ASO_CT_UPDATE_STATE(ct, ASO_CONNTRACK_READY);
1378cf756556SBing Zhao 		if (sq->elts[idx].query_data)
1379cf756556SBing Zhao 			rte_memcpy(sq->elts[idx].query_data,
1380cf756556SBing Zhao 				   (char *)((uintptr_t)sq->mr.addr + idx * 64),
1381cf756556SBing Zhao 				   64);
1382ebaf1b31SBing Zhao 	}
1383ebaf1b31SBing Zhao }
1384ebaf1b31SBing Zhao 
1385ebaf1b31SBing Zhao /*
1386cf756556SBing Zhao  * Post a WQE to the ASO CT SQ to query the current context.
1387cf756556SBing Zhao  *
13885dfa003dSMichael Baum  * @param[in] sh
13895dfa003dSMichael Baum  *   Pointer to shared device context.
1390cf756556SBing Zhao  * @param[in] ct
1391cf756556SBing Zhao  *   Pointer to the generic CT structure related to the context.
1392cf756556SBing Zhao  * @param[in] data
1393cf756556SBing Zhao  *   Pointer to data area to be filled.
1394cf756556SBing Zhao  *
1395cf756556SBing Zhao  * @return
1396cf756556SBing Zhao  *   1 on success (WQE number), 0 on failure.
1397cf756556SBing Zhao  */
1398cf756556SBing Zhao static int
13995dfa003dSMichael Baum mlx5_aso_ct_sq_query_single(struct mlx5_dev_ctx_shared *sh,
1400463170a7SSuanming Mou 			    struct mlx5_aso_sq *sq,
1401463170a7SSuanming Mou 			    struct mlx5_aso_ct_action *ct, char *data,
1402478ba4bbSSuanming Mou 			    bool need_lock,
1403478ba4bbSSuanming Mou 			    void *user_data,
1404478ba4bbSSuanming Mou 			    bool push)
1405cf756556SBing Zhao {
1406cf756556SBing Zhao 	volatile struct mlx5_aso_wqe *wqe = NULL;
1407cf756556SBing Zhao 	uint16_t size = 1 << sq->log_desc_n;
1408cf756556SBing Zhao 	uint16_t mask = size - 1;
1409cf756556SBing Zhao 	uint16_t res;
1410cf756556SBing Zhao 	uint16_t wqe_idx;
1411cf756556SBing Zhao 	struct mlx5_aso_ct_pool *pool;
1412cf756556SBing Zhao 	enum mlx5_aso_ct_state state =
1413e12a0166STyler Retzlaff 				rte_atomic_load_explicit(&ct->state, rte_memory_order_relaxed);
1414cf756556SBing Zhao 
1415cf756556SBing Zhao 	if (state == ASO_CONNTRACK_FREE) {
1416cf756556SBing Zhao 		DRV_LOG(ERR, "Fail: No context to query");
1417cf756556SBing Zhao 		return -1;
1418cf756556SBing Zhao 	} else if (state == ASO_CONNTRACK_WAIT) {
1419cf756556SBing Zhao 		return 0;
1420cf756556SBing Zhao 	}
1421463170a7SSuanming Mou 	if (need_lock)
1422cf756556SBing Zhao 		rte_spinlock_lock(&sq->sqsl);
1423cf756556SBing Zhao 	res = size - (uint16_t)(sq->head - sq->tail);
1424cf756556SBing Zhao 	if (unlikely(!res)) {
1425463170a7SSuanming Mou 		if (need_lock)
1426cf756556SBing Zhao 			rte_spinlock_unlock(&sq->sqsl);
1427cf756556SBing Zhao 		DRV_LOG(ERR, "Fail: SQ is full and no free WQE to send");
1428cf756556SBing Zhao 		return 0;
1429cf756556SBing Zhao 	}
1430478ba4bbSSuanming Mou 	MLX5_ASO_CT_UPDATE_STATE(ct,
1431478ba4bbSSuanming Mou 			user_data ? ASO_CONNTRACK_WAIT_ASYNC : ASO_CONNTRACK_QUERY);
1432cf756556SBing Zhao 	wqe = &sq->sq_obj.aso_wqes[sq->head & mask];
1433cf756556SBing Zhao 	/* Confirm the location and address of the prefetch instruction. */
1434cf756556SBing Zhao 	rte_prefetch0(&sq->sq_obj.aso_wqes[(sq->head + 1) & mask]);
1435cf756556SBing Zhao 	/* Fill next WQE. */
1436cf756556SBing Zhao 	wqe_idx = sq->head & mask;
1437478ba4bbSSuanming Mou 	/* Check if this is async mode. */
1438478ba4bbSSuanming Mou 	if (user_data) {
1439478ba4bbSSuanming Mou 		struct mlx5_hw_q_job *job = (struct mlx5_hw_q_job *)user_data;
1440478ba4bbSSuanming Mou 
1441478ba4bbSSuanming Mou 		sq->elts[wqe_idx].ct = user_data;
1442d065ecc4SGregory Etelson 		job->query.hw = (char *)((uintptr_t)sq->mr.addr + wqe_idx * 64);
1443478ba4bbSSuanming Mou 	} else {
1444cf756556SBing Zhao 		sq->elts[wqe_idx].query_data = data;
1445478ba4bbSSuanming Mou 		sq->elts[wqe_idx].ct = ct;
1446478ba4bbSSuanming Mou 	}
1447463170a7SSuanming Mou 	pool = __mlx5_aso_ct_get_pool(sh, ct);
1448cf756556SBing Zhao 	/* Each WQE will have a single CT object. */
1449cf756556SBing Zhao 	wqe->general_cseg.misc = rte_cpu_to_be_32(pool->devx_obj->id +
1450cf756556SBing Zhao 						  ct->offset);
1451cf756556SBing Zhao 	wqe->general_cseg.opcode = rte_cpu_to_be_32(MLX5_OPCODE_ACCESS_ASO |
1452cf756556SBing Zhao 			(ASO_OPC_MOD_CONNECTION_TRACKING <<
1453cf756556SBing Zhao 			 WQE_CSEG_OPC_MOD_OFFSET) |
1454cf756556SBing Zhao 			sq->pi << WQE_CSEG_WQE_INDEX_OFFSET);
1455cf756556SBing Zhao 	/*
1456cf756556SBing Zhao 	 * There is no write request is required.
1457cf756556SBing Zhao 	 * ASO_OPER_LOGICAL_AND and ASO_OP_ALWAYS_FALSE are both 0.
1458cf756556SBing Zhao 	 * "BYTEWISE_64BYTE" is needed for a whole context.
1459cf756556SBing Zhao 	 * Set to 0 directly to reduce an endian swap. (Modify should rewrite.)
1460cf756556SBing Zhao 	 * "data_mask" is ignored.
1461cf756556SBing Zhao 	 * Buffer address was already filled during initialization.
1462cf756556SBing Zhao 	 */
1463cf756556SBing Zhao 	wqe->aso_cseg.operand_masks = rte_cpu_to_be_32(BYTEWISE_64BYTE <<
1464cf756556SBing Zhao 					ASO_CSEG_DATA_MASK_MODE_OFFSET);
1465cf756556SBing Zhao 	wqe->aso_cseg.data_mask = 0;
1466cf756556SBing Zhao 	sq->head++;
1467cf756556SBing Zhao 	/*
1468cf756556SBing Zhao 	 * Each WQE contains 2 WQEBB's, even though
1469cf756556SBing Zhao 	 * data segment is not used in this case.
1470cf756556SBing Zhao 	 */
1471cf756556SBing Zhao 	sq->pi += 2;
1472478ba4bbSSuanming Mou 	if (push) {
14735dfa003dSMichael Baum 		mlx5_doorbell_ring(&sh->tx_uar.bf_db, *(volatile uint64_t *)wqe,
14745dfa003dSMichael Baum 				   sq->pi, &sq->sq_obj.db_rec[MLX5_SND_DBR],
14755dfa003dSMichael Baum 				   !sh->tx_uar.dbnc);
1476478ba4bbSSuanming Mou 		sq->db_pi = sq->pi;
1477478ba4bbSSuanming Mou 	}
1478478ba4bbSSuanming Mou 	sq->db = wqe;
1479463170a7SSuanming Mou 	if (need_lock)
1480cf756556SBing Zhao 		rte_spinlock_unlock(&sq->sqsl);
1481cf756556SBing Zhao 	return 1;
1482cf756556SBing Zhao }
1483cf756556SBing Zhao 
1484cf756556SBing Zhao /*
1485ebaf1b31SBing Zhao  * Handle completions from WQEs sent to ASO CT.
1486ebaf1b31SBing Zhao  *
1487ebaf1b31SBing Zhao  * @param[in] mng
1488ebaf1b31SBing Zhao  *   Pointer to the CT pools management structure.
1489ebaf1b31SBing Zhao  */
1490ebaf1b31SBing Zhao static void
1491463170a7SSuanming Mou mlx5_aso_ct_completion_handle(struct mlx5_dev_ctx_shared *sh __rte_unused,
1492463170a7SSuanming Mou 			      struct mlx5_aso_sq *sq,
1493463170a7SSuanming Mou 			      bool need_lock)
1494ebaf1b31SBing Zhao {
1495ebaf1b31SBing Zhao 	struct mlx5_aso_cq *cq = &sq->cq;
1496ebaf1b31SBing Zhao 	volatile struct mlx5_cqe *restrict cqe;
1497ebaf1b31SBing Zhao 	const uint32_t cq_size = 1 << cq->log_desc_n;
1498ebaf1b31SBing Zhao 	const uint32_t mask = cq_size - 1;
1499ebaf1b31SBing Zhao 	uint32_t idx;
1500ebaf1b31SBing Zhao 	uint32_t next_idx;
1501ebaf1b31SBing Zhao 	uint16_t max;
1502ebaf1b31SBing Zhao 	uint16_t n = 0;
1503ebaf1b31SBing Zhao 	int ret;
1504ebaf1b31SBing Zhao 
1505463170a7SSuanming Mou 	if (need_lock)
1506ebaf1b31SBing Zhao 		rte_spinlock_lock(&sq->sqsl);
1507ebaf1b31SBing Zhao 	max = (uint16_t)(sq->head - sq->tail);
1508ebaf1b31SBing Zhao 	if (unlikely(!max)) {
1509463170a7SSuanming Mou 		if (need_lock)
1510ebaf1b31SBing Zhao 			rte_spinlock_unlock(&sq->sqsl);
1511ebaf1b31SBing Zhao 		return;
1512ebaf1b31SBing Zhao 	}
1513ebaf1b31SBing Zhao 	next_idx = cq->cq_ci & mask;
1514ebaf1b31SBing Zhao 	do {
1515ebaf1b31SBing Zhao 		idx = next_idx;
1516ebaf1b31SBing Zhao 		next_idx = (cq->cq_ci + 1) & mask;
1517ebaf1b31SBing Zhao 		/* Need to confirm the position of the prefetch. */
1518ebaf1b31SBing Zhao 		rte_prefetch0(&cq->cq_obj.cqes[next_idx]);
1519ebaf1b31SBing Zhao 		cqe = &cq->cq_obj.cqes[idx];
1520ebaf1b31SBing Zhao 		ret = check_cqe(cqe, cq_size, cq->cq_ci);
1521ebaf1b31SBing Zhao 		/*
1522ebaf1b31SBing Zhao 		 * Be sure owner read is done before any other cookie field or
1523ebaf1b31SBing Zhao 		 * opaque field.
1524ebaf1b31SBing Zhao 		 */
1525ebaf1b31SBing Zhao 		rte_io_rmb();
1526ebaf1b31SBing Zhao 		if (unlikely(ret != MLX5_CQE_STATUS_SW_OWN)) {
1527ebaf1b31SBing Zhao 			if (likely(ret == MLX5_CQE_STATUS_HW_OWN))
1528ebaf1b31SBing Zhao 				break;
1529ebaf1b31SBing Zhao 			mlx5_aso_cqe_err_handle(sq);
1530ebaf1b31SBing Zhao 		} else {
1531ebaf1b31SBing Zhao 			n++;
1532ebaf1b31SBing Zhao 		}
1533ebaf1b31SBing Zhao 		cq->cq_ci++;
1534ebaf1b31SBing Zhao 	} while (1);
1535ebaf1b31SBing Zhao 	if (likely(n)) {
1536ebaf1b31SBing Zhao 		mlx5_aso_ct_status_update(sq, n);
1537ebaf1b31SBing Zhao 		sq->tail += n;
1538ebaf1b31SBing Zhao 		rte_io_wmb();
1539ebaf1b31SBing Zhao 		cq->cq_obj.db_rec[0] = rte_cpu_to_be_32(cq->cq_ci);
1540ebaf1b31SBing Zhao 	}
1541463170a7SSuanming Mou 	if (need_lock)
1542ebaf1b31SBing Zhao 		rte_spinlock_unlock(&sq->sqsl);
1543ebaf1b31SBing Zhao }
1544ebaf1b31SBing Zhao 
1545ebaf1b31SBing Zhao /*
1546ebaf1b31SBing Zhao  * Update connection tracking ASO context by sending WQE.
1547ebaf1b31SBing Zhao  *
1548ebaf1b31SBing Zhao  * @param[in] sh
1549ebaf1b31SBing Zhao  *   Pointer to mlx5_dev_ctx_shared object.
1550463170a7SSuanming Mou  * @param[in] queue
1551463170a7SSuanming Mou  *   The queue index.
1552ebaf1b31SBing Zhao  * @param[in] ct
1553ebaf1b31SBing Zhao  *   Pointer to connection tracking offload object.
1554ebaf1b31SBing Zhao  * @param[in] profile
1555ebaf1b31SBing Zhao  *   Pointer to connection tracking TCP parameter.
1556ebaf1b31SBing Zhao  *
1557ebaf1b31SBing Zhao  * @return
1558ebaf1b31SBing Zhao  *   0 on success, -1 on failure.
1559ebaf1b31SBing Zhao  */
1560ebaf1b31SBing Zhao int
1561ebaf1b31SBing Zhao mlx5_aso_ct_update_by_wqe(struct mlx5_dev_ctx_shared *sh,
1562463170a7SSuanming Mou 			  uint32_t queue,
1563ebaf1b31SBing Zhao 			  struct mlx5_aso_ct_action *ct,
1564478ba4bbSSuanming Mou 			  const struct rte_flow_action_conntrack *profile,
1565478ba4bbSSuanming Mou 			  void *user_data,
1566478ba4bbSSuanming Mou 			  bool push)
1567ebaf1b31SBing Zhao {
1568ebaf1b31SBing Zhao 	uint32_t poll_wqe_times = MLX5_CT_POLL_WQE_CQE_TIMES;
1569463170a7SSuanming Mou 	struct mlx5_aso_ct_pool *pool = __mlx5_aso_ct_get_pool(sh, ct);
1570463170a7SSuanming Mou 	struct mlx5_aso_sq *sq;
1571463170a7SSuanming Mou 	bool need_lock = !!(queue == MLX5_HW_INV_QUEUE);
1572478ba4bbSSuanming Mou 	int ret;
1573ebaf1b31SBing Zhao 
1574463170a7SSuanming Mou 	if (sh->config.dv_flow_en == 2)
1575463170a7SSuanming Mou 		sq = __mlx5_aso_ct_get_sq_in_hws(queue, pool);
1576463170a7SSuanming Mou 	else
1577463170a7SSuanming Mou 		sq = __mlx5_aso_ct_get_sq_in_sws(sh, ct);
1578478ba4bbSSuanming Mou 	if (queue != MLX5_HW_INV_QUEUE) {
1579478ba4bbSSuanming Mou 		ret = mlx5_aso_ct_sq_enqueue_single(sh, sq, ct, profile,
1580478ba4bbSSuanming Mou 						    need_lock, user_data, push);
1581478ba4bbSSuanming Mou 		return ret > 0 ? 0 : -1;
1582478ba4bbSSuanming Mou 	}
1583ebaf1b31SBing Zhao 	do {
1584463170a7SSuanming Mou 		mlx5_aso_ct_completion_handle(sh, sq,  need_lock);
1585478ba4bbSSuanming Mou 		if (mlx5_aso_ct_sq_enqueue_single(sh, sq, ct, profile,
1586478ba4bbSSuanming Mou 						  need_lock, NULL, true))
1587ebaf1b31SBing Zhao 			return 0;
1588ebaf1b31SBing Zhao 		/* Waiting for wqe resource. */
1589ebaf1b31SBing Zhao 		rte_delay_us_sleep(10u);
1590ebaf1b31SBing Zhao 	} while (--poll_wqe_times);
1591ebaf1b31SBing Zhao 	DRV_LOG(ERR, "Fail to send WQE for ASO CT %d in pool %d",
1592ebaf1b31SBing Zhao 		ct->offset, pool->index);
1593ebaf1b31SBing Zhao 	return -1;
1594ebaf1b31SBing Zhao }
1595cf756556SBing Zhao 
1596cf756556SBing Zhao /*
1597cf756556SBing Zhao  * The routine is used to wait for WQE completion to continue with queried data.
1598cf756556SBing Zhao  *
1599cf756556SBing Zhao  * @param[in] sh
1600cf756556SBing Zhao  *   Pointer to mlx5_dev_ctx_shared object.
1601463170a7SSuanming Mou  * @param[in] queue
1602463170a7SSuanming Mou  *   The queue which CT works on..
1603cf756556SBing Zhao  * @param[in] ct
1604cf756556SBing Zhao  *   Pointer to connection tracking offload object.
1605cf756556SBing Zhao  *
1606cf756556SBing Zhao  * @return
1607cf756556SBing Zhao  *   0 on success, -1 on failure.
1608cf756556SBing Zhao  */
1609cf756556SBing Zhao int
1610463170a7SSuanming Mou mlx5_aso_ct_wait_ready(struct mlx5_dev_ctx_shared *sh, uint32_t queue,
1611cf756556SBing Zhao 		       struct mlx5_aso_ct_action *ct)
1612cf756556SBing Zhao {
1613cf756556SBing Zhao 	uint32_t poll_cqe_times = MLX5_CT_POLL_WQE_CQE_TIMES;
1614463170a7SSuanming Mou 	struct mlx5_aso_ct_pool *pool = __mlx5_aso_ct_get_pool(sh, ct);
1615463170a7SSuanming Mou 	struct mlx5_aso_sq *sq;
1616463170a7SSuanming Mou 	bool need_lock = !!(queue == MLX5_HW_INV_QUEUE);
1617cf756556SBing Zhao 
1618463170a7SSuanming Mou 	if (sh->config.dv_flow_en == 2)
1619463170a7SSuanming Mou 		sq = __mlx5_aso_ct_get_sq_in_hws(queue, pool);
1620463170a7SSuanming Mou 	else
1621463170a7SSuanming Mou 		sq = __mlx5_aso_ct_get_sq_in_sws(sh, ct);
1622e12a0166STyler Retzlaff 	if (rte_atomic_load_explicit(&ct->state, rte_memory_order_relaxed) ==
1623cf756556SBing Zhao 	    ASO_CONNTRACK_READY)
1624cf756556SBing Zhao 		return 0;
1625cf756556SBing Zhao 	do {
1626463170a7SSuanming Mou 		mlx5_aso_ct_completion_handle(sh, sq, need_lock);
1627e12a0166STyler Retzlaff 		if (rte_atomic_load_explicit(&ct->state, rte_memory_order_relaxed) ==
1628cf756556SBing Zhao 		    ASO_CONNTRACK_READY)
1629cf756556SBing Zhao 			return 0;
1630cf756556SBing Zhao 		/* Waiting for CQE ready, consider should block or sleep. */
1631cf756556SBing Zhao 		rte_delay_us_sleep(MLX5_ASO_WQE_CQE_RESPONSE_DELAY);
1632cf756556SBing Zhao 	} while (--poll_cqe_times);
1633cf756556SBing Zhao 	DRV_LOG(ERR, "Fail to poll CQE for ASO CT %d in pool %d",
1634cf756556SBing Zhao 		ct->offset, pool->index);
1635cf756556SBing Zhao 	return -1;
1636cf756556SBing Zhao }
1637cf756556SBing Zhao 
1638cf756556SBing Zhao /*
1639cf756556SBing Zhao  * Convert the hardware conntrack data format into the profile.
1640cf756556SBing Zhao  *
1641cf756556SBing Zhao  * @param[in] profile
1642cf756556SBing Zhao  *   Pointer to conntrack profile to be filled after query.
1643cf756556SBing Zhao  * @param[in] wdata
1644cf756556SBing Zhao  *   Pointer to data fetched from hardware.
1645cf756556SBing Zhao  */
1646478ba4bbSSuanming Mou void
1647cf756556SBing Zhao mlx5_aso_ct_obj_analyze(struct rte_flow_action_conntrack *profile,
1648cf756556SBing Zhao 			char *wdata)
1649cf756556SBing Zhao {
1650cf756556SBing Zhao 	void *o_dir = MLX5_ADDR_OF(conn_track_aso, wdata, original_dir);
1651cf756556SBing Zhao 	void *r_dir = MLX5_ADDR_OF(conn_track_aso, wdata, reply_dir);
1652cf756556SBing Zhao 
1653cf756556SBing Zhao 	/* MLX5_GET16 should be taken into consideration. */
1654cf756556SBing Zhao 	profile->state = (enum rte_flow_conntrack_state)
1655cf756556SBing Zhao 			 MLX5_GET(conn_track_aso, wdata, state);
1656cf756556SBing Zhao 	profile->enable = !MLX5_GET(conn_track_aso, wdata, freeze_track);
1657cf756556SBing Zhao 	profile->selective_ack = MLX5_GET(conn_track_aso, wdata,
1658cf756556SBing Zhao 					  sack_permitted);
1659cf756556SBing Zhao 	profile->live_connection = MLX5_GET(conn_track_aso, wdata,
1660cf756556SBing Zhao 					    connection_assured);
1661cf756556SBing Zhao 	profile->challenge_ack_passed = MLX5_GET(conn_track_aso, wdata,
1662cf756556SBing Zhao 						 challenged_acked);
1663cf756556SBing Zhao 	profile->max_ack_window = MLX5_GET(conn_track_aso, wdata,
1664cf756556SBing Zhao 					   max_ack_window);
1665cf756556SBing Zhao 	profile->retransmission_limit = MLX5_GET(conn_track_aso, wdata,
1666cf756556SBing Zhao 						 retranmission_limit);
1667cf756556SBing Zhao 	profile->last_window = MLX5_GET(conn_track_aso, wdata, last_win);
1668cf756556SBing Zhao 	profile->last_direction = MLX5_GET(conn_track_aso, wdata, last_dir);
1669cf756556SBing Zhao 	profile->last_index = (enum rte_flow_conntrack_tcp_last_index)
1670cf756556SBing Zhao 			      MLX5_GET(conn_track_aso, wdata, last_index);
1671cf756556SBing Zhao 	profile->last_seq = MLX5_GET(conn_track_aso, wdata, last_seq);
1672cf756556SBing Zhao 	profile->last_ack = MLX5_GET(conn_track_aso, wdata, last_ack);
1673cf756556SBing Zhao 	profile->last_end = MLX5_GET(conn_track_aso, wdata, last_end);
1674cf756556SBing Zhao 	profile->liberal_mode = MLX5_GET(conn_track_aso, wdata,
1675cf756556SBing Zhao 				reply_direction_tcp_liberal_enabled) |
1676cf756556SBing Zhao 				MLX5_GET(conn_track_aso, wdata,
1677cf756556SBing Zhao 				original_direction_tcp_liberal_enabled);
1678cf756556SBing Zhao 	/* No liberal in the RTE structure profile. */
1679cf756556SBing Zhao 	profile->reply_dir.scale = MLX5_GET(conn_track_aso, wdata,
1680cf756556SBing Zhao 					    reply_direction_tcp_scale);
1681cf756556SBing Zhao 	profile->reply_dir.close_initiated = MLX5_GET(conn_track_aso, wdata,
1682cf756556SBing Zhao 					reply_direction_tcp_close_initiated);
1683cf756556SBing Zhao 	profile->reply_dir.data_unacked = MLX5_GET(conn_track_aso, wdata,
1684cf756556SBing Zhao 					reply_direction_tcp_data_unacked);
1685cf756556SBing Zhao 	profile->reply_dir.last_ack_seen = MLX5_GET(conn_track_aso, wdata,
1686cf756556SBing Zhao 					reply_direction_tcp_max_ack);
1687cf756556SBing Zhao 	profile->reply_dir.sent_end = MLX5_GET(tcp_window_params,
1688cf756556SBing Zhao 					       r_dir, sent_end);
1689cf756556SBing Zhao 	profile->reply_dir.reply_end = MLX5_GET(tcp_window_params,
1690cf756556SBing Zhao 						r_dir, reply_end);
1691cf756556SBing Zhao 	profile->reply_dir.max_win = MLX5_GET(tcp_window_params,
1692cf756556SBing Zhao 					      r_dir, max_win);
1693cf756556SBing Zhao 	profile->reply_dir.max_ack = MLX5_GET(tcp_window_params,
1694cf756556SBing Zhao 					      r_dir, max_ack);
1695cf756556SBing Zhao 	profile->original_dir.scale = MLX5_GET(conn_track_aso, wdata,
1696cf756556SBing Zhao 					       original_direction_tcp_scale);
1697cf756556SBing Zhao 	profile->original_dir.close_initiated = MLX5_GET(conn_track_aso, wdata,
1698cf756556SBing Zhao 					original_direction_tcp_close_initiated);
1699cf756556SBing Zhao 	profile->original_dir.data_unacked = MLX5_GET(conn_track_aso, wdata,
1700cf756556SBing Zhao 					original_direction_tcp_data_unacked);
1701cf756556SBing Zhao 	profile->original_dir.last_ack_seen = MLX5_GET(conn_track_aso, wdata,
1702cf756556SBing Zhao 					original_direction_tcp_max_ack);
1703cf756556SBing Zhao 	profile->original_dir.sent_end = MLX5_GET(tcp_window_params,
1704cf756556SBing Zhao 						  o_dir, sent_end);
1705cf756556SBing Zhao 	profile->original_dir.reply_end = MLX5_GET(tcp_window_params,
1706cf756556SBing Zhao 						   o_dir, reply_end);
1707cf756556SBing Zhao 	profile->original_dir.max_win = MLX5_GET(tcp_window_params,
1708cf756556SBing Zhao 						 o_dir, max_win);
1709cf756556SBing Zhao 	profile->original_dir.max_ack = MLX5_GET(tcp_window_params,
1710cf756556SBing Zhao 						 o_dir, max_ack);
1711cf756556SBing Zhao }
1712cf756556SBing Zhao 
1713cf756556SBing Zhao /*
1714cf756556SBing Zhao  * Query connection tracking information parameter by send WQE.
1715cf756556SBing Zhao  *
1716cf756556SBing Zhao  * @param[in] dev
1717cf756556SBing Zhao  *   Pointer to Ethernet device.
1718cf756556SBing Zhao  * @param[in] ct
1719cf756556SBing Zhao  *   Pointer to connection tracking offload object.
1720cf756556SBing Zhao  * @param[out] profile
1721cf756556SBing Zhao  *   Pointer to connection tracking TCP information.
1722cf756556SBing Zhao  *
1723cf756556SBing Zhao  * @return
1724cf756556SBing Zhao  *   0 on success, -1 on failure.
1725cf756556SBing Zhao  */
1726cf756556SBing Zhao int
1727cf756556SBing Zhao mlx5_aso_ct_query_by_wqe(struct mlx5_dev_ctx_shared *sh,
1728463170a7SSuanming Mou 			 uint32_t queue,
1729cf756556SBing Zhao 			 struct mlx5_aso_ct_action *ct,
1730478ba4bbSSuanming Mou 			 struct rte_flow_action_conntrack *profile,
1731478ba4bbSSuanming Mou 			 void *user_data, bool push)
1732cf756556SBing Zhao {
1733cf756556SBing Zhao 	uint32_t poll_wqe_times = MLX5_CT_POLL_WQE_CQE_TIMES;
1734463170a7SSuanming Mou 	struct mlx5_aso_ct_pool *pool = __mlx5_aso_ct_get_pool(sh, ct);
1735463170a7SSuanming Mou 	struct mlx5_aso_sq *sq;
1736463170a7SSuanming Mou 	bool need_lock = !!(queue == MLX5_HW_INV_QUEUE);
1737cf756556SBing Zhao 	char out_data[64 * 2];
1738cf756556SBing Zhao 	int ret;
1739cf756556SBing Zhao 
1740463170a7SSuanming Mou 	if (sh->config.dv_flow_en == 2)
1741463170a7SSuanming Mou 		sq = __mlx5_aso_ct_get_sq_in_hws(queue, pool);
1742463170a7SSuanming Mou 	else
1743463170a7SSuanming Mou 		sq = __mlx5_aso_ct_get_sq_in_sws(sh, ct);
1744478ba4bbSSuanming Mou 	if (queue != MLX5_HW_INV_QUEUE) {
1745478ba4bbSSuanming Mou 		ret = mlx5_aso_ct_sq_query_single(sh, sq, ct, out_data,
1746478ba4bbSSuanming Mou 						  need_lock, user_data, push);
1747478ba4bbSSuanming Mou 		return ret > 0 ? 0 : -1;
1748478ba4bbSSuanming Mou 	}
1749cf756556SBing Zhao 	do {
1750463170a7SSuanming Mou 		mlx5_aso_ct_completion_handle(sh, sq, need_lock);
1751478ba4bbSSuanming Mou 		ret = mlx5_aso_ct_sq_query_single(sh, sq, ct, out_data,
1752478ba4bbSSuanming Mou 				need_lock, NULL, true);
1753cf756556SBing Zhao 		if (ret < 0)
1754cf756556SBing Zhao 			return ret;
1755cf756556SBing Zhao 		else if (ret > 0)
1756cf756556SBing Zhao 			goto data_handle;
1757cf756556SBing Zhao 		/* Waiting for wqe resource or state. */
1758cf756556SBing Zhao 		else
1759cf756556SBing Zhao 			rte_delay_us_sleep(10u);
1760cf756556SBing Zhao 	} while (--poll_wqe_times);
1761cf756556SBing Zhao 	DRV_LOG(ERR, "Fail to send WQE for ASO CT %d in pool %d",
1762cf756556SBing Zhao 		ct->offset, pool->index);
1763cf756556SBing Zhao 	return -1;
1764cf756556SBing Zhao data_handle:
1765463170a7SSuanming Mou 	ret = mlx5_aso_ct_wait_ready(sh, queue, ct);
1766cf756556SBing Zhao 	if (!ret)
1767cf756556SBing Zhao 		mlx5_aso_ct_obj_analyze(profile, out_data);
1768cf756556SBing Zhao 	return ret;
1769cf756556SBing Zhao }
17702d084f69SBing Zhao 
17712d084f69SBing Zhao /*
17722d084f69SBing Zhao  * Make sure the conntrack context is synchronized with hardware before
17732d084f69SBing Zhao  * creating a flow rule that uses it.
17742d084f69SBing Zhao  *
17752d084f69SBing Zhao  * @param[in] sh
17762d084f69SBing Zhao  *   Pointer to shared device context.
17772d084f69SBing Zhao  * @param[in] ct
17782d084f69SBing Zhao  *   Pointer to connection tracking offload object.
17792d084f69SBing Zhao  *
17802d084f69SBing Zhao  * @return
17812d084f69SBing Zhao  *   0 on success, a negative errno value otherwise and rte_errno is set.
17822d084f69SBing Zhao  */
17832d084f69SBing Zhao int
17842d084f69SBing Zhao mlx5_aso_ct_available(struct mlx5_dev_ctx_shared *sh,
1785463170a7SSuanming Mou 		      uint32_t queue,
17862d084f69SBing Zhao 		      struct mlx5_aso_ct_action *ct)
17872d084f69SBing Zhao {
1788463170a7SSuanming Mou 	struct mlx5_aso_ct_pool *pool = __mlx5_aso_ct_get_pool(sh, ct);
1789463170a7SSuanming Mou 	struct mlx5_aso_sq *sq;
1790463170a7SSuanming Mou 	bool need_lock = !!(queue == MLX5_HW_INV_QUEUE);
17912d084f69SBing Zhao 	uint32_t poll_cqe_times = MLX5_CT_POLL_WQE_CQE_TIMES;
17922d084f69SBing Zhao 	enum mlx5_aso_ct_state state =
1793e12a0166STyler Retzlaff 				rte_atomic_load_explicit(&ct->state, rte_memory_order_relaxed);
17942d084f69SBing Zhao 
1795463170a7SSuanming Mou 	if (sh->config.dv_flow_en == 2)
1796463170a7SSuanming Mou 		sq = __mlx5_aso_ct_get_sq_in_hws(queue, pool);
1797463170a7SSuanming Mou 	else
1798463170a7SSuanming Mou 		sq = __mlx5_aso_ct_get_sq_in_sws(sh, ct);
17992d084f69SBing Zhao 	if (state == ASO_CONNTRACK_FREE) {
18002d084f69SBing Zhao 		rte_errno = ENXIO;
18012d084f69SBing Zhao 		return -rte_errno;
18022d084f69SBing Zhao 	} else if (state == ASO_CONNTRACK_READY ||
1803478ba4bbSSuanming Mou 		   state == ASO_CONNTRACK_QUERY ||
1804478ba4bbSSuanming Mou 		   state == ASO_CONNTRACK_WAIT_ASYNC) {
18052d084f69SBing Zhao 		return 0;
18062d084f69SBing Zhao 	}
18072d084f69SBing Zhao 	do {
1808463170a7SSuanming Mou 		mlx5_aso_ct_completion_handle(sh, sq, need_lock);
1809e12a0166STyler Retzlaff 		state = rte_atomic_load_explicit(&ct->state, rte_memory_order_relaxed);
18102d084f69SBing Zhao 		if (state == ASO_CONNTRACK_READY ||
18112d084f69SBing Zhao 		    state == ASO_CONNTRACK_QUERY)
18122d084f69SBing Zhao 			return 0;
18132d084f69SBing Zhao 		/* Waiting for CQE ready, consider should block or sleep.  */
1814463170a7SSuanming Mou 		rte_delay_us_block(MLX5_ASO_WQE_CQE_RESPONSE_DELAY);
18152d084f69SBing Zhao 	} while (--poll_cqe_times);
18162d084f69SBing Zhao 	rte_errno = EBUSY;
18172d084f69SBing Zhao 	return -rte_errno;
18182d084f69SBing Zhao }
18194d368e1dSXiaoyu Min 
18204d368e1dSXiaoyu Min int
18214d368e1dSXiaoyu Min mlx5_aso_cnt_queue_init(struct mlx5_dev_ctx_shared *sh)
18224d368e1dSXiaoyu Min {
18234d368e1dSXiaoyu Min 	struct mlx5_hws_aso_mng *aso_mng = NULL;
18244d368e1dSXiaoyu Min 	uint8_t idx;
18254d368e1dSXiaoyu Min 	struct mlx5_aso_sq *sq;
18264d368e1dSXiaoyu Min 
18274d368e1dSXiaoyu Min 	MLX5_ASSERT(sh);
18284d368e1dSXiaoyu Min 	MLX5_ASSERT(sh->cnt_svc);
18294d368e1dSXiaoyu Min 	aso_mng = &sh->cnt_svc->aso_mng;
18304d368e1dSXiaoyu Min 	aso_mng->sq_num = HWS_CNT_ASO_SQ_NUM;
18314d368e1dSXiaoyu Min 	for (idx = 0; idx < HWS_CNT_ASO_SQ_NUM; idx++) {
18324d368e1dSXiaoyu Min 		sq = &aso_mng->sqs[idx];
18334d368e1dSXiaoyu Min 		if (mlx5_aso_sq_create(sh->cdev, sq, sh->tx_uar.obj,
18344d368e1dSXiaoyu Min 					MLX5_ASO_CNT_QUEUE_LOG_DESC))
18354d368e1dSXiaoyu Min 			goto error;
18364d368e1dSXiaoyu Min 		mlx5_aso_cnt_init_sq(sq);
18374d368e1dSXiaoyu Min 	}
18384d368e1dSXiaoyu Min 	return 0;
18394d368e1dSXiaoyu Min error:
18404d368e1dSXiaoyu Min 	mlx5_aso_cnt_queue_uninit(sh);
18414d368e1dSXiaoyu Min 	return -1;
18424d368e1dSXiaoyu Min }
18434d368e1dSXiaoyu Min 
18444d368e1dSXiaoyu Min void
18454d368e1dSXiaoyu Min mlx5_aso_cnt_queue_uninit(struct mlx5_dev_ctx_shared *sh)
18464d368e1dSXiaoyu Min {
18474d368e1dSXiaoyu Min 	uint16_t idx;
18484d368e1dSXiaoyu Min 
18494d368e1dSXiaoyu Min 	for (idx = 0; idx < sh->cnt_svc->aso_mng.sq_num; idx++)
18504d368e1dSXiaoyu Min 		mlx5_aso_destroy_sq(&sh->cnt_svc->aso_mng.sqs[idx]);
18514d368e1dSXiaoyu Min 	sh->cnt_svc->aso_mng.sq_num = 0;
18524d368e1dSXiaoyu Min }
18534d368e1dSXiaoyu Min 
18544d368e1dSXiaoyu Min static uint16_t
18554d368e1dSXiaoyu Min mlx5_aso_cnt_sq_enqueue_burst(struct mlx5_hws_cnt_pool *cpool,
18564d368e1dSXiaoyu Min 		struct mlx5_dev_ctx_shared *sh,
18574d368e1dSXiaoyu Min 		struct mlx5_aso_sq *sq, uint32_t n,
18584d368e1dSXiaoyu Min 		uint32_t offset, uint32_t dcs_id_base)
18594d368e1dSXiaoyu Min {
18604d368e1dSXiaoyu Min 	volatile struct mlx5_aso_wqe *wqe;
18614d368e1dSXiaoyu Min 	uint16_t size = 1 << sq->log_desc_n;
18624d368e1dSXiaoyu Min 	uint16_t mask = size - 1;
18634d368e1dSXiaoyu Min 	uint16_t max;
18644d368e1dSXiaoyu Min 	uint32_t upper_offset = offset;
18654d368e1dSXiaoyu Min 	uint64_t addr;
18664d368e1dSXiaoyu Min 	uint32_t ctrl_gen_id = 0;
18674d368e1dSXiaoyu Min 	uint8_t opcmod = sh->cdev->config.hca_attr.flow_access_aso_opc_mod;
18684d368e1dSXiaoyu Min 	rte_be32_t lkey = rte_cpu_to_be_32(cpool->raw_mng->mr.lkey);
18694d368e1dSXiaoyu Min 	uint16_t aso_n = (uint16_t)(RTE_ALIGN_CEIL(n, 4) / 4);
18704d368e1dSXiaoyu Min 	uint32_t ccntid;
18714d368e1dSXiaoyu Min 
18724d368e1dSXiaoyu Min 	max = RTE_MIN(size - (uint16_t)(sq->head - sq->tail), aso_n);
18734d368e1dSXiaoyu Min 	if (unlikely(!max))
18744d368e1dSXiaoyu Min 		return 0;
18754d368e1dSXiaoyu Min 	upper_offset += (max * 4);
18764d368e1dSXiaoyu Min 	/* Because only one burst at one time, we can use the same elt. */
18774d368e1dSXiaoyu Min 	sq->elts[0].burst_size = max;
18784d368e1dSXiaoyu Min 	ctrl_gen_id = dcs_id_base;
18794d368e1dSXiaoyu Min 	ctrl_gen_id /= 4;
18804d368e1dSXiaoyu Min 	do {
18814d368e1dSXiaoyu Min 		ccntid = upper_offset - max * 4;
18824d368e1dSXiaoyu Min 		wqe = &sq->sq_obj.aso_wqes[sq->head & mask];
18834d368e1dSXiaoyu Min 		rte_prefetch0(&sq->sq_obj.aso_wqes[(sq->head + 1) & mask]);
18844d368e1dSXiaoyu Min 		wqe->general_cseg.misc = rte_cpu_to_be_32(ctrl_gen_id);
18854d368e1dSXiaoyu Min 		wqe->general_cseg.flags = RTE_BE32(MLX5_COMP_ONLY_FIRST_ERR <<
18864d368e1dSXiaoyu Min 							 MLX5_COMP_MODE_OFFSET);
18874d368e1dSXiaoyu Min 		wqe->general_cseg.opcode = rte_cpu_to_be_32
18884d368e1dSXiaoyu Min 						(MLX5_OPCODE_ACCESS_ASO |
18894d368e1dSXiaoyu Min 						 (opcmod <<
18904d368e1dSXiaoyu Min 						  WQE_CSEG_OPC_MOD_OFFSET) |
18914d368e1dSXiaoyu Min 						 (sq->pi <<
18924d368e1dSXiaoyu Min 						  WQE_CSEG_WQE_INDEX_OFFSET));
18934d368e1dSXiaoyu Min 		addr = (uint64_t)RTE_PTR_ADD(cpool->raw_mng->raw,
18944d368e1dSXiaoyu Min 				ccntid * sizeof(struct flow_counter_stats));
18954d368e1dSXiaoyu Min 		wqe->aso_cseg.va_h = rte_cpu_to_be_32((uint32_t)(addr >> 32));
18964d368e1dSXiaoyu Min 		wqe->aso_cseg.va_l_r = rte_cpu_to_be_32((uint32_t)addr | 1u);
18974d368e1dSXiaoyu Min 		wqe->aso_cseg.lkey = lkey;
18984d368e1dSXiaoyu Min 		sq->pi += 2; /* Each WQE contains 2 WQEBB's. */
18994d368e1dSXiaoyu Min 		sq->head++;
19004d368e1dSXiaoyu Min 		sq->next++;
19014d368e1dSXiaoyu Min 		ctrl_gen_id++;
19024d368e1dSXiaoyu Min 		max--;
19034d368e1dSXiaoyu Min 	} while (max);
19044d368e1dSXiaoyu Min 	wqe->general_cseg.flags = RTE_BE32(MLX5_COMP_ALWAYS <<
19054d368e1dSXiaoyu Min 							 MLX5_COMP_MODE_OFFSET);
19064d368e1dSXiaoyu Min 	mlx5_doorbell_ring(&sh->tx_uar.bf_db, *(volatile uint64_t *)wqe,
19074d368e1dSXiaoyu Min 			   sq->pi, &sq->sq_obj.db_rec[MLX5_SND_DBR],
19084d368e1dSXiaoyu Min 			   !sh->tx_uar.dbnc);
19094d368e1dSXiaoyu Min 	return sq->elts[0].burst_size;
19104d368e1dSXiaoyu Min }
19114d368e1dSXiaoyu Min 
19124d368e1dSXiaoyu Min static uint16_t
19134d368e1dSXiaoyu Min mlx5_aso_cnt_completion_handle(struct mlx5_aso_sq *sq)
19144d368e1dSXiaoyu Min {
19154d368e1dSXiaoyu Min 	struct mlx5_aso_cq *cq = &sq->cq;
19164d368e1dSXiaoyu Min 	volatile struct mlx5_cqe *restrict cqe;
19174d368e1dSXiaoyu Min 	const unsigned int cq_size = 1 << cq->log_desc_n;
19184d368e1dSXiaoyu Min 	const unsigned int mask = cq_size - 1;
19194d368e1dSXiaoyu Min 	uint32_t idx;
19204d368e1dSXiaoyu Min 	uint32_t next_idx = cq->cq_ci & mask;
19214d368e1dSXiaoyu Min 	const uint16_t max = (uint16_t)(sq->head - sq->tail);
19224d368e1dSXiaoyu Min 	uint16_t i = 0;
19234d368e1dSXiaoyu Min 	int ret;
19244d368e1dSXiaoyu Min 	if (unlikely(!max))
19254d368e1dSXiaoyu Min 		return 0;
19264d368e1dSXiaoyu Min 	idx = next_idx;
19274d368e1dSXiaoyu Min 	next_idx = (cq->cq_ci + 1) & mask;
19284d368e1dSXiaoyu Min 	rte_prefetch0(&cq->cq_obj.cqes[next_idx]);
19294d368e1dSXiaoyu Min 	cqe = &cq->cq_obj.cqes[idx];
19304d368e1dSXiaoyu Min 	ret = check_cqe(cqe, cq_size, cq->cq_ci);
19314d368e1dSXiaoyu Min 	/*
19324d368e1dSXiaoyu Min 	 * Be sure owner read is done before any other cookie field or
19334d368e1dSXiaoyu Min 	 * opaque field.
19344d368e1dSXiaoyu Min 	 */
19354d368e1dSXiaoyu Min 	rte_io_rmb();
19364d368e1dSXiaoyu Min 	if (unlikely(ret != MLX5_CQE_STATUS_SW_OWN)) {
19374d368e1dSXiaoyu Min 		if (likely(ret == MLX5_CQE_STATUS_HW_OWN))
19384d368e1dSXiaoyu Min 			return 0; /* return immediately. */
19394d368e1dSXiaoyu Min 		mlx5_aso_cqe_err_handle(sq);
19404d368e1dSXiaoyu Min 	}
19414d368e1dSXiaoyu Min 	i += sq->elts[0].burst_size;
19424d368e1dSXiaoyu Min 	sq->elts[0].burst_size = 0;
19434d368e1dSXiaoyu Min 	cq->cq_ci++;
19444d368e1dSXiaoyu Min 	if (likely(i)) {
19454d368e1dSXiaoyu Min 		sq->tail += i;
19464d368e1dSXiaoyu Min 		rte_io_wmb();
19474d368e1dSXiaoyu Min 		cq->cq_obj.db_rec[0] = rte_cpu_to_be_32(cq->cq_ci);
19484d368e1dSXiaoyu Min 	}
19494d368e1dSXiaoyu Min 	return i;
19504d368e1dSXiaoyu Min }
19514d368e1dSXiaoyu Min 
19524d368e1dSXiaoyu Min static uint16_t
19534d368e1dSXiaoyu Min mlx5_aso_cnt_query_one_dcs(struct mlx5_dev_ctx_shared *sh,
19544d368e1dSXiaoyu Min 			   struct mlx5_hws_cnt_pool *cpool,
19554d368e1dSXiaoyu Min 			   uint8_t dcs_idx, uint32_t num)
19564d368e1dSXiaoyu Min {
19574d368e1dSXiaoyu Min 	uint32_t dcs_id = cpool->dcs_mng.dcs[dcs_idx].obj->id;
19584d368e1dSXiaoyu Min 	uint64_t cnt_num = cpool->dcs_mng.dcs[dcs_idx].batch_sz;
19594d368e1dSXiaoyu Min 	uint64_t left;
19604d368e1dSXiaoyu Min 	uint32_t iidx = cpool->dcs_mng.dcs[dcs_idx].iidx;
19614d368e1dSXiaoyu Min 	uint32_t offset;
19624d368e1dSXiaoyu Min 	uint16_t mask;
19634d368e1dSXiaoyu Min 	uint16_t sq_idx;
19644d368e1dSXiaoyu Min 	uint64_t burst_sz = (uint64_t)(1 << MLX5_ASO_CNT_QUEUE_LOG_DESC) * 4 *
19654d368e1dSXiaoyu Min 		sh->cnt_svc->aso_mng.sq_num;
19664d368e1dSXiaoyu Min 	uint64_t qburst_sz = burst_sz / sh->cnt_svc->aso_mng.sq_num;
19674d368e1dSXiaoyu Min 	uint64_t n;
19684d368e1dSXiaoyu Min 	struct mlx5_aso_sq *sq;
19694d368e1dSXiaoyu Min 
19704d368e1dSXiaoyu Min 	cnt_num = RTE_MIN(num, cnt_num);
19714d368e1dSXiaoyu Min 	left = cnt_num;
19724d368e1dSXiaoyu Min 	while (left) {
19734d368e1dSXiaoyu Min 		mask = 0;
19744d368e1dSXiaoyu Min 		for (sq_idx = 0; sq_idx < sh->cnt_svc->aso_mng.sq_num;
19754d368e1dSXiaoyu Min 				sq_idx++) {
19764d368e1dSXiaoyu Min 			if (left == 0) {
19774d368e1dSXiaoyu Min 				mask |= (1 << sq_idx);
19784d368e1dSXiaoyu Min 				continue;
19794d368e1dSXiaoyu Min 			}
19804d368e1dSXiaoyu Min 			n = RTE_MIN(left, qburst_sz);
19814d368e1dSXiaoyu Min 			offset = cnt_num - left;
19824d368e1dSXiaoyu Min 			offset += iidx;
19834d368e1dSXiaoyu Min 			mlx5_aso_cnt_sq_enqueue_burst(cpool, sh,
19844d368e1dSXiaoyu Min 					&sh->cnt_svc->aso_mng.sqs[sq_idx], n,
19854d368e1dSXiaoyu Min 					offset, dcs_id);
19864d368e1dSXiaoyu Min 			left -= n;
19874d368e1dSXiaoyu Min 		}
19884d368e1dSXiaoyu Min 		do {
19894d368e1dSXiaoyu Min 			for (sq_idx = 0; sq_idx < sh->cnt_svc->aso_mng.sq_num;
19904d368e1dSXiaoyu Min 					sq_idx++) {
19914d368e1dSXiaoyu Min 				sq = &sh->cnt_svc->aso_mng.sqs[sq_idx];
19924d368e1dSXiaoyu Min 				if (mlx5_aso_cnt_completion_handle(sq))
19934d368e1dSXiaoyu Min 					mask |= (1 << sq_idx);
19944d368e1dSXiaoyu Min 			}
19954d368e1dSXiaoyu Min 		} while (mask < ((1 << sh->cnt_svc->aso_mng.sq_num) - 1));
19964d368e1dSXiaoyu Min 	}
19974d368e1dSXiaoyu Min 	return cnt_num;
19984d368e1dSXiaoyu Min }
19994d368e1dSXiaoyu Min 
20004d368e1dSXiaoyu Min /*
20014d368e1dSXiaoyu Min  * Query FW counter via ASO WQE.
20024d368e1dSXiaoyu Min  *
20034d368e1dSXiaoyu Min  * ASO query counter use _sync_ mode, means:
20044d368e1dSXiaoyu Min  * 1. each SQ issue one burst with several WQEs
20054d368e1dSXiaoyu Min  * 2. ask for CQE at last WQE
20064d368e1dSXiaoyu Min  * 3. busy poll CQ of each SQ's
20074d368e1dSXiaoyu Min  * 4. If all SQ's CQE are received then goto step 1, issue next burst
20084d368e1dSXiaoyu Min  *
20094d368e1dSXiaoyu Min  * @param[in] sh
20104d368e1dSXiaoyu Min  *   Pointer to shared device.
20114d368e1dSXiaoyu Min  * @param[in] cpool
20124d368e1dSXiaoyu Min  *   Pointer to counter pool.
20134d368e1dSXiaoyu Min  *
20144d368e1dSXiaoyu Min  * @return
20154d368e1dSXiaoyu Min  *   0 on success, -1 on failure.
20164d368e1dSXiaoyu Min  */
20174d368e1dSXiaoyu Min int
20184d368e1dSXiaoyu Min mlx5_aso_cnt_query(struct mlx5_dev_ctx_shared *sh,
20194d368e1dSXiaoyu Min 		   struct mlx5_hws_cnt_pool *cpool)
20204d368e1dSXiaoyu Min {
20214d368e1dSXiaoyu Min 	uint32_t idx;
20224d368e1dSXiaoyu Min 	uint32_t num;
20234d368e1dSXiaoyu Min 	uint32_t cnt_num = mlx5_hws_cnt_pool_get_size(cpool) -
20244d368e1dSXiaoyu Min 		rte_ring_count(cpool->free_list);
20254d368e1dSXiaoyu Min 
20264d368e1dSXiaoyu Min 	for (idx = 0; idx < cpool->dcs_mng.batch_total; idx++) {
20274d368e1dSXiaoyu Min 		num = RTE_MIN(cnt_num, cpool->dcs_mng.dcs[idx].batch_sz);
20284d368e1dSXiaoyu Min 		mlx5_aso_cnt_query_one_dcs(sh, cpool, idx, num);
20294d368e1dSXiaoyu Min 		cnt_num -= num;
20304d368e1dSXiaoyu Min 		if (cnt_num == 0)
20314d368e1dSXiaoyu Min 			break;
20324d368e1dSXiaoyu Min 	}
20334d368e1dSXiaoyu Min 	return 0;
20344d368e1dSXiaoyu Min }
2035