129efa63aSLi Zhang /* SPDX-License-Identifier: BSD-3-Clause 229efa63aSLi Zhang * Copyright 2020 Mellanox Technologies, Ltd 329efa63aSLi Zhang */ 429efa63aSLi Zhang #include <mlx5_prm.h> 529efa63aSLi Zhang #include <rte_malloc.h> 629efa63aSLi Zhang #include <rte_cycles.h> 729efa63aSLi Zhang #include <rte_eal_paging.h> 829efa63aSLi Zhang 929efa63aSLi Zhang #include <mlx5_malloc.h> 1029efa63aSLi Zhang #include <mlx5_common_os.h> 1129efa63aSLi Zhang #include <mlx5_common_devx.h> 1229efa63aSLi Zhang 1329efa63aSLi Zhang #include "mlx5.h" 1429efa63aSLi Zhang #include "mlx5_flow.h" 154d368e1dSXiaoyu Min #include "mlx5_hws_cnt.h" 164d368e1dSXiaoyu Min 174d368e1dSXiaoyu Min #define MLX5_ASO_CNT_QUEUE_LOG_DESC 14 1829efa63aSLi Zhang 1929efa63aSLi Zhang /** 2029efa63aSLi Zhang * Free MR resources. 2129efa63aSLi Zhang * 229f1d636fSMichael Baum * @param[in] cdev 239f1d636fSMichael Baum * Pointer to the mlx5 common device. 2429efa63aSLi Zhang * @param[in] mr 2529efa63aSLi Zhang * MR to free. 2629efa63aSLi Zhang */ 2729efa63aSLi Zhang static void 289f1d636fSMichael Baum mlx5_aso_dereg_mr(struct mlx5_common_device *cdev, struct mlx5_pmd_mr *mr) 2929efa63aSLi Zhang { 30cd414f81SMichael Baum void *addr = mr->addr; 31cd414f81SMichael Baum 329f1d636fSMichael Baum cdev->mr_scache.dereg_mr_cb(mr); 33cd414f81SMichael Baum mlx5_free(addr); 3429efa63aSLi Zhang memset(mr, 0, sizeof(*mr)); 3529efa63aSLi Zhang } 3629efa63aSLi Zhang 3729efa63aSLi Zhang /** 3829efa63aSLi Zhang * Register Memory Region. 3929efa63aSLi Zhang * 409f1d636fSMichael Baum * @param[in] cdev 419f1d636fSMichael Baum * Pointer to the mlx5 common device. 4229efa63aSLi Zhang * @param[in] length 4329efa63aSLi Zhang * Size of MR buffer. 4429efa63aSLi Zhang * @param[in/out] mr 4529efa63aSLi Zhang * Pointer to MR to create. 4629efa63aSLi Zhang * 4729efa63aSLi Zhang * @return 4829efa63aSLi Zhang * 0 on success, a negative errno value otherwise and rte_errno is set. 4929efa63aSLi Zhang */ 5029efa63aSLi Zhang static int 519f1d636fSMichael Baum mlx5_aso_reg_mr(struct mlx5_common_device *cdev, size_t length, 52147f6fb4SMichael Baum struct mlx5_pmd_mr *mr) 5329efa63aSLi Zhang { 54cd414f81SMichael Baum int ret; 55cd414f81SMichael Baum 56cd414f81SMichael Baum mr->addr = mlx5_malloc(MLX5_MEM_RTE | MLX5_MEM_ZERO, length, 4096, 57147f6fb4SMichael Baum SOCKET_ID_ANY); 58cd414f81SMichael Baum if (!mr->addr) { 59cd414f81SMichael Baum DRV_LOG(ERR, "Failed to create ASO bits mem for MR."); 6029efa63aSLi Zhang return -1; 6129efa63aSLi Zhang } 629f1d636fSMichael Baum ret = cdev->mr_scache.reg_mr_cb(cdev->pd, mr->addr, length, mr); 63cd414f81SMichael Baum if (ret) { 6429efa63aSLi Zhang DRV_LOG(ERR, "Failed to create direct Mkey."); 65cd414f81SMichael Baum mlx5_free(mr->addr); 6629efa63aSLi Zhang return -1; 6729efa63aSLi Zhang } 68cd414f81SMichael Baum return 0; 69cd414f81SMichael Baum } 7029efa63aSLi Zhang 7129efa63aSLi Zhang /** 7229efa63aSLi Zhang * Destroy Send Queue used for ASO access. 7329efa63aSLi Zhang * 7429efa63aSLi Zhang * @param[in] sq 7529efa63aSLi Zhang * ASO SQ to destroy. 7629efa63aSLi Zhang */ 7715896eafSGregory Etelson void 7829efa63aSLi Zhang mlx5_aso_destroy_sq(struct mlx5_aso_sq *sq) 7929efa63aSLi Zhang { 8029efa63aSLi Zhang mlx5_devx_sq_destroy(&sq->sq_obj); 81147f6fb4SMichael Baum mlx5_devx_cq_destroy(&sq->cq.cq_obj); 8229efa63aSLi Zhang memset(sq, 0, sizeof(*sq)); 8329efa63aSLi Zhang } 8429efa63aSLi Zhang 8529efa63aSLi Zhang /** 864d368e1dSXiaoyu Min * Initialize Send Queue used for ASO access counter. 874d368e1dSXiaoyu Min * 884d368e1dSXiaoyu Min * @param[in] sq 894d368e1dSXiaoyu Min * ASO SQ to initialize. 904d368e1dSXiaoyu Min */ 914d368e1dSXiaoyu Min static void 924d368e1dSXiaoyu Min mlx5_aso_cnt_init_sq(struct mlx5_aso_sq *sq) 934d368e1dSXiaoyu Min { 944d368e1dSXiaoyu Min volatile struct mlx5_aso_wqe *restrict wqe; 954d368e1dSXiaoyu Min int i; 964d368e1dSXiaoyu Min int size = 1 << sq->log_desc_n; 974d368e1dSXiaoyu Min 984d368e1dSXiaoyu Min /* All the next fields state should stay constant. */ 994d368e1dSXiaoyu Min for (i = 0, wqe = &sq->sq_obj.aso_wqes[0]; i < size; ++i, ++wqe) { 1004d368e1dSXiaoyu Min wqe->general_cseg.sq_ds = rte_cpu_to_be_32((sq->sqn << 8) | 1014d368e1dSXiaoyu Min (sizeof(*wqe) >> 4)); 1024d368e1dSXiaoyu Min wqe->aso_cseg.operand_masks = rte_cpu_to_be_32 1034d368e1dSXiaoyu Min (0u | 1044d368e1dSXiaoyu Min (ASO_OPER_LOGICAL_OR << ASO_CSEG_COND_OPER_OFFSET) | 1054d368e1dSXiaoyu Min (ASO_OP_ALWAYS_FALSE << ASO_CSEG_COND_1_OPER_OFFSET) | 1064d368e1dSXiaoyu Min (ASO_OP_ALWAYS_FALSE << ASO_CSEG_COND_0_OPER_OFFSET) | 1074d368e1dSXiaoyu Min (BYTEWISE_64BYTE << ASO_CSEG_DATA_MASK_MODE_OFFSET)); 1084d368e1dSXiaoyu Min wqe->aso_cseg.data_mask = RTE_BE64(UINT64_MAX); 1094d368e1dSXiaoyu Min } 1104d368e1dSXiaoyu Min } 1114d368e1dSXiaoyu Min 1124d368e1dSXiaoyu Min /** 11329efa63aSLi Zhang * Initialize Send Queue used for ASO access. 11429efa63aSLi Zhang * 11529efa63aSLi Zhang * @param[in] sq 11629efa63aSLi Zhang * ASO SQ to initialize. 11729efa63aSLi Zhang */ 11829efa63aSLi Zhang static void 11929efa63aSLi Zhang mlx5_aso_age_init_sq(struct mlx5_aso_sq *sq) 12029efa63aSLi Zhang { 12129efa63aSLi Zhang volatile struct mlx5_aso_wqe *restrict wqe; 12229efa63aSLi Zhang int i; 12329efa63aSLi Zhang int size = 1 << sq->log_desc_n; 12429efa63aSLi Zhang uint64_t addr; 12529efa63aSLi Zhang 12629efa63aSLi Zhang /* All the next fields state should stay constant. */ 12729efa63aSLi Zhang for (i = 0, wqe = &sq->sq_obj.aso_wqes[0]; i < size; ++i, ++wqe) { 12829efa63aSLi Zhang wqe->general_cseg.sq_ds = rte_cpu_to_be_32((sq->sqn << 8) | 12929efa63aSLi Zhang (sizeof(*wqe) >> 4)); 130cd414f81SMichael Baum wqe->aso_cseg.lkey = rte_cpu_to_be_32(sq->mr.lkey); 131cd414f81SMichael Baum addr = (uint64_t)((uint64_t *)sq->mr.addr + i * 13229efa63aSLi Zhang MLX5_ASO_AGE_ACTIONS_PER_POOL / 64); 13329efa63aSLi Zhang wqe->aso_cseg.va_h = rte_cpu_to_be_32((uint32_t)(addr >> 32)); 13429efa63aSLi Zhang wqe->aso_cseg.va_l_r = rte_cpu_to_be_32((uint32_t)addr | 1u); 13529efa63aSLi Zhang wqe->aso_cseg.operand_masks = rte_cpu_to_be_32 13629efa63aSLi Zhang (0u | 13729efa63aSLi Zhang (ASO_OPER_LOGICAL_OR << ASO_CSEG_COND_OPER_OFFSET) | 13829efa63aSLi Zhang (ASO_OP_ALWAYS_TRUE << ASO_CSEG_COND_1_OPER_OFFSET) | 13929efa63aSLi Zhang (ASO_OP_ALWAYS_TRUE << ASO_CSEG_COND_0_OPER_OFFSET) | 14029efa63aSLi Zhang (BYTEWISE_64BYTE << ASO_CSEG_DATA_MASK_MODE_OFFSET)); 14129efa63aSLi Zhang wqe->aso_cseg.data_mask = RTE_BE64(UINT64_MAX); 14229efa63aSLi Zhang } 14329efa63aSLi Zhang } 14429efa63aSLi Zhang 14529efa63aSLi Zhang /** 14629efa63aSLi Zhang * Initialize Send Queue used for ASO flow meter access. 14729efa63aSLi Zhang * 14829efa63aSLi Zhang * @param[in] sq 14929efa63aSLi Zhang * ASO SQ to initialize. 15029efa63aSLi Zhang */ 15115896eafSGregory Etelson void 15229efa63aSLi Zhang mlx5_aso_mtr_init_sq(struct mlx5_aso_sq *sq) 15329efa63aSLi Zhang { 15429efa63aSLi Zhang volatile struct mlx5_aso_wqe *restrict wqe; 15529efa63aSLi Zhang int i; 15629efa63aSLi Zhang int size = 1 << sq->log_desc_n; 15729efa63aSLi Zhang 15829efa63aSLi Zhang /* All the next fields state should stay constant. */ 15929efa63aSLi Zhang for (i = 0, wqe = &sq->sq_obj.aso_wqes[0]; i < size; ++i, ++wqe) { 16029efa63aSLi Zhang wqe->general_cseg.sq_ds = rte_cpu_to_be_32((sq->sqn << 8) | 16129efa63aSLi Zhang (sizeof(*wqe) >> 4)); 16229efa63aSLi Zhang wqe->aso_cseg.operand_masks = RTE_BE32(0u | 16329efa63aSLi Zhang (ASO_OPER_LOGICAL_OR << ASO_CSEG_COND_OPER_OFFSET) | 16429efa63aSLi Zhang (ASO_OP_ALWAYS_TRUE << ASO_CSEG_COND_1_OPER_OFFSET) | 16529efa63aSLi Zhang (ASO_OP_ALWAYS_TRUE << ASO_CSEG_COND_0_OPER_OFFSET) | 16629efa63aSLi Zhang (BYTEWISE_64BYTE << ASO_CSEG_DATA_MASK_MODE_OFFSET)); 16729efa63aSLi Zhang wqe->general_cseg.flags = RTE_BE32(MLX5_COMP_ALWAYS << 16829efa63aSLi Zhang MLX5_COMP_MODE_OFFSET); 16929efa63aSLi Zhang } 17029efa63aSLi Zhang } 17129efa63aSLi Zhang 172ee9e5fadSBing Zhao /* 173ee9e5fadSBing Zhao * Initialize Send Queue used for ASO connection tracking. 174ee9e5fadSBing Zhao * 175ee9e5fadSBing Zhao * @param[in] sq 176ee9e5fadSBing Zhao * ASO SQ to initialize. 177ee9e5fadSBing Zhao */ 178ee9e5fadSBing Zhao static void 179ee9e5fadSBing Zhao mlx5_aso_ct_init_sq(struct mlx5_aso_sq *sq) 180ee9e5fadSBing Zhao { 181ee9e5fadSBing Zhao volatile struct mlx5_aso_wqe *restrict wqe; 182ee9e5fadSBing Zhao int i; 183ee9e5fadSBing Zhao int size = 1 << sq->log_desc_n; 184ee9e5fadSBing Zhao uint64_t addr; 185ee9e5fadSBing Zhao 186ee9e5fadSBing Zhao /* All the next fields state should stay constant. */ 187ee9e5fadSBing Zhao for (i = 0, wqe = &sq->sq_obj.aso_wqes[0]; i < size; ++i, ++wqe) { 188ee9e5fadSBing Zhao wqe->general_cseg.sq_ds = rte_cpu_to_be_32((sq->sqn << 8) | 189ee9e5fadSBing Zhao (sizeof(*wqe) >> 4)); 190ee9e5fadSBing Zhao /* One unique MR for the query data. */ 191ee9e5fadSBing Zhao wqe->aso_cseg.lkey = rte_cpu_to_be_32(sq->mr.lkey); 192ee9e5fadSBing Zhao /* Magic number 64 represents the length of a ASO CT obj. */ 193ee9e5fadSBing Zhao addr = (uint64_t)((uintptr_t)sq->mr.addr + i * 64); 194ee9e5fadSBing Zhao wqe->aso_cseg.va_h = rte_cpu_to_be_32((uint32_t)(addr >> 32)); 195ee9e5fadSBing Zhao wqe->aso_cseg.va_l_r = rte_cpu_to_be_32((uint32_t)addr | 1u); 196ee9e5fadSBing Zhao /* 197ee9e5fadSBing Zhao * The values of operand_masks are different for modify 198ee9e5fadSBing Zhao * and query. 199ee9e5fadSBing Zhao * And data_mask may be different for each modification. In 200ee9e5fadSBing Zhao * query, it could be zero and ignored. 201ee9e5fadSBing Zhao * CQE generation is always needed, in order to decide when 202ee9e5fadSBing Zhao * it is available to create the flow or read the data. 203ee9e5fadSBing Zhao */ 204ee9e5fadSBing Zhao wqe->general_cseg.flags = RTE_BE32(MLX5_COMP_ALWAYS << 205ee9e5fadSBing Zhao MLX5_COMP_MODE_OFFSET); 206ee9e5fadSBing Zhao } 207ee9e5fadSBing Zhao } 208ee9e5fadSBing Zhao 20929efa63aSLi Zhang /** 21029efa63aSLi Zhang * Create Send Queue used for ASO access. 21129efa63aSLi Zhang * 212147f6fb4SMichael Baum * @param[in] cdev 213147f6fb4SMichael Baum * Pointer to the mlx5 common device. 21429efa63aSLi Zhang * @param[in/out] sq 21529efa63aSLi Zhang * Pointer to SQ to create. 21629efa63aSLi Zhang * @param[in] uar 21729efa63aSLi Zhang * User Access Region object. 21829efa63aSLi Zhang * 21929efa63aSLi Zhang * @return 22029efa63aSLi Zhang * 0 on success, a negative errno value otherwise and rte_errno is set. 22129efa63aSLi Zhang */ 22215896eafSGregory Etelson int 223147f6fb4SMichael Baum mlx5_aso_sq_create(struct mlx5_common_device *cdev, struct mlx5_aso_sq *sq, 2244d368e1dSXiaoyu Min void *uar, uint16_t log_desc_n) 22529efa63aSLi Zhang { 226147f6fb4SMichael Baum struct mlx5_devx_cq_attr cq_attr = { 227147f6fb4SMichael Baum .uar_page_id = mlx5_os_get_devx_uar_page_id(uar), 228147f6fb4SMichael Baum }; 229147f6fb4SMichael Baum struct mlx5_devx_create_sq_attr sq_attr = { 23029efa63aSLi Zhang .user_index = 0xFFFF, 23129efa63aSLi Zhang .wq_attr = (struct mlx5_devx_wq_attr){ 232147f6fb4SMichael Baum .pd = cdev->pdn, 23329efa63aSLi Zhang .uar_page = mlx5_os_get_devx_uar_page_id(uar), 23429efa63aSLi Zhang }, 235147f6fb4SMichael Baum .ts_format = 236147f6fb4SMichael Baum mlx5_ts_format_conv(cdev->config.hca_attr.sq_ts_format), 23729efa63aSLi Zhang }; 23829efa63aSLi Zhang struct mlx5_devx_modify_sq_attr modify_attr = { 23929efa63aSLi Zhang .state = MLX5_SQC_STATE_RDY, 24029efa63aSLi Zhang }; 24129efa63aSLi Zhang uint16_t log_wqbb_n; 24229efa63aSLi Zhang int ret; 24329efa63aSLi Zhang 244147f6fb4SMichael Baum if (mlx5_devx_cq_create(cdev->ctx, &sq->cq.cq_obj, 2454d368e1dSXiaoyu Min log_desc_n, &cq_attr, 246147f6fb4SMichael Baum SOCKET_ID_ANY)) 24729efa63aSLi Zhang goto error; 248147f6fb4SMichael Baum sq->cq.cq_ci = 0; 2494d368e1dSXiaoyu Min sq->cq.log_desc_n = log_desc_n; 2504d368e1dSXiaoyu Min sq->log_desc_n = log_desc_n; 251147f6fb4SMichael Baum sq_attr.cqn = sq->cq.cq_obj.cq->id; 25229efa63aSLi Zhang /* for mlx5_aso_wqe that is twice the size of mlx5_wqe */ 253147f6fb4SMichael Baum log_wqbb_n = sq->log_desc_n + 1; 254147f6fb4SMichael Baum ret = mlx5_devx_sq_create(cdev->ctx, &sq->sq_obj, log_wqbb_n, &sq_attr, 255147f6fb4SMichael Baum SOCKET_ID_ANY); 25629efa63aSLi Zhang if (ret) { 25729efa63aSLi Zhang DRV_LOG(ERR, "Can't create SQ object."); 25829efa63aSLi Zhang rte_errno = ENOMEM; 25929efa63aSLi Zhang goto error; 26029efa63aSLi Zhang } 26129efa63aSLi Zhang ret = mlx5_devx_cmd_modify_sq(sq->sq_obj.sq, &modify_attr); 26229efa63aSLi Zhang if (ret) { 26329efa63aSLi Zhang DRV_LOG(ERR, "Can't change SQ state to ready."); 26429efa63aSLi Zhang rte_errno = ENOMEM; 26529efa63aSLi Zhang goto error; 26629efa63aSLi Zhang } 26729efa63aSLi Zhang sq->pi = 0; 26829efa63aSLi Zhang sq->head = 0; 26929efa63aSLi Zhang sq->tail = 0; 27029efa63aSLi Zhang sq->sqn = sq->sq_obj.sq->id; 271cfd2037cSLi Zhang rte_spinlock_init(&sq->sqsl); 27229efa63aSLi Zhang return 0; 27329efa63aSLi Zhang error: 27429efa63aSLi Zhang mlx5_aso_destroy_sq(sq); 27529efa63aSLi Zhang return -1; 27629efa63aSLi Zhang } 27729efa63aSLi Zhang 27848fbb0e9SAlexander Kozyrev void 27948fbb0e9SAlexander Kozyrev mlx5_aso_mtr_queue_uninit(struct mlx5_dev_ctx_shared *sh __rte_unused, 28048fbb0e9SAlexander Kozyrev struct mlx5_aso_mtr_pool *hws_pool, 28148fbb0e9SAlexander Kozyrev struct mlx5_aso_mtr_pools_mng *pool_mng) 28248fbb0e9SAlexander Kozyrev { 28348fbb0e9SAlexander Kozyrev uint32_t i; 28448fbb0e9SAlexander Kozyrev 28548fbb0e9SAlexander Kozyrev if (hws_pool) { 28648fbb0e9SAlexander Kozyrev for (i = 0; i < hws_pool->nb_sq; i++) 28748fbb0e9SAlexander Kozyrev mlx5_aso_destroy_sq(hws_pool->sq + i); 28848fbb0e9SAlexander Kozyrev mlx5_free(hws_pool->sq); 28948fbb0e9SAlexander Kozyrev return; 29048fbb0e9SAlexander Kozyrev } 29148fbb0e9SAlexander Kozyrev if (pool_mng) 29248fbb0e9SAlexander Kozyrev mlx5_aso_destroy_sq(&pool_mng->sq); 29348fbb0e9SAlexander Kozyrev } 29448fbb0e9SAlexander Kozyrev 29548fbb0e9SAlexander Kozyrev int 29648fbb0e9SAlexander Kozyrev mlx5_aso_mtr_queue_init(struct mlx5_dev_ctx_shared *sh, 29748fbb0e9SAlexander Kozyrev struct mlx5_aso_mtr_pool *hws_pool, 29848fbb0e9SAlexander Kozyrev struct mlx5_aso_mtr_pools_mng *pool_mng, 29948fbb0e9SAlexander Kozyrev uint32_t nb_queues) 30048fbb0e9SAlexander Kozyrev { 30148fbb0e9SAlexander Kozyrev struct mlx5_common_device *cdev = sh->cdev; 30248fbb0e9SAlexander Kozyrev struct mlx5_aso_sq *sq; 30348fbb0e9SAlexander Kozyrev uint32_t i; 30448fbb0e9SAlexander Kozyrev 30548fbb0e9SAlexander Kozyrev if (hws_pool) { 30648fbb0e9SAlexander Kozyrev sq = mlx5_malloc(MLX5_MEM_ZERO, 30748fbb0e9SAlexander Kozyrev sizeof(struct mlx5_aso_sq) * nb_queues, 30848fbb0e9SAlexander Kozyrev RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY); 30948fbb0e9SAlexander Kozyrev if (!sq) 31048fbb0e9SAlexander Kozyrev return -1; 31148fbb0e9SAlexander Kozyrev hws_pool->sq = sq; 31248fbb0e9SAlexander Kozyrev for (i = 0; i < nb_queues; i++) { 31348fbb0e9SAlexander Kozyrev if (mlx5_aso_sq_create(cdev, hws_pool->sq + i, 31448fbb0e9SAlexander Kozyrev sh->tx_uar.obj, 31548fbb0e9SAlexander Kozyrev MLX5_ASO_QUEUE_LOG_DESC)) 31648fbb0e9SAlexander Kozyrev goto error; 31748fbb0e9SAlexander Kozyrev mlx5_aso_mtr_init_sq(hws_pool->sq + i); 31848fbb0e9SAlexander Kozyrev } 31948fbb0e9SAlexander Kozyrev hws_pool->nb_sq = nb_queues; 32048fbb0e9SAlexander Kozyrev } 32148fbb0e9SAlexander Kozyrev if (pool_mng) { 32248fbb0e9SAlexander Kozyrev if (mlx5_aso_sq_create(cdev, &pool_mng->sq, 32348fbb0e9SAlexander Kozyrev sh->tx_uar.obj, 32448fbb0e9SAlexander Kozyrev MLX5_ASO_QUEUE_LOG_DESC)) 32548fbb0e9SAlexander Kozyrev return -1; 32648fbb0e9SAlexander Kozyrev mlx5_aso_mtr_init_sq(&pool_mng->sq); 32748fbb0e9SAlexander Kozyrev } 32848fbb0e9SAlexander Kozyrev return 0; 32948fbb0e9SAlexander Kozyrev error: 33048fbb0e9SAlexander Kozyrev do { 33148fbb0e9SAlexander Kozyrev mlx5_aso_destroy_sq(hws_pool->sq + i); 33248fbb0e9SAlexander Kozyrev } while (i--); 33348fbb0e9SAlexander Kozyrev return -1; 33448fbb0e9SAlexander Kozyrev } 33548fbb0e9SAlexander Kozyrev 33629efa63aSLi Zhang /** 33729efa63aSLi Zhang * API to create and initialize Send Queue used for ASO access. 33829efa63aSLi Zhang * 33929efa63aSLi Zhang * @param[in] sh 34029efa63aSLi Zhang * Pointer to shared device context. 341cd414f81SMichael Baum * @param[in] aso_opc_mod 342cd414f81SMichael Baum * Mode of ASO feature. 34348fbb0e9SAlexander Kozyrev * @param[in] nb_queues 34448fbb0e9SAlexander Kozyrev * Number of Send Queues to create. 34529efa63aSLi Zhang * 34629efa63aSLi Zhang * @return 34729efa63aSLi Zhang * 0 on success, a negative errno value otherwise and rte_errno is set. 34829efa63aSLi Zhang */ 34929efa63aSLi Zhang int 35029efa63aSLi Zhang mlx5_aso_queue_init(struct mlx5_dev_ctx_shared *sh, 35148fbb0e9SAlexander Kozyrev enum mlx5_access_aso_opc_mod aso_opc_mod, 35248fbb0e9SAlexander Kozyrev uint32_t nb_queues) 35329efa63aSLi Zhang { 35429efa63aSLi Zhang uint32_t sq_desc_n = 1 << MLX5_ASO_QUEUE_LOG_DESC; 355ca1418ceSMichael Baum struct mlx5_common_device *cdev = sh->cdev; 35629efa63aSLi Zhang 35729efa63aSLi Zhang switch (aso_opc_mod) { 35829efa63aSLi Zhang case ASO_OPC_MOD_FLOW_HIT: 3599f1d636fSMichael Baum if (mlx5_aso_reg_mr(cdev, (MLX5_ASO_AGE_ACTIONS_PER_POOL / 8) * 360147f6fb4SMichael Baum sq_desc_n, &sh->aso_age_mng->aso_sq.mr)) 36129efa63aSLi Zhang return -1; 362147f6fb4SMichael Baum if (mlx5_aso_sq_create(cdev, &sh->aso_age_mng->aso_sq, 3634d368e1dSXiaoyu Min sh->tx_uar.obj, 3644d368e1dSXiaoyu Min MLX5_ASO_QUEUE_LOG_DESC)) { 3659f1d636fSMichael Baum mlx5_aso_dereg_mr(cdev, &sh->aso_age_mng->aso_sq.mr); 36629efa63aSLi Zhang return -1; 36729efa63aSLi Zhang } 36829efa63aSLi Zhang mlx5_aso_age_init_sq(&sh->aso_age_mng->aso_sq); 36929efa63aSLi Zhang break; 37029efa63aSLi Zhang case ASO_OPC_MOD_POLICER: 37148fbb0e9SAlexander Kozyrev if (mlx5_aso_mtr_queue_init(sh, NULL, 37248fbb0e9SAlexander Kozyrev &sh->mtrmng->pools_mng, nb_queues)) 37329efa63aSLi Zhang return -1; 37429efa63aSLi Zhang break; 375ee9e5fadSBing Zhao case ASO_OPC_MOD_CONNECTION_TRACKING: 376463170a7SSuanming Mou if (mlx5_aso_ct_queue_init(sh, sh->ct_mng, MLX5_ASO_CT_SQ_NUM)) 377ee9e5fadSBing Zhao return -1; 378ee9e5fadSBing Zhao break; 37929efa63aSLi Zhang default: 38029efa63aSLi Zhang DRV_LOG(ERR, "Unknown ASO operation mode"); 38129efa63aSLi Zhang return -1; 38229efa63aSLi Zhang } 38329efa63aSLi Zhang return 0; 38429efa63aSLi Zhang } 38529efa63aSLi Zhang 38629efa63aSLi Zhang /** 38729efa63aSLi Zhang * API to destroy Send Queue used for ASO access. 38829efa63aSLi Zhang * 38929efa63aSLi Zhang * @param[in] sh 39029efa63aSLi Zhang * Pointer to shared device context. 391cd414f81SMichael Baum * @param[in] aso_opc_mod 392cd414f81SMichael Baum * Mode of ASO feature. 39329efa63aSLi Zhang */ 39429efa63aSLi Zhang void 39529efa63aSLi Zhang mlx5_aso_queue_uninit(struct mlx5_dev_ctx_shared *sh, 39629efa63aSLi Zhang enum mlx5_access_aso_opc_mod aso_opc_mod) 39729efa63aSLi Zhang { 398463170a7SSuanming Mou struct mlx5_aso_sq *sq = NULL; 39929efa63aSLi Zhang 40029efa63aSLi Zhang switch (aso_opc_mod) { 40129efa63aSLi Zhang case ASO_OPC_MOD_FLOW_HIT: 4029f1d636fSMichael Baum mlx5_aso_dereg_mr(sh->cdev, &sh->aso_age_mng->aso_sq.mr); 40329efa63aSLi Zhang sq = &sh->aso_age_mng->aso_sq; 40429efa63aSLi Zhang break; 40529efa63aSLi Zhang case ASO_OPC_MOD_POLICER: 40648fbb0e9SAlexander Kozyrev mlx5_aso_mtr_queue_uninit(sh, NULL, &sh->mtrmng->pools_mng); 40729efa63aSLi Zhang break; 4080af8a229SBing Zhao case ASO_OPC_MOD_CONNECTION_TRACKING: 409463170a7SSuanming Mou mlx5_aso_ct_queue_uninit(sh, sh->ct_mng); 4100af8a229SBing Zhao break; 41129efa63aSLi Zhang default: 41229efa63aSLi Zhang DRV_LOG(ERR, "Unknown ASO operation mode"); 41329efa63aSLi Zhang return; 41429efa63aSLi Zhang } 415463170a7SSuanming Mou if (sq) 41629efa63aSLi Zhang mlx5_aso_destroy_sq(sq); 41729efa63aSLi Zhang } 41829efa63aSLi Zhang 41929efa63aSLi Zhang /** 42029efa63aSLi Zhang * Write a burst of WQEs to ASO SQ. 42129efa63aSLi Zhang * 4225dfa003dSMichael Baum * @param[in] sh 4235dfa003dSMichael Baum * Pointer to shared device context. 42429efa63aSLi Zhang * @param[in] n 42529efa63aSLi Zhang * Index of the last valid pool. 42629efa63aSLi Zhang * 42729efa63aSLi Zhang * @return 42829efa63aSLi Zhang * Number of WQEs in burst. 42929efa63aSLi Zhang */ 43029efa63aSLi Zhang static uint16_t 4315dfa003dSMichael Baum mlx5_aso_sq_enqueue_burst(struct mlx5_dev_ctx_shared *sh, uint16_t n) 43229efa63aSLi Zhang { 4335dfa003dSMichael Baum struct mlx5_aso_age_mng *mng = sh->aso_age_mng; 43429efa63aSLi Zhang volatile struct mlx5_aso_wqe *wqe; 43529efa63aSLi Zhang struct mlx5_aso_sq *sq = &mng->aso_sq; 43629efa63aSLi Zhang struct mlx5_aso_age_pool *pool; 43729efa63aSLi Zhang uint16_t size = 1 << sq->log_desc_n; 43829efa63aSLi Zhang uint16_t mask = size - 1; 43929efa63aSLi Zhang uint16_t max; 44029efa63aSLi Zhang uint16_t start_head = sq->head; 44129efa63aSLi Zhang 44229efa63aSLi Zhang max = RTE_MIN(size - (uint16_t)(sq->head - sq->tail), n - sq->next); 44329efa63aSLi Zhang if (unlikely(!max)) 44429efa63aSLi Zhang return 0; 44529efa63aSLi Zhang sq->elts[start_head & mask].burst_size = max; 44629efa63aSLi Zhang do { 44729efa63aSLi Zhang wqe = &sq->sq_obj.aso_wqes[sq->head & mask]; 44829efa63aSLi Zhang rte_prefetch0(&sq->sq_obj.aso_wqes[(sq->head + 1) & mask]); 44929efa63aSLi Zhang /* Fill next WQE. */ 4507cf2d15aSJiawei Wang rte_rwlock_read_lock(&mng->resize_rwl); 45129efa63aSLi Zhang pool = mng->pools[sq->next]; 4527cf2d15aSJiawei Wang rte_rwlock_read_unlock(&mng->resize_rwl); 45329efa63aSLi Zhang sq->elts[sq->head & mask].pool = pool; 45429efa63aSLi Zhang wqe->general_cseg.misc = 45529efa63aSLi Zhang rte_cpu_to_be_32(((struct mlx5_devx_obj *) 45629efa63aSLi Zhang (pool->flow_hit_aso_obj))->id); 45729efa63aSLi Zhang wqe->general_cseg.flags = RTE_BE32(MLX5_COMP_ONLY_FIRST_ERR << 45829efa63aSLi Zhang MLX5_COMP_MODE_OFFSET); 45929efa63aSLi Zhang wqe->general_cseg.opcode = rte_cpu_to_be_32 46029efa63aSLi Zhang (MLX5_OPCODE_ACCESS_ASO | 46129efa63aSLi Zhang (ASO_OPC_MOD_FLOW_HIT << 46229efa63aSLi Zhang WQE_CSEG_OPC_MOD_OFFSET) | 46329efa63aSLi Zhang (sq->pi << 46429efa63aSLi Zhang WQE_CSEG_WQE_INDEX_OFFSET)); 46529efa63aSLi Zhang sq->pi += 2; /* Each WQE contains 2 WQEBB's. */ 46629efa63aSLi Zhang sq->head++; 46729efa63aSLi Zhang sq->next++; 46829efa63aSLi Zhang max--; 46929efa63aSLi Zhang } while (max); 47029efa63aSLi Zhang wqe->general_cseg.flags = RTE_BE32(MLX5_COMP_ALWAYS << 47129efa63aSLi Zhang MLX5_COMP_MODE_OFFSET); 4725dfa003dSMichael Baum mlx5_doorbell_ring(&sh->tx_uar.bf_db, *(volatile uint64_t *)wqe, 4735dfa003dSMichael Baum sq->pi, &sq->sq_obj.db_rec[MLX5_SND_DBR], 4745dfa003dSMichael Baum !sh->tx_uar.dbnc); 47529efa63aSLi Zhang return sq->elts[start_head & mask].burst_size; 47629efa63aSLi Zhang } 47729efa63aSLi Zhang 47829efa63aSLi Zhang /** 47929efa63aSLi Zhang * Debug utility function. Dump contents of error CQE and WQE. 48029efa63aSLi Zhang * 48129efa63aSLi Zhang * @param[in] cqe 48229efa63aSLi Zhang * Error CQE to dump. 48329efa63aSLi Zhang * @param[in] wqe 48429efa63aSLi Zhang * Error WQE to dump. 48529efa63aSLi Zhang */ 48629efa63aSLi Zhang static void 48729efa63aSLi Zhang mlx5_aso_dump_err_objs(volatile uint32_t *cqe, volatile uint32_t *wqe) 48829efa63aSLi Zhang { 48929efa63aSLi Zhang int i; 49029efa63aSLi Zhang 49129efa63aSLi Zhang DRV_LOG(ERR, "Error cqe:"); 492*3cddeba0SAlexander Kozyrev for (i = 0; i < (int)sizeof(struct mlx5_error_cqe) / 4; i += 4) 49329efa63aSLi Zhang DRV_LOG(ERR, "%08X %08X %08X %08X", cqe[i], cqe[i + 1], 49429efa63aSLi Zhang cqe[i + 2], cqe[i + 3]); 49529efa63aSLi Zhang DRV_LOG(ERR, "\nError wqe:"); 49629efa63aSLi Zhang for (i = 0; i < (int)sizeof(struct mlx5_aso_wqe) / 4; i += 4) 49729efa63aSLi Zhang DRV_LOG(ERR, "%08X %08X %08X %08X", wqe[i], wqe[i + 1], 49829efa63aSLi Zhang wqe[i + 2], wqe[i + 3]); 49929efa63aSLi Zhang } 50029efa63aSLi Zhang 50129efa63aSLi Zhang /** 50229efa63aSLi Zhang * Handle case of error CQE. 50329efa63aSLi Zhang * 50429efa63aSLi Zhang * @param[in] sq 50529efa63aSLi Zhang * ASO SQ to use. 50629efa63aSLi Zhang */ 50715896eafSGregory Etelson void 50829efa63aSLi Zhang mlx5_aso_cqe_err_handle(struct mlx5_aso_sq *sq) 50929efa63aSLi Zhang { 51029efa63aSLi Zhang struct mlx5_aso_cq *cq = &sq->cq; 51129efa63aSLi Zhang uint32_t idx = cq->cq_ci & ((1 << cq->log_desc_n) - 1); 512*3cddeba0SAlexander Kozyrev volatile struct mlx5_error_cqe *cqe = 513*3cddeba0SAlexander Kozyrev (volatile struct mlx5_error_cqe *)&cq->cq_obj.cqes[idx]; 51429efa63aSLi Zhang 51529efa63aSLi Zhang cq->errors++; 51629efa63aSLi Zhang idx = rte_be_to_cpu_16(cqe->wqe_counter) & (1u << sq->log_desc_n); 51729efa63aSLi Zhang mlx5_aso_dump_err_objs((volatile uint32_t *)cqe, 51829efa63aSLi Zhang (volatile uint32_t *)&sq->sq_obj.aso_wqes[idx]); 51929efa63aSLi Zhang } 52029efa63aSLi Zhang 521478ba4bbSSuanming Mou int 522478ba4bbSSuanming Mou mlx5_aso_pull_completion(struct mlx5_aso_sq *sq, 523478ba4bbSSuanming Mou struct rte_flow_op_result res[], 524478ba4bbSSuanming Mou uint16_t n_res) 525478ba4bbSSuanming Mou { 526478ba4bbSSuanming Mou struct mlx5_aso_cq *cq = &sq->cq; 527478ba4bbSSuanming Mou volatile struct mlx5_cqe *restrict cqe; 528478ba4bbSSuanming Mou const uint32_t cq_size = 1 << cq->log_desc_n; 529478ba4bbSSuanming Mou const uint32_t mask = cq_size - 1; 530478ba4bbSSuanming Mou uint32_t idx; 531478ba4bbSSuanming Mou uint32_t next_idx; 532478ba4bbSSuanming Mou uint16_t max; 533478ba4bbSSuanming Mou uint16_t n = 0; 534478ba4bbSSuanming Mou int ret; 535478ba4bbSSuanming Mou 536478ba4bbSSuanming Mou max = (uint16_t)(sq->head - sq->tail); 537478ba4bbSSuanming Mou if (unlikely(!max || !n_res)) 538478ba4bbSSuanming Mou return 0; 539478ba4bbSSuanming Mou next_idx = cq->cq_ci & mask; 540478ba4bbSSuanming Mou do { 541478ba4bbSSuanming Mou idx = next_idx; 542478ba4bbSSuanming Mou next_idx = (cq->cq_ci + 1) & mask; 543478ba4bbSSuanming Mou /* Need to confirm the position of the prefetch. */ 544478ba4bbSSuanming Mou rte_prefetch0(&cq->cq_obj.cqes[next_idx]); 545478ba4bbSSuanming Mou cqe = &cq->cq_obj.cqes[idx]; 546478ba4bbSSuanming Mou ret = check_cqe(cqe, cq_size, cq->cq_ci); 547478ba4bbSSuanming Mou /* 548478ba4bbSSuanming Mou * Be sure owner read is done before any other cookie field or 549478ba4bbSSuanming Mou * opaque field. 550478ba4bbSSuanming Mou */ 551478ba4bbSSuanming Mou rte_io_rmb(); 552478ba4bbSSuanming Mou if (ret == MLX5_CQE_STATUS_HW_OWN) 553478ba4bbSSuanming Mou break; 554478ba4bbSSuanming Mou res[n].user_data = sq->elts[(uint16_t)((sq->tail + n) & mask)].user_data; 555478ba4bbSSuanming Mou if (unlikely(ret != MLX5_CQE_STATUS_SW_OWN)) { 556478ba4bbSSuanming Mou mlx5_aso_cqe_err_handle(sq); 557478ba4bbSSuanming Mou res[n].status = RTE_FLOW_OP_ERROR; 558478ba4bbSSuanming Mou } else { 559478ba4bbSSuanming Mou res[n].status = RTE_FLOW_OP_SUCCESS; 560478ba4bbSSuanming Mou } 561478ba4bbSSuanming Mou cq->cq_ci++; 562478ba4bbSSuanming Mou if (++n == n_res) 563478ba4bbSSuanming Mou break; 564478ba4bbSSuanming Mou } while (1); 565478ba4bbSSuanming Mou if (likely(n)) { 566478ba4bbSSuanming Mou sq->tail += n; 567478ba4bbSSuanming Mou rte_io_wmb(); 568478ba4bbSSuanming Mou cq->cq_obj.db_rec[0] = rte_cpu_to_be_32(cq->cq_ci); 569478ba4bbSSuanming Mou } 570478ba4bbSSuanming Mou return n; 571478ba4bbSSuanming Mou } 572478ba4bbSSuanming Mou 573478ba4bbSSuanming Mou void 574478ba4bbSSuanming Mou mlx5_aso_push_wqe(struct mlx5_dev_ctx_shared *sh, 575478ba4bbSSuanming Mou struct mlx5_aso_sq *sq) 576478ba4bbSSuanming Mou { 577478ba4bbSSuanming Mou if (sq->db_pi == sq->pi) 578478ba4bbSSuanming Mou return; 579478ba4bbSSuanming Mou mlx5_doorbell_ring(&sh->tx_uar.bf_db, *(volatile uint64_t *)sq->db, 580478ba4bbSSuanming Mou sq->pi, &sq->sq_obj.db_rec[MLX5_SND_DBR], 581478ba4bbSSuanming Mou !sh->tx_uar.dbnc); 582478ba4bbSSuanming Mou sq->db_pi = sq->pi; 583478ba4bbSSuanming Mou } 584478ba4bbSSuanming Mou 58529efa63aSLi Zhang /** 58629efa63aSLi Zhang * Update ASO objects upon completion. 58729efa63aSLi Zhang * 58829efa63aSLi Zhang * @param[in] sh 58929efa63aSLi Zhang * Shared device context. 59029efa63aSLi Zhang * @param[in] n 59129efa63aSLi Zhang * Number of completed ASO objects. 59229efa63aSLi Zhang */ 59329efa63aSLi Zhang static void 59429efa63aSLi Zhang mlx5_aso_age_action_update(struct mlx5_dev_ctx_shared *sh, uint16_t n) 59529efa63aSLi Zhang { 59629efa63aSLi Zhang struct mlx5_aso_age_mng *mng = sh->aso_age_mng; 59729efa63aSLi Zhang struct mlx5_aso_sq *sq = &mng->aso_sq; 59829efa63aSLi Zhang struct mlx5_age_info *age_info; 59929efa63aSLi Zhang const uint16_t size = 1 << sq->log_desc_n; 60029efa63aSLi Zhang const uint16_t mask = size - 1; 60129efa63aSLi Zhang const uint64_t curr = MLX5_CURR_TIME_SEC; 60229efa63aSLi Zhang uint16_t expected = AGE_CANDIDATE; 60329efa63aSLi Zhang uint16_t i; 60429efa63aSLi Zhang 60529efa63aSLi Zhang for (i = 0; i < n; ++i) { 60629efa63aSLi Zhang uint16_t idx = (sq->tail + i) & mask; 60729efa63aSLi Zhang struct mlx5_aso_age_pool *pool = sq->elts[idx].pool; 60829efa63aSLi Zhang uint64_t diff = curr - pool->time_of_last_age_check; 609cd414f81SMichael Baum uint64_t *addr = sq->mr.addr; 61029efa63aSLi Zhang int j; 61129efa63aSLi Zhang 61229efa63aSLi Zhang addr += idx * MLX5_ASO_AGE_ACTIONS_PER_POOL / 64; 61329efa63aSLi Zhang pool->time_of_last_age_check = curr; 61429efa63aSLi Zhang for (j = 0; j < MLX5_ASO_AGE_ACTIONS_PER_POOL; j++) { 61529efa63aSLi Zhang struct mlx5_aso_age_action *act = &pool->actions[j]; 61629efa63aSLi Zhang struct mlx5_age_param *ap = &act->age_params; 61729efa63aSLi Zhang uint8_t byte; 61829efa63aSLi Zhang uint8_t offset; 61929efa63aSLi Zhang uint8_t *u8addr; 62029efa63aSLi Zhang uint8_t hit; 62129efa63aSLi Zhang 622e12a0166STyler Retzlaff if (rte_atomic_load_explicit(&ap->state, rte_memory_order_relaxed) != 62329efa63aSLi Zhang AGE_CANDIDATE) 62429efa63aSLi Zhang continue; 62529efa63aSLi Zhang byte = 63 - (j / 8); 62629efa63aSLi Zhang offset = j % 8; 62729efa63aSLi Zhang u8addr = (uint8_t *)addr; 62829efa63aSLi Zhang hit = (u8addr[byte] >> offset) & 0x1; 62929efa63aSLi Zhang if (hit) { 630e12a0166STyler Retzlaff rte_atomic_store_explicit(&ap->sec_since_last_hit, 0, 631e12a0166STyler Retzlaff rte_memory_order_relaxed); 63229efa63aSLi Zhang } else { 63329efa63aSLi Zhang struct mlx5_priv *priv; 63429efa63aSLi Zhang 635e12a0166STyler Retzlaff rte_atomic_fetch_add_explicit(&ap->sec_since_last_hit, 636e12a0166STyler Retzlaff diff, rte_memory_order_relaxed); 63729efa63aSLi Zhang /* If timeout passed add to aged-out list. */ 63829efa63aSLi Zhang if (ap->sec_since_last_hit <= ap->timeout) 63929efa63aSLi Zhang continue; 64029efa63aSLi Zhang priv = 64129efa63aSLi Zhang rte_eth_devices[ap->port_id].data->dev_private; 64229efa63aSLi Zhang age_info = GET_PORT_AGE_INFO(priv); 64329efa63aSLi Zhang rte_spinlock_lock(&age_info->aged_sl); 644e12a0166STyler Retzlaff if (rte_atomic_compare_exchange_strong_explicit(&ap->state, 64529efa63aSLi Zhang &expected, 64629efa63aSLi Zhang AGE_TMOUT, 647e12a0166STyler Retzlaff rte_memory_order_relaxed, 648e12a0166STyler Retzlaff rte_memory_order_relaxed)) { 64929efa63aSLi Zhang LIST_INSERT_HEAD(&age_info->aged_aso, 65029efa63aSLi Zhang act, next); 65129efa63aSLi Zhang MLX5_AGE_SET(age_info, 65229efa63aSLi Zhang MLX5_AGE_EVENT_NEW); 65329efa63aSLi Zhang } 65429efa63aSLi Zhang rte_spinlock_unlock(&age_info->aged_sl); 65529efa63aSLi Zhang } 65629efa63aSLi Zhang } 65729efa63aSLi Zhang } 65829efa63aSLi Zhang mlx5_age_event_prepare(sh); 65929efa63aSLi Zhang } 66029efa63aSLi Zhang 66129efa63aSLi Zhang /** 66229efa63aSLi Zhang * Handle completions from WQEs sent to ASO SQ. 66329efa63aSLi Zhang * 66429efa63aSLi Zhang * @param[in] sh 66529efa63aSLi Zhang * Shared device context. 66629efa63aSLi Zhang * 66729efa63aSLi Zhang * @return 66829efa63aSLi Zhang * Number of CQEs handled. 66929efa63aSLi Zhang */ 67029efa63aSLi Zhang static uint16_t 67129efa63aSLi Zhang mlx5_aso_completion_handle(struct mlx5_dev_ctx_shared *sh) 67229efa63aSLi Zhang { 67329efa63aSLi Zhang struct mlx5_aso_age_mng *mng = sh->aso_age_mng; 67429efa63aSLi Zhang struct mlx5_aso_sq *sq = &mng->aso_sq; 67529efa63aSLi Zhang struct mlx5_aso_cq *cq = &sq->cq; 67629efa63aSLi Zhang volatile struct mlx5_cqe *restrict cqe; 67729efa63aSLi Zhang const unsigned int cq_size = 1 << cq->log_desc_n; 67829efa63aSLi Zhang const unsigned int mask = cq_size - 1; 67929efa63aSLi Zhang uint32_t idx; 68029efa63aSLi Zhang uint32_t next_idx = cq->cq_ci & mask; 68129efa63aSLi Zhang const uint16_t max = (uint16_t)(sq->head - sq->tail); 68229efa63aSLi Zhang uint16_t i = 0; 68329efa63aSLi Zhang int ret; 68429efa63aSLi Zhang if (unlikely(!max)) 68529efa63aSLi Zhang return 0; 68629efa63aSLi Zhang do { 68729efa63aSLi Zhang idx = next_idx; 68829efa63aSLi Zhang next_idx = (cq->cq_ci + 1) & mask; 68929efa63aSLi Zhang rte_prefetch0(&cq->cq_obj.cqes[next_idx]); 69029efa63aSLi Zhang cqe = &cq->cq_obj.cqes[idx]; 69129efa63aSLi Zhang ret = check_cqe(cqe, cq_size, cq->cq_ci); 69229efa63aSLi Zhang /* 69329efa63aSLi Zhang * Be sure owner read is done before any other cookie field or 69429efa63aSLi Zhang * opaque field. 69529efa63aSLi Zhang */ 69629efa63aSLi Zhang rte_io_rmb(); 69729efa63aSLi Zhang if (unlikely(ret != MLX5_CQE_STATUS_SW_OWN)) { 69829efa63aSLi Zhang if (likely(ret == MLX5_CQE_STATUS_HW_OWN)) 69929efa63aSLi Zhang break; 70029efa63aSLi Zhang mlx5_aso_cqe_err_handle(sq); 70129efa63aSLi Zhang } else { 70229efa63aSLi Zhang i += sq->elts[(sq->tail + i) & mask].burst_size; 70329efa63aSLi Zhang } 70429efa63aSLi Zhang cq->cq_ci++; 70529efa63aSLi Zhang } while (1); 70629efa63aSLi Zhang if (likely(i)) { 70729efa63aSLi Zhang mlx5_aso_age_action_update(sh, i); 70829efa63aSLi Zhang sq->tail += i; 70929efa63aSLi Zhang rte_io_wmb(); 71029efa63aSLi Zhang cq->cq_obj.db_rec[0] = rte_cpu_to_be_32(cq->cq_ci); 71129efa63aSLi Zhang } 71229efa63aSLi Zhang return i; 71329efa63aSLi Zhang } 71429efa63aSLi Zhang 71529efa63aSLi Zhang /** 71629efa63aSLi Zhang * Periodically read CQEs and send WQEs to ASO SQ. 71729efa63aSLi Zhang * 71829efa63aSLi Zhang * @param[in] arg 71929efa63aSLi Zhang * Shared device context containing the ASO SQ. 72029efa63aSLi Zhang */ 72129efa63aSLi Zhang static void 72229efa63aSLi Zhang mlx5_flow_aso_alarm(void *arg) 72329efa63aSLi Zhang { 72429efa63aSLi Zhang struct mlx5_dev_ctx_shared *sh = arg; 72529efa63aSLi Zhang struct mlx5_aso_sq *sq = &sh->aso_age_mng->aso_sq; 72629efa63aSLi Zhang uint32_t us = 100u; 72729efa63aSLi Zhang uint16_t n; 72829efa63aSLi Zhang 7297cf2d15aSJiawei Wang rte_rwlock_read_lock(&sh->aso_age_mng->resize_rwl); 73029efa63aSLi Zhang n = sh->aso_age_mng->next; 7317cf2d15aSJiawei Wang rte_rwlock_read_unlock(&sh->aso_age_mng->resize_rwl); 73229efa63aSLi Zhang mlx5_aso_completion_handle(sh); 73329efa63aSLi Zhang if (sq->next == n) { 73429efa63aSLi Zhang /* End of loop: wait 1 second. */ 73529efa63aSLi Zhang us = US_PER_S; 73629efa63aSLi Zhang sq->next = 0; 73729efa63aSLi Zhang } 7385dfa003dSMichael Baum mlx5_aso_sq_enqueue_burst(sh, n); 73929efa63aSLi Zhang if (rte_eal_alarm_set(us, mlx5_flow_aso_alarm, sh)) 74029efa63aSLi Zhang DRV_LOG(ERR, "Cannot reinitialize aso alarm."); 74129efa63aSLi Zhang } 74229efa63aSLi Zhang 74329efa63aSLi Zhang /** 74429efa63aSLi Zhang * API to start ASO access using ASO SQ. 74529efa63aSLi Zhang * 74629efa63aSLi Zhang * @param[in] sh 74729efa63aSLi Zhang * Pointer to shared device context. 74829efa63aSLi Zhang * 74929efa63aSLi Zhang * @return 75029efa63aSLi Zhang * 0 on success, a negative errno value otherwise and rte_errno is set. 75129efa63aSLi Zhang */ 75229efa63aSLi Zhang int 75329efa63aSLi Zhang mlx5_aso_flow_hit_queue_poll_start(struct mlx5_dev_ctx_shared *sh) 75429efa63aSLi Zhang { 75529efa63aSLi Zhang if (rte_eal_alarm_set(US_PER_S, mlx5_flow_aso_alarm, sh)) { 75629efa63aSLi Zhang DRV_LOG(ERR, "Cannot reinitialize ASO age alarm."); 75729efa63aSLi Zhang return -rte_errno; 75829efa63aSLi Zhang } 75929efa63aSLi Zhang return 0; 76029efa63aSLi Zhang } 76129efa63aSLi Zhang 76229efa63aSLi Zhang /** 76329efa63aSLi Zhang * API to stop ASO access using ASO SQ. 76429efa63aSLi Zhang * 76529efa63aSLi Zhang * @param[in] sh 76629efa63aSLi Zhang * Pointer to shared device context. 76729efa63aSLi Zhang * 76829efa63aSLi Zhang * @return 76929efa63aSLi Zhang * 0 on success, a negative errno value otherwise and rte_errno is set. 77029efa63aSLi Zhang */ 77129efa63aSLi Zhang int 77229efa63aSLi Zhang mlx5_aso_flow_hit_queue_poll_stop(struct mlx5_dev_ctx_shared *sh) 77329efa63aSLi Zhang { 77429efa63aSLi Zhang int retries = 1024; 77529efa63aSLi Zhang 77629efa63aSLi Zhang if (!sh->aso_age_mng->aso_sq.sq_obj.sq) 77729efa63aSLi Zhang return -EINVAL; 77829efa63aSLi Zhang rte_errno = 0; 77929efa63aSLi Zhang while (--retries) { 78029efa63aSLi Zhang rte_eal_alarm_cancel(mlx5_flow_aso_alarm, sh); 78129efa63aSLi Zhang if (rte_errno != EINPROGRESS) 78229efa63aSLi Zhang break; 78329efa63aSLi Zhang rte_pause(); 78429efa63aSLi Zhang } 78529efa63aSLi Zhang return -rte_errno; 78629efa63aSLi Zhang } 787e93c58daSLi Zhang 788e93c58daSLi Zhang static uint16_t 7895dfa003dSMichael Baum mlx5_aso_mtr_sq_enqueue_single(struct mlx5_dev_ctx_shared *sh, 7905dfa003dSMichael Baum struct mlx5_aso_sq *sq, 79124865366SAlexander Kozyrev struct mlx5_aso_mtr *aso_mtr, 79248fbb0e9SAlexander Kozyrev struct mlx5_mtr_bulk *bulk, 793478ba4bbSSuanming Mou bool need_lock, 7944359d9d1SGregory Etelson struct mlx5_hw_q_job *job, 795478ba4bbSSuanming Mou bool push) 796e93c58daSLi Zhang { 797e93c58daSLi Zhang volatile struct mlx5_aso_wqe *wqe = NULL; 798e93c58daSLi Zhang struct mlx5_flow_meter_info *fm = NULL; 799aa065a9cSLi Zhang struct mlx5_flow_meter_profile *fmp; 800e93c58daSLi Zhang uint16_t size = 1 << sq->log_desc_n; 801e93c58daSLi Zhang uint16_t mask = size - 1; 802cfd2037cSLi Zhang uint16_t res; 803e93c58daSLi Zhang uint32_t dseg_idx = 0; 804e93c58daSLi Zhang struct mlx5_aso_mtr_pool *pool = NULL; 8056b838de3SShun Hao uint32_t param_le; 80624865366SAlexander Kozyrev int id; 807e93c58daSLi Zhang 80848fbb0e9SAlexander Kozyrev if (need_lock) 809cfd2037cSLi Zhang rte_spinlock_lock(&sq->sqsl); 810cfd2037cSLi Zhang res = size - (uint16_t)(sq->head - sq->tail); 811e93c58daSLi Zhang if (unlikely(!res)) { 812e93c58daSLi Zhang DRV_LOG(ERR, "Fail: SQ is full and no free WQE to send"); 81348fbb0e9SAlexander Kozyrev if (need_lock) 814cfd2037cSLi Zhang rte_spinlock_unlock(&sq->sqsl); 815e93c58daSLi Zhang return 0; 816e93c58daSLi Zhang } 817e93c58daSLi Zhang wqe = &sq->sq_obj.aso_wqes[sq->head & mask]; 818e93c58daSLi Zhang rte_prefetch0(&sq->sq_obj.aso_wqes[(sq->head + 1) & mask]); 819e93c58daSLi Zhang /* Fill next WQE. */ 820e93c58daSLi Zhang fm = &aso_mtr->fm; 8214359d9d1SGregory Etelson sq->elts[sq->head & mask].user_data = job ? job : (void *)aso_mtr; 82224865366SAlexander Kozyrev if (aso_mtr->type == ASO_METER_INDIRECT) { 82348fbb0e9SAlexander Kozyrev if (likely(sh->config.dv_flow_en == 2)) 82448fbb0e9SAlexander Kozyrev pool = aso_mtr->pool; 82548fbb0e9SAlexander Kozyrev else 826e93c58daSLi Zhang pool = container_of(aso_mtr, struct mlx5_aso_mtr_pool, 827e93c58daSLi Zhang mtrs[aso_mtr->offset]); 82824865366SAlexander Kozyrev id = pool->devx_obj->id; 82924865366SAlexander Kozyrev } else { 83024865366SAlexander Kozyrev id = bulk->devx_obj->id; 83124865366SAlexander Kozyrev } 83224865366SAlexander Kozyrev wqe->general_cseg.misc = rte_cpu_to_be_32(id + 833e93c58daSLi Zhang (aso_mtr->offset >> 1)); 83424865366SAlexander Kozyrev wqe->general_cseg.opcode = 83524865366SAlexander Kozyrev rte_cpu_to_be_32(MLX5_OPCODE_ACCESS_ASO | 83624865366SAlexander Kozyrev (ASO_OPC_MOD_POLICER << WQE_CSEG_OPC_MOD_OFFSET) | 837e93c58daSLi Zhang sq->pi << WQE_CSEG_WQE_INDEX_OFFSET); 838e93c58daSLi Zhang /* There are 2 meters in one ASO cache line. */ 839e93c58daSLi Zhang dseg_idx = aso_mtr->offset & 0x1; 840e93c58daSLi Zhang wqe->aso_cseg.data_mask = 841e93c58daSLi Zhang RTE_BE64(MLX5_IFC_FLOW_METER_PARAM_MASK << (32 * !dseg_idx)); 842e93c58daSLi Zhang if (fm->is_enable) { 843e93c58daSLi Zhang wqe->aso_dseg.mtrs[dseg_idx].cbs_cir = 844e93c58daSLi Zhang fm->profile->srtcm_prm.cbs_cir; 845e93c58daSLi Zhang wqe->aso_dseg.mtrs[dseg_idx].ebs_eir = 846e93c58daSLi Zhang fm->profile->srtcm_prm.ebs_eir; 847e93c58daSLi Zhang } else { 848e93c58daSLi Zhang wqe->aso_dseg.mtrs[dseg_idx].cbs_cir = 849e93c58daSLi Zhang RTE_BE32(MLX5_IFC_FLOW_METER_DISABLE_CBS_CIR_VAL); 850e93c58daSLi Zhang wqe->aso_dseg.mtrs[dseg_idx].ebs_eir = 0; 851e93c58daSLi Zhang } 852aa065a9cSLi Zhang fmp = fm->profile; 8536b838de3SShun Hao param_le = (1 << ASO_DSEG_VALID_OFFSET); 8546b838de3SShun Hao if (fm->color_aware) 8556b838de3SShun Hao param_le |= (MLX5_FLOW_COLOR_UNDEFINED << ASO_DSEG_SC_OFFSET); 856aa065a9cSLi Zhang else 8576b838de3SShun Hao param_le |= (MLX5_FLOW_COLOR_GREEN << ASO_DSEG_SC_OFFSET); 8586b838de3SShun Hao if (fmp->profile.packet_mode) 8596b838de3SShun Hao param_le |= (MLX5_METER_MODE_PKT << ASO_DSEG_MTR_MODE); 8606b838de3SShun Hao wqe->aso_dseg.mtrs[dseg_idx].v_bo_sc_bbog_mm = RTE_BE32(param_le); 86133a7493cSBing Zhao switch (fmp->profile.alg) { 86233a7493cSBing Zhao case RTE_MTR_SRTCM_RFC2697: 8639b5463dfSBing Zhao /* Only needed for RFC2697. */ 8649b5463dfSBing Zhao if (fm->profile->srtcm_prm.ebs_eir) 8659b5463dfSBing Zhao wqe->aso_dseg.mtrs[dseg_idx].v_bo_sc_bbog_mm |= 8669b5463dfSBing Zhao RTE_BE32(1 << ASO_DSEG_BO_OFFSET); 86733a7493cSBing Zhao break; 86833a7493cSBing Zhao case RTE_MTR_TRTCM_RFC2698: 86933a7493cSBing Zhao wqe->aso_dseg.mtrs[dseg_idx].v_bo_sc_bbog_mm |= 87033a7493cSBing Zhao RTE_BE32(1 << ASO_DSEG_BBOG_OFFSET); 87133a7493cSBing Zhao break; 87233a7493cSBing Zhao case RTE_MTR_TRTCM_RFC4115: 8736b838de3SShun Hao wqe->aso_dseg.mtrs[dseg_idx].v_bo_sc_bbog_mm |= 8746b838de3SShun Hao RTE_BE32(1 << ASO_DSEG_BO_OFFSET); 8756b838de3SShun Hao break; 87633a7493cSBing Zhao default: 87733a7493cSBing Zhao break; 87833a7493cSBing Zhao } 87933a7493cSBing Zhao /* 88033a7493cSBing Zhao * Note: 88133a7493cSBing Zhao * Due to software performance reason, the token fields will not be 88233a7493cSBing Zhao * set when posting the WQE to ASO SQ. It will be filled by the HW 88333a7493cSBing Zhao * automatically. 88433a7493cSBing Zhao */ 885e93c58daSLi Zhang sq->head++; 886e93c58daSLi Zhang sq->pi += 2;/* Each WQE contains 2 WQEBB's. */ 887478ba4bbSSuanming Mou if (push) { 8885dfa003dSMichael Baum mlx5_doorbell_ring(&sh->tx_uar.bf_db, *(volatile uint64_t *)wqe, 8895dfa003dSMichael Baum sq->pi, &sq->sq_obj.db_rec[MLX5_SND_DBR], 8905dfa003dSMichael Baum !sh->tx_uar.dbnc); 891478ba4bbSSuanming Mou sq->db_pi = sq->pi; 892478ba4bbSSuanming Mou } 893478ba4bbSSuanming Mou sq->db = wqe; 89448fbb0e9SAlexander Kozyrev if (need_lock) 895cfd2037cSLi Zhang rte_spinlock_unlock(&sq->sqsl); 896e93c58daSLi Zhang return 1; 897e93c58daSLi Zhang } 898e93c58daSLi Zhang 899e93c58daSLi Zhang static void 90048fbb0e9SAlexander Kozyrev mlx5_aso_mtr_completion_handle(struct mlx5_aso_sq *sq, bool need_lock) 901e93c58daSLi Zhang { 902e93c58daSLi Zhang struct mlx5_aso_cq *cq = &sq->cq; 903e93c58daSLi Zhang volatile struct mlx5_cqe *restrict cqe; 904e93c58daSLi Zhang const unsigned int cq_size = 1 << cq->log_desc_n; 905e93c58daSLi Zhang const unsigned int mask = cq_size - 1; 906e93c58daSLi Zhang uint32_t idx; 907e93c58daSLi Zhang uint32_t next_idx = cq->cq_ci & mask; 908cfd2037cSLi Zhang uint16_t max; 9094359d9d1SGregory Etelson uint16_t i, n = 0; 910e93c58daSLi Zhang int ret; 911e93c58daSLi Zhang 91248fbb0e9SAlexander Kozyrev if (need_lock) 913cfd2037cSLi Zhang rte_spinlock_lock(&sq->sqsl); 914cfd2037cSLi Zhang max = (uint16_t)(sq->head - sq->tail); 915cfd2037cSLi Zhang if (unlikely(!max)) { 91629f99df4SWeiguo Li if (need_lock) 917cfd2037cSLi Zhang rte_spinlock_unlock(&sq->sqsl); 918e93c58daSLi Zhang return; 919cfd2037cSLi Zhang } 920e93c58daSLi Zhang do { 921e93c58daSLi Zhang idx = next_idx; 922e93c58daSLi Zhang next_idx = (cq->cq_ci + 1) & mask; 923e93c58daSLi Zhang rte_prefetch0(&cq->cq_obj.cqes[next_idx]); 924e93c58daSLi Zhang cqe = &cq->cq_obj.cqes[idx]; 925e93c58daSLi Zhang ret = check_cqe(cqe, cq_size, cq->cq_ci); 926e93c58daSLi Zhang /* 927e93c58daSLi Zhang * Be sure owner read is done before any other cookie field or 928e93c58daSLi Zhang * opaque field. 929e93c58daSLi Zhang */ 930e93c58daSLi Zhang rte_io_rmb(); 931e93c58daSLi Zhang if (ret != MLX5_CQE_STATUS_SW_OWN) { 932e93c58daSLi Zhang if (likely(ret == MLX5_CQE_STATUS_HW_OWN)) 933e93c58daSLi Zhang break; 934e93c58daSLi Zhang mlx5_aso_cqe_err_handle(sq); 935e93c58daSLi Zhang } else { 936e93c58daSLi Zhang n++; 937e93c58daSLi Zhang } 938e93c58daSLi Zhang cq->cq_ci++; 939e93c58daSLi Zhang } while (1); 940e93c58daSLi Zhang if (likely(n)) { 9414359d9d1SGregory Etelson uint8_t exp_state = ASO_METER_WAIT; 9424359d9d1SGregory Etelson struct mlx5_aso_mtr *aso_mtr; 9434359d9d1SGregory Etelson __rte_unused bool verdict; 9444359d9d1SGregory Etelson 9454359d9d1SGregory Etelson for (i = 0; i < n; ++i) { 9464359d9d1SGregory Etelson aso_mtr = sq->elts[(sq->tail + i) & mask].mtr; 9474359d9d1SGregory Etelson MLX5_ASSERT(aso_mtr); 948e12a0166STyler Retzlaff verdict = rte_atomic_compare_exchange_strong_explicit(&aso_mtr->state, 9494359d9d1SGregory Etelson &exp_state, ASO_METER_READY, 950e12a0166STyler Retzlaff rte_memory_order_relaxed, 951e12a0166STyler Retzlaff rte_memory_order_relaxed); 9524359d9d1SGregory Etelson MLX5_ASSERT(verdict); 9534359d9d1SGregory Etelson } 954e93c58daSLi Zhang sq->tail += n; 955e93c58daSLi Zhang rte_io_wmb(); 956e93c58daSLi Zhang cq->cq_obj.db_rec[0] = rte_cpu_to_be_32(cq->cq_ci); 957e93c58daSLi Zhang } 95848fbb0e9SAlexander Kozyrev if (need_lock) 959cfd2037cSLi Zhang rte_spinlock_unlock(&sq->sqsl); 960e93c58daSLi Zhang } 961e93c58daSLi Zhang 9624359d9d1SGregory Etelson static __rte_always_inline struct mlx5_aso_sq * 9634359d9d1SGregory Etelson mlx5_aso_mtr_select_sq(struct mlx5_dev_ctx_shared *sh, uint32_t queue, 9644359d9d1SGregory Etelson struct mlx5_aso_mtr *mtr, bool *need_lock) 9654359d9d1SGregory Etelson { 9664359d9d1SGregory Etelson struct mlx5_aso_sq *sq; 9674359d9d1SGregory Etelson 9684359d9d1SGregory Etelson if (likely(sh->config.dv_flow_en == 2) && 9694359d9d1SGregory Etelson mtr->type == ASO_METER_INDIRECT) { 9704359d9d1SGregory Etelson if (queue == MLX5_HW_INV_QUEUE) { 9714359d9d1SGregory Etelson sq = &mtr->pool->sq[mtr->pool->nb_sq - 1]; 9724359d9d1SGregory Etelson *need_lock = true; 9734359d9d1SGregory Etelson } else { 9744359d9d1SGregory Etelson sq = &mtr->pool->sq[queue]; 9754359d9d1SGregory Etelson *need_lock = false; 9764359d9d1SGregory Etelson } 9774359d9d1SGregory Etelson } else { 9784359d9d1SGregory Etelson sq = &sh->mtrmng->pools_mng.sq; 9794359d9d1SGregory Etelson *need_lock = true; 9804359d9d1SGregory Etelson } 9814359d9d1SGregory Etelson return sq; 9824359d9d1SGregory Etelson } 9834359d9d1SGregory Etelson 9844359d9d1SGregory Etelson #if defined(HAVE_MLX5_HWS_SUPPORT) 9854359d9d1SGregory Etelson static void 9864359d9d1SGregory Etelson mlx5_aso_poll_cq_mtr_hws(struct mlx5_priv *priv, struct mlx5_aso_sq *sq) 9874359d9d1SGregory Etelson { 9884359d9d1SGregory Etelson #define MLX5_HWS_MTR_CMPL_NUM 4 9894359d9d1SGregory Etelson 9904359d9d1SGregory Etelson int i, ret; 9914359d9d1SGregory Etelson struct mlx5_aso_mtr *mtr; 9924359d9d1SGregory Etelson uint8_t exp_state = ASO_METER_WAIT; 9934359d9d1SGregory Etelson struct rte_flow_op_result res[MLX5_HWS_MTR_CMPL_NUM]; 9944359d9d1SGregory Etelson __rte_unused bool verdict; 9954359d9d1SGregory Etelson 9964359d9d1SGregory Etelson rte_spinlock_lock(&sq->sqsl); 9974359d9d1SGregory Etelson repeat: 9984359d9d1SGregory Etelson ret = mlx5_aso_pull_completion(sq, res, MLX5_HWS_MTR_CMPL_NUM); 9994359d9d1SGregory Etelson if (ret) { 10004359d9d1SGregory Etelson for (i = 0; i < ret; i++) { 10014359d9d1SGregory Etelson struct mlx5_hw_q_job *job = res[i].user_data; 10024359d9d1SGregory Etelson 10034359d9d1SGregory Etelson MLX5_ASSERT(job); 10044359d9d1SGregory Etelson mtr = mlx5_ipool_get(priv->hws_mpool->idx_pool, 10054359d9d1SGregory Etelson MLX5_INDIRECT_ACTION_IDX_GET(job->action)); 10064359d9d1SGregory Etelson MLX5_ASSERT(mtr); 1007e12a0166STyler Retzlaff verdict = rte_atomic_compare_exchange_strong_explicit(&mtr->state, 10084359d9d1SGregory Etelson &exp_state, ASO_METER_READY, 1009e12a0166STyler Retzlaff rte_memory_order_relaxed, 1010e12a0166STyler Retzlaff rte_memory_order_relaxed); 10114359d9d1SGregory Etelson MLX5_ASSERT(verdict); 10124359d9d1SGregory Etelson flow_hw_job_put(priv, job, CTRL_QUEUE_ID(priv)); 10134359d9d1SGregory Etelson } 10144359d9d1SGregory Etelson if (ret == MLX5_HWS_MTR_CMPL_NUM) 10154359d9d1SGregory Etelson goto repeat; 10164359d9d1SGregory Etelson } 10174359d9d1SGregory Etelson rte_spinlock_unlock(&sq->sqsl); 10184359d9d1SGregory Etelson 10194359d9d1SGregory Etelson #undef MLX5_HWS_MTR_CMPL_NUM 10204359d9d1SGregory Etelson } 10214359d9d1SGregory Etelson #else 10224359d9d1SGregory Etelson static void 10234359d9d1SGregory Etelson mlx5_aso_poll_cq_mtr_hws(__rte_unused struct mlx5_priv *priv, __rte_unused struct mlx5_aso_sq *sq) 10244359d9d1SGregory Etelson { 10254359d9d1SGregory Etelson MLX5_ASSERT(false); 10264359d9d1SGregory Etelson } 10274359d9d1SGregory Etelson #endif 10284359d9d1SGregory Etelson 10294359d9d1SGregory Etelson static void 10304359d9d1SGregory Etelson mlx5_aso_poll_cq_mtr_sws(__rte_unused struct mlx5_priv *priv, 10314359d9d1SGregory Etelson struct mlx5_aso_sq *sq) 10324359d9d1SGregory Etelson { 10334359d9d1SGregory Etelson mlx5_aso_mtr_completion_handle(sq, true); 10344359d9d1SGregory Etelson } 10354359d9d1SGregory Etelson 10364359d9d1SGregory Etelson typedef void (*poll_cq_t)(struct mlx5_priv *, struct mlx5_aso_sq *); 10374359d9d1SGregory Etelson 1038e93c58daSLi Zhang /** 1039e93c58daSLi Zhang * Update meter parameter by send WQE. 1040e93c58daSLi Zhang * 1041e93c58daSLi Zhang * @param[in] dev 1042e93c58daSLi Zhang * Pointer to Ethernet device. 1043e93c58daSLi Zhang * @param[in] priv 1044e93c58daSLi Zhang * Pointer to mlx5 private data structure. 1045e93c58daSLi Zhang * @param[in] fm 1046e93c58daSLi Zhang * Pointer to flow meter to be modified. 1047e93c58daSLi Zhang * 1048e93c58daSLi Zhang * @return 1049e93c58daSLi Zhang * 0 on success, a negative errno value otherwise and rte_errno is set. 1050e93c58daSLi Zhang */ 1051e93c58daSLi Zhang int 10524359d9d1SGregory Etelson mlx5_aso_meter_update_by_wqe(struct mlx5_priv *priv, uint32_t queue, 105324865366SAlexander Kozyrev struct mlx5_aso_mtr *mtr, 1054478ba4bbSSuanming Mou struct mlx5_mtr_bulk *bulk, 10554359d9d1SGregory Etelson struct mlx5_hw_q_job *job, bool push) 1056e93c58daSLi Zhang { 105748fbb0e9SAlexander Kozyrev bool need_lock; 10584359d9d1SGregory Etelson struct mlx5_dev_ctx_shared *sh = priv->sh; 10594359d9d1SGregory Etelson struct mlx5_aso_sq *sq = 10604359d9d1SGregory Etelson mlx5_aso_mtr_select_sq(sh, queue, mtr, &need_lock); 10614359d9d1SGregory Etelson uint32_t poll_wqe_times = MLX5_MTR_POLL_WQE_CQE_TIMES; 10624359d9d1SGregory Etelson poll_cq_t poll_mtr_cq = 10634359d9d1SGregory Etelson job ? mlx5_aso_poll_cq_mtr_hws : mlx5_aso_poll_cq_mtr_sws; 1064478ba4bbSSuanming Mou int ret; 1065e93c58daSLi Zhang 1066478ba4bbSSuanming Mou if (queue != MLX5_HW_INV_QUEUE) { 1067478ba4bbSSuanming Mou ret = mlx5_aso_mtr_sq_enqueue_single(sh, sq, mtr, bulk, 10684359d9d1SGregory Etelson need_lock, job, push); 1069478ba4bbSSuanming Mou return ret > 0 ? 0 : -1; 1070478ba4bbSSuanming Mou } 1071e93c58daSLi Zhang do { 10724359d9d1SGregory Etelson poll_mtr_cq(priv, sq); 1073478ba4bbSSuanming Mou if (mlx5_aso_mtr_sq_enqueue_single(sh, sq, mtr, bulk, 10744359d9d1SGregory Etelson need_lock, job, true)) 1075e93c58daSLi Zhang return 0; 1076e93c58daSLi Zhang /* Waiting for wqe resource. */ 1077e93c58daSLi Zhang rte_delay_us_sleep(MLX5_ASO_WQE_CQE_RESPONSE_DELAY); 1078e93c58daSLi Zhang } while (--poll_wqe_times); 10795f0d54f3SLi Zhang DRV_LOG(ERR, "Fail to send WQE for ASO meter offset %d", 10805f0d54f3SLi Zhang mtr->offset); 1081e93c58daSLi Zhang return -1; 1082e93c58daSLi Zhang } 1083e93c58daSLi Zhang 1084e93c58daSLi Zhang /** 1085e93c58daSLi Zhang * Wait for meter to be ready. 1086e93c58daSLi Zhang * 1087e93c58daSLi Zhang * @param[in] dev 1088e93c58daSLi Zhang * Pointer to Ethernet device. 1089e93c58daSLi Zhang * @param[in] priv 1090e93c58daSLi Zhang * Pointer to mlx5 private data structure. 1091e93c58daSLi Zhang * @param[in] fm 1092e93c58daSLi Zhang * Pointer to flow meter to be modified. 1093e93c58daSLi Zhang * 1094e93c58daSLi Zhang * @return 1095e93c58daSLi Zhang * 0 on success, a negative errno value otherwise and rte_errno is set. 1096e93c58daSLi Zhang */ 1097e93c58daSLi Zhang int 10984359d9d1SGregory Etelson mlx5_aso_mtr_wait(struct mlx5_priv *priv, 10994359d9d1SGregory Etelson struct mlx5_aso_mtr *mtr, bool is_tmpl_api) 1100e93c58daSLi Zhang { 110148fbb0e9SAlexander Kozyrev bool need_lock; 11024359d9d1SGregory Etelson struct mlx5_aso_sq *sq; 11034359d9d1SGregory Etelson struct mlx5_dev_ctx_shared *sh = priv->sh; 11044359d9d1SGregory Etelson uint32_t poll_cqe_times = MLX5_MTR_POLL_WQE_CQE_TIMES; 1105e12a0166STyler Retzlaff uint8_t state = rte_atomic_load_explicit(&mtr->state, rte_memory_order_relaxed); 11064359d9d1SGregory Etelson poll_cq_t poll_mtr_cq = 11074359d9d1SGregory Etelson is_tmpl_api ? mlx5_aso_poll_cq_mtr_hws : mlx5_aso_poll_cq_mtr_sws; 1108e93c58daSLi Zhang 1109478ba4bbSSuanming Mou if (state == ASO_METER_READY || state == ASO_METER_WAIT_ASYNC) 1110e93c58daSLi Zhang return 0; 11114359d9d1SGregory Etelson sq = mlx5_aso_mtr_select_sq(sh, MLX5_HW_INV_QUEUE, mtr, &need_lock); 1112e93c58daSLi Zhang do { 11134359d9d1SGregory Etelson poll_mtr_cq(priv, sq); 1114e12a0166STyler Retzlaff if (rte_atomic_load_explicit(&mtr->state, rte_memory_order_relaxed) == 1115e93c58daSLi Zhang ASO_METER_READY) 1116e93c58daSLi Zhang return 0; 1117e93c58daSLi Zhang /* Waiting for CQE ready. */ 1118e93c58daSLi Zhang rte_delay_us_sleep(MLX5_ASO_WQE_CQE_RESPONSE_DELAY); 1119e93c58daSLi Zhang } while (--poll_cqe_times); 11205f0d54f3SLi Zhang DRV_LOG(ERR, "Fail to poll CQE ready for ASO meter offset %d", 11215f0d54f3SLi Zhang mtr->offset); 1122e93c58daSLi Zhang return -1; 1123e93c58daSLi Zhang } 1124ebaf1b31SBing Zhao 1125463170a7SSuanming Mou static inline struct mlx5_aso_sq* 1126463170a7SSuanming Mou __mlx5_aso_ct_get_sq_in_hws(uint32_t queue, 1127463170a7SSuanming Mou struct mlx5_aso_ct_pool *pool) 1128463170a7SSuanming Mou { 1129463170a7SSuanming Mou return (queue == MLX5_HW_INV_QUEUE) ? 1130463170a7SSuanming Mou pool->shared_sq : &pool->sq[queue]; 1131463170a7SSuanming Mou } 1132463170a7SSuanming Mou 1133463170a7SSuanming Mou static inline struct mlx5_aso_sq* 1134463170a7SSuanming Mou __mlx5_aso_ct_get_sq_in_sws(struct mlx5_dev_ctx_shared *sh, 1135463170a7SSuanming Mou struct mlx5_aso_ct_action *ct) 1136463170a7SSuanming Mou { 1137463170a7SSuanming Mou return &sh->ct_mng->aso_sqs[ct->offset & (MLX5_ASO_CT_SQ_NUM - 1)]; 1138463170a7SSuanming Mou } 1139463170a7SSuanming Mou 1140463170a7SSuanming Mou static inline struct mlx5_aso_ct_pool* 1141463170a7SSuanming Mou __mlx5_aso_ct_get_pool(struct mlx5_dev_ctx_shared *sh, 1142463170a7SSuanming Mou struct mlx5_aso_ct_action *ct) 1143463170a7SSuanming Mou { 1144463170a7SSuanming Mou if (likely(sh->config.dv_flow_en == 2)) 1145463170a7SSuanming Mou return ct->pool; 1146463170a7SSuanming Mou return container_of(ct, struct mlx5_aso_ct_pool, actions[ct->offset]); 1147463170a7SSuanming Mou } 1148463170a7SSuanming Mou 1149463170a7SSuanming Mou int 1150463170a7SSuanming Mou mlx5_aso_ct_queue_uninit(struct mlx5_dev_ctx_shared *sh, 1151463170a7SSuanming Mou struct mlx5_aso_ct_pools_mng *ct_mng) 1152463170a7SSuanming Mou { 1153463170a7SSuanming Mou uint32_t i; 1154463170a7SSuanming Mou 1155463170a7SSuanming Mou /* 64B per object for query. */ 1156463170a7SSuanming Mou for (i = 0; i < ct_mng->nb_sq; i++) { 1157463170a7SSuanming Mou if (ct_mng->aso_sqs[i].mr.addr) 1158463170a7SSuanming Mou mlx5_aso_dereg_mr(sh->cdev, &ct_mng->aso_sqs[i].mr); 1159463170a7SSuanming Mou mlx5_aso_destroy_sq(&ct_mng->aso_sqs[i]); 1160463170a7SSuanming Mou } 1161463170a7SSuanming Mou return 0; 1162463170a7SSuanming Mou } 1163463170a7SSuanming Mou 1164463170a7SSuanming Mou /** 1165463170a7SSuanming Mou * API to create and initialize CT Send Queue used for ASO access. 1166463170a7SSuanming Mou * 1167463170a7SSuanming Mou * @param[in] sh 1168463170a7SSuanming Mou * Pointer to shared device context. 1169463170a7SSuanming Mou * @param[in] ct_mng 1170463170a7SSuanming Mou * Pointer to the CT management struct. 1171463170a7SSuanming Mou * *param[in] nb_queues 1172463170a7SSuanming Mou * Number of queues to be allocated. 1173463170a7SSuanming Mou * 1174463170a7SSuanming Mou * @return 1175463170a7SSuanming Mou * 0 on success, a negative errno value otherwise and rte_errno is set. 1176463170a7SSuanming Mou */ 1177463170a7SSuanming Mou int 1178463170a7SSuanming Mou mlx5_aso_ct_queue_init(struct mlx5_dev_ctx_shared *sh, 1179463170a7SSuanming Mou struct mlx5_aso_ct_pools_mng *ct_mng, 1180463170a7SSuanming Mou uint32_t nb_queues) 1181463170a7SSuanming Mou { 1182463170a7SSuanming Mou uint32_t i; 1183463170a7SSuanming Mou 1184463170a7SSuanming Mou /* 64B per object for query. */ 1185463170a7SSuanming Mou for (i = 0; i < nb_queues; i++) { 1186463170a7SSuanming Mou if (mlx5_aso_reg_mr(sh->cdev, 64 * (1 << MLX5_ASO_QUEUE_LOG_DESC), 1187463170a7SSuanming Mou &ct_mng->aso_sqs[i].mr)) 1188463170a7SSuanming Mou goto error; 1189463170a7SSuanming Mou if (mlx5_aso_sq_create(sh->cdev, &ct_mng->aso_sqs[i], 1190463170a7SSuanming Mou sh->tx_uar.obj, 1191463170a7SSuanming Mou MLX5_ASO_QUEUE_LOG_DESC)) 1192463170a7SSuanming Mou goto error; 1193463170a7SSuanming Mou mlx5_aso_ct_init_sq(&ct_mng->aso_sqs[i]); 1194463170a7SSuanming Mou } 1195463170a7SSuanming Mou ct_mng->nb_sq = nb_queues; 1196463170a7SSuanming Mou return 0; 1197463170a7SSuanming Mou error: 1198463170a7SSuanming Mou do { 1199463170a7SSuanming Mou if (ct_mng->aso_sqs[i].mr.addr) 1200463170a7SSuanming Mou mlx5_aso_dereg_mr(sh->cdev, &ct_mng->aso_sqs[i].mr); 1201463170a7SSuanming Mou mlx5_aso_destroy_sq(&ct_mng->aso_sqs[i]); 1202463170a7SSuanming Mou } while (i--); 1203463170a7SSuanming Mou ct_mng->nb_sq = 0; 1204463170a7SSuanming Mou return -1; 1205463170a7SSuanming Mou } 1206463170a7SSuanming Mou 1207ebaf1b31SBing Zhao /* 1208ebaf1b31SBing Zhao * Post a WQE to the ASO CT SQ to modify the context. 1209ebaf1b31SBing Zhao * 12105dfa003dSMichael Baum * @param[in] sh 12115dfa003dSMichael Baum * Pointer to shared device context. 1212ebaf1b31SBing Zhao * @param[in] ct 1213ebaf1b31SBing Zhao * Pointer to the generic CT structure related to the context. 1214ebaf1b31SBing Zhao * @param[in] profile 1215ebaf1b31SBing Zhao * Pointer to configuration profile. 1216ebaf1b31SBing Zhao * 1217ebaf1b31SBing Zhao * @return 1218ebaf1b31SBing Zhao * 1 on success (WQE number), 0 on failure. 1219ebaf1b31SBing Zhao */ 1220ebaf1b31SBing Zhao static uint16_t 12215dfa003dSMichael Baum mlx5_aso_ct_sq_enqueue_single(struct mlx5_dev_ctx_shared *sh, 1222463170a7SSuanming Mou struct mlx5_aso_sq *sq, 1223ebaf1b31SBing Zhao struct mlx5_aso_ct_action *ct, 1224463170a7SSuanming Mou const struct rte_flow_action_conntrack *profile, 1225478ba4bbSSuanming Mou bool need_lock, 1226478ba4bbSSuanming Mou void *user_data, 1227478ba4bbSSuanming Mou bool push) 1228ebaf1b31SBing Zhao { 1229ebaf1b31SBing Zhao volatile struct mlx5_aso_wqe *wqe = NULL; 1230ebaf1b31SBing Zhao uint16_t size = 1 << sq->log_desc_n; 1231ebaf1b31SBing Zhao uint16_t mask = size - 1; 1232ebaf1b31SBing Zhao uint16_t res; 1233ebaf1b31SBing Zhao struct mlx5_aso_ct_pool *pool; 1234ebaf1b31SBing Zhao void *desg; 1235ebaf1b31SBing Zhao void *orig_dir; 1236ebaf1b31SBing Zhao void *reply_dir; 1237ebaf1b31SBing Zhao 1238463170a7SSuanming Mou if (need_lock) 1239ebaf1b31SBing Zhao rte_spinlock_lock(&sq->sqsl); 1240ebaf1b31SBing Zhao /* Prevent other threads to update the index. */ 1241ebaf1b31SBing Zhao res = size - (uint16_t)(sq->head - sq->tail); 1242ebaf1b31SBing Zhao if (unlikely(!res)) { 1243463170a7SSuanming Mou if (need_lock) 1244ebaf1b31SBing Zhao rte_spinlock_unlock(&sq->sqsl); 1245ebaf1b31SBing Zhao DRV_LOG(ERR, "Fail: SQ is full and no free WQE to send"); 1246ebaf1b31SBing Zhao return 0; 1247ebaf1b31SBing Zhao } 1248ebaf1b31SBing Zhao wqe = &sq->sq_obj.aso_wqes[sq->head & mask]; 1249ebaf1b31SBing Zhao rte_prefetch0(&sq->sq_obj.aso_wqes[(sq->head + 1) & mask]); 1250ebaf1b31SBing Zhao /* Fill next WQE. */ 1251478ba4bbSSuanming Mou MLX5_ASO_CT_UPDATE_STATE(ct, 1252478ba4bbSSuanming Mou user_data ? ASO_CONNTRACK_WAIT_ASYNC : ASO_CONNTRACK_WAIT); 1253478ba4bbSSuanming Mou if (user_data) { 1254478ba4bbSSuanming Mou sq->elts[sq->head & mask].user_data = user_data; 1255478ba4bbSSuanming Mou } else { 1256ebaf1b31SBing Zhao sq->elts[sq->head & mask].ct = ct; 1257cf756556SBing Zhao sq->elts[sq->head & mask].query_data = NULL; 1258478ba4bbSSuanming Mou } 1259463170a7SSuanming Mou pool = __mlx5_aso_ct_get_pool(sh, ct); 1260478ba4bbSSuanming Mou 1261ebaf1b31SBing Zhao /* Each WQE will have a single CT object. */ 1262ebaf1b31SBing Zhao wqe->general_cseg.misc = rte_cpu_to_be_32(pool->devx_obj->id + 1263ebaf1b31SBing Zhao ct->offset); 1264ebaf1b31SBing Zhao wqe->general_cseg.opcode = rte_cpu_to_be_32(MLX5_OPCODE_ACCESS_ASO | 1265ebaf1b31SBing Zhao (ASO_OPC_MOD_CONNECTION_TRACKING << 1266ebaf1b31SBing Zhao WQE_CSEG_OPC_MOD_OFFSET) | 1267ebaf1b31SBing Zhao sq->pi << WQE_CSEG_WQE_INDEX_OFFSET); 1268ebaf1b31SBing Zhao wqe->aso_cseg.operand_masks = rte_cpu_to_be_32 1269ebaf1b31SBing Zhao (0u | 1270ebaf1b31SBing Zhao (ASO_OPER_LOGICAL_OR << ASO_CSEG_COND_OPER_OFFSET) | 1271ebaf1b31SBing Zhao (ASO_OP_ALWAYS_TRUE << ASO_CSEG_COND_1_OPER_OFFSET) | 1272ebaf1b31SBing Zhao (ASO_OP_ALWAYS_TRUE << ASO_CSEG_COND_0_OPER_OFFSET) | 1273ebaf1b31SBing Zhao (BYTEWISE_64BYTE << ASO_CSEG_DATA_MASK_MODE_OFFSET)); 1274ebaf1b31SBing Zhao wqe->aso_cseg.data_mask = UINT64_MAX; 1275ebaf1b31SBing Zhao /* To make compiler happy. */ 1276ebaf1b31SBing Zhao desg = (void *)(uintptr_t)wqe->aso_dseg.data; 1277ebaf1b31SBing Zhao MLX5_SET(conn_track_aso, desg, valid, 1); 1278ebaf1b31SBing Zhao MLX5_SET(conn_track_aso, desg, state, profile->state); 1279ebaf1b31SBing Zhao MLX5_SET(conn_track_aso, desg, freeze_track, !profile->enable); 1280ebaf1b31SBing Zhao MLX5_SET(conn_track_aso, desg, connection_assured, 1281ebaf1b31SBing Zhao profile->live_connection); 1282ebaf1b31SBing Zhao MLX5_SET(conn_track_aso, desg, sack_permitted, profile->selective_ack); 1283ebaf1b31SBing Zhao MLX5_SET(conn_track_aso, desg, challenged_acked, 1284ebaf1b31SBing Zhao profile->challenge_ack_passed); 1285ebaf1b31SBing Zhao /* Heartbeat, retransmission_counter, retranmission_limit_exceeded: 0 */ 1286ebaf1b31SBing Zhao MLX5_SET(conn_track_aso, desg, heartbeat, 0); 1287ebaf1b31SBing Zhao MLX5_SET(conn_track_aso, desg, max_ack_window, 1288ebaf1b31SBing Zhao profile->max_ack_window); 1289ebaf1b31SBing Zhao MLX5_SET(conn_track_aso, desg, retransmission_counter, 0); 1290ebaf1b31SBing Zhao MLX5_SET(conn_track_aso, desg, retranmission_limit_exceeded, 0); 1291ebaf1b31SBing Zhao MLX5_SET(conn_track_aso, desg, retranmission_limit, 1292ebaf1b31SBing Zhao profile->retransmission_limit); 1293ebaf1b31SBing Zhao MLX5_SET(conn_track_aso, desg, reply_direction_tcp_scale, 1294ebaf1b31SBing Zhao profile->reply_dir.scale); 1295ebaf1b31SBing Zhao MLX5_SET(conn_track_aso, desg, reply_direction_tcp_close_initiated, 1296ebaf1b31SBing Zhao profile->reply_dir.close_initiated); 1297ebaf1b31SBing Zhao /* Both directions will use the same liberal mode. */ 1298ebaf1b31SBing Zhao MLX5_SET(conn_track_aso, desg, reply_direction_tcp_liberal_enabled, 1299ebaf1b31SBing Zhao profile->liberal_mode); 1300ebaf1b31SBing Zhao MLX5_SET(conn_track_aso, desg, reply_direction_tcp_data_unacked, 1301ebaf1b31SBing Zhao profile->reply_dir.data_unacked); 1302ebaf1b31SBing Zhao MLX5_SET(conn_track_aso, desg, reply_direction_tcp_max_ack, 1303ebaf1b31SBing Zhao profile->reply_dir.last_ack_seen); 1304ebaf1b31SBing Zhao MLX5_SET(conn_track_aso, desg, original_direction_tcp_scale, 1305ebaf1b31SBing Zhao profile->original_dir.scale); 1306ebaf1b31SBing Zhao MLX5_SET(conn_track_aso, desg, original_direction_tcp_close_initiated, 1307ebaf1b31SBing Zhao profile->original_dir.close_initiated); 1308ebaf1b31SBing Zhao MLX5_SET(conn_track_aso, desg, original_direction_tcp_liberal_enabled, 1309ebaf1b31SBing Zhao profile->liberal_mode); 1310ebaf1b31SBing Zhao MLX5_SET(conn_track_aso, desg, original_direction_tcp_data_unacked, 1311ebaf1b31SBing Zhao profile->original_dir.data_unacked); 1312ebaf1b31SBing Zhao MLX5_SET(conn_track_aso, desg, original_direction_tcp_max_ack, 1313ebaf1b31SBing Zhao profile->original_dir.last_ack_seen); 1314ebaf1b31SBing Zhao MLX5_SET(conn_track_aso, desg, last_win, profile->last_window); 1315ebaf1b31SBing Zhao MLX5_SET(conn_track_aso, desg, last_dir, profile->last_direction); 1316ebaf1b31SBing Zhao MLX5_SET(conn_track_aso, desg, last_index, profile->last_index); 1317ebaf1b31SBing Zhao MLX5_SET(conn_track_aso, desg, last_seq, profile->last_seq); 1318ebaf1b31SBing Zhao MLX5_SET(conn_track_aso, desg, last_ack, profile->last_ack); 1319ebaf1b31SBing Zhao MLX5_SET(conn_track_aso, desg, last_end, profile->last_end); 1320ebaf1b31SBing Zhao orig_dir = MLX5_ADDR_OF(conn_track_aso, desg, original_dir); 1321ebaf1b31SBing Zhao MLX5_SET(tcp_window_params, orig_dir, sent_end, 1322ebaf1b31SBing Zhao profile->original_dir.sent_end); 1323ebaf1b31SBing Zhao MLX5_SET(tcp_window_params, orig_dir, reply_end, 1324ebaf1b31SBing Zhao profile->original_dir.reply_end); 1325ebaf1b31SBing Zhao MLX5_SET(tcp_window_params, orig_dir, max_win, 1326ebaf1b31SBing Zhao profile->original_dir.max_win); 1327ebaf1b31SBing Zhao MLX5_SET(tcp_window_params, orig_dir, max_ack, 1328ebaf1b31SBing Zhao profile->original_dir.max_ack); 1329ebaf1b31SBing Zhao reply_dir = MLX5_ADDR_OF(conn_track_aso, desg, reply_dir); 1330ebaf1b31SBing Zhao MLX5_SET(tcp_window_params, reply_dir, sent_end, 1331ebaf1b31SBing Zhao profile->reply_dir.sent_end); 1332ebaf1b31SBing Zhao MLX5_SET(tcp_window_params, reply_dir, reply_end, 1333ebaf1b31SBing Zhao profile->reply_dir.reply_end); 1334ebaf1b31SBing Zhao MLX5_SET(tcp_window_params, reply_dir, max_win, 1335ebaf1b31SBing Zhao profile->reply_dir.max_win); 1336ebaf1b31SBing Zhao MLX5_SET(tcp_window_params, reply_dir, max_ack, 1337ebaf1b31SBing Zhao profile->reply_dir.max_ack); 1338ebaf1b31SBing Zhao sq->head++; 1339ebaf1b31SBing Zhao sq->pi += 2; /* Each WQE contains 2 WQEBB's. */ 1340478ba4bbSSuanming Mou if (push) { 13415dfa003dSMichael Baum mlx5_doorbell_ring(&sh->tx_uar.bf_db, *(volatile uint64_t *)wqe, 13425dfa003dSMichael Baum sq->pi, &sq->sq_obj.db_rec[MLX5_SND_DBR], 13435dfa003dSMichael Baum !sh->tx_uar.dbnc); 1344478ba4bbSSuanming Mou sq->db_pi = sq->pi; 1345478ba4bbSSuanming Mou } 1346478ba4bbSSuanming Mou sq->db = wqe; 1347463170a7SSuanming Mou if (need_lock) 1348ebaf1b31SBing Zhao rte_spinlock_unlock(&sq->sqsl); 1349ebaf1b31SBing Zhao return 1; 1350ebaf1b31SBing Zhao } 1351ebaf1b31SBing Zhao 1352ebaf1b31SBing Zhao /* 1353ebaf1b31SBing Zhao * Update the status field of CTs to indicate ready to be used by flows. 1354ebaf1b31SBing Zhao * A continuous number of CTs since last update. 1355ebaf1b31SBing Zhao * 1356ebaf1b31SBing Zhao * @param[in] sq 1357ebaf1b31SBing Zhao * Pointer to ASO CT SQ. 1358ebaf1b31SBing Zhao * @param[in] num 1359ebaf1b31SBing Zhao * Number of CT structures to be updated. 1360ebaf1b31SBing Zhao * 1361ebaf1b31SBing Zhao * @return 1362ebaf1b31SBing Zhao * 0 on success, a negative value. 1363ebaf1b31SBing Zhao */ 1364ebaf1b31SBing Zhao static void 1365ebaf1b31SBing Zhao mlx5_aso_ct_status_update(struct mlx5_aso_sq *sq, uint16_t num) 1366ebaf1b31SBing Zhao { 1367ebaf1b31SBing Zhao uint16_t size = 1 << sq->log_desc_n; 1368ebaf1b31SBing Zhao uint16_t mask = size - 1; 1369ebaf1b31SBing Zhao uint16_t i; 1370ebaf1b31SBing Zhao struct mlx5_aso_ct_action *ct = NULL; 1371ebaf1b31SBing Zhao uint16_t idx; 1372ebaf1b31SBing Zhao 1373ebaf1b31SBing Zhao for (i = 0; i < num; i++) { 1374ebaf1b31SBing Zhao idx = (uint16_t)((sq->tail + i) & mask); 1375ebaf1b31SBing Zhao ct = sq->elts[idx].ct; 1376ebaf1b31SBing Zhao MLX5_ASSERT(ct); 1377ebaf1b31SBing Zhao MLX5_ASO_CT_UPDATE_STATE(ct, ASO_CONNTRACK_READY); 1378cf756556SBing Zhao if (sq->elts[idx].query_data) 1379cf756556SBing Zhao rte_memcpy(sq->elts[idx].query_data, 1380cf756556SBing Zhao (char *)((uintptr_t)sq->mr.addr + idx * 64), 1381cf756556SBing Zhao 64); 1382ebaf1b31SBing Zhao } 1383ebaf1b31SBing Zhao } 1384ebaf1b31SBing Zhao 1385ebaf1b31SBing Zhao /* 1386cf756556SBing Zhao * Post a WQE to the ASO CT SQ to query the current context. 1387cf756556SBing Zhao * 13885dfa003dSMichael Baum * @param[in] sh 13895dfa003dSMichael Baum * Pointer to shared device context. 1390cf756556SBing Zhao * @param[in] ct 1391cf756556SBing Zhao * Pointer to the generic CT structure related to the context. 1392cf756556SBing Zhao * @param[in] data 1393cf756556SBing Zhao * Pointer to data area to be filled. 1394cf756556SBing Zhao * 1395cf756556SBing Zhao * @return 1396cf756556SBing Zhao * 1 on success (WQE number), 0 on failure. 1397cf756556SBing Zhao */ 1398cf756556SBing Zhao static int 13995dfa003dSMichael Baum mlx5_aso_ct_sq_query_single(struct mlx5_dev_ctx_shared *sh, 1400463170a7SSuanming Mou struct mlx5_aso_sq *sq, 1401463170a7SSuanming Mou struct mlx5_aso_ct_action *ct, char *data, 1402478ba4bbSSuanming Mou bool need_lock, 1403478ba4bbSSuanming Mou void *user_data, 1404478ba4bbSSuanming Mou bool push) 1405cf756556SBing Zhao { 1406cf756556SBing Zhao volatile struct mlx5_aso_wqe *wqe = NULL; 1407cf756556SBing Zhao uint16_t size = 1 << sq->log_desc_n; 1408cf756556SBing Zhao uint16_t mask = size - 1; 1409cf756556SBing Zhao uint16_t res; 1410cf756556SBing Zhao uint16_t wqe_idx; 1411cf756556SBing Zhao struct mlx5_aso_ct_pool *pool; 1412cf756556SBing Zhao enum mlx5_aso_ct_state state = 1413e12a0166STyler Retzlaff rte_atomic_load_explicit(&ct->state, rte_memory_order_relaxed); 1414cf756556SBing Zhao 1415cf756556SBing Zhao if (state == ASO_CONNTRACK_FREE) { 1416cf756556SBing Zhao DRV_LOG(ERR, "Fail: No context to query"); 1417cf756556SBing Zhao return -1; 1418cf756556SBing Zhao } else if (state == ASO_CONNTRACK_WAIT) { 1419cf756556SBing Zhao return 0; 1420cf756556SBing Zhao } 1421463170a7SSuanming Mou if (need_lock) 1422cf756556SBing Zhao rte_spinlock_lock(&sq->sqsl); 1423cf756556SBing Zhao res = size - (uint16_t)(sq->head - sq->tail); 1424cf756556SBing Zhao if (unlikely(!res)) { 1425463170a7SSuanming Mou if (need_lock) 1426cf756556SBing Zhao rte_spinlock_unlock(&sq->sqsl); 1427cf756556SBing Zhao DRV_LOG(ERR, "Fail: SQ is full and no free WQE to send"); 1428cf756556SBing Zhao return 0; 1429cf756556SBing Zhao } 1430478ba4bbSSuanming Mou MLX5_ASO_CT_UPDATE_STATE(ct, 1431478ba4bbSSuanming Mou user_data ? ASO_CONNTRACK_WAIT_ASYNC : ASO_CONNTRACK_QUERY); 1432cf756556SBing Zhao wqe = &sq->sq_obj.aso_wqes[sq->head & mask]; 1433cf756556SBing Zhao /* Confirm the location and address of the prefetch instruction. */ 1434cf756556SBing Zhao rte_prefetch0(&sq->sq_obj.aso_wqes[(sq->head + 1) & mask]); 1435cf756556SBing Zhao /* Fill next WQE. */ 1436cf756556SBing Zhao wqe_idx = sq->head & mask; 1437478ba4bbSSuanming Mou /* Check if this is async mode. */ 1438478ba4bbSSuanming Mou if (user_data) { 1439478ba4bbSSuanming Mou struct mlx5_hw_q_job *job = (struct mlx5_hw_q_job *)user_data; 1440478ba4bbSSuanming Mou 1441478ba4bbSSuanming Mou sq->elts[wqe_idx].ct = user_data; 1442d065ecc4SGregory Etelson job->query.hw = (char *)((uintptr_t)sq->mr.addr + wqe_idx * 64); 1443478ba4bbSSuanming Mou } else { 1444cf756556SBing Zhao sq->elts[wqe_idx].query_data = data; 1445478ba4bbSSuanming Mou sq->elts[wqe_idx].ct = ct; 1446478ba4bbSSuanming Mou } 1447463170a7SSuanming Mou pool = __mlx5_aso_ct_get_pool(sh, ct); 1448cf756556SBing Zhao /* Each WQE will have a single CT object. */ 1449cf756556SBing Zhao wqe->general_cseg.misc = rte_cpu_to_be_32(pool->devx_obj->id + 1450cf756556SBing Zhao ct->offset); 1451cf756556SBing Zhao wqe->general_cseg.opcode = rte_cpu_to_be_32(MLX5_OPCODE_ACCESS_ASO | 1452cf756556SBing Zhao (ASO_OPC_MOD_CONNECTION_TRACKING << 1453cf756556SBing Zhao WQE_CSEG_OPC_MOD_OFFSET) | 1454cf756556SBing Zhao sq->pi << WQE_CSEG_WQE_INDEX_OFFSET); 1455cf756556SBing Zhao /* 1456cf756556SBing Zhao * There is no write request is required. 1457cf756556SBing Zhao * ASO_OPER_LOGICAL_AND and ASO_OP_ALWAYS_FALSE are both 0. 1458cf756556SBing Zhao * "BYTEWISE_64BYTE" is needed for a whole context. 1459cf756556SBing Zhao * Set to 0 directly to reduce an endian swap. (Modify should rewrite.) 1460cf756556SBing Zhao * "data_mask" is ignored. 1461cf756556SBing Zhao * Buffer address was already filled during initialization. 1462cf756556SBing Zhao */ 1463cf756556SBing Zhao wqe->aso_cseg.operand_masks = rte_cpu_to_be_32(BYTEWISE_64BYTE << 1464cf756556SBing Zhao ASO_CSEG_DATA_MASK_MODE_OFFSET); 1465cf756556SBing Zhao wqe->aso_cseg.data_mask = 0; 1466cf756556SBing Zhao sq->head++; 1467cf756556SBing Zhao /* 1468cf756556SBing Zhao * Each WQE contains 2 WQEBB's, even though 1469cf756556SBing Zhao * data segment is not used in this case. 1470cf756556SBing Zhao */ 1471cf756556SBing Zhao sq->pi += 2; 1472478ba4bbSSuanming Mou if (push) { 14735dfa003dSMichael Baum mlx5_doorbell_ring(&sh->tx_uar.bf_db, *(volatile uint64_t *)wqe, 14745dfa003dSMichael Baum sq->pi, &sq->sq_obj.db_rec[MLX5_SND_DBR], 14755dfa003dSMichael Baum !sh->tx_uar.dbnc); 1476478ba4bbSSuanming Mou sq->db_pi = sq->pi; 1477478ba4bbSSuanming Mou } 1478478ba4bbSSuanming Mou sq->db = wqe; 1479463170a7SSuanming Mou if (need_lock) 1480cf756556SBing Zhao rte_spinlock_unlock(&sq->sqsl); 1481cf756556SBing Zhao return 1; 1482cf756556SBing Zhao } 1483cf756556SBing Zhao 1484cf756556SBing Zhao /* 1485ebaf1b31SBing Zhao * Handle completions from WQEs sent to ASO CT. 1486ebaf1b31SBing Zhao * 1487ebaf1b31SBing Zhao * @param[in] mng 1488ebaf1b31SBing Zhao * Pointer to the CT pools management structure. 1489ebaf1b31SBing Zhao */ 1490ebaf1b31SBing Zhao static void 1491463170a7SSuanming Mou mlx5_aso_ct_completion_handle(struct mlx5_dev_ctx_shared *sh __rte_unused, 1492463170a7SSuanming Mou struct mlx5_aso_sq *sq, 1493463170a7SSuanming Mou bool need_lock) 1494ebaf1b31SBing Zhao { 1495ebaf1b31SBing Zhao struct mlx5_aso_cq *cq = &sq->cq; 1496ebaf1b31SBing Zhao volatile struct mlx5_cqe *restrict cqe; 1497ebaf1b31SBing Zhao const uint32_t cq_size = 1 << cq->log_desc_n; 1498ebaf1b31SBing Zhao const uint32_t mask = cq_size - 1; 1499ebaf1b31SBing Zhao uint32_t idx; 1500ebaf1b31SBing Zhao uint32_t next_idx; 1501ebaf1b31SBing Zhao uint16_t max; 1502ebaf1b31SBing Zhao uint16_t n = 0; 1503ebaf1b31SBing Zhao int ret; 1504ebaf1b31SBing Zhao 1505463170a7SSuanming Mou if (need_lock) 1506ebaf1b31SBing Zhao rte_spinlock_lock(&sq->sqsl); 1507ebaf1b31SBing Zhao max = (uint16_t)(sq->head - sq->tail); 1508ebaf1b31SBing Zhao if (unlikely(!max)) { 1509463170a7SSuanming Mou if (need_lock) 1510ebaf1b31SBing Zhao rte_spinlock_unlock(&sq->sqsl); 1511ebaf1b31SBing Zhao return; 1512ebaf1b31SBing Zhao } 1513ebaf1b31SBing Zhao next_idx = cq->cq_ci & mask; 1514ebaf1b31SBing Zhao do { 1515ebaf1b31SBing Zhao idx = next_idx; 1516ebaf1b31SBing Zhao next_idx = (cq->cq_ci + 1) & mask; 1517ebaf1b31SBing Zhao /* Need to confirm the position of the prefetch. */ 1518ebaf1b31SBing Zhao rte_prefetch0(&cq->cq_obj.cqes[next_idx]); 1519ebaf1b31SBing Zhao cqe = &cq->cq_obj.cqes[idx]; 1520ebaf1b31SBing Zhao ret = check_cqe(cqe, cq_size, cq->cq_ci); 1521ebaf1b31SBing Zhao /* 1522ebaf1b31SBing Zhao * Be sure owner read is done before any other cookie field or 1523ebaf1b31SBing Zhao * opaque field. 1524ebaf1b31SBing Zhao */ 1525ebaf1b31SBing Zhao rte_io_rmb(); 1526ebaf1b31SBing Zhao if (unlikely(ret != MLX5_CQE_STATUS_SW_OWN)) { 1527ebaf1b31SBing Zhao if (likely(ret == MLX5_CQE_STATUS_HW_OWN)) 1528ebaf1b31SBing Zhao break; 1529ebaf1b31SBing Zhao mlx5_aso_cqe_err_handle(sq); 1530ebaf1b31SBing Zhao } else { 1531ebaf1b31SBing Zhao n++; 1532ebaf1b31SBing Zhao } 1533ebaf1b31SBing Zhao cq->cq_ci++; 1534ebaf1b31SBing Zhao } while (1); 1535ebaf1b31SBing Zhao if (likely(n)) { 1536ebaf1b31SBing Zhao mlx5_aso_ct_status_update(sq, n); 1537ebaf1b31SBing Zhao sq->tail += n; 1538ebaf1b31SBing Zhao rte_io_wmb(); 1539ebaf1b31SBing Zhao cq->cq_obj.db_rec[0] = rte_cpu_to_be_32(cq->cq_ci); 1540ebaf1b31SBing Zhao } 1541463170a7SSuanming Mou if (need_lock) 1542ebaf1b31SBing Zhao rte_spinlock_unlock(&sq->sqsl); 1543ebaf1b31SBing Zhao } 1544ebaf1b31SBing Zhao 1545ebaf1b31SBing Zhao /* 1546ebaf1b31SBing Zhao * Update connection tracking ASO context by sending WQE. 1547ebaf1b31SBing Zhao * 1548ebaf1b31SBing Zhao * @param[in] sh 1549ebaf1b31SBing Zhao * Pointer to mlx5_dev_ctx_shared object. 1550463170a7SSuanming Mou * @param[in] queue 1551463170a7SSuanming Mou * The queue index. 1552ebaf1b31SBing Zhao * @param[in] ct 1553ebaf1b31SBing Zhao * Pointer to connection tracking offload object. 1554ebaf1b31SBing Zhao * @param[in] profile 1555ebaf1b31SBing Zhao * Pointer to connection tracking TCP parameter. 1556ebaf1b31SBing Zhao * 1557ebaf1b31SBing Zhao * @return 1558ebaf1b31SBing Zhao * 0 on success, -1 on failure. 1559ebaf1b31SBing Zhao */ 1560ebaf1b31SBing Zhao int 1561ebaf1b31SBing Zhao mlx5_aso_ct_update_by_wqe(struct mlx5_dev_ctx_shared *sh, 1562463170a7SSuanming Mou uint32_t queue, 1563ebaf1b31SBing Zhao struct mlx5_aso_ct_action *ct, 1564478ba4bbSSuanming Mou const struct rte_flow_action_conntrack *profile, 1565478ba4bbSSuanming Mou void *user_data, 1566478ba4bbSSuanming Mou bool push) 1567ebaf1b31SBing Zhao { 1568ebaf1b31SBing Zhao uint32_t poll_wqe_times = MLX5_CT_POLL_WQE_CQE_TIMES; 1569463170a7SSuanming Mou struct mlx5_aso_ct_pool *pool = __mlx5_aso_ct_get_pool(sh, ct); 1570463170a7SSuanming Mou struct mlx5_aso_sq *sq; 1571463170a7SSuanming Mou bool need_lock = !!(queue == MLX5_HW_INV_QUEUE); 1572478ba4bbSSuanming Mou int ret; 1573ebaf1b31SBing Zhao 1574463170a7SSuanming Mou if (sh->config.dv_flow_en == 2) 1575463170a7SSuanming Mou sq = __mlx5_aso_ct_get_sq_in_hws(queue, pool); 1576463170a7SSuanming Mou else 1577463170a7SSuanming Mou sq = __mlx5_aso_ct_get_sq_in_sws(sh, ct); 1578478ba4bbSSuanming Mou if (queue != MLX5_HW_INV_QUEUE) { 1579478ba4bbSSuanming Mou ret = mlx5_aso_ct_sq_enqueue_single(sh, sq, ct, profile, 1580478ba4bbSSuanming Mou need_lock, user_data, push); 1581478ba4bbSSuanming Mou return ret > 0 ? 0 : -1; 1582478ba4bbSSuanming Mou } 1583ebaf1b31SBing Zhao do { 1584463170a7SSuanming Mou mlx5_aso_ct_completion_handle(sh, sq, need_lock); 1585478ba4bbSSuanming Mou if (mlx5_aso_ct_sq_enqueue_single(sh, sq, ct, profile, 1586478ba4bbSSuanming Mou need_lock, NULL, true)) 1587ebaf1b31SBing Zhao return 0; 1588ebaf1b31SBing Zhao /* Waiting for wqe resource. */ 1589ebaf1b31SBing Zhao rte_delay_us_sleep(10u); 1590ebaf1b31SBing Zhao } while (--poll_wqe_times); 1591ebaf1b31SBing Zhao DRV_LOG(ERR, "Fail to send WQE for ASO CT %d in pool %d", 1592ebaf1b31SBing Zhao ct->offset, pool->index); 1593ebaf1b31SBing Zhao return -1; 1594ebaf1b31SBing Zhao } 1595cf756556SBing Zhao 1596cf756556SBing Zhao /* 1597cf756556SBing Zhao * The routine is used to wait for WQE completion to continue with queried data. 1598cf756556SBing Zhao * 1599cf756556SBing Zhao * @param[in] sh 1600cf756556SBing Zhao * Pointer to mlx5_dev_ctx_shared object. 1601463170a7SSuanming Mou * @param[in] queue 1602463170a7SSuanming Mou * The queue which CT works on.. 1603cf756556SBing Zhao * @param[in] ct 1604cf756556SBing Zhao * Pointer to connection tracking offload object. 1605cf756556SBing Zhao * 1606cf756556SBing Zhao * @return 1607cf756556SBing Zhao * 0 on success, -1 on failure. 1608cf756556SBing Zhao */ 1609cf756556SBing Zhao int 1610463170a7SSuanming Mou mlx5_aso_ct_wait_ready(struct mlx5_dev_ctx_shared *sh, uint32_t queue, 1611cf756556SBing Zhao struct mlx5_aso_ct_action *ct) 1612cf756556SBing Zhao { 1613cf756556SBing Zhao uint32_t poll_cqe_times = MLX5_CT_POLL_WQE_CQE_TIMES; 1614463170a7SSuanming Mou struct mlx5_aso_ct_pool *pool = __mlx5_aso_ct_get_pool(sh, ct); 1615463170a7SSuanming Mou struct mlx5_aso_sq *sq; 1616463170a7SSuanming Mou bool need_lock = !!(queue == MLX5_HW_INV_QUEUE); 1617cf756556SBing Zhao 1618463170a7SSuanming Mou if (sh->config.dv_flow_en == 2) 1619463170a7SSuanming Mou sq = __mlx5_aso_ct_get_sq_in_hws(queue, pool); 1620463170a7SSuanming Mou else 1621463170a7SSuanming Mou sq = __mlx5_aso_ct_get_sq_in_sws(sh, ct); 1622e12a0166STyler Retzlaff if (rte_atomic_load_explicit(&ct->state, rte_memory_order_relaxed) == 1623cf756556SBing Zhao ASO_CONNTRACK_READY) 1624cf756556SBing Zhao return 0; 1625cf756556SBing Zhao do { 1626463170a7SSuanming Mou mlx5_aso_ct_completion_handle(sh, sq, need_lock); 1627e12a0166STyler Retzlaff if (rte_atomic_load_explicit(&ct->state, rte_memory_order_relaxed) == 1628cf756556SBing Zhao ASO_CONNTRACK_READY) 1629cf756556SBing Zhao return 0; 1630cf756556SBing Zhao /* Waiting for CQE ready, consider should block or sleep. */ 1631cf756556SBing Zhao rte_delay_us_sleep(MLX5_ASO_WQE_CQE_RESPONSE_DELAY); 1632cf756556SBing Zhao } while (--poll_cqe_times); 1633cf756556SBing Zhao DRV_LOG(ERR, "Fail to poll CQE for ASO CT %d in pool %d", 1634cf756556SBing Zhao ct->offset, pool->index); 1635cf756556SBing Zhao return -1; 1636cf756556SBing Zhao } 1637cf756556SBing Zhao 1638cf756556SBing Zhao /* 1639cf756556SBing Zhao * Convert the hardware conntrack data format into the profile. 1640cf756556SBing Zhao * 1641cf756556SBing Zhao * @param[in] profile 1642cf756556SBing Zhao * Pointer to conntrack profile to be filled after query. 1643cf756556SBing Zhao * @param[in] wdata 1644cf756556SBing Zhao * Pointer to data fetched from hardware. 1645cf756556SBing Zhao */ 1646478ba4bbSSuanming Mou void 1647cf756556SBing Zhao mlx5_aso_ct_obj_analyze(struct rte_flow_action_conntrack *profile, 1648cf756556SBing Zhao char *wdata) 1649cf756556SBing Zhao { 1650cf756556SBing Zhao void *o_dir = MLX5_ADDR_OF(conn_track_aso, wdata, original_dir); 1651cf756556SBing Zhao void *r_dir = MLX5_ADDR_OF(conn_track_aso, wdata, reply_dir); 1652cf756556SBing Zhao 1653cf756556SBing Zhao /* MLX5_GET16 should be taken into consideration. */ 1654cf756556SBing Zhao profile->state = (enum rte_flow_conntrack_state) 1655cf756556SBing Zhao MLX5_GET(conn_track_aso, wdata, state); 1656cf756556SBing Zhao profile->enable = !MLX5_GET(conn_track_aso, wdata, freeze_track); 1657cf756556SBing Zhao profile->selective_ack = MLX5_GET(conn_track_aso, wdata, 1658cf756556SBing Zhao sack_permitted); 1659cf756556SBing Zhao profile->live_connection = MLX5_GET(conn_track_aso, wdata, 1660cf756556SBing Zhao connection_assured); 1661cf756556SBing Zhao profile->challenge_ack_passed = MLX5_GET(conn_track_aso, wdata, 1662cf756556SBing Zhao challenged_acked); 1663cf756556SBing Zhao profile->max_ack_window = MLX5_GET(conn_track_aso, wdata, 1664cf756556SBing Zhao max_ack_window); 1665cf756556SBing Zhao profile->retransmission_limit = MLX5_GET(conn_track_aso, wdata, 1666cf756556SBing Zhao retranmission_limit); 1667cf756556SBing Zhao profile->last_window = MLX5_GET(conn_track_aso, wdata, last_win); 1668cf756556SBing Zhao profile->last_direction = MLX5_GET(conn_track_aso, wdata, last_dir); 1669cf756556SBing Zhao profile->last_index = (enum rte_flow_conntrack_tcp_last_index) 1670cf756556SBing Zhao MLX5_GET(conn_track_aso, wdata, last_index); 1671cf756556SBing Zhao profile->last_seq = MLX5_GET(conn_track_aso, wdata, last_seq); 1672cf756556SBing Zhao profile->last_ack = MLX5_GET(conn_track_aso, wdata, last_ack); 1673cf756556SBing Zhao profile->last_end = MLX5_GET(conn_track_aso, wdata, last_end); 1674cf756556SBing Zhao profile->liberal_mode = MLX5_GET(conn_track_aso, wdata, 1675cf756556SBing Zhao reply_direction_tcp_liberal_enabled) | 1676cf756556SBing Zhao MLX5_GET(conn_track_aso, wdata, 1677cf756556SBing Zhao original_direction_tcp_liberal_enabled); 1678cf756556SBing Zhao /* No liberal in the RTE structure profile. */ 1679cf756556SBing Zhao profile->reply_dir.scale = MLX5_GET(conn_track_aso, wdata, 1680cf756556SBing Zhao reply_direction_tcp_scale); 1681cf756556SBing Zhao profile->reply_dir.close_initiated = MLX5_GET(conn_track_aso, wdata, 1682cf756556SBing Zhao reply_direction_tcp_close_initiated); 1683cf756556SBing Zhao profile->reply_dir.data_unacked = MLX5_GET(conn_track_aso, wdata, 1684cf756556SBing Zhao reply_direction_tcp_data_unacked); 1685cf756556SBing Zhao profile->reply_dir.last_ack_seen = MLX5_GET(conn_track_aso, wdata, 1686cf756556SBing Zhao reply_direction_tcp_max_ack); 1687cf756556SBing Zhao profile->reply_dir.sent_end = MLX5_GET(tcp_window_params, 1688cf756556SBing Zhao r_dir, sent_end); 1689cf756556SBing Zhao profile->reply_dir.reply_end = MLX5_GET(tcp_window_params, 1690cf756556SBing Zhao r_dir, reply_end); 1691cf756556SBing Zhao profile->reply_dir.max_win = MLX5_GET(tcp_window_params, 1692cf756556SBing Zhao r_dir, max_win); 1693cf756556SBing Zhao profile->reply_dir.max_ack = MLX5_GET(tcp_window_params, 1694cf756556SBing Zhao r_dir, max_ack); 1695cf756556SBing Zhao profile->original_dir.scale = MLX5_GET(conn_track_aso, wdata, 1696cf756556SBing Zhao original_direction_tcp_scale); 1697cf756556SBing Zhao profile->original_dir.close_initiated = MLX5_GET(conn_track_aso, wdata, 1698cf756556SBing Zhao original_direction_tcp_close_initiated); 1699cf756556SBing Zhao profile->original_dir.data_unacked = MLX5_GET(conn_track_aso, wdata, 1700cf756556SBing Zhao original_direction_tcp_data_unacked); 1701cf756556SBing Zhao profile->original_dir.last_ack_seen = MLX5_GET(conn_track_aso, wdata, 1702cf756556SBing Zhao original_direction_tcp_max_ack); 1703cf756556SBing Zhao profile->original_dir.sent_end = MLX5_GET(tcp_window_params, 1704cf756556SBing Zhao o_dir, sent_end); 1705cf756556SBing Zhao profile->original_dir.reply_end = MLX5_GET(tcp_window_params, 1706cf756556SBing Zhao o_dir, reply_end); 1707cf756556SBing Zhao profile->original_dir.max_win = MLX5_GET(tcp_window_params, 1708cf756556SBing Zhao o_dir, max_win); 1709cf756556SBing Zhao profile->original_dir.max_ack = MLX5_GET(tcp_window_params, 1710cf756556SBing Zhao o_dir, max_ack); 1711cf756556SBing Zhao } 1712cf756556SBing Zhao 1713cf756556SBing Zhao /* 1714cf756556SBing Zhao * Query connection tracking information parameter by send WQE. 1715cf756556SBing Zhao * 1716cf756556SBing Zhao * @param[in] dev 1717cf756556SBing Zhao * Pointer to Ethernet device. 1718cf756556SBing Zhao * @param[in] ct 1719cf756556SBing Zhao * Pointer to connection tracking offload object. 1720cf756556SBing Zhao * @param[out] profile 1721cf756556SBing Zhao * Pointer to connection tracking TCP information. 1722cf756556SBing Zhao * 1723cf756556SBing Zhao * @return 1724cf756556SBing Zhao * 0 on success, -1 on failure. 1725cf756556SBing Zhao */ 1726cf756556SBing Zhao int 1727cf756556SBing Zhao mlx5_aso_ct_query_by_wqe(struct mlx5_dev_ctx_shared *sh, 1728463170a7SSuanming Mou uint32_t queue, 1729cf756556SBing Zhao struct mlx5_aso_ct_action *ct, 1730478ba4bbSSuanming Mou struct rte_flow_action_conntrack *profile, 1731478ba4bbSSuanming Mou void *user_data, bool push) 1732cf756556SBing Zhao { 1733cf756556SBing Zhao uint32_t poll_wqe_times = MLX5_CT_POLL_WQE_CQE_TIMES; 1734463170a7SSuanming Mou struct mlx5_aso_ct_pool *pool = __mlx5_aso_ct_get_pool(sh, ct); 1735463170a7SSuanming Mou struct mlx5_aso_sq *sq; 1736463170a7SSuanming Mou bool need_lock = !!(queue == MLX5_HW_INV_QUEUE); 1737cf756556SBing Zhao char out_data[64 * 2]; 1738cf756556SBing Zhao int ret; 1739cf756556SBing Zhao 1740463170a7SSuanming Mou if (sh->config.dv_flow_en == 2) 1741463170a7SSuanming Mou sq = __mlx5_aso_ct_get_sq_in_hws(queue, pool); 1742463170a7SSuanming Mou else 1743463170a7SSuanming Mou sq = __mlx5_aso_ct_get_sq_in_sws(sh, ct); 1744478ba4bbSSuanming Mou if (queue != MLX5_HW_INV_QUEUE) { 1745478ba4bbSSuanming Mou ret = mlx5_aso_ct_sq_query_single(sh, sq, ct, out_data, 1746478ba4bbSSuanming Mou need_lock, user_data, push); 1747478ba4bbSSuanming Mou return ret > 0 ? 0 : -1; 1748478ba4bbSSuanming Mou } 1749cf756556SBing Zhao do { 1750463170a7SSuanming Mou mlx5_aso_ct_completion_handle(sh, sq, need_lock); 1751478ba4bbSSuanming Mou ret = mlx5_aso_ct_sq_query_single(sh, sq, ct, out_data, 1752478ba4bbSSuanming Mou need_lock, NULL, true); 1753cf756556SBing Zhao if (ret < 0) 1754cf756556SBing Zhao return ret; 1755cf756556SBing Zhao else if (ret > 0) 1756cf756556SBing Zhao goto data_handle; 1757cf756556SBing Zhao /* Waiting for wqe resource or state. */ 1758cf756556SBing Zhao else 1759cf756556SBing Zhao rte_delay_us_sleep(10u); 1760cf756556SBing Zhao } while (--poll_wqe_times); 1761cf756556SBing Zhao DRV_LOG(ERR, "Fail to send WQE for ASO CT %d in pool %d", 1762cf756556SBing Zhao ct->offset, pool->index); 1763cf756556SBing Zhao return -1; 1764cf756556SBing Zhao data_handle: 1765463170a7SSuanming Mou ret = mlx5_aso_ct_wait_ready(sh, queue, ct); 1766cf756556SBing Zhao if (!ret) 1767cf756556SBing Zhao mlx5_aso_ct_obj_analyze(profile, out_data); 1768cf756556SBing Zhao return ret; 1769cf756556SBing Zhao } 17702d084f69SBing Zhao 17712d084f69SBing Zhao /* 17722d084f69SBing Zhao * Make sure the conntrack context is synchronized with hardware before 17732d084f69SBing Zhao * creating a flow rule that uses it. 17742d084f69SBing Zhao * 17752d084f69SBing Zhao * @param[in] sh 17762d084f69SBing Zhao * Pointer to shared device context. 17772d084f69SBing Zhao * @param[in] ct 17782d084f69SBing Zhao * Pointer to connection tracking offload object. 17792d084f69SBing Zhao * 17802d084f69SBing Zhao * @return 17812d084f69SBing Zhao * 0 on success, a negative errno value otherwise and rte_errno is set. 17822d084f69SBing Zhao */ 17832d084f69SBing Zhao int 17842d084f69SBing Zhao mlx5_aso_ct_available(struct mlx5_dev_ctx_shared *sh, 1785463170a7SSuanming Mou uint32_t queue, 17862d084f69SBing Zhao struct mlx5_aso_ct_action *ct) 17872d084f69SBing Zhao { 1788463170a7SSuanming Mou struct mlx5_aso_ct_pool *pool = __mlx5_aso_ct_get_pool(sh, ct); 1789463170a7SSuanming Mou struct mlx5_aso_sq *sq; 1790463170a7SSuanming Mou bool need_lock = !!(queue == MLX5_HW_INV_QUEUE); 17912d084f69SBing Zhao uint32_t poll_cqe_times = MLX5_CT_POLL_WQE_CQE_TIMES; 17922d084f69SBing Zhao enum mlx5_aso_ct_state state = 1793e12a0166STyler Retzlaff rte_atomic_load_explicit(&ct->state, rte_memory_order_relaxed); 17942d084f69SBing Zhao 1795463170a7SSuanming Mou if (sh->config.dv_flow_en == 2) 1796463170a7SSuanming Mou sq = __mlx5_aso_ct_get_sq_in_hws(queue, pool); 1797463170a7SSuanming Mou else 1798463170a7SSuanming Mou sq = __mlx5_aso_ct_get_sq_in_sws(sh, ct); 17992d084f69SBing Zhao if (state == ASO_CONNTRACK_FREE) { 18002d084f69SBing Zhao rte_errno = ENXIO; 18012d084f69SBing Zhao return -rte_errno; 18022d084f69SBing Zhao } else if (state == ASO_CONNTRACK_READY || 1803478ba4bbSSuanming Mou state == ASO_CONNTRACK_QUERY || 1804478ba4bbSSuanming Mou state == ASO_CONNTRACK_WAIT_ASYNC) { 18052d084f69SBing Zhao return 0; 18062d084f69SBing Zhao } 18072d084f69SBing Zhao do { 1808463170a7SSuanming Mou mlx5_aso_ct_completion_handle(sh, sq, need_lock); 1809e12a0166STyler Retzlaff state = rte_atomic_load_explicit(&ct->state, rte_memory_order_relaxed); 18102d084f69SBing Zhao if (state == ASO_CONNTRACK_READY || 18112d084f69SBing Zhao state == ASO_CONNTRACK_QUERY) 18122d084f69SBing Zhao return 0; 18132d084f69SBing Zhao /* Waiting for CQE ready, consider should block or sleep. */ 1814463170a7SSuanming Mou rte_delay_us_block(MLX5_ASO_WQE_CQE_RESPONSE_DELAY); 18152d084f69SBing Zhao } while (--poll_cqe_times); 18162d084f69SBing Zhao rte_errno = EBUSY; 18172d084f69SBing Zhao return -rte_errno; 18182d084f69SBing Zhao } 18194d368e1dSXiaoyu Min 18204d368e1dSXiaoyu Min int 18214d368e1dSXiaoyu Min mlx5_aso_cnt_queue_init(struct mlx5_dev_ctx_shared *sh) 18224d368e1dSXiaoyu Min { 18234d368e1dSXiaoyu Min struct mlx5_hws_aso_mng *aso_mng = NULL; 18244d368e1dSXiaoyu Min uint8_t idx; 18254d368e1dSXiaoyu Min struct mlx5_aso_sq *sq; 18264d368e1dSXiaoyu Min 18274d368e1dSXiaoyu Min MLX5_ASSERT(sh); 18284d368e1dSXiaoyu Min MLX5_ASSERT(sh->cnt_svc); 18294d368e1dSXiaoyu Min aso_mng = &sh->cnt_svc->aso_mng; 18304d368e1dSXiaoyu Min aso_mng->sq_num = HWS_CNT_ASO_SQ_NUM; 18314d368e1dSXiaoyu Min for (idx = 0; idx < HWS_CNT_ASO_SQ_NUM; idx++) { 18324d368e1dSXiaoyu Min sq = &aso_mng->sqs[idx]; 18334d368e1dSXiaoyu Min if (mlx5_aso_sq_create(sh->cdev, sq, sh->tx_uar.obj, 18344d368e1dSXiaoyu Min MLX5_ASO_CNT_QUEUE_LOG_DESC)) 18354d368e1dSXiaoyu Min goto error; 18364d368e1dSXiaoyu Min mlx5_aso_cnt_init_sq(sq); 18374d368e1dSXiaoyu Min } 18384d368e1dSXiaoyu Min return 0; 18394d368e1dSXiaoyu Min error: 18404d368e1dSXiaoyu Min mlx5_aso_cnt_queue_uninit(sh); 18414d368e1dSXiaoyu Min return -1; 18424d368e1dSXiaoyu Min } 18434d368e1dSXiaoyu Min 18444d368e1dSXiaoyu Min void 18454d368e1dSXiaoyu Min mlx5_aso_cnt_queue_uninit(struct mlx5_dev_ctx_shared *sh) 18464d368e1dSXiaoyu Min { 18474d368e1dSXiaoyu Min uint16_t idx; 18484d368e1dSXiaoyu Min 18494d368e1dSXiaoyu Min for (idx = 0; idx < sh->cnt_svc->aso_mng.sq_num; idx++) 18504d368e1dSXiaoyu Min mlx5_aso_destroy_sq(&sh->cnt_svc->aso_mng.sqs[idx]); 18514d368e1dSXiaoyu Min sh->cnt_svc->aso_mng.sq_num = 0; 18524d368e1dSXiaoyu Min } 18534d368e1dSXiaoyu Min 18544d368e1dSXiaoyu Min static uint16_t 18554d368e1dSXiaoyu Min mlx5_aso_cnt_sq_enqueue_burst(struct mlx5_hws_cnt_pool *cpool, 18564d368e1dSXiaoyu Min struct mlx5_dev_ctx_shared *sh, 18574d368e1dSXiaoyu Min struct mlx5_aso_sq *sq, uint32_t n, 18584d368e1dSXiaoyu Min uint32_t offset, uint32_t dcs_id_base) 18594d368e1dSXiaoyu Min { 18604d368e1dSXiaoyu Min volatile struct mlx5_aso_wqe *wqe; 18614d368e1dSXiaoyu Min uint16_t size = 1 << sq->log_desc_n; 18624d368e1dSXiaoyu Min uint16_t mask = size - 1; 18634d368e1dSXiaoyu Min uint16_t max; 18644d368e1dSXiaoyu Min uint32_t upper_offset = offset; 18654d368e1dSXiaoyu Min uint64_t addr; 18664d368e1dSXiaoyu Min uint32_t ctrl_gen_id = 0; 18674d368e1dSXiaoyu Min uint8_t opcmod = sh->cdev->config.hca_attr.flow_access_aso_opc_mod; 18684d368e1dSXiaoyu Min rte_be32_t lkey = rte_cpu_to_be_32(cpool->raw_mng->mr.lkey); 18694d368e1dSXiaoyu Min uint16_t aso_n = (uint16_t)(RTE_ALIGN_CEIL(n, 4) / 4); 18704d368e1dSXiaoyu Min uint32_t ccntid; 18714d368e1dSXiaoyu Min 18724d368e1dSXiaoyu Min max = RTE_MIN(size - (uint16_t)(sq->head - sq->tail), aso_n); 18734d368e1dSXiaoyu Min if (unlikely(!max)) 18744d368e1dSXiaoyu Min return 0; 18754d368e1dSXiaoyu Min upper_offset += (max * 4); 18764d368e1dSXiaoyu Min /* Because only one burst at one time, we can use the same elt. */ 18774d368e1dSXiaoyu Min sq->elts[0].burst_size = max; 18784d368e1dSXiaoyu Min ctrl_gen_id = dcs_id_base; 18794d368e1dSXiaoyu Min ctrl_gen_id /= 4; 18804d368e1dSXiaoyu Min do { 18814d368e1dSXiaoyu Min ccntid = upper_offset - max * 4; 18824d368e1dSXiaoyu Min wqe = &sq->sq_obj.aso_wqes[sq->head & mask]; 18834d368e1dSXiaoyu Min rte_prefetch0(&sq->sq_obj.aso_wqes[(sq->head + 1) & mask]); 18844d368e1dSXiaoyu Min wqe->general_cseg.misc = rte_cpu_to_be_32(ctrl_gen_id); 18854d368e1dSXiaoyu Min wqe->general_cseg.flags = RTE_BE32(MLX5_COMP_ONLY_FIRST_ERR << 18864d368e1dSXiaoyu Min MLX5_COMP_MODE_OFFSET); 18874d368e1dSXiaoyu Min wqe->general_cseg.opcode = rte_cpu_to_be_32 18884d368e1dSXiaoyu Min (MLX5_OPCODE_ACCESS_ASO | 18894d368e1dSXiaoyu Min (opcmod << 18904d368e1dSXiaoyu Min WQE_CSEG_OPC_MOD_OFFSET) | 18914d368e1dSXiaoyu Min (sq->pi << 18924d368e1dSXiaoyu Min WQE_CSEG_WQE_INDEX_OFFSET)); 18934d368e1dSXiaoyu Min addr = (uint64_t)RTE_PTR_ADD(cpool->raw_mng->raw, 18944d368e1dSXiaoyu Min ccntid * sizeof(struct flow_counter_stats)); 18954d368e1dSXiaoyu Min wqe->aso_cseg.va_h = rte_cpu_to_be_32((uint32_t)(addr >> 32)); 18964d368e1dSXiaoyu Min wqe->aso_cseg.va_l_r = rte_cpu_to_be_32((uint32_t)addr | 1u); 18974d368e1dSXiaoyu Min wqe->aso_cseg.lkey = lkey; 18984d368e1dSXiaoyu Min sq->pi += 2; /* Each WQE contains 2 WQEBB's. */ 18994d368e1dSXiaoyu Min sq->head++; 19004d368e1dSXiaoyu Min sq->next++; 19014d368e1dSXiaoyu Min ctrl_gen_id++; 19024d368e1dSXiaoyu Min max--; 19034d368e1dSXiaoyu Min } while (max); 19044d368e1dSXiaoyu Min wqe->general_cseg.flags = RTE_BE32(MLX5_COMP_ALWAYS << 19054d368e1dSXiaoyu Min MLX5_COMP_MODE_OFFSET); 19064d368e1dSXiaoyu Min mlx5_doorbell_ring(&sh->tx_uar.bf_db, *(volatile uint64_t *)wqe, 19074d368e1dSXiaoyu Min sq->pi, &sq->sq_obj.db_rec[MLX5_SND_DBR], 19084d368e1dSXiaoyu Min !sh->tx_uar.dbnc); 19094d368e1dSXiaoyu Min return sq->elts[0].burst_size; 19104d368e1dSXiaoyu Min } 19114d368e1dSXiaoyu Min 19124d368e1dSXiaoyu Min static uint16_t 19134d368e1dSXiaoyu Min mlx5_aso_cnt_completion_handle(struct mlx5_aso_sq *sq) 19144d368e1dSXiaoyu Min { 19154d368e1dSXiaoyu Min struct mlx5_aso_cq *cq = &sq->cq; 19164d368e1dSXiaoyu Min volatile struct mlx5_cqe *restrict cqe; 19174d368e1dSXiaoyu Min const unsigned int cq_size = 1 << cq->log_desc_n; 19184d368e1dSXiaoyu Min const unsigned int mask = cq_size - 1; 19194d368e1dSXiaoyu Min uint32_t idx; 19204d368e1dSXiaoyu Min uint32_t next_idx = cq->cq_ci & mask; 19214d368e1dSXiaoyu Min const uint16_t max = (uint16_t)(sq->head - sq->tail); 19224d368e1dSXiaoyu Min uint16_t i = 0; 19234d368e1dSXiaoyu Min int ret; 19244d368e1dSXiaoyu Min if (unlikely(!max)) 19254d368e1dSXiaoyu Min return 0; 19264d368e1dSXiaoyu Min idx = next_idx; 19274d368e1dSXiaoyu Min next_idx = (cq->cq_ci + 1) & mask; 19284d368e1dSXiaoyu Min rte_prefetch0(&cq->cq_obj.cqes[next_idx]); 19294d368e1dSXiaoyu Min cqe = &cq->cq_obj.cqes[idx]; 19304d368e1dSXiaoyu Min ret = check_cqe(cqe, cq_size, cq->cq_ci); 19314d368e1dSXiaoyu Min /* 19324d368e1dSXiaoyu Min * Be sure owner read is done before any other cookie field or 19334d368e1dSXiaoyu Min * opaque field. 19344d368e1dSXiaoyu Min */ 19354d368e1dSXiaoyu Min rte_io_rmb(); 19364d368e1dSXiaoyu Min if (unlikely(ret != MLX5_CQE_STATUS_SW_OWN)) { 19374d368e1dSXiaoyu Min if (likely(ret == MLX5_CQE_STATUS_HW_OWN)) 19384d368e1dSXiaoyu Min return 0; /* return immediately. */ 19394d368e1dSXiaoyu Min mlx5_aso_cqe_err_handle(sq); 19404d368e1dSXiaoyu Min } 19414d368e1dSXiaoyu Min i += sq->elts[0].burst_size; 19424d368e1dSXiaoyu Min sq->elts[0].burst_size = 0; 19434d368e1dSXiaoyu Min cq->cq_ci++; 19444d368e1dSXiaoyu Min if (likely(i)) { 19454d368e1dSXiaoyu Min sq->tail += i; 19464d368e1dSXiaoyu Min rte_io_wmb(); 19474d368e1dSXiaoyu Min cq->cq_obj.db_rec[0] = rte_cpu_to_be_32(cq->cq_ci); 19484d368e1dSXiaoyu Min } 19494d368e1dSXiaoyu Min return i; 19504d368e1dSXiaoyu Min } 19514d368e1dSXiaoyu Min 19524d368e1dSXiaoyu Min static uint16_t 19534d368e1dSXiaoyu Min mlx5_aso_cnt_query_one_dcs(struct mlx5_dev_ctx_shared *sh, 19544d368e1dSXiaoyu Min struct mlx5_hws_cnt_pool *cpool, 19554d368e1dSXiaoyu Min uint8_t dcs_idx, uint32_t num) 19564d368e1dSXiaoyu Min { 19574d368e1dSXiaoyu Min uint32_t dcs_id = cpool->dcs_mng.dcs[dcs_idx].obj->id; 19584d368e1dSXiaoyu Min uint64_t cnt_num = cpool->dcs_mng.dcs[dcs_idx].batch_sz; 19594d368e1dSXiaoyu Min uint64_t left; 19604d368e1dSXiaoyu Min uint32_t iidx = cpool->dcs_mng.dcs[dcs_idx].iidx; 19614d368e1dSXiaoyu Min uint32_t offset; 19624d368e1dSXiaoyu Min uint16_t mask; 19634d368e1dSXiaoyu Min uint16_t sq_idx; 19644d368e1dSXiaoyu Min uint64_t burst_sz = (uint64_t)(1 << MLX5_ASO_CNT_QUEUE_LOG_DESC) * 4 * 19654d368e1dSXiaoyu Min sh->cnt_svc->aso_mng.sq_num; 19664d368e1dSXiaoyu Min uint64_t qburst_sz = burst_sz / sh->cnt_svc->aso_mng.sq_num; 19674d368e1dSXiaoyu Min uint64_t n; 19684d368e1dSXiaoyu Min struct mlx5_aso_sq *sq; 19694d368e1dSXiaoyu Min 19704d368e1dSXiaoyu Min cnt_num = RTE_MIN(num, cnt_num); 19714d368e1dSXiaoyu Min left = cnt_num; 19724d368e1dSXiaoyu Min while (left) { 19734d368e1dSXiaoyu Min mask = 0; 19744d368e1dSXiaoyu Min for (sq_idx = 0; sq_idx < sh->cnt_svc->aso_mng.sq_num; 19754d368e1dSXiaoyu Min sq_idx++) { 19764d368e1dSXiaoyu Min if (left == 0) { 19774d368e1dSXiaoyu Min mask |= (1 << sq_idx); 19784d368e1dSXiaoyu Min continue; 19794d368e1dSXiaoyu Min } 19804d368e1dSXiaoyu Min n = RTE_MIN(left, qburst_sz); 19814d368e1dSXiaoyu Min offset = cnt_num - left; 19824d368e1dSXiaoyu Min offset += iidx; 19834d368e1dSXiaoyu Min mlx5_aso_cnt_sq_enqueue_burst(cpool, sh, 19844d368e1dSXiaoyu Min &sh->cnt_svc->aso_mng.sqs[sq_idx], n, 19854d368e1dSXiaoyu Min offset, dcs_id); 19864d368e1dSXiaoyu Min left -= n; 19874d368e1dSXiaoyu Min } 19884d368e1dSXiaoyu Min do { 19894d368e1dSXiaoyu Min for (sq_idx = 0; sq_idx < sh->cnt_svc->aso_mng.sq_num; 19904d368e1dSXiaoyu Min sq_idx++) { 19914d368e1dSXiaoyu Min sq = &sh->cnt_svc->aso_mng.sqs[sq_idx]; 19924d368e1dSXiaoyu Min if (mlx5_aso_cnt_completion_handle(sq)) 19934d368e1dSXiaoyu Min mask |= (1 << sq_idx); 19944d368e1dSXiaoyu Min } 19954d368e1dSXiaoyu Min } while (mask < ((1 << sh->cnt_svc->aso_mng.sq_num) - 1)); 19964d368e1dSXiaoyu Min } 19974d368e1dSXiaoyu Min return cnt_num; 19984d368e1dSXiaoyu Min } 19994d368e1dSXiaoyu Min 20004d368e1dSXiaoyu Min /* 20014d368e1dSXiaoyu Min * Query FW counter via ASO WQE. 20024d368e1dSXiaoyu Min * 20034d368e1dSXiaoyu Min * ASO query counter use _sync_ mode, means: 20044d368e1dSXiaoyu Min * 1. each SQ issue one burst with several WQEs 20054d368e1dSXiaoyu Min * 2. ask for CQE at last WQE 20064d368e1dSXiaoyu Min * 3. busy poll CQ of each SQ's 20074d368e1dSXiaoyu Min * 4. If all SQ's CQE are received then goto step 1, issue next burst 20084d368e1dSXiaoyu Min * 20094d368e1dSXiaoyu Min * @param[in] sh 20104d368e1dSXiaoyu Min * Pointer to shared device. 20114d368e1dSXiaoyu Min * @param[in] cpool 20124d368e1dSXiaoyu Min * Pointer to counter pool. 20134d368e1dSXiaoyu Min * 20144d368e1dSXiaoyu Min * @return 20154d368e1dSXiaoyu Min * 0 on success, -1 on failure. 20164d368e1dSXiaoyu Min */ 20174d368e1dSXiaoyu Min int 20184d368e1dSXiaoyu Min mlx5_aso_cnt_query(struct mlx5_dev_ctx_shared *sh, 20194d368e1dSXiaoyu Min struct mlx5_hws_cnt_pool *cpool) 20204d368e1dSXiaoyu Min { 20214d368e1dSXiaoyu Min uint32_t idx; 20224d368e1dSXiaoyu Min uint32_t num; 20234d368e1dSXiaoyu Min uint32_t cnt_num = mlx5_hws_cnt_pool_get_size(cpool) - 20244d368e1dSXiaoyu Min rte_ring_count(cpool->free_list); 20254d368e1dSXiaoyu Min 20264d368e1dSXiaoyu Min for (idx = 0; idx < cpool->dcs_mng.batch_total; idx++) { 20274d368e1dSXiaoyu Min num = RTE_MIN(cnt_num, cpool->dcs_mng.dcs[idx].batch_sz); 20284d368e1dSXiaoyu Min mlx5_aso_cnt_query_one_dcs(sh, cpool, idx, num); 20294d368e1dSXiaoyu Min cnt_num -= num; 20304d368e1dSXiaoyu Min if (cnt_num == 0) 20314d368e1dSXiaoyu Min break; 20324d368e1dSXiaoyu Min } 20334d368e1dSXiaoyu Min return 0; 20344d368e1dSXiaoyu Min } 2035