1 // SPDX-License-Identifier: BSD-3-Clause 2 /* 3 * Copyright 2018 Mellanox Technologies, Ltd 4 */ 5 #include <math.h> 6 7 #include <rte_tailq.h> 8 #include <rte_malloc.h> 9 #include <rte_mtr.h> 10 #include <rte_mtr_driver.h> 11 12 #include <mlx5_devx_cmds.h> 13 #include <mlx5_malloc.h> 14 15 #include "mlx5.h" 16 #include "mlx5_flow.h" 17 18 #ifdef HAVE_MLX5_HWS_SUPPORT 19 20 static void 21 mlx5_flow_meter_uninit_guest(struct rte_eth_dev *dev) 22 { 23 struct mlx5_priv *priv = dev->data->dev_private; 24 25 if (priv->hws_mpool) { 26 if (priv->hws_mpool->action) { 27 claim_zero(mlx5dr_action_destroy(priv->hws_mpool->action)); 28 priv->hws_mpool->action = NULL; 29 } 30 priv->hws_mpool->devx_obj = NULL; 31 priv->hws_mpool->idx_pool = NULL; 32 mlx5_free(priv->hws_mpool); 33 priv->hws_mpool = NULL; 34 } 35 } 36 37 void 38 mlx5_flow_meter_uninit(struct rte_eth_dev *dev) 39 { 40 struct mlx5_priv *priv = dev->data->dev_private; 41 42 if (priv->shared_host) { 43 mlx5_flow_meter_uninit_guest(dev); 44 return; 45 } 46 if (priv->mtr_policy_arr) { 47 mlx5_free(priv->mtr_policy_arr); 48 priv->mtr_policy_arr = NULL; 49 } 50 if (priv->mtr_profile_arr) { 51 mlx5_free(priv->mtr_profile_arr); 52 priv->mtr_profile_arr = NULL; 53 } 54 if (priv->hws_mpool) { 55 mlx5_aso_mtr_queue_uninit(priv->sh, priv->hws_mpool, NULL); 56 mlx5_ipool_destroy(priv->hws_mpool->idx_pool); 57 mlx5_free(priv->hws_mpool); 58 priv->hws_mpool = NULL; 59 } 60 if (priv->mtr_bulk.aso) { 61 mlx5_free(priv->mtr_bulk.aso); 62 priv->mtr_bulk.aso = NULL; 63 priv->mtr_bulk.size = 0; 64 mlx5_aso_queue_uninit(priv->sh, ASO_OPC_MOD_POLICER); 65 } 66 if (priv->mtr_bulk.action) { 67 mlx5dr_action_destroy(priv->mtr_bulk.action); 68 priv->mtr_bulk.action = NULL; 69 } 70 if (priv->mtr_bulk.devx_obj) { 71 claim_zero(mlx5_devx_cmd_destroy(priv->mtr_bulk.devx_obj)); 72 priv->mtr_bulk.devx_obj = NULL; 73 } 74 } 75 76 static int 77 mlx5_flow_meter_init_guest(struct rte_eth_dev *dev) 78 { 79 struct mlx5_priv *priv = dev->data->dev_private; 80 struct rte_eth_dev *host_dev = priv->shared_host; 81 struct mlx5_priv *host_priv = host_dev->data->dev_private; 82 int reg_id = 0; 83 uint32_t flags; 84 int ret = 0; 85 86 MLX5_ASSERT(priv->shared_host); 87 reg_id = mlx5_flow_get_reg_id(dev, MLX5_MTR_COLOR, 0, NULL); 88 if (reg_id < 0) { 89 rte_errno = ENOMEM; 90 ret = -rte_errno; 91 DRV_LOG(ERR, "Meter register is not available."); 92 goto err; 93 } 94 priv->hws_mpool = mlx5_malloc(MLX5_MEM_ZERO, sizeof(struct mlx5_aso_mtr_pool), 95 RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY); 96 if (!priv->hws_mpool) { 97 rte_errno = ENOMEM; 98 ret = -rte_errno; 99 DRV_LOG(ERR, "Meter ipool allocation failed."); 100 goto err; 101 } 102 MLX5_ASSERT(host_priv->hws_mpool->idx_pool); 103 MLX5_ASSERT(host_priv->hws_mpool->devx_obj); 104 priv->hws_mpool->idx_pool = host_priv->hws_mpool->idx_pool; 105 priv->hws_mpool->devx_obj = host_priv->hws_mpool->devx_obj; 106 flags = MLX5DR_ACTION_FLAG_HWS_RX | MLX5DR_ACTION_FLAG_HWS_TX; 107 if (priv->sh->config.dv_esw_en && priv->master) 108 flags |= MLX5DR_ACTION_FLAG_HWS_FDB; 109 priv->hws_mpool->action = mlx5dr_action_create_aso_meter 110 (priv->dr_ctx, (struct mlx5dr_devx_obj *)priv->hws_mpool->devx_obj, 111 reg_id - REG_C_0, flags); 112 if (!priv->hws_mpool->action) { 113 rte_errno = ENOMEM; 114 ret = -rte_errno; 115 DRV_LOG(ERR, "Meter action creation failed."); 116 goto err; 117 } 118 return 0; 119 err: 120 mlx5_flow_meter_uninit(dev); 121 return ret; 122 } 123 124 int 125 mlx5_flow_meter_init(struct rte_eth_dev *dev, 126 uint32_t nb_meters, 127 uint32_t nb_meter_profiles, 128 uint32_t nb_meter_policies, 129 uint32_t nb_queues) 130 { 131 struct mlx5_priv *priv = dev->data->dev_private; 132 struct mlx5_devx_obj *dcs = NULL; 133 uint32_t log_obj_size; 134 int ret = 0; 135 int reg_id; 136 struct mlx5_aso_mtr *aso; 137 uint32_t i; 138 struct rte_flow_error error; 139 uint32_t flags; 140 uint32_t nb_mtrs = rte_align32pow2(nb_meters); 141 struct mlx5_indexed_pool_config cfg = { 142 .size = sizeof(struct mlx5_aso_mtr), 143 .trunk_size = 1 << 12, 144 .per_core_cache = 1 << 13, 145 .need_lock = 1, 146 .release_mem_en = !!priv->sh->config.reclaim_mode, 147 .malloc = mlx5_malloc, 148 .max_idx = nb_meters, 149 .free = mlx5_free, 150 .type = "mlx5_hw_mtr_mark_action", 151 }; 152 153 if (priv->shared_host) 154 return mlx5_flow_meter_init_guest(dev); 155 if (!nb_meters) { 156 ret = ENOTSUP; 157 rte_flow_error_set(&error, ENOMEM, 158 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 159 NULL, "Meter configuration is invalid."); 160 goto err; 161 } 162 if (!priv->mtr_en || !priv->sh->meter_aso_en) { 163 ret = ENOTSUP; 164 rte_flow_error_set(&error, ENOMEM, 165 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 166 NULL, "Meter ASO is not supported."); 167 goto err; 168 } 169 priv->mtr_config.nb_meters = nb_meters; 170 log_obj_size = rte_log2_u32(nb_meters >> 1); 171 dcs = mlx5_devx_cmd_create_flow_meter_aso_obj 172 (priv->sh->cdev->ctx, priv->sh->cdev->pdn, 173 log_obj_size); 174 if (!dcs) { 175 ret = ENOMEM; 176 rte_flow_error_set(&error, ENOMEM, 177 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 178 NULL, "Meter ASO object allocation failed."); 179 goto err; 180 } 181 priv->mtr_bulk.devx_obj = dcs; 182 reg_id = mlx5_flow_get_reg_id(dev, MLX5_MTR_COLOR, 0, NULL); 183 if (reg_id < 0) { 184 ret = ENOTSUP; 185 rte_flow_error_set(&error, ENOMEM, 186 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 187 NULL, "Meter register is not available."); 188 goto err; 189 } 190 flags = MLX5DR_ACTION_FLAG_HWS_RX | MLX5DR_ACTION_FLAG_HWS_TX; 191 if (priv->sh->config.dv_esw_en && priv->master) 192 flags |= MLX5DR_ACTION_FLAG_HWS_FDB; 193 priv->mtr_bulk.action = mlx5dr_action_create_aso_meter 194 (priv->dr_ctx, (struct mlx5dr_devx_obj *)dcs, 195 reg_id - REG_C_0, flags); 196 if (!priv->mtr_bulk.action) { 197 ret = ENOMEM; 198 rte_flow_error_set(&error, ENOMEM, 199 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 200 NULL, "Meter action creation failed."); 201 goto err; 202 } 203 priv->mtr_bulk.aso = mlx5_malloc(MLX5_MEM_ZERO, 204 sizeof(struct mlx5_aso_mtr) * 205 nb_meters, 206 RTE_CACHE_LINE_SIZE, 207 SOCKET_ID_ANY); 208 if (!priv->mtr_bulk.aso) { 209 ret = ENOMEM; 210 rte_flow_error_set(&error, ENOMEM, 211 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 212 NULL, "Meter bulk ASO allocation failed."); 213 goto err; 214 } 215 priv->mtr_bulk.size = nb_meters; 216 aso = priv->mtr_bulk.aso; 217 for (i = 0; i < priv->mtr_bulk.size; i++) { 218 aso->type = ASO_METER_DIRECT; 219 aso->state = ASO_METER_WAIT; 220 aso->offset = i; 221 aso++; 222 } 223 priv->hws_mpool = mlx5_malloc(MLX5_MEM_ZERO, 224 sizeof(struct mlx5_aso_mtr_pool), 225 RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY); 226 if (!priv->hws_mpool) { 227 ret = ENOMEM; 228 rte_flow_error_set(&error, ENOMEM, 229 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 230 NULL, "Meter ipool allocation failed."); 231 goto err; 232 } 233 priv->hws_mpool->devx_obj = priv->mtr_bulk.devx_obj; 234 priv->hws_mpool->action = priv->mtr_bulk.action; 235 priv->hws_mpool->nb_sq = nb_queues; 236 if (mlx5_aso_mtr_queue_init(priv->sh, priv->hws_mpool, 237 &priv->sh->mtrmng->pools_mng, nb_queues)) { 238 ret = ENOMEM; 239 rte_flow_error_set(&error, ENOMEM, 240 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 241 NULL, "Meter ASO queue allocation failed."); 242 goto err; 243 } 244 /* 245 * No need for local cache if Meter number is a small number. 246 * Since flow insertion rate will be very limited in that case. 247 * Here let's set the number to less than default trunk size 4K. 248 */ 249 if (nb_mtrs <= cfg.trunk_size) { 250 cfg.per_core_cache = 0; 251 cfg.trunk_size = nb_mtrs; 252 } else if (nb_mtrs <= MLX5_HW_IPOOL_SIZE_THRESHOLD) { 253 cfg.per_core_cache = MLX5_HW_IPOOL_CACHE_MIN; 254 } 255 priv->hws_mpool->idx_pool = mlx5_ipool_create(&cfg); 256 if (nb_meter_profiles) { 257 priv->mtr_config.nb_meter_profiles = nb_meter_profiles; 258 priv->mtr_profile_arr = 259 mlx5_malloc(MLX5_MEM_ZERO, 260 sizeof(struct mlx5_flow_meter_profile) * 261 nb_meter_profiles, 262 RTE_CACHE_LINE_SIZE, 263 SOCKET_ID_ANY); 264 if (!priv->mtr_profile_arr) { 265 ret = ENOMEM; 266 rte_flow_error_set(&error, ENOMEM, 267 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 268 NULL, "Meter profile allocation failed."); 269 goto err; 270 } 271 } 272 if (nb_meter_policies) { 273 priv->mtr_config.nb_meter_policies = nb_meter_policies; 274 priv->mtr_policy_arr = 275 mlx5_malloc(MLX5_MEM_ZERO, 276 sizeof(struct mlx5_flow_meter_policy) * 277 nb_meter_policies, 278 RTE_CACHE_LINE_SIZE, 279 SOCKET_ID_ANY); 280 if (!priv->mtr_policy_arr) { 281 ret = ENOMEM; 282 rte_flow_error_set(&error, ENOMEM, 283 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 284 NULL, "Meter policy allocation failed."); 285 goto err; 286 } 287 } 288 return 0; 289 err: 290 mlx5_flow_meter_uninit(dev); 291 return ret; 292 } 293 294 #endif /* HAVE_MLX5_HWS_SUPPORT */ 295 296 static int mlx5_flow_meter_disable(struct rte_eth_dev *dev, 297 uint32_t meter_id, struct rte_mtr_error *error); 298 299 /** 300 * Create the meter action. 301 * 302 * @param priv 303 * Pointer to mlx5_priv. 304 * @param[in] fm 305 * Pointer to flow meter to be converted. 306 * 307 * @return 308 * Pointer to the meter action on success, NULL otherwise. 309 */ 310 static void * 311 mlx5_flow_meter_action_create(struct mlx5_priv *priv, 312 struct mlx5_flow_meter_info *fm) 313 { 314 #ifdef HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER 315 struct mlx5dv_dr_flow_meter_attr mtr_init; 316 uint32_t fmp[MLX5_ST_SZ_DW(flow_meter_parameters)]; 317 struct mlx5_flow_meter_srtcm_rfc2697_prm *srtcm = 318 &fm->profile->srtcm_prm; 319 uint32_t cbs_cir = rte_be_to_cpu_32(srtcm->cbs_cir); 320 uint32_t ebs_eir = rte_be_to_cpu_32(srtcm->ebs_eir); 321 uint32_t val; 322 enum mlx5_meter_domain domain = 323 fm->transfer ? MLX5_MTR_DOMAIN_TRANSFER : 324 fm->egress ? MLX5_MTR_DOMAIN_EGRESS : 325 MLX5_MTR_DOMAIN_INGRESS; 326 struct mlx5_flow_meter_def_policy *def_policy = 327 priv->sh->mtrmng->def_policy[domain]; 328 329 memset(fmp, 0, MLX5_ST_SZ_BYTES(flow_meter_parameters)); 330 MLX5_SET(flow_meter_parameters, fmp, valid, 1); 331 MLX5_SET(flow_meter_parameters, fmp, bucket_overflow, 1); 332 MLX5_SET(flow_meter_parameters, fmp, 333 start_color, MLX5_FLOW_COLOR_GREEN); 334 MLX5_SET(flow_meter_parameters, fmp, both_buckets_on_green, 0); 335 val = (cbs_cir >> ASO_DSEG_CBS_EXP_OFFSET) & ASO_DSEG_EXP_MASK; 336 MLX5_SET(flow_meter_parameters, fmp, cbs_exponent, val); 337 val = (cbs_cir >> ASO_DSEG_CBS_MAN_OFFSET) & ASO_DSEG_MAN_MASK; 338 MLX5_SET(flow_meter_parameters, fmp, cbs_mantissa, val); 339 val = (cbs_cir >> ASO_DSEG_XIR_EXP_OFFSET) & ASO_DSEG_EXP_MASK; 340 MLX5_SET(flow_meter_parameters, fmp, cir_exponent, val); 341 val = (cbs_cir & ASO_DSEG_MAN_MASK); 342 MLX5_SET(flow_meter_parameters, fmp, cir_mantissa, val); 343 val = (ebs_eir >> ASO_DSEG_EBS_EXP_OFFSET) & ASO_DSEG_EXP_MASK; 344 MLX5_SET(flow_meter_parameters, fmp, ebs_exponent, val); 345 val = (ebs_eir >> ASO_DSEG_EBS_MAN_OFFSET) & ASO_DSEG_MAN_MASK; 346 MLX5_SET(flow_meter_parameters, fmp, ebs_mantissa, val); 347 mtr_init.next_table = def_policy->sub_policy.tbl_rsc->obj; 348 mtr_init.reg_c_index = priv->sh->registers.aso_reg - REG_C_0; 349 mtr_init.flow_meter_parameter = fmp; 350 mtr_init.flow_meter_parameter_sz = 351 MLX5_ST_SZ_BYTES(flow_meter_parameters); 352 mtr_init.active = fm->active_state; 353 return mlx5_glue->dv_create_flow_action_meter(&mtr_init); 354 #else 355 (void)priv; 356 (void)fm; 357 return NULL; 358 #endif 359 } 360 361 /** 362 * Find meter profile by id. 363 * 364 * @param priv 365 * Pointer to mlx5_priv. 366 * @param meter_profile_id 367 * Meter profile id. 368 * 369 * @return 370 * Pointer to the profile found on success, NULL otherwise. 371 */ 372 static struct mlx5_flow_meter_profile * 373 mlx5_flow_meter_profile_find(struct mlx5_priv *priv, uint32_t meter_profile_id) 374 { 375 struct mlx5_flow_meter_profile *fmp; 376 union mlx5_l3t_data data; 377 int32_t ret; 378 379 if (priv->mtr_profile_arr) 380 return &priv->mtr_profile_arr[meter_profile_id]; 381 if (!priv->mtr_profile_tbl || 382 mlx5_l3t_get_entry(priv->mtr_profile_tbl, meter_profile_id, &data) || !data.ptr) 383 return NULL; 384 fmp = data.ptr; 385 /* Remove reference taken by the mlx5_l3t_get_entry. */ 386 ret = mlx5_l3t_clear_entry(priv->mtr_profile_tbl, 387 meter_profile_id); 388 if (!ret || ret == -1) 389 return NULL; 390 return fmp; 391 } 392 393 /** 394 * Validate the MTR profile. 395 * 396 * @param[in] dev 397 * Pointer to Ethernet device. 398 * @param[in] meter_profile_id 399 * Meter profile id. 400 * @param[in] profile 401 * Pointer to meter profile detail. 402 * @param[out] error 403 * Pointer to the error structure. 404 * 405 * @return 406 * 0 on success, a negative errno value otherwise and rte_errno is set. 407 */ 408 static int 409 mlx5_flow_meter_profile_validate(struct rte_eth_dev *dev, 410 uint32_t meter_profile_id, 411 struct rte_mtr_meter_profile *profile, 412 struct rte_mtr_error *error) 413 { 414 struct mlx5_priv *priv = dev->data->dev_private; 415 struct mlx5_flow_meter_profile *fmp; 416 uint32_t ls_factor; 417 int ret; 418 uint64_t cir, cbs; 419 uint64_t eir, ebs; 420 uint64_t pir, pbs; 421 422 /* Profile must not be NULL. */ 423 if (profile == NULL) 424 return -rte_mtr_error_set(error, EINVAL, 425 RTE_MTR_ERROR_TYPE_METER_PROFILE, 426 NULL, "Meter profile is null."); 427 /* Meter profile ID must be valid. */ 428 if (priv->mtr_profile_arr) { 429 if (meter_profile_id >= priv->mtr_config.nb_meter_profiles) 430 return -rte_mtr_error_set(error, EINVAL, 431 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, 432 NULL, "Meter profile id not valid."); 433 fmp = mlx5_flow_meter_profile_find(priv, meter_profile_id); 434 /* Meter profile must not exist. */ 435 if (fmp->initialized) 436 return -rte_mtr_error_set(error, EEXIST, 437 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, 438 NULL, "Meter profile already exists."); 439 } else { 440 if (meter_profile_id == UINT32_MAX) 441 return -rte_mtr_error_set(error, EINVAL, 442 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, 443 NULL, "Meter profile id not valid."); 444 fmp = mlx5_flow_meter_profile_find(priv, meter_profile_id); 445 /* Meter profile must not exist. */ 446 if (fmp) 447 return -rte_mtr_error_set(error, EEXIST, 448 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, 449 NULL, "Meter profile already exists."); 450 } 451 if (!priv->sh->meter_aso_en) { 452 /* Old version is even not supported. */ 453 if (!priv->sh->cdev->config.hca_attr.qos.flow_meter_old) 454 return -rte_mtr_error_set(error, ENOTSUP, 455 RTE_MTR_ERROR_TYPE_METER_PROFILE, 456 NULL, "Metering is not supported."); 457 /* Old FW metering only supports srTCM. */ 458 if (profile->alg != RTE_MTR_SRTCM_RFC2697) { 459 return -rte_mtr_error_set(error, ENOTSUP, 460 RTE_MTR_ERROR_TYPE_METER_PROFILE, 461 NULL, "Metering algorithm is not supported."); 462 } else if (profile->srtcm_rfc2697.ebs) { 463 /* EBS is not supported for old metering. */ 464 return -rte_mtr_error_set(error, ENOTSUP, 465 RTE_MTR_ERROR_TYPE_METER_PROFILE, 466 NULL, "EBS is not supported."); 467 } 468 if (profile->packet_mode) 469 return -rte_mtr_error_set(error, ENOTSUP, 470 RTE_MTR_ERROR_TYPE_METER_PROFILE, NULL, 471 "Metering algorithm packet mode is not supported."); 472 } 473 ls_factor = profile->packet_mode ? MLX5_MTRS_PPS_MAP_BPS_SHIFT : 0; 474 switch (profile->alg) { 475 case RTE_MTR_SRTCM_RFC2697: 476 cir = profile->srtcm_rfc2697.cir << ls_factor; 477 cbs = profile->srtcm_rfc2697.cbs << ls_factor; 478 ebs = profile->srtcm_rfc2697.ebs << ls_factor; 479 /* EBS could be zero for old metering. */ 480 if (cir > 0 && cir <= MLX5_SRTCM_XIR_MAX && 481 cbs > 0 && cbs <= MLX5_SRTCM_XBS_MAX && 482 ebs <= MLX5_SRTCM_XBS_MAX) { 483 ret = 0; 484 } else { 485 ret = -rte_mtr_error_set(error, ENOTSUP, 486 RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL, 487 "Profile values out of range."); 488 } 489 break; 490 case RTE_MTR_TRTCM_RFC2698: 491 cir = profile->trtcm_rfc2698.cir << ls_factor; 492 cbs = profile->trtcm_rfc2698.cbs << ls_factor; 493 pir = profile->trtcm_rfc2698.pir << ls_factor; 494 pbs = profile->trtcm_rfc2698.pbs << ls_factor; 495 if (cir > 0 && cir <= MLX5_SRTCM_XIR_MAX && 496 cbs > 0 && cbs <= MLX5_SRTCM_XBS_MAX && 497 pir >= cir && pir <= (MLX5_SRTCM_XIR_MAX * 2) && 498 pbs >= cbs && pbs <= (MLX5_SRTCM_XBS_MAX * 2)) { 499 ret = 0; 500 } else { 501 ret = -rte_mtr_error_set(error, ENOTSUP, 502 RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL, 503 "Profile values out of range."); 504 } 505 break; 506 case RTE_MTR_TRTCM_RFC4115: 507 cir = profile->trtcm_rfc4115.cir << ls_factor; 508 cbs = profile->trtcm_rfc4115.cbs << ls_factor; 509 eir = profile->trtcm_rfc4115.eir << ls_factor; 510 ebs = profile->trtcm_rfc4115.ebs << ls_factor; 511 if (cir > 0 && cir <= MLX5_SRTCM_XIR_MAX && 512 cbs > 0 && cbs <= MLX5_SRTCM_XBS_MAX && 513 eir <= MLX5_SRTCM_XIR_MAX && ebs <= MLX5_SRTCM_XBS_MAX) { 514 ret = 0; 515 } else { 516 ret = -rte_mtr_error_set(error, ENOTSUP, 517 RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL, 518 "Profile values out of range."); 519 } 520 break; 521 default: 522 ret = -rte_mtr_error_set(error, ENOTSUP, 523 RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL, 524 "Unknown metering algorithm."); 525 break; 526 } 527 return ret; 528 } 529 530 /* 531 * Calculate mantissa and exponent for cir / eir. 532 * 533 * @param[in] xir 534 * Value to be calculated. 535 * @param[out] man 536 * Pointer to the mantissa. 537 * @param[out] exp 538 * Pointer to the exp. 539 */ 540 static inline void 541 mlx5_flow_meter_xir_man_exp_calc(int64_t xir, uint8_t *man, uint8_t *exp) 542 { 543 int64_t _xir; 544 int64_t delta = INT64_MAX; 545 uint8_t _man = 0; 546 uint8_t _exp = 0; 547 uint64_t m, e; 548 549 /* Special case xir == 0 ? both exp and mantissa are 0. */ 550 if (xir == 0) { 551 *man = 0; 552 *exp = 0; 553 return; 554 } 555 for (m = 0; m <= 0xFF; m++) { /* man width 8 bit */ 556 for (e = 0; e <= 0x1F; e++) { /* exp width 5bit */ 557 _xir = (1000000000ULL * m) >> e; 558 if (llabs(xir - _xir) <= delta) { 559 delta = llabs(xir - _xir); 560 _man = m; 561 _exp = e; 562 } 563 } 564 } 565 *man = _man; 566 *exp = _exp; 567 } 568 569 /* 570 * Calculate mantissa and exponent for xbs. 571 * 572 * @param[in] xbs 573 * Value to be calculated. 574 * @param[out] man 575 * Pointer to the mantissa. 576 * @param[out] exp 577 * Pointer to the exp. 578 */ 579 static inline void 580 mlx5_flow_meter_xbs_man_exp_calc(uint64_t xbs, uint8_t *man, uint8_t *exp) 581 { 582 int _exp; 583 double _man; 584 585 /* Special case xbs == 0 ? both exp and mantissa are 0. */ 586 if (xbs == 0) { 587 *man = 0; 588 *exp = 0; 589 return; 590 } 591 /* xbs = xbs_mantissa * 2^xbs_exponent */ 592 _man = frexp(xbs, &_exp); 593 if (_exp >= MLX5_MAN_WIDTH) { 594 _man = _man * pow(2, MLX5_MAN_WIDTH); 595 _exp = _exp - MLX5_MAN_WIDTH; 596 } 597 *man = (uint8_t)ceil(_man); 598 *exp = _exp; 599 } 600 601 /** 602 * Fill the prm meter parameter. 603 * 604 * @param[in,out] fmp 605 * Pointer to meter profile to be converted. 606 * @param[out] error 607 * Pointer to the error structure. 608 * 609 * @return 610 * 0 on success, a negative errno value otherwise and rte_errno is set. 611 */ 612 static int 613 mlx5_flow_meter_param_fill(struct mlx5_flow_meter_profile *fmp, 614 struct rte_mtr_error *error) 615 { 616 struct mlx5_flow_meter_srtcm_rfc2697_prm *srtcm = &fmp->srtcm_prm; 617 uint8_t man, exp; 618 uint32_t cbs_exp, cbs_man, cir_exp, cir_man; 619 uint32_t eir_exp, eir_man, ebs_exp, ebs_man; 620 uint64_t cir, cbs, eir, ebs; 621 622 switch (fmp->profile.alg) { 623 case RTE_MTR_SRTCM_RFC2697: 624 cir = fmp->profile.srtcm_rfc2697.cir; 625 cbs = fmp->profile.srtcm_rfc2697.cbs; 626 eir = 0; 627 ebs = fmp->profile.srtcm_rfc2697.ebs; 628 break; 629 case RTE_MTR_TRTCM_RFC2698: 630 MLX5_ASSERT(fmp->profile.trtcm_rfc2698.pir > 631 fmp->profile.trtcm_rfc2698.cir && 632 fmp->profile.trtcm_rfc2698.pbs > 633 fmp->profile.trtcm_rfc2698.cbs); 634 cir = fmp->profile.trtcm_rfc2698.cir; 635 cbs = fmp->profile.trtcm_rfc2698.cbs; 636 /* EIR / EBS are filled with PIR / PBS. */ 637 eir = fmp->profile.trtcm_rfc2698.pir; 638 ebs = fmp->profile.trtcm_rfc2698.pbs; 639 break; 640 case RTE_MTR_TRTCM_RFC4115: 641 cir = fmp->profile.trtcm_rfc4115.cir; 642 cbs = fmp->profile.trtcm_rfc4115.cbs; 643 eir = fmp->profile.trtcm_rfc4115.eir; 644 ebs = fmp->profile.trtcm_rfc4115.ebs; 645 break; 646 default: 647 return -rte_mtr_error_set(error, EINVAL, 648 RTE_MTR_ERROR_TYPE_METER_PROFILE, NULL, 649 "Metering algorithm mode is invalid"); 650 } 651 /* Adjust the values for PPS mode. */ 652 if (fmp->profile.packet_mode) { 653 cir <<= MLX5_MTRS_PPS_MAP_BPS_SHIFT; 654 cbs <<= MLX5_MTRS_PPS_MAP_BPS_SHIFT; 655 eir <<= MLX5_MTRS_PPS_MAP_BPS_SHIFT; 656 ebs <<= MLX5_MTRS_PPS_MAP_BPS_SHIFT; 657 } 658 /* cir = 8G * cir_mantissa * 1/(2^cir_exponent)) Bytes/Sec */ 659 mlx5_flow_meter_xir_man_exp_calc(cir, &man, &exp); 660 /* Check if cir mantissa is too large. */ 661 if (exp > ASO_DSEG_XIR_EXP_MASK) 662 return -rte_mtr_error_set(error, ENOTSUP, 663 RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL, 664 "meter profile parameter cir is not supported."); 665 cir_man = man; 666 cir_exp = exp; 667 /* cbs = cbs_mantissa * 2^cbs_exponent */ 668 mlx5_flow_meter_xbs_man_exp_calc(cbs, &man, &exp); 669 /* Check if cbs mantissa is too large. */ 670 if (exp > ASO_DSEG_EXP_MASK) 671 return -rte_mtr_error_set(error, ENOTSUP, 672 RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL, 673 "meter profile parameter cbs is not supported."); 674 cbs_man = man; 675 cbs_exp = exp; 676 srtcm->cbs_cir = rte_cpu_to_be_32(cbs_exp << ASO_DSEG_CBS_EXP_OFFSET | 677 cbs_man << ASO_DSEG_CBS_MAN_OFFSET | 678 cir_exp << ASO_DSEG_XIR_EXP_OFFSET | 679 cir_man); 680 mlx5_flow_meter_xir_man_exp_calc(eir, &man, &exp); 681 /* Check if eir mantissa is too large. */ 682 if (exp > ASO_DSEG_XIR_EXP_MASK) 683 return -rte_mtr_error_set(error, ENOTSUP, 684 RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL, 685 "meter profile parameter eir is not supported."); 686 eir_man = man; 687 eir_exp = exp; 688 mlx5_flow_meter_xbs_man_exp_calc(ebs, &man, &exp); 689 /* Check if ebs mantissa is too large. */ 690 if (exp > ASO_DSEG_EXP_MASK) 691 return -rte_mtr_error_set(error, ENOTSUP, 692 RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL, 693 "meter profile parameter ebs is not supported."); 694 ebs_man = man; 695 ebs_exp = exp; 696 srtcm->ebs_eir = rte_cpu_to_be_32(ebs_exp << ASO_DSEG_EBS_EXP_OFFSET | 697 ebs_man << ASO_DSEG_EBS_MAN_OFFSET | 698 eir_exp << ASO_DSEG_XIR_EXP_OFFSET | 699 eir_man); 700 if (srtcm->cbs_cir) 701 fmp->g_support = 1; 702 if (srtcm->ebs_eir) 703 fmp->y_support = 1; 704 return 0; 705 } 706 707 /** 708 * Callback to get MTR maximum objects number. 709 * 710 * @param[in] priv 711 * Pointer to Ethernet device. 712 * 713 * @return 714 * Max number of meters. 715 */ 716 uint32_t 717 mlx5_flow_mtr_max_get(struct mlx5_priv *priv) 718 { 719 struct mlx5_hca_qos_attr *qattr = &priv->sh->cdev->config.hca_attr.qos; 720 721 /* Max number of meters. */ 722 return ((priv->sh->meter_aso_en) ? 723 1 << (qattr->log_max_num_meter_aso + 1) : 724 qattr->log_max_flow_meter); 725 } 726 727 /** 728 * Callback to get MTR capabilities. 729 * 730 * @param[in] dev 731 * Pointer to Ethernet device. 732 * @param[out] cap 733 * Pointer to save MTR capabilities. 734 * @param[out] error 735 * Pointer to the error structure. 736 * 737 * @return 738 * 0 on success, a negative errno value otherwise and rte_errno is set. 739 */ 740 static int 741 mlx5_flow_mtr_cap_get(struct rte_eth_dev *dev, 742 struct rte_mtr_capabilities *cap, 743 struct rte_mtr_error *error __rte_unused) 744 { 745 struct mlx5_priv *priv = dev->data->dev_private; 746 struct mlx5_hca_qos_attr *qattr = &priv->sh->cdev->config.hca_attr.qos; 747 748 if (mlx5_hws_active(dev) && !mlx5_hw_ctx_validate(dev, NULL)) 749 return -rte_mtr_error_set(error, EINVAL, 750 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 751 "non-template flow engine was not configured"); 752 if (!priv->mtr_en) 753 return -rte_mtr_error_set(error, ENOTSUP, 754 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 755 "Meter is not supported"); 756 memset(cap, 0, sizeof(*cap)); 757 cap->n_max = mlx5_flow_mtr_max_get(priv); 758 if (priv->sh->meter_aso_en) { 759 cap->srtcm_rfc2697_packet_mode_supported = 1; 760 cap->trtcm_rfc2698_packet_mode_supported = 1; 761 cap->trtcm_rfc4115_packet_mode_supported = 1; 762 } 763 cap->srtcm_rfc2697_byte_mode_supported = 1; 764 cap->trtcm_rfc2698_byte_mode_supported = 1; 765 cap->trtcm_rfc4115_byte_mode_supported = 1; 766 cap->n_shared_max = cap->n_max; 767 cap->identical = 1; 768 cap->shared_identical = 1; 769 cap->shared_n_flows_per_mtr_max = 4 << 20; 770 /* 2M flows can share the same meter. */ 771 cap->chaining_n_mtrs_per_flow_max = 1; /* Chaining is not supported. */ 772 cap->meter_srtcm_rfc2697_n_max = qattr->flow_meter_old ? cap->n_max : 0; 773 cap->meter_trtcm_rfc2698_n_max = qattr->flow_meter_old ? cap->n_max : 0; 774 cap->meter_trtcm_rfc4115_n_max = qattr->flow_meter_old ? cap->n_max : 0; 775 cap->meter_rate_max = 1ULL << 40; /* 1 Tera tokens per sec. */ 776 cap->meter_policy_n_max = cap->n_max; 777 cap->stats_mask = RTE_MTR_STATS_N_BYTES_DROPPED | 778 RTE_MTR_STATS_N_PKTS_DROPPED; 779 return 0; 780 } 781 782 /** 783 * Callback to add MTR profile. 784 * 785 * @param[in] dev 786 * Pointer to Ethernet device. 787 * @param[in] meter_profile_id 788 * Meter profile id. 789 * @param[in] profile 790 * Pointer to meter profile detail. 791 * @param[out] error 792 * Pointer to the error structure. 793 * 794 * @return 795 * 0 on success, a negative errno value otherwise and rte_errno is set. 796 */ 797 static int 798 mlx5_flow_meter_profile_add(struct rte_eth_dev *dev, 799 uint32_t meter_profile_id, 800 struct rte_mtr_meter_profile *profile, 801 struct rte_mtr_error *error) 802 { 803 struct mlx5_priv *priv = dev->data->dev_private; 804 struct mlx5_flow_meter_profile *fmp; 805 union mlx5_l3t_data data; 806 int ret; 807 808 if (!priv->mtr_en) 809 return -rte_mtr_error_set(error, ENOTSUP, 810 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 811 "Meter is not supported"); 812 /* Check input params. */ 813 ret = mlx5_flow_meter_profile_validate(dev, meter_profile_id, 814 profile, error); 815 if (ret) 816 return ret; 817 /* Meter profile memory allocation. */ 818 fmp = mlx5_malloc(MLX5_MEM_ZERO, sizeof(struct mlx5_flow_meter_profile), 819 RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY); 820 if (fmp == NULL) 821 return -rte_mtr_error_set(error, ENOMEM, 822 RTE_MTR_ERROR_TYPE_UNSPECIFIED, 823 NULL, "Meter profile memory " 824 "alloc failed."); 825 /* Fill profile info. */ 826 fmp->id = meter_profile_id; 827 fmp->profile = *profile; 828 /* Fill the flow meter parameters for the PRM. */ 829 ret = mlx5_flow_meter_param_fill(fmp, error); 830 if (ret) 831 goto error; 832 data.ptr = fmp; 833 ret = mlx5_l3t_set_entry(priv->mtr_profile_tbl, 834 meter_profile_id, &data); 835 if (ret) 836 return -rte_mtr_error_set(error, ENOTSUP, 837 RTE_MTR_ERROR_TYPE_UNSPECIFIED, 838 NULL, "Meter profile insert fail."); 839 return 0; 840 error: 841 mlx5_free(fmp); 842 return ret; 843 } 844 845 /** 846 * Callback to delete MTR profile. 847 * 848 * @param[in] dev 849 * Pointer to Ethernet device. 850 * @param[in] meter_profile_id 851 * Meter profile id. 852 * @param[out] error 853 * Pointer to the error structure. 854 * 855 * @return 856 * 0 on success, a negative errno value otherwise and rte_errno is set. 857 */ 858 static int 859 mlx5_flow_meter_profile_delete(struct rte_eth_dev *dev, 860 uint32_t meter_profile_id, 861 struct rte_mtr_error *error) 862 { 863 struct mlx5_priv *priv = dev->data->dev_private; 864 struct mlx5_flow_meter_profile *fmp; 865 866 if (!priv->mtr_en) 867 return -rte_mtr_error_set(error, ENOTSUP, 868 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 869 "Meter is not supported"); 870 /* Meter profile must exist. */ 871 fmp = mlx5_flow_meter_profile_find(priv, meter_profile_id); 872 if (fmp == NULL) 873 return -rte_mtr_error_set(error, ENOENT, 874 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, 875 &meter_profile_id, 876 "Meter profile id is invalid."); 877 /* Check profile is unused. */ 878 if (fmp->ref_cnt) 879 return -rte_mtr_error_set(error, EBUSY, 880 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, 881 NULL, "Meter profile is in use."); 882 if (mlx5_l3t_clear_entry(priv->mtr_profile_tbl, meter_profile_id)) 883 return -rte_mtr_error_set(error, EBUSY, 884 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, 885 NULL, "Meter profile remove fail."); 886 mlx5_free(fmp); 887 return 0; 888 } 889 890 /** 891 * Callback to get MTR profile. 892 * 893 * @param[in] dev 894 * Pointer to Ethernet device. 895 * @param[in] meter_profile_id 896 * Meter profile id. 897 * @param[out] error 898 * Pointer to the error structure. 899 * 900 * @return 901 * A valid handle in case of success, NULL otherwise. 902 */ 903 static struct rte_flow_meter_profile * 904 mlx5_flow_meter_profile_get(struct rte_eth_dev *dev, 905 uint32_t meter_profile_id, 906 struct rte_mtr_error *error) 907 { 908 struct mlx5_priv *priv = dev->data->dev_private; 909 910 if (mlx5_hws_active(dev) && !mlx5_hw_ctx_validate(dev, NULL)) { 911 rte_mtr_error_set(error, EINVAL, 912 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 913 "non-template flow engine was not configured"); 914 return NULL; 915 } 916 if (!priv->mtr_en) { 917 rte_mtr_error_set(error, ENOTSUP, 918 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 919 "Meter is not supported"); 920 return NULL; 921 } 922 return (void *)(uintptr_t)mlx5_flow_meter_profile_find(priv, 923 meter_profile_id); 924 } 925 926 #if defined(HAVE_MLX5_HWS_SUPPORT) 927 /** 928 * Callback to add MTR profile with HWS. 929 * 930 * @param[in] dev 931 * Pointer to Ethernet device. 932 * @param[in] meter_profile_id 933 * Meter profile id. 934 * @param[in] profile 935 * Pointer to meter profile detail. 936 * @param[out] error 937 * Pointer to the error structure. 938 * 939 * @return 940 * 0 on success, a negative errno value otherwise and rte_errno is set. 941 */ 942 static int 943 mlx5_flow_meter_profile_hws_add(struct rte_eth_dev *dev, 944 uint32_t meter_profile_id, 945 struct rte_mtr_meter_profile *profile, 946 struct rte_mtr_error *error) 947 { 948 struct mlx5_priv *priv = dev->data->dev_private; 949 struct mlx5_flow_meter_profile *fmp; 950 int ret; 951 952 if (mlx5_hws_active(dev) && !mlx5_hw_ctx_validate(dev, NULL)) 953 return -rte_mtr_error_set(error, EINVAL, 954 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 955 "non-template flow engine was not configured"); 956 if (priv->shared_host) 957 return -rte_mtr_error_set(error, ENOTSUP, RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 958 "Meter profiles cannot be created on guest port"); 959 if (!priv->mtr_profile_arr) 960 return mlx5_flow_meter_profile_add(dev, meter_profile_id, profile, error); 961 /* Check input params. */ 962 ret = mlx5_flow_meter_profile_validate(dev, meter_profile_id, 963 profile, error); 964 if (ret) 965 return ret; 966 fmp = mlx5_flow_meter_profile_find(priv, meter_profile_id); 967 /* Fill profile info. */ 968 fmp->id = meter_profile_id; 969 fmp->profile = *profile; 970 fmp->initialized = 1; 971 /* Fill the flow meter parameters for the PRM. */ 972 return mlx5_flow_meter_param_fill(fmp, error); 973 } 974 975 /** 976 * Callback to delete MTR profile with HWS. 977 * 978 * @param[in] dev 979 * Pointer to Ethernet device. 980 * @param[in] meter_profile_id 981 * Meter profile id. 982 * @param[out] error 983 * Pointer to the error structure. 984 * 985 * @return 986 * 0 on success, a negative errno value otherwise and rte_errno is set. 987 */ 988 static int 989 mlx5_flow_meter_profile_hws_delete(struct rte_eth_dev *dev, 990 uint32_t meter_profile_id, 991 struct rte_mtr_error *error) 992 { 993 struct mlx5_priv *priv = dev->data->dev_private; 994 struct mlx5_flow_meter_profile *fmp; 995 996 if (priv->shared_host) 997 return -rte_mtr_error_set(error, ENOTSUP, RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 998 "Meter profiles cannot be destroyed through guest port"); 999 if (!priv->mtr_profile_arr) 1000 return mlx5_flow_meter_profile_delete(dev, meter_profile_id, error); 1001 /* Meter profile must exist. */ 1002 fmp = mlx5_flow_meter_profile_find(priv, meter_profile_id); 1003 if (!fmp->initialized) 1004 return -rte_mtr_error_set(error, ENOENT, 1005 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, 1006 &meter_profile_id, 1007 "Meter profile id is invalid."); 1008 /* Check profile is unused. */ 1009 if (fmp->ref_cnt) 1010 return -rte_mtr_error_set(error, EBUSY, 1011 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, 1012 NULL, "Meter profile is in use."); 1013 memset(fmp, 0, sizeof(struct mlx5_flow_meter_profile)); 1014 return 0; 1015 } 1016 #endif 1017 1018 /** 1019 * Find policy by id. 1020 * 1021 * @param[in] dev 1022 * Pointer to Ethernet device. 1023 * @param policy_id 1024 * Policy id. 1025 * 1026 * @return 1027 * Pointer to the policy found on success, NULL otherwise. 1028 */ 1029 struct mlx5_flow_meter_policy * 1030 mlx5_flow_meter_policy_find(struct rte_eth_dev *dev, 1031 uint32_t policy_id, 1032 uint32_t *policy_idx) 1033 { 1034 struct mlx5_priv *priv = dev->data->dev_private; 1035 struct mlx5_flow_meter_sub_policy *sub_policy = NULL; 1036 union mlx5_l3t_data data; 1037 1038 if (priv->mtr_policy_arr) { 1039 if (policy_idx) 1040 *policy_idx = policy_id; 1041 return &priv->mtr_policy_arr[policy_id]; 1042 } 1043 if (policy_id > MLX5_MAX_SUB_POLICY_TBL_NUM || !priv->policy_idx_tbl) 1044 return NULL; 1045 if (mlx5_l3t_get_entry(priv->policy_idx_tbl, policy_id, &data) || 1046 !data.dword) 1047 return NULL; 1048 if (policy_idx) 1049 *policy_idx = data.dword; 1050 sub_policy = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MTR_POLICY], 1051 data.dword); 1052 /* Remove reference taken by the mlx5_l3t_get_entry. */ 1053 mlx5_l3t_clear_entry(priv->policy_idx_tbl, policy_id); 1054 if (sub_policy) 1055 if (sub_policy->main_policy_id) 1056 return sub_policy->main_policy; 1057 return NULL; 1058 } 1059 1060 /** 1061 * Get the next meter from one meter's policy in hierarchy chain. 1062 * Lock free, mutex should be acquired by caller. 1063 * 1064 * @param[in] priv 1065 * Pointer to mlx5_priv. 1066 * @param[in] policy 1067 * Pointer to flow meter policy. 1068 * @param[out] mtr_idx 1069 * Pointer to Meter index. 1070 * 1071 * @return 1072 * Pointer to the next meter, or NULL when fail. 1073 */ 1074 struct mlx5_flow_meter_info * 1075 mlx5_flow_meter_hierarchy_next_meter(struct mlx5_priv *priv, 1076 struct mlx5_flow_meter_policy *policy, 1077 uint32_t *mtr_idx) 1078 { 1079 int i; 1080 1081 for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) { 1082 if (policy->act_cnt[i].fate_action == MLX5_FLOW_FATE_MTR) 1083 return mlx5_flow_meter_find(priv, 1084 policy->act_cnt[i].next_mtr_id, 1085 mtr_idx); 1086 } 1087 return NULL; 1088 } 1089 1090 /** 1091 * Get the last meter's policy from one meter's policy in hierarchy. 1092 * 1093 * @param[in] dev 1094 * Pointer to Ethernet device. 1095 * @param[in] policy 1096 * Pointer to flow meter policy. 1097 * 1098 * @return 1099 * Pointer to the final meter's policy, or NULL when fail. 1100 */ 1101 struct mlx5_flow_meter_policy * 1102 mlx5_flow_meter_hierarchy_get_final_policy(struct rte_eth_dev *dev, 1103 struct mlx5_flow_meter_policy *policy) 1104 { 1105 struct mlx5_priv *priv = dev->data->dev_private; 1106 struct mlx5_flow_meter_info *next_fm; 1107 struct mlx5_flow_meter_policy *next_policy = policy; 1108 1109 while (next_policy->is_hierarchy) { 1110 rte_spinlock_lock(&next_policy->sl); 1111 next_fm = mlx5_flow_meter_hierarchy_next_meter(priv, next_policy, NULL); 1112 rte_spinlock_unlock(&next_policy->sl); 1113 if (!next_fm || next_fm->def_policy) 1114 return NULL; 1115 next_policy = mlx5_flow_meter_policy_find(dev, 1116 next_fm->policy_id, NULL); 1117 MLX5_ASSERT(next_policy); 1118 } 1119 return next_policy; 1120 } 1121 1122 /** 1123 * Callback to check MTR policy action validate 1124 * 1125 * @param[in] dev 1126 * Pointer to Ethernet device. 1127 * @param[in] actions 1128 * Pointer to meter policy action detail. 1129 * @param[out] error 1130 * Pointer to the error structure. 1131 * 1132 * @return 1133 * 0 on success, a negative errno value otherwise and rte_errno is set. 1134 */ 1135 static int 1136 mlx5_flow_meter_policy_validate(struct rte_eth_dev *dev, 1137 struct rte_mtr_meter_policy_params *policy, 1138 struct rte_mtr_error *error) 1139 { 1140 struct mlx5_priv *priv = dev->data->dev_private; 1141 struct rte_flow_attr attr = { .transfer = priv->sh->config.dv_esw_en ? 1142 1 : 0 }; 1143 bool is_rss = false; 1144 uint8_t policy_mode; 1145 uint8_t domain_bitmap; 1146 int ret; 1147 1148 if (!priv->mtr_en || !priv->sh->meter_aso_en) 1149 return -rte_mtr_error_set(error, ENOTSUP, 1150 RTE_MTR_ERROR_TYPE_METER_POLICY, 1151 NULL, "meter policy unsupported."); 1152 ret = mlx5_flow_validate_mtr_acts(dev, policy->actions, &attr, 1153 &is_rss, &domain_bitmap, &policy_mode, error); 1154 if (ret) 1155 return ret; 1156 return 0; 1157 } 1158 1159 #if defined(HAVE_MLX5_HWS_SUPPORT) 1160 /** 1161 * Callback to check MTR policy action validate for HWS 1162 * 1163 * @param[in] dev 1164 * Pointer to Ethernet device. 1165 * @param[in] actions 1166 * Pointer to meter policy action detail. 1167 * @param[out] error 1168 * Pointer to the error structure. 1169 * 1170 * @return 1171 * 0 on success, a negative errno value otherwise and rte_errno is set. 1172 */ 1173 static int 1174 mlx5_flow_meter_policy_hws_validate(struct rte_eth_dev *dev, 1175 struct rte_mtr_meter_policy_params *policy, 1176 struct rte_mtr_error *error) 1177 { 1178 struct mlx5_priv *priv = dev->data->dev_private; 1179 const struct rte_flow_actions_template_attr attr = { 1180 .transfer = priv->sh->config.dv_esw_en ? 1 : 0 }; 1181 int ret; 1182 int i; 1183 1184 if (mlx5_hws_active(dev) && !mlx5_hw_ctx_validate(dev, NULL)) 1185 return -rte_mtr_error_set(error, EINVAL, 1186 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 1187 "non-template flow engine was not configured"); 1188 if (!priv->mtr_en || !priv->sh->meter_aso_en) 1189 return -rte_mtr_error_set(error, ENOTSUP, 1190 RTE_MTR_ERROR_TYPE_METER_POLICY, 1191 NULL, "meter policy unsupported."); 1192 for (i = 0; i < RTE_COLORS; i++) { 1193 ret = mlx5_flow_actions_validate(dev, &attr, policy->actions[i], 1194 policy->actions[i], NULL); 1195 if (ret) 1196 return ret; 1197 } 1198 return 0; 1199 } 1200 #endif 1201 1202 static int 1203 __mlx5_flow_meter_policy_delete(struct rte_eth_dev *dev, 1204 uint32_t policy_id, 1205 struct mlx5_flow_meter_policy *mtr_policy, 1206 struct rte_mtr_error *error, 1207 bool clear_l3t) 1208 { 1209 struct mlx5_priv *priv = dev->data->dev_private; 1210 struct mlx5_flow_meter_sub_policy *sub_policy; 1211 uint32_t i, j; 1212 uint16_t sub_policy_num; 1213 1214 rte_spinlock_lock(&mtr_policy->sl); 1215 if (mtr_policy->ref_cnt) { 1216 rte_spinlock_unlock(&mtr_policy->sl); 1217 return -rte_mtr_error_set(error, EBUSY, 1218 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 1219 NULL, 1220 "Meter policy object is being used."); 1221 } 1222 mlx5_flow_destroy_policy_rules(dev, mtr_policy); 1223 mlx5_flow_destroy_mtr_acts(dev, mtr_policy); 1224 for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) { 1225 sub_policy_num = (mtr_policy->sub_policy_num >> 1226 (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i)) & 1227 MLX5_MTR_SUB_POLICY_NUM_MASK; 1228 if (sub_policy_num) { 1229 for (j = 0; j < sub_policy_num; j++) { 1230 sub_policy = mtr_policy->sub_policys[i][j]; 1231 if (sub_policy) 1232 mlx5_ipool_free 1233 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY], 1234 sub_policy->idx); 1235 } 1236 } 1237 } 1238 if (priv->policy_idx_tbl && clear_l3t) { 1239 if (mlx5_l3t_clear_entry(priv->policy_idx_tbl, policy_id)) { 1240 rte_spinlock_unlock(&mtr_policy->sl); 1241 return -rte_mtr_error_set(error, ENOTSUP, 1242 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL, 1243 "Fail to delete policy in index table."); 1244 } 1245 } 1246 rte_spinlock_unlock(&mtr_policy->sl); 1247 return 0; 1248 } 1249 1250 /** 1251 * Callback to add MTR policy. 1252 * 1253 * @param[in] dev 1254 * Pointer to Ethernet device. 1255 * @param[out] policy_id 1256 * Pointer to policy id 1257 * @param[in] actions 1258 * Pointer to meter policy action detail. 1259 * @param[out] error 1260 * Pointer to the error structure. 1261 * 1262 * @return 1263 * 0 on success, a negative errno value otherwise and rte_errno is set. 1264 */ 1265 static int 1266 mlx5_flow_meter_policy_add(struct rte_eth_dev *dev, 1267 uint32_t policy_id, 1268 struct rte_mtr_meter_policy_params *policy, 1269 struct rte_mtr_error *error) 1270 { 1271 struct mlx5_priv *priv = dev->data->dev_private; 1272 struct rte_flow_attr attr = { .transfer = priv->sh->config.dv_esw_en ? 1273 1 : 0 }; 1274 uint32_t sub_policy_idx = 0; 1275 uint32_t policy_idx = 0; 1276 struct mlx5_flow_meter_policy *mtr_policy = NULL; 1277 struct mlx5_flow_meter_sub_policy *sub_policy; 1278 bool is_rss = false; 1279 uint8_t policy_mode; 1280 uint32_t i; 1281 int ret; 1282 uint32_t policy_size = sizeof(struct mlx5_flow_meter_policy); 1283 uint16_t sub_policy_num; 1284 uint8_t domain_bitmap = 0; 1285 union mlx5_l3t_data data; 1286 bool skip_rule = false; 1287 1288 if (!priv->mtr_en) 1289 return -rte_mtr_error_set(error, ENOTSUP, 1290 RTE_MTR_ERROR_TYPE_METER_POLICY, 1291 NULL, "meter policy unsupported. "); 1292 if (policy_id == MLX5_INVALID_POLICY_ID) 1293 return -rte_mtr_error_set(error, ENOTSUP, 1294 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 1295 NULL, "policy ID is invalid. "); 1296 if (policy_id == priv->sh->mtrmng->def_policy_id) 1297 return -rte_mtr_error_set(error, EEXIST, 1298 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 1299 NULL, "default policy ID exists. "); 1300 mtr_policy = mlx5_flow_meter_policy_find(dev, policy_id, &policy_idx); 1301 if (mtr_policy) 1302 return -rte_mtr_error_set(error, EEXIST, 1303 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 1304 NULL, "policy ID exists. "); 1305 ret = mlx5_flow_validate_mtr_acts(dev, policy->actions, &attr, 1306 &is_rss, &domain_bitmap, 1307 &policy_mode, error); 1308 if (ret) 1309 return ret; 1310 if (!domain_bitmap) 1311 return -rte_mtr_error_set(error, ENOTSUP, 1312 RTE_MTR_ERROR_TYPE_METER_POLICY, 1313 NULL, "fail to find policy domain."); 1314 if (policy_mode == MLX5_MTR_POLICY_MODE_DEF) { 1315 if (priv->sh->mtrmng->def_policy_id != MLX5_INVALID_POLICY_ID) 1316 return -rte_mtr_error_set(error, EEXIST, 1317 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 1318 NULL, "a policy with similar actions " 1319 "is already configured"); 1320 if (mlx5_flow_create_def_policy(dev)) 1321 return -rte_mtr_error_set(error, ENOTSUP, 1322 RTE_MTR_ERROR_TYPE_METER_POLICY, 1323 NULL, 1324 "fail to create non-terminated policy."); 1325 priv->sh->mtrmng->def_policy_id = policy_id; 1326 return 0; 1327 } 1328 if (!priv->sh->meter_aso_en) 1329 return -rte_mtr_error_set(error, ENOTSUP, 1330 RTE_MTR_ERROR_TYPE_METER_POLICY, NULL, 1331 "no ASO capability to support the policy "); 1332 for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) { 1333 if (!(domain_bitmap & (1 << i))) 1334 continue; 1335 /* 1336 * If RSS is found, it means that only the ingress domain can 1337 * be supported. It is invalid to support RSS for one color 1338 * and egress / transfer domain actions for another. Drop and 1339 * jump action should have no impact. 1340 */ 1341 if (is_rss) { 1342 policy_size += 1343 sizeof(struct mlx5_flow_meter_sub_policy *) * 1344 MLX5_MTR_RSS_MAX_SUB_POLICY; 1345 break; 1346 } 1347 policy_size += sizeof(struct mlx5_flow_meter_sub_policy *); 1348 } 1349 mtr_policy = mlx5_malloc(MLX5_MEM_ZERO, policy_size, 1350 RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY); 1351 if (!mtr_policy) 1352 return -rte_mtr_error_set(error, ENOMEM, 1353 RTE_MTR_ERROR_TYPE_METER_POLICY, NULL, 1354 "Memory alloc failed for meter policy."); 1355 if (policy_mode == MLX5_MTR_POLICY_MODE_OG) 1356 mtr_policy->skip_y = 1; 1357 else if (policy_mode == MLX5_MTR_POLICY_MODE_OY) 1358 mtr_policy->skip_g = 1; 1359 policy_size = sizeof(struct mlx5_flow_meter_policy); 1360 for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) { 1361 if (!(domain_bitmap & (1 << i))) 1362 continue; 1363 if (i == MLX5_MTR_DOMAIN_INGRESS) 1364 mtr_policy->ingress = 1; 1365 if (i == MLX5_MTR_DOMAIN_EGRESS) 1366 mtr_policy->egress = 1; 1367 if (i == MLX5_MTR_DOMAIN_TRANSFER) 1368 mtr_policy->transfer = 1; 1369 sub_policy = mlx5_ipool_zmalloc 1370 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY], 1371 &sub_policy_idx); 1372 if (!sub_policy || 1373 sub_policy_idx > MLX5_MAX_SUB_POLICY_TBL_NUM) 1374 goto policy_add_err; 1375 sub_policy->idx = sub_policy_idx; 1376 sub_policy->main_policy = mtr_policy; 1377 if (!policy_idx) { 1378 policy_idx = sub_policy_idx; 1379 sub_policy->main_policy_id = 1; 1380 } 1381 mtr_policy->sub_policys[i] = 1382 (struct mlx5_flow_meter_sub_policy **) 1383 ((uint8_t *)mtr_policy + policy_size); 1384 mtr_policy->sub_policys[i][0] = sub_policy; 1385 sub_policy_num = (mtr_policy->sub_policy_num >> 1386 (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i)) & 1387 MLX5_MTR_SUB_POLICY_NUM_MASK; 1388 sub_policy_num++; 1389 mtr_policy->sub_policy_num &= ~(MLX5_MTR_SUB_POLICY_NUM_MASK << 1390 (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i)); 1391 mtr_policy->sub_policy_num |= 1392 (sub_policy_num & MLX5_MTR_SUB_POLICY_NUM_MASK) << 1393 (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i); 1394 /* 1395 * If RSS is found, it means that only the ingress domain can 1396 * be supported. It is invalid to support RSS for one color 1397 * and egress / transfer domain actions for another. Drop and 1398 * jump action should have no impact. 1399 */ 1400 if (is_rss) { 1401 mtr_policy->is_rss = 1; 1402 break; 1403 } 1404 policy_size += sizeof(struct mlx5_flow_meter_sub_policy *); 1405 } 1406 rte_spinlock_init(&mtr_policy->sl); 1407 ret = mlx5_flow_create_mtr_acts(dev, mtr_policy, 1408 policy->actions, &attr, error); 1409 if (ret) 1410 goto policy_add_err; 1411 if (mtr_policy->is_hierarchy) { 1412 struct mlx5_flow_meter_policy *final_policy; 1413 1414 final_policy = 1415 mlx5_flow_meter_hierarchy_get_final_policy(dev, mtr_policy); 1416 if (!final_policy) 1417 goto policy_add_err; 1418 skip_rule = (final_policy->is_rss || final_policy->is_queue); 1419 } 1420 /* 1421 * If either Green or Yellow has queue / RSS action, all the policy 1422 * rules will be created later in the flow splitting stage. 1423 */ 1424 if (!is_rss && !mtr_policy->is_queue && !skip_rule) { 1425 /* Create policy rules in HW. */ 1426 ret = mlx5_flow_create_policy_rules(dev, mtr_policy); 1427 if (ret) 1428 goto policy_add_err; 1429 } 1430 data.dword = policy_idx; 1431 if (!priv->policy_idx_tbl) { 1432 priv->policy_idx_tbl = mlx5_l3t_create(MLX5_L3T_TYPE_DWORD); 1433 if (!priv->policy_idx_tbl) 1434 goto policy_add_err; 1435 } 1436 if (mlx5_l3t_set_entry(priv->policy_idx_tbl, policy_id, &data)) 1437 goto policy_add_err; 1438 return 0; 1439 policy_add_err: 1440 if (mtr_policy) { 1441 ret = __mlx5_flow_meter_policy_delete(dev, policy_id, 1442 mtr_policy, error, false); 1443 mlx5_free(mtr_policy); 1444 if (ret) 1445 return ret; 1446 } 1447 return -rte_mtr_error_set(error, ENOTSUP, 1448 RTE_MTR_ERROR_TYPE_UNSPECIFIED, 1449 NULL, "Failed to create devx policy."); 1450 } 1451 1452 /** 1453 * Callback to delete MTR policy. 1454 * 1455 * @param[in] dev 1456 * Pointer to Ethernet device. 1457 * @param[in] policy_id 1458 * Meter policy id. 1459 * @param[out] error 1460 * Pointer to the error structure. 1461 * 1462 * @return 1463 * 0 on success, a negative errno value otherwise and rte_errno is set. 1464 */ 1465 static int 1466 mlx5_flow_meter_policy_delete(struct rte_eth_dev *dev, 1467 uint32_t policy_id, 1468 struct rte_mtr_error *error) 1469 { 1470 struct mlx5_priv *priv = dev->data->dev_private; 1471 struct mlx5_flow_meter_policy *mtr_policy; 1472 uint32_t policy_idx; 1473 int ret; 1474 1475 if (policy_id == priv->sh->mtrmng->def_policy_id) { 1476 if (priv->sh->mtrmng->def_policy_ref_cnt > 0) 1477 return -rte_mtr_error_set(error, ENOTSUP, 1478 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL, 1479 "Meter policy object is being used."); 1480 priv->sh->mtrmng->def_policy_id = MLX5_INVALID_POLICY_ID; 1481 return 0; 1482 } 1483 mtr_policy = mlx5_flow_meter_policy_find(dev, policy_id, &policy_idx); 1484 if (!mtr_policy) 1485 return -rte_mtr_error_set(error, ENOTSUP, 1486 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL, 1487 "Meter policy id is invalid. "); 1488 ret = __mlx5_flow_meter_policy_delete(dev, policy_id, mtr_policy, 1489 error, true); 1490 if (ret) 1491 return ret; 1492 mlx5_free(mtr_policy); 1493 return 0; 1494 } 1495 1496 /** 1497 * Callback to get MTR policy. 1498 * 1499 * @param[in] dev 1500 * Pointer to Ethernet device. 1501 * @param[in] policy_id 1502 * Meter policy id. 1503 * @param[out] error 1504 * Pointer to the error structure. 1505 * 1506 * @return 1507 * A valid handle in case of success, NULL otherwise. 1508 */ 1509 static struct rte_flow_meter_policy * 1510 mlx5_flow_meter_policy_get(struct rte_eth_dev *dev, 1511 uint32_t policy_id, 1512 struct rte_mtr_error *error) 1513 { 1514 struct mlx5_priv *priv = dev->data->dev_private; 1515 uint32_t policy_idx; 1516 1517 if (mlx5_hws_active(dev) && !mlx5_hw_ctx_validate(dev, NULL)) { 1518 rte_mtr_error_set(error, EINVAL, 1519 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 1520 "non-template flow engine was not configured"); 1521 return NULL; 1522 } 1523 if (!priv->mtr_en) { 1524 rte_mtr_error_set(error, ENOTSUP, 1525 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 1526 "Meter is not supported"); 1527 return NULL; 1528 } 1529 return (void *)(uintptr_t)mlx5_flow_meter_policy_find(dev, policy_id, 1530 &policy_idx); 1531 } 1532 1533 #if defined(HAVE_MLX5_HWS_SUPPORT) 1534 /** 1535 * Callback to delete MTR policy for HWS. 1536 * 1537 * @param[in] dev 1538 * Pointer to Ethernet device. 1539 * @param[in] policy_id 1540 * Meter policy id. 1541 * @param[out] error 1542 * Pointer to the error structure. 1543 * 1544 * @return 1545 * 0 on success, a negative errno value otherwise and rte_errno is set. 1546 */ 1547 static int 1548 mlx5_flow_meter_policy_hws_delete(struct rte_eth_dev *dev, 1549 uint32_t policy_id, 1550 struct rte_mtr_error *error) 1551 { 1552 struct mlx5_priv *priv = dev->data->dev_private; 1553 struct mlx5_flow_meter_policy *mtr_policy; 1554 uint32_t i, j; 1555 uint32_t nb_flows = 0; 1556 int ret; 1557 struct rte_flow_op_attr op_attr = { .postpone = 1 }; 1558 struct rte_flow_op_result result[RTE_COLORS * MLX5_MTR_DOMAIN_MAX]; 1559 1560 if (!priv->mtr_policy_arr) 1561 return mlx5_flow_meter_policy_delete(dev, policy_id, error); 1562 /* Meter policy must exist. */ 1563 mtr_policy = mlx5_flow_meter_policy_find(dev, policy_id, NULL); 1564 if (!mtr_policy->initialized) 1565 return -rte_mtr_error_set(error, ENOENT, 1566 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL, 1567 "Meter policy does not exists."); 1568 /* Check policy is unused. */ 1569 if (mtr_policy->ref_cnt) 1570 return -rte_mtr_error_set(error, EBUSY, 1571 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 1572 NULL, "Meter policy is in use."); 1573 rte_spinlock_lock(&priv->hw_ctrl_lock); 1574 for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) { 1575 for (j = 0; j < RTE_COLORS; j++) { 1576 if (mtr_policy->hws_flow_rule[i][j]) { 1577 ret = rte_flow_async_destroy(dev->data->port_id, 1578 CTRL_QUEUE_ID(priv), &op_attr, 1579 mtr_policy->hws_flow_rule[i][j], 1580 NULL, NULL); 1581 if (ret < 0) 1582 continue; 1583 nb_flows++; 1584 } 1585 } 1586 } 1587 ret = rte_flow_push(dev->data->port_id, CTRL_QUEUE_ID(priv), NULL); 1588 while (nb_flows && (ret >= 0)) { 1589 ret = rte_flow_pull(dev->data->port_id, 1590 CTRL_QUEUE_ID(priv), result, 1591 nb_flows, NULL); 1592 nb_flows -= ret; 1593 } 1594 for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) { 1595 if (mtr_policy->hws_flow_table[i]) 1596 rte_flow_template_table_destroy(dev->data->port_id, 1597 mtr_policy->hws_flow_table[i], NULL); 1598 } 1599 for (i = 0; i < RTE_COLORS; i++) { 1600 if (mtr_policy->hws_act_templ[i]) 1601 rte_flow_actions_template_destroy(dev->data->port_id, 1602 mtr_policy->hws_act_templ[i], NULL); 1603 } 1604 if (mtr_policy->hws_item_templ) 1605 rte_flow_pattern_template_destroy(dev->data->port_id, 1606 mtr_policy->hws_item_templ, NULL); 1607 rte_spinlock_unlock(&priv->hw_ctrl_lock); 1608 memset(mtr_policy, 0, sizeof(struct mlx5_flow_meter_policy)); 1609 return 0; 1610 } 1611 1612 /** 1613 * Callback to add MTR policy for HWS. 1614 * 1615 * @param[in] dev 1616 * Pointer to Ethernet device. 1617 * @param[out] policy_id 1618 * Pointer to policy id 1619 * @param[in] actions 1620 * Pointer to meter policy action detail. 1621 * @param[out] error 1622 * Pointer to the error structure. 1623 * 1624 * @return 1625 * 0 on success, a negative errno value otherwise and rte_errno is set. 1626 */ 1627 static int 1628 mlx5_flow_meter_policy_hws_add(struct rte_eth_dev *dev, 1629 uint32_t policy_id, 1630 struct rte_mtr_meter_policy_params *policy, 1631 struct rte_mtr_error *error) 1632 { 1633 struct mlx5_priv *priv = dev->data->dev_private; 1634 struct mlx5_flow_meter_policy *mtr_policy = NULL; 1635 const struct rte_flow_action *act; 1636 const struct rte_flow_action_meter *mtr; 1637 struct mlx5_flow_meter_info *fm; 1638 struct mlx5_flow_meter_policy *plc; 1639 uint8_t domain_color = MLX5_MTR_ALL_DOMAIN_BIT; 1640 bool is_rss = false; 1641 bool is_hierarchy = false; 1642 int i, j; 1643 uint32_t nb_colors = 0; 1644 uint32_t nb_flows = 0; 1645 int color; 1646 int ret; 1647 struct rte_flow_pattern_template_attr pta = {0}; 1648 struct rte_flow_actions_template_attr ata = {0}; 1649 struct rte_flow_template_table_attr ta = { {0}, 0 }; 1650 struct rte_flow_op_attr op_attr = { .postpone = 1 }; 1651 struct rte_flow_op_result result[RTE_COLORS * MLX5_MTR_DOMAIN_MAX]; 1652 const uint32_t color_mask = (UINT32_C(1) << MLX5_MTR_COLOR_BITS) - 1; 1653 int color_reg_c_idx = mlx5_flow_get_reg_id(dev, MLX5_MTR_COLOR, 1654 0, NULL); 1655 struct rte_flow_item_tag tag_spec = { 1656 .data = 0, 1657 .index = color_reg_c_idx 1658 }; 1659 struct rte_flow_item_tag tag_mask = { 1660 .data = color_mask, 1661 .index = 0xff}; 1662 struct rte_flow_item pattern[] = { 1663 [0] = { 1664 .type = (enum rte_flow_item_type) 1665 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 1666 .spec = &tag_spec, 1667 .mask = &tag_mask, 1668 }, 1669 [1] = { .type = RTE_FLOW_ITEM_TYPE_END } 1670 }; 1671 1672 if (mlx5_hws_active(dev) && !mlx5_hw_ctx_validate(dev, NULL)) 1673 return -rte_mtr_error_set(error, EINVAL, 1674 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 1675 "non-template flow engine was not configured"); 1676 if (!priv->mtr_policy_arr) 1677 return mlx5_flow_meter_policy_add(dev, policy_id, policy, error); 1678 mtr_policy = mlx5_flow_meter_policy_find(dev, policy_id, NULL); 1679 if (mtr_policy->initialized) 1680 return -rte_mtr_error_set(error, EEXIST, 1681 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 1682 NULL, "Meter policy already exists."); 1683 if (!policy || 1684 (!policy->actions[RTE_COLOR_RED] && 1685 !policy->actions[RTE_COLOR_YELLOW] && 1686 !policy->actions[RTE_COLOR_GREEN])) 1687 return -rte_mtr_error_set(error, EINVAL, 1688 RTE_MTR_ERROR_TYPE_METER_POLICY, 1689 NULL, "Meter policy actions are not valid."); 1690 if (policy->actions[RTE_COLOR_RED] == RTE_FLOW_ACTION_TYPE_END) 1691 mtr_policy->skip_r = 1; 1692 if (policy->actions[RTE_COLOR_YELLOW] == RTE_FLOW_ACTION_TYPE_END) 1693 mtr_policy->skip_y = 1; 1694 if (policy->actions[RTE_COLOR_GREEN] == RTE_FLOW_ACTION_TYPE_END) 1695 mtr_policy->skip_g = 1; 1696 if (mtr_policy->skip_r && mtr_policy->skip_y && mtr_policy->skip_g) 1697 return -rte_mtr_error_set(error, ENOTSUP, 1698 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 1699 NULL, "Meter policy actions are empty."); 1700 for (i = 0; i < RTE_COLORS; i++) { 1701 act = policy->actions[i]; 1702 while (act && act->type != RTE_FLOW_ACTION_TYPE_END) { 1703 switch (act->type) { 1704 case RTE_FLOW_ACTION_TYPE_PORT_ID: 1705 /* fall-through. */ 1706 case RTE_FLOW_ACTION_TYPE_REPRESENTED_PORT: 1707 domain_color &= ~(MLX5_MTR_DOMAIN_INGRESS_BIT | 1708 MLX5_MTR_DOMAIN_EGRESS_BIT); 1709 break; 1710 case RTE_FLOW_ACTION_TYPE_RSS: 1711 is_rss = true; 1712 /* fall-through. */ 1713 case RTE_FLOW_ACTION_TYPE_QUEUE: 1714 domain_color &= ~(MLX5_MTR_DOMAIN_EGRESS_BIT | 1715 MLX5_MTR_DOMAIN_TRANSFER_BIT); 1716 break; 1717 case RTE_FLOW_ACTION_TYPE_METER: 1718 is_hierarchy = true; 1719 mtr = act->conf; 1720 fm = mlx5_flow_meter_find(priv, 1721 mtr->mtr_id, NULL); 1722 if (!fm) 1723 return -rte_mtr_error_set(error, EINVAL, 1724 RTE_MTR_ERROR_TYPE_MTR_ID, NULL, 1725 "Meter not found in meter hierarchy."); 1726 plc = mlx5_flow_meter_policy_find(dev, 1727 fm->policy_id, 1728 NULL); 1729 MLX5_ASSERT(plc); 1730 domain_color &= MLX5_MTR_ALL_DOMAIN_BIT & 1731 (plc->ingress << 1732 MLX5_MTR_DOMAIN_INGRESS); 1733 domain_color &= MLX5_MTR_ALL_DOMAIN_BIT & 1734 (plc->egress << 1735 MLX5_MTR_DOMAIN_EGRESS); 1736 domain_color &= MLX5_MTR_ALL_DOMAIN_BIT & 1737 (plc->transfer << 1738 MLX5_MTR_DOMAIN_TRANSFER); 1739 break; 1740 default: 1741 break; 1742 } 1743 act++; 1744 } 1745 } 1746 if (priv->sh->config.dv_esw_en) 1747 domain_color &= ~(MLX5_MTR_DOMAIN_EGRESS_BIT | 1748 MLX5_MTR_DOMAIN_TRANSFER_BIT); 1749 else 1750 domain_color &= ~MLX5_MTR_DOMAIN_TRANSFER_BIT; 1751 if (!domain_color) 1752 return -rte_mtr_error_set(error, ENOTSUP, 1753 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 1754 NULL, "Meter policy domains are conflicting."); 1755 mtr_policy->is_rss = is_rss; 1756 mtr_policy->ingress = !!(domain_color & MLX5_MTR_DOMAIN_INGRESS_BIT); 1757 pta.ingress = mtr_policy->ingress; 1758 mtr_policy->egress = !!(domain_color & MLX5_MTR_DOMAIN_EGRESS_BIT); 1759 pta.egress = mtr_policy->egress; 1760 mtr_policy->transfer = !!(domain_color & MLX5_MTR_DOMAIN_TRANSFER_BIT); 1761 pta.transfer = mtr_policy->transfer; 1762 mtr_policy->group = MLX5_FLOW_TABLE_HWS_POLICY - policy_id; 1763 mtr_policy->is_hierarchy = is_hierarchy; 1764 mtr_policy->initialized = 1; 1765 rte_spinlock_lock(&priv->hw_ctrl_lock); 1766 mtr_policy->hws_item_templ = 1767 rte_flow_pattern_template_create(dev->data->port_id, 1768 &pta, pattern, NULL); 1769 if (!mtr_policy->hws_item_templ) 1770 goto policy_add_err; 1771 for (i = 0; i < RTE_COLORS; i++) { 1772 if (mtr_policy->skip_g && i == RTE_COLOR_GREEN) 1773 continue; 1774 if (mtr_policy->skip_y && i == RTE_COLOR_YELLOW) 1775 continue; 1776 if (mtr_policy->skip_r && i == RTE_COLOR_RED) 1777 continue; 1778 mtr_policy->hws_act_templ[nb_colors] = 1779 rte_flow_actions_template_create(dev->data->port_id, 1780 &ata, policy->actions[i], 1781 policy->actions[i], NULL); 1782 if (!mtr_policy->hws_act_templ[nb_colors]) 1783 goto policy_add_err; 1784 nb_colors++; 1785 } 1786 for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) { 1787 memset(&ta, 0, sizeof(ta)); 1788 ta.nb_flows = RTE_COLORS; 1789 ta.flow_attr.group = mtr_policy->group; 1790 if (i == MLX5_MTR_DOMAIN_INGRESS) { 1791 if (!mtr_policy->ingress) 1792 continue; 1793 ta.flow_attr.ingress = 1; 1794 } else if (i == MLX5_MTR_DOMAIN_EGRESS) { 1795 if (!mtr_policy->egress) 1796 continue; 1797 ta.flow_attr.egress = 1; 1798 } else if (i == MLX5_MTR_DOMAIN_TRANSFER) { 1799 if (!mtr_policy->transfer) 1800 continue; 1801 ta.flow_attr.transfer = 1; 1802 } 1803 mtr_policy->hws_flow_table[i] = 1804 rte_flow_template_table_create(dev->data->port_id, 1805 &ta, &mtr_policy->hws_item_templ, 1, 1806 mtr_policy->hws_act_templ, nb_colors, 1807 NULL); 1808 if (!mtr_policy->hws_flow_table[i]) 1809 goto policy_add_err; 1810 nb_colors = 0; 1811 for (j = 0; j < RTE_COLORS; j++) { 1812 if (mtr_policy->skip_g && j == RTE_COLOR_GREEN) 1813 continue; 1814 if (mtr_policy->skip_y && j == RTE_COLOR_YELLOW) 1815 continue; 1816 if (mtr_policy->skip_r && j == RTE_COLOR_RED) 1817 continue; 1818 color = rte_col_2_mlx5_col((enum rte_color)j); 1819 tag_spec.data = color; 1820 mtr_policy->hws_flow_rule[i][j] = 1821 rte_flow_async_create(dev->data->port_id, 1822 CTRL_QUEUE_ID(priv), &op_attr, 1823 mtr_policy->hws_flow_table[i], 1824 pattern, 0, policy->actions[j], 1825 nb_colors, NULL, NULL); 1826 if (!mtr_policy->hws_flow_rule[i][j]) 1827 goto policy_add_err; 1828 nb_colors++; 1829 nb_flows++; 1830 } 1831 ret = rte_flow_push(dev->data->port_id, 1832 CTRL_QUEUE_ID(priv), NULL); 1833 if (ret < 0) 1834 goto policy_add_err; 1835 while (nb_flows) { 1836 ret = rte_flow_pull(dev->data->port_id, 1837 CTRL_QUEUE_ID(priv), result, 1838 nb_flows, NULL); 1839 if (ret < 0) 1840 goto policy_add_err; 1841 for (j = 0; j < ret; j++) { 1842 if (result[j].status == RTE_FLOW_OP_ERROR) 1843 goto policy_add_err; 1844 } 1845 nb_flows -= ret; 1846 } 1847 } 1848 rte_spinlock_unlock(&priv->hw_ctrl_lock); 1849 return 0; 1850 policy_add_err: 1851 rte_spinlock_unlock(&priv->hw_ctrl_lock); 1852 ret = mlx5_flow_meter_policy_hws_delete(dev, policy_id, error); 1853 memset(mtr_policy, 0, sizeof(struct mlx5_flow_meter_policy)); 1854 if (ret) 1855 return ret; 1856 return -rte_mtr_error_set(error, ENOTSUP, 1857 RTE_MTR_ERROR_TYPE_UNSPECIFIED, 1858 NULL, "Failed to create meter policy."); 1859 } 1860 #endif 1861 /** 1862 * Check meter validation. 1863 * 1864 * @param[in] priv 1865 * Pointer to mlx5 private data structure. 1866 * @param[in] meter_id 1867 * Meter id. 1868 * @param[in] params 1869 * Pointer to rte meter parameters. 1870 * @param[out] error 1871 * Pointer to rte meter error structure. 1872 * 1873 * @return 1874 * 0 on success, a negative errno value otherwise and rte_errno is set. 1875 */ 1876 static int 1877 mlx5_flow_meter_validate(struct mlx5_priv *priv, uint32_t meter_id, 1878 struct rte_mtr_params *params, 1879 struct rte_mtr_error *error) 1880 { 1881 /* Meter must use global drop action. */ 1882 if (!priv->sh->dr_drop_action) 1883 return -rte_mtr_error_set(error, ENOTSUP, 1884 RTE_MTR_ERROR_TYPE_MTR_PARAMS, 1885 NULL, 1886 "No drop action ready for meter."); 1887 /* Meter params must not be NULL. */ 1888 if (params == NULL) 1889 return -rte_mtr_error_set(error, EINVAL, 1890 RTE_MTR_ERROR_TYPE_MTR_PARAMS, 1891 NULL, "Meter object params null."); 1892 /* Previous meter color is not supported. */ 1893 if (params->use_prev_mtr_color && !priv->sh->meter_aso_en) 1894 return -rte_mtr_error_set(error, ENOTSUP, 1895 RTE_MTR_ERROR_TYPE_MTR_PARAMS, 1896 NULL, 1897 "Previous meter color " 1898 "not supported."); 1899 if (params->meter_policy_id == MLX5_INVALID_POLICY_ID) 1900 return -rte_mtr_error_set(error, ENOENT, 1901 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 1902 NULL, "Meter policy id not valid."); 1903 /* Validate meter id. */ 1904 if (mlx5_flow_meter_find(priv, meter_id, NULL)) 1905 return -rte_mtr_error_set(error, EEXIST, 1906 RTE_MTR_ERROR_TYPE_MTR_ID, NULL, 1907 "Meter object already exists."); 1908 return 0; 1909 } 1910 1911 /** 1912 * Modify the flow meter action. 1913 * 1914 * @param[in] priv 1915 * Pointer to mlx5 private data structure. 1916 * @param[in] fm 1917 * Pointer to flow meter to be modified. 1918 * @param[in] srtcm 1919 * Pointer to meter srtcm description parameter. 1920 * @param[in] modify_bits 1921 * The bit in srtcm to be updated. 1922 * @param[in] active_state 1923 * The state to be updated. 1924 * @return 1925 * 0 on success, o negative value otherwise. 1926 */ 1927 static int 1928 mlx5_flow_meter_action_modify(struct mlx5_priv *priv, 1929 struct mlx5_flow_meter_info *fm, 1930 const struct mlx5_flow_meter_srtcm_rfc2697_prm *srtcm, 1931 uint64_t modify_bits, uint32_t active_state, uint32_t is_enable) 1932 { 1933 #ifdef HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER 1934 struct mlx5_dev_ctx_shared *sh = priv->sh; 1935 uint32_t in[MLX5_ST_SZ_DW(flow_meter_parameters)] = { 0 }; 1936 uint32_t *attr; 1937 struct mlx5dv_dr_flow_meter_attr mod_attr = { 0 }; 1938 int ret; 1939 struct mlx5_aso_mtr *aso_mtr = NULL; 1940 uint32_t cbs_cir, ebs_eir, val; 1941 1942 if (sh->meter_aso_en) { 1943 fm->is_enable = !!is_enable; 1944 aso_mtr = container_of(fm, struct mlx5_aso_mtr, fm); 1945 aso_mtr->state = ASO_METER_WAIT; 1946 ret = mlx5_aso_meter_update_by_wqe(priv, MLX5_HW_INV_QUEUE, 1947 aso_mtr, &priv->mtr_bulk, 1948 NULL, true); 1949 if (ret) 1950 return ret; 1951 ret = mlx5_aso_mtr_wait(priv, aso_mtr, false); 1952 if (ret) 1953 return ret; 1954 } else { 1955 /* Fill command parameters. */ 1956 mod_attr.reg_c_index = sh->registers.aso_reg - REG_C_0; 1957 mod_attr.flow_meter_parameter = in; 1958 mod_attr.flow_meter_parameter_sz = 1959 MLX5_ST_SZ_BYTES(flow_meter_parameters); 1960 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_ACTIVE) 1961 mod_attr.active = !!active_state; 1962 else 1963 mod_attr.active = 0; 1964 attr = in; 1965 cbs_cir = rte_be_to_cpu_32(srtcm->cbs_cir); 1966 ebs_eir = rte_be_to_cpu_32(srtcm->ebs_eir); 1967 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CBS) { 1968 val = (cbs_cir >> ASO_DSEG_CBS_EXP_OFFSET) & 1969 ASO_DSEG_EXP_MASK; 1970 MLX5_SET(flow_meter_parameters, attr, 1971 cbs_exponent, val); 1972 val = (cbs_cir >> ASO_DSEG_CBS_MAN_OFFSET) & 1973 ASO_DSEG_MAN_MASK; 1974 MLX5_SET(flow_meter_parameters, attr, 1975 cbs_mantissa, val); 1976 } 1977 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CIR) { 1978 val = (cbs_cir >> ASO_DSEG_XIR_EXP_OFFSET) & 1979 ASO_DSEG_EXP_MASK; 1980 MLX5_SET(flow_meter_parameters, attr, 1981 cir_exponent, val); 1982 val = cbs_cir & ASO_DSEG_MAN_MASK; 1983 MLX5_SET(flow_meter_parameters, attr, 1984 cir_mantissa, val); 1985 } 1986 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_EBS) { 1987 val = (ebs_eir >> ASO_DSEG_EBS_EXP_OFFSET) & 1988 ASO_DSEG_EXP_MASK; 1989 MLX5_SET(flow_meter_parameters, attr, 1990 ebs_exponent, val); 1991 val = (ebs_eir >> ASO_DSEG_EBS_MAN_OFFSET) & 1992 ASO_DSEG_MAN_MASK; 1993 MLX5_SET(flow_meter_parameters, attr, 1994 ebs_mantissa, val); 1995 } 1996 /* Apply modifications to meter only if it was created. */ 1997 if (fm->meter_action_g) { 1998 ret = mlx5_glue->dv_modify_flow_action_meter 1999 (fm->meter_action_g, &mod_attr, 2000 rte_cpu_to_be_64(modify_bits)); 2001 if (ret) 2002 return ret; 2003 } 2004 /* Update succeeded modify meter parameters. */ 2005 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_ACTIVE) 2006 fm->active_state = !!active_state; 2007 } 2008 return 0; 2009 #else 2010 (void)priv; 2011 (void)fm; 2012 (void)srtcm; 2013 (void)modify_bits; 2014 (void)active_state; 2015 (void)is_enable; 2016 return -ENOTSUP; 2017 #endif 2018 } 2019 2020 static int 2021 mlx5_flow_meter_stats_enable_update(struct rte_eth_dev *dev, 2022 struct mlx5_flow_meter_info *fm, 2023 uint64_t stats_mask) 2024 { 2025 fm->bytes_dropped = 2026 (stats_mask & RTE_MTR_STATS_N_BYTES_DROPPED) ? 1 : 0; 2027 fm->pkts_dropped = (stats_mask & RTE_MTR_STATS_N_PKTS_DROPPED) ? 1 : 0; 2028 if (fm->bytes_dropped || fm->pkts_dropped) { 2029 if (!fm->drop_cnt) { 2030 /* Alloc policer counters. */ 2031 fm->drop_cnt = mlx5_counter_alloc(dev); 2032 if (!fm->drop_cnt) 2033 return -1; 2034 } 2035 } else { 2036 if (fm->drop_cnt) { 2037 mlx5_counter_free(dev, fm->drop_cnt); 2038 fm->drop_cnt = 0; 2039 } 2040 } 2041 return 0; 2042 } 2043 2044 /** 2045 * Create meter rules. 2046 * 2047 * @param[in] dev 2048 * Pointer to Ethernet device. 2049 * @param[in] meter_id 2050 * Meter id. 2051 * @param[in] params 2052 * Pointer to rte meter parameters. 2053 * @param[in] shared 2054 * Meter shared with other flow or not. 2055 * @param[out] error 2056 * Pointer to rte meter error structure. 2057 * 2058 * @return 2059 * 0 on success, a negative errno value otherwise and rte_errno is set. 2060 */ 2061 static int 2062 mlx5_flow_meter_create(struct rte_eth_dev *dev, uint32_t meter_id, 2063 struct rte_mtr_params *params, int shared, 2064 struct rte_mtr_error *error) 2065 { 2066 struct mlx5_priv *priv = dev->data->dev_private; 2067 struct mlx5_legacy_flow_meters *fms = &priv->flow_meters; 2068 struct mlx5_flow_meter_profile *fmp; 2069 struct mlx5_flow_meter_info *fm; 2070 /* GCC fails to infer legacy_fm is set when !priv->sh->meter_aso_en. */ 2071 struct mlx5_legacy_flow_meter *legacy_fm = NULL; 2072 struct mlx5_flow_meter_policy *mtr_policy = NULL; 2073 struct mlx5_indexed_pool_config flow_ipool_cfg = { 2074 .size = 0, 2075 .trunk_size = 64, 2076 .need_lock = 1, 2077 .type = "mlx5_flow_mtr_flow_id_pool", 2078 }; 2079 struct mlx5_aso_mtr *aso_mtr; 2080 uint32_t mtr_idx, policy_idx; 2081 union mlx5_l3t_data data; 2082 int ret; 2083 uint8_t domain_bitmap; 2084 uint8_t mtr_id_bits; 2085 uint8_t mtr_reg_bits = priv->mtr_reg_share ? 2086 MLX5_MTR_IDLE_BITS_IN_COLOR_REG : MLX5_REG_BITS; 2087 2088 if (!priv->mtr_en) 2089 return -rte_mtr_error_set(error, ENOTSUP, 2090 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 2091 "Meter is not supported"); 2092 /* Validate the parameters. */ 2093 ret = mlx5_flow_meter_validate(priv, meter_id, params, error); 2094 if (ret) 2095 return ret; 2096 /* Meter profile must exist. */ 2097 fmp = mlx5_flow_meter_profile_find(priv, params->meter_profile_id); 2098 if (fmp == NULL) 2099 return -rte_mtr_error_set(error, ENOENT, 2100 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, 2101 NULL, "Meter profile id not valid."); 2102 /* Meter policy must exist. */ 2103 if (params->meter_policy_id == priv->sh->mtrmng->def_policy_id) { 2104 rte_atomic_fetch_add_explicit 2105 (&priv->sh->mtrmng->def_policy_ref_cnt, 2106 1, rte_memory_order_relaxed); 2107 domain_bitmap = MLX5_MTR_ALL_DOMAIN_BIT; 2108 if (!priv->sh->config.dv_esw_en) 2109 domain_bitmap &= ~MLX5_MTR_DOMAIN_TRANSFER_BIT; 2110 } else { 2111 if (!priv->sh->meter_aso_en) 2112 return -rte_mtr_error_set(error, ENOTSUP, 2113 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 2114 "Part of the policies cannot be " 2115 "supported without ASO "); 2116 mtr_policy = mlx5_flow_meter_policy_find(dev, 2117 params->meter_policy_id, &policy_idx); 2118 if (!mtr_policy) 2119 return -rte_mtr_error_set(error, ENOENT, 2120 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 2121 NULL, "Meter policy id not valid."); 2122 domain_bitmap = (mtr_policy->ingress ? 2123 MLX5_MTR_DOMAIN_INGRESS_BIT : 0) | 2124 (mtr_policy->egress ? 2125 MLX5_MTR_DOMAIN_EGRESS_BIT : 0) | 2126 (mtr_policy->transfer ? 2127 MLX5_MTR_DOMAIN_TRANSFER_BIT : 0); 2128 if (fmp->g_support && mtr_policy->skip_g) 2129 return -rte_mtr_error_set(error, ENOTSUP, 2130 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 2131 NULL, "Meter green policy is empty."); 2132 if (fmp->y_support && mtr_policy->skip_y) 2133 return -rte_mtr_error_set(error, ENOTSUP, 2134 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 2135 NULL, "Meter yellow policy is empty."); 2136 } 2137 /* Allocate the flow meter memory. */ 2138 if (priv->sh->meter_aso_en) { 2139 mtr_idx = mlx5_flow_mtr_alloc(dev); 2140 if (!mtr_idx) 2141 return -rte_mtr_error_set(error, ENOMEM, 2142 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 2143 "Memory alloc failed for meter."); 2144 aso_mtr = mlx5_aso_meter_by_idx(priv, mtr_idx); 2145 fm = &aso_mtr->fm; 2146 } else { 2147 if (fmp->y_support) 2148 return -rte_mtr_error_set(error, ENOMEM, 2149 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 2150 "Unsupported profile with yellow."); 2151 legacy_fm = mlx5_ipool_zmalloc 2152 (priv->sh->ipool[MLX5_IPOOL_MTR], &mtr_idx); 2153 if (legacy_fm == NULL) 2154 return -rte_mtr_error_set(error, ENOMEM, 2155 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 2156 "Memory alloc failed for meter."); 2157 legacy_fm->idx = mtr_idx; 2158 fm = &legacy_fm->fm; 2159 } 2160 mtr_id_bits = MLX5_REG_BITS - rte_clz32(mtr_idx); 2161 if ((mtr_id_bits + priv->sh->mtrmng->max_mtr_flow_bits) > 2162 mtr_reg_bits) { 2163 DRV_LOG(ERR, "Meter number exceeds max limit."); 2164 goto error; 2165 } 2166 if (mtr_id_bits > priv->sh->mtrmng->max_mtr_bits) 2167 priv->sh->mtrmng->max_mtr_bits = mtr_id_bits; 2168 /* Fill the flow meter parameters. */ 2169 fm->meter_id = meter_id; 2170 fm->policy_id = params->meter_policy_id; 2171 fm->profile = fmp; 2172 if (mlx5_flow_meter_stats_enable_update(dev, fm, params->stats_mask)) 2173 goto error; 2174 if (mlx5_flow_create_mtr_tbls(dev, fm, mtr_idx, domain_bitmap)) 2175 goto error; 2176 /* Add to the flow meter list. */ 2177 if (!priv->sh->meter_aso_en) { 2178 MLX5_ASSERT(legacy_fm != NULL); 2179 TAILQ_INSERT_TAIL(fms, legacy_fm, next); 2180 } 2181 /* Add to the flow meter list. */ 2182 fm->active_state = 1; /* Config meter starts as active. */ 2183 fm->is_enable = params->meter_enable; 2184 fm->shared = !!shared; 2185 fm->color_aware = !!params->use_prev_mtr_color; 2186 rte_atomic_fetch_add_explicit(&fm->profile->ref_cnt, 1, rte_memory_order_relaxed); 2187 if (params->meter_policy_id == priv->sh->mtrmng->def_policy_id) { 2188 fm->def_policy = 1; 2189 fm->flow_ipool = mlx5_ipool_create(&flow_ipool_cfg); 2190 if (!fm->flow_ipool) 2191 goto error; 2192 } 2193 rte_spinlock_init(&fm->sl); 2194 /* If ASO meter supported, update ASO flow meter by wqe. */ 2195 if (priv->sh->meter_aso_en) { 2196 aso_mtr = container_of(fm, struct mlx5_aso_mtr, fm); 2197 aso_mtr->state = ASO_METER_WAIT; 2198 ret = mlx5_aso_meter_update_by_wqe(priv, MLX5_HW_INV_QUEUE, 2199 aso_mtr, &priv->mtr_bulk, NULL, true); 2200 if (ret) 2201 goto error; 2202 if (!priv->mtr_idx_tbl) { 2203 priv->mtr_idx_tbl = 2204 mlx5_l3t_create(MLX5_L3T_TYPE_DWORD); 2205 if (!priv->mtr_idx_tbl) 2206 goto error; 2207 } 2208 data.dword = mtr_idx; 2209 if (mlx5_l3t_set_entry(priv->mtr_idx_tbl, meter_id, &data)) 2210 goto error; 2211 } else if (!params->meter_enable && mlx5_flow_meter_disable(dev, meter_id, error)) { 2212 goto error; 2213 } 2214 fm->active_state = params->meter_enable; 2215 if (mtr_policy) 2216 rte_atomic_fetch_add_explicit(&mtr_policy->ref_cnt, 1, rte_memory_order_relaxed); 2217 return 0; 2218 error: 2219 mlx5_flow_destroy_mtr_tbls(dev, fm); 2220 /* Free policer counters. */ 2221 if (fm->drop_cnt) 2222 mlx5_counter_free(dev, fm->drop_cnt); 2223 if (priv->sh->meter_aso_en) 2224 mlx5_flow_mtr_free(dev, mtr_idx); 2225 else 2226 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MTR], mtr_idx); 2227 return -rte_mtr_error_set(error, ENOTSUP, 2228 RTE_MTR_ERROR_TYPE_UNSPECIFIED, 2229 NULL, "Failed to create devx meter."); 2230 } 2231 2232 #if defined(HAVE_MLX5_HWS_SUPPORT) 2233 /** 2234 * Create meter rules. 2235 * 2236 * @param[in] dev 2237 * Pointer to Ethernet device. 2238 * @param[in] meter_id 2239 * Meter id. 2240 * @param[in] params 2241 * Pointer to rte meter parameters. 2242 * @param[in] shared 2243 * Meter shared with other flow or not. 2244 * @param[out] error 2245 * Pointer to rte meter error structure. 2246 * 2247 * @return 2248 * 0 on success, a negative errno value otherwise and rte_errno is set. 2249 */ 2250 static int 2251 mlx5_flow_meter_hws_create(struct rte_eth_dev *dev, uint32_t meter_id, 2252 struct rte_mtr_params *params, int shared, 2253 struct rte_mtr_error *error) 2254 { 2255 struct mlx5_priv *priv = dev->data->dev_private; 2256 struct mlx5_flow_meter_profile *profile; 2257 struct mlx5_flow_meter_info *fm; 2258 struct mlx5_flow_meter_policy *policy = NULL; 2259 struct mlx5_aso_mtr *aso_mtr; 2260 struct mlx5_hw_q_job *job; 2261 int ret; 2262 2263 if (mlx5_hws_active(dev) && !mlx5_hw_ctx_validate(dev, NULL)) 2264 return -rte_mtr_error_set(error, EINVAL, 2265 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 2266 "non-template flow engine was not configured"); 2267 if (!priv->mtr_profile_arr || 2268 !priv->mtr_policy_arr || 2269 !priv->mtr_bulk.aso) 2270 return -rte_mtr_error_set(error, ENOTSUP, 2271 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 2272 "Meter bulk array is not allocated."); 2273 /* Meter profile must exist. */ 2274 profile = mlx5_flow_meter_profile_find(priv, params->meter_profile_id); 2275 if (!profile->initialized) 2276 return -rte_mtr_error_set(error, ENOENT, 2277 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, 2278 NULL, "Meter profile id not valid."); 2279 /* Meter policy must exist. */ 2280 policy = mlx5_flow_meter_policy_find(dev, 2281 params->meter_policy_id, NULL); 2282 if (!policy->initialized) 2283 return -rte_mtr_error_set(error, ENOENT, 2284 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 2285 NULL, "Meter policy id not valid."); 2286 /* Meter ID must be valid. */ 2287 if (meter_id >= priv->mtr_config.nb_meters) 2288 return -rte_mtr_error_set(error, EINVAL, 2289 RTE_MTR_ERROR_TYPE_MTR_ID, 2290 NULL, "Meter id not valid."); 2291 /* Find ASO object. */ 2292 aso_mtr = mlx5_aso_meter_by_idx(priv, meter_id); 2293 fm = &aso_mtr->fm; 2294 if (fm->initialized) 2295 return -rte_mtr_error_set(error, ENOENT, 2296 RTE_MTR_ERROR_TYPE_MTR_ID, 2297 NULL, "Meter object already exists."); 2298 /* Fill the flow meter parameters. */ 2299 fm->meter_id = meter_id; 2300 fm->policy_id = params->meter_policy_id; 2301 fm->profile = profile; 2302 fm->meter_offset = meter_id; 2303 fm->group = policy->group; 2304 /* Add to the flow meter list. */ 2305 fm->active_state = 1; /* Config meter starts as active. */ 2306 fm->is_enable = params->meter_enable; 2307 fm->shared = !!shared; 2308 fm->initialized = 1; 2309 /* Update ASO flow meter by wqe. */ 2310 job = mlx5_flow_action_job_init(priv, MLX5_HW_INV_QUEUE, NULL, NULL, 2311 NULL, MLX5_HW_Q_JOB_TYPE_CREATE, NULL); 2312 if (!job) 2313 return -rte_mtr_error_set(error, ENOMEM, 2314 RTE_MTR_ERROR_TYPE_MTR_ID, 2315 NULL, "No job context."); 2316 ret = mlx5_aso_meter_update_by_wqe(priv, MLX5_HW_INV_QUEUE, aso_mtr, 2317 &priv->mtr_bulk, job, true); 2318 if (ret) { 2319 flow_hw_job_put(priv, job, CTRL_QUEUE_ID(priv)); 2320 return -rte_mtr_error_set(error, ENOTSUP, 2321 RTE_MTR_ERROR_TYPE_UNSPECIFIED, 2322 NULL, "Failed to create devx meter."); 2323 } 2324 fm->active_state = params->meter_enable; 2325 rte_atomic_fetch_add_explicit(&fm->profile->ref_cnt, 1, rte_memory_order_relaxed); 2326 rte_atomic_fetch_add_explicit(&policy->ref_cnt, 1, rte_memory_order_relaxed); 2327 return 0; 2328 } 2329 #endif 2330 2331 static int 2332 mlx5_flow_meter_params_flush(struct rte_eth_dev *dev, 2333 struct mlx5_flow_meter_info *fm, 2334 uint32_t mtr_idx) 2335 { 2336 struct mlx5_priv *priv = dev->data->dev_private; 2337 struct mlx5_legacy_flow_meters *fms = &priv->flow_meters; 2338 struct mlx5_flow_meter_profile *fmp; 2339 struct mlx5_legacy_flow_meter *legacy_fm = NULL; 2340 struct mlx5_flow_meter_policy *mtr_policy; 2341 2342 /* Meter object must not have any owner. */ 2343 MLX5_ASSERT(!fm->ref_cnt); 2344 /* Get meter profile. */ 2345 fmp = fm->profile; 2346 if (fmp == NULL) 2347 return -1; 2348 /* Update dependencies. */ 2349 rte_atomic_fetch_sub_explicit(&fmp->ref_cnt, 1, rte_memory_order_relaxed); 2350 fm->profile = NULL; 2351 /* Remove from list. */ 2352 if (!priv->sh->meter_aso_en) { 2353 legacy_fm = container_of(fm, 2354 struct mlx5_legacy_flow_meter, fm); 2355 TAILQ_REMOVE(fms, legacy_fm, next); 2356 } 2357 /* Free drop counters. */ 2358 if (fm->drop_cnt) 2359 mlx5_counter_free(dev, fm->drop_cnt); 2360 /* Free meter flow table. */ 2361 if (fm->flow_ipool) { 2362 mlx5_ipool_destroy(fm->flow_ipool); 2363 fm->flow_ipool = 0; 2364 } 2365 mlx5_flow_destroy_mtr_tbls(dev, fm); 2366 if (fm->def_policy) 2367 rte_atomic_fetch_sub_explicit(&priv->sh->mtrmng->def_policy_ref_cnt, 2368 1, rte_memory_order_relaxed); 2369 if (priv->sh->meter_aso_en) { 2370 if (!fm->def_policy) { 2371 mtr_policy = mlx5_flow_meter_policy_find(dev, 2372 fm->policy_id, NULL); 2373 if (mtr_policy) 2374 rte_atomic_fetch_sub_explicit(&mtr_policy->ref_cnt, 2375 1, rte_memory_order_relaxed); 2376 fm->policy_id = 0; 2377 } 2378 fm->def_policy = 0; 2379 if (mlx5_l3t_clear_entry(priv->mtr_idx_tbl, fm->meter_id)) 2380 return -1; 2381 mlx5_flow_mtr_free(dev, mtr_idx); 2382 } else { 2383 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MTR], 2384 legacy_fm->idx); 2385 } 2386 return 0; 2387 } 2388 2389 /** 2390 * Destroy meter rules. 2391 * 2392 * @param[in] dev 2393 * Pointer to Ethernet device. 2394 * @param[in] meter_id 2395 * Meter id. 2396 * @param[out] error 2397 * Pointer to rte meter error structure. 2398 * 2399 * @return 2400 * 0 on success, a negative errno value otherwise and rte_errno is set. 2401 */ 2402 static int 2403 mlx5_flow_meter_destroy(struct rte_eth_dev *dev, uint32_t meter_id, 2404 struct rte_mtr_error *error) 2405 { 2406 struct mlx5_priv *priv = dev->data->dev_private; 2407 struct mlx5_flow_meter_info *fm; 2408 uint32_t mtr_idx = 0; 2409 2410 if (!priv->mtr_en) 2411 return -rte_mtr_error_set(error, ENOTSUP, 2412 RTE_MTR_ERROR_TYPE_UNSPECIFIED, 2413 NULL, 2414 "Meter is not supported"); 2415 /* Meter object must exist. */ 2416 fm = mlx5_flow_meter_find(priv, meter_id, &mtr_idx); 2417 if (fm == NULL) 2418 return -rte_mtr_error_set(error, ENOENT, 2419 RTE_MTR_ERROR_TYPE_MTR_ID, 2420 NULL, 2421 "Meter object id not valid."); 2422 /* Meter object must not have any owner. */ 2423 if (fm->ref_cnt > 0) 2424 return -rte_mtr_error_set(error, EBUSY, 2425 RTE_MTR_ERROR_TYPE_UNSPECIFIED, 2426 NULL, 2427 "Meter object is being used."); 2428 /* Destroy the meter profile. */ 2429 if (mlx5_flow_meter_params_flush(dev, fm, mtr_idx)) 2430 return -rte_mtr_error_set(error, EINVAL, 2431 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, 2432 NULL, 2433 "MTR object meter profile invalid."); 2434 return 0; 2435 } 2436 2437 /** 2438 * Destroy meter rules. 2439 * 2440 * @param[in] dev 2441 * Pointer to Ethernet device. 2442 * @param[in] meter_id 2443 * Meter id. 2444 * @param[out] error 2445 * Pointer to rte meter error structure. 2446 * 2447 * @return 2448 * 0 on success, a negative errno value otherwise and rte_errno is set. 2449 */ 2450 static int 2451 mlx5_flow_meter_hws_destroy(struct rte_eth_dev *dev, uint32_t meter_id, 2452 struct rte_mtr_error *error) 2453 { 2454 struct mlx5_priv *priv = dev->data->dev_private; 2455 struct mlx5_aso_mtr *aso_mtr; 2456 struct mlx5_flow_meter_info *fm; 2457 struct mlx5_flow_meter_policy *policy; 2458 2459 if (!priv->mtr_profile_arr || 2460 !priv->mtr_policy_arr || 2461 !priv->mtr_bulk.aso) 2462 return -rte_mtr_error_set(error, ENOTSUP, 2463 RTE_MTR_ERROR_TYPE_METER_POLICY, NULL, 2464 "Meter bulk array is not allocated."); 2465 /* Find ASO object. */ 2466 aso_mtr = mlx5_aso_meter_by_idx(priv, meter_id); 2467 fm = &aso_mtr->fm; 2468 if (!fm->initialized) 2469 return -rte_mtr_error_set(error, ENOENT, 2470 RTE_MTR_ERROR_TYPE_MTR_ID, 2471 NULL, "Meter object id not valid."); 2472 /* Meter object must not have any owner. */ 2473 if (fm->ref_cnt > 0) 2474 return -rte_mtr_error_set(error, EBUSY, 2475 RTE_MTR_ERROR_TYPE_UNSPECIFIED, 2476 NULL, "Meter object is being used."); 2477 /* Destroy the meter profile. */ 2478 rte_atomic_fetch_sub_explicit(&fm->profile->ref_cnt, 2479 1, rte_memory_order_relaxed); 2480 /* Destroy the meter policy. */ 2481 policy = mlx5_flow_meter_policy_find(dev, 2482 fm->policy_id, NULL); 2483 rte_atomic_fetch_sub_explicit(&policy->ref_cnt, 2484 1, rte_memory_order_relaxed); 2485 memset(fm, 0, sizeof(struct mlx5_flow_meter_info)); 2486 return 0; 2487 } 2488 2489 /** 2490 * Modify meter state. 2491 * 2492 * @param[in] priv 2493 * Pointer to mlx5 private data structure. 2494 * @param[in] fm 2495 * Pointer to flow meter. 2496 * @param[in] new_state 2497 * New state to update. 2498 * @param[out] error 2499 * Pointer to rte meter error structure. 2500 * 2501 * @return 2502 * 0 on success, a negative errno value otherwise and rte_errno is set. 2503 */ 2504 static int 2505 mlx5_flow_meter_modify_state(struct mlx5_priv *priv, 2506 struct mlx5_flow_meter_info *fm, 2507 uint32_t new_state, 2508 struct rte_mtr_error *error) 2509 { 2510 static const struct mlx5_flow_meter_srtcm_rfc2697_prm srtcm = { 2511 .cbs_cir = RTE_BE32(MLX5_IFC_FLOW_METER_DISABLE_CBS_CIR_VAL), 2512 .ebs_eir = 0, 2513 }; 2514 uint64_t modify_bits = MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CBS | 2515 MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CIR; 2516 int ret; 2517 2518 if (new_state == MLX5_FLOW_METER_DISABLE) 2519 ret = mlx5_flow_meter_action_modify(priv, fm, 2520 &srtcm, modify_bits, 0, 0); 2521 else 2522 ret = mlx5_flow_meter_action_modify(priv, fm, 2523 &fm->profile->srtcm_prm, 2524 modify_bits, 0, 1); 2525 if (ret) 2526 return -rte_mtr_error_set(error, -ret, 2527 RTE_MTR_ERROR_TYPE_MTR_PARAMS, 2528 NULL, 2529 new_state ? 2530 "Failed to enable meter." : 2531 "Failed to disable meter."); 2532 return 0; 2533 } 2534 2535 /** 2536 * Callback to enable flow meter. 2537 * 2538 * @param[in] dev 2539 * Pointer to Ethernet device. 2540 * @param[in] meter_id 2541 * Meter id. 2542 * @param[out] error 2543 * Pointer to rte meter error structure. 2544 * 2545 * @return 2546 * 0 on success, a negative errno value otherwise and rte_errno is set. 2547 */ 2548 static int 2549 mlx5_flow_meter_enable(struct rte_eth_dev *dev, 2550 uint32_t meter_id, 2551 struct rte_mtr_error *error) 2552 { 2553 struct mlx5_priv *priv = dev->data->dev_private; 2554 struct mlx5_flow_meter_info *fm; 2555 int ret; 2556 2557 if (mlx5_hws_active(dev) && !mlx5_hw_ctx_validate(dev, NULL)) 2558 return -rte_mtr_error_set(error, EINVAL, 2559 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 2560 "non-template flow engine was not configured"); 2561 if (!priv->mtr_en) 2562 return -rte_mtr_error_set(error, ENOTSUP, 2563 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 2564 "Meter is not supported"); 2565 /* Meter object must exist. */ 2566 fm = mlx5_flow_meter_find(priv, meter_id, NULL); 2567 if (fm == NULL) 2568 return -rte_mtr_error_set(error, ENOENT, 2569 RTE_MTR_ERROR_TYPE_MTR_ID, 2570 NULL, "Meter not found."); 2571 if (fm->active_state == MLX5_FLOW_METER_ENABLE) 2572 return 0; 2573 ret = mlx5_flow_meter_modify_state(priv, fm, MLX5_FLOW_METER_ENABLE, 2574 error); 2575 if (!ret) 2576 fm->active_state = MLX5_FLOW_METER_ENABLE; 2577 return ret; 2578 } 2579 2580 /** 2581 * Callback to disable flow meter. 2582 * 2583 * @param[in] dev 2584 * Pointer to Ethernet device. 2585 * @param[in] meter_id 2586 * Meter id. 2587 * @param[out] error 2588 * Pointer to rte meter error structure. 2589 * 2590 * @return 2591 * 0 on success, a negative errno value otherwise and rte_errno is set. 2592 */ 2593 static int 2594 mlx5_flow_meter_disable(struct rte_eth_dev *dev, 2595 uint32_t meter_id, 2596 struct rte_mtr_error *error) 2597 { 2598 struct mlx5_priv *priv = dev->data->dev_private; 2599 struct mlx5_flow_meter_info *fm; 2600 int ret; 2601 2602 if (!priv->mtr_en) 2603 return -rte_mtr_error_set(error, ENOTSUP, 2604 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 2605 "Meter is not supported"); 2606 /* Meter object must exist. */ 2607 fm = mlx5_flow_meter_find(priv, meter_id, NULL); 2608 if (fm == NULL) 2609 return -rte_mtr_error_set(error, ENOENT, 2610 RTE_MTR_ERROR_TYPE_MTR_ID, 2611 NULL, "Meter not found."); 2612 if (fm->active_state == MLX5_FLOW_METER_DISABLE) 2613 return 0; 2614 ret = mlx5_flow_meter_modify_state(priv, fm, MLX5_FLOW_METER_DISABLE, 2615 error); 2616 if (!ret) 2617 fm->active_state = MLX5_FLOW_METER_DISABLE; 2618 return ret; 2619 } 2620 2621 /** 2622 * Callback to update meter profile. 2623 * 2624 * @param[in] dev 2625 * Pointer to Ethernet device. 2626 * @param[in] meter_id 2627 * Meter id. 2628 * @param[in] meter_profile_id 2629 * To be updated meter profile id. 2630 * @param[out] error 2631 * Pointer to rte meter error structure. 2632 * 2633 * @return 2634 * 0 on success, a negative errno value otherwise and rte_errno is set. 2635 */ 2636 static int 2637 mlx5_flow_meter_profile_update(struct rte_eth_dev *dev, 2638 uint32_t meter_id, 2639 uint32_t meter_profile_id, 2640 struct rte_mtr_error *error) 2641 { 2642 struct mlx5_priv *priv = dev->data->dev_private; 2643 struct mlx5_flow_meter_profile *fmp; 2644 struct mlx5_flow_meter_profile *old_fmp; 2645 struct mlx5_flow_meter_info *fm; 2646 uint64_t modify_bits = MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CBS | 2647 MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CIR; 2648 int ret; 2649 2650 if (mlx5_hws_active(dev) && !mlx5_hw_ctx_validate(dev, NULL)) 2651 return -rte_mtr_error_set(error, EINVAL, 2652 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 2653 "non-template flow engine was not configured"); 2654 if (!priv->mtr_en) 2655 return -rte_mtr_error_set(error, ENOTSUP, 2656 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 2657 "Meter is not supported"); 2658 /* Meter profile must exist. */ 2659 fmp = mlx5_flow_meter_profile_find(priv, meter_profile_id); 2660 if (fmp == NULL) 2661 return -rte_mtr_error_set(error, ENOENT, 2662 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, 2663 NULL, "Meter profile not found."); 2664 /* Meter object must exist. */ 2665 fm = mlx5_flow_meter_find(priv, meter_id, NULL); 2666 if (fm == NULL) 2667 return -rte_mtr_error_set(error, ENOENT, 2668 RTE_MTR_ERROR_TYPE_MTR_ID, 2669 NULL, "Meter not found."); 2670 /* MTR object already set to meter profile id. */ 2671 old_fmp = fm->profile; 2672 if (fmp == old_fmp) 2673 return 0; 2674 /* Update the profile. */ 2675 fm->profile = fmp; 2676 /* Update meter params in HW (if not disabled). */ 2677 if (fm->active_state == MLX5_FLOW_METER_DISABLE) 2678 goto dec_ref_cnt; 2679 ret = mlx5_flow_meter_action_modify(priv, fm, &fm->profile->srtcm_prm, 2680 modify_bits, fm->active_state, 1); 2681 if (ret) { 2682 fm->profile = old_fmp; 2683 return -rte_mtr_error_set(error, -ret, 2684 RTE_MTR_ERROR_TYPE_MTR_PARAMS, 2685 NULL, "Failed to update meter" 2686 " parameters in hardware."); 2687 } 2688 dec_ref_cnt: 2689 old_fmp->ref_cnt--; 2690 fmp->ref_cnt++; 2691 return 0; 2692 } 2693 2694 /** 2695 * Callback to update meter stats mask. 2696 * 2697 * @param[in] dev 2698 * Pointer to Ethernet device. 2699 * @param[in] meter_id 2700 * Meter id. 2701 * @param[in] stats_mask 2702 * To be updated stats_mask. 2703 * @param[out] error 2704 * Pointer to rte meter error structure. 2705 * 2706 * @return 2707 * 0 on success, a negative errno value otherwise and rte_errno is set. 2708 */ 2709 static int 2710 mlx5_flow_meter_stats_update(struct rte_eth_dev *dev, 2711 uint32_t meter_id, 2712 uint64_t stats_mask, 2713 struct rte_mtr_error *error) 2714 { 2715 struct mlx5_priv *priv = dev->data->dev_private; 2716 struct mlx5_flow_meter_info *fm; 2717 2718 if (!priv->mtr_en) 2719 return -rte_mtr_error_set(error, ENOTSUP, 2720 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 2721 "Meter is not supported"); 2722 /* Meter object must exist. */ 2723 fm = mlx5_flow_meter_find(priv, meter_id, NULL); 2724 if (fm == NULL) 2725 return -rte_mtr_error_set(error, ENOENT, 2726 RTE_MTR_ERROR_TYPE_MTR_ID, 2727 NULL, "Meter object id not valid."); 2728 if (mlx5_flow_meter_stats_enable_update(dev, fm, stats_mask)) 2729 return -rte_mtr_error_set(error, ENOENT, 2730 RTE_MTR_ERROR_TYPE_MTR_ID, 2731 NULL, "Fail to allocate " 2732 "counter for meter."); 2733 return 0; 2734 } 2735 2736 /** 2737 * Callback to read meter statistics. 2738 * 2739 * @param[in] dev 2740 * Pointer to Ethernet device. 2741 * @param[in] meter_id 2742 * Meter id. 2743 * @param[out] stats 2744 * Pointer to store the statistics. 2745 * @param[out] stats_mask 2746 * Pointer to store the stats_mask. 2747 * @param[in] clear 2748 * Statistic to be cleared after read or not. 2749 * @param[out] error 2750 * Pointer to rte meter error structure. 2751 * 2752 * @return 2753 * 0 on success, a negative errno value otherwise and rte_errno is set. 2754 */ 2755 static int 2756 mlx5_flow_meter_stats_read(struct rte_eth_dev *dev, 2757 uint32_t meter_id, 2758 struct rte_mtr_stats *stats, 2759 uint64_t *stats_mask, 2760 int clear, 2761 struct rte_mtr_error *error) 2762 { 2763 struct mlx5_priv *priv = dev->data->dev_private; 2764 struct mlx5_flow_meter_info *fm; 2765 uint64_t pkts; 2766 uint64_t bytes; 2767 int ret = 0; 2768 2769 if (!priv->mtr_en) 2770 return -rte_mtr_error_set(error, ENOTSUP, 2771 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, 2772 "Meter is not supported"); 2773 /* Meter object must exist. */ 2774 fm = mlx5_flow_meter_find(priv, meter_id, NULL); 2775 if (fm == NULL) 2776 return -rte_mtr_error_set(error, ENOENT, 2777 RTE_MTR_ERROR_TYPE_MTR_ID, 2778 NULL, "Meter object id not valid."); 2779 *stats_mask = 0; 2780 if (fm->bytes_dropped) 2781 *stats_mask |= RTE_MTR_STATS_N_BYTES_DROPPED; 2782 if (fm->pkts_dropped) 2783 *stats_mask |= RTE_MTR_STATS_N_PKTS_DROPPED; 2784 memset(stats, 0, sizeof(*stats)); 2785 if (fm->drop_cnt) { 2786 ret = mlx5_counter_query(dev, fm->drop_cnt, clear, &pkts, 2787 &bytes, NULL); 2788 if (ret) 2789 goto error; 2790 /* If need to read the packets, set it. */ 2791 if (fm->pkts_dropped) 2792 stats->n_pkts_dropped = pkts; 2793 /* If need to read the bytes, set it. */ 2794 if (fm->bytes_dropped) 2795 stats->n_bytes_dropped = bytes; 2796 } 2797 return 0; 2798 error: 2799 return -rte_mtr_error_set(error, ret, RTE_MTR_ERROR_TYPE_STATS, NULL, 2800 "Failed to read meter drop counters."); 2801 } 2802 2803 static const struct rte_mtr_ops mlx5_flow_mtr_ops = { 2804 .capabilities_get = mlx5_flow_mtr_cap_get, 2805 .meter_profile_add = mlx5_flow_meter_profile_add, 2806 .meter_profile_delete = mlx5_flow_meter_profile_delete, 2807 .meter_profile_get = mlx5_flow_meter_profile_get, 2808 .meter_policy_validate = mlx5_flow_meter_policy_validate, 2809 .meter_policy_add = mlx5_flow_meter_policy_add, 2810 .meter_policy_delete = mlx5_flow_meter_policy_delete, 2811 .meter_policy_get = mlx5_flow_meter_policy_get, 2812 .create = mlx5_flow_meter_create, 2813 .destroy = mlx5_flow_meter_destroy, 2814 .meter_enable = mlx5_flow_meter_enable, 2815 .meter_disable = mlx5_flow_meter_disable, 2816 .meter_profile_update = mlx5_flow_meter_profile_update, 2817 .meter_dscp_table_update = NULL, 2818 .stats_update = mlx5_flow_meter_stats_update, 2819 .stats_read = mlx5_flow_meter_stats_read, 2820 }; 2821 2822 #if defined(HAVE_MLX5_HWS_SUPPORT) 2823 static const struct rte_mtr_ops mlx5_flow_mtr_hws_ops = { 2824 .capabilities_get = mlx5_flow_mtr_cap_get, 2825 .meter_profile_add = mlx5_flow_meter_profile_hws_add, 2826 .meter_profile_delete = mlx5_flow_meter_profile_hws_delete, 2827 .meter_profile_get = mlx5_flow_meter_profile_get, 2828 .meter_policy_validate = mlx5_flow_meter_policy_hws_validate, 2829 .meter_policy_add = mlx5_flow_meter_policy_hws_add, 2830 .meter_policy_delete = mlx5_flow_meter_policy_hws_delete, 2831 .meter_policy_get = mlx5_flow_meter_policy_get, 2832 .create = mlx5_flow_meter_hws_create, 2833 .destroy = mlx5_flow_meter_hws_destroy, 2834 .meter_enable = mlx5_flow_meter_enable, 2835 .meter_disable = mlx5_flow_meter_disable, 2836 .meter_profile_update = mlx5_flow_meter_profile_update, 2837 .meter_dscp_table_update = NULL, 2838 .stats_update = NULL, 2839 .stats_read = NULL, 2840 }; 2841 #endif 2842 2843 /** 2844 * Get meter operations. 2845 * 2846 * @param dev 2847 * Pointer to Ethernet device structure. 2848 * @param arg 2849 * Pointer to set the mtr operations. 2850 * 2851 * @return 2852 * Always 0. 2853 */ 2854 int 2855 mlx5_flow_meter_ops_get(struct rte_eth_dev *dev __rte_unused, void *arg) 2856 { 2857 #if defined(HAVE_MLX5_HWS_SUPPORT) 2858 struct mlx5_priv *priv = dev->data->dev_private; 2859 2860 if (priv->sh->config.dv_flow_en == 2) 2861 *(const struct rte_mtr_ops **)arg = &mlx5_flow_mtr_hws_ops; 2862 else 2863 *(const struct rte_mtr_ops **)arg = &mlx5_flow_mtr_ops; 2864 #else 2865 *(const struct rte_mtr_ops **)arg = &mlx5_flow_mtr_ops; 2866 #endif 2867 return 0; 2868 } 2869 2870 /** 2871 * Find meter by id. 2872 * 2873 * @param priv 2874 * Pointer to mlx5_priv. 2875 * @param meter_id 2876 * Meter id. 2877 * @param mtr_idx 2878 * Pointer to Meter index. 2879 * 2880 * @return 2881 * Pointer to the meter info found on success, NULL otherwise. 2882 */ 2883 struct mlx5_flow_meter_info * 2884 mlx5_flow_meter_find(struct mlx5_priv *priv, uint32_t meter_id, 2885 uint32_t *mtr_idx) 2886 { 2887 struct mlx5_legacy_flow_meter *legacy_fm; 2888 struct mlx5_legacy_flow_meters *fms = &priv->flow_meters; 2889 struct mlx5_aso_mtr *aso_mtr; 2890 struct mlx5_aso_mtr_pools_mng *pools_mng = 2891 &priv->sh->mtrmng->pools_mng; 2892 union mlx5_l3t_data data; 2893 uint16_t n_valid; 2894 2895 if (priv->mtr_bulk.aso) { 2896 if (mtr_idx) 2897 *mtr_idx = meter_id; 2898 aso_mtr = priv->mtr_bulk.aso + meter_id; 2899 return &aso_mtr->fm; 2900 } 2901 if (priv->sh->meter_aso_en) { 2902 rte_rwlock_read_lock(&pools_mng->resize_mtrwl); 2903 n_valid = pools_mng->n_valid; 2904 rte_rwlock_read_unlock(&pools_mng->resize_mtrwl); 2905 if (!n_valid || !priv->mtr_idx_tbl || 2906 (mlx5_l3t_get_entry(priv->mtr_idx_tbl, meter_id, &data) || 2907 !data.dword)) 2908 return NULL; 2909 if (mtr_idx) 2910 *mtr_idx = data.dword; 2911 aso_mtr = mlx5_aso_meter_by_idx(priv, data.dword); 2912 /* Remove reference taken by the mlx5_l3t_get_entry. */ 2913 mlx5_l3t_clear_entry(priv->mtr_idx_tbl, meter_id); 2914 if (!aso_mtr || aso_mtr->state == ASO_METER_FREE) 2915 return NULL; 2916 return &aso_mtr->fm; 2917 } 2918 TAILQ_FOREACH(legacy_fm, fms, next) 2919 if (meter_id == legacy_fm->fm.meter_id) { 2920 if (mtr_idx) 2921 *mtr_idx = legacy_fm->idx; 2922 return &legacy_fm->fm; 2923 } 2924 return NULL; 2925 } 2926 2927 /** 2928 * Find meter by index. 2929 * 2930 * @param priv 2931 * Pointer to mlx5_priv. 2932 * @param idx 2933 * Meter index. 2934 * 2935 * @return 2936 * Pointer to the meter info found on success, NULL otherwise. 2937 */ 2938 struct mlx5_flow_meter_info * 2939 flow_dv_meter_find_by_idx(struct mlx5_priv *priv, uint32_t idx) 2940 { 2941 struct mlx5_aso_mtr *aso_mtr; 2942 2943 if (priv->sh->meter_aso_en) { 2944 aso_mtr = mlx5_aso_meter_by_idx(priv, idx); 2945 if (!aso_mtr) 2946 return NULL; 2947 return &aso_mtr->fm; 2948 } else { 2949 return mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MTR], idx); 2950 } 2951 } 2952 2953 /** 2954 * Attach meter to flow. 2955 * Unidirectional Meter creation can only be done 2956 * when flow direction is known, i.e. when calling meter_attach. 2957 * 2958 * @param [in] priv 2959 * Pointer to mlx5 private data. 2960 * @param[in] fm 2961 * Pointer to flow meter. 2962 * @param [in] attr 2963 * Pointer to flow attributes. 2964 * @param [out] error 2965 * Pointer to error structure. 2966 * 2967 * @return 2968 * 0 on success, a negative errno value otherwise and rte_errno is set. 2969 */ 2970 int 2971 mlx5_flow_meter_attach(struct mlx5_priv *priv, 2972 struct mlx5_flow_meter_info *fm, 2973 const struct rte_flow_attr *attr, 2974 struct rte_flow_error *error) 2975 { 2976 int ret = 0; 2977 2978 if (priv->sh->meter_aso_en) { 2979 struct mlx5_aso_mtr *aso_mtr; 2980 2981 aso_mtr = container_of(fm, struct mlx5_aso_mtr, fm); 2982 if (mlx5_aso_mtr_wait(priv, aso_mtr, false)) { 2983 return rte_flow_error_set(error, ENOENT, 2984 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 2985 NULL, 2986 "Timeout in meter configuration"); 2987 } 2988 rte_spinlock_lock(&fm->sl); 2989 if (fm->shared || !fm->ref_cnt) { 2990 fm->ref_cnt++; 2991 } else { 2992 rte_flow_error_set(error, EINVAL, 2993 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 2994 "Meter cannot be shared"); 2995 ret = -1; 2996 } 2997 rte_spinlock_unlock(&fm->sl); 2998 } else { 2999 rte_spinlock_lock(&fm->sl); 3000 if (fm->meter_action_g) { 3001 if (fm->shared && 3002 attr->transfer == fm->transfer && 3003 attr->ingress == fm->ingress && 3004 attr->egress == fm->egress) { 3005 fm->ref_cnt++; 3006 } else { 3007 rte_flow_error_set(error, EINVAL, 3008 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 3009 fm->shared ? 3010 "Meter attr not match." : 3011 "Meter cannot be shared."); 3012 ret = -1; 3013 } 3014 } else { 3015 fm->ingress = attr->ingress; 3016 fm->egress = attr->egress; 3017 fm->transfer = attr->transfer; 3018 fm->ref_cnt = 1; 3019 /* This also creates the meter object. */ 3020 fm->meter_action_g = mlx5_flow_meter_action_create(priv, 3021 fm); 3022 if (!fm->meter_action_g) { 3023 fm->ref_cnt = 0; 3024 fm->ingress = 0; 3025 fm->egress = 0; 3026 fm->transfer = 0; 3027 rte_flow_error_set(error, EINVAL, 3028 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 3029 "Meter action create failed."); 3030 ret = -1; 3031 } 3032 } 3033 rte_spinlock_unlock(&fm->sl); 3034 } 3035 return ret ? -rte_errno : 0; 3036 } 3037 3038 /** 3039 * Detach meter from flow. 3040 * 3041 * @param [in] priv 3042 * Pointer to mlx5 private data. 3043 * @param [in] fm 3044 * Pointer to flow meter. 3045 */ 3046 void 3047 mlx5_flow_meter_detach(struct mlx5_priv *priv, 3048 struct mlx5_flow_meter_info *fm) 3049 { 3050 #ifdef HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER 3051 rte_spinlock_lock(&fm->sl); 3052 MLX5_ASSERT(fm->ref_cnt); 3053 if (--fm->ref_cnt == 0 && !priv->sh->meter_aso_en) { 3054 mlx5_glue->destroy_flow_action(fm->meter_action_g); 3055 fm->meter_action_g = NULL; 3056 fm->ingress = 0; 3057 fm->egress = 0; 3058 fm->transfer = 0; 3059 } 3060 rte_spinlock_unlock(&fm->sl); 3061 #else 3062 (void)priv; 3063 (void)fm; 3064 #endif 3065 } 3066 3067 /** 3068 * Flush meter with Rx queue configuration. 3069 * 3070 * @param[in] dev 3071 * Pointer to Ethernet device. 3072 */ 3073 void 3074 mlx5_flow_meter_rxq_flush(struct rte_eth_dev *dev) 3075 { 3076 struct mlx5_priv *priv = dev->data->dev_private; 3077 struct mlx5_flow_meter_sub_policy *sub_policy; 3078 struct mlx5_flow_meter_policy *mtr_policy; 3079 void *entry; 3080 uint32_t i, policy_idx; 3081 3082 if (!priv->mtr_en) 3083 return; 3084 if (priv->policy_idx_tbl) { 3085 MLX5_L3T_FOREACH(priv->policy_idx_tbl, i, entry) { 3086 policy_idx = *(uint32_t *)entry; 3087 sub_policy = mlx5_ipool_get 3088 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY], 3089 policy_idx); 3090 if (!sub_policy || !sub_policy->main_policy) 3091 continue; 3092 mtr_policy = sub_policy->main_policy; 3093 if (mtr_policy->is_queue || mtr_policy->is_rss) 3094 mlx5_flow_destroy_sub_policy_with_rxq(dev, 3095 mtr_policy); 3096 } 3097 } 3098 } 3099 3100 /** 3101 * Iterate a meter hierarchy and flush all meters and policies if possible. 3102 * 3103 * @param[in] dev 3104 * Pointer to Ethernet device. 3105 * @param[in] fm 3106 * Pointer to flow meter. 3107 * @param[in] mtr_idx 3108 * .Meter's index 3109 * @param[out] error 3110 * Pointer to rte meter error structure. 3111 * 3112 * @return 3113 * 0 on success, a negative errno value otherwise and rte_errno is set. 3114 */ 3115 static int 3116 mlx5_flow_meter_flush_hierarchy(struct rte_eth_dev *dev, 3117 struct mlx5_flow_meter_info *fm, 3118 uint32_t mtr_idx, 3119 struct rte_mtr_error *error) 3120 { 3121 struct mlx5_priv *priv = dev->data->dev_private; 3122 struct mlx5_flow_meter_policy *policy; 3123 uint32_t policy_id; 3124 struct mlx5_flow_meter_info *next_fm; 3125 uint32_t next_mtr_idx; 3126 struct mlx5_flow_meter_policy *next_policy = NULL; 3127 3128 policy = mlx5_flow_meter_policy_find(dev, fm->policy_id, NULL); 3129 MLX5_ASSERT(policy); 3130 while (!fm->ref_cnt && policy->is_hierarchy) { 3131 policy_id = fm->policy_id; 3132 next_fm = mlx5_flow_meter_hierarchy_next_meter(priv, policy, &next_mtr_idx); 3133 if (next_fm) { 3134 next_policy = mlx5_flow_meter_policy_find(dev, 3135 next_fm->policy_id, 3136 NULL); 3137 MLX5_ASSERT(next_policy); 3138 } 3139 if (mlx5_flow_meter_params_flush(dev, fm, mtr_idx)) 3140 return -rte_mtr_error_set(error, ENOTSUP, 3141 RTE_MTR_ERROR_TYPE_MTR_ID, 3142 NULL, 3143 "Failed to flush meter."); 3144 if (policy->ref_cnt) 3145 break; 3146 if (__mlx5_flow_meter_policy_delete(dev, policy_id, 3147 policy, error, true)) 3148 return -rte_errno; 3149 mlx5_free(policy); 3150 if (!next_fm || !next_policy) 3151 break; 3152 fm = next_fm; 3153 mtr_idx = next_mtr_idx; 3154 policy = next_policy; 3155 } 3156 return 0; 3157 } 3158 3159 /** 3160 * Flush all the hierarchy meters and their policies. 3161 * 3162 * @param[in] dev 3163 * Pointer to Ethernet device. 3164 * @param[out] error 3165 * Pointer to rte meter error structure. 3166 * 3167 * @return 3168 * 0 on success, a negative errno value otherwise and rte_errno is set. 3169 */ 3170 static int 3171 mlx5_flow_meter_flush_all_hierarchies(struct rte_eth_dev *dev, 3172 struct rte_mtr_error *error) 3173 { 3174 struct mlx5_priv *priv = dev->data->dev_private; 3175 struct mlx5_flow_meter_info *fm; 3176 struct mlx5_flow_meter_policy *policy; 3177 struct mlx5_flow_meter_sub_policy *sub_policy; 3178 struct mlx5_flow_meter_info *next_fm; 3179 struct mlx5_aso_mtr *aso_mtr; 3180 uint32_t mtr_idx = 0; 3181 uint32_t i, policy_idx; 3182 void *entry; 3183 3184 if (!priv->mtr_idx_tbl || !priv->policy_idx_tbl) 3185 return 0; 3186 MLX5_L3T_FOREACH(priv->mtr_idx_tbl, i, entry) { 3187 mtr_idx = *(uint32_t *)entry; 3188 if (!mtr_idx) 3189 continue; 3190 aso_mtr = mlx5_aso_meter_by_idx(priv, mtr_idx); 3191 fm = &aso_mtr->fm; 3192 if (fm->ref_cnt || fm->def_policy) 3193 continue; 3194 if (mlx5_flow_meter_flush_hierarchy(dev, fm, mtr_idx, error)) 3195 return -rte_errno; 3196 } 3197 MLX5_L3T_FOREACH(priv->policy_idx_tbl, i, entry) { 3198 policy_idx = *(uint32_t *)entry; 3199 sub_policy = mlx5_ipool_get 3200 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY], 3201 policy_idx); 3202 if (!sub_policy) 3203 return -rte_mtr_error_set(error, 3204 EINVAL, 3205 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 3206 NULL, "Meter policy invalid."); 3207 policy = sub_policy->main_policy; 3208 if (!policy || !policy->is_hierarchy || policy->ref_cnt) 3209 continue; 3210 next_fm = mlx5_flow_meter_hierarchy_next_meter(priv, policy, &mtr_idx); 3211 if (__mlx5_flow_meter_policy_delete(dev, i, policy, 3212 error, true)) 3213 return -rte_mtr_error_set(error, 3214 EINVAL, 3215 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 3216 NULL, "Meter policy invalid."); 3217 mlx5_free(policy); 3218 if (!next_fm || next_fm->ref_cnt || next_fm->def_policy) 3219 continue; 3220 if (mlx5_flow_meter_flush_hierarchy(dev, next_fm, 3221 mtr_idx, error)) 3222 return -rte_errno; 3223 } 3224 return 0; 3225 } 3226 /** 3227 * Flush meter configuration. 3228 * 3229 * @param[in] dev 3230 * Pointer to Ethernet device. 3231 * @param[out] error 3232 * Pointer to rte meter error structure. 3233 * 3234 * @return 3235 * 0 on success, a negative errno value otherwise and rte_errno is set. 3236 */ 3237 int 3238 mlx5_flow_meter_flush(struct rte_eth_dev *dev, struct rte_mtr_error *error) 3239 { 3240 struct mlx5_priv *priv = dev->data->dev_private; 3241 struct mlx5_legacy_flow_meters *fms = &priv->flow_meters; 3242 struct mlx5_flow_meter_profile *fmp; 3243 struct mlx5_legacy_flow_meter *legacy_fm; 3244 struct mlx5_flow_meter_info *fm; 3245 struct mlx5_flow_meter_sub_policy *sub_policy; 3246 void *tmp; 3247 uint32_t i, mtr_idx, policy_idx; 3248 void *entry; 3249 struct mlx5_aso_mtr *aso_mtr; 3250 3251 if (!priv->mtr_en) 3252 return 0; 3253 if (priv->sh->meter_aso_en) { 3254 if (mlx5_flow_meter_flush_all_hierarchies(dev, error)) 3255 return -rte_errno; 3256 if (priv->mtr_idx_tbl) { 3257 MLX5_L3T_FOREACH(priv->mtr_idx_tbl, i, entry) { 3258 mtr_idx = *(uint32_t *)entry; 3259 if (mtr_idx) { 3260 aso_mtr = 3261 mlx5_aso_meter_by_idx(priv, mtr_idx); 3262 fm = &aso_mtr->fm; 3263 (void)mlx5_flow_meter_params_flush(dev, 3264 fm, mtr_idx); 3265 } 3266 } 3267 mlx5_l3t_destroy(priv->mtr_idx_tbl); 3268 priv->mtr_idx_tbl = NULL; 3269 } 3270 } else { 3271 RTE_TAILQ_FOREACH_SAFE(legacy_fm, fms, next, tmp) { 3272 fm = &legacy_fm->fm; 3273 if (mlx5_flow_meter_params_flush(dev, fm, 0)) 3274 return -rte_mtr_error_set(error, EINVAL, 3275 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, 3276 NULL, "MTR object meter profile invalid."); 3277 } 3278 } 3279 if (priv->mtr_bulk.aso) { 3280 for (i = 0; i < priv->mtr_config.nb_meters; i++) { 3281 aso_mtr = mlx5_aso_meter_by_idx(priv, i); 3282 fm = &aso_mtr->fm; 3283 if (fm->initialized) 3284 mlx5_flow_meter_hws_destroy(dev, i, error); 3285 } 3286 } 3287 if (priv->policy_idx_tbl) { 3288 MLX5_L3T_FOREACH(priv->policy_idx_tbl, i, entry) { 3289 policy_idx = *(uint32_t *)entry; 3290 sub_policy = mlx5_ipool_get 3291 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY], 3292 policy_idx); 3293 if (!sub_policy) 3294 return -rte_mtr_error_set(error, 3295 EINVAL, 3296 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 3297 NULL, "MTR object " 3298 "meter policy invalid."); 3299 if (__mlx5_flow_meter_policy_delete(dev, i, 3300 sub_policy->main_policy, 3301 error, true)) 3302 return -rte_mtr_error_set(error, 3303 EINVAL, 3304 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 3305 NULL, "MTR object " 3306 "meter policy invalid."); 3307 mlx5_free(sub_policy->main_policy); 3308 } 3309 mlx5_l3t_destroy(priv->policy_idx_tbl); 3310 priv->policy_idx_tbl = NULL; 3311 } 3312 #if defined(HAVE_MLX5_HWS_SUPPORT) 3313 if (priv->mtr_policy_arr) { 3314 struct mlx5_flow_meter_policy *policy; 3315 3316 for (i = 0; i < priv->mtr_config.nb_meter_policies; i++) { 3317 policy = mlx5_flow_meter_policy_find(dev, i, 3318 &policy_idx); 3319 if (policy->initialized) { 3320 mlx5_flow_meter_policy_hws_delete(dev, i, 3321 error); 3322 } 3323 } 3324 } 3325 #endif 3326 if (priv->mtr_profile_tbl) { 3327 MLX5_L3T_FOREACH(priv->mtr_profile_tbl, i, entry) { 3328 fmp = entry; 3329 if (mlx5_flow_meter_profile_delete(dev, fmp->id, 3330 error)) 3331 return -rte_mtr_error_set(error, EINVAL, 3332 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, 3333 NULL, "Fail to destroy " 3334 "meter profile."); 3335 } 3336 mlx5_l3t_destroy(priv->mtr_profile_tbl); 3337 priv->mtr_profile_tbl = NULL; 3338 } 3339 #if defined(HAVE_MLX5_HWS_SUPPORT) 3340 if (priv->mtr_profile_arr) { 3341 for (i = 0; i < priv->mtr_config.nb_meter_profiles; i++) { 3342 fmp = mlx5_flow_meter_profile_find(priv, i); 3343 if (fmp->initialized) { 3344 mlx5_flow_meter_profile_hws_delete(dev, i, 3345 error); 3346 } 3347 } 3348 } 3349 #endif 3350 /* Delete default policy table. */ 3351 mlx5_flow_destroy_def_policy(dev); 3352 if (priv->sh->refcnt == 1) 3353 mlx5_flow_destroy_mtr_drop_tbls(dev); 3354 #ifdef HAVE_MLX5_HWS_SUPPORT 3355 /* Destroy HWS configuration. */ 3356 mlx5_flow_meter_uninit(dev); 3357 #endif 3358 return 0; 3359 } 3360